xref: /linux/drivers/tty/serial/sprd_serial.c (revision 4b132aacb0768ac1e652cf517097ea6f237214b9) 
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2012-2015 Spreadtrum Communications Inc.
4  */
5 
6 #include <linux/clk.h>
7 #include <linux/console.h>
8 #include <linux/delay.h>
9 #include <linux/dmaengine.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/dma/sprd-dma.h>
12 #include <linux/io.h>
13 #include <linux/ioport.h>
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/of.h>
17 #include <linux/platform_device.h>
18 #include <linux/serial_core.h>
19 #include <linux/serial.h>
20 #include <linux/slab.h>
21 #include <linux/tty.h>
22 #include <linux/tty_flip.h>
23 
24 /* device name */
25 #define UART_NR_MAX		8
26 #define SPRD_TTY_NAME		"ttyS"
27 #define SPRD_FIFO_SIZE		128
28 #define SPRD_DEF_RATE		26000000
29 #define SPRD_BAUD_IO_LIMIT	3000000
30 #define SPRD_TIMEOUT		256000
31 
32 /* the offset of serial registers and BITs for them */
33 /* data registers */
34 #define SPRD_TXD		0x0000
35 #define SPRD_RXD		0x0004
36 
37 /* line status register and its BITs  */
38 #define SPRD_LSR		0x0008
39 #define SPRD_LSR_OE		BIT(4)
40 #define SPRD_LSR_FE		BIT(3)
41 #define SPRD_LSR_PE		BIT(2)
42 #define SPRD_LSR_BI		BIT(7)
43 #define SPRD_LSR_TX_OVER	BIT(15)
44 
45 /* data number in TX and RX fifo */
46 #define SPRD_STS1		0x000C
47 #define SPRD_RX_FIFO_CNT_MASK	GENMASK(7, 0)
48 #define SPRD_TX_FIFO_CNT_MASK	GENMASK(15, 8)
49 
50 /* interrupt enable register and its BITs */
51 #define SPRD_IEN		0x0010
52 #define SPRD_IEN_RX_FULL	BIT(0)
53 #define SPRD_IEN_TX_EMPTY	BIT(1)
54 #define SPRD_IEN_BREAK_DETECT	BIT(7)
55 #define SPRD_IEN_TIMEOUT	BIT(13)
56 
57 /* interrupt clear register */
58 #define SPRD_ICLR		0x0014
59 #define SPRD_ICLR_TIMEOUT	BIT(13)
60 
61 /* line control register */
62 #define SPRD_LCR		0x0018
63 #define SPRD_LCR_STOP_1BIT	0x10
64 #define SPRD_LCR_STOP_2BIT	0x30
65 #define SPRD_LCR_DATA_LEN	(BIT(2) | BIT(3))
66 #define SPRD_LCR_DATA_LEN5	0x0
67 #define SPRD_LCR_DATA_LEN6	0x4
68 #define SPRD_LCR_DATA_LEN7	0x8
69 #define SPRD_LCR_DATA_LEN8	0xc
70 #define SPRD_LCR_PARITY		(BIT(0) | BIT(1))
71 #define SPRD_LCR_PARITY_EN	0x2
72 #define SPRD_LCR_EVEN_PAR	0x0
73 #define SPRD_LCR_ODD_PAR	0x1
74 
75 /* control register 1 */
76 #define SPRD_CTL1		0x001C
77 #define SPRD_DMA_EN		BIT(15)
78 #define SPRD_LOOPBACK_EN	BIT(14)
79 #define RX_HW_FLOW_CTL_THLD	BIT(6)
80 #define RX_HW_FLOW_CTL_EN	BIT(7)
81 #define TX_HW_FLOW_CTL_EN	BIT(8)
82 #define RX_TOUT_THLD_DEF	0x3E00
83 #define RX_HFC_THLD_DEF		0x40
84 
85 /* fifo threshold register */
86 #define SPRD_CTL2		0x0020
87 #define THLD_TX_EMPTY		0x40
88 #define THLD_TX_EMPTY_SHIFT	8
89 #define THLD_RX_FULL		0x40
90 #define THLD_RX_FULL_MASK	GENMASK(6, 0)
91 
92 /* config baud rate register */
93 #define SPRD_CLKD0		0x0024
94 #define SPRD_CLKD0_MASK		GENMASK(15, 0)
95 #define SPRD_CLKD1		0x0028
96 #define SPRD_CLKD1_MASK		GENMASK(20, 16)
97 #define SPRD_CLKD1_SHIFT	16
98 
99 /* interrupt mask status register */
100 #define SPRD_IMSR		0x002C
101 #define SPRD_IMSR_RX_FIFO_FULL	BIT(0)
102 #define SPRD_IMSR_TX_FIFO_EMPTY	BIT(1)
103 #define SPRD_IMSR_BREAK_DETECT	BIT(7)
104 #define SPRD_IMSR_TIMEOUT	BIT(13)
105 #define SPRD_DEFAULT_SOURCE_CLK	26000000
106 
107 #define SPRD_RX_DMA_STEP	1
108 #define SPRD_RX_FIFO_FULL	1
109 #define SPRD_TX_FIFO_FULL	0x20
110 #define SPRD_UART_RX_SIZE	(UART_XMIT_SIZE / 4)
111 
112 struct sprd_uart_dma {
113 	struct dma_chan *chn;
114 	unsigned char *virt;
115 	dma_addr_t phys_addr;
116 	dma_cookie_t cookie;
117 	u32 trans_len;
118 	bool enable;
119 };
120 
121 struct sprd_uart_port {
122 	struct uart_port port;
123 	char name[16];
124 	struct clk *clk;
125 	struct sprd_uart_dma tx_dma;
126 	struct sprd_uart_dma rx_dma;
127 	dma_addr_t pos;
128 	unsigned char *rx_buf_tail;
129 };
130 
131 static struct sprd_uart_port *sprd_port[UART_NR_MAX];
132 static int sprd_ports_num;
133 
134 static int sprd_start_dma_rx(struct uart_port *port);
135 static int sprd_tx_dma_config(struct uart_port *port);
136 
137 static inline unsigned int serial_in(struct uart_port *port,
138 				     unsigned int offset)
139 {
140 	return readl_relaxed(port->membase + offset);
141 }
142 
143 static inline void serial_out(struct uart_port *port, unsigned int offset,
144 			      int value)
145 {
146 	writel_relaxed(value, port->membase + offset);
147 }
148 
149 static unsigned int sprd_tx_empty(struct uart_port *port)
150 {
151 	if (serial_in(port, SPRD_STS1) & SPRD_TX_FIFO_CNT_MASK)
152 		return 0;
153 	else
154 		return TIOCSER_TEMT;
155 }
156 
157 static unsigned int sprd_get_mctrl(struct uart_port *port)
158 {
159 	return TIOCM_DSR | TIOCM_CTS;
160 }
161 
162 static void sprd_set_mctrl(struct uart_port *port, unsigned int mctrl)
163 {
164 	u32 val = serial_in(port, SPRD_CTL1);
165 
166 	if (mctrl & TIOCM_LOOP)
167 		val |= SPRD_LOOPBACK_EN;
168 	else
169 		val &= ~SPRD_LOOPBACK_EN;
170 
171 	serial_out(port, SPRD_CTL1, val);
172 }
173 
174 static void sprd_stop_rx(struct uart_port *port)
175 {
176 	struct sprd_uart_port *sp =
177 		container_of(port, struct sprd_uart_port, port);
178 	unsigned int ien, iclr;
179 
180 	if (sp->rx_dma.enable)
181 		dmaengine_terminate_all(sp->rx_dma.chn);
182 
183 	iclr = serial_in(port, SPRD_ICLR);
184 	ien = serial_in(port, SPRD_IEN);
185 
186 	ien &= ~(SPRD_IEN_RX_FULL | SPRD_IEN_BREAK_DETECT);
187 	iclr |= SPRD_IEN_RX_FULL | SPRD_IEN_BREAK_DETECT;
188 
189 	serial_out(port, SPRD_IEN, ien);
190 	serial_out(port, SPRD_ICLR, iclr);
191 }
192 
193 static void sprd_uart_dma_enable(struct uart_port *port, bool enable)
194 {
195 	u32 val = serial_in(port, SPRD_CTL1);
196 
197 	if (enable)
198 		val |= SPRD_DMA_EN;
199 	else
200 		val &= ~SPRD_DMA_EN;
201 
202 	serial_out(port, SPRD_CTL1, val);
203 }
204 
205 static void sprd_stop_tx_dma(struct uart_port *port)
206 {
207 	struct sprd_uart_port *sp =
208 		container_of(port, struct sprd_uart_port, port);
209 	struct dma_tx_state state;
210 	u32 trans_len;
211 
212 	dmaengine_pause(sp->tx_dma.chn);
213 
214 	dmaengine_tx_status(sp->tx_dma.chn, sp->tx_dma.cookie, &state);
215 	if (state.residue) {
216 		trans_len = state.residue - sp->tx_dma.phys_addr;
217 		uart_xmit_advance(port, trans_len);
218 		dma_unmap_single(port->dev, sp->tx_dma.phys_addr,
219 				 sp->tx_dma.trans_len, DMA_TO_DEVICE);
220 	}
221 
222 	dmaengine_terminate_all(sp->tx_dma.chn);
223 	sp->tx_dma.trans_len = 0;
224 }
225 
226 static int sprd_tx_buf_remap(struct uart_port *port)
227 {
228 	struct sprd_uart_port *sp =
229 		container_of(port, struct sprd_uart_port, port);
230 	struct tty_port *tport = &port->state->port;
231 	unsigned char *tail;
232 
233 	sp->tx_dma.trans_len = kfifo_out_linear_ptr(&tport->xmit_fifo, &tail,
234 			UART_XMIT_SIZE);
235 
236 	sp->tx_dma.phys_addr = dma_map_single(port->dev, tail,
237 					      sp->tx_dma.trans_len,
238 					      DMA_TO_DEVICE);
239 	return dma_mapping_error(port->dev, sp->tx_dma.phys_addr);
240 }
241 
242 static void sprd_complete_tx_dma(void *data)
243 {
244 	struct uart_port *port = (struct uart_port *)data;
245 	struct sprd_uart_port *sp =
246 		container_of(port, struct sprd_uart_port, port);
247 	struct tty_port *tport = &port->state->port;
248 	unsigned long flags;
249 
250 	uart_port_lock_irqsave(port, &flags);
251 	dma_unmap_single(port->dev, sp->tx_dma.phys_addr,
252 			 sp->tx_dma.trans_len, DMA_TO_DEVICE);
253 
254 	uart_xmit_advance(port, sp->tx_dma.trans_len);
255 
256 	if (kfifo_len(&tport->xmit_fifo) < WAKEUP_CHARS)
257 		uart_write_wakeup(port);
258 
259 	if (kfifo_is_empty(&tport->xmit_fifo) || sprd_tx_buf_remap(port) ||
260 	    sprd_tx_dma_config(port))
261 		sp->tx_dma.trans_len = 0;
262 
263 	uart_port_unlock_irqrestore(port, flags);
264 }
265 
266 static int sprd_uart_dma_submit(struct uart_port *port,
267 				struct sprd_uart_dma *ud, u32 trans_len,
268 				enum dma_transfer_direction direction,
269 				dma_async_tx_callback callback)
270 {
271 	struct dma_async_tx_descriptor *dma_des;
272 	unsigned long flags;
273 
274 	flags = SPRD_DMA_FLAGS(SPRD_DMA_CHN_MODE_NONE,
275 			       SPRD_DMA_NO_TRG,
276 			       SPRD_DMA_FRAG_REQ,
277 			       SPRD_DMA_TRANS_INT);
278 
279 	dma_des = dmaengine_prep_slave_single(ud->chn, ud->phys_addr, trans_len,
280 					      direction, flags);
281 	if (!dma_des)
282 		return -ENODEV;
283 
284 	dma_des->callback = callback;
285 	dma_des->callback_param = port;
286 
287 	ud->cookie = dmaengine_submit(dma_des);
288 	if (dma_submit_error(ud->cookie))
289 		return dma_submit_error(ud->cookie);
290 
291 	dma_async_issue_pending(ud->chn);
292 
293 	return 0;
294 }
295 
296 static int sprd_tx_dma_config(struct uart_port *port)
297 {
298 	struct sprd_uart_port *sp =
299 		container_of(port, struct sprd_uart_port, port);
300 	u32 burst = sp->tx_dma.trans_len > SPRD_TX_FIFO_FULL ?
301 		SPRD_TX_FIFO_FULL : sp->tx_dma.trans_len;
302 	int ret;
303 	struct dma_slave_config cfg = {
304 		.dst_addr = port->mapbase + SPRD_TXD,
305 		.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
306 		.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
307 		.src_maxburst = burst,
308 	};
309 
310 	ret = dmaengine_slave_config(sp->tx_dma.chn, &cfg);
311 	if (ret < 0)
312 		return ret;
313 
314 	return sprd_uart_dma_submit(port, &sp->tx_dma, sp->tx_dma.trans_len,
315 				    DMA_MEM_TO_DEV, sprd_complete_tx_dma);
316 }
317 
318 static void sprd_start_tx_dma(struct uart_port *port)
319 {
320 	struct sprd_uart_port *sp =
321 		container_of(port, struct sprd_uart_port, port);
322 	struct tty_port *tport = &port->state->port;
323 
324 	if (port->x_char) {
325 		serial_out(port, SPRD_TXD, port->x_char);
326 		port->icount.tx++;
327 		port->x_char = 0;
328 		return;
329 	}
330 
331 	if (kfifo_is_empty(&tport->xmit_fifo) || uart_tx_stopped(port)) {
332 		sprd_stop_tx_dma(port);
333 		return;
334 	}
335 
336 	if (sp->tx_dma.trans_len)
337 		return;
338 
339 	if (sprd_tx_buf_remap(port) || sprd_tx_dma_config(port))
340 		sp->tx_dma.trans_len = 0;
341 }
342 
343 static void sprd_rx_full_thld(struct uart_port *port, u32 thld)
344 {
345 	u32 val = serial_in(port, SPRD_CTL2);
346 
347 	val &= ~THLD_RX_FULL_MASK;
348 	val |= thld & THLD_RX_FULL_MASK;
349 	serial_out(port, SPRD_CTL2, val);
350 }
351 
352 static int sprd_rx_alloc_buf(struct sprd_uart_port *sp)
353 {
354 	sp->rx_dma.virt = dma_alloc_coherent(sp->port.dev, SPRD_UART_RX_SIZE,
355 					     &sp->rx_dma.phys_addr, GFP_KERNEL);
356 	if (!sp->rx_dma.virt)
357 		return -ENOMEM;
358 
359 	return 0;
360 }
361 
362 static void sprd_rx_free_buf(struct sprd_uart_port *sp)
363 {
364 	if (sp->rx_dma.virt)
365 		dma_free_coherent(sp->port.dev, SPRD_UART_RX_SIZE,
366 				  sp->rx_dma.virt, sp->rx_dma.phys_addr);
367 	sp->rx_dma.virt = NULL;
368 }
369 
370 static int sprd_rx_dma_config(struct uart_port *port, u32 burst)
371 {
372 	struct sprd_uart_port *sp =
373 		container_of(port, struct sprd_uart_port, port);
374 	struct dma_slave_config cfg = {
375 		.src_addr = port->mapbase + SPRD_RXD,
376 		.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
377 		.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
378 		.src_maxburst = burst,
379 	};
380 
381 	return dmaengine_slave_config(sp->rx_dma.chn, &cfg);
382 }
383 
384 static void sprd_uart_dma_rx(struct uart_port *port)
385 {
386 	struct sprd_uart_port *sp =
387 		container_of(port, struct sprd_uart_port, port);
388 	struct tty_port *tty = &port->state->port;
389 
390 	port->icount.rx += sp->rx_dma.trans_len;
391 	tty_insert_flip_string(tty, sp->rx_buf_tail, sp->rx_dma.trans_len);
392 	tty_flip_buffer_push(tty);
393 }
394 
395 static void sprd_uart_dma_irq(struct uart_port *port)
396 {
397 	struct sprd_uart_port *sp =
398 		container_of(port, struct sprd_uart_port, port);
399 	struct dma_tx_state state;
400 	enum dma_status status;
401 
402 	status = dmaengine_tx_status(sp->rx_dma.chn,
403 				     sp->rx_dma.cookie, &state);
404 	if (status == DMA_ERROR)
405 		sprd_stop_rx(port);
406 
407 	if (!state.residue && sp->pos == sp->rx_dma.phys_addr)
408 		return;
409 
410 	if (!state.residue) {
411 		sp->rx_dma.trans_len = SPRD_UART_RX_SIZE +
412 			sp->rx_dma.phys_addr - sp->pos;
413 		sp->pos = sp->rx_dma.phys_addr;
414 	} else {
415 		sp->rx_dma.trans_len = state.residue - sp->pos;
416 		sp->pos = state.residue;
417 	}
418 
419 	sprd_uart_dma_rx(port);
420 	sp->rx_buf_tail += sp->rx_dma.trans_len;
421 }
422 
423 static void sprd_complete_rx_dma(void *data)
424 {
425 	struct uart_port *port = (struct uart_port *)data;
426 	struct sprd_uart_port *sp =
427 		container_of(port, struct sprd_uart_port, port);
428 	struct dma_tx_state state;
429 	enum dma_status status;
430 	unsigned long flags;
431 
432 	uart_port_lock_irqsave(port, &flags);
433 
434 	status = dmaengine_tx_status(sp->rx_dma.chn,
435 				     sp->rx_dma.cookie, &state);
436 	if (status != DMA_COMPLETE) {
437 		sprd_stop_rx(port);
438 		uart_port_unlock_irqrestore(port, flags);
439 		return;
440 	}
441 
442 	if (sp->pos != sp->rx_dma.phys_addr) {
443 		sp->rx_dma.trans_len =  SPRD_UART_RX_SIZE +
444 			sp->rx_dma.phys_addr - sp->pos;
445 		sprd_uart_dma_rx(port);
446 		sp->rx_buf_tail += sp->rx_dma.trans_len;
447 	}
448 
449 	if (sprd_start_dma_rx(port))
450 		sprd_stop_rx(port);
451 
452 	uart_port_unlock_irqrestore(port, flags);
453 }
454 
455 static int sprd_start_dma_rx(struct uart_port *port)
456 {
457 	struct sprd_uart_port *sp =
458 		container_of(port, struct sprd_uart_port, port);
459 	int ret;
460 
461 	if (!sp->rx_dma.enable)
462 		return 0;
463 
464 	sp->pos = sp->rx_dma.phys_addr;
465 	sp->rx_buf_tail = sp->rx_dma.virt;
466 	sprd_rx_full_thld(port, SPRD_RX_FIFO_FULL);
467 	ret = sprd_rx_dma_config(port, SPRD_RX_DMA_STEP);
468 	if (ret)
469 		return ret;
470 
471 	return sprd_uart_dma_submit(port, &sp->rx_dma, SPRD_UART_RX_SIZE,
472 				    DMA_DEV_TO_MEM, sprd_complete_rx_dma);
473 }
474 
475 static void sprd_release_dma(struct uart_port *port)
476 {
477 	struct sprd_uart_port *sp =
478 		container_of(port, struct sprd_uart_port, port);
479 
480 	sprd_uart_dma_enable(port, false);
481 
482 	if (sp->rx_dma.enable)
483 		dma_release_channel(sp->rx_dma.chn);
484 
485 	if (sp->tx_dma.enable)
486 		dma_release_channel(sp->tx_dma.chn);
487 
488 	sp->tx_dma.enable = false;
489 	sp->rx_dma.enable = false;
490 }
491 
492 static void sprd_request_dma(struct uart_port *port)
493 {
494 	struct sprd_uart_port *sp =
495 		container_of(port, struct sprd_uart_port, port);
496 
497 	sp->tx_dma.enable = true;
498 	sp->rx_dma.enable = true;
499 
500 	sp->tx_dma.chn = dma_request_chan(port->dev, "tx");
501 	if (IS_ERR(sp->tx_dma.chn)) {
502 		dev_err(port->dev, "request TX DMA channel failed, ret = %ld\n",
503 			PTR_ERR(sp->tx_dma.chn));
504 		sp->tx_dma.enable = false;
505 	}
506 
507 	sp->rx_dma.chn = dma_request_chan(port->dev, "rx");
508 	if (IS_ERR(sp->rx_dma.chn)) {
509 		dev_err(port->dev, "request RX DMA channel failed, ret = %ld\n",
510 			PTR_ERR(sp->rx_dma.chn));
511 		sp->rx_dma.enable = false;
512 	}
513 }
514 
515 static void sprd_stop_tx(struct uart_port *port)
516 {
517 	struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,
518 						 port);
519 	unsigned int ien, iclr;
520 
521 	if (sp->tx_dma.enable) {
522 		sprd_stop_tx_dma(port);
523 		return;
524 	}
525 
526 	iclr = serial_in(port, SPRD_ICLR);
527 	ien = serial_in(port, SPRD_IEN);
528 
529 	iclr |= SPRD_IEN_TX_EMPTY;
530 	ien &= ~SPRD_IEN_TX_EMPTY;
531 
532 	serial_out(port, SPRD_IEN, ien);
533 	serial_out(port, SPRD_ICLR, iclr);
534 }
535 
536 static void sprd_start_tx(struct uart_port *port)
537 {
538 	struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,
539 						 port);
540 	unsigned int ien;
541 
542 	if (sp->tx_dma.enable) {
543 		sprd_start_tx_dma(port);
544 		return;
545 	}
546 
547 	ien = serial_in(port, SPRD_IEN);
548 	if (!(ien & SPRD_IEN_TX_EMPTY)) {
549 		ien |= SPRD_IEN_TX_EMPTY;
550 		serial_out(port, SPRD_IEN, ien);
551 	}
552 }
553 
554 /* The Sprd serial does not support this function. */
555 static void sprd_break_ctl(struct uart_port *port, int break_state)
556 {
557 	/* nothing to do */
558 }
559 
560 static int handle_lsr_errors(struct uart_port *port,
561 			     u8 *flag,
562 			     unsigned int *lsr)
563 {
564 	int ret = 0;
565 
566 	/* statistics */
567 	if (*lsr & SPRD_LSR_BI) {
568 		*lsr &= ~(SPRD_LSR_FE | SPRD_LSR_PE);
569 		port->icount.brk++;
570 		ret = uart_handle_break(port);
571 		if (ret)
572 			return ret;
573 	} else if (*lsr & SPRD_LSR_PE)
574 		port->icount.parity++;
575 	else if (*lsr & SPRD_LSR_FE)
576 		port->icount.frame++;
577 	if (*lsr & SPRD_LSR_OE)
578 		port->icount.overrun++;
579 
580 	/* mask off conditions which should be ignored */
581 	*lsr &= port->read_status_mask;
582 	if (*lsr & SPRD_LSR_BI)
583 		*flag = TTY_BREAK;
584 	else if (*lsr & SPRD_LSR_PE)
585 		*flag = TTY_PARITY;
586 	else if (*lsr & SPRD_LSR_FE)
587 		*flag = TTY_FRAME;
588 
589 	return ret;
590 }
591 
592 static inline void sprd_rx(struct uart_port *port)
593 {
594 	struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,
595 						 port);
596 	struct tty_port *tty = &port->state->port;
597 	unsigned int lsr, max_count = SPRD_TIMEOUT;
598 	u8 ch, flag;
599 
600 	if (sp->rx_dma.enable) {
601 		sprd_uart_dma_irq(port);
602 		return;
603 	}
604 
605 	while ((serial_in(port, SPRD_STS1) & SPRD_RX_FIFO_CNT_MASK) &&
606 	       max_count--) {
607 		lsr = serial_in(port, SPRD_LSR);
608 		ch = serial_in(port, SPRD_RXD);
609 		flag = TTY_NORMAL;
610 		port->icount.rx++;
611 
612 		if (lsr & (SPRD_LSR_BI | SPRD_LSR_PE |
613 			   SPRD_LSR_FE | SPRD_LSR_OE))
614 			if (handle_lsr_errors(port, &flag, &lsr))
615 				continue;
616 		if (uart_handle_sysrq_char(port, ch))
617 			continue;
618 
619 		uart_insert_char(port, lsr, SPRD_LSR_OE, ch, flag);
620 	}
621 
622 	tty_flip_buffer_push(tty);
623 }
624 
625 static inline void sprd_tx(struct uart_port *port)
626 {
627 	u8 ch;
628 
629 	uart_port_tx_limited(port, ch, THLD_TX_EMPTY,
630 		true,
631 		serial_out(port, SPRD_TXD, ch),
632 		({}));
633 }
634 
635 /* this handles the interrupt from one port */
636 static irqreturn_t sprd_handle_irq(int irq, void *dev_id)
637 {
638 	struct uart_port *port = dev_id;
639 	unsigned int ims;
640 
641 	uart_port_lock(port);
642 
643 	ims = serial_in(port, SPRD_IMSR);
644 
645 	if (!ims) {
646 		uart_port_unlock(port);
647 		return IRQ_NONE;
648 	}
649 
650 	if (ims & SPRD_IMSR_TIMEOUT)
651 		serial_out(port, SPRD_ICLR, SPRD_ICLR_TIMEOUT);
652 
653 	if (ims & SPRD_IMSR_BREAK_DETECT)
654 		serial_out(port, SPRD_ICLR, SPRD_IMSR_BREAK_DETECT);
655 
656 	if (ims & (SPRD_IMSR_RX_FIFO_FULL | SPRD_IMSR_BREAK_DETECT |
657 		   SPRD_IMSR_TIMEOUT))
658 		sprd_rx(port);
659 
660 	if (ims & SPRD_IMSR_TX_FIFO_EMPTY)
661 		sprd_tx(port);
662 
663 	uart_port_unlock(port);
664 
665 	return IRQ_HANDLED;
666 }
667 
668 static void sprd_uart_dma_startup(struct uart_port *port,
669 				  struct sprd_uart_port *sp)
670 {
671 	int ret;
672 
673 	sprd_request_dma(port);
674 	if (!(sp->rx_dma.enable || sp->tx_dma.enable))
675 		return;
676 
677 	ret = sprd_start_dma_rx(port);
678 	if (ret) {
679 		sp->rx_dma.enable = false;
680 		dma_release_channel(sp->rx_dma.chn);
681 		dev_warn(port->dev, "fail to start RX dma mode\n");
682 	}
683 
684 	sprd_uart_dma_enable(port, true);
685 }
686 
687 static int sprd_startup(struct uart_port *port)
688 {
689 	int ret = 0;
690 	unsigned int ien, fc;
691 	unsigned int timeout;
692 	struct sprd_uart_port *sp;
693 	unsigned long flags;
694 
695 	serial_out(port, SPRD_CTL2,
696 		   THLD_TX_EMPTY << THLD_TX_EMPTY_SHIFT | THLD_RX_FULL);
697 
698 	/* clear rx fifo */
699 	timeout = SPRD_TIMEOUT;
700 	while (timeout-- && serial_in(port, SPRD_STS1) & SPRD_RX_FIFO_CNT_MASK)
701 		serial_in(port, SPRD_RXD);
702 
703 	/* clear tx fifo */
704 	timeout = SPRD_TIMEOUT;
705 	while (timeout-- && serial_in(port, SPRD_STS1) & SPRD_TX_FIFO_CNT_MASK)
706 		cpu_relax();
707 
708 	/* clear interrupt */
709 	serial_out(port, SPRD_IEN, 0);
710 	serial_out(port, SPRD_ICLR, ~0);
711 
712 	/* allocate irq */
713 	sp = container_of(port, struct sprd_uart_port, port);
714 	snprintf(sp->name, sizeof(sp->name), "sprd_serial%d", port->line);
715 
716 	sprd_uart_dma_startup(port, sp);
717 
718 	ret = devm_request_irq(port->dev, port->irq, sprd_handle_irq,
719 			       IRQF_SHARED, sp->name, port);
720 	if (ret) {
721 		dev_err(port->dev, "fail to request serial irq %d, ret=%d\n",
722 			port->irq, ret);
723 		return ret;
724 	}
725 	fc = serial_in(port, SPRD_CTL1);
726 	fc |= RX_TOUT_THLD_DEF | RX_HFC_THLD_DEF;
727 	serial_out(port, SPRD_CTL1, fc);
728 
729 	/* enable interrupt */
730 	uart_port_lock_irqsave(port, &flags);
731 	ien = serial_in(port, SPRD_IEN);
732 	ien |= SPRD_IEN_BREAK_DETECT | SPRD_IEN_TIMEOUT;
733 	if (!sp->rx_dma.enable)
734 		ien |= SPRD_IEN_RX_FULL;
735 	serial_out(port, SPRD_IEN, ien);
736 	uart_port_unlock_irqrestore(port, flags);
737 
738 	return 0;
739 }
740 
741 static void sprd_shutdown(struct uart_port *port)
742 {
743 	sprd_release_dma(port);
744 	serial_out(port, SPRD_IEN, 0);
745 	serial_out(port, SPRD_ICLR, ~0);
746 	devm_free_irq(port->dev, port->irq, port);
747 }
748 
749 static void sprd_set_termios(struct uart_port *port, struct ktermios *termios,
750 		             const struct ktermios *old)
751 {
752 	unsigned int baud, quot;
753 	unsigned int lcr = 0, fc;
754 	unsigned long flags;
755 
756 	/* ask the core to calculate the divisor for us */
757 	baud = uart_get_baud_rate(port, termios, old, 0, SPRD_BAUD_IO_LIMIT);
758 
759 	quot = port->uartclk / baud;
760 
761 	/* set data length */
762 	switch (termios->c_cflag & CSIZE) {
763 	case CS5:
764 		lcr |= SPRD_LCR_DATA_LEN5;
765 		break;
766 	case CS6:
767 		lcr |= SPRD_LCR_DATA_LEN6;
768 		break;
769 	case CS7:
770 		lcr |= SPRD_LCR_DATA_LEN7;
771 		break;
772 	case CS8:
773 	default:
774 		lcr |= SPRD_LCR_DATA_LEN8;
775 		break;
776 	}
777 
778 	/* calculate stop bits */
779 	lcr &= ~(SPRD_LCR_STOP_1BIT | SPRD_LCR_STOP_2BIT);
780 	if (termios->c_cflag & CSTOPB)
781 		lcr |= SPRD_LCR_STOP_2BIT;
782 	else
783 		lcr |= SPRD_LCR_STOP_1BIT;
784 
785 	/* calculate parity */
786 	lcr &= ~SPRD_LCR_PARITY;
787 	termios->c_cflag &= ~CMSPAR;	/* no support mark/space */
788 	if (termios->c_cflag & PARENB) {
789 		lcr |= SPRD_LCR_PARITY_EN;
790 		if (termios->c_cflag & PARODD)
791 			lcr |= SPRD_LCR_ODD_PAR;
792 		else
793 			lcr |= SPRD_LCR_EVEN_PAR;
794 	}
795 
796 	uart_port_lock_irqsave(port, &flags);
797 
798 	/* update the per-port timeout */
799 	uart_update_timeout(port, termios->c_cflag, baud);
800 
801 	port->read_status_mask = SPRD_LSR_OE;
802 	if (termios->c_iflag & INPCK)
803 		port->read_status_mask |= SPRD_LSR_FE | SPRD_LSR_PE;
804 	if (termios->c_iflag & (IGNBRK | BRKINT | PARMRK))
805 		port->read_status_mask |= SPRD_LSR_BI;
806 
807 	/* characters to ignore */
808 	port->ignore_status_mask = 0;
809 	if (termios->c_iflag & IGNPAR)
810 		port->ignore_status_mask |= SPRD_LSR_PE | SPRD_LSR_FE;
811 	if (termios->c_iflag & IGNBRK) {
812 		port->ignore_status_mask |= SPRD_LSR_BI;
813 		/*
814 		 * If we're ignoring parity and break indicators,
815 		 * ignore overruns too (for real raw support).
816 		 */
817 		if (termios->c_iflag & IGNPAR)
818 			port->ignore_status_mask |= SPRD_LSR_OE;
819 	}
820 
821 	/* flow control */
822 	fc = serial_in(port, SPRD_CTL1);
823 	fc &= ~(RX_HW_FLOW_CTL_THLD | RX_HW_FLOW_CTL_EN | TX_HW_FLOW_CTL_EN);
824 	if (termios->c_cflag & CRTSCTS) {
825 		fc |= RX_HW_FLOW_CTL_THLD;
826 		fc |= RX_HW_FLOW_CTL_EN;
827 		fc |= TX_HW_FLOW_CTL_EN;
828 	}
829 
830 	/* clock divider bit0~bit15 */
831 	serial_out(port, SPRD_CLKD0, quot & SPRD_CLKD0_MASK);
832 
833 	/* clock divider bit16~bit20 */
834 	serial_out(port, SPRD_CLKD1,
835 		   (quot & SPRD_CLKD1_MASK) >> SPRD_CLKD1_SHIFT);
836 	serial_out(port, SPRD_LCR, lcr);
837 	fc |= RX_TOUT_THLD_DEF | RX_HFC_THLD_DEF;
838 	serial_out(port, SPRD_CTL1, fc);
839 
840 	uart_port_unlock_irqrestore(port, flags);
841 
842 	/* Don't rewrite B0 */
843 	if (tty_termios_baud_rate(termios))
844 		tty_termios_encode_baud_rate(termios, baud, baud);
845 }
846 
847 static const char *sprd_type(struct uart_port *port)
848 {
849 	return "SPX";
850 }
851 
852 static void sprd_release_port(struct uart_port *port)
853 {
854 	/* nothing to do */
855 }
856 
857 static int sprd_request_port(struct uart_port *port)
858 {
859 	return 0;
860 }
861 
862 static void sprd_config_port(struct uart_port *port, int flags)
863 {
864 	if (flags & UART_CONFIG_TYPE)
865 		port->type = PORT_SPRD;
866 }
867 
868 static int sprd_verify_port(struct uart_port *port, struct serial_struct *ser)
869 {
870 	if (ser->type != PORT_SPRD)
871 		return -EINVAL;
872 	if (port->irq != ser->irq)
873 		return -EINVAL;
874 	if (port->iotype != ser->io_type)
875 		return -EINVAL;
876 	return 0;
877 }
878 
879 static void sprd_pm(struct uart_port *port, unsigned int state,
880 		unsigned int oldstate)
881 {
882 	struct sprd_uart_port *sup =
883 		container_of(port, struct sprd_uart_port, port);
884 
885 	switch (state) {
886 	case UART_PM_STATE_ON:
887 		clk_prepare_enable(sup->clk);
888 		break;
889 	case UART_PM_STATE_OFF:
890 		clk_disable_unprepare(sup->clk);
891 		break;
892 	}
893 }
894 
895 #ifdef CONFIG_CONSOLE_POLL
896 static int sprd_poll_init(struct uart_port *port)
897 {
898 	if (port->state->pm_state != UART_PM_STATE_ON) {
899 		sprd_pm(port, UART_PM_STATE_ON, 0);
900 		port->state->pm_state = UART_PM_STATE_ON;
901 	}
902 
903 	return 0;
904 }
905 
906 static int sprd_poll_get_char(struct uart_port *port)
907 {
908 	while (!(serial_in(port, SPRD_STS1) & SPRD_RX_FIFO_CNT_MASK))
909 		cpu_relax();
910 
911 	return serial_in(port, SPRD_RXD);
912 }
913 
914 static void sprd_poll_put_char(struct uart_port *port, unsigned char ch)
915 {
916 	while (serial_in(port, SPRD_STS1) & SPRD_TX_FIFO_CNT_MASK)
917 		cpu_relax();
918 
919 	serial_out(port, SPRD_TXD, ch);
920 }
921 #endif
922 
923 static const struct uart_ops serial_sprd_ops = {
924 	.tx_empty = sprd_tx_empty,
925 	.get_mctrl = sprd_get_mctrl,
926 	.set_mctrl = sprd_set_mctrl,
927 	.stop_tx = sprd_stop_tx,
928 	.start_tx = sprd_start_tx,
929 	.stop_rx = sprd_stop_rx,
930 	.break_ctl = sprd_break_ctl,
931 	.startup = sprd_startup,
932 	.shutdown = sprd_shutdown,
933 	.set_termios = sprd_set_termios,
934 	.type = sprd_type,
935 	.release_port = sprd_release_port,
936 	.request_port = sprd_request_port,
937 	.config_port = sprd_config_port,
938 	.verify_port = sprd_verify_port,
939 	.pm = sprd_pm,
940 #ifdef CONFIG_CONSOLE_POLL
941 	.poll_init	= sprd_poll_init,
942 	.poll_get_char	= sprd_poll_get_char,
943 	.poll_put_char	= sprd_poll_put_char,
944 #endif
945 };
946 
947 #ifdef CONFIG_SERIAL_SPRD_CONSOLE
948 static void wait_for_xmitr(struct uart_port *port)
949 {
950 	unsigned int status, tmout = 10000;
951 
952 	/* wait up to 10ms for the character(s) to be sent */
953 	do {
954 		status = serial_in(port, SPRD_STS1);
955 		if (--tmout == 0)
956 			break;
957 		udelay(1);
958 	} while (status & SPRD_TX_FIFO_CNT_MASK);
959 }
960 
961 static void sprd_console_putchar(struct uart_port *port, unsigned char ch)
962 {
963 	wait_for_xmitr(port);
964 	serial_out(port, SPRD_TXD, ch);
965 }
966 
967 static void sprd_console_write(struct console *co, const char *s,
968 			       unsigned int count)
969 {
970 	struct uart_port *port = &sprd_port[co->index]->port;
971 	int locked = 1;
972 	unsigned long flags;
973 
974 	if (port->sysrq)
975 		locked = 0;
976 	else if (oops_in_progress)
977 		locked = uart_port_trylock_irqsave(port, &flags);
978 	else
979 		uart_port_lock_irqsave(port, &flags);
980 
981 	uart_console_write(port, s, count, sprd_console_putchar);
982 
983 	/* wait for transmitter to become empty */
984 	wait_for_xmitr(port);
985 
986 	if (locked)
987 		uart_port_unlock_irqrestore(port, flags);
988 }
989 
990 static int sprd_console_setup(struct console *co, char *options)
991 {
992 	struct sprd_uart_port *sprd_uart_port;
993 	int baud = 115200;
994 	int bits = 8;
995 	int parity = 'n';
996 	int flow = 'n';
997 
998 	if (co->index >= UART_NR_MAX || co->index < 0)
999 		co->index = 0;
1000 
1001 	sprd_uart_port = sprd_port[co->index];
1002 	if (!sprd_uart_port || !sprd_uart_port->port.membase) {
1003 		pr_info("serial port %d not yet initialized\n", co->index);
1004 		return -ENODEV;
1005 	}
1006 
1007 	if (options)
1008 		uart_parse_options(options, &baud, &parity, &bits, &flow);
1009 
1010 	return uart_set_options(&sprd_uart_port->port, co, baud,
1011 				parity, bits, flow);
1012 }
1013 
1014 static struct uart_driver sprd_uart_driver;
1015 static struct console sprd_console = {
1016 	.name = SPRD_TTY_NAME,
1017 	.write = sprd_console_write,
1018 	.device = uart_console_device,
1019 	.setup = sprd_console_setup,
1020 	.flags = CON_PRINTBUFFER,
1021 	.index = -1,
1022 	.data = &sprd_uart_driver,
1023 };
1024 
1025 static int __init sprd_serial_console_init(void)
1026 {
1027 	register_console(&sprd_console);
1028 	return 0;
1029 }
1030 console_initcall(sprd_serial_console_init);
1031 
1032 #define SPRD_CONSOLE	(&sprd_console)
1033 
1034 /* Support for earlycon */
1035 static void sprd_putc(struct uart_port *port, unsigned char c)
1036 {
1037 	unsigned int timeout = SPRD_TIMEOUT;
1038 
1039 	while (timeout-- &&
1040 	       !(readl(port->membase + SPRD_LSR) & SPRD_LSR_TX_OVER))
1041 		cpu_relax();
1042 
1043 	writeb(c, port->membase + SPRD_TXD);
1044 }
1045 
1046 static void sprd_early_write(struct console *con, const char *s, unsigned int n)
1047 {
1048 	struct earlycon_device *dev = con->data;
1049 
1050 	uart_console_write(&dev->port, s, n, sprd_putc);
1051 }
1052 
1053 static int __init sprd_early_console_setup(struct earlycon_device *device,
1054 					   const char *opt)
1055 {
1056 	if (!device->port.membase)
1057 		return -ENODEV;
1058 
1059 	device->con->write = sprd_early_write;
1060 	return 0;
1061 }
1062 OF_EARLYCON_DECLARE(sprd_serial, "sprd,sc9836-uart",
1063 		    sprd_early_console_setup);
1064 
1065 #else /* !CONFIG_SERIAL_SPRD_CONSOLE */
1066 #define SPRD_CONSOLE		NULL
1067 #endif
1068 
1069 static struct uart_driver sprd_uart_driver = {
1070 	.owner = THIS_MODULE,
1071 	.driver_name = "sprd_serial",
1072 	.dev_name = SPRD_TTY_NAME,
1073 	.major = 0,
1074 	.minor = 0,
1075 	.nr = UART_NR_MAX,
1076 	.cons = SPRD_CONSOLE,
1077 };
1078 
1079 static void sprd_remove(struct platform_device *dev)
1080 {
1081 	struct sprd_uart_port *sup = platform_get_drvdata(dev);
1082 
1083 	if (sup) {
1084 		uart_remove_one_port(&sprd_uart_driver, &sup->port);
1085 		sprd_port[sup->port.line] = NULL;
1086 		sprd_rx_free_buf(sup);
1087 		sprd_ports_num--;
1088 	}
1089 
1090 	if (!sprd_ports_num)
1091 		uart_unregister_driver(&sprd_uart_driver);
1092 }
1093 
1094 static bool sprd_uart_is_console(struct uart_port *uport)
1095 {
1096 	struct console *cons = sprd_uart_driver.cons;
1097 
1098 	if ((cons && cons->index >= 0 && cons->index == uport->line) ||
1099 	    of_console_check(uport->dev->of_node, SPRD_TTY_NAME, uport->line))
1100 		return true;
1101 
1102 	return false;
1103 }
1104 
1105 static int sprd_clk_init(struct uart_port *uport)
1106 {
1107 	struct clk *clk_uart, *clk_parent;
1108 	struct sprd_uart_port *u = container_of(uport, struct sprd_uart_port, port);
1109 
1110 	clk_uart = devm_clk_get(uport->dev, "uart");
1111 	if (IS_ERR(clk_uart)) {
1112 		dev_warn(uport->dev, "uart%d can't get uart clock\n",
1113 			 uport->line);
1114 		clk_uart = NULL;
1115 	}
1116 
1117 	clk_parent = devm_clk_get(uport->dev, "source");
1118 	if (IS_ERR(clk_parent)) {
1119 		dev_warn(uport->dev, "uart%d can't get source clock\n",
1120 			 uport->line);
1121 		clk_parent = NULL;
1122 	}
1123 
1124 	if (!clk_uart || clk_set_parent(clk_uart, clk_parent))
1125 		uport->uartclk = SPRD_DEFAULT_SOURCE_CLK;
1126 	else
1127 		uport->uartclk = clk_get_rate(clk_uart);
1128 
1129 	u->clk = devm_clk_get(uport->dev, "enable");
1130 	if (IS_ERR(u->clk)) {
1131 		if (PTR_ERR(u->clk) == -EPROBE_DEFER)
1132 			return -EPROBE_DEFER;
1133 
1134 		dev_warn(uport->dev, "uart%d can't get enable clock\n",
1135 			uport->line);
1136 
1137 		/* To keep console alive even if the error occurred */
1138 		if (!sprd_uart_is_console(uport))
1139 			return PTR_ERR(u->clk);
1140 
1141 		u->clk = NULL;
1142 	}
1143 
1144 	return 0;
1145 }
1146 
1147 static int sprd_probe(struct platform_device *pdev)
1148 {
1149 	struct resource *res;
1150 	struct uart_port *up;
1151 	struct sprd_uart_port *sport;
1152 	int irq;
1153 	int index;
1154 	int ret;
1155 
1156 	index = of_alias_get_id(pdev->dev.of_node, "serial");
1157 	if (index < 0 || index >= UART_NR_MAX) {
1158 		dev_err(&pdev->dev, "got a wrong serial alias id %d\n", index);
1159 		return -EINVAL;
1160 	}
1161 
1162 	sport = devm_kzalloc(&pdev->dev, sizeof(*sport), GFP_KERNEL);
1163 	if (!sport)
1164 		return -ENOMEM;
1165 
1166 	up = &sport->port;
1167 	up->dev = &pdev->dev;
1168 	up->line = index;
1169 	up->type = PORT_SPRD;
1170 	up->iotype = UPIO_MEM;
1171 	up->uartclk = SPRD_DEF_RATE;
1172 	up->fifosize = SPRD_FIFO_SIZE;
1173 	up->ops = &serial_sprd_ops;
1174 	up->flags = UPF_BOOT_AUTOCONF;
1175 	up->has_sysrq = IS_ENABLED(CONFIG_SERIAL_SPRD_CONSOLE);
1176 
1177 	ret = sprd_clk_init(up);
1178 	if (ret)
1179 		return ret;
1180 
1181 	up->membase = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
1182 	if (IS_ERR(up->membase))
1183 		return PTR_ERR(up->membase);
1184 
1185 	up->mapbase = res->start;
1186 
1187 	irq = platform_get_irq(pdev, 0);
1188 	if (irq < 0)
1189 		return irq;
1190 	up->irq = irq;
1191 
1192 	/*
1193 	 * Allocate one dma buffer to prepare for receive transfer, in case
1194 	 * memory allocation failure at runtime.
1195 	 */
1196 	ret = sprd_rx_alloc_buf(sport);
1197 	if (ret)
1198 		return ret;
1199 
1200 	if (!sprd_ports_num) {
1201 		ret = uart_register_driver(&sprd_uart_driver);
1202 		if (ret < 0) {
1203 			pr_err("Failed to register SPRD-UART driver\n");
1204 			goto free_rx_buf;
1205 		}
1206 	}
1207 
1208 	sprd_ports_num++;
1209 	sprd_port[index] = sport;
1210 
1211 	ret = uart_add_one_port(&sprd_uart_driver, up);
1212 	if (ret)
1213 		goto clean_port;
1214 
1215 	platform_set_drvdata(pdev, up);
1216 
1217 	return 0;
1218 
1219 clean_port:
1220 	sprd_port[index] = NULL;
1221 	if (--sprd_ports_num == 0)
1222 		uart_unregister_driver(&sprd_uart_driver);
1223 free_rx_buf:
1224 	sprd_rx_free_buf(sport);
1225 	return ret;
1226 }
1227 
1228 #ifdef CONFIG_PM_SLEEP
1229 static int sprd_suspend(struct device *dev)
1230 {
1231 	struct sprd_uart_port *sup = dev_get_drvdata(dev);
1232 
1233 	uart_suspend_port(&sprd_uart_driver, &sup->port);
1234 
1235 	return 0;
1236 }
1237 
1238 static int sprd_resume(struct device *dev)
1239 {
1240 	struct sprd_uart_port *sup = dev_get_drvdata(dev);
1241 
1242 	uart_resume_port(&sprd_uart_driver, &sup->port);
1243 
1244 	return 0;
1245 }
1246 #endif
1247 
1248 static SIMPLE_DEV_PM_OPS(sprd_pm_ops, sprd_suspend, sprd_resume);
1249 
1250 static const struct of_device_id serial_ids[] = {
1251 	{.compatible = "sprd,sc9836-uart",},
1252 	{}
1253 };
1254 MODULE_DEVICE_TABLE(of, serial_ids);
1255 
1256 static struct platform_driver sprd_platform_driver = {
1257 	.probe		= sprd_probe,
1258 	.remove_new	= sprd_remove,
1259 	.driver		= {
1260 		.name	= "sprd_serial",
1261 		.of_match_table = serial_ids,
1262 		.pm	= &sprd_pm_ops,
1263 	},
1264 };
1265 
1266 module_platform_driver(sprd_platform_driver);
1267 
1268 MODULE_LICENSE("GPL v2");
1269 MODULE_DESCRIPTION("Spreadtrum SoC serial driver series");
1270