131 	struct uart_port port;  member
144 static int sprd_start_dma_rx(struct uart_port *port);
145 static int sprd_tx_dma_config(struct uart_port *port);
159 static inline unsigned int serial_in(struct uart_port *port,  in serial_in()  argument
162 	return readl_relaxed(port->membase + offset);  in serial_in()
165 static inline void serial_out(struct uart_port *port, unsigned int offset,  in serial_out()  argument
168 	writel_relaxed(value, port->membase + offset);  in serial_out()
171 static unsigned int sprd_tx_empty(struct uart_port *port)  in sprd_tx_empty()  argument
173 	if (serial_in(port, SPRD_STS1) & SPRD_TX_FIFO_CNT_MASK)  in sprd_tx_empty()
179 static unsigned int sprd_get_mctrl(struct uart_port *port)  in sprd_get_mctrl()  argument
184 static void sprd_set_mctrl(struct uart_port *port, unsigned int mctrl)  in sprd_set_mctrl()  argument
186 	u32 val = serial_in(port, SPRD_CTL1);  in sprd_set_mctrl()
193 	serial_out(port, SPRD_CTL1, val);  in sprd_set_mctrl()
196 static void sprd_stop_rx(struct uart_port *port)  in sprd_stop_rx()  argument
199 		container_of(port, struct sprd_uart_port, port);  in sprd_stop_rx()
205 	iclr = serial_in(port, SPRD_ICLR);  in sprd_stop_rx()
206 	ien = serial_in(port, SPRD_IEN);  in sprd_stop_rx()
211 	serial_out(port, SPRD_IEN, ien);  in sprd_stop_rx()
212 	serial_out(port, SPRD_ICLR, iclr);  in sprd_stop_rx()
215 static void sprd_uart_dma_enable(struct uart_port *port, bool enable)  in sprd_uart_dma_enable()  argument
217 	u32 val = serial_in(port, SPRD_CTL1);  in sprd_uart_dma_enable()
224 	serial_out(port, SPRD_CTL1, val);  in sprd_uart_dma_enable()
227 static void sprd_stop_tx_dma(struct uart_port *port)  in sprd_stop_tx_dma()  argument
230 		container_of(port, struct sprd_uart_port, port);  in sprd_stop_tx_dma()
239 		uart_xmit_advance(port, trans_len);  in sprd_stop_tx_dma()
240 		dma_unmap_single(port->dev, sp->tx_dma.phys_addr,  in sprd_stop_tx_dma()
248 static int sprd_tx_buf_remap(struct uart_port *port)  in sprd_tx_buf_remap()  argument
251 		container_of(port, struct sprd_uart_port, port);  in sprd_tx_buf_remap()
252 	struct tty_port *tport = &port->state->port;  in sprd_tx_buf_remap()
258 	sp->tx_dma.phys_addr = dma_map_single(port->dev, tail,  in sprd_tx_buf_remap()
261 	return dma_mapping_error(port->dev, sp->tx_dma.phys_addr);  in sprd_tx_buf_remap()
266 	struct uart_port *port = (struct uart_port *)data;  in sprd_complete_tx_dma()  local
268 		container_of(port, struct sprd_uart_port, port);  in sprd_complete_tx_dma()
269 	struct tty_port *tport = &port->state->port;  in sprd_complete_tx_dma()
272 	uart_port_lock_irqsave(port, &flags);  in sprd_complete_tx_dma()
273 	dma_unmap_single(port->dev, sp->tx_dma.phys_addr,  in sprd_complete_tx_dma()
276 	uart_xmit_advance(port, sp->tx_dma.trans_len);  in sprd_complete_tx_dma()
279 		uart_write_wakeup(port);  in sprd_complete_tx_dma()
281 	if (kfifo_is_empty(&tport->xmit_fifo) || sprd_tx_buf_remap(port) ||  in sprd_complete_tx_dma()
282 	    sprd_tx_dma_config(port))  in sprd_complete_tx_dma()
285 	uart_port_unlock_irqrestore(port, flags);  in sprd_complete_tx_dma()
288 static int sprd_uart_dma_submit(struct uart_port *port,  in sprd_uart_dma_submit()  argument
307 	dma_des->callback_param = port;  in sprd_uart_dma_submit()
318 static int sprd_tx_dma_config(struct uart_port *port)  in sprd_tx_dma_config()  argument
321 		container_of(port, struct sprd_uart_port, port);  in sprd_tx_dma_config()
326 		.dst_addr = port->mapbase + SPRD_TXD,  in sprd_tx_dma_config()
336 	return sprd_uart_dma_submit(port, &sp->tx_dma, sp->tx_dma.trans_len,  in sprd_tx_dma_config()
340 static void sprd_start_tx_dma(struct uart_port *port)  in sprd_start_tx_dma()  argument
343 		container_of(port, struct sprd_uart_port, port);  in sprd_start_tx_dma()
344 	struct tty_port *tport = &port->state->port;  in sprd_start_tx_dma()
346 	if (port->x_char) {  in sprd_start_tx_dma()
347 		serial_out(port, SPRD_TXD, port->x_char);  in sprd_start_tx_dma()
348 		port->icount.tx++;  in sprd_start_tx_dma()
349 		port->x_char = 0;  in sprd_start_tx_dma()
353 	if (kfifo_is_empty(&tport->xmit_fifo) || uart_tx_stopped(port)) {  in sprd_start_tx_dma()
354 		sprd_stop_tx_dma(port);  in sprd_start_tx_dma()
361 	if (sprd_tx_buf_remap(port) || sprd_tx_dma_config(port))  in sprd_start_tx_dma()
365 static void sprd_rx_full_thld(struct uart_port *port, u32 thld)  in sprd_rx_full_thld()  argument
367 	u32 val = serial_in(port, SPRD_CTL2);  in sprd_rx_full_thld()
371 	serial_out(port, SPRD_CTL2, val);  in sprd_rx_full_thld()
376 	sp->rx_dma.virt = dma_alloc_coherent(sp->port.dev, SPRD_UART_RX_SIZE,  in sprd_rx_alloc_buf()
387 		dma_free_coherent(sp->port.dev, SPRD_UART_RX_SIZE,  in sprd_rx_free_buf()
392 static int sprd_rx_dma_config(struct uart_port *port, u32 burst)  in sprd_rx_dma_config()  argument
395 		container_of(port, struct sprd_uart_port, port);  in sprd_rx_dma_config()
397 		.src_addr = port->mapbase + SPRD_RXD,  in sprd_rx_dma_config()
406 static void sprd_uart_dma_rx(struct uart_port *port)  in sprd_uart_dma_rx()  argument
409 		container_of(port, struct sprd_uart_port, port);  in sprd_uart_dma_rx()
410 	struct tty_port *tty = &port->state->port;  in sprd_uart_dma_rx()
412 	port->icount.rx += sp->rx_dma.trans_len;  in sprd_uart_dma_rx()
417 static void sprd_uart_dma_irq(struct uart_port *port)  in sprd_uart_dma_irq()  argument
420 		container_of(port, struct sprd_uart_port, port);  in sprd_uart_dma_irq()
427 		sprd_stop_rx(port);  in sprd_uart_dma_irq()
441 	sprd_uart_dma_rx(port);  in sprd_uart_dma_irq()
447 	struct uart_port *port = (struct uart_port *)data;  in sprd_complete_rx_dma()  local
449 		container_of(port, struct sprd_uart_port, port);  in sprd_complete_rx_dma()
454 	uart_port_lock_irqsave(port, &flags);  in sprd_complete_rx_dma()
459 		sprd_stop_rx(port);  in sprd_complete_rx_dma()
460 		uart_port_unlock_irqrestore(port, flags);  in sprd_complete_rx_dma()
467 		sprd_uart_dma_rx(port);  in sprd_complete_rx_dma()
471 	if (sprd_start_dma_rx(port))  in sprd_complete_rx_dma()
472 		sprd_stop_rx(port);  in sprd_complete_rx_dma()
474 	uart_port_unlock_irqrestore(port, flags);  in sprd_complete_rx_dma()
477 static int sprd_start_dma_rx(struct uart_port *port)  in sprd_start_dma_rx()  argument
480 		container_of(port, struct sprd_uart_port, port);  in sprd_start_dma_rx()
488 	sprd_rx_full_thld(port, SPRD_RX_FIFO_FULL);  in sprd_start_dma_rx()
489 	ret = sprd_rx_dma_config(port, SPRD_RX_DMA_STEP);  in sprd_start_dma_rx()
493 	return sprd_uart_dma_submit(port, &sp->rx_dma, SPRD_UART_RX_SIZE,  in sprd_start_dma_rx()
497 static void sprd_release_dma(struct uart_port *port)  in sprd_release_dma()  argument
500 		container_of(port, struct sprd_uart_port, port);  in sprd_release_dma()
502 	sprd_uart_dma_enable(port, false);  in sprd_release_dma()
514 static void sprd_request_dma(struct uart_port *port)  in sprd_request_dma()  argument
517 		container_of(port, struct sprd_uart_port, port);  in sprd_request_dma()
522 	sp->tx_dma.chn = dma_request_chan(port->dev, "tx");  in sprd_request_dma()
524 		dev_err(port->dev, "request TX DMA channel failed, ret = %ld\n",  in sprd_request_dma()
529 	sp->rx_dma.chn = dma_request_chan(port->dev, "rx");  in sprd_request_dma()
531 		dev_err(port->dev, "request RX DMA channel failed, ret = %ld\n",  in sprd_request_dma()
537 static void sprd_stop_tx(struct uart_port *port)  in sprd_stop_tx()  argument
539 	struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,  in sprd_stop_tx()
540 						 port);  in sprd_stop_tx()
544 		sprd_stop_tx_dma(port);  in sprd_stop_tx()
548 	iclr = serial_in(port, SPRD_ICLR);  in sprd_stop_tx()
549 	ien = serial_in(port, SPRD_IEN);  in sprd_stop_tx()
554 	serial_out(port, SPRD_IEN, ien);  in sprd_stop_tx()
555 	serial_out(port, SPRD_ICLR, iclr);  in sprd_stop_tx()
558 static void sprd_start_tx(struct uart_port *port)  in sprd_start_tx()  argument
560 	struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,  in sprd_start_tx()
561 						 port);  in sprd_start_tx()
565 		sprd_start_tx_dma(port);  in sprd_start_tx()
569 	ien = serial_in(port, SPRD_IEN);  in sprd_start_tx()
572 		serial_out(port, SPRD_IEN, ien);  in sprd_start_tx()
577 static void sprd_break_ctl(struct uart_port *port, int break_state)  in sprd_break_ctl()  argument
582 static int handle_lsr_errors(struct uart_port *port,  in handle_lsr_errors()  argument
591 		port->icount.brk++;  in handle_lsr_errors()
592 		ret = uart_handle_break(port);  in handle_lsr_errors()
596 		port->icount.parity++;  in handle_lsr_errors()
598 		port->icount.frame++;  in handle_lsr_errors()
600 		port->icount.overrun++;  in handle_lsr_errors()
603 	*lsr &= port->read_status_mask;  in handle_lsr_errors()
614 static inline void sprd_rx(struct uart_port *port)  in sprd_rx()  argument
616 	struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,  in sprd_rx()
617 						 port);  in sprd_rx()
618 	struct tty_port *tty = &port->state->port;  in sprd_rx()
623 		sprd_uart_dma_irq(port);  in sprd_rx()
627 	while ((serial_in(port, SPRD_STS1) & SPRD_RX_FIFO_CNT_MASK) &&  in sprd_rx()
629 		lsr = serial_in(port, SPRD_LSR);  in sprd_rx()
630 		ch = serial_in(port, SPRD_RXD);  in sprd_rx()
632 		port->icount.rx++;  in sprd_rx()
636 			if (handle_lsr_errors(port, &flag, &lsr))  in sprd_rx()
638 		if (uart_handle_sysrq_char(port, ch))  in sprd_rx()
641 		uart_insert_char(port, lsr, SPRD_LSR_OE, ch, flag);  in sprd_rx()
647 static inline void sprd_tx(struct uart_port *port)  in sprd_tx()  argument
651 	uart_port_tx_limited(port, ch, THLD_TX_EMPTY,  in sprd_tx()
653 		serial_out(port, SPRD_TXD, ch),  in sprd_tx()
657 /* this handles the interrupt from one port */
660 	struct uart_port *port = dev_id;  in sprd_handle_irq()  local
663 		container_of(port, struct sprd_uart_port, port);  in sprd_handle_irq()
665 	uart_port_lock(port);  in sprd_handle_irq()
667 	ims = serial_in(port, SPRD_IMSR);  in sprd_handle_irq()
670 		uart_port_unlock(port);  in sprd_handle_irq()
675 		serial_out(port, SPRD_ICLR, sp->pdata->timeout_iclr);  in sprd_handle_irq()
678 		serial_out(port, SPRD_ICLR, SPRD_IMSR_BREAK_DETECT);  in sprd_handle_irq()
682 		sprd_rx(port);  in sprd_handle_irq()
685 		sprd_tx(port);  in sprd_handle_irq()
687 	uart_port_unlock(port);  in sprd_handle_irq()
692 static void sprd_uart_dma_startup(struct uart_port *port,  in sprd_uart_dma_startup()  argument
697 	sprd_request_dma(port);  in sprd_uart_dma_startup()
701 	ret = sprd_start_dma_rx(port);  in sprd_uart_dma_startup()
705 		dev_warn(port->dev, "fail to start RX dma mode\n");  in sprd_uart_dma_startup()
708 	sprd_uart_dma_enable(port, true);  in sprd_uart_dma_startup()
711 static int sprd_startup(struct uart_port *port)  in sprd_startup()  argument
719 	serial_out(port, SPRD_CTL2,  in sprd_startup()
724 	while (timeout-- && serial_in(port, SPRD_STS1) & SPRD_RX_FIFO_CNT_MASK)  in sprd_startup()
725 		serial_in(port, SPRD_RXD);  in sprd_startup()
729 	while (timeout-- && serial_in(port, SPRD_STS1) & SPRD_TX_FIFO_CNT_MASK)  in sprd_startup()
733 	serial_out(port, SPRD_IEN, 0);  in sprd_startup()
734 	serial_out(port, SPRD_ICLR, ~0);  in sprd_startup()
737 	sp = container_of(port, struct sprd_uart_port, port);  in sprd_startup()
738 	snprintf(sp->name, sizeof(sp->name), "sprd_serial%d", port->line);  in sprd_startup()
740 	sprd_uart_dma_startup(port, sp);  in sprd_startup()
742 	ret = devm_request_irq(port->dev, port->irq, sprd_handle_irq,  in sprd_startup()
743 			       IRQF_SHARED, sp->name, port);  in sprd_startup()
745 		dev_err(port->dev, "fail to request serial irq %d, ret=%d\n",  in sprd_startup()
746 			port->irq, ret);  in sprd_startup()
749 	fc = serial_in(port, SPRD_CTL1);  in sprd_startup()
751 	serial_out(port, SPRD_CTL1, fc);  in sprd_startup()
754 	uart_port_lock_irqsave(port, &flags);  in sprd_startup()
755 	ien = serial_in(port, SPRD_IEN);  in sprd_startup()
759 	serial_out(port, SPRD_IEN, ien);  in sprd_startup()
760 	uart_port_unlock_irqrestore(port, flags);  in sprd_startup()
765 static void sprd_shutdown(struct uart_port *port)  in sprd_shutdown()  argument
767 	sprd_release_dma(port);  in sprd_shutdown()
768 	serial_out(port, SPRD_IEN, 0);  in sprd_shutdown()
769 	serial_out(port, SPRD_ICLR, ~0);  in sprd_shutdown()
770 	devm_free_irq(port->dev, port->irq, port);  in sprd_shutdown()
773 static void sprd_set_termios(struct uart_port *port, struct ktermios *termios,  in sprd_set_termios()  argument
781 	baud = uart_get_baud_rate(port, termios, old, 0, SPRD_BAUD_IO_LIMIT);  in sprd_set_termios()
783 	quot = port->uartclk / baud;  in sprd_set_termios()
820 	uart_port_lock_irqsave(port, &flags);  in sprd_set_termios()
822 	/* update the per-port timeout */  in sprd_set_termios()
823 	uart_update_timeout(port, termios->c_cflag, baud);  in sprd_set_termios()
825 	port->read_status_mask = SPRD_LSR_OE;  in sprd_set_termios()
827 		port->read_status_mask |= SPRD_LSR_FE | SPRD_LSR_PE;  in sprd_set_termios()
829 		port->read_status_mask |= SPRD_LSR_BI;  in sprd_set_termios()
832 	port->ignore_status_mask = 0;  in sprd_set_termios()
834 		port->ignore_status_mask |= SPRD_LSR_PE | SPRD_LSR_FE;  in sprd_set_termios()
836 		port->ignore_status_mask |= SPRD_LSR_BI;  in sprd_set_termios()
842 			port->ignore_status_mask |= SPRD_LSR_OE;  in sprd_set_termios()
846 	fc = serial_in(port, SPRD_CTL1);  in sprd_set_termios()
855 	serial_out(port, SPRD_CLKD0, quot & SPRD_CLKD0_MASK);  in sprd_set_termios()
858 	serial_out(port, SPRD_CLKD1,  in sprd_set_termios()
860 	serial_out(port, SPRD_LCR, lcr);  in sprd_set_termios()
862 	serial_out(port, SPRD_CTL1, fc);  in sprd_set_termios()
864 	uart_port_unlock_irqrestore(port, flags);  in sprd_set_termios()
871 static const char *sprd_type(struct uart_port *port)  in sprd_type()  argument
876 static void sprd_release_port(struct uart_port *port)  in sprd_release_port()  argument
881 static int sprd_request_port(struct uart_port *port)  in sprd_request_port()  argument
886 static void sprd_config_port(struct uart_port *port, int flags)  in sprd_config_port()  argument
889 		port->type = PORT_SPRD;  in sprd_config_port()
892 static int sprd_verify_port(struct uart_port *port, struct serial_struct *ser)  in sprd_verify_port()  argument
896 	if (port->irq != ser->irq)  in sprd_verify_port()
898 	if (port->iotype != ser->io_type)  in sprd_verify_port()
903 static void sprd_pm(struct uart_port *port, unsigned int state,  in sprd_pm()  argument
907 		container_of(port, struct sprd_uart_port, port);  in sprd_pm()
920 static int sprd_poll_init(struct uart_port *port)  in sprd_poll_init()  argument
922 	if (port->state->pm_state != UART_PM_STATE_ON) {  in sprd_poll_init()
923 		sprd_pm(port, UART_PM_STATE_ON, 0);  in sprd_poll_init()
924 		port->state->pm_state = UART_PM_STATE_ON;  in sprd_poll_init()
930 static int sprd_poll_get_char(struct uart_port *port)  in sprd_poll_get_char()  argument
932 	while (!(serial_in(port, SPRD_STS1) & SPRD_RX_FIFO_CNT_MASK))  in sprd_poll_get_char()
935 	return serial_in(port, SPRD_RXD);  in sprd_poll_get_char()
938 static void sprd_poll_put_char(struct uart_port *port, unsigned char ch)  in sprd_poll_put_char()  argument
940 	while (serial_in(port, SPRD_STS1) & SPRD_TX_FIFO_CNT_MASK)  in sprd_poll_put_char()
943 	serial_out(port, SPRD_TXD, ch);  in sprd_poll_put_char()
972 static void wait_for_xmitr(struct uart_port *port)  in wait_for_xmitr()  argument
978 		status = serial_in(port, SPRD_STS1);  in wait_for_xmitr()
985 static void sprd_console_putchar(struct uart_port *port, unsigned char ch)  in sprd_console_putchar()  argument
987 	wait_for_xmitr(port);  in sprd_console_putchar()
988 	serial_out(port, SPRD_TXD, ch);  in sprd_console_putchar()
994 	struct uart_port *port = &sprd_port[co->index]->port;  in sprd_console_write()  local
998 	if (port->sysrq)  in sprd_console_write()
1001 		locked = uart_port_trylock_irqsave(port, &flags);  in sprd_console_write()
1003 		uart_port_lock_irqsave(port, &flags);  in sprd_console_write()
1005 	uart_console_write(port, s, count, sprd_console_putchar);  in sprd_console_write()
1008 	wait_for_xmitr(port);  in sprd_console_write()
1011 		uart_port_unlock_irqrestore(port, flags);  in sprd_console_write()
1026 	if (!sprd_uart_port || !sprd_uart_port->port.membase) {  in sprd_console_setup()
1027 		pr_info("serial port %d not yet initialized\n", co->index);  in sprd_console_setup()
1034 	return uart_set_options(&sprd_uart_port->port, co, baud,  in sprd_console_setup()
1059 static void sprd_putc(struct uart_port *port, unsigned char c)  in sprd_putc()  argument
1064 	       !(readl(port->membase + SPRD_LSR) & SPRD_LSR_TX_OVER))  in sprd_putc()
1067 	writeb(c, port->membase + SPRD_TXD);  in sprd_putc()
1074 	uart_console_write(&dev->port, s, n, sprd_putc);  in sprd_early_write()
1080 	if (!device->port.membase)  in sprd_early_console_setup()
1108 		uart_remove_one_port(&sprd_uart_driver, &sup->port);  in sprd_remove()
1109 		sprd_port[sup->port.line] = NULL;  in sprd_remove()
1132 	struct sprd_uart_port *u = container_of(uport, struct sprd_uart_port, port);  in sprd_clk_init()
1190 	up = &sport->port;  in sprd_probe()
1263 	uart_suspend_port(&sprd_uart_driver, &sup->port);  in sprd_suspend()
1272 	uart_resume_port(&sprd_uart_driver, &sup->port);  in sprd_resume()