22 void usbhs_pkt_init(struct usbhs_pkt *pkt)  in usbhs_pkt_init()  argument
24 	INIT_LIST_HEAD(&pkt->node);  in usbhs_pkt_init()
30 static int usbhsf_null_handle(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_null_handle()  argument
32 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_null_handle()
45 void usbhs_pkt_push(struct usbhs_pipe *pipe, struct usbhs_pkt *pkt,  in usbhs_pkt_push()  argument
47 				 struct usbhs_pkt *pkt),  in usbhs_pkt_push()  argument
67 	list_move_tail(&pkt->node, &pipe->list);  in usbhs_pkt_push()
74 	pkt->pipe	= pipe;  in usbhs_pkt_push()
75 	pkt->buf	= buf;  in usbhs_pkt_push()
76 	pkt->handler	= pipe->handler;  in usbhs_pkt_push()
77 	pkt->length	= len;  in usbhs_pkt_push()
78 	pkt->zero	= zero;  in usbhs_pkt_push()
79 	pkt->actual	= 0;  in usbhs_pkt_push()
80 	pkt->done	= done;  in usbhs_pkt_push()
81 	pkt->sequence	= sequence;  in usbhs_pkt_push()
87 static void __usbhsf_pkt_del(struct usbhs_pkt *pkt)  in __usbhsf_pkt_del()  argument
89 	list_del_init(&pkt->node);  in __usbhsf_pkt_del()
100 					    struct usbhs_pkt *pkt);
103 static int __usbhsf_dma_map_ctrl(struct usbhs_pkt *pkt, int map);
106 struct usbhs_pkt *usbhs_pkt_pop(struct usbhs_pipe *pipe, struct usbhs_pkt *pkt)  in usbhs_pkt_pop()  argument
117 	if (!pkt)  in usbhs_pkt_pop()
118 		pkt = __usbhsf_pkt_get(pipe);  in usbhs_pkt_pop()
120 	if (pkt) {  in usbhs_pkt_pop()
124 			chan = usbhsf_dma_chan_get(fifo, pkt);  in usbhs_pkt_pop()
127 			usbhsf_dma_unmap(pkt);  in usbhs_pkt_pop()
138 		__usbhsf_pkt_del(pkt);  in usbhs_pkt_pop()
147 	return pkt;  in usbhs_pkt_pop()
159 	struct usbhs_pkt *pkt;  in usbhsf_pkt_handler()  local
161 	int (*func)(struct usbhs_pkt *pkt, int *is_done);  in usbhsf_pkt_handler()
169 	pkt = __usbhsf_pkt_get(pipe);  in usbhsf_pkt_handler()
170 	if (!pkt) {  in usbhsf_pkt_handler()
177 		func = pkt->handler->prepare;  in usbhsf_pkt_handler()
180 		func = pkt->handler->try_run;  in usbhsf_pkt_handler()
183 		func = pkt->handler->dma_done;  in usbhsf_pkt_handler()
191 		ret = func(pkt, &is_done);  in usbhsf_pkt_handler()
194 		__usbhsf_pkt_del(pkt);  in usbhsf_pkt_handler()
201 		pkt->done(priv, pkt);  in usbhsf_pkt_handler()
357 static int usbhs_dcp_dir_switch_to_write(struct usbhs_pkt *pkt, int *is_done)  in usbhs_dcp_dir_switch_to_write()  argument
359 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dcp_dir_switch_to_write()
386 static int usbhs_dcp_dir_switch_to_read(struct usbhs_pkt *pkt, int *is_done)  in usbhs_dcp_dir_switch_to_read()  argument
388 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dcp_dir_switch_to_read()
414 static int usbhs_dcp_dir_switch_done(struct usbhs_pkt *pkt, int *is_done)  in usbhs_dcp_dir_switch_done()  argument
416 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dcp_dir_switch_done()
418 	if (pkt->handler == &usbhs_dcp_status_stage_in_handler)  in usbhs_dcp_dir_switch_done()
423 	pkt->actual = pkt->length;  in usbhs_dcp_dir_switch_done()
442 static int usbhsf_dcp_data_stage_try_push(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dcp_data_stage_try_push()  argument
444 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dcp_data_stage_try_push()
451 	pkt->handler = &usbhs_fifo_pio_push_handler;  in usbhsf_dcp_data_stage_try_push()
453 	return pkt->handler->prepare(pkt, is_done);  in usbhsf_dcp_data_stage_try_push()
463 static int usbhsf_dcp_data_stage_prepare_pop(struct usbhs_pkt *pkt,  in usbhsf_dcp_data_stage_prepare_pop()  argument
466 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dcp_data_stage_prepare_pop()
490 	pkt->handler = &usbhs_fifo_pio_pop_handler;  in usbhsf_dcp_data_stage_prepare_pop()
492 	return pkt->handler->prepare(pkt, is_done);  in usbhsf_dcp_data_stage_prepare_pop()
502 static int usbhsf_pio_try_push(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_pio_try_push()  argument
504 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_pio_try_push()
515 	usbhs_pipe_data_sequence(pipe, pkt->sequence);  in usbhsf_pio_try_push()
516 	pkt->sequence = -1; /* -1 sequence will be ignored */  in usbhsf_pio_try_push()
518 	usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->length);  in usbhsf_pio_try_push()
535 	buf		= pkt->buf    + pkt->actual;  in usbhsf_pio_try_push()
536 	len		= pkt->length - pkt->actual;  in usbhsf_pio_try_push()
564 	pkt->actual += total_len;  in usbhsf_pio_try_push()
566 	if (pkt->actual < pkt->length)  in usbhsf_pio_try_push()
571 		*is_done = !pkt->zero;	/* send zero packet ? */  in usbhsf_pio_try_push()
585 		pkt->length, pkt->actual, *is_done, pkt->zero);  in usbhsf_pio_try_push()
604 static int usbhsf_pio_prepare_push(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_pio_prepare_push()  argument
606 	if (usbhs_pipe_is_running(pkt->pipe))  in usbhsf_pio_prepare_push()
609 	return usbhsf_pio_try_push(pkt, is_done);  in usbhsf_pio_prepare_push()
620 static int usbhsf_prepare_pop(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_prepare_pop()  argument
622 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_prepare_pop()
635 	usbhs_pipe_data_sequence(pipe, pkt->sequence);  in usbhsf_prepare_pop()
636 	pkt->sequence = -1; /* -1 sequence will be ignored */  in usbhsf_prepare_pop()
641 	usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->length);  in usbhsf_prepare_pop()
649 static int usbhsf_pio_try_pop(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_pio_try_pop()  argument
651 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_pio_try_pop()
673 	buf		= pkt->buf    + pkt->actual;  in usbhsf_pio_try_pop()
674 	len		= pkt->length - pkt->actual;  in usbhsf_pio_try_pop()
683 	pkt->actual += total_len;  in usbhsf_pio_try_pop()
685 	if ((pkt->actual == pkt->length) ||	/* receive all data */  in usbhsf_pio_try_pop()
707 		pkt->zero = 1;  in usbhsf_pio_try_pop()
734 		pkt->length, pkt->actual, *is_done, pkt->zero);  in usbhsf_pio_try_pop()
750 static int usbhsf_ctrl_stage_end(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_ctrl_stage_end()  argument
752 	usbhs_dcp_control_transfer_done(pkt->pipe);  in usbhsf_ctrl_stage_end()
768 					    struct usbhs_pkt *pkt)  in usbhsf_dma_chan_get()  argument
770 	if (&usbhs_fifo_dma_push_handler == pkt->handler)  in usbhsf_dma_chan_get()
773 	if (&usbhs_fifo_dma_pop_handler == pkt->handler)  in usbhsf_dma_chan_get()
780 					      struct usbhs_pkt *pkt)  in usbhsf_get_dma_fifo()  argument
786 		if (usbhsf_dma_chan_get(fifo, pkt) &&  in usbhsf_get_dma_fifo()
805 static int __usbhsf_dma_map_ctrl(struct usbhs_pkt *pkt, int map)  in __usbhsf_dma_map_ctrl()  argument
807 	struct usbhs_pipe *pipe = pkt->pipe;  in __usbhsf_dma_map_ctrl()
811 	struct dma_chan *chan = usbhsf_dma_chan_get(fifo, pkt);  in __usbhsf_dma_map_ctrl()
813 	return info->dma_map_ctrl(chan->device->dev, pkt, map);  in __usbhsf_dma_map_ctrl()
818 static void usbhsf_dma_xfer_preparing(struct usbhs_pkt *pkt)  in usbhsf_dma_xfer_preparing()  argument
820 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_xfer_preparing()
833 	chan = usbhsf_dma_chan_get(fifo, pkt);  in usbhsf_dma_xfer_preparing()
836 	desc = dmaengine_prep_slave_single(chan, pkt->dma + pkt->actual,  in usbhsf_dma_xfer_preparing()
837 					pkt->trans, dir,  in usbhsf_dma_xfer_preparing()
843 	desc->callback_param	= pkt;  in usbhsf_dma_xfer_preparing()
852 		fifo->name, usbhs_pipe_number(pipe), pkt->length, pkt->zero);  in usbhsf_dma_xfer_preparing()
855 	usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->trans);  in usbhsf_dma_xfer_preparing()
863 	struct usbhs_pkt *pkt = container_of(work, struct usbhs_pkt, work);  in xfer_work()  local
864 	struct usbhs_pipe *pipe = pkt->pipe;  in xfer_work()
869 	usbhsf_dma_xfer_preparing(pkt);  in xfer_work()
876 static int usbhsf_dma_prepare_push(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_prepare_push()  argument
878 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_prepare_push()
881 	int len = pkt->length - pkt->actual;  in usbhsf_dma_prepare_push()
900 	if ((uintptr_t)(pkt->buf + pkt->actual) & align_mask)  in usbhsf_dma_prepare_push()
908 	fifo = usbhsf_get_dma_fifo(priv, pkt);  in usbhsf_dma_prepare_push()
916 	if (usbhsf_dma_map(pkt) < 0)  in usbhsf_dma_prepare_push()
919 	pkt->trans = len;  in usbhsf_dma_prepare_push()
924 		usbhsf_dma_xfer_preparing(pkt);  in usbhsf_dma_prepare_push()
926 		INIT_WORK(&pkt->work, xfer_work);  in usbhsf_dma_prepare_push()
927 		schedule_work(&pkt->work);  in usbhsf_dma_prepare_push()
938 	pkt->handler = &usbhs_fifo_pio_push_handler;  in usbhsf_dma_prepare_push()
940 	return pkt->handler->prepare(pkt, is_done);  in usbhsf_dma_prepare_push()
943 static int usbhsf_dma_push_done(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_push_done()  argument
945 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_push_done()
946 	int is_short = pkt->trans % usbhs_pipe_get_maxpacket(pipe);  in usbhsf_dma_push_done()
948 	pkt->actual += pkt->trans;  in usbhsf_dma_push_done()
950 	if (pkt->actual < pkt->length)  in usbhsf_dma_push_done()
955 		*is_done = !pkt->zero;	/* send zero packet? */  in usbhsf_dma_push_done()
960 	usbhsf_dma_unmap(pkt);  in usbhsf_dma_push_done()
965 		pkt->handler = &usbhs_fifo_pio_push_handler;  in usbhsf_dma_push_done()
966 		return pkt->handler->try_run(pkt, is_done);  in usbhsf_dma_push_done()
981 static int usbhsf_dma_prepare_pop_with_rx_irq(struct usbhs_pkt *pkt,  in usbhsf_dma_prepare_pop_with_rx_irq()  argument
984 	return usbhsf_prepare_pop(pkt, is_done);  in usbhsf_dma_prepare_pop_with_rx_irq()
987 static int usbhsf_dma_prepare_pop_with_usb_dmac(struct usbhs_pkt *pkt,  in usbhsf_dma_prepare_pop_with_usb_dmac()  argument
990 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_prepare_pop_with_usb_dmac()
999 	if ((pkt->length < usbhs_get_dparam(priv, pio_dma_border)) ||  in usbhsf_dma_prepare_pop_with_usb_dmac()
1003 	fifo = usbhsf_get_dma_fifo(priv, pkt);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1007 	if ((uintptr_t)pkt->buf & (USBHS_USB_DMAC_XFER_SIZE - 1))  in usbhsf_dma_prepare_pop_with_usb_dmac()
1020 	if (usbhsf_dma_map(pkt) < 0)  in usbhsf_dma_prepare_pop_with_usb_dmac()
1032 	pkt->trans = pkt->length;  in usbhsf_dma_prepare_pop_with_usb_dmac()
1034 	usbhsf_dma_xfer_preparing(pkt);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1045 	pkt->handler = &usbhs_fifo_pio_pop_handler;  in usbhsf_dma_prepare_pop_with_usb_dmac()
1048 	return pkt->handler->prepare(pkt, is_done);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1051 static int usbhsf_dma_prepare_pop(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_prepare_pop()  argument
1053 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_dma_prepare_pop()
1056 		return usbhsf_dma_prepare_pop_with_usb_dmac(pkt, is_done);  in usbhsf_dma_prepare_pop()
1058 		return usbhsf_dma_prepare_pop_with_rx_irq(pkt, is_done);  in usbhsf_dma_prepare_pop()
1061 static int usbhsf_dma_try_pop_with_rx_irq(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_try_pop_with_rx_irq()  argument
1063 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_try_pop_with_rx_irq()
1075 	fifo = usbhsf_get_dma_fifo(priv, pkt);  in usbhsf_dma_try_pop_with_rx_irq()
1079 	if ((uintptr_t)(pkt->buf + pkt->actual) & 0x7) /* 8byte alignment */  in usbhsf_dma_try_pop_with_rx_irq()
1088 	len = min(pkt->length - pkt->actual, len);  in usbhsf_dma_try_pop_with_rx_irq()
1099 	if (usbhsf_dma_map(pkt) < 0)  in usbhsf_dma_try_pop_with_rx_irq()
1111 	pkt->trans = len;  in usbhsf_dma_try_pop_with_rx_irq()
1113 	INIT_WORK(&pkt->work, xfer_work);  in usbhsf_dma_try_pop_with_rx_irq()
1114 	schedule_work(&pkt->work);  in usbhsf_dma_try_pop_with_rx_irq()
1125 	pkt->handler = &usbhs_fifo_pio_pop_handler;  in usbhsf_dma_try_pop_with_rx_irq()
1127 	return pkt->handler->try_run(pkt, is_done);  in usbhsf_dma_try_pop_with_rx_irq()
1130 static int usbhsf_dma_try_pop(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_try_pop()  argument
1132 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_dma_try_pop()
1136 	return usbhsf_dma_try_pop_with_rx_irq(pkt, is_done);  in usbhsf_dma_try_pop()
1139 static int usbhsf_dma_pop_done_with_rx_irq(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_pop_done_with_rx_irq()  argument
1141 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_pop_done_with_rx_irq()
1145 	usbhsf_dma_unmap(pkt);  in usbhsf_dma_pop_done_with_rx_irq()
1148 	pkt->actual += pkt->trans;  in usbhsf_dma_pop_done_with_rx_irq()
1150 	if ((pkt->actual == pkt->length) ||	/* receive all data */  in usbhsf_dma_pop_done_with_rx_irq()
1151 	    (pkt->trans < maxp)) {		/* short packet */  in usbhsf_dma_pop_done_with_rx_irq()
1157 		usbhsf_prepare_pop(pkt, is_done);  in usbhsf_dma_pop_done_with_rx_irq()
1163 static size_t usbhs_dma_calc_received_size(struct usbhs_pkt *pkt,  in usbhs_dma_calc_received_size()  argument
1166 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dma_calc_received_size()
1170 	received_size = pkt->length - pkt->dma_result->residue;  in usbhs_dma_calc_received_size()
1181 static int usbhsf_dma_pop_done_with_usb_dmac(struct usbhs_pkt *pkt,  in usbhsf_dma_pop_done_with_usb_dmac()  argument
1184 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_pop_done_with_usb_dmac()
1187 	struct dma_chan *chan = usbhsf_dma_chan_get(fifo, pkt);  in usbhsf_dma_pop_done_with_usb_dmac()
1199 	pkt->actual = usbhs_dma_calc_received_size(pkt, chan, rcv_len);  in usbhsf_dma_pop_done_with_usb_dmac()
1203 	usbhsf_dma_unmap(pkt);  in usbhsf_dma_pop_done_with_usb_dmac()
1212 static int usbhsf_dma_pop_done(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_pop_done()  argument
1214 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_dma_pop_done()
1217 		return usbhsf_dma_pop_done_with_usb_dmac(pkt, is_done);  in usbhsf_dma_pop_done()
1219 		return usbhsf_dma_pop_done_with_rx_irq(pkt, is_done);  in usbhsf_dma_pop_done()
1379 	struct usbhs_pkt *pkt = arg;  in usbhsf_dma_complete()  local
1380 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_complete()
1385 	pkt->dma_result = result;  in usbhsf_dma_complete()