Lines Matching +full:generic +full:- +full:xhci
1 // SPDX-License-Identifier: GPL-2.0
9 * Code based on Linux XHCI driver.
28 * until you reach a non-link TRB.
59 #include <linux/dma-mapping.h>
64 #include "cdnsp-trace.h"
65 #include "cdnsp-gadget.h"
74 unsigned long segment_offset = trb - seg->trbs; in cdnsp_trb_virt_to_dma()
76 if (trb < seg->trbs || segment_offset >= TRBS_PER_SEGMENT) in cdnsp_trb_virt_to_dma()
79 return seg->dma + (segment_offset * sizeof(*trb)); in cdnsp_trb_virt_to_dma()
84 return TRB_TYPE_NOOP_LE32(trb->generic.field[3]); in cdnsp_trb_is_noop()
89 return TRB_TYPE_LINK_LE32(trb->link.control); in cdnsp_trb_is_link()
94 return trb == &seg->trbs[TRBS_PER_SEGMENT - 1]; in cdnsp_last_trb_on_seg()
101 return cdnsp_last_trb_on_seg(seg, trb) && (seg->next == ring->first_seg); in cdnsp_last_trb_on_ring()
106 return le32_to_cpu(trb->link.control) & LINK_TOGGLE; in cdnsp_link_trb_toggles_cycle()
113 trb->link.control &= cpu_to_le32(~TRB_CHAIN); in cdnsp_trb_to_noop()
115 trb->generic.field[0] = 0; in cdnsp_trb_to_noop()
116 trb->generic.field[1] = 0; in cdnsp_trb_to_noop()
117 trb->generic.field[2] = 0; in cdnsp_trb_to_noop()
119 trb->generic.field[3] &= cpu_to_le32(TRB_CYCLE); in cdnsp_trb_to_noop()
120 trb->generic.field[3] |= cpu_to_le32(TRB_TYPE(noop_type)); in cdnsp_trb_to_noop()
135 *seg = (*seg)->next; in cdnsp_next_trb()
136 *trb = ((*seg)->trbs); in cdnsp_next_trb()
149 if (ring->type == TYPE_EVENT) { in cdnsp_inc_deq()
150 if (!cdnsp_last_trb_on_seg(ring->deq_seg, ring->dequeue)) { in cdnsp_inc_deq()
151 ring->dequeue++; in cdnsp_inc_deq()
155 if (cdnsp_last_trb_on_ring(ring, ring->deq_seg, ring->dequeue)) in cdnsp_inc_deq()
156 ring->cycle_state ^= 1; in cdnsp_inc_deq()
158 ring->deq_seg = ring->deq_seg->next; in cdnsp_inc_deq()
159 ring->dequeue = ring->deq_seg->trbs; in cdnsp_inc_deq()
164 if (!cdnsp_trb_is_link(ring->dequeue)) { in cdnsp_inc_deq()
165 ring->dequeue++; in cdnsp_inc_deq()
166 ring->num_trbs_free++; in cdnsp_inc_deq()
168 while (cdnsp_trb_is_link(ring->dequeue)) { in cdnsp_inc_deq()
169 ring->deq_seg = ring->deq_seg->next; in cdnsp_inc_deq()
170 ring->dequeue = ring->deq_seg->trbs; in cdnsp_inc_deq()
194 chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN; in cdnsp_inc_enq()
197 if (!cdnsp_trb_is_link(ring->enqueue)) in cdnsp_inc_enq()
198 ring->num_trbs_free--; in cdnsp_inc_enq()
199 next = ++(ring->enqueue); in cdnsp_inc_enq()
213 next->link.control &= cpu_to_le32(~TRB_CHAIN); in cdnsp_inc_enq()
214 next->link.control |= cpu_to_le32(chain); in cdnsp_inc_enq()
218 next->link.control ^= cpu_to_le32(TRB_CYCLE); in cdnsp_inc_enq()
222 ring->cycle_state ^= 1; in cdnsp_inc_enq()
224 ring->enq_seg = ring->enq_seg->next; in cdnsp_inc_enq()
225 ring->enqueue = ring->enq_seg->trbs; in cdnsp_inc_enq()
226 next = ring->enqueue; in cdnsp_inc_enq()
242 if (ring->num_trbs_free < num_trbs) in cdnsp_room_on_ring()
245 if (ring->type != TYPE_COMMAND && ring->type != TYPE_EVENT) { in cdnsp_room_on_ring()
246 num_trbs_in_deq_seg = ring->dequeue - ring->deq_seg->trbs; in cdnsp_room_on_ring()
248 if (ring->num_trbs_free < num_trbs + num_trbs_in_deq_seg) in cdnsp_room_on_ring()
262 if (pdev->active_port == &pdev->usb2_port && pdev->gadget.lpm_capable) in cdnsp_force_l0_go()
263 cdnsp_set_link_state(pdev, &pdev->active_port->regs->portsc, XDEV_U0); in cdnsp_force_l0_go()
269 writel(DB_VALUE_CMD, &pdev->dba->cmd_db); in cdnsp_ring_cmd_db()
280 __le32 __iomem *reg_addr = &pdev->dba->ep_db; in cdnsp_ring_ep_doorbell()
281 unsigned int ep_state = pep->ep_state; in cdnsp_ring_ep_doorbell()
292 if (pep->ep_state & EP_HAS_STREAMS) { in cdnsp_ring_ep_doorbell()
293 if (pep->stream_info.drbls_count >= 2) in cdnsp_ring_ep_doorbell()
296 pep->stream_info.drbls_count++; in cdnsp_ring_ep_doorbell()
299 pep->ep_state &= ~EP_STOPPED; in cdnsp_ring_ep_doorbell()
301 if (pep->idx == 0 && pdev->ep0_stage == CDNSP_DATA_STAGE && in cdnsp_ring_ep_doorbell()
302 !pdev->ep0_expect_in) in cdnsp_ring_ep_doorbell()
303 db_value = DB_VALUE_EP0_OUT(pep->idx, stream_id); in cdnsp_ring_ep_doorbell()
305 db_value = DB_VALUE(pep->idx, stream_id); in cdnsp_ring_ep_doorbell()
326 if (!(pep->ep_state & EP_HAS_STREAMS)) in cdnsp_get_transfer_ring()
327 return pep->ring; in cdnsp_get_transfer_ring()
329 if (stream_id == 0 || stream_id >= pep->stream_info.num_streams) { in cdnsp_get_transfer_ring()
330 dev_err(pdev->dev, "ERR: %s ring doesn't exist for SID: %d.\n", in cdnsp_get_transfer_ring()
331 pep->name, stream_id); in cdnsp_get_transfer_ring()
335 return pep->stream_info.stream_rings[stream_id]; in cdnsp_get_transfer_ring()
342 return cdnsp_get_transfer_ring(pdev, preq->pep, in cdnsp_request_to_transfer_ring()
343 preq->request.stream_id); in cdnsp_request_to_transfer_ring()
354 if (pep->ep_state & EP_DIS_IN_RROGRESS) in cdnsp_ring_doorbell_for_active_rings()
358 if (!(pep->ep_state & EP_HAS_STREAMS) && pep->number) { in cdnsp_ring_doorbell_for_active_rings()
359 if (pep->ring && !list_empty(&pep->ring->td_list)) in cdnsp_ring_doorbell_for_active_rings()
364 stream_info = &pep->stream_info; in cdnsp_ring_doorbell_for_active_rings()
366 for (stream_id = 1; stream_id < stream_info->num_streams; stream_id++) { in cdnsp_ring_doorbell_for_active_rings()
370 if (stream_info->drbls_count >= 2) in cdnsp_ring_doorbell_for_active_rings()
377 if (!ep_ring->stream_active || ep_ring->stream_rejected) in cdnsp_ring_doorbell_for_active_rings()
380 list_for_each_entry_safe(td, td_temp, &ep_ring->td_list, in cdnsp_ring_doorbell_for_active_rings()
382 if (td->drbl) in cdnsp_ring_doorbell_for_active_rings()
387 td->drbl = 1; in cdnsp_ring_doorbell_for_active_rings()
405 pep = &pdev->eps[ep_index]; in cdnsp_get_hw_deq()
407 if (pep->ep_state & EP_HAS_STREAMS) { in cdnsp_get_hw_deq()
408 st_ctx = &pep->stream_info.stream_ctx_array[stream_id]; in cdnsp_get_hw_deq()
409 return le64_to_cpu(st_ctx->stream_ring); in cdnsp_get_hw_deq()
412 return le64_to_cpu(pep->out_ctx->deq); in cdnsp_get_hw_deq()
422 * - First we update our new ring state to be the same as when the
424 * - Then we traverse the ring to find the segment that contains
427 * - Finally we move the dequeue state one TRB further, toggling the cycle bit
451 hw_dequeue = cdnsp_get_hw_deq(pdev, pep->idx, stream_id); in cdnsp_find_new_dequeue_state()
452 new_seg = ep_ring->deq_seg; in cdnsp_find_new_dequeue_state()
453 new_deq = ep_ring->dequeue; in cdnsp_find_new_dequeue_state()
454 state->new_cycle_state = hw_dequeue & 0x1; in cdnsp_find_new_dequeue_state()
455 state->stream_id = stream_id; in cdnsp_find_new_dequeue_state()
472 if (new_deq == cur_td->last_trb) in cdnsp_find_new_dequeue_state()
477 state->new_cycle_state ^= 0x1; in cdnsp_find_new_dequeue_state()
482 if (new_deq == pep->ring->dequeue) { in cdnsp_find_new_dequeue_state()
483 dev_err(pdev->dev, in cdnsp_find_new_dequeue_state()
485 state->new_deq_seg = NULL; in cdnsp_find_new_dequeue_state()
486 state->new_deq_ptr = NULL; in cdnsp_find_new_dequeue_state()
492 state->new_deq_seg = new_seg; in cdnsp_find_new_dequeue_state()
493 state->new_deq_ptr = new_deq; in cdnsp_find_new_dequeue_state()
508 struct cdnsp_segment *seg = td->start_seg; in cdnsp_td_to_noop()
509 union cdnsp_trb *trb = td->first_trb; in cdnsp_td_to_noop()
515 if (flip_cycle && trb != td->first_trb && trb != td->last_trb) in cdnsp_td_to_noop()
516 trb->generic.field[3] ^= cpu_to_le32(TRB_CYCLE); in cdnsp_td_to_noop()
518 if (trb == td->last_trb) in cdnsp_td_to_noop()
550 temp_trb = &cur_seg->trbs[TRBS_PER_SEGMENT - 1]; in cdnsp_trb_in_td()
557 end_trb_dma, cur_seg->dma, in cdnsp_trb_in_td()
577 (suspect_dma >= cur_seg->dma && in cdnsp_trb_in_td()
590 cur_seg = cur_seg->next; in cdnsp_trb_in_td()
591 start_dma = cdnsp_trb_virt_to_dma(cur_seg, &cur_seg->trbs[0]); in cdnsp_trb_in_td()
601 struct cdnsp_segment *seg = td->bounce_seg; in cdnsp_unmap_td_bounce_buffer()
608 preq = td->preq; in cdnsp_unmap_td_bounce_buffer()
610 trace_cdnsp_bounce_unmap(td->preq, seg->bounce_len, seg->bounce_offs, in cdnsp_unmap_td_bounce_buffer()
611 seg->bounce_dma, 0); in cdnsp_unmap_td_bounce_buffer()
613 if (!preq->direction) { in cdnsp_unmap_td_bounce_buffer()
614 dma_unmap_single(pdev->dev, seg->bounce_dma, in cdnsp_unmap_td_bounce_buffer()
615 ring->bounce_buf_len, DMA_TO_DEVICE); in cdnsp_unmap_td_bounce_buffer()
619 dma_unmap_single(pdev->dev, seg->bounce_dma, ring->bounce_buf_len, in cdnsp_unmap_td_bounce_buffer()
623 len = sg_pcopy_from_buffer(preq->request.sg, preq->request.num_sgs, in cdnsp_unmap_td_bounce_buffer()
624 seg->bounce_buf, seg->bounce_len, in cdnsp_unmap_td_bounce_buffer()
625 seg->bounce_offs); in cdnsp_unmap_td_bounce_buffer()
626 if (len != seg->bounce_len) in cdnsp_unmap_td_bounce_buffer()
627 dev_warn(pdev->dev, "WARN Wrong bounce buffer read length: %zu != %d\n", in cdnsp_unmap_td_bounce_buffer()
628 len, seg->bounce_len); in cdnsp_unmap_td_bounce_buffer()
630 seg->bounce_len = 0; in cdnsp_unmap_td_bounce_buffer()
631 seg->bounce_offs = 0; in cdnsp_unmap_td_bounce_buffer()
641 if (!deq_state->new_deq_ptr || !deq_state->new_deq_seg) { in cdnsp_cmd_set_deq()
650 trace_cdnsp_handle_cmd_set_deq(cdnsp_get_slot_ctx(&pdev->out_ctx)); in cdnsp_cmd_set_deq()
651 trace_cdnsp_handle_cmd_set_deq_ep(pep->out_ctx); in cdnsp_cmd_set_deq()
657 ep_ring = cdnsp_get_transfer_ring(pdev, pep, deq_state->stream_id); in cdnsp_cmd_set_deq()
659 if (cdnsp_trb_is_link(ep_ring->dequeue)) { in cdnsp_cmd_set_deq()
660 ep_ring->deq_seg = ep_ring->deq_seg->next; in cdnsp_cmd_set_deq()
661 ep_ring->dequeue = ep_ring->deq_seg->trbs; in cdnsp_cmd_set_deq()
664 while (ep_ring->dequeue != deq_state->new_deq_ptr) { in cdnsp_cmd_set_deq()
665 ep_ring->num_trbs_free++; in cdnsp_cmd_set_deq()
666 ep_ring->dequeue++; in cdnsp_cmd_set_deq()
668 if (cdnsp_trb_is_link(ep_ring->dequeue)) { in cdnsp_cmd_set_deq()
669 if (ep_ring->dequeue == deq_state->new_deq_ptr) in cdnsp_cmd_set_deq()
672 ep_ring->deq_seg = ep_ring->deq_seg->next; in cdnsp_cmd_set_deq()
673 ep_ring->dequeue = ep_ring->deq_seg->trbs; in cdnsp_cmd_set_deq()
682 return -ESHUTDOWN; in cdnsp_cmd_set_deq()
698 int status = -ECONNRESET; in cdnsp_remove_request()
704 trace_cdnsp_remove_request(pep->out_ctx); in cdnsp_remove_request()
707 cur_td = &preq->td; in cdnsp_remove_request()
715 hw_deq = cdnsp_get_hw_deq(pdev, pep->idx, preq->request.stream_id); in cdnsp_remove_request()
718 seg = cdnsp_trb_in_td(pdev, cur_td->start_seg, cur_td->first_trb, in cdnsp_remove_request()
719 cur_td->last_trb, hw_deq); in cdnsp_remove_request()
721 if (seg && (pep->ep_state & EP_ENABLED) && in cdnsp_remove_request()
722 !(pep->ep_state & EP_DIS_IN_RROGRESS)) in cdnsp_remove_request()
723 cdnsp_find_new_dequeue_state(pdev, pep, preq->request.stream_id, in cdnsp_remove_request()
732 list_del_init(&cur_td->td_list); in cdnsp_remove_request()
733 ep_ring->num_tds--; in cdnsp_remove_request()
734 pep->stream_info.td_count--; in cdnsp_remove_request()
740 if (pdev->cdnsp_state & CDNSP_STATE_DISCONNECT_PENDING || in cdnsp_remove_request()
741 pep->ep_state & EP_DIS_IN_RROGRESS) { in cdnsp_remove_request()
742 status = -ESHUTDOWN; in cdnsp_remove_request()
747 cdnsp_gadget_giveback(pep, cur_td->preq, status); in cdnsp_remove_request()
754 struct cdnsp_port *port = pdev->active_port; in cdnsp_update_port_id()
757 if (port && port->port_num == port_id) in cdnsp_update_port_id()
761 old_port = port->port_num; in cdnsp_update_port_id()
763 if (port_id == pdev->usb2_port.port_num) { in cdnsp_update_port_id()
764 port = &pdev->usb2_port; in cdnsp_update_port_id()
765 } else if (port_id == pdev->usb3_port.port_num) { in cdnsp_update_port_id()
766 port = &pdev->usb3_port; in cdnsp_update_port_id()
768 dev_err(pdev->dev, "Port event with invalid port ID %d\n", in cdnsp_update_port_id()
770 return -EINVAL; in cdnsp_update_port_id()
775 pdev->active_port = port; in cdnsp_update_port_id()
779 if (port_id == pdev->usb2_port.port_num) in cdnsp_update_port_id()
783 &pdev->usb3_port.regs->portpmsc); in cdnsp_update_port_id()
798 if (GET_COMP_CODE(le32_to_cpu(event->generic.field[2])) != COMP_SUCCESS) in cdnsp_handle_port_status()
799 dev_err(pdev->dev, "ERR: incorrect PSC event\n"); in cdnsp_handle_port_status()
801 port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0])); in cdnsp_handle_port_status()
806 port_regs = pdev->active_port->regs; in cdnsp_handle_port_status()
808 if (port_id == pdev->usb2_port.port_num) in cdnsp_handle_port_status()
812 portsc = readl(&port_regs->portsc); in cdnsp_handle_port_status()
814 (portsc & PORT_CHANGE_BITS), &port_regs->portsc); in cdnsp_handle_port_status()
816 trace_cdnsp_handle_port_status(pdev->active_port->port_num, portsc); in cdnsp_handle_port_status()
818 pdev->gadget.speed = cdnsp_port_speed(portsc); in cdnsp_handle_port_status()
823 if (!(pdev->cdnsp_state & CDNSP_WAKEUP_PENDING) && in cdnsp_handle_port_status()
825 cmd_regs = readl(&pdev->op_regs->command); in cdnsp_handle_port_status()
830 cdnsp_set_link_state(pdev, &port_regs->portsc, in cdnsp_handle_port_status()
837 if ((pdev->cdnsp_state & CDNSP_WAKEUP_PENDING) && in cdnsp_handle_port_status()
839 pdev->cdnsp_state &= ~CDNSP_WAKEUP_PENDING; in cdnsp_handle_port_status()
846 if (link_state == XDEV_U0 && pdev->link_state == XDEV_U3 && in cdnsp_handle_port_status()
850 if (link_state == XDEV_U3 && pdev->link_state != XDEV_U3) in cdnsp_handle_port_status()
853 pdev->link_state = link_state; in cdnsp_handle_port_status()
858 if (pdev->gadget.connected && !(portsc & PORT_CONNECT)) in cdnsp_handle_port_status()
866 usb_gadget_set_state(&pdev->gadget, USB_STATE_ATTACHED); in cdnsp_handle_port_status()
873 pdev->u1_allowed = 0; in cdnsp_handle_port_status()
874 pdev->u2_allowed = 0; in cdnsp_handle_port_status()
875 pdev->may_wakeup = 0; in cdnsp_handle_port_status()
879 dev_err(pdev->dev, "Port Over Current detected\n"); in cdnsp_handle_port_status()
882 dev_err(pdev->dev, "Port Configure Error detected\n"); in cdnsp_handle_port_status()
884 if (readl(&port_regs->portsc) & PORT_CHANGE_BITS) in cdnsp_handle_port_status()
888 cdnsp_inc_deq(pdev, pdev->event_ring); in cdnsp_handle_port_status()
896 struct cdnsp_request *preq = td->preq; in cdnsp_td_cleanup()
905 if (preq->request.actual > preq->request.length) { in cdnsp_td_cleanup()
906 preq->request.actual = 0; in cdnsp_td_cleanup()
910 list_del_init(&td->td_list); in cdnsp_td_cleanup()
911 ep_ring->num_tds--; in cdnsp_td_cleanup()
912 preq->pep->stream_info.td_count--; in cdnsp_td_cleanup()
914 cdnsp_gadget_giveback(preq->pep, preq, *status); in cdnsp_td_cleanup()
926 ep_ring = cdnsp_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer)); in cdnsp_finish_td()
927 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); in cdnsp_finish_td()
941 while (ep_ring->dequeue != td->last_trb) in cdnsp_finish_td()
954 struct cdnsp_segment *seg = ring->deq_seg; in cdnsp_sum_trb_lengths()
955 union cdnsp_trb *trb = ring->dequeue; in cdnsp_sum_trb_lengths()
960 sum += TRB_LEN(le32_to_cpu(trb->generic.field[2])); in cdnsp_sum_trb_lengths()
978 start_trb->field[3] |= cpu_to_le32(start_cycle); in cdnsp_giveback_first_trb()
980 start_trb->field[3] &= cpu_to_le32(~TRB_CYCLE); in cdnsp_giveback_first_trb()
982 if ((pep->ep_state & EP_HAS_STREAMS) && in cdnsp_giveback_first_trb()
983 !pep->stream_info.first_prime_det) { in cdnsp_giveback_first_trb()
1005 trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event_trb->generic.field[3])); in cdnsp_process_ctrl_td()
1006 ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer)); in cdnsp_process_ctrl_td()
1007 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); in cdnsp_process_ctrl_td()
1015 td->request_length_set = true; in cdnsp_process_ctrl_td()
1016 td->preq->request.actual = td->preq->request.length - remaining; in cdnsp_process_ctrl_td()
1020 if (!td->request_length_set) in cdnsp_process_ctrl_td()
1021 td->preq->request.actual = td->preq->request.length; in cdnsp_process_ctrl_td()
1023 if (pdev->ep0_stage == CDNSP_DATA_STAGE && pep->number == 0 && in cdnsp_process_ctrl_td()
1024 pdev->three_stage_setup) { in cdnsp_process_ctrl_td()
1025 td = list_entry(ep_ring->td_list.next, struct cdnsp_td, in cdnsp_process_ctrl_td()
1027 pdev->ep0_stage = CDNSP_STATUS_STAGE; in cdnsp_process_ctrl_td()
1029 cdnsp_giveback_first_trb(pdev, pep, 0, ep_ring->cycle_state, in cdnsp_process_ctrl_td()
1030 &td->last_trb->generic); in cdnsp_process_ctrl_td()
1049 struct cdnsp_request *preq = td->preq; in cdnsp_process_isoc_td()
1056 ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer)); in cdnsp_process_isoc_td()
1057 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); in cdnsp_process_isoc_td()
1058 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); in cdnsp_process_isoc_td()
1059 ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2])); in cdnsp_process_isoc_td()
1061 requested = preq->request.length; in cdnsp_process_isoc_td()
1066 preq->request.status = 0; in cdnsp_process_isoc_td()
1069 preq->request.status = 0; in cdnsp_process_isoc_td()
1074 preq->request.status = -EOVERFLOW; in cdnsp_process_isoc_td()
1081 preq->request.status = 0; in cdnsp_process_isoc_td()
1090 preq->request.status = -1; in cdnsp_process_isoc_td()
1096 td_length += ep_trb_len - remaining; in cdnsp_process_isoc_td()
1101 td->preq->request.actual += td_length; in cdnsp_process_isoc_td()
1114 ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer)); in cdnsp_skip_isoc_td()
1115 td->preq->request.status = -EXDEV; in cdnsp_skip_isoc_td()
1116 td->preq->request.actual = 0; in cdnsp_skip_isoc_td()
1119 while (ep_ring->dequeue != td->last_trb) in cdnsp_skip_isoc_td()
1141 ep_ring = cdnsp_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer)); in cdnsp_process_bulk_intr_td()
1142 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); in cdnsp_process_bulk_intr_td()
1143 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); in cdnsp_process_bulk_intr_td()
1144 ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2])); in cdnsp_process_bulk_intr_td()
1145 requested = td->preq->request.length; in cdnsp_process_bulk_intr_td()
1153 td->preq->request.actual = remaining; in cdnsp_process_bulk_intr_td()
1162 if (ep_trb == td->last_trb) in cdnsp_process_bulk_intr_td()
1163 ep_trb_len = requested - remaining; in cdnsp_process_bulk_intr_td()
1166 ep_trb_len - remaining; in cdnsp_process_bulk_intr_td()
1167 td->preq->request.actual = ep_trb_len; in cdnsp_process_bulk_intr_td()
1170 ep->stream_info.drbls_count--; in cdnsp_process_bulk_intr_td()
1178 struct cdnsp_generic_trb *generic; in cdnsp_handle_tx_nrdy() local
1186 generic = (struct cdnsp_generic_trb *)event; in cdnsp_handle_tx_nrdy()
1187 ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1; in cdnsp_handle_tx_nrdy()
1188 dev_sid = TRB_TO_DEV_STREAM(le32_to_cpu(generic->field[0])); in cdnsp_handle_tx_nrdy()
1189 host_sid = TRB_TO_HOST_STREAM(le32_to_cpu(generic->field[2])); in cdnsp_handle_tx_nrdy()
1191 pep = &pdev->eps[ep_index]; in cdnsp_handle_tx_nrdy()
1193 if (!(pep->ep_state & EP_HAS_STREAMS)) in cdnsp_handle_tx_nrdy()
1197 pep->stream_info.first_prime_det = 1; in cdnsp_handle_tx_nrdy()
1198 for (cur_stream = 1; cur_stream < pep->stream_info.num_streams; in cdnsp_handle_tx_nrdy()
1200 ep_ring = pep->stream_info.stream_rings[cur_stream]; in cdnsp_handle_tx_nrdy()
1201 ep_ring->stream_active = 1; in cdnsp_handle_tx_nrdy()
1202 ep_ring->stream_rejected = 0; in cdnsp_handle_tx_nrdy()
1209 pep->stream_info.drbls_count--; in cdnsp_handle_tx_nrdy()
1210 ep_ring = pep->stream_info.stream_rings[dev_sid]; in cdnsp_handle_tx_nrdy()
1211 ep_ring->stream_active = 0; in cdnsp_handle_tx_nrdy()
1212 ep_ring->stream_rejected = 1; in cdnsp_handle_tx_nrdy()
1214 list_for_each_entry_safe(td, td_temp, &ep_ring->td_list, in cdnsp_handle_tx_nrdy()
1216 td->drbl = 0; in cdnsp_handle_tx_nrdy()
1234 int status = -EINPROGRESS; in cdnsp_handle_tx_event()
1243 invalidate = le32_to_cpu(event->flags) & TRB_EVENT_INVALIDATE; in cdnsp_handle_tx_event()
1244 ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1; in cdnsp_handle_tx_event()
1245 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); in cdnsp_handle_tx_event()
1246 ep_trb_dma = le64_to_cpu(event->buffer); in cdnsp_handle_tx_event()
1248 pep = &pdev->eps[ep_index]; in cdnsp_handle_tx_event()
1249 ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer)); in cdnsp_handle_tx_event()
1256 if (invalidate || !pdev->gadget.connected) in cdnsp_handle_tx_event()
1259 if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_DISABLED) { in cdnsp_handle_tx_event()
1260 trace_cdnsp_ep_disabled(pep->out_ctx); in cdnsp_handle_tx_event()
1273 dev_err(pdev->dev, "ERROR: %s event for unknown ring\n", in cdnsp_handle_tx_event()
1274 pep->name); in cdnsp_handle_tx_event()
1282 status = -EOVERFLOW; in cdnsp_handle_tx_event()
1299 pep->skip = true; in cdnsp_handle_tx_event()
1308 if (list_empty(&ep_ring->td_list)) { in cdnsp_handle_tx_event()
1318 ep_ring->last_td_was_short)) in cdnsp_handle_tx_event()
1322 if (pep->skip) { in cdnsp_handle_tx_event()
1323 pep->skip = false; in cdnsp_handle_tx_event()
1330 td = list_entry(ep_ring->td_list.next, struct cdnsp_td, in cdnsp_handle_tx_event()
1334 ep_seg = cdnsp_trb_in_td(pdev, ep_ring->deq_seg, in cdnsp_handle_tx_event()
1335 ep_ring->dequeue, td->last_trb, in cdnsp_handle_tx_event()
1338 desc = td->preq->pep->endpoint.desc; in cdnsp_handle_tx_event()
1341 ep_trb = &ep_seg->trbs[(ep_trb_dma - ep_seg->dma) in cdnsp_handle_tx_event()
1347 if (pep->skip && usb_endpoint_xfer_isoc(desc) && in cdnsp_handle_tx_event()
1348 td->last_trb != ep_trb) in cdnsp_handle_tx_event()
1349 return -EAGAIN; in cdnsp_handle_tx_event()
1354 * of FSE is not in the current TD pointed by ep_ring->dequeue in cdnsp_handle_tx_event()
1362 pep->skip = false; in cdnsp_handle_tx_event()
1367 if (!pep->skip || !usb_endpoint_xfer_isoc(desc)) { in cdnsp_handle_tx_event()
1369 dev_err(pdev->dev, in cdnsp_handle_tx_event()
1374 return -EINVAL; in cdnsp_handle_tx_event()
1382 ep_ring->last_td_was_short = true; in cdnsp_handle_tx_event()
1384 ep_ring->last_td_was_short = false; in cdnsp_handle_tx_event()
1386 if (pep->skip) { in cdnsp_handle_tx_event()
1387 pep->skip = false; in cdnsp_handle_tx_event()
1405 handling_skipped_tds = pep->skip; in cdnsp_handle_tx_event()
1412 cdnsp_inc_deq(pdev, pdev->event_ring); in cdnsp_handle_tx_event()
1415 * If ep->skip is set, it means there are missed tds on the in cdnsp_handle_tx_event()
1424 dev_err(pdev->dev, "@%016llx %08x %08x %08x %08x\n", in cdnsp_handle_tx_event()
1426 cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg, in cdnsp_handle_tx_event()
1427 pdev->event_ring->dequeue), in cdnsp_handle_tx_event()
1428 lower_32_bits(le64_to_cpu(event->buffer)), in cdnsp_handle_tx_event()
1429 upper_32_bits(le64_to_cpu(event->buffer)), in cdnsp_handle_tx_event()
1430 le32_to_cpu(event->transfer_len), in cdnsp_handle_tx_event()
1431 le32_to_cpu(event->flags)); in cdnsp_handle_tx_event()
1432 return -EINVAL; in cdnsp_handle_tx_event()
1449 event = pdev->event_ring->dequeue; in cdnsp_handle_event()
1450 flags = le32_to_cpu(event->event_cmd.flags); in cdnsp_handle_event()
1454 if (cycle_bit != pdev->event_ring->cycle_state) in cdnsp_handle_event()
1457 trace_cdnsp_handle_event(pdev->event_ring, &event->generic); in cdnsp_handle_event()
1471 cdnsp_inc_deq(pdev, pdev->cmd_ring); in cdnsp_handle_event()
1478 ret = cdnsp_handle_tx_event(pdev, &event->trans_event); in cdnsp_handle_event()
1483 pdev->ep0_stage = CDNSP_SETUP_STAGE; in cdnsp_handle_event()
1484 pdev->setup_id = TRB_SETUPID_TO_TYPE(flags); in cdnsp_handle_event()
1485 pdev->setup_speed = TRB_SETUP_SPEEDID(flags); in cdnsp_handle_event()
1486 pdev->setup = *((struct usb_ctrlrequest *) in cdnsp_handle_event()
1487 &event->trans_event.buffer); in cdnsp_handle_event()
1492 cdnsp_handle_tx_nrdy(pdev, &event->trans_event); in cdnsp_handle_event()
1495 comp_code = GET_COMP_CODE(le32_to_cpu(event->generic.field[2])); in cdnsp_handle_event()
1499 dev_err(pdev->dev, "Event Ring Full\n"); in cdnsp_handle_event()
1502 dev_err(pdev->dev, "Controller error code 0x%02x\n", in cdnsp_handle_event()
1512 dev_warn(pdev->dev, "ERROR unknown event type %ld\n", in cdnsp_handle_event()
1518 cdnsp_inc_deq(pdev, pdev->event_ring); in cdnsp_handle_event()
1535 spin_lock_irqsave(&pdev->lock, flags); in cdnsp_thread_irq_handler()
1537 if (pdev->cdnsp_state & (CDNSP_STATE_HALTED | CDNSP_STATE_DYING)) { in cdnsp_thread_irq_handler()
1543 if (pdev->gadget_driver) in cdnsp_thread_irq_handler()
1546 spin_unlock_irqrestore(&pdev->lock, flags); in cdnsp_thread_irq_handler()
1551 event_ring_deq = pdev->event_ring->dequeue; in cdnsp_thread_irq_handler()
1556 event_ring_deq = pdev->event_ring->dequeue; in cdnsp_thread_irq_handler()
1563 spin_unlock_irqrestore(&pdev->lock, flags); in cdnsp_thread_irq_handler()
1575 status = readl(&pdev->op_regs->status); in cdnsp_irq_handler()
1585 writel(status | STS_EINT, &pdev->op_regs->status); in cdnsp_irq_handler()
1586 irq_pending = readl(&pdev->ir_set->irq_pending); in cdnsp_irq_handler()
1588 writel(irq_pending, &pdev->ir_set->irq_pending); in cdnsp_irq_handler()
1599 * Generic function for queuing a TRB on a ring.
1610 trb = &ring->enqueue->generic; in cdnsp_queue_trb()
1612 trb->field[0] = cpu_to_le32(field1); in cdnsp_queue_trb()
1613 trb->field[1] = cpu_to_le32(field2); in cdnsp_queue_trb()
1614 trb->field[2] = cpu_to_le32(field3); in cdnsp_queue_trb()
1615 trb->field[3] = cpu_to_le32(field4); in cdnsp_queue_trb()
1640 dev_err(pdev->dev, "ERROR: incorrect endpoint state\n"); in cdnsp_prepare_ring()
1641 return -EINVAL; in cdnsp_prepare_ring()
1650 num_trbs_needed = num_trbs - ep_ring->num_trbs_free; in cdnsp_prepare_ring()
1653 dev_err(pdev->dev, "Ring expansion failed\n"); in cdnsp_prepare_ring()
1654 return -ENOMEM; in cdnsp_prepare_ring()
1658 while (cdnsp_trb_is_link(ep_ring->enqueue)) { in cdnsp_prepare_ring()
1659 ep_ring->enqueue->link.control |= cpu_to_le32(TRB_CHAIN); in cdnsp_prepare_ring()
1662 ep_ring->enqueue->link.control ^= cpu_to_le32(TRB_CYCLE); in cdnsp_prepare_ring()
1665 if (cdnsp_link_trb_toggles_cycle(ep_ring->enqueue)) in cdnsp_prepare_ring()
1666 ep_ring->cycle_state ^= 1; in cdnsp_prepare_ring()
1667 ep_ring->enq_seg = ep_ring->enq_seg->next; in cdnsp_prepare_ring()
1668 ep_ring->enqueue = ep_ring->enq_seg->trbs; in cdnsp_prepare_ring()
1680 ep_ring = cdnsp_get_transfer_ring(pdev, preq->pep, in cdnsp_prepare_transfer()
1681 preq->request.stream_id); in cdnsp_prepare_transfer()
1683 return -EINVAL; in cdnsp_prepare_transfer()
1686 GET_EP_CTX_STATE(preq->pep->out_ctx), in cdnsp_prepare_transfer()
1691 INIT_LIST_HEAD(&preq->td.td_list); in cdnsp_prepare_transfer()
1692 preq->td.preq = preq; in cdnsp_prepare_transfer()
1695 list_add_tail(&preq->td.td_list, &ep_ring->td_list); in cdnsp_prepare_transfer()
1696 ep_ring->num_tds++; in cdnsp_prepare_transfer()
1697 preq->pep->stream_info.td_count++; in cdnsp_prepare_transfer()
1699 preq->td.start_seg = ep_ring->enq_seg; in cdnsp_prepare_transfer()
1700 preq->td.first_trb = ep_ring->enqueue; in cdnsp_prepare_transfer()
1709 num_trbs = DIV_ROUND_UP(len + (addr & (TRB_MAX_BUFF_SIZE - 1)), in cdnsp_count_trbs()
1719 return cdnsp_count_trbs(preq->request.dma, preq->request.length); in count_trbs_needed()
1727 full_len = preq->request.length; in count_sg_trbs_needed()
1729 for_each_sg(preq->request.sg, sg, preq->request.num_sgs, i) { in count_sg_trbs_needed()
1733 full_len -= len; in count_sg_trbs_needed()
1743 if (running_total != preq->request.length) in cdnsp_check_trb_math()
1744 dev_err(preq->pep->pdev->dev, in cdnsp_check_trb_math()
1745 "%s - Miscalculated tx length, " in cdnsp_check_trb_math()
1747 preq->pep->name, running_total, in cdnsp_check_trb_math()
1748 preq->request.length, preq->request.actual); in cdnsp_check_trb_math()
1761 * TD size = total_packet_count - packets_transferred
1782 /* One TRB with a zero-length data packet. */ in cdnsp_td_remainder()
1787 maxp = usb_endpoint_maxp(preq->pep->endpoint.desc); in cdnsp_td_remainder()
1791 return (total_packet_count - ((transferred + trb_buff_len) / maxp)); in cdnsp_td_remainder()
1798 struct device *dev = pdev->dev; in cdnsp_align_td()
1803 max_pkt = usb_endpoint_maxp(preq->pep->endpoint.desc); in cdnsp_align_td()
1812 *trb_buff_len -= unalign; in cdnsp_align_td()
1823 new_buff_len = max_pkt - (enqd_len % max_pkt); in cdnsp_align_td()
1825 if (new_buff_len > (preq->request.length - enqd_len)) in cdnsp_align_td()
1826 new_buff_len = (preq->request.length - enqd_len); in cdnsp_align_td()
1829 if (preq->direction) { in cdnsp_align_td()
1830 sg_pcopy_to_buffer(preq->request.sg, in cdnsp_align_td()
1831 preq->request.num_mapped_sgs, in cdnsp_align_td()
1832 seg->bounce_buf, new_buff_len, enqd_len); in cdnsp_align_td()
1833 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf, in cdnsp_align_td()
1836 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf, in cdnsp_align_td()
1840 if (dma_mapping_error(dev, seg->bounce_dma)) { in cdnsp_align_td()
1842 dev_warn(pdev->dev, in cdnsp_align_td()
1848 seg->bounce_len = new_buff_len; in cdnsp_align_td()
1849 seg->bounce_offs = enqd_len; in cdnsp_align_td()
1851 trace_cdnsp_bounce_map(preq, new_buff_len, enqd_len, seg->bounce_dma, in cdnsp_align_td()
1855 * Bounce buffer successful aligned and seg->bounce_dma will be used in cdnsp_align_td()
1880 return -EINVAL; in cdnsp_queue_bulk_tx()
1882 full_len = preq->request.length; in cdnsp_queue_bulk_tx()
1884 if (preq->request.num_sgs) { in cdnsp_queue_bulk_tx()
1885 num_sgs = preq->request.num_sgs; in cdnsp_queue_bulk_tx()
1886 sg = preq->request.sg; in cdnsp_queue_bulk_tx()
1892 addr = (u64)preq->request.dma; in cdnsp_queue_bulk_tx()
1896 pep = preq->pep; in cdnsp_queue_bulk_tx()
1898 /* Deal with request.zero - need one more td/trb. */ in cdnsp_queue_bulk_tx()
1899 if (preq->request.zero && preq->request.length && in cdnsp_queue_bulk_tx()
1900 IS_ALIGNED(full_len, usb_endpoint_maxp(pep->endpoint.desc))) { in cdnsp_queue_bulk_tx()
1918 if (pep->wa1_nop_trb) { in cdnsp_queue_bulk_tx()
1919 field = le32_to_cpu(pep->wa1_nop_trb->trans_event.flags); in cdnsp_queue_bulk_tx()
1922 pep->wa1_nop_trb->trans_event.flags = cpu_to_le32(field); in cdnsp_queue_bulk_tx()
1923 pep->wa1_nop_trb = NULL; in cdnsp_queue_bulk_tx()
1931 start_trb = &ring->enqueue->generic; in cdnsp_queue_bulk_tx()
1932 start_cycle = ring->cycle_state; in cdnsp_queue_bulk_tx()
1935 /* Queue the TRBs, even if they are zero-length */ in cdnsp_queue_bulk_tx()
1944 trb_buff_len = full_len - enqd_len; in cdnsp_queue_bulk_tx()
1952 field |= ring->cycle_state; in cdnsp_queue_bulk_tx()
1961 if (cdnsp_trb_is_link(ring->enqueue + 1)) { in cdnsp_queue_bulk_tx()
1964 ring->enq_seg)) { in cdnsp_queue_bulk_tx()
1965 send_addr = ring->enq_seg->bounce_dma; in cdnsp_queue_bulk_tx()
1967 preq->td.bounce_seg = ring->enq_seg; in cdnsp_queue_bulk_tx()
1981 preq->td.last_trb = ring->enqueue; in cdnsp_queue_bulk_tx()
1986 if (!preq->direction) in cdnsp_queue_bulk_tx()
2008 --num_sgs; in cdnsp_queue_bulk_tx()
2009 sent_len -= block_len; in cdnsp_queue_bulk_tx()
2017 block_len -= sent_len; in cdnsp_queue_bulk_tx()
2021 if (cdnsp_trb_is_link(ring->enqueue + 1)) { in cdnsp_queue_bulk_tx()
2023 if (!ring->cycle_state) in cdnsp_queue_bulk_tx()
2026 pep->wa1_nop_trb = ring->enqueue; in cdnsp_queue_bulk_tx()
2033 ret = cdnsp_giveback_first_trb(pdev, pep, preq->request.stream_id, in cdnsp_queue_bulk_tx()
2037 preq->td.drbl = 1; in cdnsp_queue_bulk_tx()
2045 struct cdnsp_ep *pep = preq->pep; in cdnsp_queue_ctrl_tx()
2053 return -EINVAL; in cdnsp_queue_ctrl_tx()
2056 num_trbs = (pdev->three_stage_setup) ? 2 : 1; in cdnsp_queue_ctrl_tx()
2058 maxp = usb_endpoint_maxp(pep->endpoint.desc); in cdnsp_queue_ctrl_tx()
2060 if (preq->request.zero && preq->request.length && in cdnsp_queue_ctrl_tx()
2061 (preq->request.length % maxp == 0)) { in cdnsp_queue_ctrl_tx()
2071 if (preq->request.length > 0) { in cdnsp_queue_ctrl_tx()
2077 field |= TRB_IOC | (pdev->ep0_expect_in ? 0 : TRB_ISP); in cdnsp_queue_ctrl_tx()
2079 if (pdev->ep0_expect_in) in cdnsp_queue_ctrl_tx()
2082 length_field = TRB_LEN(preq->request.length) | in cdnsp_queue_ctrl_tx()
2086 lower_32_bits(preq->request.dma), in cdnsp_queue_ctrl_tx()
2087 upper_32_bits(preq->request.dma), length_field, in cdnsp_queue_ctrl_tx()
2088 field | ep_ring->cycle_state | in cdnsp_queue_ctrl_tx()
2089 TRB_SETUPID(pdev->setup_id) | in cdnsp_queue_ctrl_tx()
2090 pdev->setup_speed); in cdnsp_queue_ctrl_tx()
2095 if (!pdev->ep0_expect_in) in cdnsp_queue_ctrl_tx()
2099 lower_32_bits(preq->request.dma), in cdnsp_queue_ctrl_tx()
2100 upper_32_bits(preq->request.dma), 0, in cdnsp_queue_ctrl_tx()
2101 field | ep_ring->cycle_state | in cdnsp_queue_ctrl_tx()
2102 TRB_SETUPID(pdev->setup_id) | in cdnsp_queue_ctrl_tx()
2103 pdev->setup_speed); in cdnsp_queue_ctrl_tx()
2106 pdev->ep0_stage = CDNSP_DATA_STAGE; in cdnsp_queue_ctrl_tx()
2110 preq->td.last_trb = ep_ring->enqueue; in cdnsp_queue_ctrl_tx()
2113 if (preq->request.length == 0) in cdnsp_queue_ctrl_tx()
2114 field = ep_ring->cycle_state; in cdnsp_queue_ctrl_tx()
2116 field = (ep_ring->cycle_state ^ 1); in cdnsp_queue_ctrl_tx()
2118 if (preq->request.length > 0 && pdev->ep0_expect_in) in cdnsp_queue_ctrl_tx()
2121 if (pep->ep_state & EP0_HALTED_STATUS) { in cdnsp_queue_ctrl_tx()
2122 pep->ep_state &= ~EP0_HALTED_STATUS; in cdnsp_queue_ctrl_tx()
2129 field | TRB_IOC | TRB_SETUPID(pdev->setup_id) | in cdnsp_queue_ctrl_tx()
2130 TRB_TYPE(TRB_STATUS) | pdev->setup_speed); in cdnsp_queue_ctrl_tx()
2132 cdnsp_ring_ep_doorbell(pdev, pep, preq->request.stream_id); in cdnsp_queue_ctrl_tx()
2139 u32 ep_state = GET_EP_CTX_STATE(pep->out_ctx); in cdnsp_cmd_stop_ep()
2144 trace_cdnsp_ep_stopped_or_disabled(pep->out_ctx); in cdnsp_cmd_stop_ep()
2148 cdnsp_queue_stop_endpoint(pdev, pep->idx); in cdnsp_cmd_stop_ep()
2152 trace_cdnsp_handle_cmd_stop_ep(pep->out_ctx); in cdnsp_cmd_stop_ep()
2155 pep->ep_state |= EP_STOPPED; in cdnsp_cmd_stop_ep()
2173 if (pdev->gadget.speed < USB_SPEED_SUPER) in cdnsp_get_burst_count()
2176 max_burst = preq->pep->endpoint.comp_desc->bMaxBurst; in cdnsp_get_burst_count()
2177 return DIV_ROUND_UP(total_packet_count, max_burst + 1) - 1; in cdnsp_get_burst_count()
2196 if (pdev->gadget.speed >= USB_SPEED_SUPER) { in cdnsp_get_last_burst_packet_count()
2198 max_burst = preq->pep->endpoint.comp_desc->bMaxBurst; in cdnsp_get_last_burst_packet_count()
2203 * number of packets, but the TLBPC field is zero-based. in cdnsp_get_last_burst_packet_count()
2208 return residue - 1; in cdnsp_get_last_burst_packet_count()
2213 return total_packet_count - 1; in cdnsp_get_last_burst_packet_count()
2235 ep_ring = preq->pep->ring; in cdnsp_queue_isoc_tx()
2237 td_len = preq->request.length; in cdnsp_queue_isoc_tx()
2239 if (preq->request.num_sgs) { in cdnsp_queue_isoc_tx()
2240 num_sgs = preq->request.num_sgs; in cdnsp_queue_isoc_tx()
2241 sg = preq->request.sg; in cdnsp_queue_isoc_tx()
2246 addr = (u64)preq->request.dma; in cdnsp_queue_isoc_tx()
2255 start_trb = &ep_ring->enqueue->generic; in cdnsp_queue_isoc_tx()
2256 start_cycle = ep_ring->cycle_state; in cdnsp_queue_isoc_tx()
2260 max_pkt = usb_endpoint_maxp(preq->pep->endpoint.desc); in cdnsp_queue_isoc_tx()
2263 /* A zero-length transfer still involves at least one packet. */ in cdnsp_queue_isoc_tx()
2302 field = TRB_TYPE(TRB_NORMAL) | ep_ring->cycle_state; in cdnsp_queue_isoc_tx()
2309 if (usb_endpoint_dir_out(preq->pep->endpoint.desc)) in cdnsp_queue_isoc_tx()
2313 if (i < trbs_per_td - 1) { in cdnsp_queue_isoc_tx()
2318 preq->td.last_trb = ep_ring->enqueue; in cdnsp_queue_isoc_tx()
2328 td_remain_len -= trb_buff_len; in cdnsp_queue_isoc_tx()
2333 --num_sgs; in cdnsp_queue_isoc_tx()
2334 sent_len -= block_len; in cdnsp_queue_isoc_tx()
2342 block_len -= sent_len; in cdnsp_queue_isoc_tx()
2348 dev_err(pdev->dev, "ISOC TD length unmatch\n"); in cdnsp_queue_isoc_tx()
2349 ret = -EINVAL; in cdnsp_queue_isoc_tx()
2353 cdnsp_giveback_first_trb(pdev, preq->pep, preq->request.stream_id, in cdnsp_queue_isoc_tx()
2360 list_del_init(&preq->td.td_list); in cdnsp_queue_isoc_tx()
2361 ep_ring->num_tds--; in cdnsp_queue_isoc_tx()
2365 * queued into No-ops with a software-owned cycle bit. in cdnsp_queue_isoc_tx()
2368 * td->first_trb and td->start_seg are already set. in cdnsp_queue_isoc_tx()
2370 preq->td.last_trb = ep_ring->enqueue; in cdnsp_queue_isoc_tx()
2372 cdnsp_td_to_noop(pdev, ep_ring, &preq->td, true); in cdnsp_queue_isoc_tx()
2375 ep_ring->enqueue = preq->td.first_trb; in cdnsp_queue_isoc_tx()
2376 ep_ring->enq_seg = preq->td.start_seg; in cdnsp_queue_isoc_tx()
2377 ep_ring->cycle_state = start_cycle; in cdnsp_queue_isoc_tx()
2383 * Generic function for queuing a command TRB on the command ring.
2392 cdnsp_prepare_ring(pdev, pdev->cmd_ring, EP_STATE_RUNNING, 1, in cdnsp_queue_command()
2395 pdev->cmd.command_trb = pdev->cmd_ring->enqueue; in cdnsp_queue_command()
2397 cdnsp_queue_trb(pdev, pdev->cmd_ring, false, field1, field2, in cdnsp_queue_command()
2398 field3, field4 | pdev->cmd_ring->cycle_state); in cdnsp_queue_command()
2405 SLOT_ID_FOR_TRB(pdev->slot_id)); in cdnsp_queue_slot_control()
2416 SLOT_ID_FOR_TRB(pdev->slot_id) | in cdnsp_queue_address_device()
2424 SLOT_ID_FOR_TRB(pdev->slot_id)); in cdnsp_queue_reset_device()
2434 SLOT_ID_FOR_TRB(pdev->slot_id)); in cdnsp_queue_configure_endpoint()
2443 cdnsp_queue_command(pdev, 0, 0, 0, SLOT_ID_FOR_TRB(pdev->slot_id) | in cdnsp_queue_stop_endpoint()
2452 u32 trb_stream_id = STREAM_ID_FOR_TRB(deq_state->stream_id); in cdnsp_queue_new_dequeue_state()
2453 u32 trb_slot_id = SLOT_ID_FOR_TRB(pdev->slot_id); in cdnsp_queue_new_dequeue_state()
2458 addr = cdnsp_trb_virt_to_dma(deq_state->new_deq_seg, in cdnsp_queue_new_dequeue_state()
2459 deq_state->new_deq_ptr); in cdnsp_queue_new_dequeue_state()
2461 if (deq_state->stream_id) in cdnsp_queue_new_dequeue_state()
2465 deq_state->new_cycle_state, upper_32_bits(addr), in cdnsp_queue_new_dequeue_state()
2467 EP_ID_FOR_TRB(pep->idx) | type); in cdnsp_queue_new_dequeue_state()
2473 SLOT_ID_FOR_TRB(pdev->slot_id) | in cdnsp_queue_reset_ep()
2484 SLOT_ID_FOR_TRB(pdev->slot_id) | in cdnsp_queue_halt_endpoint()
2493 TRB_FH_TO_DEVICE_ADDRESS(pdev->device_address); in cdnsp_force_header_wakeup()