Lines Matching +full:ete +full:- +full:1
1 // SPDX-License-Identifier: GPL-2.0
13 * 1. Each segment is initialized to zero, except for link TRBs.
19 * 1. A ring is empty if enqueue == dequeue. This means there will always be at
26 * until you reach a non-link TRB.
31 * 1. When a consumer increments a dequeue pointer and encounters a toggle bit
37 * 1. Check if ring is full before you enqueue.
46 * 1. Check if TRB belongs to you. If the cycle bit == your ring cycle state,
59 #include <linux/dma-mapping.h>
61 #include "xhci-trace.h"
76 if (!seg || !trb || trb < seg->trbs) in xhci_trb_virt_to_dma()
79 segment_offset = trb - seg->trbs; in xhci_trb_virt_to_dma()
82 return seg->dma + (segment_offset * sizeof(*trb)); in xhci_trb_virt_to_dma()
92 if (in_range(dma, seg->dma, TRB_SEGMENT_SIZE)) { in xhci_dma_to_trb()
95 return &seg->trbs[(dma - seg->dma) / sizeof(union xhci_trb)]; in xhci_dma_to_trb()
104 return TRB_TYPE_NOOP_LE32(trb->generic.field[3]); in trb_is_noop()
109 return TRB_TYPE_LINK_LE32(trb->link.control); in trb_is_link()
114 return trb == &seg->trbs[TRBS_PER_SEGMENT - 1]; in last_trb_on_seg()
120 return last_trb_on_seg(seg, trb) && (seg->next == ring->first_seg); in last_trb_on_ring()
125 return le32_to_cpu(trb->link.control) & LINK_TOGGLE; in link_trb_toggles_cycle()
130 struct urb_priv *urb_priv = td->urb->hcpriv; in last_td_in_urb()
132 return urb_priv->num_tds_done == urb_priv->num_tds; in last_td_in_urb()
137 return ((le32_to_cpu(ring->dequeue->event_cmd.flags) & TRB_CYCLE) == in unhandled_event_trb()
138 ring->cycle_state); in unhandled_event_trb()
143 struct urb_priv *urb_priv = urb->hcpriv; in inc_td_cnt()
145 urb_priv->num_tds_done++; in inc_td_cnt()
152 trb->link.control &= cpu_to_le32(~TRB_CHAIN); in trb_to_noop()
154 trb->generic.field[0] = 0; in trb_to_noop()
155 trb->generic.field[1] = 0; in trb_to_noop()
156 trb->generic.field[2] = 0; in trb_to_noop()
158 trb->generic.field[3] &= cpu_to_le32(TRB_CYCLE); in trb_to_noop()
159 trb->generic.field[3] |= cpu_to_le32(TRB_TYPE(noop_type)); in trb_to_noop()
165 return seg->num * TRBS_PER_SEGMENT + (trb - seg->trbs); in trb_to_pos()
176 *seg = (*seg)->next; in next_trb()
177 *trb = ((*seg)->trbs); in next_trb()
191 if (ring->type == TYPE_EVENT) { in inc_deq()
192 if (!last_trb_on_seg(ring->deq_seg, ring->dequeue)) { in inc_deq()
193 ring->dequeue++; in inc_deq()
196 if (last_trb_on_ring(ring, ring->deq_seg, ring->dequeue)) in inc_deq()
197 ring->cycle_state ^= 1; in inc_deq()
198 ring->deq_seg = ring->deq_seg->next; in inc_deq()
199 ring->dequeue = ring->deq_seg->trbs; in inc_deq()
207 if (!trb_is_link(ring->dequeue)) { in inc_deq()
208 if (last_trb_on_seg(ring->deq_seg, ring->dequeue)) in inc_deq()
211 ring->dequeue++; in inc_deq()
214 while (trb_is_link(ring->dequeue)) { in inc_deq()
215 ring->deq_seg = ring->deq_seg->next; in inc_deq()
216 ring->dequeue = ring->deq_seg->trbs; in inc_deq()
220 if (link_trb_count++ > ring->num_segs) { in inc_deq()
236 while (trb_is_link(ring->enqueue)) { in inc_enq_past_link()
247 if (!xhci_link_chain_quirk(xhci, ring->type)) { in inc_enq_past_link()
248 ring->enqueue->link.control &= cpu_to_le32(~TRB_CHAIN); in inc_enq_past_link()
249 ring->enqueue->link.control |= cpu_to_le32(chain); in inc_enq_past_link()
254 ring->enqueue->link.control ^= cpu_to_le32(TRB_CYCLE); in inc_enq_past_link()
257 if (link_trb_toggles_cycle(ring->enqueue)) in inc_enq_past_link()
258 ring->cycle_state ^= 1; in inc_enq_past_link()
260 ring->enq_seg = ring->enq_seg->next; in inc_enq_past_link()
261 ring->enqueue = ring->enq_seg->trbs; in inc_enq_past_link()
265 if (link_trb_count++ > ring->num_segs) { in inc_enq_past_link()
288 chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN; in inc_enq()
290 if (last_trb_on_seg(ring->enq_seg, ring->enqueue)) { in inc_enq()
295 ring->enqueue++; in inc_enq()
303 if (trb_is_link(ring->enqueue) && (chain || more_trbs_coming)) in inc_enq()
333 return dma_in_range(suspect_dma, td->start_seg, td->start_trb, in trb_in_td()
334 td->end_seg, td->end_trb); in trb_in_td()
345 struct xhci_segment *enq_seg = ring->enq_seg; in xhci_num_trbs_free()
346 union xhci_trb *enq = ring->enqueue; in xhci_num_trbs_free()
353 enq_seg = enq_seg->next; in xhci_num_trbs_free()
354 enq = enq_seg->trbs; in xhci_num_trbs_free()
358 if (enq == ring->dequeue) in xhci_num_trbs_free()
359 return ring->num_segs * (TRBS_PER_SEGMENT - 1); in xhci_num_trbs_free()
362 if (ring->deq_seg == enq_seg && ring->dequeue >= enq) in xhci_num_trbs_free()
363 return free + (ring->dequeue - enq); in xhci_num_trbs_free()
364 last_on_seg = &enq_seg->trbs[TRBS_PER_SEGMENT - 1]; in xhci_num_trbs_free()
365 free += last_on_seg - enq; in xhci_num_trbs_free()
366 enq_seg = enq_seg->next; in xhci_num_trbs_free()
367 enq = enq_seg->trbs; in xhci_num_trbs_free()
368 } while (i++ < ring->num_segs); in xhci_num_trbs_free()
387 enq_used = ring->enqueue - ring->enq_seg->trbs; in xhci_ring_expansion_needed()
390 trbs_past_seg = enq_used + num_trbs - (TRBS_PER_SEGMENT - 1); in xhci_ring_expansion_needed()
402 if (trb_is_link(ring->enqueue) && ring->enq_seg->next->trbs == ring->dequeue) in xhci_ring_expansion_needed()
405 new_segs = 1 + (trbs_past_seg / (TRBS_PER_SEGMENT - 1)); in xhci_ring_expansion_needed()
406 seg = ring->enq_seg; in xhci_ring_expansion_needed()
409 seg = seg->next; in xhci_ring_expansion_needed()
410 if (seg == ring->deq_seg) { in xhci_ring_expansion_needed()
415 new_segs--; in xhci_ring_expansion_needed()
424 if (!(xhci->cmd_ring_state & CMD_RING_STATE_RUNNING)) in xhci_ring_cmd_db()
431 writel(DB_VALUE_HOST, &xhci->dba->doorbell[0]); in xhci_ring_cmd_db()
433 readl(&xhci->dba->doorbell[0]); in xhci_ring_cmd_db()
438 return mod_delayed_work(system_percpu_wq, &xhci->cmd_timer, in xhci_mod_cmd_timer()
439 msecs_to_jiffies(xhci->current_cmd->timeout_ms)); in xhci_mod_cmd_timer()
444 return list_first_entry_or_null(&xhci->cmd_list, struct xhci_command, in xhci_next_queued_cmd()
449 * Turn all commands on command ring with status set to "aborted" to no-op trbs.
451 * This must be called with command ring stopped and xhci->lock held.
458 /* Turn all aborted commands in list to no-ops, then restart */ in xhci_handle_stopped_cmd_ring()
459 list_for_each_entry(i_cmd, &xhci->cmd_list, cmd_list) { in xhci_handle_stopped_cmd_ring()
461 if (i_cmd->status != COMP_COMMAND_ABORTED) in xhci_handle_stopped_cmd_ring()
464 i_cmd->status = COMP_COMMAND_RING_STOPPED; in xhci_handle_stopped_cmd_ring()
466 xhci_dbg(xhci, "Turn aborted command %p to no-op\n", in xhci_handle_stopped_cmd_ring()
467 i_cmd->command_trb); in xhci_handle_stopped_cmd_ring()
469 trb_to_noop(i_cmd->command_trb, TRB_CMD_NOOP, false); in xhci_handle_stopped_cmd_ring()
473 * completion event is received for these no-op commands in xhci_handle_stopped_cmd_ring()
477 xhci->cmd_ring_state = CMD_RING_STATE_RUNNING; in xhci_handle_stopped_cmd_ring()
480 if ((xhci->cmd_ring->dequeue != xhci->cmd_ring->enqueue) && in xhci_handle_stopped_cmd_ring()
481 !(xhci->xhc_state & XHCI_STATE_DYING)) { in xhci_handle_stopped_cmd_ring()
482 xhci->current_cmd = cur_cmd; in xhci_handle_stopped_cmd_ring()
489 /* Must be called with xhci->lock held, releases and acquires lock back */
492 struct xhci_segment *new_seg = xhci->cmd_ring->deq_seg; in xhci_abort_cmd_ring()
493 union xhci_trb *new_deq = xhci->cmd_ring->dequeue; in xhci_abort_cmd_ring()
499 reinit_completion(&xhci->cmd_ring_stop_completion); in xhci_abort_cmd_ring()
514 xhci_write_64(xhci, crcr | CMD_RING_ABORT, &xhci->op_regs->cmd_ring); in xhci_abort_cmd_ring()
518 * seconds then driver handles it as if host died (-ENODEV). in xhci_abort_cmd_ring()
519 * In the future we should distinguish between -ENODEV and -ETIMEDOUT in xhci_abort_cmd_ring()
520 * and try to recover a -ETIMEDOUT with a host controller reset. in xhci_abort_cmd_ring()
522 ret = xhci_handshake(&xhci->op_regs->cmd_ring, in xhci_abort_cmd_ring()
536 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_abort_cmd_ring()
537 ret = wait_for_completion_timeout(&xhci->cmd_ring_stop_completion, in xhci_abort_cmd_ring()
539 spin_lock_irqsave(&xhci->lock, flags); in xhci_abort_cmd_ring()
554 __le32 __iomem *db_addr = &xhci->dba->doorbell[slot_id]; in xhci_ring_ep_doorbell()
555 struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_ring_ep_doorbell()
556 unsigned int ep_state = ep->ep_state; in xhci_ring_ep_doorbell()
583 ep = &xhci->devs[slot_id]->eps[ep_index]; in ring_doorbell_for_active_rings()
586 if (!(ep->ep_state & EP_HAS_STREAMS)) { in ring_doorbell_for_active_rings()
587 if (ep->ring && !(list_empty(&ep->ring->td_list))) in ring_doorbell_for_active_rings()
592 for (stream_id = 1; stream_id < ep->stream_info->num_streams; in ring_doorbell_for_active_rings()
594 struct xhci_stream_info *stream_info = ep->stream_info; in ring_doorbell_for_active_rings()
595 if (!list_empty(&stream_info->stream_rings[stream_id]->td_list)) in ring_doorbell_for_active_rings()
620 if (!xhci->devs[slot_id]) { in xhci_get_virt_ep()
625 return &xhci->devs[slot_id]->eps[ep_index]; in xhci_get_virt_ep()
633 if (!(ep->ep_state & EP_HAS_STREAMS)) in xhci_virt_ep_to_ring()
634 return ep->ring; in xhci_virt_ep_to_ring()
636 if (!ep->stream_info) in xhci_virt_ep_to_ring()
639 if (stream_id == 0 || stream_id >= ep->stream_info->num_streams) { in xhci_virt_ep_to_ring()
641 stream_id, ep->vdev->slot_id, ep->ep_index); in xhci_virt_ep_to_ring()
645 return ep->stream_info->stream_rings[stream_id]; in xhci_virt_ep_to_ring()
679 ep = &vdev->eps[ep_index]; in xhci_get_hw_deq()
681 if (ep->ep_state & EP_HAS_STREAMS) { in xhci_get_hw_deq()
682 st_ctx = &ep->stream_info->stream_ctx_array[stream_id]; in xhci_get_hw_deq()
683 return le64_to_cpu(st_ctx->stream_ring); in xhci_get_hw_deq()
685 ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index); in xhci_get_hw_deq()
686 return le64_to_cpu(ep_ctx->deq); in xhci_get_hw_deq()
693 struct xhci_virt_device *dev = xhci->devs[slot_id]; in xhci_move_dequeue_past_td()
694 struct xhci_virt_ep *ep = &dev->eps[ep_index]; in xhci_move_dequeue_past_td()
712 return -ENODEV; in xhci_move_dequeue_past_td()
716 new_seg = ep_ring->deq_seg; in xhci_move_dequeue_past_td()
717 new_deq = ep_ring->dequeue; in xhci_move_dequeue_past_td()
718 new_cycle = le32_to_cpu(td->end_trb->generic.field[3]) & TRB_CYCLE; in xhci_move_dequeue_past_td()
732 if (new_deq == td->end_trb) in xhci_move_dequeue_past_td()
742 if (new_deq == ep->ring->dequeue) { in xhci_move_dequeue_past_td()
744 return -EINVAL; in xhci_move_dequeue_past_td()
754 return -EINVAL; in xhci_move_dequeue_past_td()
757 if ((ep->ep_state & SET_DEQ_PENDING)) { in xhci_move_dequeue_past_td()
760 return -EBUSY; in xhci_move_dequeue_past_td()
767 return -ENOMEM; in xhci_move_dequeue_past_td()
781 ep->queued_deq_seg = new_seg; in xhci_move_dequeue_past_td()
782 ep->queued_deq_ptr = new_deq; in xhci_move_dequeue_past_td()
792 ep->ep_state |= SET_DEQ_PENDING; in xhci_move_dequeue_past_td()
805 struct xhci_segment *seg = td->start_seg; in td_to_noop()
806 union xhci_trb *trb = td->start_trb; in td_to_noop()
809 unchain_links = !xhci_link_chain_quirk(xhci, ep->ring ? ep->ring->type : TYPE_STREAM); in td_to_noop()
811 while (1) { in td_to_noop()
815 if (flip_cycle && trb != td->start_trb && trb != td->end_trb) in td_to_noop()
816 trb->generic.field[3] ^= cpu_to_le32(TRB_CYCLE); in td_to_noop()
818 if (trb == td->end_trb) in td_to_noop()
828 struct urb *urb = cur_td->urb; in xhci_giveback_urb_in_irq()
829 struct urb_priv *urb_priv = urb->hcpriv; in xhci_giveback_urb_in_irq()
830 struct usb_hcd *hcd = bus_to_hcd(urb->dev->bus); in xhci_giveback_urb_in_irq()
832 if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) { in xhci_giveback_urb_in_irq()
833 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs--; in xhci_giveback_urb_in_irq()
834 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) { in xhci_giveback_urb_in_irq()
835 if (xhci->quirks & XHCI_AMD_PLL_FIX) in xhci_giveback_urb_in_irq()
848 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_unmap_td_bounce_buffer()
849 struct xhci_segment *seg = td->bounce_seg; in xhci_unmap_td_bounce_buffer()
850 struct urb *urb = td->urb; in xhci_unmap_td_bounce_buffer()
857 dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len, in xhci_unmap_td_bounce_buffer()
862 dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len, in xhci_unmap_td_bounce_buffer()
865 if (urb->num_sgs) { in xhci_unmap_td_bounce_buffer()
866 len = sg_pcopy_from_buffer(urb->sg, urb->num_sgs, seg->bounce_buf, in xhci_unmap_td_bounce_buffer()
867 seg->bounce_len, seg->bounce_offs); in xhci_unmap_td_bounce_buffer()
868 if (len != seg->bounce_len) in xhci_unmap_td_bounce_buffer()
870 len, seg->bounce_len); in xhci_unmap_td_bounce_buffer()
872 memcpy(urb->transfer_buffer + seg->bounce_offs, seg->bounce_buf, in xhci_unmap_td_bounce_buffer()
873 seg->bounce_len); in xhci_unmap_td_bounce_buffer()
875 seg->bounce_len = 0; in xhci_unmap_td_bounce_buffer()
876 seg->bounce_offs = 0; in xhci_unmap_td_bounce_buffer()
885 urb = td->urb; in xhci_td_cleanup()
892 * length, urb->actual_length will be a very big number (since it's in xhci_td_cleanup()
895 if (urb->actual_length > urb->transfer_buffer_length) { in xhci_td_cleanup()
897 urb->transfer_buffer_length, urb->actual_length); in xhci_td_cleanup()
898 urb->actual_length = 0; in xhci_td_cleanup()
902 if (!list_empty(&td->td_list)) in xhci_td_cleanup()
903 list_del_init(&td->td_list); in xhci_td_cleanup()
905 if (!list_empty(&td->cancelled_td_list)) in xhci_td_cleanup()
906 list_del_init(&td->cancelled_td_list); in xhci_td_cleanup()
911 if ((urb->actual_length != urb->transfer_buffer_length && in xhci_td_cleanup()
912 (urb->transfer_flags & URB_SHORT_NOT_OK)) || in xhci_td_cleanup()
913 (status != 0 && !usb_endpoint_xfer_isoc(&urb->ep->desc))) in xhci_td_cleanup()
915 urb, urb->actual_length, in xhci_td_cleanup()
916 urb->transfer_buffer_length, status); in xhci_td_cleanup()
919 if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) in xhci_td_cleanup()
929 ring->dequeue = td->end_trb; in xhci_dequeue_td()
930 ring->deq_seg = td->end_seg; in xhci_dequeue_td()
942 list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list, in xhci_giveback_invalidated_tds()
945 ring = xhci_urb_to_transfer_ring(ep->xhci, td->urb); in xhci_giveback_invalidated_tds()
947 if (td->cancel_status == TD_CLEARED) { in xhci_giveback_invalidated_tds()
948 xhci_dbg(ep->xhci, "%s: Giveback cancelled URB %p TD\n", in xhci_giveback_invalidated_tds()
949 __func__, td->urb); in xhci_giveback_invalidated_tds()
950 xhci_td_cleanup(ep->xhci, td, ring, td->status); in xhci_giveback_invalidated_tds()
952 xhci_dbg(ep->xhci, "%s: Keep cancelled URB %p TD as cancel_status is %d\n", in xhci_giveback_invalidated_tds()
953 __func__, td->urb, td->cancel_status); in xhci_giveback_invalidated_tds()
955 if (ep->xhci->xhc_state & XHCI_STATE_DYING) in xhci_giveback_invalidated_tds()
968 ret = -ENOMEM; in xhci_reset_halted_ep()
972 xhci_dbg(xhci, "%s-reset ep %u, slot %u\n", in xhci_reset_halted_ep()
989 unsigned int slot_id = ep->vdev->slot_id; in xhci_handle_halted_endpoint()
996 if (ep->vdev->flags & VDEV_PORT_ERROR) in xhci_handle_halted_endpoint()
997 return -ENODEV; in xhci_handle_halted_endpoint()
1001 ep->ep_state |= EP_HARD_CLEAR_TOGGLE; in xhci_handle_halted_endpoint()
1002 if (td && list_empty(&td->cancelled_td_list)) { in xhci_handle_halted_endpoint()
1003 list_add_tail(&td->cancelled_td_list, &ep->cancelled_td_list); in xhci_handle_halted_endpoint()
1004 td->cancel_status = TD_HALTED; in xhci_handle_halted_endpoint()
1008 if (ep->ep_state & EP_HALTED) { in xhci_handle_halted_endpoint()
1010 ep->ep_index); in xhci_handle_halted_endpoint()
1014 err = xhci_reset_halted_ep(xhci, slot_id, ep->ep_index, reset_type); in xhci_handle_halted_endpoint()
1018 ep->ep_state |= EP_HALTED; in xhci_handle_halted_endpoint()
1028 * We're also in the event handler, so we can't get re-interrupted if another
1042 unsigned int slot_id = ep->vdev->slot_id; in xhci_invalidate_cancelled_tds()
1049 if (ep->ep_state & SET_DEQ_PENDING) in xhci_invalidate_cancelled_tds()
1052 xhci = ep->xhci; in xhci_invalidate_cancelled_tds()
1054 list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list, cancelled_td_list) { in xhci_invalidate_cancelled_tds()
1058 td->start_seg, td->start_trb), in xhci_invalidate_cancelled_tds()
1059 td->urb->stream_id, td->urb); in xhci_invalidate_cancelled_tds()
1060 list_del_init(&td->td_list); in xhci_invalidate_cancelled_tds()
1061 ring = xhci_urb_to_transfer_ring(xhci, td->urb); in xhci_invalidate_cancelled_tds()
1064 td->urb, td->urb->stream_id); in xhci_invalidate_cancelled_tds()
1073 hw_deq = xhci_get_hw_deq(xhci, ep->vdev, ep->ep_index, in xhci_invalidate_cancelled_tds()
1074 td->urb->stream_id); in xhci_invalidate_cancelled_tds()
1077 if (td->cancel_status == TD_HALTED || trb_in_td(td, hw_deq)) { in xhci_invalidate_cancelled_tds()
1078 switch (td->cancel_status) { in xhci_invalidate_cancelled_tds()
1079 case TD_CLEARED: /* TD is already no-op */ in xhci_invalidate_cancelled_tds()
1086 if (cached_td->urb->stream_id != td->urb->stream_id) { in xhci_invalidate_cancelled_tds()
1090 td->urb->stream_id, td->urb); in xhci_invalidate_cancelled_tds()
1091 td->cancel_status = TD_CLEARING_CACHE_DEFERRED; in xhci_invalidate_cancelled_tds()
1098 td->urb, cached_td->urb, in xhci_invalidate_cancelled_tds()
1099 td->urb->stream_id); in xhci_invalidate_cancelled_tds()
1101 cached_td->cancel_status = TD_CLEARED; in xhci_invalidate_cancelled_tds()
1104 td->cancel_status = TD_CLEARING_CACHE; in xhci_invalidate_cancelled_tds()
1110 td->cancel_status = TD_CLEARED; in xhci_invalidate_cancelled_tds()
1118 err = xhci_move_dequeue_past_td(xhci, slot_id, ep->ep_index, in xhci_invalidate_cancelled_tds()
1119 cached_td->urb->stream_id, in xhci_invalidate_cancelled_tds()
1122 /* Failed to move past cached td, just set cached TDs to no-op */ in xhci_invalidate_cancelled_tds()
1123 list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list, cancelled_td_list) { in xhci_invalidate_cancelled_tds()
1129 if (td->cancel_status != TD_CLEARING_CACHE && in xhci_invalidate_cancelled_tds()
1130 td->cancel_status != TD_CLEARING_CACHE_DEFERRED) in xhci_invalidate_cancelled_tds()
1133 td->urb); in xhci_invalidate_cancelled_tds()
1135 td->cancel_status = TD_CLEARED; in xhci_invalidate_cancelled_tds()
1146 * Call under xhci->lock on a stopped endpoint.
1156 * Only call for non-running rings without streams.
1163 if (!list_empty(&ep->ring->td_list)) { /* Not streams compatible */ in find_halted_td()
1164 hw_deq = xhci_get_hw_deq(ep->xhci, ep->vdev, ep->ep_index, 0); in find_halted_td()
1166 td = list_first_entry(&ep->ring->td_list, struct xhci_td, td_list); in find_halted_td()
1177 * 1. If the HW was in the middle of processing the TD that needs to be
1180 * 2. Otherwise, we turn all the TRBs in the TD into No-op TRBs (with the chain
1194 if (unlikely(TRB_TO_SUSPEND_PORT(le32_to_cpu(trb->generic.field[3])))) { in xhci_handle_cmd_stop_ep()
1195 if (!xhci->devs[slot_id]) in xhci_handle_cmd_stop_ep()
1201 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3])); in xhci_handle_cmd_stop_ep()
1206 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); in xhci_handle_cmd_stop_ep()
1219 * Proper error code is unknown here, it would be -EPIPE if device side in xhci_handle_cmd_stop_ep()
1220 * of enadpoit halted (aka STALL), and -EPROTO if not (transaction error) in xhci_handle_cmd_stop_ep()
1221 * We use -EPROTO, if device is stalled it should return a stall error on in xhci_handle_cmd_stop_ep()
1222 * next transfer, which then will return -EPIPE, and device side stall is in xhci_handle_cmd_stop_ep()
1232 if (ep->ep_state & EP_HALTED) in xhci_handle_cmd_stop_ep()
1235 if (ep->ep_state & EP_HAS_STREAMS) { in xhci_handle_cmd_stop_ep()
1241 td->status = -EPROTO; in xhci_handle_cmd_stop_ep()
1250 ep->ep_state &= ~EP_STOP_CMD_PENDING; in xhci_handle_cmd_stop_ep()
1261 if (ep->ep_state & EP_HALTED) in xhci_handle_cmd_stop_ep()
1275 if (time_is_before_jiffies(ep->stop_time + msecs_to_jiffies(100))) in xhci_handle_cmd_stop_ep()
1285 ep->ep_state &= ~EP_STOP_CMD_PENDING; in xhci_handle_cmd_stop_ep()
1299 ep->ep_state &= ~EP_STOP_CMD_PENDING; in xhci_handle_cmd_stop_ep()
1311 list_for_each_entry_safe(cur_td, tmp, &ring->td_list, td_list) { in xhci_kill_ring_urbs()
1312 list_del_init(&cur_td->td_list); in xhci_kill_ring_urbs()
1314 if (!list_empty(&cur_td->cancelled_td_list)) in xhci_kill_ring_urbs()
1315 list_del_init(&cur_td->cancelled_td_list); in xhci_kill_ring_urbs()
1319 inc_td_cnt(cur_td->urb); in xhci_kill_ring_urbs()
1321 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN); in xhci_kill_ring_urbs()
1337 if ((ep->ep_state & EP_HAS_STREAMS) || in xhci_kill_endpoint_urbs()
1338 (ep->ep_state & EP_GETTING_NO_STREAMS)) { in xhci_kill_endpoint_urbs()
1341 for (stream_id = 1; stream_id < ep->stream_info->num_streams; in xhci_kill_endpoint_urbs()
1343 ring = ep->stream_info->stream_rings[stream_id]; in xhci_kill_endpoint_urbs()
1353 ring = ep->ring; in xhci_kill_endpoint_urbs()
1362 list_for_each_entry_safe(cur_td, tmp, &ep->cancelled_td_list, in xhci_kill_endpoint_urbs()
1364 list_del_init(&cur_td->cancelled_td_list); in xhci_kill_endpoint_urbs()
1365 inc_td_cnt(cur_td->urb); in xhci_kill_endpoint_urbs()
1368 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN); in xhci_kill_endpoint_urbs()
1378 * Call with xhci->lock held.
1379 * lock is relased and re-acquired while giving back urb.
1386 if (xhci->xhc_state & XHCI_STATE_DYING) in xhci_hc_died()
1389 notify = !(xhci->xhc_state & XHCI_STATE_REMOVING); in xhci_hc_died()
1392 xhci->xhc_state |= XHCI_STATE_DYING; in xhci_hc_died()
1397 for (i = 0; i <= xhci->max_slots; i++) { in xhci_hc_died()
1398 if (!xhci->devs[i]) in xhci_hc_died()
1428 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3])); in xhci_handle_cmd_set_deq()
1429 stream_id = TRB_TO_STREAM_ID(le32_to_cpu(trb->generic.field[2])); in xhci_handle_cmd_set_deq()
1442 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); in xhci_handle_cmd_set_deq()
1443 slot_ctx = xhci_get_slot_ctx(xhci, ep->vdev->out_ctx); in xhci_handle_cmd_set_deq()
1447 if (ep->ep_state & EP_HAS_STREAMS) { in xhci_handle_cmd_set_deq()
1448 stream_ctx = &ep->stream_info->stream_ctx_array[stream_id]; in xhci_handle_cmd_set_deq()
1449 trace_xhci_handle_cmd_set_deq_stream(ep->stream_info, stream_id); in xhci_handle_cmd_set_deq()
1463 slot_state = le32_to_cpu(slot_ctx->dev_state); in xhci_handle_cmd_set_deq()
1487 if (ep->ep_state & EP_HAS_STREAMS) { in xhci_handle_cmd_set_deq()
1488 deq = le64_to_cpu(stream_ctx->stream_ring) & TR_DEQ_PTR_MASK; in xhci_handle_cmd_set_deq()
1499 if (xhci->quirks & XHCI_CDNS_SCTX_QUIRK) { in xhci_handle_cmd_set_deq()
1500 stream_ctx->reserved[0] = 0; in xhci_handle_cmd_set_deq()
1501 stream_ctx->reserved[1] = 0; in xhci_handle_cmd_set_deq()
1504 deq = le64_to_cpu(ep_ctx->deq) & TR_DEQ_PTR_MASK; in xhci_handle_cmd_set_deq()
1508 if (xhci_trb_virt_to_dma(ep->queued_deq_seg, in xhci_handle_cmd_set_deq()
1509 ep->queued_deq_ptr) == deq) { in xhci_handle_cmd_set_deq()
1513 ep_ring->deq_seg = ep->queued_deq_seg; in xhci_handle_cmd_set_deq()
1514 ep_ring->dequeue = ep->queued_deq_ptr; in xhci_handle_cmd_set_deq()
1518 ep->queued_deq_seg, ep->queued_deq_ptr); in xhci_handle_cmd_set_deq()
1522 list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list, in xhci_handle_cmd_set_deq()
1524 ep_ring = xhci_urb_to_transfer_ring(ep->xhci, td->urb); in xhci_handle_cmd_set_deq()
1525 if (td->cancel_status == TD_CLEARING_CACHE) { in xhci_handle_cmd_set_deq()
1526 td->cancel_status = TD_CLEARED; in xhci_handle_cmd_set_deq()
1527 xhci_dbg(ep->xhci, "%s: Giveback cancelled URB %p TD\n", in xhci_handle_cmd_set_deq()
1528 __func__, td->urb); in xhci_handle_cmd_set_deq()
1529 xhci_td_cleanup(ep->xhci, td, ep_ring, td->status); in xhci_handle_cmd_set_deq()
1531 xhci_dbg(ep->xhci, "%s: Keep cancelled URB %p TD as cancel_status is %d\n", in xhci_handle_cmd_set_deq()
1532 __func__, td->urb, td->cancel_status); in xhci_handle_cmd_set_deq()
1536 ep->ep_state &= ~SET_DEQ_PENDING; in xhci_handle_cmd_set_deq()
1537 ep->queued_deq_seg = NULL; in xhci_handle_cmd_set_deq()
1538 ep->queued_deq_ptr = NULL; in xhci_handle_cmd_set_deq()
1541 if (!list_empty(&ep->cancelled_td_list)) { in xhci_handle_cmd_set_deq()
1542 xhci_dbg(ep->xhci, "%s: Pending TDs to clear, continuing with invalidation\n", in xhci_handle_cmd_set_deq()
1551 xhci_dbg(ep->xhci, "%s: All TDs cleared, ring doorbell\n", __func__); in xhci_handle_cmd_set_deq()
1563 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3])); in xhci_handle_cmd_reset_ep()
1568 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); in xhci_handle_cmd_reset_ep()
1581 ep->ep_state &= ~EP_HALTED; in xhci_handle_cmd_reset_ep()
1586 if ((le32_to_cpu(trb->generic.field[3])) & TRB_TSP) in xhci_handle_cmd_reset_ep()
1594 command->slot_id = slot_id; in xhci_handle_cmd_enable_slot()
1596 command->slot_id = 0; in xhci_handle_cmd_enable_slot()
1605 virt_dev = xhci->devs[slot_id]; in xhci_handle_cmd_disable_slot()
1609 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx); in xhci_handle_cmd_disable_slot()
1612 if (xhci->quirks & XHCI_EP_LIMIT_QUIRK) in xhci_handle_cmd_disable_slot()
1616 xhci->dcbaa->dev_context_ptrs[slot_id] = 0; in xhci_handle_cmd_disable_slot()
1617 xhci->devs[slot_id] = NULL; in xhci_handle_cmd_disable_slot()
1634 virt_dev = xhci->devs[slot_id]; in xhci_handle_cmd_config_ep()
1637 ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx); in xhci_handle_cmd_config_ep()
1643 add_flags = le32_to_cpu(ctrl_ctx->add_flags); in xhci_handle_cmd_config_ep()
1646 ep_index = xhci_last_valid_endpoint(add_flags) - 1; in xhci_handle_cmd_config_ep()
1648 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->out_ctx, ep_index); in xhci_handle_cmd_config_ep()
1659 vdev = xhci->devs[slot_id]; in xhci_handle_cmd_addr_dev()
1662 slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx); in xhci_handle_cmd_addr_dev()
1671 vdev = xhci->devs[slot_id]; in xhci_handle_cmd_reset_dev()
1677 slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx); in xhci_handle_cmd_reset_dev()
1686 if (!(xhci->quirks & XHCI_NEC_HOST)) { in xhci_handle_cmd_nec_get_fw()
1687 xhci_warn(xhci, "WARN NEC_GET_FW command on non-NEC host\n"); in xhci_handle_cmd_nec_get_fw()
1692 NEC_FW_MAJOR(le32_to_cpu(event->status)), in xhci_handle_cmd_nec_get_fw()
1693 NEC_FW_MINOR(le32_to_cpu(event->status))); in xhci_handle_cmd_nec_get_fw()
1698 list_del(&cmd->cmd_list); in xhci_complete_del_and_free_cmd()
1700 if (cmd->completion) { in xhci_complete_del_and_free_cmd()
1701 cmd->status = comp_code; in xhci_complete_del_and_free_cmd()
1702 cmd->comp_param = comp_param; in xhci_complete_del_and_free_cmd()
1703 complete(cmd->completion); in xhci_complete_del_and_free_cmd()
1712 xhci->current_cmd = NULL; in xhci_cleanup_command_queue()
1713 list_for_each_entry_safe(cur_cmd, tmp_cmd, &xhci->cmd_list, cmd_list) in xhci_cleanup_command_queue()
1728 spin_lock_irqsave(&xhci->lock, flags); in xhci_handle_command_timeout()
1734 if (!xhci->current_cmd || delayed_work_pending(&xhci->cmd_timer)) { in xhci_handle_command_timeout()
1735 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_handle_command_timeout()
1739 cmd_field3 = le32_to_cpu(xhci->current_cmd->command_trb->generic.field[3]); in xhci_handle_command_timeout()
1740 usbsts = readl(&xhci->op_regs->status); in xhci_handle_command_timeout()
1752 ep->ep_state &= ~EP_STOP_CMD_PENDING; in xhci_handle_command_timeout()
1760 xhci->current_cmd->status = COMP_COMMAND_ABORTED; in xhci_handle_command_timeout()
1763 hw_ring_state = xhci_read_64(xhci, &xhci->op_regs->cmd_ring); in xhci_handle_command_timeout()
1769 if ((xhci->cmd_ring_state & CMD_RING_STATE_RUNNING) && in xhci_handle_command_timeout()
1772 xhci->cmd_ring_state = CMD_RING_STATE_ABORTED; in xhci_handle_command_timeout()
1779 if (xhci->xhc_state & XHCI_STATE_REMOVING) { in xhci_handle_command_timeout()
1788 xhci_handle_stopped_cmd_ring(xhci, xhci->current_cmd); in xhci_handle_command_timeout()
1791 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_handle_command_timeout()
1798 unsigned int slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags)); in handle_cmd_completion()
1799 u32 status = le32_to_cpu(event->status); in handle_cmd_completion()
1812 cmd_dma = le64_to_cpu(event->cmd_trb); in handle_cmd_completion()
1813 cmd_trb = xhci->cmd_ring->dequeue; in handle_cmd_completion()
1815 trace_xhci_handle_command(xhci->cmd_ring, &cmd_trb->generic, cmd_dma); in handle_cmd_completion()
1817 cmd_comp_code = GET_COMP_CODE(le32_to_cpu(event->status)); in handle_cmd_completion()
1821 complete_all(&xhci->cmd_ring_stop_completion); in handle_cmd_completion()
1825 cmd_dequeue_dma = xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg, in handle_cmd_completion()
1837 cmd = list_first_entry(&xhci->cmd_list, struct xhci_command, cmd_list); in handle_cmd_completion()
1839 cancel_delayed_work(&xhci->cmd_timer); in handle_cmd_completion()
1841 if (cmd->command_trb != xhci->cmd_ring->dequeue) { in handle_cmd_completion()
1854 xhci->cmd_ring_state = CMD_RING_STATE_STOPPED; in handle_cmd_completion()
1855 if (cmd->status == COMP_COMMAND_ABORTED) { in handle_cmd_completion()
1856 if (xhci->current_cmd == cmd) in handle_cmd_completion()
1857 xhci->current_cmd = NULL; in handle_cmd_completion()
1862 cmd_type = TRB_FIELD_TO_TYPE(le32_to_cpu(cmd_trb->generic.field[3])); in handle_cmd_completion()
1871 if (!cmd->completion) in handle_cmd_completion()
1881 le32_to_cpu(cmd_trb->generic.field[3]))); in handle_cmd_completion()
1882 if (!cmd->completion) in handle_cmd_completion()
1888 le32_to_cpu(cmd_trb->generic.field[3]))); in handle_cmd_completion()
1892 /* Is this an aborted command turned to NO-OP? */ in handle_cmd_completion()
1893 if (cmd->status == COMP_COMMAND_RING_STOPPED) in handle_cmd_completion()
1898 le32_to_cpu(cmd_trb->generic.field[3]))); in handle_cmd_completion()
1906 le32_to_cpu(cmd_trb->generic.field[3])); in handle_cmd_completion()
1921 if (!list_is_singular(&xhci->cmd_list)) { in handle_cmd_completion()
1922 xhci->current_cmd = list_first_entry(&cmd->cmd_list, in handle_cmd_completion()
1925 } else if (xhci->current_cmd == cmd) { in handle_cmd_completion()
1926 xhci->current_cmd = NULL; in handle_cmd_completion()
1932 inc_deq(xhci, xhci->cmd_ring); in handle_cmd_completion()
1939 if (trb_type == TRB_NEC_CMD_COMP && (xhci->quirks & XHCI_NEC_HOST)) in handle_vendor_event()
1940 handle_cmd_completion(xhci, &event->event_cmd); in handle_vendor_event()
1949 slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->generic.field[3])); in handle_device_notification()
1950 if (!xhci->devs[slot_id]) { in handle_device_notification()
1958 udev = xhci->devs[slot_id]->udev; in handle_device_notification()
1959 if (udev && udev->parent) in handle_device_notification()
1960 usb_wakeup_notification(udev->parent, udev->portnum); in handle_device_notification()
1966 * As per ThunderX2errata-129 USB 2 device may come up as USB 1
1967 * If a connection to a USB 1 device is followed by another connection
1983 writel(0x6F, hcd->regs + 0x1048); in xhci_cavium_reset_phy_quirk()
1985 /* De-assert the PHY reset */ in xhci_cavium_reset_phy_quirk()
1986 writel(0x7F, hcd->regs + 0x1048); in xhci_cavium_reset_phy_quirk()
1988 pll_lock_check = readl(hcd->regs + 0x1070); in xhci_cavium_reset_phy_quirk()
1989 } while (!(pll_lock_check & 0x1) && --retry_count); in xhci_cavium_reset_phy_quirk()
2004 if (GET_COMP_CODE(le32_to_cpu(event->generic.field[2])) != COMP_SUCCESS) in handle_port_status()
2008 port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0])); in handle_port_status()
2010 if ((port_id <= 0) || (port_id > xhci->max_ports)) { in handle_port_status()
2016 port = &xhci->hw_ports[port_id - 1]; in handle_port_status()
2017 if (!port || !port->rhub || port->hcd_portnum == DUPLICATE_ENTRY) { in handle_port_status()
2024 if (port->slot_id) in handle_port_status()
2025 vdev = xhci->devs[port->slot_id]; in handle_port_status()
2028 if (port->rhub == &xhci->usb3_rhub && xhci->shared_hcd == NULL) { in handle_port_status()
2034 hcd = port->rhub->hcd; in handle_port_status()
2035 bus_state = &port->rhub->bus_state; in handle_port_status()
2036 hcd_portnum = port->hcd_portnum; in handle_port_status()
2039 xhci_dbg(xhci, "Port change event, %d-%d, id %d, portsc: 0x%x\n", in handle_port_status()
2040 hcd->self.busnum, hcd_portnum + 1, port_id, portsc); in handle_port_status()
2044 if (hcd->state == HC_STATE_SUSPENDED) { in handle_port_status()
2051 vdev->flags |= VDEV_PORT_ERROR; in handle_port_status()
2053 vdev->flags &= ~VDEV_PORT_ERROR; in handle_port_status()
2059 cmd_reg = readl(&xhci->op_regs->command); in handle_port_status()
2071 bus_state->port_remote_wakeup |= 1 << hcd_portnum; in handle_port_status()
2073 usb_hcd_start_port_resume(&hcd->self, hcd_portnum); in handle_port_status()
2080 } else if (!test_bit(hcd_portnum, &bus_state->resuming_ports)) { in handle_port_status()
2082 port->resume_timestamp = jiffies + in handle_port_status()
2084 set_bit(hcd_portnum, &bus_state->resuming_ports); in handle_port_status()
2087 * usb device auto-resume latency around ~40ms. in handle_port_status()
2089 set_bit(HCD_FLAG_POLL_RH, &hcd->flags); in handle_port_status()
2090 mod_timer(&hcd->rh_timer, in handle_port_status()
2091 port->resume_timestamp); in handle_port_status()
2092 usb_hcd_start_port_resume(&hcd->self, hcd_portnum); in handle_port_status()
2103 complete(&port->u3exit_done); in handle_port_status()
2104 /* We've just brought the device into U0/1/2 through either the in handle_port_status()
2106 * U3Exit state after a host-initiated resume. If it's a device in handle_port_status()
2112 xhci_ring_device(xhci, port->slot_id); in handle_port_status()
2113 if (bus_state->port_remote_wakeup & (1 << hcd_portnum)) { in handle_port_status()
2115 usb_wakeup_notification(hcd->self.root_hub, in handle_port_status()
2116 hcd_portnum + 1); in handle_port_status()
2123 * Check to see if xhci-hub.c is waiting on RExit to U0 transition (or in handle_port_status()
2127 if (hcd->speed < HCD_USB3 && port->rexit_active) { in handle_port_status()
2128 complete(&port->rexit_done); in handle_port_status()
2129 port->rexit_active = false; in handle_port_status()
2134 if (hcd->speed < HCD_USB3) { in handle_port_status()
2136 if ((xhci->quirks & XHCI_RESET_PLL_ON_DISCONNECT) && in handle_port_status()
2151 * xHCI port-status-change events occur when the "or" of all the in handle_port_status()
2152 * status-change bits in the portsc register changes from 0 to 1. in handle_port_status()
2158 __func__, hcd->self.busnum); in handle_port_status()
2159 set_bit(HCD_FLAG_POLL_RH, &hcd->flags); in handle_port_status()
2160 spin_unlock(&xhci->lock); in handle_port_status()
2163 spin_lock(&xhci->lock); in handle_port_status()
2170 * As part of low/full-speed endpoint-halt processing in xhci_clear_hub_tt_buffer()
2173 if (td->urb->dev->tt && !usb_pipeint(td->urb->pipe) && in xhci_clear_hub_tt_buffer()
2174 (td->urb->dev->tt->hub != xhci_to_hcd(xhci)->self.root_hub) && in xhci_clear_hub_tt_buffer()
2175 !(ep->ep_state & EP_CLEARING_TT)) { in xhci_clear_hub_tt_buffer()
2176 ep->ep_state |= EP_CLEARING_TT; in xhci_clear_hub_tt_buffer()
2177 td->urb->ep->hcpriv = td->urb->dev; in xhci_clear_hub_tt_buffer()
2178 if (usb_hub_clear_tt_buffer(td->urb)) in xhci_clear_hub_tt_buffer()
2179 ep->ep_state &= ~EP_CLEARING_TT; in xhci_clear_hub_tt_buffer()
2194 int ep_type = CTX_TO_EP_TYPE(le32_to_cpu(ep_ctx->ep_info2)); in xhci_halted_host_endpoint()
2207 if (xhci->hci_version <= 0x95 && ep_type == CTRL_EP) in xhci_halted_host_endpoint()
2213 /* these errors halt all non-isochronous endpoints */ in xhci_halted_host_endpoint()
2224 * treat as not-an-error. in xhci_is_vendor_info_code()
2229 return 1; in xhci_is_vendor_info_code()
2240 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index); in finish_td()
2265 if (!(ep->ep_index == 0 && trb_comp_code == COMP_STALL_ERROR)) in finish_td()
2273 xhci_dequeue_td(xhci, td, ep_ring, td->status); in finish_td()
2280 union xhci_trb *trb = td->start_trb; in sum_trb_lengths()
2281 struct xhci_segment *seg = td->start_seg; in sum_trb_lengths()
2285 sum += TRB_LEN(le32_to_cpu(trb->generic.field[2])); in sum_trb_lengths()
2302 trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(ep_trb->generic.field[3])); in process_ctrl_td()
2303 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index); in process_ctrl_td()
2304 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); in process_ctrl_td()
2305 requested = td->urb->transfer_buffer_length; in process_ctrl_td()
2306 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); in process_ctrl_td()
2313 td->status = -ESHUTDOWN; in process_ctrl_td()
2316 td->status = 0; in process_ctrl_td()
2319 td->status = 0; in process_ctrl_td()
2323 td->urb->actual_length = remaining; in process_ctrl_td()
2330 td->urb->actual_length = 0; in process_ctrl_td()
2334 td->urb->actual_length = requested - remaining; in process_ctrl_td()
2337 td->urb->actual_length = requested; in process_ctrl_td()
2350 trb_comp_code, ep->ep_index); in process_ctrl_td()
2355 td->urb->actual_length = requested - remaining; in process_ctrl_td()
2356 else if (!td->urb_length_set) in process_ctrl_td()
2357 td->urb->actual_length = 0; in process_ctrl_td()
2371 td->urb_length_set = true; in process_ctrl_td()
2372 td->urb->actual_length = requested - remaining; in process_ctrl_td()
2378 if (!td->urb_length_set) in process_ctrl_td()
2379 td->urb->actual_length = requested; in process_ctrl_td()
2400 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); in process_isoc_td()
2401 urb_priv = td->urb->hcpriv; in process_isoc_td()
2402 idx = urb_priv->num_tds_done; in process_isoc_td()
2403 frame = &td->urb->iso_frame_desc[idx]; in process_isoc_td()
2404 requested = frame->length; in process_isoc_td()
2405 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); in process_isoc_td()
2406 ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2])); in process_isoc_td()
2407 short_framestatus = td->urb->transfer_flags & URB_SHORT_NOT_OK ? in process_isoc_td()
2408 -EREMOTEIO : 0; in process_isoc_td()
2413 /* Don't overwrite status if TD had an error, see xHCI 4.9.1 */ in process_isoc_td()
2414 if (td->error_mid_td) in process_isoc_td()
2417 frame->status = short_framestatus; in process_isoc_td()
2421 frame->status = 0; in process_isoc_td()
2424 frame->status = short_framestatus; in process_isoc_td()
2428 frame->status = -ECOMM; in process_isoc_td()
2434 frame->status = -EOVERFLOW; in process_isoc_td()
2435 if (ep_trb != td->end_trb) in process_isoc_td()
2436 td->error_mid_td = true; in process_isoc_td()
2439 frame->status = -EXDEV; in process_isoc_td()
2441 if (ep_trb != td->end_trb) in process_isoc_td()
2442 td->error_mid_td = true; in process_isoc_td()
2446 frame->status = -EPROTO; in process_isoc_td()
2449 frame->status = -EPROTO; in process_isoc_td()
2451 if (ep_trb != td->end_trb) in process_isoc_td()
2452 td->error_mid_td = true; in process_isoc_td()
2459 frame->status = short_framestatus; in process_isoc_td()
2470 frame->status = -1; in process_isoc_td()
2474 if (td->urb_length_set) in process_isoc_td()
2478 frame->actual_length = sum_trb_lengths(td, ep_trb) + in process_isoc_td()
2479 ep_trb_len - remaining; in process_isoc_td()
2481 frame->actual_length = requested; in process_isoc_td()
2483 td->urb->actual_length += frame->actual_length; in process_isoc_td()
2487 if (td->error_mid_td && ep_trb != td->end_trb) { in process_isoc_td()
2489 td->urb_length_set = true; in process_isoc_td()
2502 urb_priv = td->urb->hcpriv; in skip_isoc_td()
2503 idx = urb_priv->num_tds_done; in skip_isoc_td()
2504 frame = &td->urb->iso_frame_desc[idx]; in skip_isoc_td()
2507 frame->status = -EXDEV; in skip_isoc_td()
2510 frame->actual_length = 0; in skip_isoc_td()
2512 xhci_dequeue_td(xhci, td, ep->ring, status); in skip_isoc_td()
2526 slot_ctx = xhci_get_slot_ctx(xhci, ep->vdev->out_ctx); in process_bulk_intr_td()
2527 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); in process_bulk_intr_td()
2528 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); in process_bulk_intr_td()
2529 ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2])); in process_bulk_intr_td()
2530 requested = td->urb->transfer_buffer_length; in process_bulk_intr_td()
2534 ep->err_count = 0; in process_bulk_intr_td()
2536 if (ep_trb != td->end_trb || remaining) { in process_bulk_intr_td()
2538 xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n", in process_bulk_intr_td()
2539 td->urb->ep->desc.bEndpointAddress, in process_bulk_intr_td()
2542 td->status = 0; in process_bulk_intr_td()
2545 td->status = 0; in process_bulk_intr_td()
2548 td->urb->actual_length = remaining; in process_bulk_intr_td()
2552 td->urb->actual_length = sum_trb_lengths(td, ep_trb); in process_bulk_intr_td()
2555 if (xhci->quirks & XHCI_NO_SOFT_RETRY || in process_bulk_intr_td()
2556 (ep->err_count++ > MAX_SOFT_RETRY) || in process_bulk_intr_td()
2557 le32_to_cpu(slot_ctx->tt_info) & TT_SLOT) in process_bulk_intr_td()
2560 td->status = 0; in process_bulk_intr_td()
2569 if (ep_trb == td->end_trb) in process_bulk_intr_td()
2570 td->urb->actual_length = requested - remaining; in process_bulk_intr_td()
2572 td->urb->actual_length = in process_bulk_intr_td()
2574 ep_trb_len - remaining; in process_bulk_intr_td()
2579 td->urb->actual_length = 0; in process_bulk_intr_td()
2594 xhci_dbg(xhci, "Stream transaction error ep %u no id\n", ep->ep_index); in handle_transferless_tx_event()
2595 if (ep->err_count++ > MAX_SOFT_RETRY) in handle_transferless_tx_event()
2606 trb_comp_code, ep->vdev->slot_id, ep->ep_index); in handle_transferless_tx_event()
2607 return -ENODEV; in handle_transferless_tx_event()
2615 switch (ring->old_trb_comp_code) { in xhci_spurious_success_tx_event()
2617 return xhci->quirks & XHCI_SPURIOUS_SUCCESS; in xhci_spurious_success_tx_event()
2621 return xhci->quirks & XHCI_ETRON_HOST && in xhci_spurious_success_tx_event()
2622 ring->type == TYPE_ISOC; in xhci_spurious_success_tx_event()
2644 int status = -EINPROGRESS; in handle_tx_event()
2649 slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags)); in handle_tx_event()
2650 ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1; in handle_tx_event()
2651 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); in handle_tx_event()
2652 ep_trb_dma = le64_to_cpu(event->buffer); in handle_tx_event()
2661 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); in handle_tx_event()
2674 ep_trb = xhci_dma_to_trb(ep_ring->deq_seg, ep_trb_dma, NULL); in handle_tx_event()
2682 if (EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)) != 0) { in handle_tx_event()
2685 slot_id, ep_index, ep_ring->old_trb_comp_code); in handle_tx_event()
2697 "Stopped on No-op or Link TRB for slot %u ep %u\n", in handle_tx_event()
2709 status = -EPIPE; in handle_tx_event()
2714 status = -EPROTO; in handle_tx_event()
2719 status = -EPROTO; in handle_tx_event()
2724 status = -EOVERFLOW; in handle_tx_event()
2731 status = -EILSEQ; in handle_tx_event()
2738 status = -ENOSR; in handle_tx_event()
2770 ep->skip = true; in handle_tx_event()
2776 ep->skip = true; in handle_tx_event()
2787 status = -EPROTO; in handle_tx_event()
2797 if (ep->skip) in handle_tx_event()
2807 * xhci 4.9.1 states that if there are errors in mult-TRB in handle_tx_event()
2815 td = list_first_entry_or_null(&ep_ring->td_list, struct xhci_td, td_list); in handle_tx_event()
2817 if (td && td->error_mid_td && !trb_in_td(td, ep_trb_dma)) { in handle_tx_event()
2819 xhci_dequeue_td(xhci, td, ep_ring, td->status); in handle_tx_event()
2826 if (list_empty(&ep_ring->td_list)) { in handle_tx_event()
2841 ep->skip = false; in handle_tx_event()
2846 td = list_first_entry(&ep_ring->td_list, struct xhci_td, in handle_tx_event()
2852 if (ep->skip && usb_endpoint_xfer_isoc(&td->urb->ep->desc)) { in handle_tx_event()
2859 if (!list_empty(&ep_ring->td_list)) { in handle_tx_event()
2876 ep->skip = false; in handle_tx_event()
2887 * TD pointed by 'ep_ring->dequeue' because that the hardware dequeue in handle_tx_event()
2903 &ep_trb_dma, trb_comp_code, ep_ring->old_trb_comp_code); in handle_tx_event()
2904 ep_ring->old_trb_comp_code = 0; in handle_tx_event()
2912 if (ep->skip) { in handle_tx_event()
2916 ep->skip = false; in handle_tx_event()
2920 * If ep->skip is set, it means there are missed tds on the in handle_tx_event()
2925 } while (ep->skip); in handle_tx_event()
2927 ep_ring->old_trb_comp_code = trb_comp_code; in handle_tx_event()
2936 * No-op TRB could trigger interrupts in a case where a URB was killed in handle_tx_event()
2945 td->status = status; in handle_tx_event()
2948 if (usb_endpoint_xfer_control(&td->urb->ep->desc)) in handle_tx_event()
2950 else if (usb_endpoint_xfer_isoc(&td->urb->ep->desc)) in handle_tx_event()
2963 xhci_err(xhci, "Event dma %pad for ep %d status %d not part of TD at %016llx - %016llx\n", in handle_tx_event()
2965 (unsigned long long)xhci_trb_virt_to_dma(td->start_seg, td->start_trb), in handle_tx_event()
2966 (unsigned long long)xhci_trb_virt_to_dma(td->end_seg, td->end_trb)); in handle_tx_event()
2968 return -ESHUTDOWN; in handle_tx_event()
2973 ir->event_ring->deq_seg, in handle_tx_event()
2974 ir->event_ring->dequeue), in handle_tx_event()
2975 lower_32_bits(le64_to_cpu(event->buffer)), in handle_tx_event()
2976 upper_32_bits(le64_to_cpu(event->buffer)), in handle_tx_event()
2977 le32_to_cpu(event->transfer_len), in handle_tx_event()
2978 le32_to_cpu(event->flags)); in handle_tx_event()
2979 return -ENODEV; in handle_tx_event()
2983 * This function handles one OS-owned event on the event ring. It may drop
2984 * xhci->lock between event processing (e.g. to pass up port status changes).
2991 trace_xhci_handle_event(ir->event_ring, &event->generic, in xhci_handle_event_trb()
2992 xhci_trb_virt_to_dma(ir->event_ring->deq_seg, in xhci_handle_event_trb()
2993 ir->event_ring->dequeue)); in xhci_handle_event_trb()
3000 trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event->event_cmd.flags)); in xhci_handle_event_trb()
3005 handle_cmd_completion(xhci, &event->event_cmd); in xhci_handle_event_trb()
3011 handle_tx_event(xhci, ir, &event->trans_event); in xhci_handle_event_trb()
3022 /* Any of the above functions may drop and re-acquire the lock, so check in xhci_handle_event_trb()
3023 * to make sure a watchdog timer didn't mark the host as non-responsive. in xhci_handle_event_trb()
3025 if (xhci->xhc_state & XHCI_STATE_DYING) { in xhci_handle_event_trb()
3027 return -ENODEV; in xhci_handle_event_trb()
3035 * - When all events have finished
3036 * - To avoid "Event Ring Full Error" condition
3045 temp_64 = xhci_read_64(xhci, &ir->ir_set->erst_dequeue); in xhci_update_erst_dequeue()
3046 deq = xhci_trb_virt_to_dma(ir->event_ring->deq_seg, in xhci_update_erst_dequeue()
3047 ir->event_ring->dequeue); in xhci_update_erst_dequeue()
3058 temp_64 = ir->event_ring->deq_seg->num & ERST_DESI_MASK; in xhci_update_erst_dequeue()
3064 xhci_write_64(xhci, temp_64, &ir->ir_set->erst_dequeue); in xhci_update_erst_dequeue()
3070 if (!ir->ip_autoclear) { in xhci_clear_interrupt_pending()
3073 iman = readl(&ir->ir_set->iman); in xhci_clear_interrupt_pending()
3075 writel(iman, &ir->ir_set->iman); in xhci_clear_interrupt_pending()
3078 readl(&ir->ir_set->iman); in xhci_clear_interrupt_pending()
3083 * Handle all OS-owned events on an interrupter event ring. It may drop
3084 * and reaquire xhci->lock between event processing.
3096 if (!ir->event_ring || !ir->event_ring->dequeue) { in xhci_handle_events()
3098 return -ENOMEM; in xhci_handle_events()
3101 if (xhci->xhc_state & XHCI_STATE_DYING || in xhci_handle_events()
3102 xhci->xhc_state & XHCI_STATE_HALTED) { in xhci_handle_events()
3106 temp = xhci_read_64(xhci, &ir->ir_set->erst_dequeue); in xhci_handle_events()
3107 xhci_write_64(xhci, temp | ERST_EHB, &ir->ir_set->erst_dequeue); in xhci_handle_events()
3108 return -ENODEV; in xhci_handle_events()
3112 while (unhandled_event_trb(ir->event_ring)) { in xhci_handle_events()
3114 err = xhci_handle_event_trb(xhci, ir, ir->event_ring->dequeue); in xhci_handle_events()
3123 if (ir->isoc_bei_interval > AVOID_BEI_INTERVAL_MIN) in xhci_handle_events()
3124 ir->isoc_bei_interval = ir->isoc_bei_interval / 2; in xhci_handle_events()
3130 inc_deq(xhci, ir->event_ring); in xhci_handle_events()
3158 erdp_reg = xhci_read_64(xhci, &ir->ir_set->erst_dequeue); in xhci_skip_sec_intr_events()
3165 current_trb = ir->event_ring->dequeue; in xhci_skip_sec_intr_events()
3167 ring->cycle_state = le32_to_cpu(current_trb->event_cmd.flags) & TRB_CYCLE; in xhci_skip_sec_intr_events()
3183 spin_lock(&xhci->lock); in xhci_irq()
3185 status = readl(&xhci->op_regs->status); in xhci_irq()
3209 * so we can receive interrupts from other MSI-X interrupters. in xhci_irq()
3210 * Write 1 to clear the interrupt status. in xhci_irq()
3213 writel(status, &xhci->op_regs->status); in xhci_irq()
3216 xhci_handle_events(xhci, xhci->interrupters[0], false); in xhci_irq()
3218 spin_unlock(&xhci->lock); in xhci_irq()
3244 trb = &ring->enqueue->generic; in queue_trb()
3245 trb->field[0] = cpu_to_le32(field1); in queue_trb()
3246 trb->field[1] = cpu_to_le32(field2); in queue_trb()
3247 trb->field[2] = cpu_to_le32(field3); in queue_trb()
3250 trb->field[3] = cpu_to_le32(field4); in queue_trb()
3253 xhci_trb_virt_to_dma(ring->enq_seg, ring->enqueue)); in queue_trb()
3275 return -ENOENT; in prepare_ring()
3279 /* XXX not sure if this should be -ENOENT or not */ in prepare_ring()
3280 return -EINVAL; in prepare_ring()
3293 return -EINVAL; in prepare_ring()
3296 if (ep_ring != xhci->cmd_ring) { in prepare_ring()
3300 return -ENOMEM; in prepare_ring()
3308 return -ENOMEM; in prepare_ring()
3313 if (trb_is_link(ep_ring->enqueue)) in prepare_ring()
3316 if (last_trb_on_seg(ep_ring->enq_seg, ep_ring->enqueue)) { in prepare_ring()
3318 return -EINVAL; in prepare_ring()
3337 struct xhci_ep_ctx *ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); in prepare_transfer()
3339 ep_ring = xhci_triad_to_transfer_ring(xhci, xdev->slot_id, ep_index, in prepare_transfer()
3344 return -EINVAL; in prepare_transfer()
3352 urb_priv = urb->hcpriv; in prepare_transfer()
3353 td = &urb_priv->td[td_index]; in prepare_transfer()
3355 INIT_LIST_HEAD(&td->td_list); in prepare_transfer()
3356 INIT_LIST_HEAD(&td->cancelled_td_list); in prepare_transfer()
3359 ret = usb_hcd_link_urb_to_ep(bus_to_hcd(urb->dev->bus), urb); in prepare_transfer()
3364 td->urb = urb; in prepare_transfer()
3366 list_add_tail(&td->td_list, &ep_ring->td_list); in prepare_transfer()
3367 td->start_seg = ep_ring->enq_seg; in prepare_transfer()
3368 td->start_trb = ep_ring->enqueue; in prepare_transfer()
3377 num_trbs = DIV_ROUND_UP(len + (addr & (TRB_MAX_BUFF_SIZE - 1)), in count_trbs()
3387 return count_trbs(urb->transfer_dma, urb->transfer_buffer_length); in count_trbs_needed()
3395 full_len = urb->transfer_buffer_length; in count_sg_trbs_needed()
3397 for_each_sg(urb->sg, sg, urb->num_mapped_sgs, i) { in count_sg_trbs_needed()
3401 full_len -= len; in count_sg_trbs_needed()
3413 addr = (u64) (urb->transfer_dma + urb->iso_frame_desc[i].offset); in count_isoc_trbs_needed()
3414 len = urb->iso_frame_desc[i].length; in count_isoc_trbs_needed()
3421 if (unlikely(running_total != urb->transfer_buffer_length)) in check_trb_math()
3422 dev_err(&urb->dev->dev, "%s - ep %#x - Miscalculated tx length, " in check_trb_math()
3425 urb->ep->desc.bEndpointAddress, in check_trb_math()
3427 urb->transfer_buffer_length, in check_trb_math()
3428 urb->transfer_buffer_length); in check_trb_math()
3441 start_trb->field[3] |= cpu_to_le32(start_cycle); in giveback_first_trb()
3443 start_trb->field[3] &= cpu_to_le32(~TRB_CYCLE); in giveback_first_trb()
3452 xhci_interval = EP_INTERVAL_TO_UFRAMES(le32_to_cpu(ep_ctx->ep_info)); in check_interval()
3453 ep_interval = urb->interval; in check_interval()
3456 if (urb->dev->speed == USB_SPEED_LOW || in check_interval()
3457 urb->dev->speed == USB_SPEED_FULL) in check_interval()
3464 dev_dbg_ratelimited(&urb->dev->dev, in check_interval()
3468 urb->interval = xhci_interval; in check_interval()
3470 if (urb->dev->speed == USB_SPEED_LOW || in check_interval()
3471 urb->dev->speed == USB_SPEED_FULL) in check_interval()
3472 urb->interval /= 8; in check_interval()
3487 ep_ctx = xhci_get_ep_ctx(xhci, xhci->devs[slot_id]->out_ctx, ep_index); in xhci_queue_intr_tx()
3503 * TD size = total_packet_count - packets_transferred
3520 if (xhci->hci_version < 0x100 && !(xhci->quirks & XHCI_MTK_HOST)) in xhci_td_remainder()
3521 return ((td_total_len - transferred) >> 10); in xhci_td_remainder()
3523 /* One TRB with a zero-length data packet. */ in xhci_td_remainder()
3528 /* for MTK xHCI 0.96, TD size include this TRB, but not in 1.x */ in xhci_td_remainder()
3529 if ((xhci->quirks & XHCI_MTK_HOST) && (xhci->hci_version < 0x100)) in xhci_td_remainder()
3532 maxp = xhci_usb_endpoint_maxp(urb->dev, urb->ep); in xhci_td_remainder()
3536 return (total_packet_count - ((transferred + trb_buff_len) / maxp)); in xhci_td_remainder()
3543 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_align_td()
3549 max_pkt = xhci_usb_endpoint_maxp(urb->dev, urb->ep); in xhci_align_td()
3561 *trb_buff_len -= unalign; in xhci_align_td()
3571 new_buff_len = max_pkt - (enqd_len % max_pkt); in xhci_align_td()
3573 if (new_buff_len > (urb->transfer_buffer_length - enqd_len)) in xhci_align_td()
3574 new_buff_len = (urb->transfer_buffer_length - enqd_len); in xhci_align_td()
3578 if (urb->num_sgs) { in xhci_align_td()
3579 len = sg_pcopy_to_buffer(urb->sg, urb->num_sgs, in xhci_align_td()
3580 seg->bounce_buf, new_buff_len, enqd_len); in xhci_align_td()
3585 memcpy(seg->bounce_buf, urb->transfer_buffer + enqd_len, new_buff_len); in xhci_align_td()
3588 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf, in xhci_align_td()
3591 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf, in xhci_align_td()
3595 if (dma_mapping_error(dev, seg->bounce_dma)) { in xhci_align_td()
3601 seg->bounce_len = new_buff_len; in xhci_align_td()
3602 seg->bounce_offs = enqd_len; in xhci_align_td()
3606 return 1; in xhci_align_td()
3609 /* This is very similar to what ehci-q.c qtd_fill() does */
3630 return -EINVAL; in xhci_queue_bulk_tx()
3632 full_len = urb->transfer_buffer_length; in xhci_queue_bulk_tx()
3634 if (urb->num_sgs && !(urb->transfer_flags & URB_DMA_MAP_SINGLE)) { in xhci_queue_bulk_tx()
3635 num_sgs = urb->num_mapped_sgs; in xhci_queue_bulk_tx()
3636 sg = urb->sg; in xhci_queue_bulk_tx()
3642 addr = (u64) urb->transfer_dma; in xhci_queue_bulk_tx()
3645 ret = prepare_transfer(xhci, xhci->devs[slot_id], in xhci_queue_bulk_tx()
3646 ep_index, urb->stream_id, in xhci_queue_bulk_tx()
3651 urb_priv = urb->hcpriv; in xhci_queue_bulk_tx()
3653 /* Deal with URB_ZERO_PACKET - need one more td/trb */ in xhci_queue_bulk_tx()
3654 if (urb->transfer_flags & URB_ZERO_PACKET && urb_priv->num_tds > 1) in xhci_queue_bulk_tx()
3657 td = &urb_priv->td[0]; in xhci_queue_bulk_tx()
3664 start_trb = &ring->enqueue->generic; in xhci_queue_bulk_tx()
3665 start_cycle = ring->cycle_state; in xhci_queue_bulk_tx()
3668 /* Queue the TRBs, even if they are zero-length */ in xhci_queue_bulk_tx()
3678 trb_buff_len = full_len - enqd_len; in xhci_queue_bulk_tx()
3686 field |= ring->cycle_state; in xhci_queue_bulk_tx()
3693 if (trb_is_link(ring->enqueue + 1)) { in xhci_queue_bulk_tx()
3696 ring->enq_seg)) { in xhci_queue_bulk_tx()
3697 send_addr = ring->enq_seg->bounce_dma; in xhci_queue_bulk_tx()
3699 td->bounce_seg = ring->enq_seg; in xhci_queue_bulk_tx()
3707 td->end_trb = ring->enqueue; in xhci_queue_bulk_tx()
3708 td->end_seg = ring->enq_seg; in xhci_queue_bulk_tx()
3710 memcpy(&send_addr, urb->transfer_buffer, in xhci_queue_bulk_tx()
3739 --num_sgs; in xhci_queue_bulk_tx()
3740 sent_len -= block_len; in xhci_queue_bulk_tx()
3748 block_len -= sent_len; in xhci_queue_bulk_tx()
3753 ret = prepare_transfer(xhci, xhci->devs[slot_id], in xhci_queue_bulk_tx()
3754 ep_index, urb->stream_id, in xhci_queue_bulk_tx()
3755 1, urb, 1, mem_flags); in xhci_queue_bulk_tx()
3756 urb_priv->td[1].end_trb = ring->enqueue; in xhci_queue_bulk_tx()
3757 urb_priv->td[1].end_seg = ring->enq_seg; in xhci_queue_bulk_tx()
3758 field = TRB_TYPE(TRB_NORMAL) | ring->cycle_state | TRB_IOC; in xhci_queue_bulk_tx()
3763 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id, in xhci_queue_bulk_tx()
3768 /* Caller must have locked xhci->lock */
3784 return -EINVAL; in xhci_queue_ctrl_tx()
3790 if (!urb->setup_packet) in xhci_queue_ctrl_tx()
3791 return -EINVAL; in xhci_queue_ctrl_tx()
3793 if ((xhci->quirks & XHCI_ETRON_HOST) && in xhci_queue_ctrl_tx()
3794 urb->dev->speed >= USB_SPEED_SUPER) { in xhci_queue_ctrl_tx()
3800 if (last_trb_on_seg(ep_ring->enq_seg, ep_ring->enqueue + 1)) { in xhci_queue_ctrl_tx()
3801 field = TRB_TYPE(TRB_TR_NOOP) | ep_ring->cycle_state; in xhci_queue_ctrl_tx()
3807 /* 1 TRB for setup, 1 for status */ in xhci_queue_ctrl_tx()
3814 if (urb->transfer_buffer_length > 0) in xhci_queue_ctrl_tx()
3816 ret = prepare_transfer(xhci, xhci->devs[slot_id], in xhci_queue_ctrl_tx()
3817 ep_index, urb->stream_id, in xhci_queue_ctrl_tx()
3822 urb_priv = urb->hcpriv; in xhci_queue_ctrl_tx()
3823 td = &urb_priv->td[0]; in xhci_queue_ctrl_tx()
3830 start_trb = &ep_ring->enqueue->generic; in xhci_queue_ctrl_tx()
3831 start_cycle = ep_ring->cycle_state; in xhci_queue_ctrl_tx()
3833 /* Queue setup TRB - see section 6.4.1.2.1 */ in xhci_queue_ctrl_tx()
3835 setup = (struct usb_ctrlrequest *) urb->setup_packet; in xhci_queue_ctrl_tx()
3841 /* xHCI 1.0/1.1 6.4.1.2.1: Transfer Type field */ in xhci_queue_ctrl_tx()
3842 if ((xhci->hci_version >= 0x100) || (xhci->quirks & XHCI_MTK_HOST)) { in xhci_queue_ctrl_tx()
3843 if (urb->transfer_buffer_length > 0) { in xhci_queue_ctrl_tx()
3844 if (setup->bRequestType & USB_DIR_IN) in xhci_queue_ctrl_tx()
3852 setup->bRequestType | setup->bRequest << 8 | le16_to_cpu(setup->wValue) << 16, in xhci_queue_ctrl_tx()
3853 le16_to_cpu(setup->wIndex) | le16_to_cpu(setup->wLength) << 16, in xhci_queue_ctrl_tx()
3865 if (urb->transfer_buffer_length > 0) { in xhci_queue_ctrl_tx()
3870 memcpy(&addr, urb->transfer_buffer, in xhci_queue_ctrl_tx()
3871 urb->transfer_buffer_length); in xhci_queue_ctrl_tx()
3875 addr = (u64) urb->transfer_dma; in xhci_queue_ctrl_tx()
3879 urb->transfer_buffer_length, in xhci_queue_ctrl_tx()
3880 urb->transfer_buffer_length, in xhci_queue_ctrl_tx()
3881 urb, 1); in xhci_queue_ctrl_tx()
3882 length_field = TRB_LEN(urb->transfer_buffer_length) | in xhci_queue_ctrl_tx()
3885 if (setup->bRequestType & USB_DIR_IN) in xhci_queue_ctrl_tx()
3891 field | ep_ring->cycle_state); in xhci_queue_ctrl_tx()
3895 td->end_trb = ep_ring->enqueue; in xhci_queue_ctrl_tx()
3896 td->end_seg = ep_ring->enq_seg; in xhci_queue_ctrl_tx()
3898 /* Queue status TRB - see Table 7 and sections 4.11.2.2 and 6.4.1.2.3 */ in xhci_queue_ctrl_tx()
3900 if (urb->transfer_buffer_length > 0 && setup->bRequestType & USB_DIR_IN) in xhci_queue_ctrl_tx()
3909 field | TRB_IOC | TRB_TYPE(TRB_STATUS) | ep_ring->cycle_state); in xhci_queue_ctrl_tx()
3929 if (xhci->hci_version < 0x100 || urb->dev->speed < USB_SPEED_SUPER) in xhci_get_burst_count()
3932 max_burst = urb->ep->ss_ep_comp.bMaxBurst; in xhci_get_burst_count()
3933 return DIV_ROUND_UP(total_packet_count, max_burst + 1) - 1; in xhci_get_burst_count()
3941 * must contain (bMaxBurst + 1) number of packets, but the last burst can
3942 * contain 1 to (bMaxBurst + 1) packets.
3950 if (xhci->hci_version < 0x100) in xhci_get_last_burst_packet_count()
3953 if (urb->dev->speed >= USB_SPEED_SUPER) { in xhci_get_last_burst_packet_count()
3954 /* bMaxBurst is zero based: 0 means 1 packet per burst */ in xhci_get_last_burst_packet_count()
3955 max_burst = urb->ep->ss_ep_comp.bMaxBurst; in xhci_get_last_burst_packet_count()
3956 residue = total_packet_count % (max_burst + 1); in xhci_get_last_burst_packet_count()
3957 /* If residue is zero, the last burst contains (max_burst + 1) in xhci_get_last_burst_packet_count()
3958 * number of packets, but the TLBPC field is zero-based. in xhci_get_last_burst_packet_count()
3962 return residue - 1; in xhci_get_last_burst_packet_count()
3966 return total_packet_count - 1; in xhci_get_last_burst_packet_count()
3969 /* Returns the Isochronous Scheduling Threshold in Microframes. 1 Frame is 8 Microframes. */
3972 int ist = HCS_IST_VALUE(xhci->hcs_params2); in xhci_ist_microframes()
3974 if (xhci->hcs_params2 & HCS_IST_UNIT) in xhci_ist_microframes()
3992 if (urb->dev->speed == USB_SPEED_LOW || in xhci_get_isoc_frame_id()
3993 urb->dev->speed == USB_SPEED_FULL) in xhci_get_isoc_frame_id()
3994 start_frame = urb->start_frame + index * urb->interval; in xhci_get_isoc_frame_id()
3996 start_frame = (urb->start_frame + index * urb->interval) >> 3; in xhci_get_isoc_frame_id()
4005 * Start Frame ID = (Current MFINDEX register value + IST + 1) MOD 2048 in xhci_get_isoc_frame_id()
4013 current_frame_id = readl(&xhci->run_regs->microframe_index); in xhci_get_isoc_frame_id()
4014 start_frame_id = roundup(current_frame_id + ist + 1, 8); in xhci_get_isoc_frame_id()
4024 ret = -EINVAL; in xhci_get_isoc_frame_id()
4028 ret = -EINVAL; in xhci_get_isoc_frame_id()
4030 ret = -EINVAL; in xhci_get_isoc_frame_id()
4034 if (ret == -EINVAL || start_frame == start_frame_id) { in xhci_get_isoc_frame_id()
4035 start_frame = start_frame_id + 1; in xhci_get_isoc_frame_id()
4036 if (urb->dev->speed == USB_SPEED_LOW || in xhci_get_isoc_frame_id()
4037 urb->dev->speed == USB_SPEED_FULL) in xhci_get_isoc_frame_id()
4038 urb->start_frame = start_frame; in xhci_get_isoc_frame_id()
4040 urb->start_frame = start_frame << 3; in xhci_get_isoc_frame_id()
4060 if (xhci->hci_version < 0x100) in trb_block_event_intr()
4063 if (i == num_tds - 1) in trb_block_event_intr()
4069 if (i && ir->isoc_bei_interval && xhci->quirks & XHCI_AVOID_BEI) in trb_block_event_intr()
4070 return !!(i % ir->isoc_bei_interval); in trb_block_event_intr()
4095 xep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_queue_isoc_tx()
4096 ep_ring = xhci->devs[slot_id]->eps[ep_index].ring; in xhci_queue_isoc_tx()
4097 ir = xhci->interrupters[0]; in xhci_queue_isoc_tx()
4099 num_tds = urb->number_of_packets; in xhci_queue_isoc_tx()
4100 if (num_tds < 1) { in xhci_queue_isoc_tx()
4102 return -EINVAL; in xhci_queue_isoc_tx()
4104 start_addr = (u64) urb->transfer_dma; in xhci_queue_isoc_tx()
4105 start_trb = &ep_ring->enqueue->generic; in xhci_queue_isoc_tx()
4106 start_cycle = ep_ring->cycle_state; in xhci_queue_isoc_tx()
4108 urb_priv = urb->hcpriv; in xhci_queue_isoc_tx()
4109 /* Queue the TRBs for each TD, even if they are zero-length */ in xhci_queue_isoc_tx()
4117 addr = start_addr + urb->iso_frame_desc[i].offset; in xhci_queue_isoc_tx()
4118 td_len = urb->iso_frame_desc[i].length; in xhci_queue_isoc_tx()
4120 max_pkt = xhci_usb_endpoint_maxp(urb->dev, urb->ep); in xhci_queue_isoc_tx()
4123 /* A zero-length transfer still involves at least one packet. */ in xhci_queue_isoc_tx()
4132 ret = prepare_transfer(xhci, xhci->devs[slot_id], ep_index, in xhci_queue_isoc_tx()
4133 urb->stream_id, trbs_per_td, urb, i, mem_flags); in xhci_queue_isoc_tx()
4139 td = &urb_priv->td[i]; in xhci_queue_isoc_tx()
4142 if (!(urb->transfer_flags & URB_ISO_ASAP) && in xhci_queue_isoc_tx()
4143 (xhci->hcc_params & HCC_CFC)) { in xhci_queue_isoc_tx()
4156 (i ? ep_ring->cycle_state : !start_cycle); in xhci_queue_isoc_tx()
4158 /* xhci 1.1 with ETE uses TD_Size field for TBC, old is Rsvdz */ in xhci_queue_isoc_tx()
4159 if (!xep->use_extended_tbc) in xhci_queue_isoc_tx()
4169 ep_ring->cycle_state; in xhci_queue_isoc_tx()
4176 if (j < trbs_per_td - 1) { in xhci_queue_isoc_tx()
4181 td->end_trb = ep_ring->enqueue; in xhci_queue_isoc_tx()
4182 td->end_seg = ep_ring->enq_seg; in xhci_queue_isoc_tx()
4200 /* xhci 1.1 with ETE uses TD Size field for TBC */ in xhci_queue_isoc_tx()
4201 if (first_trb && xep->use_extended_tbc) in xhci_queue_isoc_tx()
4215 td_remain_len -= trb_buff_len; in xhci_queue_isoc_tx()
4221 ret = -EINVAL; in xhci_queue_isoc_tx()
4227 if (xhci->hcc_params & HCC_CFC) in xhci_queue_isoc_tx()
4228 xep->next_frame_id = urb->start_frame + num_tds * urb->interval; in xhci_queue_isoc_tx()
4230 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) { in xhci_queue_isoc_tx()
4231 if (xhci->quirks & XHCI_AMD_PLL_FIX) in xhci_queue_isoc_tx()
4234 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs++; in xhci_queue_isoc_tx()
4236 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id, in xhci_queue_isoc_tx()
4242 for (i--; i >= 0; i--) in xhci_queue_isoc_tx()
4243 list_del_init(&urb_priv->td[i].td_list); in xhci_queue_isoc_tx()
4246 * into No-ops with a software-owned cycle bit. That way the hardware in xhci_queue_isoc_tx()
4248 * overwrite them. td->start_trb and td->start_seg are already set. in xhci_queue_isoc_tx()
4250 urb_priv->td[0].end_trb = ep_ring->enqueue; in xhci_queue_isoc_tx()
4252 td_to_noop(xhci, xep, &urb_priv->td[0], true); in xhci_queue_isoc_tx()
4255 ep_ring->enqueue = urb_priv->td[0].start_trb; in xhci_queue_isoc_tx()
4256 ep_ring->enq_seg = urb_priv->td[0].start_seg; in xhci_queue_isoc_tx()
4257 ep_ring->cycle_state = start_cycle; in xhci_queue_isoc_tx()
4258 usb_hcd_unlink_urb_from_ep(bus_to_hcd(urb->dev->bus), urb); in xhci_queue_isoc_tx()
4266 * update urb->start_frame if URB_ISO_ASAP is set in transfer_flags or
4281 xdev = xhci->devs[slot_id]; in xhci_queue_isoc_tx_prepare()
4282 xep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_queue_isoc_tx_prepare()
4283 ep_ring = xdev->eps[ep_index].ring; in xhci_queue_isoc_tx_prepare()
4284 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); in xhci_queue_isoc_tx_prepare()
4287 num_tds = urb->number_of_packets; in xhci_queue_isoc_tx_prepare()
4305 /* Calculate the start frame and put it in urb->start_frame. */ in xhci_queue_isoc_tx_prepare()
4306 if ((xhci->hcc_params & HCC_CFC) && !list_empty(&ep_ring->td_list)) { in xhci_queue_isoc_tx_prepare()
4308 urb->start_frame = xep->next_frame_id; in xhci_queue_isoc_tx_prepare()
4313 start_frame = readl(&xhci->run_regs->microframe_index); in xhci_queue_isoc_tx_prepare()
4327 if (urb->dev->speed == USB_SPEED_LOW || in xhci_queue_isoc_tx_prepare()
4328 urb->dev->speed == USB_SPEED_FULL) { in xhci_queue_isoc_tx_prepare()
4329 start_frame = roundup(start_frame, urb->interval << 3); in xhci_queue_isoc_tx_prepare()
4330 urb->start_frame = start_frame >> 3; in xhci_queue_isoc_tx_prepare()
4332 start_frame = roundup(start_frame, urb->interval); in xhci_queue_isoc_tx_prepare()
4333 urb->start_frame = start_frame; in xhci_queue_isoc_tx_prepare()
4348 * Don't decrement xhci->cmd_ring_reserved_trbs after we've queued the TRB
4355 int reserved_trbs = xhci->cmd_ring_reserved_trbs; in queue_command()
4358 if ((xhci->xhc_state & XHCI_STATE_DYING) || in queue_command()
4359 (xhci->xhc_state & XHCI_STATE_HALTED)) { in queue_command()
4361 xhci->xhc_state); in queue_command()
4362 return -ESHUTDOWN; in queue_command()
4368 ret = prepare_ring(xhci, xhci->cmd_ring, EP_STATE_RUNNING, in queue_command()
4378 cmd->command_trb = xhci->cmd_ring->enqueue; in queue_command()
4381 if (list_empty(&xhci->cmd_list)) { in queue_command()
4382 xhci->current_cmd = cmd; in queue_command()
4386 list_add_tail(&cmd->cmd_list, &xhci->cmd_list); in queue_command()
4388 queue_trb(xhci, xhci->cmd_ring, false, field1, field2, field3, in queue_command()
4389 field4 | xhci->cmd_ring->cycle_state); in queue_command()