Lines Matching full:xhci
3 * xHCI host controller driver
58 #include "xhci.h"
59 #include "xhci-trace.h"
61 static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd,
148 static void next_trb(struct xhci_hcd *xhci, in next_trb() argument
164 void inc_deq(struct xhci_hcd *xhci, struct xhci_ring *ring) in inc_deq() argument
184 xhci_warn(xhci, "Missing link TRB at end of segment\n"); in inc_deq()
194 xhci_warn(xhci, "Ring is an endless link TRB loop\n"); in inc_deq()
215 * xHCI hardware can't handle the chain bit being cleared on a link TRB.
220 static void inc_enq(struct xhci_hcd *xhci, struct xhci_ring *ring, in inc_enq() argument
230 xhci_err(xhci, "Tried to move enqueue past ring segment\n"); in inc_enq()
253 if (!xhci_link_chain_quirk(xhci, ring->type)) { in inc_enq()
270 xhci_warn(xhci, "%s: Ring link TRB loop\n", __func__); in inc_enq()
320 static unsigned int xhci_ring_expansion_needed(struct xhci_hcd *xhci, struct xhci_ring *ring, in xhci_ring_expansion_needed() argument
352 xhci_dbg(xhci, "Adding %d trbs requires expanding ring by %d segments\n", in xhci_ring_expansion_needed()
363 void xhci_ring_cmd_db(struct xhci_hcd *xhci) in xhci_ring_cmd_db() argument
365 if (!(xhci->cmd_ring_state & CMD_RING_STATE_RUNNING)) in xhci_ring_cmd_db()
368 xhci_dbg(xhci, "// Ding dong!\n"); in xhci_ring_cmd_db()
372 writel(DB_VALUE_HOST, &xhci->dba->doorbell[0]); in xhci_ring_cmd_db()
374 readl(&xhci->dba->doorbell[0]); in xhci_ring_cmd_db()
377 static bool xhci_mod_cmd_timer(struct xhci_hcd *xhci) in xhci_mod_cmd_timer() argument
379 return mod_delayed_work(system_wq, &xhci->cmd_timer, in xhci_mod_cmd_timer()
380 msecs_to_jiffies(xhci->current_cmd->timeout_ms)); in xhci_mod_cmd_timer()
383 static struct xhci_command *xhci_next_queued_cmd(struct xhci_hcd *xhci) in xhci_next_queued_cmd() argument
385 return list_first_entry_or_null(&xhci->cmd_list, struct xhci_command, in xhci_next_queued_cmd()
392 * This must be called with command ring stopped and xhci->lock held.
394 static void xhci_handle_stopped_cmd_ring(struct xhci_hcd *xhci, in xhci_handle_stopped_cmd_ring() argument
400 list_for_each_entry(i_cmd, &xhci->cmd_list, cmd_list) { in xhci_handle_stopped_cmd_ring()
407 xhci_dbg(xhci, "Turn aborted command %p to no-op\n", in xhci_handle_stopped_cmd_ring()
418 xhci->cmd_ring_state = CMD_RING_STATE_RUNNING; in xhci_handle_stopped_cmd_ring()
421 if ((xhci->cmd_ring->dequeue != xhci->cmd_ring->enqueue) && in xhci_handle_stopped_cmd_ring()
422 !(xhci->xhc_state & XHCI_STATE_DYING)) { in xhci_handle_stopped_cmd_ring()
423 xhci->current_cmd = cur_cmd; in xhci_handle_stopped_cmd_ring()
424 xhci_mod_cmd_timer(xhci); in xhci_handle_stopped_cmd_ring()
425 xhci_ring_cmd_db(xhci); in xhci_handle_stopped_cmd_ring()
429 /* Must be called with xhci->lock held, releases and aquires lock back */
430 static int xhci_abort_cmd_ring(struct xhci_hcd *xhci, unsigned long flags) in xhci_abort_cmd_ring() argument
432 struct xhci_segment *new_seg = xhci->cmd_ring->deq_seg; in xhci_abort_cmd_ring()
433 union xhci_trb *new_deq = xhci->cmd_ring->dequeue; in xhci_abort_cmd_ring()
437 xhci_dbg(xhci, "Abort command ring\n"); in xhci_abort_cmd_ring()
439 reinit_completion(&xhci->cmd_ring_stop_completion); in xhci_abort_cmd_ring()
449 next_trb(xhci, NULL, &new_seg, &new_deq); in xhci_abort_cmd_ring()
451 next_trb(xhci, NULL, &new_seg, &new_deq); in xhci_abort_cmd_ring()
454 xhci_write_64(xhci, crcr | CMD_RING_ABORT, &xhci->op_regs->cmd_ring); in xhci_abort_cmd_ring()
456 /* Section 4.6.1.2 of xHCI 1.0 spec says software should also time the in xhci_abort_cmd_ring()
462 ret = xhci_handshake_check_state(xhci, &xhci->op_regs->cmd_ring, in xhci_abort_cmd_ring()
466 xhci_err(xhci, "Abort failed to stop command ring: %d\n", ret); in xhci_abort_cmd_ring()
467 xhci_halt(xhci); in xhci_abort_cmd_ring()
468 xhci_hc_died(xhci); in xhci_abort_cmd_ring()
477 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_abort_cmd_ring()
478 ret = wait_for_completion_timeout(&xhci->cmd_ring_stop_completion, in xhci_abort_cmd_ring()
480 spin_lock_irqsave(&xhci->lock, flags); in xhci_abort_cmd_ring()
482 xhci_dbg(xhci, "No stop event for abort, ring start fail?\n"); in xhci_abort_cmd_ring()
483 xhci_cleanup_command_queue(xhci); in xhci_abort_cmd_ring()
485 xhci_handle_stopped_cmd_ring(xhci, xhci_next_queued_cmd(xhci)); in xhci_abort_cmd_ring()
490 void xhci_ring_ep_doorbell(struct xhci_hcd *xhci, in xhci_ring_ep_doorbell() argument
495 __le32 __iomem *db_addr = &xhci->dba->doorbell[slot_id]; in xhci_ring_ep_doorbell()
496 struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_ring_ep_doorbell()
517 static void ring_doorbell_for_active_rings(struct xhci_hcd *xhci, in ring_doorbell_for_active_rings() argument
524 ep = &xhci->devs[slot_id]->eps[ep_index]; in ring_doorbell_for_active_rings()
529 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, 0); in ring_doorbell_for_active_rings()
537 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, in ring_doorbell_for_active_rings()
542 void xhci_ring_doorbell_for_active_rings(struct xhci_hcd *xhci, in xhci_ring_doorbell_for_active_rings() argument
546 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); in xhci_ring_doorbell_for_active_rings()
549 static struct xhci_virt_ep *xhci_get_virt_ep(struct xhci_hcd *xhci, in xhci_get_virt_ep() argument
554 xhci_warn(xhci, "Invalid slot_id %u\n", slot_id); in xhci_get_virt_ep()
558 xhci_warn(xhci, "Invalid endpoint index %u\n", ep_index); in xhci_get_virt_ep()
561 if (!xhci->devs[slot_id]) { in xhci_get_virt_ep()
562 xhci_warn(xhci, "No xhci virt device for slot_id %u\n", slot_id); in xhci_get_virt_ep()
566 return &xhci->devs[slot_id]->eps[ep_index]; in xhci_get_virt_ep()
569 static struct xhci_ring *xhci_virt_ep_to_ring(struct xhci_hcd *xhci, in xhci_virt_ep_to_ring() argument
581 xhci_warn(xhci, "Invalid stream_id %u request for slot_id %u ep_index %u\n", in xhci_virt_ep_to_ring()
593 struct xhci_ring *xhci_triad_to_transfer_ring(struct xhci_hcd *xhci, in xhci_triad_to_transfer_ring() argument
599 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); in xhci_triad_to_transfer_ring()
603 return xhci_virt_ep_to_ring(xhci, ep, stream_id); in xhci_triad_to_transfer_ring()
613 static u64 xhci_get_hw_deq(struct xhci_hcd *xhci, struct xhci_virt_device *vdev, in xhci_get_hw_deq() argument
626 ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index); in xhci_get_hw_deq()
630 static int xhci_move_dequeue_past_td(struct xhci_hcd *xhci, in xhci_move_dequeue_past_td() argument
634 struct xhci_virt_device *dev = xhci->devs[slot_id]; in xhci_move_dequeue_past_td()
648 ep_ring = xhci_triad_to_transfer_ring(xhci, slot_id, in xhci_move_dequeue_past_td()
651 xhci_warn(xhci, "WARN can't find new dequeue, invalid stream ID %u\n", in xhci_move_dequeue_past_td()
656 hw_dequeue = xhci_get_hw_deq(xhci, dev, ep_index, stream_id); in xhci_move_dequeue_past_td()
681 next_trb(xhci, ep_ring, &new_seg, &new_deq); in xhci_move_dequeue_past_td()
685 xhci_err(xhci, "Error: Failed finding new dequeue state\n"); in xhci_move_dequeue_past_td()
694 xhci_warn(xhci, "Can't find dma of new dequeue ptr\n"); in xhci_move_dequeue_past_td()
695 xhci_warn(xhci, "deq seg = %p, deq ptr = %p\n", new_seg, new_deq); in xhci_move_dequeue_past_td()
700 xhci_warn(xhci, "Set TR Deq already pending, don't submit for 0x%pad\n", in xhci_move_dequeue_past_td()
706 cmd = xhci_alloc_command(xhci, false, GFP_ATOMIC); in xhci_move_dequeue_past_td()
708 xhci_warn(xhci, "Can't alloc Set TR Deq cmd 0x%pad\n", &addr); in xhci_move_dequeue_past_td()
714 ret = queue_command(xhci, cmd, in xhci_move_dequeue_past_td()
720 xhci_free_command(xhci, cmd); in xhci_move_dequeue_past_td()
726 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_move_dequeue_past_td()
735 xhci_ring_cmd_db(xhci); in xhci_move_dequeue_past_td()
743 static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring, in td_to_noop() argument
759 next_trb(xhci, ep_ring, &seg, &trb); in td_to_noop()
763 static void xhci_giveback_urb_in_irq(struct xhci_hcd *xhci, in xhci_giveback_urb_in_irq() argument
771 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs--; in xhci_giveback_urb_in_irq()
772 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) { in xhci_giveback_urb_in_irq()
773 if (xhci->quirks & XHCI_AMD_PLL_FIX) in xhci_giveback_urb_in_irq()
783 static void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci, in xhci_unmap_td_bounce_buffer() argument
786 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_unmap_td_bounce_buffer()
807 xhci_warn(xhci, "WARN Wrong bounce buffer read length: %zu != %d\n", in xhci_unmap_td_bounce_buffer()
817 static int xhci_td_cleanup(struct xhci_hcd *xhci, struct xhci_td *td, in xhci_td_cleanup() argument
826 xhci_unmap_td_bounce_buffer(xhci, ep_ring, td); in xhci_td_cleanup()
834 xhci_warn(xhci, "URB req %u and actual %u transfer length mismatch\n", in xhci_td_cleanup()
852 xhci_dbg(xhci, "Giveback URB %p, len = %d, expected = %d, status = %d\n", in xhci_td_cleanup()
859 xhci_giveback_urb_in_irq(xhci, td, status); in xhci_td_cleanup()
875 ring = xhci_urb_to_transfer_ring(ep->xhci, td->urb); in xhci_giveback_invalidated_tds()
878 xhci_dbg(ep->xhci, "%s: Giveback cancelled URB %p TD\n", in xhci_giveback_invalidated_tds()
880 xhci_td_cleanup(ep->xhci, td, ring, td->status); in xhci_giveback_invalidated_tds()
882 xhci_dbg(ep->xhci, "%s: Keep cancelled URB %p TD as cancel_status is %d\n", in xhci_giveback_invalidated_tds()
885 if (ep->xhci->xhc_state & XHCI_STATE_DYING) in xhci_giveback_invalidated_tds()
890 static int xhci_reset_halted_ep(struct xhci_hcd *xhci, unsigned int slot_id, in xhci_reset_halted_ep() argument
896 command = xhci_alloc_command(xhci, false, GFP_ATOMIC); in xhci_reset_halted_ep()
902 xhci_dbg(xhci, "%s-reset ep %u, slot %u\n", in xhci_reset_halted_ep()
906 ret = xhci_queue_reset_ep(xhci, command, slot_id, ep_index, reset_type); in xhci_reset_halted_ep()
909 xhci_err(xhci, "ERROR queuing reset endpoint for slot %d ep_index %d, %d\n", in xhci_reset_halted_ep()
914 static int xhci_handle_halted_endpoint(struct xhci_hcd *xhci, in xhci_handle_halted_endpoint() argument
939 xhci_dbg(xhci, "Reset ep command for ep_index %d already pending\n", in xhci_handle_halted_endpoint()
944 err = xhci_reset_halted_ep(xhci, slot_id, ep->ep_index, reset_type); in xhci_handle_halted_endpoint()
950 xhci_ring_cmd_db(xhci); in xhci_handle_halted_endpoint()
957 * We have the xHCI lock, so nothing can modify this list until we drop it.
966 struct xhci_hcd *xhci; in xhci_invalidate_cancelled_tds() local
975 xhci = ep->xhci; in xhci_invalidate_cancelled_tds()
978 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_invalidate_cancelled_tds()
984 ring = xhci_urb_to_transfer_ring(xhci, td->urb); in xhci_invalidate_cancelled_tds()
986 xhci_warn(xhci, "WARN Cancelled URB %p has invalid stream ID %u.\n", in xhci_invalidate_cancelled_tds()
996 hw_deq = xhci_get_hw_deq(xhci, ep->vdev, ep->ep_index, in xhci_invalidate_cancelled_tds()
1000 if (td->cancel_status == TD_HALTED || trb_in_td(xhci, td, hw_deq, false)) { in xhci_invalidate_cancelled_tds()
1011 xhci_dbg(xhci, in xhci_invalidate_cancelled_tds()
1019 xhci_warn(xhci, in xhci_invalidate_cancelled_tds()
1023 td_to_noop(xhci, ring, cached_td, false); in xhci_invalidate_cancelled_tds()
1026 td_to_noop(xhci, ring, td, false); in xhci_invalidate_cancelled_tds()
1032 td_to_noop(xhci, ring, td, false); in xhci_invalidate_cancelled_tds()
1041 err = xhci_move_dequeue_past_td(xhci, slot_id, ep->ep_index, in xhci_invalidate_cancelled_tds()
1055 xhci_warn(xhci, "Failed to clear cancelled cached URB %p, mark clear anyway\n", in xhci_invalidate_cancelled_tds()
1057 td_to_noop(xhci, ring, td, false); in xhci_invalidate_cancelled_tds()
1074 hw_deq = xhci_get_hw_deq(ep->xhci, ep->vdev, ep->ep_index, 0); in find_halted_td()
1077 if (trb_in_td(ep->xhci, td, hw_deq, false)) in find_halted_td()
1093 static void xhci_handle_cmd_stop_ep(struct xhci_hcd *xhci, int slot_id, in xhci_handle_cmd_stop_ep() argument
1105 if (!xhci->devs[slot_id]) in xhci_handle_cmd_stop_ep()
1106 xhci_warn(xhci, "Stop endpoint command completion for disabled slot %u\n", in xhci_handle_cmd_stop_ep()
1112 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); in xhci_handle_cmd_stop_ep()
1116 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); in xhci_handle_cmd_stop_ep()
1137 xhci_dbg(xhci, "Stop ep completion raced with stall, reset ep\n"); in xhci_handle_cmd_stop_ep()
1147 err = xhci_handle_halted_endpoint(xhci, ep, td, reset_type); in xhci_handle_cmd_stop_ep()
1158 if (!(xhci->quirks & XHCI_NEC_HOST)) in xhci_handle_cmd_stop_ep()
1163 xhci_dbg(xhci, "Stop ep completion ctx error, ep is running\n"); in xhci_handle_cmd_stop_ep()
1165 command = xhci_alloc_command(xhci, false, GFP_ATOMIC); in xhci_handle_cmd_stop_ep()
1170 xhci_queue_stop_endpoint(xhci, command, slot_id, ep_index, 0); in xhci_handle_cmd_stop_ep()
1171 xhci_ring_cmd_db(xhci); in xhci_handle_cmd_stop_ep()
1185 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); in xhci_handle_cmd_stop_ep()
1188 static void xhci_kill_ring_urbs(struct xhci_hcd *xhci, struct xhci_ring *ring) in xhci_kill_ring_urbs() argument
1199 xhci_unmap_td_bounce_buffer(xhci, ring, cur_td); in xhci_kill_ring_urbs()
1203 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN); in xhci_kill_ring_urbs()
1207 static void xhci_kill_endpoint_urbs(struct xhci_hcd *xhci, in xhci_kill_endpoint_urbs() argument
1215 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); in xhci_kill_endpoint_urbs()
1229 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_kill_endpoint_urbs()
1232 xhci_kill_ring_urbs(xhci, ring); in xhci_kill_endpoint_urbs()
1238 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_kill_endpoint_urbs()
1241 xhci_kill_ring_urbs(xhci, ring); in xhci_kill_endpoint_urbs()
1250 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN); in xhci_kill_endpoint_urbs()
1260 * Call with xhci->lock held.
1263 void xhci_hc_died(struct xhci_hcd *xhci) in xhci_hc_died() argument
1267 if (xhci->xhc_state & XHCI_STATE_DYING) in xhci_hc_died()
1270 xhci_err(xhci, "xHCI host controller not responding, assume dead\n"); in xhci_hc_died()
1271 xhci->xhc_state |= XHCI_STATE_DYING; in xhci_hc_died()
1273 xhci_cleanup_command_queue(xhci); in xhci_hc_died()
1276 for (i = 0; i <= HCS_MAX_SLOTS(xhci->hcs_params1); i++) { in xhci_hc_died()
1277 if (!xhci->devs[i]) in xhci_hc_died()
1280 xhci_kill_endpoint_urbs(xhci, i, j); in xhci_hc_died()
1284 if (!(xhci->xhc_state & XHCI_STATE_REMOVING)) in xhci_hc_died()
1285 usb_hc_died(xhci_to_hcd(xhci)); in xhci_hc_died()
1288 static void update_ring_for_set_deq_completion(struct xhci_hcd *xhci, in update_ring_for_set_deq_completion() argument
1319 xhci_dbg(xhci, "Unable to find new dequeue pointer\n"); in update_ring_for_set_deq_completion()
1332 static void xhci_handle_cmd_set_deq(struct xhci_hcd *xhci, int slot_id, in xhci_handle_cmd_set_deq() argument
1346 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); in xhci_handle_cmd_set_deq()
1350 ep_ring = xhci_virt_ep_to_ring(xhci, ep, stream_id); in xhci_handle_cmd_set_deq()
1352 xhci_warn(xhci, "WARN Set TR deq ptr command for freed stream ID %u\n", in xhci_handle_cmd_set_deq()
1358 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); in xhci_handle_cmd_set_deq()
1359 slot_ctx = xhci_get_slot_ctx(xhci, ep->vdev->out_ctx); in xhci_handle_cmd_set_deq()
1369 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd invalid because of stream ID configuration\n"); in xhci_handle_cmd_set_deq()
1372 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed due to incorrect slot or ep state.\n"); in xhci_handle_cmd_set_deq()
1376 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_handle_cmd_set_deq()
1381 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed because slot %u was not enabled.\n", in xhci_handle_cmd_set_deq()
1385 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd with unknown completion code of %u.\n", in xhci_handle_cmd_set_deq()
1404 * Cadence xHCI controllers store some endpoint state in xhci_handle_cmd_set_deq()
1412 if (xhci->quirks & XHCI_CDNS_SCTX_QUIRK) { in xhci_handle_cmd_set_deq()
1419 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_handle_cmd_set_deq()
1426 update_ring_for_set_deq_completion(xhci, ep->vdev, in xhci_handle_cmd_set_deq()
1429 xhci_warn(xhci, "Mismatch between completed Set TR Deq Ptr command & xHCI internal state.\n"); in xhci_handle_cmd_set_deq()
1430 xhci_warn(xhci, "ep deq seg = %p, deq ptr = %p\n", in xhci_handle_cmd_set_deq()
1437 ep_ring = xhci_urb_to_transfer_ring(ep->xhci, td->urb); in xhci_handle_cmd_set_deq()
1440 xhci_dbg(ep->xhci, "%s: Giveback cancelled URB %p TD\n", in xhci_handle_cmd_set_deq()
1442 xhci_td_cleanup(ep->xhci, td, ep_ring, td->status); in xhci_handle_cmd_set_deq()
1446 xhci_dbg(ep->xhci, "%s: Keep cancelled URB %p TD as cancel_status is %d\n", in xhci_handle_cmd_set_deq()
1457 xhci_dbg(ep->xhci, "%s: Pending TDs to clear, continuing with invalidation\n", in xhci_handle_cmd_set_deq()
1462 xhci_dbg(ep->xhci, "%s: All TDs cleared, ring doorbell\n", __func__); in xhci_handle_cmd_set_deq()
1463 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); in xhci_handle_cmd_set_deq()
1467 static void xhci_handle_cmd_reset_ep(struct xhci_hcd *xhci, int slot_id, in xhci_handle_cmd_reset_ep() argument
1475 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); in xhci_handle_cmd_reset_ep()
1479 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); in xhci_handle_cmd_reset_ep()
1485 xhci_dbg_trace(xhci, trace_xhci_dbg_reset_ep, in xhci_handle_cmd_reset_ep()
1498 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); in xhci_handle_cmd_reset_ep()
1510 static void xhci_handle_cmd_disable_slot(struct xhci_hcd *xhci, int slot_id) in xhci_handle_cmd_disable_slot() argument
1515 virt_dev = xhci->devs[slot_id]; in xhci_handle_cmd_disable_slot()
1519 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx); in xhci_handle_cmd_disable_slot()
1522 if (xhci->quirks & XHCI_EP_LIMIT_QUIRK) in xhci_handle_cmd_disable_slot()
1524 xhci_free_device_endpoint_resources(xhci, virt_dev, true); in xhci_handle_cmd_disable_slot()
1527 static void xhci_handle_cmd_config_ep(struct xhci_hcd *xhci, int slot_id) in xhci_handle_cmd_config_ep() argument
1540 virt_dev = xhci->devs[slot_id]; in xhci_handle_cmd_config_ep()
1545 xhci_warn(xhci, "Could not get input context, bad type.\n"); in xhci_handle_cmd_config_ep()
1554 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->out_ctx, ep_index); in xhci_handle_cmd_config_ep()
1560 static void xhci_handle_cmd_addr_dev(struct xhci_hcd *xhci, int slot_id) in xhci_handle_cmd_addr_dev() argument
1565 vdev = xhci->devs[slot_id]; in xhci_handle_cmd_addr_dev()
1568 slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx); in xhci_handle_cmd_addr_dev()
1572 static void xhci_handle_cmd_reset_dev(struct xhci_hcd *xhci, int slot_id) in xhci_handle_cmd_reset_dev() argument
1577 vdev = xhci->devs[slot_id]; in xhci_handle_cmd_reset_dev()
1579 xhci_warn(xhci, "Reset device command completion for disabled slot %u\n", in xhci_handle_cmd_reset_dev()
1583 slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx); in xhci_handle_cmd_reset_dev()
1586 xhci_dbg(xhci, "Completed reset device command.\n"); in xhci_handle_cmd_reset_dev()
1589 static void xhci_handle_cmd_nec_get_fw(struct xhci_hcd *xhci, in xhci_handle_cmd_nec_get_fw() argument
1592 if (!(xhci->quirks & XHCI_NEC_HOST)) { in xhci_handle_cmd_nec_get_fw()
1593 xhci_warn(xhci, "WARN NEC_GET_FW command on non-NEC host\n"); in xhci_handle_cmd_nec_get_fw()
1596 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, in xhci_handle_cmd_nec_get_fw()
1614 void xhci_cleanup_command_queue(struct xhci_hcd *xhci) in xhci_cleanup_command_queue() argument
1617 xhci->current_cmd = NULL; in xhci_cleanup_command_queue()
1618 list_for_each_entry_safe(cur_cmd, tmp_cmd, &xhci->cmd_list, cmd_list) in xhci_cleanup_command_queue()
1624 struct xhci_hcd *xhci; in xhci_handle_command_timeout() local
1631 xhci = container_of(to_delayed_work(work), struct xhci_hcd, cmd_timer); in xhci_handle_command_timeout()
1633 spin_lock_irqsave(&xhci->lock, flags); in xhci_handle_command_timeout()
1639 if (!xhci->current_cmd || delayed_work_pending(&xhci->cmd_timer)) { in xhci_handle_command_timeout()
1640 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_handle_command_timeout()
1644 cmd_field3 = le32_to_cpu(xhci->current_cmd->command_trb->generic.field[3]); in xhci_handle_command_timeout()
1645 usbsts = readl(&xhci->op_regs->status); in xhci_handle_command_timeout()
1646 xhci_dbg(xhci, "Command timeout, USBSTS:%s\n", xhci_decode_usbsts(str, usbsts)); in xhci_handle_command_timeout()
1648 /* Bail out and tear down xhci if a stop endpoint command failed */ in xhci_handle_command_timeout()
1652 xhci_warn(xhci, "xHCI host not responding to stop endpoint command\n"); in xhci_handle_command_timeout()
1654 ep = xhci_get_virt_ep(xhci, TRB_TO_SLOT_ID(cmd_field3), in xhci_handle_command_timeout()
1659 xhci_halt(xhci); in xhci_handle_command_timeout()
1660 xhci_hc_died(xhci); in xhci_handle_command_timeout()
1665 xhci->current_cmd->status = COMP_COMMAND_ABORTED; in xhci_handle_command_timeout()
1668 hw_ring_state = xhci_read_64(xhci, &xhci->op_regs->cmd_ring); in xhci_handle_command_timeout()
1670 xhci_hc_died(xhci); in xhci_handle_command_timeout()
1674 if ((xhci->cmd_ring_state & CMD_RING_STATE_RUNNING) && in xhci_handle_command_timeout()
1677 xhci->cmd_ring_state = CMD_RING_STATE_ABORTED; in xhci_handle_command_timeout()
1678 xhci_dbg(xhci, "Command timeout\n"); in xhci_handle_command_timeout()
1679 xhci_abort_cmd_ring(xhci, flags); in xhci_handle_command_timeout()
1684 if (xhci->xhc_state & XHCI_STATE_REMOVING) { in xhci_handle_command_timeout()
1685 xhci_dbg(xhci, "host removed, ring start fail?\n"); in xhci_handle_command_timeout()
1686 xhci_cleanup_command_queue(xhci); in xhci_handle_command_timeout()
1692 xhci_dbg(xhci, "Command timeout on stopped ring\n"); in xhci_handle_command_timeout()
1693 xhci_handle_stopped_cmd_ring(xhci, xhci->current_cmd); in xhci_handle_command_timeout()
1696 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_handle_command_timeout()
1700 static void handle_cmd_completion(struct xhci_hcd *xhci, in handle_cmd_completion() argument
1712 xhci_warn(xhci, "Invalid slot_id %u\n", slot_id); in handle_cmd_completion()
1717 cmd_trb = xhci->cmd_ring->dequeue; in handle_cmd_completion()
1719 trace_xhci_handle_command(xhci->cmd_ring, &cmd_trb->generic); in handle_cmd_completion()
1725 complete_all(&xhci->cmd_ring_stop_completion); in handle_cmd_completion()
1729 cmd_dequeue_dma = xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg, in handle_cmd_completion()
1736 xhci_warn(xhci, in handle_cmd_completion()
1741 cmd = list_first_entry(&xhci->cmd_list, struct xhci_command, cmd_list); in handle_cmd_completion()
1743 cancel_delayed_work(&xhci->cmd_timer); in handle_cmd_completion()
1745 if (cmd->command_trb != xhci->cmd_ring->dequeue) { in handle_cmd_completion()
1746 xhci_err(xhci, in handle_cmd_completion()
1758 xhci->cmd_ring_state = CMD_RING_STATE_STOPPED; in handle_cmd_completion()
1760 if (xhci->current_cmd == cmd) in handle_cmd_completion()
1761 xhci->current_cmd = NULL; in handle_cmd_completion()
1772 xhci_handle_cmd_disable_slot(xhci, slot_id); in handle_cmd_completion()
1776 xhci_handle_cmd_config_ep(xhci, slot_id); in handle_cmd_completion()
1781 xhci_handle_cmd_addr_dev(xhci, slot_id); in handle_cmd_completion()
1787 xhci_handle_cmd_stop_ep(xhci, slot_id, cmd_trb, in handle_cmd_completion()
1793 xhci_handle_cmd_set_deq(xhci, slot_id, cmd_trb, cmd_comp_code); in handle_cmd_completion()
1803 xhci_handle_cmd_reset_ep(xhci, slot_id, cmd_trb, cmd_comp_code); in handle_cmd_completion()
1807 * Use the SLOT_ID from the command TRB instead (xhci 4.6.11) in handle_cmd_completion()
1811 xhci_handle_cmd_reset_dev(xhci, slot_id); in handle_cmd_completion()
1814 xhci_handle_cmd_nec_get_fw(xhci, event); in handle_cmd_completion()
1818 xhci_info(xhci, "INFO unknown command type %d\n", cmd_type); in handle_cmd_completion()
1823 if (!list_is_singular(&xhci->cmd_list)) { in handle_cmd_completion()
1824 xhci->current_cmd = list_first_entry(&cmd->cmd_list, in handle_cmd_completion()
1826 xhci_mod_cmd_timer(xhci); in handle_cmd_completion()
1827 } else if (xhci->current_cmd == cmd) { in handle_cmd_completion()
1828 xhci->current_cmd = NULL; in handle_cmd_completion()
1834 inc_deq(xhci, xhci->cmd_ring); in handle_cmd_completion()
1837 static void handle_vendor_event(struct xhci_hcd *xhci, in handle_vendor_event() argument
1840 xhci_dbg(xhci, "Vendor specific event TRB type = %u\n", trb_type); in handle_vendor_event()
1841 if (trb_type == TRB_NEC_CMD_COMP && (xhci->quirks & XHCI_NEC_HOST)) in handle_vendor_event()
1842 handle_cmd_completion(xhci, &event->event_cmd); in handle_vendor_event()
1845 static void handle_device_notification(struct xhci_hcd *xhci, in handle_device_notification() argument
1852 if (!xhci->devs[slot_id]) { in handle_device_notification()
1853 xhci_warn(xhci, "Device Notification event for " in handle_device_notification()
1858 xhci_dbg(xhci, "Device Wake Notification event for slot ID %u\n", in handle_device_notification()
1860 udev = xhci->devs[slot_id]->udev; in handle_device_notification()
1866 * Quirk hanlder for errata seen on Cavium ThunderX2 processor XHCI
1877 static void xhci_cavium_reset_phy_quirk(struct xhci_hcd *xhci) in xhci_cavium_reset_phy_quirk() argument
1879 struct usb_hcd *hcd = xhci_to_hcd(xhci); in xhci_cavium_reset_phy_quirk()
1894 static void handle_port_status(struct xhci_hcd *xhci, union xhci_trb *event) in handle_port_status() argument
1907 xhci_warn(xhci, in handle_port_status()
1911 max_ports = HCS_MAX_PORTS(xhci->hcs_params1); in handle_port_status()
1914 xhci_warn(xhci, "Port change event with invalid port ID %d\n", in handle_port_status()
1919 port = &xhci->hw_ports[port_id - 1]; in handle_port_status()
1921 xhci_warn(xhci, "Port change event, no port for port ID %u\n", in handle_port_status()
1928 if (port->rhub == &xhci->usb3_rhub && xhci->shared_hcd == NULL) { in handle_port_status()
1929 xhci_dbg(xhci, "ignore port event for removed USB3 hcd\n"); in handle_port_status()
1939 xhci_dbg(xhci, "Port change event, %d-%d, id %d, portsc: 0x%x\n", in handle_port_status()
1945 xhci_dbg(xhci, "resume root hub\n"); in handle_port_status()
1951 if (port->slot_id && xhci->devs[port->slot_id]) in handle_port_status()
1952 xhci->devs[port->slot_id]->flags |= VDEV_PORT_ERROR; in handle_port_status()
1956 xhci_dbg(xhci, "port resume event for port %d\n", port_id); in handle_port_status()
1958 cmd_reg = readl(&xhci->op_regs->command); in handle_port_status()
1960 xhci_warn(xhci, "xHC is not running.\n"); in handle_port_status()
1965 xhci_dbg(xhci, "remote wake SS port %d\n", port_id); in handle_port_status()
1971 xhci_test_and_clear_bit(xhci, port, PORT_PLC); in handle_port_status()
1973 xhci_set_link_state(xhci, port, XDEV_U0); in handle_port_status()
1980 xhci_dbg(xhci, "resume HS port %d\n", port_id); in handle_port_status()
2001 xhci_dbg(xhci, "resume SS port %d finished\n", port_id); in handle_port_status()
2010 if (port->slot_id && xhci->devs[port->slot_id]) in handle_port_status()
2011 xhci_ring_device(xhci, port->slot_id); in handle_port_status()
2013 xhci_test_and_clear_bit(xhci, port, PORT_PLC); in handle_port_status()
2022 * Check to see if xhci-hub.c is waiting on RExit to U0 transition (or in handle_port_status()
2034 xhci_test_and_clear_bit(xhci, port, PORT_PLC); in handle_port_status()
2035 if ((xhci->quirks & XHCI_RESET_PLL_ON_DISCONNECT) && in handle_port_status()
2037 xhci_cavium_reset_phy_quirk(xhci); in handle_port_status()
2050 * xHCI port-status-change events occur when the "or" of all the in handle_port_status()
2056 xhci_dbg(xhci, "%s: starting usb%d port polling.\n", in handle_port_status()
2059 spin_unlock(&xhci->lock); in handle_port_status()
2062 spin_lock(&xhci->lock); in handle_port_status()
2069 struct xhci_segment *trb_in_td(struct xhci_hcd *xhci, struct xhci_td *td, dma_addr_t suspect_dma, in trb_in_td() argument
2090 xhci_warn(xhci, in trb_in_td()
2126 static void xhci_clear_hub_tt_buffer(struct xhci_hcd *xhci, struct xhci_td *td, in xhci_clear_hub_tt_buffer() argument
2134 (td->urb->dev->tt->hub != xhci_to_hcd(xhci)->self.root_hub) && in xhci_clear_hub_tt_buffer()
2144 * Check if xhci internal endpoint state has gone to a "halt" state due to an
2173 int xhci_is_vendor_info_code(struct xhci_hcd *xhci, unsigned int trb_comp_code) in xhci_is_vendor_info_code() argument
2179 xhci_dbg(xhci, "Vendor defined info completion code %u\n", in xhci_is_vendor_info_code()
2181 xhci_dbg(xhci, "Treating code as success.\n"); in xhci_is_vendor_info_code()
2187 static int finish_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, in finish_td() argument
2193 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index); in finish_td()
2227 xhci_dbg(xhci, "Already resolving halted ep for 0x%llx\n", in finish_td()
2236 xhci_clear_hub_tt_buffer(xhci, td, ep); in finish_td()
2237 xhci_handle_halted_endpoint(xhci, ep, td, EP_HARD_RESET); in finish_td()
2241 * xhci internal endpoint state will go to a "halt" state for in finish_td()
2251 xhci_clear_hub_tt_buffer(xhci, td, ep); in finish_td()
2253 xhci_handle_halted_endpoint(xhci, ep, td, EP_HARD_RESET); in finish_td()
2263 inc_deq(xhci, ep_ring); in finish_td()
2265 return xhci_td_cleanup(xhci, td, ep_ring, td->status); in finish_td()
2269 static int sum_trb_lengths(struct xhci_hcd *xhci, struct xhci_ring *ring, in sum_trb_lengths() argument
2276 for (sum = 0; trb != stop_trb; next_trb(xhci, ring, &seg, &trb)) { in sum_trb_lengths()
2286 static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, in process_ctrl_td() argument
2296 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index); in process_ctrl_td()
2304 xhci_warn(xhci, "WARN: Success on ctrl %s TRB without IOC set?\n", in process_ctrl_td()
2318 xhci_warn(xhci, "WARN: Stopped Short Packet on ctrl setup or status TRB\n"); in process_ctrl_td()
2333 xhci_warn(xhci, "WARN: unexpected TRB Type %d\n", in process_ctrl_td()
2342 xhci_dbg(xhci, "TRB error %u, halted endpoint index = %u\n", in process_ctrl_td()
2366 xhci_dbg(xhci, "Waiting for status stage event\n"); in process_ctrl_td()
2375 return finish_td(xhci, ep, ep_ring, td, trb_comp_code); in process_ctrl_td()
2381 static int process_isoc_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, in process_isoc_td() argument
2406 /* Don't overwrite status if TD had an error, see xHCI 4.9.1 */ in process_isoc_td()
2465 frame->actual_length = sum_trb_lengths(xhci, ep->ring, ep_trb) + in process_isoc_td()
2475 xhci_dbg(xhci, "Error mid isoc TD, wait for final completion event\n"); in process_isoc_td()
2480 return finish_td(xhci, ep, ep_ring, td, trb_comp_code); in process_isoc_td()
2483 static int skip_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td, in skip_isoc_td() argument
2503 inc_deq(xhci, ep->ring); in skip_isoc_td()
2505 return xhci_td_cleanup(xhci, td, ep->ring, status); in skip_isoc_td()
2511 static int process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, in process_bulk_intr_td() argument
2519 slot_ctx = xhci_get_slot_ctx(xhci, ep->vdev->out_ctx); in process_bulk_intr_td()
2530 xhci_warn(xhci, "WARN Successful completion on short TX\n"); in process_bulk_intr_td()
2531 xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n", in process_bulk_intr_td()
2545 td->urb->actual_length = sum_trb_lengths(xhci, ep_ring, ep_trb); in process_bulk_intr_td()
2548 if (xhci->quirks & XHCI_NO_SOFT_RETRY || in process_bulk_intr_td()
2555 xhci_handle_halted_endpoint(xhci, ep, td, EP_SOFT_RESET); in process_bulk_intr_td()
2566 sum_trb_lengths(xhci, ep_ring, ep_trb) + in process_bulk_intr_td()
2570 xhci_warn(xhci, "bad transfer trb length %d in event trb\n", in process_bulk_intr_td()
2575 return finish_td(xhci, ep, ep_ring, td, trb_comp_code); in process_bulk_intr_td()
2578 /* Transfer events which don't point to a transfer TRB, see xhci 4.17.4 */
2579 static int handle_transferless_tx_event(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, in handle_transferless_tx_event() argument
2587 xhci_dbg(xhci, "Stream transaction error ep %u no id\n", ep->ep_index); in handle_transferless_tx_event()
2589 xhci_handle_halted_endpoint(xhci, ep, NULL, EP_HARD_RESET); in handle_transferless_tx_event()
2591 xhci_handle_halted_endpoint(xhci, ep, NULL, EP_SOFT_RESET); in handle_transferless_tx_event()
2598 xhci_err(xhci, "Transfer event %u for unknown stream ring slot %u ep %u\n", in handle_transferless_tx_event()
2610 static int handle_tx_event(struct xhci_hcd *xhci, in handle_tx_event() argument
2631 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); in handle_tx_event()
2633 xhci_err(xhci, "ERROR Invalid Transfer event\n"); in handle_tx_event()
2638 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); in handle_tx_event()
2641 xhci_err(xhci, in handle_tx_event()
2648 return handle_transferless_tx_event(xhci, ep, trb_comp_code); in handle_tx_event()
2658 xhci_dbg(xhci, "Successful completion on short TX for slot %u ep %u with last td short %d\n", in handle_tx_event()
2666 xhci_dbg(xhci, "Stopped on Transfer TRB for slot %u ep %u\n", in handle_tx_event()
2670 xhci_dbg(xhci, in handle_tx_event()
2675 xhci_dbg(xhci, in handle_tx_event()
2681 xhci_dbg(xhci, "Stalled endpoint for slot %u ep %u\n", slot_id, in handle_tx_event()
2686 xhci_dbg(xhci, "Split transaction error for slot %u ep %u\n", in handle_tx_event()
2691 xhci_dbg(xhci, "Transfer error for slot %u ep %u on endpoint\n", in handle_tx_event()
2696 xhci_dbg(xhci, "Babble error for slot %u ep %u on endpoint\n", in handle_tx_event()
2702 xhci_warn(xhci, in handle_tx_event()
2709 xhci_warn(xhci, in handle_tx_event()
2715 xhci_warn(xhci, in handle_tx_event()
2720 xhci_warn(xhci, in handle_tx_event()
2730 xhci_dbg(xhci, "Underrun event on slot %u ep %u\n", slot_id, ep_index); in handle_tx_event()
2735 xhci_dbg(xhci, "Overrun event on slot %u ep %u\n", slot_id, ep_index); in handle_tx_event()
2747 xhci_dbg(xhci, in handle_tx_event()
2753 xhci_dbg(xhci, in handle_tx_event()
2760 xhci_warn(xhci, in handle_tx_event()
2766 if (xhci_is_vendor_info_code(xhci, trb_comp_code)) { in handle_tx_event()
2770 xhci_warn(xhci, in handle_tx_event()
2779 * xhci 4.10.2 states isoc endpoints should continue in handle_tx_event()
2783 * xhci 4.9.1 states that if there are errors in mult-TRB in handle_tx_event()
2793 if (td && td->error_mid_td && !trb_in_td(xhci, td, ep_trb_dma, false)) { in handle_tx_event()
2794 xhci_dbg(xhci, "Missing TD completion event after mid TD error\n"); in handle_tx_event()
2797 inc_deq(xhci, ep_ring); in handle_tx_event()
2798 xhci_td_cleanup(xhci, td, ep_ring, td->status); in handle_tx_event()
2811 xhci_warn(xhci, "Event TRB for slot %u ep %u with no TDs queued\n", in handle_tx_event()
2824 ep_seg = trb_in_td(xhci, td, ep_trb_dma, false); in handle_tx_event()
2829 skip_isoc_td(xhci, td, ep, status); in handle_tx_event()
2833 xhci_dbg(xhci, "All TDs skipped for slot %u ep %u. Clear skip flag.\n", in handle_tx_event()
2856 if ((xhci->quirks & XHCI_SPURIOUS_SUCCESS) && in handle_tx_event()
2863 xhci_err(xhci, in handle_tx_event()
2866 trb_in_td(xhci, td, ep_trb_dma, true); in handle_tx_event()
2872 xhci_dbg(xhci, in handle_tx_event()
2908 process_ctrl_td(xhci, ep, ep_ring, td, ep_trb, event); in handle_tx_event()
2910 process_isoc_td(xhci, ep, ep_ring, td, ep_trb, event); in handle_tx_event()
2912 process_bulk_intr_td(xhci, ep, ep_ring, td, ep_trb, event); in handle_tx_event()
2917 xhci_handle_halted_endpoint(xhci, ep, td, EP_HARD_RESET); in handle_tx_event()
2922 xhci_err(xhci, "@%016llx %08x %08x %08x %08x\n", in handle_tx_event()
2935 * xhci->lock between event processing (e.g. to pass up port status changes).
2937 static int xhci_handle_event_trb(struct xhci_hcd *xhci, struct xhci_interrupter *ir, in xhci_handle_event_trb() argument
2954 handle_cmd_completion(xhci, &event->event_cmd); in xhci_handle_event_trb()
2957 handle_port_status(xhci, event); in xhci_handle_event_trb()
2960 handle_tx_event(xhci, ir, &event->trans_event); in xhci_handle_event_trb()
2963 handle_device_notification(xhci, event); in xhci_handle_event_trb()
2967 handle_vendor_event(xhci, event, trb_type); in xhci_handle_event_trb()
2969 xhci_warn(xhci, "ERROR unknown event type %d\n", trb_type); in xhci_handle_event_trb()
2974 if (xhci->xhc_state & XHCI_STATE_DYING) { in xhci_handle_event_trb()
2975 xhci_dbg(xhci, "xHCI host dying, returning from event handler.\n"); in xhci_handle_event_trb()
2987 static void xhci_update_erst_dequeue(struct xhci_hcd *xhci, in xhci_update_erst_dequeue() argument
2994 temp_64 = xhci_read_64(xhci, &ir->ir_set->erst_dequeue); in xhci_update_erst_dequeue()
2998 xhci_warn(xhci, "WARN something wrong with SW event ring dequeue ptr\n"); in xhci_update_erst_dequeue()
3013 xhci_write_64(xhci, temp_64, &ir->ir_set->erst_dequeue); in xhci_update_erst_dequeue()
3030 * and reaquire xhci->lock between event processing.
3032 static int xhci_handle_events(struct xhci_hcd *xhci, struct xhci_interrupter *ir) in xhci_handle_events() argument
3042 xhci_err(xhci, "ERROR interrupter event ring not ready\n"); in xhci_handle_events()
3046 if (xhci->xhc_state & XHCI_STATE_DYING || in xhci_handle_events()
3047 xhci->xhc_state & XHCI_STATE_HALTED) { in xhci_handle_events()
3048 xhci_dbg(xhci, "xHCI dying, ignoring interrupt. Shouldn't IRQs be disabled?\n"); in xhci_handle_events()
3051 temp = xhci_read_64(xhci, &ir->ir_set->erst_dequeue); in xhci_handle_events()
3052 xhci_write_64(xhci, temp | ERST_EHB, &ir->ir_set->erst_dequeue); in xhci_handle_events()
3058 err = xhci_handle_event_trb(xhci, ir, ir->event_ring->dequeue); in xhci_handle_events()
3065 xhci_update_erst_dequeue(xhci, ir, false); in xhci_handle_events()
3074 inc_deq(xhci, ir->event_ring); in xhci_handle_events()
3080 xhci_update_erst_dequeue(xhci, ir, true); in xhci_handle_events()
3086 * xHCI spec says we can get an interrupt, and if the HC has an error condition,
3092 struct xhci_hcd *xhci = hcd_to_xhci(hcd); in xhci_irq() local
3096 spin_lock(&xhci->lock); in xhci_irq()
3098 status = readl(&xhci->op_regs->status); in xhci_irq()
3100 xhci_hc_died(xhci); in xhci_irq()
3110 xhci_warn(xhci, "WARNING: Host Controller Error\n"); in xhci_irq()
3115 xhci_warn(xhci, "WARNING: Host System Error\n"); in xhci_irq()
3116 xhci_halt(xhci); in xhci_irq()
3126 writel(status, &xhci->op_regs->status); in xhci_irq()
3129 xhci_handle_events(xhci, xhci->interrupters[0]); in xhci_irq()
3131 spin_unlock(&xhci->lock); in xhci_irq()
3151 static void queue_trb(struct xhci_hcd *xhci, struct xhci_ring *ring, in queue_trb() argument
3167 inc_enq(xhci, ring, more_trbs_coming); in queue_trb()
3174 static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring, in prepare_ring() argument
3187 xhci_warn(xhci, "WARN urb submitted to disabled ep\n"); in prepare_ring()
3190 xhci_warn(xhci, "WARN waiting for error on ep to be cleared\n"); in prepare_ring()
3195 xhci_dbg(xhci, "WARN halted endpoint, queueing URB anyway.\n"); in prepare_ring()
3201 xhci_err(xhci, "ERROR unknown endpoint state for ep\n"); in prepare_ring()
3209 if (ep_ring != xhci->cmd_ring) { in prepare_ring()
3210 new_segs = xhci_ring_expansion_needed(xhci, ep_ring, num_trbs); in prepare_ring()
3212 xhci_err(xhci, "Do not support expand command ring\n"); in prepare_ring()
3217 xhci_dbg_trace(xhci, trace_xhci_dbg_ring_expansion, in prepare_ring()
3219 if (xhci_ring_expansion(xhci, ep_ring, new_segs, mem_flags)) { in prepare_ring()
3220 xhci_err(xhci, "Ring expansion failed\n"); in prepare_ring()
3229 if (!xhci_link_chain_quirk(xhci, ep_ring->type)) in prepare_ring()
3248 xhci_warn(xhci, "Ring is an endless link TRB loop\n"); in prepare_ring()
3254 xhci_warn(xhci, "Missing link TRB at end of ring segment\n"); in prepare_ring()
3261 static int prepare_transfer(struct xhci_hcd *xhci, in prepare_transfer() argument
3274 struct xhci_ep_ctx *ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); in prepare_transfer()
3276 ep_ring = xhci_triad_to_transfer_ring(xhci, xdev->slot_id, ep_index, in prepare_transfer()
3279 xhci_dbg(xhci, "Can't prepare ring for bad stream ID %u\n", in prepare_transfer()
3284 ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx), in prepare_transfer()
3368 static void giveback_first_trb(struct xhci_hcd *xhci, int slot_id, in giveback_first_trb() argument
3381 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, stream_id); in giveback_first_trb()
3402 "Driver uses different interval (%d microframe%s) than xHCI (%d microframe%s)\n", in check_interval()
3414 * xHCI uses normal TRBs for both bulk and interrupt. When the interrupt
3419 int xhci_queue_intr_tx(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_intr_tx() argument
3424 ep_ctx = xhci_get_ep_ctx(xhci, xhci->devs[slot_id]->out_ctx, ep_index); in xhci_queue_intr_tx()
3427 return xhci_queue_bulk_tx(xhci, mem_flags, urb, slot_id, ep_index); in xhci_queue_intr_tx()
3431 * For xHCI 1.0 host controllers, TD size is the number of max packet sized
3442 * For xHCI 0.96 and older, TD size field should be the remaining bytes
3450 static u32 xhci_td_remainder(struct xhci_hcd *xhci, int transferred, in xhci_td_remainder() argument
3456 /* MTK xHCI 0.96 contains some features from 1.0 */ in xhci_td_remainder()
3457 if (xhci->hci_version < 0x100 && !(xhci->quirks & XHCI_MTK_HOST)) in xhci_td_remainder()
3465 /* for MTK xHCI 0.96, TD size include this TRB, but not in 1.x */ in xhci_td_remainder()
3466 if ((xhci->quirks & XHCI_MTK_HOST) && (xhci->hci_version < 0x100)) in xhci_td_remainder()
3477 static int xhci_align_td(struct xhci_hcd *xhci, struct urb *urb, u32 enqd_len, in xhci_align_td() argument
3480 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_align_td()
3493 xhci_dbg(xhci, "Unaligned %d bytes, buff len %d\n", in xhci_align_td()
3499 xhci_dbg(xhci, "split align, new buff len %d\n", *trb_buff_len); in xhci_align_td()
3519 xhci_warn(xhci, "WARN Wrong bounce buffer write length: %zu != %d\n", in xhci_align_td()
3534 xhci_warn(xhci, "Failed mapping bounce buffer, not aligning\n"); in xhci_align_td()
3541 xhci_dbg(xhci, "Bounce align, new buff len %d\n", *trb_buff_len); in xhci_align_td()
3547 int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_bulk_tx() argument
3565 ring = xhci_urb_to_transfer_ring(xhci, urb); in xhci_queue_bulk_tx()
3582 ret = prepare_transfer(xhci, xhci->devs[slot_id], in xhci_queue_bulk_tx()
3631 if (xhci_align_td(xhci, urb, enqd_len, in xhci_queue_bulk_tx()
3659 remainder = xhci_td_remainder(xhci, enqd_len, trb_buff_len, in xhci_queue_bulk_tx()
3666 queue_trb(xhci, ring, more_trbs_coming | need_zero_pkt, in xhci_queue_bulk_tx()
3690 ret = prepare_transfer(xhci, xhci->devs[slot_id], in xhci_queue_bulk_tx()
3696 queue_trb(xhci, ring, 0, 0, 0, TRB_INTR_TARGET(0), field); in xhci_queue_bulk_tx()
3700 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id, in xhci_queue_bulk_tx()
3705 /* Caller must have locked xhci->lock */
3706 int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_ctrl_tx() argument
3719 ep_ring = xhci_urb_to_transfer_ring(xhci, urb); in xhci_queue_ctrl_tx()
3739 ret = prepare_transfer(xhci, xhci->devs[slot_id], in xhci_queue_ctrl_tx()
3764 /* xHCI 1.0/1.1 6.4.1.2.1: Transfer Type field */ in xhci_queue_ctrl_tx()
3765 if ((xhci->hci_version >= 0x100) || (xhci->quirks & XHCI_MTK_HOST)) { in xhci_queue_ctrl_tx()
3774 queue_trb(xhci, ep_ring, true, in xhci_queue_ctrl_tx()
3801 remainder = xhci_td_remainder(xhci, 0, in xhci_queue_ctrl_tx()
3810 queue_trb(xhci, ep_ring, true, in xhci_queue_ctrl_tx()
3827 queue_trb(xhci, ep_ring, false, in xhci_queue_ctrl_tx()
3834 giveback_first_trb(xhci, slot_id, ep_index, 0, in xhci_queue_ctrl_tx()
3845 * zero. Only xHCI 1.0 host controllers support this field.
3847 static unsigned int xhci_get_burst_count(struct xhci_hcd *xhci, in xhci_get_burst_count() argument
3852 if (xhci->hci_version < 0x100 || urb->dev->speed < USB_SPEED_SUPER) in xhci_get_burst_count()
3867 static unsigned int xhci_get_last_burst_packet_count(struct xhci_hcd *xhci, in xhci_get_last_burst_packet_count() argument
3873 if (xhci->hci_version < 0x100) in xhci_get_last_burst_packet_count()
3899 static int xhci_get_isoc_frame_id(struct xhci_hcd *xhci, in xhci_get_isoc_frame_id() argument
3919 ist = HCS_IST(xhci->hcs_params2) & 0x7; in xhci_get_isoc_frame_id()
3920 if (HCS_IST(xhci->hcs_params2) & (1 << 3)) in xhci_get_isoc_frame_id()
3936 current_frame_id = readl(&xhci->run_regs->microframe_index); in xhci_get_isoc_frame_id()
3969 xhci_warn(xhci, "Frame ID %d (reg %d, index %d) beyond range (%d, %d)\n", in xhci_get_isoc_frame_id()
3972 xhci_warn(xhci, "Ignore frame ID field, use SIA bit instead\n"); in xhci_get_isoc_frame_id()
3980 static bool trb_block_event_intr(struct xhci_hcd *xhci, int num_tds, int i, in trb_block_event_intr() argument
3983 if (xhci->hci_version < 0x100) in trb_block_event_intr()
3992 if (i && ir->isoc_bei_interval && xhci->quirks & XHCI_AVOID_BEI) in trb_block_event_intr()
3999 static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_isoc_tx() argument
4018 xep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_queue_isoc_tx()
4019 ep_ring = xhci->devs[slot_id]->eps[ep_index].ring; in xhci_queue_isoc_tx()
4020 ir = xhci->interrupters[0]; in xhci_queue_isoc_tx()
4024 xhci_dbg(xhci, "Isoc URB with zero packets?\n"); in xhci_queue_isoc_tx()
4049 burst_count = xhci_get_burst_count(xhci, urb, total_pkt_count); in xhci_queue_isoc_tx()
4050 last_burst_pkt_count = xhci_get_last_burst_packet_count(xhci, in xhci_queue_isoc_tx()
4055 ret = prepare_transfer(xhci, xhci->devs[slot_id], ep_index, in xhci_queue_isoc_tx()
4066 HCC_CFC(xhci->hcc_params)) { in xhci_queue_isoc_tx()
4067 frame_id = xhci_get_isoc_frame_id(xhci, urb, i); in xhci_queue_isoc_tx()
4081 /* xhci 1.1 with ETE uses TD_Size field for TBC, old is Rsvdz */ in xhci_queue_isoc_tx()
4107 if (trb_block_event_intr(xhci, num_tds, i, ir)) in xhci_queue_isoc_tx()
4116 remainder = xhci_td_remainder(xhci, running_total, in xhci_queue_isoc_tx()
4123 /* xhci 1.1 with ETE uses TD Size field for TBC */ in xhci_queue_isoc_tx()
4130 queue_trb(xhci, ep_ring, more_trbs_coming, in xhci_queue_isoc_tx()
4143 xhci_err(xhci, "ISOC TD length unmatch\n"); in xhci_queue_isoc_tx()
4150 if (HCC_CFC(xhci->hcc_params)) in xhci_queue_isoc_tx()
4153 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) { in xhci_queue_isoc_tx()
4154 if (xhci->quirks & XHCI_AMD_PLL_FIX) in xhci_queue_isoc_tx()
4157 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs++; in xhci_queue_isoc_tx()
4159 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id, in xhci_queue_isoc_tx()
4175 td_to_noop(xhci, ep_ring, &urb_priv->td[0], true); in xhci_queue_isoc_tx()
4188 * Update interval as xhci_queue_intr_tx does. Use xhci frame_index to
4192 int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_isoc_tx_prepare() argument
4204 xdev = xhci->devs[slot_id]; in xhci_queue_isoc_tx_prepare()
4205 xep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_queue_isoc_tx_prepare()
4207 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); in xhci_queue_isoc_tx_prepare()
4217 ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx), in xhci_queue_isoc_tx_prepare()
4229 if (HCC_CFC(xhci->hcc_params) && !list_empty(&ep_ring->td_list)) { in xhci_queue_isoc_tx_prepare()
4236 start_frame = readl(&xhci->run_regs->microframe_index); in xhci_queue_isoc_tx_prepare()
4242 ist = HCS_IST(xhci->hcs_params2) & 0x7; in xhci_queue_isoc_tx_prepare()
4243 if (HCS_IST(xhci->hcs_params2) & (1 << 3)) in xhci_queue_isoc_tx_prepare()
4263 return xhci_queue_isoc_tx(xhci, mem_flags, urb, slot_id, ep_index); in xhci_queue_isoc_tx_prepare()
4273 * Don't decrement xhci->cmd_ring_reserved_trbs after we've queued the TRB
4276 static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd, in queue_command() argument
4280 int reserved_trbs = xhci->cmd_ring_reserved_trbs; in queue_command()
4283 if ((xhci->xhc_state & XHCI_STATE_DYING) || in queue_command()
4284 (xhci->xhc_state & XHCI_STATE_HALTED)) { in queue_command()
4285 xhci_dbg(xhci, "xHCI dying or halted, can't queue_command\n"); in queue_command()
4292 ret = prepare_ring(xhci, xhci->cmd_ring, EP_STATE_RUNNING, in queue_command()
4295 xhci_err(xhci, "ERR: No room for command on command ring\n"); in queue_command()
4297 xhci_err(xhci, "ERR: Reserved TRB counting for " in queue_command()
4302 cmd->command_trb = xhci->cmd_ring->enqueue; in queue_command()
4305 if (list_empty(&xhci->cmd_list)) { in queue_command()
4306 xhci->current_cmd = cmd; in queue_command()
4307 xhci_mod_cmd_timer(xhci); in queue_command()
4310 list_add_tail(&cmd->cmd_list, &xhci->cmd_list); in queue_command()
4312 queue_trb(xhci, xhci->cmd_ring, false, field1, field2, field3, in queue_command()
4313 field4 | xhci->cmd_ring->cycle_state); in queue_command()
4318 int xhci_queue_slot_control(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_slot_control() argument
4321 return queue_command(xhci, cmd, 0, 0, 0, in xhci_queue_slot_control()
4326 int xhci_queue_address_device(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_address_device() argument
4329 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr), in xhci_queue_address_device()
4335 int xhci_queue_vendor_command(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_vendor_command() argument
4338 return queue_command(xhci, cmd, field1, field2, field3, field4, false); in xhci_queue_vendor_command()
4342 int xhci_queue_reset_device(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_reset_device() argument
4345 return queue_command(xhci, cmd, 0, 0, 0, in xhci_queue_reset_device()
4351 int xhci_queue_configure_endpoint(struct xhci_hcd *xhci, in xhci_queue_configure_endpoint() argument
4355 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr), in xhci_queue_configure_endpoint()
4362 int xhci_queue_evaluate_context(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_evaluate_context() argument
4365 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr), in xhci_queue_evaluate_context()
4375 int xhci_queue_stop_endpoint(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_stop_endpoint() argument
4383 return queue_command(xhci, cmd, 0, 0, 0, in xhci_queue_stop_endpoint()
4387 int xhci_queue_reset_ep(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_reset_ep() argument
4398 return queue_command(xhci, cmd, 0, 0, 0, in xhci_queue_reset_ep()