Lines Matching full:xhci
3 * xHCI host controller driver
18 #include "xhci.h"
19 #include "xhci-trace.h"
20 #include "xhci-debugfs.h"
29 static struct xhci_segment *xhci_segment_alloc(struct xhci_hcd *xhci, in xhci_segment_alloc() argument
36 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_segment_alloc()
42 seg->trbs = dma_pool_zalloc(xhci->segment_pool, flags, &dma); in xhci_segment_alloc()
52 dma_pool_free(xhci->segment_pool, seg->trbs, dma); in xhci_segment_alloc()
64 static void xhci_segment_free(struct xhci_hcd *xhci, struct xhci_segment *seg) in xhci_segment_free() argument
67 dma_pool_free(xhci->segment_pool, seg->trbs, seg->dma); in xhci_segment_free()
74 static void xhci_ring_segments_free(struct xhci_hcd *xhci, struct xhci_ring *ring) in xhci_ring_segments_free() argument
83 xhci_segment_free(xhci, seg); in xhci_ring_segments_free()
116 static void xhci_initialize_ring_segments(struct xhci_hcd *xhci, struct xhci_ring *ring) in xhci_initialize_ring_segments() argument
124 chain_links = xhci_link_chain_quirk(xhci, ring->type); in xhci_initialize_ring_segments()
136 static void xhci_link_rings(struct xhci_hcd *xhci, struct xhci_ring *src, struct xhci_ring *dst) in xhci_link_rings() argument
155 chain_links = xhci_link_chain_quirk(xhci, dst->type); in xhci_link_rings()
289 void xhci_ring_free(struct xhci_hcd *xhci, struct xhci_ring *ring) in xhci_ring_free() argument
299 xhci_ring_segments_free(xhci, ring); in xhci_ring_free()
330 static int xhci_alloc_segments_for_ring(struct xhci_hcd *xhci, struct xhci_ring *ring, gfp_t flags) in xhci_alloc_segments_for_ring() argument
335 prev = xhci_segment_alloc(xhci, ring->bounce_buf_len, num, flags); in xhci_alloc_segments_for_ring()
344 next = xhci_segment_alloc(xhci, ring->bounce_buf_len, num, flags); in xhci_alloc_segments_for_ring()
359 xhci_ring_segments_free(xhci, ring); in xhci_alloc_segments_for_ring()
370 struct xhci_ring *xhci_ring_alloc(struct xhci_hcd *xhci, unsigned int num_segs, in xhci_ring_alloc() argument
375 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_ring_alloc()
388 ret = xhci_alloc_segments_for_ring(xhci, ring, flags); in xhci_ring_alloc()
392 xhci_initialize_ring_segments(xhci, ring); in xhci_ring_alloc()
402 void xhci_free_endpoint_ring(struct xhci_hcd *xhci, in xhci_free_endpoint_ring() argument
406 xhci_ring_free(xhci, virt_dev->eps[ep_index].ring); in xhci_free_endpoint_ring()
414 int xhci_ring_expansion(struct xhci_hcd *xhci, struct xhci_ring *ring, in xhci_ring_expansion() argument
426 ret = xhci_alloc_segments_for_ring(xhci, &new_ring, flags); in xhci_ring_expansion()
430 xhci_initialize_ring_segments(xhci, &new_ring); in xhci_ring_expansion()
439 xhci_link_rings(xhci, &new_ring, ring); in xhci_ring_expansion()
441 xhci_dbg_trace(xhci, trace_xhci_dbg_ring_expansion, in xhci_ring_expansion()
448 xhci_ring_segments_free(xhci, &new_ring); in xhci_ring_expansion()
452 struct xhci_container_ctx *xhci_alloc_container_ctx(struct xhci_hcd *xhci, in xhci_alloc_container_ctx() argument
456 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_alloc_container_ctx()
466 ctx->size = xhci->hcc_params & HCC_64BYTE_CONTEXT ? 2048 : 1024; in xhci_alloc_container_ctx()
468 ctx->size += CTX_SIZE(xhci->hcc_params); in xhci_alloc_container_ctx()
470 ctx->bytes = dma_pool_zalloc(xhci->device_pool, flags, &ctx->dma); in xhci_alloc_container_ctx()
478 void xhci_free_container_ctx(struct xhci_hcd *xhci, in xhci_free_container_ctx() argument
483 dma_pool_free(xhci->device_pool, ctx->bytes, ctx->dma); in xhci_free_container_ctx()
487 struct xhci_container_ctx *xhci_alloc_port_bw_ctx(struct xhci_hcd *xhci, in xhci_alloc_port_bw_ctx() argument
491 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_alloc_port_bw_ctx()
499 ctx->bytes = dma_pool_zalloc(xhci->port_bw_pool, flags, &ctx->dma); in xhci_alloc_port_bw_ctx()
507 void xhci_free_port_bw_ctx(struct xhci_hcd *xhci, in xhci_free_port_bw_ctx() argument
512 dma_pool_free(xhci->port_bw_pool, ctx->bytes, ctx->dma); in xhci_free_port_bw_ctx()
525 struct xhci_slot_ctx *xhci_get_slot_ctx(struct xhci_hcd *xhci, in xhci_get_slot_ctx() argument
532 (ctx->bytes + CTX_SIZE(xhci->hcc_params)); in xhci_get_slot_ctx()
535 struct xhci_ep_ctx *xhci_get_ep_ctx(struct xhci_hcd *xhci, in xhci_get_ep_ctx() argument
545 (ctx->bytes + (ep_index * CTX_SIZE(xhci->hcc_params))); in xhci_get_ep_ctx()
551 static void xhci_free_stream_ctx(struct xhci_hcd *xhci, in xhci_free_stream_ctx() argument
555 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_free_stream_ctx()
561 dma_pool_free(xhci->medium_streams_pool, stream_ctx, dma); in xhci_free_stream_ctx()
563 dma_pool_free(xhci->small_streams_pool, stream_ctx, dma); in xhci_free_stream_ctx()
576 static struct xhci_stream_ctx *xhci_alloc_stream_ctx(struct xhci_hcd *xhci, in xhci_alloc_stream_ctx() argument
580 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_alloc_stream_ctx()
586 return dma_pool_zalloc(xhci->medium_streams_pool, mem_flags, dma); in xhci_alloc_stream_ctx()
588 return dma_pool_zalloc(xhci->small_streams_pool, mem_flags, dma); in xhci_alloc_stream_ctx()
610 struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci, in xhci_alloc_stream_info() argument
620 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_alloc_stream_info()
622 xhci_dbg(xhci, "Allocating %u streams and %u stream context array entries.\n", in xhci_alloc_stream_info()
624 if (xhci->cmd_ring_reserved_trbs == MAX_RSVD_CMD_TRBS) { in xhci_alloc_stream_info()
625 xhci_dbg(xhci, "Command ring has no reserved TRBs available\n"); in xhci_alloc_stream_info()
628 xhci->cmd_ring_reserved_trbs++; in xhci_alloc_stream_info()
646 stream_info->stream_ctx_array = xhci_alloc_stream_ctx(xhci, in xhci_alloc_stream_info()
654 xhci_alloc_command_with_ctx(xhci, true, mem_flags); in xhci_alloc_stream_info()
667 xhci_ring_alloc(xhci, 2, TYPE_STREAM, max_packet, mem_flags); in xhci_alloc_stream_info()
679 xhci_dbg(xhci, "Setting stream %d ring ptr to 0x%08llx\n", cur_stream, addr); in xhci_alloc_stream_info()
685 xhci_ring_free(xhci, cur_ring); in xhci_alloc_stream_info()
703 xhci_ring_free(xhci, cur_ring); in xhci_alloc_stream_info()
707 xhci_free_command(xhci, stream_info->free_streams_command); in xhci_alloc_stream_info()
709 xhci_free_stream_ctx(xhci, in xhci_alloc_stream_info()
718 xhci->cmd_ring_reserved_trbs--; in xhci_alloc_stream_info()
725 void xhci_setup_streams_ep_input_ctx(struct xhci_hcd *xhci, in xhci_setup_streams_ep_input_ctx() argument
735 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change, in xhci_setup_streams_ep_input_ctx()
762 void xhci_free_stream_info(struct xhci_hcd *xhci, in xhci_free_stream_info() argument
775 xhci_ring_free(xhci, cur_ring); in xhci_free_stream_info()
779 xhci_free_command(xhci, stream_info->free_streams_command); in xhci_free_stream_info()
780 xhci->cmd_ring_reserved_trbs--; in xhci_free_stream_info()
782 xhci_free_stream_ctx(xhci, in xhci_free_stream_info()
794 static void xhci_free_tt_info(struct xhci_hcd *xhci, in xhci_free_tt_info() argument
806 xhci_dbg(xhci, "Bad rhub port.\n"); in xhci_free_tt_info()
810 tt_list_head = &(xhci->rh_bw[virt_dev->rhub_port->hw_portnum].tts); in xhci_free_tt_info()
823 int xhci_alloc_tt_info(struct xhci_hcd *xhci, in xhci_alloc_tt_info() argument
831 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_alloc_tt_info()
847 &xhci->rh_bw[virt_dev->rhub_port->hw_portnum].tts); in xhci_alloc_tt_info()
858 xhci_free_tt_info(xhci, virt_dev, virt_dev->udev->slot_id); in xhci_alloc_tt_info()
864 * Should be called with xhci->lock held if there is any chance the TT lists
868 void xhci_free_virt_device(struct xhci_hcd *xhci, struct xhci_virt_device *dev, in xhci_free_virt_device() argument
880 xhci->dcbaa->dev_context_ptrs[slot_id] == cpu_to_le64(dev->out_ctx->dma)) in xhci_free_virt_device()
881 xhci->dcbaa->dev_context_ptrs[slot_id] = 0; in xhci_free_virt_device()
890 xhci_ring_free(xhci, dev->eps[i].ring); in xhci_free_virt_device()
892 xhci_free_stream_info(xhci, in xhci_free_virt_device()
904 xhci_dbg(xhci, "Slot %u endpoint %u not removed from BW list!\n", in xhci_free_virt_device()
909 xhci_free_tt_info(xhci, dev, slot_id); in xhci_free_virt_device()
911 xhci_update_tt_active_eps(xhci, dev, old_active_eps); in xhci_free_virt_device()
914 xhci_free_container_ctx(xhci, dev->in_ctx); in xhci_free_virt_device()
916 xhci_free_container_ctx(xhci, dev->out_ctx); in xhci_free_virt_device()
922 if (xhci->devs[slot_id] == dev) in xhci_free_virt_device()
923 xhci->devs[slot_id] = NULL; in xhci_free_virt_device()
933 static void xhci_free_virt_devices_depth_first(struct xhci_hcd *xhci, int slot_id) in xhci_free_virt_devices_depth_first() argument
940 vdev = xhci->devs[slot_id]; in xhci_free_virt_devices_depth_first()
945 xhci_dbg(xhci, "Bad rhub port.\n"); in xhci_free_virt_devices_depth_first()
949 tt_list_head = &(xhci->rh_bw[vdev->rhub_port->hw_portnum].tts); in xhci_free_virt_devices_depth_first()
954 for (i = 1; i < xhci->max_slots; i++) { in xhci_free_virt_devices_depth_first()
955 vdev = xhci->devs[i]; in xhci_free_virt_devices_depth_first()
958 xhci, i); in xhci_free_virt_devices_depth_first()
964 xhci_debugfs_remove_slot(xhci, slot_id); in xhci_free_virt_devices_depth_first()
965 xhci_free_virt_device(xhci, xhci->devs[slot_id], slot_id); in xhci_free_virt_devices_depth_first()
968 int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id, in xhci_alloc_virt_device() argument
975 if (slot_id == 0 || xhci->devs[slot_id]) { in xhci_alloc_virt_device()
976 xhci_warn(xhci, "Bad Slot ID %d\n", slot_id); in xhci_alloc_virt_device()
987 dev->out_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_DEVICE, flags); in xhci_alloc_virt_device()
991 xhci_dbg(xhci, "Slot %d output ctx = 0x%pad (dma)\n", slot_id, &dev->out_ctx->dma); in xhci_alloc_virt_device()
994 dev->in_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT, flags); in xhci_alloc_virt_device()
998 xhci_dbg(xhci, "Slot %d input ctx = 0x%pad (dma)\n", slot_id, &dev->in_ctx->dma); in xhci_alloc_virt_device()
1004 dev->eps[i].xhci = xhci; in xhci_alloc_virt_device()
1010 dev->eps[0].ring = xhci_ring_alloc(xhci, 2, TYPE_CTRL, 0, flags); in xhci_alloc_virt_device()
1017 xhci->dcbaa->dev_context_ptrs[slot_id] = cpu_to_le64(dev->out_ctx->dma); in xhci_alloc_virt_device()
1018 xhci_dbg(xhci, "Set slot id %d dcbaa entry %p to 0x%llx\n", in xhci_alloc_virt_device()
1020 &xhci->dcbaa->dev_context_ptrs[slot_id], in xhci_alloc_virt_device()
1021 le64_to_cpu(xhci->dcbaa->dev_context_ptrs[slot_id])); in xhci_alloc_virt_device()
1025 xhci->devs[slot_id] = dev; in xhci_alloc_virt_device()
1031 xhci_free_container_ctx(xhci, dev->in_ctx); in xhci_alloc_virt_device()
1033 xhci_free_container_ctx(xhci, dev->out_ctx); in xhci_alloc_virt_device()
1039 void xhci_copy_ep0_dequeue_into_input_ctx(struct xhci_hcd *xhci, in xhci_copy_ep0_dequeue_into_input_ctx() argument
1046 virt_dev = xhci->devs[udev->slot_id]; in xhci_copy_ep0_dequeue_into_input_ctx()
1047 ep0_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, 0); in xhci_copy_ep0_dequeue_into_input_ctx()
1062 * The xHCI roothub may have ports of differing speeds in any order in the port
1065 * The xHCI hardware wants to know the roothub port that the USB device
1071 static struct xhci_port *xhci_find_rhub_port(struct xhci_hcd *xhci, struct usb_device *udev) in xhci_find_rhub_port() argument
1078 hcd = xhci_get_usb3_hcd(xhci); in xhci_find_rhub_port()
1080 hcd = xhci->main_hcd; in xhci_find_rhub_port()
1090 /* Setup an xHCI virtual device for a Set Address command */
1091 int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *udev) in xhci_setup_addressable_virt_dev() argument
1098 dev = xhci->devs[udev->slot_id]; in xhci_setup_addressable_virt_dev()
1101 xhci_warn(xhci, "Slot ID %d is not assigned to this device\n", in xhci_setup_addressable_virt_dev()
1105 ep0_ctx = xhci_get_ep_ctx(xhci, dev->in_ctx, 0); in xhci_setup_addressable_virt_dev()
1106 slot_ctx = xhci_get_slot_ctx(xhci, dev->in_ctx); in xhci_setup_addressable_virt_dev()
1137 dev->rhub_port = xhci_find_rhub_port(xhci, udev); in xhci_setup_addressable_virt_dev()
1144 xhci_dbg(xhci, "Slot ID %d: HW portnum %d, hcd portnum %d\n", in xhci_setup_addressable_virt_dev()
1154 dev->bw_table = &xhci->rh_bw[dev->rhub_port->hw_portnum].bw_table; in xhci_setup_addressable_virt_dev()
1159 rh_bw = &xhci->rh_bw[dev->rhub_port->hw_portnum]; in xhci_setup_addressable_virt_dev()
1174 xhci_warn(xhci, "WARN: Didn't find a matching TT\n"); in xhci_setup_addressable_virt_dev()
1184 xhci_dbg(xhci, "udev->tt = %p\n", udev->tt); in xhci_setup_addressable_virt_dev()
1185 xhci_dbg(xhci, "udev->ttport = 0x%x\n", udev->ttport); in xhci_setup_addressable_virt_dev()
1278 * The polling interval is expressed in "microframes". If xHCI's Interval field
1340 static u32 xhci_get_endpoint_mult(struct xhci_hcd *xhci, in xhci_get_endpoint_mult() argument
1346 /* xHCI 1.1 with LEC set does not use mult field, except intel eUSB2 */ in xhci_get_endpoint_mult()
1347 lec = xhci->hci_version > 0x100 && (xhci->hcc_params2 & HCC2_LEC); in xhci_get_endpoint_mult()
1351 (!lec || xhci->quirks & XHCI_INTEL_HOST)) in xhci_get_endpoint_mult()
1407 int xhci_endpoint_init(struct xhci_hcd *xhci, in xhci_endpoint_init() argument
1427 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index); in xhci_endpoint_init()
1436 if (usb_endpoint_is_hs_isoc_double(udev, ep) && !(xhci->hcc_params2 & HCC2_EUSB2_DIC)) { in xhci_endpoint_init()
1437 dev_dbg(&udev->dev, "Double Isoc Bandwidth not supported by xhci\n"); in xhci_endpoint_init()
1445 * set it to max available. See xHCI 1.1 spec 4.14.1.1 for details. in xhci_endpoint_init()
1453 if ((xhci->quirks & XHCI_LIMIT_ENDPOINT_INTERVAL_9) && in xhci_endpoint_init()
1457 if ((xhci->quirks & XHCI_LIMIT_ENDPOINT_INTERVAL_7) && in xhci_endpoint_init()
1464 mult = xhci_get_endpoint_mult(xhci, udev, ep); in xhci_endpoint_init()
1483 /* xHCI 1.0 and 1.1 indicates that ctrl ep avg TRB Length should be 8 */ in xhci_endpoint_init()
1484 if (usb_endpoint_xfer_control(&ep->desc) && xhci->hci_version >= 0x100) in xhci_endpoint_init()
1489 xhci_ring_alloc(xhci, 2, ring_type, max_packet, mem_flags); in xhci_endpoint_init()
1513 void xhci_endpoint_zero(struct xhci_hcd *xhci, in xhci_endpoint_zero() argument
1521 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index); in xhci_endpoint_zero()
1542 void xhci_update_bw_info(struct xhci_hcd *xhci, in xhci_update_bw_info() argument
1567 ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, i); in xhci_update_bw_info()
1600 void xhci_endpoint_copy(struct xhci_hcd *xhci, in xhci_endpoint_copy() argument
1608 out_ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index); in xhci_endpoint_copy()
1609 in_ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, ep_index); in xhci_endpoint_copy()
1615 if (xhci->quirks & XHCI_MTK_HOST) { in xhci_endpoint_copy()
1626 void xhci_slot_copy(struct xhci_hcd *xhci, in xhci_slot_copy() argument
1633 in_slot_ctx = xhci_get_slot_ctx(xhci, in_ctx); in xhci_slot_copy()
1634 out_slot_ctx = xhci_get_slot_ctx(xhci, out_ctx); in xhci_slot_copy()
1643 static int scratchpad_alloc(struct xhci_hcd *xhci, gfp_t flags) in scratchpad_alloc() argument
1646 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in scratchpad_alloc()
1647 int num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2); in scratchpad_alloc()
1649 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in scratchpad_alloc()
1655 xhci->scratchpad = kzalloc_node(sizeof(*xhci->scratchpad), flags, in scratchpad_alloc()
1657 if (!xhci->scratchpad) in scratchpad_alloc()
1660 xhci->scratchpad->sp_array = dma_alloc_coherent(dev, in scratchpad_alloc()
1662 &xhci->scratchpad->sp_dma, flags); in scratchpad_alloc()
1663 if (!xhci->scratchpad->sp_array) in scratchpad_alloc()
1666 xhci->scratchpad->sp_buffers = kcalloc_node(num_sp, sizeof(void *), in scratchpad_alloc()
1668 if (!xhci->scratchpad->sp_buffers) in scratchpad_alloc()
1671 xhci->dcbaa->dev_context_ptrs[0] = cpu_to_le64(xhci->scratchpad->sp_dma); in scratchpad_alloc()
1674 void *buf = dma_alloc_coherent(dev, xhci->page_size, &dma, in scratchpad_alloc()
1679 xhci->scratchpad->sp_array[i] = dma; in scratchpad_alloc()
1680 xhci->scratchpad->sp_buffers[i] = buf; in scratchpad_alloc()
1687 dma_free_coherent(dev, xhci->page_size, in scratchpad_alloc()
1688 xhci->scratchpad->sp_buffers[i], in scratchpad_alloc()
1689 xhci->scratchpad->sp_array[i]); in scratchpad_alloc()
1691 kfree(xhci->scratchpad->sp_buffers); in scratchpad_alloc()
1695 xhci->scratchpad->sp_array, in scratchpad_alloc()
1696 xhci->scratchpad->sp_dma); in scratchpad_alloc()
1699 kfree(xhci->scratchpad); in scratchpad_alloc()
1700 xhci->scratchpad = NULL; in scratchpad_alloc()
1706 static void scratchpad_free(struct xhci_hcd *xhci) in scratchpad_free() argument
1710 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in scratchpad_free()
1712 if (!xhci->scratchpad) in scratchpad_free()
1715 num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2); in scratchpad_free()
1718 dma_free_coherent(dev, xhci->page_size, in scratchpad_free()
1719 xhci->scratchpad->sp_buffers[i], in scratchpad_free()
1720 xhci->scratchpad->sp_array[i]); in scratchpad_free()
1722 kfree(xhci->scratchpad->sp_buffers); in scratchpad_free()
1724 xhci->scratchpad->sp_array, in scratchpad_free()
1725 xhci->scratchpad->sp_dma); in scratchpad_free()
1726 kfree(xhci->scratchpad); in scratchpad_free()
1727 xhci->scratchpad = NULL; in scratchpad_free()
1730 struct xhci_command *xhci_alloc_command(struct xhci_hcd *xhci, in xhci_alloc_command() argument
1734 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_alloc_command()
1758 struct xhci_command *xhci_alloc_command_with_ctx(struct xhci_hcd *xhci, in xhci_alloc_command_with_ctx() argument
1763 command = xhci_alloc_command(xhci, allocate_completion, mem_flags); in xhci_alloc_command_with_ctx()
1767 command->in_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT, in xhci_alloc_command_with_ctx()
1782 void xhci_free_command(struct xhci_hcd *xhci, in xhci_free_command() argument
1785 xhci_free_container_ctx(xhci, in xhci_free_command()
1791 static int xhci_alloc_erst(struct xhci_hcd *xhci, in xhci_alloc_erst() argument
1802 erst->entries = dma_alloc_coherent(xhci_to_hcd(xhci)->self.sysdev, in xhci_alloc_erst()
1822 xhci_remove_interrupter(struct xhci_hcd *xhci, struct xhci_interrupter *ir) in xhci_remove_interrupter() argument
1839 xhci_update_erst_dequeue(xhci, ir, true); in xhci_remove_interrupter()
1844 xhci_free_interrupter(struct xhci_hcd *xhci, struct xhci_interrupter *ir) in xhci_free_interrupter() argument
1846 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_free_interrupter()
1861 xhci_ring_free(xhci, ir->event_ring); in xhci_free_interrupter()
1870 struct xhci_hcd *xhci = hcd_to_xhci(hcd); in xhci_remove_secondary_interrupter() local
1873 spin_lock_irq(&xhci->lock); in xhci_remove_secondary_interrupter()
1876 if (!ir || !ir->intr_num || ir->intr_num >= xhci->max_interrupters) { in xhci_remove_secondary_interrupter()
1877 xhci_dbg(xhci, "Invalid secondary interrupter, can't remove\n"); in xhci_remove_secondary_interrupter()
1878 spin_unlock_irq(&xhci->lock); in xhci_remove_secondary_interrupter()
1886 xhci_skip_sec_intr_events(xhci, ir->event_ring, ir); in xhci_remove_secondary_interrupter()
1889 xhci_remove_interrupter(xhci, ir); in xhci_remove_secondary_interrupter()
1890 xhci->interrupters[intr_num] = NULL; in xhci_remove_secondary_interrupter()
1892 spin_unlock_irq(&xhci->lock); in xhci_remove_secondary_interrupter()
1894 xhci_free_interrupter(xhci, ir); in xhci_remove_secondary_interrupter()
1898 void xhci_mem_cleanup(struct xhci_hcd *xhci) in xhci_mem_cleanup() argument
1900 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_mem_cleanup()
1903 cancel_delayed_work_sync(&xhci->cmd_timer); in xhci_mem_cleanup()
1905 for (i = 0; xhci->interrupters && i < xhci->max_interrupters; i++) { in xhci_mem_cleanup()
1906 if (xhci->interrupters[i]) { in xhci_mem_cleanup()
1907 xhci_remove_interrupter(xhci, xhci->interrupters[i]); in xhci_mem_cleanup()
1908 xhci_free_interrupter(xhci, xhci->interrupters[i]); in xhci_mem_cleanup()
1909 xhci->interrupters[i] = NULL; in xhci_mem_cleanup()
1912 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed interrupters"); in xhci_mem_cleanup()
1914 if (xhci->cmd_ring) in xhci_mem_cleanup()
1915 xhci_ring_free(xhci, xhci->cmd_ring); in xhci_mem_cleanup()
1916 xhci->cmd_ring = NULL; in xhci_mem_cleanup()
1917 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed command ring"); in xhci_mem_cleanup()
1918 xhci_cleanup_command_queue(xhci); in xhci_mem_cleanup()
1920 for (i = 0; i < xhci->max_ports && xhci->rh_bw; i++) { in xhci_mem_cleanup()
1921 struct xhci_interval_bw_table *bwt = &xhci->rh_bw[i].bw_table; in xhci_mem_cleanup()
1929 for (i = xhci->max_slots; i > 0; i--) in xhci_mem_cleanup()
1930 xhci_free_virt_devices_depth_first(xhci, i); in xhci_mem_cleanup()
1932 dma_pool_destroy(xhci->segment_pool); in xhci_mem_cleanup()
1933 xhci->segment_pool = NULL; in xhci_mem_cleanup()
1934 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed segment pool"); in xhci_mem_cleanup()
1936 dma_pool_destroy(xhci->device_pool); in xhci_mem_cleanup()
1937 xhci->device_pool = NULL; in xhci_mem_cleanup()
1938 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed device context pool"); in xhci_mem_cleanup()
1940 dma_pool_destroy(xhci->small_streams_pool); in xhci_mem_cleanup()
1941 xhci->small_streams_pool = NULL; in xhci_mem_cleanup()
1942 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_cleanup()
1945 dma_pool_destroy(xhci->port_bw_pool); in xhci_mem_cleanup()
1946 xhci->port_bw_pool = NULL; in xhci_mem_cleanup()
1947 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_cleanup()
1948 "Freed xhci port bw array pool"); in xhci_mem_cleanup()
1950 dma_pool_destroy(xhci->medium_streams_pool); in xhci_mem_cleanup()
1951 xhci->medium_streams_pool = NULL; in xhci_mem_cleanup()
1952 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_cleanup()
1955 if (xhci->dcbaa) in xhci_mem_cleanup()
1956 dma_free_coherent(dev, sizeof(*xhci->dcbaa), in xhci_mem_cleanup()
1957 xhci->dcbaa, xhci->dcbaa->dma); in xhci_mem_cleanup()
1958 xhci->dcbaa = NULL; in xhci_mem_cleanup()
1960 scratchpad_free(xhci); in xhci_mem_cleanup()
1962 if (!xhci->rh_bw) in xhci_mem_cleanup()
1965 for (i = 0; i < xhci->max_ports; i++) { in xhci_mem_cleanup()
1967 list_for_each_entry_safe(tt, n, &xhci->rh_bw[i].tts, tt_list) { in xhci_mem_cleanup()
1974 xhci->cmd_ring_reserved_trbs = 0; in xhci_mem_cleanup()
1975 xhci->usb2_rhub.num_ports = 0; in xhci_mem_cleanup()
1976 xhci->usb3_rhub.num_ports = 0; in xhci_mem_cleanup()
1977 xhci->num_active_eps = 0; in xhci_mem_cleanup()
1978 kfree(xhci->usb2_rhub.ports); in xhci_mem_cleanup()
1979 kfree(xhci->usb3_rhub.ports); in xhci_mem_cleanup()
1980 kfree(xhci->hw_ports); in xhci_mem_cleanup()
1981 kfree(xhci->rh_bw); in xhci_mem_cleanup()
1982 for (i = 0; i < xhci->num_port_caps; i++) in xhci_mem_cleanup()
1983 kfree(xhci->port_caps[i].psi); in xhci_mem_cleanup()
1984 kfree(xhci->port_caps); in xhci_mem_cleanup()
1985 kfree(xhci->interrupters); in xhci_mem_cleanup()
1986 xhci->num_port_caps = 0; in xhci_mem_cleanup()
1988 xhci->usb2_rhub.ports = NULL; in xhci_mem_cleanup()
1989 xhci->usb3_rhub.ports = NULL; in xhci_mem_cleanup()
1990 xhci->hw_ports = NULL; in xhci_mem_cleanup()
1991 xhci->rh_bw = NULL; in xhci_mem_cleanup()
1992 xhci->port_caps = NULL; in xhci_mem_cleanup()
1993 xhci->interrupters = NULL; in xhci_mem_cleanup()
1995 xhci->page_size = 0; in xhci_mem_cleanup()
1996 xhci->usb2_rhub.bus_state.bus_suspended = 0; in xhci_mem_cleanup()
1997 xhci->usb3_rhub.bus_state.bus_suspended = 0; in xhci_mem_cleanup()
2000 static void xhci_set_hc_event_deq(struct xhci_hcd *xhci, struct xhci_interrupter *ir) in xhci_set_hc_event_deq() argument
2007 xhci_warn(xhci, "WARN something wrong with SW event ring dequeue ptr.\n"); in xhci_set_hc_event_deq()
2012 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_set_hc_event_deq()
2014 xhci_write_64(xhci, deq & ERST_PTR_MASK, &ir->ir_set->erst_dequeue); in xhci_set_hc_event_deq()
2017 static void xhci_add_in_port(struct xhci_hcd *xhci, unsigned int num_ports, in xhci_add_in_port() argument
2024 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_add_in_port()
2032 rhub = &xhci->usb3_rhub; in xhci_add_in_port()
2043 * Some zhaoxin's xHCI controller that follow usb3.1 spec in xhci_add_in_port()
2046 if (xhci->quirks & XHCI_ZHAOXIN_HOST) { in xhci_add_in_port()
2052 rhub = &xhci->usb2_rhub; in xhci_add_in_port()
2054 xhci_warn(xhci, "Ignoring unknown port speed, Ext Cap %p, revision = 0x%x\n", in xhci_add_in_port()
2064 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_add_in_port()
2072 port_cap = &xhci->port_caps[xhci->num_port_caps++]; in xhci_add_in_port()
2073 if (xhci->num_port_caps > max_caps) in xhci_add_in_port()
2096 if (xhci->quirks & XHCI_ZHAOXIN_HOST && in xhci_add_in_port()
2101 xhci_dbg(xhci, "PSIV:%d PSIE:%d PLT:%d PFD:%d LP:%d PSIM:%d\n", in xhci_add_in_port()
2120 if ((xhci->hci_version >= 0x100) && (major_revision != 0x03) && in xhci_add_in_port()
2122 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_add_in_port()
2123 "xHCI 1.0: support USB2 hardware lpm"); in xhci_add_in_port()
2124 xhci->hw_lpm_support = 1; in xhci_add_in_port()
2129 struct xhci_port *hw_port = &xhci->hw_ports[i]; in xhci_add_in_port()
2132 xhci_warn(xhci, "Duplicate port entry, Ext Cap %p, port %u\n", addr, i); in xhci_add_in_port()
2133 xhci_warn(xhci, "Port was marked as USB %u, duplicated as USB %u\n", in xhci_add_in_port()
2152 static void xhci_create_rhub_port_array(struct xhci_hcd *xhci, in xhci_create_rhub_port_array() argument
2157 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_create_rhub_port_array()
2166 for (i = 0; i < xhci->max_ports; i++) { in xhci_create_rhub_port_array()
2167 if (xhci->hw_ports[i].rhub != rhub || in xhci_create_rhub_port_array()
2168 xhci->hw_ports[i].hcd_portnum == DUPLICATE_ENTRY) in xhci_create_rhub_port_array()
2170 xhci->hw_ports[i].hcd_portnum = port_index; in xhci_create_rhub_port_array()
2171 rhub->ports[port_index] = &xhci->hw_ports[i]; in xhci_create_rhub_port_array()
2185 static int xhci_setup_port_arrays(struct xhci_hcd *xhci, gfp_t flags) in xhci_setup_port_arrays() argument
2192 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_setup_port_arrays()
2194 xhci->hw_ports = kcalloc_node(xhci->max_ports, sizeof(*xhci->hw_ports), in xhci_setup_port_arrays()
2196 if (!xhci->hw_ports) in xhci_setup_port_arrays()
2199 for (i = 0; i < xhci->max_ports; i++) { in xhci_setup_port_arrays()
2200 xhci->hw_ports[i].port_reg = &xhci->op_regs->port_regs[i]; in xhci_setup_port_arrays()
2201 xhci->hw_ports[i].hw_portnum = i; in xhci_setup_port_arrays()
2203 init_completion(&xhci->hw_ports[i].rexit_done); in xhci_setup_port_arrays()
2204 init_completion(&xhci->hw_ports[i].u3exit_done); in xhci_setup_port_arrays()
2207 xhci->rh_bw = kcalloc_node(xhci->max_ports, sizeof(*xhci->rh_bw), flags, dev_to_node(dev)); in xhci_setup_port_arrays()
2208 if (!xhci->rh_bw) in xhci_setup_port_arrays()
2210 for (i = 0; i < xhci->max_ports; i++) { in xhci_setup_port_arrays()
2213 INIT_LIST_HEAD(&xhci->rh_bw[i].tts); in xhci_setup_port_arrays()
2214 bw_table = &xhci->rh_bw[i].bw_table; in xhci_setup_port_arrays()
2218 base = &xhci->cap_regs->hc_capbase; in xhci_setup_port_arrays()
2222 xhci_err(xhci, "No Extended Capability registers, unable to set up roothub\n"); in xhci_setup_port_arrays()
2234 xhci->port_caps = kcalloc_node(cap_count, sizeof(*xhci->port_caps), in xhci_setup_port_arrays()
2236 if (!xhci->port_caps) in xhci_setup_port_arrays()
2242 xhci_add_in_port(xhci, xhci->max_ports, base + offset, cap_count); in xhci_setup_port_arrays()
2243 if (xhci->usb2_rhub.num_ports + xhci->usb3_rhub.num_ports == xhci->max_ports) in xhci_setup_port_arrays()
2248 if (xhci->usb2_rhub.num_ports == 0 && xhci->usb3_rhub.num_ports == 0) { in xhci_setup_port_arrays()
2249 xhci_warn(xhci, "No ports on the roothubs?\n"); in xhci_setup_port_arrays()
2252 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_setup_port_arrays()
2254 xhci->usb2_rhub.num_ports, xhci->usb3_rhub.num_ports); in xhci_setup_port_arrays()
2259 if (xhci->usb3_rhub.num_ports > USB_SS_MAXPORTS) { in xhci_setup_port_arrays()
2260 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_setup_port_arrays()
2263 xhci->usb3_rhub.num_ports = USB_SS_MAXPORTS; in xhci_setup_port_arrays()
2265 if (xhci->usb2_rhub.num_ports > USB_MAXCHILDREN) { in xhci_setup_port_arrays()
2266 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_setup_port_arrays()
2269 xhci->usb2_rhub.num_ports = USB_MAXCHILDREN; in xhci_setup_port_arrays()
2272 if (!xhci->usb2_rhub.num_ports) in xhci_setup_port_arrays()
2273 xhci_info(xhci, "USB2 root hub has no ports\n"); in xhci_setup_port_arrays()
2275 if (!xhci->usb3_rhub.num_ports) in xhci_setup_port_arrays()
2276 xhci_info(xhci, "USB3 root hub has no ports\n"); in xhci_setup_port_arrays()
2278 xhci_create_rhub_port_array(xhci, &xhci->usb2_rhub, flags); in xhci_setup_port_arrays()
2279 xhci_create_rhub_port_array(xhci, &xhci->usb3_rhub, flags); in xhci_setup_port_arrays()
2285 xhci_alloc_interrupter(struct xhci_hcd *xhci, unsigned int segs, gfp_t flags) in xhci_alloc_interrupter() argument
2287 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_alloc_interrupter()
2295 max_segs = BIT(HCS_ERST_MAX(xhci->hcs_params2)); in xhci_alloc_interrupter()
2302 ir->event_ring = xhci_ring_alloc(xhci, segs, TYPE_EVENT, 0, flags); in xhci_alloc_interrupter()
2304 xhci_warn(xhci, "Failed to allocate interrupter event ring\n"); in xhci_alloc_interrupter()
2309 ret = xhci_alloc_erst(xhci, ir->event_ring, &ir->erst, flags); in xhci_alloc_interrupter()
2311 xhci_warn(xhci, "Failed to allocate interrupter erst\n"); in xhci_alloc_interrupter()
2312 xhci_ring_free(xhci, ir->event_ring); in xhci_alloc_interrupter()
2320 void xhci_add_interrupter(struct xhci_hcd *xhci, unsigned int intr_num) in xhci_add_interrupter() argument
2326 ir = xhci->interrupters[intr_num]; in xhci_add_interrupter()
2328 ir->ir_set = &xhci->run_regs->ir_set[intr_num]; in xhci_add_interrupter()
2336 erst_base = xhci_read_64(xhci, &ir->ir_set->erst_base); in xhci_add_interrupter()
2339 if (xhci->quirks & XHCI_WRITE_64_HI_LO) in xhci_add_interrupter()
2342 xhci_write_64(xhci, erst_base, &ir->ir_set->erst_base); in xhci_add_interrupter()
2345 xhci_set_hc_event_deq(xhci, ir); in xhci_add_interrupter()
2352 struct xhci_hcd *xhci = hcd_to_xhci(hcd); in xhci_create_secondary_interrupter() local
2357 if (!xhci->interrupters || xhci->max_interrupters <= 1 || in xhci_create_secondary_interrupter()
2358 intr_num >= xhci->max_interrupters) in xhci_create_secondary_interrupter()
2361 ir = xhci_alloc_interrupter(xhci, segs, GFP_KERNEL); in xhci_create_secondary_interrupter()
2365 spin_lock_irq(&xhci->lock); in xhci_create_secondary_interrupter()
2368 for (i = 1; i < xhci->max_interrupters; i++) { in xhci_create_secondary_interrupter()
2369 if (!xhci->interrupters[i]) { in xhci_create_secondary_interrupter()
2370 xhci->interrupters[i] = ir; in xhci_create_secondary_interrupter()
2371 xhci_add_interrupter(xhci, i); in xhci_create_secondary_interrupter()
2377 if (!xhci->interrupters[intr_num]) { in xhci_create_secondary_interrupter()
2378 xhci->interrupters[intr_num] = ir; in xhci_create_secondary_interrupter()
2379 xhci_add_interrupter(xhci, intr_num); in xhci_create_secondary_interrupter()
2383 spin_unlock_irq(&xhci->lock); in xhci_create_secondary_interrupter()
2386 xhci_warn(xhci, "Failed to add secondary interrupter, max interrupters %d\n", in xhci_create_secondary_interrupter()
2387 xhci->max_interrupters); in xhci_create_secondary_interrupter()
2388 xhci_free_interrupter(xhci, ir); in xhci_create_secondary_interrupter()
2394 xhci_dbg(xhci, "Add secondary interrupter %d, max interrupters %d\n", in xhci_create_secondary_interrupter()
2395 ir->intr_num, xhci->max_interrupters); in xhci_create_secondary_interrupter()
2401 int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) in xhci_mem_init() argument
2403 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_mem_init()
2407 * xHCI section 5.4.6 - Device Context array must be in xhci_mem_init()
2410 xhci->dcbaa = dma_alloc_coherent(dev, sizeof(*xhci->dcbaa), &dma, flags); in xhci_mem_init()
2411 if (!xhci->dcbaa) in xhci_mem_init()
2414 xhci->dcbaa->dma = dma; in xhci_mem_init()
2415 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2417 &xhci->dcbaa->dma, xhci->dcbaa); in xhci_mem_init()
2426 if (xhci->quirks & XHCI_TRB_OVERFETCH) in xhci_mem_init()
2428 xhci->segment_pool = dma_pool_create("xHCI ring segments", dev, in xhci_mem_init()
2429 TRB_SEGMENT_SIZE * 2, TRB_SEGMENT_SIZE * 2, xhci->page_size * 2); in xhci_mem_init()
2431 xhci->segment_pool = dma_pool_create("xHCI ring segments", dev, in xhci_mem_init()
2432 TRB_SEGMENT_SIZE, TRB_SEGMENT_SIZE, xhci->page_size); in xhci_mem_init()
2433 if (!xhci->segment_pool) in xhci_mem_init()
2437 xhci->device_pool = dma_pool_create("xHCI input/output contexts", dev, 2112, 64, in xhci_mem_init()
2438 xhci->page_size); in xhci_mem_init()
2439 if (!xhci->device_pool) in xhci_mem_init()
2446 xhci->small_streams_pool = dma_pool_create("xHCI 256 byte stream ctx arrays", in xhci_mem_init()
2448 if (!xhci->small_streams_pool) in xhci_mem_init()
2456 xhci->medium_streams_pool = dma_pool_create("xHCI 1KB stream ctx arrays", in xhci_mem_init()
2458 if (!xhci->medium_streams_pool) in xhci_mem_init()
2462 * refer to xhci rev1_2 protocol 5.3.3 max ports is 255. in xhci_mem_init()
2463 * refer to xhci rev1_2 protocol 6.4.3.14 port bandwidth buffer need in xhci_mem_init()
2466 xhci->port_bw_pool = dma_pool_create("xHCI 256 port bw ctx arrays", in xhci_mem_init()
2468 if (!xhci->port_bw_pool) in xhci_mem_init()
2472 xhci->cmd_ring = xhci_ring_alloc(xhci, 1, TYPE_COMMAND, 0, flags); in xhci_mem_init()
2473 if (!xhci->cmd_ring) in xhci_mem_init()
2476 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Allocated command ring at %p", xhci->cmd_ring); in xhci_mem_init()
2477 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "First segment DMA is 0x%pad", in xhci_mem_init()
2478 &xhci->cmd_ring->first_seg->dma); in xhci_mem_init()
2485 xhci->cmd_ring_reserved_trbs++; in xhci_mem_init()
2488 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Allocating primary event ring"); in xhci_mem_init()
2489 xhci->interrupters = kcalloc_node(xhci->max_interrupters, sizeof(*xhci->interrupters), in xhci_mem_init()
2491 if (!xhci->interrupters) in xhci_mem_init()
2494 xhci->interrupters[0] = xhci_alloc_interrupter(xhci, 0, flags); in xhci_mem_init()
2495 if (!xhci->interrupters[0]) in xhci_mem_init()
2498 if (scratchpad_alloc(xhci, flags)) in xhci_mem_init()
2501 if (xhci_setup_port_arrays(xhci, flags)) in xhci_mem_init()
2507 xhci_halt(xhci); in xhci_mem_init()
2508 xhci_reset(xhci, XHCI_RESET_SHORT_USEC); in xhci_mem_init()
2509 xhci_mem_cleanup(xhci); in xhci_mem_init()