1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * xHCI host controller driver 4 * 5 * Copyright (C) 2008 Intel Corp. 6 * 7 * Author: Sarah Sharp 8 * Some code borrowed from the Linux EHCI driver. 9 */ 10 11 #include <linux/jiffies.h> 12 #include <linux/pci.h> 13 #include <linux/iommu.h> 14 #include <linux/iopoll.h> 15 #include <linux/irq.h> 16 #include <linux/log2.h> 17 #include <linux/module.h> 18 #include <linux/moduleparam.h> 19 #include <linux/slab.h> 20 #include <linux/string_choices.h> 21 #include <linux/dmi.h> 22 #include <linux/dma-mapping.h> 23 24 #include "xhci.h" 25 #include "xhci-trace.h" 26 #include "xhci-debugfs.h" 27 #include "xhci-dbgcap.h" 28 29 #define DRIVER_AUTHOR "Sarah Sharp" 30 #define DRIVER_DESC "'eXtensible' Host Controller (xHC) Driver" 31 32 #define PORT_WAKE_BITS (PORT_WKOC_E | PORT_WKDISC_E | PORT_WKCONN_E) 33 34 /* Some 0.95 hardware can't handle the chain bit on a Link TRB being cleared */ 35 static int link_quirk; 36 module_param(link_quirk, int, S_IRUGO | S_IWUSR); 37 MODULE_PARM_DESC(link_quirk, "Don't clear the chain bit on a link TRB"); 38 39 static unsigned long long quirks; 40 module_param(quirks, ullong, S_IRUGO); 41 MODULE_PARM_DESC(quirks, "Bit flags for quirks to be enabled as default"); 42 43 static bool td_on_ring(struct xhci_td *td, struct xhci_ring *ring) 44 { 45 struct xhci_segment *seg; 46 47 if (!td || !td->start_seg) 48 return false; 49 50 xhci_for_each_ring_seg(ring->first_seg, seg) { 51 if (seg == td->start_seg) 52 return true; 53 } 54 55 return false; 56 } 57 58 /* 59 * xhci_handshake - spin reading hc until handshake completes or fails 60 * @ptr: address of hc register to be read 61 * @mask: bits to look at in result of read 62 * @done: value of those bits when handshake succeeds 63 * @usec: timeout in microseconds 64 * 65 * Returns negative errno, or zero on success 66 * 67 * Success happens when the "mask" bits have the specified value (hardware 68 * handshake done). There are two failure modes: "usec" have passed (major 69 * hardware flakeout), or the register reads as all-ones (hardware removed). 70 */ 71 int xhci_handshake(void __iomem *ptr, u32 mask, u32 done, u64 timeout_us) 72 { 73 u32 result; 74 int ret; 75 76 ret = readl_poll_timeout_atomic(ptr, result, 77 (result & mask) == done || 78 result == U32_MAX, 79 1, timeout_us); 80 if (result == U32_MAX) /* card removed */ 81 return -ENODEV; 82 83 return ret; 84 } 85 86 /* 87 * xhci_handshake_check_state - same as xhci_handshake but takes an additional 88 * exit_state parameter, and bails out with an error immediately when xhc_state 89 * has exit_state flag set. 90 */ 91 int xhci_handshake_check_state(struct xhci_hcd *xhci, void __iomem *ptr, 92 u32 mask, u32 done, int usec, unsigned int exit_state) 93 { 94 u32 result; 95 int ret; 96 97 ret = readl_poll_timeout_atomic(ptr, result, 98 (result & mask) == done || 99 result == U32_MAX || 100 xhci->xhc_state & exit_state, 101 1, usec); 102 103 if (result == U32_MAX || xhci->xhc_state & exit_state) 104 return -ENODEV; 105 106 return ret; 107 } 108 109 /* 110 * Disable interrupts and begin the xHCI halting process. 111 */ 112 void xhci_quiesce(struct xhci_hcd *xhci) 113 { 114 u32 halted; 115 u32 cmd; 116 u32 mask; 117 118 mask = ~(XHCI_IRQS); 119 halted = readl(&xhci->op_regs->status) & STS_HALT; 120 if (!halted) 121 mask &= ~CMD_RUN; 122 123 cmd = readl(&xhci->op_regs->command); 124 cmd &= mask; 125 writel(cmd, &xhci->op_regs->command); 126 } 127 128 /* 129 * Force HC into halt state. 130 * 131 * Disable any IRQs and clear the run/stop bit. 132 * HC will complete any current and actively pipelined transactions, and 133 * should halt within 16 ms of the run/stop bit being cleared. 134 * Read HC Halted bit in the status register to see when the HC is finished. 135 */ 136 int xhci_halt(struct xhci_hcd *xhci) 137 { 138 int ret; 139 140 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Halt the HC"); 141 xhci_quiesce(xhci); 142 143 ret = xhci_handshake(&xhci->op_regs->status, 144 STS_HALT, STS_HALT, XHCI_MAX_HALT_USEC); 145 if (ret) { 146 xhci_warn(xhci, "Host halt failed, %d\n", ret); 147 return ret; 148 } 149 150 xhci->xhc_state |= XHCI_STATE_HALTED; 151 xhci->cmd_ring_state = CMD_RING_STATE_STOPPED; 152 153 return ret; 154 } 155 156 /* 157 * Set the run bit and wait for the host to be running. 158 */ 159 int xhci_start(struct xhci_hcd *xhci) 160 { 161 u32 temp; 162 int ret; 163 164 temp = readl(&xhci->op_regs->command); 165 temp |= (CMD_RUN); 166 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Turn on HC, cmd = 0x%x.", 167 temp); 168 writel(temp, &xhci->op_regs->command); 169 170 /* 171 * Wait for the HCHalted Status bit to be 0 to indicate the host is 172 * running. 173 */ 174 ret = xhci_handshake(&xhci->op_regs->status, 175 STS_HALT, 0, XHCI_MAX_HALT_USEC); 176 if (ret == -ETIMEDOUT) 177 xhci_err(xhci, "Host took too long to start, " 178 "waited %u microseconds.\n", 179 XHCI_MAX_HALT_USEC); 180 if (!ret) { 181 /* clear state flags. Including dying, halted or removing */ 182 xhci->xhc_state = 0; 183 xhci->run_graceperiod = jiffies + msecs_to_jiffies(500); 184 } 185 186 return ret; 187 } 188 189 /* 190 * Reset a halted HC. 191 * 192 * This resets pipelines, timers, counters, state machines, etc. 193 * Transactions will be terminated immediately, and operational registers 194 * will be set to their defaults. 195 */ 196 int xhci_reset(struct xhci_hcd *xhci, u64 timeout_us) 197 { 198 u32 command; 199 u32 state; 200 int ret; 201 202 state = readl(&xhci->op_regs->status); 203 204 if (state == ~(u32)0) { 205 xhci_warn(xhci, "Host not accessible, reset failed.\n"); 206 return -ENODEV; 207 } 208 209 if ((state & STS_HALT) == 0) { 210 xhci_warn(xhci, "Host controller not halted, aborting reset.\n"); 211 return 0; 212 } 213 214 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Reset the HC"); 215 command = readl(&xhci->op_regs->command); 216 command |= CMD_RESET; 217 writel(command, &xhci->op_regs->command); 218 219 /* Existing Intel xHCI controllers require a delay of 1 mS, 220 * after setting the CMD_RESET bit, and before accessing any 221 * HC registers. This allows the HC to complete the 222 * reset operation and be ready for HC register access. 223 * Without this delay, the subsequent HC register access, 224 * may result in a system hang very rarely. 225 */ 226 if (xhci->quirks & XHCI_INTEL_HOST) 227 udelay(1000); 228 229 ret = xhci_handshake_check_state(xhci, &xhci->op_regs->command, 230 CMD_RESET, 0, timeout_us, XHCI_STATE_REMOVING); 231 if (ret) 232 return ret; 233 234 if (xhci->quirks & XHCI_ASMEDIA_MODIFY_FLOWCONTROL) 235 usb_asmedia_modifyflowcontrol(to_pci_dev(xhci_to_hcd(xhci)->self.controller)); 236 237 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 238 "Wait for controller to be ready for doorbell rings"); 239 /* 240 * xHCI cannot write to any doorbells or operational registers other 241 * than status until the "Controller Not Ready" flag is cleared. 242 */ 243 ret = xhci_handshake(&xhci->op_regs->status, STS_CNR, 0, timeout_us); 244 245 xhci->usb2_rhub.bus_state.port_c_suspend = 0; 246 xhci->usb2_rhub.bus_state.suspended_ports = 0; 247 xhci->usb2_rhub.bus_state.resuming_ports = 0; 248 xhci->usb3_rhub.bus_state.port_c_suspend = 0; 249 xhci->usb3_rhub.bus_state.suspended_ports = 0; 250 xhci->usb3_rhub.bus_state.resuming_ports = 0; 251 252 return ret; 253 } 254 255 static void xhci_zero_64b_regs(struct xhci_hcd *xhci) 256 { 257 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; 258 struct iommu_domain *domain; 259 int err, i; 260 u64 val; 261 u32 intrs; 262 263 /* 264 * Some Renesas controllers get into a weird state if they are 265 * reset while programmed with 64bit addresses (they will preserve 266 * the top half of the address in internal, non visible 267 * registers). You end up with half the address coming from the 268 * kernel, and the other half coming from the firmware. Also, 269 * changing the programming leads to extra accesses even if the 270 * controller is supposed to be halted. The controller ends up with 271 * a fatal fault, and is then ripe for being properly reset. 272 * 273 * Special care is taken to only apply this if the device is behind 274 * an iommu. Doing anything when there is no iommu is definitely 275 * unsafe... 276 */ 277 domain = iommu_get_domain_for_dev(dev); 278 if (!(xhci->quirks & XHCI_ZERO_64B_REGS) || !domain || 279 domain->type == IOMMU_DOMAIN_IDENTITY) 280 return; 281 282 xhci_info(xhci, "Zeroing 64bit base registers, expecting fault\n"); 283 284 /* Clear HSEIE so that faults do not get signaled */ 285 val = readl(&xhci->op_regs->command); 286 val &= ~CMD_HSEIE; 287 writel(val, &xhci->op_regs->command); 288 289 /* Clear HSE (aka FATAL) */ 290 val = readl(&xhci->op_regs->status); 291 val |= STS_FATAL; 292 writel(val, &xhci->op_regs->status); 293 294 /* Now zero the registers, and brace for impact */ 295 val = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr); 296 if (upper_32_bits(val)) 297 xhci_write_64(xhci, 0, &xhci->op_regs->dcbaa_ptr); 298 val = xhci_read_64(xhci, &xhci->op_regs->cmd_ring); 299 if (upper_32_bits(val)) 300 xhci_write_64(xhci, 0, &xhci->op_regs->cmd_ring); 301 302 intrs = min_t(u32, HCS_MAX_INTRS(xhci->hcs_params1), 303 ARRAY_SIZE(xhci->run_regs->ir_set)); 304 305 for (i = 0; i < intrs; i++) { 306 struct xhci_intr_reg __iomem *ir; 307 308 ir = &xhci->run_regs->ir_set[i]; 309 val = xhci_read_64(xhci, &ir->erst_base); 310 if (upper_32_bits(val)) 311 xhci_write_64(xhci, 0, &ir->erst_base); 312 val= xhci_read_64(xhci, &ir->erst_dequeue); 313 if (upper_32_bits(val)) 314 xhci_write_64(xhci, 0, &ir->erst_dequeue); 315 } 316 317 /* Wait for the fault to appear. It will be cleared on reset */ 318 err = xhci_handshake(&xhci->op_regs->status, 319 STS_FATAL, STS_FATAL, 320 XHCI_MAX_HALT_USEC); 321 if (!err) 322 xhci_info(xhci, "Fault detected\n"); 323 } 324 325 static int xhci_enable_interrupter(struct xhci_interrupter *ir) 326 { 327 u32 iman; 328 329 if (!ir || !ir->ir_set) 330 return -EINVAL; 331 332 iman = readl(&ir->ir_set->irq_pending); 333 writel(ER_IRQ_ENABLE(iman), &ir->ir_set->irq_pending); 334 335 return 0; 336 } 337 338 static int xhci_disable_interrupter(struct xhci_interrupter *ir) 339 { 340 u32 iman; 341 342 if (!ir || !ir->ir_set) 343 return -EINVAL; 344 345 iman = readl(&ir->ir_set->irq_pending); 346 writel(ER_IRQ_DISABLE(iman), &ir->ir_set->irq_pending); 347 348 return 0; 349 } 350 351 /* interrupt moderation interval imod_interval in nanoseconds */ 352 int xhci_set_interrupter_moderation(struct xhci_interrupter *ir, 353 u32 imod_interval) 354 { 355 u32 imod; 356 357 if (!ir || !ir->ir_set || imod_interval > U16_MAX * 250) 358 return -EINVAL; 359 360 imod = readl(&ir->ir_set->irq_control); 361 imod &= ~ER_IRQ_INTERVAL_MASK; 362 imod |= (imod_interval / 250) & ER_IRQ_INTERVAL_MASK; 363 writel(imod, &ir->ir_set->irq_control); 364 365 return 0; 366 } 367 368 static void compliance_mode_recovery(struct timer_list *t) 369 { 370 struct xhci_hcd *xhci; 371 struct usb_hcd *hcd; 372 struct xhci_hub *rhub; 373 u32 temp; 374 int i; 375 376 xhci = from_timer(xhci, t, comp_mode_recovery_timer); 377 rhub = &xhci->usb3_rhub; 378 hcd = rhub->hcd; 379 380 if (!hcd) 381 return; 382 383 for (i = 0; i < rhub->num_ports; i++) { 384 temp = readl(rhub->ports[i]->addr); 385 if ((temp & PORT_PLS_MASK) == USB_SS_PORT_LS_COMP_MOD) { 386 /* 387 * Compliance Mode Detected. Letting USB Core 388 * handle the Warm Reset 389 */ 390 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, 391 "Compliance mode detected->port %d", 392 i + 1); 393 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, 394 "Attempting compliance mode recovery"); 395 396 if (hcd->state == HC_STATE_SUSPENDED) 397 usb_hcd_resume_root_hub(hcd); 398 399 usb_hcd_poll_rh_status(hcd); 400 } 401 } 402 403 if (xhci->port_status_u0 != ((1 << rhub->num_ports) - 1)) 404 mod_timer(&xhci->comp_mode_recovery_timer, 405 jiffies + msecs_to_jiffies(COMP_MODE_RCVRY_MSECS)); 406 } 407 408 /* 409 * Quirk to work around issue generated by the SN65LVPE502CP USB3.0 re-driver 410 * that causes ports behind that hardware to enter compliance mode sometimes. 411 * The quirk creates a timer that polls every 2 seconds the link state of 412 * each host controller's port and recovers it by issuing a Warm reset 413 * if Compliance mode is detected, otherwise the port will become "dead" (no 414 * device connections or disconnections will be detected anymore). Becasue no 415 * status event is generated when entering compliance mode (per xhci spec), 416 * this quirk is needed on systems that have the failing hardware installed. 417 */ 418 static void compliance_mode_recovery_timer_init(struct xhci_hcd *xhci) 419 { 420 xhci->port_status_u0 = 0; 421 timer_setup(&xhci->comp_mode_recovery_timer, compliance_mode_recovery, 422 0); 423 xhci->comp_mode_recovery_timer.expires = jiffies + 424 msecs_to_jiffies(COMP_MODE_RCVRY_MSECS); 425 426 add_timer(&xhci->comp_mode_recovery_timer); 427 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, 428 "Compliance mode recovery timer initialized"); 429 } 430 431 /* 432 * This function identifies the systems that have installed the SN65LVPE502CP 433 * USB3.0 re-driver and that need the Compliance Mode Quirk. 434 * Systems: 435 * Vendor: Hewlett-Packard -> System Models: Z420, Z620 and Z820 436 */ 437 static bool xhci_compliance_mode_recovery_timer_quirk_check(void) 438 { 439 const char *dmi_product_name, *dmi_sys_vendor; 440 441 dmi_product_name = dmi_get_system_info(DMI_PRODUCT_NAME); 442 dmi_sys_vendor = dmi_get_system_info(DMI_SYS_VENDOR); 443 if (!dmi_product_name || !dmi_sys_vendor) 444 return false; 445 446 if (!(strstr(dmi_sys_vendor, "Hewlett-Packard"))) 447 return false; 448 449 if (strstr(dmi_product_name, "Z420") || 450 strstr(dmi_product_name, "Z620") || 451 strstr(dmi_product_name, "Z820") || 452 strstr(dmi_product_name, "Z1 Workstation")) 453 return true; 454 455 return false; 456 } 457 458 static int xhci_all_ports_seen_u0(struct xhci_hcd *xhci) 459 { 460 return (xhci->port_status_u0 == ((1 << xhci->usb3_rhub.num_ports) - 1)); 461 } 462 463 464 /* 465 * Initialize memory for HCD and xHC (one-time init). 466 * 467 * Program the PAGESIZE register, initialize the device context array, create 468 * device contexts (?), set up a command ring segment (or two?), create event 469 * ring (one for now). 470 */ 471 static int xhci_init(struct usb_hcd *hcd) 472 { 473 struct xhci_hcd *xhci = hcd_to_xhci(hcd); 474 int retval; 475 476 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "xhci_init"); 477 spin_lock_init(&xhci->lock); 478 479 retval = xhci_mem_init(xhci, GFP_KERNEL); 480 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Finished xhci_init"); 481 482 /* Initializing Compliance Mode Recovery Data If Needed */ 483 if (xhci_compliance_mode_recovery_timer_quirk_check()) { 484 xhci->quirks |= XHCI_COMP_MODE_QUIRK; 485 compliance_mode_recovery_timer_init(xhci); 486 } 487 488 return retval; 489 } 490 491 /*-------------------------------------------------------------------------*/ 492 493 static int xhci_run_finished(struct xhci_hcd *xhci) 494 { 495 struct xhci_interrupter *ir = xhci->interrupters[0]; 496 unsigned long flags; 497 u32 temp; 498 499 /* 500 * Enable interrupts before starting the host (xhci 4.2 and 5.5.2). 501 * Protect the short window before host is running with a lock 502 */ 503 spin_lock_irqsave(&xhci->lock, flags); 504 505 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Enable interrupts"); 506 temp = readl(&xhci->op_regs->command); 507 temp |= (CMD_EIE); 508 writel(temp, &xhci->op_regs->command); 509 510 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Enable primary interrupter"); 511 xhci_enable_interrupter(ir); 512 513 if (xhci_start(xhci)) { 514 xhci_halt(xhci); 515 spin_unlock_irqrestore(&xhci->lock, flags); 516 return -ENODEV; 517 } 518 519 xhci->cmd_ring_state = CMD_RING_STATE_RUNNING; 520 521 if (xhci->quirks & XHCI_NEC_HOST) 522 xhci_ring_cmd_db(xhci); 523 524 spin_unlock_irqrestore(&xhci->lock, flags); 525 526 return 0; 527 } 528 529 /* 530 * Start the HC after it was halted. 531 * 532 * This function is called by the USB core when the HC driver is added. 533 * Its opposite is xhci_stop(). 534 * 535 * xhci_init() must be called once before this function can be called. 536 * Reset the HC, enable device slot contexts, program DCBAAP, and 537 * set command ring pointer and event ring pointer. 538 * 539 * Setup MSI-X vectors and enable interrupts. 540 */ 541 int xhci_run(struct usb_hcd *hcd) 542 { 543 u64 temp_64; 544 int ret; 545 struct xhci_hcd *xhci = hcd_to_xhci(hcd); 546 struct xhci_interrupter *ir = xhci->interrupters[0]; 547 /* Start the xHCI host controller running only after the USB 2.0 roothub 548 * is setup. 549 */ 550 551 hcd->uses_new_polling = 1; 552 if (hcd->msi_enabled) 553 ir->ip_autoclear = true; 554 555 if (!usb_hcd_is_primary_hcd(hcd)) 556 return xhci_run_finished(xhci); 557 558 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "xhci_run"); 559 560 temp_64 = xhci_read_64(xhci, &ir->ir_set->erst_dequeue); 561 temp_64 &= ERST_PTR_MASK; 562 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 563 "ERST deq = 64'h%0lx", (long unsigned int) temp_64); 564 565 xhci_set_interrupter_moderation(ir, xhci->imod_interval); 566 567 if (xhci->quirks & XHCI_NEC_HOST) { 568 struct xhci_command *command; 569 570 command = xhci_alloc_command(xhci, false, GFP_KERNEL); 571 if (!command) 572 return -ENOMEM; 573 574 ret = xhci_queue_vendor_command(xhci, command, 0, 0, 0, 575 TRB_TYPE(TRB_NEC_GET_FW)); 576 if (ret) 577 xhci_free_command(xhci, command); 578 } 579 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 580 "Finished %s for main hcd", __func__); 581 582 xhci_create_dbc_dev(xhci); 583 584 xhci_debugfs_init(xhci); 585 586 if (xhci_has_one_roothub(xhci)) 587 return xhci_run_finished(xhci); 588 589 set_bit(HCD_FLAG_DEFER_RH_REGISTER, &hcd->flags); 590 591 return 0; 592 } 593 EXPORT_SYMBOL_GPL(xhci_run); 594 595 /* 596 * Stop xHCI driver. 597 * 598 * This function is called by the USB core when the HC driver is removed. 599 * Its opposite is xhci_run(). 600 * 601 * Disable device contexts, disable IRQs, and quiesce the HC. 602 * Reset the HC, finish any completed transactions, and cleanup memory. 603 */ 604 void xhci_stop(struct usb_hcd *hcd) 605 { 606 u32 temp; 607 struct xhci_hcd *xhci = hcd_to_xhci(hcd); 608 struct xhci_interrupter *ir = xhci->interrupters[0]; 609 610 mutex_lock(&xhci->mutex); 611 612 /* Only halt host and free memory after both hcds are removed */ 613 if (!usb_hcd_is_primary_hcd(hcd)) { 614 mutex_unlock(&xhci->mutex); 615 return; 616 } 617 618 xhci_remove_dbc_dev(xhci); 619 620 spin_lock_irq(&xhci->lock); 621 xhci->xhc_state |= XHCI_STATE_HALTED; 622 xhci->cmd_ring_state = CMD_RING_STATE_STOPPED; 623 xhci_halt(xhci); 624 xhci_reset(xhci, XHCI_RESET_SHORT_USEC); 625 spin_unlock_irq(&xhci->lock); 626 627 /* Deleting Compliance Mode Recovery Timer */ 628 if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) && 629 (!(xhci_all_ports_seen_u0(xhci)))) { 630 del_timer_sync(&xhci->comp_mode_recovery_timer); 631 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, 632 "%s: compliance mode recovery timer deleted", 633 __func__); 634 } 635 636 if (xhci->quirks & XHCI_AMD_PLL_FIX) 637 usb_amd_dev_put(); 638 639 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 640 "// Disabling event ring interrupts"); 641 temp = readl(&xhci->op_regs->status); 642 writel((temp & ~0x1fff) | STS_EINT, &xhci->op_regs->status); 643 xhci_disable_interrupter(ir); 644 645 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "cleaning up memory"); 646 xhci_mem_cleanup(xhci); 647 xhci_debugfs_exit(xhci); 648 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 649 "xhci_stop completed - status = %x", 650 readl(&xhci->op_regs->status)); 651 mutex_unlock(&xhci->mutex); 652 } 653 EXPORT_SYMBOL_GPL(xhci_stop); 654 655 /* 656 * Shutdown HC (not bus-specific) 657 * 658 * This is called when the machine is rebooting or halting. We assume that the 659 * machine will be powered off, and the HC's internal state will be reset. 660 * Don't bother to free memory. 661 * 662 * This will only ever be called with the main usb_hcd (the USB3 roothub). 663 */ 664 void xhci_shutdown(struct usb_hcd *hcd) 665 { 666 struct xhci_hcd *xhci = hcd_to_xhci(hcd); 667 668 if (xhci->quirks & XHCI_SPURIOUS_REBOOT) 669 usb_disable_xhci_ports(to_pci_dev(hcd->self.sysdev)); 670 671 /* Don't poll the roothubs after shutdown. */ 672 xhci_dbg(xhci, "%s: stopping usb%d port polling.\n", 673 __func__, hcd->self.busnum); 674 clear_bit(HCD_FLAG_POLL_RH, &hcd->flags); 675 del_timer_sync(&hcd->rh_timer); 676 677 if (xhci->shared_hcd) { 678 clear_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags); 679 del_timer_sync(&xhci->shared_hcd->rh_timer); 680 } 681 682 spin_lock_irq(&xhci->lock); 683 xhci_halt(xhci); 684 685 /* 686 * Workaround for spurious wakeps at shutdown with HSW, and for boot 687 * firmware delay in ADL-P PCH if port are left in U3 at shutdown 688 */ 689 if (xhci->quirks & XHCI_SPURIOUS_WAKEUP || 690 xhci->quirks & XHCI_RESET_TO_DEFAULT) 691 xhci_reset(xhci, XHCI_RESET_SHORT_USEC); 692 693 spin_unlock_irq(&xhci->lock); 694 695 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 696 "xhci_shutdown completed - status = %x", 697 readl(&xhci->op_regs->status)); 698 } 699 EXPORT_SYMBOL_GPL(xhci_shutdown); 700 701 #ifdef CONFIG_PM 702 static void xhci_save_registers(struct xhci_hcd *xhci) 703 { 704 struct xhci_interrupter *ir; 705 unsigned int i; 706 707 xhci->s3.command = readl(&xhci->op_regs->command); 708 xhci->s3.dev_nt = readl(&xhci->op_regs->dev_notification); 709 xhci->s3.dcbaa_ptr = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr); 710 xhci->s3.config_reg = readl(&xhci->op_regs->config_reg); 711 712 /* save both primary and all secondary interrupters */ 713 /* fixme, shold we lock to prevent race with remove secondary interrupter? */ 714 for (i = 0; i < xhci->max_interrupters; i++) { 715 ir = xhci->interrupters[i]; 716 if (!ir) 717 continue; 718 719 ir->s3_erst_size = readl(&ir->ir_set->erst_size); 720 ir->s3_erst_base = xhci_read_64(xhci, &ir->ir_set->erst_base); 721 ir->s3_erst_dequeue = xhci_read_64(xhci, &ir->ir_set->erst_dequeue); 722 ir->s3_irq_pending = readl(&ir->ir_set->irq_pending); 723 ir->s3_irq_control = readl(&ir->ir_set->irq_control); 724 } 725 } 726 727 static void xhci_restore_registers(struct xhci_hcd *xhci) 728 { 729 struct xhci_interrupter *ir; 730 unsigned int i; 731 732 writel(xhci->s3.command, &xhci->op_regs->command); 733 writel(xhci->s3.dev_nt, &xhci->op_regs->dev_notification); 734 xhci_write_64(xhci, xhci->s3.dcbaa_ptr, &xhci->op_regs->dcbaa_ptr); 735 writel(xhci->s3.config_reg, &xhci->op_regs->config_reg); 736 737 /* FIXME should we lock to protect against freeing of interrupters */ 738 for (i = 0; i < xhci->max_interrupters; i++) { 739 ir = xhci->interrupters[i]; 740 if (!ir) 741 continue; 742 743 writel(ir->s3_erst_size, &ir->ir_set->erst_size); 744 xhci_write_64(xhci, ir->s3_erst_base, &ir->ir_set->erst_base); 745 xhci_write_64(xhci, ir->s3_erst_dequeue, &ir->ir_set->erst_dequeue); 746 writel(ir->s3_irq_pending, &ir->ir_set->irq_pending); 747 writel(ir->s3_irq_control, &ir->ir_set->irq_control); 748 } 749 } 750 751 static void xhci_set_cmd_ring_deq(struct xhci_hcd *xhci) 752 { 753 u64 val_64; 754 755 /* step 2: initialize command ring buffer */ 756 val_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring); 757 val_64 = (val_64 & (u64) CMD_RING_RSVD_BITS) | 758 (xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg, 759 xhci->cmd_ring->dequeue) & 760 (u64) ~CMD_RING_RSVD_BITS) | 761 xhci->cmd_ring->cycle_state; 762 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 763 "// Setting command ring address to 0x%llx", 764 (long unsigned long) val_64); 765 xhci_write_64(xhci, val_64, &xhci->op_regs->cmd_ring); 766 } 767 768 /* 769 * The whole command ring must be cleared to zero when we suspend the host. 770 * 771 * The host doesn't save the command ring pointer in the suspend well, so we 772 * need to re-program it on resume. Unfortunately, the pointer must be 64-byte 773 * aligned, because of the reserved bits in the command ring dequeue pointer 774 * register. Therefore, we can't just set the dequeue pointer back in the 775 * middle of the ring (TRBs are 16-byte aligned). 776 */ 777 static void xhci_clear_command_ring(struct xhci_hcd *xhci) 778 { 779 struct xhci_ring *ring; 780 struct xhci_segment *seg; 781 782 ring = xhci->cmd_ring; 783 xhci_for_each_ring_seg(ring->first_seg, seg) 784 memset(seg->trbs, 0, sizeof(union xhci_trb) * (TRBS_PER_SEGMENT - 1)); 785 786 xhci_initialize_ring_info(ring); 787 /* 788 * Reset the hardware dequeue pointer. 789 * Yes, this will need to be re-written after resume, but we're paranoid 790 * and want to make sure the hardware doesn't access bogus memory 791 * because, say, the BIOS or an SMI started the host without changing 792 * the command ring pointers. 793 */ 794 xhci_set_cmd_ring_deq(xhci); 795 } 796 797 /* 798 * Disable port wake bits if do_wakeup is not set. 799 * 800 * Also clear a possible internal port wake state left hanging for ports that 801 * detected termination but never successfully enumerated (trained to 0U). 802 * Internal wake causes immediate xHCI wake after suspend. PORT_CSC write done 803 * at enumeration clears this wake, force one here as well for unconnected ports 804 */ 805 806 static void xhci_disable_hub_port_wake(struct xhci_hcd *xhci, 807 struct xhci_hub *rhub, 808 bool do_wakeup) 809 { 810 unsigned long flags; 811 u32 t1, t2, portsc; 812 int i; 813 814 spin_lock_irqsave(&xhci->lock, flags); 815 816 for (i = 0; i < rhub->num_ports; i++) { 817 portsc = readl(rhub->ports[i]->addr); 818 t1 = xhci_port_state_to_neutral(portsc); 819 t2 = t1; 820 821 /* clear wake bits if do_wake is not set */ 822 if (!do_wakeup) 823 t2 &= ~PORT_WAKE_BITS; 824 825 /* Don't touch csc bit if connected or connect change is set */ 826 if (!(portsc & (PORT_CSC | PORT_CONNECT))) 827 t2 |= PORT_CSC; 828 829 if (t1 != t2) { 830 writel(t2, rhub->ports[i]->addr); 831 xhci_dbg(xhci, "config port %d-%d wake bits, portsc: 0x%x, write: 0x%x\n", 832 rhub->hcd->self.busnum, i + 1, portsc, t2); 833 } 834 } 835 spin_unlock_irqrestore(&xhci->lock, flags); 836 } 837 838 static bool xhci_pending_portevent(struct xhci_hcd *xhci) 839 { 840 struct xhci_port **ports; 841 int port_index; 842 u32 status; 843 u32 portsc; 844 845 status = readl(&xhci->op_regs->status); 846 if (status & STS_EINT) 847 return true; 848 /* 849 * Checking STS_EINT is not enough as there is a lag between a change 850 * bit being set and the Port Status Change Event that it generated 851 * being written to the Event Ring. See note in xhci 1.1 section 4.19.2. 852 */ 853 854 port_index = xhci->usb2_rhub.num_ports; 855 ports = xhci->usb2_rhub.ports; 856 while (port_index--) { 857 portsc = readl(ports[port_index]->addr); 858 if (portsc & PORT_CHANGE_MASK || 859 (portsc & PORT_PLS_MASK) == XDEV_RESUME) 860 return true; 861 } 862 port_index = xhci->usb3_rhub.num_ports; 863 ports = xhci->usb3_rhub.ports; 864 while (port_index--) { 865 portsc = readl(ports[port_index]->addr); 866 if (portsc & (PORT_CHANGE_MASK | PORT_CAS) || 867 (portsc & PORT_PLS_MASK) == XDEV_RESUME) 868 return true; 869 } 870 return false; 871 } 872 873 /* 874 * Stop HC (not bus-specific) 875 * 876 * This is called when the machine transition into S3/S4 mode. 877 * 878 */ 879 int xhci_suspend(struct xhci_hcd *xhci, bool do_wakeup) 880 { 881 int rc = 0; 882 unsigned int delay = XHCI_MAX_HALT_USEC * 2; 883 struct usb_hcd *hcd = xhci_to_hcd(xhci); 884 u32 command; 885 u32 res; 886 887 if (!hcd->state) 888 return 0; 889 890 if (hcd->state != HC_STATE_SUSPENDED || 891 (xhci->shared_hcd && xhci->shared_hcd->state != HC_STATE_SUSPENDED)) 892 return -EINVAL; 893 894 /* Clear root port wake on bits if wakeup not allowed. */ 895 xhci_disable_hub_port_wake(xhci, &xhci->usb3_rhub, do_wakeup); 896 xhci_disable_hub_port_wake(xhci, &xhci->usb2_rhub, do_wakeup); 897 898 if (!HCD_HW_ACCESSIBLE(hcd)) 899 return 0; 900 901 xhci_dbc_suspend(xhci); 902 903 /* Don't poll the roothubs on bus suspend. */ 904 xhci_dbg(xhci, "%s: stopping usb%d port polling.\n", 905 __func__, hcd->self.busnum); 906 clear_bit(HCD_FLAG_POLL_RH, &hcd->flags); 907 del_timer_sync(&hcd->rh_timer); 908 if (xhci->shared_hcd) { 909 clear_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags); 910 del_timer_sync(&xhci->shared_hcd->rh_timer); 911 } 912 913 if (xhci->quirks & XHCI_SUSPEND_DELAY) 914 usleep_range(1000, 1500); 915 916 spin_lock_irq(&xhci->lock); 917 clear_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags); 918 if (xhci->shared_hcd) 919 clear_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags); 920 /* step 1: stop endpoint */ 921 /* skipped assuming that port suspend has done */ 922 923 /* step 2: clear Run/Stop bit */ 924 command = readl(&xhci->op_regs->command); 925 command &= ~CMD_RUN; 926 writel(command, &xhci->op_regs->command); 927 928 /* Some chips from Fresco Logic need an extraordinary delay */ 929 delay *= (xhci->quirks & XHCI_SLOW_SUSPEND) ? 10 : 1; 930 931 if (xhci_handshake(&xhci->op_regs->status, 932 STS_HALT, STS_HALT, delay)) { 933 xhci_warn(xhci, "WARN: xHC CMD_RUN timeout\n"); 934 spin_unlock_irq(&xhci->lock); 935 return -ETIMEDOUT; 936 } 937 xhci_clear_command_ring(xhci); 938 939 /* step 3: save registers */ 940 xhci_save_registers(xhci); 941 942 /* step 4: set CSS flag */ 943 command = readl(&xhci->op_regs->command); 944 command |= CMD_CSS; 945 writel(command, &xhci->op_regs->command); 946 xhci->broken_suspend = 0; 947 if (xhci_handshake(&xhci->op_regs->status, 948 STS_SAVE, 0, 20 * 1000)) { 949 /* 950 * AMD SNPS xHC 3.0 occasionally does not clear the 951 * SSS bit of USBSTS and when driver tries to poll 952 * to see if the xHC clears BIT(8) which never happens 953 * and driver assumes that controller is not responding 954 * and times out. To workaround this, its good to check 955 * if SRE and HCE bits are not set (as per xhci 956 * Section 5.4.2) and bypass the timeout. 957 */ 958 res = readl(&xhci->op_regs->status); 959 if ((xhci->quirks & XHCI_SNPS_BROKEN_SUSPEND) && 960 (((res & STS_SRE) == 0) && 961 ((res & STS_HCE) == 0))) { 962 xhci->broken_suspend = 1; 963 } else { 964 xhci_warn(xhci, "WARN: xHC save state timeout\n"); 965 spin_unlock_irq(&xhci->lock); 966 return -ETIMEDOUT; 967 } 968 } 969 spin_unlock_irq(&xhci->lock); 970 971 /* 972 * Deleting Compliance Mode Recovery Timer because the xHCI Host 973 * is about to be suspended. 974 */ 975 if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) && 976 (!(xhci_all_ports_seen_u0(xhci)))) { 977 del_timer_sync(&xhci->comp_mode_recovery_timer); 978 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, 979 "%s: compliance mode recovery timer deleted", 980 __func__); 981 } 982 983 return rc; 984 } 985 EXPORT_SYMBOL_GPL(xhci_suspend); 986 987 /* 988 * start xHC (not bus-specific) 989 * 990 * This is called when the machine transition from S3/S4 mode. 991 * 992 */ 993 int xhci_resume(struct xhci_hcd *xhci, pm_message_t msg) 994 { 995 bool hibernated = (msg.event == PM_EVENT_RESTORE); 996 u32 command, temp = 0; 997 struct usb_hcd *hcd = xhci_to_hcd(xhci); 998 int retval = 0; 999 bool comp_timer_running = false; 1000 bool pending_portevent = false; 1001 bool suspended_usb3_devs = false; 1002 bool reinit_xhc = false; 1003 1004 if (!hcd->state) 1005 return 0; 1006 1007 /* Wait a bit if either of the roothubs need to settle from the 1008 * transition into bus suspend. 1009 */ 1010 1011 if (time_before(jiffies, xhci->usb2_rhub.bus_state.next_statechange) || 1012 time_before(jiffies, xhci->usb3_rhub.bus_state.next_statechange)) 1013 msleep(100); 1014 1015 set_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags); 1016 if (xhci->shared_hcd) 1017 set_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags); 1018 1019 spin_lock_irq(&xhci->lock); 1020 1021 if (hibernated || xhci->quirks & XHCI_RESET_ON_RESUME || xhci->broken_suspend) 1022 reinit_xhc = true; 1023 1024 if (!reinit_xhc) { 1025 /* 1026 * Some controllers might lose power during suspend, so wait 1027 * for controller not ready bit to clear, just as in xHC init. 1028 */ 1029 retval = xhci_handshake(&xhci->op_regs->status, 1030 STS_CNR, 0, 10 * 1000 * 1000); 1031 if (retval) { 1032 xhci_warn(xhci, "Controller not ready at resume %d\n", 1033 retval); 1034 spin_unlock_irq(&xhci->lock); 1035 return retval; 1036 } 1037 /* step 1: restore register */ 1038 xhci_restore_registers(xhci); 1039 /* step 2: initialize command ring buffer */ 1040 xhci_set_cmd_ring_deq(xhci); 1041 /* step 3: restore state and start state*/ 1042 /* step 3: set CRS flag */ 1043 command = readl(&xhci->op_regs->command); 1044 command |= CMD_CRS; 1045 writel(command, &xhci->op_regs->command); 1046 /* 1047 * Some controllers take up to 55+ ms to complete the controller 1048 * restore so setting the timeout to 100ms. Xhci specification 1049 * doesn't mention any timeout value. 1050 */ 1051 if (xhci_handshake(&xhci->op_regs->status, 1052 STS_RESTORE, 0, 100 * 1000)) { 1053 xhci_warn(xhci, "WARN: xHC restore state timeout\n"); 1054 spin_unlock_irq(&xhci->lock); 1055 return -ETIMEDOUT; 1056 } 1057 } 1058 1059 temp = readl(&xhci->op_regs->status); 1060 1061 /* re-initialize the HC on Restore Error, or Host Controller Error */ 1062 if ((temp & (STS_SRE | STS_HCE)) && 1063 !(xhci->xhc_state & XHCI_STATE_REMOVING)) { 1064 reinit_xhc = true; 1065 if (!xhci->broken_suspend) 1066 xhci_warn(xhci, "xHC error in resume, USBSTS 0x%x, Reinit\n", temp); 1067 } 1068 1069 if (reinit_xhc) { 1070 if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) && 1071 !(xhci_all_ports_seen_u0(xhci))) { 1072 del_timer_sync(&xhci->comp_mode_recovery_timer); 1073 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, 1074 "Compliance Mode Recovery Timer deleted!"); 1075 } 1076 1077 /* Let the USB core know _both_ roothubs lost power. */ 1078 usb_root_hub_lost_power(xhci->main_hcd->self.root_hub); 1079 if (xhci->shared_hcd) 1080 usb_root_hub_lost_power(xhci->shared_hcd->self.root_hub); 1081 1082 xhci_dbg(xhci, "Stop HCD\n"); 1083 xhci_halt(xhci); 1084 xhci_zero_64b_regs(xhci); 1085 retval = xhci_reset(xhci, XHCI_RESET_LONG_USEC); 1086 spin_unlock_irq(&xhci->lock); 1087 if (retval) 1088 return retval; 1089 1090 xhci_dbg(xhci, "// Disabling event ring interrupts\n"); 1091 temp = readl(&xhci->op_regs->status); 1092 writel((temp & ~0x1fff) | STS_EINT, &xhci->op_regs->status); 1093 xhci_disable_interrupter(xhci->interrupters[0]); 1094 1095 xhci_dbg(xhci, "cleaning up memory\n"); 1096 xhci_mem_cleanup(xhci); 1097 xhci_debugfs_exit(xhci); 1098 xhci_dbg(xhci, "xhci_stop completed - status = %x\n", 1099 readl(&xhci->op_regs->status)); 1100 1101 /* USB core calls the PCI reinit and start functions twice: 1102 * first with the primary HCD, and then with the secondary HCD. 1103 * If we don't do the same, the host will never be started. 1104 */ 1105 xhci_dbg(xhci, "Initialize the xhci_hcd\n"); 1106 retval = xhci_init(hcd); 1107 if (retval) 1108 return retval; 1109 comp_timer_running = true; 1110 1111 xhci_dbg(xhci, "Start the primary HCD\n"); 1112 retval = xhci_run(hcd); 1113 if (!retval && xhci->shared_hcd) { 1114 xhci_dbg(xhci, "Start the secondary HCD\n"); 1115 retval = xhci_run(xhci->shared_hcd); 1116 } 1117 if (retval) 1118 return retval; 1119 /* 1120 * Resume roothubs unconditionally as PORTSC change bits are not 1121 * immediately visible after xHC reset 1122 */ 1123 hcd->state = HC_STATE_SUSPENDED; 1124 1125 if (xhci->shared_hcd) { 1126 xhci->shared_hcd->state = HC_STATE_SUSPENDED; 1127 usb_hcd_resume_root_hub(xhci->shared_hcd); 1128 } 1129 usb_hcd_resume_root_hub(hcd); 1130 1131 goto done; 1132 } 1133 1134 /* step 4: set Run/Stop bit */ 1135 command = readl(&xhci->op_regs->command); 1136 command |= CMD_RUN; 1137 writel(command, &xhci->op_regs->command); 1138 xhci_handshake(&xhci->op_regs->status, STS_HALT, 1139 0, 250 * 1000); 1140 1141 /* step 5: walk topology and initialize portsc, 1142 * portpmsc and portli 1143 */ 1144 /* this is done in bus_resume */ 1145 1146 /* step 6: restart each of the previously 1147 * Running endpoints by ringing their doorbells 1148 */ 1149 1150 spin_unlock_irq(&xhci->lock); 1151 1152 xhci_dbc_resume(xhci); 1153 1154 if (retval == 0) { 1155 /* 1156 * Resume roothubs only if there are pending events. 1157 * USB 3 devices resend U3 LFPS wake after a 100ms delay if 1158 * the first wake signalling failed, give it that chance if 1159 * there are suspended USB 3 devices. 1160 */ 1161 if (xhci->usb3_rhub.bus_state.suspended_ports || 1162 xhci->usb3_rhub.bus_state.bus_suspended) 1163 suspended_usb3_devs = true; 1164 1165 pending_portevent = xhci_pending_portevent(xhci); 1166 1167 if (suspended_usb3_devs && !pending_portevent && 1168 msg.event == PM_EVENT_AUTO_RESUME) { 1169 msleep(120); 1170 pending_portevent = xhci_pending_portevent(xhci); 1171 } 1172 1173 if (pending_portevent) { 1174 if (xhci->shared_hcd) 1175 usb_hcd_resume_root_hub(xhci->shared_hcd); 1176 usb_hcd_resume_root_hub(hcd); 1177 } 1178 } 1179 done: 1180 /* 1181 * If system is subject to the Quirk, Compliance Mode Timer needs to 1182 * be re-initialized Always after a system resume. Ports are subject 1183 * to suffer the Compliance Mode issue again. It doesn't matter if 1184 * ports have entered previously to U0 before system's suspension. 1185 */ 1186 if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) && !comp_timer_running) 1187 compliance_mode_recovery_timer_init(xhci); 1188 1189 if (xhci->quirks & XHCI_ASMEDIA_MODIFY_FLOWCONTROL) 1190 usb_asmedia_modifyflowcontrol(to_pci_dev(hcd->self.controller)); 1191 1192 /* Re-enable port polling. */ 1193 xhci_dbg(xhci, "%s: starting usb%d port polling.\n", 1194 __func__, hcd->self.busnum); 1195 if (xhci->shared_hcd) { 1196 set_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags); 1197 usb_hcd_poll_rh_status(xhci->shared_hcd); 1198 } 1199 set_bit(HCD_FLAG_POLL_RH, &hcd->flags); 1200 usb_hcd_poll_rh_status(hcd); 1201 1202 return retval; 1203 } 1204 EXPORT_SYMBOL_GPL(xhci_resume); 1205 #endif /* CONFIG_PM */ 1206 1207 /*-------------------------------------------------------------------------*/ 1208 1209 static int xhci_map_temp_buffer(struct usb_hcd *hcd, struct urb *urb) 1210 { 1211 void *temp; 1212 int ret = 0; 1213 unsigned int buf_len; 1214 enum dma_data_direction dir; 1215 1216 dir = usb_urb_dir_in(urb) ? DMA_FROM_DEVICE : DMA_TO_DEVICE; 1217 buf_len = urb->transfer_buffer_length; 1218 1219 temp = kzalloc_node(buf_len, GFP_ATOMIC, 1220 dev_to_node(hcd->self.sysdev)); 1221 if (!temp) 1222 return -ENOMEM; 1223 1224 if (usb_urb_dir_out(urb)) 1225 sg_pcopy_to_buffer(urb->sg, urb->num_sgs, 1226 temp, buf_len, 0); 1227 1228 urb->transfer_buffer = temp; 1229 urb->transfer_dma = dma_map_single(hcd->self.sysdev, 1230 urb->transfer_buffer, 1231 urb->transfer_buffer_length, 1232 dir); 1233 1234 if (dma_mapping_error(hcd->self.sysdev, 1235 urb->transfer_dma)) { 1236 ret = -EAGAIN; 1237 kfree(temp); 1238 } else { 1239 urb->transfer_flags |= URB_DMA_MAP_SINGLE; 1240 } 1241 1242 return ret; 1243 } 1244 1245 static bool xhci_urb_temp_buffer_required(struct usb_hcd *hcd, 1246 struct urb *urb) 1247 { 1248 bool ret = false; 1249 unsigned int i; 1250 unsigned int len = 0; 1251 unsigned int trb_size; 1252 unsigned int max_pkt; 1253 struct scatterlist *sg; 1254 struct scatterlist *tail_sg; 1255 1256 tail_sg = urb->sg; 1257 max_pkt = usb_endpoint_maxp(&urb->ep->desc); 1258 1259 if (!urb->num_sgs) 1260 return ret; 1261 1262 if (urb->dev->speed >= USB_SPEED_SUPER) 1263 trb_size = TRB_CACHE_SIZE_SS; 1264 else 1265 trb_size = TRB_CACHE_SIZE_HS; 1266 1267 if (urb->transfer_buffer_length != 0 && 1268 !(urb->transfer_flags & URB_NO_TRANSFER_DMA_MAP)) { 1269 for_each_sg(urb->sg, sg, urb->num_sgs, i) { 1270 len = len + sg->length; 1271 if (i > trb_size - 2) { 1272 len = len - tail_sg->length; 1273 if (len < max_pkt) { 1274 ret = true; 1275 break; 1276 } 1277 1278 tail_sg = sg_next(tail_sg); 1279 } 1280 } 1281 } 1282 return ret; 1283 } 1284 1285 static void xhci_unmap_temp_buf(struct usb_hcd *hcd, struct urb *urb) 1286 { 1287 unsigned int len; 1288 unsigned int buf_len; 1289 enum dma_data_direction dir; 1290 1291 dir = usb_urb_dir_in(urb) ? DMA_FROM_DEVICE : DMA_TO_DEVICE; 1292 1293 buf_len = urb->transfer_buffer_length; 1294 1295 if (IS_ENABLED(CONFIG_HAS_DMA) && 1296 (urb->transfer_flags & URB_DMA_MAP_SINGLE)) 1297 dma_unmap_single(hcd->self.sysdev, 1298 urb->transfer_dma, 1299 urb->transfer_buffer_length, 1300 dir); 1301 1302 if (usb_urb_dir_in(urb)) { 1303 len = sg_pcopy_from_buffer(urb->sg, urb->num_sgs, 1304 urb->transfer_buffer, 1305 buf_len, 1306 0); 1307 if (len != buf_len) { 1308 xhci_dbg(hcd_to_xhci(hcd), 1309 "Copy from tmp buf to urb sg list failed\n"); 1310 urb->actual_length = len; 1311 } 1312 } 1313 urb->transfer_flags &= ~URB_DMA_MAP_SINGLE; 1314 kfree(urb->transfer_buffer); 1315 urb->transfer_buffer = NULL; 1316 } 1317 1318 /* 1319 * Bypass the DMA mapping if URB is suitable for Immediate Transfer (IDT), 1320 * we'll copy the actual data into the TRB address register. This is limited to 1321 * transfers up to 8 bytes on output endpoints of any kind with wMaxPacketSize 1322 * >= 8 bytes. If suitable for IDT only one Transfer TRB per TD is allowed. 1323 */ 1324 static int xhci_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb, 1325 gfp_t mem_flags) 1326 { 1327 struct xhci_hcd *xhci; 1328 1329 xhci = hcd_to_xhci(hcd); 1330 1331 if (xhci_urb_suitable_for_idt(urb)) 1332 return 0; 1333 1334 if (xhci->quirks & XHCI_SG_TRB_CACHE_SIZE_QUIRK) { 1335 if (xhci_urb_temp_buffer_required(hcd, urb)) 1336 return xhci_map_temp_buffer(hcd, urb); 1337 } 1338 return usb_hcd_map_urb_for_dma(hcd, urb, mem_flags); 1339 } 1340 1341 static void xhci_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb) 1342 { 1343 struct xhci_hcd *xhci; 1344 bool unmap_temp_buf = false; 1345 1346 xhci = hcd_to_xhci(hcd); 1347 1348 if (urb->num_sgs && (urb->transfer_flags & URB_DMA_MAP_SINGLE)) 1349 unmap_temp_buf = true; 1350 1351 if ((xhci->quirks & XHCI_SG_TRB_CACHE_SIZE_QUIRK) && unmap_temp_buf) 1352 xhci_unmap_temp_buf(hcd, urb); 1353 else 1354 usb_hcd_unmap_urb_for_dma(hcd, urb); 1355 } 1356 1357 /** 1358 * xhci_get_endpoint_index - Used for passing endpoint bitmasks between the core and 1359 * HCDs. Find the index for an endpoint given its descriptor. Use the return 1360 * value to right shift 1 for the bitmask. 1361 * 1362 * Index = (epnum * 2) + direction - 1, 1363 * where direction = 0 for OUT, 1 for IN. 1364 * For control endpoints, the IN index is used (OUT index is unused), so 1365 * index = (epnum * 2) + direction - 1 = (epnum * 2) + 1 - 1 = (epnum * 2) 1366 */ 1367 unsigned int xhci_get_endpoint_index(struct usb_endpoint_descriptor *desc) 1368 { 1369 unsigned int index; 1370 if (usb_endpoint_xfer_control(desc)) 1371 index = (unsigned int) (usb_endpoint_num(desc)*2); 1372 else 1373 index = (unsigned int) (usb_endpoint_num(desc)*2) + 1374 (usb_endpoint_dir_in(desc) ? 1 : 0) - 1; 1375 return index; 1376 } 1377 EXPORT_SYMBOL_GPL(xhci_get_endpoint_index); 1378 1379 /* The reverse operation to xhci_get_endpoint_index. Calculate the USB endpoint 1380 * address from the XHCI endpoint index. 1381 */ 1382 static unsigned int xhci_get_endpoint_address(unsigned int ep_index) 1383 { 1384 unsigned int number = DIV_ROUND_UP(ep_index, 2); 1385 unsigned int direction = ep_index % 2 ? USB_DIR_OUT : USB_DIR_IN; 1386 return direction | number; 1387 } 1388 1389 /* Find the flag for this endpoint (for use in the control context). Use the 1390 * endpoint index to create a bitmask. The slot context is bit 0, endpoint 0 is 1391 * bit 1, etc. 1392 */ 1393 static unsigned int xhci_get_endpoint_flag(struct usb_endpoint_descriptor *desc) 1394 { 1395 return 1 << (xhci_get_endpoint_index(desc) + 1); 1396 } 1397 1398 /* Compute the last valid endpoint context index. Basically, this is the 1399 * endpoint index plus one. For slot contexts with more than valid endpoint, 1400 * we find the most significant bit set in the added contexts flags. 1401 * e.g. ep 1 IN (with epnum 0x81) => added_ctxs = 0b1000 1402 * fls(0b1000) = 4, but the endpoint context index is 3, so subtract one. 1403 */ 1404 unsigned int xhci_last_valid_endpoint(u32 added_ctxs) 1405 { 1406 return fls(added_ctxs) - 1; 1407 } 1408 1409 /* Returns 1 if the arguments are OK; 1410 * returns 0 this is a root hub; returns -EINVAL for NULL pointers. 1411 */ 1412 static int xhci_check_args(struct usb_hcd *hcd, struct usb_device *udev, 1413 struct usb_host_endpoint *ep, int check_ep, bool check_virt_dev, 1414 const char *func) { 1415 struct xhci_hcd *xhci; 1416 struct xhci_virt_device *virt_dev; 1417 1418 if (!hcd || (check_ep && !ep) || !udev) { 1419 pr_debug("xHCI %s called with invalid args\n", func); 1420 return -EINVAL; 1421 } 1422 if (!udev->parent) { 1423 pr_debug("xHCI %s called for root hub\n", func); 1424 return 0; 1425 } 1426 1427 xhci = hcd_to_xhci(hcd); 1428 if (check_virt_dev) { 1429 if (!udev->slot_id || !xhci->devs[udev->slot_id]) { 1430 xhci_dbg(xhci, "xHCI %s called with unaddressed device\n", 1431 func); 1432 return -EINVAL; 1433 } 1434 1435 virt_dev = xhci->devs[udev->slot_id]; 1436 if (virt_dev->udev != udev) { 1437 xhci_dbg(xhci, "xHCI %s called with udev and " 1438 "virt_dev does not match\n", func); 1439 return -EINVAL; 1440 } 1441 } 1442 1443 if (xhci->xhc_state & XHCI_STATE_HALTED) 1444 return -ENODEV; 1445 1446 return 1; 1447 } 1448 1449 static int xhci_configure_endpoint(struct xhci_hcd *xhci, 1450 struct usb_device *udev, struct xhci_command *command, 1451 bool ctx_change, bool must_succeed); 1452 1453 /* 1454 * Full speed devices may have a max packet size greater than 8 bytes, but the 1455 * USB core doesn't know that until it reads the first 8 bytes of the 1456 * descriptor. If the usb_device's max packet size changes after that point, 1457 * we need to issue an evaluate context command and wait on it. 1458 */ 1459 static int xhci_check_ep0_maxpacket(struct xhci_hcd *xhci, struct xhci_virt_device *vdev) 1460 { 1461 struct xhci_input_control_ctx *ctrl_ctx; 1462 struct xhci_ep_ctx *ep_ctx; 1463 struct xhci_command *command; 1464 int max_packet_size; 1465 int hw_max_packet_size; 1466 int ret = 0; 1467 1468 ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, 0); 1469 hw_max_packet_size = MAX_PACKET_DECODED(le32_to_cpu(ep_ctx->ep_info2)); 1470 max_packet_size = usb_endpoint_maxp(&vdev->udev->ep0.desc); 1471 1472 if (hw_max_packet_size == max_packet_size) 1473 return 0; 1474 1475 switch (max_packet_size) { 1476 case 8: case 16: case 32: case 64: case 9: 1477 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change, 1478 "Max Packet Size for ep 0 changed."); 1479 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change, 1480 "Max packet size in usb_device = %d", 1481 max_packet_size); 1482 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change, 1483 "Max packet size in xHCI HW = %d", 1484 hw_max_packet_size); 1485 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change, 1486 "Issuing evaluate context command."); 1487 1488 command = xhci_alloc_command(xhci, true, GFP_KERNEL); 1489 if (!command) 1490 return -ENOMEM; 1491 1492 command->in_ctx = vdev->in_ctx; 1493 ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx); 1494 if (!ctrl_ctx) { 1495 xhci_warn(xhci, "%s: Could not get input context, bad type.\n", 1496 __func__); 1497 ret = -ENOMEM; 1498 break; 1499 } 1500 /* Set up the modified control endpoint 0 */ 1501 xhci_endpoint_copy(xhci, vdev->in_ctx, vdev->out_ctx, 0); 1502 1503 ep_ctx = xhci_get_ep_ctx(xhci, command->in_ctx, 0); 1504 ep_ctx->ep_info &= cpu_to_le32(~EP_STATE_MASK);/* must clear */ 1505 ep_ctx->ep_info2 &= cpu_to_le32(~MAX_PACKET_MASK); 1506 ep_ctx->ep_info2 |= cpu_to_le32(MAX_PACKET(max_packet_size)); 1507 1508 ctrl_ctx->add_flags = cpu_to_le32(EP0_FLAG); 1509 ctrl_ctx->drop_flags = 0; 1510 1511 ret = xhci_configure_endpoint(xhci, vdev->udev, command, 1512 true, false); 1513 /* Clean up the input context for later use by bandwidth functions */ 1514 ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG); 1515 break; 1516 default: 1517 dev_dbg(&vdev->udev->dev, "incorrect max packet size %d for ep0\n", 1518 max_packet_size); 1519 return -EINVAL; 1520 } 1521 1522 kfree(command->completion); 1523 kfree(command); 1524 1525 return ret; 1526 } 1527 1528 /* 1529 * non-error returns are a promise to giveback() the urb later 1530 * we drop ownership so next owner (or urb unlink) can get it 1531 */ 1532 static int xhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flags) 1533 { 1534 struct xhci_hcd *xhci = hcd_to_xhci(hcd); 1535 unsigned long flags; 1536 int ret = 0; 1537 unsigned int slot_id, ep_index; 1538 unsigned int *ep_state; 1539 struct urb_priv *urb_priv; 1540 int num_tds; 1541 1542 ep_index = xhci_get_endpoint_index(&urb->ep->desc); 1543 1544 if (usb_endpoint_xfer_isoc(&urb->ep->desc)) 1545 num_tds = urb->number_of_packets; 1546 else if (usb_endpoint_is_bulk_out(&urb->ep->desc) && 1547 urb->transfer_buffer_length > 0 && 1548 urb->transfer_flags & URB_ZERO_PACKET && 1549 !(urb->transfer_buffer_length % usb_endpoint_maxp(&urb->ep->desc))) 1550 num_tds = 2; 1551 else 1552 num_tds = 1; 1553 1554 urb_priv = kzalloc(struct_size(urb_priv, td, num_tds), mem_flags); 1555 if (!urb_priv) 1556 return -ENOMEM; 1557 1558 urb_priv->num_tds = num_tds; 1559 urb_priv->num_tds_done = 0; 1560 urb->hcpriv = urb_priv; 1561 1562 trace_xhci_urb_enqueue(urb); 1563 1564 spin_lock_irqsave(&xhci->lock, flags); 1565 1566 ret = xhci_check_args(hcd, urb->dev, urb->ep, 1567 true, true, __func__); 1568 if (ret <= 0) { 1569 ret = ret ? ret : -EINVAL; 1570 goto free_priv; 1571 } 1572 1573 slot_id = urb->dev->slot_id; 1574 1575 if (!HCD_HW_ACCESSIBLE(hcd)) { 1576 ret = -ESHUTDOWN; 1577 goto free_priv; 1578 } 1579 1580 if (xhci->devs[slot_id]->flags & VDEV_PORT_ERROR) { 1581 xhci_dbg(xhci, "Can't queue urb, port error, link inactive\n"); 1582 ret = -ENODEV; 1583 goto free_priv; 1584 } 1585 1586 if (xhci->xhc_state & XHCI_STATE_DYING) { 1587 xhci_dbg(xhci, "Ep 0x%x: URB %p submitted for non-responsive xHCI host.\n", 1588 urb->ep->desc.bEndpointAddress, urb); 1589 ret = -ESHUTDOWN; 1590 goto free_priv; 1591 } 1592 1593 ep_state = &xhci->devs[slot_id]->eps[ep_index].ep_state; 1594 1595 if (*ep_state & (EP_GETTING_STREAMS | EP_GETTING_NO_STREAMS)) { 1596 xhci_warn(xhci, "WARN: Can't enqueue URB, ep in streams transition state %x\n", 1597 *ep_state); 1598 ret = -EINVAL; 1599 goto free_priv; 1600 } 1601 if (*ep_state & EP_SOFT_CLEAR_TOGGLE) { 1602 xhci_warn(xhci, "Can't enqueue URB while manually clearing toggle\n"); 1603 ret = -EINVAL; 1604 goto free_priv; 1605 } 1606 1607 switch (usb_endpoint_type(&urb->ep->desc)) { 1608 1609 case USB_ENDPOINT_XFER_CONTROL: 1610 ret = xhci_queue_ctrl_tx(xhci, GFP_ATOMIC, urb, 1611 slot_id, ep_index); 1612 break; 1613 case USB_ENDPOINT_XFER_BULK: 1614 ret = xhci_queue_bulk_tx(xhci, GFP_ATOMIC, urb, 1615 slot_id, ep_index); 1616 break; 1617 case USB_ENDPOINT_XFER_INT: 1618 ret = xhci_queue_intr_tx(xhci, GFP_ATOMIC, urb, 1619 slot_id, ep_index); 1620 break; 1621 case USB_ENDPOINT_XFER_ISOC: 1622 ret = xhci_queue_isoc_tx_prepare(xhci, GFP_ATOMIC, urb, 1623 slot_id, ep_index); 1624 } 1625 1626 if (ret) { 1627 free_priv: 1628 xhci_urb_free_priv(urb_priv); 1629 urb->hcpriv = NULL; 1630 } 1631 spin_unlock_irqrestore(&xhci->lock, flags); 1632 return ret; 1633 } 1634 1635 /* 1636 * Remove the URB's TD from the endpoint ring. This may cause the HC to stop 1637 * USB transfers, potentially stopping in the middle of a TRB buffer. The HC 1638 * should pick up where it left off in the TD, unless a Set Transfer Ring 1639 * Dequeue Pointer is issued. 1640 * 1641 * The TRBs that make up the buffers for the canceled URB will be "removed" from 1642 * the ring. Since the ring is a contiguous structure, they can't be physically 1643 * removed. Instead, there are two options: 1644 * 1645 * 1) If the HC is in the middle of processing the URB to be canceled, we 1646 * simply move the ring's dequeue pointer past those TRBs using the Set 1647 * Transfer Ring Dequeue Pointer command. This will be the common case, 1648 * when drivers timeout on the last submitted URB and attempt to cancel. 1649 * 1650 * 2) If the HC is in the middle of a different TD, we turn the TRBs into a 1651 * series of 1-TRB transfer no-op TDs. (No-ops shouldn't be chained.) The 1652 * HC will need to invalidate the any TRBs it has cached after the stop 1653 * endpoint command, as noted in the xHCI 0.95 errata. 1654 * 1655 * 3) The TD may have completed by the time the Stop Endpoint Command 1656 * completes, so software needs to handle that case too. 1657 * 1658 * This function should protect against the TD enqueueing code ringing the 1659 * doorbell while this code is waiting for a Stop Endpoint command to complete. 1660 * It also needs to account for multiple cancellations on happening at the same 1661 * time for the same endpoint. 1662 * 1663 * Note that this function can be called in any context, or so says 1664 * usb_hcd_unlink_urb() 1665 */ 1666 static int xhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status) 1667 { 1668 unsigned long flags; 1669 int ret, i; 1670 u32 temp; 1671 struct xhci_hcd *xhci; 1672 struct urb_priv *urb_priv; 1673 struct xhci_td *td; 1674 unsigned int ep_index; 1675 struct xhci_ring *ep_ring; 1676 struct xhci_virt_ep *ep; 1677 struct xhci_command *command; 1678 struct xhci_virt_device *vdev; 1679 1680 xhci = hcd_to_xhci(hcd); 1681 spin_lock_irqsave(&xhci->lock, flags); 1682 1683 trace_xhci_urb_dequeue(urb); 1684 1685 /* Make sure the URB hasn't completed or been unlinked already */ 1686 ret = usb_hcd_check_unlink_urb(hcd, urb, status); 1687 if (ret) 1688 goto done; 1689 1690 /* give back URB now if we can't queue it for cancel */ 1691 vdev = xhci->devs[urb->dev->slot_id]; 1692 urb_priv = urb->hcpriv; 1693 if (!vdev || !urb_priv) 1694 goto err_giveback; 1695 1696 ep_index = xhci_get_endpoint_index(&urb->ep->desc); 1697 ep = &vdev->eps[ep_index]; 1698 ep_ring = xhci_urb_to_transfer_ring(xhci, urb); 1699 if (!ep || !ep_ring) 1700 goto err_giveback; 1701 1702 /* If xHC is dead take it down and return ALL URBs in xhci_hc_died() */ 1703 temp = readl(&xhci->op_regs->status); 1704 if (temp == ~(u32)0 || xhci->xhc_state & XHCI_STATE_DYING) { 1705 xhci_hc_died(xhci); 1706 goto done; 1707 } 1708 1709 /* 1710 * check ring is not re-allocated since URB was enqueued. If it is, then 1711 * make sure none of the ring related pointers in this URB private data 1712 * are touched, such as td_list, otherwise we overwrite freed data 1713 */ 1714 if (!td_on_ring(&urb_priv->td[0], ep_ring)) { 1715 xhci_err(xhci, "Canceled URB td not found on endpoint ring"); 1716 for (i = urb_priv->num_tds_done; i < urb_priv->num_tds; i++) { 1717 td = &urb_priv->td[i]; 1718 if (!list_empty(&td->cancelled_td_list)) 1719 list_del_init(&td->cancelled_td_list); 1720 } 1721 goto err_giveback; 1722 } 1723 1724 if (xhci->xhc_state & XHCI_STATE_HALTED) { 1725 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, 1726 "HC halted, freeing TD manually."); 1727 for (i = urb_priv->num_tds_done; 1728 i < urb_priv->num_tds; 1729 i++) { 1730 td = &urb_priv->td[i]; 1731 if (!list_empty(&td->td_list)) 1732 list_del_init(&td->td_list); 1733 if (!list_empty(&td->cancelled_td_list)) 1734 list_del_init(&td->cancelled_td_list); 1735 } 1736 goto err_giveback; 1737 } 1738 1739 i = urb_priv->num_tds_done; 1740 if (i < urb_priv->num_tds) 1741 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, 1742 "Cancel URB %p, dev %s, ep 0x%x, " 1743 "starting at offset 0x%llx", 1744 urb, urb->dev->devpath, 1745 urb->ep->desc.bEndpointAddress, 1746 (unsigned long long) xhci_trb_virt_to_dma( 1747 urb_priv->td[i].start_seg, 1748 urb_priv->td[i].start_trb)); 1749 1750 for (; i < urb_priv->num_tds; i++) { 1751 td = &urb_priv->td[i]; 1752 /* TD can already be on cancelled list if ep halted on it */ 1753 if (list_empty(&td->cancelled_td_list)) { 1754 td->cancel_status = TD_DIRTY; 1755 list_add_tail(&td->cancelled_td_list, 1756 &ep->cancelled_td_list); 1757 } 1758 } 1759 1760 /* These completion handlers will sort out cancelled TDs for us */ 1761 if (ep->ep_state & (EP_STOP_CMD_PENDING | EP_HALTED | SET_DEQ_PENDING)) { 1762 xhci_dbg(xhci, "Not queuing Stop Endpoint on slot %d ep %d in state 0x%x\n", 1763 urb->dev->slot_id, ep_index, ep->ep_state); 1764 goto done; 1765 } 1766 1767 /* In this case no commands are pending but the endpoint is stopped */ 1768 if (ep->ep_state & EP_CLEARING_TT) { 1769 /* and cancelled TDs can be given back right away */ 1770 xhci_dbg(xhci, "Invalidating TDs instantly on slot %d ep %d in state 0x%x\n", 1771 urb->dev->slot_id, ep_index, ep->ep_state); 1772 xhci_process_cancelled_tds(ep); 1773 } else { 1774 /* Otherwise, queue a new Stop Endpoint command */ 1775 command = xhci_alloc_command(xhci, false, GFP_ATOMIC); 1776 if (!command) { 1777 ret = -ENOMEM; 1778 goto done; 1779 } 1780 ep->stop_time = jiffies; 1781 ep->ep_state |= EP_STOP_CMD_PENDING; 1782 xhci_queue_stop_endpoint(xhci, command, urb->dev->slot_id, 1783 ep_index, 0); 1784 xhci_ring_cmd_db(xhci); 1785 } 1786 done: 1787 spin_unlock_irqrestore(&xhci->lock, flags); 1788 return ret; 1789 1790 err_giveback: 1791 if (urb_priv) 1792 xhci_urb_free_priv(urb_priv); 1793 usb_hcd_unlink_urb_from_ep(hcd, urb); 1794 spin_unlock_irqrestore(&xhci->lock, flags); 1795 usb_hcd_giveback_urb(hcd, urb, -ESHUTDOWN); 1796 return ret; 1797 } 1798 1799 /* Drop an endpoint from a new bandwidth configuration for this device. 1800 * Only one call to this function is allowed per endpoint before 1801 * check_bandwidth() or reset_bandwidth() must be called. 1802 * A call to xhci_drop_endpoint() followed by a call to xhci_add_endpoint() will 1803 * add the endpoint to the schedule with possibly new parameters denoted by a 1804 * different endpoint descriptor in usb_host_endpoint. 1805 * A call to xhci_add_endpoint() followed by a call to xhci_drop_endpoint() is 1806 * not allowed. 1807 * 1808 * The USB core will not allow URBs to be queued to an endpoint that is being 1809 * disabled, so there's no need for mutual exclusion to protect 1810 * the xhci->devs[slot_id] structure. 1811 */ 1812 int xhci_drop_endpoint(struct usb_hcd *hcd, struct usb_device *udev, 1813 struct usb_host_endpoint *ep) 1814 { 1815 struct xhci_hcd *xhci; 1816 struct xhci_container_ctx *in_ctx, *out_ctx; 1817 struct xhci_input_control_ctx *ctrl_ctx; 1818 unsigned int ep_index; 1819 struct xhci_ep_ctx *ep_ctx; 1820 u32 drop_flag; 1821 u32 new_add_flags, new_drop_flags; 1822 int ret; 1823 1824 ret = xhci_check_args(hcd, udev, ep, 1, true, __func__); 1825 if (ret <= 0) 1826 return ret; 1827 xhci = hcd_to_xhci(hcd); 1828 if (xhci->xhc_state & XHCI_STATE_DYING) 1829 return -ENODEV; 1830 1831 xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev); 1832 drop_flag = xhci_get_endpoint_flag(&ep->desc); 1833 if (drop_flag == SLOT_FLAG || drop_flag == EP0_FLAG) { 1834 xhci_dbg(xhci, "xHCI %s - can't drop slot or ep 0 %#x\n", 1835 __func__, drop_flag); 1836 return 0; 1837 } 1838 1839 in_ctx = xhci->devs[udev->slot_id]->in_ctx; 1840 out_ctx = xhci->devs[udev->slot_id]->out_ctx; 1841 ctrl_ctx = xhci_get_input_control_ctx(in_ctx); 1842 if (!ctrl_ctx) { 1843 xhci_warn(xhci, "%s: Could not get input context, bad type.\n", 1844 __func__); 1845 return 0; 1846 } 1847 1848 ep_index = xhci_get_endpoint_index(&ep->desc); 1849 ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index); 1850 /* If the HC already knows the endpoint is disabled, 1851 * or the HCD has noted it is disabled, ignore this request 1852 */ 1853 if ((GET_EP_CTX_STATE(ep_ctx) == EP_STATE_DISABLED) || 1854 le32_to_cpu(ctrl_ctx->drop_flags) & 1855 xhci_get_endpoint_flag(&ep->desc)) { 1856 /* Do not warn when called after a usb_device_reset */ 1857 if (xhci->devs[udev->slot_id]->eps[ep_index].ring != NULL) 1858 xhci_warn(xhci, "xHCI %s called with disabled ep %p\n", 1859 __func__, ep); 1860 return 0; 1861 } 1862 1863 ctrl_ctx->drop_flags |= cpu_to_le32(drop_flag); 1864 new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags); 1865 1866 ctrl_ctx->add_flags &= cpu_to_le32(~drop_flag); 1867 new_add_flags = le32_to_cpu(ctrl_ctx->add_flags); 1868 1869 xhci_debugfs_remove_endpoint(xhci, xhci->devs[udev->slot_id], ep_index); 1870 1871 xhci_endpoint_zero(xhci, xhci->devs[udev->slot_id], ep); 1872 1873 xhci_dbg(xhci, "drop ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x\n", 1874 (unsigned int) ep->desc.bEndpointAddress, 1875 udev->slot_id, 1876 (unsigned int) new_drop_flags, 1877 (unsigned int) new_add_flags); 1878 return 0; 1879 } 1880 EXPORT_SYMBOL_GPL(xhci_drop_endpoint); 1881 1882 /* Add an endpoint to a new possible bandwidth configuration for this device. 1883 * Only one call to this function is allowed per endpoint before 1884 * check_bandwidth() or reset_bandwidth() must be called. 1885 * A call to xhci_drop_endpoint() followed by a call to xhci_add_endpoint() will 1886 * add the endpoint to the schedule with possibly new parameters denoted by a 1887 * different endpoint descriptor in usb_host_endpoint. 1888 * A call to xhci_add_endpoint() followed by a call to xhci_drop_endpoint() is 1889 * not allowed. 1890 * 1891 * The USB core will not allow URBs to be queued to an endpoint until the 1892 * configuration or alt setting is installed in the device, so there's no need 1893 * for mutual exclusion to protect the xhci->devs[slot_id] structure. 1894 */ 1895 int xhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev, 1896 struct usb_host_endpoint *ep) 1897 { 1898 struct xhci_hcd *xhci; 1899 struct xhci_container_ctx *in_ctx; 1900 unsigned int ep_index; 1901 struct xhci_input_control_ctx *ctrl_ctx; 1902 struct xhci_ep_ctx *ep_ctx; 1903 u32 added_ctxs; 1904 u32 new_add_flags, new_drop_flags; 1905 struct xhci_virt_device *virt_dev; 1906 int ret = 0; 1907 1908 ret = xhci_check_args(hcd, udev, ep, 1, true, __func__); 1909 if (ret <= 0) { 1910 /* So we won't queue a reset ep command for a root hub */ 1911 ep->hcpriv = NULL; 1912 return ret; 1913 } 1914 xhci = hcd_to_xhci(hcd); 1915 if (xhci->xhc_state & XHCI_STATE_DYING) 1916 return -ENODEV; 1917 1918 added_ctxs = xhci_get_endpoint_flag(&ep->desc); 1919 if (added_ctxs == SLOT_FLAG || added_ctxs == EP0_FLAG) { 1920 /* FIXME when we have to issue an evaluate endpoint command to 1921 * deal with ep0 max packet size changing once we get the 1922 * descriptors 1923 */ 1924 xhci_dbg(xhci, "xHCI %s - can't add slot or ep 0 %#x\n", 1925 __func__, added_ctxs); 1926 return 0; 1927 } 1928 1929 virt_dev = xhci->devs[udev->slot_id]; 1930 in_ctx = virt_dev->in_ctx; 1931 ctrl_ctx = xhci_get_input_control_ctx(in_ctx); 1932 if (!ctrl_ctx) { 1933 xhci_warn(xhci, "%s: Could not get input context, bad type.\n", 1934 __func__); 1935 return 0; 1936 } 1937 1938 ep_index = xhci_get_endpoint_index(&ep->desc); 1939 /* If this endpoint is already in use, and the upper layers are trying 1940 * to add it again without dropping it, reject the addition. 1941 */ 1942 if (virt_dev->eps[ep_index].ring && 1943 !(le32_to_cpu(ctrl_ctx->drop_flags) & added_ctxs)) { 1944 xhci_warn(xhci, "Trying to add endpoint 0x%x " 1945 "without dropping it.\n", 1946 (unsigned int) ep->desc.bEndpointAddress); 1947 return -EINVAL; 1948 } 1949 1950 /* If the HCD has already noted the endpoint is enabled, 1951 * ignore this request. 1952 */ 1953 if (le32_to_cpu(ctrl_ctx->add_flags) & added_ctxs) { 1954 xhci_warn(xhci, "xHCI %s called with enabled ep %p\n", 1955 __func__, ep); 1956 return 0; 1957 } 1958 1959 /* 1960 * Configuration and alternate setting changes must be done in 1961 * process context, not interrupt context (or so documenation 1962 * for usb_set_interface() and usb_set_configuration() claim). 1963 */ 1964 if (xhci_endpoint_init(xhci, virt_dev, udev, ep, GFP_NOIO) < 0) { 1965 dev_dbg(&udev->dev, "%s - could not initialize ep %#x\n", 1966 __func__, ep->desc.bEndpointAddress); 1967 return -ENOMEM; 1968 } 1969 1970 ctrl_ctx->add_flags |= cpu_to_le32(added_ctxs); 1971 new_add_flags = le32_to_cpu(ctrl_ctx->add_flags); 1972 1973 /* If xhci_endpoint_disable() was called for this endpoint, but the 1974 * xHC hasn't been notified yet through the check_bandwidth() call, 1975 * this re-adds a new state for the endpoint from the new endpoint 1976 * descriptors. We must drop and re-add this endpoint, so we leave the 1977 * drop flags alone. 1978 */ 1979 new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags); 1980 1981 /* Store the usb_device pointer for later use */ 1982 ep->hcpriv = udev; 1983 1984 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index); 1985 trace_xhci_add_endpoint(ep_ctx); 1986 1987 xhci_dbg(xhci, "add ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x\n", 1988 (unsigned int) ep->desc.bEndpointAddress, 1989 udev->slot_id, 1990 (unsigned int) new_drop_flags, 1991 (unsigned int) new_add_flags); 1992 return 0; 1993 } 1994 EXPORT_SYMBOL_GPL(xhci_add_endpoint); 1995 1996 static void xhci_zero_in_ctx(struct xhci_hcd *xhci, struct xhci_virt_device *virt_dev) 1997 { 1998 struct xhci_input_control_ctx *ctrl_ctx; 1999 struct xhci_ep_ctx *ep_ctx; 2000 struct xhci_slot_ctx *slot_ctx; 2001 int i; 2002 2003 ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx); 2004 if (!ctrl_ctx) { 2005 xhci_warn(xhci, "%s: Could not get input context, bad type.\n", 2006 __func__); 2007 return; 2008 } 2009 2010 /* When a device's add flag and drop flag are zero, any subsequent 2011 * configure endpoint command will leave that endpoint's state 2012 * untouched. Make sure we don't leave any old state in the input 2013 * endpoint contexts. 2014 */ 2015 ctrl_ctx->drop_flags = 0; 2016 ctrl_ctx->add_flags = 0; 2017 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx); 2018 slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK); 2019 /* Endpoint 0 is always valid */ 2020 slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1)); 2021 for (i = 1; i < 31; i++) { 2022 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, i); 2023 ep_ctx->ep_info = 0; 2024 ep_ctx->ep_info2 = 0; 2025 ep_ctx->deq = 0; 2026 ep_ctx->tx_info = 0; 2027 } 2028 } 2029 2030 static int xhci_configure_endpoint_result(struct xhci_hcd *xhci, 2031 struct usb_device *udev, u32 *cmd_status) 2032 { 2033 int ret; 2034 2035 switch (*cmd_status) { 2036 case COMP_COMMAND_ABORTED: 2037 case COMP_COMMAND_RING_STOPPED: 2038 xhci_warn(xhci, "Timeout while waiting for configure endpoint command\n"); 2039 ret = -ETIME; 2040 break; 2041 case COMP_RESOURCE_ERROR: 2042 dev_warn(&udev->dev, 2043 "Not enough host controller resources for new device state.\n"); 2044 ret = -ENOMEM; 2045 /* FIXME: can we allocate more resources for the HC? */ 2046 break; 2047 case COMP_BANDWIDTH_ERROR: 2048 case COMP_SECONDARY_BANDWIDTH_ERROR: 2049 dev_warn(&udev->dev, 2050 "Not enough bandwidth for new device state.\n"); 2051 ret = -ENOSPC; 2052 /* FIXME: can we go back to the old state? */ 2053 break; 2054 case COMP_TRB_ERROR: 2055 /* the HCD set up something wrong */ 2056 dev_warn(&udev->dev, "ERROR: Endpoint drop flag = 0, " 2057 "add flag = 1, " 2058 "and endpoint is not disabled.\n"); 2059 ret = -EINVAL; 2060 break; 2061 case COMP_INCOMPATIBLE_DEVICE_ERROR: 2062 dev_warn(&udev->dev, 2063 "ERROR: Incompatible device for endpoint configure command.\n"); 2064 ret = -ENODEV; 2065 break; 2066 case COMP_SUCCESS: 2067 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change, 2068 "Successful Endpoint Configure command"); 2069 ret = 0; 2070 break; 2071 default: 2072 xhci_err(xhci, "ERROR: unexpected command completion code 0x%x.\n", 2073 *cmd_status); 2074 ret = -EINVAL; 2075 break; 2076 } 2077 return ret; 2078 } 2079 2080 static int xhci_evaluate_context_result(struct xhci_hcd *xhci, 2081 struct usb_device *udev, u32 *cmd_status) 2082 { 2083 int ret; 2084 2085 switch (*cmd_status) { 2086 case COMP_COMMAND_ABORTED: 2087 case COMP_COMMAND_RING_STOPPED: 2088 xhci_warn(xhci, "Timeout while waiting for evaluate context command\n"); 2089 ret = -ETIME; 2090 break; 2091 case COMP_PARAMETER_ERROR: 2092 dev_warn(&udev->dev, 2093 "WARN: xHCI driver setup invalid evaluate context command.\n"); 2094 ret = -EINVAL; 2095 break; 2096 case COMP_SLOT_NOT_ENABLED_ERROR: 2097 dev_warn(&udev->dev, 2098 "WARN: slot not enabled for evaluate context command.\n"); 2099 ret = -EINVAL; 2100 break; 2101 case COMP_CONTEXT_STATE_ERROR: 2102 dev_warn(&udev->dev, 2103 "WARN: invalid context state for evaluate context command.\n"); 2104 ret = -EINVAL; 2105 break; 2106 case COMP_INCOMPATIBLE_DEVICE_ERROR: 2107 dev_warn(&udev->dev, 2108 "ERROR: Incompatible device for evaluate context command.\n"); 2109 ret = -ENODEV; 2110 break; 2111 case COMP_MAX_EXIT_LATENCY_TOO_LARGE_ERROR: 2112 /* Max Exit Latency too large error */ 2113 dev_warn(&udev->dev, "WARN: Max Exit Latency too large\n"); 2114 ret = -EINVAL; 2115 break; 2116 case COMP_SUCCESS: 2117 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change, 2118 "Successful evaluate context command"); 2119 ret = 0; 2120 break; 2121 default: 2122 xhci_err(xhci, "ERROR: unexpected command completion code 0x%x.\n", 2123 *cmd_status); 2124 ret = -EINVAL; 2125 break; 2126 } 2127 return ret; 2128 } 2129 2130 static u32 xhci_count_num_new_endpoints(struct xhci_hcd *xhci, 2131 struct xhci_input_control_ctx *ctrl_ctx) 2132 { 2133 u32 valid_add_flags; 2134 u32 valid_drop_flags; 2135 2136 /* Ignore the slot flag (bit 0), and the default control endpoint flag 2137 * (bit 1). The default control endpoint is added during the Address 2138 * Device command and is never removed until the slot is disabled. 2139 */ 2140 valid_add_flags = le32_to_cpu(ctrl_ctx->add_flags) >> 2; 2141 valid_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags) >> 2; 2142 2143 /* Use hweight32 to count the number of ones in the add flags, or 2144 * number of endpoints added. Don't count endpoints that are changed 2145 * (both added and dropped). 2146 */ 2147 return hweight32(valid_add_flags) - 2148 hweight32(valid_add_flags & valid_drop_flags); 2149 } 2150 2151 static unsigned int xhci_count_num_dropped_endpoints(struct xhci_hcd *xhci, 2152 struct xhci_input_control_ctx *ctrl_ctx) 2153 { 2154 u32 valid_add_flags; 2155 u32 valid_drop_flags; 2156 2157 valid_add_flags = le32_to_cpu(ctrl_ctx->add_flags) >> 2; 2158 valid_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags) >> 2; 2159 2160 return hweight32(valid_drop_flags) - 2161 hweight32(valid_add_flags & valid_drop_flags); 2162 } 2163 2164 /* 2165 * We need to reserve the new number of endpoints before the configure endpoint 2166 * command completes. We can't subtract the dropped endpoints from the number 2167 * of active endpoints until the command completes because we can oversubscribe 2168 * the host in this case: 2169 * 2170 * - the first configure endpoint command drops more endpoints than it adds 2171 * - a second configure endpoint command that adds more endpoints is queued 2172 * - the first configure endpoint command fails, so the config is unchanged 2173 * - the second command may succeed, even though there isn't enough resources 2174 * 2175 * Must be called with xhci->lock held. 2176 */ 2177 static int xhci_reserve_host_resources(struct xhci_hcd *xhci, 2178 struct xhci_input_control_ctx *ctrl_ctx) 2179 { 2180 u32 added_eps; 2181 2182 added_eps = xhci_count_num_new_endpoints(xhci, ctrl_ctx); 2183 if (xhci->num_active_eps + added_eps > xhci->limit_active_eps) { 2184 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, 2185 "Not enough ep ctxs: " 2186 "%u active, need to add %u, limit is %u.", 2187 xhci->num_active_eps, added_eps, 2188 xhci->limit_active_eps); 2189 return -ENOMEM; 2190 } 2191 xhci->num_active_eps += added_eps; 2192 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, 2193 "Adding %u ep ctxs, %u now active.", added_eps, 2194 xhci->num_active_eps); 2195 return 0; 2196 } 2197 2198 /* 2199 * The configure endpoint was failed by the xHC for some other reason, so we 2200 * need to revert the resources that failed configuration would have used. 2201 * 2202 * Must be called with xhci->lock held. 2203 */ 2204 static void xhci_free_host_resources(struct xhci_hcd *xhci, 2205 struct xhci_input_control_ctx *ctrl_ctx) 2206 { 2207 u32 num_failed_eps; 2208 2209 num_failed_eps = xhci_count_num_new_endpoints(xhci, ctrl_ctx); 2210 xhci->num_active_eps -= num_failed_eps; 2211 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, 2212 "Removing %u failed ep ctxs, %u now active.", 2213 num_failed_eps, 2214 xhci->num_active_eps); 2215 } 2216 2217 /* 2218 * Now that the command has completed, clean up the active endpoint count by 2219 * subtracting out the endpoints that were dropped (but not changed). 2220 * 2221 * Must be called with xhci->lock held. 2222 */ 2223 static void xhci_finish_resource_reservation(struct xhci_hcd *xhci, 2224 struct xhci_input_control_ctx *ctrl_ctx) 2225 { 2226 u32 num_dropped_eps; 2227 2228 num_dropped_eps = xhci_count_num_dropped_endpoints(xhci, ctrl_ctx); 2229 xhci->num_active_eps -= num_dropped_eps; 2230 if (num_dropped_eps) 2231 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, 2232 "Removing %u dropped ep ctxs, %u now active.", 2233 num_dropped_eps, 2234 xhci->num_active_eps); 2235 } 2236 2237 static unsigned int xhci_get_block_size(struct usb_device *udev) 2238 { 2239 switch (udev->speed) { 2240 case USB_SPEED_LOW: 2241 case USB_SPEED_FULL: 2242 return FS_BLOCK; 2243 case USB_SPEED_HIGH: 2244 return HS_BLOCK; 2245 case USB_SPEED_SUPER: 2246 case USB_SPEED_SUPER_PLUS: 2247 return SS_BLOCK; 2248 case USB_SPEED_UNKNOWN: 2249 default: 2250 /* Should never happen */ 2251 return 1; 2252 } 2253 } 2254 2255 static unsigned int 2256 xhci_get_largest_overhead(struct xhci_interval_bw *interval_bw) 2257 { 2258 if (interval_bw->overhead[LS_OVERHEAD_TYPE]) 2259 return LS_OVERHEAD; 2260 if (interval_bw->overhead[FS_OVERHEAD_TYPE]) 2261 return FS_OVERHEAD; 2262 return HS_OVERHEAD; 2263 } 2264 2265 /* If we are changing a LS/FS device under a HS hub, 2266 * make sure (if we are activating a new TT) that the HS bus has enough 2267 * bandwidth for this new TT. 2268 */ 2269 static int xhci_check_tt_bw_table(struct xhci_hcd *xhci, 2270 struct xhci_virt_device *virt_dev, 2271 int old_active_eps) 2272 { 2273 struct xhci_interval_bw_table *bw_table; 2274 struct xhci_tt_bw_info *tt_info; 2275 2276 /* Find the bandwidth table for the root port this TT is attached to. */ 2277 bw_table = &xhci->rh_bw[virt_dev->rhub_port->hw_portnum].bw_table; 2278 tt_info = virt_dev->tt_info; 2279 /* If this TT already had active endpoints, the bandwidth for this TT 2280 * has already been added. Removing all periodic endpoints (and thus 2281 * making the TT enactive) will only decrease the bandwidth used. 2282 */ 2283 if (old_active_eps) 2284 return 0; 2285 if (old_active_eps == 0 && tt_info->active_eps != 0) { 2286 if (bw_table->bw_used + TT_HS_OVERHEAD > HS_BW_LIMIT) 2287 return -ENOMEM; 2288 return 0; 2289 } 2290 /* Not sure why we would have no new active endpoints... 2291 * 2292 * Maybe because of an Evaluate Context change for a hub update or a 2293 * control endpoint 0 max packet size change? 2294 * FIXME: skip the bandwidth calculation in that case. 2295 */ 2296 return 0; 2297 } 2298 2299 static int xhci_check_ss_bw(struct xhci_hcd *xhci, 2300 struct xhci_virt_device *virt_dev) 2301 { 2302 unsigned int bw_reserved; 2303 2304 bw_reserved = DIV_ROUND_UP(SS_BW_RESERVED*SS_BW_LIMIT_IN, 100); 2305 if (virt_dev->bw_table->ss_bw_in > (SS_BW_LIMIT_IN - bw_reserved)) 2306 return -ENOMEM; 2307 2308 bw_reserved = DIV_ROUND_UP(SS_BW_RESERVED*SS_BW_LIMIT_OUT, 100); 2309 if (virt_dev->bw_table->ss_bw_out > (SS_BW_LIMIT_OUT - bw_reserved)) 2310 return -ENOMEM; 2311 2312 return 0; 2313 } 2314 2315 /* 2316 * This algorithm is a very conservative estimate of the worst-case scheduling 2317 * scenario for any one interval. The hardware dynamically schedules the 2318 * packets, so we can't tell which microframe could be the limiting factor in 2319 * the bandwidth scheduling. This only takes into account periodic endpoints. 2320 * 2321 * Obviously, we can't solve an NP complete problem to find the minimum worst 2322 * case scenario. Instead, we come up with an estimate that is no less than 2323 * the worst case bandwidth used for any one microframe, but may be an 2324 * over-estimate. 2325 * 2326 * We walk the requirements for each endpoint by interval, starting with the 2327 * smallest interval, and place packets in the schedule where there is only one 2328 * possible way to schedule packets for that interval. In order to simplify 2329 * this algorithm, we record the largest max packet size for each interval, and 2330 * assume all packets will be that size. 2331 * 2332 * For interval 0, we obviously must schedule all packets for each interval. 2333 * The bandwidth for interval 0 is just the amount of data to be transmitted 2334 * (the sum of all max ESIT payload sizes, plus any overhead per packet times 2335 * the number of packets). 2336 * 2337 * For interval 1, we have two possible microframes to schedule those packets 2338 * in. For this algorithm, if we can schedule the same number of packets for 2339 * each possible scheduling opportunity (each microframe), we will do so. The 2340 * remaining number of packets will be saved to be transmitted in the gaps in 2341 * the next interval's scheduling sequence. 2342 * 2343 * As we move those remaining packets to be scheduled with interval 2 packets, 2344 * we have to double the number of remaining packets to transmit. This is 2345 * because the intervals are actually powers of 2, and we would be transmitting 2346 * the previous interval's packets twice in this interval. We also have to be 2347 * sure that when we look at the largest max packet size for this interval, we 2348 * also look at the largest max packet size for the remaining packets and take 2349 * the greater of the two. 2350 * 2351 * The algorithm continues to evenly distribute packets in each scheduling 2352 * opportunity, and push the remaining packets out, until we get to the last 2353 * interval. Then those packets and their associated overhead are just added 2354 * to the bandwidth used. 2355 */ 2356 static int xhci_check_bw_table(struct xhci_hcd *xhci, 2357 struct xhci_virt_device *virt_dev, 2358 int old_active_eps) 2359 { 2360 unsigned int bw_reserved; 2361 unsigned int max_bandwidth; 2362 unsigned int bw_used; 2363 unsigned int block_size; 2364 struct xhci_interval_bw_table *bw_table; 2365 unsigned int packet_size = 0; 2366 unsigned int overhead = 0; 2367 unsigned int packets_transmitted = 0; 2368 unsigned int packets_remaining = 0; 2369 unsigned int i; 2370 2371 if (virt_dev->udev->speed >= USB_SPEED_SUPER) 2372 return xhci_check_ss_bw(xhci, virt_dev); 2373 2374 if (virt_dev->udev->speed == USB_SPEED_HIGH) { 2375 max_bandwidth = HS_BW_LIMIT; 2376 /* Convert percent of bus BW reserved to blocks reserved */ 2377 bw_reserved = DIV_ROUND_UP(HS_BW_RESERVED * max_bandwidth, 100); 2378 } else { 2379 max_bandwidth = FS_BW_LIMIT; 2380 bw_reserved = DIV_ROUND_UP(FS_BW_RESERVED * max_bandwidth, 100); 2381 } 2382 2383 bw_table = virt_dev->bw_table; 2384 /* We need to translate the max packet size and max ESIT payloads into 2385 * the units the hardware uses. 2386 */ 2387 block_size = xhci_get_block_size(virt_dev->udev); 2388 2389 /* If we are manipulating a LS/FS device under a HS hub, double check 2390 * that the HS bus has enough bandwidth if we are activing a new TT. 2391 */ 2392 if (virt_dev->tt_info) { 2393 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, 2394 "Recalculating BW for rootport %u", 2395 virt_dev->rhub_port->hw_portnum + 1); 2396 if (xhci_check_tt_bw_table(xhci, virt_dev, old_active_eps)) { 2397 xhci_warn(xhci, "Not enough bandwidth on HS bus for " 2398 "newly activated TT.\n"); 2399 return -ENOMEM; 2400 } 2401 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, 2402 "Recalculating BW for TT slot %u port %u", 2403 virt_dev->tt_info->slot_id, 2404 virt_dev->tt_info->ttport); 2405 } else { 2406 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, 2407 "Recalculating BW for rootport %u", 2408 virt_dev->rhub_port->hw_portnum + 1); 2409 } 2410 2411 /* Add in how much bandwidth will be used for interval zero, or the 2412 * rounded max ESIT payload + number of packets * largest overhead. 2413 */ 2414 bw_used = DIV_ROUND_UP(bw_table->interval0_esit_payload, block_size) + 2415 bw_table->interval_bw[0].num_packets * 2416 xhci_get_largest_overhead(&bw_table->interval_bw[0]); 2417 2418 for (i = 1; i < XHCI_MAX_INTERVAL; i++) { 2419 unsigned int bw_added; 2420 unsigned int largest_mps; 2421 unsigned int interval_overhead; 2422 2423 /* 2424 * How many packets could we transmit in this interval? 2425 * If packets didn't fit in the previous interval, we will need 2426 * to transmit that many packets twice within this interval. 2427 */ 2428 packets_remaining = 2 * packets_remaining + 2429 bw_table->interval_bw[i].num_packets; 2430 2431 /* Find the largest max packet size of this or the previous 2432 * interval. 2433 */ 2434 if (list_empty(&bw_table->interval_bw[i].endpoints)) 2435 largest_mps = 0; 2436 else { 2437 struct xhci_virt_ep *virt_ep; 2438 struct list_head *ep_entry; 2439 2440 ep_entry = bw_table->interval_bw[i].endpoints.next; 2441 virt_ep = list_entry(ep_entry, 2442 struct xhci_virt_ep, bw_endpoint_list); 2443 /* Convert to blocks, rounding up */ 2444 largest_mps = DIV_ROUND_UP( 2445 virt_ep->bw_info.max_packet_size, 2446 block_size); 2447 } 2448 if (largest_mps > packet_size) 2449 packet_size = largest_mps; 2450 2451 /* Use the larger overhead of this or the previous interval. */ 2452 interval_overhead = xhci_get_largest_overhead( 2453 &bw_table->interval_bw[i]); 2454 if (interval_overhead > overhead) 2455 overhead = interval_overhead; 2456 2457 /* How many packets can we evenly distribute across 2458 * (1 << (i + 1)) possible scheduling opportunities? 2459 */ 2460 packets_transmitted = packets_remaining >> (i + 1); 2461 2462 /* Add in the bandwidth used for those scheduled packets */ 2463 bw_added = packets_transmitted * (overhead + packet_size); 2464 2465 /* How many packets do we have remaining to transmit? */ 2466 packets_remaining = packets_remaining % (1 << (i + 1)); 2467 2468 /* What largest max packet size should those packets have? */ 2469 /* If we've transmitted all packets, don't carry over the 2470 * largest packet size. 2471 */ 2472 if (packets_remaining == 0) { 2473 packet_size = 0; 2474 overhead = 0; 2475 } else if (packets_transmitted > 0) { 2476 /* Otherwise if we do have remaining packets, and we've 2477 * scheduled some packets in this interval, take the 2478 * largest max packet size from endpoints with this 2479 * interval. 2480 */ 2481 packet_size = largest_mps; 2482 overhead = interval_overhead; 2483 } 2484 /* Otherwise carry over packet_size and overhead from the last 2485 * time we had a remainder. 2486 */ 2487 bw_used += bw_added; 2488 if (bw_used > max_bandwidth) { 2489 xhci_warn(xhci, "Not enough bandwidth. " 2490 "Proposed: %u, Max: %u\n", 2491 bw_used, max_bandwidth); 2492 return -ENOMEM; 2493 } 2494 } 2495 /* 2496 * Ok, we know we have some packets left over after even-handedly 2497 * scheduling interval 15. We don't know which microframes they will 2498 * fit into, so we over-schedule and say they will be scheduled every 2499 * microframe. 2500 */ 2501 if (packets_remaining > 0) 2502 bw_used += overhead + packet_size; 2503 2504 if (!virt_dev->tt_info && virt_dev->udev->speed == USB_SPEED_HIGH) { 2505 /* OK, we're manipulating a HS device attached to a 2506 * root port bandwidth domain. Include the number of active TTs 2507 * in the bandwidth used. 2508 */ 2509 bw_used += TT_HS_OVERHEAD * 2510 xhci->rh_bw[virt_dev->rhub_port->hw_portnum].num_active_tts; 2511 } 2512 2513 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, 2514 "Final bandwidth: %u, Limit: %u, Reserved: %u, " 2515 "Available: %u " "percent", 2516 bw_used, max_bandwidth, bw_reserved, 2517 (max_bandwidth - bw_used - bw_reserved) * 100 / 2518 max_bandwidth); 2519 2520 bw_used += bw_reserved; 2521 if (bw_used > max_bandwidth) { 2522 xhci_warn(xhci, "Not enough bandwidth. Proposed: %u, Max: %u\n", 2523 bw_used, max_bandwidth); 2524 return -ENOMEM; 2525 } 2526 2527 bw_table->bw_used = bw_used; 2528 return 0; 2529 } 2530 2531 static bool xhci_is_async_ep(unsigned int ep_type) 2532 { 2533 return (ep_type != ISOC_OUT_EP && ep_type != INT_OUT_EP && 2534 ep_type != ISOC_IN_EP && 2535 ep_type != INT_IN_EP); 2536 } 2537 2538 static bool xhci_is_sync_in_ep(unsigned int ep_type) 2539 { 2540 return (ep_type == ISOC_IN_EP || ep_type == INT_IN_EP); 2541 } 2542 2543 static unsigned int xhci_get_ss_bw_consumed(struct xhci_bw_info *ep_bw) 2544 { 2545 unsigned int mps = DIV_ROUND_UP(ep_bw->max_packet_size, SS_BLOCK); 2546 2547 if (ep_bw->ep_interval == 0) 2548 return SS_OVERHEAD_BURST + 2549 (ep_bw->mult * ep_bw->num_packets * 2550 (SS_OVERHEAD + mps)); 2551 return DIV_ROUND_UP(ep_bw->mult * ep_bw->num_packets * 2552 (SS_OVERHEAD + mps + SS_OVERHEAD_BURST), 2553 1 << ep_bw->ep_interval); 2554 2555 } 2556 2557 static void xhci_drop_ep_from_interval_table(struct xhci_hcd *xhci, 2558 struct xhci_bw_info *ep_bw, 2559 struct xhci_interval_bw_table *bw_table, 2560 struct usb_device *udev, 2561 struct xhci_virt_ep *virt_ep, 2562 struct xhci_tt_bw_info *tt_info) 2563 { 2564 struct xhci_interval_bw *interval_bw; 2565 int normalized_interval; 2566 2567 if (xhci_is_async_ep(ep_bw->type)) 2568 return; 2569 2570 if (udev->speed >= USB_SPEED_SUPER) { 2571 if (xhci_is_sync_in_ep(ep_bw->type)) 2572 xhci->devs[udev->slot_id]->bw_table->ss_bw_in -= 2573 xhci_get_ss_bw_consumed(ep_bw); 2574 else 2575 xhci->devs[udev->slot_id]->bw_table->ss_bw_out -= 2576 xhci_get_ss_bw_consumed(ep_bw); 2577 return; 2578 } 2579 2580 /* SuperSpeed endpoints never get added to intervals in the table, so 2581 * this check is only valid for HS/FS/LS devices. 2582 */ 2583 if (list_empty(&virt_ep->bw_endpoint_list)) 2584 return; 2585 /* For LS/FS devices, we need to translate the interval expressed in 2586 * microframes to frames. 2587 */ 2588 if (udev->speed == USB_SPEED_HIGH) 2589 normalized_interval = ep_bw->ep_interval; 2590 else 2591 normalized_interval = ep_bw->ep_interval - 3; 2592 2593 if (normalized_interval == 0) 2594 bw_table->interval0_esit_payload -= ep_bw->max_esit_payload; 2595 interval_bw = &bw_table->interval_bw[normalized_interval]; 2596 interval_bw->num_packets -= ep_bw->num_packets; 2597 switch (udev->speed) { 2598 case USB_SPEED_LOW: 2599 interval_bw->overhead[LS_OVERHEAD_TYPE] -= 1; 2600 break; 2601 case USB_SPEED_FULL: 2602 interval_bw->overhead[FS_OVERHEAD_TYPE] -= 1; 2603 break; 2604 case USB_SPEED_HIGH: 2605 interval_bw->overhead[HS_OVERHEAD_TYPE] -= 1; 2606 break; 2607 default: 2608 /* Should never happen because only LS/FS/HS endpoints will get 2609 * added to the endpoint list. 2610 */ 2611 return; 2612 } 2613 if (tt_info) 2614 tt_info->active_eps -= 1; 2615 list_del_init(&virt_ep->bw_endpoint_list); 2616 } 2617 2618 static void xhci_add_ep_to_interval_table(struct xhci_hcd *xhci, 2619 struct xhci_bw_info *ep_bw, 2620 struct xhci_interval_bw_table *bw_table, 2621 struct usb_device *udev, 2622 struct xhci_virt_ep *virt_ep, 2623 struct xhci_tt_bw_info *tt_info) 2624 { 2625 struct xhci_interval_bw *interval_bw; 2626 struct xhci_virt_ep *smaller_ep; 2627 int normalized_interval; 2628 2629 if (xhci_is_async_ep(ep_bw->type)) 2630 return; 2631 2632 if (udev->speed == USB_SPEED_SUPER) { 2633 if (xhci_is_sync_in_ep(ep_bw->type)) 2634 xhci->devs[udev->slot_id]->bw_table->ss_bw_in += 2635 xhci_get_ss_bw_consumed(ep_bw); 2636 else 2637 xhci->devs[udev->slot_id]->bw_table->ss_bw_out += 2638 xhci_get_ss_bw_consumed(ep_bw); 2639 return; 2640 } 2641 2642 /* For LS/FS devices, we need to translate the interval expressed in 2643 * microframes to frames. 2644 */ 2645 if (udev->speed == USB_SPEED_HIGH) 2646 normalized_interval = ep_bw->ep_interval; 2647 else 2648 normalized_interval = ep_bw->ep_interval - 3; 2649 2650 if (normalized_interval == 0) 2651 bw_table->interval0_esit_payload += ep_bw->max_esit_payload; 2652 interval_bw = &bw_table->interval_bw[normalized_interval]; 2653 interval_bw->num_packets += ep_bw->num_packets; 2654 switch (udev->speed) { 2655 case USB_SPEED_LOW: 2656 interval_bw->overhead[LS_OVERHEAD_TYPE] += 1; 2657 break; 2658 case USB_SPEED_FULL: 2659 interval_bw->overhead[FS_OVERHEAD_TYPE] += 1; 2660 break; 2661 case USB_SPEED_HIGH: 2662 interval_bw->overhead[HS_OVERHEAD_TYPE] += 1; 2663 break; 2664 default: 2665 /* Should never happen because only LS/FS/HS endpoints will get 2666 * added to the endpoint list. 2667 */ 2668 return; 2669 } 2670 2671 if (tt_info) 2672 tt_info->active_eps += 1; 2673 /* Insert the endpoint into the list, largest max packet size first. */ 2674 list_for_each_entry(smaller_ep, &interval_bw->endpoints, 2675 bw_endpoint_list) { 2676 if (ep_bw->max_packet_size >= 2677 smaller_ep->bw_info.max_packet_size) { 2678 /* Add the new ep before the smaller endpoint */ 2679 list_add_tail(&virt_ep->bw_endpoint_list, 2680 &smaller_ep->bw_endpoint_list); 2681 return; 2682 } 2683 } 2684 /* Add the new endpoint at the end of the list. */ 2685 list_add_tail(&virt_ep->bw_endpoint_list, 2686 &interval_bw->endpoints); 2687 } 2688 2689 void xhci_update_tt_active_eps(struct xhci_hcd *xhci, 2690 struct xhci_virt_device *virt_dev, 2691 int old_active_eps) 2692 { 2693 struct xhci_root_port_bw_info *rh_bw_info; 2694 if (!virt_dev->tt_info) 2695 return; 2696 2697 rh_bw_info = &xhci->rh_bw[virt_dev->rhub_port->hw_portnum]; 2698 if (old_active_eps == 0 && 2699 virt_dev->tt_info->active_eps != 0) { 2700 rh_bw_info->num_active_tts += 1; 2701 rh_bw_info->bw_table.bw_used += TT_HS_OVERHEAD; 2702 } else if (old_active_eps != 0 && 2703 virt_dev->tt_info->active_eps == 0) { 2704 rh_bw_info->num_active_tts -= 1; 2705 rh_bw_info->bw_table.bw_used -= TT_HS_OVERHEAD; 2706 } 2707 } 2708 2709 static int xhci_reserve_bandwidth(struct xhci_hcd *xhci, 2710 struct xhci_virt_device *virt_dev, 2711 struct xhci_container_ctx *in_ctx) 2712 { 2713 struct xhci_bw_info ep_bw_info[31]; 2714 int i; 2715 struct xhci_input_control_ctx *ctrl_ctx; 2716 int old_active_eps = 0; 2717 2718 if (virt_dev->tt_info) 2719 old_active_eps = virt_dev->tt_info->active_eps; 2720 2721 ctrl_ctx = xhci_get_input_control_ctx(in_ctx); 2722 if (!ctrl_ctx) { 2723 xhci_warn(xhci, "%s: Could not get input context, bad type.\n", 2724 __func__); 2725 return -ENOMEM; 2726 } 2727 2728 for (i = 0; i < 31; i++) { 2729 if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i)) 2730 continue; 2731 2732 /* Make a copy of the BW info in case we need to revert this */ 2733 memcpy(&ep_bw_info[i], &virt_dev->eps[i].bw_info, 2734 sizeof(ep_bw_info[i])); 2735 /* Drop the endpoint from the interval table if the endpoint is 2736 * being dropped or changed. 2737 */ 2738 if (EP_IS_DROPPED(ctrl_ctx, i)) 2739 xhci_drop_ep_from_interval_table(xhci, 2740 &virt_dev->eps[i].bw_info, 2741 virt_dev->bw_table, 2742 virt_dev->udev, 2743 &virt_dev->eps[i], 2744 virt_dev->tt_info); 2745 } 2746 /* Overwrite the information stored in the endpoints' bw_info */ 2747 xhci_update_bw_info(xhci, virt_dev->in_ctx, ctrl_ctx, virt_dev); 2748 for (i = 0; i < 31; i++) { 2749 /* Add any changed or added endpoints to the interval table */ 2750 if (EP_IS_ADDED(ctrl_ctx, i)) 2751 xhci_add_ep_to_interval_table(xhci, 2752 &virt_dev->eps[i].bw_info, 2753 virt_dev->bw_table, 2754 virt_dev->udev, 2755 &virt_dev->eps[i], 2756 virt_dev->tt_info); 2757 } 2758 2759 if (!xhci_check_bw_table(xhci, virt_dev, old_active_eps)) { 2760 /* Ok, this fits in the bandwidth we have. 2761 * Update the number of active TTs. 2762 */ 2763 xhci_update_tt_active_eps(xhci, virt_dev, old_active_eps); 2764 return 0; 2765 } 2766 2767 /* We don't have enough bandwidth for this, revert the stored info. */ 2768 for (i = 0; i < 31; i++) { 2769 if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i)) 2770 continue; 2771 2772 /* Drop the new copies of any added or changed endpoints from 2773 * the interval table. 2774 */ 2775 if (EP_IS_ADDED(ctrl_ctx, i)) { 2776 xhci_drop_ep_from_interval_table(xhci, 2777 &virt_dev->eps[i].bw_info, 2778 virt_dev->bw_table, 2779 virt_dev->udev, 2780 &virt_dev->eps[i], 2781 virt_dev->tt_info); 2782 } 2783 /* Revert the endpoint back to its old information */ 2784 memcpy(&virt_dev->eps[i].bw_info, &ep_bw_info[i], 2785 sizeof(ep_bw_info[i])); 2786 /* Add any changed or dropped endpoints back into the table */ 2787 if (EP_IS_DROPPED(ctrl_ctx, i)) 2788 xhci_add_ep_to_interval_table(xhci, 2789 &virt_dev->eps[i].bw_info, 2790 virt_dev->bw_table, 2791 virt_dev->udev, 2792 &virt_dev->eps[i], 2793 virt_dev->tt_info); 2794 } 2795 return -ENOMEM; 2796 } 2797 2798 /* 2799 * Synchronous XHCI stop endpoint helper. Issues the stop endpoint command and 2800 * waits for the command completion before returning. This does not call 2801 * xhci_handle_cmd_stop_ep(), which has additional handling for 'context error' 2802 * cases, along with transfer ring cleanup. 2803 * 2804 * xhci_stop_endpoint_sync() is intended to be utilized by clients that manage 2805 * their own transfer ring, such as offload situations. 2806 */ 2807 int xhci_stop_endpoint_sync(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, int suspend, 2808 gfp_t gfp_flags) 2809 { 2810 struct xhci_command *command; 2811 unsigned long flags; 2812 int ret; 2813 2814 command = xhci_alloc_command(xhci, true, gfp_flags); 2815 if (!command) 2816 return -ENOMEM; 2817 2818 spin_lock_irqsave(&xhci->lock, flags); 2819 ret = xhci_queue_stop_endpoint(xhci, command, ep->vdev->slot_id, 2820 ep->ep_index, suspend); 2821 if (ret < 0) { 2822 spin_unlock_irqrestore(&xhci->lock, flags); 2823 goto out; 2824 } 2825 2826 xhci_ring_cmd_db(xhci); 2827 spin_unlock_irqrestore(&xhci->lock, flags); 2828 2829 wait_for_completion(command->completion); 2830 2831 /* No handling for COMP_CONTEXT_STATE_ERROR done at command completion*/ 2832 if (command->status == COMP_COMMAND_ABORTED || 2833 command->status == COMP_COMMAND_RING_STOPPED) { 2834 xhci_warn(xhci, "Timeout while waiting for stop endpoint command\n"); 2835 ret = -ETIME; 2836 } 2837 out: 2838 xhci_free_command(xhci, command); 2839 2840 return ret; 2841 } 2842 EXPORT_SYMBOL_GPL(xhci_stop_endpoint_sync); 2843 2844 /* Issue a configure endpoint command or evaluate context command 2845 * and wait for it to finish. 2846 */ 2847 static int xhci_configure_endpoint(struct xhci_hcd *xhci, 2848 struct usb_device *udev, 2849 struct xhci_command *command, 2850 bool ctx_change, bool must_succeed) 2851 { 2852 int ret; 2853 unsigned long flags; 2854 struct xhci_input_control_ctx *ctrl_ctx; 2855 struct xhci_virt_device *virt_dev; 2856 struct xhci_slot_ctx *slot_ctx; 2857 2858 if (!command) 2859 return -EINVAL; 2860 2861 spin_lock_irqsave(&xhci->lock, flags); 2862 2863 if (xhci->xhc_state & XHCI_STATE_DYING) { 2864 spin_unlock_irqrestore(&xhci->lock, flags); 2865 return -ESHUTDOWN; 2866 } 2867 2868 virt_dev = xhci->devs[udev->slot_id]; 2869 2870 ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx); 2871 if (!ctrl_ctx) { 2872 spin_unlock_irqrestore(&xhci->lock, flags); 2873 xhci_warn(xhci, "%s: Could not get input context, bad type.\n", 2874 __func__); 2875 return -ENOMEM; 2876 } 2877 2878 if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK) && 2879 xhci_reserve_host_resources(xhci, ctrl_ctx)) { 2880 spin_unlock_irqrestore(&xhci->lock, flags); 2881 xhci_warn(xhci, "Not enough host resources, " 2882 "active endpoint contexts = %u\n", 2883 xhci->num_active_eps); 2884 return -ENOMEM; 2885 } 2886 if ((xhci->quirks & XHCI_SW_BW_CHECKING) && !ctx_change && 2887 xhci_reserve_bandwidth(xhci, virt_dev, command->in_ctx)) { 2888 if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) 2889 xhci_free_host_resources(xhci, ctrl_ctx); 2890 spin_unlock_irqrestore(&xhci->lock, flags); 2891 xhci_warn(xhci, "Not enough bandwidth\n"); 2892 return -ENOMEM; 2893 } 2894 2895 slot_ctx = xhci_get_slot_ctx(xhci, command->in_ctx); 2896 2897 trace_xhci_configure_endpoint_ctrl_ctx(ctrl_ctx); 2898 trace_xhci_configure_endpoint(slot_ctx); 2899 2900 if (!ctx_change) 2901 ret = xhci_queue_configure_endpoint(xhci, command, 2902 command->in_ctx->dma, 2903 udev->slot_id, must_succeed); 2904 else 2905 ret = xhci_queue_evaluate_context(xhci, command, 2906 command->in_ctx->dma, 2907 udev->slot_id, must_succeed); 2908 if (ret < 0) { 2909 if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) 2910 xhci_free_host_resources(xhci, ctrl_ctx); 2911 spin_unlock_irqrestore(&xhci->lock, flags); 2912 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change, 2913 "FIXME allocate a new ring segment"); 2914 return -ENOMEM; 2915 } 2916 xhci_ring_cmd_db(xhci); 2917 spin_unlock_irqrestore(&xhci->lock, flags); 2918 2919 /* Wait for the configure endpoint command to complete */ 2920 wait_for_completion(command->completion); 2921 2922 if (!ctx_change) 2923 ret = xhci_configure_endpoint_result(xhci, udev, 2924 &command->status); 2925 else 2926 ret = xhci_evaluate_context_result(xhci, udev, 2927 &command->status); 2928 2929 if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) { 2930 spin_lock_irqsave(&xhci->lock, flags); 2931 /* If the command failed, remove the reserved resources. 2932 * Otherwise, clean up the estimate to include dropped eps. 2933 */ 2934 if (ret) 2935 xhci_free_host_resources(xhci, ctrl_ctx); 2936 else 2937 xhci_finish_resource_reservation(xhci, ctrl_ctx); 2938 spin_unlock_irqrestore(&xhci->lock, flags); 2939 } 2940 return ret; 2941 } 2942 2943 static void xhci_check_bw_drop_ep_streams(struct xhci_hcd *xhci, 2944 struct xhci_virt_device *vdev, int i) 2945 { 2946 struct xhci_virt_ep *ep = &vdev->eps[i]; 2947 2948 if (ep->ep_state & EP_HAS_STREAMS) { 2949 xhci_warn(xhci, "WARN: endpoint 0x%02x has streams on set_interface, freeing streams.\n", 2950 xhci_get_endpoint_address(i)); 2951 xhci_free_stream_info(xhci, ep->stream_info); 2952 ep->stream_info = NULL; 2953 ep->ep_state &= ~EP_HAS_STREAMS; 2954 } 2955 } 2956 2957 /* Called after one or more calls to xhci_add_endpoint() or 2958 * xhci_drop_endpoint(). If this call fails, the USB core is expected 2959 * to call xhci_reset_bandwidth(). 2960 * 2961 * Since we are in the middle of changing either configuration or 2962 * installing a new alt setting, the USB core won't allow URBs to be 2963 * enqueued for any endpoint on the old config or interface. Nothing 2964 * else should be touching the xhci->devs[slot_id] structure, so we 2965 * don't need to take the xhci->lock for manipulating that. 2966 */ 2967 int xhci_check_bandwidth(struct usb_hcd *hcd, struct usb_device *udev) 2968 { 2969 int i; 2970 int ret = 0; 2971 struct xhci_hcd *xhci; 2972 struct xhci_virt_device *virt_dev; 2973 struct xhci_input_control_ctx *ctrl_ctx; 2974 struct xhci_slot_ctx *slot_ctx; 2975 struct xhci_command *command; 2976 2977 ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__); 2978 if (ret <= 0) 2979 return ret; 2980 xhci = hcd_to_xhci(hcd); 2981 if ((xhci->xhc_state & XHCI_STATE_DYING) || 2982 (xhci->xhc_state & XHCI_STATE_REMOVING)) 2983 return -ENODEV; 2984 2985 xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev); 2986 virt_dev = xhci->devs[udev->slot_id]; 2987 2988 command = xhci_alloc_command(xhci, true, GFP_KERNEL); 2989 if (!command) 2990 return -ENOMEM; 2991 2992 command->in_ctx = virt_dev->in_ctx; 2993 2994 /* See section 4.6.6 - A0 = 1; A1 = D0 = D1 = 0 */ 2995 ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx); 2996 if (!ctrl_ctx) { 2997 xhci_warn(xhci, "%s: Could not get input context, bad type.\n", 2998 __func__); 2999 ret = -ENOMEM; 3000 goto command_cleanup; 3001 } 3002 ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG); 3003 ctrl_ctx->add_flags &= cpu_to_le32(~EP0_FLAG); 3004 ctrl_ctx->drop_flags &= cpu_to_le32(~(SLOT_FLAG | EP0_FLAG)); 3005 3006 /* Don't issue the command if there's no endpoints to update. */ 3007 if (ctrl_ctx->add_flags == cpu_to_le32(SLOT_FLAG) && 3008 ctrl_ctx->drop_flags == 0) { 3009 ret = 0; 3010 goto command_cleanup; 3011 } 3012 /* Fix up Context Entries field. Minimum value is EP0 == BIT(1). */ 3013 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx); 3014 for (i = 31; i >= 1; i--) { 3015 __le32 le32 = cpu_to_le32(BIT(i)); 3016 3017 if ((virt_dev->eps[i-1].ring && !(ctrl_ctx->drop_flags & le32)) 3018 || (ctrl_ctx->add_flags & le32) || i == 1) { 3019 slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK); 3020 slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(i)); 3021 break; 3022 } 3023 } 3024 3025 ret = xhci_configure_endpoint(xhci, udev, command, 3026 false, false); 3027 if (ret) 3028 /* Callee should call reset_bandwidth() */ 3029 goto command_cleanup; 3030 3031 /* Free any rings that were dropped, but not changed. */ 3032 for (i = 1; i < 31; i++) { 3033 if ((le32_to_cpu(ctrl_ctx->drop_flags) & (1 << (i + 1))) && 3034 !(le32_to_cpu(ctrl_ctx->add_flags) & (1 << (i + 1)))) { 3035 xhci_free_endpoint_ring(xhci, virt_dev, i); 3036 xhci_check_bw_drop_ep_streams(xhci, virt_dev, i); 3037 } 3038 } 3039 xhci_zero_in_ctx(xhci, virt_dev); 3040 /* 3041 * Install any rings for completely new endpoints or changed endpoints, 3042 * and free any old rings from changed endpoints. 3043 */ 3044 for (i = 1; i < 31; i++) { 3045 if (!virt_dev->eps[i].new_ring) 3046 continue; 3047 /* Only free the old ring if it exists. 3048 * It may not if this is the first add of an endpoint. 3049 */ 3050 if (virt_dev->eps[i].ring) { 3051 xhci_free_endpoint_ring(xhci, virt_dev, i); 3052 } 3053 xhci_check_bw_drop_ep_streams(xhci, virt_dev, i); 3054 virt_dev->eps[i].ring = virt_dev->eps[i].new_ring; 3055 virt_dev->eps[i].new_ring = NULL; 3056 xhci_debugfs_create_endpoint(xhci, virt_dev, i); 3057 } 3058 command_cleanup: 3059 kfree(command->completion); 3060 kfree(command); 3061 3062 return ret; 3063 } 3064 EXPORT_SYMBOL_GPL(xhci_check_bandwidth); 3065 3066 void xhci_reset_bandwidth(struct usb_hcd *hcd, struct usb_device *udev) 3067 { 3068 struct xhci_hcd *xhci; 3069 struct xhci_virt_device *virt_dev; 3070 int i, ret; 3071 3072 ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__); 3073 if (ret <= 0) 3074 return; 3075 xhci = hcd_to_xhci(hcd); 3076 3077 xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev); 3078 virt_dev = xhci->devs[udev->slot_id]; 3079 /* Free any rings allocated for added endpoints */ 3080 for (i = 0; i < 31; i++) { 3081 if (virt_dev->eps[i].new_ring) { 3082 xhci_debugfs_remove_endpoint(xhci, virt_dev, i); 3083 xhci_ring_free(xhci, virt_dev->eps[i].new_ring); 3084 virt_dev->eps[i].new_ring = NULL; 3085 } 3086 } 3087 xhci_zero_in_ctx(xhci, virt_dev); 3088 } 3089 EXPORT_SYMBOL_GPL(xhci_reset_bandwidth); 3090 3091 static void xhci_setup_input_ctx_for_config_ep(struct xhci_hcd *xhci, 3092 struct xhci_container_ctx *in_ctx, 3093 struct xhci_container_ctx *out_ctx, 3094 struct xhci_input_control_ctx *ctrl_ctx, 3095 u32 add_flags, u32 drop_flags) 3096 { 3097 ctrl_ctx->add_flags = cpu_to_le32(add_flags); 3098 ctrl_ctx->drop_flags = cpu_to_le32(drop_flags); 3099 xhci_slot_copy(xhci, in_ctx, out_ctx); 3100 ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG); 3101 } 3102 3103 static void xhci_endpoint_disable(struct usb_hcd *hcd, 3104 struct usb_host_endpoint *host_ep) 3105 { 3106 struct xhci_hcd *xhci; 3107 struct xhci_virt_device *vdev; 3108 struct xhci_virt_ep *ep; 3109 struct usb_device *udev; 3110 unsigned long flags; 3111 unsigned int ep_index; 3112 3113 xhci = hcd_to_xhci(hcd); 3114 rescan: 3115 spin_lock_irqsave(&xhci->lock, flags); 3116 3117 udev = (struct usb_device *)host_ep->hcpriv; 3118 if (!udev || !udev->slot_id) 3119 goto done; 3120 3121 vdev = xhci->devs[udev->slot_id]; 3122 if (!vdev) 3123 goto done; 3124 3125 ep_index = xhci_get_endpoint_index(&host_ep->desc); 3126 ep = &vdev->eps[ep_index]; 3127 3128 /* wait for hub_tt_work to finish clearing hub TT */ 3129 if (ep->ep_state & EP_CLEARING_TT) { 3130 spin_unlock_irqrestore(&xhci->lock, flags); 3131 schedule_timeout_uninterruptible(1); 3132 goto rescan; 3133 } 3134 3135 if (ep->ep_state) 3136 xhci_dbg(xhci, "endpoint disable with ep_state 0x%x\n", 3137 ep->ep_state); 3138 done: 3139 host_ep->hcpriv = NULL; 3140 spin_unlock_irqrestore(&xhci->lock, flags); 3141 } 3142 3143 /* 3144 * Called after usb core issues a clear halt control message. 3145 * The host side of the halt should already be cleared by a reset endpoint 3146 * command issued when the STALL event was received. 3147 * 3148 * The reset endpoint command may only be issued to endpoints in the halted 3149 * state. For software that wishes to reset the data toggle or sequence number 3150 * of an endpoint that isn't in the halted state this function will issue a 3151 * configure endpoint command with the Drop and Add bits set for the target 3152 * endpoint. Refer to the additional note in xhci spcification section 4.6.8. 3153 * 3154 * vdev may be lost due to xHC restore error and re-initialization during S3/S4 3155 * resume. A new vdev will be allocated later by xhci_discover_or_reset_device() 3156 */ 3157 3158 static void xhci_endpoint_reset(struct usb_hcd *hcd, 3159 struct usb_host_endpoint *host_ep) 3160 { 3161 struct xhci_hcd *xhci; 3162 struct usb_device *udev; 3163 struct xhci_virt_device *vdev; 3164 struct xhci_virt_ep *ep; 3165 struct xhci_input_control_ctx *ctrl_ctx; 3166 struct xhci_command *stop_cmd, *cfg_cmd; 3167 unsigned int ep_index; 3168 unsigned long flags; 3169 u32 ep_flag; 3170 int err; 3171 3172 xhci = hcd_to_xhci(hcd); 3173 ep_index = xhci_get_endpoint_index(&host_ep->desc); 3174 3175 /* 3176 * Usb core assumes a max packet value for ep0 on FS devices until the 3177 * real value is read from the descriptor. Core resets Ep0 if values 3178 * mismatch. Reconfigure the xhci ep0 endpoint context here in that case 3179 */ 3180 if (usb_endpoint_xfer_control(&host_ep->desc) && ep_index == 0) { 3181 3182 udev = container_of(host_ep, struct usb_device, ep0); 3183 if (udev->speed != USB_SPEED_FULL || !udev->slot_id) 3184 return; 3185 3186 vdev = xhci->devs[udev->slot_id]; 3187 if (!vdev || vdev->udev != udev) 3188 return; 3189 3190 xhci_check_ep0_maxpacket(xhci, vdev); 3191 3192 /* Nothing else should be done here for ep0 during ep reset */ 3193 return; 3194 } 3195 3196 if (!host_ep->hcpriv) 3197 return; 3198 udev = (struct usb_device *) host_ep->hcpriv; 3199 vdev = xhci->devs[udev->slot_id]; 3200 3201 if (!udev->slot_id || !vdev) 3202 return; 3203 3204 ep = &vdev->eps[ep_index]; 3205 3206 /* Bail out if toggle is already being cleared by a endpoint reset */ 3207 spin_lock_irqsave(&xhci->lock, flags); 3208 if (ep->ep_state & EP_HARD_CLEAR_TOGGLE) { 3209 ep->ep_state &= ~EP_HARD_CLEAR_TOGGLE; 3210 spin_unlock_irqrestore(&xhci->lock, flags); 3211 return; 3212 } 3213 spin_unlock_irqrestore(&xhci->lock, flags); 3214 /* Only interrupt and bulk ep's use data toggle, USB2 spec 5.5.4-> */ 3215 if (usb_endpoint_xfer_control(&host_ep->desc) || 3216 usb_endpoint_xfer_isoc(&host_ep->desc)) 3217 return; 3218 3219 ep_flag = xhci_get_endpoint_flag(&host_ep->desc); 3220 3221 if (ep_flag == SLOT_FLAG || ep_flag == EP0_FLAG) 3222 return; 3223 3224 stop_cmd = xhci_alloc_command(xhci, true, GFP_NOWAIT); 3225 if (!stop_cmd) 3226 return; 3227 3228 cfg_cmd = xhci_alloc_command_with_ctx(xhci, true, GFP_NOWAIT); 3229 if (!cfg_cmd) 3230 goto cleanup; 3231 3232 spin_lock_irqsave(&xhci->lock, flags); 3233 3234 /* block queuing new trbs and ringing ep doorbell */ 3235 ep->ep_state |= EP_SOFT_CLEAR_TOGGLE; 3236 3237 /* 3238 * Make sure endpoint ring is empty before resetting the toggle/seq. 3239 * Driver is required to synchronously cancel all transfer request. 3240 * Stop the endpoint to force xHC to update the output context 3241 */ 3242 3243 if (!list_empty(&ep->ring->td_list)) { 3244 dev_err(&udev->dev, "EP not empty, refuse reset\n"); 3245 spin_unlock_irqrestore(&xhci->lock, flags); 3246 xhci_free_command(xhci, cfg_cmd); 3247 goto cleanup; 3248 } 3249 3250 err = xhci_queue_stop_endpoint(xhci, stop_cmd, udev->slot_id, 3251 ep_index, 0); 3252 if (err < 0) { 3253 spin_unlock_irqrestore(&xhci->lock, flags); 3254 xhci_free_command(xhci, cfg_cmd); 3255 xhci_dbg(xhci, "%s: Failed to queue stop ep command, %d ", 3256 __func__, err); 3257 goto cleanup; 3258 } 3259 3260 xhci_ring_cmd_db(xhci); 3261 spin_unlock_irqrestore(&xhci->lock, flags); 3262 3263 wait_for_completion(stop_cmd->completion); 3264 3265 spin_lock_irqsave(&xhci->lock, flags); 3266 3267 /* config ep command clears toggle if add and drop ep flags are set */ 3268 ctrl_ctx = xhci_get_input_control_ctx(cfg_cmd->in_ctx); 3269 if (!ctrl_ctx) { 3270 spin_unlock_irqrestore(&xhci->lock, flags); 3271 xhci_free_command(xhci, cfg_cmd); 3272 xhci_warn(xhci, "%s: Could not get input context, bad type.\n", 3273 __func__); 3274 goto cleanup; 3275 } 3276 3277 xhci_setup_input_ctx_for_config_ep(xhci, cfg_cmd->in_ctx, vdev->out_ctx, 3278 ctrl_ctx, ep_flag, ep_flag); 3279 xhci_endpoint_copy(xhci, cfg_cmd->in_ctx, vdev->out_ctx, ep_index); 3280 3281 err = xhci_queue_configure_endpoint(xhci, cfg_cmd, cfg_cmd->in_ctx->dma, 3282 udev->slot_id, false); 3283 if (err < 0) { 3284 spin_unlock_irqrestore(&xhci->lock, flags); 3285 xhci_free_command(xhci, cfg_cmd); 3286 xhci_dbg(xhci, "%s: Failed to queue config ep command, %d ", 3287 __func__, err); 3288 goto cleanup; 3289 } 3290 3291 xhci_ring_cmd_db(xhci); 3292 spin_unlock_irqrestore(&xhci->lock, flags); 3293 3294 wait_for_completion(cfg_cmd->completion); 3295 3296 xhci_free_command(xhci, cfg_cmd); 3297 cleanup: 3298 xhci_free_command(xhci, stop_cmd); 3299 spin_lock_irqsave(&xhci->lock, flags); 3300 if (ep->ep_state & EP_SOFT_CLEAR_TOGGLE) 3301 ep->ep_state &= ~EP_SOFT_CLEAR_TOGGLE; 3302 spin_unlock_irqrestore(&xhci->lock, flags); 3303 } 3304 3305 static int xhci_check_streams_endpoint(struct xhci_hcd *xhci, 3306 struct usb_device *udev, struct usb_host_endpoint *ep, 3307 unsigned int slot_id) 3308 { 3309 int ret; 3310 unsigned int ep_index; 3311 unsigned int ep_state; 3312 3313 if (!ep) 3314 return -EINVAL; 3315 ret = xhci_check_args(xhci_to_hcd(xhci), udev, ep, 1, true, __func__); 3316 if (ret <= 0) 3317 return ret ? ret : -EINVAL; 3318 if (usb_ss_max_streams(&ep->ss_ep_comp) == 0) { 3319 xhci_warn(xhci, "WARN: SuperSpeed Endpoint Companion" 3320 " descriptor for ep 0x%x does not support streams\n", 3321 ep->desc.bEndpointAddress); 3322 return -EINVAL; 3323 } 3324 3325 ep_index = xhci_get_endpoint_index(&ep->desc); 3326 ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state; 3327 if (ep_state & EP_HAS_STREAMS || 3328 ep_state & EP_GETTING_STREAMS) { 3329 xhci_warn(xhci, "WARN: SuperSpeed bulk endpoint 0x%x " 3330 "already has streams set up.\n", 3331 ep->desc.bEndpointAddress); 3332 xhci_warn(xhci, "Send email to xHCI maintainer and ask for " 3333 "dynamic stream context array reallocation.\n"); 3334 return -EINVAL; 3335 } 3336 if (!list_empty(&xhci->devs[slot_id]->eps[ep_index].ring->td_list)) { 3337 xhci_warn(xhci, "Cannot setup streams for SuperSpeed bulk " 3338 "endpoint 0x%x; URBs are pending.\n", 3339 ep->desc.bEndpointAddress); 3340 return -EINVAL; 3341 } 3342 return 0; 3343 } 3344 3345 static void xhci_calculate_streams_entries(struct xhci_hcd *xhci, 3346 unsigned int *num_streams, unsigned int *num_stream_ctxs) 3347 { 3348 unsigned int max_streams; 3349 3350 /* The stream context array size must be a power of two */ 3351 *num_stream_ctxs = roundup_pow_of_two(*num_streams); 3352 /* 3353 * Find out how many primary stream array entries the host controller 3354 * supports. Later we may use secondary stream arrays (similar to 2nd 3355 * level page entries), but that's an optional feature for xHCI host 3356 * controllers. xHCs must support at least 4 stream IDs. 3357 */ 3358 max_streams = HCC_MAX_PSA(xhci->hcc_params); 3359 if (*num_stream_ctxs > max_streams) { 3360 xhci_dbg(xhci, "xHCI HW only supports %u stream ctx entries.\n", 3361 max_streams); 3362 *num_stream_ctxs = max_streams; 3363 *num_streams = max_streams; 3364 } 3365 } 3366 3367 /* Returns an error code if one of the endpoint already has streams. 3368 * This does not change any data structures, it only checks and gathers 3369 * information. 3370 */ 3371 static int xhci_calculate_streams_and_bitmask(struct xhci_hcd *xhci, 3372 struct usb_device *udev, 3373 struct usb_host_endpoint **eps, unsigned int num_eps, 3374 unsigned int *num_streams, u32 *changed_ep_bitmask) 3375 { 3376 unsigned int max_streams; 3377 unsigned int endpoint_flag; 3378 int i; 3379 int ret; 3380 3381 for (i = 0; i < num_eps; i++) { 3382 ret = xhci_check_streams_endpoint(xhci, udev, 3383 eps[i], udev->slot_id); 3384 if (ret < 0) 3385 return ret; 3386 3387 max_streams = usb_ss_max_streams(&eps[i]->ss_ep_comp); 3388 if (max_streams < (*num_streams - 1)) { 3389 xhci_dbg(xhci, "Ep 0x%x only supports %u stream IDs.\n", 3390 eps[i]->desc.bEndpointAddress, 3391 max_streams); 3392 *num_streams = max_streams+1; 3393 } 3394 3395 endpoint_flag = xhci_get_endpoint_flag(&eps[i]->desc); 3396 if (*changed_ep_bitmask & endpoint_flag) 3397 return -EINVAL; 3398 *changed_ep_bitmask |= endpoint_flag; 3399 } 3400 return 0; 3401 } 3402 3403 static u32 xhci_calculate_no_streams_bitmask(struct xhci_hcd *xhci, 3404 struct usb_device *udev, 3405 struct usb_host_endpoint **eps, unsigned int num_eps) 3406 { 3407 u32 changed_ep_bitmask = 0; 3408 unsigned int slot_id; 3409 unsigned int ep_index; 3410 unsigned int ep_state; 3411 int i; 3412 3413 slot_id = udev->slot_id; 3414 if (!xhci->devs[slot_id]) 3415 return 0; 3416 3417 for (i = 0; i < num_eps; i++) { 3418 ep_index = xhci_get_endpoint_index(&eps[i]->desc); 3419 ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state; 3420 /* Are streams already being freed for the endpoint? */ 3421 if (ep_state & EP_GETTING_NO_STREAMS) { 3422 xhci_warn(xhci, "WARN Can't disable streams for " 3423 "endpoint 0x%x, " 3424 "streams are being disabled already\n", 3425 eps[i]->desc.bEndpointAddress); 3426 return 0; 3427 } 3428 /* Are there actually any streams to free? */ 3429 if (!(ep_state & EP_HAS_STREAMS) && 3430 !(ep_state & EP_GETTING_STREAMS)) { 3431 xhci_warn(xhci, "WARN Can't disable streams for " 3432 "endpoint 0x%x, " 3433 "streams are already disabled!\n", 3434 eps[i]->desc.bEndpointAddress); 3435 xhci_warn(xhci, "WARN xhci_free_streams() called " 3436 "with non-streams endpoint\n"); 3437 return 0; 3438 } 3439 changed_ep_bitmask |= xhci_get_endpoint_flag(&eps[i]->desc); 3440 } 3441 return changed_ep_bitmask; 3442 } 3443 3444 /* 3445 * The USB device drivers use this function (through the HCD interface in USB 3446 * core) to prepare a set of bulk endpoints to use streams. Streams are used to 3447 * coordinate mass storage command queueing across multiple endpoints (basically 3448 * a stream ID == a task ID). 3449 * 3450 * Setting up streams involves allocating the same size stream context array 3451 * for each endpoint and issuing a configure endpoint command for all endpoints. 3452 * 3453 * Don't allow the call to succeed if one endpoint only supports one stream 3454 * (which means it doesn't support streams at all). 3455 * 3456 * Drivers may get less stream IDs than they asked for, if the host controller 3457 * hardware or endpoints claim they can't support the number of requested 3458 * stream IDs. 3459 */ 3460 static int xhci_alloc_streams(struct usb_hcd *hcd, struct usb_device *udev, 3461 struct usb_host_endpoint **eps, unsigned int num_eps, 3462 unsigned int num_streams, gfp_t mem_flags) 3463 { 3464 int i, ret; 3465 struct xhci_hcd *xhci; 3466 struct xhci_virt_device *vdev; 3467 struct xhci_command *config_cmd; 3468 struct xhci_input_control_ctx *ctrl_ctx; 3469 unsigned int ep_index; 3470 unsigned int num_stream_ctxs; 3471 unsigned int max_packet; 3472 unsigned long flags; 3473 u32 changed_ep_bitmask = 0; 3474 3475 if (!eps) 3476 return -EINVAL; 3477 3478 /* Add one to the number of streams requested to account for 3479 * stream 0 that is reserved for xHCI usage. 3480 */ 3481 num_streams += 1; 3482 xhci = hcd_to_xhci(hcd); 3483 xhci_dbg(xhci, "Driver wants %u stream IDs (including stream 0).\n", 3484 num_streams); 3485 3486 /* MaxPSASize value 0 (2 streams) means streams are not supported */ 3487 if ((xhci->quirks & XHCI_BROKEN_STREAMS) || 3488 HCC_MAX_PSA(xhci->hcc_params) < 4) { 3489 xhci_dbg(xhci, "xHCI controller does not support streams.\n"); 3490 return -ENOSYS; 3491 } 3492 3493 config_cmd = xhci_alloc_command_with_ctx(xhci, true, mem_flags); 3494 if (!config_cmd) 3495 return -ENOMEM; 3496 3497 ctrl_ctx = xhci_get_input_control_ctx(config_cmd->in_ctx); 3498 if (!ctrl_ctx) { 3499 xhci_warn(xhci, "%s: Could not get input context, bad type.\n", 3500 __func__); 3501 xhci_free_command(xhci, config_cmd); 3502 return -ENOMEM; 3503 } 3504 3505 /* Check to make sure all endpoints are not already configured for 3506 * streams. While we're at it, find the maximum number of streams that 3507 * all the endpoints will support and check for duplicate endpoints. 3508 */ 3509 spin_lock_irqsave(&xhci->lock, flags); 3510 ret = xhci_calculate_streams_and_bitmask(xhci, udev, eps, 3511 num_eps, &num_streams, &changed_ep_bitmask); 3512 if (ret < 0) { 3513 xhci_free_command(xhci, config_cmd); 3514 spin_unlock_irqrestore(&xhci->lock, flags); 3515 return ret; 3516 } 3517 if (num_streams <= 1) { 3518 xhci_warn(xhci, "WARN: endpoints can't handle " 3519 "more than one stream.\n"); 3520 xhci_free_command(xhci, config_cmd); 3521 spin_unlock_irqrestore(&xhci->lock, flags); 3522 return -EINVAL; 3523 } 3524 vdev = xhci->devs[udev->slot_id]; 3525 /* Mark each endpoint as being in transition, so 3526 * xhci_urb_enqueue() will reject all URBs. 3527 */ 3528 for (i = 0; i < num_eps; i++) { 3529 ep_index = xhci_get_endpoint_index(&eps[i]->desc); 3530 vdev->eps[ep_index].ep_state |= EP_GETTING_STREAMS; 3531 } 3532 spin_unlock_irqrestore(&xhci->lock, flags); 3533 3534 /* Setup internal data structures and allocate HW data structures for 3535 * streams (but don't install the HW structures in the input context 3536 * until we're sure all memory allocation succeeded). 3537 */ 3538 xhci_calculate_streams_entries(xhci, &num_streams, &num_stream_ctxs); 3539 xhci_dbg(xhci, "Need %u stream ctx entries for %u stream IDs.\n", 3540 num_stream_ctxs, num_streams); 3541 3542 for (i = 0; i < num_eps; i++) { 3543 ep_index = xhci_get_endpoint_index(&eps[i]->desc); 3544 max_packet = usb_endpoint_maxp(&eps[i]->desc); 3545 vdev->eps[ep_index].stream_info = xhci_alloc_stream_info(xhci, 3546 num_stream_ctxs, 3547 num_streams, 3548 max_packet, mem_flags); 3549 if (!vdev->eps[ep_index].stream_info) 3550 goto cleanup; 3551 /* Set maxPstreams in endpoint context and update deq ptr to 3552 * point to stream context array. FIXME 3553 */ 3554 } 3555 3556 /* Set up the input context for a configure endpoint command. */ 3557 for (i = 0; i < num_eps; i++) { 3558 struct xhci_ep_ctx *ep_ctx; 3559 3560 ep_index = xhci_get_endpoint_index(&eps[i]->desc); 3561 ep_ctx = xhci_get_ep_ctx(xhci, config_cmd->in_ctx, ep_index); 3562 3563 xhci_endpoint_copy(xhci, config_cmd->in_ctx, 3564 vdev->out_ctx, ep_index); 3565 xhci_setup_streams_ep_input_ctx(xhci, ep_ctx, 3566 vdev->eps[ep_index].stream_info); 3567 } 3568 /* Tell the HW to drop its old copy of the endpoint context info 3569 * and add the updated copy from the input context. 3570 */ 3571 xhci_setup_input_ctx_for_config_ep(xhci, config_cmd->in_ctx, 3572 vdev->out_ctx, ctrl_ctx, 3573 changed_ep_bitmask, changed_ep_bitmask); 3574 3575 /* Issue and wait for the configure endpoint command */ 3576 ret = xhci_configure_endpoint(xhci, udev, config_cmd, 3577 false, false); 3578 3579 /* xHC rejected the configure endpoint command for some reason, so we 3580 * leave the old ring intact and free our internal streams data 3581 * structure. 3582 */ 3583 if (ret < 0) 3584 goto cleanup; 3585 3586 spin_lock_irqsave(&xhci->lock, flags); 3587 for (i = 0; i < num_eps; i++) { 3588 ep_index = xhci_get_endpoint_index(&eps[i]->desc); 3589 vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS; 3590 xhci_dbg(xhci, "Slot %u ep ctx %u now has streams.\n", 3591 udev->slot_id, ep_index); 3592 vdev->eps[ep_index].ep_state |= EP_HAS_STREAMS; 3593 } 3594 xhci_free_command(xhci, config_cmd); 3595 spin_unlock_irqrestore(&xhci->lock, flags); 3596 3597 for (i = 0; i < num_eps; i++) { 3598 ep_index = xhci_get_endpoint_index(&eps[i]->desc); 3599 xhci_debugfs_create_stream_files(xhci, vdev, ep_index); 3600 } 3601 /* Subtract 1 for stream 0, which drivers can't use */ 3602 return num_streams - 1; 3603 3604 cleanup: 3605 /* If it didn't work, free the streams! */ 3606 for (i = 0; i < num_eps; i++) { 3607 ep_index = xhci_get_endpoint_index(&eps[i]->desc); 3608 xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info); 3609 vdev->eps[ep_index].stream_info = NULL; 3610 /* FIXME Unset maxPstreams in endpoint context and 3611 * update deq ptr to point to normal string ring. 3612 */ 3613 vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS; 3614 vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS; 3615 xhci_endpoint_zero(xhci, vdev, eps[i]); 3616 } 3617 xhci_free_command(xhci, config_cmd); 3618 return -ENOMEM; 3619 } 3620 3621 /* Transition the endpoint from using streams to being a "normal" endpoint 3622 * without streams. 3623 * 3624 * Modify the endpoint context state, submit a configure endpoint command, 3625 * and free all endpoint rings for streams if that completes successfully. 3626 */ 3627 static int xhci_free_streams(struct usb_hcd *hcd, struct usb_device *udev, 3628 struct usb_host_endpoint **eps, unsigned int num_eps, 3629 gfp_t mem_flags) 3630 { 3631 int i, ret; 3632 struct xhci_hcd *xhci; 3633 struct xhci_virt_device *vdev; 3634 struct xhci_command *command; 3635 struct xhci_input_control_ctx *ctrl_ctx; 3636 unsigned int ep_index; 3637 unsigned long flags; 3638 u32 changed_ep_bitmask; 3639 3640 xhci = hcd_to_xhci(hcd); 3641 vdev = xhci->devs[udev->slot_id]; 3642 3643 /* Set up a configure endpoint command to remove the streams rings */ 3644 spin_lock_irqsave(&xhci->lock, flags); 3645 changed_ep_bitmask = xhci_calculate_no_streams_bitmask(xhci, 3646 udev, eps, num_eps); 3647 if (changed_ep_bitmask == 0) { 3648 spin_unlock_irqrestore(&xhci->lock, flags); 3649 return -EINVAL; 3650 } 3651 3652 /* Use the xhci_command structure from the first endpoint. We may have 3653 * allocated too many, but the driver may call xhci_free_streams() for 3654 * each endpoint it grouped into one call to xhci_alloc_streams(). 3655 */ 3656 ep_index = xhci_get_endpoint_index(&eps[0]->desc); 3657 command = vdev->eps[ep_index].stream_info->free_streams_command; 3658 ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx); 3659 if (!ctrl_ctx) { 3660 spin_unlock_irqrestore(&xhci->lock, flags); 3661 xhci_warn(xhci, "%s: Could not get input context, bad type.\n", 3662 __func__); 3663 return -EINVAL; 3664 } 3665 3666 for (i = 0; i < num_eps; i++) { 3667 struct xhci_ep_ctx *ep_ctx; 3668 3669 ep_index = xhci_get_endpoint_index(&eps[i]->desc); 3670 ep_ctx = xhci_get_ep_ctx(xhci, command->in_ctx, ep_index); 3671 xhci->devs[udev->slot_id]->eps[ep_index].ep_state |= 3672 EP_GETTING_NO_STREAMS; 3673 3674 xhci_endpoint_copy(xhci, command->in_ctx, 3675 vdev->out_ctx, ep_index); 3676 xhci_setup_no_streams_ep_input_ctx(ep_ctx, 3677 &vdev->eps[ep_index]); 3678 } 3679 xhci_setup_input_ctx_for_config_ep(xhci, command->in_ctx, 3680 vdev->out_ctx, ctrl_ctx, 3681 changed_ep_bitmask, changed_ep_bitmask); 3682 spin_unlock_irqrestore(&xhci->lock, flags); 3683 3684 /* Issue and wait for the configure endpoint command, 3685 * which must succeed. 3686 */ 3687 ret = xhci_configure_endpoint(xhci, udev, command, 3688 false, true); 3689 3690 /* xHC rejected the configure endpoint command for some reason, so we 3691 * leave the streams rings intact. 3692 */ 3693 if (ret < 0) 3694 return ret; 3695 3696 spin_lock_irqsave(&xhci->lock, flags); 3697 for (i = 0; i < num_eps; i++) { 3698 ep_index = xhci_get_endpoint_index(&eps[i]->desc); 3699 xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info); 3700 vdev->eps[ep_index].stream_info = NULL; 3701 /* FIXME Unset maxPstreams in endpoint context and 3702 * update deq ptr to point to normal string ring. 3703 */ 3704 vdev->eps[ep_index].ep_state &= ~EP_GETTING_NO_STREAMS; 3705 vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS; 3706 } 3707 spin_unlock_irqrestore(&xhci->lock, flags); 3708 3709 return 0; 3710 } 3711 3712 /* 3713 * Deletes endpoint resources for endpoints that were active before a Reset 3714 * Device command, or a Disable Slot command. The Reset Device command leaves 3715 * the control endpoint intact, whereas the Disable Slot command deletes it. 3716 * 3717 * Must be called with xhci->lock held. 3718 */ 3719 void xhci_free_device_endpoint_resources(struct xhci_hcd *xhci, 3720 struct xhci_virt_device *virt_dev, bool drop_control_ep) 3721 { 3722 int i; 3723 unsigned int num_dropped_eps = 0; 3724 unsigned int drop_flags = 0; 3725 3726 for (i = (drop_control_ep ? 0 : 1); i < 31; i++) { 3727 if (virt_dev->eps[i].ring) { 3728 drop_flags |= 1 << i; 3729 num_dropped_eps++; 3730 } 3731 } 3732 xhci->num_active_eps -= num_dropped_eps; 3733 if (num_dropped_eps) 3734 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, 3735 "Dropped %u ep ctxs, flags = 0x%x, " 3736 "%u now active.", 3737 num_dropped_eps, drop_flags, 3738 xhci->num_active_eps); 3739 } 3740 3741 static void xhci_free_dev(struct usb_hcd *hcd, struct usb_device *udev); 3742 3743 /* 3744 * This submits a Reset Device Command, which will set the device state to 0, 3745 * set the device address to 0, and disable all the endpoints except the default 3746 * control endpoint. The USB core should come back and call 3747 * xhci_address_device(), and then re-set up the configuration. If this is 3748 * called because of a usb_reset_and_verify_device(), then the old alternate 3749 * settings will be re-installed through the normal bandwidth allocation 3750 * functions. 3751 * 3752 * Wait for the Reset Device command to finish. Remove all structures 3753 * associated with the endpoints that were disabled. Clear the input device 3754 * structure? Reset the control endpoint 0 max packet size? 3755 * 3756 * If the virt_dev to be reset does not exist or does not match the udev, 3757 * it means the device is lost, possibly due to the xHC restore error and 3758 * re-initialization during S3/S4. In this case, call xhci_alloc_dev() to 3759 * re-allocate the device. 3760 */ 3761 static int xhci_discover_or_reset_device(struct usb_hcd *hcd, 3762 struct usb_device *udev) 3763 { 3764 int ret, i; 3765 unsigned long flags; 3766 struct xhci_hcd *xhci; 3767 unsigned int slot_id; 3768 struct xhci_virt_device *virt_dev; 3769 struct xhci_command *reset_device_cmd; 3770 struct xhci_slot_ctx *slot_ctx; 3771 int old_active_eps = 0; 3772 3773 ret = xhci_check_args(hcd, udev, NULL, 0, false, __func__); 3774 if (ret <= 0) 3775 return ret; 3776 xhci = hcd_to_xhci(hcd); 3777 slot_id = udev->slot_id; 3778 virt_dev = xhci->devs[slot_id]; 3779 if (!virt_dev) { 3780 xhci_dbg(xhci, "The device to be reset with slot ID %u does " 3781 "not exist. Re-allocate the device\n", slot_id); 3782 ret = xhci_alloc_dev(hcd, udev); 3783 if (ret == 1) 3784 return 0; 3785 else 3786 return -EINVAL; 3787 } 3788 3789 if (virt_dev->tt_info) 3790 old_active_eps = virt_dev->tt_info->active_eps; 3791 3792 if (virt_dev->udev != udev) { 3793 /* If the virt_dev and the udev does not match, this virt_dev 3794 * may belong to another udev. 3795 * Re-allocate the device. 3796 */ 3797 xhci_dbg(xhci, "The device to be reset with slot ID %u does " 3798 "not match the udev. Re-allocate the device\n", 3799 slot_id); 3800 ret = xhci_alloc_dev(hcd, udev); 3801 if (ret == 1) 3802 return 0; 3803 else 3804 return -EINVAL; 3805 } 3806 3807 /* If device is not setup, there is no point in resetting it */ 3808 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx); 3809 if (GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)) == 3810 SLOT_STATE_DISABLED) 3811 return 0; 3812 3813 if (xhci->quirks & XHCI_ETRON_HOST) { 3814 /* 3815 * Obtaining a new device slot to inform the xHCI host that 3816 * the USB device has been reset. 3817 */ 3818 ret = xhci_disable_slot(xhci, udev->slot_id); 3819 xhci_free_virt_device(xhci, udev->slot_id); 3820 if (!ret) { 3821 ret = xhci_alloc_dev(hcd, udev); 3822 if (ret == 1) 3823 ret = 0; 3824 else 3825 ret = -EINVAL; 3826 } 3827 return ret; 3828 } 3829 3830 trace_xhci_discover_or_reset_device(slot_ctx); 3831 3832 xhci_dbg(xhci, "Resetting device with slot ID %u\n", slot_id); 3833 /* Allocate the command structure that holds the struct completion. 3834 * Assume we're in process context, since the normal device reset 3835 * process has to wait for the device anyway. Storage devices are 3836 * reset as part of error handling, so use GFP_NOIO instead of 3837 * GFP_KERNEL. 3838 */ 3839 reset_device_cmd = xhci_alloc_command(xhci, true, GFP_NOIO); 3840 if (!reset_device_cmd) { 3841 xhci_dbg(xhci, "Couldn't allocate command structure.\n"); 3842 return -ENOMEM; 3843 } 3844 3845 /* Attempt to submit the Reset Device command to the command ring */ 3846 spin_lock_irqsave(&xhci->lock, flags); 3847 3848 ret = xhci_queue_reset_device(xhci, reset_device_cmd, slot_id); 3849 if (ret) { 3850 xhci_dbg(xhci, "FIXME: allocate a command ring segment\n"); 3851 spin_unlock_irqrestore(&xhci->lock, flags); 3852 goto command_cleanup; 3853 } 3854 xhci_ring_cmd_db(xhci); 3855 spin_unlock_irqrestore(&xhci->lock, flags); 3856 3857 /* Wait for the Reset Device command to finish */ 3858 wait_for_completion(reset_device_cmd->completion); 3859 3860 /* The Reset Device command can't fail, according to the 0.95/0.96 spec, 3861 * unless we tried to reset a slot ID that wasn't enabled, 3862 * or the device wasn't in the addressed or configured state. 3863 */ 3864 ret = reset_device_cmd->status; 3865 switch (ret) { 3866 case COMP_COMMAND_ABORTED: 3867 case COMP_COMMAND_RING_STOPPED: 3868 xhci_warn(xhci, "Timeout waiting for reset device command\n"); 3869 ret = -ETIME; 3870 goto command_cleanup; 3871 case COMP_SLOT_NOT_ENABLED_ERROR: /* 0.95 completion for bad slot ID */ 3872 case COMP_CONTEXT_STATE_ERROR: /* 0.96 completion code for same thing */ 3873 xhci_dbg(xhci, "Can't reset device (slot ID %u) in %s state\n", 3874 slot_id, 3875 xhci_get_slot_state(xhci, virt_dev->out_ctx)); 3876 xhci_dbg(xhci, "Not freeing device rings.\n"); 3877 /* Don't treat this as an error. May change my mind later. */ 3878 ret = 0; 3879 goto command_cleanup; 3880 case COMP_SUCCESS: 3881 xhci_dbg(xhci, "Successful reset device command.\n"); 3882 break; 3883 default: 3884 if (xhci_is_vendor_info_code(xhci, ret)) 3885 break; 3886 xhci_warn(xhci, "Unknown completion code %u for " 3887 "reset device command.\n", ret); 3888 ret = -EINVAL; 3889 goto command_cleanup; 3890 } 3891 3892 /* Free up host controller endpoint resources */ 3893 if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) { 3894 spin_lock_irqsave(&xhci->lock, flags); 3895 /* Don't delete the default control endpoint resources */ 3896 xhci_free_device_endpoint_resources(xhci, virt_dev, false); 3897 spin_unlock_irqrestore(&xhci->lock, flags); 3898 } 3899 3900 /* Everything but endpoint 0 is disabled, so free the rings. */ 3901 for (i = 1; i < 31; i++) { 3902 struct xhci_virt_ep *ep = &virt_dev->eps[i]; 3903 3904 if (ep->ep_state & EP_HAS_STREAMS) { 3905 xhci_warn(xhci, "WARN: endpoint 0x%02x has streams on device reset, freeing streams.\n", 3906 xhci_get_endpoint_address(i)); 3907 xhci_free_stream_info(xhci, ep->stream_info); 3908 ep->stream_info = NULL; 3909 ep->ep_state &= ~EP_HAS_STREAMS; 3910 } 3911 3912 if (ep->ring) { 3913 xhci_debugfs_remove_endpoint(xhci, virt_dev, i); 3914 xhci_free_endpoint_ring(xhci, virt_dev, i); 3915 } 3916 if (!list_empty(&virt_dev->eps[i].bw_endpoint_list)) 3917 xhci_drop_ep_from_interval_table(xhci, 3918 &virt_dev->eps[i].bw_info, 3919 virt_dev->bw_table, 3920 udev, 3921 &virt_dev->eps[i], 3922 virt_dev->tt_info); 3923 xhci_clear_endpoint_bw_info(&virt_dev->eps[i].bw_info); 3924 } 3925 /* If necessary, update the number of active TTs on this root port */ 3926 xhci_update_tt_active_eps(xhci, virt_dev, old_active_eps); 3927 virt_dev->flags = 0; 3928 ret = 0; 3929 3930 command_cleanup: 3931 xhci_free_command(xhci, reset_device_cmd); 3932 return ret; 3933 } 3934 3935 /* 3936 * At this point, the struct usb_device is about to go away, the device has 3937 * disconnected, and all traffic has been stopped and the endpoints have been 3938 * disabled. Free any HC data structures associated with that device. 3939 */ 3940 static void xhci_free_dev(struct usb_hcd *hcd, struct usb_device *udev) 3941 { 3942 struct xhci_hcd *xhci = hcd_to_xhci(hcd); 3943 struct xhci_virt_device *virt_dev; 3944 struct xhci_slot_ctx *slot_ctx; 3945 unsigned long flags; 3946 int i, ret; 3947 3948 /* 3949 * We called pm_runtime_get_noresume when the device was attached. 3950 * Decrement the counter here to allow controller to runtime suspend 3951 * if no devices remain. 3952 */ 3953 if (xhci->quirks & XHCI_RESET_ON_RESUME) 3954 pm_runtime_put_noidle(hcd->self.controller); 3955 3956 ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__); 3957 /* If the host is halted due to driver unload, we still need to free the 3958 * device. 3959 */ 3960 if (ret <= 0 && ret != -ENODEV) 3961 return; 3962 3963 virt_dev = xhci->devs[udev->slot_id]; 3964 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx); 3965 trace_xhci_free_dev(slot_ctx); 3966 3967 /* Stop any wayward timer functions (which may grab the lock) */ 3968 for (i = 0; i < 31; i++) 3969 virt_dev->eps[i].ep_state &= ~EP_STOP_CMD_PENDING; 3970 virt_dev->udev = NULL; 3971 xhci_disable_slot(xhci, udev->slot_id); 3972 3973 spin_lock_irqsave(&xhci->lock, flags); 3974 xhci_free_virt_device(xhci, udev->slot_id); 3975 spin_unlock_irqrestore(&xhci->lock, flags); 3976 3977 } 3978 3979 int xhci_disable_slot(struct xhci_hcd *xhci, u32 slot_id) 3980 { 3981 struct xhci_command *command; 3982 unsigned long flags; 3983 u32 state; 3984 int ret; 3985 3986 command = xhci_alloc_command(xhci, true, GFP_KERNEL); 3987 if (!command) 3988 return -ENOMEM; 3989 3990 xhci_debugfs_remove_slot(xhci, slot_id); 3991 3992 spin_lock_irqsave(&xhci->lock, flags); 3993 /* Don't disable the slot if the host controller is dead. */ 3994 state = readl(&xhci->op_regs->status); 3995 if (state == 0xffffffff || (xhci->xhc_state & XHCI_STATE_DYING) || 3996 (xhci->xhc_state & XHCI_STATE_HALTED)) { 3997 spin_unlock_irqrestore(&xhci->lock, flags); 3998 kfree(command); 3999 return -ENODEV; 4000 } 4001 4002 ret = xhci_queue_slot_control(xhci, command, TRB_DISABLE_SLOT, 4003 slot_id); 4004 if (ret) { 4005 spin_unlock_irqrestore(&xhci->lock, flags); 4006 kfree(command); 4007 return ret; 4008 } 4009 xhci_ring_cmd_db(xhci); 4010 spin_unlock_irqrestore(&xhci->lock, flags); 4011 4012 wait_for_completion(command->completion); 4013 4014 if (command->status != COMP_SUCCESS) 4015 xhci_warn(xhci, "Unsuccessful disable slot %u command, status %d\n", 4016 slot_id, command->status); 4017 4018 xhci_free_command(xhci, command); 4019 4020 return 0; 4021 } 4022 4023 /* 4024 * Checks if we have enough host controller resources for the default control 4025 * endpoint. 4026 * 4027 * Must be called with xhci->lock held. 4028 */ 4029 static int xhci_reserve_host_control_ep_resources(struct xhci_hcd *xhci) 4030 { 4031 if (xhci->num_active_eps + 1 > xhci->limit_active_eps) { 4032 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, 4033 "Not enough ep ctxs: " 4034 "%u active, need to add 1, limit is %u.", 4035 xhci->num_active_eps, xhci->limit_active_eps); 4036 return -ENOMEM; 4037 } 4038 xhci->num_active_eps += 1; 4039 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, 4040 "Adding 1 ep ctx, %u now active.", 4041 xhci->num_active_eps); 4042 return 0; 4043 } 4044 4045 4046 /* 4047 * Returns 0 if the xHC ran out of device slots, the Enable Slot command 4048 * timed out, or allocating memory failed. Returns 1 on success. 4049 */ 4050 int xhci_alloc_dev(struct usb_hcd *hcd, struct usb_device *udev) 4051 { 4052 struct xhci_hcd *xhci = hcd_to_xhci(hcd); 4053 struct xhci_virt_device *vdev; 4054 struct xhci_slot_ctx *slot_ctx; 4055 unsigned long flags; 4056 int ret, slot_id; 4057 struct xhci_command *command; 4058 4059 command = xhci_alloc_command(xhci, true, GFP_KERNEL); 4060 if (!command) 4061 return 0; 4062 4063 spin_lock_irqsave(&xhci->lock, flags); 4064 ret = xhci_queue_slot_control(xhci, command, TRB_ENABLE_SLOT, 0); 4065 if (ret) { 4066 spin_unlock_irqrestore(&xhci->lock, flags); 4067 xhci_dbg(xhci, "FIXME: allocate a command ring segment\n"); 4068 xhci_free_command(xhci, command); 4069 return 0; 4070 } 4071 xhci_ring_cmd_db(xhci); 4072 spin_unlock_irqrestore(&xhci->lock, flags); 4073 4074 wait_for_completion(command->completion); 4075 slot_id = command->slot_id; 4076 4077 if (!slot_id || command->status != COMP_SUCCESS) { 4078 xhci_err(xhci, "Error while assigning device slot ID: %s\n", 4079 xhci_trb_comp_code_string(command->status)); 4080 xhci_err(xhci, "Max number of devices this xHCI host supports is %u.\n", 4081 HCS_MAX_SLOTS( 4082 readl(&xhci->cap_regs->hcs_params1))); 4083 xhci_free_command(xhci, command); 4084 return 0; 4085 } 4086 4087 xhci_free_command(xhci, command); 4088 4089 if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) { 4090 spin_lock_irqsave(&xhci->lock, flags); 4091 ret = xhci_reserve_host_control_ep_resources(xhci); 4092 if (ret) { 4093 spin_unlock_irqrestore(&xhci->lock, flags); 4094 xhci_warn(xhci, "Not enough host resources, " 4095 "active endpoint contexts = %u\n", 4096 xhci->num_active_eps); 4097 goto disable_slot; 4098 } 4099 spin_unlock_irqrestore(&xhci->lock, flags); 4100 } 4101 /* Use GFP_NOIO, since this function can be called from 4102 * xhci_discover_or_reset_device(), which may be called as part of 4103 * mass storage driver error handling. 4104 */ 4105 if (!xhci_alloc_virt_device(xhci, slot_id, udev, GFP_NOIO)) { 4106 xhci_warn(xhci, "Could not allocate xHCI USB device data structures\n"); 4107 goto disable_slot; 4108 } 4109 vdev = xhci->devs[slot_id]; 4110 slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx); 4111 trace_xhci_alloc_dev(slot_ctx); 4112 4113 udev->slot_id = slot_id; 4114 4115 xhci_debugfs_create_slot(xhci, slot_id); 4116 4117 /* 4118 * If resetting upon resume, we can't put the controller into runtime 4119 * suspend if there is a device attached. 4120 */ 4121 if (xhci->quirks & XHCI_RESET_ON_RESUME) 4122 pm_runtime_get_noresume(hcd->self.controller); 4123 4124 /* Is this a LS or FS device under a HS hub? */ 4125 /* Hub or peripherial? */ 4126 return 1; 4127 4128 disable_slot: 4129 xhci_disable_slot(xhci, udev->slot_id); 4130 xhci_free_virt_device(xhci, udev->slot_id); 4131 4132 return 0; 4133 } 4134 4135 /** 4136 * xhci_setup_device - issues an Address Device command to assign a unique 4137 * USB bus address. 4138 * @hcd: USB host controller data structure. 4139 * @udev: USB dev structure representing the connected device. 4140 * @setup: Enum specifying setup mode: address only or with context. 4141 * @timeout_ms: Max wait time (ms) for the command operation to complete. 4142 * 4143 * Return: 0 if successful; otherwise, negative error code. 4144 */ 4145 static int xhci_setup_device(struct usb_hcd *hcd, struct usb_device *udev, 4146 enum xhci_setup_dev setup, unsigned int timeout_ms) 4147 { 4148 const char *act = setup == SETUP_CONTEXT_ONLY ? "context" : "address"; 4149 unsigned long flags; 4150 struct xhci_virt_device *virt_dev; 4151 int ret = 0; 4152 struct xhci_hcd *xhci = hcd_to_xhci(hcd); 4153 struct xhci_slot_ctx *slot_ctx; 4154 struct xhci_input_control_ctx *ctrl_ctx; 4155 u64 temp_64; 4156 struct xhci_command *command = NULL; 4157 4158 mutex_lock(&xhci->mutex); 4159 4160 if (xhci->xhc_state) { /* dying, removing or halted */ 4161 ret = -ESHUTDOWN; 4162 goto out; 4163 } 4164 4165 if (!udev->slot_id) { 4166 xhci_dbg_trace(xhci, trace_xhci_dbg_address, 4167 "Bad Slot ID %d", udev->slot_id); 4168 ret = -EINVAL; 4169 goto out; 4170 } 4171 4172 virt_dev = xhci->devs[udev->slot_id]; 4173 4174 if (WARN_ON(!virt_dev)) { 4175 /* 4176 * In plug/unplug torture test with an NEC controller, 4177 * a zero-dereference was observed once due to virt_dev = 0. 4178 * Print useful debug rather than crash if it is observed again! 4179 */ 4180 xhci_warn(xhci, "Virt dev invalid for slot_id 0x%x!\n", 4181 udev->slot_id); 4182 ret = -EINVAL; 4183 goto out; 4184 } 4185 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx); 4186 trace_xhci_setup_device_slot(slot_ctx); 4187 4188 if (setup == SETUP_CONTEXT_ONLY) { 4189 if (GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)) == 4190 SLOT_STATE_DEFAULT) { 4191 xhci_dbg(xhci, "Slot already in default state\n"); 4192 goto out; 4193 } 4194 } 4195 4196 command = xhci_alloc_command(xhci, true, GFP_KERNEL); 4197 if (!command) { 4198 ret = -ENOMEM; 4199 goto out; 4200 } 4201 4202 command->in_ctx = virt_dev->in_ctx; 4203 command->timeout_ms = timeout_ms; 4204 4205 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx); 4206 ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx); 4207 if (!ctrl_ctx) { 4208 xhci_warn(xhci, "%s: Could not get input context, bad type.\n", 4209 __func__); 4210 ret = -EINVAL; 4211 goto out; 4212 } 4213 /* 4214 * If this is the first Set Address since device plug-in or 4215 * virt_device realloaction after a resume with an xHCI power loss, 4216 * then set up the slot context. 4217 */ 4218 if (!slot_ctx->dev_info) 4219 xhci_setup_addressable_virt_dev(xhci, udev); 4220 /* Otherwise, update the control endpoint ring enqueue pointer. */ 4221 else 4222 xhci_copy_ep0_dequeue_into_input_ctx(xhci, udev); 4223 ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG | EP0_FLAG); 4224 ctrl_ctx->drop_flags = 0; 4225 4226 trace_xhci_address_ctx(xhci, virt_dev->in_ctx, 4227 le32_to_cpu(slot_ctx->dev_info) >> 27); 4228 4229 trace_xhci_address_ctrl_ctx(ctrl_ctx); 4230 spin_lock_irqsave(&xhci->lock, flags); 4231 trace_xhci_setup_device(virt_dev); 4232 ret = xhci_queue_address_device(xhci, command, virt_dev->in_ctx->dma, 4233 udev->slot_id, setup); 4234 if (ret) { 4235 spin_unlock_irqrestore(&xhci->lock, flags); 4236 xhci_dbg_trace(xhci, trace_xhci_dbg_address, 4237 "FIXME: allocate a command ring segment"); 4238 goto out; 4239 } 4240 xhci_ring_cmd_db(xhci); 4241 spin_unlock_irqrestore(&xhci->lock, flags); 4242 4243 /* ctrl tx can take up to 5 sec; XXX: need more time for xHC? */ 4244 wait_for_completion(command->completion); 4245 4246 /* FIXME: From section 4.3.4: "Software shall be responsible for timing 4247 * the SetAddress() "recovery interval" required by USB and aborting the 4248 * command on a timeout. 4249 */ 4250 switch (command->status) { 4251 case COMP_COMMAND_ABORTED: 4252 case COMP_COMMAND_RING_STOPPED: 4253 xhci_warn(xhci, "Timeout while waiting for setup device command\n"); 4254 ret = -ETIME; 4255 break; 4256 case COMP_CONTEXT_STATE_ERROR: 4257 case COMP_SLOT_NOT_ENABLED_ERROR: 4258 xhci_err(xhci, "Setup ERROR: setup %s command for slot %d.\n", 4259 act, udev->slot_id); 4260 ret = -EINVAL; 4261 break; 4262 case COMP_USB_TRANSACTION_ERROR: 4263 dev_warn(&udev->dev, "Device not responding to setup %s.\n", act); 4264 4265 mutex_unlock(&xhci->mutex); 4266 ret = xhci_disable_slot(xhci, udev->slot_id); 4267 xhci_free_virt_device(xhci, udev->slot_id); 4268 if (!ret) { 4269 if (xhci_alloc_dev(hcd, udev) == 1) 4270 xhci_setup_addressable_virt_dev(xhci, udev); 4271 } 4272 kfree(command->completion); 4273 kfree(command); 4274 return -EPROTO; 4275 case COMP_INCOMPATIBLE_DEVICE_ERROR: 4276 dev_warn(&udev->dev, 4277 "ERROR: Incompatible device for setup %s command\n", act); 4278 ret = -ENODEV; 4279 break; 4280 case COMP_SUCCESS: 4281 xhci_dbg_trace(xhci, trace_xhci_dbg_address, 4282 "Successful setup %s command", act); 4283 break; 4284 default: 4285 xhci_err(xhci, 4286 "ERROR: unexpected setup %s command completion code 0x%x.\n", 4287 act, command->status); 4288 trace_xhci_address_ctx(xhci, virt_dev->out_ctx, 1); 4289 ret = -EINVAL; 4290 break; 4291 } 4292 if (ret) 4293 goto out; 4294 temp_64 = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr); 4295 xhci_dbg_trace(xhci, trace_xhci_dbg_address, 4296 "Op regs DCBAA ptr = %#016llx", temp_64); 4297 xhci_dbg_trace(xhci, trace_xhci_dbg_address, 4298 "Slot ID %d dcbaa entry @%p = %#016llx", 4299 udev->slot_id, 4300 &xhci->dcbaa->dev_context_ptrs[udev->slot_id], 4301 (unsigned long long) 4302 le64_to_cpu(xhci->dcbaa->dev_context_ptrs[udev->slot_id])); 4303 xhci_dbg_trace(xhci, trace_xhci_dbg_address, 4304 "Output Context DMA address = %#08llx", 4305 (unsigned long long)virt_dev->out_ctx->dma); 4306 trace_xhci_address_ctx(xhci, virt_dev->in_ctx, 4307 le32_to_cpu(slot_ctx->dev_info) >> 27); 4308 /* 4309 * USB core uses address 1 for the roothubs, so we add one to the 4310 * address given back to us by the HC. 4311 */ 4312 trace_xhci_address_ctx(xhci, virt_dev->out_ctx, 4313 le32_to_cpu(slot_ctx->dev_info) >> 27); 4314 /* Zero the input context control for later use */ 4315 ctrl_ctx->add_flags = 0; 4316 ctrl_ctx->drop_flags = 0; 4317 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx); 4318 udev->devaddr = (u8)(le32_to_cpu(slot_ctx->dev_state) & DEV_ADDR_MASK); 4319 4320 xhci_dbg_trace(xhci, trace_xhci_dbg_address, 4321 "Internal device address = %d", 4322 le32_to_cpu(slot_ctx->dev_state) & DEV_ADDR_MASK); 4323 out: 4324 mutex_unlock(&xhci->mutex); 4325 if (command) { 4326 kfree(command->completion); 4327 kfree(command); 4328 } 4329 return ret; 4330 } 4331 4332 static int xhci_address_device(struct usb_hcd *hcd, struct usb_device *udev, 4333 unsigned int timeout_ms) 4334 { 4335 return xhci_setup_device(hcd, udev, SETUP_CONTEXT_ADDRESS, timeout_ms); 4336 } 4337 4338 static int xhci_enable_device(struct usb_hcd *hcd, struct usb_device *udev) 4339 { 4340 return xhci_setup_device(hcd, udev, SETUP_CONTEXT_ONLY, 4341 XHCI_CMD_DEFAULT_TIMEOUT); 4342 } 4343 4344 /* 4345 * Transfer the port index into real index in the HW port status 4346 * registers. Caculate offset between the port's PORTSC register 4347 * and port status base. Divide the number of per port register 4348 * to get the real index. The raw port number bases 1. 4349 */ 4350 int xhci_find_raw_port_number(struct usb_hcd *hcd, int port1) 4351 { 4352 struct xhci_hub *rhub; 4353 4354 rhub = xhci_get_rhub(hcd); 4355 return rhub->ports[port1 - 1]->hw_portnum + 1; 4356 } 4357 4358 /* 4359 * Issue an Evaluate Context command to change the Maximum Exit Latency in the 4360 * slot context. If that succeeds, store the new MEL in the xhci_virt_device. 4361 */ 4362 static int __maybe_unused xhci_change_max_exit_latency(struct xhci_hcd *xhci, 4363 struct usb_device *udev, u16 max_exit_latency) 4364 { 4365 struct xhci_virt_device *virt_dev; 4366 struct xhci_command *command; 4367 struct xhci_input_control_ctx *ctrl_ctx; 4368 struct xhci_slot_ctx *slot_ctx; 4369 unsigned long flags; 4370 int ret; 4371 4372 command = xhci_alloc_command_with_ctx(xhci, true, GFP_KERNEL); 4373 if (!command) 4374 return -ENOMEM; 4375 4376 spin_lock_irqsave(&xhci->lock, flags); 4377 4378 virt_dev = xhci->devs[udev->slot_id]; 4379 4380 /* 4381 * virt_dev might not exists yet if xHC resumed from hibernate (S4) and 4382 * xHC was re-initialized. Exit latency will be set later after 4383 * hub_port_finish_reset() is done and xhci->devs[] are re-allocated 4384 */ 4385 4386 if (!virt_dev || max_exit_latency == virt_dev->current_mel) { 4387 spin_unlock_irqrestore(&xhci->lock, flags); 4388 xhci_free_command(xhci, command); 4389 return 0; 4390 } 4391 4392 /* Attempt to issue an Evaluate Context command to change the MEL. */ 4393 ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx); 4394 if (!ctrl_ctx) { 4395 spin_unlock_irqrestore(&xhci->lock, flags); 4396 xhci_free_command(xhci, command); 4397 xhci_warn(xhci, "%s: Could not get input context, bad type.\n", 4398 __func__); 4399 return -ENOMEM; 4400 } 4401 4402 xhci_slot_copy(xhci, command->in_ctx, virt_dev->out_ctx); 4403 spin_unlock_irqrestore(&xhci->lock, flags); 4404 4405 ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG); 4406 slot_ctx = xhci_get_slot_ctx(xhci, command->in_ctx); 4407 slot_ctx->dev_info2 &= cpu_to_le32(~((u32) MAX_EXIT)); 4408 slot_ctx->dev_info2 |= cpu_to_le32(max_exit_latency); 4409 slot_ctx->dev_state = 0; 4410 4411 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change, 4412 "Set up evaluate context for LPM MEL change."); 4413 4414 /* Issue and wait for the evaluate context command. */ 4415 ret = xhci_configure_endpoint(xhci, udev, command, 4416 true, true); 4417 4418 if (!ret) { 4419 spin_lock_irqsave(&xhci->lock, flags); 4420 virt_dev->current_mel = max_exit_latency; 4421 spin_unlock_irqrestore(&xhci->lock, flags); 4422 } 4423 4424 xhci_free_command(xhci, command); 4425 4426 return ret; 4427 } 4428 4429 #ifdef CONFIG_PM 4430 4431 /* BESL to HIRD Encoding array for USB2 LPM */ 4432 static int xhci_besl_encoding[16] = {125, 150, 200, 300, 400, 500, 1000, 2000, 4433 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000}; 4434 4435 /* Calculate HIRD/BESL for USB2 PORTPMSC*/ 4436 static int xhci_calculate_hird_besl(struct xhci_hcd *xhci, 4437 struct usb_device *udev) 4438 { 4439 int u2del, besl, besl_host; 4440 int besl_device = 0; 4441 u32 field; 4442 4443 u2del = HCS_U2_LATENCY(xhci->hcs_params3); 4444 field = le32_to_cpu(udev->bos->ext_cap->bmAttributes); 4445 4446 if (field & USB_BESL_SUPPORT) { 4447 for (besl_host = 0; besl_host < 16; besl_host++) { 4448 if (xhci_besl_encoding[besl_host] >= u2del) 4449 break; 4450 } 4451 /* Use baseline BESL value as default */ 4452 if (field & USB_BESL_BASELINE_VALID) 4453 besl_device = USB_GET_BESL_BASELINE(field); 4454 else if (field & USB_BESL_DEEP_VALID) 4455 besl_device = USB_GET_BESL_DEEP(field); 4456 } else { 4457 if (u2del <= 50) 4458 besl_host = 0; 4459 else 4460 besl_host = (u2del - 51) / 75 + 1; 4461 } 4462 4463 besl = besl_host + besl_device; 4464 if (besl > 15) 4465 besl = 15; 4466 4467 return besl; 4468 } 4469 4470 /* Calculate BESLD, L1 timeout and HIRDM for USB2 PORTHLPMC */ 4471 static int xhci_calculate_usb2_hw_lpm_params(struct usb_device *udev) 4472 { 4473 u32 field; 4474 int l1; 4475 int besld = 0; 4476 int hirdm = 0; 4477 4478 field = le32_to_cpu(udev->bos->ext_cap->bmAttributes); 4479 4480 /* xHCI l1 is set in steps of 256us, xHCI 1.0 section 5.4.11.2 */ 4481 l1 = udev->l1_params.timeout / 256; 4482 4483 /* device has preferred BESLD */ 4484 if (field & USB_BESL_DEEP_VALID) { 4485 besld = USB_GET_BESL_DEEP(field); 4486 hirdm = 1; 4487 } 4488 4489 return PORT_BESLD(besld) | PORT_L1_TIMEOUT(l1) | PORT_HIRDM(hirdm); 4490 } 4491 4492 static int xhci_set_usb2_hardware_lpm(struct usb_hcd *hcd, 4493 struct usb_device *udev, int enable) 4494 { 4495 struct xhci_hcd *xhci = hcd_to_xhci(hcd); 4496 struct xhci_port **ports; 4497 __le32 __iomem *pm_addr, *hlpm_addr; 4498 u32 pm_val, hlpm_val, field; 4499 unsigned int port_num; 4500 unsigned long flags; 4501 int hird, exit_latency; 4502 int ret; 4503 4504 if (xhci->quirks & XHCI_HW_LPM_DISABLE) 4505 return -EPERM; 4506 4507 if (hcd->speed >= HCD_USB3 || !xhci->hw_lpm_support || 4508 !udev->lpm_capable) 4509 return -EPERM; 4510 4511 if (!udev->parent || udev->parent->parent || 4512 udev->descriptor.bDeviceClass == USB_CLASS_HUB) 4513 return -EPERM; 4514 4515 if (udev->usb2_hw_lpm_capable != 1) 4516 return -EPERM; 4517 4518 spin_lock_irqsave(&xhci->lock, flags); 4519 4520 ports = xhci->usb2_rhub.ports; 4521 port_num = udev->portnum - 1; 4522 pm_addr = ports[port_num]->addr + PORTPMSC; 4523 pm_val = readl(pm_addr); 4524 hlpm_addr = ports[port_num]->addr + PORTHLPMC; 4525 4526 xhci_dbg(xhci, "%s port %d USB2 hardware LPM\n", 4527 str_enable_disable(enable), port_num + 1); 4528 4529 if (enable) { 4530 /* Host supports BESL timeout instead of HIRD */ 4531 if (udev->usb2_hw_lpm_besl_capable) { 4532 /* if device doesn't have a preferred BESL value use a 4533 * default one which works with mixed HIRD and BESL 4534 * systems. See XHCI_DEFAULT_BESL definition in xhci.h 4535 */ 4536 field = le32_to_cpu(udev->bos->ext_cap->bmAttributes); 4537 if ((field & USB_BESL_SUPPORT) && 4538 (field & USB_BESL_BASELINE_VALID)) 4539 hird = USB_GET_BESL_BASELINE(field); 4540 else 4541 hird = udev->l1_params.besl; 4542 4543 exit_latency = xhci_besl_encoding[hird]; 4544 spin_unlock_irqrestore(&xhci->lock, flags); 4545 4546 ret = xhci_change_max_exit_latency(xhci, udev, 4547 exit_latency); 4548 if (ret < 0) 4549 return ret; 4550 spin_lock_irqsave(&xhci->lock, flags); 4551 4552 hlpm_val = xhci_calculate_usb2_hw_lpm_params(udev); 4553 writel(hlpm_val, hlpm_addr); 4554 /* flush write */ 4555 readl(hlpm_addr); 4556 } else { 4557 hird = xhci_calculate_hird_besl(xhci, udev); 4558 } 4559 4560 pm_val &= ~PORT_HIRD_MASK; 4561 pm_val |= PORT_HIRD(hird) | PORT_RWE | PORT_L1DS(udev->slot_id); 4562 writel(pm_val, pm_addr); 4563 pm_val = readl(pm_addr); 4564 pm_val |= PORT_HLE; 4565 writel(pm_val, pm_addr); 4566 /* flush write */ 4567 readl(pm_addr); 4568 } else { 4569 pm_val &= ~(PORT_HLE | PORT_RWE | PORT_HIRD_MASK | PORT_L1DS_MASK); 4570 writel(pm_val, pm_addr); 4571 /* flush write */ 4572 readl(pm_addr); 4573 if (udev->usb2_hw_lpm_besl_capable) { 4574 spin_unlock_irqrestore(&xhci->lock, flags); 4575 xhci_change_max_exit_latency(xhci, udev, 0); 4576 readl_poll_timeout(ports[port_num]->addr, pm_val, 4577 (pm_val & PORT_PLS_MASK) == XDEV_U0, 4578 100, 10000); 4579 return 0; 4580 } 4581 } 4582 4583 spin_unlock_irqrestore(&xhci->lock, flags); 4584 return 0; 4585 } 4586 4587 static int xhci_update_device(struct usb_hcd *hcd, struct usb_device *udev) 4588 { 4589 struct xhci_hcd *xhci = hcd_to_xhci(hcd); 4590 struct xhci_port *port; 4591 u32 capability; 4592 4593 /* Check if USB3 device at root port is tunneled over USB4 */ 4594 if (hcd->speed >= HCD_USB3 && !udev->parent->parent) { 4595 port = xhci->usb3_rhub.ports[udev->portnum - 1]; 4596 4597 udev->tunnel_mode = xhci_port_is_tunneled(xhci, port); 4598 if (udev->tunnel_mode == USB_LINK_UNKNOWN) 4599 dev_dbg(&udev->dev, "link tunnel state unknown\n"); 4600 else if (udev->tunnel_mode == USB_LINK_TUNNELED) 4601 dev_dbg(&udev->dev, "tunneled over USB4 link\n"); 4602 else if (udev->tunnel_mode == USB_LINK_NATIVE) 4603 dev_dbg(&udev->dev, "native USB 3.x link\n"); 4604 return 0; 4605 } 4606 4607 if (hcd->speed >= HCD_USB3 || !udev->lpm_capable || !xhci->hw_lpm_support) 4608 return 0; 4609 4610 /* we only support lpm for non-hub device connected to root hub yet */ 4611 if (!udev->parent || udev->parent->parent || 4612 udev->descriptor.bDeviceClass == USB_CLASS_HUB) 4613 return 0; 4614 4615 port = xhci->usb2_rhub.ports[udev->portnum - 1]; 4616 capability = port->port_cap->protocol_caps; 4617 4618 if (capability & XHCI_HLC) { 4619 udev->usb2_hw_lpm_capable = 1; 4620 udev->l1_params.timeout = XHCI_L1_TIMEOUT; 4621 udev->l1_params.besl = XHCI_DEFAULT_BESL; 4622 if (capability & XHCI_BLC) 4623 udev->usb2_hw_lpm_besl_capable = 1; 4624 } 4625 4626 return 0; 4627 } 4628 4629 /*---------------------- USB 3.0 Link PM functions ------------------------*/ 4630 4631 /* Service interval in nanoseconds = 2^(bInterval - 1) * 125us * 1000ns / 1us */ 4632 static unsigned long long xhci_service_interval_to_ns( 4633 struct usb_endpoint_descriptor *desc) 4634 { 4635 return (1ULL << (desc->bInterval - 1)) * 125 * 1000; 4636 } 4637 4638 static u16 xhci_get_timeout_no_hub_lpm(struct usb_device *udev, 4639 enum usb3_link_state state) 4640 { 4641 unsigned long long sel; 4642 unsigned long long pel; 4643 unsigned int max_sel_pel; 4644 char *state_name; 4645 4646 switch (state) { 4647 case USB3_LPM_U1: 4648 /* Convert SEL and PEL stored in nanoseconds to microseconds */ 4649 sel = DIV_ROUND_UP(udev->u1_params.sel, 1000); 4650 pel = DIV_ROUND_UP(udev->u1_params.pel, 1000); 4651 max_sel_pel = USB3_LPM_MAX_U1_SEL_PEL; 4652 state_name = "U1"; 4653 break; 4654 case USB3_LPM_U2: 4655 sel = DIV_ROUND_UP(udev->u2_params.sel, 1000); 4656 pel = DIV_ROUND_UP(udev->u2_params.pel, 1000); 4657 max_sel_pel = USB3_LPM_MAX_U2_SEL_PEL; 4658 state_name = "U2"; 4659 break; 4660 default: 4661 dev_warn(&udev->dev, "%s: Can't get timeout for non-U1 or U2 state.\n", 4662 __func__); 4663 return USB3_LPM_DISABLED; 4664 } 4665 4666 if (sel <= max_sel_pel && pel <= max_sel_pel) 4667 return USB3_LPM_DEVICE_INITIATED; 4668 4669 if (sel > max_sel_pel) 4670 dev_dbg(&udev->dev, "Device-initiated %s disabled " 4671 "due to long SEL %llu ms\n", 4672 state_name, sel); 4673 else 4674 dev_dbg(&udev->dev, "Device-initiated %s disabled " 4675 "due to long PEL %llu ms\n", 4676 state_name, pel); 4677 return USB3_LPM_DISABLED; 4678 } 4679 4680 /* The U1 timeout should be the maximum of the following values: 4681 * - For control endpoints, U1 system exit latency (SEL) * 3 4682 * - For bulk endpoints, U1 SEL * 5 4683 * - For interrupt endpoints: 4684 * - Notification EPs, U1 SEL * 3 4685 * - Periodic EPs, max(105% of bInterval, U1 SEL * 2) 4686 * - For isochronous endpoints, max(105% of bInterval, U1 SEL * 2) 4687 */ 4688 static unsigned long long xhci_calculate_intel_u1_timeout( 4689 struct usb_device *udev, 4690 struct usb_endpoint_descriptor *desc) 4691 { 4692 unsigned long long timeout_ns; 4693 int ep_type; 4694 int intr_type; 4695 4696 ep_type = usb_endpoint_type(desc); 4697 switch (ep_type) { 4698 case USB_ENDPOINT_XFER_CONTROL: 4699 timeout_ns = udev->u1_params.sel * 3; 4700 break; 4701 case USB_ENDPOINT_XFER_BULK: 4702 timeout_ns = udev->u1_params.sel * 5; 4703 break; 4704 case USB_ENDPOINT_XFER_INT: 4705 intr_type = usb_endpoint_interrupt_type(desc); 4706 if (intr_type == USB_ENDPOINT_INTR_NOTIFICATION) { 4707 timeout_ns = udev->u1_params.sel * 3; 4708 break; 4709 } 4710 /* Otherwise the calculation is the same as isoc eps */ 4711 fallthrough; 4712 case USB_ENDPOINT_XFER_ISOC: 4713 timeout_ns = xhci_service_interval_to_ns(desc); 4714 timeout_ns = DIV_ROUND_UP_ULL(timeout_ns * 105, 100); 4715 if (timeout_ns < udev->u1_params.sel * 2) 4716 timeout_ns = udev->u1_params.sel * 2; 4717 break; 4718 default: 4719 return 0; 4720 } 4721 4722 return timeout_ns; 4723 } 4724 4725 /* Returns the hub-encoded U1 timeout value. */ 4726 static u16 xhci_calculate_u1_timeout(struct xhci_hcd *xhci, 4727 struct usb_device *udev, 4728 struct usb_endpoint_descriptor *desc) 4729 { 4730 unsigned long long timeout_ns; 4731 4732 /* Prevent U1 if service interval is shorter than U1 exit latency */ 4733 if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) { 4734 if (xhci_service_interval_to_ns(desc) <= udev->u1_params.mel) { 4735 dev_dbg(&udev->dev, "Disable U1, ESIT shorter than exit latency\n"); 4736 return USB3_LPM_DISABLED; 4737 } 4738 } 4739 4740 if (xhci->quirks & (XHCI_INTEL_HOST | XHCI_ZHAOXIN_HOST)) 4741 timeout_ns = xhci_calculate_intel_u1_timeout(udev, desc); 4742 else 4743 timeout_ns = udev->u1_params.sel; 4744 4745 /* The U1 timeout is encoded in 1us intervals. 4746 * Don't return a timeout of zero, because that's USB3_LPM_DISABLED. 4747 */ 4748 if (timeout_ns == USB3_LPM_DISABLED) 4749 timeout_ns = 1; 4750 else 4751 timeout_ns = DIV_ROUND_UP_ULL(timeout_ns, 1000); 4752 4753 /* If the necessary timeout value is bigger than what we can set in the 4754 * USB 3.0 hub, we have to disable hub-initiated U1. 4755 */ 4756 if (timeout_ns <= USB3_LPM_U1_MAX_TIMEOUT) 4757 return timeout_ns; 4758 dev_dbg(&udev->dev, "Hub-initiated U1 disabled " 4759 "due to long timeout %llu ms\n", timeout_ns); 4760 return xhci_get_timeout_no_hub_lpm(udev, USB3_LPM_U1); 4761 } 4762 4763 /* The U2 timeout should be the maximum of: 4764 * - 10 ms (to avoid the bandwidth impact on the scheduler) 4765 * - largest bInterval of any active periodic endpoint (to avoid going 4766 * into lower power link states between intervals). 4767 * - the U2 Exit Latency of the device 4768 */ 4769 static unsigned long long xhci_calculate_intel_u2_timeout( 4770 struct usb_device *udev, 4771 struct usb_endpoint_descriptor *desc) 4772 { 4773 unsigned long long timeout_ns; 4774 unsigned long long u2_del_ns; 4775 4776 timeout_ns = 10 * 1000 * 1000; 4777 4778 if ((usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) && 4779 (xhci_service_interval_to_ns(desc) > timeout_ns)) 4780 timeout_ns = xhci_service_interval_to_ns(desc); 4781 4782 u2_del_ns = le16_to_cpu(udev->bos->ss_cap->bU2DevExitLat) * 1000ULL; 4783 if (u2_del_ns > timeout_ns) 4784 timeout_ns = u2_del_ns; 4785 4786 return timeout_ns; 4787 } 4788 4789 /* Returns the hub-encoded U2 timeout value. */ 4790 static u16 xhci_calculate_u2_timeout(struct xhci_hcd *xhci, 4791 struct usb_device *udev, 4792 struct usb_endpoint_descriptor *desc) 4793 { 4794 unsigned long long timeout_ns; 4795 4796 /* Prevent U2 if service interval is shorter than U2 exit latency */ 4797 if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) { 4798 if (xhci_service_interval_to_ns(desc) <= udev->u2_params.mel) { 4799 dev_dbg(&udev->dev, "Disable U2, ESIT shorter than exit latency\n"); 4800 return USB3_LPM_DISABLED; 4801 } 4802 } 4803 4804 if (xhci->quirks & (XHCI_INTEL_HOST | XHCI_ZHAOXIN_HOST)) 4805 timeout_ns = xhci_calculate_intel_u2_timeout(udev, desc); 4806 else 4807 timeout_ns = udev->u2_params.sel; 4808 4809 /* The U2 timeout is encoded in 256us intervals */ 4810 timeout_ns = DIV_ROUND_UP_ULL(timeout_ns, 256 * 1000); 4811 /* If the necessary timeout value is bigger than what we can set in the 4812 * USB 3.0 hub, we have to disable hub-initiated U2. 4813 */ 4814 if (timeout_ns <= USB3_LPM_U2_MAX_TIMEOUT) 4815 return timeout_ns; 4816 dev_dbg(&udev->dev, "Hub-initiated U2 disabled " 4817 "due to long timeout %llu ms\n", timeout_ns); 4818 return xhci_get_timeout_no_hub_lpm(udev, USB3_LPM_U2); 4819 } 4820 4821 static u16 xhci_call_host_update_timeout_for_endpoint(struct xhci_hcd *xhci, 4822 struct usb_device *udev, 4823 struct usb_endpoint_descriptor *desc, 4824 enum usb3_link_state state, 4825 u16 *timeout) 4826 { 4827 if (state == USB3_LPM_U1) 4828 return xhci_calculate_u1_timeout(xhci, udev, desc); 4829 else if (state == USB3_LPM_U2) 4830 return xhci_calculate_u2_timeout(xhci, udev, desc); 4831 4832 return USB3_LPM_DISABLED; 4833 } 4834 4835 static int xhci_update_timeout_for_endpoint(struct xhci_hcd *xhci, 4836 struct usb_device *udev, 4837 struct usb_endpoint_descriptor *desc, 4838 enum usb3_link_state state, 4839 u16 *timeout) 4840 { 4841 u16 alt_timeout; 4842 4843 alt_timeout = xhci_call_host_update_timeout_for_endpoint(xhci, udev, 4844 desc, state, timeout); 4845 4846 /* If we found we can't enable hub-initiated LPM, and 4847 * the U1 or U2 exit latency was too high to allow 4848 * device-initiated LPM as well, then we will disable LPM 4849 * for this device, so stop searching any further. 4850 */ 4851 if (alt_timeout == USB3_LPM_DISABLED) { 4852 *timeout = alt_timeout; 4853 return -E2BIG; 4854 } 4855 if (alt_timeout > *timeout) 4856 *timeout = alt_timeout; 4857 return 0; 4858 } 4859 4860 static int xhci_update_timeout_for_interface(struct xhci_hcd *xhci, 4861 struct usb_device *udev, 4862 struct usb_host_interface *alt, 4863 enum usb3_link_state state, 4864 u16 *timeout) 4865 { 4866 int j; 4867 4868 for (j = 0; j < alt->desc.bNumEndpoints; j++) { 4869 if (xhci_update_timeout_for_endpoint(xhci, udev, 4870 &alt->endpoint[j].desc, state, timeout)) 4871 return -E2BIG; 4872 } 4873 return 0; 4874 } 4875 4876 static int xhci_check_tier_policy(struct xhci_hcd *xhci, 4877 struct usb_device *udev, 4878 enum usb3_link_state state) 4879 { 4880 struct usb_device *parent = udev->parent; 4881 int tier = 1; /* roothub is tier1 */ 4882 4883 while (parent) { 4884 parent = parent->parent; 4885 tier++; 4886 } 4887 4888 if (xhci->quirks & XHCI_INTEL_HOST && tier > 3) 4889 goto fail; 4890 if (xhci->quirks & XHCI_ZHAOXIN_HOST && tier > 2) 4891 goto fail; 4892 4893 return 0; 4894 fail: 4895 dev_dbg(&udev->dev, "Tier policy prevents U1/U2 LPM states for devices at tier %d\n", 4896 tier); 4897 return -E2BIG; 4898 } 4899 4900 /* Returns the U1 or U2 timeout that should be enabled. 4901 * If the tier check or timeout setting functions return with a non-zero exit 4902 * code, that means the timeout value has been finalized and we shouldn't look 4903 * at any more endpoints. 4904 */ 4905 static u16 xhci_calculate_lpm_timeout(struct usb_hcd *hcd, 4906 struct usb_device *udev, enum usb3_link_state state) 4907 { 4908 struct xhci_hcd *xhci = hcd_to_xhci(hcd); 4909 struct usb_host_config *config; 4910 char *state_name; 4911 int i; 4912 u16 timeout = USB3_LPM_DISABLED; 4913 4914 if (state == USB3_LPM_U1) 4915 state_name = "U1"; 4916 else if (state == USB3_LPM_U2) 4917 state_name = "U2"; 4918 else { 4919 dev_warn(&udev->dev, "Can't enable unknown link state %i\n", 4920 state); 4921 return timeout; 4922 } 4923 4924 /* Gather some information about the currently installed configuration 4925 * and alternate interface settings. 4926 */ 4927 if (xhci_update_timeout_for_endpoint(xhci, udev, &udev->ep0.desc, 4928 state, &timeout)) 4929 return timeout; 4930 4931 config = udev->actconfig; 4932 if (!config) 4933 return timeout; 4934 4935 for (i = 0; i < config->desc.bNumInterfaces; i++) { 4936 struct usb_driver *driver; 4937 struct usb_interface *intf = config->interface[i]; 4938 4939 if (!intf) 4940 continue; 4941 4942 /* Check if any currently bound drivers want hub-initiated LPM 4943 * disabled. 4944 */ 4945 if (intf->dev.driver) { 4946 driver = to_usb_driver(intf->dev.driver); 4947 if (driver && driver->disable_hub_initiated_lpm) { 4948 dev_dbg(&udev->dev, "Hub-initiated %s disabled at request of driver %s\n", 4949 state_name, driver->name); 4950 timeout = xhci_get_timeout_no_hub_lpm(udev, 4951 state); 4952 if (timeout == USB3_LPM_DISABLED) 4953 return timeout; 4954 } 4955 } 4956 4957 /* Not sure how this could happen... */ 4958 if (!intf->cur_altsetting) 4959 continue; 4960 4961 if (xhci_update_timeout_for_interface(xhci, udev, 4962 intf->cur_altsetting, 4963 state, &timeout)) 4964 return timeout; 4965 } 4966 return timeout; 4967 } 4968 4969 static int calculate_max_exit_latency(struct usb_device *udev, 4970 enum usb3_link_state state_changed, 4971 u16 hub_encoded_timeout) 4972 { 4973 unsigned long long u1_mel_us = 0; 4974 unsigned long long u2_mel_us = 0; 4975 unsigned long long mel_us = 0; 4976 bool disabling_u1; 4977 bool disabling_u2; 4978 bool enabling_u1; 4979 bool enabling_u2; 4980 4981 disabling_u1 = (state_changed == USB3_LPM_U1 && 4982 hub_encoded_timeout == USB3_LPM_DISABLED); 4983 disabling_u2 = (state_changed == USB3_LPM_U2 && 4984 hub_encoded_timeout == USB3_LPM_DISABLED); 4985 4986 enabling_u1 = (state_changed == USB3_LPM_U1 && 4987 hub_encoded_timeout != USB3_LPM_DISABLED); 4988 enabling_u2 = (state_changed == USB3_LPM_U2 && 4989 hub_encoded_timeout != USB3_LPM_DISABLED); 4990 4991 /* If U1 was already enabled and we're not disabling it, 4992 * or we're going to enable U1, account for the U1 max exit latency. 4993 */ 4994 if ((udev->u1_params.timeout != USB3_LPM_DISABLED && !disabling_u1) || 4995 enabling_u1) 4996 u1_mel_us = DIV_ROUND_UP(udev->u1_params.mel, 1000); 4997 if ((udev->u2_params.timeout != USB3_LPM_DISABLED && !disabling_u2) || 4998 enabling_u2) 4999 u2_mel_us = DIV_ROUND_UP(udev->u2_params.mel, 1000); 5000 5001 mel_us = max(u1_mel_us, u2_mel_us); 5002 5003 /* xHCI host controller max exit latency field is only 16 bits wide. */ 5004 if (mel_us > MAX_EXIT) { 5005 dev_warn(&udev->dev, "Link PM max exit latency of %lluus " 5006 "is too big.\n", mel_us); 5007 return -E2BIG; 5008 } 5009 return mel_us; 5010 } 5011 5012 /* Returns the USB3 hub-encoded value for the U1/U2 timeout. */ 5013 static int xhci_enable_usb3_lpm_timeout(struct usb_hcd *hcd, 5014 struct usb_device *udev, enum usb3_link_state state) 5015 { 5016 struct xhci_hcd *xhci; 5017 struct xhci_port *port; 5018 u16 hub_encoded_timeout; 5019 int mel; 5020 int ret; 5021 5022 xhci = hcd_to_xhci(hcd); 5023 /* The LPM timeout values are pretty host-controller specific, so don't 5024 * enable hub-initiated timeouts unless the vendor has provided 5025 * information about their timeout algorithm. 5026 */ 5027 if (!xhci || !(xhci->quirks & XHCI_LPM_SUPPORT) || 5028 !xhci->devs[udev->slot_id]) 5029 return USB3_LPM_DISABLED; 5030 5031 if (xhci_check_tier_policy(xhci, udev, state) < 0) 5032 return USB3_LPM_DISABLED; 5033 5034 /* If connected to root port then check port can handle lpm */ 5035 if (udev->parent && !udev->parent->parent) { 5036 port = xhci->usb3_rhub.ports[udev->portnum - 1]; 5037 if (port->lpm_incapable) 5038 return USB3_LPM_DISABLED; 5039 } 5040 5041 hub_encoded_timeout = xhci_calculate_lpm_timeout(hcd, udev, state); 5042 mel = calculate_max_exit_latency(udev, state, hub_encoded_timeout); 5043 if (mel < 0) { 5044 /* Max Exit Latency is too big, disable LPM. */ 5045 hub_encoded_timeout = USB3_LPM_DISABLED; 5046 mel = 0; 5047 } 5048 5049 ret = xhci_change_max_exit_latency(xhci, udev, mel); 5050 if (ret) 5051 return ret; 5052 return hub_encoded_timeout; 5053 } 5054 5055 static int xhci_disable_usb3_lpm_timeout(struct usb_hcd *hcd, 5056 struct usb_device *udev, enum usb3_link_state state) 5057 { 5058 struct xhci_hcd *xhci; 5059 u16 mel; 5060 5061 xhci = hcd_to_xhci(hcd); 5062 if (!xhci || !(xhci->quirks & XHCI_LPM_SUPPORT) || 5063 !xhci->devs[udev->slot_id]) 5064 return 0; 5065 5066 mel = calculate_max_exit_latency(udev, state, USB3_LPM_DISABLED); 5067 return xhci_change_max_exit_latency(xhci, udev, mel); 5068 } 5069 #else /* CONFIG_PM */ 5070 5071 static int xhci_set_usb2_hardware_lpm(struct usb_hcd *hcd, 5072 struct usb_device *udev, int enable) 5073 { 5074 return 0; 5075 } 5076 5077 static int xhci_update_device(struct usb_hcd *hcd, struct usb_device *udev) 5078 { 5079 return 0; 5080 } 5081 5082 static int xhci_enable_usb3_lpm_timeout(struct usb_hcd *hcd, 5083 struct usb_device *udev, enum usb3_link_state state) 5084 { 5085 return USB3_LPM_DISABLED; 5086 } 5087 5088 static int xhci_disable_usb3_lpm_timeout(struct usb_hcd *hcd, 5089 struct usb_device *udev, enum usb3_link_state state) 5090 { 5091 return 0; 5092 } 5093 #endif /* CONFIG_PM */ 5094 5095 /*-------------------------------------------------------------------------*/ 5096 5097 /* Once a hub descriptor is fetched for a device, we need to update the xHC's 5098 * internal data structures for the device. 5099 */ 5100 int xhci_update_hub_device(struct usb_hcd *hcd, struct usb_device *hdev, 5101 struct usb_tt *tt, gfp_t mem_flags) 5102 { 5103 struct xhci_hcd *xhci = hcd_to_xhci(hcd); 5104 struct xhci_virt_device *vdev; 5105 struct xhci_command *config_cmd; 5106 struct xhci_input_control_ctx *ctrl_ctx; 5107 struct xhci_slot_ctx *slot_ctx; 5108 unsigned long flags; 5109 unsigned think_time; 5110 int ret; 5111 5112 /* Ignore root hubs */ 5113 if (!hdev->parent) 5114 return 0; 5115 5116 vdev = xhci->devs[hdev->slot_id]; 5117 if (!vdev) { 5118 xhci_warn(xhci, "Cannot update hub desc for unknown device.\n"); 5119 return -EINVAL; 5120 } 5121 5122 config_cmd = xhci_alloc_command_with_ctx(xhci, true, mem_flags); 5123 if (!config_cmd) 5124 return -ENOMEM; 5125 5126 ctrl_ctx = xhci_get_input_control_ctx(config_cmd->in_ctx); 5127 if (!ctrl_ctx) { 5128 xhci_warn(xhci, "%s: Could not get input context, bad type.\n", 5129 __func__); 5130 xhci_free_command(xhci, config_cmd); 5131 return -ENOMEM; 5132 } 5133 5134 spin_lock_irqsave(&xhci->lock, flags); 5135 if (hdev->speed == USB_SPEED_HIGH && 5136 xhci_alloc_tt_info(xhci, vdev, hdev, tt, GFP_ATOMIC)) { 5137 xhci_dbg(xhci, "Could not allocate xHCI TT structure.\n"); 5138 xhci_free_command(xhci, config_cmd); 5139 spin_unlock_irqrestore(&xhci->lock, flags); 5140 return -ENOMEM; 5141 } 5142 5143 xhci_slot_copy(xhci, config_cmd->in_ctx, vdev->out_ctx); 5144 ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG); 5145 slot_ctx = xhci_get_slot_ctx(xhci, config_cmd->in_ctx); 5146 slot_ctx->dev_info |= cpu_to_le32(DEV_HUB); 5147 /* 5148 * refer to section 6.2.2: MTT should be 0 for full speed hub, 5149 * but it may be already set to 1 when setup an xHCI virtual 5150 * device, so clear it anyway. 5151 */ 5152 if (tt->multi) 5153 slot_ctx->dev_info |= cpu_to_le32(DEV_MTT); 5154 else if (hdev->speed == USB_SPEED_FULL) 5155 slot_ctx->dev_info &= cpu_to_le32(~DEV_MTT); 5156 5157 if (xhci->hci_version > 0x95) { 5158 xhci_dbg(xhci, "xHCI version %x needs hub " 5159 "TT think time and number of ports\n", 5160 (unsigned int) xhci->hci_version); 5161 slot_ctx->dev_info2 |= cpu_to_le32(XHCI_MAX_PORTS(hdev->maxchild)); 5162 /* Set TT think time - convert from ns to FS bit times. 5163 * 0 = 8 FS bit times, 1 = 16 FS bit times, 5164 * 2 = 24 FS bit times, 3 = 32 FS bit times. 5165 * 5166 * xHCI 1.0: this field shall be 0 if the device is not a 5167 * High-spped hub. 5168 */ 5169 think_time = tt->think_time; 5170 if (think_time != 0) 5171 think_time = (think_time / 666) - 1; 5172 if (xhci->hci_version < 0x100 || hdev->speed == USB_SPEED_HIGH) 5173 slot_ctx->tt_info |= 5174 cpu_to_le32(TT_THINK_TIME(think_time)); 5175 } else { 5176 xhci_dbg(xhci, "xHCI version %x doesn't need hub " 5177 "TT think time or number of ports\n", 5178 (unsigned int) xhci->hci_version); 5179 } 5180 slot_ctx->dev_state = 0; 5181 spin_unlock_irqrestore(&xhci->lock, flags); 5182 5183 xhci_dbg(xhci, "Set up %s for hub device.\n", 5184 (xhci->hci_version > 0x95) ? 5185 "configure endpoint" : "evaluate context"); 5186 5187 /* Issue and wait for the configure endpoint or 5188 * evaluate context command. 5189 */ 5190 if (xhci->hci_version > 0x95) 5191 ret = xhci_configure_endpoint(xhci, hdev, config_cmd, 5192 false, false); 5193 else 5194 ret = xhci_configure_endpoint(xhci, hdev, config_cmd, 5195 true, false); 5196 5197 xhci_free_command(xhci, config_cmd); 5198 return ret; 5199 } 5200 EXPORT_SYMBOL_GPL(xhci_update_hub_device); 5201 5202 static int xhci_get_frame(struct usb_hcd *hcd) 5203 { 5204 struct xhci_hcd *xhci = hcd_to_xhci(hcd); 5205 /* EHCI mods by the periodic size. Why? */ 5206 return readl(&xhci->run_regs->microframe_index) >> 3; 5207 } 5208 5209 static void xhci_hcd_init_usb2_data(struct xhci_hcd *xhci, struct usb_hcd *hcd) 5210 { 5211 xhci->usb2_rhub.hcd = hcd; 5212 hcd->speed = HCD_USB2; 5213 hcd->self.root_hub->speed = USB_SPEED_HIGH; 5214 /* 5215 * USB 2.0 roothub under xHCI has an integrated TT, 5216 * (rate matching hub) as opposed to having an OHCI/UHCI 5217 * companion controller. 5218 */ 5219 hcd->has_tt = 1; 5220 } 5221 5222 static void xhci_hcd_init_usb3_data(struct xhci_hcd *xhci, struct usb_hcd *hcd) 5223 { 5224 unsigned int minor_rev; 5225 5226 /* 5227 * Early xHCI 1.1 spec did not mention USB 3.1 capable hosts 5228 * should return 0x31 for sbrn, or that the minor revision 5229 * is a two digit BCD containig minor and sub-minor numbers. 5230 * This was later clarified in xHCI 1.2. 5231 * 5232 * Some USB 3.1 capable hosts therefore have sbrn 0x30, and 5233 * minor revision set to 0x1 instead of 0x10. 5234 */ 5235 if (xhci->usb3_rhub.min_rev == 0x1) 5236 minor_rev = 1; 5237 else 5238 minor_rev = xhci->usb3_rhub.min_rev / 0x10; 5239 5240 switch (minor_rev) { 5241 case 2: 5242 hcd->speed = HCD_USB32; 5243 hcd->self.root_hub->speed = USB_SPEED_SUPER_PLUS; 5244 hcd->self.root_hub->rx_lanes = 2; 5245 hcd->self.root_hub->tx_lanes = 2; 5246 hcd->self.root_hub->ssp_rate = USB_SSP_GEN_2x2; 5247 break; 5248 case 1: 5249 hcd->speed = HCD_USB31; 5250 hcd->self.root_hub->speed = USB_SPEED_SUPER_PLUS; 5251 hcd->self.root_hub->ssp_rate = USB_SSP_GEN_2x1; 5252 break; 5253 } 5254 xhci_info(xhci, "Host supports USB 3.%x %sSuperSpeed\n", 5255 minor_rev, minor_rev ? "Enhanced " : ""); 5256 5257 xhci->usb3_rhub.hcd = hcd; 5258 } 5259 5260 int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks) 5261 { 5262 struct xhci_hcd *xhci; 5263 /* 5264 * TODO: Check with DWC3 clients for sysdev according to 5265 * quirks 5266 */ 5267 struct device *dev = hcd->self.sysdev; 5268 int retval; 5269 5270 /* Accept arbitrarily long scatter-gather lists */ 5271 hcd->self.sg_tablesize = ~0; 5272 5273 /* support to build packet from discontinuous buffers */ 5274 hcd->self.no_sg_constraint = 1; 5275 5276 /* XHCI controllers don't stop the ep queue on short packets :| */ 5277 hcd->self.no_stop_on_short = 1; 5278 5279 xhci = hcd_to_xhci(hcd); 5280 5281 if (!usb_hcd_is_primary_hcd(hcd)) { 5282 xhci_hcd_init_usb3_data(xhci, hcd); 5283 return 0; 5284 } 5285 5286 mutex_init(&xhci->mutex); 5287 xhci->main_hcd = hcd; 5288 xhci->cap_regs = hcd->regs; 5289 xhci->op_regs = hcd->regs + 5290 HC_LENGTH(readl(&xhci->cap_regs->hc_capbase)); 5291 xhci->run_regs = hcd->regs + 5292 (readl(&xhci->cap_regs->run_regs_off) & RTSOFF_MASK); 5293 /* Cache read-only capability registers */ 5294 xhci->hcs_params1 = readl(&xhci->cap_regs->hcs_params1); 5295 xhci->hcs_params2 = readl(&xhci->cap_regs->hcs_params2); 5296 xhci->hcs_params3 = readl(&xhci->cap_regs->hcs_params3); 5297 xhci->hci_version = HC_VERSION(readl(&xhci->cap_regs->hc_capbase)); 5298 xhci->hcc_params = readl(&xhci->cap_regs->hcc_params); 5299 if (xhci->hci_version > 0x100) 5300 xhci->hcc_params2 = readl(&xhci->cap_regs->hcc_params2); 5301 5302 /* xhci-plat or xhci-pci might have set max_interrupters already */ 5303 if ((!xhci->max_interrupters) || 5304 xhci->max_interrupters > HCS_MAX_INTRS(xhci->hcs_params1)) 5305 xhci->max_interrupters = HCS_MAX_INTRS(xhci->hcs_params1); 5306 5307 xhci->quirks |= quirks; 5308 5309 if (get_quirks) 5310 get_quirks(dev, xhci); 5311 5312 /* In xhci controllers which follow xhci 1.0 spec gives a spurious 5313 * success event after a short transfer. This quirk will ignore such 5314 * spurious event. 5315 */ 5316 if (xhci->hci_version > 0x96) 5317 xhci->quirks |= XHCI_SPURIOUS_SUCCESS; 5318 5319 if (xhci->hci_version == 0x95 && link_quirk) { 5320 xhci_dbg(xhci, "QUIRK: Not clearing Link TRB chain bits"); 5321 xhci->quirks |= XHCI_LINK_TRB_QUIRK; 5322 } 5323 5324 /* Make sure the HC is halted. */ 5325 retval = xhci_halt(xhci); 5326 if (retval) 5327 return retval; 5328 5329 xhci_zero_64b_regs(xhci); 5330 5331 xhci_dbg(xhci, "Resetting HCD\n"); 5332 /* Reset the internal HC memory state and registers. */ 5333 retval = xhci_reset(xhci, XHCI_RESET_LONG_USEC); 5334 if (retval) 5335 return retval; 5336 xhci_dbg(xhci, "Reset complete\n"); 5337 5338 /* 5339 * On some xHCI controllers (e.g. R-Car SoCs), the AC64 bit (bit 0) 5340 * of HCCPARAMS1 is set to 1. However, the xHCs don't support 64-bit 5341 * address memory pointers actually. So, this driver clears the AC64 5342 * bit of xhci->hcc_params to call dma_set_coherent_mask(dev, 5343 * DMA_BIT_MASK(32)) in this xhci_gen_setup(). 5344 */ 5345 if (xhci->quirks & XHCI_NO_64BIT_SUPPORT) 5346 xhci->hcc_params &= ~BIT(0); 5347 5348 /* Set dma_mask and coherent_dma_mask to 64-bits, 5349 * if xHC supports 64-bit addressing */ 5350 if (HCC_64BIT_ADDR(xhci->hcc_params) && 5351 !dma_set_mask(dev, DMA_BIT_MASK(64))) { 5352 xhci_dbg(xhci, "Enabling 64-bit DMA addresses.\n"); 5353 dma_set_coherent_mask(dev, DMA_BIT_MASK(64)); 5354 } else { 5355 /* 5356 * This is to avoid error in cases where a 32-bit USB 5357 * controller is used on a 64-bit capable system. 5358 */ 5359 retval = dma_set_mask(dev, DMA_BIT_MASK(32)); 5360 if (retval) 5361 return retval; 5362 xhci_dbg(xhci, "Enabling 32-bit DMA addresses.\n"); 5363 dma_set_coherent_mask(dev, DMA_BIT_MASK(32)); 5364 } 5365 5366 xhci_dbg(xhci, "Calling HCD init\n"); 5367 /* Initialize HCD and host controller data structures. */ 5368 retval = xhci_init(hcd); 5369 if (retval) 5370 return retval; 5371 xhci_dbg(xhci, "Called HCD init\n"); 5372 5373 if (xhci_hcd_is_usb3(hcd)) 5374 xhci_hcd_init_usb3_data(xhci, hcd); 5375 else 5376 xhci_hcd_init_usb2_data(xhci, hcd); 5377 5378 xhci_info(xhci, "hcc params 0x%08x hci version 0x%x quirks 0x%016llx\n", 5379 xhci->hcc_params, xhci->hci_version, xhci->quirks); 5380 5381 return 0; 5382 } 5383 EXPORT_SYMBOL_GPL(xhci_gen_setup); 5384 5385 static void xhci_clear_tt_buffer_complete(struct usb_hcd *hcd, 5386 struct usb_host_endpoint *ep) 5387 { 5388 struct xhci_hcd *xhci; 5389 struct usb_device *udev; 5390 unsigned int slot_id; 5391 unsigned int ep_index; 5392 unsigned long flags; 5393 5394 xhci = hcd_to_xhci(hcd); 5395 5396 spin_lock_irqsave(&xhci->lock, flags); 5397 udev = (struct usb_device *)ep->hcpriv; 5398 slot_id = udev->slot_id; 5399 ep_index = xhci_get_endpoint_index(&ep->desc); 5400 5401 xhci->devs[slot_id]->eps[ep_index].ep_state &= ~EP_CLEARING_TT; 5402 xhci_ring_doorbell_for_active_rings(xhci, slot_id, ep_index); 5403 spin_unlock_irqrestore(&xhci->lock, flags); 5404 } 5405 5406 static const struct hc_driver xhci_hc_driver = { 5407 .description = "xhci-hcd", 5408 .product_desc = "xHCI Host Controller", 5409 .hcd_priv_size = sizeof(struct xhci_hcd), 5410 5411 /* 5412 * generic hardware linkage 5413 */ 5414 .irq = xhci_irq, 5415 .flags = HCD_MEMORY | HCD_DMA | HCD_USB3 | HCD_SHARED | 5416 HCD_BH, 5417 5418 /* 5419 * basic lifecycle operations 5420 */ 5421 .reset = NULL, /* set in xhci_init_driver() */ 5422 .start = xhci_run, 5423 .stop = xhci_stop, 5424 .shutdown = xhci_shutdown, 5425 5426 /* 5427 * managing i/o requests and associated device resources 5428 */ 5429 .map_urb_for_dma = xhci_map_urb_for_dma, 5430 .unmap_urb_for_dma = xhci_unmap_urb_for_dma, 5431 .urb_enqueue = xhci_urb_enqueue, 5432 .urb_dequeue = xhci_urb_dequeue, 5433 .alloc_dev = xhci_alloc_dev, 5434 .free_dev = xhci_free_dev, 5435 .alloc_streams = xhci_alloc_streams, 5436 .free_streams = xhci_free_streams, 5437 .add_endpoint = xhci_add_endpoint, 5438 .drop_endpoint = xhci_drop_endpoint, 5439 .endpoint_disable = xhci_endpoint_disable, 5440 .endpoint_reset = xhci_endpoint_reset, 5441 .check_bandwidth = xhci_check_bandwidth, 5442 .reset_bandwidth = xhci_reset_bandwidth, 5443 .address_device = xhci_address_device, 5444 .enable_device = xhci_enable_device, 5445 .update_hub_device = xhci_update_hub_device, 5446 .reset_device = xhci_discover_or_reset_device, 5447 5448 /* 5449 * scheduling support 5450 */ 5451 .get_frame_number = xhci_get_frame, 5452 5453 /* 5454 * root hub support 5455 */ 5456 .hub_control = xhci_hub_control, 5457 .hub_status_data = xhci_hub_status_data, 5458 .bus_suspend = xhci_bus_suspend, 5459 .bus_resume = xhci_bus_resume, 5460 .get_resuming_ports = xhci_get_resuming_ports, 5461 5462 /* 5463 * call back when device connected and addressed 5464 */ 5465 .update_device = xhci_update_device, 5466 .set_usb2_hw_lpm = xhci_set_usb2_hardware_lpm, 5467 .enable_usb3_lpm_timeout = xhci_enable_usb3_lpm_timeout, 5468 .disable_usb3_lpm_timeout = xhci_disable_usb3_lpm_timeout, 5469 .find_raw_port_number = xhci_find_raw_port_number, 5470 .clear_tt_buffer_complete = xhci_clear_tt_buffer_complete, 5471 }; 5472 5473 void xhci_init_driver(struct hc_driver *drv, 5474 const struct xhci_driver_overrides *over) 5475 { 5476 BUG_ON(!over); 5477 5478 /* Copy the generic table to drv then apply the overrides */ 5479 *drv = xhci_hc_driver; 5480 5481 if (over) { 5482 drv->hcd_priv_size += over->extra_priv_size; 5483 if (over->reset) 5484 drv->reset = over->reset; 5485 if (over->start) 5486 drv->start = over->start; 5487 if (over->add_endpoint) 5488 drv->add_endpoint = over->add_endpoint; 5489 if (over->drop_endpoint) 5490 drv->drop_endpoint = over->drop_endpoint; 5491 if (over->check_bandwidth) 5492 drv->check_bandwidth = over->check_bandwidth; 5493 if (over->reset_bandwidth) 5494 drv->reset_bandwidth = over->reset_bandwidth; 5495 if (over->update_hub_device) 5496 drv->update_hub_device = over->update_hub_device; 5497 if (over->hub_control) 5498 drv->hub_control = over->hub_control; 5499 } 5500 } 5501 EXPORT_SYMBOL_GPL(xhci_init_driver); 5502 5503 MODULE_DESCRIPTION(DRIVER_DESC); 5504 MODULE_AUTHOR(DRIVER_AUTHOR); 5505 MODULE_LICENSE("GPL"); 5506 5507 static int __init xhci_hcd_init(void) 5508 { 5509 /* 5510 * Check the compiler generated sizes of structures that must be laid 5511 * out in specific ways for hardware access. 5512 */ 5513 BUILD_BUG_ON(sizeof(struct xhci_doorbell_array) != 256*32/8); 5514 BUILD_BUG_ON(sizeof(struct xhci_slot_ctx) != 8*32/8); 5515 BUILD_BUG_ON(sizeof(struct xhci_ep_ctx) != 8*32/8); 5516 /* xhci_device_control has eight fields, and also 5517 * embeds one xhci_slot_ctx and 31 xhci_ep_ctx 5518 */ 5519 BUILD_BUG_ON(sizeof(struct xhci_stream_ctx) != 4*32/8); 5520 BUILD_BUG_ON(sizeof(union xhci_trb) != 4*32/8); 5521 BUILD_BUG_ON(sizeof(struct xhci_erst_entry) != 4*32/8); 5522 BUILD_BUG_ON(sizeof(struct xhci_cap_regs) != 8*32/8); 5523 BUILD_BUG_ON(sizeof(struct xhci_intr_reg) != 8*32/8); 5524 /* xhci_run_regs has eight fields and embeds 128 xhci_intr_regs */ 5525 BUILD_BUG_ON(sizeof(struct xhci_run_regs) != (8+8*128)*32/8); 5526 5527 if (usb_disabled()) 5528 return -ENODEV; 5529 5530 xhci_debugfs_create_root(); 5531 xhci_dbc_init(); 5532 5533 return 0; 5534 } 5535 5536 /* 5537 * If an init function is provided, an exit function must also be provided 5538 * to allow module unload. 5539 */ 5540 static void __exit xhci_hcd_fini(void) 5541 { 5542 xhci_debugfs_remove_root(); 5543 xhci_dbc_exit(); 5544 } 5545 5546 module_init(xhci_hcd_init); 5547 module_exit(xhci_hcd_fini); 5548