xref: /linux/drivers/usb/host/xhci.c (revision 7fe03f8ff55d33fe6398637f78a8620dd2a78b38)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * xHCI host controller driver
4  *
5  * Copyright (C) 2008 Intel Corp.
6  *
7  * Author: Sarah Sharp
8  * Some code borrowed from the Linux EHCI driver.
9  */
10 
11 #include <linux/jiffies.h>
12 #include <linux/pci.h>
13 #include <linux/iommu.h>
14 #include <linux/iopoll.h>
15 #include <linux/irq.h>
16 #include <linux/log2.h>
17 #include <linux/module.h>
18 #include <linux/moduleparam.h>
19 #include <linux/slab.h>
20 #include <linux/string_choices.h>
21 #include <linux/dmi.h>
22 #include <linux/dma-mapping.h>
23 
24 #include "xhci.h"
25 #include "xhci-trace.h"
26 #include "xhci-debugfs.h"
27 #include "xhci-dbgcap.h"
28 
29 #define DRIVER_AUTHOR "Sarah Sharp"
30 #define DRIVER_DESC "'eXtensible' Host Controller (xHC) Driver"
31 
32 #define	PORT_WAKE_BITS	(PORT_WKOC_E | PORT_WKDISC_E | PORT_WKCONN_E)
33 
34 /* Some 0.95 hardware can't handle the chain bit on a Link TRB being cleared */
35 static int link_quirk;
36 module_param(link_quirk, int, S_IRUGO | S_IWUSR);
37 MODULE_PARM_DESC(link_quirk, "Don't clear the chain bit on a link TRB");
38 
39 static unsigned long long quirks;
40 module_param(quirks, ullong, S_IRUGO);
41 MODULE_PARM_DESC(quirks, "Bit flags for quirks to be enabled as default");
42 
43 static bool td_on_ring(struct xhci_td *td, struct xhci_ring *ring)
44 {
45 	struct xhci_segment *seg;
46 
47 	if (!td || !td->start_seg)
48 		return false;
49 
50 	xhci_for_each_ring_seg(ring->first_seg, seg) {
51 		if (seg == td->start_seg)
52 			return true;
53 	}
54 
55 	return false;
56 }
57 
58 /*
59  * xhci_handshake - spin reading hc until handshake completes or fails
60  * @ptr: address of hc register to be read
61  * @mask: bits to look at in result of read
62  * @done: value of those bits when handshake succeeds
63  * @usec: timeout in microseconds
64  *
65  * Returns negative errno, or zero on success
66  *
67  * Success happens when the "mask" bits have the specified value (hardware
68  * handshake done).  There are two failure modes:  "usec" have passed (major
69  * hardware flakeout), or the register reads as all-ones (hardware removed).
70  */
71 int xhci_handshake(void __iomem *ptr, u32 mask, u32 done, u64 timeout_us)
72 {
73 	u32	result;
74 	int	ret;
75 
76 	ret = readl_poll_timeout_atomic(ptr, result,
77 					(result & mask) == done ||
78 					result == U32_MAX,
79 					1, timeout_us);
80 	if (result == U32_MAX)		/* card removed */
81 		return -ENODEV;
82 
83 	return ret;
84 }
85 
86 /*
87  * xhci_handshake_check_state - same as xhci_handshake but takes an additional
88  * exit_state parameter, and bails out with an error immediately when xhc_state
89  * has exit_state flag set.
90  */
91 int xhci_handshake_check_state(struct xhci_hcd *xhci, void __iomem *ptr,
92 		u32 mask, u32 done, int usec, unsigned int exit_state)
93 {
94 	u32	result;
95 	int	ret;
96 
97 	ret = readl_poll_timeout_atomic(ptr, result,
98 				(result & mask) == done ||
99 				result == U32_MAX ||
100 				xhci->xhc_state & exit_state,
101 				1, usec);
102 
103 	if (result == U32_MAX || xhci->xhc_state & exit_state)
104 		return -ENODEV;
105 
106 	return ret;
107 }
108 
109 /*
110  * Disable interrupts and begin the xHCI halting process.
111  */
112 void xhci_quiesce(struct xhci_hcd *xhci)
113 {
114 	u32 halted;
115 	u32 cmd;
116 	u32 mask;
117 
118 	mask = ~(XHCI_IRQS);
119 	halted = readl(&xhci->op_regs->status) & STS_HALT;
120 	if (!halted)
121 		mask &= ~CMD_RUN;
122 
123 	cmd = readl(&xhci->op_regs->command);
124 	cmd &= mask;
125 	writel(cmd, &xhci->op_regs->command);
126 }
127 
128 /*
129  * Force HC into halt state.
130  *
131  * Disable any IRQs and clear the run/stop bit.
132  * HC will complete any current and actively pipelined transactions, and
133  * should halt within 16 ms of the run/stop bit being cleared.
134  * Read HC Halted bit in the status register to see when the HC is finished.
135  */
136 int xhci_halt(struct xhci_hcd *xhci)
137 {
138 	int ret;
139 
140 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Halt the HC");
141 	xhci_quiesce(xhci);
142 
143 	ret = xhci_handshake(&xhci->op_regs->status,
144 			STS_HALT, STS_HALT, XHCI_MAX_HALT_USEC);
145 	if (ret) {
146 		xhci_warn(xhci, "Host halt failed, %d\n", ret);
147 		return ret;
148 	}
149 
150 	xhci->xhc_state |= XHCI_STATE_HALTED;
151 	xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
152 
153 	return ret;
154 }
155 
156 /*
157  * Set the run bit and wait for the host to be running.
158  */
159 int xhci_start(struct xhci_hcd *xhci)
160 {
161 	u32 temp;
162 	int ret;
163 
164 	temp = readl(&xhci->op_regs->command);
165 	temp |= (CMD_RUN);
166 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Turn on HC, cmd = 0x%x.",
167 			temp);
168 	writel(temp, &xhci->op_regs->command);
169 
170 	/*
171 	 * Wait for the HCHalted Status bit to be 0 to indicate the host is
172 	 * running.
173 	 */
174 	ret = xhci_handshake(&xhci->op_regs->status,
175 			STS_HALT, 0, XHCI_MAX_HALT_USEC);
176 	if (ret == -ETIMEDOUT)
177 		xhci_err(xhci, "Host took too long to start, "
178 				"waited %u microseconds.\n",
179 				XHCI_MAX_HALT_USEC);
180 	if (!ret) {
181 		/* clear state flags. Including dying, halted or removing */
182 		xhci->xhc_state = 0;
183 		xhci->run_graceperiod = jiffies + msecs_to_jiffies(500);
184 	}
185 
186 	return ret;
187 }
188 
189 /*
190  * Reset a halted HC.
191  *
192  * This resets pipelines, timers, counters, state machines, etc.
193  * Transactions will be terminated immediately, and operational registers
194  * will be set to their defaults.
195  */
196 int xhci_reset(struct xhci_hcd *xhci, u64 timeout_us)
197 {
198 	u32 command;
199 	u32 state;
200 	int ret;
201 
202 	state = readl(&xhci->op_regs->status);
203 
204 	if (state == ~(u32)0) {
205 		xhci_warn(xhci, "Host not accessible, reset failed.\n");
206 		return -ENODEV;
207 	}
208 
209 	if ((state & STS_HALT) == 0) {
210 		xhci_warn(xhci, "Host controller not halted, aborting reset.\n");
211 		return 0;
212 	}
213 
214 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Reset the HC");
215 	command = readl(&xhci->op_regs->command);
216 	command |= CMD_RESET;
217 	writel(command, &xhci->op_regs->command);
218 
219 	/* Existing Intel xHCI controllers require a delay of 1 mS,
220 	 * after setting the CMD_RESET bit, and before accessing any
221 	 * HC registers. This allows the HC to complete the
222 	 * reset operation and be ready for HC register access.
223 	 * Without this delay, the subsequent HC register access,
224 	 * may result in a system hang very rarely.
225 	 */
226 	if (xhci->quirks & XHCI_INTEL_HOST)
227 		udelay(1000);
228 
229 	ret = xhci_handshake_check_state(xhci, &xhci->op_regs->command,
230 				CMD_RESET, 0, timeout_us, XHCI_STATE_REMOVING);
231 	if (ret)
232 		return ret;
233 
234 	if (xhci->quirks & XHCI_ASMEDIA_MODIFY_FLOWCONTROL)
235 		usb_asmedia_modifyflowcontrol(to_pci_dev(xhci_to_hcd(xhci)->self.controller));
236 
237 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
238 			 "Wait for controller to be ready for doorbell rings");
239 	/*
240 	 * xHCI cannot write to any doorbells or operational registers other
241 	 * than status until the "Controller Not Ready" flag is cleared.
242 	 */
243 	ret = xhci_handshake(&xhci->op_regs->status, STS_CNR, 0, timeout_us);
244 
245 	xhci->usb2_rhub.bus_state.port_c_suspend = 0;
246 	xhci->usb2_rhub.bus_state.suspended_ports = 0;
247 	xhci->usb2_rhub.bus_state.resuming_ports = 0;
248 	xhci->usb3_rhub.bus_state.port_c_suspend = 0;
249 	xhci->usb3_rhub.bus_state.suspended_ports = 0;
250 	xhci->usb3_rhub.bus_state.resuming_ports = 0;
251 
252 	return ret;
253 }
254 
255 static void xhci_zero_64b_regs(struct xhci_hcd *xhci)
256 {
257 	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
258 	struct iommu_domain *domain;
259 	int err, i;
260 	u64 val;
261 	u32 intrs;
262 
263 	/*
264 	 * Some Renesas controllers get into a weird state if they are
265 	 * reset while programmed with 64bit addresses (they will preserve
266 	 * the top half of the address in internal, non visible
267 	 * registers). You end up with half the address coming from the
268 	 * kernel, and the other half coming from the firmware. Also,
269 	 * changing the programming leads to extra accesses even if the
270 	 * controller is supposed to be halted. The controller ends up with
271 	 * a fatal fault, and is then ripe for being properly reset.
272 	 *
273 	 * Special care is taken to only apply this if the device is behind
274 	 * an iommu. Doing anything when there is no iommu is definitely
275 	 * unsafe...
276 	 */
277 	domain = iommu_get_domain_for_dev(dev);
278 	if (!(xhci->quirks & XHCI_ZERO_64B_REGS) || !domain ||
279 	    domain->type == IOMMU_DOMAIN_IDENTITY)
280 		return;
281 
282 	xhci_info(xhci, "Zeroing 64bit base registers, expecting fault\n");
283 
284 	/* Clear HSEIE so that faults do not get signaled */
285 	val = readl(&xhci->op_regs->command);
286 	val &= ~CMD_HSEIE;
287 	writel(val, &xhci->op_regs->command);
288 
289 	/* Clear HSE (aka FATAL) */
290 	val = readl(&xhci->op_regs->status);
291 	val |= STS_FATAL;
292 	writel(val, &xhci->op_regs->status);
293 
294 	/* Now zero the registers, and brace for impact */
295 	val = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
296 	if (upper_32_bits(val))
297 		xhci_write_64(xhci, 0, &xhci->op_regs->dcbaa_ptr);
298 	val = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
299 	if (upper_32_bits(val))
300 		xhci_write_64(xhci, 0, &xhci->op_regs->cmd_ring);
301 
302 	intrs = min_t(u32, HCS_MAX_INTRS(xhci->hcs_params1),
303 		      ARRAY_SIZE(xhci->run_regs->ir_set));
304 
305 	for (i = 0; i < intrs; i++) {
306 		struct xhci_intr_reg __iomem *ir;
307 
308 		ir = &xhci->run_regs->ir_set[i];
309 		val = xhci_read_64(xhci, &ir->erst_base);
310 		if (upper_32_bits(val))
311 			xhci_write_64(xhci, 0, &ir->erst_base);
312 		val= xhci_read_64(xhci, &ir->erst_dequeue);
313 		if (upper_32_bits(val))
314 			xhci_write_64(xhci, 0, &ir->erst_dequeue);
315 	}
316 
317 	/* Wait for the fault to appear. It will be cleared on reset */
318 	err = xhci_handshake(&xhci->op_regs->status,
319 			     STS_FATAL, STS_FATAL,
320 			     XHCI_MAX_HALT_USEC);
321 	if (!err)
322 		xhci_info(xhci, "Fault detected\n");
323 }
324 
325 static int xhci_enable_interrupter(struct xhci_interrupter *ir)
326 {
327 	u32 iman;
328 
329 	if (!ir || !ir->ir_set)
330 		return -EINVAL;
331 
332 	iman = readl(&ir->ir_set->irq_pending);
333 	writel(ER_IRQ_ENABLE(iman), &ir->ir_set->irq_pending);
334 
335 	return 0;
336 }
337 
338 static int xhci_disable_interrupter(struct xhci_interrupter *ir)
339 {
340 	u32 iman;
341 
342 	if (!ir || !ir->ir_set)
343 		return -EINVAL;
344 
345 	iman = readl(&ir->ir_set->irq_pending);
346 	writel(ER_IRQ_DISABLE(iman), &ir->ir_set->irq_pending);
347 
348 	return 0;
349 }
350 
351 /* interrupt moderation interval imod_interval in nanoseconds */
352 int xhci_set_interrupter_moderation(struct xhci_interrupter *ir,
353 				    u32 imod_interval)
354 {
355 	u32 imod;
356 
357 	if (!ir || !ir->ir_set || imod_interval > U16_MAX * 250)
358 		return -EINVAL;
359 
360 	imod = readl(&ir->ir_set->irq_control);
361 	imod &= ~ER_IRQ_INTERVAL_MASK;
362 	imod |= (imod_interval / 250) & ER_IRQ_INTERVAL_MASK;
363 	writel(imod, &ir->ir_set->irq_control);
364 
365 	return 0;
366 }
367 
368 static void compliance_mode_recovery(struct timer_list *t)
369 {
370 	struct xhci_hcd *xhci;
371 	struct usb_hcd *hcd;
372 	struct xhci_hub *rhub;
373 	u32 temp;
374 	int i;
375 
376 	xhci = from_timer(xhci, t, comp_mode_recovery_timer);
377 	rhub = &xhci->usb3_rhub;
378 	hcd = rhub->hcd;
379 
380 	if (!hcd)
381 		return;
382 
383 	for (i = 0; i < rhub->num_ports; i++) {
384 		temp = readl(rhub->ports[i]->addr);
385 		if ((temp & PORT_PLS_MASK) == USB_SS_PORT_LS_COMP_MOD) {
386 			/*
387 			 * Compliance Mode Detected. Letting USB Core
388 			 * handle the Warm Reset
389 			 */
390 			xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
391 					"Compliance mode detected->port %d",
392 					i + 1);
393 			xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
394 					"Attempting compliance mode recovery");
395 
396 			if (hcd->state == HC_STATE_SUSPENDED)
397 				usb_hcd_resume_root_hub(hcd);
398 
399 			usb_hcd_poll_rh_status(hcd);
400 		}
401 	}
402 
403 	if (xhci->port_status_u0 != ((1 << rhub->num_ports) - 1))
404 		mod_timer(&xhci->comp_mode_recovery_timer,
405 			jiffies + msecs_to_jiffies(COMP_MODE_RCVRY_MSECS));
406 }
407 
408 /*
409  * Quirk to work around issue generated by the SN65LVPE502CP USB3.0 re-driver
410  * that causes ports behind that hardware to enter compliance mode sometimes.
411  * The quirk creates a timer that polls every 2 seconds the link state of
412  * each host controller's port and recovers it by issuing a Warm reset
413  * if Compliance mode is detected, otherwise the port will become "dead" (no
414  * device connections or disconnections will be detected anymore). Becasue no
415  * status event is generated when entering compliance mode (per xhci spec),
416  * this quirk is needed on systems that have the failing hardware installed.
417  */
418 static void compliance_mode_recovery_timer_init(struct xhci_hcd *xhci)
419 {
420 	xhci->port_status_u0 = 0;
421 	timer_setup(&xhci->comp_mode_recovery_timer, compliance_mode_recovery,
422 		    0);
423 	xhci->comp_mode_recovery_timer.expires = jiffies +
424 			msecs_to_jiffies(COMP_MODE_RCVRY_MSECS);
425 
426 	add_timer(&xhci->comp_mode_recovery_timer);
427 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
428 			"Compliance mode recovery timer initialized");
429 }
430 
431 /*
432  * This function identifies the systems that have installed the SN65LVPE502CP
433  * USB3.0 re-driver and that need the Compliance Mode Quirk.
434  * Systems:
435  * Vendor: Hewlett-Packard -> System Models: Z420, Z620 and Z820
436  */
437 static bool xhci_compliance_mode_recovery_timer_quirk_check(void)
438 {
439 	const char *dmi_product_name, *dmi_sys_vendor;
440 
441 	dmi_product_name = dmi_get_system_info(DMI_PRODUCT_NAME);
442 	dmi_sys_vendor = dmi_get_system_info(DMI_SYS_VENDOR);
443 	if (!dmi_product_name || !dmi_sys_vendor)
444 		return false;
445 
446 	if (!(strstr(dmi_sys_vendor, "Hewlett-Packard")))
447 		return false;
448 
449 	if (strstr(dmi_product_name, "Z420") ||
450 			strstr(dmi_product_name, "Z620") ||
451 			strstr(dmi_product_name, "Z820") ||
452 			strstr(dmi_product_name, "Z1 Workstation"))
453 		return true;
454 
455 	return false;
456 }
457 
458 static int xhci_all_ports_seen_u0(struct xhci_hcd *xhci)
459 {
460 	return (xhci->port_status_u0 == ((1 << xhci->usb3_rhub.num_ports) - 1));
461 }
462 
463 
464 /*
465  * Initialize memory for HCD and xHC (one-time init).
466  *
467  * Program the PAGESIZE register, initialize the device context array, create
468  * device contexts (?), set up a command ring segment (or two?), create event
469  * ring (one for now).
470  */
471 static int xhci_init(struct usb_hcd *hcd)
472 {
473 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
474 	int retval;
475 
476 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "xhci_init");
477 	spin_lock_init(&xhci->lock);
478 
479 	retval = xhci_mem_init(xhci, GFP_KERNEL);
480 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Finished xhci_init");
481 
482 	/* Initializing Compliance Mode Recovery Data If Needed */
483 	if (xhci_compliance_mode_recovery_timer_quirk_check()) {
484 		xhci->quirks |= XHCI_COMP_MODE_QUIRK;
485 		compliance_mode_recovery_timer_init(xhci);
486 	}
487 
488 	return retval;
489 }
490 
491 /*-------------------------------------------------------------------------*/
492 
493 static int xhci_run_finished(struct xhci_hcd *xhci)
494 {
495 	struct xhci_interrupter *ir = xhci->interrupters[0];
496 	unsigned long	flags;
497 	u32		temp;
498 
499 	/*
500 	 * Enable interrupts before starting the host (xhci 4.2 and 5.5.2).
501 	 * Protect the short window before host is running with a lock
502 	 */
503 	spin_lock_irqsave(&xhci->lock, flags);
504 
505 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Enable interrupts");
506 	temp = readl(&xhci->op_regs->command);
507 	temp |= (CMD_EIE);
508 	writel(temp, &xhci->op_regs->command);
509 
510 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Enable primary interrupter");
511 	xhci_enable_interrupter(ir);
512 
513 	if (xhci_start(xhci)) {
514 		xhci_halt(xhci);
515 		spin_unlock_irqrestore(&xhci->lock, flags);
516 		return -ENODEV;
517 	}
518 
519 	xhci->cmd_ring_state = CMD_RING_STATE_RUNNING;
520 
521 	if (xhci->quirks & XHCI_NEC_HOST)
522 		xhci_ring_cmd_db(xhci);
523 
524 	spin_unlock_irqrestore(&xhci->lock, flags);
525 
526 	return 0;
527 }
528 
529 /*
530  * Start the HC after it was halted.
531  *
532  * This function is called by the USB core when the HC driver is added.
533  * Its opposite is xhci_stop().
534  *
535  * xhci_init() must be called once before this function can be called.
536  * Reset the HC, enable device slot contexts, program DCBAAP, and
537  * set command ring pointer and event ring pointer.
538  *
539  * Setup MSI-X vectors and enable interrupts.
540  */
541 int xhci_run(struct usb_hcd *hcd)
542 {
543 	u64 temp_64;
544 	int ret;
545 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
546 	struct xhci_interrupter *ir = xhci->interrupters[0];
547 	/* Start the xHCI host controller running only after the USB 2.0 roothub
548 	 * is setup.
549 	 */
550 
551 	hcd->uses_new_polling = 1;
552 	if (hcd->msi_enabled)
553 		ir->ip_autoclear = true;
554 
555 	if (!usb_hcd_is_primary_hcd(hcd))
556 		return xhci_run_finished(xhci);
557 
558 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "xhci_run");
559 
560 	temp_64 = xhci_read_64(xhci, &ir->ir_set->erst_dequeue);
561 	temp_64 &= ERST_PTR_MASK;
562 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
563 			"ERST deq = 64'h%0lx", (long unsigned int) temp_64);
564 
565 	xhci_set_interrupter_moderation(ir, xhci->imod_interval);
566 
567 	if (xhci->quirks & XHCI_NEC_HOST) {
568 		struct xhci_command *command;
569 
570 		command = xhci_alloc_command(xhci, false, GFP_KERNEL);
571 		if (!command)
572 			return -ENOMEM;
573 
574 		ret = xhci_queue_vendor_command(xhci, command, 0, 0, 0,
575 				TRB_TYPE(TRB_NEC_GET_FW));
576 		if (ret)
577 			xhci_free_command(xhci, command);
578 	}
579 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
580 			"Finished %s for main hcd", __func__);
581 
582 	xhci_create_dbc_dev(xhci);
583 
584 	xhci_debugfs_init(xhci);
585 
586 	if (xhci_has_one_roothub(xhci))
587 		return xhci_run_finished(xhci);
588 
589 	set_bit(HCD_FLAG_DEFER_RH_REGISTER, &hcd->flags);
590 
591 	return 0;
592 }
593 EXPORT_SYMBOL_GPL(xhci_run);
594 
595 /*
596  * Stop xHCI driver.
597  *
598  * This function is called by the USB core when the HC driver is removed.
599  * Its opposite is xhci_run().
600  *
601  * Disable device contexts, disable IRQs, and quiesce the HC.
602  * Reset the HC, finish any completed transactions, and cleanup memory.
603  */
604 void xhci_stop(struct usb_hcd *hcd)
605 {
606 	u32 temp;
607 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
608 	struct xhci_interrupter *ir = xhci->interrupters[0];
609 
610 	mutex_lock(&xhci->mutex);
611 
612 	/* Only halt host and free memory after both hcds are removed */
613 	if (!usb_hcd_is_primary_hcd(hcd)) {
614 		mutex_unlock(&xhci->mutex);
615 		return;
616 	}
617 
618 	xhci_remove_dbc_dev(xhci);
619 
620 	spin_lock_irq(&xhci->lock);
621 	xhci->xhc_state |= XHCI_STATE_HALTED;
622 	xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
623 	xhci_halt(xhci);
624 	xhci_reset(xhci, XHCI_RESET_SHORT_USEC);
625 	spin_unlock_irq(&xhci->lock);
626 
627 	/* Deleting Compliance Mode Recovery Timer */
628 	if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
629 			(!(xhci_all_ports_seen_u0(xhci)))) {
630 		del_timer_sync(&xhci->comp_mode_recovery_timer);
631 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
632 				"%s: compliance mode recovery timer deleted",
633 				__func__);
634 	}
635 
636 	if (xhci->quirks & XHCI_AMD_PLL_FIX)
637 		usb_amd_dev_put();
638 
639 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
640 			"// Disabling event ring interrupts");
641 	temp = readl(&xhci->op_regs->status);
642 	writel((temp & ~0x1fff) | STS_EINT, &xhci->op_regs->status);
643 	xhci_disable_interrupter(ir);
644 
645 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "cleaning up memory");
646 	xhci_mem_cleanup(xhci);
647 	xhci_debugfs_exit(xhci);
648 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
649 			"xhci_stop completed - status = %x",
650 			readl(&xhci->op_regs->status));
651 	mutex_unlock(&xhci->mutex);
652 }
653 EXPORT_SYMBOL_GPL(xhci_stop);
654 
655 /*
656  * Shutdown HC (not bus-specific)
657  *
658  * This is called when the machine is rebooting or halting.  We assume that the
659  * machine will be powered off, and the HC's internal state will be reset.
660  * Don't bother to free memory.
661  *
662  * This will only ever be called with the main usb_hcd (the USB3 roothub).
663  */
664 void xhci_shutdown(struct usb_hcd *hcd)
665 {
666 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
667 
668 	if (xhci->quirks & XHCI_SPURIOUS_REBOOT)
669 		usb_disable_xhci_ports(to_pci_dev(hcd->self.sysdev));
670 
671 	/* Don't poll the roothubs after shutdown. */
672 	xhci_dbg(xhci, "%s: stopping usb%d port polling.\n",
673 			__func__, hcd->self.busnum);
674 	clear_bit(HCD_FLAG_POLL_RH, &hcd->flags);
675 	del_timer_sync(&hcd->rh_timer);
676 
677 	if (xhci->shared_hcd) {
678 		clear_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags);
679 		del_timer_sync(&xhci->shared_hcd->rh_timer);
680 	}
681 
682 	spin_lock_irq(&xhci->lock);
683 	xhci_halt(xhci);
684 
685 	/*
686 	 * Workaround for spurious wakeps at shutdown with HSW, and for boot
687 	 * firmware delay in ADL-P PCH if port are left in U3 at shutdown
688 	 */
689 	if (xhci->quirks & XHCI_SPURIOUS_WAKEUP ||
690 	    xhci->quirks & XHCI_RESET_TO_DEFAULT)
691 		xhci_reset(xhci, XHCI_RESET_SHORT_USEC);
692 
693 	spin_unlock_irq(&xhci->lock);
694 
695 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
696 			"xhci_shutdown completed - status = %x",
697 			readl(&xhci->op_regs->status));
698 }
699 EXPORT_SYMBOL_GPL(xhci_shutdown);
700 
701 #ifdef CONFIG_PM
702 static void xhci_save_registers(struct xhci_hcd *xhci)
703 {
704 	struct xhci_interrupter *ir;
705 	unsigned int i;
706 
707 	xhci->s3.command = readl(&xhci->op_regs->command);
708 	xhci->s3.dev_nt = readl(&xhci->op_regs->dev_notification);
709 	xhci->s3.dcbaa_ptr = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
710 	xhci->s3.config_reg = readl(&xhci->op_regs->config_reg);
711 
712 	/* save both primary and all secondary interrupters */
713 	/* fixme, shold we lock  to prevent race with remove secondary interrupter? */
714 	for (i = 0; i < xhci->max_interrupters; i++) {
715 		ir = xhci->interrupters[i];
716 		if (!ir)
717 			continue;
718 
719 		ir->s3_erst_size = readl(&ir->ir_set->erst_size);
720 		ir->s3_erst_base = xhci_read_64(xhci, &ir->ir_set->erst_base);
721 		ir->s3_erst_dequeue = xhci_read_64(xhci, &ir->ir_set->erst_dequeue);
722 		ir->s3_irq_pending = readl(&ir->ir_set->irq_pending);
723 		ir->s3_irq_control = readl(&ir->ir_set->irq_control);
724 	}
725 }
726 
727 static void xhci_restore_registers(struct xhci_hcd *xhci)
728 {
729 	struct xhci_interrupter *ir;
730 	unsigned int i;
731 
732 	writel(xhci->s3.command, &xhci->op_regs->command);
733 	writel(xhci->s3.dev_nt, &xhci->op_regs->dev_notification);
734 	xhci_write_64(xhci, xhci->s3.dcbaa_ptr, &xhci->op_regs->dcbaa_ptr);
735 	writel(xhci->s3.config_reg, &xhci->op_regs->config_reg);
736 
737 	/* FIXME should we lock to protect against freeing of interrupters */
738 	for (i = 0; i < xhci->max_interrupters; i++) {
739 		ir = xhci->interrupters[i];
740 		if (!ir)
741 			continue;
742 
743 		writel(ir->s3_erst_size, &ir->ir_set->erst_size);
744 		xhci_write_64(xhci, ir->s3_erst_base, &ir->ir_set->erst_base);
745 		xhci_write_64(xhci, ir->s3_erst_dequeue, &ir->ir_set->erst_dequeue);
746 		writel(ir->s3_irq_pending, &ir->ir_set->irq_pending);
747 		writel(ir->s3_irq_control, &ir->ir_set->irq_control);
748 	}
749 }
750 
751 static void xhci_set_cmd_ring_deq(struct xhci_hcd *xhci)
752 {
753 	u64	val_64;
754 
755 	/* step 2: initialize command ring buffer */
756 	val_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
757 	val_64 = (val_64 & (u64) CMD_RING_RSVD_BITS) |
758 		(xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg,
759 				      xhci->cmd_ring->dequeue) &
760 		 (u64) ~CMD_RING_RSVD_BITS) |
761 		xhci->cmd_ring->cycle_state;
762 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
763 			"// Setting command ring address to 0x%llx",
764 			(long unsigned long) val_64);
765 	xhci_write_64(xhci, val_64, &xhci->op_regs->cmd_ring);
766 }
767 
768 /*
769  * The whole command ring must be cleared to zero when we suspend the host.
770  *
771  * The host doesn't save the command ring pointer in the suspend well, so we
772  * need to re-program it on resume.  Unfortunately, the pointer must be 64-byte
773  * aligned, because of the reserved bits in the command ring dequeue pointer
774  * register.  Therefore, we can't just set the dequeue pointer back in the
775  * middle of the ring (TRBs are 16-byte aligned).
776  */
777 static void xhci_clear_command_ring(struct xhci_hcd *xhci)
778 {
779 	struct xhci_ring *ring;
780 	struct xhci_segment *seg;
781 
782 	ring = xhci->cmd_ring;
783 	xhci_for_each_ring_seg(ring->first_seg, seg)
784 		memset(seg->trbs, 0, sizeof(union xhci_trb) * (TRBS_PER_SEGMENT - 1));
785 
786 	xhci_initialize_ring_info(ring);
787 	/*
788 	 * Reset the hardware dequeue pointer.
789 	 * Yes, this will need to be re-written after resume, but we're paranoid
790 	 * and want to make sure the hardware doesn't access bogus memory
791 	 * because, say, the BIOS or an SMI started the host without changing
792 	 * the command ring pointers.
793 	 */
794 	xhci_set_cmd_ring_deq(xhci);
795 }
796 
797 /*
798  * Disable port wake bits if do_wakeup is not set.
799  *
800  * Also clear a possible internal port wake state left hanging for ports that
801  * detected termination but never successfully enumerated (trained to 0U).
802  * Internal wake causes immediate xHCI wake after suspend. PORT_CSC write done
803  * at enumeration clears this wake, force one here as well for unconnected ports
804  */
805 
806 static void xhci_disable_hub_port_wake(struct xhci_hcd *xhci,
807 				       struct xhci_hub *rhub,
808 				       bool do_wakeup)
809 {
810 	unsigned long flags;
811 	u32 t1, t2, portsc;
812 	int i;
813 
814 	spin_lock_irqsave(&xhci->lock, flags);
815 
816 	for (i = 0; i < rhub->num_ports; i++) {
817 		portsc = readl(rhub->ports[i]->addr);
818 		t1 = xhci_port_state_to_neutral(portsc);
819 		t2 = t1;
820 
821 		/* clear wake bits if do_wake is not set */
822 		if (!do_wakeup)
823 			t2 &= ~PORT_WAKE_BITS;
824 
825 		/* Don't touch csc bit if connected or connect change is set */
826 		if (!(portsc & (PORT_CSC | PORT_CONNECT)))
827 			t2 |= PORT_CSC;
828 
829 		if (t1 != t2) {
830 			writel(t2, rhub->ports[i]->addr);
831 			xhci_dbg(xhci, "config port %d-%d wake bits, portsc: 0x%x, write: 0x%x\n",
832 				 rhub->hcd->self.busnum, i + 1, portsc, t2);
833 		}
834 	}
835 	spin_unlock_irqrestore(&xhci->lock, flags);
836 }
837 
838 static bool xhci_pending_portevent(struct xhci_hcd *xhci)
839 {
840 	struct xhci_port	**ports;
841 	int			port_index;
842 	u32			status;
843 	u32			portsc;
844 
845 	status = readl(&xhci->op_regs->status);
846 	if (status & STS_EINT)
847 		return true;
848 	/*
849 	 * Checking STS_EINT is not enough as there is a lag between a change
850 	 * bit being set and the Port Status Change Event that it generated
851 	 * being written to the Event Ring. See note in xhci 1.1 section 4.19.2.
852 	 */
853 
854 	port_index = xhci->usb2_rhub.num_ports;
855 	ports = xhci->usb2_rhub.ports;
856 	while (port_index--) {
857 		portsc = readl(ports[port_index]->addr);
858 		if (portsc & PORT_CHANGE_MASK ||
859 		    (portsc & PORT_PLS_MASK) == XDEV_RESUME)
860 			return true;
861 	}
862 	port_index = xhci->usb3_rhub.num_ports;
863 	ports = xhci->usb3_rhub.ports;
864 	while (port_index--) {
865 		portsc = readl(ports[port_index]->addr);
866 		if (portsc & (PORT_CHANGE_MASK | PORT_CAS) ||
867 		    (portsc & PORT_PLS_MASK) == XDEV_RESUME)
868 			return true;
869 	}
870 	return false;
871 }
872 
873 /*
874  * Stop HC (not bus-specific)
875  *
876  * This is called when the machine transition into S3/S4 mode.
877  *
878  */
879 int xhci_suspend(struct xhci_hcd *xhci, bool do_wakeup)
880 {
881 	int			rc = 0;
882 	unsigned int		delay = XHCI_MAX_HALT_USEC * 2;
883 	struct usb_hcd		*hcd = xhci_to_hcd(xhci);
884 	u32			command;
885 	u32			res;
886 
887 	if (!hcd->state)
888 		return 0;
889 
890 	if (hcd->state != HC_STATE_SUSPENDED ||
891 	    (xhci->shared_hcd && xhci->shared_hcd->state != HC_STATE_SUSPENDED))
892 		return -EINVAL;
893 
894 	/* Clear root port wake on bits if wakeup not allowed. */
895 	xhci_disable_hub_port_wake(xhci, &xhci->usb3_rhub, do_wakeup);
896 	xhci_disable_hub_port_wake(xhci, &xhci->usb2_rhub, do_wakeup);
897 
898 	if (!HCD_HW_ACCESSIBLE(hcd))
899 		return 0;
900 
901 	xhci_dbc_suspend(xhci);
902 
903 	/* Don't poll the roothubs on bus suspend. */
904 	xhci_dbg(xhci, "%s: stopping usb%d port polling.\n",
905 		 __func__, hcd->self.busnum);
906 	clear_bit(HCD_FLAG_POLL_RH, &hcd->flags);
907 	del_timer_sync(&hcd->rh_timer);
908 	if (xhci->shared_hcd) {
909 		clear_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags);
910 		del_timer_sync(&xhci->shared_hcd->rh_timer);
911 	}
912 
913 	if (xhci->quirks & XHCI_SUSPEND_DELAY)
914 		usleep_range(1000, 1500);
915 
916 	spin_lock_irq(&xhci->lock);
917 	clear_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
918 	if (xhci->shared_hcd)
919 		clear_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags);
920 	/* step 1: stop endpoint */
921 	/* skipped assuming that port suspend has done */
922 
923 	/* step 2: clear Run/Stop bit */
924 	command = readl(&xhci->op_regs->command);
925 	command &= ~CMD_RUN;
926 	writel(command, &xhci->op_regs->command);
927 
928 	/* Some chips from Fresco Logic need an extraordinary delay */
929 	delay *= (xhci->quirks & XHCI_SLOW_SUSPEND) ? 10 : 1;
930 
931 	if (xhci_handshake(&xhci->op_regs->status,
932 		      STS_HALT, STS_HALT, delay)) {
933 		xhci_warn(xhci, "WARN: xHC CMD_RUN timeout\n");
934 		spin_unlock_irq(&xhci->lock);
935 		return -ETIMEDOUT;
936 	}
937 	xhci_clear_command_ring(xhci);
938 
939 	/* step 3: save registers */
940 	xhci_save_registers(xhci);
941 
942 	/* step 4: set CSS flag */
943 	command = readl(&xhci->op_regs->command);
944 	command |= CMD_CSS;
945 	writel(command, &xhci->op_regs->command);
946 	xhci->broken_suspend = 0;
947 	if (xhci_handshake(&xhci->op_regs->status,
948 				STS_SAVE, 0, 20 * 1000)) {
949 	/*
950 	 * AMD SNPS xHC 3.0 occasionally does not clear the
951 	 * SSS bit of USBSTS and when driver tries to poll
952 	 * to see if the xHC clears BIT(8) which never happens
953 	 * and driver assumes that controller is not responding
954 	 * and times out. To workaround this, its good to check
955 	 * if SRE and HCE bits are not set (as per xhci
956 	 * Section 5.4.2) and bypass the timeout.
957 	 */
958 		res = readl(&xhci->op_regs->status);
959 		if ((xhci->quirks & XHCI_SNPS_BROKEN_SUSPEND) &&
960 		    (((res & STS_SRE) == 0) &&
961 				((res & STS_HCE) == 0))) {
962 			xhci->broken_suspend = 1;
963 		} else {
964 			xhci_warn(xhci, "WARN: xHC save state timeout\n");
965 			spin_unlock_irq(&xhci->lock);
966 			return -ETIMEDOUT;
967 		}
968 	}
969 	spin_unlock_irq(&xhci->lock);
970 
971 	/*
972 	 * Deleting Compliance Mode Recovery Timer because the xHCI Host
973 	 * is about to be suspended.
974 	 */
975 	if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
976 			(!(xhci_all_ports_seen_u0(xhci)))) {
977 		del_timer_sync(&xhci->comp_mode_recovery_timer);
978 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
979 				"%s: compliance mode recovery timer deleted",
980 				__func__);
981 	}
982 
983 	return rc;
984 }
985 EXPORT_SYMBOL_GPL(xhci_suspend);
986 
987 /*
988  * start xHC (not bus-specific)
989  *
990  * This is called when the machine transition from S3/S4 mode.
991  *
992  */
993 int xhci_resume(struct xhci_hcd *xhci, pm_message_t msg)
994 {
995 	bool			hibernated = (msg.event == PM_EVENT_RESTORE);
996 	u32			command, temp = 0;
997 	struct usb_hcd		*hcd = xhci_to_hcd(xhci);
998 	int			retval = 0;
999 	bool			comp_timer_running = false;
1000 	bool			pending_portevent = false;
1001 	bool			suspended_usb3_devs = false;
1002 	bool			reinit_xhc = false;
1003 
1004 	if (!hcd->state)
1005 		return 0;
1006 
1007 	/* Wait a bit if either of the roothubs need to settle from the
1008 	 * transition into bus suspend.
1009 	 */
1010 
1011 	if (time_before(jiffies, xhci->usb2_rhub.bus_state.next_statechange) ||
1012 	    time_before(jiffies, xhci->usb3_rhub.bus_state.next_statechange))
1013 		msleep(100);
1014 
1015 	set_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
1016 	if (xhci->shared_hcd)
1017 		set_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags);
1018 
1019 	spin_lock_irq(&xhci->lock);
1020 
1021 	if (hibernated || xhci->quirks & XHCI_RESET_ON_RESUME || xhci->broken_suspend)
1022 		reinit_xhc = true;
1023 
1024 	if (!reinit_xhc) {
1025 		/*
1026 		 * Some controllers might lose power during suspend, so wait
1027 		 * for controller not ready bit to clear, just as in xHC init.
1028 		 */
1029 		retval = xhci_handshake(&xhci->op_regs->status,
1030 					STS_CNR, 0, 10 * 1000 * 1000);
1031 		if (retval) {
1032 			xhci_warn(xhci, "Controller not ready at resume %d\n",
1033 				  retval);
1034 			spin_unlock_irq(&xhci->lock);
1035 			return retval;
1036 		}
1037 		/* step 1: restore register */
1038 		xhci_restore_registers(xhci);
1039 		/* step 2: initialize command ring buffer */
1040 		xhci_set_cmd_ring_deq(xhci);
1041 		/* step 3: restore state and start state*/
1042 		/* step 3: set CRS flag */
1043 		command = readl(&xhci->op_regs->command);
1044 		command |= CMD_CRS;
1045 		writel(command, &xhci->op_regs->command);
1046 		/*
1047 		 * Some controllers take up to 55+ ms to complete the controller
1048 		 * restore so setting the timeout to 100ms. Xhci specification
1049 		 * doesn't mention any timeout value.
1050 		 */
1051 		if (xhci_handshake(&xhci->op_regs->status,
1052 			      STS_RESTORE, 0, 100 * 1000)) {
1053 			xhci_warn(xhci, "WARN: xHC restore state timeout\n");
1054 			spin_unlock_irq(&xhci->lock);
1055 			return -ETIMEDOUT;
1056 		}
1057 	}
1058 
1059 	temp = readl(&xhci->op_regs->status);
1060 
1061 	/* re-initialize the HC on Restore Error, or Host Controller Error */
1062 	if ((temp & (STS_SRE | STS_HCE)) &&
1063 	    !(xhci->xhc_state & XHCI_STATE_REMOVING)) {
1064 		reinit_xhc = true;
1065 		if (!xhci->broken_suspend)
1066 			xhci_warn(xhci, "xHC error in resume, USBSTS 0x%x, Reinit\n", temp);
1067 	}
1068 
1069 	if (reinit_xhc) {
1070 		if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
1071 				!(xhci_all_ports_seen_u0(xhci))) {
1072 			del_timer_sync(&xhci->comp_mode_recovery_timer);
1073 			xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1074 				"Compliance Mode Recovery Timer deleted!");
1075 		}
1076 
1077 		/* Let the USB core know _both_ roothubs lost power. */
1078 		usb_root_hub_lost_power(xhci->main_hcd->self.root_hub);
1079 		if (xhci->shared_hcd)
1080 			usb_root_hub_lost_power(xhci->shared_hcd->self.root_hub);
1081 
1082 		xhci_dbg(xhci, "Stop HCD\n");
1083 		xhci_halt(xhci);
1084 		xhci_zero_64b_regs(xhci);
1085 		retval = xhci_reset(xhci, XHCI_RESET_LONG_USEC);
1086 		spin_unlock_irq(&xhci->lock);
1087 		if (retval)
1088 			return retval;
1089 
1090 		xhci_dbg(xhci, "// Disabling event ring interrupts\n");
1091 		temp = readl(&xhci->op_regs->status);
1092 		writel((temp & ~0x1fff) | STS_EINT, &xhci->op_regs->status);
1093 		xhci_disable_interrupter(xhci->interrupters[0]);
1094 
1095 		xhci_dbg(xhci, "cleaning up memory\n");
1096 		xhci_mem_cleanup(xhci);
1097 		xhci_debugfs_exit(xhci);
1098 		xhci_dbg(xhci, "xhci_stop completed - status = %x\n",
1099 			    readl(&xhci->op_regs->status));
1100 
1101 		/* USB core calls the PCI reinit and start functions twice:
1102 		 * first with the primary HCD, and then with the secondary HCD.
1103 		 * If we don't do the same, the host will never be started.
1104 		 */
1105 		xhci_dbg(xhci, "Initialize the xhci_hcd\n");
1106 		retval = xhci_init(hcd);
1107 		if (retval)
1108 			return retval;
1109 		comp_timer_running = true;
1110 
1111 		xhci_dbg(xhci, "Start the primary HCD\n");
1112 		retval = xhci_run(hcd);
1113 		if (!retval && xhci->shared_hcd) {
1114 			xhci_dbg(xhci, "Start the secondary HCD\n");
1115 			retval = xhci_run(xhci->shared_hcd);
1116 		}
1117 		if (retval)
1118 			return retval;
1119 		/*
1120 		 * Resume roothubs unconditionally as PORTSC change bits are not
1121 		 * immediately visible after xHC reset
1122 		 */
1123 		hcd->state = HC_STATE_SUSPENDED;
1124 
1125 		if (xhci->shared_hcd) {
1126 			xhci->shared_hcd->state = HC_STATE_SUSPENDED;
1127 			usb_hcd_resume_root_hub(xhci->shared_hcd);
1128 		}
1129 		usb_hcd_resume_root_hub(hcd);
1130 
1131 		goto done;
1132 	}
1133 
1134 	/* step 4: set Run/Stop bit */
1135 	command = readl(&xhci->op_regs->command);
1136 	command |= CMD_RUN;
1137 	writel(command, &xhci->op_regs->command);
1138 	xhci_handshake(&xhci->op_regs->status, STS_HALT,
1139 		  0, 250 * 1000);
1140 
1141 	/* step 5: walk topology and initialize portsc,
1142 	 * portpmsc and portli
1143 	 */
1144 	/* this is done in bus_resume */
1145 
1146 	/* step 6: restart each of the previously
1147 	 * Running endpoints by ringing their doorbells
1148 	 */
1149 
1150 	spin_unlock_irq(&xhci->lock);
1151 
1152 	xhci_dbc_resume(xhci);
1153 
1154 	if (retval == 0) {
1155 		/*
1156 		 * Resume roothubs only if there are pending events.
1157 		 * USB 3 devices resend U3 LFPS wake after a 100ms delay if
1158 		 * the first wake signalling failed, give it that chance if
1159 		 * there are suspended USB 3 devices.
1160 		 */
1161 		if (xhci->usb3_rhub.bus_state.suspended_ports ||
1162 		    xhci->usb3_rhub.bus_state.bus_suspended)
1163 			suspended_usb3_devs = true;
1164 
1165 		pending_portevent = xhci_pending_portevent(xhci);
1166 
1167 		if (suspended_usb3_devs && !pending_portevent &&
1168 		    msg.event == PM_EVENT_AUTO_RESUME) {
1169 			msleep(120);
1170 			pending_portevent = xhci_pending_portevent(xhci);
1171 		}
1172 
1173 		if (pending_portevent) {
1174 			if (xhci->shared_hcd)
1175 				usb_hcd_resume_root_hub(xhci->shared_hcd);
1176 			usb_hcd_resume_root_hub(hcd);
1177 		}
1178 	}
1179 done:
1180 	/*
1181 	 * If system is subject to the Quirk, Compliance Mode Timer needs to
1182 	 * be re-initialized Always after a system resume. Ports are subject
1183 	 * to suffer the Compliance Mode issue again. It doesn't matter if
1184 	 * ports have entered previously to U0 before system's suspension.
1185 	 */
1186 	if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) && !comp_timer_running)
1187 		compliance_mode_recovery_timer_init(xhci);
1188 
1189 	if (xhci->quirks & XHCI_ASMEDIA_MODIFY_FLOWCONTROL)
1190 		usb_asmedia_modifyflowcontrol(to_pci_dev(hcd->self.controller));
1191 
1192 	/* Re-enable port polling. */
1193 	xhci_dbg(xhci, "%s: starting usb%d port polling.\n",
1194 		 __func__, hcd->self.busnum);
1195 	if (xhci->shared_hcd) {
1196 		set_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags);
1197 		usb_hcd_poll_rh_status(xhci->shared_hcd);
1198 	}
1199 	set_bit(HCD_FLAG_POLL_RH, &hcd->flags);
1200 	usb_hcd_poll_rh_status(hcd);
1201 
1202 	return retval;
1203 }
1204 EXPORT_SYMBOL_GPL(xhci_resume);
1205 #endif	/* CONFIG_PM */
1206 
1207 /*-------------------------------------------------------------------------*/
1208 
1209 static int xhci_map_temp_buffer(struct usb_hcd *hcd, struct urb *urb)
1210 {
1211 	void *temp;
1212 	int ret = 0;
1213 	unsigned int buf_len;
1214 	enum dma_data_direction dir;
1215 
1216 	dir = usb_urb_dir_in(urb) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
1217 	buf_len = urb->transfer_buffer_length;
1218 
1219 	temp = kzalloc_node(buf_len, GFP_ATOMIC,
1220 			    dev_to_node(hcd->self.sysdev));
1221 	if (!temp)
1222 		return -ENOMEM;
1223 
1224 	if (usb_urb_dir_out(urb))
1225 		sg_pcopy_to_buffer(urb->sg, urb->num_sgs,
1226 				   temp, buf_len, 0);
1227 
1228 	urb->transfer_buffer = temp;
1229 	urb->transfer_dma = dma_map_single(hcd->self.sysdev,
1230 					   urb->transfer_buffer,
1231 					   urb->transfer_buffer_length,
1232 					   dir);
1233 
1234 	if (dma_mapping_error(hcd->self.sysdev,
1235 			      urb->transfer_dma)) {
1236 		ret = -EAGAIN;
1237 		kfree(temp);
1238 	} else {
1239 		urb->transfer_flags |= URB_DMA_MAP_SINGLE;
1240 	}
1241 
1242 	return ret;
1243 }
1244 
1245 static bool xhci_urb_temp_buffer_required(struct usb_hcd *hcd,
1246 					  struct urb *urb)
1247 {
1248 	bool ret = false;
1249 	unsigned int i;
1250 	unsigned int len = 0;
1251 	unsigned int trb_size;
1252 	unsigned int max_pkt;
1253 	struct scatterlist *sg;
1254 	struct scatterlist *tail_sg;
1255 
1256 	tail_sg = urb->sg;
1257 	max_pkt = usb_endpoint_maxp(&urb->ep->desc);
1258 
1259 	if (!urb->num_sgs)
1260 		return ret;
1261 
1262 	if (urb->dev->speed >= USB_SPEED_SUPER)
1263 		trb_size = TRB_CACHE_SIZE_SS;
1264 	else
1265 		trb_size = TRB_CACHE_SIZE_HS;
1266 
1267 	if (urb->transfer_buffer_length != 0 &&
1268 	    !(urb->transfer_flags & URB_NO_TRANSFER_DMA_MAP)) {
1269 		for_each_sg(urb->sg, sg, urb->num_sgs, i) {
1270 			len = len + sg->length;
1271 			if (i > trb_size - 2) {
1272 				len = len - tail_sg->length;
1273 				if (len < max_pkt) {
1274 					ret = true;
1275 					break;
1276 				}
1277 
1278 				tail_sg = sg_next(tail_sg);
1279 			}
1280 		}
1281 	}
1282 	return ret;
1283 }
1284 
1285 static void xhci_unmap_temp_buf(struct usb_hcd *hcd, struct urb *urb)
1286 {
1287 	unsigned int len;
1288 	unsigned int buf_len;
1289 	enum dma_data_direction dir;
1290 
1291 	dir = usb_urb_dir_in(urb) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
1292 
1293 	buf_len = urb->transfer_buffer_length;
1294 
1295 	if (IS_ENABLED(CONFIG_HAS_DMA) &&
1296 	    (urb->transfer_flags & URB_DMA_MAP_SINGLE))
1297 		dma_unmap_single(hcd->self.sysdev,
1298 				 urb->transfer_dma,
1299 				 urb->transfer_buffer_length,
1300 				 dir);
1301 
1302 	if (usb_urb_dir_in(urb)) {
1303 		len = sg_pcopy_from_buffer(urb->sg, urb->num_sgs,
1304 					   urb->transfer_buffer,
1305 					   buf_len,
1306 					   0);
1307 		if (len != buf_len) {
1308 			xhci_dbg(hcd_to_xhci(hcd),
1309 				 "Copy from tmp buf to urb sg list failed\n");
1310 			urb->actual_length = len;
1311 		}
1312 	}
1313 	urb->transfer_flags &= ~URB_DMA_MAP_SINGLE;
1314 	kfree(urb->transfer_buffer);
1315 	urb->transfer_buffer = NULL;
1316 }
1317 
1318 /*
1319  * Bypass the DMA mapping if URB is suitable for Immediate Transfer (IDT),
1320  * we'll copy the actual data into the TRB address register. This is limited to
1321  * transfers up to 8 bytes on output endpoints of any kind with wMaxPacketSize
1322  * >= 8 bytes. If suitable for IDT only one Transfer TRB per TD is allowed.
1323  */
1324 static int xhci_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
1325 				gfp_t mem_flags)
1326 {
1327 	struct xhci_hcd *xhci;
1328 
1329 	xhci = hcd_to_xhci(hcd);
1330 
1331 	if (xhci_urb_suitable_for_idt(urb))
1332 		return 0;
1333 
1334 	if (xhci->quirks & XHCI_SG_TRB_CACHE_SIZE_QUIRK) {
1335 		if (xhci_urb_temp_buffer_required(hcd, urb))
1336 			return xhci_map_temp_buffer(hcd, urb);
1337 	}
1338 	return usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);
1339 }
1340 
1341 static void xhci_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb)
1342 {
1343 	struct xhci_hcd *xhci;
1344 	bool unmap_temp_buf = false;
1345 
1346 	xhci = hcd_to_xhci(hcd);
1347 
1348 	if (urb->num_sgs && (urb->transfer_flags & URB_DMA_MAP_SINGLE))
1349 		unmap_temp_buf = true;
1350 
1351 	if ((xhci->quirks & XHCI_SG_TRB_CACHE_SIZE_QUIRK) && unmap_temp_buf)
1352 		xhci_unmap_temp_buf(hcd, urb);
1353 	else
1354 		usb_hcd_unmap_urb_for_dma(hcd, urb);
1355 }
1356 
1357 /**
1358  * xhci_get_endpoint_index - Used for passing endpoint bitmasks between the core and
1359  * HCDs.  Find the index for an endpoint given its descriptor.  Use the return
1360  * value to right shift 1 for the bitmask.
1361  *
1362  * Index  = (epnum * 2) + direction - 1,
1363  * where direction = 0 for OUT, 1 for IN.
1364  * For control endpoints, the IN index is used (OUT index is unused), so
1365  * index = (epnum * 2) + direction - 1 = (epnum * 2) + 1 - 1 = (epnum * 2)
1366  */
1367 unsigned int xhci_get_endpoint_index(struct usb_endpoint_descriptor *desc)
1368 {
1369 	unsigned int index;
1370 	if (usb_endpoint_xfer_control(desc))
1371 		index = (unsigned int) (usb_endpoint_num(desc)*2);
1372 	else
1373 		index = (unsigned int) (usb_endpoint_num(desc)*2) +
1374 			(usb_endpoint_dir_in(desc) ? 1 : 0) - 1;
1375 	return index;
1376 }
1377 EXPORT_SYMBOL_GPL(xhci_get_endpoint_index);
1378 
1379 /* The reverse operation to xhci_get_endpoint_index. Calculate the USB endpoint
1380  * address from the XHCI endpoint index.
1381  */
1382 static unsigned int xhci_get_endpoint_address(unsigned int ep_index)
1383 {
1384 	unsigned int number = DIV_ROUND_UP(ep_index, 2);
1385 	unsigned int direction = ep_index % 2 ? USB_DIR_OUT : USB_DIR_IN;
1386 	return direction | number;
1387 }
1388 
1389 /* Find the flag for this endpoint (for use in the control context).  Use the
1390  * endpoint index to create a bitmask.  The slot context is bit 0, endpoint 0 is
1391  * bit 1, etc.
1392  */
1393 static unsigned int xhci_get_endpoint_flag(struct usb_endpoint_descriptor *desc)
1394 {
1395 	return 1 << (xhci_get_endpoint_index(desc) + 1);
1396 }
1397 
1398 /* Compute the last valid endpoint context index.  Basically, this is the
1399  * endpoint index plus one.  For slot contexts with more than valid endpoint,
1400  * we find the most significant bit set in the added contexts flags.
1401  * e.g. ep 1 IN (with epnum 0x81) => added_ctxs = 0b1000
1402  * fls(0b1000) = 4, but the endpoint context index is 3, so subtract one.
1403  */
1404 unsigned int xhci_last_valid_endpoint(u32 added_ctxs)
1405 {
1406 	return fls(added_ctxs) - 1;
1407 }
1408 
1409 /* Returns 1 if the arguments are OK;
1410  * returns 0 this is a root hub; returns -EINVAL for NULL pointers.
1411  */
1412 static int xhci_check_args(struct usb_hcd *hcd, struct usb_device *udev,
1413 		struct usb_host_endpoint *ep, int check_ep, bool check_virt_dev,
1414 		const char *func) {
1415 	struct xhci_hcd	*xhci;
1416 	struct xhci_virt_device	*virt_dev;
1417 
1418 	if (!hcd || (check_ep && !ep) || !udev) {
1419 		pr_debug("xHCI %s called with invalid args\n", func);
1420 		return -EINVAL;
1421 	}
1422 	if (!udev->parent) {
1423 		pr_debug("xHCI %s called for root hub\n", func);
1424 		return 0;
1425 	}
1426 
1427 	xhci = hcd_to_xhci(hcd);
1428 	if (check_virt_dev) {
1429 		if (!udev->slot_id || !xhci->devs[udev->slot_id]) {
1430 			xhci_dbg(xhci, "xHCI %s called with unaddressed device\n",
1431 					func);
1432 			return -EINVAL;
1433 		}
1434 
1435 		virt_dev = xhci->devs[udev->slot_id];
1436 		if (virt_dev->udev != udev) {
1437 			xhci_dbg(xhci, "xHCI %s called with udev and "
1438 					  "virt_dev does not match\n", func);
1439 			return -EINVAL;
1440 		}
1441 	}
1442 
1443 	if (xhci->xhc_state & XHCI_STATE_HALTED)
1444 		return -ENODEV;
1445 
1446 	return 1;
1447 }
1448 
1449 static int xhci_configure_endpoint(struct xhci_hcd *xhci,
1450 		struct usb_device *udev, struct xhci_command *command,
1451 		bool ctx_change, bool must_succeed);
1452 
1453 /*
1454  * Full speed devices may have a max packet size greater than 8 bytes, but the
1455  * USB core doesn't know that until it reads the first 8 bytes of the
1456  * descriptor.  If the usb_device's max packet size changes after that point,
1457  * we need to issue an evaluate context command and wait on it.
1458  */
1459 static int xhci_check_ep0_maxpacket(struct xhci_hcd *xhci, struct xhci_virt_device *vdev)
1460 {
1461 	struct xhci_input_control_ctx *ctrl_ctx;
1462 	struct xhci_ep_ctx *ep_ctx;
1463 	struct xhci_command *command;
1464 	int max_packet_size;
1465 	int hw_max_packet_size;
1466 	int ret = 0;
1467 
1468 	ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, 0);
1469 	hw_max_packet_size = MAX_PACKET_DECODED(le32_to_cpu(ep_ctx->ep_info2));
1470 	max_packet_size = usb_endpoint_maxp(&vdev->udev->ep0.desc);
1471 
1472 	if (hw_max_packet_size == max_packet_size)
1473 		return 0;
1474 
1475 	switch (max_packet_size) {
1476 	case 8: case 16: case 32: case 64: case 9:
1477 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
1478 				"Max Packet Size for ep 0 changed.");
1479 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
1480 				"Max packet size in usb_device = %d",
1481 				max_packet_size);
1482 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
1483 				"Max packet size in xHCI HW = %d",
1484 				hw_max_packet_size);
1485 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
1486 				"Issuing evaluate context command.");
1487 
1488 		command = xhci_alloc_command(xhci, true, GFP_KERNEL);
1489 		if (!command)
1490 			return -ENOMEM;
1491 
1492 		command->in_ctx = vdev->in_ctx;
1493 		ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
1494 		if (!ctrl_ctx) {
1495 			xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
1496 					__func__);
1497 			ret = -ENOMEM;
1498 			break;
1499 		}
1500 		/* Set up the modified control endpoint 0 */
1501 		xhci_endpoint_copy(xhci, vdev->in_ctx, vdev->out_ctx, 0);
1502 
1503 		ep_ctx = xhci_get_ep_ctx(xhci, command->in_ctx, 0);
1504 		ep_ctx->ep_info &= cpu_to_le32(~EP_STATE_MASK);/* must clear */
1505 		ep_ctx->ep_info2 &= cpu_to_le32(~MAX_PACKET_MASK);
1506 		ep_ctx->ep_info2 |= cpu_to_le32(MAX_PACKET(max_packet_size));
1507 
1508 		ctrl_ctx->add_flags = cpu_to_le32(EP0_FLAG);
1509 		ctrl_ctx->drop_flags = 0;
1510 
1511 		ret = xhci_configure_endpoint(xhci, vdev->udev, command,
1512 					      true, false);
1513 		/* Clean up the input context for later use by bandwidth functions */
1514 		ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG);
1515 		break;
1516 	default:
1517 		dev_dbg(&vdev->udev->dev, "incorrect max packet size %d for ep0\n",
1518 			max_packet_size);
1519 		return -EINVAL;
1520 	}
1521 
1522 	kfree(command->completion);
1523 	kfree(command);
1524 
1525 	return ret;
1526 }
1527 
1528 /*
1529  * non-error returns are a promise to giveback() the urb later
1530  * we drop ownership so next owner (or urb unlink) can get it
1531  */
1532 static int xhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flags)
1533 {
1534 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
1535 	unsigned long flags;
1536 	int ret = 0;
1537 	unsigned int slot_id, ep_index;
1538 	unsigned int *ep_state;
1539 	struct urb_priv	*urb_priv;
1540 	int num_tds;
1541 
1542 	ep_index = xhci_get_endpoint_index(&urb->ep->desc);
1543 
1544 	if (usb_endpoint_xfer_isoc(&urb->ep->desc))
1545 		num_tds = urb->number_of_packets;
1546 	else if (usb_endpoint_is_bulk_out(&urb->ep->desc) &&
1547 	    urb->transfer_buffer_length > 0 &&
1548 	    urb->transfer_flags & URB_ZERO_PACKET &&
1549 	    !(urb->transfer_buffer_length % usb_endpoint_maxp(&urb->ep->desc)))
1550 		num_tds = 2;
1551 	else
1552 		num_tds = 1;
1553 
1554 	urb_priv = kzalloc(struct_size(urb_priv, td, num_tds), mem_flags);
1555 	if (!urb_priv)
1556 		return -ENOMEM;
1557 
1558 	urb_priv->num_tds = num_tds;
1559 	urb_priv->num_tds_done = 0;
1560 	urb->hcpriv = urb_priv;
1561 
1562 	trace_xhci_urb_enqueue(urb);
1563 
1564 	spin_lock_irqsave(&xhci->lock, flags);
1565 
1566 	ret = xhci_check_args(hcd, urb->dev, urb->ep,
1567 			      true, true, __func__);
1568 	if (ret <= 0) {
1569 		ret = ret ? ret : -EINVAL;
1570 		goto free_priv;
1571 	}
1572 
1573 	slot_id = urb->dev->slot_id;
1574 
1575 	if (!HCD_HW_ACCESSIBLE(hcd)) {
1576 		ret = -ESHUTDOWN;
1577 		goto free_priv;
1578 	}
1579 
1580 	if (xhci->devs[slot_id]->flags & VDEV_PORT_ERROR) {
1581 		xhci_dbg(xhci, "Can't queue urb, port error, link inactive\n");
1582 		ret = -ENODEV;
1583 		goto free_priv;
1584 	}
1585 
1586 	if (xhci->xhc_state & XHCI_STATE_DYING) {
1587 		xhci_dbg(xhci, "Ep 0x%x: URB %p submitted for non-responsive xHCI host.\n",
1588 			 urb->ep->desc.bEndpointAddress, urb);
1589 		ret = -ESHUTDOWN;
1590 		goto free_priv;
1591 	}
1592 
1593 	ep_state = &xhci->devs[slot_id]->eps[ep_index].ep_state;
1594 
1595 	if (*ep_state & (EP_GETTING_STREAMS | EP_GETTING_NO_STREAMS)) {
1596 		xhci_warn(xhci, "WARN: Can't enqueue URB, ep in streams transition state %x\n",
1597 			  *ep_state);
1598 		ret = -EINVAL;
1599 		goto free_priv;
1600 	}
1601 	if (*ep_state & EP_SOFT_CLEAR_TOGGLE) {
1602 		xhci_warn(xhci, "Can't enqueue URB while manually clearing toggle\n");
1603 		ret = -EINVAL;
1604 		goto free_priv;
1605 	}
1606 
1607 	switch (usb_endpoint_type(&urb->ep->desc)) {
1608 
1609 	case USB_ENDPOINT_XFER_CONTROL:
1610 		ret = xhci_queue_ctrl_tx(xhci, GFP_ATOMIC, urb,
1611 					 slot_id, ep_index);
1612 		break;
1613 	case USB_ENDPOINT_XFER_BULK:
1614 		ret = xhci_queue_bulk_tx(xhci, GFP_ATOMIC, urb,
1615 					 slot_id, ep_index);
1616 		break;
1617 	case USB_ENDPOINT_XFER_INT:
1618 		ret = xhci_queue_intr_tx(xhci, GFP_ATOMIC, urb,
1619 				slot_id, ep_index);
1620 		break;
1621 	case USB_ENDPOINT_XFER_ISOC:
1622 		ret = xhci_queue_isoc_tx_prepare(xhci, GFP_ATOMIC, urb,
1623 				slot_id, ep_index);
1624 	}
1625 
1626 	if (ret) {
1627 free_priv:
1628 		xhci_urb_free_priv(urb_priv);
1629 		urb->hcpriv = NULL;
1630 	}
1631 	spin_unlock_irqrestore(&xhci->lock, flags);
1632 	return ret;
1633 }
1634 
1635 /*
1636  * Remove the URB's TD from the endpoint ring.  This may cause the HC to stop
1637  * USB transfers, potentially stopping in the middle of a TRB buffer.  The HC
1638  * should pick up where it left off in the TD, unless a Set Transfer Ring
1639  * Dequeue Pointer is issued.
1640  *
1641  * The TRBs that make up the buffers for the canceled URB will be "removed" from
1642  * the ring.  Since the ring is a contiguous structure, they can't be physically
1643  * removed.  Instead, there are two options:
1644  *
1645  *  1) If the HC is in the middle of processing the URB to be canceled, we
1646  *     simply move the ring's dequeue pointer past those TRBs using the Set
1647  *     Transfer Ring Dequeue Pointer command.  This will be the common case,
1648  *     when drivers timeout on the last submitted URB and attempt to cancel.
1649  *
1650  *  2) If the HC is in the middle of a different TD, we turn the TRBs into a
1651  *     series of 1-TRB transfer no-op TDs.  (No-ops shouldn't be chained.)  The
1652  *     HC will need to invalidate the any TRBs it has cached after the stop
1653  *     endpoint command, as noted in the xHCI 0.95 errata.
1654  *
1655  *  3) The TD may have completed by the time the Stop Endpoint Command
1656  *     completes, so software needs to handle that case too.
1657  *
1658  * This function should protect against the TD enqueueing code ringing the
1659  * doorbell while this code is waiting for a Stop Endpoint command to complete.
1660  * It also needs to account for multiple cancellations on happening at the same
1661  * time for the same endpoint.
1662  *
1663  * Note that this function can be called in any context, or so says
1664  * usb_hcd_unlink_urb()
1665  */
1666 static int xhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
1667 {
1668 	unsigned long flags;
1669 	int ret, i;
1670 	u32 temp;
1671 	struct xhci_hcd *xhci;
1672 	struct urb_priv	*urb_priv;
1673 	struct xhci_td *td;
1674 	unsigned int ep_index;
1675 	struct xhci_ring *ep_ring;
1676 	struct xhci_virt_ep *ep;
1677 	struct xhci_command *command;
1678 	struct xhci_virt_device *vdev;
1679 
1680 	xhci = hcd_to_xhci(hcd);
1681 	spin_lock_irqsave(&xhci->lock, flags);
1682 
1683 	trace_xhci_urb_dequeue(urb);
1684 
1685 	/* Make sure the URB hasn't completed or been unlinked already */
1686 	ret = usb_hcd_check_unlink_urb(hcd, urb, status);
1687 	if (ret)
1688 		goto done;
1689 
1690 	/* give back URB now if we can't queue it for cancel */
1691 	vdev = xhci->devs[urb->dev->slot_id];
1692 	urb_priv = urb->hcpriv;
1693 	if (!vdev || !urb_priv)
1694 		goto err_giveback;
1695 
1696 	ep_index = xhci_get_endpoint_index(&urb->ep->desc);
1697 	ep = &vdev->eps[ep_index];
1698 	ep_ring = xhci_urb_to_transfer_ring(xhci, urb);
1699 	if (!ep || !ep_ring)
1700 		goto err_giveback;
1701 
1702 	/* If xHC is dead take it down and return ALL URBs in xhci_hc_died() */
1703 	temp = readl(&xhci->op_regs->status);
1704 	if (temp == ~(u32)0 || xhci->xhc_state & XHCI_STATE_DYING) {
1705 		xhci_hc_died(xhci);
1706 		goto done;
1707 	}
1708 
1709 	/*
1710 	 * check ring is not re-allocated since URB was enqueued. If it is, then
1711 	 * make sure none of the ring related pointers in this URB private data
1712 	 * are touched, such as td_list, otherwise we overwrite freed data
1713 	 */
1714 	if (!td_on_ring(&urb_priv->td[0], ep_ring)) {
1715 		xhci_err(xhci, "Canceled URB td not found on endpoint ring");
1716 		for (i = urb_priv->num_tds_done; i < urb_priv->num_tds; i++) {
1717 			td = &urb_priv->td[i];
1718 			if (!list_empty(&td->cancelled_td_list))
1719 				list_del_init(&td->cancelled_td_list);
1720 		}
1721 		goto err_giveback;
1722 	}
1723 
1724 	if (xhci->xhc_state & XHCI_STATE_HALTED) {
1725 		xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1726 				"HC halted, freeing TD manually.");
1727 		for (i = urb_priv->num_tds_done;
1728 		     i < urb_priv->num_tds;
1729 		     i++) {
1730 			td = &urb_priv->td[i];
1731 			if (!list_empty(&td->td_list))
1732 				list_del_init(&td->td_list);
1733 			if (!list_empty(&td->cancelled_td_list))
1734 				list_del_init(&td->cancelled_td_list);
1735 		}
1736 		goto err_giveback;
1737 	}
1738 
1739 	i = urb_priv->num_tds_done;
1740 	if (i < urb_priv->num_tds)
1741 		xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1742 				"Cancel URB %p, dev %s, ep 0x%x, "
1743 				"starting at offset 0x%llx",
1744 				urb, urb->dev->devpath,
1745 				urb->ep->desc.bEndpointAddress,
1746 				(unsigned long long) xhci_trb_virt_to_dma(
1747 					urb_priv->td[i].start_seg,
1748 					urb_priv->td[i].start_trb));
1749 
1750 	for (; i < urb_priv->num_tds; i++) {
1751 		td = &urb_priv->td[i];
1752 		/* TD can already be on cancelled list if ep halted on it */
1753 		if (list_empty(&td->cancelled_td_list)) {
1754 			td->cancel_status = TD_DIRTY;
1755 			list_add_tail(&td->cancelled_td_list,
1756 				      &ep->cancelled_td_list);
1757 		}
1758 	}
1759 
1760 	/* These completion handlers will sort out cancelled TDs for us */
1761 	if (ep->ep_state & (EP_STOP_CMD_PENDING | EP_HALTED | SET_DEQ_PENDING)) {
1762 		xhci_dbg(xhci, "Not queuing Stop Endpoint on slot %d ep %d in state 0x%x\n",
1763 				urb->dev->slot_id, ep_index, ep->ep_state);
1764 		goto done;
1765 	}
1766 
1767 	/* In this case no commands are pending but the endpoint is stopped */
1768 	if (ep->ep_state & EP_CLEARING_TT) {
1769 		/* and cancelled TDs can be given back right away */
1770 		xhci_dbg(xhci, "Invalidating TDs instantly on slot %d ep %d in state 0x%x\n",
1771 				urb->dev->slot_id, ep_index, ep->ep_state);
1772 		xhci_process_cancelled_tds(ep);
1773 	} else {
1774 		/* Otherwise, queue a new Stop Endpoint command */
1775 		command = xhci_alloc_command(xhci, false, GFP_ATOMIC);
1776 		if (!command) {
1777 			ret = -ENOMEM;
1778 			goto done;
1779 		}
1780 		ep->stop_time = jiffies;
1781 		ep->ep_state |= EP_STOP_CMD_PENDING;
1782 		xhci_queue_stop_endpoint(xhci, command, urb->dev->slot_id,
1783 					 ep_index, 0);
1784 		xhci_ring_cmd_db(xhci);
1785 	}
1786 done:
1787 	spin_unlock_irqrestore(&xhci->lock, flags);
1788 	return ret;
1789 
1790 err_giveback:
1791 	if (urb_priv)
1792 		xhci_urb_free_priv(urb_priv);
1793 	usb_hcd_unlink_urb_from_ep(hcd, urb);
1794 	spin_unlock_irqrestore(&xhci->lock, flags);
1795 	usb_hcd_giveback_urb(hcd, urb, -ESHUTDOWN);
1796 	return ret;
1797 }
1798 
1799 /* Drop an endpoint from a new bandwidth configuration for this device.
1800  * Only one call to this function is allowed per endpoint before
1801  * check_bandwidth() or reset_bandwidth() must be called.
1802  * A call to xhci_drop_endpoint() followed by a call to xhci_add_endpoint() will
1803  * add the endpoint to the schedule with possibly new parameters denoted by a
1804  * different endpoint descriptor in usb_host_endpoint.
1805  * A call to xhci_add_endpoint() followed by a call to xhci_drop_endpoint() is
1806  * not allowed.
1807  *
1808  * The USB core will not allow URBs to be queued to an endpoint that is being
1809  * disabled, so there's no need for mutual exclusion to protect
1810  * the xhci->devs[slot_id] structure.
1811  */
1812 int xhci_drop_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
1813 		       struct usb_host_endpoint *ep)
1814 {
1815 	struct xhci_hcd *xhci;
1816 	struct xhci_container_ctx *in_ctx, *out_ctx;
1817 	struct xhci_input_control_ctx *ctrl_ctx;
1818 	unsigned int ep_index;
1819 	struct xhci_ep_ctx *ep_ctx;
1820 	u32 drop_flag;
1821 	u32 new_add_flags, new_drop_flags;
1822 	int ret;
1823 
1824 	ret = xhci_check_args(hcd, udev, ep, 1, true, __func__);
1825 	if (ret <= 0)
1826 		return ret;
1827 	xhci = hcd_to_xhci(hcd);
1828 	if (xhci->xhc_state & XHCI_STATE_DYING)
1829 		return -ENODEV;
1830 
1831 	xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
1832 	drop_flag = xhci_get_endpoint_flag(&ep->desc);
1833 	if (drop_flag == SLOT_FLAG || drop_flag == EP0_FLAG) {
1834 		xhci_dbg(xhci, "xHCI %s - can't drop slot or ep 0 %#x\n",
1835 				__func__, drop_flag);
1836 		return 0;
1837 	}
1838 
1839 	in_ctx = xhci->devs[udev->slot_id]->in_ctx;
1840 	out_ctx = xhci->devs[udev->slot_id]->out_ctx;
1841 	ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
1842 	if (!ctrl_ctx) {
1843 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
1844 				__func__);
1845 		return 0;
1846 	}
1847 
1848 	ep_index = xhci_get_endpoint_index(&ep->desc);
1849 	ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
1850 	/* If the HC already knows the endpoint is disabled,
1851 	 * or the HCD has noted it is disabled, ignore this request
1852 	 */
1853 	if ((GET_EP_CTX_STATE(ep_ctx) == EP_STATE_DISABLED) ||
1854 	    le32_to_cpu(ctrl_ctx->drop_flags) &
1855 	    xhci_get_endpoint_flag(&ep->desc)) {
1856 		/* Do not warn when called after a usb_device_reset */
1857 		if (xhci->devs[udev->slot_id]->eps[ep_index].ring != NULL)
1858 			xhci_warn(xhci, "xHCI %s called with disabled ep %p\n",
1859 				  __func__, ep);
1860 		return 0;
1861 	}
1862 
1863 	ctrl_ctx->drop_flags |= cpu_to_le32(drop_flag);
1864 	new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
1865 
1866 	ctrl_ctx->add_flags &= cpu_to_le32(~drop_flag);
1867 	new_add_flags = le32_to_cpu(ctrl_ctx->add_flags);
1868 
1869 	xhci_debugfs_remove_endpoint(xhci, xhci->devs[udev->slot_id], ep_index);
1870 
1871 	xhci_endpoint_zero(xhci, xhci->devs[udev->slot_id], ep);
1872 
1873 	xhci_dbg(xhci, "drop ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x\n",
1874 			(unsigned int) ep->desc.bEndpointAddress,
1875 			udev->slot_id,
1876 			(unsigned int) new_drop_flags,
1877 			(unsigned int) new_add_flags);
1878 	return 0;
1879 }
1880 EXPORT_SYMBOL_GPL(xhci_drop_endpoint);
1881 
1882 /* Add an endpoint to a new possible bandwidth configuration for this device.
1883  * Only one call to this function is allowed per endpoint before
1884  * check_bandwidth() or reset_bandwidth() must be called.
1885  * A call to xhci_drop_endpoint() followed by a call to xhci_add_endpoint() will
1886  * add the endpoint to the schedule with possibly new parameters denoted by a
1887  * different endpoint descriptor in usb_host_endpoint.
1888  * A call to xhci_add_endpoint() followed by a call to xhci_drop_endpoint() is
1889  * not allowed.
1890  *
1891  * The USB core will not allow URBs to be queued to an endpoint until the
1892  * configuration or alt setting is installed in the device, so there's no need
1893  * for mutual exclusion to protect the xhci->devs[slot_id] structure.
1894  */
1895 int xhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
1896 		      struct usb_host_endpoint *ep)
1897 {
1898 	struct xhci_hcd *xhci;
1899 	struct xhci_container_ctx *in_ctx;
1900 	unsigned int ep_index;
1901 	struct xhci_input_control_ctx *ctrl_ctx;
1902 	struct xhci_ep_ctx *ep_ctx;
1903 	u32 added_ctxs;
1904 	u32 new_add_flags, new_drop_flags;
1905 	struct xhci_virt_device *virt_dev;
1906 	int ret = 0;
1907 
1908 	ret = xhci_check_args(hcd, udev, ep, 1, true, __func__);
1909 	if (ret <= 0) {
1910 		/* So we won't queue a reset ep command for a root hub */
1911 		ep->hcpriv = NULL;
1912 		return ret;
1913 	}
1914 	xhci = hcd_to_xhci(hcd);
1915 	if (xhci->xhc_state & XHCI_STATE_DYING)
1916 		return -ENODEV;
1917 
1918 	added_ctxs = xhci_get_endpoint_flag(&ep->desc);
1919 	if (added_ctxs == SLOT_FLAG || added_ctxs == EP0_FLAG) {
1920 		/* FIXME when we have to issue an evaluate endpoint command to
1921 		 * deal with ep0 max packet size changing once we get the
1922 		 * descriptors
1923 		 */
1924 		xhci_dbg(xhci, "xHCI %s - can't add slot or ep 0 %#x\n",
1925 				__func__, added_ctxs);
1926 		return 0;
1927 	}
1928 
1929 	virt_dev = xhci->devs[udev->slot_id];
1930 	in_ctx = virt_dev->in_ctx;
1931 	ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
1932 	if (!ctrl_ctx) {
1933 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
1934 				__func__);
1935 		return 0;
1936 	}
1937 
1938 	ep_index = xhci_get_endpoint_index(&ep->desc);
1939 	/* If this endpoint is already in use, and the upper layers are trying
1940 	 * to add it again without dropping it, reject the addition.
1941 	 */
1942 	if (virt_dev->eps[ep_index].ring &&
1943 			!(le32_to_cpu(ctrl_ctx->drop_flags) & added_ctxs)) {
1944 		xhci_warn(xhci, "Trying to add endpoint 0x%x "
1945 				"without dropping it.\n",
1946 				(unsigned int) ep->desc.bEndpointAddress);
1947 		return -EINVAL;
1948 	}
1949 
1950 	/* If the HCD has already noted the endpoint is enabled,
1951 	 * ignore this request.
1952 	 */
1953 	if (le32_to_cpu(ctrl_ctx->add_flags) & added_ctxs) {
1954 		xhci_warn(xhci, "xHCI %s called with enabled ep %p\n",
1955 				__func__, ep);
1956 		return 0;
1957 	}
1958 
1959 	/*
1960 	 * Configuration and alternate setting changes must be done in
1961 	 * process context, not interrupt context (or so documenation
1962 	 * for usb_set_interface() and usb_set_configuration() claim).
1963 	 */
1964 	if (xhci_endpoint_init(xhci, virt_dev, udev, ep, GFP_NOIO) < 0) {
1965 		dev_dbg(&udev->dev, "%s - could not initialize ep %#x\n",
1966 				__func__, ep->desc.bEndpointAddress);
1967 		return -ENOMEM;
1968 	}
1969 
1970 	ctrl_ctx->add_flags |= cpu_to_le32(added_ctxs);
1971 	new_add_flags = le32_to_cpu(ctrl_ctx->add_flags);
1972 
1973 	/* If xhci_endpoint_disable() was called for this endpoint, but the
1974 	 * xHC hasn't been notified yet through the check_bandwidth() call,
1975 	 * this re-adds a new state for the endpoint from the new endpoint
1976 	 * descriptors.  We must drop and re-add this endpoint, so we leave the
1977 	 * drop flags alone.
1978 	 */
1979 	new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
1980 
1981 	/* Store the usb_device pointer for later use */
1982 	ep->hcpriv = udev;
1983 
1984 	ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index);
1985 	trace_xhci_add_endpoint(ep_ctx);
1986 
1987 	xhci_dbg(xhci, "add ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x\n",
1988 			(unsigned int) ep->desc.bEndpointAddress,
1989 			udev->slot_id,
1990 			(unsigned int) new_drop_flags,
1991 			(unsigned int) new_add_flags);
1992 	return 0;
1993 }
1994 EXPORT_SYMBOL_GPL(xhci_add_endpoint);
1995 
1996 static void xhci_zero_in_ctx(struct xhci_hcd *xhci, struct xhci_virt_device *virt_dev)
1997 {
1998 	struct xhci_input_control_ctx *ctrl_ctx;
1999 	struct xhci_ep_ctx *ep_ctx;
2000 	struct xhci_slot_ctx *slot_ctx;
2001 	int i;
2002 
2003 	ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
2004 	if (!ctrl_ctx) {
2005 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
2006 				__func__);
2007 		return;
2008 	}
2009 
2010 	/* When a device's add flag and drop flag are zero, any subsequent
2011 	 * configure endpoint command will leave that endpoint's state
2012 	 * untouched.  Make sure we don't leave any old state in the input
2013 	 * endpoint contexts.
2014 	 */
2015 	ctrl_ctx->drop_flags = 0;
2016 	ctrl_ctx->add_flags = 0;
2017 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
2018 	slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
2019 	/* Endpoint 0 is always valid */
2020 	slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1));
2021 	for (i = 1; i < 31; i++) {
2022 		ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, i);
2023 		ep_ctx->ep_info = 0;
2024 		ep_ctx->ep_info2 = 0;
2025 		ep_ctx->deq = 0;
2026 		ep_ctx->tx_info = 0;
2027 	}
2028 }
2029 
2030 static int xhci_configure_endpoint_result(struct xhci_hcd *xhci,
2031 		struct usb_device *udev, u32 *cmd_status)
2032 {
2033 	int ret;
2034 
2035 	switch (*cmd_status) {
2036 	case COMP_COMMAND_ABORTED:
2037 	case COMP_COMMAND_RING_STOPPED:
2038 		xhci_warn(xhci, "Timeout while waiting for configure endpoint command\n");
2039 		ret = -ETIME;
2040 		break;
2041 	case COMP_RESOURCE_ERROR:
2042 		dev_warn(&udev->dev,
2043 			 "Not enough host controller resources for new device state.\n");
2044 		ret = -ENOMEM;
2045 		/* FIXME: can we allocate more resources for the HC? */
2046 		break;
2047 	case COMP_BANDWIDTH_ERROR:
2048 	case COMP_SECONDARY_BANDWIDTH_ERROR:
2049 		dev_warn(&udev->dev,
2050 			 "Not enough bandwidth for new device state.\n");
2051 		ret = -ENOSPC;
2052 		/* FIXME: can we go back to the old state? */
2053 		break;
2054 	case COMP_TRB_ERROR:
2055 		/* the HCD set up something wrong */
2056 		dev_warn(&udev->dev, "ERROR: Endpoint drop flag = 0, "
2057 				"add flag = 1, "
2058 				"and endpoint is not disabled.\n");
2059 		ret = -EINVAL;
2060 		break;
2061 	case COMP_INCOMPATIBLE_DEVICE_ERROR:
2062 		dev_warn(&udev->dev,
2063 			 "ERROR: Incompatible device for endpoint configure command.\n");
2064 		ret = -ENODEV;
2065 		break;
2066 	case COMP_SUCCESS:
2067 		xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
2068 				"Successful Endpoint Configure command");
2069 		ret = 0;
2070 		break;
2071 	default:
2072 		xhci_err(xhci, "ERROR: unexpected command completion code 0x%x.\n",
2073 				*cmd_status);
2074 		ret = -EINVAL;
2075 		break;
2076 	}
2077 	return ret;
2078 }
2079 
2080 static int xhci_evaluate_context_result(struct xhci_hcd *xhci,
2081 		struct usb_device *udev, u32 *cmd_status)
2082 {
2083 	int ret;
2084 
2085 	switch (*cmd_status) {
2086 	case COMP_COMMAND_ABORTED:
2087 	case COMP_COMMAND_RING_STOPPED:
2088 		xhci_warn(xhci, "Timeout while waiting for evaluate context command\n");
2089 		ret = -ETIME;
2090 		break;
2091 	case COMP_PARAMETER_ERROR:
2092 		dev_warn(&udev->dev,
2093 			 "WARN: xHCI driver setup invalid evaluate context command.\n");
2094 		ret = -EINVAL;
2095 		break;
2096 	case COMP_SLOT_NOT_ENABLED_ERROR:
2097 		dev_warn(&udev->dev,
2098 			"WARN: slot not enabled for evaluate context command.\n");
2099 		ret = -EINVAL;
2100 		break;
2101 	case COMP_CONTEXT_STATE_ERROR:
2102 		dev_warn(&udev->dev,
2103 			"WARN: invalid context state for evaluate context command.\n");
2104 		ret = -EINVAL;
2105 		break;
2106 	case COMP_INCOMPATIBLE_DEVICE_ERROR:
2107 		dev_warn(&udev->dev,
2108 			"ERROR: Incompatible device for evaluate context command.\n");
2109 		ret = -ENODEV;
2110 		break;
2111 	case COMP_MAX_EXIT_LATENCY_TOO_LARGE_ERROR:
2112 		/* Max Exit Latency too large error */
2113 		dev_warn(&udev->dev, "WARN: Max Exit Latency too large\n");
2114 		ret = -EINVAL;
2115 		break;
2116 	case COMP_SUCCESS:
2117 		xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
2118 				"Successful evaluate context command");
2119 		ret = 0;
2120 		break;
2121 	default:
2122 		xhci_err(xhci, "ERROR: unexpected command completion code 0x%x.\n",
2123 			*cmd_status);
2124 		ret = -EINVAL;
2125 		break;
2126 	}
2127 	return ret;
2128 }
2129 
2130 static u32 xhci_count_num_new_endpoints(struct xhci_hcd *xhci,
2131 		struct xhci_input_control_ctx *ctrl_ctx)
2132 {
2133 	u32 valid_add_flags;
2134 	u32 valid_drop_flags;
2135 
2136 	/* Ignore the slot flag (bit 0), and the default control endpoint flag
2137 	 * (bit 1).  The default control endpoint is added during the Address
2138 	 * Device command and is never removed until the slot is disabled.
2139 	 */
2140 	valid_add_flags = le32_to_cpu(ctrl_ctx->add_flags) >> 2;
2141 	valid_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags) >> 2;
2142 
2143 	/* Use hweight32 to count the number of ones in the add flags, or
2144 	 * number of endpoints added.  Don't count endpoints that are changed
2145 	 * (both added and dropped).
2146 	 */
2147 	return hweight32(valid_add_flags) -
2148 		hweight32(valid_add_flags & valid_drop_flags);
2149 }
2150 
2151 static unsigned int xhci_count_num_dropped_endpoints(struct xhci_hcd *xhci,
2152 		struct xhci_input_control_ctx *ctrl_ctx)
2153 {
2154 	u32 valid_add_flags;
2155 	u32 valid_drop_flags;
2156 
2157 	valid_add_flags = le32_to_cpu(ctrl_ctx->add_flags) >> 2;
2158 	valid_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags) >> 2;
2159 
2160 	return hweight32(valid_drop_flags) -
2161 		hweight32(valid_add_flags & valid_drop_flags);
2162 }
2163 
2164 /*
2165  * We need to reserve the new number of endpoints before the configure endpoint
2166  * command completes.  We can't subtract the dropped endpoints from the number
2167  * of active endpoints until the command completes because we can oversubscribe
2168  * the host in this case:
2169  *
2170  *  - the first configure endpoint command drops more endpoints than it adds
2171  *  - a second configure endpoint command that adds more endpoints is queued
2172  *  - the first configure endpoint command fails, so the config is unchanged
2173  *  - the second command may succeed, even though there isn't enough resources
2174  *
2175  * Must be called with xhci->lock held.
2176  */
2177 static int xhci_reserve_host_resources(struct xhci_hcd *xhci,
2178 		struct xhci_input_control_ctx *ctrl_ctx)
2179 {
2180 	u32 added_eps;
2181 
2182 	added_eps = xhci_count_num_new_endpoints(xhci, ctrl_ctx);
2183 	if (xhci->num_active_eps + added_eps > xhci->limit_active_eps) {
2184 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2185 				"Not enough ep ctxs: "
2186 				"%u active, need to add %u, limit is %u.",
2187 				xhci->num_active_eps, added_eps,
2188 				xhci->limit_active_eps);
2189 		return -ENOMEM;
2190 	}
2191 	xhci->num_active_eps += added_eps;
2192 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2193 			"Adding %u ep ctxs, %u now active.", added_eps,
2194 			xhci->num_active_eps);
2195 	return 0;
2196 }
2197 
2198 /*
2199  * The configure endpoint was failed by the xHC for some other reason, so we
2200  * need to revert the resources that failed configuration would have used.
2201  *
2202  * Must be called with xhci->lock held.
2203  */
2204 static void xhci_free_host_resources(struct xhci_hcd *xhci,
2205 		struct xhci_input_control_ctx *ctrl_ctx)
2206 {
2207 	u32 num_failed_eps;
2208 
2209 	num_failed_eps = xhci_count_num_new_endpoints(xhci, ctrl_ctx);
2210 	xhci->num_active_eps -= num_failed_eps;
2211 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2212 			"Removing %u failed ep ctxs, %u now active.",
2213 			num_failed_eps,
2214 			xhci->num_active_eps);
2215 }
2216 
2217 /*
2218  * Now that the command has completed, clean up the active endpoint count by
2219  * subtracting out the endpoints that were dropped (but not changed).
2220  *
2221  * Must be called with xhci->lock held.
2222  */
2223 static void xhci_finish_resource_reservation(struct xhci_hcd *xhci,
2224 		struct xhci_input_control_ctx *ctrl_ctx)
2225 {
2226 	u32 num_dropped_eps;
2227 
2228 	num_dropped_eps = xhci_count_num_dropped_endpoints(xhci, ctrl_ctx);
2229 	xhci->num_active_eps -= num_dropped_eps;
2230 	if (num_dropped_eps)
2231 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2232 				"Removing %u dropped ep ctxs, %u now active.",
2233 				num_dropped_eps,
2234 				xhci->num_active_eps);
2235 }
2236 
2237 static unsigned int xhci_get_block_size(struct usb_device *udev)
2238 {
2239 	switch (udev->speed) {
2240 	case USB_SPEED_LOW:
2241 	case USB_SPEED_FULL:
2242 		return FS_BLOCK;
2243 	case USB_SPEED_HIGH:
2244 		return HS_BLOCK;
2245 	case USB_SPEED_SUPER:
2246 	case USB_SPEED_SUPER_PLUS:
2247 		return SS_BLOCK;
2248 	case USB_SPEED_UNKNOWN:
2249 	default:
2250 		/* Should never happen */
2251 		return 1;
2252 	}
2253 }
2254 
2255 static unsigned int
2256 xhci_get_largest_overhead(struct xhci_interval_bw *interval_bw)
2257 {
2258 	if (interval_bw->overhead[LS_OVERHEAD_TYPE])
2259 		return LS_OVERHEAD;
2260 	if (interval_bw->overhead[FS_OVERHEAD_TYPE])
2261 		return FS_OVERHEAD;
2262 	return HS_OVERHEAD;
2263 }
2264 
2265 /* If we are changing a LS/FS device under a HS hub,
2266  * make sure (if we are activating a new TT) that the HS bus has enough
2267  * bandwidth for this new TT.
2268  */
2269 static int xhci_check_tt_bw_table(struct xhci_hcd *xhci,
2270 		struct xhci_virt_device *virt_dev,
2271 		int old_active_eps)
2272 {
2273 	struct xhci_interval_bw_table *bw_table;
2274 	struct xhci_tt_bw_info *tt_info;
2275 
2276 	/* Find the bandwidth table for the root port this TT is attached to. */
2277 	bw_table = &xhci->rh_bw[virt_dev->rhub_port->hw_portnum].bw_table;
2278 	tt_info = virt_dev->tt_info;
2279 	/* If this TT already had active endpoints, the bandwidth for this TT
2280 	 * has already been added.  Removing all periodic endpoints (and thus
2281 	 * making the TT enactive) will only decrease the bandwidth used.
2282 	 */
2283 	if (old_active_eps)
2284 		return 0;
2285 	if (old_active_eps == 0 && tt_info->active_eps != 0) {
2286 		if (bw_table->bw_used + TT_HS_OVERHEAD > HS_BW_LIMIT)
2287 			return -ENOMEM;
2288 		return 0;
2289 	}
2290 	/* Not sure why we would have no new active endpoints...
2291 	 *
2292 	 * Maybe because of an Evaluate Context change for a hub update or a
2293 	 * control endpoint 0 max packet size change?
2294 	 * FIXME: skip the bandwidth calculation in that case.
2295 	 */
2296 	return 0;
2297 }
2298 
2299 static int xhci_check_ss_bw(struct xhci_hcd *xhci,
2300 		struct xhci_virt_device *virt_dev)
2301 {
2302 	unsigned int bw_reserved;
2303 
2304 	bw_reserved = DIV_ROUND_UP(SS_BW_RESERVED*SS_BW_LIMIT_IN, 100);
2305 	if (virt_dev->bw_table->ss_bw_in > (SS_BW_LIMIT_IN - bw_reserved))
2306 		return -ENOMEM;
2307 
2308 	bw_reserved = DIV_ROUND_UP(SS_BW_RESERVED*SS_BW_LIMIT_OUT, 100);
2309 	if (virt_dev->bw_table->ss_bw_out > (SS_BW_LIMIT_OUT - bw_reserved))
2310 		return -ENOMEM;
2311 
2312 	return 0;
2313 }
2314 
2315 /*
2316  * This algorithm is a very conservative estimate of the worst-case scheduling
2317  * scenario for any one interval.  The hardware dynamically schedules the
2318  * packets, so we can't tell which microframe could be the limiting factor in
2319  * the bandwidth scheduling.  This only takes into account periodic endpoints.
2320  *
2321  * Obviously, we can't solve an NP complete problem to find the minimum worst
2322  * case scenario.  Instead, we come up with an estimate that is no less than
2323  * the worst case bandwidth used for any one microframe, but may be an
2324  * over-estimate.
2325  *
2326  * We walk the requirements for each endpoint by interval, starting with the
2327  * smallest interval, and place packets in the schedule where there is only one
2328  * possible way to schedule packets for that interval.  In order to simplify
2329  * this algorithm, we record the largest max packet size for each interval, and
2330  * assume all packets will be that size.
2331  *
2332  * For interval 0, we obviously must schedule all packets for each interval.
2333  * The bandwidth for interval 0 is just the amount of data to be transmitted
2334  * (the sum of all max ESIT payload sizes, plus any overhead per packet times
2335  * the number of packets).
2336  *
2337  * For interval 1, we have two possible microframes to schedule those packets
2338  * in.  For this algorithm, if we can schedule the same number of packets for
2339  * each possible scheduling opportunity (each microframe), we will do so.  The
2340  * remaining number of packets will be saved to be transmitted in the gaps in
2341  * the next interval's scheduling sequence.
2342  *
2343  * As we move those remaining packets to be scheduled with interval 2 packets,
2344  * we have to double the number of remaining packets to transmit.  This is
2345  * because the intervals are actually powers of 2, and we would be transmitting
2346  * the previous interval's packets twice in this interval.  We also have to be
2347  * sure that when we look at the largest max packet size for this interval, we
2348  * also look at the largest max packet size for the remaining packets and take
2349  * the greater of the two.
2350  *
2351  * The algorithm continues to evenly distribute packets in each scheduling
2352  * opportunity, and push the remaining packets out, until we get to the last
2353  * interval.  Then those packets and their associated overhead are just added
2354  * to the bandwidth used.
2355  */
2356 static int xhci_check_bw_table(struct xhci_hcd *xhci,
2357 		struct xhci_virt_device *virt_dev,
2358 		int old_active_eps)
2359 {
2360 	unsigned int bw_reserved;
2361 	unsigned int max_bandwidth;
2362 	unsigned int bw_used;
2363 	unsigned int block_size;
2364 	struct xhci_interval_bw_table *bw_table;
2365 	unsigned int packet_size = 0;
2366 	unsigned int overhead = 0;
2367 	unsigned int packets_transmitted = 0;
2368 	unsigned int packets_remaining = 0;
2369 	unsigned int i;
2370 
2371 	if (virt_dev->udev->speed >= USB_SPEED_SUPER)
2372 		return xhci_check_ss_bw(xhci, virt_dev);
2373 
2374 	if (virt_dev->udev->speed == USB_SPEED_HIGH) {
2375 		max_bandwidth = HS_BW_LIMIT;
2376 		/* Convert percent of bus BW reserved to blocks reserved */
2377 		bw_reserved = DIV_ROUND_UP(HS_BW_RESERVED * max_bandwidth, 100);
2378 	} else {
2379 		max_bandwidth = FS_BW_LIMIT;
2380 		bw_reserved = DIV_ROUND_UP(FS_BW_RESERVED * max_bandwidth, 100);
2381 	}
2382 
2383 	bw_table = virt_dev->bw_table;
2384 	/* We need to translate the max packet size and max ESIT payloads into
2385 	 * the units the hardware uses.
2386 	 */
2387 	block_size = xhci_get_block_size(virt_dev->udev);
2388 
2389 	/* If we are manipulating a LS/FS device under a HS hub, double check
2390 	 * that the HS bus has enough bandwidth if we are activing a new TT.
2391 	 */
2392 	if (virt_dev->tt_info) {
2393 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2394 				"Recalculating BW for rootport %u",
2395 				virt_dev->rhub_port->hw_portnum + 1);
2396 		if (xhci_check_tt_bw_table(xhci, virt_dev, old_active_eps)) {
2397 			xhci_warn(xhci, "Not enough bandwidth on HS bus for "
2398 					"newly activated TT.\n");
2399 			return -ENOMEM;
2400 		}
2401 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2402 				"Recalculating BW for TT slot %u port %u",
2403 				virt_dev->tt_info->slot_id,
2404 				virt_dev->tt_info->ttport);
2405 	} else {
2406 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2407 				"Recalculating BW for rootport %u",
2408 				virt_dev->rhub_port->hw_portnum + 1);
2409 	}
2410 
2411 	/* Add in how much bandwidth will be used for interval zero, or the
2412 	 * rounded max ESIT payload + number of packets * largest overhead.
2413 	 */
2414 	bw_used = DIV_ROUND_UP(bw_table->interval0_esit_payload, block_size) +
2415 		bw_table->interval_bw[0].num_packets *
2416 		xhci_get_largest_overhead(&bw_table->interval_bw[0]);
2417 
2418 	for (i = 1; i < XHCI_MAX_INTERVAL; i++) {
2419 		unsigned int bw_added;
2420 		unsigned int largest_mps;
2421 		unsigned int interval_overhead;
2422 
2423 		/*
2424 		 * How many packets could we transmit in this interval?
2425 		 * If packets didn't fit in the previous interval, we will need
2426 		 * to transmit that many packets twice within this interval.
2427 		 */
2428 		packets_remaining = 2 * packets_remaining +
2429 			bw_table->interval_bw[i].num_packets;
2430 
2431 		/* Find the largest max packet size of this or the previous
2432 		 * interval.
2433 		 */
2434 		if (list_empty(&bw_table->interval_bw[i].endpoints))
2435 			largest_mps = 0;
2436 		else {
2437 			struct xhci_virt_ep *virt_ep;
2438 			struct list_head *ep_entry;
2439 
2440 			ep_entry = bw_table->interval_bw[i].endpoints.next;
2441 			virt_ep = list_entry(ep_entry,
2442 					struct xhci_virt_ep, bw_endpoint_list);
2443 			/* Convert to blocks, rounding up */
2444 			largest_mps = DIV_ROUND_UP(
2445 					virt_ep->bw_info.max_packet_size,
2446 					block_size);
2447 		}
2448 		if (largest_mps > packet_size)
2449 			packet_size = largest_mps;
2450 
2451 		/* Use the larger overhead of this or the previous interval. */
2452 		interval_overhead = xhci_get_largest_overhead(
2453 				&bw_table->interval_bw[i]);
2454 		if (interval_overhead > overhead)
2455 			overhead = interval_overhead;
2456 
2457 		/* How many packets can we evenly distribute across
2458 		 * (1 << (i + 1)) possible scheduling opportunities?
2459 		 */
2460 		packets_transmitted = packets_remaining >> (i + 1);
2461 
2462 		/* Add in the bandwidth used for those scheduled packets */
2463 		bw_added = packets_transmitted * (overhead + packet_size);
2464 
2465 		/* How many packets do we have remaining to transmit? */
2466 		packets_remaining = packets_remaining % (1 << (i + 1));
2467 
2468 		/* What largest max packet size should those packets have? */
2469 		/* If we've transmitted all packets, don't carry over the
2470 		 * largest packet size.
2471 		 */
2472 		if (packets_remaining == 0) {
2473 			packet_size = 0;
2474 			overhead = 0;
2475 		} else if (packets_transmitted > 0) {
2476 			/* Otherwise if we do have remaining packets, and we've
2477 			 * scheduled some packets in this interval, take the
2478 			 * largest max packet size from endpoints with this
2479 			 * interval.
2480 			 */
2481 			packet_size = largest_mps;
2482 			overhead = interval_overhead;
2483 		}
2484 		/* Otherwise carry over packet_size and overhead from the last
2485 		 * time we had a remainder.
2486 		 */
2487 		bw_used += bw_added;
2488 		if (bw_used > max_bandwidth) {
2489 			xhci_warn(xhci, "Not enough bandwidth. "
2490 					"Proposed: %u, Max: %u\n",
2491 				bw_used, max_bandwidth);
2492 			return -ENOMEM;
2493 		}
2494 	}
2495 	/*
2496 	 * Ok, we know we have some packets left over after even-handedly
2497 	 * scheduling interval 15.  We don't know which microframes they will
2498 	 * fit into, so we over-schedule and say they will be scheduled every
2499 	 * microframe.
2500 	 */
2501 	if (packets_remaining > 0)
2502 		bw_used += overhead + packet_size;
2503 
2504 	if (!virt_dev->tt_info && virt_dev->udev->speed == USB_SPEED_HIGH) {
2505 		/* OK, we're manipulating a HS device attached to a
2506 		 * root port bandwidth domain.  Include the number of active TTs
2507 		 * in the bandwidth used.
2508 		 */
2509 		bw_used += TT_HS_OVERHEAD *
2510 			xhci->rh_bw[virt_dev->rhub_port->hw_portnum].num_active_tts;
2511 	}
2512 
2513 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2514 		"Final bandwidth: %u, Limit: %u, Reserved: %u, "
2515 		"Available: %u " "percent",
2516 		bw_used, max_bandwidth, bw_reserved,
2517 		(max_bandwidth - bw_used - bw_reserved) * 100 /
2518 		max_bandwidth);
2519 
2520 	bw_used += bw_reserved;
2521 	if (bw_used > max_bandwidth) {
2522 		xhci_warn(xhci, "Not enough bandwidth. Proposed: %u, Max: %u\n",
2523 				bw_used, max_bandwidth);
2524 		return -ENOMEM;
2525 	}
2526 
2527 	bw_table->bw_used = bw_used;
2528 	return 0;
2529 }
2530 
2531 static bool xhci_is_async_ep(unsigned int ep_type)
2532 {
2533 	return (ep_type != ISOC_OUT_EP && ep_type != INT_OUT_EP &&
2534 					ep_type != ISOC_IN_EP &&
2535 					ep_type != INT_IN_EP);
2536 }
2537 
2538 static bool xhci_is_sync_in_ep(unsigned int ep_type)
2539 {
2540 	return (ep_type == ISOC_IN_EP || ep_type == INT_IN_EP);
2541 }
2542 
2543 static unsigned int xhci_get_ss_bw_consumed(struct xhci_bw_info *ep_bw)
2544 {
2545 	unsigned int mps = DIV_ROUND_UP(ep_bw->max_packet_size, SS_BLOCK);
2546 
2547 	if (ep_bw->ep_interval == 0)
2548 		return SS_OVERHEAD_BURST +
2549 			(ep_bw->mult * ep_bw->num_packets *
2550 					(SS_OVERHEAD + mps));
2551 	return DIV_ROUND_UP(ep_bw->mult * ep_bw->num_packets *
2552 				(SS_OVERHEAD + mps + SS_OVERHEAD_BURST),
2553 				1 << ep_bw->ep_interval);
2554 
2555 }
2556 
2557 static void xhci_drop_ep_from_interval_table(struct xhci_hcd *xhci,
2558 		struct xhci_bw_info *ep_bw,
2559 		struct xhci_interval_bw_table *bw_table,
2560 		struct usb_device *udev,
2561 		struct xhci_virt_ep *virt_ep,
2562 		struct xhci_tt_bw_info *tt_info)
2563 {
2564 	struct xhci_interval_bw	*interval_bw;
2565 	int normalized_interval;
2566 
2567 	if (xhci_is_async_ep(ep_bw->type))
2568 		return;
2569 
2570 	if (udev->speed >= USB_SPEED_SUPER) {
2571 		if (xhci_is_sync_in_ep(ep_bw->type))
2572 			xhci->devs[udev->slot_id]->bw_table->ss_bw_in -=
2573 				xhci_get_ss_bw_consumed(ep_bw);
2574 		else
2575 			xhci->devs[udev->slot_id]->bw_table->ss_bw_out -=
2576 				xhci_get_ss_bw_consumed(ep_bw);
2577 		return;
2578 	}
2579 
2580 	/* SuperSpeed endpoints never get added to intervals in the table, so
2581 	 * this check is only valid for HS/FS/LS devices.
2582 	 */
2583 	if (list_empty(&virt_ep->bw_endpoint_list))
2584 		return;
2585 	/* For LS/FS devices, we need to translate the interval expressed in
2586 	 * microframes to frames.
2587 	 */
2588 	if (udev->speed == USB_SPEED_HIGH)
2589 		normalized_interval = ep_bw->ep_interval;
2590 	else
2591 		normalized_interval = ep_bw->ep_interval - 3;
2592 
2593 	if (normalized_interval == 0)
2594 		bw_table->interval0_esit_payload -= ep_bw->max_esit_payload;
2595 	interval_bw = &bw_table->interval_bw[normalized_interval];
2596 	interval_bw->num_packets -= ep_bw->num_packets;
2597 	switch (udev->speed) {
2598 	case USB_SPEED_LOW:
2599 		interval_bw->overhead[LS_OVERHEAD_TYPE] -= 1;
2600 		break;
2601 	case USB_SPEED_FULL:
2602 		interval_bw->overhead[FS_OVERHEAD_TYPE] -= 1;
2603 		break;
2604 	case USB_SPEED_HIGH:
2605 		interval_bw->overhead[HS_OVERHEAD_TYPE] -= 1;
2606 		break;
2607 	default:
2608 		/* Should never happen because only LS/FS/HS endpoints will get
2609 		 * added to the endpoint list.
2610 		 */
2611 		return;
2612 	}
2613 	if (tt_info)
2614 		tt_info->active_eps -= 1;
2615 	list_del_init(&virt_ep->bw_endpoint_list);
2616 }
2617 
2618 static void xhci_add_ep_to_interval_table(struct xhci_hcd *xhci,
2619 		struct xhci_bw_info *ep_bw,
2620 		struct xhci_interval_bw_table *bw_table,
2621 		struct usb_device *udev,
2622 		struct xhci_virt_ep *virt_ep,
2623 		struct xhci_tt_bw_info *tt_info)
2624 {
2625 	struct xhci_interval_bw	*interval_bw;
2626 	struct xhci_virt_ep *smaller_ep;
2627 	int normalized_interval;
2628 
2629 	if (xhci_is_async_ep(ep_bw->type))
2630 		return;
2631 
2632 	if (udev->speed == USB_SPEED_SUPER) {
2633 		if (xhci_is_sync_in_ep(ep_bw->type))
2634 			xhci->devs[udev->slot_id]->bw_table->ss_bw_in +=
2635 				xhci_get_ss_bw_consumed(ep_bw);
2636 		else
2637 			xhci->devs[udev->slot_id]->bw_table->ss_bw_out +=
2638 				xhci_get_ss_bw_consumed(ep_bw);
2639 		return;
2640 	}
2641 
2642 	/* For LS/FS devices, we need to translate the interval expressed in
2643 	 * microframes to frames.
2644 	 */
2645 	if (udev->speed == USB_SPEED_HIGH)
2646 		normalized_interval = ep_bw->ep_interval;
2647 	else
2648 		normalized_interval = ep_bw->ep_interval - 3;
2649 
2650 	if (normalized_interval == 0)
2651 		bw_table->interval0_esit_payload += ep_bw->max_esit_payload;
2652 	interval_bw = &bw_table->interval_bw[normalized_interval];
2653 	interval_bw->num_packets += ep_bw->num_packets;
2654 	switch (udev->speed) {
2655 	case USB_SPEED_LOW:
2656 		interval_bw->overhead[LS_OVERHEAD_TYPE] += 1;
2657 		break;
2658 	case USB_SPEED_FULL:
2659 		interval_bw->overhead[FS_OVERHEAD_TYPE] += 1;
2660 		break;
2661 	case USB_SPEED_HIGH:
2662 		interval_bw->overhead[HS_OVERHEAD_TYPE] += 1;
2663 		break;
2664 	default:
2665 		/* Should never happen because only LS/FS/HS endpoints will get
2666 		 * added to the endpoint list.
2667 		 */
2668 		return;
2669 	}
2670 
2671 	if (tt_info)
2672 		tt_info->active_eps += 1;
2673 	/* Insert the endpoint into the list, largest max packet size first. */
2674 	list_for_each_entry(smaller_ep, &interval_bw->endpoints,
2675 			bw_endpoint_list) {
2676 		if (ep_bw->max_packet_size >=
2677 				smaller_ep->bw_info.max_packet_size) {
2678 			/* Add the new ep before the smaller endpoint */
2679 			list_add_tail(&virt_ep->bw_endpoint_list,
2680 					&smaller_ep->bw_endpoint_list);
2681 			return;
2682 		}
2683 	}
2684 	/* Add the new endpoint at the end of the list. */
2685 	list_add_tail(&virt_ep->bw_endpoint_list,
2686 			&interval_bw->endpoints);
2687 }
2688 
2689 void xhci_update_tt_active_eps(struct xhci_hcd *xhci,
2690 		struct xhci_virt_device *virt_dev,
2691 		int old_active_eps)
2692 {
2693 	struct xhci_root_port_bw_info *rh_bw_info;
2694 	if (!virt_dev->tt_info)
2695 		return;
2696 
2697 	rh_bw_info = &xhci->rh_bw[virt_dev->rhub_port->hw_portnum];
2698 	if (old_active_eps == 0 &&
2699 				virt_dev->tt_info->active_eps != 0) {
2700 		rh_bw_info->num_active_tts += 1;
2701 		rh_bw_info->bw_table.bw_used += TT_HS_OVERHEAD;
2702 	} else if (old_active_eps != 0 &&
2703 				virt_dev->tt_info->active_eps == 0) {
2704 		rh_bw_info->num_active_tts -= 1;
2705 		rh_bw_info->bw_table.bw_used -= TT_HS_OVERHEAD;
2706 	}
2707 }
2708 
2709 static int xhci_reserve_bandwidth(struct xhci_hcd *xhci,
2710 		struct xhci_virt_device *virt_dev,
2711 		struct xhci_container_ctx *in_ctx)
2712 {
2713 	struct xhci_bw_info ep_bw_info[31];
2714 	int i;
2715 	struct xhci_input_control_ctx *ctrl_ctx;
2716 	int old_active_eps = 0;
2717 
2718 	if (virt_dev->tt_info)
2719 		old_active_eps = virt_dev->tt_info->active_eps;
2720 
2721 	ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
2722 	if (!ctrl_ctx) {
2723 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
2724 				__func__);
2725 		return -ENOMEM;
2726 	}
2727 
2728 	for (i = 0; i < 31; i++) {
2729 		if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i))
2730 			continue;
2731 
2732 		/* Make a copy of the BW info in case we need to revert this */
2733 		memcpy(&ep_bw_info[i], &virt_dev->eps[i].bw_info,
2734 				sizeof(ep_bw_info[i]));
2735 		/* Drop the endpoint from the interval table if the endpoint is
2736 		 * being dropped or changed.
2737 		 */
2738 		if (EP_IS_DROPPED(ctrl_ctx, i))
2739 			xhci_drop_ep_from_interval_table(xhci,
2740 					&virt_dev->eps[i].bw_info,
2741 					virt_dev->bw_table,
2742 					virt_dev->udev,
2743 					&virt_dev->eps[i],
2744 					virt_dev->tt_info);
2745 	}
2746 	/* Overwrite the information stored in the endpoints' bw_info */
2747 	xhci_update_bw_info(xhci, virt_dev->in_ctx, ctrl_ctx, virt_dev);
2748 	for (i = 0; i < 31; i++) {
2749 		/* Add any changed or added endpoints to the interval table */
2750 		if (EP_IS_ADDED(ctrl_ctx, i))
2751 			xhci_add_ep_to_interval_table(xhci,
2752 					&virt_dev->eps[i].bw_info,
2753 					virt_dev->bw_table,
2754 					virt_dev->udev,
2755 					&virt_dev->eps[i],
2756 					virt_dev->tt_info);
2757 	}
2758 
2759 	if (!xhci_check_bw_table(xhci, virt_dev, old_active_eps)) {
2760 		/* Ok, this fits in the bandwidth we have.
2761 		 * Update the number of active TTs.
2762 		 */
2763 		xhci_update_tt_active_eps(xhci, virt_dev, old_active_eps);
2764 		return 0;
2765 	}
2766 
2767 	/* We don't have enough bandwidth for this, revert the stored info. */
2768 	for (i = 0; i < 31; i++) {
2769 		if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i))
2770 			continue;
2771 
2772 		/* Drop the new copies of any added or changed endpoints from
2773 		 * the interval table.
2774 		 */
2775 		if (EP_IS_ADDED(ctrl_ctx, i)) {
2776 			xhci_drop_ep_from_interval_table(xhci,
2777 					&virt_dev->eps[i].bw_info,
2778 					virt_dev->bw_table,
2779 					virt_dev->udev,
2780 					&virt_dev->eps[i],
2781 					virt_dev->tt_info);
2782 		}
2783 		/* Revert the endpoint back to its old information */
2784 		memcpy(&virt_dev->eps[i].bw_info, &ep_bw_info[i],
2785 				sizeof(ep_bw_info[i]));
2786 		/* Add any changed or dropped endpoints back into the table */
2787 		if (EP_IS_DROPPED(ctrl_ctx, i))
2788 			xhci_add_ep_to_interval_table(xhci,
2789 					&virt_dev->eps[i].bw_info,
2790 					virt_dev->bw_table,
2791 					virt_dev->udev,
2792 					&virt_dev->eps[i],
2793 					virt_dev->tt_info);
2794 	}
2795 	return -ENOMEM;
2796 }
2797 
2798 /*
2799  * Synchronous XHCI stop endpoint helper.  Issues the stop endpoint command and
2800  * waits for the command completion before returning.  This does not call
2801  * xhci_handle_cmd_stop_ep(), which has additional handling for 'context error'
2802  * cases, along with transfer ring cleanup.
2803  *
2804  * xhci_stop_endpoint_sync() is intended to be utilized by clients that manage
2805  * their own transfer ring, such as offload situations.
2806  */
2807 int xhci_stop_endpoint_sync(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, int suspend,
2808 			    gfp_t gfp_flags)
2809 {
2810 	struct xhci_command *command;
2811 	unsigned long flags;
2812 	int ret;
2813 
2814 	command = xhci_alloc_command(xhci, true, gfp_flags);
2815 	if (!command)
2816 		return -ENOMEM;
2817 
2818 	spin_lock_irqsave(&xhci->lock, flags);
2819 	ret = xhci_queue_stop_endpoint(xhci, command, ep->vdev->slot_id,
2820 				       ep->ep_index, suspend);
2821 	if (ret < 0) {
2822 		spin_unlock_irqrestore(&xhci->lock, flags);
2823 		goto out;
2824 	}
2825 
2826 	xhci_ring_cmd_db(xhci);
2827 	spin_unlock_irqrestore(&xhci->lock, flags);
2828 
2829 	wait_for_completion(command->completion);
2830 
2831 	/* No handling for COMP_CONTEXT_STATE_ERROR done at command completion*/
2832 	if (command->status == COMP_COMMAND_ABORTED ||
2833 	    command->status == COMP_COMMAND_RING_STOPPED) {
2834 		xhci_warn(xhci, "Timeout while waiting for stop endpoint command\n");
2835 		ret = -ETIME;
2836 	}
2837 out:
2838 	xhci_free_command(xhci, command);
2839 
2840 	return ret;
2841 }
2842 EXPORT_SYMBOL_GPL(xhci_stop_endpoint_sync);
2843 
2844 /* Issue a configure endpoint command or evaluate context command
2845  * and wait for it to finish.
2846  */
2847 static int xhci_configure_endpoint(struct xhci_hcd *xhci,
2848 		struct usb_device *udev,
2849 		struct xhci_command *command,
2850 		bool ctx_change, bool must_succeed)
2851 {
2852 	int ret;
2853 	unsigned long flags;
2854 	struct xhci_input_control_ctx *ctrl_ctx;
2855 	struct xhci_virt_device *virt_dev;
2856 	struct xhci_slot_ctx *slot_ctx;
2857 
2858 	if (!command)
2859 		return -EINVAL;
2860 
2861 	spin_lock_irqsave(&xhci->lock, flags);
2862 
2863 	if (xhci->xhc_state & XHCI_STATE_DYING) {
2864 		spin_unlock_irqrestore(&xhci->lock, flags);
2865 		return -ESHUTDOWN;
2866 	}
2867 
2868 	virt_dev = xhci->devs[udev->slot_id];
2869 
2870 	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
2871 	if (!ctrl_ctx) {
2872 		spin_unlock_irqrestore(&xhci->lock, flags);
2873 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
2874 				__func__);
2875 		return -ENOMEM;
2876 	}
2877 
2878 	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK) &&
2879 			xhci_reserve_host_resources(xhci, ctrl_ctx)) {
2880 		spin_unlock_irqrestore(&xhci->lock, flags);
2881 		xhci_warn(xhci, "Not enough host resources, "
2882 				"active endpoint contexts = %u\n",
2883 				xhci->num_active_eps);
2884 		return -ENOMEM;
2885 	}
2886 	if ((xhci->quirks & XHCI_SW_BW_CHECKING) && !ctx_change &&
2887 	    xhci_reserve_bandwidth(xhci, virt_dev, command->in_ctx)) {
2888 		if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK))
2889 			xhci_free_host_resources(xhci, ctrl_ctx);
2890 		spin_unlock_irqrestore(&xhci->lock, flags);
2891 		xhci_warn(xhci, "Not enough bandwidth\n");
2892 		return -ENOMEM;
2893 	}
2894 
2895 	slot_ctx = xhci_get_slot_ctx(xhci, command->in_ctx);
2896 
2897 	trace_xhci_configure_endpoint_ctrl_ctx(ctrl_ctx);
2898 	trace_xhci_configure_endpoint(slot_ctx);
2899 
2900 	if (!ctx_change)
2901 		ret = xhci_queue_configure_endpoint(xhci, command,
2902 				command->in_ctx->dma,
2903 				udev->slot_id, must_succeed);
2904 	else
2905 		ret = xhci_queue_evaluate_context(xhci, command,
2906 				command->in_ctx->dma,
2907 				udev->slot_id, must_succeed);
2908 	if (ret < 0) {
2909 		if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK))
2910 			xhci_free_host_resources(xhci, ctrl_ctx);
2911 		spin_unlock_irqrestore(&xhci->lock, flags);
2912 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
2913 				"FIXME allocate a new ring segment");
2914 		return -ENOMEM;
2915 	}
2916 	xhci_ring_cmd_db(xhci);
2917 	spin_unlock_irqrestore(&xhci->lock, flags);
2918 
2919 	/* Wait for the configure endpoint command to complete */
2920 	wait_for_completion(command->completion);
2921 
2922 	if (!ctx_change)
2923 		ret = xhci_configure_endpoint_result(xhci, udev,
2924 						     &command->status);
2925 	else
2926 		ret = xhci_evaluate_context_result(xhci, udev,
2927 						   &command->status);
2928 
2929 	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
2930 		spin_lock_irqsave(&xhci->lock, flags);
2931 		/* If the command failed, remove the reserved resources.
2932 		 * Otherwise, clean up the estimate to include dropped eps.
2933 		 */
2934 		if (ret)
2935 			xhci_free_host_resources(xhci, ctrl_ctx);
2936 		else
2937 			xhci_finish_resource_reservation(xhci, ctrl_ctx);
2938 		spin_unlock_irqrestore(&xhci->lock, flags);
2939 	}
2940 	return ret;
2941 }
2942 
2943 static void xhci_check_bw_drop_ep_streams(struct xhci_hcd *xhci,
2944 	struct xhci_virt_device *vdev, int i)
2945 {
2946 	struct xhci_virt_ep *ep = &vdev->eps[i];
2947 
2948 	if (ep->ep_state & EP_HAS_STREAMS) {
2949 		xhci_warn(xhci, "WARN: endpoint 0x%02x has streams on set_interface, freeing streams.\n",
2950 				xhci_get_endpoint_address(i));
2951 		xhci_free_stream_info(xhci, ep->stream_info);
2952 		ep->stream_info = NULL;
2953 		ep->ep_state &= ~EP_HAS_STREAMS;
2954 	}
2955 }
2956 
2957 /* Called after one or more calls to xhci_add_endpoint() or
2958  * xhci_drop_endpoint().  If this call fails, the USB core is expected
2959  * to call xhci_reset_bandwidth().
2960  *
2961  * Since we are in the middle of changing either configuration or
2962  * installing a new alt setting, the USB core won't allow URBs to be
2963  * enqueued for any endpoint on the old config or interface.  Nothing
2964  * else should be touching the xhci->devs[slot_id] structure, so we
2965  * don't need to take the xhci->lock for manipulating that.
2966  */
2967 int xhci_check_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
2968 {
2969 	int i;
2970 	int ret = 0;
2971 	struct xhci_hcd *xhci;
2972 	struct xhci_virt_device	*virt_dev;
2973 	struct xhci_input_control_ctx *ctrl_ctx;
2974 	struct xhci_slot_ctx *slot_ctx;
2975 	struct xhci_command *command;
2976 
2977 	ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
2978 	if (ret <= 0)
2979 		return ret;
2980 	xhci = hcd_to_xhci(hcd);
2981 	if ((xhci->xhc_state & XHCI_STATE_DYING) ||
2982 		(xhci->xhc_state & XHCI_STATE_REMOVING))
2983 		return -ENODEV;
2984 
2985 	xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
2986 	virt_dev = xhci->devs[udev->slot_id];
2987 
2988 	command = xhci_alloc_command(xhci, true, GFP_KERNEL);
2989 	if (!command)
2990 		return -ENOMEM;
2991 
2992 	command->in_ctx = virt_dev->in_ctx;
2993 
2994 	/* See section 4.6.6 - A0 = 1; A1 = D0 = D1 = 0 */
2995 	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
2996 	if (!ctrl_ctx) {
2997 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
2998 				__func__);
2999 		ret = -ENOMEM;
3000 		goto command_cleanup;
3001 	}
3002 	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
3003 	ctrl_ctx->add_flags &= cpu_to_le32(~EP0_FLAG);
3004 	ctrl_ctx->drop_flags &= cpu_to_le32(~(SLOT_FLAG | EP0_FLAG));
3005 
3006 	/* Don't issue the command if there's no endpoints to update. */
3007 	if (ctrl_ctx->add_flags == cpu_to_le32(SLOT_FLAG) &&
3008 	    ctrl_ctx->drop_flags == 0) {
3009 		ret = 0;
3010 		goto command_cleanup;
3011 	}
3012 	/* Fix up Context Entries field. Minimum value is EP0 == BIT(1). */
3013 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
3014 	for (i = 31; i >= 1; i--) {
3015 		__le32 le32 = cpu_to_le32(BIT(i));
3016 
3017 		if ((virt_dev->eps[i-1].ring && !(ctrl_ctx->drop_flags & le32))
3018 		    || (ctrl_ctx->add_flags & le32) || i == 1) {
3019 			slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
3020 			slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(i));
3021 			break;
3022 		}
3023 	}
3024 
3025 	ret = xhci_configure_endpoint(xhci, udev, command,
3026 			false, false);
3027 	if (ret)
3028 		/* Callee should call reset_bandwidth() */
3029 		goto command_cleanup;
3030 
3031 	/* Free any rings that were dropped, but not changed. */
3032 	for (i = 1; i < 31; i++) {
3033 		if ((le32_to_cpu(ctrl_ctx->drop_flags) & (1 << (i + 1))) &&
3034 		    !(le32_to_cpu(ctrl_ctx->add_flags) & (1 << (i + 1)))) {
3035 			xhci_free_endpoint_ring(xhci, virt_dev, i);
3036 			xhci_check_bw_drop_ep_streams(xhci, virt_dev, i);
3037 		}
3038 	}
3039 	xhci_zero_in_ctx(xhci, virt_dev);
3040 	/*
3041 	 * Install any rings for completely new endpoints or changed endpoints,
3042 	 * and free any old rings from changed endpoints.
3043 	 */
3044 	for (i = 1; i < 31; i++) {
3045 		if (!virt_dev->eps[i].new_ring)
3046 			continue;
3047 		/* Only free the old ring if it exists.
3048 		 * It may not if this is the first add of an endpoint.
3049 		 */
3050 		if (virt_dev->eps[i].ring) {
3051 			xhci_free_endpoint_ring(xhci, virt_dev, i);
3052 		}
3053 		xhci_check_bw_drop_ep_streams(xhci, virt_dev, i);
3054 		virt_dev->eps[i].ring = virt_dev->eps[i].new_ring;
3055 		virt_dev->eps[i].new_ring = NULL;
3056 		xhci_debugfs_create_endpoint(xhci, virt_dev, i);
3057 	}
3058 command_cleanup:
3059 	kfree(command->completion);
3060 	kfree(command);
3061 
3062 	return ret;
3063 }
3064 EXPORT_SYMBOL_GPL(xhci_check_bandwidth);
3065 
3066 void xhci_reset_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
3067 {
3068 	struct xhci_hcd *xhci;
3069 	struct xhci_virt_device	*virt_dev;
3070 	int i, ret;
3071 
3072 	ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
3073 	if (ret <= 0)
3074 		return;
3075 	xhci = hcd_to_xhci(hcd);
3076 
3077 	xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
3078 	virt_dev = xhci->devs[udev->slot_id];
3079 	/* Free any rings allocated for added endpoints */
3080 	for (i = 0; i < 31; i++) {
3081 		if (virt_dev->eps[i].new_ring) {
3082 			xhci_debugfs_remove_endpoint(xhci, virt_dev, i);
3083 			xhci_ring_free(xhci, virt_dev->eps[i].new_ring);
3084 			virt_dev->eps[i].new_ring = NULL;
3085 		}
3086 	}
3087 	xhci_zero_in_ctx(xhci, virt_dev);
3088 }
3089 EXPORT_SYMBOL_GPL(xhci_reset_bandwidth);
3090 
3091 static void xhci_setup_input_ctx_for_config_ep(struct xhci_hcd *xhci,
3092 		struct xhci_container_ctx *in_ctx,
3093 		struct xhci_container_ctx *out_ctx,
3094 		struct xhci_input_control_ctx *ctrl_ctx,
3095 		u32 add_flags, u32 drop_flags)
3096 {
3097 	ctrl_ctx->add_flags = cpu_to_le32(add_flags);
3098 	ctrl_ctx->drop_flags = cpu_to_le32(drop_flags);
3099 	xhci_slot_copy(xhci, in_ctx, out_ctx);
3100 	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
3101 }
3102 
3103 static void xhci_endpoint_disable(struct usb_hcd *hcd,
3104 				  struct usb_host_endpoint *host_ep)
3105 {
3106 	struct xhci_hcd		*xhci;
3107 	struct xhci_virt_device	*vdev;
3108 	struct xhci_virt_ep	*ep;
3109 	struct usb_device	*udev;
3110 	unsigned long		flags;
3111 	unsigned int		ep_index;
3112 
3113 	xhci = hcd_to_xhci(hcd);
3114 rescan:
3115 	spin_lock_irqsave(&xhci->lock, flags);
3116 
3117 	udev = (struct usb_device *)host_ep->hcpriv;
3118 	if (!udev || !udev->slot_id)
3119 		goto done;
3120 
3121 	vdev = xhci->devs[udev->slot_id];
3122 	if (!vdev)
3123 		goto done;
3124 
3125 	ep_index = xhci_get_endpoint_index(&host_ep->desc);
3126 	ep = &vdev->eps[ep_index];
3127 
3128 	/* wait for hub_tt_work to finish clearing hub TT */
3129 	if (ep->ep_state & EP_CLEARING_TT) {
3130 		spin_unlock_irqrestore(&xhci->lock, flags);
3131 		schedule_timeout_uninterruptible(1);
3132 		goto rescan;
3133 	}
3134 
3135 	if (ep->ep_state)
3136 		xhci_dbg(xhci, "endpoint disable with ep_state 0x%x\n",
3137 			 ep->ep_state);
3138 done:
3139 	host_ep->hcpriv = NULL;
3140 	spin_unlock_irqrestore(&xhci->lock, flags);
3141 }
3142 
3143 /*
3144  * Called after usb core issues a clear halt control message.
3145  * The host side of the halt should already be cleared by a reset endpoint
3146  * command issued when the STALL event was received.
3147  *
3148  * The reset endpoint command may only be issued to endpoints in the halted
3149  * state. For software that wishes to reset the data toggle or sequence number
3150  * of an endpoint that isn't in the halted state this function will issue a
3151  * configure endpoint command with the Drop and Add bits set for the target
3152  * endpoint. Refer to the additional note in xhci spcification section 4.6.8.
3153  *
3154  * vdev may be lost due to xHC restore error and re-initialization during S3/S4
3155  * resume. A new vdev will be allocated later by xhci_discover_or_reset_device()
3156  */
3157 
3158 static void xhci_endpoint_reset(struct usb_hcd *hcd,
3159 		struct usb_host_endpoint *host_ep)
3160 {
3161 	struct xhci_hcd *xhci;
3162 	struct usb_device *udev;
3163 	struct xhci_virt_device *vdev;
3164 	struct xhci_virt_ep *ep;
3165 	struct xhci_input_control_ctx *ctrl_ctx;
3166 	struct xhci_command *stop_cmd, *cfg_cmd;
3167 	unsigned int ep_index;
3168 	unsigned long flags;
3169 	u32 ep_flag;
3170 	int err;
3171 
3172 	xhci = hcd_to_xhci(hcd);
3173 	ep_index = xhci_get_endpoint_index(&host_ep->desc);
3174 
3175 	/*
3176 	 * Usb core assumes a max packet value for ep0 on FS devices until the
3177 	 * real value is read from the descriptor. Core resets Ep0 if values
3178 	 * mismatch. Reconfigure the xhci ep0 endpoint context here in that case
3179 	 */
3180 	if (usb_endpoint_xfer_control(&host_ep->desc) && ep_index == 0) {
3181 
3182 		udev = container_of(host_ep, struct usb_device, ep0);
3183 		if (udev->speed != USB_SPEED_FULL || !udev->slot_id)
3184 			return;
3185 
3186 		vdev = xhci->devs[udev->slot_id];
3187 		if (!vdev || vdev->udev != udev)
3188 			return;
3189 
3190 		xhci_check_ep0_maxpacket(xhci, vdev);
3191 
3192 		/* Nothing else should be done here for ep0 during ep reset */
3193 		return;
3194 	}
3195 
3196 	if (!host_ep->hcpriv)
3197 		return;
3198 	udev = (struct usb_device *) host_ep->hcpriv;
3199 	vdev = xhci->devs[udev->slot_id];
3200 
3201 	if (!udev->slot_id || !vdev)
3202 		return;
3203 
3204 	ep = &vdev->eps[ep_index];
3205 
3206 	/* Bail out if toggle is already being cleared by a endpoint reset */
3207 	spin_lock_irqsave(&xhci->lock, flags);
3208 	if (ep->ep_state & EP_HARD_CLEAR_TOGGLE) {
3209 		ep->ep_state &= ~EP_HARD_CLEAR_TOGGLE;
3210 		spin_unlock_irqrestore(&xhci->lock, flags);
3211 		return;
3212 	}
3213 	spin_unlock_irqrestore(&xhci->lock, flags);
3214 	/* Only interrupt and bulk ep's use data toggle, USB2 spec 5.5.4-> */
3215 	if (usb_endpoint_xfer_control(&host_ep->desc) ||
3216 	    usb_endpoint_xfer_isoc(&host_ep->desc))
3217 		return;
3218 
3219 	ep_flag = xhci_get_endpoint_flag(&host_ep->desc);
3220 
3221 	if (ep_flag == SLOT_FLAG || ep_flag == EP0_FLAG)
3222 		return;
3223 
3224 	stop_cmd = xhci_alloc_command(xhci, true, GFP_NOWAIT);
3225 	if (!stop_cmd)
3226 		return;
3227 
3228 	cfg_cmd = xhci_alloc_command_with_ctx(xhci, true, GFP_NOWAIT);
3229 	if (!cfg_cmd)
3230 		goto cleanup;
3231 
3232 	spin_lock_irqsave(&xhci->lock, flags);
3233 
3234 	/* block queuing new trbs and ringing ep doorbell */
3235 	ep->ep_state |= EP_SOFT_CLEAR_TOGGLE;
3236 
3237 	/*
3238 	 * Make sure endpoint ring is empty before resetting the toggle/seq.
3239 	 * Driver is required to synchronously cancel all transfer request.
3240 	 * Stop the endpoint to force xHC to update the output context
3241 	 */
3242 
3243 	if (!list_empty(&ep->ring->td_list)) {
3244 		dev_err(&udev->dev, "EP not empty, refuse reset\n");
3245 		spin_unlock_irqrestore(&xhci->lock, flags);
3246 		xhci_free_command(xhci, cfg_cmd);
3247 		goto cleanup;
3248 	}
3249 
3250 	err = xhci_queue_stop_endpoint(xhci, stop_cmd, udev->slot_id,
3251 					ep_index, 0);
3252 	if (err < 0) {
3253 		spin_unlock_irqrestore(&xhci->lock, flags);
3254 		xhci_free_command(xhci, cfg_cmd);
3255 		xhci_dbg(xhci, "%s: Failed to queue stop ep command, %d ",
3256 				__func__, err);
3257 		goto cleanup;
3258 	}
3259 
3260 	xhci_ring_cmd_db(xhci);
3261 	spin_unlock_irqrestore(&xhci->lock, flags);
3262 
3263 	wait_for_completion(stop_cmd->completion);
3264 
3265 	spin_lock_irqsave(&xhci->lock, flags);
3266 
3267 	/* config ep command clears toggle if add and drop ep flags are set */
3268 	ctrl_ctx = xhci_get_input_control_ctx(cfg_cmd->in_ctx);
3269 	if (!ctrl_ctx) {
3270 		spin_unlock_irqrestore(&xhci->lock, flags);
3271 		xhci_free_command(xhci, cfg_cmd);
3272 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
3273 				__func__);
3274 		goto cleanup;
3275 	}
3276 
3277 	xhci_setup_input_ctx_for_config_ep(xhci, cfg_cmd->in_ctx, vdev->out_ctx,
3278 					   ctrl_ctx, ep_flag, ep_flag);
3279 	xhci_endpoint_copy(xhci, cfg_cmd->in_ctx, vdev->out_ctx, ep_index);
3280 
3281 	err = xhci_queue_configure_endpoint(xhci, cfg_cmd, cfg_cmd->in_ctx->dma,
3282 				      udev->slot_id, false);
3283 	if (err < 0) {
3284 		spin_unlock_irqrestore(&xhci->lock, flags);
3285 		xhci_free_command(xhci, cfg_cmd);
3286 		xhci_dbg(xhci, "%s: Failed to queue config ep command, %d ",
3287 				__func__, err);
3288 		goto cleanup;
3289 	}
3290 
3291 	xhci_ring_cmd_db(xhci);
3292 	spin_unlock_irqrestore(&xhci->lock, flags);
3293 
3294 	wait_for_completion(cfg_cmd->completion);
3295 
3296 	xhci_free_command(xhci, cfg_cmd);
3297 cleanup:
3298 	xhci_free_command(xhci, stop_cmd);
3299 	spin_lock_irqsave(&xhci->lock, flags);
3300 	if (ep->ep_state & EP_SOFT_CLEAR_TOGGLE)
3301 		ep->ep_state &= ~EP_SOFT_CLEAR_TOGGLE;
3302 	spin_unlock_irqrestore(&xhci->lock, flags);
3303 }
3304 
3305 static int xhci_check_streams_endpoint(struct xhci_hcd *xhci,
3306 		struct usb_device *udev, struct usb_host_endpoint *ep,
3307 		unsigned int slot_id)
3308 {
3309 	int ret;
3310 	unsigned int ep_index;
3311 	unsigned int ep_state;
3312 
3313 	if (!ep)
3314 		return -EINVAL;
3315 	ret = xhci_check_args(xhci_to_hcd(xhci), udev, ep, 1, true, __func__);
3316 	if (ret <= 0)
3317 		return ret ? ret : -EINVAL;
3318 	if (usb_ss_max_streams(&ep->ss_ep_comp) == 0) {
3319 		xhci_warn(xhci, "WARN: SuperSpeed Endpoint Companion"
3320 				" descriptor for ep 0x%x does not support streams\n",
3321 				ep->desc.bEndpointAddress);
3322 		return -EINVAL;
3323 	}
3324 
3325 	ep_index = xhci_get_endpoint_index(&ep->desc);
3326 	ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;
3327 	if (ep_state & EP_HAS_STREAMS ||
3328 			ep_state & EP_GETTING_STREAMS) {
3329 		xhci_warn(xhci, "WARN: SuperSpeed bulk endpoint 0x%x "
3330 				"already has streams set up.\n",
3331 				ep->desc.bEndpointAddress);
3332 		xhci_warn(xhci, "Send email to xHCI maintainer and ask for "
3333 				"dynamic stream context array reallocation.\n");
3334 		return -EINVAL;
3335 	}
3336 	if (!list_empty(&xhci->devs[slot_id]->eps[ep_index].ring->td_list)) {
3337 		xhci_warn(xhci, "Cannot setup streams for SuperSpeed bulk "
3338 				"endpoint 0x%x; URBs are pending.\n",
3339 				ep->desc.bEndpointAddress);
3340 		return -EINVAL;
3341 	}
3342 	return 0;
3343 }
3344 
3345 static void xhci_calculate_streams_entries(struct xhci_hcd *xhci,
3346 		unsigned int *num_streams, unsigned int *num_stream_ctxs)
3347 {
3348 	unsigned int max_streams;
3349 
3350 	/* The stream context array size must be a power of two */
3351 	*num_stream_ctxs = roundup_pow_of_two(*num_streams);
3352 	/*
3353 	 * Find out how many primary stream array entries the host controller
3354 	 * supports.  Later we may use secondary stream arrays (similar to 2nd
3355 	 * level page entries), but that's an optional feature for xHCI host
3356 	 * controllers. xHCs must support at least 4 stream IDs.
3357 	 */
3358 	max_streams = HCC_MAX_PSA(xhci->hcc_params);
3359 	if (*num_stream_ctxs > max_streams) {
3360 		xhci_dbg(xhci, "xHCI HW only supports %u stream ctx entries.\n",
3361 				max_streams);
3362 		*num_stream_ctxs = max_streams;
3363 		*num_streams = max_streams;
3364 	}
3365 }
3366 
3367 /* Returns an error code if one of the endpoint already has streams.
3368  * This does not change any data structures, it only checks and gathers
3369  * information.
3370  */
3371 static int xhci_calculate_streams_and_bitmask(struct xhci_hcd *xhci,
3372 		struct usb_device *udev,
3373 		struct usb_host_endpoint **eps, unsigned int num_eps,
3374 		unsigned int *num_streams, u32 *changed_ep_bitmask)
3375 {
3376 	unsigned int max_streams;
3377 	unsigned int endpoint_flag;
3378 	int i;
3379 	int ret;
3380 
3381 	for (i = 0; i < num_eps; i++) {
3382 		ret = xhci_check_streams_endpoint(xhci, udev,
3383 				eps[i], udev->slot_id);
3384 		if (ret < 0)
3385 			return ret;
3386 
3387 		max_streams = usb_ss_max_streams(&eps[i]->ss_ep_comp);
3388 		if (max_streams < (*num_streams - 1)) {
3389 			xhci_dbg(xhci, "Ep 0x%x only supports %u stream IDs.\n",
3390 					eps[i]->desc.bEndpointAddress,
3391 					max_streams);
3392 			*num_streams = max_streams+1;
3393 		}
3394 
3395 		endpoint_flag = xhci_get_endpoint_flag(&eps[i]->desc);
3396 		if (*changed_ep_bitmask & endpoint_flag)
3397 			return -EINVAL;
3398 		*changed_ep_bitmask |= endpoint_flag;
3399 	}
3400 	return 0;
3401 }
3402 
3403 static u32 xhci_calculate_no_streams_bitmask(struct xhci_hcd *xhci,
3404 		struct usb_device *udev,
3405 		struct usb_host_endpoint **eps, unsigned int num_eps)
3406 {
3407 	u32 changed_ep_bitmask = 0;
3408 	unsigned int slot_id;
3409 	unsigned int ep_index;
3410 	unsigned int ep_state;
3411 	int i;
3412 
3413 	slot_id = udev->slot_id;
3414 	if (!xhci->devs[slot_id])
3415 		return 0;
3416 
3417 	for (i = 0; i < num_eps; i++) {
3418 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3419 		ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;
3420 		/* Are streams already being freed for the endpoint? */
3421 		if (ep_state & EP_GETTING_NO_STREAMS) {
3422 			xhci_warn(xhci, "WARN Can't disable streams for "
3423 					"endpoint 0x%x, "
3424 					"streams are being disabled already\n",
3425 					eps[i]->desc.bEndpointAddress);
3426 			return 0;
3427 		}
3428 		/* Are there actually any streams to free? */
3429 		if (!(ep_state & EP_HAS_STREAMS) &&
3430 				!(ep_state & EP_GETTING_STREAMS)) {
3431 			xhci_warn(xhci, "WARN Can't disable streams for "
3432 					"endpoint 0x%x, "
3433 					"streams are already disabled!\n",
3434 					eps[i]->desc.bEndpointAddress);
3435 			xhci_warn(xhci, "WARN xhci_free_streams() called "
3436 					"with non-streams endpoint\n");
3437 			return 0;
3438 		}
3439 		changed_ep_bitmask |= xhci_get_endpoint_flag(&eps[i]->desc);
3440 	}
3441 	return changed_ep_bitmask;
3442 }
3443 
3444 /*
3445  * The USB device drivers use this function (through the HCD interface in USB
3446  * core) to prepare a set of bulk endpoints to use streams.  Streams are used to
3447  * coordinate mass storage command queueing across multiple endpoints (basically
3448  * a stream ID == a task ID).
3449  *
3450  * Setting up streams involves allocating the same size stream context array
3451  * for each endpoint and issuing a configure endpoint command for all endpoints.
3452  *
3453  * Don't allow the call to succeed if one endpoint only supports one stream
3454  * (which means it doesn't support streams at all).
3455  *
3456  * Drivers may get less stream IDs than they asked for, if the host controller
3457  * hardware or endpoints claim they can't support the number of requested
3458  * stream IDs.
3459  */
3460 static int xhci_alloc_streams(struct usb_hcd *hcd, struct usb_device *udev,
3461 		struct usb_host_endpoint **eps, unsigned int num_eps,
3462 		unsigned int num_streams, gfp_t mem_flags)
3463 {
3464 	int i, ret;
3465 	struct xhci_hcd *xhci;
3466 	struct xhci_virt_device *vdev;
3467 	struct xhci_command *config_cmd;
3468 	struct xhci_input_control_ctx *ctrl_ctx;
3469 	unsigned int ep_index;
3470 	unsigned int num_stream_ctxs;
3471 	unsigned int max_packet;
3472 	unsigned long flags;
3473 	u32 changed_ep_bitmask = 0;
3474 
3475 	if (!eps)
3476 		return -EINVAL;
3477 
3478 	/* Add one to the number of streams requested to account for
3479 	 * stream 0 that is reserved for xHCI usage.
3480 	 */
3481 	num_streams += 1;
3482 	xhci = hcd_to_xhci(hcd);
3483 	xhci_dbg(xhci, "Driver wants %u stream IDs (including stream 0).\n",
3484 			num_streams);
3485 
3486 	/* MaxPSASize value 0 (2 streams) means streams are not supported */
3487 	if ((xhci->quirks & XHCI_BROKEN_STREAMS) ||
3488 			HCC_MAX_PSA(xhci->hcc_params) < 4) {
3489 		xhci_dbg(xhci, "xHCI controller does not support streams.\n");
3490 		return -ENOSYS;
3491 	}
3492 
3493 	config_cmd = xhci_alloc_command_with_ctx(xhci, true, mem_flags);
3494 	if (!config_cmd)
3495 		return -ENOMEM;
3496 
3497 	ctrl_ctx = xhci_get_input_control_ctx(config_cmd->in_ctx);
3498 	if (!ctrl_ctx) {
3499 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
3500 				__func__);
3501 		xhci_free_command(xhci, config_cmd);
3502 		return -ENOMEM;
3503 	}
3504 
3505 	/* Check to make sure all endpoints are not already configured for
3506 	 * streams.  While we're at it, find the maximum number of streams that
3507 	 * all the endpoints will support and check for duplicate endpoints.
3508 	 */
3509 	spin_lock_irqsave(&xhci->lock, flags);
3510 	ret = xhci_calculate_streams_and_bitmask(xhci, udev, eps,
3511 			num_eps, &num_streams, &changed_ep_bitmask);
3512 	if (ret < 0) {
3513 		xhci_free_command(xhci, config_cmd);
3514 		spin_unlock_irqrestore(&xhci->lock, flags);
3515 		return ret;
3516 	}
3517 	if (num_streams <= 1) {
3518 		xhci_warn(xhci, "WARN: endpoints can't handle "
3519 				"more than one stream.\n");
3520 		xhci_free_command(xhci, config_cmd);
3521 		spin_unlock_irqrestore(&xhci->lock, flags);
3522 		return -EINVAL;
3523 	}
3524 	vdev = xhci->devs[udev->slot_id];
3525 	/* Mark each endpoint as being in transition, so
3526 	 * xhci_urb_enqueue() will reject all URBs.
3527 	 */
3528 	for (i = 0; i < num_eps; i++) {
3529 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3530 		vdev->eps[ep_index].ep_state |= EP_GETTING_STREAMS;
3531 	}
3532 	spin_unlock_irqrestore(&xhci->lock, flags);
3533 
3534 	/* Setup internal data structures and allocate HW data structures for
3535 	 * streams (but don't install the HW structures in the input context
3536 	 * until we're sure all memory allocation succeeded).
3537 	 */
3538 	xhci_calculate_streams_entries(xhci, &num_streams, &num_stream_ctxs);
3539 	xhci_dbg(xhci, "Need %u stream ctx entries for %u stream IDs.\n",
3540 			num_stream_ctxs, num_streams);
3541 
3542 	for (i = 0; i < num_eps; i++) {
3543 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3544 		max_packet = usb_endpoint_maxp(&eps[i]->desc);
3545 		vdev->eps[ep_index].stream_info = xhci_alloc_stream_info(xhci,
3546 				num_stream_ctxs,
3547 				num_streams,
3548 				max_packet, mem_flags);
3549 		if (!vdev->eps[ep_index].stream_info)
3550 			goto cleanup;
3551 		/* Set maxPstreams in endpoint context and update deq ptr to
3552 		 * point to stream context array. FIXME
3553 		 */
3554 	}
3555 
3556 	/* Set up the input context for a configure endpoint command. */
3557 	for (i = 0; i < num_eps; i++) {
3558 		struct xhci_ep_ctx *ep_ctx;
3559 
3560 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3561 		ep_ctx = xhci_get_ep_ctx(xhci, config_cmd->in_ctx, ep_index);
3562 
3563 		xhci_endpoint_copy(xhci, config_cmd->in_ctx,
3564 				vdev->out_ctx, ep_index);
3565 		xhci_setup_streams_ep_input_ctx(xhci, ep_ctx,
3566 				vdev->eps[ep_index].stream_info);
3567 	}
3568 	/* Tell the HW to drop its old copy of the endpoint context info
3569 	 * and add the updated copy from the input context.
3570 	 */
3571 	xhci_setup_input_ctx_for_config_ep(xhci, config_cmd->in_ctx,
3572 			vdev->out_ctx, ctrl_ctx,
3573 			changed_ep_bitmask, changed_ep_bitmask);
3574 
3575 	/* Issue and wait for the configure endpoint command */
3576 	ret = xhci_configure_endpoint(xhci, udev, config_cmd,
3577 			false, false);
3578 
3579 	/* xHC rejected the configure endpoint command for some reason, so we
3580 	 * leave the old ring intact and free our internal streams data
3581 	 * structure.
3582 	 */
3583 	if (ret < 0)
3584 		goto cleanup;
3585 
3586 	spin_lock_irqsave(&xhci->lock, flags);
3587 	for (i = 0; i < num_eps; i++) {
3588 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3589 		vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS;
3590 		xhci_dbg(xhci, "Slot %u ep ctx %u now has streams.\n",
3591 			 udev->slot_id, ep_index);
3592 		vdev->eps[ep_index].ep_state |= EP_HAS_STREAMS;
3593 	}
3594 	xhci_free_command(xhci, config_cmd);
3595 	spin_unlock_irqrestore(&xhci->lock, flags);
3596 
3597 	for (i = 0; i < num_eps; i++) {
3598 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3599 		xhci_debugfs_create_stream_files(xhci, vdev, ep_index);
3600 	}
3601 	/* Subtract 1 for stream 0, which drivers can't use */
3602 	return num_streams - 1;
3603 
3604 cleanup:
3605 	/* If it didn't work, free the streams! */
3606 	for (i = 0; i < num_eps; i++) {
3607 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3608 		xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info);
3609 		vdev->eps[ep_index].stream_info = NULL;
3610 		/* FIXME Unset maxPstreams in endpoint context and
3611 		 * update deq ptr to point to normal string ring.
3612 		 */
3613 		vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS;
3614 		vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS;
3615 		xhci_endpoint_zero(xhci, vdev, eps[i]);
3616 	}
3617 	xhci_free_command(xhci, config_cmd);
3618 	return -ENOMEM;
3619 }
3620 
3621 /* Transition the endpoint from using streams to being a "normal" endpoint
3622  * without streams.
3623  *
3624  * Modify the endpoint context state, submit a configure endpoint command,
3625  * and free all endpoint rings for streams if that completes successfully.
3626  */
3627 static int xhci_free_streams(struct usb_hcd *hcd, struct usb_device *udev,
3628 		struct usb_host_endpoint **eps, unsigned int num_eps,
3629 		gfp_t mem_flags)
3630 {
3631 	int i, ret;
3632 	struct xhci_hcd *xhci;
3633 	struct xhci_virt_device *vdev;
3634 	struct xhci_command *command;
3635 	struct xhci_input_control_ctx *ctrl_ctx;
3636 	unsigned int ep_index;
3637 	unsigned long flags;
3638 	u32 changed_ep_bitmask;
3639 
3640 	xhci = hcd_to_xhci(hcd);
3641 	vdev = xhci->devs[udev->slot_id];
3642 
3643 	/* Set up a configure endpoint command to remove the streams rings */
3644 	spin_lock_irqsave(&xhci->lock, flags);
3645 	changed_ep_bitmask = xhci_calculate_no_streams_bitmask(xhci,
3646 			udev, eps, num_eps);
3647 	if (changed_ep_bitmask == 0) {
3648 		spin_unlock_irqrestore(&xhci->lock, flags);
3649 		return -EINVAL;
3650 	}
3651 
3652 	/* Use the xhci_command structure from the first endpoint.  We may have
3653 	 * allocated too many, but the driver may call xhci_free_streams() for
3654 	 * each endpoint it grouped into one call to xhci_alloc_streams().
3655 	 */
3656 	ep_index = xhci_get_endpoint_index(&eps[0]->desc);
3657 	command = vdev->eps[ep_index].stream_info->free_streams_command;
3658 	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
3659 	if (!ctrl_ctx) {
3660 		spin_unlock_irqrestore(&xhci->lock, flags);
3661 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
3662 				__func__);
3663 		return -EINVAL;
3664 	}
3665 
3666 	for (i = 0; i < num_eps; i++) {
3667 		struct xhci_ep_ctx *ep_ctx;
3668 
3669 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3670 		ep_ctx = xhci_get_ep_ctx(xhci, command->in_ctx, ep_index);
3671 		xhci->devs[udev->slot_id]->eps[ep_index].ep_state |=
3672 			EP_GETTING_NO_STREAMS;
3673 
3674 		xhci_endpoint_copy(xhci, command->in_ctx,
3675 				vdev->out_ctx, ep_index);
3676 		xhci_setup_no_streams_ep_input_ctx(ep_ctx,
3677 				&vdev->eps[ep_index]);
3678 	}
3679 	xhci_setup_input_ctx_for_config_ep(xhci, command->in_ctx,
3680 			vdev->out_ctx, ctrl_ctx,
3681 			changed_ep_bitmask, changed_ep_bitmask);
3682 	spin_unlock_irqrestore(&xhci->lock, flags);
3683 
3684 	/* Issue and wait for the configure endpoint command,
3685 	 * which must succeed.
3686 	 */
3687 	ret = xhci_configure_endpoint(xhci, udev, command,
3688 			false, true);
3689 
3690 	/* xHC rejected the configure endpoint command for some reason, so we
3691 	 * leave the streams rings intact.
3692 	 */
3693 	if (ret < 0)
3694 		return ret;
3695 
3696 	spin_lock_irqsave(&xhci->lock, flags);
3697 	for (i = 0; i < num_eps; i++) {
3698 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3699 		xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info);
3700 		vdev->eps[ep_index].stream_info = NULL;
3701 		/* FIXME Unset maxPstreams in endpoint context and
3702 		 * update deq ptr to point to normal string ring.
3703 		 */
3704 		vdev->eps[ep_index].ep_state &= ~EP_GETTING_NO_STREAMS;
3705 		vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS;
3706 	}
3707 	spin_unlock_irqrestore(&xhci->lock, flags);
3708 
3709 	return 0;
3710 }
3711 
3712 /*
3713  * Deletes endpoint resources for endpoints that were active before a Reset
3714  * Device command, or a Disable Slot command.  The Reset Device command leaves
3715  * the control endpoint intact, whereas the Disable Slot command deletes it.
3716  *
3717  * Must be called with xhci->lock held.
3718  */
3719 void xhci_free_device_endpoint_resources(struct xhci_hcd *xhci,
3720 	struct xhci_virt_device *virt_dev, bool drop_control_ep)
3721 {
3722 	int i;
3723 	unsigned int num_dropped_eps = 0;
3724 	unsigned int drop_flags = 0;
3725 
3726 	for (i = (drop_control_ep ? 0 : 1); i < 31; i++) {
3727 		if (virt_dev->eps[i].ring) {
3728 			drop_flags |= 1 << i;
3729 			num_dropped_eps++;
3730 		}
3731 	}
3732 	xhci->num_active_eps -= num_dropped_eps;
3733 	if (num_dropped_eps)
3734 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
3735 				"Dropped %u ep ctxs, flags = 0x%x, "
3736 				"%u now active.",
3737 				num_dropped_eps, drop_flags,
3738 				xhci->num_active_eps);
3739 }
3740 
3741 static void xhci_free_dev(struct usb_hcd *hcd, struct usb_device *udev);
3742 
3743 /*
3744  * This submits a Reset Device Command, which will set the device state to 0,
3745  * set the device address to 0, and disable all the endpoints except the default
3746  * control endpoint.  The USB core should come back and call
3747  * xhci_address_device(), and then re-set up the configuration.  If this is
3748  * called because of a usb_reset_and_verify_device(), then the old alternate
3749  * settings will be re-installed through the normal bandwidth allocation
3750  * functions.
3751  *
3752  * Wait for the Reset Device command to finish.  Remove all structures
3753  * associated with the endpoints that were disabled.  Clear the input device
3754  * structure? Reset the control endpoint 0 max packet size?
3755  *
3756  * If the virt_dev to be reset does not exist or does not match the udev,
3757  * it means the device is lost, possibly due to the xHC restore error and
3758  * re-initialization during S3/S4. In this case, call xhci_alloc_dev() to
3759  * re-allocate the device.
3760  */
3761 static int xhci_discover_or_reset_device(struct usb_hcd *hcd,
3762 		struct usb_device *udev)
3763 {
3764 	int ret, i;
3765 	unsigned long flags;
3766 	struct xhci_hcd *xhci;
3767 	unsigned int slot_id;
3768 	struct xhci_virt_device *virt_dev;
3769 	struct xhci_command *reset_device_cmd;
3770 	struct xhci_slot_ctx *slot_ctx;
3771 	int old_active_eps = 0;
3772 
3773 	ret = xhci_check_args(hcd, udev, NULL, 0, false, __func__);
3774 	if (ret <= 0)
3775 		return ret;
3776 	xhci = hcd_to_xhci(hcd);
3777 	slot_id = udev->slot_id;
3778 	virt_dev = xhci->devs[slot_id];
3779 	if (!virt_dev) {
3780 		xhci_dbg(xhci, "The device to be reset with slot ID %u does "
3781 				"not exist. Re-allocate the device\n", slot_id);
3782 		ret = xhci_alloc_dev(hcd, udev);
3783 		if (ret == 1)
3784 			return 0;
3785 		else
3786 			return -EINVAL;
3787 	}
3788 
3789 	if (virt_dev->tt_info)
3790 		old_active_eps = virt_dev->tt_info->active_eps;
3791 
3792 	if (virt_dev->udev != udev) {
3793 		/* If the virt_dev and the udev does not match, this virt_dev
3794 		 * may belong to another udev.
3795 		 * Re-allocate the device.
3796 		 */
3797 		xhci_dbg(xhci, "The device to be reset with slot ID %u does "
3798 				"not match the udev. Re-allocate the device\n",
3799 				slot_id);
3800 		ret = xhci_alloc_dev(hcd, udev);
3801 		if (ret == 1)
3802 			return 0;
3803 		else
3804 			return -EINVAL;
3805 	}
3806 
3807 	/* If device is not setup, there is no point in resetting it */
3808 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
3809 	if (GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)) ==
3810 						SLOT_STATE_DISABLED)
3811 		return 0;
3812 
3813 	if (xhci->quirks & XHCI_ETRON_HOST) {
3814 		/*
3815 		 * Obtaining a new device slot to inform the xHCI host that
3816 		 * the USB device has been reset.
3817 		 */
3818 		ret = xhci_disable_slot(xhci, udev->slot_id);
3819 		xhci_free_virt_device(xhci, udev->slot_id);
3820 		if (!ret) {
3821 			ret = xhci_alloc_dev(hcd, udev);
3822 			if (ret == 1)
3823 				ret = 0;
3824 			else
3825 				ret = -EINVAL;
3826 		}
3827 		return ret;
3828 	}
3829 
3830 	trace_xhci_discover_or_reset_device(slot_ctx);
3831 
3832 	xhci_dbg(xhci, "Resetting device with slot ID %u\n", slot_id);
3833 	/* Allocate the command structure that holds the struct completion.
3834 	 * Assume we're in process context, since the normal device reset
3835 	 * process has to wait for the device anyway.  Storage devices are
3836 	 * reset as part of error handling, so use GFP_NOIO instead of
3837 	 * GFP_KERNEL.
3838 	 */
3839 	reset_device_cmd = xhci_alloc_command(xhci, true, GFP_NOIO);
3840 	if (!reset_device_cmd) {
3841 		xhci_dbg(xhci, "Couldn't allocate command structure.\n");
3842 		return -ENOMEM;
3843 	}
3844 
3845 	/* Attempt to submit the Reset Device command to the command ring */
3846 	spin_lock_irqsave(&xhci->lock, flags);
3847 
3848 	ret = xhci_queue_reset_device(xhci, reset_device_cmd, slot_id);
3849 	if (ret) {
3850 		xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
3851 		spin_unlock_irqrestore(&xhci->lock, flags);
3852 		goto command_cleanup;
3853 	}
3854 	xhci_ring_cmd_db(xhci);
3855 	spin_unlock_irqrestore(&xhci->lock, flags);
3856 
3857 	/* Wait for the Reset Device command to finish */
3858 	wait_for_completion(reset_device_cmd->completion);
3859 
3860 	/* The Reset Device command can't fail, according to the 0.95/0.96 spec,
3861 	 * unless we tried to reset a slot ID that wasn't enabled,
3862 	 * or the device wasn't in the addressed or configured state.
3863 	 */
3864 	ret = reset_device_cmd->status;
3865 	switch (ret) {
3866 	case COMP_COMMAND_ABORTED:
3867 	case COMP_COMMAND_RING_STOPPED:
3868 		xhci_warn(xhci, "Timeout waiting for reset device command\n");
3869 		ret = -ETIME;
3870 		goto command_cleanup;
3871 	case COMP_SLOT_NOT_ENABLED_ERROR: /* 0.95 completion for bad slot ID */
3872 	case COMP_CONTEXT_STATE_ERROR: /* 0.96 completion code for same thing */
3873 		xhci_dbg(xhci, "Can't reset device (slot ID %u) in %s state\n",
3874 				slot_id,
3875 				xhci_get_slot_state(xhci, virt_dev->out_ctx));
3876 		xhci_dbg(xhci, "Not freeing device rings.\n");
3877 		/* Don't treat this as an error.  May change my mind later. */
3878 		ret = 0;
3879 		goto command_cleanup;
3880 	case COMP_SUCCESS:
3881 		xhci_dbg(xhci, "Successful reset device command.\n");
3882 		break;
3883 	default:
3884 		if (xhci_is_vendor_info_code(xhci, ret))
3885 			break;
3886 		xhci_warn(xhci, "Unknown completion code %u for "
3887 				"reset device command.\n", ret);
3888 		ret = -EINVAL;
3889 		goto command_cleanup;
3890 	}
3891 
3892 	/* Free up host controller endpoint resources */
3893 	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
3894 		spin_lock_irqsave(&xhci->lock, flags);
3895 		/* Don't delete the default control endpoint resources */
3896 		xhci_free_device_endpoint_resources(xhci, virt_dev, false);
3897 		spin_unlock_irqrestore(&xhci->lock, flags);
3898 	}
3899 
3900 	/* Everything but endpoint 0 is disabled, so free the rings. */
3901 	for (i = 1; i < 31; i++) {
3902 		struct xhci_virt_ep *ep = &virt_dev->eps[i];
3903 
3904 		if (ep->ep_state & EP_HAS_STREAMS) {
3905 			xhci_warn(xhci, "WARN: endpoint 0x%02x has streams on device reset, freeing streams.\n",
3906 					xhci_get_endpoint_address(i));
3907 			xhci_free_stream_info(xhci, ep->stream_info);
3908 			ep->stream_info = NULL;
3909 			ep->ep_state &= ~EP_HAS_STREAMS;
3910 		}
3911 
3912 		if (ep->ring) {
3913 			xhci_debugfs_remove_endpoint(xhci, virt_dev, i);
3914 			xhci_free_endpoint_ring(xhci, virt_dev, i);
3915 		}
3916 		if (!list_empty(&virt_dev->eps[i].bw_endpoint_list))
3917 			xhci_drop_ep_from_interval_table(xhci,
3918 					&virt_dev->eps[i].bw_info,
3919 					virt_dev->bw_table,
3920 					udev,
3921 					&virt_dev->eps[i],
3922 					virt_dev->tt_info);
3923 		xhci_clear_endpoint_bw_info(&virt_dev->eps[i].bw_info);
3924 	}
3925 	/* If necessary, update the number of active TTs on this root port */
3926 	xhci_update_tt_active_eps(xhci, virt_dev, old_active_eps);
3927 	virt_dev->flags = 0;
3928 	ret = 0;
3929 
3930 command_cleanup:
3931 	xhci_free_command(xhci, reset_device_cmd);
3932 	return ret;
3933 }
3934 
3935 /*
3936  * At this point, the struct usb_device is about to go away, the device has
3937  * disconnected, and all traffic has been stopped and the endpoints have been
3938  * disabled.  Free any HC data structures associated with that device.
3939  */
3940 static void xhci_free_dev(struct usb_hcd *hcd, struct usb_device *udev)
3941 {
3942 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
3943 	struct xhci_virt_device *virt_dev;
3944 	struct xhci_slot_ctx *slot_ctx;
3945 	unsigned long flags;
3946 	int i, ret;
3947 
3948 	/*
3949 	 * We called pm_runtime_get_noresume when the device was attached.
3950 	 * Decrement the counter here to allow controller to runtime suspend
3951 	 * if no devices remain.
3952 	 */
3953 	if (xhci->quirks & XHCI_RESET_ON_RESUME)
3954 		pm_runtime_put_noidle(hcd->self.controller);
3955 
3956 	ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
3957 	/* If the host is halted due to driver unload, we still need to free the
3958 	 * device.
3959 	 */
3960 	if (ret <= 0 && ret != -ENODEV)
3961 		return;
3962 
3963 	virt_dev = xhci->devs[udev->slot_id];
3964 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
3965 	trace_xhci_free_dev(slot_ctx);
3966 
3967 	/* Stop any wayward timer functions (which may grab the lock) */
3968 	for (i = 0; i < 31; i++)
3969 		virt_dev->eps[i].ep_state &= ~EP_STOP_CMD_PENDING;
3970 	virt_dev->udev = NULL;
3971 	xhci_disable_slot(xhci, udev->slot_id);
3972 
3973 	spin_lock_irqsave(&xhci->lock, flags);
3974 	xhci_free_virt_device(xhci, udev->slot_id);
3975 	spin_unlock_irqrestore(&xhci->lock, flags);
3976 
3977 }
3978 
3979 int xhci_disable_slot(struct xhci_hcd *xhci, u32 slot_id)
3980 {
3981 	struct xhci_command *command;
3982 	unsigned long flags;
3983 	u32 state;
3984 	int ret;
3985 
3986 	command = xhci_alloc_command(xhci, true, GFP_KERNEL);
3987 	if (!command)
3988 		return -ENOMEM;
3989 
3990 	xhci_debugfs_remove_slot(xhci, slot_id);
3991 
3992 	spin_lock_irqsave(&xhci->lock, flags);
3993 	/* Don't disable the slot if the host controller is dead. */
3994 	state = readl(&xhci->op_regs->status);
3995 	if (state == 0xffffffff || (xhci->xhc_state & XHCI_STATE_DYING) ||
3996 			(xhci->xhc_state & XHCI_STATE_HALTED)) {
3997 		spin_unlock_irqrestore(&xhci->lock, flags);
3998 		kfree(command);
3999 		return -ENODEV;
4000 	}
4001 
4002 	ret = xhci_queue_slot_control(xhci, command, TRB_DISABLE_SLOT,
4003 				slot_id);
4004 	if (ret) {
4005 		spin_unlock_irqrestore(&xhci->lock, flags);
4006 		kfree(command);
4007 		return ret;
4008 	}
4009 	xhci_ring_cmd_db(xhci);
4010 	spin_unlock_irqrestore(&xhci->lock, flags);
4011 
4012 	wait_for_completion(command->completion);
4013 
4014 	if (command->status != COMP_SUCCESS)
4015 		xhci_warn(xhci, "Unsuccessful disable slot %u command, status %d\n",
4016 			  slot_id, command->status);
4017 
4018 	xhci_free_command(xhci, command);
4019 
4020 	return 0;
4021 }
4022 
4023 /*
4024  * Checks if we have enough host controller resources for the default control
4025  * endpoint.
4026  *
4027  * Must be called with xhci->lock held.
4028  */
4029 static int xhci_reserve_host_control_ep_resources(struct xhci_hcd *xhci)
4030 {
4031 	if (xhci->num_active_eps + 1 > xhci->limit_active_eps) {
4032 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
4033 				"Not enough ep ctxs: "
4034 				"%u active, need to add 1, limit is %u.",
4035 				xhci->num_active_eps, xhci->limit_active_eps);
4036 		return -ENOMEM;
4037 	}
4038 	xhci->num_active_eps += 1;
4039 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
4040 			"Adding 1 ep ctx, %u now active.",
4041 			xhci->num_active_eps);
4042 	return 0;
4043 }
4044 
4045 
4046 /*
4047  * Returns 0 if the xHC ran out of device slots, the Enable Slot command
4048  * timed out, or allocating memory failed.  Returns 1 on success.
4049  */
4050 int xhci_alloc_dev(struct usb_hcd *hcd, struct usb_device *udev)
4051 {
4052 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
4053 	struct xhci_virt_device *vdev;
4054 	struct xhci_slot_ctx *slot_ctx;
4055 	unsigned long flags;
4056 	int ret, slot_id;
4057 	struct xhci_command *command;
4058 
4059 	command = xhci_alloc_command(xhci, true, GFP_KERNEL);
4060 	if (!command)
4061 		return 0;
4062 
4063 	spin_lock_irqsave(&xhci->lock, flags);
4064 	ret = xhci_queue_slot_control(xhci, command, TRB_ENABLE_SLOT, 0);
4065 	if (ret) {
4066 		spin_unlock_irqrestore(&xhci->lock, flags);
4067 		xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
4068 		xhci_free_command(xhci, command);
4069 		return 0;
4070 	}
4071 	xhci_ring_cmd_db(xhci);
4072 	spin_unlock_irqrestore(&xhci->lock, flags);
4073 
4074 	wait_for_completion(command->completion);
4075 	slot_id = command->slot_id;
4076 
4077 	if (!slot_id || command->status != COMP_SUCCESS) {
4078 		xhci_err(xhci, "Error while assigning device slot ID: %s\n",
4079 			 xhci_trb_comp_code_string(command->status));
4080 		xhci_err(xhci, "Max number of devices this xHCI host supports is %u.\n",
4081 				HCS_MAX_SLOTS(
4082 					readl(&xhci->cap_regs->hcs_params1)));
4083 		xhci_free_command(xhci, command);
4084 		return 0;
4085 	}
4086 
4087 	xhci_free_command(xhci, command);
4088 
4089 	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
4090 		spin_lock_irqsave(&xhci->lock, flags);
4091 		ret = xhci_reserve_host_control_ep_resources(xhci);
4092 		if (ret) {
4093 			spin_unlock_irqrestore(&xhci->lock, flags);
4094 			xhci_warn(xhci, "Not enough host resources, "
4095 					"active endpoint contexts = %u\n",
4096 					xhci->num_active_eps);
4097 			goto disable_slot;
4098 		}
4099 		spin_unlock_irqrestore(&xhci->lock, flags);
4100 	}
4101 	/* Use GFP_NOIO, since this function can be called from
4102 	 * xhci_discover_or_reset_device(), which may be called as part of
4103 	 * mass storage driver error handling.
4104 	 */
4105 	if (!xhci_alloc_virt_device(xhci, slot_id, udev, GFP_NOIO)) {
4106 		xhci_warn(xhci, "Could not allocate xHCI USB device data structures\n");
4107 		goto disable_slot;
4108 	}
4109 	vdev = xhci->devs[slot_id];
4110 	slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);
4111 	trace_xhci_alloc_dev(slot_ctx);
4112 
4113 	udev->slot_id = slot_id;
4114 
4115 	xhci_debugfs_create_slot(xhci, slot_id);
4116 
4117 	/*
4118 	 * If resetting upon resume, we can't put the controller into runtime
4119 	 * suspend if there is a device attached.
4120 	 */
4121 	if (xhci->quirks & XHCI_RESET_ON_RESUME)
4122 		pm_runtime_get_noresume(hcd->self.controller);
4123 
4124 	/* Is this a LS or FS device under a HS hub? */
4125 	/* Hub or peripherial? */
4126 	return 1;
4127 
4128 disable_slot:
4129 	xhci_disable_slot(xhci, udev->slot_id);
4130 	xhci_free_virt_device(xhci, udev->slot_id);
4131 
4132 	return 0;
4133 }
4134 
4135 /**
4136  * xhci_setup_device - issues an Address Device command to assign a unique
4137  *			USB bus address.
4138  * @hcd: USB host controller data structure.
4139  * @udev: USB dev structure representing the connected device.
4140  * @setup: Enum specifying setup mode: address only or with context.
4141  * @timeout_ms: Max wait time (ms) for the command operation to complete.
4142  *
4143  * Return: 0 if successful; otherwise, negative error code.
4144  */
4145 static int xhci_setup_device(struct usb_hcd *hcd, struct usb_device *udev,
4146 			     enum xhci_setup_dev setup, unsigned int timeout_ms)
4147 {
4148 	const char *act = setup == SETUP_CONTEXT_ONLY ? "context" : "address";
4149 	unsigned long flags;
4150 	struct xhci_virt_device *virt_dev;
4151 	int ret = 0;
4152 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
4153 	struct xhci_slot_ctx *slot_ctx;
4154 	struct xhci_input_control_ctx *ctrl_ctx;
4155 	u64 temp_64;
4156 	struct xhci_command *command = NULL;
4157 
4158 	mutex_lock(&xhci->mutex);
4159 
4160 	if (xhci->xhc_state) {	/* dying, removing or halted */
4161 		ret = -ESHUTDOWN;
4162 		goto out;
4163 	}
4164 
4165 	if (!udev->slot_id) {
4166 		xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4167 				"Bad Slot ID %d", udev->slot_id);
4168 		ret = -EINVAL;
4169 		goto out;
4170 	}
4171 
4172 	virt_dev = xhci->devs[udev->slot_id];
4173 
4174 	if (WARN_ON(!virt_dev)) {
4175 		/*
4176 		 * In plug/unplug torture test with an NEC controller,
4177 		 * a zero-dereference was observed once due to virt_dev = 0.
4178 		 * Print useful debug rather than crash if it is observed again!
4179 		 */
4180 		xhci_warn(xhci, "Virt dev invalid for slot_id 0x%x!\n",
4181 			udev->slot_id);
4182 		ret = -EINVAL;
4183 		goto out;
4184 	}
4185 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
4186 	trace_xhci_setup_device_slot(slot_ctx);
4187 
4188 	if (setup == SETUP_CONTEXT_ONLY) {
4189 		if (GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)) ==
4190 		    SLOT_STATE_DEFAULT) {
4191 			xhci_dbg(xhci, "Slot already in default state\n");
4192 			goto out;
4193 		}
4194 	}
4195 
4196 	command = xhci_alloc_command(xhci, true, GFP_KERNEL);
4197 	if (!command) {
4198 		ret = -ENOMEM;
4199 		goto out;
4200 	}
4201 
4202 	command->in_ctx = virt_dev->in_ctx;
4203 	command->timeout_ms = timeout_ms;
4204 
4205 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
4206 	ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
4207 	if (!ctrl_ctx) {
4208 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
4209 				__func__);
4210 		ret = -EINVAL;
4211 		goto out;
4212 	}
4213 	/*
4214 	 * If this is the first Set Address since device plug-in or
4215 	 * virt_device realloaction after a resume with an xHCI power loss,
4216 	 * then set up the slot context.
4217 	 */
4218 	if (!slot_ctx->dev_info)
4219 		xhci_setup_addressable_virt_dev(xhci, udev);
4220 	/* Otherwise, update the control endpoint ring enqueue pointer. */
4221 	else
4222 		xhci_copy_ep0_dequeue_into_input_ctx(xhci, udev);
4223 	ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG | EP0_FLAG);
4224 	ctrl_ctx->drop_flags = 0;
4225 
4226 	trace_xhci_address_ctx(xhci, virt_dev->in_ctx,
4227 				le32_to_cpu(slot_ctx->dev_info) >> 27);
4228 
4229 	trace_xhci_address_ctrl_ctx(ctrl_ctx);
4230 	spin_lock_irqsave(&xhci->lock, flags);
4231 	trace_xhci_setup_device(virt_dev);
4232 	ret = xhci_queue_address_device(xhci, command, virt_dev->in_ctx->dma,
4233 					udev->slot_id, setup);
4234 	if (ret) {
4235 		spin_unlock_irqrestore(&xhci->lock, flags);
4236 		xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4237 				"FIXME: allocate a command ring segment");
4238 		goto out;
4239 	}
4240 	xhci_ring_cmd_db(xhci);
4241 	spin_unlock_irqrestore(&xhci->lock, flags);
4242 
4243 	/* ctrl tx can take up to 5 sec; XXX: need more time for xHC? */
4244 	wait_for_completion(command->completion);
4245 
4246 	/* FIXME: From section 4.3.4: "Software shall be responsible for timing
4247 	 * the SetAddress() "recovery interval" required by USB and aborting the
4248 	 * command on a timeout.
4249 	 */
4250 	switch (command->status) {
4251 	case COMP_COMMAND_ABORTED:
4252 	case COMP_COMMAND_RING_STOPPED:
4253 		xhci_warn(xhci, "Timeout while waiting for setup device command\n");
4254 		ret = -ETIME;
4255 		break;
4256 	case COMP_CONTEXT_STATE_ERROR:
4257 	case COMP_SLOT_NOT_ENABLED_ERROR:
4258 		xhci_err(xhci, "Setup ERROR: setup %s command for slot %d.\n",
4259 			 act, udev->slot_id);
4260 		ret = -EINVAL;
4261 		break;
4262 	case COMP_USB_TRANSACTION_ERROR:
4263 		dev_warn(&udev->dev, "Device not responding to setup %s.\n", act);
4264 
4265 		mutex_unlock(&xhci->mutex);
4266 		ret = xhci_disable_slot(xhci, udev->slot_id);
4267 		xhci_free_virt_device(xhci, udev->slot_id);
4268 		if (!ret) {
4269 			if (xhci_alloc_dev(hcd, udev) == 1)
4270 				xhci_setup_addressable_virt_dev(xhci, udev);
4271 		}
4272 		kfree(command->completion);
4273 		kfree(command);
4274 		return -EPROTO;
4275 	case COMP_INCOMPATIBLE_DEVICE_ERROR:
4276 		dev_warn(&udev->dev,
4277 			 "ERROR: Incompatible device for setup %s command\n", act);
4278 		ret = -ENODEV;
4279 		break;
4280 	case COMP_SUCCESS:
4281 		xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4282 			       "Successful setup %s command", act);
4283 		break;
4284 	default:
4285 		xhci_err(xhci,
4286 			 "ERROR: unexpected setup %s command completion code 0x%x.\n",
4287 			 act, command->status);
4288 		trace_xhci_address_ctx(xhci, virt_dev->out_ctx, 1);
4289 		ret = -EINVAL;
4290 		break;
4291 	}
4292 	if (ret)
4293 		goto out;
4294 	temp_64 = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
4295 	xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4296 			"Op regs DCBAA ptr = %#016llx", temp_64);
4297 	xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4298 		"Slot ID %d dcbaa entry @%p = %#016llx",
4299 		udev->slot_id,
4300 		&xhci->dcbaa->dev_context_ptrs[udev->slot_id],
4301 		(unsigned long long)
4302 		le64_to_cpu(xhci->dcbaa->dev_context_ptrs[udev->slot_id]));
4303 	xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4304 			"Output Context DMA address = %#08llx",
4305 			(unsigned long long)virt_dev->out_ctx->dma);
4306 	trace_xhci_address_ctx(xhci, virt_dev->in_ctx,
4307 				le32_to_cpu(slot_ctx->dev_info) >> 27);
4308 	/*
4309 	 * USB core uses address 1 for the roothubs, so we add one to the
4310 	 * address given back to us by the HC.
4311 	 */
4312 	trace_xhci_address_ctx(xhci, virt_dev->out_ctx,
4313 				le32_to_cpu(slot_ctx->dev_info) >> 27);
4314 	/* Zero the input context control for later use */
4315 	ctrl_ctx->add_flags = 0;
4316 	ctrl_ctx->drop_flags = 0;
4317 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
4318 	udev->devaddr = (u8)(le32_to_cpu(slot_ctx->dev_state) & DEV_ADDR_MASK);
4319 
4320 	xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4321 		       "Internal device address = %d",
4322 		       le32_to_cpu(slot_ctx->dev_state) & DEV_ADDR_MASK);
4323 out:
4324 	mutex_unlock(&xhci->mutex);
4325 	if (command) {
4326 		kfree(command->completion);
4327 		kfree(command);
4328 	}
4329 	return ret;
4330 }
4331 
4332 static int xhci_address_device(struct usb_hcd *hcd, struct usb_device *udev,
4333 			       unsigned int timeout_ms)
4334 {
4335 	return xhci_setup_device(hcd, udev, SETUP_CONTEXT_ADDRESS, timeout_ms);
4336 }
4337 
4338 static int xhci_enable_device(struct usb_hcd *hcd, struct usb_device *udev)
4339 {
4340 	return xhci_setup_device(hcd, udev, SETUP_CONTEXT_ONLY,
4341 				 XHCI_CMD_DEFAULT_TIMEOUT);
4342 }
4343 
4344 /*
4345  * Transfer the port index into real index in the HW port status
4346  * registers. Caculate offset between the port's PORTSC register
4347  * and port status base. Divide the number of per port register
4348  * to get the real index. The raw port number bases 1.
4349  */
4350 int xhci_find_raw_port_number(struct usb_hcd *hcd, int port1)
4351 {
4352 	struct xhci_hub *rhub;
4353 
4354 	rhub = xhci_get_rhub(hcd);
4355 	return rhub->ports[port1 - 1]->hw_portnum + 1;
4356 }
4357 
4358 /*
4359  * Issue an Evaluate Context command to change the Maximum Exit Latency in the
4360  * slot context.  If that succeeds, store the new MEL in the xhci_virt_device.
4361  */
4362 static int __maybe_unused xhci_change_max_exit_latency(struct xhci_hcd *xhci,
4363 			struct usb_device *udev, u16 max_exit_latency)
4364 {
4365 	struct xhci_virt_device *virt_dev;
4366 	struct xhci_command *command;
4367 	struct xhci_input_control_ctx *ctrl_ctx;
4368 	struct xhci_slot_ctx *slot_ctx;
4369 	unsigned long flags;
4370 	int ret;
4371 
4372 	command = xhci_alloc_command_with_ctx(xhci, true, GFP_KERNEL);
4373 	if (!command)
4374 		return -ENOMEM;
4375 
4376 	spin_lock_irqsave(&xhci->lock, flags);
4377 
4378 	virt_dev = xhci->devs[udev->slot_id];
4379 
4380 	/*
4381 	 * virt_dev might not exists yet if xHC resumed from hibernate (S4) and
4382 	 * xHC was re-initialized. Exit latency will be set later after
4383 	 * hub_port_finish_reset() is done and xhci->devs[] are re-allocated
4384 	 */
4385 
4386 	if (!virt_dev || max_exit_latency == virt_dev->current_mel) {
4387 		spin_unlock_irqrestore(&xhci->lock, flags);
4388 		xhci_free_command(xhci, command);
4389 		return 0;
4390 	}
4391 
4392 	/* Attempt to issue an Evaluate Context command to change the MEL. */
4393 	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
4394 	if (!ctrl_ctx) {
4395 		spin_unlock_irqrestore(&xhci->lock, flags);
4396 		xhci_free_command(xhci, command);
4397 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
4398 				__func__);
4399 		return -ENOMEM;
4400 	}
4401 
4402 	xhci_slot_copy(xhci, command->in_ctx, virt_dev->out_ctx);
4403 	spin_unlock_irqrestore(&xhci->lock, flags);
4404 
4405 	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
4406 	slot_ctx = xhci_get_slot_ctx(xhci, command->in_ctx);
4407 	slot_ctx->dev_info2 &= cpu_to_le32(~((u32) MAX_EXIT));
4408 	slot_ctx->dev_info2 |= cpu_to_le32(max_exit_latency);
4409 	slot_ctx->dev_state = 0;
4410 
4411 	xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
4412 			"Set up evaluate context for LPM MEL change.");
4413 
4414 	/* Issue and wait for the evaluate context command. */
4415 	ret = xhci_configure_endpoint(xhci, udev, command,
4416 			true, true);
4417 
4418 	if (!ret) {
4419 		spin_lock_irqsave(&xhci->lock, flags);
4420 		virt_dev->current_mel = max_exit_latency;
4421 		spin_unlock_irqrestore(&xhci->lock, flags);
4422 	}
4423 
4424 	xhci_free_command(xhci, command);
4425 
4426 	return ret;
4427 }
4428 
4429 #ifdef CONFIG_PM
4430 
4431 /* BESL to HIRD Encoding array for USB2 LPM */
4432 static int xhci_besl_encoding[16] = {125, 150, 200, 300, 400, 500, 1000, 2000,
4433 	3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000};
4434 
4435 /* Calculate HIRD/BESL for USB2 PORTPMSC*/
4436 static int xhci_calculate_hird_besl(struct xhci_hcd *xhci,
4437 					struct usb_device *udev)
4438 {
4439 	int u2del, besl, besl_host;
4440 	int besl_device = 0;
4441 	u32 field;
4442 
4443 	u2del = HCS_U2_LATENCY(xhci->hcs_params3);
4444 	field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);
4445 
4446 	if (field & USB_BESL_SUPPORT) {
4447 		for (besl_host = 0; besl_host < 16; besl_host++) {
4448 			if (xhci_besl_encoding[besl_host] >= u2del)
4449 				break;
4450 		}
4451 		/* Use baseline BESL value as default */
4452 		if (field & USB_BESL_BASELINE_VALID)
4453 			besl_device = USB_GET_BESL_BASELINE(field);
4454 		else if (field & USB_BESL_DEEP_VALID)
4455 			besl_device = USB_GET_BESL_DEEP(field);
4456 	} else {
4457 		if (u2del <= 50)
4458 			besl_host = 0;
4459 		else
4460 			besl_host = (u2del - 51) / 75 + 1;
4461 	}
4462 
4463 	besl = besl_host + besl_device;
4464 	if (besl > 15)
4465 		besl = 15;
4466 
4467 	return besl;
4468 }
4469 
4470 /* Calculate BESLD, L1 timeout and HIRDM for USB2 PORTHLPMC */
4471 static int xhci_calculate_usb2_hw_lpm_params(struct usb_device *udev)
4472 {
4473 	u32 field;
4474 	int l1;
4475 	int besld = 0;
4476 	int hirdm = 0;
4477 
4478 	field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);
4479 
4480 	/* xHCI l1 is set in steps of 256us, xHCI 1.0 section 5.4.11.2 */
4481 	l1 = udev->l1_params.timeout / 256;
4482 
4483 	/* device has preferred BESLD */
4484 	if (field & USB_BESL_DEEP_VALID) {
4485 		besld = USB_GET_BESL_DEEP(field);
4486 		hirdm = 1;
4487 	}
4488 
4489 	return PORT_BESLD(besld) | PORT_L1_TIMEOUT(l1) | PORT_HIRDM(hirdm);
4490 }
4491 
4492 static int xhci_set_usb2_hardware_lpm(struct usb_hcd *hcd,
4493 			struct usb_device *udev, int enable)
4494 {
4495 	struct xhci_hcd	*xhci = hcd_to_xhci(hcd);
4496 	struct xhci_port **ports;
4497 	__le32 __iomem	*pm_addr, *hlpm_addr;
4498 	u32		pm_val, hlpm_val, field;
4499 	unsigned int	port_num;
4500 	unsigned long	flags;
4501 	int		hird, exit_latency;
4502 	int		ret;
4503 
4504 	if (xhci->quirks & XHCI_HW_LPM_DISABLE)
4505 		return -EPERM;
4506 
4507 	if (hcd->speed >= HCD_USB3 || !xhci->hw_lpm_support ||
4508 			!udev->lpm_capable)
4509 		return -EPERM;
4510 
4511 	if (!udev->parent || udev->parent->parent ||
4512 			udev->descriptor.bDeviceClass == USB_CLASS_HUB)
4513 		return -EPERM;
4514 
4515 	if (udev->usb2_hw_lpm_capable != 1)
4516 		return -EPERM;
4517 
4518 	spin_lock_irqsave(&xhci->lock, flags);
4519 
4520 	ports = xhci->usb2_rhub.ports;
4521 	port_num = udev->portnum - 1;
4522 	pm_addr = ports[port_num]->addr + PORTPMSC;
4523 	pm_val = readl(pm_addr);
4524 	hlpm_addr = ports[port_num]->addr + PORTHLPMC;
4525 
4526 	xhci_dbg(xhci, "%s port %d USB2 hardware LPM\n",
4527 		 str_enable_disable(enable), port_num + 1);
4528 
4529 	if (enable) {
4530 		/* Host supports BESL timeout instead of HIRD */
4531 		if (udev->usb2_hw_lpm_besl_capable) {
4532 			/* if device doesn't have a preferred BESL value use a
4533 			 * default one which works with mixed HIRD and BESL
4534 			 * systems. See XHCI_DEFAULT_BESL definition in xhci.h
4535 			 */
4536 			field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);
4537 			if ((field & USB_BESL_SUPPORT) &&
4538 			    (field & USB_BESL_BASELINE_VALID))
4539 				hird = USB_GET_BESL_BASELINE(field);
4540 			else
4541 				hird = udev->l1_params.besl;
4542 
4543 			exit_latency = xhci_besl_encoding[hird];
4544 			spin_unlock_irqrestore(&xhci->lock, flags);
4545 
4546 			ret = xhci_change_max_exit_latency(xhci, udev,
4547 							   exit_latency);
4548 			if (ret < 0)
4549 				return ret;
4550 			spin_lock_irqsave(&xhci->lock, flags);
4551 
4552 			hlpm_val = xhci_calculate_usb2_hw_lpm_params(udev);
4553 			writel(hlpm_val, hlpm_addr);
4554 			/* flush write */
4555 			readl(hlpm_addr);
4556 		} else {
4557 			hird = xhci_calculate_hird_besl(xhci, udev);
4558 		}
4559 
4560 		pm_val &= ~PORT_HIRD_MASK;
4561 		pm_val |= PORT_HIRD(hird) | PORT_RWE | PORT_L1DS(udev->slot_id);
4562 		writel(pm_val, pm_addr);
4563 		pm_val = readl(pm_addr);
4564 		pm_val |= PORT_HLE;
4565 		writel(pm_val, pm_addr);
4566 		/* flush write */
4567 		readl(pm_addr);
4568 	} else {
4569 		pm_val &= ~(PORT_HLE | PORT_RWE | PORT_HIRD_MASK | PORT_L1DS_MASK);
4570 		writel(pm_val, pm_addr);
4571 		/* flush write */
4572 		readl(pm_addr);
4573 		if (udev->usb2_hw_lpm_besl_capable) {
4574 			spin_unlock_irqrestore(&xhci->lock, flags);
4575 			xhci_change_max_exit_latency(xhci, udev, 0);
4576 			readl_poll_timeout(ports[port_num]->addr, pm_val,
4577 					   (pm_val & PORT_PLS_MASK) == XDEV_U0,
4578 					   100, 10000);
4579 			return 0;
4580 		}
4581 	}
4582 
4583 	spin_unlock_irqrestore(&xhci->lock, flags);
4584 	return 0;
4585 }
4586 
4587 static int xhci_update_device(struct usb_hcd *hcd, struct usb_device *udev)
4588 {
4589 	struct xhci_hcd	*xhci = hcd_to_xhci(hcd);
4590 	struct xhci_port *port;
4591 	u32 capability;
4592 
4593 	/* Check if USB3 device at root port is tunneled over USB4 */
4594 	if (hcd->speed >= HCD_USB3 && !udev->parent->parent) {
4595 		port = xhci->usb3_rhub.ports[udev->portnum - 1];
4596 
4597 		udev->tunnel_mode = xhci_port_is_tunneled(xhci, port);
4598 		if (udev->tunnel_mode == USB_LINK_UNKNOWN)
4599 			dev_dbg(&udev->dev, "link tunnel state unknown\n");
4600 		else if (udev->tunnel_mode == USB_LINK_TUNNELED)
4601 			dev_dbg(&udev->dev, "tunneled over USB4 link\n");
4602 		else if (udev->tunnel_mode == USB_LINK_NATIVE)
4603 			dev_dbg(&udev->dev, "native USB 3.x link\n");
4604 		return 0;
4605 	}
4606 
4607 	if (hcd->speed >= HCD_USB3 || !udev->lpm_capable || !xhci->hw_lpm_support)
4608 		return 0;
4609 
4610 	/* we only support lpm for non-hub device connected to root hub yet */
4611 	if (!udev->parent || udev->parent->parent ||
4612 			udev->descriptor.bDeviceClass == USB_CLASS_HUB)
4613 		return 0;
4614 
4615 	port = xhci->usb2_rhub.ports[udev->portnum - 1];
4616 	capability = port->port_cap->protocol_caps;
4617 
4618 	if (capability & XHCI_HLC) {
4619 		udev->usb2_hw_lpm_capable = 1;
4620 		udev->l1_params.timeout = XHCI_L1_TIMEOUT;
4621 		udev->l1_params.besl = XHCI_DEFAULT_BESL;
4622 		if (capability & XHCI_BLC)
4623 			udev->usb2_hw_lpm_besl_capable = 1;
4624 	}
4625 
4626 	return 0;
4627 }
4628 
4629 /*---------------------- USB 3.0 Link PM functions ------------------------*/
4630 
4631 /* Service interval in nanoseconds = 2^(bInterval - 1) * 125us * 1000ns / 1us */
4632 static unsigned long long xhci_service_interval_to_ns(
4633 		struct usb_endpoint_descriptor *desc)
4634 {
4635 	return (1ULL << (desc->bInterval - 1)) * 125 * 1000;
4636 }
4637 
4638 static u16 xhci_get_timeout_no_hub_lpm(struct usb_device *udev,
4639 		enum usb3_link_state state)
4640 {
4641 	unsigned long long sel;
4642 	unsigned long long pel;
4643 	unsigned int max_sel_pel;
4644 	char *state_name;
4645 
4646 	switch (state) {
4647 	case USB3_LPM_U1:
4648 		/* Convert SEL and PEL stored in nanoseconds to microseconds */
4649 		sel = DIV_ROUND_UP(udev->u1_params.sel, 1000);
4650 		pel = DIV_ROUND_UP(udev->u1_params.pel, 1000);
4651 		max_sel_pel = USB3_LPM_MAX_U1_SEL_PEL;
4652 		state_name = "U1";
4653 		break;
4654 	case USB3_LPM_U2:
4655 		sel = DIV_ROUND_UP(udev->u2_params.sel, 1000);
4656 		pel = DIV_ROUND_UP(udev->u2_params.pel, 1000);
4657 		max_sel_pel = USB3_LPM_MAX_U2_SEL_PEL;
4658 		state_name = "U2";
4659 		break;
4660 	default:
4661 		dev_warn(&udev->dev, "%s: Can't get timeout for non-U1 or U2 state.\n",
4662 				__func__);
4663 		return USB3_LPM_DISABLED;
4664 	}
4665 
4666 	if (sel <= max_sel_pel && pel <= max_sel_pel)
4667 		return USB3_LPM_DEVICE_INITIATED;
4668 
4669 	if (sel > max_sel_pel)
4670 		dev_dbg(&udev->dev, "Device-initiated %s disabled "
4671 				"due to long SEL %llu ms\n",
4672 				state_name, sel);
4673 	else
4674 		dev_dbg(&udev->dev, "Device-initiated %s disabled "
4675 				"due to long PEL %llu ms\n",
4676 				state_name, pel);
4677 	return USB3_LPM_DISABLED;
4678 }
4679 
4680 /* The U1 timeout should be the maximum of the following values:
4681  *  - For control endpoints, U1 system exit latency (SEL) * 3
4682  *  - For bulk endpoints, U1 SEL * 5
4683  *  - For interrupt endpoints:
4684  *    - Notification EPs, U1 SEL * 3
4685  *    - Periodic EPs, max(105% of bInterval, U1 SEL * 2)
4686  *  - For isochronous endpoints, max(105% of bInterval, U1 SEL * 2)
4687  */
4688 static unsigned long long xhci_calculate_intel_u1_timeout(
4689 		struct usb_device *udev,
4690 		struct usb_endpoint_descriptor *desc)
4691 {
4692 	unsigned long long timeout_ns;
4693 	int ep_type;
4694 	int intr_type;
4695 
4696 	ep_type = usb_endpoint_type(desc);
4697 	switch (ep_type) {
4698 	case USB_ENDPOINT_XFER_CONTROL:
4699 		timeout_ns = udev->u1_params.sel * 3;
4700 		break;
4701 	case USB_ENDPOINT_XFER_BULK:
4702 		timeout_ns = udev->u1_params.sel * 5;
4703 		break;
4704 	case USB_ENDPOINT_XFER_INT:
4705 		intr_type = usb_endpoint_interrupt_type(desc);
4706 		if (intr_type == USB_ENDPOINT_INTR_NOTIFICATION) {
4707 			timeout_ns = udev->u1_params.sel * 3;
4708 			break;
4709 		}
4710 		/* Otherwise the calculation is the same as isoc eps */
4711 		fallthrough;
4712 	case USB_ENDPOINT_XFER_ISOC:
4713 		timeout_ns = xhci_service_interval_to_ns(desc);
4714 		timeout_ns = DIV_ROUND_UP_ULL(timeout_ns * 105, 100);
4715 		if (timeout_ns < udev->u1_params.sel * 2)
4716 			timeout_ns = udev->u1_params.sel * 2;
4717 		break;
4718 	default:
4719 		return 0;
4720 	}
4721 
4722 	return timeout_ns;
4723 }
4724 
4725 /* Returns the hub-encoded U1 timeout value. */
4726 static u16 xhci_calculate_u1_timeout(struct xhci_hcd *xhci,
4727 		struct usb_device *udev,
4728 		struct usb_endpoint_descriptor *desc)
4729 {
4730 	unsigned long long timeout_ns;
4731 
4732 	/* Prevent U1 if service interval is shorter than U1 exit latency */
4733 	if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) {
4734 		if (xhci_service_interval_to_ns(desc) <= udev->u1_params.mel) {
4735 			dev_dbg(&udev->dev, "Disable U1, ESIT shorter than exit latency\n");
4736 			return USB3_LPM_DISABLED;
4737 		}
4738 	}
4739 
4740 	if (xhci->quirks & (XHCI_INTEL_HOST | XHCI_ZHAOXIN_HOST))
4741 		timeout_ns = xhci_calculate_intel_u1_timeout(udev, desc);
4742 	else
4743 		timeout_ns = udev->u1_params.sel;
4744 
4745 	/* The U1 timeout is encoded in 1us intervals.
4746 	 * Don't return a timeout of zero, because that's USB3_LPM_DISABLED.
4747 	 */
4748 	if (timeout_ns == USB3_LPM_DISABLED)
4749 		timeout_ns = 1;
4750 	else
4751 		timeout_ns = DIV_ROUND_UP_ULL(timeout_ns, 1000);
4752 
4753 	/* If the necessary timeout value is bigger than what we can set in the
4754 	 * USB 3.0 hub, we have to disable hub-initiated U1.
4755 	 */
4756 	if (timeout_ns <= USB3_LPM_U1_MAX_TIMEOUT)
4757 		return timeout_ns;
4758 	dev_dbg(&udev->dev, "Hub-initiated U1 disabled "
4759 			"due to long timeout %llu ms\n", timeout_ns);
4760 	return xhci_get_timeout_no_hub_lpm(udev, USB3_LPM_U1);
4761 }
4762 
4763 /* The U2 timeout should be the maximum of:
4764  *  - 10 ms (to avoid the bandwidth impact on the scheduler)
4765  *  - largest bInterval of any active periodic endpoint (to avoid going
4766  *    into lower power link states between intervals).
4767  *  - the U2 Exit Latency of the device
4768  */
4769 static unsigned long long xhci_calculate_intel_u2_timeout(
4770 		struct usb_device *udev,
4771 		struct usb_endpoint_descriptor *desc)
4772 {
4773 	unsigned long long timeout_ns;
4774 	unsigned long long u2_del_ns;
4775 
4776 	timeout_ns = 10 * 1000 * 1000;
4777 
4778 	if ((usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) &&
4779 			(xhci_service_interval_to_ns(desc) > timeout_ns))
4780 		timeout_ns = xhci_service_interval_to_ns(desc);
4781 
4782 	u2_del_ns = le16_to_cpu(udev->bos->ss_cap->bU2DevExitLat) * 1000ULL;
4783 	if (u2_del_ns > timeout_ns)
4784 		timeout_ns = u2_del_ns;
4785 
4786 	return timeout_ns;
4787 }
4788 
4789 /* Returns the hub-encoded U2 timeout value. */
4790 static u16 xhci_calculate_u2_timeout(struct xhci_hcd *xhci,
4791 		struct usb_device *udev,
4792 		struct usb_endpoint_descriptor *desc)
4793 {
4794 	unsigned long long timeout_ns;
4795 
4796 	/* Prevent U2 if service interval is shorter than U2 exit latency */
4797 	if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) {
4798 		if (xhci_service_interval_to_ns(desc) <= udev->u2_params.mel) {
4799 			dev_dbg(&udev->dev, "Disable U2, ESIT shorter than exit latency\n");
4800 			return USB3_LPM_DISABLED;
4801 		}
4802 	}
4803 
4804 	if (xhci->quirks & (XHCI_INTEL_HOST | XHCI_ZHAOXIN_HOST))
4805 		timeout_ns = xhci_calculate_intel_u2_timeout(udev, desc);
4806 	else
4807 		timeout_ns = udev->u2_params.sel;
4808 
4809 	/* The U2 timeout is encoded in 256us intervals */
4810 	timeout_ns = DIV_ROUND_UP_ULL(timeout_ns, 256 * 1000);
4811 	/* If the necessary timeout value is bigger than what we can set in the
4812 	 * USB 3.0 hub, we have to disable hub-initiated U2.
4813 	 */
4814 	if (timeout_ns <= USB3_LPM_U2_MAX_TIMEOUT)
4815 		return timeout_ns;
4816 	dev_dbg(&udev->dev, "Hub-initiated U2 disabled "
4817 			"due to long timeout %llu ms\n", timeout_ns);
4818 	return xhci_get_timeout_no_hub_lpm(udev, USB3_LPM_U2);
4819 }
4820 
4821 static u16 xhci_call_host_update_timeout_for_endpoint(struct xhci_hcd *xhci,
4822 		struct usb_device *udev,
4823 		struct usb_endpoint_descriptor *desc,
4824 		enum usb3_link_state state,
4825 		u16 *timeout)
4826 {
4827 	if (state == USB3_LPM_U1)
4828 		return xhci_calculate_u1_timeout(xhci, udev, desc);
4829 	else if (state == USB3_LPM_U2)
4830 		return xhci_calculate_u2_timeout(xhci, udev, desc);
4831 
4832 	return USB3_LPM_DISABLED;
4833 }
4834 
4835 static int xhci_update_timeout_for_endpoint(struct xhci_hcd *xhci,
4836 		struct usb_device *udev,
4837 		struct usb_endpoint_descriptor *desc,
4838 		enum usb3_link_state state,
4839 		u16 *timeout)
4840 {
4841 	u16 alt_timeout;
4842 
4843 	alt_timeout = xhci_call_host_update_timeout_for_endpoint(xhci, udev,
4844 		desc, state, timeout);
4845 
4846 	/* If we found we can't enable hub-initiated LPM, and
4847 	 * the U1 or U2 exit latency was too high to allow
4848 	 * device-initiated LPM as well, then we will disable LPM
4849 	 * for this device, so stop searching any further.
4850 	 */
4851 	if (alt_timeout == USB3_LPM_DISABLED) {
4852 		*timeout = alt_timeout;
4853 		return -E2BIG;
4854 	}
4855 	if (alt_timeout > *timeout)
4856 		*timeout = alt_timeout;
4857 	return 0;
4858 }
4859 
4860 static int xhci_update_timeout_for_interface(struct xhci_hcd *xhci,
4861 		struct usb_device *udev,
4862 		struct usb_host_interface *alt,
4863 		enum usb3_link_state state,
4864 		u16 *timeout)
4865 {
4866 	int j;
4867 
4868 	for (j = 0; j < alt->desc.bNumEndpoints; j++) {
4869 		if (xhci_update_timeout_for_endpoint(xhci, udev,
4870 					&alt->endpoint[j].desc, state, timeout))
4871 			return -E2BIG;
4872 	}
4873 	return 0;
4874 }
4875 
4876 static int xhci_check_tier_policy(struct xhci_hcd *xhci,
4877 		struct usb_device *udev,
4878 		enum usb3_link_state state)
4879 {
4880 	struct usb_device *parent = udev->parent;
4881 	int tier = 1; /* roothub is tier1 */
4882 
4883 	while (parent) {
4884 		parent = parent->parent;
4885 		tier++;
4886 	}
4887 
4888 	if (xhci->quirks & XHCI_INTEL_HOST && tier > 3)
4889 		goto fail;
4890 	if (xhci->quirks & XHCI_ZHAOXIN_HOST && tier > 2)
4891 		goto fail;
4892 
4893 	return 0;
4894 fail:
4895 	dev_dbg(&udev->dev, "Tier policy prevents U1/U2 LPM states for devices at tier %d\n",
4896 			tier);
4897 	return -E2BIG;
4898 }
4899 
4900 /* Returns the U1 or U2 timeout that should be enabled.
4901  * If the tier check or timeout setting functions return with a non-zero exit
4902  * code, that means the timeout value has been finalized and we shouldn't look
4903  * at any more endpoints.
4904  */
4905 static u16 xhci_calculate_lpm_timeout(struct usb_hcd *hcd,
4906 			struct usb_device *udev, enum usb3_link_state state)
4907 {
4908 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
4909 	struct usb_host_config *config;
4910 	char *state_name;
4911 	int i;
4912 	u16 timeout = USB3_LPM_DISABLED;
4913 
4914 	if (state == USB3_LPM_U1)
4915 		state_name = "U1";
4916 	else if (state == USB3_LPM_U2)
4917 		state_name = "U2";
4918 	else {
4919 		dev_warn(&udev->dev, "Can't enable unknown link state %i\n",
4920 				state);
4921 		return timeout;
4922 	}
4923 
4924 	/* Gather some information about the currently installed configuration
4925 	 * and alternate interface settings.
4926 	 */
4927 	if (xhci_update_timeout_for_endpoint(xhci, udev, &udev->ep0.desc,
4928 			state, &timeout))
4929 		return timeout;
4930 
4931 	config = udev->actconfig;
4932 	if (!config)
4933 		return timeout;
4934 
4935 	for (i = 0; i < config->desc.bNumInterfaces; i++) {
4936 		struct usb_driver *driver;
4937 		struct usb_interface *intf = config->interface[i];
4938 
4939 		if (!intf)
4940 			continue;
4941 
4942 		/* Check if any currently bound drivers want hub-initiated LPM
4943 		 * disabled.
4944 		 */
4945 		if (intf->dev.driver) {
4946 			driver = to_usb_driver(intf->dev.driver);
4947 			if (driver && driver->disable_hub_initiated_lpm) {
4948 				dev_dbg(&udev->dev, "Hub-initiated %s disabled at request of driver %s\n",
4949 					state_name, driver->name);
4950 				timeout = xhci_get_timeout_no_hub_lpm(udev,
4951 								      state);
4952 				if (timeout == USB3_LPM_DISABLED)
4953 					return timeout;
4954 			}
4955 		}
4956 
4957 		/* Not sure how this could happen... */
4958 		if (!intf->cur_altsetting)
4959 			continue;
4960 
4961 		if (xhci_update_timeout_for_interface(xhci, udev,
4962 					intf->cur_altsetting,
4963 					state, &timeout))
4964 			return timeout;
4965 	}
4966 	return timeout;
4967 }
4968 
4969 static int calculate_max_exit_latency(struct usb_device *udev,
4970 		enum usb3_link_state state_changed,
4971 		u16 hub_encoded_timeout)
4972 {
4973 	unsigned long long u1_mel_us = 0;
4974 	unsigned long long u2_mel_us = 0;
4975 	unsigned long long mel_us = 0;
4976 	bool disabling_u1;
4977 	bool disabling_u2;
4978 	bool enabling_u1;
4979 	bool enabling_u2;
4980 
4981 	disabling_u1 = (state_changed == USB3_LPM_U1 &&
4982 			hub_encoded_timeout == USB3_LPM_DISABLED);
4983 	disabling_u2 = (state_changed == USB3_LPM_U2 &&
4984 			hub_encoded_timeout == USB3_LPM_DISABLED);
4985 
4986 	enabling_u1 = (state_changed == USB3_LPM_U1 &&
4987 			hub_encoded_timeout != USB3_LPM_DISABLED);
4988 	enabling_u2 = (state_changed == USB3_LPM_U2 &&
4989 			hub_encoded_timeout != USB3_LPM_DISABLED);
4990 
4991 	/* If U1 was already enabled and we're not disabling it,
4992 	 * or we're going to enable U1, account for the U1 max exit latency.
4993 	 */
4994 	if ((udev->u1_params.timeout != USB3_LPM_DISABLED && !disabling_u1) ||
4995 			enabling_u1)
4996 		u1_mel_us = DIV_ROUND_UP(udev->u1_params.mel, 1000);
4997 	if ((udev->u2_params.timeout != USB3_LPM_DISABLED && !disabling_u2) ||
4998 			enabling_u2)
4999 		u2_mel_us = DIV_ROUND_UP(udev->u2_params.mel, 1000);
5000 
5001 	mel_us = max(u1_mel_us, u2_mel_us);
5002 
5003 	/* xHCI host controller max exit latency field is only 16 bits wide. */
5004 	if (mel_us > MAX_EXIT) {
5005 		dev_warn(&udev->dev, "Link PM max exit latency of %lluus "
5006 				"is too big.\n", mel_us);
5007 		return -E2BIG;
5008 	}
5009 	return mel_us;
5010 }
5011 
5012 /* Returns the USB3 hub-encoded value for the U1/U2 timeout. */
5013 static int xhci_enable_usb3_lpm_timeout(struct usb_hcd *hcd,
5014 			struct usb_device *udev, enum usb3_link_state state)
5015 {
5016 	struct xhci_hcd	*xhci;
5017 	struct xhci_port *port;
5018 	u16 hub_encoded_timeout;
5019 	int mel;
5020 	int ret;
5021 
5022 	xhci = hcd_to_xhci(hcd);
5023 	/* The LPM timeout values are pretty host-controller specific, so don't
5024 	 * enable hub-initiated timeouts unless the vendor has provided
5025 	 * information about their timeout algorithm.
5026 	 */
5027 	if (!xhci || !(xhci->quirks & XHCI_LPM_SUPPORT) ||
5028 			!xhci->devs[udev->slot_id])
5029 		return USB3_LPM_DISABLED;
5030 
5031 	if (xhci_check_tier_policy(xhci, udev, state) < 0)
5032 		return USB3_LPM_DISABLED;
5033 
5034 	/* If connected to root port then check port can handle lpm */
5035 	if (udev->parent && !udev->parent->parent) {
5036 		port = xhci->usb3_rhub.ports[udev->portnum - 1];
5037 		if (port->lpm_incapable)
5038 			return USB3_LPM_DISABLED;
5039 	}
5040 
5041 	hub_encoded_timeout = xhci_calculate_lpm_timeout(hcd, udev, state);
5042 	mel = calculate_max_exit_latency(udev, state, hub_encoded_timeout);
5043 	if (mel < 0) {
5044 		/* Max Exit Latency is too big, disable LPM. */
5045 		hub_encoded_timeout = USB3_LPM_DISABLED;
5046 		mel = 0;
5047 	}
5048 
5049 	ret = xhci_change_max_exit_latency(xhci, udev, mel);
5050 	if (ret)
5051 		return ret;
5052 	return hub_encoded_timeout;
5053 }
5054 
5055 static int xhci_disable_usb3_lpm_timeout(struct usb_hcd *hcd,
5056 			struct usb_device *udev, enum usb3_link_state state)
5057 {
5058 	struct xhci_hcd	*xhci;
5059 	u16 mel;
5060 
5061 	xhci = hcd_to_xhci(hcd);
5062 	if (!xhci || !(xhci->quirks & XHCI_LPM_SUPPORT) ||
5063 			!xhci->devs[udev->slot_id])
5064 		return 0;
5065 
5066 	mel = calculate_max_exit_latency(udev, state, USB3_LPM_DISABLED);
5067 	return xhci_change_max_exit_latency(xhci, udev, mel);
5068 }
5069 #else /* CONFIG_PM */
5070 
5071 static int xhci_set_usb2_hardware_lpm(struct usb_hcd *hcd,
5072 				struct usb_device *udev, int enable)
5073 {
5074 	return 0;
5075 }
5076 
5077 static int xhci_update_device(struct usb_hcd *hcd, struct usb_device *udev)
5078 {
5079 	return 0;
5080 }
5081 
5082 static int xhci_enable_usb3_lpm_timeout(struct usb_hcd *hcd,
5083 			struct usb_device *udev, enum usb3_link_state state)
5084 {
5085 	return USB3_LPM_DISABLED;
5086 }
5087 
5088 static int xhci_disable_usb3_lpm_timeout(struct usb_hcd *hcd,
5089 			struct usb_device *udev, enum usb3_link_state state)
5090 {
5091 	return 0;
5092 }
5093 #endif	/* CONFIG_PM */
5094 
5095 /*-------------------------------------------------------------------------*/
5096 
5097 /* Once a hub descriptor is fetched for a device, we need to update the xHC's
5098  * internal data structures for the device.
5099  */
5100 int xhci_update_hub_device(struct usb_hcd *hcd, struct usb_device *hdev,
5101 			struct usb_tt *tt, gfp_t mem_flags)
5102 {
5103 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
5104 	struct xhci_virt_device *vdev;
5105 	struct xhci_command *config_cmd;
5106 	struct xhci_input_control_ctx *ctrl_ctx;
5107 	struct xhci_slot_ctx *slot_ctx;
5108 	unsigned long flags;
5109 	unsigned think_time;
5110 	int ret;
5111 
5112 	/* Ignore root hubs */
5113 	if (!hdev->parent)
5114 		return 0;
5115 
5116 	vdev = xhci->devs[hdev->slot_id];
5117 	if (!vdev) {
5118 		xhci_warn(xhci, "Cannot update hub desc for unknown device.\n");
5119 		return -EINVAL;
5120 	}
5121 
5122 	config_cmd = xhci_alloc_command_with_ctx(xhci, true, mem_flags);
5123 	if (!config_cmd)
5124 		return -ENOMEM;
5125 
5126 	ctrl_ctx = xhci_get_input_control_ctx(config_cmd->in_ctx);
5127 	if (!ctrl_ctx) {
5128 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
5129 				__func__);
5130 		xhci_free_command(xhci, config_cmd);
5131 		return -ENOMEM;
5132 	}
5133 
5134 	spin_lock_irqsave(&xhci->lock, flags);
5135 	if (hdev->speed == USB_SPEED_HIGH &&
5136 			xhci_alloc_tt_info(xhci, vdev, hdev, tt, GFP_ATOMIC)) {
5137 		xhci_dbg(xhci, "Could not allocate xHCI TT structure.\n");
5138 		xhci_free_command(xhci, config_cmd);
5139 		spin_unlock_irqrestore(&xhci->lock, flags);
5140 		return -ENOMEM;
5141 	}
5142 
5143 	xhci_slot_copy(xhci, config_cmd->in_ctx, vdev->out_ctx);
5144 	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
5145 	slot_ctx = xhci_get_slot_ctx(xhci, config_cmd->in_ctx);
5146 	slot_ctx->dev_info |= cpu_to_le32(DEV_HUB);
5147 	/*
5148 	 * refer to section 6.2.2: MTT should be 0 for full speed hub,
5149 	 * but it may be already set to 1 when setup an xHCI virtual
5150 	 * device, so clear it anyway.
5151 	 */
5152 	if (tt->multi)
5153 		slot_ctx->dev_info |= cpu_to_le32(DEV_MTT);
5154 	else if (hdev->speed == USB_SPEED_FULL)
5155 		slot_ctx->dev_info &= cpu_to_le32(~DEV_MTT);
5156 
5157 	if (xhci->hci_version > 0x95) {
5158 		xhci_dbg(xhci, "xHCI version %x needs hub "
5159 				"TT think time and number of ports\n",
5160 				(unsigned int) xhci->hci_version);
5161 		slot_ctx->dev_info2 |= cpu_to_le32(XHCI_MAX_PORTS(hdev->maxchild));
5162 		/* Set TT think time - convert from ns to FS bit times.
5163 		 * 0 = 8 FS bit times, 1 = 16 FS bit times,
5164 		 * 2 = 24 FS bit times, 3 = 32 FS bit times.
5165 		 *
5166 		 * xHCI 1.0: this field shall be 0 if the device is not a
5167 		 * High-spped hub.
5168 		 */
5169 		think_time = tt->think_time;
5170 		if (think_time != 0)
5171 			think_time = (think_time / 666) - 1;
5172 		if (xhci->hci_version < 0x100 || hdev->speed == USB_SPEED_HIGH)
5173 			slot_ctx->tt_info |=
5174 				cpu_to_le32(TT_THINK_TIME(think_time));
5175 	} else {
5176 		xhci_dbg(xhci, "xHCI version %x doesn't need hub "
5177 				"TT think time or number of ports\n",
5178 				(unsigned int) xhci->hci_version);
5179 	}
5180 	slot_ctx->dev_state = 0;
5181 	spin_unlock_irqrestore(&xhci->lock, flags);
5182 
5183 	xhci_dbg(xhci, "Set up %s for hub device.\n",
5184 			(xhci->hci_version > 0x95) ?
5185 			"configure endpoint" : "evaluate context");
5186 
5187 	/* Issue and wait for the configure endpoint or
5188 	 * evaluate context command.
5189 	 */
5190 	if (xhci->hci_version > 0x95)
5191 		ret = xhci_configure_endpoint(xhci, hdev, config_cmd,
5192 				false, false);
5193 	else
5194 		ret = xhci_configure_endpoint(xhci, hdev, config_cmd,
5195 				true, false);
5196 
5197 	xhci_free_command(xhci, config_cmd);
5198 	return ret;
5199 }
5200 EXPORT_SYMBOL_GPL(xhci_update_hub_device);
5201 
5202 static int xhci_get_frame(struct usb_hcd *hcd)
5203 {
5204 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
5205 	/* EHCI mods by the periodic size.  Why? */
5206 	return readl(&xhci->run_regs->microframe_index) >> 3;
5207 }
5208 
5209 static void xhci_hcd_init_usb2_data(struct xhci_hcd *xhci, struct usb_hcd *hcd)
5210 {
5211 	xhci->usb2_rhub.hcd = hcd;
5212 	hcd->speed = HCD_USB2;
5213 	hcd->self.root_hub->speed = USB_SPEED_HIGH;
5214 	/*
5215 	 * USB 2.0 roothub under xHCI has an integrated TT,
5216 	 * (rate matching hub) as opposed to having an OHCI/UHCI
5217 	 * companion controller.
5218 	 */
5219 	hcd->has_tt = 1;
5220 }
5221 
5222 static void xhci_hcd_init_usb3_data(struct xhci_hcd *xhci, struct usb_hcd *hcd)
5223 {
5224 	unsigned int minor_rev;
5225 
5226 	/*
5227 	 * Early xHCI 1.1 spec did not mention USB 3.1 capable hosts
5228 	 * should return 0x31 for sbrn, or that the minor revision
5229 	 * is a two digit BCD containig minor and sub-minor numbers.
5230 	 * This was later clarified in xHCI 1.2.
5231 	 *
5232 	 * Some USB 3.1 capable hosts therefore have sbrn 0x30, and
5233 	 * minor revision set to 0x1 instead of 0x10.
5234 	 */
5235 	if (xhci->usb3_rhub.min_rev == 0x1)
5236 		minor_rev = 1;
5237 	else
5238 		minor_rev = xhci->usb3_rhub.min_rev / 0x10;
5239 
5240 	switch (minor_rev) {
5241 	case 2:
5242 		hcd->speed = HCD_USB32;
5243 		hcd->self.root_hub->speed = USB_SPEED_SUPER_PLUS;
5244 		hcd->self.root_hub->rx_lanes = 2;
5245 		hcd->self.root_hub->tx_lanes = 2;
5246 		hcd->self.root_hub->ssp_rate = USB_SSP_GEN_2x2;
5247 		break;
5248 	case 1:
5249 		hcd->speed = HCD_USB31;
5250 		hcd->self.root_hub->speed = USB_SPEED_SUPER_PLUS;
5251 		hcd->self.root_hub->ssp_rate = USB_SSP_GEN_2x1;
5252 		break;
5253 	}
5254 	xhci_info(xhci, "Host supports USB 3.%x %sSuperSpeed\n",
5255 		  minor_rev, minor_rev ? "Enhanced " : "");
5256 
5257 	xhci->usb3_rhub.hcd = hcd;
5258 }
5259 
5260 int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks)
5261 {
5262 	struct xhci_hcd		*xhci;
5263 	/*
5264 	 * TODO: Check with DWC3 clients for sysdev according to
5265 	 * quirks
5266 	 */
5267 	struct device		*dev = hcd->self.sysdev;
5268 	int			retval;
5269 
5270 	/* Accept arbitrarily long scatter-gather lists */
5271 	hcd->self.sg_tablesize = ~0;
5272 
5273 	/* support to build packet from discontinuous buffers */
5274 	hcd->self.no_sg_constraint = 1;
5275 
5276 	/* XHCI controllers don't stop the ep queue on short packets :| */
5277 	hcd->self.no_stop_on_short = 1;
5278 
5279 	xhci = hcd_to_xhci(hcd);
5280 
5281 	if (!usb_hcd_is_primary_hcd(hcd)) {
5282 		xhci_hcd_init_usb3_data(xhci, hcd);
5283 		return 0;
5284 	}
5285 
5286 	mutex_init(&xhci->mutex);
5287 	xhci->main_hcd = hcd;
5288 	xhci->cap_regs = hcd->regs;
5289 	xhci->op_regs = hcd->regs +
5290 		HC_LENGTH(readl(&xhci->cap_regs->hc_capbase));
5291 	xhci->run_regs = hcd->regs +
5292 		(readl(&xhci->cap_regs->run_regs_off) & RTSOFF_MASK);
5293 	/* Cache read-only capability registers */
5294 	xhci->hcs_params1 = readl(&xhci->cap_regs->hcs_params1);
5295 	xhci->hcs_params2 = readl(&xhci->cap_regs->hcs_params2);
5296 	xhci->hcs_params3 = readl(&xhci->cap_regs->hcs_params3);
5297 	xhci->hci_version = HC_VERSION(readl(&xhci->cap_regs->hc_capbase));
5298 	xhci->hcc_params = readl(&xhci->cap_regs->hcc_params);
5299 	if (xhci->hci_version > 0x100)
5300 		xhci->hcc_params2 = readl(&xhci->cap_regs->hcc_params2);
5301 
5302 	/* xhci-plat or xhci-pci might have set max_interrupters already */
5303 	if ((!xhci->max_interrupters) ||
5304 	    xhci->max_interrupters > HCS_MAX_INTRS(xhci->hcs_params1))
5305 		xhci->max_interrupters = HCS_MAX_INTRS(xhci->hcs_params1);
5306 
5307 	xhci->quirks |= quirks;
5308 
5309 	if (get_quirks)
5310 		get_quirks(dev, xhci);
5311 
5312 	/* In xhci controllers which follow xhci 1.0 spec gives a spurious
5313 	 * success event after a short transfer. This quirk will ignore such
5314 	 * spurious event.
5315 	 */
5316 	if (xhci->hci_version > 0x96)
5317 		xhci->quirks |= XHCI_SPURIOUS_SUCCESS;
5318 
5319 	if (xhci->hci_version == 0x95 && link_quirk) {
5320 		xhci_dbg(xhci, "QUIRK: Not clearing Link TRB chain bits");
5321 		xhci->quirks |= XHCI_LINK_TRB_QUIRK;
5322 	}
5323 
5324 	/* Make sure the HC is halted. */
5325 	retval = xhci_halt(xhci);
5326 	if (retval)
5327 		return retval;
5328 
5329 	xhci_zero_64b_regs(xhci);
5330 
5331 	xhci_dbg(xhci, "Resetting HCD\n");
5332 	/* Reset the internal HC memory state and registers. */
5333 	retval = xhci_reset(xhci, XHCI_RESET_LONG_USEC);
5334 	if (retval)
5335 		return retval;
5336 	xhci_dbg(xhci, "Reset complete\n");
5337 
5338 	/*
5339 	 * On some xHCI controllers (e.g. R-Car SoCs), the AC64 bit (bit 0)
5340 	 * of HCCPARAMS1 is set to 1. However, the xHCs don't support 64-bit
5341 	 * address memory pointers actually. So, this driver clears the AC64
5342 	 * bit of xhci->hcc_params to call dma_set_coherent_mask(dev,
5343 	 * DMA_BIT_MASK(32)) in this xhci_gen_setup().
5344 	 */
5345 	if (xhci->quirks & XHCI_NO_64BIT_SUPPORT)
5346 		xhci->hcc_params &= ~BIT(0);
5347 
5348 	/* Set dma_mask and coherent_dma_mask to 64-bits,
5349 	 * if xHC supports 64-bit addressing */
5350 	if (HCC_64BIT_ADDR(xhci->hcc_params) &&
5351 			!dma_set_mask(dev, DMA_BIT_MASK(64))) {
5352 		xhci_dbg(xhci, "Enabling 64-bit DMA addresses.\n");
5353 		dma_set_coherent_mask(dev, DMA_BIT_MASK(64));
5354 	} else {
5355 		/*
5356 		 * This is to avoid error in cases where a 32-bit USB
5357 		 * controller is used on a 64-bit capable system.
5358 		 */
5359 		retval = dma_set_mask(dev, DMA_BIT_MASK(32));
5360 		if (retval)
5361 			return retval;
5362 		xhci_dbg(xhci, "Enabling 32-bit DMA addresses.\n");
5363 		dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
5364 	}
5365 
5366 	xhci_dbg(xhci, "Calling HCD init\n");
5367 	/* Initialize HCD and host controller data structures. */
5368 	retval = xhci_init(hcd);
5369 	if (retval)
5370 		return retval;
5371 	xhci_dbg(xhci, "Called HCD init\n");
5372 
5373 	if (xhci_hcd_is_usb3(hcd))
5374 		xhci_hcd_init_usb3_data(xhci, hcd);
5375 	else
5376 		xhci_hcd_init_usb2_data(xhci, hcd);
5377 
5378 	xhci_info(xhci, "hcc params 0x%08x hci version 0x%x quirks 0x%016llx\n",
5379 		  xhci->hcc_params, xhci->hci_version, xhci->quirks);
5380 
5381 	return 0;
5382 }
5383 EXPORT_SYMBOL_GPL(xhci_gen_setup);
5384 
5385 static void xhci_clear_tt_buffer_complete(struct usb_hcd *hcd,
5386 		struct usb_host_endpoint *ep)
5387 {
5388 	struct xhci_hcd *xhci;
5389 	struct usb_device *udev;
5390 	unsigned int slot_id;
5391 	unsigned int ep_index;
5392 	unsigned long flags;
5393 
5394 	xhci = hcd_to_xhci(hcd);
5395 
5396 	spin_lock_irqsave(&xhci->lock, flags);
5397 	udev = (struct usb_device *)ep->hcpriv;
5398 	slot_id = udev->slot_id;
5399 	ep_index = xhci_get_endpoint_index(&ep->desc);
5400 
5401 	xhci->devs[slot_id]->eps[ep_index].ep_state &= ~EP_CLEARING_TT;
5402 	xhci_ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
5403 	spin_unlock_irqrestore(&xhci->lock, flags);
5404 }
5405 
5406 static const struct hc_driver xhci_hc_driver = {
5407 	.description =		"xhci-hcd",
5408 	.product_desc =		"xHCI Host Controller",
5409 	.hcd_priv_size =	sizeof(struct xhci_hcd),
5410 
5411 	/*
5412 	 * generic hardware linkage
5413 	 */
5414 	.irq =			xhci_irq,
5415 	.flags =		HCD_MEMORY | HCD_DMA | HCD_USB3 | HCD_SHARED |
5416 				HCD_BH,
5417 
5418 	/*
5419 	 * basic lifecycle operations
5420 	 */
5421 	.reset =		NULL, /* set in xhci_init_driver() */
5422 	.start =		xhci_run,
5423 	.stop =			xhci_stop,
5424 	.shutdown =		xhci_shutdown,
5425 
5426 	/*
5427 	 * managing i/o requests and associated device resources
5428 	 */
5429 	.map_urb_for_dma =      xhci_map_urb_for_dma,
5430 	.unmap_urb_for_dma =    xhci_unmap_urb_for_dma,
5431 	.urb_enqueue =		xhci_urb_enqueue,
5432 	.urb_dequeue =		xhci_urb_dequeue,
5433 	.alloc_dev =		xhci_alloc_dev,
5434 	.free_dev =		xhci_free_dev,
5435 	.alloc_streams =	xhci_alloc_streams,
5436 	.free_streams =		xhci_free_streams,
5437 	.add_endpoint =		xhci_add_endpoint,
5438 	.drop_endpoint =	xhci_drop_endpoint,
5439 	.endpoint_disable =	xhci_endpoint_disable,
5440 	.endpoint_reset =	xhci_endpoint_reset,
5441 	.check_bandwidth =	xhci_check_bandwidth,
5442 	.reset_bandwidth =	xhci_reset_bandwidth,
5443 	.address_device =	xhci_address_device,
5444 	.enable_device =	xhci_enable_device,
5445 	.update_hub_device =	xhci_update_hub_device,
5446 	.reset_device =		xhci_discover_or_reset_device,
5447 
5448 	/*
5449 	 * scheduling support
5450 	 */
5451 	.get_frame_number =	xhci_get_frame,
5452 
5453 	/*
5454 	 * root hub support
5455 	 */
5456 	.hub_control =		xhci_hub_control,
5457 	.hub_status_data =	xhci_hub_status_data,
5458 	.bus_suspend =		xhci_bus_suspend,
5459 	.bus_resume =		xhci_bus_resume,
5460 	.get_resuming_ports =	xhci_get_resuming_ports,
5461 
5462 	/*
5463 	 * call back when device connected and addressed
5464 	 */
5465 	.update_device =        xhci_update_device,
5466 	.set_usb2_hw_lpm =	xhci_set_usb2_hardware_lpm,
5467 	.enable_usb3_lpm_timeout =	xhci_enable_usb3_lpm_timeout,
5468 	.disable_usb3_lpm_timeout =	xhci_disable_usb3_lpm_timeout,
5469 	.find_raw_port_number =	xhci_find_raw_port_number,
5470 	.clear_tt_buffer_complete = xhci_clear_tt_buffer_complete,
5471 };
5472 
5473 void xhci_init_driver(struct hc_driver *drv,
5474 		      const struct xhci_driver_overrides *over)
5475 {
5476 	BUG_ON(!over);
5477 
5478 	/* Copy the generic table to drv then apply the overrides */
5479 	*drv = xhci_hc_driver;
5480 
5481 	if (over) {
5482 		drv->hcd_priv_size += over->extra_priv_size;
5483 		if (over->reset)
5484 			drv->reset = over->reset;
5485 		if (over->start)
5486 			drv->start = over->start;
5487 		if (over->add_endpoint)
5488 			drv->add_endpoint = over->add_endpoint;
5489 		if (over->drop_endpoint)
5490 			drv->drop_endpoint = over->drop_endpoint;
5491 		if (over->check_bandwidth)
5492 			drv->check_bandwidth = over->check_bandwidth;
5493 		if (over->reset_bandwidth)
5494 			drv->reset_bandwidth = over->reset_bandwidth;
5495 		if (over->update_hub_device)
5496 			drv->update_hub_device = over->update_hub_device;
5497 		if (over->hub_control)
5498 			drv->hub_control = over->hub_control;
5499 	}
5500 }
5501 EXPORT_SYMBOL_GPL(xhci_init_driver);
5502 
5503 MODULE_DESCRIPTION(DRIVER_DESC);
5504 MODULE_AUTHOR(DRIVER_AUTHOR);
5505 MODULE_LICENSE("GPL");
5506 
5507 static int __init xhci_hcd_init(void)
5508 {
5509 	/*
5510 	 * Check the compiler generated sizes of structures that must be laid
5511 	 * out in specific ways for hardware access.
5512 	 */
5513 	BUILD_BUG_ON(sizeof(struct xhci_doorbell_array) != 256*32/8);
5514 	BUILD_BUG_ON(sizeof(struct xhci_slot_ctx) != 8*32/8);
5515 	BUILD_BUG_ON(sizeof(struct xhci_ep_ctx) != 8*32/8);
5516 	/* xhci_device_control has eight fields, and also
5517 	 * embeds one xhci_slot_ctx and 31 xhci_ep_ctx
5518 	 */
5519 	BUILD_BUG_ON(sizeof(struct xhci_stream_ctx) != 4*32/8);
5520 	BUILD_BUG_ON(sizeof(union xhci_trb) != 4*32/8);
5521 	BUILD_BUG_ON(sizeof(struct xhci_erst_entry) != 4*32/8);
5522 	BUILD_BUG_ON(sizeof(struct xhci_cap_regs) != 8*32/8);
5523 	BUILD_BUG_ON(sizeof(struct xhci_intr_reg) != 8*32/8);
5524 	/* xhci_run_regs has eight fields and embeds 128 xhci_intr_regs */
5525 	BUILD_BUG_ON(sizeof(struct xhci_run_regs) != (8+8*128)*32/8);
5526 
5527 	if (usb_disabled())
5528 		return -ENODEV;
5529 
5530 	xhci_debugfs_create_root();
5531 	xhci_dbc_init();
5532 
5533 	return 0;
5534 }
5535 
5536 /*
5537  * If an init function is provided, an exit function must also be provided
5538  * to allow module unload.
5539  */
5540 static void __exit xhci_hcd_fini(void)
5541 {
5542 	xhci_debugfs_remove_root();
5543 	xhci_dbc_exit();
5544 }
5545 
5546 module_init(xhci_hcd_init);
5547 module_exit(xhci_hcd_fini);
5548