xref: /linux/drivers/usb/host/xhci.c (revision 96f30c8f0aa9923aa39b30bcaefeacf88b490231)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * xHCI host controller driver
4  *
5  * Copyright (C) 2008 Intel Corp.
6  *
7  * Author: Sarah Sharp
8  * Some code borrowed from the Linux EHCI driver.
9  */
10 
11 #include <linux/pci.h>
12 #include <linux/iommu.h>
13 #include <linux/iopoll.h>
14 #include <linux/irq.h>
15 #include <linux/log2.h>
16 #include <linux/module.h>
17 #include <linux/moduleparam.h>
18 #include <linux/slab.h>
19 #include <linux/dmi.h>
20 #include <linux/dma-mapping.h>
21 
22 #include "xhci.h"
23 #include "xhci-trace.h"
24 #include "xhci-debugfs.h"
25 #include "xhci-dbgcap.h"
26 
27 #define DRIVER_AUTHOR "Sarah Sharp"
28 #define DRIVER_DESC "'eXtensible' Host Controller (xHC) Driver"
29 
30 #define	PORT_WAKE_BITS	(PORT_WKOC_E | PORT_WKDISC_E | PORT_WKCONN_E)
31 
32 /* Some 0.95 hardware can't handle the chain bit on a Link TRB being cleared */
33 static int link_quirk;
34 module_param(link_quirk, int, S_IRUGO | S_IWUSR);
35 MODULE_PARM_DESC(link_quirk, "Don't clear the chain bit on a link TRB");
36 
37 static unsigned long long quirks;
38 module_param(quirks, ullong, S_IRUGO);
39 MODULE_PARM_DESC(quirks, "Bit flags for quirks to be enabled as default");
40 
41 static bool td_on_ring(struct xhci_td *td, struct xhci_ring *ring)
42 {
43 	struct xhci_segment *seg = ring->first_seg;
44 
45 	if (!td || !td->start_seg)
46 		return false;
47 	do {
48 		if (seg == td->start_seg)
49 			return true;
50 		seg = seg->next;
51 	} while (seg && seg != ring->first_seg);
52 
53 	return false;
54 }
55 
56 /*
57  * xhci_handshake - spin reading hc until handshake completes or fails
58  * @ptr: address of hc register to be read
59  * @mask: bits to look at in result of read
60  * @done: value of those bits when handshake succeeds
61  * @usec: timeout in microseconds
62  *
63  * Returns negative errno, or zero on success
64  *
65  * Success happens when the "mask" bits have the specified value (hardware
66  * handshake done).  There are two failure modes:  "usec" have passed (major
67  * hardware flakeout), or the register reads as all-ones (hardware removed).
68  */
69 int xhci_handshake(void __iomem *ptr, u32 mask, u32 done, u64 timeout_us)
70 {
71 	u32	result;
72 	int	ret;
73 
74 	ret = readl_poll_timeout_atomic(ptr, result,
75 					(result & mask) == done ||
76 					result == U32_MAX,
77 					1, timeout_us);
78 	if (result == U32_MAX)		/* card removed */
79 		return -ENODEV;
80 
81 	return ret;
82 }
83 
84 /*
85  * xhci_handshake_check_state - same as xhci_handshake but takes an additional
86  * exit_state parameter, and bails out with an error immediately when xhc_state
87  * has exit_state flag set.
88  */
89 int xhci_handshake_check_state(struct xhci_hcd *xhci, void __iomem *ptr,
90 		u32 mask, u32 done, int usec, unsigned int exit_state)
91 {
92 	u32	result;
93 	int	ret;
94 
95 	ret = readl_poll_timeout_atomic(ptr, result,
96 				(result & mask) == done ||
97 				result == U32_MAX ||
98 				xhci->xhc_state & exit_state,
99 				1, usec);
100 
101 	if (result == U32_MAX || xhci->xhc_state & exit_state)
102 		return -ENODEV;
103 
104 	return ret;
105 }
106 
107 /*
108  * Disable interrupts and begin the xHCI halting process.
109  */
110 void xhci_quiesce(struct xhci_hcd *xhci)
111 {
112 	u32 halted;
113 	u32 cmd;
114 	u32 mask;
115 
116 	mask = ~(XHCI_IRQS);
117 	halted = readl(&xhci->op_regs->status) & STS_HALT;
118 	if (!halted)
119 		mask &= ~CMD_RUN;
120 
121 	cmd = readl(&xhci->op_regs->command);
122 	cmd &= mask;
123 	writel(cmd, &xhci->op_regs->command);
124 }
125 
126 /*
127  * Force HC into halt state.
128  *
129  * Disable any IRQs and clear the run/stop bit.
130  * HC will complete any current and actively pipelined transactions, and
131  * should halt within 16 ms of the run/stop bit being cleared.
132  * Read HC Halted bit in the status register to see when the HC is finished.
133  */
134 int xhci_halt(struct xhci_hcd *xhci)
135 {
136 	int ret;
137 
138 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Halt the HC");
139 	xhci_quiesce(xhci);
140 
141 	ret = xhci_handshake(&xhci->op_regs->status,
142 			STS_HALT, STS_HALT, XHCI_MAX_HALT_USEC);
143 	if (ret) {
144 		xhci_warn(xhci, "Host halt failed, %d\n", ret);
145 		return ret;
146 	}
147 
148 	xhci->xhc_state |= XHCI_STATE_HALTED;
149 	xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
150 
151 	return ret;
152 }
153 
154 /*
155  * Set the run bit and wait for the host to be running.
156  */
157 int xhci_start(struct xhci_hcd *xhci)
158 {
159 	u32 temp;
160 	int ret;
161 
162 	temp = readl(&xhci->op_regs->command);
163 	temp |= (CMD_RUN);
164 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Turn on HC, cmd = 0x%x.",
165 			temp);
166 	writel(temp, &xhci->op_regs->command);
167 
168 	/*
169 	 * Wait for the HCHalted Status bit to be 0 to indicate the host is
170 	 * running.
171 	 */
172 	ret = xhci_handshake(&xhci->op_regs->status,
173 			STS_HALT, 0, XHCI_MAX_HALT_USEC);
174 	if (ret == -ETIMEDOUT)
175 		xhci_err(xhci, "Host took too long to start, "
176 				"waited %u microseconds.\n",
177 				XHCI_MAX_HALT_USEC);
178 	if (!ret) {
179 		/* clear state flags. Including dying, halted or removing */
180 		xhci->xhc_state = 0;
181 		xhci->run_graceperiod = jiffies + msecs_to_jiffies(500);
182 	}
183 
184 	return ret;
185 }
186 
187 /*
188  * Reset a halted HC.
189  *
190  * This resets pipelines, timers, counters, state machines, etc.
191  * Transactions will be terminated immediately, and operational registers
192  * will be set to their defaults.
193  */
194 int xhci_reset(struct xhci_hcd *xhci, u64 timeout_us)
195 {
196 	u32 command;
197 	u32 state;
198 	int ret;
199 
200 	state = readl(&xhci->op_regs->status);
201 
202 	if (state == ~(u32)0) {
203 		xhci_warn(xhci, "Host not accessible, reset failed.\n");
204 		return -ENODEV;
205 	}
206 
207 	if ((state & STS_HALT) == 0) {
208 		xhci_warn(xhci, "Host controller not halted, aborting reset.\n");
209 		return 0;
210 	}
211 
212 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Reset the HC");
213 	command = readl(&xhci->op_regs->command);
214 	command |= CMD_RESET;
215 	writel(command, &xhci->op_regs->command);
216 
217 	/* Existing Intel xHCI controllers require a delay of 1 mS,
218 	 * after setting the CMD_RESET bit, and before accessing any
219 	 * HC registers. This allows the HC to complete the
220 	 * reset operation and be ready for HC register access.
221 	 * Without this delay, the subsequent HC register access,
222 	 * may result in a system hang very rarely.
223 	 */
224 	if (xhci->quirks & XHCI_INTEL_HOST)
225 		udelay(1000);
226 
227 	ret = xhci_handshake_check_state(xhci, &xhci->op_regs->command,
228 				CMD_RESET, 0, timeout_us, XHCI_STATE_REMOVING);
229 	if (ret)
230 		return ret;
231 
232 	if (xhci->quirks & XHCI_ASMEDIA_MODIFY_FLOWCONTROL)
233 		usb_asmedia_modifyflowcontrol(to_pci_dev(xhci_to_hcd(xhci)->self.controller));
234 
235 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
236 			 "Wait for controller to be ready for doorbell rings");
237 	/*
238 	 * xHCI cannot write to any doorbells or operational registers other
239 	 * than status until the "Controller Not Ready" flag is cleared.
240 	 */
241 	ret = xhci_handshake(&xhci->op_regs->status, STS_CNR, 0, timeout_us);
242 
243 	xhci->usb2_rhub.bus_state.port_c_suspend = 0;
244 	xhci->usb2_rhub.bus_state.suspended_ports = 0;
245 	xhci->usb2_rhub.bus_state.resuming_ports = 0;
246 	xhci->usb3_rhub.bus_state.port_c_suspend = 0;
247 	xhci->usb3_rhub.bus_state.suspended_ports = 0;
248 	xhci->usb3_rhub.bus_state.resuming_ports = 0;
249 
250 	return ret;
251 }
252 
253 static void xhci_zero_64b_regs(struct xhci_hcd *xhci)
254 {
255 	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
256 	struct iommu_domain *domain;
257 	int err, i;
258 	u64 val;
259 	u32 intrs;
260 
261 	/*
262 	 * Some Renesas controllers get into a weird state if they are
263 	 * reset while programmed with 64bit addresses (they will preserve
264 	 * the top half of the address in internal, non visible
265 	 * registers). You end up with half the address coming from the
266 	 * kernel, and the other half coming from the firmware. Also,
267 	 * changing the programming leads to extra accesses even if the
268 	 * controller is supposed to be halted. The controller ends up with
269 	 * a fatal fault, and is then ripe for being properly reset.
270 	 *
271 	 * Special care is taken to only apply this if the device is behind
272 	 * an iommu. Doing anything when there is no iommu is definitely
273 	 * unsafe...
274 	 */
275 	domain = iommu_get_domain_for_dev(dev);
276 	if (!(xhci->quirks & XHCI_ZERO_64B_REGS) || !domain ||
277 	    domain->type == IOMMU_DOMAIN_IDENTITY)
278 		return;
279 
280 	xhci_info(xhci, "Zeroing 64bit base registers, expecting fault\n");
281 
282 	/* Clear HSEIE so that faults do not get signaled */
283 	val = readl(&xhci->op_regs->command);
284 	val &= ~CMD_HSEIE;
285 	writel(val, &xhci->op_regs->command);
286 
287 	/* Clear HSE (aka FATAL) */
288 	val = readl(&xhci->op_regs->status);
289 	val |= STS_FATAL;
290 	writel(val, &xhci->op_regs->status);
291 
292 	/* Now zero the registers, and brace for impact */
293 	val = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
294 	if (upper_32_bits(val))
295 		xhci_write_64(xhci, 0, &xhci->op_regs->dcbaa_ptr);
296 	val = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
297 	if (upper_32_bits(val))
298 		xhci_write_64(xhci, 0, &xhci->op_regs->cmd_ring);
299 
300 	intrs = min_t(u32, HCS_MAX_INTRS(xhci->hcs_params1),
301 		      ARRAY_SIZE(xhci->run_regs->ir_set));
302 
303 	for (i = 0; i < intrs; i++) {
304 		struct xhci_intr_reg __iomem *ir;
305 
306 		ir = &xhci->run_regs->ir_set[i];
307 		val = xhci_read_64(xhci, &ir->erst_base);
308 		if (upper_32_bits(val))
309 			xhci_write_64(xhci, 0, &ir->erst_base);
310 		val= xhci_read_64(xhci, &ir->erst_dequeue);
311 		if (upper_32_bits(val))
312 			xhci_write_64(xhci, 0, &ir->erst_dequeue);
313 	}
314 
315 	/* Wait for the fault to appear. It will be cleared on reset */
316 	err = xhci_handshake(&xhci->op_regs->status,
317 			     STS_FATAL, STS_FATAL,
318 			     XHCI_MAX_HALT_USEC);
319 	if (!err)
320 		xhci_info(xhci, "Fault detected\n");
321 }
322 
323 static int xhci_enable_interrupter(struct xhci_interrupter *ir)
324 {
325 	u32 iman;
326 
327 	if (!ir || !ir->ir_set)
328 		return -EINVAL;
329 
330 	iman = readl(&ir->ir_set->irq_pending);
331 	writel(ER_IRQ_ENABLE(iman), &ir->ir_set->irq_pending);
332 
333 	return 0;
334 }
335 
336 static int xhci_disable_interrupter(struct xhci_interrupter *ir)
337 {
338 	u32 iman;
339 
340 	if (!ir || !ir->ir_set)
341 		return -EINVAL;
342 
343 	iman = readl(&ir->ir_set->irq_pending);
344 	writel(ER_IRQ_DISABLE(iman), &ir->ir_set->irq_pending);
345 
346 	return 0;
347 }
348 
349 /* interrupt moderation interval imod_interval in nanoseconds */
350 static int xhci_set_interrupter_moderation(struct xhci_interrupter *ir,
351 					   u32 imod_interval)
352 {
353 	u32 imod;
354 
355 	if (!ir || !ir->ir_set || imod_interval > U16_MAX * 250)
356 		return -EINVAL;
357 
358 	imod = readl(&ir->ir_set->irq_control);
359 	imod &= ~ER_IRQ_INTERVAL_MASK;
360 	imod |= (imod_interval / 250) & ER_IRQ_INTERVAL_MASK;
361 	writel(imod, &ir->ir_set->irq_control);
362 
363 	return 0;
364 }
365 
366 static void compliance_mode_recovery(struct timer_list *t)
367 {
368 	struct xhci_hcd *xhci;
369 	struct usb_hcd *hcd;
370 	struct xhci_hub *rhub;
371 	u32 temp;
372 	int i;
373 
374 	xhci = from_timer(xhci, t, comp_mode_recovery_timer);
375 	rhub = &xhci->usb3_rhub;
376 	hcd = rhub->hcd;
377 
378 	if (!hcd)
379 		return;
380 
381 	for (i = 0; i < rhub->num_ports; i++) {
382 		temp = readl(rhub->ports[i]->addr);
383 		if ((temp & PORT_PLS_MASK) == USB_SS_PORT_LS_COMP_MOD) {
384 			/*
385 			 * Compliance Mode Detected. Letting USB Core
386 			 * handle the Warm Reset
387 			 */
388 			xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
389 					"Compliance mode detected->port %d",
390 					i + 1);
391 			xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
392 					"Attempting compliance mode recovery");
393 
394 			if (hcd->state == HC_STATE_SUSPENDED)
395 				usb_hcd_resume_root_hub(hcd);
396 
397 			usb_hcd_poll_rh_status(hcd);
398 		}
399 	}
400 
401 	if (xhci->port_status_u0 != ((1 << rhub->num_ports) - 1))
402 		mod_timer(&xhci->comp_mode_recovery_timer,
403 			jiffies + msecs_to_jiffies(COMP_MODE_RCVRY_MSECS));
404 }
405 
406 /*
407  * Quirk to work around issue generated by the SN65LVPE502CP USB3.0 re-driver
408  * that causes ports behind that hardware to enter compliance mode sometimes.
409  * The quirk creates a timer that polls every 2 seconds the link state of
410  * each host controller's port and recovers it by issuing a Warm reset
411  * if Compliance mode is detected, otherwise the port will become "dead" (no
412  * device connections or disconnections will be detected anymore). Becasue no
413  * status event is generated when entering compliance mode (per xhci spec),
414  * this quirk is needed on systems that have the failing hardware installed.
415  */
416 static void compliance_mode_recovery_timer_init(struct xhci_hcd *xhci)
417 {
418 	xhci->port_status_u0 = 0;
419 	timer_setup(&xhci->comp_mode_recovery_timer, compliance_mode_recovery,
420 		    0);
421 	xhci->comp_mode_recovery_timer.expires = jiffies +
422 			msecs_to_jiffies(COMP_MODE_RCVRY_MSECS);
423 
424 	add_timer(&xhci->comp_mode_recovery_timer);
425 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
426 			"Compliance mode recovery timer initialized");
427 }
428 
429 /*
430  * This function identifies the systems that have installed the SN65LVPE502CP
431  * USB3.0 re-driver and that need the Compliance Mode Quirk.
432  * Systems:
433  * Vendor: Hewlett-Packard -> System Models: Z420, Z620 and Z820
434  */
435 static bool xhci_compliance_mode_recovery_timer_quirk_check(void)
436 {
437 	const char *dmi_product_name, *dmi_sys_vendor;
438 
439 	dmi_product_name = dmi_get_system_info(DMI_PRODUCT_NAME);
440 	dmi_sys_vendor = dmi_get_system_info(DMI_SYS_VENDOR);
441 	if (!dmi_product_name || !dmi_sys_vendor)
442 		return false;
443 
444 	if (!(strstr(dmi_sys_vendor, "Hewlett-Packard")))
445 		return false;
446 
447 	if (strstr(dmi_product_name, "Z420") ||
448 			strstr(dmi_product_name, "Z620") ||
449 			strstr(dmi_product_name, "Z820") ||
450 			strstr(dmi_product_name, "Z1 Workstation"))
451 		return true;
452 
453 	return false;
454 }
455 
456 static int xhci_all_ports_seen_u0(struct xhci_hcd *xhci)
457 {
458 	return (xhci->port_status_u0 == ((1 << xhci->usb3_rhub.num_ports) - 1));
459 }
460 
461 
462 /*
463  * Initialize memory for HCD and xHC (one-time init).
464  *
465  * Program the PAGESIZE register, initialize the device context array, create
466  * device contexts (?), set up a command ring segment (or two?), create event
467  * ring (one for now).
468  */
469 static int xhci_init(struct usb_hcd *hcd)
470 {
471 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
472 	int retval;
473 
474 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "xhci_init");
475 	spin_lock_init(&xhci->lock);
476 	if (xhci->hci_version == 0x95 && link_quirk) {
477 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
478 				"QUIRK: Not clearing Link TRB chain bits.");
479 		xhci->quirks |= XHCI_LINK_TRB_QUIRK;
480 	} else {
481 		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
482 				"xHCI doesn't need link TRB QUIRK");
483 	}
484 	retval = xhci_mem_init(xhci, GFP_KERNEL);
485 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Finished xhci_init");
486 
487 	/* Initializing Compliance Mode Recovery Data If Needed */
488 	if (xhci_compliance_mode_recovery_timer_quirk_check()) {
489 		xhci->quirks |= XHCI_COMP_MODE_QUIRK;
490 		compliance_mode_recovery_timer_init(xhci);
491 	}
492 
493 	return retval;
494 }
495 
496 /*-------------------------------------------------------------------------*/
497 
498 static int xhci_run_finished(struct xhci_hcd *xhci)
499 {
500 	struct xhci_interrupter *ir = xhci->interrupters[0];
501 	unsigned long	flags;
502 	u32		temp;
503 
504 	/*
505 	 * Enable interrupts before starting the host (xhci 4.2 and 5.5.2).
506 	 * Protect the short window before host is running with a lock
507 	 */
508 	spin_lock_irqsave(&xhci->lock, flags);
509 
510 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Enable interrupts");
511 	temp = readl(&xhci->op_regs->command);
512 	temp |= (CMD_EIE);
513 	writel(temp, &xhci->op_regs->command);
514 
515 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Enable primary interrupter");
516 	xhci_enable_interrupter(ir);
517 
518 	if (xhci_start(xhci)) {
519 		xhci_halt(xhci);
520 		spin_unlock_irqrestore(&xhci->lock, flags);
521 		return -ENODEV;
522 	}
523 
524 	xhci->cmd_ring_state = CMD_RING_STATE_RUNNING;
525 
526 	if (xhci->quirks & XHCI_NEC_HOST)
527 		xhci_ring_cmd_db(xhci);
528 
529 	spin_unlock_irqrestore(&xhci->lock, flags);
530 
531 	return 0;
532 }
533 
534 /*
535  * Start the HC after it was halted.
536  *
537  * This function is called by the USB core when the HC driver is added.
538  * Its opposite is xhci_stop().
539  *
540  * xhci_init() must be called once before this function can be called.
541  * Reset the HC, enable device slot contexts, program DCBAAP, and
542  * set command ring pointer and event ring pointer.
543  *
544  * Setup MSI-X vectors and enable interrupts.
545  */
546 int xhci_run(struct usb_hcd *hcd)
547 {
548 	u64 temp_64;
549 	int ret;
550 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
551 	struct xhci_interrupter *ir = xhci->interrupters[0];
552 	/* Start the xHCI host controller running only after the USB 2.0 roothub
553 	 * is setup.
554 	 */
555 
556 	hcd->uses_new_polling = 1;
557 	if (hcd->msi_enabled)
558 		ir->ip_autoclear = true;
559 
560 	if (!usb_hcd_is_primary_hcd(hcd))
561 		return xhci_run_finished(xhci);
562 
563 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "xhci_run");
564 
565 	temp_64 = xhci_read_64(xhci, &ir->ir_set->erst_dequeue);
566 	temp_64 &= ERST_PTR_MASK;
567 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
568 			"ERST deq = 64'h%0lx", (long unsigned int) temp_64);
569 
570 	xhci_set_interrupter_moderation(ir, xhci->imod_interval);
571 
572 	if (xhci->quirks & XHCI_NEC_HOST) {
573 		struct xhci_command *command;
574 
575 		command = xhci_alloc_command(xhci, false, GFP_KERNEL);
576 		if (!command)
577 			return -ENOMEM;
578 
579 		ret = xhci_queue_vendor_command(xhci, command, 0, 0, 0,
580 				TRB_TYPE(TRB_NEC_GET_FW));
581 		if (ret)
582 			xhci_free_command(xhci, command);
583 	}
584 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
585 			"Finished %s for main hcd", __func__);
586 
587 	xhci_create_dbc_dev(xhci);
588 
589 	xhci_debugfs_init(xhci);
590 
591 	if (xhci_has_one_roothub(xhci))
592 		return xhci_run_finished(xhci);
593 
594 	set_bit(HCD_FLAG_DEFER_RH_REGISTER, &hcd->flags);
595 
596 	return 0;
597 }
598 EXPORT_SYMBOL_GPL(xhci_run);
599 
600 /*
601  * Stop xHCI driver.
602  *
603  * This function is called by the USB core when the HC driver is removed.
604  * Its opposite is xhci_run().
605  *
606  * Disable device contexts, disable IRQs, and quiesce the HC.
607  * Reset the HC, finish any completed transactions, and cleanup memory.
608  */
609 void xhci_stop(struct usb_hcd *hcd)
610 {
611 	u32 temp;
612 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
613 	struct xhci_interrupter *ir = xhci->interrupters[0];
614 
615 	mutex_lock(&xhci->mutex);
616 
617 	/* Only halt host and free memory after both hcds are removed */
618 	if (!usb_hcd_is_primary_hcd(hcd)) {
619 		mutex_unlock(&xhci->mutex);
620 		return;
621 	}
622 
623 	xhci_remove_dbc_dev(xhci);
624 
625 	spin_lock_irq(&xhci->lock);
626 	xhci->xhc_state |= XHCI_STATE_HALTED;
627 	xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
628 	xhci_halt(xhci);
629 	xhci_reset(xhci, XHCI_RESET_SHORT_USEC);
630 	spin_unlock_irq(&xhci->lock);
631 
632 	/* Deleting Compliance Mode Recovery Timer */
633 	if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
634 			(!(xhci_all_ports_seen_u0(xhci)))) {
635 		del_timer_sync(&xhci->comp_mode_recovery_timer);
636 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
637 				"%s: compliance mode recovery timer deleted",
638 				__func__);
639 	}
640 
641 	if (xhci->quirks & XHCI_AMD_PLL_FIX)
642 		usb_amd_dev_put();
643 
644 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
645 			"// Disabling event ring interrupts");
646 	temp = readl(&xhci->op_regs->status);
647 	writel((temp & ~0x1fff) | STS_EINT, &xhci->op_regs->status);
648 	xhci_disable_interrupter(ir);
649 
650 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "cleaning up memory");
651 	xhci_mem_cleanup(xhci);
652 	xhci_debugfs_exit(xhci);
653 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
654 			"xhci_stop completed - status = %x",
655 			readl(&xhci->op_regs->status));
656 	mutex_unlock(&xhci->mutex);
657 }
658 EXPORT_SYMBOL_GPL(xhci_stop);
659 
660 /*
661  * Shutdown HC (not bus-specific)
662  *
663  * This is called when the machine is rebooting or halting.  We assume that the
664  * machine will be powered off, and the HC's internal state will be reset.
665  * Don't bother to free memory.
666  *
667  * This will only ever be called with the main usb_hcd (the USB3 roothub).
668  */
669 void xhci_shutdown(struct usb_hcd *hcd)
670 {
671 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
672 
673 	if (xhci->quirks & XHCI_SPURIOUS_REBOOT)
674 		usb_disable_xhci_ports(to_pci_dev(hcd->self.sysdev));
675 
676 	/* Don't poll the roothubs after shutdown. */
677 	xhci_dbg(xhci, "%s: stopping usb%d port polling.\n",
678 			__func__, hcd->self.busnum);
679 	clear_bit(HCD_FLAG_POLL_RH, &hcd->flags);
680 	del_timer_sync(&hcd->rh_timer);
681 
682 	if (xhci->shared_hcd) {
683 		clear_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags);
684 		del_timer_sync(&xhci->shared_hcd->rh_timer);
685 	}
686 
687 	spin_lock_irq(&xhci->lock);
688 	xhci_halt(xhci);
689 
690 	/*
691 	 * Workaround for spurious wakeps at shutdown with HSW, and for boot
692 	 * firmware delay in ADL-P PCH if port are left in U3 at shutdown
693 	 */
694 	if (xhci->quirks & XHCI_SPURIOUS_WAKEUP ||
695 	    xhci->quirks & XHCI_RESET_TO_DEFAULT)
696 		xhci_reset(xhci, XHCI_RESET_SHORT_USEC);
697 
698 	spin_unlock_irq(&xhci->lock);
699 
700 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
701 			"xhci_shutdown completed - status = %x",
702 			readl(&xhci->op_regs->status));
703 }
704 EXPORT_SYMBOL_GPL(xhci_shutdown);
705 
706 #ifdef CONFIG_PM
707 static void xhci_save_registers(struct xhci_hcd *xhci)
708 {
709 	struct xhci_interrupter *ir;
710 	unsigned int i;
711 
712 	xhci->s3.command = readl(&xhci->op_regs->command);
713 	xhci->s3.dev_nt = readl(&xhci->op_regs->dev_notification);
714 	xhci->s3.dcbaa_ptr = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
715 	xhci->s3.config_reg = readl(&xhci->op_regs->config_reg);
716 
717 	/* save both primary and all secondary interrupters */
718 	/* fixme, shold we lock  to prevent race with remove secondary interrupter? */
719 	for (i = 0; i < xhci->max_interrupters; i++) {
720 		ir = xhci->interrupters[i];
721 		if (!ir)
722 			continue;
723 
724 		ir->s3_erst_size = readl(&ir->ir_set->erst_size);
725 		ir->s3_erst_base = xhci_read_64(xhci, &ir->ir_set->erst_base);
726 		ir->s3_erst_dequeue = xhci_read_64(xhci, &ir->ir_set->erst_dequeue);
727 		ir->s3_irq_pending = readl(&ir->ir_set->irq_pending);
728 		ir->s3_irq_control = readl(&ir->ir_set->irq_control);
729 	}
730 }
731 
732 static void xhci_restore_registers(struct xhci_hcd *xhci)
733 {
734 	struct xhci_interrupter *ir;
735 	unsigned int i;
736 
737 	writel(xhci->s3.command, &xhci->op_regs->command);
738 	writel(xhci->s3.dev_nt, &xhci->op_regs->dev_notification);
739 	xhci_write_64(xhci, xhci->s3.dcbaa_ptr, &xhci->op_regs->dcbaa_ptr);
740 	writel(xhci->s3.config_reg, &xhci->op_regs->config_reg);
741 
742 	/* FIXME should we lock to protect against freeing of interrupters */
743 	for (i = 0; i < xhci->max_interrupters; i++) {
744 		ir = xhci->interrupters[i];
745 		if (!ir)
746 			continue;
747 
748 		writel(ir->s3_erst_size, &ir->ir_set->erst_size);
749 		xhci_write_64(xhci, ir->s3_erst_base, &ir->ir_set->erst_base);
750 		xhci_write_64(xhci, ir->s3_erst_dequeue, &ir->ir_set->erst_dequeue);
751 		writel(ir->s3_irq_pending, &ir->ir_set->irq_pending);
752 		writel(ir->s3_irq_control, &ir->ir_set->irq_control);
753 	}
754 }
755 
756 static void xhci_set_cmd_ring_deq(struct xhci_hcd *xhci)
757 {
758 	u64	val_64;
759 
760 	/* step 2: initialize command ring buffer */
761 	val_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
762 	val_64 = (val_64 & (u64) CMD_RING_RSVD_BITS) |
763 		(xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg,
764 				      xhci->cmd_ring->dequeue) &
765 		 (u64) ~CMD_RING_RSVD_BITS) |
766 		xhci->cmd_ring->cycle_state;
767 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
768 			"// Setting command ring address to 0x%llx",
769 			(long unsigned long) val_64);
770 	xhci_write_64(xhci, val_64, &xhci->op_regs->cmd_ring);
771 }
772 
773 /*
774  * The whole command ring must be cleared to zero when we suspend the host.
775  *
776  * The host doesn't save the command ring pointer in the suspend well, so we
777  * need to re-program it on resume.  Unfortunately, the pointer must be 64-byte
778  * aligned, because of the reserved bits in the command ring dequeue pointer
779  * register.  Therefore, we can't just set the dequeue pointer back in the
780  * middle of the ring (TRBs are 16-byte aligned).
781  */
782 static void xhci_clear_command_ring(struct xhci_hcd *xhci)
783 {
784 	struct xhci_ring *ring;
785 	struct xhci_segment *seg;
786 
787 	ring = xhci->cmd_ring;
788 	seg = ring->deq_seg;
789 	do {
790 		memset(seg->trbs, 0,
791 			sizeof(union xhci_trb) * (TRBS_PER_SEGMENT - 1));
792 		seg->trbs[TRBS_PER_SEGMENT - 1].link.control &=
793 			cpu_to_le32(~TRB_CYCLE);
794 		seg = seg->next;
795 	} while (seg != ring->deq_seg);
796 
797 	xhci_initialize_ring_info(ring, 1);
798 	/*
799 	 * Reset the hardware dequeue pointer.
800 	 * Yes, this will need to be re-written after resume, but we're paranoid
801 	 * and want to make sure the hardware doesn't access bogus memory
802 	 * because, say, the BIOS or an SMI started the host without changing
803 	 * the command ring pointers.
804 	 */
805 	xhci_set_cmd_ring_deq(xhci);
806 }
807 
808 /*
809  * Disable port wake bits if do_wakeup is not set.
810  *
811  * Also clear a possible internal port wake state left hanging for ports that
812  * detected termination but never successfully enumerated (trained to 0U).
813  * Internal wake causes immediate xHCI wake after suspend. PORT_CSC write done
814  * at enumeration clears this wake, force one here as well for unconnected ports
815  */
816 
817 static void xhci_disable_hub_port_wake(struct xhci_hcd *xhci,
818 				       struct xhci_hub *rhub,
819 				       bool do_wakeup)
820 {
821 	unsigned long flags;
822 	u32 t1, t2, portsc;
823 	int i;
824 
825 	spin_lock_irqsave(&xhci->lock, flags);
826 
827 	for (i = 0; i < rhub->num_ports; i++) {
828 		portsc = readl(rhub->ports[i]->addr);
829 		t1 = xhci_port_state_to_neutral(portsc);
830 		t2 = t1;
831 
832 		/* clear wake bits if do_wake is not set */
833 		if (!do_wakeup)
834 			t2 &= ~PORT_WAKE_BITS;
835 
836 		/* Don't touch csc bit if connected or connect change is set */
837 		if (!(portsc & (PORT_CSC | PORT_CONNECT)))
838 			t2 |= PORT_CSC;
839 
840 		if (t1 != t2) {
841 			writel(t2, rhub->ports[i]->addr);
842 			xhci_dbg(xhci, "config port %d-%d wake bits, portsc: 0x%x, write: 0x%x\n",
843 				 rhub->hcd->self.busnum, i + 1, portsc, t2);
844 		}
845 	}
846 	spin_unlock_irqrestore(&xhci->lock, flags);
847 }
848 
849 static bool xhci_pending_portevent(struct xhci_hcd *xhci)
850 {
851 	struct xhci_port	**ports;
852 	int			port_index;
853 	u32			status;
854 	u32			portsc;
855 
856 	status = readl(&xhci->op_regs->status);
857 	if (status & STS_EINT)
858 		return true;
859 	/*
860 	 * Checking STS_EINT is not enough as there is a lag between a change
861 	 * bit being set and the Port Status Change Event that it generated
862 	 * being written to the Event Ring. See note in xhci 1.1 section 4.19.2.
863 	 */
864 
865 	port_index = xhci->usb2_rhub.num_ports;
866 	ports = xhci->usb2_rhub.ports;
867 	while (port_index--) {
868 		portsc = readl(ports[port_index]->addr);
869 		if (portsc & PORT_CHANGE_MASK ||
870 		    (portsc & PORT_PLS_MASK) == XDEV_RESUME)
871 			return true;
872 	}
873 	port_index = xhci->usb3_rhub.num_ports;
874 	ports = xhci->usb3_rhub.ports;
875 	while (port_index--) {
876 		portsc = readl(ports[port_index]->addr);
877 		if (portsc & (PORT_CHANGE_MASK | PORT_CAS) ||
878 		    (portsc & PORT_PLS_MASK) == XDEV_RESUME)
879 			return true;
880 	}
881 	return false;
882 }
883 
884 /*
885  * Stop HC (not bus-specific)
886  *
887  * This is called when the machine transition into S3/S4 mode.
888  *
889  */
890 int xhci_suspend(struct xhci_hcd *xhci, bool do_wakeup)
891 {
892 	int			rc = 0;
893 	unsigned int		delay = XHCI_MAX_HALT_USEC * 2;
894 	struct usb_hcd		*hcd = xhci_to_hcd(xhci);
895 	u32			command;
896 	u32			res;
897 
898 	if (!hcd->state)
899 		return 0;
900 
901 	if (hcd->state != HC_STATE_SUSPENDED ||
902 	    (xhci->shared_hcd && xhci->shared_hcd->state != HC_STATE_SUSPENDED))
903 		return -EINVAL;
904 
905 	/* Clear root port wake on bits if wakeup not allowed. */
906 	xhci_disable_hub_port_wake(xhci, &xhci->usb3_rhub, do_wakeup);
907 	xhci_disable_hub_port_wake(xhci, &xhci->usb2_rhub, do_wakeup);
908 
909 	if (!HCD_HW_ACCESSIBLE(hcd))
910 		return 0;
911 
912 	xhci_dbc_suspend(xhci);
913 
914 	/* Don't poll the roothubs on bus suspend. */
915 	xhci_dbg(xhci, "%s: stopping usb%d port polling.\n",
916 		 __func__, hcd->self.busnum);
917 	clear_bit(HCD_FLAG_POLL_RH, &hcd->flags);
918 	del_timer_sync(&hcd->rh_timer);
919 	if (xhci->shared_hcd) {
920 		clear_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags);
921 		del_timer_sync(&xhci->shared_hcd->rh_timer);
922 	}
923 
924 	if (xhci->quirks & XHCI_SUSPEND_DELAY)
925 		usleep_range(1000, 1500);
926 
927 	spin_lock_irq(&xhci->lock);
928 	clear_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
929 	if (xhci->shared_hcd)
930 		clear_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags);
931 	/* step 1: stop endpoint */
932 	/* skipped assuming that port suspend has done */
933 
934 	/* step 2: clear Run/Stop bit */
935 	command = readl(&xhci->op_regs->command);
936 	command &= ~CMD_RUN;
937 	writel(command, &xhci->op_regs->command);
938 
939 	/* Some chips from Fresco Logic need an extraordinary delay */
940 	delay *= (xhci->quirks & XHCI_SLOW_SUSPEND) ? 10 : 1;
941 
942 	if (xhci_handshake(&xhci->op_regs->status,
943 		      STS_HALT, STS_HALT, delay)) {
944 		xhci_warn(xhci, "WARN: xHC CMD_RUN timeout\n");
945 		spin_unlock_irq(&xhci->lock);
946 		return -ETIMEDOUT;
947 	}
948 	xhci_clear_command_ring(xhci);
949 
950 	/* step 3: save registers */
951 	xhci_save_registers(xhci);
952 
953 	/* step 4: set CSS flag */
954 	command = readl(&xhci->op_regs->command);
955 	command |= CMD_CSS;
956 	writel(command, &xhci->op_regs->command);
957 	xhci->broken_suspend = 0;
958 	if (xhci_handshake(&xhci->op_regs->status,
959 				STS_SAVE, 0, 20 * 1000)) {
960 	/*
961 	 * AMD SNPS xHC 3.0 occasionally does not clear the
962 	 * SSS bit of USBSTS and when driver tries to poll
963 	 * to see if the xHC clears BIT(8) which never happens
964 	 * and driver assumes that controller is not responding
965 	 * and times out. To workaround this, its good to check
966 	 * if SRE and HCE bits are not set (as per xhci
967 	 * Section 5.4.2) and bypass the timeout.
968 	 */
969 		res = readl(&xhci->op_regs->status);
970 		if ((xhci->quirks & XHCI_SNPS_BROKEN_SUSPEND) &&
971 		    (((res & STS_SRE) == 0) &&
972 				((res & STS_HCE) == 0))) {
973 			xhci->broken_suspend = 1;
974 		} else {
975 			xhci_warn(xhci, "WARN: xHC save state timeout\n");
976 			spin_unlock_irq(&xhci->lock);
977 			return -ETIMEDOUT;
978 		}
979 	}
980 	spin_unlock_irq(&xhci->lock);
981 
982 	/*
983 	 * Deleting Compliance Mode Recovery Timer because the xHCI Host
984 	 * is about to be suspended.
985 	 */
986 	if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
987 			(!(xhci_all_ports_seen_u0(xhci)))) {
988 		del_timer_sync(&xhci->comp_mode_recovery_timer);
989 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
990 				"%s: compliance mode recovery timer deleted",
991 				__func__);
992 	}
993 
994 	return rc;
995 }
996 EXPORT_SYMBOL_GPL(xhci_suspend);
997 
998 /*
999  * start xHC (not bus-specific)
1000  *
1001  * This is called when the machine transition from S3/S4 mode.
1002  *
1003  */
1004 int xhci_resume(struct xhci_hcd *xhci, pm_message_t msg)
1005 {
1006 	bool			hibernated = (msg.event == PM_EVENT_RESTORE);
1007 	u32			command, temp = 0;
1008 	struct usb_hcd		*hcd = xhci_to_hcd(xhci);
1009 	int			retval = 0;
1010 	bool			comp_timer_running = false;
1011 	bool			pending_portevent = false;
1012 	bool			suspended_usb3_devs = false;
1013 	bool			reinit_xhc = false;
1014 
1015 	if (!hcd->state)
1016 		return 0;
1017 
1018 	/* Wait a bit if either of the roothubs need to settle from the
1019 	 * transition into bus suspend.
1020 	 */
1021 
1022 	if (time_before(jiffies, xhci->usb2_rhub.bus_state.next_statechange) ||
1023 	    time_before(jiffies, xhci->usb3_rhub.bus_state.next_statechange))
1024 		msleep(100);
1025 
1026 	set_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
1027 	if (xhci->shared_hcd)
1028 		set_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags);
1029 
1030 	spin_lock_irq(&xhci->lock);
1031 
1032 	if (hibernated || xhci->quirks & XHCI_RESET_ON_RESUME || xhci->broken_suspend)
1033 		reinit_xhc = true;
1034 
1035 	if (!reinit_xhc) {
1036 		/*
1037 		 * Some controllers might lose power during suspend, so wait
1038 		 * for controller not ready bit to clear, just as in xHC init.
1039 		 */
1040 		retval = xhci_handshake(&xhci->op_regs->status,
1041 					STS_CNR, 0, 10 * 1000 * 1000);
1042 		if (retval) {
1043 			xhci_warn(xhci, "Controller not ready at resume %d\n",
1044 				  retval);
1045 			spin_unlock_irq(&xhci->lock);
1046 			return retval;
1047 		}
1048 		/* step 1: restore register */
1049 		xhci_restore_registers(xhci);
1050 		/* step 2: initialize command ring buffer */
1051 		xhci_set_cmd_ring_deq(xhci);
1052 		/* step 3: restore state and start state*/
1053 		/* step 3: set CRS flag */
1054 		command = readl(&xhci->op_regs->command);
1055 		command |= CMD_CRS;
1056 		writel(command, &xhci->op_regs->command);
1057 		/*
1058 		 * Some controllers take up to 55+ ms to complete the controller
1059 		 * restore so setting the timeout to 100ms. Xhci specification
1060 		 * doesn't mention any timeout value.
1061 		 */
1062 		if (xhci_handshake(&xhci->op_regs->status,
1063 			      STS_RESTORE, 0, 100 * 1000)) {
1064 			xhci_warn(xhci, "WARN: xHC restore state timeout\n");
1065 			spin_unlock_irq(&xhci->lock);
1066 			return -ETIMEDOUT;
1067 		}
1068 	}
1069 
1070 	temp = readl(&xhci->op_regs->status);
1071 
1072 	/* re-initialize the HC on Restore Error, or Host Controller Error */
1073 	if ((temp & (STS_SRE | STS_HCE)) &&
1074 	    !(xhci->xhc_state & XHCI_STATE_REMOVING)) {
1075 		reinit_xhc = true;
1076 		if (!xhci->broken_suspend)
1077 			xhci_warn(xhci, "xHC error in resume, USBSTS 0x%x, Reinit\n", temp);
1078 	}
1079 
1080 	if (reinit_xhc) {
1081 		if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
1082 				!(xhci_all_ports_seen_u0(xhci))) {
1083 			del_timer_sync(&xhci->comp_mode_recovery_timer);
1084 			xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1085 				"Compliance Mode Recovery Timer deleted!");
1086 		}
1087 
1088 		/* Let the USB core know _both_ roothubs lost power. */
1089 		usb_root_hub_lost_power(xhci->main_hcd->self.root_hub);
1090 		if (xhci->shared_hcd)
1091 			usb_root_hub_lost_power(xhci->shared_hcd->self.root_hub);
1092 
1093 		xhci_dbg(xhci, "Stop HCD\n");
1094 		xhci_halt(xhci);
1095 		xhci_zero_64b_regs(xhci);
1096 		retval = xhci_reset(xhci, XHCI_RESET_LONG_USEC);
1097 		spin_unlock_irq(&xhci->lock);
1098 		if (retval)
1099 			return retval;
1100 
1101 		xhci_dbg(xhci, "// Disabling event ring interrupts\n");
1102 		temp = readl(&xhci->op_regs->status);
1103 		writel((temp & ~0x1fff) | STS_EINT, &xhci->op_regs->status);
1104 		xhci_disable_interrupter(xhci->interrupters[0]);
1105 
1106 		xhci_dbg(xhci, "cleaning up memory\n");
1107 		xhci_mem_cleanup(xhci);
1108 		xhci_debugfs_exit(xhci);
1109 		xhci_dbg(xhci, "xhci_stop completed - status = %x\n",
1110 			    readl(&xhci->op_regs->status));
1111 
1112 		/* USB core calls the PCI reinit and start functions twice:
1113 		 * first with the primary HCD, and then with the secondary HCD.
1114 		 * If we don't do the same, the host will never be started.
1115 		 */
1116 		xhci_dbg(xhci, "Initialize the xhci_hcd\n");
1117 		retval = xhci_init(hcd);
1118 		if (retval)
1119 			return retval;
1120 		comp_timer_running = true;
1121 
1122 		xhci_dbg(xhci, "Start the primary HCD\n");
1123 		retval = xhci_run(hcd);
1124 		if (!retval && xhci->shared_hcd) {
1125 			xhci_dbg(xhci, "Start the secondary HCD\n");
1126 			retval = xhci_run(xhci->shared_hcd);
1127 		}
1128 		if (retval)
1129 			return retval;
1130 		/*
1131 		 * Resume roothubs unconditionally as PORTSC change bits are not
1132 		 * immediately visible after xHC reset
1133 		 */
1134 		hcd->state = HC_STATE_SUSPENDED;
1135 
1136 		if (xhci->shared_hcd) {
1137 			xhci->shared_hcd->state = HC_STATE_SUSPENDED;
1138 			usb_hcd_resume_root_hub(xhci->shared_hcd);
1139 		}
1140 		usb_hcd_resume_root_hub(hcd);
1141 
1142 		goto done;
1143 	}
1144 
1145 	/* step 4: set Run/Stop bit */
1146 	command = readl(&xhci->op_regs->command);
1147 	command |= CMD_RUN;
1148 	writel(command, &xhci->op_regs->command);
1149 	xhci_handshake(&xhci->op_regs->status, STS_HALT,
1150 		  0, 250 * 1000);
1151 
1152 	/* step 5: walk topology and initialize portsc,
1153 	 * portpmsc and portli
1154 	 */
1155 	/* this is done in bus_resume */
1156 
1157 	/* step 6: restart each of the previously
1158 	 * Running endpoints by ringing their doorbells
1159 	 */
1160 
1161 	spin_unlock_irq(&xhci->lock);
1162 
1163 	xhci_dbc_resume(xhci);
1164 
1165 	if (retval == 0) {
1166 		/*
1167 		 * Resume roothubs only if there are pending events.
1168 		 * USB 3 devices resend U3 LFPS wake after a 100ms delay if
1169 		 * the first wake signalling failed, give it that chance if
1170 		 * there are suspended USB 3 devices.
1171 		 */
1172 		if (xhci->usb3_rhub.bus_state.suspended_ports ||
1173 		    xhci->usb3_rhub.bus_state.bus_suspended)
1174 			suspended_usb3_devs = true;
1175 
1176 		pending_portevent = xhci_pending_portevent(xhci);
1177 
1178 		if (suspended_usb3_devs && !pending_portevent &&
1179 		    msg.event == PM_EVENT_AUTO_RESUME) {
1180 			msleep(120);
1181 			pending_portevent = xhci_pending_portevent(xhci);
1182 		}
1183 
1184 		if (pending_portevent) {
1185 			if (xhci->shared_hcd)
1186 				usb_hcd_resume_root_hub(xhci->shared_hcd);
1187 			usb_hcd_resume_root_hub(hcd);
1188 		}
1189 	}
1190 done:
1191 	/*
1192 	 * If system is subject to the Quirk, Compliance Mode Timer needs to
1193 	 * be re-initialized Always after a system resume. Ports are subject
1194 	 * to suffer the Compliance Mode issue again. It doesn't matter if
1195 	 * ports have entered previously to U0 before system's suspension.
1196 	 */
1197 	if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) && !comp_timer_running)
1198 		compliance_mode_recovery_timer_init(xhci);
1199 
1200 	if (xhci->quirks & XHCI_ASMEDIA_MODIFY_FLOWCONTROL)
1201 		usb_asmedia_modifyflowcontrol(to_pci_dev(hcd->self.controller));
1202 
1203 	/* Re-enable port polling. */
1204 	xhci_dbg(xhci, "%s: starting usb%d port polling.\n",
1205 		 __func__, hcd->self.busnum);
1206 	if (xhci->shared_hcd) {
1207 		set_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags);
1208 		usb_hcd_poll_rh_status(xhci->shared_hcd);
1209 	}
1210 	set_bit(HCD_FLAG_POLL_RH, &hcd->flags);
1211 	usb_hcd_poll_rh_status(hcd);
1212 
1213 	return retval;
1214 }
1215 EXPORT_SYMBOL_GPL(xhci_resume);
1216 #endif	/* CONFIG_PM */
1217 
1218 /*-------------------------------------------------------------------------*/
1219 
1220 static int xhci_map_temp_buffer(struct usb_hcd *hcd, struct urb *urb)
1221 {
1222 	void *temp;
1223 	int ret = 0;
1224 	unsigned int buf_len;
1225 	enum dma_data_direction dir;
1226 
1227 	dir = usb_urb_dir_in(urb) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
1228 	buf_len = urb->transfer_buffer_length;
1229 
1230 	temp = kzalloc_node(buf_len, GFP_ATOMIC,
1231 			    dev_to_node(hcd->self.sysdev));
1232 	if (!temp)
1233 		return -ENOMEM;
1234 
1235 	if (usb_urb_dir_out(urb))
1236 		sg_pcopy_to_buffer(urb->sg, urb->num_sgs,
1237 				   temp, buf_len, 0);
1238 
1239 	urb->transfer_buffer = temp;
1240 	urb->transfer_dma = dma_map_single(hcd->self.sysdev,
1241 					   urb->transfer_buffer,
1242 					   urb->transfer_buffer_length,
1243 					   dir);
1244 
1245 	if (dma_mapping_error(hcd->self.sysdev,
1246 			      urb->transfer_dma)) {
1247 		ret = -EAGAIN;
1248 		kfree(temp);
1249 	} else {
1250 		urb->transfer_flags |= URB_DMA_MAP_SINGLE;
1251 	}
1252 
1253 	return ret;
1254 }
1255 
1256 static bool xhci_urb_temp_buffer_required(struct usb_hcd *hcd,
1257 					  struct urb *urb)
1258 {
1259 	bool ret = false;
1260 	unsigned int i;
1261 	unsigned int len = 0;
1262 	unsigned int trb_size;
1263 	unsigned int max_pkt;
1264 	struct scatterlist *sg;
1265 	struct scatterlist *tail_sg;
1266 
1267 	tail_sg = urb->sg;
1268 	max_pkt = usb_endpoint_maxp(&urb->ep->desc);
1269 
1270 	if (!urb->num_sgs)
1271 		return ret;
1272 
1273 	if (urb->dev->speed >= USB_SPEED_SUPER)
1274 		trb_size = TRB_CACHE_SIZE_SS;
1275 	else
1276 		trb_size = TRB_CACHE_SIZE_HS;
1277 
1278 	if (urb->transfer_buffer_length != 0 &&
1279 	    !(urb->transfer_flags & URB_NO_TRANSFER_DMA_MAP)) {
1280 		for_each_sg(urb->sg, sg, urb->num_sgs, i) {
1281 			len = len + sg->length;
1282 			if (i > trb_size - 2) {
1283 				len = len - tail_sg->length;
1284 				if (len < max_pkt) {
1285 					ret = true;
1286 					break;
1287 				}
1288 
1289 				tail_sg = sg_next(tail_sg);
1290 			}
1291 		}
1292 	}
1293 	return ret;
1294 }
1295 
1296 static void xhci_unmap_temp_buf(struct usb_hcd *hcd, struct urb *urb)
1297 {
1298 	unsigned int len;
1299 	unsigned int buf_len;
1300 	enum dma_data_direction dir;
1301 
1302 	dir = usb_urb_dir_in(urb) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
1303 
1304 	buf_len = urb->transfer_buffer_length;
1305 
1306 	if (IS_ENABLED(CONFIG_HAS_DMA) &&
1307 	    (urb->transfer_flags & URB_DMA_MAP_SINGLE))
1308 		dma_unmap_single(hcd->self.sysdev,
1309 				 urb->transfer_dma,
1310 				 urb->transfer_buffer_length,
1311 				 dir);
1312 
1313 	if (usb_urb_dir_in(urb)) {
1314 		len = sg_pcopy_from_buffer(urb->sg, urb->num_sgs,
1315 					   urb->transfer_buffer,
1316 					   buf_len,
1317 					   0);
1318 		if (len != buf_len) {
1319 			xhci_dbg(hcd_to_xhci(hcd),
1320 				 "Copy from tmp buf to urb sg list failed\n");
1321 			urb->actual_length = len;
1322 		}
1323 	}
1324 	urb->transfer_flags &= ~URB_DMA_MAP_SINGLE;
1325 	kfree(urb->transfer_buffer);
1326 	urb->transfer_buffer = NULL;
1327 }
1328 
1329 /*
1330  * Bypass the DMA mapping if URB is suitable for Immediate Transfer (IDT),
1331  * we'll copy the actual data into the TRB address register. This is limited to
1332  * transfers up to 8 bytes on output endpoints of any kind with wMaxPacketSize
1333  * >= 8 bytes. If suitable for IDT only one Transfer TRB per TD is allowed.
1334  */
1335 static int xhci_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
1336 				gfp_t mem_flags)
1337 {
1338 	struct xhci_hcd *xhci;
1339 
1340 	xhci = hcd_to_xhci(hcd);
1341 
1342 	if (xhci_urb_suitable_for_idt(urb))
1343 		return 0;
1344 
1345 	if (xhci->quirks & XHCI_SG_TRB_CACHE_SIZE_QUIRK) {
1346 		if (xhci_urb_temp_buffer_required(hcd, urb))
1347 			return xhci_map_temp_buffer(hcd, urb);
1348 	}
1349 	return usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);
1350 }
1351 
1352 static void xhci_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb)
1353 {
1354 	struct xhci_hcd *xhci;
1355 	bool unmap_temp_buf = false;
1356 
1357 	xhci = hcd_to_xhci(hcd);
1358 
1359 	if (urb->num_sgs && (urb->transfer_flags & URB_DMA_MAP_SINGLE))
1360 		unmap_temp_buf = true;
1361 
1362 	if ((xhci->quirks & XHCI_SG_TRB_CACHE_SIZE_QUIRK) && unmap_temp_buf)
1363 		xhci_unmap_temp_buf(hcd, urb);
1364 	else
1365 		usb_hcd_unmap_urb_for_dma(hcd, urb);
1366 }
1367 
1368 /**
1369  * xhci_get_endpoint_index - Used for passing endpoint bitmasks between the core and
1370  * HCDs.  Find the index for an endpoint given its descriptor.  Use the return
1371  * value to right shift 1 for the bitmask.
1372  *
1373  * Index  = (epnum * 2) + direction - 1,
1374  * where direction = 0 for OUT, 1 for IN.
1375  * For control endpoints, the IN index is used (OUT index is unused), so
1376  * index = (epnum * 2) + direction - 1 = (epnum * 2) + 1 - 1 = (epnum * 2)
1377  */
1378 unsigned int xhci_get_endpoint_index(struct usb_endpoint_descriptor *desc)
1379 {
1380 	unsigned int index;
1381 	if (usb_endpoint_xfer_control(desc))
1382 		index = (unsigned int) (usb_endpoint_num(desc)*2);
1383 	else
1384 		index = (unsigned int) (usb_endpoint_num(desc)*2) +
1385 			(usb_endpoint_dir_in(desc) ? 1 : 0) - 1;
1386 	return index;
1387 }
1388 EXPORT_SYMBOL_GPL(xhci_get_endpoint_index);
1389 
1390 /* The reverse operation to xhci_get_endpoint_index. Calculate the USB endpoint
1391  * address from the XHCI endpoint index.
1392  */
1393 static unsigned int xhci_get_endpoint_address(unsigned int ep_index)
1394 {
1395 	unsigned int number = DIV_ROUND_UP(ep_index, 2);
1396 	unsigned int direction = ep_index % 2 ? USB_DIR_OUT : USB_DIR_IN;
1397 	return direction | number;
1398 }
1399 
1400 /* Find the flag for this endpoint (for use in the control context).  Use the
1401  * endpoint index to create a bitmask.  The slot context is bit 0, endpoint 0 is
1402  * bit 1, etc.
1403  */
1404 static unsigned int xhci_get_endpoint_flag(struct usb_endpoint_descriptor *desc)
1405 {
1406 	return 1 << (xhci_get_endpoint_index(desc) + 1);
1407 }
1408 
1409 /* Compute the last valid endpoint context index.  Basically, this is the
1410  * endpoint index plus one.  For slot contexts with more than valid endpoint,
1411  * we find the most significant bit set in the added contexts flags.
1412  * e.g. ep 1 IN (with epnum 0x81) => added_ctxs = 0b1000
1413  * fls(0b1000) = 4, but the endpoint context index is 3, so subtract one.
1414  */
1415 unsigned int xhci_last_valid_endpoint(u32 added_ctxs)
1416 {
1417 	return fls(added_ctxs) - 1;
1418 }
1419 
1420 /* Returns 1 if the arguments are OK;
1421  * returns 0 this is a root hub; returns -EINVAL for NULL pointers.
1422  */
1423 static int xhci_check_args(struct usb_hcd *hcd, struct usb_device *udev,
1424 		struct usb_host_endpoint *ep, int check_ep, bool check_virt_dev,
1425 		const char *func) {
1426 	struct xhci_hcd	*xhci;
1427 	struct xhci_virt_device	*virt_dev;
1428 
1429 	if (!hcd || (check_ep && !ep) || !udev) {
1430 		pr_debug("xHCI %s called with invalid args\n", func);
1431 		return -EINVAL;
1432 	}
1433 	if (!udev->parent) {
1434 		pr_debug("xHCI %s called for root hub\n", func);
1435 		return 0;
1436 	}
1437 
1438 	xhci = hcd_to_xhci(hcd);
1439 	if (check_virt_dev) {
1440 		if (!udev->slot_id || !xhci->devs[udev->slot_id]) {
1441 			xhci_dbg(xhci, "xHCI %s called with unaddressed device\n",
1442 					func);
1443 			return -EINVAL;
1444 		}
1445 
1446 		virt_dev = xhci->devs[udev->slot_id];
1447 		if (virt_dev->udev != udev) {
1448 			xhci_dbg(xhci, "xHCI %s called with udev and "
1449 					  "virt_dev does not match\n", func);
1450 			return -EINVAL;
1451 		}
1452 	}
1453 
1454 	if (xhci->xhc_state & XHCI_STATE_HALTED)
1455 		return -ENODEV;
1456 
1457 	return 1;
1458 }
1459 
1460 static int xhci_configure_endpoint(struct xhci_hcd *xhci,
1461 		struct usb_device *udev, struct xhci_command *command,
1462 		bool ctx_change, bool must_succeed);
1463 
1464 /*
1465  * Full speed devices may have a max packet size greater than 8 bytes, but the
1466  * USB core doesn't know that until it reads the first 8 bytes of the
1467  * descriptor.  If the usb_device's max packet size changes after that point,
1468  * we need to issue an evaluate context command and wait on it.
1469  */
1470 static int xhci_check_ep0_maxpacket(struct xhci_hcd *xhci, struct xhci_virt_device *vdev)
1471 {
1472 	struct xhci_input_control_ctx *ctrl_ctx;
1473 	struct xhci_ep_ctx *ep_ctx;
1474 	struct xhci_command *command;
1475 	int max_packet_size;
1476 	int hw_max_packet_size;
1477 	int ret = 0;
1478 
1479 	ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, 0);
1480 	hw_max_packet_size = MAX_PACKET_DECODED(le32_to_cpu(ep_ctx->ep_info2));
1481 	max_packet_size = usb_endpoint_maxp(&vdev->udev->ep0.desc);
1482 
1483 	if (hw_max_packet_size == max_packet_size)
1484 		return 0;
1485 
1486 	switch (max_packet_size) {
1487 	case 8: case 16: case 32: case 64: case 9:
1488 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
1489 				"Max Packet Size for ep 0 changed.");
1490 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
1491 				"Max packet size in usb_device = %d",
1492 				max_packet_size);
1493 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
1494 				"Max packet size in xHCI HW = %d",
1495 				hw_max_packet_size);
1496 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
1497 				"Issuing evaluate context command.");
1498 
1499 		command = xhci_alloc_command(xhci, true, GFP_KERNEL);
1500 		if (!command)
1501 			return -ENOMEM;
1502 
1503 		command->in_ctx = vdev->in_ctx;
1504 		ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
1505 		if (!ctrl_ctx) {
1506 			xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
1507 					__func__);
1508 			ret = -ENOMEM;
1509 			break;
1510 		}
1511 		/* Set up the modified control endpoint 0 */
1512 		xhci_endpoint_copy(xhci, vdev->in_ctx, vdev->out_ctx, 0);
1513 
1514 		ep_ctx = xhci_get_ep_ctx(xhci, command->in_ctx, 0);
1515 		ep_ctx->ep_info &= cpu_to_le32(~EP_STATE_MASK);/* must clear */
1516 		ep_ctx->ep_info2 &= cpu_to_le32(~MAX_PACKET_MASK);
1517 		ep_ctx->ep_info2 |= cpu_to_le32(MAX_PACKET(max_packet_size));
1518 
1519 		ctrl_ctx->add_flags = cpu_to_le32(EP0_FLAG);
1520 		ctrl_ctx->drop_flags = 0;
1521 
1522 		ret = xhci_configure_endpoint(xhci, vdev->udev, command,
1523 					      true, false);
1524 		/* Clean up the input context for later use by bandwidth functions */
1525 		ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG);
1526 		break;
1527 	default:
1528 		dev_dbg(&vdev->udev->dev, "incorrect max packet size %d for ep0\n",
1529 			max_packet_size);
1530 		return -EINVAL;
1531 	}
1532 
1533 	kfree(command->completion);
1534 	kfree(command);
1535 
1536 	return ret;
1537 }
1538 
1539 /*
1540  * non-error returns are a promise to giveback() the urb later
1541  * we drop ownership so next owner (or urb unlink) can get it
1542  */
1543 static int xhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flags)
1544 {
1545 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
1546 	unsigned long flags;
1547 	int ret = 0;
1548 	unsigned int slot_id, ep_index;
1549 	unsigned int *ep_state;
1550 	struct urb_priv	*urb_priv;
1551 	int num_tds;
1552 
1553 	ep_index = xhci_get_endpoint_index(&urb->ep->desc);
1554 
1555 	if (usb_endpoint_xfer_isoc(&urb->ep->desc))
1556 		num_tds = urb->number_of_packets;
1557 	else if (usb_endpoint_is_bulk_out(&urb->ep->desc) &&
1558 	    urb->transfer_buffer_length > 0 &&
1559 	    urb->transfer_flags & URB_ZERO_PACKET &&
1560 	    !(urb->transfer_buffer_length % usb_endpoint_maxp(&urb->ep->desc)))
1561 		num_tds = 2;
1562 	else
1563 		num_tds = 1;
1564 
1565 	urb_priv = kzalloc(struct_size(urb_priv, td, num_tds), mem_flags);
1566 	if (!urb_priv)
1567 		return -ENOMEM;
1568 
1569 	urb_priv->num_tds = num_tds;
1570 	urb_priv->num_tds_done = 0;
1571 	urb->hcpriv = urb_priv;
1572 
1573 	trace_xhci_urb_enqueue(urb);
1574 
1575 	spin_lock_irqsave(&xhci->lock, flags);
1576 
1577 	ret = xhci_check_args(hcd, urb->dev, urb->ep,
1578 			      true, true, __func__);
1579 	if (ret <= 0) {
1580 		ret = ret ? ret : -EINVAL;
1581 		goto free_priv;
1582 	}
1583 
1584 	slot_id = urb->dev->slot_id;
1585 
1586 	if (!HCD_HW_ACCESSIBLE(hcd)) {
1587 		ret = -ESHUTDOWN;
1588 		goto free_priv;
1589 	}
1590 
1591 	if (xhci->devs[slot_id]->flags & VDEV_PORT_ERROR) {
1592 		xhci_dbg(xhci, "Can't queue urb, port error, link inactive\n");
1593 		ret = -ENODEV;
1594 		goto free_priv;
1595 	}
1596 
1597 	if (xhci->xhc_state & XHCI_STATE_DYING) {
1598 		xhci_dbg(xhci, "Ep 0x%x: URB %p submitted for non-responsive xHCI host.\n",
1599 			 urb->ep->desc.bEndpointAddress, urb);
1600 		ret = -ESHUTDOWN;
1601 		goto free_priv;
1602 	}
1603 
1604 	ep_state = &xhci->devs[slot_id]->eps[ep_index].ep_state;
1605 
1606 	if (*ep_state & (EP_GETTING_STREAMS | EP_GETTING_NO_STREAMS)) {
1607 		xhci_warn(xhci, "WARN: Can't enqueue URB, ep in streams transition state %x\n",
1608 			  *ep_state);
1609 		ret = -EINVAL;
1610 		goto free_priv;
1611 	}
1612 	if (*ep_state & EP_SOFT_CLEAR_TOGGLE) {
1613 		xhci_warn(xhci, "Can't enqueue URB while manually clearing toggle\n");
1614 		ret = -EINVAL;
1615 		goto free_priv;
1616 	}
1617 
1618 	switch (usb_endpoint_type(&urb->ep->desc)) {
1619 
1620 	case USB_ENDPOINT_XFER_CONTROL:
1621 		ret = xhci_queue_ctrl_tx(xhci, GFP_ATOMIC, urb,
1622 					 slot_id, ep_index);
1623 		break;
1624 	case USB_ENDPOINT_XFER_BULK:
1625 		ret = xhci_queue_bulk_tx(xhci, GFP_ATOMIC, urb,
1626 					 slot_id, ep_index);
1627 		break;
1628 	case USB_ENDPOINT_XFER_INT:
1629 		ret = xhci_queue_intr_tx(xhci, GFP_ATOMIC, urb,
1630 				slot_id, ep_index);
1631 		break;
1632 	case USB_ENDPOINT_XFER_ISOC:
1633 		ret = xhci_queue_isoc_tx_prepare(xhci, GFP_ATOMIC, urb,
1634 				slot_id, ep_index);
1635 	}
1636 
1637 	if (ret) {
1638 free_priv:
1639 		xhci_urb_free_priv(urb_priv);
1640 		urb->hcpriv = NULL;
1641 	}
1642 	spin_unlock_irqrestore(&xhci->lock, flags);
1643 	return ret;
1644 }
1645 
1646 /*
1647  * Remove the URB's TD from the endpoint ring.  This may cause the HC to stop
1648  * USB transfers, potentially stopping in the middle of a TRB buffer.  The HC
1649  * should pick up where it left off in the TD, unless a Set Transfer Ring
1650  * Dequeue Pointer is issued.
1651  *
1652  * The TRBs that make up the buffers for the canceled URB will be "removed" from
1653  * the ring.  Since the ring is a contiguous structure, they can't be physically
1654  * removed.  Instead, there are two options:
1655  *
1656  *  1) If the HC is in the middle of processing the URB to be canceled, we
1657  *     simply move the ring's dequeue pointer past those TRBs using the Set
1658  *     Transfer Ring Dequeue Pointer command.  This will be the common case,
1659  *     when drivers timeout on the last submitted URB and attempt to cancel.
1660  *
1661  *  2) If the HC is in the middle of a different TD, we turn the TRBs into a
1662  *     series of 1-TRB transfer no-op TDs.  (No-ops shouldn't be chained.)  The
1663  *     HC will need to invalidate the any TRBs it has cached after the stop
1664  *     endpoint command, as noted in the xHCI 0.95 errata.
1665  *
1666  *  3) The TD may have completed by the time the Stop Endpoint Command
1667  *     completes, so software needs to handle that case too.
1668  *
1669  * This function should protect against the TD enqueueing code ringing the
1670  * doorbell while this code is waiting for a Stop Endpoint command to complete.
1671  * It also needs to account for multiple cancellations on happening at the same
1672  * time for the same endpoint.
1673  *
1674  * Note that this function can be called in any context, or so says
1675  * usb_hcd_unlink_urb()
1676  */
1677 static int xhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
1678 {
1679 	unsigned long flags;
1680 	int ret, i;
1681 	u32 temp;
1682 	struct xhci_hcd *xhci;
1683 	struct urb_priv	*urb_priv;
1684 	struct xhci_td *td;
1685 	unsigned int ep_index;
1686 	struct xhci_ring *ep_ring;
1687 	struct xhci_virt_ep *ep;
1688 	struct xhci_command *command;
1689 	struct xhci_virt_device *vdev;
1690 
1691 	xhci = hcd_to_xhci(hcd);
1692 	spin_lock_irqsave(&xhci->lock, flags);
1693 
1694 	trace_xhci_urb_dequeue(urb);
1695 
1696 	/* Make sure the URB hasn't completed or been unlinked already */
1697 	ret = usb_hcd_check_unlink_urb(hcd, urb, status);
1698 	if (ret)
1699 		goto done;
1700 
1701 	/* give back URB now if we can't queue it for cancel */
1702 	vdev = xhci->devs[urb->dev->slot_id];
1703 	urb_priv = urb->hcpriv;
1704 	if (!vdev || !urb_priv)
1705 		goto err_giveback;
1706 
1707 	ep_index = xhci_get_endpoint_index(&urb->ep->desc);
1708 	ep = &vdev->eps[ep_index];
1709 	ep_ring = xhci_urb_to_transfer_ring(xhci, urb);
1710 	if (!ep || !ep_ring)
1711 		goto err_giveback;
1712 
1713 	/* If xHC is dead take it down and return ALL URBs in xhci_hc_died() */
1714 	temp = readl(&xhci->op_regs->status);
1715 	if (temp == ~(u32)0 || xhci->xhc_state & XHCI_STATE_DYING) {
1716 		xhci_hc_died(xhci);
1717 		goto done;
1718 	}
1719 
1720 	/*
1721 	 * check ring is not re-allocated since URB was enqueued. If it is, then
1722 	 * make sure none of the ring related pointers in this URB private data
1723 	 * are touched, such as td_list, otherwise we overwrite freed data
1724 	 */
1725 	if (!td_on_ring(&urb_priv->td[0], ep_ring)) {
1726 		xhci_err(xhci, "Canceled URB td not found on endpoint ring");
1727 		for (i = urb_priv->num_tds_done; i < urb_priv->num_tds; i++) {
1728 			td = &urb_priv->td[i];
1729 			if (!list_empty(&td->cancelled_td_list))
1730 				list_del_init(&td->cancelled_td_list);
1731 		}
1732 		goto err_giveback;
1733 	}
1734 
1735 	if (xhci->xhc_state & XHCI_STATE_HALTED) {
1736 		xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1737 				"HC halted, freeing TD manually.");
1738 		for (i = urb_priv->num_tds_done;
1739 		     i < urb_priv->num_tds;
1740 		     i++) {
1741 			td = &urb_priv->td[i];
1742 			if (!list_empty(&td->td_list))
1743 				list_del_init(&td->td_list);
1744 			if (!list_empty(&td->cancelled_td_list))
1745 				list_del_init(&td->cancelled_td_list);
1746 		}
1747 		goto err_giveback;
1748 	}
1749 
1750 	i = urb_priv->num_tds_done;
1751 	if (i < urb_priv->num_tds)
1752 		xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1753 				"Cancel URB %p, dev %s, ep 0x%x, "
1754 				"starting at offset 0x%llx",
1755 				urb, urb->dev->devpath,
1756 				urb->ep->desc.bEndpointAddress,
1757 				(unsigned long long) xhci_trb_virt_to_dma(
1758 					urb_priv->td[i].start_seg,
1759 					urb_priv->td[i].first_trb));
1760 
1761 	for (; i < urb_priv->num_tds; i++) {
1762 		td = &urb_priv->td[i];
1763 		/* TD can already be on cancelled list if ep halted on it */
1764 		if (list_empty(&td->cancelled_td_list)) {
1765 			td->cancel_status = TD_DIRTY;
1766 			list_add_tail(&td->cancelled_td_list,
1767 				      &ep->cancelled_td_list);
1768 		}
1769 	}
1770 
1771 	/* Queue a stop endpoint command, but only if this is
1772 	 * the first cancellation to be handled.
1773 	 */
1774 	if (!(ep->ep_state & EP_STOP_CMD_PENDING)) {
1775 		command = xhci_alloc_command(xhci, false, GFP_ATOMIC);
1776 		if (!command) {
1777 			ret = -ENOMEM;
1778 			goto done;
1779 		}
1780 		ep->ep_state |= EP_STOP_CMD_PENDING;
1781 		xhci_queue_stop_endpoint(xhci, command, urb->dev->slot_id,
1782 					 ep_index, 0);
1783 		xhci_ring_cmd_db(xhci);
1784 	}
1785 done:
1786 	spin_unlock_irqrestore(&xhci->lock, flags);
1787 	return ret;
1788 
1789 err_giveback:
1790 	if (urb_priv)
1791 		xhci_urb_free_priv(urb_priv);
1792 	usb_hcd_unlink_urb_from_ep(hcd, urb);
1793 	spin_unlock_irqrestore(&xhci->lock, flags);
1794 	usb_hcd_giveback_urb(hcd, urb, -ESHUTDOWN);
1795 	return ret;
1796 }
1797 
1798 /* Drop an endpoint from a new bandwidth configuration for this device.
1799  * Only one call to this function is allowed per endpoint before
1800  * check_bandwidth() or reset_bandwidth() must be called.
1801  * A call to xhci_drop_endpoint() followed by a call to xhci_add_endpoint() will
1802  * add the endpoint to the schedule with possibly new parameters denoted by a
1803  * different endpoint descriptor in usb_host_endpoint.
1804  * A call to xhci_add_endpoint() followed by a call to xhci_drop_endpoint() is
1805  * not allowed.
1806  *
1807  * The USB core will not allow URBs to be queued to an endpoint that is being
1808  * disabled, so there's no need for mutual exclusion to protect
1809  * the xhci->devs[slot_id] structure.
1810  */
1811 int xhci_drop_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
1812 		       struct usb_host_endpoint *ep)
1813 {
1814 	struct xhci_hcd *xhci;
1815 	struct xhci_container_ctx *in_ctx, *out_ctx;
1816 	struct xhci_input_control_ctx *ctrl_ctx;
1817 	unsigned int ep_index;
1818 	struct xhci_ep_ctx *ep_ctx;
1819 	u32 drop_flag;
1820 	u32 new_add_flags, new_drop_flags;
1821 	int ret;
1822 
1823 	ret = xhci_check_args(hcd, udev, ep, 1, true, __func__);
1824 	if (ret <= 0)
1825 		return ret;
1826 	xhci = hcd_to_xhci(hcd);
1827 	if (xhci->xhc_state & XHCI_STATE_DYING)
1828 		return -ENODEV;
1829 
1830 	xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
1831 	drop_flag = xhci_get_endpoint_flag(&ep->desc);
1832 	if (drop_flag == SLOT_FLAG || drop_flag == EP0_FLAG) {
1833 		xhci_dbg(xhci, "xHCI %s - can't drop slot or ep 0 %#x\n",
1834 				__func__, drop_flag);
1835 		return 0;
1836 	}
1837 
1838 	in_ctx = xhci->devs[udev->slot_id]->in_ctx;
1839 	out_ctx = xhci->devs[udev->slot_id]->out_ctx;
1840 	ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
1841 	if (!ctrl_ctx) {
1842 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
1843 				__func__);
1844 		return 0;
1845 	}
1846 
1847 	ep_index = xhci_get_endpoint_index(&ep->desc);
1848 	ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
1849 	/* If the HC already knows the endpoint is disabled,
1850 	 * or the HCD has noted it is disabled, ignore this request
1851 	 */
1852 	if ((GET_EP_CTX_STATE(ep_ctx) == EP_STATE_DISABLED) ||
1853 	    le32_to_cpu(ctrl_ctx->drop_flags) &
1854 	    xhci_get_endpoint_flag(&ep->desc)) {
1855 		/* Do not warn when called after a usb_device_reset */
1856 		if (xhci->devs[udev->slot_id]->eps[ep_index].ring != NULL)
1857 			xhci_warn(xhci, "xHCI %s called with disabled ep %p\n",
1858 				  __func__, ep);
1859 		return 0;
1860 	}
1861 
1862 	ctrl_ctx->drop_flags |= cpu_to_le32(drop_flag);
1863 	new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
1864 
1865 	ctrl_ctx->add_flags &= cpu_to_le32(~drop_flag);
1866 	new_add_flags = le32_to_cpu(ctrl_ctx->add_flags);
1867 
1868 	xhci_debugfs_remove_endpoint(xhci, xhci->devs[udev->slot_id], ep_index);
1869 
1870 	xhci_endpoint_zero(xhci, xhci->devs[udev->slot_id], ep);
1871 
1872 	xhci_dbg(xhci, "drop ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x\n",
1873 			(unsigned int) ep->desc.bEndpointAddress,
1874 			udev->slot_id,
1875 			(unsigned int) new_drop_flags,
1876 			(unsigned int) new_add_flags);
1877 	return 0;
1878 }
1879 EXPORT_SYMBOL_GPL(xhci_drop_endpoint);
1880 
1881 /* Add an endpoint to a new possible bandwidth configuration for this device.
1882  * Only one call to this function is allowed per endpoint before
1883  * check_bandwidth() or reset_bandwidth() must be called.
1884  * A call to xhci_drop_endpoint() followed by a call to xhci_add_endpoint() will
1885  * add the endpoint to the schedule with possibly new parameters denoted by a
1886  * different endpoint descriptor in usb_host_endpoint.
1887  * A call to xhci_add_endpoint() followed by a call to xhci_drop_endpoint() is
1888  * not allowed.
1889  *
1890  * The USB core will not allow URBs to be queued to an endpoint until the
1891  * configuration or alt setting is installed in the device, so there's no need
1892  * for mutual exclusion to protect the xhci->devs[slot_id] structure.
1893  */
1894 int xhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
1895 		      struct usb_host_endpoint *ep)
1896 {
1897 	struct xhci_hcd *xhci;
1898 	struct xhci_container_ctx *in_ctx;
1899 	unsigned int ep_index;
1900 	struct xhci_input_control_ctx *ctrl_ctx;
1901 	struct xhci_ep_ctx *ep_ctx;
1902 	u32 added_ctxs;
1903 	u32 new_add_flags, new_drop_flags;
1904 	struct xhci_virt_device *virt_dev;
1905 	int ret = 0;
1906 
1907 	ret = xhci_check_args(hcd, udev, ep, 1, true, __func__);
1908 	if (ret <= 0) {
1909 		/* So we won't queue a reset ep command for a root hub */
1910 		ep->hcpriv = NULL;
1911 		return ret;
1912 	}
1913 	xhci = hcd_to_xhci(hcd);
1914 	if (xhci->xhc_state & XHCI_STATE_DYING)
1915 		return -ENODEV;
1916 
1917 	added_ctxs = xhci_get_endpoint_flag(&ep->desc);
1918 	if (added_ctxs == SLOT_FLAG || added_ctxs == EP0_FLAG) {
1919 		/* FIXME when we have to issue an evaluate endpoint command to
1920 		 * deal with ep0 max packet size changing once we get the
1921 		 * descriptors
1922 		 */
1923 		xhci_dbg(xhci, "xHCI %s - can't add slot or ep 0 %#x\n",
1924 				__func__, added_ctxs);
1925 		return 0;
1926 	}
1927 
1928 	virt_dev = xhci->devs[udev->slot_id];
1929 	in_ctx = virt_dev->in_ctx;
1930 	ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
1931 	if (!ctrl_ctx) {
1932 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
1933 				__func__);
1934 		return 0;
1935 	}
1936 
1937 	ep_index = xhci_get_endpoint_index(&ep->desc);
1938 	/* If this endpoint is already in use, and the upper layers are trying
1939 	 * to add it again without dropping it, reject the addition.
1940 	 */
1941 	if (virt_dev->eps[ep_index].ring &&
1942 			!(le32_to_cpu(ctrl_ctx->drop_flags) & added_ctxs)) {
1943 		xhci_warn(xhci, "Trying to add endpoint 0x%x "
1944 				"without dropping it.\n",
1945 				(unsigned int) ep->desc.bEndpointAddress);
1946 		return -EINVAL;
1947 	}
1948 
1949 	/* If the HCD has already noted the endpoint is enabled,
1950 	 * ignore this request.
1951 	 */
1952 	if (le32_to_cpu(ctrl_ctx->add_flags) & added_ctxs) {
1953 		xhci_warn(xhci, "xHCI %s called with enabled ep %p\n",
1954 				__func__, ep);
1955 		return 0;
1956 	}
1957 
1958 	/*
1959 	 * Configuration and alternate setting changes must be done in
1960 	 * process context, not interrupt context (or so documenation
1961 	 * for usb_set_interface() and usb_set_configuration() claim).
1962 	 */
1963 	if (xhci_endpoint_init(xhci, virt_dev, udev, ep, GFP_NOIO) < 0) {
1964 		dev_dbg(&udev->dev, "%s - could not initialize ep %#x\n",
1965 				__func__, ep->desc.bEndpointAddress);
1966 		return -ENOMEM;
1967 	}
1968 
1969 	ctrl_ctx->add_flags |= cpu_to_le32(added_ctxs);
1970 	new_add_flags = le32_to_cpu(ctrl_ctx->add_flags);
1971 
1972 	/* If xhci_endpoint_disable() was called for this endpoint, but the
1973 	 * xHC hasn't been notified yet through the check_bandwidth() call,
1974 	 * this re-adds a new state for the endpoint from the new endpoint
1975 	 * descriptors.  We must drop and re-add this endpoint, so we leave the
1976 	 * drop flags alone.
1977 	 */
1978 	new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
1979 
1980 	/* Store the usb_device pointer for later use */
1981 	ep->hcpriv = udev;
1982 
1983 	ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index);
1984 	trace_xhci_add_endpoint(ep_ctx);
1985 
1986 	xhci_dbg(xhci, "add ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x\n",
1987 			(unsigned int) ep->desc.bEndpointAddress,
1988 			udev->slot_id,
1989 			(unsigned int) new_drop_flags,
1990 			(unsigned int) new_add_flags);
1991 	return 0;
1992 }
1993 EXPORT_SYMBOL_GPL(xhci_add_endpoint);
1994 
1995 static void xhci_zero_in_ctx(struct xhci_hcd *xhci, struct xhci_virt_device *virt_dev)
1996 {
1997 	struct xhci_input_control_ctx *ctrl_ctx;
1998 	struct xhci_ep_ctx *ep_ctx;
1999 	struct xhci_slot_ctx *slot_ctx;
2000 	int i;
2001 
2002 	ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
2003 	if (!ctrl_ctx) {
2004 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
2005 				__func__);
2006 		return;
2007 	}
2008 
2009 	/* When a device's add flag and drop flag are zero, any subsequent
2010 	 * configure endpoint command will leave that endpoint's state
2011 	 * untouched.  Make sure we don't leave any old state in the input
2012 	 * endpoint contexts.
2013 	 */
2014 	ctrl_ctx->drop_flags = 0;
2015 	ctrl_ctx->add_flags = 0;
2016 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
2017 	slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
2018 	/* Endpoint 0 is always valid */
2019 	slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1));
2020 	for (i = 1; i < 31; i++) {
2021 		ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, i);
2022 		ep_ctx->ep_info = 0;
2023 		ep_ctx->ep_info2 = 0;
2024 		ep_ctx->deq = 0;
2025 		ep_ctx->tx_info = 0;
2026 	}
2027 }
2028 
2029 static int xhci_configure_endpoint_result(struct xhci_hcd *xhci,
2030 		struct usb_device *udev, u32 *cmd_status)
2031 {
2032 	int ret;
2033 
2034 	switch (*cmd_status) {
2035 	case COMP_COMMAND_ABORTED:
2036 	case COMP_COMMAND_RING_STOPPED:
2037 		xhci_warn(xhci, "Timeout while waiting for configure endpoint command\n");
2038 		ret = -ETIME;
2039 		break;
2040 	case COMP_RESOURCE_ERROR:
2041 		dev_warn(&udev->dev,
2042 			 "Not enough host controller resources for new device state.\n");
2043 		ret = -ENOMEM;
2044 		/* FIXME: can we allocate more resources for the HC? */
2045 		break;
2046 	case COMP_BANDWIDTH_ERROR:
2047 	case COMP_SECONDARY_BANDWIDTH_ERROR:
2048 		dev_warn(&udev->dev,
2049 			 "Not enough bandwidth for new device state.\n");
2050 		ret = -ENOSPC;
2051 		/* FIXME: can we go back to the old state? */
2052 		break;
2053 	case COMP_TRB_ERROR:
2054 		/* the HCD set up something wrong */
2055 		dev_warn(&udev->dev, "ERROR: Endpoint drop flag = 0, "
2056 				"add flag = 1, "
2057 				"and endpoint is not disabled.\n");
2058 		ret = -EINVAL;
2059 		break;
2060 	case COMP_INCOMPATIBLE_DEVICE_ERROR:
2061 		dev_warn(&udev->dev,
2062 			 "ERROR: Incompatible device for endpoint configure command.\n");
2063 		ret = -ENODEV;
2064 		break;
2065 	case COMP_SUCCESS:
2066 		xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
2067 				"Successful Endpoint Configure command");
2068 		ret = 0;
2069 		break;
2070 	default:
2071 		xhci_err(xhci, "ERROR: unexpected command completion code 0x%x.\n",
2072 				*cmd_status);
2073 		ret = -EINVAL;
2074 		break;
2075 	}
2076 	return ret;
2077 }
2078 
2079 static int xhci_evaluate_context_result(struct xhci_hcd *xhci,
2080 		struct usb_device *udev, u32 *cmd_status)
2081 {
2082 	int ret;
2083 
2084 	switch (*cmd_status) {
2085 	case COMP_COMMAND_ABORTED:
2086 	case COMP_COMMAND_RING_STOPPED:
2087 		xhci_warn(xhci, "Timeout while waiting for evaluate context command\n");
2088 		ret = -ETIME;
2089 		break;
2090 	case COMP_PARAMETER_ERROR:
2091 		dev_warn(&udev->dev,
2092 			 "WARN: xHCI driver setup invalid evaluate context command.\n");
2093 		ret = -EINVAL;
2094 		break;
2095 	case COMP_SLOT_NOT_ENABLED_ERROR:
2096 		dev_warn(&udev->dev,
2097 			"WARN: slot not enabled for evaluate context command.\n");
2098 		ret = -EINVAL;
2099 		break;
2100 	case COMP_CONTEXT_STATE_ERROR:
2101 		dev_warn(&udev->dev,
2102 			"WARN: invalid context state for evaluate context command.\n");
2103 		ret = -EINVAL;
2104 		break;
2105 	case COMP_INCOMPATIBLE_DEVICE_ERROR:
2106 		dev_warn(&udev->dev,
2107 			"ERROR: Incompatible device for evaluate context command.\n");
2108 		ret = -ENODEV;
2109 		break;
2110 	case COMP_MAX_EXIT_LATENCY_TOO_LARGE_ERROR:
2111 		/* Max Exit Latency too large error */
2112 		dev_warn(&udev->dev, "WARN: Max Exit Latency too large\n");
2113 		ret = -EINVAL;
2114 		break;
2115 	case COMP_SUCCESS:
2116 		xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
2117 				"Successful evaluate context command");
2118 		ret = 0;
2119 		break;
2120 	default:
2121 		xhci_err(xhci, "ERROR: unexpected command completion code 0x%x.\n",
2122 			*cmd_status);
2123 		ret = -EINVAL;
2124 		break;
2125 	}
2126 	return ret;
2127 }
2128 
2129 static u32 xhci_count_num_new_endpoints(struct xhci_hcd *xhci,
2130 		struct xhci_input_control_ctx *ctrl_ctx)
2131 {
2132 	u32 valid_add_flags;
2133 	u32 valid_drop_flags;
2134 
2135 	/* Ignore the slot flag (bit 0), and the default control endpoint flag
2136 	 * (bit 1).  The default control endpoint is added during the Address
2137 	 * Device command and is never removed until the slot is disabled.
2138 	 */
2139 	valid_add_flags = le32_to_cpu(ctrl_ctx->add_flags) >> 2;
2140 	valid_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags) >> 2;
2141 
2142 	/* Use hweight32 to count the number of ones in the add flags, or
2143 	 * number of endpoints added.  Don't count endpoints that are changed
2144 	 * (both added and dropped).
2145 	 */
2146 	return hweight32(valid_add_flags) -
2147 		hweight32(valid_add_flags & valid_drop_flags);
2148 }
2149 
2150 static unsigned int xhci_count_num_dropped_endpoints(struct xhci_hcd *xhci,
2151 		struct xhci_input_control_ctx *ctrl_ctx)
2152 {
2153 	u32 valid_add_flags;
2154 	u32 valid_drop_flags;
2155 
2156 	valid_add_flags = le32_to_cpu(ctrl_ctx->add_flags) >> 2;
2157 	valid_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags) >> 2;
2158 
2159 	return hweight32(valid_drop_flags) -
2160 		hweight32(valid_add_flags & valid_drop_flags);
2161 }
2162 
2163 /*
2164  * We need to reserve the new number of endpoints before the configure endpoint
2165  * command completes.  We can't subtract the dropped endpoints from the number
2166  * of active endpoints until the command completes because we can oversubscribe
2167  * the host in this case:
2168  *
2169  *  - the first configure endpoint command drops more endpoints than it adds
2170  *  - a second configure endpoint command that adds more endpoints is queued
2171  *  - the first configure endpoint command fails, so the config is unchanged
2172  *  - the second command may succeed, even though there isn't enough resources
2173  *
2174  * Must be called with xhci->lock held.
2175  */
2176 static int xhci_reserve_host_resources(struct xhci_hcd *xhci,
2177 		struct xhci_input_control_ctx *ctrl_ctx)
2178 {
2179 	u32 added_eps;
2180 
2181 	added_eps = xhci_count_num_new_endpoints(xhci, ctrl_ctx);
2182 	if (xhci->num_active_eps + added_eps > xhci->limit_active_eps) {
2183 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2184 				"Not enough ep ctxs: "
2185 				"%u active, need to add %u, limit is %u.",
2186 				xhci->num_active_eps, added_eps,
2187 				xhci->limit_active_eps);
2188 		return -ENOMEM;
2189 	}
2190 	xhci->num_active_eps += added_eps;
2191 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2192 			"Adding %u ep ctxs, %u now active.", added_eps,
2193 			xhci->num_active_eps);
2194 	return 0;
2195 }
2196 
2197 /*
2198  * The configure endpoint was failed by the xHC for some other reason, so we
2199  * need to revert the resources that failed configuration would have used.
2200  *
2201  * Must be called with xhci->lock held.
2202  */
2203 static void xhci_free_host_resources(struct xhci_hcd *xhci,
2204 		struct xhci_input_control_ctx *ctrl_ctx)
2205 {
2206 	u32 num_failed_eps;
2207 
2208 	num_failed_eps = xhci_count_num_new_endpoints(xhci, ctrl_ctx);
2209 	xhci->num_active_eps -= num_failed_eps;
2210 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2211 			"Removing %u failed ep ctxs, %u now active.",
2212 			num_failed_eps,
2213 			xhci->num_active_eps);
2214 }
2215 
2216 /*
2217  * Now that the command has completed, clean up the active endpoint count by
2218  * subtracting out the endpoints that were dropped (but not changed).
2219  *
2220  * Must be called with xhci->lock held.
2221  */
2222 static void xhci_finish_resource_reservation(struct xhci_hcd *xhci,
2223 		struct xhci_input_control_ctx *ctrl_ctx)
2224 {
2225 	u32 num_dropped_eps;
2226 
2227 	num_dropped_eps = xhci_count_num_dropped_endpoints(xhci, ctrl_ctx);
2228 	xhci->num_active_eps -= num_dropped_eps;
2229 	if (num_dropped_eps)
2230 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2231 				"Removing %u dropped ep ctxs, %u now active.",
2232 				num_dropped_eps,
2233 				xhci->num_active_eps);
2234 }
2235 
2236 static unsigned int xhci_get_block_size(struct usb_device *udev)
2237 {
2238 	switch (udev->speed) {
2239 	case USB_SPEED_LOW:
2240 	case USB_SPEED_FULL:
2241 		return FS_BLOCK;
2242 	case USB_SPEED_HIGH:
2243 		return HS_BLOCK;
2244 	case USB_SPEED_SUPER:
2245 	case USB_SPEED_SUPER_PLUS:
2246 		return SS_BLOCK;
2247 	case USB_SPEED_UNKNOWN:
2248 	default:
2249 		/* Should never happen */
2250 		return 1;
2251 	}
2252 }
2253 
2254 static unsigned int
2255 xhci_get_largest_overhead(struct xhci_interval_bw *interval_bw)
2256 {
2257 	if (interval_bw->overhead[LS_OVERHEAD_TYPE])
2258 		return LS_OVERHEAD;
2259 	if (interval_bw->overhead[FS_OVERHEAD_TYPE])
2260 		return FS_OVERHEAD;
2261 	return HS_OVERHEAD;
2262 }
2263 
2264 /* If we are changing a LS/FS device under a HS hub,
2265  * make sure (if we are activating a new TT) that the HS bus has enough
2266  * bandwidth for this new TT.
2267  */
2268 static int xhci_check_tt_bw_table(struct xhci_hcd *xhci,
2269 		struct xhci_virt_device *virt_dev,
2270 		int old_active_eps)
2271 {
2272 	struct xhci_interval_bw_table *bw_table;
2273 	struct xhci_tt_bw_info *tt_info;
2274 
2275 	/* Find the bandwidth table for the root port this TT is attached to. */
2276 	bw_table = &xhci->rh_bw[virt_dev->rhub_port->hw_portnum].bw_table;
2277 	tt_info = virt_dev->tt_info;
2278 	/* If this TT already had active endpoints, the bandwidth for this TT
2279 	 * has already been added.  Removing all periodic endpoints (and thus
2280 	 * making the TT enactive) will only decrease the bandwidth used.
2281 	 */
2282 	if (old_active_eps)
2283 		return 0;
2284 	if (old_active_eps == 0 && tt_info->active_eps != 0) {
2285 		if (bw_table->bw_used + TT_HS_OVERHEAD > HS_BW_LIMIT)
2286 			return -ENOMEM;
2287 		return 0;
2288 	}
2289 	/* Not sure why we would have no new active endpoints...
2290 	 *
2291 	 * Maybe because of an Evaluate Context change for a hub update or a
2292 	 * control endpoint 0 max packet size change?
2293 	 * FIXME: skip the bandwidth calculation in that case.
2294 	 */
2295 	return 0;
2296 }
2297 
2298 static int xhci_check_ss_bw(struct xhci_hcd *xhci,
2299 		struct xhci_virt_device *virt_dev)
2300 {
2301 	unsigned int bw_reserved;
2302 
2303 	bw_reserved = DIV_ROUND_UP(SS_BW_RESERVED*SS_BW_LIMIT_IN, 100);
2304 	if (virt_dev->bw_table->ss_bw_in > (SS_BW_LIMIT_IN - bw_reserved))
2305 		return -ENOMEM;
2306 
2307 	bw_reserved = DIV_ROUND_UP(SS_BW_RESERVED*SS_BW_LIMIT_OUT, 100);
2308 	if (virt_dev->bw_table->ss_bw_out > (SS_BW_LIMIT_OUT - bw_reserved))
2309 		return -ENOMEM;
2310 
2311 	return 0;
2312 }
2313 
2314 /*
2315  * This algorithm is a very conservative estimate of the worst-case scheduling
2316  * scenario for any one interval.  The hardware dynamically schedules the
2317  * packets, so we can't tell which microframe could be the limiting factor in
2318  * the bandwidth scheduling.  This only takes into account periodic endpoints.
2319  *
2320  * Obviously, we can't solve an NP complete problem to find the minimum worst
2321  * case scenario.  Instead, we come up with an estimate that is no less than
2322  * the worst case bandwidth used for any one microframe, but may be an
2323  * over-estimate.
2324  *
2325  * We walk the requirements for each endpoint by interval, starting with the
2326  * smallest interval, and place packets in the schedule where there is only one
2327  * possible way to schedule packets for that interval.  In order to simplify
2328  * this algorithm, we record the largest max packet size for each interval, and
2329  * assume all packets will be that size.
2330  *
2331  * For interval 0, we obviously must schedule all packets for each interval.
2332  * The bandwidth for interval 0 is just the amount of data to be transmitted
2333  * (the sum of all max ESIT payload sizes, plus any overhead per packet times
2334  * the number of packets).
2335  *
2336  * For interval 1, we have two possible microframes to schedule those packets
2337  * in.  For this algorithm, if we can schedule the same number of packets for
2338  * each possible scheduling opportunity (each microframe), we will do so.  The
2339  * remaining number of packets will be saved to be transmitted in the gaps in
2340  * the next interval's scheduling sequence.
2341  *
2342  * As we move those remaining packets to be scheduled with interval 2 packets,
2343  * we have to double the number of remaining packets to transmit.  This is
2344  * because the intervals are actually powers of 2, and we would be transmitting
2345  * the previous interval's packets twice in this interval.  We also have to be
2346  * sure that when we look at the largest max packet size for this interval, we
2347  * also look at the largest max packet size for the remaining packets and take
2348  * the greater of the two.
2349  *
2350  * The algorithm continues to evenly distribute packets in each scheduling
2351  * opportunity, and push the remaining packets out, until we get to the last
2352  * interval.  Then those packets and their associated overhead are just added
2353  * to the bandwidth used.
2354  */
2355 static int xhci_check_bw_table(struct xhci_hcd *xhci,
2356 		struct xhci_virt_device *virt_dev,
2357 		int old_active_eps)
2358 {
2359 	unsigned int bw_reserved;
2360 	unsigned int max_bandwidth;
2361 	unsigned int bw_used;
2362 	unsigned int block_size;
2363 	struct xhci_interval_bw_table *bw_table;
2364 	unsigned int packet_size = 0;
2365 	unsigned int overhead = 0;
2366 	unsigned int packets_transmitted = 0;
2367 	unsigned int packets_remaining = 0;
2368 	unsigned int i;
2369 
2370 	if (virt_dev->udev->speed >= USB_SPEED_SUPER)
2371 		return xhci_check_ss_bw(xhci, virt_dev);
2372 
2373 	if (virt_dev->udev->speed == USB_SPEED_HIGH) {
2374 		max_bandwidth = HS_BW_LIMIT;
2375 		/* Convert percent of bus BW reserved to blocks reserved */
2376 		bw_reserved = DIV_ROUND_UP(HS_BW_RESERVED * max_bandwidth, 100);
2377 	} else {
2378 		max_bandwidth = FS_BW_LIMIT;
2379 		bw_reserved = DIV_ROUND_UP(FS_BW_RESERVED * max_bandwidth, 100);
2380 	}
2381 
2382 	bw_table = virt_dev->bw_table;
2383 	/* We need to translate the max packet size and max ESIT payloads into
2384 	 * the units the hardware uses.
2385 	 */
2386 	block_size = xhci_get_block_size(virt_dev->udev);
2387 
2388 	/* If we are manipulating a LS/FS device under a HS hub, double check
2389 	 * that the HS bus has enough bandwidth if we are activing a new TT.
2390 	 */
2391 	if (virt_dev->tt_info) {
2392 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2393 				"Recalculating BW for rootport %u",
2394 				virt_dev->rhub_port->hw_portnum + 1);
2395 		if (xhci_check_tt_bw_table(xhci, virt_dev, old_active_eps)) {
2396 			xhci_warn(xhci, "Not enough bandwidth on HS bus for "
2397 					"newly activated TT.\n");
2398 			return -ENOMEM;
2399 		}
2400 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2401 				"Recalculating BW for TT slot %u port %u",
2402 				virt_dev->tt_info->slot_id,
2403 				virt_dev->tt_info->ttport);
2404 	} else {
2405 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2406 				"Recalculating BW for rootport %u",
2407 				virt_dev->rhub_port->hw_portnum + 1);
2408 	}
2409 
2410 	/* Add in how much bandwidth will be used for interval zero, or the
2411 	 * rounded max ESIT payload + number of packets * largest overhead.
2412 	 */
2413 	bw_used = DIV_ROUND_UP(bw_table->interval0_esit_payload, block_size) +
2414 		bw_table->interval_bw[0].num_packets *
2415 		xhci_get_largest_overhead(&bw_table->interval_bw[0]);
2416 
2417 	for (i = 1; i < XHCI_MAX_INTERVAL; i++) {
2418 		unsigned int bw_added;
2419 		unsigned int largest_mps;
2420 		unsigned int interval_overhead;
2421 
2422 		/*
2423 		 * How many packets could we transmit in this interval?
2424 		 * If packets didn't fit in the previous interval, we will need
2425 		 * to transmit that many packets twice within this interval.
2426 		 */
2427 		packets_remaining = 2 * packets_remaining +
2428 			bw_table->interval_bw[i].num_packets;
2429 
2430 		/* Find the largest max packet size of this or the previous
2431 		 * interval.
2432 		 */
2433 		if (list_empty(&bw_table->interval_bw[i].endpoints))
2434 			largest_mps = 0;
2435 		else {
2436 			struct xhci_virt_ep *virt_ep;
2437 			struct list_head *ep_entry;
2438 
2439 			ep_entry = bw_table->interval_bw[i].endpoints.next;
2440 			virt_ep = list_entry(ep_entry,
2441 					struct xhci_virt_ep, bw_endpoint_list);
2442 			/* Convert to blocks, rounding up */
2443 			largest_mps = DIV_ROUND_UP(
2444 					virt_ep->bw_info.max_packet_size,
2445 					block_size);
2446 		}
2447 		if (largest_mps > packet_size)
2448 			packet_size = largest_mps;
2449 
2450 		/* Use the larger overhead of this or the previous interval. */
2451 		interval_overhead = xhci_get_largest_overhead(
2452 				&bw_table->interval_bw[i]);
2453 		if (interval_overhead > overhead)
2454 			overhead = interval_overhead;
2455 
2456 		/* How many packets can we evenly distribute across
2457 		 * (1 << (i + 1)) possible scheduling opportunities?
2458 		 */
2459 		packets_transmitted = packets_remaining >> (i + 1);
2460 
2461 		/* Add in the bandwidth used for those scheduled packets */
2462 		bw_added = packets_transmitted * (overhead + packet_size);
2463 
2464 		/* How many packets do we have remaining to transmit? */
2465 		packets_remaining = packets_remaining % (1 << (i + 1));
2466 
2467 		/* What largest max packet size should those packets have? */
2468 		/* If we've transmitted all packets, don't carry over the
2469 		 * largest packet size.
2470 		 */
2471 		if (packets_remaining == 0) {
2472 			packet_size = 0;
2473 			overhead = 0;
2474 		} else if (packets_transmitted > 0) {
2475 			/* Otherwise if we do have remaining packets, and we've
2476 			 * scheduled some packets in this interval, take the
2477 			 * largest max packet size from endpoints with this
2478 			 * interval.
2479 			 */
2480 			packet_size = largest_mps;
2481 			overhead = interval_overhead;
2482 		}
2483 		/* Otherwise carry over packet_size and overhead from the last
2484 		 * time we had a remainder.
2485 		 */
2486 		bw_used += bw_added;
2487 		if (bw_used > max_bandwidth) {
2488 			xhci_warn(xhci, "Not enough bandwidth. "
2489 					"Proposed: %u, Max: %u\n",
2490 				bw_used, max_bandwidth);
2491 			return -ENOMEM;
2492 		}
2493 	}
2494 	/*
2495 	 * Ok, we know we have some packets left over after even-handedly
2496 	 * scheduling interval 15.  We don't know which microframes they will
2497 	 * fit into, so we over-schedule and say they will be scheduled every
2498 	 * microframe.
2499 	 */
2500 	if (packets_remaining > 0)
2501 		bw_used += overhead + packet_size;
2502 
2503 	if (!virt_dev->tt_info && virt_dev->udev->speed == USB_SPEED_HIGH) {
2504 		/* OK, we're manipulating a HS device attached to a
2505 		 * root port bandwidth domain.  Include the number of active TTs
2506 		 * in the bandwidth used.
2507 		 */
2508 		bw_used += TT_HS_OVERHEAD *
2509 			xhci->rh_bw[virt_dev->rhub_port->hw_portnum].num_active_tts;
2510 	}
2511 
2512 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2513 		"Final bandwidth: %u, Limit: %u, Reserved: %u, "
2514 		"Available: %u " "percent",
2515 		bw_used, max_bandwidth, bw_reserved,
2516 		(max_bandwidth - bw_used - bw_reserved) * 100 /
2517 		max_bandwidth);
2518 
2519 	bw_used += bw_reserved;
2520 	if (bw_used > max_bandwidth) {
2521 		xhci_warn(xhci, "Not enough bandwidth. Proposed: %u, Max: %u\n",
2522 				bw_used, max_bandwidth);
2523 		return -ENOMEM;
2524 	}
2525 
2526 	bw_table->bw_used = bw_used;
2527 	return 0;
2528 }
2529 
2530 static bool xhci_is_async_ep(unsigned int ep_type)
2531 {
2532 	return (ep_type != ISOC_OUT_EP && ep_type != INT_OUT_EP &&
2533 					ep_type != ISOC_IN_EP &&
2534 					ep_type != INT_IN_EP);
2535 }
2536 
2537 static bool xhci_is_sync_in_ep(unsigned int ep_type)
2538 {
2539 	return (ep_type == ISOC_IN_EP || ep_type == INT_IN_EP);
2540 }
2541 
2542 static unsigned int xhci_get_ss_bw_consumed(struct xhci_bw_info *ep_bw)
2543 {
2544 	unsigned int mps = DIV_ROUND_UP(ep_bw->max_packet_size, SS_BLOCK);
2545 
2546 	if (ep_bw->ep_interval == 0)
2547 		return SS_OVERHEAD_BURST +
2548 			(ep_bw->mult * ep_bw->num_packets *
2549 					(SS_OVERHEAD + mps));
2550 	return DIV_ROUND_UP(ep_bw->mult * ep_bw->num_packets *
2551 				(SS_OVERHEAD + mps + SS_OVERHEAD_BURST),
2552 				1 << ep_bw->ep_interval);
2553 
2554 }
2555 
2556 static void xhci_drop_ep_from_interval_table(struct xhci_hcd *xhci,
2557 		struct xhci_bw_info *ep_bw,
2558 		struct xhci_interval_bw_table *bw_table,
2559 		struct usb_device *udev,
2560 		struct xhci_virt_ep *virt_ep,
2561 		struct xhci_tt_bw_info *tt_info)
2562 {
2563 	struct xhci_interval_bw	*interval_bw;
2564 	int normalized_interval;
2565 
2566 	if (xhci_is_async_ep(ep_bw->type))
2567 		return;
2568 
2569 	if (udev->speed >= USB_SPEED_SUPER) {
2570 		if (xhci_is_sync_in_ep(ep_bw->type))
2571 			xhci->devs[udev->slot_id]->bw_table->ss_bw_in -=
2572 				xhci_get_ss_bw_consumed(ep_bw);
2573 		else
2574 			xhci->devs[udev->slot_id]->bw_table->ss_bw_out -=
2575 				xhci_get_ss_bw_consumed(ep_bw);
2576 		return;
2577 	}
2578 
2579 	/* SuperSpeed endpoints never get added to intervals in the table, so
2580 	 * this check is only valid for HS/FS/LS devices.
2581 	 */
2582 	if (list_empty(&virt_ep->bw_endpoint_list))
2583 		return;
2584 	/* For LS/FS devices, we need to translate the interval expressed in
2585 	 * microframes to frames.
2586 	 */
2587 	if (udev->speed == USB_SPEED_HIGH)
2588 		normalized_interval = ep_bw->ep_interval;
2589 	else
2590 		normalized_interval = ep_bw->ep_interval - 3;
2591 
2592 	if (normalized_interval == 0)
2593 		bw_table->interval0_esit_payload -= ep_bw->max_esit_payload;
2594 	interval_bw = &bw_table->interval_bw[normalized_interval];
2595 	interval_bw->num_packets -= ep_bw->num_packets;
2596 	switch (udev->speed) {
2597 	case USB_SPEED_LOW:
2598 		interval_bw->overhead[LS_OVERHEAD_TYPE] -= 1;
2599 		break;
2600 	case USB_SPEED_FULL:
2601 		interval_bw->overhead[FS_OVERHEAD_TYPE] -= 1;
2602 		break;
2603 	case USB_SPEED_HIGH:
2604 		interval_bw->overhead[HS_OVERHEAD_TYPE] -= 1;
2605 		break;
2606 	default:
2607 		/* Should never happen because only LS/FS/HS endpoints will get
2608 		 * added to the endpoint list.
2609 		 */
2610 		return;
2611 	}
2612 	if (tt_info)
2613 		tt_info->active_eps -= 1;
2614 	list_del_init(&virt_ep->bw_endpoint_list);
2615 }
2616 
2617 static void xhci_add_ep_to_interval_table(struct xhci_hcd *xhci,
2618 		struct xhci_bw_info *ep_bw,
2619 		struct xhci_interval_bw_table *bw_table,
2620 		struct usb_device *udev,
2621 		struct xhci_virt_ep *virt_ep,
2622 		struct xhci_tt_bw_info *tt_info)
2623 {
2624 	struct xhci_interval_bw	*interval_bw;
2625 	struct xhci_virt_ep *smaller_ep;
2626 	int normalized_interval;
2627 
2628 	if (xhci_is_async_ep(ep_bw->type))
2629 		return;
2630 
2631 	if (udev->speed == USB_SPEED_SUPER) {
2632 		if (xhci_is_sync_in_ep(ep_bw->type))
2633 			xhci->devs[udev->slot_id]->bw_table->ss_bw_in +=
2634 				xhci_get_ss_bw_consumed(ep_bw);
2635 		else
2636 			xhci->devs[udev->slot_id]->bw_table->ss_bw_out +=
2637 				xhci_get_ss_bw_consumed(ep_bw);
2638 		return;
2639 	}
2640 
2641 	/* For LS/FS devices, we need to translate the interval expressed in
2642 	 * microframes to frames.
2643 	 */
2644 	if (udev->speed == USB_SPEED_HIGH)
2645 		normalized_interval = ep_bw->ep_interval;
2646 	else
2647 		normalized_interval = ep_bw->ep_interval - 3;
2648 
2649 	if (normalized_interval == 0)
2650 		bw_table->interval0_esit_payload += ep_bw->max_esit_payload;
2651 	interval_bw = &bw_table->interval_bw[normalized_interval];
2652 	interval_bw->num_packets += ep_bw->num_packets;
2653 	switch (udev->speed) {
2654 	case USB_SPEED_LOW:
2655 		interval_bw->overhead[LS_OVERHEAD_TYPE] += 1;
2656 		break;
2657 	case USB_SPEED_FULL:
2658 		interval_bw->overhead[FS_OVERHEAD_TYPE] += 1;
2659 		break;
2660 	case USB_SPEED_HIGH:
2661 		interval_bw->overhead[HS_OVERHEAD_TYPE] += 1;
2662 		break;
2663 	default:
2664 		/* Should never happen because only LS/FS/HS endpoints will get
2665 		 * added to the endpoint list.
2666 		 */
2667 		return;
2668 	}
2669 
2670 	if (tt_info)
2671 		tt_info->active_eps += 1;
2672 	/* Insert the endpoint into the list, largest max packet size first. */
2673 	list_for_each_entry(smaller_ep, &interval_bw->endpoints,
2674 			bw_endpoint_list) {
2675 		if (ep_bw->max_packet_size >=
2676 				smaller_ep->bw_info.max_packet_size) {
2677 			/* Add the new ep before the smaller endpoint */
2678 			list_add_tail(&virt_ep->bw_endpoint_list,
2679 					&smaller_ep->bw_endpoint_list);
2680 			return;
2681 		}
2682 	}
2683 	/* Add the new endpoint at the end of the list. */
2684 	list_add_tail(&virt_ep->bw_endpoint_list,
2685 			&interval_bw->endpoints);
2686 }
2687 
2688 void xhci_update_tt_active_eps(struct xhci_hcd *xhci,
2689 		struct xhci_virt_device *virt_dev,
2690 		int old_active_eps)
2691 {
2692 	struct xhci_root_port_bw_info *rh_bw_info;
2693 	if (!virt_dev->tt_info)
2694 		return;
2695 
2696 	rh_bw_info = &xhci->rh_bw[virt_dev->rhub_port->hw_portnum];
2697 	if (old_active_eps == 0 &&
2698 				virt_dev->tt_info->active_eps != 0) {
2699 		rh_bw_info->num_active_tts += 1;
2700 		rh_bw_info->bw_table.bw_used += TT_HS_OVERHEAD;
2701 	} else if (old_active_eps != 0 &&
2702 				virt_dev->tt_info->active_eps == 0) {
2703 		rh_bw_info->num_active_tts -= 1;
2704 		rh_bw_info->bw_table.bw_used -= TT_HS_OVERHEAD;
2705 	}
2706 }
2707 
2708 static int xhci_reserve_bandwidth(struct xhci_hcd *xhci,
2709 		struct xhci_virt_device *virt_dev,
2710 		struct xhci_container_ctx *in_ctx)
2711 {
2712 	struct xhci_bw_info ep_bw_info[31];
2713 	int i;
2714 	struct xhci_input_control_ctx *ctrl_ctx;
2715 	int old_active_eps = 0;
2716 
2717 	if (virt_dev->tt_info)
2718 		old_active_eps = virt_dev->tt_info->active_eps;
2719 
2720 	ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
2721 	if (!ctrl_ctx) {
2722 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
2723 				__func__);
2724 		return -ENOMEM;
2725 	}
2726 
2727 	for (i = 0; i < 31; i++) {
2728 		if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i))
2729 			continue;
2730 
2731 		/* Make a copy of the BW info in case we need to revert this */
2732 		memcpy(&ep_bw_info[i], &virt_dev->eps[i].bw_info,
2733 				sizeof(ep_bw_info[i]));
2734 		/* Drop the endpoint from the interval table if the endpoint is
2735 		 * being dropped or changed.
2736 		 */
2737 		if (EP_IS_DROPPED(ctrl_ctx, i))
2738 			xhci_drop_ep_from_interval_table(xhci,
2739 					&virt_dev->eps[i].bw_info,
2740 					virt_dev->bw_table,
2741 					virt_dev->udev,
2742 					&virt_dev->eps[i],
2743 					virt_dev->tt_info);
2744 	}
2745 	/* Overwrite the information stored in the endpoints' bw_info */
2746 	xhci_update_bw_info(xhci, virt_dev->in_ctx, ctrl_ctx, virt_dev);
2747 	for (i = 0; i < 31; i++) {
2748 		/* Add any changed or added endpoints to the interval table */
2749 		if (EP_IS_ADDED(ctrl_ctx, i))
2750 			xhci_add_ep_to_interval_table(xhci,
2751 					&virt_dev->eps[i].bw_info,
2752 					virt_dev->bw_table,
2753 					virt_dev->udev,
2754 					&virt_dev->eps[i],
2755 					virt_dev->tt_info);
2756 	}
2757 
2758 	if (!xhci_check_bw_table(xhci, virt_dev, old_active_eps)) {
2759 		/* Ok, this fits in the bandwidth we have.
2760 		 * Update the number of active TTs.
2761 		 */
2762 		xhci_update_tt_active_eps(xhci, virt_dev, old_active_eps);
2763 		return 0;
2764 	}
2765 
2766 	/* We don't have enough bandwidth for this, revert the stored info. */
2767 	for (i = 0; i < 31; i++) {
2768 		if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i))
2769 			continue;
2770 
2771 		/* Drop the new copies of any added or changed endpoints from
2772 		 * the interval table.
2773 		 */
2774 		if (EP_IS_ADDED(ctrl_ctx, i)) {
2775 			xhci_drop_ep_from_interval_table(xhci,
2776 					&virt_dev->eps[i].bw_info,
2777 					virt_dev->bw_table,
2778 					virt_dev->udev,
2779 					&virt_dev->eps[i],
2780 					virt_dev->tt_info);
2781 		}
2782 		/* Revert the endpoint back to its old information */
2783 		memcpy(&virt_dev->eps[i].bw_info, &ep_bw_info[i],
2784 				sizeof(ep_bw_info[i]));
2785 		/* Add any changed or dropped endpoints back into the table */
2786 		if (EP_IS_DROPPED(ctrl_ctx, i))
2787 			xhci_add_ep_to_interval_table(xhci,
2788 					&virt_dev->eps[i].bw_info,
2789 					virt_dev->bw_table,
2790 					virt_dev->udev,
2791 					&virt_dev->eps[i],
2792 					virt_dev->tt_info);
2793 	}
2794 	return -ENOMEM;
2795 }
2796 
2797 
2798 /* Issue a configure endpoint command or evaluate context command
2799  * and wait for it to finish.
2800  */
2801 static int xhci_configure_endpoint(struct xhci_hcd *xhci,
2802 		struct usb_device *udev,
2803 		struct xhci_command *command,
2804 		bool ctx_change, bool must_succeed)
2805 {
2806 	int ret;
2807 	unsigned long flags;
2808 	struct xhci_input_control_ctx *ctrl_ctx;
2809 	struct xhci_virt_device *virt_dev;
2810 	struct xhci_slot_ctx *slot_ctx;
2811 
2812 	if (!command)
2813 		return -EINVAL;
2814 
2815 	spin_lock_irqsave(&xhci->lock, flags);
2816 
2817 	if (xhci->xhc_state & XHCI_STATE_DYING) {
2818 		spin_unlock_irqrestore(&xhci->lock, flags);
2819 		return -ESHUTDOWN;
2820 	}
2821 
2822 	virt_dev = xhci->devs[udev->slot_id];
2823 
2824 	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
2825 	if (!ctrl_ctx) {
2826 		spin_unlock_irqrestore(&xhci->lock, flags);
2827 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
2828 				__func__);
2829 		return -ENOMEM;
2830 	}
2831 
2832 	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK) &&
2833 			xhci_reserve_host_resources(xhci, ctrl_ctx)) {
2834 		spin_unlock_irqrestore(&xhci->lock, flags);
2835 		xhci_warn(xhci, "Not enough host resources, "
2836 				"active endpoint contexts = %u\n",
2837 				xhci->num_active_eps);
2838 		return -ENOMEM;
2839 	}
2840 	if ((xhci->quirks & XHCI_SW_BW_CHECKING) &&
2841 	    xhci_reserve_bandwidth(xhci, virt_dev, command->in_ctx)) {
2842 		if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK))
2843 			xhci_free_host_resources(xhci, ctrl_ctx);
2844 		spin_unlock_irqrestore(&xhci->lock, flags);
2845 		xhci_warn(xhci, "Not enough bandwidth\n");
2846 		return -ENOMEM;
2847 	}
2848 
2849 	slot_ctx = xhci_get_slot_ctx(xhci, command->in_ctx);
2850 
2851 	trace_xhci_configure_endpoint_ctrl_ctx(ctrl_ctx);
2852 	trace_xhci_configure_endpoint(slot_ctx);
2853 
2854 	if (!ctx_change)
2855 		ret = xhci_queue_configure_endpoint(xhci, command,
2856 				command->in_ctx->dma,
2857 				udev->slot_id, must_succeed);
2858 	else
2859 		ret = xhci_queue_evaluate_context(xhci, command,
2860 				command->in_ctx->dma,
2861 				udev->slot_id, must_succeed);
2862 	if (ret < 0) {
2863 		if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK))
2864 			xhci_free_host_resources(xhci, ctrl_ctx);
2865 		spin_unlock_irqrestore(&xhci->lock, flags);
2866 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
2867 				"FIXME allocate a new ring segment");
2868 		return -ENOMEM;
2869 	}
2870 	xhci_ring_cmd_db(xhci);
2871 	spin_unlock_irqrestore(&xhci->lock, flags);
2872 
2873 	/* Wait for the configure endpoint command to complete */
2874 	wait_for_completion(command->completion);
2875 
2876 	if (!ctx_change)
2877 		ret = xhci_configure_endpoint_result(xhci, udev,
2878 						     &command->status);
2879 	else
2880 		ret = xhci_evaluate_context_result(xhci, udev,
2881 						   &command->status);
2882 
2883 	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
2884 		spin_lock_irqsave(&xhci->lock, flags);
2885 		/* If the command failed, remove the reserved resources.
2886 		 * Otherwise, clean up the estimate to include dropped eps.
2887 		 */
2888 		if (ret)
2889 			xhci_free_host_resources(xhci, ctrl_ctx);
2890 		else
2891 			xhci_finish_resource_reservation(xhci, ctrl_ctx);
2892 		spin_unlock_irqrestore(&xhci->lock, flags);
2893 	}
2894 	return ret;
2895 }
2896 
2897 static void xhci_check_bw_drop_ep_streams(struct xhci_hcd *xhci,
2898 	struct xhci_virt_device *vdev, int i)
2899 {
2900 	struct xhci_virt_ep *ep = &vdev->eps[i];
2901 
2902 	if (ep->ep_state & EP_HAS_STREAMS) {
2903 		xhci_warn(xhci, "WARN: endpoint 0x%02x has streams on set_interface, freeing streams.\n",
2904 				xhci_get_endpoint_address(i));
2905 		xhci_free_stream_info(xhci, ep->stream_info);
2906 		ep->stream_info = NULL;
2907 		ep->ep_state &= ~EP_HAS_STREAMS;
2908 	}
2909 }
2910 
2911 /* Called after one or more calls to xhci_add_endpoint() or
2912  * xhci_drop_endpoint().  If this call fails, the USB core is expected
2913  * to call xhci_reset_bandwidth().
2914  *
2915  * Since we are in the middle of changing either configuration or
2916  * installing a new alt setting, the USB core won't allow URBs to be
2917  * enqueued for any endpoint on the old config or interface.  Nothing
2918  * else should be touching the xhci->devs[slot_id] structure, so we
2919  * don't need to take the xhci->lock for manipulating that.
2920  */
2921 int xhci_check_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
2922 {
2923 	int i;
2924 	int ret = 0;
2925 	struct xhci_hcd *xhci;
2926 	struct xhci_virt_device	*virt_dev;
2927 	struct xhci_input_control_ctx *ctrl_ctx;
2928 	struct xhci_slot_ctx *slot_ctx;
2929 	struct xhci_command *command;
2930 
2931 	ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
2932 	if (ret <= 0)
2933 		return ret;
2934 	xhci = hcd_to_xhci(hcd);
2935 	if ((xhci->xhc_state & XHCI_STATE_DYING) ||
2936 		(xhci->xhc_state & XHCI_STATE_REMOVING))
2937 		return -ENODEV;
2938 
2939 	xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
2940 	virt_dev = xhci->devs[udev->slot_id];
2941 
2942 	command = xhci_alloc_command(xhci, true, GFP_KERNEL);
2943 	if (!command)
2944 		return -ENOMEM;
2945 
2946 	command->in_ctx = virt_dev->in_ctx;
2947 
2948 	/* See section 4.6.6 - A0 = 1; A1 = D0 = D1 = 0 */
2949 	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
2950 	if (!ctrl_ctx) {
2951 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
2952 				__func__);
2953 		ret = -ENOMEM;
2954 		goto command_cleanup;
2955 	}
2956 	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
2957 	ctrl_ctx->add_flags &= cpu_to_le32(~EP0_FLAG);
2958 	ctrl_ctx->drop_flags &= cpu_to_le32(~(SLOT_FLAG | EP0_FLAG));
2959 
2960 	/* Don't issue the command if there's no endpoints to update. */
2961 	if (ctrl_ctx->add_flags == cpu_to_le32(SLOT_FLAG) &&
2962 	    ctrl_ctx->drop_flags == 0) {
2963 		ret = 0;
2964 		goto command_cleanup;
2965 	}
2966 	/* Fix up Context Entries field. Minimum value is EP0 == BIT(1). */
2967 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
2968 	for (i = 31; i >= 1; i--) {
2969 		__le32 le32 = cpu_to_le32(BIT(i));
2970 
2971 		if ((virt_dev->eps[i-1].ring && !(ctrl_ctx->drop_flags & le32))
2972 		    || (ctrl_ctx->add_flags & le32) || i == 1) {
2973 			slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
2974 			slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(i));
2975 			break;
2976 		}
2977 	}
2978 
2979 	ret = xhci_configure_endpoint(xhci, udev, command,
2980 			false, false);
2981 	if (ret)
2982 		/* Callee should call reset_bandwidth() */
2983 		goto command_cleanup;
2984 
2985 	/* Free any rings that were dropped, but not changed. */
2986 	for (i = 1; i < 31; i++) {
2987 		if ((le32_to_cpu(ctrl_ctx->drop_flags) & (1 << (i + 1))) &&
2988 		    !(le32_to_cpu(ctrl_ctx->add_flags) & (1 << (i + 1)))) {
2989 			xhci_free_endpoint_ring(xhci, virt_dev, i);
2990 			xhci_check_bw_drop_ep_streams(xhci, virt_dev, i);
2991 		}
2992 	}
2993 	xhci_zero_in_ctx(xhci, virt_dev);
2994 	/*
2995 	 * Install any rings for completely new endpoints or changed endpoints,
2996 	 * and free any old rings from changed endpoints.
2997 	 */
2998 	for (i = 1; i < 31; i++) {
2999 		if (!virt_dev->eps[i].new_ring)
3000 			continue;
3001 		/* Only free the old ring if it exists.
3002 		 * It may not if this is the first add of an endpoint.
3003 		 */
3004 		if (virt_dev->eps[i].ring) {
3005 			xhci_free_endpoint_ring(xhci, virt_dev, i);
3006 		}
3007 		xhci_check_bw_drop_ep_streams(xhci, virt_dev, i);
3008 		virt_dev->eps[i].ring = virt_dev->eps[i].new_ring;
3009 		virt_dev->eps[i].new_ring = NULL;
3010 		xhci_debugfs_create_endpoint(xhci, virt_dev, i);
3011 	}
3012 command_cleanup:
3013 	kfree(command->completion);
3014 	kfree(command);
3015 
3016 	return ret;
3017 }
3018 EXPORT_SYMBOL_GPL(xhci_check_bandwidth);
3019 
3020 void xhci_reset_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
3021 {
3022 	struct xhci_hcd *xhci;
3023 	struct xhci_virt_device	*virt_dev;
3024 	int i, ret;
3025 
3026 	ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
3027 	if (ret <= 0)
3028 		return;
3029 	xhci = hcd_to_xhci(hcd);
3030 
3031 	xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
3032 	virt_dev = xhci->devs[udev->slot_id];
3033 	/* Free any rings allocated for added endpoints */
3034 	for (i = 0; i < 31; i++) {
3035 		if (virt_dev->eps[i].new_ring) {
3036 			xhci_debugfs_remove_endpoint(xhci, virt_dev, i);
3037 			xhci_ring_free(xhci, virt_dev->eps[i].new_ring);
3038 			virt_dev->eps[i].new_ring = NULL;
3039 		}
3040 	}
3041 	xhci_zero_in_ctx(xhci, virt_dev);
3042 }
3043 EXPORT_SYMBOL_GPL(xhci_reset_bandwidth);
3044 
3045 static void xhci_setup_input_ctx_for_config_ep(struct xhci_hcd *xhci,
3046 		struct xhci_container_ctx *in_ctx,
3047 		struct xhci_container_ctx *out_ctx,
3048 		struct xhci_input_control_ctx *ctrl_ctx,
3049 		u32 add_flags, u32 drop_flags)
3050 {
3051 	ctrl_ctx->add_flags = cpu_to_le32(add_flags);
3052 	ctrl_ctx->drop_flags = cpu_to_le32(drop_flags);
3053 	xhci_slot_copy(xhci, in_ctx, out_ctx);
3054 	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
3055 }
3056 
3057 static void xhci_endpoint_disable(struct usb_hcd *hcd,
3058 				  struct usb_host_endpoint *host_ep)
3059 {
3060 	struct xhci_hcd		*xhci;
3061 	struct xhci_virt_device	*vdev;
3062 	struct xhci_virt_ep	*ep;
3063 	struct usb_device	*udev;
3064 	unsigned long		flags;
3065 	unsigned int		ep_index;
3066 
3067 	xhci = hcd_to_xhci(hcd);
3068 rescan:
3069 	spin_lock_irqsave(&xhci->lock, flags);
3070 
3071 	udev = (struct usb_device *)host_ep->hcpriv;
3072 	if (!udev || !udev->slot_id)
3073 		goto done;
3074 
3075 	vdev = xhci->devs[udev->slot_id];
3076 	if (!vdev)
3077 		goto done;
3078 
3079 	ep_index = xhci_get_endpoint_index(&host_ep->desc);
3080 	ep = &vdev->eps[ep_index];
3081 
3082 	/* wait for hub_tt_work to finish clearing hub TT */
3083 	if (ep->ep_state & EP_CLEARING_TT) {
3084 		spin_unlock_irqrestore(&xhci->lock, flags);
3085 		schedule_timeout_uninterruptible(1);
3086 		goto rescan;
3087 	}
3088 
3089 	if (ep->ep_state)
3090 		xhci_dbg(xhci, "endpoint disable with ep_state 0x%x\n",
3091 			 ep->ep_state);
3092 done:
3093 	host_ep->hcpriv = NULL;
3094 	spin_unlock_irqrestore(&xhci->lock, flags);
3095 }
3096 
3097 /*
3098  * Called after usb core issues a clear halt control message.
3099  * The host side of the halt should already be cleared by a reset endpoint
3100  * command issued when the STALL event was received.
3101  *
3102  * The reset endpoint command may only be issued to endpoints in the halted
3103  * state. For software that wishes to reset the data toggle or sequence number
3104  * of an endpoint that isn't in the halted state this function will issue a
3105  * configure endpoint command with the Drop and Add bits set for the target
3106  * endpoint. Refer to the additional note in xhci spcification section 4.6.8.
3107  *
3108  * vdev may be lost due to xHC restore error and re-initialization during S3/S4
3109  * resume. A new vdev will be allocated later by xhci_discover_or_reset_device()
3110  */
3111 
3112 static void xhci_endpoint_reset(struct usb_hcd *hcd,
3113 		struct usb_host_endpoint *host_ep)
3114 {
3115 	struct xhci_hcd *xhci;
3116 	struct usb_device *udev;
3117 	struct xhci_virt_device *vdev;
3118 	struct xhci_virt_ep *ep;
3119 	struct xhci_input_control_ctx *ctrl_ctx;
3120 	struct xhci_command *stop_cmd, *cfg_cmd;
3121 	unsigned int ep_index;
3122 	unsigned long flags;
3123 	u32 ep_flag;
3124 	int err;
3125 
3126 	xhci = hcd_to_xhci(hcd);
3127 	ep_index = xhci_get_endpoint_index(&host_ep->desc);
3128 
3129 	/*
3130 	 * Usb core assumes a max packet value for ep0 on FS devices until the
3131 	 * real value is read from the descriptor. Core resets Ep0 if values
3132 	 * mismatch. Reconfigure the xhci ep0 endpoint context here in that case
3133 	 */
3134 	if (usb_endpoint_xfer_control(&host_ep->desc) && ep_index == 0) {
3135 
3136 		udev = container_of(host_ep, struct usb_device, ep0);
3137 		if (udev->speed != USB_SPEED_FULL || !udev->slot_id)
3138 			return;
3139 
3140 		vdev = xhci->devs[udev->slot_id];
3141 		if (!vdev || vdev->udev != udev)
3142 			return;
3143 
3144 		xhci_check_ep0_maxpacket(xhci, vdev);
3145 
3146 		/* Nothing else should be done here for ep0 during ep reset */
3147 		return;
3148 	}
3149 
3150 	if (!host_ep->hcpriv)
3151 		return;
3152 	udev = (struct usb_device *) host_ep->hcpriv;
3153 	vdev = xhci->devs[udev->slot_id];
3154 
3155 	if (!udev->slot_id || !vdev)
3156 		return;
3157 
3158 	ep = &vdev->eps[ep_index];
3159 
3160 	/* Bail out if toggle is already being cleared by a endpoint reset */
3161 	spin_lock_irqsave(&xhci->lock, flags);
3162 	if (ep->ep_state & EP_HARD_CLEAR_TOGGLE) {
3163 		ep->ep_state &= ~EP_HARD_CLEAR_TOGGLE;
3164 		spin_unlock_irqrestore(&xhci->lock, flags);
3165 		return;
3166 	}
3167 	spin_unlock_irqrestore(&xhci->lock, flags);
3168 	/* Only interrupt and bulk ep's use data toggle, USB2 spec 5.5.4-> */
3169 	if (usb_endpoint_xfer_control(&host_ep->desc) ||
3170 	    usb_endpoint_xfer_isoc(&host_ep->desc))
3171 		return;
3172 
3173 	ep_flag = xhci_get_endpoint_flag(&host_ep->desc);
3174 
3175 	if (ep_flag == SLOT_FLAG || ep_flag == EP0_FLAG)
3176 		return;
3177 
3178 	stop_cmd = xhci_alloc_command(xhci, true, GFP_NOWAIT);
3179 	if (!stop_cmd)
3180 		return;
3181 
3182 	cfg_cmd = xhci_alloc_command_with_ctx(xhci, true, GFP_NOWAIT);
3183 	if (!cfg_cmd)
3184 		goto cleanup;
3185 
3186 	spin_lock_irqsave(&xhci->lock, flags);
3187 
3188 	/* block queuing new trbs and ringing ep doorbell */
3189 	ep->ep_state |= EP_SOFT_CLEAR_TOGGLE;
3190 
3191 	/*
3192 	 * Make sure endpoint ring is empty before resetting the toggle/seq.
3193 	 * Driver is required to synchronously cancel all transfer request.
3194 	 * Stop the endpoint to force xHC to update the output context
3195 	 */
3196 
3197 	if (!list_empty(&ep->ring->td_list)) {
3198 		dev_err(&udev->dev, "EP not empty, refuse reset\n");
3199 		spin_unlock_irqrestore(&xhci->lock, flags);
3200 		xhci_free_command(xhci, cfg_cmd);
3201 		goto cleanup;
3202 	}
3203 
3204 	err = xhci_queue_stop_endpoint(xhci, stop_cmd, udev->slot_id,
3205 					ep_index, 0);
3206 	if (err < 0) {
3207 		spin_unlock_irqrestore(&xhci->lock, flags);
3208 		xhci_free_command(xhci, cfg_cmd);
3209 		xhci_dbg(xhci, "%s: Failed to queue stop ep command, %d ",
3210 				__func__, err);
3211 		goto cleanup;
3212 	}
3213 
3214 	xhci_ring_cmd_db(xhci);
3215 	spin_unlock_irqrestore(&xhci->lock, flags);
3216 
3217 	wait_for_completion(stop_cmd->completion);
3218 
3219 	spin_lock_irqsave(&xhci->lock, flags);
3220 
3221 	/* config ep command clears toggle if add and drop ep flags are set */
3222 	ctrl_ctx = xhci_get_input_control_ctx(cfg_cmd->in_ctx);
3223 	if (!ctrl_ctx) {
3224 		spin_unlock_irqrestore(&xhci->lock, flags);
3225 		xhci_free_command(xhci, cfg_cmd);
3226 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
3227 				__func__);
3228 		goto cleanup;
3229 	}
3230 
3231 	xhci_setup_input_ctx_for_config_ep(xhci, cfg_cmd->in_ctx, vdev->out_ctx,
3232 					   ctrl_ctx, ep_flag, ep_flag);
3233 	xhci_endpoint_copy(xhci, cfg_cmd->in_ctx, vdev->out_ctx, ep_index);
3234 
3235 	err = xhci_queue_configure_endpoint(xhci, cfg_cmd, cfg_cmd->in_ctx->dma,
3236 				      udev->slot_id, false);
3237 	if (err < 0) {
3238 		spin_unlock_irqrestore(&xhci->lock, flags);
3239 		xhci_free_command(xhci, cfg_cmd);
3240 		xhci_dbg(xhci, "%s: Failed to queue config ep command, %d ",
3241 				__func__, err);
3242 		goto cleanup;
3243 	}
3244 
3245 	xhci_ring_cmd_db(xhci);
3246 	spin_unlock_irqrestore(&xhci->lock, flags);
3247 
3248 	wait_for_completion(cfg_cmd->completion);
3249 
3250 	xhci_free_command(xhci, cfg_cmd);
3251 cleanup:
3252 	xhci_free_command(xhci, stop_cmd);
3253 	spin_lock_irqsave(&xhci->lock, flags);
3254 	if (ep->ep_state & EP_SOFT_CLEAR_TOGGLE)
3255 		ep->ep_state &= ~EP_SOFT_CLEAR_TOGGLE;
3256 	spin_unlock_irqrestore(&xhci->lock, flags);
3257 }
3258 
3259 static int xhci_check_streams_endpoint(struct xhci_hcd *xhci,
3260 		struct usb_device *udev, struct usb_host_endpoint *ep,
3261 		unsigned int slot_id)
3262 {
3263 	int ret;
3264 	unsigned int ep_index;
3265 	unsigned int ep_state;
3266 
3267 	if (!ep)
3268 		return -EINVAL;
3269 	ret = xhci_check_args(xhci_to_hcd(xhci), udev, ep, 1, true, __func__);
3270 	if (ret <= 0)
3271 		return ret ? ret : -EINVAL;
3272 	if (usb_ss_max_streams(&ep->ss_ep_comp) == 0) {
3273 		xhci_warn(xhci, "WARN: SuperSpeed Endpoint Companion"
3274 				" descriptor for ep 0x%x does not support streams\n",
3275 				ep->desc.bEndpointAddress);
3276 		return -EINVAL;
3277 	}
3278 
3279 	ep_index = xhci_get_endpoint_index(&ep->desc);
3280 	ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;
3281 	if (ep_state & EP_HAS_STREAMS ||
3282 			ep_state & EP_GETTING_STREAMS) {
3283 		xhci_warn(xhci, "WARN: SuperSpeed bulk endpoint 0x%x "
3284 				"already has streams set up.\n",
3285 				ep->desc.bEndpointAddress);
3286 		xhci_warn(xhci, "Send email to xHCI maintainer and ask for "
3287 				"dynamic stream context array reallocation.\n");
3288 		return -EINVAL;
3289 	}
3290 	if (!list_empty(&xhci->devs[slot_id]->eps[ep_index].ring->td_list)) {
3291 		xhci_warn(xhci, "Cannot setup streams for SuperSpeed bulk "
3292 				"endpoint 0x%x; URBs are pending.\n",
3293 				ep->desc.bEndpointAddress);
3294 		return -EINVAL;
3295 	}
3296 	return 0;
3297 }
3298 
3299 static void xhci_calculate_streams_entries(struct xhci_hcd *xhci,
3300 		unsigned int *num_streams, unsigned int *num_stream_ctxs)
3301 {
3302 	unsigned int max_streams;
3303 
3304 	/* The stream context array size must be a power of two */
3305 	*num_stream_ctxs = roundup_pow_of_two(*num_streams);
3306 	/*
3307 	 * Find out how many primary stream array entries the host controller
3308 	 * supports.  Later we may use secondary stream arrays (similar to 2nd
3309 	 * level page entries), but that's an optional feature for xHCI host
3310 	 * controllers. xHCs must support at least 4 stream IDs.
3311 	 */
3312 	max_streams = HCC_MAX_PSA(xhci->hcc_params);
3313 	if (*num_stream_ctxs > max_streams) {
3314 		xhci_dbg(xhci, "xHCI HW only supports %u stream ctx entries.\n",
3315 				max_streams);
3316 		*num_stream_ctxs = max_streams;
3317 		*num_streams = max_streams;
3318 	}
3319 }
3320 
3321 /* Returns an error code if one of the endpoint already has streams.
3322  * This does not change any data structures, it only checks and gathers
3323  * information.
3324  */
3325 static int xhci_calculate_streams_and_bitmask(struct xhci_hcd *xhci,
3326 		struct usb_device *udev,
3327 		struct usb_host_endpoint **eps, unsigned int num_eps,
3328 		unsigned int *num_streams, u32 *changed_ep_bitmask)
3329 {
3330 	unsigned int max_streams;
3331 	unsigned int endpoint_flag;
3332 	int i;
3333 	int ret;
3334 
3335 	for (i = 0; i < num_eps; i++) {
3336 		ret = xhci_check_streams_endpoint(xhci, udev,
3337 				eps[i], udev->slot_id);
3338 		if (ret < 0)
3339 			return ret;
3340 
3341 		max_streams = usb_ss_max_streams(&eps[i]->ss_ep_comp);
3342 		if (max_streams < (*num_streams - 1)) {
3343 			xhci_dbg(xhci, "Ep 0x%x only supports %u stream IDs.\n",
3344 					eps[i]->desc.bEndpointAddress,
3345 					max_streams);
3346 			*num_streams = max_streams+1;
3347 		}
3348 
3349 		endpoint_flag = xhci_get_endpoint_flag(&eps[i]->desc);
3350 		if (*changed_ep_bitmask & endpoint_flag)
3351 			return -EINVAL;
3352 		*changed_ep_bitmask |= endpoint_flag;
3353 	}
3354 	return 0;
3355 }
3356 
3357 static u32 xhci_calculate_no_streams_bitmask(struct xhci_hcd *xhci,
3358 		struct usb_device *udev,
3359 		struct usb_host_endpoint **eps, unsigned int num_eps)
3360 {
3361 	u32 changed_ep_bitmask = 0;
3362 	unsigned int slot_id;
3363 	unsigned int ep_index;
3364 	unsigned int ep_state;
3365 	int i;
3366 
3367 	slot_id = udev->slot_id;
3368 	if (!xhci->devs[slot_id])
3369 		return 0;
3370 
3371 	for (i = 0; i < num_eps; i++) {
3372 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3373 		ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;
3374 		/* Are streams already being freed for the endpoint? */
3375 		if (ep_state & EP_GETTING_NO_STREAMS) {
3376 			xhci_warn(xhci, "WARN Can't disable streams for "
3377 					"endpoint 0x%x, "
3378 					"streams are being disabled already\n",
3379 					eps[i]->desc.bEndpointAddress);
3380 			return 0;
3381 		}
3382 		/* Are there actually any streams to free? */
3383 		if (!(ep_state & EP_HAS_STREAMS) &&
3384 				!(ep_state & EP_GETTING_STREAMS)) {
3385 			xhci_warn(xhci, "WARN Can't disable streams for "
3386 					"endpoint 0x%x, "
3387 					"streams are already disabled!\n",
3388 					eps[i]->desc.bEndpointAddress);
3389 			xhci_warn(xhci, "WARN xhci_free_streams() called "
3390 					"with non-streams endpoint\n");
3391 			return 0;
3392 		}
3393 		changed_ep_bitmask |= xhci_get_endpoint_flag(&eps[i]->desc);
3394 	}
3395 	return changed_ep_bitmask;
3396 }
3397 
3398 /*
3399  * The USB device drivers use this function (through the HCD interface in USB
3400  * core) to prepare a set of bulk endpoints to use streams.  Streams are used to
3401  * coordinate mass storage command queueing across multiple endpoints (basically
3402  * a stream ID == a task ID).
3403  *
3404  * Setting up streams involves allocating the same size stream context array
3405  * for each endpoint and issuing a configure endpoint command for all endpoints.
3406  *
3407  * Don't allow the call to succeed if one endpoint only supports one stream
3408  * (which means it doesn't support streams at all).
3409  *
3410  * Drivers may get less stream IDs than they asked for, if the host controller
3411  * hardware or endpoints claim they can't support the number of requested
3412  * stream IDs.
3413  */
3414 static int xhci_alloc_streams(struct usb_hcd *hcd, struct usb_device *udev,
3415 		struct usb_host_endpoint **eps, unsigned int num_eps,
3416 		unsigned int num_streams, gfp_t mem_flags)
3417 {
3418 	int i, ret;
3419 	struct xhci_hcd *xhci;
3420 	struct xhci_virt_device *vdev;
3421 	struct xhci_command *config_cmd;
3422 	struct xhci_input_control_ctx *ctrl_ctx;
3423 	unsigned int ep_index;
3424 	unsigned int num_stream_ctxs;
3425 	unsigned int max_packet;
3426 	unsigned long flags;
3427 	u32 changed_ep_bitmask = 0;
3428 
3429 	if (!eps)
3430 		return -EINVAL;
3431 
3432 	/* Add one to the number of streams requested to account for
3433 	 * stream 0 that is reserved for xHCI usage.
3434 	 */
3435 	num_streams += 1;
3436 	xhci = hcd_to_xhci(hcd);
3437 	xhci_dbg(xhci, "Driver wants %u stream IDs (including stream 0).\n",
3438 			num_streams);
3439 
3440 	/* MaxPSASize value 0 (2 streams) means streams are not supported */
3441 	if ((xhci->quirks & XHCI_BROKEN_STREAMS) ||
3442 			HCC_MAX_PSA(xhci->hcc_params) < 4) {
3443 		xhci_dbg(xhci, "xHCI controller does not support streams.\n");
3444 		return -ENOSYS;
3445 	}
3446 
3447 	config_cmd = xhci_alloc_command_with_ctx(xhci, true, mem_flags);
3448 	if (!config_cmd)
3449 		return -ENOMEM;
3450 
3451 	ctrl_ctx = xhci_get_input_control_ctx(config_cmd->in_ctx);
3452 	if (!ctrl_ctx) {
3453 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
3454 				__func__);
3455 		xhci_free_command(xhci, config_cmd);
3456 		return -ENOMEM;
3457 	}
3458 
3459 	/* Check to make sure all endpoints are not already configured for
3460 	 * streams.  While we're at it, find the maximum number of streams that
3461 	 * all the endpoints will support and check for duplicate endpoints.
3462 	 */
3463 	spin_lock_irqsave(&xhci->lock, flags);
3464 	ret = xhci_calculate_streams_and_bitmask(xhci, udev, eps,
3465 			num_eps, &num_streams, &changed_ep_bitmask);
3466 	if (ret < 0) {
3467 		xhci_free_command(xhci, config_cmd);
3468 		spin_unlock_irqrestore(&xhci->lock, flags);
3469 		return ret;
3470 	}
3471 	if (num_streams <= 1) {
3472 		xhci_warn(xhci, "WARN: endpoints can't handle "
3473 				"more than one stream.\n");
3474 		xhci_free_command(xhci, config_cmd);
3475 		spin_unlock_irqrestore(&xhci->lock, flags);
3476 		return -EINVAL;
3477 	}
3478 	vdev = xhci->devs[udev->slot_id];
3479 	/* Mark each endpoint as being in transition, so
3480 	 * xhci_urb_enqueue() will reject all URBs.
3481 	 */
3482 	for (i = 0; i < num_eps; i++) {
3483 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3484 		vdev->eps[ep_index].ep_state |= EP_GETTING_STREAMS;
3485 	}
3486 	spin_unlock_irqrestore(&xhci->lock, flags);
3487 
3488 	/* Setup internal data structures and allocate HW data structures for
3489 	 * streams (but don't install the HW structures in the input context
3490 	 * until we're sure all memory allocation succeeded).
3491 	 */
3492 	xhci_calculate_streams_entries(xhci, &num_streams, &num_stream_ctxs);
3493 	xhci_dbg(xhci, "Need %u stream ctx entries for %u stream IDs.\n",
3494 			num_stream_ctxs, num_streams);
3495 
3496 	for (i = 0; i < num_eps; i++) {
3497 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3498 		max_packet = usb_endpoint_maxp(&eps[i]->desc);
3499 		vdev->eps[ep_index].stream_info = xhci_alloc_stream_info(xhci,
3500 				num_stream_ctxs,
3501 				num_streams,
3502 				max_packet, mem_flags);
3503 		if (!vdev->eps[ep_index].stream_info)
3504 			goto cleanup;
3505 		/* Set maxPstreams in endpoint context and update deq ptr to
3506 		 * point to stream context array. FIXME
3507 		 */
3508 	}
3509 
3510 	/* Set up the input context for a configure endpoint command. */
3511 	for (i = 0; i < num_eps; i++) {
3512 		struct xhci_ep_ctx *ep_ctx;
3513 
3514 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3515 		ep_ctx = xhci_get_ep_ctx(xhci, config_cmd->in_ctx, ep_index);
3516 
3517 		xhci_endpoint_copy(xhci, config_cmd->in_ctx,
3518 				vdev->out_ctx, ep_index);
3519 		xhci_setup_streams_ep_input_ctx(xhci, ep_ctx,
3520 				vdev->eps[ep_index].stream_info);
3521 	}
3522 	/* Tell the HW to drop its old copy of the endpoint context info
3523 	 * and add the updated copy from the input context.
3524 	 */
3525 	xhci_setup_input_ctx_for_config_ep(xhci, config_cmd->in_ctx,
3526 			vdev->out_ctx, ctrl_ctx,
3527 			changed_ep_bitmask, changed_ep_bitmask);
3528 
3529 	/* Issue and wait for the configure endpoint command */
3530 	ret = xhci_configure_endpoint(xhci, udev, config_cmd,
3531 			false, false);
3532 
3533 	/* xHC rejected the configure endpoint command for some reason, so we
3534 	 * leave the old ring intact and free our internal streams data
3535 	 * structure.
3536 	 */
3537 	if (ret < 0)
3538 		goto cleanup;
3539 
3540 	spin_lock_irqsave(&xhci->lock, flags);
3541 	for (i = 0; i < num_eps; i++) {
3542 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3543 		vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS;
3544 		xhci_dbg(xhci, "Slot %u ep ctx %u now has streams.\n",
3545 			 udev->slot_id, ep_index);
3546 		vdev->eps[ep_index].ep_state |= EP_HAS_STREAMS;
3547 	}
3548 	xhci_free_command(xhci, config_cmd);
3549 	spin_unlock_irqrestore(&xhci->lock, flags);
3550 
3551 	for (i = 0; i < num_eps; i++) {
3552 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3553 		xhci_debugfs_create_stream_files(xhci, vdev, ep_index);
3554 	}
3555 	/* Subtract 1 for stream 0, which drivers can't use */
3556 	return num_streams - 1;
3557 
3558 cleanup:
3559 	/* If it didn't work, free the streams! */
3560 	for (i = 0; i < num_eps; i++) {
3561 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3562 		xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info);
3563 		vdev->eps[ep_index].stream_info = NULL;
3564 		/* FIXME Unset maxPstreams in endpoint context and
3565 		 * update deq ptr to point to normal string ring.
3566 		 */
3567 		vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS;
3568 		vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS;
3569 		xhci_endpoint_zero(xhci, vdev, eps[i]);
3570 	}
3571 	xhci_free_command(xhci, config_cmd);
3572 	return -ENOMEM;
3573 }
3574 
3575 /* Transition the endpoint from using streams to being a "normal" endpoint
3576  * without streams.
3577  *
3578  * Modify the endpoint context state, submit a configure endpoint command,
3579  * and free all endpoint rings for streams if that completes successfully.
3580  */
3581 static int xhci_free_streams(struct usb_hcd *hcd, struct usb_device *udev,
3582 		struct usb_host_endpoint **eps, unsigned int num_eps,
3583 		gfp_t mem_flags)
3584 {
3585 	int i, ret;
3586 	struct xhci_hcd *xhci;
3587 	struct xhci_virt_device *vdev;
3588 	struct xhci_command *command;
3589 	struct xhci_input_control_ctx *ctrl_ctx;
3590 	unsigned int ep_index;
3591 	unsigned long flags;
3592 	u32 changed_ep_bitmask;
3593 
3594 	xhci = hcd_to_xhci(hcd);
3595 	vdev = xhci->devs[udev->slot_id];
3596 
3597 	/* Set up a configure endpoint command to remove the streams rings */
3598 	spin_lock_irqsave(&xhci->lock, flags);
3599 	changed_ep_bitmask = xhci_calculate_no_streams_bitmask(xhci,
3600 			udev, eps, num_eps);
3601 	if (changed_ep_bitmask == 0) {
3602 		spin_unlock_irqrestore(&xhci->lock, flags);
3603 		return -EINVAL;
3604 	}
3605 
3606 	/* Use the xhci_command structure from the first endpoint.  We may have
3607 	 * allocated too many, but the driver may call xhci_free_streams() for
3608 	 * each endpoint it grouped into one call to xhci_alloc_streams().
3609 	 */
3610 	ep_index = xhci_get_endpoint_index(&eps[0]->desc);
3611 	command = vdev->eps[ep_index].stream_info->free_streams_command;
3612 	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
3613 	if (!ctrl_ctx) {
3614 		spin_unlock_irqrestore(&xhci->lock, flags);
3615 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
3616 				__func__);
3617 		return -EINVAL;
3618 	}
3619 
3620 	for (i = 0; i < num_eps; i++) {
3621 		struct xhci_ep_ctx *ep_ctx;
3622 
3623 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3624 		ep_ctx = xhci_get_ep_ctx(xhci, command->in_ctx, ep_index);
3625 		xhci->devs[udev->slot_id]->eps[ep_index].ep_state |=
3626 			EP_GETTING_NO_STREAMS;
3627 
3628 		xhci_endpoint_copy(xhci, command->in_ctx,
3629 				vdev->out_ctx, ep_index);
3630 		xhci_setup_no_streams_ep_input_ctx(ep_ctx,
3631 				&vdev->eps[ep_index]);
3632 	}
3633 	xhci_setup_input_ctx_for_config_ep(xhci, command->in_ctx,
3634 			vdev->out_ctx, ctrl_ctx,
3635 			changed_ep_bitmask, changed_ep_bitmask);
3636 	spin_unlock_irqrestore(&xhci->lock, flags);
3637 
3638 	/* Issue and wait for the configure endpoint command,
3639 	 * which must succeed.
3640 	 */
3641 	ret = xhci_configure_endpoint(xhci, udev, command,
3642 			false, true);
3643 
3644 	/* xHC rejected the configure endpoint command for some reason, so we
3645 	 * leave the streams rings intact.
3646 	 */
3647 	if (ret < 0)
3648 		return ret;
3649 
3650 	spin_lock_irqsave(&xhci->lock, flags);
3651 	for (i = 0; i < num_eps; i++) {
3652 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3653 		xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info);
3654 		vdev->eps[ep_index].stream_info = NULL;
3655 		/* FIXME Unset maxPstreams in endpoint context and
3656 		 * update deq ptr to point to normal string ring.
3657 		 */
3658 		vdev->eps[ep_index].ep_state &= ~EP_GETTING_NO_STREAMS;
3659 		vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS;
3660 	}
3661 	spin_unlock_irqrestore(&xhci->lock, flags);
3662 
3663 	return 0;
3664 }
3665 
3666 /*
3667  * Deletes endpoint resources for endpoints that were active before a Reset
3668  * Device command, or a Disable Slot command.  The Reset Device command leaves
3669  * the control endpoint intact, whereas the Disable Slot command deletes it.
3670  *
3671  * Must be called with xhci->lock held.
3672  */
3673 void xhci_free_device_endpoint_resources(struct xhci_hcd *xhci,
3674 	struct xhci_virt_device *virt_dev, bool drop_control_ep)
3675 {
3676 	int i;
3677 	unsigned int num_dropped_eps = 0;
3678 	unsigned int drop_flags = 0;
3679 
3680 	for (i = (drop_control_ep ? 0 : 1); i < 31; i++) {
3681 		if (virt_dev->eps[i].ring) {
3682 			drop_flags |= 1 << i;
3683 			num_dropped_eps++;
3684 		}
3685 	}
3686 	xhci->num_active_eps -= num_dropped_eps;
3687 	if (num_dropped_eps)
3688 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
3689 				"Dropped %u ep ctxs, flags = 0x%x, "
3690 				"%u now active.",
3691 				num_dropped_eps, drop_flags,
3692 				xhci->num_active_eps);
3693 }
3694 
3695 /*
3696  * This submits a Reset Device Command, which will set the device state to 0,
3697  * set the device address to 0, and disable all the endpoints except the default
3698  * control endpoint.  The USB core should come back and call
3699  * xhci_address_device(), and then re-set up the configuration.  If this is
3700  * called because of a usb_reset_and_verify_device(), then the old alternate
3701  * settings will be re-installed through the normal bandwidth allocation
3702  * functions.
3703  *
3704  * Wait for the Reset Device command to finish.  Remove all structures
3705  * associated with the endpoints that were disabled.  Clear the input device
3706  * structure? Reset the control endpoint 0 max packet size?
3707  *
3708  * If the virt_dev to be reset does not exist or does not match the udev,
3709  * it means the device is lost, possibly due to the xHC restore error and
3710  * re-initialization during S3/S4. In this case, call xhci_alloc_dev() to
3711  * re-allocate the device.
3712  */
3713 static int xhci_discover_or_reset_device(struct usb_hcd *hcd,
3714 		struct usb_device *udev)
3715 {
3716 	int ret, i;
3717 	unsigned long flags;
3718 	struct xhci_hcd *xhci;
3719 	unsigned int slot_id;
3720 	struct xhci_virt_device *virt_dev;
3721 	struct xhci_command *reset_device_cmd;
3722 	struct xhci_slot_ctx *slot_ctx;
3723 	int old_active_eps = 0;
3724 
3725 	ret = xhci_check_args(hcd, udev, NULL, 0, false, __func__);
3726 	if (ret <= 0)
3727 		return ret;
3728 	xhci = hcd_to_xhci(hcd);
3729 	slot_id = udev->slot_id;
3730 	virt_dev = xhci->devs[slot_id];
3731 	if (!virt_dev) {
3732 		xhci_dbg(xhci, "The device to be reset with slot ID %u does "
3733 				"not exist. Re-allocate the device\n", slot_id);
3734 		ret = xhci_alloc_dev(hcd, udev);
3735 		if (ret == 1)
3736 			return 0;
3737 		else
3738 			return -EINVAL;
3739 	}
3740 
3741 	if (virt_dev->tt_info)
3742 		old_active_eps = virt_dev->tt_info->active_eps;
3743 
3744 	if (virt_dev->udev != udev) {
3745 		/* If the virt_dev and the udev does not match, this virt_dev
3746 		 * may belong to another udev.
3747 		 * Re-allocate the device.
3748 		 */
3749 		xhci_dbg(xhci, "The device to be reset with slot ID %u does "
3750 				"not match the udev. Re-allocate the device\n",
3751 				slot_id);
3752 		ret = xhci_alloc_dev(hcd, udev);
3753 		if (ret == 1)
3754 			return 0;
3755 		else
3756 			return -EINVAL;
3757 	}
3758 
3759 	/* If device is not setup, there is no point in resetting it */
3760 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
3761 	if (GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)) ==
3762 						SLOT_STATE_DISABLED)
3763 		return 0;
3764 
3765 	trace_xhci_discover_or_reset_device(slot_ctx);
3766 
3767 	xhci_dbg(xhci, "Resetting device with slot ID %u\n", slot_id);
3768 	/* Allocate the command structure that holds the struct completion.
3769 	 * Assume we're in process context, since the normal device reset
3770 	 * process has to wait for the device anyway.  Storage devices are
3771 	 * reset as part of error handling, so use GFP_NOIO instead of
3772 	 * GFP_KERNEL.
3773 	 */
3774 	reset_device_cmd = xhci_alloc_command(xhci, true, GFP_NOIO);
3775 	if (!reset_device_cmd) {
3776 		xhci_dbg(xhci, "Couldn't allocate command structure.\n");
3777 		return -ENOMEM;
3778 	}
3779 
3780 	/* Attempt to submit the Reset Device command to the command ring */
3781 	spin_lock_irqsave(&xhci->lock, flags);
3782 
3783 	ret = xhci_queue_reset_device(xhci, reset_device_cmd, slot_id);
3784 	if (ret) {
3785 		xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
3786 		spin_unlock_irqrestore(&xhci->lock, flags);
3787 		goto command_cleanup;
3788 	}
3789 	xhci_ring_cmd_db(xhci);
3790 	spin_unlock_irqrestore(&xhci->lock, flags);
3791 
3792 	/* Wait for the Reset Device command to finish */
3793 	wait_for_completion(reset_device_cmd->completion);
3794 
3795 	/* The Reset Device command can't fail, according to the 0.95/0.96 spec,
3796 	 * unless we tried to reset a slot ID that wasn't enabled,
3797 	 * or the device wasn't in the addressed or configured state.
3798 	 */
3799 	ret = reset_device_cmd->status;
3800 	switch (ret) {
3801 	case COMP_COMMAND_ABORTED:
3802 	case COMP_COMMAND_RING_STOPPED:
3803 		xhci_warn(xhci, "Timeout waiting for reset device command\n");
3804 		ret = -ETIME;
3805 		goto command_cleanup;
3806 	case COMP_SLOT_NOT_ENABLED_ERROR: /* 0.95 completion for bad slot ID */
3807 	case COMP_CONTEXT_STATE_ERROR: /* 0.96 completion code for same thing */
3808 		xhci_dbg(xhci, "Can't reset device (slot ID %u) in %s state\n",
3809 				slot_id,
3810 				xhci_get_slot_state(xhci, virt_dev->out_ctx));
3811 		xhci_dbg(xhci, "Not freeing device rings.\n");
3812 		/* Don't treat this as an error.  May change my mind later. */
3813 		ret = 0;
3814 		goto command_cleanup;
3815 	case COMP_SUCCESS:
3816 		xhci_dbg(xhci, "Successful reset device command.\n");
3817 		break;
3818 	default:
3819 		if (xhci_is_vendor_info_code(xhci, ret))
3820 			break;
3821 		xhci_warn(xhci, "Unknown completion code %u for "
3822 				"reset device command.\n", ret);
3823 		ret = -EINVAL;
3824 		goto command_cleanup;
3825 	}
3826 
3827 	/* Free up host controller endpoint resources */
3828 	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
3829 		spin_lock_irqsave(&xhci->lock, flags);
3830 		/* Don't delete the default control endpoint resources */
3831 		xhci_free_device_endpoint_resources(xhci, virt_dev, false);
3832 		spin_unlock_irqrestore(&xhci->lock, flags);
3833 	}
3834 
3835 	/* Everything but endpoint 0 is disabled, so free the rings. */
3836 	for (i = 1; i < 31; i++) {
3837 		struct xhci_virt_ep *ep = &virt_dev->eps[i];
3838 
3839 		if (ep->ep_state & EP_HAS_STREAMS) {
3840 			xhci_warn(xhci, "WARN: endpoint 0x%02x has streams on device reset, freeing streams.\n",
3841 					xhci_get_endpoint_address(i));
3842 			xhci_free_stream_info(xhci, ep->stream_info);
3843 			ep->stream_info = NULL;
3844 			ep->ep_state &= ~EP_HAS_STREAMS;
3845 		}
3846 
3847 		if (ep->ring) {
3848 			xhci_debugfs_remove_endpoint(xhci, virt_dev, i);
3849 			xhci_free_endpoint_ring(xhci, virt_dev, i);
3850 		}
3851 		if (!list_empty(&virt_dev->eps[i].bw_endpoint_list))
3852 			xhci_drop_ep_from_interval_table(xhci,
3853 					&virt_dev->eps[i].bw_info,
3854 					virt_dev->bw_table,
3855 					udev,
3856 					&virt_dev->eps[i],
3857 					virt_dev->tt_info);
3858 		xhci_clear_endpoint_bw_info(&virt_dev->eps[i].bw_info);
3859 	}
3860 	/* If necessary, update the number of active TTs on this root port */
3861 	xhci_update_tt_active_eps(xhci, virt_dev, old_active_eps);
3862 	virt_dev->flags = 0;
3863 	ret = 0;
3864 
3865 command_cleanup:
3866 	xhci_free_command(xhci, reset_device_cmd);
3867 	return ret;
3868 }
3869 
3870 /*
3871  * At this point, the struct usb_device is about to go away, the device has
3872  * disconnected, and all traffic has been stopped and the endpoints have been
3873  * disabled.  Free any HC data structures associated with that device.
3874  */
3875 static void xhci_free_dev(struct usb_hcd *hcd, struct usb_device *udev)
3876 {
3877 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
3878 	struct xhci_virt_device *virt_dev;
3879 	struct xhci_slot_ctx *slot_ctx;
3880 	unsigned long flags;
3881 	int i, ret;
3882 
3883 	/*
3884 	 * We called pm_runtime_get_noresume when the device was attached.
3885 	 * Decrement the counter here to allow controller to runtime suspend
3886 	 * if no devices remain.
3887 	 */
3888 	if (xhci->quirks & XHCI_RESET_ON_RESUME)
3889 		pm_runtime_put_noidle(hcd->self.controller);
3890 
3891 	ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
3892 	/* If the host is halted due to driver unload, we still need to free the
3893 	 * device.
3894 	 */
3895 	if (ret <= 0 && ret != -ENODEV)
3896 		return;
3897 
3898 	virt_dev = xhci->devs[udev->slot_id];
3899 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
3900 	trace_xhci_free_dev(slot_ctx);
3901 
3902 	/* Stop any wayward timer functions (which may grab the lock) */
3903 	for (i = 0; i < 31; i++)
3904 		virt_dev->eps[i].ep_state &= ~EP_STOP_CMD_PENDING;
3905 	virt_dev->udev = NULL;
3906 	xhci_disable_slot(xhci, udev->slot_id);
3907 
3908 	spin_lock_irqsave(&xhci->lock, flags);
3909 	xhci_free_virt_device(xhci, udev->slot_id);
3910 	spin_unlock_irqrestore(&xhci->lock, flags);
3911 
3912 }
3913 
3914 int xhci_disable_slot(struct xhci_hcd *xhci, u32 slot_id)
3915 {
3916 	struct xhci_command *command;
3917 	unsigned long flags;
3918 	u32 state;
3919 	int ret;
3920 
3921 	command = xhci_alloc_command(xhci, true, GFP_KERNEL);
3922 	if (!command)
3923 		return -ENOMEM;
3924 
3925 	xhci_debugfs_remove_slot(xhci, slot_id);
3926 
3927 	spin_lock_irqsave(&xhci->lock, flags);
3928 	/* Don't disable the slot if the host controller is dead. */
3929 	state = readl(&xhci->op_regs->status);
3930 	if (state == 0xffffffff || (xhci->xhc_state & XHCI_STATE_DYING) ||
3931 			(xhci->xhc_state & XHCI_STATE_HALTED)) {
3932 		spin_unlock_irqrestore(&xhci->lock, flags);
3933 		kfree(command);
3934 		return -ENODEV;
3935 	}
3936 
3937 	ret = xhci_queue_slot_control(xhci, command, TRB_DISABLE_SLOT,
3938 				slot_id);
3939 	if (ret) {
3940 		spin_unlock_irqrestore(&xhci->lock, flags);
3941 		kfree(command);
3942 		return ret;
3943 	}
3944 	xhci_ring_cmd_db(xhci);
3945 	spin_unlock_irqrestore(&xhci->lock, flags);
3946 
3947 	wait_for_completion(command->completion);
3948 
3949 	if (command->status != COMP_SUCCESS)
3950 		xhci_warn(xhci, "Unsuccessful disable slot %u command, status %d\n",
3951 			  slot_id, command->status);
3952 
3953 	xhci_free_command(xhci, command);
3954 
3955 	return 0;
3956 }
3957 
3958 /*
3959  * Checks if we have enough host controller resources for the default control
3960  * endpoint.
3961  *
3962  * Must be called with xhci->lock held.
3963  */
3964 static int xhci_reserve_host_control_ep_resources(struct xhci_hcd *xhci)
3965 {
3966 	if (xhci->num_active_eps + 1 > xhci->limit_active_eps) {
3967 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
3968 				"Not enough ep ctxs: "
3969 				"%u active, need to add 1, limit is %u.",
3970 				xhci->num_active_eps, xhci->limit_active_eps);
3971 		return -ENOMEM;
3972 	}
3973 	xhci->num_active_eps += 1;
3974 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
3975 			"Adding 1 ep ctx, %u now active.",
3976 			xhci->num_active_eps);
3977 	return 0;
3978 }
3979 
3980 
3981 /*
3982  * Returns 0 if the xHC ran out of device slots, the Enable Slot command
3983  * timed out, or allocating memory failed.  Returns 1 on success.
3984  */
3985 int xhci_alloc_dev(struct usb_hcd *hcd, struct usb_device *udev)
3986 {
3987 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
3988 	struct xhci_virt_device *vdev;
3989 	struct xhci_slot_ctx *slot_ctx;
3990 	unsigned long flags;
3991 	int ret, slot_id;
3992 	struct xhci_command *command;
3993 
3994 	command = xhci_alloc_command(xhci, true, GFP_KERNEL);
3995 	if (!command)
3996 		return 0;
3997 
3998 	spin_lock_irqsave(&xhci->lock, flags);
3999 	ret = xhci_queue_slot_control(xhci, command, TRB_ENABLE_SLOT, 0);
4000 	if (ret) {
4001 		spin_unlock_irqrestore(&xhci->lock, flags);
4002 		xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
4003 		xhci_free_command(xhci, command);
4004 		return 0;
4005 	}
4006 	xhci_ring_cmd_db(xhci);
4007 	spin_unlock_irqrestore(&xhci->lock, flags);
4008 
4009 	wait_for_completion(command->completion);
4010 	slot_id = command->slot_id;
4011 
4012 	if (!slot_id || command->status != COMP_SUCCESS) {
4013 		xhci_err(xhci, "Error while assigning device slot ID: %s\n",
4014 			 xhci_trb_comp_code_string(command->status));
4015 		xhci_err(xhci, "Max number of devices this xHCI host supports is %u.\n",
4016 				HCS_MAX_SLOTS(
4017 					readl(&xhci->cap_regs->hcs_params1)));
4018 		xhci_free_command(xhci, command);
4019 		return 0;
4020 	}
4021 
4022 	xhci_free_command(xhci, command);
4023 
4024 	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
4025 		spin_lock_irqsave(&xhci->lock, flags);
4026 		ret = xhci_reserve_host_control_ep_resources(xhci);
4027 		if (ret) {
4028 			spin_unlock_irqrestore(&xhci->lock, flags);
4029 			xhci_warn(xhci, "Not enough host resources, "
4030 					"active endpoint contexts = %u\n",
4031 					xhci->num_active_eps);
4032 			goto disable_slot;
4033 		}
4034 		spin_unlock_irqrestore(&xhci->lock, flags);
4035 	}
4036 	/* Use GFP_NOIO, since this function can be called from
4037 	 * xhci_discover_or_reset_device(), which may be called as part of
4038 	 * mass storage driver error handling.
4039 	 */
4040 	if (!xhci_alloc_virt_device(xhci, slot_id, udev, GFP_NOIO)) {
4041 		xhci_warn(xhci, "Could not allocate xHCI USB device data structures\n");
4042 		goto disable_slot;
4043 	}
4044 	vdev = xhci->devs[slot_id];
4045 	slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);
4046 	trace_xhci_alloc_dev(slot_ctx);
4047 
4048 	udev->slot_id = slot_id;
4049 
4050 	xhci_debugfs_create_slot(xhci, slot_id);
4051 
4052 	/*
4053 	 * If resetting upon resume, we can't put the controller into runtime
4054 	 * suspend if there is a device attached.
4055 	 */
4056 	if (xhci->quirks & XHCI_RESET_ON_RESUME)
4057 		pm_runtime_get_noresume(hcd->self.controller);
4058 
4059 	/* Is this a LS or FS device under a HS hub? */
4060 	/* Hub or peripherial? */
4061 	return 1;
4062 
4063 disable_slot:
4064 	xhci_disable_slot(xhci, udev->slot_id);
4065 	xhci_free_virt_device(xhci, udev->slot_id);
4066 
4067 	return 0;
4068 }
4069 
4070 /**
4071  * xhci_setup_device - issues an Address Device command to assign a unique
4072  *			USB bus address.
4073  * @hcd: USB host controller data structure.
4074  * @udev: USB dev structure representing the connected device.
4075  * @setup: Enum specifying setup mode: address only or with context.
4076  * @timeout_ms: Max wait time (ms) for the command operation to complete.
4077  *
4078  * Return: 0 if successful; otherwise, negative error code.
4079  */
4080 static int xhci_setup_device(struct usb_hcd *hcd, struct usb_device *udev,
4081 			     enum xhci_setup_dev setup, unsigned int timeout_ms)
4082 {
4083 	const char *act = setup == SETUP_CONTEXT_ONLY ? "context" : "address";
4084 	unsigned long flags;
4085 	struct xhci_virt_device *virt_dev;
4086 	int ret = 0;
4087 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
4088 	struct xhci_slot_ctx *slot_ctx;
4089 	struct xhci_input_control_ctx *ctrl_ctx;
4090 	u64 temp_64;
4091 	struct xhci_command *command = NULL;
4092 
4093 	mutex_lock(&xhci->mutex);
4094 
4095 	if (xhci->xhc_state) {	/* dying, removing or halted */
4096 		ret = -ESHUTDOWN;
4097 		goto out;
4098 	}
4099 
4100 	if (!udev->slot_id) {
4101 		xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4102 				"Bad Slot ID %d", udev->slot_id);
4103 		ret = -EINVAL;
4104 		goto out;
4105 	}
4106 
4107 	virt_dev = xhci->devs[udev->slot_id];
4108 
4109 	if (WARN_ON(!virt_dev)) {
4110 		/*
4111 		 * In plug/unplug torture test with an NEC controller,
4112 		 * a zero-dereference was observed once due to virt_dev = 0.
4113 		 * Print useful debug rather than crash if it is observed again!
4114 		 */
4115 		xhci_warn(xhci, "Virt dev invalid for slot_id 0x%x!\n",
4116 			udev->slot_id);
4117 		ret = -EINVAL;
4118 		goto out;
4119 	}
4120 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
4121 	trace_xhci_setup_device_slot(slot_ctx);
4122 
4123 	if (setup == SETUP_CONTEXT_ONLY) {
4124 		if (GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)) ==
4125 		    SLOT_STATE_DEFAULT) {
4126 			xhci_dbg(xhci, "Slot already in default state\n");
4127 			goto out;
4128 		}
4129 	}
4130 
4131 	command = xhci_alloc_command(xhci, true, GFP_KERNEL);
4132 	if (!command) {
4133 		ret = -ENOMEM;
4134 		goto out;
4135 	}
4136 
4137 	command->in_ctx = virt_dev->in_ctx;
4138 	command->timeout_ms = timeout_ms;
4139 
4140 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
4141 	ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
4142 	if (!ctrl_ctx) {
4143 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
4144 				__func__);
4145 		ret = -EINVAL;
4146 		goto out;
4147 	}
4148 	/*
4149 	 * If this is the first Set Address since device plug-in or
4150 	 * virt_device realloaction after a resume with an xHCI power loss,
4151 	 * then set up the slot context.
4152 	 */
4153 	if (!slot_ctx->dev_info)
4154 		xhci_setup_addressable_virt_dev(xhci, udev);
4155 	/* Otherwise, update the control endpoint ring enqueue pointer. */
4156 	else
4157 		xhci_copy_ep0_dequeue_into_input_ctx(xhci, udev);
4158 	ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG | EP0_FLAG);
4159 	ctrl_ctx->drop_flags = 0;
4160 
4161 	trace_xhci_address_ctx(xhci, virt_dev->in_ctx,
4162 				le32_to_cpu(slot_ctx->dev_info) >> 27);
4163 
4164 	trace_xhci_address_ctrl_ctx(ctrl_ctx);
4165 	spin_lock_irqsave(&xhci->lock, flags);
4166 	trace_xhci_setup_device(virt_dev);
4167 	ret = xhci_queue_address_device(xhci, command, virt_dev->in_ctx->dma,
4168 					udev->slot_id, setup);
4169 	if (ret) {
4170 		spin_unlock_irqrestore(&xhci->lock, flags);
4171 		xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4172 				"FIXME: allocate a command ring segment");
4173 		goto out;
4174 	}
4175 	xhci_ring_cmd_db(xhci);
4176 	spin_unlock_irqrestore(&xhci->lock, flags);
4177 
4178 	/* ctrl tx can take up to 5 sec; XXX: need more time for xHC? */
4179 	wait_for_completion(command->completion);
4180 
4181 	/* FIXME: From section 4.3.4: "Software shall be responsible for timing
4182 	 * the SetAddress() "recovery interval" required by USB and aborting the
4183 	 * command on a timeout.
4184 	 */
4185 	switch (command->status) {
4186 	case COMP_COMMAND_ABORTED:
4187 	case COMP_COMMAND_RING_STOPPED:
4188 		xhci_warn(xhci, "Timeout while waiting for setup device command\n");
4189 		ret = -ETIME;
4190 		break;
4191 	case COMP_CONTEXT_STATE_ERROR:
4192 	case COMP_SLOT_NOT_ENABLED_ERROR:
4193 		xhci_err(xhci, "Setup ERROR: setup %s command for slot %d.\n",
4194 			 act, udev->slot_id);
4195 		ret = -EINVAL;
4196 		break;
4197 	case COMP_USB_TRANSACTION_ERROR:
4198 		dev_warn(&udev->dev, "Device not responding to setup %s.\n", act);
4199 
4200 		mutex_unlock(&xhci->mutex);
4201 		ret = xhci_disable_slot(xhci, udev->slot_id);
4202 		xhci_free_virt_device(xhci, udev->slot_id);
4203 		if (!ret)
4204 			xhci_alloc_dev(hcd, udev);
4205 		kfree(command->completion);
4206 		kfree(command);
4207 		return -EPROTO;
4208 	case COMP_INCOMPATIBLE_DEVICE_ERROR:
4209 		dev_warn(&udev->dev,
4210 			 "ERROR: Incompatible device for setup %s command\n", act);
4211 		ret = -ENODEV;
4212 		break;
4213 	case COMP_SUCCESS:
4214 		xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4215 			       "Successful setup %s command", act);
4216 		break;
4217 	default:
4218 		xhci_err(xhci,
4219 			 "ERROR: unexpected setup %s command completion code 0x%x.\n",
4220 			 act, command->status);
4221 		trace_xhci_address_ctx(xhci, virt_dev->out_ctx, 1);
4222 		ret = -EINVAL;
4223 		break;
4224 	}
4225 	if (ret)
4226 		goto out;
4227 	temp_64 = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
4228 	xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4229 			"Op regs DCBAA ptr = %#016llx", temp_64);
4230 	xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4231 		"Slot ID %d dcbaa entry @%p = %#016llx",
4232 		udev->slot_id,
4233 		&xhci->dcbaa->dev_context_ptrs[udev->slot_id],
4234 		(unsigned long long)
4235 		le64_to_cpu(xhci->dcbaa->dev_context_ptrs[udev->slot_id]));
4236 	xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4237 			"Output Context DMA address = %#08llx",
4238 			(unsigned long long)virt_dev->out_ctx->dma);
4239 	trace_xhci_address_ctx(xhci, virt_dev->in_ctx,
4240 				le32_to_cpu(slot_ctx->dev_info) >> 27);
4241 	/*
4242 	 * USB core uses address 1 for the roothubs, so we add one to the
4243 	 * address given back to us by the HC.
4244 	 */
4245 	trace_xhci_address_ctx(xhci, virt_dev->out_ctx,
4246 				le32_to_cpu(slot_ctx->dev_info) >> 27);
4247 	/* Zero the input context control for later use */
4248 	ctrl_ctx->add_flags = 0;
4249 	ctrl_ctx->drop_flags = 0;
4250 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
4251 	udev->devaddr = (u8)(le32_to_cpu(slot_ctx->dev_state) & DEV_ADDR_MASK);
4252 
4253 	xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4254 		       "Internal device address = %d",
4255 		       le32_to_cpu(slot_ctx->dev_state) & DEV_ADDR_MASK);
4256 out:
4257 	mutex_unlock(&xhci->mutex);
4258 	if (command) {
4259 		kfree(command->completion);
4260 		kfree(command);
4261 	}
4262 	return ret;
4263 }
4264 
4265 static int xhci_address_device(struct usb_hcd *hcd, struct usb_device *udev,
4266 			       unsigned int timeout_ms)
4267 {
4268 	return xhci_setup_device(hcd, udev, SETUP_CONTEXT_ADDRESS, timeout_ms);
4269 }
4270 
4271 static int xhci_enable_device(struct usb_hcd *hcd, struct usb_device *udev)
4272 {
4273 	return xhci_setup_device(hcd, udev, SETUP_CONTEXT_ONLY,
4274 				 XHCI_CMD_DEFAULT_TIMEOUT);
4275 }
4276 
4277 /*
4278  * Transfer the port index into real index in the HW port status
4279  * registers. Caculate offset between the port's PORTSC register
4280  * and port status base. Divide the number of per port register
4281  * to get the real index. The raw port number bases 1.
4282  */
4283 int xhci_find_raw_port_number(struct usb_hcd *hcd, int port1)
4284 {
4285 	struct xhci_hub *rhub;
4286 
4287 	rhub = xhci_get_rhub(hcd);
4288 	return rhub->ports[port1 - 1]->hw_portnum + 1;
4289 }
4290 
4291 /*
4292  * Issue an Evaluate Context command to change the Maximum Exit Latency in the
4293  * slot context.  If that succeeds, store the new MEL in the xhci_virt_device.
4294  */
4295 static int __maybe_unused xhci_change_max_exit_latency(struct xhci_hcd *xhci,
4296 			struct usb_device *udev, u16 max_exit_latency)
4297 {
4298 	struct xhci_virt_device *virt_dev;
4299 	struct xhci_command *command;
4300 	struct xhci_input_control_ctx *ctrl_ctx;
4301 	struct xhci_slot_ctx *slot_ctx;
4302 	unsigned long flags;
4303 	int ret;
4304 
4305 	command = xhci_alloc_command_with_ctx(xhci, true, GFP_KERNEL);
4306 	if (!command)
4307 		return -ENOMEM;
4308 
4309 	spin_lock_irqsave(&xhci->lock, flags);
4310 
4311 	virt_dev = xhci->devs[udev->slot_id];
4312 
4313 	/*
4314 	 * virt_dev might not exists yet if xHC resumed from hibernate (S4) and
4315 	 * xHC was re-initialized. Exit latency will be set later after
4316 	 * hub_port_finish_reset() is done and xhci->devs[] are re-allocated
4317 	 */
4318 
4319 	if (!virt_dev || max_exit_latency == virt_dev->current_mel) {
4320 		spin_unlock_irqrestore(&xhci->lock, flags);
4321 		xhci_free_command(xhci, command);
4322 		return 0;
4323 	}
4324 
4325 	/* Attempt to issue an Evaluate Context command to change the MEL. */
4326 	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
4327 	if (!ctrl_ctx) {
4328 		spin_unlock_irqrestore(&xhci->lock, flags);
4329 		xhci_free_command(xhci, command);
4330 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
4331 				__func__);
4332 		return -ENOMEM;
4333 	}
4334 
4335 	xhci_slot_copy(xhci, command->in_ctx, virt_dev->out_ctx);
4336 	spin_unlock_irqrestore(&xhci->lock, flags);
4337 
4338 	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
4339 	slot_ctx = xhci_get_slot_ctx(xhci, command->in_ctx);
4340 	slot_ctx->dev_info2 &= cpu_to_le32(~((u32) MAX_EXIT));
4341 	slot_ctx->dev_info2 |= cpu_to_le32(max_exit_latency);
4342 	slot_ctx->dev_state = 0;
4343 
4344 	xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
4345 			"Set up evaluate context for LPM MEL change.");
4346 
4347 	/* Issue and wait for the evaluate context command. */
4348 	ret = xhci_configure_endpoint(xhci, udev, command,
4349 			true, true);
4350 
4351 	if (!ret) {
4352 		spin_lock_irqsave(&xhci->lock, flags);
4353 		virt_dev->current_mel = max_exit_latency;
4354 		spin_unlock_irqrestore(&xhci->lock, flags);
4355 	}
4356 
4357 	xhci_free_command(xhci, command);
4358 
4359 	return ret;
4360 }
4361 
4362 #ifdef CONFIG_PM
4363 
4364 /* BESL to HIRD Encoding array for USB2 LPM */
4365 static int xhci_besl_encoding[16] = {125, 150, 200, 300, 400, 500, 1000, 2000,
4366 	3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000};
4367 
4368 /* Calculate HIRD/BESL for USB2 PORTPMSC*/
4369 static int xhci_calculate_hird_besl(struct xhci_hcd *xhci,
4370 					struct usb_device *udev)
4371 {
4372 	int u2del, besl, besl_host;
4373 	int besl_device = 0;
4374 	u32 field;
4375 
4376 	u2del = HCS_U2_LATENCY(xhci->hcs_params3);
4377 	field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);
4378 
4379 	if (field & USB_BESL_SUPPORT) {
4380 		for (besl_host = 0; besl_host < 16; besl_host++) {
4381 			if (xhci_besl_encoding[besl_host] >= u2del)
4382 				break;
4383 		}
4384 		/* Use baseline BESL value as default */
4385 		if (field & USB_BESL_BASELINE_VALID)
4386 			besl_device = USB_GET_BESL_BASELINE(field);
4387 		else if (field & USB_BESL_DEEP_VALID)
4388 			besl_device = USB_GET_BESL_DEEP(field);
4389 	} else {
4390 		if (u2del <= 50)
4391 			besl_host = 0;
4392 		else
4393 			besl_host = (u2del - 51) / 75 + 1;
4394 	}
4395 
4396 	besl = besl_host + besl_device;
4397 	if (besl > 15)
4398 		besl = 15;
4399 
4400 	return besl;
4401 }
4402 
4403 /* Calculate BESLD, L1 timeout and HIRDM for USB2 PORTHLPMC */
4404 static int xhci_calculate_usb2_hw_lpm_params(struct usb_device *udev)
4405 {
4406 	u32 field;
4407 	int l1;
4408 	int besld = 0;
4409 	int hirdm = 0;
4410 
4411 	field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);
4412 
4413 	/* xHCI l1 is set in steps of 256us, xHCI 1.0 section 5.4.11.2 */
4414 	l1 = udev->l1_params.timeout / 256;
4415 
4416 	/* device has preferred BESLD */
4417 	if (field & USB_BESL_DEEP_VALID) {
4418 		besld = USB_GET_BESL_DEEP(field);
4419 		hirdm = 1;
4420 	}
4421 
4422 	return PORT_BESLD(besld) | PORT_L1_TIMEOUT(l1) | PORT_HIRDM(hirdm);
4423 }
4424 
4425 static int xhci_set_usb2_hardware_lpm(struct usb_hcd *hcd,
4426 			struct usb_device *udev, int enable)
4427 {
4428 	struct xhci_hcd	*xhci = hcd_to_xhci(hcd);
4429 	struct xhci_port **ports;
4430 	__le32 __iomem	*pm_addr, *hlpm_addr;
4431 	u32		pm_val, hlpm_val, field;
4432 	unsigned int	port_num;
4433 	unsigned long	flags;
4434 	int		hird, exit_latency;
4435 	int		ret;
4436 
4437 	if (xhci->quirks & XHCI_HW_LPM_DISABLE)
4438 		return -EPERM;
4439 
4440 	if (hcd->speed >= HCD_USB3 || !xhci->hw_lpm_support ||
4441 			!udev->lpm_capable)
4442 		return -EPERM;
4443 
4444 	if (!udev->parent || udev->parent->parent ||
4445 			udev->descriptor.bDeviceClass == USB_CLASS_HUB)
4446 		return -EPERM;
4447 
4448 	if (udev->usb2_hw_lpm_capable != 1)
4449 		return -EPERM;
4450 
4451 	spin_lock_irqsave(&xhci->lock, flags);
4452 
4453 	ports = xhci->usb2_rhub.ports;
4454 	port_num = udev->portnum - 1;
4455 	pm_addr = ports[port_num]->addr + PORTPMSC;
4456 	pm_val = readl(pm_addr);
4457 	hlpm_addr = ports[port_num]->addr + PORTHLPMC;
4458 
4459 	xhci_dbg(xhci, "%s port %d USB2 hardware LPM\n",
4460 			enable ? "enable" : "disable", port_num + 1);
4461 
4462 	if (enable) {
4463 		/* Host supports BESL timeout instead of HIRD */
4464 		if (udev->usb2_hw_lpm_besl_capable) {
4465 			/* if device doesn't have a preferred BESL value use a
4466 			 * default one which works with mixed HIRD and BESL
4467 			 * systems. See XHCI_DEFAULT_BESL definition in xhci.h
4468 			 */
4469 			field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);
4470 			if ((field & USB_BESL_SUPPORT) &&
4471 			    (field & USB_BESL_BASELINE_VALID))
4472 				hird = USB_GET_BESL_BASELINE(field);
4473 			else
4474 				hird = udev->l1_params.besl;
4475 
4476 			exit_latency = xhci_besl_encoding[hird];
4477 			spin_unlock_irqrestore(&xhci->lock, flags);
4478 
4479 			ret = xhci_change_max_exit_latency(xhci, udev,
4480 							   exit_latency);
4481 			if (ret < 0)
4482 				return ret;
4483 			spin_lock_irqsave(&xhci->lock, flags);
4484 
4485 			hlpm_val = xhci_calculate_usb2_hw_lpm_params(udev);
4486 			writel(hlpm_val, hlpm_addr);
4487 			/* flush write */
4488 			readl(hlpm_addr);
4489 		} else {
4490 			hird = xhci_calculate_hird_besl(xhci, udev);
4491 		}
4492 
4493 		pm_val &= ~PORT_HIRD_MASK;
4494 		pm_val |= PORT_HIRD(hird) | PORT_RWE | PORT_L1DS(udev->slot_id);
4495 		writel(pm_val, pm_addr);
4496 		pm_val = readl(pm_addr);
4497 		pm_val |= PORT_HLE;
4498 		writel(pm_val, pm_addr);
4499 		/* flush write */
4500 		readl(pm_addr);
4501 	} else {
4502 		pm_val &= ~(PORT_HLE | PORT_RWE | PORT_HIRD_MASK | PORT_L1DS_MASK);
4503 		writel(pm_val, pm_addr);
4504 		/* flush write */
4505 		readl(pm_addr);
4506 		if (udev->usb2_hw_lpm_besl_capable) {
4507 			spin_unlock_irqrestore(&xhci->lock, flags);
4508 			xhci_change_max_exit_latency(xhci, udev, 0);
4509 			readl_poll_timeout(ports[port_num]->addr, pm_val,
4510 					   (pm_val & PORT_PLS_MASK) == XDEV_U0,
4511 					   100, 10000);
4512 			return 0;
4513 		}
4514 	}
4515 
4516 	spin_unlock_irqrestore(&xhci->lock, flags);
4517 	return 0;
4518 }
4519 
4520 static int xhci_update_device(struct usb_hcd *hcd, struct usb_device *udev)
4521 {
4522 	struct xhci_hcd	*xhci = hcd_to_xhci(hcd);
4523 	struct xhci_port *port;
4524 	u32 capability;
4525 
4526 	if (hcd->speed >= HCD_USB3 || !udev->lpm_capable || !xhci->hw_lpm_support)
4527 		return 0;
4528 
4529 	/* we only support lpm for non-hub device connected to root hub yet */
4530 	if (!udev->parent || udev->parent->parent ||
4531 			udev->descriptor.bDeviceClass == USB_CLASS_HUB)
4532 		return 0;
4533 
4534 	port = xhci->usb2_rhub.ports[udev->portnum - 1];
4535 	capability = port->port_cap->protocol_caps;
4536 
4537 	if (capability & XHCI_HLC) {
4538 		udev->usb2_hw_lpm_capable = 1;
4539 		udev->l1_params.timeout = XHCI_L1_TIMEOUT;
4540 		udev->l1_params.besl = XHCI_DEFAULT_BESL;
4541 		if (capability & XHCI_BLC)
4542 			udev->usb2_hw_lpm_besl_capable = 1;
4543 	}
4544 
4545 	return 0;
4546 }
4547 
4548 /*---------------------- USB 3.0 Link PM functions ------------------------*/
4549 
4550 /* Service interval in nanoseconds = 2^(bInterval - 1) * 125us * 1000ns / 1us */
4551 static unsigned long long xhci_service_interval_to_ns(
4552 		struct usb_endpoint_descriptor *desc)
4553 {
4554 	return (1ULL << (desc->bInterval - 1)) * 125 * 1000;
4555 }
4556 
4557 static u16 xhci_get_timeout_no_hub_lpm(struct usb_device *udev,
4558 		enum usb3_link_state state)
4559 {
4560 	unsigned long long sel;
4561 	unsigned long long pel;
4562 	unsigned int max_sel_pel;
4563 	char *state_name;
4564 
4565 	switch (state) {
4566 	case USB3_LPM_U1:
4567 		/* Convert SEL and PEL stored in nanoseconds to microseconds */
4568 		sel = DIV_ROUND_UP(udev->u1_params.sel, 1000);
4569 		pel = DIV_ROUND_UP(udev->u1_params.pel, 1000);
4570 		max_sel_pel = USB3_LPM_MAX_U1_SEL_PEL;
4571 		state_name = "U1";
4572 		break;
4573 	case USB3_LPM_U2:
4574 		sel = DIV_ROUND_UP(udev->u2_params.sel, 1000);
4575 		pel = DIV_ROUND_UP(udev->u2_params.pel, 1000);
4576 		max_sel_pel = USB3_LPM_MAX_U2_SEL_PEL;
4577 		state_name = "U2";
4578 		break;
4579 	default:
4580 		dev_warn(&udev->dev, "%s: Can't get timeout for non-U1 or U2 state.\n",
4581 				__func__);
4582 		return USB3_LPM_DISABLED;
4583 	}
4584 
4585 	if (sel <= max_sel_pel && pel <= max_sel_pel)
4586 		return USB3_LPM_DEVICE_INITIATED;
4587 
4588 	if (sel > max_sel_pel)
4589 		dev_dbg(&udev->dev, "Device-initiated %s disabled "
4590 				"due to long SEL %llu ms\n",
4591 				state_name, sel);
4592 	else
4593 		dev_dbg(&udev->dev, "Device-initiated %s disabled "
4594 				"due to long PEL %llu ms\n",
4595 				state_name, pel);
4596 	return USB3_LPM_DISABLED;
4597 }
4598 
4599 /* The U1 timeout should be the maximum of the following values:
4600  *  - For control endpoints, U1 system exit latency (SEL) * 3
4601  *  - For bulk endpoints, U1 SEL * 5
4602  *  - For interrupt endpoints:
4603  *    - Notification EPs, U1 SEL * 3
4604  *    - Periodic EPs, max(105% of bInterval, U1 SEL * 2)
4605  *  - For isochronous endpoints, max(105% of bInterval, U1 SEL * 2)
4606  */
4607 static unsigned long long xhci_calculate_intel_u1_timeout(
4608 		struct usb_device *udev,
4609 		struct usb_endpoint_descriptor *desc)
4610 {
4611 	unsigned long long timeout_ns;
4612 	int ep_type;
4613 	int intr_type;
4614 
4615 	ep_type = usb_endpoint_type(desc);
4616 	switch (ep_type) {
4617 	case USB_ENDPOINT_XFER_CONTROL:
4618 		timeout_ns = udev->u1_params.sel * 3;
4619 		break;
4620 	case USB_ENDPOINT_XFER_BULK:
4621 		timeout_ns = udev->u1_params.sel * 5;
4622 		break;
4623 	case USB_ENDPOINT_XFER_INT:
4624 		intr_type = usb_endpoint_interrupt_type(desc);
4625 		if (intr_type == USB_ENDPOINT_INTR_NOTIFICATION) {
4626 			timeout_ns = udev->u1_params.sel * 3;
4627 			break;
4628 		}
4629 		/* Otherwise the calculation is the same as isoc eps */
4630 		fallthrough;
4631 	case USB_ENDPOINT_XFER_ISOC:
4632 		timeout_ns = xhci_service_interval_to_ns(desc);
4633 		timeout_ns = DIV_ROUND_UP_ULL(timeout_ns * 105, 100);
4634 		if (timeout_ns < udev->u1_params.sel * 2)
4635 			timeout_ns = udev->u1_params.sel * 2;
4636 		break;
4637 	default:
4638 		return 0;
4639 	}
4640 
4641 	return timeout_ns;
4642 }
4643 
4644 /* Returns the hub-encoded U1 timeout value. */
4645 static u16 xhci_calculate_u1_timeout(struct xhci_hcd *xhci,
4646 		struct usb_device *udev,
4647 		struct usb_endpoint_descriptor *desc)
4648 {
4649 	unsigned long long timeout_ns;
4650 
4651 	/* Prevent U1 if service interval is shorter than U1 exit latency */
4652 	if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) {
4653 		if (xhci_service_interval_to_ns(desc) <= udev->u1_params.mel) {
4654 			dev_dbg(&udev->dev, "Disable U1, ESIT shorter than exit latency\n");
4655 			return USB3_LPM_DISABLED;
4656 		}
4657 	}
4658 
4659 	if (xhci->quirks & (XHCI_INTEL_HOST | XHCI_ZHAOXIN_HOST))
4660 		timeout_ns = xhci_calculate_intel_u1_timeout(udev, desc);
4661 	else
4662 		timeout_ns = udev->u1_params.sel;
4663 
4664 	/* The U1 timeout is encoded in 1us intervals.
4665 	 * Don't return a timeout of zero, because that's USB3_LPM_DISABLED.
4666 	 */
4667 	if (timeout_ns == USB3_LPM_DISABLED)
4668 		timeout_ns = 1;
4669 	else
4670 		timeout_ns = DIV_ROUND_UP_ULL(timeout_ns, 1000);
4671 
4672 	/* If the necessary timeout value is bigger than what we can set in the
4673 	 * USB 3.0 hub, we have to disable hub-initiated U1.
4674 	 */
4675 	if (timeout_ns <= USB3_LPM_U1_MAX_TIMEOUT)
4676 		return timeout_ns;
4677 	dev_dbg(&udev->dev, "Hub-initiated U1 disabled "
4678 			"due to long timeout %llu ms\n", timeout_ns);
4679 	return xhci_get_timeout_no_hub_lpm(udev, USB3_LPM_U1);
4680 }
4681 
4682 /* The U2 timeout should be the maximum of:
4683  *  - 10 ms (to avoid the bandwidth impact on the scheduler)
4684  *  - largest bInterval of any active periodic endpoint (to avoid going
4685  *    into lower power link states between intervals).
4686  *  - the U2 Exit Latency of the device
4687  */
4688 static unsigned long long xhci_calculate_intel_u2_timeout(
4689 		struct usb_device *udev,
4690 		struct usb_endpoint_descriptor *desc)
4691 {
4692 	unsigned long long timeout_ns;
4693 	unsigned long long u2_del_ns;
4694 
4695 	timeout_ns = 10 * 1000 * 1000;
4696 
4697 	if ((usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) &&
4698 			(xhci_service_interval_to_ns(desc) > timeout_ns))
4699 		timeout_ns = xhci_service_interval_to_ns(desc);
4700 
4701 	u2_del_ns = le16_to_cpu(udev->bos->ss_cap->bU2DevExitLat) * 1000ULL;
4702 	if (u2_del_ns > timeout_ns)
4703 		timeout_ns = u2_del_ns;
4704 
4705 	return timeout_ns;
4706 }
4707 
4708 /* Returns the hub-encoded U2 timeout value. */
4709 static u16 xhci_calculate_u2_timeout(struct xhci_hcd *xhci,
4710 		struct usb_device *udev,
4711 		struct usb_endpoint_descriptor *desc)
4712 {
4713 	unsigned long long timeout_ns;
4714 
4715 	/* Prevent U2 if service interval is shorter than U2 exit latency */
4716 	if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) {
4717 		if (xhci_service_interval_to_ns(desc) <= udev->u2_params.mel) {
4718 			dev_dbg(&udev->dev, "Disable U2, ESIT shorter than exit latency\n");
4719 			return USB3_LPM_DISABLED;
4720 		}
4721 	}
4722 
4723 	if (xhci->quirks & (XHCI_INTEL_HOST | XHCI_ZHAOXIN_HOST))
4724 		timeout_ns = xhci_calculate_intel_u2_timeout(udev, desc);
4725 	else
4726 		timeout_ns = udev->u2_params.sel;
4727 
4728 	/* The U2 timeout is encoded in 256us intervals */
4729 	timeout_ns = DIV_ROUND_UP_ULL(timeout_ns, 256 * 1000);
4730 	/* If the necessary timeout value is bigger than what we can set in the
4731 	 * USB 3.0 hub, we have to disable hub-initiated U2.
4732 	 */
4733 	if (timeout_ns <= USB3_LPM_U2_MAX_TIMEOUT)
4734 		return timeout_ns;
4735 	dev_dbg(&udev->dev, "Hub-initiated U2 disabled "
4736 			"due to long timeout %llu ms\n", timeout_ns);
4737 	return xhci_get_timeout_no_hub_lpm(udev, USB3_LPM_U2);
4738 }
4739 
4740 static u16 xhci_call_host_update_timeout_for_endpoint(struct xhci_hcd *xhci,
4741 		struct usb_device *udev,
4742 		struct usb_endpoint_descriptor *desc,
4743 		enum usb3_link_state state,
4744 		u16 *timeout)
4745 {
4746 	if (state == USB3_LPM_U1)
4747 		return xhci_calculate_u1_timeout(xhci, udev, desc);
4748 	else if (state == USB3_LPM_U2)
4749 		return xhci_calculate_u2_timeout(xhci, udev, desc);
4750 
4751 	return USB3_LPM_DISABLED;
4752 }
4753 
4754 static int xhci_update_timeout_for_endpoint(struct xhci_hcd *xhci,
4755 		struct usb_device *udev,
4756 		struct usb_endpoint_descriptor *desc,
4757 		enum usb3_link_state state,
4758 		u16 *timeout)
4759 {
4760 	u16 alt_timeout;
4761 
4762 	alt_timeout = xhci_call_host_update_timeout_for_endpoint(xhci, udev,
4763 		desc, state, timeout);
4764 
4765 	/* If we found we can't enable hub-initiated LPM, and
4766 	 * the U1 or U2 exit latency was too high to allow
4767 	 * device-initiated LPM as well, then we will disable LPM
4768 	 * for this device, so stop searching any further.
4769 	 */
4770 	if (alt_timeout == USB3_LPM_DISABLED) {
4771 		*timeout = alt_timeout;
4772 		return -E2BIG;
4773 	}
4774 	if (alt_timeout > *timeout)
4775 		*timeout = alt_timeout;
4776 	return 0;
4777 }
4778 
4779 static int xhci_update_timeout_for_interface(struct xhci_hcd *xhci,
4780 		struct usb_device *udev,
4781 		struct usb_host_interface *alt,
4782 		enum usb3_link_state state,
4783 		u16 *timeout)
4784 {
4785 	int j;
4786 
4787 	for (j = 0; j < alt->desc.bNumEndpoints; j++) {
4788 		if (xhci_update_timeout_for_endpoint(xhci, udev,
4789 					&alt->endpoint[j].desc, state, timeout))
4790 			return -E2BIG;
4791 	}
4792 	return 0;
4793 }
4794 
4795 static int xhci_check_tier_policy(struct xhci_hcd *xhci,
4796 		struct usb_device *udev,
4797 		enum usb3_link_state state)
4798 {
4799 	struct usb_device *parent = udev->parent;
4800 	int tier = 1; /* roothub is tier1 */
4801 
4802 	while (parent) {
4803 		parent = parent->parent;
4804 		tier++;
4805 	}
4806 
4807 	if (xhci->quirks & XHCI_INTEL_HOST && tier > 3)
4808 		goto fail;
4809 	if (xhci->quirks & XHCI_ZHAOXIN_HOST && tier > 2)
4810 		goto fail;
4811 
4812 	return 0;
4813 fail:
4814 	dev_dbg(&udev->dev, "Tier policy prevents U1/U2 LPM states for devices at tier %d\n",
4815 			tier);
4816 	return -E2BIG;
4817 }
4818 
4819 /* Returns the U1 or U2 timeout that should be enabled.
4820  * If the tier check or timeout setting functions return with a non-zero exit
4821  * code, that means the timeout value has been finalized and we shouldn't look
4822  * at any more endpoints.
4823  */
4824 static u16 xhci_calculate_lpm_timeout(struct usb_hcd *hcd,
4825 			struct usb_device *udev, enum usb3_link_state state)
4826 {
4827 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
4828 	struct usb_host_config *config;
4829 	char *state_name;
4830 	int i;
4831 	u16 timeout = USB3_LPM_DISABLED;
4832 
4833 	if (state == USB3_LPM_U1)
4834 		state_name = "U1";
4835 	else if (state == USB3_LPM_U2)
4836 		state_name = "U2";
4837 	else {
4838 		dev_warn(&udev->dev, "Can't enable unknown link state %i\n",
4839 				state);
4840 		return timeout;
4841 	}
4842 
4843 	/* Gather some information about the currently installed configuration
4844 	 * and alternate interface settings.
4845 	 */
4846 	if (xhci_update_timeout_for_endpoint(xhci, udev, &udev->ep0.desc,
4847 			state, &timeout))
4848 		return timeout;
4849 
4850 	config = udev->actconfig;
4851 	if (!config)
4852 		return timeout;
4853 
4854 	for (i = 0; i < config->desc.bNumInterfaces; i++) {
4855 		struct usb_driver *driver;
4856 		struct usb_interface *intf = config->interface[i];
4857 
4858 		if (!intf)
4859 			continue;
4860 
4861 		/* Check if any currently bound drivers want hub-initiated LPM
4862 		 * disabled.
4863 		 */
4864 		if (intf->dev.driver) {
4865 			driver = to_usb_driver(intf->dev.driver);
4866 			if (driver && driver->disable_hub_initiated_lpm) {
4867 				dev_dbg(&udev->dev, "Hub-initiated %s disabled at request of driver %s\n",
4868 					state_name, driver->name);
4869 				timeout = xhci_get_timeout_no_hub_lpm(udev,
4870 								      state);
4871 				if (timeout == USB3_LPM_DISABLED)
4872 					return timeout;
4873 			}
4874 		}
4875 
4876 		/* Not sure how this could happen... */
4877 		if (!intf->cur_altsetting)
4878 			continue;
4879 
4880 		if (xhci_update_timeout_for_interface(xhci, udev,
4881 					intf->cur_altsetting,
4882 					state, &timeout))
4883 			return timeout;
4884 	}
4885 	return timeout;
4886 }
4887 
4888 static int calculate_max_exit_latency(struct usb_device *udev,
4889 		enum usb3_link_state state_changed,
4890 		u16 hub_encoded_timeout)
4891 {
4892 	unsigned long long u1_mel_us = 0;
4893 	unsigned long long u2_mel_us = 0;
4894 	unsigned long long mel_us = 0;
4895 	bool disabling_u1;
4896 	bool disabling_u2;
4897 	bool enabling_u1;
4898 	bool enabling_u2;
4899 
4900 	disabling_u1 = (state_changed == USB3_LPM_U1 &&
4901 			hub_encoded_timeout == USB3_LPM_DISABLED);
4902 	disabling_u2 = (state_changed == USB3_LPM_U2 &&
4903 			hub_encoded_timeout == USB3_LPM_DISABLED);
4904 
4905 	enabling_u1 = (state_changed == USB3_LPM_U1 &&
4906 			hub_encoded_timeout != USB3_LPM_DISABLED);
4907 	enabling_u2 = (state_changed == USB3_LPM_U2 &&
4908 			hub_encoded_timeout != USB3_LPM_DISABLED);
4909 
4910 	/* If U1 was already enabled and we're not disabling it,
4911 	 * or we're going to enable U1, account for the U1 max exit latency.
4912 	 */
4913 	if ((udev->u1_params.timeout != USB3_LPM_DISABLED && !disabling_u1) ||
4914 			enabling_u1)
4915 		u1_mel_us = DIV_ROUND_UP(udev->u1_params.mel, 1000);
4916 	if ((udev->u2_params.timeout != USB3_LPM_DISABLED && !disabling_u2) ||
4917 			enabling_u2)
4918 		u2_mel_us = DIV_ROUND_UP(udev->u2_params.mel, 1000);
4919 
4920 	mel_us = max(u1_mel_us, u2_mel_us);
4921 
4922 	/* xHCI host controller max exit latency field is only 16 bits wide. */
4923 	if (mel_us > MAX_EXIT) {
4924 		dev_warn(&udev->dev, "Link PM max exit latency of %lluus "
4925 				"is too big.\n", mel_us);
4926 		return -E2BIG;
4927 	}
4928 	return mel_us;
4929 }
4930 
4931 /* Returns the USB3 hub-encoded value for the U1/U2 timeout. */
4932 static int xhci_enable_usb3_lpm_timeout(struct usb_hcd *hcd,
4933 			struct usb_device *udev, enum usb3_link_state state)
4934 {
4935 	struct xhci_hcd	*xhci;
4936 	struct xhci_port *port;
4937 	u16 hub_encoded_timeout;
4938 	int mel;
4939 	int ret;
4940 
4941 	xhci = hcd_to_xhci(hcd);
4942 	/* The LPM timeout values are pretty host-controller specific, so don't
4943 	 * enable hub-initiated timeouts unless the vendor has provided
4944 	 * information about their timeout algorithm.
4945 	 */
4946 	if (!xhci || !(xhci->quirks & XHCI_LPM_SUPPORT) ||
4947 			!xhci->devs[udev->slot_id])
4948 		return USB3_LPM_DISABLED;
4949 
4950 	if (xhci_check_tier_policy(xhci, udev, state) < 0)
4951 		return USB3_LPM_DISABLED;
4952 
4953 	/* If connected to root port then check port can handle lpm */
4954 	if (udev->parent && !udev->parent->parent) {
4955 		port = xhci->usb3_rhub.ports[udev->portnum - 1];
4956 		if (port->lpm_incapable)
4957 			return USB3_LPM_DISABLED;
4958 	}
4959 
4960 	hub_encoded_timeout = xhci_calculate_lpm_timeout(hcd, udev, state);
4961 	mel = calculate_max_exit_latency(udev, state, hub_encoded_timeout);
4962 	if (mel < 0) {
4963 		/* Max Exit Latency is too big, disable LPM. */
4964 		hub_encoded_timeout = USB3_LPM_DISABLED;
4965 		mel = 0;
4966 	}
4967 
4968 	ret = xhci_change_max_exit_latency(xhci, udev, mel);
4969 	if (ret)
4970 		return ret;
4971 	return hub_encoded_timeout;
4972 }
4973 
4974 static int xhci_disable_usb3_lpm_timeout(struct usb_hcd *hcd,
4975 			struct usb_device *udev, enum usb3_link_state state)
4976 {
4977 	struct xhci_hcd	*xhci;
4978 	u16 mel;
4979 
4980 	xhci = hcd_to_xhci(hcd);
4981 	if (!xhci || !(xhci->quirks & XHCI_LPM_SUPPORT) ||
4982 			!xhci->devs[udev->slot_id])
4983 		return 0;
4984 
4985 	mel = calculate_max_exit_latency(udev, state, USB3_LPM_DISABLED);
4986 	return xhci_change_max_exit_latency(xhci, udev, mel);
4987 }
4988 #else /* CONFIG_PM */
4989 
4990 static int xhci_set_usb2_hardware_lpm(struct usb_hcd *hcd,
4991 				struct usb_device *udev, int enable)
4992 {
4993 	return 0;
4994 }
4995 
4996 static int xhci_update_device(struct usb_hcd *hcd, struct usb_device *udev)
4997 {
4998 	return 0;
4999 }
5000 
5001 static int xhci_enable_usb3_lpm_timeout(struct usb_hcd *hcd,
5002 			struct usb_device *udev, enum usb3_link_state state)
5003 {
5004 	return USB3_LPM_DISABLED;
5005 }
5006 
5007 static int xhci_disable_usb3_lpm_timeout(struct usb_hcd *hcd,
5008 			struct usb_device *udev, enum usb3_link_state state)
5009 {
5010 	return 0;
5011 }
5012 #endif	/* CONFIG_PM */
5013 
5014 /*-------------------------------------------------------------------------*/
5015 
5016 /* Once a hub descriptor is fetched for a device, we need to update the xHC's
5017  * internal data structures for the device.
5018  */
5019 int xhci_update_hub_device(struct usb_hcd *hcd, struct usb_device *hdev,
5020 			struct usb_tt *tt, gfp_t mem_flags)
5021 {
5022 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
5023 	struct xhci_virt_device *vdev;
5024 	struct xhci_command *config_cmd;
5025 	struct xhci_input_control_ctx *ctrl_ctx;
5026 	struct xhci_slot_ctx *slot_ctx;
5027 	unsigned long flags;
5028 	unsigned think_time;
5029 	int ret;
5030 
5031 	/* Ignore root hubs */
5032 	if (!hdev->parent)
5033 		return 0;
5034 
5035 	vdev = xhci->devs[hdev->slot_id];
5036 	if (!vdev) {
5037 		xhci_warn(xhci, "Cannot update hub desc for unknown device.\n");
5038 		return -EINVAL;
5039 	}
5040 
5041 	config_cmd = xhci_alloc_command_with_ctx(xhci, true, mem_flags);
5042 	if (!config_cmd)
5043 		return -ENOMEM;
5044 
5045 	ctrl_ctx = xhci_get_input_control_ctx(config_cmd->in_ctx);
5046 	if (!ctrl_ctx) {
5047 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
5048 				__func__);
5049 		xhci_free_command(xhci, config_cmd);
5050 		return -ENOMEM;
5051 	}
5052 
5053 	spin_lock_irqsave(&xhci->lock, flags);
5054 	if (hdev->speed == USB_SPEED_HIGH &&
5055 			xhci_alloc_tt_info(xhci, vdev, hdev, tt, GFP_ATOMIC)) {
5056 		xhci_dbg(xhci, "Could not allocate xHCI TT structure.\n");
5057 		xhci_free_command(xhci, config_cmd);
5058 		spin_unlock_irqrestore(&xhci->lock, flags);
5059 		return -ENOMEM;
5060 	}
5061 
5062 	xhci_slot_copy(xhci, config_cmd->in_ctx, vdev->out_ctx);
5063 	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
5064 	slot_ctx = xhci_get_slot_ctx(xhci, config_cmd->in_ctx);
5065 	slot_ctx->dev_info |= cpu_to_le32(DEV_HUB);
5066 	/*
5067 	 * refer to section 6.2.2: MTT should be 0 for full speed hub,
5068 	 * but it may be already set to 1 when setup an xHCI virtual
5069 	 * device, so clear it anyway.
5070 	 */
5071 	if (tt->multi)
5072 		slot_ctx->dev_info |= cpu_to_le32(DEV_MTT);
5073 	else if (hdev->speed == USB_SPEED_FULL)
5074 		slot_ctx->dev_info &= cpu_to_le32(~DEV_MTT);
5075 
5076 	if (xhci->hci_version > 0x95) {
5077 		xhci_dbg(xhci, "xHCI version %x needs hub "
5078 				"TT think time and number of ports\n",
5079 				(unsigned int) xhci->hci_version);
5080 		slot_ctx->dev_info2 |= cpu_to_le32(XHCI_MAX_PORTS(hdev->maxchild));
5081 		/* Set TT think time - convert from ns to FS bit times.
5082 		 * 0 = 8 FS bit times, 1 = 16 FS bit times,
5083 		 * 2 = 24 FS bit times, 3 = 32 FS bit times.
5084 		 *
5085 		 * xHCI 1.0: this field shall be 0 if the device is not a
5086 		 * High-spped hub.
5087 		 */
5088 		think_time = tt->think_time;
5089 		if (think_time != 0)
5090 			think_time = (think_time / 666) - 1;
5091 		if (xhci->hci_version < 0x100 || hdev->speed == USB_SPEED_HIGH)
5092 			slot_ctx->tt_info |=
5093 				cpu_to_le32(TT_THINK_TIME(think_time));
5094 	} else {
5095 		xhci_dbg(xhci, "xHCI version %x doesn't need hub "
5096 				"TT think time or number of ports\n",
5097 				(unsigned int) xhci->hci_version);
5098 	}
5099 	slot_ctx->dev_state = 0;
5100 	spin_unlock_irqrestore(&xhci->lock, flags);
5101 
5102 	xhci_dbg(xhci, "Set up %s for hub device.\n",
5103 			(xhci->hci_version > 0x95) ?
5104 			"configure endpoint" : "evaluate context");
5105 
5106 	/* Issue and wait for the configure endpoint or
5107 	 * evaluate context command.
5108 	 */
5109 	if (xhci->hci_version > 0x95)
5110 		ret = xhci_configure_endpoint(xhci, hdev, config_cmd,
5111 				false, false);
5112 	else
5113 		ret = xhci_configure_endpoint(xhci, hdev, config_cmd,
5114 				true, false);
5115 
5116 	xhci_free_command(xhci, config_cmd);
5117 	return ret;
5118 }
5119 EXPORT_SYMBOL_GPL(xhci_update_hub_device);
5120 
5121 static int xhci_get_frame(struct usb_hcd *hcd)
5122 {
5123 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
5124 	/* EHCI mods by the periodic size.  Why? */
5125 	return readl(&xhci->run_regs->microframe_index) >> 3;
5126 }
5127 
5128 static void xhci_hcd_init_usb2_data(struct xhci_hcd *xhci, struct usb_hcd *hcd)
5129 {
5130 	xhci->usb2_rhub.hcd = hcd;
5131 	hcd->speed = HCD_USB2;
5132 	hcd->self.root_hub->speed = USB_SPEED_HIGH;
5133 	/*
5134 	 * USB 2.0 roothub under xHCI has an integrated TT,
5135 	 * (rate matching hub) as opposed to having an OHCI/UHCI
5136 	 * companion controller.
5137 	 */
5138 	hcd->has_tt = 1;
5139 }
5140 
5141 static void xhci_hcd_init_usb3_data(struct xhci_hcd *xhci, struct usb_hcd *hcd)
5142 {
5143 	unsigned int minor_rev;
5144 
5145 	/*
5146 	 * Early xHCI 1.1 spec did not mention USB 3.1 capable hosts
5147 	 * should return 0x31 for sbrn, or that the minor revision
5148 	 * is a two digit BCD containig minor and sub-minor numbers.
5149 	 * This was later clarified in xHCI 1.2.
5150 	 *
5151 	 * Some USB 3.1 capable hosts therefore have sbrn 0x30, and
5152 	 * minor revision set to 0x1 instead of 0x10.
5153 	 */
5154 	if (xhci->usb3_rhub.min_rev == 0x1)
5155 		minor_rev = 1;
5156 	else
5157 		minor_rev = xhci->usb3_rhub.min_rev / 0x10;
5158 
5159 	switch (minor_rev) {
5160 	case 2:
5161 		hcd->speed = HCD_USB32;
5162 		hcd->self.root_hub->speed = USB_SPEED_SUPER_PLUS;
5163 		hcd->self.root_hub->rx_lanes = 2;
5164 		hcd->self.root_hub->tx_lanes = 2;
5165 		hcd->self.root_hub->ssp_rate = USB_SSP_GEN_2x2;
5166 		break;
5167 	case 1:
5168 		hcd->speed = HCD_USB31;
5169 		hcd->self.root_hub->speed = USB_SPEED_SUPER_PLUS;
5170 		hcd->self.root_hub->ssp_rate = USB_SSP_GEN_2x1;
5171 		break;
5172 	}
5173 	xhci_info(xhci, "Host supports USB 3.%x %sSuperSpeed\n",
5174 		  minor_rev, minor_rev ? "Enhanced " : "");
5175 
5176 	xhci->usb3_rhub.hcd = hcd;
5177 }
5178 
5179 int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks)
5180 {
5181 	struct xhci_hcd		*xhci;
5182 	/*
5183 	 * TODO: Check with DWC3 clients for sysdev according to
5184 	 * quirks
5185 	 */
5186 	struct device		*dev = hcd->self.sysdev;
5187 	int			retval;
5188 
5189 	/* Accept arbitrarily long scatter-gather lists */
5190 	hcd->self.sg_tablesize = ~0;
5191 
5192 	/* support to build packet from discontinuous buffers */
5193 	hcd->self.no_sg_constraint = 1;
5194 
5195 	/* XHCI controllers don't stop the ep queue on short packets :| */
5196 	hcd->self.no_stop_on_short = 1;
5197 
5198 	xhci = hcd_to_xhci(hcd);
5199 
5200 	if (!usb_hcd_is_primary_hcd(hcd)) {
5201 		xhci_hcd_init_usb3_data(xhci, hcd);
5202 		return 0;
5203 	}
5204 
5205 	mutex_init(&xhci->mutex);
5206 	xhci->main_hcd = hcd;
5207 	xhci->cap_regs = hcd->regs;
5208 	xhci->op_regs = hcd->regs +
5209 		HC_LENGTH(readl(&xhci->cap_regs->hc_capbase));
5210 	xhci->run_regs = hcd->regs +
5211 		(readl(&xhci->cap_regs->run_regs_off) & RTSOFF_MASK);
5212 	/* Cache read-only capability registers */
5213 	xhci->hcs_params1 = readl(&xhci->cap_regs->hcs_params1);
5214 	xhci->hcs_params2 = readl(&xhci->cap_regs->hcs_params2);
5215 	xhci->hcs_params3 = readl(&xhci->cap_regs->hcs_params3);
5216 	xhci->hci_version = HC_VERSION(readl(&xhci->cap_regs->hc_capbase));
5217 	xhci->hcc_params = readl(&xhci->cap_regs->hcc_params);
5218 	if (xhci->hci_version > 0x100)
5219 		xhci->hcc_params2 = readl(&xhci->cap_regs->hcc_params2);
5220 
5221 	/* xhci-plat or xhci-pci might have set max_interrupters already */
5222 	if ((!xhci->max_interrupters) ||
5223 	    xhci->max_interrupters > HCS_MAX_INTRS(xhci->hcs_params1))
5224 		xhci->max_interrupters = HCS_MAX_INTRS(xhci->hcs_params1);
5225 
5226 	xhci->quirks |= quirks;
5227 
5228 	if (get_quirks)
5229 		get_quirks(dev, xhci);
5230 
5231 	/* In xhci controllers which follow xhci 1.0 spec gives a spurious
5232 	 * success event after a short transfer. This quirk will ignore such
5233 	 * spurious event.
5234 	 */
5235 	if (xhci->hci_version > 0x96)
5236 		xhci->quirks |= XHCI_SPURIOUS_SUCCESS;
5237 
5238 	/* Make sure the HC is halted. */
5239 	retval = xhci_halt(xhci);
5240 	if (retval)
5241 		return retval;
5242 
5243 	xhci_zero_64b_regs(xhci);
5244 
5245 	xhci_dbg(xhci, "Resetting HCD\n");
5246 	/* Reset the internal HC memory state and registers. */
5247 	retval = xhci_reset(xhci, XHCI_RESET_LONG_USEC);
5248 	if (retval)
5249 		return retval;
5250 	xhci_dbg(xhci, "Reset complete\n");
5251 
5252 	/*
5253 	 * On some xHCI controllers (e.g. R-Car SoCs), the AC64 bit (bit 0)
5254 	 * of HCCPARAMS1 is set to 1. However, the xHCs don't support 64-bit
5255 	 * address memory pointers actually. So, this driver clears the AC64
5256 	 * bit of xhci->hcc_params to call dma_set_coherent_mask(dev,
5257 	 * DMA_BIT_MASK(32)) in this xhci_gen_setup().
5258 	 */
5259 	if (xhci->quirks & XHCI_NO_64BIT_SUPPORT)
5260 		xhci->hcc_params &= ~BIT(0);
5261 
5262 	/* Set dma_mask and coherent_dma_mask to 64-bits,
5263 	 * if xHC supports 64-bit addressing */
5264 	if (HCC_64BIT_ADDR(xhci->hcc_params) &&
5265 			!dma_set_mask(dev, DMA_BIT_MASK(64))) {
5266 		xhci_dbg(xhci, "Enabling 64-bit DMA addresses.\n");
5267 		dma_set_coherent_mask(dev, DMA_BIT_MASK(64));
5268 	} else {
5269 		/*
5270 		 * This is to avoid error in cases where a 32-bit USB
5271 		 * controller is used on a 64-bit capable system.
5272 		 */
5273 		retval = dma_set_mask(dev, DMA_BIT_MASK(32));
5274 		if (retval)
5275 			return retval;
5276 		xhci_dbg(xhci, "Enabling 32-bit DMA addresses.\n");
5277 		dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
5278 	}
5279 
5280 	xhci_dbg(xhci, "Calling HCD init\n");
5281 	/* Initialize HCD and host controller data structures. */
5282 	retval = xhci_init(hcd);
5283 	if (retval)
5284 		return retval;
5285 	xhci_dbg(xhci, "Called HCD init\n");
5286 
5287 	if (xhci_hcd_is_usb3(hcd))
5288 		xhci_hcd_init_usb3_data(xhci, hcd);
5289 	else
5290 		xhci_hcd_init_usb2_data(xhci, hcd);
5291 
5292 	xhci_info(xhci, "hcc params 0x%08x hci version 0x%x quirks 0x%016llx\n",
5293 		  xhci->hcc_params, xhci->hci_version, xhci->quirks);
5294 
5295 	return 0;
5296 }
5297 EXPORT_SYMBOL_GPL(xhci_gen_setup);
5298 
5299 static void xhci_clear_tt_buffer_complete(struct usb_hcd *hcd,
5300 		struct usb_host_endpoint *ep)
5301 {
5302 	struct xhci_hcd *xhci;
5303 	struct usb_device *udev;
5304 	unsigned int slot_id;
5305 	unsigned int ep_index;
5306 	unsigned long flags;
5307 
5308 	xhci = hcd_to_xhci(hcd);
5309 
5310 	spin_lock_irqsave(&xhci->lock, flags);
5311 	udev = (struct usb_device *)ep->hcpriv;
5312 	slot_id = udev->slot_id;
5313 	ep_index = xhci_get_endpoint_index(&ep->desc);
5314 
5315 	xhci->devs[slot_id]->eps[ep_index].ep_state &= ~EP_CLEARING_TT;
5316 	xhci_ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
5317 	spin_unlock_irqrestore(&xhci->lock, flags);
5318 }
5319 
5320 static const struct hc_driver xhci_hc_driver = {
5321 	.description =		"xhci-hcd",
5322 	.product_desc =		"xHCI Host Controller",
5323 	.hcd_priv_size =	sizeof(struct xhci_hcd),
5324 
5325 	/*
5326 	 * generic hardware linkage
5327 	 */
5328 	.irq =			xhci_irq,
5329 	.flags =		HCD_MEMORY | HCD_DMA | HCD_USB3 | HCD_SHARED |
5330 				HCD_BH,
5331 
5332 	/*
5333 	 * basic lifecycle operations
5334 	 */
5335 	.reset =		NULL, /* set in xhci_init_driver() */
5336 	.start =		xhci_run,
5337 	.stop =			xhci_stop,
5338 	.shutdown =		xhci_shutdown,
5339 
5340 	/*
5341 	 * managing i/o requests and associated device resources
5342 	 */
5343 	.map_urb_for_dma =      xhci_map_urb_for_dma,
5344 	.unmap_urb_for_dma =    xhci_unmap_urb_for_dma,
5345 	.urb_enqueue =		xhci_urb_enqueue,
5346 	.urb_dequeue =		xhci_urb_dequeue,
5347 	.alloc_dev =		xhci_alloc_dev,
5348 	.free_dev =		xhci_free_dev,
5349 	.alloc_streams =	xhci_alloc_streams,
5350 	.free_streams =		xhci_free_streams,
5351 	.add_endpoint =		xhci_add_endpoint,
5352 	.drop_endpoint =	xhci_drop_endpoint,
5353 	.endpoint_disable =	xhci_endpoint_disable,
5354 	.endpoint_reset =	xhci_endpoint_reset,
5355 	.check_bandwidth =	xhci_check_bandwidth,
5356 	.reset_bandwidth =	xhci_reset_bandwidth,
5357 	.address_device =	xhci_address_device,
5358 	.enable_device =	xhci_enable_device,
5359 	.update_hub_device =	xhci_update_hub_device,
5360 	.reset_device =		xhci_discover_or_reset_device,
5361 
5362 	/*
5363 	 * scheduling support
5364 	 */
5365 	.get_frame_number =	xhci_get_frame,
5366 
5367 	/*
5368 	 * root hub support
5369 	 */
5370 	.hub_control =		xhci_hub_control,
5371 	.hub_status_data =	xhci_hub_status_data,
5372 	.bus_suspend =		xhci_bus_suspend,
5373 	.bus_resume =		xhci_bus_resume,
5374 	.get_resuming_ports =	xhci_get_resuming_ports,
5375 
5376 	/*
5377 	 * call back when device connected and addressed
5378 	 */
5379 	.update_device =        xhci_update_device,
5380 	.set_usb2_hw_lpm =	xhci_set_usb2_hardware_lpm,
5381 	.enable_usb3_lpm_timeout =	xhci_enable_usb3_lpm_timeout,
5382 	.disable_usb3_lpm_timeout =	xhci_disable_usb3_lpm_timeout,
5383 	.find_raw_port_number =	xhci_find_raw_port_number,
5384 	.clear_tt_buffer_complete = xhci_clear_tt_buffer_complete,
5385 };
5386 
5387 void xhci_init_driver(struct hc_driver *drv,
5388 		      const struct xhci_driver_overrides *over)
5389 {
5390 	BUG_ON(!over);
5391 
5392 	/* Copy the generic table to drv then apply the overrides */
5393 	*drv = xhci_hc_driver;
5394 
5395 	if (over) {
5396 		drv->hcd_priv_size += over->extra_priv_size;
5397 		if (over->reset)
5398 			drv->reset = over->reset;
5399 		if (over->start)
5400 			drv->start = over->start;
5401 		if (over->add_endpoint)
5402 			drv->add_endpoint = over->add_endpoint;
5403 		if (over->drop_endpoint)
5404 			drv->drop_endpoint = over->drop_endpoint;
5405 		if (over->check_bandwidth)
5406 			drv->check_bandwidth = over->check_bandwidth;
5407 		if (over->reset_bandwidth)
5408 			drv->reset_bandwidth = over->reset_bandwidth;
5409 		if (over->update_hub_device)
5410 			drv->update_hub_device = over->update_hub_device;
5411 		if (over->hub_control)
5412 			drv->hub_control = over->hub_control;
5413 	}
5414 }
5415 EXPORT_SYMBOL_GPL(xhci_init_driver);
5416 
5417 MODULE_DESCRIPTION(DRIVER_DESC);
5418 MODULE_AUTHOR(DRIVER_AUTHOR);
5419 MODULE_LICENSE("GPL");
5420 
5421 static int __init xhci_hcd_init(void)
5422 {
5423 	/*
5424 	 * Check the compiler generated sizes of structures that must be laid
5425 	 * out in specific ways for hardware access.
5426 	 */
5427 	BUILD_BUG_ON(sizeof(struct xhci_doorbell_array) != 256*32/8);
5428 	BUILD_BUG_ON(sizeof(struct xhci_slot_ctx) != 8*32/8);
5429 	BUILD_BUG_ON(sizeof(struct xhci_ep_ctx) != 8*32/8);
5430 	/* xhci_device_control has eight fields, and also
5431 	 * embeds one xhci_slot_ctx and 31 xhci_ep_ctx
5432 	 */
5433 	BUILD_BUG_ON(sizeof(struct xhci_stream_ctx) != 4*32/8);
5434 	BUILD_BUG_ON(sizeof(union xhci_trb) != 4*32/8);
5435 	BUILD_BUG_ON(sizeof(struct xhci_erst_entry) != 4*32/8);
5436 	BUILD_BUG_ON(sizeof(struct xhci_cap_regs) != 8*32/8);
5437 	BUILD_BUG_ON(sizeof(struct xhci_intr_reg) != 8*32/8);
5438 	/* xhci_run_regs has eight fields and embeds 128 xhci_intr_regs */
5439 	BUILD_BUG_ON(sizeof(struct xhci_run_regs) != (8+8*128)*32/8);
5440 
5441 	if (usb_disabled())
5442 		return -ENODEV;
5443 
5444 	xhci_debugfs_create_root();
5445 	xhci_dbc_init();
5446 
5447 	return 0;
5448 }
5449 
5450 /*
5451  * If an init function is provided, an exit function must also be provided
5452  * to allow module unload.
5453  */
5454 static void __exit xhci_hcd_fini(void)
5455 {
5456 	xhci_debugfs_remove_root();
5457 	xhci_dbc_exit();
5458 }
5459 
5460 module_init(xhci_hcd_init);
5461 module_exit(xhci_hcd_fini);
5462