xref: /freebsd/sys/dev/usb/controller/xhci.c (revision a521f2116473fbd8c09db395518f060a27d02334)
1 /* $FreeBSD$ */
2 /*-
3  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4  *
5  * Copyright (c) 2010 Hans Petter Selasky. All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 
29 /*
30  * USB eXtensible Host Controller Interface, a.k.a. USB 3.0 controller.
31  *
32  * The XHCI 1.0 spec can be found at
33  * http://www.intel.com/technology/usb/download/xHCI_Specification_for_USB.pdf
34  * and the USB 3.0 spec at
35  * http://www.usb.org/developers/docs/usb_30_spec_060910.zip
36  */
37 
38 /*
39  * A few words about the design implementation: This driver emulates
40  * the concept about TDs which is found in EHCI specification. This
41  * way we achieve that the USB controller drivers look similar to
42  * eachother which makes it easier to understand the code.
43  */
44 
45 #ifdef USB_GLOBAL_INCLUDE_FILE
46 #include USB_GLOBAL_INCLUDE_FILE
47 #else
48 #include <sys/stdint.h>
49 #include <sys/stddef.h>
50 #include <sys/param.h>
51 #include <sys/queue.h>
52 #include <sys/types.h>
53 #include <sys/systm.h>
54 #include <sys/kernel.h>
55 #include <sys/bus.h>
56 #include <sys/module.h>
57 #include <sys/lock.h>
58 #include <sys/mutex.h>
59 #include <sys/condvar.h>
60 #include <sys/sysctl.h>
61 #include <sys/sx.h>
62 #include <sys/unistd.h>
63 #include <sys/callout.h>
64 #include <sys/malloc.h>
65 #include <sys/priv.h>
66 
67 #include <dev/usb/usb.h>
68 #include <dev/usb/usbdi.h>
69 
70 #define	USB_DEBUG_VAR xhcidebug
71 
72 #include <dev/usb/usb_core.h>
73 #include <dev/usb/usb_debug.h>
74 #include <dev/usb/usb_busdma.h>
75 #include <dev/usb/usb_process.h>
76 #include <dev/usb/usb_transfer.h>
77 #include <dev/usb/usb_device.h>
78 #include <dev/usb/usb_hub.h>
79 #include <dev/usb/usb_util.h>
80 
81 #include <dev/usb/usb_controller.h>
82 #include <dev/usb/usb_bus.h>
83 #endif			/* USB_GLOBAL_INCLUDE_FILE */
84 
85 #include <dev/usb/controller/xhci.h>
86 #include <dev/usb/controller/xhcireg.h>
87 
88 #define	XHCI_BUS2SC(bus) \
89    ((struct xhci_softc *)(((uint8_t *)(bus)) - \
90     ((uint8_t *)&(((struct xhci_softc *)0)->sc_bus))))
91 
92 static SYSCTL_NODE(_hw_usb, OID_AUTO, xhci, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
93     "USB XHCI");
94 
95 static int xhcistreams;
96 SYSCTL_INT(_hw_usb_xhci, OID_AUTO, streams, CTLFLAG_RWTUN,
97     &xhcistreams, 0, "Set to enable streams mode support");
98 
99 static int xhcictlquirk = 1;
100 SYSCTL_INT(_hw_usb_xhci, OID_AUTO, ctlquirk, CTLFLAG_RWTUN,
101     &xhcictlquirk, 0, "Set to enable control endpoint quirk");
102 
103 #ifdef USB_DEBUG
104 static int xhcidebug;
105 static int xhciroute;
106 static int xhcipolling;
107 static int xhcidma32;
108 static int xhcictlstep;
109 
110 SYSCTL_INT(_hw_usb_xhci, OID_AUTO, debug, CTLFLAG_RWTUN,
111     &xhcidebug, 0, "Debug level");
112 SYSCTL_INT(_hw_usb_xhci, OID_AUTO, xhci_port_route, CTLFLAG_RWTUN,
113     &xhciroute, 0, "Routing bitmap for switching EHCI ports to the XHCI controller");
114 SYSCTL_INT(_hw_usb_xhci, OID_AUTO, use_polling, CTLFLAG_RWTUN,
115     &xhcipolling, 0, "Set to enable software interrupt polling for the XHCI controller");
116 SYSCTL_INT(_hw_usb_xhci, OID_AUTO, dma32, CTLFLAG_RWTUN,
117     &xhcidma32, 0, "Set to only use 32-bit DMA for the XHCI controller");
118 SYSCTL_INT(_hw_usb_xhci, OID_AUTO, ctlstep, CTLFLAG_RWTUN,
119     &xhcictlstep, 0, "Set to enable control endpoint status stage stepping");
120 #else
121 #define	xhciroute 0
122 #define	xhcidma32 0
123 #define	xhcictlstep 0
124 #endif
125 
126 #define	XHCI_INTR_ENDPT 1
127 
128 struct xhci_std_temp {
129 	struct xhci_softc	*sc;
130 	struct usb_page_cache	*pc;
131 	struct xhci_td		*td;
132 	struct xhci_td		*td_next;
133 	uint32_t		len;
134 	uint32_t		offset;
135 	uint32_t		max_packet_size;
136 	uint32_t		average;
137 	uint16_t		isoc_delta;
138 	uint16_t		isoc_frame;
139 	uint8_t			shortpkt;
140 	uint8_t			multishort;
141 	uint8_t			last_frame;
142 	uint8_t			trb_type;
143 	uint8_t			direction;
144 	uint8_t			tbc;
145 	uint8_t			tlbpc;
146 	uint8_t			step_td;
147 	uint8_t			do_isoc_sync;
148 };
149 
150 static void	xhci_do_poll(struct usb_bus *);
151 static void	xhci_device_done(struct usb_xfer *, usb_error_t);
152 static void	xhci_root_intr(struct xhci_softc *);
153 static void	xhci_free_device_ext(struct usb_device *);
154 static struct xhci_endpoint_ext *xhci_get_endpoint_ext(struct usb_device *,
155 		    struct usb_endpoint_descriptor *);
156 static usb_proc_callback_t xhci_configure_msg;
157 static usb_error_t xhci_configure_device(struct usb_device *);
158 static usb_error_t xhci_configure_endpoint(struct usb_device *,
159 		   struct usb_endpoint_descriptor *, struct xhci_endpoint_ext *,
160 		   uint16_t, uint8_t, uint8_t, uint8_t, uint16_t, uint16_t,
161 		   uint8_t);
162 static usb_error_t xhci_configure_mask(struct usb_device *,
163 		    uint32_t, uint8_t);
164 static usb_error_t xhci_cmd_evaluate_ctx(struct xhci_softc *,
165 		    uint64_t, uint8_t);
166 static void xhci_endpoint_doorbell(struct usb_xfer *);
167 static void xhci_ctx_set_le32(struct xhci_softc *sc, volatile uint32_t *ptr, uint32_t val);
168 static uint32_t xhci_ctx_get_le32(struct xhci_softc *sc, volatile uint32_t *ptr);
169 static void xhci_ctx_set_le64(struct xhci_softc *sc, volatile uint64_t *ptr, uint64_t val);
170 #ifdef USB_DEBUG
171 static uint64_t xhci_ctx_get_le64(struct xhci_softc *sc, volatile uint64_t *ptr);
172 #endif
173 
174 static const struct usb_bus_methods xhci_bus_methods;
175 
176 #ifdef USB_DEBUG
177 static void
178 xhci_dump_trb(struct xhci_trb *trb)
179 {
180 	DPRINTFN(5, "trb = %p\n", trb);
181 	DPRINTFN(5, "qwTrb0 = 0x%016llx\n", (long long)le64toh(trb->qwTrb0));
182 	DPRINTFN(5, "dwTrb2 = 0x%08x\n", le32toh(trb->dwTrb2));
183 	DPRINTFN(5, "dwTrb3 = 0x%08x\n", le32toh(trb->dwTrb3));
184 }
185 
186 static void
187 xhci_dump_endpoint(struct xhci_softc *sc, struct xhci_endp_ctx *pep)
188 {
189 	DPRINTFN(5, "pep = %p\n", pep);
190 	DPRINTFN(5, "dwEpCtx0=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx0));
191 	DPRINTFN(5, "dwEpCtx1=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx1));
192 	DPRINTFN(5, "qwEpCtx2=0x%016llx\n", (long long)xhci_ctx_get_le64(sc, &pep->qwEpCtx2));
193 	DPRINTFN(5, "dwEpCtx4=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx4));
194 	DPRINTFN(5, "dwEpCtx5=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx5));
195 	DPRINTFN(5, "dwEpCtx6=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx6));
196 	DPRINTFN(5, "dwEpCtx7=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx7));
197 }
198 
199 static void
200 xhci_dump_device(struct xhci_softc *sc, struct xhci_slot_ctx *psl)
201 {
202 	DPRINTFN(5, "psl = %p\n", psl);
203 	DPRINTFN(5, "dwSctx0=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx0));
204 	DPRINTFN(5, "dwSctx1=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx1));
205 	DPRINTFN(5, "dwSctx2=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx2));
206 	DPRINTFN(5, "dwSctx3=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx3));
207 }
208 #endif
209 
210 uint8_t
211 xhci_use_polling(void)
212 {
213 #ifdef USB_DEBUG
214 	return (xhcipolling != 0);
215 #else
216 	return (0);
217 #endif
218 }
219 
220 static void
221 xhci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb)
222 {
223 	struct xhci_softc *sc = XHCI_BUS2SC(bus);
224 	uint16_t i;
225 
226 	cb(bus, &sc->sc_hw.root_pc, &sc->sc_hw.root_pg,
227 	   sizeof(struct xhci_hw_root), XHCI_PAGE_SIZE);
228 
229 	cb(bus, &sc->sc_hw.ctx_pc, &sc->sc_hw.ctx_pg,
230 	   sizeof(struct xhci_dev_ctx_addr), XHCI_PAGE_SIZE);
231 
232 	for (i = 0; i != sc->sc_noscratch; i++) {
233 		cb(bus, &sc->sc_hw.scratch_pc[i], &sc->sc_hw.scratch_pg[i],
234 		    XHCI_PAGE_SIZE, XHCI_PAGE_SIZE);
235 	}
236 }
237 
238 static void
239 xhci_ctx_set_le32(struct xhci_softc *sc, volatile uint32_t *ptr, uint32_t val)
240 {
241 	if (sc->sc_ctx_is_64_byte) {
242 		uint32_t offset;
243 		/* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */
244 		/* all contexts are initially 32-bytes */
245 		offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U));
246 		ptr = (volatile uint32_t *)(((volatile uint8_t *)ptr) + offset);
247 	}
248 	*ptr = htole32(val);
249 }
250 
251 static uint32_t
252 xhci_ctx_get_le32(struct xhci_softc *sc, volatile uint32_t *ptr)
253 {
254 	if (sc->sc_ctx_is_64_byte) {
255 		uint32_t offset;
256 		/* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */
257 		/* all contexts are initially 32-bytes */
258 		offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U));
259 		ptr = (volatile uint32_t *)(((volatile uint8_t *)ptr) + offset);
260 	}
261 	return (le32toh(*ptr));
262 }
263 
264 static void
265 xhci_ctx_set_le64(struct xhci_softc *sc, volatile uint64_t *ptr, uint64_t val)
266 {
267 	if (sc->sc_ctx_is_64_byte) {
268 		uint32_t offset;
269 		/* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */
270 		/* all contexts are initially 32-bytes */
271 		offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U));
272 		ptr = (volatile uint64_t *)(((volatile uint8_t *)ptr) + offset);
273 	}
274 	*ptr = htole64(val);
275 }
276 
277 #ifdef USB_DEBUG
278 static uint64_t
279 xhci_ctx_get_le64(struct xhci_softc *sc, volatile uint64_t *ptr)
280 {
281 	if (sc->sc_ctx_is_64_byte) {
282 		uint32_t offset;
283 		/* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */
284 		/* all contexts are initially 32-bytes */
285 		offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U));
286 		ptr = (volatile uint64_t *)(((volatile uint8_t *)ptr) + offset);
287 	}
288 	return (le64toh(*ptr));
289 }
290 #endif
291 
292 static int
293 xhci_reset_command_queue_locked(struct xhci_softc *sc)
294 {
295 	struct usb_page_search buf_res;
296 	struct xhci_hw_root *phwr;
297 	uint64_t addr;
298 	uint32_t temp;
299 
300 	DPRINTF("\n");
301 
302 	temp = XREAD4(sc, oper, XHCI_CRCR_LO);
303 	if (temp & XHCI_CRCR_LO_CRR) {
304 		DPRINTF("Command ring running\n");
305 		temp &= ~(XHCI_CRCR_LO_CS | XHCI_CRCR_LO_CA);
306 
307 		/*
308 		 * Try to abort the last command as per section
309 		 * 4.6.1.2 "Aborting a Command" of the XHCI
310 		 * specification:
311 		 */
312 
313 		/* stop and cancel */
314 		XWRITE4(sc, oper, XHCI_CRCR_LO, temp | XHCI_CRCR_LO_CS);
315 		XWRITE4(sc, oper, XHCI_CRCR_HI, 0);
316 
317 		XWRITE4(sc, oper, XHCI_CRCR_LO, temp | XHCI_CRCR_LO_CA);
318 		XWRITE4(sc, oper, XHCI_CRCR_HI, 0);
319 
320  		/* wait 250ms */
321  		usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 4);
322 
323 		/* check if command ring is still running */
324 		temp = XREAD4(sc, oper, XHCI_CRCR_LO);
325 		if (temp & XHCI_CRCR_LO_CRR) {
326 			DPRINTF("Comand ring still running\n");
327 			return (USB_ERR_IOERROR);
328 		}
329 	}
330 
331 	/* reset command ring */
332 	sc->sc_command_ccs = 1;
333 	sc->sc_command_idx = 0;
334 
335 	usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res);
336 
337 	/* set up command ring control base address */
338 	addr = buf_res.physaddr;
339 	phwr = buf_res.buffer;
340 	addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_commands[0];
341 
342 	DPRINTF("CRCR=0x%016llx\n", (unsigned long long)addr);
343 
344 	memset(phwr->hwr_commands, 0, sizeof(phwr->hwr_commands));
345 	phwr->hwr_commands[XHCI_MAX_COMMANDS - 1].qwTrb0 = htole64(addr);
346 
347 	usb_pc_cpu_flush(&sc->sc_hw.root_pc);
348 
349 	XWRITE4(sc, oper, XHCI_CRCR_LO, ((uint32_t)addr) | XHCI_CRCR_LO_RCS);
350 	XWRITE4(sc, oper, XHCI_CRCR_HI, (uint32_t)(addr >> 32));
351 
352 	return (0);
353 }
354 
355 usb_error_t
356 xhci_start_controller(struct xhci_softc *sc)
357 {
358 	struct usb_page_search buf_res;
359 	struct xhci_hw_root *phwr;
360 	struct xhci_dev_ctx_addr *pdctxa;
361 	usb_error_t err;
362 	uint64_t addr;
363 	uint32_t temp;
364 	uint16_t i;
365 
366 	DPRINTF("\n");
367 
368 	sc->sc_event_ccs = 1;
369 	sc->sc_event_idx = 0;
370 	sc->sc_command_ccs = 1;
371 	sc->sc_command_idx = 0;
372 
373 	err = xhci_reset_controller(sc);
374 	if (err)
375 		return (err);
376 
377 	/* set up number of device slots */
378 	DPRINTF("CONFIG=0x%08x -> 0x%08x\n",
379 	    XREAD4(sc, oper, XHCI_CONFIG), sc->sc_noslot);
380 
381 	XWRITE4(sc, oper, XHCI_CONFIG, sc->sc_noslot);
382 
383 	temp = XREAD4(sc, oper, XHCI_USBSTS);
384 
385 	/* clear interrupts */
386 	XWRITE4(sc, oper, XHCI_USBSTS, temp);
387 	/* disable all device notifications */
388 	XWRITE4(sc, oper, XHCI_DNCTRL, 0);
389 
390 	/* set up device context base address */
391 	usbd_get_page(&sc->sc_hw.ctx_pc, 0, &buf_res);
392 	pdctxa = buf_res.buffer;
393 	memset(pdctxa, 0, sizeof(*pdctxa));
394 
395 	addr = buf_res.physaddr;
396 	addr += (uintptr_t)&((struct xhci_dev_ctx_addr *)0)->qwSpBufPtr[0];
397 
398 	/* slot 0 points to the table of scratchpad pointers */
399 	pdctxa->qwBaaDevCtxAddr[0] = htole64(addr);
400 
401 	for (i = 0; i != sc->sc_noscratch; i++) {
402 		struct usb_page_search buf_scp;
403 		usbd_get_page(&sc->sc_hw.scratch_pc[i], 0, &buf_scp);
404 		pdctxa->qwSpBufPtr[i] = htole64((uint64_t)buf_scp.physaddr);
405 	}
406 
407 	addr = buf_res.physaddr;
408 
409 	XWRITE4(sc, oper, XHCI_DCBAAP_LO, (uint32_t)addr);
410 	XWRITE4(sc, oper, XHCI_DCBAAP_HI, (uint32_t)(addr >> 32));
411 	XWRITE4(sc, oper, XHCI_DCBAAP_LO, (uint32_t)addr);
412 	XWRITE4(sc, oper, XHCI_DCBAAP_HI, (uint32_t)(addr >> 32));
413 
414 	/* set up event table size */
415 	DPRINTF("ERSTSZ=0x%08x -> 0x%08x\n",
416 	    XREAD4(sc, runt, XHCI_ERSTSZ(0)), sc->sc_erst_max);
417 
418 	XWRITE4(sc, runt, XHCI_ERSTSZ(0), XHCI_ERSTS_SET(sc->sc_erst_max));
419 
420 	/* set up interrupt rate */
421 	XWRITE4(sc, runt, XHCI_IMOD(0), sc->sc_imod_default);
422 
423 	usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res);
424 
425 	phwr = buf_res.buffer;
426 	addr = buf_res.physaddr;
427 	addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_events[0];
428 
429 	/* reset hardware root structure */
430 	memset(phwr, 0, sizeof(*phwr));
431 
432 	phwr->hwr_ring_seg[0].qwEvrsTablePtr = htole64(addr);
433 	phwr->hwr_ring_seg[0].dwEvrsTableSize = htole32(XHCI_MAX_EVENTS);
434 
435 	/*
436 	 * PR 237666:
437 	 *
438 	 * According to the XHCI specification, the XWRITE4's to
439 	 * XHCI_ERSTBA_LO and _HI lead to the XHCI to copy the
440 	 * qwEvrsTablePtr and dwEvrsTableSize values above at that
441 	 * time, as the XHCI initializes its event ring support. This
442 	 * is before the event ring starts to pay attention to the
443 	 * RUN/STOP bit. Thus, make sure the values are observable to
444 	 * the XHCI before that point.
445 	 */
446 	usb_bus_mem_flush_all(&sc->sc_bus, &xhci_iterate_hw_softc);
447 
448 	DPRINTF("ERDP(0)=0x%016llx\n", (unsigned long long)addr);
449 
450 	XWRITE4(sc, runt, XHCI_ERDP_LO(0), (uint32_t)addr);
451 	XWRITE4(sc, runt, XHCI_ERDP_HI(0), (uint32_t)(addr >> 32));
452 
453 	addr = buf_res.physaddr;
454 
455 	DPRINTF("ERSTBA(0)=0x%016llx\n", (unsigned long long)addr);
456 
457 	XWRITE4(sc, runt, XHCI_ERSTBA_LO(0), (uint32_t)addr);
458 	XWRITE4(sc, runt, XHCI_ERSTBA_HI(0), (uint32_t)(addr >> 32));
459 
460 	/* set up interrupter registers */
461 	temp = XREAD4(sc, runt, XHCI_IMAN(0));
462 	temp |= XHCI_IMAN_INTR_ENA;
463 	XWRITE4(sc, runt, XHCI_IMAN(0), temp);
464 
465 	/* set up command ring control base address */
466 	addr = buf_res.physaddr;
467 	addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_commands[0];
468 
469 	DPRINTF("CRCR=0x%016llx\n", (unsigned long long)addr);
470 
471 	XWRITE4(sc, oper, XHCI_CRCR_LO, ((uint32_t)addr) | XHCI_CRCR_LO_RCS);
472 	XWRITE4(sc, oper, XHCI_CRCR_HI, (uint32_t)(addr >> 32));
473 
474 	phwr->hwr_commands[XHCI_MAX_COMMANDS - 1].qwTrb0 = htole64(addr);
475 
476 	usb_bus_mem_flush_all(&sc->sc_bus, &xhci_iterate_hw_softc);
477 
478 	/* Go! */
479 	XWRITE4(sc, oper, XHCI_USBCMD, XHCI_CMD_RS |
480 	    XHCI_CMD_INTE | XHCI_CMD_HSEE);
481 
482 	for (i = 0; i != 100; i++) {
483 		usb_pause_mtx(NULL, hz / 100);
484 		temp = XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_HCH;
485 		if (!temp)
486 			break;
487 	}
488 	if (temp) {
489 		XWRITE4(sc, oper, XHCI_USBCMD, 0);
490 		device_printf(sc->sc_bus.parent, "Run timeout.\n");
491 		return (USB_ERR_IOERROR);
492 	}
493 
494 	/* catch any lost interrupts */
495 	xhci_do_poll(&sc->sc_bus);
496 
497 	if (sc->sc_port_route != NULL) {
498 		/* Route all ports to the XHCI by default */
499 		sc->sc_port_route(sc->sc_bus.parent,
500 		    ~xhciroute, xhciroute);
501 	}
502 	return (0);
503 }
504 
505 usb_error_t
506 xhci_halt_controller(struct xhci_softc *sc)
507 {
508 	uint32_t temp;
509 	uint16_t i;
510 
511 	DPRINTF("\n");
512 
513 	sc->sc_capa_off = 0;
514 	sc->sc_oper_off = XREAD1(sc, capa, XHCI_CAPLENGTH);
515 	sc->sc_runt_off = XREAD4(sc, capa, XHCI_RTSOFF) & ~0xF;
516 	sc->sc_door_off = XREAD4(sc, capa, XHCI_DBOFF) & ~0x3;
517 
518 	/* Halt controller */
519 	XWRITE4(sc, oper, XHCI_USBCMD, 0);
520 
521 	for (i = 0; i != 100; i++) {
522 		usb_pause_mtx(NULL, hz / 100);
523 		temp = XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_HCH;
524 		if (temp)
525 			break;
526 	}
527 
528 	if (!temp) {
529 		device_printf(sc->sc_bus.parent, "Controller halt timeout.\n");
530 		return (USB_ERR_IOERROR);
531 	}
532 	return (0);
533 }
534 
535 usb_error_t
536 xhci_reset_controller(struct xhci_softc *sc)
537 {
538 	uint32_t temp = 0;
539 	uint16_t i;
540 
541 	DPRINTF("\n");
542 
543 	/* Reset controller */
544 	XWRITE4(sc, oper, XHCI_USBCMD, XHCI_CMD_HCRST);
545 
546 	for (i = 0; i != 100; i++) {
547 		usb_pause_mtx(NULL, hz / 100);
548 		temp = (XREAD4(sc, oper, XHCI_USBCMD) & XHCI_CMD_HCRST) |
549 		    (XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_CNR);
550 		if (!temp)
551 			break;
552 	}
553 
554 	if (temp) {
555 		device_printf(sc->sc_bus.parent, "Controller "
556 		    "reset timeout.\n");
557 		return (USB_ERR_IOERROR);
558 	}
559 	return (0);
560 }
561 
562 usb_error_t
563 xhci_init(struct xhci_softc *sc, device_t self, uint8_t dma32)
564 {
565 	uint32_t temp;
566 
567 	DPRINTF("\n");
568 
569 	/* initialize some bus fields */
570 	sc->sc_bus.parent = self;
571 
572 	/* set the bus revision */
573 	sc->sc_bus.usbrev = USB_REV_3_0;
574 
575 	/* set up the bus struct */
576 	sc->sc_bus.methods = &xhci_bus_methods;
577 
578 	/* set up devices array */
579 	sc->sc_bus.devices = sc->sc_devices;
580 	sc->sc_bus.devices_max = XHCI_MAX_DEVICES;
581 
582 	/* set default cycle state in case of early interrupts */
583 	sc->sc_event_ccs = 1;
584 	sc->sc_command_ccs = 1;
585 
586 	/* set up bus space offsets */
587 	sc->sc_capa_off = 0;
588 	sc->sc_oper_off = XREAD1(sc, capa, XHCI_CAPLENGTH);
589 	sc->sc_runt_off = XREAD4(sc, capa, XHCI_RTSOFF) & ~0x1F;
590 	sc->sc_door_off = XREAD4(sc, capa, XHCI_DBOFF) & ~0x3;
591 
592 	DPRINTF("CAPLENGTH=0x%x\n", sc->sc_oper_off);
593 	DPRINTF("RUNTIMEOFFSET=0x%x\n", sc->sc_runt_off);
594 	DPRINTF("DOOROFFSET=0x%x\n", sc->sc_door_off);
595 
596 	DPRINTF("xHCI version = 0x%04x\n", XREAD2(sc, capa, XHCI_HCIVERSION));
597 
598 	if (!(XREAD4(sc, oper, XHCI_PAGESIZE) & XHCI_PAGESIZE_4K)) {
599 		device_printf(sc->sc_bus.parent, "Controller does "
600 		    "not support 4K page size.\n");
601 		return (ENXIO);
602 	}
603 
604 	temp = XREAD4(sc, capa, XHCI_HCSPARAMS0);
605 
606 	DPRINTF("HCS0 = 0x%08x\n", temp);
607 
608 	/* set up context size */
609 	if (XHCI_HCS0_CSZ(temp)) {
610 		sc->sc_ctx_is_64_byte = 1;
611 	} else {
612 		sc->sc_ctx_is_64_byte = 0;
613 	}
614 
615 	/* get DMA bits */
616 	sc->sc_bus.dma_bits = (XHCI_HCS0_AC64(temp) &&
617 	    xhcidma32 == 0 && dma32 == 0) ? 64 : 32;
618 
619 	device_printf(self, "%d bytes context size, %d-bit DMA\n",
620 	    sc->sc_ctx_is_64_byte ? 64 : 32, (int)sc->sc_bus.dma_bits);
621 
622 	/* enable 64Kbyte control endpoint quirk */
623 	sc->sc_bus.control_ep_quirk = (xhcictlquirk ? 1 : 0);
624 
625 	temp = XREAD4(sc, capa, XHCI_HCSPARAMS1);
626 
627 	/* get number of device slots */
628 	sc->sc_noport = XHCI_HCS1_N_PORTS(temp);
629 
630 	if (sc->sc_noport == 0) {
631 		device_printf(sc->sc_bus.parent, "Invalid number "
632 		    "of ports: %u\n", sc->sc_noport);
633 		return (ENXIO);
634 	}
635 
636 	sc->sc_noport = sc->sc_noport;
637 	sc->sc_noslot = XHCI_HCS1_DEVSLOT_MAX(temp);
638 
639 	DPRINTF("Max slots: %u\n", sc->sc_noslot);
640 
641 	if (sc->sc_noslot > XHCI_MAX_DEVICES)
642 		sc->sc_noslot = XHCI_MAX_DEVICES;
643 
644 	temp = XREAD4(sc, capa, XHCI_HCSPARAMS2);
645 
646 	DPRINTF("HCS2=0x%08x\n", temp);
647 
648 	/* get number of scratchpads */
649 	sc->sc_noscratch = XHCI_HCS2_SPB_MAX(temp);
650 
651 	if (sc->sc_noscratch > XHCI_MAX_SCRATCHPADS) {
652 		device_printf(sc->sc_bus.parent, "XHCI request "
653 		    "too many scratchpads\n");
654 		return (ENOMEM);
655 	}
656 
657 	DPRINTF("Max scratch: %u\n", sc->sc_noscratch);
658 
659 	/* get event table size */
660 	sc->sc_erst_max = 1U << XHCI_HCS2_ERST_MAX(temp);
661 	if (sc->sc_erst_max > XHCI_MAX_RSEG)
662 		sc->sc_erst_max = XHCI_MAX_RSEG;
663 
664 	temp = XREAD4(sc, capa, XHCI_HCSPARAMS3);
665 
666 	/* get maximum exit latency */
667 	sc->sc_exit_lat_max = XHCI_HCS3_U1_DEL(temp) +
668 	    XHCI_HCS3_U2_DEL(temp) + 250 /* us */;
669 
670 	/* Check if we should use the default IMOD value. */
671 	if (sc->sc_imod_default == 0)
672 		sc->sc_imod_default = XHCI_IMOD_DEFAULT;
673 
674 	/* get all DMA memory */
675 	if (usb_bus_mem_alloc_all(&sc->sc_bus,
676 	    USB_GET_DMA_TAG(self), &xhci_iterate_hw_softc)) {
677 		return (ENOMEM);
678 	}
679 
680 	/* set up command queue mutex and condition varible */
681 	cv_init(&sc->sc_cmd_cv, "CMDQ");
682 	sx_init(&sc->sc_cmd_sx, "CMDQ lock");
683 
684 	sc->sc_config_msg[0].hdr.pm_callback = &xhci_configure_msg;
685 	sc->sc_config_msg[0].bus = &sc->sc_bus;
686 	sc->sc_config_msg[1].hdr.pm_callback = &xhci_configure_msg;
687 	sc->sc_config_msg[1].bus = &sc->sc_bus;
688 
689 	return (0);
690 }
691 
692 void
693 xhci_uninit(struct xhci_softc *sc)
694 {
695 	/*
696 	 * NOTE: At this point the control transfer process is gone
697 	 * and "xhci_configure_msg" is no longer called. Consequently
698 	 * waiting for the configuration messages to complete is not
699 	 * needed.
700 	 */
701 	usb_bus_mem_free_all(&sc->sc_bus, &xhci_iterate_hw_softc);
702 
703 	cv_destroy(&sc->sc_cmd_cv);
704 	sx_destroy(&sc->sc_cmd_sx);
705 }
706 
707 static void
708 xhci_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
709 {
710 	struct xhci_softc *sc = XHCI_BUS2SC(bus);
711 
712 	switch (state) {
713 	case USB_HW_POWER_SUSPEND:
714 		DPRINTF("Stopping the XHCI\n");
715 		xhci_halt_controller(sc);
716 		xhci_reset_controller(sc);
717 		break;
718 	case USB_HW_POWER_SHUTDOWN:
719 		DPRINTF("Stopping the XHCI\n");
720 		xhci_halt_controller(sc);
721 		xhci_reset_controller(sc);
722 		break;
723 	case USB_HW_POWER_RESUME:
724 		DPRINTF("Starting the XHCI\n");
725 		xhci_start_controller(sc);
726 		break;
727 	default:
728 		break;
729 	}
730 }
731 
732 static usb_error_t
733 xhci_generic_done_sub(struct usb_xfer *xfer)
734 {
735 	struct xhci_td *td;
736 	struct xhci_td *td_alt_next;
737 	uint32_t len;
738 	uint8_t status;
739 
740 	td = xfer->td_transfer_cache;
741 	td_alt_next = td->alt_next;
742 
743 	if (xfer->aframes != xfer->nframes)
744 		usbd_xfer_set_frame_len(xfer, xfer->aframes, 0);
745 
746 	while (1) {
747 		usb_pc_cpu_invalidate(td->page_cache);
748 
749 		status = td->status;
750 		len = td->remainder;
751 
752 		DPRINTFN(4, "xfer=%p[%u/%u] rem=%u/%u status=%u\n",
753 		    xfer, (unsigned int)xfer->aframes,
754 		    (unsigned int)xfer->nframes,
755 		    (unsigned int)len, (unsigned int)td->len,
756 		    (unsigned int)status);
757 
758 		/*
759 	         * Verify the status length and
760 		 * add the length to "frlengths[]":
761 	         */
762 		if (len > td->len) {
763 			/* should not happen */
764 			DPRINTF("Invalid status length, "
765 			    "0x%04x/0x%04x bytes\n", len, td->len);
766 			status = XHCI_TRB_ERROR_LENGTH;
767 		} else if (xfer->aframes != xfer->nframes) {
768 			xfer->frlengths[xfer->aframes] += td->len - len;
769 		}
770 		/* Check for last transfer */
771 		if (((void *)td) == xfer->td_transfer_last) {
772 			td = NULL;
773 			break;
774 		}
775 		/* Check for transfer error */
776 		if (status != XHCI_TRB_ERROR_SHORT_PKT &&
777 		    status != XHCI_TRB_ERROR_SUCCESS) {
778 			/* the transfer is finished */
779 			td = NULL;
780 			break;
781 		}
782 		/* Check for short transfer */
783 		if (len > 0) {
784 			if (xfer->flags_int.short_frames_ok ||
785 			    xfer->flags_int.isochronous_xfr ||
786 			    xfer->flags_int.control_xfr) {
787 				/* follow alt next */
788 				td = td->alt_next;
789 			} else {
790 				/* the transfer is finished */
791 				td = NULL;
792 			}
793 			break;
794 		}
795 		td = td->obj_next;
796 
797 		if (td->alt_next != td_alt_next) {
798 			/* this USB frame is complete */
799 			break;
800 		}
801 	}
802 
803 	/* update transfer cache */
804 
805 	xfer->td_transfer_cache = td;
806 
807 	return ((status == XHCI_TRB_ERROR_STALL) ? USB_ERR_STALLED :
808 	    (status != XHCI_TRB_ERROR_SHORT_PKT &&
809 	    status != XHCI_TRB_ERROR_SUCCESS) ? USB_ERR_IOERROR :
810 	    USB_ERR_NORMAL_COMPLETION);
811 }
812 
813 static void
814 xhci_generic_done(struct usb_xfer *xfer)
815 {
816 	usb_error_t err = 0;
817 
818 	DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
819 	    xfer, xfer->endpoint);
820 
821 	/* reset scanner */
822 
823 	xfer->td_transfer_cache = xfer->td_transfer_first;
824 
825 	if (xfer->flags_int.control_xfr) {
826 		if (xfer->flags_int.control_hdr)
827 			err = xhci_generic_done_sub(xfer);
828 
829 		xfer->aframes = 1;
830 
831 		if (xfer->td_transfer_cache == NULL)
832 			goto done;
833 	}
834 
835 	while (xfer->aframes != xfer->nframes) {
836 		err = xhci_generic_done_sub(xfer);
837 		xfer->aframes++;
838 
839 		if (xfer->td_transfer_cache == NULL)
840 			goto done;
841 	}
842 
843 	if (xfer->flags_int.control_xfr &&
844 	    !xfer->flags_int.control_act)
845 		err = xhci_generic_done_sub(xfer);
846 done:
847 	/* transfer is complete */
848 	xhci_device_done(xfer, err);
849 }
850 
851 static void
852 xhci_activate_transfer(struct usb_xfer *xfer)
853 {
854 	struct xhci_td *td;
855 
856 	td = xfer->td_transfer_cache;
857 
858 	usb_pc_cpu_invalidate(td->page_cache);
859 
860 	if (!(td->td_trb[0].dwTrb3 & htole32(XHCI_TRB_3_CYCLE_BIT))) {
861 		/* activate the transfer */
862 
863 		td->td_trb[0].dwTrb3 |= htole32(XHCI_TRB_3_CYCLE_BIT);
864 		usb_pc_cpu_flush(td->page_cache);
865 
866 		xhci_endpoint_doorbell(xfer);
867 	}
868 }
869 
870 static void
871 xhci_skip_transfer(struct usb_xfer *xfer)
872 {
873 	struct xhci_td *td;
874 	struct xhci_td *td_last;
875 
876 	td = xfer->td_transfer_cache;
877 	td_last = xfer->td_transfer_last;
878 
879 	td = td->alt_next;
880 
881 	usb_pc_cpu_invalidate(td->page_cache);
882 
883 	if (!(td->td_trb[0].dwTrb3 & htole32(XHCI_TRB_3_CYCLE_BIT))) {
884 		usb_pc_cpu_invalidate(td_last->page_cache);
885 
886 		/* copy LINK TRB to current waiting location */
887 
888 		td->td_trb[0].qwTrb0 = td_last->td_trb[td_last->ntrb].qwTrb0;
889 		td->td_trb[0].dwTrb2 = td_last->td_trb[td_last->ntrb].dwTrb2;
890 		usb_pc_cpu_flush(td->page_cache);
891 
892 		td->td_trb[0].dwTrb3 = td_last->td_trb[td_last->ntrb].dwTrb3;
893 		usb_pc_cpu_flush(td->page_cache);
894 
895 		xhci_endpoint_doorbell(xfer);
896 	}
897 }
898 
899 /*------------------------------------------------------------------------*
900  *	xhci_check_transfer
901  *------------------------------------------------------------------------*/
902 static void
903 xhci_check_transfer(struct xhci_softc *sc, struct xhci_trb *trb)
904 {
905 	struct xhci_endpoint_ext *pepext;
906 	int64_t offset;
907 	uint64_t td_event;
908 	uint32_t temp;
909 	uint32_t remainder;
910 	uint16_t stream_id = 0;
911 	uint16_t i;
912 	uint8_t status;
913 	uint8_t halted;
914 	uint8_t epno;
915 	uint8_t index;
916 
917 	/* decode TRB */
918 	td_event = le64toh(trb->qwTrb0);
919 	temp = le32toh(trb->dwTrb2);
920 
921 	remainder = XHCI_TRB_2_REM_GET(temp);
922 	status = XHCI_TRB_2_ERROR_GET(temp);
923 
924 	temp = le32toh(trb->dwTrb3);
925 	epno = XHCI_TRB_3_EP_GET(temp);
926 	index = XHCI_TRB_3_SLOT_GET(temp);
927 
928 	/* check if error means halted */
929 	halted = (status != XHCI_TRB_ERROR_SHORT_PKT &&
930 	    status != XHCI_TRB_ERROR_SUCCESS);
931 
932 	DPRINTF("slot=%u epno=%u remainder=%u status=%u\n",
933 	    index, epno, remainder, status);
934 
935 	if (index > sc->sc_noslot) {
936 		DPRINTF("Invalid slot.\n");
937 		return;
938 	}
939 
940 	if ((epno == 0) || (epno >= XHCI_MAX_ENDPOINTS)) {
941 		DPRINTF("Invalid endpoint.\n");
942 		return;
943 	}
944 
945 	pepext = &sc->sc_hw.devs[index].endp[epno];
946 
947 	/* try to find the USB transfer that generated the event */
948 	for (i = 0;; i++) {
949 		struct usb_xfer *xfer;
950 		struct xhci_td *td;
951 
952 		if (i == (XHCI_MAX_TRANSFERS - 1)) {
953 			if (pepext->trb_ep_mode != USB_EP_MODE_STREAMS ||
954 			    stream_id == (XHCI_MAX_STREAMS - 1))
955 				break;
956 			stream_id++;
957 			i = 0;
958 			DPRINTFN(5, "stream_id=%u\n", stream_id);
959 		}
960 
961 		xfer = pepext->xfer[i + (XHCI_MAX_TRANSFERS * stream_id)];
962 		if (xfer == NULL)
963 			continue;
964 
965 		td = xfer->td_transfer_cache;
966 
967 		DPRINTFN(5, "Checking if 0x%016llx == (0x%016llx .. 0x%016llx)\n",
968 			(long long)td_event,
969 			(long long)td->td_self,
970 			(long long)td->td_self + sizeof(td->td_trb));
971 
972 		/*
973 		 * NOTE: Some XHCI implementations might not trigger
974 		 * an event on the last LINK TRB so we need to
975 		 * consider both the last and second last event
976 		 * address as conditions for a successful transfer.
977 		 *
978 		 * NOTE: We assume that the XHCI will only trigger one
979 		 * event per chain of TRBs.
980 		 */
981 
982 		offset = td_event - td->td_self;
983 
984 		if (offset >= 0 &&
985 		    offset < (int64_t)sizeof(td->td_trb)) {
986 			usb_pc_cpu_invalidate(td->page_cache);
987 
988 			/* compute rest of remainder, if any */
989 			for (i = (offset / 16) + 1; i < td->ntrb; i++) {
990 				temp = le32toh(td->td_trb[i].dwTrb2);
991 				remainder += XHCI_TRB_2_BYTES_GET(temp);
992 			}
993 
994 			DPRINTFN(5, "New remainder: %u\n", remainder);
995 
996 			/* clear isochronous transfer errors */
997 			if (xfer->flags_int.isochronous_xfr) {
998 				if (halted) {
999 					halted = 0;
1000 					status = XHCI_TRB_ERROR_SUCCESS;
1001 					remainder = td->len;
1002 				}
1003 			}
1004 
1005 			/* "td->remainder" is verified later */
1006 			td->remainder = remainder;
1007 			td->status = status;
1008 
1009 			usb_pc_cpu_flush(td->page_cache);
1010 
1011 			/*
1012 			 * 1) Last transfer descriptor makes the
1013 			 * transfer done
1014 			 */
1015 			if (((void *)td) == xfer->td_transfer_last) {
1016 				DPRINTF("TD is last\n");
1017 				xhci_generic_done(xfer);
1018 				break;
1019 			}
1020 
1021 			/*
1022 			 * 2) Any kind of error makes the transfer
1023 			 * done
1024 			 */
1025 			if (halted) {
1026 				DPRINTF("TD has I/O error\n");
1027 				xhci_generic_done(xfer);
1028 				break;
1029 			}
1030 
1031 			/*
1032 			 * 3) If there is no alternate next transfer,
1033 			 * a short packet also makes the transfer done
1034 			 */
1035 			if (td->remainder > 0) {
1036 				if (td->alt_next == NULL) {
1037 					DPRINTF(
1038 					    "short TD has no alternate next\n");
1039 					xhci_generic_done(xfer);
1040 					break;
1041 				}
1042 				DPRINTF("TD has short pkt\n");
1043 				if (xfer->flags_int.short_frames_ok ||
1044 				    xfer->flags_int.isochronous_xfr ||
1045 				    xfer->flags_int.control_xfr) {
1046 					/* follow the alt next */
1047 					xfer->td_transfer_cache = td->alt_next;
1048 					xhci_activate_transfer(xfer);
1049 					break;
1050 				}
1051 				xhci_skip_transfer(xfer);
1052 				xhci_generic_done(xfer);
1053 				break;
1054 			}
1055 
1056 			/*
1057 			 * 4) Transfer complete - go to next TD
1058 			 */
1059 			DPRINTF("Following next TD\n");
1060 			xfer->td_transfer_cache = td->obj_next;
1061 			xhci_activate_transfer(xfer);
1062 			break;		/* there should only be one match */
1063 		}
1064 	}
1065 }
1066 
1067 static int
1068 xhci_check_command(struct xhci_softc *sc, struct xhci_trb *trb)
1069 {
1070 	if (sc->sc_cmd_addr == trb->qwTrb0) {
1071 		DPRINTF("Received command event\n");
1072 		sc->sc_cmd_result[0] = trb->dwTrb2;
1073 		sc->sc_cmd_result[1] = trb->dwTrb3;
1074 		cv_signal(&sc->sc_cmd_cv);
1075 		return (1);	/* command match */
1076 	}
1077 	return (0);
1078 }
1079 
1080 static int
1081 xhci_interrupt_poll(struct xhci_softc *sc)
1082 {
1083 	struct usb_page_search buf_res;
1084 	struct xhci_hw_root *phwr;
1085 	uint64_t addr;
1086 	uint32_t temp;
1087 	int retval = 0;
1088 	uint16_t i;
1089 	uint8_t event;
1090 	uint8_t j;
1091 	uint8_t k;
1092 	uint8_t t;
1093 
1094 	usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res);
1095 
1096 	phwr = buf_res.buffer;
1097 
1098 	/* Receive any events */
1099 
1100 	usb_pc_cpu_invalidate(&sc->sc_hw.root_pc);
1101 
1102 	i = sc->sc_event_idx;
1103 	j = sc->sc_event_ccs;
1104 	t = 2;
1105 
1106 	while (1) {
1107 		temp = le32toh(phwr->hwr_events[i].dwTrb3);
1108 
1109 		k = (temp & XHCI_TRB_3_CYCLE_BIT) ? 1 : 0;
1110 
1111 		if (j != k)
1112 			break;
1113 
1114 		event = XHCI_TRB_3_TYPE_GET(temp);
1115 
1116 		DPRINTFN(10, "event[%u] = %u (0x%016llx 0x%08lx 0x%08lx)\n",
1117 		    i, event, (long long)le64toh(phwr->hwr_events[i].qwTrb0),
1118 		    (long)le32toh(phwr->hwr_events[i].dwTrb2),
1119 		    (long)le32toh(phwr->hwr_events[i].dwTrb3));
1120 
1121 		switch (event) {
1122 		case XHCI_TRB_EVENT_TRANSFER:
1123 			xhci_check_transfer(sc, &phwr->hwr_events[i]);
1124 			break;
1125 		case XHCI_TRB_EVENT_CMD_COMPLETE:
1126 			retval |= xhci_check_command(sc, &phwr->hwr_events[i]);
1127 			break;
1128 		default:
1129 			DPRINTF("Unhandled event = %u\n", event);
1130 			break;
1131 		}
1132 
1133 		i++;
1134 
1135 		if (i == XHCI_MAX_EVENTS) {
1136 			i = 0;
1137 			j ^= 1;
1138 
1139 			/* check for timeout */
1140 			if (!--t)
1141 				break;
1142 		}
1143 	}
1144 
1145 	sc->sc_event_idx = i;
1146 	sc->sc_event_ccs = j;
1147 
1148 	/*
1149 	 * NOTE: The Event Ring Dequeue Pointer Register is 64-bit
1150 	 * latched. That means to activate the register we need to
1151 	 * write both the low and high double word of the 64-bit
1152 	 * register.
1153 	 */
1154 
1155 	addr = buf_res.physaddr;
1156 	addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_events[i];
1157 
1158 	/* try to clear busy bit */
1159 	addr |= XHCI_ERDP_LO_BUSY;
1160 
1161 	XWRITE4(sc, runt, XHCI_ERDP_LO(0), (uint32_t)addr);
1162 	XWRITE4(sc, runt, XHCI_ERDP_HI(0), (uint32_t)(addr >> 32));
1163 
1164 	return (retval);
1165 }
1166 
1167 static usb_error_t
1168 xhci_do_command(struct xhci_softc *sc, struct xhci_trb *trb,
1169     uint16_t timeout_ms)
1170 {
1171 	struct usb_page_search buf_res;
1172 	struct xhci_hw_root *phwr;
1173 	uint64_t addr;
1174 	uint32_t temp;
1175 	uint8_t i;
1176 	uint8_t j;
1177 	uint8_t timeout = 0;
1178 	int err;
1179 
1180 	XHCI_CMD_ASSERT_LOCKED(sc);
1181 
1182 	/* get hardware root structure */
1183 
1184 	usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res);
1185 
1186 	phwr = buf_res.buffer;
1187 
1188 	/* Queue command */
1189 
1190 	USB_BUS_LOCK(&sc->sc_bus);
1191 retry:
1192 	i = sc->sc_command_idx;
1193 	j = sc->sc_command_ccs;
1194 
1195 	DPRINTFN(10, "command[%u] = %u (0x%016llx, 0x%08lx, 0x%08lx)\n",
1196 	    i, XHCI_TRB_3_TYPE_GET(le32toh(trb->dwTrb3)),
1197 	    (long long)le64toh(trb->qwTrb0),
1198 	    (long)le32toh(trb->dwTrb2),
1199 	    (long)le32toh(trb->dwTrb3));
1200 
1201 	phwr->hwr_commands[i].qwTrb0 = trb->qwTrb0;
1202 	phwr->hwr_commands[i].dwTrb2 = trb->dwTrb2;
1203 
1204 	usb_pc_cpu_flush(&sc->sc_hw.root_pc);
1205 
1206 	temp = trb->dwTrb3;
1207 
1208 	if (j)
1209 		temp |= htole32(XHCI_TRB_3_CYCLE_BIT);
1210 	else
1211 		temp &= ~htole32(XHCI_TRB_3_CYCLE_BIT);
1212 
1213 	temp &= ~htole32(XHCI_TRB_3_TC_BIT);
1214 
1215 	phwr->hwr_commands[i].dwTrb3 = temp;
1216 
1217 	usb_pc_cpu_flush(&sc->sc_hw.root_pc);
1218 
1219 	addr = buf_res.physaddr;
1220 	addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_commands[i];
1221 
1222 	sc->sc_cmd_addr = htole64(addr);
1223 
1224 	i++;
1225 
1226 	if (i == (XHCI_MAX_COMMANDS - 1)) {
1227 		if (j) {
1228 			temp = htole32(XHCI_TRB_3_TC_BIT |
1229 			    XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) |
1230 			    XHCI_TRB_3_CYCLE_BIT);
1231 		} else {
1232 			temp = htole32(XHCI_TRB_3_TC_BIT |
1233 			    XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK));
1234 		}
1235 
1236 		phwr->hwr_commands[i].dwTrb3 = temp;
1237 
1238 		usb_pc_cpu_flush(&sc->sc_hw.root_pc);
1239 
1240 		i = 0;
1241 		j ^= 1;
1242 	}
1243 
1244 	sc->sc_command_idx = i;
1245 	sc->sc_command_ccs = j;
1246 
1247 	XWRITE4(sc, door, XHCI_DOORBELL(0), 0);
1248 
1249 	err = cv_timedwait(&sc->sc_cmd_cv, &sc->sc_bus.bus_mtx,
1250 	    USB_MS_TO_TICKS(timeout_ms));
1251 
1252 	/*
1253 	 * In some error cases event interrupts are not generated.
1254 	 * Poll one time to see if the command has completed.
1255 	 */
1256 	if (err != 0 && xhci_interrupt_poll(sc) != 0) {
1257 		DPRINTF("Command was completed when polling\n");
1258 		err = 0;
1259 	}
1260 	if (err != 0) {
1261 		DPRINTF("Command timeout!\n");
1262 		/*
1263 		 * After some weeks of continuous operation, it has
1264 		 * been observed that the ASMedia Technology, ASM1042
1265 		 * SuperSpeed USB Host Controller can suddenly stop
1266 		 * accepting commands via the command queue. Try to
1267 		 * first reset the command queue. If that fails do a
1268 		 * host controller reset.
1269 		 */
1270 		if (timeout == 0 &&
1271 		    xhci_reset_command_queue_locked(sc) == 0) {
1272 			temp = le32toh(trb->dwTrb3);
1273 
1274 			/*
1275 			 * Avoid infinite XHCI reset loops if the set
1276 			 * address command fails to respond due to a
1277 			 * non-enumerating device:
1278 			 */
1279 			if (XHCI_TRB_3_TYPE_GET(temp) == XHCI_TRB_TYPE_ADDRESS_DEVICE &&
1280 			    (temp & XHCI_TRB_3_BSR_BIT) == 0) {
1281 				DPRINTF("Set address timeout\n");
1282 			} else {
1283 				timeout = 1;
1284 				goto retry;
1285 			}
1286 		} else {
1287 			DPRINTF("Controller reset!\n");
1288 			usb_bus_reset_async_locked(&sc->sc_bus);
1289 		}
1290 		err = USB_ERR_TIMEOUT;
1291 		trb->dwTrb2 = 0;
1292 		trb->dwTrb3 = 0;
1293 	} else {
1294 		temp = le32toh(sc->sc_cmd_result[0]);
1295 		if (XHCI_TRB_2_ERROR_GET(temp) != XHCI_TRB_ERROR_SUCCESS)
1296 			err = USB_ERR_IOERROR;
1297 
1298 		trb->dwTrb2 = sc->sc_cmd_result[0];
1299 		trb->dwTrb3 = sc->sc_cmd_result[1];
1300 	}
1301 
1302 	USB_BUS_UNLOCK(&sc->sc_bus);
1303 
1304 	return (err);
1305 }
1306 
1307 #if 0
1308 static usb_error_t
1309 xhci_cmd_nop(struct xhci_softc *sc)
1310 {
1311 	struct xhci_trb trb;
1312 	uint32_t temp;
1313 
1314 	DPRINTF("\n");
1315 
1316 	trb.qwTrb0 = 0;
1317 	trb.dwTrb2 = 0;
1318 	temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NOOP);
1319 
1320 	trb.dwTrb3 = htole32(temp);
1321 
1322 	return (xhci_do_command(sc, &trb, 100 /* ms */));
1323 }
1324 #endif
1325 
1326 static usb_error_t
1327 xhci_cmd_enable_slot(struct xhci_softc *sc, uint8_t *pslot)
1328 {
1329 	struct xhci_trb trb;
1330 	uint32_t temp;
1331 	usb_error_t err;
1332 
1333 	DPRINTF("\n");
1334 
1335 	trb.qwTrb0 = 0;
1336 	trb.dwTrb2 = 0;
1337 	trb.dwTrb3 = htole32(XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ENABLE_SLOT));
1338 
1339 	err = xhci_do_command(sc, &trb, 100 /* ms */);
1340 	if (err)
1341 		goto done;
1342 
1343 	temp = le32toh(trb.dwTrb3);
1344 
1345 	*pslot = XHCI_TRB_3_SLOT_GET(temp);
1346 
1347 done:
1348 	return (err);
1349 }
1350 
1351 static usb_error_t
1352 xhci_cmd_disable_slot(struct xhci_softc *sc, uint8_t slot_id)
1353 {
1354 	struct xhci_trb trb;
1355 	uint32_t temp;
1356 
1357 	DPRINTF("\n");
1358 
1359 	trb.qwTrb0 = 0;
1360 	trb.dwTrb2 = 0;
1361 	temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DISABLE_SLOT) |
1362 	    XHCI_TRB_3_SLOT_SET(slot_id);
1363 
1364 	trb.dwTrb3 = htole32(temp);
1365 
1366 	return (xhci_do_command(sc, &trb, 100 /* ms */));
1367 }
1368 
1369 static usb_error_t
1370 xhci_cmd_set_address(struct xhci_softc *sc, uint64_t input_ctx,
1371     uint8_t bsr, uint8_t slot_id)
1372 {
1373 	struct xhci_trb trb;
1374 	uint32_t temp;
1375 
1376 	DPRINTF("\n");
1377 
1378 	trb.qwTrb0 = htole64(input_ctx);
1379 	trb.dwTrb2 = 0;
1380 	temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ADDRESS_DEVICE) |
1381 	    XHCI_TRB_3_SLOT_SET(slot_id);
1382 
1383 	if (bsr)
1384 		temp |= XHCI_TRB_3_BSR_BIT;
1385 
1386 	trb.dwTrb3 = htole32(temp);
1387 
1388 	return (xhci_do_command(sc, &trb, 500 /* ms */));
1389 }
1390 
1391 static usb_error_t
1392 xhci_set_address(struct usb_device *udev, struct mtx *mtx, uint16_t address)
1393 {
1394 	struct usb_page_search buf_inp;
1395 	struct usb_page_search buf_dev;
1396 	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
1397 	struct xhci_hw_dev *hdev;
1398 	struct xhci_dev_ctx *pdev;
1399 	struct xhci_endpoint_ext *pepext;
1400 	uint32_t temp;
1401 	uint16_t mps;
1402 	usb_error_t err;
1403 	uint8_t index;
1404 
1405 	/* the root HUB case is not handled here */
1406 	if (udev->parent_hub == NULL)
1407 		return (USB_ERR_INVAL);
1408 
1409 	index = udev->controller_slot_id;
1410 
1411 	hdev = 	&sc->sc_hw.devs[index];
1412 
1413 	if (mtx != NULL)
1414 		mtx_unlock(mtx);
1415 
1416 	XHCI_CMD_LOCK(sc);
1417 
1418 	switch (hdev->state) {
1419 	case XHCI_ST_DEFAULT:
1420 	case XHCI_ST_ENABLED:
1421 
1422 		hdev->state = XHCI_ST_ENABLED;
1423 
1424 		/* set configure mask to slot and EP0 */
1425 		xhci_configure_mask(udev, 3, 0);
1426 
1427 		/* configure input slot context structure */
1428 		err = xhci_configure_device(udev);
1429 
1430 		if (err != 0) {
1431 			DPRINTF("Could not configure device\n");
1432 			break;
1433 		}
1434 
1435 		/* configure input endpoint context structure */
1436 		switch (udev->speed) {
1437 		case USB_SPEED_LOW:
1438 		case USB_SPEED_FULL:
1439 			mps = 8;
1440 			break;
1441 		case USB_SPEED_HIGH:
1442 			mps = 64;
1443 			break;
1444 		default:
1445 			mps = 512;
1446 			break;
1447 		}
1448 
1449 		pepext = xhci_get_endpoint_ext(udev,
1450 		    &udev->ctrl_ep_desc);
1451 
1452 		/* ensure the control endpoint is setup again */
1453 		USB_BUS_LOCK(udev->bus);
1454 		pepext->trb_halted = 1;
1455 		pepext->trb_running = 0;
1456 		USB_BUS_UNLOCK(udev->bus);
1457 
1458 		err = xhci_configure_endpoint(udev,
1459 		    &udev->ctrl_ep_desc, pepext,
1460 		    0, 1, 1, 0, mps, mps, USB_EP_MODE_DEFAULT);
1461 
1462 		if (err != 0) {
1463 			DPRINTF("Could not configure default endpoint\n");
1464 			break;
1465 		}
1466 
1467 		/* execute set address command */
1468 		usbd_get_page(&hdev->input_pc, 0, &buf_inp);
1469 
1470 		err = xhci_cmd_set_address(sc, buf_inp.physaddr,
1471 		    (address == 0), index);
1472 
1473 		if (err != 0) {
1474 			temp = le32toh(sc->sc_cmd_result[0]);
1475 			if (address == 0 && sc->sc_port_route != NULL &&
1476 			    XHCI_TRB_2_ERROR_GET(temp) ==
1477 			    XHCI_TRB_ERROR_PARAMETER) {
1478 				/* LynxPoint XHCI - ports are not switchable */
1479 				/* Un-route all ports from the XHCI */
1480 				sc->sc_port_route(sc->sc_bus.parent, 0, ~0);
1481 			}
1482 			DPRINTF("Could not set address "
1483 			    "for slot %u.\n", index);
1484 			if (address != 0)
1485 				break;
1486 		}
1487 
1488 		/* update device address to new value */
1489 
1490 		usbd_get_page(&hdev->device_pc, 0, &buf_dev);
1491 		pdev = buf_dev.buffer;
1492 		usb_pc_cpu_invalidate(&hdev->device_pc);
1493 
1494 		temp = xhci_ctx_get_le32(sc, &pdev->ctx_slot.dwSctx3);
1495 		udev->address = XHCI_SCTX_3_DEV_ADDR_GET(temp);
1496 
1497 		/* update device state to new value */
1498 
1499 		if (address != 0)
1500 			hdev->state = XHCI_ST_ADDRESSED;
1501 		else
1502 			hdev->state = XHCI_ST_DEFAULT;
1503 		break;
1504 
1505 	default:
1506 		DPRINTF("Wrong state for set address.\n");
1507 		err = USB_ERR_IOERROR;
1508 		break;
1509 	}
1510 	XHCI_CMD_UNLOCK(sc);
1511 
1512 	if (mtx != NULL)
1513 		mtx_lock(mtx);
1514 
1515 	return (err);
1516 }
1517 
1518 static usb_error_t
1519 xhci_cmd_configure_ep(struct xhci_softc *sc, uint64_t input_ctx,
1520     uint8_t deconfigure, uint8_t slot_id)
1521 {
1522 	struct xhci_trb trb;
1523 	uint32_t temp;
1524 
1525 	DPRINTF("\n");
1526 
1527 	trb.qwTrb0 = htole64(input_ctx);
1528 	trb.dwTrb2 = 0;
1529 	temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP) |
1530 	    XHCI_TRB_3_SLOT_SET(slot_id);
1531 
1532 	if (deconfigure)
1533 		temp |= XHCI_TRB_3_DCEP_BIT;
1534 
1535 	trb.dwTrb3 = htole32(temp);
1536 
1537 	return (xhci_do_command(sc, &trb, 100 /* ms */));
1538 }
1539 
1540 static usb_error_t
1541 xhci_cmd_evaluate_ctx(struct xhci_softc *sc, uint64_t input_ctx,
1542     uint8_t slot_id)
1543 {
1544 	struct xhci_trb trb;
1545 	uint32_t temp;
1546 
1547 	DPRINTF("\n");
1548 
1549 	trb.qwTrb0 = htole64(input_ctx);
1550 	trb.dwTrb2 = 0;
1551 	temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVALUATE_CTX) |
1552 	    XHCI_TRB_3_SLOT_SET(slot_id);
1553 	trb.dwTrb3 = htole32(temp);
1554 
1555 	return (xhci_do_command(sc, &trb, 100 /* ms */));
1556 }
1557 
1558 static usb_error_t
1559 xhci_cmd_reset_ep(struct xhci_softc *sc, uint8_t preserve,
1560     uint8_t ep_id, uint8_t slot_id)
1561 {
1562 	struct xhci_trb trb;
1563 	uint32_t temp;
1564 
1565 	DPRINTF("\n");
1566 
1567 	trb.qwTrb0 = 0;
1568 	trb.dwTrb2 = 0;
1569 	temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_EP) |
1570 	    XHCI_TRB_3_SLOT_SET(slot_id) |
1571 	    XHCI_TRB_3_EP_SET(ep_id);
1572 
1573 	if (preserve)
1574 		temp |= XHCI_TRB_3_PRSV_BIT;
1575 
1576 	trb.dwTrb3 = htole32(temp);
1577 
1578 	return (xhci_do_command(sc, &trb, 100 /* ms */));
1579 }
1580 
1581 static usb_error_t
1582 xhci_cmd_set_tr_dequeue_ptr(struct xhci_softc *sc, uint64_t dequeue_ptr,
1583     uint16_t stream_id, uint8_t ep_id, uint8_t slot_id)
1584 {
1585 	struct xhci_trb trb;
1586 	uint32_t temp;
1587 
1588 	DPRINTF("\n");
1589 
1590 	trb.qwTrb0 = htole64(dequeue_ptr);
1591 
1592 	temp = XHCI_TRB_2_STREAM_SET(stream_id);
1593 	trb.dwTrb2 = htole32(temp);
1594 
1595 	temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SET_TR_DEQUEUE) |
1596 	    XHCI_TRB_3_SLOT_SET(slot_id) |
1597 	    XHCI_TRB_3_EP_SET(ep_id);
1598 	trb.dwTrb3 = htole32(temp);
1599 
1600 	return (xhci_do_command(sc, &trb, 100 /* ms */));
1601 }
1602 
1603 static usb_error_t
1604 xhci_cmd_stop_ep(struct xhci_softc *sc, uint8_t suspend,
1605     uint8_t ep_id, uint8_t slot_id)
1606 {
1607 	struct xhci_trb trb;
1608 	uint32_t temp;
1609 
1610 	DPRINTF("\n");
1611 
1612 	trb.qwTrb0 = 0;
1613 	trb.dwTrb2 = 0;
1614 	temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STOP_EP) |
1615 	    XHCI_TRB_3_SLOT_SET(slot_id) |
1616 	    XHCI_TRB_3_EP_SET(ep_id);
1617 
1618 	if (suspend)
1619 		temp |= XHCI_TRB_3_SUSP_EP_BIT;
1620 
1621 	trb.dwTrb3 = htole32(temp);
1622 
1623 	return (xhci_do_command(sc, &trb, 100 /* ms */));
1624 }
1625 
1626 static usb_error_t
1627 xhci_cmd_reset_dev(struct xhci_softc *sc, uint8_t slot_id)
1628 {
1629 	struct xhci_trb trb;
1630 	uint32_t temp;
1631 
1632 	DPRINTF("\n");
1633 
1634 	trb.qwTrb0 = 0;
1635 	trb.dwTrb2 = 0;
1636 	temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_DEVICE) |
1637 	    XHCI_TRB_3_SLOT_SET(slot_id);
1638 
1639 	trb.dwTrb3 = htole32(temp);
1640 
1641 	return (xhci_do_command(sc, &trb, 100 /* ms */));
1642 }
1643 
1644 /*------------------------------------------------------------------------*
1645  *	xhci_interrupt - XHCI interrupt handler
1646  *------------------------------------------------------------------------*/
1647 void
1648 xhci_interrupt(struct xhci_softc *sc)
1649 {
1650 	uint32_t status;
1651 	uint32_t temp;
1652 
1653 	USB_BUS_LOCK(&sc->sc_bus);
1654 
1655 	status = XREAD4(sc, oper, XHCI_USBSTS);
1656 
1657 	/* acknowledge interrupts, if any */
1658 	if (status != 0) {
1659 		XWRITE4(sc, oper, XHCI_USBSTS, status);
1660 		DPRINTFN(16, "real interrupt (status=0x%08x)\n", status);
1661 	}
1662 
1663 	temp = XREAD4(sc, runt, XHCI_IMAN(0));
1664 
1665 	/* force clearing of pending interrupts */
1666 	if (temp & XHCI_IMAN_INTR_PEND)
1667 		XWRITE4(sc, runt, XHCI_IMAN(0), temp);
1668 
1669 	/* check for event(s) */
1670 	xhci_interrupt_poll(sc);
1671 
1672 	if (status & (XHCI_STS_PCD | XHCI_STS_HCH |
1673 	    XHCI_STS_HSE | XHCI_STS_HCE)) {
1674 		if (status & XHCI_STS_PCD) {
1675 			xhci_root_intr(sc);
1676 		}
1677 
1678 		if (status & XHCI_STS_HCH) {
1679 			printf("%s: host controller halted\n",
1680 			    __FUNCTION__);
1681 		}
1682 
1683 		if (status & XHCI_STS_HSE) {
1684 			printf("%s: host system error\n",
1685 			    __FUNCTION__);
1686 		}
1687 
1688 		if (status & XHCI_STS_HCE) {
1689 			printf("%s: host controller error\n",
1690 			   __FUNCTION__);
1691 		}
1692 	}
1693 	USB_BUS_UNLOCK(&sc->sc_bus);
1694 }
1695 
1696 /*------------------------------------------------------------------------*
1697  *	xhci_timeout - XHCI timeout handler
1698  *------------------------------------------------------------------------*/
1699 static void
1700 xhci_timeout(void *arg)
1701 {
1702 	struct usb_xfer *xfer = arg;
1703 
1704 	DPRINTF("xfer=%p\n", xfer);
1705 
1706 	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
1707 
1708 	/* transfer is transferred */
1709 	xhci_device_done(xfer, USB_ERR_TIMEOUT);
1710 }
1711 
1712 static void
1713 xhci_do_poll(struct usb_bus *bus)
1714 {
1715 	struct xhci_softc *sc = XHCI_BUS2SC(bus);
1716 
1717 	USB_BUS_LOCK(&sc->sc_bus);
1718 	xhci_interrupt_poll(sc);
1719 	USB_BUS_UNLOCK(&sc->sc_bus);
1720 }
1721 
1722 static void
1723 xhci_setup_generic_chain_sub(struct xhci_std_temp *temp)
1724 {
1725 	struct usb_page_search buf_res;
1726 	struct xhci_td *td;
1727 	struct xhci_td *td_next;
1728 	struct xhci_td *td_alt_next;
1729 	struct xhci_td *td_first;
1730 	uint32_t buf_offset;
1731 	uint32_t average;
1732 	uint32_t len_old;
1733 	uint32_t npkt_off;
1734 	uint32_t dword;
1735 	uint8_t shortpkt_old;
1736 	uint8_t precompute;
1737 	uint8_t x;
1738 
1739 	td_alt_next = NULL;
1740 	buf_offset = 0;
1741 	shortpkt_old = temp->shortpkt;
1742 	len_old = temp->len;
1743 	npkt_off = 0;
1744 	precompute = 1;
1745 
1746 restart:
1747 
1748 	td = temp->td;
1749 	td_next = td_first = temp->td_next;
1750 
1751 	while (1) {
1752 		if (temp->len == 0) {
1753 			if (temp->shortpkt)
1754 				break;
1755 
1756 			/* send a Zero Length Packet, ZLP, last */
1757 
1758 			temp->shortpkt = 1;
1759 			average = 0;
1760 
1761 		} else {
1762 			average = temp->average;
1763 
1764 			if (temp->len < average) {
1765 				if (temp->len % temp->max_packet_size) {
1766 					temp->shortpkt = 1;
1767 				}
1768 				average = temp->len;
1769 			}
1770 		}
1771 
1772 		if (td_next == NULL)
1773 			panic("%s: out of XHCI transfer descriptors!", __FUNCTION__);
1774 
1775 		/* get next TD */
1776 
1777 		td = td_next;
1778 		td_next = td->obj_next;
1779 
1780 		/* check if we are pre-computing */
1781 
1782 		if (precompute) {
1783 			/* update remaining length */
1784 
1785 			temp->len -= average;
1786 
1787 			continue;
1788 		}
1789 		/* fill out current TD */
1790 
1791 		td->len = average;
1792 		td->remainder = 0;
1793 		td->status = 0;
1794 
1795 		/* update remaining length */
1796 
1797 		temp->len -= average;
1798 
1799 		/* reset TRB index */
1800 
1801 		x = 0;
1802 
1803 		if (temp->trb_type == XHCI_TRB_TYPE_SETUP_STAGE) {
1804 			/* immediate data */
1805 
1806 			if (average > 8)
1807 				average = 8;
1808 
1809 			td->td_trb[0].qwTrb0 = 0;
1810 
1811 			usbd_copy_out(temp->pc, temp->offset + buf_offset,
1812 			   (uint8_t *)(uintptr_t)&td->td_trb[0].qwTrb0,
1813 			   average);
1814 
1815 			dword = XHCI_TRB_2_BYTES_SET(8) |
1816 			    XHCI_TRB_2_TDSZ_SET(0) |
1817 			    XHCI_TRB_2_IRQ_SET(0);
1818 
1819 			td->td_trb[0].dwTrb2 = htole32(dword);
1820 
1821 			dword = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SETUP_STAGE) |
1822 			  XHCI_TRB_3_IDT_BIT | XHCI_TRB_3_CYCLE_BIT;
1823 
1824 			/* check wLength */
1825 			if (td->td_trb[0].qwTrb0 &
1826 			   htole64(XHCI_TRB_0_WLENGTH_MASK)) {
1827 				if (td->td_trb[0].qwTrb0 &
1828 				    htole64(XHCI_TRB_0_DIR_IN_MASK))
1829 					dword |= XHCI_TRB_3_TRT_IN;
1830 				else
1831 					dword |= XHCI_TRB_3_TRT_OUT;
1832 			}
1833 
1834 			td->td_trb[0].dwTrb3 = htole32(dword);
1835 #ifdef USB_DEBUG
1836 			xhci_dump_trb(&td->td_trb[x]);
1837 #endif
1838 			x++;
1839 
1840 		} else do {
1841 			uint32_t npkt;
1842 
1843 			/* fill out buffer pointers */
1844 
1845 			if (average == 0) {
1846 				memset(&buf_res, 0, sizeof(buf_res));
1847 			} else {
1848 				usbd_get_page(temp->pc, temp->offset +
1849 				    buf_offset, &buf_res);
1850 
1851 				/* get length to end of page */
1852 				if (buf_res.length > average)
1853 					buf_res.length = average;
1854 
1855 				/* check for maximum length */
1856 				if (buf_res.length > XHCI_TD_PAGE_SIZE)
1857 					buf_res.length = XHCI_TD_PAGE_SIZE;
1858 
1859 				npkt_off += buf_res.length;
1860 			}
1861 
1862 			/* set up npkt */
1863 			npkt = howmany(len_old - npkt_off,
1864 				       temp->max_packet_size);
1865 
1866 			if (npkt == 0)
1867 				npkt = 1;
1868 			else if (npkt > 31)
1869 				npkt = 31;
1870 
1871 			/* fill out TRB's */
1872 			td->td_trb[x].qwTrb0 =
1873 			    htole64((uint64_t)buf_res.physaddr);
1874 
1875 			dword =
1876 			  XHCI_TRB_2_BYTES_SET(buf_res.length) |
1877 			  XHCI_TRB_2_TDSZ_SET(npkt) |
1878 			  XHCI_TRB_2_IRQ_SET(0);
1879 
1880 			td->td_trb[x].dwTrb2 = htole32(dword);
1881 
1882 			switch (temp->trb_type) {
1883 			case XHCI_TRB_TYPE_ISOCH:
1884 				dword = XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_CYCLE_BIT |
1885 				    XHCI_TRB_3_TBC_SET(temp->tbc) |
1886 				    XHCI_TRB_3_TLBPC_SET(temp->tlbpc);
1887 				if (td != td_first) {
1888 					dword |= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL);
1889 				} else if (temp->do_isoc_sync != 0) {
1890 					temp->do_isoc_sync = 0;
1891 					/* wait until "isoc_frame" */
1892 					dword |= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ISOCH) |
1893 					    XHCI_TRB_3_FRID_SET(temp->isoc_frame / 8);
1894 				} else {
1895 					/* start data transfer at next interval */
1896 					dword |= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ISOCH) |
1897 					    XHCI_TRB_3_ISO_SIA_BIT;
1898 				}
1899 				if (temp->direction == UE_DIR_IN)
1900 					dword |= XHCI_TRB_3_ISP_BIT;
1901 				break;
1902 			case XHCI_TRB_TYPE_DATA_STAGE:
1903 				dword = XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_CYCLE_BIT |
1904 				    XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DATA_STAGE);
1905 				if (temp->direction == UE_DIR_IN)
1906 					dword |= XHCI_TRB_3_DIR_IN | XHCI_TRB_3_ISP_BIT;
1907 				/*
1908 				 * Section 3.2.9 in the XHCI
1909 				 * specification about control
1910 				 * transfers says that we should use a
1911 				 * normal-TRB if there are more TRBs
1912 				 * extending the data-stage
1913 				 * TRB. Update the "trb_type".
1914 				 */
1915 				temp->trb_type = XHCI_TRB_TYPE_NORMAL;
1916 				break;
1917 			case XHCI_TRB_TYPE_STATUS_STAGE:
1918 				dword = XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_CYCLE_BIT |
1919 				    XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STATUS_STAGE);
1920 				if (temp->direction == UE_DIR_IN)
1921 					dword |= XHCI_TRB_3_DIR_IN;
1922 				break;
1923 			default:	/* XHCI_TRB_TYPE_NORMAL */
1924 				dword = XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_CYCLE_BIT |
1925 				    XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL);
1926 				if (temp->direction == UE_DIR_IN)
1927 					dword |= XHCI_TRB_3_ISP_BIT;
1928 				break;
1929 			}
1930 			td->td_trb[x].dwTrb3 = htole32(dword);
1931 
1932 			average -= buf_res.length;
1933 			buf_offset += buf_res.length;
1934 #ifdef USB_DEBUG
1935 			xhci_dump_trb(&td->td_trb[x]);
1936 #endif
1937 			x++;
1938 
1939 		} while (average != 0);
1940 
1941 		td->td_trb[x-1].dwTrb3 |= htole32(XHCI_TRB_3_IOC_BIT);
1942 
1943 		/* store number of data TRB's */
1944 
1945 		td->ntrb = x;
1946 
1947 		DPRINTF("NTRB=%u\n", x);
1948 
1949 		/* fill out link TRB */
1950 
1951 		if (td_next != NULL) {
1952 			/* link the current TD with the next one */
1953 			td->td_trb[x].qwTrb0 = htole64((uint64_t)td_next->td_self);
1954 			DPRINTF("LINK=0x%08llx\n", (long long)td_next->td_self);
1955 		} else {
1956 			/* this field will get updated later */
1957 			DPRINTF("NOLINK\n");
1958 		}
1959 
1960 		dword = XHCI_TRB_2_IRQ_SET(0);
1961 
1962 		td->td_trb[x].dwTrb2 = htole32(dword);
1963 
1964 		dword = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) |
1965 		    XHCI_TRB_3_CYCLE_BIT | XHCI_TRB_3_IOC_BIT |
1966 		    /*
1967 		     * CHAIN-BIT: Ensure that a multi-TRB IN-endpoint
1968 		     * frame only receives a single short packet event
1969 		     * by setting the CHAIN bit in the LINK field. In
1970 		     * addition some XHCI controllers have problems
1971 		     * sending a ZLP unless the CHAIN-BIT is set in
1972 		     * the LINK TRB.
1973 		     */
1974 		    XHCI_TRB_3_CHAIN_BIT;
1975 
1976 		td->td_trb[x].dwTrb3 = htole32(dword);
1977 
1978 		td->alt_next = td_alt_next;
1979 #ifdef USB_DEBUG
1980 		xhci_dump_trb(&td->td_trb[x]);
1981 #endif
1982 		usb_pc_cpu_flush(td->page_cache);
1983 	}
1984 
1985 	if (precompute) {
1986 		precompute = 0;
1987 
1988 		/* set up alt next pointer, if any */
1989 		if (temp->last_frame) {
1990 			td_alt_next = NULL;
1991 		} else {
1992 			/* we use this field internally */
1993 			td_alt_next = td_next;
1994 		}
1995 
1996 		/* restore */
1997 		temp->shortpkt = shortpkt_old;
1998 		temp->len = len_old;
1999 		goto restart;
2000 	}
2001 
2002 	/*
2003 	 * Remove cycle bit from the first TRB if we are
2004 	 * stepping them:
2005 	 */
2006 	if (temp->step_td != 0) {
2007 		td_first->td_trb[0].dwTrb3 &= ~htole32(XHCI_TRB_3_CYCLE_BIT);
2008 		usb_pc_cpu_flush(td_first->page_cache);
2009 	}
2010 
2011 	/* clear TD SIZE to zero, hence this is the last TRB */
2012 	/* remove chain bit because this is the last data TRB in the chain */
2013 	td->td_trb[td->ntrb - 1].dwTrb2 &= ~htole32(XHCI_TRB_2_TDSZ_SET(31));
2014 	td->td_trb[td->ntrb - 1].dwTrb3 &= ~htole32(XHCI_TRB_3_CHAIN_BIT);
2015 	/* remove CHAIN-BIT from last LINK TRB */
2016 	td->td_trb[td->ntrb].dwTrb3 &= ~htole32(XHCI_TRB_3_CHAIN_BIT);
2017 
2018 	usb_pc_cpu_flush(td->page_cache);
2019 
2020 	temp->td = td;
2021 	temp->td_next = td_next;
2022 }
2023 
2024 static void
2025 xhci_setup_generic_chain(struct usb_xfer *xfer)
2026 {
2027 	struct xhci_std_temp temp;
2028 	struct xhci_td *td;
2029 	uint32_t x;
2030 	uint32_t y;
2031 	uint8_t mult;
2032 
2033 	temp.do_isoc_sync = 0;
2034 	temp.step_td = 0;
2035 	temp.tbc = 0;
2036 	temp.tlbpc = 0;
2037 	temp.average = xfer->max_hc_frame_size;
2038 	temp.max_packet_size = xfer->max_packet_size;
2039 	temp.sc = XHCI_BUS2SC(xfer->xroot->bus);
2040 	temp.pc = NULL;
2041 	temp.last_frame = 0;
2042 	temp.offset = 0;
2043 	temp.multishort = xfer->flags_int.isochronous_xfr ||
2044 	    xfer->flags_int.control_xfr ||
2045 	    xfer->flags_int.short_frames_ok;
2046 
2047 	/* toggle the DMA set we are using */
2048 	xfer->flags_int.curr_dma_set ^= 1;
2049 
2050 	/* get next DMA set */
2051 	td = xfer->td_start[xfer->flags_int.curr_dma_set];
2052 
2053 	temp.td = NULL;
2054 	temp.td_next = td;
2055 
2056 	xfer->td_transfer_first = td;
2057 	xfer->td_transfer_cache = td;
2058 
2059 	if (xfer->flags_int.isochronous_xfr) {
2060 		uint8_t shift;
2061 
2062 		/* compute multiplier for ISOCHRONOUS transfers */
2063 		mult = xfer->endpoint->ecomp ?
2064 		    UE_GET_SS_ISO_MULT(xfer->endpoint->ecomp->bmAttributes)
2065 		    : 0;
2066 		/* check for USB 2.0 multiplier */
2067 		if (mult == 0) {
2068 			mult = (xfer->endpoint->edesc->
2069 			    wMaxPacketSize[1] >> 3) & 3;
2070 		}
2071 		/* range check */
2072 		if (mult > 2)
2073 			mult = 3;
2074 		else
2075 			mult++;
2076 
2077 		x = XREAD4(temp.sc, runt, XHCI_MFINDEX);
2078 
2079 		DPRINTF("MFINDEX=0x%08x\n", x);
2080 
2081 		switch (usbd_get_speed(xfer->xroot->udev)) {
2082 		case USB_SPEED_FULL:
2083 			shift = 3;
2084 			temp.isoc_delta = 8;	/* 1ms */
2085 			x += temp.isoc_delta - 1;
2086 			x &= ~(temp.isoc_delta - 1);
2087 			break;
2088 		default:
2089 			shift = usbd_xfer_get_fps_shift(xfer);
2090 			temp.isoc_delta = 1U << shift;
2091 			x += temp.isoc_delta - 1;
2092 			x &= ~(temp.isoc_delta - 1);
2093 			/* simple frame load balancing */
2094 			x += xfer->endpoint->usb_uframe;
2095 			break;
2096 		}
2097 
2098 		y = XHCI_MFINDEX_GET(x - xfer->endpoint->isoc_next);
2099 
2100 		if ((xfer->endpoint->is_synced == 0) ||
2101 		    (y < (xfer->nframes << shift)) ||
2102 		    (XHCI_MFINDEX_GET(-y) >= (128 * 8))) {
2103 			/*
2104 			 * If there is data underflow or the pipe
2105 			 * queue is empty we schedule the transfer a
2106 			 * few frames ahead of the current frame
2107 			 * position. Else two isochronous transfers
2108 			 * might overlap.
2109 			 */
2110 			xfer->endpoint->isoc_next = XHCI_MFINDEX_GET(x + (3 * 8));
2111 			xfer->endpoint->is_synced = 1;
2112 			temp.do_isoc_sync = 1;
2113 
2114 			DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2115 		}
2116 
2117 		/* compute isochronous completion time */
2118 
2119 		y = XHCI_MFINDEX_GET(xfer->endpoint->isoc_next - (x & ~7));
2120 
2121 		xfer->isoc_time_complete =
2122 		    usb_isoc_time_expand(&temp.sc->sc_bus, x / 8) +
2123 		    (y / 8) + (((xfer->nframes << shift) + 7) / 8);
2124 
2125 		x = 0;
2126 		temp.isoc_frame = xfer->endpoint->isoc_next;
2127 		temp.trb_type = XHCI_TRB_TYPE_ISOCH;
2128 
2129 		xfer->endpoint->isoc_next += xfer->nframes << shift;
2130 
2131 	} else if (xfer->flags_int.control_xfr) {
2132 		/* check if we should prepend a setup message */
2133 
2134 		if (xfer->flags_int.control_hdr) {
2135 			temp.len = xfer->frlengths[0];
2136 			temp.pc = xfer->frbuffers + 0;
2137 			temp.shortpkt = temp.len ? 1 : 0;
2138 			temp.trb_type = XHCI_TRB_TYPE_SETUP_STAGE;
2139 			temp.direction = 0;
2140 
2141 			/* check for last frame */
2142 			if (xfer->nframes == 1) {
2143 				/* no STATUS stage yet, SETUP is last */
2144 				if (xfer->flags_int.control_act)
2145 					temp.last_frame = 1;
2146 			}
2147 
2148 			xhci_setup_generic_chain_sub(&temp);
2149 		}
2150 		x = 1;
2151 		mult = 1;
2152 		temp.isoc_delta = 0;
2153 		temp.isoc_frame = 0;
2154 		temp.trb_type = xfer->flags_int.control_did_data ?
2155 		    XHCI_TRB_TYPE_NORMAL : XHCI_TRB_TYPE_DATA_STAGE;
2156 	} else {
2157 		x = 0;
2158 		mult = 1;
2159 		temp.isoc_delta = 0;
2160 		temp.isoc_frame = 0;
2161 		temp.trb_type = XHCI_TRB_TYPE_NORMAL;
2162 	}
2163 
2164 	if (x != xfer->nframes) {
2165                 /* set up page_cache pointer */
2166                 temp.pc = xfer->frbuffers + x;
2167 		/* set endpoint direction */
2168 		temp.direction = UE_GET_DIR(xfer->endpointno);
2169 	}
2170 
2171 	while (x != xfer->nframes) {
2172 		/* DATA0 / DATA1 message */
2173 
2174 		temp.len = xfer->frlengths[x];
2175 		temp.step_td = ((xfer->endpointno & UE_DIR_IN) &&
2176 		    x != 0 && temp.multishort == 0);
2177 
2178 		x++;
2179 
2180 		if (x == xfer->nframes) {
2181 			if (xfer->flags_int.control_xfr) {
2182 				/* no STATUS stage yet, DATA is last */
2183 				if (xfer->flags_int.control_act)
2184 					temp.last_frame = 1;
2185 			} else {
2186 				temp.last_frame = 1;
2187 			}
2188 		}
2189 		if (temp.len == 0) {
2190 			/* make sure that we send an USB packet */
2191 
2192 			temp.shortpkt = 0;
2193 
2194 			temp.tbc = 0;
2195 			temp.tlbpc = mult - 1;
2196 
2197 		} else if (xfer->flags_int.isochronous_xfr) {
2198 			uint8_t tdpc;
2199 
2200 			/*
2201 			 * Isochronous transfers don't have short
2202 			 * packet termination:
2203 			 */
2204 
2205 			temp.shortpkt = 1;
2206 
2207 			/* isochronous transfers have a transfer limit */
2208 
2209 			if (temp.len > xfer->max_frame_size)
2210 				temp.len = xfer->max_frame_size;
2211 
2212 			/* compute TD packet count */
2213 			tdpc = howmany(temp.len, xfer->max_packet_size);
2214 
2215 			temp.tbc = howmany(tdpc, mult) - 1;
2216 			temp.tlbpc = (tdpc % mult);
2217 
2218 			if (temp.tlbpc == 0)
2219 				temp.tlbpc = mult - 1;
2220 			else
2221 				temp.tlbpc--;
2222 		} else {
2223 			/* regular data transfer */
2224 
2225 			temp.shortpkt = xfer->flags.force_short_xfer ? 0 : 1;
2226 		}
2227 
2228 		xhci_setup_generic_chain_sub(&temp);
2229 
2230 		if (xfer->flags_int.isochronous_xfr) {
2231 			temp.offset += xfer->frlengths[x - 1];
2232 			temp.isoc_frame += temp.isoc_delta;
2233 		} else {
2234 			/* get next Page Cache pointer */
2235 			temp.pc = xfer->frbuffers + x;
2236 		}
2237 	}
2238 
2239 	/* check if we should append a status stage */
2240 
2241 	if (xfer->flags_int.control_xfr &&
2242 	    !xfer->flags_int.control_act) {
2243 		/*
2244 		 * Send a DATA1 message and invert the current
2245 		 * endpoint direction.
2246 		 */
2247 		if (xhcictlstep || temp.sc->sc_ctlstep) {
2248 			/*
2249 			 * Some XHCI controllers will not delay the
2250 			 * status stage until the next SOF. Force this
2251 			 * behaviour to avoid failed control
2252 			 * transfers.
2253 			 */
2254 			temp.step_td = (xfer->nframes != 0);
2255 		} else {
2256 			temp.step_td = 0;
2257 		}
2258 		temp.direction = UE_GET_DIR(xfer->endpointno) ^ UE_DIR_IN;
2259 		temp.len = 0;
2260 		temp.pc = NULL;
2261 		temp.shortpkt = 0;
2262 		temp.last_frame = 1;
2263 		temp.trb_type = XHCI_TRB_TYPE_STATUS_STAGE;
2264 
2265 		xhci_setup_generic_chain_sub(&temp);
2266 	}
2267 
2268 	td = temp.td;
2269 
2270 	/* must have at least one frame! */
2271 
2272 	xfer->td_transfer_last = td;
2273 
2274 	DPRINTF("first=%p last=%p\n", xfer->td_transfer_first, td);
2275 }
2276 
2277 static void
2278 xhci_set_slot_pointer(struct xhci_softc *sc, uint8_t index, uint64_t dev_addr)
2279 {
2280 	struct usb_page_search buf_res;
2281 	struct xhci_dev_ctx_addr *pdctxa;
2282 
2283 	usbd_get_page(&sc->sc_hw.ctx_pc, 0, &buf_res);
2284 
2285 	pdctxa = buf_res.buffer;
2286 
2287 	DPRINTF("addr[%u]=0x%016llx\n", index, (long long)dev_addr);
2288 
2289 	pdctxa->qwBaaDevCtxAddr[index] = htole64(dev_addr);
2290 
2291 	usb_pc_cpu_flush(&sc->sc_hw.ctx_pc);
2292 }
2293 
2294 static usb_error_t
2295 xhci_configure_mask(struct usb_device *udev, uint32_t mask, uint8_t drop)
2296 {
2297 	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2298 	struct usb_page_search buf_inp;
2299 	struct xhci_input_dev_ctx *pinp;
2300 	uint32_t temp;
2301 	uint8_t index;
2302 	uint8_t x;
2303 
2304 	index = udev->controller_slot_id;
2305 
2306 	usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp);
2307 
2308 	pinp = buf_inp.buffer;
2309 
2310 	if (drop) {
2311 		mask &= XHCI_INCTX_NON_CTRL_MASK;
2312 		xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx0, mask);
2313 		xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx1, 0);
2314 	} else {
2315 		/*
2316 		 * Some hardware requires that we drop the endpoint
2317 		 * context before adding it again:
2318 		 */
2319 		xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx0,
2320 		    mask & XHCI_INCTX_NON_CTRL_MASK);
2321 
2322 		/* Add new endpoint context */
2323 		xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx1, mask);
2324 
2325 		/* find most significant set bit */
2326 		for (x = 31; x != 1; x--) {
2327 			if (mask & (1 << x))
2328 				break;
2329 		}
2330 
2331 		/* adjust */
2332 		x--;
2333 
2334 		/* figure out the maximum number of contexts */
2335 		if (x > sc->sc_hw.devs[index].context_num)
2336 			sc->sc_hw.devs[index].context_num = x;
2337 		else
2338 			x = sc->sc_hw.devs[index].context_num;
2339 
2340 		/* update number of contexts */
2341 		temp = xhci_ctx_get_le32(sc, &pinp->ctx_slot.dwSctx0);
2342 		temp &= ~XHCI_SCTX_0_CTX_NUM_SET(31);
2343 		temp |= XHCI_SCTX_0_CTX_NUM_SET(x + 1);
2344 		xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx0, temp);
2345 	}
2346 	usb_pc_cpu_flush(&sc->sc_hw.devs[index].input_pc);
2347 	return (0);
2348 }
2349 
2350 static usb_error_t
2351 xhci_configure_endpoint(struct usb_device *udev,
2352     struct usb_endpoint_descriptor *edesc, struct xhci_endpoint_ext *pepext,
2353     uint16_t interval, uint8_t max_packet_count,
2354     uint8_t mult, uint8_t fps_shift, uint16_t max_packet_size,
2355     uint16_t max_frame_size, uint8_t ep_mode)
2356 {
2357 	struct usb_page_search buf_inp;
2358 	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2359 	struct xhci_input_dev_ctx *pinp;
2360 	uint64_t ring_addr = pepext->physaddr;
2361 	uint32_t temp;
2362 	uint8_t index;
2363 	uint8_t epno;
2364 	uint8_t type;
2365 
2366 	index = udev->controller_slot_id;
2367 
2368 	usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp);
2369 
2370 	pinp = buf_inp.buffer;
2371 
2372 	epno = edesc->bEndpointAddress;
2373 	type = edesc->bmAttributes & UE_XFERTYPE;
2374 
2375 	if (type == UE_CONTROL)
2376 		epno |= UE_DIR_IN;
2377 
2378 	epno = XHCI_EPNO2EPID(epno);
2379 
2380  	if (epno == 0)
2381 		return (USB_ERR_NO_PIPE);		/* invalid */
2382 
2383 	if (max_packet_count == 0)
2384 		return (USB_ERR_BAD_BUFSIZE);
2385 
2386 	max_packet_count--;
2387 
2388 	if (mult == 0)
2389 		return (USB_ERR_BAD_BUFSIZE);
2390 
2391 	/* store endpoint mode */
2392 	pepext->trb_ep_mode = ep_mode;
2393 	/* store bMaxPacketSize for control endpoints */
2394 	pepext->trb_ep_maxp = edesc->wMaxPacketSize[0];
2395 	usb_pc_cpu_flush(pepext->page_cache);
2396 
2397 	if (ep_mode == USB_EP_MODE_STREAMS) {
2398 		temp = XHCI_EPCTX_0_EPSTATE_SET(0) |
2399 		    XHCI_EPCTX_0_MAXP_STREAMS_SET(XHCI_MAX_STREAMS_LOG - 1) |
2400 		    XHCI_EPCTX_0_LSA_SET(1);
2401 
2402 		ring_addr += sizeof(struct xhci_trb) *
2403 		    XHCI_MAX_TRANSFERS * XHCI_MAX_STREAMS;
2404 	} else {
2405 		temp = XHCI_EPCTX_0_EPSTATE_SET(0) |
2406 		    XHCI_EPCTX_0_MAXP_STREAMS_SET(0) |
2407 		    XHCI_EPCTX_0_LSA_SET(0);
2408 
2409 		ring_addr |= XHCI_EPCTX_2_DCS_SET(1);
2410 	}
2411 
2412 	switch (udev->speed) {
2413 	case USB_SPEED_FULL:
2414 	case USB_SPEED_LOW:
2415 		/* 1ms -> 125us */
2416 		fps_shift += 3;
2417 		break;
2418 	default:
2419 		break;
2420 	}
2421 
2422 	switch (type) {
2423 	case UE_INTERRUPT:
2424 		if (fps_shift > 3)
2425 			fps_shift--;
2426 		temp |= XHCI_EPCTX_0_IVAL_SET(fps_shift);
2427 		break;
2428 	case UE_ISOCHRONOUS:
2429 		temp |= XHCI_EPCTX_0_IVAL_SET(fps_shift);
2430 
2431 		switch (udev->speed) {
2432 		case USB_SPEED_SUPER:
2433 			if (mult > 3)
2434 				mult = 3;
2435 			temp |= XHCI_EPCTX_0_MULT_SET(mult - 1);
2436 			max_packet_count /= mult;
2437 			break;
2438 		default:
2439 			break;
2440 		}
2441 		break;
2442 	default:
2443 		break;
2444 	}
2445 
2446 	xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx0, temp);
2447 
2448 	temp =
2449 	    XHCI_EPCTX_1_HID_SET(0) |
2450 	    XHCI_EPCTX_1_MAXB_SET(max_packet_count) |
2451 	    XHCI_EPCTX_1_MAXP_SIZE_SET(max_packet_size);
2452 
2453 	/*
2454 	 * Always enable the "three strikes and you are gone" feature
2455 	 * except for ISOCHRONOUS endpoints. This is suggested by
2456 	 * section 4.3.3 in the XHCI specification about device slot
2457 	 * initialisation.
2458 	 */
2459 	if (type != UE_ISOCHRONOUS)
2460 		temp |= XHCI_EPCTX_1_CERR_SET(3);
2461 
2462 	switch (type) {
2463 	case UE_CONTROL:
2464 		temp |= XHCI_EPCTX_1_EPTYPE_SET(4);
2465 		break;
2466 	case UE_ISOCHRONOUS:
2467 		temp |= XHCI_EPCTX_1_EPTYPE_SET(1);
2468 		break;
2469 	case UE_BULK:
2470 		temp |= XHCI_EPCTX_1_EPTYPE_SET(2);
2471 		break;
2472 	default:
2473 		temp |= XHCI_EPCTX_1_EPTYPE_SET(3);
2474 		break;
2475 	}
2476 
2477 	/* check for IN direction */
2478 	if (epno & 1)
2479 		temp |= XHCI_EPCTX_1_EPTYPE_SET(4);
2480 
2481 	xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx1, temp);
2482 	xhci_ctx_set_le64(sc, &pinp->ctx_ep[epno - 1].qwEpCtx2, ring_addr);
2483 
2484 	switch (edesc->bmAttributes & UE_XFERTYPE) {
2485 	case UE_INTERRUPT:
2486 	case UE_ISOCHRONOUS:
2487 		temp = XHCI_EPCTX_4_MAX_ESIT_PAYLOAD_SET(max_frame_size) |
2488 		    XHCI_EPCTX_4_AVG_TRB_LEN_SET(MIN(XHCI_PAGE_SIZE,
2489 		    max_frame_size));
2490 		break;
2491 	case UE_CONTROL:
2492 		temp = XHCI_EPCTX_4_AVG_TRB_LEN_SET(8);
2493 		break;
2494 	default:
2495 		temp = XHCI_EPCTX_4_AVG_TRB_LEN_SET(XHCI_PAGE_SIZE);
2496 		break;
2497 	}
2498 
2499 	xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx4, temp);
2500 
2501 #ifdef USB_DEBUG
2502 	xhci_dump_endpoint(sc, &pinp->ctx_ep[epno - 1]);
2503 #endif
2504 	usb_pc_cpu_flush(&sc->sc_hw.devs[index].input_pc);
2505 
2506 	return (0);		/* success */
2507 }
2508 
2509 static usb_error_t
2510 xhci_configure_endpoint_by_xfer(struct usb_xfer *xfer)
2511 {
2512 	struct xhci_endpoint_ext *pepext;
2513 	struct usb_endpoint_ss_comp_descriptor *ecomp;
2514 	usb_stream_t x;
2515 
2516 	pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
2517 	    xfer->endpoint->edesc);
2518 
2519 	ecomp = xfer->endpoint->ecomp;
2520 
2521 	for (x = 0; x != XHCI_MAX_STREAMS; x++) {
2522 		uint64_t temp;
2523 
2524 		/* halt any transfers */
2525 		pepext->trb[x * XHCI_MAX_TRANSFERS].dwTrb3 = 0;
2526 
2527 		/* compute start of TRB ring for stream "x" */
2528 		temp = pepext->physaddr +
2529 		    (x * XHCI_MAX_TRANSFERS * sizeof(struct xhci_trb)) +
2530 		    XHCI_SCTX_0_SCT_SEC_TR_RING;
2531 
2532 		/* make tree structure */
2533 		pepext->trb[(XHCI_MAX_TRANSFERS *
2534 		    XHCI_MAX_STREAMS) + x].qwTrb0 = htole64(temp);
2535 
2536 		/* reserved fields */
2537 		pepext->trb[(XHCI_MAX_TRANSFERS *
2538                     XHCI_MAX_STREAMS) + x].dwTrb2 = 0;
2539 		pepext->trb[(XHCI_MAX_TRANSFERS *
2540 		    XHCI_MAX_STREAMS) + x].dwTrb3 = 0;
2541 	}
2542 	usb_pc_cpu_flush(pepext->page_cache);
2543 
2544 	return (xhci_configure_endpoint(xfer->xroot->udev,
2545 	    xfer->endpoint->edesc, pepext,
2546 	    xfer->interval, xfer->max_packet_count,
2547 	    (ecomp != NULL) ? UE_GET_SS_ISO_MULT(ecomp->bmAttributes) + 1 : 1,
2548 	    usbd_xfer_get_fps_shift(xfer), xfer->max_packet_size,
2549 	    xfer->max_frame_size, xfer->endpoint->ep_mode));
2550 }
2551 
2552 static usb_error_t
2553 xhci_configure_device(struct usb_device *udev)
2554 {
2555 	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2556 	struct usb_page_search buf_inp;
2557 	struct usb_page_cache *pcinp;
2558 	struct xhci_input_dev_ctx *pinp;
2559 	struct usb_device *hubdev;
2560 	uint32_t temp;
2561 	uint32_t route;
2562 	uint32_t rh_port;
2563 	uint8_t is_hub;
2564 	uint8_t index;
2565 	uint8_t depth;
2566 
2567 	index = udev->controller_slot_id;
2568 
2569 	DPRINTF("index=%u\n", index);
2570 
2571 	pcinp = &sc->sc_hw.devs[index].input_pc;
2572 
2573 	usbd_get_page(pcinp, 0, &buf_inp);
2574 
2575 	pinp = buf_inp.buffer;
2576 
2577 	rh_port = 0;
2578 	route = 0;
2579 
2580 	/* figure out route string and root HUB port number */
2581 
2582 	for (hubdev = udev; hubdev != NULL; hubdev = hubdev->parent_hub) {
2583 		if (hubdev->parent_hub == NULL)
2584 			break;
2585 
2586 		depth = hubdev->parent_hub->depth;
2587 
2588 		/*
2589 		 * NOTE: HS/FS/LS devices and the SS root HUB can have
2590 		 * more than 15 ports
2591 		 */
2592 
2593 		rh_port = hubdev->port_no;
2594 
2595 		if (depth == 0)
2596 			break;
2597 
2598 		if (rh_port > 15)
2599 			rh_port = 15;
2600 
2601 		if (depth < 6)
2602 			route |= rh_port << (4 * (depth - 1));
2603 	}
2604 
2605 	DPRINTF("Route=0x%08x\n", route);
2606 
2607 	temp = XHCI_SCTX_0_ROUTE_SET(route) |
2608 	    XHCI_SCTX_0_CTX_NUM_SET(
2609 	    sc->sc_hw.devs[index].context_num + 1);
2610 
2611 	switch (udev->speed) {
2612 	case USB_SPEED_LOW:
2613 		temp |= XHCI_SCTX_0_SPEED_SET(2);
2614 		if (udev->parent_hs_hub != NULL &&
2615 		    udev->parent_hs_hub->ddesc.bDeviceProtocol ==
2616 		    UDPROTO_HSHUBMTT) {
2617 			DPRINTF("Device inherits MTT\n");
2618 			temp |= XHCI_SCTX_0_MTT_SET(1);
2619 		}
2620 		break;
2621 	case USB_SPEED_HIGH:
2622 		temp |= XHCI_SCTX_0_SPEED_SET(3);
2623 		if (sc->sc_hw.devs[index].nports != 0 &&
2624 		    udev->ddesc.bDeviceProtocol == UDPROTO_HSHUBMTT) {
2625 			DPRINTF("HUB supports MTT\n");
2626 			temp |= XHCI_SCTX_0_MTT_SET(1);
2627 		}
2628 		break;
2629 	case USB_SPEED_FULL:
2630 		temp |= XHCI_SCTX_0_SPEED_SET(1);
2631 		if (udev->parent_hs_hub != NULL &&
2632 		    udev->parent_hs_hub->ddesc.bDeviceProtocol ==
2633 		    UDPROTO_HSHUBMTT) {
2634 			DPRINTF("Device inherits MTT\n");
2635 			temp |= XHCI_SCTX_0_MTT_SET(1);
2636 		}
2637 		break;
2638 	default:
2639 		temp |= XHCI_SCTX_0_SPEED_SET(4);
2640 		break;
2641 	}
2642 
2643 	is_hub = sc->sc_hw.devs[index].nports != 0 &&
2644 	    (udev->speed == USB_SPEED_SUPER ||
2645 	    udev->speed == USB_SPEED_HIGH);
2646 
2647 	if (is_hub)
2648 		temp |= XHCI_SCTX_0_HUB_SET(1);
2649 
2650 	xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx0, temp);
2651 
2652 	temp = XHCI_SCTX_1_RH_PORT_SET(rh_port);
2653 
2654 	if (is_hub) {
2655 		temp |= XHCI_SCTX_1_NUM_PORTS_SET(
2656 		    sc->sc_hw.devs[index].nports);
2657 	}
2658 
2659 	xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx1, temp);
2660 
2661 	temp = XHCI_SCTX_2_IRQ_TARGET_SET(0);
2662 
2663 	if (is_hub) {
2664 		temp |= XHCI_SCTX_2_TT_THINK_TIME_SET(
2665 		    sc->sc_hw.devs[index].tt);
2666 	}
2667 
2668 	hubdev = udev->parent_hs_hub;
2669 
2670 	/* check if we should activate the transaction translator */
2671 	switch (udev->speed) {
2672 	case USB_SPEED_FULL:
2673 	case USB_SPEED_LOW:
2674 		if (hubdev != NULL) {
2675 			temp |= XHCI_SCTX_2_TT_HUB_SID_SET(
2676 			    hubdev->controller_slot_id);
2677 			temp |= XHCI_SCTX_2_TT_PORT_NUM_SET(
2678 			    udev->hs_port_no);
2679 		}
2680 		break;
2681 	default:
2682 		break;
2683 	}
2684 
2685 	xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx2, temp);
2686 
2687 	/*
2688 	 * These fields should be initialized to zero, according to
2689 	 * XHCI section 6.2.2 - slot context:
2690 	 */
2691 	temp = XHCI_SCTX_3_DEV_ADDR_SET(0) |
2692 	    XHCI_SCTX_3_SLOT_STATE_SET(0);
2693 
2694 	xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx3, temp);
2695 
2696 #ifdef USB_DEBUG
2697 	xhci_dump_device(sc, &pinp->ctx_slot);
2698 #endif
2699 	usb_pc_cpu_flush(pcinp);
2700 
2701 	return (0);		/* success */
2702 }
2703 
2704 static usb_error_t
2705 xhci_alloc_device_ext(struct usb_device *udev)
2706 {
2707 	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2708 	struct usb_page_search buf_dev;
2709 	struct usb_page_search buf_ep;
2710 	struct xhci_trb *trb;
2711 	struct usb_page_cache *pc;
2712 	struct usb_page *pg;
2713 	uint64_t addr;
2714 	uint8_t index;
2715 	uint8_t i;
2716 
2717 	index = udev->controller_slot_id;
2718 
2719 	pc = &sc->sc_hw.devs[index].device_pc;
2720 	pg = &sc->sc_hw.devs[index].device_pg;
2721 
2722 	/* need to initialize the page cache */
2723 	pc->tag_parent = sc->sc_bus.dma_parent_tag;
2724 
2725 	if (usb_pc_alloc_mem(pc, pg, sc->sc_ctx_is_64_byte ?
2726 	    (2 * sizeof(struct xhci_dev_ctx)) :
2727 	    sizeof(struct xhci_dev_ctx), XHCI_PAGE_SIZE))
2728 		goto error;
2729 
2730 	usbd_get_page(pc, 0, &buf_dev);
2731 
2732 	pc = &sc->sc_hw.devs[index].input_pc;
2733 	pg = &sc->sc_hw.devs[index].input_pg;
2734 
2735 	/* need to initialize the page cache */
2736 	pc->tag_parent = sc->sc_bus.dma_parent_tag;
2737 
2738 	if (usb_pc_alloc_mem(pc, pg, sc->sc_ctx_is_64_byte ?
2739 	    (2 * sizeof(struct xhci_input_dev_ctx)) :
2740 	    sizeof(struct xhci_input_dev_ctx), XHCI_PAGE_SIZE)) {
2741 		goto error;
2742 	}
2743 
2744 	/* initialize all endpoint LINK TRBs */
2745 
2746 	for (i = 0; i != XHCI_MAX_ENDPOINTS; i++) {
2747 		pc = &sc->sc_hw.devs[index].endpoint_pc[i];
2748 		pg = &sc->sc_hw.devs[index].endpoint_pg[i];
2749 
2750 		/* need to initialize the page cache */
2751 		pc->tag_parent = sc->sc_bus.dma_parent_tag;
2752 
2753 		if (usb_pc_alloc_mem(pc, pg,
2754 		    sizeof(struct xhci_dev_endpoint_trbs), XHCI_TRB_ALIGN)) {
2755 			goto error;
2756 		}
2757 
2758 		/* lookup endpoint TRB ring */
2759 		usbd_get_page(pc, 0, &buf_ep);
2760 
2761 		/* get TRB pointer */
2762 		trb = buf_ep.buffer;
2763 		trb += XHCI_MAX_TRANSFERS - 1;
2764 
2765 		/* get TRB start address */
2766 		addr = buf_ep.physaddr;
2767 
2768 		/* create LINK TRB */
2769 		trb->qwTrb0 = htole64(addr);
2770 		trb->dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0));
2771 		trb->dwTrb3 = htole32(XHCI_TRB_3_CYCLE_BIT |
2772 		    XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK));
2773 
2774 		usb_pc_cpu_flush(pc);
2775 	}
2776 
2777 	xhci_set_slot_pointer(sc, index, buf_dev.physaddr);
2778 
2779 	return (0);
2780 
2781 error:
2782 	xhci_free_device_ext(udev);
2783 
2784 	return (USB_ERR_NOMEM);
2785 }
2786 
2787 static void
2788 xhci_free_device_ext(struct usb_device *udev)
2789 {
2790 	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2791 	uint8_t index;
2792 	uint8_t i;
2793 
2794 	index = udev->controller_slot_id;
2795 	xhci_set_slot_pointer(sc, index, 0);
2796 
2797 	usb_pc_free_mem(&sc->sc_hw.devs[index].device_pc);
2798 	usb_pc_free_mem(&sc->sc_hw.devs[index].input_pc);
2799 	for (i = 0; i != XHCI_MAX_ENDPOINTS; i++)
2800 		usb_pc_free_mem(&sc->sc_hw.devs[index].endpoint_pc[i]);
2801 }
2802 
2803 static struct xhci_endpoint_ext *
2804 xhci_get_endpoint_ext(struct usb_device *udev, struct usb_endpoint_descriptor *edesc)
2805 {
2806 	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2807 	struct xhci_endpoint_ext *pepext;
2808 	struct usb_page_cache *pc;
2809 	struct usb_page_search buf_ep;
2810 	uint8_t epno;
2811 	uint8_t index;
2812 
2813 	epno = edesc->bEndpointAddress;
2814 	if ((edesc->bmAttributes & UE_XFERTYPE) == UE_CONTROL)
2815 		epno |= UE_DIR_IN;
2816 
2817 	epno = XHCI_EPNO2EPID(epno);
2818 
2819 	index = udev->controller_slot_id;
2820 
2821 	pc = &sc->sc_hw.devs[index].endpoint_pc[epno];
2822 
2823 	usbd_get_page(pc, 0, &buf_ep);
2824 
2825 	pepext = &sc->sc_hw.devs[index].endp[epno];
2826 	pepext->page_cache = pc;
2827 	pepext->trb = buf_ep.buffer;
2828 	pepext->physaddr = buf_ep.physaddr;
2829 
2830 	return (pepext);
2831 }
2832 
2833 static void
2834 xhci_endpoint_doorbell(struct usb_xfer *xfer)
2835 {
2836 	struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus);
2837 	uint8_t epno;
2838 	uint8_t index;
2839 
2840 	epno = xfer->endpointno;
2841 	if (xfer->flags_int.control_xfr)
2842 		epno |= UE_DIR_IN;
2843 
2844 	epno = XHCI_EPNO2EPID(epno);
2845 	index = xfer->xroot->udev->controller_slot_id;
2846 
2847 	if (xfer->xroot->udev->flags.self_suspended == 0) {
2848 		XWRITE4(sc, door, XHCI_DOORBELL(index),
2849 		    epno | XHCI_DB_SID_SET(xfer->stream_id));
2850 	}
2851 }
2852 
2853 static void
2854 xhci_transfer_remove(struct usb_xfer *xfer, usb_error_t error)
2855 {
2856 	struct xhci_endpoint_ext *pepext;
2857 
2858 	if (xfer->flags_int.bandwidth_reclaimed) {
2859 		xfer->flags_int.bandwidth_reclaimed = 0;
2860 
2861 		pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
2862 		    xfer->endpoint->edesc);
2863 
2864 		pepext->trb_used[xfer->stream_id]--;
2865 
2866 		pepext->xfer[xfer->qh_pos] = NULL;
2867 
2868 		if (error && pepext->trb_running != 0) {
2869 			pepext->trb_halted = 1;
2870 			pepext->trb_running = 0;
2871 		}
2872 	}
2873 }
2874 
2875 static usb_error_t
2876 xhci_transfer_insert(struct usb_xfer *xfer)
2877 {
2878 	struct xhci_td *td_first;
2879 	struct xhci_td *td_last;
2880 	struct xhci_trb *trb_link;
2881 	struct xhci_endpoint_ext *pepext;
2882 	uint64_t addr;
2883 	usb_stream_t id;
2884 	uint8_t i;
2885 	uint8_t inext;
2886 	uint8_t trb_limit;
2887 
2888 	DPRINTFN(8, "\n");
2889 
2890 	id = xfer->stream_id;
2891 
2892 	/* check if already inserted */
2893 	if (xfer->flags_int.bandwidth_reclaimed) {
2894 		DPRINTFN(8, "Already in schedule\n");
2895 		return (0);
2896 	}
2897 
2898 	pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
2899 	    xfer->endpoint->edesc);
2900 
2901 	td_first = xfer->td_transfer_first;
2902 	td_last = xfer->td_transfer_last;
2903 	addr = pepext->physaddr;
2904 
2905 	switch (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE) {
2906 	case UE_CONTROL:
2907 	case UE_INTERRUPT:
2908 		/* single buffered */
2909 		trb_limit = 1;
2910 		break;
2911 	default:
2912 		/* multi buffered */
2913 		trb_limit = (XHCI_MAX_TRANSFERS - 2);
2914 		break;
2915 	}
2916 
2917 	if (pepext->trb_used[id] >= trb_limit) {
2918 		DPRINTFN(8, "Too many TDs queued.\n");
2919 		return (USB_ERR_NOMEM);
2920 	}
2921 
2922 	/* check if bMaxPacketSize changed */
2923 	if (xfer->flags_int.control_xfr != 0 &&
2924 	    pepext->trb_ep_maxp != xfer->endpoint->edesc->wMaxPacketSize[0]) {
2925 		DPRINTFN(8, "Reconfigure control endpoint\n");
2926 
2927 		/* force driver to reconfigure endpoint */
2928 		pepext->trb_halted = 1;
2929 		pepext->trb_running = 0;
2930 	}
2931 
2932 	/* check for stopped condition, after putting transfer on interrupt queue */
2933 	if (pepext->trb_running == 0) {
2934 		struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus);
2935 
2936 		DPRINTFN(8, "Not running\n");
2937 
2938 		/* start configuration */
2939 		(void)usb_proc_msignal(USB_BUS_CONTROL_XFER_PROC(&sc->sc_bus),
2940 		    &sc->sc_config_msg[0], &sc->sc_config_msg[1]);
2941 		return (0);
2942 	}
2943 
2944 	pepext->trb_used[id]++;
2945 
2946 	/* get current TRB index */
2947 	i = pepext->trb_index[id];
2948 
2949 	/* get next TRB index */
2950 	inext = (i + 1);
2951 
2952 	/* the last entry of the ring is a hardcoded link TRB */
2953 	if (inext >= (XHCI_MAX_TRANSFERS - 1))
2954 		inext = 0;
2955 
2956 	/* store next TRB index, before stream ID offset is added */
2957 	pepext->trb_index[id] = inext;
2958 
2959 	/* offset for stream */
2960 	i += id * XHCI_MAX_TRANSFERS;
2961 	inext += id * XHCI_MAX_TRANSFERS;
2962 
2963 	/* compute terminating return address */
2964 	addr += (inext * sizeof(struct xhci_trb));
2965 
2966 	/* compute link TRB pointer */
2967 	trb_link = td_last->td_trb + td_last->ntrb;
2968 
2969 	/* update next pointer of last link TRB */
2970 	trb_link->qwTrb0 = htole64(addr);
2971 	trb_link->dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0));
2972 	trb_link->dwTrb3 = htole32(XHCI_TRB_3_IOC_BIT |
2973 	    XHCI_TRB_3_CYCLE_BIT |
2974 	    XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK));
2975 
2976 #ifdef USB_DEBUG
2977 	xhci_dump_trb(&td_last->td_trb[td_last->ntrb]);
2978 #endif
2979 	usb_pc_cpu_flush(td_last->page_cache);
2980 
2981 	/* write ahead chain end marker */
2982 
2983 	pepext->trb[inext].qwTrb0 = 0;
2984 	pepext->trb[inext].dwTrb2 = 0;
2985 	pepext->trb[inext].dwTrb3 = 0;
2986 
2987 	/* update next pointer of link TRB */
2988 
2989 	pepext->trb[i].qwTrb0 = htole64((uint64_t)td_first->td_self);
2990 	pepext->trb[i].dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0));
2991 
2992 #ifdef USB_DEBUG
2993 	xhci_dump_trb(&pepext->trb[i]);
2994 #endif
2995 	usb_pc_cpu_flush(pepext->page_cache);
2996 
2997 	/* toggle cycle bit which activates the transfer chain */
2998 
2999 	pepext->trb[i].dwTrb3 = htole32(XHCI_TRB_3_CYCLE_BIT |
3000 	    XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK));
3001 
3002 	usb_pc_cpu_flush(pepext->page_cache);
3003 
3004 	DPRINTF("qh_pos = %u\n", i);
3005 
3006 	pepext->xfer[i] = xfer;
3007 
3008 	xfer->qh_pos = i;
3009 
3010 	xfer->flags_int.bandwidth_reclaimed = 1;
3011 
3012 	xhci_endpoint_doorbell(xfer);
3013 
3014 	return (0);
3015 }
3016 
3017 static void
3018 xhci_root_intr(struct xhci_softc *sc)
3019 {
3020 	uint16_t i;
3021 
3022 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3023 
3024 	/* clear any old interrupt data */
3025 	memset(sc->sc_hub_idata, 0, sizeof(sc->sc_hub_idata));
3026 
3027 	for (i = 1; i <= sc->sc_noport; i++) {
3028 		/* pick out CHANGE bits from the status register */
3029 		if (XREAD4(sc, oper, XHCI_PORTSC(i)) & (
3030 		    XHCI_PS_CSC | XHCI_PS_PEC |
3031 		    XHCI_PS_OCC | XHCI_PS_WRC |
3032 		    XHCI_PS_PRC | XHCI_PS_PLC |
3033 		    XHCI_PS_CEC)) {
3034 			sc->sc_hub_idata[i / 8] |= 1 << (i % 8);
3035 			DPRINTF("port %d changed\n", i);
3036 		}
3037 	}
3038 	uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
3039 	    sizeof(sc->sc_hub_idata));
3040 }
3041 
3042 /*------------------------------------------------------------------------*
3043  *	xhci_device_done - XHCI done handler
3044  *
3045  * NOTE: This function can be called two times in a row on
3046  * the same USB transfer. From close and from interrupt.
3047  *------------------------------------------------------------------------*/
3048 static void
3049 xhci_device_done(struct usb_xfer *xfer, usb_error_t error)
3050 {
3051 	DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n",
3052 	    xfer, xfer->endpoint, error);
3053 
3054 	/* remove transfer from HW queue */
3055 	xhci_transfer_remove(xfer, error);
3056 
3057 	/* dequeue transfer and start next transfer */
3058 	usbd_transfer_done(xfer, error);
3059 }
3060 
3061 /*------------------------------------------------------------------------*
3062  * XHCI data transfer support (generic type)
3063  *------------------------------------------------------------------------*/
3064 static void
3065 xhci_device_generic_open(struct usb_xfer *xfer)
3066 {
3067 	if (xfer->flags_int.isochronous_xfr) {
3068 		switch (xfer->xroot->udev->speed) {
3069 		case USB_SPEED_FULL:
3070 			break;
3071 		default:
3072 			usb_hs_bandwidth_alloc(xfer);
3073 			break;
3074 		}
3075 	}
3076 }
3077 
3078 static void
3079 xhci_device_generic_close(struct usb_xfer *xfer)
3080 {
3081 	DPRINTF("\n");
3082 
3083 	xhci_device_done(xfer, USB_ERR_CANCELLED);
3084 
3085 	if (xfer->flags_int.isochronous_xfr) {
3086 		switch (xfer->xroot->udev->speed) {
3087 		case USB_SPEED_FULL:
3088 			break;
3089 		default:
3090 			usb_hs_bandwidth_free(xfer);
3091 			break;
3092 		}
3093 	}
3094 }
3095 
3096 static void
3097 xhci_device_generic_multi_enter(struct usb_endpoint *ep,
3098     usb_stream_t stream_id, struct usb_xfer *enter_xfer)
3099 {
3100 	struct usb_xfer *xfer;
3101 
3102 	/* check if there is a current transfer */
3103 	xfer = ep->endpoint_q[stream_id].curr;
3104 	if (xfer == NULL)
3105 		return;
3106 
3107 	/*
3108 	 * Check if the current transfer is started and then pickup
3109 	 * the next one, if any. Else wait for next start event due to
3110 	 * block on failure feature.
3111 	 */
3112 	if (!xfer->flags_int.bandwidth_reclaimed)
3113 		return;
3114 
3115 	xfer = TAILQ_FIRST(&ep->endpoint_q[stream_id].head);
3116 	if (xfer == NULL) {
3117 		/*
3118 		 * In case of enter we have to consider that the
3119 		 * transfer is queued by the USB core after the enter
3120 		 * method is called.
3121 		 */
3122 		xfer = enter_xfer;
3123 
3124 		if (xfer == NULL)
3125 			return;
3126 	}
3127 
3128 	/* try to multi buffer */
3129 	xhci_transfer_insert(xfer);
3130 }
3131 
3132 static void
3133 xhci_device_generic_enter(struct usb_xfer *xfer)
3134 {
3135 	DPRINTF("\n");
3136 
3137 	/* set up TD's and QH */
3138 	xhci_setup_generic_chain(xfer);
3139 
3140 	xhci_device_generic_multi_enter(xfer->endpoint,
3141 	    xfer->stream_id, xfer);
3142 }
3143 
3144 static void
3145 xhci_device_generic_start(struct usb_xfer *xfer)
3146 {
3147 	DPRINTF("\n");
3148 
3149 	/* try to insert xfer on HW queue */
3150 	xhci_transfer_insert(xfer);
3151 
3152 	/* try to multi buffer */
3153 	xhci_device_generic_multi_enter(xfer->endpoint,
3154 	    xfer->stream_id, NULL);
3155 
3156 	/* add transfer last on interrupt queue */
3157 	usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
3158 
3159 	/* start timeout, if any */
3160 	if (xfer->timeout != 0)
3161 		usbd_transfer_timeout_ms(xfer, &xhci_timeout, xfer->timeout);
3162 }
3163 
3164 static const struct usb_pipe_methods xhci_device_generic_methods =
3165 {
3166 	.open = xhci_device_generic_open,
3167 	.close = xhci_device_generic_close,
3168 	.enter = xhci_device_generic_enter,
3169 	.start = xhci_device_generic_start,
3170 };
3171 
3172 /*------------------------------------------------------------------------*
3173  * xhci root HUB support
3174  *------------------------------------------------------------------------*
3175  * Simulate a hardware HUB by handling all the necessary requests.
3176  *------------------------------------------------------------------------*/
3177 #define	HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) }
3178 
3179 static const
3180 struct usb_device_descriptor xhci_devd =
3181 {
3182 	.bLength = sizeof(xhci_devd),
3183 	.bDescriptorType = UDESC_DEVICE,	/* type */
3184 	HSETW(.bcdUSB, 0x0300),			/* USB version */
3185 	.bDeviceClass = UDCLASS_HUB,		/* class */
3186 	.bDeviceSubClass = UDSUBCLASS_HUB,	/* subclass */
3187 	.bDeviceProtocol = UDPROTO_SSHUB,	/* protocol */
3188 	.bMaxPacketSize = 9,			/* max packet size */
3189 	HSETW(.idVendor, 0x0000),		/* vendor */
3190 	HSETW(.idProduct, 0x0000),		/* product */
3191 	HSETW(.bcdDevice, 0x0100),		/* device version */
3192 	.iManufacturer = 1,
3193 	.iProduct = 2,
3194 	.iSerialNumber = 0,
3195 	.bNumConfigurations = 1,		/* # of configurations */
3196 };
3197 
3198 static const
3199 struct xhci_bos_desc xhci_bosd = {
3200 	.bosd = {
3201 		.bLength = sizeof(xhci_bosd.bosd),
3202 		.bDescriptorType = UDESC_BOS,
3203 		HSETW(.wTotalLength, sizeof(xhci_bosd)),
3204 		.bNumDeviceCaps = 3,
3205 	},
3206 	.usb2extd = {
3207 		.bLength = sizeof(xhci_bosd.usb2extd),
3208 		.bDescriptorType = 1,
3209 		.bDevCapabilityType = 2,
3210 		.bmAttributes[0] = 2,
3211 	},
3212 	.usbdcd = {
3213 		.bLength = sizeof(xhci_bosd.usbdcd),
3214 		.bDescriptorType = UDESC_DEVICE_CAPABILITY,
3215 		.bDevCapabilityType = 3,
3216 		.bmAttributes = 0, /* XXX */
3217 		HSETW(.wSpeedsSupported, 0x000C),
3218 		.bFunctionalitySupport = 8,
3219 		.bU1DevExitLat = 255,	/* dummy - not used */
3220 		.wU2DevExitLat = { 0x00, 0x08 },
3221 	},
3222 	.cidd = {
3223 		.bLength = sizeof(xhci_bosd.cidd),
3224 		.bDescriptorType = 1,
3225 		.bDevCapabilityType = 4,
3226 		.bReserved = 0,
3227 		.bContainerID = 0, /* XXX */
3228 	},
3229 };
3230 
3231 static const
3232 struct xhci_config_desc xhci_confd = {
3233 	.confd = {
3234 		.bLength = sizeof(xhci_confd.confd),
3235 		.bDescriptorType = UDESC_CONFIG,
3236 		.wTotalLength[0] = sizeof(xhci_confd),
3237 		.bNumInterface = 1,
3238 		.bConfigurationValue = 1,
3239 		.iConfiguration = 0,
3240 		.bmAttributes = UC_SELF_POWERED,
3241 		.bMaxPower = 0		/* max power */
3242 	},
3243 	.ifcd = {
3244 		.bLength = sizeof(xhci_confd.ifcd),
3245 		.bDescriptorType = UDESC_INTERFACE,
3246 		.bNumEndpoints = 1,
3247 		.bInterfaceClass = UICLASS_HUB,
3248 		.bInterfaceSubClass = UISUBCLASS_HUB,
3249 		.bInterfaceProtocol = 0,
3250 	},
3251 	.endpd = {
3252 		.bLength = sizeof(xhci_confd.endpd),
3253 		.bDescriptorType = UDESC_ENDPOINT,
3254 		.bEndpointAddress = UE_DIR_IN | XHCI_INTR_ENDPT,
3255 		.bmAttributes = UE_INTERRUPT,
3256 		.wMaxPacketSize[0] = 2,		/* max 15 ports */
3257 		.bInterval = 255,
3258 	},
3259 	.endpcd = {
3260 		.bLength = sizeof(xhci_confd.endpcd),
3261 		.bDescriptorType = UDESC_ENDPOINT_SS_COMP,
3262 		.bMaxBurst = 0,
3263 		.bmAttributes = 0,
3264 	},
3265 };
3266 
3267 static const
3268 struct usb_hub_ss_descriptor xhci_hubd = {
3269 	.bLength = sizeof(xhci_hubd),
3270 	.bDescriptorType = UDESC_SS_HUB,
3271 };
3272 
3273 static usb_error_t
3274 xhci_roothub_exec(struct usb_device *udev,
3275     struct usb_device_request *req, const void **pptr, uint16_t *plength)
3276 {
3277 	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
3278 	const char *str_ptr;
3279 	const void *ptr;
3280 	uint32_t port;
3281 	uint32_t v;
3282 	uint16_t len;
3283 	uint16_t i;
3284 	uint16_t value;
3285 	uint16_t index;
3286 	uint8_t j;
3287 	usb_error_t err;
3288 
3289 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3290 
3291 	/* buffer reset */
3292 	ptr = (const void *)&sc->sc_hub_desc;
3293 	len = 0;
3294 	err = 0;
3295 
3296 	value = UGETW(req->wValue);
3297 	index = UGETW(req->wIndex);
3298 
3299 	DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x "
3300 	    "wValue=0x%04x wIndex=0x%04x\n",
3301 	    req->bmRequestType, req->bRequest,
3302 	    UGETW(req->wLength), value, index);
3303 
3304 #define	C(x,y) ((x) | ((y) << 8))
3305 	switch (C(req->bRequest, req->bmRequestType)) {
3306 	case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
3307 	case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
3308 	case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
3309 		/*
3310 		 * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
3311 		 * for the integrated root hub.
3312 		 */
3313 		break;
3314 	case C(UR_GET_CONFIG, UT_READ_DEVICE):
3315 		len = 1;
3316 		sc->sc_hub_desc.temp[0] = sc->sc_conf;
3317 		break;
3318 	case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
3319 		switch (value >> 8) {
3320 		case UDESC_DEVICE:
3321 			if ((value & 0xff) != 0) {
3322 				err = USB_ERR_IOERROR;
3323 				goto done;
3324 			}
3325 			len = sizeof(xhci_devd);
3326 			ptr = (const void *)&xhci_devd;
3327 			break;
3328 
3329 		case UDESC_BOS:
3330 			if ((value & 0xff) != 0) {
3331 				err = USB_ERR_IOERROR;
3332 				goto done;
3333 			}
3334 			len = sizeof(xhci_bosd);
3335 			ptr = (const void *)&xhci_bosd;
3336 			break;
3337 
3338 		case UDESC_CONFIG:
3339 			if ((value & 0xff) != 0) {
3340 				err = USB_ERR_IOERROR;
3341 				goto done;
3342 			}
3343 			len = sizeof(xhci_confd);
3344 			ptr = (const void *)&xhci_confd;
3345 			break;
3346 
3347 		case UDESC_STRING:
3348 			switch (value & 0xff) {
3349 			case 0:	/* Language table */
3350 				str_ptr = "\001";
3351 				break;
3352 
3353 			case 1:	/* Vendor */
3354 				str_ptr = sc->sc_vendor;
3355 				break;
3356 
3357 			case 2:	/* Product */
3358 				str_ptr = "XHCI root HUB";
3359 				break;
3360 
3361 			default:
3362 				str_ptr = "";
3363 				break;
3364 			}
3365 
3366 			len = usb_make_str_desc(
3367 			    sc->sc_hub_desc.temp,
3368 			    sizeof(sc->sc_hub_desc.temp),
3369 			    str_ptr);
3370 			break;
3371 
3372 		default:
3373 			err = USB_ERR_IOERROR;
3374 			goto done;
3375 		}
3376 		break;
3377 	case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
3378 		len = 1;
3379 		sc->sc_hub_desc.temp[0] = 0;
3380 		break;
3381 	case C(UR_GET_STATUS, UT_READ_DEVICE):
3382 		len = 2;
3383 		USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED);
3384 		break;
3385 	case C(UR_GET_STATUS, UT_READ_INTERFACE):
3386 	case C(UR_GET_STATUS, UT_READ_ENDPOINT):
3387 		len = 2;
3388 		USETW(sc->sc_hub_desc.stat.wStatus, 0);
3389 		break;
3390 	case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
3391 		if (value >= XHCI_MAX_DEVICES) {
3392 			err = USB_ERR_IOERROR;
3393 			goto done;
3394 		}
3395 		break;
3396 	case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
3397 		if (value != 0 && value != 1) {
3398 			err = USB_ERR_IOERROR;
3399 			goto done;
3400 		}
3401 		sc->sc_conf = value;
3402 		break;
3403 	case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
3404 		break;
3405 	case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
3406 	case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
3407 	case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
3408 		err = USB_ERR_IOERROR;
3409 		goto done;
3410 	case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
3411 		break;
3412 	case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
3413 		break;
3414 		/* Hub requests */
3415 	case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
3416 		break;
3417 	case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
3418 		DPRINTFN(9, "UR_CLEAR_PORT_FEATURE\n");
3419 
3420 		if ((index < 1) ||
3421 		    (index > sc->sc_noport)) {
3422 			err = USB_ERR_IOERROR;
3423 			goto done;
3424 		}
3425 		port = XHCI_PORTSC(index);
3426 
3427 		v = XREAD4(sc, oper, port);
3428 		i = XHCI_PS_PLS_GET(v);
3429 		v &= ~XHCI_PS_CLEAR;
3430 
3431 		switch (value) {
3432 		case UHF_C_BH_PORT_RESET:
3433 			XWRITE4(sc, oper, port, v | XHCI_PS_WRC);
3434 			break;
3435 		case UHF_C_PORT_CONFIG_ERROR:
3436 			XWRITE4(sc, oper, port, v | XHCI_PS_CEC);
3437 			break;
3438 		case UHF_C_PORT_SUSPEND:
3439 		case UHF_C_PORT_LINK_STATE:
3440 			XWRITE4(sc, oper, port, v | XHCI_PS_PLC);
3441 			break;
3442 		case UHF_C_PORT_CONNECTION:
3443 			XWRITE4(sc, oper, port, v | XHCI_PS_CSC);
3444 			break;
3445 		case UHF_C_PORT_ENABLE:
3446 			XWRITE4(sc, oper, port, v | XHCI_PS_PEC);
3447 			break;
3448 		case UHF_C_PORT_OVER_CURRENT:
3449 			XWRITE4(sc, oper, port, v | XHCI_PS_OCC);
3450 			break;
3451 		case UHF_C_PORT_RESET:
3452 			XWRITE4(sc, oper, port, v | XHCI_PS_PRC);
3453 			break;
3454 		case UHF_PORT_ENABLE:
3455 			XWRITE4(sc, oper, port, v | XHCI_PS_PED);
3456 			break;
3457 		case UHF_PORT_POWER:
3458 			XWRITE4(sc, oper, port, v & ~XHCI_PS_PP);
3459 			break;
3460 		case UHF_PORT_INDICATOR:
3461 			XWRITE4(sc, oper, port, v & ~XHCI_PS_PIC_SET(3));
3462 			break;
3463 		case UHF_PORT_SUSPEND:
3464 
3465 			/* U3 -> U15 */
3466 			if (i == 3) {
3467 				XWRITE4(sc, oper, port, v |
3468 				    XHCI_PS_PLS_SET(0xF) | XHCI_PS_LWS);
3469 			}
3470 
3471 			/* wait 20ms for resume sequence to complete */
3472 			usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50);
3473 
3474 			/* U0 */
3475 			XWRITE4(sc, oper, port, v |
3476 			    XHCI_PS_PLS_SET(0) | XHCI_PS_LWS);
3477 			break;
3478 		default:
3479 			err = USB_ERR_IOERROR;
3480 			goto done;
3481 		}
3482 		break;
3483 
3484 	case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
3485 		if ((value & 0xff) != 0) {
3486 			err = USB_ERR_IOERROR;
3487 			goto done;
3488 		}
3489 
3490 		v = XREAD4(sc, capa, XHCI_HCSPARAMS0);
3491 
3492 		sc->sc_hub_desc.hubd = xhci_hubd;
3493 
3494 		sc->sc_hub_desc.hubd.bNbrPorts = sc->sc_noport;
3495 
3496 		if (XHCI_HCS0_PPC(v))
3497 			i = UHD_PWR_INDIVIDUAL;
3498 		else
3499 			i = UHD_PWR_GANGED;
3500 
3501 		if (XHCI_HCS0_PIND(v))
3502 			i |= UHD_PORT_IND;
3503 
3504 		i |= UHD_OC_INDIVIDUAL;
3505 
3506 		USETW(sc->sc_hub_desc.hubd.wHubCharacteristics, i);
3507 
3508 		/* see XHCI section 5.4.9: */
3509 		sc->sc_hub_desc.hubd.bPwrOn2PwrGood = 10;
3510 
3511 		for (j = 1; j <= sc->sc_noport; j++) {
3512 			v = XREAD4(sc, oper, XHCI_PORTSC(j));
3513 			if (v & XHCI_PS_DR) {
3514 				sc->sc_hub_desc.hubd.
3515 				    DeviceRemovable[j / 8] |= 1U << (j % 8);
3516 			}
3517 		}
3518 		len = sc->sc_hub_desc.hubd.bLength;
3519 		break;
3520 
3521 	case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
3522 		len = 16;
3523 		memset(sc->sc_hub_desc.temp, 0, 16);
3524 		break;
3525 
3526 	case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
3527 		DPRINTFN(9, "UR_GET_STATUS i=%d\n", index);
3528 
3529 		if ((index < 1) ||
3530 		    (index > sc->sc_noport)) {
3531 			err = USB_ERR_IOERROR;
3532 			goto done;
3533 		}
3534 
3535 		v = XREAD4(sc, oper, XHCI_PORTSC(index));
3536 
3537 		DPRINTFN(9, "port status=0x%08x\n", v);
3538 
3539 		i = UPS_PORT_LINK_STATE_SET(XHCI_PS_PLS_GET(v));
3540 
3541 		switch (XHCI_PS_SPEED_GET(v)) {
3542 		case 3:
3543 			i |= UPS_HIGH_SPEED;
3544 			break;
3545 		case 2:
3546 			i |= UPS_LOW_SPEED;
3547 			break;
3548 		case 1:
3549 			/* FULL speed */
3550 			break;
3551 		default:
3552 			i |= UPS_OTHER_SPEED;
3553 			break;
3554 		}
3555 
3556 		if (v & XHCI_PS_CCS)
3557 			i |= UPS_CURRENT_CONNECT_STATUS;
3558 		if (v & XHCI_PS_PED)
3559 			i |= UPS_PORT_ENABLED;
3560 		if (v & XHCI_PS_OCA)
3561 			i |= UPS_OVERCURRENT_INDICATOR;
3562 		if (v & XHCI_PS_PR)
3563 			i |= UPS_RESET;
3564 #if 0
3565 		if (v & XHCI_PS_PP)
3566 			/* XXX undefined */
3567 #endif
3568 		USETW(sc->sc_hub_desc.ps.wPortStatus, i);
3569 
3570 		i = 0;
3571 		if (v & XHCI_PS_CSC)
3572 			i |= UPS_C_CONNECT_STATUS;
3573 		if (v & XHCI_PS_PEC)
3574 			i |= UPS_C_PORT_ENABLED;
3575 		if (v & XHCI_PS_OCC)
3576 			i |= UPS_C_OVERCURRENT_INDICATOR;
3577 		if (v & XHCI_PS_WRC)
3578 			i |= UPS_C_BH_PORT_RESET;
3579 		if (v & XHCI_PS_PRC)
3580 			i |= UPS_C_PORT_RESET;
3581 		if (v & XHCI_PS_PLC)
3582 			i |= UPS_C_PORT_LINK_STATE;
3583 		if (v & XHCI_PS_CEC)
3584 			i |= UPS_C_PORT_CONFIG_ERROR;
3585 
3586 		USETW(sc->sc_hub_desc.ps.wPortChange, i);
3587 		len = sizeof(sc->sc_hub_desc.ps);
3588 		break;
3589 
3590 	case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
3591 		err = USB_ERR_IOERROR;
3592 		goto done;
3593 
3594 	case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
3595 		break;
3596 
3597 	case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
3598 
3599 		i = index >> 8;
3600 		index &= 0x00FF;
3601 
3602 		if ((index < 1) ||
3603 		    (index > sc->sc_noport)) {
3604 			err = USB_ERR_IOERROR;
3605 			goto done;
3606 		}
3607 
3608 		port = XHCI_PORTSC(index);
3609 		v = XREAD4(sc, oper, port) & ~XHCI_PS_CLEAR;
3610 
3611 		switch (value) {
3612 		case UHF_PORT_U1_TIMEOUT:
3613 			if (XHCI_PS_SPEED_GET(v) != 4) {
3614 				err = USB_ERR_IOERROR;
3615 				goto done;
3616 			}
3617 			port = XHCI_PORTPMSC(index);
3618 			v = XREAD4(sc, oper, port);
3619 			v &= ~XHCI_PM3_U1TO_SET(0xFF);
3620 			v |= XHCI_PM3_U1TO_SET(i);
3621 			XWRITE4(sc, oper, port, v);
3622 			break;
3623 		case UHF_PORT_U2_TIMEOUT:
3624 			if (XHCI_PS_SPEED_GET(v) != 4) {
3625 				err = USB_ERR_IOERROR;
3626 				goto done;
3627 			}
3628 			port = XHCI_PORTPMSC(index);
3629 			v = XREAD4(sc, oper, port);
3630 			v &= ~XHCI_PM3_U2TO_SET(0xFF);
3631 			v |= XHCI_PM3_U2TO_SET(i);
3632 			XWRITE4(sc, oper, port, v);
3633 			break;
3634 		case UHF_BH_PORT_RESET:
3635 			XWRITE4(sc, oper, port, v | XHCI_PS_WPR);
3636 			break;
3637 		case UHF_PORT_LINK_STATE:
3638 			XWRITE4(sc, oper, port, v |
3639 			    XHCI_PS_PLS_SET(i) | XHCI_PS_LWS);
3640 			/* 4ms settle time */
3641 			usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
3642 			break;
3643 		case UHF_PORT_ENABLE:
3644 			DPRINTFN(3, "set port enable %d\n", index);
3645 			break;
3646 		case UHF_PORT_SUSPEND:
3647 			DPRINTFN(6, "suspend port %u (LPM=%u)\n", index, i);
3648 			j = XHCI_PS_SPEED_GET(v);
3649 			if ((j < 1) || (j > 3)) {
3650 				/* non-supported speed */
3651 				err = USB_ERR_IOERROR;
3652 				goto done;
3653 			}
3654 			XWRITE4(sc, oper, port, v |
3655 			    XHCI_PS_PLS_SET(i ? 2 /* LPM */ : 3) | XHCI_PS_LWS);
3656 			break;
3657 		case UHF_PORT_RESET:
3658 			DPRINTFN(6, "reset port %d\n", index);
3659 			XWRITE4(sc, oper, port, v | XHCI_PS_PR);
3660 			break;
3661 		case UHF_PORT_POWER:
3662 			DPRINTFN(3, "set port power %d\n", index);
3663 			XWRITE4(sc, oper, port, v | XHCI_PS_PP);
3664 			break;
3665 		case UHF_PORT_TEST:
3666 			DPRINTFN(3, "set port test %d\n", index);
3667 			break;
3668 		case UHF_PORT_INDICATOR:
3669 			DPRINTFN(3, "set port indicator %d\n", index);
3670 
3671 			v &= ~XHCI_PS_PIC_SET(3);
3672 			v |= XHCI_PS_PIC_SET(1);
3673 
3674 			XWRITE4(sc, oper, port, v);
3675 			break;
3676 		default:
3677 			err = USB_ERR_IOERROR;
3678 			goto done;
3679 		}
3680 		break;
3681 
3682 	case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
3683 	case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
3684 	case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
3685 	case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
3686 		break;
3687 	default:
3688 		err = USB_ERR_IOERROR;
3689 		goto done;
3690 	}
3691 done:
3692 	*plength = len;
3693 	*pptr = ptr;
3694 	return (err);
3695 }
3696 
3697 static void
3698 xhci_xfer_setup(struct usb_setup_params *parm)
3699 {
3700 	struct usb_page_search page_info;
3701 	struct usb_page_cache *pc;
3702 	struct usb_xfer *xfer;
3703 	void *last_obj;
3704 	uint32_t ntd;
3705 	uint32_t n;
3706 
3707 	xfer = parm->curr_xfer;
3708 
3709 	/*
3710 	 * The proof for the "ntd" formula is illustrated like this:
3711 	 *
3712 	 * +------------------------------------+
3713 	 * |                                    |
3714 	 * |         |remainder ->              |
3715 	 * |   +-----+---+                      |
3716 	 * |   | xxx | x | frm 0                |
3717 	 * |   +-----+---++                     |
3718 	 * |   | xxx | xx | frm 1               |
3719 	 * |   +-----+----+                     |
3720 	 * |            ...                     |
3721 	 * +------------------------------------+
3722 	 *
3723 	 * "xxx" means a completely full USB transfer descriptor
3724 	 *
3725 	 * "x" and "xx" means a short USB packet
3726 	 *
3727 	 * For the remainder of an USB transfer modulo
3728 	 * "max_data_length" we need two USB transfer descriptors.
3729 	 * One to transfer the remaining data and one to finalise with
3730 	 * a zero length packet in case the "force_short_xfer" flag is
3731 	 * set. We only need two USB transfer descriptors in the case
3732 	 * where the transfer length of the first one is a factor of
3733 	 * "max_frame_size". The rest of the needed USB transfer
3734 	 * descriptors is given by the buffer size divided by the
3735 	 * maximum data payload.
3736 	 */
3737 	parm->hc_max_packet_size = 0x400;
3738 	parm->hc_max_packet_count = 16 * 3;
3739 	parm->hc_max_frame_size = XHCI_TD_PAYLOAD_MAX;
3740 
3741 	xfer->flags_int.bdma_enable = 1;
3742 
3743 	usbd_transfer_setup_sub(parm);
3744 
3745 	if (xfer->flags_int.isochronous_xfr) {
3746 		ntd = ((1 * xfer->nframes)
3747 		    + (xfer->max_data_length / xfer->max_hc_frame_size));
3748 	} else if (xfer->flags_int.control_xfr) {
3749 		ntd = ((2 * xfer->nframes) + 1	/* STATUS */
3750 		    + (xfer->max_data_length / xfer->max_hc_frame_size));
3751 	} else {
3752 		ntd = ((2 * xfer->nframes)
3753 		    + (xfer->max_data_length / xfer->max_hc_frame_size));
3754 	}
3755 
3756 alloc_dma_set:
3757 
3758 	if (parm->err)
3759 		return;
3760 
3761 	/*
3762 	 * Allocate queue heads and transfer descriptors
3763 	 */
3764 	last_obj = NULL;
3765 
3766 	if (usbd_transfer_setup_sub_malloc(
3767 	    parm, &pc, sizeof(struct xhci_td),
3768 	    XHCI_TD_ALIGN, ntd)) {
3769 		parm->err = USB_ERR_NOMEM;
3770 		return;
3771 	}
3772 	if (parm->buf) {
3773 		for (n = 0; n != ntd; n++) {
3774 			struct xhci_td *td;
3775 
3776 			usbd_get_page(pc + n, 0, &page_info);
3777 
3778 			td = page_info.buffer;
3779 
3780 			/* init TD */
3781 			td->td_self = page_info.physaddr;
3782 			td->obj_next = last_obj;
3783 			td->page_cache = pc + n;
3784 
3785 			last_obj = td;
3786 
3787 			usb_pc_cpu_flush(pc + n);
3788 		}
3789 	}
3790 	xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj;
3791 
3792 	if (!xfer->flags_int.curr_dma_set) {
3793 		xfer->flags_int.curr_dma_set = 1;
3794 		goto alloc_dma_set;
3795 	}
3796 }
3797 
3798 static uint8_t
3799 xhci_get_endpoint_state(struct usb_device *udev, uint8_t epno)
3800 {
3801 	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
3802 	struct usb_page_search buf_dev;
3803 	struct xhci_hw_dev *hdev;
3804 	struct xhci_dev_ctx *pdev;
3805 	uint32_t temp;
3806 
3807 	MPASS(epno != 0);
3808 
3809 	hdev =	&sc->sc_hw.devs[udev->controller_slot_id];
3810 
3811 	usbd_get_page(&hdev->device_pc, 0, &buf_dev);
3812 	pdev = buf_dev.buffer;
3813 	usb_pc_cpu_invalidate(&hdev->device_pc);
3814 
3815 	temp = xhci_ctx_get_le32(sc, &pdev->ctx_ep[epno - 1].dwEpCtx0);
3816 
3817 	return (XHCI_EPCTX_0_EPSTATE_GET(temp));
3818 }
3819 
3820 static usb_error_t
3821 xhci_configure_reset_endpoint(struct usb_xfer *xfer)
3822 {
3823 	struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus);
3824 	struct usb_page_search buf_inp;
3825 	struct usb_device *udev;
3826 	struct xhci_endpoint_ext *pepext;
3827 	struct usb_endpoint_descriptor *edesc;
3828 	struct usb_page_cache *pcinp;
3829 	usb_error_t err;
3830 	usb_stream_t stream_id;
3831 	uint32_t mask;
3832 	uint8_t index;
3833 	uint8_t epno;
3834 
3835 	pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
3836 	    xfer->endpoint->edesc);
3837 
3838 	udev = xfer->xroot->udev;
3839 	index = udev->controller_slot_id;
3840 
3841 	pcinp = &sc->sc_hw.devs[index].input_pc;
3842 
3843 	usbd_get_page(pcinp, 0, &buf_inp);
3844 
3845 	edesc = xfer->endpoint->edesc;
3846 
3847 	epno = edesc->bEndpointAddress;
3848 	stream_id = xfer->stream_id;
3849 
3850 	if ((edesc->bmAttributes & UE_XFERTYPE) == UE_CONTROL)
3851 		epno |= UE_DIR_IN;
3852 
3853 	epno = XHCI_EPNO2EPID(epno);
3854 
3855  	if (epno == 0)
3856 		return (USB_ERR_NO_PIPE);		/* invalid */
3857 
3858 	XHCI_CMD_LOCK(sc);
3859 
3860 	/* configure endpoint */
3861 
3862 	err = xhci_configure_endpoint_by_xfer(xfer);
3863 
3864 	if (err != 0) {
3865 		XHCI_CMD_UNLOCK(sc);
3866 		return (err);
3867 	}
3868 
3869 	/*
3870 	 * Get the endpoint into the stopped state according to the
3871 	 * endpoint context state diagram in the XHCI specification:
3872 	 */
3873 	switch (xhci_get_endpoint_state(udev, epno)) {
3874 	case XHCI_EPCTX_0_EPSTATE_STOPPED:
3875 		break;
3876 	case XHCI_EPCTX_0_EPSTATE_HALTED:
3877 		err = xhci_cmd_reset_ep(sc, 0, epno, index);
3878 		if (err != 0)
3879 			DPRINTF("Could not reset endpoint %u\n", epno);
3880 		break;
3881 	default:
3882 		err = xhci_cmd_stop_ep(sc, 0, epno, index);
3883 		if (err != 0)
3884 			DPRINTF("Could not stop endpoint %u\n", epno);
3885 		break;
3886 	}
3887 
3888 	err = xhci_cmd_set_tr_dequeue_ptr(sc,
3889 	    (pepext->physaddr + (stream_id * sizeof(struct xhci_trb) *
3890 	    XHCI_MAX_TRANSFERS)) | XHCI_EPCTX_2_DCS_SET(1),
3891 	    stream_id, epno, index);
3892 
3893 	if (err != 0)
3894 		DPRINTF("Could not set dequeue ptr for endpoint %u\n", epno);
3895 
3896 	/*
3897 	 * Get the endpoint into the running state according to the
3898 	 * endpoint context state diagram in the XHCI specification:
3899 	 */
3900 
3901 	mask = (1U << epno);
3902 	xhci_configure_mask(udev, mask | 1U, 0);
3903 
3904 	if (!(sc->sc_hw.devs[index].ep_configured & mask)) {
3905 		sc->sc_hw.devs[index].ep_configured |= mask;
3906 		err = xhci_cmd_configure_ep(sc, buf_inp.physaddr, 0, index);
3907 	} else {
3908 		err = xhci_cmd_evaluate_ctx(sc, buf_inp.physaddr, index);
3909 	}
3910 
3911 	if (err != 0) {
3912 		DPRINTF("Could not configure "
3913 		    "endpoint %u at slot %u.\n", epno, index);
3914 	}
3915 	XHCI_CMD_UNLOCK(sc);
3916 
3917 	return (0);
3918 }
3919 
3920 static void
3921 xhci_xfer_unsetup(struct usb_xfer *xfer)
3922 {
3923 	return;
3924 }
3925 
3926 static void
3927 xhci_start_dma_delay(struct usb_xfer *xfer)
3928 {
3929 	struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus);
3930 
3931 	/* put transfer on interrupt queue (again) */
3932 	usbd_transfer_enqueue(&sc->sc_bus.intr_q, xfer);
3933 
3934 	(void)usb_proc_msignal(USB_BUS_CONTROL_XFER_PROC(&sc->sc_bus),
3935 	    &sc->sc_config_msg[0], &sc->sc_config_msg[1]);
3936 }
3937 
3938 static void
3939 xhci_configure_msg(struct usb_proc_msg *pm)
3940 {
3941 	struct xhci_softc *sc;
3942 	struct xhci_endpoint_ext *pepext;
3943 	struct usb_xfer *xfer;
3944 
3945 	sc = XHCI_BUS2SC(((struct usb_bus_msg *)pm)->bus);
3946 
3947 restart:
3948 	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3949 		pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
3950 		    xfer->endpoint->edesc);
3951 
3952 		if ((pepext->trb_halted != 0) ||
3953 		    (pepext->trb_running == 0)) {
3954 			uint16_t i;
3955 
3956 			/* clear halted and running */
3957 			pepext->trb_halted = 0;
3958 			pepext->trb_running = 0;
3959 
3960 			/* nuke remaining buffered transfers */
3961 
3962 			for (i = 0; i != (XHCI_MAX_TRANSFERS *
3963 			    XHCI_MAX_STREAMS); i++) {
3964 				/*
3965 				 * NOTE: We need to use the timeout
3966 				 * error code here else existing
3967 				 * isochronous clients can get
3968 				 * confused:
3969 				 */
3970 				if (pepext->xfer[i] != NULL) {
3971 					xhci_device_done(pepext->xfer[i],
3972 					    USB_ERR_TIMEOUT);
3973 				}
3974 			}
3975 
3976 			/*
3977 			 * NOTE: The USB transfer cannot vanish in
3978 			 * this state!
3979 			 */
3980 
3981 			USB_BUS_UNLOCK(&sc->sc_bus);
3982 
3983 			xhci_configure_reset_endpoint(xfer);
3984 
3985 			USB_BUS_LOCK(&sc->sc_bus);
3986 
3987 			/* check if halted is still cleared */
3988 			if (pepext->trb_halted == 0) {
3989 				pepext->trb_running = 1;
3990 				memset(pepext->trb_index, 0,
3991 				    sizeof(pepext->trb_index));
3992 			}
3993 			goto restart;
3994 		}
3995 
3996 		if (xfer->flags_int.did_dma_delay) {
3997 			/* remove transfer from interrupt queue (again) */
3998 			usbd_transfer_dequeue(xfer);
3999 
4000 			/* we are finally done */
4001 			usb_dma_delay_done_cb(xfer);
4002 
4003 			/* queue changed - restart */
4004 			goto restart;
4005 		}
4006 	}
4007 
4008 	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
4009 		/* try to insert xfer on HW queue */
4010 		xhci_transfer_insert(xfer);
4011 
4012 		/* try to multi buffer */
4013 		xhci_device_generic_multi_enter(xfer->endpoint,
4014 		    xfer->stream_id, NULL);
4015 	}
4016 }
4017 
4018 static void
4019 xhci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
4020     struct usb_endpoint *ep)
4021 {
4022 	struct xhci_endpoint_ext *pepext;
4023 	struct xhci_softc *sc;
4024 	uint8_t index;
4025 	uint8_t epno;
4026 
4027 	DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d\n",
4028 	    ep, udev->address, edesc->bEndpointAddress, udev->flags.usb_mode);
4029 
4030 	if (udev->parent_hub == NULL) {
4031 		/* root HUB has special endpoint handling */
4032 		return;
4033 	}
4034 
4035 	ep->methods = &xhci_device_generic_methods;
4036 
4037 	pepext = xhci_get_endpoint_ext(udev, edesc);
4038 
4039 	USB_BUS_LOCK(udev->bus);
4040 	pepext->trb_halted = 1;
4041 	pepext->trb_running = 0;
4042 
4043 	/*
4044 	 * When doing an alternate setting, except for control
4045 	 * endpoints, we need to re-configure the XHCI endpoint
4046 	 * context:
4047 	 */
4048 	if ((edesc->bEndpointAddress & UE_ADDR) != 0) {
4049 		sc = XHCI_BUS2SC(udev->bus);
4050 		index = udev->controller_slot_id;
4051 		epno = XHCI_EPNO2EPID(edesc->bEndpointAddress);
4052 		sc->sc_hw.devs[index].ep_configured &= ~(1U << epno);
4053 	}
4054 	USB_BUS_UNLOCK(udev->bus);
4055 }
4056 
4057 static void
4058 xhci_ep_uninit(struct usb_device *udev, struct usb_endpoint *ep)
4059 {
4060 
4061 }
4062 
4063 static void
4064 xhci_ep_clear_stall(struct usb_device *udev, struct usb_endpoint *ep)
4065 {
4066 	struct xhci_endpoint_ext *pepext;
4067 
4068 	DPRINTF("\n");
4069 
4070 	if (udev->flags.usb_mode != USB_MODE_HOST) {
4071 		/* not supported */
4072 		return;
4073 	}
4074 	if (udev->parent_hub == NULL) {
4075 		/* root HUB has special endpoint handling */
4076 		return;
4077 	}
4078 
4079 	pepext = xhci_get_endpoint_ext(udev, ep->edesc);
4080 
4081 	USB_BUS_LOCK(udev->bus);
4082 	pepext->trb_halted = 1;
4083 	pepext->trb_running = 0;
4084 	USB_BUS_UNLOCK(udev->bus);
4085 }
4086 
4087 static usb_error_t
4088 xhci_device_init(struct usb_device *udev)
4089 {
4090 	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
4091 	usb_error_t err;
4092 	uint8_t temp;
4093 
4094 	/* no init for root HUB */
4095 	if (udev->parent_hub == NULL)
4096 		return (0);
4097 
4098 	XHCI_CMD_LOCK(sc);
4099 
4100 	/* set invalid default */
4101 
4102 	udev->controller_slot_id = sc->sc_noslot + 1;
4103 
4104 	/* try to get a new slot ID from the XHCI */
4105 
4106 	err = xhci_cmd_enable_slot(sc, &temp);
4107 
4108 	if (err) {
4109 		XHCI_CMD_UNLOCK(sc);
4110 		return (err);
4111 	}
4112 
4113 	if (temp > sc->sc_noslot) {
4114 		XHCI_CMD_UNLOCK(sc);
4115 		return (USB_ERR_BAD_ADDRESS);
4116 	}
4117 
4118 	if (sc->sc_hw.devs[temp].state != XHCI_ST_DISABLED) {
4119 		DPRINTF("slot %u already allocated.\n", temp);
4120 		XHCI_CMD_UNLOCK(sc);
4121 		return (USB_ERR_BAD_ADDRESS);
4122 	}
4123 
4124 	/* store slot ID for later reference */
4125 
4126 	udev->controller_slot_id = temp;
4127 
4128 	/* reset data structure */
4129 
4130 	memset(&sc->sc_hw.devs[temp], 0, sizeof(sc->sc_hw.devs[0]));
4131 
4132 	/* set mark slot allocated */
4133 
4134 	sc->sc_hw.devs[temp].state = XHCI_ST_ENABLED;
4135 
4136 	err = xhci_alloc_device_ext(udev);
4137 
4138 	XHCI_CMD_UNLOCK(sc);
4139 
4140 	/* get device into default state */
4141 
4142 	if (err == 0)
4143 		err = xhci_set_address(udev, NULL, 0);
4144 
4145 	return (err);
4146 }
4147 
4148 static void
4149 xhci_device_uninit(struct usb_device *udev)
4150 {
4151 	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
4152 	uint8_t index;
4153 
4154 	/* no init for root HUB */
4155 	if (udev->parent_hub == NULL)
4156 		return;
4157 
4158 	XHCI_CMD_LOCK(sc);
4159 
4160 	index = udev->controller_slot_id;
4161 
4162 	if (index <= sc->sc_noslot) {
4163 		xhci_cmd_disable_slot(sc, index);
4164 		sc->sc_hw.devs[index].state = XHCI_ST_DISABLED;
4165 
4166 		/* free device extension */
4167 		xhci_free_device_ext(udev);
4168 	}
4169 
4170 	XHCI_CMD_UNLOCK(sc);
4171 }
4172 
4173 static void
4174 xhci_get_dma_delay(struct usb_device *udev, uint32_t *pus)
4175 {
4176 	/*
4177 	 * Wait until the hardware has finished any possible use of
4178 	 * the transfer descriptor(s)
4179 	 */
4180 	*pus = 2048;			/* microseconds */
4181 }
4182 
4183 static void
4184 xhci_device_resume(struct usb_device *udev)
4185 {
4186 	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
4187 	uint8_t index;
4188 	uint8_t n;
4189 	uint8_t p;
4190 
4191 	DPRINTF("\n");
4192 
4193 	/* check for root HUB */
4194 	if (udev->parent_hub == NULL)
4195 		return;
4196 
4197 	index = udev->controller_slot_id;
4198 
4199 	XHCI_CMD_LOCK(sc);
4200 
4201 	/* blindly resume all endpoints */
4202 
4203 	USB_BUS_LOCK(udev->bus);
4204 
4205 	for (n = 1; n != XHCI_MAX_ENDPOINTS; n++) {
4206 		for (p = 0; p != XHCI_MAX_STREAMS; p++) {
4207 			XWRITE4(sc, door, XHCI_DOORBELL(index),
4208 			    n | XHCI_DB_SID_SET(p));
4209 		}
4210 	}
4211 
4212 	USB_BUS_UNLOCK(udev->bus);
4213 
4214 	XHCI_CMD_UNLOCK(sc);
4215 }
4216 
4217 static void
4218 xhci_device_suspend(struct usb_device *udev)
4219 {
4220 	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
4221 	uint8_t index;
4222 	uint8_t n;
4223 	usb_error_t err;
4224 
4225 	DPRINTF("\n");
4226 
4227 	/* check for root HUB */
4228 	if (udev->parent_hub == NULL)
4229 		return;
4230 
4231 	index = udev->controller_slot_id;
4232 
4233 	XHCI_CMD_LOCK(sc);
4234 
4235 	/* blindly suspend all endpoints */
4236 
4237 	for (n = 1; n != XHCI_MAX_ENDPOINTS; n++) {
4238 		err = xhci_cmd_stop_ep(sc, 1, n, index);
4239 		if (err != 0) {
4240 			DPRINTF("Failed to suspend endpoint "
4241 			    "%u on slot %u (ignored).\n", n, index);
4242 		}
4243 	}
4244 
4245 	XHCI_CMD_UNLOCK(sc);
4246 }
4247 
4248 static void
4249 xhci_set_hw_power(struct usb_bus *bus)
4250 {
4251 	DPRINTF("\n");
4252 }
4253 
4254 static void
4255 xhci_device_state_change(struct usb_device *udev)
4256 {
4257 	struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
4258 	struct usb_page_search buf_inp;
4259 	usb_error_t err;
4260 	uint8_t index;
4261 
4262 	/* check for root HUB */
4263 	if (udev->parent_hub == NULL)
4264 		return;
4265 
4266 	index = udev->controller_slot_id;
4267 
4268 	DPRINTF("\n");
4269 
4270 	if (usb_get_device_state(udev) == USB_STATE_CONFIGURED) {
4271 		err = uhub_query_info(udev, &sc->sc_hw.devs[index].nports,
4272 		    &sc->sc_hw.devs[index].tt);
4273 		if (err != 0)
4274 			sc->sc_hw.devs[index].nports = 0;
4275 	}
4276 
4277 	XHCI_CMD_LOCK(sc);
4278 
4279 	switch (usb_get_device_state(udev)) {
4280 	case USB_STATE_POWERED:
4281 		if (sc->sc_hw.devs[index].state == XHCI_ST_DEFAULT)
4282 			break;
4283 
4284 		/* set default state */
4285 		sc->sc_hw.devs[index].state = XHCI_ST_DEFAULT;
4286 		sc->sc_hw.devs[index].ep_configured = 3U;
4287 
4288 		/* reset number of contexts */
4289 		sc->sc_hw.devs[index].context_num = 0;
4290 
4291 		err = xhci_cmd_reset_dev(sc, index);
4292 
4293 		if (err != 0) {
4294 			DPRINTF("Device reset failed "
4295 			    "for slot %u.\n", index);
4296 		}
4297 		break;
4298 
4299 	case USB_STATE_ADDRESSED:
4300 		if (sc->sc_hw.devs[index].state == XHCI_ST_ADDRESSED)
4301 			break;
4302 
4303 		sc->sc_hw.devs[index].state = XHCI_ST_ADDRESSED;
4304 		sc->sc_hw.devs[index].ep_configured = 3U;
4305 
4306 		/* set configure mask to slot only */
4307 		xhci_configure_mask(udev, 1, 0);
4308 
4309 		/* deconfigure all endpoints, except EP0 */
4310 		err = xhci_cmd_configure_ep(sc, 0, 1, index);
4311 
4312 		if (err) {
4313 			DPRINTF("Failed to deconfigure "
4314 			    "slot %u.\n", index);
4315 		}
4316 		break;
4317 
4318 	case USB_STATE_CONFIGURED:
4319 		if (sc->sc_hw.devs[index].state == XHCI_ST_CONFIGURED) {
4320 			/* deconfigure all endpoints, except EP0 */
4321 			err = xhci_cmd_configure_ep(sc, 0, 1, index);
4322 
4323 			if (err) {
4324 				DPRINTF("Failed to deconfigure "
4325 				    "slot %u.\n", index);
4326 			}
4327 		}
4328 
4329 		/* set configured state */
4330 		sc->sc_hw.devs[index].state = XHCI_ST_CONFIGURED;
4331 		sc->sc_hw.devs[index].ep_configured = 3U;
4332 
4333 		/* reset number of contexts */
4334 		sc->sc_hw.devs[index].context_num = 0;
4335 
4336 		usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp);
4337 
4338 		xhci_configure_mask(udev, 3, 0);
4339 
4340 		err = xhci_configure_device(udev);
4341 		if (err != 0) {
4342 			DPRINTF("Could not configure device "
4343 			    "at slot %u.\n", index);
4344 		}
4345 
4346 		err = xhci_cmd_evaluate_ctx(sc, buf_inp.physaddr, index);
4347 		if (err != 0) {
4348 			DPRINTF("Could not evaluate device "
4349 			    "context at slot %u.\n", index);
4350 		}
4351 		break;
4352 
4353 	default:
4354 		break;
4355 	}
4356 	XHCI_CMD_UNLOCK(sc);
4357 }
4358 
4359 static usb_error_t
4360 xhci_set_endpoint_mode(struct usb_device *udev, struct usb_endpoint *ep,
4361     uint8_t ep_mode)
4362 {
4363 	switch (ep_mode) {
4364 	case USB_EP_MODE_DEFAULT:
4365 		return (0);
4366 	case USB_EP_MODE_STREAMS:
4367 		if (xhcistreams == 0 ||
4368 		    (ep->edesc->bmAttributes & UE_XFERTYPE) != UE_BULK ||
4369 		    udev->speed != USB_SPEED_SUPER)
4370 			return (USB_ERR_INVAL);
4371 		return (0);
4372 	default:
4373 		return (USB_ERR_INVAL);
4374 	}
4375 }
4376 
4377 static const struct usb_bus_methods xhci_bus_methods = {
4378 	.endpoint_init = xhci_ep_init,
4379 	.endpoint_uninit = xhci_ep_uninit,
4380 	.xfer_setup = xhci_xfer_setup,
4381 	.xfer_unsetup = xhci_xfer_unsetup,
4382 	.get_dma_delay = xhci_get_dma_delay,
4383 	.device_init = xhci_device_init,
4384 	.device_uninit = xhci_device_uninit,
4385 	.device_resume = xhci_device_resume,
4386 	.device_suspend = xhci_device_suspend,
4387 	.set_hw_power = xhci_set_hw_power,
4388 	.roothub_exec = xhci_roothub_exec,
4389 	.xfer_poll = xhci_do_poll,
4390 	.start_dma_delay = xhci_start_dma_delay,
4391 	.set_address = xhci_set_address,
4392 	.clear_stall = xhci_ep_clear_stall,
4393 	.device_state_change = xhci_device_state_change,
4394 	.set_hw_power_sleep = xhci_set_hw_power_sleep,
4395 	.set_endpoint_mode = xhci_set_endpoint_mode,
4396 };
4397