xref: /freebsd/sys/dev/usb/controller/ehci.c (revision d4ae33f0721c1b170fe37d97e395228ffcfb3f80)
1 /* $FreeBSD$ */
2 /*-
3  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
4  * Copyright (c) 2004 The NetBSD Foundation, Inc. All rights reserved.
5  * Copyright (c) 2004 Lennart Augustsson. All rights reserved.
6  * Copyright (c) 2004 Charles M. Hannum. All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 /*
31  * USB Enhanced Host Controller Driver, a.k.a. USB 2.0 controller.
32  *
33  * The EHCI 0.96 spec can be found at
34  * http://developer.intel.com/technology/usb/download/ehci-r096.pdf
35  * The EHCI 1.0 spec can be found at
36  * http://developer.intel.com/technology/usb/download/ehci-r10.pdf
37  * and the USB 2.0 spec at
38  * http://www.usb.org/developers/docs/usb_20.zip
39  *
40  */
41 
42 /*
43  * TODO:
44  * 1) command failures are not recovered correctly
45  */
46 
47 #ifdef USB_GLOBAL_INCLUDE_FILE
48 #include USB_GLOBAL_INCLUDE_FILE
49 #else
50 #include <sys/stdint.h>
51 #include <sys/stddef.h>
52 #include <sys/param.h>
53 #include <sys/queue.h>
54 #include <sys/types.h>
55 #include <sys/systm.h>
56 #include <sys/kernel.h>
57 #include <sys/bus.h>
58 #include <sys/module.h>
59 #include <sys/lock.h>
60 #include <sys/mutex.h>
61 #include <sys/condvar.h>
62 #include <sys/sysctl.h>
63 #include <sys/sx.h>
64 #include <sys/unistd.h>
65 #include <sys/callout.h>
66 #include <sys/malloc.h>
67 #include <sys/priv.h>
68 
69 #include <dev/usb/usb.h>
70 #include <dev/usb/usbdi.h>
71 
72 #define	USB_DEBUG_VAR ehcidebug
73 
74 #include <dev/usb/usb_core.h>
75 #include <dev/usb/usb_debug.h>
76 #include <dev/usb/usb_busdma.h>
77 #include <dev/usb/usb_process.h>
78 #include <dev/usb/usb_transfer.h>
79 #include <dev/usb/usb_device.h>
80 #include <dev/usb/usb_hub.h>
81 #include <dev/usb/usb_util.h>
82 
83 #include <dev/usb/usb_controller.h>
84 #include <dev/usb/usb_bus.h>
85 #endif			/* USB_GLOBAL_INCLUDE_FILE */
86 
87 #include <dev/usb/controller/ehci.h>
88 #include <dev/usb/controller/ehcireg.h>
89 
90 #define	EHCI_BUS2SC(bus) \
91    ((ehci_softc_t *)(((uint8_t *)(bus)) - \
92     ((uint8_t *)&(((ehci_softc_t *)0)->sc_bus))))
93 
94 #ifdef USB_DEBUG
95 static int ehcidebug = 0;
96 static int ehcinohighspeed = 0;
97 static int ehciiaadbug = 0;
98 static int ehcilostintrbug = 0;
99 
100 static SYSCTL_NODE(_hw_usb, OID_AUTO, ehci, CTLFLAG_RW, 0, "USB ehci");
101 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_TUN,
102     &ehcidebug, 0, "Debug level");
103 TUNABLE_INT("hw.usb.ehci.debug", &ehcidebug);
104 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, no_hs, CTLFLAG_RW | CTLFLAG_TUN,
105     &ehcinohighspeed, 0, "Disable High Speed USB");
106 TUNABLE_INT("hw.usb.ehci.no_hs", &ehcinohighspeed);
107 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, iaadbug, CTLFLAG_RW | CTLFLAG_TUN,
108     &ehciiaadbug, 0, "Enable doorbell bug workaround");
109 TUNABLE_INT("hw.usb.ehci.iaadbug", &ehciiaadbug);
110 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, lostintrbug, CTLFLAG_RW | CTLFLAG_TUN,
111     &ehcilostintrbug, 0, "Enable lost interrupt bug workaround");
112 TUNABLE_INT("hw.usb.ehci.lostintrbug", &ehcilostintrbug);
113 
114 
115 static void ehci_dump_regs(ehci_softc_t *sc);
116 static void ehci_dump_sqh(ehci_softc_t *sc, ehci_qh_t *sqh);
117 
118 #endif
119 
120 #define	EHCI_INTR_ENDPT 1
121 
122 extern struct usb_bus_methods ehci_bus_methods;
123 extern struct usb_pipe_methods ehci_device_bulk_methods;
124 extern struct usb_pipe_methods ehci_device_ctrl_methods;
125 extern struct usb_pipe_methods ehci_device_intr_methods;
126 extern struct usb_pipe_methods ehci_device_isoc_fs_methods;
127 extern struct usb_pipe_methods ehci_device_isoc_hs_methods;
128 
129 static void ehci_do_poll(struct usb_bus *);
130 static void ehci_device_done(struct usb_xfer *, usb_error_t);
131 static uint8_t ehci_check_transfer(struct usb_xfer *);
132 static void ehci_timeout(void *);
133 static void ehci_poll_timeout(void *);
134 
135 static void ehci_root_intr(ehci_softc_t *sc);
136 
137 struct ehci_std_temp {
138 	ehci_softc_t *sc;
139 	struct usb_page_cache *pc;
140 	ehci_qtd_t *td;
141 	ehci_qtd_t *td_next;
142 	uint32_t average;
143 	uint32_t qtd_status;
144 	uint32_t len;
145 	uint16_t max_frame_size;
146 	uint8_t	shortpkt;
147 	uint8_t	auto_data_toggle;
148 	uint8_t	setup_alt_next;
149 	uint8_t	last_frame;
150 };
151 
152 void
153 ehci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb)
154 {
155 	ehci_softc_t *sc = EHCI_BUS2SC(bus);
156 	uint32_t i;
157 
158 	cb(bus, &sc->sc_hw.pframes_pc, &sc->sc_hw.pframes_pg,
159 	    sizeof(uint32_t) * EHCI_FRAMELIST_COUNT, EHCI_FRAMELIST_ALIGN);
160 
161 	cb(bus, &sc->sc_hw.terminate_pc, &sc->sc_hw.terminate_pg,
162 	    sizeof(struct ehci_qh_sub), EHCI_QH_ALIGN);
163 
164 	cb(bus, &sc->sc_hw.async_start_pc, &sc->sc_hw.async_start_pg,
165 	    sizeof(ehci_qh_t), EHCI_QH_ALIGN);
166 
167 	for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
168 		cb(bus, sc->sc_hw.intr_start_pc + i,
169 		    sc->sc_hw.intr_start_pg + i,
170 		    sizeof(ehci_qh_t), EHCI_QH_ALIGN);
171 	}
172 
173 	for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
174 		cb(bus, sc->sc_hw.isoc_hs_start_pc + i,
175 		    sc->sc_hw.isoc_hs_start_pg + i,
176 		    sizeof(ehci_itd_t), EHCI_ITD_ALIGN);
177 	}
178 
179 	for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
180 		cb(bus, sc->sc_hw.isoc_fs_start_pc + i,
181 		    sc->sc_hw.isoc_fs_start_pg + i,
182 		    sizeof(ehci_sitd_t), EHCI_SITD_ALIGN);
183 	}
184 }
185 
186 usb_error_t
187 ehci_reset(ehci_softc_t *sc)
188 {
189 	uint32_t hcr;
190 	int i;
191 
192 	EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
193 	for (i = 0; i < 100; i++) {
194 		usb_pause_mtx(NULL, hz / 128);
195 		hcr = EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_HCRESET;
196 		if (!hcr) {
197 			if (sc->sc_flags & (EHCI_SCFLG_SETMODE | EHCI_SCFLG_BIGEMMIO)) {
198 				/*
199 				 * Force USBMODE as requested.  Controllers
200 				 * may have multiple operating modes.
201 				 */
202 				uint32_t usbmode = EOREAD4(sc, EHCI_USBMODE);
203 				if (sc->sc_flags & EHCI_SCFLG_SETMODE) {
204 					usbmode = (usbmode &~ EHCI_UM_CM) | EHCI_UM_CM_HOST;
205 					device_printf(sc->sc_bus.bdev,
206 					    "set host controller mode\n");
207 				}
208 				if (sc->sc_flags & EHCI_SCFLG_BIGEMMIO) {
209 					usbmode = (usbmode &~ EHCI_UM_ES) | EHCI_UM_ES_BE;
210 					device_printf(sc->sc_bus.bdev,
211 					    "set big-endian mode\n");
212 				}
213 				EOWRITE4(sc,  EHCI_USBMODE, usbmode);
214 			}
215 			return (0);
216 		}
217 	}
218 	device_printf(sc->sc_bus.bdev, "Reset timeout\n");
219 	return (USB_ERR_IOERROR);
220 }
221 
222 static usb_error_t
223 ehci_hcreset(ehci_softc_t *sc)
224 {
225 	uint32_t hcr;
226 	int i;
227 
228 	EOWRITE4(sc, EHCI_USBCMD, 0);	/* Halt controller */
229 	for (i = 0; i < 100; i++) {
230 		usb_pause_mtx(NULL, hz / 128);
231 		hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
232 		if (hcr)
233 			break;
234 	}
235 	if (!hcr)
236 		/*
237                  * Fall through and try reset anyway even though
238                  * Table 2-9 in the EHCI spec says this will result
239                  * in undefined behavior.
240                  */
241 		device_printf(sc->sc_bus.bdev, "stop timeout\n");
242 
243 	return (ehci_reset(sc));
244 }
245 
246 static int
247 ehci_init_sub(struct ehci_softc *sc)
248 {
249 	struct usb_page_search buf_res;
250 	uint32_t cparams;
251   	uint32_t hcr;
252 	uint8_t i;
253 
254 	cparams = EREAD4(sc, EHCI_HCCPARAMS);
255 
256 	DPRINTF("cparams=0x%x\n", cparams);
257 
258 	if (EHCI_HCC_64BIT(cparams)) {
259 		DPRINTF("HCC uses 64-bit structures\n");
260 
261 		/* MUST clear segment register if 64 bit capable */
262 		EOWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
263 	}
264 
265 	usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
266 	EOWRITE4(sc, EHCI_PERIODICLISTBASE, buf_res.physaddr);
267 
268 	usbd_get_page(&sc->sc_hw.async_start_pc, 0, &buf_res);
269 	EOWRITE4(sc, EHCI_ASYNCLISTADDR, buf_res.physaddr | EHCI_LINK_QH);
270 
271 	/* enable interrupts */
272 	EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
273 
274 	/* turn on controller */
275 	EOWRITE4(sc, EHCI_USBCMD,
276 	    EHCI_CMD_ITC_1 |		/* 1 microframes interrupt delay */
277 	    (EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_FLS_M) |
278 	    EHCI_CMD_ASE |
279 	    EHCI_CMD_PSE |
280 	    EHCI_CMD_RS);
281 
282 	/* Take over port ownership */
283 	EOWRITE4(sc, EHCI_CONFIGFLAG, EHCI_CONF_CF);
284 
285 	for (i = 0; i < 100; i++) {
286 		usb_pause_mtx(NULL, hz / 128);
287 		hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
288 		if (!hcr) {
289 			break;
290 		}
291 	}
292 	if (hcr) {
293 		device_printf(sc->sc_bus.bdev, "Run timeout\n");
294 		return (USB_ERR_IOERROR);
295 	}
296 	return (USB_ERR_NORMAL_COMPLETION);
297 }
298 
299 usb_error_t
300 ehci_init(ehci_softc_t *sc)
301 {
302 	struct usb_page_search buf_res;
303 	uint32_t version;
304 	uint32_t sparams;
305 	uint16_t i;
306 	uint16_t x;
307 	uint16_t y;
308 	uint16_t bit;
309 	usb_error_t err = 0;
310 
311 	DPRINTF("start\n");
312 
313 	usb_callout_init_mtx(&sc->sc_tmo_pcd, &sc->sc_bus.bus_mtx, 0);
314 	usb_callout_init_mtx(&sc->sc_tmo_poll, &sc->sc_bus.bus_mtx, 0);
315 
316 	sc->sc_offs = EHCI_CAPLENGTH(EREAD4(sc, EHCI_CAPLEN_HCIVERSION));
317 
318 #ifdef USB_DEBUG
319 	if (ehciiaadbug)
320 		sc->sc_flags |= EHCI_SCFLG_IAADBUG;
321 	if (ehcilostintrbug)
322 		sc->sc_flags |= EHCI_SCFLG_LOSTINTRBUG;
323 	if (ehcidebug > 2) {
324 		ehci_dump_regs(sc);
325 	}
326 #endif
327 
328 	version = EHCI_HCIVERSION(EREAD4(sc, EHCI_CAPLEN_HCIVERSION));
329 	device_printf(sc->sc_bus.bdev, "EHCI version %x.%x\n",
330 	    version >> 8, version & 0xff);
331 
332 	sparams = EREAD4(sc, EHCI_HCSPARAMS);
333 	DPRINTF("sparams=0x%x\n", sparams);
334 
335 	sc->sc_noport = EHCI_HCS_N_PORTS(sparams);
336 	sc->sc_bus.usbrev = USB_REV_2_0;
337 
338 	if (!(sc->sc_flags & EHCI_SCFLG_DONTRESET)) {
339 		/* Reset the controller */
340 		DPRINTF("%s: resetting\n",
341 		    device_get_nameunit(sc->sc_bus.bdev));
342 
343 		err = ehci_hcreset(sc);
344 		if (err) {
345 			device_printf(sc->sc_bus.bdev, "reset timeout\n");
346 			return (err);
347 		}
348 	}
349 
350 	/*
351 	 * use current frame-list-size selection 0: 1024*4 bytes 1:  512*4
352 	 * bytes 2:  256*4 bytes 3:      unknown
353 	 */
354 	if (EHCI_CMD_FLS(EOREAD4(sc, EHCI_USBCMD)) == 3) {
355 		device_printf(sc->sc_bus.bdev, "invalid frame-list-size\n");
356 		return (USB_ERR_IOERROR);
357 	}
358 	/* set up the bus struct */
359 	sc->sc_bus.methods = &ehci_bus_methods;
360 
361 	sc->sc_eintrs = EHCI_NORMAL_INTRS;
362 
363 	if (1) {
364 		struct ehci_qh_sub *qh;
365 
366 		usbd_get_page(&sc->sc_hw.terminate_pc, 0, &buf_res);
367 
368 		qh = buf_res.buffer;
369 
370 		sc->sc_terminate_self = htohc32(sc, buf_res.physaddr);
371 
372 		/* init terminate TD */
373 		qh->qtd_next =
374 		    htohc32(sc, EHCI_LINK_TERMINATE);
375 		qh->qtd_altnext =
376 		    htohc32(sc, EHCI_LINK_TERMINATE);
377 		qh->qtd_status =
378 		    htohc32(sc, EHCI_QTD_HALTED);
379 	}
380 
381 	for (i = 0; i < EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
382 		ehci_qh_t *qh;
383 
384 		usbd_get_page(sc->sc_hw.intr_start_pc + i, 0, &buf_res);
385 
386 		qh = buf_res.buffer;
387 
388 		/* initialize page cache pointer */
389 
390 		qh->page_cache = sc->sc_hw.intr_start_pc + i;
391 
392 		/* store a pointer to queue head */
393 
394 		sc->sc_intr_p_last[i] = qh;
395 
396 		qh->qh_self =
397 		    htohc32(sc, buf_res.physaddr) |
398 		    htohc32(sc, EHCI_LINK_QH);
399 
400 		qh->qh_endp =
401 		    htohc32(sc, EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH));
402 		qh->qh_endphub =
403 		    htohc32(sc, EHCI_QH_SET_MULT(1));
404 		qh->qh_curqtd = 0;
405 
406 		qh->qh_qtd.qtd_next =
407 		    htohc32(sc, EHCI_LINK_TERMINATE);
408 		qh->qh_qtd.qtd_altnext =
409 		    htohc32(sc, EHCI_LINK_TERMINATE);
410 		qh->qh_qtd.qtd_status =
411 		    htohc32(sc, EHCI_QTD_HALTED);
412 	}
413 
414 	/*
415 	 * the QHs are arranged to give poll intervals that are
416 	 * powers of 2 times 1ms
417 	 */
418 	bit = EHCI_VIRTUAL_FRAMELIST_COUNT / 2;
419 	while (bit) {
420 		x = bit;
421 		while (x & bit) {
422 			ehci_qh_t *qh_x;
423 			ehci_qh_t *qh_y;
424 
425 			y = (x ^ bit) | (bit / 2);
426 
427 			qh_x = sc->sc_intr_p_last[x];
428 			qh_y = sc->sc_intr_p_last[y];
429 
430 			/*
431 			 * the next QH has half the poll interval
432 			 */
433 			qh_x->qh_link = qh_y->qh_self;
434 
435 			x++;
436 		}
437 		bit >>= 1;
438 	}
439 
440 	if (1) {
441 		ehci_qh_t *qh;
442 
443 		qh = sc->sc_intr_p_last[0];
444 
445 		/* the last (1ms) QH terminates */
446 		qh->qh_link = htohc32(sc, EHCI_LINK_TERMINATE);
447 	}
448 	for (i = 0; i < EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
449 		ehci_sitd_t *sitd;
450 		ehci_itd_t *itd;
451 
452 		usbd_get_page(sc->sc_hw.isoc_fs_start_pc + i, 0, &buf_res);
453 
454 		sitd = buf_res.buffer;
455 
456 		/* initialize page cache pointer */
457 
458 		sitd->page_cache = sc->sc_hw.isoc_fs_start_pc + i;
459 
460 		/* store a pointer to the transfer descriptor */
461 
462 		sc->sc_isoc_fs_p_last[i] = sitd;
463 
464 		/* initialize full speed isochronous */
465 
466 		sitd->sitd_self =
467 		    htohc32(sc, buf_res.physaddr) |
468 		    htohc32(sc, EHCI_LINK_SITD);
469 
470 		sitd->sitd_back =
471 		    htohc32(sc, EHCI_LINK_TERMINATE);
472 
473 		sitd->sitd_next =
474 		    sc->sc_intr_p_last[i | (EHCI_VIRTUAL_FRAMELIST_COUNT / 2)]->qh_self;
475 
476 
477 		usbd_get_page(sc->sc_hw.isoc_hs_start_pc + i, 0, &buf_res);
478 
479 		itd = buf_res.buffer;
480 
481 		/* initialize page cache pointer */
482 
483 		itd->page_cache = sc->sc_hw.isoc_hs_start_pc + i;
484 
485 		/* store a pointer to the transfer descriptor */
486 
487 		sc->sc_isoc_hs_p_last[i] = itd;
488 
489 		/* initialize high speed isochronous */
490 
491 		itd->itd_self =
492 		    htohc32(sc, buf_res.physaddr) |
493 		    htohc32(sc, EHCI_LINK_ITD);
494 
495 		itd->itd_next =
496 		    sitd->sitd_self;
497 	}
498 
499 	usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
500 
501 	if (1) {
502 		uint32_t *pframes;
503 
504 		pframes = buf_res.buffer;
505 
506 		/*
507 		 * execution order:
508 		 * pframes -> high speed isochronous ->
509 		 *    full speed isochronous -> interrupt QH's
510 		 */
511 		for (i = 0; i < EHCI_FRAMELIST_COUNT; i++) {
512 			pframes[i] = sc->sc_isoc_hs_p_last
513 			    [i & (EHCI_VIRTUAL_FRAMELIST_COUNT - 1)]->itd_self;
514 		}
515 	}
516 	usbd_get_page(&sc->sc_hw.async_start_pc, 0, &buf_res);
517 
518 	if (1) {
519 
520 		ehci_qh_t *qh;
521 
522 		qh = buf_res.buffer;
523 
524 		/* initialize page cache pointer */
525 
526 		qh->page_cache = &sc->sc_hw.async_start_pc;
527 
528 		/* store a pointer to the queue head */
529 
530 		sc->sc_async_p_last = qh;
531 
532 		/* init dummy QH that starts the async list */
533 
534 		qh->qh_self =
535 		    htohc32(sc, buf_res.physaddr) |
536 		    htohc32(sc, EHCI_LINK_QH);
537 
538 		/* fill the QH */
539 		qh->qh_endp =
540 		    htohc32(sc, EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH) | EHCI_QH_HRECL);
541 		qh->qh_endphub = htohc32(sc, EHCI_QH_SET_MULT(1));
542 		qh->qh_link = qh->qh_self;
543 		qh->qh_curqtd = 0;
544 
545 		/* fill the overlay qTD */
546 		qh->qh_qtd.qtd_next = htohc32(sc, EHCI_LINK_TERMINATE);
547 		qh->qh_qtd.qtd_altnext = htohc32(sc, EHCI_LINK_TERMINATE);
548 		qh->qh_qtd.qtd_status = htohc32(sc, EHCI_QTD_HALTED);
549 	}
550 	/* flush all cache into memory */
551 
552 	usb_bus_mem_flush_all(&sc->sc_bus, &ehci_iterate_hw_softc);
553 
554 #ifdef USB_DEBUG
555 	if (ehcidebug) {
556 		ehci_dump_sqh(sc, sc->sc_async_p_last);
557 	}
558 #endif
559 
560 	/* finial setup */
561  	err = ehci_init_sub(sc);
562 
563 	if (!err) {
564 		/* catch any lost interrupts */
565 		ehci_do_poll(&sc->sc_bus);
566 	}
567 	return (err);
568 }
569 
570 /*
571  * shut down the controller when the system is going down
572  */
573 void
574 ehci_detach(ehci_softc_t *sc)
575 {
576 	USB_BUS_LOCK(&sc->sc_bus);
577 
578 	usb_callout_stop(&sc->sc_tmo_pcd);
579 	usb_callout_stop(&sc->sc_tmo_poll);
580 
581 	EOWRITE4(sc, EHCI_USBINTR, 0);
582 	USB_BUS_UNLOCK(&sc->sc_bus);
583 
584 	if (ehci_hcreset(sc)) {
585 		DPRINTF("reset failed!\n");
586 	}
587 
588 	/* XXX let stray task complete */
589 	usb_pause_mtx(NULL, hz / 20);
590 
591 	usb_callout_drain(&sc->sc_tmo_pcd);
592 	usb_callout_drain(&sc->sc_tmo_poll);
593 }
594 
595 static void
596 ehci_suspend(ehci_softc_t *sc)
597 {
598 	DPRINTF("stopping the HC\n");
599 
600 	/* reset HC */
601 	ehci_hcreset(sc);
602 }
603 
604 static void
605 ehci_resume(ehci_softc_t *sc)
606 {
607 	/* reset HC */
608 	ehci_hcreset(sc);
609 
610 	/* setup HC */
611 	ehci_init_sub(sc);
612 
613 	/* catch any lost interrupts */
614 	ehci_do_poll(&sc->sc_bus);
615 }
616 
617 #ifdef USB_DEBUG
618 static void
619 ehci_dump_regs(ehci_softc_t *sc)
620 {
621 	uint32_t i;
622 
623 	i = EOREAD4(sc, EHCI_USBCMD);
624 	printf("cmd=0x%08x\n", i);
625 
626 	if (i & EHCI_CMD_ITC_1)
627 		printf(" EHCI_CMD_ITC_1\n");
628 	if (i & EHCI_CMD_ITC_2)
629 		printf(" EHCI_CMD_ITC_2\n");
630 	if (i & EHCI_CMD_ITC_4)
631 		printf(" EHCI_CMD_ITC_4\n");
632 	if (i & EHCI_CMD_ITC_8)
633 		printf(" EHCI_CMD_ITC_8\n");
634 	if (i & EHCI_CMD_ITC_16)
635 		printf(" EHCI_CMD_ITC_16\n");
636 	if (i & EHCI_CMD_ITC_32)
637 		printf(" EHCI_CMD_ITC_32\n");
638 	if (i & EHCI_CMD_ITC_64)
639 		printf(" EHCI_CMD_ITC_64\n");
640 	if (i & EHCI_CMD_ASPME)
641 		printf(" EHCI_CMD_ASPME\n");
642 	if (i & EHCI_CMD_ASPMC)
643 		printf(" EHCI_CMD_ASPMC\n");
644 	if (i & EHCI_CMD_LHCR)
645 		printf(" EHCI_CMD_LHCR\n");
646 	if (i & EHCI_CMD_IAAD)
647 		printf(" EHCI_CMD_IAAD\n");
648 	if (i & EHCI_CMD_ASE)
649 		printf(" EHCI_CMD_ASE\n");
650 	if (i & EHCI_CMD_PSE)
651 		printf(" EHCI_CMD_PSE\n");
652 	if (i & EHCI_CMD_FLS_M)
653 		printf(" EHCI_CMD_FLS_M\n");
654 	if (i & EHCI_CMD_HCRESET)
655 		printf(" EHCI_CMD_HCRESET\n");
656 	if (i & EHCI_CMD_RS)
657 		printf(" EHCI_CMD_RS\n");
658 
659 	i = EOREAD4(sc, EHCI_USBSTS);
660 
661 	printf("sts=0x%08x\n", i);
662 
663 	if (i & EHCI_STS_ASS)
664 		printf(" EHCI_STS_ASS\n");
665 	if (i & EHCI_STS_PSS)
666 		printf(" EHCI_STS_PSS\n");
667 	if (i & EHCI_STS_REC)
668 		printf(" EHCI_STS_REC\n");
669 	if (i & EHCI_STS_HCH)
670 		printf(" EHCI_STS_HCH\n");
671 	if (i & EHCI_STS_IAA)
672 		printf(" EHCI_STS_IAA\n");
673 	if (i & EHCI_STS_HSE)
674 		printf(" EHCI_STS_HSE\n");
675 	if (i & EHCI_STS_FLR)
676 		printf(" EHCI_STS_FLR\n");
677 	if (i & EHCI_STS_PCD)
678 		printf(" EHCI_STS_PCD\n");
679 	if (i & EHCI_STS_ERRINT)
680 		printf(" EHCI_STS_ERRINT\n");
681 	if (i & EHCI_STS_INT)
682 		printf(" EHCI_STS_INT\n");
683 
684 	printf("ien=0x%08x\n",
685 	    EOREAD4(sc, EHCI_USBINTR));
686 	printf("frindex=0x%08x ctrdsegm=0x%08x periodic=0x%08x async=0x%08x\n",
687 	    EOREAD4(sc, EHCI_FRINDEX),
688 	    EOREAD4(sc, EHCI_CTRLDSSEGMENT),
689 	    EOREAD4(sc, EHCI_PERIODICLISTBASE),
690 	    EOREAD4(sc, EHCI_ASYNCLISTADDR));
691 	for (i = 1; i <= sc->sc_noport; i++) {
692 		printf("port %d status=0x%08x\n", i,
693 		    EOREAD4(sc, EHCI_PORTSC(i)));
694 	}
695 }
696 
697 static void
698 ehci_dump_link(ehci_softc_t *sc, uint32_t link, int type)
699 {
700 	link = hc32toh(sc, link);
701 	printf("0x%08x", link);
702 	if (link & EHCI_LINK_TERMINATE)
703 		printf("<T>");
704 	else {
705 		printf("<");
706 		if (type) {
707 			switch (EHCI_LINK_TYPE(link)) {
708 			case EHCI_LINK_ITD:
709 				printf("ITD");
710 				break;
711 			case EHCI_LINK_QH:
712 				printf("QH");
713 				break;
714 			case EHCI_LINK_SITD:
715 				printf("SITD");
716 				break;
717 			case EHCI_LINK_FSTN:
718 				printf("FSTN");
719 				break;
720 			}
721 		}
722 		printf(">");
723 	}
724 }
725 
726 static void
727 ehci_dump_qtd(ehci_softc_t *sc, ehci_qtd_t *qtd)
728 {
729 	uint32_t s;
730 
731 	printf("  next=");
732 	ehci_dump_link(sc, qtd->qtd_next, 0);
733 	printf(" altnext=");
734 	ehci_dump_link(sc, qtd->qtd_altnext, 0);
735 	printf("\n");
736 	s = hc32toh(sc, qtd->qtd_status);
737 	printf("  status=0x%08x: toggle=%d bytes=0x%x ioc=%d c_page=0x%x\n",
738 	    s, EHCI_QTD_GET_TOGGLE(s), EHCI_QTD_GET_BYTES(s),
739 	    EHCI_QTD_GET_IOC(s), EHCI_QTD_GET_C_PAGE(s));
740 	printf("    cerr=%d pid=%d stat=%s%s%s%s%s%s%s%s\n",
741 	    EHCI_QTD_GET_CERR(s), EHCI_QTD_GET_PID(s),
742 	    (s & EHCI_QTD_ACTIVE) ? "ACTIVE" : "NOT_ACTIVE",
743 	    (s & EHCI_QTD_HALTED) ? "-HALTED" : "",
744 	    (s & EHCI_QTD_BUFERR) ? "-BUFERR" : "",
745 	    (s & EHCI_QTD_BABBLE) ? "-BABBLE" : "",
746 	    (s & EHCI_QTD_XACTERR) ? "-XACTERR" : "",
747 	    (s & EHCI_QTD_MISSEDMICRO) ? "-MISSED" : "",
748 	    (s & EHCI_QTD_SPLITXSTATE) ? "-SPLIT" : "",
749 	    (s & EHCI_QTD_PINGSTATE) ? "-PING" : "");
750 
751 	for (s = 0; s < 5; s++) {
752 		printf("  buffer[%d]=0x%08x\n", s,
753 		    hc32toh(sc, qtd->qtd_buffer[s]));
754 	}
755 	for (s = 0; s < 5; s++) {
756 		printf("  buffer_hi[%d]=0x%08x\n", s,
757 		    hc32toh(sc, qtd->qtd_buffer_hi[s]));
758 	}
759 }
760 
761 static uint8_t
762 ehci_dump_sqtd(ehci_softc_t *sc, ehci_qtd_t *sqtd)
763 {
764 	uint8_t temp;
765 
766 	usb_pc_cpu_invalidate(sqtd->page_cache);
767 	printf("QTD(%p) at 0x%08x:\n", sqtd, hc32toh(sc, sqtd->qtd_self));
768 	ehci_dump_qtd(sc, sqtd);
769 	temp = (sqtd->qtd_next & htohc32(sc, EHCI_LINK_TERMINATE)) ? 1 : 0;
770 	return (temp);
771 }
772 
773 static void
774 ehci_dump_sqtds(ehci_softc_t *sc, ehci_qtd_t *sqtd)
775 {
776 	uint16_t i;
777 	uint8_t stop;
778 
779 	stop = 0;
780 	for (i = 0; sqtd && (i < 20) && !stop; sqtd = sqtd->obj_next, i++) {
781 		stop = ehci_dump_sqtd(sc, sqtd);
782 	}
783 	if (sqtd) {
784 		printf("dump aborted, too many TDs\n");
785 	}
786 }
787 
788 static void
789 ehci_dump_sqh(ehci_softc_t *sc, ehci_qh_t *qh)
790 {
791 	uint32_t endp;
792 	uint32_t endphub;
793 
794 	usb_pc_cpu_invalidate(qh->page_cache);
795 	printf("QH(%p) at 0x%08x:\n", qh, hc32toh(sc, qh->qh_self) & ~0x1F);
796 	printf("  link=");
797 	ehci_dump_link(sc, qh->qh_link, 1);
798 	printf("\n");
799 	endp = hc32toh(sc, qh->qh_endp);
800 	printf("  endp=0x%08x\n", endp);
801 	printf("    addr=0x%02x inact=%d endpt=%d eps=%d dtc=%d hrecl=%d\n",
802 	    EHCI_QH_GET_ADDR(endp), EHCI_QH_GET_INACT(endp),
803 	    EHCI_QH_GET_ENDPT(endp), EHCI_QH_GET_EPS(endp),
804 	    EHCI_QH_GET_DTC(endp), EHCI_QH_GET_HRECL(endp));
805 	printf("    mpl=0x%x ctl=%d nrl=%d\n",
806 	    EHCI_QH_GET_MPL(endp), EHCI_QH_GET_CTL(endp),
807 	    EHCI_QH_GET_NRL(endp));
808 	endphub = hc32toh(sc, qh->qh_endphub);
809 	printf("  endphub=0x%08x\n", endphub);
810 	printf("    smask=0x%02x cmask=0x%02x huba=0x%02x port=%d mult=%d\n",
811 	    EHCI_QH_GET_SMASK(endphub), EHCI_QH_GET_CMASK(endphub),
812 	    EHCI_QH_GET_HUBA(endphub), EHCI_QH_GET_PORT(endphub),
813 	    EHCI_QH_GET_MULT(endphub));
814 	printf("  curqtd=");
815 	ehci_dump_link(sc, qh->qh_curqtd, 0);
816 	printf("\n");
817 	printf("Overlay qTD:\n");
818 	ehci_dump_qtd(sc, (void *)&qh->qh_qtd);
819 }
820 
821 static void
822 ehci_dump_sitd(ehci_softc_t *sc, ehci_sitd_t *sitd)
823 {
824 	usb_pc_cpu_invalidate(sitd->page_cache);
825 	printf("SITD(%p) at 0x%08x\n", sitd, hc32toh(sc, sitd->sitd_self) & ~0x1F);
826 	printf(" next=0x%08x\n", hc32toh(sc, sitd->sitd_next));
827 	printf(" portaddr=0x%08x dir=%s addr=%d endpt=0x%x port=0x%x huba=0x%x\n",
828 	    hc32toh(sc, sitd->sitd_portaddr),
829 	    (sitd->sitd_portaddr & htohc32(sc, EHCI_SITD_SET_DIR_IN))
830 	    ? "in" : "out",
831 	    EHCI_SITD_GET_ADDR(hc32toh(sc, sitd->sitd_portaddr)),
832 	    EHCI_SITD_GET_ENDPT(hc32toh(sc, sitd->sitd_portaddr)),
833 	    EHCI_SITD_GET_PORT(hc32toh(sc, sitd->sitd_portaddr)),
834 	    EHCI_SITD_GET_HUBA(hc32toh(sc, sitd->sitd_portaddr)));
835 	printf(" mask=0x%08x\n", hc32toh(sc, sitd->sitd_mask));
836 	printf(" status=0x%08x <%s> len=0x%x\n", hc32toh(sc, sitd->sitd_status),
837 	    (sitd->sitd_status & htohc32(sc, EHCI_SITD_ACTIVE)) ? "ACTIVE" : "",
838 	    EHCI_SITD_GET_LEN(hc32toh(sc, sitd->sitd_status)));
839 	printf(" back=0x%08x, bp=0x%08x,0x%08x,0x%08x,0x%08x\n",
840 	    hc32toh(sc, sitd->sitd_back),
841 	    hc32toh(sc, sitd->sitd_bp[0]),
842 	    hc32toh(sc, sitd->sitd_bp[1]),
843 	    hc32toh(sc, sitd->sitd_bp_hi[0]),
844 	    hc32toh(sc, sitd->sitd_bp_hi[1]));
845 }
846 
847 static void
848 ehci_dump_itd(ehci_softc_t *sc, ehci_itd_t *itd)
849 {
850 	usb_pc_cpu_invalidate(itd->page_cache);
851 	printf("ITD(%p) at 0x%08x\n", itd, hc32toh(sc, itd->itd_self) & ~0x1F);
852 	printf(" next=0x%08x\n", hc32toh(sc, itd->itd_next));
853 	printf(" status[0]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[0]),
854 	    (itd->itd_status[0] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
855 	printf(" status[1]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[1]),
856 	    (itd->itd_status[1] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
857 	printf(" status[2]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[2]),
858 	    (itd->itd_status[2] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
859 	printf(" status[3]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[3]),
860 	    (itd->itd_status[3] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
861 	printf(" status[4]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[4]),
862 	    (itd->itd_status[4] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
863 	printf(" status[5]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[5]),
864 	    (itd->itd_status[5] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
865 	printf(" status[6]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[6]),
866 	    (itd->itd_status[6] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
867 	printf(" status[7]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[7]),
868 	    (itd->itd_status[7] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
869 	printf(" bp[0]=0x%08x\n", hc32toh(sc, itd->itd_bp[0]));
870 	printf("  addr=0x%02x; endpt=0x%01x\n",
871 	    EHCI_ITD_GET_ADDR(hc32toh(sc, itd->itd_bp[0])),
872 	    EHCI_ITD_GET_ENDPT(hc32toh(sc, itd->itd_bp[0])));
873 	printf(" bp[1]=0x%08x\n", hc32toh(sc, itd->itd_bp[1]));
874 	printf(" dir=%s; mpl=0x%02x\n",
875 	    (hc32toh(sc, itd->itd_bp[1]) & EHCI_ITD_SET_DIR_IN) ? "in" : "out",
876 	    EHCI_ITD_GET_MPL(hc32toh(sc, itd->itd_bp[1])));
877 	printf(" bp[2..6]=0x%08x,0x%08x,0x%08x,0x%08x,0x%08x\n",
878 	    hc32toh(sc, itd->itd_bp[2]),
879 	    hc32toh(sc, itd->itd_bp[3]),
880 	    hc32toh(sc, itd->itd_bp[4]),
881 	    hc32toh(sc, itd->itd_bp[5]),
882 	    hc32toh(sc, itd->itd_bp[6]));
883 	printf(" bp_hi=0x%08x,0x%08x,0x%08x,0x%08x,\n"
884 	    "       0x%08x,0x%08x,0x%08x\n",
885 	    hc32toh(sc, itd->itd_bp_hi[0]),
886 	    hc32toh(sc, itd->itd_bp_hi[1]),
887 	    hc32toh(sc, itd->itd_bp_hi[2]),
888 	    hc32toh(sc, itd->itd_bp_hi[3]),
889 	    hc32toh(sc, itd->itd_bp_hi[4]),
890 	    hc32toh(sc, itd->itd_bp_hi[5]),
891 	    hc32toh(sc, itd->itd_bp_hi[6]));
892 }
893 
894 static void
895 ehci_dump_isoc(ehci_softc_t *sc)
896 {
897 	ehci_itd_t *itd;
898 	ehci_sitd_t *sitd;
899 	uint16_t max = 1000;
900 	uint16_t pos;
901 
902 	pos = (EOREAD4(sc, EHCI_FRINDEX) / 8) &
903 	    (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
904 
905 	printf("%s: isochronous dump from frame 0x%03x:\n",
906 	    __FUNCTION__, pos);
907 
908 	itd = sc->sc_isoc_hs_p_last[pos];
909 	sitd = sc->sc_isoc_fs_p_last[pos];
910 
911 	while (itd && max && max--) {
912 		ehci_dump_itd(sc, itd);
913 		itd = itd->prev;
914 	}
915 
916 	while (sitd && max && max--) {
917 		ehci_dump_sitd(sc, sitd);
918 		sitd = sitd->prev;
919 	}
920 }
921 
922 #endif
923 
924 static void
925 ehci_transfer_intr_enqueue(struct usb_xfer *xfer)
926 {
927 	/* check for early completion */
928 	if (ehci_check_transfer(xfer)) {
929 		return;
930 	}
931 	/* put transfer on interrupt queue */
932 	usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
933 
934 	/* start timeout, if any */
935 	if (xfer->timeout != 0) {
936 		usbd_transfer_timeout_ms(xfer, &ehci_timeout, xfer->timeout);
937 	}
938 }
939 
940 #define	EHCI_APPEND_FS_TD(std,last) (last) = _ehci_append_fs_td(std,last)
941 static ehci_sitd_t *
942 _ehci_append_fs_td(ehci_sitd_t *std, ehci_sitd_t *last)
943 {
944 	DPRINTFN(11, "%p to %p\n", std, last);
945 
946 	/* (sc->sc_bus.mtx) must be locked */
947 
948 	std->next = last->next;
949 	std->sitd_next = last->sitd_next;
950 
951 	std->prev = last;
952 
953 	usb_pc_cpu_flush(std->page_cache);
954 
955 	/*
956 	 * the last->next->prev is never followed: std->next->prev = std;
957 	 */
958 	last->next = std;
959 	last->sitd_next = std->sitd_self;
960 
961 	usb_pc_cpu_flush(last->page_cache);
962 
963 	return (std);
964 }
965 
966 #define	EHCI_APPEND_HS_TD(std,last) (last) = _ehci_append_hs_td(std,last)
967 static ehci_itd_t *
968 _ehci_append_hs_td(ehci_itd_t *std, ehci_itd_t *last)
969 {
970 	DPRINTFN(11, "%p to %p\n", std, last);
971 
972 	/* (sc->sc_bus.mtx) must be locked */
973 
974 	std->next = last->next;
975 	std->itd_next = last->itd_next;
976 
977 	std->prev = last;
978 
979 	usb_pc_cpu_flush(std->page_cache);
980 
981 	/*
982 	 * the last->next->prev is never followed: std->next->prev = std;
983 	 */
984 	last->next = std;
985 	last->itd_next = std->itd_self;
986 
987 	usb_pc_cpu_flush(last->page_cache);
988 
989 	return (std);
990 }
991 
992 #define	EHCI_APPEND_QH(sqh,last) (last) = _ehci_append_qh(sqh,last)
993 static ehci_qh_t *
994 _ehci_append_qh(ehci_qh_t *sqh, ehci_qh_t *last)
995 {
996 	DPRINTFN(11, "%p to %p\n", sqh, last);
997 
998 	if (sqh->prev != NULL) {
999 		/* should not happen */
1000 		DPRINTFN(0, "QH already linked!\n");
1001 		return (last);
1002 	}
1003 	/* (sc->sc_bus.mtx) must be locked */
1004 
1005 	sqh->next = last->next;
1006 	sqh->qh_link = last->qh_link;
1007 
1008 	sqh->prev = last;
1009 
1010 	usb_pc_cpu_flush(sqh->page_cache);
1011 
1012 	/*
1013 	 * the last->next->prev is never followed: sqh->next->prev = sqh;
1014 	 */
1015 
1016 	last->next = sqh;
1017 	last->qh_link = sqh->qh_self;
1018 
1019 	usb_pc_cpu_flush(last->page_cache);
1020 
1021 	return (sqh);
1022 }
1023 
1024 #define	EHCI_REMOVE_FS_TD(std,last) (last) = _ehci_remove_fs_td(std,last)
1025 static ehci_sitd_t *
1026 _ehci_remove_fs_td(ehci_sitd_t *std, ehci_sitd_t *last)
1027 {
1028 	DPRINTFN(11, "%p from %p\n", std, last);
1029 
1030 	/* (sc->sc_bus.mtx) must be locked */
1031 
1032 	std->prev->next = std->next;
1033 	std->prev->sitd_next = std->sitd_next;
1034 
1035 	usb_pc_cpu_flush(std->prev->page_cache);
1036 
1037 	if (std->next) {
1038 		std->next->prev = std->prev;
1039 		usb_pc_cpu_flush(std->next->page_cache);
1040 	}
1041 	return ((last == std) ? std->prev : last);
1042 }
1043 
1044 #define	EHCI_REMOVE_HS_TD(std,last) (last) = _ehci_remove_hs_td(std,last)
1045 static ehci_itd_t *
1046 _ehci_remove_hs_td(ehci_itd_t *std, ehci_itd_t *last)
1047 {
1048 	DPRINTFN(11, "%p from %p\n", std, last);
1049 
1050 	/* (sc->sc_bus.mtx) must be locked */
1051 
1052 	std->prev->next = std->next;
1053 	std->prev->itd_next = std->itd_next;
1054 
1055 	usb_pc_cpu_flush(std->prev->page_cache);
1056 
1057 	if (std->next) {
1058 		std->next->prev = std->prev;
1059 		usb_pc_cpu_flush(std->next->page_cache);
1060 	}
1061 	return ((last == std) ? std->prev : last);
1062 }
1063 
1064 #define	EHCI_REMOVE_QH(sqh,last) (last) = _ehci_remove_qh(sqh,last)
1065 static ehci_qh_t *
1066 _ehci_remove_qh(ehci_qh_t *sqh, ehci_qh_t *last)
1067 {
1068 	DPRINTFN(11, "%p from %p\n", sqh, last);
1069 
1070 	/* (sc->sc_bus.mtx) must be locked */
1071 
1072 	/* only remove if not removed from a queue */
1073 	if (sqh->prev) {
1074 
1075 		sqh->prev->next = sqh->next;
1076 		sqh->prev->qh_link = sqh->qh_link;
1077 
1078 		usb_pc_cpu_flush(sqh->prev->page_cache);
1079 
1080 		if (sqh->next) {
1081 			sqh->next->prev = sqh->prev;
1082 			usb_pc_cpu_flush(sqh->next->page_cache);
1083 		}
1084 		last = ((last == sqh) ? sqh->prev : last);
1085 
1086 		sqh->prev = 0;
1087 
1088 		usb_pc_cpu_flush(sqh->page_cache);
1089 	}
1090 	return (last);
1091 }
1092 
1093 static void
1094 ehci_data_toggle_update(struct usb_xfer *xfer, uint16_t actlen, uint16_t xlen)
1095 {
1096 	uint16_t rem;
1097 	uint8_t dt;
1098 
1099 	/* count number of full packets */
1100 	dt = (actlen / xfer->max_packet_size) & 1;
1101 
1102 	/* compute remainder */
1103 	rem = actlen % xfer->max_packet_size;
1104 
1105 	if (rem > 0)
1106 		dt ^= 1;	/* short packet at the end */
1107 	else if (actlen != xlen)
1108 		dt ^= 1;	/* zero length packet at the end */
1109 	else if (xlen == 0)
1110 		dt ^= 1;	/* zero length transfer */
1111 
1112 	xfer->endpoint->toggle_next ^= dt;
1113 }
1114 
1115 static usb_error_t
1116 ehci_non_isoc_done_sub(struct usb_xfer *xfer)
1117 {
1118 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1119 	ehci_qtd_t *td;
1120 	ehci_qtd_t *td_alt_next;
1121 	uint32_t status;
1122 	uint16_t len;
1123 
1124 	td = xfer->td_transfer_cache;
1125 	td_alt_next = td->alt_next;
1126 
1127 	if (xfer->aframes != xfer->nframes) {
1128 		usbd_xfer_set_frame_len(xfer, xfer->aframes, 0);
1129 	}
1130 	while (1) {
1131 
1132 		usb_pc_cpu_invalidate(td->page_cache);
1133 		status = hc32toh(sc, td->qtd_status);
1134 
1135 		len = EHCI_QTD_GET_BYTES(status);
1136 
1137 		/*
1138 	         * Verify the status length and
1139 		 * add the length to "frlengths[]":
1140 	         */
1141 		if (len > td->len) {
1142 			/* should not happen */
1143 			DPRINTF("Invalid status length, "
1144 			    "0x%04x/0x%04x bytes\n", len, td->len);
1145 			status |= EHCI_QTD_HALTED;
1146 		} else if (xfer->aframes != xfer->nframes) {
1147 			xfer->frlengths[xfer->aframes] += td->len - len;
1148 			/* manually update data toggle */
1149 			ehci_data_toggle_update(xfer, td->len - len, td->len);
1150 		}
1151 
1152 		/* Check for last transfer */
1153 		if (((void *)td) == xfer->td_transfer_last) {
1154 			td = NULL;
1155 			break;
1156 		}
1157 		/* Check for transfer error */
1158 		if (status & EHCI_QTD_HALTED) {
1159 			/* the transfer is finished */
1160 			td = NULL;
1161 			break;
1162 		}
1163 		/* Check for short transfer */
1164 		if (len > 0) {
1165 			if (xfer->flags_int.short_frames_ok) {
1166 				/* follow alt next */
1167 				td = td->alt_next;
1168 			} else {
1169 				/* the transfer is finished */
1170 				td = NULL;
1171 			}
1172 			break;
1173 		}
1174 		td = td->obj_next;
1175 
1176 		if (td->alt_next != td_alt_next) {
1177 			/* this USB frame is complete */
1178 			break;
1179 		}
1180 	}
1181 
1182 	/* update transfer cache */
1183 
1184 	xfer->td_transfer_cache = td;
1185 
1186 #ifdef USB_DEBUG
1187 	if (status & EHCI_QTD_STATERRS) {
1188 		DPRINTFN(11, "error, addr=%d, endpt=0x%02x, frame=0x%02x"
1189 		    "status=%s%s%s%s%s%s%s%s\n",
1190 		    xfer->address, xfer->endpointno, xfer->aframes,
1191 		    (status & EHCI_QTD_ACTIVE) ? "[ACTIVE]" : "[NOT_ACTIVE]",
1192 		    (status & EHCI_QTD_HALTED) ? "[HALTED]" : "",
1193 		    (status & EHCI_QTD_BUFERR) ? "[BUFERR]" : "",
1194 		    (status & EHCI_QTD_BABBLE) ? "[BABBLE]" : "",
1195 		    (status & EHCI_QTD_XACTERR) ? "[XACTERR]" : "",
1196 		    (status & EHCI_QTD_MISSEDMICRO) ? "[MISSED]" : "",
1197 		    (status & EHCI_QTD_SPLITXSTATE) ? "[SPLIT]" : "",
1198 		    (status & EHCI_QTD_PINGSTATE) ? "[PING]" : "");
1199 	}
1200 #endif
1201 
1202 	return ((status & EHCI_QTD_HALTED) ?
1203 	    USB_ERR_STALLED : USB_ERR_NORMAL_COMPLETION);
1204 }
1205 
1206 static void
1207 ehci_non_isoc_done(struct usb_xfer *xfer)
1208 {
1209 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1210 	ehci_qh_t *qh;
1211 	uint32_t status;
1212 	usb_error_t err = 0;
1213 
1214 	DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
1215 	    xfer, xfer->endpoint);
1216 
1217 #ifdef USB_DEBUG
1218 	if (ehcidebug > 10) {
1219 		ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1220 
1221 		ehci_dump_sqtds(sc, xfer->td_transfer_first);
1222 	}
1223 #endif
1224 
1225 	/* extract data toggle directly from the QH's overlay area */
1226 
1227 	qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1228 
1229 	usb_pc_cpu_invalidate(qh->page_cache);
1230 
1231 	status = hc32toh(sc, qh->qh_qtd.qtd_status);
1232 
1233 	/* reset scanner */
1234 
1235 	xfer->td_transfer_cache = xfer->td_transfer_first;
1236 
1237 	if (xfer->flags_int.control_xfr) {
1238 
1239 		if (xfer->flags_int.control_hdr) {
1240 
1241 			err = ehci_non_isoc_done_sub(xfer);
1242 		}
1243 		xfer->aframes = 1;
1244 
1245 		if (xfer->td_transfer_cache == NULL) {
1246 			goto done;
1247 		}
1248 	}
1249 	while (xfer->aframes != xfer->nframes) {
1250 
1251 		err = ehci_non_isoc_done_sub(xfer);
1252 		xfer->aframes++;
1253 
1254 		if (xfer->td_transfer_cache == NULL) {
1255 			goto done;
1256 		}
1257 	}
1258 
1259 	if (xfer->flags_int.control_xfr &&
1260 	    !xfer->flags_int.control_act) {
1261 
1262 		err = ehci_non_isoc_done_sub(xfer);
1263 	}
1264 done:
1265 	ehci_device_done(xfer, err);
1266 }
1267 
1268 /*------------------------------------------------------------------------*
1269  *	ehci_check_transfer
1270  *
1271  * Return values:
1272  *    0: USB transfer is not finished
1273  * Else: USB transfer is finished
1274  *------------------------------------------------------------------------*/
1275 static uint8_t
1276 ehci_check_transfer(struct usb_xfer *xfer)
1277 {
1278 	struct usb_pipe_methods *methods = xfer->endpoint->methods;
1279 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1280 
1281 	uint32_t status;
1282 
1283 	DPRINTFN(13, "xfer=%p checking transfer\n", xfer);
1284 
1285 	if (methods == &ehci_device_isoc_fs_methods) {
1286 		ehci_sitd_t *td;
1287 
1288 		/* isochronous full speed transfer */
1289 
1290 		td = xfer->td_transfer_last;
1291 		usb_pc_cpu_invalidate(td->page_cache);
1292 		status = hc32toh(sc, td->sitd_status);
1293 
1294 		/* also check if first is complete */
1295 
1296 		td = xfer->td_transfer_first;
1297 		usb_pc_cpu_invalidate(td->page_cache);
1298 		status |= hc32toh(sc, td->sitd_status);
1299 
1300 		if (!(status & EHCI_SITD_ACTIVE)) {
1301 			ehci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1302 			goto transferred;
1303 		}
1304 	} else if (methods == &ehci_device_isoc_hs_methods) {
1305 		ehci_itd_t *td;
1306 
1307 		/* isochronous high speed transfer */
1308 
1309 		/* check last transfer */
1310 		td = xfer->td_transfer_last;
1311 		usb_pc_cpu_invalidate(td->page_cache);
1312 		status = td->itd_status[0];
1313 		status |= td->itd_status[1];
1314 		status |= td->itd_status[2];
1315 		status |= td->itd_status[3];
1316 		status |= td->itd_status[4];
1317 		status |= td->itd_status[5];
1318 		status |= td->itd_status[6];
1319 		status |= td->itd_status[7];
1320 
1321 		/* also check first transfer */
1322 		td = xfer->td_transfer_first;
1323 		usb_pc_cpu_invalidate(td->page_cache);
1324 		status |= td->itd_status[0];
1325 		status |= td->itd_status[1];
1326 		status |= td->itd_status[2];
1327 		status |= td->itd_status[3];
1328 		status |= td->itd_status[4];
1329 		status |= td->itd_status[5];
1330 		status |= td->itd_status[6];
1331 		status |= td->itd_status[7];
1332 
1333 		/* if no transactions are active we continue */
1334 		if (!(status & htohc32(sc, EHCI_ITD_ACTIVE))) {
1335 			ehci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1336 			goto transferred;
1337 		}
1338 	} else {
1339 		ehci_qtd_t *td;
1340 		ehci_qh_t *qh;
1341 
1342 		/* non-isochronous transfer */
1343 
1344 		/*
1345 		 * check whether there is an error somewhere in the middle,
1346 		 * or whether there was a short packet (SPD and not ACTIVE)
1347 		 */
1348 		td = xfer->td_transfer_cache;
1349 
1350 		qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1351 
1352 		usb_pc_cpu_invalidate(qh->page_cache);
1353 
1354 		status = hc32toh(sc, qh->qh_qtd.qtd_status);
1355 		if (status & EHCI_QTD_ACTIVE) {
1356 			/* transfer is pending */
1357 			goto done;
1358 		}
1359 
1360 		while (1) {
1361 			usb_pc_cpu_invalidate(td->page_cache);
1362 			status = hc32toh(sc, td->qtd_status);
1363 
1364 			/*
1365 			 * Check if there is an active TD which
1366 			 * indicates that the transfer isn't done.
1367 			 */
1368 			if (status & EHCI_QTD_ACTIVE) {
1369 				/* update cache */
1370 				xfer->td_transfer_cache = td;
1371 				goto done;
1372 			}
1373 			/*
1374 			 * last transfer descriptor makes the transfer done
1375 			 */
1376 			if (((void *)td) == xfer->td_transfer_last) {
1377 				break;
1378 			}
1379 			/*
1380 			 * any kind of error makes the transfer done
1381 			 */
1382 			if (status & EHCI_QTD_HALTED) {
1383 				break;
1384 			}
1385 			/*
1386 			 * if there is no alternate next transfer, a short
1387 			 * packet also makes the transfer done
1388 			 */
1389 			if (EHCI_QTD_GET_BYTES(status)) {
1390 				if (xfer->flags_int.short_frames_ok) {
1391 					/* follow alt next */
1392 					if (td->alt_next) {
1393 						td = td->alt_next;
1394 						continue;
1395 					}
1396 				}
1397 				/* transfer is done */
1398 				break;
1399 			}
1400 			td = td->obj_next;
1401 		}
1402 		ehci_non_isoc_done(xfer);
1403 		goto transferred;
1404 	}
1405 
1406 done:
1407 	DPRINTFN(13, "xfer=%p is still active\n", xfer);
1408 	return (0);
1409 
1410 transferred:
1411 	return (1);
1412 }
1413 
1414 static void
1415 ehci_pcd_enable(ehci_softc_t *sc)
1416 {
1417 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
1418 
1419 	sc->sc_eintrs |= EHCI_STS_PCD;
1420 	EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1421 
1422 	/* acknowledge any PCD interrupt */
1423 	EOWRITE4(sc, EHCI_USBSTS, EHCI_STS_PCD);
1424 
1425 	ehci_root_intr(sc);
1426 }
1427 
1428 static void
1429 ehci_interrupt_poll(ehci_softc_t *sc)
1430 {
1431 	struct usb_xfer *xfer;
1432 
1433 repeat:
1434 	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
1435 		/*
1436 		 * check if transfer is transferred
1437 		 */
1438 		if (ehci_check_transfer(xfer)) {
1439 			/* queue has been modified */
1440 			goto repeat;
1441 		}
1442 	}
1443 }
1444 
1445 /*
1446  * Some EHCI chips from VIA / ATI seem to trigger interrupts before
1447  * writing back the qTD status, or miss signalling occasionally under
1448  * heavy load.  If the host machine is too fast, we can miss
1449  * transaction completion - when we scan the active list the
1450  * transaction still seems to be active. This generally exhibits
1451  * itself as a umass stall that never recovers.
1452  *
1453  * We work around this behaviour by setting up this callback after any
1454  * softintr that completes with transactions still pending, giving us
1455  * another chance to check for completion after the writeback has
1456  * taken place.
1457  */
1458 static void
1459 ehci_poll_timeout(void *arg)
1460 {
1461 	ehci_softc_t *sc = arg;
1462 
1463 	DPRINTFN(3, "\n");
1464 	ehci_interrupt_poll(sc);
1465 }
1466 
1467 /*------------------------------------------------------------------------*
1468  *	ehci_interrupt - EHCI interrupt handler
1469  *
1470  * NOTE: Do not access "sc->sc_bus.bdev" inside the interrupt handler,
1471  * hence the interrupt handler will be setup before "sc->sc_bus.bdev"
1472  * is present !
1473  *------------------------------------------------------------------------*/
1474 void
1475 ehci_interrupt(ehci_softc_t *sc)
1476 {
1477 	uint32_t status;
1478 
1479 	USB_BUS_LOCK(&sc->sc_bus);
1480 
1481 	DPRINTFN(16, "real interrupt\n");
1482 
1483 #ifdef USB_DEBUG
1484 	if (ehcidebug > 15) {
1485 		ehci_dump_regs(sc);
1486 	}
1487 #endif
1488 
1489 	status = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
1490 	if (status == 0) {
1491 		/* the interrupt was not for us */
1492 		goto done;
1493 	}
1494 	if (!(status & sc->sc_eintrs)) {
1495 		goto done;
1496 	}
1497 	EOWRITE4(sc, EHCI_USBSTS, status);	/* acknowledge */
1498 
1499 	status &= sc->sc_eintrs;
1500 
1501 	if (status & EHCI_STS_HSE) {
1502 		printf("%s: unrecoverable error, "
1503 		    "controller halted\n", __FUNCTION__);
1504 #ifdef USB_DEBUG
1505 		ehci_dump_regs(sc);
1506 		ehci_dump_isoc(sc);
1507 #endif
1508 	}
1509 	if (status & EHCI_STS_PCD) {
1510 		/*
1511 		 * Disable PCD interrupt for now, because it will be
1512 		 * on until the port has been reset.
1513 		 */
1514 		sc->sc_eintrs &= ~EHCI_STS_PCD;
1515 		EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1516 
1517 		ehci_root_intr(sc);
1518 
1519 		/* do not allow RHSC interrupts > 1 per second */
1520 		usb_callout_reset(&sc->sc_tmo_pcd, hz,
1521 		    (void *)&ehci_pcd_enable, sc);
1522 	}
1523 	status &= ~(EHCI_STS_INT | EHCI_STS_ERRINT | EHCI_STS_PCD | EHCI_STS_IAA);
1524 
1525 	if (status != 0) {
1526 		/* block unprocessed interrupts */
1527 		sc->sc_eintrs &= ~status;
1528 		EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1529 		printf("%s: blocking interrupts 0x%x\n", __FUNCTION__, status);
1530 	}
1531 	/* poll all the USB transfers */
1532 	ehci_interrupt_poll(sc);
1533 
1534 	if (sc->sc_flags & EHCI_SCFLG_LOSTINTRBUG) {
1535 		usb_callout_reset(&sc->sc_tmo_poll, hz / 128,
1536 		    (void *)&ehci_poll_timeout, sc);
1537 	}
1538 
1539 done:
1540 	USB_BUS_UNLOCK(&sc->sc_bus);
1541 }
1542 
1543 /*
1544  * called when a request does not complete
1545  */
1546 static void
1547 ehci_timeout(void *arg)
1548 {
1549 	struct usb_xfer *xfer = arg;
1550 
1551 	DPRINTF("xfer=%p\n", xfer);
1552 
1553 	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
1554 
1555 	/* transfer is transferred */
1556 	ehci_device_done(xfer, USB_ERR_TIMEOUT);
1557 }
1558 
1559 static void
1560 ehci_do_poll(struct usb_bus *bus)
1561 {
1562 	ehci_softc_t *sc = EHCI_BUS2SC(bus);
1563 
1564 	USB_BUS_LOCK(&sc->sc_bus);
1565 	ehci_interrupt_poll(sc);
1566 	USB_BUS_UNLOCK(&sc->sc_bus);
1567 }
1568 
1569 static void
1570 ehci_setup_standard_chain_sub(struct ehci_std_temp *temp)
1571 {
1572 	struct usb_page_search buf_res;
1573 	ehci_qtd_t *td;
1574 	ehci_qtd_t *td_next;
1575 	ehci_qtd_t *td_alt_next;
1576 	uint32_t buf_offset;
1577 	uint32_t average;
1578 	uint32_t len_old;
1579 	uint32_t terminate;
1580 	uint32_t qtd_altnext;
1581 	uint8_t shortpkt_old;
1582 	uint8_t precompute;
1583 
1584 	terminate = temp->sc->sc_terminate_self;
1585 	qtd_altnext = temp->sc->sc_terminate_self;
1586 	td_alt_next = NULL;
1587 	buf_offset = 0;
1588 	shortpkt_old = temp->shortpkt;
1589 	len_old = temp->len;
1590 	precompute = 1;
1591 
1592 restart:
1593 
1594 	td = temp->td;
1595 	td_next = temp->td_next;
1596 
1597 	while (1) {
1598 
1599 		if (temp->len == 0) {
1600 
1601 			if (temp->shortpkt) {
1602 				break;
1603 			}
1604 			/* send a Zero Length Packet, ZLP, last */
1605 
1606 			temp->shortpkt = 1;
1607 			average = 0;
1608 
1609 		} else {
1610 
1611 			average = temp->average;
1612 
1613 			if (temp->len < average) {
1614 				if (temp->len % temp->max_frame_size) {
1615 					temp->shortpkt = 1;
1616 				}
1617 				average = temp->len;
1618 			}
1619 		}
1620 
1621 		if (td_next == NULL) {
1622 			panic("%s: out of EHCI transfer descriptors!", __FUNCTION__);
1623 		}
1624 		/* get next TD */
1625 
1626 		td = td_next;
1627 		td_next = td->obj_next;
1628 
1629 		/* check if we are pre-computing */
1630 
1631 		if (precompute) {
1632 
1633 			/* update remaining length */
1634 
1635 			temp->len -= average;
1636 
1637 			continue;
1638 		}
1639 		/* fill out current TD */
1640 
1641 		td->qtd_status =
1642 		    temp->qtd_status |
1643 		    htohc32(temp->sc, EHCI_QTD_IOC |
1644 			EHCI_QTD_SET_BYTES(average));
1645 
1646 		if (average == 0) {
1647 
1648 			if (temp->auto_data_toggle == 0) {
1649 
1650 				/* update data toggle, ZLP case */
1651 
1652 				temp->qtd_status ^=
1653 				    htohc32(temp->sc, EHCI_QTD_TOGGLE_MASK);
1654 			}
1655 			td->len = 0;
1656 
1657 			td->qtd_buffer[0] = 0;
1658 			td->qtd_buffer_hi[0] = 0;
1659 
1660 			td->qtd_buffer[1] = 0;
1661 			td->qtd_buffer_hi[1] = 0;
1662 
1663 		} else {
1664 
1665 			uint8_t x;
1666 
1667 			if (temp->auto_data_toggle == 0) {
1668 
1669 				/* update data toggle */
1670 
1671 				if (((average + temp->max_frame_size - 1) /
1672 				    temp->max_frame_size) & 1) {
1673 					temp->qtd_status ^=
1674 					    htohc32(temp->sc, EHCI_QTD_TOGGLE_MASK);
1675 				}
1676 			}
1677 			td->len = average;
1678 
1679 			/* update remaining length */
1680 
1681 			temp->len -= average;
1682 
1683 			/* fill out buffer pointers */
1684 
1685 			usbd_get_page(temp->pc, buf_offset, &buf_res);
1686 			td->qtd_buffer[0] =
1687 			    htohc32(temp->sc, buf_res.physaddr);
1688 			td->qtd_buffer_hi[0] = 0;
1689 
1690 			x = 1;
1691 
1692 			while (average > EHCI_PAGE_SIZE) {
1693 				average -= EHCI_PAGE_SIZE;
1694 				buf_offset += EHCI_PAGE_SIZE;
1695 				usbd_get_page(temp->pc, buf_offset, &buf_res);
1696 				td->qtd_buffer[x] =
1697 				    htohc32(temp->sc,
1698 				    buf_res.physaddr & (~0xFFF));
1699 				td->qtd_buffer_hi[x] = 0;
1700 				x++;
1701 			}
1702 
1703 			/*
1704 			 * NOTE: The "average" variable is never zero after
1705 			 * exiting the loop above !
1706 			 *
1707 			 * NOTE: We have to subtract one from the offset to
1708 			 * ensure that we are computing the physical address
1709 			 * of a valid page !
1710 			 */
1711 			buf_offset += average;
1712 			usbd_get_page(temp->pc, buf_offset - 1, &buf_res);
1713 			td->qtd_buffer[x] =
1714 			    htohc32(temp->sc,
1715 			    buf_res.physaddr & (~0xFFF));
1716 			td->qtd_buffer_hi[x] = 0;
1717 		}
1718 
1719 		if (td_next) {
1720 			/* link the current TD with the next one */
1721 			td->qtd_next = td_next->qtd_self;
1722 		}
1723 		td->qtd_altnext = qtd_altnext;
1724 		td->alt_next = td_alt_next;
1725 
1726 		usb_pc_cpu_flush(td->page_cache);
1727 	}
1728 
1729 	if (precompute) {
1730 		precompute = 0;
1731 
1732 		/* setup alt next pointer, if any */
1733 		if (temp->last_frame) {
1734 			td_alt_next = NULL;
1735 			qtd_altnext = terminate;
1736 		} else {
1737 			/* we use this field internally */
1738 			td_alt_next = td_next;
1739 			if (temp->setup_alt_next) {
1740 				qtd_altnext = td_next->qtd_self;
1741 			} else {
1742 				qtd_altnext = terminate;
1743 			}
1744 		}
1745 
1746 		/* restore */
1747 		temp->shortpkt = shortpkt_old;
1748 		temp->len = len_old;
1749 		goto restart;
1750 	}
1751 	temp->td = td;
1752 	temp->td_next = td_next;
1753 }
1754 
1755 static void
1756 ehci_setup_standard_chain(struct usb_xfer *xfer, ehci_qh_t **qh_last)
1757 {
1758 	struct ehci_std_temp temp;
1759 	struct usb_pipe_methods *methods;
1760 	ehci_qh_t *qh;
1761 	ehci_qtd_t *td;
1762 	uint32_t qh_endp;
1763 	uint32_t qh_endphub;
1764 	uint32_t x;
1765 
1766 	DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
1767 	    xfer->address, UE_GET_ADDR(xfer->endpointno),
1768 	    xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
1769 
1770 	temp.average = xfer->max_hc_frame_size;
1771 	temp.max_frame_size = xfer->max_frame_size;
1772 	temp.sc = EHCI_BUS2SC(xfer->xroot->bus);
1773 
1774 	/* toggle the DMA set we are using */
1775 	xfer->flags_int.curr_dma_set ^= 1;
1776 
1777 	/* get next DMA set */
1778 	td = xfer->td_start[xfer->flags_int.curr_dma_set];
1779 
1780 	xfer->td_transfer_first = td;
1781 	xfer->td_transfer_cache = td;
1782 
1783 	temp.td = NULL;
1784 	temp.td_next = td;
1785 	temp.qtd_status = 0;
1786 	temp.last_frame = 0;
1787 	temp.setup_alt_next = xfer->flags_int.short_frames_ok;
1788 
1789 	if (xfer->flags_int.control_xfr) {
1790 		if (xfer->endpoint->toggle_next) {
1791 			/* DATA1 is next */
1792 			temp.qtd_status |=
1793 			    htohc32(temp.sc, EHCI_QTD_SET_TOGGLE(1));
1794 		}
1795 		temp.auto_data_toggle = 0;
1796 	} else {
1797 		temp.auto_data_toggle = 1;
1798 	}
1799 
1800 	if ((xfer->xroot->udev->parent_hs_hub != NULL) ||
1801 	    (xfer->xroot->udev->address != 0)) {
1802 		/* max 3 retries */
1803 		temp.qtd_status |=
1804 		    htohc32(temp.sc, EHCI_QTD_SET_CERR(3));
1805 	}
1806 	/* check if we should prepend a setup message */
1807 
1808 	if (xfer->flags_int.control_xfr) {
1809 		if (xfer->flags_int.control_hdr) {
1810 
1811 			xfer->endpoint->toggle_next = 0;
1812 
1813 			temp.qtd_status &=
1814 			    htohc32(temp.sc, EHCI_QTD_SET_CERR(3));
1815 			temp.qtd_status |= htohc32(temp.sc,
1816 			    EHCI_QTD_ACTIVE |
1817 			    EHCI_QTD_SET_PID(EHCI_QTD_PID_SETUP) |
1818 			    EHCI_QTD_SET_TOGGLE(0));
1819 
1820 			temp.len = xfer->frlengths[0];
1821 			temp.pc = xfer->frbuffers + 0;
1822 			temp.shortpkt = temp.len ? 1 : 0;
1823 			/* check for last frame */
1824 			if (xfer->nframes == 1) {
1825 				/* no STATUS stage yet, SETUP is last */
1826 				if (xfer->flags_int.control_act) {
1827 					temp.last_frame = 1;
1828 					temp.setup_alt_next = 0;
1829 				}
1830 			}
1831 			ehci_setup_standard_chain_sub(&temp);
1832 		}
1833 		x = 1;
1834 	} else {
1835 		x = 0;
1836 	}
1837 
1838 	while (x != xfer->nframes) {
1839 
1840 		/* DATA0 / DATA1 message */
1841 
1842 		temp.len = xfer->frlengths[x];
1843 		temp.pc = xfer->frbuffers + x;
1844 
1845 		x++;
1846 
1847 		if (x == xfer->nframes) {
1848 			if (xfer->flags_int.control_xfr) {
1849 				/* no STATUS stage yet, DATA is last */
1850 				if (xfer->flags_int.control_act) {
1851 					temp.last_frame = 1;
1852 					temp.setup_alt_next = 0;
1853 				}
1854 			} else {
1855 				temp.last_frame = 1;
1856 				temp.setup_alt_next = 0;
1857 			}
1858 		}
1859 		/* keep previous data toggle and error count */
1860 
1861 		temp.qtd_status &=
1862 		    htohc32(temp.sc, EHCI_QTD_SET_CERR(3) |
1863 		    EHCI_QTD_SET_TOGGLE(1));
1864 
1865 		if (temp.len == 0) {
1866 
1867 			/* make sure that we send an USB packet */
1868 
1869 			temp.shortpkt = 0;
1870 
1871 		} else {
1872 
1873 			/* regular data transfer */
1874 
1875 			temp.shortpkt = (xfer->flags.force_short_xfer) ? 0 : 1;
1876 		}
1877 
1878 		/* set endpoint direction */
1879 
1880 		temp.qtd_status |=
1881 		    (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) ?
1882 		    htohc32(temp.sc, EHCI_QTD_ACTIVE |
1883 		    EHCI_QTD_SET_PID(EHCI_QTD_PID_IN)) :
1884 		    htohc32(temp.sc, EHCI_QTD_ACTIVE |
1885 		    EHCI_QTD_SET_PID(EHCI_QTD_PID_OUT));
1886 
1887 		ehci_setup_standard_chain_sub(&temp);
1888 	}
1889 
1890 	/* check if we should append a status stage */
1891 
1892 	if (xfer->flags_int.control_xfr &&
1893 	    !xfer->flags_int.control_act) {
1894 
1895 		/*
1896 		 * Send a DATA1 message and invert the current endpoint
1897 		 * direction.
1898 		 */
1899 
1900 		temp.qtd_status &= htohc32(temp.sc, EHCI_QTD_SET_CERR(3) |
1901 		    EHCI_QTD_SET_TOGGLE(1));
1902 		temp.qtd_status |=
1903 		    (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) ?
1904 		    htohc32(temp.sc, EHCI_QTD_ACTIVE |
1905 		    EHCI_QTD_SET_PID(EHCI_QTD_PID_IN) |
1906 		    EHCI_QTD_SET_TOGGLE(1)) :
1907 		    htohc32(temp.sc, EHCI_QTD_ACTIVE |
1908 		    EHCI_QTD_SET_PID(EHCI_QTD_PID_OUT) |
1909 		    EHCI_QTD_SET_TOGGLE(1));
1910 
1911 		temp.len = 0;
1912 		temp.pc = NULL;
1913 		temp.shortpkt = 0;
1914 		temp.last_frame = 1;
1915 		temp.setup_alt_next = 0;
1916 
1917 		ehci_setup_standard_chain_sub(&temp);
1918 	}
1919 	td = temp.td;
1920 
1921 	/* the last TD terminates the transfer: */
1922 	td->qtd_next = htohc32(temp.sc, EHCI_LINK_TERMINATE);
1923 	td->qtd_altnext = htohc32(temp.sc, EHCI_LINK_TERMINATE);
1924 
1925 	usb_pc_cpu_flush(td->page_cache);
1926 
1927 	/* must have at least one frame! */
1928 
1929 	xfer->td_transfer_last = td;
1930 
1931 #ifdef USB_DEBUG
1932 	if (ehcidebug > 8) {
1933 		DPRINTF("nexttog=%d; data before transfer:\n",
1934 		    xfer->endpoint->toggle_next);
1935 		ehci_dump_sqtds(temp.sc,
1936 		    xfer->td_transfer_first);
1937 	}
1938 #endif
1939 
1940 	methods = xfer->endpoint->methods;
1941 
1942 	qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1943 
1944 	/* the "qh_link" field is filled when the QH is added */
1945 
1946 	qh_endp =
1947 	    (EHCI_QH_SET_ADDR(xfer->address) |
1948 	    EHCI_QH_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)) |
1949 	    EHCI_QH_SET_MPL(xfer->max_packet_size));
1950 
1951 	if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_HIGH) {
1952 		qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH);
1953 		if (methods != &ehci_device_intr_methods)
1954 			qh_endp |= EHCI_QH_SET_NRL(8);
1955 	} else {
1956 
1957 		if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_FULL) {
1958 			qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_FULL);
1959 		} else {
1960 			qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_LOW);
1961 		}
1962 
1963 		if (methods == &ehci_device_ctrl_methods) {
1964 			qh_endp |= EHCI_QH_CTL;
1965 		}
1966 		if (methods != &ehci_device_intr_methods) {
1967 			/* Only try one time per microframe! */
1968 			qh_endp |= EHCI_QH_SET_NRL(1);
1969 		}
1970 	}
1971 
1972 	if (temp.auto_data_toggle == 0) {
1973 		/* software computes the data toggle */
1974 		qh_endp |= EHCI_QH_DTC;
1975 	}
1976 
1977 	qh->qh_endp = htohc32(temp.sc, qh_endp);
1978 
1979 	qh_endphub =
1980 	    (EHCI_QH_SET_MULT(xfer->max_packet_count & 3) |
1981 	    EHCI_QH_SET_CMASK(xfer->endpoint->usb_cmask) |
1982 	    EHCI_QH_SET_SMASK(xfer->endpoint->usb_smask) |
1983 	    EHCI_QH_SET_HUBA(xfer->xroot->udev->hs_hub_addr) |
1984 	    EHCI_QH_SET_PORT(xfer->xroot->udev->hs_port_no));
1985 
1986 	qh->qh_endphub = htohc32(temp.sc, qh_endphub);
1987 	qh->qh_curqtd = 0;
1988 
1989 	/* fill the overlay qTD */
1990 
1991 	if (temp.auto_data_toggle && xfer->endpoint->toggle_next) {
1992 		/* DATA1 is next */
1993 		qh->qh_qtd.qtd_status = htohc32(temp.sc, EHCI_QTD_SET_TOGGLE(1));
1994 	} else {
1995 		qh->qh_qtd.qtd_status = 0;
1996 	}
1997 
1998 	td = xfer->td_transfer_first;
1999 
2000 	qh->qh_qtd.qtd_next = td->qtd_self;
2001 	qh->qh_qtd.qtd_altnext =
2002 	    htohc32(temp.sc, EHCI_LINK_TERMINATE);
2003 
2004 	usb_pc_cpu_flush(qh->page_cache);
2005 
2006 	if (xfer->xroot->udev->flags.self_suspended == 0) {
2007 		EHCI_APPEND_QH(qh, *qh_last);
2008 	}
2009 }
2010 
2011 static void
2012 ehci_root_intr(ehci_softc_t *sc)
2013 {
2014 	uint16_t i;
2015 	uint16_t m;
2016 
2017 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2018 
2019 	/* clear any old interrupt data */
2020 	memset(sc->sc_hub_idata, 0, sizeof(sc->sc_hub_idata));
2021 
2022 	/* set bits */
2023 	m = (sc->sc_noport + 1);
2024 	if (m > (8 * sizeof(sc->sc_hub_idata))) {
2025 		m = (8 * sizeof(sc->sc_hub_idata));
2026 	}
2027 	for (i = 1; i < m; i++) {
2028 		/* pick out CHANGE bits from the status register */
2029 		if (EOREAD4(sc, EHCI_PORTSC(i)) & EHCI_PS_CLEAR) {
2030 			sc->sc_hub_idata[i / 8] |= 1 << (i % 8);
2031 			DPRINTF("port %d changed\n", i);
2032 		}
2033 	}
2034 	uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
2035 	    sizeof(sc->sc_hub_idata));
2036 }
2037 
2038 static void
2039 ehci_isoc_fs_done(ehci_softc_t *sc, struct usb_xfer *xfer)
2040 {
2041 	uint32_t nframes = xfer->nframes;
2042 	uint32_t status;
2043 	uint32_t *plen = xfer->frlengths;
2044 	uint16_t len = 0;
2045 	ehci_sitd_t *td = xfer->td_transfer_first;
2046 	ehci_sitd_t **pp_last = &sc->sc_isoc_fs_p_last[xfer->qh_pos];
2047 
2048 	DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
2049 	    xfer, xfer->endpoint);
2050 
2051 	while (nframes--) {
2052 		if (td == NULL) {
2053 			panic("%s:%d: out of TD's\n",
2054 			    __FUNCTION__, __LINE__);
2055 		}
2056 		if (pp_last >= &sc->sc_isoc_fs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2057 			pp_last = &sc->sc_isoc_fs_p_last[0];
2058 		}
2059 #ifdef USB_DEBUG
2060 		if (ehcidebug > 15) {
2061 			DPRINTF("isoc FS-TD\n");
2062 			ehci_dump_sitd(sc, td);
2063 		}
2064 #endif
2065 		usb_pc_cpu_invalidate(td->page_cache);
2066 		status = hc32toh(sc, td->sitd_status);
2067 
2068 		len = EHCI_SITD_GET_LEN(status);
2069 
2070 		DPRINTFN(2, "status=0x%08x, rem=%u\n", status, len);
2071 
2072 		if (*plen >= len) {
2073 			len = *plen - len;
2074 		} else {
2075 			len = 0;
2076 		}
2077 
2078 		*plen = len;
2079 
2080 		/* remove FS-TD from schedule */
2081 		EHCI_REMOVE_FS_TD(td, *pp_last);
2082 
2083 		pp_last++;
2084 		plen++;
2085 		td = td->obj_next;
2086 	}
2087 
2088 	xfer->aframes = xfer->nframes;
2089 }
2090 
2091 static void
2092 ehci_isoc_hs_done(ehci_softc_t *sc, struct usb_xfer *xfer)
2093 {
2094 	uint32_t nframes = xfer->nframes;
2095 	uint32_t status;
2096 	uint32_t *plen = xfer->frlengths;
2097 	uint16_t len = 0;
2098 	uint8_t td_no = 0;
2099 	ehci_itd_t *td = xfer->td_transfer_first;
2100 	ehci_itd_t **pp_last = &sc->sc_isoc_hs_p_last[xfer->qh_pos];
2101 
2102 	DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
2103 	    xfer, xfer->endpoint);
2104 
2105 	while (nframes) {
2106 		if (td == NULL) {
2107 			panic("%s:%d: out of TD's\n",
2108 			    __FUNCTION__, __LINE__);
2109 		}
2110 		if (pp_last >= &sc->sc_isoc_hs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2111 			pp_last = &sc->sc_isoc_hs_p_last[0];
2112 		}
2113 #ifdef USB_DEBUG
2114 		if (ehcidebug > 15) {
2115 			DPRINTF("isoc HS-TD\n");
2116 			ehci_dump_itd(sc, td);
2117 		}
2118 #endif
2119 
2120 		usb_pc_cpu_invalidate(td->page_cache);
2121 		status = hc32toh(sc, td->itd_status[td_no]);
2122 
2123 		len = EHCI_ITD_GET_LEN(status);
2124 
2125 		DPRINTFN(2, "status=0x%08x, len=%u\n", status, len);
2126 
2127 		if (xfer->endpoint->usb_smask & (1 << td_no)) {
2128 
2129 			if (*plen >= len) {
2130 				/*
2131 				 * The length is valid. NOTE: The
2132 				 * complete length is written back
2133 				 * into the status field, and not the
2134 				 * remainder like with other transfer
2135 				 * descriptor types.
2136 				 */
2137 			} else {
2138 				/* Invalid length - truncate */
2139 				len = 0;
2140 			}
2141 
2142 			*plen = len;
2143 			plen++;
2144 			nframes--;
2145 		}
2146 
2147 		td_no++;
2148 
2149 		if ((td_no == 8) || (nframes == 0)) {
2150 			/* remove HS-TD from schedule */
2151 			EHCI_REMOVE_HS_TD(td, *pp_last);
2152 			pp_last++;
2153 
2154 			td_no = 0;
2155 			td = td->obj_next;
2156 		}
2157 	}
2158 	xfer->aframes = xfer->nframes;
2159 }
2160 
2161 /* NOTE: "done" can be run two times in a row,
2162  * from close and from interrupt
2163  */
2164 static void
2165 ehci_device_done(struct usb_xfer *xfer, usb_error_t error)
2166 {
2167 	struct usb_pipe_methods *methods = xfer->endpoint->methods;
2168 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2169 
2170 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2171 
2172 	DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n",
2173 	    xfer, xfer->endpoint, error);
2174 
2175 	if ((methods == &ehci_device_bulk_methods) ||
2176 	    (methods == &ehci_device_ctrl_methods)) {
2177 #ifdef USB_DEBUG
2178 		if (ehcidebug > 8) {
2179 			DPRINTF("nexttog=%d; data after transfer:\n",
2180 			    xfer->endpoint->toggle_next);
2181 			ehci_dump_sqtds(sc,
2182 			    xfer->td_transfer_first);
2183 		}
2184 #endif
2185 
2186 		EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
2187 		    sc->sc_async_p_last);
2188 	}
2189 	if (methods == &ehci_device_intr_methods) {
2190 		EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
2191 		    sc->sc_intr_p_last[xfer->qh_pos]);
2192 	}
2193 	/*
2194 	 * Only finish isochronous transfers once which will update
2195 	 * "xfer->frlengths".
2196 	 */
2197 	if (xfer->td_transfer_first &&
2198 	    xfer->td_transfer_last) {
2199 		if (methods == &ehci_device_isoc_fs_methods) {
2200 			ehci_isoc_fs_done(sc, xfer);
2201 		}
2202 		if (methods == &ehci_device_isoc_hs_methods) {
2203 			ehci_isoc_hs_done(sc, xfer);
2204 		}
2205 		xfer->td_transfer_first = NULL;
2206 		xfer->td_transfer_last = NULL;
2207 	}
2208 	/* dequeue transfer and start next transfer */
2209 	usbd_transfer_done(xfer, error);
2210 }
2211 
2212 /*------------------------------------------------------------------------*
2213  * ehci bulk support
2214  *------------------------------------------------------------------------*/
2215 static void
2216 ehci_device_bulk_open(struct usb_xfer *xfer)
2217 {
2218 	return;
2219 }
2220 
2221 static void
2222 ehci_device_bulk_close(struct usb_xfer *xfer)
2223 {
2224 	ehci_device_done(xfer, USB_ERR_CANCELLED);
2225 }
2226 
2227 static void
2228 ehci_device_bulk_enter(struct usb_xfer *xfer)
2229 {
2230 	return;
2231 }
2232 
2233 static void
2234 ehci_device_bulk_start(struct usb_xfer *xfer)
2235 {
2236 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2237 	uint32_t temp;
2238 
2239 	/* setup TD's and QH */
2240 	ehci_setup_standard_chain(xfer, &sc->sc_async_p_last);
2241 
2242 	/* put transfer on interrupt queue */
2243 	ehci_transfer_intr_enqueue(xfer);
2244 
2245 	/*
2246 	 * XXX Certain nVidia chipsets choke when using the IAAD
2247 	 * feature too frequently.
2248 	 */
2249 	if (sc->sc_flags & EHCI_SCFLG_IAADBUG)
2250 		return;
2251 
2252 	/* XXX Performance quirk: Some Host Controllers have a too low
2253 	 * interrupt rate. Issue an IAAD to stimulate the Host
2254 	 * Controller after queueing the BULK transfer.
2255 	 */
2256 	temp = EOREAD4(sc, EHCI_USBCMD);
2257 	if (!(temp & EHCI_CMD_IAAD))
2258 		EOWRITE4(sc, EHCI_USBCMD, temp | EHCI_CMD_IAAD);
2259 }
2260 
2261 struct usb_pipe_methods ehci_device_bulk_methods =
2262 {
2263 	.open = ehci_device_bulk_open,
2264 	.close = ehci_device_bulk_close,
2265 	.enter = ehci_device_bulk_enter,
2266 	.start = ehci_device_bulk_start,
2267 };
2268 
2269 /*------------------------------------------------------------------------*
2270  * ehci control support
2271  *------------------------------------------------------------------------*/
2272 static void
2273 ehci_device_ctrl_open(struct usb_xfer *xfer)
2274 {
2275 	return;
2276 }
2277 
2278 static void
2279 ehci_device_ctrl_close(struct usb_xfer *xfer)
2280 {
2281 	ehci_device_done(xfer, USB_ERR_CANCELLED);
2282 }
2283 
2284 static void
2285 ehci_device_ctrl_enter(struct usb_xfer *xfer)
2286 {
2287 	return;
2288 }
2289 
2290 static void
2291 ehci_device_ctrl_start(struct usb_xfer *xfer)
2292 {
2293 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2294 
2295 	/* setup TD's and QH */
2296 	ehci_setup_standard_chain(xfer, &sc->sc_async_p_last);
2297 
2298 	/* put transfer on interrupt queue */
2299 	ehci_transfer_intr_enqueue(xfer);
2300 }
2301 
2302 struct usb_pipe_methods ehci_device_ctrl_methods =
2303 {
2304 	.open = ehci_device_ctrl_open,
2305 	.close = ehci_device_ctrl_close,
2306 	.enter = ehci_device_ctrl_enter,
2307 	.start = ehci_device_ctrl_start,
2308 };
2309 
2310 /*------------------------------------------------------------------------*
2311  * ehci interrupt support
2312  *------------------------------------------------------------------------*/
2313 static void
2314 ehci_device_intr_open(struct usb_xfer *xfer)
2315 {
2316 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2317 	uint16_t best;
2318 	uint16_t bit;
2319 	uint16_t x;
2320 
2321 	usb_hs_bandwidth_alloc(xfer);
2322 
2323 	/*
2324 	 * Find the best QH position corresponding to the given interval:
2325 	 */
2326 
2327 	best = 0;
2328 	bit = EHCI_VIRTUAL_FRAMELIST_COUNT / 2;
2329 	while (bit) {
2330 		if (xfer->interval >= bit) {
2331 			x = bit;
2332 			best = bit;
2333 			while (x & bit) {
2334 				if (sc->sc_intr_stat[x] <
2335 				    sc->sc_intr_stat[best]) {
2336 					best = x;
2337 				}
2338 				x++;
2339 			}
2340 			break;
2341 		}
2342 		bit >>= 1;
2343 	}
2344 
2345 	sc->sc_intr_stat[best]++;
2346 	xfer->qh_pos = best;
2347 
2348 	DPRINTFN(3, "best=%d interval=%d\n",
2349 	    best, xfer->interval);
2350 }
2351 
2352 static void
2353 ehci_device_intr_close(struct usb_xfer *xfer)
2354 {
2355 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2356 
2357 	sc->sc_intr_stat[xfer->qh_pos]--;
2358 
2359 	ehci_device_done(xfer, USB_ERR_CANCELLED);
2360 
2361 	/* bandwidth must be freed after device done */
2362 	usb_hs_bandwidth_free(xfer);
2363 }
2364 
2365 static void
2366 ehci_device_intr_enter(struct usb_xfer *xfer)
2367 {
2368 	return;
2369 }
2370 
2371 static void
2372 ehci_device_intr_start(struct usb_xfer *xfer)
2373 {
2374 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2375 
2376 	/* setup TD's and QH */
2377 	ehci_setup_standard_chain(xfer, &sc->sc_intr_p_last[xfer->qh_pos]);
2378 
2379 	/* put transfer on interrupt queue */
2380 	ehci_transfer_intr_enqueue(xfer);
2381 }
2382 
2383 struct usb_pipe_methods ehci_device_intr_methods =
2384 {
2385 	.open = ehci_device_intr_open,
2386 	.close = ehci_device_intr_close,
2387 	.enter = ehci_device_intr_enter,
2388 	.start = ehci_device_intr_start,
2389 };
2390 
2391 /*------------------------------------------------------------------------*
2392  * ehci full speed isochronous support
2393  *------------------------------------------------------------------------*/
2394 static void
2395 ehci_device_isoc_fs_open(struct usb_xfer *xfer)
2396 {
2397 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2398 	ehci_sitd_t *td;
2399 	uint32_t sitd_portaddr;
2400 	uint8_t ds;
2401 
2402 	sitd_portaddr =
2403 	    EHCI_SITD_SET_ADDR(xfer->address) |
2404 	    EHCI_SITD_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)) |
2405 	    EHCI_SITD_SET_HUBA(xfer->xroot->udev->hs_hub_addr) |
2406 	    EHCI_SITD_SET_PORT(xfer->xroot->udev->hs_port_no);
2407 
2408 	if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN)
2409 		sitd_portaddr |= EHCI_SITD_SET_DIR_IN;
2410 
2411 	sitd_portaddr = htohc32(sc, sitd_portaddr);
2412 
2413 	/* initialize all TD's */
2414 
2415 	for (ds = 0; ds != 2; ds++) {
2416 
2417 		for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2418 
2419 			td->sitd_portaddr = sitd_portaddr;
2420 
2421 			/*
2422 			 * TODO: make some kind of automatic
2423 			 * SMASK/CMASK selection based on micro-frame
2424 			 * usage
2425 			 *
2426 			 * micro-frame usage (8 microframes per 1ms)
2427 			 */
2428 			td->sitd_back = htohc32(sc, EHCI_LINK_TERMINATE);
2429 
2430 			usb_pc_cpu_flush(td->page_cache);
2431 		}
2432 	}
2433 }
2434 
2435 static void
2436 ehci_device_isoc_fs_close(struct usb_xfer *xfer)
2437 {
2438 	ehci_device_done(xfer, USB_ERR_CANCELLED);
2439 }
2440 
2441 static void
2442 ehci_device_isoc_fs_enter(struct usb_xfer *xfer)
2443 {
2444 	struct usb_page_search buf_res;
2445 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2446 	ehci_sitd_t *td;
2447 	ehci_sitd_t *td_last = NULL;
2448 	ehci_sitd_t **pp_last;
2449 	uint32_t *plen;
2450 	uint32_t buf_offset;
2451 	uint32_t nframes;
2452 	uint32_t temp;
2453 	uint32_t sitd_mask;
2454 	uint16_t tlen;
2455 	uint8_t sa;
2456 	uint8_t sb;
2457 
2458 #ifdef USB_DEBUG
2459 	uint8_t once = 1;
2460 
2461 #endif
2462 
2463 	DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
2464 	    xfer, xfer->endpoint->isoc_next, xfer->nframes);
2465 
2466 	/* get the current frame index */
2467 
2468 	nframes = EOREAD4(sc, EHCI_FRINDEX) / 8;
2469 
2470 	/*
2471 	 * check if the frame index is within the window where the frames
2472 	 * will be inserted
2473 	 */
2474 	buf_offset = (nframes - xfer->endpoint->isoc_next) &
2475 	    (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2476 
2477 	if ((xfer->endpoint->is_synced == 0) ||
2478 	    (buf_offset < xfer->nframes)) {
2479 		/*
2480 		 * If there is data underflow or the pipe queue is empty we
2481 		 * schedule the transfer a few frames ahead of the current
2482 		 * frame position. Else two isochronous transfers might
2483 		 * overlap.
2484 		 */
2485 		xfer->endpoint->isoc_next = (nframes + 3) &
2486 		    (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2487 		xfer->endpoint->is_synced = 1;
2488 		DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2489 	}
2490 	/*
2491 	 * compute how many milliseconds the insertion is ahead of the
2492 	 * current frame position:
2493 	 */
2494 	buf_offset = (xfer->endpoint->isoc_next - nframes) &
2495 	    (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2496 
2497 	/*
2498 	 * pre-compute when the isochronous transfer will be finished:
2499 	 */
2500 	xfer->isoc_time_complete =
2501 	    usb_isoc_time_expand(&sc->sc_bus, nframes) +
2502 	    buf_offset + xfer->nframes;
2503 
2504 	/* get the real number of frames */
2505 
2506 	nframes = xfer->nframes;
2507 
2508 	buf_offset = 0;
2509 
2510 	plen = xfer->frlengths;
2511 
2512 	/* toggle the DMA set we are using */
2513 	xfer->flags_int.curr_dma_set ^= 1;
2514 
2515 	/* get next DMA set */
2516 	td = xfer->td_start[xfer->flags_int.curr_dma_set];
2517 	xfer->td_transfer_first = td;
2518 
2519 	pp_last = &sc->sc_isoc_fs_p_last[xfer->endpoint->isoc_next];
2520 
2521 	/* store starting position */
2522 
2523 	xfer->qh_pos = xfer->endpoint->isoc_next;
2524 
2525 	while (nframes--) {
2526 		if (td == NULL) {
2527 			panic("%s:%d: out of TD's\n",
2528 			    __FUNCTION__, __LINE__);
2529 		}
2530 		if (pp_last >= &sc->sc_isoc_fs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT])
2531 			pp_last = &sc->sc_isoc_fs_p_last[0];
2532 
2533 		/* reuse sitd_portaddr and sitd_back from last transfer */
2534 
2535 		if (*plen > xfer->max_frame_size) {
2536 #ifdef USB_DEBUG
2537 			if (once) {
2538 				once = 0;
2539 				printf("%s: frame length(%d) exceeds %d "
2540 				    "bytes (frame truncated)\n",
2541 				    __FUNCTION__, *plen,
2542 				    xfer->max_frame_size);
2543 			}
2544 #endif
2545 			*plen = xfer->max_frame_size;
2546 		}
2547 
2548 		/* allocate a slot */
2549 
2550 		sa = usbd_fs_isoc_schedule_alloc_slot(xfer,
2551 		    xfer->isoc_time_complete - nframes - 1);
2552 
2553 		if (sa == 255) {
2554 			/*
2555 			 * Schedule is FULL, set length to zero:
2556 			 */
2557 
2558 			*plen = 0;
2559 			sa = USB_FS_ISOC_UFRAME_MAX - 1;
2560 		}
2561 		if (*plen) {
2562 			/*
2563 			 * only call "usbd_get_page()" when we have a
2564 			 * non-zero length
2565 			 */
2566 			usbd_get_page(xfer->frbuffers, buf_offset, &buf_res);
2567 			td->sitd_bp[0] = htohc32(sc, buf_res.physaddr);
2568 			buf_offset += *plen;
2569 			/*
2570 			 * NOTE: We need to subtract one from the offset so
2571 			 * that we are on a valid page!
2572 			 */
2573 			usbd_get_page(xfer->frbuffers, buf_offset - 1,
2574 			    &buf_res);
2575 			temp = buf_res.physaddr & ~0xFFF;
2576 		} else {
2577 			td->sitd_bp[0] = 0;
2578 			temp = 0;
2579 		}
2580 
2581 		if (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) {
2582 			tlen = *plen;
2583 			if (tlen <= 188) {
2584 				temp |= 1;	/* T-count = 1, TP = ALL */
2585 				tlen = 1;
2586 			} else {
2587 				tlen += 187;
2588 				tlen /= 188;
2589 				temp |= tlen;	/* T-count = [1..6] */
2590 				temp |= 8;	/* TP = Begin */
2591 			}
2592 
2593 			tlen += sa;
2594 
2595 			if (tlen >= 8) {
2596 				sb = 0;
2597 			} else {
2598 				sb = (1 << tlen);
2599 			}
2600 
2601 			sa = (1 << sa);
2602 			sa = (sb - sa) & 0x3F;
2603 			sb = 0;
2604 		} else {
2605 			sb = (-(4 << sa)) & 0xFE;
2606 			sa = (1 << sa) & 0x3F;
2607 		}
2608 
2609 		sitd_mask = (EHCI_SITD_SET_SMASK(sa) |
2610 		    EHCI_SITD_SET_CMASK(sb));
2611 
2612 		td->sitd_bp[1] = htohc32(sc, temp);
2613 
2614 		td->sitd_mask = htohc32(sc, sitd_mask);
2615 
2616 		if (nframes == 0) {
2617 			td->sitd_status = htohc32(sc,
2618 			    EHCI_SITD_IOC |
2619 			    EHCI_SITD_ACTIVE |
2620 			    EHCI_SITD_SET_LEN(*plen));
2621 		} else {
2622 			td->sitd_status = htohc32(sc,
2623 			    EHCI_SITD_ACTIVE |
2624 			    EHCI_SITD_SET_LEN(*plen));
2625 		}
2626 		usb_pc_cpu_flush(td->page_cache);
2627 
2628 #ifdef USB_DEBUG
2629 		if (ehcidebug > 15) {
2630 			DPRINTF("FS-TD %d\n", nframes);
2631 			ehci_dump_sitd(sc, td);
2632 		}
2633 #endif
2634 		/* insert TD into schedule */
2635 		EHCI_APPEND_FS_TD(td, *pp_last);
2636 		pp_last++;
2637 
2638 		plen++;
2639 		td_last = td;
2640 		td = td->obj_next;
2641 	}
2642 
2643 	xfer->td_transfer_last = td_last;
2644 
2645 	/* update isoc_next */
2646 	xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_fs_p_last[0]) &
2647 	    (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2648 
2649 	/*
2650 	 * We don't allow cancelling of the SPLIT transaction USB FULL
2651 	 * speed transfer, because it disturbs the bandwidth
2652 	 * computation algorithm.
2653 	 */
2654 	xfer->flags_int.can_cancel_immed = 0;
2655 }
2656 
2657 static void
2658 ehci_device_isoc_fs_start(struct usb_xfer *xfer)
2659 {
2660 	/*
2661 	 * We don't allow cancelling of the SPLIT transaction USB FULL
2662 	 * speed transfer, because it disturbs the bandwidth
2663 	 * computation algorithm.
2664 	 */
2665 	xfer->flags_int.can_cancel_immed = 0;
2666 
2667 	/* set a default timeout */
2668 	if (xfer->timeout == 0)
2669 		xfer->timeout = 500; /* ms */
2670 
2671 	/* put transfer on interrupt queue */
2672 	ehci_transfer_intr_enqueue(xfer);
2673 }
2674 
2675 struct usb_pipe_methods ehci_device_isoc_fs_methods =
2676 {
2677 	.open = ehci_device_isoc_fs_open,
2678 	.close = ehci_device_isoc_fs_close,
2679 	.enter = ehci_device_isoc_fs_enter,
2680 	.start = ehci_device_isoc_fs_start,
2681 };
2682 
2683 /*------------------------------------------------------------------------*
2684  * ehci high speed isochronous support
2685  *------------------------------------------------------------------------*/
2686 static void
2687 ehci_device_isoc_hs_open(struct usb_xfer *xfer)
2688 {
2689 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2690 	ehci_itd_t *td;
2691 	uint32_t temp;
2692 	uint8_t ds;
2693 
2694 	usb_hs_bandwidth_alloc(xfer);
2695 
2696 	/* initialize all TD's */
2697 
2698 	for (ds = 0; ds != 2; ds++) {
2699 
2700 		for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2701 
2702 			/* set TD inactive */
2703 			td->itd_status[0] = 0;
2704 			td->itd_status[1] = 0;
2705 			td->itd_status[2] = 0;
2706 			td->itd_status[3] = 0;
2707 			td->itd_status[4] = 0;
2708 			td->itd_status[5] = 0;
2709 			td->itd_status[6] = 0;
2710 			td->itd_status[7] = 0;
2711 
2712 			/* set endpoint and address */
2713 			td->itd_bp[0] = htohc32(sc,
2714 			    EHCI_ITD_SET_ADDR(xfer->address) |
2715 			    EHCI_ITD_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)));
2716 
2717 			temp =
2718 			    EHCI_ITD_SET_MPL(xfer->max_packet_size & 0x7FF);
2719 
2720 			/* set direction */
2721 			if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) {
2722 				temp |= EHCI_ITD_SET_DIR_IN;
2723 			}
2724 			/* set maximum packet size */
2725 			td->itd_bp[1] = htohc32(sc, temp);
2726 
2727 			/* set transfer multiplier */
2728 			td->itd_bp[2] = htohc32(sc, xfer->max_packet_count & 3);
2729 
2730 			usb_pc_cpu_flush(td->page_cache);
2731 		}
2732 	}
2733 }
2734 
2735 static void
2736 ehci_device_isoc_hs_close(struct usb_xfer *xfer)
2737 {
2738 	ehci_device_done(xfer, USB_ERR_CANCELLED);
2739 
2740 	/* bandwidth must be freed after device done */
2741 	usb_hs_bandwidth_free(xfer);
2742 }
2743 
2744 static void
2745 ehci_device_isoc_hs_enter(struct usb_xfer *xfer)
2746 {
2747 	struct usb_page_search buf_res;
2748 	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2749 	ehci_itd_t *td;
2750 	ehci_itd_t *td_last = NULL;
2751 	ehci_itd_t **pp_last;
2752 	bus_size_t page_addr;
2753 	uint32_t *plen;
2754 	uint32_t status;
2755 	uint32_t buf_offset;
2756 	uint32_t nframes;
2757 	uint32_t itd_offset[8 + 1];
2758 	uint8_t x;
2759 	uint8_t td_no;
2760 	uint8_t page_no;
2761 	uint8_t shift = usbd_xfer_get_fps_shift(xfer);
2762 
2763 #ifdef USB_DEBUG
2764 	uint8_t once = 1;
2765 
2766 #endif
2767 
2768 	DPRINTFN(6, "xfer=%p next=%d nframes=%d shift=%d\n",
2769 	    xfer, xfer->endpoint->isoc_next, xfer->nframes, (int)shift);
2770 
2771 	/* get the current frame index */
2772 
2773 	nframes = EOREAD4(sc, EHCI_FRINDEX) / 8;
2774 
2775 	/*
2776 	 * check if the frame index is within the window where the frames
2777 	 * will be inserted
2778 	 */
2779 	buf_offset = (nframes - xfer->endpoint->isoc_next) &
2780 	    (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2781 
2782 	if ((xfer->endpoint->is_synced == 0) ||
2783 	    (buf_offset < (((xfer->nframes << shift) + 7) / 8))) {
2784 		/*
2785 		 * If there is data underflow or the pipe queue is empty we
2786 		 * schedule the transfer a few frames ahead of the current
2787 		 * frame position. Else two isochronous transfers might
2788 		 * overlap.
2789 		 */
2790 		xfer->endpoint->isoc_next = (nframes + 3) &
2791 		    (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2792 		xfer->endpoint->is_synced = 1;
2793 		DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2794 	}
2795 	/*
2796 	 * compute how many milliseconds the insertion is ahead of the
2797 	 * current frame position:
2798 	 */
2799 	buf_offset = (xfer->endpoint->isoc_next - nframes) &
2800 	    (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2801 
2802 	/*
2803 	 * pre-compute when the isochronous transfer will be finished:
2804 	 */
2805 	xfer->isoc_time_complete =
2806 	    usb_isoc_time_expand(&sc->sc_bus, nframes) + buf_offset +
2807 	    (((xfer->nframes << shift) + 7) / 8);
2808 
2809 	/* get the real number of frames */
2810 
2811 	nframes = xfer->nframes;
2812 
2813 	buf_offset = 0;
2814 	td_no = 0;
2815 
2816 	plen = xfer->frlengths;
2817 
2818 	/* toggle the DMA set we are using */
2819 	xfer->flags_int.curr_dma_set ^= 1;
2820 
2821 	/* get next DMA set */
2822 	td = xfer->td_start[xfer->flags_int.curr_dma_set];
2823 	xfer->td_transfer_first = td;
2824 
2825 	pp_last = &sc->sc_isoc_hs_p_last[xfer->endpoint->isoc_next];
2826 
2827 	/* store starting position */
2828 
2829 	xfer->qh_pos = xfer->endpoint->isoc_next;
2830 
2831 	while (nframes) {
2832 		if (td == NULL) {
2833 			panic("%s:%d: out of TD's\n",
2834 			    __FUNCTION__, __LINE__);
2835 		}
2836 		if (pp_last >= &sc->sc_isoc_hs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2837 			pp_last = &sc->sc_isoc_hs_p_last[0];
2838 		}
2839 		/* range check */
2840 		if (*plen > xfer->max_frame_size) {
2841 #ifdef USB_DEBUG
2842 			if (once) {
2843 				once = 0;
2844 				printf("%s: frame length(%d) exceeds %d bytes "
2845 				    "(frame truncated)\n",
2846 				    __FUNCTION__, *plen, xfer->max_frame_size);
2847 			}
2848 #endif
2849 			*plen = xfer->max_frame_size;
2850 		}
2851 
2852 		if (xfer->endpoint->usb_smask & (1 << td_no)) {
2853 			status = (EHCI_ITD_SET_LEN(*plen) |
2854 			    EHCI_ITD_ACTIVE |
2855 			    EHCI_ITD_SET_PG(0));
2856 			td->itd_status[td_no] = htohc32(sc, status);
2857 			itd_offset[td_no] = buf_offset;
2858 			buf_offset += *plen;
2859 			plen++;
2860 			nframes --;
2861 		} else {
2862 			td->itd_status[td_no] = 0;	/* not active */
2863 			itd_offset[td_no] = buf_offset;
2864 		}
2865 
2866 		td_no++;
2867 
2868 		if ((td_no == 8) || (nframes == 0)) {
2869 
2870 			/* the rest of the transfers are not active, if any */
2871 			for (x = td_no; x != 8; x++) {
2872 				td->itd_status[x] = 0;	/* not active */
2873 			}
2874 
2875 			/* check if there is any data to be transferred */
2876 			if (itd_offset[0] != buf_offset) {
2877 				page_no = 0;
2878 				itd_offset[td_no] = buf_offset;
2879 
2880 				/* get first page offset */
2881 				usbd_get_page(xfer->frbuffers, itd_offset[0], &buf_res);
2882 				/* get page address */
2883 				page_addr = buf_res.physaddr & ~0xFFF;
2884 				/* update page address */
2885 				td->itd_bp[0] &= htohc32(sc, 0xFFF);
2886 				td->itd_bp[0] |= htohc32(sc, page_addr);
2887 
2888 				for (x = 0; x != td_no; x++) {
2889 					/* set page number and page offset */
2890 					status = (EHCI_ITD_SET_PG(page_no) |
2891 					    (buf_res.physaddr & 0xFFF));
2892 					td->itd_status[x] |= htohc32(sc, status);
2893 
2894 					/* get next page offset */
2895 					if (itd_offset[x + 1] == buf_offset) {
2896 						/*
2897 						 * We subtract one so that
2898 						 * we don't go off the last
2899 						 * page!
2900 						 */
2901 						usbd_get_page(xfer->frbuffers, buf_offset - 1, &buf_res);
2902 					} else {
2903 						usbd_get_page(xfer->frbuffers, itd_offset[x + 1], &buf_res);
2904 					}
2905 
2906 					/* check if we need a new page */
2907 					if ((buf_res.physaddr ^ page_addr) & ~0xFFF) {
2908 						/* new page needed */
2909 						page_addr = buf_res.physaddr & ~0xFFF;
2910 						if (page_no == 6) {
2911 							panic("%s: too many pages\n", __FUNCTION__);
2912 						}
2913 						page_no++;
2914 						/* update page address */
2915 						td->itd_bp[page_no] &= htohc32(sc, 0xFFF);
2916 						td->itd_bp[page_no] |= htohc32(sc, page_addr);
2917 					}
2918 				}
2919 			}
2920 			/* set IOC bit if we are complete */
2921 			if (nframes == 0) {
2922 				td->itd_status[td_no - 1] |= htohc32(sc, EHCI_ITD_IOC);
2923 			}
2924 			usb_pc_cpu_flush(td->page_cache);
2925 #ifdef USB_DEBUG
2926 			if (ehcidebug > 15) {
2927 				DPRINTF("HS-TD %d\n", nframes);
2928 				ehci_dump_itd(sc, td);
2929 			}
2930 #endif
2931 			/* insert TD into schedule */
2932 			EHCI_APPEND_HS_TD(td, *pp_last);
2933 			pp_last++;
2934 
2935 			td_no = 0;
2936 			td_last = td;
2937 			td = td->obj_next;
2938 		}
2939 	}
2940 
2941 	xfer->td_transfer_last = td_last;
2942 
2943 	/* update isoc_next */
2944 	xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_hs_p_last[0]) &
2945 	    (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2946 }
2947 
2948 static void
2949 ehci_device_isoc_hs_start(struct usb_xfer *xfer)
2950 {
2951 	/* put transfer on interrupt queue */
2952 	ehci_transfer_intr_enqueue(xfer);
2953 }
2954 
2955 struct usb_pipe_methods ehci_device_isoc_hs_methods =
2956 {
2957 	.open = ehci_device_isoc_hs_open,
2958 	.close = ehci_device_isoc_hs_close,
2959 	.enter = ehci_device_isoc_hs_enter,
2960 	.start = ehci_device_isoc_hs_start,
2961 };
2962 
2963 /*------------------------------------------------------------------------*
2964  * ehci root control support
2965  *------------------------------------------------------------------------*
2966  * Simulate a hardware hub by handling all the necessary requests.
2967  *------------------------------------------------------------------------*/
2968 
2969 static const
2970 struct usb_device_descriptor ehci_devd =
2971 {
2972 	sizeof(struct usb_device_descriptor),
2973 	UDESC_DEVICE,			/* type */
2974 	{0x00, 0x02},			/* USB version */
2975 	UDCLASS_HUB,			/* class */
2976 	UDSUBCLASS_HUB,			/* subclass */
2977 	UDPROTO_HSHUBSTT,		/* protocol */
2978 	64,				/* max packet */
2979 	{0}, {0}, {0x00, 0x01},		/* device id */
2980 	1, 2, 0,			/* string indicies */
2981 	1				/* # of configurations */
2982 };
2983 
2984 static const
2985 struct usb_device_qualifier ehci_odevd =
2986 {
2987 	sizeof(struct usb_device_qualifier),
2988 	UDESC_DEVICE_QUALIFIER,		/* type */
2989 	{0x00, 0x02},			/* USB version */
2990 	UDCLASS_HUB,			/* class */
2991 	UDSUBCLASS_HUB,			/* subclass */
2992 	UDPROTO_FSHUB,			/* protocol */
2993 	0,				/* max packet */
2994 	0,				/* # of configurations */
2995 	0
2996 };
2997 
2998 static const struct ehci_config_desc ehci_confd = {
2999 	.confd = {
3000 		.bLength = sizeof(struct usb_config_descriptor),
3001 		.bDescriptorType = UDESC_CONFIG,
3002 		.wTotalLength[0] = sizeof(ehci_confd),
3003 		.bNumInterface = 1,
3004 		.bConfigurationValue = 1,
3005 		.iConfiguration = 0,
3006 		.bmAttributes = UC_SELF_POWERED,
3007 		.bMaxPower = 0		/* max power */
3008 	},
3009 	.ifcd = {
3010 		.bLength = sizeof(struct usb_interface_descriptor),
3011 		.bDescriptorType = UDESC_INTERFACE,
3012 		.bNumEndpoints = 1,
3013 		.bInterfaceClass = UICLASS_HUB,
3014 		.bInterfaceSubClass = UISUBCLASS_HUB,
3015 		.bInterfaceProtocol = 0,
3016 	},
3017 	.endpd = {
3018 		.bLength = sizeof(struct usb_endpoint_descriptor),
3019 		.bDescriptorType = UDESC_ENDPOINT,
3020 		.bEndpointAddress = UE_DIR_IN | EHCI_INTR_ENDPT,
3021 		.bmAttributes = UE_INTERRUPT,
3022 		.wMaxPacketSize[0] = 8,	/* max packet (63 ports) */
3023 		.bInterval = 255,
3024 	},
3025 };
3026 
3027 static const
3028 struct usb_hub_descriptor ehci_hubd =
3029 {
3030 	.bDescLength = 0,		/* dynamic length */
3031 	.bDescriptorType = UDESC_HUB,
3032 };
3033 
3034 static void
3035 ehci_disown(ehci_softc_t *sc, uint16_t index, uint8_t lowspeed)
3036 {
3037 	uint32_t port;
3038 	uint32_t v;
3039 
3040 	DPRINTF("index=%d lowspeed=%d\n", index, lowspeed);
3041 
3042 	port = EHCI_PORTSC(index);
3043 	v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3044 	EOWRITE4(sc, port, v | EHCI_PS_PO);
3045 }
3046 
3047 static usb_error_t
3048 ehci_roothub_exec(struct usb_device *udev,
3049     struct usb_device_request *req, const void **pptr, uint16_t *plength)
3050 {
3051 	ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3052 	const char *str_ptr;
3053 	const void *ptr;
3054 	uint32_t port;
3055 	uint32_t v;
3056 	uint16_t len;
3057 	uint16_t i;
3058 	uint16_t value;
3059 	uint16_t index;
3060 	usb_error_t err;
3061 
3062 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3063 
3064 	/* buffer reset */
3065 	ptr = (const void *)&sc->sc_hub_desc;
3066 	len = 0;
3067 	err = 0;
3068 
3069 	value = UGETW(req->wValue);
3070 	index = UGETW(req->wIndex);
3071 
3072 	DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x "
3073 	    "wValue=0x%04x wIndex=0x%04x\n",
3074 	    req->bmRequestType, req->bRequest,
3075 	    UGETW(req->wLength), value, index);
3076 
3077 #define	C(x,y) ((x) | ((y) << 8))
3078 	switch (C(req->bRequest, req->bmRequestType)) {
3079 	case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
3080 	case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
3081 	case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
3082 		/*
3083 		 * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
3084 		 * for the integrated root hub.
3085 		 */
3086 		break;
3087 	case C(UR_GET_CONFIG, UT_READ_DEVICE):
3088 		len = 1;
3089 		sc->sc_hub_desc.temp[0] = sc->sc_conf;
3090 		break;
3091 	case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
3092 		switch (value >> 8) {
3093 		case UDESC_DEVICE:
3094 			if ((value & 0xff) != 0) {
3095 				err = USB_ERR_IOERROR;
3096 				goto done;
3097 			}
3098 			len = sizeof(ehci_devd);
3099 			ptr = (const void *)&ehci_devd;
3100 			break;
3101 			/*
3102 			 * We can't really operate at another speed,
3103 			 * but the specification says we need this
3104 			 * descriptor:
3105 			 */
3106 		case UDESC_DEVICE_QUALIFIER:
3107 			if ((value & 0xff) != 0) {
3108 				err = USB_ERR_IOERROR;
3109 				goto done;
3110 			}
3111 			len = sizeof(ehci_odevd);
3112 			ptr = (const void *)&ehci_odevd;
3113 			break;
3114 
3115 		case UDESC_CONFIG:
3116 			if ((value & 0xff) != 0) {
3117 				err = USB_ERR_IOERROR;
3118 				goto done;
3119 			}
3120 			len = sizeof(ehci_confd);
3121 			ptr = (const void *)&ehci_confd;
3122 			break;
3123 
3124 		case UDESC_STRING:
3125 			switch (value & 0xff) {
3126 			case 0:	/* Language table */
3127 				str_ptr = "\001";
3128 				break;
3129 
3130 			case 1:	/* Vendor */
3131 				str_ptr = sc->sc_vendor;
3132 				break;
3133 
3134 			case 2:	/* Product */
3135 				str_ptr = "EHCI root HUB";
3136 				break;
3137 
3138 			default:
3139 				str_ptr = "";
3140 				break;
3141 			}
3142 
3143 			len = usb_make_str_desc(
3144 			    sc->sc_hub_desc.temp,
3145 			    sizeof(sc->sc_hub_desc.temp),
3146 			    str_ptr);
3147 			break;
3148 		default:
3149 			err = USB_ERR_IOERROR;
3150 			goto done;
3151 		}
3152 		break;
3153 	case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
3154 		len = 1;
3155 		sc->sc_hub_desc.temp[0] = 0;
3156 		break;
3157 	case C(UR_GET_STATUS, UT_READ_DEVICE):
3158 		len = 2;
3159 		USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED);
3160 		break;
3161 	case C(UR_GET_STATUS, UT_READ_INTERFACE):
3162 	case C(UR_GET_STATUS, UT_READ_ENDPOINT):
3163 		len = 2;
3164 		USETW(sc->sc_hub_desc.stat.wStatus, 0);
3165 		break;
3166 	case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
3167 		if (value >= EHCI_MAX_DEVICES) {
3168 			err = USB_ERR_IOERROR;
3169 			goto done;
3170 		}
3171 		sc->sc_addr = value;
3172 		break;
3173 	case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
3174 		if ((value != 0) && (value != 1)) {
3175 			err = USB_ERR_IOERROR;
3176 			goto done;
3177 		}
3178 		sc->sc_conf = value;
3179 		break;
3180 	case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
3181 		break;
3182 	case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
3183 	case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
3184 	case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
3185 		err = USB_ERR_IOERROR;
3186 		goto done;
3187 	case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
3188 		break;
3189 	case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
3190 		break;
3191 		/* Hub requests */
3192 	case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
3193 		break;
3194 	case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
3195 		DPRINTFN(9, "UR_CLEAR_PORT_FEATURE\n");
3196 
3197 		if ((index < 1) ||
3198 		    (index > sc->sc_noport)) {
3199 			err = USB_ERR_IOERROR;
3200 			goto done;
3201 		}
3202 		port = EHCI_PORTSC(index);
3203 		v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3204 		switch (value) {
3205 		case UHF_PORT_ENABLE:
3206 			EOWRITE4(sc, port, v & ~EHCI_PS_PE);
3207 			break;
3208 		case UHF_PORT_SUSPEND:
3209 			if ((v & EHCI_PS_SUSP) && (!(v & EHCI_PS_FPR))) {
3210 
3211 				/*
3212 				 * waking up a High Speed device is rather
3213 				 * complicated if
3214 				 */
3215 				EOWRITE4(sc, port, v | EHCI_PS_FPR);
3216 			}
3217 			/* wait 20ms for resume sequence to complete */
3218 			usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50);
3219 
3220 			EOWRITE4(sc, port, v & ~(EHCI_PS_SUSP |
3221 			    EHCI_PS_FPR | (3 << 10) /* High Speed */ ));
3222 
3223 			/* 4ms settle time */
3224 			usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
3225 			break;
3226 		case UHF_PORT_POWER:
3227 			EOWRITE4(sc, port, v & ~EHCI_PS_PP);
3228 			break;
3229 		case UHF_PORT_TEST:
3230 			DPRINTFN(3, "clear port test "
3231 			    "%d\n", index);
3232 			break;
3233 		case UHF_PORT_INDICATOR:
3234 			DPRINTFN(3, "clear port ind "
3235 			    "%d\n", index);
3236 			EOWRITE4(sc, port, v & ~EHCI_PS_PIC);
3237 			break;
3238 		case UHF_C_PORT_CONNECTION:
3239 			EOWRITE4(sc, port, v | EHCI_PS_CSC);
3240 			break;
3241 		case UHF_C_PORT_ENABLE:
3242 			EOWRITE4(sc, port, v | EHCI_PS_PEC);
3243 			break;
3244 		case UHF_C_PORT_SUSPEND:
3245 			EOWRITE4(sc, port, v | EHCI_PS_SUSP);
3246 			break;
3247 		case UHF_C_PORT_OVER_CURRENT:
3248 			EOWRITE4(sc, port, v | EHCI_PS_OCC);
3249 			break;
3250 		case UHF_C_PORT_RESET:
3251 			sc->sc_isreset = 0;
3252 			break;
3253 		default:
3254 			err = USB_ERR_IOERROR;
3255 			goto done;
3256 		}
3257 		break;
3258 	case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
3259 		if ((value & 0xff) != 0) {
3260 			err = USB_ERR_IOERROR;
3261 			goto done;
3262 		}
3263 		v = EREAD4(sc, EHCI_HCSPARAMS);
3264 
3265 		sc->sc_hub_desc.hubd = ehci_hubd;
3266 		sc->sc_hub_desc.hubd.bNbrPorts = sc->sc_noport;
3267 
3268 		if (EHCI_HCS_PPC(v))
3269 			i = UHD_PWR_INDIVIDUAL;
3270 		else
3271 			i = UHD_PWR_NO_SWITCH;
3272 
3273 		if (EHCI_HCS_P_INDICATOR(v))
3274 			i |= UHD_PORT_IND;
3275 
3276 		USETW(sc->sc_hub_desc.hubd.wHubCharacteristics, i);
3277 		/* XXX can't find out? */
3278 		sc->sc_hub_desc.hubd.bPwrOn2PwrGood = 200;
3279 		/* XXX don't know if ports are removable or not */
3280 		sc->sc_hub_desc.hubd.bDescLength =
3281 		    8 + ((sc->sc_noport + 7) / 8);
3282 		len = sc->sc_hub_desc.hubd.bDescLength;
3283 		break;
3284 	case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
3285 		len = 16;
3286 		memset(sc->sc_hub_desc.temp, 0, 16);
3287 		break;
3288 	case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
3289 		DPRINTFN(9, "get port status i=%d\n",
3290 		    index);
3291 		if ((index < 1) ||
3292 		    (index > sc->sc_noport)) {
3293 			err = USB_ERR_IOERROR;
3294 			goto done;
3295 		}
3296 		v = EOREAD4(sc, EHCI_PORTSC(index));
3297 		DPRINTFN(9, "port status=0x%04x\n", v);
3298 		if (sc->sc_flags & (EHCI_SCFLG_FORCESPEED | EHCI_SCFLG_TT)) {
3299 			if ((v & 0xc000000) == 0x8000000)
3300 				i = UPS_HIGH_SPEED;
3301 			else if ((v & 0xc000000) == 0x4000000)
3302 				i = UPS_LOW_SPEED;
3303 			else
3304 				i = 0;
3305 		} else {
3306 			i = UPS_HIGH_SPEED;
3307 		}
3308 		if (v & EHCI_PS_CS)
3309 			i |= UPS_CURRENT_CONNECT_STATUS;
3310 		if (v & EHCI_PS_PE)
3311 			i |= UPS_PORT_ENABLED;
3312 		if ((v & EHCI_PS_SUSP) && !(v & EHCI_PS_FPR))
3313 			i |= UPS_SUSPEND;
3314 		if (v & EHCI_PS_OCA)
3315 			i |= UPS_OVERCURRENT_INDICATOR;
3316 		if (v & EHCI_PS_PR)
3317 			i |= UPS_RESET;
3318 		if (v & EHCI_PS_PP)
3319 			i |= UPS_PORT_POWER;
3320 		USETW(sc->sc_hub_desc.ps.wPortStatus, i);
3321 		i = 0;
3322 		if (v & EHCI_PS_CSC)
3323 			i |= UPS_C_CONNECT_STATUS;
3324 		if (v & EHCI_PS_PEC)
3325 			i |= UPS_C_PORT_ENABLED;
3326 		if (v & EHCI_PS_OCC)
3327 			i |= UPS_C_OVERCURRENT_INDICATOR;
3328 		if (v & EHCI_PS_FPR)
3329 			i |= UPS_C_SUSPEND;
3330 		if (sc->sc_isreset)
3331 			i |= UPS_C_PORT_RESET;
3332 		USETW(sc->sc_hub_desc.ps.wPortChange, i);
3333 		len = sizeof(sc->sc_hub_desc.ps);
3334 		break;
3335 	case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
3336 		err = USB_ERR_IOERROR;
3337 		goto done;
3338 	case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
3339 		break;
3340 	case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
3341 		if ((index < 1) ||
3342 		    (index > sc->sc_noport)) {
3343 			err = USB_ERR_IOERROR;
3344 			goto done;
3345 		}
3346 		port = EHCI_PORTSC(index);
3347 		v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3348 		switch (value) {
3349 		case UHF_PORT_ENABLE:
3350 			EOWRITE4(sc, port, v | EHCI_PS_PE);
3351 			break;
3352 		case UHF_PORT_SUSPEND:
3353 			EOWRITE4(sc, port, v | EHCI_PS_SUSP);
3354 			break;
3355 		case UHF_PORT_RESET:
3356 			DPRINTFN(6, "reset port %d\n", index);
3357 #ifdef USB_DEBUG
3358 			if (ehcinohighspeed) {
3359 				/*
3360 				 * Connect USB device to companion
3361 				 * controller.
3362 				 */
3363 				ehci_disown(sc, index, 1);
3364 				break;
3365 			}
3366 #endif
3367 			if (EHCI_PS_IS_LOWSPEED(v) &&
3368 			    (sc->sc_flags & EHCI_SCFLG_TT) == 0) {
3369 				/* Low speed device, give up ownership. */
3370 				ehci_disown(sc, index, 1);
3371 				break;
3372 			}
3373 			/* Start reset sequence. */
3374 			v &= ~(EHCI_PS_PE | EHCI_PS_PR);
3375 			EOWRITE4(sc, port, v | EHCI_PS_PR);
3376 
3377 			/* Wait for reset to complete. */
3378 			usb_pause_mtx(&sc->sc_bus.bus_mtx,
3379 			    USB_MS_TO_TICKS(usb_port_root_reset_delay));
3380 
3381 			/* Terminate reset sequence. */
3382 			if (!(sc->sc_flags & EHCI_SCFLG_NORESTERM))
3383 				EOWRITE4(sc, port, v);
3384 
3385 			/* Wait for HC to complete reset. */
3386 			usb_pause_mtx(&sc->sc_bus.bus_mtx,
3387 			    USB_MS_TO_TICKS(EHCI_PORT_RESET_COMPLETE));
3388 
3389 			v = EOREAD4(sc, port);
3390 			DPRINTF("ehci after reset, status=0x%08x\n", v);
3391 			if (v & EHCI_PS_PR) {
3392 				device_printf(sc->sc_bus.bdev,
3393 				    "port reset timeout\n");
3394 				err = USB_ERR_TIMEOUT;
3395 				goto done;
3396 			}
3397 			if (!(v & EHCI_PS_PE) &&
3398 			    (sc->sc_flags & EHCI_SCFLG_TT) == 0) {
3399 				/* Not a high speed device, give up ownership.*/
3400 				ehci_disown(sc, index, 0);
3401 				break;
3402 			}
3403 			sc->sc_isreset = 1;
3404 			DPRINTF("ehci port %d reset, status = 0x%08x\n",
3405 			    index, v);
3406 			break;
3407 
3408 		case UHF_PORT_POWER:
3409 			DPRINTFN(3, "set port power %d\n", index);
3410 			EOWRITE4(sc, port, v | EHCI_PS_PP);
3411 			break;
3412 
3413 		case UHF_PORT_TEST:
3414 			DPRINTFN(3, "set port test %d\n", index);
3415 			break;
3416 
3417 		case UHF_PORT_INDICATOR:
3418 			DPRINTFN(3, "set port ind %d\n", index);
3419 			EOWRITE4(sc, port, v | EHCI_PS_PIC);
3420 			break;
3421 
3422 		default:
3423 			err = USB_ERR_IOERROR;
3424 			goto done;
3425 		}
3426 		break;
3427 	case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
3428 	case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
3429 	case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
3430 	case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
3431 		break;
3432 	default:
3433 		err = USB_ERR_IOERROR;
3434 		goto done;
3435 	}
3436 done:
3437 	*plength = len;
3438 	*pptr = ptr;
3439 	return (err);
3440 }
3441 
3442 static void
3443 ehci_xfer_setup(struct usb_setup_params *parm)
3444 {
3445 	struct usb_page_search page_info;
3446 	struct usb_page_cache *pc;
3447 	ehci_softc_t *sc;
3448 	struct usb_xfer *xfer;
3449 	void *last_obj;
3450 	uint32_t nqtd;
3451 	uint32_t nqh;
3452 	uint32_t nsitd;
3453 	uint32_t nitd;
3454 	uint32_t n;
3455 
3456 	sc = EHCI_BUS2SC(parm->udev->bus);
3457 	xfer = parm->curr_xfer;
3458 
3459 	nqtd = 0;
3460 	nqh = 0;
3461 	nsitd = 0;
3462 	nitd = 0;
3463 
3464 	/*
3465 	 * compute maximum number of some structures
3466 	 */
3467 	if (parm->methods == &ehci_device_ctrl_methods) {
3468 
3469 		/*
3470 		 * The proof for the "nqtd" formula is illustrated like
3471 		 * this:
3472 		 *
3473 		 * +------------------------------------+
3474 		 * |                                    |
3475 		 * |         |remainder ->              |
3476 		 * |   +-----+---+                      |
3477 		 * |   | xxx | x | frm 0                |
3478 		 * |   +-----+---++                     |
3479 		 * |   | xxx | xx | frm 1               |
3480 		 * |   +-----+----+                     |
3481 		 * |            ...                     |
3482 		 * +------------------------------------+
3483 		 *
3484 		 * "xxx" means a completely full USB transfer descriptor
3485 		 *
3486 		 * "x" and "xx" means a short USB packet
3487 		 *
3488 		 * For the remainder of an USB transfer modulo
3489 		 * "max_data_length" we need two USB transfer descriptors.
3490 		 * One to transfer the remaining data and one to finalise
3491 		 * with a zero length packet in case the "force_short_xfer"
3492 		 * flag is set. We only need two USB transfer descriptors in
3493 		 * the case where the transfer length of the first one is a
3494 		 * factor of "max_frame_size". The rest of the needed USB
3495 		 * transfer descriptors is given by the buffer size divided
3496 		 * by the maximum data payload.
3497 		 */
3498 		parm->hc_max_packet_size = 0x400;
3499 		parm->hc_max_packet_count = 1;
3500 		parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3501 		xfer->flags_int.bdma_enable = 1;
3502 
3503 		usbd_transfer_setup_sub(parm);
3504 
3505 		nqh = 1;
3506 		nqtd = ((2 * xfer->nframes) + 1	/* STATUS */
3507 		    + (xfer->max_data_length / xfer->max_hc_frame_size));
3508 
3509 	} else if (parm->methods == &ehci_device_bulk_methods) {
3510 
3511 		parm->hc_max_packet_size = 0x400;
3512 		parm->hc_max_packet_count = 1;
3513 		parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3514 		xfer->flags_int.bdma_enable = 1;
3515 
3516 		usbd_transfer_setup_sub(parm);
3517 
3518 		nqh = 1;
3519 		nqtd = ((2 * xfer->nframes)
3520 		    + (xfer->max_data_length / xfer->max_hc_frame_size));
3521 
3522 	} else if (parm->methods == &ehci_device_intr_methods) {
3523 
3524 		if (parm->speed == USB_SPEED_HIGH) {
3525 			parm->hc_max_packet_size = 0x400;
3526 			parm->hc_max_packet_count = 3;
3527 		} else if (parm->speed == USB_SPEED_FULL) {
3528 			parm->hc_max_packet_size = USB_FS_BYTES_PER_HS_UFRAME;
3529 			parm->hc_max_packet_count = 1;
3530 		} else {
3531 			parm->hc_max_packet_size = USB_FS_BYTES_PER_HS_UFRAME / 8;
3532 			parm->hc_max_packet_count = 1;
3533 		}
3534 
3535 		parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3536 		xfer->flags_int.bdma_enable = 1;
3537 
3538 		usbd_transfer_setup_sub(parm);
3539 
3540 		nqh = 1;
3541 		nqtd = ((2 * xfer->nframes)
3542 		    + (xfer->max_data_length / xfer->max_hc_frame_size));
3543 
3544 	} else if (parm->methods == &ehci_device_isoc_fs_methods) {
3545 
3546 		parm->hc_max_packet_size = 0x3FF;
3547 		parm->hc_max_packet_count = 1;
3548 		parm->hc_max_frame_size = 0x3FF;
3549 		xfer->flags_int.bdma_enable = 1;
3550 
3551 		usbd_transfer_setup_sub(parm);
3552 
3553 		nsitd = xfer->nframes;
3554 
3555 	} else if (parm->methods == &ehci_device_isoc_hs_methods) {
3556 
3557 		parm->hc_max_packet_size = 0x400;
3558 		parm->hc_max_packet_count = 3;
3559 		parm->hc_max_frame_size = 0xC00;
3560 		xfer->flags_int.bdma_enable = 1;
3561 
3562 		usbd_transfer_setup_sub(parm);
3563 
3564 		nitd = ((xfer->nframes + 7) / 8) <<
3565 		    usbd_xfer_get_fps_shift(xfer);
3566 
3567 	} else {
3568 
3569 		parm->hc_max_packet_size = 0x400;
3570 		parm->hc_max_packet_count = 1;
3571 		parm->hc_max_frame_size = 0x400;
3572 
3573 		usbd_transfer_setup_sub(parm);
3574 	}
3575 
3576 alloc_dma_set:
3577 
3578 	if (parm->err) {
3579 		return;
3580 	}
3581 	/*
3582 	 * Allocate queue heads and transfer descriptors
3583 	 */
3584 	last_obj = NULL;
3585 
3586 	if (usbd_transfer_setup_sub_malloc(
3587 	    parm, &pc, sizeof(ehci_itd_t),
3588 	    EHCI_ITD_ALIGN, nitd)) {
3589 		parm->err = USB_ERR_NOMEM;
3590 		return;
3591 	}
3592 	if (parm->buf) {
3593 		for (n = 0; n != nitd; n++) {
3594 			ehci_itd_t *td;
3595 
3596 			usbd_get_page(pc + n, 0, &page_info);
3597 
3598 			td = page_info.buffer;
3599 
3600 			/* init TD */
3601 			td->itd_self = htohc32(sc, page_info.physaddr | EHCI_LINK_ITD);
3602 			td->obj_next = last_obj;
3603 			td->page_cache = pc + n;
3604 
3605 			last_obj = td;
3606 
3607 			usb_pc_cpu_flush(pc + n);
3608 		}
3609 	}
3610 	if (usbd_transfer_setup_sub_malloc(
3611 	    parm, &pc, sizeof(ehci_sitd_t),
3612 	    EHCI_SITD_ALIGN, nsitd)) {
3613 		parm->err = USB_ERR_NOMEM;
3614 		return;
3615 	}
3616 	if (parm->buf) {
3617 		for (n = 0; n != nsitd; n++) {
3618 			ehci_sitd_t *td;
3619 
3620 			usbd_get_page(pc + n, 0, &page_info);
3621 
3622 			td = page_info.buffer;
3623 
3624 			/* init TD */
3625 			td->sitd_self = htohc32(sc, page_info.physaddr | EHCI_LINK_SITD);
3626 			td->obj_next = last_obj;
3627 			td->page_cache = pc + n;
3628 
3629 			last_obj = td;
3630 
3631 			usb_pc_cpu_flush(pc + n);
3632 		}
3633 	}
3634 	if (usbd_transfer_setup_sub_malloc(
3635 	    parm, &pc, sizeof(ehci_qtd_t),
3636 	    EHCI_QTD_ALIGN, nqtd)) {
3637 		parm->err = USB_ERR_NOMEM;
3638 		return;
3639 	}
3640 	if (parm->buf) {
3641 		for (n = 0; n != nqtd; n++) {
3642 			ehci_qtd_t *qtd;
3643 
3644 			usbd_get_page(pc + n, 0, &page_info);
3645 
3646 			qtd = page_info.buffer;
3647 
3648 			/* init TD */
3649 			qtd->qtd_self = htohc32(sc, page_info.physaddr);
3650 			qtd->obj_next = last_obj;
3651 			qtd->page_cache = pc + n;
3652 
3653 			last_obj = qtd;
3654 
3655 			usb_pc_cpu_flush(pc + n);
3656 		}
3657 	}
3658 	xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj;
3659 
3660 	last_obj = NULL;
3661 
3662 	if (usbd_transfer_setup_sub_malloc(
3663 	    parm, &pc, sizeof(ehci_qh_t),
3664 	    EHCI_QH_ALIGN, nqh)) {
3665 		parm->err = USB_ERR_NOMEM;
3666 		return;
3667 	}
3668 	if (parm->buf) {
3669 		for (n = 0; n != nqh; n++) {
3670 			ehci_qh_t *qh;
3671 
3672 			usbd_get_page(pc + n, 0, &page_info);
3673 
3674 			qh = page_info.buffer;
3675 
3676 			/* init QH */
3677 			qh->qh_self = htohc32(sc, page_info.physaddr | EHCI_LINK_QH);
3678 			qh->obj_next = last_obj;
3679 			qh->page_cache = pc + n;
3680 
3681 			last_obj = qh;
3682 
3683 			usb_pc_cpu_flush(pc + n);
3684 		}
3685 	}
3686 	xfer->qh_start[xfer->flags_int.curr_dma_set] = last_obj;
3687 
3688 	if (!xfer->flags_int.curr_dma_set) {
3689 		xfer->flags_int.curr_dma_set = 1;
3690 		goto alloc_dma_set;
3691 	}
3692 }
3693 
3694 static void
3695 ehci_xfer_unsetup(struct usb_xfer *xfer)
3696 {
3697 	return;
3698 }
3699 
3700 static void
3701 ehci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
3702     struct usb_endpoint *ep)
3703 {
3704 	ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3705 
3706 	DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d)\n",
3707 	    ep, udev->address,
3708 	    edesc->bEndpointAddress, udev->flags.usb_mode,
3709 	    sc->sc_addr);
3710 
3711 	if (udev->device_index != sc->sc_addr) {
3712 
3713 		if ((udev->speed != USB_SPEED_HIGH) &&
3714 		    ((udev->hs_hub_addr == 0) ||
3715 		    (udev->hs_port_no == 0) ||
3716 		    (udev->parent_hs_hub == NULL) ||
3717 		    (udev->parent_hs_hub->hub == NULL))) {
3718 			/* We need a transaction translator */
3719 			goto done;
3720 		}
3721 		switch (edesc->bmAttributes & UE_XFERTYPE) {
3722 		case UE_CONTROL:
3723 			ep->methods = &ehci_device_ctrl_methods;
3724 			break;
3725 		case UE_INTERRUPT:
3726 			ep->methods = &ehci_device_intr_methods;
3727 			break;
3728 		case UE_ISOCHRONOUS:
3729 			if (udev->speed == USB_SPEED_HIGH) {
3730 				ep->methods = &ehci_device_isoc_hs_methods;
3731 			} else if (udev->speed == USB_SPEED_FULL) {
3732 				ep->methods = &ehci_device_isoc_fs_methods;
3733 			}
3734 			break;
3735 		case UE_BULK:
3736 			ep->methods = &ehci_device_bulk_methods;
3737 			break;
3738 		default:
3739 			/* do nothing */
3740 			break;
3741 		}
3742 	}
3743 done:
3744 	return;
3745 }
3746 
3747 static void
3748 ehci_get_dma_delay(struct usb_device *udev, uint32_t *pus)
3749 {
3750 	/*
3751 	 * Wait until the hardware has finished any possible use of
3752 	 * the transfer descriptor(s) and QH
3753 	 */
3754 	*pus = (188);			/* microseconds */
3755 }
3756 
3757 static void
3758 ehci_device_resume(struct usb_device *udev)
3759 {
3760 	ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3761 	struct usb_xfer *xfer;
3762 	struct usb_pipe_methods *methods;
3763 
3764 	DPRINTF("\n");
3765 
3766 	USB_BUS_LOCK(udev->bus);
3767 
3768 	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3769 
3770 		if (xfer->xroot->udev == udev) {
3771 
3772 			methods = xfer->endpoint->methods;
3773 
3774 			if ((methods == &ehci_device_bulk_methods) ||
3775 			    (methods == &ehci_device_ctrl_methods)) {
3776 				EHCI_APPEND_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3777 				    sc->sc_async_p_last);
3778 			}
3779 			if (methods == &ehci_device_intr_methods) {
3780 				EHCI_APPEND_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3781 				    sc->sc_intr_p_last[xfer->qh_pos]);
3782 			}
3783 		}
3784 	}
3785 
3786 	USB_BUS_UNLOCK(udev->bus);
3787 
3788 	return;
3789 }
3790 
3791 static void
3792 ehci_device_suspend(struct usb_device *udev)
3793 {
3794 	ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3795 	struct usb_xfer *xfer;
3796 	struct usb_pipe_methods *methods;
3797 
3798 	DPRINTF("\n");
3799 
3800 	USB_BUS_LOCK(udev->bus);
3801 
3802 	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3803 
3804 		if (xfer->xroot->udev == udev) {
3805 
3806 			methods = xfer->endpoint->methods;
3807 
3808 			if ((methods == &ehci_device_bulk_methods) ||
3809 			    (methods == &ehci_device_ctrl_methods)) {
3810 				EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3811 				    sc->sc_async_p_last);
3812 			}
3813 			if (methods == &ehci_device_intr_methods) {
3814 				EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3815 				    sc->sc_intr_p_last[xfer->qh_pos]);
3816 			}
3817 		}
3818 	}
3819 
3820 	USB_BUS_UNLOCK(udev->bus);
3821 }
3822 
3823 static void
3824 ehci_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
3825 {
3826 	struct ehci_softc *sc = EHCI_BUS2SC(bus);
3827 
3828 	switch (state) {
3829 	case USB_HW_POWER_SUSPEND:
3830 	case USB_HW_POWER_SHUTDOWN:
3831 		ehci_suspend(sc);
3832 		break;
3833 	case USB_HW_POWER_RESUME:
3834 		ehci_resume(sc);
3835 		break;
3836 	default:
3837 		break;
3838 	}
3839 }
3840 
3841 static void
3842 ehci_set_hw_power(struct usb_bus *bus)
3843 {
3844 	ehci_softc_t *sc = EHCI_BUS2SC(bus);
3845 	uint32_t temp;
3846 	uint32_t flags;
3847 
3848 	DPRINTF("\n");
3849 
3850 	USB_BUS_LOCK(bus);
3851 
3852 	flags = bus->hw_power_state;
3853 
3854 	temp = EOREAD4(sc, EHCI_USBCMD);
3855 
3856 	temp &= ~(EHCI_CMD_ASE | EHCI_CMD_PSE);
3857 
3858 	if (flags & (USB_HW_POWER_CONTROL |
3859 	    USB_HW_POWER_BULK)) {
3860 		DPRINTF("Async is active\n");
3861 		temp |= EHCI_CMD_ASE;
3862 	}
3863 	if (flags & (USB_HW_POWER_INTERRUPT |
3864 	    USB_HW_POWER_ISOC)) {
3865 		DPRINTF("Periodic is active\n");
3866 		temp |= EHCI_CMD_PSE;
3867 	}
3868 	EOWRITE4(sc, EHCI_USBCMD, temp);
3869 
3870 	USB_BUS_UNLOCK(bus);
3871 
3872 	return;
3873 }
3874 
3875 struct usb_bus_methods ehci_bus_methods =
3876 {
3877 	.endpoint_init = ehci_ep_init,
3878 	.xfer_setup = ehci_xfer_setup,
3879 	.xfer_unsetup = ehci_xfer_unsetup,
3880 	.get_dma_delay = ehci_get_dma_delay,
3881 	.device_resume = ehci_device_resume,
3882 	.device_suspend = ehci_device_suspend,
3883 	.set_hw_power = ehci_set_hw_power,
3884 	.set_hw_power_sleep = ehci_set_hw_power_sleep,
3885 	.roothub_exec = ehci_roothub_exec,
3886 	.xfer_poll = ehci_do_poll,
3887 };
3888