xref: /freebsd/sys/dev/usb/controller/dwc_otg.c (revision 39ee7a7a6bdd1557b1c3532abf60d139798ac88b)
1 /* $FreeBSD$ */
2 /*-
3  * Copyright (c) 2015 Daisuke Aoyama. All rights reserved.
4  * Copyright (c) 2012-2015 Hans Petter Selasky. All rights reserved.
5  * Copyright (c) 2010-2011 Aleksandr Rybalko. All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 
29 /*
30  * This file contains the driver for the DesignWare series USB 2.0 OTG
31  * Controller.
32  */
33 
34 /*
35  * LIMITATION: Drivers must be bound to all OUT endpoints in the
36  * active configuration for this driver to work properly. Blocking any
37  * OUT endpoint will block all OUT endpoints including the control
38  * endpoint. Usually this is not a problem.
39  */
40 
41 /*
42  * NOTE: Writing to non-existing registers appears to cause an
43  * internal reset.
44  */
45 
46 #ifdef USB_GLOBAL_INCLUDE_FILE
47 #include USB_GLOBAL_INCLUDE_FILE
48 #else
49 #include <sys/stdint.h>
50 #include <sys/stddef.h>
51 #include <sys/param.h>
52 #include <sys/queue.h>
53 #include <sys/types.h>
54 #include <sys/systm.h>
55 #include <sys/kernel.h>
56 #include <sys/bus.h>
57 #include <sys/module.h>
58 #include <sys/lock.h>
59 #include <sys/mutex.h>
60 #include <sys/condvar.h>
61 #include <sys/sysctl.h>
62 #include <sys/sx.h>
63 #include <sys/unistd.h>
64 #include <sys/callout.h>
65 #include <sys/malloc.h>
66 #include <sys/priv.h>
67 
68 #include <dev/usb/usb.h>
69 #include <dev/usb/usbdi.h>
70 
71 #define	USB_DEBUG_VAR dwc_otg_debug
72 
73 #include <dev/usb/usb_core.h>
74 #include <dev/usb/usb_debug.h>
75 #include <dev/usb/usb_busdma.h>
76 #include <dev/usb/usb_process.h>
77 #include <dev/usb/usb_transfer.h>
78 #include <dev/usb/usb_device.h>
79 #include <dev/usb/usb_hub.h>
80 #include <dev/usb/usb_util.h>
81 
82 #include <dev/usb/usb_controller.h>
83 #include <dev/usb/usb_bus.h>
84 #endif			/* USB_GLOBAL_INCLUDE_FILE */
85 
86 #include <dev/usb/controller/dwc_otg.h>
87 #include <dev/usb/controller/dwc_otgreg.h>
88 
89 #define	DWC_OTG_BUS2SC(bus) \
90    ((struct dwc_otg_softc *)(((uint8_t *)(bus)) - \
91     ((uint8_t *)&(((struct dwc_otg_softc *)0)->sc_bus))))
92 
93 #define	DWC_OTG_PC2UDEV(pc) \
94    (USB_DMATAG_TO_XROOT((pc)->tag_parent)->udev)
95 
96 #define	DWC_OTG_MSK_GINT_ENABLED	\
97    (GINTMSK_ENUMDONEMSK |		\
98    GINTMSK_USBRSTMSK |			\
99    GINTMSK_USBSUSPMSK |			\
100    GINTMSK_IEPINTMSK |			\
101    GINTMSK_SESSREQINTMSK |		\
102    GINTMSK_RXFLVLMSK |			\
103    GINTMSK_HCHINTMSK |			\
104    GINTMSK_OTGINTMSK |			\
105    GINTMSK_PRTINTMSK)
106 
107 #define	DWC_OTG_MSK_GINT_THREAD_IRQ				\
108    (GINTSTS_USBRST | GINTSTS_ENUMDONE | GINTSTS_PRTINT |	\
109    GINTSTS_WKUPINT | GINTSTS_USBSUSP | GINTMSK_OTGINTMSK |	\
110    GINTSTS_SESSREQINT)
111 
112 #define	DWC_OTG_PHY_ULPI 0
113 #define	DWC_OTG_PHY_HSIC 1
114 #define	DWC_OTG_PHY_INTERNAL 2
115 
116 #ifndef DWC_OTG_PHY_DEFAULT
117 #define	DWC_OTG_PHY_DEFAULT DWC_OTG_PHY_ULPI
118 #endif
119 
120 static int dwc_otg_phy_type = DWC_OTG_PHY_DEFAULT;
121 
122 static SYSCTL_NODE(_hw_usb, OID_AUTO, dwc_otg, CTLFLAG_RW, 0, "USB DWC OTG");
123 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, phy_type, CTLFLAG_RDTUN,
124     &dwc_otg_phy_type, 0, "DWC OTG PHY TYPE - 0/1/2 - ULPI/HSIC/INTERNAL");
125 
126 #ifdef USB_DEBUG
127 static int dwc_otg_debug;
128 
129 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, debug, CTLFLAG_RWTUN,
130     &dwc_otg_debug, 0, "DWC OTG debug level");
131 #endif
132 
133 #define	DWC_OTG_INTR_ENDPT 1
134 
135 /* prototypes */
136 
137 static const struct usb_bus_methods dwc_otg_bus_methods;
138 static const struct usb_pipe_methods dwc_otg_device_non_isoc_methods;
139 static const struct usb_pipe_methods dwc_otg_device_isoc_methods;
140 
141 static dwc_otg_cmd_t dwc_otg_setup_rx;
142 static dwc_otg_cmd_t dwc_otg_data_rx;
143 static dwc_otg_cmd_t dwc_otg_data_tx;
144 static dwc_otg_cmd_t dwc_otg_data_tx_sync;
145 
146 static dwc_otg_cmd_t dwc_otg_host_setup_tx;
147 static dwc_otg_cmd_t dwc_otg_host_data_tx;
148 static dwc_otg_cmd_t dwc_otg_host_data_rx;
149 
150 static void dwc_otg_device_done(struct usb_xfer *, usb_error_t);
151 static void dwc_otg_do_poll(struct usb_bus *);
152 static void dwc_otg_standard_done(struct usb_xfer *);
153 static void dwc_otg_root_intr(struct dwc_otg_softc *);
154 static void dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *);
155 
156 /*
157  * Here is a configuration that the chip supports.
158  */
159 static const struct usb_hw_ep_profile dwc_otg_ep_profile[1] = {
160 
161 	[0] = {
162 		.max_in_frame_size = 64,/* fixed */
163 		.max_out_frame_size = 64,	/* fixed */
164 		.is_simplex = 1,
165 		.support_control = 1,
166 	}
167 };
168 
169 static void
170 dwc_otg_get_hw_ep_profile(struct usb_device *udev,
171     const struct usb_hw_ep_profile **ppf, uint8_t ep_addr)
172 {
173 	struct dwc_otg_softc *sc;
174 
175 	sc = DWC_OTG_BUS2SC(udev->bus);
176 
177 	if (ep_addr < sc->sc_dev_ep_max)
178 		*ppf = &sc->sc_hw_ep_profile[ep_addr].usb;
179 	else
180 		*ppf = NULL;
181 }
182 
183 static void
184 dwc_otg_tx_fifo_reset(struct dwc_otg_softc *sc, uint32_t value)
185 {
186 	uint32_t temp;
187 
188   	/* reset FIFO */
189 	DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL, value);
190 
191 	/* wait for reset to complete */
192 	for (temp = 0; temp != 16; temp++) {
193 		value = DWC_OTG_READ_4(sc, DOTG_GRSTCTL);
194 		if (!(value & (GRSTCTL_TXFFLSH | GRSTCTL_RXFFLSH)))
195 			break;
196 	}
197 }
198 
199 static int
200 dwc_otg_init_fifo(struct dwc_otg_softc *sc, uint8_t mode)
201 {
202 	struct dwc_otg_profile *pf;
203 	uint32_t fifo_size;
204 	uint32_t fifo_regs;
205 	uint32_t tx_start;
206 	uint8_t x;
207 
208 	fifo_size = sc->sc_fifo_size;
209 
210 	/*
211 	 * NOTE: Reserved fixed size area at end of RAM, which must
212 	 * not be allocated to the FIFOs:
213 	 */
214 	fifo_regs = 4 * 16;
215 
216 	if (fifo_size < fifo_regs) {
217 		DPRINTF("Too little FIFO\n");
218 		return (EINVAL);
219 	}
220 
221 	/* subtract FIFO regs from total once */
222 	fifo_size -= fifo_regs;
223 
224 	/* split equally for IN and OUT */
225 	fifo_size /= 2;
226 
227 	/* Align to 4 bytes boundary (refer to PGM) */
228 	fifo_size &= ~3;
229 
230 	/* set global receive FIFO size */
231 	DWC_OTG_WRITE_4(sc, DOTG_GRXFSIZ, fifo_size / 4);
232 
233 	tx_start = fifo_size;
234 
235 	if (fifo_size < 64) {
236 		DPRINTFN(-1, "Not enough data space for EP0 FIFO.\n");
237 		return (EINVAL);
238 	}
239 
240 	if (mode == DWC_MODE_HOST) {
241 
242 		/* reset active endpoints */
243 		sc->sc_active_rx_ep = 0;
244 
245 		/* split equally for periodic and non-periodic */
246 		fifo_size /= 2;
247 
248 		DPRINTF("PTX/NPTX FIFO=%u\n", fifo_size);
249 
250 		/* align to 4 bytes boundary */
251 		fifo_size &= ~3;
252 
253 		DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
254 		    ((fifo_size / 4) << 16) |
255 		    (tx_start / 4));
256 
257 		tx_start += fifo_size;
258 
259 		for (x = 0; x != sc->sc_host_ch_max; x++) {
260 			/* enable all host interrupts */
261 			DWC_OTG_WRITE_4(sc, DOTG_HCINTMSK(x),
262 			    HCINT_DEFAULT_MASK);
263 		}
264 
265 		DWC_OTG_WRITE_4(sc, DOTG_HPTXFSIZ,
266 		    ((fifo_size / 4) << 16) |
267 		    (tx_start / 4));
268 
269 		/* reset host channel state */
270 		memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
271 
272 		/* enable all host channel interrupts */
273 		DWC_OTG_WRITE_4(sc, DOTG_HAINTMSK,
274 		    (1U << sc->sc_host_ch_max) - 1U);
275 	}
276 
277 	if (mode == DWC_MODE_DEVICE) {
278 
279 	    DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
280 		(0x10 << 16) | (tx_start / 4));
281 	    fifo_size -= 0x40;
282 	    tx_start += 0x40;
283 
284 	    /* setup control endpoint profile */
285 	    sc->sc_hw_ep_profile[0].usb = dwc_otg_ep_profile[0];
286 
287 	    /* reset active endpoints */
288 	    sc->sc_active_rx_ep = 1;
289 
290 	    for (x = 1; x != sc->sc_dev_ep_max; x++) {
291 
292 		pf = sc->sc_hw_ep_profile + x;
293 
294 		pf->usb.max_out_frame_size = 1024 * 3;
295 		pf->usb.is_simplex = 0;	/* assume duplex */
296 		pf->usb.support_bulk = 1;
297 		pf->usb.support_interrupt = 1;
298 		pf->usb.support_isochronous = 1;
299 		pf->usb.support_out = 1;
300 
301 		if (x < sc->sc_dev_in_ep_max) {
302 			uint32_t limit;
303 
304 			limit = (x == 1) ? MIN(DWC_OTG_TX_MAX_FIFO_SIZE,
305 			    DWC_OTG_MAX_TXN) : MIN(DWC_OTG_MAX_TXN / 2,
306 			    DWC_OTG_TX_MAX_FIFO_SIZE);
307 
308 			/* see if there is enough FIFO space */
309 			if (limit <= fifo_size) {
310 				pf->max_buffer = limit;
311 				pf->usb.support_in = 1;
312 			} else {
313 				limit = MIN(DWC_OTG_TX_MAX_FIFO_SIZE, 0x40);
314 				if (limit <= fifo_size) {
315 					pf->usb.support_in = 1;
316 				} else {
317 					pf->usb.is_simplex = 1;
318 					limit = 0;
319 				}
320 			}
321 			/* set FIFO size */
322 			DWC_OTG_WRITE_4(sc, DOTG_DIEPTXF(x),
323 			    ((limit / 4) << 16) | (tx_start / 4));
324 			tx_start += limit;
325 			fifo_size -= limit;
326 			pf->usb.max_in_frame_size = limit;
327 		} else {
328 			pf->usb.is_simplex = 1;
329 		}
330 
331 		DPRINTF("FIFO%d = IN:%d / OUT:%d\n", x,
332 		    pf->usb.max_in_frame_size,
333 		    pf->usb.max_out_frame_size);
334 	    }
335 	}
336 
337 	/* reset RX FIFO */
338 	dwc_otg_tx_fifo_reset(sc, GRSTCTL_RXFFLSH);
339 
340 	if (mode != DWC_MODE_OTG) {
341 		/* reset all TX FIFOs */
342 		dwc_otg_tx_fifo_reset(sc,
343 		    GRSTCTL_TXFIFO(0x10) |
344 		    GRSTCTL_TXFFLSH);
345 	} else {
346 		/* reset active endpoints */
347 		sc->sc_active_rx_ep = 0;
348 
349 		/* reset host channel state */
350 		memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
351 	}
352 	return (0);
353 }
354 
355 static void
356 dwc_otg_update_host_frame_interval(struct dwc_otg_softc *sc)
357 {
358 
359   /*
360    * Disabled until further. Assuming that the register is already
361    * programmed correctly by the boot loader.
362    */
363 #if 0
364 	uint32_t temp;
365 
366 	/* setup HOST frame interval register, based on existing value */
367 	temp = DWC_OTG_READ_4(sc, DOTG_HFIR) & HFIR_FRINT_MASK;
368 	if (temp >= 10000)
369 		temp /= 1000;
370 	else
371 		temp /= 125;
372 
373 	/* figure out nearest X-tal value */
374 	if (temp >= 54)
375 		temp = 60;	/* MHz */
376 	else if (temp >= 39)
377 		temp = 48;	/* MHz */
378 	else
379 		temp = 30;	/* MHz */
380 
381 	if (sc->sc_flags.status_high_speed)
382 		temp *= 125;
383 	else
384 		temp *= 1000;
385 
386 	DPRINTF("HFIR=0x%08x\n", temp);
387 
388 	DWC_OTG_WRITE_4(sc, DOTG_HFIR, temp);
389 #endif
390 }
391 
392 static void
393 dwc_otg_clocks_on(struct dwc_otg_softc *sc)
394 {
395 	if (sc->sc_flags.clocks_off &&
396 	    sc->sc_flags.port_powered) {
397 
398 		DPRINTFN(5, "\n");
399 
400 		/* TODO - platform specific */
401 
402 		sc->sc_flags.clocks_off = 0;
403 	}
404 }
405 
406 static void
407 dwc_otg_clocks_off(struct dwc_otg_softc *sc)
408 {
409 	if (!sc->sc_flags.clocks_off) {
410 
411 		DPRINTFN(5, "\n");
412 
413 		/* TODO - platform specific */
414 
415 		sc->sc_flags.clocks_off = 1;
416 	}
417 }
418 
419 static void
420 dwc_otg_pull_up(struct dwc_otg_softc *sc)
421 {
422 	uint32_t temp;
423 
424 	/* pullup D+, if possible */
425 
426 	if (!sc->sc_flags.d_pulled_up &&
427 	    sc->sc_flags.port_powered) {
428 		sc->sc_flags.d_pulled_up = 1;
429 
430 		temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
431 		temp &= ~DCTL_SFTDISCON;
432 		DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
433 	}
434 }
435 
436 static void
437 dwc_otg_pull_down(struct dwc_otg_softc *sc)
438 {
439 	uint32_t temp;
440 
441 	/* pulldown D+, if possible */
442 
443 	if (sc->sc_flags.d_pulled_up) {
444 		sc->sc_flags.d_pulled_up = 0;
445 
446 		temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
447 		temp |= DCTL_SFTDISCON;
448 		DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
449 	}
450 }
451 
452 static void
453 dwc_otg_enable_sof_irq(struct dwc_otg_softc *sc)
454 {
455 	/* In device mode we don't use the SOF interrupt */
456 	if (sc->sc_flags.status_device_mode != 0)
457 		return;
458 	/* Ensure the SOF interrupt is not disabled */
459 	sc->sc_needsof = 1;
460 	/* Check if the SOF interrupt is already enabled */
461 	if ((sc->sc_irq_mask & GINTMSK_SOFMSK) != 0)
462 		return;
463 	sc->sc_irq_mask |= GINTMSK_SOFMSK;
464 	DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
465 }
466 
467 static void
468 dwc_otg_resume_irq(struct dwc_otg_softc *sc)
469 {
470 	if (sc->sc_flags.status_suspend) {
471 		/* update status bits */
472 		sc->sc_flags.status_suspend = 0;
473 		sc->sc_flags.change_suspend = 1;
474 
475 		if (sc->sc_flags.status_device_mode) {
476 			/*
477 			 * Disable resume interrupt and enable suspend
478 			 * interrupt:
479 			 */
480 			sc->sc_irq_mask &= ~GINTMSK_WKUPINTMSK;
481 			sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
482 			DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
483 		}
484 
485 		/* complete root HUB interrupt endpoint */
486 		dwc_otg_root_intr(sc);
487 	}
488 }
489 
490 static void
491 dwc_otg_suspend_irq(struct dwc_otg_softc *sc)
492 {
493 	if (!sc->sc_flags.status_suspend) {
494 		/* update status bits */
495 		sc->sc_flags.status_suspend = 1;
496 		sc->sc_flags.change_suspend = 1;
497 
498 		if (sc->sc_flags.status_device_mode) {
499 			/*
500 			 * Disable suspend interrupt and enable resume
501 			 * interrupt:
502 			 */
503 			sc->sc_irq_mask &= ~GINTMSK_USBSUSPMSK;
504 			sc->sc_irq_mask |= GINTMSK_WKUPINTMSK;
505 			DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
506 		}
507 
508 		/* complete root HUB interrupt endpoint */
509 		dwc_otg_root_intr(sc);
510 	}
511 }
512 
513 static void
514 dwc_otg_wakeup_peer(struct dwc_otg_softc *sc)
515 {
516 	if (!sc->sc_flags.status_suspend)
517 		return;
518 
519 	DPRINTFN(5, "Remote wakeup\n");
520 
521 	if (sc->sc_flags.status_device_mode) {
522 		uint32_t temp;
523 
524 		/* enable remote wakeup signalling */
525 		temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
526 		temp |= DCTL_RMTWKUPSIG;
527 		DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
528 
529 		/* Wait 8ms for remote wakeup to complete. */
530 		usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 125);
531 
532 		temp &= ~DCTL_RMTWKUPSIG;
533 		DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
534 	} else {
535 		/* enable USB port */
536 		DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
537 
538 		/* wait 10ms */
539 		usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
540 
541 		/* resume port */
542 		sc->sc_hprt_val |= HPRT_PRTRES;
543 		DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
544 
545 		/* Wait 100ms for resume signalling to complete. */
546 		usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 10);
547 
548 		/* clear suspend and resume */
549 		sc->sc_hprt_val &= ~(HPRT_PRTSUSP | HPRT_PRTRES);
550 		DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
551 
552 		/* Wait 4ms */
553 		usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
554 	}
555 
556 	/* need to fake resume IRQ */
557 	dwc_otg_resume_irq(sc);
558 }
559 
560 static void
561 dwc_otg_set_address(struct dwc_otg_softc *sc, uint8_t addr)
562 {
563 	uint32_t temp;
564 
565 	DPRINTFN(5, "addr=%d\n", addr);
566 
567 	temp = DWC_OTG_READ_4(sc, DOTG_DCFG);
568 	temp &= ~DCFG_DEVADDR_SET(0x7F);
569 	temp |= DCFG_DEVADDR_SET(addr);
570 	DWC_OTG_WRITE_4(sc, DOTG_DCFG, temp);
571 }
572 
573 static void
574 dwc_otg_common_rx_ack(struct dwc_otg_softc *sc)
575 {
576 	DPRINTFN(5, "RX status clear\n");
577 
578 	/* enable RX FIFO level interrupt */
579 	sc->sc_irq_mask |= GINTMSK_RXFLVLMSK;
580 	DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
581 
582 	/* clear cached status */
583 	sc->sc_last_rx_status = 0;
584 }
585 
586 static void
587 dwc_otg_clear_hcint(struct dwc_otg_softc *sc, uint8_t x)
588 {
589 	uint32_t hcint;
590 
591 	/* clear all pending interrupts */
592 	hcint = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
593 	DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), hcint);
594 
595 	/* clear buffered interrupts */
596 	sc->sc_chan_state[x].hcint = 0;
597 }
598 
599 static uint8_t
600 dwc_otg_host_check_tx_fifo_empty(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
601 {
602 	uint32_t temp;
603 
604 	temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
605 
606 	if (td->ep_type == UE_ISOCHRONOUS) {
607 		/*
608 		 * NOTE: USB INTERRUPT transactions are executed like
609 		 * USB CONTROL transactions! See the setup standard
610 		 * chain function for more information.
611 		 */
612 		if (!(temp & GINTSTS_PTXFEMP)) {
613 			DPRINTF("Periodic TX FIFO is not empty\n");
614 			if (!(sc->sc_irq_mask & GINTMSK_PTXFEMPMSK)) {
615 				sc->sc_irq_mask |= GINTMSK_PTXFEMPMSK;
616 				DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
617 			}
618 			return (1);	/* busy */
619 		}
620 	} else {
621 		if (!(temp & GINTSTS_NPTXFEMP)) {
622 			DPRINTF("Non-periodic TX FIFO is not empty\n");
623 			if (!(sc->sc_irq_mask & GINTMSK_NPTXFEMPMSK)) {
624 				sc->sc_irq_mask |= GINTMSK_NPTXFEMPMSK;
625 				DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
626 			}
627 			return (1);	/* busy */
628 		}
629 	}
630 	return (0);	/* ready for transmit */
631 }
632 
633 static uint8_t
634 dwc_otg_host_channel_alloc(struct dwc_otg_softc *sc,
635     struct dwc_otg_td *td, uint8_t is_out)
636 {
637 	uint8_t x;
638 	uint8_t y;
639 	uint8_t z;
640 
641 	if (td->channel[0] < DWC_OTG_MAX_CHANNELS)
642 		return (0);		/* already allocated */
643 
644 	/* check if device is suspended */
645 	if (DWC_OTG_PC2UDEV(td->pc)->flags.self_suspended != 0)
646 		return (1);		/* busy - cannot transfer data */
647 
648 	/* compute needed TX FIFO size */
649 	if (is_out != 0) {
650 		if (dwc_otg_host_check_tx_fifo_empty(sc, td) != 0)
651 			return (1);	/* busy - cannot transfer data */
652 	}
653 	z = td->max_packet_count;
654 	for (x = y = 0; x != sc->sc_host_ch_max; x++) {
655 		/* check if channel is allocated */
656 		if (sc->sc_chan_state[x].allocated != 0)
657 			continue;
658 		/* check if channel is still enabled */
659 		if (sc->sc_chan_state[x].wait_halted != 0)
660 			continue;
661 		/* store channel number */
662 		td->channel[y++] = x;
663 		/* check if we got all channels */
664 		if (y == z)
665 			break;
666 	}
667 	if (y != z) {
668 		/* reset channel variable */
669 		td->channel[0] = DWC_OTG_MAX_CHANNELS;
670 		td->channel[1] = DWC_OTG_MAX_CHANNELS;
671 		td->channel[2] = DWC_OTG_MAX_CHANNELS;
672 		/* wait a bit */
673 		dwc_otg_enable_sof_irq(sc);
674 		return (1);	/* busy - not enough channels */
675 	}
676 
677 	for (y = 0; y != z; y++) {
678 		x = td->channel[y];
679 
680 		/* set allocated */
681 		sc->sc_chan_state[x].allocated = 1;
682 
683 		/* set wait halted */
684 		sc->sc_chan_state[x].wait_halted = 1;
685 
686 		/* clear interrupts */
687 		dwc_otg_clear_hcint(sc, x);
688 
689 		DPRINTF("CH=%d HCCHAR=0x%08x "
690 		    "HCSPLT=0x%08x\n", x, td->hcchar, td->hcsplt);
691 
692 		/* set active channel */
693 		sc->sc_active_rx_ep |= (1 << x);
694 	}
695 	return (0);	/* allocated */
696 }
697 
698 static void
699 dwc_otg_host_channel_free_sub(struct dwc_otg_softc *sc, struct dwc_otg_td *td, uint8_t index)
700 {
701 	uint32_t hcchar;
702 	uint8_t x;
703 
704 	if (td->channel[index] >= DWC_OTG_MAX_CHANNELS)
705 		return;		/* already freed */
706 
707 	/* free channel */
708 	x = td->channel[index];
709 	td->channel[index] = DWC_OTG_MAX_CHANNELS;
710 
711 	DPRINTF("CH=%d\n", x);
712 
713 	/*
714 	 * We need to let programmed host channels run till complete
715 	 * else the host channel will stop functioning.
716 	 */
717 	sc->sc_chan_state[x].allocated = 0;
718 
719 	/* ack any pending messages */
720 	if (sc->sc_last_rx_status != 0 &&
721 	    GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) == x) {
722 		dwc_otg_common_rx_ack(sc);
723 	}
724 
725 	/* clear active channel */
726 	sc->sc_active_rx_ep &= ~(1 << x);
727 
728 	/* check if already halted */
729 	if (sc->sc_chan_state[x].wait_halted == 0)
730 		return;
731 
732 	/* disable host channel */
733 	hcchar = DWC_OTG_READ_4(sc, DOTG_HCCHAR(x));
734 	if (hcchar & HCCHAR_CHENA) {
735 		DPRINTF("Halting channel %d\n", x);
736 		DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(x),
737 		    hcchar | HCCHAR_CHDIS);
738 		/* don't write HCCHAR until the channel is halted */
739 	} else {
740 		sc->sc_chan_state[x].wait_halted = 0;
741 	}
742 }
743 
744 static void
745 dwc_otg_host_channel_free(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
746 {
747 	uint8_t x;
748 	for (x = 0; x != td->max_packet_count; x++)
749 		dwc_otg_host_channel_free_sub(sc, td, x);
750 }
751 
752 static void
753 dwc_otg_host_dump_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
754 {
755 	uint8_t x;
756 	/* dump any pending messages */
757 	if (sc->sc_last_rx_status == 0)
758 		return;
759 	for (x = 0; x != td->max_packet_count; x++) {
760 		if (td->channel[x] >= DWC_OTG_MAX_CHANNELS ||
761 		    td->channel[x] != GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status))
762 			continue;
763 		dwc_otg_common_rx_ack(sc);
764 	}
765 }
766 
767 static uint8_t
768 dwc_otg_host_setup_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
769 {
770 	struct usb_device_request req __aligned(4);
771 	uint32_t hcint;
772 	uint32_t hcchar;
773 	uint8_t delta;
774 
775 	dwc_otg_host_dump_rx(sc, td);
776 
777 	if (td->channel[0] < DWC_OTG_MAX_CHANNELS) {
778 		hcint = sc->sc_chan_state[td->channel[0]].hcint;
779 
780 		DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
781 		    td->channel[0], td->state, hcint,
782 		    DWC_OTG_READ_4(sc, DOTG_HCCHAR(td->channel[0])),
783 		    DWC_OTG_READ_4(sc, DOTG_HCTSIZ(td->channel[0])));
784 	} else {
785 		hcint = 0;
786 		goto check_state;
787 	}
788 
789 	if (hcint & (HCINT_RETRY |
790 	    HCINT_ACK | HCINT_NYET)) {
791 		/* give success bits priority over failure bits */
792 	} else if (hcint & HCINT_STALL) {
793 		DPRINTF("CH=%d STALL\n", td->channel[0]);
794 		td->error_stall = 1;
795 		td->error_any = 1;
796 		goto complete;
797 	} else if (hcint & HCINT_ERRORS) {
798 		DPRINTF("CH=%d ERROR\n", td->channel[0]);
799 		td->errcnt++;
800 		if (td->hcsplt != 0 || td->errcnt >= 3) {
801 			td->error_any = 1;
802 			goto complete;
803 		}
804 	}
805 
806 	if (hcint & (HCINT_ERRORS | HCINT_RETRY |
807 	    HCINT_ACK | HCINT_NYET)) {
808 		if (!(hcint & HCINT_ERRORS))
809 			td->errcnt = 0;
810 	}
811 
812 check_state:
813 	switch (td->state) {
814 	case DWC_CHAN_ST_START:
815 		goto send_pkt;
816 
817 	case DWC_CHAN_ST_WAIT_ANE:
818 		if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
819 			td->did_nak = 1;
820 			td->tt_scheduled = 0;
821 			goto send_pkt;
822 		} else if (hcint & (HCINT_ACK | HCINT_NYET)) {
823 			td->offset += td->tx_bytes;
824 			td->remainder -= td->tx_bytes;
825 			td->toggle = 1;
826 			td->tt_scheduled = 0;
827 			goto complete;
828 		}
829 		break;
830 
831 	case DWC_CHAN_ST_WAIT_S_ANE:
832 		if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
833 			td->did_nak = 1;
834 			td->tt_scheduled = 0;
835 			goto send_pkt;
836 		} else if (hcint & (HCINT_ACK | HCINT_NYET)) {
837 			goto send_cpkt;
838 		}
839 		break;
840 
841 	case DWC_CHAN_ST_WAIT_C_ANE:
842 		if (hcint & HCINT_NYET) {
843 			goto send_cpkt;
844 		} else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
845 			td->did_nak = 1;
846 			td->tt_scheduled = 0;
847 			goto send_pkt;
848 		} else if (hcint & HCINT_ACK) {
849 			td->offset += td->tx_bytes;
850 			td->remainder -= td->tx_bytes;
851 			td->toggle = 1;
852 			goto complete;
853 		}
854 		break;
855 
856 	case DWC_CHAN_ST_WAIT_C_PKT:
857 		goto send_cpkt;
858 
859 	default:
860 		break;
861 	}
862 	goto busy;
863 
864 send_pkt:
865 	/* free existing channel, if any */
866 	dwc_otg_host_channel_free(sc, td);
867 
868 	if (sizeof(req) != td->remainder) {
869 		td->error_any = 1;
870 		goto complete;
871 	}
872 
873 	if (td->hcsplt != 0) {
874 		delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
875 		if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
876 			td->state = DWC_CHAN_ST_START;
877 			goto busy;
878 		}
879 		delta = sc->sc_last_frame_num - td->tt_start_slot;
880 		if (delta > 5) {
881 			/* missed it */
882 			td->tt_scheduled = 0;
883 			td->state = DWC_CHAN_ST_START;
884 			goto busy;
885 		}
886 	}
887 
888 	/* allocate a new channel */
889 	if (dwc_otg_host_channel_alloc(sc, td, 1)) {
890 		td->state = DWC_CHAN_ST_START;
891 		goto busy;
892 	}
893 
894 	if (td->hcsplt != 0) {
895 		td->hcsplt &= ~HCSPLT_COMPSPLT;
896 		td->state = DWC_CHAN_ST_WAIT_S_ANE;
897 	} else {
898 		td->state = DWC_CHAN_ST_WAIT_ANE;
899 	}
900 
901 	usbd_copy_out(td->pc, 0, &req, sizeof(req));
902 
903 	DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel[0]),
904 	    (sizeof(req) << HCTSIZ_XFERSIZE_SHIFT) |
905 	    (1 << HCTSIZ_PKTCNT_SHIFT) |
906 	    (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
907 
908 	DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel[0]), td->hcsplt);
909 
910 	hcchar = td->hcchar;
911 	hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
912 	hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
913 
914 	/* must enable channel before writing data to FIFO */
915 	DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel[0]), hcchar);
916 
917 	/* transfer data into FIFO */
918 	bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
919 	    DOTG_DFIFO(td->channel[0]), (uint32_t *)&req, sizeof(req) / 4);
920 
921 	/* wait until next slot before trying complete split */
922 	td->tt_complete_slot = sc->sc_last_frame_num + 1;
923 
924 	/* store number of bytes transmitted */
925 	td->tx_bytes = sizeof(req);
926 	goto busy;
927 
928 send_cpkt:
929 	/* free existing channel, if any */
930 	dwc_otg_host_channel_free(sc, td);
931 
932 	delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
933 	if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
934 		td->state = DWC_CHAN_ST_WAIT_C_PKT;
935 		goto busy;
936 	}
937 	delta = sc->sc_last_frame_num - td->tt_start_slot;
938 	if (delta > DWC_OTG_TT_SLOT_MAX) {
939 		/* we missed the service interval */
940 		if (td->ep_type != UE_ISOCHRONOUS)
941 			td->error_any = 1;
942 		goto complete;
943 	}
944 	/* allocate a new channel */
945 	if (dwc_otg_host_channel_alloc(sc, td, 0)) {
946 		td->state = DWC_CHAN_ST_WAIT_C_PKT;
947 		goto busy;
948 	}
949 
950 	/* wait until next slot before trying complete split */
951 	td->tt_complete_slot = sc->sc_last_frame_num + 1;
952 
953 	td->hcsplt |= HCSPLT_COMPSPLT;
954 	td->state = DWC_CHAN_ST_WAIT_C_ANE;
955 
956 	DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel[0]),
957 	    (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
958 
959 	DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel[0]), td->hcsplt);
960 
961 	hcchar = td->hcchar;
962 	hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
963 	hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
964 
965 	/* must enable channel before writing data to FIFO */
966 	DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel[0]), hcchar);
967 
968 busy:
969 	return (1);	/* busy */
970 
971 complete:
972 	dwc_otg_host_channel_free(sc, td);
973 	return (0);	/* complete */
974 }
975 
976 static uint8_t
977 dwc_otg_setup_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
978 {
979 	struct usb_device_request req __aligned(4);
980 	uint32_t temp;
981 	uint16_t count;
982 
983 	/* check endpoint status */
984 
985 	if (sc->sc_last_rx_status == 0)
986 		goto not_complete;
987 
988 	if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != 0)
989 		goto not_complete;
990 
991 	if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
992 	    GRXSTSRD_STP_DATA) {
993 		if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
994 		    GRXSTSRD_STP_COMPLETE || td->remainder != 0) {
995 			/* release FIFO */
996 			dwc_otg_common_rx_ack(sc);
997 			goto not_complete;
998 		}
999 		/* release FIFO */
1000 		dwc_otg_common_rx_ack(sc);
1001 		return (0);     /* complete */
1002 	}
1003 
1004 	if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
1005 	    GRXSTSRD_DPID_DATA0) {
1006 		/* release FIFO */
1007 		dwc_otg_common_rx_ack(sc);
1008 		goto not_complete;
1009 	}
1010 
1011 	DPRINTFN(5, "GRXSTSR=0x%08x\n", sc->sc_last_rx_status);
1012 
1013 	/* clear did stall */
1014 	td->did_stall = 0;
1015 
1016 	/* get the packet byte count */
1017 	count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1018 
1019 	if (count != sizeof(req)) {
1020 		DPRINTFN(0, "Unsupported SETUP packet "
1021 		    "length, %d bytes\n", count);
1022 		/* release FIFO */
1023 		dwc_otg_common_rx_ack(sc);
1024 		goto not_complete;
1025 	}
1026 
1027 	/* copy in control request */
1028 	memcpy(&req, sc->sc_rx_bounce_buffer, sizeof(req));
1029 
1030 	/* copy data into real buffer */
1031 	usbd_copy_in(td->pc, 0, &req, sizeof(req));
1032 
1033 	td->offset = sizeof(req);
1034 	td->remainder = 0;
1035 
1036 	/* sneak peek the set address */
1037 	if ((req.bmRequestType == UT_WRITE_DEVICE) &&
1038 	    (req.bRequest == UR_SET_ADDRESS)) {
1039 		/* must write address before ZLP */
1040 		dwc_otg_set_address(sc, req.wValue[0] & 0x7F);
1041 	}
1042 
1043 	/* don't send any data by default */
1044 	DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(0), DIEPCTL_EPDIS);
1045 	DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0), DOEPCTL_EPDIS);
1046 
1047 	/* reset IN endpoint buffer */
1048 	dwc_otg_tx_fifo_reset(sc,
1049 	    GRSTCTL_TXFIFO(0) |
1050 	    GRSTCTL_TXFFLSH);
1051 
1052 	/* acknowledge RX status */
1053 	dwc_otg_common_rx_ack(sc);
1054 	td->did_stall = 1;
1055 
1056 not_complete:
1057 	/* abort any ongoing transfer, before enabling again */
1058 	if (!td->did_stall) {
1059 		td->did_stall = 1;
1060 
1061 		DPRINTFN(5, "stalling IN and OUT direction\n");
1062 
1063 		temp = sc->sc_out_ctl[0];
1064 
1065 		/* set stall after enabling endpoint */
1066 		DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(0),
1067 		    temp | DOEPCTL_STALL);
1068 
1069 		temp = sc->sc_in_ctl[0];
1070 
1071 		/* set stall assuming endpoint is enabled */
1072 		DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(0),
1073 		    temp | DIEPCTL_STALL);
1074 	}
1075 	return (1);			/* not complete */
1076 }
1077 
1078 static uint8_t
1079 dwc_otg_host_rate_check_interrupt(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1080 {
1081 	uint8_t delta;
1082 
1083 	delta = sc->sc_tmr_val - td->tmr_val;
1084 	if (delta >= 128)
1085 		return (1);	/* busy */
1086 
1087 	td->tmr_val = sc->sc_tmr_val + td->tmr_res;
1088 
1089 	/* set toggle, if any */
1090 	if (td->set_toggle) {
1091 		td->set_toggle = 0;
1092 		td->toggle = 1;
1093 	}
1094 	return (0);
1095 }
1096 
1097 static uint8_t
1098 dwc_otg_host_rate_check(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1099 {
1100 	uint8_t frame_num = (uint8_t)sc->sc_last_frame_num;
1101 
1102 	if (td->ep_type == UE_ISOCHRONOUS) {
1103 		/* non TT isochronous traffic */
1104 		if (frame_num & (td->tmr_res - 1))
1105 			goto busy;
1106 		if ((frame_num ^ td->tmr_val) & td->tmr_res)
1107 			goto busy;
1108 		td->tmr_val = td->tmr_res + sc->sc_last_frame_num;
1109 		td->toggle = 0;
1110 		return (0);
1111 	} else if (td->ep_type == UE_INTERRUPT) {
1112 		if (!td->tt_scheduled)
1113 			goto busy;
1114 		td->tt_scheduled = 0;
1115 		return (0);
1116 	} else if (td->did_nak != 0) {
1117 		/* check if we should pause sending queries for 125us */
1118 		if (td->tmr_res == frame_num) {
1119 			/* wait a bit */
1120 			dwc_otg_enable_sof_irq(sc);
1121 			goto busy;
1122 		}
1123 	} else if (td->set_toggle) {
1124 		td->set_toggle = 0;
1125 		td->toggle = 1;
1126 	}
1127 	/* query for data one more time */
1128 	td->tmr_res = frame_num;
1129 	td->did_nak = 0;
1130 	return (0);
1131 busy:
1132 	return (1);
1133 }
1134 
1135 static uint8_t
1136 dwc_otg_host_data_rx_sub(struct dwc_otg_softc *sc, struct dwc_otg_td *td,
1137     uint8_t channel)
1138 {
1139 	uint32_t count;
1140 
1141 	/* check endpoint status */
1142 	if (sc->sc_last_rx_status == 0)
1143 		goto busy;
1144 
1145 	if (channel >= DWC_OTG_MAX_CHANNELS)
1146 		goto busy;
1147 
1148 	if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != channel)
1149 		goto busy;
1150 
1151 	switch (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) {
1152 	case GRXSTSRH_IN_DATA:
1153 
1154 		DPRINTF("DATA ST=%d STATUS=0x%08x\n",
1155 		    (int)td->state, (int)sc->sc_last_rx_status);
1156 
1157 		if (sc->sc_chan_state[channel].hcint & HCINT_SOFTWARE_ONLY) {
1158 			/*
1159 			 * When using SPLIT transactions on interrupt
1160 			 * endpoints, sometimes data occurs twice.
1161 			 */
1162 			DPRINTF("Data already received\n");
1163 			break;
1164 		}
1165 
1166 		/* get the packet byte count */
1167 		count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1168 
1169 		/* check for ISOCHRONOUS endpoint */
1170 		if (td->ep_type == UE_ISOCHRONOUS) {
1171 			if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
1172 			    GRXSTSRD_DPID_DATA0) {
1173 				/* more data to be received */
1174 				td->tt_xactpos = HCSPLT_XACTPOS_MIDDLE;
1175 			} else {
1176 				/* all data received */
1177 				td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
1178 				/* verify the packet byte count */
1179 				if (count != td->remainder) {
1180 					/* we have a short packet */
1181 					td->short_pkt = 1;
1182 					td->got_short = 1;
1183 				}
1184 			}
1185 		} else {
1186 			/* verify the packet byte count */
1187 			if (count != td->max_packet_size) {
1188 				if (count < td->max_packet_size) {
1189 					/* we have a short packet */
1190 					td->short_pkt = 1;
1191 					td->got_short = 1;
1192 				} else {
1193 					/* invalid USB packet */
1194 					td->error_any = 1;
1195 
1196 					/* release FIFO */
1197 					dwc_otg_common_rx_ack(sc);
1198 					goto complete;
1199 				}
1200 			}
1201 			td->toggle ^= 1;
1202 			td->tt_scheduled = 0;
1203 		}
1204 
1205 		/* verify the packet byte count */
1206 		if (count > td->remainder) {
1207 			/* invalid USB packet */
1208 			td->error_any = 1;
1209 
1210 			/* release FIFO */
1211 			dwc_otg_common_rx_ack(sc);
1212 			goto complete;
1213 		}
1214 
1215 		usbd_copy_in(td->pc, td->offset,
1216 		    sc->sc_rx_bounce_buffer, count);
1217 
1218 		td->remainder -= count;
1219 		td->offset += count;
1220 		sc->sc_chan_state[channel].hcint |= HCINT_SOFTWARE_ONLY;
1221 		break;
1222 	default:
1223 		break;
1224 	}
1225 	/* release FIFO */
1226 	dwc_otg_common_rx_ack(sc);
1227 busy:
1228 	return (0);
1229 complete:
1230 	return (1);
1231 }
1232 
1233 static uint8_t
1234 dwc_otg_host_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1235 {
1236 	uint32_t hcint = 0;
1237 	uint32_t hcchar;
1238 	uint8_t delta;
1239 	uint8_t channel;
1240 	uint8_t x;
1241 
1242 	for (x = 0; x != td->max_packet_count; x++) {
1243 		channel = td->channel[x];
1244 		if (channel >= DWC_OTG_MAX_CHANNELS)
1245 			continue;
1246 		hcint |= sc->sc_chan_state[channel].hcint;
1247 
1248 		DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1249 		    channel, td->state, hcint,
1250 		    DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1251 		    DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1252 
1253 		/* check interrupt bits */
1254 		if (hcint & (HCINT_RETRY |
1255 		    HCINT_ACK | HCINT_NYET)) {
1256 			/* give success bits priority over failure bits */
1257 		} else if (hcint & HCINT_STALL) {
1258 			DPRINTF("CH=%d STALL\n", channel);
1259 			td->error_stall = 1;
1260 			td->error_any = 1;
1261 			goto complete;
1262 		} else if (hcint & HCINT_ERRORS) {
1263 			DPRINTF("CH=%d ERROR\n", channel);
1264 			td->errcnt++;
1265 			if (td->hcsplt != 0 || td->errcnt >= 3) {
1266 				if (td->ep_type != UE_ISOCHRONOUS) {
1267 					td->error_any = 1;
1268 					goto complete;
1269 				}
1270 			}
1271 		}
1272 
1273 		/* check channels for data, if any */
1274 		if (dwc_otg_host_data_rx_sub(sc, td, channel))
1275 			goto complete;
1276 
1277 		/* refresh interrupt status */
1278 		hcint |= sc->sc_chan_state[channel].hcint;
1279 
1280 		if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1281 		    HCINT_ACK | HCINT_NYET)) {
1282 			if (!(hcint & HCINT_ERRORS))
1283 				td->errcnt = 0;
1284 		}
1285 	}
1286 
1287 	switch (td->state) {
1288 	case DWC_CHAN_ST_START:
1289 		if (td->hcsplt != 0)
1290 			goto receive_spkt;
1291 		else
1292 			goto receive_pkt;
1293 
1294 	case DWC_CHAN_ST_WAIT_ANE:
1295 		if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1296 			if (td->ep_type == UE_INTERRUPT) {
1297 				/*
1298 				 * The USB specification does not
1299 				 * mandate a particular data toggle
1300 				 * value for USB INTERRUPT
1301 				 * transfers. Switch the data toggle
1302 				 * value to receive the packet
1303 				 * correctly:
1304 				 */
1305 				if (hcint & HCINT_DATATGLERR) {
1306 					DPRINTF("Retrying packet due to "
1307 					    "data toggle error\n");
1308 					td->toggle ^= 1;
1309 					goto receive_pkt;
1310 				}
1311 			} else if (td->ep_type == UE_ISOCHRONOUS) {
1312 				goto complete;
1313 			}
1314 			td->did_nak = 1;
1315 			td->tt_scheduled = 0;
1316 			if (td->hcsplt != 0)
1317 				goto receive_spkt;
1318 			else
1319 				goto receive_pkt;
1320 		} else if (hcint & HCINT_NYET) {
1321 			if (td->hcsplt != 0) {
1322 				/* try again */
1323 				goto receive_pkt;
1324 			} else {
1325 				/* not a valid token for IN endpoints */
1326 				td->error_any = 1;
1327 				goto complete;
1328 			}
1329 		} else if (hcint & HCINT_ACK) {
1330 			/* wait for data - ACK arrived first */
1331 			if (!(hcint & HCINT_SOFTWARE_ONLY))
1332 				goto busy;
1333 
1334 			if (td->ep_type == UE_ISOCHRONOUS) {
1335 				/* check if we are complete */
1336 				if (td->tt_xactpos == HCSPLT_XACTPOS_BEGIN) {
1337 					goto complete;
1338 				} else {
1339 					/* get more packets */
1340 					goto busy;
1341 				}
1342 			} else {
1343 				/* check if we are complete */
1344 				if ((td->remainder == 0) || (td->got_short != 0)) {
1345 					if (td->short_pkt)
1346 						goto complete;
1347 
1348 					/*
1349 					 * Else need to receive a zero length
1350 					 * packet.
1351 					 */
1352 				}
1353 				td->tt_scheduled = 0;
1354 				td->did_nak = 0;
1355 				if (td->hcsplt != 0)
1356 					goto receive_spkt;
1357 				else
1358 					goto receive_pkt;
1359 			}
1360 		}
1361 		break;
1362 
1363 	case DWC_CHAN_ST_WAIT_S_ANE:
1364 		/*
1365 		 * NOTE: The DWC OTG hardware provides a fake ACK in
1366 		 * case of interrupt and isochronous transfers:
1367 		 */
1368 		if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1369 			td->did_nak = 1;
1370 			td->tt_scheduled = 0;
1371 			goto receive_spkt;
1372 		} else if (hcint & HCINT_NYET) {
1373 			td->tt_scheduled = 0;
1374 			goto receive_spkt;
1375 		} else if (hcint & HCINT_ACK) {
1376 			td->did_nak = 0;
1377 			goto receive_pkt;
1378 		}
1379 		break;
1380 
1381 	case DWC_CHAN_ST_WAIT_C_PKT:
1382 		goto receive_pkt;
1383 
1384 	default:
1385 		break;
1386 	}
1387 	goto busy;
1388 
1389 receive_pkt:
1390 	/* free existing channel, if any */
1391 	dwc_otg_host_channel_free(sc, td);
1392 
1393   	if (td->hcsplt != 0) {
1394 		delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1395 		if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1396 			td->state = DWC_CHAN_ST_WAIT_C_PKT;
1397 			goto busy;
1398 		}
1399 		delta = sc->sc_last_frame_num - td->tt_start_slot;
1400 		if (delta > DWC_OTG_TT_SLOT_MAX) {
1401 			if (td->ep_type != UE_ISOCHRONOUS) {
1402 				/* we missed the service interval */
1403 				td->error_any = 1;
1404 			}
1405 			goto complete;
1406 		}
1407 		/* complete split */
1408 		td->hcsplt |= HCSPLT_COMPSPLT;
1409 	} else if (dwc_otg_host_rate_check(sc, td)) {
1410 		td->state = DWC_CHAN_ST_WAIT_C_PKT;
1411 		goto busy;
1412 	}
1413 
1414 	/* allocate a new channel */
1415 	if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1416 		td->state = DWC_CHAN_ST_WAIT_C_PKT;
1417 		goto busy;
1418 	}
1419 
1420 	/* set toggle, if any */
1421 	if (td->set_toggle) {
1422 		td->set_toggle = 0;
1423 		td->toggle = 1;
1424 	}
1425 
1426 	td->state = DWC_CHAN_ST_WAIT_ANE;
1427 
1428 	for (x = 0; x != td->max_packet_count; x++) {
1429 	  	channel = td->channel[x];
1430 
1431 		/* receive one packet */
1432 		DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1433 		    (td->max_packet_size << HCTSIZ_XFERSIZE_SHIFT) |
1434 		    (1 << HCTSIZ_PKTCNT_SHIFT) |
1435 		    (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1436 		    (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1437 
1438 		DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1439 
1440 		hcchar = td->hcchar;
1441 		hcchar |= HCCHAR_EPDIR_IN;
1442 
1443 		/* receive complete split ASAP */
1444 		if ((sc->sc_last_frame_num & 1) != 0 &&
1445 		    td->ep_type == UE_ISOCHRONOUS)
1446 			hcchar |= HCCHAR_ODDFRM;
1447 		else
1448 			hcchar &= ~HCCHAR_ODDFRM;
1449 
1450 		/* must enable channel before data can be received */
1451 		DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1452 	}
1453 	/* wait until next slot before trying complete split */
1454 	td->tt_complete_slot = sc->sc_last_frame_num + 1;
1455 
1456 	goto busy;
1457 
1458 receive_spkt:
1459 	/* free existing channel(s), if any */
1460 	dwc_otg_host_channel_free(sc, td);
1461 
1462 	delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1463 	if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1464 		td->state = DWC_CHAN_ST_START;
1465 		goto busy;
1466 	}
1467 	delta = sc->sc_last_frame_num - td->tt_start_slot;
1468 	if (delta > 5) {
1469 		/* missed it */
1470 		td->tt_scheduled = 0;
1471 		td->state = DWC_CHAN_ST_START;
1472 		goto busy;
1473 	}
1474 
1475 	/* allocate a new channel */
1476 	if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1477 		td->state = DWC_CHAN_ST_START;
1478 		goto busy;
1479 	}
1480 
1481 	channel = td->channel[0];
1482 
1483 	td->hcsplt &= ~HCSPLT_COMPSPLT;
1484 	td->state = DWC_CHAN_ST_WAIT_S_ANE;
1485 
1486 	/* receive one packet */
1487 	DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1488 	    (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
1489 
1490 	DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1491 
1492 	/* send after next SOF event */
1493 	if ((sc->sc_last_frame_num & 1) == 0 &&
1494 	    td->ep_type == UE_ISOCHRONOUS)
1495 		td->hcchar |= HCCHAR_ODDFRM;
1496 	else
1497 		td->hcchar &= ~HCCHAR_ODDFRM;
1498 
1499 	hcchar = td->hcchar;
1500 	hcchar |= HCCHAR_EPDIR_IN;
1501 
1502 	/* wait until next slot before trying complete split */
1503 	td->tt_complete_slot = sc->sc_last_frame_num + 1;
1504 
1505 	/* must enable channel before data can be received */
1506 	DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1507 busy:
1508 	return (1);	/* busy */
1509 
1510 complete:
1511 	dwc_otg_host_channel_free(sc, td);
1512 	return (0);	/* complete */
1513 }
1514 
1515 static uint8_t
1516 dwc_otg_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1517 {
1518 	uint32_t temp;
1519 	uint16_t count;
1520 	uint8_t got_short;
1521 
1522 	got_short = 0;
1523 
1524 	/* check endpoint status */
1525 	if (sc->sc_last_rx_status == 0)
1526 		goto not_complete;
1527 
1528 	if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != td->ep_no)
1529 		goto not_complete;
1530 
1531 	/* check for SETUP packet */
1532 	if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1533 	    GRXSTSRD_STP_DATA ||
1534 	    (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1535 	    GRXSTSRD_STP_COMPLETE) {
1536 		if (td->remainder == 0) {
1537 			/*
1538 			 * We are actually complete and have
1539 			 * received the next SETUP
1540 			 */
1541 			DPRINTFN(5, "faking complete\n");
1542 			return (0);	/* complete */
1543 		}
1544 		/*
1545 		 * USB Host Aborted the transfer.
1546 		 */
1547 		td->error_any = 1;
1548 		return (0);		/* complete */
1549 	}
1550 
1551 	if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1552 	    GRXSTSRD_OUT_DATA) {
1553 		/* release FIFO */
1554 		dwc_otg_common_rx_ack(sc);
1555 		goto not_complete;
1556 	}
1557 
1558 	/* get the packet byte count */
1559 	count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1560 
1561 	/* verify the packet byte count */
1562 	if (count != td->max_packet_size) {
1563 		if (count < td->max_packet_size) {
1564 			/* we have a short packet */
1565 			td->short_pkt = 1;
1566 			got_short = 1;
1567 		} else {
1568 			/* invalid USB packet */
1569 			td->error_any = 1;
1570 
1571 			/* release FIFO */
1572 			dwc_otg_common_rx_ack(sc);
1573 			return (0);	/* we are complete */
1574 		}
1575 	}
1576 	/* verify the packet byte count */
1577 	if (count > td->remainder) {
1578 		/* invalid USB packet */
1579 		td->error_any = 1;
1580 
1581 		/* release FIFO */
1582 		dwc_otg_common_rx_ack(sc);
1583 		return (0);		/* we are complete */
1584 	}
1585 
1586 	usbd_copy_in(td->pc, td->offset, sc->sc_rx_bounce_buffer, count);
1587 	td->remainder -= count;
1588 	td->offset += count;
1589 
1590 	/* release FIFO */
1591 	dwc_otg_common_rx_ack(sc);
1592 
1593 	temp = sc->sc_out_ctl[td->ep_no];
1594 
1595 	/* check for isochronous mode */
1596 	if ((temp & DIEPCTL_EPTYPE_MASK) ==
1597 	    (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
1598 		/* toggle odd or even frame bit */
1599 		if (temp & DIEPCTL_SETD1PID) {
1600 			temp &= ~DIEPCTL_SETD1PID;
1601 			temp |= DIEPCTL_SETD0PID;
1602 		} else {
1603 			temp &= ~DIEPCTL_SETD0PID;
1604 			temp |= DIEPCTL_SETD1PID;
1605 		}
1606 		sc->sc_out_ctl[td->ep_no] = temp;
1607 	}
1608 
1609 	/* check if we are complete */
1610 	if ((td->remainder == 0) || got_short) {
1611 		if (td->short_pkt) {
1612 			/* we are complete */
1613 			return (0);
1614 		}
1615 		/* else need to receive a zero length packet */
1616 	}
1617 
1618 not_complete:
1619 
1620 	/* enable SETUP and transfer complete interrupt */
1621 	if (td->ep_no == 0) {
1622 		DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0),
1623 		    DXEPTSIZ_SET_MULTI(3) |
1624 		    DXEPTSIZ_SET_NPKT(1) |
1625 		    DXEPTSIZ_SET_NBYTES(td->max_packet_size));
1626 	} else {
1627 		/* allow reception of multiple packets */
1628 		DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(td->ep_no),
1629 		    DXEPTSIZ_SET_MULTI(1) |
1630 		    DXEPTSIZ_SET_NPKT(4) |
1631 		    DXEPTSIZ_SET_NBYTES(4 *
1632 		        ((td->max_packet_size + 3) & ~3)));
1633 	}
1634 	temp = sc->sc_out_ctl[td->ep_no];
1635 	DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(td->ep_no), temp |
1636 	    DOEPCTL_EPENA | DOEPCTL_CNAK);
1637 
1638 	return (1);			/* not complete */
1639 }
1640 
1641 static uint8_t
1642 dwc_otg_host_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1643 {
1644 	uint32_t count;
1645 	uint32_t hcint;
1646 	uint32_t hcchar;
1647 	uint8_t delta;
1648 	uint8_t channel;
1649 	uint8_t x;
1650 
1651 	dwc_otg_host_dump_rx(sc, td);
1652 
1653 	/* check that last channel is complete */
1654 	channel = td->channel[td->npkt];
1655 
1656 	if (channel < DWC_OTG_MAX_CHANNELS) {
1657 		hcint = sc->sc_chan_state[channel].hcint;
1658 
1659 		DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1660 		    channel, td->state, hcint,
1661 		    DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1662 		    DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1663 
1664 		if (hcint & (HCINT_RETRY |
1665 		    HCINT_ACK | HCINT_NYET)) {
1666 			/* give success bits priority over failure bits */
1667 		} else if (hcint & HCINT_STALL) {
1668 			DPRINTF("CH=%d STALL\n", channel);
1669 			td->error_stall = 1;
1670 			td->error_any = 1;
1671 			goto complete;
1672 		} else if (hcint & HCINT_ERRORS) {
1673 			DPRINTF("CH=%d ERROR\n", channel);
1674 			td->errcnt++;
1675 			if (td->hcsplt != 0 || td->errcnt >= 3) {
1676 				td->error_any = 1;
1677 				goto complete;
1678 			}
1679 		}
1680 
1681 		if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1682 		    HCINT_ACK | HCINT_NYET)) {
1683 
1684 			if (!(hcint & HCINT_ERRORS))
1685 				td->errcnt = 0;
1686 		}
1687 	} else {
1688 		hcint = 0;
1689 	}
1690 
1691 	switch (td->state) {
1692 	case DWC_CHAN_ST_START:
1693 		goto send_pkt;
1694 
1695 	case DWC_CHAN_ST_WAIT_ANE:
1696 		if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1697 			td->did_nak = 1;
1698 			td->tt_scheduled = 0;
1699 			goto send_pkt;
1700 		} else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1701 			td->offset += td->tx_bytes;
1702 			td->remainder -= td->tx_bytes;
1703 			td->toggle ^= 1;
1704 			/* check if next response will be a NAK */
1705 			if (hcint & HCINT_NYET)
1706 				td->did_nak = 1;
1707 			else
1708 				td->did_nak = 0;
1709 			td->tt_scheduled = 0;
1710 
1711 			/* check remainder */
1712 			if (td->remainder == 0) {
1713 				if (td->short_pkt)
1714 					goto complete;
1715 
1716 				/*
1717 				 * Else we need to transmit a short
1718 				 * packet:
1719 				 */
1720 			}
1721 			goto send_pkt;
1722 		}
1723 		break;
1724 
1725 	case DWC_CHAN_ST_WAIT_S_ANE:
1726 		if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1727 			td->did_nak = 1;
1728 			td->tt_scheduled = 0;
1729 			goto send_pkt;
1730 		} else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1731 			td->did_nak = 0;
1732 			goto send_cpkt;
1733 		}
1734 		break;
1735 
1736 	case DWC_CHAN_ST_WAIT_C_ANE:
1737 		if (hcint & HCINT_NYET) {
1738 			goto send_cpkt;
1739 		} else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1740 			td->did_nak = 1;
1741 			td->tt_scheduled = 0;
1742 			goto send_pkt;
1743 		} else if (hcint & HCINT_ACK) {
1744 			td->offset += td->tx_bytes;
1745 			td->remainder -= td->tx_bytes;
1746 			td->toggle ^= 1;
1747 			td->did_nak = 0;
1748 			td->tt_scheduled = 0;
1749 
1750 			/* check remainder */
1751 			if (td->remainder == 0) {
1752 				if (td->short_pkt)
1753 					goto complete;
1754 
1755 				/* else we need to transmit a short packet */
1756 			}
1757 			goto send_pkt;
1758 		}
1759 		break;
1760 
1761 	case DWC_CHAN_ST_WAIT_C_PKT:
1762 		goto send_cpkt;
1763 
1764 	case DWC_CHAN_ST_TX_WAIT_ISOC:
1765 		/* Check if ISOCHRONOUS OUT traffic is complete */
1766 		if ((hcint & HCINT_HCH_DONE_MASK) == 0)
1767 			break;
1768 
1769 		td->offset += td->tx_bytes;
1770 		td->remainder -= td->tx_bytes;
1771 		goto complete;
1772 	default:
1773 		break;
1774 	}
1775 	goto busy;
1776 
1777 send_pkt:
1778 	/* free existing channel(s), if any */
1779 	dwc_otg_host_channel_free(sc, td);
1780 
1781 	if (td->hcsplt != 0) {
1782 		delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1783 		if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1784 			td->state = DWC_CHAN_ST_START;
1785 			goto busy;
1786 		}
1787 		delta = sc->sc_last_frame_num - td->tt_start_slot;
1788 		if (delta > 5) {
1789 			/* missed it */
1790 			td->tt_scheduled = 0;
1791 			td->state = DWC_CHAN_ST_START;
1792 			goto busy;
1793 		}
1794 	} else if (dwc_otg_host_rate_check(sc, td)) {
1795 		td->state = DWC_CHAN_ST_START;
1796 		goto busy;
1797 	}
1798 
1799 	/* allocate a new channel */
1800 	if (dwc_otg_host_channel_alloc(sc, td, 1)) {
1801 		td->state = DWC_CHAN_ST_START;
1802 		goto busy;
1803 	}
1804 
1805 	/* set toggle, if any */
1806 	if (td->set_toggle) {
1807 		td->set_toggle = 0;
1808 		td->toggle = 1;
1809 	}
1810 
1811 	if (td->ep_type == UE_ISOCHRONOUS) {
1812 		/* ISOCHRONOUS OUT transfers don't have any ACKs */
1813 		td->state = DWC_CHAN_ST_TX_WAIT_ISOC;
1814 		td->hcsplt &= ~HCSPLT_COMPSPLT;
1815 		if (td->hcsplt != 0) {
1816 			/* get maximum transfer length */
1817 			count = td->remainder;
1818 			if (count > HCSPLT_XACTLEN_BURST) {
1819 				DPRINTF("TT overflow\n");
1820 				td->error_any = 1;
1821 				goto complete;
1822 			}
1823 			/* Update transaction position */
1824 			td->hcsplt &= ~HCSPLT_XACTPOS_MASK;
1825 			td->hcsplt |= (HCSPLT_XACTPOS_ALL << HCSPLT_XACTPOS_SHIFT);
1826 		}
1827 	} else if (td->hcsplt != 0) {
1828 		td->hcsplt &= ~HCSPLT_COMPSPLT;
1829 		/* Wait for ACK/NAK/ERR from TT */
1830 		td->state = DWC_CHAN_ST_WAIT_S_ANE;
1831 	} else {
1832 		/* Wait for ACK/NAK/STALL from device */
1833 		td->state = DWC_CHAN_ST_WAIT_ANE;
1834 	}
1835 
1836 	td->tx_bytes = 0;
1837 
1838 	for (x = 0; x != td->max_packet_count; x++) {
1839 		uint32_t rem_bytes;
1840 
1841 		channel = td->channel[x];
1842 
1843 		/* send one packet at a time */
1844 		count = td->max_packet_size;
1845 		rem_bytes = td->remainder - td->tx_bytes;
1846 		if (rem_bytes < count) {
1847 			/* we have a short packet */
1848 			td->short_pkt = 1;
1849 			count = rem_bytes;
1850 		}
1851 		if (count == rem_bytes) {
1852 			/* last packet */
1853 			switch (x) {
1854 			case 0:
1855 				DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1856 				    (count << HCTSIZ_XFERSIZE_SHIFT) |
1857 				    (1 << HCTSIZ_PKTCNT_SHIFT) |
1858 				    (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1859 				    (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1860 				break;
1861 			case 1:
1862 				DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1863 				    (count << HCTSIZ_XFERSIZE_SHIFT) |
1864 				    (1 << HCTSIZ_PKTCNT_SHIFT) |
1865 				    (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT));
1866 				break;
1867 			default:
1868 				DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1869 				    (count << HCTSIZ_XFERSIZE_SHIFT) |
1870 				    (1 << HCTSIZ_PKTCNT_SHIFT) |
1871 				    (HCTSIZ_PID_DATA2 << HCTSIZ_PID_SHIFT));
1872 				break;
1873 			}
1874 		} else if (td->ep_type == UE_ISOCHRONOUS &&
1875 			   td->max_packet_count > 1) {
1876 			/* ISOCHRONOUS multi packet */
1877 			DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1878 			    (count << HCTSIZ_XFERSIZE_SHIFT) |
1879 			    (1 << HCTSIZ_PKTCNT_SHIFT) |
1880 			    (HCTSIZ_PID_MDATA << HCTSIZ_PID_SHIFT));
1881 		} else {
1882 			/* TODO: HCTSIZ_DOPNG */
1883 			/* standard BULK/INTERRUPT/CONTROL packet */
1884 			DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1885 			    (count << HCTSIZ_XFERSIZE_SHIFT) |
1886 			    (1 << HCTSIZ_PKTCNT_SHIFT) |
1887 			    (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1888 			    (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1889 		}
1890 
1891 		DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1892 
1893 		hcchar = td->hcchar;
1894 		hcchar &= ~HCCHAR_EPDIR_IN;
1895 
1896 		/* send after next SOF event */
1897 		if ((sc->sc_last_frame_num & 1) == 0 &&
1898 		    td->ep_type == UE_ISOCHRONOUS)
1899 			hcchar |= HCCHAR_ODDFRM;
1900 		else
1901 			hcchar &= ~HCCHAR_ODDFRM;
1902 
1903 		/* must enable before writing data to FIFO */
1904 		DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1905 
1906 		if (count != 0) {
1907 			/* clear topmost word before copy */
1908 			sc->sc_tx_bounce_buffer[(count - 1) / 4] = 0;
1909 
1910 			/* copy out data */
1911 			usbd_copy_out(td->pc, td->offset + td->tx_bytes,
1912 			    sc->sc_tx_bounce_buffer, count);
1913 
1914 			/* transfer data into FIFO */
1915 			bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
1916 			    DOTG_DFIFO(channel),
1917 			    sc->sc_tx_bounce_buffer, (count + 3) / 4);
1918 		}
1919 
1920 		/* store number of bytes transmitted */
1921 		td->tx_bytes += count;
1922 
1923 		/* store last packet index */
1924 		td->npkt = x;
1925 
1926 		/* check for last packet */
1927 		if (count == rem_bytes)
1928 			break;
1929 	}
1930 	goto busy;
1931 
1932 send_cpkt:
1933 	/* free existing channel, if any */
1934 	dwc_otg_host_channel_free(sc, td);
1935 
1936 	delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1937 	if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1938 		td->state = DWC_CHAN_ST_WAIT_C_PKT;
1939 		goto busy;
1940 	}
1941 	delta = sc->sc_last_frame_num - td->tt_start_slot;
1942 	if (delta > DWC_OTG_TT_SLOT_MAX) {
1943 		/* we missed the service interval */
1944 		if (td->ep_type != UE_ISOCHRONOUS)
1945 			td->error_any = 1;
1946 		goto complete;
1947 	}
1948 
1949 	/* allocate a new channel */
1950 	if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1951 		td->state = DWC_CHAN_ST_WAIT_C_PKT;
1952 		goto busy;
1953 	}
1954 
1955 	channel = td->channel[0];
1956 
1957  	td->hcsplt |= HCSPLT_COMPSPLT;
1958 	td->state = DWC_CHAN_ST_WAIT_C_ANE;
1959 
1960 	DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1961 	    (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
1962 
1963 	DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1964 
1965 	hcchar = td->hcchar;
1966 	hcchar &= ~HCCHAR_EPDIR_IN;
1967 
1968 	/* receive complete split ASAP */
1969 	if ((sc->sc_last_frame_num & 1) != 0 &&
1970 	    td->ep_type == UE_ISOCHRONOUS)
1971 		hcchar |= HCCHAR_ODDFRM;
1972 	else
1973 		hcchar &= ~HCCHAR_ODDFRM;
1974 
1975 	/* must enable channel before data can be received */
1976 	DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1977 
1978 	/* wait until next slot before trying complete split */
1979 	td->tt_complete_slot = sc->sc_last_frame_num + 1;
1980 busy:
1981 	return (1);	/* busy */
1982 
1983 complete:
1984 	dwc_otg_host_channel_free(sc, td);
1985 	return (0);	/* complete */
1986 }
1987 
1988 static uint8_t
1989 dwc_otg_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1990 {
1991 	uint32_t max_buffer;
1992 	uint32_t count;
1993 	uint32_t fifo_left;
1994 	uint32_t mpkt;
1995 	uint32_t temp;
1996 	uint8_t to;
1997 
1998 	to = 3;				/* don't loop forever! */
1999 
2000 	max_buffer = sc->sc_hw_ep_profile[td->ep_no].max_buffer;
2001 
2002 repeat:
2003 	/* check for for endpoint 0 data */
2004 
2005 	temp = sc->sc_last_rx_status;
2006 
2007 	if ((td->ep_no == 0) && (temp != 0) &&
2008 	    (GRXSTSRD_CHNUM_GET(temp) == 0)) {
2009 
2010 		if ((temp & GRXSTSRD_PKTSTS_MASK) !=
2011 		    GRXSTSRD_STP_DATA &&
2012 		    (temp & GRXSTSRD_PKTSTS_MASK) !=
2013 		    GRXSTSRD_STP_COMPLETE) {
2014 
2015 			/* dump data - wrong direction */
2016 			dwc_otg_common_rx_ack(sc);
2017 		} else {
2018 			/*
2019 			 * The current transfer was cancelled
2020 			 * by the USB Host:
2021 			 */
2022 			td->error_any = 1;
2023 			return (0);		/* complete */
2024 		}
2025 	}
2026 
2027 	/* fill in more TX data, if possible */
2028 	if (td->tx_bytes != 0) {
2029 
2030 		uint16_t cpkt;
2031 
2032 		/* check if packets have been transferred */
2033 		temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2034 
2035 		/* get current packet number */
2036 		cpkt = DXEPTSIZ_GET_NPKT(temp);
2037 
2038 		if (cpkt >= td->npkt) {
2039 			fifo_left = 0;
2040 		} else {
2041 			if (max_buffer != 0) {
2042 				fifo_left = (td->npkt - cpkt) *
2043 				    td->max_packet_size;
2044 
2045 				if (fifo_left > max_buffer)
2046 					fifo_left = max_buffer;
2047 			} else {
2048 				fifo_left = td->max_packet_size;
2049 			}
2050 		}
2051 
2052 		count = td->tx_bytes;
2053 		if (count > fifo_left)
2054 			count = fifo_left;
2055 
2056 		if (count != 0) {
2057 
2058 			/* clear topmost word before copy */
2059 			sc->sc_tx_bounce_buffer[(count - 1) / 4] = 0;
2060 
2061 			/* copy out data */
2062 			usbd_copy_out(td->pc, td->offset,
2063 			    sc->sc_tx_bounce_buffer, count);
2064 
2065 			/* transfer data into FIFO */
2066 			bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
2067 			    DOTG_DFIFO(td->ep_no),
2068 			    sc->sc_tx_bounce_buffer, (count + 3) / 4);
2069 
2070 			td->tx_bytes -= count;
2071 			td->remainder -= count;
2072 			td->offset += count;
2073 			td->npkt = cpkt;
2074 		}
2075 		if (td->tx_bytes != 0)
2076 			goto not_complete;
2077 
2078 		/* check remainder */
2079 		if (td->remainder == 0) {
2080 			if (td->short_pkt)
2081 				return (0);	/* complete */
2082 
2083 			/* else we need to transmit a short packet */
2084 		}
2085 	}
2086 
2087 	if (!to--)
2088 		goto not_complete;
2089 
2090 	/* check if not all packets have been transferred */
2091 	temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2092 
2093 	if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2094 
2095 		DPRINTFN(5, "busy ep=%d npkt=%d DIEPTSIZ=0x%08x "
2096 		    "DIEPCTL=0x%08x\n", td->ep_no,
2097 		    DXEPTSIZ_GET_NPKT(temp),
2098 		    temp, DWC_OTG_READ_4(sc, DOTG_DIEPCTL(td->ep_no)));
2099 
2100 		goto not_complete;
2101 	}
2102 
2103 	DPRINTFN(5, "rem=%u ep=%d\n", td->remainder, td->ep_no);
2104 
2105 	/* try to optimise by sending more data */
2106 	if ((max_buffer != 0) && ((td->max_packet_size & 3) == 0)) {
2107 
2108 		/* send multiple packets at the same time */
2109 		mpkt = max_buffer / td->max_packet_size;
2110 
2111 		if (mpkt > 0x3FE)
2112 			mpkt = 0x3FE;
2113 
2114 		count = td->remainder;
2115 		if (count > 0x7FFFFF)
2116 			count = 0x7FFFFF - (0x7FFFFF % td->max_packet_size);
2117 
2118 		td->npkt = count / td->max_packet_size;
2119 
2120 		/*
2121 		 * NOTE: We could use 0x3FE instead of "mpkt" in the
2122 		 * check below to get more throughput, but then we
2123 		 * have a dependency towards non-generic chip features
2124 		 * to disable the TX-FIFO-EMPTY interrupts on a per
2125 		 * endpoint basis. Increase the maximum buffer size of
2126 		 * the IN endpoint to increase the performance.
2127 		 */
2128 		if (td->npkt > mpkt) {
2129 			td->npkt = mpkt;
2130 			count = td->max_packet_size * mpkt;
2131 		} else if ((count == 0) || (count % td->max_packet_size)) {
2132 			/* we are transmitting a short packet */
2133 			td->npkt++;
2134 			td->short_pkt = 1;
2135 		}
2136 	} else {
2137 		/* send one packet at a time */
2138 		mpkt = 1;
2139 		count = td->max_packet_size;
2140 		if (td->remainder < count) {
2141 			/* we have a short packet */
2142 			td->short_pkt = 1;
2143 			count = td->remainder;
2144 		}
2145 		td->npkt = 1;
2146 	}
2147 	DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(td->ep_no),
2148 	    DXEPTSIZ_SET_MULTI(1) |
2149 	    DXEPTSIZ_SET_NPKT(td->npkt) |
2150 	    DXEPTSIZ_SET_NBYTES(count));
2151 
2152 	/* make room for buffering */
2153 	td->npkt += mpkt;
2154 
2155 	temp = sc->sc_in_ctl[td->ep_no];
2156 
2157 	/* check for isochronous mode */
2158 	if ((temp & DIEPCTL_EPTYPE_MASK) ==
2159 	    (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
2160 		/* toggle odd or even frame bit */
2161 		if (temp & DIEPCTL_SETD1PID) {
2162 			temp &= ~DIEPCTL_SETD1PID;
2163 			temp |= DIEPCTL_SETD0PID;
2164 		} else {
2165 			temp &= ~DIEPCTL_SETD0PID;
2166 			temp |= DIEPCTL_SETD1PID;
2167 		}
2168 		sc->sc_in_ctl[td->ep_no] = temp;
2169 	}
2170 
2171 	/* must enable before writing data to FIFO */
2172 	DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(td->ep_no), temp |
2173 	    DIEPCTL_EPENA | DIEPCTL_CNAK);
2174 
2175 	td->tx_bytes = count;
2176 
2177 	/* check remainder */
2178 	if (td->tx_bytes == 0 &&
2179 	    td->remainder == 0) {
2180 		if (td->short_pkt)
2181 			return (0);	/* complete */
2182 
2183 		/* else we need to transmit a short packet */
2184 	}
2185 	goto repeat;
2186 
2187 not_complete:
2188 	return (1);			/* not complete */
2189 }
2190 
2191 static uint8_t
2192 dwc_otg_data_tx_sync(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
2193 {
2194 	uint32_t temp;
2195 
2196 	/*
2197 	 * If all packets are transferred we are complete:
2198 	 */
2199 	temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2200 
2201 	/* check that all packets have been transferred */
2202 	if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2203 		DPRINTFN(5, "busy ep=%d\n", td->ep_no);
2204 		goto not_complete;
2205 	}
2206 	return (0);
2207 
2208 not_complete:
2209 
2210 	/* we only want to know if there is a SETUP packet or free IN packet */
2211 
2212 	temp = sc->sc_last_rx_status;
2213 
2214 	if ((td->ep_no == 0) && (temp != 0) &&
2215 	    (GRXSTSRD_CHNUM_GET(temp) == 0)) {
2216 
2217 		if ((temp & GRXSTSRD_PKTSTS_MASK) ==
2218 		    GRXSTSRD_STP_DATA ||
2219 		    (temp & GRXSTSRD_PKTSTS_MASK) ==
2220 		    GRXSTSRD_STP_COMPLETE) {
2221 			DPRINTFN(5, "faking complete\n");
2222 			/*
2223 			 * Race condition: We are complete!
2224 			 */
2225 			return (0);
2226 		} else {
2227 			/* dump data - wrong direction */
2228 			dwc_otg_common_rx_ack(sc);
2229 		}
2230 	}
2231 	return (1);			/* not complete */
2232 }
2233 
2234 static void
2235 dwc_otg_xfer_do_fifo(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2236 {
2237 	struct dwc_otg_td *td;
2238 	uint8_t toggle;
2239 	uint8_t tmr_val;
2240 	uint8_t tmr_res;
2241 
2242 	DPRINTFN(9, "\n");
2243 
2244 	td = xfer->td_transfer_cache;
2245 	if (td == NULL)
2246 		return;
2247 
2248 	while (1) {
2249 		if ((td->func) (sc, td)) {
2250 			/* operation in progress */
2251 			break;
2252 		}
2253 		if (((void *)td) == xfer->td_transfer_last) {
2254 			goto done;
2255 		}
2256 		if (td->error_any) {
2257 			goto done;
2258 		} else if (td->remainder > 0) {
2259 			/*
2260 			 * We had a short transfer. If there is no alternate
2261 			 * next, stop processing !
2262 			 */
2263 			if (!td->alt_next)
2264 				goto done;
2265 		}
2266 
2267 		/*
2268 		 * Fetch the next transfer descriptor and transfer
2269 		 * some flags to the next transfer descriptor
2270 		 */
2271 		tmr_res = td->tmr_res;
2272 		tmr_val = td->tmr_val;
2273 		toggle = td->toggle;
2274 		td = td->obj_next;
2275 		xfer->td_transfer_cache = td;
2276 		td->toggle = toggle;	/* transfer toggle */
2277 		td->tmr_res = tmr_res;
2278 		td->tmr_val = tmr_val;
2279 	}
2280 	return;
2281 
2282 done:
2283 	xfer->td_transfer_cache = NULL;
2284 	sc->sc_xfer_complete = 1;
2285 }
2286 
2287 static uint8_t
2288 dwc_otg_xfer_do_complete_locked(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2289 {
2290 	struct dwc_otg_td *td;
2291 
2292 	DPRINTFN(9, "\n");
2293 
2294 	td = xfer->td_transfer_cache;
2295 	if (td == NULL) {
2296 		/* compute all actual lengths */
2297 		dwc_otg_standard_done(xfer);
2298 		return (1);
2299 	}
2300 	return (0);
2301 }
2302 
2303 static void
2304 dwc_otg_timer(void *_sc)
2305 {
2306 	struct dwc_otg_softc *sc = _sc;
2307 
2308 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2309 
2310 	DPRINTF("\n");
2311 
2312 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
2313 
2314 	/* increment timer value */
2315 	sc->sc_tmr_val++;
2316 
2317 	/* enable SOF interrupt, which will poll jobs */
2318 	dwc_otg_enable_sof_irq(sc);
2319 
2320 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2321 
2322 	if (sc->sc_timer_active) {
2323 		/* restart timer */
2324 		usb_callout_reset(&sc->sc_timer,
2325 		    hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2326 		    &dwc_otg_timer, sc);
2327 	}
2328 }
2329 
2330 static void
2331 dwc_otg_timer_start(struct dwc_otg_softc *sc)
2332 {
2333 	if (sc->sc_timer_active != 0)
2334 		return;
2335 
2336 	sc->sc_timer_active = 1;
2337 
2338 	/* restart timer */
2339 	usb_callout_reset(&sc->sc_timer,
2340 	    hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2341 	    &dwc_otg_timer, sc);
2342 }
2343 
2344 static void
2345 dwc_otg_timer_stop(struct dwc_otg_softc *sc)
2346 {
2347 	if (sc->sc_timer_active == 0)
2348 		return;
2349 
2350 	sc->sc_timer_active = 0;
2351 
2352 	/* stop timer */
2353 	usb_callout_stop(&sc->sc_timer);
2354 }
2355 
2356 static uint16_t
2357 dwc_otg_compute_isoc_rx_tt_slot(struct dwc_otg_tt_info *pinfo)
2358 {
2359 	if (pinfo->slot_index < DWC_OTG_TT_SLOT_MAX)
2360 		pinfo->slot_index++;
2361 	return (pinfo->slot_index);
2362 }
2363 
2364 static uint8_t
2365 dwc_otg_update_host_transfer_schedule_locked(struct dwc_otg_softc *sc)
2366 {
2367 	TAILQ_HEAD(, usb_xfer) head;
2368 	struct usb_xfer *xfer;
2369 	struct usb_xfer *xfer_next;
2370 	struct dwc_otg_td *td;
2371 	uint16_t temp;
2372 	uint16_t slot;
2373 
2374 	temp = DWC_OTG_READ_4(sc, DOTG_HFNUM) & DWC_OTG_FRAME_MASK;
2375 
2376 	if (sc->sc_last_frame_num == temp)
2377 		return (0);
2378 
2379 	sc->sc_last_frame_num = temp;
2380 
2381 	TAILQ_INIT(&head);
2382 
2383 	if ((temp & 7) == 0) {
2384 
2385 		/* reset the schedule */
2386 		memset(sc->sc_tt_info, 0, sizeof(sc->sc_tt_info));
2387 
2388 		TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2389 			td = xfer->td_transfer_cache;
2390 			if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2391 				continue;
2392 
2393 			/* check for IN direction */
2394 			if ((td->hcchar & HCCHAR_EPDIR_IN) != 0)
2395 				continue;
2396 
2397 			sc->sc_needsof = 1;
2398 
2399 			if (td->hcsplt == 0 || td->tt_scheduled != 0)
2400 				continue;
2401 
2402 			/* compute slot */
2403 			slot = dwc_otg_compute_isoc_rx_tt_slot(
2404 			    sc->sc_tt_info + td->tt_index);
2405 			if (slot > 3) {
2406 				/*
2407 				 * Not enough time to get complete
2408 				 * split executed.
2409 				 */
2410 				continue;
2411 			}
2412 			/* Delayed start */
2413 			td->tt_start_slot = temp + slot;
2414 			td->tt_scheduled = 1;
2415 			TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2416 			TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2417 		}
2418 
2419 		TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2420 			td = xfer->td_transfer_cache;
2421 			if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2422 				continue;
2423 
2424 			/* check for OUT direction */
2425 			if ((td->hcchar & HCCHAR_EPDIR_IN) == 0)
2426 				continue;
2427 
2428 			sc->sc_needsof = 1;
2429 
2430 			if (td->hcsplt == 0 || td->tt_scheduled != 0)
2431 				continue;
2432 
2433 			/* Start ASAP */
2434 			td->tt_start_slot = temp;
2435 			td->tt_scheduled = 1;
2436 			TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2437 			TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2438 		}
2439 
2440 		TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2441 			td = xfer->td_transfer_cache;
2442 			if (td == NULL || td->ep_type != UE_INTERRUPT)
2443 				continue;
2444 
2445 			if (td->tt_scheduled != 0) {
2446 				sc->sc_needsof = 1;
2447 				continue;
2448 			}
2449 
2450 			if (dwc_otg_host_rate_check_interrupt(sc, td))
2451 				continue;
2452 
2453 			if (td->hcsplt == 0) {
2454 				sc->sc_needsof = 1;
2455 				td->tt_scheduled = 1;
2456 				continue;
2457 			}
2458 
2459 			/* start ASAP */
2460 			td->tt_start_slot = temp;
2461 			sc->sc_needsof = 1;
2462 			td->tt_scheduled = 1;
2463 			TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2464 			TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2465 		}
2466 
2467 		TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2468 			td = xfer->td_transfer_cache;
2469 			if (td == NULL ||
2470 			    td->ep_type != UE_CONTROL) {
2471 				continue;
2472 			}
2473 
2474 			sc->sc_needsof = 1;
2475 
2476 			if (td->hcsplt == 0 || td->tt_scheduled != 0)
2477 				continue;
2478 
2479 			/* start ASAP */
2480 			td->tt_start_slot = temp;
2481 			td->tt_scheduled = 1;
2482 			TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2483 			TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2484 		}
2485 	}
2486 	if ((temp & 7) < 6) {
2487 		TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2488 			td = xfer->td_transfer_cache;
2489 			if (td == NULL ||
2490 			    td->ep_type != UE_BULK) {
2491 				continue;
2492 			}
2493 
2494 			sc->sc_needsof = 1;
2495 
2496 			if (td->hcsplt == 0 || td->tt_scheduled != 0)
2497 				continue;
2498 
2499 			/* start ASAP */
2500 			td->tt_start_slot = temp;
2501 			td->tt_scheduled = 1;
2502 			TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2503 			TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2504 		}
2505 	}
2506 
2507 	/* Put TT transfers in execution order at the end */
2508 	TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2509 
2510 	/* move all TT transfers in front, keeping the current order */
2511 	TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2512 		td = xfer->td_transfer_cache;
2513 		if (td == NULL || td->hcsplt == 0)
2514 			continue;
2515 		TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2516 		TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2517 	}
2518 	TAILQ_CONCAT(&head, &sc->sc_bus.intr_q.head, wait_entry);
2519 	TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2520 
2521 	/* put non-TT non-ISOCHRONOUS transfers last */
2522 	TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2523 		td = xfer->td_transfer_cache;
2524 		if (td == NULL || td->hcsplt != 0 || td->ep_type == UE_ISOCHRONOUS)
2525 			continue;
2526 		TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2527 		TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2528 	}
2529 	TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2530 
2531 	if ((temp & 7) == 0) {
2532 
2533 		DPRINTFN(12, "SOF interrupt #%d, needsof=%d\n",
2534 		    (int)temp, (int)sc->sc_needsof);
2535 
2536 		/* update SOF IRQ mask */
2537 		if (sc->sc_irq_mask & GINTMSK_SOFMSK) {
2538 			if (sc->sc_needsof == 0) {
2539 				sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2540 				DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2541 			}
2542 		} else {
2543 			if (sc->sc_needsof != 0) {
2544 				sc->sc_irq_mask |= GINTMSK_SOFMSK;
2545 				DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2546 			}
2547 		}
2548 
2549 		/* clear need SOF flag */
2550 		sc->sc_needsof = 0;
2551 	}
2552 	return (1);
2553 }
2554 
2555 static void
2556 dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *sc)
2557 {
2558 	struct usb_xfer *xfer;
2559 	uint32_t count;
2560 	uint32_t temp;
2561 	uint8_t got_rx_status;
2562 	uint8_t x;
2563 
2564 	if (sc->sc_flags.status_device_mode == 0) {
2565 		/*
2566 		 * Update host transfer schedule, so that new
2567 		 * transfers can be issued:
2568 		 */
2569 		dwc_otg_update_host_transfer_schedule_locked(sc);
2570 	}
2571 	count = 0;
2572 repeat:
2573 	if (++count == 16) {
2574 		/* give other interrupts a chance */
2575 		DPRINTF("Yield\n");
2576 		return;
2577 	}
2578 	/* get all host channel interrupts */
2579 	for (x = 0; x != sc->sc_host_ch_max; x++) {
2580 		temp = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
2581 		if (temp != 0) {
2582 			DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), temp);
2583 			temp &= ~HCINT_SOFTWARE_ONLY;
2584 			sc->sc_chan_state[x].hcint |= temp;
2585 		}
2586 	}
2587 
2588 	if (sc->sc_last_rx_status == 0) {
2589 
2590 		temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2591 		if (temp & GINTSTS_RXFLVL) {
2592 			/* pop current status */
2593 			sc->sc_last_rx_status =
2594 			    DWC_OTG_READ_4(sc, DOTG_GRXSTSPD);
2595 		}
2596 
2597 		if (sc->sc_last_rx_status != 0) {
2598 
2599 			uint8_t ep_no;
2600 
2601 			temp = sc->sc_last_rx_status &
2602 			    GRXSTSRD_PKTSTS_MASK;
2603 
2604 			/* non-data messages we simply skip */
2605 			if (temp != GRXSTSRD_STP_DATA &&
2606 			    temp != GRXSTSRD_STP_COMPLETE &&
2607 			    temp != GRXSTSRD_OUT_DATA) {
2608 				/* check for halted channel */
2609 				if (temp == GRXSTSRH_HALTED) {
2610 					ep_no = GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status);
2611 					sc->sc_chan_state[ep_no].wait_halted = 0;
2612 					DPRINTFN(5, "channel halt complete ch=%u\n", ep_no);
2613 				}
2614 				dwc_otg_common_rx_ack(sc);
2615 				goto repeat;
2616 			}
2617 
2618 			temp = GRXSTSRD_BCNT_GET(
2619 			    sc->sc_last_rx_status);
2620 			ep_no = GRXSTSRD_CHNUM_GET(
2621 			    sc->sc_last_rx_status);
2622 
2623 			/* receive data, if any */
2624 			if (temp != 0) {
2625 				DPRINTF("Reading %d bytes from ep %d\n", temp, ep_no);
2626 				bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
2627 				    DOTG_DFIFO(ep_no),
2628 				    sc->sc_rx_bounce_buffer, (temp + 3) / 4);
2629 			}
2630 
2631 			/* check if we should dump the data */
2632 			if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2633 				dwc_otg_common_rx_ack(sc);
2634 				goto repeat;
2635 			}
2636 
2637 			got_rx_status = 1;
2638 
2639 			DPRINTFN(5, "RX status = 0x%08x: ch=%d pid=%d bytes=%d sts=%d\n",
2640 			    sc->sc_last_rx_status, ep_no,
2641 			    (sc->sc_last_rx_status >> 15) & 3,
2642 			    GRXSTSRD_BCNT_GET(sc->sc_last_rx_status),
2643 			    (sc->sc_last_rx_status >> 17) & 15);
2644 		} else {
2645 			got_rx_status = 0;
2646 		}
2647 	} else {
2648 		uint8_t ep_no;
2649 
2650 		ep_no = GRXSTSRD_CHNUM_GET(
2651 		    sc->sc_last_rx_status);
2652 
2653 		/* check if we should dump the data */
2654 		if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2655 			dwc_otg_common_rx_ack(sc);
2656 			goto repeat;
2657 		}
2658 
2659 		got_rx_status = 1;
2660 	}
2661 
2662 	/* execute FIFOs */
2663 	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry)
2664 		dwc_otg_xfer_do_fifo(sc, xfer);
2665 
2666 	if (got_rx_status) {
2667 		/* check if data was consumed */
2668 		if (sc->sc_last_rx_status == 0)
2669 			goto repeat;
2670 
2671 		/* disable RX FIFO level interrupt */
2672 		sc->sc_irq_mask &= ~GINTMSK_RXFLVLMSK;
2673 		DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2674 	}
2675 }
2676 
2677 static void
2678 dwc_otg_interrupt_complete_locked(struct dwc_otg_softc *sc)
2679 {
2680 	struct usb_xfer *xfer;
2681 repeat:
2682 	/* scan for completion events */
2683 	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
2684 		if (dwc_otg_xfer_do_complete_locked(sc, xfer))
2685 			goto repeat;
2686 	}
2687 }
2688 
2689 static void
2690 dwc_otg_vbus_interrupt(struct dwc_otg_softc *sc, uint8_t is_on)
2691 {
2692 	DPRINTFN(5, "vbus = %u\n", is_on);
2693 
2694 	/*
2695 	 * If the USB host mode is forced, then assume VBUS is always
2696 	 * present else rely on the input to this function:
2697 	 */
2698 	if ((is_on != 0) || (sc->sc_mode == DWC_MODE_HOST)) {
2699 
2700 		if (!sc->sc_flags.status_vbus) {
2701 			sc->sc_flags.status_vbus = 1;
2702 
2703 			/* complete root HUB interrupt endpoint */
2704 
2705 			dwc_otg_root_intr(sc);
2706 		}
2707 	} else {
2708 		if (sc->sc_flags.status_vbus) {
2709 			sc->sc_flags.status_vbus = 0;
2710 			sc->sc_flags.status_bus_reset = 0;
2711 			sc->sc_flags.status_suspend = 0;
2712 			sc->sc_flags.change_suspend = 0;
2713 			sc->sc_flags.change_connect = 1;
2714 
2715 			/* complete root HUB interrupt endpoint */
2716 
2717 			dwc_otg_root_intr(sc);
2718 		}
2719 	}
2720 }
2721 
2722 int
2723 dwc_otg_filter_interrupt(void *arg)
2724 {
2725 	struct dwc_otg_softc *sc = arg;
2726 	int retval = FILTER_HANDLED;
2727 	uint32_t status;
2728 
2729 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
2730 
2731 	/* read and clear interrupt status */
2732 	status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2733 
2734 	/* clear interrupts we are handling here */
2735 	DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & ~DWC_OTG_MSK_GINT_THREAD_IRQ);
2736 
2737 	/* check for USB state change interrupts */
2738 	if ((status & DWC_OTG_MSK_GINT_THREAD_IRQ) != 0)
2739 		retval = FILTER_SCHEDULE_THREAD;
2740 
2741 	/* clear FIFO empty interrupts */
2742 	if (status & sc->sc_irq_mask &
2743 	    (GINTSTS_PTXFEMP | GINTSTS_NPTXFEMP)) {
2744 		sc->sc_irq_mask &= ~(GINTSTS_PTXFEMP | GINTSTS_NPTXFEMP);
2745 		DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2746 	}
2747 	/* clear all IN endpoint interrupts */
2748 	if (status & GINTSTS_IEPINT) {
2749 		uint32_t temp;
2750 		uint8_t x;
2751 
2752 		for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
2753 			temp = DWC_OTG_READ_4(sc, DOTG_DIEPINT(x));
2754 			if (temp & DIEPMSK_XFERCOMPLMSK) {
2755 				DWC_OTG_WRITE_4(sc, DOTG_DIEPINT(x),
2756 				    DIEPMSK_XFERCOMPLMSK);
2757 			}
2758 		}
2759 	}
2760 
2761 	/* poll FIFOs, if any */
2762 	dwc_otg_interrupt_poll_locked(sc);
2763 
2764 	if (sc->sc_xfer_complete != 0)
2765 		retval = FILTER_SCHEDULE_THREAD;
2766 
2767 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2768 
2769 	return (retval);
2770 }
2771 
2772 void
2773 dwc_otg_interrupt(void *arg)
2774 {
2775 	struct dwc_otg_softc *sc = arg;
2776 	uint32_t status;
2777 
2778 	USB_BUS_LOCK(&sc->sc_bus);
2779 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
2780 
2781 	/* read and clear interrupt status */
2782 	status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2783 
2784 	/* clear interrupts we are handling here */
2785 	DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & DWC_OTG_MSK_GINT_THREAD_IRQ);
2786 
2787 	DPRINTFN(14, "GINTSTS=0x%08x HAINT=0x%08x HFNUM=0x%08x\n",
2788 	    status, DWC_OTG_READ_4(sc, DOTG_HAINT),
2789 	    DWC_OTG_READ_4(sc, DOTG_HFNUM));
2790 
2791 	if (status & GINTSTS_USBRST) {
2792 
2793 		/* set correct state */
2794 		sc->sc_flags.status_device_mode = 1;
2795 		sc->sc_flags.status_bus_reset = 0;
2796 		sc->sc_flags.status_suspend = 0;
2797 		sc->sc_flags.change_suspend = 0;
2798 		sc->sc_flags.change_connect = 1;
2799 
2800 		/* Disable SOF interrupt */
2801 		sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2802 		DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2803 
2804 		/* complete root HUB interrupt endpoint */
2805 		dwc_otg_root_intr(sc);
2806 	}
2807 
2808 	/* check for any bus state change interrupts */
2809 	if (status & GINTSTS_ENUMDONE) {
2810 
2811 		uint32_t temp;
2812 
2813 		DPRINTFN(5, "end of reset\n");
2814 
2815 		/* set correct state */
2816 		sc->sc_flags.status_device_mode = 1;
2817 		sc->sc_flags.status_bus_reset = 1;
2818 		sc->sc_flags.status_suspend = 0;
2819 		sc->sc_flags.change_suspend = 0;
2820 		sc->sc_flags.change_connect = 1;
2821 		sc->sc_flags.status_low_speed = 0;
2822 		sc->sc_flags.port_enabled = 1;
2823 
2824 		/* reset FIFOs */
2825 		(void) dwc_otg_init_fifo(sc, DWC_MODE_DEVICE);
2826 
2827 		/* reset function address */
2828 		dwc_otg_set_address(sc, 0);
2829 
2830 		/* figure out enumeration speed */
2831 		temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
2832 		if (DSTS_ENUMSPD_GET(temp) == DSTS_ENUMSPD_HI)
2833 			sc->sc_flags.status_high_speed = 1;
2834 		else
2835 			sc->sc_flags.status_high_speed = 0;
2836 
2837 		/*
2838 		 * Disable resume and SOF interrupt, and enable
2839 		 * suspend and RX frame interrupt:
2840 		 */
2841 		sc->sc_irq_mask &= ~(GINTMSK_WKUPINTMSK | GINTMSK_SOFMSK);
2842 		sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
2843 		DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2844 
2845 		/* complete root HUB interrupt endpoint */
2846 		dwc_otg_root_intr(sc);
2847 	}
2848 
2849 	if (status & GINTSTS_PRTINT) {
2850 		uint32_t hprt;
2851 
2852 		hprt = DWC_OTG_READ_4(sc, DOTG_HPRT);
2853 
2854 		/* clear change bits */
2855 		DWC_OTG_WRITE_4(sc, DOTG_HPRT, (hprt & (
2856 		    HPRT_PRTPWR | HPRT_PRTENCHNG |
2857 		    HPRT_PRTCONNDET | HPRT_PRTOVRCURRCHNG)) |
2858 		    sc->sc_hprt_val);
2859 
2860 		DPRINTFN(12, "GINTSTS=0x%08x, HPRT=0x%08x\n", status, hprt);
2861 
2862 		sc->sc_flags.status_device_mode = 0;
2863 
2864 		if (hprt & HPRT_PRTCONNSTS)
2865 			sc->sc_flags.status_bus_reset = 1;
2866 		else
2867 			sc->sc_flags.status_bus_reset = 0;
2868 
2869 		if (hprt & HPRT_PRTENCHNG)
2870 			sc->sc_flags.change_enabled = 1;
2871 
2872 		if (hprt & HPRT_PRTENA)
2873 			sc->sc_flags.port_enabled = 1;
2874 		else
2875 			sc->sc_flags.port_enabled = 0;
2876 
2877 		if (hprt & HPRT_PRTOVRCURRCHNG)
2878 			sc->sc_flags.change_over_current = 1;
2879 
2880 		if (hprt & HPRT_PRTOVRCURRACT)
2881 			sc->sc_flags.port_over_current = 1;
2882 		else
2883 			sc->sc_flags.port_over_current = 0;
2884 
2885 		if (hprt & HPRT_PRTPWR)
2886 			sc->sc_flags.port_powered = 1;
2887 		else
2888 			sc->sc_flags.port_powered = 0;
2889 
2890 		if (((hprt & HPRT_PRTSPD_MASK)
2891 		    >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_LOW)
2892 			sc->sc_flags.status_low_speed = 1;
2893 		else
2894 			sc->sc_flags.status_low_speed = 0;
2895 
2896 		if (((hprt & HPRT_PRTSPD_MASK)
2897 		    >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_HIGH)
2898 			sc->sc_flags.status_high_speed = 1;
2899 		else
2900 			sc->sc_flags.status_high_speed = 0;
2901 
2902 		if (hprt & HPRT_PRTCONNDET)
2903 			sc->sc_flags.change_connect = 1;
2904 
2905 		if (hprt & HPRT_PRTSUSP)
2906 			dwc_otg_suspend_irq(sc);
2907 		else
2908 			dwc_otg_resume_irq(sc);
2909 
2910 		/* complete root HUB interrupt endpoint */
2911 		dwc_otg_root_intr(sc);
2912 
2913 		/* update host frame interval */
2914 		dwc_otg_update_host_frame_interval(sc);
2915 	}
2916 
2917 	/*
2918 	 * If resume and suspend is set at the same time we interpret
2919 	 * that like RESUME. Resume is set when there is at least 3
2920 	 * milliseconds of inactivity on the USB BUS.
2921 	 */
2922 	if (status & GINTSTS_WKUPINT) {
2923 
2924 		DPRINTFN(5, "resume interrupt\n");
2925 
2926 		dwc_otg_resume_irq(sc);
2927 
2928 	} else if (status & GINTSTS_USBSUSP) {
2929 
2930 		DPRINTFN(5, "suspend interrupt\n");
2931 
2932 		dwc_otg_suspend_irq(sc);
2933 	}
2934 	/* check VBUS */
2935 	if (status & (GINTSTS_USBSUSP |
2936 	    GINTSTS_USBRST |
2937 	    GINTMSK_OTGINTMSK |
2938 	    GINTSTS_SESSREQINT)) {
2939 		uint32_t temp;
2940 
2941 		temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
2942 
2943 		DPRINTFN(5, "GOTGCTL=0x%08x\n", temp);
2944 
2945 		dwc_otg_vbus_interrupt(sc,
2946 		    (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
2947 	}
2948 
2949 	if (sc->sc_xfer_complete != 0) {
2950 		sc->sc_xfer_complete = 0;
2951 
2952 		/* complete FIFOs, if any */
2953 		dwc_otg_interrupt_complete_locked(sc);
2954 	}
2955 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2956 	USB_BUS_UNLOCK(&sc->sc_bus);
2957 }
2958 
2959 static void
2960 dwc_otg_setup_standard_chain_sub(struct dwc_otg_std_temp *temp)
2961 {
2962 	struct dwc_otg_td *td;
2963 
2964 	/* get current Transfer Descriptor */
2965 	td = temp->td_next;
2966 	temp->td = td;
2967 
2968 	/* prepare for next TD */
2969 	temp->td_next = td->obj_next;
2970 
2971 	/* fill out the Transfer Descriptor */
2972 	td->func = temp->func;
2973 	td->pc = temp->pc;
2974 	td->offset = temp->offset;
2975 	td->remainder = temp->len;
2976 	td->tx_bytes = 0;
2977 	td->error_any = 0;
2978 	td->error_stall = 0;
2979 	td->npkt = 0;
2980 	td->did_stall = temp->did_stall;
2981 	td->short_pkt = temp->short_pkt;
2982 	td->alt_next = temp->setup_alt_next;
2983 	td->set_toggle = 0;
2984 	td->got_short = 0;
2985 	td->did_nak = 0;
2986 	td->channel[0] = DWC_OTG_MAX_CHANNELS;
2987 	td->channel[1] = DWC_OTG_MAX_CHANNELS;
2988 	td->channel[2] = DWC_OTG_MAX_CHANNELS;
2989 	td->state = 0;
2990 	td->errcnt = 0;
2991 	td->tt_scheduled = 0;
2992 	td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
2993 }
2994 
2995 static void
2996 dwc_otg_setup_standard_chain(struct usb_xfer *xfer)
2997 {
2998 	struct dwc_otg_std_temp temp;
2999 	struct dwc_otg_td *td;
3000 	uint32_t x;
3001 	uint8_t need_sync;
3002 	uint8_t is_host;
3003 
3004 	DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
3005 	    xfer->address, UE_GET_ADDR(xfer->endpointno),
3006 	    xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
3007 
3008 	temp.max_frame_size = xfer->max_frame_size;
3009 
3010 	td = xfer->td_start[0];
3011 	xfer->td_transfer_first = td;
3012 	xfer->td_transfer_cache = td;
3013 
3014 	/* setup temp */
3015 
3016 	temp.pc = NULL;
3017 	temp.td = NULL;
3018 	temp.td_next = xfer->td_start[0];
3019 	temp.offset = 0;
3020 	temp.setup_alt_next = xfer->flags_int.short_frames_ok ||
3021 	    xfer->flags_int.isochronous_xfr;
3022 	temp.did_stall = !xfer->flags_int.control_stall;
3023 
3024 	is_host = (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST);
3025 
3026 	/* check if we should prepend a setup message */
3027 
3028 	if (xfer->flags_int.control_xfr) {
3029 		if (xfer->flags_int.control_hdr) {
3030 
3031 			if (is_host)
3032 				temp.func = &dwc_otg_host_setup_tx;
3033 			else
3034 				temp.func = &dwc_otg_setup_rx;
3035 
3036 			temp.len = xfer->frlengths[0];
3037 			temp.pc = xfer->frbuffers + 0;
3038 			temp.short_pkt = temp.len ? 1 : 0;
3039 
3040 			/* check for last frame */
3041 			if (xfer->nframes == 1) {
3042 				/* no STATUS stage yet, SETUP is last */
3043 				if (xfer->flags_int.control_act)
3044 					temp.setup_alt_next = 0;
3045 			}
3046 
3047 			dwc_otg_setup_standard_chain_sub(&temp);
3048 		}
3049 		x = 1;
3050 	} else {
3051 		x = 0;
3052 	}
3053 
3054 	if (x != xfer->nframes) {
3055 		if (xfer->endpointno & UE_DIR_IN) {
3056 			if (is_host) {
3057 				temp.func = &dwc_otg_host_data_rx;
3058 				need_sync = 0;
3059 			} else {
3060 				temp.func = &dwc_otg_data_tx;
3061 				need_sync = 1;
3062 			}
3063 		} else {
3064 			if (is_host) {
3065 				temp.func = &dwc_otg_host_data_tx;
3066 				need_sync = 0;
3067 			} else {
3068 				temp.func = &dwc_otg_data_rx;
3069 				need_sync = 0;
3070 			}
3071 		}
3072 
3073 		/* setup "pc" pointer */
3074 		temp.pc = xfer->frbuffers + x;
3075 	} else {
3076 		need_sync = 0;
3077 	}
3078 	while (x != xfer->nframes) {
3079 
3080 		/* DATA0 / DATA1 message */
3081 
3082 		temp.len = xfer->frlengths[x];
3083 
3084 		x++;
3085 
3086 		if (x == xfer->nframes) {
3087 			if (xfer->flags_int.control_xfr) {
3088 				if (xfer->flags_int.control_act) {
3089 					temp.setup_alt_next = 0;
3090 				}
3091 			} else {
3092 				temp.setup_alt_next = 0;
3093 			}
3094 		}
3095 		if (temp.len == 0) {
3096 
3097 			/* make sure that we send an USB packet */
3098 
3099 			temp.short_pkt = 0;
3100 
3101 		} else {
3102 
3103 			/* regular data transfer */
3104 
3105 			temp.short_pkt = (xfer->flags.force_short_xfer ? 0 : 1);
3106 		}
3107 
3108 		dwc_otg_setup_standard_chain_sub(&temp);
3109 
3110 		if (xfer->flags_int.isochronous_xfr) {
3111 			temp.offset += temp.len;
3112 		} else {
3113 			/* get next Page Cache pointer */
3114 			temp.pc = xfer->frbuffers + x;
3115 		}
3116 	}
3117 
3118 	if (xfer->flags_int.control_xfr) {
3119 
3120 		/* always setup a valid "pc" pointer for status and sync */
3121 		temp.pc = xfer->frbuffers + 0;
3122 		temp.len = 0;
3123 		temp.short_pkt = 0;
3124 		temp.setup_alt_next = 0;
3125 
3126 		/* check if we need to sync */
3127 		if (need_sync) {
3128 			/* we need a SYNC point after TX */
3129 			temp.func = &dwc_otg_data_tx_sync;
3130 			dwc_otg_setup_standard_chain_sub(&temp);
3131 		}
3132 
3133 		/* check if we should append a status stage */
3134 		if (!xfer->flags_int.control_act) {
3135 
3136 			/*
3137 			 * Send a DATA1 message and invert the current
3138 			 * endpoint direction.
3139 			 */
3140 			if (xfer->endpointno & UE_DIR_IN) {
3141 				if (is_host) {
3142 					temp.func = &dwc_otg_host_data_tx;
3143 					need_sync = 0;
3144 				} else {
3145 					temp.func = &dwc_otg_data_rx;
3146 					need_sync = 0;
3147 				}
3148 			} else {
3149 				if (is_host) {
3150 					temp.func = &dwc_otg_host_data_rx;
3151 					need_sync = 0;
3152 				} else {
3153 					temp.func = &dwc_otg_data_tx;
3154 					need_sync = 1;
3155 				}
3156 			}
3157 
3158 			dwc_otg_setup_standard_chain_sub(&temp);
3159 
3160 			/* data toggle should be DATA1 */
3161 			td = temp.td;
3162 			td->set_toggle = 1;
3163 
3164 			if (need_sync) {
3165 				/* we need a SYNC point after TX */
3166 				temp.func = &dwc_otg_data_tx_sync;
3167 				dwc_otg_setup_standard_chain_sub(&temp);
3168 			}
3169 		}
3170 	} else {
3171 		/* check if we need to sync */
3172 		if (need_sync) {
3173 
3174 			temp.pc = xfer->frbuffers + 0;
3175 			temp.len = 0;
3176 			temp.short_pkt = 0;
3177 			temp.setup_alt_next = 0;
3178 
3179 			/* we need a SYNC point after TX */
3180 			temp.func = &dwc_otg_data_tx_sync;
3181 			dwc_otg_setup_standard_chain_sub(&temp);
3182 		}
3183 	}
3184 
3185 	/* must have at least one frame! */
3186 	td = temp.td;
3187 	xfer->td_transfer_last = td;
3188 
3189 	if (is_host) {
3190 
3191 		struct dwc_otg_softc *sc;
3192 		uint32_t hcchar;
3193 		uint32_t hcsplt;
3194 
3195 		sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3196 
3197 		/* get first again */
3198 		td = xfer->td_transfer_first;
3199 		td->toggle = (xfer->endpoint->toggle_next ? 1 : 0);
3200 
3201 		hcchar =
3202 			(xfer->address << HCCHAR_DEVADDR_SHIFT) |
3203 			((xfer->endpointno & UE_ADDR) << HCCHAR_EPNUM_SHIFT) |
3204 			(xfer->max_packet_size << HCCHAR_MPS_SHIFT) |
3205 			HCCHAR_CHENA;
3206 
3207 		/*
3208 		 * We are not always able to meet the timing
3209 		 * requirements of the USB interrupt endpoint's
3210 		 * complete split token, when doing transfers going
3211 		 * via a transaction translator. Use the CONTROL
3212 		 * transfer type instead of the INTERRUPT transfer
3213 		 * type in general, as a means to workaround
3214 		 * that. This trick should work for both FULL and LOW
3215 		 * speed USB traffic going through a TT. For non-TT
3216 		 * traffic it works aswell. The reason for using
3217 		 * CONTROL type instead of BULK is that some TTs might
3218 		 * reject LOW speed BULK traffic.
3219 		 */
3220 		if (td->ep_type == UE_INTERRUPT)
3221 			hcchar |= (UE_CONTROL << HCCHAR_EPTYPE_SHIFT);
3222 		else
3223 			hcchar |= (td->ep_type << HCCHAR_EPTYPE_SHIFT);
3224 
3225 		if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_LOW)
3226 			hcchar |= HCCHAR_LSPDDEV;
3227 		if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN)
3228 			hcchar |= HCCHAR_EPDIR_IN;
3229 
3230 		switch (xfer->xroot->udev->speed) {
3231 		case USB_SPEED_FULL:
3232 		case USB_SPEED_LOW:
3233 			/* check if root HUB port is running High Speed */
3234 			if (xfer->xroot->udev->parent_hs_hub != NULL) {
3235 				hcsplt = HCSPLT_SPLTENA |
3236 				    (xfer->xroot->udev->hs_port_no <<
3237 				    HCSPLT_PRTADDR_SHIFT) |
3238 				    (xfer->xroot->udev->hs_hub_addr <<
3239 				    HCSPLT_HUBADDR_SHIFT);
3240 			} else {
3241 				hcsplt = 0;
3242 			}
3243 			if (td->ep_type == UE_INTERRUPT) {
3244 				uint32_t ival;
3245 				ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3246 				if (ival == 0)
3247 					ival = 1;
3248 				else if (ival > 127)
3249 					ival = 127;
3250 				td->tmr_val = sc->sc_tmr_val + ival;
3251 				td->tmr_res = ival;
3252 			} else if (td->ep_type == UE_ISOCHRONOUS) {
3253 				td->tmr_res = 1;
3254 				td->tmr_val = sc->sc_last_frame_num;
3255 				if (td->hcchar & HCCHAR_EPDIR_IN)
3256 					td->tmr_val++;
3257 			} else {
3258 				td->tmr_val = 0;
3259 				td->tmr_res = (uint8_t)sc->sc_last_frame_num;
3260 			}
3261 			break;
3262 		case USB_SPEED_HIGH:
3263 			hcsplt = 0;
3264 			if (td->ep_type == UE_INTERRUPT) {
3265 				uint32_t ival;
3266 				hcchar |= ((xfer->max_packet_count & 3)
3267 				    << HCCHAR_MC_SHIFT);
3268 				ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3269 				if (ival == 0)
3270 					ival = 1;
3271 				else if (ival > 127)
3272 					ival = 127;
3273 				td->tmr_val = sc->sc_tmr_val + ival;
3274 				td->tmr_res = ival;
3275 			} else if (td->ep_type == UE_ISOCHRONOUS) {
3276 				hcchar |= ((xfer->max_packet_count & 3)
3277 				    << HCCHAR_MC_SHIFT);
3278 				td->tmr_res = 1 << usbd_xfer_get_fps_shift(xfer);
3279 				td->tmr_val = sc->sc_last_frame_num;
3280 				if (td->hcchar & HCCHAR_EPDIR_IN)
3281 					td->tmr_val += td->tmr_res;
3282 
3283 			} else {
3284 				td->tmr_val = 0;
3285 				td->tmr_res = (uint8_t)sc->sc_last_frame_num;
3286 			}
3287 			break;
3288 		default:
3289 			hcsplt = 0;
3290 			td->tmr_val = 0;
3291 			td->tmr_res = 0;
3292 			break;
3293 		}
3294 
3295 		/* store configuration in all TD's */
3296 		while (1) {
3297 			td->hcchar = hcchar;
3298 			td->hcsplt = hcsplt;
3299 
3300 			if (((void *)td) == xfer->td_transfer_last)
3301 				break;
3302 
3303 			td = td->obj_next;
3304 		}
3305 	}
3306 }
3307 
3308 static void
3309 dwc_otg_timeout(void *arg)
3310 {
3311 	struct usb_xfer *xfer = arg;
3312 
3313 	DPRINTF("xfer=%p\n", xfer);
3314 
3315 	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
3316 
3317 	/* transfer is transferred */
3318 	dwc_otg_device_done(xfer, USB_ERR_TIMEOUT);
3319 }
3320 
3321 static void
3322 dwc_otg_start_standard_chain(struct usb_xfer *xfer)
3323 {
3324 	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3325 
3326 	DPRINTFN(9, "\n");
3327 
3328 	/*
3329 	 * Poll one time in device mode, which will turn on the
3330 	 * endpoint interrupts. Else wait for SOF interrupt in host
3331 	 * mode.
3332 	 */
3333 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
3334 
3335 	if (sc->sc_flags.status_device_mode != 0) {
3336 		dwc_otg_xfer_do_fifo(sc, xfer);
3337 		if (dwc_otg_xfer_do_complete_locked(sc, xfer))
3338 			goto done;
3339 	} else {
3340 		struct dwc_otg_td *td = xfer->td_transfer_cache;
3341 		if (td->ep_type == UE_ISOCHRONOUS &&
3342 		    (td->hcchar & HCCHAR_EPDIR_IN) == 0) {
3343 			/*
3344 			 * Need to start ISOCHRONOUS OUT transfer ASAP
3345 			 * because execution is delayed by one 125us
3346 			 * microframe:
3347 			 */
3348 			dwc_otg_xfer_do_fifo(sc, xfer);
3349 			if (dwc_otg_xfer_do_complete_locked(sc, xfer))
3350 				goto done;
3351 		}
3352 	}
3353 
3354 	/* put transfer on interrupt queue */
3355 	usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
3356 
3357 	/* start timeout, if any */
3358 	if (xfer->timeout != 0) {
3359 		usbd_transfer_timeout_ms(xfer,
3360 		    &dwc_otg_timeout, xfer->timeout);
3361 	}
3362 
3363 	if (sc->sc_flags.status_device_mode != 0)
3364 		goto done;
3365 
3366 	/* enable SOF interrupt, if any */
3367 	dwc_otg_enable_sof_irq(sc);
3368 done:
3369 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3370 }
3371 
3372 static void
3373 dwc_otg_root_intr(struct dwc_otg_softc *sc)
3374 {
3375 	DPRINTFN(9, "\n");
3376 
3377 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3378 
3379 	/* set port bit */
3380 	sc->sc_hub_idata[0] = 0x02;	/* we only have one port */
3381 
3382 	uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
3383 	    sizeof(sc->sc_hub_idata));
3384 }
3385 
3386 static usb_error_t
3387 dwc_otg_standard_done_sub(struct usb_xfer *xfer)
3388 {
3389 	struct dwc_otg_td *td;
3390 	uint32_t len;
3391 	usb_error_t error;
3392 
3393 	DPRINTFN(9, "\n");
3394 
3395 	td = xfer->td_transfer_cache;
3396 
3397 	do {
3398 		len = td->remainder;
3399 
3400 		/* store last data toggle */
3401 		xfer->endpoint->toggle_next = td->toggle;
3402 
3403 		if (xfer->aframes != xfer->nframes) {
3404 			/*
3405 			 * Verify the length and subtract
3406 			 * the remainder from "frlengths[]":
3407 			 */
3408 			if (len > xfer->frlengths[xfer->aframes]) {
3409 				td->error_any = 1;
3410 			} else {
3411 				xfer->frlengths[xfer->aframes] -= len;
3412 			}
3413 		}
3414 		/* Check for transfer error */
3415 		if (td->error_any) {
3416 			/* the transfer is finished */
3417 			error = (td->error_stall ?
3418 			    USB_ERR_STALLED : USB_ERR_IOERROR);
3419 			td = NULL;
3420 			break;
3421 		}
3422 		/* Check for short transfer */
3423 		if (len > 0) {
3424 			if (xfer->flags_int.short_frames_ok ||
3425 			    xfer->flags_int.isochronous_xfr) {
3426 				/* follow alt next */
3427 				if (td->alt_next) {
3428 					td = td->obj_next;
3429 				} else {
3430 					td = NULL;
3431 				}
3432 			} else {
3433 				/* the transfer is finished */
3434 				td = NULL;
3435 			}
3436 			error = 0;
3437 			break;
3438 		}
3439 		td = td->obj_next;
3440 
3441 		/* this USB frame is complete */
3442 		error = 0;
3443 		break;
3444 
3445 	} while (0);
3446 
3447 	/* update transfer cache */
3448 
3449 	xfer->td_transfer_cache = td;
3450 
3451 	return (error);
3452 }
3453 
3454 static void
3455 dwc_otg_standard_done(struct usb_xfer *xfer)
3456 {
3457 	usb_error_t err = 0;
3458 
3459 	DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
3460 	    xfer, xfer->endpoint);
3461 
3462 	/* reset scanner */
3463 
3464 	xfer->td_transfer_cache = xfer->td_transfer_first;
3465 
3466 	if (xfer->flags_int.control_xfr) {
3467 
3468 		if (xfer->flags_int.control_hdr) {
3469 
3470 			err = dwc_otg_standard_done_sub(xfer);
3471 		}
3472 		xfer->aframes = 1;
3473 
3474 		if (xfer->td_transfer_cache == NULL) {
3475 			goto done;
3476 		}
3477 	}
3478 	while (xfer->aframes != xfer->nframes) {
3479 
3480 		err = dwc_otg_standard_done_sub(xfer);
3481 		xfer->aframes++;
3482 
3483 		if (xfer->td_transfer_cache == NULL) {
3484 			goto done;
3485 		}
3486 	}
3487 
3488 	if (xfer->flags_int.control_xfr &&
3489 	    !xfer->flags_int.control_act) {
3490 
3491 		err = dwc_otg_standard_done_sub(xfer);
3492 	}
3493 done:
3494 	dwc_otg_device_done(xfer, err);
3495 }
3496 
3497 /*------------------------------------------------------------------------*
3498  *	dwc_otg_device_done
3499  *
3500  * NOTE: this function can be called more than one time on the
3501  * same USB transfer!
3502  *------------------------------------------------------------------------*/
3503 static void
3504 dwc_otg_device_done(struct usb_xfer *xfer, usb_error_t error)
3505 {
3506 	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3507 
3508 	DPRINTFN(9, "xfer=%p, endpoint=%p, error=%d\n",
3509 	    xfer, xfer->endpoint, error);
3510 
3511 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
3512 
3513 	if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
3514 		/* Interrupts are cleared by the interrupt handler */
3515 	} else {
3516 		struct dwc_otg_td *td;
3517 
3518 		td = xfer->td_transfer_cache;
3519  		if (td != NULL)
3520 			dwc_otg_host_channel_free(sc, td);
3521 	}
3522 	/* dequeue transfer and start next transfer */
3523 	usbd_transfer_done(xfer, error);
3524 
3525 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3526 }
3527 
3528 static void
3529 dwc_otg_xfer_stall(struct usb_xfer *xfer)
3530 {
3531 	dwc_otg_device_done(xfer, USB_ERR_STALLED);
3532 }
3533 
3534 static void
3535 dwc_otg_set_stall(struct usb_device *udev,
3536     struct usb_endpoint *ep, uint8_t *did_stall)
3537 {
3538 	struct dwc_otg_softc *sc;
3539 	uint32_t temp;
3540 	uint32_t reg;
3541 	uint8_t ep_no;
3542 
3543 	USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3544 
3545 	/* check mode */
3546 	if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3547 		/* not supported */
3548 		return;
3549 	}
3550 
3551 	sc = DWC_OTG_BUS2SC(udev->bus);
3552 
3553 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
3554 
3555 	/* get endpoint address */
3556 	ep_no = ep->edesc->bEndpointAddress;
3557 
3558 	DPRINTFN(5, "endpoint=0x%x\n", ep_no);
3559 
3560 	if (ep_no & UE_DIR_IN) {
3561 		reg = DOTG_DIEPCTL(ep_no & UE_ADDR);
3562 		temp = sc->sc_in_ctl[ep_no & UE_ADDR];
3563 	} else {
3564 		reg = DOTG_DOEPCTL(ep_no & UE_ADDR);
3565 		temp = sc->sc_out_ctl[ep_no & UE_ADDR];
3566 	}
3567 
3568 	/* disable and stall endpoint */
3569 	DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3570 	DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_STALL);
3571 
3572 	/* clear active OUT ep */
3573 	if (!(ep_no & UE_DIR_IN)) {
3574 
3575 		sc->sc_active_rx_ep &= ~(1U << (ep_no & UE_ADDR));
3576 
3577 		if (sc->sc_last_rx_status != 0 &&
3578 		    (ep_no & UE_ADDR) == GRXSTSRD_CHNUM_GET(
3579 		    sc->sc_last_rx_status)) {
3580 			/* dump data */
3581 			dwc_otg_common_rx_ack(sc);
3582 			/* poll interrupt */
3583 			dwc_otg_interrupt_poll_locked(sc);
3584 			dwc_otg_interrupt_complete_locked(sc);
3585 		}
3586 	}
3587 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3588 }
3589 
3590 static void
3591 dwc_otg_clear_stall_sub_locked(struct dwc_otg_softc *sc, uint32_t mps,
3592     uint8_t ep_no, uint8_t ep_type, uint8_t ep_dir)
3593 {
3594 	uint32_t reg;
3595 	uint32_t temp;
3596 
3597 	if (ep_type == UE_CONTROL) {
3598 		/* clearing stall is not needed */
3599 		return;
3600 	}
3601 
3602 	if (ep_dir) {
3603 		reg = DOTG_DIEPCTL(ep_no);
3604 	} else {
3605 		reg = DOTG_DOEPCTL(ep_no);
3606 		sc->sc_active_rx_ep |= (1U << ep_no);
3607 	}
3608 
3609 	/* round up and mask away the multiplier count */
3610 	mps = (mps + 3) & 0x7FC;
3611 
3612 	if (ep_type == UE_BULK) {
3613 		temp = DIEPCTL_EPTYPE_SET(
3614 		    DIEPCTL_EPTYPE_BULK) |
3615 		    DIEPCTL_USBACTEP;
3616 	} else if (ep_type == UE_INTERRUPT) {
3617 		temp = DIEPCTL_EPTYPE_SET(
3618 		    DIEPCTL_EPTYPE_INTERRUPT) |
3619 		    DIEPCTL_USBACTEP;
3620 	} else {
3621 		temp = DIEPCTL_EPTYPE_SET(
3622 		    DIEPCTL_EPTYPE_ISOC) |
3623 		    DIEPCTL_USBACTEP;
3624 	}
3625 
3626 	temp |= DIEPCTL_MPS_SET(mps);
3627 	temp |= DIEPCTL_TXFNUM_SET(ep_no);
3628 
3629 	if (ep_dir)
3630 		sc->sc_in_ctl[ep_no] = temp;
3631 	else
3632 		sc->sc_out_ctl[ep_no] = temp;
3633 
3634 	DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3635 	DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_SETD0PID);
3636 	DWC_OTG_WRITE_4(sc, reg, temp | DIEPCTL_SNAK);
3637 
3638 	/* we only reset the transmit FIFO */
3639 	if (ep_dir) {
3640 		dwc_otg_tx_fifo_reset(sc,
3641 		    GRSTCTL_TXFIFO(ep_no) |
3642 		    GRSTCTL_TXFFLSH);
3643 
3644 		DWC_OTG_WRITE_4(sc,
3645 		    DOTG_DIEPTSIZ(ep_no), 0);
3646 	}
3647 
3648 	/* poll interrupt */
3649 	dwc_otg_interrupt_poll_locked(sc);
3650 	dwc_otg_interrupt_complete_locked(sc);
3651 }
3652 
3653 static void
3654 dwc_otg_clear_stall(struct usb_device *udev, struct usb_endpoint *ep)
3655 {
3656 	struct dwc_otg_softc *sc;
3657 	struct usb_endpoint_descriptor *ed;
3658 
3659 	DPRINTFN(5, "endpoint=%p\n", ep);
3660 
3661 	USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3662 
3663 	/* check mode */
3664 	if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3665 		/* not supported */
3666 		return;
3667 	}
3668 	/* get softc */
3669 	sc = DWC_OTG_BUS2SC(udev->bus);
3670 
3671 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
3672 
3673 	/* get endpoint descriptor */
3674 	ed = ep->edesc;
3675 
3676 	/* reset endpoint */
3677 	dwc_otg_clear_stall_sub_locked(sc,
3678 	    UGETW(ed->wMaxPacketSize),
3679 	    (ed->bEndpointAddress & UE_ADDR),
3680 	    (ed->bmAttributes & UE_XFERTYPE),
3681 	    (ed->bEndpointAddress & (UE_DIR_IN | UE_DIR_OUT)));
3682 
3683 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3684 }
3685 
3686 static void
3687 dwc_otg_device_state_change(struct usb_device *udev)
3688 {
3689 	struct dwc_otg_softc *sc;
3690 	uint8_t x;
3691 
3692 	/* check mode */
3693 	if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3694 		/* not supported */
3695 		return;
3696 	}
3697 
3698 	/* get softc */
3699 	sc = DWC_OTG_BUS2SC(udev->bus);
3700 
3701 	/* deactivate all other endpoint but the control endpoint */
3702 	if (udev->state == USB_STATE_CONFIGURED ||
3703 	    udev->state == USB_STATE_ADDRESSED) {
3704 
3705 		USB_BUS_LOCK(&sc->sc_bus);
3706 
3707 		for (x = 1; x != sc->sc_dev_ep_max; x++) {
3708 
3709 			if (x < sc->sc_dev_in_ep_max) {
3710 				DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x),
3711 				    DIEPCTL_EPDIS);
3712 				DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x), 0);
3713 			}
3714 
3715 			DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x),
3716 			    DOEPCTL_EPDIS);
3717 			DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x), 0);
3718 		}
3719 		USB_BUS_UNLOCK(&sc->sc_bus);
3720 	}
3721 }
3722 
3723 int
3724 dwc_otg_init(struct dwc_otg_softc *sc)
3725 {
3726 	uint32_t temp;
3727 
3728 	DPRINTF("start\n");
3729 
3730 	/* set up the bus structure */
3731 	sc->sc_bus.usbrev = USB_REV_2_0;
3732 	sc->sc_bus.methods = &dwc_otg_bus_methods;
3733 
3734 	usb_callout_init_mtx(&sc->sc_timer,
3735 	    &sc->sc_bus.bus_mtx, 0);
3736 
3737 	USB_BUS_LOCK(&sc->sc_bus);
3738 
3739 	/* turn on clocks */
3740 	dwc_otg_clocks_on(sc);
3741 
3742 	temp = DWC_OTG_READ_4(sc, DOTG_GSNPSID);
3743 	DPRINTF("Version = 0x%08x\n", temp);
3744 
3745 	/* disconnect */
3746 	DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3747 	    DCTL_SFTDISCON);
3748 
3749 	/* wait for host to detect disconnect */
3750 	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 32);
3751 
3752 	DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
3753 	    GRSTCTL_CSFTRST);
3754 
3755 	/* wait a little bit for block to reset */
3756 	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 128);
3757 
3758 	switch (sc->sc_mode) {
3759 	case DWC_MODE_DEVICE:
3760 		temp = GUSBCFG_FORCEDEVMODE;
3761 		break;
3762 	case DWC_MODE_HOST:
3763 		temp = GUSBCFG_FORCEHOSTMODE;
3764 		break;
3765 	default:
3766 		temp = 0;
3767 		break;
3768 	}
3769 
3770 	/* select HSIC, ULPI or internal PHY mode */
3771 	switch (dwc_otg_phy_type) {
3772 	case DWC_OTG_PHY_HSIC:
3773 		DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3774 		    GUSBCFG_PHYIF |
3775 		    GUSBCFG_TRD_TIM_SET(5) | temp);
3776 		DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL,
3777 		    0x000000EC);
3778 
3779 		temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3780 		DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3781 		    temp & ~GLPMCFG_HSIC_CONN);
3782 		DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3783 		    temp | GLPMCFG_HSIC_CONN);
3784 		break;
3785 	case DWC_OTG_PHY_ULPI:
3786 		DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3787 		    GUSBCFG_ULPI_UTMI_SEL |
3788 		    GUSBCFG_TRD_TIM_SET(5) | temp);
3789 		DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3790 
3791 		temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3792 		DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3793 		    temp & ~GLPMCFG_HSIC_CONN);
3794 		break;
3795 	case DWC_OTG_PHY_INTERNAL:
3796 		DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3797 		    GUSBCFG_PHYSEL |
3798 		    GUSBCFG_TRD_TIM_SET(5) | temp);
3799 		DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3800 
3801 		temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3802 		DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3803 		    temp & ~GLPMCFG_HSIC_CONN);
3804 
3805 		temp = DWC_OTG_READ_4(sc, DOTG_GGPIO);
3806 		temp &= ~(DOTG_GGPIO_NOVBUSSENS | DOTG_GGPIO_I2CPADEN);
3807 		temp |= (DOTG_GGPIO_VBUSASEN | DOTG_GGPIO_VBUSBSEN |
3808 		    DOTG_GGPIO_PWRDWN);
3809 		DWC_OTG_WRITE_4(sc, DOTG_GGPIO, temp);
3810 		break;
3811 	default:
3812 		break;
3813 	}
3814 
3815 	/* clear global nak */
3816 	DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3817 	    DCTL_CGOUTNAK |
3818 	    DCTL_CGNPINNAK);
3819 
3820 	/* disable USB port */
3821 	DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0xFFFFFFFF);
3822 
3823 	/* wait 10ms */
3824 	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3825 
3826 	/* enable USB port */
3827 	DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
3828 
3829 	/* wait 10ms */
3830 	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3831 
3832 	temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG3);
3833 
3834 	sc->sc_fifo_size = 4 * GHWCFG3_DFIFODEPTH_GET(temp);
3835 
3836 	temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
3837 
3838 	sc->sc_dev_ep_max = GHWCFG2_NUMDEVEPS_GET(temp);
3839 
3840 	if (sc->sc_dev_ep_max > DWC_OTG_MAX_ENDPOINTS)
3841 		sc->sc_dev_ep_max = DWC_OTG_MAX_ENDPOINTS;
3842 
3843 	sc->sc_host_ch_max = GHWCFG2_NUMHSTCHNL_GET(temp);
3844 
3845 	if (sc->sc_host_ch_max > DWC_OTG_MAX_CHANNELS)
3846 		sc->sc_host_ch_max = DWC_OTG_MAX_CHANNELS;
3847 
3848 	temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG4);
3849 
3850 	sc->sc_dev_in_ep_max = GHWCFG4_NUM_IN_EP_GET(temp);
3851 
3852 	DPRINTF("Total FIFO size = %d bytes, Device EPs = %d/%d Host CHs = %d\n",
3853 	    sc->sc_fifo_size, sc->sc_dev_ep_max, sc->sc_dev_in_ep_max,
3854 	    sc->sc_host_ch_max);
3855 
3856 	/* setup FIFO */
3857 	if (dwc_otg_init_fifo(sc, sc->sc_mode)) {
3858 		USB_BUS_UNLOCK(&sc->sc_bus);
3859 		return (EINVAL);
3860 	}
3861 
3862 	/* enable interrupts */
3863 	sc->sc_irq_mask = DWC_OTG_MSK_GINT_ENABLED;
3864 	DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
3865 
3866 	if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_DEVICE) {
3867 
3868 		/* enable all endpoint interrupts */
3869 		temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
3870 		if (temp & GHWCFG2_MPI) {
3871 			uint8_t x;
3872 
3873 			DPRINTF("Disable Multi Process Interrupts\n");
3874 
3875 			for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
3876 				DWC_OTG_WRITE_4(sc, DOTG_DIEPEACHINTMSK(x), 0);
3877 				DWC_OTG_WRITE_4(sc, DOTG_DOEPEACHINTMSK(x), 0);
3878 			}
3879 			DWC_OTG_WRITE_4(sc, DOTG_DEACHINTMSK, 0);
3880 		}
3881 		DWC_OTG_WRITE_4(sc, DOTG_DIEPMSK,
3882 		    DIEPMSK_XFERCOMPLMSK);
3883 		DWC_OTG_WRITE_4(sc, DOTG_DOEPMSK, 0);
3884 		DWC_OTG_WRITE_4(sc, DOTG_DAINTMSK, 0xFFFF);
3885 	}
3886 
3887 	if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_HOST) {
3888 		/* setup clocks */
3889 		temp = DWC_OTG_READ_4(sc, DOTG_HCFG);
3890 		temp &= ~(HCFG_FSLSSUPP | HCFG_FSLSPCLKSEL_MASK);
3891 		temp |= (1 << HCFG_FSLSPCLKSEL_SHIFT);
3892 		DWC_OTG_WRITE_4(sc, DOTG_HCFG, temp);
3893 	}
3894 
3895 	/* only enable global IRQ */
3896 	DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG,
3897 	    GAHBCFG_GLBLINTRMSK);
3898 
3899 	/* turn off clocks */
3900 	dwc_otg_clocks_off(sc);
3901 
3902 	/* read initial VBUS state */
3903 
3904 	temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
3905 
3906 	DPRINTFN(5, "GOTCTL=0x%08x\n", temp);
3907 
3908 	dwc_otg_vbus_interrupt(sc,
3909 	    (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
3910 
3911 	USB_BUS_UNLOCK(&sc->sc_bus);
3912 
3913 	/* catch any lost interrupts */
3914 
3915 	dwc_otg_do_poll(&sc->sc_bus);
3916 
3917 	return (0);			/* success */
3918 }
3919 
3920 void
3921 dwc_otg_uninit(struct dwc_otg_softc *sc)
3922 {
3923 	USB_BUS_LOCK(&sc->sc_bus);
3924 
3925 	/* stop host timer */
3926 	dwc_otg_timer_stop(sc);
3927 
3928 	/* set disconnect */
3929 	DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3930 	    DCTL_SFTDISCON);
3931 
3932 	/* turn off global IRQ */
3933 	DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG, 0);
3934 
3935 	sc->sc_flags.port_enabled = 0;
3936 	sc->sc_flags.port_powered = 0;
3937 	sc->sc_flags.status_vbus = 0;
3938 	sc->sc_flags.status_bus_reset = 0;
3939 	sc->sc_flags.status_suspend = 0;
3940 	sc->sc_flags.change_suspend = 0;
3941 	sc->sc_flags.change_connect = 1;
3942 
3943 	dwc_otg_pull_down(sc);
3944 	dwc_otg_clocks_off(sc);
3945 
3946 	USB_BUS_UNLOCK(&sc->sc_bus);
3947 
3948 	usb_callout_drain(&sc->sc_timer);
3949 }
3950 
3951 static void
3952 dwc_otg_suspend(struct dwc_otg_softc *sc)
3953 {
3954 	return;
3955 }
3956 
3957 static void
3958 dwc_otg_resume(struct dwc_otg_softc *sc)
3959 {
3960 	return;
3961 }
3962 
3963 static void
3964 dwc_otg_do_poll(struct usb_bus *bus)
3965 {
3966 	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
3967 
3968 	USB_BUS_LOCK(&sc->sc_bus);
3969 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
3970 	dwc_otg_interrupt_poll_locked(sc);
3971 	dwc_otg_interrupt_complete_locked(sc);
3972 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3973 	USB_BUS_UNLOCK(&sc->sc_bus);
3974 }
3975 
3976 /*------------------------------------------------------------------------*
3977  * DWC OTG bulk support
3978  * DWC OTG control support
3979  * DWC OTG interrupt support
3980  *------------------------------------------------------------------------*/
3981 static void
3982 dwc_otg_device_non_isoc_open(struct usb_xfer *xfer)
3983 {
3984 }
3985 
3986 static void
3987 dwc_otg_device_non_isoc_close(struct usb_xfer *xfer)
3988 {
3989 	dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
3990 }
3991 
3992 static void
3993 dwc_otg_device_non_isoc_enter(struct usb_xfer *xfer)
3994 {
3995 }
3996 
3997 static void
3998 dwc_otg_device_non_isoc_start(struct usb_xfer *xfer)
3999 {
4000 	/* setup TDs */
4001 	dwc_otg_setup_standard_chain(xfer);
4002 	dwc_otg_start_standard_chain(xfer);
4003 }
4004 
4005 static const struct usb_pipe_methods dwc_otg_device_non_isoc_methods =
4006 {
4007 	.open = dwc_otg_device_non_isoc_open,
4008 	.close = dwc_otg_device_non_isoc_close,
4009 	.enter = dwc_otg_device_non_isoc_enter,
4010 	.start = dwc_otg_device_non_isoc_start,
4011 };
4012 
4013 /*------------------------------------------------------------------------*
4014  * DWC OTG full speed isochronous support
4015  *------------------------------------------------------------------------*/
4016 static void
4017 dwc_otg_device_isoc_open(struct usb_xfer *xfer)
4018 {
4019 }
4020 
4021 static void
4022 dwc_otg_device_isoc_close(struct usb_xfer *xfer)
4023 {
4024 	dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
4025 }
4026 
4027 static void
4028 dwc_otg_device_isoc_enter(struct usb_xfer *xfer)
4029 {
4030 }
4031 
4032 static void
4033 dwc_otg_device_isoc_start(struct usb_xfer *xfer)
4034 {
4035 	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
4036 	uint32_t temp;
4037 	uint32_t msframes;
4038 	uint32_t framenum;
4039 	uint8_t shift = usbd_xfer_get_fps_shift(xfer);
4040 
4041 	DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
4042 	    xfer, xfer->endpoint->isoc_next, xfer->nframes);
4043 
4044 	if (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST) {
4045 		temp = DWC_OTG_READ_4(sc, DOTG_HFNUM);
4046 
4047 		/* get the current frame index */
4048 		framenum = (temp & HFNUM_FRNUM_MASK);
4049 	} else {
4050 		temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
4051 
4052 		/* get the current frame index */
4053 		framenum = DSTS_SOFFN_GET(temp);
4054 	}
4055 
4056 	if (xfer->xroot->udev->parent_hs_hub != NULL)
4057 		framenum /= 8;
4058 
4059 	framenum &= DWC_OTG_FRAME_MASK;
4060 
4061 	/*
4062 	 * Compute number of milliseconds worth of data traffic for
4063 	 * this USB transfer:
4064 	 */
4065 	if (xfer->xroot->udev->speed == USB_SPEED_HIGH)
4066 		msframes = ((xfer->nframes << shift) + 7) / 8;
4067 	else
4068 		msframes = xfer->nframes;
4069 
4070 	/*
4071 	 * check if the frame index is within the window where the frames
4072 	 * will be inserted
4073 	 */
4074 	temp = (framenum - xfer->endpoint->isoc_next) & DWC_OTG_FRAME_MASK;
4075 
4076 	if ((xfer->endpoint->is_synced == 0) || (temp < msframes)) {
4077 		/*
4078 		 * If there is data underflow or the pipe queue is
4079 		 * empty we schedule the transfer a few frames ahead
4080 		 * of the current frame position. Else two isochronous
4081 		 * transfers might overlap.
4082 		 */
4083 		xfer->endpoint->isoc_next = (framenum + 3) & DWC_OTG_FRAME_MASK;
4084 		xfer->endpoint->is_synced = 1;
4085 		DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
4086 	}
4087 	/*
4088 	 * compute how many milliseconds the insertion is ahead of the
4089 	 * current frame position:
4090 	 */
4091 	temp = (xfer->endpoint->isoc_next - framenum) & DWC_OTG_FRAME_MASK;
4092 
4093 	/*
4094 	 * pre-compute when the isochronous transfer will be finished:
4095 	 */
4096 	xfer->isoc_time_complete =
4097 		usb_isoc_time_expand(&sc->sc_bus, framenum) + temp + msframes;
4098 
4099 	/* setup TDs */
4100 	dwc_otg_setup_standard_chain(xfer);
4101 
4102 	/* compute frame number for next insertion */
4103 	xfer->endpoint->isoc_next += msframes;
4104 
4105 	/* start TD chain */
4106 	dwc_otg_start_standard_chain(xfer);
4107 }
4108 
4109 static const struct usb_pipe_methods dwc_otg_device_isoc_methods =
4110 {
4111 	.open = dwc_otg_device_isoc_open,
4112 	.close = dwc_otg_device_isoc_close,
4113 	.enter = dwc_otg_device_isoc_enter,
4114 	.start = dwc_otg_device_isoc_start,
4115 };
4116 
4117 /*------------------------------------------------------------------------*
4118  * DWC OTG root control support
4119  *------------------------------------------------------------------------*
4120  * Simulate a hardware HUB by handling all the necessary requests.
4121  *------------------------------------------------------------------------*/
4122 
4123 static const struct usb_device_descriptor dwc_otg_devd = {
4124 	.bLength = sizeof(struct usb_device_descriptor),
4125 	.bDescriptorType = UDESC_DEVICE,
4126 	.bcdUSB = {0x00, 0x02},
4127 	.bDeviceClass = UDCLASS_HUB,
4128 	.bDeviceSubClass = UDSUBCLASS_HUB,
4129 	.bDeviceProtocol = UDPROTO_HSHUBSTT,
4130 	.bMaxPacketSize = 64,
4131 	.bcdDevice = {0x00, 0x01},
4132 	.iManufacturer = 1,
4133 	.iProduct = 2,
4134 	.bNumConfigurations = 1,
4135 };
4136 
4137 static const struct dwc_otg_config_desc dwc_otg_confd = {
4138 	.confd = {
4139 		.bLength = sizeof(struct usb_config_descriptor),
4140 		.bDescriptorType = UDESC_CONFIG,
4141 		.wTotalLength[0] = sizeof(dwc_otg_confd),
4142 		.bNumInterface = 1,
4143 		.bConfigurationValue = 1,
4144 		.iConfiguration = 0,
4145 		.bmAttributes = UC_SELF_POWERED,
4146 		.bMaxPower = 0,
4147 	},
4148 	.ifcd = {
4149 		.bLength = sizeof(struct usb_interface_descriptor),
4150 		.bDescriptorType = UDESC_INTERFACE,
4151 		.bNumEndpoints = 1,
4152 		.bInterfaceClass = UICLASS_HUB,
4153 		.bInterfaceSubClass = UISUBCLASS_HUB,
4154 		.bInterfaceProtocol = 0,
4155 	},
4156 	.endpd = {
4157 		.bLength = sizeof(struct usb_endpoint_descriptor),
4158 		.bDescriptorType = UDESC_ENDPOINT,
4159 		.bEndpointAddress = (UE_DIR_IN | DWC_OTG_INTR_ENDPT),
4160 		.bmAttributes = UE_INTERRUPT,
4161 		.wMaxPacketSize[0] = 8,
4162 		.bInterval = 255,
4163 	},
4164 };
4165 
4166 #define	HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) }
4167 
4168 static const struct usb_hub_descriptor_min dwc_otg_hubd = {
4169 	.bDescLength = sizeof(dwc_otg_hubd),
4170 	.bDescriptorType = UDESC_HUB,
4171 	.bNbrPorts = 1,
4172 	HSETW(.wHubCharacteristics, (UHD_PWR_NO_SWITCH | UHD_OC_INDIVIDUAL)),
4173 	.bPwrOn2PwrGood = 50,
4174 	.bHubContrCurrent = 0,
4175 	.DeviceRemovable = {0},		/* port is removable */
4176 };
4177 
4178 #define	STRING_VENDOR \
4179   "D\0W\0C\0O\0T\0G"
4180 
4181 #define	STRING_PRODUCT \
4182   "O\0T\0G\0 \0R\0o\0o\0t\0 \0H\0U\0B"
4183 
4184 USB_MAKE_STRING_DESC(STRING_VENDOR, dwc_otg_vendor);
4185 USB_MAKE_STRING_DESC(STRING_PRODUCT, dwc_otg_product);
4186 
4187 static usb_error_t
4188 dwc_otg_roothub_exec(struct usb_device *udev,
4189     struct usb_device_request *req, const void **pptr, uint16_t *plength)
4190 {
4191 	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4192 	const void *ptr;
4193 	uint16_t len;
4194 	uint16_t value;
4195 	uint16_t index;
4196 	usb_error_t err;
4197 
4198 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
4199 
4200 	/* buffer reset */
4201 	ptr = (const void *)&sc->sc_hub_temp;
4202 	len = 0;
4203 	err = 0;
4204 
4205 	value = UGETW(req->wValue);
4206 	index = UGETW(req->wIndex);
4207 
4208 	/* demultiplex the control request */
4209 
4210 	switch (req->bmRequestType) {
4211 	case UT_READ_DEVICE:
4212 		switch (req->bRequest) {
4213 		case UR_GET_DESCRIPTOR:
4214 			goto tr_handle_get_descriptor;
4215 		case UR_GET_CONFIG:
4216 			goto tr_handle_get_config;
4217 		case UR_GET_STATUS:
4218 			goto tr_handle_get_status;
4219 		default:
4220 			goto tr_stalled;
4221 		}
4222 		break;
4223 
4224 	case UT_WRITE_DEVICE:
4225 		switch (req->bRequest) {
4226 		case UR_SET_ADDRESS:
4227 			goto tr_handle_set_address;
4228 		case UR_SET_CONFIG:
4229 			goto tr_handle_set_config;
4230 		case UR_CLEAR_FEATURE:
4231 			goto tr_valid;	/* nop */
4232 		case UR_SET_DESCRIPTOR:
4233 			goto tr_valid;	/* nop */
4234 		case UR_SET_FEATURE:
4235 		default:
4236 			goto tr_stalled;
4237 		}
4238 		break;
4239 
4240 	case UT_WRITE_ENDPOINT:
4241 		switch (req->bRequest) {
4242 		case UR_CLEAR_FEATURE:
4243 			switch (UGETW(req->wValue)) {
4244 			case UF_ENDPOINT_HALT:
4245 				goto tr_handle_clear_halt;
4246 			case UF_DEVICE_REMOTE_WAKEUP:
4247 				goto tr_handle_clear_wakeup;
4248 			default:
4249 				goto tr_stalled;
4250 			}
4251 			break;
4252 		case UR_SET_FEATURE:
4253 			switch (UGETW(req->wValue)) {
4254 			case UF_ENDPOINT_HALT:
4255 				goto tr_handle_set_halt;
4256 			case UF_DEVICE_REMOTE_WAKEUP:
4257 				goto tr_handle_set_wakeup;
4258 			default:
4259 				goto tr_stalled;
4260 			}
4261 			break;
4262 		case UR_SYNCH_FRAME:
4263 			goto tr_valid;	/* nop */
4264 		default:
4265 			goto tr_stalled;
4266 		}
4267 		break;
4268 
4269 	case UT_READ_ENDPOINT:
4270 		switch (req->bRequest) {
4271 		case UR_GET_STATUS:
4272 			goto tr_handle_get_ep_status;
4273 		default:
4274 			goto tr_stalled;
4275 		}
4276 		break;
4277 
4278 	case UT_WRITE_INTERFACE:
4279 		switch (req->bRequest) {
4280 		case UR_SET_INTERFACE:
4281 			goto tr_handle_set_interface;
4282 		case UR_CLEAR_FEATURE:
4283 			goto tr_valid;	/* nop */
4284 		case UR_SET_FEATURE:
4285 		default:
4286 			goto tr_stalled;
4287 		}
4288 		break;
4289 
4290 	case UT_READ_INTERFACE:
4291 		switch (req->bRequest) {
4292 		case UR_GET_INTERFACE:
4293 			goto tr_handle_get_interface;
4294 		case UR_GET_STATUS:
4295 			goto tr_handle_get_iface_status;
4296 		default:
4297 			goto tr_stalled;
4298 		}
4299 		break;
4300 
4301 	case UT_WRITE_CLASS_INTERFACE:
4302 	case UT_WRITE_VENDOR_INTERFACE:
4303 		/* XXX forward */
4304 		break;
4305 
4306 	case UT_READ_CLASS_INTERFACE:
4307 	case UT_READ_VENDOR_INTERFACE:
4308 		/* XXX forward */
4309 		break;
4310 
4311 	case UT_WRITE_CLASS_DEVICE:
4312 		switch (req->bRequest) {
4313 		case UR_CLEAR_FEATURE:
4314 			goto tr_valid;
4315 		case UR_SET_DESCRIPTOR:
4316 		case UR_SET_FEATURE:
4317 			break;
4318 		default:
4319 			goto tr_stalled;
4320 		}
4321 		break;
4322 
4323 	case UT_WRITE_CLASS_OTHER:
4324 		switch (req->bRequest) {
4325 		case UR_CLEAR_FEATURE:
4326 			goto tr_handle_clear_port_feature;
4327 		case UR_SET_FEATURE:
4328 			goto tr_handle_set_port_feature;
4329 		case UR_CLEAR_TT_BUFFER:
4330 		case UR_RESET_TT:
4331 		case UR_STOP_TT:
4332 			goto tr_valid;
4333 
4334 		default:
4335 			goto tr_stalled;
4336 		}
4337 		break;
4338 
4339 	case UT_READ_CLASS_OTHER:
4340 		switch (req->bRequest) {
4341 		case UR_GET_TT_STATE:
4342 			goto tr_handle_get_tt_state;
4343 		case UR_GET_STATUS:
4344 			goto tr_handle_get_port_status;
4345 		default:
4346 			goto tr_stalled;
4347 		}
4348 		break;
4349 
4350 	case UT_READ_CLASS_DEVICE:
4351 		switch (req->bRequest) {
4352 		case UR_GET_DESCRIPTOR:
4353 			goto tr_handle_get_class_descriptor;
4354 		case UR_GET_STATUS:
4355 			goto tr_handle_get_class_status;
4356 
4357 		default:
4358 			goto tr_stalled;
4359 		}
4360 		break;
4361 	default:
4362 		goto tr_stalled;
4363 	}
4364 	goto tr_valid;
4365 
4366 tr_handle_get_descriptor:
4367 	switch (value >> 8) {
4368 	case UDESC_DEVICE:
4369 		if (value & 0xff) {
4370 			goto tr_stalled;
4371 		}
4372 		len = sizeof(dwc_otg_devd);
4373 		ptr = (const void *)&dwc_otg_devd;
4374 		goto tr_valid;
4375 	case UDESC_CONFIG:
4376 		if (value & 0xff) {
4377 			goto tr_stalled;
4378 		}
4379 		len = sizeof(dwc_otg_confd);
4380 		ptr = (const void *)&dwc_otg_confd;
4381 		goto tr_valid;
4382 	case UDESC_STRING:
4383 		switch (value & 0xff) {
4384 		case 0:		/* Language table */
4385 			len = sizeof(usb_string_lang_en);
4386 			ptr = (const void *)&usb_string_lang_en;
4387 			goto tr_valid;
4388 
4389 		case 1:		/* Vendor */
4390 			len = sizeof(dwc_otg_vendor);
4391 			ptr = (const void *)&dwc_otg_vendor;
4392 			goto tr_valid;
4393 
4394 		case 2:		/* Product */
4395 			len = sizeof(dwc_otg_product);
4396 			ptr = (const void *)&dwc_otg_product;
4397 			goto tr_valid;
4398 		default:
4399 			break;
4400 		}
4401 		break;
4402 	default:
4403 		goto tr_stalled;
4404 	}
4405 	goto tr_stalled;
4406 
4407 tr_handle_get_config:
4408 	len = 1;
4409 	sc->sc_hub_temp.wValue[0] = sc->sc_conf;
4410 	goto tr_valid;
4411 
4412 tr_handle_get_status:
4413 	len = 2;
4414 	USETW(sc->sc_hub_temp.wValue, UDS_SELF_POWERED);
4415 	goto tr_valid;
4416 
4417 tr_handle_set_address:
4418 	if (value & 0xFF00) {
4419 		goto tr_stalled;
4420 	}
4421 	sc->sc_rt_addr = value;
4422 	goto tr_valid;
4423 
4424 tr_handle_set_config:
4425 	if (value >= 2) {
4426 		goto tr_stalled;
4427 	}
4428 	sc->sc_conf = value;
4429 	goto tr_valid;
4430 
4431 tr_handle_get_interface:
4432 	len = 1;
4433 	sc->sc_hub_temp.wValue[0] = 0;
4434 	goto tr_valid;
4435 
4436 tr_handle_get_tt_state:
4437 tr_handle_get_class_status:
4438 tr_handle_get_iface_status:
4439 tr_handle_get_ep_status:
4440 	len = 2;
4441 	USETW(sc->sc_hub_temp.wValue, 0);
4442 	goto tr_valid;
4443 
4444 tr_handle_set_halt:
4445 tr_handle_set_interface:
4446 tr_handle_set_wakeup:
4447 tr_handle_clear_wakeup:
4448 tr_handle_clear_halt:
4449 	goto tr_valid;
4450 
4451 tr_handle_clear_port_feature:
4452 	if (index != 1)
4453 		goto tr_stalled;
4454 
4455 	DPRINTFN(9, "UR_CLEAR_PORT_FEATURE on port %d\n", index);
4456 
4457 	switch (value) {
4458 	case UHF_PORT_SUSPEND:
4459 		dwc_otg_wakeup_peer(sc);
4460 		break;
4461 
4462 	case UHF_PORT_ENABLE:
4463 		if (sc->sc_flags.status_device_mode == 0) {
4464 			DWC_OTG_WRITE_4(sc, DOTG_HPRT,
4465 			    sc->sc_hprt_val | HPRT_PRTENA);
4466 		}
4467 		sc->sc_flags.port_enabled = 0;
4468 		break;
4469 
4470 	case UHF_C_PORT_RESET:
4471 		sc->sc_flags.change_reset = 0;
4472 		break;
4473 
4474 	case UHF_C_PORT_ENABLE:
4475 		sc->sc_flags.change_enabled = 0;
4476 		break;
4477 
4478 	case UHF_C_PORT_OVER_CURRENT:
4479 		sc->sc_flags.change_over_current = 0;
4480 		break;
4481 
4482 	case UHF_PORT_TEST:
4483 	case UHF_PORT_INDICATOR:
4484 		/* nops */
4485 		break;
4486 
4487 	case UHF_PORT_POWER:
4488 		sc->sc_flags.port_powered = 0;
4489 		if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4490 			sc->sc_hprt_val = 0;
4491 			DWC_OTG_WRITE_4(sc, DOTG_HPRT, HPRT_PRTENA);
4492 		}
4493 		dwc_otg_pull_down(sc);
4494 		dwc_otg_clocks_off(sc);
4495 		break;
4496 
4497 	case UHF_C_PORT_CONNECTION:
4498 		/* clear connect change flag */
4499 		sc->sc_flags.change_connect = 0;
4500 		break;
4501 
4502 	case UHF_C_PORT_SUSPEND:
4503 		sc->sc_flags.change_suspend = 0;
4504 		break;
4505 
4506 	default:
4507 		err = USB_ERR_IOERROR;
4508 		goto done;
4509 	}
4510 	goto tr_valid;
4511 
4512 tr_handle_set_port_feature:
4513 	if (index != 1) {
4514 		goto tr_stalled;
4515 	}
4516 	DPRINTFN(9, "UR_SET_PORT_FEATURE\n");
4517 
4518 	switch (value) {
4519 	case UHF_PORT_ENABLE:
4520 		break;
4521 
4522 	case UHF_PORT_SUSPEND:
4523 		if (sc->sc_flags.status_device_mode == 0) {
4524 			/* set suspend BIT */
4525 			sc->sc_hprt_val |= HPRT_PRTSUSP;
4526 			DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4527 
4528 			/* generate HUB suspend event */
4529 			dwc_otg_suspend_irq(sc);
4530 		}
4531 		break;
4532 
4533 	case UHF_PORT_RESET:
4534 		if (sc->sc_flags.status_device_mode == 0) {
4535 
4536 			DPRINTF("PORT RESET\n");
4537 
4538 			/* enable PORT reset */
4539 			DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val | HPRT_PRTRST);
4540 
4541 			/* Wait 62.5ms for reset to complete */
4542 			usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4543 
4544 			DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4545 
4546 			/* Wait 62.5ms for reset to complete */
4547 			usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4548 
4549 			/* reset FIFOs */
4550 			(void) dwc_otg_init_fifo(sc, DWC_MODE_HOST);
4551 
4552 			sc->sc_flags.change_reset = 1;
4553 		} else {
4554 			err = USB_ERR_IOERROR;
4555 		}
4556 		break;
4557 
4558 	case UHF_PORT_TEST:
4559 	case UHF_PORT_INDICATOR:
4560 		/* nops */
4561 		break;
4562 	case UHF_PORT_POWER:
4563 		sc->sc_flags.port_powered = 1;
4564 		if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4565 			sc->sc_hprt_val |= HPRT_PRTPWR;
4566 			DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4567 		}
4568 		if (sc->sc_mode == DWC_MODE_DEVICE || sc->sc_mode == DWC_MODE_OTG) {
4569 			/* pull up D+, if any */
4570 			dwc_otg_pull_up(sc);
4571 		}
4572 		break;
4573 	default:
4574 		err = USB_ERR_IOERROR;
4575 		goto done;
4576 	}
4577 	goto tr_valid;
4578 
4579 tr_handle_get_port_status:
4580 
4581 	DPRINTFN(9, "UR_GET_PORT_STATUS\n");
4582 
4583 	if (index != 1)
4584 		goto tr_stalled;
4585 
4586 	if (sc->sc_flags.status_vbus)
4587 		dwc_otg_clocks_on(sc);
4588 	else
4589 		dwc_otg_clocks_off(sc);
4590 
4591 	/* Select Device Side Mode */
4592 
4593 	if (sc->sc_flags.status_device_mode) {
4594 		value = UPS_PORT_MODE_DEVICE;
4595 		dwc_otg_timer_stop(sc);
4596 	} else {
4597 		value = 0;
4598 		dwc_otg_timer_start(sc);
4599 	}
4600 
4601 	if (sc->sc_flags.status_high_speed)
4602 		value |= UPS_HIGH_SPEED;
4603 	else if (sc->sc_flags.status_low_speed)
4604 		value |= UPS_LOW_SPEED;
4605 
4606 	if (sc->sc_flags.port_powered)
4607 		value |= UPS_PORT_POWER;
4608 
4609 	if (sc->sc_flags.port_enabled)
4610 		value |= UPS_PORT_ENABLED;
4611 
4612 	if (sc->sc_flags.port_over_current)
4613 		value |= UPS_OVERCURRENT_INDICATOR;
4614 
4615 	if (sc->sc_flags.status_vbus &&
4616 	    sc->sc_flags.status_bus_reset)
4617 		value |= UPS_CURRENT_CONNECT_STATUS;
4618 
4619 	if (sc->sc_flags.status_suspend)
4620 		value |= UPS_SUSPEND;
4621 
4622 	USETW(sc->sc_hub_temp.ps.wPortStatus, value);
4623 
4624 	value = 0;
4625 
4626 	if (sc->sc_flags.change_connect)
4627 		value |= UPS_C_CONNECT_STATUS;
4628 	if (sc->sc_flags.change_suspend)
4629 		value |= UPS_C_SUSPEND;
4630 	if (sc->sc_flags.change_reset)
4631 		value |= UPS_C_PORT_RESET;
4632 	if (sc->sc_flags.change_over_current)
4633 		value |= UPS_C_OVERCURRENT_INDICATOR;
4634 
4635 	USETW(sc->sc_hub_temp.ps.wPortChange, value);
4636 	len = sizeof(sc->sc_hub_temp.ps);
4637 	goto tr_valid;
4638 
4639 tr_handle_get_class_descriptor:
4640 	if (value & 0xFF) {
4641 		goto tr_stalled;
4642 	}
4643 	ptr = (const void *)&dwc_otg_hubd;
4644 	len = sizeof(dwc_otg_hubd);
4645 	goto tr_valid;
4646 
4647 tr_stalled:
4648 	err = USB_ERR_STALLED;
4649 tr_valid:
4650 done:
4651 	*plength = len;
4652 	*pptr = ptr;
4653 	return (err);
4654 }
4655 
4656 static void
4657 dwc_otg_xfer_setup(struct usb_setup_params *parm)
4658 {
4659 	struct usb_xfer *xfer;
4660 	void *last_obj;
4661 	uint32_t ntd;
4662 	uint32_t n;
4663 	uint8_t ep_no;
4664 	uint8_t ep_type;
4665 
4666 	xfer = parm->curr_xfer;
4667 
4668 	/*
4669 	 * NOTE: This driver does not use any of the parameters that
4670 	 * are computed from the following values. Just set some
4671 	 * reasonable dummies:
4672 	 */
4673 	parm->hc_max_packet_size = 0x500;
4674 	parm->hc_max_packet_count = 3;
4675 	parm->hc_max_frame_size = 3 * 0x500;
4676 
4677 	usbd_transfer_setup_sub(parm);
4678 
4679 	/*
4680 	 * compute maximum number of TDs
4681 	 */
4682 	ep_type = (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE);
4683 
4684 	if (ep_type == UE_CONTROL) {
4685 
4686 		ntd = xfer->nframes + 1 /* STATUS */ + 1 /* SYNC 1 */
4687 		    + 1 /* SYNC 2 */ + 1 /* SYNC 3 */;
4688 	} else {
4689 
4690 		ntd = xfer->nframes + 1 /* SYNC */ ;
4691 	}
4692 
4693 	/*
4694 	 * check if "usbd_transfer_setup_sub" set an error
4695 	 */
4696 	if (parm->err)
4697 		return;
4698 
4699 	/*
4700 	 * allocate transfer descriptors
4701 	 */
4702 	last_obj = NULL;
4703 
4704 	ep_no = xfer->endpointno & UE_ADDR;
4705 
4706 	/*
4707 	 * Check for a valid endpoint profile in USB device mode:
4708 	 */
4709 	if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
4710 		const struct usb_hw_ep_profile *pf;
4711 
4712 		dwc_otg_get_hw_ep_profile(parm->udev, &pf, ep_no);
4713 
4714 		if (pf == NULL) {
4715 			/* should not happen */
4716 			parm->err = USB_ERR_INVAL;
4717 			return;
4718 		}
4719 	}
4720 
4721 	/* align data */
4722 	parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
4723 
4724 	for (n = 0; n != ntd; n++) {
4725 
4726 		struct dwc_otg_td *td;
4727 
4728 		if (parm->buf) {
4729 
4730 			td = USB_ADD_BYTES(parm->buf, parm->size[0]);
4731 
4732 			/* compute shared bandwidth resource index for TT */
4733 			if (parm->udev->parent_hs_hub != NULL && parm->udev->speed != USB_SPEED_HIGH) {
4734 				if (parm->udev->parent_hs_hub->ddesc.bDeviceProtocol == UDPROTO_HSHUBMTT)
4735 					td->tt_index = parm->udev->device_index;
4736 				else
4737 					td->tt_index = parm->udev->parent_hs_hub->device_index;
4738 			} else {
4739 				td->tt_index = parm->udev->device_index;
4740 			}
4741 
4742 			/* init TD */
4743 			td->max_packet_size = xfer->max_packet_size;
4744 			td->max_packet_count = xfer->max_packet_count;
4745 			/* range check */
4746 			if (td->max_packet_count == 0 || td->max_packet_count > 3)
4747 				td->max_packet_count = 1;
4748 			td->ep_no = ep_no;
4749 			td->ep_type = ep_type;
4750 			td->obj_next = last_obj;
4751 
4752 			last_obj = td;
4753 		}
4754 		parm->size[0] += sizeof(*td);
4755 	}
4756 
4757 	xfer->td_start[0] = last_obj;
4758 }
4759 
4760 static void
4761 dwc_otg_xfer_unsetup(struct usb_xfer *xfer)
4762 {
4763 	return;
4764 }
4765 
4766 static void
4767 dwc_otg_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
4768     struct usb_endpoint *ep)
4769 {
4770 	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4771 
4772 	DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d,%d)\n",
4773 	    ep, udev->address,
4774 	    edesc->bEndpointAddress, udev->flags.usb_mode,
4775 	    sc->sc_rt_addr, udev->device_index);
4776 
4777 	if (udev->device_index != sc->sc_rt_addr) {
4778 
4779 		if (udev->flags.usb_mode == USB_MODE_DEVICE) {
4780 			if (udev->speed != USB_SPEED_FULL &&
4781 			    udev->speed != USB_SPEED_HIGH) {
4782 				/* not supported */
4783 				return;
4784 			}
4785 		} else {
4786 			if (udev->speed == USB_SPEED_HIGH &&
4787 			    (edesc->wMaxPacketSize[1] & 0x18) != 0 &&
4788 			    (edesc->bmAttributes & UE_XFERTYPE) != UE_ISOCHRONOUS) {
4789 				/* not supported */
4790 				DPRINTFN(-1, "Non-isochronous high bandwidth "
4791 				    "endpoint not supported\n");
4792 				return;
4793 			}
4794 		}
4795 		if ((edesc->bmAttributes & UE_XFERTYPE) == UE_ISOCHRONOUS)
4796 			ep->methods = &dwc_otg_device_isoc_methods;
4797 		else
4798 			ep->methods = &dwc_otg_device_non_isoc_methods;
4799 	}
4800 }
4801 
4802 static void
4803 dwc_otg_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
4804 {
4805 	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
4806 
4807 	switch (state) {
4808 	case USB_HW_POWER_SUSPEND:
4809 		dwc_otg_suspend(sc);
4810 		break;
4811 	case USB_HW_POWER_SHUTDOWN:
4812 		dwc_otg_uninit(sc);
4813 		break;
4814 	case USB_HW_POWER_RESUME:
4815 		dwc_otg_resume(sc);
4816 		break;
4817 	default:
4818 		break;
4819 	}
4820 }
4821 
4822 static void
4823 dwc_otg_get_dma_delay(struct usb_device *udev, uint32_t *pus)
4824 {
4825 	/* DMA delay - wait until any use of memory is finished */
4826 	*pus = (2125);			/* microseconds */
4827 }
4828 
4829 static void
4830 dwc_otg_device_resume(struct usb_device *udev)
4831 {
4832 	DPRINTF("\n");
4833 
4834 	/* poll all transfers again to restart resumed ones */
4835 	dwc_otg_do_poll(udev->bus);
4836 }
4837 
4838 static void
4839 dwc_otg_device_suspend(struct usb_device *udev)
4840 {
4841 	DPRINTF("\n");
4842 }
4843 
4844 static const struct usb_bus_methods dwc_otg_bus_methods =
4845 {
4846 	.endpoint_init = &dwc_otg_ep_init,
4847 	.xfer_setup = &dwc_otg_xfer_setup,
4848 	.xfer_unsetup = &dwc_otg_xfer_unsetup,
4849 	.get_hw_ep_profile = &dwc_otg_get_hw_ep_profile,
4850 	.xfer_stall = &dwc_otg_xfer_stall,
4851 	.set_stall = &dwc_otg_set_stall,
4852 	.clear_stall = &dwc_otg_clear_stall,
4853 	.roothub_exec = &dwc_otg_roothub_exec,
4854 	.xfer_poll = &dwc_otg_do_poll,
4855 	.device_state_change = &dwc_otg_device_state_change,
4856 	.set_hw_power_sleep = &dwc_otg_set_hw_power_sleep,
4857 	.get_dma_delay = &dwc_otg_get_dma_delay,
4858 	.device_resume = &dwc_otg_device_resume,
4859 	.device_suspend = &dwc_otg_device_suspend,
4860 };
4861