xref: /freebsd/sys/dev/usb/controller/dwc_otg.c (revision 298f5fdc242b760e70cd3494e3a4f1f50b20664d)
1 /* $FreeBSD$ */
2 /*-
3  * Copyright (c) 2012 Hans Petter Selasky. All rights reserved.
4  * Copyright (c) 2010-2011 Aleksandr Rybalko. All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  */
27 
28 /*
29  * This file contains the driver for the DesignWare series USB 2.0 OTG
30  * Controller.
31  */
32 
33 /*
34  * LIMITATION: Drivers must be bound to all OUT endpoints in the
35  * active configuration for this driver to work properly. Blocking any
36  * OUT endpoint will block all OUT endpoints including the control
37  * endpoint. Usually this is not a problem.
38  */
39 
40 /*
41  * NOTE: Writing to non-existing registers appears to cause an
42  * internal reset.
43  */
44 
45 #ifdef USB_GLOBAL_INCLUDE_FILE
46 #include USB_GLOBAL_INCLUDE_FILE
47 #else
48 #include <sys/stdint.h>
49 #include <sys/stddef.h>
50 #include <sys/param.h>
51 #include <sys/queue.h>
52 #include <sys/types.h>
53 #include <sys/systm.h>
54 #include <sys/kernel.h>
55 #include <sys/bus.h>
56 #include <sys/module.h>
57 #include <sys/lock.h>
58 #include <sys/mutex.h>
59 #include <sys/condvar.h>
60 #include <sys/sysctl.h>
61 #include <sys/sx.h>
62 #include <sys/unistd.h>
63 #include <sys/callout.h>
64 #include <sys/malloc.h>
65 #include <sys/priv.h>
66 
67 #include <dev/usb/usb.h>
68 #include <dev/usb/usbdi.h>
69 
70 #define	USB_DEBUG_VAR dwc_otg_debug
71 
72 #include <dev/usb/usb_core.h>
73 #include <dev/usb/usb_debug.h>
74 #include <dev/usb/usb_busdma.h>
75 #include <dev/usb/usb_process.h>
76 #include <dev/usb/usb_transfer.h>
77 #include <dev/usb/usb_device.h>
78 #include <dev/usb/usb_hub.h>
79 #include <dev/usb/usb_util.h>
80 
81 #include <dev/usb/usb_controller.h>
82 #include <dev/usb/usb_bus.h>
83 #endif			/* USB_GLOBAL_INCLUDE_FILE */
84 
85 #include <dev/usb/controller/dwc_otg.h>
86 #include <dev/usb/controller/dwc_otgreg.h>
87 
88 #define	DWC_OTG_BUS2SC(bus) \
89    ((struct dwc_otg_softc *)(((uint8_t *)(bus)) - \
90     ((uint8_t *)&(((struct dwc_otg_softc *)0)->sc_bus))))
91 
92 #define	DWC_OTG_PC2UDEV(pc) \
93    (USB_DMATAG_TO_XROOT((pc)->tag_parent)->udev)
94 
95 #define	DWC_OTG_MSK_GINT_ENABLED	\
96    (GINTMSK_ENUMDONEMSK |		\
97    GINTMSK_USBRSTMSK |			\
98    GINTMSK_USBSUSPMSK |			\
99    GINTMSK_IEPINTMSK |			\
100    GINTMSK_SESSREQINTMSK |		\
101    GINTMSK_RXFLVLMSK |			\
102    GINTMSK_HCHINTMSK |			\
103    GINTMSK_OTGINTMSK |			\
104    GINTMSK_PRTINTMSK)
105 
106 #define	DWC_OTG_MSK_GINT_THREAD_IRQ				\
107    (GINTSTS_USBRST | GINTSTS_ENUMDONE | GINTSTS_PRTINT |	\
108    GINTSTS_WKUPINT | GINTSTS_USBSUSP | GINTMSK_OTGINTMSK |	\
109    GINTSTS_SESSREQINT)
110 
111 static int dwc_otg_use_hsic;
112 
113 static SYSCTL_NODE(_hw_usb, OID_AUTO, dwc_otg, CTLFLAG_RW, 0, "USB DWC OTG");
114 
115 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, use_hsic, CTLFLAG_RDTUN,
116     &dwc_otg_use_hsic, 0, "DWC OTG uses HSIC interface");
117 
118 #ifdef USB_DEBUG
119 static int dwc_otg_debug;
120 
121 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, debug, CTLFLAG_RWTUN,
122     &dwc_otg_debug, 0, "DWC OTG debug level");
123 #endif
124 
125 #define	DWC_OTG_INTR_ENDPT 1
126 
127 /* prototypes */
128 
129 static const struct usb_bus_methods dwc_otg_bus_methods;
130 static const struct usb_pipe_methods dwc_otg_device_non_isoc_methods;
131 static const struct usb_pipe_methods dwc_otg_device_isoc_methods;
132 
133 static dwc_otg_cmd_t dwc_otg_setup_rx;
134 static dwc_otg_cmd_t dwc_otg_data_rx;
135 static dwc_otg_cmd_t dwc_otg_data_tx;
136 static dwc_otg_cmd_t dwc_otg_data_tx_sync;
137 
138 static dwc_otg_cmd_t dwc_otg_host_setup_tx;
139 static dwc_otg_cmd_t dwc_otg_host_data_tx;
140 static dwc_otg_cmd_t dwc_otg_host_data_rx;
141 
142 static void dwc_otg_device_done(struct usb_xfer *, usb_error_t);
143 static void dwc_otg_do_poll(struct usb_bus *);
144 static void dwc_otg_standard_done(struct usb_xfer *);
145 static void dwc_otg_root_intr(struct dwc_otg_softc *);
146 static void dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *);
147 static void dwc_otg_host_channel_disable(struct dwc_otg_softc *, uint8_t);
148 
149 /*
150  * Here is a configuration that the chip supports.
151  */
152 static const struct usb_hw_ep_profile dwc_otg_ep_profile[1] = {
153 
154 	[0] = {
155 		.max_in_frame_size = 64,/* fixed */
156 		.max_out_frame_size = 64,	/* fixed */
157 		.is_simplex = 1,
158 		.support_control = 1,
159 	}
160 };
161 
162 static void
163 dwc_otg_get_hw_ep_profile(struct usb_device *udev,
164     const struct usb_hw_ep_profile **ppf, uint8_t ep_addr)
165 {
166 	struct dwc_otg_softc *sc;
167 
168 	sc = DWC_OTG_BUS2SC(udev->bus);
169 
170 	if (ep_addr < sc->sc_dev_ep_max)
171 		*ppf = &sc->sc_hw_ep_profile[ep_addr].usb;
172 	else
173 		*ppf = NULL;
174 }
175 
176 static int
177 dwc_otg_init_fifo(struct dwc_otg_softc *sc, uint8_t mode)
178 {
179 	struct dwc_otg_profile *pf;
180 	uint32_t fifo_size;
181 	uint32_t fifo_regs;
182 	uint32_t tx_start;
183 	uint8_t x;
184 
185 	fifo_size = sc->sc_fifo_size;
186 
187 	/*
188 	 * NOTE: Reserved fixed size area at end of RAM, which must
189 	 * not be allocated to the FIFOs:
190 	 */
191 	fifo_regs = 4 * 16;
192 
193 	if (fifo_size < fifo_regs) {
194 		DPRINTF("Too little FIFO\n");
195 		return (EINVAL);
196 	}
197 
198 	/* subtract FIFO regs from total once */
199 	fifo_size -= fifo_regs;
200 
201 	/* split equally for IN and OUT */
202 	fifo_size /= 2;
203 
204 	/* align to 4 bytes boundary */
205 	fifo_size &= ~3;
206 
207 	/* set global receive FIFO size */
208 	DWC_OTG_WRITE_4(sc, DOTG_GRXFSIZ, fifo_size / 4);
209 
210 	tx_start = fifo_size;
211 
212 	if (fifo_size < 64) {
213 		DPRINTFN(-1, "Not enough data space for EP0 FIFO.\n");
214 		return (EINVAL);
215 	}
216 
217 	/* disable any leftover host channels */
218 	for (x = 0; x != sc->sc_host_ch_max; x++) {
219 		if (sc->sc_chan_state[x].wait_sof == 0)
220 			continue;
221 		dwc_otg_host_channel_disable(sc, x);
222 	}
223 
224 	if (mode == DWC_MODE_HOST) {
225 
226 		/* reset active endpoints */
227 		sc->sc_active_rx_ep = 0;
228 
229 		/* split equally for periodic and non-periodic */
230 		fifo_size /= 2;
231 
232 		/* align to 4 bytes boundary */
233 		fifo_size &= ~3;
234 
235 		DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
236 		    ((fifo_size / 4) << 16) |
237 		    (tx_start / 4));
238 
239 		tx_start += fifo_size;
240 
241 		for (x = 0; x != sc->sc_host_ch_max; x++) {
242 			/* disable all host interrupts */
243 			DWC_OTG_WRITE_4(sc, DOTG_HCINTMSK(x),
244 			    HCINT_DEFAULT_MASK);
245 		}
246 
247 		DWC_OTG_WRITE_4(sc, DOTG_HPTXFSIZ,
248 		    ((fifo_size / 4) << 16) |
249 		    (tx_start / 4));
250 
251 		/* reset host channel state */
252 		memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
253 
254 		/* reset FIFO TX levels */
255 		sc->sc_tx_cur_p_level = 0;
256 		sc->sc_tx_cur_np_level = 0;
257 
258 		/* store maximum periodic and non-periodic FIFO TX size */
259 		sc->sc_tx_max_size = fifo_size;
260 
261 		/* enable all host channel interrupts */
262 		DWC_OTG_WRITE_4(sc, DOTG_HAINTMSK,
263 		    (1U << sc->sc_host_ch_max) - 1U);
264 	}
265 
266 	if (mode == DWC_MODE_DEVICE) {
267 
268 	    DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
269 		(0x10 << 16) | (tx_start / 4));
270 	    fifo_size -= 0x40;
271 	    tx_start += 0x40;
272 
273 	    /* setup control endpoint profile */
274 	    sc->sc_hw_ep_profile[0].usb = dwc_otg_ep_profile[0];
275 
276 	    /* reset active endpoints */
277 	    sc->sc_active_rx_ep = 1;
278 
279 	    for (x = 1; x != sc->sc_dev_ep_max; x++) {
280 
281 		pf = sc->sc_hw_ep_profile + x;
282 
283 		pf->usb.max_out_frame_size = 1024 * 3;
284 		pf->usb.is_simplex = 0;	/* assume duplex */
285 		pf->usb.support_bulk = 1;
286 		pf->usb.support_interrupt = 1;
287 		pf->usb.support_isochronous = 1;
288 		pf->usb.support_out = 1;
289 
290 		if (x < sc->sc_dev_in_ep_max) {
291 			uint32_t limit;
292 
293 			limit = (x == 1) ? DWC_OTG_MAX_TXN :
294 			    (DWC_OTG_MAX_TXN / 2);
295 
296 			if (fifo_size >= limit) {
297 				DWC_OTG_WRITE_4(sc, DOTG_DIEPTXF(x),
298 				    ((limit / 4) << 16) |
299 				    (tx_start / 4));
300 				tx_start += limit;
301 				fifo_size -= limit;
302 				pf->usb.max_in_frame_size = 0x200;
303 				pf->usb.support_in = 1;
304 				pf->max_buffer = limit;
305 
306 			} else if (fifo_size >= 0x80) {
307 				DWC_OTG_WRITE_4(sc, DOTG_DIEPTXF(x),
308 				    ((0x80 / 4) << 16) | (tx_start / 4));
309 				tx_start += 0x80;
310 				fifo_size -= 0x80;
311 				pf->usb.max_in_frame_size = 0x40;
312 				pf->usb.support_in = 1;
313 
314 			} else {
315 				pf->usb.is_simplex = 1;
316 				DWC_OTG_WRITE_4(sc, DOTG_DIEPTXF(x),
317 				    (0x0 << 16) | (tx_start / 4));
318 			}
319 		} else {
320 			pf->usb.is_simplex = 1;
321 		}
322 
323 		DPRINTF("FIFO%d = IN:%d / OUT:%d\n", x,
324 		    pf->usb.max_in_frame_size,
325 		    pf->usb.max_out_frame_size);
326 	    }
327 	}
328 
329 	/* reset RX FIFO */
330 	DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
331 	    GRSTCTL_RXFFLSH);
332 
333 	if (mode != DWC_MODE_OTG) {
334 		/* reset all TX FIFOs */
335 		DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
336 		    GRSTCTL_TXFIFO(0x10) |
337 		    GRSTCTL_TXFFLSH);
338 	} else {
339 		/* reset active endpoints */
340 		sc->sc_active_rx_ep = 0;
341 
342 		/* reset periodic and non-periodic FIFO TX size */
343 		sc->sc_tx_max_size = fifo_size;
344 
345 		/* reset host channel state */
346 		memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
347 
348 		/* reset FIFO TX levels */
349 		sc->sc_tx_cur_p_level = 0;
350 		sc->sc_tx_cur_np_level = 0;
351 	}
352 	return (0);
353 }
354 
355 static void
356 dwc_otg_update_host_frame_interval(struct dwc_otg_softc *sc)
357 {
358 
359   /*
360    * Disabled until further. Assuming that the register is already
361    * programmed correctly by the boot loader.
362    */
363 #if 0
364 	uint32_t temp;
365 
366 	/* setup HOST frame interval register, based on existing value */
367 	temp = DWC_OTG_READ_4(sc, DOTG_HFIR) & HFIR_FRINT_MASK;
368 	if (temp >= 10000)
369 		temp /= 1000;
370 	else
371 		temp /= 125;
372 
373 	/* figure out nearest X-tal value */
374 	if (temp >= 54)
375 		temp = 60;	/* MHz */
376 	else if (temp >= 39)
377 		temp = 48;	/* MHz */
378 	else
379 		temp = 30;	/* MHz */
380 
381 	if (sc->sc_flags.status_high_speed)
382 		temp *= 125;
383 	else
384 		temp *= 1000;
385 
386 	DPRINTF("HFIR=0x%08x\n", temp);
387 
388 	DWC_OTG_WRITE_4(sc, DOTG_HFIR, temp);
389 #endif
390 }
391 
392 static void
393 dwc_otg_clocks_on(struct dwc_otg_softc *sc)
394 {
395 	if (sc->sc_flags.clocks_off &&
396 	    sc->sc_flags.port_powered) {
397 
398 		DPRINTFN(5, "\n");
399 
400 		/* TODO - platform specific */
401 
402 		sc->sc_flags.clocks_off = 0;
403 	}
404 }
405 
406 static void
407 dwc_otg_clocks_off(struct dwc_otg_softc *sc)
408 {
409 	if (!sc->sc_flags.clocks_off) {
410 
411 		DPRINTFN(5, "\n");
412 
413 		/* TODO - platform specific */
414 
415 		sc->sc_flags.clocks_off = 1;
416 	}
417 }
418 
419 static void
420 dwc_otg_pull_up(struct dwc_otg_softc *sc)
421 {
422 	uint32_t temp;
423 
424 	/* pullup D+, if possible */
425 
426 	if (!sc->sc_flags.d_pulled_up &&
427 	    sc->sc_flags.port_powered) {
428 		sc->sc_flags.d_pulled_up = 1;
429 
430 		temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
431 		temp &= ~DCTL_SFTDISCON;
432 		DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
433 	}
434 }
435 
436 static void
437 dwc_otg_pull_down(struct dwc_otg_softc *sc)
438 {
439 	uint32_t temp;
440 
441 	/* pulldown D+, if possible */
442 
443 	if (sc->sc_flags.d_pulled_up) {
444 		sc->sc_flags.d_pulled_up = 0;
445 
446 		temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
447 		temp |= DCTL_SFTDISCON;
448 		DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
449 	}
450 }
451 
452 static void
453 dwc_otg_enable_sof_irq(struct dwc_otg_softc *sc)
454 {
455 	/* In device mode we don't use the SOF interrupt */
456 	if (sc->sc_flags.status_device_mode != 0 ||
457 	    (sc->sc_irq_mask & GINTMSK_SOFMSK) != 0)
458 		return;
459 	sc->sc_irq_mask |= GINTMSK_SOFMSK;
460 	DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
461 }
462 
463 static void
464 dwc_otg_resume_irq(struct dwc_otg_softc *sc)
465 {
466 	if (sc->sc_flags.status_suspend) {
467 		/* update status bits */
468 		sc->sc_flags.status_suspend = 0;
469 		sc->sc_flags.change_suspend = 1;
470 
471 		if (sc->sc_flags.status_device_mode) {
472 			/*
473 			 * Disable resume interrupt and enable suspend
474 			 * interrupt:
475 			 */
476 			sc->sc_irq_mask &= ~GINTMSK_WKUPINTMSK;
477 			sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
478 			DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
479 		}
480 
481 		/* complete root HUB interrupt endpoint */
482 		dwc_otg_root_intr(sc);
483 	}
484 }
485 
486 static void
487 dwc_otg_suspend_irq(struct dwc_otg_softc *sc)
488 {
489 	if (!sc->sc_flags.status_suspend) {
490 		/* update status bits */
491 		sc->sc_flags.status_suspend = 1;
492 		sc->sc_flags.change_suspend = 1;
493 
494 		if (sc->sc_flags.status_device_mode) {
495 			/*
496 			 * Disable suspend interrupt and enable resume
497 			 * interrupt:
498 			 */
499 			sc->sc_irq_mask &= ~GINTMSK_USBSUSPMSK;
500 			sc->sc_irq_mask |= GINTMSK_WKUPINTMSK;
501 			DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
502 		}
503 
504 		/* complete root HUB interrupt endpoint */
505 		dwc_otg_root_intr(sc);
506 	}
507 }
508 
509 static void
510 dwc_otg_wakeup_peer(struct dwc_otg_softc *sc)
511 {
512 	if (!sc->sc_flags.status_suspend)
513 		return;
514 
515 	DPRINTFN(5, "Remote wakeup\n");
516 
517 	if (sc->sc_flags.status_device_mode) {
518 		uint32_t temp;
519 
520 		/* enable remote wakeup signalling */
521 		temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
522 		temp |= DCTL_RMTWKUPSIG;
523 		DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
524 
525 		/* Wait 8ms for remote wakeup to complete. */
526 		usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 125);
527 
528 		temp &= ~DCTL_RMTWKUPSIG;
529 		DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
530 	} else {
531 		/* enable USB port */
532 		DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
533 
534 		/* wait 10ms */
535 		usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
536 
537 		/* resume port */
538 		sc->sc_hprt_val |= HPRT_PRTRES;
539 		DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
540 
541 		/* Wait 100ms for resume signalling to complete. */
542 		usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 10);
543 
544 		/* clear suspend and resume */
545 		sc->sc_hprt_val &= ~(HPRT_PRTSUSP | HPRT_PRTRES);
546 		DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
547 
548 		/* Wait 4ms */
549 		usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
550 	}
551 
552 	/* need to fake resume IRQ */
553 	dwc_otg_resume_irq(sc);
554 }
555 
556 static void
557 dwc_otg_set_address(struct dwc_otg_softc *sc, uint8_t addr)
558 {
559 	uint32_t temp;
560 
561 	DPRINTFN(5, "addr=%d\n", addr);
562 
563 	temp = DWC_OTG_READ_4(sc, DOTG_DCFG);
564 	temp &= ~DCFG_DEVADDR_SET(0x7F);
565 	temp |= DCFG_DEVADDR_SET(addr);
566 	DWC_OTG_WRITE_4(sc, DOTG_DCFG, temp);
567 }
568 
569 static void
570 dwc_otg_common_rx_ack(struct dwc_otg_softc *sc)
571 {
572 	DPRINTFN(5, "RX status clear\n");
573 
574 	/* enable RX FIFO level interrupt */
575 	sc->sc_irq_mask |= GINTMSK_RXFLVLMSK;
576 	DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
577 
578 	/* clear cached status */
579 	sc->sc_last_rx_status = 0;
580 }
581 
582 static void
583 dwc_otg_clear_hcint(struct dwc_otg_softc *sc, uint8_t x)
584 {
585 	uint32_t hcint;
586 
587 	/* clear all pending interrupts */
588 	hcint = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
589 	DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), hcint);
590 
591 	/* clear buffered interrupts */
592 	sc->sc_chan_state[x].hcint = 0;
593 }
594 
595 static uint8_t
596 dwc_otg_host_channel_alloc(struct dwc_otg_softc *sc, struct dwc_otg_td *td, uint8_t is_out)
597 {
598 	uint32_t tx_p_size;
599 	uint32_t tx_np_size;
600 	uint8_t x;
601 
602 	if (td->channel < DWC_OTG_MAX_CHANNELS)
603 		return (0);		/* already allocated */
604 
605 	/* check if device is suspended */
606 	if (DWC_OTG_PC2UDEV(td->pc)->flags.self_suspended != 0)
607 		return (1);		/* busy - cannot transfer data */
608 
609 	/* compute needed TX FIFO size */
610 	if (is_out != 0) {
611 		if (td->ep_type == UE_ISOCHRONOUS) {
612 			tx_p_size = td->max_packet_size;
613 			tx_np_size = 0;
614 			if (td->hcsplt != 0 && tx_p_size > HCSPLT_XACTLEN_BURST)
615 				tx_p_size = HCSPLT_XACTLEN_BURST;
616 			if ((sc->sc_tx_cur_p_level + tx_p_size) > sc->sc_tx_max_size) {
617 				DPRINTF("Too little FIFO space\n");
618 				return (1);	/* too little FIFO */
619 			}
620 		} else {
621 			tx_p_size = 0;
622 			tx_np_size = td->max_packet_size;
623 			if (td->hcsplt != 0 && tx_np_size > HCSPLT_XACTLEN_BURST)
624 				tx_np_size = HCSPLT_XACTLEN_BURST;
625 			if ((sc->sc_tx_cur_np_level + tx_np_size) > sc->sc_tx_max_size) {
626 				DPRINTF("Too little FIFO space\n");
627 				return (1);	/* too little FIFO */
628 			}
629 		}
630 	} else {
631 		/* not a TX transaction */
632 		tx_p_size = 0;
633 		tx_np_size = 0;
634 	}
635 
636 	for (x = 0; x != sc->sc_host_ch_max; x++) {
637 		if (sc->sc_chan_state[x].allocated != 0)
638 			continue;
639 		/* check if channel is still enabled */
640 		if (sc->sc_chan_state[x].wait_sof != 0)
641 			continue;
642 
643 		sc->sc_chan_state[x].allocated = 1;
644 		sc->sc_chan_state[x].tx_p_size = tx_p_size;
645 		sc->sc_chan_state[x].tx_np_size = tx_np_size;
646 
647 		/* keep track of used TX FIFO, if any */
648 		sc->sc_tx_cur_p_level += tx_p_size;
649 		sc->sc_tx_cur_np_level += tx_np_size;
650 
651 		/* clear interrupts */
652 		dwc_otg_clear_hcint(sc, x);
653 
654 		DPRINTF("CH=%d HCCHAR=0x%08x "
655 		    "HCSPLT=0x%08x\n", x, td->hcchar, td->hcsplt);
656 
657 		/* set active channel */
658 		sc->sc_active_rx_ep |= (1 << x);
659 
660 		/* set channel */
661 		td->channel = x;
662 
663 		return (0);	/* allocated */
664 	}
665 	/* wait a bit */
666 	dwc_otg_enable_sof_irq(sc);
667 	return (1);	/* busy */
668 }
669 
670 static void
671 dwc_otg_host_channel_free(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
672 {
673 	uint8_t x;
674 
675 	if (td->channel >= DWC_OTG_MAX_CHANNELS)
676 		return;		/* already freed */
677 
678 	/* free channel */
679 	x = td->channel;
680 	td->channel = DWC_OTG_MAX_CHANNELS;
681 
682 	DPRINTF("CH=%d\n", x);
683 
684 	/*
685 	 * We need to let programmed host channels run till complete
686 	 * else the host channel will stop functioning. Assume that
687 	 * after a fixed given amount of time the host channel is no
688 	 * longer doing any USB traffic:
689 	 */
690 	if (td->ep_type == UE_ISOCHRONOUS) {
691 		/* double buffered */
692 		sc->sc_chan_state[x].wait_sof = DWC_OTG_SLOT_IDLE_MAX;
693 	} else {
694 		/* single buffered */
695 		sc->sc_chan_state[x].wait_sof = DWC_OTG_SLOT_IDLE_MIN;
696 	}
697 
698 	sc->sc_chan_state[x].allocated = 0;
699 
700 	/* ack any pending messages */
701 	if (sc->sc_last_rx_status != 0 &&
702 	    GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) == x) {
703 		dwc_otg_common_rx_ack(sc);
704 	}
705 
706 	/* clear active channel */
707 	sc->sc_active_rx_ep &= ~(1 << x);
708 }
709 
710 static void
711 dwc_otg_host_dump_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
712 {
713 	/* dump any pending messages */
714 	if (sc->sc_last_rx_status != 0) {
715 		if (td->channel < DWC_OTG_MAX_CHANNELS &&
716 		    td->channel == GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status)) {
717 			dwc_otg_common_rx_ack(sc);
718 		}
719 	}
720 }
721 
722 static uint8_t
723 dwc_otg_host_setup_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
724 {
725 	struct usb_device_request req __aligned(4);
726 	uint32_t hcint;
727 	uint32_t hcchar;
728 	uint8_t delta;
729 
730 	dwc_otg_host_dump_rx(sc, td);
731 
732 	if (td->channel < DWC_OTG_MAX_CHANNELS) {
733 		hcint = sc->sc_chan_state[td->channel].hcint;
734 
735 		DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
736 		    td->channel, td->state, hcint,
737 		    DWC_OTG_READ_4(sc, DOTG_HCCHAR(td->channel)),
738 		    DWC_OTG_READ_4(sc, DOTG_HCTSIZ(td->channel)));
739 	} else {
740 		hcint = 0;
741 		goto check_state;
742 	}
743 
744 	if (hcint & (HCINT_RETRY |
745 	    HCINT_ACK | HCINT_NYET)) {
746 		/* give success bits priority over failure bits */
747 	} else if (hcint & HCINT_STALL) {
748 		DPRINTF("CH=%d STALL\n", td->channel);
749 		td->error_stall = 1;
750 		td->error_any = 1;
751 		goto complete;
752 	} else if (hcint & HCINT_ERRORS) {
753 		DPRINTF("CH=%d ERROR\n", td->channel);
754 		td->errcnt++;
755 		if (td->hcsplt != 0 || td->errcnt >= 3) {
756 			td->error_any = 1;
757 			goto complete;
758 		}
759 	}
760 
761 	if (hcint & (HCINT_ERRORS | HCINT_RETRY |
762 	    HCINT_ACK | HCINT_NYET)) {
763 		if (!(hcint & HCINT_ERRORS))
764 			td->errcnt = 0;
765 	}
766 
767 check_state:
768 	switch (td->state) {
769 	case DWC_CHAN_ST_START:
770 		goto send_pkt;
771 
772 	case DWC_CHAN_ST_WAIT_ANE:
773 		if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
774 			td->did_nak++;
775 			td->tt_scheduled = 0;
776 			goto send_pkt;
777 		} else if (hcint & (HCINT_ACK | HCINT_NYET)) {
778 			td->offset += td->tx_bytes;
779 			td->remainder -= td->tx_bytes;
780 			td->toggle = 1;
781 			td->tt_scheduled = 0;
782 			goto complete;
783 		}
784 		break;
785 
786 	case DWC_CHAN_ST_WAIT_S_ANE:
787 		if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
788 			td->did_nak++;
789 			td->tt_scheduled = 0;
790 			goto send_pkt;
791 		} else if (hcint & (HCINT_ACK | HCINT_NYET)) {
792 			goto send_cpkt;
793 		}
794 		break;
795 
796 	case DWC_CHAN_ST_WAIT_C_ANE:
797 		if (hcint & HCINT_NYET) {
798 			goto send_cpkt;
799 		} else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
800 			td->did_nak++;
801 			td->tt_scheduled = 0;
802 			goto send_pkt;
803 		} else if (hcint & HCINT_ACK) {
804 			td->offset += td->tx_bytes;
805 			td->remainder -= td->tx_bytes;
806 			td->toggle = 1;
807 			goto complete;
808 		}
809 		break;
810 
811 	case DWC_CHAN_ST_WAIT_C_PKT:
812 		goto send_cpkt;
813 
814 	default:
815 		break;
816 	}
817 	goto busy;
818 
819 send_pkt:
820 	/* free existing channel, if any */
821 	dwc_otg_host_channel_free(sc, td);
822 
823 	if (sizeof(req) != td->remainder) {
824 		td->error_any = 1;
825 		goto complete;
826 	}
827 
828 	if (td->hcsplt != 0) {
829 		delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
830 		if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
831 			td->state = DWC_CHAN_ST_START;
832 			goto busy;
833 		}
834 		delta = sc->sc_last_frame_num - td->tt_start_slot;
835 		if (delta > 5) {
836 			/* missed it */
837 			td->tt_scheduled = 0;
838 			td->state = DWC_CHAN_ST_START;
839 			goto busy;
840 		}
841 	}
842 
843 	/* allocate a new channel */
844 	if (dwc_otg_host_channel_alloc(sc, td, 1)) {
845 		td->state = DWC_CHAN_ST_START;
846 		goto busy;
847 	}
848 
849 	if (td->hcsplt != 0) {
850 		td->hcsplt &= ~HCSPLT_COMPSPLT;
851 		td->state = DWC_CHAN_ST_WAIT_S_ANE;
852 	} else {
853 		td->state = DWC_CHAN_ST_WAIT_ANE;
854 	}
855 
856 	usbd_copy_out(td->pc, 0, &req, sizeof(req));
857 
858 	DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel),
859 	    (sizeof(req) << HCTSIZ_XFERSIZE_SHIFT) |
860 	    (1 << HCTSIZ_PKTCNT_SHIFT) |
861 	    (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
862 
863 	DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel), td->hcsplt);
864 
865 	hcchar = td->hcchar;
866 	hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
867 	hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
868 
869 	/* must enable channel before writing data to FIFO */
870 	DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel), hcchar);
871 
872 	/* transfer data into FIFO */
873 	bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
874 	    DOTG_DFIFO(td->channel), (uint32_t *)&req, sizeof(req) / 4);
875 
876 	/* wait until next slot before trying complete split */
877 	td->tt_complete_slot = sc->sc_last_frame_num + 1;
878 
879 	/* store number of bytes transmitted */
880 	td->tx_bytes = sizeof(req);
881 	goto busy;
882 
883 send_cpkt:
884 	/* free existing channel, if any */
885 	dwc_otg_host_channel_free(sc, td);
886 
887 	delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
888 	if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
889 		td->state = DWC_CHAN_ST_WAIT_C_PKT;
890 		goto busy;
891 	}
892 	delta = sc->sc_last_frame_num - td->tt_start_slot;
893 	if (delta > DWC_OTG_TT_SLOT_MAX) {
894 		/* we missed the service interval */
895 		if (td->ep_type != UE_ISOCHRONOUS)
896 			td->error_any = 1;
897 		goto complete;
898 	}
899 	/* allocate a new channel */
900 	if (dwc_otg_host_channel_alloc(sc, td, 0)) {
901 		td->state = DWC_CHAN_ST_WAIT_C_PKT;
902 		goto busy;
903 	}
904 
905 	/* wait until next slot before trying complete split */
906 	td->tt_complete_slot = sc->sc_last_frame_num + 1;
907 
908 	td->hcsplt |= HCSPLT_COMPSPLT;
909 	td->state = DWC_CHAN_ST_WAIT_C_ANE;
910 
911 	DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel),
912 	    (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
913 
914 	DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel), td->hcsplt);
915 
916 	hcchar = td->hcchar;
917 	hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
918 	hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
919 
920 	/* must enable channel before writing data to FIFO */
921 	DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel), hcchar);
922 
923 busy:
924 	return (1);	/* busy */
925 
926 complete:
927 	dwc_otg_host_channel_free(sc, td);
928 	return (0);	/* complete */
929 }
930 
931 static uint8_t
932 dwc_otg_setup_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
933 {
934 	struct usb_device_request req __aligned(4);
935 	uint32_t temp;
936 	uint16_t count;
937 
938 	/* check endpoint status */
939 
940 	if (sc->sc_last_rx_status == 0)
941 		goto not_complete;
942 
943 	if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != 0)
944 		goto not_complete;
945 
946 	if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
947 	    GRXSTSRD_DPID_DATA0) {
948 		/* release FIFO */
949 		dwc_otg_common_rx_ack(sc);
950 		goto not_complete;
951 	}
952 
953 	if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
954 	    GRXSTSRD_STP_DATA) {
955 		/* release FIFO */
956 		dwc_otg_common_rx_ack(sc);
957 		goto not_complete;
958 	}
959 
960 	DPRINTFN(5, "GRXSTSR=0x%08x\n", sc->sc_last_rx_status);
961 
962 	/* clear did stall */
963 	td->did_stall = 0;
964 
965 	/* get the packet byte count */
966 	count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
967 
968 	/* verify data length */
969 	if (count != td->remainder) {
970 		DPRINTFN(0, "Invalid SETUP packet "
971 		    "length, %d bytes\n", count);
972 		/* release FIFO */
973 		dwc_otg_common_rx_ack(sc);
974 		goto not_complete;
975 	}
976 	if (count != sizeof(req)) {
977 		DPRINTFN(0, "Unsupported SETUP packet "
978 		    "length, %d bytes\n", count);
979 		/* release FIFO */
980 		dwc_otg_common_rx_ack(sc);
981 		goto not_complete;
982 	}
983 
984 	/* copy in control request */
985 	memcpy(&req, sc->sc_rx_bounce_buffer, sizeof(req));
986 
987 	/* copy data into real buffer */
988 	usbd_copy_in(td->pc, 0, &req, sizeof(req));
989 
990 	td->offset = sizeof(req);
991 	td->remainder = 0;
992 
993 	/* sneak peek the set address */
994 	if ((req.bmRequestType == UT_WRITE_DEVICE) &&
995 	    (req.bRequest == UR_SET_ADDRESS)) {
996 		/* must write address before ZLP */
997 		dwc_otg_set_address(sc, req.wValue[0] & 0x7F);
998 	}
999 
1000 	/* don't send any data by default */
1001 	DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(0),
1002 	    DXEPTSIZ_SET_NPKT(0) |
1003 	    DXEPTSIZ_SET_NBYTES(0));
1004 
1005 	temp = sc->sc_in_ctl[0];
1006 
1007 	/* enable IN endpoint */
1008 	DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(0),
1009 	    temp | DIEPCTL_EPENA);
1010 	DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(0),
1011 	    temp | DIEPCTL_SNAK);
1012 
1013 	/* reset IN endpoint buffer */
1014 	DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
1015 	    GRSTCTL_TXFIFO(0) |
1016 	    GRSTCTL_TXFFLSH);
1017 
1018 	/* acknowledge RX status */
1019 	dwc_otg_common_rx_ack(sc);
1020 	return (0);			/* complete */
1021 
1022 not_complete:
1023 	/* abort any ongoing transfer, before enabling again */
1024 
1025 	temp = sc->sc_out_ctl[0];
1026 
1027 	temp |= DOEPCTL_EPENA |
1028 	    DOEPCTL_SNAK;
1029 
1030 	/* enable OUT endpoint */
1031 	DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(0), temp);
1032 
1033 	if (!td->did_stall) {
1034 		td->did_stall = 1;
1035 
1036 		DPRINTFN(5, "stalling IN and OUT direction\n");
1037 
1038 		/* set stall after enabling endpoint */
1039 		DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(0),
1040 		    temp | DOEPCTL_STALL);
1041 
1042 		temp = sc->sc_in_ctl[0];
1043 
1044 		/* set stall assuming endpoint is enabled */
1045 		DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(0),
1046 		    temp | DIEPCTL_STALL);
1047 	}
1048 
1049 	/* setup number of buffers to receive */
1050 	DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0),
1051 	    DXEPTSIZ_SET_MULTI(3) |
1052 	    DXEPTSIZ_SET_NPKT(1) |
1053 	    DXEPTSIZ_SET_NBYTES(sizeof(req)));
1054 
1055 	return (1);			/* not complete */
1056 }
1057 
1058 static uint8_t
1059 dwc_otg_host_rate_check_interrupt(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1060 {
1061 	uint8_t delta;
1062 
1063 	delta = sc->sc_tmr_val - td->tmr_val;
1064 	if (delta >= 128)
1065 		return (1);	/* busy */
1066 
1067 	td->tmr_val = sc->sc_tmr_val + td->tmr_res;
1068 
1069 	/* set toggle, if any */
1070 	if (td->set_toggle) {
1071 		td->set_toggle = 0;
1072 		td->toggle = 1;
1073 	}
1074 	return (0);
1075 }
1076 
1077 static uint8_t
1078 dwc_otg_host_rate_check(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1079 {
1080 	if (td->ep_type == UE_ISOCHRONOUS) {
1081 		/* non TT isochronous traffic */
1082 		if ((td->tmr_val != 0) ||
1083 		    (sc->sc_last_frame_num & (td->tmr_res - 1))) {
1084 			goto busy;
1085 		}
1086 		td->tmr_val = 1;	/* executed */
1087 		td->toggle = 0;
1088 
1089 	} else if (td->ep_type == UE_INTERRUPT) {
1090 		if (!td->tt_scheduled)
1091 			goto busy;
1092 		td->tt_scheduled = 0;
1093 	} else if (td->did_nak >= DWC_OTG_NAK_MAX) {
1094 		goto busy;
1095 	} else if (td->set_toggle) {
1096 		td->set_toggle = 0;
1097 		td->toggle = 1;
1098 	}
1099 	return (0);
1100 busy:
1101 	return (1);
1102 }
1103 
1104 static uint8_t
1105 dwc_otg_host_data_rx_sub(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1106 {
1107 	uint32_t count;
1108 	uint8_t channel;
1109 
1110 	/* check endpoint status */
1111 	if (sc->sc_last_rx_status == 0)
1112 		goto busy;
1113 
1114 	channel = td->channel;
1115 	if (channel >= DWC_OTG_MAX_CHANNELS)
1116 		goto busy;
1117 
1118 	if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != channel)
1119 		goto busy;
1120 
1121 	switch (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) {
1122 	case GRXSTSRH_IN_DATA:
1123 
1124 		DPRINTF("DATA ST=%d STATUS=0x%08x\n",
1125 		    (int)td->state, (int)sc->sc_last_rx_status);
1126 
1127 		if (sc->sc_chan_state[channel].hcint & HCINT_SOFTWARE_ONLY) {
1128 			/*
1129 			 * When using SPLIT transactions on interrupt
1130 			 * endpoints, sometimes data occurs twice.
1131 			 */
1132 			DPRINTF("Data already received\n");
1133 			break;
1134 		}
1135 
1136 		/* get the packet byte count */
1137 		count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1138 
1139 		/* check for isochronous transfer or high-speed bandwidth endpoint */
1140 		if (td->ep_type == UE_ISOCHRONOUS || td->max_packet_count > 1) {
1141 			if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) != GRXSTSRD_DPID_DATA0) {
1142 				td->tt_xactpos = HCSPLT_XACTPOS_MIDDLE;
1143 			} else {
1144 				td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
1145 
1146 				/* verify the packet byte count */
1147 				if (count < td->max_packet_size) {
1148 					/* we have a short packet */
1149 					td->short_pkt = 1;
1150 					td->got_short = 1;
1151 				}
1152 			}
1153 			td->toggle = 0;
1154 		} else {
1155 			/* verify the packet byte count */
1156 			if (count != td->max_packet_size) {
1157 				if (count < td->max_packet_size) {
1158 					/* we have a short packet */
1159 					td->short_pkt = 1;
1160 					td->got_short = 1;
1161 				} else {
1162 					/* invalid USB packet */
1163 					td->error_any = 1;
1164 
1165 					/* release FIFO */
1166 					dwc_otg_common_rx_ack(sc);
1167 					goto complete;
1168 				}
1169 			}
1170 			td->toggle ^= 1;
1171 			td->tt_scheduled = 0;
1172 		}
1173 
1174 		/* verify the packet byte count */
1175 		if (count > td->remainder) {
1176 			/* invalid USB packet */
1177 			td->error_any = 1;
1178 
1179 			/* release FIFO */
1180 			dwc_otg_common_rx_ack(sc);
1181 			goto complete;
1182 		}
1183 
1184 		usbd_copy_in(td->pc, td->offset,
1185 		    sc->sc_rx_bounce_buffer, count);
1186 
1187 		td->remainder -= count;
1188 		td->offset += count;
1189 		sc->sc_chan_state[channel].hcint |= HCINT_SOFTWARE_ONLY;
1190 		break;
1191 	default:
1192 		break;
1193 	}
1194 	/* release FIFO */
1195 	dwc_otg_common_rx_ack(sc);
1196 busy:
1197 	return (0);
1198 complete:
1199 	return (1);
1200 }
1201 
1202 static uint8_t
1203 dwc_otg_host_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1204 {
1205 	uint32_t hcint;
1206 	uint32_t hcchar;
1207 	uint8_t delta;
1208 	uint8_t channel;
1209 
1210 	channel = td->channel;
1211 
1212 	if (channel < DWC_OTG_MAX_CHANNELS) {
1213 		hcint = sc->sc_chan_state[channel].hcint;
1214 
1215 		DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1216 		    channel, td->state, hcint,
1217 		    DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1218 		    DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1219 
1220 		/* check interrupt bits */
1221 		if (hcint & (HCINT_RETRY |
1222 		    HCINT_ACK | HCINT_NYET)) {
1223 			/* give success bits priority over failure bits */
1224 		} else if (hcint & HCINT_STALL) {
1225 			DPRINTF("CH=%d STALL\n", channel);
1226 			td->error_stall = 1;
1227 			td->error_any = 1;
1228 			goto complete;
1229 		} else if (hcint & HCINT_ERRORS) {
1230 			DPRINTF("CH=%d ERROR\n", channel);
1231 			td->errcnt++;
1232 			if (td->hcsplt != 0 || td->errcnt >= 3) {
1233 				if (td->ep_type != UE_ISOCHRONOUS) {
1234 					td->error_any = 1;
1235 					goto complete;
1236 				}
1237 			}
1238 		}
1239 
1240 		/* check channels for data, if any */
1241 		if (dwc_otg_host_data_rx_sub(sc, td))
1242 			goto complete;
1243 
1244 		/* refresh interrupt status */
1245 		hcint = sc->sc_chan_state[channel].hcint;
1246 
1247 		if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1248 		    HCINT_ACK | HCINT_NYET)) {
1249 			if (!(hcint & HCINT_ERRORS))
1250 				td->errcnt = 0;
1251 		}
1252 	} else {
1253 		hcint = 0;
1254 	}
1255 
1256 	switch (td->state) {
1257 	case DWC_CHAN_ST_START:
1258 		if (td->hcsplt != 0)
1259 			goto receive_spkt;
1260 		else
1261 			goto receive_pkt;
1262 
1263 	case DWC_CHAN_ST_WAIT_ANE:
1264 		if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1265 			if (td->ep_type == UE_INTERRUPT) {
1266 				/*
1267 				 * The USB specification does not
1268 				 * mandate a particular data toggle
1269 				 * value for USB INTERRUPT
1270 				 * transfers. Switch the data toggle
1271 				 * value to receive the packet
1272 				 * correctly:
1273 				 */
1274 				if (hcint & HCINT_DATATGLERR) {
1275 					DPRINTF("Retrying packet due to "
1276 					    "data toggle error\n");
1277 					td->toggle ^= 1;
1278 					goto receive_pkt;
1279 				}
1280 			}
1281 			td->did_nak++;
1282 			td->tt_scheduled = 0;
1283 			if (td->hcsplt != 0)
1284 				goto receive_spkt;
1285 			else
1286 				goto receive_pkt;
1287 		} else if (hcint & HCINT_NYET) {
1288 			if (td->hcsplt != 0) {
1289 				/* try again */
1290 				goto receive_pkt;
1291 			} else {
1292 				/* not a valid token for IN endpoints */
1293 				td->error_any = 1;
1294 				goto complete;
1295 			}
1296 		} else if (hcint & HCINT_ACK) {
1297 			/* wait for data - ACK arrived first */
1298 			if (!(hcint & HCINT_SOFTWARE_ONLY))
1299 				goto busy;
1300 
1301 			if (td->ep_type == UE_ISOCHRONOUS) {
1302 				/* check if we are complete */
1303 				if ((td->remainder == 0) ||
1304 				    (td->tt_xactpos == HCSPLT_XACTPOS_BEGIN)) {
1305 					goto complete;
1306 				}
1307 				/* get another packet */
1308 				goto receive_pkt;
1309 			} else {
1310 				/* check if we are complete */
1311 				if ((td->remainder == 0) || (td->got_short != 0)) {
1312 					if (td->short_pkt)
1313 						goto complete;
1314 
1315 					/*
1316 					 * Else need to receive a zero length
1317 					 * packet.
1318 					 */
1319 				}
1320 				td->tt_scheduled = 0;
1321 				td->did_nak = 0;
1322 				if (td->hcsplt != 0)
1323 					goto receive_spkt;
1324 				else
1325 					goto receive_pkt;
1326 			}
1327 		}
1328 		break;
1329 
1330 	case DWC_CHAN_ST_WAIT_S_ANE:
1331 		/*
1332 		 * NOTE: The DWC OTG hardware provides a fake ACK in
1333 		 * case of interrupt and isochronous transfers:
1334 		 */
1335 		if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1336 			td->did_nak++;
1337 			td->tt_scheduled = 0;
1338 			goto receive_spkt;
1339 		} else if (hcint & HCINT_NYET) {
1340 			td->tt_scheduled = 0;
1341 			goto receive_spkt;
1342 		} else if (hcint & HCINT_ACK) {
1343 			td->did_nak = 0;
1344 			goto receive_pkt;
1345 		}
1346 		break;
1347 
1348 	case DWC_CHAN_ST_WAIT_C_PKT:
1349 		goto receive_pkt;
1350 
1351 	default:
1352 		break;
1353 	}
1354 	goto busy;
1355 
1356 receive_pkt:
1357 	/* free existing channel, if any */
1358 	dwc_otg_host_channel_free(sc, td);
1359 
1360   	if (td->hcsplt != 0) {
1361 		delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1362 		if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1363 			td->state = DWC_CHAN_ST_WAIT_C_PKT;
1364 			goto busy;
1365 		}
1366 		delta = sc->sc_last_frame_num - td->tt_start_slot;
1367 		if (delta > DWC_OTG_TT_SLOT_MAX) {
1368 			if (td->ep_type != UE_ISOCHRONOUS) {
1369 				/* we missed the service interval */
1370 				td->error_any = 1;
1371 			}
1372 			goto complete;
1373 		}
1374 		/* complete split */
1375 		td->hcsplt |= HCSPLT_COMPSPLT;
1376 	} else if (td->tt_xactpos == HCSPLT_XACTPOS_BEGIN &&
1377 	    dwc_otg_host_rate_check(sc, td)) {
1378 		td->state = DWC_CHAN_ST_WAIT_C_PKT;
1379 		goto busy;
1380 	}
1381 
1382 	/* allocate a new channel */
1383 	if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1384 		td->state = DWC_CHAN_ST_WAIT_C_PKT;
1385 		goto busy;
1386 	}
1387 
1388 	channel = td->channel;
1389 
1390 	/* set toggle, if any */
1391 	if (td->set_toggle) {
1392 		td->set_toggle = 0;
1393 		td->toggle = 1;
1394 	}
1395 
1396 	td->state = DWC_CHAN_ST_WAIT_ANE;
1397 
1398 	/* receive one packet */
1399 	DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1400 	    (td->max_packet_size << HCTSIZ_XFERSIZE_SHIFT) |
1401 	    (1 << HCTSIZ_PKTCNT_SHIFT) |
1402 	    (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1403 	    (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1404 
1405 	DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1406 
1407 	hcchar = td->hcchar;
1408 	hcchar |= HCCHAR_EPDIR_IN;
1409 
1410 	/* receive complete split ASAP */
1411 	if ((sc->sc_last_frame_num & 1) != 0)
1412 		hcchar |= HCCHAR_ODDFRM;
1413 	else
1414 		hcchar &= ~HCCHAR_ODDFRM;
1415 
1416 	/* must enable channel before data can be received */
1417 	DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1418 
1419 	/* wait until next slot before trying complete split */
1420 	td->tt_complete_slot = sc->sc_last_frame_num + 1;
1421 
1422 	goto busy;
1423 
1424 receive_spkt:
1425 	/* free existing channel(s), if any */
1426 	dwc_otg_host_channel_free(sc, td);
1427 
1428 	delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1429 	if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1430 		td->state = DWC_CHAN_ST_START;
1431 		goto busy;
1432 	}
1433 	delta = sc->sc_last_frame_num - td->tt_start_slot;
1434 	if (delta > 5) {
1435 		/* missed it */
1436 		td->tt_scheduled = 0;
1437 		td->state = DWC_CHAN_ST_START;
1438 		goto busy;
1439 	}
1440 
1441 	/* allocate a new channel */
1442 	if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1443 		td->state = DWC_CHAN_ST_START;
1444 		goto busy;
1445 	}
1446 
1447 	channel = td->channel;
1448 
1449 	td->hcsplt &= ~HCSPLT_COMPSPLT;
1450 	td->state = DWC_CHAN_ST_WAIT_S_ANE;
1451 
1452 	/* receive one packet */
1453 	DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1454 	    (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
1455 
1456 	DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1457 
1458 	/* send after next SOF event */
1459 	if ((sc->sc_last_frame_num & 1) == 0)
1460 		td->hcchar |= HCCHAR_ODDFRM;
1461 	else
1462 		td->hcchar &= ~HCCHAR_ODDFRM;
1463 
1464 	hcchar = td->hcchar;
1465 	hcchar |= HCCHAR_EPDIR_IN;
1466 
1467 	/* wait until next slot before trying complete split */
1468 	td->tt_complete_slot = sc->sc_last_frame_num + 1;
1469 
1470 	/* must enable channel before data can be received */
1471 	DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1472 busy:
1473 	return (1);	/* busy */
1474 
1475 complete:
1476 	dwc_otg_host_channel_free(sc, td);
1477 	return (0);	/* complete */
1478 }
1479 
1480 static uint8_t
1481 dwc_otg_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1482 {
1483 	uint32_t temp;
1484 	uint16_t count;
1485 	uint8_t got_short;
1486 
1487 	got_short = 0;
1488 
1489 	/* check endpoint status */
1490 	if (sc->sc_last_rx_status == 0)
1491 		goto not_complete;
1492 
1493 	if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != td->ep_no)
1494 		goto not_complete;
1495 
1496 	/* check for SETUP packet */
1497 	if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1498 	    GRXSTSRD_STP_DATA) {
1499 		if (td->remainder == 0) {
1500 			/*
1501 			 * We are actually complete and have
1502 			 * received the next SETUP
1503 			 */
1504 			DPRINTFN(5, "faking complete\n");
1505 			return (0);	/* complete */
1506 		}
1507 		/*
1508 		 * USB Host Aborted the transfer.
1509 		 */
1510 		td->error_any = 1;
1511 		return (0);		/* complete */
1512 	}
1513 
1514 	if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1515 	    GRXSTSRD_OUT_DATA) {
1516 		/* release FIFO */
1517 		dwc_otg_common_rx_ack(sc);
1518 		goto not_complete;
1519 	}
1520 
1521 	/* get the packet byte count */
1522 	count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1523 
1524 	/* verify the packet byte count */
1525 	if (count != td->max_packet_size) {
1526 		if (count < td->max_packet_size) {
1527 			/* we have a short packet */
1528 			td->short_pkt = 1;
1529 			got_short = 1;
1530 		} else {
1531 			/* invalid USB packet */
1532 			td->error_any = 1;
1533 
1534 			/* release FIFO */
1535 			dwc_otg_common_rx_ack(sc);
1536 			return (0);	/* we are complete */
1537 		}
1538 	}
1539 	/* verify the packet byte count */
1540 	if (count > td->remainder) {
1541 		/* invalid USB packet */
1542 		td->error_any = 1;
1543 
1544 		/* release FIFO */
1545 		dwc_otg_common_rx_ack(sc);
1546 		return (0);		/* we are complete */
1547 	}
1548 
1549 	usbd_copy_in(td->pc, td->offset, sc->sc_rx_bounce_buffer, count);
1550 	td->remainder -= count;
1551 	td->offset += count;
1552 
1553 	/* release FIFO */
1554 	dwc_otg_common_rx_ack(sc);
1555 
1556 	/* check if we are complete */
1557 	if ((td->remainder == 0) || got_short) {
1558 		if (td->short_pkt) {
1559 			/* we are complete */
1560 			return (0);
1561 		}
1562 		/* else need to receive a zero length packet */
1563 	}
1564 
1565 not_complete:
1566 
1567 	temp = sc->sc_out_ctl[td->ep_no];
1568 
1569 	temp |= DOEPCTL_EPENA | DOEPCTL_CNAK;
1570 
1571 	DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(td->ep_no), temp);
1572 
1573 	/* enable SETUP and transfer complete interrupt */
1574 	if (td->ep_no == 0) {
1575 		DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0),
1576 		    DXEPTSIZ_SET_NPKT(1) |
1577 		    DXEPTSIZ_SET_NBYTES(td->max_packet_size));
1578 	} else {
1579 		/* allow reception of multiple packets */
1580 		DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(td->ep_no),
1581 		    DXEPTSIZ_SET_MULTI(1) |
1582 		    DXEPTSIZ_SET_NPKT(4) |
1583 		    DXEPTSIZ_SET_NBYTES(4 *
1584 		    ((td->max_packet_size + 3) & ~3)));
1585 	}
1586 	return (1);			/* not complete */
1587 }
1588 
1589 static uint8_t
1590 dwc_otg_host_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1591 {
1592 	uint32_t count;
1593 	uint32_t hcint;
1594 	uint32_t hcchar;
1595 	uint8_t delta;
1596 	uint8_t channel;
1597 
1598 	dwc_otg_host_dump_rx(sc, td);
1599 
1600 	channel = td->channel;
1601 
1602 	if (channel < DWC_OTG_MAX_CHANNELS) {
1603 		hcint = sc->sc_chan_state[channel].hcint;
1604 
1605 		DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1606 		    channel, td->state, hcint,
1607 		    DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1608 		    DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1609 
1610 		if (hcint & (HCINT_RETRY |
1611 		    HCINT_ACK | HCINT_NYET)) {
1612 			/* give success bits priority over failure bits */
1613 		} else if (hcint & HCINT_STALL) {
1614 			DPRINTF("CH=%d STALL\n", channel);
1615 			td->error_stall = 1;
1616 			td->error_any = 1;
1617 			goto complete;
1618 		} else if (hcint & HCINT_ERRORS) {
1619 			DPRINTF("CH=%d ERROR\n", channel);
1620 			td->errcnt++;
1621 			if (td->hcsplt != 0 || td->errcnt >= 3) {
1622 				td->error_any = 1;
1623 				goto complete;
1624 			}
1625 		}
1626 
1627 		if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1628 		    HCINT_ACK | HCINT_NYET)) {
1629 
1630 			if (!(hcint & HCINT_ERRORS))
1631 				td->errcnt = 0;
1632 		}
1633 	} else {
1634 		hcint = 0;
1635 	}
1636 
1637 	switch (td->state) {
1638 	case DWC_CHAN_ST_START:
1639 		goto send_pkt;
1640 
1641 	case DWC_CHAN_ST_WAIT_ANE:
1642 		if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1643 			td->did_nak++;
1644 			td->tt_scheduled = 0;
1645 			goto send_pkt;
1646 		} else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1647 			td->offset += td->tx_bytes;
1648 			td->remainder -= td->tx_bytes;
1649 			td->toggle ^= 1;
1650 			td->did_nak = 0;
1651 			td->tt_scheduled = 0;
1652 
1653 			/* check remainder */
1654 			if (td->remainder == 0) {
1655 				if (td->short_pkt)
1656 					goto complete;
1657 
1658 				/*
1659 				 * Else we need to transmit a short
1660 				 * packet:
1661 				 */
1662 			}
1663 			goto send_pkt;
1664 		}
1665 		break;
1666 
1667 	case DWC_CHAN_ST_WAIT_S_ANE:
1668 		if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1669 			td->did_nak++;
1670 			td->tt_scheduled = 0;
1671 			goto send_pkt;
1672 		} else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1673 			td->did_nak = 0;
1674 			goto send_cpkt;
1675 		}
1676 		break;
1677 
1678 	case DWC_CHAN_ST_WAIT_C_ANE:
1679 		if (hcint & HCINT_NYET) {
1680 			goto send_cpkt;
1681 		} else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1682 			td->did_nak++;
1683 			td->tt_scheduled = 0;
1684 			goto send_pkt;
1685 		} else if (hcint & HCINT_ACK) {
1686 			td->offset += td->tx_bytes;
1687 			td->remainder -= td->tx_bytes;
1688 			td->toggle ^= 1;
1689 			td->did_nak = 0;
1690 			td->tt_scheduled = 0;
1691 
1692 			/* check remainder */
1693 			if (td->remainder == 0) {
1694 				if (td->short_pkt)
1695 					goto complete;
1696 
1697 				/* else we need to transmit a short packet */
1698 			}
1699 			goto send_pkt;
1700 		}
1701 		break;
1702 
1703 	case DWC_CHAN_ST_WAIT_C_PKT:
1704 		goto send_cpkt;
1705 
1706 	case DWC_CHAN_ST_TX_WAIT_ISOC:
1707 
1708 		/* Check if isochronous OUT traffic is complete */
1709 		if ((hcint & HCINT_HCH_DONE_MASK) == 0)
1710 			break;
1711 
1712 		td->offset += td->tx_bytes;
1713 		td->remainder -= td->tx_bytes;
1714 
1715 		if (td->hcsplt != 0 || td->remainder == 0)
1716 			goto complete;
1717 
1718 		/* check for next packet */
1719 		if (td->max_packet_count > 1)
1720 			td->tt_xactpos++;
1721 
1722 		/* free existing channel, if any */
1723 		dwc_otg_host_channel_free(sc, td);
1724 
1725 		td->state = DWC_CHAN_ST_TX_PKT_ISOC;
1726 
1727 		/* FALLTHROUGH */
1728 
1729 	case DWC_CHAN_ST_TX_PKT_ISOC:
1730 		if (dwc_otg_host_channel_alloc(sc, td, 1))
1731 			break;
1732 		channel = td->channel;
1733 		goto send_isoc_pkt;
1734 	default:
1735 		break;
1736 	}
1737 	goto busy;
1738 
1739 send_pkt:
1740 	/* free existing channel(s), if any */
1741 	dwc_otg_host_channel_free(sc, td);
1742 
1743 	if (td->hcsplt != 0) {
1744 		delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1745 		if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1746 			td->state = DWC_CHAN_ST_START;
1747 			goto busy;
1748 		}
1749 		delta = sc->sc_last_frame_num - td->tt_start_slot;
1750 		if (delta > 5) {
1751 			/* missed it */
1752 			td->tt_scheduled = 0;
1753 			td->state = DWC_CHAN_ST_START;
1754 			goto busy;
1755 		}
1756 	} else if (dwc_otg_host_rate_check(sc, td)) {
1757 		td->state = DWC_CHAN_ST_START;
1758 		goto busy;
1759 	}
1760 
1761 	/* allocate a new channel */
1762 	if (dwc_otg_host_channel_alloc(sc, td, 1)) {
1763 		td->state = DWC_CHAN_ST_START;
1764 		goto busy;
1765 	}
1766 
1767 	channel = td->channel;
1768 
1769 	/* set toggle, if any */
1770 	if (td->set_toggle) {
1771 		td->set_toggle = 0;
1772 		td->toggle = 1;
1773 	}
1774 
1775 	if (td->ep_type == UE_ISOCHRONOUS) {
1776 send_isoc_pkt:
1777 		/* Isochronous OUT transfers don't have any ACKs */
1778 		td->state = DWC_CHAN_ST_TX_WAIT_ISOC;
1779 		td->hcsplt &= ~HCSPLT_COMPSPLT;
1780 		if (td->hcsplt != 0) {
1781 			/* get maximum transfer length */
1782 			count = td->remainder;
1783 			if (count > HCSPLT_XACTLEN_BURST) {
1784 				DPRINTF("TT overflow\n");
1785 				td->error_any = 1;
1786 				goto complete;
1787 			}
1788 			/* Update transaction position */
1789 			td->hcsplt &= ~HCSPLT_XACTPOS_MASK;
1790 			td->hcsplt |= (HCSPLT_XACTPOS_ALL << HCSPLT_XACTPOS_SHIFT);
1791 		} else {
1792 			/* send one packet at a time */
1793 			count = td->max_packet_size;
1794 			if (td->remainder < count) {
1795 				/* we have a short packet */
1796 				td->short_pkt = 1;
1797 				count = td->remainder;
1798 			}
1799 		}
1800 	} else if (td->hcsplt != 0) {
1801 
1802 		td->hcsplt &= ~HCSPLT_COMPSPLT;
1803 
1804 		/* Wait for ACK/NAK/ERR from TT */
1805 		td->state = DWC_CHAN_ST_WAIT_S_ANE;
1806 
1807 		/* send one packet at a time */
1808 		count = td->max_packet_size;
1809 		if (td->remainder < count) {
1810 			/* we have a short packet */
1811 			td->short_pkt = 1;
1812 			count = td->remainder;
1813 		}
1814 	} else {
1815 		/* Wait for ACK/NAK/STALL from device */
1816 		td->state = DWC_CHAN_ST_WAIT_ANE;
1817 
1818 		/* send one packet at a time */
1819 		count = td->max_packet_size;
1820 		if (td->remainder < count) {
1821 			/* we have a short packet */
1822 			td->short_pkt = 1;
1823 			count = td->remainder;
1824 		}
1825 	}
1826 
1827 	/* check for High-Speed multi-packets */
1828 	if ((td->hcsplt == 0) && (td->max_packet_count > 1)) {
1829 		if (td->npkt == 0) {
1830 			if (td->remainder >= (3 * td->max_packet_size))
1831 				td->npkt = 3;
1832 			else if (td->remainder >= (2 * td->max_packet_size))
1833 				td->npkt = 2;
1834 			else
1835 				td->npkt = 1;
1836 
1837 			if (td->npkt > td->max_packet_count)
1838 				td->npkt = td->max_packet_count;
1839 
1840 			td->tt_xactpos = 1;	/* overload */
1841 		}
1842 		if (td->tt_xactpos == td->npkt) {
1843 			if (td->npkt == 1) {
1844 				DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1845 				    (count << HCTSIZ_XFERSIZE_SHIFT) |
1846 				    (1 << HCTSIZ_PKTCNT_SHIFT) |
1847 				    (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
1848 			} else if (td->npkt == 2) {
1849 				DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1850 				    (count << HCTSIZ_XFERSIZE_SHIFT) |
1851 				    (1 << HCTSIZ_PKTCNT_SHIFT) |
1852 				    (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT));
1853 			} else {
1854 				DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1855 				    (count << HCTSIZ_XFERSIZE_SHIFT) |
1856 				    (1 << HCTSIZ_PKTCNT_SHIFT) |
1857 				    (HCTSIZ_PID_DATA2 << HCTSIZ_PID_SHIFT));
1858 			}
1859 			td->npkt = 0;
1860 		} else {
1861 			DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1862 			    (count << HCTSIZ_XFERSIZE_SHIFT) |
1863 			    (1 << HCTSIZ_PKTCNT_SHIFT) |
1864 			    (HCTSIZ_PID_MDATA << HCTSIZ_PID_SHIFT));
1865 		}
1866 	} else {
1867 		/* TODO: HCTSIZ_DOPNG */
1868 
1869 		DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1870 		    (count << HCTSIZ_XFERSIZE_SHIFT) |
1871 		    (1 << HCTSIZ_PKTCNT_SHIFT) |
1872 		    (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1873 		    (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1874 	}
1875 
1876 	DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1877 
1878 	hcchar = td->hcchar;
1879 	hcchar &= ~HCCHAR_EPDIR_IN;
1880 
1881 	/* send after next SOF event */
1882 	if ((sc->sc_last_frame_num & 1) == 0)
1883 		hcchar |= HCCHAR_ODDFRM;
1884 	else
1885 		hcchar &= ~HCCHAR_ODDFRM;
1886 
1887 	/* must enable before writing data to FIFO */
1888 	DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1889 
1890 	if (count != 0) {
1891 
1892 		/* clear topmost word before copy */
1893 		sc->sc_tx_bounce_buffer[(count - 1) / 4] = 0;
1894 
1895 		/* copy out data */
1896 		usbd_copy_out(td->pc, td->offset,
1897 		    sc->sc_tx_bounce_buffer, count);
1898 
1899 		/* transfer data into FIFO */
1900 		bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
1901 		    DOTG_DFIFO(channel),
1902 		    sc->sc_tx_bounce_buffer, (count + 3) / 4);
1903 	}
1904 
1905 	/* store number of bytes transmitted */
1906 	td->tx_bytes = count;
1907 	goto busy;
1908 
1909 send_cpkt:
1910 	/* free existing channel, if any */
1911 	dwc_otg_host_channel_free(sc, td);
1912 
1913 	delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1914 	if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1915 		td->state = DWC_CHAN_ST_WAIT_C_PKT;
1916 		goto busy;
1917 	}
1918 	delta = sc->sc_last_frame_num - td->tt_start_slot;
1919 	if (delta > DWC_OTG_TT_SLOT_MAX) {
1920 		/* we missed the service interval */
1921 		if (td->ep_type != UE_ISOCHRONOUS)
1922 			td->error_any = 1;
1923 		goto complete;
1924 	}
1925 
1926 	/* allocate a new channel */
1927 	if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1928 		td->state = DWC_CHAN_ST_WAIT_C_PKT;
1929 		goto busy;
1930 	}
1931 
1932 	channel = td->channel;
1933 
1934  	td->hcsplt |= HCSPLT_COMPSPLT;
1935 	td->state = DWC_CHAN_ST_WAIT_C_ANE;
1936 
1937 	DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1938 	    (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
1939 
1940 	DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1941 
1942 	hcchar = td->hcchar;
1943 	hcchar &= ~HCCHAR_EPDIR_IN;
1944 
1945 	/* receive complete split ASAP */
1946 	if ((sc->sc_last_frame_num & 1) != 0)
1947 		hcchar |= HCCHAR_ODDFRM;
1948 	else
1949 		hcchar &= ~HCCHAR_ODDFRM;
1950 
1951 	/* must enable channel before data can be received */
1952 	DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1953 
1954 	/* wait until next slot before trying complete split */
1955 	td->tt_complete_slot = sc->sc_last_frame_num + 1;
1956 busy:
1957 	return (1);	/* busy */
1958 
1959 complete:
1960 	dwc_otg_host_channel_free(sc, td);
1961 	return (0);	/* complete */
1962 }
1963 
1964 static uint8_t
1965 dwc_otg_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1966 {
1967 	uint32_t max_buffer;
1968 	uint32_t count;
1969 	uint32_t fifo_left;
1970 	uint32_t mpkt;
1971 	uint32_t temp;
1972 	uint8_t to;
1973 
1974 	to = 3;				/* don't loop forever! */
1975 
1976 	max_buffer = sc->sc_hw_ep_profile[td->ep_no].max_buffer;
1977 
1978 repeat:
1979 	/* check for for endpoint 0 data */
1980 
1981 	temp = sc->sc_last_rx_status;
1982 
1983 	if ((td->ep_no == 0) && (temp != 0) &&
1984 	    (GRXSTSRD_CHNUM_GET(temp) == 0)) {
1985 
1986 		if ((temp & GRXSTSRD_PKTSTS_MASK) !=
1987 		    GRXSTSRD_STP_DATA) {
1988 
1989 			/* dump data - wrong direction */
1990 			dwc_otg_common_rx_ack(sc);
1991 		} else {
1992 			/*
1993 			 * The current transfer was cancelled
1994 			 * by the USB Host:
1995 			 */
1996 			td->error_any = 1;
1997 			return (0);		/* complete */
1998 		}
1999 	}
2000 
2001 	/* fill in more TX data, if possible */
2002 	if (td->tx_bytes != 0) {
2003 
2004 		uint16_t cpkt;
2005 
2006 		/* check if packets have been transferred */
2007 		temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2008 
2009 		/* get current packet number */
2010 		cpkt = DXEPTSIZ_GET_NPKT(temp);
2011 
2012 		if (cpkt >= td->npkt) {
2013 			fifo_left = 0;
2014 		} else {
2015 			if (max_buffer != 0) {
2016 				fifo_left = (td->npkt - cpkt) *
2017 				    td->max_packet_size;
2018 
2019 				if (fifo_left > max_buffer)
2020 					fifo_left = max_buffer;
2021 			} else {
2022 				fifo_left = td->max_packet_size;
2023 			}
2024 		}
2025 
2026 		count = td->tx_bytes;
2027 		if (count > fifo_left)
2028 			count = fifo_left;
2029 
2030 		if (count != 0) {
2031 
2032 			/* clear topmost word before copy */
2033 			sc->sc_tx_bounce_buffer[(count - 1) / 4] = 0;
2034 
2035 			/* copy out data */
2036 			usbd_copy_out(td->pc, td->offset,
2037 			    sc->sc_tx_bounce_buffer, count);
2038 
2039 			/* transfer data into FIFO */
2040 			bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
2041 			    DOTG_DFIFO(td->ep_no),
2042 			    sc->sc_tx_bounce_buffer, (count + 3) / 4);
2043 
2044 			td->tx_bytes -= count;
2045 			td->remainder -= count;
2046 			td->offset += count;
2047 			td->npkt = cpkt;
2048 		}
2049 		if (td->tx_bytes != 0)
2050 			goto not_complete;
2051 
2052 		/* check remainder */
2053 		if (td->remainder == 0) {
2054 			if (td->short_pkt)
2055 				return (0);	/* complete */
2056 
2057 			/* else we need to transmit a short packet */
2058 		}
2059 	}
2060 
2061 	if (!to--)
2062 		goto not_complete;
2063 
2064 	/* check if not all packets have been transferred */
2065 	temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2066 
2067 	if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2068 
2069 		DPRINTFN(5, "busy ep=%d npkt=%d DIEPTSIZ=0x%08x "
2070 		    "DIEPCTL=0x%08x\n", td->ep_no,
2071 		    DXEPTSIZ_GET_NPKT(temp),
2072 		    temp, DWC_OTG_READ_4(sc, DOTG_DIEPCTL(td->ep_no)));
2073 
2074 		goto not_complete;
2075 	}
2076 
2077 	DPRINTFN(5, "rem=%u ep=%d\n", td->remainder, td->ep_no);
2078 
2079 	/* try to optimise by sending more data */
2080 	if ((max_buffer != 0) && ((td->max_packet_size & 3) == 0)) {
2081 
2082 		/* send multiple packets at the same time */
2083 		mpkt = max_buffer / td->max_packet_size;
2084 
2085 		if (mpkt > 0x3FE)
2086 			mpkt = 0x3FE;
2087 
2088 		count = td->remainder;
2089 		if (count > 0x7FFFFF)
2090 			count = 0x7FFFFF - (0x7FFFFF % td->max_packet_size);
2091 
2092 		td->npkt = count / td->max_packet_size;
2093 
2094 		/*
2095 		 * NOTE: We could use 0x3FE instead of "mpkt" in the
2096 		 * check below to get more throughput, but then we
2097 		 * have a dependency towards non-generic chip features
2098 		 * to disable the TX-FIFO-EMPTY interrupts on a per
2099 		 * endpoint basis. Increase the maximum buffer size of
2100 		 * the IN endpoint to increase the performance.
2101 		 */
2102 		if (td->npkt > mpkt) {
2103 			td->npkt = mpkt;
2104 			count = td->max_packet_size * mpkt;
2105 		} else if ((count == 0) || (count % td->max_packet_size)) {
2106 			/* we are transmitting a short packet */
2107 			td->npkt++;
2108 			td->short_pkt = 1;
2109 		}
2110 	} else {
2111 		/* send one packet at a time */
2112 		mpkt = 1;
2113 		count = td->max_packet_size;
2114 		if (td->remainder < count) {
2115 			/* we have a short packet */
2116 			td->short_pkt = 1;
2117 			count = td->remainder;
2118 		}
2119 		td->npkt = 1;
2120 	}
2121 	DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(td->ep_no),
2122 	    DXEPTSIZ_SET_MULTI(1) |
2123 	    DXEPTSIZ_SET_NPKT(td->npkt) |
2124 	    DXEPTSIZ_SET_NBYTES(count));
2125 
2126 	/* make room for buffering */
2127 	td->npkt += mpkt;
2128 
2129 	temp = sc->sc_in_ctl[td->ep_no];
2130 
2131 	/* must enable before writing data to FIFO */
2132 	DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(td->ep_no), temp |
2133 	    DIEPCTL_EPENA |
2134 	    DIEPCTL_CNAK);
2135 
2136 	td->tx_bytes = count;
2137 
2138 	/* check remainder */
2139 	if (td->tx_bytes == 0 &&
2140 	    td->remainder == 0) {
2141 		if (td->short_pkt)
2142 			return (0);	/* complete */
2143 
2144 		/* else we need to transmit a short packet */
2145 	}
2146 	goto repeat;
2147 
2148 not_complete:
2149 	return (1);			/* not complete */
2150 }
2151 
2152 static uint8_t
2153 dwc_otg_data_tx_sync(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
2154 {
2155 	uint32_t temp;
2156 
2157 	/*
2158 	 * If all packets are transferred we are complete:
2159 	 */
2160 	temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2161 
2162 	/* check that all packets have been transferred */
2163 	if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2164 		DPRINTFN(5, "busy ep=%d\n", td->ep_no);
2165 		goto not_complete;
2166 	}
2167 	return (0);
2168 
2169 not_complete:
2170 
2171 	/* we only want to know if there is a SETUP packet or free IN packet */
2172 
2173 	temp = sc->sc_last_rx_status;
2174 
2175 	if ((td->ep_no == 0) && (temp != 0) &&
2176 	    (GRXSTSRD_CHNUM_GET(temp) == 0)) {
2177 
2178 		if ((temp & GRXSTSRD_PKTSTS_MASK) ==
2179 		    GRXSTSRD_STP_DATA) {
2180 			DPRINTFN(5, "faking complete\n");
2181 			/*
2182 			 * Race condition: We are complete!
2183 			 */
2184 			return (0);
2185 		} else {
2186 			/* dump data - wrong direction */
2187 			dwc_otg_common_rx_ack(sc);
2188 		}
2189 	}
2190 	return (1);			/* not complete */
2191 }
2192 
2193 static void
2194 dwc_otg_xfer_do_fifo(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2195 {
2196 	struct dwc_otg_td *td;
2197 	uint8_t toggle;
2198 	uint8_t tmr_val;
2199 	uint8_t tmr_res;
2200 
2201 	DPRINTFN(9, "\n");
2202 
2203 	td = xfer->td_transfer_cache;
2204 	if (td == NULL)
2205 		return;
2206 
2207 	while (1) {
2208 		if ((td->func) (sc, td)) {
2209 			/* operation in progress */
2210 			break;
2211 		}
2212 		if (((void *)td) == xfer->td_transfer_last) {
2213 			goto done;
2214 		}
2215 		if (td->error_any) {
2216 			goto done;
2217 		} else if (td->remainder > 0) {
2218 			/*
2219 			 * We had a short transfer. If there is no alternate
2220 			 * next, stop processing !
2221 			 */
2222 			if (!td->alt_next)
2223 				goto done;
2224 		}
2225 
2226 		/*
2227 		 * Fetch the next transfer descriptor and transfer
2228 		 * some flags to the next transfer descriptor
2229 		 */
2230 		tmr_res = td->tmr_res;
2231 		tmr_val = td->tmr_val;
2232 		toggle = td->toggle;
2233 		td = td->obj_next;
2234 		xfer->td_transfer_cache = td;
2235 		td->toggle = toggle;	/* transfer toggle */
2236 		td->tmr_res = tmr_res;
2237 		td->tmr_val = tmr_val;
2238 	}
2239 	return;
2240 
2241 done:
2242 	xfer->td_transfer_cache = NULL;
2243 	sc->sc_xfer_complete = 1;
2244 }
2245 
2246 static uint8_t
2247 dwc_otg_xfer_do_complete_locked(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2248 {
2249 	struct dwc_otg_td *td;
2250 
2251 	DPRINTFN(9, "\n");
2252 
2253 	td = xfer->td_transfer_cache;
2254 	if (td == NULL) {
2255 		/* compute all actual lengths */
2256 		dwc_otg_standard_done(xfer);
2257 		return (1);
2258 	}
2259 	return (0);
2260 }
2261 
2262 static void
2263 dwc_otg_timer(void *_sc)
2264 {
2265 	struct dwc_otg_softc *sc = _sc;
2266 	struct usb_xfer *xfer;
2267 	struct dwc_otg_td *td;
2268 
2269 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2270 
2271 	DPRINTF("\n");
2272 
2273 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
2274 
2275 	/* increment timer value */
2276 	sc->sc_tmr_val++;
2277 
2278 	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
2279 		td = xfer->td_transfer_cache;
2280 		if (td != NULL) {
2281 			/* reset NAK counter */
2282 			td->did_nak = 0;
2283 		}
2284 	}
2285 
2286 	/* enable SOF interrupt, which will poll jobs */
2287 	dwc_otg_enable_sof_irq(sc);
2288 
2289 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2290 
2291 	if (sc->sc_timer_active) {
2292 		/* restart timer */
2293 		usb_callout_reset(&sc->sc_timer,
2294 		    hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2295 		    &dwc_otg_timer, sc);
2296 	}
2297 }
2298 
2299 static void
2300 dwc_otg_timer_start(struct dwc_otg_softc *sc)
2301 {
2302 	if (sc->sc_timer_active != 0)
2303 		return;
2304 
2305 	sc->sc_timer_active = 1;
2306 
2307 	/* restart timer */
2308 	usb_callout_reset(&sc->sc_timer,
2309 	    hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2310 	    &dwc_otg_timer, sc);
2311 }
2312 
2313 static void
2314 dwc_otg_timer_stop(struct dwc_otg_softc *sc)
2315 {
2316 	if (sc->sc_timer_active == 0)
2317 		return;
2318 
2319 	sc->sc_timer_active = 0;
2320 
2321 	/* stop timer */
2322 	usb_callout_stop(&sc->sc_timer);
2323 }
2324 
2325 static void
2326 dwc_otg_host_channel_disable(struct dwc_otg_softc *sc, uint8_t x)
2327 {
2328 	uint32_t hcchar;
2329 
2330 	hcchar = DWC_OTG_READ_4(sc, DOTG_HCCHAR(x));
2331 
2332 	/* disable host channel, if any */
2333 	if (hcchar & (HCCHAR_CHENA | HCCHAR_CHDIS)) {
2334 		/* disable channel */
2335 		DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(x),
2336 		    HCCHAR_CHENA | HCCHAR_CHDIS);
2337 		/* wait for chip to get its brains in order */
2338 		sc->sc_chan_state[x].wait_sof = 2;
2339 	}
2340 
2341 	/* release TX FIFO usage, if any */
2342 	sc->sc_tx_cur_p_level -= sc->sc_chan_state[x].tx_p_size;
2343 	sc->sc_tx_cur_np_level -= sc->sc_chan_state[x].tx_np_size;
2344 
2345 	/* don't release TX FIFO usage twice */
2346 	sc->sc_chan_state[x].tx_p_size = 0;
2347 	sc->sc_chan_state[x].tx_np_size = 0;
2348 }
2349 
2350 static uint16_t
2351 dwc_otg_compute_isoc_rx_tt_slot(struct dwc_otg_tt_info *pinfo)
2352 {
2353 	if (pinfo->slot_index < DWC_OTG_TT_SLOT_MAX)
2354 		pinfo->slot_index++;
2355 	return (pinfo->slot_index);
2356 }
2357 
2358 static uint8_t
2359 dwc_otg_update_host_transfer_schedule_locked(struct dwc_otg_softc *sc)
2360 {
2361 	TAILQ_HEAD(, usb_xfer) head;
2362 	struct usb_xfer *xfer;
2363 	struct usb_xfer *xfer_next;
2364 	struct dwc_otg_td *td;
2365 	uint16_t temp;
2366 	uint16_t slot;
2367 	uint8_t x;
2368 
2369 	temp = DWC_OTG_READ_4(sc, DOTG_HFNUM) & DWC_OTG_FRAME_MASK;
2370 
2371 	if (sc->sc_last_frame_num == temp)
2372 		return (0);
2373 
2374 	sc->sc_last_frame_num = temp;
2375 
2376 	TAILQ_INIT(&head);
2377 
2378 	for (x = 0; x != sc->sc_host_ch_max; x++) {
2379 		if (sc->sc_chan_state[x].wait_sof == 0)
2380 			continue;
2381 
2382 		sc->sc_needsof = 1;
2383 		if (--(sc->sc_chan_state[x].wait_sof) == 0)
2384 			dwc_otg_host_channel_disable(sc, x);
2385 	}
2386 
2387 	if ((temp & 7) == 0) {
2388 
2389 		/* reset the schedule */
2390 		memset(sc->sc_tt_info, 0, sizeof(sc->sc_tt_info));
2391 
2392 		TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2393 			td = xfer->td_transfer_cache;
2394 			if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2395 				continue;
2396 
2397 			/* check for IN direction */
2398 			if ((td->hcchar & HCCHAR_EPDIR_IN) != 0)
2399 				continue;
2400 
2401 			/* execute more frames */
2402 			td->tmr_val = 0;
2403 
2404 			sc->sc_needsof = 1;
2405 
2406 			if (td->hcsplt == 0 || td->tt_scheduled != 0)
2407 				continue;
2408 
2409 			/* compute slot */
2410 			slot = dwc_otg_compute_isoc_rx_tt_slot(
2411 			    sc->sc_tt_info + td->tt_index);
2412 			if (slot > 3) {
2413 				/*
2414 				 * Not enough time to get complete
2415 				 * split executed.
2416 				 */
2417 				continue;
2418 			}
2419 			/* Delayed start */
2420 			td->tt_start_slot = temp + slot;
2421 			td->tt_scheduled = 1;
2422 			TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2423 			TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2424 		}
2425 
2426 		TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2427 			td = xfer->td_transfer_cache;
2428 			if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2429 				continue;
2430 
2431 			/* check for OUT direction */
2432 			if ((td->hcchar & HCCHAR_EPDIR_IN) == 0)
2433 				continue;
2434 
2435 			/* execute more frames */
2436 			td->tmr_val = 0;
2437 
2438 			sc->sc_needsof = 1;
2439 
2440 			if (td->hcsplt == 0 || td->tt_scheduled != 0)
2441 				continue;
2442 
2443 			/* Start ASAP */
2444 			td->tt_start_slot = temp;
2445 			td->tt_scheduled = 1;
2446 			TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2447 			TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2448 		}
2449 
2450 		TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2451 			td = xfer->td_transfer_cache;
2452 			if (td == NULL || td->ep_type != UE_INTERRUPT)
2453 				continue;
2454 
2455 			if (td->tt_scheduled != 0) {
2456 				sc->sc_needsof = 1;
2457 				continue;
2458 			}
2459 
2460 			if (dwc_otg_host_rate_check_interrupt(sc, td))
2461 				continue;
2462 
2463 			if (td->hcsplt == 0) {
2464 				sc->sc_needsof = 1;
2465 				td->tt_scheduled = 1;
2466 				continue;
2467 			}
2468 
2469 			/* start ASAP */
2470 			td->tt_start_slot = temp;
2471 			sc->sc_needsof = 1;
2472 			td->tt_scheduled = 1;
2473 			TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2474 			TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2475 		}
2476 
2477 		TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2478 			td = xfer->td_transfer_cache;
2479 			if (td == NULL ||
2480 			    td->ep_type != UE_CONTROL ||
2481 			    td->did_nak >= DWC_OTG_NAK_MAX) {
2482 				continue;
2483 			}
2484 
2485 			sc->sc_needsof = 1;
2486 
2487 			if (td->hcsplt == 0 || td->tt_scheduled != 0)
2488 				continue;
2489 
2490 			/* start ASAP */
2491 			td->tt_start_slot = temp;
2492 			td->tt_scheduled = 1;
2493 			TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2494 			TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2495 		}
2496 	}
2497 	if ((temp & 7) < 6) {
2498 		TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2499 			td = xfer->td_transfer_cache;
2500 			if (td == NULL ||
2501 			    td->ep_type != UE_BULK ||
2502 			    td->did_nak >= DWC_OTG_NAK_MAX) {
2503 				continue;
2504 			}
2505 
2506 			sc->sc_needsof = 1;
2507 
2508 			if (td->hcsplt == 0 || td->tt_scheduled != 0)
2509 				continue;
2510 
2511 			/* start ASAP */
2512 			td->tt_start_slot = temp;
2513 			td->tt_scheduled = 1;
2514 			TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2515 			TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2516 		}
2517 	}
2518 
2519 	/* Put TT transfers in execution order at the end */
2520 	TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2521 
2522 	/* move all TT transfers in front, keeping the current order */
2523 	TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2524 		td = xfer->td_transfer_cache;
2525 		if (td == NULL || td->hcsplt == 0)
2526 			continue;
2527 		TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2528 		TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2529 	}
2530 	TAILQ_CONCAT(&head, &sc->sc_bus.intr_q.head, wait_entry);
2531 	TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2532 
2533 	/* put non-TT BULK transfers last */
2534 	TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2535 		td = xfer->td_transfer_cache;
2536 		if (td == NULL || td->hcsplt != 0 || td->ep_type != UE_BULK)
2537 			continue;
2538 		TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2539 		TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2540 	}
2541 	TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2542 
2543 	if ((temp & 7) == 0) {
2544 
2545 		DPRINTFN(12, "SOF interrupt #%d, needsof=%d\n",
2546 		    (int)temp, (int)sc->sc_needsof);
2547 
2548 		/* update SOF IRQ mask */
2549 		if (sc->sc_irq_mask & GINTMSK_SOFMSK) {
2550 			if (sc->sc_needsof == 0) {
2551 				sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2552 				DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2553 			}
2554 		} else {
2555 			if (sc->sc_needsof != 0) {
2556 				sc->sc_irq_mask |= GINTMSK_SOFMSK;
2557 				DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2558 			}
2559 		}
2560 
2561 		/* clear need SOF flag */
2562 		sc->sc_needsof = 0;
2563 	}
2564 	return (1);
2565 }
2566 
2567 static void
2568 dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *sc)
2569 {
2570 	struct usb_xfer *xfer;
2571 	uint32_t temp;
2572 	uint8_t got_rx_status;
2573 	uint8_t x;
2574 
2575 repeat:
2576 	/* get all channel interrupts */
2577 	for (x = 0; x != sc->sc_host_ch_max; x++) {
2578 		temp = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
2579 		if (temp != 0) {
2580 			DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), temp);
2581 			temp &= ~HCINT_SOFTWARE_ONLY;
2582 			sc->sc_chan_state[x].hcint |= temp;
2583 		}
2584 	}
2585 
2586 	if (sc->sc_last_rx_status == 0) {
2587 
2588 		temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2589 		if (temp & GINTSTS_RXFLVL) {
2590 			/* pop current status */
2591 			sc->sc_last_rx_status =
2592 			    DWC_OTG_READ_4(sc, DOTG_GRXSTSPD);
2593 		}
2594 
2595 		if (sc->sc_last_rx_status != 0) {
2596 
2597 			uint8_t ep_no;
2598 
2599 			temp = sc->sc_last_rx_status &
2600 			    GRXSTSRD_PKTSTS_MASK;
2601 
2602 			/* non-data messages we simply skip */
2603 			if (temp != GRXSTSRD_STP_DATA &&
2604 			    temp != GRXSTSRD_OUT_DATA) {
2605 				dwc_otg_common_rx_ack(sc);
2606 				goto repeat;
2607 			}
2608 
2609 			temp = GRXSTSRD_BCNT_GET(
2610 			    sc->sc_last_rx_status);
2611 			ep_no = GRXSTSRD_CHNUM_GET(
2612 			    sc->sc_last_rx_status);
2613 
2614 			/* receive data, if any */
2615 			if (temp != 0) {
2616 				DPRINTF("Reading %d bytes from ep %d\n", temp, ep_no);
2617 				bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
2618 				    DOTG_DFIFO(ep_no),
2619 				    sc->sc_rx_bounce_buffer, (temp + 3) / 4);
2620 			}
2621 
2622 			/* check if we should dump the data */
2623 			if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2624 				dwc_otg_common_rx_ack(sc);
2625 				goto repeat;
2626 			}
2627 
2628 			got_rx_status = 1;
2629 
2630 			DPRINTFN(5, "RX status = 0x%08x: ch=%d pid=%d bytes=%d sts=%d\n",
2631 			    sc->sc_last_rx_status, ep_no,
2632 			    (sc->sc_last_rx_status >> 15) & 3,
2633 			    GRXSTSRD_BCNT_GET(sc->sc_last_rx_status),
2634 			    (sc->sc_last_rx_status >> 17) & 15);
2635 		} else {
2636 			got_rx_status = 0;
2637 		}
2638 	} else {
2639 		uint8_t ep_no;
2640 
2641 		ep_no = GRXSTSRD_CHNUM_GET(
2642 		    sc->sc_last_rx_status);
2643 
2644 		/* check if we should dump the data */
2645 		if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2646 			dwc_otg_common_rx_ack(sc);
2647 			goto repeat;
2648 		}
2649 
2650 		got_rx_status = 1;
2651 	}
2652 
2653 	/* execute FIFOs */
2654 	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry)
2655 		dwc_otg_xfer_do_fifo(sc, xfer);
2656 
2657 	if (got_rx_status) {
2658 		/* check if data was consumed */
2659 		if (sc->sc_last_rx_status == 0)
2660 			goto repeat;
2661 
2662 		/* disable RX FIFO level interrupt */
2663 		sc->sc_irq_mask &= ~GINTMSK_RXFLVLMSK;
2664 		DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2665 	}
2666 
2667 	if (sc->sc_flags.status_device_mode == 0 && sc->sc_xfer_complete == 0) {
2668 		/* update host transfer schedule, so that new transfers can be issued */
2669 		if (dwc_otg_update_host_transfer_schedule_locked(sc))
2670 			goto repeat;
2671 	}
2672 }
2673 
2674 static void
2675 dwc_otg_interrupt_complete_locked(struct dwc_otg_softc *sc)
2676 {
2677 	struct usb_xfer *xfer;
2678 repeat:
2679 	/* scan for completion events */
2680 	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
2681 		if (dwc_otg_xfer_do_complete_locked(sc, xfer))
2682 			goto repeat;
2683 	}
2684 }
2685 
2686 static void
2687 dwc_otg_vbus_interrupt(struct dwc_otg_softc *sc, uint8_t is_on)
2688 {
2689 	DPRINTFN(5, "vbus = %u\n", is_on);
2690 
2691 	/*
2692 	 * If the USB host mode is forced, then assume VBUS is always
2693 	 * present else rely on the input to this function:
2694 	 */
2695 	if ((is_on != 0) || (sc->sc_mode == DWC_MODE_HOST)) {
2696 
2697 		if (!sc->sc_flags.status_vbus) {
2698 			sc->sc_flags.status_vbus = 1;
2699 
2700 			/* complete root HUB interrupt endpoint */
2701 
2702 			dwc_otg_root_intr(sc);
2703 		}
2704 	} else {
2705 		if (sc->sc_flags.status_vbus) {
2706 			sc->sc_flags.status_vbus = 0;
2707 			sc->sc_flags.status_bus_reset = 0;
2708 			sc->sc_flags.status_suspend = 0;
2709 			sc->sc_flags.change_suspend = 0;
2710 			sc->sc_flags.change_connect = 1;
2711 
2712 			/* complete root HUB interrupt endpoint */
2713 
2714 			dwc_otg_root_intr(sc);
2715 		}
2716 	}
2717 }
2718 
2719 int
2720 dwc_otg_filter_interrupt(void *arg)
2721 {
2722 	struct dwc_otg_softc *sc = arg;
2723 	int retval = FILTER_HANDLED;
2724 	uint32_t status;
2725 
2726 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
2727 
2728 	/* read and clear interrupt status */
2729 	status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2730 
2731 	/* clear interrupts we are handling here */
2732 	DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & ~DWC_OTG_MSK_GINT_THREAD_IRQ);
2733 
2734 	/* check for USB state change interrupts */
2735 	if ((status & DWC_OTG_MSK_GINT_THREAD_IRQ) != 0)
2736 		retval = FILTER_SCHEDULE_THREAD;
2737 
2738 	/* clear all IN endpoint interrupts */
2739 	if (status & GINTSTS_IEPINT) {
2740 		uint32_t temp;
2741 		uint8_t x;
2742 
2743 		for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
2744 			temp = DWC_OTG_READ_4(sc, DOTG_DIEPINT(x));
2745 			if (temp & DIEPMSK_XFERCOMPLMSK) {
2746 				DWC_OTG_WRITE_4(sc, DOTG_DIEPINT(x),
2747 				    DIEPMSK_XFERCOMPLMSK);
2748 			}
2749 		}
2750 	}
2751 
2752 	/* poll FIFOs, if any */
2753 	dwc_otg_interrupt_poll_locked(sc);
2754 
2755 	if (sc->sc_xfer_complete != 0)
2756 		retval = FILTER_SCHEDULE_THREAD;
2757 
2758 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2759 
2760 	return (retval);
2761 }
2762 
2763 void
2764 dwc_otg_interrupt(void *arg)
2765 {
2766 	struct dwc_otg_softc *sc = arg;
2767 	uint32_t status;
2768 
2769 	USB_BUS_LOCK(&sc->sc_bus);
2770 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
2771 
2772 	/* read and clear interrupt status */
2773 	status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2774 
2775 	/* clear interrupts we are handling here */
2776 	DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & DWC_OTG_MSK_GINT_THREAD_IRQ);
2777 
2778 	DPRINTFN(14, "GINTSTS=0x%08x HAINT=0x%08x HFNUM=0x%08x\n",
2779 	    status, DWC_OTG_READ_4(sc, DOTG_HAINT),
2780 	    DWC_OTG_READ_4(sc, DOTG_HFNUM));
2781 
2782 	if (status & GINTSTS_USBRST) {
2783 
2784 		/* set correct state */
2785 		sc->sc_flags.status_device_mode = 1;
2786 		sc->sc_flags.status_bus_reset = 0;
2787 		sc->sc_flags.status_suspend = 0;
2788 		sc->sc_flags.change_suspend = 0;
2789 		sc->sc_flags.change_connect = 1;
2790 
2791 		/* Disable SOF interrupt */
2792 		sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2793 		DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2794 
2795 		/* complete root HUB interrupt endpoint */
2796 		dwc_otg_root_intr(sc);
2797 	}
2798 
2799 	/* check for any bus state change interrupts */
2800 	if (status & GINTSTS_ENUMDONE) {
2801 
2802 		uint32_t temp;
2803 
2804 		DPRINTFN(5, "end of reset\n");
2805 
2806 		/* set correct state */
2807 		sc->sc_flags.status_device_mode = 1;
2808 		sc->sc_flags.status_bus_reset = 1;
2809 		sc->sc_flags.status_suspend = 0;
2810 		sc->sc_flags.change_suspend = 0;
2811 		sc->sc_flags.change_connect = 1;
2812 		sc->sc_flags.status_low_speed = 0;
2813 		sc->sc_flags.port_enabled = 1;
2814 
2815 		/* reset FIFOs */
2816 		(void) dwc_otg_init_fifo(sc, DWC_MODE_DEVICE);
2817 
2818 		/* reset function address */
2819 		dwc_otg_set_address(sc, 0);
2820 
2821 		/* figure out enumeration speed */
2822 		temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
2823 		if (DSTS_ENUMSPD_GET(temp) == DSTS_ENUMSPD_HI)
2824 			sc->sc_flags.status_high_speed = 1;
2825 		else
2826 			sc->sc_flags.status_high_speed = 0;
2827 
2828 		/*
2829 		 * Disable resume and SOF interrupt, and enable
2830 		 * suspend and RX frame interrupt:
2831 		 */
2832 		sc->sc_irq_mask &= ~(GINTMSK_WKUPINTMSK | GINTMSK_SOFMSK);
2833 		sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
2834 		DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2835 
2836 		/* complete root HUB interrupt endpoint */
2837 		dwc_otg_root_intr(sc);
2838 	}
2839 
2840 	if (status & GINTSTS_PRTINT) {
2841 		uint32_t hprt;
2842 
2843 		hprt = DWC_OTG_READ_4(sc, DOTG_HPRT);
2844 
2845 		/* clear change bits */
2846 		DWC_OTG_WRITE_4(sc, DOTG_HPRT, (hprt & (
2847 		    HPRT_PRTPWR | HPRT_PRTENCHNG |
2848 		    HPRT_PRTCONNDET | HPRT_PRTOVRCURRCHNG)) |
2849 		    sc->sc_hprt_val);
2850 
2851 		DPRINTFN(12, "GINTSTS=0x%08x, HPRT=0x%08x\n", status, hprt);
2852 
2853 		sc->sc_flags.status_device_mode = 0;
2854 
2855 		if (hprt & HPRT_PRTCONNSTS)
2856 			sc->sc_flags.status_bus_reset = 1;
2857 		else
2858 			sc->sc_flags.status_bus_reset = 0;
2859 
2860 		if (hprt & HPRT_PRTENCHNG)
2861 			sc->sc_flags.change_enabled = 1;
2862 
2863 		if (hprt & HPRT_PRTENA)
2864 			sc->sc_flags.port_enabled = 1;
2865 		else
2866 			sc->sc_flags.port_enabled = 0;
2867 
2868 		if (hprt & HPRT_PRTOVRCURRCHNG)
2869 			sc->sc_flags.change_over_current = 1;
2870 
2871 		if (hprt & HPRT_PRTOVRCURRACT)
2872 			sc->sc_flags.port_over_current = 1;
2873 		else
2874 			sc->sc_flags.port_over_current = 0;
2875 
2876 		if (hprt & HPRT_PRTPWR)
2877 			sc->sc_flags.port_powered = 1;
2878 		else
2879 			sc->sc_flags.port_powered = 0;
2880 
2881 		if (((hprt & HPRT_PRTSPD_MASK)
2882 		    >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_LOW)
2883 			sc->sc_flags.status_low_speed = 1;
2884 		else
2885 			sc->sc_flags.status_low_speed = 0;
2886 
2887 		if (((hprt & HPRT_PRTSPD_MASK)
2888 		    >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_HIGH)
2889 			sc->sc_flags.status_high_speed = 1;
2890 		else
2891 			sc->sc_flags.status_high_speed = 0;
2892 
2893 		if (hprt & HPRT_PRTCONNDET)
2894 			sc->sc_flags.change_connect = 1;
2895 
2896 		if (hprt & HPRT_PRTSUSP)
2897 			dwc_otg_suspend_irq(sc);
2898 		else
2899 			dwc_otg_resume_irq(sc);
2900 
2901 		/* complete root HUB interrupt endpoint */
2902 		dwc_otg_root_intr(sc);
2903 
2904 		/* update host frame interval */
2905 		dwc_otg_update_host_frame_interval(sc);
2906 	}
2907 
2908 	/*
2909 	 * If resume and suspend is set at the same time we interpret
2910 	 * that like RESUME. Resume is set when there is at least 3
2911 	 * milliseconds of inactivity on the USB BUS.
2912 	 */
2913 	if (status & GINTSTS_WKUPINT) {
2914 
2915 		DPRINTFN(5, "resume interrupt\n");
2916 
2917 		dwc_otg_resume_irq(sc);
2918 
2919 	} else if (status & GINTSTS_USBSUSP) {
2920 
2921 		DPRINTFN(5, "suspend interrupt\n");
2922 
2923 		dwc_otg_suspend_irq(sc);
2924 	}
2925 	/* check VBUS */
2926 	if (status & (GINTSTS_USBSUSP |
2927 	    GINTSTS_USBRST |
2928 	    GINTMSK_OTGINTMSK |
2929 	    GINTSTS_SESSREQINT)) {
2930 		uint32_t temp;
2931 
2932 		temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
2933 
2934 		DPRINTFN(5, "GOTGCTL=0x%08x\n", temp);
2935 
2936 		dwc_otg_vbus_interrupt(sc,
2937 		    (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
2938 	}
2939 
2940 	if (sc->sc_xfer_complete != 0) {
2941 		sc->sc_xfer_complete = 0;
2942 
2943 		/* complete FIFOs, if any */
2944 		dwc_otg_interrupt_complete_locked(sc);
2945 
2946 		if (sc->sc_flags.status_device_mode == 0) {
2947 			/* update host transfer schedule, so that new transfers can be issued */
2948 			if (dwc_otg_update_host_transfer_schedule_locked(sc))
2949 				dwc_otg_interrupt_poll_locked(sc);
2950 		}
2951 	}
2952 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2953 	USB_BUS_UNLOCK(&sc->sc_bus);
2954 }
2955 
2956 static void
2957 dwc_otg_setup_standard_chain_sub(struct dwc_otg_std_temp *temp)
2958 {
2959 	struct dwc_otg_td *td;
2960 
2961 	/* get current Transfer Descriptor */
2962 	td = temp->td_next;
2963 	temp->td = td;
2964 
2965 	/* prepare for next TD */
2966 	temp->td_next = td->obj_next;
2967 
2968 	/* fill out the Transfer Descriptor */
2969 	td->func = temp->func;
2970 	td->pc = temp->pc;
2971 	td->offset = temp->offset;
2972 	td->remainder = temp->len;
2973 	td->tx_bytes = 0;
2974 	td->error_any = 0;
2975 	td->error_stall = 0;
2976 	td->npkt = 0;
2977 	td->did_stall = temp->did_stall;
2978 	td->short_pkt = temp->short_pkt;
2979 	td->alt_next = temp->setup_alt_next;
2980 	td->set_toggle = 0;
2981 	td->got_short = 0;
2982 	td->did_nak = 0;
2983 	td->channel = DWC_OTG_MAX_CHANNELS;
2984 	td->state = 0;
2985 	td->errcnt = 0;
2986 	td->tt_scheduled = 0;
2987 	td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
2988 }
2989 
2990 static void
2991 dwc_otg_setup_standard_chain(struct usb_xfer *xfer)
2992 {
2993 	struct dwc_otg_std_temp temp;
2994 	struct dwc_otg_td *td;
2995 	uint32_t x;
2996 	uint8_t need_sync;
2997 	uint8_t is_host;
2998 
2999 	DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
3000 	    xfer->address, UE_GET_ADDR(xfer->endpointno),
3001 	    xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
3002 
3003 	temp.max_frame_size = xfer->max_frame_size;
3004 
3005 	td = xfer->td_start[0];
3006 	xfer->td_transfer_first = td;
3007 	xfer->td_transfer_cache = td;
3008 
3009 	/* setup temp */
3010 
3011 	temp.pc = NULL;
3012 	temp.td = NULL;
3013 	temp.td_next = xfer->td_start[0];
3014 	temp.offset = 0;
3015 	temp.setup_alt_next = xfer->flags_int.short_frames_ok ||
3016 	    xfer->flags_int.isochronous_xfr;
3017 	temp.did_stall = !xfer->flags_int.control_stall;
3018 
3019 	is_host = (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST);
3020 
3021 	/* check if we should prepend a setup message */
3022 
3023 	if (xfer->flags_int.control_xfr) {
3024 		if (xfer->flags_int.control_hdr) {
3025 
3026 			if (is_host)
3027 				temp.func = &dwc_otg_host_setup_tx;
3028 			else
3029 				temp.func = &dwc_otg_setup_rx;
3030 
3031 			temp.len = xfer->frlengths[0];
3032 			temp.pc = xfer->frbuffers + 0;
3033 			temp.short_pkt = temp.len ? 1 : 0;
3034 
3035 			/* check for last frame */
3036 			if (xfer->nframes == 1) {
3037 				/* no STATUS stage yet, SETUP is last */
3038 				if (xfer->flags_int.control_act)
3039 					temp.setup_alt_next = 0;
3040 			}
3041 
3042 			dwc_otg_setup_standard_chain_sub(&temp);
3043 		}
3044 		x = 1;
3045 	} else {
3046 		x = 0;
3047 	}
3048 
3049 	if (x != xfer->nframes) {
3050 		if (xfer->endpointno & UE_DIR_IN) {
3051 			if (is_host) {
3052 				temp.func = &dwc_otg_host_data_rx;
3053 				need_sync = 0;
3054 			} else {
3055 				temp.func = &dwc_otg_data_tx;
3056 				need_sync = 1;
3057 			}
3058 		} else {
3059 			if (is_host) {
3060 				temp.func = &dwc_otg_host_data_tx;
3061 				need_sync = 0;
3062 			} else {
3063 				temp.func = &dwc_otg_data_rx;
3064 				need_sync = 0;
3065 			}
3066 		}
3067 
3068 		/* setup "pc" pointer */
3069 		temp.pc = xfer->frbuffers + x;
3070 	} else {
3071 		need_sync = 0;
3072 	}
3073 	while (x != xfer->nframes) {
3074 
3075 		/* DATA0 / DATA1 message */
3076 
3077 		temp.len = xfer->frlengths[x];
3078 
3079 		x++;
3080 
3081 		if (x == xfer->nframes) {
3082 			if (xfer->flags_int.control_xfr) {
3083 				if (xfer->flags_int.control_act) {
3084 					temp.setup_alt_next = 0;
3085 				}
3086 			} else {
3087 				temp.setup_alt_next = 0;
3088 			}
3089 		}
3090 		if (temp.len == 0) {
3091 
3092 			/* make sure that we send an USB packet */
3093 
3094 			temp.short_pkt = 0;
3095 
3096 		} else {
3097 
3098 			/* regular data transfer */
3099 
3100 			temp.short_pkt = (xfer->flags.force_short_xfer ? 0 : 1);
3101 		}
3102 
3103 		dwc_otg_setup_standard_chain_sub(&temp);
3104 
3105 		if (xfer->flags_int.isochronous_xfr) {
3106 			temp.offset += temp.len;
3107 		} else {
3108 			/* get next Page Cache pointer */
3109 			temp.pc = xfer->frbuffers + x;
3110 		}
3111 	}
3112 
3113 	if (xfer->flags_int.control_xfr) {
3114 
3115 		/* always setup a valid "pc" pointer for status and sync */
3116 		temp.pc = xfer->frbuffers + 0;
3117 		temp.len = 0;
3118 		temp.short_pkt = 0;
3119 		temp.setup_alt_next = 0;
3120 
3121 		/* check if we need to sync */
3122 		if (need_sync) {
3123 			/* we need a SYNC point after TX */
3124 			temp.func = &dwc_otg_data_tx_sync;
3125 			dwc_otg_setup_standard_chain_sub(&temp);
3126 		}
3127 
3128 		/* check if we should append a status stage */
3129 		if (!xfer->flags_int.control_act) {
3130 
3131 			/*
3132 			 * Send a DATA1 message and invert the current
3133 			 * endpoint direction.
3134 			 */
3135 			if (xfer->endpointno & UE_DIR_IN) {
3136 				if (is_host) {
3137 					temp.func = &dwc_otg_host_data_tx;
3138 					need_sync = 0;
3139 				} else {
3140 					temp.func = &dwc_otg_data_rx;
3141 					need_sync = 0;
3142 				}
3143 			} else {
3144 				if (is_host) {
3145 					temp.func = &dwc_otg_host_data_rx;
3146 					need_sync = 0;
3147 				} else {
3148 					temp.func = &dwc_otg_data_tx;
3149 					need_sync = 1;
3150 				}
3151 			}
3152 
3153 			dwc_otg_setup_standard_chain_sub(&temp);
3154 
3155 			/* data toggle should be DATA1 */
3156 			td = temp.td;
3157 			td->set_toggle = 1;
3158 
3159 			if (need_sync) {
3160 				/* we need a SYNC point after TX */
3161 				temp.func = &dwc_otg_data_tx_sync;
3162 				dwc_otg_setup_standard_chain_sub(&temp);
3163 			}
3164 		}
3165 	} else {
3166 		/* check if we need to sync */
3167 		if (need_sync) {
3168 
3169 			temp.pc = xfer->frbuffers + 0;
3170 			temp.len = 0;
3171 			temp.short_pkt = 0;
3172 			temp.setup_alt_next = 0;
3173 
3174 			/* we need a SYNC point after TX */
3175 			temp.func = &dwc_otg_data_tx_sync;
3176 			dwc_otg_setup_standard_chain_sub(&temp);
3177 		}
3178 	}
3179 
3180 	/* must have at least one frame! */
3181 	td = temp.td;
3182 	xfer->td_transfer_last = td;
3183 
3184 	if (is_host) {
3185 
3186 		struct dwc_otg_softc *sc;
3187 		uint32_t hcchar;
3188 		uint32_t hcsplt;
3189 
3190 		sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3191 
3192 		/* get first again */
3193 		td = xfer->td_transfer_first;
3194 		td->toggle = (xfer->endpoint->toggle_next ? 1 : 0);
3195 
3196 		hcchar =
3197 			(xfer->address << HCCHAR_DEVADDR_SHIFT) |
3198 			((xfer->endpointno & UE_ADDR) << HCCHAR_EPNUM_SHIFT) |
3199 			(xfer->max_packet_size << HCCHAR_MPS_SHIFT) |
3200 			HCCHAR_CHENA;
3201 
3202 		/*
3203 		 * We are not always able to meet the timing
3204 		 * requirements of the USB interrupt endpoint's
3205 		 * complete split token, when doing transfers going
3206 		 * via a transaction translator. Use the CONTROL
3207 		 * transfer type instead of the INTERRUPT transfer
3208 		 * type in general, as a means to workaround
3209 		 * that. This trick should work for both FULL and LOW
3210 		 * speed USB traffic going through a TT. For non-TT
3211 		 * traffic it works aswell. The reason for using
3212 		 * CONTROL type instead of BULK is that some TTs might
3213 		 * reject LOW speed BULK traffic.
3214 		 */
3215 		if (td->ep_type == UE_INTERRUPT)
3216 			hcchar |= (UE_CONTROL << HCCHAR_EPTYPE_SHIFT);
3217 		else
3218 			hcchar |= (td->ep_type << HCCHAR_EPTYPE_SHIFT);
3219 
3220 		if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_LOW)
3221 			hcchar |= HCCHAR_LSPDDEV;
3222 		if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN)
3223 			hcchar |= HCCHAR_EPDIR_IN;
3224 
3225 		switch (xfer->xroot->udev->speed) {
3226 		case USB_SPEED_FULL:
3227 		case USB_SPEED_LOW:
3228 			/* check if root HUB port is running High Speed */
3229 			if (xfer->xroot->udev->parent_hs_hub != NULL) {
3230 				hcsplt = HCSPLT_SPLTENA |
3231 				    (xfer->xroot->udev->hs_port_no <<
3232 				    HCSPLT_PRTADDR_SHIFT) |
3233 				    (xfer->xroot->udev->hs_hub_addr <<
3234 				    HCSPLT_HUBADDR_SHIFT);
3235 			} else {
3236 				hcsplt = 0;
3237 			}
3238 			if (td->ep_type == UE_INTERRUPT) {
3239 				uint32_t ival;
3240 				ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3241 				if (ival == 0)
3242 					ival = 1;
3243 				else if (ival > 127)
3244 					ival = 127;
3245 				td->tmr_val = sc->sc_tmr_val + ival;
3246 				td->tmr_res = ival;
3247 			} else if (td->ep_type == UE_ISOCHRONOUS) {
3248 				td->tmr_val = 0;
3249 				td->tmr_res = 1;
3250 			} else {
3251 				td->tmr_val = 0;
3252 				td->tmr_res = 0;
3253 			}
3254 			break;
3255 		case USB_SPEED_HIGH:
3256 			hcsplt = 0;
3257 			if (td->ep_type == UE_INTERRUPT) {
3258 				uint32_t ival;
3259 #if 0
3260 				hcchar |= ((xfer->max_packet_count & 3)
3261 				    << HCCHAR_MC_SHIFT);
3262 #endif
3263 				ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3264 				if (ival == 0)
3265 					ival = 1;
3266 				else if (ival > 127)
3267 					ival = 127;
3268 				td->tmr_val = sc->sc_tmr_val + ival;
3269 				td->tmr_res = ival;
3270 			} else if (td->ep_type == UE_ISOCHRONOUS) {
3271 				hcchar |= ((xfer->max_packet_count & 3)
3272 				    << HCCHAR_MC_SHIFT);
3273 				td->tmr_val = 0;
3274 				td->tmr_res = 1 << usbd_xfer_get_fps_shift(xfer);
3275 			} else {
3276 				td->tmr_val = 0;
3277 				td->tmr_res = 0;
3278 			}
3279 			break;
3280 		default:
3281 			hcsplt = 0;
3282 			td->tmr_val = 0;
3283 			td->tmr_res = 0;
3284 			break;
3285 		}
3286 
3287 		/* store configuration in all TD's */
3288 		while (1) {
3289 			td->hcchar = hcchar;
3290 			td->hcsplt = hcsplt;
3291 
3292 			if (((void *)td) == xfer->td_transfer_last)
3293 				break;
3294 
3295 			td = td->obj_next;
3296 		}
3297 	}
3298 }
3299 
3300 static void
3301 dwc_otg_timeout(void *arg)
3302 {
3303 	struct usb_xfer *xfer = arg;
3304 
3305 	DPRINTF("xfer=%p\n", xfer);
3306 
3307 	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
3308 
3309 	/* transfer is transferred */
3310 	dwc_otg_device_done(xfer, USB_ERR_TIMEOUT);
3311 }
3312 
3313 static void
3314 dwc_otg_start_standard_chain(struct usb_xfer *xfer)
3315 {
3316 	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3317 	struct usb_xfer_root *xroot;
3318 	struct dwc_otg_td *td;
3319 
3320 	DPRINTFN(9, "\n");
3321 
3322 	/*
3323 	 * Poll one time in device mode, which will turn on the
3324 	 * endpoint interrupts. Else wait for SOF interrupt in host
3325 	 * mode.
3326 	 */
3327 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
3328 
3329 	if (sc->sc_flags.status_device_mode != 0) {
3330 		dwc_otg_xfer_do_fifo(sc, xfer);
3331 		if (dwc_otg_xfer_do_complete_locked(sc, xfer))
3332 			goto done;
3333 	}
3334 
3335 	/* put transfer on interrupt queue */
3336 	usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
3337 
3338 	/* start timeout, if any */
3339 	if (xfer->timeout != 0) {
3340 		usbd_transfer_timeout_ms(xfer,
3341 		    &dwc_otg_timeout, xfer->timeout);
3342 	}
3343 
3344 	if (sc->sc_flags.status_device_mode != 0)
3345 		goto done;
3346 
3347 	/* enable SOF interrupt, if any */
3348 	dwc_otg_enable_sof_irq(sc);
3349 
3350 	td = xfer->td_transfer_cache;
3351 	if (td->ep_type != UE_BULK)
3352 		goto done;
3353 
3354 	xroot = xfer->xroot;
3355 
3356 	/*
3357 	 * Optimise the ping-pong effect by waking up other BULK
3358 	 * transfers belonging to the same device group:
3359 	 */
3360 	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3361 		td = xfer->td_transfer_cache;
3362 		if (td == NULL || td->ep_type != UE_BULK || xfer->xroot != xroot)
3363 			continue;
3364 		/* reset NAK counter */
3365 		td->did_nak = 0;
3366 	}
3367 done:
3368 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3369 }
3370 
3371 static void
3372 dwc_otg_root_intr(struct dwc_otg_softc *sc)
3373 {
3374 	DPRINTFN(9, "\n");
3375 
3376 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3377 
3378 	/* set port bit */
3379 	sc->sc_hub_idata[0] = 0x02;	/* we only have one port */
3380 
3381 	uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
3382 	    sizeof(sc->sc_hub_idata));
3383 }
3384 
3385 static usb_error_t
3386 dwc_otg_standard_done_sub(struct usb_xfer *xfer)
3387 {
3388 	struct dwc_otg_td *td;
3389 	uint32_t len;
3390 	usb_error_t error;
3391 
3392 	DPRINTFN(9, "\n");
3393 
3394 	td = xfer->td_transfer_cache;
3395 
3396 	do {
3397 		len = td->remainder;
3398 
3399 		/* store last data toggle */
3400 		xfer->endpoint->toggle_next = td->toggle;
3401 
3402 		if (xfer->aframes != xfer->nframes) {
3403 			/*
3404 			 * Verify the length and subtract
3405 			 * the remainder from "frlengths[]":
3406 			 */
3407 			if (len > xfer->frlengths[xfer->aframes]) {
3408 				td->error_any = 1;
3409 			} else {
3410 				xfer->frlengths[xfer->aframes] -= len;
3411 			}
3412 		}
3413 		/* Check for transfer error */
3414 		if (td->error_any) {
3415 			/* the transfer is finished */
3416 			error = (td->error_stall ?
3417 			    USB_ERR_STALLED : USB_ERR_IOERROR);
3418 			td = NULL;
3419 			break;
3420 		}
3421 		/* Check for short transfer */
3422 		if (len > 0) {
3423 			if (xfer->flags_int.short_frames_ok ||
3424 			    xfer->flags_int.isochronous_xfr) {
3425 				/* follow alt next */
3426 				if (td->alt_next) {
3427 					td = td->obj_next;
3428 				} else {
3429 					td = NULL;
3430 				}
3431 			} else {
3432 				/* the transfer is finished */
3433 				td = NULL;
3434 			}
3435 			error = 0;
3436 			break;
3437 		}
3438 		td = td->obj_next;
3439 
3440 		/* this USB frame is complete */
3441 		error = 0;
3442 		break;
3443 
3444 	} while (0);
3445 
3446 	/* update transfer cache */
3447 
3448 	xfer->td_transfer_cache = td;
3449 
3450 	return (error);
3451 }
3452 
3453 static void
3454 dwc_otg_standard_done(struct usb_xfer *xfer)
3455 {
3456 	usb_error_t err = 0;
3457 
3458 	DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
3459 	    xfer, xfer->endpoint);
3460 
3461 	/* reset scanner */
3462 
3463 	xfer->td_transfer_cache = xfer->td_transfer_first;
3464 
3465 	if (xfer->flags_int.control_xfr) {
3466 
3467 		if (xfer->flags_int.control_hdr) {
3468 
3469 			err = dwc_otg_standard_done_sub(xfer);
3470 		}
3471 		xfer->aframes = 1;
3472 
3473 		if (xfer->td_transfer_cache == NULL) {
3474 			goto done;
3475 		}
3476 	}
3477 	while (xfer->aframes != xfer->nframes) {
3478 
3479 		err = dwc_otg_standard_done_sub(xfer);
3480 		xfer->aframes++;
3481 
3482 		if (xfer->td_transfer_cache == NULL) {
3483 			goto done;
3484 		}
3485 	}
3486 
3487 	if (xfer->flags_int.control_xfr &&
3488 	    !xfer->flags_int.control_act) {
3489 
3490 		err = dwc_otg_standard_done_sub(xfer);
3491 	}
3492 done:
3493 	dwc_otg_device_done(xfer, err);
3494 }
3495 
3496 /*------------------------------------------------------------------------*
3497  *	dwc_otg_device_done
3498  *
3499  * NOTE: this function can be called more than one time on the
3500  * same USB transfer!
3501  *------------------------------------------------------------------------*/
3502 static void
3503 dwc_otg_device_done(struct usb_xfer *xfer, usb_error_t error)
3504 {
3505 	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3506 
3507 	DPRINTFN(9, "xfer=%p, endpoint=%p, error=%d\n",
3508 	    xfer, xfer->endpoint, error);
3509 
3510 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
3511 
3512 	if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
3513 		/* Interrupts are cleared by the interrupt handler */
3514 	} else {
3515 		struct dwc_otg_td *td;
3516 
3517 		td = xfer->td_transfer_cache;
3518  		if (td != NULL)
3519 			dwc_otg_host_channel_free(sc, td);
3520 	}
3521 	/* dequeue transfer and start next transfer */
3522 	usbd_transfer_done(xfer, error);
3523 
3524 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3525 }
3526 
3527 static void
3528 dwc_otg_xfer_stall(struct usb_xfer *xfer)
3529 {
3530 	dwc_otg_device_done(xfer, USB_ERR_STALLED);
3531 }
3532 
3533 static void
3534 dwc_otg_set_stall(struct usb_device *udev,
3535     struct usb_endpoint *ep, uint8_t *did_stall)
3536 {
3537 	struct dwc_otg_softc *sc;
3538 	uint32_t temp;
3539 	uint32_t reg;
3540 	uint8_t ep_no;
3541 
3542 	USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3543 
3544 	/* check mode */
3545 	if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3546 		/* not supported */
3547 		return;
3548 	}
3549 
3550 	sc = DWC_OTG_BUS2SC(udev->bus);
3551 
3552 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
3553 
3554 	/* get endpoint address */
3555 	ep_no = ep->edesc->bEndpointAddress;
3556 
3557 	DPRINTFN(5, "endpoint=0x%x\n", ep_no);
3558 
3559 	if (ep_no & UE_DIR_IN) {
3560 		reg = DOTG_DIEPCTL(ep_no & UE_ADDR);
3561 		temp = sc->sc_in_ctl[ep_no & UE_ADDR];
3562 	} else {
3563 		reg = DOTG_DOEPCTL(ep_no & UE_ADDR);
3564 		temp = sc->sc_out_ctl[ep_no & UE_ADDR];
3565 	}
3566 
3567 	/* disable and stall endpoint */
3568 	DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3569 	DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_STALL);
3570 
3571 	/* clear active OUT ep */
3572 	if (!(ep_no & UE_DIR_IN)) {
3573 
3574 		sc->sc_active_rx_ep &= ~(1U << (ep_no & UE_ADDR));
3575 
3576 		if (sc->sc_last_rx_status != 0 &&
3577 		    (ep_no & UE_ADDR) == GRXSTSRD_CHNUM_GET(
3578 		    sc->sc_last_rx_status)) {
3579 			/* dump data */
3580 			dwc_otg_common_rx_ack(sc);
3581 			/* poll interrupt */
3582 			dwc_otg_interrupt_poll_locked(sc);
3583 			dwc_otg_interrupt_complete_locked(sc);
3584 		}
3585 	}
3586 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3587 }
3588 
3589 static void
3590 dwc_otg_clear_stall_sub_locked(struct dwc_otg_softc *sc, uint32_t mps,
3591     uint8_t ep_no, uint8_t ep_type, uint8_t ep_dir)
3592 {
3593 	uint32_t reg;
3594 	uint32_t temp;
3595 
3596 	if (ep_type == UE_CONTROL) {
3597 		/* clearing stall is not needed */
3598 		return;
3599 	}
3600 
3601 	if (ep_dir) {
3602 		reg = DOTG_DIEPCTL(ep_no);
3603 	} else {
3604 		reg = DOTG_DOEPCTL(ep_no);
3605 		sc->sc_active_rx_ep |= (1U << ep_no);
3606 	}
3607 
3608 	/* round up and mask away the multiplier count */
3609 	mps = (mps + 3) & 0x7FC;
3610 
3611 	if (ep_type == UE_BULK) {
3612 		temp = DIEPCTL_EPTYPE_SET(
3613 		    DIEPCTL_EPTYPE_BULK) |
3614 		    DIEPCTL_USBACTEP;
3615 	} else if (ep_type == UE_INTERRUPT) {
3616 		temp = DIEPCTL_EPTYPE_SET(
3617 		    DIEPCTL_EPTYPE_INTERRUPT) |
3618 		    DIEPCTL_USBACTEP;
3619 	} else {
3620 		temp = DIEPCTL_EPTYPE_SET(
3621 		    DIEPCTL_EPTYPE_ISOC) |
3622 		    DIEPCTL_USBACTEP;
3623 	}
3624 
3625 	temp |= DIEPCTL_MPS_SET(mps);
3626 	temp |= DIEPCTL_TXFNUM_SET(ep_no);
3627 
3628 	if (ep_dir)
3629 		sc->sc_in_ctl[ep_no] = temp;
3630 	else
3631 		sc->sc_out_ctl[ep_no] = temp;
3632 
3633 	DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3634 	DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_SETD0PID);
3635 	DWC_OTG_WRITE_4(sc, reg, temp | DIEPCTL_SNAK);
3636 
3637 	/* we only reset the transmit FIFO */
3638 	if (ep_dir) {
3639 		DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
3640 		    GRSTCTL_TXFIFO(ep_no) |
3641 		    GRSTCTL_TXFFLSH);
3642 
3643 		DWC_OTG_WRITE_4(sc,
3644 		    DOTG_DIEPTSIZ(ep_no), 0);
3645 	}
3646 
3647 	/* poll interrupt */
3648 	dwc_otg_interrupt_poll_locked(sc);
3649 	dwc_otg_interrupt_complete_locked(sc);
3650 }
3651 
3652 static void
3653 dwc_otg_clear_stall(struct usb_device *udev, struct usb_endpoint *ep)
3654 {
3655 	struct dwc_otg_softc *sc;
3656 	struct usb_endpoint_descriptor *ed;
3657 
3658 	DPRINTFN(5, "endpoint=%p\n", ep);
3659 
3660 	USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3661 
3662 	/* check mode */
3663 	if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3664 		/* not supported */
3665 		return;
3666 	}
3667 	/* get softc */
3668 	sc = DWC_OTG_BUS2SC(udev->bus);
3669 
3670 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
3671 
3672 	/* get endpoint descriptor */
3673 	ed = ep->edesc;
3674 
3675 	/* reset endpoint */
3676 	dwc_otg_clear_stall_sub_locked(sc,
3677 	    UGETW(ed->wMaxPacketSize),
3678 	    (ed->bEndpointAddress & UE_ADDR),
3679 	    (ed->bmAttributes & UE_XFERTYPE),
3680 	    (ed->bEndpointAddress & (UE_DIR_IN | UE_DIR_OUT)));
3681 
3682 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3683 }
3684 
3685 static void
3686 dwc_otg_device_state_change(struct usb_device *udev)
3687 {
3688 	struct dwc_otg_softc *sc;
3689 	uint8_t x;
3690 
3691 	/* check mode */
3692 	if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3693 		/* not supported */
3694 		return;
3695 	}
3696 
3697 	/* get softc */
3698 	sc = DWC_OTG_BUS2SC(udev->bus);
3699 
3700 	/* deactivate all other endpoint but the control endpoint */
3701 	if (udev->state == USB_STATE_CONFIGURED ||
3702 	    udev->state == USB_STATE_ADDRESSED) {
3703 
3704 		USB_BUS_LOCK(&sc->sc_bus);
3705 
3706 		for (x = 1; x != sc->sc_dev_ep_max; x++) {
3707 
3708 			if (x < sc->sc_dev_in_ep_max) {
3709 				DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x),
3710 				    DIEPCTL_EPDIS);
3711 				DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x), 0);
3712 			}
3713 
3714 			DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x),
3715 			    DOEPCTL_EPDIS);
3716 			DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x), 0);
3717 		}
3718 		USB_BUS_UNLOCK(&sc->sc_bus);
3719 	}
3720 }
3721 
3722 int
3723 dwc_otg_init(struct dwc_otg_softc *sc)
3724 {
3725 	uint32_t temp;
3726 
3727 	DPRINTF("start\n");
3728 
3729 	/* set up the bus structure */
3730 	sc->sc_bus.usbrev = USB_REV_2_0;
3731 	sc->sc_bus.methods = &dwc_otg_bus_methods;
3732 
3733 	usb_callout_init_mtx(&sc->sc_timer,
3734 	    &sc->sc_bus.bus_mtx, 0);
3735 
3736 	USB_BUS_LOCK(&sc->sc_bus);
3737 
3738 	/* turn on clocks */
3739 	dwc_otg_clocks_on(sc);
3740 
3741 	temp = DWC_OTG_READ_4(sc, DOTG_GSNPSID);
3742 	DPRINTF("Version = 0x%08x\n", temp);
3743 
3744 	/* disconnect */
3745 	DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3746 	    DCTL_SFTDISCON);
3747 
3748 	/* wait for host to detect disconnect */
3749 	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 32);
3750 
3751 	DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
3752 	    GRSTCTL_CSFTRST);
3753 
3754 	/* wait a little bit for block to reset */
3755 	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 128);
3756 
3757 	switch (sc->sc_mode) {
3758 	case DWC_MODE_DEVICE:
3759 		temp = GUSBCFG_FORCEDEVMODE;
3760 		break;
3761 	case DWC_MODE_HOST:
3762 		temp = GUSBCFG_FORCEHOSTMODE;
3763 		break;
3764 	default:
3765 		temp = 0;
3766 		break;
3767 	}
3768 
3769 	/* select HSIC or non-HSIC mode */
3770 	if (dwc_otg_use_hsic) {
3771 		DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3772 		    GUSBCFG_PHYIF |
3773 		    GUSBCFG_TRD_TIM_SET(5) | temp);
3774 		DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL,
3775 		    0x000000EC);
3776 
3777 		temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3778 		DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3779 		    temp & ~GLPMCFG_HSIC_CONN);
3780 		DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3781 		    temp | GLPMCFG_HSIC_CONN);
3782 	} else {
3783 		DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3784 		    GUSBCFG_ULPI_UTMI_SEL |
3785 		    GUSBCFG_TRD_TIM_SET(5) | temp);
3786 		DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3787 
3788 		temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3789 		DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3790 		    temp & ~GLPMCFG_HSIC_CONN);
3791 	}
3792 
3793 	/* clear global nak */
3794 	DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3795 	    DCTL_CGOUTNAK |
3796 	    DCTL_CGNPINNAK);
3797 
3798 	/* disable USB port */
3799 	DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0xFFFFFFFF);
3800 
3801 	/* wait 10ms */
3802 	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3803 
3804 	/* enable USB port */
3805 	DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
3806 
3807 	/* wait 10ms */
3808 	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3809 
3810 	/* pull up D+ */
3811 	dwc_otg_pull_up(sc);
3812 
3813 	temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG3);
3814 
3815 	sc->sc_fifo_size = 4 * GHWCFG3_DFIFODEPTH_GET(temp);
3816 
3817 	temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
3818 
3819 	sc->sc_dev_ep_max = GHWCFG2_NUMDEVEPS_GET(temp);
3820 
3821 	if (sc->sc_dev_ep_max > DWC_OTG_MAX_ENDPOINTS)
3822 		sc->sc_dev_ep_max = DWC_OTG_MAX_ENDPOINTS;
3823 
3824 	sc->sc_host_ch_max = GHWCFG2_NUMHSTCHNL_GET(temp);
3825 
3826 	if (sc->sc_host_ch_max > DWC_OTG_MAX_CHANNELS)
3827 		sc->sc_host_ch_max = DWC_OTG_MAX_CHANNELS;
3828 
3829 	temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG4);
3830 
3831 	sc->sc_dev_in_ep_max = GHWCFG4_NUM_IN_EP_GET(temp);
3832 
3833 	DPRINTF("Total FIFO size = %d bytes, Device EPs = %d/%d Host CHs = %d\n",
3834 	    sc->sc_fifo_size, sc->sc_dev_ep_max, sc->sc_dev_in_ep_max,
3835 	    sc->sc_host_ch_max);
3836 
3837 	/* setup FIFO */
3838 	if (dwc_otg_init_fifo(sc, sc->sc_mode)) {
3839 		USB_BUS_UNLOCK(&sc->sc_bus);
3840 		return (EINVAL);
3841 	}
3842 
3843 	/* enable interrupts */
3844 	sc->sc_irq_mask = DWC_OTG_MSK_GINT_ENABLED;
3845 	DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
3846 
3847 	if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_DEVICE) {
3848 
3849 		/* enable all endpoint interrupts */
3850 		temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
3851 		if (temp & GHWCFG2_MPI) {
3852 			uint8_t x;
3853 
3854 			DPRINTF("Multi Process Interrupts\n");
3855 
3856 			for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
3857 				DWC_OTG_WRITE_4(sc, DOTG_DIEPEACHINTMSK(x),
3858 				    DIEPMSK_XFERCOMPLMSK);
3859 				DWC_OTG_WRITE_4(sc, DOTG_DOEPEACHINTMSK(x), 0);
3860 			}
3861 			DWC_OTG_WRITE_4(sc, DOTG_DEACHINTMSK, 0xFFFF);
3862 		} else {
3863 			DWC_OTG_WRITE_4(sc, DOTG_DIEPMSK,
3864 			    DIEPMSK_XFERCOMPLMSK);
3865 			DWC_OTG_WRITE_4(sc, DOTG_DOEPMSK, 0);
3866 			DWC_OTG_WRITE_4(sc, DOTG_DAINTMSK, 0xFFFF);
3867 		}
3868 	}
3869 
3870 	if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_HOST) {
3871 		/* setup clocks */
3872 		temp = DWC_OTG_READ_4(sc, DOTG_HCFG);
3873 		temp &= ~(HCFG_FSLSSUPP | HCFG_FSLSPCLKSEL_MASK);
3874 		temp |= (1 << HCFG_FSLSPCLKSEL_SHIFT);
3875 		DWC_OTG_WRITE_4(sc, DOTG_HCFG, temp);
3876 	}
3877 
3878 	/* only enable global IRQ */
3879 	DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG,
3880 	    GAHBCFG_GLBLINTRMSK);
3881 
3882 	/* turn off clocks */
3883 	dwc_otg_clocks_off(sc);
3884 
3885 	/* read initial VBUS state */
3886 
3887 	temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
3888 
3889 	DPRINTFN(5, "GOTCTL=0x%08x\n", temp);
3890 
3891 	dwc_otg_vbus_interrupt(sc,
3892 	    (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
3893 
3894 	USB_BUS_UNLOCK(&sc->sc_bus);
3895 
3896 	/* catch any lost interrupts */
3897 
3898 	dwc_otg_do_poll(&sc->sc_bus);
3899 
3900 	return (0);			/* success */
3901 }
3902 
3903 void
3904 dwc_otg_uninit(struct dwc_otg_softc *sc)
3905 {
3906 	USB_BUS_LOCK(&sc->sc_bus);
3907 
3908 	/* stop host timer */
3909 	dwc_otg_timer_stop(sc);
3910 
3911 	/* set disconnect */
3912 	DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3913 	    DCTL_SFTDISCON);
3914 
3915 	/* turn off global IRQ */
3916 	DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG, 0);
3917 
3918 	sc->sc_flags.port_enabled = 0;
3919 	sc->sc_flags.port_powered = 0;
3920 	sc->sc_flags.status_vbus = 0;
3921 	sc->sc_flags.status_bus_reset = 0;
3922 	sc->sc_flags.status_suspend = 0;
3923 	sc->sc_flags.change_suspend = 0;
3924 	sc->sc_flags.change_connect = 1;
3925 
3926 	dwc_otg_pull_down(sc);
3927 	dwc_otg_clocks_off(sc);
3928 
3929 	USB_BUS_UNLOCK(&sc->sc_bus);
3930 
3931 	usb_callout_drain(&sc->sc_timer);
3932 }
3933 
3934 static void
3935 dwc_otg_suspend(struct dwc_otg_softc *sc)
3936 {
3937 	return;
3938 }
3939 
3940 static void
3941 dwc_otg_resume(struct dwc_otg_softc *sc)
3942 {
3943 	return;
3944 }
3945 
3946 static void
3947 dwc_otg_do_poll(struct usb_bus *bus)
3948 {
3949 	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
3950 
3951 	USB_BUS_LOCK(&sc->sc_bus);
3952 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
3953 	dwc_otg_interrupt_poll_locked(sc);
3954 	dwc_otg_interrupt_complete_locked(sc);
3955 	if (sc->sc_flags.status_device_mode == 0) {
3956 		/* update host transfer schedule, so that new transfers can be issued */
3957 		if (dwc_otg_update_host_transfer_schedule_locked(sc))
3958 			dwc_otg_interrupt_poll_locked(sc);
3959 	}
3960 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3961 	USB_BUS_UNLOCK(&sc->sc_bus);
3962 }
3963 
3964 /*------------------------------------------------------------------------*
3965  * DWC OTG bulk support
3966  * DWC OTG control support
3967  * DWC OTG interrupt support
3968  *------------------------------------------------------------------------*/
3969 static void
3970 dwc_otg_device_non_isoc_open(struct usb_xfer *xfer)
3971 {
3972 }
3973 
3974 static void
3975 dwc_otg_device_non_isoc_close(struct usb_xfer *xfer)
3976 {
3977 	dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
3978 }
3979 
3980 static void
3981 dwc_otg_device_non_isoc_enter(struct usb_xfer *xfer)
3982 {
3983 }
3984 
3985 static void
3986 dwc_otg_device_non_isoc_start(struct usb_xfer *xfer)
3987 {
3988 	/* setup TDs */
3989 	dwc_otg_setup_standard_chain(xfer);
3990 	dwc_otg_start_standard_chain(xfer);
3991 }
3992 
3993 static const struct usb_pipe_methods dwc_otg_device_non_isoc_methods =
3994 {
3995 	.open = dwc_otg_device_non_isoc_open,
3996 	.close = dwc_otg_device_non_isoc_close,
3997 	.enter = dwc_otg_device_non_isoc_enter,
3998 	.start = dwc_otg_device_non_isoc_start,
3999 };
4000 
4001 /*------------------------------------------------------------------------*
4002  * DWC OTG full speed isochronous support
4003  *------------------------------------------------------------------------*/
4004 static void
4005 dwc_otg_device_isoc_open(struct usb_xfer *xfer)
4006 {
4007 }
4008 
4009 static void
4010 dwc_otg_device_isoc_close(struct usb_xfer *xfer)
4011 {
4012 	dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
4013 }
4014 
4015 static void
4016 dwc_otg_device_isoc_enter(struct usb_xfer *xfer)
4017 {
4018 }
4019 
4020 static void
4021 dwc_otg_device_isoc_start(struct usb_xfer *xfer)
4022 {
4023 	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
4024 	uint32_t temp;
4025 	uint32_t msframes;
4026 	uint32_t framenum;
4027 	uint8_t shift = usbd_xfer_get_fps_shift(xfer);
4028 
4029 	DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
4030 	    xfer, xfer->endpoint->isoc_next, xfer->nframes);
4031 
4032 	if (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST) {
4033 		temp = DWC_OTG_READ_4(sc, DOTG_HFNUM);
4034 
4035 		/* get the current frame index */
4036 		framenum = (temp & HFNUM_FRNUM_MASK);
4037 	} else {
4038 		temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
4039 
4040 		/* get the current frame index */
4041 		framenum = DSTS_SOFFN_GET(temp);
4042 	}
4043 
4044 	if (xfer->xroot->udev->parent_hs_hub != NULL)
4045 		framenum /= 8;
4046 
4047 	framenum &= DWC_OTG_FRAME_MASK;
4048 
4049 	/*
4050 	 * Compute number of milliseconds worth of data traffic for
4051 	 * this USB transfer:
4052 	 */
4053 	if (xfer->xroot->udev->speed == USB_SPEED_HIGH)
4054 		msframes = ((xfer->nframes << shift) + 7) / 8;
4055 	else
4056 		msframes = xfer->nframes;
4057 
4058 	/*
4059 	 * check if the frame index is within the window where the frames
4060 	 * will be inserted
4061 	 */
4062 	temp = (framenum - xfer->endpoint->isoc_next) & DWC_OTG_FRAME_MASK;
4063 
4064 	if ((xfer->endpoint->is_synced == 0) || (temp < msframes)) {
4065 		/*
4066 		 * If there is data underflow or the pipe queue is
4067 		 * empty we schedule the transfer a few frames ahead
4068 		 * of the current frame position. Else two isochronous
4069 		 * transfers might overlap.
4070 		 */
4071 		xfer->endpoint->isoc_next = (framenum + 3) & DWC_OTG_FRAME_MASK;
4072 		xfer->endpoint->is_synced = 1;
4073 		DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
4074 	}
4075 	/*
4076 	 * compute how many milliseconds the insertion is ahead of the
4077 	 * current frame position:
4078 	 */
4079 	temp = (xfer->endpoint->isoc_next - framenum) & DWC_OTG_FRAME_MASK;
4080 
4081 	/*
4082 	 * pre-compute when the isochronous transfer will be finished:
4083 	 */
4084 	xfer->isoc_time_complete =
4085 		usb_isoc_time_expand(&sc->sc_bus, framenum) + temp + msframes;
4086 
4087 	/* setup TDs */
4088 	dwc_otg_setup_standard_chain(xfer);
4089 
4090 	/* compute frame number for next insertion */
4091 	xfer->endpoint->isoc_next += msframes;
4092 
4093 	/* start TD chain */
4094 	dwc_otg_start_standard_chain(xfer);
4095 }
4096 
4097 static const struct usb_pipe_methods dwc_otg_device_isoc_methods =
4098 {
4099 	.open = dwc_otg_device_isoc_open,
4100 	.close = dwc_otg_device_isoc_close,
4101 	.enter = dwc_otg_device_isoc_enter,
4102 	.start = dwc_otg_device_isoc_start,
4103 };
4104 
4105 /*------------------------------------------------------------------------*
4106  * DWC OTG root control support
4107  *------------------------------------------------------------------------*
4108  * Simulate a hardware HUB by handling all the necessary requests.
4109  *------------------------------------------------------------------------*/
4110 
4111 static const struct usb_device_descriptor dwc_otg_devd = {
4112 	.bLength = sizeof(struct usb_device_descriptor),
4113 	.bDescriptorType = UDESC_DEVICE,
4114 	.bcdUSB = {0x00, 0x02},
4115 	.bDeviceClass = UDCLASS_HUB,
4116 	.bDeviceSubClass = UDSUBCLASS_HUB,
4117 	.bDeviceProtocol = UDPROTO_HSHUBSTT,
4118 	.bMaxPacketSize = 64,
4119 	.bcdDevice = {0x00, 0x01},
4120 	.iManufacturer = 1,
4121 	.iProduct = 2,
4122 	.bNumConfigurations = 1,
4123 };
4124 
4125 static const struct dwc_otg_config_desc dwc_otg_confd = {
4126 	.confd = {
4127 		.bLength = sizeof(struct usb_config_descriptor),
4128 		.bDescriptorType = UDESC_CONFIG,
4129 		.wTotalLength[0] = sizeof(dwc_otg_confd),
4130 		.bNumInterface = 1,
4131 		.bConfigurationValue = 1,
4132 		.iConfiguration = 0,
4133 		.bmAttributes = UC_SELF_POWERED,
4134 		.bMaxPower = 0,
4135 	},
4136 	.ifcd = {
4137 		.bLength = sizeof(struct usb_interface_descriptor),
4138 		.bDescriptorType = UDESC_INTERFACE,
4139 		.bNumEndpoints = 1,
4140 		.bInterfaceClass = UICLASS_HUB,
4141 		.bInterfaceSubClass = UISUBCLASS_HUB,
4142 		.bInterfaceProtocol = 0,
4143 	},
4144 	.endpd = {
4145 		.bLength = sizeof(struct usb_endpoint_descriptor),
4146 		.bDescriptorType = UDESC_ENDPOINT,
4147 		.bEndpointAddress = (UE_DIR_IN | DWC_OTG_INTR_ENDPT),
4148 		.bmAttributes = UE_INTERRUPT,
4149 		.wMaxPacketSize[0] = 8,
4150 		.bInterval = 255,
4151 	},
4152 };
4153 
4154 #define	HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) }
4155 
4156 static const struct usb_hub_descriptor_min dwc_otg_hubd = {
4157 	.bDescLength = sizeof(dwc_otg_hubd),
4158 	.bDescriptorType = UDESC_HUB,
4159 	.bNbrPorts = 1,
4160 	HSETW(.wHubCharacteristics, (UHD_PWR_NO_SWITCH | UHD_OC_INDIVIDUAL)),
4161 	.bPwrOn2PwrGood = 50,
4162 	.bHubContrCurrent = 0,
4163 	.DeviceRemovable = {0},		/* port is removable */
4164 };
4165 
4166 #define	STRING_VENDOR \
4167   "D\0W\0C\0O\0T\0G"
4168 
4169 #define	STRING_PRODUCT \
4170   "O\0T\0G\0 \0R\0o\0o\0t\0 \0H\0U\0B"
4171 
4172 USB_MAKE_STRING_DESC(STRING_VENDOR, dwc_otg_vendor);
4173 USB_MAKE_STRING_DESC(STRING_PRODUCT, dwc_otg_product);
4174 
4175 static usb_error_t
4176 dwc_otg_roothub_exec(struct usb_device *udev,
4177     struct usb_device_request *req, const void **pptr, uint16_t *plength)
4178 {
4179 	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4180 	const void *ptr;
4181 	uint16_t len;
4182 	uint16_t value;
4183 	uint16_t index;
4184 	usb_error_t err;
4185 
4186 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
4187 
4188 	/* buffer reset */
4189 	ptr = (const void *)&sc->sc_hub_temp;
4190 	len = 0;
4191 	err = 0;
4192 
4193 	value = UGETW(req->wValue);
4194 	index = UGETW(req->wIndex);
4195 
4196 	/* demultiplex the control request */
4197 
4198 	switch (req->bmRequestType) {
4199 	case UT_READ_DEVICE:
4200 		switch (req->bRequest) {
4201 		case UR_GET_DESCRIPTOR:
4202 			goto tr_handle_get_descriptor;
4203 		case UR_GET_CONFIG:
4204 			goto tr_handle_get_config;
4205 		case UR_GET_STATUS:
4206 			goto tr_handle_get_status;
4207 		default:
4208 			goto tr_stalled;
4209 		}
4210 		break;
4211 
4212 	case UT_WRITE_DEVICE:
4213 		switch (req->bRequest) {
4214 		case UR_SET_ADDRESS:
4215 			goto tr_handle_set_address;
4216 		case UR_SET_CONFIG:
4217 			goto tr_handle_set_config;
4218 		case UR_CLEAR_FEATURE:
4219 			goto tr_valid;	/* nop */
4220 		case UR_SET_DESCRIPTOR:
4221 			goto tr_valid;	/* nop */
4222 		case UR_SET_FEATURE:
4223 		default:
4224 			goto tr_stalled;
4225 		}
4226 		break;
4227 
4228 	case UT_WRITE_ENDPOINT:
4229 		switch (req->bRequest) {
4230 		case UR_CLEAR_FEATURE:
4231 			switch (UGETW(req->wValue)) {
4232 			case UF_ENDPOINT_HALT:
4233 				goto tr_handle_clear_halt;
4234 			case UF_DEVICE_REMOTE_WAKEUP:
4235 				goto tr_handle_clear_wakeup;
4236 			default:
4237 				goto tr_stalled;
4238 			}
4239 			break;
4240 		case UR_SET_FEATURE:
4241 			switch (UGETW(req->wValue)) {
4242 			case UF_ENDPOINT_HALT:
4243 				goto tr_handle_set_halt;
4244 			case UF_DEVICE_REMOTE_WAKEUP:
4245 				goto tr_handle_set_wakeup;
4246 			default:
4247 				goto tr_stalled;
4248 			}
4249 			break;
4250 		case UR_SYNCH_FRAME:
4251 			goto tr_valid;	/* nop */
4252 		default:
4253 			goto tr_stalled;
4254 		}
4255 		break;
4256 
4257 	case UT_READ_ENDPOINT:
4258 		switch (req->bRequest) {
4259 		case UR_GET_STATUS:
4260 			goto tr_handle_get_ep_status;
4261 		default:
4262 			goto tr_stalled;
4263 		}
4264 		break;
4265 
4266 	case UT_WRITE_INTERFACE:
4267 		switch (req->bRequest) {
4268 		case UR_SET_INTERFACE:
4269 			goto tr_handle_set_interface;
4270 		case UR_CLEAR_FEATURE:
4271 			goto tr_valid;	/* nop */
4272 		case UR_SET_FEATURE:
4273 		default:
4274 			goto tr_stalled;
4275 		}
4276 		break;
4277 
4278 	case UT_READ_INTERFACE:
4279 		switch (req->bRequest) {
4280 		case UR_GET_INTERFACE:
4281 			goto tr_handle_get_interface;
4282 		case UR_GET_STATUS:
4283 			goto tr_handle_get_iface_status;
4284 		default:
4285 			goto tr_stalled;
4286 		}
4287 		break;
4288 
4289 	case UT_WRITE_CLASS_INTERFACE:
4290 	case UT_WRITE_VENDOR_INTERFACE:
4291 		/* XXX forward */
4292 		break;
4293 
4294 	case UT_READ_CLASS_INTERFACE:
4295 	case UT_READ_VENDOR_INTERFACE:
4296 		/* XXX forward */
4297 		break;
4298 
4299 	case UT_WRITE_CLASS_DEVICE:
4300 		switch (req->bRequest) {
4301 		case UR_CLEAR_FEATURE:
4302 			goto tr_valid;
4303 		case UR_SET_DESCRIPTOR:
4304 		case UR_SET_FEATURE:
4305 			break;
4306 		default:
4307 			goto tr_stalled;
4308 		}
4309 		break;
4310 
4311 	case UT_WRITE_CLASS_OTHER:
4312 		switch (req->bRequest) {
4313 		case UR_CLEAR_FEATURE:
4314 			goto tr_handle_clear_port_feature;
4315 		case UR_SET_FEATURE:
4316 			goto tr_handle_set_port_feature;
4317 		case UR_CLEAR_TT_BUFFER:
4318 		case UR_RESET_TT:
4319 		case UR_STOP_TT:
4320 			goto tr_valid;
4321 
4322 		default:
4323 			goto tr_stalled;
4324 		}
4325 		break;
4326 
4327 	case UT_READ_CLASS_OTHER:
4328 		switch (req->bRequest) {
4329 		case UR_GET_TT_STATE:
4330 			goto tr_handle_get_tt_state;
4331 		case UR_GET_STATUS:
4332 			goto tr_handle_get_port_status;
4333 		default:
4334 			goto tr_stalled;
4335 		}
4336 		break;
4337 
4338 	case UT_READ_CLASS_DEVICE:
4339 		switch (req->bRequest) {
4340 		case UR_GET_DESCRIPTOR:
4341 			goto tr_handle_get_class_descriptor;
4342 		case UR_GET_STATUS:
4343 			goto tr_handle_get_class_status;
4344 
4345 		default:
4346 			goto tr_stalled;
4347 		}
4348 		break;
4349 	default:
4350 		goto tr_stalled;
4351 	}
4352 	goto tr_valid;
4353 
4354 tr_handle_get_descriptor:
4355 	switch (value >> 8) {
4356 	case UDESC_DEVICE:
4357 		if (value & 0xff) {
4358 			goto tr_stalled;
4359 		}
4360 		len = sizeof(dwc_otg_devd);
4361 		ptr = (const void *)&dwc_otg_devd;
4362 		goto tr_valid;
4363 	case UDESC_CONFIG:
4364 		if (value & 0xff) {
4365 			goto tr_stalled;
4366 		}
4367 		len = sizeof(dwc_otg_confd);
4368 		ptr = (const void *)&dwc_otg_confd;
4369 		goto tr_valid;
4370 	case UDESC_STRING:
4371 		switch (value & 0xff) {
4372 		case 0:		/* Language table */
4373 			len = sizeof(usb_string_lang_en);
4374 			ptr = (const void *)&usb_string_lang_en;
4375 			goto tr_valid;
4376 
4377 		case 1:		/* Vendor */
4378 			len = sizeof(dwc_otg_vendor);
4379 			ptr = (const void *)&dwc_otg_vendor;
4380 			goto tr_valid;
4381 
4382 		case 2:		/* Product */
4383 			len = sizeof(dwc_otg_product);
4384 			ptr = (const void *)&dwc_otg_product;
4385 			goto tr_valid;
4386 		default:
4387 			break;
4388 		}
4389 		break;
4390 	default:
4391 		goto tr_stalled;
4392 	}
4393 	goto tr_stalled;
4394 
4395 tr_handle_get_config:
4396 	len = 1;
4397 	sc->sc_hub_temp.wValue[0] = sc->sc_conf;
4398 	goto tr_valid;
4399 
4400 tr_handle_get_status:
4401 	len = 2;
4402 	USETW(sc->sc_hub_temp.wValue, UDS_SELF_POWERED);
4403 	goto tr_valid;
4404 
4405 tr_handle_set_address:
4406 	if (value & 0xFF00) {
4407 		goto tr_stalled;
4408 	}
4409 	sc->sc_rt_addr = value;
4410 	goto tr_valid;
4411 
4412 tr_handle_set_config:
4413 	if (value >= 2) {
4414 		goto tr_stalled;
4415 	}
4416 	sc->sc_conf = value;
4417 	goto tr_valid;
4418 
4419 tr_handle_get_interface:
4420 	len = 1;
4421 	sc->sc_hub_temp.wValue[0] = 0;
4422 	goto tr_valid;
4423 
4424 tr_handle_get_tt_state:
4425 tr_handle_get_class_status:
4426 tr_handle_get_iface_status:
4427 tr_handle_get_ep_status:
4428 	len = 2;
4429 	USETW(sc->sc_hub_temp.wValue, 0);
4430 	goto tr_valid;
4431 
4432 tr_handle_set_halt:
4433 tr_handle_set_interface:
4434 tr_handle_set_wakeup:
4435 tr_handle_clear_wakeup:
4436 tr_handle_clear_halt:
4437 	goto tr_valid;
4438 
4439 tr_handle_clear_port_feature:
4440 	if (index != 1)
4441 		goto tr_stalled;
4442 
4443 	DPRINTFN(9, "UR_CLEAR_PORT_FEATURE on port %d\n", index);
4444 
4445 	switch (value) {
4446 	case UHF_PORT_SUSPEND:
4447 		dwc_otg_wakeup_peer(sc);
4448 		break;
4449 
4450 	case UHF_PORT_ENABLE:
4451 		if (sc->sc_flags.status_device_mode == 0) {
4452 			DWC_OTG_WRITE_4(sc, DOTG_HPRT,
4453 			    sc->sc_hprt_val | HPRT_PRTENA);
4454 		}
4455 		sc->sc_flags.port_enabled = 0;
4456 		break;
4457 
4458 	case UHF_C_PORT_RESET:
4459 		sc->sc_flags.change_reset = 0;
4460 		break;
4461 
4462 	case UHF_C_PORT_ENABLE:
4463 		sc->sc_flags.change_enabled = 0;
4464 		break;
4465 
4466 	case UHF_C_PORT_OVER_CURRENT:
4467 		sc->sc_flags.change_over_current = 0;
4468 		break;
4469 
4470 	case UHF_PORT_TEST:
4471 	case UHF_PORT_INDICATOR:
4472 		/* nops */
4473 		break;
4474 
4475 	case UHF_PORT_POWER:
4476 		sc->sc_flags.port_powered = 0;
4477 		if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4478 			sc->sc_hprt_val = 0;
4479 			DWC_OTG_WRITE_4(sc, DOTG_HPRT, HPRT_PRTENA);
4480 		}
4481 		dwc_otg_pull_down(sc);
4482 		dwc_otg_clocks_off(sc);
4483 		break;
4484 
4485 	case UHF_C_PORT_CONNECTION:
4486 		/* clear connect change flag */
4487 		sc->sc_flags.change_connect = 0;
4488 		break;
4489 
4490 	case UHF_C_PORT_SUSPEND:
4491 		sc->sc_flags.change_suspend = 0;
4492 		break;
4493 
4494 	default:
4495 		err = USB_ERR_IOERROR;
4496 		goto done;
4497 	}
4498 	goto tr_valid;
4499 
4500 tr_handle_set_port_feature:
4501 	if (index != 1) {
4502 		goto tr_stalled;
4503 	}
4504 	DPRINTFN(9, "UR_SET_PORT_FEATURE\n");
4505 
4506 	switch (value) {
4507 	case UHF_PORT_ENABLE:
4508 		break;
4509 
4510 	case UHF_PORT_SUSPEND:
4511 		if (sc->sc_flags.status_device_mode == 0) {
4512 			/* set suspend BIT */
4513 			sc->sc_hprt_val |= HPRT_PRTSUSP;
4514 			DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4515 
4516 			/* generate HUB suspend event */
4517 			dwc_otg_suspend_irq(sc);
4518 		}
4519 		break;
4520 
4521 	case UHF_PORT_RESET:
4522 		if (sc->sc_flags.status_device_mode == 0) {
4523 
4524 			DPRINTF("PORT RESET\n");
4525 
4526 			/* enable PORT reset */
4527 			DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val | HPRT_PRTRST);
4528 
4529 			/* Wait 62.5ms for reset to complete */
4530 			usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4531 
4532 			DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4533 
4534 			/* Wait 62.5ms for reset to complete */
4535 			usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4536 
4537 			/* reset FIFOs */
4538 			(void) dwc_otg_init_fifo(sc, DWC_MODE_HOST);
4539 
4540 			sc->sc_flags.change_reset = 1;
4541 		} else {
4542 			err = USB_ERR_IOERROR;
4543 		}
4544 		break;
4545 
4546 	case UHF_PORT_TEST:
4547 	case UHF_PORT_INDICATOR:
4548 		/* nops */
4549 		break;
4550 	case UHF_PORT_POWER:
4551 		if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4552 			sc->sc_hprt_val |= HPRT_PRTPWR;
4553 			DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4554 		}
4555 		sc->sc_flags.port_powered = 1;
4556 		break;
4557 	default:
4558 		err = USB_ERR_IOERROR;
4559 		goto done;
4560 	}
4561 	goto tr_valid;
4562 
4563 tr_handle_get_port_status:
4564 
4565 	DPRINTFN(9, "UR_GET_PORT_STATUS\n");
4566 
4567 	if (index != 1)
4568 		goto tr_stalled;
4569 
4570 	if (sc->sc_flags.status_vbus)
4571 		dwc_otg_clocks_on(sc);
4572 	else
4573 		dwc_otg_clocks_off(sc);
4574 
4575 	/* Select Device Side Mode */
4576 
4577 	if (sc->sc_flags.status_device_mode) {
4578 		value = UPS_PORT_MODE_DEVICE;
4579 		dwc_otg_timer_stop(sc);
4580 	} else {
4581 		value = 0;
4582 		dwc_otg_timer_start(sc);
4583 	}
4584 
4585 	if (sc->sc_flags.status_high_speed)
4586 		value |= UPS_HIGH_SPEED;
4587 	else if (sc->sc_flags.status_low_speed)
4588 		value |= UPS_LOW_SPEED;
4589 
4590 	if (sc->sc_flags.port_powered)
4591 		value |= UPS_PORT_POWER;
4592 
4593 	if (sc->sc_flags.port_enabled)
4594 		value |= UPS_PORT_ENABLED;
4595 
4596 	if (sc->sc_flags.port_over_current)
4597 		value |= UPS_OVERCURRENT_INDICATOR;
4598 
4599 	if (sc->sc_flags.status_vbus &&
4600 	    sc->sc_flags.status_bus_reset)
4601 		value |= UPS_CURRENT_CONNECT_STATUS;
4602 
4603 	if (sc->sc_flags.status_suspend)
4604 		value |= UPS_SUSPEND;
4605 
4606 	USETW(sc->sc_hub_temp.ps.wPortStatus, value);
4607 
4608 	value = 0;
4609 
4610 	if (sc->sc_flags.change_connect)
4611 		value |= UPS_C_CONNECT_STATUS;
4612 	if (sc->sc_flags.change_suspend)
4613 		value |= UPS_C_SUSPEND;
4614 	if (sc->sc_flags.change_reset)
4615 		value |= UPS_C_PORT_RESET;
4616 	if (sc->sc_flags.change_over_current)
4617 		value |= UPS_C_OVERCURRENT_INDICATOR;
4618 
4619 	USETW(sc->sc_hub_temp.ps.wPortChange, value);
4620 	len = sizeof(sc->sc_hub_temp.ps);
4621 	goto tr_valid;
4622 
4623 tr_handle_get_class_descriptor:
4624 	if (value & 0xFF) {
4625 		goto tr_stalled;
4626 	}
4627 	ptr = (const void *)&dwc_otg_hubd;
4628 	len = sizeof(dwc_otg_hubd);
4629 	goto tr_valid;
4630 
4631 tr_stalled:
4632 	err = USB_ERR_STALLED;
4633 tr_valid:
4634 done:
4635 	*plength = len;
4636 	*pptr = ptr;
4637 	return (err);
4638 }
4639 
4640 static void
4641 dwc_otg_xfer_setup(struct usb_setup_params *parm)
4642 {
4643 	struct usb_xfer *xfer;
4644 	void *last_obj;
4645 	uint32_t ntd;
4646 	uint32_t n;
4647 	uint8_t ep_no;
4648 	uint8_t ep_type;
4649 
4650 	xfer = parm->curr_xfer;
4651 
4652 	/*
4653 	 * NOTE: This driver does not use any of the parameters that
4654 	 * are computed from the following values. Just set some
4655 	 * reasonable dummies:
4656 	 */
4657 	parm->hc_max_packet_size = 0x500;
4658 	parm->hc_max_packet_count = 3;
4659 	parm->hc_max_frame_size = 3 * 0x500;
4660 
4661 	usbd_transfer_setup_sub(parm);
4662 
4663 	/*
4664 	 * compute maximum number of TDs
4665 	 */
4666 	ep_type = (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE);
4667 
4668 	if (ep_type == UE_CONTROL) {
4669 
4670 		ntd = xfer->nframes + 1 /* STATUS */ + 1 /* SYNC 1 */
4671 		    + 1 /* SYNC 2 */ + 1 /* SYNC 3 */;
4672 	} else {
4673 
4674 		ntd = xfer->nframes + 1 /* SYNC */ ;
4675 	}
4676 
4677 	/*
4678 	 * check if "usbd_transfer_setup_sub" set an error
4679 	 */
4680 	if (parm->err)
4681 		return;
4682 
4683 	/*
4684 	 * allocate transfer descriptors
4685 	 */
4686 	last_obj = NULL;
4687 
4688 	ep_no = xfer->endpointno & UE_ADDR;
4689 
4690 	/*
4691 	 * Check for a valid endpoint profile in USB device mode:
4692 	 */
4693 	if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
4694 		const struct usb_hw_ep_profile *pf;
4695 
4696 		dwc_otg_get_hw_ep_profile(parm->udev, &pf, ep_no);
4697 
4698 		if (pf == NULL) {
4699 			/* should not happen */
4700 			parm->err = USB_ERR_INVAL;
4701 			return;
4702 		}
4703 	}
4704 
4705 	/* align data */
4706 	parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
4707 
4708 	for (n = 0; n != ntd; n++) {
4709 
4710 		struct dwc_otg_td *td;
4711 
4712 		if (parm->buf) {
4713 
4714 			td = USB_ADD_BYTES(parm->buf, parm->size[0]);
4715 
4716 			/* compute shared bandwidth resource index for TT */
4717 			if (parm->udev->parent_hs_hub != NULL && parm->udev->speed != USB_SPEED_HIGH) {
4718 				if (parm->udev->parent_hs_hub->ddesc.bDeviceProtocol == UDPROTO_HSHUBMTT)
4719 					td->tt_index = parm->udev->device_index;
4720 				else
4721 					td->tt_index = parm->udev->parent_hs_hub->device_index;
4722 			} else {
4723 				td->tt_index = parm->udev->device_index;
4724 			}
4725 
4726 			/* init TD */
4727 			td->max_packet_size = xfer->max_packet_size;
4728 			td->max_packet_count = xfer->max_packet_count;
4729 			td->ep_no = ep_no;
4730 			td->ep_type = ep_type;
4731 			td->obj_next = last_obj;
4732 
4733 			last_obj = td;
4734 		}
4735 		parm->size[0] += sizeof(*td);
4736 	}
4737 
4738 	xfer->td_start[0] = last_obj;
4739 }
4740 
4741 static void
4742 dwc_otg_xfer_unsetup(struct usb_xfer *xfer)
4743 {
4744 	return;
4745 }
4746 
4747 static void
4748 dwc_otg_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
4749     struct usb_endpoint *ep)
4750 {
4751 	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4752 
4753 	DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d,%d)\n",
4754 	    ep, udev->address,
4755 	    edesc->bEndpointAddress, udev->flags.usb_mode,
4756 	    sc->sc_rt_addr, udev->device_index);
4757 
4758 	if (udev->device_index != sc->sc_rt_addr) {
4759 
4760 		if (udev->flags.usb_mode == USB_MODE_DEVICE) {
4761 			if (udev->speed != USB_SPEED_FULL &&
4762 			    udev->speed != USB_SPEED_HIGH) {
4763 				/* not supported */
4764 				return;
4765 			}
4766 		} else {
4767 			if (udev->speed == USB_SPEED_HIGH) {
4768 				if ((UGETW(edesc->wMaxPacketSize) >> 11) & 3) {
4769 					/* high bandwidth endpoint - not tested */
4770 					DPRINTF("High Bandwidth Endpoint - not tested\n");
4771 					return;
4772 				}
4773 			}
4774 		}
4775 		if ((edesc->bmAttributes & UE_XFERTYPE) == UE_ISOCHRONOUS)
4776 			ep->methods = &dwc_otg_device_isoc_methods;
4777 		else
4778 			ep->methods = &dwc_otg_device_non_isoc_methods;
4779 	}
4780 }
4781 
4782 static void
4783 dwc_otg_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
4784 {
4785 	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
4786 
4787 	switch (state) {
4788 	case USB_HW_POWER_SUSPEND:
4789 		dwc_otg_suspend(sc);
4790 		break;
4791 	case USB_HW_POWER_SHUTDOWN:
4792 		dwc_otg_uninit(sc);
4793 		break;
4794 	case USB_HW_POWER_RESUME:
4795 		dwc_otg_resume(sc);
4796 		break;
4797 	default:
4798 		break;
4799 	}
4800 }
4801 
4802 static void
4803 dwc_otg_get_dma_delay(struct usb_device *udev, uint32_t *pus)
4804 {
4805 	/* DMA delay - wait until any use of memory is finished */
4806 	*pus = (2125);			/* microseconds */
4807 }
4808 
4809 static void
4810 dwc_otg_device_resume(struct usb_device *udev)
4811 {
4812 	DPRINTF("\n");
4813 
4814 	/* poll all transfers again to restart resumed ones */
4815 	dwc_otg_do_poll(udev->bus);
4816 }
4817 
4818 static void
4819 dwc_otg_device_suspend(struct usb_device *udev)
4820 {
4821 	DPRINTF("\n");
4822 }
4823 
4824 static const struct usb_bus_methods dwc_otg_bus_methods =
4825 {
4826 	.endpoint_init = &dwc_otg_ep_init,
4827 	.xfer_setup = &dwc_otg_xfer_setup,
4828 	.xfer_unsetup = &dwc_otg_xfer_unsetup,
4829 	.get_hw_ep_profile = &dwc_otg_get_hw_ep_profile,
4830 	.xfer_stall = &dwc_otg_xfer_stall,
4831 	.set_stall = &dwc_otg_set_stall,
4832 	.clear_stall = &dwc_otg_clear_stall,
4833 	.roothub_exec = &dwc_otg_roothub_exec,
4834 	.xfer_poll = &dwc_otg_do_poll,
4835 	.device_state_change = &dwc_otg_device_state_change,
4836 	.set_hw_power_sleep = &dwc_otg_set_hw_power_sleep,
4837 	.get_dma_delay = &dwc_otg_get_dma_delay,
4838 	.device_resume = &dwc_otg_device_resume,
4839 	.device_suspend = &dwc_otg_device_suspend,
4840 };
4841