xref: /freebsd/sys/dev/usb/controller/dwc_otg.c (revision e92ffd9b626833ebdbf2742c8ffddc6cd94b963e)
1 /* $FreeBSD$ */
2 /*-
3  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4  *
5  * Copyright (c) 2015 Daisuke Aoyama. All rights reserved.
6  * Copyright (c) 2012-2015 Hans Petter Selasky. All rights reserved.
7  * Copyright (c) 2010-2011 Aleksandr Rybalko. All rights reserved.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28  * SUCH DAMAGE.
29  */
30 
31 /*
32  * This file contains the driver for the DesignWare series USB 2.0 OTG
33  * Controller.
34  */
35 
36 /*
37  * LIMITATION: Drivers must be bound to all OUT endpoints in the
38  * active configuration for this driver to work properly. Blocking any
39  * OUT endpoint will block all OUT endpoints including the control
40  * endpoint. Usually this is not a problem.
41  */
42 
43 /*
44  * NOTE: Writing to non-existing registers appears to cause an
45  * internal reset.
46  */
47 
48 #ifdef USB_GLOBAL_INCLUDE_FILE
49 #include USB_GLOBAL_INCLUDE_FILE
50 #else
51 #include <sys/stdint.h>
52 #include <sys/stddef.h>
53 #include <sys/param.h>
54 #include <sys/queue.h>
55 #include <sys/types.h>
56 #include <sys/systm.h>
57 #include <sys/kernel.h>
58 #include <sys/bus.h>
59 #include <sys/module.h>
60 #include <sys/lock.h>
61 #include <sys/mutex.h>
62 #include <sys/condvar.h>
63 #include <sys/sysctl.h>
64 #include <sys/sx.h>
65 #include <sys/unistd.h>
66 #include <sys/callout.h>
67 #include <sys/malloc.h>
68 #include <sys/priv.h>
69 #include <sys/rman.h>
70 
71 #include <dev/usb/usb.h>
72 #include <dev/usb/usbdi.h>
73 
74 #define	USB_DEBUG_VAR dwc_otg_debug
75 
76 #include <dev/usb/usb_core.h>
77 #include <dev/usb/usb_debug.h>
78 #include <dev/usb/usb_busdma.h>
79 #include <dev/usb/usb_process.h>
80 #include <dev/usb/usb_transfer.h>
81 #include <dev/usb/usb_device.h>
82 #include <dev/usb/usb_hub.h>
83 #include <dev/usb/usb_util.h>
84 
85 #include <dev/usb/usb_controller.h>
86 #include <dev/usb/usb_bus.h>
87 #endif			/* USB_GLOBAL_INCLUDE_FILE */
88 
89 #include <dev/usb/controller/dwc_otg.h>
90 #include <dev/usb/controller/dwc_otgreg.h>
91 
92 #define	DWC_OTG_BUS2SC(bus) \
93     __containerof(bus, struct dwc_otg_softc, sc_bus)
94 
95 #define	DWC_OTG_PC2UDEV(pc) \
96    (USB_DMATAG_TO_XROOT((pc)->tag_parent)->udev)
97 
98 #define	DWC_OTG_MSK_GINT_THREAD_IRQ				\
99    (GINTSTS_USBRST | GINTSTS_ENUMDONE | GINTSTS_PRTINT |	\
100    GINTSTS_WKUPINT | GINTSTS_USBSUSP | GINTMSK_OTGINTMSK |	\
101    GINTSTS_SESSREQINT)
102 
103 #ifndef DWC_OTG_PHY_DEFAULT
104 #define	DWC_OTG_PHY_DEFAULT DWC_OTG_PHY_ULPI
105 #endif
106 
107 static int dwc_otg_phy_type = DWC_OTG_PHY_DEFAULT;
108 
109 static SYSCTL_NODE(_hw_usb, OID_AUTO, dwc_otg, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
110     "USB DWC OTG");
111 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, phy_type, CTLFLAG_RDTUN,
112     &dwc_otg_phy_type, 0, "DWC OTG PHY TYPE - 0/1/2/3 - ULPI/HSIC/INTERNAL/UTMI+");
113 
114 #ifdef USB_DEBUG
115 static int dwc_otg_debug = 0;
116 
117 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, debug, CTLFLAG_RWTUN,
118     &dwc_otg_debug, 0, "DWC OTG debug level");
119 #endif
120 
121 #define	DWC_OTG_INTR_ENDPT 1
122 
123 /* prototypes */
124 
125 static const struct usb_bus_methods dwc_otg_bus_methods;
126 static const struct usb_pipe_methods dwc_otg_device_non_isoc_methods;
127 static const struct usb_pipe_methods dwc_otg_device_isoc_methods;
128 
129 static dwc_otg_cmd_t dwc_otg_setup_rx;
130 static dwc_otg_cmd_t dwc_otg_data_rx;
131 static dwc_otg_cmd_t dwc_otg_data_tx;
132 static dwc_otg_cmd_t dwc_otg_data_tx_sync;
133 
134 static dwc_otg_cmd_t dwc_otg_host_setup_tx;
135 static dwc_otg_cmd_t dwc_otg_host_data_tx;
136 static dwc_otg_cmd_t dwc_otg_host_data_rx;
137 
138 static void dwc_otg_device_done(struct usb_xfer *, usb_error_t);
139 static void dwc_otg_do_poll(struct usb_bus *);
140 static void dwc_otg_standard_done(struct usb_xfer *);
141 static void dwc_otg_root_intr(struct dwc_otg_softc *);
142 static void dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *);
143 
144 /*
145  * Here is a configuration that the chip supports.
146  */
147 static const struct usb_hw_ep_profile dwc_otg_ep_profile[1] = {
148 	[0] = {
149 		.max_in_frame_size = 64,/* fixed */
150 		.max_out_frame_size = 64,	/* fixed */
151 		.is_simplex = 1,
152 		.support_control = 1,
153 	}
154 };
155 
156 static void
157 dwc_otg_get_hw_ep_profile(struct usb_device *udev,
158     const struct usb_hw_ep_profile **ppf, uint8_t ep_addr)
159 {
160 	struct dwc_otg_softc *sc;
161 
162 	sc = DWC_OTG_BUS2SC(udev->bus);
163 
164 	if (ep_addr < sc->sc_dev_ep_max)
165 		*ppf = &sc->sc_hw_ep_profile[ep_addr].usb;
166 	else
167 		*ppf = NULL;
168 }
169 
170 static void
171 dwc_otg_write_fifo(struct dwc_otg_softc *sc, struct usb_page_cache *pc,
172     uint32_t offset, uint32_t fifo, uint32_t count)
173 {
174 	uint32_t temp;
175 
176 	/* round down length to nearest 4-bytes */
177 	temp = count & ~3;
178 
179 	/* check if we can write the data directly */
180 	if (temp != 0 && usb_pc_buffer_is_aligned(pc, offset, temp, 3)) {
181 		struct usb_page_search buf_res;
182 
183 		/* pre-subtract length */
184 		count -= temp;
185 
186 		/* iterate buffer list */
187 		do {
188 			/* get current buffer pointer */
189 			usbd_get_page(pc, offset, &buf_res);
190 
191 			if (buf_res.length > temp)
192 				buf_res.length = temp;
193 
194 			/* transfer data into FIFO */
195 			bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
196 			    fifo, buf_res.buffer, buf_res.length / 4);
197 
198 			offset += buf_res.length;
199 			fifo += buf_res.length;
200 			temp -= buf_res.length;
201 		} while (temp != 0);
202 	}
203 
204 	/* check for remainder */
205 	if (count != 0) {
206 		/* clear topmost word before copy */
207 		sc->sc_bounce_buffer[(count - 1) / 4] = 0;
208 
209 		/* copy out data */
210 		usbd_copy_out(pc, offset,
211 		    sc->sc_bounce_buffer, count);
212 
213 		/* transfer data into FIFO */
214 		bus_space_write_region_4(sc->sc_io_tag,
215 		    sc->sc_io_hdl, fifo, sc->sc_bounce_buffer,
216 		    (count + 3) / 4);
217 	}
218 }
219 
220 static void
221 dwc_otg_read_fifo(struct dwc_otg_softc *sc, struct usb_page_cache *pc,
222     uint32_t offset, uint32_t count)
223 {
224 	uint32_t temp;
225 
226 	/* round down length to nearest 4-bytes */
227 	temp = count & ~3;
228 
229 	/* check if we can read the data directly */
230 	if (temp != 0 && usb_pc_buffer_is_aligned(pc, offset, temp, 3)) {
231 		struct usb_page_search buf_res;
232 
233 		/* pre-subtract length */
234 		count -= temp;
235 
236 		/* iterate buffer list */
237 		do {
238 			/* get current buffer pointer */
239 			usbd_get_page(pc, offset, &buf_res);
240 
241 			if (buf_res.length > temp)
242 				buf_res.length = temp;
243 
244 			/* transfer data from FIFO */
245 			bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
246 			    sc->sc_current_rx_fifo, buf_res.buffer, buf_res.length / 4);
247 
248 			offset += buf_res.length;
249 			sc->sc_current_rx_fifo += buf_res.length;
250 			sc->sc_current_rx_bytes -= buf_res.length;
251 			temp -= buf_res.length;
252 		} while (temp != 0);
253 	}
254 
255 	/* check for remainder */
256 	if (count != 0) {
257 		/* read data into bounce buffer */
258 		bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
259 			sc->sc_current_rx_fifo,
260 			sc->sc_bounce_buffer, (count + 3) / 4);
261 
262 		/* store data into proper buffer */
263 		usbd_copy_in(pc, offset, sc->sc_bounce_buffer, count);
264 
265 		/* round length up to nearest 4 bytes */
266 		count = (count + 3) & ~3;
267 
268 		/* update counters */
269 		sc->sc_current_rx_bytes -= count;
270 		sc->sc_current_rx_fifo += count;
271 	}
272 }
273 
274 static void
275 dwc_otg_tx_fifo_reset(struct dwc_otg_softc *sc, uint32_t value)
276 {
277 	uint32_t temp;
278 
279   	/* reset FIFO */
280 	DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL, value);
281 
282 	/* wait for reset to complete */
283 	for (temp = 0; temp != 16; temp++) {
284 		value = DWC_OTG_READ_4(sc, DOTG_GRSTCTL);
285 		if (!(value & (GRSTCTL_TXFFLSH | GRSTCTL_RXFFLSH)))
286 			break;
287 	}
288 }
289 
290 static int
291 dwc_otg_init_fifo(struct dwc_otg_softc *sc, uint8_t mode)
292 {
293 	struct dwc_otg_profile *pf;
294 	uint32_t fifo_size;
295 	uint32_t fifo_regs;
296 	uint32_t tx_start;
297 	uint8_t x;
298 
299 	fifo_size = sc->sc_fifo_size;
300 
301 	/*
302 	 * NOTE: Reserved fixed size area at end of RAM, which must
303 	 * not be allocated to the FIFOs:
304 	 */
305 	fifo_regs = 4 * 16;
306 
307 	if (fifo_size < fifo_regs) {
308 		DPRINTF("Too little FIFO\n");
309 		return (EINVAL);
310 	}
311 
312 	/* subtract FIFO regs from total once */
313 	fifo_size -= fifo_regs;
314 
315 	/* split equally for IN and OUT */
316 	fifo_size /= 2;
317 
318 	/* Align to 4 bytes boundary (refer to PGM) */
319 	fifo_size &= ~3;
320 
321 	/* set global receive FIFO size */
322 	DWC_OTG_WRITE_4(sc, DOTG_GRXFSIZ, fifo_size / 4);
323 
324 	tx_start = fifo_size;
325 
326 	if (fifo_size < 64) {
327 		DPRINTFN(-1, "Not enough data space for EP0 FIFO.\n");
328 		return (EINVAL);
329 	}
330 
331 	if (mode == DWC_MODE_HOST) {
332 		/* reset active endpoints */
333 		sc->sc_active_rx_ep = 0;
334 
335 		/* split equally for periodic and non-periodic */
336 		fifo_size /= 2;
337 
338 		DPRINTF("PTX/NPTX FIFO=%u\n", fifo_size);
339 
340 		/* align to 4 bytes boundary */
341 		fifo_size &= ~3;
342 
343 		DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
344 		    ((fifo_size / 4) << 16) |
345 		    (tx_start / 4));
346 
347 		tx_start += fifo_size;
348 
349 		for (x = 0; x != sc->sc_host_ch_max; x++) {
350 			/* enable all host interrupts */
351 			DWC_OTG_WRITE_4(sc, DOTG_HCINTMSK(x),
352 			    HCINT_DEFAULT_MASK);
353 		}
354 
355 		DWC_OTG_WRITE_4(sc, DOTG_HPTXFSIZ,
356 		    ((fifo_size / 4) << 16) |
357 		    (tx_start / 4));
358 
359 		/* reset host channel state */
360 		memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
361 
362 		/* enable all host channel interrupts */
363 		DWC_OTG_WRITE_4(sc, DOTG_HAINTMSK,
364 		    (1U << sc->sc_host_ch_max) - 1U);
365 
366 		/* enable proper host channel interrupts */
367 		sc->sc_irq_mask |= GINTMSK_HCHINTMSK;
368 		sc->sc_irq_mask &= ~GINTMSK_IEPINTMSK;
369 		DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
370 	}
371 
372 	if (mode == DWC_MODE_DEVICE) {
373 	    DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
374 		(0x10 << 16) | (tx_start / 4));
375 	    fifo_size -= 0x40;
376 	    tx_start += 0x40;
377 
378 	    /* setup control endpoint profile */
379 	    sc->sc_hw_ep_profile[0].usb = dwc_otg_ep_profile[0];
380 
381 	    /* reset active endpoints */
382 	    sc->sc_active_rx_ep = 1;
383 
384 	    for (x = 1; x != sc->sc_dev_ep_max; x++) {
385 		pf = sc->sc_hw_ep_profile + x;
386 
387 		pf->usb.max_out_frame_size = 1024 * 3;
388 		pf->usb.is_simplex = 0;	/* assume duplex */
389 		pf->usb.support_bulk = 1;
390 		pf->usb.support_interrupt = 1;
391 		pf->usb.support_isochronous = 1;
392 		pf->usb.support_out = 1;
393 
394 		if (x < sc->sc_dev_in_ep_max) {
395 			uint32_t limit;
396 
397 			limit = (x == 1) ? MIN(DWC_OTG_TX_MAX_FIFO_SIZE,
398 			    DWC_OTG_MAX_TXN) : MIN(DWC_OTG_MAX_TXN / 2,
399 			    DWC_OTG_TX_MAX_FIFO_SIZE);
400 
401 			/* see if there is enough FIFO space */
402 			if (limit <= fifo_size) {
403 				pf->max_buffer = limit;
404 				pf->usb.support_in = 1;
405 			} else {
406 				limit = MIN(DWC_OTG_TX_MAX_FIFO_SIZE, 0x40);
407 				if (limit <= fifo_size) {
408 					pf->usb.support_in = 1;
409 				} else {
410 					pf->usb.is_simplex = 1;
411 					limit = 0;
412 				}
413 			}
414 			/* set FIFO size */
415 			DWC_OTG_WRITE_4(sc, DOTG_DIEPTXF(x),
416 			    ((limit / 4) << 16) | (tx_start / 4));
417 			tx_start += limit;
418 			fifo_size -= limit;
419 			pf->usb.max_in_frame_size = limit;
420 		} else {
421 			pf->usb.is_simplex = 1;
422 		}
423 
424 		DPRINTF("FIFO%d = IN:%d / OUT:%d\n", x,
425 		    pf->usb.max_in_frame_size,
426 		    pf->usb.max_out_frame_size);
427 	    }
428 
429 	    /* enable proper device channel interrupts */
430 	    sc->sc_irq_mask &= ~GINTMSK_HCHINTMSK;
431 	    sc->sc_irq_mask |= GINTMSK_IEPINTMSK;
432 	    DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
433 	}
434 
435 	/* reset RX FIFO */
436 	dwc_otg_tx_fifo_reset(sc, GRSTCTL_RXFFLSH);
437 
438 	if (mode != DWC_MODE_OTG) {
439 		/* reset all TX FIFOs */
440 		dwc_otg_tx_fifo_reset(sc,
441 		    GRSTCTL_TXFIFO(0x10) |
442 		    GRSTCTL_TXFFLSH);
443 	} else {
444 		/* reset active endpoints */
445 		sc->sc_active_rx_ep = 0;
446 
447 		/* reset host channel state */
448 		memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
449 	}
450 	return (0);
451 }
452 
453 static uint8_t
454 dwc_otg_uses_split(struct usb_device *udev)
455 {
456 	/*
457 	 * When a LOW or FULL speed device is connected directly to
458 	 * the USB port we don't use split transactions:
459 	 */
460 	return (udev->speed != USB_SPEED_HIGH &&
461 	    udev->parent_hs_hub != NULL &&
462 	    udev->parent_hs_hub->parent_hub != NULL);
463 }
464 
465 static void
466 dwc_otg_update_host_frame_interval(struct dwc_otg_softc *sc)
467 {
468 
469   /*
470    * Disabled until further. Assuming that the register is already
471    * programmed correctly by the boot loader.
472    */
473 #if 0
474 	uint32_t temp;
475 
476 	/* setup HOST frame interval register, based on existing value */
477 	temp = DWC_OTG_READ_4(sc, DOTG_HFIR) & HFIR_FRINT_MASK;
478 	if (temp >= 10000)
479 		temp /= 1000;
480 	else
481 		temp /= 125;
482 
483 	/* figure out nearest X-tal value */
484 	if (temp >= 54)
485 		temp = 60;	/* MHz */
486 	else if (temp >= 39)
487 		temp = 48;	/* MHz */
488 	else
489 		temp = 30;	/* MHz */
490 
491 	if (sc->sc_flags.status_high_speed)
492 		temp *= 125;
493 	else
494 		temp *= 1000;
495 
496 	DPRINTF("HFIR=0x%08x\n", temp);
497 
498 	DWC_OTG_WRITE_4(sc, DOTG_HFIR, temp);
499 #endif
500 }
501 
502 static void
503 dwc_otg_clocks_on(struct dwc_otg_softc *sc)
504 {
505 	if (sc->sc_flags.clocks_off &&
506 	    sc->sc_flags.port_powered) {
507 		DPRINTFN(5, "\n");
508 
509 		/* TODO - platform specific */
510 
511 		sc->sc_flags.clocks_off = 0;
512 	}
513 }
514 
515 static void
516 dwc_otg_clocks_off(struct dwc_otg_softc *sc)
517 {
518 	if (!sc->sc_flags.clocks_off) {
519 		DPRINTFN(5, "\n");
520 
521 		/* TODO - platform specific */
522 
523 		sc->sc_flags.clocks_off = 1;
524 	}
525 }
526 
527 static void
528 dwc_otg_pull_up(struct dwc_otg_softc *sc)
529 {
530 	uint32_t temp;
531 
532 	/* pullup D+, if possible */
533 
534 	if (!sc->sc_flags.d_pulled_up &&
535 	    sc->sc_flags.port_powered) {
536 		sc->sc_flags.d_pulled_up = 1;
537 
538 		temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
539 		temp &= ~DCTL_SFTDISCON;
540 		DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
541 	}
542 }
543 
544 static void
545 dwc_otg_pull_down(struct dwc_otg_softc *sc)
546 {
547 	uint32_t temp;
548 
549 	/* pulldown D+, if possible */
550 
551 	if (sc->sc_flags.d_pulled_up) {
552 		sc->sc_flags.d_pulled_up = 0;
553 
554 		temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
555 		temp |= DCTL_SFTDISCON;
556 		DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
557 	}
558 }
559 
560 static void
561 dwc_otg_enable_sof_irq(struct dwc_otg_softc *sc)
562 {
563 	/* In device mode we don't use the SOF interrupt */
564 	if (sc->sc_flags.status_device_mode != 0)
565 		return;
566 	/* Ensure the SOF interrupt is not disabled */
567 	sc->sc_needsof = 1;
568 	/* Check if the SOF interrupt is already enabled */
569 	if ((sc->sc_irq_mask & GINTMSK_SOFMSK) != 0)
570 		return;
571 	sc->sc_irq_mask |= GINTMSK_SOFMSK;
572 	DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
573 }
574 
575 static void
576 dwc_otg_resume_irq(struct dwc_otg_softc *sc)
577 {
578 	if (sc->sc_flags.status_suspend) {
579 		/* update status bits */
580 		sc->sc_flags.status_suspend = 0;
581 		sc->sc_flags.change_suspend = 1;
582 
583 		if (sc->sc_flags.status_device_mode) {
584 			/*
585 			 * Disable resume interrupt and enable suspend
586 			 * interrupt:
587 			 */
588 			sc->sc_irq_mask &= ~GINTMSK_WKUPINTMSK;
589 			sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
590 			DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
591 		}
592 
593 		/* complete root HUB interrupt endpoint */
594 		dwc_otg_root_intr(sc);
595 	}
596 }
597 
598 static void
599 dwc_otg_suspend_irq(struct dwc_otg_softc *sc)
600 {
601 	if (!sc->sc_flags.status_suspend) {
602 		/* update status bits */
603 		sc->sc_flags.status_suspend = 1;
604 		sc->sc_flags.change_suspend = 1;
605 
606 		if (sc->sc_flags.status_device_mode) {
607 			/*
608 			 * Disable suspend interrupt and enable resume
609 			 * interrupt:
610 			 */
611 			sc->sc_irq_mask &= ~GINTMSK_USBSUSPMSK;
612 			sc->sc_irq_mask |= GINTMSK_WKUPINTMSK;
613 			DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
614 		}
615 
616 		/* complete root HUB interrupt endpoint */
617 		dwc_otg_root_intr(sc);
618 	}
619 }
620 
621 static void
622 dwc_otg_wakeup_peer(struct dwc_otg_softc *sc)
623 {
624 	if (!sc->sc_flags.status_suspend)
625 		return;
626 
627 	DPRINTFN(5, "Remote wakeup\n");
628 
629 	if (sc->sc_flags.status_device_mode) {
630 		uint32_t temp;
631 
632 		/* enable remote wakeup signalling */
633 		temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
634 		temp |= DCTL_RMTWKUPSIG;
635 		DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
636 
637 		/* Wait 8ms for remote wakeup to complete. */
638 		usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 125);
639 
640 		temp &= ~DCTL_RMTWKUPSIG;
641 		DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
642 	} else {
643 		/* enable USB port */
644 		DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
645 
646 		/* wait 10ms */
647 		usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
648 
649 		/* resume port */
650 		sc->sc_hprt_val |= HPRT_PRTRES;
651 		DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
652 
653 		/* Wait 100ms for resume signalling to complete. */
654 		usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 10);
655 
656 		/* clear suspend and resume */
657 		sc->sc_hprt_val &= ~(HPRT_PRTSUSP | HPRT_PRTRES);
658 		DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
659 
660 		/* Wait 4ms */
661 		usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
662 	}
663 
664 	/* need to fake resume IRQ */
665 	dwc_otg_resume_irq(sc);
666 }
667 
668 static void
669 dwc_otg_set_address(struct dwc_otg_softc *sc, uint8_t addr)
670 {
671 	uint32_t temp;
672 
673 	DPRINTFN(5, "addr=%d\n", addr);
674 
675 	temp = DWC_OTG_READ_4(sc, DOTG_DCFG);
676 	temp &= ~DCFG_DEVADDR_SET(0x7F);
677 	temp |= DCFG_DEVADDR_SET(addr);
678 	DWC_OTG_WRITE_4(sc, DOTG_DCFG, temp);
679 }
680 
681 static void
682 dwc_otg_common_rx_ack(struct dwc_otg_softc *sc)
683 {
684 	DPRINTFN(5, "RX status clear\n");
685 
686 	/* enable RX FIFO level interrupt */
687 	sc->sc_irq_mask |= GINTMSK_RXFLVLMSK;
688 	DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
689 
690 	if (sc->sc_current_rx_bytes != 0) {
691 		/* need to dump remaining data */
692 		bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
693 		    sc->sc_current_rx_fifo, sc->sc_bounce_buffer,
694 		    sc->sc_current_rx_bytes / 4);
695 		/* clear number of active bytes to receive */
696 		sc->sc_current_rx_bytes = 0;
697 	}
698 	/* clear cached status */
699 	sc->sc_last_rx_status = 0;
700 }
701 
702 static void
703 dwc_otg_clear_hcint(struct dwc_otg_softc *sc, uint8_t x)
704 {
705 	uint32_t hcint;
706 
707 	/* clear all pending interrupts */
708 	hcint = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
709 	DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), hcint);
710 
711 	/* clear buffered interrupts */
712 	sc->sc_chan_state[x].hcint = 0;
713 }
714 
715 static uint8_t
716 dwc_otg_host_check_tx_fifo_empty(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
717 {
718 	uint32_t temp;
719 
720 	temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
721 
722 	if (td->ep_type == UE_ISOCHRONOUS) {
723 		/*
724 		 * NOTE: USB INTERRUPT transactions are executed like
725 		 * USB CONTROL transactions! See the setup standard
726 		 * chain function for more information.
727 		 */
728 		if (!(temp & GINTSTS_PTXFEMP)) {
729 			DPRINTF("Periodic TX FIFO is not empty\n");
730 			if (!(sc->sc_irq_mask & GINTMSK_PTXFEMPMSK)) {
731 				sc->sc_irq_mask |= GINTMSK_PTXFEMPMSK;
732 				DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
733 			}
734 			return (1);	/* busy */
735 		}
736 	} else {
737 		if (!(temp & GINTSTS_NPTXFEMP)) {
738 			DPRINTF("Non-periodic TX FIFO is not empty\n");
739 			if (!(sc->sc_irq_mask & GINTMSK_NPTXFEMPMSK)) {
740 				sc->sc_irq_mask |= GINTMSK_NPTXFEMPMSK;
741 				DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
742 			}
743 			return (1);	/* busy */
744 		}
745 	}
746 	return (0);	/* ready for transmit */
747 }
748 
749 static uint8_t
750 dwc_otg_host_channel_alloc(struct dwc_otg_softc *sc,
751     struct dwc_otg_td *td, uint8_t is_out)
752 {
753 	uint8_t x;
754 	uint8_t y;
755 	uint8_t z;
756 
757 	if (td->channel[0] < DWC_OTG_MAX_CHANNELS)
758 		return (0);		/* already allocated */
759 
760 	/* check if device is suspended */
761 	if (DWC_OTG_PC2UDEV(td->pc)->flags.self_suspended != 0)
762 		return (1);		/* busy - cannot transfer data */
763 
764 	/* compute needed TX FIFO size */
765 	if (is_out != 0) {
766 		if (dwc_otg_host_check_tx_fifo_empty(sc, td) != 0)
767 			return (1);	/* busy - cannot transfer data */
768 	}
769 	z = td->max_packet_count;
770 	for (x = y = 0; x != sc->sc_host_ch_max; x++) {
771 		/* check if channel is allocated */
772 		if (sc->sc_chan_state[x].allocated != 0)
773 			continue;
774 		/* check if channel is still enabled */
775 		if (sc->sc_chan_state[x].wait_halted != 0)
776 			continue;
777 		/* store channel number */
778 		td->channel[y++] = x;
779 		/* check if we got all channels */
780 		if (y == z)
781 			break;
782 	}
783 	if (y != z) {
784 		/* reset channel variable */
785 		td->channel[0] = DWC_OTG_MAX_CHANNELS;
786 		td->channel[1] = DWC_OTG_MAX_CHANNELS;
787 		td->channel[2] = DWC_OTG_MAX_CHANNELS;
788 		/* wait a bit */
789 		dwc_otg_enable_sof_irq(sc);
790 		return (1);	/* busy - not enough channels */
791 	}
792 
793 	for (y = 0; y != z; y++) {
794 		x = td->channel[y];
795 
796 		/* set allocated */
797 		sc->sc_chan_state[x].allocated = 1;
798 
799 		/* set wait halted */
800 		sc->sc_chan_state[x].wait_halted = 1;
801 
802 		/* clear interrupts */
803 		dwc_otg_clear_hcint(sc, x);
804 
805 		DPRINTF("CH=%d HCCHAR=0x%08x "
806 		    "HCSPLT=0x%08x\n", x, td->hcchar, td->hcsplt);
807 
808 		/* set active channel */
809 		sc->sc_active_rx_ep |= (1 << x);
810 	}
811 	return (0);	/* allocated */
812 }
813 
814 static void
815 dwc_otg_host_channel_free_sub(struct dwc_otg_softc *sc, struct dwc_otg_td *td, uint8_t index)
816 {
817 	uint32_t hcchar;
818 	uint8_t x;
819 
820 	if (td->channel[index] >= DWC_OTG_MAX_CHANNELS)
821 		return;		/* already freed */
822 
823 	/* free channel */
824 	x = td->channel[index];
825 	td->channel[index] = DWC_OTG_MAX_CHANNELS;
826 
827 	DPRINTF("CH=%d\n", x);
828 
829 	/*
830 	 * We need to let programmed host channels run till complete
831 	 * else the host channel will stop functioning.
832 	 */
833 	sc->sc_chan_state[x].allocated = 0;
834 
835 	/* ack any pending messages */
836 	if (sc->sc_last_rx_status != 0 &&
837 	    GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) == x) {
838 		dwc_otg_common_rx_ack(sc);
839 	}
840 
841 	/* clear active channel */
842 	sc->sc_active_rx_ep &= ~(1 << x);
843 
844 	/* check if already halted */
845 	if (sc->sc_chan_state[x].wait_halted == 0)
846 		return;
847 
848 	/* disable host channel */
849 	hcchar = DWC_OTG_READ_4(sc, DOTG_HCCHAR(x));
850 	if (hcchar & HCCHAR_CHENA) {
851 		DPRINTF("Halting channel %d\n", x);
852 		DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(x),
853 		    hcchar | HCCHAR_CHDIS);
854 		/* don't write HCCHAR until the channel is halted */
855 	} else {
856 		sc->sc_chan_state[x].wait_halted = 0;
857 	}
858 }
859 
860 static void
861 dwc_otg_host_channel_free(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
862 {
863 	uint8_t x;
864 	for (x = 0; x != td->max_packet_count; x++)
865 		dwc_otg_host_channel_free_sub(sc, td, x);
866 }
867 
868 static void
869 dwc_otg_host_dump_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
870 {
871 	uint8_t x;
872 	/* dump any pending messages */
873 	if (sc->sc_last_rx_status == 0)
874 		return;
875 	for (x = 0; x != td->max_packet_count; x++) {
876 		if (td->channel[x] >= DWC_OTG_MAX_CHANNELS ||
877 		    td->channel[x] != GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status))
878 			continue;
879 		dwc_otg_common_rx_ack(sc);
880 		break;
881 	}
882 }
883 
884 static uint8_t
885 dwc_otg_host_setup_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
886 {
887 	struct usb_device_request req __aligned(4);
888 	uint32_t hcint;
889 	uint32_t hcchar;
890 	uint8_t delta;
891 
892 	dwc_otg_host_dump_rx(sc, td);
893 
894 	if (td->channel[0] < DWC_OTG_MAX_CHANNELS) {
895 		hcint = sc->sc_chan_state[td->channel[0]].hcint;
896 
897 		DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
898 		    td->channel[0], td->state, hcint,
899 		    DWC_OTG_READ_4(sc, DOTG_HCCHAR(td->channel[0])),
900 		    DWC_OTG_READ_4(sc, DOTG_HCTSIZ(td->channel[0])));
901 	} else {
902 		hcint = 0;
903 		goto check_state;
904 	}
905 
906 	if (hcint & (HCINT_RETRY |
907 	    HCINT_ACK | HCINT_NYET)) {
908 		/* give success bits priority over failure bits */
909 	} else if (hcint & HCINT_STALL) {
910 		DPRINTF("CH=%d STALL\n", td->channel[0]);
911 		td->error_stall = 1;
912 		td->error_any = 1;
913 		goto complete;
914 	} else if (hcint & HCINT_ERRORS) {
915 		DPRINTF("CH=%d ERROR\n", td->channel[0]);
916 		td->errcnt++;
917 		if (td->hcsplt != 0 || td->errcnt >= 3) {
918 			td->error_any = 1;
919 			goto complete;
920 		}
921 	}
922 
923 	if (hcint & (HCINT_ERRORS | HCINT_RETRY |
924 	    HCINT_ACK | HCINT_NYET)) {
925 		if (!(hcint & HCINT_ERRORS))
926 			td->errcnt = 0;
927 	}
928 
929 check_state:
930 	switch (td->state) {
931 	case DWC_CHAN_ST_START:
932 		goto send_pkt;
933 
934 	case DWC_CHAN_ST_WAIT_ANE:
935 		if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
936 			td->did_nak = 1;
937 			td->tt_scheduled = 0;
938 			goto send_pkt;
939 		} else if (hcint & (HCINT_ACK | HCINT_NYET)) {
940 			td->offset += td->tx_bytes;
941 			td->remainder -= td->tx_bytes;
942 			td->toggle = 1;
943 			td->tt_scheduled = 0;
944 			goto complete;
945 		}
946 		break;
947 
948 	case DWC_CHAN_ST_WAIT_S_ANE:
949 		if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
950 			td->did_nak = 1;
951 			td->tt_scheduled = 0;
952 			goto send_pkt;
953 		} else if (hcint & (HCINT_ACK | HCINT_NYET)) {
954 			goto send_cpkt;
955 		}
956 		break;
957 
958 	case DWC_CHAN_ST_WAIT_C_ANE:
959 		if (hcint & HCINT_NYET) {
960 			goto send_cpkt;
961 		} else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
962 			td->did_nak = 1;
963 			td->tt_scheduled = 0;
964 			goto send_pkt;
965 		} else if (hcint & HCINT_ACK) {
966 			td->offset += td->tx_bytes;
967 			td->remainder -= td->tx_bytes;
968 			td->toggle = 1;
969 			goto complete;
970 		}
971 		break;
972 
973 	case DWC_CHAN_ST_WAIT_C_PKT:
974 		goto send_cpkt;
975 
976 	default:
977 		break;
978 	}
979 	goto busy;
980 
981 send_pkt:
982 	/* free existing channel, if any */
983 	dwc_otg_host_channel_free(sc, td);
984 
985 	if (sizeof(req) != td->remainder) {
986 		td->error_any = 1;
987 		goto complete;
988 	}
989 
990 	if (td->hcsplt != 0) {
991 		delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
992 		if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
993 			td->state = DWC_CHAN_ST_START;
994 			goto busy;
995 		}
996 		delta = sc->sc_last_frame_num - td->tt_start_slot;
997 		if (delta > 5) {
998 			/* missed it */
999 			td->tt_scheduled = 0;
1000 			td->state = DWC_CHAN_ST_START;
1001 			goto busy;
1002 		}
1003 	}
1004 
1005 	/* allocate a new channel */
1006 	if (dwc_otg_host_channel_alloc(sc, td, 1)) {
1007 		td->state = DWC_CHAN_ST_START;
1008 		goto busy;
1009 	}
1010 
1011 	if (td->hcsplt != 0) {
1012 		td->hcsplt &= ~HCSPLT_COMPSPLT;
1013 		td->state = DWC_CHAN_ST_WAIT_S_ANE;
1014 	} else {
1015 		td->state = DWC_CHAN_ST_WAIT_ANE;
1016 	}
1017 
1018 	/* copy out control request */
1019 	usbd_copy_out(td->pc, 0, &req, sizeof(req));
1020 
1021 	DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel[0]),
1022 	    (sizeof(req) << HCTSIZ_XFERSIZE_SHIFT) |
1023 	    (1 << HCTSIZ_PKTCNT_SHIFT) |
1024 	    (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
1025 
1026 	DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel[0]), td->hcsplt);
1027 
1028 	hcchar = td->hcchar;
1029 	hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
1030 	hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
1031 
1032 	/* must enable channel before writing data to FIFO */
1033 	DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel[0]), hcchar);
1034 
1035 	/* transfer data into FIFO */
1036 	bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
1037 	    DOTG_DFIFO(td->channel[0]), (uint32_t *)&req, sizeof(req) / 4);
1038 
1039 	/* wait until next slot before trying complete split */
1040 	td->tt_complete_slot = sc->sc_last_frame_num + 1;
1041 
1042 	/* store number of bytes transmitted */
1043 	td->tx_bytes = sizeof(req);
1044 	goto busy;
1045 
1046 send_cpkt:
1047 	/* free existing channel, if any */
1048 	dwc_otg_host_channel_free(sc, td);
1049 
1050 	delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1051 	if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1052 		td->state = DWC_CHAN_ST_WAIT_C_PKT;
1053 		goto busy;
1054 	}
1055 	delta = sc->sc_last_frame_num - td->tt_start_slot;
1056 	if (delta > DWC_OTG_TT_SLOT_MAX) {
1057 		/* we missed the service interval */
1058 		if (td->ep_type != UE_ISOCHRONOUS)
1059 			td->error_any = 1;
1060 		goto complete;
1061 	}
1062 	/* allocate a new channel */
1063 	if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1064 		td->state = DWC_CHAN_ST_WAIT_C_PKT;
1065 		goto busy;
1066 	}
1067 
1068 	/* wait until next slot before trying complete split */
1069 	td->tt_complete_slot = sc->sc_last_frame_num + 1;
1070 
1071 	td->hcsplt |= HCSPLT_COMPSPLT;
1072 	td->state = DWC_CHAN_ST_WAIT_C_ANE;
1073 
1074 	DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel[0]),
1075 	    (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
1076 
1077 	DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel[0]), td->hcsplt);
1078 
1079 	hcchar = td->hcchar;
1080 	hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
1081 	hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
1082 
1083 	/* must enable channel before writing data to FIFO */
1084 	DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel[0]), hcchar);
1085 
1086 busy:
1087 	return (1);	/* busy */
1088 
1089 complete:
1090 	dwc_otg_host_channel_free(sc, td);
1091 	return (0);	/* complete */
1092 }
1093 
1094 static uint8_t
1095 dwc_otg_setup_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1096 {
1097 	struct usb_device_request req __aligned(4);
1098 	uint32_t temp;
1099 	uint16_t count;
1100 
1101 	/* check endpoint status */
1102 
1103 	if (sc->sc_last_rx_status == 0)
1104 		goto not_complete;
1105 
1106 	if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != 0)
1107 		goto not_complete;
1108 
1109 	if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1110 	    GRXSTSRD_STP_DATA) {
1111 		if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1112 		    GRXSTSRD_STP_COMPLETE || td->remainder != 0) {
1113 			/* release FIFO */
1114 			dwc_otg_common_rx_ack(sc);
1115 			goto not_complete;
1116 		}
1117 		/* release FIFO */
1118 		dwc_otg_common_rx_ack(sc);
1119 		return (0);     /* complete */
1120 	}
1121 
1122 	if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
1123 	    GRXSTSRD_DPID_DATA0) {
1124 		/* release FIFO */
1125 		dwc_otg_common_rx_ack(sc);
1126 		goto not_complete;
1127 	}
1128 
1129 	DPRINTFN(5, "GRXSTSR=0x%08x\n", sc->sc_last_rx_status);
1130 
1131 	/* clear did stall */
1132 	td->did_stall = 0;
1133 
1134 	/* get the packet byte count */
1135 	count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1136 
1137 	if (count != sizeof(req)) {
1138 		DPRINTFN(0, "Unsupported SETUP packet "
1139 		    "length, %d bytes\n", count);
1140 		/* release FIFO */
1141 		dwc_otg_common_rx_ack(sc);
1142 		goto not_complete;
1143 	}
1144 
1145 	/* read FIFO */
1146 	dwc_otg_read_fifo(sc, td->pc, 0, sizeof(req));
1147 
1148 	/* copy out control request */
1149 	usbd_copy_out(td->pc, 0, &req, sizeof(req));
1150 
1151 	td->offset = sizeof(req);
1152 	td->remainder = 0;
1153 
1154 	/* sneak peek the set address */
1155 	if ((req.bmRequestType == UT_WRITE_DEVICE) &&
1156 	    (req.bRequest == UR_SET_ADDRESS)) {
1157 		/* must write address before ZLP */
1158 		dwc_otg_set_address(sc, req.wValue[0] & 0x7F);
1159 	}
1160 
1161 	/* don't send any data by default */
1162 	DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(0), DIEPCTL_EPDIS);
1163 	DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0), DOEPCTL_EPDIS);
1164 
1165 	/* reset IN endpoint buffer */
1166 	dwc_otg_tx_fifo_reset(sc,
1167 	    GRSTCTL_TXFIFO(0) |
1168 	    GRSTCTL_TXFFLSH);
1169 
1170 	/* acknowledge RX status */
1171 	dwc_otg_common_rx_ack(sc);
1172 	td->did_stall = 1;
1173 
1174 not_complete:
1175 	/* abort any ongoing transfer, before enabling again */
1176 	if (!td->did_stall) {
1177 		td->did_stall = 1;
1178 
1179 		DPRINTFN(5, "stalling IN and OUT direction\n");
1180 
1181 		temp = sc->sc_out_ctl[0];
1182 
1183 		/* set stall after enabling endpoint */
1184 		DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(0),
1185 		    temp | DOEPCTL_STALL);
1186 
1187 		temp = sc->sc_in_ctl[0];
1188 
1189 		/* set stall assuming endpoint is enabled */
1190 		DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(0),
1191 		    temp | DIEPCTL_STALL);
1192 	}
1193 	return (1);			/* not complete */
1194 }
1195 
1196 static uint8_t
1197 dwc_otg_host_rate_check_interrupt(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1198 {
1199 	uint8_t delta;
1200 
1201 	delta = sc->sc_tmr_val - td->tmr_val;
1202 	if (delta >= 128)
1203 		return (1);	/* busy */
1204 
1205 	td->tmr_val = sc->sc_tmr_val + td->tmr_res;
1206 
1207 	/* set toggle, if any */
1208 	if (td->set_toggle) {
1209 		td->set_toggle = 0;
1210 		td->toggle = 1;
1211 	}
1212 	return (0);
1213 }
1214 
1215 static uint8_t
1216 dwc_otg_host_rate_check(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1217 {
1218 	uint8_t frame_num = (uint8_t)sc->sc_last_frame_num;
1219 
1220 	if (td->ep_type == UE_ISOCHRONOUS) {
1221 		/* non TT isochronous traffic */
1222 		if (frame_num & (td->tmr_res - 1))
1223 			goto busy;
1224 		if ((frame_num ^ td->tmr_val) & td->tmr_res)
1225 			goto busy;
1226 		td->tmr_val = td->tmr_res + sc->sc_last_frame_num;
1227 		td->toggle = 0;
1228 		return (0);
1229 	} else if (td->ep_type == UE_INTERRUPT) {
1230 		if (!td->tt_scheduled)
1231 			goto busy;
1232 		td->tt_scheduled = 0;
1233 		return (0);
1234 	} else if (td->did_nak != 0) {
1235 		/* check if we should pause sending queries for 125us */
1236 		if (td->tmr_res == frame_num) {
1237 			/* wait a bit */
1238 			dwc_otg_enable_sof_irq(sc);
1239 			goto busy;
1240 		}
1241 	} else if (td->set_toggle) {
1242 		td->set_toggle = 0;
1243 		td->toggle = 1;
1244 	}
1245 	/* query for data one more time */
1246 	td->tmr_res = frame_num;
1247 	td->did_nak = 0;
1248 	return (0);
1249 busy:
1250 	return (1);
1251 }
1252 
1253 static uint8_t
1254 dwc_otg_host_data_rx_sub(struct dwc_otg_softc *sc, struct dwc_otg_td *td,
1255     uint8_t channel)
1256 {
1257 	uint32_t count;
1258 
1259 	/* check endpoint status */
1260 	if (sc->sc_last_rx_status == 0)
1261 		goto busy;
1262 
1263 	if (channel >= DWC_OTG_MAX_CHANNELS)
1264 		goto busy;
1265 
1266 	if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != channel)
1267 		goto busy;
1268 
1269 	switch (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) {
1270 	case GRXSTSRH_IN_DATA:
1271 
1272 		DPRINTF("DATA ST=%d STATUS=0x%08x\n",
1273 		    (int)td->state, (int)sc->sc_last_rx_status);
1274 
1275 		if (sc->sc_chan_state[channel].hcint & HCINT_SOFTWARE_ONLY) {
1276 			/*
1277 			 * When using SPLIT transactions on interrupt
1278 			 * endpoints, sometimes data occurs twice.
1279 			 */
1280 			DPRINTF("Data already received\n");
1281 			break;
1282 		}
1283 
1284 		/* get the packet byte count */
1285 		count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1286 
1287 		/* check for ISOCHRONOUS endpoint */
1288 		if (td->ep_type == UE_ISOCHRONOUS) {
1289 			if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
1290 			    GRXSTSRD_DPID_DATA0) {
1291 				/* more data to be received */
1292 				td->tt_xactpos = HCSPLT_XACTPOS_MIDDLE;
1293 			} else {
1294 				/* all data received */
1295 				td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
1296 				/* verify the packet byte count */
1297 				if (count != td->remainder) {
1298 					/* we have a short packet */
1299 					td->short_pkt = 1;
1300 					td->got_short = 1;
1301 				}
1302 			}
1303 		} else {
1304 			/* verify the packet byte count */
1305 			if (count != td->max_packet_size) {
1306 				if (count < td->max_packet_size) {
1307 					/* we have a short packet */
1308 					td->short_pkt = 1;
1309 					td->got_short = 1;
1310 				} else {
1311 					/* invalid USB packet */
1312 					td->error_any = 1;
1313 
1314 					/* release FIFO */
1315 					dwc_otg_common_rx_ack(sc);
1316 					goto complete;
1317 				}
1318 			}
1319 			td->toggle ^= 1;
1320 			td->tt_scheduled = 0;
1321 		}
1322 
1323 		/* verify the packet byte count */
1324 		if (count > td->remainder) {
1325 			/* invalid USB packet */
1326 			td->error_any = 1;
1327 
1328 			/* release FIFO */
1329 			dwc_otg_common_rx_ack(sc);
1330 			goto complete;
1331 		}
1332 
1333 		/* read data from FIFO */
1334 		dwc_otg_read_fifo(sc, td->pc, td->offset, count);
1335 
1336 		td->remainder -= count;
1337 		td->offset += count;
1338 		sc->sc_chan_state[channel].hcint |= HCINT_SOFTWARE_ONLY;
1339 		break;
1340 	default:
1341 		break;
1342 	}
1343 	/* release FIFO */
1344 	dwc_otg_common_rx_ack(sc);
1345 busy:
1346 	return (0);
1347 complete:
1348 	return (1);
1349 }
1350 
1351 static uint8_t
1352 dwc_otg_host_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1353 {
1354 	uint32_t hcint = 0;
1355 	uint32_t hcchar;
1356 	uint8_t delta;
1357 	uint8_t channel;
1358 	uint8_t x;
1359 
1360 	for (x = 0; x != td->max_packet_count; x++) {
1361 		channel = td->channel[x];
1362 		if (channel >= DWC_OTG_MAX_CHANNELS)
1363 			continue;
1364 		hcint |= sc->sc_chan_state[channel].hcint;
1365 
1366 		DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1367 		    channel, td->state, hcint,
1368 		    DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1369 		    DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1370 
1371 		/* check interrupt bits */
1372 		if (hcint & (HCINT_RETRY |
1373 		    HCINT_ACK | HCINT_NYET)) {
1374 			/* give success bits priority over failure bits */
1375 		} else if (hcint & HCINT_STALL) {
1376 			DPRINTF("CH=%d STALL\n", channel);
1377 			td->error_stall = 1;
1378 			td->error_any = 1;
1379 			goto complete;
1380 		} else if (hcint & HCINT_ERRORS) {
1381 			DPRINTF("CH=%d ERROR\n", channel);
1382 			td->errcnt++;
1383 			if (td->hcsplt != 0 || td->errcnt >= 3) {
1384 				if (td->ep_type != UE_ISOCHRONOUS) {
1385 					td->error_any = 1;
1386 					goto complete;
1387 				}
1388 			}
1389 		}
1390 
1391 		/* check channels for data, if any */
1392 		if (dwc_otg_host_data_rx_sub(sc, td, channel))
1393 			goto complete;
1394 
1395 		/* refresh interrupt status */
1396 		hcint |= sc->sc_chan_state[channel].hcint;
1397 
1398 		if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1399 		    HCINT_ACK | HCINT_NYET)) {
1400 			if (!(hcint & HCINT_ERRORS))
1401 				td->errcnt = 0;
1402 		}
1403 	}
1404 
1405 	switch (td->state) {
1406 	case DWC_CHAN_ST_START:
1407 		if (td->hcsplt != 0)
1408 			goto receive_spkt;
1409 		else
1410 			goto receive_pkt;
1411 
1412 	case DWC_CHAN_ST_WAIT_ANE:
1413 		if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1414 			if (td->ep_type == UE_INTERRUPT) {
1415 				/*
1416 				 * The USB specification does not
1417 				 * mandate a particular data toggle
1418 				 * value for USB INTERRUPT
1419 				 * transfers. Switch the data toggle
1420 				 * value to receive the packet
1421 				 * correctly:
1422 				 */
1423 				if (hcint & HCINT_DATATGLERR) {
1424 					DPRINTF("Retrying packet due to "
1425 					    "data toggle error\n");
1426 					td->toggle ^= 1;
1427 					goto receive_pkt;
1428 				}
1429 			} else if (td->ep_type == UE_ISOCHRONOUS) {
1430 				if (td->hcsplt != 0) {
1431 					/*
1432 					 * Sometimes the complete
1433 					 * split packet may be queued
1434 					 * too early and the
1435 					 * transaction translator will
1436 					 * return a NAK. Ignore
1437 					 * this message and retry the
1438 					 * complete split instead.
1439 					 */
1440 					DPRINTF("Retrying complete split\n");
1441 					goto receive_pkt;
1442 				}
1443 				goto complete;
1444 			}
1445 			td->did_nak = 1;
1446 			td->tt_scheduled = 0;
1447 			if (td->hcsplt != 0)
1448 				goto receive_spkt;
1449 			else
1450 				goto receive_pkt;
1451 		} else if (hcint & HCINT_NYET) {
1452 			if (td->hcsplt != 0) {
1453 				/* try again */
1454 				goto receive_pkt;
1455 			} else {
1456 				/* not a valid token for IN endpoints */
1457 				td->error_any = 1;
1458 				goto complete;
1459 			}
1460 		} else if (hcint & HCINT_ACK) {
1461 			/* wait for data - ACK arrived first */
1462 			if (!(hcint & HCINT_SOFTWARE_ONLY))
1463 				goto busy;
1464 
1465 			if (td->ep_type == UE_ISOCHRONOUS) {
1466 				/* check if we are complete */
1467 				if (td->tt_xactpos == HCSPLT_XACTPOS_BEGIN) {
1468 					goto complete;
1469 				} else if (td->hcsplt != 0) {
1470 					goto receive_pkt;
1471 				} else {
1472 					/* get more packets */
1473 					goto busy;
1474 				}
1475 			} else {
1476 				/* check if we are complete */
1477 				if ((td->remainder == 0) || (td->got_short != 0)) {
1478 					if (td->short_pkt)
1479 						goto complete;
1480 
1481 					/*
1482 					 * Else need to receive a zero length
1483 					 * packet.
1484 					 */
1485 				}
1486 				td->tt_scheduled = 0;
1487 				td->did_nak = 0;
1488 				if (td->hcsplt != 0)
1489 					goto receive_spkt;
1490 				else
1491 					goto receive_pkt;
1492 			}
1493 		}
1494 		break;
1495 
1496 	case DWC_CHAN_ST_WAIT_S_ANE:
1497 		/*
1498 		 * NOTE: The DWC OTG hardware provides a fake ACK in
1499 		 * case of interrupt and isochronous transfers:
1500 		 */
1501 		if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1502 			td->did_nak = 1;
1503 			td->tt_scheduled = 0;
1504 			goto receive_spkt;
1505 		} else if (hcint & HCINT_NYET) {
1506 			td->tt_scheduled = 0;
1507 			goto receive_spkt;
1508 		} else if (hcint & HCINT_ACK) {
1509 			td->did_nak = 0;
1510 			goto receive_pkt;
1511 		}
1512 		break;
1513 
1514 	case DWC_CHAN_ST_WAIT_C_PKT:
1515 		goto receive_pkt;
1516 
1517 	default:
1518 		break;
1519 	}
1520 	goto busy;
1521 
1522 receive_pkt:
1523 	/* free existing channel, if any */
1524 	dwc_otg_host_channel_free(sc, td);
1525 
1526   	if (td->hcsplt != 0) {
1527 		delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1528 		if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1529 			if (td->ep_type != UE_ISOCHRONOUS) {
1530 				td->state = DWC_CHAN_ST_WAIT_C_PKT;
1531 				goto busy;
1532 			}
1533 		}
1534 		delta = sc->sc_last_frame_num - td->tt_start_slot;
1535 		if (delta > DWC_OTG_TT_SLOT_MAX) {
1536 			if (td->ep_type != UE_ISOCHRONOUS) {
1537 				/* we missed the service interval */
1538 				td->error_any = 1;
1539 			}
1540 			goto complete;
1541 		}
1542 		/* complete split */
1543 		td->hcsplt |= HCSPLT_COMPSPLT;
1544 	} else if (dwc_otg_host_rate_check(sc, td)) {
1545 		td->state = DWC_CHAN_ST_WAIT_C_PKT;
1546 		goto busy;
1547 	}
1548 
1549 	/* allocate a new channel */
1550 	if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1551 		td->state = DWC_CHAN_ST_WAIT_C_PKT;
1552 		goto busy;
1553 	}
1554 
1555 	/* set toggle, if any */
1556 	if (td->set_toggle) {
1557 		td->set_toggle = 0;
1558 		td->toggle = 1;
1559 	}
1560 
1561 	td->state = DWC_CHAN_ST_WAIT_ANE;
1562 
1563 	for (x = 0; x != td->max_packet_count; x++) {
1564 	  	channel = td->channel[x];
1565 
1566 		/* receive one packet */
1567 		DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1568 		    (td->max_packet_size << HCTSIZ_XFERSIZE_SHIFT) |
1569 		    (1 << HCTSIZ_PKTCNT_SHIFT) |
1570 		    (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1571 		    (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1572 
1573 		DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1574 
1575 		hcchar = td->hcchar;
1576 		hcchar |= HCCHAR_EPDIR_IN;
1577 
1578 		if (td->ep_type == UE_ISOCHRONOUS) {
1579 			if (td->hcsplt != 0) {
1580 				/* continously buffer */
1581 				if (sc->sc_last_frame_num & 1)
1582 					hcchar &= ~HCCHAR_ODDFRM;
1583 				else
1584 					hcchar |= HCCHAR_ODDFRM;
1585 			} else {
1586 				/* multi buffer, if any */
1587 				if (sc->sc_last_frame_num & 1)
1588 					hcchar |= HCCHAR_ODDFRM;
1589 				else
1590 					hcchar &= ~HCCHAR_ODDFRM;
1591 			}
1592 		} else {
1593 			hcchar &= ~HCCHAR_ODDFRM;
1594 		}
1595 
1596 		/* must enable channel before data can be received */
1597 		DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1598 	}
1599 	/* wait until next slot before trying complete split */
1600 	td->tt_complete_slot = sc->sc_last_frame_num + 1;
1601 
1602 	goto busy;
1603 
1604 receive_spkt:
1605 	/* free existing channel(s), if any */
1606 	dwc_otg_host_channel_free(sc, td);
1607 
1608 	delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1609 	if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1610 		td->state = DWC_CHAN_ST_START;
1611 		goto busy;
1612 	}
1613 	delta = sc->sc_last_frame_num - td->tt_start_slot;
1614 	if (delta > 5) {
1615 		/* missed it */
1616 		td->tt_scheduled = 0;
1617 		td->state = DWC_CHAN_ST_START;
1618 		goto busy;
1619 	}
1620 
1621 	/* allocate a new channel */
1622 	if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1623 		td->state = DWC_CHAN_ST_START;
1624 		goto busy;
1625 	}
1626 
1627 	channel = td->channel[0];
1628 
1629 	td->hcsplt &= ~HCSPLT_COMPSPLT;
1630 	td->state = DWC_CHAN_ST_WAIT_S_ANE;
1631 
1632 	/* receive one packet */
1633 	DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1634 	    (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
1635 
1636 	DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1637 
1638 	/* send after next SOF event */
1639 	if ((sc->sc_last_frame_num & 1) == 0 &&
1640 	    td->ep_type == UE_ISOCHRONOUS)
1641 		td->hcchar |= HCCHAR_ODDFRM;
1642 	else
1643 		td->hcchar &= ~HCCHAR_ODDFRM;
1644 
1645 	hcchar = td->hcchar;
1646 	hcchar |= HCCHAR_EPDIR_IN;
1647 
1648 	/* wait until next slot before trying complete split */
1649 	td->tt_complete_slot = sc->sc_last_frame_num + 1;
1650 
1651 	/* must enable channel before data can be received */
1652 	DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1653 busy:
1654 	return (1);	/* busy */
1655 
1656 complete:
1657 	dwc_otg_host_channel_free(sc, td);
1658 	return (0);	/* complete */
1659 }
1660 
1661 static uint8_t
1662 dwc_otg_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1663 {
1664 	uint32_t temp;
1665 	uint16_t count;
1666 	uint8_t got_short;
1667 
1668 	got_short = 0;
1669 
1670 	/* check endpoint status */
1671 	if (sc->sc_last_rx_status == 0)
1672 		goto not_complete;
1673 
1674 	if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != td->ep_no)
1675 		goto not_complete;
1676 
1677 	/* check for SETUP packet */
1678 	if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1679 	    GRXSTSRD_STP_DATA ||
1680 	    (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1681 	    GRXSTSRD_STP_COMPLETE) {
1682 		if (td->remainder == 0) {
1683 			/*
1684 			 * We are actually complete and have
1685 			 * received the next SETUP
1686 			 */
1687 			DPRINTFN(5, "faking complete\n");
1688 			return (0);	/* complete */
1689 		}
1690 		/*
1691 		 * USB Host Aborted the transfer.
1692 		 */
1693 		td->error_any = 1;
1694 		return (0);		/* complete */
1695 	}
1696 
1697 	if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1698 	    GRXSTSRD_OUT_DATA) {
1699 		/* release FIFO */
1700 		dwc_otg_common_rx_ack(sc);
1701 		goto not_complete;
1702 	}
1703 
1704 	/* get the packet byte count */
1705 	count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1706 
1707 	/* verify the packet byte count */
1708 	if (count != td->max_packet_size) {
1709 		if (count < td->max_packet_size) {
1710 			/* we have a short packet */
1711 			td->short_pkt = 1;
1712 			got_short = 1;
1713 		} else {
1714 			/* invalid USB packet */
1715 			td->error_any = 1;
1716 
1717 			/* release FIFO */
1718 			dwc_otg_common_rx_ack(sc);
1719 			return (0);	/* we are complete */
1720 		}
1721 	}
1722 	/* verify the packet byte count */
1723 	if (count > td->remainder) {
1724 		/* invalid USB packet */
1725 		td->error_any = 1;
1726 
1727 		/* release FIFO */
1728 		dwc_otg_common_rx_ack(sc);
1729 		return (0);		/* we are complete */
1730 	}
1731 
1732 	/* read data from FIFO */
1733 	dwc_otg_read_fifo(sc, td->pc, td->offset, count);
1734 
1735 	td->remainder -= count;
1736 	td->offset += count;
1737 
1738 	/* release FIFO */
1739 	dwc_otg_common_rx_ack(sc);
1740 
1741 	temp = sc->sc_out_ctl[td->ep_no];
1742 
1743 	/* check for isochronous mode */
1744 	if ((temp & DIEPCTL_EPTYPE_MASK) ==
1745 	    (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
1746 		/* toggle odd or even frame bit */
1747 		if (temp & DIEPCTL_SETD1PID) {
1748 			temp &= ~DIEPCTL_SETD1PID;
1749 			temp |= DIEPCTL_SETD0PID;
1750 		} else {
1751 			temp &= ~DIEPCTL_SETD0PID;
1752 			temp |= DIEPCTL_SETD1PID;
1753 		}
1754 		sc->sc_out_ctl[td->ep_no] = temp;
1755 	}
1756 
1757 	/* check if we are complete */
1758 	if ((td->remainder == 0) || got_short) {
1759 		if (td->short_pkt) {
1760 			/* we are complete */
1761 			return (0);
1762 		}
1763 		/* else need to receive a zero length packet */
1764 	}
1765 
1766 not_complete:
1767 
1768 	/* enable SETUP and transfer complete interrupt */
1769 	if (td->ep_no == 0) {
1770 		DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0),
1771 		    DXEPTSIZ_SET_MULTI(3) |
1772 		    DXEPTSIZ_SET_NPKT(1) |
1773 		    DXEPTSIZ_SET_NBYTES(td->max_packet_size));
1774 	} else {
1775 		/* allow reception of multiple packets */
1776 		DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(td->ep_no),
1777 		    DXEPTSIZ_SET_MULTI(1) |
1778 		    DXEPTSIZ_SET_NPKT(4) |
1779 		    DXEPTSIZ_SET_NBYTES(4 *
1780 		        ((td->max_packet_size + 3) & ~3)));
1781 	}
1782 	temp = sc->sc_out_ctl[td->ep_no];
1783 	DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(td->ep_no), temp |
1784 	    DOEPCTL_EPENA | DOEPCTL_CNAK);
1785 
1786 	return (1);			/* not complete */
1787 }
1788 
1789 static uint8_t
1790 dwc_otg_host_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1791 {
1792 	uint32_t count;
1793 	uint32_t hcint;
1794 	uint32_t hcchar;
1795 	uint8_t delta;
1796 	uint8_t channel;
1797 	uint8_t x;
1798 
1799 	dwc_otg_host_dump_rx(sc, td);
1800 
1801 	/* check that last channel is complete */
1802 	channel = td->channel[td->npkt];
1803 
1804 	if (channel < DWC_OTG_MAX_CHANNELS) {
1805 		hcint = sc->sc_chan_state[channel].hcint;
1806 
1807 		DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1808 		    channel, td->state, hcint,
1809 		    DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1810 		    DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1811 
1812 		if (hcint & (HCINT_RETRY |
1813 		    HCINT_ACK | HCINT_NYET)) {
1814 			/* give success bits priority over failure bits */
1815 		} else if (hcint & HCINT_STALL) {
1816 			DPRINTF("CH=%d STALL\n", channel);
1817 			td->error_stall = 1;
1818 			td->error_any = 1;
1819 			goto complete;
1820 		} else if (hcint & HCINT_ERRORS) {
1821 			DPRINTF("CH=%d ERROR\n", channel);
1822 			td->errcnt++;
1823 			if (td->hcsplt != 0 || td->errcnt >= 3) {
1824 				td->error_any = 1;
1825 				goto complete;
1826 			}
1827 		}
1828 
1829 		if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1830 		    HCINT_ACK | HCINT_NYET)) {
1831 			if (!(hcint & HCINT_ERRORS))
1832 				td->errcnt = 0;
1833 		}
1834 	} else {
1835 		hcint = 0;
1836 	}
1837 
1838 	switch (td->state) {
1839 	case DWC_CHAN_ST_START:
1840 		goto send_pkt;
1841 
1842 	case DWC_CHAN_ST_WAIT_ANE:
1843 		if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1844 			td->did_nak = 1;
1845 			td->tt_scheduled = 0;
1846 			goto send_pkt;
1847 		} else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1848 			td->offset += td->tx_bytes;
1849 			td->remainder -= td->tx_bytes;
1850 			td->toggle ^= 1;
1851 			/* check if next response will be a NAK */
1852 			if (hcint & HCINT_NYET)
1853 				td->did_nak = 1;
1854 			else
1855 				td->did_nak = 0;
1856 			td->tt_scheduled = 0;
1857 
1858 			/* check remainder */
1859 			if (td->remainder == 0) {
1860 				if (td->short_pkt)
1861 					goto complete;
1862 
1863 				/*
1864 				 * Else we need to transmit a short
1865 				 * packet:
1866 				 */
1867 			}
1868 			goto send_pkt;
1869 		}
1870 		break;
1871 
1872 	case DWC_CHAN_ST_WAIT_S_ANE:
1873 		if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1874 			td->did_nak = 1;
1875 			td->tt_scheduled = 0;
1876 			goto send_pkt;
1877 		} else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1878 			td->did_nak = 0;
1879 			goto send_cpkt;
1880 		}
1881 		break;
1882 
1883 	case DWC_CHAN_ST_WAIT_C_ANE:
1884 		if (hcint & HCINT_NYET) {
1885 			goto send_cpkt;
1886 		} else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1887 			td->did_nak = 1;
1888 			td->tt_scheduled = 0;
1889 			goto send_pkt;
1890 		} else if (hcint & HCINT_ACK) {
1891 			td->offset += td->tx_bytes;
1892 			td->remainder -= td->tx_bytes;
1893 			td->toggle ^= 1;
1894 			td->did_nak = 0;
1895 			td->tt_scheduled = 0;
1896 
1897 			/* check remainder */
1898 			if (td->remainder == 0) {
1899 				if (td->short_pkt)
1900 					goto complete;
1901 
1902 				/* else we need to transmit a short packet */
1903 			}
1904 			goto send_pkt;
1905 		}
1906 		break;
1907 
1908 	case DWC_CHAN_ST_WAIT_C_PKT:
1909 		goto send_cpkt;
1910 
1911 	case DWC_CHAN_ST_TX_WAIT_ISOC:
1912 		/* Check if ISOCHRONOUS OUT traffic is complete */
1913 		if ((hcint & HCINT_HCH_DONE_MASK) == 0)
1914 			break;
1915 
1916 		td->offset += td->tx_bytes;
1917 		td->remainder -= td->tx_bytes;
1918 		goto complete;
1919 	default:
1920 		break;
1921 	}
1922 	goto busy;
1923 
1924 send_pkt:
1925 	/* free existing channel(s), if any */
1926 	dwc_otg_host_channel_free(sc, td);
1927 
1928 	if (td->hcsplt != 0) {
1929 		delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1930 		if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1931 			td->state = DWC_CHAN_ST_START;
1932 			goto busy;
1933 		}
1934 		delta = sc->sc_last_frame_num - td->tt_start_slot;
1935 		if (delta > 5) {
1936 			/* missed it */
1937 			td->tt_scheduled = 0;
1938 			td->state = DWC_CHAN_ST_START;
1939 			goto busy;
1940 		}
1941 	} else if (dwc_otg_host_rate_check(sc, td)) {
1942 		td->state = DWC_CHAN_ST_START;
1943 		goto busy;
1944 	}
1945 
1946 	/* allocate a new channel */
1947 	if (dwc_otg_host_channel_alloc(sc, td, 1)) {
1948 		td->state = DWC_CHAN_ST_START;
1949 		goto busy;
1950 	}
1951 
1952 	/* set toggle, if any */
1953 	if (td->set_toggle) {
1954 		td->set_toggle = 0;
1955 		td->toggle = 1;
1956 	}
1957 
1958 	if (td->ep_type == UE_ISOCHRONOUS) {
1959 		/* ISOCHRONOUS OUT transfers don't have any ACKs */
1960 		td->state = DWC_CHAN_ST_TX_WAIT_ISOC;
1961 		td->hcsplt &= ~HCSPLT_COMPSPLT;
1962 		if (td->hcsplt != 0) {
1963 			/* get maximum transfer length */
1964 			count = td->remainder;
1965 			if (count > HCSPLT_XACTLEN_BURST) {
1966 				DPRINTF("TT overflow\n");
1967 				td->error_any = 1;
1968 				goto complete;
1969 			}
1970 			/* Update transaction position */
1971 			td->hcsplt &= ~HCSPLT_XACTPOS_MASK;
1972 			td->hcsplt |= (HCSPLT_XACTPOS_ALL << HCSPLT_XACTPOS_SHIFT);
1973 		}
1974 	} else if (td->hcsplt != 0) {
1975 		td->hcsplt &= ~HCSPLT_COMPSPLT;
1976 		/* Wait for ACK/NAK/ERR from TT */
1977 		td->state = DWC_CHAN_ST_WAIT_S_ANE;
1978 	} else {
1979 		/* Wait for ACK/NAK/STALL from device */
1980 		td->state = DWC_CHAN_ST_WAIT_ANE;
1981 	}
1982 
1983 	td->tx_bytes = 0;
1984 
1985 	for (x = 0; x != td->max_packet_count; x++) {
1986 		uint32_t rem_bytes;
1987 
1988 		channel = td->channel[x];
1989 
1990 		/* send one packet at a time */
1991 		count = td->max_packet_size;
1992 		rem_bytes = td->remainder - td->tx_bytes;
1993 		if (rem_bytes < count) {
1994 			/* we have a short packet */
1995 			td->short_pkt = 1;
1996 			count = rem_bytes;
1997 		}
1998 		if (count == rem_bytes) {
1999 			/* last packet */
2000 			switch (x) {
2001 			case 0:
2002 				DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2003 				    (count << HCTSIZ_XFERSIZE_SHIFT) |
2004 				    (1 << HCTSIZ_PKTCNT_SHIFT) |
2005 				    (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
2006 				    (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
2007 				break;
2008 			case 1:
2009 				DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2010 				    (count << HCTSIZ_XFERSIZE_SHIFT) |
2011 				    (1 << HCTSIZ_PKTCNT_SHIFT) |
2012 				    (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT));
2013 				break;
2014 			default:
2015 				DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2016 				    (count << HCTSIZ_XFERSIZE_SHIFT) |
2017 				    (1 << HCTSIZ_PKTCNT_SHIFT) |
2018 				    (HCTSIZ_PID_DATA2 << HCTSIZ_PID_SHIFT));
2019 				break;
2020 			}
2021 		} else if (td->ep_type == UE_ISOCHRONOUS &&
2022 			   td->max_packet_count > 1) {
2023 			/* ISOCHRONOUS multi packet */
2024 			DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2025 			    (count << HCTSIZ_XFERSIZE_SHIFT) |
2026 			    (1 << HCTSIZ_PKTCNT_SHIFT) |
2027 			    (HCTSIZ_PID_MDATA << HCTSIZ_PID_SHIFT));
2028 		} else {
2029 			/* TODO: HCTSIZ_DOPNG */
2030 			/* standard BULK/INTERRUPT/CONTROL packet */
2031 			DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2032 			    (count << HCTSIZ_XFERSIZE_SHIFT) |
2033 			    (1 << HCTSIZ_PKTCNT_SHIFT) |
2034 			    (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
2035 			    (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
2036 		}
2037 
2038 		DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
2039 
2040 		hcchar = td->hcchar;
2041 		hcchar &= ~HCCHAR_EPDIR_IN;
2042 
2043 		/* send after next SOF event */
2044 		if ((sc->sc_last_frame_num & 1) == 0 &&
2045 		    td->ep_type == UE_ISOCHRONOUS)
2046 			hcchar |= HCCHAR_ODDFRM;
2047 		else
2048 			hcchar &= ~HCCHAR_ODDFRM;
2049 
2050 		/* must enable before writing data to FIFO */
2051 		DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
2052 
2053 		if (count != 0) {
2054 			/* write data into FIFO */
2055 			dwc_otg_write_fifo(sc, td->pc, td->offset +
2056 			    td->tx_bytes, DOTG_DFIFO(channel), count);
2057 		}
2058 
2059 		/* store number of bytes transmitted */
2060 		td->tx_bytes += count;
2061 
2062 		/* store last packet index */
2063 		td->npkt = x;
2064 
2065 		/* check for last packet */
2066 		if (count == rem_bytes)
2067 			break;
2068 	}
2069 	goto busy;
2070 
2071 send_cpkt:
2072 	/* free existing channel, if any */
2073 	dwc_otg_host_channel_free(sc, td);
2074 
2075 	delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
2076 	if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
2077 		td->state = DWC_CHAN_ST_WAIT_C_PKT;
2078 		goto busy;
2079 	}
2080 	delta = sc->sc_last_frame_num - td->tt_start_slot;
2081 	if (delta > DWC_OTG_TT_SLOT_MAX) {
2082 		/* we missed the service interval */
2083 		if (td->ep_type != UE_ISOCHRONOUS)
2084 			td->error_any = 1;
2085 		goto complete;
2086 	}
2087 
2088 	/* allocate a new channel */
2089 	if (dwc_otg_host_channel_alloc(sc, td, 0)) {
2090 		td->state = DWC_CHAN_ST_WAIT_C_PKT;
2091 		goto busy;
2092 	}
2093 
2094 	channel = td->channel[0];
2095 
2096  	td->hcsplt |= HCSPLT_COMPSPLT;
2097 	td->state = DWC_CHAN_ST_WAIT_C_ANE;
2098 
2099 	DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2100 	    (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
2101 
2102 	DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
2103 
2104 	hcchar = td->hcchar;
2105 	hcchar &= ~HCCHAR_EPDIR_IN;
2106 
2107 	/* receive complete split ASAP */
2108 	if ((sc->sc_last_frame_num & 1) != 0 &&
2109 	    td->ep_type == UE_ISOCHRONOUS)
2110 		hcchar |= HCCHAR_ODDFRM;
2111 	else
2112 		hcchar &= ~HCCHAR_ODDFRM;
2113 
2114 	/* must enable channel before data can be received */
2115 	DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
2116 
2117 	/* wait until next slot before trying complete split */
2118 	td->tt_complete_slot = sc->sc_last_frame_num + 1;
2119 busy:
2120 	return (1);	/* busy */
2121 
2122 complete:
2123 	dwc_otg_host_channel_free(sc, td);
2124 	return (0);	/* complete */
2125 }
2126 
2127 static uint8_t
2128 dwc_otg_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
2129 {
2130 	uint32_t max_buffer;
2131 	uint32_t count;
2132 	uint32_t fifo_left;
2133 	uint32_t mpkt;
2134 	uint32_t temp;
2135 	uint8_t to;
2136 
2137 	to = 3;				/* don't loop forever! */
2138 
2139 	max_buffer = sc->sc_hw_ep_profile[td->ep_no].max_buffer;
2140 
2141 repeat:
2142 	/* check for for endpoint 0 data */
2143 
2144 	temp = sc->sc_last_rx_status;
2145 
2146 	if ((td->ep_no == 0) && (temp != 0) &&
2147 	    (GRXSTSRD_CHNUM_GET(temp) == 0)) {
2148 		if ((temp & GRXSTSRD_PKTSTS_MASK) !=
2149 		    GRXSTSRD_STP_DATA &&
2150 		    (temp & GRXSTSRD_PKTSTS_MASK) !=
2151 		    GRXSTSRD_STP_COMPLETE) {
2152 			/* dump data - wrong direction */
2153 			dwc_otg_common_rx_ack(sc);
2154 		} else {
2155 			/*
2156 			 * The current transfer was cancelled
2157 			 * by the USB Host:
2158 			 */
2159 			td->error_any = 1;
2160 			return (0);		/* complete */
2161 		}
2162 	}
2163 
2164 	/* fill in more TX data, if possible */
2165 	if (td->tx_bytes != 0) {
2166 		uint16_t cpkt;
2167 
2168 		/* check if packets have been transferred */
2169 		temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2170 
2171 		/* get current packet number */
2172 		cpkt = DXEPTSIZ_GET_NPKT(temp);
2173 
2174 		if (cpkt >= td->npkt) {
2175 			fifo_left = 0;
2176 		} else {
2177 			if (max_buffer != 0) {
2178 				fifo_left = (td->npkt - cpkt) *
2179 				    td->max_packet_size;
2180 
2181 				if (fifo_left > max_buffer)
2182 					fifo_left = max_buffer;
2183 			} else {
2184 				fifo_left = td->max_packet_size;
2185 			}
2186 		}
2187 
2188 		count = td->tx_bytes;
2189 		if (count > fifo_left)
2190 			count = fifo_left;
2191 
2192 		if (count != 0) {
2193 			/* write data into FIFO */
2194 			dwc_otg_write_fifo(sc, td->pc, td->offset,
2195 			    DOTG_DFIFO(td->ep_no), count);
2196 
2197 			td->tx_bytes -= count;
2198 			td->remainder -= count;
2199 			td->offset += count;
2200 			td->npkt = cpkt;
2201 		}
2202 		if (td->tx_bytes != 0)
2203 			goto not_complete;
2204 
2205 		/* check remainder */
2206 		if (td->remainder == 0) {
2207 			if (td->short_pkt)
2208 				return (0);	/* complete */
2209 
2210 			/* else we need to transmit a short packet */
2211 		}
2212 	}
2213 
2214 	if (!to--)
2215 		goto not_complete;
2216 
2217 	/* check if not all packets have been transferred */
2218 	temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2219 
2220 	if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2221 		DPRINTFN(5, "busy ep=%d npkt=%d DIEPTSIZ=0x%08x "
2222 		    "DIEPCTL=0x%08x\n", td->ep_no,
2223 		    DXEPTSIZ_GET_NPKT(temp),
2224 		    temp, DWC_OTG_READ_4(sc, DOTG_DIEPCTL(td->ep_no)));
2225 
2226 		goto not_complete;
2227 	}
2228 
2229 	DPRINTFN(5, "rem=%u ep=%d\n", td->remainder, td->ep_no);
2230 
2231 	/* try to optimise by sending more data */
2232 	if ((max_buffer != 0) && ((td->max_packet_size & 3) == 0)) {
2233 		/* send multiple packets at the same time */
2234 		mpkt = max_buffer / td->max_packet_size;
2235 
2236 		if (mpkt > 0x3FE)
2237 			mpkt = 0x3FE;
2238 
2239 		count = td->remainder;
2240 		if (count > 0x7FFFFF)
2241 			count = 0x7FFFFF - (0x7FFFFF % td->max_packet_size);
2242 
2243 		td->npkt = count / td->max_packet_size;
2244 
2245 		/*
2246 		 * NOTE: We could use 0x3FE instead of "mpkt" in the
2247 		 * check below to get more throughput, but then we
2248 		 * have a dependency towards non-generic chip features
2249 		 * to disable the TX-FIFO-EMPTY interrupts on a per
2250 		 * endpoint basis. Increase the maximum buffer size of
2251 		 * the IN endpoint to increase the performance.
2252 		 */
2253 		if (td->npkt > mpkt) {
2254 			td->npkt = mpkt;
2255 			count = td->max_packet_size * mpkt;
2256 		} else if ((count == 0) || (count % td->max_packet_size)) {
2257 			/* we are transmitting a short packet */
2258 			td->npkt++;
2259 			td->short_pkt = 1;
2260 		}
2261 	} else {
2262 		/* send one packet at a time */
2263 		mpkt = 1;
2264 		count = td->max_packet_size;
2265 		if (td->remainder < count) {
2266 			/* we have a short packet */
2267 			td->short_pkt = 1;
2268 			count = td->remainder;
2269 		}
2270 		td->npkt = 1;
2271 	}
2272 	DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(td->ep_no),
2273 	    DXEPTSIZ_SET_MULTI(1) |
2274 	    DXEPTSIZ_SET_NPKT(td->npkt) |
2275 	    DXEPTSIZ_SET_NBYTES(count));
2276 
2277 	/* make room for buffering */
2278 	td->npkt += mpkt;
2279 
2280 	temp = sc->sc_in_ctl[td->ep_no];
2281 
2282 	/* check for isochronous mode */
2283 	if ((temp & DIEPCTL_EPTYPE_MASK) ==
2284 	    (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
2285 		/* toggle odd or even frame bit */
2286 		if (temp & DIEPCTL_SETD1PID) {
2287 			temp &= ~DIEPCTL_SETD1PID;
2288 			temp |= DIEPCTL_SETD0PID;
2289 		} else {
2290 			temp &= ~DIEPCTL_SETD0PID;
2291 			temp |= DIEPCTL_SETD1PID;
2292 		}
2293 		sc->sc_in_ctl[td->ep_no] = temp;
2294 	}
2295 
2296 	/* must enable before writing data to FIFO */
2297 	DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(td->ep_no), temp |
2298 	    DIEPCTL_EPENA | DIEPCTL_CNAK);
2299 
2300 	td->tx_bytes = count;
2301 
2302 	/* check remainder */
2303 	if (td->tx_bytes == 0 &&
2304 	    td->remainder == 0) {
2305 		if (td->short_pkt)
2306 			return (0);	/* complete */
2307 
2308 		/* else we need to transmit a short packet */
2309 	}
2310 	goto repeat;
2311 
2312 not_complete:
2313 	return (1);			/* not complete */
2314 }
2315 
2316 static uint8_t
2317 dwc_otg_data_tx_sync(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
2318 {
2319 	uint32_t temp;
2320 
2321 	/*
2322 	 * If all packets are transferred we are complete:
2323 	 */
2324 	temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2325 
2326 	/* check that all packets have been transferred */
2327 	if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2328 		DPRINTFN(5, "busy ep=%d\n", td->ep_no);
2329 		goto not_complete;
2330 	}
2331 	return (0);
2332 
2333 not_complete:
2334 
2335 	/* we only want to know if there is a SETUP packet or free IN packet */
2336 
2337 	temp = sc->sc_last_rx_status;
2338 
2339 	if ((td->ep_no == 0) && (temp != 0) &&
2340 	    (GRXSTSRD_CHNUM_GET(temp) == 0)) {
2341 		if ((temp & GRXSTSRD_PKTSTS_MASK) ==
2342 		    GRXSTSRD_STP_DATA ||
2343 		    (temp & GRXSTSRD_PKTSTS_MASK) ==
2344 		    GRXSTSRD_STP_COMPLETE) {
2345 			DPRINTFN(5, "faking complete\n");
2346 			/*
2347 			 * Race condition: We are complete!
2348 			 */
2349 			return (0);
2350 		} else {
2351 			/* dump data - wrong direction */
2352 			dwc_otg_common_rx_ack(sc);
2353 		}
2354 	}
2355 	return (1);			/* not complete */
2356 }
2357 
2358 static void
2359 dwc_otg_xfer_do_fifo(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2360 {
2361 	struct dwc_otg_td *td;
2362 	uint8_t toggle;
2363 	uint8_t tmr_val;
2364 	uint8_t tmr_res;
2365 
2366 	DPRINTFN(9, "\n");
2367 
2368 	td = xfer->td_transfer_cache;
2369 	if (td == NULL)
2370 		return;
2371 
2372 	while (1) {
2373 		if ((td->func) (sc, td)) {
2374 			/* operation in progress */
2375 			break;
2376 		}
2377 		if (((void *)td) == xfer->td_transfer_last) {
2378 			goto done;
2379 		}
2380 		if (td->error_any) {
2381 			goto done;
2382 		} else if (td->remainder > 0) {
2383 			/*
2384 			 * We had a short transfer. If there is no alternate
2385 			 * next, stop processing !
2386 			 */
2387 			if (!td->alt_next)
2388 				goto done;
2389 		}
2390 
2391 		/*
2392 		 * Fetch the next transfer descriptor and transfer
2393 		 * some flags to the next transfer descriptor
2394 		 */
2395 		tmr_res = td->tmr_res;
2396 		tmr_val = td->tmr_val;
2397 		toggle = td->toggle;
2398 		td = td->obj_next;
2399 		xfer->td_transfer_cache = td;
2400 		td->toggle = toggle;	/* transfer toggle */
2401 		td->tmr_res = tmr_res;
2402 		td->tmr_val = tmr_val;
2403 	}
2404 	return;
2405 
2406 done:
2407 	xfer->td_transfer_cache = NULL;
2408 	sc->sc_xfer_complete = 1;
2409 }
2410 
2411 static uint8_t
2412 dwc_otg_xfer_do_complete_locked(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2413 {
2414 	struct dwc_otg_td *td;
2415 
2416 	DPRINTFN(9, "\n");
2417 
2418 	td = xfer->td_transfer_cache;
2419 	if (td == NULL) {
2420 		/* compute all actual lengths */
2421 		dwc_otg_standard_done(xfer);
2422 		return (1);
2423 	}
2424 	return (0);
2425 }
2426 
2427 static void
2428 dwc_otg_timer(void *_sc)
2429 {
2430 	struct dwc_otg_softc *sc = _sc;
2431 
2432 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2433 
2434 	DPRINTF("\n");
2435 
2436 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
2437 
2438 	/* increment timer value */
2439 	sc->sc_tmr_val++;
2440 
2441 	/* enable SOF interrupt, which will poll jobs */
2442 	dwc_otg_enable_sof_irq(sc);
2443 
2444 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2445 
2446 	if (sc->sc_timer_active) {
2447 		/* restart timer */
2448 		usb_callout_reset(&sc->sc_timer,
2449 		    hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2450 		    &dwc_otg_timer, sc);
2451 	}
2452 }
2453 
2454 static void
2455 dwc_otg_timer_start(struct dwc_otg_softc *sc)
2456 {
2457 	if (sc->sc_timer_active != 0)
2458 		return;
2459 
2460 	sc->sc_timer_active = 1;
2461 
2462 	/* restart timer */
2463 	usb_callout_reset(&sc->sc_timer,
2464 	    hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2465 	    &dwc_otg_timer, sc);
2466 }
2467 
2468 static void
2469 dwc_otg_timer_stop(struct dwc_otg_softc *sc)
2470 {
2471 	if (sc->sc_timer_active == 0)
2472 		return;
2473 
2474 	sc->sc_timer_active = 0;
2475 
2476 	/* stop timer */
2477 	usb_callout_stop(&sc->sc_timer);
2478 }
2479 
2480 static uint16_t
2481 dwc_otg_compute_isoc_rx_tt_slot(struct dwc_otg_tt_info *pinfo)
2482 {
2483 	if (pinfo->slot_index < DWC_OTG_TT_SLOT_MAX)
2484 		pinfo->slot_index++;
2485 	return (pinfo->slot_index);
2486 }
2487 
2488 static uint8_t
2489 dwc_otg_update_host_transfer_schedule_locked(struct dwc_otg_softc *sc)
2490 {
2491 	TAILQ_HEAD(, usb_xfer) head;
2492 	struct usb_xfer *xfer;
2493 	struct usb_xfer *xfer_next;
2494 	struct dwc_otg_td *td;
2495 	uint16_t temp;
2496 	uint16_t slot;
2497 
2498 	temp = DWC_OTG_READ_4(sc, DOTG_HFNUM) & DWC_OTG_FRAME_MASK;
2499 
2500 	if (sc->sc_last_frame_num == temp)
2501 		return (0);
2502 
2503 	sc->sc_last_frame_num = temp;
2504 
2505 	TAILQ_INIT(&head);
2506 
2507 	if ((temp & 7) == 0) {
2508 		/* reset the schedule */
2509 		memset(sc->sc_tt_info, 0, sizeof(sc->sc_tt_info));
2510 
2511 		TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2512 			td = xfer->td_transfer_cache;
2513 			if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2514 				continue;
2515 
2516 			/* check for IN direction */
2517 			if ((td->hcchar & HCCHAR_EPDIR_IN) != 0)
2518 				continue;
2519 
2520 			sc->sc_needsof = 1;
2521 
2522 			if (td->hcsplt == 0 || td->tt_scheduled != 0)
2523 				continue;
2524 
2525 			/* compute slot */
2526 			slot = dwc_otg_compute_isoc_rx_tt_slot(
2527 			    sc->sc_tt_info + td->tt_index);
2528 			if (slot > 3) {
2529 				/*
2530 				 * Not enough time to get complete
2531 				 * split executed.
2532 				 */
2533 				continue;
2534 			}
2535 			/* Delayed start */
2536 			td->tt_start_slot = temp + slot;
2537 			td->tt_scheduled = 1;
2538 			TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2539 			TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2540 		}
2541 
2542 		TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2543 			td = xfer->td_transfer_cache;
2544 			if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2545 				continue;
2546 
2547 			/* check for OUT direction */
2548 			if ((td->hcchar & HCCHAR_EPDIR_IN) == 0)
2549 				continue;
2550 
2551 			sc->sc_needsof = 1;
2552 
2553 			if (td->hcsplt == 0 || td->tt_scheduled != 0)
2554 				continue;
2555 
2556 			/* Start ASAP */
2557 			td->tt_start_slot = temp;
2558 			td->tt_scheduled = 1;
2559 			TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2560 			TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2561 		}
2562 
2563 		TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2564 			td = xfer->td_transfer_cache;
2565 			if (td == NULL || td->ep_type != UE_INTERRUPT)
2566 				continue;
2567 
2568 			if (td->tt_scheduled != 0) {
2569 				sc->sc_needsof = 1;
2570 				continue;
2571 			}
2572 
2573 			if (dwc_otg_host_rate_check_interrupt(sc, td))
2574 				continue;
2575 
2576 			if (td->hcsplt == 0) {
2577 				sc->sc_needsof = 1;
2578 				td->tt_scheduled = 1;
2579 				continue;
2580 			}
2581 
2582 			/* start ASAP */
2583 			td->tt_start_slot = temp;
2584 			sc->sc_needsof = 1;
2585 			td->tt_scheduled = 1;
2586 			TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2587 			TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2588 		}
2589 
2590 		TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2591 			td = xfer->td_transfer_cache;
2592 			if (td == NULL ||
2593 			    td->ep_type != UE_CONTROL) {
2594 				continue;
2595 			}
2596 
2597 			sc->sc_needsof = 1;
2598 
2599 			if (td->hcsplt == 0 || td->tt_scheduled != 0)
2600 				continue;
2601 
2602 			/* start ASAP */
2603 			td->tt_start_slot = temp;
2604 			td->tt_scheduled = 1;
2605 			TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2606 			TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2607 		}
2608 	}
2609 	if ((temp & 7) < 6) {
2610 		TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2611 			td = xfer->td_transfer_cache;
2612 			if (td == NULL ||
2613 			    td->ep_type != UE_BULK) {
2614 				continue;
2615 			}
2616 
2617 			sc->sc_needsof = 1;
2618 
2619 			if (td->hcsplt == 0 || td->tt_scheduled != 0)
2620 				continue;
2621 
2622 			/* start ASAP */
2623 			td->tt_start_slot = temp;
2624 			td->tt_scheduled = 1;
2625 			TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2626 			TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2627 		}
2628 	}
2629 
2630 	/* Put TT transfers in execution order at the end */
2631 	TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2632 
2633 	/* move all TT transfers in front, keeping the current order */
2634 	TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2635 		td = xfer->td_transfer_cache;
2636 		if (td == NULL || td->hcsplt == 0)
2637 			continue;
2638 		TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2639 		TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2640 	}
2641 	TAILQ_CONCAT(&head, &sc->sc_bus.intr_q.head, wait_entry);
2642 	TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2643 
2644 	/* put non-TT non-ISOCHRONOUS transfers last */
2645 	TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2646 		td = xfer->td_transfer_cache;
2647 		if (td == NULL || td->hcsplt != 0 || td->ep_type == UE_ISOCHRONOUS)
2648 			continue;
2649 		TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2650 		TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2651 	}
2652 	TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2653 
2654 	if ((temp & 7) == 0) {
2655 		DPRINTFN(12, "SOF interrupt #%d, needsof=%d\n",
2656 		    (int)temp, (int)sc->sc_needsof);
2657 
2658 		/* update SOF IRQ mask */
2659 		if (sc->sc_irq_mask & GINTMSK_SOFMSK) {
2660 			if (sc->sc_needsof == 0) {
2661 				sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2662 				DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2663 			}
2664 		} else {
2665 			if (sc->sc_needsof != 0) {
2666 				sc->sc_irq_mask |= GINTMSK_SOFMSK;
2667 				DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2668 			}
2669 		}
2670 
2671 		/* clear need SOF flag */
2672 		sc->sc_needsof = 0;
2673 	}
2674 	return (1);
2675 }
2676 
2677 static void
2678 dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *sc)
2679 {
2680 	struct usb_xfer *xfer;
2681 	uint32_t count;
2682 	uint32_t temp;
2683 	uint32_t haint;
2684 	uint8_t got_rx_status;
2685 	uint8_t x;
2686 
2687 	if (sc->sc_flags.status_device_mode == 0) {
2688 		/*
2689 		 * Update host transfer schedule, so that new
2690 		 * transfers can be issued:
2691 		 */
2692 		dwc_otg_update_host_transfer_schedule_locked(sc);
2693 	}
2694 	count = 0;
2695 repeat:
2696 	if (++count == 16) {
2697 		/* give other interrupts a chance */
2698 		DPRINTF("Yield\n");
2699 		return;
2700 	}
2701 
2702 	/* get all host channel interrupts */
2703 	haint = DWC_OTG_READ_4(sc, DOTG_HAINT);
2704 	while (1) {
2705 		x = ffs(haint) - 1;
2706 		if (x >= sc->sc_host_ch_max)
2707 			break;
2708 		temp = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
2709 		DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), temp);
2710 		temp &= ~HCINT_SOFTWARE_ONLY;
2711 		sc->sc_chan_state[x].hcint |= temp;
2712 		haint &= ~(1U << x);
2713 	}
2714 
2715 	if (sc->sc_last_rx_status == 0) {
2716 		temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2717 		if (temp & GINTSTS_RXFLVL) {
2718 			/* pop current status */
2719 			sc->sc_last_rx_status =
2720 			    DWC_OTG_READ_4(sc, DOTG_GRXSTSPD);
2721 		}
2722 
2723 		if (sc->sc_last_rx_status != 0) {
2724 			uint8_t ep_no;
2725 
2726 			temp = sc->sc_last_rx_status &
2727 			    GRXSTSRD_PKTSTS_MASK;
2728 
2729 			/* non-data messages we simply skip */
2730 			if (temp != GRXSTSRD_STP_DATA &&
2731 			    temp != GRXSTSRD_STP_COMPLETE &&
2732 			    temp != GRXSTSRD_OUT_DATA) {
2733 				/* check for halted channel */
2734 				if (temp == GRXSTSRH_HALTED) {
2735 					ep_no = GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status);
2736 					sc->sc_chan_state[ep_no].wait_halted = 0;
2737 					DPRINTFN(5, "channel halt complete ch=%u\n", ep_no);
2738 				}
2739 				/* store bytes and FIFO offset */
2740 				sc->sc_current_rx_bytes = 0;
2741 				sc->sc_current_rx_fifo = 0;
2742 
2743 				/* acknowledge status */
2744 				dwc_otg_common_rx_ack(sc);
2745 				goto repeat;
2746 			}
2747 
2748 			temp = GRXSTSRD_BCNT_GET(
2749 			    sc->sc_last_rx_status);
2750 			ep_no = GRXSTSRD_CHNUM_GET(
2751 			    sc->sc_last_rx_status);
2752 
2753 			/* store bytes and FIFO offset */
2754 			sc->sc_current_rx_bytes = (temp + 3) & ~3;
2755 			sc->sc_current_rx_fifo = DOTG_DFIFO(ep_no);
2756 
2757 			DPRINTF("Reading %d bytes from ep %d\n", temp, ep_no);
2758 
2759 			/* check if we should dump the data */
2760 			if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2761 				dwc_otg_common_rx_ack(sc);
2762 				goto repeat;
2763 			}
2764 
2765 			got_rx_status = 1;
2766 
2767 			DPRINTFN(5, "RX status = 0x%08x: ch=%d pid=%d bytes=%d sts=%d\n",
2768 			    sc->sc_last_rx_status, ep_no,
2769 			    (sc->sc_last_rx_status >> 15) & 3,
2770 			    GRXSTSRD_BCNT_GET(sc->sc_last_rx_status),
2771 			    (sc->sc_last_rx_status >> 17) & 15);
2772 		} else {
2773 			got_rx_status = 0;
2774 		}
2775 	} else {
2776 		uint8_t ep_no;
2777 
2778 		ep_no = GRXSTSRD_CHNUM_GET(
2779 		    sc->sc_last_rx_status);
2780 
2781 		/* check if we should dump the data */
2782 		if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2783 			dwc_otg_common_rx_ack(sc);
2784 			goto repeat;
2785 		}
2786 
2787 		got_rx_status = 1;
2788 	}
2789 
2790 	/* execute FIFOs */
2791 	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry)
2792 		dwc_otg_xfer_do_fifo(sc, xfer);
2793 
2794 	if (got_rx_status) {
2795 		/* check if data was consumed */
2796 		if (sc->sc_last_rx_status == 0)
2797 			goto repeat;
2798 
2799 		/* disable RX FIFO level interrupt */
2800 		sc->sc_irq_mask &= ~GINTMSK_RXFLVLMSK;
2801 		DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2802 	}
2803 }
2804 
2805 static void
2806 dwc_otg_interrupt_complete_locked(struct dwc_otg_softc *sc)
2807 {
2808 	struct usb_xfer *xfer;
2809 repeat:
2810 	/* scan for completion events */
2811 	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
2812 		if (dwc_otg_xfer_do_complete_locked(sc, xfer))
2813 			goto repeat;
2814 	}
2815 }
2816 
2817 static void
2818 dwc_otg_vbus_interrupt(struct dwc_otg_softc *sc, uint8_t is_on)
2819 {
2820 	DPRINTFN(5, "vbus = %u\n", is_on);
2821 
2822 	/*
2823 	 * If the USB host mode is forced, then assume VBUS is always
2824 	 * present else rely on the input to this function:
2825 	 */
2826 	if ((is_on != 0) || (sc->sc_mode == DWC_MODE_HOST)) {
2827 		if (!sc->sc_flags.status_vbus) {
2828 			sc->sc_flags.status_vbus = 1;
2829 
2830 			/* complete root HUB interrupt endpoint */
2831 
2832 			dwc_otg_root_intr(sc);
2833 		}
2834 	} else {
2835 		if (sc->sc_flags.status_vbus) {
2836 			sc->sc_flags.status_vbus = 0;
2837 			sc->sc_flags.status_bus_reset = 0;
2838 			sc->sc_flags.status_suspend = 0;
2839 			sc->sc_flags.change_suspend = 0;
2840 			sc->sc_flags.change_connect = 1;
2841 
2842 			/* complete root HUB interrupt endpoint */
2843 
2844 			dwc_otg_root_intr(sc);
2845 		}
2846 	}
2847 }
2848 
2849 int
2850 dwc_otg_filter_interrupt(void *arg)
2851 {
2852 	struct dwc_otg_softc *sc = arg;
2853 	int retval = FILTER_HANDLED;
2854 	uint32_t status;
2855 
2856 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
2857 
2858 	/* read and clear interrupt status */
2859 	status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2860 
2861 	/* clear interrupts we are handling here */
2862 	DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & ~DWC_OTG_MSK_GINT_THREAD_IRQ);
2863 
2864 	/* check for USB state change interrupts */
2865 	if ((status & DWC_OTG_MSK_GINT_THREAD_IRQ) != 0)
2866 		retval = FILTER_SCHEDULE_THREAD;
2867 
2868 	/* clear FIFO empty interrupts */
2869 	if (status & sc->sc_irq_mask &
2870 	    (GINTSTS_PTXFEMP | GINTSTS_NPTXFEMP)) {
2871 		sc->sc_irq_mask &= ~(GINTSTS_PTXFEMP | GINTSTS_NPTXFEMP);
2872 		DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2873 	}
2874 	/* clear all IN endpoint interrupts */
2875 	if (status & GINTSTS_IEPINT) {
2876 		uint32_t temp;
2877 		uint8_t x;
2878 
2879 		for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
2880 			temp = DWC_OTG_READ_4(sc, DOTG_DIEPINT(x));
2881 			/*
2882 			 * NOTE: Need to clear all interrupt bits,
2883 			 * because some appears to be unmaskable and
2884 			 * can cause an interrupt loop:
2885 			 */
2886 			if (temp != 0)
2887 				DWC_OTG_WRITE_4(sc, DOTG_DIEPINT(x), temp);
2888 		}
2889 	}
2890 
2891 	/* poll FIFOs, if any */
2892 	dwc_otg_interrupt_poll_locked(sc);
2893 
2894 	if (sc->sc_xfer_complete != 0)
2895 		retval = FILTER_SCHEDULE_THREAD;
2896 
2897 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2898 
2899 	return (retval);
2900 }
2901 
2902 void
2903 dwc_otg_interrupt(void *arg)
2904 {
2905 	struct dwc_otg_softc *sc = arg;
2906 	uint32_t status;
2907 
2908 	USB_BUS_LOCK(&sc->sc_bus);
2909 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
2910 
2911 	/* read and clear interrupt status */
2912 	status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2913 
2914 	/* clear interrupts we are handling here */
2915 	DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & DWC_OTG_MSK_GINT_THREAD_IRQ);
2916 
2917 	DPRINTFN(14, "GINTSTS=0x%08x HAINT=0x%08x HFNUM=0x%08x\n",
2918 	    status, DWC_OTG_READ_4(sc, DOTG_HAINT),
2919 	    DWC_OTG_READ_4(sc, DOTG_HFNUM));
2920 
2921 	if (status & GINTSTS_USBRST) {
2922 		/* set correct state */
2923 		sc->sc_flags.status_device_mode = 1;
2924 		sc->sc_flags.status_bus_reset = 0;
2925 		sc->sc_flags.status_suspend = 0;
2926 		sc->sc_flags.change_suspend = 0;
2927 		sc->sc_flags.change_connect = 1;
2928 
2929 		/* Disable SOF interrupt */
2930 		sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2931 		DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2932 
2933 		/* complete root HUB interrupt endpoint */
2934 		dwc_otg_root_intr(sc);
2935 	}
2936 
2937 	/* check for any bus state change interrupts */
2938 	if (status & GINTSTS_ENUMDONE) {
2939 		uint32_t temp;
2940 
2941 		DPRINTFN(5, "end of reset\n");
2942 
2943 		/* set correct state */
2944 		sc->sc_flags.status_device_mode = 1;
2945 		sc->sc_flags.status_bus_reset = 1;
2946 		sc->sc_flags.status_suspend = 0;
2947 		sc->sc_flags.change_suspend = 0;
2948 		sc->sc_flags.change_connect = 1;
2949 		sc->sc_flags.status_low_speed = 0;
2950 		sc->sc_flags.port_enabled = 1;
2951 
2952 		/* reset FIFOs */
2953 		(void) dwc_otg_init_fifo(sc, DWC_MODE_DEVICE);
2954 
2955 		/* reset function address */
2956 		dwc_otg_set_address(sc, 0);
2957 
2958 		/* figure out enumeration speed */
2959 		temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
2960 		if (DSTS_ENUMSPD_GET(temp) == DSTS_ENUMSPD_HI)
2961 			sc->sc_flags.status_high_speed = 1;
2962 		else
2963 			sc->sc_flags.status_high_speed = 0;
2964 
2965 		/*
2966 		 * Disable resume and SOF interrupt, and enable
2967 		 * suspend and RX frame interrupt:
2968 		 */
2969 		sc->sc_irq_mask &= ~(GINTMSK_WKUPINTMSK | GINTMSK_SOFMSK);
2970 		sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
2971 		DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2972 
2973 		/* complete root HUB interrupt endpoint */
2974 		dwc_otg_root_intr(sc);
2975 	}
2976 
2977 	if (status & GINTSTS_PRTINT) {
2978 		uint32_t hprt;
2979 
2980 		hprt = DWC_OTG_READ_4(sc, DOTG_HPRT);
2981 
2982 		/* clear change bits */
2983 		DWC_OTG_WRITE_4(sc, DOTG_HPRT, (hprt & (
2984 		    HPRT_PRTPWR | HPRT_PRTENCHNG |
2985 		    HPRT_PRTCONNDET | HPRT_PRTOVRCURRCHNG)) |
2986 		    sc->sc_hprt_val);
2987 
2988 		DPRINTFN(12, "GINTSTS=0x%08x, HPRT=0x%08x\n", status, hprt);
2989 
2990 		sc->sc_flags.status_device_mode = 0;
2991 
2992 		if (hprt & HPRT_PRTCONNSTS)
2993 			sc->sc_flags.status_bus_reset = 1;
2994 		else
2995 			sc->sc_flags.status_bus_reset = 0;
2996 
2997 		if ((hprt & HPRT_PRTENCHNG) &&
2998 		    (hprt & HPRT_PRTENA) == 0)
2999 			sc->sc_flags.change_enabled = 1;
3000 
3001 		if (hprt & HPRT_PRTENA)
3002 			sc->sc_flags.port_enabled = 1;
3003 		else
3004 			sc->sc_flags.port_enabled = 0;
3005 
3006 		if (hprt & HPRT_PRTOVRCURRCHNG)
3007 			sc->sc_flags.change_over_current = 1;
3008 
3009 		if (hprt & HPRT_PRTOVRCURRACT)
3010 			sc->sc_flags.port_over_current = 1;
3011 		else
3012 			sc->sc_flags.port_over_current = 0;
3013 
3014 		if (hprt & HPRT_PRTPWR)
3015 			sc->sc_flags.port_powered = 1;
3016 		else
3017 			sc->sc_flags.port_powered = 0;
3018 
3019 		if (((hprt & HPRT_PRTSPD_MASK)
3020 		    >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_LOW)
3021 			sc->sc_flags.status_low_speed = 1;
3022 		else
3023 			sc->sc_flags.status_low_speed = 0;
3024 
3025 		if (((hprt & HPRT_PRTSPD_MASK)
3026 		    >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_HIGH)
3027 			sc->sc_flags.status_high_speed = 1;
3028 		else
3029 			sc->sc_flags.status_high_speed = 0;
3030 
3031 		if (hprt & HPRT_PRTCONNDET)
3032 			sc->sc_flags.change_connect = 1;
3033 
3034 		if (hprt & HPRT_PRTSUSP)
3035 			dwc_otg_suspend_irq(sc);
3036 		else
3037 			dwc_otg_resume_irq(sc);
3038 
3039 		/* complete root HUB interrupt endpoint */
3040 		dwc_otg_root_intr(sc);
3041 
3042 		/* update host frame interval */
3043 		dwc_otg_update_host_frame_interval(sc);
3044 	}
3045 
3046 	/*
3047 	 * If resume and suspend is set at the same time we interpret
3048 	 * that like RESUME. Resume is set when there is at least 3
3049 	 * milliseconds of inactivity on the USB BUS.
3050 	 */
3051 	if (status & GINTSTS_WKUPINT) {
3052 		DPRINTFN(5, "resume interrupt\n");
3053 
3054 		dwc_otg_resume_irq(sc);
3055 
3056 	} else if (status & GINTSTS_USBSUSP) {
3057 		DPRINTFN(5, "suspend interrupt\n");
3058 
3059 		dwc_otg_suspend_irq(sc);
3060 	}
3061 	/* check VBUS */
3062 	if (status & (GINTSTS_USBSUSP |
3063 	    GINTSTS_USBRST |
3064 	    GINTMSK_OTGINTMSK |
3065 	    GINTSTS_SESSREQINT)) {
3066 		uint32_t temp;
3067 
3068 		temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
3069 
3070 		DPRINTFN(5, "GOTGCTL=0x%08x\n", temp);
3071 
3072 		dwc_otg_vbus_interrupt(sc,
3073 		    (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
3074 	}
3075 
3076 	if (sc->sc_xfer_complete != 0) {
3077 		sc->sc_xfer_complete = 0;
3078 
3079 		/* complete FIFOs, if any */
3080 		dwc_otg_interrupt_complete_locked(sc);
3081 	}
3082 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3083 	USB_BUS_UNLOCK(&sc->sc_bus);
3084 }
3085 
3086 static void
3087 dwc_otg_setup_standard_chain_sub(struct dwc_otg_std_temp *temp)
3088 {
3089 	struct dwc_otg_td *td;
3090 
3091 	/* get current Transfer Descriptor */
3092 	td = temp->td_next;
3093 	temp->td = td;
3094 
3095 	/* prepare for next TD */
3096 	temp->td_next = td->obj_next;
3097 
3098 	/* fill out the Transfer Descriptor */
3099 	td->func = temp->func;
3100 	td->pc = temp->pc;
3101 	td->offset = temp->offset;
3102 	td->remainder = temp->len;
3103 	td->tx_bytes = 0;
3104 	td->error_any = 0;
3105 	td->error_stall = 0;
3106 	td->npkt = 0;
3107 	td->did_stall = temp->did_stall;
3108 	td->short_pkt = temp->short_pkt;
3109 	td->alt_next = temp->setup_alt_next;
3110 	td->set_toggle = 0;
3111 	td->got_short = 0;
3112 	td->did_nak = 0;
3113 	td->channel[0] = DWC_OTG_MAX_CHANNELS;
3114 	td->channel[1] = DWC_OTG_MAX_CHANNELS;
3115 	td->channel[2] = DWC_OTG_MAX_CHANNELS;
3116 	td->state = 0;
3117 	td->errcnt = 0;
3118 	td->tt_scheduled = 0;
3119 	td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
3120 }
3121 
3122 static void
3123 dwc_otg_setup_standard_chain(struct usb_xfer *xfer)
3124 {
3125 	struct dwc_otg_std_temp temp;
3126 	struct dwc_otg_td *td;
3127 	uint32_t x;
3128 	uint8_t need_sync;
3129 	uint8_t is_host;
3130 
3131 	DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
3132 	    xfer->address, UE_GET_ADDR(xfer->endpointno),
3133 	    xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
3134 
3135 	temp.max_frame_size = xfer->max_frame_size;
3136 
3137 	td = xfer->td_start[0];
3138 	xfer->td_transfer_first = td;
3139 	xfer->td_transfer_cache = td;
3140 
3141 	/* setup temp */
3142 
3143 	temp.pc = NULL;
3144 	temp.td = NULL;
3145 	temp.td_next = xfer->td_start[0];
3146 	temp.offset = 0;
3147 	temp.setup_alt_next = xfer->flags_int.short_frames_ok ||
3148 	    xfer->flags_int.isochronous_xfr;
3149 	temp.did_stall = !xfer->flags_int.control_stall;
3150 
3151 	is_host = (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST);
3152 
3153 	/* check if we should prepend a setup message */
3154 
3155 	if (xfer->flags_int.control_xfr) {
3156 		if (xfer->flags_int.control_hdr) {
3157 			if (is_host)
3158 				temp.func = &dwc_otg_host_setup_tx;
3159 			else
3160 				temp.func = &dwc_otg_setup_rx;
3161 
3162 			temp.len = xfer->frlengths[0];
3163 			temp.pc = xfer->frbuffers + 0;
3164 			temp.short_pkt = temp.len ? 1 : 0;
3165 
3166 			/* check for last frame */
3167 			if (xfer->nframes == 1) {
3168 				/* no STATUS stage yet, SETUP is last */
3169 				if (xfer->flags_int.control_act)
3170 					temp.setup_alt_next = 0;
3171 			}
3172 
3173 			dwc_otg_setup_standard_chain_sub(&temp);
3174 		}
3175 		x = 1;
3176 	} else {
3177 		x = 0;
3178 	}
3179 
3180 	if (x != xfer->nframes) {
3181 		if (xfer->endpointno & UE_DIR_IN) {
3182 			if (is_host) {
3183 				temp.func = &dwc_otg_host_data_rx;
3184 				need_sync = 0;
3185 			} else {
3186 				temp.func = &dwc_otg_data_tx;
3187 				need_sync = 1;
3188 			}
3189 		} else {
3190 			if (is_host) {
3191 				temp.func = &dwc_otg_host_data_tx;
3192 				need_sync = 0;
3193 			} else {
3194 				temp.func = &dwc_otg_data_rx;
3195 				need_sync = 0;
3196 			}
3197 		}
3198 
3199 		/* setup "pc" pointer */
3200 		temp.pc = xfer->frbuffers + x;
3201 	} else {
3202 		need_sync = 0;
3203 	}
3204 	while (x != xfer->nframes) {
3205 		/* DATA0 / DATA1 message */
3206 
3207 		temp.len = xfer->frlengths[x];
3208 
3209 		x++;
3210 
3211 		if (x == xfer->nframes) {
3212 			if (xfer->flags_int.control_xfr) {
3213 				if (xfer->flags_int.control_act) {
3214 					temp.setup_alt_next = 0;
3215 				}
3216 			} else {
3217 				temp.setup_alt_next = 0;
3218 			}
3219 		}
3220 		if (temp.len == 0) {
3221 			/* make sure that we send an USB packet */
3222 
3223 			temp.short_pkt = 0;
3224 
3225 		} else {
3226 			/* regular data transfer */
3227 
3228 			temp.short_pkt = (xfer->flags.force_short_xfer ? 0 : 1);
3229 		}
3230 
3231 		dwc_otg_setup_standard_chain_sub(&temp);
3232 
3233 		if (xfer->flags_int.isochronous_xfr) {
3234 			temp.offset += temp.len;
3235 		} else {
3236 			/* get next Page Cache pointer */
3237 			temp.pc = xfer->frbuffers + x;
3238 		}
3239 	}
3240 
3241 	if (xfer->flags_int.control_xfr) {
3242 		/* always setup a valid "pc" pointer for status and sync */
3243 		temp.pc = xfer->frbuffers + 0;
3244 		temp.len = 0;
3245 		temp.short_pkt = 0;
3246 		temp.setup_alt_next = 0;
3247 
3248 		/* check if we need to sync */
3249 		if (need_sync) {
3250 			/* we need a SYNC point after TX */
3251 			temp.func = &dwc_otg_data_tx_sync;
3252 			dwc_otg_setup_standard_chain_sub(&temp);
3253 		}
3254 
3255 		/* check if we should append a status stage */
3256 		if (!xfer->flags_int.control_act) {
3257 			/*
3258 			 * Send a DATA1 message and invert the current
3259 			 * endpoint direction.
3260 			 */
3261 			if (xfer->endpointno & UE_DIR_IN) {
3262 				if (is_host) {
3263 					temp.func = &dwc_otg_host_data_tx;
3264 					need_sync = 0;
3265 				} else {
3266 					temp.func = &dwc_otg_data_rx;
3267 					need_sync = 0;
3268 				}
3269 			} else {
3270 				if (is_host) {
3271 					temp.func = &dwc_otg_host_data_rx;
3272 					need_sync = 0;
3273 				} else {
3274 					temp.func = &dwc_otg_data_tx;
3275 					need_sync = 1;
3276 				}
3277 			}
3278 
3279 			dwc_otg_setup_standard_chain_sub(&temp);
3280 
3281 			/* data toggle should be DATA1 */
3282 			td = temp.td;
3283 			td->set_toggle = 1;
3284 
3285 			if (need_sync) {
3286 				/* we need a SYNC point after TX */
3287 				temp.func = &dwc_otg_data_tx_sync;
3288 				dwc_otg_setup_standard_chain_sub(&temp);
3289 			}
3290 		}
3291 	} else {
3292 		/* check if we need to sync */
3293 		if (need_sync) {
3294 			temp.pc = xfer->frbuffers + 0;
3295 			temp.len = 0;
3296 			temp.short_pkt = 0;
3297 			temp.setup_alt_next = 0;
3298 
3299 			/* we need a SYNC point after TX */
3300 			temp.func = &dwc_otg_data_tx_sync;
3301 			dwc_otg_setup_standard_chain_sub(&temp);
3302 		}
3303 	}
3304 
3305 	/* must have at least one frame! */
3306 	td = temp.td;
3307 	xfer->td_transfer_last = td;
3308 
3309 	if (is_host) {
3310 		struct dwc_otg_softc *sc;
3311 		uint32_t hcchar;
3312 		uint32_t hcsplt;
3313 
3314 		sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3315 
3316 		/* get first again */
3317 		td = xfer->td_transfer_first;
3318 		td->toggle = (xfer->endpoint->toggle_next ? 1 : 0);
3319 
3320 		hcchar =
3321 			(xfer->address << HCCHAR_DEVADDR_SHIFT) |
3322 			((xfer->endpointno & UE_ADDR) << HCCHAR_EPNUM_SHIFT) |
3323 			(xfer->max_packet_size << HCCHAR_MPS_SHIFT) |
3324 			HCCHAR_CHENA;
3325 
3326 		/*
3327 		 * We are not always able to meet the timing
3328 		 * requirements of the USB interrupt endpoint's
3329 		 * complete split token, when doing transfers going
3330 		 * via a transaction translator. Use the CONTROL
3331 		 * transfer type instead of the INTERRUPT transfer
3332 		 * type in general, as a means to workaround
3333 		 * that. This trick should work for both FULL and LOW
3334 		 * speed USB traffic going through a TT. For non-TT
3335 		 * traffic it works as well. The reason for using
3336 		 * CONTROL type instead of BULK is that some TTs might
3337 		 * reject LOW speed BULK traffic.
3338 		 */
3339 		if (td->ep_type == UE_INTERRUPT)
3340 			hcchar |= (UE_CONTROL << HCCHAR_EPTYPE_SHIFT);
3341 		else
3342 			hcchar |= (td->ep_type << HCCHAR_EPTYPE_SHIFT);
3343 
3344 		if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN)
3345 			hcchar |= HCCHAR_EPDIR_IN;
3346 
3347 		switch (xfer->xroot->udev->speed) {
3348 		case USB_SPEED_LOW:
3349 			hcchar |= HCCHAR_LSPDDEV;
3350 			/* FALLTHROUGH */
3351 		case USB_SPEED_FULL:
3352 			/* check if root HUB port is running High Speed */
3353 			if (dwc_otg_uses_split(xfer->xroot->udev)) {
3354 				hcsplt = HCSPLT_SPLTENA |
3355 				    (xfer->xroot->udev->hs_port_no <<
3356 				    HCSPLT_PRTADDR_SHIFT) |
3357 				    (xfer->xroot->udev->hs_hub_addr <<
3358 				    HCSPLT_HUBADDR_SHIFT);
3359 			} else {
3360 				hcsplt = 0;
3361 			}
3362 			if (td->ep_type == UE_INTERRUPT) {
3363 				uint32_t ival;
3364 				ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3365 				if (ival == 0)
3366 					ival = 1;
3367 				else if (ival > 127)
3368 					ival = 127;
3369 				td->tmr_val = sc->sc_tmr_val + ival;
3370 				td->tmr_res = ival;
3371 			} else if (td->ep_type == UE_ISOCHRONOUS) {
3372 				td->tmr_res = 1;
3373 				td->tmr_val = sc->sc_last_frame_num;
3374 				if (td->hcchar & HCCHAR_EPDIR_IN)
3375 					td->tmr_val++;
3376 			} else {
3377 				td->tmr_val = 0;
3378 				td->tmr_res = (uint8_t)sc->sc_last_frame_num;
3379 			}
3380 			break;
3381 		case USB_SPEED_HIGH:
3382 			hcsplt = 0;
3383 			if (td->ep_type == UE_INTERRUPT) {
3384 				uint32_t ival;
3385 				hcchar |= ((xfer->max_packet_count & 3)
3386 				    << HCCHAR_MC_SHIFT);
3387 				ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3388 				if (ival == 0)
3389 					ival = 1;
3390 				else if (ival > 127)
3391 					ival = 127;
3392 				td->tmr_val = sc->sc_tmr_val + ival;
3393 				td->tmr_res = ival;
3394 			} else if (td->ep_type == UE_ISOCHRONOUS) {
3395 				hcchar |= ((xfer->max_packet_count & 3)
3396 				    << HCCHAR_MC_SHIFT);
3397 				td->tmr_res = 1 << usbd_xfer_get_fps_shift(xfer);
3398 				td->tmr_val = sc->sc_last_frame_num;
3399 				if (td->hcchar & HCCHAR_EPDIR_IN)
3400 					td->tmr_val += td->tmr_res;
3401 
3402 			} else {
3403 				td->tmr_val = 0;
3404 				td->tmr_res = (uint8_t)sc->sc_last_frame_num;
3405 			}
3406 			break;
3407 		default:
3408 			hcsplt = 0;
3409 			td->tmr_val = 0;
3410 			td->tmr_res = 0;
3411 			break;
3412 		}
3413 
3414 		/* store configuration in all TD's */
3415 		while (1) {
3416 			td->hcchar = hcchar;
3417 			td->hcsplt = hcsplt;
3418 
3419 			if (((void *)td) == xfer->td_transfer_last)
3420 				break;
3421 
3422 			td = td->obj_next;
3423 		}
3424 	}
3425 }
3426 
3427 static void
3428 dwc_otg_timeout(void *arg)
3429 {
3430 	struct usb_xfer *xfer = arg;
3431 
3432 	DPRINTF("xfer=%p\n", xfer);
3433 
3434 	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
3435 
3436 	/* transfer is transferred */
3437 	dwc_otg_device_done(xfer, USB_ERR_TIMEOUT);
3438 }
3439 
3440 static void
3441 dwc_otg_start_standard_chain(struct usb_xfer *xfer)
3442 {
3443 	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3444 
3445 	DPRINTFN(9, "\n");
3446 
3447 	/*
3448 	 * Poll one time in device mode, which will turn on the
3449 	 * endpoint interrupts. Else wait for SOF interrupt in host
3450 	 * mode.
3451 	 */
3452 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
3453 
3454 	if (sc->sc_flags.status_device_mode != 0) {
3455 		dwc_otg_xfer_do_fifo(sc, xfer);
3456 		if (dwc_otg_xfer_do_complete_locked(sc, xfer))
3457 			goto done;
3458 	} else {
3459 		struct dwc_otg_td *td = xfer->td_transfer_cache;
3460 		if (td->ep_type == UE_ISOCHRONOUS &&
3461 		    (td->hcchar & HCCHAR_EPDIR_IN) == 0) {
3462 			/*
3463 			 * Need to start ISOCHRONOUS OUT transfer ASAP
3464 			 * because execution is delayed by one 125us
3465 			 * microframe:
3466 			 */
3467 			dwc_otg_xfer_do_fifo(sc, xfer);
3468 			if (dwc_otg_xfer_do_complete_locked(sc, xfer))
3469 				goto done;
3470 		}
3471 	}
3472 
3473 	/* put transfer on interrupt queue */
3474 	usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
3475 
3476 	/* start timeout, if any */
3477 	if (xfer->timeout != 0) {
3478 		usbd_transfer_timeout_ms(xfer,
3479 		    &dwc_otg_timeout, xfer->timeout);
3480 	}
3481 
3482 	if (sc->sc_flags.status_device_mode != 0)
3483 		goto done;
3484 
3485 	/* enable SOF interrupt, if any */
3486 	dwc_otg_enable_sof_irq(sc);
3487 done:
3488 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3489 }
3490 
3491 static void
3492 dwc_otg_root_intr(struct dwc_otg_softc *sc)
3493 {
3494 	DPRINTFN(9, "\n");
3495 
3496 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3497 
3498 	/* set port bit */
3499 	sc->sc_hub_idata[0] = 0x02;	/* we only have one port */
3500 
3501 	uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
3502 	    sizeof(sc->sc_hub_idata));
3503 }
3504 
3505 static usb_error_t
3506 dwc_otg_standard_done_sub(struct usb_xfer *xfer)
3507 {
3508 	struct dwc_otg_td *td;
3509 	uint32_t len;
3510 	usb_error_t error;
3511 
3512 	DPRINTFN(9, "\n");
3513 
3514 	td = xfer->td_transfer_cache;
3515 
3516 	do {
3517 		len = td->remainder;
3518 
3519 		/* store last data toggle */
3520 		xfer->endpoint->toggle_next = td->toggle;
3521 
3522 		if (xfer->aframes != xfer->nframes) {
3523 			/*
3524 			 * Verify the length and subtract
3525 			 * the remainder from "frlengths[]":
3526 			 */
3527 			if (len > xfer->frlengths[xfer->aframes]) {
3528 				td->error_any = 1;
3529 			} else {
3530 				xfer->frlengths[xfer->aframes] -= len;
3531 			}
3532 		}
3533 		/* Check for transfer error */
3534 		if (td->error_any) {
3535 			/* the transfer is finished */
3536 			error = (td->error_stall ?
3537 			    USB_ERR_STALLED : USB_ERR_IOERROR);
3538 			td = NULL;
3539 			break;
3540 		}
3541 		/* Check for short transfer */
3542 		if (len > 0) {
3543 			if (xfer->flags_int.short_frames_ok ||
3544 			    xfer->flags_int.isochronous_xfr) {
3545 				/* follow alt next */
3546 				if (td->alt_next) {
3547 					td = td->obj_next;
3548 				} else {
3549 					td = NULL;
3550 				}
3551 			} else {
3552 				/* the transfer is finished */
3553 				td = NULL;
3554 			}
3555 			error = 0;
3556 			break;
3557 		}
3558 		td = td->obj_next;
3559 
3560 		/* this USB frame is complete */
3561 		error = 0;
3562 		break;
3563 
3564 	} while (0);
3565 
3566 	/* update transfer cache */
3567 
3568 	xfer->td_transfer_cache = td;
3569 
3570 	return (error);
3571 }
3572 
3573 static void
3574 dwc_otg_standard_done(struct usb_xfer *xfer)
3575 {
3576 	usb_error_t err = 0;
3577 
3578 	DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
3579 	    xfer, xfer->endpoint);
3580 
3581 	/* reset scanner */
3582 
3583 	xfer->td_transfer_cache = xfer->td_transfer_first;
3584 
3585 	if (xfer->flags_int.control_xfr) {
3586 		if (xfer->flags_int.control_hdr) {
3587 			err = dwc_otg_standard_done_sub(xfer);
3588 		}
3589 		xfer->aframes = 1;
3590 
3591 		if (xfer->td_transfer_cache == NULL) {
3592 			goto done;
3593 		}
3594 	}
3595 	while (xfer->aframes != xfer->nframes) {
3596 		err = dwc_otg_standard_done_sub(xfer);
3597 		xfer->aframes++;
3598 
3599 		if (xfer->td_transfer_cache == NULL) {
3600 			goto done;
3601 		}
3602 	}
3603 
3604 	if (xfer->flags_int.control_xfr &&
3605 	    !xfer->flags_int.control_act) {
3606 		err = dwc_otg_standard_done_sub(xfer);
3607 	}
3608 done:
3609 	dwc_otg_device_done(xfer, err);
3610 }
3611 
3612 /*------------------------------------------------------------------------*
3613  *	dwc_otg_device_done
3614  *
3615  * NOTE: this function can be called more than one time on the
3616  * same USB transfer!
3617  *------------------------------------------------------------------------*/
3618 static void
3619 dwc_otg_device_done(struct usb_xfer *xfer, usb_error_t error)
3620 {
3621 	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3622 
3623 	DPRINTFN(9, "xfer=%p, endpoint=%p, error=%d\n",
3624 	    xfer, xfer->endpoint, error);
3625 
3626 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
3627 
3628 	if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
3629 		/* Interrupts are cleared by the interrupt handler */
3630 	} else {
3631 		struct dwc_otg_td *td;
3632 
3633 		td = xfer->td_transfer_cache;
3634  		if (td != NULL)
3635 			dwc_otg_host_channel_free(sc, td);
3636 	}
3637 	/* dequeue transfer and start next transfer */
3638 	usbd_transfer_done(xfer, error);
3639 
3640 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3641 }
3642 
3643 static void
3644 dwc_otg_xfer_stall(struct usb_xfer *xfer)
3645 {
3646 	dwc_otg_device_done(xfer, USB_ERR_STALLED);
3647 }
3648 
3649 static void
3650 dwc_otg_set_stall(struct usb_device *udev,
3651     struct usb_endpoint *ep, uint8_t *did_stall)
3652 {
3653 	struct dwc_otg_softc *sc;
3654 	uint32_t temp;
3655 	uint32_t reg;
3656 	uint8_t ep_no;
3657 
3658 	USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3659 
3660 	/* check mode */
3661 	if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3662 		/* not supported */
3663 		return;
3664 	}
3665 
3666 	sc = DWC_OTG_BUS2SC(udev->bus);
3667 
3668 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
3669 
3670 	/* get endpoint address */
3671 	ep_no = ep->edesc->bEndpointAddress;
3672 
3673 	DPRINTFN(5, "endpoint=0x%x\n", ep_no);
3674 
3675 	if (ep_no & UE_DIR_IN) {
3676 		reg = DOTG_DIEPCTL(ep_no & UE_ADDR);
3677 		temp = sc->sc_in_ctl[ep_no & UE_ADDR];
3678 	} else {
3679 		reg = DOTG_DOEPCTL(ep_no & UE_ADDR);
3680 		temp = sc->sc_out_ctl[ep_no & UE_ADDR];
3681 	}
3682 
3683 	/* disable and stall endpoint */
3684 	DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3685 	DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_STALL);
3686 
3687 	/* clear active OUT ep */
3688 	if (!(ep_no & UE_DIR_IN)) {
3689 		sc->sc_active_rx_ep &= ~(1U << (ep_no & UE_ADDR));
3690 
3691 		if (sc->sc_last_rx_status != 0 &&
3692 		    (ep_no & UE_ADDR) == GRXSTSRD_CHNUM_GET(
3693 		    sc->sc_last_rx_status)) {
3694 			/* dump data */
3695 			dwc_otg_common_rx_ack(sc);
3696 			/* poll interrupt */
3697 			dwc_otg_interrupt_poll_locked(sc);
3698 			dwc_otg_interrupt_complete_locked(sc);
3699 		}
3700 	}
3701 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3702 }
3703 
3704 static void
3705 dwc_otg_clear_stall_sub_locked(struct dwc_otg_softc *sc, uint32_t mps,
3706     uint8_t ep_no, uint8_t ep_type, uint8_t ep_dir)
3707 {
3708 	uint32_t reg;
3709 	uint32_t temp;
3710 
3711 	if (ep_type == UE_CONTROL) {
3712 		/* clearing stall is not needed */
3713 		return;
3714 	}
3715 
3716 	if (ep_dir) {
3717 		reg = DOTG_DIEPCTL(ep_no);
3718 	} else {
3719 		reg = DOTG_DOEPCTL(ep_no);
3720 		sc->sc_active_rx_ep |= (1U << ep_no);
3721 	}
3722 
3723 	/* round up and mask away the multiplier count */
3724 	mps = (mps + 3) & 0x7FC;
3725 
3726 	if (ep_type == UE_BULK) {
3727 		temp = DIEPCTL_EPTYPE_SET(
3728 		    DIEPCTL_EPTYPE_BULK) |
3729 		    DIEPCTL_USBACTEP;
3730 	} else if (ep_type == UE_INTERRUPT) {
3731 		temp = DIEPCTL_EPTYPE_SET(
3732 		    DIEPCTL_EPTYPE_INTERRUPT) |
3733 		    DIEPCTL_USBACTEP;
3734 	} else {
3735 		temp = DIEPCTL_EPTYPE_SET(
3736 		    DIEPCTL_EPTYPE_ISOC) |
3737 		    DIEPCTL_USBACTEP;
3738 	}
3739 
3740 	temp |= DIEPCTL_MPS_SET(mps);
3741 	temp |= DIEPCTL_TXFNUM_SET(ep_no);
3742 
3743 	if (ep_dir)
3744 		sc->sc_in_ctl[ep_no] = temp;
3745 	else
3746 		sc->sc_out_ctl[ep_no] = temp;
3747 
3748 	DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3749 	DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_SETD0PID);
3750 	DWC_OTG_WRITE_4(sc, reg, temp | DIEPCTL_SNAK);
3751 
3752 	/* we only reset the transmit FIFO */
3753 	if (ep_dir) {
3754 		dwc_otg_tx_fifo_reset(sc,
3755 		    GRSTCTL_TXFIFO(ep_no) |
3756 		    GRSTCTL_TXFFLSH);
3757 
3758 		DWC_OTG_WRITE_4(sc,
3759 		    DOTG_DIEPTSIZ(ep_no), 0);
3760 	}
3761 
3762 	/* poll interrupt */
3763 	dwc_otg_interrupt_poll_locked(sc);
3764 	dwc_otg_interrupt_complete_locked(sc);
3765 }
3766 
3767 static void
3768 dwc_otg_clear_stall(struct usb_device *udev, struct usb_endpoint *ep)
3769 {
3770 	struct dwc_otg_softc *sc;
3771 	struct usb_endpoint_descriptor *ed;
3772 
3773 	DPRINTFN(5, "endpoint=%p\n", ep);
3774 
3775 	USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3776 
3777 	/* check mode */
3778 	if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3779 		/* not supported */
3780 		return;
3781 	}
3782 	/* get softc */
3783 	sc = DWC_OTG_BUS2SC(udev->bus);
3784 
3785 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
3786 
3787 	/* get endpoint descriptor */
3788 	ed = ep->edesc;
3789 
3790 	/* reset endpoint */
3791 	dwc_otg_clear_stall_sub_locked(sc,
3792 	    UGETW(ed->wMaxPacketSize),
3793 	    (ed->bEndpointAddress & UE_ADDR),
3794 	    (ed->bmAttributes & UE_XFERTYPE),
3795 	    (ed->bEndpointAddress & (UE_DIR_IN | UE_DIR_OUT)));
3796 
3797 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3798 }
3799 
3800 static void
3801 dwc_otg_device_state_change(struct usb_device *udev)
3802 {
3803 	struct dwc_otg_softc *sc;
3804 	uint8_t x;
3805 
3806 	/* check mode */
3807 	if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3808 		/* not supported */
3809 		return;
3810 	}
3811 
3812 	/* get softc */
3813 	sc = DWC_OTG_BUS2SC(udev->bus);
3814 
3815 	/* deactivate all other endpoint but the control endpoint */
3816 	if (udev->state == USB_STATE_CONFIGURED ||
3817 	    udev->state == USB_STATE_ADDRESSED) {
3818 		USB_BUS_LOCK(&sc->sc_bus);
3819 
3820 		for (x = 1; x != sc->sc_dev_ep_max; x++) {
3821 			if (x < sc->sc_dev_in_ep_max) {
3822 				DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x),
3823 				    DIEPCTL_EPDIS);
3824 				DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x), 0);
3825 			}
3826 
3827 			DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x),
3828 			    DOEPCTL_EPDIS);
3829 			DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x), 0);
3830 		}
3831 		USB_BUS_UNLOCK(&sc->sc_bus);
3832 	}
3833 }
3834 
3835 int
3836 dwc_otg_init(struct dwc_otg_softc *sc)
3837 {
3838 	uint32_t temp;
3839 	int err;
3840 
3841 	DPRINTF("start\n");
3842 
3843 	sc->sc_io_tag = rman_get_bustag(sc->sc_io_res);
3844 	sc->sc_io_hdl = rman_get_bushandle(sc->sc_io_res);
3845 	sc->sc_io_size = rman_get_size(sc->sc_io_res);
3846 
3847 	/* set up the bus structure */
3848 	sc->sc_bus.devices = sc->sc_devices;
3849 	sc->sc_bus.devices_max = DWC_OTG_MAX_DEVICES;
3850 	sc->sc_bus.dma_bits = 32;
3851 	sc->sc_bus.usbrev = USB_REV_2_0;
3852 	sc->sc_bus.methods = &dwc_otg_bus_methods;
3853 
3854 	/* get all DMA memory */
3855 	if (usb_bus_mem_alloc_all(&sc->sc_bus,
3856 	    USB_GET_DMA_TAG(sc->sc_bus.parent), NULL)) {
3857 		return (ENOMEM);
3858 	}
3859 
3860 	sc->sc_bus.bdev = device_add_child(sc->sc_bus.parent, "usbus", -1);
3861 	if (sc->sc_bus.bdev == NULL)
3862 		return (ENXIO);
3863 
3864 	device_set_ivars(sc->sc_bus.bdev, &sc->sc_bus);
3865 
3866 	err = bus_setup_intr(sc->sc_bus.parent, sc->sc_irq_res,
3867 	    INTR_TYPE_TTY | INTR_MPSAFE, &dwc_otg_filter_interrupt,
3868 	    &dwc_otg_interrupt, sc, &sc->sc_intr_hdl);
3869 	if (err) {
3870 		sc->sc_intr_hdl = NULL;
3871 		return (ENXIO);
3872 	}
3873 
3874 	usb_callout_init_mtx(&sc->sc_timer,
3875 	    &sc->sc_bus.bus_mtx, 0);
3876 
3877 	USB_BUS_LOCK(&sc->sc_bus);
3878 
3879 	/* turn on clocks */
3880 	dwc_otg_clocks_on(sc);
3881 
3882 	temp = DWC_OTG_READ_4(sc, DOTG_GSNPSID);
3883 	DPRINTF("Version = 0x%08x\n", temp);
3884 
3885 	/* disconnect */
3886 	DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3887 	    DCTL_SFTDISCON);
3888 
3889 	/* wait for host to detect disconnect */
3890 	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 32);
3891 
3892 	DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
3893 	    GRSTCTL_CSFTRST);
3894 
3895 	/* wait a little bit for block to reset */
3896 	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 128);
3897 
3898 	switch (sc->sc_mode) {
3899 	case DWC_MODE_DEVICE:
3900 		temp = GUSBCFG_FORCEDEVMODE;
3901 		break;
3902 	case DWC_MODE_HOST:
3903 		temp = GUSBCFG_FORCEHOSTMODE;
3904 		break;
3905 	default:
3906 		temp = 0;
3907 		break;
3908 	}
3909 
3910 	if (sc->sc_phy_type == 0)
3911 		sc->sc_phy_type = dwc_otg_phy_type + 1;
3912 	if (sc->sc_phy_bits == 0)
3913 		sc->sc_phy_bits = 16;
3914 
3915 	/* select HSIC, ULPI, UTMI+ or internal PHY mode */
3916 	switch (sc->sc_phy_type) {
3917 	case DWC_OTG_PHY_HSIC:
3918 		DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3919 		    GUSBCFG_PHYIF |
3920 		    GUSBCFG_TRD_TIM_SET(5) | temp);
3921 		DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL,
3922 		    0x000000EC);
3923 
3924 		temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3925 		DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3926 		    temp & ~GLPMCFG_HSIC_CONN);
3927 		DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3928 		    temp | GLPMCFG_HSIC_CONN);
3929 		break;
3930 	case DWC_OTG_PHY_ULPI:
3931 		DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3932 		    GUSBCFG_ULPI_UTMI_SEL |
3933 		    GUSBCFG_TRD_TIM_SET(5) | temp);
3934 		DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3935 
3936 		temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3937 		DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3938 		    temp & ~GLPMCFG_HSIC_CONN);
3939 		break;
3940 	case DWC_OTG_PHY_UTMI:
3941 		DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3942 		    (sc->sc_phy_bits == 16 ? GUSBCFG_PHYIF : 0) |
3943 		    GUSBCFG_TRD_TIM_SET(5) | temp);
3944 		DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3945 
3946 		temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3947 		DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3948 		    temp & ~GLPMCFG_HSIC_CONN);
3949 		break;
3950 	case DWC_OTG_PHY_INTERNAL:
3951 		DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3952 		    GUSBCFG_PHYSEL |
3953 		    GUSBCFG_TRD_TIM_SET(5) | temp);
3954 		DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3955 
3956 		temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3957 		DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3958 		    temp & ~GLPMCFG_HSIC_CONN);
3959 
3960 		temp = DWC_OTG_READ_4(sc, DOTG_GGPIO);
3961 		temp &= ~(DOTG_GGPIO_NOVBUSSENS | DOTG_GGPIO_I2CPADEN);
3962 		temp |= (DOTG_GGPIO_VBUSASEN | DOTG_GGPIO_VBUSBSEN |
3963 		    DOTG_GGPIO_PWRDWN);
3964 		DWC_OTG_WRITE_4(sc, DOTG_GGPIO, temp);
3965 		break;
3966 	default:
3967 		break;
3968 	}
3969 
3970 	/* clear global nak */
3971 	DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3972 	    DCTL_CGOUTNAK |
3973 	    DCTL_CGNPINNAK);
3974 
3975 	/* disable USB port */
3976 	DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0xFFFFFFFF);
3977 
3978 	/* wait 10ms */
3979 	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3980 
3981 	/* enable USB port */
3982 	DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
3983 
3984 	/* wait 10ms */
3985 	usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3986 
3987 	temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG3);
3988 
3989 	sc->sc_fifo_size = 4 * GHWCFG3_DFIFODEPTH_GET(temp);
3990 
3991 	temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
3992 
3993 	sc->sc_dev_ep_max = GHWCFG2_NUMDEVEPS_GET(temp);
3994 
3995 	if (sc->sc_dev_ep_max > DWC_OTG_MAX_ENDPOINTS)
3996 		sc->sc_dev_ep_max = DWC_OTG_MAX_ENDPOINTS;
3997 
3998 	sc->sc_host_ch_max = GHWCFG2_NUMHSTCHNL_GET(temp);
3999 
4000 	if (sc->sc_host_ch_max > DWC_OTG_MAX_CHANNELS)
4001 		sc->sc_host_ch_max = DWC_OTG_MAX_CHANNELS;
4002 
4003 	temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG4);
4004 
4005 	sc->sc_dev_in_ep_max = GHWCFG4_NUM_IN_EP_GET(temp);
4006 
4007 	DPRINTF("Total FIFO size = %d bytes, Device EPs = %d/%d Host CHs = %d\n",
4008 	    sc->sc_fifo_size, sc->sc_dev_ep_max, sc->sc_dev_in_ep_max,
4009 	    sc->sc_host_ch_max);
4010 
4011 	/* setup FIFO */
4012 	if (dwc_otg_init_fifo(sc, sc->sc_mode)) {
4013 		USB_BUS_UNLOCK(&sc->sc_bus);
4014 		return (EINVAL);
4015 	}
4016 
4017 	/* enable interrupts */
4018 	sc->sc_irq_mask |= DWC_OTG_MSK_GINT_THREAD_IRQ;
4019 	DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
4020 
4021 	if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_DEVICE) {
4022 		/* enable all endpoint interrupts */
4023 		temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
4024 		if (temp & GHWCFG2_MPI) {
4025 			uint8_t x;
4026 
4027 			DPRINTF("Disable Multi Process Interrupts\n");
4028 
4029 			for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
4030 				DWC_OTG_WRITE_4(sc, DOTG_DIEPEACHINTMSK(x), 0);
4031 				DWC_OTG_WRITE_4(sc, DOTG_DOEPEACHINTMSK(x), 0);
4032 			}
4033 			DWC_OTG_WRITE_4(sc, DOTG_DEACHINTMSK, 0);
4034 		}
4035 		DWC_OTG_WRITE_4(sc, DOTG_DIEPMSK,
4036 		    DIEPMSK_XFERCOMPLMSK);
4037 		DWC_OTG_WRITE_4(sc, DOTG_DOEPMSK, 0);
4038 		DWC_OTG_WRITE_4(sc, DOTG_DAINTMSK, 0xFFFF);
4039 	}
4040 
4041 	if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_HOST) {
4042 		/* setup clocks */
4043 		temp = DWC_OTG_READ_4(sc, DOTG_HCFG);
4044 		temp &= ~(HCFG_FSLSSUPP | HCFG_FSLSPCLKSEL_MASK);
4045 		temp |= (1 << HCFG_FSLSPCLKSEL_SHIFT);
4046 		DWC_OTG_WRITE_4(sc, DOTG_HCFG, temp);
4047 	}
4048 
4049 	/* only enable global IRQ */
4050 	DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG,
4051 	    GAHBCFG_GLBLINTRMSK);
4052 
4053 	/* turn off clocks */
4054 	dwc_otg_clocks_off(sc);
4055 
4056 	/* read initial VBUS state */
4057 
4058 	temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
4059 
4060 	DPRINTFN(5, "GOTCTL=0x%08x\n", temp);
4061 
4062 	dwc_otg_vbus_interrupt(sc,
4063 	    (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
4064 
4065 	USB_BUS_UNLOCK(&sc->sc_bus);
4066 
4067 	/* catch any lost interrupts */
4068 
4069 	dwc_otg_do_poll(&sc->sc_bus);
4070 
4071 	return (0);			/* success */
4072 }
4073 
4074 void
4075 dwc_otg_uninit(struct dwc_otg_softc *sc)
4076 {
4077 	USB_BUS_LOCK(&sc->sc_bus);
4078 
4079 	/* stop host timer */
4080 	dwc_otg_timer_stop(sc);
4081 
4082 	/* set disconnect */
4083 	DWC_OTG_WRITE_4(sc, DOTG_DCTL,
4084 	    DCTL_SFTDISCON);
4085 
4086 	/* turn off global IRQ */
4087 	DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG, 0);
4088 
4089 	sc->sc_flags.port_enabled = 0;
4090 	sc->sc_flags.port_powered = 0;
4091 	sc->sc_flags.status_vbus = 0;
4092 	sc->sc_flags.status_bus_reset = 0;
4093 	sc->sc_flags.status_suspend = 0;
4094 	sc->sc_flags.change_suspend = 0;
4095 	sc->sc_flags.change_connect = 1;
4096 
4097 	dwc_otg_pull_down(sc);
4098 	dwc_otg_clocks_off(sc);
4099 
4100 	USB_BUS_UNLOCK(&sc->sc_bus);
4101 
4102 	usb_callout_drain(&sc->sc_timer);
4103 }
4104 
4105 static void
4106 dwc_otg_suspend(struct dwc_otg_softc *sc)
4107 {
4108 	return;
4109 }
4110 
4111 static void
4112 dwc_otg_resume(struct dwc_otg_softc *sc)
4113 {
4114 	return;
4115 }
4116 
4117 static void
4118 dwc_otg_do_poll(struct usb_bus *bus)
4119 {
4120 	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
4121 
4122 	USB_BUS_LOCK(&sc->sc_bus);
4123 	USB_BUS_SPIN_LOCK(&sc->sc_bus);
4124 	dwc_otg_interrupt_poll_locked(sc);
4125 	dwc_otg_interrupt_complete_locked(sc);
4126 	USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
4127 	USB_BUS_UNLOCK(&sc->sc_bus);
4128 }
4129 
4130 /*------------------------------------------------------------------------*
4131  * DWC OTG bulk support
4132  * DWC OTG control support
4133  * DWC OTG interrupt support
4134  *------------------------------------------------------------------------*/
4135 static void
4136 dwc_otg_device_non_isoc_open(struct usb_xfer *xfer)
4137 {
4138 }
4139 
4140 static void
4141 dwc_otg_device_non_isoc_close(struct usb_xfer *xfer)
4142 {
4143 	dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
4144 }
4145 
4146 static void
4147 dwc_otg_device_non_isoc_enter(struct usb_xfer *xfer)
4148 {
4149 }
4150 
4151 static void
4152 dwc_otg_device_non_isoc_start(struct usb_xfer *xfer)
4153 {
4154 	/* setup TDs */
4155 	dwc_otg_setup_standard_chain(xfer);
4156 	dwc_otg_start_standard_chain(xfer);
4157 }
4158 
4159 static const struct usb_pipe_methods dwc_otg_device_non_isoc_methods =
4160 {
4161 	.open = dwc_otg_device_non_isoc_open,
4162 	.close = dwc_otg_device_non_isoc_close,
4163 	.enter = dwc_otg_device_non_isoc_enter,
4164 	.start = dwc_otg_device_non_isoc_start,
4165 };
4166 
4167 /*------------------------------------------------------------------------*
4168  * DWC OTG full speed isochronous support
4169  *------------------------------------------------------------------------*/
4170 static void
4171 dwc_otg_device_isoc_open(struct usb_xfer *xfer)
4172 {
4173 }
4174 
4175 static void
4176 dwc_otg_device_isoc_close(struct usb_xfer *xfer)
4177 {
4178 	dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
4179 }
4180 
4181 static void
4182 dwc_otg_device_isoc_enter(struct usb_xfer *xfer)
4183 {
4184 }
4185 
4186 static void
4187 dwc_otg_device_isoc_start(struct usb_xfer *xfer)
4188 {
4189 	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
4190 	uint32_t temp;
4191 	uint32_t framenum;
4192 
4193 	DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
4194 	    xfer, xfer->endpoint->isoc_next, xfer->nframes);
4195 
4196 	if (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST) {
4197 		temp = DWC_OTG_READ_4(sc, DOTG_HFNUM);
4198 
4199 		/* get the current frame index */
4200 		framenum = (temp & HFNUM_FRNUM_MASK);
4201 	} else {
4202 		temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
4203 
4204 		/* get the current frame index */
4205 		framenum = DSTS_SOFFN_GET(temp);
4206 	}
4207 
4208 	/*
4209 	 * Check if port is doing 8000 or 1000 frames per second:
4210 	 */
4211 	if (sc->sc_flags.status_high_speed)
4212 		framenum /= 8;
4213 
4214 	if (usbd_xfer_get_isochronous_start_frame(
4215 	    xfer, framenum, 0, 1, DWC_OTG_FRAME_MASK, NULL))
4216 		DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
4217 
4218 	/* setup TDs */
4219 	dwc_otg_setup_standard_chain(xfer);
4220 
4221 	/* start TD chain */
4222 	dwc_otg_start_standard_chain(xfer);
4223 }
4224 
4225 static const struct usb_pipe_methods dwc_otg_device_isoc_methods =
4226 {
4227 	.open = dwc_otg_device_isoc_open,
4228 	.close = dwc_otg_device_isoc_close,
4229 	.enter = dwc_otg_device_isoc_enter,
4230 	.start = dwc_otg_device_isoc_start,
4231 };
4232 
4233 /*------------------------------------------------------------------------*
4234  * DWC OTG root control support
4235  *------------------------------------------------------------------------*
4236  * Simulate a hardware HUB by handling all the necessary requests.
4237  *------------------------------------------------------------------------*/
4238 
4239 static const struct usb_device_descriptor dwc_otg_devd = {
4240 	.bLength = sizeof(struct usb_device_descriptor),
4241 	.bDescriptorType = UDESC_DEVICE,
4242 	.bcdUSB = {0x00, 0x02},
4243 	.bDeviceClass = UDCLASS_HUB,
4244 	.bDeviceSubClass = UDSUBCLASS_HUB,
4245 	.bDeviceProtocol = UDPROTO_HSHUBSTT,
4246 	.bMaxPacketSize = 64,
4247 	.bcdDevice = {0x00, 0x01},
4248 	.iManufacturer = 1,
4249 	.iProduct = 2,
4250 	.bNumConfigurations = 1,
4251 };
4252 
4253 static const struct dwc_otg_config_desc dwc_otg_confd = {
4254 	.confd = {
4255 		.bLength = sizeof(struct usb_config_descriptor),
4256 		.bDescriptorType = UDESC_CONFIG,
4257 		.wTotalLength[0] = sizeof(dwc_otg_confd),
4258 		.bNumInterface = 1,
4259 		.bConfigurationValue = 1,
4260 		.iConfiguration = 0,
4261 		.bmAttributes = UC_SELF_POWERED,
4262 		.bMaxPower = 0,
4263 	},
4264 	.ifcd = {
4265 		.bLength = sizeof(struct usb_interface_descriptor),
4266 		.bDescriptorType = UDESC_INTERFACE,
4267 		.bNumEndpoints = 1,
4268 		.bInterfaceClass = UICLASS_HUB,
4269 		.bInterfaceSubClass = UISUBCLASS_HUB,
4270 		.bInterfaceProtocol = 0,
4271 	},
4272 	.endpd = {
4273 		.bLength = sizeof(struct usb_endpoint_descriptor),
4274 		.bDescriptorType = UDESC_ENDPOINT,
4275 		.bEndpointAddress = (UE_DIR_IN | DWC_OTG_INTR_ENDPT),
4276 		.bmAttributes = UE_INTERRUPT,
4277 		.wMaxPacketSize[0] = 8,
4278 		.bInterval = 255,
4279 	},
4280 };
4281 #define	HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) }
4282 
4283 static const struct usb_hub_descriptor_min dwc_otg_hubd = {
4284 	.bDescLength = sizeof(dwc_otg_hubd),
4285 	.bDescriptorType = UDESC_HUB,
4286 	.bNbrPorts = 1,
4287 	HSETW(.wHubCharacteristics, (UHD_PWR_NO_SWITCH | UHD_OC_INDIVIDUAL)),
4288 	.bPwrOn2PwrGood = 50,
4289 	.bHubContrCurrent = 0,
4290 	.DeviceRemovable = {0},		/* port is removable */
4291 };
4292 
4293 #define	STRING_VENDOR \
4294   "D\0W\0C\0O\0T\0G"
4295 
4296 #define	STRING_PRODUCT \
4297   "O\0T\0G\0 \0R\0o\0o\0t\0 \0H\0U\0B"
4298 
4299 USB_MAKE_STRING_DESC(STRING_VENDOR, dwc_otg_vendor);
4300 USB_MAKE_STRING_DESC(STRING_PRODUCT, dwc_otg_product);
4301 
4302 static usb_error_t
4303 dwc_otg_roothub_exec(struct usb_device *udev,
4304     struct usb_device_request *req, const void **pptr, uint16_t *plength)
4305 {
4306 	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4307 	const void *ptr;
4308 	uint16_t len;
4309 	uint16_t value;
4310 	uint16_t index;
4311 	usb_error_t err;
4312 
4313 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
4314 
4315 	/* buffer reset */
4316 	ptr = (const void *)&sc->sc_hub_temp;
4317 	len = 0;
4318 	err = 0;
4319 
4320 	value = UGETW(req->wValue);
4321 	index = UGETW(req->wIndex);
4322 
4323 	/* demultiplex the control request */
4324 
4325 	switch (req->bmRequestType) {
4326 	case UT_READ_DEVICE:
4327 		switch (req->bRequest) {
4328 		case UR_GET_DESCRIPTOR:
4329 			goto tr_handle_get_descriptor;
4330 		case UR_GET_CONFIG:
4331 			goto tr_handle_get_config;
4332 		case UR_GET_STATUS:
4333 			goto tr_handle_get_status;
4334 		default:
4335 			goto tr_stalled;
4336 		}
4337 		break;
4338 
4339 	case UT_WRITE_DEVICE:
4340 		switch (req->bRequest) {
4341 		case UR_SET_ADDRESS:
4342 			goto tr_handle_set_address;
4343 		case UR_SET_CONFIG:
4344 			goto tr_handle_set_config;
4345 		case UR_CLEAR_FEATURE:
4346 			goto tr_valid;	/* nop */
4347 		case UR_SET_DESCRIPTOR:
4348 			goto tr_valid;	/* nop */
4349 		case UR_SET_FEATURE:
4350 		default:
4351 			goto tr_stalled;
4352 		}
4353 		break;
4354 
4355 	case UT_WRITE_ENDPOINT:
4356 		switch (req->bRequest) {
4357 		case UR_CLEAR_FEATURE:
4358 			switch (UGETW(req->wValue)) {
4359 			case UF_ENDPOINT_HALT:
4360 				goto tr_handle_clear_halt;
4361 			case UF_DEVICE_REMOTE_WAKEUP:
4362 				goto tr_handle_clear_wakeup;
4363 			default:
4364 				goto tr_stalled;
4365 			}
4366 			break;
4367 		case UR_SET_FEATURE:
4368 			switch (UGETW(req->wValue)) {
4369 			case UF_ENDPOINT_HALT:
4370 				goto tr_handle_set_halt;
4371 			case UF_DEVICE_REMOTE_WAKEUP:
4372 				goto tr_handle_set_wakeup;
4373 			default:
4374 				goto tr_stalled;
4375 			}
4376 			break;
4377 		case UR_SYNCH_FRAME:
4378 			goto tr_valid;	/* nop */
4379 		default:
4380 			goto tr_stalled;
4381 		}
4382 		break;
4383 
4384 	case UT_READ_ENDPOINT:
4385 		switch (req->bRequest) {
4386 		case UR_GET_STATUS:
4387 			goto tr_handle_get_ep_status;
4388 		default:
4389 			goto tr_stalled;
4390 		}
4391 		break;
4392 
4393 	case UT_WRITE_INTERFACE:
4394 		switch (req->bRequest) {
4395 		case UR_SET_INTERFACE:
4396 			goto tr_handle_set_interface;
4397 		case UR_CLEAR_FEATURE:
4398 			goto tr_valid;	/* nop */
4399 		case UR_SET_FEATURE:
4400 		default:
4401 			goto tr_stalled;
4402 		}
4403 		break;
4404 
4405 	case UT_READ_INTERFACE:
4406 		switch (req->bRequest) {
4407 		case UR_GET_INTERFACE:
4408 			goto tr_handle_get_interface;
4409 		case UR_GET_STATUS:
4410 			goto tr_handle_get_iface_status;
4411 		default:
4412 			goto tr_stalled;
4413 		}
4414 		break;
4415 
4416 	case UT_WRITE_CLASS_INTERFACE:
4417 	case UT_WRITE_VENDOR_INTERFACE:
4418 		/* XXX forward */
4419 		break;
4420 
4421 	case UT_READ_CLASS_INTERFACE:
4422 	case UT_READ_VENDOR_INTERFACE:
4423 		/* XXX forward */
4424 		break;
4425 
4426 	case UT_WRITE_CLASS_DEVICE:
4427 		switch (req->bRequest) {
4428 		case UR_CLEAR_FEATURE:
4429 			goto tr_valid;
4430 		case UR_SET_DESCRIPTOR:
4431 		case UR_SET_FEATURE:
4432 			break;
4433 		default:
4434 			goto tr_stalled;
4435 		}
4436 		break;
4437 
4438 	case UT_WRITE_CLASS_OTHER:
4439 		switch (req->bRequest) {
4440 		case UR_CLEAR_FEATURE:
4441 			goto tr_handle_clear_port_feature;
4442 		case UR_SET_FEATURE:
4443 			goto tr_handle_set_port_feature;
4444 		case UR_CLEAR_TT_BUFFER:
4445 		case UR_RESET_TT:
4446 		case UR_STOP_TT:
4447 			goto tr_valid;
4448 
4449 		default:
4450 			goto tr_stalled;
4451 		}
4452 		break;
4453 
4454 	case UT_READ_CLASS_OTHER:
4455 		switch (req->bRequest) {
4456 		case UR_GET_TT_STATE:
4457 			goto tr_handle_get_tt_state;
4458 		case UR_GET_STATUS:
4459 			goto tr_handle_get_port_status;
4460 		default:
4461 			goto tr_stalled;
4462 		}
4463 		break;
4464 
4465 	case UT_READ_CLASS_DEVICE:
4466 		switch (req->bRequest) {
4467 		case UR_GET_DESCRIPTOR:
4468 			goto tr_handle_get_class_descriptor;
4469 		case UR_GET_STATUS:
4470 			goto tr_handle_get_class_status;
4471 
4472 		default:
4473 			goto tr_stalled;
4474 		}
4475 		break;
4476 	default:
4477 		goto tr_stalled;
4478 	}
4479 	goto tr_valid;
4480 
4481 tr_handle_get_descriptor:
4482 	switch (value >> 8) {
4483 	case UDESC_DEVICE:
4484 		if (value & 0xff) {
4485 			goto tr_stalled;
4486 		}
4487 		len = sizeof(dwc_otg_devd);
4488 		ptr = (const void *)&dwc_otg_devd;
4489 		goto tr_valid;
4490 	case UDESC_CONFIG:
4491 		if (value & 0xff) {
4492 			goto tr_stalled;
4493 		}
4494 		len = sizeof(dwc_otg_confd);
4495 		ptr = (const void *)&dwc_otg_confd;
4496 		goto tr_valid;
4497 	case UDESC_STRING:
4498 		switch (value & 0xff) {
4499 		case 0:		/* Language table */
4500 			len = sizeof(usb_string_lang_en);
4501 			ptr = (const void *)&usb_string_lang_en;
4502 			goto tr_valid;
4503 
4504 		case 1:		/* Vendor */
4505 			len = sizeof(dwc_otg_vendor);
4506 			ptr = (const void *)&dwc_otg_vendor;
4507 			goto tr_valid;
4508 
4509 		case 2:		/* Product */
4510 			len = sizeof(dwc_otg_product);
4511 			ptr = (const void *)&dwc_otg_product;
4512 			goto tr_valid;
4513 		default:
4514 			break;
4515 		}
4516 		break;
4517 	default:
4518 		goto tr_stalled;
4519 	}
4520 	goto tr_stalled;
4521 
4522 tr_handle_get_config:
4523 	len = 1;
4524 	sc->sc_hub_temp.wValue[0] = sc->sc_conf;
4525 	goto tr_valid;
4526 
4527 tr_handle_get_status:
4528 	len = 2;
4529 	USETW(sc->sc_hub_temp.wValue, UDS_SELF_POWERED);
4530 	goto tr_valid;
4531 
4532 tr_handle_set_address:
4533 	if (value & 0xFF00) {
4534 		goto tr_stalled;
4535 	}
4536 	sc->sc_rt_addr = value;
4537 	goto tr_valid;
4538 
4539 tr_handle_set_config:
4540 	if (value >= 2) {
4541 		goto tr_stalled;
4542 	}
4543 	sc->sc_conf = value;
4544 	goto tr_valid;
4545 
4546 tr_handle_get_interface:
4547 	len = 1;
4548 	sc->sc_hub_temp.wValue[0] = 0;
4549 	goto tr_valid;
4550 
4551 tr_handle_get_tt_state:
4552 tr_handle_get_class_status:
4553 tr_handle_get_iface_status:
4554 tr_handle_get_ep_status:
4555 	len = 2;
4556 	USETW(sc->sc_hub_temp.wValue, 0);
4557 	goto tr_valid;
4558 
4559 tr_handle_set_halt:
4560 tr_handle_set_interface:
4561 tr_handle_set_wakeup:
4562 tr_handle_clear_wakeup:
4563 tr_handle_clear_halt:
4564 	goto tr_valid;
4565 
4566 tr_handle_clear_port_feature:
4567 	if (index != 1)
4568 		goto tr_stalled;
4569 
4570 	DPRINTFN(9, "UR_CLEAR_PORT_FEATURE on port %d\n", index);
4571 
4572 	switch (value) {
4573 	case UHF_PORT_SUSPEND:
4574 		dwc_otg_wakeup_peer(sc);
4575 		break;
4576 
4577 	case UHF_PORT_ENABLE:
4578 		if (sc->sc_flags.status_device_mode == 0) {
4579 			DWC_OTG_WRITE_4(sc, DOTG_HPRT,
4580 			    sc->sc_hprt_val | HPRT_PRTENA);
4581 		}
4582 		sc->sc_flags.port_enabled = 0;
4583 		break;
4584 
4585 	case UHF_C_PORT_RESET:
4586 		sc->sc_flags.change_reset = 0;
4587 		break;
4588 
4589 	case UHF_C_PORT_ENABLE:
4590 		sc->sc_flags.change_enabled = 0;
4591 		break;
4592 
4593 	case UHF_C_PORT_OVER_CURRENT:
4594 		sc->sc_flags.change_over_current = 0;
4595 		break;
4596 
4597 	case UHF_PORT_TEST:
4598 	case UHF_PORT_INDICATOR:
4599 		/* nops */
4600 		break;
4601 
4602 	case UHF_PORT_POWER:
4603 		sc->sc_flags.port_powered = 0;
4604 		if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4605 			sc->sc_hprt_val = 0;
4606 			DWC_OTG_WRITE_4(sc, DOTG_HPRT, HPRT_PRTENA);
4607 		}
4608 		dwc_otg_pull_down(sc);
4609 		dwc_otg_clocks_off(sc);
4610 		break;
4611 
4612 	case UHF_C_PORT_CONNECTION:
4613 		/* clear connect change flag */
4614 		sc->sc_flags.change_connect = 0;
4615 		break;
4616 
4617 	case UHF_C_PORT_SUSPEND:
4618 		sc->sc_flags.change_suspend = 0;
4619 		break;
4620 
4621 	default:
4622 		err = USB_ERR_IOERROR;
4623 		goto done;
4624 	}
4625 	goto tr_valid;
4626 
4627 tr_handle_set_port_feature:
4628 	if (index != 1) {
4629 		goto tr_stalled;
4630 	}
4631 	DPRINTFN(9, "UR_SET_PORT_FEATURE\n");
4632 
4633 	switch (value) {
4634 	case UHF_PORT_ENABLE:
4635 		break;
4636 
4637 	case UHF_PORT_SUSPEND:
4638 		if (sc->sc_flags.status_device_mode == 0) {
4639 			/* set suspend BIT */
4640 			sc->sc_hprt_val |= HPRT_PRTSUSP;
4641 			DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4642 
4643 			/* generate HUB suspend event */
4644 			dwc_otg_suspend_irq(sc);
4645 		}
4646 		break;
4647 
4648 	case UHF_PORT_RESET:
4649 		if (sc->sc_flags.status_device_mode == 0) {
4650 			DPRINTF("PORT RESET\n");
4651 
4652 			/* enable PORT reset */
4653 			DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val | HPRT_PRTRST);
4654 
4655 			/* Wait 62.5ms for reset to complete */
4656 			usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4657 
4658 			DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4659 
4660 			/* Wait 62.5ms for reset to complete */
4661 			usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4662 
4663 			/* reset FIFOs */
4664 			(void) dwc_otg_init_fifo(sc, DWC_MODE_HOST);
4665 
4666 			sc->sc_flags.change_reset = 1;
4667 		} else {
4668 			err = USB_ERR_IOERROR;
4669 		}
4670 		break;
4671 
4672 	case UHF_PORT_TEST:
4673 	case UHF_PORT_INDICATOR:
4674 		/* nops */
4675 		break;
4676 	case UHF_PORT_POWER:
4677 		sc->sc_flags.port_powered = 1;
4678 		if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4679 			sc->sc_hprt_val |= HPRT_PRTPWR;
4680 			DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4681 		}
4682 		if (sc->sc_mode == DWC_MODE_DEVICE || sc->sc_mode == DWC_MODE_OTG) {
4683 			/* pull up D+, if any */
4684 			dwc_otg_pull_up(sc);
4685 		}
4686 		break;
4687 	default:
4688 		err = USB_ERR_IOERROR;
4689 		goto done;
4690 	}
4691 	goto tr_valid;
4692 
4693 tr_handle_get_port_status:
4694 
4695 	DPRINTFN(9, "UR_GET_PORT_STATUS\n");
4696 
4697 	if (index != 1)
4698 		goto tr_stalled;
4699 
4700 	if (sc->sc_flags.status_vbus)
4701 		dwc_otg_clocks_on(sc);
4702 	else
4703 		dwc_otg_clocks_off(sc);
4704 
4705 	/* Select Device Side Mode */
4706 
4707 	if (sc->sc_flags.status_device_mode) {
4708 		value = UPS_PORT_MODE_DEVICE;
4709 		dwc_otg_timer_stop(sc);
4710 	} else {
4711 		value = 0;
4712 		dwc_otg_timer_start(sc);
4713 	}
4714 
4715 	if (sc->sc_flags.status_high_speed)
4716 		value |= UPS_HIGH_SPEED;
4717 	else if (sc->sc_flags.status_low_speed)
4718 		value |= UPS_LOW_SPEED;
4719 
4720 	if (sc->sc_flags.port_powered)
4721 		value |= UPS_PORT_POWER;
4722 
4723 	if (sc->sc_flags.port_enabled)
4724 		value |= UPS_PORT_ENABLED;
4725 
4726 	if (sc->sc_flags.port_over_current)
4727 		value |= UPS_OVERCURRENT_INDICATOR;
4728 
4729 	if (sc->sc_flags.status_vbus &&
4730 	    sc->sc_flags.status_bus_reset)
4731 		value |= UPS_CURRENT_CONNECT_STATUS;
4732 
4733 	if (sc->sc_flags.status_suspend)
4734 		value |= UPS_SUSPEND;
4735 
4736 	USETW(sc->sc_hub_temp.ps.wPortStatus, value);
4737 
4738 	value = 0;
4739 
4740 	if (sc->sc_flags.change_enabled)
4741 		value |= UPS_C_PORT_ENABLED;
4742 	if (sc->sc_flags.change_connect)
4743 		value |= UPS_C_CONNECT_STATUS;
4744 	if (sc->sc_flags.change_suspend)
4745 		value |= UPS_C_SUSPEND;
4746 	if (sc->sc_flags.change_reset)
4747 		value |= UPS_C_PORT_RESET;
4748 	if (sc->sc_flags.change_over_current)
4749 		value |= UPS_C_OVERCURRENT_INDICATOR;
4750 
4751 	USETW(sc->sc_hub_temp.ps.wPortChange, value);
4752 	len = sizeof(sc->sc_hub_temp.ps);
4753 	goto tr_valid;
4754 
4755 tr_handle_get_class_descriptor:
4756 	if (value & 0xFF) {
4757 		goto tr_stalled;
4758 	}
4759 	ptr = (const void *)&dwc_otg_hubd;
4760 	len = sizeof(dwc_otg_hubd);
4761 	goto tr_valid;
4762 
4763 tr_stalled:
4764 	err = USB_ERR_STALLED;
4765 tr_valid:
4766 done:
4767 	*plength = len;
4768 	*pptr = ptr;
4769 	return (err);
4770 }
4771 
4772 static void
4773 dwc_otg_xfer_setup(struct usb_setup_params *parm)
4774 {
4775 	struct usb_xfer *xfer;
4776 	void *last_obj;
4777 	uint32_t ntd;
4778 	uint32_t n;
4779 	uint8_t ep_no;
4780 	uint8_t ep_type;
4781 
4782 	xfer = parm->curr_xfer;
4783 
4784 	/*
4785 	 * NOTE: This driver does not use any of the parameters that
4786 	 * are computed from the following values. Just set some
4787 	 * reasonable dummies:
4788 	 */
4789 	parm->hc_max_packet_size = 0x500;
4790 	parm->hc_max_packet_count = 3;
4791 	parm->hc_max_frame_size = 3 * 0x500;
4792 
4793 	usbd_transfer_setup_sub(parm);
4794 
4795 	/*
4796 	 * compute maximum number of TDs
4797 	 */
4798 	ep_type = (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE);
4799 
4800 	if (ep_type == UE_CONTROL) {
4801 		ntd = xfer->nframes + 1 /* STATUS */ + 1 /* SYNC 1 */
4802 		    + 1 /* SYNC 2 */ + 1 /* SYNC 3 */;
4803 	} else {
4804 		ntd = xfer->nframes + 1 /* SYNC */ ;
4805 	}
4806 
4807 	/*
4808 	 * check if "usbd_transfer_setup_sub" set an error
4809 	 */
4810 	if (parm->err)
4811 		return;
4812 
4813 	/*
4814 	 * allocate transfer descriptors
4815 	 */
4816 	last_obj = NULL;
4817 
4818 	ep_no = xfer->endpointno & UE_ADDR;
4819 
4820 	/*
4821 	 * Check for a valid endpoint profile in USB device mode:
4822 	 */
4823 	if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
4824 		const struct usb_hw_ep_profile *pf;
4825 
4826 		dwc_otg_get_hw_ep_profile(parm->udev, &pf, ep_no);
4827 
4828 		if (pf == NULL) {
4829 			/* should not happen */
4830 			parm->err = USB_ERR_INVAL;
4831 			return;
4832 		}
4833 	}
4834 
4835 	/* align data */
4836 	parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
4837 
4838 	for (n = 0; n != ntd; n++) {
4839 		struct dwc_otg_td *td;
4840 
4841 		if (parm->buf) {
4842 			td = USB_ADD_BYTES(parm->buf, parm->size[0]);
4843 
4844 			/* compute shared bandwidth resource index for TT */
4845 			if (dwc_otg_uses_split(parm->udev)) {
4846 				if (parm->udev->parent_hs_hub->ddesc.bDeviceProtocol == UDPROTO_HSHUBMTT)
4847 					td->tt_index = parm->udev->device_index;
4848 				else
4849 					td->tt_index = parm->udev->parent_hs_hub->device_index;
4850 			} else {
4851 				td->tt_index = parm->udev->device_index;
4852 			}
4853 
4854 			/* init TD */
4855 			td->max_packet_size = xfer->max_packet_size;
4856 			td->max_packet_count = xfer->max_packet_count;
4857 			/* range check */
4858 			if (td->max_packet_count == 0 || td->max_packet_count > 3)
4859 				td->max_packet_count = 1;
4860 			td->ep_no = ep_no;
4861 			td->ep_type = ep_type;
4862 			td->obj_next = last_obj;
4863 
4864 			last_obj = td;
4865 		}
4866 		parm->size[0] += sizeof(*td);
4867 	}
4868 
4869 	xfer->td_start[0] = last_obj;
4870 }
4871 
4872 static void
4873 dwc_otg_xfer_unsetup(struct usb_xfer *xfer)
4874 {
4875 	return;
4876 }
4877 
4878 static void
4879 dwc_otg_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
4880     struct usb_endpoint *ep)
4881 {
4882 	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4883 
4884 	DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d,%d)\n",
4885 	    ep, udev->address,
4886 	    edesc->bEndpointAddress, udev->flags.usb_mode,
4887 	    sc->sc_rt_addr, udev->device_index);
4888 
4889 	if (udev->device_index != sc->sc_rt_addr) {
4890 		if (udev->flags.usb_mode == USB_MODE_DEVICE) {
4891 			if (udev->speed != USB_SPEED_FULL &&
4892 			    udev->speed != USB_SPEED_HIGH) {
4893 				/* not supported */
4894 				return;
4895 			}
4896 		} else {
4897 			if (udev->speed == USB_SPEED_HIGH &&
4898 			    (edesc->wMaxPacketSize[1] & 0x18) != 0 &&
4899 			    (edesc->bmAttributes & UE_XFERTYPE) != UE_ISOCHRONOUS) {
4900 				/* not supported */
4901 				DPRINTFN(-1, "Non-isochronous high bandwidth "
4902 				    "endpoint not supported\n");
4903 				return;
4904 			}
4905 		}
4906 		if ((edesc->bmAttributes & UE_XFERTYPE) == UE_ISOCHRONOUS)
4907 			ep->methods = &dwc_otg_device_isoc_methods;
4908 		else
4909 			ep->methods = &dwc_otg_device_non_isoc_methods;
4910 	}
4911 }
4912 
4913 static void
4914 dwc_otg_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
4915 {
4916 	struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
4917 
4918 	switch (state) {
4919 	case USB_HW_POWER_SUSPEND:
4920 		dwc_otg_suspend(sc);
4921 		break;
4922 	case USB_HW_POWER_SHUTDOWN:
4923 		dwc_otg_uninit(sc);
4924 		break;
4925 	case USB_HW_POWER_RESUME:
4926 		dwc_otg_resume(sc);
4927 		break;
4928 	default:
4929 		break;
4930 	}
4931 }
4932 
4933 static void
4934 dwc_otg_get_dma_delay(struct usb_device *udev, uint32_t *pus)
4935 {
4936 	/* DMA delay - wait until any use of memory is finished */
4937 	*pus = (2125);			/* microseconds */
4938 }
4939 
4940 static void
4941 dwc_otg_device_resume(struct usb_device *udev)
4942 {
4943 	DPRINTF("\n");
4944 
4945 	/* poll all transfers again to restart resumed ones */
4946 	dwc_otg_do_poll(udev->bus);
4947 }
4948 
4949 static void
4950 dwc_otg_device_suspend(struct usb_device *udev)
4951 {
4952 	DPRINTF("\n");
4953 }
4954 
4955 static const struct usb_bus_methods dwc_otg_bus_methods =
4956 {
4957 	.endpoint_init = &dwc_otg_ep_init,
4958 	.xfer_setup = &dwc_otg_xfer_setup,
4959 	.xfer_unsetup = &dwc_otg_xfer_unsetup,
4960 	.get_hw_ep_profile = &dwc_otg_get_hw_ep_profile,
4961 	.xfer_stall = &dwc_otg_xfer_stall,
4962 	.set_stall = &dwc_otg_set_stall,
4963 	.clear_stall = &dwc_otg_clear_stall,
4964 	.roothub_exec = &dwc_otg_roothub_exec,
4965 	.xfer_poll = &dwc_otg_do_poll,
4966 	.device_state_change = &dwc_otg_device_state_change,
4967 	.set_hw_power_sleep = &dwc_otg_set_hw_power_sleep,
4968 	.get_dma_delay = &dwc_otg_get_dma_delay,
4969 	.device_resume = &dwc_otg_device_resume,
4970 	.device_suspend = &dwc_otg_device_suspend,
4971 };
4972