1 /* $FreeBSD$ */ 2 /*- 3 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 4 * 5 * Copyright (c) 2010 Hans Petter Selasky. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * USB eXtensible Host Controller Interface, a.k.a. USB 3.0 controller. 31 * 32 * The XHCI 1.0 spec can be found at 33 * http://www.intel.com/technology/usb/download/xHCI_Specification_for_USB.pdf 34 * and the USB 3.0 spec at 35 * http://www.usb.org/developers/docs/usb_30_spec_060910.zip 36 */ 37 38 /* 39 * A few words about the design implementation: This driver emulates 40 * the concept about TDs which is found in EHCI specification. This 41 * way we achieve that the USB controller drivers look similar to 42 * eachother which makes it easier to understand the code. 43 */ 44 45 #ifdef USB_GLOBAL_INCLUDE_FILE 46 #include USB_GLOBAL_INCLUDE_FILE 47 #else 48 #include <sys/stdint.h> 49 #include <sys/stddef.h> 50 #include <sys/param.h> 51 #include <sys/queue.h> 52 #include <sys/types.h> 53 #include <sys/systm.h> 54 #include <sys/kernel.h> 55 #include <sys/bus.h> 56 #include <sys/module.h> 57 #include <sys/lock.h> 58 #include <sys/mutex.h> 59 #include <sys/condvar.h> 60 #include <sys/sysctl.h> 61 #include <sys/sx.h> 62 #include <sys/unistd.h> 63 #include <sys/callout.h> 64 #include <sys/malloc.h> 65 #include <sys/priv.h> 66 67 #include <dev/usb/usb.h> 68 #include <dev/usb/usbdi.h> 69 70 #define USB_DEBUG_VAR xhcidebug 71 72 #include <dev/usb/usb_core.h> 73 #include <dev/usb/usb_debug.h> 74 #include <dev/usb/usb_busdma.h> 75 #include <dev/usb/usb_process.h> 76 #include <dev/usb/usb_transfer.h> 77 #include <dev/usb/usb_device.h> 78 #include <dev/usb/usb_hub.h> 79 #include <dev/usb/usb_util.h> 80 81 #include <dev/usb/usb_controller.h> 82 #include <dev/usb/usb_bus.h> 83 #endif /* USB_GLOBAL_INCLUDE_FILE */ 84 85 #include <dev/usb/controller/xhci.h> 86 #include <dev/usb/controller/xhcireg.h> 87 88 #define XHCI_BUS2SC(bus) \ 89 ((struct xhci_softc *)(((uint8_t *)(bus)) - \ 90 ((uint8_t *)&(((struct xhci_softc *)0)->sc_bus)))) 91 92 static SYSCTL_NODE(_hw_usb, OID_AUTO, xhci, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 93 "USB XHCI"); 94 95 static int xhcistreams; 96 SYSCTL_INT(_hw_usb_xhci, OID_AUTO, streams, CTLFLAG_RWTUN, 97 &xhcistreams, 0, "Set to enable streams mode support"); 98 99 static int xhcictlquirk = 1; 100 SYSCTL_INT(_hw_usb_xhci, OID_AUTO, ctlquirk, CTLFLAG_RWTUN, 101 &xhcictlquirk, 0, "Set to enable control endpoint quirk"); 102 103 #ifdef USB_DEBUG 104 static int xhcidebug; 105 static int xhciroute; 106 static int xhcipolling; 107 static int xhcidma32; 108 static int xhcictlstep; 109 110 SYSCTL_INT(_hw_usb_xhci, OID_AUTO, debug, CTLFLAG_RWTUN, 111 &xhcidebug, 0, "Debug level"); 112 SYSCTL_INT(_hw_usb_xhci, OID_AUTO, xhci_port_route, CTLFLAG_RWTUN, 113 &xhciroute, 0, "Routing bitmap for switching EHCI ports to the XHCI controller"); 114 SYSCTL_INT(_hw_usb_xhci, OID_AUTO, use_polling, CTLFLAG_RWTUN, 115 &xhcipolling, 0, "Set to enable software interrupt polling for the XHCI controller"); 116 SYSCTL_INT(_hw_usb_xhci, OID_AUTO, dma32, CTLFLAG_RWTUN, 117 &xhcidma32, 0, "Set to only use 32-bit DMA for the XHCI controller"); 118 SYSCTL_INT(_hw_usb_xhci, OID_AUTO, ctlstep, CTLFLAG_RWTUN, 119 &xhcictlstep, 0, "Set to enable control endpoint status stage stepping"); 120 #else 121 #define xhciroute 0 122 #define xhcidma32 0 123 #define xhcictlstep 0 124 #endif 125 126 #define XHCI_INTR_ENDPT 1 127 128 struct xhci_std_temp { 129 struct xhci_softc *sc; 130 struct usb_page_cache *pc; 131 struct xhci_td *td; 132 struct xhci_td *td_next; 133 uint32_t len; 134 uint32_t offset; 135 uint32_t max_packet_size; 136 uint32_t average; 137 uint16_t isoc_delta; 138 uint16_t isoc_frame; 139 uint8_t shortpkt; 140 uint8_t multishort; 141 uint8_t last_frame; 142 uint8_t trb_type; 143 uint8_t direction; 144 uint8_t tbc; 145 uint8_t tlbpc; 146 uint8_t step_td; 147 uint8_t do_isoc_sync; 148 }; 149 150 static void xhci_do_poll(struct usb_bus *); 151 static void xhci_device_done(struct usb_xfer *, usb_error_t); 152 static void xhci_root_intr(struct xhci_softc *); 153 static void xhci_free_device_ext(struct usb_device *); 154 static struct xhci_endpoint_ext *xhci_get_endpoint_ext(struct usb_device *, 155 struct usb_endpoint_descriptor *); 156 static usb_proc_callback_t xhci_configure_msg; 157 static usb_error_t xhci_configure_device(struct usb_device *); 158 static usb_error_t xhci_configure_endpoint(struct usb_device *, 159 struct usb_endpoint_descriptor *, struct xhci_endpoint_ext *, 160 uint16_t, uint8_t, uint8_t, uint8_t, uint16_t, uint16_t, 161 uint8_t); 162 static usb_error_t xhci_configure_mask(struct usb_device *, 163 uint32_t, uint8_t); 164 static usb_error_t xhci_cmd_evaluate_ctx(struct xhci_softc *, 165 uint64_t, uint8_t); 166 static void xhci_endpoint_doorbell(struct usb_xfer *); 167 static void xhci_ctx_set_le32(struct xhci_softc *sc, volatile uint32_t *ptr, uint32_t val); 168 static uint32_t xhci_ctx_get_le32(struct xhci_softc *sc, volatile uint32_t *ptr); 169 static void xhci_ctx_set_le64(struct xhci_softc *sc, volatile uint64_t *ptr, uint64_t val); 170 #ifdef USB_DEBUG 171 static uint64_t xhci_ctx_get_le64(struct xhci_softc *sc, volatile uint64_t *ptr); 172 #endif 173 174 static const struct usb_bus_methods xhci_bus_methods; 175 176 #ifdef USB_DEBUG 177 static void 178 xhci_dump_trb(struct xhci_trb *trb) 179 { 180 DPRINTFN(5, "trb = %p\n", trb); 181 DPRINTFN(5, "qwTrb0 = 0x%016llx\n", (long long)le64toh(trb->qwTrb0)); 182 DPRINTFN(5, "dwTrb2 = 0x%08x\n", le32toh(trb->dwTrb2)); 183 DPRINTFN(5, "dwTrb3 = 0x%08x\n", le32toh(trb->dwTrb3)); 184 } 185 186 static void 187 xhci_dump_endpoint(struct xhci_softc *sc, struct xhci_endp_ctx *pep) 188 { 189 DPRINTFN(5, "pep = %p\n", pep); 190 DPRINTFN(5, "dwEpCtx0=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx0)); 191 DPRINTFN(5, "dwEpCtx1=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx1)); 192 DPRINTFN(5, "qwEpCtx2=0x%016llx\n", (long long)xhci_ctx_get_le64(sc, &pep->qwEpCtx2)); 193 DPRINTFN(5, "dwEpCtx4=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx4)); 194 DPRINTFN(5, "dwEpCtx5=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx5)); 195 DPRINTFN(5, "dwEpCtx6=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx6)); 196 DPRINTFN(5, "dwEpCtx7=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx7)); 197 } 198 199 static void 200 xhci_dump_device(struct xhci_softc *sc, struct xhci_slot_ctx *psl) 201 { 202 DPRINTFN(5, "psl = %p\n", psl); 203 DPRINTFN(5, "dwSctx0=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx0)); 204 DPRINTFN(5, "dwSctx1=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx1)); 205 DPRINTFN(5, "dwSctx2=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx2)); 206 DPRINTFN(5, "dwSctx3=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx3)); 207 } 208 #endif 209 210 uint8_t 211 xhci_use_polling(void) 212 { 213 #ifdef USB_DEBUG 214 return (xhcipolling != 0); 215 #else 216 return (0); 217 #endif 218 } 219 220 static void 221 xhci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb) 222 { 223 struct xhci_softc *sc = XHCI_BUS2SC(bus); 224 uint16_t i; 225 226 cb(bus, &sc->sc_hw.root_pc, &sc->sc_hw.root_pg, 227 sizeof(struct xhci_hw_root), XHCI_PAGE_SIZE); 228 229 cb(bus, &sc->sc_hw.ctx_pc, &sc->sc_hw.ctx_pg, 230 sizeof(struct xhci_dev_ctx_addr), XHCI_PAGE_SIZE); 231 232 for (i = 0; i != sc->sc_noscratch; i++) { 233 cb(bus, &sc->sc_hw.scratch_pc[i], &sc->sc_hw.scratch_pg[i], 234 XHCI_PAGE_SIZE, XHCI_PAGE_SIZE); 235 } 236 } 237 238 static void 239 xhci_ctx_set_le32(struct xhci_softc *sc, volatile uint32_t *ptr, uint32_t val) 240 { 241 if (sc->sc_ctx_is_64_byte) { 242 uint32_t offset; 243 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */ 244 /* all contexts are initially 32-bytes */ 245 offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U)); 246 ptr = (volatile uint32_t *)(((volatile uint8_t *)ptr) + offset); 247 } 248 *ptr = htole32(val); 249 } 250 251 static uint32_t 252 xhci_ctx_get_le32(struct xhci_softc *sc, volatile uint32_t *ptr) 253 { 254 if (sc->sc_ctx_is_64_byte) { 255 uint32_t offset; 256 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */ 257 /* all contexts are initially 32-bytes */ 258 offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U)); 259 ptr = (volatile uint32_t *)(((volatile uint8_t *)ptr) + offset); 260 } 261 return (le32toh(*ptr)); 262 } 263 264 static void 265 xhci_ctx_set_le64(struct xhci_softc *sc, volatile uint64_t *ptr, uint64_t val) 266 { 267 if (sc->sc_ctx_is_64_byte) { 268 uint32_t offset; 269 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */ 270 /* all contexts are initially 32-bytes */ 271 offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U)); 272 ptr = (volatile uint64_t *)(((volatile uint8_t *)ptr) + offset); 273 } 274 *ptr = htole64(val); 275 } 276 277 #ifdef USB_DEBUG 278 static uint64_t 279 xhci_ctx_get_le64(struct xhci_softc *sc, volatile uint64_t *ptr) 280 { 281 if (sc->sc_ctx_is_64_byte) { 282 uint32_t offset; 283 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */ 284 /* all contexts are initially 32-bytes */ 285 offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U)); 286 ptr = (volatile uint64_t *)(((volatile uint8_t *)ptr) + offset); 287 } 288 return (le64toh(*ptr)); 289 } 290 #endif 291 292 static int 293 xhci_reset_command_queue_locked(struct xhci_softc *sc) 294 { 295 struct usb_page_search buf_res; 296 struct xhci_hw_root *phwr; 297 uint64_t addr; 298 uint32_t temp; 299 300 DPRINTF("\n"); 301 302 temp = XREAD4(sc, oper, XHCI_CRCR_LO); 303 if (temp & XHCI_CRCR_LO_CRR) { 304 DPRINTF("Command ring running\n"); 305 temp &= ~(XHCI_CRCR_LO_CS | XHCI_CRCR_LO_CA); 306 307 /* 308 * Try to abort the last command as per section 309 * 4.6.1.2 "Aborting a Command" of the XHCI 310 * specification: 311 */ 312 313 /* stop and cancel */ 314 XWRITE4(sc, oper, XHCI_CRCR_LO, temp | XHCI_CRCR_LO_CS); 315 XWRITE4(sc, oper, XHCI_CRCR_HI, 0); 316 317 XWRITE4(sc, oper, XHCI_CRCR_LO, temp | XHCI_CRCR_LO_CA); 318 XWRITE4(sc, oper, XHCI_CRCR_HI, 0); 319 320 /* wait 250ms */ 321 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 4); 322 323 /* check if command ring is still running */ 324 temp = XREAD4(sc, oper, XHCI_CRCR_LO); 325 if (temp & XHCI_CRCR_LO_CRR) { 326 DPRINTF("Comand ring still running\n"); 327 return (USB_ERR_IOERROR); 328 } 329 } 330 331 /* reset command ring */ 332 sc->sc_command_ccs = 1; 333 sc->sc_command_idx = 0; 334 335 usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res); 336 337 /* set up command ring control base address */ 338 addr = buf_res.physaddr; 339 phwr = buf_res.buffer; 340 addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_commands[0]; 341 342 DPRINTF("CRCR=0x%016llx\n", (unsigned long long)addr); 343 344 memset(phwr->hwr_commands, 0, sizeof(phwr->hwr_commands)); 345 phwr->hwr_commands[XHCI_MAX_COMMANDS - 1].qwTrb0 = htole64(addr); 346 347 usb_pc_cpu_flush(&sc->sc_hw.root_pc); 348 349 XWRITE4(sc, oper, XHCI_CRCR_LO, ((uint32_t)addr) | XHCI_CRCR_LO_RCS); 350 XWRITE4(sc, oper, XHCI_CRCR_HI, (uint32_t)(addr >> 32)); 351 352 return (0); 353 } 354 355 usb_error_t 356 xhci_start_controller(struct xhci_softc *sc) 357 { 358 struct usb_page_search buf_res; 359 struct xhci_hw_root *phwr; 360 struct xhci_dev_ctx_addr *pdctxa; 361 usb_error_t err; 362 uint64_t addr; 363 uint32_t temp; 364 uint16_t i; 365 366 DPRINTF("\n"); 367 368 sc->sc_event_ccs = 1; 369 sc->sc_event_idx = 0; 370 sc->sc_command_ccs = 1; 371 sc->sc_command_idx = 0; 372 373 err = xhci_reset_controller(sc); 374 if (err) 375 return (err); 376 377 /* set up number of device slots */ 378 DPRINTF("CONFIG=0x%08x -> 0x%08x\n", 379 XREAD4(sc, oper, XHCI_CONFIG), sc->sc_noslot); 380 381 XWRITE4(sc, oper, XHCI_CONFIG, sc->sc_noslot); 382 383 temp = XREAD4(sc, oper, XHCI_USBSTS); 384 385 /* clear interrupts */ 386 XWRITE4(sc, oper, XHCI_USBSTS, temp); 387 /* disable all device notifications */ 388 XWRITE4(sc, oper, XHCI_DNCTRL, 0); 389 390 /* set up device context base address */ 391 usbd_get_page(&sc->sc_hw.ctx_pc, 0, &buf_res); 392 pdctxa = buf_res.buffer; 393 memset(pdctxa, 0, sizeof(*pdctxa)); 394 395 addr = buf_res.physaddr; 396 addr += (uintptr_t)&((struct xhci_dev_ctx_addr *)0)->qwSpBufPtr[0]; 397 398 /* slot 0 points to the table of scratchpad pointers */ 399 pdctxa->qwBaaDevCtxAddr[0] = htole64(addr); 400 401 for (i = 0; i != sc->sc_noscratch; i++) { 402 struct usb_page_search buf_scp; 403 usbd_get_page(&sc->sc_hw.scratch_pc[i], 0, &buf_scp); 404 pdctxa->qwSpBufPtr[i] = htole64((uint64_t)buf_scp.physaddr); 405 } 406 407 addr = buf_res.physaddr; 408 409 XWRITE4(sc, oper, XHCI_DCBAAP_LO, (uint32_t)addr); 410 XWRITE4(sc, oper, XHCI_DCBAAP_HI, (uint32_t)(addr >> 32)); 411 XWRITE4(sc, oper, XHCI_DCBAAP_LO, (uint32_t)addr); 412 XWRITE4(sc, oper, XHCI_DCBAAP_HI, (uint32_t)(addr >> 32)); 413 414 /* set up event table size */ 415 DPRINTF("ERSTSZ=0x%08x -> 0x%08x\n", 416 XREAD4(sc, runt, XHCI_ERSTSZ(0)), sc->sc_erst_max); 417 418 XWRITE4(sc, runt, XHCI_ERSTSZ(0), XHCI_ERSTS_SET(sc->sc_erst_max)); 419 420 /* set up interrupt rate */ 421 XWRITE4(sc, runt, XHCI_IMOD(0), sc->sc_imod_default); 422 423 usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res); 424 425 phwr = buf_res.buffer; 426 addr = buf_res.physaddr; 427 addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_events[0]; 428 429 /* reset hardware root structure */ 430 memset(phwr, 0, sizeof(*phwr)); 431 432 phwr->hwr_ring_seg[0].qwEvrsTablePtr = htole64(addr); 433 phwr->hwr_ring_seg[0].dwEvrsTableSize = htole32(XHCI_MAX_EVENTS); 434 435 DPRINTF("ERDP(0)=0x%016llx\n", (unsigned long long)addr); 436 437 XWRITE4(sc, runt, XHCI_ERDP_LO(0), (uint32_t)addr); 438 XWRITE4(sc, runt, XHCI_ERDP_HI(0), (uint32_t)(addr >> 32)); 439 440 addr = buf_res.physaddr; 441 442 DPRINTF("ERSTBA(0)=0x%016llx\n", (unsigned long long)addr); 443 444 XWRITE4(sc, runt, XHCI_ERSTBA_LO(0), (uint32_t)addr); 445 XWRITE4(sc, runt, XHCI_ERSTBA_HI(0), (uint32_t)(addr >> 32)); 446 447 /* set up interrupter registers */ 448 temp = XREAD4(sc, runt, XHCI_IMAN(0)); 449 temp |= XHCI_IMAN_INTR_ENA; 450 XWRITE4(sc, runt, XHCI_IMAN(0), temp); 451 452 /* set up command ring control base address */ 453 addr = buf_res.physaddr; 454 addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_commands[0]; 455 456 DPRINTF("CRCR=0x%016llx\n", (unsigned long long)addr); 457 458 XWRITE4(sc, oper, XHCI_CRCR_LO, ((uint32_t)addr) | XHCI_CRCR_LO_RCS); 459 XWRITE4(sc, oper, XHCI_CRCR_HI, (uint32_t)(addr >> 32)); 460 461 phwr->hwr_commands[XHCI_MAX_COMMANDS - 1].qwTrb0 = htole64(addr); 462 463 usb_bus_mem_flush_all(&sc->sc_bus, &xhci_iterate_hw_softc); 464 465 /* Go! */ 466 XWRITE4(sc, oper, XHCI_USBCMD, XHCI_CMD_RS | 467 XHCI_CMD_INTE | XHCI_CMD_HSEE); 468 469 for (i = 0; i != 100; i++) { 470 usb_pause_mtx(NULL, hz / 100); 471 temp = XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_HCH; 472 if (!temp) 473 break; 474 } 475 if (temp) { 476 XWRITE4(sc, oper, XHCI_USBCMD, 0); 477 device_printf(sc->sc_bus.parent, "Run timeout.\n"); 478 return (USB_ERR_IOERROR); 479 } 480 481 /* catch any lost interrupts */ 482 xhci_do_poll(&sc->sc_bus); 483 484 if (sc->sc_port_route != NULL) { 485 /* Route all ports to the XHCI by default */ 486 sc->sc_port_route(sc->sc_bus.parent, 487 ~xhciroute, xhciroute); 488 } 489 return (0); 490 } 491 492 usb_error_t 493 xhci_halt_controller(struct xhci_softc *sc) 494 { 495 uint32_t temp; 496 uint16_t i; 497 498 DPRINTF("\n"); 499 500 sc->sc_capa_off = 0; 501 sc->sc_oper_off = XREAD1(sc, capa, XHCI_CAPLENGTH); 502 sc->sc_runt_off = XREAD4(sc, capa, XHCI_RTSOFF) & ~0xF; 503 sc->sc_door_off = XREAD4(sc, capa, XHCI_DBOFF) & ~0x3; 504 505 /* Halt controller */ 506 XWRITE4(sc, oper, XHCI_USBCMD, 0); 507 508 for (i = 0; i != 100; i++) { 509 usb_pause_mtx(NULL, hz / 100); 510 temp = XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_HCH; 511 if (temp) 512 break; 513 } 514 515 if (!temp) { 516 device_printf(sc->sc_bus.parent, "Controller halt timeout.\n"); 517 return (USB_ERR_IOERROR); 518 } 519 return (0); 520 } 521 522 usb_error_t 523 xhci_reset_controller(struct xhci_softc *sc) 524 { 525 uint32_t temp = 0; 526 uint16_t i; 527 528 DPRINTF("\n"); 529 530 /* Reset controller */ 531 XWRITE4(sc, oper, XHCI_USBCMD, XHCI_CMD_HCRST); 532 533 for (i = 0; i != 100; i++) { 534 usb_pause_mtx(NULL, hz / 100); 535 temp = (XREAD4(sc, oper, XHCI_USBCMD) & XHCI_CMD_HCRST) | 536 (XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_CNR); 537 if (!temp) 538 break; 539 } 540 541 if (temp) { 542 device_printf(sc->sc_bus.parent, "Controller " 543 "reset timeout.\n"); 544 return (USB_ERR_IOERROR); 545 } 546 return (0); 547 } 548 549 usb_error_t 550 xhci_init(struct xhci_softc *sc, device_t self, uint8_t dma32) 551 { 552 uint32_t temp; 553 554 DPRINTF("\n"); 555 556 /* initialize some bus fields */ 557 sc->sc_bus.parent = self; 558 559 /* set the bus revision */ 560 sc->sc_bus.usbrev = USB_REV_3_0; 561 562 /* set up the bus struct */ 563 sc->sc_bus.methods = &xhci_bus_methods; 564 565 /* set up devices array */ 566 sc->sc_bus.devices = sc->sc_devices; 567 sc->sc_bus.devices_max = XHCI_MAX_DEVICES; 568 569 /* set default cycle state in case of early interrupts */ 570 sc->sc_event_ccs = 1; 571 sc->sc_command_ccs = 1; 572 573 /* set up bus space offsets */ 574 sc->sc_capa_off = 0; 575 sc->sc_oper_off = XREAD1(sc, capa, XHCI_CAPLENGTH); 576 sc->sc_runt_off = XREAD4(sc, capa, XHCI_RTSOFF) & ~0x1F; 577 sc->sc_door_off = XREAD4(sc, capa, XHCI_DBOFF) & ~0x3; 578 579 DPRINTF("CAPLENGTH=0x%x\n", sc->sc_oper_off); 580 DPRINTF("RUNTIMEOFFSET=0x%x\n", sc->sc_runt_off); 581 DPRINTF("DOOROFFSET=0x%x\n", sc->sc_door_off); 582 583 DPRINTF("xHCI version = 0x%04x\n", XREAD2(sc, capa, XHCI_HCIVERSION)); 584 585 if (!(XREAD4(sc, oper, XHCI_PAGESIZE) & XHCI_PAGESIZE_4K)) { 586 device_printf(sc->sc_bus.parent, "Controller does " 587 "not support 4K page size.\n"); 588 return (ENXIO); 589 } 590 591 temp = XREAD4(sc, capa, XHCI_HCSPARAMS0); 592 593 DPRINTF("HCS0 = 0x%08x\n", temp); 594 595 /* set up context size */ 596 if (XHCI_HCS0_CSZ(temp)) { 597 sc->sc_ctx_is_64_byte = 1; 598 } else { 599 sc->sc_ctx_is_64_byte = 0; 600 } 601 602 /* get DMA bits */ 603 sc->sc_bus.dma_bits = (XHCI_HCS0_AC64(temp) && 604 xhcidma32 == 0 && dma32 == 0) ? 64 : 32; 605 606 device_printf(self, "%d bytes context size, %d-bit DMA\n", 607 sc->sc_ctx_is_64_byte ? 64 : 32, (int)sc->sc_bus.dma_bits); 608 609 /* enable 64Kbyte control endpoint quirk */ 610 sc->sc_bus.control_ep_quirk = (xhcictlquirk ? 1 : 0); 611 612 temp = XREAD4(sc, capa, XHCI_HCSPARAMS1); 613 614 /* get number of device slots */ 615 sc->sc_noport = XHCI_HCS1_N_PORTS(temp); 616 617 if (sc->sc_noport == 0) { 618 device_printf(sc->sc_bus.parent, "Invalid number " 619 "of ports: %u\n", sc->sc_noport); 620 return (ENXIO); 621 } 622 623 sc->sc_noport = sc->sc_noport; 624 sc->sc_noslot = XHCI_HCS1_DEVSLOT_MAX(temp); 625 626 DPRINTF("Max slots: %u\n", sc->sc_noslot); 627 628 if (sc->sc_noslot > XHCI_MAX_DEVICES) 629 sc->sc_noslot = XHCI_MAX_DEVICES; 630 631 temp = XREAD4(sc, capa, XHCI_HCSPARAMS2); 632 633 DPRINTF("HCS2=0x%08x\n", temp); 634 635 /* get number of scratchpads */ 636 sc->sc_noscratch = XHCI_HCS2_SPB_MAX(temp); 637 638 if (sc->sc_noscratch > XHCI_MAX_SCRATCHPADS) { 639 device_printf(sc->sc_bus.parent, "XHCI request " 640 "too many scratchpads\n"); 641 return (ENOMEM); 642 } 643 644 DPRINTF("Max scratch: %u\n", sc->sc_noscratch); 645 646 /* get event table size */ 647 sc->sc_erst_max = 1U << XHCI_HCS2_ERST_MAX(temp); 648 if (sc->sc_erst_max > XHCI_MAX_RSEG) 649 sc->sc_erst_max = XHCI_MAX_RSEG; 650 651 temp = XREAD4(sc, capa, XHCI_HCSPARAMS3); 652 653 /* get maximum exit latency */ 654 sc->sc_exit_lat_max = XHCI_HCS3_U1_DEL(temp) + 655 XHCI_HCS3_U2_DEL(temp) + 250 /* us */; 656 657 /* Check if we should use the default IMOD value. */ 658 if (sc->sc_imod_default == 0) 659 sc->sc_imod_default = XHCI_IMOD_DEFAULT; 660 661 /* get all DMA memory */ 662 if (usb_bus_mem_alloc_all(&sc->sc_bus, 663 USB_GET_DMA_TAG(self), &xhci_iterate_hw_softc)) { 664 return (ENOMEM); 665 } 666 667 /* set up command queue mutex and condition varible */ 668 cv_init(&sc->sc_cmd_cv, "CMDQ"); 669 sx_init(&sc->sc_cmd_sx, "CMDQ lock"); 670 671 sc->sc_config_msg[0].hdr.pm_callback = &xhci_configure_msg; 672 sc->sc_config_msg[0].bus = &sc->sc_bus; 673 sc->sc_config_msg[1].hdr.pm_callback = &xhci_configure_msg; 674 sc->sc_config_msg[1].bus = &sc->sc_bus; 675 676 return (0); 677 } 678 679 void 680 xhci_uninit(struct xhci_softc *sc) 681 { 682 /* 683 * NOTE: At this point the control transfer process is gone 684 * and "xhci_configure_msg" is no longer called. Consequently 685 * waiting for the configuration messages to complete is not 686 * needed. 687 */ 688 usb_bus_mem_free_all(&sc->sc_bus, &xhci_iterate_hw_softc); 689 690 cv_destroy(&sc->sc_cmd_cv); 691 sx_destroy(&sc->sc_cmd_sx); 692 } 693 694 static void 695 xhci_set_hw_power_sleep(struct usb_bus *bus, uint32_t state) 696 { 697 struct xhci_softc *sc = XHCI_BUS2SC(bus); 698 699 switch (state) { 700 case USB_HW_POWER_SUSPEND: 701 DPRINTF("Stopping the XHCI\n"); 702 xhci_halt_controller(sc); 703 xhci_reset_controller(sc); 704 break; 705 case USB_HW_POWER_SHUTDOWN: 706 DPRINTF("Stopping the XHCI\n"); 707 xhci_halt_controller(sc); 708 xhci_reset_controller(sc); 709 break; 710 case USB_HW_POWER_RESUME: 711 DPRINTF("Starting the XHCI\n"); 712 xhci_start_controller(sc); 713 break; 714 default: 715 break; 716 } 717 } 718 719 static usb_error_t 720 xhci_generic_done_sub(struct usb_xfer *xfer) 721 { 722 struct xhci_td *td; 723 struct xhci_td *td_alt_next; 724 uint32_t len; 725 uint8_t status; 726 727 td = xfer->td_transfer_cache; 728 td_alt_next = td->alt_next; 729 730 if (xfer->aframes != xfer->nframes) 731 usbd_xfer_set_frame_len(xfer, xfer->aframes, 0); 732 733 while (1) { 734 usb_pc_cpu_invalidate(td->page_cache); 735 736 status = td->status; 737 len = td->remainder; 738 739 DPRINTFN(4, "xfer=%p[%u/%u] rem=%u/%u status=%u\n", 740 xfer, (unsigned int)xfer->aframes, 741 (unsigned int)xfer->nframes, 742 (unsigned int)len, (unsigned int)td->len, 743 (unsigned int)status); 744 745 /* 746 * Verify the status length and 747 * add the length to "frlengths[]": 748 */ 749 if (len > td->len) { 750 /* should not happen */ 751 DPRINTF("Invalid status length, " 752 "0x%04x/0x%04x bytes\n", len, td->len); 753 status = XHCI_TRB_ERROR_LENGTH; 754 } else if (xfer->aframes != xfer->nframes) { 755 xfer->frlengths[xfer->aframes] += td->len - len; 756 } 757 /* Check for last transfer */ 758 if (((void *)td) == xfer->td_transfer_last) { 759 td = NULL; 760 break; 761 } 762 /* Check for transfer error */ 763 if (status != XHCI_TRB_ERROR_SHORT_PKT && 764 status != XHCI_TRB_ERROR_SUCCESS) { 765 /* the transfer is finished */ 766 td = NULL; 767 break; 768 } 769 /* Check for short transfer */ 770 if (len > 0) { 771 if (xfer->flags_int.short_frames_ok || 772 xfer->flags_int.isochronous_xfr || 773 xfer->flags_int.control_xfr) { 774 /* follow alt next */ 775 td = td->alt_next; 776 } else { 777 /* the transfer is finished */ 778 td = NULL; 779 } 780 break; 781 } 782 td = td->obj_next; 783 784 if (td->alt_next != td_alt_next) { 785 /* this USB frame is complete */ 786 break; 787 } 788 } 789 790 /* update transfer cache */ 791 792 xfer->td_transfer_cache = td; 793 794 return ((status == XHCI_TRB_ERROR_STALL) ? USB_ERR_STALLED : 795 (status != XHCI_TRB_ERROR_SHORT_PKT && 796 status != XHCI_TRB_ERROR_SUCCESS) ? USB_ERR_IOERROR : 797 USB_ERR_NORMAL_COMPLETION); 798 } 799 800 static void 801 xhci_generic_done(struct usb_xfer *xfer) 802 { 803 usb_error_t err = 0; 804 805 DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n", 806 xfer, xfer->endpoint); 807 808 /* reset scanner */ 809 810 xfer->td_transfer_cache = xfer->td_transfer_first; 811 812 if (xfer->flags_int.control_xfr) { 813 if (xfer->flags_int.control_hdr) 814 err = xhci_generic_done_sub(xfer); 815 816 xfer->aframes = 1; 817 818 if (xfer->td_transfer_cache == NULL) 819 goto done; 820 } 821 822 while (xfer->aframes != xfer->nframes) { 823 err = xhci_generic_done_sub(xfer); 824 xfer->aframes++; 825 826 if (xfer->td_transfer_cache == NULL) 827 goto done; 828 } 829 830 if (xfer->flags_int.control_xfr && 831 !xfer->flags_int.control_act) 832 err = xhci_generic_done_sub(xfer); 833 done: 834 /* transfer is complete */ 835 xhci_device_done(xfer, err); 836 } 837 838 static void 839 xhci_activate_transfer(struct usb_xfer *xfer) 840 { 841 struct xhci_td *td; 842 843 td = xfer->td_transfer_cache; 844 845 usb_pc_cpu_invalidate(td->page_cache); 846 847 if (!(td->td_trb[0].dwTrb3 & htole32(XHCI_TRB_3_CYCLE_BIT))) { 848 /* activate the transfer */ 849 850 td->td_trb[0].dwTrb3 |= htole32(XHCI_TRB_3_CYCLE_BIT); 851 usb_pc_cpu_flush(td->page_cache); 852 853 xhci_endpoint_doorbell(xfer); 854 } 855 } 856 857 static void 858 xhci_skip_transfer(struct usb_xfer *xfer) 859 { 860 struct xhci_td *td; 861 struct xhci_td *td_last; 862 863 td = xfer->td_transfer_cache; 864 td_last = xfer->td_transfer_last; 865 866 td = td->alt_next; 867 868 usb_pc_cpu_invalidate(td->page_cache); 869 870 if (!(td->td_trb[0].dwTrb3 & htole32(XHCI_TRB_3_CYCLE_BIT))) { 871 usb_pc_cpu_invalidate(td_last->page_cache); 872 873 /* copy LINK TRB to current waiting location */ 874 875 td->td_trb[0].qwTrb0 = td_last->td_trb[td_last->ntrb].qwTrb0; 876 td->td_trb[0].dwTrb2 = td_last->td_trb[td_last->ntrb].dwTrb2; 877 usb_pc_cpu_flush(td->page_cache); 878 879 td->td_trb[0].dwTrb3 = td_last->td_trb[td_last->ntrb].dwTrb3; 880 usb_pc_cpu_flush(td->page_cache); 881 882 xhci_endpoint_doorbell(xfer); 883 } 884 } 885 886 /*------------------------------------------------------------------------* 887 * xhci_check_transfer 888 *------------------------------------------------------------------------*/ 889 static void 890 xhci_check_transfer(struct xhci_softc *sc, struct xhci_trb *trb) 891 { 892 struct xhci_endpoint_ext *pepext; 893 int64_t offset; 894 uint64_t td_event; 895 uint32_t temp; 896 uint32_t remainder; 897 uint16_t stream_id = 0; 898 uint16_t i; 899 uint8_t status; 900 uint8_t halted; 901 uint8_t epno; 902 uint8_t index; 903 904 /* decode TRB */ 905 td_event = le64toh(trb->qwTrb0); 906 temp = le32toh(trb->dwTrb2); 907 908 remainder = XHCI_TRB_2_REM_GET(temp); 909 status = XHCI_TRB_2_ERROR_GET(temp); 910 911 temp = le32toh(trb->dwTrb3); 912 epno = XHCI_TRB_3_EP_GET(temp); 913 index = XHCI_TRB_3_SLOT_GET(temp); 914 915 /* check if error means halted */ 916 halted = (status != XHCI_TRB_ERROR_SHORT_PKT && 917 status != XHCI_TRB_ERROR_SUCCESS); 918 919 DPRINTF("slot=%u epno=%u remainder=%u status=%u\n", 920 index, epno, remainder, status); 921 922 if (index > sc->sc_noslot) { 923 DPRINTF("Invalid slot.\n"); 924 return; 925 } 926 927 if ((epno == 0) || (epno >= XHCI_MAX_ENDPOINTS)) { 928 DPRINTF("Invalid endpoint.\n"); 929 return; 930 } 931 932 pepext = &sc->sc_hw.devs[index].endp[epno]; 933 934 /* try to find the USB transfer that generated the event */ 935 for (i = 0;; i++) { 936 struct usb_xfer *xfer; 937 struct xhci_td *td; 938 939 if (i == (XHCI_MAX_TRANSFERS - 1)) { 940 if (pepext->trb_ep_mode != USB_EP_MODE_STREAMS || 941 stream_id == (XHCI_MAX_STREAMS - 1)) 942 break; 943 stream_id++; 944 i = 0; 945 DPRINTFN(5, "stream_id=%u\n", stream_id); 946 } 947 948 xfer = pepext->xfer[i + (XHCI_MAX_TRANSFERS * stream_id)]; 949 if (xfer == NULL) 950 continue; 951 952 td = xfer->td_transfer_cache; 953 954 DPRINTFN(5, "Checking if 0x%016llx == (0x%016llx .. 0x%016llx)\n", 955 (long long)td_event, 956 (long long)td->td_self, 957 (long long)td->td_self + sizeof(td->td_trb)); 958 959 /* 960 * NOTE: Some XHCI implementations might not trigger 961 * an event on the last LINK TRB so we need to 962 * consider both the last and second last event 963 * address as conditions for a successful transfer. 964 * 965 * NOTE: We assume that the XHCI will only trigger one 966 * event per chain of TRBs. 967 */ 968 969 offset = td_event - td->td_self; 970 971 if (offset >= 0 && 972 offset < (int64_t)sizeof(td->td_trb)) { 973 usb_pc_cpu_invalidate(td->page_cache); 974 975 /* compute rest of remainder, if any */ 976 for (i = (offset / 16) + 1; i < td->ntrb; i++) { 977 temp = le32toh(td->td_trb[i].dwTrb2); 978 remainder += XHCI_TRB_2_BYTES_GET(temp); 979 } 980 981 DPRINTFN(5, "New remainder: %u\n", remainder); 982 983 /* clear isochronous transfer errors */ 984 if (xfer->flags_int.isochronous_xfr) { 985 if (halted) { 986 halted = 0; 987 status = XHCI_TRB_ERROR_SUCCESS; 988 remainder = td->len; 989 } 990 } 991 992 /* "td->remainder" is verified later */ 993 td->remainder = remainder; 994 td->status = status; 995 996 usb_pc_cpu_flush(td->page_cache); 997 998 /* 999 * 1) Last transfer descriptor makes the 1000 * transfer done 1001 */ 1002 if (((void *)td) == xfer->td_transfer_last) { 1003 DPRINTF("TD is last\n"); 1004 xhci_generic_done(xfer); 1005 break; 1006 } 1007 1008 /* 1009 * 2) Any kind of error makes the transfer 1010 * done 1011 */ 1012 if (halted) { 1013 DPRINTF("TD has I/O error\n"); 1014 xhci_generic_done(xfer); 1015 break; 1016 } 1017 1018 /* 1019 * 3) If there is no alternate next transfer, 1020 * a short packet also makes the transfer done 1021 */ 1022 if (td->remainder > 0) { 1023 if (td->alt_next == NULL) { 1024 DPRINTF( 1025 "short TD has no alternate next\n"); 1026 xhci_generic_done(xfer); 1027 break; 1028 } 1029 DPRINTF("TD has short pkt\n"); 1030 if (xfer->flags_int.short_frames_ok || 1031 xfer->flags_int.isochronous_xfr || 1032 xfer->flags_int.control_xfr) { 1033 /* follow the alt next */ 1034 xfer->td_transfer_cache = td->alt_next; 1035 xhci_activate_transfer(xfer); 1036 break; 1037 } 1038 xhci_skip_transfer(xfer); 1039 xhci_generic_done(xfer); 1040 break; 1041 } 1042 1043 /* 1044 * 4) Transfer complete - go to next TD 1045 */ 1046 DPRINTF("Following next TD\n"); 1047 xfer->td_transfer_cache = td->obj_next; 1048 xhci_activate_transfer(xfer); 1049 break; /* there should only be one match */ 1050 } 1051 } 1052 } 1053 1054 static int 1055 xhci_check_command(struct xhci_softc *sc, struct xhci_trb *trb) 1056 { 1057 if (sc->sc_cmd_addr == trb->qwTrb0) { 1058 DPRINTF("Received command event\n"); 1059 sc->sc_cmd_result[0] = trb->dwTrb2; 1060 sc->sc_cmd_result[1] = trb->dwTrb3; 1061 cv_signal(&sc->sc_cmd_cv); 1062 return (1); /* command match */ 1063 } 1064 return (0); 1065 } 1066 1067 static int 1068 xhci_interrupt_poll(struct xhci_softc *sc) 1069 { 1070 struct usb_page_search buf_res; 1071 struct xhci_hw_root *phwr; 1072 uint64_t addr; 1073 uint32_t temp; 1074 int retval = 0; 1075 uint16_t i; 1076 uint8_t event; 1077 uint8_t j; 1078 uint8_t k; 1079 uint8_t t; 1080 1081 usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res); 1082 1083 phwr = buf_res.buffer; 1084 1085 /* Receive any events */ 1086 1087 usb_pc_cpu_invalidate(&sc->sc_hw.root_pc); 1088 1089 i = sc->sc_event_idx; 1090 j = sc->sc_event_ccs; 1091 t = 2; 1092 1093 while (1) { 1094 temp = le32toh(phwr->hwr_events[i].dwTrb3); 1095 1096 k = (temp & XHCI_TRB_3_CYCLE_BIT) ? 1 : 0; 1097 1098 if (j != k) 1099 break; 1100 1101 event = XHCI_TRB_3_TYPE_GET(temp); 1102 1103 DPRINTFN(10, "event[%u] = %u (0x%016llx 0x%08lx 0x%08lx)\n", 1104 i, event, (long long)le64toh(phwr->hwr_events[i].qwTrb0), 1105 (long)le32toh(phwr->hwr_events[i].dwTrb2), 1106 (long)le32toh(phwr->hwr_events[i].dwTrb3)); 1107 1108 switch (event) { 1109 case XHCI_TRB_EVENT_TRANSFER: 1110 xhci_check_transfer(sc, &phwr->hwr_events[i]); 1111 break; 1112 case XHCI_TRB_EVENT_CMD_COMPLETE: 1113 retval |= xhci_check_command(sc, &phwr->hwr_events[i]); 1114 break; 1115 default: 1116 DPRINTF("Unhandled event = %u\n", event); 1117 break; 1118 } 1119 1120 i++; 1121 1122 if (i == XHCI_MAX_EVENTS) { 1123 i = 0; 1124 j ^= 1; 1125 1126 /* check for timeout */ 1127 if (!--t) 1128 break; 1129 } 1130 } 1131 1132 sc->sc_event_idx = i; 1133 sc->sc_event_ccs = j; 1134 1135 /* 1136 * NOTE: The Event Ring Dequeue Pointer Register is 64-bit 1137 * latched. That means to activate the register we need to 1138 * write both the low and high double word of the 64-bit 1139 * register. 1140 */ 1141 1142 addr = buf_res.physaddr; 1143 addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_events[i]; 1144 1145 /* try to clear busy bit */ 1146 addr |= XHCI_ERDP_LO_BUSY; 1147 1148 XWRITE4(sc, runt, XHCI_ERDP_LO(0), (uint32_t)addr); 1149 XWRITE4(sc, runt, XHCI_ERDP_HI(0), (uint32_t)(addr >> 32)); 1150 1151 return (retval); 1152 } 1153 1154 static usb_error_t 1155 xhci_do_command(struct xhci_softc *sc, struct xhci_trb *trb, 1156 uint16_t timeout_ms) 1157 { 1158 struct usb_page_search buf_res; 1159 struct xhci_hw_root *phwr; 1160 uint64_t addr; 1161 uint32_t temp; 1162 uint8_t i; 1163 uint8_t j; 1164 uint8_t timeout = 0; 1165 int err; 1166 1167 XHCI_CMD_ASSERT_LOCKED(sc); 1168 1169 /* get hardware root structure */ 1170 1171 usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res); 1172 1173 phwr = buf_res.buffer; 1174 1175 /* Queue command */ 1176 1177 USB_BUS_LOCK(&sc->sc_bus); 1178 retry: 1179 i = sc->sc_command_idx; 1180 j = sc->sc_command_ccs; 1181 1182 DPRINTFN(10, "command[%u] = %u (0x%016llx, 0x%08lx, 0x%08lx)\n", 1183 i, XHCI_TRB_3_TYPE_GET(le32toh(trb->dwTrb3)), 1184 (long long)le64toh(trb->qwTrb0), 1185 (long)le32toh(trb->dwTrb2), 1186 (long)le32toh(trb->dwTrb3)); 1187 1188 phwr->hwr_commands[i].qwTrb0 = trb->qwTrb0; 1189 phwr->hwr_commands[i].dwTrb2 = trb->dwTrb2; 1190 1191 usb_pc_cpu_flush(&sc->sc_hw.root_pc); 1192 1193 temp = trb->dwTrb3; 1194 1195 if (j) 1196 temp |= htole32(XHCI_TRB_3_CYCLE_BIT); 1197 else 1198 temp &= ~htole32(XHCI_TRB_3_CYCLE_BIT); 1199 1200 temp &= ~htole32(XHCI_TRB_3_TC_BIT); 1201 1202 phwr->hwr_commands[i].dwTrb3 = temp; 1203 1204 usb_pc_cpu_flush(&sc->sc_hw.root_pc); 1205 1206 addr = buf_res.physaddr; 1207 addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_commands[i]; 1208 1209 sc->sc_cmd_addr = htole64(addr); 1210 1211 i++; 1212 1213 if (i == (XHCI_MAX_COMMANDS - 1)) { 1214 if (j) { 1215 temp = htole32(XHCI_TRB_3_TC_BIT | 1216 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) | 1217 XHCI_TRB_3_CYCLE_BIT); 1218 } else { 1219 temp = htole32(XHCI_TRB_3_TC_BIT | 1220 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK)); 1221 } 1222 1223 phwr->hwr_commands[i].dwTrb3 = temp; 1224 1225 usb_pc_cpu_flush(&sc->sc_hw.root_pc); 1226 1227 i = 0; 1228 j ^= 1; 1229 } 1230 1231 sc->sc_command_idx = i; 1232 sc->sc_command_ccs = j; 1233 1234 XWRITE4(sc, door, XHCI_DOORBELL(0), 0); 1235 1236 err = cv_timedwait(&sc->sc_cmd_cv, &sc->sc_bus.bus_mtx, 1237 USB_MS_TO_TICKS(timeout_ms)); 1238 1239 /* 1240 * In some error cases event interrupts are not generated. 1241 * Poll one time to see if the command has completed. 1242 */ 1243 if (err != 0 && xhci_interrupt_poll(sc) != 0) { 1244 DPRINTF("Command was completed when polling\n"); 1245 err = 0; 1246 } 1247 if (err != 0) { 1248 DPRINTF("Command timeout!\n"); 1249 /* 1250 * After some weeks of continuous operation, it has 1251 * been observed that the ASMedia Technology, ASM1042 1252 * SuperSpeed USB Host Controller can suddenly stop 1253 * accepting commands via the command queue. Try to 1254 * first reset the command queue. If that fails do a 1255 * host controller reset. 1256 */ 1257 if (timeout == 0 && 1258 xhci_reset_command_queue_locked(sc) == 0) { 1259 temp = le32toh(trb->dwTrb3); 1260 1261 /* 1262 * Avoid infinite XHCI reset loops if the set 1263 * address command fails to respond due to a 1264 * non-enumerating device: 1265 */ 1266 if (XHCI_TRB_3_TYPE_GET(temp) == XHCI_TRB_TYPE_ADDRESS_DEVICE && 1267 (temp & XHCI_TRB_3_BSR_BIT) == 0) { 1268 DPRINTF("Set address timeout\n"); 1269 } else { 1270 timeout = 1; 1271 goto retry; 1272 } 1273 } else { 1274 DPRINTF("Controller reset!\n"); 1275 usb_bus_reset_async_locked(&sc->sc_bus); 1276 } 1277 err = USB_ERR_TIMEOUT; 1278 trb->dwTrb2 = 0; 1279 trb->dwTrb3 = 0; 1280 } else { 1281 temp = le32toh(sc->sc_cmd_result[0]); 1282 if (XHCI_TRB_2_ERROR_GET(temp) != XHCI_TRB_ERROR_SUCCESS) 1283 err = USB_ERR_IOERROR; 1284 1285 trb->dwTrb2 = sc->sc_cmd_result[0]; 1286 trb->dwTrb3 = sc->sc_cmd_result[1]; 1287 } 1288 1289 USB_BUS_UNLOCK(&sc->sc_bus); 1290 1291 return (err); 1292 } 1293 1294 #if 0 1295 static usb_error_t 1296 xhci_cmd_nop(struct xhci_softc *sc) 1297 { 1298 struct xhci_trb trb; 1299 uint32_t temp; 1300 1301 DPRINTF("\n"); 1302 1303 trb.qwTrb0 = 0; 1304 trb.dwTrb2 = 0; 1305 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NOOP); 1306 1307 trb.dwTrb3 = htole32(temp); 1308 1309 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1310 } 1311 #endif 1312 1313 static usb_error_t 1314 xhci_cmd_enable_slot(struct xhci_softc *sc, uint8_t *pslot) 1315 { 1316 struct xhci_trb trb; 1317 uint32_t temp; 1318 usb_error_t err; 1319 1320 DPRINTF("\n"); 1321 1322 trb.qwTrb0 = 0; 1323 trb.dwTrb2 = 0; 1324 trb.dwTrb3 = htole32(XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ENABLE_SLOT)); 1325 1326 err = xhci_do_command(sc, &trb, 100 /* ms */); 1327 if (err) 1328 goto done; 1329 1330 temp = le32toh(trb.dwTrb3); 1331 1332 *pslot = XHCI_TRB_3_SLOT_GET(temp); 1333 1334 done: 1335 return (err); 1336 } 1337 1338 static usb_error_t 1339 xhci_cmd_disable_slot(struct xhci_softc *sc, uint8_t slot_id) 1340 { 1341 struct xhci_trb trb; 1342 uint32_t temp; 1343 1344 DPRINTF("\n"); 1345 1346 trb.qwTrb0 = 0; 1347 trb.dwTrb2 = 0; 1348 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DISABLE_SLOT) | 1349 XHCI_TRB_3_SLOT_SET(slot_id); 1350 1351 trb.dwTrb3 = htole32(temp); 1352 1353 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1354 } 1355 1356 static usb_error_t 1357 xhci_cmd_set_address(struct xhci_softc *sc, uint64_t input_ctx, 1358 uint8_t bsr, uint8_t slot_id) 1359 { 1360 struct xhci_trb trb; 1361 uint32_t temp; 1362 1363 DPRINTF("\n"); 1364 1365 trb.qwTrb0 = htole64(input_ctx); 1366 trb.dwTrb2 = 0; 1367 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ADDRESS_DEVICE) | 1368 XHCI_TRB_3_SLOT_SET(slot_id); 1369 1370 if (bsr) 1371 temp |= XHCI_TRB_3_BSR_BIT; 1372 1373 trb.dwTrb3 = htole32(temp); 1374 1375 return (xhci_do_command(sc, &trb, 500 /* ms */)); 1376 } 1377 1378 static usb_error_t 1379 xhci_set_address(struct usb_device *udev, struct mtx *mtx, uint16_t address) 1380 { 1381 struct usb_page_search buf_inp; 1382 struct usb_page_search buf_dev; 1383 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 1384 struct xhci_hw_dev *hdev; 1385 struct xhci_dev_ctx *pdev; 1386 struct xhci_endpoint_ext *pepext; 1387 uint32_t temp; 1388 uint16_t mps; 1389 usb_error_t err; 1390 uint8_t index; 1391 1392 /* the root HUB case is not handled here */ 1393 if (udev->parent_hub == NULL) 1394 return (USB_ERR_INVAL); 1395 1396 index = udev->controller_slot_id; 1397 1398 hdev = &sc->sc_hw.devs[index]; 1399 1400 if (mtx != NULL) 1401 mtx_unlock(mtx); 1402 1403 XHCI_CMD_LOCK(sc); 1404 1405 switch (hdev->state) { 1406 case XHCI_ST_DEFAULT: 1407 case XHCI_ST_ENABLED: 1408 1409 hdev->state = XHCI_ST_ENABLED; 1410 1411 /* set configure mask to slot and EP0 */ 1412 xhci_configure_mask(udev, 3, 0); 1413 1414 /* configure input slot context structure */ 1415 err = xhci_configure_device(udev); 1416 1417 if (err != 0) { 1418 DPRINTF("Could not configure device\n"); 1419 break; 1420 } 1421 1422 /* configure input endpoint context structure */ 1423 switch (udev->speed) { 1424 case USB_SPEED_LOW: 1425 case USB_SPEED_FULL: 1426 mps = 8; 1427 break; 1428 case USB_SPEED_HIGH: 1429 mps = 64; 1430 break; 1431 default: 1432 mps = 512; 1433 break; 1434 } 1435 1436 pepext = xhci_get_endpoint_ext(udev, 1437 &udev->ctrl_ep_desc); 1438 1439 /* ensure the control endpoint is setup again */ 1440 USB_BUS_LOCK(udev->bus); 1441 pepext->trb_halted = 1; 1442 pepext->trb_running = 0; 1443 USB_BUS_UNLOCK(udev->bus); 1444 1445 err = xhci_configure_endpoint(udev, 1446 &udev->ctrl_ep_desc, pepext, 1447 0, 1, 1, 0, mps, mps, USB_EP_MODE_DEFAULT); 1448 1449 if (err != 0) { 1450 DPRINTF("Could not configure default endpoint\n"); 1451 break; 1452 } 1453 1454 /* execute set address command */ 1455 usbd_get_page(&hdev->input_pc, 0, &buf_inp); 1456 1457 err = xhci_cmd_set_address(sc, buf_inp.physaddr, 1458 (address == 0), index); 1459 1460 if (err != 0) { 1461 temp = le32toh(sc->sc_cmd_result[0]); 1462 if (address == 0 && sc->sc_port_route != NULL && 1463 XHCI_TRB_2_ERROR_GET(temp) == 1464 XHCI_TRB_ERROR_PARAMETER) { 1465 /* LynxPoint XHCI - ports are not switchable */ 1466 /* Un-route all ports from the XHCI */ 1467 sc->sc_port_route(sc->sc_bus.parent, 0, ~0); 1468 } 1469 DPRINTF("Could not set address " 1470 "for slot %u.\n", index); 1471 if (address != 0) 1472 break; 1473 } 1474 1475 /* update device address to new value */ 1476 1477 usbd_get_page(&hdev->device_pc, 0, &buf_dev); 1478 pdev = buf_dev.buffer; 1479 usb_pc_cpu_invalidate(&hdev->device_pc); 1480 1481 temp = xhci_ctx_get_le32(sc, &pdev->ctx_slot.dwSctx3); 1482 udev->address = XHCI_SCTX_3_DEV_ADDR_GET(temp); 1483 1484 /* update device state to new value */ 1485 1486 if (address != 0) 1487 hdev->state = XHCI_ST_ADDRESSED; 1488 else 1489 hdev->state = XHCI_ST_DEFAULT; 1490 break; 1491 1492 default: 1493 DPRINTF("Wrong state for set address.\n"); 1494 err = USB_ERR_IOERROR; 1495 break; 1496 } 1497 XHCI_CMD_UNLOCK(sc); 1498 1499 if (mtx != NULL) 1500 mtx_lock(mtx); 1501 1502 return (err); 1503 } 1504 1505 static usb_error_t 1506 xhci_cmd_configure_ep(struct xhci_softc *sc, uint64_t input_ctx, 1507 uint8_t deconfigure, uint8_t slot_id) 1508 { 1509 struct xhci_trb trb; 1510 uint32_t temp; 1511 1512 DPRINTF("\n"); 1513 1514 trb.qwTrb0 = htole64(input_ctx); 1515 trb.dwTrb2 = 0; 1516 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP) | 1517 XHCI_TRB_3_SLOT_SET(slot_id); 1518 1519 if (deconfigure) 1520 temp |= XHCI_TRB_3_DCEP_BIT; 1521 1522 trb.dwTrb3 = htole32(temp); 1523 1524 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1525 } 1526 1527 static usb_error_t 1528 xhci_cmd_evaluate_ctx(struct xhci_softc *sc, uint64_t input_ctx, 1529 uint8_t slot_id) 1530 { 1531 struct xhci_trb trb; 1532 uint32_t temp; 1533 1534 DPRINTF("\n"); 1535 1536 trb.qwTrb0 = htole64(input_ctx); 1537 trb.dwTrb2 = 0; 1538 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVALUATE_CTX) | 1539 XHCI_TRB_3_SLOT_SET(slot_id); 1540 trb.dwTrb3 = htole32(temp); 1541 1542 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1543 } 1544 1545 static usb_error_t 1546 xhci_cmd_reset_ep(struct xhci_softc *sc, uint8_t preserve, 1547 uint8_t ep_id, uint8_t slot_id) 1548 { 1549 struct xhci_trb trb; 1550 uint32_t temp; 1551 1552 DPRINTF("\n"); 1553 1554 trb.qwTrb0 = 0; 1555 trb.dwTrb2 = 0; 1556 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_EP) | 1557 XHCI_TRB_3_SLOT_SET(slot_id) | 1558 XHCI_TRB_3_EP_SET(ep_id); 1559 1560 if (preserve) 1561 temp |= XHCI_TRB_3_PRSV_BIT; 1562 1563 trb.dwTrb3 = htole32(temp); 1564 1565 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1566 } 1567 1568 static usb_error_t 1569 xhci_cmd_set_tr_dequeue_ptr(struct xhci_softc *sc, uint64_t dequeue_ptr, 1570 uint16_t stream_id, uint8_t ep_id, uint8_t slot_id) 1571 { 1572 struct xhci_trb trb; 1573 uint32_t temp; 1574 1575 DPRINTF("\n"); 1576 1577 trb.qwTrb0 = htole64(dequeue_ptr); 1578 1579 temp = XHCI_TRB_2_STREAM_SET(stream_id); 1580 trb.dwTrb2 = htole32(temp); 1581 1582 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SET_TR_DEQUEUE) | 1583 XHCI_TRB_3_SLOT_SET(slot_id) | 1584 XHCI_TRB_3_EP_SET(ep_id); 1585 trb.dwTrb3 = htole32(temp); 1586 1587 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1588 } 1589 1590 static usb_error_t 1591 xhci_cmd_stop_ep(struct xhci_softc *sc, uint8_t suspend, 1592 uint8_t ep_id, uint8_t slot_id) 1593 { 1594 struct xhci_trb trb; 1595 uint32_t temp; 1596 1597 DPRINTF("\n"); 1598 1599 trb.qwTrb0 = 0; 1600 trb.dwTrb2 = 0; 1601 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STOP_EP) | 1602 XHCI_TRB_3_SLOT_SET(slot_id) | 1603 XHCI_TRB_3_EP_SET(ep_id); 1604 1605 if (suspend) 1606 temp |= XHCI_TRB_3_SUSP_EP_BIT; 1607 1608 trb.dwTrb3 = htole32(temp); 1609 1610 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1611 } 1612 1613 static usb_error_t 1614 xhci_cmd_reset_dev(struct xhci_softc *sc, uint8_t slot_id) 1615 { 1616 struct xhci_trb trb; 1617 uint32_t temp; 1618 1619 DPRINTF("\n"); 1620 1621 trb.qwTrb0 = 0; 1622 trb.dwTrb2 = 0; 1623 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_DEVICE) | 1624 XHCI_TRB_3_SLOT_SET(slot_id); 1625 1626 trb.dwTrb3 = htole32(temp); 1627 1628 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1629 } 1630 1631 /*------------------------------------------------------------------------* 1632 * xhci_interrupt - XHCI interrupt handler 1633 *------------------------------------------------------------------------*/ 1634 void 1635 xhci_interrupt(struct xhci_softc *sc) 1636 { 1637 uint32_t status; 1638 uint32_t temp; 1639 1640 USB_BUS_LOCK(&sc->sc_bus); 1641 1642 status = XREAD4(sc, oper, XHCI_USBSTS); 1643 1644 /* acknowledge interrupts, if any */ 1645 if (status != 0) { 1646 XWRITE4(sc, oper, XHCI_USBSTS, status); 1647 DPRINTFN(16, "real interrupt (status=0x%08x)\n", status); 1648 } 1649 1650 temp = XREAD4(sc, runt, XHCI_IMAN(0)); 1651 1652 /* force clearing of pending interrupts */ 1653 if (temp & XHCI_IMAN_INTR_PEND) 1654 XWRITE4(sc, runt, XHCI_IMAN(0), temp); 1655 1656 /* check for event(s) */ 1657 xhci_interrupt_poll(sc); 1658 1659 if (status & (XHCI_STS_PCD | XHCI_STS_HCH | 1660 XHCI_STS_HSE | XHCI_STS_HCE)) { 1661 if (status & XHCI_STS_PCD) { 1662 xhci_root_intr(sc); 1663 } 1664 1665 if (status & XHCI_STS_HCH) { 1666 printf("%s: host controller halted\n", 1667 __FUNCTION__); 1668 } 1669 1670 if (status & XHCI_STS_HSE) { 1671 printf("%s: host system error\n", 1672 __FUNCTION__); 1673 } 1674 1675 if (status & XHCI_STS_HCE) { 1676 printf("%s: host controller error\n", 1677 __FUNCTION__); 1678 } 1679 } 1680 USB_BUS_UNLOCK(&sc->sc_bus); 1681 } 1682 1683 /*------------------------------------------------------------------------* 1684 * xhci_timeout - XHCI timeout handler 1685 *------------------------------------------------------------------------*/ 1686 static void 1687 xhci_timeout(void *arg) 1688 { 1689 struct usb_xfer *xfer = arg; 1690 1691 DPRINTF("xfer=%p\n", xfer); 1692 1693 USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED); 1694 1695 /* transfer is transferred */ 1696 xhci_device_done(xfer, USB_ERR_TIMEOUT); 1697 } 1698 1699 static void 1700 xhci_do_poll(struct usb_bus *bus) 1701 { 1702 struct xhci_softc *sc = XHCI_BUS2SC(bus); 1703 1704 USB_BUS_LOCK(&sc->sc_bus); 1705 xhci_interrupt_poll(sc); 1706 USB_BUS_UNLOCK(&sc->sc_bus); 1707 } 1708 1709 static void 1710 xhci_setup_generic_chain_sub(struct xhci_std_temp *temp) 1711 { 1712 struct usb_page_search buf_res; 1713 struct xhci_td *td; 1714 struct xhci_td *td_next; 1715 struct xhci_td *td_alt_next; 1716 struct xhci_td *td_first; 1717 uint32_t buf_offset; 1718 uint32_t average; 1719 uint32_t len_old; 1720 uint32_t npkt_off; 1721 uint32_t dword; 1722 uint8_t shortpkt_old; 1723 uint8_t precompute; 1724 uint8_t x; 1725 1726 td_alt_next = NULL; 1727 buf_offset = 0; 1728 shortpkt_old = temp->shortpkt; 1729 len_old = temp->len; 1730 npkt_off = 0; 1731 precompute = 1; 1732 1733 restart: 1734 1735 td = temp->td; 1736 td_next = td_first = temp->td_next; 1737 1738 while (1) { 1739 if (temp->len == 0) { 1740 if (temp->shortpkt) 1741 break; 1742 1743 /* send a Zero Length Packet, ZLP, last */ 1744 1745 temp->shortpkt = 1; 1746 average = 0; 1747 1748 } else { 1749 average = temp->average; 1750 1751 if (temp->len < average) { 1752 if (temp->len % temp->max_packet_size) { 1753 temp->shortpkt = 1; 1754 } 1755 average = temp->len; 1756 } 1757 } 1758 1759 if (td_next == NULL) 1760 panic("%s: out of XHCI transfer descriptors!", __FUNCTION__); 1761 1762 /* get next TD */ 1763 1764 td = td_next; 1765 td_next = td->obj_next; 1766 1767 /* check if we are pre-computing */ 1768 1769 if (precompute) { 1770 /* update remaining length */ 1771 1772 temp->len -= average; 1773 1774 continue; 1775 } 1776 /* fill out current TD */ 1777 1778 td->len = average; 1779 td->remainder = 0; 1780 td->status = 0; 1781 1782 /* update remaining length */ 1783 1784 temp->len -= average; 1785 1786 /* reset TRB index */ 1787 1788 x = 0; 1789 1790 if (temp->trb_type == XHCI_TRB_TYPE_SETUP_STAGE) { 1791 /* immediate data */ 1792 1793 if (average > 8) 1794 average = 8; 1795 1796 td->td_trb[0].qwTrb0 = 0; 1797 1798 usbd_copy_out(temp->pc, temp->offset + buf_offset, 1799 (uint8_t *)(uintptr_t)&td->td_trb[0].qwTrb0, 1800 average); 1801 1802 dword = XHCI_TRB_2_BYTES_SET(8) | 1803 XHCI_TRB_2_TDSZ_SET(0) | 1804 XHCI_TRB_2_IRQ_SET(0); 1805 1806 td->td_trb[0].dwTrb2 = htole32(dword); 1807 1808 dword = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SETUP_STAGE) | 1809 XHCI_TRB_3_IDT_BIT | XHCI_TRB_3_CYCLE_BIT; 1810 1811 /* check wLength */ 1812 if (td->td_trb[0].qwTrb0 & 1813 htole64(XHCI_TRB_0_WLENGTH_MASK)) { 1814 if (td->td_trb[0].qwTrb0 & 1815 htole64(XHCI_TRB_0_DIR_IN_MASK)) 1816 dword |= XHCI_TRB_3_TRT_IN; 1817 else 1818 dword |= XHCI_TRB_3_TRT_OUT; 1819 } 1820 1821 td->td_trb[0].dwTrb3 = htole32(dword); 1822 #ifdef USB_DEBUG 1823 xhci_dump_trb(&td->td_trb[x]); 1824 #endif 1825 x++; 1826 1827 } else do { 1828 uint32_t npkt; 1829 1830 /* fill out buffer pointers */ 1831 1832 if (average == 0) { 1833 memset(&buf_res, 0, sizeof(buf_res)); 1834 } else { 1835 usbd_get_page(temp->pc, temp->offset + 1836 buf_offset, &buf_res); 1837 1838 /* get length to end of page */ 1839 if (buf_res.length > average) 1840 buf_res.length = average; 1841 1842 /* check for maximum length */ 1843 if (buf_res.length > XHCI_TD_PAGE_SIZE) 1844 buf_res.length = XHCI_TD_PAGE_SIZE; 1845 1846 npkt_off += buf_res.length; 1847 } 1848 1849 /* set up npkt */ 1850 npkt = howmany(len_old - npkt_off, 1851 temp->max_packet_size); 1852 1853 if (npkt == 0) 1854 npkt = 1; 1855 else if (npkt > 31) 1856 npkt = 31; 1857 1858 /* fill out TRB's */ 1859 td->td_trb[x].qwTrb0 = 1860 htole64((uint64_t)buf_res.physaddr); 1861 1862 dword = 1863 XHCI_TRB_2_BYTES_SET(buf_res.length) | 1864 XHCI_TRB_2_TDSZ_SET(npkt) | 1865 XHCI_TRB_2_IRQ_SET(0); 1866 1867 td->td_trb[x].dwTrb2 = htole32(dword); 1868 1869 switch (temp->trb_type) { 1870 case XHCI_TRB_TYPE_ISOCH: 1871 dword = XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_CYCLE_BIT | 1872 XHCI_TRB_3_TBC_SET(temp->tbc) | 1873 XHCI_TRB_3_TLBPC_SET(temp->tlbpc); 1874 if (td != td_first) { 1875 dword |= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL); 1876 } else if (temp->do_isoc_sync != 0) { 1877 temp->do_isoc_sync = 0; 1878 /* wait until "isoc_frame" */ 1879 dword |= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ISOCH) | 1880 XHCI_TRB_3_FRID_SET(temp->isoc_frame / 8); 1881 } else { 1882 /* start data transfer at next interval */ 1883 dword |= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ISOCH) | 1884 XHCI_TRB_3_ISO_SIA_BIT; 1885 } 1886 if (temp->direction == UE_DIR_IN) 1887 dword |= XHCI_TRB_3_ISP_BIT; 1888 break; 1889 case XHCI_TRB_TYPE_DATA_STAGE: 1890 dword = XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_CYCLE_BIT | 1891 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DATA_STAGE); 1892 if (temp->direction == UE_DIR_IN) 1893 dword |= XHCI_TRB_3_DIR_IN | XHCI_TRB_3_ISP_BIT; 1894 /* 1895 * Section 3.2.9 in the XHCI 1896 * specification about control 1897 * transfers says that we should use a 1898 * normal-TRB if there are more TRBs 1899 * extending the data-stage 1900 * TRB. Update the "trb_type". 1901 */ 1902 temp->trb_type = XHCI_TRB_TYPE_NORMAL; 1903 break; 1904 case XHCI_TRB_TYPE_STATUS_STAGE: 1905 dword = XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_CYCLE_BIT | 1906 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STATUS_STAGE); 1907 if (temp->direction == UE_DIR_IN) 1908 dword |= XHCI_TRB_3_DIR_IN; 1909 break; 1910 default: /* XHCI_TRB_TYPE_NORMAL */ 1911 dword = XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_CYCLE_BIT | 1912 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL); 1913 if (temp->direction == UE_DIR_IN) 1914 dword |= XHCI_TRB_3_ISP_BIT; 1915 break; 1916 } 1917 td->td_trb[x].dwTrb3 = htole32(dword); 1918 1919 average -= buf_res.length; 1920 buf_offset += buf_res.length; 1921 #ifdef USB_DEBUG 1922 xhci_dump_trb(&td->td_trb[x]); 1923 #endif 1924 x++; 1925 1926 } while (average != 0); 1927 1928 td->td_trb[x-1].dwTrb3 |= htole32(XHCI_TRB_3_IOC_BIT); 1929 1930 /* store number of data TRB's */ 1931 1932 td->ntrb = x; 1933 1934 DPRINTF("NTRB=%u\n", x); 1935 1936 /* fill out link TRB */ 1937 1938 if (td_next != NULL) { 1939 /* link the current TD with the next one */ 1940 td->td_trb[x].qwTrb0 = htole64((uint64_t)td_next->td_self); 1941 DPRINTF("LINK=0x%08llx\n", (long long)td_next->td_self); 1942 } else { 1943 /* this field will get updated later */ 1944 DPRINTF("NOLINK\n"); 1945 } 1946 1947 dword = XHCI_TRB_2_IRQ_SET(0); 1948 1949 td->td_trb[x].dwTrb2 = htole32(dword); 1950 1951 dword = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) | 1952 XHCI_TRB_3_CYCLE_BIT | XHCI_TRB_3_IOC_BIT | 1953 /* 1954 * CHAIN-BIT: Ensure that a multi-TRB IN-endpoint 1955 * frame only receives a single short packet event 1956 * by setting the CHAIN bit in the LINK field. In 1957 * addition some XHCI controllers have problems 1958 * sending a ZLP unless the CHAIN-BIT is set in 1959 * the LINK TRB. 1960 */ 1961 XHCI_TRB_3_CHAIN_BIT; 1962 1963 td->td_trb[x].dwTrb3 = htole32(dword); 1964 1965 td->alt_next = td_alt_next; 1966 #ifdef USB_DEBUG 1967 xhci_dump_trb(&td->td_trb[x]); 1968 #endif 1969 usb_pc_cpu_flush(td->page_cache); 1970 } 1971 1972 if (precompute) { 1973 precompute = 0; 1974 1975 /* set up alt next pointer, if any */ 1976 if (temp->last_frame) { 1977 td_alt_next = NULL; 1978 } else { 1979 /* we use this field internally */ 1980 td_alt_next = td_next; 1981 } 1982 1983 /* restore */ 1984 temp->shortpkt = shortpkt_old; 1985 temp->len = len_old; 1986 goto restart; 1987 } 1988 1989 /* 1990 * Remove cycle bit from the first TRB if we are 1991 * stepping them: 1992 */ 1993 if (temp->step_td != 0) { 1994 td_first->td_trb[0].dwTrb3 &= ~htole32(XHCI_TRB_3_CYCLE_BIT); 1995 usb_pc_cpu_flush(td_first->page_cache); 1996 } 1997 1998 /* clear TD SIZE to zero, hence this is the last TRB */ 1999 /* remove chain bit because this is the last data TRB in the chain */ 2000 td->td_trb[td->ntrb - 1].dwTrb2 &= ~htole32(XHCI_TRB_2_TDSZ_SET(31)); 2001 td->td_trb[td->ntrb - 1].dwTrb3 &= ~htole32(XHCI_TRB_3_CHAIN_BIT); 2002 /* remove CHAIN-BIT from last LINK TRB */ 2003 td->td_trb[td->ntrb].dwTrb3 &= ~htole32(XHCI_TRB_3_CHAIN_BIT); 2004 2005 usb_pc_cpu_flush(td->page_cache); 2006 2007 temp->td = td; 2008 temp->td_next = td_next; 2009 } 2010 2011 static void 2012 xhci_setup_generic_chain(struct usb_xfer *xfer) 2013 { 2014 struct xhci_std_temp temp; 2015 struct xhci_td *td; 2016 uint32_t x; 2017 uint32_t y; 2018 uint8_t mult; 2019 2020 temp.do_isoc_sync = 0; 2021 temp.step_td = 0; 2022 temp.tbc = 0; 2023 temp.tlbpc = 0; 2024 temp.average = xfer->max_hc_frame_size; 2025 temp.max_packet_size = xfer->max_packet_size; 2026 temp.sc = XHCI_BUS2SC(xfer->xroot->bus); 2027 temp.pc = NULL; 2028 temp.last_frame = 0; 2029 temp.offset = 0; 2030 temp.multishort = xfer->flags_int.isochronous_xfr || 2031 xfer->flags_int.control_xfr || 2032 xfer->flags_int.short_frames_ok; 2033 2034 /* toggle the DMA set we are using */ 2035 xfer->flags_int.curr_dma_set ^= 1; 2036 2037 /* get next DMA set */ 2038 td = xfer->td_start[xfer->flags_int.curr_dma_set]; 2039 2040 temp.td = NULL; 2041 temp.td_next = td; 2042 2043 xfer->td_transfer_first = td; 2044 xfer->td_transfer_cache = td; 2045 2046 if (xfer->flags_int.isochronous_xfr) { 2047 uint8_t shift; 2048 2049 /* compute multiplier for ISOCHRONOUS transfers */ 2050 mult = xfer->endpoint->ecomp ? 2051 UE_GET_SS_ISO_MULT(xfer->endpoint->ecomp->bmAttributes) 2052 : 0; 2053 /* check for USB 2.0 multiplier */ 2054 if (mult == 0) { 2055 mult = (xfer->endpoint->edesc-> 2056 wMaxPacketSize[1] >> 3) & 3; 2057 } 2058 /* range check */ 2059 if (mult > 2) 2060 mult = 3; 2061 else 2062 mult++; 2063 2064 x = XREAD4(temp.sc, runt, XHCI_MFINDEX); 2065 2066 DPRINTF("MFINDEX=0x%08x\n", x); 2067 2068 switch (usbd_get_speed(xfer->xroot->udev)) { 2069 case USB_SPEED_FULL: 2070 shift = 3; 2071 temp.isoc_delta = 8; /* 1ms */ 2072 x += temp.isoc_delta - 1; 2073 x &= ~(temp.isoc_delta - 1); 2074 break; 2075 default: 2076 shift = usbd_xfer_get_fps_shift(xfer); 2077 temp.isoc_delta = 1U << shift; 2078 x += temp.isoc_delta - 1; 2079 x &= ~(temp.isoc_delta - 1); 2080 /* simple frame load balancing */ 2081 x += xfer->endpoint->usb_uframe; 2082 break; 2083 } 2084 2085 y = XHCI_MFINDEX_GET(x - xfer->endpoint->isoc_next); 2086 2087 if ((xfer->endpoint->is_synced == 0) || 2088 (y < (xfer->nframes << shift)) || 2089 (XHCI_MFINDEX_GET(-y) >= (128 * 8))) { 2090 /* 2091 * If there is data underflow or the pipe 2092 * queue is empty we schedule the transfer a 2093 * few frames ahead of the current frame 2094 * position. Else two isochronous transfers 2095 * might overlap. 2096 */ 2097 xfer->endpoint->isoc_next = XHCI_MFINDEX_GET(x + (3 * 8)); 2098 xfer->endpoint->is_synced = 1; 2099 temp.do_isoc_sync = 1; 2100 2101 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next); 2102 } 2103 2104 /* compute isochronous completion time */ 2105 2106 y = XHCI_MFINDEX_GET(xfer->endpoint->isoc_next - (x & ~7)); 2107 2108 xfer->isoc_time_complete = 2109 usb_isoc_time_expand(&temp.sc->sc_bus, x / 8) + 2110 (y / 8) + (((xfer->nframes << shift) + 7) / 8); 2111 2112 x = 0; 2113 temp.isoc_frame = xfer->endpoint->isoc_next; 2114 temp.trb_type = XHCI_TRB_TYPE_ISOCH; 2115 2116 xfer->endpoint->isoc_next += xfer->nframes << shift; 2117 2118 } else if (xfer->flags_int.control_xfr) { 2119 /* check if we should prepend a setup message */ 2120 2121 if (xfer->flags_int.control_hdr) { 2122 temp.len = xfer->frlengths[0]; 2123 temp.pc = xfer->frbuffers + 0; 2124 temp.shortpkt = temp.len ? 1 : 0; 2125 temp.trb_type = XHCI_TRB_TYPE_SETUP_STAGE; 2126 temp.direction = 0; 2127 2128 /* check for last frame */ 2129 if (xfer->nframes == 1) { 2130 /* no STATUS stage yet, SETUP is last */ 2131 if (xfer->flags_int.control_act) 2132 temp.last_frame = 1; 2133 } 2134 2135 xhci_setup_generic_chain_sub(&temp); 2136 } 2137 x = 1; 2138 mult = 1; 2139 temp.isoc_delta = 0; 2140 temp.isoc_frame = 0; 2141 temp.trb_type = xfer->flags_int.control_did_data ? 2142 XHCI_TRB_TYPE_NORMAL : XHCI_TRB_TYPE_DATA_STAGE; 2143 } else { 2144 x = 0; 2145 mult = 1; 2146 temp.isoc_delta = 0; 2147 temp.isoc_frame = 0; 2148 temp.trb_type = XHCI_TRB_TYPE_NORMAL; 2149 } 2150 2151 if (x != xfer->nframes) { 2152 /* set up page_cache pointer */ 2153 temp.pc = xfer->frbuffers + x; 2154 /* set endpoint direction */ 2155 temp.direction = UE_GET_DIR(xfer->endpointno); 2156 } 2157 2158 while (x != xfer->nframes) { 2159 /* DATA0 / DATA1 message */ 2160 2161 temp.len = xfer->frlengths[x]; 2162 temp.step_td = ((xfer->endpointno & UE_DIR_IN) && 2163 x != 0 && temp.multishort == 0); 2164 2165 x++; 2166 2167 if (x == xfer->nframes) { 2168 if (xfer->flags_int.control_xfr) { 2169 /* no STATUS stage yet, DATA is last */ 2170 if (xfer->flags_int.control_act) 2171 temp.last_frame = 1; 2172 } else { 2173 temp.last_frame = 1; 2174 } 2175 } 2176 if (temp.len == 0) { 2177 /* make sure that we send an USB packet */ 2178 2179 temp.shortpkt = 0; 2180 2181 temp.tbc = 0; 2182 temp.tlbpc = mult - 1; 2183 2184 } else if (xfer->flags_int.isochronous_xfr) { 2185 uint8_t tdpc; 2186 2187 /* 2188 * Isochronous transfers don't have short 2189 * packet termination: 2190 */ 2191 2192 temp.shortpkt = 1; 2193 2194 /* isochronous transfers have a transfer limit */ 2195 2196 if (temp.len > xfer->max_frame_size) 2197 temp.len = xfer->max_frame_size; 2198 2199 /* compute TD packet count */ 2200 tdpc = howmany(temp.len, xfer->max_packet_size); 2201 2202 temp.tbc = howmany(tdpc, mult) - 1; 2203 temp.tlbpc = (tdpc % mult); 2204 2205 if (temp.tlbpc == 0) 2206 temp.tlbpc = mult - 1; 2207 else 2208 temp.tlbpc--; 2209 } else { 2210 /* regular data transfer */ 2211 2212 temp.shortpkt = xfer->flags.force_short_xfer ? 0 : 1; 2213 } 2214 2215 xhci_setup_generic_chain_sub(&temp); 2216 2217 if (xfer->flags_int.isochronous_xfr) { 2218 temp.offset += xfer->frlengths[x - 1]; 2219 temp.isoc_frame += temp.isoc_delta; 2220 } else { 2221 /* get next Page Cache pointer */ 2222 temp.pc = xfer->frbuffers + x; 2223 } 2224 } 2225 2226 /* check if we should append a status stage */ 2227 2228 if (xfer->flags_int.control_xfr && 2229 !xfer->flags_int.control_act) { 2230 /* 2231 * Send a DATA1 message and invert the current 2232 * endpoint direction. 2233 */ 2234 if (xhcictlstep || temp.sc->sc_ctlstep) { 2235 /* 2236 * Some XHCI controllers will not delay the 2237 * status stage until the next SOF. Force this 2238 * behaviour to avoid failed control 2239 * transfers. 2240 */ 2241 temp.step_td = (xfer->nframes != 0); 2242 } else { 2243 temp.step_td = 0; 2244 } 2245 temp.direction = UE_GET_DIR(xfer->endpointno) ^ UE_DIR_IN; 2246 temp.len = 0; 2247 temp.pc = NULL; 2248 temp.shortpkt = 0; 2249 temp.last_frame = 1; 2250 temp.trb_type = XHCI_TRB_TYPE_STATUS_STAGE; 2251 2252 xhci_setup_generic_chain_sub(&temp); 2253 } 2254 2255 td = temp.td; 2256 2257 /* must have at least one frame! */ 2258 2259 xfer->td_transfer_last = td; 2260 2261 DPRINTF("first=%p last=%p\n", xfer->td_transfer_first, td); 2262 } 2263 2264 static void 2265 xhci_set_slot_pointer(struct xhci_softc *sc, uint8_t index, uint64_t dev_addr) 2266 { 2267 struct usb_page_search buf_res; 2268 struct xhci_dev_ctx_addr *pdctxa; 2269 2270 usbd_get_page(&sc->sc_hw.ctx_pc, 0, &buf_res); 2271 2272 pdctxa = buf_res.buffer; 2273 2274 DPRINTF("addr[%u]=0x%016llx\n", index, (long long)dev_addr); 2275 2276 pdctxa->qwBaaDevCtxAddr[index] = htole64(dev_addr); 2277 2278 usb_pc_cpu_flush(&sc->sc_hw.ctx_pc); 2279 } 2280 2281 static usb_error_t 2282 xhci_configure_mask(struct usb_device *udev, uint32_t mask, uint8_t drop) 2283 { 2284 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2285 struct usb_page_search buf_inp; 2286 struct xhci_input_dev_ctx *pinp; 2287 uint32_t temp; 2288 uint8_t index; 2289 uint8_t x; 2290 2291 index = udev->controller_slot_id; 2292 2293 usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp); 2294 2295 pinp = buf_inp.buffer; 2296 2297 if (drop) { 2298 mask &= XHCI_INCTX_NON_CTRL_MASK; 2299 xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx0, mask); 2300 xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx1, 0); 2301 } else { 2302 /* 2303 * Some hardware requires that we drop the endpoint 2304 * context before adding it again: 2305 */ 2306 xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx0, 2307 mask & XHCI_INCTX_NON_CTRL_MASK); 2308 2309 /* Add new endpoint context */ 2310 xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx1, mask); 2311 2312 /* find most significant set bit */ 2313 for (x = 31; x != 1; x--) { 2314 if (mask & (1 << x)) 2315 break; 2316 } 2317 2318 /* adjust */ 2319 x--; 2320 2321 /* figure out the maximum number of contexts */ 2322 if (x > sc->sc_hw.devs[index].context_num) 2323 sc->sc_hw.devs[index].context_num = x; 2324 else 2325 x = sc->sc_hw.devs[index].context_num; 2326 2327 /* update number of contexts */ 2328 temp = xhci_ctx_get_le32(sc, &pinp->ctx_slot.dwSctx0); 2329 temp &= ~XHCI_SCTX_0_CTX_NUM_SET(31); 2330 temp |= XHCI_SCTX_0_CTX_NUM_SET(x + 1); 2331 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx0, temp); 2332 } 2333 usb_pc_cpu_flush(&sc->sc_hw.devs[index].input_pc); 2334 return (0); 2335 } 2336 2337 static usb_error_t 2338 xhci_configure_endpoint(struct usb_device *udev, 2339 struct usb_endpoint_descriptor *edesc, struct xhci_endpoint_ext *pepext, 2340 uint16_t interval, uint8_t max_packet_count, 2341 uint8_t mult, uint8_t fps_shift, uint16_t max_packet_size, 2342 uint16_t max_frame_size, uint8_t ep_mode) 2343 { 2344 struct usb_page_search buf_inp; 2345 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2346 struct xhci_input_dev_ctx *pinp; 2347 uint64_t ring_addr = pepext->physaddr; 2348 uint32_t temp; 2349 uint8_t index; 2350 uint8_t epno; 2351 uint8_t type; 2352 2353 index = udev->controller_slot_id; 2354 2355 usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp); 2356 2357 pinp = buf_inp.buffer; 2358 2359 epno = edesc->bEndpointAddress; 2360 type = edesc->bmAttributes & UE_XFERTYPE; 2361 2362 if (type == UE_CONTROL) 2363 epno |= UE_DIR_IN; 2364 2365 epno = XHCI_EPNO2EPID(epno); 2366 2367 if (epno == 0) 2368 return (USB_ERR_NO_PIPE); /* invalid */ 2369 2370 if (max_packet_count == 0) 2371 return (USB_ERR_BAD_BUFSIZE); 2372 2373 max_packet_count--; 2374 2375 if (mult == 0) 2376 return (USB_ERR_BAD_BUFSIZE); 2377 2378 /* store endpoint mode */ 2379 pepext->trb_ep_mode = ep_mode; 2380 /* store bMaxPacketSize for control endpoints */ 2381 pepext->trb_ep_maxp = edesc->wMaxPacketSize[0]; 2382 usb_pc_cpu_flush(pepext->page_cache); 2383 2384 if (ep_mode == USB_EP_MODE_STREAMS) { 2385 temp = XHCI_EPCTX_0_EPSTATE_SET(0) | 2386 XHCI_EPCTX_0_MAXP_STREAMS_SET(XHCI_MAX_STREAMS_LOG - 1) | 2387 XHCI_EPCTX_0_LSA_SET(1); 2388 2389 ring_addr += sizeof(struct xhci_trb) * 2390 XHCI_MAX_TRANSFERS * XHCI_MAX_STREAMS; 2391 } else { 2392 temp = XHCI_EPCTX_0_EPSTATE_SET(0) | 2393 XHCI_EPCTX_0_MAXP_STREAMS_SET(0) | 2394 XHCI_EPCTX_0_LSA_SET(0); 2395 2396 ring_addr |= XHCI_EPCTX_2_DCS_SET(1); 2397 } 2398 2399 switch (udev->speed) { 2400 case USB_SPEED_FULL: 2401 case USB_SPEED_LOW: 2402 /* 1ms -> 125us */ 2403 fps_shift += 3; 2404 break; 2405 default: 2406 break; 2407 } 2408 2409 switch (type) { 2410 case UE_INTERRUPT: 2411 if (fps_shift > 3) 2412 fps_shift--; 2413 temp |= XHCI_EPCTX_0_IVAL_SET(fps_shift); 2414 break; 2415 case UE_ISOCHRONOUS: 2416 temp |= XHCI_EPCTX_0_IVAL_SET(fps_shift); 2417 2418 switch (udev->speed) { 2419 case USB_SPEED_SUPER: 2420 if (mult > 3) 2421 mult = 3; 2422 temp |= XHCI_EPCTX_0_MULT_SET(mult - 1); 2423 max_packet_count /= mult; 2424 break; 2425 default: 2426 break; 2427 } 2428 break; 2429 default: 2430 break; 2431 } 2432 2433 xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx0, temp); 2434 2435 temp = 2436 XHCI_EPCTX_1_HID_SET(0) | 2437 XHCI_EPCTX_1_MAXB_SET(max_packet_count) | 2438 XHCI_EPCTX_1_MAXP_SIZE_SET(max_packet_size); 2439 2440 /* 2441 * Always enable the "three strikes and you are gone" feature 2442 * except for ISOCHRONOUS endpoints. This is suggested by 2443 * section 4.3.3 in the XHCI specification about device slot 2444 * initialisation. 2445 */ 2446 if (type != UE_ISOCHRONOUS) 2447 temp |= XHCI_EPCTX_1_CERR_SET(3); 2448 2449 switch (type) { 2450 case UE_CONTROL: 2451 temp |= XHCI_EPCTX_1_EPTYPE_SET(4); 2452 break; 2453 case UE_ISOCHRONOUS: 2454 temp |= XHCI_EPCTX_1_EPTYPE_SET(1); 2455 break; 2456 case UE_BULK: 2457 temp |= XHCI_EPCTX_1_EPTYPE_SET(2); 2458 break; 2459 default: 2460 temp |= XHCI_EPCTX_1_EPTYPE_SET(3); 2461 break; 2462 } 2463 2464 /* check for IN direction */ 2465 if (epno & 1) 2466 temp |= XHCI_EPCTX_1_EPTYPE_SET(4); 2467 2468 xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx1, temp); 2469 xhci_ctx_set_le64(sc, &pinp->ctx_ep[epno - 1].qwEpCtx2, ring_addr); 2470 2471 switch (edesc->bmAttributes & UE_XFERTYPE) { 2472 case UE_INTERRUPT: 2473 case UE_ISOCHRONOUS: 2474 temp = XHCI_EPCTX_4_MAX_ESIT_PAYLOAD_SET(max_frame_size) | 2475 XHCI_EPCTX_4_AVG_TRB_LEN_SET(MIN(XHCI_PAGE_SIZE, 2476 max_frame_size)); 2477 break; 2478 case UE_CONTROL: 2479 temp = XHCI_EPCTX_4_AVG_TRB_LEN_SET(8); 2480 break; 2481 default: 2482 temp = XHCI_EPCTX_4_AVG_TRB_LEN_SET(XHCI_PAGE_SIZE); 2483 break; 2484 } 2485 2486 xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx4, temp); 2487 2488 #ifdef USB_DEBUG 2489 xhci_dump_endpoint(sc, &pinp->ctx_ep[epno - 1]); 2490 #endif 2491 usb_pc_cpu_flush(&sc->sc_hw.devs[index].input_pc); 2492 2493 return (0); /* success */ 2494 } 2495 2496 static usb_error_t 2497 xhci_configure_endpoint_by_xfer(struct usb_xfer *xfer) 2498 { 2499 struct xhci_endpoint_ext *pepext; 2500 struct usb_endpoint_ss_comp_descriptor *ecomp; 2501 usb_stream_t x; 2502 2503 pepext = xhci_get_endpoint_ext(xfer->xroot->udev, 2504 xfer->endpoint->edesc); 2505 2506 ecomp = xfer->endpoint->ecomp; 2507 2508 for (x = 0; x != XHCI_MAX_STREAMS; x++) { 2509 uint64_t temp; 2510 2511 /* halt any transfers */ 2512 pepext->trb[x * XHCI_MAX_TRANSFERS].dwTrb3 = 0; 2513 2514 /* compute start of TRB ring for stream "x" */ 2515 temp = pepext->physaddr + 2516 (x * XHCI_MAX_TRANSFERS * sizeof(struct xhci_trb)) + 2517 XHCI_SCTX_0_SCT_SEC_TR_RING; 2518 2519 /* make tree structure */ 2520 pepext->trb[(XHCI_MAX_TRANSFERS * 2521 XHCI_MAX_STREAMS) + x].qwTrb0 = htole64(temp); 2522 2523 /* reserved fields */ 2524 pepext->trb[(XHCI_MAX_TRANSFERS * 2525 XHCI_MAX_STREAMS) + x].dwTrb2 = 0; 2526 pepext->trb[(XHCI_MAX_TRANSFERS * 2527 XHCI_MAX_STREAMS) + x].dwTrb3 = 0; 2528 } 2529 usb_pc_cpu_flush(pepext->page_cache); 2530 2531 return (xhci_configure_endpoint(xfer->xroot->udev, 2532 xfer->endpoint->edesc, pepext, 2533 xfer->interval, xfer->max_packet_count, 2534 (ecomp != NULL) ? UE_GET_SS_ISO_MULT(ecomp->bmAttributes) + 1 : 1, 2535 usbd_xfer_get_fps_shift(xfer), xfer->max_packet_size, 2536 xfer->max_frame_size, xfer->endpoint->ep_mode)); 2537 } 2538 2539 static usb_error_t 2540 xhci_configure_device(struct usb_device *udev) 2541 { 2542 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2543 struct usb_page_search buf_inp; 2544 struct usb_page_cache *pcinp; 2545 struct xhci_input_dev_ctx *pinp; 2546 struct usb_device *hubdev; 2547 uint32_t temp; 2548 uint32_t route; 2549 uint32_t rh_port; 2550 uint8_t is_hub; 2551 uint8_t index; 2552 uint8_t depth; 2553 2554 index = udev->controller_slot_id; 2555 2556 DPRINTF("index=%u\n", index); 2557 2558 pcinp = &sc->sc_hw.devs[index].input_pc; 2559 2560 usbd_get_page(pcinp, 0, &buf_inp); 2561 2562 pinp = buf_inp.buffer; 2563 2564 rh_port = 0; 2565 route = 0; 2566 2567 /* figure out route string and root HUB port number */ 2568 2569 for (hubdev = udev; hubdev != NULL; hubdev = hubdev->parent_hub) { 2570 if (hubdev->parent_hub == NULL) 2571 break; 2572 2573 depth = hubdev->parent_hub->depth; 2574 2575 /* 2576 * NOTE: HS/FS/LS devices and the SS root HUB can have 2577 * more than 15 ports 2578 */ 2579 2580 rh_port = hubdev->port_no; 2581 2582 if (depth == 0) 2583 break; 2584 2585 if (rh_port > 15) 2586 rh_port = 15; 2587 2588 if (depth < 6) 2589 route |= rh_port << (4 * (depth - 1)); 2590 } 2591 2592 DPRINTF("Route=0x%08x\n", route); 2593 2594 temp = XHCI_SCTX_0_ROUTE_SET(route) | 2595 XHCI_SCTX_0_CTX_NUM_SET( 2596 sc->sc_hw.devs[index].context_num + 1); 2597 2598 switch (udev->speed) { 2599 case USB_SPEED_LOW: 2600 temp |= XHCI_SCTX_0_SPEED_SET(2); 2601 if (udev->parent_hs_hub != NULL && 2602 udev->parent_hs_hub->ddesc.bDeviceProtocol == 2603 UDPROTO_HSHUBMTT) { 2604 DPRINTF("Device inherits MTT\n"); 2605 temp |= XHCI_SCTX_0_MTT_SET(1); 2606 } 2607 break; 2608 case USB_SPEED_HIGH: 2609 temp |= XHCI_SCTX_0_SPEED_SET(3); 2610 if (sc->sc_hw.devs[index].nports != 0 && 2611 udev->ddesc.bDeviceProtocol == UDPROTO_HSHUBMTT) { 2612 DPRINTF("HUB supports MTT\n"); 2613 temp |= XHCI_SCTX_0_MTT_SET(1); 2614 } 2615 break; 2616 case USB_SPEED_FULL: 2617 temp |= XHCI_SCTX_0_SPEED_SET(1); 2618 if (udev->parent_hs_hub != NULL && 2619 udev->parent_hs_hub->ddesc.bDeviceProtocol == 2620 UDPROTO_HSHUBMTT) { 2621 DPRINTF("Device inherits MTT\n"); 2622 temp |= XHCI_SCTX_0_MTT_SET(1); 2623 } 2624 break; 2625 default: 2626 temp |= XHCI_SCTX_0_SPEED_SET(4); 2627 break; 2628 } 2629 2630 is_hub = sc->sc_hw.devs[index].nports != 0 && 2631 (udev->speed == USB_SPEED_SUPER || 2632 udev->speed == USB_SPEED_HIGH); 2633 2634 if (is_hub) 2635 temp |= XHCI_SCTX_0_HUB_SET(1); 2636 2637 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx0, temp); 2638 2639 temp = XHCI_SCTX_1_RH_PORT_SET(rh_port); 2640 2641 if (is_hub) { 2642 temp |= XHCI_SCTX_1_NUM_PORTS_SET( 2643 sc->sc_hw.devs[index].nports); 2644 } 2645 2646 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx1, temp); 2647 2648 temp = XHCI_SCTX_2_IRQ_TARGET_SET(0); 2649 2650 if (is_hub) { 2651 temp |= XHCI_SCTX_2_TT_THINK_TIME_SET( 2652 sc->sc_hw.devs[index].tt); 2653 } 2654 2655 hubdev = udev->parent_hs_hub; 2656 2657 /* check if we should activate the transaction translator */ 2658 switch (udev->speed) { 2659 case USB_SPEED_FULL: 2660 case USB_SPEED_LOW: 2661 if (hubdev != NULL) { 2662 temp |= XHCI_SCTX_2_TT_HUB_SID_SET( 2663 hubdev->controller_slot_id); 2664 temp |= XHCI_SCTX_2_TT_PORT_NUM_SET( 2665 udev->hs_port_no); 2666 } 2667 break; 2668 default: 2669 break; 2670 } 2671 2672 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx2, temp); 2673 2674 /* 2675 * These fields should be initialized to zero, according to 2676 * XHCI section 6.2.2 - slot context: 2677 */ 2678 temp = XHCI_SCTX_3_DEV_ADDR_SET(0) | 2679 XHCI_SCTX_3_SLOT_STATE_SET(0); 2680 2681 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx3, temp); 2682 2683 #ifdef USB_DEBUG 2684 xhci_dump_device(sc, &pinp->ctx_slot); 2685 #endif 2686 usb_pc_cpu_flush(pcinp); 2687 2688 return (0); /* success */ 2689 } 2690 2691 static usb_error_t 2692 xhci_alloc_device_ext(struct usb_device *udev) 2693 { 2694 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2695 struct usb_page_search buf_dev; 2696 struct usb_page_search buf_ep; 2697 struct xhci_trb *trb; 2698 struct usb_page_cache *pc; 2699 struct usb_page *pg; 2700 uint64_t addr; 2701 uint8_t index; 2702 uint8_t i; 2703 2704 index = udev->controller_slot_id; 2705 2706 pc = &sc->sc_hw.devs[index].device_pc; 2707 pg = &sc->sc_hw.devs[index].device_pg; 2708 2709 /* need to initialize the page cache */ 2710 pc->tag_parent = sc->sc_bus.dma_parent_tag; 2711 2712 if (usb_pc_alloc_mem(pc, pg, sc->sc_ctx_is_64_byte ? 2713 (2 * sizeof(struct xhci_dev_ctx)) : 2714 sizeof(struct xhci_dev_ctx), XHCI_PAGE_SIZE)) 2715 goto error; 2716 2717 usbd_get_page(pc, 0, &buf_dev); 2718 2719 pc = &sc->sc_hw.devs[index].input_pc; 2720 pg = &sc->sc_hw.devs[index].input_pg; 2721 2722 /* need to initialize the page cache */ 2723 pc->tag_parent = sc->sc_bus.dma_parent_tag; 2724 2725 if (usb_pc_alloc_mem(pc, pg, sc->sc_ctx_is_64_byte ? 2726 (2 * sizeof(struct xhci_input_dev_ctx)) : 2727 sizeof(struct xhci_input_dev_ctx), XHCI_PAGE_SIZE)) { 2728 goto error; 2729 } 2730 2731 /* initialize all endpoint LINK TRBs */ 2732 2733 for (i = 0; i != XHCI_MAX_ENDPOINTS; i++) { 2734 pc = &sc->sc_hw.devs[index].endpoint_pc[i]; 2735 pg = &sc->sc_hw.devs[index].endpoint_pg[i]; 2736 2737 /* need to initialize the page cache */ 2738 pc->tag_parent = sc->sc_bus.dma_parent_tag; 2739 2740 if (usb_pc_alloc_mem(pc, pg, 2741 sizeof(struct xhci_dev_endpoint_trbs), XHCI_TRB_ALIGN)) { 2742 goto error; 2743 } 2744 2745 /* lookup endpoint TRB ring */ 2746 usbd_get_page(pc, 0, &buf_ep); 2747 2748 /* get TRB pointer */ 2749 trb = buf_ep.buffer; 2750 trb += XHCI_MAX_TRANSFERS - 1; 2751 2752 /* get TRB start address */ 2753 addr = buf_ep.physaddr; 2754 2755 /* create LINK TRB */ 2756 trb->qwTrb0 = htole64(addr); 2757 trb->dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0)); 2758 trb->dwTrb3 = htole32(XHCI_TRB_3_CYCLE_BIT | 2759 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK)); 2760 2761 usb_pc_cpu_flush(pc); 2762 } 2763 2764 xhci_set_slot_pointer(sc, index, buf_dev.physaddr); 2765 2766 return (0); 2767 2768 error: 2769 xhci_free_device_ext(udev); 2770 2771 return (USB_ERR_NOMEM); 2772 } 2773 2774 static void 2775 xhci_free_device_ext(struct usb_device *udev) 2776 { 2777 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2778 uint8_t index; 2779 uint8_t i; 2780 2781 index = udev->controller_slot_id; 2782 xhci_set_slot_pointer(sc, index, 0); 2783 2784 usb_pc_free_mem(&sc->sc_hw.devs[index].device_pc); 2785 usb_pc_free_mem(&sc->sc_hw.devs[index].input_pc); 2786 for (i = 0; i != XHCI_MAX_ENDPOINTS; i++) 2787 usb_pc_free_mem(&sc->sc_hw.devs[index].endpoint_pc[i]); 2788 } 2789 2790 static struct xhci_endpoint_ext * 2791 xhci_get_endpoint_ext(struct usb_device *udev, struct usb_endpoint_descriptor *edesc) 2792 { 2793 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2794 struct xhci_endpoint_ext *pepext; 2795 struct usb_page_cache *pc; 2796 struct usb_page_search buf_ep; 2797 uint8_t epno; 2798 uint8_t index; 2799 2800 epno = edesc->bEndpointAddress; 2801 if ((edesc->bmAttributes & UE_XFERTYPE) == UE_CONTROL) 2802 epno |= UE_DIR_IN; 2803 2804 epno = XHCI_EPNO2EPID(epno); 2805 2806 index = udev->controller_slot_id; 2807 2808 pc = &sc->sc_hw.devs[index].endpoint_pc[epno]; 2809 2810 usbd_get_page(pc, 0, &buf_ep); 2811 2812 pepext = &sc->sc_hw.devs[index].endp[epno]; 2813 pepext->page_cache = pc; 2814 pepext->trb = buf_ep.buffer; 2815 pepext->physaddr = buf_ep.physaddr; 2816 2817 return (pepext); 2818 } 2819 2820 static void 2821 xhci_endpoint_doorbell(struct usb_xfer *xfer) 2822 { 2823 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus); 2824 uint8_t epno; 2825 uint8_t index; 2826 2827 epno = xfer->endpointno; 2828 if (xfer->flags_int.control_xfr) 2829 epno |= UE_DIR_IN; 2830 2831 epno = XHCI_EPNO2EPID(epno); 2832 index = xfer->xroot->udev->controller_slot_id; 2833 2834 if (xfer->xroot->udev->flags.self_suspended == 0) { 2835 XWRITE4(sc, door, XHCI_DOORBELL(index), 2836 epno | XHCI_DB_SID_SET(xfer->stream_id)); 2837 } 2838 } 2839 2840 static void 2841 xhci_transfer_remove(struct usb_xfer *xfer, usb_error_t error) 2842 { 2843 struct xhci_endpoint_ext *pepext; 2844 2845 if (xfer->flags_int.bandwidth_reclaimed) { 2846 xfer->flags_int.bandwidth_reclaimed = 0; 2847 2848 pepext = xhci_get_endpoint_ext(xfer->xroot->udev, 2849 xfer->endpoint->edesc); 2850 2851 pepext->trb_used[xfer->stream_id]--; 2852 2853 pepext->xfer[xfer->qh_pos] = NULL; 2854 2855 if (error && pepext->trb_running != 0) { 2856 pepext->trb_halted = 1; 2857 pepext->trb_running = 0; 2858 } 2859 } 2860 } 2861 2862 static usb_error_t 2863 xhci_transfer_insert(struct usb_xfer *xfer) 2864 { 2865 struct xhci_td *td_first; 2866 struct xhci_td *td_last; 2867 struct xhci_trb *trb_link; 2868 struct xhci_endpoint_ext *pepext; 2869 uint64_t addr; 2870 usb_stream_t id; 2871 uint8_t i; 2872 uint8_t inext; 2873 uint8_t trb_limit; 2874 2875 DPRINTFN(8, "\n"); 2876 2877 id = xfer->stream_id; 2878 2879 /* check if already inserted */ 2880 if (xfer->flags_int.bandwidth_reclaimed) { 2881 DPRINTFN(8, "Already in schedule\n"); 2882 return (0); 2883 } 2884 2885 pepext = xhci_get_endpoint_ext(xfer->xroot->udev, 2886 xfer->endpoint->edesc); 2887 2888 td_first = xfer->td_transfer_first; 2889 td_last = xfer->td_transfer_last; 2890 addr = pepext->physaddr; 2891 2892 switch (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE) { 2893 case UE_CONTROL: 2894 case UE_INTERRUPT: 2895 /* single buffered */ 2896 trb_limit = 1; 2897 break; 2898 default: 2899 /* multi buffered */ 2900 trb_limit = (XHCI_MAX_TRANSFERS - 2); 2901 break; 2902 } 2903 2904 if (pepext->trb_used[id] >= trb_limit) { 2905 DPRINTFN(8, "Too many TDs queued.\n"); 2906 return (USB_ERR_NOMEM); 2907 } 2908 2909 /* check if bMaxPacketSize changed */ 2910 if (xfer->flags_int.control_xfr != 0 && 2911 pepext->trb_ep_maxp != xfer->endpoint->edesc->wMaxPacketSize[0]) { 2912 DPRINTFN(8, "Reconfigure control endpoint\n"); 2913 2914 /* force driver to reconfigure endpoint */ 2915 pepext->trb_halted = 1; 2916 pepext->trb_running = 0; 2917 } 2918 2919 /* check for stopped condition, after putting transfer on interrupt queue */ 2920 if (pepext->trb_running == 0) { 2921 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus); 2922 2923 DPRINTFN(8, "Not running\n"); 2924 2925 /* start configuration */ 2926 (void)usb_proc_msignal(USB_BUS_CONTROL_XFER_PROC(&sc->sc_bus), 2927 &sc->sc_config_msg[0], &sc->sc_config_msg[1]); 2928 return (0); 2929 } 2930 2931 pepext->trb_used[id]++; 2932 2933 /* get current TRB index */ 2934 i = pepext->trb_index[id]; 2935 2936 /* get next TRB index */ 2937 inext = (i + 1); 2938 2939 /* the last entry of the ring is a hardcoded link TRB */ 2940 if (inext >= (XHCI_MAX_TRANSFERS - 1)) 2941 inext = 0; 2942 2943 /* store next TRB index, before stream ID offset is added */ 2944 pepext->trb_index[id] = inext; 2945 2946 /* offset for stream */ 2947 i += id * XHCI_MAX_TRANSFERS; 2948 inext += id * XHCI_MAX_TRANSFERS; 2949 2950 /* compute terminating return address */ 2951 addr += (inext * sizeof(struct xhci_trb)); 2952 2953 /* compute link TRB pointer */ 2954 trb_link = td_last->td_trb + td_last->ntrb; 2955 2956 /* update next pointer of last link TRB */ 2957 trb_link->qwTrb0 = htole64(addr); 2958 trb_link->dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0)); 2959 trb_link->dwTrb3 = htole32(XHCI_TRB_3_IOC_BIT | 2960 XHCI_TRB_3_CYCLE_BIT | 2961 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK)); 2962 2963 #ifdef USB_DEBUG 2964 xhci_dump_trb(&td_last->td_trb[td_last->ntrb]); 2965 #endif 2966 usb_pc_cpu_flush(td_last->page_cache); 2967 2968 /* write ahead chain end marker */ 2969 2970 pepext->trb[inext].qwTrb0 = 0; 2971 pepext->trb[inext].dwTrb2 = 0; 2972 pepext->trb[inext].dwTrb3 = 0; 2973 2974 /* update next pointer of link TRB */ 2975 2976 pepext->trb[i].qwTrb0 = htole64((uint64_t)td_first->td_self); 2977 pepext->trb[i].dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0)); 2978 2979 #ifdef USB_DEBUG 2980 xhci_dump_trb(&pepext->trb[i]); 2981 #endif 2982 usb_pc_cpu_flush(pepext->page_cache); 2983 2984 /* toggle cycle bit which activates the transfer chain */ 2985 2986 pepext->trb[i].dwTrb3 = htole32(XHCI_TRB_3_CYCLE_BIT | 2987 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK)); 2988 2989 usb_pc_cpu_flush(pepext->page_cache); 2990 2991 DPRINTF("qh_pos = %u\n", i); 2992 2993 pepext->xfer[i] = xfer; 2994 2995 xfer->qh_pos = i; 2996 2997 xfer->flags_int.bandwidth_reclaimed = 1; 2998 2999 xhci_endpoint_doorbell(xfer); 3000 3001 return (0); 3002 } 3003 3004 static void 3005 xhci_root_intr(struct xhci_softc *sc) 3006 { 3007 uint16_t i; 3008 3009 USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED); 3010 3011 /* clear any old interrupt data */ 3012 memset(sc->sc_hub_idata, 0, sizeof(sc->sc_hub_idata)); 3013 3014 for (i = 1; i <= sc->sc_noport; i++) { 3015 /* pick out CHANGE bits from the status register */ 3016 if (XREAD4(sc, oper, XHCI_PORTSC(i)) & ( 3017 XHCI_PS_CSC | XHCI_PS_PEC | 3018 XHCI_PS_OCC | XHCI_PS_WRC | 3019 XHCI_PS_PRC | XHCI_PS_PLC | 3020 XHCI_PS_CEC)) { 3021 sc->sc_hub_idata[i / 8] |= 1 << (i % 8); 3022 DPRINTF("port %d changed\n", i); 3023 } 3024 } 3025 uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata, 3026 sizeof(sc->sc_hub_idata)); 3027 } 3028 3029 /*------------------------------------------------------------------------* 3030 * xhci_device_done - XHCI done handler 3031 * 3032 * NOTE: This function can be called two times in a row on 3033 * the same USB transfer. From close and from interrupt. 3034 *------------------------------------------------------------------------*/ 3035 static void 3036 xhci_device_done(struct usb_xfer *xfer, usb_error_t error) 3037 { 3038 DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n", 3039 xfer, xfer->endpoint, error); 3040 3041 /* remove transfer from HW queue */ 3042 xhci_transfer_remove(xfer, error); 3043 3044 /* dequeue transfer and start next transfer */ 3045 usbd_transfer_done(xfer, error); 3046 } 3047 3048 /*------------------------------------------------------------------------* 3049 * XHCI data transfer support (generic type) 3050 *------------------------------------------------------------------------*/ 3051 static void 3052 xhci_device_generic_open(struct usb_xfer *xfer) 3053 { 3054 if (xfer->flags_int.isochronous_xfr) { 3055 switch (xfer->xroot->udev->speed) { 3056 case USB_SPEED_FULL: 3057 break; 3058 default: 3059 usb_hs_bandwidth_alloc(xfer); 3060 break; 3061 } 3062 } 3063 } 3064 3065 static void 3066 xhci_device_generic_close(struct usb_xfer *xfer) 3067 { 3068 DPRINTF("\n"); 3069 3070 xhci_device_done(xfer, USB_ERR_CANCELLED); 3071 3072 if (xfer->flags_int.isochronous_xfr) { 3073 switch (xfer->xroot->udev->speed) { 3074 case USB_SPEED_FULL: 3075 break; 3076 default: 3077 usb_hs_bandwidth_free(xfer); 3078 break; 3079 } 3080 } 3081 } 3082 3083 static void 3084 xhci_device_generic_multi_enter(struct usb_endpoint *ep, 3085 usb_stream_t stream_id, struct usb_xfer *enter_xfer) 3086 { 3087 struct usb_xfer *xfer; 3088 3089 /* check if there is a current transfer */ 3090 xfer = ep->endpoint_q[stream_id].curr; 3091 if (xfer == NULL) 3092 return; 3093 3094 /* 3095 * Check if the current transfer is started and then pickup 3096 * the next one, if any. Else wait for next start event due to 3097 * block on failure feature. 3098 */ 3099 if (!xfer->flags_int.bandwidth_reclaimed) 3100 return; 3101 3102 xfer = TAILQ_FIRST(&ep->endpoint_q[stream_id].head); 3103 if (xfer == NULL) { 3104 /* 3105 * In case of enter we have to consider that the 3106 * transfer is queued by the USB core after the enter 3107 * method is called. 3108 */ 3109 xfer = enter_xfer; 3110 3111 if (xfer == NULL) 3112 return; 3113 } 3114 3115 /* try to multi buffer */ 3116 xhci_transfer_insert(xfer); 3117 } 3118 3119 static void 3120 xhci_device_generic_enter(struct usb_xfer *xfer) 3121 { 3122 DPRINTF("\n"); 3123 3124 /* set up TD's and QH */ 3125 xhci_setup_generic_chain(xfer); 3126 3127 xhci_device_generic_multi_enter(xfer->endpoint, 3128 xfer->stream_id, xfer); 3129 } 3130 3131 static void 3132 xhci_device_generic_start(struct usb_xfer *xfer) 3133 { 3134 DPRINTF("\n"); 3135 3136 /* try to insert xfer on HW queue */ 3137 xhci_transfer_insert(xfer); 3138 3139 /* try to multi buffer */ 3140 xhci_device_generic_multi_enter(xfer->endpoint, 3141 xfer->stream_id, NULL); 3142 3143 /* add transfer last on interrupt queue */ 3144 usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer); 3145 3146 /* start timeout, if any */ 3147 if (xfer->timeout != 0) 3148 usbd_transfer_timeout_ms(xfer, &xhci_timeout, xfer->timeout); 3149 } 3150 3151 static const struct usb_pipe_methods xhci_device_generic_methods = 3152 { 3153 .open = xhci_device_generic_open, 3154 .close = xhci_device_generic_close, 3155 .enter = xhci_device_generic_enter, 3156 .start = xhci_device_generic_start, 3157 }; 3158 3159 /*------------------------------------------------------------------------* 3160 * xhci root HUB support 3161 *------------------------------------------------------------------------* 3162 * Simulate a hardware HUB by handling all the necessary requests. 3163 *------------------------------------------------------------------------*/ 3164 #define HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) } 3165 3166 static const 3167 struct usb_device_descriptor xhci_devd = 3168 { 3169 .bLength = sizeof(xhci_devd), 3170 .bDescriptorType = UDESC_DEVICE, /* type */ 3171 HSETW(.bcdUSB, 0x0300), /* USB version */ 3172 .bDeviceClass = UDCLASS_HUB, /* class */ 3173 .bDeviceSubClass = UDSUBCLASS_HUB, /* subclass */ 3174 .bDeviceProtocol = UDPROTO_SSHUB, /* protocol */ 3175 .bMaxPacketSize = 9, /* max packet size */ 3176 HSETW(.idVendor, 0x0000), /* vendor */ 3177 HSETW(.idProduct, 0x0000), /* product */ 3178 HSETW(.bcdDevice, 0x0100), /* device version */ 3179 .iManufacturer = 1, 3180 .iProduct = 2, 3181 .iSerialNumber = 0, 3182 .bNumConfigurations = 1, /* # of configurations */ 3183 }; 3184 3185 static const 3186 struct xhci_bos_desc xhci_bosd = { 3187 .bosd = { 3188 .bLength = sizeof(xhci_bosd.bosd), 3189 .bDescriptorType = UDESC_BOS, 3190 HSETW(.wTotalLength, sizeof(xhci_bosd)), 3191 .bNumDeviceCaps = 3, 3192 }, 3193 .usb2extd = { 3194 .bLength = sizeof(xhci_bosd.usb2extd), 3195 .bDescriptorType = 1, 3196 .bDevCapabilityType = 2, 3197 .bmAttributes[0] = 2, 3198 }, 3199 .usbdcd = { 3200 .bLength = sizeof(xhci_bosd.usbdcd), 3201 .bDescriptorType = UDESC_DEVICE_CAPABILITY, 3202 .bDevCapabilityType = 3, 3203 .bmAttributes = 0, /* XXX */ 3204 HSETW(.wSpeedsSupported, 0x000C), 3205 .bFunctionalitySupport = 8, 3206 .bU1DevExitLat = 255, /* dummy - not used */ 3207 .wU2DevExitLat = { 0x00, 0x08 }, 3208 }, 3209 .cidd = { 3210 .bLength = sizeof(xhci_bosd.cidd), 3211 .bDescriptorType = 1, 3212 .bDevCapabilityType = 4, 3213 .bReserved = 0, 3214 .bContainerID = 0, /* XXX */ 3215 }, 3216 }; 3217 3218 static const 3219 struct xhci_config_desc xhci_confd = { 3220 .confd = { 3221 .bLength = sizeof(xhci_confd.confd), 3222 .bDescriptorType = UDESC_CONFIG, 3223 .wTotalLength[0] = sizeof(xhci_confd), 3224 .bNumInterface = 1, 3225 .bConfigurationValue = 1, 3226 .iConfiguration = 0, 3227 .bmAttributes = UC_SELF_POWERED, 3228 .bMaxPower = 0 /* max power */ 3229 }, 3230 .ifcd = { 3231 .bLength = sizeof(xhci_confd.ifcd), 3232 .bDescriptorType = UDESC_INTERFACE, 3233 .bNumEndpoints = 1, 3234 .bInterfaceClass = UICLASS_HUB, 3235 .bInterfaceSubClass = UISUBCLASS_HUB, 3236 .bInterfaceProtocol = 0, 3237 }, 3238 .endpd = { 3239 .bLength = sizeof(xhci_confd.endpd), 3240 .bDescriptorType = UDESC_ENDPOINT, 3241 .bEndpointAddress = UE_DIR_IN | XHCI_INTR_ENDPT, 3242 .bmAttributes = UE_INTERRUPT, 3243 .wMaxPacketSize[0] = 2, /* max 15 ports */ 3244 .bInterval = 255, 3245 }, 3246 .endpcd = { 3247 .bLength = sizeof(xhci_confd.endpcd), 3248 .bDescriptorType = UDESC_ENDPOINT_SS_COMP, 3249 .bMaxBurst = 0, 3250 .bmAttributes = 0, 3251 }, 3252 }; 3253 3254 static const 3255 struct usb_hub_ss_descriptor xhci_hubd = { 3256 .bLength = sizeof(xhci_hubd), 3257 .bDescriptorType = UDESC_SS_HUB, 3258 }; 3259 3260 static usb_error_t 3261 xhci_roothub_exec(struct usb_device *udev, 3262 struct usb_device_request *req, const void **pptr, uint16_t *plength) 3263 { 3264 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 3265 const char *str_ptr; 3266 const void *ptr; 3267 uint32_t port; 3268 uint32_t v; 3269 uint16_t len; 3270 uint16_t i; 3271 uint16_t value; 3272 uint16_t index; 3273 uint8_t j; 3274 usb_error_t err; 3275 3276 USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED); 3277 3278 /* buffer reset */ 3279 ptr = (const void *)&sc->sc_hub_desc; 3280 len = 0; 3281 err = 0; 3282 3283 value = UGETW(req->wValue); 3284 index = UGETW(req->wIndex); 3285 3286 DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x " 3287 "wValue=0x%04x wIndex=0x%04x\n", 3288 req->bmRequestType, req->bRequest, 3289 UGETW(req->wLength), value, index); 3290 3291 #define C(x,y) ((x) | ((y) << 8)) 3292 switch (C(req->bRequest, req->bmRequestType)) { 3293 case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE): 3294 case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE): 3295 case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT): 3296 /* 3297 * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops 3298 * for the integrated root hub. 3299 */ 3300 break; 3301 case C(UR_GET_CONFIG, UT_READ_DEVICE): 3302 len = 1; 3303 sc->sc_hub_desc.temp[0] = sc->sc_conf; 3304 break; 3305 case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE): 3306 switch (value >> 8) { 3307 case UDESC_DEVICE: 3308 if ((value & 0xff) != 0) { 3309 err = USB_ERR_IOERROR; 3310 goto done; 3311 } 3312 len = sizeof(xhci_devd); 3313 ptr = (const void *)&xhci_devd; 3314 break; 3315 3316 case UDESC_BOS: 3317 if ((value & 0xff) != 0) { 3318 err = USB_ERR_IOERROR; 3319 goto done; 3320 } 3321 len = sizeof(xhci_bosd); 3322 ptr = (const void *)&xhci_bosd; 3323 break; 3324 3325 case UDESC_CONFIG: 3326 if ((value & 0xff) != 0) { 3327 err = USB_ERR_IOERROR; 3328 goto done; 3329 } 3330 len = sizeof(xhci_confd); 3331 ptr = (const void *)&xhci_confd; 3332 break; 3333 3334 case UDESC_STRING: 3335 switch (value & 0xff) { 3336 case 0: /* Language table */ 3337 str_ptr = "\001"; 3338 break; 3339 3340 case 1: /* Vendor */ 3341 str_ptr = sc->sc_vendor; 3342 break; 3343 3344 case 2: /* Product */ 3345 str_ptr = "XHCI root HUB"; 3346 break; 3347 3348 default: 3349 str_ptr = ""; 3350 break; 3351 } 3352 3353 len = usb_make_str_desc( 3354 sc->sc_hub_desc.temp, 3355 sizeof(sc->sc_hub_desc.temp), 3356 str_ptr); 3357 break; 3358 3359 default: 3360 err = USB_ERR_IOERROR; 3361 goto done; 3362 } 3363 break; 3364 case C(UR_GET_INTERFACE, UT_READ_INTERFACE): 3365 len = 1; 3366 sc->sc_hub_desc.temp[0] = 0; 3367 break; 3368 case C(UR_GET_STATUS, UT_READ_DEVICE): 3369 len = 2; 3370 USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED); 3371 break; 3372 case C(UR_GET_STATUS, UT_READ_INTERFACE): 3373 case C(UR_GET_STATUS, UT_READ_ENDPOINT): 3374 len = 2; 3375 USETW(sc->sc_hub_desc.stat.wStatus, 0); 3376 break; 3377 case C(UR_SET_ADDRESS, UT_WRITE_DEVICE): 3378 if (value >= XHCI_MAX_DEVICES) { 3379 err = USB_ERR_IOERROR; 3380 goto done; 3381 } 3382 break; 3383 case C(UR_SET_CONFIG, UT_WRITE_DEVICE): 3384 if (value != 0 && value != 1) { 3385 err = USB_ERR_IOERROR; 3386 goto done; 3387 } 3388 sc->sc_conf = value; 3389 break; 3390 case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE): 3391 break; 3392 case C(UR_SET_FEATURE, UT_WRITE_DEVICE): 3393 case C(UR_SET_FEATURE, UT_WRITE_INTERFACE): 3394 case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT): 3395 err = USB_ERR_IOERROR; 3396 goto done; 3397 case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE): 3398 break; 3399 case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT): 3400 break; 3401 /* Hub requests */ 3402 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE): 3403 break; 3404 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER): 3405 DPRINTFN(9, "UR_CLEAR_PORT_FEATURE\n"); 3406 3407 if ((index < 1) || 3408 (index > sc->sc_noport)) { 3409 err = USB_ERR_IOERROR; 3410 goto done; 3411 } 3412 port = XHCI_PORTSC(index); 3413 3414 v = XREAD4(sc, oper, port); 3415 i = XHCI_PS_PLS_GET(v); 3416 v &= ~XHCI_PS_CLEAR; 3417 3418 switch (value) { 3419 case UHF_C_BH_PORT_RESET: 3420 XWRITE4(sc, oper, port, v | XHCI_PS_WRC); 3421 break; 3422 case UHF_C_PORT_CONFIG_ERROR: 3423 XWRITE4(sc, oper, port, v | XHCI_PS_CEC); 3424 break; 3425 case UHF_C_PORT_SUSPEND: 3426 case UHF_C_PORT_LINK_STATE: 3427 XWRITE4(sc, oper, port, v | XHCI_PS_PLC); 3428 break; 3429 case UHF_C_PORT_CONNECTION: 3430 XWRITE4(sc, oper, port, v | XHCI_PS_CSC); 3431 break; 3432 case UHF_C_PORT_ENABLE: 3433 XWRITE4(sc, oper, port, v | XHCI_PS_PEC); 3434 break; 3435 case UHF_C_PORT_OVER_CURRENT: 3436 XWRITE4(sc, oper, port, v | XHCI_PS_OCC); 3437 break; 3438 case UHF_C_PORT_RESET: 3439 XWRITE4(sc, oper, port, v | XHCI_PS_PRC); 3440 break; 3441 case UHF_PORT_ENABLE: 3442 XWRITE4(sc, oper, port, v | XHCI_PS_PED); 3443 break; 3444 case UHF_PORT_POWER: 3445 XWRITE4(sc, oper, port, v & ~XHCI_PS_PP); 3446 break; 3447 case UHF_PORT_INDICATOR: 3448 XWRITE4(sc, oper, port, v & ~XHCI_PS_PIC_SET(3)); 3449 break; 3450 case UHF_PORT_SUSPEND: 3451 3452 /* U3 -> U15 */ 3453 if (i == 3) { 3454 XWRITE4(sc, oper, port, v | 3455 XHCI_PS_PLS_SET(0xF) | XHCI_PS_LWS); 3456 } 3457 3458 /* wait 20ms for resume sequence to complete */ 3459 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50); 3460 3461 /* U0 */ 3462 XWRITE4(sc, oper, port, v | 3463 XHCI_PS_PLS_SET(0) | XHCI_PS_LWS); 3464 break; 3465 default: 3466 err = USB_ERR_IOERROR; 3467 goto done; 3468 } 3469 break; 3470 3471 case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE): 3472 if ((value & 0xff) != 0) { 3473 err = USB_ERR_IOERROR; 3474 goto done; 3475 } 3476 3477 v = XREAD4(sc, capa, XHCI_HCSPARAMS0); 3478 3479 sc->sc_hub_desc.hubd = xhci_hubd; 3480 3481 sc->sc_hub_desc.hubd.bNbrPorts = sc->sc_noport; 3482 3483 if (XHCI_HCS0_PPC(v)) 3484 i = UHD_PWR_INDIVIDUAL; 3485 else 3486 i = UHD_PWR_GANGED; 3487 3488 if (XHCI_HCS0_PIND(v)) 3489 i |= UHD_PORT_IND; 3490 3491 i |= UHD_OC_INDIVIDUAL; 3492 3493 USETW(sc->sc_hub_desc.hubd.wHubCharacteristics, i); 3494 3495 /* see XHCI section 5.4.9: */ 3496 sc->sc_hub_desc.hubd.bPwrOn2PwrGood = 10; 3497 3498 for (j = 1; j <= sc->sc_noport; j++) { 3499 v = XREAD4(sc, oper, XHCI_PORTSC(j)); 3500 if (v & XHCI_PS_DR) { 3501 sc->sc_hub_desc.hubd. 3502 DeviceRemovable[j / 8] |= 1U << (j % 8); 3503 } 3504 } 3505 len = sc->sc_hub_desc.hubd.bLength; 3506 break; 3507 3508 case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE): 3509 len = 16; 3510 memset(sc->sc_hub_desc.temp, 0, 16); 3511 break; 3512 3513 case C(UR_GET_STATUS, UT_READ_CLASS_OTHER): 3514 DPRINTFN(9, "UR_GET_STATUS i=%d\n", index); 3515 3516 if ((index < 1) || 3517 (index > sc->sc_noport)) { 3518 err = USB_ERR_IOERROR; 3519 goto done; 3520 } 3521 3522 v = XREAD4(sc, oper, XHCI_PORTSC(index)); 3523 3524 DPRINTFN(9, "port status=0x%08x\n", v); 3525 3526 i = UPS_PORT_LINK_STATE_SET(XHCI_PS_PLS_GET(v)); 3527 3528 switch (XHCI_PS_SPEED_GET(v)) { 3529 case 3: 3530 i |= UPS_HIGH_SPEED; 3531 break; 3532 case 2: 3533 i |= UPS_LOW_SPEED; 3534 break; 3535 case 1: 3536 /* FULL speed */ 3537 break; 3538 default: 3539 i |= UPS_OTHER_SPEED; 3540 break; 3541 } 3542 3543 if (v & XHCI_PS_CCS) 3544 i |= UPS_CURRENT_CONNECT_STATUS; 3545 if (v & XHCI_PS_PED) 3546 i |= UPS_PORT_ENABLED; 3547 if (v & XHCI_PS_OCA) 3548 i |= UPS_OVERCURRENT_INDICATOR; 3549 if (v & XHCI_PS_PR) 3550 i |= UPS_RESET; 3551 #if 0 3552 if (v & XHCI_PS_PP) 3553 /* XXX undefined */ 3554 #endif 3555 USETW(sc->sc_hub_desc.ps.wPortStatus, i); 3556 3557 i = 0; 3558 if (v & XHCI_PS_CSC) 3559 i |= UPS_C_CONNECT_STATUS; 3560 if (v & XHCI_PS_PEC) 3561 i |= UPS_C_PORT_ENABLED; 3562 if (v & XHCI_PS_OCC) 3563 i |= UPS_C_OVERCURRENT_INDICATOR; 3564 if (v & XHCI_PS_WRC) 3565 i |= UPS_C_BH_PORT_RESET; 3566 if (v & XHCI_PS_PRC) 3567 i |= UPS_C_PORT_RESET; 3568 if (v & XHCI_PS_PLC) 3569 i |= UPS_C_PORT_LINK_STATE; 3570 if (v & XHCI_PS_CEC) 3571 i |= UPS_C_PORT_CONFIG_ERROR; 3572 3573 USETW(sc->sc_hub_desc.ps.wPortChange, i); 3574 len = sizeof(sc->sc_hub_desc.ps); 3575 break; 3576 3577 case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE): 3578 err = USB_ERR_IOERROR; 3579 goto done; 3580 3581 case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE): 3582 break; 3583 3584 case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER): 3585 3586 i = index >> 8; 3587 index &= 0x00FF; 3588 3589 if ((index < 1) || 3590 (index > sc->sc_noport)) { 3591 err = USB_ERR_IOERROR; 3592 goto done; 3593 } 3594 3595 port = XHCI_PORTSC(index); 3596 v = XREAD4(sc, oper, port) & ~XHCI_PS_CLEAR; 3597 3598 switch (value) { 3599 case UHF_PORT_U1_TIMEOUT: 3600 if (XHCI_PS_SPEED_GET(v) != 4) { 3601 err = USB_ERR_IOERROR; 3602 goto done; 3603 } 3604 port = XHCI_PORTPMSC(index); 3605 v = XREAD4(sc, oper, port); 3606 v &= ~XHCI_PM3_U1TO_SET(0xFF); 3607 v |= XHCI_PM3_U1TO_SET(i); 3608 XWRITE4(sc, oper, port, v); 3609 break; 3610 case UHF_PORT_U2_TIMEOUT: 3611 if (XHCI_PS_SPEED_GET(v) != 4) { 3612 err = USB_ERR_IOERROR; 3613 goto done; 3614 } 3615 port = XHCI_PORTPMSC(index); 3616 v = XREAD4(sc, oper, port); 3617 v &= ~XHCI_PM3_U2TO_SET(0xFF); 3618 v |= XHCI_PM3_U2TO_SET(i); 3619 XWRITE4(sc, oper, port, v); 3620 break; 3621 case UHF_BH_PORT_RESET: 3622 XWRITE4(sc, oper, port, v | XHCI_PS_WPR); 3623 break; 3624 case UHF_PORT_LINK_STATE: 3625 XWRITE4(sc, oper, port, v | 3626 XHCI_PS_PLS_SET(i) | XHCI_PS_LWS); 3627 /* 4ms settle time */ 3628 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250); 3629 break; 3630 case UHF_PORT_ENABLE: 3631 DPRINTFN(3, "set port enable %d\n", index); 3632 break; 3633 case UHF_PORT_SUSPEND: 3634 DPRINTFN(6, "suspend port %u (LPM=%u)\n", index, i); 3635 j = XHCI_PS_SPEED_GET(v); 3636 if ((j < 1) || (j > 3)) { 3637 /* non-supported speed */ 3638 err = USB_ERR_IOERROR; 3639 goto done; 3640 } 3641 XWRITE4(sc, oper, port, v | 3642 XHCI_PS_PLS_SET(i ? 2 /* LPM */ : 3) | XHCI_PS_LWS); 3643 break; 3644 case UHF_PORT_RESET: 3645 DPRINTFN(6, "reset port %d\n", index); 3646 XWRITE4(sc, oper, port, v | XHCI_PS_PR); 3647 break; 3648 case UHF_PORT_POWER: 3649 DPRINTFN(3, "set port power %d\n", index); 3650 XWRITE4(sc, oper, port, v | XHCI_PS_PP); 3651 break; 3652 case UHF_PORT_TEST: 3653 DPRINTFN(3, "set port test %d\n", index); 3654 break; 3655 case UHF_PORT_INDICATOR: 3656 DPRINTFN(3, "set port indicator %d\n", index); 3657 3658 v &= ~XHCI_PS_PIC_SET(3); 3659 v |= XHCI_PS_PIC_SET(1); 3660 3661 XWRITE4(sc, oper, port, v); 3662 break; 3663 default: 3664 err = USB_ERR_IOERROR; 3665 goto done; 3666 } 3667 break; 3668 3669 case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER): 3670 case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER): 3671 case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER): 3672 case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER): 3673 break; 3674 default: 3675 err = USB_ERR_IOERROR; 3676 goto done; 3677 } 3678 done: 3679 *plength = len; 3680 *pptr = ptr; 3681 return (err); 3682 } 3683 3684 static void 3685 xhci_xfer_setup(struct usb_setup_params *parm) 3686 { 3687 struct usb_page_search page_info; 3688 struct usb_page_cache *pc; 3689 struct usb_xfer *xfer; 3690 void *last_obj; 3691 uint32_t ntd; 3692 uint32_t n; 3693 3694 xfer = parm->curr_xfer; 3695 3696 /* 3697 * The proof for the "ntd" formula is illustrated like this: 3698 * 3699 * +------------------------------------+ 3700 * | | 3701 * | |remainder -> | 3702 * | +-----+---+ | 3703 * | | xxx | x | frm 0 | 3704 * | +-----+---++ | 3705 * | | xxx | xx | frm 1 | 3706 * | +-----+----+ | 3707 * | ... | 3708 * +------------------------------------+ 3709 * 3710 * "xxx" means a completely full USB transfer descriptor 3711 * 3712 * "x" and "xx" means a short USB packet 3713 * 3714 * For the remainder of an USB transfer modulo 3715 * "max_data_length" we need two USB transfer descriptors. 3716 * One to transfer the remaining data and one to finalise with 3717 * a zero length packet in case the "force_short_xfer" flag is 3718 * set. We only need two USB transfer descriptors in the case 3719 * where the transfer length of the first one is a factor of 3720 * "max_frame_size". The rest of the needed USB transfer 3721 * descriptors is given by the buffer size divided by the 3722 * maximum data payload. 3723 */ 3724 parm->hc_max_packet_size = 0x400; 3725 parm->hc_max_packet_count = 16 * 3; 3726 parm->hc_max_frame_size = XHCI_TD_PAYLOAD_MAX; 3727 3728 xfer->flags_int.bdma_enable = 1; 3729 3730 usbd_transfer_setup_sub(parm); 3731 3732 if (xfer->flags_int.isochronous_xfr) { 3733 ntd = ((1 * xfer->nframes) 3734 + (xfer->max_data_length / xfer->max_hc_frame_size)); 3735 } else if (xfer->flags_int.control_xfr) { 3736 ntd = ((2 * xfer->nframes) + 1 /* STATUS */ 3737 + (xfer->max_data_length / xfer->max_hc_frame_size)); 3738 } else { 3739 ntd = ((2 * xfer->nframes) 3740 + (xfer->max_data_length / xfer->max_hc_frame_size)); 3741 } 3742 3743 alloc_dma_set: 3744 3745 if (parm->err) 3746 return; 3747 3748 /* 3749 * Allocate queue heads and transfer descriptors 3750 */ 3751 last_obj = NULL; 3752 3753 if (usbd_transfer_setup_sub_malloc( 3754 parm, &pc, sizeof(struct xhci_td), 3755 XHCI_TD_ALIGN, ntd)) { 3756 parm->err = USB_ERR_NOMEM; 3757 return; 3758 } 3759 if (parm->buf) { 3760 for (n = 0; n != ntd; n++) { 3761 struct xhci_td *td; 3762 3763 usbd_get_page(pc + n, 0, &page_info); 3764 3765 td = page_info.buffer; 3766 3767 /* init TD */ 3768 td->td_self = page_info.physaddr; 3769 td->obj_next = last_obj; 3770 td->page_cache = pc + n; 3771 3772 last_obj = td; 3773 3774 usb_pc_cpu_flush(pc + n); 3775 } 3776 } 3777 xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj; 3778 3779 if (!xfer->flags_int.curr_dma_set) { 3780 xfer->flags_int.curr_dma_set = 1; 3781 goto alloc_dma_set; 3782 } 3783 } 3784 3785 static uint8_t 3786 xhci_get_endpoint_state(struct usb_device *udev, uint8_t epno) 3787 { 3788 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 3789 struct usb_page_search buf_dev; 3790 struct xhci_hw_dev *hdev; 3791 struct xhci_dev_ctx *pdev; 3792 uint32_t temp; 3793 3794 MPASS(epno != 0); 3795 3796 hdev = &sc->sc_hw.devs[udev->controller_slot_id]; 3797 3798 usbd_get_page(&hdev->device_pc, 0, &buf_dev); 3799 pdev = buf_dev.buffer; 3800 usb_pc_cpu_invalidate(&hdev->device_pc); 3801 3802 temp = xhci_ctx_get_le32(sc, &pdev->ctx_ep[epno - 1].dwEpCtx0); 3803 3804 return (XHCI_EPCTX_0_EPSTATE_GET(temp)); 3805 } 3806 3807 static usb_error_t 3808 xhci_configure_reset_endpoint(struct usb_xfer *xfer) 3809 { 3810 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus); 3811 struct usb_page_search buf_inp; 3812 struct usb_device *udev; 3813 struct xhci_endpoint_ext *pepext; 3814 struct usb_endpoint_descriptor *edesc; 3815 struct usb_page_cache *pcinp; 3816 usb_error_t err; 3817 usb_stream_t stream_id; 3818 uint32_t mask; 3819 uint8_t index; 3820 uint8_t epno; 3821 3822 pepext = xhci_get_endpoint_ext(xfer->xroot->udev, 3823 xfer->endpoint->edesc); 3824 3825 udev = xfer->xroot->udev; 3826 index = udev->controller_slot_id; 3827 3828 pcinp = &sc->sc_hw.devs[index].input_pc; 3829 3830 usbd_get_page(pcinp, 0, &buf_inp); 3831 3832 edesc = xfer->endpoint->edesc; 3833 3834 epno = edesc->bEndpointAddress; 3835 stream_id = xfer->stream_id; 3836 3837 if ((edesc->bmAttributes & UE_XFERTYPE) == UE_CONTROL) 3838 epno |= UE_DIR_IN; 3839 3840 epno = XHCI_EPNO2EPID(epno); 3841 3842 if (epno == 0) 3843 return (USB_ERR_NO_PIPE); /* invalid */ 3844 3845 XHCI_CMD_LOCK(sc); 3846 3847 /* configure endpoint */ 3848 3849 err = xhci_configure_endpoint_by_xfer(xfer); 3850 3851 if (err != 0) { 3852 XHCI_CMD_UNLOCK(sc); 3853 return (err); 3854 } 3855 3856 /* 3857 * Get the endpoint into the stopped state according to the 3858 * endpoint context state diagram in the XHCI specification: 3859 */ 3860 switch (xhci_get_endpoint_state(udev, epno)) { 3861 case XHCI_EPCTX_0_EPSTATE_STOPPED: 3862 break; 3863 case XHCI_EPCTX_0_EPSTATE_HALTED: 3864 err = xhci_cmd_reset_ep(sc, 0, epno, index); 3865 if (err != 0) 3866 DPRINTF("Could not reset endpoint %u\n", epno); 3867 break; 3868 default: 3869 err = xhci_cmd_stop_ep(sc, 0, epno, index); 3870 if (err != 0) 3871 DPRINTF("Could not stop endpoint %u\n", epno); 3872 break; 3873 } 3874 3875 err = xhci_cmd_set_tr_dequeue_ptr(sc, 3876 (pepext->physaddr + (stream_id * sizeof(struct xhci_trb) * 3877 XHCI_MAX_TRANSFERS)) | XHCI_EPCTX_2_DCS_SET(1), 3878 stream_id, epno, index); 3879 3880 if (err != 0) 3881 DPRINTF("Could not set dequeue ptr for endpoint %u\n", epno); 3882 3883 /* 3884 * Get the endpoint into the running state according to the 3885 * endpoint context state diagram in the XHCI specification: 3886 */ 3887 3888 mask = (1U << epno); 3889 xhci_configure_mask(udev, mask | 1U, 0); 3890 3891 if (!(sc->sc_hw.devs[index].ep_configured & mask)) { 3892 sc->sc_hw.devs[index].ep_configured |= mask; 3893 err = xhci_cmd_configure_ep(sc, buf_inp.physaddr, 0, index); 3894 } else { 3895 err = xhci_cmd_evaluate_ctx(sc, buf_inp.physaddr, index); 3896 } 3897 3898 if (err != 0) { 3899 DPRINTF("Could not configure " 3900 "endpoint %u at slot %u.\n", epno, index); 3901 } 3902 XHCI_CMD_UNLOCK(sc); 3903 3904 return (0); 3905 } 3906 3907 static void 3908 xhci_xfer_unsetup(struct usb_xfer *xfer) 3909 { 3910 return; 3911 } 3912 3913 static void 3914 xhci_start_dma_delay(struct usb_xfer *xfer) 3915 { 3916 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus); 3917 3918 /* put transfer on interrupt queue (again) */ 3919 usbd_transfer_enqueue(&sc->sc_bus.intr_q, xfer); 3920 3921 (void)usb_proc_msignal(USB_BUS_CONTROL_XFER_PROC(&sc->sc_bus), 3922 &sc->sc_config_msg[0], &sc->sc_config_msg[1]); 3923 } 3924 3925 static void 3926 xhci_configure_msg(struct usb_proc_msg *pm) 3927 { 3928 struct xhci_softc *sc; 3929 struct xhci_endpoint_ext *pepext; 3930 struct usb_xfer *xfer; 3931 3932 sc = XHCI_BUS2SC(((struct usb_bus_msg *)pm)->bus); 3933 3934 restart: 3935 TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) { 3936 pepext = xhci_get_endpoint_ext(xfer->xroot->udev, 3937 xfer->endpoint->edesc); 3938 3939 if ((pepext->trb_halted != 0) || 3940 (pepext->trb_running == 0)) { 3941 uint16_t i; 3942 3943 /* clear halted and running */ 3944 pepext->trb_halted = 0; 3945 pepext->trb_running = 0; 3946 3947 /* nuke remaining buffered transfers */ 3948 3949 for (i = 0; i != (XHCI_MAX_TRANSFERS * 3950 XHCI_MAX_STREAMS); i++) { 3951 /* 3952 * NOTE: We need to use the timeout 3953 * error code here else existing 3954 * isochronous clients can get 3955 * confused: 3956 */ 3957 if (pepext->xfer[i] != NULL) { 3958 xhci_device_done(pepext->xfer[i], 3959 USB_ERR_TIMEOUT); 3960 } 3961 } 3962 3963 /* 3964 * NOTE: The USB transfer cannot vanish in 3965 * this state! 3966 */ 3967 3968 USB_BUS_UNLOCK(&sc->sc_bus); 3969 3970 xhci_configure_reset_endpoint(xfer); 3971 3972 USB_BUS_LOCK(&sc->sc_bus); 3973 3974 /* check if halted is still cleared */ 3975 if (pepext->trb_halted == 0) { 3976 pepext->trb_running = 1; 3977 memset(pepext->trb_index, 0, 3978 sizeof(pepext->trb_index)); 3979 } 3980 goto restart; 3981 } 3982 3983 if (xfer->flags_int.did_dma_delay) { 3984 /* remove transfer from interrupt queue (again) */ 3985 usbd_transfer_dequeue(xfer); 3986 3987 /* we are finally done */ 3988 usb_dma_delay_done_cb(xfer); 3989 3990 /* queue changed - restart */ 3991 goto restart; 3992 } 3993 } 3994 3995 TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) { 3996 /* try to insert xfer on HW queue */ 3997 xhci_transfer_insert(xfer); 3998 3999 /* try to multi buffer */ 4000 xhci_device_generic_multi_enter(xfer->endpoint, 4001 xfer->stream_id, NULL); 4002 } 4003 } 4004 4005 static void 4006 xhci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc, 4007 struct usb_endpoint *ep) 4008 { 4009 struct xhci_endpoint_ext *pepext; 4010 4011 DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d\n", 4012 ep, udev->address, edesc->bEndpointAddress, udev->flags.usb_mode); 4013 4014 if (udev->parent_hub == NULL) { 4015 /* root HUB has special endpoint handling */ 4016 return; 4017 } 4018 4019 ep->methods = &xhci_device_generic_methods; 4020 4021 pepext = xhci_get_endpoint_ext(udev, edesc); 4022 4023 USB_BUS_LOCK(udev->bus); 4024 pepext->trb_halted = 1; 4025 pepext->trb_running = 0; 4026 USB_BUS_UNLOCK(udev->bus); 4027 } 4028 4029 static void 4030 xhci_ep_uninit(struct usb_device *udev, struct usb_endpoint *ep) 4031 { 4032 4033 } 4034 4035 static void 4036 xhci_ep_clear_stall(struct usb_device *udev, struct usb_endpoint *ep) 4037 { 4038 struct xhci_endpoint_ext *pepext; 4039 4040 DPRINTF("\n"); 4041 4042 if (udev->flags.usb_mode != USB_MODE_HOST) { 4043 /* not supported */ 4044 return; 4045 } 4046 if (udev->parent_hub == NULL) { 4047 /* root HUB has special endpoint handling */ 4048 return; 4049 } 4050 4051 pepext = xhci_get_endpoint_ext(udev, ep->edesc); 4052 4053 USB_BUS_LOCK(udev->bus); 4054 pepext->trb_halted = 1; 4055 pepext->trb_running = 0; 4056 USB_BUS_UNLOCK(udev->bus); 4057 } 4058 4059 static usb_error_t 4060 xhci_device_init(struct usb_device *udev) 4061 { 4062 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 4063 usb_error_t err; 4064 uint8_t temp; 4065 4066 /* no init for root HUB */ 4067 if (udev->parent_hub == NULL) 4068 return (0); 4069 4070 XHCI_CMD_LOCK(sc); 4071 4072 /* set invalid default */ 4073 4074 udev->controller_slot_id = sc->sc_noslot + 1; 4075 4076 /* try to get a new slot ID from the XHCI */ 4077 4078 err = xhci_cmd_enable_slot(sc, &temp); 4079 4080 if (err) { 4081 XHCI_CMD_UNLOCK(sc); 4082 return (err); 4083 } 4084 4085 if (temp > sc->sc_noslot) { 4086 XHCI_CMD_UNLOCK(sc); 4087 return (USB_ERR_BAD_ADDRESS); 4088 } 4089 4090 if (sc->sc_hw.devs[temp].state != XHCI_ST_DISABLED) { 4091 DPRINTF("slot %u already allocated.\n", temp); 4092 XHCI_CMD_UNLOCK(sc); 4093 return (USB_ERR_BAD_ADDRESS); 4094 } 4095 4096 /* store slot ID for later reference */ 4097 4098 udev->controller_slot_id = temp; 4099 4100 /* reset data structure */ 4101 4102 memset(&sc->sc_hw.devs[temp], 0, sizeof(sc->sc_hw.devs[0])); 4103 4104 /* set mark slot allocated */ 4105 4106 sc->sc_hw.devs[temp].state = XHCI_ST_ENABLED; 4107 4108 err = xhci_alloc_device_ext(udev); 4109 4110 XHCI_CMD_UNLOCK(sc); 4111 4112 /* get device into default state */ 4113 4114 if (err == 0) 4115 err = xhci_set_address(udev, NULL, 0); 4116 4117 return (err); 4118 } 4119 4120 static void 4121 xhci_device_uninit(struct usb_device *udev) 4122 { 4123 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 4124 uint8_t index; 4125 4126 /* no init for root HUB */ 4127 if (udev->parent_hub == NULL) 4128 return; 4129 4130 XHCI_CMD_LOCK(sc); 4131 4132 index = udev->controller_slot_id; 4133 4134 if (index <= sc->sc_noslot) { 4135 xhci_cmd_disable_slot(sc, index); 4136 sc->sc_hw.devs[index].state = XHCI_ST_DISABLED; 4137 4138 /* free device extension */ 4139 xhci_free_device_ext(udev); 4140 } 4141 4142 XHCI_CMD_UNLOCK(sc); 4143 } 4144 4145 static void 4146 xhci_get_dma_delay(struct usb_device *udev, uint32_t *pus) 4147 { 4148 /* 4149 * Wait until the hardware has finished any possible use of 4150 * the transfer descriptor(s) 4151 */ 4152 *pus = 2048; /* microseconds */ 4153 } 4154 4155 static void 4156 xhci_device_resume(struct usb_device *udev) 4157 { 4158 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 4159 uint8_t index; 4160 uint8_t n; 4161 uint8_t p; 4162 4163 DPRINTF("\n"); 4164 4165 /* check for root HUB */ 4166 if (udev->parent_hub == NULL) 4167 return; 4168 4169 index = udev->controller_slot_id; 4170 4171 XHCI_CMD_LOCK(sc); 4172 4173 /* blindly resume all endpoints */ 4174 4175 USB_BUS_LOCK(udev->bus); 4176 4177 for (n = 1; n != XHCI_MAX_ENDPOINTS; n++) { 4178 for (p = 0; p != XHCI_MAX_STREAMS; p++) { 4179 XWRITE4(sc, door, XHCI_DOORBELL(index), 4180 n | XHCI_DB_SID_SET(p)); 4181 } 4182 } 4183 4184 USB_BUS_UNLOCK(udev->bus); 4185 4186 XHCI_CMD_UNLOCK(sc); 4187 } 4188 4189 static void 4190 xhci_device_suspend(struct usb_device *udev) 4191 { 4192 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 4193 uint8_t index; 4194 uint8_t n; 4195 usb_error_t err; 4196 4197 DPRINTF("\n"); 4198 4199 /* check for root HUB */ 4200 if (udev->parent_hub == NULL) 4201 return; 4202 4203 index = udev->controller_slot_id; 4204 4205 XHCI_CMD_LOCK(sc); 4206 4207 /* blindly suspend all endpoints */ 4208 4209 for (n = 1; n != XHCI_MAX_ENDPOINTS; n++) { 4210 err = xhci_cmd_stop_ep(sc, 1, n, index); 4211 if (err != 0) { 4212 DPRINTF("Failed to suspend endpoint " 4213 "%u on slot %u (ignored).\n", n, index); 4214 } 4215 } 4216 4217 XHCI_CMD_UNLOCK(sc); 4218 } 4219 4220 static void 4221 xhci_set_hw_power(struct usb_bus *bus) 4222 { 4223 DPRINTF("\n"); 4224 } 4225 4226 static void 4227 xhci_device_state_change(struct usb_device *udev) 4228 { 4229 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 4230 struct usb_page_search buf_inp; 4231 usb_error_t err; 4232 uint8_t index; 4233 4234 /* check for root HUB */ 4235 if (udev->parent_hub == NULL) 4236 return; 4237 4238 index = udev->controller_slot_id; 4239 4240 DPRINTF("\n"); 4241 4242 if (usb_get_device_state(udev) == USB_STATE_CONFIGURED) { 4243 err = uhub_query_info(udev, &sc->sc_hw.devs[index].nports, 4244 &sc->sc_hw.devs[index].tt); 4245 if (err != 0) 4246 sc->sc_hw.devs[index].nports = 0; 4247 } 4248 4249 XHCI_CMD_LOCK(sc); 4250 4251 switch (usb_get_device_state(udev)) { 4252 case USB_STATE_POWERED: 4253 if (sc->sc_hw.devs[index].state == XHCI_ST_DEFAULT) 4254 break; 4255 4256 /* set default state */ 4257 sc->sc_hw.devs[index].state = XHCI_ST_DEFAULT; 4258 sc->sc_hw.devs[index].ep_configured = 3U; 4259 4260 /* reset number of contexts */ 4261 sc->sc_hw.devs[index].context_num = 0; 4262 4263 err = xhci_cmd_reset_dev(sc, index); 4264 4265 if (err != 0) { 4266 DPRINTF("Device reset failed " 4267 "for slot %u.\n", index); 4268 } 4269 break; 4270 4271 case USB_STATE_ADDRESSED: 4272 if (sc->sc_hw.devs[index].state == XHCI_ST_ADDRESSED) 4273 break; 4274 4275 sc->sc_hw.devs[index].state = XHCI_ST_ADDRESSED; 4276 sc->sc_hw.devs[index].ep_configured = 3U; 4277 4278 /* set configure mask to slot only */ 4279 xhci_configure_mask(udev, 1, 0); 4280 4281 /* deconfigure all endpoints, except EP0 */ 4282 err = xhci_cmd_configure_ep(sc, 0, 1, index); 4283 4284 if (err) { 4285 DPRINTF("Failed to deconfigure " 4286 "slot %u.\n", index); 4287 } 4288 break; 4289 4290 case USB_STATE_CONFIGURED: 4291 if (sc->sc_hw.devs[index].state == XHCI_ST_CONFIGURED) { 4292 /* deconfigure all endpoints, except EP0 */ 4293 err = xhci_cmd_configure_ep(sc, 0, 1, index); 4294 4295 if (err) { 4296 DPRINTF("Failed to deconfigure " 4297 "slot %u.\n", index); 4298 } 4299 } 4300 4301 /* set configured state */ 4302 sc->sc_hw.devs[index].state = XHCI_ST_CONFIGURED; 4303 sc->sc_hw.devs[index].ep_configured = 3U; 4304 4305 /* reset number of contexts */ 4306 sc->sc_hw.devs[index].context_num = 0; 4307 4308 usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp); 4309 4310 xhci_configure_mask(udev, 3, 0); 4311 4312 err = xhci_configure_device(udev); 4313 if (err != 0) { 4314 DPRINTF("Could not configure device " 4315 "at slot %u.\n", index); 4316 } 4317 4318 err = xhci_cmd_evaluate_ctx(sc, buf_inp.physaddr, index); 4319 if (err != 0) { 4320 DPRINTF("Could not evaluate device " 4321 "context at slot %u.\n", index); 4322 } 4323 break; 4324 4325 default: 4326 break; 4327 } 4328 XHCI_CMD_UNLOCK(sc); 4329 } 4330 4331 static usb_error_t 4332 xhci_set_endpoint_mode(struct usb_device *udev, struct usb_endpoint *ep, 4333 uint8_t ep_mode) 4334 { 4335 switch (ep_mode) { 4336 case USB_EP_MODE_DEFAULT: 4337 return (0); 4338 case USB_EP_MODE_STREAMS: 4339 if (xhcistreams == 0 || 4340 (ep->edesc->bmAttributes & UE_XFERTYPE) != UE_BULK || 4341 udev->speed != USB_SPEED_SUPER) 4342 return (USB_ERR_INVAL); 4343 return (0); 4344 default: 4345 return (USB_ERR_INVAL); 4346 } 4347 } 4348 4349 static const struct usb_bus_methods xhci_bus_methods = { 4350 .endpoint_init = xhci_ep_init, 4351 .endpoint_uninit = xhci_ep_uninit, 4352 .xfer_setup = xhci_xfer_setup, 4353 .xfer_unsetup = xhci_xfer_unsetup, 4354 .get_dma_delay = xhci_get_dma_delay, 4355 .device_init = xhci_device_init, 4356 .device_uninit = xhci_device_uninit, 4357 .device_resume = xhci_device_resume, 4358 .device_suspend = xhci_device_suspend, 4359 .set_hw_power = xhci_set_hw_power, 4360 .roothub_exec = xhci_roothub_exec, 4361 .xfer_poll = xhci_do_poll, 4362 .start_dma_delay = xhci_start_dma_delay, 4363 .set_address = xhci_set_address, 4364 .clear_stall = xhci_ep_clear_stall, 4365 .device_state_change = xhci_device_state_change, 4366 .set_hw_power_sleep = xhci_set_hw_power_sleep, 4367 .set_endpoint_mode = xhci_set_endpoint_mode, 4368 }; 4369