1 /* $FreeBSD$ */ 2 /*- 3 * Copyright (c) 2010 Hans Petter Selasky. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 /* 28 * USB eXtensible Host Controller Interface, a.k.a. USB 3.0 controller. 29 * 30 * The XHCI 1.0 spec can be found at 31 * http://www.intel.com/technology/usb/download/xHCI_Specification_for_USB.pdf 32 * and the USB 3.0 spec at 33 * http://www.usb.org/developers/docs/usb_30_spec_060910.zip 34 */ 35 36 /* 37 * A few words about the design implementation: This driver emulates 38 * the concept about TDs which is found in EHCI specification. This 39 * way we achieve that the USB controller drivers look similar to 40 * eachother which makes it easier to understand the code. 41 */ 42 43 #ifdef USB_GLOBAL_INCLUDE_FILE 44 #include USB_GLOBAL_INCLUDE_FILE 45 #else 46 #include <sys/stdint.h> 47 #include <sys/stddef.h> 48 #include <sys/param.h> 49 #include <sys/queue.h> 50 #include <sys/types.h> 51 #include <sys/systm.h> 52 #include <sys/kernel.h> 53 #include <sys/bus.h> 54 #include <sys/module.h> 55 #include <sys/lock.h> 56 #include <sys/mutex.h> 57 #include <sys/condvar.h> 58 #include <sys/sysctl.h> 59 #include <sys/sx.h> 60 #include <sys/unistd.h> 61 #include <sys/callout.h> 62 #include <sys/malloc.h> 63 #include <sys/priv.h> 64 65 #include <dev/usb/usb.h> 66 #include <dev/usb/usbdi.h> 67 68 #define USB_DEBUG_VAR xhcidebug 69 70 #include <dev/usb/usb_core.h> 71 #include <dev/usb/usb_debug.h> 72 #include <dev/usb/usb_busdma.h> 73 #include <dev/usb/usb_process.h> 74 #include <dev/usb/usb_transfer.h> 75 #include <dev/usb/usb_device.h> 76 #include <dev/usb/usb_hub.h> 77 #include <dev/usb/usb_util.h> 78 79 #include <dev/usb/usb_controller.h> 80 #include <dev/usb/usb_bus.h> 81 #endif /* USB_GLOBAL_INCLUDE_FILE */ 82 83 #include <dev/usb/controller/xhci.h> 84 #include <dev/usb/controller/xhcireg.h> 85 86 #define XHCI_BUS2SC(bus) \ 87 ((struct xhci_softc *)(((uint8_t *)(bus)) - \ 88 ((uint8_t *)&(((struct xhci_softc *)0)->sc_bus)))) 89 90 #ifdef USB_DEBUG 91 static int xhcidebug; 92 static int xhciroute; 93 94 static SYSCTL_NODE(_hw_usb, OID_AUTO, xhci, CTLFLAG_RW, 0, "USB XHCI"); 95 SYSCTL_INT(_hw_usb_xhci, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_TUN, 96 &xhcidebug, 0, "Debug level"); 97 TUNABLE_INT("hw.usb.xhci.debug", &xhcidebug); 98 SYSCTL_INT(_hw_usb_xhci, OID_AUTO, xhci_port_route, CTLFLAG_RW | CTLFLAG_TUN, 99 &xhciroute, 0, "Routing bitmap for switching EHCI ports to XHCI controller"); 100 TUNABLE_INT("hw.usb.xhci.xhci_port_route", &xhciroute); 101 #endif 102 103 #define XHCI_INTR_ENDPT 1 104 105 struct xhci_std_temp { 106 struct xhci_softc *sc; 107 struct usb_page_cache *pc; 108 struct xhci_td *td; 109 struct xhci_td *td_next; 110 uint32_t len; 111 uint32_t offset; 112 uint32_t max_packet_size; 113 uint32_t average; 114 uint16_t isoc_delta; 115 uint16_t isoc_frame; 116 uint8_t shortpkt; 117 uint8_t multishort; 118 uint8_t last_frame; 119 uint8_t trb_type; 120 uint8_t direction; 121 uint8_t tbc; 122 uint8_t tlbpc; 123 uint8_t step_td; 124 uint8_t do_isoc_sync; 125 }; 126 127 static void xhci_do_poll(struct usb_bus *); 128 static void xhci_device_done(struct usb_xfer *, usb_error_t); 129 static void xhci_root_intr(struct xhci_softc *); 130 static void xhci_free_device_ext(struct usb_device *); 131 static struct xhci_endpoint_ext *xhci_get_endpoint_ext(struct usb_device *, 132 struct usb_endpoint_descriptor *); 133 static usb_proc_callback_t xhci_configure_msg; 134 static usb_error_t xhci_configure_device(struct usb_device *); 135 static usb_error_t xhci_configure_endpoint(struct usb_device *, 136 struct usb_endpoint_descriptor *, uint64_t, uint16_t, 137 uint8_t, uint8_t, uint8_t, uint16_t, uint16_t, uint8_t); 138 static usb_error_t xhci_configure_mask(struct usb_device *, 139 uint32_t, uint8_t); 140 static usb_error_t xhci_cmd_evaluate_ctx(struct xhci_softc *, 141 uint64_t, uint8_t); 142 static void xhci_endpoint_doorbell(struct usb_xfer *); 143 static void xhci_ctx_set_le32(struct xhci_softc *sc, volatile uint32_t *ptr, uint32_t val); 144 static uint32_t xhci_ctx_get_le32(struct xhci_softc *sc, volatile uint32_t *ptr); 145 static void xhci_ctx_set_le64(struct xhci_softc *sc, volatile uint64_t *ptr, uint64_t val); 146 #ifdef USB_DEBUG 147 static uint64_t xhci_ctx_get_le64(struct xhci_softc *sc, volatile uint64_t *ptr); 148 #endif 149 150 extern struct usb_bus_methods xhci_bus_methods; 151 152 #ifdef USB_DEBUG 153 static void 154 xhci_dump_trb(struct xhci_trb *trb) 155 { 156 DPRINTFN(5, "trb = %p\n", trb); 157 DPRINTFN(5, "qwTrb0 = 0x%016llx\n", (long long)le64toh(trb->qwTrb0)); 158 DPRINTFN(5, "dwTrb2 = 0x%08x\n", le32toh(trb->dwTrb2)); 159 DPRINTFN(5, "dwTrb3 = 0x%08x\n", le32toh(trb->dwTrb3)); 160 } 161 162 static void 163 xhci_dump_endpoint(struct xhci_softc *sc, struct xhci_endp_ctx *pep) 164 { 165 DPRINTFN(5, "pep = %p\n", pep); 166 DPRINTFN(5, "dwEpCtx0=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx0)); 167 DPRINTFN(5, "dwEpCtx1=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx1)); 168 DPRINTFN(5, "qwEpCtx2=0x%016llx\n", (long long)xhci_ctx_get_le64(sc, &pep->qwEpCtx2)); 169 DPRINTFN(5, "dwEpCtx4=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx4)); 170 DPRINTFN(5, "dwEpCtx5=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx5)); 171 DPRINTFN(5, "dwEpCtx6=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx6)); 172 DPRINTFN(5, "dwEpCtx7=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx7)); 173 } 174 175 static void 176 xhci_dump_device(struct xhci_softc *sc, struct xhci_slot_ctx *psl) 177 { 178 DPRINTFN(5, "psl = %p\n", psl); 179 DPRINTFN(5, "dwSctx0=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx0)); 180 DPRINTFN(5, "dwSctx1=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx1)); 181 DPRINTFN(5, "dwSctx2=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx2)); 182 DPRINTFN(5, "dwSctx3=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx3)); 183 } 184 #endif 185 186 uint32_t 187 xhci_get_port_route(void) 188 { 189 #ifdef USB_DEBUG 190 return (0xFFFFFFFFU ^ ((uint32_t)xhciroute)); 191 #else 192 return (0xFFFFFFFFU); 193 #endif 194 } 195 196 static void 197 xhci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb) 198 { 199 struct xhci_softc *sc = XHCI_BUS2SC(bus); 200 uint8_t i; 201 202 cb(bus, &sc->sc_hw.root_pc, &sc->sc_hw.root_pg, 203 sizeof(struct xhci_hw_root), XHCI_PAGE_SIZE); 204 205 cb(bus, &sc->sc_hw.ctx_pc, &sc->sc_hw.ctx_pg, 206 sizeof(struct xhci_dev_ctx_addr), XHCI_PAGE_SIZE); 207 208 for (i = 0; i != XHCI_MAX_SCRATCHPADS; i++) { 209 cb(bus, &sc->sc_hw.scratch_pc[i], &sc->sc_hw.scratch_pg[i], 210 XHCI_PAGE_SIZE, XHCI_PAGE_SIZE); 211 } 212 } 213 214 static void 215 xhci_ctx_set_le32(struct xhci_softc *sc, volatile uint32_t *ptr, uint32_t val) 216 { 217 if (sc->sc_ctx_is_64_byte) { 218 uint32_t offset; 219 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */ 220 /* all contexts are initially 32-bytes */ 221 offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U)); 222 ptr = (volatile uint32_t *)(((volatile uint8_t *)ptr) + offset); 223 } 224 *ptr = htole32(val); 225 } 226 227 static uint32_t 228 xhci_ctx_get_le32(struct xhci_softc *sc, volatile uint32_t *ptr) 229 { 230 if (sc->sc_ctx_is_64_byte) { 231 uint32_t offset; 232 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */ 233 /* all contexts are initially 32-bytes */ 234 offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U)); 235 ptr = (volatile uint32_t *)(((volatile uint8_t *)ptr) + offset); 236 } 237 return (le32toh(*ptr)); 238 } 239 240 static void 241 xhci_ctx_set_le64(struct xhci_softc *sc, volatile uint64_t *ptr, uint64_t val) 242 { 243 if (sc->sc_ctx_is_64_byte) { 244 uint32_t offset; 245 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */ 246 /* all contexts are initially 32-bytes */ 247 offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U)); 248 ptr = (volatile uint64_t *)(((volatile uint8_t *)ptr) + offset); 249 } 250 *ptr = htole64(val); 251 } 252 253 #ifdef USB_DEBUG 254 static uint64_t 255 xhci_ctx_get_le64(struct xhci_softc *sc, volatile uint64_t *ptr) 256 { 257 if (sc->sc_ctx_is_64_byte) { 258 uint32_t offset; 259 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */ 260 /* all contexts are initially 32-bytes */ 261 offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U)); 262 ptr = (volatile uint64_t *)(((volatile uint8_t *)ptr) + offset); 263 } 264 return (le64toh(*ptr)); 265 } 266 #endif 267 268 usb_error_t 269 xhci_start_controller(struct xhci_softc *sc) 270 { 271 struct usb_page_search buf_res; 272 struct xhci_hw_root *phwr; 273 struct xhci_dev_ctx_addr *pdctxa; 274 uint64_t addr; 275 uint32_t temp; 276 uint16_t i; 277 278 DPRINTF("\n"); 279 280 sc->sc_capa_off = 0; 281 sc->sc_oper_off = XREAD1(sc, capa, XHCI_CAPLENGTH); 282 sc->sc_runt_off = XREAD4(sc, capa, XHCI_RTSOFF) & ~0x1F; 283 sc->sc_door_off = XREAD4(sc, capa, XHCI_DBOFF) & ~0x3; 284 285 DPRINTF("CAPLENGTH=0x%x\n", sc->sc_oper_off); 286 DPRINTF("RUNTIMEOFFSET=0x%x\n", sc->sc_runt_off); 287 DPRINTF("DOOROFFSET=0x%x\n", sc->sc_door_off); 288 289 sc->sc_event_ccs = 1; 290 sc->sc_event_idx = 0; 291 sc->sc_command_ccs = 1; 292 sc->sc_command_idx = 0; 293 294 DPRINTF("xHCI version = 0x%04x\n", XREAD2(sc, capa, XHCI_HCIVERSION)); 295 296 temp = XREAD4(sc, capa, XHCI_HCSPARAMS0); 297 298 DPRINTF("HCS0 = 0x%08x\n", temp); 299 300 if (XHCI_HCS0_CSZ(temp)) { 301 sc->sc_ctx_is_64_byte = 1; 302 device_printf(sc->sc_bus.parent, "64 byte context size.\n"); 303 } else { 304 sc->sc_ctx_is_64_byte = 0; 305 device_printf(sc->sc_bus.parent, "32 byte context size.\n"); 306 } 307 308 /* Reset controller */ 309 XWRITE4(sc, oper, XHCI_USBCMD, XHCI_CMD_HCRST); 310 311 for (i = 0; i != 100; i++) { 312 usb_pause_mtx(NULL, hz / 100); 313 temp = XREAD4(sc, oper, XHCI_USBCMD) & 314 (XHCI_CMD_HCRST | XHCI_STS_CNR); 315 if (!temp) 316 break; 317 } 318 319 if (temp) { 320 device_printf(sc->sc_bus.parent, "Controller " 321 "reset timeout.\n"); 322 return (USB_ERR_IOERROR); 323 } 324 325 if (!(XREAD4(sc, oper, XHCI_PAGESIZE) & XHCI_PAGESIZE_4K)) { 326 device_printf(sc->sc_bus.parent, "Controller does " 327 "not support 4K page size.\n"); 328 return (USB_ERR_IOERROR); 329 } 330 331 temp = XREAD4(sc, capa, XHCI_HCSPARAMS1); 332 333 i = XHCI_HCS1_N_PORTS(temp); 334 335 if (i == 0) { 336 device_printf(sc->sc_bus.parent, "Invalid number " 337 "of ports: %u\n", i); 338 return (USB_ERR_IOERROR); 339 } 340 341 sc->sc_noport = i; 342 sc->sc_noslot = XHCI_HCS1_DEVSLOT_MAX(temp); 343 344 if (sc->sc_noslot > XHCI_MAX_DEVICES) 345 sc->sc_noslot = XHCI_MAX_DEVICES; 346 347 /* setup number of device slots */ 348 349 DPRINTF("CONFIG=0x%08x -> 0x%08x\n", 350 XREAD4(sc, oper, XHCI_CONFIG), sc->sc_noslot); 351 352 XWRITE4(sc, oper, XHCI_CONFIG, sc->sc_noslot); 353 354 DPRINTF("Max slots: %u\n", sc->sc_noslot); 355 356 temp = XREAD4(sc, capa, XHCI_HCSPARAMS2); 357 358 sc->sc_noscratch = XHCI_HCS2_SPB_MAX(temp); 359 360 if (sc->sc_noscratch > XHCI_MAX_SCRATCHPADS) { 361 device_printf(sc->sc_bus.parent, "XHCI request " 362 "too many scratchpads\n"); 363 return (USB_ERR_NOMEM); 364 } 365 366 DPRINTF("Max scratch: %u\n", sc->sc_noscratch); 367 368 temp = XREAD4(sc, capa, XHCI_HCSPARAMS3); 369 370 sc->sc_exit_lat_max = XHCI_HCS3_U1_DEL(temp) + 371 XHCI_HCS3_U2_DEL(temp) + 250 /* us */; 372 373 temp = XREAD4(sc, oper, XHCI_USBSTS); 374 375 /* clear interrupts */ 376 XWRITE4(sc, oper, XHCI_USBSTS, temp); 377 /* disable all device notifications */ 378 XWRITE4(sc, oper, XHCI_DNCTRL, 0); 379 380 /* setup device context base address */ 381 usbd_get_page(&sc->sc_hw.ctx_pc, 0, &buf_res); 382 pdctxa = buf_res.buffer; 383 memset(pdctxa, 0, sizeof(*pdctxa)); 384 385 addr = buf_res.physaddr; 386 addr += (uintptr_t)&((struct xhci_dev_ctx_addr *)0)->qwSpBufPtr[0]; 387 388 /* slot 0 points to the table of scratchpad pointers */ 389 pdctxa->qwBaaDevCtxAddr[0] = htole64(addr); 390 391 for (i = 0; i != sc->sc_noscratch; i++) { 392 struct usb_page_search buf_scp; 393 usbd_get_page(&sc->sc_hw.scratch_pc[i], 0, &buf_scp); 394 pdctxa->qwSpBufPtr[i] = htole64((uint64_t)buf_scp.physaddr); 395 } 396 397 addr = buf_res.physaddr; 398 399 XWRITE4(sc, oper, XHCI_DCBAAP_LO, (uint32_t)addr); 400 XWRITE4(sc, oper, XHCI_DCBAAP_HI, (uint32_t)(addr >> 32)); 401 XWRITE4(sc, oper, XHCI_DCBAAP_LO, (uint32_t)addr); 402 XWRITE4(sc, oper, XHCI_DCBAAP_HI, (uint32_t)(addr >> 32)); 403 404 /* Setup event table size */ 405 406 temp = XREAD4(sc, capa, XHCI_HCSPARAMS2); 407 408 DPRINTF("HCS2=0x%08x\n", temp); 409 410 temp = XHCI_HCS2_ERST_MAX(temp); 411 temp = 1U << temp; 412 if (temp > XHCI_MAX_RSEG) 413 temp = XHCI_MAX_RSEG; 414 415 sc->sc_erst_max = temp; 416 417 DPRINTF("ERSTSZ=0x%08x -> 0x%08x\n", 418 XREAD4(sc, runt, XHCI_ERSTSZ(0)), temp); 419 420 XWRITE4(sc, runt, XHCI_ERSTSZ(0), XHCI_ERSTS_SET(temp)); 421 422 /* Setup interrupt rate */ 423 XWRITE4(sc, runt, XHCI_IMOD(0), XHCI_IMOD_DEFAULT); 424 425 usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res); 426 427 phwr = buf_res.buffer; 428 addr = buf_res.physaddr; 429 addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_events[0]; 430 431 /* reset hardware root structure */ 432 memset(phwr, 0, sizeof(*phwr)); 433 434 phwr->hwr_ring_seg[0].qwEvrsTablePtr = htole64(addr); 435 phwr->hwr_ring_seg[0].dwEvrsTableSize = htole32(XHCI_MAX_EVENTS); 436 437 DPRINTF("ERDP(0)=0x%016llx\n", (unsigned long long)addr); 438 439 XWRITE4(sc, runt, XHCI_ERDP_LO(0), (uint32_t)addr); 440 XWRITE4(sc, runt, XHCI_ERDP_HI(0), (uint32_t)(addr >> 32)); 441 442 addr = (uint64_t)buf_res.physaddr; 443 444 DPRINTF("ERSTBA(0)=0x%016llx\n", (unsigned long long)addr); 445 446 XWRITE4(sc, runt, XHCI_ERSTBA_LO(0), (uint32_t)addr); 447 XWRITE4(sc, runt, XHCI_ERSTBA_HI(0), (uint32_t)(addr >> 32)); 448 449 /* Setup interrupter registers */ 450 451 temp = XREAD4(sc, runt, XHCI_IMAN(0)); 452 temp |= XHCI_IMAN_INTR_ENA; 453 XWRITE4(sc, runt, XHCI_IMAN(0), temp); 454 455 /* setup command ring control base address */ 456 addr = buf_res.physaddr; 457 addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_commands[0]; 458 459 DPRINTF("CRCR=0x%016llx\n", (unsigned long long)addr); 460 461 XWRITE4(sc, oper, XHCI_CRCR_LO, ((uint32_t)addr) | XHCI_CRCR_LO_RCS); 462 XWRITE4(sc, oper, XHCI_CRCR_HI, (uint32_t)(addr >> 32)); 463 464 phwr->hwr_commands[XHCI_MAX_COMMANDS - 1].qwTrb0 = htole64(addr); 465 466 usb_bus_mem_flush_all(&sc->sc_bus, &xhci_iterate_hw_softc); 467 468 /* Go! */ 469 XWRITE4(sc, oper, XHCI_USBCMD, XHCI_CMD_RS | 470 XHCI_CMD_INTE | XHCI_CMD_HSEE); 471 472 for (i = 0; i != 100; i++) { 473 usb_pause_mtx(NULL, hz / 100); 474 temp = XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_HCH; 475 if (!temp) 476 break; 477 } 478 if (temp) { 479 XWRITE4(sc, oper, XHCI_USBCMD, 0); 480 device_printf(sc->sc_bus.parent, "Run timeout.\n"); 481 return (USB_ERR_IOERROR); 482 } 483 484 /* catch any lost interrupts */ 485 xhci_do_poll(&sc->sc_bus); 486 487 return (0); 488 } 489 490 usb_error_t 491 xhci_halt_controller(struct xhci_softc *sc) 492 { 493 uint32_t temp; 494 uint16_t i; 495 496 DPRINTF("\n"); 497 498 sc->sc_capa_off = 0; 499 sc->sc_oper_off = XREAD1(sc, capa, XHCI_CAPLENGTH); 500 sc->sc_runt_off = XREAD4(sc, capa, XHCI_RTSOFF) & ~0xF; 501 sc->sc_door_off = XREAD4(sc, capa, XHCI_DBOFF) & ~0x3; 502 503 /* Halt controller */ 504 XWRITE4(sc, oper, XHCI_USBCMD, 0); 505 506 for (i = 0; i != 100; i++) { 507 usb_pause_mtx(NULL, hz / 100); 508 temp = XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_HCH; 509 if (temp) 510 break; 511 } 512 513 if (!temp) { 514 device_printf(sc->sc_bus.parent, "Controller halt timeout.\n"); 515 return (USB_ERR_IOERROR); 516 } 517 return (0); 518 } 519 520 usb_error_t 521 xhci_init(struct xhci_softc *sc, device_t self) 522 { 523 /* initialise some bus fields */ 524 sc->sc_bus.parent = self; 525 526 /* set the bus revision */ 527 sc->sc_bus.usbrev = USB_REV_3_0; 528 529 /* set up the bus struct */ 530 sc->sc_bus.methods = &xhci_bus_methods; 531 532 /* setup devices array */ 533 sc->sc_bus.devices = sc->sc_devices; 534 sc->sc_bus.devices_max = XHCI_MAX_DEVICES; 535 536 /* setup command queue mutex and condition varible */ 537 cv_init(&sc->sc_cmd_cv, "CMDQ"); 538 sx_init(&sc->sc_cmd_sx, "CMDQ lock"); 539 540 /* get all DMA memory */ 541 if (usb_bus_mem_alloc_all(&sc->sc_bus, 542 USB_GET_DMA_TAG(self), &xhci_iterate_hw_softc)) { 543 return (ENOMEM); 544 } 545 546 sc->sc_config_msg[0].hdr.pm_callback = &xhci_configure_msg; 547 sc->sc_config_msg[0].bus = &sc->sc_bus; 548 sc->sc_config_msg[1].hdr.pm_callback = &xhci_configure_msg; 549 sc->sc_config_msg[1].bus = &sc->sc_bus; 550 551 return (0); 552 } 553 554 void 555 xhci_uninit(struct xhci_softc *sc) 556 { 557 usb_bus_mem_free_all(&sc->sc_bus, &xhci_iterate_hw_softc); 558 559 cv_destroy(&sc->sc_cmd_cv); 560 sx_destroy(&sc->sc_cmd_sx); 561 } 562 563 static void 564 xhci_set_hw_power_sleep(struct usb_bus *bus, uint32_t state) 565 { 566 struct xhci_softc *sc = XHCI_BUS2SC(bus); 567 568 switch (state) { 569 case USB_HW_POWER_SUSPEND: 570 DPRINTF("Stopping the XHCI\n"); 571 xhci_halt_controller(sc); 572 break; 573 case USB_HW_POWER_SHUTDOWN: 574 DPRINTF("Stopping the XHCI\n"); 575 xhci_halt_controller(sc); 576 break; 577 case USB_HW_POWER_RESUME: 578 DPRINTF("Starting the XHCI\n"); 579 xhci_start_controller(sc); 580 break; 581 default: 582 break; 583 } 584 } 585 586 static usb_error_t 587 xhci_generic_done_sub(struct usb_xfer *xfer) 588 { 589 struct xhci_td *td; 590 struct xhci_td *td_alt_next; 591 uint32_t len; 592 uint8_t status; 593 594 td = xfer->td_transfer_cache; 595 td_alt_next = td->alt_next; 596 597 if (xfer->aframes != xfer->nframes) 598 usbd_xfer_set_frame_len(xfer, xfer->aframes, 0); 599 600 while (1) { 601 602 usb_pc_cpu_invalidate(td->page_cache); 603 604 status = td->status; 605 len = td->remainder; 606 607 DPRINTFN(4, "xfer=%p[%u/%u] rem=%u/%u status=%u\n", 608 xfer, (unsigned int)xfer->aframes, 609 (unsigned int)xfer->nframes, 610 (unsigned int)len, (unsigned int)td->len, 611 (unsigned int)status); 612 613 /* 614 * Verify the status length and 615 * add the length to "frlengths[]": 616 */ 617 if (len > td->len) { 618 /* should not happen */ 619 DPRINTF("Invalid status length, " 620 "0x%04x/0x%04x bytes\n", len, td->len); 621 status = XHCI_TRB_ERROR_LENGTH; 622 } else if (xfer->aframes != xfer->nframes) { 623 xfer->frlengths[xfer->aframes] += td->len - len; 624 } 625 /* Check for last transfer */ 626 if (((void *)td) == xfer->td_transfer_last) { 627 td = NULL; 628 break; 629 } 630 /* Check for transfer error */ 631 if (status != XHCI_TRB_ERROR_SHORT_PKT && 632 status != XHCI_TRB_ERROR_SUCCESS) { 633 /* the transfer is finished */ 634 td = NULL; 635 break; 636 } 637 /* Check for short transfer */ 638 if (len > 0) { 639 if (xfer->flags_int.short_frames_ok || 640 xfer->flags_int.isochronous_xfr || 641 xfer->flags_int.control_xfr) { 642 /* follow alt next */ 643 td = td->alt_next; 644 } else { 645 /* the transfer is finished */ 646 td = NULL; 647 } 648 break; 649 } 650 td = td->obj_next; 651 652 if (td->alt_next != td_alt_next) { 653 /* this USB frame is complete */ 654 break; 655 } 656 } 657 658 /* update transfer cache */ 659 660 xfer->td_transfer_cache = td; 661 662 return ((status == XHCI_TRB_ERROR_STALL) ? USB_ERR_STALLED : 663 (status != XHCI_TRB_ERROR_SHORT_PKT && 664 status != XHCI_TRB_ERROR_SUCCESS) ? USB_ERR_IOERROR : 665 USB_ERR_NORMAL_COMPLETION); 666 } 667 668 static void 669 xhci_generic_done(struct usb_xfer *xfer) 670 { 671 usb_error_t err = 0; 672 673 DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n", 674 xfer, xfer->endpoint); 675 676 /* reset scanner */ 677 678 xfer->td_transfer_cache = xfer->td_transfer_first; 679 680 if (xfer->flags_int.control_xfr) { 681 682 if (xfer->flags_int.control_hdr) 683 err = xhci_generic_done_sub(xfer); 684 685 xfer->aframes = 1; 686 687 if (xfer->td_transfer_cache == NULL) 688 goto done; 689 } 690 691 while (xfer->aframes != xfer->nframes) { 692 693 err = xhci_generic_done_sub(xfer); 694 xfer->aframes++; 695 696 if (xfer->td_transfer_cache == NULL) 697 goto done; 698 } 699 700 if (xfer->flags_int.control_xfr && 701 !xfer->flags_int.control_act) 702 err = xhci_generic_done_sub(xfer); 703 done: 704 /* transfer is complete */ 705 xhci_device_done(xfer, err); 706 } 707 708 static void 709 xhci_activate_transfer(struct usb_xfer *xfer) 710 { 711 struct xhci_td *td; 712 713 td = xfer->td_transfer_cache; 714 715 usb_pc_cpu_invalidate(td->page_cache); 716 717 if (!(td->td_trb[0].dwTrb3 & htole32(XHCI_TRB_3_CYCLE_BIT))) { 718 719 /* activate the transfer */ 720 721 td->td_trb[0].dwTrb3 |= htole32(XHCI_TRB_3_CYCLE_BIT); 722 usb_pc_cpu_flush(td->page_cache); 723 724 xhci_endpoint_doorbell(xfer); 725 } 726 } 727 728 static void 729 xhci_skip_transfer(struct usb_xfer *xfer) 730 { 731 struct xhci_td *td; 732 struct xhci_td *td_last; 733 734 td = xfer->td_transfer_cache; 735 td_last = xfer->td_transfer_last; 736 737 td = td->alt_next; 738 739 usb_pc_cpu_invalidate(td->page_cache); 740 741 if (!(td->td_trb[0].dwTrb3 & htole32(XHCI_TRB_3_CYCLE_BIT))) { 742 743 usb_pc_cpu_invalidate(td_last->page_cache); 744 745 /* copy LINK TRB to current waiting location */ 746 747 td->td_trb[0].qwTrb0 = td_last->td_trb[td_last->ntrb].qwTrb0; 748 td->td_trb[0].dwTrb2 = td_last->td_trb[td_last->ntrb].dwTrb2; 749 usb_pc_cpu_flush(td->page_cache); 750 751 td->td_trb[0].dwTrb3 = td_last->td_trb[td_last->ntrb].dwTrb3; 752 usb_pc_cpu_flush(td->page_cache); 753 754 xhci_endpoint_doorbell(xfer); 755 } 756 } 757 758 /*------------------------------------------------------------------------* 759 * xhci_check_transfer 760 *------------------------------------------------------------------------*/ 761 static void 762 xhci_check_transfer(struct xhci_softc *sc, struct xhci_trb *trb) 763 { 764 int64_t offset; 765 uint64_t td_event; 766 uint32_t temp; 767 uint32_t remainder; 768 uint8_t status; 769 uint8_t halted; 770 uint8_t epno; 771 uint8_t index; 772 uint8_t i; 773 774 /* decode TRB */ 775 td_event = le64toh(trb->qwTrb0); 776 temp = le32toh(trb->dwTrb2); 777 778 remainder = XHCI_TRB_2_REM_GET(temp); 779 status = XHCI_TRB_2_ERROR_GET(temp); 780 781 temp = le32toh(trb->dwTrb3); 782 epno = XHCI_TRB_3_EP_GET(temp); 783 index = XHCI_TRB_3_SLOT_GET(temp); 784 785 /* check if error means halted */ 786 halted = (status != XHCI_TRB_ERROR_SHORT_PKT && 787 status != XHCI_TRB_ERROR_SUCCESS); 788 789 DPRINTF("slot=%u epno=%u remainder=%u status=%u\n", 790 index, epno, remainder, status); 791 792 if (index > sc->sc_noslot) { 793 DPRINTF("Invalid slot.\n"); 794 return; 795 } 796 797 if ((epno == 0) || (epno >= XHCI_MAX_ENDPOINTS)) { 798 DPRINTF("Invalid endpoint.\n"); 799 return; 800 } 801 802 /* try to find the USB transfer that generated the event */ 803 for (i = 0; i != (XHCI_MAX_TRANSFERS - 1); i++) { 804 struct usb_xfer *xfer; 805 struct xhci_td *td; 806 struct xhci_endpoint_ext *pepext; 807 808 pepext = &sc->sc_hw.devs[index].endp[epno]; 809 810 xfer = pepext->xfer[i]; 811 if (xfer == NULL) 812 continue; 813 814 td = xfer->td_transfer_cache; 815 816 DPRINTFN(5, "Checking if 0x%016llx == (0x%016llx .. 0x%016llx)\n", 817 (long long)td_event, 818 (long long)td->td_self, 819 (long long)td->td_self + sizeof(td->td_trb)); 820 821 /* 822 * NOTE: Some XHCI implementations might not trigger 823 * an event on the last LINK TRB so we need to 824 * consider both the last and second last event 825 * address as conditions for a successful transfer. 826 * 827 * NOTE: We assume that the XHCI will only trigger one 828 * event per chain of TRBs. 829 */ 830 831 offset = td_event - td->td_self; 832 833 if (offset >= 0 && 834 offset < (int64_t)sizeof(td->td_trb)) { 835 836 usb_pc_cpu_invalidate(td->page_cache); 837 838 /* compute rest of remainder, if any */ 839 for (i = (offset / 16) + 1; i < td->ntrb; i++) { 840 temp = le32toh(td->td_trb[i].dwTrb2); 841 remainder += XHCI_TRB_2_BYTES_GET(temp); 842 } 843 844 DPRINTFN(5, "New remainder: %u\n", remainder); 845 846 /* clear isochronous transfer errors */ 847 if (xfer->flags_int.isochronous_xfr) { 848 if (halted) { 849 halted = 0; 850 status = XHCI_TRB_ERROR_SUCCESS; 851 remainder = td->len; 852 } 853 } 854 855 /* "td->remainder" is verified later */ 856 td->remainder = remainder; 857 td->status = status; 858 859 usb_pc_cpu_flush(td->page_cache); 860 861 /* 862 * 1) Last transfer descriptor makes the 863 * transfer done 864 */ 865 if (((void *)td) == xfer->td_transfer_last) { 866 DPRINTF("TD is last\n"); 867 xhci_generic_done(xfer); 868 break; 869 } 870 871 /* 872 * 2) Any kind of error makes the transfer 873 * done 874 */ 875 if (halted) { 876 DPRINTF("TD has I/O error\n"); 877 xhci_generic_done(xfer); 878 break; 879 } 880 881 /* 882 * 3) If there is no alternate next transfer, 883 * a short packet also makes the transfer done 884 */ 885 if (td->remainder > 0) { 886 if (td->alt_next == NULL) { 887 DPRINTF( 888 "short TD has no alternate next\n"); 889 xhci_generic_done(xfer); 890 break; 891 } 892 DPRINTF("TD has short pkt\n"); 893 if (xfer->flags_int.short_frames_ok || 894 xfer->flags_int.isochronous_xfr || 895 xfer->flags_int.control_xfr) { 896 /* follow the alt next */ 897 xfer->td_transfer_cache = td->alt_next; 898 xhci_activate_transfer(xfer); 899 break; 900 } 901 xhci_skip_transfer(xfer); 902 xhci_generic_done(xfer); 903 break; 904 } 905 906 /* 907 * 4) Transfer complete - go to next TD 908 */ 909 DPRINTF("Following next TD\n"); 910 xfer->td_transfer_cache = td->obj_next; 911 xhci_activate_transfer(xfer); 912 break; /* there should only be one match */ 913 } 914 } 915 } 916 917 static void 918 xhci_check_command(struct xhci_softc *sc, struct xhci_trb *trb) 919 { 920 if (sc->sc_cmd_addr == trb->qwTrb0) { 921 DPRINTF("Received command event\n"); 922 sc->sc_cmd_result[0] = trb->dwTrb2; 923 sc->sc_cmd_result[1] = trb->dwTrb3; 924 cv_signal(&sc->sc_cmd_cv); 925 } 926 } 927 928 static void 929 xhci_interrupt_poll(struct xhci_softc *sc) 930 { 931 struct usb_page_search buf_res; 932 struct xhci_hw_root *phwr; 933 uint64_t addr; 934 uint32_t temp; 935 uint16_t i; 936 uint8_t event; 937 uint8_t j; 938 uint8_t k; 939 uint8_t t; 940 941 usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res); 942 943 phwr = buf_res.buffer; 944 945 /* Receive any events */ 946 947 usb_pc_cpu_invalidate(&sc->sc_hw.root_pc); 948 949 i = sc->sc_event_idx; 950 j = sc->sc_event_ccs; 951 t = 2; 952 953 while (1) { 954 955 temp = le32toh(phwr->hwr_events[i].dwTrb3); 956 957 k = (temp & XHCI_TRB_3_CYCLE_BIT) ? 1 : 0; 958 959 if (j != k) 960 break; 961 962 event = XHCI_TRB_3_TYPE_GET(temp); 963 964 DPRINTFN(10, "event[%u] = %u (0x%016llx 0x%08lx 0x%08lx)\n", 965 i, event, (long long)le64toh(phwr->hwr_events[i].qwTrb0), 966 (long)le32toh(phwr->hwr_events[i].dwTrb2), 967 (long)le32toh(phwr->hwr_events[i].dwTrb3)); 968 969 switch (event) { 970 case XHCI_TRB_EVENT_TRANSFER: 971 xhci_check_transfer(sc, &phwr->hwr_events[i]); 972 break; 973 case XHCI_TRB_EVENT_CMD_COMPLETE: 974 xhci_check_command(sc, &phwr->hwr_events[i]); 975 break; 976 default: 977 DPRINTF("Unhandled event = %u\n", event); 978 break; 979 } 980 981 i++; 982 983 if (i == XHCI_MAX_EVENTS) { 984 i = 0; 985 j ^= 1; 986 987 /* check for timeout */ 988 if (!--t) 989 break; 990 } 991 } 992 993 sc->sc_event_idx = i; 994 sc->sc_event_ccs = j; 995 996 /* 997 * NOTE: The Event Ring Dequeue Pointer Register is 64-bit 998 * latched. That means to activate the register we need to 999 * write both the low and high double word of the 64-bit 1000 * register. 1001 */ 1002 1003 addr = (uint32_t)buf_res.physaddr; 1004 addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_events[i]; 1005 1006 /* try to clear busy bit */ 1007 addr |= XHCI_ERDP_LO_BUSY; 1008 1009 XWRITE4(sc, runt, XHCI_ERDP_LO(0), (uint32_t)addr); 1010 XWRITE4(sc, runt, XHCI_ERDP_HI(0), (uint32_t)(addr >> 32)); 1011 } 1012 1013 static usb_error_t 1014 xhci_do_command(struct xhci_softc *sc, struct xhci_trb *trb, 1015 uint16_t timeout_ms) 1016 { 1017 struct usb_page_search buf_res; 1018 struct xhci_hw_root *phwr; 1019 uint64_t addr; 1020 uint32_t temp; 1021 uint8_t i; 1022 uint8_t j; 1023 int err; 1024 1025 XHCI_CMD_ASSERT_LOCKED(sc); 1026 1027 /* get hardware root structure */ 1028 1029 usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res); 1030 1031 phwr = buf_res.buffer; 1032 1033 /* Queue command */ 1034 1035 USB_BUS_LOCK(&sc->sc_bus); 1036 1037 i = sc->sc_command_idx; 1038 j = sc->sc_command_ccs; 1039 1040 DPRINTFN(10, "command[%u] = %u (0x%016llx, 0x%08lx, 0x%08lx)\n", 1041 i, XHCI_TRB_3_TYPE_GET(le32toh(trb->dwTrb3)), 1042 (long long)le64toh(trb->qwTrb0), 1043 (long)le32toh(trb->dwTrb2), 1044 (long)le32toh(trb->dwTrb3)); 1045 1046 phwr->hwr_commands[i].qwTrb0 = trb->qwTrb0; 1047 phwr->hwr_commands[i].dwTrb2 = trb->dwTrb2; 1048 1049 usb_pc_cpu_flush(&sc->sc_hw.root_pc); 1050 1051 temp = trb->dwTrb3; 1052 1053 if (j) 1054 temp |= htole32(XHCI_TRB_3_CYCLE_BIT); 1055 else 1056 temp &= ~htole32(XHCI_TRB_3_CYCLE_BIT); 1057 1058 temp &= ~htole32(XHCI_TRB_3_TC_BIT); 1059 1060 phwr->hwr_commands[i].dwTrb3 = temp; 1061 1062 usb_pc_cpu_flush(&sc->sc_hw.root_pc); 1063 1064 addr = buf_res.physaddr; 1065 addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_commands[i]; 1066 1067 sc->sc_cmd_addr = htole64(addr); 1068 1069 i++; 1070 1071 if (i == (XHCI_MAX_COMMANDS - 1)) { 1072 1073 if (j) { 1074 temp = htole32(XHCI_TRB_3_TC_BIT | 1075 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) | 1076 XHCI_TRB_3_CYCLE_BIT); 1077 } else { 1078 temp = htole32(XHCI_TRB_3_TC_BIT | 1079 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK)); 1080 } 1081 1082 phwr->hwr_commands[i].dwTrb3 = temp; 1083 1084 usb_pc_cpu_flush(&sc->sc_hw.root_pc); 1085 1086 i = 0; 1087 j ^= 1; 1088 } 1089 1090 sc->sc_command_idx = i; 1091 sc->sc_command_ccs = j; 1092 1093 XWRITE4(sc, door, XHCI_DOORBELL(0), 0); 1094 1095 err = cv_timedwait(&sc->sc_cmd_cv, &sc->sc_bus.bus_mtx, 1096 USB_MS_TO_TICKS(timeout_ms)); 1097 1098 if (err) { 1099 DPRINTFN(0, "Command timeout!\n"); 1100 err = USB_ERR_TIMEOUT; 1101 trb->dwTrb2 = 0; 1102 trb->dwTrb3 = 0; 1103 } else { 1104 temp = le32toh(sc->sc_cmd_result[0]); 1105 if (XHCI_TRB_2_ERROR_GET(temp) != XHCI_TRB_ERROR_SUCCESS) 1106 err = USB_ERR_IOERROR; 1107 1108 trb->dwTrb2 = sc->sc_cmd_result[0]; 1109 trb->dwTrb3 = sc->sc_cmd_result[1]; 1110 } 1111 1112 USB_BUS_UNLOCK(&sc->sc_bus); 1113 1114 return (err); 1115 } 1116 1117 #if 0 1118 static usb_error_t 1119 xhci_cmd_nop(struct xhci_softc *sc) 1120 { 1121 struct xhci_trb trb; 1122 uint32_t temp; 1123 1124 DPRINTF("\n"); 1125 1126 trb.qwTrb0 = 0; 1127 trb.dwTrb2 = 0; 1128 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NOOP); 1129 1130 trb.dwTrb3 = htole32(temp); 1131 1132 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1133 } 1134 #endif 1135 1136 static usb_error_t 1137 xhci_cmd_enable_slot(struct xhci_softc *sc, uint8_t *pslot) 1138 { 1139 struct xhci_trb trb; 1140 uint32_t temp; 1141 usb_error_t err; 1142 1143 DPRINTF("\n"); 1144 1145 trb.qwTrb0 = 0; 1146 trb.dwTrb2 = 0; 1147 trb.dwTrb3 = htole32(XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ENABLE_SLOT)); 1148 1149 err = xhci_do_command(sc, &trb, 100 /* ms */); 1150 if (err) 1151 goto done; 1152 1153 temp = le32toh(trb.dwTrb3); 1154 1155 *pslot = XHCI_TRB_3_SLOT_GET(temp); 1156 1157 done: 1158 return (err); 1159 } 1160 1161 static usb_error_t 1162 xhci_cmd_disable_slot(struct xhci_softc *sc, uint8_t slot_id) 1163 { 1164 struct xhci_trb trb; 1165 uint32_t temp; 1166 1167 DPRINTF("\n"); 1168 1169 trb.qwTrb0 = 0; 1170 trb.dwTrb2 = 0; 1171 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DISABLE_SLOT) | 1172 XHCI_TRB_3_SLOT_SET(slot_id); 1173 1174 trb.dwTrb3 = htole32(temp); 1175 1176 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1177 } 1178 1179 static usb_error_t 1180 xhci_cmd_set_address(struct xhci_softc *sc, uint64_t input_ctx, 1181 uint8_t bsr, uint8_t slot_id) 1182 { 1183 struct xhci_trb trb; 1184 uint32_t temp; 1185 1186 DPRINTF("\n"); 1187 1188 trb.qwTrb0 = htole64(input_ctx); 1189 trb.dwTrb2 = 0; 1190 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ADDRESS_DEVICE) | 1191 XHCI_TRB_3_SLOT_SET(slot_id); 1192 1193 if (bsr) 1194 temp |= XHCI_TRB_3_BSR_BIT; 1195 1196 trb.dwTrb3 = htole32(temp); 1197 1198 return (xhci_do_command(sc, &trb, 500 /* ms */)); 1199 } 1200 1201 static usb_error_t 1202 xhci_set_address(struct usb_device *udev, struct mtx *mtx, uint16_t address) 1203 { 1204 struct usb_page_search buf_inp; 1205 struct usb_page_search buf_dev; 1206 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 1207 struct xhci_hw_dev *hdev; 1208 struct xhci_dev_ctx *pdev; 1209 struct xhci_endpoint_ext *pepext; 1210 uint32_t temp; 1211 uint16_t mps; 1212 usb_error_t err; 1213 uint8_t index; 1214 1215 /* the root HUB case is not handled here */ 1216 if (udev->parent_hub == NULL) 1217 return (USB_ERR_INVAL); 1218 1219 index = udev->controller_slot_id; 1220 1221 hdev = &sc->sc_hw.devs[index]; 1222 1223 if (mtx != NULL) 1224 mtx_unlock(mtx); 1225 1226 XHCI_CMD_LOCK(sc); 1227 1228 switch (hdev->state) { 1229 case XHCI_ST_DEFAULT: 1230 case XHCI_ST_ENABLED: 1231 1232 hdev->state = XHCI_ST_ENABLED; 1233 1234 /* set configure mask to slot and EP0 */ 1235 xhci_configure_mask(udev, 3, 0); 1236 1237 /* configure input slot context structure */ 1238 err = xhci_configure_device(udev); 1239 1240 if (err != 0) { 1241 DPRINTF("Could not configure device\n"); 1242 break; 1243 } 1244 1245 /* configure input endpoint context structure */ 1246 switch (udev->speed) { 1247 case USB_SPEED_LOW: 1248 case USB_SPEED_FULL: 1249 mps = 8; 1250 break; 1251 case USB_SPEED_HIGH: 1252 mps = 64; 1253 break; 1254 default: 1255 mps = 512; 1256 break; 1257 } 1258 1259 pepext = xhci_get_endpoint_ext(udev, 1260 &udev->ctrl_ep_desc); 1261 err = xhci_configure_endpoint(udev, 1262 &udev->ctrl_ep_desc, pepext->physaddr, 1263 0, 1, 1, 0, mps, mps, USB_EP_MODE_DEFAULT); 1264 1265 if (err != 0) { 1266 DPRINTF("Could not configure default endpoint\n"); 1267 break; 1268 } 1269 1270 /* execute set address command */ 1271 usbd_get_page(&hdev->input_pc, 0, &buf_inp); 1272 1273 err = xhci_cmd_set_address(sc, buf_inp.physaddr, 1274 (address == 0), index); 1275 1276 if (err != 0) { 1277 DPRINTF("Could not set address " 1278 "for slot %u.\n", index); 1279 if (address != 0) 1280 break; 1281 } 1282 1283 /* update device address to new value */ 1284 1285 usbd_get_page(&hdev->device_pc, 0, &buf_dev); 1286 pdev = buf_dev.buffer; 1287 usb_pc_cpu_invalidate(&hdev->device_pc); 1288 1289 temp = xhci_ctx_get_le32(sc, &pdev->ctx_slot.dwSctx3); 1290 udev->address = XHCI_SCTX_3_DEV_ADDR_GET(temp); 1291 1292 /* update device state to new value */ 1293 1294 if (address != 0) 1295 hdev->state = XHCI_ST_ADDRESSED; 1296 else 1297 hdev->state = XHCI_ST_DEFAULT; 1298 break; 1299 1300 default: 1301 DPRINTF("Wrong state for set address.\n"); 1302 err = USB_ERR_IOERROR; 1303 break; 1304 } 1305 XHCI_CMD_UNLOCK(sc); 1306 1307 if (mtx != NULL) 1308 mtx_lock(mtx); 1309 1310 return (err); 1311 } 1312 1313 static usb_error_t 1314 xhci_cmd_configure_ep(struct xhci_softc *sc, uint64_t input_ctx, 1315 uint8_t deconfigure, uint8_t slot_id) 1316 { 1317 struct xhci_trb trb; 1318 uint32_t temp; 1319 1320 DPRINTF("\n"); 1321 1322 trb.qwTrb0 = htole64(input_ctx); 1323 trb.dwTrb2 = 0; 1324 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP) | 1325 XHCI_TRB_3_SLOT_SET(slot_id); 1326 1327 if (deconfigure) 1328 temp |= XHCI_TRB_3_DCEP_BIT; 1329 1330 trb.dwTrb3 = htole32(temp); 1331 1332 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1333 } 1334 1335 static usb_error_t 1336 xhci_cmd_evaluate_ctx(struct xhci_softc *sc, uint64_t input_ctx, 1337 uint8_t slot_id) 1338 { 1339 struct xhci_trb trb; 1340 uint32_t temp; 1341 1342 DPRINTF("\n"); 1343 1344 trb.qwTrb0 = htole64(input_ctx); 1345 trb.dwTrb2 = 0; 1346 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVALUATE_CTX) | 1347 XHCI_TRB_3_SLOT_SET(slot_id); 1348 trb.dwTrb3 = htole32(temp); 1349 1350 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1351 } 1352 1353 static usb_error_t 1354 xhci_cmd_reset_ep(struct xhci_softc *sc, uint8_t preserve, 1355 uint8_t ep_id, uint8_t slot_id) 1356 { 1357 struct xhci_trb trb; 1358 uint32_t temp; 1359 1360 DPRINTF("\n"); 1361 1362 trb.qwTrb0 = 0; 1363 trb.dwTrb2 = 0; 1364 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_EP) | 1365 XHCI_TRB_3_SLOT_SET(slot_id) | 1366 XHCI_TRB_3_EP_SET(ep_id); 1367 1368 if (preserve) 1369 temp |= XHCI_TRB_3_PRSV_BIT; 1370 1371 trb.dwTrb3 = htole32(temp); 1372 1373 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1374 } 1375 1376 static usb_error_t 1377 xhci_cmd_set_tr_dequeue_ptr(struct xhci_softc *sc, uint64_t dequeue_ptr, 1378 uint16_t stream_id, uint8_t ep_id, uint8_t slot_id) 1379 { 1380 struct xhci_trb trb; 1381 uint32_t temp; 1382 1383 DPRINTF("\n"); 1384 1385 trb.qwTrb0 = htole64(dequeue_ptr); 1386 1387 temp = XHCI_TRB_2_STREAM_SET(stream_id); 1388 trb.dwTrb2 = htole32(temp); 1389 1390 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SET_TR_DEQUEUE) | 1391 XHCI_TRB_3_SLOT_SET(slot_id) | 1392 XHCI_TRB_3_EP_SET(ep_id); 1393 trb.dwTrb3 = htole32(temp); 1394 1395 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1396 } 1397 1398 static usb_error_t 1399 xhci_cmd_stop_ep(struct xhci_softc *sc, uint8_t suspend, 1400 uint8_t ep_id, uint8_t slot_id) 1401 { 1402 struct xhci_trb trb; 1403 uint32_t temp; 1404 1405 DPRINTF("\n"); 1406 1407 trb.qwTrb0 = 0; 1408 trb.dwTrb2 = 0; 1409 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STOP_EP) | 1410 XHCI_TRB_3_SLOT_SET(slot_id) | 1411 XHCI_TRB_3_EP_SET(ep_id); 1412 1413 if (suspend) 1414 temp |= XHCI_TRB_3_SUSP_EP_BIT; 1415 1416 trb.dwTrb3 = htole32(temp); 1417 1418 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1419 } 1420 1421 static usb_error_t 1422 xhci_cmd_reset_dev(struct xhci_softc *sc, uint8_t slot_id) 1423 { 1424 struct xhci_trb trb; 1425 uint32_t temp; 1426 1427 DPRINTF("\n"); 1428 1429 trb.qwTrb0 = 0; 1430 trb.dwTrb2 = 0; 1431 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_DEVICE) | 1432 XHCI_TRB_3_SLOT_SET(slot_id); 1433 1434 trb.dwTrb3 = htole32(temp); 1435 1436 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1437 } 1438 1439 /*------------------------------------------------------------------------* 1440 * xhci_interrupt - XHCI interrupt handler 1441 *------------------------------------------------------------------------*/ 1442 void 1443 xhci_interrupt(struct xhci_softc *sc) 1444 { 1445 uint32_t status; 1446 uint32_t iman; 1447 1448 USB_BUS_LOCK(&sc->sc_bus); 1449 1450 status = XREAD4(sc, oper, XHCI_USBSTS); 1451 if (status == 0) 1452 goto done; 1453 1454 /* acknowledge interrupts */ 1455 1456 XWRITE4(sc, oper, XHCI_USBSTS, status); 1457 1458 DPRINTFN(16, "real interrupt (status=0x%08x)\n", status); 1459 1460 if (status & XHCI_STS_EINT) { 1461 1462 /* acknowledge pending event */ 1463 iman = XREAD4(sc, runt, XHCI_IMAN(0)); 1464 1465 /* reset interrupt */ 1466 XWRITE4(sc, runt, XHCI_IMAN(0), iman); 1467 1468 DPRINTFN(16, "real interrupt (iman=0x%08x)\n", iman); 1469 1470 /* check for event(s) */ 1471 xhci_interrupt_poll(sc); 1472 } 1473 1474 if (status & (XHCI_STS_PCD | XHCI_STS_HCH | 1475 XHCI_STS_HSE | XHCI_STS_HCE)) { 1476 1477 if (status & XHCI_STS_PCD) { 1478 xhci_root_intr(sc); 1479 } 1480 1481 if (status & XHCI_STS_HCH) { 1482 printf("%s: host controller halted\n", 1483 __FUNCTION__); 1484 } 1485 1486 if (status & XHCI_STS_HSE) { 1487 printf("%s: host system error\n", 1488 __FUNCTION__); 1489 } 1490 1491 if (status & XHCI_STS_HCE) { 1492 printf("%s: host controller error\n", 1493 __FUNCTION__); 1494 } 1495 } 1496 done: 1497 USB_BUS_UNLOCK(&sc->sc_bus); 1498 } 1499 1500 /*------------------------------------------------------------------------* 1501 * xhci_timeout - XHCI timeout handler 1502 *------------------------------------------------------------------------*/ 1503 static void 1504 xhci_timeout(void *arg) 1505 { 1506 struct usb_xfer *xfer = arg; 1507 1508 DPRINTF("xfer=%p\n", xfer); 1509 1510 USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED); 1511 1512 /* transfer is transferred */ 1513 xhci_device_done(xfer, USB_ERR_TIMEOUT); 1514 } 1515 1516 static void 1517 xhci_do_poll(struct usb_bus *bus) 1518 { 1519 struct xhci_softc *sc = XHCI_BUS2SC(bus); 1520 1521 USB_BUS_LOCK(&sc->sc_bus); 1522 xhci_interrupt_poll(sc); 1523 USB_BUS_UNLOCK(&sc->sc_bus); 1524 } 1525 1526 static void 1527 xhci_setup_generic_chain_sub(struct xhci_std_temp *temp) 1528 { 1529 struct usb_page_search buf_res; 1530 struct xhci_td *td; 1531 struct xhci_td *td_next; 1532 struct xhci_td *td_alt_next; 1533 uint32_t buf_offset; 1534 uint32_t average; 1535 uint32_t len_old; 1536 uint32_t dword; 1537 uint8_t shortpkt_old; 1538 uint8_t precompute; 1539 uint8_t x; 1540 1541 td_alt_next = NULL; 1542 buf_offset = 0; 1543 shortpkt_old = temp->shortpkt; 1544 len_old = temp->len; 1545 precompute = 1; 1546 1547 restart: 1548 1549 td = temp->td; 1550 td_next = temp->td_next; 1551 1552 while (1) { 1553 1554 if (temp->len == 0) { 1555 1556 if (temp->shortpkt) 1557 break; 1558 1559 /* send a Zero Length Packet, ZLP, last */ 1560 1561 temp->shortpkt = 1; 1562 average = 0; 1563 1564 } else { 1565 1566 average = temp->average; 1567 1568 if (temp->len < average) { 1569 if (temp->len % temp->max_packet_size) { 1570 temp->shortpkt = 1; 1571 } 1572 average = temp->len; 1573 } 1574 } 1575 1576 if (td_next == NULL) 1577 panic("%s: out of XHCI transfer descriptors!", __FUNCTION__); 1578 1579 /* get next TD */ 1580 1581 td = td_next; 1582 td_next = td->obj_next; 1583 1584 /* check if we are pre-computing */ 1585 1586 if (precompute) { 1587 1588 /* update remaining length */ 1589 1590 temp->len -= average; 1591 1592 continue; 1593 } 1594 /* fill out current TD */ 1595 1596 td->len = average; 1597 td->remainder = 0; 1598 td->status = 0; 1599 1600 /* update remaining length */ 1601 1602 temp->len -= average; 1603 1604 /* reset TRB index */ 1605 1606 x = 0; 1607 1608 if (temp->trb_type == XHCI_TRB_TYPE_SETUP_STAGE) { 1609 /* immediate data */ 1610 1611 if (average > 8) 1612 average = 8; 1613 1614 td->td_trb[0].qwTrb0 = 0; 1615 1616 usbd_copy_out(temp->pc, temp->offset + buf_offset, 1617 (uint8_t *)(uintptr_t)&td->td_trb[0].qwTrb0, 1618 average); 1619 1620 dword = XHCI_TRB_2_BYTES_SET(8) | 1621 XHCI_TRB_2_TDSZ_SET(0) | 1622 XHCI_TRB_2_IRQ_SET(0); 1623 1624 td->td_trb[0].dwTrb2 = htole32(dword); 1625 1626 dword = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SETUP_STAGE) | 1627 XHCI_TRB_3_IDT_BIT | XHCI_TRB_3_CYCLE_BIT; 1628 1629 /* check wLength */ 1630 if (td->td_trb[0].qwTrb0 & 1631 htole64(XHCI_TRB_0_WLENGTH_MASK)) { 1632 if (td->td_trb[0].qwTrb0 & htole64(1)) 1633 dword |= XHCI_TRB_3_TRT_IN; 1634 else 1635 dword |= XHCI_TRB_3_TRT_OUT; 1636 } 1637 1638 td->td_trb[0].dwTrb3 = htole32(dword); 1639 #ifdef USB_DEBUG 1640 xhci_dump_trb(&td->td_trb[x]); 1641 #endif 1642 x++; 1643 1644 } else do { 1645 1646 uint32_t npkt; 1647 1648 /* fill out buffer pointers */ 1649 1650 if (average == 0) { 1651 npkt = 1; 1652 memset(&buf_res, 0, sizeof(buf_res)); 1653 } else { 1654 usbd_get_page(temp->pc, temp->offset + 1655 buf_offset, &buf_res); 1656 1657 /* get length to end of page */ 1658 if (buf_res.length > average) 1659 buf_res.length = average; 1660 1661 /* check for maximum length */ 1662 if (buf_res.length > XHCI_TD_PAGE_SIZE) 1663 buf_res.length = XHCI_TD_PAGE_SIZE; 1664 1665 /* setup npkt */ 1666 npkt = (average + temp->max_packet_size - 1) / 1667 temp->max_packet_size; 1668 1669 if (npkt > 31) 1670 npkt = 31; 1671 } 1672 1673 /* fill out TRB's */ 1674 td->td_trb[x].qwTrb0 = 1675 htole64((uint64_t)buf_res.physaddr); 1676 1677 dword = 1678 XHCI_TRB_2_BYTES_SET(buf_res.length) | 1679 XHCI_TRB_2_TDSZ_SET(npkt) | 1680 XHCI_TRB_2_IRQ_SET(0); 1681 1682 td->td_trb[x].dwTrb2 = htole32(dword); 1683 1684 dword = XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_CYCLE_BIT | 1685 XHCI_TRB_3_TYPE_SET(temp->trb_type) | 1686 (temp->do_isoc_sync ? 1687 XHCI_TRB_3_FRID_SET(temp->isoc_frame / 8) : 1688 XHCI_TRB_3_ISO_SIA_BIT) | 1689 XHCI_TRB_3_TBC_SET(temp->tbc) | 1690 XHCI_TRB_3_TLBPC_SET(temp->tlbpc); 1691 1692 temp->do_isoc_sync = 0; 1693 1694 if (temp->direction == UE_DIR_IN) { 1695 dword |= XHCI_TRB_3_DIR_IN; 1696 1697 /* 1698 * NOTE: Only the SETUP stage should 1699 * use the IDT bit. Else transactions 1700 * can be sent using the wrong data 1701 * toggle value. 1702 */ 1703 if (temp->trb_type != 1704 XHCI_TRB_TYPE_SETUP_STAGE && 1705 temp->trb_type != 1706 XHCI_TRB_TYPE_STATUS_STAGE) 1707 dword |= XHCI_TRB_3_ISP_BIT; 1708 } 1709 1710 td->td_trb[x].dwTrb3 = htole32(dword); 1711 1712 average -= buf_res.length; 1713 buf_offset += buf_res.length; 1714 #ifdef USB_DEBUG 1715 xhci_dump_trb(&td->td_trb[x]); 1716 #endif 1717 x++; 1718 1719 } while (average != 0); 1720 1721 td->td_trb[x-1].dwTrb3 |= htole32(XHCI_TRB_3_IOC_BIT); 1722 1723 /* store number of data TRB's */ 1724 1725 td->ntrb = x; 1726 1727 DPRINTF("NTRB=%u\n", x); 1728 1729 /* fill out link TRB */ 1730 1731 if (td_next != NULL) { 1732 /* link the current TD with the next one */ 1733 td->td_trb[x].qwTrb0 = htole64((uint64_t)td_next->td_self); 1734 DPRINTF("LINK=0x%08llx\n", (long long)td_next->td_self); 1735 } else { 1736 /* this field will get updated later */ 1737 DPRINTF("NOLINK\n"); 1738 } 1739 1740 dword = XHCI_TRB_2_IRQ_SET(0); 1741 1742 td->td_trb[x].dwTrb2 = htole32(dword); 1743 1744 dword = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) | 1745 XHCI_TRB_3_CYCLE_BIT | XHCI_TRB_3_IOC_BIT; 1746 1747 td->td_trb[x].dwTrb3 = htole32(dword); 1748 1749 td->alt_next = td_alt_next; 1750 #ifdef USB_DEBUG 1751 xhci_dump_trb(&td->td_trb[x]); 1752 #endif 1753 usb_pc_cpu_flush(td->page_cache); 1754 } 1755 1756 if (precompute) { 1757 precompute = 0; 1758 1759 /* setup alt next pointer, if any */ 1760 if (temp->last_frame) { 1761 td_alt_next = NULL; 1762 } else { 1763 /* we use this field internally */ 1764 td_alt_next = td_next; 1765 } 1766 1767 /* restore */ 1768 temp->shortpkt = shortpkt_old; 1769 temp->len = len_old; 1770 goto restart; 1771 } 1772 1773 /* remove cycle bit from first if we are stepping the TRBs */ 1774 if (temp->step_td) 1775 td->td_trb[0].dwTrb3 &= ~htole32(XHCI_TRB_3_CYCLE_BIT); 1776 1777 /* remove chain bit because this is the last TRB in the chain */ 1778 td->td_trb[td->ntrb - 1].dwTrb2 &= ~htole32(XHCI_TRB_2_TDSZ_SET(15)); 1779 td->td_trb[td->ntrb - 1].dwTrb3 &= ~htole32(XHCI_TRB_3_CHAIN_BIT); 1780 1781 usb_pc_cpu_flush(td->page_cache); 1782 1783 temp->td = td; 1784 temp->td_next = td_next; 1785 } 1786 1787 static void 1788 xhci_setup_generic_chain(struct usb_xfer *xfer) 1789 { 1790 struct xhci_std_temp temp; 1791 struct xhci_td *td; 1792 uint32_t x; 1793 uint32_t y; 1794 uint8_t mult; 1795 1796 temp.do_isoc_sync = 0; 1797 temp.step_td = 0; 1798 temp.tbc = 0; 1799 temp.tlbpc = 0; 1800 temp.average = xfer->max_hc_frame_size; 1801 temp.max_packet_size = xfer->max_packet_size; 1802 temp.sc = XHCI_BUS2SC(xfer->xroot->bus); 1803 temp.pc = NULL; 1804 temp.last_frame = 0; 1805 temp.offset = 0; 1806 temp.multishort = xfer->flags_int.isochronous_xfr || 1807 xfer->flags_int.control_xfr || 1808 xfer->flags_int.short_frames_ok; 1809 1810 /* toggle the DMA set we are using */ 1811 xfer->flags_int.curr_dma_set ^= 1; 1812 1813 /* get next DMA set */ 1814 td = xfer->td_start[xfer->flags_int.curr_dma_set]; 1815 1816 temp.td = NULL; 1817 temp.td_next = td; 1818 1819 xfer->td_transfer_first = td; 1820 xfer->td_transfer_cache = td; 1821 1822 if (xfer->flags_int.isochronous_xfr) { 1823 uint8_t shift; 1824 1825 /* compute multiplier for ISOCHRONOUS transfers */ 1826 mult = xfer->endpoint->ecomp ? 1827 UE_GET_SS_ISO_MULT(xfer->endpoint->ecomp->bmAttributes) 1828 : 0; 1829 /* check for USB 2.0 multiplier */ 1830 if (mult == 0) { 1831 mult = (xfer->endpoint->edesc-> 1832 wMaxPacketSize[1] >> 3) & 3; 1833 } 1834 /* range check */ 1835 if (mult > 2) 1836 mult = 3; 1837 else 1838 mult++; 1839 1840 x = XREAD4(temp.sc, runt, XHCI_MFINDEX); 1841 1842 DPRINTF("MFINDEX=0x%08x\n", x); 1843 1844 switch (usbd_get_speed(xfer->xroot->udev)) { 1845 case USB_SPEED_FULL: 1846 shift = 3; 1847 temp.isoc_delta = 8; /* 1ms */ 1848 x += temp.isoc_delta - 1; 1849 x &= ~(temp.isoc_delta - 1); 1850 break; 1851 default: 1852 shift = usbd_xfer_get_fps_shift(xfer); 1853 temp.isoc_delta = 1U << shift; 1854 x += temp.isoc_delta - 1; 1855 x &= ~(temp.isoc_delta - 1); 1856 /* simple frame load balancing */ 1857 x += xfer->endpoint->usb_uframe; 1858 break; 1859 } 1860 1861 y = XHCI_MFINDEX_GET(x - xfer->endpoint->isoc_next); 1862 1863 if ((xfer->endpoint->is_synced == 0) || 1864 (y < (xfer->nframes << shift)) || 1865 (XHCI_MFINDEX_GET(-y) >= (128 * 8))) { 1866 /* 1867 * If there is data underflow or the pipe 1868 * queue is empty we schedule the transfer a 1869 * few frames ahead of the current frame 1870 * position. Else two isochronous transfers 1871 * might overlap. 1872 */ 1873 xfer->endpoint->isoc_next = XHCI_MFINDEX_GET(x + (3 * 8)); 1874 xfer->endpoint->is_synced = 1; 1875 temp.do_isoc_sync = 1; 1876 1877 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next); 1878 } 1879 1880 /* compute isochronous completion time */ 1881 1882 y = XHCI_MFINDEX_GET(xfer->endpoint->isoc_next - (x & ~7)); 1883 1884 xfer->isoc_time_complete = 1885 usb_isoc_time_expand(&temp.sc->sc_bus, x / 8) + 1886 (y / 8) + (((xfer->nframes << shift) + 7) / 8); 1887 1888 x = 0; 1889 temp.isoc_frame = xfer->endpoint->isoc_next; 1890 temp.trb_type = XHCI_TRB_TYPE_ISOCH; 1891 1892 xfer->endpoint->isoc_next += xfer->nframes << shift; 1893 1894 } else if (xfer->flags_int.control_xfr) { 1895 1896 /* check if we should prepend a setup message */ 1897 1898 if (xfer->flags_int.control_hdr) { 1899 1900 temp.len = xfer->frlengths[0]; 1901 temp.pc = xfer->frbuffers + 0; 1902 temp.shortpkt = temp.len ? 1 : 0; 1903 temp.trb_type = XHCI_TRB_TYPE_SETUP_STAGE; 1904 temp.direction = 0; 1905 1906 /* check for last frame */ 1907 if (xfer->nframes == 1) { 1908 /* no STATUS stage yet, SETUP is last */ 1909 if (xfer->flags_int.control_act) 1910 temp.last_frame = 1; 1911 } 1912 1913 xhci_setup_generic_chain_sub(&temp); 1914 } 1915 x = 1; 1916 mult = 1; 1917 temp.isoc_delta = 0; 1918 temp.isoc_frame = 0; 1919 temp.trb_type = XHCI_TRB_TYPE_DATA_STAGE; 1920 } else { 1921 x = 0; 1922 mult = 1; 1923 temp.isoc_delta = 0; 1924 temp.isoc_frame = 0; 1925 temp.trb_type = XHCI_TRB_TYPE_NORMAL; 1926 } 1927 1928 if (x != xfer->nframes) { 1929 /* setup page_cache pointer */ 1930 temp.pc = xfer->frbuffers + x; 1931 /* set endpoint direction */ 1932 temp.direction = UE_GET_DIR(xfer->endpointno); 1933 } 1934 1935 while (x != xfer->nframes) { 1936 1937 /* DATA0 / DATA1 message */ 1938 1939 temp.len = xfer->frlengths[x]; 1940 temp.step_td = ((xfer->endpointno & UE_DIR_IN) && 1941 x != 0 && temp.multishort == 0); 1942 1943 x++; 1944 1945 if (x == xfer->nframes) { 1946 if (xfer->flags_int.control_xfr) { 1947 /* no STATUS stage yet, DATA is last */ 1948 if (xfer->flags_int.control_act) 1949 temp.last_frame = 1; 1950 } else { 1951 temp.last_frame = 1; 1952 } 1953 } 1954 if (temp.len == 0) { 1955 1956 /* make sure that we send an USB packet */ 1957 1958 temp.shortpkt = 0; 1959 1960 temp.tbc = 0; 1961 temp.tlbpc = mult - 1; 1962 1963 } else if (xfer->flags_int.isochronous_xfr) { 1964 1965 uint8_t tdpc; 1966 1967 /* 1968 * Isochronous transfers don't have short 1969 * packet termination: 1970 */ 1971 1972 temp.shortpkt = 1; 1973 1974 /* isochronous transfers have a transfer limit */ 1975 1976 if (temp.len > xfer->max_frame_size) 1977 temp.len = xfer->max_frame_size; 1978 1979 /* compute TD packet count */ 1980 tdpc = (temp.len + xfer->max_packet_size - 1) / 1981 xfer->max_packet_size; 1982 1983 temp.tbc = ((tdpc + mult - 1) / mult) - 1; 1984 temp.tlbpc = (tdpc % mult); 1985 1986 if (temp.tlbpc == 0) 1987 temp.tlbpc = mult - 1; 1988 else 1989 temp.tlbpc--; 1990 } else { 1991 1992 /* regular data transfer */ 1993 1994 temp.shortpkt = xfer->flags.force_short_xfer ? 0 : 1; 1995 } 1996 1997 xhci_setup_generic_chain_sub(&temp); 1998 1999 if (xfer->flags_int.isochronous_xfr) { 2000 temp.offset += xfer->frlengths[x - 1]; 2001 temp.isoc_frame += temp.isoc_delta; 2002 } else { 2003 /* get next Page Cache pointer */ 2004 temp.pc = xfer->frbuffers + x; 2005 } 2006 } 2007 2008 /* check if we should append a status stage */ 2009 2010 if (xfer->flags_int.control_xfr && 2011 !xfer->flags_int.control_act) { 2012 2013 /* 2014 * Send a DATA1 message and invert the current 2015 * endpoint direction. 2016 */ 2017 temp.step_td = (xfer->nframes != 0); 2018 temp.direction = UE_GET_DIR(xfer->endpointno) ^ UE_DIR_IN; 2019 temp.len = 0; 2020 temp.pc = NULL; 2021 temp.shortpkt = 0; 2022 temp.last_frame = 1; 2023 temp.trb_type = XHCI_TRB_TYPE_STATUS_STAGE; 2024 2025 xhci_setup_generic_chain_sub(&temp); 2026 } 2027 2028 td = temp.td; 2029 2030 /* must have at least one frame! */ 2031 2032 xfer->td_transfer_last = td; 2033 2034 DPRINTF("first=%p last=%p\n", xfer->td_transfer_first, td); 2035 } 2036 2037 static void 2038 xhci_set_slot_pointer(struct xhci_softc *sc, uint8_t index, uint64_t dev_addr) 2039 { 2040 struct usb_page_search buf_res; 2041 struct xhci_dev_ctx_addr *pdctxa; 2042 2043 usbd_get_page(&sc->sc_hw.ctx_pc, 0, &buf_res); 2044 2045 pdctxa = buf_res.buffer; 2046 2047 DPRINTF("addr[%u]=0x%016llx\n", index, (long long)dev_addr); 2048 2049 pdctxa->qwBaaDevCtxAddr[index] = htole64(dev_addr); 2050 2051 usb_pc_cpu_flush(&sc->sc_hw.ctx_pc); 2052 } 2053 2054 static usb_error_t 2055 xhci_configure_mask(struct usb_device *udev, uint32_t mask, uint8_t drop) 2056 { 2057 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2058 struct usb_page_search buf_inp; 2059 struct xhci_input_dev_ctx *pinp; 2060 uint32_t temp; 2061 uint8_t index; 2062 uint8_t x; 2063 2064 index = udev->controller_slot_id; 2065 2066 usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp); 2067 2068 pinp = buf_inp.buffer; 2069 2070 if (drop) { 2071 mask &= XHCI_INCTX_NON_CTRL_MASK; 2072 xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx0, mask); 2073 xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx1, 0); 2074 } else { 2075 xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx0, 0); 2076 xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx1, mask); 2077 2078 /* find most significant set bit */ 2079 for (x = 31; x != 1; x--) { 2080 if (mask & (1 << x)) 2081 break; 2082 } 2083 2084 /* adjust */ 2085 x--; 2086 2087 /* figure out maximum */ 2088 if (x > sc->sc_hw.devs[index].context_num) { 2089 sc->sc_hw.devs[index].context_num = x; 2090 temp = xhci_ctx_get_le32(sc, &pinp->ctx_slot.dwSctx0); 2091 temp &= ~XHCI_SCTX_0_CTX_NUM_SET(31); 2092 temp |= XHCI_SCTX_0_CTX_NUM_SET(x + 1); 2093 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx0, temp); 2094 } 2095 } 2096 return (0); 2097 } 2098 2099 static usb_error_t 2100 xhci_configure_endpoint(struct usb_device *udev, 2101 struct usb_endpoint_descriptor *edesc, uint64_t ring_addr, 2102 uint16_t interval, uint8_t max_packet_count, uint8_t mult, 2103 uint8_t fps_shift, uint16_t max_packet_size, 2104 uint16_t max_frame_size, uint8_t ep_mode) 2105 { 2106 struct usb_page_search buf_inp; 2107 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2108 struct xhci_input_dev_ctx *pinp; 2109 uint32_t temp; 2110 uint8_t index; 2111 uint8_t epno; 2112 uint8_t type; 2113 2114 index = udev->controller_slot_id; 2115 2116 usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp); 2117 2118 pinp = buf_inp.buffer; 2119 2120 epno = edesc->bEndpointAddress; 2121 type = edesc->bmAttributes & UE_XFERTYPE; 2122 2123 if (type == UE_CONTROL) 2124 epno |= UE_DIR_IN; 2125 2126 epno = XHCI_EPNO2EPID(epno); 2127 2128 if (epno == 0) 2129 return (USB_ERR_NO_PIPE); /* invalid */ 2130 2131 if (max_packet_count == 0) 2132 return (USB_ERR_BAD_BUFSIZE); 2133 2134 max_packet_count--; 2135 2136 if (mult == 0) 2137 return (USB_ERR_BAD_BUFSIZE); 2138 2139 if (ep_mode == USB_EP_MODE_STREAMS) { 2140 temp = XHCI_EPCTX_0_EPSTATE_SET(0) | 2141 XHCI_EPCTX_0_MAXP_STREAMS_SET(XHCI_MAX_STREAMS_LOG - 1) | 2142 XHCI_EPCTX_0_LSA_SET(1); 2143 2144 ring_addr += sizeof(struct xhci_trb) * 2145 XHCI_MAX_TRANSFERS * XHCI_MAX_STREAMS; 2146 } else { 2147 temp = XHCI_EPCTX_0_EPSTATE_SET(0) | 2148 XHCI_EPCTX_0_MAXP_STREAMS_SET(0) | 2149 XHCI_EPCTX_0_LSA_SET(0); 2150 2151 ring_addr |= XHCI_EPCTX_2_DCS_SET(1); 2152 } 2153 2154 switch (udev->speed) { 2155 case USB_SPEED_FULL: 2156 case USB_SPEED_LOW: 2157 /* 1ms -> 125us */ 2158 fps_shift += 3; 2159 break; 2160 default: 2161 break; 2162 } 2163 2164 switch (type) { 2165 case UE_INTERRUPT: 2166 if (fps_shift > 3) 2167 fps_shift--; 2168 temp |= XHCI_EPCTX_0_IVAL_SET(fps_shift); 2169 break; 2170 case UE_ISOCHRONOUS: 2171 temp |= XHCI_EPCTX_0_IVAL_SET(fps_shift); 2172 2173 switch (udev->speed) { 2174 case USB_SPEED_SUPER: 2175 if (mult > 3) 2176 mult = 3; 2177 temp |= XHCI_EPCTX_0_MULT_SET(mult - 1); 2178 max_packet_count /= mult; 2179 break; 2180 default: 2181 break; 2182 } 2183 break; 2184 default: 2185 break; 2186 } 2187 2188 xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx0, temp); 2189 2190 temp = 2191 XHCI_EPCTX_1_HID_SET(0) | 2192 XHCI_EPCTX_1_MAXB_SET(max_packet_count) | 2193 XHCI_EPCTX_1_MAXP_SIZE_SET(max_packet_size); 2194 2195 if ((udev->parent_hs_hub != NULL) || (udev->address != 0)) { 2196 if (type != UE_ISOCHRONOUS) 2197 temp |= XHCI_EPCTX_1_CERR_SET(3); 2198 } 2199 2200 switch (type) { 2201 case UE_CONTROL: 2202 temp |= XHCI_EPCTX_1_EPTYPE_SET(4); 2203 break; 2204 case UE_ISOCHRONOUS: 2205 temp |= XHCI_EPCTX_1_EPTYPE_SET(1); 2206 break; 2207 case UE_BULK: 2208 temp |= XHCI_EPCTX_1_EPTYPE_SET(2); 2209 break; 2210 default: 2211 temp |= XHCI_EPCTX_1_EPTYPE_SET(3); 2212 break; 2213 } 2214 2215 /* check for IN direction */ 2216 if (epno & 1) 2217 temp |= XHCI_EPCTX_1_EPTYPE_SET(4); 2218 2219 xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx1, temp); 2220 xhci_ctx_set_le64(sc, &pinp->ctx_ep[epno - 1].qwEpCtx2, ring_addr); 2221 2222 switch (edesc->bmAttributes & UE_XFERTYPE) { 2223 case UE_INTERRUPT: 2224 case UE_ISOCHRONOUS: 2225 temp = XHCI_EPCTX_4_MAX_ESIT_PAYLOAD_SET(max_frame_size) | 2226 XHCI_EPCTX_4_AVG_TRB_LEN_SET(MIN(XHCI_PAGE_SIZE, 2227 max_frame_size)); 2228 break; 2229 case UE_CONTROL: 2230 temp = XHCI_EPCTX_4_AVG_TRB_LEN_SET(8); 2231 break; 2232 default: 2233 temp = XHCI_EPCTX_4_AVG_TRB_LEN_SET(XHCI_PAGE_SIZE); 2234 break; 2235 } 2236 2237 xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx4, temp); 2238 2239 #ifdef USB_DEBUG 2240 xhci_dump_endpoint(sc, &pinp->ctx_ep[epno - 1]); 2241 #endif 2242 usb_pc_cpu_flush(&sc->sc_hw.devs[index].input_pc); 2243 2244 return (0); /* success */ 2245 } 2246 2247 static usb_error_t 2248 xhci_configure_endpoint_by_xfer(struct usb_xfer *xfer) 2249 { 2250 struct xhci_endpoint_ext *pepext; 2251 struct usb_endpoint_ss_comp_descriptor *ecomp; 2252 usb_stream_t x; 2253 2254 pepext = xhci_get_endpoint_ext(xfer->xroot->udev, 2255 xfer->endpoint->edesc); 2256 2257 ecomp = xfer->endpoint->ecomp; 2258 2259 for (x = 0; x != XHCI_MAX_STREAMS; x++) { 2260 uint64_t temp; 2261 2262 /* halt any transfers */ 2263 pepext->trb[x * XHCI_MAX_TRANSFERS].dwTrb3 = 0; 2264 2265 /* compute start of TRB ring for stream "x" */ 2266 temp = pepext->physaddr + 2267 (x * XHCI_MAX_TRANSFERS * sizeof(struct xhci_trb)) + 2268 XHCI_SCTX_0_SCT_SEC_TR_RING; 2269 2270 /* make tree structure */ 2271 pepext->trb[(XHCI_MAX_TRANSFERS * 2272 XHCI_MAX_STREAMS) + x].qwTrb0 = htole64(temp); 2273 2274 /* reserved fields */ 2275 pepext->trb[(XHCI_MAX_TRANSFERS * 2276 XHCI_MAX_STREAMS) + x].dwTrb2 = 0; 2277 pepext->trb[(XHCI_MAX_TRANSFERS * 2278 XHCI_MAX_STREAMS) + x].dwTrb3 = 0; 2279 } 2280 usb_pc_cpu_flush(pepext->page_cache); 2281 2282 return (xhci_configure_endpoint(xfer->xroot->udev, 2283 xfer->endpoint->edesc, pepext->physaddr, 2284 xfer->interval, xfer->max_packet_count, 2285 (ecomp != NULL) ? UE_GET_SS_ISO_MULT(ecomp->bmAttributes) + 1 : 1, 2286 usbd_xfer_get_fps_shift(xfer), xfer->max_packet_size, 2287 xfer->max_frame_size, xfer->endpoint->ep_mode)); 2288 } 2289 2290 static usb_error_t 2291 xhci_configure_device(struct usb_device *udev) 2292 { 2293 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2294 struct usb_page_search buf_inp; 2295 struct usb_page_cache *pcinp; 2296 struct xhci_input_dev_ctx *pinp; 2297 struct usb_device *hubdev; 2298 uint32_t temp; 2299 uint32_t route; 2300 uint32_t rh_port; 2301 uint8_t is_hub; 2302 uint8_t index; 2303 uint8_t depth; 2304 2305 index = udev->controller_slot_id; 2306 2307 DPRINTF("index=%u\n", index); 2308 2309 pcinp = &sc->sc_hw.devs[index].input_pc; 2310 2311 usbd_get_page(pcinp, 0, &buf_inp); 2312 2313 pinp = buf_inp.buffer; 2314 2315 rh_port = 0; 2316 route = 0; 2317 2318 /* figure out route string and root HUB port number */ 2319 2320 for (hubdev = udev; hubdev != NULL; hubdev = hubdev->parent_hub) { 2321 2322 if (hubdev->parent_hub == NULL) 2323 break; 2324 2325 depth = hubdev->parent_hub->depth; 2326 2327 /* 2328 * NOTE: HS/FS/LS devices and the SS root HUB can have 2329 * more than 15 ports 2330 */ 2331 2332 rh_port = hubdev->port_no; 2333 2334 if (depth == 0) 2335 break; 2336 2337 if (rh_port > 15) 2338 rh_port = 15; 2339 2340 if (depth < 6) 2341 route |= rh_port << (4 * (depth - 1)); 2342 } 2343 2344 DPRINTF("Route=0x%08x\n", route); 2345 2346 temp = XHCI_SCTX_0_ROUTE_SET(route) | 2347 XHCI_SCTX_0_CTX_NUM_SET( 2348 sc->sc_hw.devs[index].context_num + 1); 2349 2350 switch (udev->speed) { 2351 case USB_SPEED_LOW: 2352 temp |= XHCI_SCTX_0_SPEED_SET(2); 2353 if (udev->parent_hs_hub != NULL && 2354 udev->parent_hs_hub->ddesc.bDeviceProtocol == 2355 UDPROTO_HSHUBMTT) { 2356 DPRINTF("Device inherits MTT\n"); 2357 temp |= XHCI_SCTX_0_MTT_SET(1); 2358 } 2359 break; 2360 case USB_SPEED_HIGH: 2361 temp |= XHCI_SCTX_0_SPEED_SET(3); 2362 if (sc->sc_hw.devs[index].nports != 0 && 2363 udev->ddesc.bDeviceProtocol == UDPROTO_HSHUBMTT) { 2364 DPRINTF("HUB supports MTT\n"); 2365 temp |= XHCI_SCTX_0_MTT_SET(1); 2366 } 2367 break; 2368 case USB_SPEED_FULL: 2369 temp |= XHCI_SCTX_0_SPEED_SET(1); 2370 if (udev->parent_hs_hub != NULL && 2371 udev->parent_hs_hub->ddesc.bDeviceProtocol == 2372 UDPROTO_HSHUBMTT) { 2373 DPRINTF("Device inherits MTT\n"); 2374 temp |= XHCI_SCTX_0_MTT_SET(1); 2375 } 2376 break; 2377 default: 2378 temp |= XHCI_SCTX_0_SPEED_SET(4); 2379 break; 2380 } 2381 2382 is_hub = sc->sc_hw.devs[index].nports != 0 && 2383 (udev->speed == USB_SPEED_SUPER || 2384 udev->speed == USB_SPEED_HIGH); 2385 2386 if (is_hub) 2387 temp |= XHCI_SCTX_0_HUB_SET(1); 2388 2389 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx0, temp); 2390 2391 temp = XHCI_SCTX_1_RH_PORT_SET(rh_port); 2392 2393 if (is_hub) { 2394 temp |= XHCI_SCTX_1_NUM_PORTS_SET( 2395 sc->sc_hw.devs[index].nports); 2396 } 2397 2398 switch (udev->speed) { 2399 case USB_SPEED_SUPER: 2400 switch (sc->sc_hw.devs[index].state) { 2401 case XHCI_ST_ADDRESSED: 2402 case XHCI_ST_CONFIGURED: 2403 /* enable power save */ 2404 temp |= XHCI_SCTX_1_MAX_EL_SET(sc->sc_exit_lat_max); 2405 break; 2406 default: 2407 /* disable power save */ 2408 break; 2409 } 2410 break; 2411 default: 2412 break; 2413 } 2414 2415 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx1, temp); 2416 2417 temp = XHCI_SCTX_2_IRQ_TARGET_SET(0); 2418 2419 if (is_hub) { 2420 temp |= XHCI_SCTX_2_TT_THINK_TIME_SET( 2421 sc->sc_hw.devs[index].tt); 2422 } 2423 2424 hubdev = udev->parent_hs_hub; 2425 2426 /* check if we should activate the transaction translator */ 2427 switch (udev->speed) { 2428 case USB_SPEED_FULL: 2429 case USB_SPEED_LOW: 2430 if (hubdev != NULL) { 2431 temp |= XHCI_SCTX_2_TT_HUB_SID_SET( 2432 hubdev->controller_slot_id); 2433 temp |= XHCI_SCTX_2_TT_PORT_NUM_SET( 2434 udev->hs_port_no); 2435 } 2436 break; 2437 default: 2438 break; 2439 } 2440 2441 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx2, temp); 2442 2443 temp = XHCI_SCTX_3_DEV_ADDR_SET(udev->address) | 2444 XHCI_SCTX_3_SLOT_STATE_SET(0); 2445 2446 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx3, temp); 2447 2448 #ifdef USB_DEBUG 2449 xhci_dump_device(sc, &pinp->ctx_slot); 2450 #endif 2451 usb_pc_cpu_flush(pcinp); 2452 2453 return (0); /* success */ 2454 } 2455 2456 static usb_error_t 2457 xhci_alloc_device_ext(struct usb_device *udev) 2458 { 2459 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2460 struct usb_page_search buf_dev; 2461 struct usb_page_search buf_ep; 2462 struct xhci_trb *trb; 2463 struct usb_page_cache *pc; 2464 struct usb_page *pg; 2465 uint64_t addr; 2466 uint8_t index; 2467 uint8_t i; 2468 2469 index = udev->controller_slot_id; 2470 2471 pc = &sc->sc_hw.devs[index].device_pc; 2472 pg = &sc->sc_hw.devs[index].device_pg; 2473 2474 /* need to initialize the page cache */ 2475 pc->tag_parent = sc->sc_bus.dma_parent_tag; 2476 2477 if (usb_pc_alloc_mem(pc, pg, sc->sc_ctx_is_64_byte ? 2478 (2 * sizeof(struct xhci_dev_ctx)) : 2479 sizeof(struct xhci_dev_ctx), XHCI_PAGE_SIZE)) 2480 goto error; 2481 2482 usbd_get_page(pc, 0, &buf_dev); 2483 2484 pc = &sc->sc_hw.devs[index].input_pc; 2485 pg = &sc->sc_hw.devs[index].input_pg; 2486 2487 /* need to initialize the page cache */ 2488 pc->tag_parent = sc->sc_bus.dma_parent_tag; 2489 2490 if (usb_pc_alloc_mem(pc, pg, sc->sc_ctx_is_64_byte ? 2491 (2 * sizeof(struct xhci_input_dev_ctx)) : 2492 sizeof(struct xhci_input_dev_ctx), XHCI_PAGE_SIZE)) { 2493 goto error; 2494 } 2495 2496 pc = &sc->sc_hw.devs[index].endpoint_pc; 2497 pg = &sc->sc_hw.devs[index].endpoint_pg; 2498 2499 /* need to initialize the page cache */ 2500 pc->tag_parent = sc->sc_bus.dma_parent_tag; 2501 2502 if (usb_pc_alloc_mem(pc, pg, 2503 sizeof(struct xhci_dev_endpoint_trbs), XHCI_PAGE_SIZE)) { 2504 goto error; 2505 } 2506 2507 /* initialise all endpoint LINK TRBs */ 2508 2509 for (i = 0; i != XHCI_MAX_ENDPOINTS; i++) { 2510 2511 /* lookup endpoint TRB ring */ 2512 usbd_get_page(pc, (uintptr_t)& 2513 ((struct xhci_dev_endpoint_trbs *)0)->trb[i][0], &buf_ep); 2514 2515 /* get TRB pointer */ 2516 trb = buf_ep.buffer; 2517 trb += XHCI_MAX_TRANSFERS - 1; 2518 2519 /* get TRB start address */ 2520 addr = buf_ep.physaddr; 2521 2522 /* create LINK TRB */ 2523 trb->qwTrb0 = htole64(addr); 2524 trb->dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0)); 2525 trb->dwTrb3 = htole32(XHCI_TRB_3_CYCLE_BIT | 2526 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK)); 2527 } 2528 2529 usb_pc_cpu_flush(pc); 2530 2531 xhci_set_slot_pointer(sc, index, buf_dev.physaddr); 2532 2533 return (0); 2534 2535 error: 2536 xhci_free_device_ext(udev); 2537 2538 return (USB_ERR_NOMEM); 2539 } 2540 2541 static void 2542 xhci_free_device_ext(struct usb_device *udev) 2543 { 2544 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2545 uint8_t index; 2546 2547 index = udev->controller_slot_id; 2548 xhci_set_slot_pointer(sc, index, 0); 2549 2550 usb_pc_free_mem(&sc->sc_hw.devs[index].device_pc); 2551 usb_pc_free_mem(&sc->sc_hw.devs[index].input_pc); 2552 usb_pc_free_mem(&sc->sc_hw.devs[index].endpoint_pc); 2553 } 2554 2555 static struct xhci_endpoint_ext * 2556 xhci_get_endpoint_ext(struct usb_device *udev, struct usb_endpoint_descriptor *edesc) 2557 { 2558 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2559 struct xhci_endpoint_ext *pepext; 2560 struct usb_page_cache *pc; 2561 struct usb_page_search buf_ep; 2562 uint8_t epno; 2563 uint8_t index; 2564 2565 epno = edesc->bEndpointAddress; 2566 if ((edesc->bmAttributes & UE_XFERTYPE) == UE_CONTROL) 2567 epno |= UE_DIR_IN; 2568 2569 epno = XHCI_EPNO2EPID(epno); 2570 2571 index = udev->controller_slot_id; 2572 2573 pc = &sc->sc_hw.devs[index].endpoint_pc; 2574 2575 usbd_get_page(pc, (uintptr_t)&((struct xhci_dev_endpoint_trbs *)0)-> 2576 trb[epno][0], &buf_ep); 2577 2578 pepext = &sc->sc_hw.devs[index].endp[epno]; 2579 pepext->page_cache = pc; 2580 pepext->trb = buf_ep.buffer; 2581 pepext->physaddr = buf_ep.physaddr; 2582 2583 return (pepext); 2584 } 2585 2586 static void 2587 xhci_endpoint_doorbell(struct usb_xfer *xfer) 2588 { 2589 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus); 2590 uint8_t epno; 2591 uint8_t index; 2592 2593 epno = xfer->endpointno; 2594 if (xfer->flags_int.control_xfr) 2595 epno |= UE_DIR_IN; 2596 2597 epno = XHCI_EPNO2EPID(epno); 2598 index = xfer->xroot->udev->controller_slot_id; 2599 2600 if (xfer->xroot->udev->flags.self_suspended == 0) { 2601 XWRITE4(sc, door, XHCI_DOORBELL(index), 2602 epno | XHCI_DB_SID_SET(xfer->stream_id)); 2603 } 2604 } 2605 2606 static void 2607 xhci_transfer_remove(struct usb_xfer *xfer, usb_error_t error) 2608 { 2609 struct xhci_endpoint_ext *pepext; 2610 2611 if (xfer->flags_int.bandwidth_reclaimed) { 2612 xfer->flags_int.bandwidth_reclaimed = 0; 2613 2614 pepext = xhci_get_endpoint_ext(xfer->xroot->udev, 2615 xfer->endpoint->edesc); 2616 2617 pepext->trb_used[xfer->stream_id]--; 2618 2619 pepext->xfer[xfer->qh_pos] = NULL; 2620 2621 if (error && pepext->trb_running != 0) { 2622 pepext->trb_halted = 1; 2623 pepext->trb_running = 0; 2624 } 2625 } 2626 } 2627 2628 static usb_error_t 2629 xhci_transfer_insert(struct usb_xfer *xfer) 2630 { 2631 struct xhci_td *td_first; 2632 struct xhci_td *td_last; 2633 struct xhci_endpoint_ext *pepext; 2634 uint64_t addr; 2635 usb_stream_t id; 2636 uint8_t i; 2637 uint8_t inext; 2638 uint8_t trb_limit; 2639 2640 DPRINTFN(8, "\n"); 2641 2642 id = xfer->stream_id; 2643 2644 /* check if already inserted */ 2645 if (xfer->flags_int.bandwidth_reclaimed) { 2646 DPRINTFN(8, "Already in schedule\n"); 2647 return (0); 2648 } 2649 2650 pepext = xhci_get_endpoint_ext(xfer->xroot->udev, 2651 xfer->endpoint->edesc); 2652 2653 td_first = xfer->td_transfer_first; 2654 td_last = xfer->td_transfer_last; 2655 addr = pepext->physaddr; 2656 2657 switch (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE) { 2658 case UE_CONTROL: 2659 case UE_INTERRUPT: 2660 /* single buffered */ 2661 trb_limit = 1; 2662 break; 2663 default: 2664 /* multi buffered */ 2665 trb_limit = (XHCI_MAX_TRANSFERS - 2); 2666 break; 2667 } 2668 2669 if (pepext->trb_used[id] >= trb_limit) { 2670 DPRINTFN(8, "Too many TDs queued.\n"); 2671 return (USB_ERR_NOMEM); 2672 } 2673 2674 /* check for stopped condition, after putting transfer on interrupt queue */ 2675 if (pepext->trb_running == 0) { 2676 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus); 2677 2678 DPRINTFN(8, "Not running\n"); 2679 2680 /* start configuration */ 2681 (void)usb_proc_msignal(USB_BUS_CONTROL_XFER_PROC(&sc->sc_bus), 2682 &sc->sc_config_msg[0], &sc->sc_config_msg[1]); 2683 return (0); 2684 } 2685 2686 pepext->trb_used[id]++; 2687 2688 /* get current TRB index */ 2689 i = pepext->trb_index[id]; 2690 2691 /* get next TRB index */ 2692 inext = (i + 1); 2693 2694 /* the last entry of the ring is a hardcoded link TRB */ 2695 if (inext >= (XHCI_MAX_TRANSFERS - 1)) 2696 inext = 0; 2697 2698 /* offset for stream */ 2699 i += id * XHCI_MAX_TRANSFERS; 2700 inext += id * XHCI_MAX_TRANSFERS; 2701 2702 /* compute terminating return address */ 2703 addr += (inext * sizeof(struct xhci_trb)); 2704 2705 /* update next pointer of last link TRB */ 2706 td_last->td_trb[td_last->ntrb].qwTrb0 = htole64(addr); 2707 td_last->td_trb[td_last->ntrb].dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0)); 2708 td_last->td_trb[td_last->ntrb].dwTrb3 = htole32(XHCI_TRB_3_IOC_BIT | 2709 XHCI_TRB_3_CYCLE_BIT | XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK)); 2710 2711 #ifdef USB_DEBUG 2712 xhci_dump_trb(&td_last->td_trb[td_last->ntrb]); 2713 #endif 2714 usb_pc_cpu_flush(td_last->page_cache); 2715 2716 /* write ahead chain end marker */ 2717 2718 pepext->trb[inext].qwTrb0 = 0; 2719 pepext->trb[inext].dwTrb2 = 0; 2720 pepext->trb[inext].dwTrb3 = 0; 2721 2722 /* update next pointer of link TRB */ 2723 2724 pepext->trb[i].qwTrb0 = htole64((uint64_t)td_first->td_self); 2725 pepext->trb[i].dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0)); 2726 2727 #ifdef USB_DEBUG 2728 xhci_dump_trb(&pepext->trb[i]); 2729 #endif 2730 usb_pc_cpu_flush(pepext->page_cache); 2731 2732 /* toggle cycle bit which activates the transfer chain */ 2733 2734 pepext->trb[i].dwTrb3 = htole32(XHCI_TRB_3_CYCLE_BIT | 2735 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK)); 2736 2737 usb_pc_cpu_flush(pepext->page_cache); 2738 2739 DPRINTF("qh_pos = %u\n", i); 2740 2741 pepext->xfer[i] = xfer; 2742 2743 xfer->qh_pos = i; 2744 2745 xfer->flags_int.bandwidth_reclaimed = 1; 2746 2747 pepext->trb_index[id] = inext; 2748 2749 xhci_endpoint_doorbell(xfer); 2750 2751 return (0); 2752 } 2753 2754 static void 2755 xhci_root_intr(struct xhci_softc *sc) 2756 { 2757 uint16_t i; 2758 2759 USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED); 2760 2761 /* clear any old interrupt data */ 2762 memset(sc->sc_hub_idata, 0, sizeof(sc->sc_hub_idata)); 2763 2764 for (i = 1; i <= sc->sc_noport; i++) { 2765 /* pick out CHANGE bits from the status register */ 2766 if (XREAD4(sc, oper, XHCI_PORTSC(i)) & ( 2767 XHCI_PS_CSC | XHCI_PS_PEC | 2768 XHCI_PS_OCC | XHCI_PS_WRC | 2769 XHCI_PS_PRC | XHCI_PS_PLC | 2770 XHCI_PS_CEC)) { 2771 sc->sc_hub_idata[i / 8] |= 1 << (i % 8); 2772 DPRINTF("port %d changed\n", i); 2773 } 2774 } 2775 uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata, 2776 sizeof(sc->sc_hub_idata)); 2777 } 2778 2779 /*------------------------------------------------------------------------* 2780 * xhci_device_done - XHCI done handler 2781 * 2782 * NOTE: This function can be called two times in a row on 2783 * the same USB transfer. From close and from interrupt. 2784 *------------------------------------------------------------------------*/ 2785 static void 2786 xhci_device_done(struct usb_xfer *xfer, usb_error_t error) 2787 { 2788 DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n", 2789 xfer, xfer->endpoint, error); 2790 2791 /* remove transfer from HW queue */ 2792 xhci_transfer_remove(xfer, error); 2793 2794 /* dequeue transfer and start next transfer */ 2795 usbd_transfer_done(xfer, error); 2796 } 2797 2798 /*------------------------------------------------------------------------* 2799 * XHCI data transfer support (generic type) 2800 *------------------------------------------------------------------------*/ 2801 static void 2802 xhci_device_generic_open(struct usb_xfer *xfer) 2803 { 2804 if (xfer->flags_int.isochronous_xfr) { 2805 switch (xfer->xroot->udev->speed) { 2806 case USB_SPEED_FULL: 2807 break; 2808 default: 2809 usb_hs_bandwidth_alloc(xfer); 2810 break; 2811 } 2812 } 2813 } 2814 2815 static void 2816 xhci_device_generic_close(struct usb_xfer *xfer) 2817 { 2818 DPRINTF("\n"); 2819 2820 xhci_device_done(xfer, USB_ERR_CANCELLED); 2821 2822 if (xfer->flags_int.isochronous_xfr) { 2823 switch (xfer->xroot->udev->speed) { 2824 case USB_SPEED_FULL: 2825 break; 2826 default: 2827 usb_hs_bandwidth_free(xfer); 2828 break; 2829 } 2830 } 2831 } 2832 2833 static void 2834 xhci_device_generic_multi_enter(struct usb_endpoint *ep, 2835 usb_stream_t stream_id, struct usb_xfer *enter_xfer) 2836 { 2837 struct usb_xfer *xfer; 2838 2839 /* check if there is a current transfer */ 2840 xfer = ep->endpoint_q[stream_id].curr; 2841 if (xfer == NULL) 2842 return; 2843 2844 /* 2845 * Check if the current transfer is started and then pickup 2846 * the next one, if any. Else wait for next start event due to 2847 * block on failure feature. 2848 */ 2849 if (!xfer->flags_int.bandwidth_reclaimed) 2850 return; 2851 2852 xfer = TAILQ_FIRST(&ep->endpoint_q[stream_id].head); 2853 if (xfer == NULL) { 2854 /* 2855 * In case of enter we have to consider that the 2856 * transfer is queued by the USB core after the enter 2857 * method is called. 2858 */ 2859 xfer = enter_xfer; 2860 2861 if (xfer == NULL) 2862 return; 2863 } 2864 2865 /* try to multi buffer */ 2866 xhci_transfer_insert(xfer); 2867 } 2868 2869 static void 2870 xhci_device_generic_enter(struct usb_xfer *xfer) 2871 { 2872 DPRINTF("\n"); 2873 2874 /* setup TD's and QH */ 2875 xhci_setup_generic_chain(xfer); 2876 2877 xhci_device_generic_multi_enter(xfer->endpoint, 2878 xfer->stream_id, xfer); 2879 } 2880 2881 static void 2882 xhci_device_generic_start(struct usb_xfer *xfer) 2883 { 2884 DPRINTF("\n"); 2885 2886 /* try to insert xfer on HW queue */ 2887 xhci_transfer_insert(xfer); 2888 2889 /* try to multi buffer */ 2890 xhci_device_generic_multi_enter(xfer->endpoint, 2891 xfer->stream_id, NULL); 2892 2893 /* add transfer last on interrupt queue */ 2894 usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer); 2895 2896 /* start timeout, if any */ 2897 if (xfer->timeout != 0) 2898 usbd_transfer_timeout_ms(xfer, &xhci_timeout, xfer->timeout); 2899 } 2900 2901 struct usb_pipe_methods xhci_device_generic_methods = 2902 { 2903 .open = xhci_device_generic_open, 2904 .close = xhci_device_generic_close, 2905 .enter = xhci_device_generic_enter, 2906 .start = xhci_device_generic_start, 2907 }; 2908 2909 /*------------------------------------------------------------------------* 2910 * xhci root HUB support 2911 *------------------------------------------------------------------------* 2912 * Simulate a hardware HUB by handling all the necessary requests. 2913 *------------------------------------------------------------------------*/ 2914 2915 #define HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) } 2916 2917 static const 2918 struct usb_device_descriptor xhci_devd = 2919 { 2920 .bLength = sizeof(xhci_devd), 2921 .bDescriptorType = UDESC_DEVICE, /* type */ 2922 HSETW(.bcdUSB, 0x0300), /* USB version */ 2923 .bDeviceClass = UDCLASS_HUB, /* class */ 2924 .bDeviceSubClass = UDSUBCLASS_HUB, /* subclass */ 2925 .bDeviceProtocol = UDPROTO_SSHUB, /* protocol */ 2926 .bMaxPacketSize = 9, /* max packet size */ 2927 HSETW(.idVendor, 0x0000), /* vendor */ 2928 HSETW(.idProduct, 0x0000), /* product */ 2929 HSETW(.bcdDevice, 0x0100), /* device version */ 2930 .iManufacturer = 1, 2931 .iProduct = 2, 2932 .iSerialNumber = 0, 2933 .bNumConfigurations = 1, /* # of configurations */ 2934 }; 2935 2936 static const 2937 struct xhci_bos_desc xhci_bosd = { 2938 .bosd = { 2939 .bLength = sizeof(xhci_bosd.bosd), 2940 .bDescriptorType = UDESC_BOS, 2941 HSETW(.wTotalLength, sizeof(xhci_bosd)), 2942 .bNumDeviceCaps = 3, 2943 }, 2944 .usb2extd = { 2945 .bLength = sizeof(xhci_bosd.usb2extd), 2946 .bDescriptorType = 1, 2947 .bDevCapabilityType = 2, 2948 .bmAttributes[0] = 2, 2949 }, 2950 .usbdcd = { 2951 .bLength = sizeof(xhci_bosd.usbdcd), 2952 .bDescriptorType = UDESC_DEVICE_CAPABILITY, 2953 .bDevCapabilityType = 3, 2954 .bmAttributes = 0, /* XXX */ 2955 HSETW(.wSpeedsSupported, 0x000C), 2956 .bFunctionalitySupport = 8, 2957 .bU1DevExitLat = 255, /* dummy - not used */ 2958 .wU2DevExitLat = { 0x00, 0x08 }, 2959 }, 2960 .cidd = { 2961 .bLength = sizeof(xhci_bosd.cidd), 2962 .bDescriptorType = 1, 2963 .bDevCapabilityType = 4, 2964 .bReserved = 0, 2965 .bContainerID = 0, /* XXX */ 2966 }, 2967 }; 2968 2969 static const 2970 struct xhci_config_desc xhci_confd = { 2971 .confd = { 2972 .bLength = sizeof(xhci_confd.confd), 2973 .bDescriptorType = UDESC_CONFIG, 2974 .wTotalLength[0] = sizeof(xhci_confd), 2975 .bNumInterface = 1, 2976 .bConfigurationValue = 1, 2977 .iConfiguration = 0, 2978 .bmAttributes = UC_SELF_POWERED, 2979 .bMaxPower = 0 /* max power */ 2980 }, 2981 .ifcd = { 2982 .bLength = sizeof(xhci_confd.ifcd), 2983 .bDescriptorType = UDESC_INTERFACE, 2984 .bNumEndpoints = 1, 2985 .bInterfaceClass = UICLASS_HUB, 2986 .bInterfaceSubClass = UISUBCLASS_HUB, 2987 .bInterfaceProtocol = 0, 2988 }, 2989 .endpd = { 2990 .bLength = sizeof(xhci_confd.endpd), 2991 .bDescriptorType = UDESC_ENDPOINT, 2992 .bEndpointAddress = UE_DIR_IN | XHCI_INTR_ENDPT, 2993 .bmAttributes = UE_INTERRUPT, 2994 .wMaxPacketSize[0] = 2, /* max 15 ports */ 2995 .bInterval = 255, 2996 }, 2997 .endpcd = { 2998 .bLength = sizeof(xhci_confd.endpcd), 2999 .bDescriptorType = UDESC_ENDPOINT_SS_COMP, 3000 .bMaxBurst = 0, 3001 .bmAttributes = 0, 3002 }, 3003 }; 3004 3005 static const 3006 struct usb_hub_ss_descriptor xhci_hubd = { 3007 .bLength = sizeof(xhci_hubd), 3008 .bDescriptorType = UDESC_SS_HUB, 3009 }; 3010 3011 static usb_error_t 3012 xhci_roothub_exec(struct usb_device *udev, 3013 struct usb_device_request *req, const void **pptr, uint16_t *plength) 3014 { 3015 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 3016 const char *str_ptr; 3017 const void *ptr; 3018 uint32_t port; 3019 uint32_t v; 3020 uint16_t len; 3021 uint16_t i; 3022 uint16_t value; 3023 uint16_t index; 3024 uint8_t j; 3025 usb_error_t err; 3026 3027 USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED); 3028 3029 /* buffer reset */ 3030 ptr = (const void *)&sc->sc_hub_desc; 3031 len = 0; 3032 err = 0; 3033 3034 value = UGETW(req->wValue); 3035 index = UGETW(req->wIndex); 3036 3037 DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x " 3038 "wValue=0x%04x wIndex=0x%04x\n", 3039 req->bmRequestType, req->bRequest, 3040 UGETW(req->wLength), value, index); 3041 3042 #define C(x,y) ((x) | ((y) << 8)) 3043 switch (C(req->bRequest, req->bmRequestType)) { 3044 case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE): 3045 case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE): 3046 case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT): 3047 /* 3048 * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops 3049 * for the integrated root hub. 3050 */ 3051 break; 3052 case C(UR_GET_CONFIG, UT_READ_DEVICE): 3053 len = 1; 3054 sc->sc_hub_desc.temp[0] = sc->sc_conf; 3055 break; 3056 case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE): 3057 switch (value >> 8) { 3058 case UDESC_DEVICE: 3059 if ((value & 0xff) != 0) { 3060 err = USB_ERR_IOERROR; 3061 goto done; 3062 } 3063 len = sizeof(xhci_devd); 3064 ptr = (const void *)&xhci_devd; 3065 break; 3066 3067 case UDESC_BOS: 3068 if ((value & 0xff) != 0) { 3069 err = USB_ERR_IOERROR; 3070 goto done; 3071 } 3072 len = sizeof(xhci_bosd); 3073 ptr = (const void *)&xhci_bosd; 3074 break; 3075 3076 case UDESC_CONFIG: 3077 if ((value & 0xff) != 0) { 3078 err = USB_ERR_IOERROR; 3079 goto done; 3080 } 3081 len = sizeof(xhci_confd); 3082 ptr = (const void *)&xhci_confd; 3083 break; 3084 3085 case UDESC_STRING: 3086 switch (value & 0xff) { 3087 case 0: /* Language table */ 3088 str_ptr = "\001"; 3089 break; 3090 3091 case 1: /* Vendor */ 3092 str_ptr = sc->sc_vendor; 3093 break; 3094 3095 case 2: /* Product */ 3096 str_ptr = "XHCI root HUB"; 3097 break; 3098 3099 default: 3100 str_ptr = ""; 3101 break; 3102 } 3103 3104 len = usb_make_str_desc( 3105 sc->sc_hub_desc.temp, 3106 sizeof(sc->sc_hub_desc.temp), 3107 str_ptr); 3108 break; 3109 3110 default: 3111 err = USB_ERR_IOERROR; 3112 goto done; 3113 } 3114 break; 3115 case C(UR_GET_INTERFACE, UT_READ_INTERFACE): 3116 len = 1; 3117 sc->sc_hub_desc.temp[0] = 0; 3118 break; 3119 case C(UR_GET_STATUS, UT_READ_DEVICE): 3120 len = 2; 3121 USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED); 3122 break; 3123 case C(UR_GET_STATUS, UT_READ_INTERFACE): 3124 case C(UR_GET_STATUS, UT_READ_ENDPOINT): 3125 len = 2; 3126 USETW(sc->sc_hub_desc.stat.wStatus, 0); 3127 break; 3128 case C(UR_SET_ADDRESS, UT_WRITE_DEVICE): 3129 if (value >= XHCI_MAX_DEVICES) { 3130 err = USB_ERR_IOERROR; 3131 goto done; 3132 } 3133 break; 3134 case C(UR_SET_CONFIG, UT_WRITE_DEVICE): 3135 if (value != 0 && value != 1) { 3136 err = USB_ERR_IOERROR; 3137 goto done; 3138 } 3139 sc->sc_conf = value; 3140 break; 3141 case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE): 3142 break; 3143 case C(UR_SET_FEATURE, UT_WRITE_DEVICE): 3144 case C(UR_SET_FEATURE, UT_WRITE_INTERFACE): 3145 case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT): 3146 err = USB_ERR_IOERROR; 3147 goto done; 3148 case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE): 3149 break; 3150 case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT): 3151 break; 3152 /* Hub requests */ 3153 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE): 3154 break; 3155 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER): 3156 DPRINTFN(9, "UR_CLEAR_PORT_FEATURE\n"); 3157 3158 if ((index < 1) || 3159 (index > sc->sc_noport)) { 3160 err = USB_ERR_IOERROR; 3161 goto done; 3162 } 3163 port = XHCI_PORTSC(index); 3164 3165 v = XREAD4(sc, oper, port); 3166 i = XHCI_PS_PLS_GET(v); 3167 v &= ~XHCI_PS_CLEAR; 3168 3169 switch (value) { 3170 case UHF_C_BH_PORT_RESET: 3171 XWRITE4(sc, oper, port, v | XHCI_PS_WRC); 3172 break; 3173 case UHF_C_PORT_CONFIG_ERROR: 3174 XWRITE4(sc, oper, port, v | XHCI_PS_CEC); 3175 break; 3176 case UHF_C_PORT_SUSPEND: 3177 case UHF_C_PORT_LINK_STATE: 3178 XWRITE4(sc, oper, port, v | XHCI_PS_PLC); 3179 break; 3180 case UHF_C_PORT_CONNECTION: 3181 XWRITE4(sc, oper, port, v | XHCI_PS_CSC); 3182 break; 3183 case UHF_C_PORT_ENABLE: 3184 XWRITE4(sc, oper, port, v | XHCI_PS_PEC); 3185 break; 3186 case UHF_C_PORT_OVER_CURRENT: 3187 XWRITE4(sc, oper, port, v | XHCI_PS_OCC); 3188 break; 3189 case UHF_C_PORT_RESET: 3190 XWRITE4(sc, oper, port, v | XHCI_PS_PRC); 3191 break; 3192 case UHF_PORT_ENABLE: 3193 XWRITE4(sc, oper, port, v | XHCI_PS_PED); 3194 break; 3195 case UHF_PORT_POWER: 3196 XWRITE4(sc, oper, port, v & ~XHCI_PS_PP); 3197 break; 3198 case UHF_PORT_INDICATOR: 3199 XWRITE4(sc, oper, port, v & ~XHCI_PS_PIC_SET(3)); 3200 break; 3201 case UHF_PORT_SUSPEND: 3202 3203 /* U3 -> U15 */ 3204 if (i == 3) { 3205 XWRITE4(sc, oper, port, v | 3206 XHCI_PS_PLS_SET(0xF) | XHCI_PS_LWS); 3207 } 3208 3209 /* wait 20ms for resume sequence to complete */ 3210 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50); 3211 3212 /* U0 */ 3213 XWRITE4(sc, oper, port, v | 3214 XHCI_PS_PLS_SET(0) | XHCI_PS_LWS); 3215 break; 3216 default: 3217 err = USB_ERR_IOERROR; 3218 goto done; 3219 } 3220 break; 3221 3222 case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE): 3223 if ((value & 0xff) != 0) { 3224 err = USB_ERR_IOERROR; 3225 goto done; 3226 } 3227 3228 v = XREAD4(sc, capa, XHCI_HCSPARAMS0); 3229 3230 sc->sc_hub_desc.hubd = xhci_hubd; 3231 3232 sc->sc_hub_desc.hubd.bNbrPorts = sc->sc_noport; 3233 3234 if (XHCI_HCS0_PPC(v)) 3235 i = UHD_PWR_INDIVIDUAL; 3236 else 3237 i = UHD_PWR_GANGED; 3238 3239 if (XHCI_HCS0_PIND(v)) 3240 i |= UHD_PORT_IND; 3241 3242 i |= UHD_OC_INDIVIDUAL; 3243 3244 USETW(sc->sc_hub_desc.hubd.wHubCharacteristics, i); 3245 3246 /* see XHCI section 5.4.9: */ 3247 sc->sc_hub_desc.hubd.bPwrOn2PwrGood = 10; 3248 3249 for (j = 1; j <= sc->sc_noport; j++) { 3250 3251 v = XREAD4(sc, oper, XHCI_PORTSC(j)); 3252 if (v & XHCI_PS_DR) { 3253 sc->sc_hub_desc.hubd. 3254 DeviceRemovable[j / 8] |= 1U << (j % 8); 3255 } 3256 } 3257 len = sc->sc_hub_desc.hubd.bLength; 3258 break; 3259 3260 case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE): 3261 len = 16; 3262 memset(sc->sc_hub_desc.temp, 0, 16); 3263 break; 3264 3265 case C(UR_GET_STATUS, UT_READ_CLASS_OTHER): 3266 DPRINTFN(9, "UR_GET_STATUS i=%d\n", index); 3267 3268 if ((index < 1) || 3269 (index > sc->sc_noport)) { 3270 err = USB_ERR_IOERROR; 3271 goto done; 3272 } 3273 3274 v = XREAD4(sc, oper, XHCI_PORTSC(index)); 3275 3276 DPRINTFN(9, "port status=0x%08x\n", v); 3277 3278 i = UPS_PORT_LINK_STATE_SET(XHCI_PS_PLS_GET(v)); 3279 3280 switch (XHCI_PS_SPEED_GET(v)) { 3281 case 3: 3282 i |= UPS_HIGH_SPEED; 3283 break; 3284 case 2: 3285 i |= UPS_LOW_SPEED; 3286 break; 3287 case 1: 3288 /* FULL speed */ 3289 break; 3290 default: 3291 i |= UPS_OTHER_SPEED; 3292 break; 3293 } 3294 3295 if (v & XHCI_PS_CCS) 3296 i |= UPS_CURRENT_CONNECT_STATUS; 3297 if (v & XHCI_PS_PED) 3298 i |= UPS_PORT_ENABLED; 3299 if (v & XHCI_PS_OCA) 3300 i |= UPS_OVERCURRENT_INDICATOR; 3301 if (v & XHCI_PS_PR) 3302 i |= UPS_RESET; 3303 if (v & XHCI_PS_PP) { 3304 /* 3305 * The USB 3.0 RH is using the 3306 * USB 2.0's power bit 3307 */ 3308 i |= UPS_PORT_POWER; 3309 } 3310 USETW(sc->sc_hub_desc.ps.wPortStatus, i); 3311 3312 i = 0; 3313 if (v & XHCI_PS_CSC) 3314 i |= UPS_C_CONNECT_STATUS; 3315 if (v & XHCI_PS_PEC) 3316 i |= UPS_C_PORT_ENABLED; 3317 if (v & XHCI_PS_OCC) 3318 i |= UPS_C_OVERCURRENT_INDICATOR; 3319 if (v & XHCI_PS_WRC) 3320 i |= UPS_C_BH_PORT_RESET; 3321 if (v & XHCI_PS_PRC) 3322 i |= UPS_C_PORT_RESET; 3323 if (v & XHCI_PS_PLC) 3324 i |= UPS_C_PORT_LINK_STATE; 3325 if (v & XHCI_PS_CEC) 3326 i |= UPS_C_PORT_CONFIG_ERROR; 3327 3328 USETW(sc->sc_hub_desc.ps.wPortChange, i); 3329 len = sizeof(sc->sc_hub_desc.ps); 3330 break; 3331 3332 case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE): 3333 err = USB_ERR_IOERROR; 3334 goto done; 3335 3336 case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE): 3337 break; 3338 3339 case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER): 3340 3341 i = index >> 8; 3342 index &= 0x00FF; 3343 3344 if ((index < 1) || 3345 (index > sc->sc_noport)) { 3346 err = USB_ERR_IOERROR; 3347 goto done; 3348 } 3349 3350 port = XHCI_PORTSC(index); 3351 v = XREAD4(sc, oper, port) & ~XHCI_PS_CLEAR; 3352 3353 switch (value) { 3354 case UHF_PORT_U1_TIMEOUT: 3355 if (XHCI_PS_SPEED_GET(v) != 4) { 3356 err = USB_ERR_IOERROR; 3357 goto done; 3358 } 3359 port = XHCI_PORTPMSC(index); 3360 v = XREAD4(sc, oper, port); 3361 v &= ~XHCI_PM3_U1TO_SET(0xFF); 3362 v |= XHCI_PM3_U1TO_SET(i); 3363 XWRITE4(sc, oper, port, v); 3364 break; 3365 case UHF_PORT_U2_TIMEOUT: 3366 if (XHCI_PS_SPEED_GET(v) != 4) { 3367 err = USB_ERR_IOERROR; 3368 goto done; 3369 } 3370 port = XHCI_PORTPMSC(index); 3371 v = XREAD4(sc, oper, port); 3372 v &= ~XHCI_PM3_U2TO_SET(0xFF); 3373 v |= XHCI_PM3_U2TO_SET(i); 3374 XWRITE4(sc, oper, port, v); 3375 break; 3376 case UHF_BH_PORT_RESET: 3377 XWRITE4(sc, oper, port, v | XHCI_PS_WPR); 3378 break; 3379 case UHF_PORT_LINK_STATE: 3380 XWRITE4(sc, oper, port, v | 3381 XHCI_PS_PLS_SET(i) | XHCI_PS_LWS); 3382 /* 4ms settle time */ 3383 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250); 3384 break; 3385 case UHF_PORT_ENABLE: 3386 DPRINTFN(3, "set port enable %d\n", index); 3387 break; 3388 case UHF_PORT_SUSPEND: 3389 DPRINTFN(6, "suspend port %u (LPM=%u)\n", index, i); 3390 j = XHCI_PS_SPEED_GET(v); 3391 if ((j < 1) || (j > 3)) { 3392 /* non-supported speed */ 3393 err = USB_ERR_IOERROR; 3394 goto done; 3395 } 3396 XWRITE4(sc, oper, port, v | 3397 XHCI_PS_PLS_SET(i ? 2 /* LPM */ : 3) | XHCI_PS_LWS); 3398 break; 3399 case UHF_PORT_RESET: 3400 DPRINTFN(6, "reset port %d\n", index); 3401 XWRITE4(sc, oper, port, v | XHCI_PS_PR); 3402 break; 3403 case UHF_PORT_POWER: 3404 DPRINTFN(3, "set port power %d\n", index); 3405 XWRITE4(sc, oper, port, v | XHCI_PS_PP); 3406 break; 3407 case UHF_PORT_TEST: 3408 DPRINTFN(3, "set port test %d\n", index); 3409 break; 3410 case UHF_PORT_INDICATOR: 3411 DPRINTFN(3, "set port indicator %d\n", index); 3412 3413 v &= ~XHCI_PS_PIC_SET(3); 3414 v |= XHCI_PS_PIC_SET(1); 3415 3416 XWRITE4(sc, oper, port, v); 3417 break; 3418 default: 3419 err = USB_ERR_IOERROR; 3420 goto done; 3421 } 3422 break; 3423 3424 case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER): 3425 case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER): 3426 case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER): 3427 case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER): 3428 break; 3429 default: 3430 err = USB_ERR_IOERROR; 3431 goto done; 3432 } 3433 done: 3434 *plength = len; 3435 *pptr = ptr; 3436 return (err); 3437 } 3438 3439 static void 3440 xhci_xfer_setup(struct usb_setup_params *parm) 3441 { 3442 struct usb_page_search page_info; 3443 struct usb_page_cache *pc; 3444 struct xhci_softc *sc; 3445 struct usb_xfer *xfer; 3446 void *last_obj; 3447 uint32_t ntd; 3448 uint32_t n; 3449 3450 sc = XHCI_BUS2SC(parm->udev->bus); 3451 xfer = parm->curr_xfer; 3452 3453 /* 3454 * The proof for the "ntd" formula is illustrated like this: 3455 * 3456 * +------------------------------------+ 3457 * | | 3458 * | |remainder -> | 3459 * | +-----+---+ | 3460 * | | xxx | x | frm 0 | 3461 * | +-----+---++ | 3462 * | | xxx | xx | frm 1 | 3463 * | +-----+----+ | 3464 * | ... | 3465 * +------------------------------------+ 3466 * 3467 * "xxx" means a completely full USB transfer descriptor 3468 * 3469 * "x" and "xx" means a short USB packet 3470 * 3471 * For the remainder of an USB transfer modulo 3472 * "max_data_length" we need two USB transfer descriptors. 3473 * One to transfer the remaining data and one to finalise with 3474 * a zero length packet in case the "force_short_xfer" flag is 3475 * set. We only need two USB transfer descriptors in the case 3476 * where the transfer length of the first one is a factor of 3477 * "max_frame_size". The rest of the needed USB transfer 3478 * descriptors is given by the buffer size divided by the 3479 * maximum data payload. 3480 */ 3481 parm->hc_max_packet_size = 0x400; 3482 parm->hc_max_packet_count = 16 * 3; 3483 parm->hc_max_frame_size = XHCI_TD_PAYLOAD_MAX; 3484 3485 xfer->flags_int.bdma_enable = 1; 3486 3487 usbd_transfer_setup_sub(parm); 3488 3489 if (xfer->flags_int.isochronous_xfr) { 3490 ntd = ((1 * xfer->nframes) 3491 + (xfer->max_data_length / xfer->max_hc_frame_size)); 3492 } else if (xfer->flags_int.control_xfr) { 3493 ntd = ((2 * xfer->nframes) + 1 /* STATUS */ 3494 + (xfer->max_data_length / xfer->max_hc_frame_size)); 3495 } else { 3496 ntd = ((2 * xfer->nframes) 3497 + (xfer->max_data_length / xfer->max_hc_frame_size)); 3498 } 3499 3500 alloc_dma_set: 3501 3502 if (parm->err) 3503 return; 3504 3505 /* 3506 * Allocate queue heads and transfer descriptors 3507 */ 3508 last_obj = NULL; 3509 3510 if (usbd_transfer_setup_sub_malloc( 3511 parm, &pc, sizeof(struct xhci_td), 3512 XHCI_TD_ALIGN, ntd)) { 3513 parm->err = USB_ERR_NOMEM; 3514 return; 3515 } 3516 if (parm->buf) { 3517 for (n = 0; n != ntd; n++) { 3518 struct xhci_td *td; 3519 3520 usbd_get_page(pc + n, 0, &page_info); 3521 3522 td = page_info.buffer; 3523 3524 /* init TD */ 3525 td->td_self = page_info.physaddr; 3526 td->obj_next = last_obj; 3527 td->page_cache = pc + n; 3528 3529 last_obj = td; 3530 3531 usb_pc_cpu_flush(pc + n); 3532 } 3533 } 3534 xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj; 3535 3536 if (!xfer->flags_int.curr_dma_set) { 3537 xfer->flags_int.curr_dma_set = 1; 3538 goto alloc_dma_set; 3539 } 3540 } 3541 3542 static usb_error_t 3543 xhci_configure_reset_endpoint(struct usb_xfer *xfer) 3544 { 3545 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus); 3546 struct usb_page_search buf_inp; 3547 struct usb_device *udev; 3548 struct xhci_endpoint_ext *pepext; 3549 struct usb_endpoint_descriptor *edesc; 3550 struct usb_page_cache *pcinp; 3551 usb_error_t err; 3552 usb_stream_t stream_id; 3553 uint8_t index; 3554 uint8_t epno; 3555 3556 pepext = xhci_get_endpoint_ext(xfer->xroot->udev, 3557 xfer->endpoint->edesc); 3558 3559 udev = xfer->xroot->udev; 3560 index = udev->controller_slot_id; 3561 3562 pcinp = &sc->sc_hw.devs[index].input_pc; 3563 3564 usbd_get_page(pcinp, 0, &buf_inp); 3565 3566 edesc = xfer->endpoint->edesc; 3567 3568 epno = edesc->bEndpointAddress; 3569 stream_id = xfer->stream_id; 3570 3571 if ((edesc->bmAttributes & UE_XFERTYPE) == UE_CONTROL) 3572 epno |= UE_DIR_IN; 3573 3574 epno = XHCI_EPNO2EPID(epno); 3575 3576 if (epno == 0) 3577 return (USB_ERR_NO_PIPE); /* invalid */ 3578 3579 XHCI_CMD_LOCK(sc); 3580 3581 /* configure endpoint */ 3582 3583 err = xhci_configure_endpoint_by_xfer(xfer); 3584 3585 if (err != 0) { 3586 XHCI_CMD_UNLOCK(sc); 3587 return (err); 3588 } 3589 3590 /* 3591 * Get the endpoint into the stopped state according to the 3592 * endpoint context state diagram in the XHCI specification: 3593 */ 3594 3595 err = xhci_cmd_stop_ep(sc, 0, epno, index); 3596 3597 if (err != 0) 3598 DPRINTF("Could not stop endpoint %u\n", epno); 3599 3600 err = xhci_cmd_reset_ep(sc, 0, epno, index); 3601 3602 if (err != 0) 3603 DPRINTF("Could not reset endpoint %u\n", epno); 3604 3605 err = xhci_cmd_set_tr_dequeue_ptr(sc, 3606 (pepext->physaddr + (stream_id * sizeof(struct xhci_trb) * 3607 XHCI_MAX_TRANSFERS)) | XHCI_EPCTX_2_DCS_SET(1), 3608 stream_id, epno, index); 3609 3610 if (err != 0) 3611 DPRINTF("Could not set dequeue ptr for endpoint %u\n", epno); 3612 3613 /* 3614 * Get the endpoint into the running state according to the 3615 * endpoint context state diagram in the XHCI specification: 3616 */ 3617 3618 xhci_configure_mask(udev, (1U << epno) | 1U, 0); 3619 3620 err = xhci_cmd_evaluate_ctx(sc, buf_inp.physaddr, index); 3621 3622 if (err != 0) 3623 DPRINTF("Could not configure endpoint %u\n", epno); 3624 3625 err = xhci_cmd_configure_ep(sc, buf_inp.physaddr, 0, index); 3626 3627 if (err != 0) 3628 DPRINTF("Could not configure endpoint %u\n", epno); 3629 3630 XHCI_CMD_UNLOCK(sc); 3631 3632 return (0); 3633 } 3634 3635 static void 3636 xhci_xfer_unsetup(struct usb_xfer *xfer) 3637 { 3638 return; 3639 } 3640 3641 static void 3642 xhci_start_dma_delay(struct usb_xfer *xfer) 3643 { 3644 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus); 3645 3646 /* put transfer on interrupt queue (again) */ 3647 usbd_transfer_enqueue(&sc->sc_bus.intr_q, xfer); 3648 3649 (void)usb_proc_msignal(USB_BUS_CONTROL_XFER_PROC(&sc->sc_bus), 3650 &sc->sc_config_msg[0], &sc->sc_config_msg[1]); 3651 } 3652 3653 static void 3654 xhci_configure_msg(struct usb_proc_msg *pm) 3655 { 3656 struct xhci_softc *sc; 3657 struct xhci_endpoint_ext *pepext; 3658 struct usb_xfer *xfer; 3659 3660 sc = XHCI_BUS2SC(((struct usb_bus_msg *)pm)->bus); 3661 3662 restart: 3663 TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) { 3664 3665 pepext = xhci_get_endpoint_ext(xfer->xroot->udev, 3666 xfer->endpoint->edesc); 3667 3668 if ((pepext->trb_halted != 0) || 3669 (pepext->trb_running == 0)) { 3670 3671 uint8_t i; 3672 3673 /* clear halted and running */ 3674 pepext->trb_halted = 0; 3675 pepext->trb_running = 0; 3676 3677 /* nuke remaining buffered transfers */ 3678 3679 for (i = 0; i != (XHCI_MAX_TRANSFERS - 1); i++) { 3680 /* 3681 * NOTE: We need to use the timeout 3682 * error code here else existing 3683 * isochronous clients can get 3684 * confused: 3685 */ 3686 if (pepext->xfer[i] != NULL) { 3687 xhci_device_done(pepext->xfer[i], 3688 USB_ERR_TIMEOUT); 3689 } 3690 } 3691 3692 /* 3693 * NOTE: The USB transfer cannot vanish in 3694 * this state! 3695 */ 3696 3697 USB_BUS_UNLOCK(&sc->sc_bus); 3698 3699 xhci_configure_reset_endpoint(xfer); 3700 3701 USB_BUS_LOCK(&sc->sc_bus); 3702 3703 /* check if halted is still cleared */ 3704 if (pepext->trb_halted == 0) { 3705 pepext->trb_running = 1; 3706 memset(pepext->trb_index, 0, 3707 sizeof(pepext->trb_index)); 3708 } 3709 goto restart; 3710 } 3711 3712 if (xfer->flags_int.did_dma_delay) { 3713 3714 /* remove transfer from interrupt queue (again) */ 3715 usbd_transfer_dequeue(xfer); 3716 3717 /* we are finally done */ 3718 usb_dma_delay_done_cb(xfer); 3719 3720 /* queue changed - restart */ 3721 goto restart; 3722 } 3723 } 3724 3725 TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) { 3726 3727 /* try to insert xfer on HW queue */ 3728 xhci_transfer_insert(xfer); 3729 3730 /* try to multi buffer */ 3731 xhci_device_generic_multi_enter(xfer->endpoint, 3732 xfer->stream_id, NULL); 3733 } 3734 } 3735 3736 static void 3737 xhci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc, 3738 struct usb_endpoint *ep) 3739 { 3740 struct xhci_endpoint_ext *pepext; 3741 3742 DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d\n", 3743 ep, udev->address, edesc->bEndpointAddress, udev->flags.usb_mode); 3744 3745 if (udev->parent_hub == NULL) { 3746 /* root HUB has special endpoint handling */ 3747 return; 3748 } 3749 3750 ep->methods = &xhci_device_generic_methods; 3751 3752 pepext = xhci_get_endpoint_ext(udev, edesc); 3753 3754 USB_BUS_LOCK(udev->bus); 3755 pepext->trb_halted = 1; 3756 pepext->trb_running = 0; 3757 USB_BUS_UNLOCK(udev->bus); 3758 } 3759 3760 static void 3761 xhci_ep_uninit(struct usb_device *udev, struct usb_endpoint *ep) 3762 { 3763 3764 } 3765 3766 static void 3767 xhci_ep_clear_stall(struct usb_device *udev, struct usb_endpoint *ep) 3768 { 3769 struct xhci_endpoint_ext *pepext; 3770 3771 DPRINTF("\n"); 3772 3773 if (udev->flags.usb_mode != USB_MODE_HOST) { 3774 /* not supported */ 3775 return; 3776 } 3777 if (udev->parent_hub == NULL) { 3778 /* root HUB has special endpoint handling */ 3779 return; 3780 } 3781 3782 pepext = xhci_get_endpoint_ext(udev, ep->edesc); 3783 3784 USB_BUS_LOCK(udev->bus); 3785 pepext->trb_halted = 1; 3786 pepext->trb_running = 0; 3787 USB_BUS_UNLOCK(udev->bus); 3788 } 3789 3790 static usb_error_t 3791 xhci_device_init(struct usb_device *udev) 3792 { 3793 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 3794 usb_error_t err; 3795 uint8_t temp; 3796 3797 /* no init for root HUB */ 3798 if (udev->parent_hub == NULL) 3799 return (0); 3800 3801 XHCI_CMD_LOCK(sc); 3802 3803 /* set invalid default */ 3804 3805 udev->controller_slot_id = sc->sc_noslot + 1; 3806 3807 /* try to get a new slot ID from the XHCI */ 3808 3809 err = xhci_cmd_enable_slot(sc, &temp); 3810 3811 if (err) { 3812 XHCI_CMD_UNLOCK(sc); 3813 return (err); 3814 } 3815 3816 if (temp > sc->sc_noslot) { 3817 XHCI_CMD_UNLOCK(sc); 3818 return (USB_ERR_BAD_ADDRESS); 3819 } 3820 3821 if (sc->sc_hw.devs[temp].state != XHCI_ST_DISABLED) { 3822 DPRINTF("slot %u already allocated.\n", temp); 3823 XHCI_CMD_UNLOCK(sc); 3824 return (USB_ERR_BAD_ADDRESS); 3825 } 3826 3827 /* store slot ID for later reference */ 3828 3829 udev->controller_slot_id = temp; 3830 3831 /* reset data structure */ 3832 3833 memset(&sc->sc_hw.devs[temp], 0, sizeof(sc->sc_hw.devs[0])); 3834 3835 /* set mark slot allocated */ 3836 3837 sc->sc_hw.devs[temp].state = XHCI_ST_ENABLED; 3838 3839 err = xhci_alloc_device_ext(udev); 3840 3841 XHCI_CMD_UNLOCK(sc); 3842 3843 /* get device into default state */ 3844 3845 if (err == 0) 3846 err = xhci_set_address(udev, NULL, 0); 3847 3848 return (err); 3849 } 3850 3851 static void 3852 xhci_device_uninit(struct usb_device *udev) 3853 { 3854 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 3855 uint8_t index; 3856 3857 /* no init for root HUB */ 3858 if (udev->parent_hub == NULL) 3859 return; 3860 3861 XHCI_CMD_LOCK(sc); 3862 3863 index = udev->controller_slot_id; 3864 3865 if (index <= sc->sc_noslot) { 3866 xhci_cmd_disable_slot(sc, index); 3867 sc->sc_hw.devs[index].state = XHCI_ST_DISABLED; 3868 3869 /* free device extension */ 3870 xhci_free_device_ext(udev); 3871 } 3872 3873 XHCI_CMD_UNLOCK(sc); 3874 } 3875 3876 static void 3877 xhci_get_dma_delay(struct usb_device *udev, uint32_t *pus) 3878 { 3879 /* 3880 * Wait until the hardware has finished any possible use of 3881 * the transfer descriptor(s) 3882 */ 3883 *pus = 2048; /* microseconds */ 3884 } 3885 3886 static void 3887 xhci_device_resume(struct usb_device *udev) 3888 { 3889 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 3890 uint8_t index; 3891 uint8_t n; 3892 uint8_t p; 3893 3894 DPRINTF("\n"); 3895 3896 /* check for root HUB */ 3897 if (udev->parent_hub == NULL) 3898 return; 3899 3900 index = udev->controller_slot_id; 3901 3902 XHCI_CMD_LOCK(sc); 3903 3904 /* blindly resume all endpoints */ 3905 3906 USB_BUS_LOCK(udev->bus); 3907 3908 for (n = 1; n != XHCI_MAX_ENDPOINTS; n++) { 3909 for (p = 0; p != XHCI_MAX_STREAMS; p++) { 3910 XWRITE4(sc, door, XHCI_DOORBELL(index), 3911 n | XHCI_DB_SID_SET(p)); 3912 } 3913 } 3914 3915 USB_BUS_UNLOCK(udev->bus); 3916 3917 XHCI_CMD_UNLOCK(sc); 3918 } 3919 3920 static void 3921 xhci_device_suspend(struct usb_device *udev) 3922 { 3923 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 3924 uint8_t index; 3925 uint8_t n; 3926 usb_error_t err; 3927 3928 DPRINTF("\n"); 3929 3930 /* check for root HUB */ 3931 if (udev->parent_hub == NULL) 3932 return; 3933 3934 index = udev->controller_slot_id; 3935 3936 XHCI_CMD_LOCK(sc); 3937 3938 /* blindly suspend all endpoints */ 3939 3940 for (n = 1; n != XHCI_MAX_ENDPOINTS; n++) { 3941 err = xhci_cmd_stop_ep(sc, 1, n, index); 3942 if (err != 0) { 3943 DPRINTF("Failed to suspend endpoint " 3944 "%u on slot %u (ignored).\n", n, index); 3945 } 3946 } 3947 3948 XHCI_CMD_UNLOCK(sc); 3949 } 3950 3951 static void 3952 xhci_set_hw_power(struct usb_bus *bus) 3953 { 3954 DPRINTF("\n"); 3955 } 3956 3957 static void 3958 xhci_device_state_change(struct usb_device *udev) 3959 { 3960 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 3961 struct usb_page_search buf_inp; 3962 usb_error_t err; 3963 uint8_t index; 3964 3965 /* check for root HUB */ 3966 if (udev->parent_hub == NULL) 3967 return; 3968 3969 index = udev->controller_slot_id; 3970 3971 DPRINTF("\n"); 3972 3973 if (usb_get_device_state(udev) == USB_STATE_CONFIGURED) { 3974 err = uhub_query_info(udev, &sc->sc_hw.devs[index].nports, 3975 &sc->sc_hw.devs[index].tt); 3976 if (err != 0) 3977 sc->sc_hw.devs[index].nports = 0; 3978 } 3979 3980 XHCI_CMD_LOCK(sc); 3981 3982 switch (usb_get_device_state(udev)) { 3983 case USB_STATE_POWERED: 3984 if (sc->sc_hw.devs[index].state == XHCI_ST_DEFAULT) 3985 break; 3986 3987 /* set default state */ 3988 sc->sc_hw.devs[index].state = XHCI_ST_DEFAULT; 3989 3990 /* reset number of contexts */ 3991 sc->sc_hw.devs[index].context_num = 0; 3992 3993 err = xhci_cmd_reset_dev(sc, index); 3994 3995 if (err != 0) { 3996 DPRINTF("Device reset failed " 3997 "for slot %u.\n", index); 3998 } 3999 break; 4000 4001 case USB_STATE_ADDRESSED: 4002 if (sc->sc_hw.devs[index].state == XHCI_ST_ADDRESSED) 4003 break; 4004 4005 sc->sc_hw.devs[index].state = XHCI_ST_ADDRESSED; 4006 4007 err = xhci_cmd_configure_ep(sc, 0, 1, index); 4008 4009 if (err) { 4010 DPRINTF("Failed to deconfigure " 4011 "slot %u.\n", index); 4012 } 4013 break; 4014 4015 case USB_STATE_CONFIGURED: 4016 if (sc->sc_hw.devs[index].state == XHCI_ST_CONFIGURED) 4017 break; 4018 4019 /* set configured state */ 4020 sc->sc_hw.devs[index].state = XHCI_ST_CONFIGURED; 4021 4022 /* reset number of contexts */ 4023 sc->sc_hw.devs[index].context_num = 0; 4024 4025 usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp); 4026 4027 xhci_configure_mask(udev, 3, 0); 4028 4029 err = xhci_configure_device(udev); 4030 if (err != 0) { 4031 DPRINTF("Could not configure device " 4032 "at slot %u.\n", index); 4033 } 4034 4035 err = xhci_cmd_evaluate_ctx(sc, buf_inp.physaddr, index); 4036 if (err != 0) { 4037 DPRINTF("Could not evaluate device " 4038 "context at slot %u.\n", index); 4039 } 4040 break; 4041 4042 default: 4043 break; 4044 } 4045 XHCI_CMD_UNLOCK(sc); 4046 } 4047 4048 static usb_error_t 4049 xhci_set_endpoint_mode(struct usb_device *udev, struct usb_endpoint *ep, 4050 uint8_t ep_mode) 4051 { 4052 switch (ep_mode) { 4053 case USB_EP_MODE_DEFAULT: 4054 return (0); 4055 case USB_EP_MODE_STREAMS: 4056 if ((ep->edesc->bmAttributes & UE_XFERTYPE) != UE_BULK || 4057 udev->speed != USB_SPEED_SUPER) 4058 return (USB_ERR_INVAL); 4059 return (0); 4060 default: 4061 return (USB_ERR_INVAL); 4062 } 4063 } 4064 4065 struct usb_bus_methods xhci_bus_methods = { 4066 .endpoint_init = xhci_ep_init, 4067 .endpoint_uninit = xhci_ep_uninit, 4068 .xfer_setup = xhci_xfer_setup, 4069 .xfer_unsetup = xhci_xfer_unsetup, 4070 .get_dma_delay = xhci_get_dma_delay, 4071 .device_init = xhci_device_init, 4072 .device_uninit = xhci_device_uninit, 4073 .device_resume = xhci_device_resume, 4074 .device_suspend = xhci_device_suspend, 4075 .set_hw_power = xhci_set_hw_power, 4076 .roothub_exec = xhci_roothub_exec, 4077 .xfer_poll = xhci_do_poll, 4078 .start_dma_delay = xhci_start_dma_delay, 4079 .set_address = xhci_set_address, 4080 .clear_stall = xhci_ep_clear_stall, 4081 .device_state_change = xhci_device_state_change, 4082 .set_hw_power_sleep = xhci_set_hw_power_sleep, 4083 .set_endpoint_mode = xhci_set_endpoint_mode, 4084 }; 4085