1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright (c) 2018, Joyent, Inc. 14 * Copyright (c) 2019 by Western Digital Corporation 15 * Copyright 2024 Oxide Computer Company 16 */ 17 18 /* 19 * illumos USB framework endpoints and functions for xHCI. 20 * 21 * Please see the big theory statement in xhci.c for more information. 22 */ 23 24 #include <sys/usb/hcd/xhci/xhci.h> 25 #include <sys/sysmacros.h> 26 #include <sys/strsun.h> 27 #include <sys/strsubr.h> 28 29 xhci_t * 30 xhci_hcdi_get_xhcip_from_dev(usba_device_t *ud) 31 { 32 dev_info_t *dip = ud->usb_root_hub_dip; 33 xhci_t *xhcip = ddi_get_soft_state(xhci_soft_state, 34 ddi_get_instance(dip)); 35 VERIFY(xhcip != NULL); 36 return (xhcip); 37 } 38 39 static xhci_t * 40 xhci_hcdi_get_xhcip(usba_pipe_handle_data_t *ph) 41 { 42 return (xhci_hcdi_get_xhcip_from_dev(ph->p_usba_device)); 43 } 44 45 /* 46 * While the xHCI hardware is capable of supporting power management, we don't 47 * in the driver right now. Note, USBA doesn't seem to end up calling this entry 48 * point. 49 */ 50 /* ARGSUSED */ 51 static int 52 xhci_hcdi_pm_support(dev_info_t *dip) 53 { 54 return (USB_FAILURE); 55 } 56 57 static int 58 xhci_hcdi_pipe_open(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags) 59 { 60 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph); 61 xhci_pipe_t *pipe; 62 xhci_endpoint_t *xep = NULL; 63 xhci_device_t *xd; 64 int kmflags = usb_flags & USB_FLAGS_SLEEP ? KM_SLEEP : KM_NOSLEEP; 65 int ret; 66 uint_t epid; 67 68 mutex_enter(&xhcip->xhci_lock); 69 if (xhcip->xhci_state & XHCI_S_ERROR) { 70 mutex_exit(&xhcip->xhci_lock); 71 return (USB_HC_HARDWARE_ERROR); 72 } 73 mutex_exit(&xhcip->xhci_lock); 74 75 /* 76 * If we're here, something must be trying to open an already-opened 77 * pipe which is bad news. 78 */ 79 if (ph->p_hcd_private != NULL) { 80 return (USB_FAILURE); 81 } 82 83 pipe = kmem_zalloc(sizeof (xhci_pipe_t), kmflags); 84 if (pipe == NULL) { 85 return (USB_NO_RESOURCES); 86 } 87 pipe->xp_opentime = gethrtime(); 88 pipe->xp_pipe = ph; 89 90 /* 91 * If this is the root hub, there's nothing special to do on open. Just 92 * go ahead and allow it to be opened. All we have to do is add this to 93 * the list of our tracking structures for open pipes. 94 */ 95 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) { 96 xep = NULL; 97 goto add; 98 } 99 100 /* 101 * Now that we're here, we're being asked to open up an endpoint of some 102 * kind. Because we've already handled the case of the root hub, 103 * everything should have a device. 104 */ 105 epid = xhci_endpoint_pipe_to_epid(ph); 106 xd = usba_hcdi_get_device_private(ph->p_usba_device); 107 if (xd == NULL) { 108 xhci_error(xhcip, "!encountered endpoint (%d) without device " 109 "during pipe open", epid); 110 kmem_free(pipe, sizeof (xhci_pipe_t)); 111 return (USB_FAILURE); 112 } 113 114 /* 115 * See if this endpoint exists or not, in general endpoints should not 116 * exist except for the default control endpoint, which we don't tear 117 * down until the device itself is cleaned up. Otherwise, a given pipe 118 * can only be open once. 119 */ 120 mutex_enter(&xhcip->xhci_lock); 121 if (epid == XHCI_DEFAULT_ENDPOINT) { 122 xep = xd->xd_endpoints[epid]; 123 VERIFY(xep != NULL); 124 VERIFY(xep->xep_pipe == NULL); 125 xep->xep_pipe = ph; 126 mutex_exit(&xhcip->xhci_lock); 127 ret = xhci_endpoint_update_default(xhcip, xd, xep); 128 if (ret != USB_SUCCESS) { 129 kmem_free(pipe, sizeof (xhci_pipe_t)); 130 return (ret); 131 } 132 goto add; 133 } 134 135 /* 136 * If we're opening an endpoint other than the default control endpoint, 137 * then the device should have had a USB address assigned by the 138 * controller. Sanity check that before continuing. 139 */ 140 VERIFY(xd->xd_addressed == B_TRUE); 141 142 /* 143 * We may have already initialized the endpoint with a previous pipe 144 * open. 145 */ 146 if ((xep = xd->xd_endpoints[epid]) != NULL && 147 (xep->xep_state & XHCI_ENDPOINT_OPEN)) { 148 mutex_exit(&xhcip->xhci_lock); 149 150 kmem_free(pipe, sizeof (xhci_pipe_t)); 151 xhci_log(xhcip, "!asked to open endpoint %d on slot %d and " 152 "port %d, but endpoint already exists", epid, xd->xd_slot, 153 xd->xd_port); 154 return (USB_FAILURE); 155 } 156 157 if (xep != NULL) { 158 /* 159 * The endpoint is already initialized but is not presently 160 * open so we can take it over here. 161 */ 162 if ((ret = xhci_endpoint_reinit(xhcip, xd, xep, ph) != 0)) { 163 mutex_exit(&xhcip->xhci_lock); 164 165 kmem_free(pipe, sizeof (xhci_pipe_t)); 166 xhci_log(xhcip, "!asked to reopen endpoint %d on " 167 "slot %d and port %d, but reinit failed (%d)", 168 epid, xd->xd_slot, xd->xd_port, ret); 169 return (ret); 170 } 171 172 /* 173 * We need to ensure the endpoint is stopped before we try to 174 * reset the transfer ring. 175 */ 176 xep->xep_state |= XHCI_ENDPOINT_QUIESCE; 177 if ((ret = xhci_endpoint_quiesce(xhcip, xd, xep)) != 178 USB_SUCCESS) { 179 /* 180 * If we could not quiesce the endpoint, release it so 181 * that another open can try again. 182 */ 183 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE; 184 xhci_endpoint_release(xhcip, xep); 185 mutex_exit(&xhcip->xhci_lock); 186 187 kmem_free(pipe, sizeof (xhci_pipe_t)); 188 xhci_log(xhcip, "!asked to reopen endpoint %d on " 189 "slot %d and port %d, but quiesce failed (%d)", 190 epid, xd->xd_slot, xd->xd_port, ret); 191 return (ret); 192 } 193 194 /* 195 * Reset the transfer ring dequeue pointer. The initial 196 * Configure Endpoint command leaves the endpoint in the 197 * Running state (xHCI 1.2 / 4.6.6), so even though the ring is 198 * still empty we ring the doorbell to end up in the same state 199 * (Running but Inactive). 200 */ 201 mutex_exit(&xhcip->xhci_lock); 202 if ((ret = xhci_command_set_tr_dequeue(xhcip, xd, xep)) != 0 || 203 (ret = xhci_endpoint_ring(xhcip, xd, xep)) != 0) { 204 mutex_enter(&xhcip->xhci_lock); 205 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE; 206 xhci_endpoint_release(xhcip, xep); 207 mutex_exit(&xhcip->xhci_lock); 208 209 kmem_free(pipe, sizeof (xhci_pipe_t)); 210 xhci_log(xhcip, "!asked to open endpoint %d on " 211 "slot %d and port %d, but restart failed (%d)", 212 epid, xd->xd_slot, xd->xd_port, ret); 213 return (USB_FAILURE); 214 } 215 mutex_enter(&xhcip->xhci_lock); 216 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE; 217 mutex_exit(&xhcip->xhci_lock); 218 219 goto add; 220 } 221 222 /* 223 * Okay, at this point we need to go create and set up an endpoint from 224 * scratch. Once we're done, we'll try to install it and make sure 225 * that it doesn't conflict with something else going on. 226 */ 227 ret = xhci_endpoint_init(xhcip, xd, ph); 228 if (ret != 0) { 229 mutex_exit(&xhcip->xhci_lock); 230 kmem_free(pipe, sizeof (xhci_pipe_t)); 231 if (ret == EIO) { 232 xhci_error(xhcip, "failed to initialize endpoint %d " 233 "on device slot %d and port %d: encountered fatal " 234 "FM error, resetting device", epid, xd->xd_slot, 235 xd->xd_port); 236 xhci_fm_runtime_reset(xhcip); 237 } 238 return (USB_HC_HARDWARE_ERROR); 239 } 240 xep = xd->xd_endpoints[epid]; 241 242 mutex_enter(&xd->xd_imtx); 243 mutex_exit(&xhcip->xhci_lock); 244 245 /* 246 * Update the slot and input context for this endpoint. We make sure to 247 * always set the slot as having changed in the context field as the 248 * specification suggests we should and some hardware requires it. 249 */ 250 xd->xd_input->xic_drop_flags = LE_32(0); 251 xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0) | 252 XHCI_INCTX_MASK_DCI(epid + 1)); 253 254 if (epid + 1 > XHCI_SCTX_GET_DCI(LE_32(xd->xd_slotin->xsc_info))) { 255 uint32_t info; 256 257 info = xd->xd_slotin->xsc_info; 258 info &= ~XHCI_SCTX_DCI_MASK; 259 info |= XHCI_SCTX_SET_DCI(epid + 1); 260 xd->xd_slotin->xsc_info = info; 261 } 262 263 XHCI_DMA_SYNC(xd->xd_ictx, DDI_DMA_SYNC_FORDEV); 264 if (xhci_check_dma_handle(xhcip, &xd->xd_ictx) != DDI_FM_OK) { 265 mutex_exit(&xd->xd_imtx); 266 xhci_endpoint_fini(xd, epid); 267 kmem_free(pipe, sizeof (xhci_pipe_t)); 268 xhci_error(xhcip, "failed to open pipe on endpoint %d of " 269 "device with slot %d and port %d: encountered fatal FM " 270 "error syncing device input context, resetting device", 271 epid, xd->xd_slot, xd->xd_port); 272 xhci_fm_runtime_reset(xhcip); 273 return (USB_HC_HARDWARE_ERROR); 274 } 275 276 if ((ret = xhci_command_configure_endpoint(xhcip, xd)) != USB_SUCCESS) { 277 mutex_exit(&xd->xd_imtx); 278 xhci_endpoint_fini(xd, epid); 279 kmem_free(pipe, sizeof (xhci_pipe_t)); 280 return (ret); 281 } 282 283 mutex_exit(&xd->xd_imtx); 284 add: 285 pipe->xp_ep = xep; 286 ph->p_hcd_private = (usb_opaque_t)pipe; 287 mutex_enter(&xhcip->xhci_lock); 288 list_insert_tail(&xhcip->xhci_usba.xa_pipes, pipe); 289 mutex_exit(&xhcip->xhci_lock); 290 291 return (USB_SUCCESS); 292 } 293 294 static void 295 xhci_hcdi_periodic_free(xhci_t *xhcip, xhci_pipe_t *xp) 296 { 297 int i; 298 xhci_periodic_pipe_t *xpp = &xp->xp_periodic; 299 300 if (xpp->xpp_tsize == 0) 301 return; 302 303 for (i = 0; i < xpp->xpp_ntransfers; i++) { 304 if (xpp->xpp_transfers[i] == NULL) 305 continue; 306 xhci_transfer_free(xhcip, xpp->xpp_transfers[i]); 307 xpp->xpp_transfers[i] = NULL; 308 } 309 310 xpp->xpp_ntransfers = 0; 311 xpp->xpp_tsize = 0; 312 } 313 314 /* 315 * Iterate over all transfers and free everything on the pipe. Once done, update 316 * the ring to basically 'consume' everything. For periodic IN endpoints, we 317 * need to handle this somewhat differently and actually close the original 318 * request and not deallocate the related pieces as those exist for the lifetime 319 * of the endpoint and are constantly reused. While interrupt IN endpoints are 320 * mostly seen as periodic, oneshot transfers on those endpoints still bear a 321 * USB request that must be completed, and are a special case where we must 322 * still free the transfer. 323 */ 324 static void 325 xhci_hcdi_pipe_flush(xhci_t *xhcip, xhci_endpoint_t *xep, int intr_code) 326 { 327 xhci_transfer_t *xt; 328 329 ASSERT(MUTEX_HELD(&xhcip->xhci_lock)); 330 331 while ((xt = list_remove_head(&xep->xep_transfers)) != NULL) { 332 if (xhci_endpoint_is_periodic_in(xep) == B_FALSE || 333 XHCI_IS_ONESHOT_XFER(xt)) { 334 usba_hcdi_cb(xep->xep_pipe, xt->xt_usba_req, 335 USB_CR_FLUSHED); 336 xhci_transfer_free(xhcip, xt); 337 } 338 } 339 340 if (xhci_endpoint_is_periodic_in(xep) == B_TRUE) { 341 xhci_pipe_t *xp = (xhci_pipe_t *)xep->xep_pipe->p_hcd_private; 342 xhci_periodic_pipe_t *xpp = &xp->xp_periodic; 343 344 if (xpp->xpp_usb_req != NULL) { 345 usba_hcdi_cb(xep->xep_pipe, xpp->xpp_usb_req, 346 intr_code); 347 xpp->xpp_usb_req = NULL; 348 } 349 } 350 } 351 352 /* 353 * We've been asked to terminate some set of regular I/O on an interrupt pipe. 354 * If this is for the root device, e.g. the xhci driver itself, then we remove 355 * our interrupt callback. Otherwise we stop the device for interrupt polling as 356 * follows: 357 * 358 * 1. Issue a stop endpoint command 359 * 2. Check to make sure that the endpoint stopped and reset it if needed. 360 * 3. Any thing that gets resolved can callback in the interim. 361 * 4. Ensure that nothing is scheduled on the ring 362 * 5. Skip the contents of the ring and set the TR dequeue pointer. 363 * 6. Return the original callback with a USB_CR_STOPPED_POLLING, NULL out the 364 * callback in the process. 365 */ 366 static int 367 xhci_hcdi_pipe_poll_fini(usba_pipe_handle_data_t *ph, boolean_t is_close) 368 { 369 int ret; 370 uint_t epid; 371 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph); 372 xhci_device_t *xd; 373 xhci_endpoint_t *xep; 374 xhci_pipe_t *xp; 375 xhci_periodic_pipe_t *xpp; 376 usb_opaque_t urp; 377 378 mutex_enter(&xhcip->xhci_lock); 379 if (xhcip->xhci_state & XHCI_S_ERROR) { 380 mutex_exit(&xhcip->xhci_lock); 381 return (USB_HC_HARDWARE_ERROR); 382 } 383 384 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) { 385 xhci_root_hub_intr_root_disable(xhcip); 386 ret = USB_SUCCESS; 387 mutex_exit(&xhcip->xhci_lock); 388 return (ret); 389 } 390 391 xd = usba_hcdi_get_device_private(ph->p_usba_device); 392 epid = xhci_endpoint_pipe_to_epid(ph); 393 if (xd->xd_endpoints[epid] == NULL) { 394 mutex_exit(&xhcip->xhci_lock); 395 xhci_error(xhcip, "asked to stop intr polling on slot %d, " 396 "port %d, endpoint: %d, but no endpoint structure", 397 xd->xd_slot, xd->xd_port, epid); 398 return (USB_FAILURE); 399 } 400 xep = xd->xd_endpoints[epid]; 401 xp = (xhci_pipe_t *)ph->p_hcd_private; 402 if (xp == NULL) { 403 mutex_exit(&xhcip->xhci_lock); 404 xhci_error(xhcip, "asked to do finish polling on slot %d, " 405 "port %d, endpoint: %d, but no pipe structure", 406 xd->xd_slot, xd->xd_port, epid); 407 return (USB_FAILURE); 408 } 409 xpp = &xp->xp_periodic; 410 411 /* 412 * Ensure that no other resets or time outs are going on right now. 413 */ 414 xhci_endpoint_serialize(xhcip, xep); 415 416 if (xpp->xpp_poll_state == XHCI_PERIODIC_POLL_IDLE) { 417 mutex_exit(&xhcip->xhci_lock); 418 return (USB_SUCCESS); 419 } 420 421 if (xpp->xpp_poll_state == XHCI_PERIODIC_POLL_STOPPING) { 422 mutex_exit(&xhcip->xhci_lock); 423 return (USB_FAILURE); 424 } 425 426 xpp->xpp_poll_state = XHCI_PERIODIC_POLL_STOPPING; 427 xep->xep_state |= XHCI_ENDPOINT_QUIESCE; 428 ret = xhci_endpoint_quiesce(xhcip, xd, xep); 429 if (ret != USB_SUCCESS) { 430 xhci_error(xhcip, "!failed to quiesce endpoint on slot %d, " 431 "port %d, endpoint: %d, failed with %d.", 432 xd->xd_slot, xd->xd_port, epid, ret); 433 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE; 434 cv_broadcast(&xep->xep_state_cv); 435 mutex_exit(&xhcip->xhci_lock); 436 return (ret); 437 } 438 439 /* 440 * Okay, we've stopped this ring time to wrap it all up. Remove all the 441 * transfers, note they aren't freed like a pipe reset. 442 */ 443 while (list_is_empty(&xep->xep_transfers) == 0) 444 (void) list_remove_head(&xep->xep_transfers); 445 xhci_ring_skip(&xep->xep_ring); 446 mutex_exit(&xhcip->xhci_lock); 447 448 if ((ret = xhci_command_set_tr_dequeue(xhcip, xd, xep)) != 449 USB_SUCCESS) { 450 xhci_error(xhcip, "!failed to reset endpoint ring on slot %d, " 451 "port %d, endpoint: %d, failed with %d.", 452 xd->xd_slot, xd->xd_port, epid, ret); 453 mutex_enter(&xhcip->xhci_lock); 454 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE; 455 cv_broadcast(&xep->xep_state_cv); 456 mutex_exit(&xhcip->xhci_lock); 457 return (ret); 458 } 459 460 mutex_enter(&xhcip->xhci_lock); 461 urp = xpp->xpp_usb_req; 462 xpp->xpp_usb_req = NULL; 463 xpp->xpp_poll_state = XHCI_PERIODIC_POLL_IDLE; 464 xep->xep_state &= ~XHCI_ENDPOINT_PERIODIC; 465 mutex_exit(&xhcip->xhci_lock); 466 467 /* 468 * It's possible that with a persistent pipe, we may not actually have 469 * anything left to call back on, because we already had. 470 */ 471 if (urp != NULL) { 472 usba_hcdi_cb(ph, urp, is_close == B_TRUE ? 473 USB_CR_PIPE_CLOSING : USB_CR_STOPPED_POLLING); 474 } 475 476 /* 477 * Notify anything waiting for us that we're done quiescing this device. 478 */ 479 mutex_enter(&xhcip->xhci_lock); 480 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE; 481 cv_broadcast(&xep->xep_state_cv); 482 mutex_exit(&xhcip->xhci_lock); 483 484 return (USB_SUCCESS); 485 486 } 487 488 /* 489 * Tear down everything that we did in open. After this, the consumer of this 490 * USB device is done. 491 */ 492 static int 493 xhci_hcdi_pipe_close(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags) 494 { 495 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph); 496 xhci_pipe_t *xp; 497 xhci_device_t *xd; 498 xhci_endpoint_t *xep; 499 int ret; 500 uint_t epid; 501 502 if ((ph->p_ep.bmAttributes & USB_EP_ATTR_MASK) == USB_EP_ATTR_INTR && 503 xhcip->xhci_usba.xa_intr_cb_ph != NULL) { 504 if ((ret = xhci_hcdi_pipe_poll_fini(ph, B_TRUE)) != 505 USB_SUCCESS) { 506 return (ret); 507 } 508 } 509 510 mutex_enter(&xhcip->xhci_lock); 511 512 xp = (xhci_pipe_t *)ph->p_hcd_private; 513 VERIFY(xp != NULL); 514 515 /* 516 * The default endpoint is special. It is created and destroyed with the 517 * device. So like with open, closing it is just state tracking. The 518 * same is true for the root hub. 519 */ 520 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) 521 goto remove; 522 523 xd = usba_hcdi_get_device_private(ph->p_usba_device); 524 epid = xhci_endpoint_pipe_to_epid(ph); 525 if (xd->xd_endpoints[epid] == NULL) { 526 mutex_exit(&xhcip->xhci_lock); 527 xhci_error(xhcip, "asked to do close pipe on slot %d, " 528 "port %d, endpoint: %d, but no endpoint structure", 529 xd->xd_slot, xd->xd_port, epid); 530 return (USB_FAILURE); 531 } 532 xep = xd->xd_endpoints[epid]; 533 534 if (xp->xp_ep != NULL && xp->xp_ep->xep_num == XHCI_DEFAULT_ENDPOINT) { 535 xep->xep_pipe = NULL; 536 goto remove; 537 } 538 539 /* 540 * We clean up the endpoint by stopping it and cancelling any transfers 541 * that were in flight at the time. The endpoint is not unconfigured 542 * until the device is torn down later. 543 */ 544 xhci_endpoint_timeout_cancel(xhcip, xep); 545 xep->xep_state |= XHCI_ENDPOINT_QUIESCE; 546 if ((ret = xhci_endpoint_quiesce(xhcip, xd, xep)) != USB_SUCCESS) { 547 /* 548 * If we cannot stop the ring, it is not safe to proceed and we 549 * must keep the pipe open. 550 */ 551 xep->xep_state &= 552 ~(XHCI_ENDPOINT_TEARDOWN | XHCI_ENDPOINT_QUIESCE); 553 cv_broadcast(&xep->xep_state_cv); 554 mutex_exit(&xhcip->xhci_lock); 555 xhci_error(xhcip, "asked to do close pipe on slot %d, " 556 "port %d, endpoint: %d, but quiesce failed %d", 557 xd->xd_slot, xd->xd_port, epid, ret); 558 return (USB_FAILURE); 559 } 560 561 /* 562 * Now that we've stopped the endpoint, see if we need to flush any 563 * transfers. 564 */ 565 xhci_hcdi_pipe_flush(xhcip, xep, USB_CR_PIPE_CLOSING); 566 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) { 567 xhci_hcdi_periodic_free(xhcip, xp); 568 } 569 570 xhci_endpoint_release(xhcip, xep); 571 572 remove: 573 ph->p_hcd_private = NULL; 574 list_remove(&xhcip->xhci_usba.xa_pipes, xp); 575 kmem_free(xp, sizeof (xhci_pipe_t)); 576 577 mutex_exit(&xhcip->xhci_lock); 578 579 return (USB_SUCCESS); 580 } 581 582 /* 583 * We've been asked to reset a pipe aka an endpoint. This endpoint may be in an 584 * arbitrary state, it may be running or it may be halted. In this case, we go 585 * through and check whether or not we know it's been halted or not. If it has 586 * not, then we stop the endpoint. 587 * 588 * Once the endpoint has been stopped, walk all transfers and go ahead and 589 * basically return them as being flushed. Then finally set the dequeue point 590 * for this endpoint. 591 */ 592 /* ARGSUSED */ 593 static int 594 xhci_hcdi_pipe_reset(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags) 595 { 596 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph); 597 xhci_device_t *xd; 598 xhci_endpoint_t *xep; 599 uint_t epid; 600 int ret; 601 602 mutex_enter(&xhcip->xhci_lock); 603 if (xhcip->xhci_state & XHCI_S_ERROR) { 604 mutex_exit(&xhcip->xhci_lock); 605 return (USB_HC_HARDWARE_ERROR); 606 } 607 608 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) { 609 mutex_exit(&xhcip->xhci_lock); 610 return (USB_NOT_SUPPORTED); 611 } 612 613 xd = usba_hcdi_get_device_private(ph->p_usba_device); 614 epid = xhci_endpoint_pipe_to_epid(ph); 615 if (xd->xd_endpoints[epid] == NULL) { 616 mutex_exit(&xhcip->xhci_lock); 617 xhci_error(xhcip, "asked to do reset pipe on slot %d, " 618 "port %d, endpoint: %d, but no endpoint structure", 619 xd->xd_slot, xd->xd_port, epid); 620 return (USB_FAILURE); 621 } 622 623 xep = xd->xd_endpoints[epid]; 624 625 /* 626 * Ensure that no other resets or time outs are going on right now. 627 */ 628 xhci_endpoint_serialize(xhcip, xep); 629 630 xep->xep_state |= XHCI_ENDPOINT_QUIESCE; 631 ret = xhci_endpoint_quiesce(xhcip, xd, xep); 632 if (ret != USB_SUCCESS) { 633 /* 634 * We failed to quiesce for some reason, remove the flag and let 635 * someone else give it a shot. 636 */ 637 xhci_error(xhcip, "!failed to quiesce endpoint on slot %d, " 638 "port %d, endpoint: %d, failed with %d.", 639 xd->xd_slot, xd->xd_port, epid, ret); 640 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE; 641 cv_broadcast(&xep->xep_state_cv); 642 mutex_exit(&xhcip->xhci_lock); 643 return (ret); 644 } 645 646 xhci_ring_skip(&xep->xep_ring); 647 648 mutex_exit(&xhcip->xhci_lock); 649 if ((ret = xhci_command_set_tr_dequeue(xhcip, xd, xep)) != 650 USB_SUCCESS) { 651 xhci_error(xhcip, "!failed to reset endpoint ring on slot %d, " 652 "port %d, endpoint: %d, failed setting ring dequeue with " 653 "%d.", xd->xd_slot, xd->xd_port, epid, ret); 654 mutex_enter(&xhcip->xhci_lock); 655 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE; 656 cv_broadcast(&xep->xep_state_cv); 657 mutex_exit(&xhcip->xhci_lock); 658 return (ret); 659 } 660 661 mutex_enter(&xhcip->xhci_lock); 662 xhci_hcdi_pipe_flush(xhcip, xep, USB_CR_PIPE_RESET); 663 664 /* 665 * We need to remove the periodic flag as part of resetting, as if this 666 * was used for periodic activity, it no longer is and therefore can now 667 * be used for such purposes. 668 * 669 * Notify anything waiting for us that we're done quiescing this device. 670 */ 671 xep->xep_state &= ~(XHCI_ENDPOINT_QUIESCE | XHCI_ENDPOINT_PERIODIC); 672 cv_broadcast(&xep->xep_state_cv); 673 mutex_exit(&xhcip->xhci_lock); 674 675 return (USB_SUCCESS); 676 } 677 678 /* 679 * We're asked to reset or change the data toggle, which is used in a few cases. 680 * However, there doesn't seem to be a good way to do this in xHCI as the data 681 * toggle isn't exposed. It seems that dropping a reset endpoint would 682 * theoretically do this; however, that can only be used when in the HALTED 683 * state. As such, for now we just return. 684 */ 685 /* ARGSUSED */ 686 void 687 xhci_hcdi_pipe_reset_data_toggle(usba_pipe_handle_data_t *pipe_handle) 688 { 689 } 690 691 /* 692 * We need to convert the USB request to an 8-byte little endian value. If we 693 * didn't have to think about big endian systems, this would be fine. 694 * Unfortunately, with them, this is a bit confusing. The problem is that if you 695 * think of this as a struct layout, the order that we or things together 696 * represents their byte layout. e.g. ctrl_bRequest is at offset 1 in the SETUP 697 * STAGE trb. However, when it becomes a part of a 64-bit big endian number, if 698 * ends up at byte 7, where as it needs to be at one. Hence why we do a final 699 * LE_64 at the end of this, to convert this into the byte order that it's 700 * expected to be in. 701 */ 702 static uint64_t 703 xhci_hcdi_ctrl_req_to_trb(usb_ctrl_req_t *ucrp) 704 { 705 uint64_t ret = ucrp->ctrl_bmRequestType | 706 (ucrp->ctrl_bRequest << 8) | 707 ((uint64_t)LE_16(ucrp->ctrl_wValue) << 16) | 708 ((uint64_t)LE_16(ucrp->ctrl_wIndex) << 32) | 709 ((uint64_t)LE_16(ucrp->ctrl_wLength) << 48); 710 return (LE_64(ret)); 711 } 712 713 /* 714 * USBA calls us in order to make a specific control type request to a device, 715 * potentially even the root hub. If the request is for the root hub, then we 716 * need to intercept this and cons up the requested data. 717 */ 718 static int 719 xhci_hcdi_pipe_ctrl_xfer(usba_pipe_handle_data_t *ph, usb_ctrl_req_t *ucrp, 720 usb_flags_t usb_flags) 721 { 722 int ret, statusdir, trt; 723 uint_t ep; 724 xhci_device_t *xd; 725 xhci_endpoint_t *xep; 726 xhci_transfer_t *xt; 727 boolean_t datain; 728 729 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph); 730 731 mutex_enter(&xhcip->xhci_lock); 732 if (xhcip->xhci_state & XHCI_S_ERROR) { 733 mutex_exit(&xhcip->xhci_lock); 734 return (USB_HC_HARDWARE_ERROR); 735 } 736 737 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) { 738 ret = xhci_root_hub_ctrl_req(xhcip, ph, ucrp); 739 mutex_exit(&xhcip->xhci_lock); 740 return (ret); 741 } 742 743 /* 744 * Determine the device and endpoint. 745 */ 746 xd = usba_hcdi_get_device_private(ph->p_usba_device); 747 ep = xhci_endpoint_pipe_to_epid(ph); 748 if (xd->xd_endpoints[ep] == NULL) { 749 mutex_exit(&xhcip->xhci_lock); 750 xhci_error(xhcip, "asked to do control transfer on slot %d, " 751 "port %d, endpoint: %d, but no endpoint structure", 752 xd->xd_slot, xd->xd_port, ep); 753 return (USB_FAILURE); 754 } 755 xep = xd->xd_endpoints[ep]; 756 757 /* 758 * There are several types of requests that we have to handle in special 759 * ways in xHCI. If we have one of those requests, then we don't 760 * necessarily go through the normal path. These special cases are all 761 * documented in xHCI 1.1 / 4.5.4. 762 * 763 * Looking at that, you may ask why aren't SET_CONFIGURATION and SET_IF 764 * special cased here. This action is a little confusing by default. The 765 * xHCI specification requires that we may need to issue a configure 766 * endpoint command as part of this. However, the xHCI 1.1 / 4.5.4.2 767 * states that we don't actually need to if nothing in the endpoint 768 * configuration context has changed. Because nothing in it should have 769 * changed as part of this, we don't need to do anything and instead 770 * just can issue the request normally. We're also assuming in the 771 * USB_REQ_SET_IF case that if something's changing the interface, the 772 * non-default endpoint will have yet to be opened. 773 */ 774 if (ucrp->ctrl_bmRequestType == USB_DEV_REQ_HOST_TO_DEV && 775 ucrp->ctrl_bRequest == USB_REQ_SET_ADDRESS) { 776 /* 777 * As we've defined an explicit set-address endpoint, we should 778 * never call this function. If we get here, always fail. 779 */ 780 mutex_exit(&xhcip->xhci_lock); 781 usba_hcdi_cb(ph, (usb_opaque_t)ucrp, USB_CR_NOT_SUPPORTED); 782 return (USB_SUCCESS); 783 } 784 785 mutex_exit(&xhcip->xhci_lock); 786 787 /* 788 * Allocate the transfer memory, etc. 789 */ 790 xt = xhci_transfer_alloc(xhcip, xep, ucrp->ctrl_wLength, 2, usb_flags); 791 if (xt == NULL) { 792 return (USB_NO_RESOURCES); 793 } 794 xt->xt_usba_req = (usb_opaque_t)ucrp; 795 xt->xt_timeout = ucrp->ctrl_timeout; 796 if (xt->xt_timeout == 0) { 797 xt->xt_timeout = HCDI_DEFAULT_TIMEOUT; 798 } 799 800 if (ucrp->ctrl_wLength > 0) { 801 if ((ucrp->ctrl_bmRequestType & USB_DEV_REQ_DEV_TO_HOST) != 0) { 802 trt = XHCI_TRB_TRT_IN; 803 datain = B_TRUE; 804 statusdir = 0; 805 } else { 806 trt = XHCI_TRB_TRT_OUT; 807 datain = B_FALSE; 808 statusdir = XHCI_TRB_DIR_IN; 809 810 xhci_transfer_copy(xt, ucrp->ctrl_data->b_rptr, 811 ucrp->ctrl_wLength, B_FALSE); 812 if (xhci_transfer_sync(xhcip, xt, 813 DDI_DMA_SYNC_FORDEV) != DDI_FM_OK) { 814 xhci_transfer_free(xhcip, xt); 815 xhci_error(xhcip, "failed to synchronize ctrl " 816 "transfer DMA memory on endpoint %u of " 817 "device on slot %d and port %d: resetting " 818 "device", xep->xep_num, xd->xd_slot, 819 xd->xd_port); 820 xhci_fm_runtime_reset(xhcip); 821 return (USB_HC_HARDWARE_ERROR); 822 } 823 } 824 } else { 825 trt = 0; 826 datain = B_FALSE; 827 statusdir = XHCI_TRB_DIR_IN; 828 } 829 830 /* 831 * We always fill in the required setup and status TRBs ourselves; 832 * however, to minimize our knowledge about how the data has been split 833 * across multiple DMA cookies in an SGL, we leave that to the transfer 834 * logic to fill in. 835 */ 836 xt->xt_trbs[0].trb_addr = xhci_hcdi_ctrl_req_to_trb(ucrp); 837 xt->xt_trbs[0].trb_status = LE_32(XHCI_TRB_LEN(8) | XHCI_TRB_INTR(0)); 838 xt->xt_trbs[0].trb_flags = LE_32(trt | XHCI_TRB_IDT | 839 XHCI_TRB_TYPE_SETUP); 840 841 if (ucrp->ctrl_wLength > 0) 842 xhci_transfer_trb_fill_data(xep, xt, 1, datain); 843 844 xt->xt_trbs[xt->xt_ntrbs - 1].trb_addr = 0; 845 xt->xt_trbs[xt->xt_ntrbs - 1].trb_status = LE_32(XHCI_TRB_INTR(0)); 846 xt->xt_trbs[xt->xt_ntrbs - 1].trb_flags = LE_32(XHCI_TRB_TYPE_STATUS | 847 XHCI_TRB_IOC | statusdir); 848 849 850 mutex_enter(&xhcip->xhci_lock); 851 852 /* 853 * Schedule the transfer, allocating resources in the process. 854 */ 855 if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) { 856 xhci_transfer_free(xhcip, xt); 857 mutex_exit(&xhcip->xhci_lock); 858 return (USB_NO_RESOURCES); 859 } 860 861 mutex_exit(&xhcip->xhci_lock); 862 863 return (USB_SUCCESS); 864 } 865 866 /* 867 * This request is trying to get the upper bound on the amount of data we're 868 * willing transfer in one go. Note that this amount can be broken down into 869 * multiple SGL entries, this interface doesn't particularly care about that. 870 */ 871 /* ARGSUSED */ 872 static int 873 xhci_hcdi_bulk_transfer_size(usba_device_t *ud, size_t *sizep) 874 { 875 if (sizep != NULL) 876 *sizep = XHCI_MAX_TRANSFER; 877 return (USB_SUCCESS); 878 } 879 880 /* 881 * Perform a bulk transfer. This is a pretty straightforward action. We 882 * basically just allocate the appropriate transfer and try to schedule it, 883 * hoping there is enough space. 884 */ 885 static int 886 xhci_hcdi_pipe_bulk_xfer(usba_pipe_handle_data_t *ph, usb_bulk_req_t *ubrp, 887 usb_flags_t usb_flags) 888 { 889 uint_t epid; 890 xhci_device_t *xd; 891 xhci_endpoint_t *xep; 892 xhci_transfer_t *xt; 893 boolean_t datain; 894 895 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph); 896 897 mutex_enter(&xhcip->xhci_lock); 898 if (xhcip->xhci_state & XHCI_S_ERROR) { 899 mutex_exit(&xhcip->xhci_lock); 900 return (USB_HC_HARDWARE_ERROR); 901 } 902 903 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) { 904 mutex_exit(&xhcip->xhci_lock); 905 return (USB_NOT_SUPPORTED); 906 } 907 908 xd = usba_hcdi_get_device_private(ph->p_usba_device); 909 epid = xhci_endpoint_pipe_to_epid(ph); 910 if (xd->xd_endpoints[epid] == NULL) { 911 mutex_exit(&xhcip->xhci_lock); 912 xhci_error(xhcip, "asked to do bulk transfer on slot %d, " 913 "port %d, endpoint: %d, but no endpoint structure", 914 xd->xd_slot, xd->xd_port, epid); 915 return (USB_FAILURE); 916 } 917 xep = xd->xd_endpoints[epid]; 918 mutex_exit(&xhcip->xhci_lock); 919 920 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) { 921 datain = B_TRUE; 922 } else { 923 datain = B_FALSE; 924 } 925 926 xt = xhci_transfer_alloc(xhcip, xep, ubrp->bulk_len, 0, usb_flags); 927 if (xt == NULL) { 928 return (USB_NO_RESOURCES); 929 } 930 xt->xt_usba_req = (usb_opaque_t)ubrp; 931 xt->xt_timeout = ubrp->bulk_timeout; 932 if (xt->xt_timeout == 0) { 933 xt->xt_timeout = HCDI_DEFAULT_TIMEOUT; 934 } 935 936 if (ubrp->bulk_len > 0 && datain == B_FALSE) { 937 xhci_transfer_copy(xt, ubrp->bulk_data->b_rptr, ubrp->bulk_len, 938 B_FALSE); 939 if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) != 940 DDI_FM_OK) { 941 xhci_transfer_free(xhcip, xt); 942 xhci_error(xhcip, "failed to synchronize bulk " 943 "transfer DMA memory on endpoint %u of " 944 "device on slot %d and port %d: resetting " 945 "device", xep->xep_num, xd->xd_slot, 946 xd->xd_port); 947 xhci_fm_runtime_reset(xhcip); 948 return (USB_HC_HARDWARE_ERROR); 949 } 950 } 951 952 xhci_transfer_trb_fill_data(xep, xt, 0, datain); 953 mutex_enter(&xhcip->xhci_lock); 954 if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) { 955 xhci_transfer_free(xhcip, xt); 956 mutex_exit(&xhcip->xhci_lock); 957 return (USB_NO_RESOURCES); 958 } 959 mutex_exit(&xhcip->xhci_lock); 960 961 return (USB_SUCCESS); 962 } 963 964 static void 965 xhci_hcdi_isoc_transfer_fill(xhci_device_t *xd, xhci_endpoint_t *xep, 966 xhci_transfer_t *xt, usb_isoc_req_t *usrp) 967 { 968 int i; 969 uintptr_t buf; 970 971 buf = xt->xt_buffer.xdb_cookies[0].dmac_laddress; 972 for (i = 0; i < usrp->isoc_pkts_count; i++) { 973 int flags; 974 uint_t tbc, tlbpc; 975 976 ushort_t len = usrp->isoc_pkt_descr[i].isoc_pkt_length; 977 xhci_trb_t *trb = &xt->xt_trbs[i]; 978 979 trb->trb_addr = LE_64(buf); 980 981 /* 982 * Because we know that a single frame can have all of its data 983 * in a single instance, we know that we don't need to do 984 * anything special here. 985 */ 986 trb->trb_status = LE_32(XHCI_TRB_LEN(len) | XHCI_TRB_TDREM(0) | 987 XHCI_TRB_INTR(0)); 988 989 /* 990 * Always enable SIA to start the frame ASAP. We also always 991 * enable an interrupt on a short packet. If this is the last 992 * trb, then we will set IOC. Each TRB created here is really 993 * its own TD. However, we only set an interrupt on the last 994 * entry to better deal with scheduling. 995 */ 996 flags = XHCI_TRB_SIA | XHCI_TRB_ISP | XHCI_TRB_SET_FRAME(0); 997 flags |= XHCI_TRB_TYPE_ISOCH; 998 999 if (i + 1 == usrp->isoc_pkts_count) 1000 flags |= XHCI_TRB_IOC; 1001 1002 /* 1003 * Now we need to calculate the TBC and the TLBPC. 1004 */ 1005 xhci_transfer_calculate_isoc(xd, xep, len, &tbc, &tlbpc); 1006 flags |= XHCI_TRB_SET_TBC(tbc); 1007 flags |= XHCI_TRB_SET_TLBPC(tlbpc); 1008 1009 trb->trb_flags = LE_32(flags); 1010 buf += len; 1011 1012 /* 1013 * Go through and copy the required data to our local copy of 1014 * the isoc descriptor. By default, we assume that all data will 1015 * be copied and the status set to OK. This mirrors the fact 1016 * that we won't get a notification unless there's been an 1017 * error or short packet transfer. 1018 */ 1019 xt->xt_isoc[i].isoc_pkt_length = len; 1020 xt->xt_isoc[i].isoc_pkt_actual_length = len; 1021 xt->xt_isoc[i].isoc_pkt_status = USB_CR_OK; 1022 } 1023 } 1024 1025 /* 1026 * Initialize periodic IN requests (both interrupt and isochronous) 1027 */ 1028 static int 1029 xhci_hcdi_periodic_init(xhci_t *xhcip, usba_pipe_handle_data_t *ph, 1030 usb_opaque_t usb_req, size_t len, int usb_flags) 1031 { 1032 int i, ret; 1033 uint_t epid; 1034 xhci_device_t *xd; 1035 xhci_endpoint_t *xep; 1036 xhci_pipe_t *xp; 1037 xhci_periodic_pipe_t *xpp; 1038 1039 mutex_enter(&xhcip->xhci_lock); 1040 if (xhcip->xhci_state & XHCI_S_ERROR) { 1041 mutex_exit(&xhcip->xhci_lock); 1042 return (USB_HC_HARDWARE_ERROR); 1043 } 1044 1045 xd = usba_hcdi_get_device_private(ph->p_usba_device); 1046 epid = xhci_endpoint_pipe_to_epid(ph); 1047 if (xd->xd_endpoints[epid] == NULL) { 1048 xhci_error(xhcip, "asked to do periodic transfer on slot %d, " 1049 "port %d, endpoint: %d, but no endpoint structure", 1050 xd->xd_slot, xd->xd_port, epid); 1051 mutex_exit(&xhcip->xhci_lock); 1052 return (USB_FAILURE); 1053 } 1054 xep = xd->xd_endpoints[epid]; 1055 xp = (xhci_pipe_t *)ph->p_hcd_private; 1056 if (xp == NULL) { 1057 xhci_error(xhcip, "asked to do periodic transfer on slot %d, " 1058 "port %d, endpoint: %d, but no pipe structure", 1059 xd->xd_slot, xd->xd_port, epid); 1060 mutex_exit(&xhcip->xhci_lock); 1061 return (USB_FAILURE); 1062 } 1063 xpp = &xp->xp_periodic; 1064 1065 /* 1066 * Only allow a single polling request at any given time. 1067 */ 1068 if (xpp->xpp_usb_req != NULL) { 1069 mutex_exit(&xhcip->xhci_lock); 1070 return (USB_BUSY); 1071 } 1072 1073 /* 1074 * We keep allocations around in case we restart polling, which most 1075 * devices do (not really caring about a lost event). However, we don't 1076 * support a driver changing that size on us, which it probably won't. 1077 * If we stumble across driver that does, then this will need to become 1078 * a lot more complicated. 1079 */ 1080 if (xpp->xpp_tsize > 0 && xpp->xpp_tsize < len) { 1081 mutex_exit(&xhcip->xhci_lock); 1082 return (USB_INVALID_REQUEST); 1083 } 1084 1085 if (xpp->xpp_tsize == 0) { 1086 int ntrbs; 1087 int ntransfers; 1088 1089 /* 1090 * What we allocate varies based on whether or not this is an 1091 * isochronous or interrupt IN periodic. 1092 */ 1093 if (xep->xep_type == USB_EP_ATTR_INTR) { 1094 ntrbs = 0; 1095 ntransfers = XHCI_INTR_IN_NTRANSFERS; 1096 } else { 1097 usb_isoc_req_t *usrp; 1098 ASSERT(xep->xep_type == USB_EP_ATTR_ISOCH); 1099 1100 usrp = (usb_isoc_req_t *)usb_req; 1101 ntrbs = usrp->isoc_pkts_count; 1102 ntransfers = XHCI_ISOC_IN_NTRANSFERS; 1103 } 1104 1105 xpp->xpp_tsize = len; 1106 xpp->xpp_ntransfers = ntransfers; 1107 1108 for (i = 0; i < xpp->xpp_ntransfers; i++) { 1109 xhci_transfer_t *xt = xhci_transfer_alloc(xhcip, xep, 1110 len, ntrbs, usb_flags); 1111 if (xt == NULL) { 1112 xhci_hcdi_periodic_free(xhcip, xp); 1113 mutex_exit(&xhcip->xhci_lock); 1114 return (USB_NO_RESOURCES); 1115 } 1116 1117 if (xep->xep_type == USB_EP_ATTR_INTR) { 1118 xhci_transfer_trb_fill_data(xep, xt, 0, B_TRUE); 1119 } else { 1120 usb_isoc_req_t *usrp; 1121 usrp = (usb_isoc_req_t *)usb_req; 1122 xhci_hcdi_isoc_transfer_fill(xd, xep, xt, usrp); 1123 xt->xt_data_tohost = B_TRUE; 1124 } 1125 xpp->xpp_transfers[i] = xt; 1126 } 1127 } 1128 1129 /* 1130 * Mark the endpoint as periodic so we don't have timeouts at play. 1131 */ 1132 xep->xep_state |= XHCI_ENDPOINT_PERIODIC; 1133 1134 /* 1135 * Now that we've allocated everything, go ahead and schedule them and 1136 * kick off the ring. 1137 */ 1138 for (i = 0; i < xpp->xpp_ntransfers; i++) { 1139 int ret; 1140 ret = xhci_endpoint_schedule(xhcip, xd, xep, 1141 xpp->xpp_transfers[i], B_FALSE); 1142 if (ret != 0) { 1143 (void) xhci_ring_reset(xhcip, &xep->xep_ring); 1144 xep->xep_state &= ~XHCI_ENDPOINT_PERIODIC; 1145 mutex_exit(&xhcip->xhci_lock); 1146 return (ret); 1147 } 1148 } 1149 1150 /* 1151 * Don't worry about freeing memory, it'll be done when the endpoint 1152 * closes and the whole system is reset. 1153 */ 1154 xpp->xpp_usb_req = usb_req; 1155 xpp->xpp_poll_state = XHCI_PERIODIC_POLL_ACTIVE; 1156 1157 ret = xhci_endpoint_ring(xhcip, xd, xep); 1158 mutex_exit(&xhcip->xhci_lock); 1159 return (ret); 1160 } 1161 1162 static int 1163 xhci_hcdi_intr_oneshot(xhci_t *xhcip, usba_pipe_handle_data_t *ph, 1164 usb_intr_req_t *uirp, usb_flags_t usb_flags) 1165 { 1166 uint_t epid; 1167 xhci_device_t *xd; 1168 xhci_endpoint_t *xep; 1169 xhci_transfer_t *xt; 1170 boolean_t datain; 1171 mblk_t *mp = NULL; 1172 1173 mutex_enter(&xhcip->xhci_lock); 1174 if (xhcip->xhci_state & XHCI_S_ERROR) { 1175 mutex_exit(&xhcip->xhci_lock); 1176 return (USB_HC_HARDWARE_ERROR); 1177 } 1178 1179 xd = usba_hcdi_get_device_private(ph->p_usba_device); 1180 epid = xhci_endpoint_pipe_to_epid(ph); 1181 if (xd->xd_endpoints[epid] == NULL) { 1182 xhci_error(xhcip, "asked to do interrupt transfer on slot %d, " 1183 "port %d, endpoint: %d, but no endpoint structure", 1184 xd->xd_slot, xd->xd_port, epid); 1185 mutex_exit(&xhcip->xhci_lock); 1186 return (USB_FAILURE); 1187 } 1188 xep = xd->xd_endpoints[epid]; 1189 1190 mutex_exit(&xhcip->xhci_lock); 1191 1192 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) { 1193 datain = B_TRUE; 1194 } else { 1195 datain = B_FALSE; 1196 } 1197 1198 xt = xhci_transfer_alloc(xhcip, xep, uirp->intr_len, 0, usb_flags); 1199 if (xt == NULL) { 1200 return (USB_NO_RESOURCES); 1201 } 1202 1203 xt->xt_usba_req = (usb_opaque_t)uirp; 1204 xt->xt_timeout = uirp->intr_timeout; 1205 if (xt->xt_timeout == 0) { 1206 xt->xt_timeout = HCDI_DEFAULT_TIMEOUT; 1207 } 1208 1209 /* 1210 * Unlike other request types, USB Interrupt-IN requests aren't required 1211 * to have allocated the message block for data. If they haven't, we 1212 * take care of that now. 1213 */ 1214 if (uirp->intr_len > 0 && datain == B_TRUE && uirp->intr_data == NULL) { 1215 if (usb_flags & USB_FLAGS_SLEEP) { 1216 mp = allocb_wait(uirp->intr_len, BPRI_LO, STR_NOSIG, 1217 NULL); 1218 } else { 1219 mp = allocb(uirp->intr_len, 0); 1220 } 1221 if (mp == NULL) { 1222 xhci_transfer_free(xhcip, xt); 1223 mutex_exit(&xhcip->xhci_lock); 1224 return (USB_NO_RESOURCES); 1225 } 1226 uirp->intr_data = mp; 1227 } 1228 1229 if (uirp->intr_len > 0 && datain == B_FALSE) { 1230 xhci_transfer_copy(xt, uirp->intr_data->b_rptr, uirp->intr_len, 1231 B_FALSE); 1232 if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) != 1233 DDI_FM_OK) { 1234 xhci_transfer_free(xhcip, xt); 1235 xhci_error(xhcip, "failed to synchronize interrupt " 1236 "transfer DMA memory on endpoint %u of " 1237 "device on slot %d and port %d: resetting " 1238 "device", xep->xep_num, xd->xd_slot, 1239 xd->xd_port); 1240 xhci_fm_runtime_reset(xhcip); 1241 return (USB_HC_HARDWARE_ERROR); 1242 } 1243 } 1244 1245 xhci_transfer_trb_fill_data(xep, xt, 0, datain); 1246 mutex_enter(&xhcip->xhci_lock); 1247 if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) { 1248 if (mp != NULL) { 1249 uirp->intr_data = NULL; 1250 freemsg(mp); 1251 } 1252 xhci_transfer_free(xhcip, xt); 1253 mutex_exit(&xhcip->xhci_lock); 1254 return (USB_NO_RESOURCES); 1255 } 1256 mutex_exit(&xhcip->xhci_lock); 1257 1258 return (USB_SUCCESS); 1259 } 1260 1261 /* 1262 * We've been asked to perform an interrupt transfer. When this is an interrupt 1263 * IN endpoint, that means that the hcd is being asked to start polling on the 1264 * endpoint. When the endpoint is the root hub, it effectively becomes synthetic 1265 * polling. 1266 * 1267 * When we have an interrupt out endpoint, then this is just a single simple 1268 * interrupt request that we send out and there isn't much special to do beyond 1269 * the normal activity. 1270 */ 1271 static int 1272 xhci_hcdi_pipe_intr_xfer(usba_pipe_handle_data_t *ph, usb_intr_req_t *uirp, 1273 usb_flags_t usb_flags) 1274 { 1275 int ret; 1276 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph); 1277 1278 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) { 1279 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) { 1280 ret = xhci_root_hub_intr_root_enable(xhcip, ph, uirp); 1281 } else if (uirp->intr_attributes & USB_ATTRS_ONE_XFER) { 1282 ret = xhci_hcdi_intr_oneshot(xhcip, ph, uirp, 1283 usb_flags); 1284 } else { 1285 ret = xhci_hcdi_periodic_init(xhcip, ph, 1286 (usb_opaque_t)uirp, uirp->intr_len, usb_flags); 1287 } 1288 } else { 1289 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) { 1290 return (USB_NOT_SUPPORTED); 1291 } 1292 ret = xhci_hcdi_intr_oneshot(xhcip, ph, uirp, usb_flags); 1293 } 1294 1295 return (ret); 1296 } 1297 1298 /* ARGSUSED */ 1299 static int 1300 xhci_hcdi_pipe_stop_intr_polling(usba_pipe_handle_data_t *ph, 1301 usb_flags_t usb_flags) 1302 { 1303 return (xhci_hcdi_pipe_poll_fini(ph, B_FALSE)); 1304 } 1305 1306 static int 1307 xhci_hcdi_isoc_periodic(xhci_t *xhcip, usba_pipe_handle_data_t *ph, 1308 usb_isoc_req_t *usrp, usb_flags_t usb_flags) 1309 { 1310 int i; 1311 size_t count; 1312 1313 count = 0; 1314 for (i = 0; i < usrp->isoc_pkts_count; i++) { 1315 count += usrp->isoc_pkt_descr[i].isoc_pkt_length; 1316 } 1317 1318 return (xhci_hcdi_periodic_init(xhcip, ph, (usb_opaque_t)usrp, count, 1319 usb_flags)); 1320 } 1321 1322 /* 1323 * This is used to create an isochronous request to send data out to the device. 1324 * This is a single one shot request, it is not something that we'll have to 1325 * repeat over and over. 1326 */ 1327 static int 1328 xhci_hcdi_isoc_oneshot(xhci_t *xhcip, usba_pipe_handle_data_t *ph, 1329 usb_isoc_req_t *usrp, usb_flags_t usb_flags) 1330 { 1331 int i, ret; 1332 uint_t epid; 1333 size_t count, mblen; 1334 xhci_device_t *xd; 1335 xhci_endpoint_t *xep; 1336 xhci_transfer_t *xt; 1337 1338 count = 0; 1339 for (i = 0; i < usrp->isoc_pkts_count; i++) { 1340 count += usrp->isoc_pkt_descr[i].isoc_pkt_length; 1341 } 1342 mblen = MBLKL(usrp->isoc_data); 1343 1344 if (count != mblen) { 1345 return (USB_INVALID_ARGS); 1346 } 1347 1348 mutex_enter(&xhcip->xhci_lock); 1349 if (xhcip->xhci_state & XHCI_S_ERROR) { 1350 mutex_exit(&xhcip->xhci_lock); 1351 return (USB_HC_HARDWARE_ERROR); 1352 } 1353 1354 xd = usba_hcdi_get_device_private(ph->p_usba_device); 1355 epid = xhci_endpoint_pipe_to_epid(ph); 1356 if (xd->xd_endpoints[epid] == NULL) { 1357 xhci_error(xhcip, "asked to do isochronous transfer on slot " 1358 "%d, port %d, endpoint: %d, but no endpoint structure", 1359 xd->xd_slot, xd->xd_port, epid); 1360 mutex_exit(&xhcip->xhci_lock); 1361 return (USB_FAILURE); 1362 } 1363 xep = xd->xd_endpoints[epid]; 1364 mutex_exit(&xhcip->xhci_lock); 1365 1366 xt = xhci_transfer_alloc(xhcip, xep, mblen, usrp->isoc_pkts_count, 1367 usb_flags); 1368 if (xt == NULL) { 1369 return (USB_NO_RESOURCES); 1370 } 1371 xt->xt_usba_req = (usb_opaque_t)usrp; 1372 1373 /* 1374 * USBA doesn't provide any real way for a timeout to be defined for an 1375 * isochronous event. However, since we technically aren't a periodic 1376 * endpoint, go ahead and always set the default timeout. It's better 1377 * than nothing. 1378 */ 1379 xt->xt_timeout = HCDI_DEFAULT_TIMEOUT; 1380 1381 xhci_transfer_copy(xt, usrp->isoc_data->b_rptr, mblen, B_FALSE); 1382 if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) != DDI_FM_OK) { 1383 xhci_transfer_free(xhcip, xt); 1384 xhci_error(xhcip, "failed to synchronize isochronous " 1385 "transfer DMA memory on endpoint %u of " 1386 "device on slot %d and port %d: resetting " 1387 "device", xep->xep_num, xd->xd_slot, 1388 xd->xd_port); 1389 xhci_fm_runtime_reset(xhcip); 1390 return (USB_HC_HARDWARE_ERROR); 1391 } 1392 1393 /* 1394 * Fill in the ISOC data. Note, that we always use ASAP scheduling and 1395 * we don't support specifying the frame at this time, for better or 1396 * worse. 1397 */ 1398 xhci_hcdi_isoc_transfer_fill(xd, xep, xt, usrp); 1399 1400 mutex_enter(&xhcip->xhci_lock); 1401 ret = xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE); 1402 mutex_exit(&xhcip->xhci_lock); 1403 1404 return (ret); 1405 } 1406 1407 static int 1408 xhci_hcdi_pipe_isoc_xfer(usba_pipe_handle_data_t *ph, usb_isoc_req_t *usrp, 1409 usb_flags_t usb_flags) 1410 { 1411 int ret; 1412 xhci_t *xhcip; 1413 1414 xhcip = xhci_hcdi_get_xhcip(ph); 1415 1416 /* 1417 * We don't support isochronous transactions on the root hub at all. 1418 * Always fail them if for some reason we end up here. 1419 */ 1420 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) { 1421 return (USB_NOT_SUPPORTED); 1422 } 1423 1424 /* 1425 * We do not support being asked to set the frame ID at this time. We 1426 * require that everything specify the attribute 1427 * USB_ATTRS_ISOC_XFER_ASAP. 1428 */ 1429 if (!(usrp->isoc_attributes & USB_ATTRS_ISOC_XFER_ASAP)) { 1430 return (USB_NOT_SUPPORTED); 1431 } 1432 1433 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) { 1434 /* 1435 * Note, there is no such thing as a non-periodic isochronous 1436 * incoming transfer. 1437 */ 1438 ret = xhci_hcdi_isoc_periodic(xhcip, ph, usrp, usb_flags); 1439 } else { 1440 ret = xhci_hcdi_isoc_oneshot(xhcip, ph, usrp, usb_flags); 1441 } 1442 1443 return (ret); 1444 } 1445 1446 /* ARGSUSED */ 1447 static int 1448 xhci_hcdi_pipe_stop_isoc_polling(usba_pipe_handle_data_t *ph, 1449 usb_flags_t usb_flags) 1450 { 1451 return (xhci_hcdi_pipe_poll_fini(ph, B_FALSE)); 1452 } 1453 1454 /* 1455 * This is asking us for the current frame number. The USBA expects this to 1456 * actually be a bit of a fiction, as it tries to maintain a frame number well 1457 * beyond what the hardware actually contains in its registers. Hardware 1458 * basically has a 14-bit counter, whereas we need to have a constant amount of 1459 * milliseconds. 1460 * 1461 * Today, no client drivers actually use this API and everyone specifies the 1462 * attribute to say that we should schedule things ASAP. So until we have some 1463 * real device that want this functionality, we're going to fail. 1464 */ 1465 /* ARGSUSED */ 1466 static int 1467 xhci_hcdi_get_current_frame_number(usba_device_t *usba_device, 1468 usb_frame_number_t *frame_number) 1469 { 1470 return (USB_FAILURE); 1471 } 1472 1473 /* 1474 * See the comments around the XHCI_ISOC_MAX_TRB macro for more information. 1475 */ 1476 /* ARGSUSED */ 1477 static int 1478 xhci_hcdi_get_max_isoc_pkts(usba_device_t *usba_device, 1479 uint_t *max_isoc_pkts_per_request) 1480 { 1481 *max_isoc_pkts_per_request = XHCI_ISOC_MAX_TRB; 1482 return (USB_SUCCESS); 1483 } 1484 1485 /* 1486 * VERSION 2 ops and helpers 1487 */ 1488 1489 static void 1490 xhci_hcdi_device_free(xhci_device_t *xd) 1491 { 1492 xhci_dma_free(&xd->xd_ictx); 1493 xhci_dma_free(&xd->xd_octx); 1494 mutex_destroy(&xd->xd_imtx); 1495 kmem_free(xd, sizeof (xhci_device_t)); 1496 } 1497 1498 /* 1499 * Calculate the device's route string. In USB 3.0 the route string is a 20-bit 1500 * number. Each four bits represent a port number attached to a deeper hub. 1501 * Particularly it represents the port on that current hub that you need to go 1502 * down to reach the next device. Bits 0-3 represent the first *external* hub. 1503 * So a device connected to a root hub has a route string of zero. Imagine the 1504 * following set of devices: 1505 * 1506 * . port 2 . port 5 1507 * . . 1508 * +----------+ . +--------+ . +-------+ 1509 * | root hub |-*->| hub 1 |-*->| hub 2 | 1510 * +----------+ +--------+ +-------+ 1511 * * . port 12 * . port 8 * . port 1 1512 * v v v 1513 * +-------+ +-------+ +-------+ 1514 * | dev a | | dev b | | dev c | 1515 * +-------+ +-------+ +-------+ 1516 * 1517 * So, based on the above diagram, device a should have a route string of 0, 1518 * because it's directly connected to the root port. Device b would simply have 1519 * a route string of 8. This is because it travels through one non-root hub, hub 1520 * 1, and it does so on port 8. The root ports basically don't matter. Device c 1521 * would then have a route string of 0x15, as it's first traversing through hub 1522 * 1 on port 2 and then hub 2 on port 5. 1523 * 1524 * Finally, it's worth mentioning that because it's a four bit field, if for 1525 * some reason a device has more than 15 ports, we just treat the value as 15. 1526 * 1527 * Note, as part of this, we also grab what port on the root hub this whole 1528 * chain is on, as we're required to store that information in the slot context. 1529 */ 1530 static void 1531 xhci_hcdi_device_route(usba_device_t *ud, uint32_t *routep, uint32_t *root_port) 1532 { 1533 uint32_t route = 0; 1534 usba_device_t *hub = ud->usb_parent_hub; 1535 usba_device_t *port_dev = ud; 1536 1537 ASSERT(hub != NULL); 1538 1539 /* 1540 * Iterate over every hub, updating the route as we go. When we 1541 * encounter a hub without a parent, then we're at the root hub. At 1542 * which point, the port we want is on port_dev (the child of hub). 1543 */ 1544 while (hub->usb_parent_hub != NULL) { 1545 uint32_t p; 1546 1547 p = port_dev->usb_port; 1548 if (p > 15) 1549 p = 15; 1550 route <<= 4; 1551 route |= p & 0xf; 1552 port_dev = hub; 1553 hub = hub->usb_parent_hub; 1554 } 1555 1556 ASSERT(port_dev->usb_parent_hub == hub); 1557 *root_port = port_dev->usb_port; 1558 *routep = XHCI_ROUTE_MASK(route); 1559 } 1560 1561 /* 1562 * If a low or full speed device is behind a high-speed device that is not a 1563 * root hub, then we must include the port and slot of that device. USBA already 1564 * stores this device in the usb_hs_hub_usba_dev member. 1565 */ 1566 static uint32_t 1567 xhci_hcdi_device_tt(usba_device_t *ud) 1568 { 1569 uint32_t ret; 1570 xhci_device_t *xd; 1571 1572 if (ud->usb_port_status >= USBA_HIGH_SPEED_DEV) 1573 return (0); 1574 1575 if (ud->usb_hs_hub_usba_dev == NULL) 1576 return (0); 1577 1578 ASSERT(ud->usb_hs_hub_usba_dev != NULL); 1579 ASSERT(ud->usb_hs_hub_usba_dev->usb_parent_hub != NULL); 1580 xd = usba_hcdi_get_device_private(ud->usb_hs_hub_usba_dev); 1581 ASSERT(xd != NULL); 1582 1583 ret = XHCI_SCTX_SET_TT_HUB_SID(xd->xd_slot); 1584 ret |= XHCI_SCTX_SET_TT_PORT_NUM(ud->usb_hs_hub_usba_dev->usb_port); 1585 1586 return (ret); 1587 } 1588 1589 /* 1590 * Initialize a new device. This allocates a device slot from the controller, 1591 * which tranfers it to our control. 1592 */ 1593 static int 1594 xhci_hcdi_device_init(usba_device_t *ud, usb_port_t port, void **hcdpp) 1595 { 1596 int ret, i; 1597 xhci_device_t *xd; 1598 ddi_device_acc_attr_t acc; 1599 ddi_dma_attr_t attr; 1600 xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud); 1601 size_t isize, osize, incr; 1602 uint32_t route, rp, info, info2, tt; 1603 1604 xd = kmem_zalloc(sizeof (xhci_device_t), KM_SLEEP); 1605 xd->xd_port = port; 1606 xd->xd_usbdev = ud; 1607 mutex_init(&xd->xd_imtx, NULL, MUTEX_DRIVER, 1608 (void *)(uintptr_t)xhcip->xhci_intr_pri); 1609 1610 /* 1611 * The size of the context structures is based upon the presence of the 1612 * context flag which determines whether we have a 32-byte or 64-byte 1613 * context. Note that the input context always has to account for the 1614 * entire size of the xhci_input_contex_t, which is 32-bytes by default. 1615 */ 1616 if (xhcip->xhci_caps.xcap_flags & XCAP_CSZ) { 1617 incr = 64; 1618 osize = XHCI_DEVICE_CONTEXT_64; 1619 isize = XHCI_DEVICE_CONTEXT_64 + incr; 1620 } else { 1621 incr = 32; 1622 osize = XHCI_DEVICE_CONTEXT_32; 1623 isize = XHCI_DEVICE_CONTEXT_32 + incr; 1624 } 1625 1626 xhci_dma_acc_attr(xhcip, &acc); 1627 xhci_dma_dma_attr(xhcip, &attr); 1628 if (xhci_dma_alloc(xhcip, &xd->xd_ictx, &attr, &acc, B_TRUE, 1629 isize, B_FALSE) == B_FALSE) { 1630 xhci_hcdi_device_free(xd); 1631 return (USB_NO_RESOURCES); 1632 } 1633 1634 xd->xd_input = (xhci_input_context_t *)xd->xd_ictx.xdb_va; 1635 xd->xd_slotin = (xhci_slot_context_t *)(xd->xd_ictx.xdb_va + incr); 1636 for (i = 0; i < XHCI_NUM_ENDPOINTS; i++) { 1637 xd->xd_endin[i] = 1638 (xhci_endpoint_context_t *)(xd->xd_ictx.xdb_va + 1639 (i + 2) * incr); 1640 } 1641 1642 if (xhci_dma_alloc(xhcip, &xd->xd_octx, &attr, &acc, B_TRUE, 1643 osize, B_FALSE) == B_FALSE) { 1644 xhci_hcdi_device_free(xd); 1645 return (USB_NO_RESOURCES); 1646 } 1647 xd->xd_slotout = (xhci_slot_context_t *)xd->xd_octx.xdb_va; 1648 for (i = 0; i < XHCI_NUM_ENDPOINTS; i++) { 1649 xd->xd_endout[i] = 1650 (xhci_endpoint_context_t *)(xd->xd_octx.xdb_va + 1651 (i + 1) * incr); 1652 } 1653 1654 ret = xhci_command_enable_slot(xhcip, &xd->xd_slot); 1655 if (ret != USB_SUCCESS) { 1656 xhci_hcdi_device_free(xd); 1657 return (ret); 1658 } 1659 1660 /* 1661 * These are the default slot context and the endpoint zero context that 1662 * we're enabling. See 4.3.3. 1663 */ 1664 xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0) | 1665 XHCI_INCTX_MASK_DCI(1)); 1666 1667 /* 1668 * Note, we never need to set the MTT bit as illumos never enables the 1669 * alternate MTT interface. 1670 */ 1671 xhci_hcdi_device_route(ud, &route, &rp); 1672 info = XHCI_SCTX_SET_ROUTE(route) | XHCI_SCTX_SET_DCI(1); 1673 switch (ud->usb_port_status) { 1674 case USBA_LOW_SPEED_DEV: 1675 info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_LOW); 1676 break; 1677 case USBA_HIGH_SPEED_DEV: 1678 info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_HIGH); 1679 break; 1680 case USBA_FULL_SPEED_DEV: 1681 info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_FULL); 1682 break; 1683 case USBA_SUPER_SPEED_DEV: 1684 default: 1685 info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_SUPER); 1686 break; 1687 } 1688 info2 = XHCI_SCTX_SET_RHPORT(rp); 1689 tt = XHCI_SCTX_SET_IRQ_TARGET(0); 1690 tt |= xhci_hcdi_device_tt(ud); 1691 1692 xd->xd_slotin->xsc_info = LE_32(info); 1693 xd->xd_slotin->xsc_info2 = LE_32(info2); 1694 xd->xd_slotin->xsc_tt = LE_32(tt); 1695 1696 if ((ret = xhci_endpoint_init(xhcip, xd, NULL)) != 0) { 1697 (void) xhci_command_disable_slot(xhcip, xd->xd_slot); 1698 xhci_hcdi_device_free(xd); 1699 return (USB_HC_HARDWARE_ERROR); 1700 } 1701 1702 if (xhci_context_slot_output_init(xhcip, xd) != B_TRUE) { 1703 (void) xhci_command_disable_slot(xhcip, xd->xd_slot); 1704 xhci_endpoint_fini(xd, 0); 1705 xhci_hcdi_device_free(xd); 1706 return (USB_HC_HARDWARE_ERROR); 1707 } 1708 1709 if ((ret = xhci_command_set_address(xhcip, xd, B_TRUE)) != 0) { 1710 (void) xhci_command_disable_slot(xhcip, xd->xd_slot); 1711 xhci_context_slot_output_fini(xhcip, xd); 1712 xhci_endpoint_fini(xd, 0); 1713 xhci_hcdi_device_free(xd); 1714 return (ret); 1715 } 1716 1717 mutex_enter(&xhcip->xhci_lock); 1718 list_insert_tail(&xhcip->xhci_usba.xa_devices, xd); 1719 mutex_exit(&xhcip->xhci_lock); 1720 1721 *hcdpp = xd; 1722 return (ret); 1723 } 1724 1725 /* 1726 * We're tearing down a device now. 1727 */ 1728 static void 1729 xhci_hcdi_device_fini(usba_device_t *ud, void *hcdp) 1730 { 1731 int ret; 1732 xhci_endpoint_t *xep; 1733 xhci_device_t *xd; 1734 xhci_t *xhcip; 1735 1736 /* 1737 * Right now, it's theoretically possible that USBA may try and call 1738 * us here even if we hadn't successfully finished the device_init() 1739 * endpoint. We should probably modify the USBA to make sure that this 1740 * can't happen. 1741 */ 1742 if (hcdp == NULL) 1743 return; 1744 1745 xd = hcdp; 1746 xhcip = xhci_hcdi_get_xhcip_from_dev(ud); 1747 1748 /* 1749 * Make sure we have no timeout running on the default endpoint still. 1750 */ 1751 xep = xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT]; 1752 mutex_enter(&xhcip->xhci_lock); 1753 xep->xep_state |= XHCI_ENDPOINT_TEARDOWN; 1754 mutex_exit(&xhcip->xhci_lock); 1755 (void) untimeout(xep->xep_timeout); 1756 1757 /* 1758 * Go ahead and disable the slot. There's no reason to do anything 1759 * special about the default endpoint as it will be disabled as a part 1760 * of the slot disabling. However, if this all fails, we'll leave this 1761 * sitting here in a failed state, eating up a device slot. It is 1762 * unlikely this will occur. 1763 */ 1764 ret = xhci_command_disable_slot(xhcip, xd->xd_slot); 1765 if (ret != USB_SUCCESS) { 1766 xhci_error(xhcip, "failed to disable slot %d: %d", 1767 xd->xd_slot, ret); 1768 return; 1769 } 1770 1771 xhci_context_slot_output_fini(xhcip, xd); 1772 1773 /* 1774 * Once the slot is disabled, we can free any endpoints that were 1775 * opened. 1776 */ 1777 for (uint_t n = 0; n < XHCI_NUM_ENDPOINTS; n++) { 1778 if (xd->xd_endpoints[n] != NULL) { 1779 xhci_endpoint_fini(xd, n); 1780 } 1781 } 1782 1783 mutex_enter(&xhcip->xhci_lock); 1784 list_remove(&xhcip->xhci_usba.xa_devices, xd); 1785 mutex_exit(&xhcip->xhci_lock); 1786 1787 xhci_hcdi_device_free(xd); 1788 } 1789 1790 /* 1791 * Synchronously attempt to set the device address. For xHCI this involves it 1792 * deciding what address to use. 1793 */ 1794 static int 1795 xhci_hcdi_device_address(usba_device_t *ud) 1796 { 1797 int ret; 1798 xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud); 1799 xhci_device_t *xd = usba_hcdi_get_device_private(ud); 1800 xhci_endpoint_t *xep; 1801 1802 mutex_enter(&xhcip->xhci_lock); 1803 1804 /* 1805 * This device may already be addressed from the perspective of the xhci 1806 * controller. For example, the device this represents may have been 1807 * unconfigured, which does not actually remove the slot or other 1808 * information, merely tears down all the active use of it and the child 1809 * driver. In such cases, if we're already addressed, just return 1810 * success. The actual USB address is a fiction for USBA anyways. 1811 */ 1812 if (xd->xd_addressed == B_TRUE) { 1813 mutex_exit(&xhcip->xhci_lock); 1814 return (USB_SUCCESS); 1815 } 1816 1817 ASSERT(xd->xd_addressed == B_FALSE); 1818 xd->xd_addressed = B_TRUE; 1819 VERIFY3P(xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT], !=, NULL); 1820 xep = xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT]; 1821 mutex_exit(&xhcip->xhci_lock); 1822 1823 if ((ret = xhci_endpoint_setup_default_context(xhcip, xd, xep)) != 0) { 1824 ASSERT(ret == EIO); 1825 return (USB_HC_HARDWARE_ERROR); 1826 } 1827 1828 ret = xhci_command_set_address(xhcip, xd, B_FALSE); 1829 1830 if (ret != USB_SUCCESS) { 1831 mutex_enter(&xhcip->xhci_lock); 1832 xd->xd_addressed = B_FALSE; 1833 mutex_exit(&xhcip->xhci_lock); 1834 } 1835 1836 return (ret); 1837 } 1838 1839 /* 1840 * This is called relatively early on in a hub's life time. At this point, it's 1841 * descriptors have all been pulled and the default control pipe is still open. 1842 * What we need to do is go through and update the slot context to indicate that 1843 * this is a hub, otherwise, the controller will never let us speak to 1844 * downstream ports. 1845 */ 1846 static int 1847 xhci_hcdi_hub_update(usba_device_t *ud, uint8_t nports, uint8_t tt) 1848 { 1849 int ret; 1850 xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud); 1851 xhci_device_t *xd = usba_hcdi_get_device_private(ud); 1852 1853 if (xd == NULL) 1854 return (USB_FAILURE); 1855 1856 if (ud->usb_hubdi == NULL) { 1857 return (USB_FAILURE); 1858 } 1859 1860 mutex_enter(&xd->xd_imtx); 1861 1862 /* 1863 * Note, that usba never sets the interface of a hub to Multi TT. Hence 1864 * why we're never setting the MTT bit in xsc_info. 1865 */ 1866 xd->xd_slotin->xsc_info |= LE_32(XHCI_SCTX_SET_HUB(1)); 1867 xd->xd_slotin->xsc_info2 |= LE_32(XHCI_SCTX_SET_NPORTS(nports)); 1868 if (ud->usb_port_status == USBA_HIGH_SPEED_DEV) 1869 xd->xd_slotin->xsc_tt |= LE_32(XHCI_SCTX_SET_TT_THINK_TIME(tt)); 1870 1871 /* 1872 * We're only updating the slot context, no endpoint contexts should be 1873 * touched. 1874 */ 1875 xd->xd_input->xic_drop_flags = LE_32(0); 1876 xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0)); 1877 1878 ret = xhci_command_evaluate_context(xhcip, xd); 1879 mutex_exit(&xd->xd_imtx); 1880 return (ret); 1881 } 1882 1883 void 1884 xhci_hcd_fini(xhci_t *xhcip) 1885 { 1886 usba_hcdi_unregister(xhcip->xhci_dip); 1887 usba_free_hcdi_ops(xhcip->xhci_usba.xa_ops); 1888 list_destroy(&xhcip->xhci_usba.xa_pipes); 1889 list_destroy(&xhcip->xhci_usba.xa_devices); 1890 } 1891 1892 int 1893 xhci_hcd_init(xhci_t *xhcip) 1894 { 1895 usba_hcdi_register_args_t hreg; 1896 usba_hcdi_ops_t *ops; 1897 1898 ops = usba_alloc_hcdi_ops(); 1899 VERIFY(ops != NULL); 1900 1901 ops->usba_hcdi_ops_version = HCDI_OPS_VERSION; 1902 ops->usba_hcdi_dip = xhcip->xhci_dip; 1903 1904 ops->usba_hcdi_pm_support = xhci_hcdi_pm_support; 1905 ops->usba_hcdi_pipe_open = xhci_hcdi_pipe_open; 1906 ops->usba_hcdi_pipe_close = xhci_hcdi_pipe_close; 1907 ops->usba_hcdi_pipe_reset = xhci_hcdi_pipe_reset; 1908 ops->usba_hcdi_pipe_reset_data_toggle = 1909 xhci_hcdi_pipe_reset_data_toggle; 1910 ops->usba_hcdi_pipe_ctrl_xfer = xhci_hcdi_pipe_ctrl_xfer; 1911 ops->usba_hcdi_bulk_transfer_size = xhci_hcdi_bulk_transfer_size; 1912 ops->usba_hcdi_pipe_bulk_xfer = xhci_hcdi_pipe_bulk_xfer; 1913 ops->usba_hcdi_pipe_intr_xfer = xhci_hcdi_pipe_intr_xfer; 1914 ops->usba_hcdi_pipe_stop_intr_polling = 1915 xhci_hcdi_pipe_stop_intr_polling; 1916 ops->usba_hcdi_pipe_isoc_xfer = xhci_hcdi_pipe_isoc_xfer; 1917 ops->usba_hcdi_pipe_stop_isoc_polling = 1918 xhci_hcdi_pipe_stop_isoc_polling; 1919 ops->usba_hcdi_get_current_frame_number = 1920 xhci_hcdi_get_current_frame_number; 1921 ops->usba_hcdi_get_max_isoc_pkts = xhci_hcdi_get_max_isoc_pkts; 1922 ops->usba_hcdi_console_input_init = xhci_hcdi_console_input_init; 1923 ops->usba_hcdi_console_input_fini = xhci_hcdi_console_input_fini; 1924 ops->usba_hcdi_console_input_enter = xhci_hcdi_console_input_enter; 1925 ops->usba_hcdi_console_read = xhci_hcdi_console_read; 1926 ops->usba_hcdi_console_input_exit = xhci_hcdi_console_input_exit; 1927 1928 ops->usba_hcdi_console_output_init = xhci_hcdi_console_output_init; 1929 ops->usba_hcdi_console_output_fini = xhci_hcdi_console_output_fini; 1930 ops->usba_hcdi_console_output_enter = xhci_hcdi_console_output_enter; 1931 ops->usba_hcdi_console_write = xhci_hcdi_console_write; 1932 ops->usba_hcdi_console_output_exit = xhci_hcdi_console_output_exit; 1933 1934 ops->usba_hcdi_device_init = xhci_hcdi_device_init; 1935 ops->usba_hcdi_device_fini = xhci_hcdi_device_fini; 1936 ops->usba_hcdi_device_address = xhci_hcdi_device_address; 1937 ops->usba_hcdi_hub_update = xhci_hcdi_hub_update; 1938 1939 hreg.usba_hcdi_register_version = HCDI_REGISTER_VERSION; 1940 hreg.usba_hcdi_register_dip = xhcip->xhci_dip; 1941 hreg.usba_hcdi_register_ops = ops; 1942 1943 /* 1944 * We're required to give xhci a set of DMA attributes that it may loan 1945 * out to other devices. Therefore we'll be conservative with what we 1946 * end up giving it. 1947 */ 1948 xhci_dma_dma_attr(xhcip, &xhcip->xhci_usba.xa_dma_attr); 1949 hreg.usba_hcdi_register_dma_attr = &xhcip->xhci_usba.xa_dma_attr; 1950 1951 hreg.usba_hcdi_register_iblock_cookie = 1952 (ddi_iblock_cookie_t)(uintptr_t)xhcip->xhci_intr_pri; 1953 1954 if (usba_hcdi_register(&hreg, 0) != DDI_SUCCESS) { 1955 usba_free_hcdi_ops(ops); 1956 return (DDI_FAILURE); 1957 } 1958 1959 xhcip->xhci_usba.xa_ops = ops; 1960 1961 list_create(&xhcip->xhci_usba.xa_devices, sizeof (xhci_device_t), 1962 offsetof(xhci_device_t, xd_link)); 1963 list_create(&xhcip->xhci_usba.xa_pipes, sizeof (xhci_pipe_t), 1964 offsetof(xhci_pipe_t, xp_link)); 1965 1966 1967 return (DDI_SUCCESS); 1968 } 1969