1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright (c) 2018, Joyent, Inc. 14 * Copyright (c) 2019 by Western Digital Corporation 15 * Copyright 2022 Oxide Computer Company 16 */ 17 18 /* 19 * illumos USB framework endpoints and functions for xHCI. 20 * 21 * Please see the big theory statement in xhci.c for more information. 22 */ 23 24 #include <sys/usb/hcd/xhci/xhci.h> 25 #include <sys/sysmacros.h> 26 #include <sys/strsun.h> 27 #include <sys/strsubr.h> 28 29 xhci_t * 30 xhci_hcdi_get_xhcip_from_dev(usba_device_t *ud) 31 { 32 dev_info_t *dip = ud->usb_root_hub_dip; 33 xhci_t *xhcip = ddi_get_soft_state(xhci_soft_state, 34 ddi_get_instance(dip)); 35 VERIFY(xhcip != NULL); 36 return (xhcip); 37 } 38 39 static xhci_t * 40 xhci_hcdi_get_xhcip(usba_pipe_handle_data_t *ph) 41 { 42 return (xhci_hcdi_get_xhcip_from_dev(ph->p_usba_device)); 43 } 44 45 /* 46 * While the xHCI hardware is capable of supporting power management, we don't 47 * in the driver right now. Note, USBA doesn't seem to end up calling this entry 48 * point. 49 */ 50 /* ARGSUSED */ 51 static int 52 xhci_hcdi_pm_support(dev_info_t *dip) 53 { 54 return (USB_FAILURE); 55 } 56 57 static int 58 xhci_hcdi_pipe_open(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags) 59 { 60 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph); 61 xhci_pipe_t *pipe; 62 xhci_endpoint_t *xep = NULL; 63 xhci_device_t *xd; 64 int kmflags = usb_flags & USB_FLAGS_SLEEP ? KM_SLEEP : KM_NOSLEEP; 65 int ret; 66 uint_t epid; 67 68 mutex_enter(&xhcip->xhci_lock); 69 if (xhcip->xhci_state & XHCI_S_ERROR) { 70 mutex_exit(&xhcip->xhci_lock); 71 return (USB_HC_HARDWARE_ERROR); 72 } 73 mutex_exit(&xhcip->xhci_lock); 74 75 /* 76 * If we're here, something must be trying to open an already-opened 77 * pipe which is bad news. 78 */ 79 if (ph->p_hcd_private != NULL) { 80 return (USB_FAILURE); 81 } 82 83 pipe = kmem_zalloc(sizeof (xhci_pipe_t), kmflags); 84 if (pipe == NULL) { 85 return (USB_NO_RESOURCES); 86 } 87 pipe->xp_opentime = gethrtime(); 88 pipe->xp_pipe = ph; 89 90 /* 91 * If this is the root hub, there's nothing special to do on open. Just 92 * go ahead and allow it to be opened. All we have to do is add this to 93 * the list of our tracking structures for open pipes. 94 */ 95 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) { 96 xep = NULL; 97 goto add; 98 } 99 100 /* 101 * Now that we're here, we're being asked to open up an endpoint of some 102 * kind. Because we've already handled the case of the root hub, 103 * everything should have a device. 104 */ 105 epid = xhci_endpoint_pipe_to_epid(ph); 106 xd = usba_hcdi_get_device_private(ph->p_usba_device); 107 if (xd == NULL) { 108 xhci_error(xhcip, "!encountered endpoint (%d) without device " 109 "during pipe open", epid); 110 kmem_free(pipe, sizeof (xhci_pipe_t)); 111 return (USB_FAILURE); 112 } 113 114 /* 115 * See if this endpoint exists or not, in general endpoints should not 116 * exist except for the default control endpoint, which we don't tear 117 * down until the device itself is cleaned up. Otherwise, a given pipe 118 * can only be open once. 119 */ 120 mutex_enter(&xhcip->xhci_lock); 121 if (epid == XHCI_DEFAULT_ENDPOINT) { 122 xep = xd->xd_endpoints[epid]; 123 VERIFY(xep != NULL); 124 VERIFY(xep->xep_pipe == NULL); 125 xep->xep_pipe = ph; 126 mutex_exit(&xhcip->xhci_lock); 127 ret = xhci_endpoint_update_default(xhcip, xd, xep); 128 if (ret != USB_SUCCESS) { 129 kmem_free(pipe, sizeof (xhci_pipe_t)); 130 return (ret); 131 } 132 goto add; 133 } 134 135 /* 136 * If we're opening an endpoint other than the default control endpoint, 137 * then the device should have had a USB address assigned by the 138 * controller. Sanity check that before continuing. 139 */ 140 VERIFY(xd->xd_addressed == B_TRUE); 141 142 /* 143 * We may have already initialized the endpoint with a previous pipe 144 * open. 145 */ 146 if ((xep = xd->xd_endpoints[epid]) != NULL && 147 (xep->xep_state & XHCI_ENDPOINT_OPEN)) { 148 mutex_exit(&xhcip->xhci_lock); 149 150 kmem_free(pipe, sizeof (xhci_pipe_t)); 151 xhci_log(xhcip, "!asked to open endpoint %d on slot %d and " 152 "port %d, but endpoint already exists", epid, xd->xd_slot, 153 xd->xd_port); 154 return (USB_FAILURE); 155 } 156 157 if (xep != NULL) { 158 /* 159 * The endpoint is already initialized but is not presently 160 * open so we can take it over here. 161 */ 162 if ((ret = xhci_endpoint_reinit(xhcip, xd, xep, ph) != 0)) { 163 mutex_exit(&xhcip->xhci_lock); 164 165 kmem_free(pipe, sizeof (xhci_pipe_t)); 166 xhci_log(xhcip, "!asked to reopen endpoint %d on " 167 "slot %d and port %d, but reinit failed (%d)", 168 epid, xd->xd_slot, xd->xd_port, ret); 169 return (ret); 170 } 171 172 /* 173 * We need to ensure the endpoint is stopped before we try to 174 * reset the transfer ring. 175 */ 176 xep->xep_state |= XHCI_ENDPOINT_QUIESCE; 177 if ((ret = xhci_endpoint_quiesce(xhcip, xd, xep)) != 178 USB_SUCCESS) { 179 /* 180 * If we could not quiesce the endpoint, release it so 181 * that another open can try again. 182 */ 183 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE; 184 xhci_endpoint_release(xhcip, xep); 185 mutex_exit(&xhcip->xhci_lock); 186 187 kmem_free(pipe, sizeof (xhci_pipe_t)); 188 xhci_log(xhcip, "!asked to reopen endpoint %d on " 189 "slot %d and port %d, but quiesce failed (%d)", 190 epid, xd->xd_slot, xd->xd_port, ret); 191 return (ret); 192 } 193 194 /* 195 * Reset the transfer ring dequeue pointer. The initial 196 * Configure Endpoint command leaves the endpoint in the 197 * Running state (xHCI 1.2 / 4.6.6), so even though the ring is 198 * still empty we ring the doorbell to end up in the same state 199 * (Running but Inactive). 200 */ 201 mutex_exit(&xhcip->xhci_lock); 202 if ((ret = xhci_command_set_tr_dequeue(xhcip, xd, xep)) != 0 || 203 (ret = xhci_endpoint_ring(xhcip, xd, xep)) != 0) { 204 mutex_enter(&xhcip->xhci_lock); 205 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE; 206 xhci_endpoint_release(xhcip, xep); 207 mutex_exit(&xhcip->xhci_lock); 208 209 kmem_free(pipe, sizeof (xhci_pipe_t)); 210 xhci_log(xhcip, "!asked to open endpoint %d on " 211 "slot %d and port %d, but restart failed (%d)", 212 epid, xd->xd_slot, xd->xd_port, ret); 213 return (USB_FAILURE); 214 } 215 mutex_enter(&xhcip->xhci_lock); 216 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE; 217 mutex_exit(&xhcip->xhci_lock); 218 219 goto add; 220 } 221 222 /* 223 * Okay, at this point we need to go create and set up an endpoint from 224 * scratch. Once we're done, we'll try to install it and make sure 225 * that it doesn't conflict with something else going on. 226 */ 227 ret = xhci_endpoint_init(xhcip, xd, ph); 228 if (ret != 0) { 229 mutex_exit(&xhcip->xhci_lock); 230 kmem_free(pipe, sizeof (xhci_pipe_t)); 231 if (ret == EIO) { 232 xhci_error(xhcip, "failed to initialize endpoint %d " 233 "on device slot %d and port %d: encountered fatal " 234 "FM error, resetting device", epid, xd->xd_slot, 235 xd->xd_port); 236 xhci_fm_runtime_reset(xhcip); 237 } 238 return (USB_HC_HARDWARE_ERROR); 239 } 240 xep = xd->xd_endpoints[epid]; 241 242 mutex_enter(&xd->xd_imtx); 243 mutex_exit(&xhcip->xhci_lock); 244 245 /* 246 * Update the slot and input context for this endpoint. We make sure to 247 * always set the slot as having changed in the context field as the 248 * specification suggests we should and some hardware requires it. 249 */ 250 xd->xd_input->xic_drop_flags = LE_32(0); 251 xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0) | 252 XHCI_INCTX_MASK_DCI(epid + 1)); 253 254 if (epid + 1 > XHCI_SCTX_GET_DCI(LE_32(xd->xd_slotin->xsc_info))) { 255 uint32_t info; 256 257 info = xd->xd_slotin->xsc_info; 258 info &= ~XHCI_SCTX_DCI_MASK; 259 info |= XHCI_SCTX_SET_DCI(epid + 1); 260 xd->xd_slotin->xsc_info = info; 261 } 262 263 XHCI_DMA_SYNC(xd->xd_ictx, DDI_DMA_SYNC_FORDEV); 264 if (xhci_check_dma_handle(xhcip, &xd->xd_ictx) != DDI_FM_OK) { 265 mutex_exit(&xd->xd_imtx); 266 xhci_endpoint_fini(xd, epid); 267 kmem_free(pipe, sizeof (xhci_pipe_t)); 268 xhci_error(xhcip, "failed to open pipe on endpoint %d of " 269 "device with slot %d and port %d: encountered fatal FM " 270 "error syncing device input context, resetting device", 271 epid, xd->xd_slot, xd->xd_port); 272 xhci_fm_runtime_reset(xhcip); 273 return (USB_HC_HARDWARE_ERROR); 274 } 275 276 if ((ret = xhci_command_configure_endpoint(xhcip, xd)) != USB_SUCCESS) { 277 mutex_exit(&xd->xd_imtx); 278 xhci_endpoint_fini(xd, epid); 279 kmem_free(pipe, sizeof (xhci_pipe_t)); 280 return (ret); 281 } 282 283 mutex_exit(&xd->xd_imtx); 284 add: 285 pipe->xp_ep = xep; 286 ph->p_hcd_private = (usb_opaque_t)pipe; 287 mutex_enter(&xhcip->xhci_lock); 288 list_insert_tail(&xhcip->xhci_usba.xa_pipes, pipe); 289 mutex_exit(&xhcip->xhci_lock); 290 291 return (USB_SUCCESS); 292 } 293 294 static void 295 xhci_hcdi_periodic_free(xhci_t *xhcip, xhci_pipe_t *xp) 296 { 297 int i; 298 xhci_periodic_pipe_t *xpp = &xp->xp_periodic; 299 300 if (xpp->xpp_tsize == 0) 301 return; 302 303 for (i = 0; i < xpp->xpp_ntransfers; i++) { 304 if (xpp->xpp_transfers[i] == NULL) 305 continue; 306 xhci_transfer_free(xhcip, xpp->xpp_transfers[i]); 307 xpp->xpp_transfers[i] = NULL; 308 } 309 310 xpp->xpp_ntransfers = 0; 311 xpp->xpp_tsize = 0; 312 } 313 314 /* 315 * Iterate over all transfers and free everything on the pipe. Once done, update 316 * the ring to basically 'consume' everything. For periodic IN endpoints, we 317 * need to handle this somewhat differently and actually close the original 318 * request and not deallocate the related pieces as those exist for the lifetime 319 * of the endpoint and are constantly reused. 320 */ 321 static void 322 xhci_hcdi_pipe_flush(xhci_t *xhcip, xhci_endpoint_t *xep, int intr_code) 323 { 324 xhci_transfer_t *xt; 325 326 ASSERT(MUTEX_HELD(&xhcip->xhci_lock)); 327 328 while ((xt = list_remove_head(&xep->xep_transfers)) != NULL) { 329 if (xhci_endpoint_is_periodic_in(xep) == B_FALSE) { 330 usba_hcdi_cb(xep->xep_pipe, xt->xt_usba_req, 331 USB_CR_FLUSHED); 332 xhci_transfer_free(xhcip, xt); 333 } 334 } 335 336 if (xhci_endpoint_is_periodic_in(xep) == B_TRUE) { 337 xhci_pipe_t *xp = (xhci_pipe_t *)xep->xep_pipe->p_hcd_private; 338 xhci_periodic_pipe_t *xpp = &xp->xp_periodic; 339 340 if (xpp->xpp_usb_req != NULL) { 341 usba_hcdi_cb(xep->xep_pipe, xpp->xpp_usb_req, 342 intr_code); 343 xpp->xpp_usb_req = NULL; 344 } 345 } 346 } 347 348 /* 349 * We've been asked to terminate some set of regular I/O on an interrupt pipe. 350 * If this is for the root device, e.g. the xhci driver itself, then we remove 351 * our interrupt callback. Otherwise we stop the device for interrupt polling as 352 * follows: 353 * 354 * 1. Issue a stop endpoint command 355 * 2. Check to make sure that the endpoint stopped and reset it if needed. 356 * 3. Any thing that gets resolved can callback in the interim. 357 * 4. Ensure that nothing is scheduled on the ring 358 * 5. Skip the contents of the ring and set the TR dequeue pointer. 359 * 6. Return the original callback with a USB_CR_STOPPED_POLLING, NULL out the 360 * callback in the process. 361 */ 362 static int 363 xhci_hcdi_pipe_poll_fini(usba_pipe_handle_data_t *ph, boolean_t is_close) 364 { 365 int ret; 366 uint_t epid; 367 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph); 368 xhci_device_t *xd; 369 xhci_endpoint_t *xep; 370 xhci_pipe_t *xp; 371 xhci_periodic_pipe_t *xpp; 372 usb_opaque_t urp; 373 374 mutex_enter(&xhcip->xhci_lock); 375 if (xhcip->xhci_state & XHCI_S_ERROR) { 376 mutex_exit(&xhcip->xhci_lock); 377 return (USB_HC_HARDWARE_ERROR); 378 } 379 380 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) { 381 xhci_root_hub_intr_root_disable(xhcip); 382 ret = USB_SUCCESS; 383 mutex_exit(&xhcip->xhci_lock); 384 return (ret); 385 } 386 387 xd = usba_hcdi_get_device_private(ph->p_usba_device); 388 epid = xhci_endpoint_pipe_to_epid(ph); 389 if (xd->xd_endpoints[epid] == NULL) { 390 mutex_exit(&xhcip->xhci_lock); 391 xhci_error(xhcip, "asked to stop intr polling on slot %d, " 392 "port %d, endpoint: %d, but no endpoint structure", 393 xd->xd_slot, xd->xd_port, epid); 394 return (USB_FAILURE); 395 } 396 xep = xd->xd_endpoints[epid]; 397 xp = (xhci_pipe_t *)ph->p_hcd_private; 398 if (xp == NULL) { 399 mutex_exit(&xhcip->xhci_lock); 400 xhci_error(xhcip, "asked to do finish polling on slot %d, " 401 "port %d, endpoint: %d, but no pipe structure", 402 xd->xd_slot, xd->xd_port, epid); 403 return (USB_FAILURE); 404 } 405 xpp = &xp->xp_periodic; 406 407 /* 408 * Ensure that no other resets or time outs are going on right now. 409 */ 410 xhci_endpoint_serialize(xhcip, xep); 411 412 if (xpp->xpp_poll_state == XHCI_PERIODIC_POLL_IDLE) { 413 mutex_exit(&xhcip->xhci_lock); 414 return (USB_SUCCESS); 415 } 416 417 if (xpp->xpp_poll_state == XHCI_PERIODIC_POLL_STOPPING) { 418 mutex_exit(&xhcip->xhci_lock); 419 return (USB_FAILURE); 420 } 421 422 xpp->xpp_poll_state = XHCI_PERIODIC_POLL_STOPPING; 423 xep->xep_state |= XHCI_ENDPOINT_QUIESCE; 424 ret = xhci_endpoint_quiesce(xhcip, xd, xep); 425 if (ret != USB_SUCCESS) { 426 xhci_error(xhcip, "!failed to quiesce endpoint on slot %d, " 427 "port %d, endpoint: %d, failed with %d.", 428 xd->xd_slot, xd->xd_port, epid, ret); 429 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE; 430 cv_broadcast(&xep->xep_state_cv); 431 mutex_exit(&xhcip->xhci_lock); 432 return (ret); 433 } 434 435 /* 436 * Okay, we've stopped this ring time to wrap it all up. Remove all the 437 * transfers, note they aren't freed like a pipe reset. 438 */ 439 while (list_is_empty(&xep->xep_transfers) == 0) 440 (void) list_remove_head(&xep->xep_transfers); 441 xhci_ring_skip(&xep->xep_ring); 442 mutex_exit(&xhcip->xhci_lock); 443 444 if ((ret = xhci_command_set_tr_dequeue(xhcip, xd, xep)) != 445 USB_SUCCESS) { 446 xhci_error(xhcip, "!failed to reset endpoint ring on slot %d, " 447 "port %d, endpoint: %d, failed with %d.", 448 xd->xd_slot, xd->xd_port, epid, ret); 449 mutex_enter(&xhcip->xhci_lock); 450 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE; 451 cv_broadcast(&xep->xep_state_cv); 452 mutex_exit(&xhcip->xhci_lock); 453 return (ret); 454 } 455 456 mutex_enter(&xhcip->xhci_lock); 457 urp = xpp->xpp_usb_req; 458 xpp->xpp_usb_req = NULL; 459 xpp->xpp_poll_state = XHCI_PERIODIC_POLL_IDLE; 460 xep->xep_state &= ~XHCI_ENDPOINT_PERIODIC; 461 mutex_exit(&xhcip->xhci_lock); 462 463 /* 464 * It's possible that with a persistent pipe, we may not actually have 465 * anything left to call back on, because we already had. 466 */ 467 if (urp != NULL) { 468 usba_hcdi_cb(ph, urp, is_close == B_TRUE ? 469 USB_CR_PIPE_CLOSING : USB_CR_STOPPED_POLLING); 470 } 471 472 /* 473 * Notify anything waiting for us that we're done quiescing this device. 474 */ 475 mutex_enter(&xhcip->xhci_lock); 476 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE; 477 cv_broadcast(&xep->xep_state_cv); 478 mutex_exit(&xhcip->xhci_lock); 479 480 return (USB_SUCCESS); 481 482 } 483 484 /* 485 * Tear down everything that we did in open. After this, the consumer of this 486 * USB device is done. 487 */ 488 static int 489 xhci_hcdi_pipe_close(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags) 490 { 491 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph); 492 xhci_pipe_t *xp; 493 xhci_device_t *xd; 494 xhci_endpoint_t *xep; 495 int ret; 496 uint_t epid; 497 498 if ((ph->p_ep.bmAttributes & USB_EP_ATTR_MASK) == USB_EP_ATTR_INTR && 499 xhcip->xhci_usba.xa_intr_cb_ph != NULL) { 500 if ((ret = xhci_hcdi_pipe_poll_fini(ph, B_TRUE)) != 501 USB_SUCCESS) { 502 return (ret); 503 } 504 } 505 506 mutex_enter(&xhcip->xhci_lock); 507 508 xp = (xhci_pipe_t *)ph->p_hcd_private; 509 VERIFY(xp != NULL); 510 511 /* 512 * The default endpoint is special. It is created and destroyed with the 513 * device. So like with open, closing it is just state tracking. The 514 * same is true for the root hub. 515 */ 516 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) 517 goto remove; 518 519 xd = usba_hcdi_get_device_private(ph->p_usba_device); 520 epid = xhci_endpoint_pipe_to_epid(ph); 521 if (xd->xd_endpoints[epid] == NULL) { 522 mutex_exit(&xhcip->xhci_lock); 523 xhci_error(xhcip, "asked to do close pipe on slot %d, " 524 "port %d, endpoint: %d, but no endpoint structure", 525 xd->xd_slot, xd->xd_port, epid); 526 return (USB_FAILURE); 527 } 528 xep = xd->xd_endpoints[epid]; 529 530 if (xp->xp_ep != NULL && xp->xp_ep->xep_num == XHCI_DEFAULT_ENDPOINT) { 531 xep->xep_pipe = NULL; 532 goto remove; 533 } 534 535 /* 536 * We clean up the endpoint by stopping it and cancelling any transfers 537 * that were in flight at the time. The endpoint is not unconfigured 538 * until the device is torn down later. 539 */ 540 xhci_endpoint_timeout_cancel(xhcip, xep); 541 xep->xep_state |= XHCI_ENDPOINT_QUIESCE; 542 if ((ret = xhci_endpoint_quiesce(xhcip, xd, xep)) != USB_SUCCESS) { 543 /* 544 * If we cannot stop the ring, it is not safe to proceed and we 545 * must keep the pipe open. 546 */ 547 xep->xep_state &= 548 ~(XHCI_ENDPOINT_TEARDOWN | XHCI_ENDPOINT_QUIESCE); 549 cv_broadcast(&xep->xep_state_cv); 550 mutex_exit(&xhcip->xhci_lock); 551 xhci_error(xhcip, "asked to do close pipe on slot %d, " 552 "port %d, endpoint: %d, but quiesce failed %d", 553 xd->xd_slot, xd->xd_port, epid, ret); 554 return (USB_FAILURE); 555 } 556 557 /* 558 * Now that we've stopped the endpoint, see if we need to flush any 559 * transfers. 560 */ 561 xhci_hcdi_pipe_flush(xhcip, xep, USB_CR_PIPE_CLOSING); 562 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) { 563 xhci_hcdi_periodic_free(xhcip, xp); 564 } 565 566 xhci_endpoint_release(xhcip, xep); 567 568 remove: 569 ph->p_hcd_private = NULL; 570 list_remove(&xhcip->xhci_usba.xa_pipes, xp); 571 kmem_free(xp, sizeof (xhci_pipe_t)); 572 573 mutex_exit(&xhcip->xhci_lock); 574 575 return (USB_SUCCESS); 576 } 577 578 /* 579 * We've been asked to reset a pipe aka an endpoint. This endpoint may be in an 580 * arbitrary state, it may be running or it may be halted. In this case, we go 581 * through and check whether or not we know it's been halted or not. If it has 582 * not, then we stop the endpoint. 583 * 584 * Once the endpoint has been stopped, walk all transfers and go ahead and 585 * basically return them as being flushed. Then finally set the dequeue point 586 * for this endpoint. 587 */ 588 /* ARGSUSED */ 589 static int 590 xhci_hcdi_pipe_reset(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags) 591 { 592 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph); 593 xhci_device_t *xd; 594 xhci_endpoint_t *xep; 595 uint_t epid; 596 int ret; 597 598 mutex_enter(&xhcip->xhci_lock); 599 if (xhcip->xhci_state & XHCI_S_ERROR) { 600 mutex_exit(&xhcip->xhci_lock); 601 return (USB_HC_HARDWARE_ERROR); 602 } 603 604 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) { 605 mutex_exit(&xhcip->xhci_lock); 606 return (USB_NOT_SUPPORTED); 607 } 608 609 xd = usba_hcdi_get_device_private(ph->p_usba_device); 610 epid = xhci_endpoint_pipe_to_epid(ph); 611 if (xd->xd_endpoints[epid] == NULL) { 612 mutex_exit(&xhcip->xhci_lock); 613 xhci_error(xhcip, "asked to do reset pipe on slot %d, " 614 "port %d, endpoint: %d, but no endpoint structure", 615 xd->xd_slot, xd->xd_port, epid); 616 return (USB_FAILURE); 617 } 618 619 xep = xd->xd_endpoints[epid]; 620 621 /* 622 * Ensure that no other resets or time outs are going on right now. 623 */ 624 xhci_endpoint_serialize(xhcip, xep); 625 626 xep->xep_state |= XHCI_ENDPOINT_QUIESCE; 627 ret = xhci_endpoint_quiesce(xhcip, xd, xep); 628 if (ret != USB_SUCCESS) { 629 /* 630 * We failed to quiesce for some reason, remove the flag and let 631 * someone else give it a shot. 632 */ 633 xhci_error(xhcip, "!failed to quiesce endpoint on slot %d, " 634 "port %d, endpoint: %d, failed with %d.", 635 xd->xd_slot, xd->xd_port, epid, ret); 636 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE; 637 cv_broadcast(&xep->xep_state_cv); 638 mutex_exit(&xhcip->xhci_lock); 639 return (ret); 640 } 641 642 xhci_ring_skip(&xep->xep_ring); 643 644 mutex_exit(&xhcip->xhci_lock); 645 if ((ret = xhci_command_set_tr_dequeue(xhcip, xd, xep)) != 646 USB_SUCCESS) { 647 xhci_error(xhcip, "!failed to reset endpoint ring on slot %d, " 648 "port %d, endpoint: %d, failed setting ring dequeue with " 649 "%d.", xd->xd_slot, xd->xd_port, epid, ret); 650 mutex_enter(&xhcip->xhci_lock); 651 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE; 652 cv_broadcast(&xep->xep_state_cv); 653 mutex_exit(&xhcip->xhci_lock); 654 return (ret); 655 } 656 657 mutex_enter(&xhcip->xhci_lock); 658 xhci_hcdi_pipe_flush(xhcip, xep, USB_CR_PIPE_RESET); 659 660 /* 661 * We need to remove the periodic flag as part of resetting, as if this 662 * was used for periodic activity, it no longer is and therefore can now 663 * be used for such purposes. 664 * 665 * Notify anything waiting for us that we're done quiescing this device. 666 */ 667 xep->xep_state &= ~(XHCI_ENDPOINT_QUIESCE | XHCI_ENDPOINT_PERIODIC); 668 cv_broadcast(&xep->xep_state_cv); 669 mutex_exit(&xhcip->xhci_lock); 670 671 return (USB_SUCCESS); 672 } 673 674 /* 675 * We're asked to reset or change the data toggle, which is used in a few cases. 676 * However, there doesn't seem to be a good way to do this in xHCI as the data 677 * toggle isn't exposed. It seems that dropping a reset endpoint would 678 * theoretically do this; however, that can only be used when in the HALTED 679 * state. As such, for now we just return. 680 */ 681 /* ARGSUSED */ 682 void 683 xhci_hcdi_pipe_reset_data_toggle(usba_pipe_handle_data_t *pipe_handle) 684 { 685 } 686 687 /* 688 * We need to convert the USB request to an 8-byte little endian value. If we 689 * didn't have to think about big endian systems, this would be fine. 690 * Unfortunately, with them, this is a bit confusing. The problem is that if you 691 * think of this as a struct layout, the order that we or things together 692 * represents their byte layout. e.g. ctrl_bRequest is at offset 1 in the SETUP 693 * STAGE trb. However, when it becomes a part of a 64-bit big endian number, if 694 * ends up at byte 7, where as it needs to be at one. Hence why we do a final 695 * LE_64 at the end of this, to convert this into the byte order that it's 696 * expected to be in. 697 */ 698 static uint64_t 699 xhci_hcdi_ctrl_req_to_trb(usb_ctrl_req_t *ucrp) 700 { 701 uint64_t ret = ucrp->ctrl_bmRequestType | 702 (ucrp->ctrl_bRequest << 8) | 703 ((uint64_t)LE_16(ucrp->ctrl_wValue) << 16) | 704 ((uint64_t)LE_16(ucrp->ctrl_wIndex) << 32) | 705 ((uint64_t)LE_16(ucrp->ctrl_wLength) << 48); 706 return (LE_64(ret)); 707 } 708 709 /* 710 * USBA calls us in order to make a specific control type request to a device, 711 * potentially even the root hub. If the request is for the root hub, then we 712 * need to intercept this and cons up the requested data. 713 */ 714 static int 715 xhci_hcdi_pipe_ctrl_xfer(usba_pipe_handle_data_t *ph, usb_ctrl_req_t *ucrp, 716 usb_flags_t usb_flags) 717 { 718 int ret, statusdir, trt; 719 uint_t ep; 720 xhci_device_t *xd; 721 xhci_endpoint_t *xep; 722 xhci_transfer_t *xt; 723 boolean_t datain; 724 725 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph); 726 727 mutex_enter(&xhcip->xhci_lock); 728 if (xhcip->xhci_state & XHCI_S_ERROR) { 729 mutex_exit(&xhcip->xhci_lock); 730 return (USB_HC_HARDWARE_ERROR); 731 } 732 733 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) { 734 ret = xhci_root_hub_ctrl_req(xhcip, ph, ucrp); 735 mutex_exit(&xhcip->xhci_lock); 736 return (ret); 737 } 738 739 /* 740 * Determine the device and endpoint. 741 */ 742 xd = usba_hcdi_get_device_private(ph->p_usba_device); 743 ep = xhci_endpoint_pipe_to_epid(ph); 744 if (xd->xd_endpoints[ep] == NULL) { 745 mutex_exit(&xhcip->xhci_lock); 746 xhci_error(xhcip, "asked to do control transfer on slot %d, " 747 "port %d, endpoint: %d, but no endpoint structure", 748 xd->xd_slot, xd->xd_port, ep); 749 return (USB_FAILURE); 750 } 751 xep = xd->xd_endpoints[ep]; 752 753 /* 754 * There are several types of requests that we have to handle in special 755 * ways in xHCI. If we have one of those requests, then we don't 756 * necessarily go through the normal path. These special cases are all 757 * documented in xHCI 1.1 / 4.5.4. 758 * 759 * Looking at that, you may ask why aren't SET_CONFIGURATION and SET_IF 760 * special cased here. This action is a little confusing by default. The 761 * xHCI specification requires that we may need to issue a configure 762 * endpoint command as part of this. However, the xHCI 1.1 / 4.5.4.2 763 * states that we don't actually need to if nothing in the endpoint 764 * configuration context has changed. Because nothing in it should have 765 * changed as part of this, we don't need to do anything and instead 766 * just can issue the request normally. We're also assuming in the 767 * USB_REQ_SET_IF case that if something's changing the interface, the 768 * non-default endpoint will have yet to be opened. 769 */ 770 if (ucrp->ctrl_bmRequestType == USB_DEV_REQ_HOST_TO_DEV && 771 ucrp->ctrl_bRequest == USB_REQ_SET_ADDRESS) { 772 /* 773 * As we've defined an explicit set-address endpoint, we should 774 * never call this function. If we get here, always fail. 775 */ 776 mutex_exit(&xhcip->xhci_lock); 777 usba_hcdi_cb(ph, (usb_opaque_t)ucrp, USB_CR_NOT_SUPPORTED); 778 return (USB_SUCCESS); 779 } 780 781 mutex_exit(&xhcip->xhci_lock); 782 783 /* 784 * Allocate the transfer memory, etc. 785 */ 786 xt = xhci_transfer_alloc(xhcip, xep, ucrp->ctrl_wLength, 2, usb_flags); 787 if (xt == NULL) { 788 return (USB_NO_RESOURCES); 789 } 790 xt->xt_usba_req = (usb_opaque_t)ucrp; 791 xt->xt_timeout = ucrp->ctrl_timeout; 792 if (xt->xt_timeout == 0) { 793 xt->xt_timeout = HCDI_DEFAULT_TIMEOUT; 794 } 795 796 if (ucrp->ctrl_wLength > 0) { 797 if ((ucrp->ctrl_bmRequestType & USB_DEV_REQ_DEV_TO_HOST) != 0) { 798 trt = XHCI_TRB_TRT_IN; 799 datain = B_TRUE; 800 statusdir = 0; 801 } else { 802 trt = XHCI_TRB_TRT_OUT; 803 datain = B_FALSE; 804 statusdir = XHCI_TRB_DIR_IN; 805 806 xhci_transfer_copy(xt, ucrp->ctrl_data->b_rptr, 807 ucrp->ctrl_wLength, B_FALSE); 808 if (xhci_transfer_sync(xhcip, xt, 809 DDI_DMA_SYNC_FORDEV) != DDI_FM_OK) { 810 xhci_transfer_free(xhcip, xt); 811 xhci_error(xhcip, "failed to synchronize ctrl " 812 "transfer DMA memory on endpoint %u of " 813 "device on slot %d and port %d: resetting " 814 "device", xep->xep_num, xd->xd_slot, 815 xd->xd_port); 816 xhci_fm_runtime_reset(xhcip); 817 return (USB_HC_HARDWARE_ERROR); 818 } 819 } 820 } else { 821 trt = 0; 822 datain = B_FALSE; 823 statusdir = XHCI_TRB_DIR_IN; 824 } 825 826 /* 827 * We always fill in the required setup and status TRBs ourselves; 828 * however, to minimize our knowledge about how the data has been split 829 * across multiple DMA cookies in an SGL, we leave that to the transfer 830 * logic to fill in. 831 */ 832 xt->xt_trbs[0].trb_addr = xhci_hcdi_ctrl_req_to_trb(ucrp); 833 xt->xt_trbs[0].trb_status = LE_32(XHCI_TRB_LEN(8) | XHCI_TRB_INTR(0)); 834 xt->xt_trbs[0].trb_flags = LE_32(trt | XHCI_TRB_IDT | 835 XHCI_TRB_TYPE_SETUP); 836 837 if (ucrp->ctrl_wLength > 0) 838 xhci_transfer_trb_fill_data(xep, xt, 1, datain); 839 840 xt->xt_trbs[xt->xt_ntrbs - 1].trb_addr = 0; 841 xt->xt_trbs[xt->xt_ntrbs - 1].trb_status = LE_32(XHCI_TRB_INTR(0)); 842 xt->xt_trbs[xt->xt_ntrbs - 1].trb_flags = LE_32(XHCI_TRB_TYPE_STATUS | 843 XHCI_TRB_IOC | statusdir); 844 845 846 mutex_enter(&xhcip->xhci_lock); 847 848 /* 849 * Schedule the transfer, allocating resources in the process. 850 */ 851 if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) { 852 xhci_transfer_free(xhcip, xt); 853 mutex_exit(&xhcip->xhci_lock); 854 return (USB_NO_RESOURCES); 855 } 856 857 mutex_exit(&xhcip->xhci_lock); 858 859 return (USB_SUCCESS); 860 } 861 862 /* 863 * This request is trying to get the upper bound on the amount of data we're 864 * willing transfer in one go. Note that this amount can be broken down into 865 * multiple SGL entries, this interface doesn't particularly care about that. 866 */ 867 /* ARGSUSED */ 868 static int 869 xhci_hcdi_bulk_transfer_size(usba_device_t *ud, size_t *sizep) 870 { 871 if (sizep != NULL) 872 *sizep = XHCI_MAX_TRANSFER; 873 return (USB_SUCCESS); 874 } 875 876 /* 877 * Perform a bulk transfer. This is a pretty straightforward action. We 878 * basically just allocate the appropriate transfer and try to schedule it, 879 * hoping there is enough space. 880 */ 881 static int 882 xhci_hcdi_pipe_bulk_xfer(usba_pipe_handle_data_t *ph, usb_bulk_req_t *ubrp, 883 usb_flags_t usb_flags) 884 { 885 uint_t epid; 886 xhci_device_t *xd; 887 xhci_endpoint_t *xep; 888 xhci_transfer_t *xt; 889 boolean_t datain; 890 891 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph); 892 893 mutex_enter(&xhcip->xhci_lock); 894 if (xhcip->xhci_state & XHCI_S_ERROR) { 895 mutex_exit(&xhcip->xhci_lock); 896 return (USB_HC_HARDWARE_ERROR); 897 } 898 899 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) { 900 mutex_exit(&xhcip->xhci_lock); 901 return (USB_NOT_SUPPORTED); 902 } 903 904 xd = usba_hcdi_get_device_private(ph->p_usba_device); 905 epid = xhci_endpoint_pipe_to_epid(ph); 906 if (xd->xd_endpoints[epid] == NULL) { 907 mutex_exit(&xhcip->xhci_lock); 908 xhci_error(xhcip, "asked to do control transfer on slot %d, " 909 "port %d, endpoint: %d, but no endpoint structure", 910 xd->xd_slot, xd->xd_port, epid); 911 return (USB_FAILURE); 912 } 913 xep = xd->xd_endpoints[epid]; 914 mutex_exit(&xhcip->xhci_lock); 915 916 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) { 917 datain = B_TRUE; 918 } else { 919 datain = B_FALSE; 920 } 921 922 xt = xhci_transfer_alloc(xhcip, xep, ubrp->bulk_len, 0, usb_flags); 923 if (xt == NULL) { 924 return (USB_NO_RESOURCES); 925 } 926 xt->xt_usba_req = (usb_opaque_t)ubrp; 927 xt->xt_timeout = ubrp->bulk_timeout; 928 if (xt->xt_timeout == 0) { 929 xt->xt_timeout = HCDI_DEFAULT_TIMEOUT; 930 } 931 932 if (ubrp->bulk_len > 0 && datain == B_FALSE) { 933 xhci_transfer_copy(xt, ubrp->bulk_data->b_rptr, ubrp->bulk_len, 934 B_FALSE); 935 if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) != 936 DDI_FM_OK) { 937 xhci_transfer_free(xhcip, xt); 938 xhci_error(xhcip, "failed to synchronize bulk " 939 "transfer DMA memory on endpoint %u of " 940 "device on slot %d and port %d: resetting " 941 "device", xep->xep_num, xd->xd_slot, 942 xd->xd_port); 943 xhci_fm_runtime_reset(xhcip); 944 return (USB_HC_HARDWARE_ERROR); 945 } 946 } 947 948 xhci_transfer_trb_fill_data(xep, xt, 0, datain); 949 mutex_enter(&xhcip->xhci_lock); 950 if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) { 951 xhci_transfer_free(xhcip, xt); 952 mutex_exit(&xhcip->xhci_lock); 953 return (USB_NO_RESOURCES); 954 } 955 mutex_exit(&xhcip->xhci_lock); 956 957 return (USB_SUCCESS); 958 } 959 960 static void 961 xhci_hcdi_isoc_transfer_fill(xhci_device_t *xd, xhci_endpoint_t *xep, 962 xhci_transfer_t *xt, usb_isoc_req_t *usrp) 963 { 964 int i; 965 uintptr_t buf; 966 967 buf = xt->xt_buffer.xdb_cookies[0].dmac_laddress; 968 for (i = 0; i < usrp->isoc_pkts_count; i++) { 969 int flags; 970 uint_t tbc, tlbpc; 971 972 ushort_t len = usrp->isoc_pkt_descr[i].isoc_pkt_length; 973 xhci_trb_t *trb = &xt->xt_trbs[i]; 974 975 trb->trb_addr = LE_64(buf); 976 977 /* 978 * Because we know that a single frame can have all of its data 979 * in a single instance, we know that we don't need to do 980 * anything special here. 981 */ 982 trb->trb_status = LE_32(XHCI_TRB_LEN(len) | XHCI_TRB_TDREM(0) | 983 XHCI_TRB_INTR(0)); 984 985 /* 986 * Always enable SIA to start the frame ASAP. We also always 987 * enable an interrupt on a short packet. If this is the last 988 * trb, then we will set IOC. Each TRB created here is really 989 * its own TD. However, we only set an interrupt on the last 990 * entry to better deal with scheduling. 991 */ 992 flags = XHCI_TRB_SIA | XHCI_TRB_ISP | XHCI_TRB_SET_FRAME(0); 993 flags |= XHCI_TRB_TYPE_ISOCH; 994 995 if (i + 1 == usrp->isoc_pkts_count) 996 flags |= XHCI_TRB_IOC; 997 998 /* 999 * Now we need to calculate the TBC and the TLBPC. 1000 */ 1001 xhci_transfer_calculate_isoc(xd, xep, len, &tbc, &tlbpc); 1002 flags |= XHCI_TRB_SET_TBC(tbc); 1003 flags |= XHCI_TRB_SET_TLBPC(tlbpc); 1004 1005 trb->trb_flags = LE_32(flags); 1006 buf += len; 1007 1008 /* 1009 * Go through and copy the required data to our local copy of 1010 * the isoc descriptor. By default, we assume that all data will 1011 * be copied and the status set to OK. This mirrors the fact 1012 * that we won't get a notification unless there's been an 1013 * error or short packet transfer. 1014 */ 1015 xt->xt_isoc[i].isoc_pkt_length = len; 1016 xt->xt_isoc[i].isoc_pkt_actual_length = len; 1017 xt->xt_isoc[i].isoc_pkt_status = USB_CR_OK; 1018 } 1019 } 1020 1021 /* 1022 * Initialize periodic IN requests (both interrupt and isochronous) 1023 */ 1024 static int 1025 xhci_hcdi_periodic_init(xhci_t *xhcip, usba_pipe_handle_data_t *ph, 1026 usb_opaque_t usb_req, size_t len, int usb_flags) 1027 { 1028 int i, ret; 1029 uint_t epid; 1030 xhci_device_t *xd; 1031 xhci_endpoint_t *xep; 1032 xhci_pipe_t *xp; 1033 xhci_periodic_pipe_t *xpp; 1034 1035 mutex_enter(&xhcip->xhci_lock); 1036 if (xhcip->xhci_state & XHCI_S_ERROR) { 1037 mutex_exit(&xhcip->xhci_lock); 1038 return (USB_HC_HARDWARE_ERROR); 1039 } 1040 1041 xd = usba_hcdi_get_device_private(ph->p_usba_device); 1042 epid = xhci_endpoint_pipe_to_epid(ph); 1043 if (xd->xd_endpoints[epid] == NULL) { 1044 xhci_error(xhcip, "asked to do periodic transfer on slot %d, " 1045 "port %d, endpoint: %d, but no endpoint structure", 1046 xd->xd_slot, xd->xd_port, epid); 1047 mutex_exit(&xhcip->xhci_lock); 1048 return (USB_FAILURE); 1049 } 1050 xep = xd->xd_endpoints[epid]; 1051 xp = (xhci_pipe_t *)ph->p_hcd_private; 1052 if (xp == NULL) { 1053 xhci_error(xhcip, "asked to do periodic transfer on slot %d, " 1054 "port %d, endpoint: %d, but no pipe structure", 1055 xd->xd_slot, xd->xd_port, epid); 1056 mutex_exit(&xhcip->xhci_lock); 1057 return (USB_FAILURE); 1058 } 1059 xpp = &xp->xp_periodic; 1060 1061 /* 1062 * Only allow a single polling request at any given time. 1063 */ 1064 if (xpp->xpp_usb_req != NULL) { 1065 mutex_exit(&xhcip->xhci_lock); 1066 return (USB_BUSY); 1067 } 1068 1069 /* 1070 * We keep allocations around in case we restart polling, which most 1071 * devices do (not really caring about a lost event). However, we don't 1072 * support a driver changing that size on us, which it probably won't. 1073 * If we stumble across driver that does, then this will need to become 1074 * a lot more complicated. 1075 */ 1076 if (xpp->xpp_tsize > 0 && xpp->xpp_tsize < len) { 1077 mutex_exit(&xhcip->xhci_lock); 1078 return (USB_INVALID_REQUEST); 1079 } 1080 1081 if (xpp->xpp_tsize == 0) { 1082 int ntrbs; 1083 int ntransfers; 1084 1085 /* 1086 * What we allocate varies based on whether or not this is an 1087 * isochronous or interrupt IN periodic. 1088 */ 1089 if (xep->xep_type == USB_EP_ATTR_INTR) { 1090 ntrbs = 0; 1091 ntransfers = XHCI_INTR_IN_NTRANSFERS; 1092 } else { 1093 usb_isoc_req_t *usrp; 1094 ASSERT(xep->xep_type == USB_EP_ATTR_ISOCH); 1095 1096 usrp = (usb_isoc_req_t *)usb_req; 1097 ntrbs = usrp->isoc_pkts_count; 1098 ntransfers = XHCI_ISOC_IN_NTRANSFERS; 1099 } 1100 1101 xpp->xpp_tsize = len; 1102 xpp->xpp_ntransfers = ntransfers; 1103 1104 for (i = 0; i < xpp->xpp_ntransfers; i++) { 1105 xhci_transfer_t *xt = xhci_transfer_alloc(xhcip, xep, 1106 len, ntrbs, usb_flags); 1107 if (xt == NULL) { 1108 xhci_hcdi_periodic_free(xhcip, xp); 1109 mutex_exit(&xhcip->xhci_lock); 1110 return (USB_NO_RESOURCES); 1111 } 1112 1113 if (xep->xep_type == USB_EP_ATTR_INTR) { 1114 xhci_transfer_trb_fill_data(xep, xt, 0, B_TRUE); 1115 } else { 1116 usb_isoc_req_t *usrp; 1117 usrp = (usb_isoc_req_t *)usb_req; 1118 xhci_hcdi_isoc_transfer_fill(xd, xep, xt, usrp); 1119 xt->xt_data_tohost = B_TRUE; 1120 } 1121 xpp->xpp_transfers[i] = xt; 1122 } 1123 } 1124 1125 /* 1126 * Mark the endpoint as periodic so we don't have timeouts at play. 1127 */ 1128 xep->xep_state |= XHCI_ENDPOINT_PERIODIC; 1129 1130 /* 1131 * Now that we've allocated everything, go ahead and schedule them and 1132 * kick off the ring. 1133 */ 1134 for (i = 0; i < xpp->xpp_ntransfers; i++) { 1135 int ret; 1136 ret = xhci_endpoint_schedule(xhcip, xd, xep, 1137 xpp->xpp_transfers[i], B_FALSE); 1138 if (ret != 0) { 1139 (void) xhci_ring_reset(xhcip, &xep->xep_ring); 1140 xep->xep_state &= ~XHCI_ENDPOINT_PERIODIC; 1141 mutex_exit(&xhcip->xhci_lock); 1142 return (ret); 1143 } 1144 } 1145 1146 /* 1147 * Don't worry about freeing memory, it'll be done when the endpoint 1148 * closes and the whole system is reset. 1149 */ 1150 xpp->xpp_usb_req = usb_req; 1151 xpp->xpp_poll_state = XHCI_PERIODIC_POLL_ACTIVE; 1152 1153 ret = xhci_endpoint_ring(xhcip, xd, xep); 1154 mutex_exit(&xhcip->xhci_lock); 1155 return (ret); 1156 } 1157 1158 static int 1159 xhci_hcdi_intr_oneshot(xhci_t *xhcip, usba_pipe_handle_data_t *ph, 1160 usb_intr_req_t *uirp, usb_flags_t usb_flags) 1161 { 1162 uint_t epid; 1163 xhci_device_t *xd; 1164 xhci_endpoint_t *xep; 1165 xhci_transfer_t *xt; 1166 boolean_t datain; 1167 mblk_t *mp = NULL; 1168 1169 mutex_enter(&xhcip->xhci_lock); 1170 if (xhcip->xhci_state & XHCI_S_ERROR) { 1171 mutex_exit(&xhcip->xhci_lock); 1172 return (USB_HC_HARDWARE_ERROR); 1173 } 1174 1175 xd = usba_hcdi_get_device_private(ph->p_usba_device); 1176 epid = xhci_endpoint_pipe_to_epid(ph); 1177 if (xd->xd_endpoints[epid] == NULL) { 1178 xhci_error(xhcip, "asked to do interrupt transfer on slot %d, " 1179 "port %d, endpoint: %d, but no endpoint structure", 1180 xd->xd_slot, xd->xd_port, epid); 1181 mutex_exit(&xhcip->xhci_lock); 1182 return (USB_FAILURE); 1183 } 1184 xep = xd->xd_endpoints[epid]; 1185 1186 mutex_exit(&xhcip->xhci_lock); 1187 1188 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) { 1189 datain = B_TRUE; 1190 } else { 1191 datain = B_FALSE; 1192 } 1193 1194 xt = xhci_transfer_alloc(xhcip, xep, uirp->intr_len, 0, usb_flags); 1195 if (xt == NULL) { 1196 return (USB_NO_RESOURCES); 1197 } 1198 1199 xt->xt_usba_req = (usb_opaque_t)uirp; 1200 xt->xt_timeout = uirp->intr_timeout; 1201 if (xt->xt_timeout == 0) { 1202 xt->xt_timeout = HCDI_DEFAULT_TIMEOUT; 1203 } 1204 1205 /* 1206 * Unlike other request types, USB Interrupt-IN requests aren't required 1207 * to have allocated the message block for data. If they haven't, we 1208 * take care of that now. 1209 */ 1210 if (uirp->intr_len > 0 && datain == B_TRUE && uirp->intr_data == NULL) { 1211 if (usb_flags & USB_FLAGS_SLEEP) { 1212 mp = allocb_wait(uirp->intr_len, BPRI_LO, STR_NOSIG, 1213 NULL); 1214 } else { 1215 mp = allocb(uirp->intr_len, 0); 1216 } 1217 if (mp == NULL) { 1218 xhci_transfer_free(xhcip, xt); 1219 mutex_exit(&xhcip->xhci_lock); 1220 return (USB_NO_RESOURCES); 1221 } 1222 uirp->intr_data = mp; 1223 } 1224 1225 if (uirp->intr_len > 0 && datain == B_FALSE) { 1226 xhci_transfer_copy(xt, uirp->intr_data->b_rptr, uirp->intr_len, 1227 B_FALSE); 1228 if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) != 1229 DDI_FM_OK) { 1230 xhci_transfer_free(xhcip, xt); 1231 xhci_error(xhcip, "failed to synchronize interrupt " 1232 "transfer DMA memory on endpoint %u of " 1233 "device on slot %d and port %d: resetting " 1234 "device", xep->xep_num, xd->xd_slot, 1235 xd->xd_port); 1236 xhci_fm_runtime_reset(xhcip); 1237 return (USB_HC_HARDWARE_ERROR); 1238 } 1239 } 1240 1241 xhci_transfer_trb_fill_data(xep, xt, 0, datain); 1242 mutex_enter(&xhcip->xhci_lock); 1243 if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) { 1244 if (mp != NULL) { 1245 uirp->intr_data = NULL; 1246 freemsg(mp); 1247 } 1248 xhci_transfer_free(xhcip, xt); 1249 mutex_exit(&xhcip->xhci_lock); 1250 return (USB_NO_RESOURCES); 1251 } 1252 mutex_exit(&xhcip->xhci_lock); 1253 1254 return (USB_SUCCESS); 1255 } 1256 1257 /* 1258 * We've been asked to perform an interrupt transfer. When this is an interrupt 1259 * IN endpoint, that means that the hcd is being asked to start polling on the 1260 * endpoint. When the endpoint is the root hub, it effectively becomes synthetic 1261 * polling. 1262 * 1263 * When we have an interrupt out endpoint, then this is just a single simple 1264 * interrupt request that we send out and there isn't much special to do beyond 1265 * the normal activity. 1266 */ 1267 static int 1268 xhci_hcdi_pipe_intr_xfer(usba_pipe_handle_data_t *ph, usb_intr_req_t *uirp, 1269 usb_flags_t usb_flags) 1270 { 1271 int ret; 1272 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph); 1273 1274 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) { 1275 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) { 1276 ret = xhci_root_hub_intr_root_enable(xhcip, ph, uirp); 1277 } else if (uirp->intr_attributes & USB_ATTRS_ONE_XFER) { 1278 ret = xhci_hcdi_intr_oneshot(xhcip, ph, uirp, 1279 usb_flags); 1280 } else { 1281 ret = xhci_hcdi_periodic_init(xhcip, ph, 1282 (usb_opaque_t)uirp, uirp->intr_len, usb_flags); 1283 } 1284 } else { 1285 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) { 1286 return (USB_NOT_SUPPORTED); 1287 } 1288 ret = xhci_hcdi_intr_oneshot(xhcip, ph, uirp, usb_flags); 1289 } 1290 1291 return (ret); 1292 } 1293 1294 /* ARGSUSED */ 1295 static int 1296 xhci_hcdi_pipe_stop_intr_polling(usba_pipe_handle_data_t *ph, 1297 usb_flags_t usb_flags) 1298 { 1299 return (xhci_hcdi_pipe_poll_fini(ph, B_FALSE)); 1300 } 1301 1302 static int 1303 xhci_hcdi_isoc_periodic(xhci_t *xhcip, usba_pipe_handle_data_t *ph, 1304 usb_isoc_req_t *usrp, usb_flags_t usb_flags) 1305 { 1306 int i; 1307 size_t count; 1308 1309 count = 0; 1310 for (i = 0; i < usrp->isoc_pkts_count; i++) { 1311 count += usrp->isoc_pkt_descr[i].isoc_pkt_length; 1312 } 1313 1314 return (xhci_hcdi_periodic_init(xhcip, ph, (usb_opaque_t)usrp, count, 1315 usb_flags)); 1316 } 1317 1318 /* 1319 * This is used to create an isochronous request to send data out to the device. 1320 * This is a single one shot request, it is not something that we'll have to 1321 * repeat over and over. 1322 */ 1323 static int 1324 xhci_hcdi_isoc_oneshot(xhci_t *xhcip, usba_pipe_handle_data_t *ph, 1325 usb_isoc_req_t *usrp, usb_flags_t usb_flags) 1326 { 1327 int i, ret; 1328 uint_t epid; 1329 size_t count, mblen; 1330 xhci_device_t *xd; 1331 xhci_endpoint_t *xep; 1332 xhci_transfer_t *xt; 1333 1334 count = 0; 1335 for (i = 0; i < usrp->isoc_pkts_count; i++) { 1336 count += usrp->isoc_pkt_descr[i].isoc_pkt_length; 1337 } 1338 mblen = MBLKL(usrp->isoc_data); 1339 1340 if (count != mblen) { 1341 return (USB_INVALID_ARGS); 1342 } 1343 1344 mutex_enter(&xhcip->xhci_lock); 1345 if (xhcip->xhci_state & XHCI_S_ERROR) { 1346 mutex_exit(&xhcip->xhci_lock); 1347 return (USB_HC_HARDWARE_ERROR); 1348 } 1349 1350 xd = usba_hcdi_get_device_private(ph->p_usba_device); 1351 epid = xhci_endpoint_pipe_to_epid(ph); 1352 if (xd->xd_endpoints[epid] == NULL) { 1353 xhci_error(xhcip, "asked to do isochronous transfer on slot " 1354 "%d, port %d, endpoint: %d, but no endpoint structure", 1355 xd->xd_slot, xd->xd_port, epid); 1356 mutex_exit(&xhcip->xhci_lock); 1357 return (USB_FAILURE); 1358 } 1359 xep = xd->xd_endpoints[epid]; 1360 mutex_exit(&xhcip->xhci_lock); 1361 1362 xt = xhci_transfer_alloc(xhcip, xep, mblen, usrp->isoc_pkts_count, 1363 usb_flags); 1364 if (xt == NULL) { 1365 return (USB_NO_RESOURCES); 1366 } 1367 xt->xt_usba_req = (usb_opaque_t)usrp; 1368 1369 /* 1370 * USBA doesn't provide any real way for a timeout to be defined for an 1371 * isochronous event. However, since we technically aren't a periodic 1372 * endpoint, go ahead and always set the default timeout. It's better 1373 * than nothing. 1374 */ 1375 xt->xt_timeout = HCDI_DEFAULT_TIMEOUT; 1376 1377 xhci_transfer_copy(xt, usrp->isoc_data->b_rptr, mblen, B_FALSE); 1378 if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) != DDI_FM_OK) { 1379 xhci_transfer_free(xhcip, xt); 1380 xhci_error(xhcip, "failed to synchronize isochronous " 1381 "transfer DMA memory on endpoint %u of " 1382 "device on slot %d and port %d: resetting " 1383 "device", xep->xep_num, xd->xd_slot, 1384 xd->xd_port); 1385 xhci_fm_runtime_reset(xhcip); 1386 return (USB_HC_HARDWARE_ERROR); 1387 } 1388 1389 /* 1390 * Fill in the ISOC data. Note, that we always use ASAP scheduling and 1391 * we don't support specifying the frame at this time, for better or 1392 * worse. 1393 */ 1394 xhci_hcdi_isoc_transfer_fill(xd, xep, xt, usrp); 1395 1396 mutex_enter(&xhcip->xhci_lock); 1397 ret = xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE); 1398 mutex_exit(&xhcip->xhci_lock); 1399 1400 return (ret); 1401 } 1402 1403 static int 1404 xhci_hcdi_pipe_isoc_xfer(usba_pipe_handle_data_t *ph, usb_isoc_req_t *usrp, 1405 usb_flags_t usb_flags) 1406 { 1407 int ret; 1408 xhci_t *xhcip; 1409 1410 xhcip = xhci_hcdi_get_xhcip(ph); 1411 1412 /* 1413 * We don't support isochronous transactions on the root hub at all. 1414 * Always fail them if for some reason we end up here. 1415 */ 1416 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) { 1417 return (USB_NOT_SUPPORTED); 1418 } 1419 1420 /* 1421 * We do not support being asked to set the frame ID at this time. We 1422 * require that everything specify the attribute 1423 * USB_ATTRS_ISOC_XFER_ASAP. 1424 */ 1425 if (!(usrp->isoc_attributes & USB_ATTRS_ISOC_XFER_ASAP)) { 1426 return (USB_NOT_SUPPORTED); 1427 } 1428 1429 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) { 1430 /* 1431 * Note, there is no such thing as a non-periodic isochronous 1432 * incoming transfer. 1433 */ 1434 ret = xhci_hcdi_isoc_periodic(xhcip, ph, usrp, usb_flags); 1435 } else { 1436 ret = xhci_hcdi_isoc_oneshot(xhcip, ph, usrp, usb_flags); 1437 } 1438 1439 return (ret); 1440 } 1441 1442 /* ARGSUSED */ 1443 static int 1444 xhci_hcdi_pipe_stop_isoc_polling(usba_pipe_handle_data_t *ph, 1445 usb_flags_t usb_flags) 1446 { 1447 return (xhci_hcdi_pipe_poll_fini(ph, B_FALSE)); 1448 } 1449 1450 /* 1451 * This is asking us for the current frame number. The USBA expects this to 1452 * actually be a bit of a fiction, as it tries to maintain a frame number well 1453 * beyond what the hardware actually contains in its registers. Hardware 1454 * basically has a 14-bit counter, whereas we need to have a constant amount of 1455 * milliseconds. 1456 * 1457 * Today, no client drivers actually use this API and everyone specifies the 1458 * attribute to say that we should schedule things ASAP. So until we have some 1459 * real device that want this functionality, we're going to fail. 1460 */ 1461 /* ARGSUSED */ 1462 static int 1463 xhci_hcdi_get_current_frame_number(usba_device_t *usba_device, 1464 usb_frame_number_t *frame_number) 1465 { 1466 return (USB_FAILURE); 1467 } 1468 1469 /* 1470 * See the comments around the XHCI_ISOC_MAX_TRB macro for more information. 1471 */ 1472 /* ARGSUSED */ 1473 static int 1474 xhci_hcdi_get_max_isoc_pkts(usba_device_t *usba_device, 1475 uint_t *max_isoc_pkts_per_request) 1476 { 1477 *max_isoc_pkts_per_request = XHCI_ISOC_MAX_TRB; 1478 return (USB_SUCCESS); 1479 } 1480 1481 /* 1482 * VERSION 2 ops and helpers 1483 */ 1484 1485 static void 1486 xhci_hcdi_device_free(xhci_device_t *xd) 1487 { 1488 xhci_dma_free(&xd->xd_ictx); 1489 xhci_dma_free(&xd->xd_octx); 1490 mutex_destroy(&xd->xd_imtx); 1491 kmem_free(xd, sizeof (xhci_device_t)); 1492 } 1493 1494 /* 1495 * Calculate the device's route string. In USB 3.0 the route string is a 20-bit 1496 * number. Each four bits represent a port number attached to a deeper hub. 1497 * Particularly it represents the port on that current hub that you need to go 1498 * down to reach the next device. Bits 0-3 represent the first *external* hub. 1499 * So a device connected to a root hub has a route string of zero. Imagine the 1500 * following set of devices: 1501 * 1502 * . port 2 . port 5 1503 * . . 1504 * +----------+ . +--------+ . +-------+ 1505 * | root hub |-*->| hub 1 |-*->| hub 2 | 1506 * +----------+ +--------+ +-------+ 1507 * * . port 12 * . port 8 * . port 1 1508 * v v v 1509 * +-------+ +-------+ +-------+ 1510 * | dev a | | dev b | | dev c | 1511 * +-------+ +-------+ +-------+ 1512 * 1513 * So, based on the above diagram, device a should have a route string of 0, 1514 * because it's directly connected to the root port. Device b would simply have 1515 * a route string of 8. This is because it travels through one non-root hub, hub 1516 * 1, and it does so on port 8. The root ports basically don't matter. Device c 1517 * would then have a route string of 0x15, as it's first traversing through hub 1518 * 1 on port 2 and then hub 2 on port 5. 1519 * 1520 * Finally, it's worth mentioning that because it's a four bit field, if for 1521 * some reason a device has more than 15 ports, we just treat the value as 15. 1522 * 1523 * Note, as part of this, we also grab what port on the root hub this whole 1524 * chain is on, as we're required to store that information in the slot context. 1525 */ 1526 static void 1527 xhci_hcdi_device_route(usba_device_t *ud, uint32_t *routep, uint32_t *root_port) 1528 { 1529 uint32_t route = 0; 1530 usba_device_t *hub = ud->usb_parent_hub; 1531 usba_device_t *port_dev = ud; 1532 1533 ASSERT(hub != NULL); 1534 1535 /* 1536 * Iterate over every hub, updating the route as we go. When we 1537 * encounter a hub without a parent, then we're at the root hub. At 1538 * which point, the port we want is on port_dev (the child of hub). 1539 */ 1540 while (hub->usb_parent_hub != NULL) { 1541 uint32_t p; 1542 1543 p = port_dev->usb_port; 1544 if (p > 15) 1545 p = 15; 1546 route <<= 4; 1547 route |= p & 0xf; 1548 port_dev = hub; 1549 hub = hub->usb_parent_hub; 1550 } 1551 1552 ASSERT(port_dev->usb_parent_hub == hub); 1553 *root_port = port_dev->usb_port; 1554 *routep = XHCI_ROUTE_MASK(route); 1555 } 1556 1557 /* 1558 * If a low or full speed device is behind a high-speed device that is not a 1559 * root hub, then we must include the port and slot of that device. USBA already 1560 * stores this device in the usb_hs_hub_usba_dev member. 1561 */ 1562 static uint32_t 1563 xhci_hcdi_device_tt(usba_device_t *ud) 1564 { 1565 uint32_t ret; 1566 xhci_device_t *xd; 1567 1568 if (ud->usb_port_status >= USBA_HIGH_SPEED_DEV) 1569 return (0); 1570 1571 if (ud->usb_hs_hub_usba_dev == NULL) 1572 return (0); 1573 1574 ASSERT(ud->usb_hs_hub_usba_dev != NULL); 1575 ASSERT(ud->usb_hs_hub_usba_dev->usb_parent_hub != NULL); 1576 xd = usba_hcdi_get_device_private(ud->usb_hs_hub_usba_dev); 1577 ASSERT(xd != NULL); 1578 1579 ret = XHCI_SCTX_SET_TT_HUB_SID(xd->xd_slot); 1580 ret |= XHCI_SCTX_SET_TT_PORT_NUM(ud->usb_hs_hub_usba_dev->usb_port); 1581 1582 return (ret); 1583 } 1584 1585 /* 1586 * Initialize a new device. This allocates a device slot from the controller, 1587 * which tranfers it to our control. 1588 */ 1589 static int 1590 xhci_hcdi_device_init(usba_device_t *ud, usb_port_t port, void **hcdpp) 1591 { 1592 int ret, i; 1593 xhci_device_t *xd; 1594 ddi_device_acc_attr_t acc; 1595 ddi_dma_attr_t attr; 1596 xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud); 1597 size_t isize, osize, incr; 1598 uint32_t route, rp, info, info2, tt; 1599 1600 xd = kmem_zalloc(sizeof (xhci_device_t), KM_SLEEP); 1601 xd->xd_port = port; 1602 xd->xd_usbdev = ud; 1603 mutex_init(&xd->xd_imtx, NULL, MUTEX_DRIVER, 1604 (void *)(uintptr_t)xhcip->xhci_intr_pri); 1605 1606 /* 1607 * The size of the context structures is based upon the presence of the 1608 * context flag which determines whether we have a 32-byte or 64-byte 1609 * context. Note that the input context always has to account for the 1610 * entire size of the xhci_input_contex_t, which is 32-bytes by default. 1611 */ 1612 if (xhcip->xhci_caps.xcap_flags & XCAP_CSZ) { 1613 incr = 64; 1614 osize = XHCI_DEVICE_CONTEXT_64; 1615 isize = XHCI_DEVICE_CONTEXT_64 + incr; 1616 } else { 1617 incr = 32; 1618 osize = XHCI_DEVICE_CONTEXT_32; 1619 isize = XHCI_DEVICE_CONTEXT_32 + incr; 1620 } 1621 1622 xhci_dma_acc_attr(xhcip, &acc); 1623 xhci_dma_dma_attr(xhcip, &attr); 1624 if (xhci_dma_alloc(xhcip, &xd->xd_ictx, &attr, &acc, B_TRUE, 1625 isize, B_FALSE) == B_FALSE) { 1626 xhci_hcdi_device_free(xd); 1627 return (USB_NO_RESOURCES); 1628 } 1629 1630 xd->xd_input = (xhci_input_context_t *)xd->xd_ictx.xdb_va; 1631 xd->xd_slotin = (xhci_slot_context_t *)(xd->xd_ictx.xdb_va + incr); 1632 for (i = 0; i < XHCI_NUM_ENDPOINTS; i++) { 1633 xd->xd_endin[i] = 1634 (xhci_endpoint_context_t *)(xd->xd_ictx.xdb_va + 1635 (i + 2) * incr); 1636 } 1637 1638 if (xhci_dma_alloc(xhcip, &xd->xd_octx, &attr, &acc, B_TRUE, 1639 osize, B_FALSE) == B_FALSE) { 1640 xhci_hcdi_device_free(xd); 1641 return (USB_NO_RESOURCES); 1642 } 1643 xd->xd_slotout = (xhci_slot_context_t *)xd->xd_octx.xdb_va; 1644 for (i = 0; i < XHCI_NUM_ENDPOINTS; i++) { 1645 xd->xd_endout[i] = 1646 (xhci_endpoint_context_t *)(xd->xd_octx.xdb_va + 1647 (i + 1) * incr); 1648 } 1649 1650 ret = xhci_command_enable_slot(xhcip, &xd->xd_slot); 1651 if (ret != USB_SUCCESS) { 1652 xhci_hcdi_device_free(xd); 1653 return (ret); 1654 } 1655 1656 /* 1657 * These are the default slot context and the endpoint zero context that 1658 * we're enabling. See 4.3.3. 1659 */ 1660 xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0) | 1661 XHCI_INCTX_MASK_DCI(1)); 1662 1663 /* 1664 * Note, we never need to set the MTT bit as illumos never enables the 1665 * alternate MTT interface. 1666 */ 1667 xhci_hcdi_device_route(ud, &route, &rp); 1668 info = XHCI_SCTX_SET_ROUTE(route) | XHCI_SCTX_SET_DCI(1); 1669 switch (ud->usb_port_status) { 1670 case USBA_LOW_SPEED_DEV: 1671 info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_LOW); 1672 break; 1673 case USBA_HIGH_SPEED_DEV: 1674 info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_HIGH); 1675 break; 1676 case USBA_FULL_SPEED_DEV: 1677 info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_FULL); 1678 break; 1679 case USBA_SUPER_SPEED_DEV: 1680 default: 1681 info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_SUPER); 1682 break; 1683 } 1684 info2 = XHCI_SCTX_SET_RHPORT(rp); 1685 tt = XHCI_SCTX_SET_IRQ_TARGET(0); 1686 tt |= xhci_hcdi_device_tt(ud); 1687 1688 xd->xd_slotin->xsc_info = LE_32(info); 1689 xd->xd_slotin->xsc_info2 = LE_32(info2); 1690 xd->xd_slotin->xsc_tt = LE_32(tt); 1691 1692 if ((ret = xhci_endpoint_init(xhcip, xd, NULL)) != 0) { 1693 (void) xhci_command_disable_slot(xhcip, xd->xd_slot); 1694 xhci_hcdi_device_free(xd); 1695 return (USB_HC_HARDWARE_ERROR); 1696 } 1697 1698 if (xhci_context_slot_output_init(xhcip, xd) != B_TRUE) { 1699 (void) xhci_command_disable_slot(xhcip, xd->xd_slot); 1700 xhci_endpoint_fini(xd, 0); 1701 xhci_hcdi_device_free(xd); 1702 return (USB_HC_HARDWARE_ERROR); 1703 } 1704 1705 if ((ret = xhci_command_set_address(xhcip, xd, B_TRUE)) != 0) { 1706 (void) xhci_command_disable_slot(xhcip, xd->xd_slot); 1707 xhci_context_slot_output_fini(xhcip, xd); 1708 xhci_endpoint_fini(xd, 0); 1709 xhci_hcdi_device_free(xd); 1710 return (ret); 1711 } 1712 1713 mutex_enter(&xhcip->xhci_lock); 1714 list_insert_tail(&xhcip->xhci_usba.xa_devices, xd); 1715 mutex_exit(&xhcip->xhci_lock); 1716 1717 *hcdpp = xd; 1718 return (ret); 1719 } 1720 1721 /* 1722 * We're tearing down a device now. 1723 */ 1724 static void 1725 xhci_hcdi_device_fini(usba_device_t *ud, void *hcdp) 1726 { 1727 int ret; 1728 xhci_endpoint_t *xep; 1729 xhci_device_t *xd; 1730 xhci_t *xhcip; 1731 1732 /* 1733 * Right now, it's theoretically possible that USBA may try and call 1734 * us here even if we hadn't successfully finished the device_init() 1735 * endpoint. We should probably modify the USBA to make sure that this 1736 * can't happen. 1737 */ 1738 if (hcdp == NULL) 1739 return; 1740 1741 xd = hcdp; 1742 xhcip = xhci_hcdi_get_xhcip_from_dev(ud); 1743 1744 /* 1745 * Make sure we have no timeout running on the default endpoint still. 1746 */ 1747 xep = xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT]; 1748 mutex_enter(&xhcip->xhci_lock); 1749 xep->xep_state |= XHCI_ENDPOINT_TEARDOWN; 1750 mutex_exit(&xhcip->xhci_lock); 1751 (void) untimeout(xep->xep_timeout); 1752 1753 /* 1754 * Go ahead and disable the slot. There's no reason to do anything 1755 * special about the default endpoint as it will be disabled as a part 1756 * of the slot disabling. However, if this all fails, we'll leave this 1757 * sitting here in a failed state, eating up a device slot. It is 1758 * unlikely this will occur. 1759 */ 1760 ret = xhci_command_disable_slot(xhcip, xd->xd_slot); 1761 if (ret != USB_SUCCESS) { 1762 xhci_error(xhcip, "failed to disable slot %d: %d", 1763 xd->xd_slot, ret); 1764 return; 1765 } 1766 1767 xhci_context_slot_output_fini(xhcip, xd); 1768 1769 /* 1770 * Once the slot is disabled, we can free any endpoints that were 1771 * opened. 1772 */ 1773 for (uint_t n = 0; n < XHCI_NUM_ENDPOINTS; n++) { 1774 if (xd->xd_endpoints[n] != NULL) { 1775 xhci_endpoint_fini(xd, n); 1776 } 1777 } 1778 1779 mutex_enter(&xhcip->xhci_lock); 1780 list_remove(&xhcip->xhci_usba.xa_devices, xd); 1781 mutex_exit(&xhcip->xhci_lock); 1782 1783 xhci_hcdi_device_free(xd); 1784 } 1785 1786 /* 1787 * Synchronously attempt to set the device address. For xHCI this involves it 1788 * deciding what address to use. 1789 */ 1790 static int 1791 xhci_hcdi_device_address(usba_device_t *ud) 1792 { 1793 int ret; 1794 xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud); 1795 xhci_device_t *xd = usba_hcdi_get_device_private(ud); 1796 xhci_endpoint_t *xep; 1797 1798 mutex_enter(&xhcip->xhci_lock); 1799 1800 /* 1801 * This device may already be addressed from the perspective of the xhci 1802 * controller. For example, the device this represents may have been 1803 * unconfigured, which does not actually remove the slot or other 1804 * information, merely tears down all the active use of it and the child 1805 * driver. In such cases, if we're already addressed, just return 1806 * success. The actual USB address is a fiction for USBA anyways. 1807 */ 1808 if (xd->xd_addressed == B_TRUE) { 1809 mutex_exit(&xhcip->xhci_lock); 1810 return (USB_SUCCESS); 1811 } 1812 1813 ASSERT(xd->xd_addressed == B_FALSE); 1814 xd->xd_addressed = B_TRUE; 1815 VERIFY3P(xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT], !=, NULL); 1816 xep = xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT]; 1817 mutex_exit(&xhcip->xhci_lock); 1818 1819 if ((ret = xhci_endpoint_setup_default_context(xhcip, xd, xep)) != 0) { 1820 ASSERT(ret == EIO); 1821 return (USB_HC_HARDWARE_ERROR); 1822 } 1823 1824 ret = xhci_command_set_address(xhcip, xd, B_FALSE); 1825 1826 if (ret != USB_SUCCESS) { 1827 mutex_enter(&xhcip->xhci_lock); 1828 xd->xd_addressed = B_FALSE; 1829 mutex_exit(&xhcip->xhci_lock); 1830 } 1831 1832 return (ret); 1833 } 1834 1835 /* 1836 * This is called relatively early on in a hub's life time. At this point, it's 1837 * descriptors have all been pulled and the default control pipe is still open. 1838 * What we need to do is go through and update the slot context to indicate that 1839 * this is a hub, otherwise, the controller will never let us speak to 1840 * downstream ports. 1841 */ 1842 static int 1843 xhci_hcdi_hub_update(usba_device_t *ud, uint8_t nports, uint8_t tt) 1844 { 1845 int ret; 1846 xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud); 1847 xhci_device_t *xd = usba_hcdi_get_device_private(ud); 1848 1849 if (xd == NULL) 1850 return (USB_FAILURE); 1851 1852 if (ud->usb_hubdi == NULL) { 1853 return (USB_FAILURE); 1854 } 1855 1856 mutex_enter(&xd->xd_imtx); 1857 1858 /* 1859 * Note, that usba never sets the interface of a hub to Multi TT. Hence 1860 * why we're never setting the MTT bit in xsc_info. 1861 */ 1862 xd->xd_slotin->xsc_info |= LE_32(XHCI_SCTX_SET_HUB(1)); 1863 xd->xd_slotin->xsc_info2 |= LE_32(XHCI_SCTX_SET_NPORTS(nports)); 1864 if (ud->usb_port_status == USBA_HIGH_SPEED_DEV) 1865 xd->xd_slotin->xsc_tt |= LE_32(XHCI_SCTX_SET_TT_THINK_TIME(tt)); 1866 1867 /* 1868 * We're only updating the slot context, no endpoint contexts should be 1869 * touched. 1870 */ 1871 xd->xd_input->xic_drop_flags = LE_32(0); 1872 xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0)); 1873 1874 ret = xhci_command_evaluate_context(xhcip, xd); 1875 mutex_exit(&xd->xd_imtx); 1876 return (ret); 1877 } 1878 1879 void 1880 xhci_hcd_fini(xhci_t *xhcip) 1881 { 1882 usba_hcdi_unregister(xhcip->xhci_dip); 1883 usba_free_hcdi_ops(xhcip->xhci_usba.xa_ops); 1884 list_destroy(&xhcip->xhci_usba.xa_pipes); 1885 list_destroy(&xhcip->xhci_usba.xa_devices); 1886 } 1887 1888 int 1889 xhci_hcd_init(xhci_t *xhcip) 1890 { 1891 usba_hcdi_register_args_t hreg; 1892 usba_hcdi_ops_t *ops; 1893 1894 ops = usba_alloc_hcdi_ops(); 1895 VERIFY(ops != NULL); 1896 1897 ops->usba_hcdi_ops_version = HCDI_OPS_VERSION; 1898 ops->usba_hcdi_dip = xhcip->xhci_dip; 1899 1900 ops->usba_hcdi_pm_support = xhci_hcdi_pm_support; 1901 ops->usba_hcdi_pipe_open = xhci_hcdi_pipe_open; 1902 ops->usba_hcdi_pipe_close = xhci_hcdi_pipe_close; 1903 ops->usba_hcdi_pipe_reset = xhci_hcdi_pipe_reset; 1904 ops->usba_hcdi_pipe_reset_data_toggle = 1905 xhci_hcdi_pipe_reset_data_toggle; 1906 ops->usba_hcdi_pipe_ctrl_xfer = xhci_hcdi_pipe_ctrl_xfer; 1907 ops->usba_hcdi_bulk_transfer_size = xhci_hcdi_bulk_transfer_size; 1908 ops->usba_hcdi_pipe_bulk_xfer = xhci_hcdi_pipe_bulk_xfer; 1909 ops->usba_hcdi_pipe_intr_xfer = xhci_hcdi_pipe_intr_xfer; 1910 ops->usba_hcdi_pipe_stop_intr_polling = 1911 xhci_hcdi_pipe_stop_intr_polling; 1912 ops->usba_hcdi_pipe_isoc_xfer = xhci_hcdi_pipe_isoc_xfer; 1913 ops->usba_hcdi_pipe_stop_isoc_polling = 1914 xhci_hcdi_pipe_stop_isoc_polling; 1915 ops->usba_hcdi_get_current_frame_number = 1916 xhci_hcdi_get_current_frame_number; 1917 ops->usba_hcdi_get_max_isoc_pkts = xhci_hcdi_get_max_isoc_pkts; 1918 ops->usba_hcdi_console_input_init = xhci_hcdi_console_input_init; 1919 ops->usba_hcdi_console_input_fini = xhci_hcdi_console_input_fini; 1920 ops->usba_hcdi_console_input_enter = xhci_hcdi_console_input_enter; 1921 ops->usba_hcdi_console_read = xhci_hcdi_console_read; 1922 ops->usba_hcdi_console_input_exit = xhci_hcdi_console_input_exit; 1923 1924 ops->usba_hcdi_console_output_init = xhci_hcdi_console_output_init; 1925 ops->usba_hcdi_console_output_fini = xhci_hcdi_console_output_fini; 1926 ops->usba_hcdi_console_output_enter = xhci_hcdi_console_output_enter; 1927 ops->usba_hcdi_console_write = xhci_hcdi_console_write; 1928 ops->usba_hcdi_console_output_exit = xhci_hcdi_console_output_exit; 1929 1930 ops->usba_hcdi_device_init = xhci_hcdi_device_init; 1931 ops->usba_hcdi_device_fini = xhci_hcdi_device_fini; 1932 ops->usba_hcdi_device_address = xhci_hcdi_device_address; 1933 ops->usba_hcdi_hub_update = xhci_hcdi_hub_update; 1934 1935 hreg.usba_hcdi_register_version = HCDI_REGISTER_VERSION; 1936 hreg.usba_hcdi_register_dip = xhcip->xhci_dip; 1937 hreg.usba_hcdi_register_ops = ops; 1938 1939 /* 1940 * We're required to give xhci a set of DMA attributes that it may loan 1941 * out to other devices. Therefore we'll be conservative with what we 1942 * end up giving it. 1943 */ 1944 xhci_dma_dma_attr(xhcip, &xhcip->xhci_usba.xa_dma_attr); 1945 hreg.usba_hcdi_register_dma_attr = &xhcip->xhci_usba.xa_dma_attr; 1946 1947 hreg.usba_hcdi_register_iblock_cookie = 1948 (ddi_iblock_cookie_t)(uintptr_t)xhcip->xhci_intr_pri; 1949 1950 if (usba_hcdi_register(&hreg, 0) != DDI_SUCCESS) { 1951 usba_free_hcdi_ops(ops); 1952 return (DDI_FAILURE); 1953 } 1954 1955 xhcip->xhci_usba.xa_ops = ops; 1956 1957 list_create(&xhcip->xhci_usba.xa_devices, sizeof (xhci_device_t), 1958 offsetof(xhci_device_t, xd_link)); 1959 list_create(&xhcip->xhci_usba.xa_pipes, sizeof (xhci_pipe_t), 1960 offsetof(xhci_pipe_t, xp_link)); 1961 1962 1963 return (DDI_SUCCESS); 1964 } 1965