1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2016 Joyent, Inc. 14 */ 15 16 /* 17 * xHCI Endpoint Initialization and Management 18 * 19 * Please see the big theory statement in xhci.c for more information. 20 */ 21 22 #include <sys/usb/hcd/xhci/xhci.h> 23 #include <sys/sdt.h> 24 25 boolean_t 26 xhci_endpoint_is_periodic_in(xhci_endpoint_t *xep) 27 { 28 usba_pipe_handle_data_t *ph; 29 30 ASSERT(xep != NULL); 31 ph = xep->xep_pipe; 32 ASSERT(ph != NULL); 33 34 return ((xep->xep_type == USB_EP_ATTR_INTR || 35 xep->xep_type == USB_EP_ATTR_ISOCH) && 36 (ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN); 37 } 38 39 /* 40 * Endpoints are a bit weirdly numbered. Endpoint zero is the default control 41 * endpoint, so the direction doesn't matter. For all the others, they're 42 * arranged as ep 1 out, ep 1 in, ep 2 out, ep 2 in. This is based on the layout 43 * of the Device Context Structure in xHCI 1.1 / 6.2.1. Therefore to go from the 44 * endpoint and direction, we know that endpoint n starts at 2n - 1. e.g. 45 * endpoint 1 starts at entry 1, endpoint 2 at entry 3, etc. Finally, the OUT 46 * direction comes first, followed by the IN direction. So if we're getting the 47 * endpoint for one of those, then we have to deal with that. 48 */ 49 uint_t 50 xhci_endpoint_pipe_to_epid(usba_pipe_handle_data_t *ph) 51 { 52 int ep; 53 54 ep = ph->p_ep.bEndpointAddress & USB_EP_NUM_MASK; 55 if (ep == 0) 56 return (ep); 57 ep = ep * 2 - 1; 58 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) 59 ep++; 60 61 VERIFY(ep < XHCI_NUM_ENDPOINTS); 62 return (ep); 63 } 64 65 /* 66 * The assumption is that someone calling this owns this endpoint / device and 67 * that it's in a state where it's safe to zero out that information. 68 */ 69 void 70 xhci_endpoint_fini(xhci_device_t *xd, int endpoint) 71 { 72 xhci_endpoint_t *xep = xd->xd_endpoints[endpoint]; 73 74 VERIFY(xep != NULL); 75 xd->xd_endpoints[endpoint] = NULL; 76 77 xhci_ring_free(&xep->xep_ring); 78 cv_destroy(&xep->xep_state_cv); 79 list_destroy(&xep->xep_transfers); 80 kmem_free(xep, sizeof (xhci_endpoint_t)); 81 } 82 83 /* 84 * Set up the default control endpoint input context. This needs to be done 85 * before we address the device. Note, we separate out the default endpoint from 86 * others, as we must set this up before we have a pipe handle. 87 */ 88 int 89 xhci_endpoint_setup_default_context(xhci_t *xhcip, xhci_device_t *xd, 90 xhci_endpoint_t *xep) 91 { 92 uint_t mps; 93 xhci_endpoint_context_t *ectx; 94 uint64_t deq; 95 96 ectx = xd->xd_endin[xep->xep_num]; 97 VERIFY(ectx != NULL); 98 99 /* 100 * We may or may not have a device descriptor. This should match the 101 * same initial sizes that are done in hubd_create_child(). 102 * 103 * Note, since we don't necessarily have an endpoint descriptor yet to 104 * base this on we instead use the device's defaults if available. This 105 * is different from normal endpoints for which there's always a 106 * specific descriptor. 107 */ 108 switch (xd->xd_usbdev->usb_port_status) { 109 case USBA_LOW_SPEED_DEV: 110 if (xd->xd_usbdev->usb_dev_descr != NULL) { 111 mps = xd->xd_usbdev->usb_dev_descr->bMaxPacketSize0; 112 } else { 113 mps = 8; 114 } 115 break; 116 case USBA_FULL_SPEED_DEV: 117 case USBA_HIGH_SPEED_DEV: 118 if (xd->xd_usbdev->usb_dev_descr != NULL) { 119 mps = xd->xd_usbdev->usb_dev_descr->bMaxPacketSize0; 120 } else { 121 mps = 64; 122 } 123 break; 124 case USBA_SUPER_SPEED_DEV: 125 default: 126 if (xd->xd_usbdev->usb_dev_descr != NULL) { 127 mps = xd->xd_usbdev->usb_dev_descr->bMaxPacketSize0; 128 mps = 1 << mps; 129 } else { 130 mps = 512; 131 } 132 break; 133 } 134 135 bzero(ectx, sizeof (xhci_endpoint_context_t)); 136 ectx->xec_info = LE_32(0); 137 ectx->xec_info2 = LE_32(XHCI_EPCTX_SET_CERR(3) | 138 XHCI_EPCTX_SET_EPTYPE(XHCI_EPCTX_TYPE_CTRL) | 139 XHCI_EPCTX_SET_MAXB(0) | XHCI_EPCTX_SET_MPS(mps)); 140 deq = xhci_dma_pa(&xep->xep_ring.xr_dma) + sizeof (xhci_trb_t) * 141 xep->xep_ring.xr_tail; 142 ectx->xec_dequeue = LE_64(deq | xep->xep_ring.xr_cycle); 143 ectx->xec_txinfo = LE_32(XHCI_EPCTX_MAX_ESIT_PAYLOAD(0) | 144 XHCI_EPCTX_AVG_TRB_LEN(XHCI_CONTEXT_DEF_CTRL_ATL)); 145 146 XHCI_DMA_SYNC(xd->xd_ictx, DDI_DMA_SYNC_FORDEV); 147 if (xhci_check_dma_handle(xhcip, &xd->xd_ictx) != DDI_FM_OK) { 148 xhci_error(xhcip, "failed to initialize default device input " 149 "context on slot %d and port %d for endpoint %u: " 150 "encountered fatal FM error synchronizing input context " 151 "DMA memory", xd->xd_slot, xd->xd_port, xep->xep_num); 152 xhci_fm_runtime_reset(xhcip); 153 return (EIO); 154 } 155 156 return (0); 157 } 158 159 /* 160 * Determine if we need to update the maximum packet size of the default 161 * control endpoint. This may happen because we start with the default size 162 * before we have a descriptor and then it may change. For example, with 163 * full-speed devices that may have either an 8 or 64 byte maximum packet size. 164 */ 165 int 166 xhci_endpoint_update_default(xhci_t *xhcip, xhci_device_t *xd, 167 xhci_endpoint_t *xep) 168 { 169 int mps, desc, info, ret; 170 ASSERT(xd->xd_usbdev != NULL); 171 172 mps = XHCI_EPCTX_GET_MPS(xd->xd_endout[xep->xep_num]->xec_info2); 173 desc = xd->xd_usbdev->usb_dev_descr->bMaxPacketSize0; 174 if (xd->xd_usbdev->usb_port_status >= USBA_SUPER_SPEED_DEV) { 175 desc = 1 << desc; 176 } 177 178 if (mps == desc) 179 return (USB_SUCCESS); 180 181 /* 182 * Update only the context for the default control endpoint. 183 */ 184 mutex_enter(&xd->xd_imtx); 185 info = LE_32(xd->xd_endout[xep->xep_num]->xec_info2); 186 info &= ~XHCI_EPCTX_SET_MPS(mps); 187 info |= XHCI_EPCTX_SET_MPS(desc); 188 xd->xd_endin[xep->xep_num]->xec_info2 = LE_32(info); 189 xd->xd_input->xic_drop_flags = LE_32(0); 190 xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(1)); 191 192 ret = xhci_command_evaluate_context(xhcip, xd); 193 mutex_exit(&xd->xd_imtx); 194 195 return (ret); 196 } 197 198 static uint_t 199 xhci_endpoint_epdesc_to_type(usb_ep_descr_t *ep) 200 { 201 int type = ep->bmAttributes & USB_EP_ATTR_MASK; 202 boolean_t in = (ep->bEndpointAddress & USB_EP_DIR_MASK) == 203 USB_EP_DIR_IN; 204 205 switch (type) { 206 case USB_EP_ATTR_CONTROL: 207 return (XHCI_EPCTX_TYPE_CTRL); 208 case USB_EP_ATTR_ISOCH: 209 if (in == B_TRUE) 210 return (XHCI_EPCTX_TYPE_ISOCH_IN); 211 return (XHCI_EPCTX_TYPE_ISOCH_OUT); 212 case USB_EP_ATTR_BULK: 213 if (in == B_TRUE) 214 return (XHCI_EPCTX_TYPE_BULK_IN); 215 return (XHCI_EPCTX_TYPE_BULK_OUT); 216 case USB_EP_ATTR_INTR: 217 if (in == B_TRUE) 218 return (XHCI_EPCTX_TYPE_INTR_IN); 219 return (XHCI_EPCTX_TYPE_INTR_OUT); 220 default: 221 panic("bad USB attribute type: %d", type); 222 } 223 224 /* LINTED: E_FUNC_NO_RET_VAL */ 225 } 226 227 static uint_t 228 xhci_endpoint_determine_burst(xhci_device_t *xd, xhci_endpoint_t *xep) 229 { 230 switch (xd->xd_usbdev->usb_port_status) { 231 case USBA_LOW_SPEED_DEV: 232 case USBA_FULL_SPEED_DEV: 233 /* 234 * Per xHCI 1.1 / 6.2.3.4, burst is always zero for these 235 * devices. 236 */ 237 return (0); 238 case USBA_HIGH_SPEED_DEV: 239 if (xep->xep_type == USB_EP_ATTR_CONTROL || 240 xep->xep_type == USB_EP_ATTR_BULK) 241 return (0); 242 return ((xep->xep_pipe->p_xep.uex_ep.wMaxPacketSize & 243 XHCI_CONTEXT_BURST_MASK) >> XHCI_CONTEXT_BURST_SHIFT); 244 default: 245 /* 246 * For these USB >= 3.0, this comes from the companion 247 * descriptor. 248 */ 249 ASSERT(xep->xep_pipe->p_xep.uex_flags & USB_EP_XFLAGS_SS_COMP); 250 return (xep->xep_pipe->p_xep.uex_ep_ss.bMaxBurst); 251 } 252 } 253 254 /* 255 * Convert a linear mapping of values that are in in the range of 1-255 into a 256 * 2^x value. Because we're supposed to round down for these calculations (see 257 * the note in xHCI 1.1 / 6.2.3.6) we can do this simply with a fls() and 258 * subtracting one. 259 */ 260 static uint_t 261 xhci_endpoint_linear_interval(usb_ep_descr_t *ep) 262 { 263 int exp; 264 int ival = ep->bInterval; 265 if (ival < 1) 266 ival = 1; 267 if (ival > 255) 268 ival = 255; 269 exp = ddi_fls(ival) - 1; 270 ASSERT(exp >= 0 && exp <= 7); 271 return (exp); 272 } 273 274 /* 275 * Convert the set of values that use a 2^(x-1) value for interval into a 2^x 276 * range. Note the valid input range is 1-16, so we clamp values based on this. 277 * See xHCI 1.1 / 6.2.3.6 for more information. 278 */ 279 static uint_t 280 xhci_endpoint_exponential_interval(usb_ep_descr_t *ep) 281 { 282 int ival; 283 284 ival = ep->bInterval; 285 if (ival < 1) 286 ival = 1; 287 if (ival > 16) 288 ival = 16; 289 ival--; 290 ASSERT(ival >= 0 && ival <= 15); 291 return (ival); 292 } 293 294 295 /* 296 * Determining the interval is unfortunately somewhat complicated as there are 297 * many differnet forms that things can take. This is all summarized in a 298 * somewhat helpful table, number 65, in xHCI 1.1 / 6.2.3.6. But here's 299 * basically the six different cases we have to consider: 300 * 301 * Case 1: Non-High Speed Bulk and Control Endpoints 302 * Always return 0. 303 * 304 * Case 2: Super Speed and High Speed Isoch and Intr endpoints 305 * Convert from a 2^(x-1) range to a 2^x range. 306 * 307 * Case 3: Full Speed Isochronous Endpoints 308 * As case 2, but add 3 as its values are in frames and we need to convert 309 * to microframes. Adding three to the result is the same as multiplying 310 * the initial value by 8. 311 * 312 * Case 4: Full speed and Low Speed Interrupt Endpoints 313 * These have a 1-255 ms range that we need to convert to a 2^x * 128 us 314 * range. We use the linear conversion and then add 3 to account for the 315 * multiplying by 8 conversion from frames to microframes. 316 * 317 * Case 5: High Speed Interrupt and Bulk Output 318 * These are a bit of a weird case. The spec and other implementations make 319 * it seem that it's similar to case 4, but without the fixed addition as 320 * its interpreted differently due to NAKs. 321 * 322 * Case 6: Low Speed Isochronous Endpoints 323 * These are not actually defined; however, like other implementations we 324 * treat them like case 4. 325 */ 326 static uint_t 327 xhci_endpoint_interval(xhci_device_t *xd, usb_ep_descr_t *ep) 328 { 329 int type = ep->bmAttributes & USB_EP_ATTR_MASK; 330 int speed = xd->xd_usbdev->usb_port_status; 331 332 /* 333 * Handle Cases 1 and 5 first. 334 */ 335 if (type == USB_EP_ATTR_CONTROL || type == USB_EP_ATTR_BULK) { 336 if (speed != USBA_HIGH_SPEED_DEV) 337 return (0); 338 return (xhci_endpoint_linear_interval(ep)); 339 } 340 341 /* 342 * Handle Isoch and Intr cases next. 343 */ 344 switch (speed) { 345 case USBA_LOW_SPEED_DEV: 346 /* 347 * Interrupt endpoints at low speed are the same as full speed, 348 * hence the fall through. 349 */ 350 if (type == USB_EP_ATTR_ISOCH) { 351 return (xhci_endpoint_exponential_interval(ep) + 3); 352 } 353 /* FALLTHROUGH */ 354 case USBA_FULL_SPEED_DEV: 355 return (xhci_endpoint_linear_interval(ep) + 3); 356 case USBA_HIGH_SPEED_DEV: 357 case USBA_SUPER_SPEED_DEV: 358 default: 359 /* 360 * Case 2. Treat any newer and faster speeds as Super Speed by 361 * default as USB 3.1 is effectively treated the same here. 362 */ 363 return (xhci_endpoint_exponential_interval(ep)); 364 } 365 } 366 367 /* 368 * The way to calculate the Maximum ESIT is described in xHCI 1.1 / 4.14.2. 369 * First off, this only applies to Interrupt and Isochronous descriptors. For 370 * Super Speed and newer things, it comes out of a descriptor. Otherwise we 371 * calculate it by doing 'Max Packet Size' * ('Max Burst' + 1). 372 */ 373 static uint_t 374 xhci_endpoint_max_esit(xhci_device_t *xd, xhci_endpoint_t *xep, uint_t mps, 375 uint_t burst) 376 { 377 if (xep->xep_type == USB_EP_ATTR_CONTROL || 378 xep->xep_type == USB_EP_ATTR_BULK) { 379 return (0); 380 } 381 382 /* 383 * Note that this will need to be updated for SuperSpeedPlus ISOC 384 * devices to pull from the secondary companion descriptor they use. 385 */ 386 if (xd->xd_usbdev->usb_port_status >= USBA_SUPER_SPEED_DEV) { 387 usb_ep_xdescr_t *ep_xdesc = &xep->xep_pipe->p_xep; 388 ASSERT(xep->xep_pipe->p_xep.uex_flags & USB_EP_XFLAGS_SS_COMP); 389 return (ep_xdesc->uex_ep_ss.wBytesPerInterval); 390 } 391 392 return (mps * (burst + 1)); 393 } 394 395 /* 396 * We've been asked to calculate and tell the xHCI controller an average TRB 397 * data length. This is talked about in an implementation note in xHCI 1.1 / 398 * 4.14.1.1. So, the reality is that it's hard to actually calculate this, as 399 * we're supposed to take into account all of the TRBs that we use on that ring. 400 * 401 * Surveying other xHCI drivers, they all agree on using the default of 8 for 402 * control endpoints; however, from there things get a little more fluid. For 403 * interrupt and isochronous endpoints, many device use the minimum of the max 404 * packet size and the device's pagesize. For bulk endpoints some folks punt and 405 * don't set anything and others try and set it to the pagesize. The xHCI 406 * implementation note suggests a 3k size here initially. For now, we'll just 407 * guess for bulk endpoints and use our page size as a determining factor for 408 * this and use the BSD style for others. Note Linux here only sets this value 409 * for control devices. 410 */ 411 static uint_t 412 xhci_endpoint_avg_trb(xhci_t *xhcip, usb_ep_descr_t *ep, int mps) 413 { 414 int type = ep->bmAttributes & USB_EP_ATTR_MASK; 415 416 switch (type) { 417 case USB_EP_ATTR_ISOCH: 418 case USB_EP_ATTR_INTR: 419 return (MIN(xhcip->xhci_caps.xcap_pagesize, mps)); 420 case USB_EP_ATTR_CONTROL: 421 return (XHCI_CONTEXT_DEF_CTRL_ATL); 422 case USB_EP_ATTR_BULK: 423 return (xhcip->xhci_caps.xcap_pagesize); 424 default: 425 panic("bad USB endpoint type: %d", type); 426 } 427 428 /* LINTED: E_FUNC_NO_RET_VAL */ 429 } 430 431 int 432 xhci_endpoint_setup_context(xhci_t *xhcip, xhci_device_t *xd, 433 xhci_endpoint_t *xep) 434 { 435 uint_t eptype, burst, ival, max_esit, avgtrb, mps, mult, cerr; 436 xhci_endpoint_context_t *ectx; 437 uint64_t deq; 438 439 /* 440 * For a USB >=3.0 device we should always have its companion descriptor 441 * provided for us by USBA. If it's not here, complain loudly and fail. 442 */ 443 if (xd->xd_usbdev->usb_port_status >= USBA_SUPER_SPEED_DEV && 444 (xep->xep_pipe->p_xep.uex_flags & USB_EP_XFLAGS_SS_COMP) == 0) { 445 const char *prod, *mfg; 446 447 prod = xd->xd_usbdev->usb_product_str; 448 if (prod == NULL) 449 prod = "Unknown Device"; 450 mfg = xd->xd_usbdev->usb_mfg_str; 451 if (mfg == NULL) 452 mfg = "Unknown Manufacturer"; 453 454 xhci_log(xhcip, "Encountered USB >=3.0 device without endpoint " 455 "companion descriptor. Ensure driver %s is properly using " 456 "usb_pipe_xopen() for device %s %s", 457 ddi_driver_name(xd->xd_usbdev->usb_dip), prod, mfg); 458 return (EINVAL); 459 } 460 461 ectx = xd->xd_endin[xep->xep_num]; 462 VERIFY(ectx != NULL); 463 VERIFY(xd->xd_usbdev->usb_dev_descr != NULL); 464 VERIFY(xep->xep_pipe != NULL); 465 466 mps = xep->xep_pipe->p_ep.wMaxPacketSize & XHCI_CONTEXT_MPS_MASK; 467 mult = XHCI_CONTEXT_DEF_MULT; 468 cerr = XHCI_CONTEXT_DEF_CERR; 469 470 switch (xep->xep_type) { 471 case USB_EP_ATTR_ISOCH: 472 /* 473 * When we have support for USB 3.1 SuperSpeedPlus devices, 474 * we'll need to make sure that we also check for its secondary 475 * endpoint companion descriptor here. 476 */ 477 /* 478 * Super Speed devices nominally have these xHCI super speed 479 * companion descriptors. We know that we're not properly 480 * grabbing them right now, so until we do, we should basically 481 * error about it. 482 */ 483 if (xd->xd_usbdev->usb_port_status >= USBA_SUPER_SPEED_DEV) { 484 ASSERT(xep->xep_pipe->p_xep.uex_flags & 485 USB_EP_XFLAGS_SS_COMP); 486 mult = xep->xep_pipe->p_xep.uex_ep_ss.bmAttributes & 487 USB_EP_SS_COMP_ISOC_MULT_MASK; 488 } 489 490 mps &= XHCI_CONTEXT_MPS_MASK; 491 cerr = XHCI_CONTEXT_ISOCH_CERR; 492 break; 493 default: 494 /* 495 * No explicit changes needed for CONTROL, INTR, and BULK 496 * endpoints. They've been handled already and don't have any 497 * differences. 498 */ 499 break; 500 } 501 502 eptype = xhci_endpoint_epdesc_to_type(&xep->xep_pipe->p_xep.uex_ep); 503 burst = xhci_endpoint_determine_burst(xd, xep); 504 ival = xhci_endpoint_interval(xd, &xep->xep_pipe->p_xep.uex_ep); 505 max_esit = xhci_endpoint_max_esit(xd, xep, mps, burst); 506 avgtrb = xhci_endpoint_avg_trb(xhcip, &xep->xep_pipe->p_xep.uex_ep, 507 mps); 508 509 /* 510 * The multi field may be reserved as zero if the LEC feature flag is 511 * set. See the description of mult in xHCI 1.1 / 6.2.3. 512 */ 513 if (xhcip->xhci_caps.xcap_flags2 & XCAP2_LEC) 514 mult = 0; 515 516 bzero(ectx, sizeof (xhci_endpoint_context_t)); 517 518 ectx->xec_info = LE_32(XHCI_EPCTX_SET_MULT(mult) | 519 XHCI_EPCTX_SET_IVAL(ival)); 520 if (xhcip->xhci_caps.xcap_flags2 & XCAP2_LEC) 521 ectx->xec_info |= LE_32(XHCI_EPCTX_SET_MAX_ESIT_HI(max_esit)); 522 523 ectx->xec_info2 = LE_32(XHCI_EPCTX_SET_CERR(cerr) | 524 XHCI_EPCTX_SET_EPTYPE(eptype) | XHCI_EPCTX_SET_MAXB(burst) | 525 XHCI_EPCTX_SET_MPS(mps)); 526 527 deq = xhci_dma_pa(&xep->xep_ring.xr_dma) + sizeof (xhci_trb_t) * 528 xep->xep_ring.xr_tail; 529 ectx->xec_dequeue = LE_64(deq | xep->xep_ring.xr_cycle); 530 531 ectx->xec_txinfo = LE_32(XHCI_EPCTX_MAX_ESIT_PAYLOAD(max_esit) | 532 XHCI_EPCTX_AVG_TRB_LEN(avgtrb)); 533 534 XHCI_DMA_SYNC(xd->xd_ictx, DDI_DMA_SYNC_FORDEV); 535 if (xhci_check_dma_handle(xhcip, &xd->xd_ictx) != DDI_FM_OK) { 536 xhci_error(xhcip, "failed to initialize device input " 537 "context on slot %d and port %d for endpoint %u: " 538 "encountered fatal FM error synchronizing input context " 539 "DMA memory", xd->xd_slot, xd->xd_port, xep->xep_num); 540 xhci_fm_runtime_reset(xhcip); 541 return (EIO); 542 } 543 544 return (0); 545 } 546 547 /* 548 * Initialize the endpoint and its input context for a given device. This is 549 * called from two different contexts: 550 * 551 * 1. Initializing a device 552 * 2. Opening a USB pipe 553 * 554 * In the second case, we need to worry about locking around the device. We 555 * don't need to worry about the locking in the first case because the USBA 556 * doesn't know about it yet. 557 */ 558 int 559 xhci_endpoint_init(xhci_t *xhcip, xhci_device_t *xd, 560 usba_pipe_handle_data_t *ph) 561 { 562 int ret; 563 uint_t epid; 564 xhci_endpoint_t *xep; 565 566 if (ph == NULL) { 567 epid = XHCI_DEFAULT_ENDPOINT; 568 } else { 569 ASSERT(MUTEX_HELD(&xhcip->xhci_lock)); 570 epid = xhci_endpoint_pipe_to_epid(ph); 571 } 572 VERIFY(xd->xd_endpoints[epid] == NULL); 573 574 xep = kmem_zalloc(sizeof (xhci_endpoint_t), KM_SLEEP); 575 list_create(&xep->xep_transfers, sizeof (xhci_transfer_t), 576 offsetof(xhci_transfer_t, xt_link)); 577 cv_init(&xep->xep_state_cv, NULL, CV_DRIVER, NULL); 578 xep->xep_xd = xd; 579 xep->xep_xhci = xhcip; 580 xep->xep_num = epid; 581 if (ph == NULL) { 582 xep->xep_pipe = NULL; 583 xep->xep_type = USB_EP_ATTR_CONTROL; 584 } else { 585 xep->xep_pipe = ph; 586 xep->xep_type = ph->p_ep.bmAttributes & USB_EP_ATTR_MASK; 587 } 588 589 if ((ret = xhci_ring_alloc(xhcip, &xep->xep_ring)) != 0) { 590 cv_destroy(&xep->xep_state_cv); 591 list_destroy(&xep->xep_transfers); 592 kmem_free(xep, sizeof (xhci_endpoint_t)); 593 return (ret); 594 } 595 596 if ((ret = xhci_ring_reset(xhcip, &xep->xep_ring)) != 0) { 597 xhci_ring_free(&xep->xep_ring); 598 cv_destroy(&xep->xep_state_cv); 599 list_destroy(&xep->xep_transfers); 600 kmem_free(xep, sizeof (xhci_endpoint_t)); 601 return (ret); 602 } 603 604 xd->xd_endpoints[epid] = xep; 605 if (ph == NULL) { 606 ret = xhci_endpoint_setup_default_context(xhcip, xd, xep); 607 } else { 608 ret = xhci_endpoint_setup_context(xhcip, xd, xep); 609 } 610 if (ret != 0) { 611 xhci_endpoint_fini(xd, xep->xep_num); 612 return (ret); 613 } 614 615 return (0); 616 } 617 618 /* 619 * Attempt to quiesce an endpoint. Depending on the state of the endpoint, we 620 * may need to simply stop it. Alternatively, we may need to explicitly reset 621 * the endpoint. Once done, this endpoint should be stopped and can be 622 * manipulated. 623 */ 624 int 625 xhci_endpoint_quiesce(xhci_t *xhcip, xhci_device_t *xd, xhci_endpoint_t *xep) 626 { 627 int ret = USB_SUCCESS; 628 xhci_endpoint_context_t *epctx = xd->xd_endout[xep->xep_num]; 629 630 ASSERT(MUTEX_HELD(&xhcip->xhci_lock)); 631 ASSERT(xep->xep_state & XHCI_ENDPOINT_QUIESCE); 632 633 /* 634 * First attempt to stop the endpoint, unless it's halted. We don't 635 * really care what state it is in. Note that because other activity 636 * could be going on, the state may change on us; however, if it's 637 * running, it will always transition to a stopped state and none of the 638 * other valid states will allow transitions without us taking an active 639 * action. 640 */ 641 if (!(xep->xep_state & XHCI_ENDPOINT_HALTED)) { 642 mutex_exit(&xhcip->xhci_lock); 643 ret = xhci_command_stop_endpoint(xhcip, xd, xep); 644 mutex_enter(&xhcip->xhci_lock); 645 646 if (ret == USB_INVALID_CONTEXT) { 647 XHCI_DMA_SYNC(xd->xd_octx, DDI_DMA_SYNC_FORKERNEL); 648 } 649 } 650 651 /* 652 * Now, if we had the HALTED flag set or we failed to stop it due to a 653 * context error and we're in the HALTED state now, reset the end point. 654 */ 655 if ((xep->xep_state & XHCI_ENDPOINT_HALTED) || 656 (ret == USB_INVALID_CONTEXT && 657 XHCI_EPCTX_STATE(LE_32(epctx->xec_info)) == XHCI_EP_HALTED)) { 658 mutex_exit(&xhcip->xhci_lock); 659 ret = xhci_command_reset_endpoint(xhcip, xd, xep); 660 mutex_enter(&xhcip->xhci_lock); 661 } 662 663 /* 664 * Ideally, one of the two commands should have worked; however, we 665 * could have had a context error due to being in the wrong state. 666 * Verify that we're either in the ERROR or STOPPED state and treat both 667 * as success. All callers are assumed to be doing this so they can 668 * change the dequeue pointer. 669 */ 670 if (ret != USB_SUCCESS && ret != USB_INVALID_CONTEXT) { 671 return (ret); 672 } 673 674 if (ret == USB_INVALID_CONTEXT) { 675 XHCI_DMA_SYNC(xd->xd_octx, DDI_DMA_SYNC_FORKERNEL); 676 677 switch (XHCI_EPCTX_STATE(LE_32(epctx->xec_info))) { 678 case XHCI_EP_STOPPED: 679 case XHCI_EP_ERROR: 680 /* 681 * This is where we wanted to go, so let's just take it. 682 */ 683 ret = USB_SUCCESS; 684 break; 685 case XHCI_EP_DISABLED: 686 case XHCI_EP_RUNNING: 687 case XHCI_EP_HALTED: 688 default: 689 /* 690 * If we're in any of these, something really weird has 691 * happened and it's not worth trying to recover at this 692 * point. 693 */ 694 xhci_error(xhcip, "!asked to stop endpoint %u on slot " 695 "%d and port %d: ended up in unexpected state %d", 696 xep->xep_num, xd->xd_slot, xd->xd_port, 697 XHCI_EPCTX_STATE(LE_32(epctx->xec_info))); 698 return (ret); 699 } 700 } 701 702 /* 703 * Now that we're successful, we can clear any possible halted state 704 * tracking that we might have had. 705 */ 706 if (ret == USB_SUCCESS) { 707 xep->xep_state &= ~XHCI_ENDPOINT_HALTED; 708 } 709 710 return (ret); 711 } 712 713 int 714 xhci_endpoint_ring(xhci_t *xhcip, xhci_device_t *xd, xhci_endpoint_t *xep) 715 { 716 /* 717 * The doorbell ID's are offset by one from the endpoint numbers that we 718 * keep. 719 */ 720 xhci_put32(xhcip, XHCI_R_DOOR, XHCI_DOORBELL(xd->xd_slot), 721 xep->xep_num + 1); 722 if (xhci_check_regs_acc(xhcip) != DDI_FM_OK) { 723 xhci_error(xhcip, "failed to ring doorbell for slot %d and " 724 "endpoint %u: encountered fatal FM register access error", 725 xd->xd_slot, xep->xep_num); 726 xhci_fm_runtime_reset(xhcip); 727 return (USB_HC_HARDWARE_ERROR); 728 } 729 730 DTRACE_PROBE3(xhci__doorbell__ring, xhci_t *, xhcip, uint32_t, 731 XHCI_DOORBELL(xd->xd_slot), uint32_t, xep->xep_num + 1); 732 733 return (USB_SUCCESS); 734 } 735 736 static void 737 xhci_endpoint_tick(void *arg) 738 { 739 int ret; 740 xhci_transfer_t *xt; 741 xhci_endpoint_t *xep = arg; 742 xhci_device_t *xd = xep->xep_xd; 743 xhci_t *xhcip = xep->xep_xhci; 744 745 mutex_enter(&xhcip->xhci_lock); 746 747 /* 748 * If we have the teardown flag set, then this is going away, don't try 749 * to do anything. Also, if somehow a periodic endpoint has something 750 * scheduled, just quit now and don't bother. 751 */ 752 if (xep->xep_state & (XHCI_ENDPOINT_TEARDOWN | 753 XHCI_ENDPOINT_PERIODIC)) { 754 xep->xep_timeout = 0; 755 mutex_exit(&xhcip->xhci_lock); 756 return; 757 } 758 759 /* 760 * If something else has already kicked off, something potentially 761 * dangerous, just don't bother waiting for it and reschedule. 762 */ 763 if (xep->xep_state & XHCI_ENDPOINT_DONT_SCHEDULE) { 764 xep->xep_timeout = timeout(xhci_endpoint_tick, xep, 765 drv_usectohz(XHCI_TICK_TIMEOUT_US)); 766 mutex_exit(&xhcip->xhci_lock); 767 return; 768 } 769 770 /* 771 * At this point, we have an endpoint that we need to consider. See if 772 * there are any transfers on it, if none, we're done. If so, check if 773 * we have exceeded the timeout. If we have, then we have some work to 774 * do. 775 */ 776 xt = list_head(&xep->xep_transfers); 777 if (xt == NULL) { 778 xep->xep_timeout = 0; 779 mutex_exit(&xhcip->xhci_lock); 780 return; 781 } 782 783 if (xt->xt_timeout > 0) { 784 xt->xt_timeout--; 785 xep->xep_timeout = timeout(xhci_endpoint_tick, xep, 786 drv_usectohz(XHCI_TICK_TIMEOUT_US)); 787 mutex_exit(&xhcip->xhci_lock); 788 return; 789 } 790 791 /* 792 * This item has timed out. We need to stop the ring and take action. 793 */ 794 xep->xep_state |= XHCI_ENDPOINT_TIMED_OUT | XHCI_ENDPOINT_QUIESCE; 795 ret = xhci_endpoint_quiesce(xhcip, xd, xep); 796 if (ret != USB_SUCCESS) { 797 /* 798 * If we fail to quiesce during the timeout, then remove the 799 * state flags and hopefully we'll be able to the next time 800 * around or if a reset or polling stop comes in, maybe it can 801 * deal with it. 802 */ 803 xep->xep_state &= ~(XHCI_ENDPOINT_QUIESCE | 804 XHCI_ENDPOINT_TIMED_OUT); 805 xep->xep_timeout = timeout(xhci_endpoint_tick, xep, 806 drv_usectohz(XHCI_TICK_TIMEOUT_US)); 807 mutex_exit(&xhcip->xhci_lock); 808 cv_broadcast(&xep->xep_state_cv); 809 xhci_error(xhcip, "failed to successfully quiesce timed out " 810 "endpoint %u of device on slot %d and port %d: device " 811 "remains timed out", xep->xep_num, xd->xd_slot, 812 xd->xd_port); 813 return; 814 } 815 816 xhci_ring_skip_transfer(&xep->xep_ring, xt); 817 (void) list_remove_head(&xep->xep_transfers); 818 mutex_exit(&xhcip->xhci_lock); 819 820 /* 821 * At this point, we try and set the ring's dequeue pointer. If this 822 * fails, we're left in an awkward state. We've already adjusted the 823 * ring and removed the transfer. All we can really do is go through and 824 * return the transfer and hope that they perhaps attempt to reset the 825 * ring and that will succeed at this point. Based on everything we've 826 * done to set things up, it'd be odd if this did fail. 827 */ 828 ret = xhci_command_set_tr_dequeue(xhcip, xd, xep); 829 mutex_enter(&xhcip->xhci_lock); 830 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE; 831 if (ret == USB_SUCCESS) { 832 xep->xep_state &= ~XHCI_ENDPOINT_TIMED_OUT; 833 } else { 834 xhci_error(xhcip, "failed to successfully set transfer ring " 835 "dequeue pointer of timed out endpoint %u of " 836 "device on slot %d and port %d: device remains timed out, " 837 "please use cfgadm to recover", xep->xep_num, xd->xd_slot, 838 xd->xd_port); 839 } 840 xep->xep_timeout = timeout(xhci_endpoint_tick, xep, 841 drv_usectohz(XHCI_TICK_TIMEOUT_US)); 842 mutex_exit(&xhcip->xhci_lock); 843 cv_broadcast(&xep->xep_state_cv); 844 845 /* 846 * Because we never time out periodic related activity, we will always 847 * have the request on the transfer. 848 */ 849 ASSERT(xt->xt_usba_req != NULL); 850 usba_hcdi_cb(xep->xep_pipe, xt->xt_usba_req, USB_CR_TIMEOUT); 851 xhci_transfer_free(xhcip, xt); 852 } 853 854 /* 855 * We've been asked to schedule a series of frames onto the specified endpoint. 856 * We need to make sure that there is enough room, at which point we can queue 857 * it and then ring the door bell. Note that we queue in reverse order to make 858 * sure that if the ring moves on, it won't see the correct cycle bit. 859 */ 860 int 861 xhci_endpoint_schedule(xhci_t *xhcip, xhci_device_t *xd, xhci_endpoint_t *xep, 862 xhci_transfer_t *xt, boolean_t ring) 863 { 864 int i; 865 xhci_ring_t *rp = &xep->xep_ring; 866 867 ASSERT(MUTEX_HELD(&xhcip->xhci_lock)); 868 ASSERT(xt->xt_ntrbs > 0); 869 ASSERT(xt->xt_trbs != NULL); 870 871 if ((xep->xep_state & XHCI_ENDPOINT_DONT_SCHEDULE) != 0) 872 return (USB_FAILURE); 873 874 if (xhci_ring_trb_space(rp, xt->xt_ntrbs) == B_FALSE) 875 return (USB_NO_RESOURCES); 876 877 for (i = xt->xt_ntrbs - 1; i > 0; i--) { 878 xhci_ring_trb_fill(rp, i, &xt->xt_trbs[i], B_TRUE); 879 } 880 xhci_ring_trb_fill(rp, 0U, &xt->xt_trbs[0], B_FALSE); 881 882 XHCI_DMA_SYNC(rp->xr_dma, DDI_DMA_SYNC_FORDEV); 883 xhci_ring_trb_produce(rp, xt->xt_ntrbs); 884 list_insert_tail(&xep->xep_transfers, xt); 885 886 XHCI_DMA_SYNC(rp->xr_dma, DDI_DMA_SYNC_FORDEV); 887 if (xhci_check_dma_handle(xhcip, &rp->xr_dma) != DDI_FM_OK) { 888 xhci_error(xhcip, "failed to write out TRB for device on slot " 889 "%d, port %d, and endpoint %u: encountered fatal FM error " 890 "synchronizing ring DMA memory", xd->xd_slot, xd->xd_port, 891 xep->xep_num); 892 xhci_fm_runtime_reset(xhcip); 893 return (USB_HC_HARDWARE_ERROR); 894 } 895 896 if (xep->xep_timeout == 0 && 897 !(xep->xep_state & XHCI_ENDPOINT_PERIODIC)) { 898 xep->xep_timeout = timeout(xhci_endpoint_tick, xep, 899 drv_usectohz(XHCI_TICK_TIMEOUT_US)); 900 } 901 902 xt->xt_sched_time = gethrtime(); 903 904 if (ring == B_FALSE) 905 return (USB_SUCCESS); 906 907 return (xhci_endpoint_ring(xhcip, xd, xep)); 908 } 909 910 static xhci_transfer_t * 911 xhci_endpoint_determine_transfer(xhci_t *xhcip, xhci_endpoint_t *xep, 912 xhci_trb_t *trb, int *offp) 913 { 914 xhci_transfer_t *xt; 915 916 ASSERT(xhcip != NULL); 917 ASSERT(offp != NULL); 918 ASSERT(xep != NULL); 919 ASSERT(trb != NULL); 920 ASSERT(MUTEX_HELD(&xhcip->xhci_lock)); 921 922 if ((xt = list_head(&xep->xep_transfers)) == NULL) 923 return (NULL); 924 925 *offp = xhci_ring_trb_valid_range(&xep->xep_ring, LE_64(trb->trb_addr), 926 xt->xt_ntrbs); 927 if (*offp == -1) 928 return (NULL); 929 return (xt); 930 } 931 932 static void 933 xhci_endpoint_reschedule_periodic(xhci_t *xhcip, xhci_device_t *xd, 934 xhci_endpoint_t *xep, xhci_transfer_t *xt) 935 { 936 int ret; 937 xhci_pipe_t *xp = (xhci_pipe_t *)xep->xep_pipe->p_hcd_private; 938 xhci_periodic_pipe_t *xpp = &xp->xp_periodic; 939 940 ASSERT3U(xpp->xpp_tsize, >, 0); 941 942 xt->xt_short = 0; 943 xt->xt_cr = USB_CR_OK; 944 945 mutex_enter(&xhcip->xhci_lock); 946 947 /* 948 * If we don't have an active poll, then we shouldn't bother trying to 949 * reschedule it. This means that we're trying to stop or we ran out of 950 * memory. 951 */ 952 if (xpp->xpp_poll_state != XHCI_PERIODIC_POLL_ACTIVE) { 953 mutex_exit(&xhcip->xhci_lock); 954 return; 955 } 956 957 if (xep->xep_type == USB_EP_ATTR_ISOCH) { 958 int i; 959 for (i = 0; i < xt->xt_ntrbs; i++) { 960 xt->xt_isoc[i].isoc_pkt_actual_length = 961 xt->xt_isoc[i].isoc_pkt_length; 962 xt->xt_isoc[i].isoc_pkt_status = USB_CR_OK; 963 } 964 } 965 966 /* 967 * In general, there should always be space on the ring for this. The 968 * only reason that rescheduling an existing transfer for a periodic 969 * endpoint wouldn't work is because we have a hardware error, at which 970 * point we're going to be going down hard anyways. We log and bump a 971 * stat here to make this case discoverable in case our assumptions our 972 * wrong. 973 */ 974 ret = xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE); 975 if (ret != 0) { 976 xhci_log(xhcip, "!failed to reschedule periodic endpoint %u " 977 "(type %u) on slot %d: %d\n", xep->xep_num, xep->xep_type, 978 xd->xd_slot, ret); 979 } 980 mutex_exit(&xhcip->xhci_lock); 981 } 982 983 /* 984 * We're dealing with a message on a control endpoint. This may be a default 985 * endpoint or otherwise. These usually come in groups of 3+ TRBs where you have 986 * a setup stage, data stage (which may have one or more other TRBs) and then a 987 * final status stage. 988 * 989 * We generally set ourselves up such that we get interrupted and notified only 990 * on the status stage and for short transfers in the data stage. If we 991 * encounter a short transfer in the data stage, then we need to go through and 992 * check whether or not the short transfer is allowed. If it is, then there's 993 * nothing to do. We'll update everything and call back the framework once we 994 * get the status stage. 995 */ 996 static boolean_t 997 xhci_endpoint_control_callback(xhci_t *xhcip, xhci_device_t *xd, 998 xhci_endpoint_t *xep, xhci_transfer_t *xt, int off, xhci_trb_t *trb) 999 { 1000 int code; 1001 usb_ctrl_req_t *ucrp; 1002 xhci_transfer_t *rem; 1003 1004 ASSERT(MUTEX_HELD(&xhcip->xhci_lock)); 1005 1006 code = XHCI_TRB_GET_CODE(LE_32(trb->trb_status)); 1007 ucrp = (usb_ctrl_req_t *)xt->xt_usba_req; 1008 1009 /* 1010 * Now that we know what this TRB is for, was it for a data/normal stage 1011 * or is it the status stage. We cheat by looking at the last entry. If 1012 * it's a data stage, then we must have gotten a short write. In that 1013 * case, we should go through and check to make sure it's allowed. If 1014 * not, we need to fail the transfer, try to stop the ring, and make 1015 * callbacks. We'll clean up the xhci transfer at this time. 1016 */ 1017 if (off != xt->xt_ntrbs - 1) { 1018 uint_t remain; 1019 usb_ctrl_req_t *ucrp = (usb_ctrl_req_t *)xt->xt_usba_req; 1020 1021 /* 1022 * This is a data stage TRB. The only reason we should have 1023 * gotten something for this is beacuse it was short. Make sure 1024 * it's okay before we continue. 1025 */ 1026 VERIFY3S(code, ==, XHCI_CODE_SHORT_XFER); 1027 if (!(ucrp->ctrl_attributes & USB_ATTRS_SHORT_XFER_OK)) { 1028 xt->xt_cr = USB_CR_DATA_UNDERRUN; 1029 mutex_exit(&xhcip->xhci_lock); 1030 return (B_TRUE); 1031 } 1032 1033 /* 1034 * The value in the resulting trb is how much data remained to 1035 * be transferred. Normalize that against the original buffer 1036 * size. 1037 */ 1038 remain = XHCI_TRB_REMAIN(LE_32(trb->trb_status)); 1039 xt->xt_short = xt->xt_buffer.xdb_len - remain; 1040 mutex_exit(&xhcip->xhci_lock); 1041 return (B_TRUE); 1042 } 1043 1044 /* 1045 * Okay, this is a status stage trb that's in good health. We should 1046 * finally go ahead, sync data and try and finally do the callback. If 1047 * we have short data, then xt->xt_short will be non-zero. 1048 */ 1049 if (xt->xt_data_tohost == B_TRUE) { 1050 size_t len; 1051 if (xt->xt_short != 0) { 1052 len = xt->xt_short; 1053 } else { 1054 len = xt->xt_buffer.xdb_len; 1055 } 1056 1057 if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORCPU) != 1058 DDI_FM_OK) { 1059 xhci_error(xhcip, "failed to process control transfer " 1060 "callback for endpoint %u of device on slot %d and " 1061 "port %d: encountered fatal FM error synchronizing " 1062 "DMA memory, resetting device", xep->xep_num, 1063 xd->xd_slot, xd->xd_port); 1064 xhci_fm_runtime_reset(xhcip); 1065 mutex_exit(&xhcip->xhci_lock); 1066 return (B_FALSE); 1067 } 1068 1069 xhci_transfer_copy(xt, ucrp->ctrl_data->b_rptr, len, B_TRUE); 1070 ucrp->ctrl_data->b_wptr += len; 1071 } 1072 1073 /* 1074 * Now we're done. We can go ahead and bump the ring. Free the transfer 1075 * outside of the lock and call back into the framework. 1076 */ 1077 VERIFY(xhci_ring_trb_consumed(&xep->xep_ring, LE_64(trb->trb_addr))); 1078 rem = list_remove_head(&xep->xep_transfers); 1079 VERIFY3P(rem, ==, xt); 1080 mutex_exit(&xhcip->xhci_lock); 1081 1082 usba_hcdi_cb(xep->xep_pipe, (usb_opaque_t)ucrp, xt->xt_cr); 1083 xhci_transfer_free(xhcip, xt); 1084 1085 return (B_TRUE); 1086 } 1087 1088 /* 1089 * Cons up a new usb request for the periodic data transfer if we can. If there 1090 * isn't one available, change the return code to NO_RESOURCES and stop polling 1091 * on this endpoint, thus using and consuming the original request. 1092 */ 1093 static usb_opaque_t 1094 xhci_endpoint_dup_periodic(xhci_endpoint_t *xep, xhci_transfer_t *xt, 1095 usb_cr_t *cr) 1096 { 1097 usb_opaque_t urp; 1098 1099 xhci_pipe_t *xp = (xhci_pipe_t *)xep->xep_pipe->p_hcd_private; 1100 xhci_periodic_pipe_t *xpp = &xp->xp_periodic; 1101 1102 /* 1103 * In general, transfers shouldn't have a usb request. However, oneshot 1104 * Interrupt IN ones will, so we use this as a way to shortcut out of 1105 * here. 1106 */ 1107 if (xt->xt_usba_req != NULL) 1108 return (xt->xt_usba_req); 1109 1110 if (xep->xep_type == USB_EP_ATTR_INTR) { 1111 urp = (usb_opaque_t)usba_hcdi_dup_intr_req(xep->xep_pipe->p_dip, 1112 (usb_intr_req_t *)xpp->xpp_usb_req, xpp->xpp_tsize, 0); 1113 } else { 1114 urp = (usb_opaque_t)usba_hcdi_dup_isoc_req(xep->xep_pipe->p_dip, 1115 (usb_isoc_req_t *)xpp->xpp_usb_req, 0); 1116 } 1117 if (urp == NULL) { 1118 xpp->xpp_poll_state = XHCI_PERIODIC_POLL_NOMEM; 1119 urp = xpp->xpp_usb_req; 1120 xpp->xpp_usb_req = NULL; 1121 *cr = USB_CR_NO_RESOURCES; 1122 } else { 1123 mutex_enter(&xep->xep_pipe->p_mutex); 1124 xep->xep_pipe->p_req_count++; 1125 mutex_exit(&xep->xep_pipe->p_mutex); 1126 } 1127 1128 return (urp); 1129 } 1130 1131 static xhci_device_t * 1132 xhci_device_lookup_by_slot(xhci_t *xhcip, int slot) 1133 { 1134 xhci_device_t *xd; 1135 1136 ASSERT(MUTEX_HELD(&xhcip->xhci_lock)); 1137 1138 for (xd = list_head(&xhcip->xhci_usba.xa_devices); xd != NULL; 1139 xd = list_next(&xhcip->xhci_usba.xa_devices, xd)) { 1140 if (xd->xd_slot == slot) 1141 return (xd); 1142 } 1143 1144 return (NULL); 1145 } 1146 1147 /* 1148 * Handle things which consist solely of normal tranfers, in other words, bulk 1149 * and interrupt transfers. 1150 */ 1151 static boolean_t 1152 xhci_endpoint_norm_callback(xhci_t *xhcip, xhci_device_t *xd, 1153 xhci_endpoint_t *xep, xhci_transfer_t *xt, int off, xhci_trb_t *trb) 1154 { 1155 int code; 1156 usb_cr_t cr; 1157 xhci_transfer_t *rem; 1158 int attrs; 1159 mblk_t *mp; 1160 boolean_t periodic = B_FALSE; 1161 usb_opaque_t urp; 1162 1163 ASSERT(MUTEX_HELD(&xhcip->xhci_lock)); 1164 ASSERT(xep->xep_type == USB_EP_ATTR_BULK || 1165 xep->xep_type == USB_EP_ATTR_INTR); 1166 1167 code = XHCI_TRB_GET_CODE(LE_32(trb->trb_status)); 1168 1169 if (code == XHCI_CODE_SHORT_XFER) { 1170 int residue; 1171 residue = XHCI_TRB_REMAIN(LE_32(trb->trb_status)); 1172 xt->xt_short = xt->xt_buffer.xdb_len - residue; 1173 } 1174 1175 /* 1176 * If we have an interrupt from something that's not the last entry, 1177 * that must mean we had a short transfer, so there's nothing more for 1178 * us to do at the moment. We won't call back until everything's 1179 * finished for the general transfer. 1180 */ 1181 if (off < xt->xt_ntrbs - 1) { 1182 mutex_exit(&xhcip->xhci_lock); 1183 return (B_TRUE); 1184 } 1185 1186 urp = xt->xt_usba_req; 1187 if (xep->xep_type == USB_EP_ATTR_BULK) { 1188 usb_bulk_req_t *ubrp = (usb_bulk_req_t *)xt->xt_usba_req; 1189 attrs = ubrp->bulk_attributes; 1190 mp = ubrp->bulk_data; 1191 } else { 1192 usb_intr_req_t *uirp = (usb_intr_req_t *)xt->xt_usba_req; 1193 1194 if (uirp == NULL) { 1195 periodic = B_TRUE; 1196 urp = xhci_endpoint_dup_periodic(xep, xt, &cr); 1197 uirp = (usb_intr_req_t *)urp; 1198 1199 /* 1200 * If we weren't able to duplicate the interrupt, then 1201 * we can't put any data in it. 1202 */ 1203 if (cr == USB_CR_NO_RESOURCES) 1204 goto out; 1205 } 1206 1207 attrs = uirp->intr_attributes; 1208 mp = uirp->intr_data; 1209 } 1210 1211 if (xt->xt_data_tohost == B_TRUE) { 1212 size_t len; 1213 if (xt->xt_short != 0) { 1214 if (!(attrs & USB_ATTRS_SHORT_XFER_OK)) { 1215 cr = USB_CR_DATA_UNDERRUN; 1216 goto out; 1217 } 1218 len = xt->xt_short; 1219 } else { 1220 len = xt->xt_buffer.xdb_len; 1221 } 1222 1223 if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORCPU) != 1224 DDI_FM_OK) { 1225 xhci_error(xhcip, "failed to process normal transfer " 1226 "callback for endpoint %u of device on slot %d and " 1227 "port %d: encountered fatal FM error synchronizing " 1228 "DMA memory, resetting device", xep->xep_num, 1229 xd->xd_slot, xd->xd_port); 1230 xhci_fm_runtime_reset(xhcip); 1231 mutex_exit(&xhcip->xhci_lock); 1232 return (B_FALSE); 1233 } 1234 1235 xhci_transfer_copy(xt, mp->b_rptr, len, B_TRUE); 1236 mp->b_wptr += len; 1237 } 1238 cr = USB_CR_OK; 1239 1240 out: 1241 VERIFY(xhci_ring_trb_consumed(&xep->xep_ring, LE_64(trb->trb_addr))); 1242 rem = list_remove_head(&xep->xep_transfers); 1243 VERIFY3P(rem, ==, xt); 1244 mutex_exit(&xhcip->xhci_lock); 1245 1246 usba_hcdi_cb(xep->xep_pipe, urp, cr); 1247 if (periodic == B_TRUE) { 1248 xhci_endpoint_reschedule_periodic(xhcip, xd, xep, xt); 1249 } else { 1250 xhci_transfer_free(xhcip, xt); 1251 } 1252 1253 return (B_TRUE); 1254 } 1255 1256 static boolean_t 1257 xhci_endpoint_isoch_callback(xhci_t *xhcip, xhci_device_t *xd, 1258 xhci_endpoint_t *xep, xhci_transfer_t *xt, int off, xhci_trb_t *trb) 1259 { 1260 int code; 1261 usb_cr_t cr; 1262 xhci_transfer_t *rem; 1263 usb_isoc_pkt_descr_t *desc; 1264 usb_isoc_req_t *usrp; 1265 1266 ASSERT(MUTEX_HELD(&xhcip->xhci_lock)); 1267 ASSERT3S(xep->xep_type, ==, USB_EP_ATTR_ISOCH); 1268 1269 code = XHCI_TRB_GET_CODE(LE_32(trb->trb_status)); 1270 1271 /* 1272 * The descriptors that we copy the data from are set up to assume that 1273 * everything was OK and we transferred all the requested data. 1274 */ 1275 desc = &xt->xt_isoc[off]; 1276 if (code == XHCI_CODE_SHORT_XFER) { 1277 int residue = XHCI_TRB_REMAIN(LE_32(trb->trb_status)); 1278 desc->isoc_pkt_actual_length -= residue; 1279 } 1280 1281 /* 1282 * We don't perform the callback until the very last TRB is returned 1283 * here. If we have a TRB report on something else, that means that we 1284 * had a short transfer. 1285 */ 1286 if (off < xt->xt_ntrbs - 1) { 1287 mutex_exit(&xhcip->xhci_lock); 1288 return (B_TRUE); 1289 } 1290 1291 VERIFY(xhci_ring_trb_consumed(&xep->xep_ring, LE_64(trb->trb_addr))); 1292 rem = list_remove_head(&xep->xep_transfers); 1293 VERIFY3P(rem, ==, xt); 1294 mutex_exit(&xhcip->xhci_lock); 1295 1296 cr = USB_CR_OK; 1297 1298 if (xt->xt_data_tohost == B_TRUE) { 1299 usb_opaque_t urp; 1300 urp = xhci_endpoint_dup_periodic(xep, xt, &cr); 1301 usrp = (usb_isoc_req_t *)urp; 1302 1303 if (cr == USB_CR_OK) { 1304 mblk_t *mp; 1305 size_t len; 1306 if (xhci_transfer_sync(xhcip, xt, 1307 DDI_DMA_SYNC_FORCPU) != DDI_FM_OK) { 1308 xhci_error(xhcip, "failed to process " 1309 "isochronous transfer callback for " 1310 "endpoint %u of device on slot %d and port " 1311 "%d: encountered fatal FM error " 1312 "synchronizing DMA memory, resetting " 1313 "device", 1314 xep->xep_num, xd->xd_slot, xd->xd_port); 1315 xhci_fm_runtime_reset(xhcip); 1316 mutex_exit(&xhcip->xhci_lock); 1317 return (B_FALSE); 1318 } 1319 1320 mp = usrp->isoc_data; 1321 len = xt->xt_buffer.xdb_len; 1322 xhci_transfer_copy(xt, mp->b_rptr, len, B_TRUE); 1323 mp->b_wptr += len; 1324 } 1325 } else { 1326 usrp = (usb_isoc_req_t *)xt->xt_usba_req; 1327 } 1328 1329 if (cr == USB_CR_OK) { 1330 bcopy(xt->xt_isoc, usrp->isoc_pkt_descr, 1331 sizeof (usb_isoc_pkt_descr_t) * usrp->isoc_pkts_count); 1332 } 1333 1334 usba_hcdi_cb(xep->xep_pipe, (usb_opaque_t)usrp, cr); 1335 if (xt->xt_data_tohost == B_TRUE) { 1336 xhci_endpoint_reschedule_periodic(xhcip, xd, xep, xt); 1337 } else { 1338 xhci_transfer_free(xhcip, xt); 1339 } 1340 1341 return (B_TRUE); 1342 } 1343 1344 boolean_t 1345 xhci_endpoint_transfer_callback(xhci_t *xhcip, xhci_trb_t *trb) 1346 { 1347 boolean_t ret; 1348 int slot, endpoint, code, off; 1349 xhci_device_t *xd; 1350 xhci_endpoint_t *xep; 1351 xhci_transfer_t *xt; 1352 boolean_t transfer_done; 1353 1354 endpoint = XHCI_TRB_GET_EP(LE_32(trb->trb_flags)); 1355 slot = XHCI_TRB_GET_SLOT(LE_32(trb->trb_flags)); 1356 code = XHCI_TRB_GET_CODE(LE_32(trb->trb_status)); 1357 1358 mutex_enter(&xhcip->xhci_lock); 1359 xd = xhci_device_lookup_by_slot(xhcip, slot); 1360 if (xd == NULL) { 1361 xhci_error(xhcip, "received transfer trb with code %d for " 1362 "unknown slot %d and endpoint %d: resetting device", code, 1363 slot, endpoint); 1364 mutex_exit(&xhcip->xhci_lock); 1365 xhci_fm_runtime_reset(xhcip); 1366 return (B_FALSE); 1367 } 1368 1369 /* 1370 * Endpoint IDs are indexed based on their Device Context Index, which 1371 * means that we need to subtract one to get the actual ID that we use. 1372 */ 1373 xep = xd->xd_endpoints[endpoint - 1]; 1374 if (xep == NULL) { 1375 xhci_error(xhcip, "received transfer trb with code %d, slot " 1376 "%d, and unknown endpoint %d: resetting device", code, 1377 slot, endpoint); 1378 mutex_exit(&xhcip->xhci_lock); 1379 xhci_fm_runtime_reset(xhcip); 1380 return (B_FALSE); 1381 } 1382 1383 /* 1384 * This TRB should be part of a transfer. If it's not, then we ignore 1385 * it. We also check whether or not it's for the first transfer. Because 1386 * the rings are serviced in order, it should be. 1387 */ 1388 if ((xt = xhci_endpoint_determine_transfer(xhcip, xep, trb, &off)) == 1389 NULL) { 1390 mutex_exit(&xhcip->xhci_lock); 1391 return (B_TRUE); 1392 } 1393 1394 transfer_done = B_FALSE; 1395 1396 switch (code) { 1397 case XHCI_CODE_SUCCESS: 1398 case XHCI_CODE_SHORT_XFER: 1399 /* Handled by endpoint logic */ 1400 break; 1401 case XHCI_CODE_XFER_STOPPED: 1402 case XHCI_CODE_XFER_STOPINV: 1403 case XHCI_CODE_XFER_STOPSHORT: 1404 /* 1405 * This causes us to transition the endpoint to a stopped state. 1406 * Each of these indicate a different possible state that we 1407 * have to deal with. Effectively we're going to drop it and 1408 * leave it up to the consumers to figure out what to do. For 1409 * the moment, that's generally okay because stops are only used 1410 * in cases where we're cleaning up outstanding reqs, etc. 1411 */ 1412 mutex_exit(&xhcip->xhci_lock); 1413 return (B_TRUE); 1414 case XHCI_CODE_STALL: 1415 /* 1416 * This causes us to transition to the halted state; 1417 * however, downstream clients are able to handle this just 1418 * fine. 1419 */ 1420 xep->xep_state |= XHCI_ENDPOINT_HALTED; 1421 xt->xt_cr = USB_CR_STALL; 1422 transfer_done = B_TRUE; 1423 break; 1424 case XHCI_CODE_BABBLE: 1425 transfer_done = B_TRUE; 1426 xt->xt_cr = USB_CR_DATA_OVERRUN; 1427 xep->xep_state |= XHCI_ENDPOINT_HALTED; 1428 break; 1429 case XHCI_CODE_TXERR: 1430 case XHCI_CODE_SPLITERR: 1431 transfer_done = B_TRUE; 1432 xt->xt_cr = USB_CR_DEV_NOT_RESP; 1433 xep->xep_state |= XHCI_ENDPOINT_HALTED; 1434 break; 1435 default: 1436 /* 1437 * Treat these as general unspecified errors that don't cause a 1438 * stop of the ring. Even if it does, a subsequent timeout 1439 * should occur which causes us to end up dropping a pipe reset 1440 * or at least issuing a reset of the device as part of 1441 * quiescing. 1442 */ 1443 transfer_done = B_TRUE; 1444 break; 1445 } 1446 1447 if (transfer_done == B_TRUE) { 1448 xhci_transfer_t *alt; 1449 1450 alt = list_remove_head(&xep->xep_transfers); 1451 VERIFY3P(alt, ==, xt); 1452 mutex_exit(&xhcip->xhci_lock); 1453 if (xt->xt_usba_req == NULL) { 1454 usb_opaque_t urp; 1455 1456 urp = xhci_endpoint_dup_periodic(xep, xt, &xt->xt_cr); 1457 usba_hcdi_cb(xep->xep_pipe, urp, xt->xt_cr); 1458 } else { 1459 usba_hcdi_cb(xep->xep_pipe, 1460 (usb_opaque_t)xt->xt_usba_req, xt->xt_cr); 1461 xhci_transfer_free(xhcip, xt); 1462 } 1463 return (B_TRUE); 1464 } 1465 1466 /* 1467 * Process the transfer callback based on the type of endpoint. Each of 1468 * these callback functions will end up calling back into USBA via 1469 * usba_hcdi_cb() to return transfer information (whether successful or 1470 * not). Because we can't hold any locks across a call to that function, 1471 * all of these callbacks will drop the xhci_t`xhci_lock by the time 1472 * they return. This is why there's no mutex_exit() call before we 1473 * return. 1474 */ 1475 switch (xep->xep_type) { 1476 case USB_EP_ATTR_CONTROL: 1477 ret = xhci_endpoint_control_callback(xhcip, xd, xep, xt, off, 1478 trb); 1479 break; 1480 case USB_EP_ATTR_BULK: 1481 ret = xhci_endpoint_norm_callback(xhcip, xd, xep, xt, off, trb); 1482 break; 1483 case USB_EP_ATTR_INTR: 1484 ret = xhci_endpoint_norm_callback(xhcip, xd, xep, xt, off, 1485 trb); 1486 break; 1487 case USB_EP_ATTR_ISOCH: 1488 ret = xhci_endpoint_isoch_callback(xhcip, xd, xep, xt, off, 1489 trb); 1490 break; 1491 default: 1492 panic("bad endpoint type: %u", xep->xep_type); 1493 } 1494 1495 return (ret); 1496 } 1497