1*993e3fafSRobert Mustacchi /* 2*993e3fafSRobert Mustacchi * This file and its contents are supplied under the terms of the 3*993e3fafSRobert Mustacchi * Common Development and Distribution License ("CDDL"), version 1.0. 4*993e3fafSRobert Mustacchi * You may only use this file in accordance with the terms of version 5*993e3fafSRobert Mustacchi * 1.0 of the CDDL. 6*993e3fafSRobert Mustacchi * 7*993e3fafSRobert Mustacchi * A full copy of the text of the CDDL should have accompanied this 8*993e3fafSRobert Mustacchi * source. A copy of the CDDL is also available via the Internet at 9*993e3fafSRobert Mustacchi * http://www.illumos.org/license/CDDL. 10*993e3fafSRobert Mustacchi */ 11*993e3fafSRobert Mustacchi 12*993e3fafSRobert Mustacchi /* 13*993e3fafSRobert Mustacchi * Copyright 2016 Joyent, Inc. 14*993e3fafSRobert Mustacchi */ 15*993e3fafSRobert Mustacchi 16*993e3fafSRobert Mustacchi /* 17*993e3fafSRobert Mustacchi * xHCI DMA Management Routines 18*993e3fafSRobert Mustacchi * 19*993e3fafSRobert Mustacchi * Please see the big theory statement in xhci.c for more information. 20*993e3fafSRobert Mustacchi */ 21*993e3fafSRobert Mustacchi 22*993e3fafSRobert Mustacchi #include <sys/usb/hcd/xhci/xhci.h> 23*993e3fafSRobert Mustacchi 24*993e3fafSRobert Mustacchi int 25*993e3fafSRobert Mustacchi xhci_check_dma_handle(xhci_t *xhcip, xhci_dma_buffer_t *xdb) 26*993e3fafSRobert Mustacchi { 27*993e3fafSRobert Mustacchi ddi_fm_error_t de; 28*993e3fafSRobert Mustacchi 29*993e3fafSRobert Mustacchi if (!DDI_FM_DMA_ERR_CAP(xhcip->xhci_fm_caps)) 30*993e3fafSRobert Mustacchi return (0); 31*993e3fafSRobert Mustacchi 32*993e3fafSRobert Mustacchi ddi_fm_dma_err_get(xdb->xdb_dma_handle, &de, DDI_FME_VERSION); 33*993e3fafSRobert Mustacchi return (de.fme_status); 34*993e3fafSRobert Mustacchi } 35*993e3fafSRobert Mustacchi 36*993e3fafSRobert Mustacchi void 37*993e3fafSRobert Mustacchi xhci_dma_acc_attr(xhci_t *xhcip, ddi_device_acc_attr_t *accp) 38*993e3fafSRobert Mustacchi { 39*993e3fafSRobert Mustacchi accp->devacc_attr_version = DDI_DEVICE_ATTR_V0; 40*993e3fafSRobert Mustacchi accp->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 41*993e3fafSRobert Mustacchi accp->devacc_attr_dataorder = DDI_STRICTORDER_ACC; 42*993e3fafSRobert Mustacchi 43*993e3fafSRobert Mustacchi if (DDI_FM_DMA_ERR_CAP(xhcip->xhci_fm_caps)) { 44*993e3fafSRobert Mustacchi accp->devacc_attr_access = DDI_FLAGERR_ACC; 45*993e3fafSRobert Mustacchi } else { 46*993e3fafSRobert Mustacchi accp->devacc_attr_access = DDI_DEFAULT_ACC; 47*993e3fafSRobert Mustacchi } 48*993e3fafSRobert Mustacchi } 49*993e3fafSRobert Mustacchi 50*993e3fafSRobert Mustacchi /* 51*993e3fafSRobert Mustacchi * These are DMA attributes that we assign when making a transfer. The SGL is 52*993e3fafSRobert Mustacchi * variable and based on the caller, which varies based on the type of transfer 53*993e3fafSRobert Mustacchi * we're doing. 54*993e3fafSRobert Mustacchi */ 55*993e3fafSRobert Mustacchi void 56*993e3fafSRobert Mustacchi xhci_dma_transfer_attr(xhci_t *xhcip, ddi_dma_attr_t *attrp, uint_t sgl) 57*993e3fafSRobert Mustacchi { 58*993e3fafSRobert Mustacchi VERIFY3U(sgl, >, 0); 59*993e3fafSRobert Mustacchi VERIFY3U(sgl, <=, XHCI_TRANSFER_DMA_SGL); 60*993e3fafSRobert Mustacchi attrp->dma_attr_version = DMA_ATTR_V0; 61*993e3fafSRobert Mustacchi 62*993e3fafSRobert Mustacchi /* 63*993e3fafSRobert Mustacchi * The range of data that we can use is based on what hardware supports. 64*993e3fafSRobert Mustacchi */ 65*993e3fafSRobert Mustacchi attrp->dma_attr_addr_lo = 0x0; 66*993e3fafSRobert Mustacchi if (xhcip->xhci_caps.xcap_flags & XCAP_AC64) { 67*993e3fafSRobert Mustacchi attrp->dma_attr_addr_hi = UINT64_MAX; 68*993e3fafSRobert Mustacchi } else { 69*993e3fafSRobert Mustacchi attrp->dma_attr_addr_hi = UINT32_MAX; 70*993e3fafSRobert Mustacchi } 71*993e3fafSRobert Mustacchi 72*993e3fafSRobert Mustacchi /* 73*993e3fafSRobert Mustacchi * The count max indicates the total amount that will fit into one 74*993e3fafSRobert Mustacchi * cookie, which is one TRB in our world. In other words 64k. 75*993e3fafSRobert Mustacchi */ 76*993e3fafSRobert Mustacchi attrp->dma_attr_count_max = XHCI_TRB_MAX_TRANSFER; 77*993e3fafSRobert Mustacchi 78*993e3fafSRobert Mustacchi /* 79*993e3fafSRobert Mustacchi * The alignment and segment are related. The alignment describes the 80*993e3fafSRobert Mustacchi * alignment of the PA. The segment describes a boundary that the DMA 81*993e3fafSRobert Mustacchi * allocation cannot cross. In other words, for a given chunk of memory 82*993e3fafSRobert Mustacchi * it cannot cross a 64-byte boundary. However, the physical address 83*993e3fafSRobert Mustacchi * only needs to be aligned to align bytes. 84*993e3fafSRobert Mustacchi */ 85*993e3fafSRobert Mustacchi attrp->dma_attr_align = XHCI_DMA_ALIGN; 86*993e3fafSRobert Mustacchi attrp->dma_attr_seg = XHCI_TRB_MAX_TRANSFER - 1; 87*993e3fafSRobert Mustacchi 88*993e3fafSRobert Mustacchi 89*993e3fafSRobert Mustacchi attrp->dma_attr_burstsizes = 0xfff; 90*993e3fafSRobert Mustacchi 91*993e3fafSRobert Mustacchi /* 92*993e3fafSRobert Mustacchi * This is the maximum we can send. Technically this is limited by the 93*993e3fafSRobert Mustacchi * descriptors and not by hardware, hence why we use a large value for 94*993e3fafSRobert Mustacchi * the max that'll be less than any memory allocation we ever throw at 95*993e3fafSRobert Mustacchi * it. 96*993e3fafSRobert Mustacchi */ 97*993e3fafSRobert Mustacchi attrp->dma_attr_minxfer = 0x1; 98*993e3fafSRobert Mustacchi attrp->dma_attr_maxxfer = UINT32_MAX; 99*993e3fafSRobert Mustacchi 100*993e3fafSRobert Mustacchi /* 101*993e3fafSRobert Mustacchi * This is determined by the caller. 102*993e3fafSRobert Mustacchi */ 103*993e3fafSRobert Mustacchi attrp->dma_attr_sgllen = sgl; 104*993e3fafSRobert Mustacchi 105*993e3fafSRobert Mustacchi /* 106*993e3fafSRobert Mustacchi * The granularity describes the addressing granularity. e.g. can things 107*993e3fafSRobert Mustacchi * ask for chunks in units of this number of bytes. For PCI this should 108*993e3fafSRobert Mustacchi * always be one. 109*993e3fafSRobert Mustacchi */ 110*993e3fafSRobert Mustacchi attrp->dma_attr_granular = 1; 111*993e3fafSRobert Mustacchi 112*993e3fafSRobert Mustacchi if (DDI_FM_DMA_ERR_CAP(xhcip->xhci_fm_caps)) { 113*993e3fafSRobert Mustacchi attrp->dma_attr_flags = DDI_DMA_FLAGERR; 114*993e3fafSRobert Mustacchi } else { 115*993e3fafSRobert Mustacchi attrp->dma_attr_flags = 0; 116*993e3fafSRobert Mustacchi } 117*993e3fafSRobert Mustacchi } 118*993e3fafSRobert Mustacchi 119*993e3fafSRobert Mustacchi /* 120*993e3fafSRobert Mustacchi * This routine tries to create DMA attributes for normal allocations for data 121*993e3fafSRobert Mustacchi * structures and the like. By default we use the same values as the transfer 122*993e3fafSRobert Mustacchi * attributes, but have explicit comments about how they're different. 123*993e3fafSRobert Mustacchi */ 124*993e3fafSRobert Mustacchi void 125*993e3fafSRobert Mustacchi xhci_dma_dma_attr(xhci_t *xhcip, ddi_dma_attr_t *attrp) 126*993e3fafSRobert Mustacchi { 127*993e3fafSRobert Mustacchi /* 128*993e3fafSRobert Mustacchi * Note, we always use a single SGL for these DMA allocations as these 129*993e3fafSRobert Mustacchi * are used for small data structures. 130*993e3fafSRobert Mustacchi */ 131*993e3fafSRobert Mustacchi xhci_dma_transfer_attr(xhcip, attrp, XHCI_DEF_DMA_SGL); 132*993e3fafSRobert Mustacchi 133*993e3fafSRobert Mustacchi /* 134*993e3fafSRobert Mustacchi * The maximum size of any of these structures is 4k as opposed to the 135*993e3fafSRobert Mustacchi * 64K max described above. Similarly the boundary requirement is 136*993e3fafSRobert Mustacchi * reduced to 4k. 137*993e3fafSRobert Mustacchi */ 138*993e3fafSRobert Mustacchi attrp->dma_attr_count_max = xhcip->xhci_caps.xcap_pagesize; 139*993e3fafSRobert Mustacchi attrp->dma_attr_maxxfer = xhcip->xhci_caps.xcap_pagesize; 140*993e3fafSRobert Mustacchi attrp->dma_attr_seg = xhcip->xhci_caps.xcap_pagesize - 1; 141*993e3fafSRobert Mustacchi } 142*993e3fafSRobert Mustacchi 143*993e3fafSRobert Mustacchi /* 144*993e3fafSRobert Mustacchi * Fill in attributes for a scratchpad entry. The scratchpad entries are 145*993e3fafSRobert Mustacchi * somewhat different in so far as they are closest to a normal DMA attribute, 146*993e3fafSRobert Mustacchi * except they have stricter alignments, needing to be page sized. 147*993e3fafSRobert Mustacchi * 148*993e3fafSRobert Mustacchi * In addition, because we never access this memory ourselves, we can just mark 149*993e3fafSRobert Mustacchi * it all as relaxed ordering. 150*993e3fafSRobert Mustacchi */ 151*993e3fafSRobert Mustacchi void 152*993e3fafSRobert Mustacchi xhci_dma_scratchpad_attr(xhci_t *xhcip, ddi_dma_attr_t *attrp) 153*993e3fafSRobert Mustacchi { 154*993e3fafSRobert Mustacchi xhci_dma_dma_attr(xhcip, attrp); 155*993e3fafSRobert Mustacchi attrp->dma_attr_align = xhcip->xhci_caps.xcap_pagesize; 156*993e3fafSRobert Mustacchi attrp->dma_attr_flags |= DDI_DMA_RELAXED_ORDERING; 157*993e3fafSRobert Mustacchi } 158*993e3fafSRobert Mustacchi 159*993e3fafSRobert Mustacchi /* 160*993e3fafSRobert Mustacchi * This should be used for the simple case of a single SGL entry, which is the 161*993e3fafSRobert Mustacchi * vast majority of the non-transfer allocations. 162*993e3fafSRobert Mustacchi */ 163*993e3fafSRobert Mustacchi uint64_t 164*993e3fafSRobert Mustacchi xhci_dma_pa(xhci_dma_buffer_t *xdb) 165*993e3fafSRobert Mustacchi { 166*993e3fafSRobert Mustacchi ASSERT(xdb->xdb_ncookies == 1); 167*993e3fafSRobert Mustacchi return (xdb->xdb_cookies[0].dmac_laddress); 168*993e3fafSRobert Mustacchi } 169*993e3fafSRobert Mustacchi 170*993e3fafSRobert Mustacchi void 171*993e3fafSRobert Mustacchi xhci_dma_free(xhci_dma_buffer_t *xdb) 172*993e3fafSRobert Mustacchi { 173*993e3fafSRobert Mustacchi if (xdb->xdb_ncookies != 0) { 174*993e3fafSRobert Mustacchi VERIFY(xdb->xdb_dma_handle != NULL); 175*993e3fafSRobert Mustacchi (void) ddi_dma_unbind_handle(xdb->xdb_dma_handle); 176*993e3fafSRobert Mustacchi xdb->xdb_ncookies = 0; 177*993e3fafSRobert Mustacchi bzero(xdb->xdb_cookies, sizeof (ddi_dma_cookie_t) * 178*993e3fafSRobert Mustacchi XHCI_TRANSFER_DMA_SGL); 179*993e3fafSRobert Mustacchi xdb->xdb_len = 0; 180*993e3fafSRobert Mustacchi } 181*993e3fafSRobert Mustacchi 182*993e3fafSRobert Mustacchi if (xdb->xdb_acc_handle != NULL) { 183*993e3fafSRobert Mustacchi ddi_dma_mem_free(&xdb->xdb_acc_handle); 184*993e3fafSRobert Mustacchi xdb->xdb_acc_handle = NULL; 185*993e3fafSRobert Mustacchi xdb->xdb_va = NULL; 186*993e3fafSRobert Mustacchi } 187*993e3fafSRobert Mustacchi 188*993e3fafSRobert Mustacchi if (xdb->xdb_dma_handle != NULL) { 189*993e3fafSRobert Mustacchi ddi_dma_free_handle(&xdb->xdb_dma_handle); 190*993e3fafSRobert Mustacchi xdb->xdb_dma_handle = NULL; 191*993e3fafSRobert Mustacchi } 192*993e3fafSRobert Mustacchi 193*993e3fafSRobert Mustacchi ASSERT(xdb->xdb_va == NULL); 194*993e3fafSRobert Mustacchi ASSERT(xdb->xdb_ncookies == 0); 195*993e3fafSRobert Mustacchi ASSERT(xdb->xdb_cookies[0].dmac_laddress == 0); 196*993e3fafSRobert Mustacchi ASSERT(xdb->xdb_len == 0); 197*993e3fafSRobert Mustacchi } 198*993e3fafSRobert Mustacchi 199*993e3fafSRobert Mustacchi boolean_t 200*993e3fafSRobert Mustacchi xhci_dma_alloc(xhci_t *xhcip, xhci_dma_buffer_t *xdb, 201*993e3fafSRobert Mustacchi ddi_dma_attr_t *attrp, ddi_device_acc_attr_t *accp, boolean_t zero, 202*993e3fafSRobert Mustacchi size_t size, boolean_t wait) 203*993e3fafSRobert Mustacchi { 204*993e3fafSRobert Mustacchi int ret, i; 205*993e3fafSRobert Mustacchi uint_t flags = DDI_DMA_CONSISTENT; 206*993e3fafSRobert Mustacchi size_t len; 207*993e3fafSRobert Mustacchi ddi_dma_cookie_t cookie; 208*993e3fafSRobert Mustacchi uint_t ncookies; 209*993e3fafSRobert Mustacchi int (*memcb)(caddr_t); 210*993e3fafSRobert Mustacchi 211*993e3fafSRobert Mustacchi if (wait == B_TRUE) { 212*993e3fafSRobert Mustacchi memcb = DDI_DMA_SLEEP; 213*993e3fafSRobert Mustacchi } else { 214*993e3fafSRobert Mustacchi memcb = DDI_DMA_DONTWAIT; 215*993e3fafSRobert Mustacchi } 216*993e3fafSRobert Mustacchi 217*993e3fafSRobert Mustacchi ret = ddi_dma_alloc_handle(xhcip->xhci_dip, attrp, memcb, NULL, 218*993e3fafSRobert Mustacchi &xdb->xdb_dma_handle); 219*993e3fafSRobert Mustacchi if (ret != 0) { 220*993e3fafSRobert Mustacchi xhci_log(xhcip, "!failed to allocate DMA handle: %d", ret); 221*993e3fafSRobert Mustacchi xdb->xdb_dma_handle = NULL; 222*993e3fafSRobert Mustacchi return (B_FALSE); 223*993e3fafSRobert Mustacchi } 224*993e3fafSRobert Mustacchi 225*993e3fafSRobert Mustacchi ret = ddi_dma_mem_alloc(xdb->xdb_dma_handle, size, accp, flags, memcb, 226*993e3fafSRobert Mustacchi NULL, &xdb->xdb_va, &len, &xdb->xdb_acc_handle); 227*993e3fafSRobert Mustacchi if (ret != DDI_SUCCESS) { 228*993e3fafSRobert Mustacchi xhci_log(xhcip, "!failed to allocate DMA memory: %d", ret); 229*993e3fafSRobert Mustacchi xdb->xdb_va = NULL; 230*993e3fafSRobert Mustacchi xdb->xdb_acc_handle = NULL; 231*993e3fafSRobert Mustacchi xhci_dma_free(xdb); 232*993e3fafSRobert Mustacchi return (B_FALSE); 233*993e3fafSRobert Mustacchi } 234*993e3fafSRobert Mustacchi 235*993e3fafSRobert Mustacchi if (zero == B_TRUE) 236*993e3fafSRobert Mustacchi bzero(xdb->xdb_va, len); 237*993e3fafSRobert Mustacchi 238*993e3fafSRobert Mustacchi ret = ddi_dma_addr_bind_handle(xdb->xdb_dma_handle, NULL, 239*993e3fafSRobert Mustacchi xdb->xdb_va, len, DDI_DMA_RDWR | flags, memcb, NULL, &cookie, 240*993e3fafSRobert Mustacchi &ncookies); 241*993e3fafSRobert Mustacchi if (ret != 0) { 242*993e3fafSRobert Mustacchi xhci_log(xhcip, "!failed to bind DMA memory: %d", ret); 243*993e3fafSRobert Mustacchi xhci_dma_free(xdb); 244*993e3fafSRobert Mustacchi return (B_FALSE); 245*993e3fafSRobert Mustacchi } 246*993e3fafSRobert Mustacchi 247*993e3fafSRobert Mustacchi /* 248*993e3fafSRobert Mustacchi * Note we explicitly store the logical length of this allocation. The 249*993e3fafSRobert Mustacchi * physical length is available via the cookies. 250*993e3fafSRobert Mustacchi */ 251*993e3fafSRobert Mustacchi xdb->xdb_len = size; 252*993e3fafSRobert Mustacchi xdb->xdb_ncookies = ncookies; 253*993e3fafSRobert Mustacchi xdb->xdb_cookies[0] = cookie; 254*993e3fafSRobert Mustacchi for (i = 1; i < ncookies; i++) { 255*993e3fafSRobert Mustacchi ddi_dma_nextcookie(xdb->xdb_dma_handle, &xdb->xdb_cookies[i]); 256*993e3fafSRobert Mustacchi } 257*993e3fafSRobert Mustacchi 258*993e3fafSRobert Mustacchi 259*993e3fafSRobert Mustacchi return (B_TRUE); 260*993e3fafSRobert Mustacchi } 261*993e3fafSRobert Mustacchi 262*993e3fafSRobert Mustacchi void 263*993e3fafSRobert Mustacchi xhci_transfer_free(xhci_t *xhcip, xhci_transfer_t *xt) 264*993e3fafSRobert Mustacchi { 265*993e3fafSRobert Mustacchi if (xt == NULL) 266*993e3fafSRobert Mustacchi return; 267*993e3fafSRobert Mustacchi 268*993e3fafSRobert Mustacchi VERIFY(xhcip != NULL); 269*993e3fafSRobert Mustacchi xhci_dma_free(&xt->xt_buffer); 270*993e3fafSRobert Mustacchi if (xt->xt_isoc != NULL) { 271*993e3fafSRobert Mustacchi ASSERT(xt->xt_ntrbs > 0); 272*993e3fafSRobert Mustacchi kmem_free(xt->xt_isoc, sizeof (usb_isoc_pkt_descr_t) * 273*993e3fafSRobert Mustacchi xt->xt_ntrbs); 274*993e3fafSRobert Mustacchi xt->xt_isoc = NULL; 275*993e3fafSRobert Mustacchi } 276*993e3fafSRobert Mustacchi if (xt->xt_trbs != NULL) { 277*993e3fafSRobert Mustacchi ASSERT(xt->xt_ntrbs > 0); 278*993e3fafSRobert Mustacchi kmem_free(xt->xt_trbs, sizeof (xhci_trb_t) * xt->xt_ntrbs); 279*993e3fafSRobert Mustacchi xt->xt_trbs = NULL; 280*993e3fafSRobert Mustacchi } 281*993e3fafSRobert Mustacchi kmem_free(xt, sizeof (xhci_transfer_t)); 282*993e3fafSRobert Mustacchi } 283*993e3fafSRobert Mustacchi 284*993e3fafSRobert Mustacchi xhci_transfer_t * 285*993e3fafSRobert Mustacchi xhci_transfer_alloc(xhci_t *xhcip, xhci_endpoint_t *xep, size_t size, int trbs, 286*993e3fafSRobert Mustacchi int usb_flags) 287*993e3fafSRobert Mustacchi { 288*993e3fafSRobert Mustacchi int kmflags; 289*993e3fafSRobert Mustacchi boolean_t dmawait; 290*993e3fafSRobert Mustacchi xhci_transfer_t *xt; 291*993e3fafSRobert Mustacchi ddi_device_acc_attr_t acc; 292*993e3fafSRobert Mustacchi ddi_dma_attr_t attr; 293*993e3fafSRobert Mustacchi 294*993e3fafSRobert Mustacchi if (usb_flags & USB_FLAGS_SLEEP) { 295*993e3fafSRobert Mustacchi kmflags = KM_SLEEP; 296*993e3fafSRobert Mustacchi dmawait = B_TRUE; 297*993e3fafSRobert Mustacchi } else { 298*993e3fafSRobert Mustacchi kmflags = KM_NOSLEEP; 299*993e3fafSRobert Mustacchi dmawait = B_FALSE; 300*993e3fafSRobert Mustacchi } 301*993e3fafSRobert Mustacchi 302*993e3fafSRobert Mustacchi xt = kmem_zalloc(sizeof (xhci_transfer_t), kmflags); 303*993e3fafSRobert Mustacchi if (xt == NULL) 304*993e3fafSRobert Mustacchi return (NULL); 305*993e3fafSRobert Mustacchi 306*993e3fafSRobert Mustacchi if (size != 0) { 307*993e3fafSRobert Mustacchi int sgl = XHCI_DEF_DMA_SGL; 308*993e3fafSRobert Mustacchi 309*993e3fafSRobert Mustacchi /* 310*993e3fafSRobert Mustacchi * For BULK transfers, we always increase the number of SGL 311*993e3fafSRobert Mustacchi * entries that we support to make things easier for the kernel. 312*993e3fafSRobert Mustacchi * However, for control transfers, we currently opt to keep 313*993e3fafSRobert Mustacchi * things a bit simpler and use our default of one SGL. There's 314*993e3fafSRobert Mustacchi * no good technical reason for this, rather it just keeps 315*993e3fafSRobert Mustacchi * things a bit easier. 316*993e3fafSRobert Mustacchi * 317*993e3fafSRobert Mustacchi * To simplify things, we don't use additional SGL entries for 318*993e3fafSRobert Mustacchi * ISOC transfers. While this isn't the best, it isn't too far 319*993e3fafSRobert Mustacchi * off from what ehci and co. have done before. If this becomes 320*993e3fafSRobert Mustacchi * a technical issue, it's certainly possible to increase the 321*993e3fafSRobert Mustacchi * SGL entry count. 322*993e3fafSRobert Mustacchi */ 323*993e3fafSRobert Mustacchi if (xep->xep_type == USB_EP_ATTR_BULK) 324*993e3fafSRobert Mustacchi sgl = XHCI_TRANSFER_DMA_SGL; 325*993e3fafSRobert Mustacchi 326*993e3fafSRobert Mustacchi xhci_dma_acc_attr(xhcip, &acc); 327*993e3fafSRobert Mustacchi xhci_dma_transfer_attr(xhcip, &attr, sgl); 328*993e3fafSRobert Mustacchi if (xhci_dma_alloc(xhcip, &xt->xt_buffer, &attr, &acc, B_FALSE, 329*993e3fafSRobert Mustacchi size, dmawait) == B_FALSE) { 330*993e3fafSRobert Mustacchi kmem_free(xt, sizeof (xhci_transfer_t)); 331*993e3fafSRobert Mustacchi return (NULL); 332*993e3fafSRobert Mustacchi } 333*993e3fafSRobert Mustacchi 334*993e3fafSRobert Mustacchi /* 335*993e3fafSRobert Mustacchi * ISOC transfers are a bit special and don't need additional 336*993e3fafSRobert Mustacchi * TRBs for data. 337*993e3fafSRobert Mustacchi */ 338*993e3fafSRobert Mustacchi if (xep->xep_type != USB_EP_ATTR_ISOCH) 339*993e3fafSRobert Mustacchi trbs += xt->xt_buffer.xdb_ncookies; 340*993e3fafSRobert Mustacchi } 341*993e3fafSRobert Mustacchi 342*993e3fafSRobert Mustacchi xt->xt_trbs = kmem_zalloc(sizeof (xhci_trb_t) * trbs, kmflags); 343*993e3fafSRobert Mustacchi if (xt->xt_trbs == NULL) { 344*993e3fafSRobert Mustacchi xhci_dma_free(&xt->xt_buffer); 345*993e3fafSRobert Mustacchi kmem_free(xt, sizeof (xhci_transfer_t)); 346*993e3fafSRobert Mustacchi return (NULL); 347*993e3fafSRobert Mustacchi } 348*993e3fafSRobert Mustacchi 349*993e3fafSRobert Mustacchi /* 350*993e3fafSRobert Mustacchi * For ISOCH transfers, we need to also allocate the results data. 351*993e3fafSRobert Mustacchi */ 352*993e3fafSRobert Mustacchi if (xep->xep_type == USB_EP_ATTR_ISOCH) { 353*993e3fafSRobert Mustacchi xt->xt_isoc = kmem_zalloc(sizeof (usb_isoc_pkt_descr_t) * trbs, 354*993e3fafSRobert Mustacchi kmflags); 355*993e3fafSRobert Mustacchi if (xt->xt_isoc == NULL) { 356*993e3fafSRobert Mustacchi kmem_free(xt->xt_trbs, sizeof (xhci_trb_t) * trbs); 357*993e3fafSRobert Mustacchi xhci_dma_free(&xt->xt_buffer); 358*993e3fafSRobert Mustacchi kmem_free(xt, sizeof (xhci_transfer_t)); 359*993e3fafSRobert Mustacchi return (NULL); 360*993e3fafSRobert Mustacchi } 361*993e3fafSRobert Mustacchi } 362*993e3fafSRobert Mustacchi 363*993e3fafSRobert Mustacchi xt->xt_ntrbs = trbs; 364*993e3fafSRobert Mustacchi xt->xt_cr = USB_CR_OK; 365*993e3fafSRobert Mustacchi 366*993e3fafSRobert Mustacchi return (xt); 367*993e3fafSRobert Mustacchi } 368*993e3fafSRobert Mustacchi 369*993e3fafSRobert Mustacchi /* 370*993e3fafSRobert Mustacchi * Abstract the notion of copying out to handle the case of multiple DMA 371*993e3fafSRobert Mustacchi * cookies. If tobuf is true, we are copying to the kernel provided buffer, 372*993e3fafSRobert Mustacchi * otherwise we're copying into the DMA memory. 373*993e3fafSRobert Mustacchi */ 374*993e3fafSRobert Mustacchi void 375*993e3fafSRobert Mustacchi xhci_transfer_copy(xhci_transfer_t *xt, void *buf, size_t len, 376*993e3fafSRobert Mustacchi boolean_t tobuf) 377*993e3fafSRobert Mustacchi { 378*993e3fafSRobert Mustacchi void *dmabuf = xt->xt_buffer.xdb_va; 379*993e3fafSRobert Mustacchi if (tobuf == B_TRUE) 380*993e3fafSRobert Mustacchi bcopy(dmabuf, buf, len); 381*993e3fafSRobert Mustacchi else 382*993e3fafSRobert Mustacchi bcopy(buf, dmabuf, len); 383*993e3fafSRobert Mustacchi } 384*993e3fafSRobert Mustacchi 385*993e3fafSRobert Mustacchi int 386*993e3fafSRobert Mustacchi xhci_transfer_sync(xhci_t *xhcip, xhci_transfer_t *xt, uint_t type) 387*993e3fafSRobert Mustacchi { 388*993e3fafSRobert Mustacchi XHCI_DMA_SYNC(xt->xt_buffer, type); 389*993e3fafSRobert Mustacchi return (xhci_check_dma_handle(xhcip, &xt->xt_buffer)); 390*993e3fafSRobert Mustacchi } 391*993e3fafSRobert Mustacchi 392*993e3fafSRobert Mustacchi /* 393*993e3fafSRobert Mustacchi * We're required to try and inform the xHCI controller about the number of data 394*993e3fafSRobert Mustacchi * packets that are required. The algorithm to use is described in xHCI 1.1 / 395*993e3fafSRobert Mustacchi * 4.11.2.4. While it might be tempting to just try and calculate the number of 396*993e3fafSRobert Mustacchi * packets based on simple rounding of the remaining number of bytes, that 397*993e3fafSRobert Mustacchi * misses a critical problem -- DMA boundaries may cause us to need additional 398*993e3fafSRobert Mustacchi * packets that are missed initially. Consider a transfer made up of four 399*993e3fafSRobert Mustacchi * different DMA buffers sized in bytes: 4096, 4096, 256, 256, with a 512 byte 400*993e3fafSRobert Mustacchi * packet size. 401*993e3fafSRobert Mustacchi * 402*993e3fafSRobert Mustacchi * Remain 4608 512 256 0 403*993e3fafSRobert Mustacchi * Bytes 4096 4096 256 256 404*993e3fafSRobert Mustacchi * Naive TD 9 1 1 0 405*993e3fafSRobert Mustacchi * Act TD 10 2 1 0 406*993e3fafSRobert Mustacchi * 407*993e3fafSRobert Mustacchi * This means that the only safe way forward here is to work backwards and see 408*993e3fafSRobert Mustacchi * how many we need to work up to this point. 409*993e3fafSRobert Mustacchi */ 410*993e3fafSRobert Mustacchi static int 411*993e3fafSRobert Mustacchi xhci_transfer_get_tdsize(xhci_transfer_t *xt, uint_t off, uint_t mps) 412*993e3fafSRobert Mustacchi { 413*993e3fafSRobert Mustacchi int i; 414*993e3fafSRobert Mustacchi uint_t npkt = 0; 415*993e3fafSRobert Mustacchi 416*993e3fafSRobert Mustacchi /* 417*993e3fafSRobert Mustacchi * There are always zero packets for the last TRB. 418*993e3fafSRobert Mustacchi */ 419*993e3fafSRobert Mustacchi ASSERT(xt->xt_buffer.xdb_ncookies > 0); 420*993e3fafSRobert Mustacchi for (i = xt->xt_buffer.xdb_ncookies - 1; i > off; i--) { 421*993e3fafSRobert Mustacchi size_t len; 422*993e3fafSRobert Mustacchi 423*993e3fafSRobert Mustacchi /* 424*993e3fafSRobert Mustacchi * The maximum value we can return is 31 packets. So, in that 425*993e3fafSRobert Mustacchi * case we short-circuit and return. 426*993e3fafSRobert Mustacchi */ 427*993e3fafSRobert Mustacchi if (npkt >= 31) 428*993e3fafSRobert Mustacchi return (31); 429*993e3fafSRobert Mustacchi 430*993e3fafSRobert Mustacchi len = roundup(xt->xt_buffer.xdb_cookies[i].dmac_size, mps); 431*993e3fafSRobert Mustacchi npkt += len / mps; 432*993e3fafSRobert Mustacchi } 433*993e3fafSRobert Mustacchi 434*993e3fafSRobert Mustacchi return (npkt); 435*993e3fafSRobert Mustacchi } 436*993e3fafSRobert Mustacchi 437*993e3fafSRobert Mustacchi void 438*993e3fafSRobert Mustacchi xhci_transfer_trb_fill_data(xhci_endpoint_t *xep, xhci_transfer_t *xt, int off, 439*993e3fafSRobert Mustacchi boolean_t in) 440*993e3fafSRobert Mustacchi { 441*993e3fafSRobert Mustacchi uint_t mps, tdsize, flags; 442*993e3fafSRobert Mustacchi int i; 443*993e3fafSRobert Mustacchi 444*993e3fafSRobert Mustacchi VERIFY(xt->xt_buffer.xdb_ncookies > 0); 445*993e3fafSRobert Mustacchi VERIFY(xep->xep_pipe != NULL); 446*993e3fafSRobert Mustacchi VERIFY(off + xt->xt_buffer.xdb_ncookies <= xt->xt_ntrbs); 447*993e3fafSRobert Mustacchi mps = xep->xep_pipe->p_ep.wMaxPacketSize; 448*993e3fafSRobert Mustacchi 449*993e3fafSRobert Mustacchi for (i = 0; i < xt->xt_buffer.xdb_ncookies; i++) { 450*993e3fafSRobert Mustacchi uint64_t pa, dmasz; 451*993e3fafSRobert Mustacchi 452*993e3fafSRobert Mustacchi pa = xt->xt_buffer.xdb_cookies[i].dmac_laddress; 453*993e3fafSRobert Mustacchi dmasz = xt->xt_buffer.xdb_cookies[i].dmac_size; 454*993e3fafSRobert Mustacchi 455*993e3fafSRobert Mustacchi tdsize = xhci_transfer_get_tdsize(xt, i, mps); 456*993e3fafSRobert Mustacchi 457*993e3fafSRobert Mustacchi flags = XHCI_TRB_TYPE_NORMAL; 458*993e3fafSRobert Mustacchi if (i == 0 && xep->xep_type == USB_EP_ATTR_CONTROL) { 459*993e3fafSRobert Mustacchi flags = XHCI_TRB_TYPE_DATA; 460*993e3fafSRobert Mustacchi if (in == B_TRUE) 461*993e3fafSRobert Mustacchi flags |= XHCI_TRB_DIR_IN; 462*993e3fafSRobert Mustacchi } 463*993e3fafSRobert Mustacchi 464*993e3fafSRobert Mustacchi /* 465*993e3fafSRobert Mustacchi * When reading data in (from the device), we may get shorter 466*993e3fafSRobert Mustacchi * transfers than the buffer allowed for. To make sure we get 467*993e3fafSRobert Mustacchi * notified about that and handle that, we need to set the ISP 468*993e3fafSRobert Mustacchi * flag. 469*993e3fafSRobert Mustacchi */ 470*993e3fafSRobert Mustacchi if (in == B_TRUE) { 471*993e3fafSRobert Mustacchi flags |= XHCI_TRB_ISP; 472*993e3fafSRobert Mustacchi xt->xt_data_tohost = B_TRUE; 473*993e3fafSRobert Mustacchi } 474*993e3fafSRobert Mustacchi 475*993e3fafSRobert Mustacchi /* 476*993e3fafSRobert Mustacchi * When we have more than one cookie, we are technically 477*993e3fafSRobert Mustacchi * chaining together things according to the controllers view, 478*993e3fafSRobert Mustacchi * hence why we need to set the chain flag. 479*993e3fafSRobert Mustacchi */ 480*993e3fafSRobert Mustacchi if (xt->xt_buffer.xdb_ncookies > 1 && 481*993e3fafSRobert Mustacchi i != (xt->xt_buffer.xdb_ncookies - 1)) { 482*993e3fafSRobert Mustacchi flags |= XHCI_TRB_CHAIN; 483*993e3fafSRobert Mustacchi } 484*993e3fafSRobert Mustacchi 485*993e3fafSRobert Mustacchi /* 486*993e3fafSRobert Mustacchi * If we have a non-control transfer, then we need to make sure 487*993e3fafSRobert Mustacchi * that we set ourselves up to be interrupted, which we set for 488*993e3fafSRobert Mustacchi * the last entry. 489*993e3fafSRobert Mustacchi */ 490*993e3fafSRobert Mustacchi if (i + 1 == xt->xt_buffer.xdb_ncookies && 491*993e3fafSRobert Mustacchi xep->xep_type != USB_EP_ATTR_CONTROL) { 492*993e3fafSRobert Mustacchi flags |= XHCI_TRB_IOC; 493*993e3fafSRobert Mustacchi } 494*993e3fafSRobert Mustacchi 495*993e3fafSRobert Mustacchi xt->xt_trbs[off + i].trb_addr = LE_64(pa); 496*993e3fafSRobert Mustacchi xt->xt_trbs[off + i].trb_status = LE_32(XHCI_TRB_LEN(dmasz) | 497*993e3fafSRobert Mustacchi XHCI_TRB_TDREM(tdsize) | XHCI_TRB_INTR(0)); 498*993e3fafSRobert Mustacchi xt->xt_trbs[off + i].trb_flags = LE_32(flags); 499*993e3fafSRobert Mustacchi } 500*993e3fafSRobert Mustacchi } 501*993e3fafSRobert Mustacchi 502*993e3fafSRobert Mustacchi /* 503*993e3fafSRobert Mustacchi * These are utility functions for isochronus transfers to help calculate the 504*993e3fafSRobert Mustacchi * transfer burst count (TBC) and transfer last burst packet count (TLPBC) 505*993e3fafSRobert Mustacchi * entries for an isochronus entry. See xHCI 1.1 / 4.11.2.3 for how to calculate 506*993e3fafSRobert Mustacchi * them. 507*993e3fafSRobert Mustacchi */ 508*993e3fafSRobert Mustacchi void 509*993e3fafSRobert Mustacchi xhci_transfer_calculate_isoc(xhci_device_t *xd, xhci_endpoint_t *xep, 510*993e3fafSRobert Mustacchi uint_t trb_len, uint_t *tbc, uint_t *tlbpc) 511*993e3fafSRobert Mustacchi { 512*993e3fafSRobert Mustacchi uint_t mps, tdpc, burst; 513*993e3fafSRobert Mustacchi 514*993e3fafSRobert Mustacchi /* 515*993e3fafSRobert Mustacchi * Even if we're asked to send no data, that actually requires the 516*993e3fafSRobert Mustacchi * equivalent of sending one byte of data. 517*993e3fafSRobert Mustacchi */ 518*993e3fafSRobert Mustacchi if (trb_len == 0) 519*993e3fafSRobert Mustacchi trb_len = 1; 520*993e3fafSRobert Mustacchi 521*993e3fafSRobert Mustacchi mps = XHCI_EPCTX_GET_MPS(xd->xd_endout[xep->xep_num]->xec_info2); 522*993e3fafSRobert Mustacchi burst = XHCI_EPCTX_GET_MAXB(xd->xd_endout[xep->xep_num]->xec_info2); 523*993e3fafSRobert Mustacchi 524*993e3fafSRobert Mustacchi /* 525*993e3fafSRobert Mustacchi * This is supposed to correspond to the Transfer Descriptor Packet 526*993e3fafSRobert Mustacchi * Count from xHCI 1.1 / 4.14.1. 527*993e3fafSRobert Mustacchi */ 528*993e3fafSRobert Mustacchi tdpc = howmany(trb_len, mps); 529*993e3fafSRobert Mustacchi *tbc = howmany(tdpc, burst + 1) - 1; 530*993e3fafSRobert Mustacchi 531*993e3fafSRobert Mustacchi if ((tdpc % (burst + 1)) == 0) 532*993e3fafSRobert Mustacchi *tlbpc = burst; 533*993e3fafSRobert Mustacchi else 534*993e3fafSRobert Mustacchi *tlbpc = (tdpc % (burst + 1)) - 1; 535*993e3fafSRobert Mustacchi } 536