1993e3fafSRobert Mustacchi /* 2993e3fafSRobert Mustacchi * This file and its contents are supplied under the terms of the 3993e3fafSRobert Mustacchi * Common Development and Distribution License ("CDDL"), version 1.0. 4993e3fafSRobert Mustacchi * You may only use this file in accordance with the terms of version 5993e3fafSRobert Mustacchi * 1.0 of the CDDL. 6993e3fafSRobert Mustacchi * 7993e3fafSRobert Mustacchi * A full copy of the text of the CDDL should have accompanied this 8993e3fafSRobert Mustacchi * source. A copy of the CDDL is also available via the Internet at 9993e3fafSRobert Mustacchi * http://www.illumos.org/license/CDDL. 10993e3fafSRobert Mustacchi */ 11993e3fafSRobert Mustacchi 12993e3fafSRobert Mustacchi /* 13*2aba3acdSRobert Mustacchi * Copyright (c) 2018, Joyent, Inc. 14993e3fafSRobert Mustacchi */ 15993e3fafSRobert Mustacchi 16993e3fafSRobert Mustacchi /* 17993e3fafSRobert Mustacchi * xHCI DMA Management Routines 18993e3fafSRobert Mustacchi * 19993e3fafSRobert Mustacchi * Please see the big theory statement in xhci.c for more information. 20993e3fafSRobert Mustacchi */ 21993e3fafSRobert Mustacchi 22993e3fafSRobert Mustacchi #include <sys/usb/hcd/xhci/xhci.h> 23993e3fafSRobert Mustacchi 24993e3fafSRobert Mustacchi int 25993e3fafSRobert Mustacchi xhci_check_dma_handle(xhci_t *xhcip, xhci_dma_buffer_t *xdb) 26993e3fafSRobert Mustacchi { 27993e3fafSRobert Mustacchi ddi_fm_error_t de; 28993e3fafSRobert Mustacchi 29993e3fafSRobert Mustacchi if (!DDI_FM_DMA_ERR_CAP(xhcip->xhci_fm_caps)) 30993e3fafSRobert Mustacchi return (0); 31993e3fafSRobert Mustacchi 32993e3fafSRobert Mustacchi ddi_fm_dma_err_get(xdb->xdb_dma_handle, &de, DDI_FME_VERSION); 33993e3fafSRobert Mustacchi return (de.fme_status); 34993e3fafSRobert Mustacchi } 35993e3fafSRobert Mustacchi 36993e3fafSRobert Mustacchi void 37993e3fafSRobert Mustacchi xhci_dma_acc_attr(xhci_t *xhcip, ddi_device_acc_attr_t *accp) 38993e3fafSRobert Mustacchi { 39993e3fafSRobert Mustacchi accp->devacc_attr_version = DDI_DEVICE_ATTR_V0; 40993e3fafSRobert Mustacchi accp->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 41993e3fafSRobert Mustacchi accp->devacc_attr_dataorder = DDI_STRICTORDER_ACC; 42993e3fafSRobert Mustacchi 43993e3fafSRobert Mustacchi if (DDI_FM_DMA_ERR_CAP(xhcip->xhci_fm_caps)) { 44993e3fafSRobert Mustacchi accp->devacc_attr_access = DDI_FLAGERR_ACC; 45993e3fafSRobert Mustacchi } else { 46993e3fafSRobert Mustacchi accp->devacc_attr_access = DDI_DEFAULT_ACC; 47993e3fafSRobert Mustacchi } 48993e3fafSRobert Mustacchi } 49993e3fafSRobert Mustacchi 50993e3fafSRobert Mustacchi /* 51993e3fafSRobert Mustacchi * These are DMA attributes that we assign when making a transfer. The SGL is 52993e3fafSRobert Mustacchi * variable and based on the caller, which varies based on the type of transfer 53993e3fafSRobert Mustacchi * we're doing. 54993e3fafSRobert Mustacchi */ 55993e3fafSRobert Mustacchi void 56993e3fafSRobert Mustacchi xhci_dma_transfer_attr(xhci_t *xhcip, ddi_dma_attr_t *attrp, uint_t sgl) 57993e3fafSRobert Mustacchi { 58993e3fafSRobert Mustacchi VERIFY3U(sgl, >, 0); 59993e3fafSRobert Mustacchi VERIFY3U(sgl, <=, XHCI_TRANSFER_DMA_SGL); 60993e3fafSRobert Mustacchi attrp->dma_attr_version = DMA_ATTR_V0; 61993e3fafSRobert Mustacchi 62993e3fafSRobert Mustacchi /* 63993e3fafSRobert Mustacchi * The range of data that we can use is based on what hardware supports. 64993e3fafSRobert Mustacchi */ 65993e3fafSRobert Mustacchi attrp->dma_attr_addr_lo = 0x0; 66993e3fafSRobert Mustacchi if (xhcip->xhci_caps.xcap_flags & XCAP_AC64) { 67993e3fafSRobert Mustacchi attrp->dma_attr_addr_hi = UINT64_MAX; 68993e3fafSRobert Mustacchi } else { 69993e3fafSRobert Mustacchi attrp->dma_attr_addr_hi = UINT32_MAX; 70993e3fafSRobert Mustacchi } 71993e3fafSRobert Mustacchi 72993e3fafSRobert Mustacchi /* 73993e3fafSRobert Mustacchi * The count max indicates the total amount that will fit into one 74993e3fafSRobert Mustacchi * cookie, which is one TRB in our world. In other words 64k. 75993e3fafSRobert Mustacchi */ 76993e3fafSRobert Mustacchi attrp->dma_attr_count_max = XHCI_TRB_MAX_TRANSFER; 77993e3fafSRobert Mustacchi 78993e3fafSRobert Mustacchi /* 79993e3fafSRobert Mustacchi * The alignment and segment are related. The alignment describes the 80993e3fafSRobert Mustacchi * alignment of the PA. The segment describes a boundary that the DMA 81993e3fafSRobert Mustacchi * allocation cannot cross. In other words, for a given chunk of memory 82993e3fafSRobert Mustacchi * it cannot cross a 64-byte boundary. However, the physical address 83993e3fafSRobert Mustacchi * only needs to be aligned to align bytes. 84993e3fafSRobert Mustacchi */ 85993e3fafSRobert Mustacchi attrp->dma_attr_align = XHCI_DMA_ALIGN; 86993e3fafSRobert Mustacchi attrp->dma_attr_seg = XHCI_TRB_MAX_TRANSFER - 1; 87993e3fafSRobert Mustacchi 88993e3fafSRobert Mustacchi 89993e3fafSRobert Mustacchi attrp->dma_attr_burstsizes = 0xfff; 90993e3fafSRobert Mustacchi 91993e3fafSRobert Mustacchi /* 92993e3fafSRobert Mustacchi * This is the maximum we can send. Technically this is limited by the 93993e3fafSRobert Mustacchi * descriptors and not by hardware, hence why we use a large value for 94993e3fafSRobert Mustacchi * the max that'll be less than any memory allocation we ever throw at 95993e3fafSRobert Mustacchi * it. 96993e3fafSRobert Mustacchi */ 97993e3fafSRobert Mustacchi attrp->dma_attr_minxfer = 0x1; 98993e3fafSRobert Mustacchi attrp->dma_attr_maxxfer = UINT32_MAX; 99993e3fafSRobert Mustacchi 100993e3fafSRobert Mustacchi /* 101993e3fafSRobert Mustacchi * This is determined by the caller. 102993e3fafSRobert Mustacchi */ 103993e3fafSRobert Mustacchi attrp->dma_attr_sgllen = sgl; 104993e3fafSRobert Mustacchi 105993e3fafSRobert Mustacchi /* 106993e3fafSRobert Mustacchi * The granularity describes the addressing granularity. e.g. can things 107993e3fafSRobert Mustacchi * ask for chunks in units of this number of bytes. For PCI this should 108993e3fafSRobert Mustacchi * always be one. 109993e3fafSRobert Mustacchi */ 110993e3fafSRobert Mustacchi attrp->dma_attr_granular = 1; 111993e3fafSRobert Mustacchi 112993e3fafSRobert Mustacchi if (DDI_FM_DMA_ERR_CAP(xhcip->xhci_fm_caps)) { 113993e3fafSRobert Mustacchi attrp->dma_attr_flags = DDI_DMA_FLAGERR; 114993e3fafSRobert Mustacchi } else { 115993e3fafSRobert Mustacchi attrp->dma_attr_flags = 0; 116993e3fafSRobert Mustacchi } 117993e3fafSRobert Mustacchi } 118993e3fafSRobert Mustacchi 119993e3fafSRobert Mustacchi /* 120993e3fafSRobert Mustacchi * This routine tries to create DMA attributes for normal allocations for data 121993e3fafSRobert Mustacchi * structures and the like. By default we use the same values as the transfer 122993e3fafSRobert Mustacchi * attributes, but have explicit comments about how they're different. 123993e3fafSRobert Mustacchi */ 124993e3fafSRobert Mustacchi void 125993e3fafSRobert Mustacchi xhci_dma_dma_attr(xhci_t *xhcip, ddi_dma_attr_t *attrp) 126993e3fafSRobert Mustacchi { 127993e3fafSRobert Mustacchi /* 128993e3fafSRobert Mustacchi * Note, we always use a single SGL for these DMA allocations as these 129993e3fafSRobert Mustacchi * are used for small data structures. 130993e3fafSRobert Mustacchi */ 131993e3fafSRobert Mustacchi xhci_dma_transfer_attr(xhcip, attrp, XHCI_DEF_DMA_SGL); 132993e3fafSRobert Mustacchi 133993e3fafSRobert Mustacchi /* 134993e3fafSRobert Mustacchi * The maximum size of any of these structures is 4k as opposed to the 135993e3fafSRobert Mustacchi * 64K max described above. Similarly the boundary requirement is 136993e3fafSRobert Mustacchi * reduced to 4k. 137993e3fafSRobert Mustacchi */ 138993e3fafSRobert Mustacchi attrp->dma_attr_count_max = xhcip->xhci_caps.xcap_pagesize; 139993e3fafSRobert Mustacchi attrp->dma_attr_maxxfer = xhcip->xhci_caps.xcap_pagesize; 140993e3fafSRobert Mustacchi attrp->dma_attr_seg = xhcip->xhci_caps.xcap_pagesize - 1; 141993e3fafSRobert Mustacchi } 142993e3fafSRobert Mustacchi 143993e3fafSRobert Mustacchi /* 144993e3fafSRobert Mustacchi * Fill in attributes for a scratchpad entry. The scratchpad entries are 145993e3fafSRobert Mustacchi * somewhat different in so far as they are closest to a normal DMA attribute, 146993e3fafSRobert Mustacchi * except they have stricter alignments, needing to be page sized. 147993e3fafSRobert Mustacchi * 148993e3fafSRobert Mustacchi * In addition, because we never access this memory ourselves, we can just mark 149993e3fafSRobert Mustacchi * it all as relaxed ordering. 150993e3fafSRobert Mustacchi */ 151993e3fafSRobert Mustacchi void 152993e3fafSRobert Mustacchi xhci_dma_scratchpad_attr(xhci_t *xhcip, ddi_dma_attr_t *attrp) 153993e3fafSRobert Mustacchi { 154993e3fafSRobert Mustacchi xhci_dma_dma_attr(xhcip, attrp); 155993e3fafSRobert Mustacchi attrp->dma_attr_align = xhcip->xhci_caps.xcap_pagesize; 156993e3fafSRobert Mustacchi attrp->dma_attr_flags |= DDI_DMA_RELAXED_ORDERING; 157993e3fafSRobert Mustacchi } 158993e3fafSRobert Mustacchi 159993e3fafSRobert Mustacchi /* 160993e3fafSRobert Mustacchi * This should be used for the simple case of a single SGL entry, which is the 161993e3fafSRobert Mustacchi * vast majority of the non-transfer allocations. 162993e3fafSRobert Mustacchi */ 163993e3fafSRobert Mustacchi uint64_t 164993e3fafSRobert Mustacchi xhci_dma_pa(xhci_dma_buffer_t *xdb) 165993e3fafSRobert Mustacchi { 166993e3fafSRobert Mustacchi ASSERT(xdb->xdb_ncookies == 1); 167993e3fafSRobert Mustacchi return (xdb->xdb_cookies[0].dmac_laddress); 168993e3fafSRobert Mustacchi } 169993e3fafSRobert Mustacchi 170993e3fafSRobert Mustacchi void 171993e3fafSRobert Mustacchi xhci_dma_free(xhci_dma_buffer_t *xdb) 172993e3fafSRobert Mustacchi { 173993e3fafSRobert Mustacchi if (xdb->xdb_ncookies != 0) { 174993e3fafSRobert Mustacchi VERIFY(xdb->xdb_dma_handle != NULL); 175993e3fafSRobert Mustacchi (void) ddi_dma_unbind_handle(xdb->xdb_dma_handle); 176993e3fafSRobert Mustacchi xdb->xdb_ncookies = 0; 177993e3fafSRobert Mustacchi bzero(xdb->xdb_cookies, sizeof (ddi_dma_cookie_t) * 178993e3fafSRobert Mustacchi XHCI_TRANSFER_DMA_SGL); 179993e3fafSRobert Mustacchi xdb->xdb_len = 0; 180993e3fafSRobert Mustacchi } 181993e3fafSRobert Mustacchi 182993e3fafSRobert Mustacchi if (xdb->xdb_acc_handle != NULL) { 183993e3fafSRobert Mustacchi ddi_dma_mem_free(&xdb->xdb_acc_handle); 184993e3fafSRobert Mustacchi xdb->xdb_acc_handle = NULL; 185993e3fafSRobert Mustacchi xdb->xdb_va = NULL; 186993e3fafSRobert Mustacchi } 187993e3fafSRobert Mustacchi 188993e3fafSRobert Mustacchi if (xdb->xdb_dma_handle != NULL) { 189993e3fafSRobert Mustacchi ddi_dma_free_handle(&xdb->xdb_dma_handle); 190993e3fafSRobert Mustacchi xdb->xdb_dma_handle = NULL; 191993e3fafSRobert Mustacchi } 192993e3fafSRobert Mustacchi 193993e3fafSRobert Mustacchi ASSERT(xdb->xdb_va == NULL); 194993e3fafSRobert Mustacchi ASSERT(xdb->xdb_ncookies == 0); 195993e3fafSRobert Mustacchi ASSERT(xdb->xdb_cookies[0].dmac_laddress == 0); 196993e3fafSRobert Mustacchi ASSERT(xdb->xdb_len == 0); 197993e3fafSRobert Mustacchi } 198993e3fafSRobert Mustacchi 199993e3fafSRobert Mustacchi boolean_t 200993e3fafSRobert Mustacchi xhci_dma_alloc(xhci_t *xhcip, xhci_dma_buffer_t *xdb, 201993e3fafSRobert Mustacchi ddi_dma_attr_t *attrp, ddi_device_acc_attr_t *accp, boolean_t zero, 202993e3fafSRobert Mustacchi size_t size, boolean_t wait) 203993e3fafSRobert Mustacchi { 204993e3fafSRobert Mustacchi int ret, i; 205993e3fafSRobert Mustacchi uint_t flags = DDI_DMA_CONSISTENT; 206993e3fafSRobert Mustacchi size_t len; 207993e3fafSRobert Mustacchi ddi_dma_cookie_t cookie; 208993e3fafSRobert Mustacchi uint_t ncookies; 209993e3fafSRobert Mustacchi int (*memcb)(caddr_t); 210993e3fafSRobert Mustacchi 211993e3fafSRobert Mustacchi if (wait == B_TRUE) { 212993e3fafSRobert Mustacchi memcb = DDI_DMA_SLEEP; 213993e3fafSRobert Mustacchi } else { 214993e3fafSRobert Mustacchi memcb = DDI_DMA_DONTWAIT; 215993e3fafSRobert Mustacchi } 216993e3fafSRobert Mustacchi 217993e3fafSRobert Mustacchi ret = ddi_dma_alloc_handle(xhcip->xhci_dip, attrp, memcb, NULL, 218993e3fafSRobert Mustacchi &xdb->xdb_dma_handle); 219993e3fafSRobert Mustacchi if (ret != 0) { 220993e3fafSRobert Mustacchi xhci_log(xhcip, "!failed to allocate DMA handle: %d", ret); 221993e3fafSRobert Mustacchi xdb->xdb_dma_handle = NULL; 222993e3fafSRobert Mustacchi return (B_FALSE); 223993e3fafSRobert Mustacchi } 224993e3fafSRobert Mustacchi 225993e3fafSRobert Mustacchi ret = ddi_dma_mem_alloc(xdb->xdb_dma_handle, size, accp, flags, memcb, 226993e3fafSRobert Mustacchi NULL, &xdb->xdb_va, &len, &xdb->xdb_acc_handle); 227993e3fafSRobert Mustacchi if (ret != DDI_SUCCESS) { 228993e3fafSRobert Mustacchi xhci_log(xhcip, "!failed to allocate DMA memory: %d", ret); 229993e3fafSRobert Mustacchi xdb->xdb_va = NULL; 230993e3fafSRobert Mustacchi xdb->xdb_acc_handle = NULL; 231993e3fafSRobert Mustacchi xhci_dma_free(xdb); 232993e3fafSRobert Mustacchi return (B_FALSE); 233993e3fafSRobert Mustacchi } 234993e3fafSRobert Mustacchi 235993e3fafSRobert Mustacchi if (zero == B_TRUE) 236993e3fafSRobert Mustacchi bzero(xdb->xdb_va, len); 237993e3fafSRobert Mustacchi 238993e3fafSRobert Mustacchi ret = ddi_dma_addr_bind_handle(xdb->xdb_dma_handle, NULL, 239993e3fafSRobert Mustacchi xdb->xdb_va, len, DDI_DMA_RDWR | flags, memcb, NULL, &cookie, 240993e3fafSRobert Mustacchi &ncookies); 241993e3fafSRobert Mustacchi if (ret != 0) { 242993e3fafSRobert Mustacchi xhci_log(xhcip, "!failed to bind DMA memory: %d", ret); 243993e3fafSRobert Mustacchi xhci_dma_free(xdb); 244993e3fafSRobert Mustacchi return (B_FALSE); 245993e3fafSRobert Mustacchi } 246993e3fafSRobert Mustacchi 247993e3fafSRobert Mustacchi /* 248993e3fafSRobert Mustacchi * Note we explicitly store the logical length of this allocation. The 249993e3fafSRobert Mustacchi * physical length is available via the cookies. 250993e3fafSRobert Mustacchi */ 251993e3fafSRobert Mustacchi xdb->xdb_len = size; 252993e3fafSRobert Mustacchi xdb->xdb_ncookies = ncookies; 253993e3fafSRobert Mustacchi xdb->xdb_cookies[0] = cookie; 254993e3fafSRobert Mustacchi for (i = 1; i < ncookies; i++) { 255993e3fafSRobert Mustacchi ddi_dma_nextcookie(xdb->xdb_dma_handle, &xdb->xdb_cookies[i]); 256993e3fafSRobert Mustacchi } 257993e3fafSRobert Mustacchi 258993e3fafSRobert Mustacchi 259993e3fafSRobert Mustacchi return (B_TRUE); 260993e3fafSRobert Mustacchi } 261993e3fafSRobert Mustacchi 262993e3fafSRobert Mustacchi void 263993e3fafSRobert Mustacchi xhci_transfer_free(xhci_t *xhcip, xhci_transfer_t *xt) 264993e3fafSRobert Mustacchi { 265993e3fafSRobert Mustacchi if (xt == NULL) 266993e3fafSRobert Mustacchi return; 267993e3fafSRobert Mustacchi 268993e3fafSRobert Mustacchi VERIFY(xhcip != NULL); 269993e3fafSRobert Mustacchi xhci_dma_free(&xt->xt_buffer); 270993e3fafSRobert Mustacchi if (xt->xt_isoc != NULL) { 271*2aba3acdSRobert Mustacchi ASSERT3U(xt->xt_ntrbs, >, 0); 272993e3fafSRobert Mustacchi kmem_free(xt->xt_isoc, sizeof (usb_isoc_pkt_descr_t) * 273993e3fafSRobert Mustacchi xt->xt_ntrbs); 274993e3fafSRobert Mustacchi xt->xt_isoc = NULL; 275993e3fafSRobert Mustacchi } 276993e3fafSRobert Mustacchi if (xt->xt_trbs != NULL) { 277*2aba3acdSRobert Mustacchi ASSERT3U(xt->xt_ntrbs, >, 0); 278993e3fafSRobert Mustacchi kmem_free(xt->xt_trbs, sizeof (xhci_trb_t) * xt->xt_ntrbs); 279993e3fafSRobert Mustacchi xt->xt_trbs = NULL; 280993e3fafSRobert Mustacchi } 281*2aba3acdSRobert Mustacchi if (xt->xt_trbs_pa != NULL) { 282*2aba3acdSRobert Mustacchi ASSERT3U(xt->xt_ntrbs, >, 0); 283*2aba3acdSRobert Mustacchi kmem_free(xt->xt_trbs_pa, sizeof (uint64_t) * xt->xt_ntrbs); 284*2aba3acdSRobert Mustacchi xt->xt_trbs_pa = NULL; 285*2aba3acdSRobert Mustacchi } 286993e3fafSRobert Mustacchi kmem_free(xt, sizeof (xhci_transfer_t)); 287993e3fafSRobert Mustacchi } 288993e3fafSRobert Mustacchi 289993e3fafSRobert Mustacchi xhci_transfer_t * 290*2aba3acdSRobert Mustacchi xhci_transfer_alloc(xhci_t *xhcip, xhci_endpoint_t *xep, size_t size, 291*2aba3acdSRobert Mustacchi uint_t trbs, int usb_flags) 292993e3fafSRobert Mustacchi { 293993e3fafSRobert Mustacchi int kmflags; 294993e3fafSRobert Mustacchi boolean_t dmawait; 295993e3fafSRobert Mustacchi xhci_transfer_t *xt; 296993e3fafSRobert Mustacchi ddi_device_acc_attr_t acc; 297993e3fafSRobert Mustacchi ddi_dma_attr_t attr; 298993e3fafSRobert Mustacchi 299993e3fafSRobert Mustacchi if (usb_flags & USB_FLAGS_SLEEP) { 300993e3fafSRobert Mustacchi kmflags = KM_SLEEP; 301993e3fafSRobert Mustacchi dmawait = B_TRUE; 302993e3fafSRobert Mustacchi } else { 303993e3fafSRobert Mustacchi kmflags = KM_NOSLEEP; 304993e3fafSRobert Mustacchi dmawait = B_FALSE; 305993e3fafSRobert Mustacchi } 306993e3fafSRobert Mustacchi 307993e3fafSRobert Mustacchi xt = kmem_zalloc(sizeof (xhci_transfer_t), kmflags); 308993e3fafSRobert Mustacchi if (xt == NULL) 309993e3fafSRobert Mustacchi return (NULL); 310993e3fafSRobert Mustacchi 311993e3fafSRobert Mustacchi if (size != 0) { 312993e3fafSRobert Mustacchi int sgl = XHCI_DEF_DMA_SGL; 313993e3fafSRobert Mustacchi 314993e3fafSRobert Mustacchi /* 315993e3fafSRobert Mustacchi * For BULK transfers, we always increase the number of SGL 316993e3fafSRobert Mustacchi * entries that we support to make things easier for the kernel. 317993e3fafSRobert Mustacchi * However, for control transfers, we currently opt to keep 318993e3fafSRobert Mustacchi * things a bit simpler and use our default of one SGL. There's 319993e3fafSRobert Mustacchi * no good technical reason for this, rather it just keeps 320993e3fafSRobert Mustacchi * things a bit easier. 321993e3fafSRobert Mustacchi * 322993e3fafSRobert Mustacchi * To simplify things, we don't use additional SGL entries for 323993e3fafSRobert Mustacchi * ISOC transfers. While this isn't the best, it isn't too far 324993e3fafSRobert Mustacchi * off from what ehci and co. have done before. If this becomes 325993e3fafSRobert Mustacchi * a technical issue, it's certainly possible to increase the 326993e3fafSRobert Mustacchi * SGL entry count. 327*2aba3acdSRobert Mustacchi * 328*2aba3acdSRobert Mustacchi * When we use the larger SGL count, we change our strategy for 329*2aba3acdSRobert Mustacchi * being notified. In such a case we will opt to use an event 330*2aba3acdSRobert Mustacchi * data packet. This helps deal with cases where some 331*2aba3acdSRobert Mustacchi * controllers don't properly generate events for the last entry 332*2aba3acdSRobert Mustacchi * in a TD with IOC when IOSP is set. 333993e3fafSRobert Mustacchi */ 334*2aba3acdSRobert Mustacchi if (xep->xep_type == USB_EP_ATTR_BULK) { 335993e3fafSRobert Mustacchi sgl = XHCI_TRANSFER_DMA_SGL; 336*2aba3acdSRobert Mustacchi trbs++; 337*2aba3acdSRobert Mustacchi } 338993e3fafSRobert Mustacchi 339993e3fafSRobert Mustacchi xhci_dma_acc_attr(xhcip, &acc); 340993e3fafSRobert Mustacchi xhci_dma_transfer_attr(xhcip, &attr, sgl); 341993e3fafSRobert Mustacchi if (xhci_dma_alloc(xhcip, &xt->xt_buffer, &attr, &acc, B_FALSE, 342993e3fafSRobert Mustacchi size, dmawait) == B_FALSE) { 343993e3fafSRobert Mustacchi kmem_free(xt, sizeof (xhci_transfer_t)); 344993e3fafSRobert Mustacchi return (NULL); 345993e3fafSRobert Mustacchi } 346993e3fafSRobert Mustacchi 347993e3fafSRobert Mustacchi /* 348993e3fafSRobert Mustacchi * ISOC transfers are a bit special and don't need additional 349993e3fafSRobert Mustacchi * TRBs for data. 350993e3fafSRobert Mustacchi */ 351993e3fafSRobert Mustacchi if (xep->xep_type != USB_EP_ATTR_ISOCH) 352993e3fafSRobert Mustacchi trbs += xt->xt_buffer.xdb_ncookies; 353993e3fafSRobert Mustacchi } 354993e3fafSRobert Mustacchi 355993e3fafSRobert Mustacchi xt->xt_trbs = kmem_zalloc(sizeof (xhci_trb_t) * trbs, kmflags); 356993e3fafSRobert Mustacchi if (xt->xt_trbs == NULL) { 357993e3fafSRobert Mustacchi xhci_dma_free(&xt->xt_buffer); 358993e3fafSRobert Mustacchi kmem_free(xt, sizeof (xhci_transfer_t)); 359993e3fafSRobert Mustacchi return (NULL); 360993e3fafSRobert Mustacchi } 361993e3fafSRobert Mustacchi 362*2aba3acdSRobert Mustacchi xt->xt_trbs_pa = kmem_zalloc(sizeof (uint64_t) * trbs, kmflags); 363*2aba3acdSRobert Mustacchi if (xt->xt_trbs_pa == NULL) { 364*2aba3acdSRobert Mustacchi kmem_free(xt->xt_trbs, sizeof (xhci_trb_t) * trbs); 365*2aba3acdSRobert Mustacchi xhci_dma_free(&xt->xt_buffer); 366*2aba3acdSRobert Mustacchi kmem_free(xt, sizeof (xhci_transfer_t)); 367*2aba3acdSRobert Mustacchi return (NULL); 368*2aba3acdSRobert Mustacchi } 369*2aba3acdSRobert Mustacchi 370993e3fafSRobert Mustacchi /* 371993e3fafSRobert Mustacchi * For ISOCH transfers, we need to also allocate the results data. 372993e3fafSRobert Mustacchi */ 373993e3fafSRobert Mustacchi if (xep->xep_type == USB_EP_ATTR_ISOCH) { 374993e3fafSRobert Mustacchi xt->xt_isoc = kmem_zalloc(sizeof (usb_isoc_pkt_descr_t) * trbs, 375993e3fafSRobert Mustacchi kmflags); 376993e3fafSRobert Mustacchi if (xt->xt_isoc == NULL) { 377*2aba3acdSRobert Mustacchi kmem_free(xt->xt_trbs_pa, sizeof (uint64_t) * trbs); 378993e3fafSRobert Mustacchi kmem_free(xt->xt_trbs, sizeof (xhci_trb_t) * trbs); 379993e3fafSRobert Mustacchi xhci_dma_free(&xt->xt_buffer); 380993e3fafSRobert Mustacchi kmem_free(xt, sizeof (xhci_transfer_t)); 381993e3fafSRobert Mustacchi return (NULL); 382993e3fafSRobert Mustacchi } 383993e3fafSRobert Mustacchi } 384993e3fafSRobert Mustacchi 385993e3fafSRobert Mustacchi xt->xt_ntrbs = trbs; 386993e3fafSRobert Mustacchi xt->xt_cr = USB_CR_OK; 387993e3fafSRobert Mustacchi 388993e3fafSRobert Mustacchi return (xt); 389993e3fafSRobert Mustacchi } 390993e3fafSRobert Mustacchi 391993e3fafSRobert Mustacchi /* 392993e3fafSRobert Mustacchi * Abstract the notion of copying out to handle the case of multiple DMA 393993e3fafSRobert Mustacchi * cookies. If tobuf is true, we are copying to the kernel provided buffer, 394993e3fafSRobert Mustacchi * otherwise we're copying into the DMA memory. 395993e3fafSRobert Mustacchi */ 396993e3fafSRobert Mustacchi void 397993e3fafSRobert Mustacchi xhci_transfer_copy(xhci_transfer_t *xt, void *buf, size_t len, 398993e3fafSRobert Mustacchi boolean_t tobuf) 399993e3fafSRobert Mustacchi { 400993e3fafSRobert Mustacchi void *dmabuf = xt->xt_buffer.xdb_va; 401993e3fafSRobert Mustacchi if (tobuf == B_TRUE) 402993e3fafSRobert Mustacchi bcopy(dmabuf, buf, len); 403993e3fafSRobert Mustacchi else 404993e3fafSRobert Mustacchi bcopy(buf, dmabuf, len); 405993e3fafSRobert Mustacchi } 406993e3fafSRobert Mustacchi 407993e3fafSRobert Mustacchi int 408993e3fafSRobert Mustacchi xhci_transfer_sync(xhci_t *xhcip, xhci_transfer_t *xt, uint_t type) 409993e3fafSRobert Mustacchi { 410993e3fafSRobert Mustacchi XHCI_DMA_SYNC(xt->xt_buffer, type); 411993e3fafSRobert Mustacchi return (xhci_check_dma_handle(xhcip, &xt->xt_buffer)); 412993e3fafSRobert Mustacchi } 413993e3fafSRobert Mustacchi 414993e3fafSRobert Mustacchi /* 415993e3fafSRobert Mustacchi * We're required to try and inform the xHCI controller about the number of data 416993e3fafSRobert Mustacchi * packets that are required. The algorithm to use is described in xHCI 1.1 / 417993e3fafSRobert Mustacchi * 4.11.2.4. While it might be tempting to just try and calculate the number of 418993e3fafSRobert Mustacchi * packets based on simple rounding of the remaining number of bytes, that 419993e3fafSRobert Mustacchi * misses a critical problem -- DMA boundaries may cause us to need additional 420993e3fafSRobert Mustacchi * packets that are missed initially. Consider a transfer made up of four 421993e3fafSRobert Mustacchi * different DMA buffers sized in bytes: 4096, 4096, 256, 256, with a 512 byte 422993e3fafSRobert Mustacchi * packet size. 423993e3fafSRobert Mustacchi * 424993e3fafSRobert Mustacchi * Remain 4608 512 256 0 425993e3fafSRobert Mustacchi * Bytes 4096 4096 256 256 426993e3fafSRobert Mustacchi * Naive TD 9 1 1 0 427993e3fafSRobert Mustacchi * Act TD 10 2 1 0 428993e3fafSRobert Mustacchi * 429993e3fafSRobert Mustacchi * This means that the only safe way forward here is to work backwards and see 430993e3fafSRobert Mustacchi * how many we need to work up to this point. 431993e3fafSRobert Mustacchi */ 432*2aba3acdSRobert Mustacchi static uint_t 433993e3fafSRobert Mustacchi xhci_transfer_get_tdsize(xhci_transfer_t *xt, uint_t off, uint_t mps) 434993e3fafSRobert Mustacchi { 435993e3fafSRobert Mustacchi int i; 436993e3fafSRobert Mustacchi uint_t npkt = 0; 437993e3fafSRobert Mustacchi 438993e3fafSRobert Mustacchi /* 439993e3fafSRobert Mustacchi * There are always zero packets for the last TRB. 440993e3fafSRobert Mustacchi */ 441993e3fafSRobert Mustacchi ASSERT(xt->xt_buffer.xdb_ncookies > 0); 442993e3fafSRobert Mustacchi for (i = xt->xt_buffer.xdb_ncookies - 1; i > off; i--) { 443*2aba3acdSRobert Mustacchi size_t len = roundup(xt->xt_buffer.xdb_cookies[i].dmac_size, 444*2aba3acdSRobert Mustacchi mps); 445993e3fafSRobert Mustacchi npkt += len / mps; 446993e3fafSRobert Mustacchi } 447993e3fafSRobert Mustacchi 448*2aba3acdSRobert Mustacchi /* 449*2aba3acdSRobert Mustacchi * Make sure to clamp this value otherwise we risk truncation. 450*2aba3acdSRobert Mustacchi */ 451*2aba3acdSRobert Mustacchi if (npkt >= XHCI_MAX_TDSIZE) 452*2aba3acdSRobert Mustacchi return (XHCI_MAX_TDSIZE); 453*2aba3acdSRobert Mustacchi 454993e3fafSRobert Mustacchi return (npkt); 455993e3fafSRobert Mustacchi } 456993e3fafSRobert Mustacchi 457993e3fafSRobert Mustacchi void 458993e3fafSRobert Mustacchi xhci_transfer_trb_fill_data(xhci_endpoint_t *xep, xhci_transfer_t *xt, int off, 459993e3fafSRobert Mustacchi boolean_t in) 460993e3fafSRobert Mustacchi { 461993e3fafSRobert Mustacchi uint_t mps, tdsize, flags; 462993e3fafSRobert Mustacchi int i; 463993e3fafSRobert Mustacchi 464993e3fafSRobert Mustacchi VERIFY(xt->xt_buffer.xdb_ncookies > 0); 465993e3fafSRobert Mustacchi VERIFY(xep->xep_pipe != NULL); 466993e3fafSRobert Mustacchi VERIFY(off + xt->xt_buffer.xdb_ncookies <= xt->xt_ntrbs); 467993e3fafSRobert Mustacchi mps = xep->xep_pipe->p_ep.wMaxPacketSize; 468993e3fafSRobert Mustacchi 469*2aba3acdSRobert Mustacchi if (in == B_TRUE) { 470*2aba3acdSRobert Mustacchi xt->xt_data_tohost = B_TRUE; 471*2aba3acdSRobert Mustacchi } 472*2aba3acdSRobert Mustacchi 473*2aba3acdSRobert Mustacchi /* 474*2aba3acdSRobert Mustacchi * We assume that if we have a non-bulk endpoint, then we should only 475*2aba3acdSRobert Mustacchi * have a single cookie. This falls out from the default SGL length that 476*2aba3acdSRobert Mustacchi * we use for these other device types. 477*2aba3acdSRobert Mustacchi */ 478*2aba3acdSRobert Mustacchi if (xep->xep_type != USB_EP_ATTR_BULK) { 479*2aba3acdSRobert Mustacchi VERIFY3U(xt->xt_buffer.xdb_ncookies, ==, 1); 480*2aba3acdSRobert Mustacchi } 481*2aba3acdSRobert Mustacchi 482993e3fafSRobert Mustacchi for (i = 0; i < xt->xt_buffer.xdb_ncookies; i++) { 483993e3fafSRobert Mustacchi uint64_t pa, dmasz; 484993e3fafSRobert Mustacchi 485993e3fafSRobert Mustacchi pa = xt->xt_buffer.xdb_cookies[i].dmac_laddress; 486993e3fafSRobert Mustacchi dmasz = xt->xt_buffer.xdb_cookies[i].dmac_size; 487993e3fafSRobert Mustacchi 488993e3fafSRobert Mustacchi tdsize = xhci_transfer_get_tdsize(xt, i, mps); 489993e3fafSRobert Mustacchi 490993e3fafSRobert Mustacchi flags = XHCI_TRB_TYPE_NORMAL; 491993e3fafSRobert Mustacchi if (i == 0 && xep->xep_type == USB_EP_ATTR_CONTROL) { 492993e3fafSRobert Mustacchi flags = XHCI_TRB_TYPE_DATA; 493993e3fafSRobert Mustacchi if (in == B_TRUE) 494993e3fafSRobert Mustacchi flags |= XHCI_TRB_DIR_IN; 495993e3fafSRobert Mustacchi } 496993e3fafSRobert Mustacchi 497993e3fafSRobert Mustacchi /* 498*2aba3acdSRobert Mustacchi * If we have more than one cookie, then we need to set chaining 499*2aba3acdSRobert Mustacchi * on every TRB and the last TRB will turn into an event data 500*2aba3acdSRobert Mustacchi * TRB. If we only have a single TRB, then we just set interrupt 501*2aba3acdSRobert Mustacchi * on completion (IOC). There's no need to specifically set 502*2aba3acdSRobert Mustacchi * interrupt on short packet (IOSP) in that case, as we'll 503*2aba3acdSRobert Mustacchi * always get the event notification. We still need the chain 504*2aba3acdSRobert Mustacchi * bit set on the last packet, so we can chain into the event 505*2aba3acdSRobert Mustacchi * data. Even if all the data on a bulk endpoint (the only 506*2aba3acdSRobert Mustacchi * endpoint type that uses chaining today) has only one cookie, 507*2aba3acdSRobert Mustacchi * then we'll still schedule an event data block. 508993e3fafSRobert Mustacchi */ 509*2aba3acdSRobert Mustacchi if (xep->xep_type == USB_EP_ATTR_BULK || 510*2aba3acdSRobert Mustacchi xt->xt_buffer.xdb_ncookies > 1) { 511993e3fafSRobert Mustacchi flags |= XHCI_TRB_CHAIN; 512993e3fafSRobert Mustacchi } 513993e3fafSRobert Mustacchi 514993e3fafSRobert Mustacchi /* 515*2aba3acdSRobert Mustacchi * What we set for the last TRB depends on the type of the 516*2aba3acdSRobert Mustacchi * endpoint. If it's a bulk endpoint, then we have to set 517*2aba3acdSRobert Mustacchi * evaluate next trb (ENT) so we successfully process the event 518*2aba3acdSRobert Mustacchi * data TRB we'll set up. Otherwise, we need to make sure that 519*2aba3acdSRobert Mustacchi * we set interrupt on completion, so we get the event. However, 520*2aba3acdSRobert Mustacchi * we don't set the event on control endpoints, as the status 521*2aba3acdSRobert Mustacchi * stage TD will be the one where we get the event. But, we do 522*2aba3acdSRobert Mustacchi * still need an interrupt on short packet, because technically 523*2aba3acdSRobert Mustacchi * the status stage is in its own TD. 524993e3fafSRobert Mustacchi */ 525*2aba3acdSRobert Mustacchi if (i + 1 == xt->xt_buffer.xdb_ncookies) { 526*2aba3acdSRobert Mustacchi switch (xep->xep_type) { 527*2aba3acdSRobert Mustacchi case USB_EP_ATTR_BULK: 528*2aba3acdSRobert Mustacchi flags |= XHCI_TRB_ENT; 529*2aba3acdSRobert Mustacchi break; 530*2aba3acdSRobert Mustacchi case USB_EP_ATTR_CONTROL: 531*2aba3acdSRobert Mustacchi flags |= XHCI_TRB_ISP; 532*2aba3acdSRobert Mustacchi break; 533*2aba3acdSRobert Mustacchi default: 534993e3fafSRobert Mustacchi flags |= XHCI_TRB_IOC; 535*2aba3acdSRobert Mustacchi break; 536*2aba3acdSRobert Mustacchi } 537993e3fafSRobert Mustacchi } 538993e3fafSRobert Mustacchi 539993e3fafSRobert Mustacchi xt->xt_trbs[off + i].trb_addr = LE_64(pa); 540993e3fafSRobert Mustacchi xt->xt_trbs[off + i].trb_status = LE_32(XHCI_TRB_LEN(dmasz) | 541993e3fafSRobert Mustacchi XHCI_TRB_TDREM(tdsize) | XHCI_TRB_INTR(0)); 542993e3fafSRobert Mustacchi xt->xt_trbs[off + i].trb_flags = LE_32(flags); 543993e3fafSRobert Mustacchi } 544*2aba3acdSRobert Mustacchi 545*2aba3acdSRobert Mustacchi /* 546*2aba3acdSRobert Mustacchi * The last TRB in any bulk transfer is the Event Data TRB. 547*2aba3acdSRobert Mustacchi */ 548*2aba3acdSRobert Mustacchi if (xep->xep_type == USB_EP_ATTR_BULK) { 549*2aba3acdSRobert Mustacchi VERIFY(off + xt->xt_buffer.xdb_ncookies + 1 <= xt->xt_ntrbs); 550*2aba3acdSRobert Mustacchi xt->xt_trbs[off + i].trb_addr = LE_64((uintptr_t)xt); 551*2aba3acdSRobert Mustacchi xt->xt_trbs[off + i].trb_status = LE_32(XHCI_TRB_INTR(0)); 552*2aba3acdSRobert Mustacchi xt->xt_trbs[off + i].trb_flags = LE_32(XHCI_TRB_TYPE_EVENT | 553*2aba3acdSRobert Mustacchi XHCI_TRB_IOC); 554*2aba3acdSRobert Mustacchi } 555993e3fafSRobert Mustacchi } 556993e3fafSRobert Mustacchi 557993e3fafSRobert Mustacchi /* 558993e3fafSRobert Mustacchi * These are utility functions for isochronus transfers to help calculate the 559993e3fafSRobert Mustacchi * transfer burst count (TBC) and transfer last burst packet count (TLPBC) 560993e3fafSRobert Mustacchi * entries for an isochronus entry. See xHCI 1.1 / 4.11.2.3 for how to calculate 561993e3fafSRobert Mustacchi * them. 562993e3fafSRobert Mustacchi */ 563993e3fafSRobert Mustacchi void 564993e3fafSRobert Mustacchi xhci_transfer_calculate_isoc(xhci_device_t *xd, xhci_endpoint_t *xep, 565993e3fafSRobert Mustacchi uint_t trb_len, uint_t *tbc, uint_t *tlbpc) 566993e3fafSRobert Mustacchi { 567993e3fafSRobert Mustacchi uint_t mps, tdpc, burst; 568993e3fafSRobert Mustacchi 569993e3fafSRobert Mustacchi /* 570993e3fafSRobert Mustacchi * Even if we're asked to send no data, that actually requires the 571993e3fafSRobert Mustacchi * equivalent of sending one byte of data. 572993e3fafSRobert Mustacchi */ 573993e3fafSRobert Mustacchi if (trb_len == 0) 574993e3fafSRobert Mustacchi trb_len = 1; 575993e3fafSRobert Mustacchi 576993e3fafSRobert Mustacchi mps = XHCI_EPCTX_GET_MPS(xd->xd_endout[xep->xep_num]->xec_info2); 577993e3fafSRobert Mustacchi burst = XHCI_EPCTX_GET_MAXB(xd->xd_endout[xep->xep_num]->xec_info2); 578993e3fafSRobert Mustacchi 579993e3fafSRobert Mustacchi /* 580993e3fafSRobert Mustacchi * This is supposed to correspond to the Transfer Descriptor Packet 581993e3fafSRobert Mustacchi * Count from xHCI 1.1 / 4.14.1. 582993e3fafSRobert Mustacchi */ 583993e3fafSRobert Mustacchi tdpc = howmany(trb_len, mps); 584993e3fafSRobert Mustacchi *tbc = howmany(tdpc, burst + 1) - 1; 585993e3fafSRobert Mustacchi 586993e3fafSRobert Mustacchi if ((tdpc % (burst + 1)) == 0) 587993e3fafSRobert Mustacchi *tlbpc = burst; 588993e3fafSRobert Mustacchi else 589993e3fafSRobert Mustacchi *tlbpc = (tdpc % (burst + 1)) - 1; 590993e3fafSRobert Mustacchi } 591