1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 /* 26 * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved. 27 */ 28 29 /* 30 * PCI nexus DVMA and DMA core routines: 31 * dma_map/dma_bind_handle implementation 32 * bypass and peer-to-peer support 33 * fast track DVMA space allocation 34 * runtime DVMA debug 35 */ 36 #include <sys/types.h> 37 #include <sys/kmem.h> 38 #include <sys/async.h> 39 #include <sys/sysmacros.h> 40 #include <sys/sunddi.h> 41 #include <sys/machsystm.h> /* lddphys() */ 42 #include <sys/ddi_impldefs.h> 43 #include <vm/hat.h> 44 #include <sys/pci/pci_obj.h> 45 46 /*LINTLIBRARY*/ 47 48 static void 49 pci_sc_pg_inv(dev_info_t *dip, sc_t *sc_p, ddi_dma_impl_t *mp, off_t off, 50 size_t len) 51 { 52 dvma_addr_t dvma_addr, pg_off; 53 volatile uint64_t *invl_va = sc_p->sc_invl_reg; 54 55 if (!len) 56 len = mp->dmai_size; 57 58 pg_off = mp->dmai_offset; /* start min */ 59 dvma_addr = MAX(off, pg_off); /* lo */ 60 pg_off += mp->dmai_size; /* end max */ 61 pg_off = MIN(off + len, pg_off); /* hi */ 62 if (dvma_addr >= pg_off) { /* lo >= hi ? */ 63 DEBUG4(DBG_SC, dip, "%x+%x out of window [%x,%x)\n", 64 off, len, mp->dmai_offset, 65 mp->dmai_offset + mp->dmai_size); 66 return; 67 } 68 69 len = pg_off - dvma_addr; /* sz = hi - lo */ 70 dvma_addr += mp->dmai_mapping; /* start addr */ 71 pg_off = dvma_addr & IOMMU_PAGE_OFFSET; /* offset in 1st pg */ 72 len = IOMMU_BTOPR(len + pg_off); /* # of pages */ 73 dvma_addr ^= pg_off; 74 75 DEBUG2(DBG_SC, dip, "addr=%x+%x pages: \n", dvma_addr, len); 76 for (; len; len--, dvma_addr += IOMMU_PAGE_SIZE) { 77 DEBUG1(DBG_SC|DBG_CONT, dip, " %x", dvma_addr); 78 *invl_va = (uint64_t)dvma_addr; 79 } 80 DEBUG0(DBG_SC|DBG_CONT, dip, "\n"); 81 } 82 83 static void 84 pci_dma_sync_flag_wait(ddi_dma_impl_t *mp, sc_t *sc_p, uint32_t onstack) 85 { 86 hrtime_t start_time; 87 uint64_t loops = 0; 88 uint64_t sync_flag_pa = SYNC_BUF_PA(mp); 89 uint64_t sync_reg_pa = sc_p->sc_sync_reg_pa; 90 uint8_t stack_buf[128]; 91 92 stack_buf[0] = DDI_SUCCESS; 93 94 /* check for handle specific sync flag */ 95 if (sync_flag_pa) 96 goto start; 97 98 sync_flag_pa = sc_p->sc_sync_flag_pa; 99 100 if (onstack) { 101 sync_flag_pa = va_to_pa(stack_buf); 102 sync_flag_pa += PCI_SYNC_FLAG_SIZE; 103 sync_flag_pa >>= PCI_SYNC_FLAG_SZSHIFT; 104 sync_flag_pa <<= PCI_SYNC_FLAG_SZSHIFT; 105 goto start; 106 } 107 stack_buf[0] |= PCI_SYNC_FLAG_LOCKED; 108 mutex_enter(&sc_p->sc_sync_mutex); 109 start: 110 ASSERT(!(sync_flag_pa & PCI_SYNC_FLAG_SIZE - 1)); 111 stdphys(sync_flag_pa, 0); /* reset sync flag to 0 */ 112 /* membar #LoadStore|#StoreStore */ 113 stdphysio(sync_reg_pa, sync_flag_pa); 114 start_time = gethrtime(); 115 116 for (; gethrtime() - start_time < pci_sync_buf_timeout; loops++) 117 if (lddphys(sync_flag_pa)) 118 goto done; 119 120 if (!lddphys(sync_flag_pa)) 121 stack_buf[0] |= PCI_SYNC_FLAG_FAILED; 122 done: 123 DEBUG3(DBG_SC|DBG_CONT, 0, "flag wait loops=%lu ticks=%lu status=%x\n", 124 loops, gethrtime() - start_time, stack_buf[0]); 125 126 if (stack_buf[0] & PCI_SYNC_FLAG_LOCKED) 127 mutex_exit(&sc_p->sc_sync_mutex); 128 129 if (stack_buf[0] & PCI_SYNC_FLAG_FAILED) 130 cmn_err(CE_PANIC, "%p pci dma sync %lx %lx timeout!", 131 mp, sync_flag_pa, loops); 132 } 133 134 /* 135 * Cache RW Before During After 136 * 137 * STREAMING read no/no pg/no ctx,pg/no 138 * STREAMING write no/no pg/yes ctx,pg/yes 139 * CONSISTENT read no/no yes,no/no yes,no/no 140 * CONSISTENT write no/no yes,yes/yes yes,yes/yes 141 * 142 * STREAMING read ctx,pg/no 143 * STREAMING write ctx,pg/yes 144 * CONSISTENT read yes,no/no 145 * CONSISTENT write yes,yes/yes 146 */ 147 int 148 pci_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 149 off_t off, size_t len, uint32_t sync_flag) 150 { 151 ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle; 152 int ret = ddi_get_instance(dip); 153 pci_t *pci_p = get_pci_soft_state(ret); 154 pbm_t *pbm_p = pci_p->pci_pbm_p; 155 uint32_t dev_flag = mp->dmai_rflags; 156 sc_t *sc_p; 157 158 DEBUG4(DBG_DMA_SYNC, dip, "%s%d flags=%x,%x\n", ddi_driver_name(rdip), 159 ddi_get_instance(rdip), dev_flag, sync_flag); 160 DEBUG4(DBG_SC, dip, "dmai_mapping=%x, dmai_sz=%x off=%x len=%x\n", 161 mp->dmai_mapping, mp->dmai_size, off, len); 162 DEBUG2(DBG_SC, dip, "mp=%p, ctx=%x\n", mp, MP2CTX(mp)); 163 164 if (!(mp->dmai_flags & DMAI_FLAGS_INUSE)) { 165 cmn_err(CE_WARN, "Unbound dma handle %p from %s%d", mp, 166 ddi_driver_name(rdip), ddi_get_instance(rdip)); 167 return (DDI_FAILURE); 168 } 169 170 if (mp->dmai_flags & DMAI_FLAGS_NOSYNC) 171 return (DDI_SUCCESS); 172 173 if (!(dev_flag & DDI_DMA_CONSISTENT)) 174 goto streaming; 175 176 if (sync_flag & PCI_DMA_SYNC_EXT) { 177 if (sync_flag & (PCI_DMA_SYNC_BEFORE | PCI_DMA_SYNC_POST) || 178 !(sync_flag & PCI_DMA_SYNC_WRITE)) 179 return (DDI_SUCCESS); 180 } else { 181 if (!(dev_flag & DDI_DMA_READ) || 182 ((sync_flag & PCI_DMA_SYNC_DDI_FLAGS) == 183 DDI_DMA_SYNC_FORDEV)) 184 return (DDI_SUCCESS); 185 } 186 187 pci_pbm_dma_sync(pbm_p, pbm_p->pbm_sync_ino); 188 return (DDI_SUCCESS); 189 190 streaming: 191 ASSERT(pci_stream_buf_exists && (pci_stream_buf_enable & 1 << ret)); 192 sc_p = pci_p->pci_sc_p; 193 ret = DDI_FAILURE; 194 195 if (sync_flag & PCI_DMA_SYNC_EXT) 196 goto ext; 197 198 if (mp->dmai_flags & DMAI_FLAGS_CONTEXT && pci_sc_use_contexts) 199 ret = pci_sc_ctx_inv(dip, sc_p, mp); 200 if (ret) 201 pci_sc_pg_inv(dip, sc_p, mp, off, len); 202 203 if ((dev_flag & DDI_DMA_READ) && 204 ((sync_flag & PCI_DMA_SYNC_DDI_FLAGS) != DDI_DMA_SYNC_FORDEV)) 205 goto wait; 206 207 return (DDI_SUCCESS); 208 ext: 209 if (sync_flag & PCI_DMA_SYNC_BEFORE) 210 return (DDI_SUCCESS); 211 if (sync_flag & PCI_DMA_SYNC_BAR) 212 goto wait_check; 213 if (sync_flag & PCI_DMA_SYNC_AFTER && 214 mp->dmai_flags & DMAI_FLAGS_CONTEXT && pci_sc_use_contexts) 215 ret = pci_sc_ctx_inv(dip, sc_p, mp); 216 if (ret) 217 pci_sc_pg_inv(dip, sc_p, mp, off, len); 218 wait_check: 219 if (sync_flag & PCI_DMA_SYNC_POST || !(sync_flag & PCI_DMA_SYNC_WRITE)) 220 return (DDI_SUCCESS); 221 wait: 222 pci_dma_sync_flag_wait(mp, sc_p, sync_flag & PCI_DMA_SYNC_PRIVATE); 223 return (DDI_SUCCESS); 224 } 225 226 int 227 pci_dma_handle_clean(dev_info_t *rdip, ddi_dma_handle_t h) 228 { 229 ddi_dma_impl_t *mp = (ddi_dma_impl_t *)h; 230 if ((mp->dmai_flags & DMAI_FLAGS_INUSE) == 0) 231 return (DDI_FAILURE); 232 mp->dmai_rflags |= DMP_NOSYNC; 233 mp->dmai_flags |= DMAI_FLAGS_NOSYNC; 234 return (DDI_SUCCESS); 235 } 236 237 /* 238 * pci_dma_allocmp - Allocate a pci dma implementation structure 239 * 240 * An extra ddi_dma_attr structure is bundled with the usual ddi_dma_impl 241 * to hold unmodified device limits. The ddi_dma_attr inside the 242 * ddi_dma_impl structure is augumented with system limits to enhance 243 * DVMA performance at runtime. The unaugumented device limits saved 244 * right after (accessed through the DEV_ATTR macro) is used 245 * strictly for peer-to-peer transfers which do not obey system limits. 246 * 247 * return: DDI_SUCCESS DDI_DMA_NORESOURCES 248 */ 249 ddi_dma_impl_t * 250 pci_dma_allocmp(dev_info_t *dip, dev_info_t *rdip, int (*waitfp)(caddr_t), 251 caddr_t arg) 252 { 253 ddi_dma_impl_t *mp; 254 int sleep = (waitfp == DDI_DMA_SLEEP) ? KM_SLEEP : KM_NOSLEEP; 255 256 /* Caution: we don't use zalloc to enhance performance! */ 257 if ((mp = kmem_alloc(sizeof (pci_dma_hdl_t), sleep)) == 0) { 258 DEBUG0(DBG_DMA_MAP, dip, "can't alloc dma_handle\n"); 259 if (waitfp != DDI_DMA_DONTWAIT) { 260 DEBUG0(DBG_DMA_MAP, dip, "alloc_mp kmem cb\n"); 261 ddi_set_callback(waitfp, arg, &pci_kmem_clid); 262 } 263 return (mp); 264 } 265 266 mp->dmai_rdip = rdip; 267 mp->dmai_flags = 0; 268 mp->dmai_pfnlst = NULL; 269 mp->dmai_winlst = NULL; 270 mp->dmai_ncookies = 0; 271 mp->dmai_curcookie = 0; 272 273 /* 274 * kmem_alloc debug: the following fields are not zero-ed 275 * mp->dmai_mapping = 0; 276 * mp->dmai_size = 0; 277 * mp->dmai_offset = 0; 278 * mp->dmai_minxfer = 0; 279 * mp->dmai_burstsizes = 0; 280 * mp->dmai_ndvmapages = 0; 281 * mp->dmai_pool/roffset = 0; 282 * mp->dmai_rflags = 0; 283 * mp->dmai_inuse/flags 284 * mp->dmai_nwin = 0; 285 * mp->dmai_winsize = 0; 286 * mp->dmai_nexus_private/tte = 0; 287 * mp->dmai_iopte/pfnlst 288 * mp->dmai_sbi/pfn0 = 0; 289 * mp->dmai_minfo/winlst/fdvma 290 * mp->dmai_rdip 291 * bzero(&mp->dmai_object, sizeof (ddi_dma_obj_t)); 292 * mp->dmai_cookie = 0; 293 */ 294 295 mp->dmai_attr.dma_attr_version = (uint_t)DMA_ATTR_VERSION; 296 mp->dmai_attr.dma_attr_flags = (uint_t)0; 297 mp->dmai_fault = 0; 298 mp->dmai_fault_check = NULL; 299 mp->dmai_fault_notify = NULL; 300 301 mp->dmai_error.err_ena = 0; 302 mp->dmai_error.err_status = DDI_FM_OK; 303 mp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 304 mp->dmai_error.err_ontrap = NULL; 305 mp->dmai_error.err_fep = NULL; 306 mp->dmai_error.err_cf = NULL; 307 ndi_fmc_insert(rdip, DMA_HANDLE, mp, NULL); 308 309 SYNC_BUF_PA(mp) = 0ull; 310 return (mp); 311 } 312 313 void 314 pci_dma_freemp(ddi_dma_impl_t *mp) 315 { 316 ndi_fmc_remove(mp->dmai_rdip, DMA_HANDLE, mp); 317 if (mp->dmai_ndvmapages > 1) 318 pci_dma_freepfn(mp); 319 if (mp->dmai_winlst) 320 pci_dma_freewin(mp); 321 kmem_free(mp, sizeof (pci_dma_hdl_t)); 322 } 323 324 void 325 pci_dma_freepfn(ddi_dma_impl_t *mp) 326 { 327 void *addr = mp->dmai_pfnlst; 328 ASSERT(!PCI_DMA_CANRELOC(mp)); 329 if (addr) { 330 size_t npages = mp->dmai_ndvmapages; 331 if (npages > 1) 332 kmem_free(addr, npages * sizeof (iopfn_t)); 333 mp->dmai_pfnlst = NULL; 334 } 335 mp->dmai_ndvmapages = 0; 336 } 337 338 /* 339 * pci_dma_lmts2hdl - alloate a ddi_dma_impl_t, validate practical limits 340 * and convert dmareq->dmar_limits to mp->dmai_attr 341 * 342 * ddi_dma_impl_t member modified input 343 * ------------------------------------------------------------------------ 344 * mp->dmai_minxfer - dev 345 * mp->dmai_burstsizes - dev 346 * mp->dmai_flags - no limit? peer-to-peer only? 347 * 348 * ddi_dma_attr member modified input 349 * ------------------------------------------------------------------------ 350 * mp->dmai_attr.dma_attr_addr_lo - dev lo, sys lo 351 * mp->dmai_attr.dma_attr_addr_hi - dev hi, sys hi 352 * mp->dmai_attr.dma_attr_count_max - dev count max, dev/sys lo/hi delta 353 * mp->dmai_attr.dma_attr_seg - 0 (no nocross restriction) 354 * mp->dmai_attr.dma_attr_align - 1 (no alignment restriction) 355 * 356 * The dlim_dmaspeed member of dmareq->dmar_limits is ignored. 357 */ 358 ddi_dma_impl_t * 359 pci_dma_lmts2hdl(dev_info_t *dip, dev_info_t *rdip, iommu_t *iommu_p, 360 ddi_dma_req_t *dmareq) 361 { 362 ddi_dma_impl_t *mp; 363 ddi_dma_attr_t *attr_p; 364 uint64_t syslo = iommu_p->iommu_dvma_base; 365 uint64_t syshi = iommu_p->iommu_dvma_end; 366 uint64_t fasthi = iommu_p->iommu_dvma_fast_end; 367 ddi_dma_lim_t *lim_p = dmareq->dmar_limits; 368 uint32_t count_max = lim_p->dlim_cntr_max; 369 uint64_t lo = lim_p->dlim_addr_lo; 370 uint64_t hi = lim_p->dlim_addr_hi; 371 if (hi <= lo) { 372 DEBUG0(DBG_DMA_MAP, dip, "Bad limits\n"); 373 return ((ddi_dma_impl_t *)DDI_DMA_NOMAPPING); 374 } 375 if (!count_max) 376 count_max--; 377 378 if (!(mp = pci_dma_allocmp(dip, rdip, dmareq->dmar_fp, 379 dmareq->dmar_arg))) 380 return (NULL); 381 382 /* store original dev input at the 2nd ddi_dma_attr */ 383 attr_p = DEV_ATTR(mp); 384 SET_DMAATTR(attr_p, lo, hi, -1, count_max); 385 SET_DMAALIGN(attr_p, 1); 386 387 lo = MAX(lo, syslo); 388 hi = MIN(hi, syshi); 389 if (hi <= lo) 390 mp->dmai_flags |= DMAI_FLAGS_PEER_ONLY; 391 count_max = MIN(count_max, hi - lo); 392 393 if (DEV_NOSYSLIMIT(lo, hi, syslo, fasthi, 1)) 394 mp->dmai_flags |= DMAI_FLAGS_NOFASTLIMIT | 395 DMAI_FLAGS_NOSYSLIMIT; 396 else { 397 if (DEV_NOFASTLIMIT(lo, hi, syslo, syshi, 1)) 398 mp->dmai_flags |= DMAI_FLAGS_NOFASTLIMIT; 399 } 400 if (PCI_DMA_NOCTX(rdip)) 401 mp->dmai_flags |= DMAI_FLAGS_NOCTX; 402 403 /* store augumented dev input to mp->dmai_attr */ 404 mp->dmai_minxfer = lim_p->dlim_minxfer; 405 mp->dmai_burstsizes = lim_p->dlim_burstsizes; 406 attr_p = &mp->dmai_attr; 407 SET_DMAATTR(attr_p, lo, hi, -1, count_max); 408 SET_DMAALIGN(attr_p, 1); 409 return (mp); 410 } 411 412 /* 413 * pci_dma_attr2hdl 414 * 415 * This routine is called from the alloc handle entry point to sanity check the 416 * dma attribute structure. 417 * 418 * use by: pci_dma_allochdl() 419 * 420 * return value: 421 * 422 * DDI_SUCCESS - on success 423 * DDI_DMA_BADATTR - attribute has invalid version number 424 * or address limits exclude dvma space 425 */ 426 int 427 pci_dma_attr2hdl(pci_t *pci_p, ddi_dma_impl_t *mp) 428 { 429 iommu_t *iommu_p = pci_p->pci_iommu_p; 430 uint64_t syslo, syshi; 431 ddi_dma_attr_t *attrp = DEV_ATTR(mp); 432 uint64_t hi = attrp->dma_attr_addr_hi; 433 uint64_t lo = attrp->dma_attr_addr_lo; 434 uint64_t align = attrp->dma_attr_align; 435 uint64_t nocross = attrp->dma_attr_seg; 436 uint64_t count_max = attrp->dma_attr_count_max; 437 438 DEBUG3(DBG_DMA_ALLOCH, pci_p->pci_dip, "attrp=%p cntr_max=%x.%08x\n", 439 attrp, HI32(count_max), LO32(count_max)); 440 DEBUG4(DBG_DMA_ALLOCH, pci_p->pci_dip, "hi=%x.%08x lo=%x.%08x\n", 441 HI32(hi), LO32(hi), HI32(lo), LO32(lo)); 442 DEBUG4(DBG_DMA_ALLOCH, pci_p->pci_dip, "seg=%x.%08x align=%x.%08x\n", 443 HI32(nocross), LO32(nocross), HI32(align), LO32(align)); 444 445 if (!nocross) 446 nocross--; 447 if (attrp->dma_attr_flags & DDI_DMA_FORCE_PHYSICAL) { /* BYPASS */ 448 449 DEBUG0(DBG_DMA_ALLOCH, pci_p->pci_dip, "bypass mode\n"); 450 /* if tomatillo ver <= 2.3 don't allow bypass */ 451 if (tomatillo_disallow_bypass) 452 return (DDI_DMA_BADATTR); 453 454 mp->dmai_flags |= DMAI_FLAGS_BYPASSREQ; 455 if (nocross != UINT64_MAX) 456 return (DDI_DMA_BADATTR); 457 if (align && (align > IOMMU_PAGE_SIZE)) 458 return (DDI_DMA_BADATTR); 459 align = 1; /* align on 1 page boundary */ 460 syslo = iommu_p->iommu_dma_bypass_base; 461 syshi = iommu_p->iommu_dma_bypass_end; 462 463 } else { /* IOMMU_XLATE or PEER_TO_PEER */ 464 align = MAX(align, IOMMU_PAGE_SIZE) - 1; 465 if ((align & nocross) != align) { 466 dev_info_t *rdip = mp->dmai_rdip; 467 cmn_err(CE_WARN, "%s%d dma_attr_seg not aligned", 468 NAMEINST(rdip)); 469 return (DDI_DMA_BADATTR); 470 } 471 align = IOMMU_BTOP(align + 1); 472 syslo = iommu_p->iommu_dvma_base; 473 syshi = iommu_p->iommu_dvma_end; 474 } 475 if (hi <= lo) { 476 dev_info_t *rdip = mp->dmai_rdip; 477 cmn_err(CE_WARN, "%s%d limits out of range", NAMEINST(rdip)); 478 return (DDI_DMA_BADATTR); 479 } 480 lo = MAX(lo, syslo); 481 hi = MIN(hi, syshi); 482 if (!count_max) 483 count_max--; 484 485 DEBUG4(DBG_DMA_ALLOCH, pci_p->pci_dip, "hi=%x.%08x, lo=%x.%08x\n", 486 HI32(hi), LO32(hi), HI32(lo), LO32(lo)); 487 if (hi <= lo) { /* peer transfers cannot have alignment & nocross */ 488 dev_info_t *rdip = mp->dmai_rdip; 489 cmn_err(CE_WARN, "%s%d peer only dev %p", NAMEINST(rdip), mp); 490 if ((nocross < UINT32_MAX) || (align > 1)) { 491 cmn_err(CE_WARN, "%s%d peer only device bad attr", 492 NAMEINST(rdip)); 493 return (DDI_DMA_BADATTR); 494 } 495 mp->dmai_flags |= DMAI_FLAGS_PEER_ONLY; 496 } else /* set practical counter_max value */ 497 count_max = MIN(count_max, hi - lo); 498 499 if (DEV_NOSYSLIMIT(lo, hi, syslo, syshi, align)) 500 mp->dmai_flags |= DMAI_FLAGS_NOSYSLIMIT | 501 DMAI_FLAGS_NOFASTLIMIT; 502 else { 503 syshi = iommu_p->iommu_dvma_fast_end; 504 if (DEV_NOFASTLIMIT(lo, hi, syslo, syshi, align)) 505 mp->dmai_flags |= DMAI_FLAGS_NOFASTLIMIT; 506 } 507 if (PCI_DMA_NOCTX(mp->dmai_rdip)) 508 mp->dmai_flags |= DMAI_FLAGS_NOCTX; 509 510 mp->dmai_minxfer = attrp->dma_attr_minxfer; 511 mp->dmai_burstsizes = attrp->dma_attr_burstsizes; 512 attrp = &mp->dmai_attr; 513 SET_DMAATTR(attrp, lo, hi, nocross, count_max); 514 return (DDI_SUCCESS); 515 } 516 517 /* 518 * set up consistent dma flags according to hardware capability 519 */ 520 uint32_t 521 pci_dma_consist_check(uint32_t req_flags, pbm_t *pbm_p) 522 { 523 if (!pci_stream_buf_enable || !pci_stream_buf_exists) 524 req_flags |= DDI_DMA_CONSISTENT; 525 if (req_flags & DDI_DMA_CONSISTENT && !pbm_p->pbm_sync_reg_pa) 526 req_flags |= DMP_NOSYNC; 527 return (req_flags); 528 } 529 530 #define TGT_PFN_INBETWEEN(pfn, bgn, end) ((pfn >= bgn) && (pfn <= end)) 531 532 /* 533 * pci_dma_type - determine which of the three types DMA (peer-to-peer, 534 * iommu bypass, or iommu translate) we are asked to do. 535 * Also checks pfn0 and rejects any non-peer-to-peer 536 * requests for peer-only devices. 537 * 538 * return values: 539 * DDI_DMA_NOMAPPING - can't get valid pfn0, or bad dma type 540 * DDI_SUCCESS 541 * 542 * dma handle members affected (set on exit): 543 * mp->dmai_object - dmareq->dmar_object 544 * mp->dmai_rflags - consistent?, nosync?, dmareq->dmar_flags 545 * mp->dmai_flags - DMA type 546 * mp->dmai_pfn0 - 1st page pfn (if va/size pair and not shadow) 547 * mp->dmai_roffset - initialized to starting IOMMU page offset 548 * mp->dmai_ndvmapages - # of total IOMMU pages of entire object 549 * mp->pdh_sync_buf_pa - dma sync buffer PA is DMA flow is supported 550 */ 551 int 552 pci_dma_type(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 553 { 554 dev_info_t *dip = pci_p->pci_dip; 555 ddi_dma_obj_t *dobj_p = &dmareq->dmar_object; 556 pbm_t *pbm_p = pci_p->pci_pbm_p; 557 page_t **pplist; 558 struct as *as_p; 559 uint32_t offset; 560 caddr_t vaddr; 561 pfn_t pfn0; 562 563 mp->dmai_rflags = pci_dma_consist_check(dmareq->dmar_flags, pbm_p); 564 mp->dmai_flags |= mp->dmai_rflags & DMP_NOSYNC ? DMAI_FLAGS_NOSYNC : 0; 565 566 switch (dobj_p->dmao_type) { 567 case DMA_OTYP_BUFVADDR: 568 case DMA_OTYP_VADDR: { 569 vaddr = dobj_p->dmao_obj.virt_obj.v_addr; 570 pplist = dobj_p->dmao_obj.virt_obj.v_priv; 571 as_p = dobj_p->dmao_obj.virt_obj.v_as; 572 if (as_p == NULL) 573 as_p = &kas; 574 575 DEBUG2(DBG_DMA_MAP, dip, "vaddr=%p pplist=%p\n", vaddr, pplist); 576 offset = (ulong_t)vaddr & IOMMU_PAGE_OFFSET; 577 578 if (pplist) { /* shadow list */ 579 mp->dmai_flags |= DMAI_FLAGS_PGPFN; 580 ASSERT(PAGE_LOCKED(*pplist)); 581 pfn0 = page_pptonum(*pplist); 582 } else if (pci_dvma_remap_enabled && as_p == &kas && 583 dobj_p->dmao_type != DMA_OTYP_BUFVADDR) { 584 int (*waitfp)(caddr_t) = dmareq->dmar_fp; 585 uint_t flags = ((waitfp == DDI_DMA_SLEEP)? 586 HAC_SLEEP : HAC_NOSLEEP) | HAC_PAGELOCK; 587 int ret; 588 589 ret = hat_add_callback(pci_dvma_cbid, vaddr, 590 IOMMU_PAGE_SIZE - offset, flags, mp, &pfn0, 591 MP_HAT_CB_COOKIE_PTR(mp, 0)); 592 593 if (pfn0 == PFN_INVALID && ret == ENOMEM) { 594 ASSERT(waitfp != DDI_DMA_SLEEP); 595 if (waitfp != DDI_DMA_DONTWAIT) { 596 ddi_set_callback(waitfp, 597 dmareq->dmar_arg, 598 &pci_kmem_clid); 599 return (DDI_DMA_NORESOURCES); 600 } 601 } 602 mp->dmai_flags |= DMAI_FLAGS_RELOC; 603 } else 604 pfn0 = hat_getpfnum(as_p->a_hat, vaddr); 605 } 606 break; 607 608 case DMA_OTYP_PAGES: 609 offset = dobj_p->dmao_obj.pp_obj.pp_offset; 610 mp->dmai_flags |= DMAI_FLAGS_PGPFN; 611 pfn0 = page_pptonum(dobj_p->dmao_obj.pp_obj.pp_pp); 612 ASSERT(PAGE_LOCKED(dobj_p->dmao_obj.pp_obj.pp_pp)); 613 break; 614 615 case DMA_OTYP_PADDR: 616 default: 617 cmn_err(CE_WARN, "%s%d requested unsupported dma type %x", 618 NAMEINST(mp->dmai_rdip), dobj_p->dmao_type); 619 return (DDI_DMA_NOMAPPING); 620 } 621 if (pfn0 == PFN_INVALID) { 622 cmn_err(CE_WARN, "%s%d: invalid pfn0 for DMA object %p", 623 NAMEINST(dip), dobj_p); 624 return (DDI_DMA_NOMAPPING); 625 } 626 if (TGT_PFN_INBETWEEN(pfn0, pbm_p->pbm_base_pfn, pbm_p->pbm_last_pfn)) { 627 mp->dmai_flags |= DMAI_FLAGS_PEER_TO_PEER; 628 goto done; /* leave bypass and dvma flag as 0 */ 629 } 630 if (PCI_DMA_ISPEERONLY(mp)) { 631 dev_info_t *rdip = mp->dmai_rdip; 632 cmn_err(CE_WARN, "Bad peer-to-peer req %s%d", NAMEINST(rdip)); 633 return (DDI_DMA_NOMAPPING); 634 } 635 mp->dmai_flags |= (mp->dmai_flags & DMAI_FLAGS_BYPASSREQ) ? 636 DMAI_FLAGS_BYPASS : DMAI_FLAGS_DVMA; 637 done: 638 mp->dmai_object = *dobj_p; /* whole object */ 639 mp->dmai_pfn0 = (void *)pfn0; /* cache pfn0 */ 640 mp->dmai_roffset = offset; /* win0 pg0 offset */ 641 mp->dmai_ndvmapages = IOMMU_BTOPR(offset + mp->dmai_object.dmao_size); 642 643 return (DDI_SUCCESS); 644 } 645 646 /* 647 * pci_dma_pgpfn - set up pfnlst array according to pages 648 * VA/size pair: <shadow IO, bypass, peer-to-peer>, or OTYP_PAGES 649 */ 650 /*ARGSUSED*/ 651 static int 652 pci_dma_pgpfn(pci_t *pci_p, ddi_dma_impl_t *mp, uint_t npages) 653 { 654 int i; 655 #ifdef DEBUG 656 dev_info_t *dip = pci_p->pci_dip; 657 #endif 658 switch (mp->dmai_object.dmao_type) { 659 case DMA_OTYP_BUFVADDR: 660 case DMA_OTYP_VADDR: { 661 page_t **pplist = mp->dmai_object.dmao_obj.virt_obj.v_priv; 662 DEBUG2(DBG_DMA_MAP, dip, "shadow pplist=%p, %x pages, pfns=", 663 pplist, npages); 664 for (i = 1; i < npages; i++) { 665 iopfn_t pfn = page_pptonum(pplist[i]); 666 ASSERT(PAGE_LOCKED(pplist[i])); 667 PCI_SET_MP_PFN1(mp, i, pfn); 668 DEBUG1(DBG_DMA_MAP|DBG_CONT, dip, "%x ", pfn); 669 } 670 DEBUG0(DBG_DMA_MAP|DBG_CONT, dip, "\n"); 671 } 672 break; 673 674 case DMA_OTYP_PAGES: { 675 page_t *pp = mp->dmai_object.dmao_obj.pp_obj.pp_pp->p_next; 676 DEBUG1(DBG_DMA_MAP, dip, "pp=%p pfns=", pp); 677 for (i = 1; i < npages; i++, pp = pp->p_next) { 678 iopfn_t pfn = page_pptonum(pp); 679 ASSERT(PAGE_LOCKED(pp)); 680 PCI_SET_MP_PFN1(mp, i, pfn); 681 DEBUG1(DBG_DMA_MAP|DBG_CONT, dip, "%x ", pfn); 682 } 683 DEBUG0(DBG_DMA_MAP|DBG_CONT, dip, "\n"); 684 } 685 break; 686 687 default: /* check is already done by pci_dma_type */ 688 ASSERT(0); 689 break; 690 } 691 return (DDI_SUCCESS); 692 } 693 694 /* 695 * pci_dma_vapfn - set up pfnlst array according to VA 696 * VA/size pair: <normal, bypass, peer-to-peer> 697 * pfn0 is skipped as it is already done. 698 * In this case, the cached pfn0 is used to fill pfnlst[0] 699 */ 700 static int 701 pci_dma_vapfn(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, 702 uint_t npages) 703 { 704 dev_info_t *dip = pci_p->pci_dip; 705 int i; 706 caddr_t vaddr = (caddr_t)mp->dmai_object.dmao_obj.virt_obj.v_as; 707 struct hat *hat_p = vaddr ? ((struct as *)vaddr)->a_hat : kas.a_hat; 708 caddr_t sva; 709 int needcb = 0; 710 711 sva = (caddr_t)(((uintptr_t)mp->dmai_object.dmao_obj.virt_obj.v_addr + 712 IOMMU_PAGE_SIZE) & IOMMU_PAGE_MASK); 713 714 if (pci_dvma_remap_enabled && hat_p == kas.a_hat && 715 mp->dmai_object.dmao_type != DMA_OTYP_BUFVADDR) 716 needcb = 1; 717 718 for (vaddr = sva, i = 1; i < npages; i++, vaddr += IOMMU_PAGE_SIZE) { 719 pfn_t pfn; 720 721 if (needcb) { 722 int (*waitfp)(caddr_t) = dmareq->dmar_fp; 723 uint_t flags = ((waitfp == DDI_DMA_SLEEP)? 724 HAC_SLEEP : HAC_NOSLEEP) | HAC_PAGELOCK; 725 int ret; 726 727 ret = hat_add_callback(pci_dvma_cbid, vaddr, 728 IOMMU_PAGE_SIZE, flags, mp, &pfn, 729 MP_HAT_CB_COOKIE_PTR(mp, i)); 730 731 if (pfn == PFN_INVALID && ret == ENOMEM) { 732 ASSERT(waitfp != DDI_DMA_SLEEP); 733 if (waitfp != DDI_DMA_DONTWAIT) 734 ddi_set_callback(waitfp, 735 dmareq->dmar_arg, &pci_kmem_clid); 736 return (DDI_DMA_NORESOURCES); 737 } 738 } else 739 pfn = hat_getpfnum(hat_p, vaddr); 740 if (pfn == PFN_INVALID) 741 goto err_badpfn; 742 PCI_SET_MP_PFN1(mp, i, (iopfn_t)pfn); 743 DEBUG3(DBG_DMA_MAP, dip, "pci_dma_vapfn: mp=%p pfnlst[%x]=%x\n", 744 mp, i, (iopfn_t)pfn); 745 } 746 return (DDI_SUCCESS); 747 err_badpfn: 748 cmn_err(CE_WARN, "%s%d: bad page frame vaddr=%p", NAMEINST(dip), vaddr); 749 return (DDI_DMA_NOMAPPING); 750 } 751 752 /* 753 * pci_dma_pfn - Fills pfn list for all pages being DMA-ed. 754 * 755 * dependencies: 756 * mp->dmai_ndvmapages - set to total # of dma pages 757 * 758 * return value: 759 * DDI_SUCCESS 760 * DDI_DMA_NOMAPPING 761 */ 762 int 763 pci_dma_pfn(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 764 { 765 uint32_t npages = mp->dmai_ndvmapages; 766 int (*waitfp)(caddr_t) = dmareq->dmar_fp; 767 int i, ret, peer = PCI_DMA_ISPTP(mp); 768 769 pbm_t *pbm_p = pci_p->pci_pbm_p; 770 iopfn_t pfn_base = pbm_p->pbm_base_pfn; 771 iopfn_t pfn_last = pbm_p->pbm_last_pfn; 772 iopfn_t pfn_adj = peer ? pfn_base : 0; 773 774 DEBUG2(DBG_DMA_MAP, pci_p->pci_dip, "pci_dma_pfn: mp=%p pfn0=%x\n", 775 mp, MP_PFN0(mp) - pfn_adj); 776 /* 1 page: no array alloc/fill, no mixed mode check */ 777 if (npages == 1) { 778 PCI_SET_MP_PFN(mp, 0, MP_PFN0(mp) - pfn_adj); 779 return (DDI_SUCCESS); 780 } 781 /* allocate pfn array */ 782 if (!(mp->dmai_pfnlst = kmem_alloc(npages * sizeof (iopfn_t), 783 waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP))) { 784 if (waitfp != DDI_DMA_DONTWAIT) 785 ddi_set_callback(waitfp, dmareq->dmar_arg, 786 &pci_kmem_clid); 787 return (DDI_DMA_NORESOURCES); 788 } 789 /* fill pfn array */ 790 PCI_SET_MP_PFN(mp, 0, MP_PFN0(mp) - pfn_adj); /* pfnlst[0] */ 791 if ((ret = PCI_DMA_ISPGPFN(mp) ? pci_dma_pgpfn(pci_p, mp, npages) : 792 pci_dma_vapfn(pci_p, dmareq, mp, npages)) != DDI_SUCCESS) 793 goto err; 794 795 /* skip pfn0, check mixed mode and adjust peer to peer pfn */ 796 for (i = 1; i < npages; i++) { 797 iopfn_t pfn = PCI_GET_MP_PFN1(mp, i); 798 if (peer ^ TGT_PFN_INBETWEEN(pfn, pfn_base, pfn_last)) { 799 cmn_err(CE_WARN, "%s%d mixed mode DMA %lx %lx", 800 NAMEINST(mp->dmai_rdip), MP_PFN0(mp), pfn); 801 ret = DDI_DMA_NOMAPPING; /* mixed mode */ 802 goto err; 803 } 804 DEBUG3(DBG_DMA_MAP, pci_p->pci_dip, 805 "pci_dma_pfn: pfnlst[%x]=%x-%x\n", i, pfn, pfn_adj); 806 if (pfn_adj) 807 PCI_SET_MP_PFN1(mp, i, pfn - pfn_adj); 808 } 809 return (DDI_SUCCESS); 810 err: 811 pci_dvma_unregister_callbacks(pci_p, mp); 812 pci_dma_freepfn(mp); 813 return (ret); 814 } 815 816 /* 817 * pci_dvma_win() - trim requested DVMA size down to window size 818 * The 1st window starts from offset and ends at page-aligned boundary. 819 * From the 2nd window on, each window starts and ends at page-aligned 820 * boundary except the last window ends at wherever requested. 821 * 822 * accesses the following mp-> members: 823 * mp->dmai_attr.dma_attr_count_max 824 * mp->dmai_attr.dma_attr_seg 825 * mp->dmai_roffset - start offset of 1st window 826 * mp->dmai_rflags (redzone) 827 * mp->dmai_ndvmapages (for 1 page fast path) 828 * 829 * sets the following mp-> members: 830 * mp->dmai_size - xfer size, != winsize if 1st/last win (not fixed) 831 * mp->dmai_winsize - window size (no redzone), n * page size (fixed) 832 * mp->dmai_nwin - # of DMA windows of entire object (fixed) 833 * mp->dmai_rflags - remove partial flag if nwin == 1 (fixed) 834 * mp->dmai_winlst - NULL, window objects not used for DVMA (fixed) 835 * 836 * fixed - not changed across different DMA windows 837 */ 838 /*ARGSUSED*/ 839 int 840 pci_dvma_win(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 841 { 842 uint32_t redzone_sz = HAS_REDZONE(mp) ? IOMMU_PAGE_SIZE : 0; 843 size_t obj_sz = mp->dmai_object.dmao_size; 844 size_t xfer_sz; 845 ulong_t pg_off; 846 847 if ((mp->dmai_ndvmapages == 1) && !redzone_sz) { 848 mp->dmai_rflags &= ~DDI_DMA_PARTIAL; 849 mp->dmai_size = obj_sz; 850 mp->dmai_winsize = IOMMU_PAGE_SIZE; 851 mp->dmai_nwin = 1; 852 goto done; 853 } 854 855 pg_off = mp->dmai_roffset; 856 xfer_sz = obj_sz + redzone_sz; 857 858 /* include redzone in nocross check */ 859 { 860 uint64_t nocross = mp->dmai_attr.dma_attr_seg; 861 if (xfer_sz + pg_off - 1 > nocross) 862 xfer_sz = nocross - pg_off + 1; 863 if (redzone_sz && (xfer_sz <= redzone_sz)) { 864 DEBUG5(DBG_DMA_MAP, pci_p->pci_dip, 865 "nocross too small %lx(%lx)+%lx+%x < %" PRIx64 "\n", 866 xfer_sz, obj_sz, pg_off, redzone_sz, nocross); 867 return (DDI_DMA_TOOBIG); 868 } 869 } 870 xfer_sz -= redzone_sz; /* restore transfer size */ 871 /* check counter max */ 872 { 873 uint32_t count_max = mp->dmai_attr.dma_attr_count_max; 874 if (xfer_sz - 1 > count_max) 875 xfer_sz = count_max + 1; 876 } 877 if (xfer_sz >= obj_sz) { 878 mp->dmai_rflags &= ~DDI_DMA_PARTIAL; 879 mp->dmai_size = xfer_sz; 880 mp->dmai_winsize = P2ROUNDUP(xfer_sz + pg_off, IOMMU_PAGE_SIZE); 881 mp->dmai_nwin = 1; 882 goto done; 883 } 884 if (!(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 885 DEBUG4(DBG_DMA_MAP, pci_p->pci_dip, 886 "too big: %lx+%lx+%x > %lx\n", 887 obj_sz, pg_off, redzone_sz, xfer_sz); 888 return (DDI_DMA_TOOBIG); 889 } 890 891 xfer_sz = IOMMU_PTOB(IOMMU_BTOP(xfer_sz + pg_off)); /* page align */ 892 mp->dmai_size = xfer_sz - pg_off; /* 1st window xferrable size */ 893 mp->dmai_winsize = xfer_sz; /* redzone not in winsize */ 894 mp->dmai_nwin = (obj_sz + pg_off + xfer_sz - 1) / xfer_sz; 895 done: 896 mp->dmai_winlst = NULL; 897 dump_dma_handle(DBG_DMA_MAP, pci_p->pci_dip, mp); 898 return (DDI_SUCCESS); 899 } 900 901 /* 902 * fast track cache entry to iommu context, inserts 3 0 bits between 903 * upper 6-bits and lower 3-bits of the 9-bit cache entry 904 */ 905 #define IOMMU_FCE_TO_CTX(i) (((i) << 3) | ((i) & 0x7) | 0x38) 906 907 /* 908 * pci_dvma_map_fast - attempts to map fast trackable DVMA 909 */ 910 int 911 pci_dvma_map_fast(iommu_t *iommu_p, ddi_dma_impl_t *mp) 912 { 913 uint_t clustsz = pci_dvma_page_cache_clustsz; 914 uint_t entries = pci_dvma_page_cache_entries; 915 uint64_t *tte_addr; 916 uint64_t tte = GET_TTE_TEMPLATE(mp); 917 int i = iommu_p->iommu_dvma_addr_scan_start; 918 uint8_t *lock_addr = iommu_p->iommu_dvma_cache_locks + i; 919 iopfn_t *pfn_addr; 920 dvma_addr_t dvma_pg; 921 size_t npages = IOMMU_BTOP(mp->dmai_winsize); 922 #ifdef DEBUG 923 dev_info_t *dip = mp->dmai_rdip; 924 #endif 925 extern uint8_t ldstub(uint8_t *); 926 ASSERT(IOMMU_PTOB(npages) == mp->dmai_winsize); 927 ASSERT(npages + HAS_REDZONE(mp) <= clustsz); 928 929 for (; i < entries && ldstub(lock_addr); i++, lock_addr++) 930 ; 931 if (i >= entries) { 932 lock_addr = iommu_p->iommu_dvma_cache_locks; 933 i = 0; 934 for (; i < entries && ldstub(lock_addr); i++, lock_addr++) 935 ; 936 if (i >= entries) { 937 #ifdef PCI_DMA_PROF 938 pci_dvmaft_exhaust++; 939 #endif 940 return (DDI_DMA_NORESOURCES); 941 } 942 } 943 iommu_p->iommu_dvma_addr_scan_start = (i + 1) & (entries - 1); 944 if (PCI_DMA_USECTX(mp)) { 945 dvma_context_t ctx = IOMMU_FCE_TO_CTX(i); 946 tte |= IOMMU_CTX2TTE(ctx); 947 mp->dmai_flags |= DMAI_FLAGS_CONTEXT; 948 DEBUG1(DBG_DMA_MAP, dip, "fast: ctx=0x%x\n", ctx); 949 } 950 i *= clustsz; 951 tte_addr = iommu_p->iommu_tsb_vaddr + i; 952 dvma_pg = iommu_p->dvma_base_pg + i; 953 #ifdef DEBUG 954 for (i = 0; i < clustsz; i++) 955 ASSERT(TTE_IS_INVALID(tte_addr[i])); 956 #endif 957 *tte_addr = tte | IOMMU_PTOB(MP_PFN0(mp)); /* map page 0 */ 958 DEBUG5(DBG_DMA_MAP, dip, "fast %p:dvma_pg=%x tte0(%p)=%08x.%08x\n", mp, 959 dvma_pg, tte_addr, HI32(*tte_addr), LO32(*tte_addr)); 960 if (npages == 1) 961 goto tte_done; 962 pfn_addr = PCI_GET_MP_PFN1_ADDR(mp); /* short iommu_map_pages() */ 963 for (tte_addr++, i = 1; i < npages; i++, tte_addr++, pfn_addr++) { 964 *tte_addr = tte | IOMMU_PTOB(*pfn_addr); 965 DEBUG5(DBG_DMA_MAP, dip, "fast %p:tte(%p, %p)=%08x.%08x\n", mp, 966 tte_addr, pfn_addr, HI32(*tte_addr), LO32(*tte_addr)); 967 } 968 tte_done: 969 #ifdef PCI_DMA_PROF 970 pci_dvmaft_success++; 971 #endif 972 mp->dmai_mapping = mp->dmai_roffset | IOMMU_PTOB(dvma_pg); 973 mp->dmai_offset = 0; 974 mp->dmai_flags |= DMAI_FLAGS_FASTTRACK; 975 PCI_SAVE_MP_TTE(mp, tte); /* save TTE template for unmapping */ 976 if (DVMA_DBG_ON(iommu_p)) 977 pci_dvma_alloc_debug(iommu_p, (char *)mp->dmai_mapping, 978 mp->dmai_size, mp); 979 return (DDI_SUCCESS); 980 } 981 982 /* 983 * pci_dvma_map: map non-fasttrack DMA 984 * Use quantum cache if single page DMA. 985 */ 986 int 987 pci_dvma_map(ddi_dma_impl_t *mp, ddi_dma_req_t *dmareq, iommu_t *iommu_p) 988 { 989 uint_t npages = PCI_DMA_WINNPGS(mp); 990 dvma_addr_t dvma_pg, dvma_pg_index; 991 void *dvma_addr; 992 uint64_t tte = GET_TTE_TEMPLATE(mp); 993 int sleep = dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP; 994 #ifdef DEBUG 995 dev_info_t *dip = mp->dmai_rdip; 996 #endif 997 /* 998 * allocate dvma space resource and map in the first window. 999 * (vmem_t *vmp, size_t size, 1000 * size_t align, size_t phase, size_t nocross, 1001 * void *minaddr, void *maxaddr, int vmflag) 1002 */ 1003 if ((npages == 1) && !HAS_REDZONE(mp) && HAS_NOSYSLIMIT(mp)) { 1004 dvma_addr = vmem_alloc(iommu_p->iommu_dvma_map, 1005 IOMMU_PAGE_SIZE, sleep); 1006 mp->dmai_flags |= DMAI_FLAGS_VMEMCACHE; 1007 #ifdef PCI_DMA_PROF 1008 pci_dvma_vmem_alloc++; 1009 #endif 1010 } else { 1011 dvma_addr = vmem_xalloc(iommu_p->iommu_dvma_map, 1012 IOMMU_PTOB(npages + HAS_REDZONE(mp)), 1013 MAX(mp->dmai_attr.dma_attr_align, IOMMU_PAGE_SIZE), 1014 0, 1015 mp->dmai_attr.dma_attr_seg + 1, 1016 (void *)mp->dmai_attr.dma_attr_addr_lo, 1017 (void *)(mp->dmai_attr.dma_attr_addr_hi + 1), 1018 sleep); 1019 #ifdef PCI_DMA_PROF 1020 pci_dvma_vmem_xalloc++; 1021 #endif 1022 } 1023 dvma_pg = IOMMU_BTOP((ulong_t)dvma_addr); 1024 dvma_pg_index = dvma_pg - iommu_p->dvma_base_pg; 1025 DEBUG2(DBG_DMA_MAP, dip, "fallback dvma_pages: dvma_pg=%x index=%x\n", 1026 dvma_pg, dvma_pg_index); 1027 if (dvma_pg == 0) 1028 goto noresource; 1029 1030 /* allocate DVMA context */ 1031 if ((npages >= pci_context_minpages) && PCI_DMA_USECTX(mp)) { 1032 dvma_context_t ctx; 1033 if (ctx = pci_iommu_get_dvma_context(iommu_p, dvma_pg_index)) { 1034 tte |= IOMMU_CTX2TTE(ctx); 1035 mp->dmai_flags |= DMAI_FLAGS_CONTEXT; 1036 } 1037 } 1038 mp->dmai_mapping = mp->dmai_roffset | IOMMU_PTOB(dvma_pg); 1039 mp->dmai_offset = 0; 1040 PCI_SAVE_MP_TTE(mp, tte); /* mp->dmai_tte = tte */ 1041 iommu_map_pages(iommu_p, mp, dvma_pg, npages, 0); 1042 return (DDI_SUCCESS); 1043 noresource: 1044 if (dmareq->dmar_fp != DDI_DMA_DONTWAIT) { 1045 DEBUG0(DBG_DMA_MAP, dip, "dvma_pg 0 - set callback\n"); 1046 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg, 1047 &iommu_p->iommu_dvma_clid); 1048 } 1049 DEBUG0(DBG_DMA_MAP, dip, "vmem_xalloc - DDI_DMA_NORESOURCES\n"); 1050 return (DDI_DMA_NORESOURCES); 1051 } 1052 1053 void 1054 pci_dvma_unmap(iommu_t *iommu_p, ddi_dma_impl_t *mp) 1055 { 1056 size_t npages; 1057 dvma_addr_t dvma_addr = (dvma_addr_t)mp->dmai_mapping; 1058 dvma_addr_t dvma_pg = IOMMU_BTOP(dvma_addr); 1059 dvma_addr = IOMMU_PTOB(dvma_pg); 1060 1061 if (mp->dmai_flags & DMAI_FLAGS_FASTTRACK) { 1062 iopfn_t index = dvma_pg - iommu_p->dvma_base_pg; 1063 ASSERT(index % pci_dvma_page_cache_clustsz == 0); 1064 index /= pci_dvma_page_cache_clustsz; 1065 ASSERT(index < pci_dvma_page_cache_entries); 1066 iommu_p->iommu_dvma_cache_locks[index] = 0; 1067 #ifdef PCI_DMA_PROF 1068 pci_dvmaft_free++; 1069 #endif 1070 return; 1071 } 1072 npages = IOMMU_BTOP(mp->dmai_winsize) + HAS_REDZONE(mp); 1073 pci_vmem_free(iommu_p, mp, (void *)dvma_addr, npages); 1074 1075 if (mp->dmai_flags & DMAI_FLAGS_CONTEXT) 1076 pci_iommu_free_dvma_context(iommu_p, MP2CTX(mp)); 1077 } 1078 1079 void 1080 pci_dma_sync_unmap(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp) 1081 { 1082 pci_t *pci_p = get_pci_soft_state(ddi_get_instance(dip)); 1083 iommu_t *iommu_p = pci_p->pci_iommu_p; 1084 uint64_t sync_buf_save = SYNC_BUF_PA(mp); 1085 uint32_t fast_track = mp->dmai_flags & DMAI_FLAGS_FASTTRACK; 1086 1087 if (fast_track) { 1088 dvma_addr_t dvma_pg = IOMMU_BTOP(mp->dmai_mapping); 1089 1090 SYNC_BUF_PA(mp) = IOMMU_PAGE_TTEPA(iommu_p, dvma_pg); 1091 ASSERT(!(SYNC_BUF_PA(mp) & PCI_SYNC_FLAG_SIZE - 1)); 1092 } 1093 1094 if (pci_dvma_sync_before_unmap) { 1095 pci_dma_sync(dip, rdip, (ddi_dma_handle_t)mp, 0, 0, 1096 DDI_DMA_SYNC_FORCPU); 1097 iommu_unmap_window(iommu_p, mp); 1098 } else { 1099 iommu_unmap_window(iommu_p, mp); 1100 pci_dma_sync(dip, rdip, (ddi_dma_handle_t)mp, 0, 0, 1101 DDI_DMA_SYNC_FORCPU); 1102 } 1103 1104 if (fast_track) 1105 SYNC_BUF_PA(mp) = sync_buf_save; 1106 } 1107 1108 /* 1109 * DVMA mappings may have multiple windows, but each window always have 1110 * one segment. 1111 */ 1112 int 1113 pci_dvma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp, 1114 enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp, 1115 uint_t cache_flags) 1116 { 1117 switch (cmd) { 1118 1119 case DDI_DMA_REMAP: 1120 if (pci_dvma_remap_enabled) 1121 return (pci_dvma_remap(dip, rdip, mp, *offp, *lenp)); 1122 return (DDI_FAILURE); 1123 1124 default: 1125 DEBUG3(DBG_DMA_CTL, dip, "unknown command (%x): rdip=%s%d\n", 1126 cmd, ddi_driver_name(rdip), ddi_get_instance(rdip)); 1127 break; 1128 } 1129 return (DDI_FAILURE); 1130 } 1131 1132 void 1133 pci_dma_freewin(ddi_dma_impl_t *mp) 1134 { 1135 pci_dma_win_t *win_p = mp->dmai_winlst, *win2_p; 1136 for (win2_p = win_p; win_p; win2_p = win_p) { 1137 win_p = win2_p->win_next; 1138 kmem_free(win2_p, sizeof (pci_dma_win_t) + 1139 sizeof (ddi_dma_cookie_t) * win2_p->win_ncookies); 1140 } 1141 mp->dmai_nwin = 0; 1142 mp->dmai_winlst = NULL; 1143 } 1144 1145 /* 1146 * pci_dma_newwin - create a dma window object and cookies 1147 * 1148 * After the initial scan in pci_dma_physwin(), which identifies 1149 * a portion of the pfn array that belongs to a dma window, 1150 * we are called to allocate and initialize representing memory 1151 * resources. We know from the 1st scan the number of cookies 1152 * or dma segment in this window so we can allocate a contiguous 1153 * memory array for the dma cookies (The implementation of 1154 * ddi_dma_nextcookie(9f) dictates dma cookies be contiguous). 1155 * 1156 * A second round scan is done on the pfn array to identify 1157 * each dma segment and initialize its corresponding dma cookie. 1158 * We don't need to do all the safety checking and we know they 1159 * all belong to the same dma window. 1160 * 1161 * Input: cookie_no - # of cookies identified by the 1st scan 1162 * start_idx - subscript of the pfn array for the starting pfn 1163 * end_idx - subscript of the last pfn in dma window 1164 * win_pp - pointer to win_next member of previous window 1165 * Return: DDI_SUCCESS - with **win_pp as newly created window object 1166 * DDI_DMA_NORESROUCE - caller frees all previous window objs 1167 * Note: Each cookie and window size are all initialized on page 1168 * boundary. This is not true for the 1st cookie of the 1st 1169 * window and the last cookie of the last window. 1170 * We fix that later in upper layer which has access to size 1171 * and offset info. 1172 * 1173 */ 1174 static int 1175 pci_dma_newwin(ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, uint32_t cookie_no, 1176 uint32_t start_idx, uint32_t end_idx, pci_dma_win_t **win_pp, 1177 uint64_t count_max, uint64_t bypass_prefix) 1178 { 1179 int (*waitfp)(caddr_t) = dmareq->dmar_fp; 1180 ddi_dma_cookie_t *cookie_p; 1181 uint32_t pfn_no = 1; 1182 iopfn_t pfn = PCI_GET_MP_PFN(mp, start_idx); 1183 iopfn_t prev_pfn = pfn; 1184 uint64_t seg_pfn0 = pfn; 1185 size_t sz = cookie_no * sizeof (ddi_dma_cookie_t); 1186 pci_dma_win_t *win_p = kmem_alloc(sizeof (pci_dma_win_t) + sz, 1187 waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP); 1188 if (!win_p) 1189 goto noresource; 1190 1191 win_p->win_next = NULL; 1192 win_p->win_ncookies = cookie_no; 1193 win_p->win_curseg = 0; /* start from segment 0 */ 1194 win_p->win_size = IOMMU_PTOB(end_idx - start_idx + 1); 1195 /* win_p->win_offset is left uninitialized */ 1196 1197 cookie_p = (ddi_dma_cookie_t *)(win_p + 1); 1198 start_idx++; 1199 for (; start_idx <= end_idx; start_idx++, prev_pfn = pfn, pfn_no++) { 1200 pfn = PCI_GET_MP_PFN1(mp, start_idx); 1201 if ((pfn == prev_pfn + 1) && 1202 (IOMMU_PTOB(pfn_no + 1) - 1 <= count_max)) 1203 continue; 1204 1205 /* close up the cookie up to (including) prev_pfn */ 1206 MAKE_DMA_COOKIE(cookie_p, IOMMU_PTOB(seg_pfn0) | bypass_prefix, 1207 IOMMU_PTOB(pfn_no)); 1208 DEBUG2(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages)\n", 1209 IOMMU_PTOB(seg_pfn0) | bypass_prefix, pfn_no); 1210 1211 cookie_p++; /* advance to next available cookie cell */ 1212 pfn_no = 0; 1213 seg_pfn0 = pfn; /* start a new segment from current pfn */ 1214 } 1215 MAKE_DMA_COOKIE(cookie_p, IOMMU_PTOB(seg_pfn0) | bypass_prefix, 1216 IOMMU_PTOB(pfn_no)); 1217 DEBUG3(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages) of total %x\n", 1218 IOMMU_PTOB(seg_pfn0) | bypass_prefix, pfn_no, cookie_no); 1219 #ifdef DEBUG 1220 cookie_p++; 1221 ASSERT((cookie_p - (ddi_dma_cookie_t *)(win_p + 1)) == cookie_no); 1222 #endif 1223 *win_pp = win_p; 1224 return (DDI_SUCCESS); 1225 noresource: 1226 if (waitfp != DDI_DMA_DONTWAIT) 1227 ddi_set_callback(waitfp, dmareq->dmar_arg, &pci_kmem_clid); 1228 return (DDI_DMA_NORESOURCES); 1229 } 1230 1231 /* 1232 * pci_dma_adjust - adjust 1st and last cookie and window sizes 1233 * remove initial dma page offset from 1st cookie and window size 1234 * remove last dma page remainder from last cookie and window size 1235 * fill win_offset of each dma window according to just fixed up 1236 * each window sizes 1237 * pci_dma_win_t members modified: 1238 * win_p->win_offset - this window's offset within entire DMA object 1239 * win_p->win_size - xferrable size (in bytes) for this window 1240 * 1241 * ddi_dma_impl_t members modified: 1242 * mp->dmai_size - 1st window xferrable size 1243 * mp->dmai_offset - 0, which is the dma offset of the 1st window 1244 * 1245 * ddi_dma_cookie_t members modified: 1246 * cookie_p->dmac_size - 1st and last cookie remove offset or remainder 1247 * cookie_p->dmac_laddress - 1st cookie add page offset 1248 */ 1249 static void 1250 pci_dma_adjust(ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, pci_dma_win_t *win_p) 1251 { 1252 ddi_dma_cookie_t *cookie_p = (ddi_dma_cookie_t *)(win_p + 1); 1253 size_t pg_offset = mp->dmai_roffset; 1254 size_t win_offset = 0; 1255 1256 cookie_p->dmac_size -= pg_offset; 1257 cookie_p->dmac_laddress |= pg_offset; 1258 win_p->win_size -= pg_offset; 1259 DEBUG1(DBG_BYPASS, mp->dmai_rdip, "pg0 adjust %lx\n", pg_offset); 1260 1261 mp->dmai_size = win_p->win_size; 1262 mp->dmai_offset = 0; 1263 1264 pg_offset += mp->dmai_object.dmao_size; 1265 pg_offset &= IOMMU_PAGE_OFFSET; 1266 if (pg_offset) 1267 pg_offset = IOMMU_PAGE_SIZE - pg_offset; 1268 DEBUG1(DBG_BYPASS, mp->dmai_rdip, "last pg adjust %lx\n", pg_offset); 1269 1270 for (; win_p->win_next; win_p = win_p->win_next) { 1271 DEBUG1(DBG_BYPASS, mp->dmai_rdip, "win off %p\n", win_offset); 1272 win_p->win_offset = win_offset; 1273 win_offset += win_p->win_size; 1274 } 1275 /* last window */ 1276 win_p->win_offset = win_offset; 1277 cookie_p = (ddi_dma_cookie_t *)(win_p + 1); 1278 cookie_p[win_p->win_ncookies - 1].dmac_size -= pg_offset; 1279 win_p->win_size -= pg_offset; 1280 ASSERT((win_offset + win_p->win_size) == mp->dmai_object.dmao_size); 1281 } 1282 1283 /* 1284 * pci_dma_physwin() - carve up dma windows using physical addresses. 1285 * Called to handle iommu bypass and pci peer-to-peer transfers. 1286 * Calls pci_dma_newwin() to allocate window objects. 1287 * 1288 * Dependency: mp->dmai_pfnlst points to an array of pfns 1289 * 1290 * 1. Each dma window is represented by a pci_dma_win_t object. 1291 * The object will be casted to ddi_dma_win_t and returned 1292 * to leaf driver through the DDI interface. 1293 * 2. Each dma window can have several dma segments with each 1294 * segment representing a physically contiguous either memory 1295 * space (if we are doing an iommu bypass transfer) or pci address 1296 * space (if we are doing a peer-to-peer transfer). 1297 * 3. Each segment has a DMA cookie to program the DMA engine. 1298 * The cookies within each DMA window must be located in a 1299 * contiguous array per ddi_dma_nextcookie(9f). 1300 * 4. The number of DMA segments within each DMA window cannot exceed 1301 * mp->dmai_attr.dma_attr_sgllen. If the transfer size is 1302 * too large to fit in the sgllen, the rest needs to be 1303 * relocated to the next dma window. 1304 * 5. Peer-to-peer DMA segment follows device hi, lo, count_max, 1305 * and nocross restrictions while bypass DMA follows the set of 1306 * restrictions with system limits factored in. 1307 * 1308 * Return: 1309 * mp->dmai_winlst - points to a link list of pci_dma_win_t objects. 1310 * Each pci_dma_win_t object on the link list contains 1311 * infomation such as its window size (# of pages), 1312 * starting offset (also see Restriction), an array of 1313 * DMA cookies, and # of cookies in the array. 1314 * mp->dmai_pfnlst - NULL, the pfn list is freed to conserve memory. 1315 * mp->dmai_nwin - # of total DMA windows on mp->dmai_winlst. 1316 * mp->dmai_mapping - starting cookie address 1317 * mp->dmai_rflags - consistent, nosync, no redzone 1318 * mp->dmai_cookie - start of cookie table of the 1st DMA window 1319 * 1320 * Restriction: 1321 * Each pci_dma_win_t object can theoratically start from any offset 1322 * since the iommu is not involved. However, this implementation 1323 * always make windows start from page aligned offset (except 1324 * the 1st window, which follows the requested offset) due to the 1325 * fact that we are handed a pfn list. This does require device's 1326 * count_max and attr_seg to be at least IOMMU_PAGE_SIZE aligned. 1327 */ 1328 int 1329 pci_dma_physwin(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 1330 { 1331 uint_t npages = mp->dmai_ndvmapages; 1332 int ret, sgllen = mp->dmai_attr.dma_attr_sgllen; 1333 iopfn_t pfn_lo, pfn_hi, prev_pfn, bypass_pfn; 1334 iopfn_t pfn = PCI_GET_MP_PFN(mp, 0); 1335 uint32_t i, win_no = 0, pfn_no = 1, win_pfn0_index = 0, cookie_no = 0; 1336 uint64_t count_max, bypass = PCI_DMA_BYPASS_PREFIX(mp, pfn); 1337 pci_dma_win_t **win_pp = (pci_dma_win_t **)&mp->dmai_winlst; 1338 ddi_dma_cookie_t *cookie0_p; 1339 1340 if (PCI_DMA_ISPTP(mp)) { /* ignore sys limits for peer-to-peer */ 1341 ddi_dma_attr_t *dev_attr_p = DEV_ATTR(mp); 1342 iopfn_t pfn_base = pci_p->pci_pbm_p->pbm_base_pfn; 1343 iopfn_t pfn_last = pci_p->pci_pbm_p->pbm_last_pfn - pfn_base; 1344 uint64_t nocross = dev_attr_p->dma_attr_seg; 1345 if (nocross && (nocross < UINT32_MAX)) 1346 return (DDI_DMA_NOMAPPING); 1347 if (dev_attr_p->dma_attr_align > IOMMU_PAGE_SIZE) 1348 return (DDI_DMA_NOMAPPING); 1349 pfn_lo = IOMMU_BTOP(dev_attr_p->dma_attr_addr_lo); 1350 pfn_hi = IOMMU_BTOP(dev_attr_p->dma_attr_addr_hi); 1351 pfn_hi = MIN(pfn_hi, pfn_last); 1352 if ((pfn_lo > pfn_hi) || (pfn < pfn_lo)) 1353 return (DDI_DMA_NOMAPPING); 1354 count_max = dev_attr_p->dma_attr_count_max; 1355 count_max = MIN(count_max, nocross); 1356 /* 1357 * the following count_max trim is not done because we are 1358 * making sure pfn_lo <= pfn <= pfn_hi inside the loop 1359 * count_max=MIN(count_max, IOMMU_PTOB(pfn_hi - pfn_lo + 1)-1); 1360 */ 1361 } else { /* bypass hi/lo/count_max have been processed by attr2hdl() */ 1362 count_max = mp->dmai_attr.dma_attr_count_max; 1363 pfn_lo = IOMMU_BTOP(mp->dmai_attr.dma_attr_addr_lo); 1364 pfn_hi = IOMMU_BTOP(mp->dmai_attr.dma_attr_addr_hi); 1365 } 1366 1367 bypass_pfn = IOMMU_BTOP(bypass); 1368 1369 for (prev_pfn = (bypass_pfn | pfn), i = 1; i < npages; 1370 i++, prev_pfn = pfn, pfn_no++) { 1371 pfn = bypass_pfn | PCI_GET_MP_PFN1(mp, i); 1372 if ((pfn == prev_pfn + 1) && 1373 (IOMMU_PTOB(pfn_no + 1) - 1 <= count_max)) 1374 continue; 1375 if ((pfn < pfn_lo) || (prev_pfn > pfn_hi)) { 1376 ret = DDI_DMA_NOMAPPING; 1377 goto err; 1378 } 1379 cookie_no++; 1380 pfn_no = 0; 1381 if (cookie_no < sgllen) 1382 continue; 1383 1384 DEBUG3(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n", 1385 win_pfn0_index, i - 1, cookie_no); 1386 if (ret = pci_dma_newwin(dmareq, mp, cookie_no, 1387 win_pfn0_index, i - 1, win_pp, count_max, bypass)) 1388 goto err; 1389 1390 win_pp = &(*win_pp)->win_next; /* win_pp = *(win_pp) */ 1391 win_no++; 1392 win_pfn0_index = i; 1393 cookie_no = 0; 1394 } 1395 if (pfn > pfn_hi) { 1396 ret = DDI_DMA_NOMAPPING; 1397 goto err; 1398 } 1399 cookie_no++; 1400 DEBUG3(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n", 1401 win_pfn0_index, i - 1, cookie_no); 1402 if (ret = pci_dma_newwin(dmareq, mp, cookie_no, win_pfn0_index, 1403 i - 1, win_pp, count_max, bypass)) 1404 goto err; 1405 win_no++; 1406 pci_dma_adjust(dmareq, mp, mp->dmai_winlst); 1407 mp->dmai_nwin = win_no; 1408 mp->dmai_rflags |= DDI_DMA_CONSISTENT; 1409 if (!pci_p->pci_pbm_p->pbm_sync_reg_pa) { 1410 mp->dmai_rflags |= DMP_NOSYNC; 1411 mp->dmai_flags |= DMAI_FLAGS_NOSYNC; 1412 } 1413 mp->dmai_rflags &= ~DDI_DMA_REDZONE; 1414 cookie0_p = (ddi_dma_cookie_t *)(WINLST(mp) + 1); 1415 mp->dmai_cookie = cookie0_p + 1; 1416 mp->dmai_mapping = cookie0_p->dmac_laddress; 1417 mp->dmai_ncookies = WINLST(mp)->win_ncookies; 1418 mp->dmai_curcookie = 1; 1419 1420 pci_dma_freepfn(mp); 1421 return (DDI_DMA_MAPPED); 1422 err: 1423 pci_dma_freewin(mp); 1424 return (ret); 1425 } 1426 1427 /*ARGSUSED*/ 1428 int 1429 pci_dma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp, 1430 enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp, 1431 uint_t cache_flags) 1432 { 1433 switch (cmd) { 1434 1435 case DDI_DMA_HTOC: { 1436 off_t off = *offp; 1437 ddi_dma_cookie_t *loop_cp, *cp; 1438 pci_dma_win_t *win_p = mp->dmai_winlst; 1439 1440 if (off >= mp->dmai_object.dmao_size) 1441 return (DDI_FAILURE); 1442 1443 /* locate window */ 1444 while (win_p->win_offset + win_p->win_size <= off) 1445 win_p = win_p->win_next; 1446 1447 loop_cp = cp = (ddi_dma_cookie_t *)(win_p + 1); 1448 mp->dmai_offset = win_p->win_offset; 1449 mp->dmai_size = win_p->win_size; 1450 mp->dmai_mapping = cp->dmac_laddress; /* cookie0 start addr */ 1451 1452 /* adjust cookie addr/len if we are not on cookie boundary */ 1453 off -= win_p->win_offset; /* offset within window */ 1454 for (; off >= loop_cp->dmac_size; loop_cp++) 1455 off -= loop_cp->dmac_size; /* offset within cookie */ 1456 1457 mp->dmai_cookie = loop_cp + 1; 1458 win_p->win_curseg = loop_cp - cp; 1459 cp = (ddi_dma_cookie_t *)objp; 1460 MAKE_DMA_COOKIE(cp, loop_cp->dmac_laddress + off, 1461 loop_cp->dmac_size - off); 1462 1463 DEBUG2(DBG_DMA_CTL, dip, 1464 "HTOC: cookie - dmac_laddress=%p dmac_size=%x\n", 1465 cp->dmac_laddress, cp->dmac_size); 1466 } 1467 return (DDI_SUCCESS); 1468 1469 case DDI_DMA_COFF: { 1470 pci_dma_win_t *win_p; 1471 ddi_dma_cookie_t *cp; 1472 uint64_t addr, key = ((ddi_dma_cookie_t *)offp)->dmac_laddress; 1473 size_t win_off; 1474 1475 for (win_p = mp->dmai_winlst; win_p; win_p = win_p->win_next) { 1476 int i; 1477 win_off = 0; 1478 cp = (ddi_dma_cookie_t *)(win_p + 1); 1479 for (i = 0; i < win_p->win_ncookies; i++, cp++) { 1480 size_t sz = cp->dmac_size; 1481 1482 addr = cp->dmac_laddress; 1483 if ((addr <= key) && (addr + sz >= key)) 1484 goto found; 1485 win_off += sz; 1486 } 1487 } 1488 return (DDI_FAILURE); 1489 found: 1490 *objp = (caddr_t)(win_p->win_offset + win_off + (key - addr)); 1491 return (DDI_SUCCESS); 1492 } 1493 1494 case DDI_DMA_REMAP: 1495 return (DDI_FAILURE); 1496 1497 default: 1498 DEBUG3(DBG_DMA_CTL, dip, "unknown command (%x): rdip=%s%d\n", 1499 cmd, ddi_driver_name(rdip), ddi_get_instance(rdip)); 1500 break; 1501 } 1502 return (DDI_FAILURE); 1503 } 1504 1505 static void 1506 pci_dvma_debug_init(iommu_t *iommu_p) 1507 { 1508 size_t sz = sizeof (struct dvma_rec) * pci_dvma_debug_rec; 1509 ASSERT(MUTEX_HELD(&iommu_p->dvma_debug_lock)); 1510 cmn_err(CE_NOTE, "PCI DVMA %p stat ON", iommu_p); 1511 1512 iommu_p->dvma_alloc_rec = kmem_zalloc(sz, KM_SLEEP); 1513 iommu_p->dvma_free_rec = kmem_zalloc(sz, KM_SLEEP); 1514 1515 iommu_p->dvma_active_list = NULL; 1516 iommu_p->dvma_alloc_rec_index = 0; 1517 iommu_p->dvma_free_rec_index = 0; 1518 iommu_p->dvma_active_count = 0; 1519 } 1520 1521 void 1522 pci_dvma_debug_fini(iommu_t *iommu_p) 1523 { 1524 struct dvma_rec *prev, *ptr; 1525 size_t sz = sizeof (struct dvma_rec) * pci_dvma_debug_rec; 1526 uint64_t mask = ~(1ull << iommu_p->iommu_inst); 1527 cmn_err(CE_NOTE, "PCI DVMA %p stat OFF", iommu_p); 1528 1529 kmem_free(iommu_p->dvma_alloc_rec, sz); 1530 kmem_free(iommu_p->dvma_free_rec, sz); 1531 iommu_p->dvma_alloc_rec = iommu_p->dvma_free_rec = NULL; 1532 1533 prev = iommu_p->dvma_active_list; 1534 if (!prev) 1535 return; 1536 for (ptr = prev->next; ptr; prev = ptr, ptr = ptr->next) 1537 kmem_free(prev, sizeof (struct dvma_rec)); 1538 kmem_free(prev, sizeof (struct dvma_rec)); 1539 1540 iommu_p->dvma_active_list = NULL; 1541 iommu_p->dvma_alloc_rec_index = 0; 1542 iommu_p->dvma_free_rec_index = 0; 1543 iommu_p->dvma_active_count = 0; 1544 1545 pci_dvma_debug_on &= mask; 1546 pci_dvma_debug_off &= mask; 1547 } 1548 1549 void 1550 pci_dvma_alloc_debug(iommu_t *iommu_p, char *address, uint_t len, 1551 ddi_dma_impl_t *mp) 1552 { 1553 struct dvma_rec *ptr; 1554 mutex_enter(&iommu_p->dvma_debug_lock); 1555 1556 if (!iommu_p->dvma_alloc_rec) 1557 pci_dvma_debug_init(iommu_p); 1558 if (DVMA_DBG_OFF(iommu_p)) { 1559 pci_dvma_debug_fini(iommu_p); 1560 goto done; 1561 } 1562 1563 ptr = &iommu_p->dvma_alloc_rec[iommu_p->dvma_alloc_rec_index]; 1564 ptr->dvma_addr = address; 1565 ptr->len = len; 1566 ptr->mp = mp; 1567 if (++iommu_p->dvma_alloc_rec_index == pci_dvma_debug_rec) 1568 iommu_p->dvma_alloc_rec_index = 0; 1569 1570 ptr = kmem_alloc(sizeof (struct dvma_rec), KM_SLEEP); 1571 ptr->dvma_addr = address; 1572 ptr->len = len; 1573 ptr->mp = mp; 1574 1575 ptr->next = iommu_p->dvma_active_list; 1576 iommu_p->dvma_active_list = ptr; 1577 iommu_p->dvma_active_count++; 1578 done: 1579 mutex_exit(&iommu_p->dvma_debug_lock); 1580 } 1581 1582 void 1583 pci_dvma_free_debug(iommu_t *iommu_p, char *address, uint_t len, 1584 ddi_dma_impl_t *mp) 1585 { 1586 struct dvma_rec *ptr, *ptr_save; 1587 mutex_enter(&iommu_p->dvma_debug_lock); 1588 1589 if (!iommu_p->dvma_alloc_rec) 1590 pci_dvma_debug_init(iommu_p); 1591 if (DVMA_DBG_OFF(iommu_p)) { 1592 pci_dvma_debug_fini(iommu_p); 1593 goto done; 1594 } 1595 1596 ptr = &iommu_p->dvma_free_rec[iommu_p->dvma_free_rec_index]; 1597 ptr->dvma_addr = address; 1598 ptr->len = len; 1599 ptr->mp = mp; 1600 if (++iommu_p->dvma_free_rec_index == pci_dvma_debug_rec) 1601 iommu_p->dvma_free_rec_index = 0; 1602 1603 ptr_save = iommu_p->dvma_active_list; 1604 for (ptr = ptr_save; ptr; ptr = ptr->next) { 1605 if ((ptr->dvma_addr == address) && (ptr->len = len)) 1606 break; 1607 ptr_save = ptr; 1608 } 1609 if (!ptr) { 1610 cmn_err(CE_WARN, "bad dvma free addr=%lx len=%x", 1611 (long)address, len); 1612 goto done; 1613 } 1614 if (ptr == iommu_p->dvma_active_list) 1615 iommu_p->dvma_active_list = ptr->next; 1616 else 1617 ptr_save->next = ptr->next; 1618 kmem_free(ptr, sizeof (struct dvma_rec)); 1619 iommu_p->dvma_active_count--; 1620 done: 1621 mutex_exit(&iommu_p->dvma_debug_lock); 1622 } 1623 1624 #ifdef DEBUG 1625 void 1626 dump_dma_handle(uint64_t flag, dev_info_t *dip, ddi_dma_impl_t *hp) 1627 { 1628 DEBUG4(flag, dip, "mp(%p): flags=%x mapping=%lx xfer_size=%x\n", 1629 hp, hp->dmai_inuse, hp->dmai_mapping, hp->dmai_size); 1630 DEBUG4(flag|DBG_CONT, dip, "\tnpages=%x roffset=%x rflags=%x nwin=%x\n", 1631 hp->dmai_ndvmapages, hp->dmai_roffset, hp->dmai_rflags, 1632 hp->dmai_nwin); 1633 DEBUG4(flag|DBG_CONT, dip, "\twinsize=%x tte=%p pfnlst=%p pfn0=%p\n", 1634 hp->dmai_winsize, hp->dmai_tte, hp->dmai_pfnlst, hp->dmai_pfn0); 1635 DEBUG4(flag|DBG_CONT, dip, "\twinlst=%x obj=%p attr=%p ckp=%p\n", 1636 hp->dmai_winlst, &hp->dmai_object, &hp->dmai_attr, 1637 hp->dmai_cookie); 1638 } 1639 #endif 1640 1641 void 1642 pci_vmem_do_free(iommu_t *iommu_p, void *base_addr, size_t npages, 1643 int vmemcache) 1644 { 1645 vmem_t *map_p = iommu_p->iommu_dvma_map; 1646 1647 if (vmemcache) { 1648 vmem_free(map_p, base_addr, IOMMU_PAGE_SIZE); 1649 #ifdef PCI_DMA_PROF 1650 pci_dvma_vmem_free++; 1651 #endif 1652 return; 1653 } 1654 1655 vmem_xfree(map_p, base_addr, IOMMU_PTOB(npages)); 1656 #ifdef PCI_DMA_PROF 1657 pci_dvma_vmem_xfree++; 1658 #endif 1659 } 1660