1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * PCI nexus DVMA and DMA core routines: 28 * dma_map/dma_bind_handle implementation 29 * bypass and peer-to-peer support 30 * fast track DVMA space allocation 31 * runtime DVMA debug 32 */ 33 #include <sys/types.h> 34 #include <sys/kmem.h> 35 #include <sys/async.h> 36 #include <sys/sysmacros.h> 37 #include <sys/sunddi.h> 38 #include <sys/machsystm.h> /* lddphys() */ 39 #include <sys/ddi_impldefs.h> 40 #include <vm/hat.h> 41 #include <sys/pci/pci_obj.h> 42 43 /*LINTLIBRARY*/ 44 45 static void 46 pci_sc_pg_inv(dev_info_t *dip, sc_t *sc_p, ddi_dma_impl_t *mp, off_t off, 47 size_t len) 48 { 49 dvma_addr_t dvma_addr, pg_off; 50 volatile uint64_t *invl_va = sc_p->sc_invl_reg; 51 52 if (!len) 53 len = mp->dmai_size; 54 55 pg_off = mp->dmai_offset; /* start min */ 56 dvma_addr = MAX(off, pg_off); /* lo */ 57 pg_off += mp->dmai_size; /* end max */ 58 pg_off = MIN(off + len, pg_off); /* hi */ 59 if (dvma_addr >= pg_off) { /* lo >= hi ? */ 60 DEBUG4(DBG_SC, dip, "%x+%x out of window [%x,%x)\n", 61 off, len, mp->dmai_offset, 62 mp->dmai_offset + mp->dmai_size); 63 return; 64 } 65 66 len = pg_off - dvma_addr; /* sz = hi - lo */ 67 dvma_addr += mp->dmai_mapping; /* start addr */ 68 pg_off = dvma_addr & IOMMU_PAGE_OFFSET; /* offset in 1st pg */ 69 len = IOMMU_BTOPR(len + pg_off); /* # of pages */ 70 dvma_addr ^= pg_off; 71 72 DEBUG2(DBG_SC, dip, "addr=%x+%x pages: \n", dvma_addr, len); 73 for (; len; len--, dvma_addr += IOMMU_PAGE_SIZE) { 74 DEBUG1(DBG_SC|DBG_CONT, dip, " %x", dvma_addr); 75 *invl_va = (uint64_t)dvma_addr; 76 } 77 DEBUG0(DBG_SC|DBG_CONT, dip, "\n"); 78 } 79 80 static void 81 pci_dma_sync_flag_wait(ddi_dma_impl_t *mp, sc_t *sc_p, uint32_t onstack) 82 { 83 hrtime_t start_time; 84 uint64_t loops = 0; 85 uint64_t sync_flag_pa = SYNC_BUF_PA(mp); 86 uint64_t sync_reg_pa = sc_p->sc_sync_reg_pa; 87 uint8_t stack_buf[128]; 88 89 stack_buf[0] = DDI_SUCCESS; 90 91 /* check for handle specific sync flag */ 92 if (sync_flag_pa) 93 goto start; 94 95 sync_flag_pa = sc_p->sc_sync_flag_pa; 96 97 if (onstack) { 98 sync_flag_pa = va_to_pa(stack_buf); 99 sync_flag_pa += PCI_SYNC_FLAG_SIZE; 100 sync_flag_pa >>= PCI_SYNC_FLAG_SZSHIFT; 101 sync_flag_pa <<= PCI_SYNC_FLAG_SZSHIFT; 102 goto start; 103 } 104 stack_buf[0] |= PCI_SYNC_FLAG_LOCKED; 105 mutex_enter(&sc_p->sc_sync_mutex); 106 start: 107 ASSERT(!(sync_flag_pa & PCI_SYNC_FLAG_SIZE - 1)); 108 stdphys(sync_flag_pa, 0); /* reset sync flag to 0 */ 109 /* membar #LoadStore|#StoreStore */ 110 stdphysio(sync_reg_pa, sync_flag_pa); 111 start_time = gethrtime(); 112 113 for (; gethrtime() - start_time < pci_sync_buf_timeout; loops++) 114 if (lddphys(sync_flag_pa)) 115 goto done; 116 117 if (!lddphys(sync_flag_pa)) 118 stack_buf[0] |= PCI_SYNC_FLAG_FAILED; 119 done: 120 DEBUG3(DBG_SC|DBG_CONT, 0, "flag wait loops=%lu ticks=%lu status=%x\n", 121 loops, gethrtime() - start_time, stack_buf[0]); 122 123 if (stack_buf[0] & PCI_SYNC_FLAG_LOCKED) 124 mutex_exit(&sc_p->sc_sync_mutex); 125 126 if (stack_buf[0] & PCI_SYNC_FLAG_FAILED) 127 cmn_err(CE_PANIC, "%p pci dma sync %lx %lx timeout!", 128 mp, sync_flag_pa, loops); 129 } 130 131 /* 132 * Cache RW Before During After 133 * 134 * STREAMING read no/no pg/no ctx,pg/no 135 * STREAMING write no/no pg/yes ctx,pg/yes 136 * CONSISTENT read no/no yes,no/no yes,no/no 137 * CONSISTENT write no/no yes,yes/yes yes,yes/yes 138 * 139 * STREAMING read ctx,pg/no 140 * STREAMING write ctx,pg/yes 141 * CONSISTENT read yes,no/no 142 * CONSISTENT write yes,yes/yes 143 */ 144 int 145 pci_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 146 off_t off, size_t len, uint32_t sync_flag) 147 { 148 ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle; 149 int ret = ddi_get_instance(dip); 150 pci_t *pci_p = get_pci_soft_state(ret); 151 pbm_t *pbm_p = pci_p->pci_pbm_p; 152 uint32_t dev_flag = mp->dmai_rflags; 153 sc_t *sc_p; 154 155 DEBUG4(DBG_DMA_SYNC, dip, "%s%d flags=%x,%x\n", ddi_driver_name(rdip), 156 ddi_get_instance(rdip), dev_flag, sync_flag); 157 DEBUG4(DBG_SC, dip, "dmai_mapping=%x, dmai_sz=%x off=%x len=%x\n", 158 mp->dmai_mapping, mp->dmai_size, off, len); 159 DEBUG2(DBG_SC, dip, "mp=%p, ctx=%x\n", mp, MP2CTX(mp)); 160 161 if (!(mp->dmai_flags & DMAI_FLAGS_INUSE)) { 162 cmn_err(CE_WARN, "Unbound dma handle %p from %s%d", mp, 163 ddi_driver_name(rdip), ddi_get_instance(rdip)); 164 return (DDI_FAILURE); 165 } 166 167 if (mp->dmai_flags & DMAI_FLAGS_NOSYNC) 168 return (DDI_SUCCESS); 169 170 if (!(dev_flag & DDI_DMA_CONSISTENT)) 171 goto streaming; 172 173 if (sync_flag & PCI_DMA_SYNC_EXT) { 174 if (sync_flag & (PCI_DMA_SYNC_BEFORE | PCI_DMA_SYNC_POST) || 175 !(sync_flag & PCI_DMA_SYNC_WRITE)) 176 return (DDI_SUCCESS); 177 } else { 178 if (!(dev_flag & DDI_DMA_READ) || 179 ((sync_flag & PCI_DMA_SYNC_DDI_FLAGS) == 180 DDI_DMA_SYNC_FORDEV)) 181 return (DDI_SUCCESS); 182 } 183 184 pci_pbm_dma_sync(pbm_p, pbm_p->pbm_sync_ino); 185 return (DDI_SUCCESS); 186 187 streaming: 188 ASSERT(pci_stream_buf_exists && (pci_stream_buf_enable & 1 << ret)); 189 sc_p = pci_p->pci_sc_p; 190 ret = DDI_FAILURE; 191 192 if (sync_flag & PCI_DMA_SYNC_EXT) 193 goto ext; 194 195 if (mp->dmai_flags & DMAI_FLAGS_CONTEXT && pci_sc_use_contexts) 196 ret = pci_sc_ctx_inv(dip, sc_p, mp); 197 if (ret) 198 pci_sc_pg_inv(dip, sc_p, mp, off, len); 199 200 if ((dev_flag & DDI_DMA_READ) && 201 ((sync_flag & PCI_DMA_SYNC_DDI_FLAGS) != DDI_DMA_SYNC_FORDEV)) 202 goto wait; 203 204 return (DDI_SUCCESS); 205 ext: 206 if (sync_flag & PCI_DMA_SYNC_BEFORE) 207 return (DDI_SUCCESS); 208 if (sync_flag & PCI_DMA_SYNC_BAR) 209 goto wait_check; 210 if (sync_flag & PCI_DMA_SYNC_AFTER && 211 mp->dmai_flags & DMAI_FLAGS_CONTEXT && pci_sc_use_contexts) 212 ret = pci_sc_ctx_inv(dip, sc_p, mp); 213 if (ret) 214 pci_sc_pg_inv(dip, sc_p, mp, off, len); 215 wait_check: 216 if (sync_flag & PCI_DMA_SYNC_POST || !(sync_flag & PCI_DMA_SYNC_WRITE)) 217 return (DDI_SUCCESS); 218 wait: 219 pci_dma_sync_flag_wait(mp, sc_p, sync_flag & PCI_DMA_SYNC_PRIVATE); 220 return (DDI_SUCCESS); 221 } 222 223 int 224 pci_dma_handle_clean(dev_info_t *rdip, ddi_dma_handle_t h) 225 { 226 ddi_dma_impl_t *mp = (ddi_dma_impl_t *)h; 227 if ((mp->dmai_flags & DMAI_FLAGS_INUSE) == 0) 228 return (DDI_FAILURE); 229 mp->dmai_rflags |= DMP_NOSYNC; 230 mp->dmai_flags |= DMAI_FLAGS_NOSYNC; 231 return (DDI_SUCCESS); 232 } 233 234 /* 235 * pci_dma_allocmp - Allocate a pci dma implementation structure 236 * 237 * An extra ddi_dma_attr structure is bundled with the usual ddi_dma_impl 238 * to hold unmodified device limits. The ddi_dma_attr inside the 239 * ddi_dma_impl structure is augumented with system limits to enhance 240 * DVMA performance at runtime. The unaugumented device limits saved 241 * right after (accessed through the DEV_ATTR macro) is used 242 * strictly for peer-to-peer transfers which do not obey system limits. 243 * 244 * return: DDI_SUCCESS DDI_DMA_NORESOURCES 245 */ 246 ddi_dma_impl_t * 247 pci_dma_allocmp(dev_info_t *dip, dev_info_t *rdip, int (*waitfp)(caddr_t), 248 caddr_t arg) 249 { 250 ddi_dma_impl_t *mp; 251 int sleep = (waitfp == DDI_DMA_SLEEP) ? KM_SLEEP : KM_NOSLEEP; 252 253 /* Caution: we don't use zalloc to enhance performance! */ 254 if ((mp = kmem_alloc(sizeof (pci_dma_hdl_t), sleep)) == 0) { 255 DEBUG0(DBG_DMA_MAP, dip, "can't alloc dma_handle\n"); 256 if (waitfp != DDI_DMA_DONTWAIT) { 257 DEBUG0(DBG_DMA_MAP, dip, "alloc_mp kmem cb\n"); 258 ddi_set_callback(waitfp, arg, &pci_kmem_clid); 259 } 260 return (mp); 261 } 262 263 mp->dmai_rdip = rdip; 264 mp->dmai_flags = 0; 265 mp->dmai_pfnlst = NULL; 266 mp->dmai_winlst = NULL; 267 268 /* 269 * kmem_alloc debug: the following fields are not zero-ed 270 * mp->dmai_mapping = 0; 271 * mp->dmai_size = 0; 272 * mp->dmai_offset = 0; 273 * mp->dmai_minxfer = 0; 274 * mp->dmai_burstsizes = 0; 275 * mp->dmai_ndvmapages = 0; 276 * mp->dmai_pool/roffset = 0; 277 * mp->dmai_rflags = 0; 278 * mp->dmai_inuse/flags 279 * mp->dmai_nwin = 0; 280 * mp->dmai_winsize = 0; 281 * mp->dmai_nexus_private/tte = 0; 282 * mp->dmai_iopte/pfnlst 283 * mp->dmai_sbi/pfn0 = 0; 284 * mp->dmai_minfo/winlst/fdvma 285 * mp->dmai_rdip 286 * bzero(&mp->dmai_object, sizeof (ddi_dma_obj_t)); 287 * mp->dmai_cookie = 0; 288 */ 289 290 mp->dmai_attr.dma_attr_version = (uint_t)DMA_ATTR_VERSION; 291 mp->dmai_attr.dma_attr_flags = (uint_t)0; 292 mp->dmai_fault = 0; 293 mp->dmai_fault_check = NULL; 294 mp->dmai_fault_notify = NULL; 295 296 mp->dmai_error.err_ena = 0; 297 mp->dmai_error.err_status = DDI_FM_OK; 298 mp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 299 mp->dmai_error.err_ontrap = NULL; 300 mp->dmai_error.err_fep = NULL; 301 mp->dmai_error.err_cf = NULL; 302 ndi_fmc_insert(rdip, DMA_HANDLE, mp, NULL); 303 304 SYNC_BUF_PA(mp) = 0ull; 305 return (mp); 306 } 307 308 void 309 pci_dma_freemp(ddi_dma_impl_t *mp) 310 { 311 ndi_fmc_remove(mp->dmai_rdip, DMA_HANDLE, mp); 312 if (mp->dmai_ndvmapages > 1) 313 pci_dma_freepfn(mp); 314 if (mp->dmai_winlst) 315 pci_dma_freewin(mp); 316 kmem_free(mp, sizeof (pci_dma_hdl_t)); 317 } 318 319 void 320 pci_dma_freepfn(ddi_dma_impl_t *mp) 321 { 322 void *addr = mp->dmai_pfnlst; 323 ASSERT(!PCI_DMA_CANRELOC(mp)); 324 if (addr) { 325 size_t npages = mp->dmai_ndvmapages; 326 if (npages > 1) 327 kmem_free(addr, npages * sizeof (iopfn_t)); 328 mp->dmai_pfnlst = NULL; 329 } 330 mp->dmai_ndvmapages = 0; 331 } 332 333 /* 334 * pci_dma_lmts2hdl - alloate a ddi_dma_impl_t, validate practical limits 335 * and convert dmareq->dmar_limits to mp->dmai_attr 336 * 337 * ddi_dma_impl_t member modified input 338 * ------------------------------------------------------------------------ 339 * mp->dmai_minxfer - dev 340 * mp->dmai_burstsizes - dev 341 * mp->dmai_flags - no limit? peer-to-peer only? 342 * 343 * ddi_dma_attr member modified input 344 * ------------------------------------------------------------------------ 345 * mp->dmai_attr.dma_attr_addr_lo - dev lo, sys lo 346 * mp->dmai_attr.dma_attr_addr_hi - dev hi, sys hi 347 * mp->dmai_attr.dma_attr_count_max - dev count max, dev/sys lo/hi delta 348 * mp->dmai_attr.dma_attr_seg - 0 (no nocross restriction) 349 * mp->dmai_attr.dma_attr_align - 1 (no alignment restriction) 350 * 351 * The dlim_dmaspeed member of dmareq->dmar_limits is ignored. 352 */ 353 ddi_dma_impl_t * 354 pci_dma_lmts2hdl(dev_info_t *dip, dev_info_t *rdip, iommu_t *iommu_p, 355 ddi_dma_req_t *dmareq) 356 { 357 ddi_dma_impl_t *mp; 358 ddi_dma_attr_t *attr_p; 359 uint64_t syslo = iommu_p->iommu_dvma_base; 360 uint64_t syshi = iommu_p->iommu_dvma_end; 361 uint64_t fasthi = iommu_p->iommu_dvma_fast_end; 362 ddi_dma_lim_t *lim_p = dmareq->dmar_limits; 363 uint32_t count_max = lim_p->dlim_cntr_max; 364 uint64_t lo = lim_p->dlim_addr_lo; 365 uint64_t hi = lim_p->dlim_addr_hi; 366 if (hi <= lo) { 367 DEBUG0(DBG_DMA_MAP, dip, "Bad limits\n"); 368 return ((ddi_dma_impl_t *)DDI_DMA_NOMAPPING); 369 } 370 if (!count_max) 371 count_max--; 372 373 if (!(mp = pci_dma_allocmp(dip, rdip, dmareq->dmar_fp, 374 dmareq->dmar_arg))) 375 return (NULL); 376 377 /* store original dev input at the 2nd ddi_dma_attr */ 378 attr_p = DEV_ATTR(mp); 379 SET_DMAATTR(attr_p, lo, hi, -1, count_max); 380 SET_DMAALIGN(attr_p, 1); 381 382 lo = MAX(lo, syslo); 383 hi = MIN(hi, syshi); 384 if (hi <= lo) 385 mp->dmai_flags |= DMAI_FLAGS_PEER_ONLY; 386 count_max = MIN(count_max, hi - lo); 387 388 if (DEV_NOSYSLIMIT(lo, hi, syslo, fasthi, 1)) 389 mp->dmai_flags |= DMAI_FLAGS_NOFASTLIMIT | 390 DMAI_FLAGS_NOSYSLIMIT; 391 else { 392 if (DEV_NOFASTLIMIT(lo, hi, syslo, syshi, 1)) 393 mp->dmai_flags |= DMAI_FLAGS_NOFASTLIMIT; 394 } 395 if (PCI_DMA_NOCTX(rdip)) 396 mp->dmai_flags |= DMAI_FLAGS_NOCTX; 397 398 /* store augumented dev input to mp->dmai_attr */ 399 mp->dmai_minxfer = lim_p->dlim_minxfer; 400 mp->dmai_burstsizes = lim_p->dlim_burstsizes; 401 attr_p = &mp->dmai_attr; 402 SET_DMAATTR(attr_p, lo, hi, -1, count_max); 403 SET_DMAALIGN(attr_p, 1); 404 return (mp); 405 } 406 407 /* 408 * pci_dma_attr2hdl 409 * 410 * This routine is called from the alloc handle entry point to sanity check the 411 * dma attribute structure. 412 * 413 * use by: pci_dma_allochdl() 414 * 415 * return value: 416 * 417 * DDI_SUCCESS - on success 418 * DDI_DMA_BADATTR - attribute has invalid version number 419 * or address limits exclude dvma space 420 */ 421 int 422 pci_dma_attr2hdl(pci_t *pci_p, ddi_dma_impl_t *mp) 423 { 424 iommu_t *iommu_p = pci_p->pci_iommu_p; 425 uint64_t syslo, syshi; 426 ddi_dma_attr_t *attrp = DEV_ATTR(mp); 427 uint64_t hi = attrp->dma_attr_addr_hi; 428 uint64_t lo = attrp->dma_attr_addr_lo; 429 uint64_t align = attrp->dma_attr_align; 430 uint64_t nocross = attrp->dma_attr_seg; 431 uint64_t count_max = attrp->dma_attr_count_max; 432 433 DEBUG3(DBG_DMA_ALLOCH, pci_p->pci_dip, "attrp=%p cntr_max=%x.%08x\n", 434 attrp, HI32(count_max), LO32(count_max)); 435 DEBUG4(DBG_DMA_ALLOCH, pci_p->pci_dip, "hi=%x.%08x lo=%x.%08x\n", 436 HI32(hi), LO32(hi), HI32(lo), LO32(lo)); 437 DEBUG4(DBG_DMA_ALLOCH, pci_p->pci_dip, "seg=%x.%08x align=%x.%08x\n", 438 HI32(nocross), LO32(nocross), HI32(align), LO32(align)); 439 440 if (!nocross) 441 nocross--; 442 if (attrp->dma_attr_flags & DDI_DMA_FORCE_PHYSICAL) { /* BYPASS */ 443 444 DEBUG0(DBG_DMA_ALLOCH, pci_p->pci_dip, "bypass mode\n"); 445 /* if tomatillo ver <= 2.3 don't allow bypass */ 446 if (tomatillo_disallow_bypass) 447 return (DDI_DMA_BADATTR); 448 449 mp->dmai_flags |= DMAI_FLAGS_BYPASSREQ; 450 if (nocross != UINT64_MAX) 451 return (DDI_DMA_BADATTR); 452 if (align && (align > IOMMU_PAGE_SIZE)) 453 return (DDI_DMA_BADATTR); 454 align = 1; /* align on 1 page boundary */ 455 syslo = iommu_p->iommu_dma_bypass_base; 456 syshi = iommu_p->iommu_dma_bypass_end; 457 458 } else { /* IOMMU_XLATE or PEER_TO_PEER */ 459 align = MAX(align, IOMMU_PAGE_SIZE) - 1; 460 if ((align & nocross) != align) { 461 dev_info_t *rdip = mp->dmai_rdip; 462 cmn_err(CE_WARN, "%s%d dma_attr_seg not aligned", 463 NAMEINST(rdip)); 464 return (DDI_DMA_BADATTR); 465 } 466 align = IOMMU_BTOP(align + 1); 467 syslo = iommu_p->iommu_dvma_base; 468 syshi = iommu_p->iommu_dvma_end; 469 } 470 if (hi <= lo) { 471 dev_info_t *rdip = mp->dmai_rdip; 472 cmn_err(CE_WARN, "%s%d limits out of range", NAMEINST(rdip)); 473 return (DDI_DMA_BADATTR); 474 } 475 lo = MAX(lo, syslo); 476 hi = MIN(hi, syshi); 477 if (!count_max) 478 count_max--; 479 480 DEBUG4(DBG_DMA_ALLOCH, pci_p->pci_dip, "hi=%x.%08x, lo=%x.%08x\n", 481 HI32(hi), LO32(hi), HI32(lo), LO32(lo)); 482 if (hi <= lo) { /* peer transfers cannot have alignment & nocross */ 483 dev_info_t *rdip = mp->dmai_rdip; 484 cmn_err(CE_WARN, "%s%d peer only dev %p", NAMEINST(rdip), mp); 485 if ((nocross < UINT32_MAX) || (align > 1)) { 486 cmn_err(CE_WARN, "%s%d peer only device bad attr", 487 NAMEINST(rdip)); 488 return (DDI_DMA_BADATTR); 489 } 490 mp->dmai_flags |= DMAI_FLAGS_PEER_ONLY; 491 } else /* set practical counter_max value */ 492 count_max = MIN(count_max, hi - lo); 493 494 if (DEV_NOSYSLIMIT(lo, hi, syslo, syshi, align)) 495 mp->dmai_flags |= DMAI_FLAGS_NOSYSLIMIT | 496 DMAI_FLAGS_NOFASTLIMIT; 497 else { 498 syshi = iommu_p->iommu_dvma_fast_end; 499 if (DEV_NOFASTLIMIT(lo, hi, syslo, syshi, align)) 500 mp->dmai_flags |= DMAI_FLAGS_NOFASTLIMIT; 501 } 502 if (PCI_DMA_NOCTX(mp->dmai_rdip)) 503 mp->dmai_flags |= DMAI_FLAGS_NOCTX; 504 505 mp->dmai_minxfer = attrp->dma_attr_minxfer; 506 mp->dmai_burstsizes = attrp->dma_attr_burstsizes; 507 attrp = &mp->dmai_attr; 508 SET_DMAATTR(attrp, lo, hi, nocross, count_max); 509 return (DDI_SUCCESS); 510 } 511 512 /* 513 * set up consistent dma flags according to hardware capability 514 */ 515 uint32_t 516 pci_dma_consist_check(uint32_t req_flags, pbm_t *pbm_p) 517 { 518 if (!pci_stream_buf_enable || !pci_stream_buf_exists) 519 req_flags |= DDI_DMA_CONSISTENT; 520 if (req_flags & DDI_DMA_CONSISTENT && !pbm_p->pbm_sync_reg_pa) 521 req_flags |= DMP_NOSYNC; 522 return (req_flags); 523 } 524 525 #define TGT_PFN_INBETWEEN(pfn, bgn, end) ((pfn >= bgn) && (pfn <= end)) 526 527 /* 528 * pci_dma_type - determine which of the three types DMA (peer-to-peer, 529 * iommu bypass, or iommu translate) we are asked to do. 530 * Also checks pfn0 and rejects any non-peer-to-peer 531 * requests for peer-only devices. 532 * 533 * return values: 534 * DDI_DMA_NOMAPPING - can't get valid pfn0, or bad dma type 535 * DDI_SUCCESS 536 * 537 * dma handle members affected (set on exit): 538 * mp->dmai_object - dmareq->dmar_object 539 * mp->dmai_rflags - consistent?, nosync?, dmareq->dmar_flags 540 * mp->dmai_flags - DMA type 541 * mp->dmai_pfn0 - 1st page pfn (if va/size pair and not shadow) 542 * mp->dmai_roffset - initialized to starting IOMMU page offset 543 * mp->dmai_ndvmapages - # of total IOMMU pages of entire object 544 * mp->pdh_sync_buf_pa - dma sync buffer PA is DMA flow is supported 545 */ 546 int 547 pci_dma_type(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 548 { 549 dev_info_t *dip = pci_p->pci_dip; 550 ddi_dma_obj_t *dobj_p = &dmareq->dmar_object; 551 pbm_t *pbm_p = pci_p->pci_pbm_p; 552 page_t **pplist; 553 struct as *as_p; 554 uint32_t offset; 555 caddr_t vaddr; 556 pfn_t pfn0; 557 558 mp->dmai_rflags = pci_dma_consist_check(dmareq->dmar_flags, pbm_p); 559 mp->dmai_flags |= mp->dmai_rflags & DMP_NOSYNC ? DMAI_FLAGS_NOSYNC : 0; 560 561 switch (dobj_p->dmao_type) { 562 case DMA_OTYP_BUFVADDR: 563 case DMA_OTYP_VADDR: { 564 vaddr = dobj_p->dmao_obj.virt_obj.v_addr; 565 pplist = dobj_p->dmao_obj.virt_obj.v_priv; 566 as_p = dobj_p->dmao_obj.virt_obj.v_as; 567 if (as_p == NULL) 568 as_p = &kas; 569 570 DEBUG2(DBG_DMA_MAP, dip, "vaddr=%p pplist=%p\n", vaddr, pplist); 571 offset = (ulong_t)vaddr & IOMMU_PAGE_OFFSET; 572 573 if (pplist) { /* shadow list */ 574 mp->dmai_flags |= DMAI_FLAGS_PGPFN; 575 ASSERT(PAGE_LOCKED(*pplist)); 576 pfn0 = page_pptonum(*pplist); 577 } else if (pci_dvma_remap_enabled && as_p == &kas && 578 dobj_p->dmao_type != DMA_OTYP_BUFVADDR) { 579 int (*waitfp)(caddr_t) = dmareq->dmar_fp; 580 uint_t flags = ((waitfp == DDI_DMA_SLEEP)? 581 HAC_SLEEP : HAC_NOSLEEP) | HAC_PAGELOCK; 582 int ret; 583 584 ret = hat_add_callback(pci_dvma_cbid, vaddr, 585 IOMMU_PAGE_SIZE - offset, flags, mp, &pfn0, 586 MP_HAT_CB_COOKIE_PTR(mp, 0)); 587 588 if (pfn0 == PFN_INVALID && ret == ENOMEM) { 589 ASSERT(waitfp != DDI_DMA_SLEEP); 590 if (waitfp != DDI_DMA_DONTWAIT) { 591 ddi_set_callback(waitfp, 592 dmareq->dmar_arg, 593 &pci_kmem_clid); 594 return (DDI_DMA_NORESOURCES); 595 } 596 } 597 mp->dmai_flags |= DMAI_FLAGS_RELOC; 598 } else 599 pfn0 = hat_getpfnum(as_p->a_hat, vaddr); 600 } 601 break; 602 603 case DMA_OTYP_PAGES: 604 offset = dobj_p->dmao_obj.pp_obj.pp_offset; 605 mp->dmai_flags |= DMAI_FLAGS_PGPFN; 606 pfn0 = page_pptonum(dobj_p->dmao_obj.pp_obj.pp_pp); 607 ASSERT(PAGE_LOCKED(dobj_p->dmao_obj.pp_obj.pp_pp)); 608 break; 609 610 case DMA_OTYP_PADDR: 611 default: 612 cmn_err(CE_WARN, "%s%d requested unsupported dma type %x", 613 NAMEINST(mp->dmai_rdip), dobj_p->dmao_type); 614 return (DDI_DMA_NOMAPPING); 615 } 616 if (pfn0 == PFN_INVALID) { 617 cmn_err(CE_WARN, "%s%d: invalid pfn0 for DMA object %p", 618 NAMEINST(dip), dobj_p); 619 return (DDI_DMA_NOMAPPING); 620 } 621 if (TGT_PFN_INBETWEEN(pfn0, pbm_p->pbm_base_pfn, pbm_p->pbm_last_pfn)) { 622 mp->dmai_flags |= DMAI_FLAGS_PEER_TO_PEER; 623 goto done; /* leave bypass and dvma flag as 0 */ 624 } 625 if (PCI_DMA_ISPEERONLY(mp)) { 626 dev_info_t *rdip = mp->dmai_rdip; 627 cmn_err(CE_WARN, "Bad peer-to-peer req %s%d", NAMEINST(rdip)); 628 return (DDI_DMA_NOMAPPING); 629 } 630 mp->dmai_flags |= (mp->dmai_flags & DMAI_FLAGS_BYPASSREQ) ? 631 DMAI_FLAGS_BYPASS : DMAI_FLAGS_DVMA; 632 done: 633 mp->dmai_object = *dobj_p; /* whole object */ 634 mp->dmai_pfn0 = (void *)pfn0; /* cache pfn0 */ 635 mp->dmai_roffset = offset; /* win0 pg0 offset */ 636 mp->dmai_ndvmapages = IOMMU_BTOPR(offset + mp->dmai_object.dmao_size); 637 638 return (DDI_SUCCESS); 639 } 640 641 /* 642 * pci_dma_pgpfn - set up pfnlst array according to pages 643 * VA/size pair: <shadow IO, bypass, peer-to-peer>, or OTYP_PAGES 644 */ 645 /*ARGSUSED*/ 646 static int 647 pci_dma_pgpfn(pci_t *pci_p, ddi_dma_impl_t *mp, uint_t npages) 648 { 649 int i; 650 #ifdef DEBUG 651 dev_info_t *dip = pci_p->pci_dip; 652 #endif 653 switch (mp->dmai_object.dmao_type) { 654 case DMA_OTYP_BUFVADDR: 655 case DMA_OTYP_VADDR: { 656 page_t **pplist = mp->dmai_object.dmao_obj.virt_obj.v_priv; 657 DEBUG2(DBG_DMA_MAP, dip, "shadow pplist=%p, %x pages, pfns=", 658 pplist, npages); 659 for (i = 1; i < npages; i++) { 660 iopfn_t pfn = page_pptonum(pplist[i]); 661 ASSERT(PAGE_LOCKED(pplist[i])); 662 PCI_SET_MP_PFN1(mp, i, pfn); 663 DEBUG1(DBG_DMA_MAP|DBG_CONT, dip, "%x ", pfn); 664 } 665 DEBUG0(DBG_DMA_MAP|DBG_CONT, dip, "\n"); 666 } 667 break; 668 669 case DMA_OTYP_PAGES: { 670 page_t *pp = mp->dmai_object.dmao_obj.pp_obj.pp_pp->p_next; 671 DEBUG1(DBG_DMA_MAP, dip, "pp=%p pfns=", pp); 672 for (i = 1; i < npages; i++, pp = pp->p_next) { 673 iopfn_t pfn = page_pptonum(pp); 674 ASSERT(PAGE_LOCKED(pp)); 675 PCI_SET_MP_PFN1(mp, i, pfn); 676 DEBUG1(DBG_DMA_MAP|DBG_CONT, dip, "%x ", pfn); 677 } 678 DEBUG0(DBG_DMA_MAP|DBG_CONT, dip, "\n"); 679 } 680 break; 681 682 default: /* check is already done by pci_dma_type */ 683 ASSERT(0); 684 break; 685 } 686 return (DDI_SUCCESS); 687 } 688 689 /* 690 * pci_dma_vapfn - set up pfnlst array according to VA 691 * VA/size pair: <normal, bypass, peer-to-peer> 692 * pfn0 is skipped as it is already done. 693 * In this case, the cached pfn0 is used to fill pfnlst[0] 694 */ 695 static int 696 pci_dma_vapfn(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, 697 uint_t npages) 698 { 699 dev_info_t *dip = pci_p->pci_dip; 700 int i; 701 caddr_t vaddr = (caddr_t)mp->dmai_object.dmao_obj.virt_obj.v_as; 702 struct hat *hat_p = vaddr ? ((struct as *)vaddr)->a_hat : kas.a_hat; 703 caddr_t sva; 704 int needcb = 0; 705 706 sva = (caddr_t)(((uintptr_t)mp->dmai_object.dmao_obj.virt_obj.v_addr + 707 IOMMU_PAGE_SIZE) & IOMMU_PAGE_MASK); 708 709 if (pci_dvma_remap_enabled && hat_p == kas.a_hat && 710 mp->dmai_object.dmao_type != DMA_OTYP_BUFVADDR) 711 needcb = 1; 712 713 for (vaddr = sva, i = 1; i < npages; i++, vaddr += IOMMU_PAGE_SIZE) { 714 pfn_t pfn; 715 716 if (needcb) { 717 int (*waitfp)(caddr_t) = dmareq->dmar_fp; 718 uint_t flags = ((waitfp == DDI_DMA_SLEEP)? 719 HAC_SLEEP : HAC_NOSLEEP) | HAC_PAGELOCK; 720 int ret; 721 722 ret = hat_add_callback(pci_dvma_cbid, vaddr, 723 IOMMU_PAGE_SIZE, flags, mp, &pfn, 724 MP_HAT_CB_COOKIE_PTR(mp, i)); 725 726 if (pfn == PFN_INVALID && ret == ENOMEM) { 727 ASSERT(waitfp != DDI_DMA_SLEEP); 728 if (waitfp != DDI_DMA_DONTWAIT) 729 ddi_set_callback(waitfp, 730 dmareq->dmar_arg, &pci_kmem_clid); 731 return (DDI_DMA_NORESOURCES); 732 } 733 } else 734 pfn = hat_getpfnum(hat_p, vaddr); 735 if (pfn == PFN_INVALID) 736 goto err_badpfn; 737 PCI_SET_MP_PFN1(mp, i, (iopfn_t)pfn); 738 DEBUG3(DBG_DMA_MAP, dip, "pci_dma_vapfn: mp=%p pfnlst[%x]=%x\n", 739 mp, i, (iopfn_t)pfn); 740 } 741 return (DDI_SUCCESS); 742 err_badpfn: 743 cmn_err(CE_WARN, "%s%d: bad page frame vaddr=%p", NAMEINST(dip), vaddr); 744 return (DDI_DMA_NOMAPPING); 745 } 746 747 /* 748 * pci_dma_pfn - Fills pfn list for all pages being DMA-ed. 749 * 750 * dependencies: 751 * mp->dmai_ndvmapages - set to total # of dma pages 752 * 753 * return value: 754 * DDI_SUCCESS 755 * DDI_DMA_NOMAPPING 756 */ 757 int 758 pci_dma_pfn(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 759 { 760 uint32_t npages = mp->dmai_ndvmapages; 761 int (*waitfp)(caddr_t) = dmareq->dmar_fp; 762 int i, ret, peer = PCI_DMA_ISPTP(mp); 763 764 pbm_t *pbm_p = pci_p->pci_pbm_p; 765 iopfn_t pfn_base = pbm_p->pbm_base_pfn; 766 iopfn_t pfn_last = pbm_p->pbm_last_pfn; 767 iopfn_t pfn_adj = peer ? pfn_base : 0; 768 769 DEBUG2(DBG_DMA_MAP, pci_p->pci_dip, "pci_dma_pfn: mp=%p pfn0=%x\n", 770 mp, MP_PFN0(mp) - pfn_adj); 771 /* 1 page: no array alloc/fill, no mixed mode check */ 772 if (npages == 1) { 773 PCI_SET_MP_PFN(mp, 0, MP_PFN0(mp) - pfn_adj); 774 return (DDI_SUCCESS); 775 } 776 /* allocate pfn array */ 777 if (!(mp->dmai_pfnlst = kmem_alloc(npages * sizeof (iopfn_t), 778 waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP))) { 779 if (waitfp != DDI_DMA_DONTWAIT) 780 ddi_set_callback(waitfp, dmareq->dmar_arg, 781 &pci_kmem_clid); 782 return (DDI_DMA_NORESOURCES); 783 } 784 /* fill pfn array */ 785 PCI_SET_MP_PFN(mp, 0, MP_PFN0(mp) - pfn_adj); /* pfnlst[0] */ 786 if ((ret = PCI_DMA_ISPGPFN(mp) ? pci_dma_pgpfn(pci_p, mp, npages) : 787 pci_dma_vapfn(pci_p, dmareq, mp, npages)) != DDI_SUCCESS) 788 goto err; 789 790 /* skip pfn0, check mixed mode and adjust peer to peer pfn */ 791 for (i = 1; i < npages; i++) { 792 iopfn_t pfn = PCI_GET_MP_PFN1(mp, i); 793 if (peer ^ TGT_PFN_INBETWEEN(pfn, pfn_base, pfn_last)) { 794 cmn_err(CE_WARN, "%s%d mixed mode DMA %lx %lx", 795 NAMEINST(mp->dmai_rdip), MP_PFN0(mp), pfn); 796 ret = DDI_DMA_NOMAPPING; /* mixed mode */ 797 goto err; 798 } 799 DEBUG3(DBG_DMA_MAP, pci_p->pci_dip, 800 "pci_dma_pfn: pfnlst[%x]=%x-%x\n", i, pfn, pfn_adj); 801 if (pfn_adj) 802 PCI_SET_MP_PFN1(mp, i, pfn - pfn_adj); 803 } 804 return (DDI_SUCCESS); 805 err: 806 pci_dvma_unregister_callbacks(pci_p, mp); 807 pci_dma_freepfn(mp); 808 return (ret); 809 } 810 811 /* 812 * pci_dvma_win() - trim requested DVMA size down to window size 813 * The 1st window starts from offset and ends at page-aligned boundary. 814 * From the 2nd window on, each window starts and ends at page-aligned 815 * boundary except the last window ends at wherever requested. 816 * 817 * accesses the following mp-> members: 818 * mp->dmai_attr.dma_attr_count_max 819 * mp->dmai_attr.dma_attr_seg 820 * mp->dmai_roffset - start offset of 1st window 821 * mp->dmai_rflags (redzone) 822 * mp->dmai_ndvmapages (for 1 page fast path) 823 * 824 * sets the following mp-> members: 825 * mp->dmai_size - xfer size, != winsize if 1st/last win (not fixed) 826 * mp->dmai_winsize - window size (no redzone), n * page size (fixed) 827 * mp->dmai_nwin - # of DMA windows of entire object (fixed) 828 * mp->dmai_rflags - remove partial flag if nwin == 1 (fixed) 829 * mp->dmai_winlst - NULL, window objects not used for DVMA (fixed) 830 * 831 * fixed - not changed across different DMA windows 832 */ 833 /*ARGSUSED*/ 834 int 835 pci_dvma_win(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 836 { 837 uint32_t redzone_sz = HAS_REDZONE(mp) ? IOMMU_PAGE_SIZE : 0; 838 size_t obj_sz = mp->dmai_object.dmao_size; 839 size_t xfer_sz; 840 ulong_t pg_off; 841 842 if ((mp->dmai_ndvmapages == 1) && !redzone_sz) { 843 mp->dmai_rflags &= ~DDI_DMA_PARTIAL; 844 mp->dmai_size = obj_sz; 845 mp->dmai_winsize = IOMMU_PAGE_SIZE; 846 mp->dmai_nwin = 1; 847 goto done; 848 } 849 850 pg_off = mp->dmai_roffset; 851 xfer_sz = obj_sz + redzone_sz; 852 853 /* include redzone in nocross check */ 854 { 855 uint64_t nocross = mp->dmai_attr.dma_attr_seg; 856 if (xfer_sz + pg_off - 1 > nocross) 857 xfer_sz = nocross - pg_off + 1; 858 if (redzone_sz && (xfer_sz <= redzone_sz)) { 859 DEBUG5(DBG_DMA_MAP, pci_p->pci_dip, 860 "nocross too small %lx(%lx)+%lx+%x < %" PRIx64 "\n", 861 xfer_sz, obj_sz, pg_off, redzone_sz, nocross); 862 return (DDI_DMA_TOOBIG); 863 } 864 } 865 xfer_sz -= redzone_sz; /* restore transfer size */ 866 /* check counter max */ 867 { 868 uint32_t count_max = mp->dmai_attr.dma_attr_count_max; 869 if (xfer_sz - 1 > count_max) 870 xfer_sz = count_max + 1; 871 } 872 if (xfer_sz >= obj_sz) { 873 mp->dmai_rflags &= ~DDI_DMA_PARTIAL; 874 mp->dmai_size = xfer_sz; 875 mp->dmai_winsize = P2ROUNDUP(xfer_sz + pg_off, IOMMU_PAGE_SIZE); 876 mp->dmai_nwin = 1; 877 goto done; 878 } 879 if (!(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 880 DEBUG4(DBG_DMA_MAP, pci_p->pci_dip, 881 "too big: %lx+%lx+%x > %lx\n", 882 obj_sz, pg_off, redzone_sz, xfer_sz); 883 return (DDI_DMA_TOOBIG); 884 } 885 886 xfer_sz = IOMMU_PTOB(IOMMU_BTOP(xfer_sz + pg_off)); /* page align */ 887 mp->dmai_size = xfer_sz - pg_off; /* 1st window xferrable size */ 888 mp->dmai_winsize = xfer_sz; /* redzone not in winsize */ 889 mp->dmai_nwin = (obj_sz + pg_off + xfer_sz - 1) / xfer_sz; 890 done: 891 mp->dmai_winlst = NULL; 892 dump_dma_handle(DBG_DMA_MAP, pci_p->pci_dip, mp); 893 return (DDI_SUCCESS); 894 } 895 896 /* 897 * fast track cache entry to iommu context, inserts 3 0 bits between 898 * upper 6-bits and lower 3-bits of the 9-bit cache entry 899 */ 900 #define IOMMU_FCE_TO_CTX(i) (((i) << 3) | ((i) & 0x7) | 0x38) 901 902 /* 903 * pci_dvma_map_fast - attempts to map fast trackable DVMA 904 */ 905 int 906 pci_dvma_map_fast(iommu_t *iommu_p, ddi_dma_impl_t *mp) 907 { 908 uint_t clustsz = pci_dvma_page_cache_clustsz; 909 uint_t entries = pci_dvma_page_cache_entries; 910 uint64_t *tte_addr; 911 uint64_t tte = GET_TTE_TEMPLATE(mp); 912 int i = iommu_p->iommu_dvma_addr_scan_start; 913 uint8_t *lock_addr = iommu_p->iommu_dvma_cache_locks + i; 914 iopfn_t *pfn_addr; 915 dvma_addr_t dvma_pg; 916 size_t npages = IOMMU_BTOP(mp->dmai_winsize); 917 #ifdef DEBUG 918 dev_info_t *dip = mp->dmai_rdip; 919 #endif 920 extern uint8_t ldstub(uint8_t *); 921 ASSERT(IOMMU_PTOB(npages) == mp->dmai_winsize); 922 ASSERT(npages + HAS_REDZONE(mp) <= clustsz); 923 924 for (; i < entries && ldstub(lock_addr); i++, lock_addr++) 925 ; 926 if (i >= entries) { 927 lock_addr = iommu_p->iommu_dvma_cache_locks; 928 i = 0; 929 for (; i < entries && ldstub(lock_addr); i++, lock_addr++) 930 ; 931 if (i >= entries) { 932 #ifdef PCI_DMA_PROF 933 pci_dvmaft_exhaust++; 934 #endif 935 return (DDI_DMA_NORESOURCES); 936 } 937 } 938 iommu_p->iommu_dvma_addr_scan_start = (i + 1) & (entries - 1); 939 if (PCI_DMA_USECTX(mp)) { 940 dvma_context_t ctx = IOMMU_FCE_TO_CTX(i); 941 tte |= IOMMU_CTX2TTE(ctx); 942 mp->dmai_flags |= DMAI_FLAGS_CONTEXT; 943 DEBUG1(DBG_DMA_MAP, dip, "fast: ctx=0x%x\n", ctx); 944 } 945 i *= clustsz; 946 tte_addr = iommu_p->iommu_tsb_vaddr + i; 947 dvma_pg = iommu_p->dvma_base_pg + i; 948 #ifdef DEBUG 949 for (i = 0; i < clustsz; i++) 950 ASSERT(TTE_IS_INVALID(tte_addr[i])); 951 #endif 952 *tte_addr = tte | IOMMU_PTOB(MP_PFN0(mp)); /* map page 0 */ 953 DEBUG5(DBG_DMA_MAP, dip, "fast %p:dvma_pg=%x tte0(%p)=%08x.%08x\n", mp, 954 dvma_pg, tte_addr, HI32(*tte_addr), LO32(*tte_addr)); 955 if (npages == 1) 956 goto tte_done; 957 pfn_addr = PCI_GET_MP_PFN1_ADDR(mp); /* short iommu_map_pages() */ 958 for (tte_addr++, i = 1; i < npages; i++, tte_addr++, pfn_addr++) { 959 *tte_addr = tte | IOMMU_PTOB(*pfn_addr); 960 DEBUG5(DBG_DMA_MAP, dip, "fast %p:tte(%p, %p)=%08x.%08x\n", mp, 961 tte_addr, pfn_addr, HI32(*tte_addr), LO32(*tte_addr)); 962 } 963 tte_done: 964 #ifdef PCI_DMA_PROF 965 pci_dvmaft_success++; 966 #endif 967 mp->dmai_mapping = mp->dmai_roffset | IOMMU_PTOB(dvma_pg); 968 mp->dmai_offset = 0; 969 mp->dmai_flags |= DMAI_FLAGS_FASTTRACK; 970 PCI_SAVE_MP_TTE(mp, tte); /* save TTE template for unmapping */ 971 if (DVMA_DBG_ON(iommu_p)) 972 pci_dvma_alloc_debug(iommu_p, (char *)mp->dmai_mapping, 973 mp->dmai_size, mp); 974 return (DDI_SUCCESS); 975 } 976 977 /* 978 * pci_dvma_map: map non-fasttrack DMA 979 * Use quantum cache if single page DMA. 980 */ 981 int 982 pci_dvma_map(ddi_dma_impl_t *mp, ddi_dma_req_t *dmareq, iommu_t *iommu_p) 983 { 984 uint_t npages = PCI_DMA_WINNPGS(mp); 985 dvma_addr_t dvma_pg, dvma_pg_index; 986 void *dvma_addr; 987 uint64_t tte = GET_TTE_TEMPLATE(mp); 988 int sleep = dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP; 989 #ifdef DEBUG 990 dev_info_t *dip = mp->dmai_rdip; 991 #endif 992 /* 993 * allocate dvma space resource and map in the first window. 994 * (vmem_t *vmp, size_t size, 995 * size_t align, size_t phase, size_t nocross, 996 * void *minaddr, void *maxaddr, int vmflag) 997 */ 998 if ((npages == 1) && !HAS_REDZONE(mp) && HAS_NOSYSLIMIT(mp)) { 999 dvma_addr = vmem_alloc(iommu_p->iommu_dvma_map, 1000 IOMMU_PAGE_SIZE, sleep); 1001 mp->dmai_flags |= DMAI_FLAGS_VMEMCACHE; 1002 #ifdef PCI_DMA_PROF 1003 pci_dvma_vmem_alloc++; 1004 #endif 1005 } else { 1006 dvma_addr = vmem_xalloc(iommu_p->iommu_dvma_map, 1007 IOMMU_PTOB(npages + HAS_REDZONE(mp)), 1008 MAX(mp->dmai_attr.dma_attr_align, IOMMU_PAGE_SIZE), 1009 0, 1010 mp->dmai_attr.dma_attr_seg + 1, 1011 (void *)mp->dmai_attr.dma_attr_addr_lo, 1012 (void *)(mp->dmai_attr.dma_attr_addr_hi + 1), 1013 sleep); 1014 #ifdef PCI_DMA_PROF 1015 pci_dvma_vmem_xalloc++; 1016 #endif 1017 } 1018 dvma_pg = IOMMU_BTOP((ulong_t)dvma_addr); 1019 dvma_pg_index = dvma_pg - iommu_p->dvma_base_pg; 1020 DEBUG2(DBG_DMA_MAP, dip, "fallback dvma_pages: dvma_pg=%x index=%x\n", 1021 dvma_pg, dvma_pg_index); 1022 if (dvma_pg == 0) 1023 goto noresource; 1024 1025 /* allocate DVMA context */ 1026 if ((npages >= pci_context_minpages) && PCI_DMA_USECTX(mp)) { 1027 dvma_context_t ctx; 1028 if (ctx = pci_iommu_get_dvma_context(iommu_p, dvma_pg_index)) { 1029 tte |= IOMMU_CTX2TTE(ctx); 1030 mp->dmai_flags |= DMAI_FLAGS_CONTEXT; 1031 } 1032 } 1033 mp->dmai_mapping = mp->dmai_roffset | IOMMU_PTOB(dvma_pg); 1034 mp->dmai_offset = 0; 1035 PCI_SAVE_MP_TTE(mp, tte); /* mp->dmai_tte = tte */ 1036 iommu_map_pages(iommu_p, mp, dvma_pg, npages, 0); 1037 return (DDI_SUCCESS); 1038 noresource: 1039 if (dmareq->dmar_fp != DDI_DMA_DONTWAIT) { 1040 DEBUG0(DBG_DMA_MAP, dip, "dvma_pg 0 - set callback\n"); 1041 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg, 1042 &iommu_p->iommu_dvma_clid); 1043 } 1044 DEBUG0(DBG_DMA_MAP, dip, "vmem_xalloc - DDI_DMA_NORESOURCES\n"); 1045 return (DDI_DMA_NORESOURCES); 1046 } 1047 1048 void 1049 pci_dvma_unmap(iommu_t *iommu_p, ddi_dma_impl_t *mp) 1050 { 1051 size_t npages; 1052 dvma_addr_t dvma_addr = (dvma_addr_t)mp->dmai_mapping; 1053 dvma_addr_t dvma_pg = IOMMU_BTOP(dvma_addr); 1054 dvma_addr = IOMMU_PTOB(dvma_pg); 1055 1056 if (mp->dmai_flags & DMAI_FLAGS_FASTTRACK) { 1057 iopfn_t index = dvma_pg - iommu_p->dvma_base_pg; 1058 ASSERT(index % pci_dvma_page_cache_clustsz == 0); 1059 index /= pci_dvma_page_cache_clustsz; 1060 ASSERT(index < pci_dvma_page_cache_entries); 1061 iommu_p->iommu_dvma_cache_locks[index] = 0; 1062 #ifdef PCI_DMA_PROF 1063 pci_dvmaft_free++; 1064 #endif 1065 return; 1066 } 1067 npages = IOMMU_BTOP(mp->dmai_winsize) + HAS_REDZONE(mp); 1068 pci_vmem_free(iommu_p, mp, (void *)dvma_addr, npages); 1069 1070 if (mp->dmai_flags & DMAI_FLAGS_CONTEXT) 1071 pci_iommu_free_dvma_context(iommu_p, MP2CTX(mp)); 1072 } 1073 1074 void 1075 pci_dma_sync_unmap(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp) 1076 { 1077 pci_t *pci_p = get_pci_soft_state(ddi_get_instance(dip)); 1078 iommu_t *iommu_p = pci_p->pci_iommu_p; 1079 uint64_t sync_buf_save = SYNC_BUF_PA(mp); 1080 uint32_t fast_track = mp->dmai_flags & DMAI_FLAGS_FASTTRACK; 1081 1082 if (fast_track) { 1083 dvma_addr_t dvma_pg = IOMMU_BTOP(mp->dmai_mapping); 1084 1085 SYNC_BUF_PA(mp) = IOMMU_PAGE_TTEPA(iommu_p, dvma_pg); 1086 ASSERT(!(SYNC_BUF_PA(mp) & PCI_SYNC_FLAG_SIZE - 1)); 1087 } 1088 1089 if (pci_dvma_sync_before_unmap) { 1090 pci_dma_sync(dip, rdip, (ddi_dma_handle_t)mp, 0, 0, 1091 DDI_DMA_SYNC_FORCPU); 1092 iommu_unmap_window(iommu_p, mp); 1093 } else { 1094 iommu_unmap_window(iommu_p, mp); 1095 pci_dma_sync(dip, rdip, (ddi_dma_handle_t)mp, 0, 0, 1096 DDI_DMA_SYNC_FORCPU); 1097 } 1098 1099 if (fast_track) 1100 SYNC_BUF_PA(mp) = sync_buf_save; 1101 } 1102 1103 /* 1104 * DVMA mappings may have multiple windows, but each window always have 1105 * one segment. 1106 */ 1107 int 1108 pci_dvma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp, 1109 enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp, 1110 uint_t cache_flags) 1111 { 1112 switch (cmd) { 1113 case DDI_DMA_SYNC: 1114 return (pci_dma_sync(dip, rdip, (ddi_dma_handle_t)mp, 1115 *offp, *lenp, cache_flags)); 1116 1117 case DDI_DMA_HTOC: { 1118 int ret; 1119 off_t wo_off, off = *offp; /* wo_off: wnd's obj offset */ 1120 uint_t win_size = mp->dmai_winsize; 1121 ddi_dma_cookie_t *cp = (ddi_dma_cookie_t *)objp; 1122 1123 if (off >= mp->dmai_object.dmao_size) { 1124 cmn_err(CE_WARN, "%s%d invalid dma_htoc offset %lx", 1125 NAMEINST(mp->dmai_rdip), off); 1126 return (DDI_FAILURE); 1127 } 1128 off += mp->dmai_roffset; 1129 ret = pci_dma_win(dip, rdip, (ddi_dma_handle_t)mp, 1130 off / win_size, &wo_off, NULL, cp, NULL); /* lenp == NULL */ 1131 if (ret) 1132 return (ret); 1133 DEBUG4(DBG_DMA_CTL, dip, "HTOC:cookie=%x+%lx off=%lx,%lx\n", 1134 cp->dmac_address, cp->dmac_size, off, *offp); 1135 1136 /* adjust cookie addr/len if we are not on window boundary */ 1137 ASSERT((off % win_size) == (off - 1138 (PCI_DMA_CURWIN(mp) ? mp->dmai_roffset : 0) - wo_off)); 1139 off = PCI_DMA_CURWIN(mp) ? off % win_size : *offp; 1140 ASSERT(cp->dmac_size > off); 1141 cp->dmac_laddress += off; 1142 cp->dmac_size -= off; 1143 DEBUG5(DBG_DMA_CTL, dip, 1144 "HTOC:mp=%p cookie=%x+%lx off=%lx,%lx\n", 1145 mp, cp->dmac_address, cp->dmac_size, off, wo_off); 1146 } 1147 return (DDI_SUCCESS); 1148 1149 case DDI_DMA_REPWIN: 1150 *offp = mp->dmai_offset; 1151 *lenp = mp->dmai_size; 1152 return (DDI_SUCCESS); 1153 1154 case DDI_DMA_MOVWIN: { 1155 off_t off = *offp; 1156 if (off >= mp->dmai_object.dmao_size) 1157 return (DDI_FAILURE); 1158 off += mp->dmai_roffset; 1159 return (pci_dma_win(dip, rdip, (ddi_dma_handle_t)mp, 1160 off / mp->dmai_winsize, offp, lenp, 1161 (ddi_dma_cookie_t *)objp, NULL)); 1162 } 1163 1164 case DDI_DMA_NEXTWIN: { 1165 window_t win = PCI_DMA_CURWIN(mp); 1166 if (offp) { 1167 if (*(window_t *)offp != win) { /* window not active */ 1168 *(window_t *)objp = win; /* return cur win */ 1169 return (DDI_DMA_STALE); 1170 } 1171 win++; 1172 } else /* map win 0 */ 1173 win = 0; 1174 if (win >= mp->dmai_nwin) { 1175 *(window_t *)objp = win - 1; 1176 return (DDI_DMA_DONE); 1177 } 1178 if (pci_dma_win(dip, rdip, (ddi_dma_handle_t)mp, 1179 win, 0, 0, 0, 0)) { 1180 *(window_t *)objp = win - 1; 1181 return (DDI_FAILURE); 1182 } 1183 *(window_t *)objp = win; 1184 } 1185 return (DDI_SUCCESS); 1186 1187 case DDI_DMA_NEXTSEG: 1188 if (*(window_t *)offp != PCI_DMA_CURWIN(mp)) 1189 return (DDI_DMA_STALE); 1190 if (lenp) /* only 1 seg allowed */ 1191 return (DDI_DMA_DONE); 1192 /* return mp as seg 0 */ 1193 *(ddi_dma_seg_t *)objp = (ddi_dma_seg_t)mp; 1194 return (DDI_SUCCESS); 1195 1196 case DDI_DMA_SEGTOC: 1197 MAKE_DMA_COOKIE((ddi_dma_cookie_t *)objp, mp->dmai_mapping, 1198 mp->dmai_size); 1199 *offp = mp->dmai_offset; 1200 *lenp = mp->dmai_size; 1201 return (DDI_SUCCESS); 1202 1203 case DDI_DMA_COFF: { 1204 ddi_dma_cookie_t *cp = (ddi_dma_cookie_t *)offp; 1205 if (cp->dmac_address < mp->dmai_mapping || 1206 (cp->dmac_address + cp->dmac_size) > 1207 (mp->dmai_mapping + mp->dmai_size)) 1208 return (DDI_FAILURE); 1209 *objp = (caddr_t)(cp->dmac_address - mp->dmai_mapping + 1210 mp->dmai_offset); 1211 } 1212 return (DDI_SUCCESS); 1213 1214 case DDI_DMA_REMAP: 1215 if (pci_dvma_remap_enabled) 1216 return (pci_dvma_remap(dip, rdip, mp, *offp, *lenp)); 1217 return (DDI_FAILURE); 1218 1219 default: 1220 DEBUG3(DBG_DMA_CTL, dip, "unknown command (%x): rdip=%s%d\n", 1221 cmd, ddi_driver_name(rdip), ddi_get_instance(rdip)); 1222 break; 1223 } 1224 return (DDI_FAILURE); 1225 } 1226 1227 void 1228 pci_dma_freewin(ddi_dma_impl_t *mp) 1229 { 1230 pci_dma_win_t *win_p = mp->dmai_winlst, *win2_p; 1231 for (win2_p = win_p; win_p; win2_p = win_p) { 1232 win_p = win2_p->win_next; 1233 kmem_free(win2_p, sizeof (pci_dma_win_t) + 1234 sizeof (ddi_dma_cookie_t) * win2_p->win_ncookies); 1235 } 1236 mp->dmai_nwin = 0; 1237 mp->dmai_winlst = NULL; 1238 } 1239 1240 /* 1241 * pci_dma_newwin - create a dma window object and cookies 1242 * 1243 * After the initial scan in pci_dma_physwin(), which identifies 1244 * a portion of the pfn array that belongs to a dma window, 1245 * we are called to allocate and initialize representing memory 1246 * resources. We know from the 1st scan the number of cookies 1247 * or dma segment in this window so we can allocate a contiguous 1248 * memory array for the dma cookies (The implementation of 1249 * ddi_dma_nextcookie(9f) dictates dma cookies be contiguous). 1250 * 1251 * A second round scan is done on the pfn array to identify 1252 * each dma segment and initialize its corresponding dma cookie. 1253 * We don't need to do all the safety checking and we know they 1254 * all belong to the same dma window. 1255 * 1256 * Input: cookie_no - # of cookies identified by the 1st scan 1257 * start_idx - subscript of the pfn array for the starting pfn 1258 * end_idx - subscript of the last pfn in dma window 1259 * win_pp - pointer to win_next member of previous window 1260 * Return: DDI_SUCCESS - with **win_pp as newly created window object 1261 * DDI_DMA_NORESROUCE - caller frees all previous window objs 1262 * Note: Each cookie and window size are all initialized on page 1263 * boundary. This is not true for the 1st cookie of the 1st 1264 * window and the last cookie of the last window. 1265 * We fix that later in upper layer which has access to size 1266 * and offset info. 1267 * 1268 */ 1269 static int 1270 pci_dma_newwin(ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, uint32_t cookie_no, 1271 uint32_t start_idx, uint32_t end_idx, pci_dma_win_t **win_pp, 1272 uint64_t count_max, uint64_t bypass_prefix) 1273 { 1274 int (*waitfp)(caddr_t) = dmareq->dmar_fp; 1275 ddi_dma_cookie_t *cookie_p; 1276 uint32_t pfn_no = 1; 1277 iopfn_t pfn = PCI_GET_MP_PFN(mp, start_idx); 1278 iopfn_t prev_pfn = pfn; 1279 uint64_t seg_pfn0 = pfn; 1280 size_t sz = cookie_no * sizeof (ddi_dma_cookie_t); 1281 pci_dma_win_t *win_p = kmem_alloc(sizeof (pci_dma_win_t) + sz, 1282 waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP); 1283 if (!win_p) 1284 goto noresource; 1285 1286 win_p->win_next = NULL; 1287 win_p->win_ncookies = cookie_no; 1288 win_p->win_curseg = 0; /* start from segment 0 */ 1289 win_p->win_size = IOMMU_PTOB(end_idx - start_idx + 1); 1290 /* win_p->win_offset is left uninitialized */ 1291 1292 cookie_p = (ddi_dma_cookie_t *)(win_p + 1); 1293 start_idx++; 1294 for (; start_idx <= end_idx; start_idx++, prev_pfn = pfn, pfn_no++) { 1295 pfn = PCI_GET_MP_PFN1(mp, start_idx); 1296 if ((pfn == prev_pfn + 1) && 1297 (IOMMU_PTOB(pfn_no + 1) - 1 <= count_max)) 1298 continue; 1299 1300 /* close up the cookie up to (including) prev_pfn */ 1301 MAKE_DMA_COOKIE(cookie_p, IOMMU_PTOB(seg_pfn0) | bypass_prefix, 1302 IOMMU_PTOB(pfn_no)); 1303 DEBUG2(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages)\n", 1304 IOMMU_PTOB(seg_pfn0) | bypass_prefix, pfn_no); 1305 1306 cookie_p++; /* advance to next available cookie cell */ 1307 pfn_no = 0; 1308 seg_pfn0 = pfn; /* start a new segment from current pfn */ 1309 } 1310 MAKE_DMA_COOKIE(cookie_p, IOMMU_PTOB(seg_pfn0) | bypass_prefix, 1311 IOMMU_PTOB(pfn_no)); 1312 DEBUG3(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages) of total %x\n", 1313 IOMMU_PTOB(seg_pfn0) | bypass_prefix, pfn_no, cookie_no); 1314 #ifdef DEBUG 1315 cookie_p++; 1316 ASSERT((cookie_p - (ddi_dma_cookie_t *)(win_p + 1)) == cookie_no); 1317 #endif 1318 *win_pp = win_p; 1319 return (DDI_SUCCESS); 1320 noresource: 1321 if (waitfp != DDI_DMA_DONTWAIT) 1322 ddi_set_callback(waitfp, dmareq->dmar_arg, &pci_kmem_clid); 1323 return (DDI_DMA_NORESOURCES); 1324 } 1325 1326 /* 1327 * pci_dma_adjust - adjust 1st and last cookie and window sizes 1328 * remove initial dma page offset from 1st cookie and window size 1329 * remove last dma page remainder from last cookie and window size 1330 * fill win_offset of each dma window according to just fixed up 1331 * each window sizes 1332 * pci_dma_win_t members modified: 1333 * win_p->win_offset - this window's offset within entire DMA object 1334 * win_p->win_size - xferrable size (in bytes) for this window 1335 * 1336 * ddi_dma_impl_t members modified: 1337 * mp->dmai_size - 1st window xferrable size 1338 * mp->dmai_offset - 0, which is the dma offset of the 1st window 1339 * 1340 * ddi_dma_cookie_t members modified: 1341 * cookie_p->dmac_size - 1st and last cookie remove offset or remainder 1342 * cookie_p->dmac_laddress - 1st cookie add page offset 1343 */ 1344 static void 1345 pci_dma_adjust(ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, pci_dma_win_t *win_p) 1346 { 1347 ddi_dma_cookie_t *cookie_p = (ddi_dma_cookie_t *)(win_p + 1); 1348 size_t pg_offset = mp->dmai_roffset; 1349 size_t win_offset = 0; 1350 1351 cookie_p->dmac_size -= pg_offset; 1352 cookie_p->dmac_laddress |= pg_offset; 1353 win_p->win_size -= pg_offset; 1354 DEBUG1(DBG_BYPASS, mp->dmai_rdip, "pg0 adjust %lx\n", pg_offset); 1355 1356 mp->dmai_size = win_p->win_size; 1357 mp->dmai_offset = 0; 1358 1359 pg_offset += mp->dmai_object.dmao_size; 1360 pg_offset &= IOMMU_PAGE_OFFSET; 1361 if (pg_offset) 1362 pg_offset = IOMMU_PAGE_SIZE - pg_offset; 1363 DEBUG1(DBG_BYPASS, mp->dmai_rdip, "last pg adjust %lx\n", pg_offset); 1364 1365 for (; win_p->win_next; win_p = win_p->win_next) { 1366 DEBUG1(DBG_BYPASS, mp->dmai_rdip, "win off %p\n", win_offset); 1367 win_p->win_offset = win_offset; 1368 win_offset += win_p->win_size; 1369 } 1370 /* last window */ 1371 win_p->win_offset = win_offset; 1372 cookie_p = (ddi_dma_cookie_t *)(win_p + 1); 1373 cookie_p[win_p->win_ncookies - 1].dmac_size -= pg_offset; 1374 win_p->win_size -= pg_offset; 1375 ASSERT((win_offset + win_p->win_size) == mp->dmai_object.dmao_size); 1376 } 1377 1378 /* 1379 * pci_dma_physwin() - carve up dma windows using physical addresses. 1380 * Called to handle iommu bypass and pci peer-to-peer transfers. 1381 * Calls pci_dma_newwin() to allocate window objects. 1382 * 1383 * Dependency: mp->dmai_pfnlst points to an array of pfns 1384 * 1385 * 1. Each dma window is represented by a pci_dma_win_t object. 1386 * The object will be casted to ddi_dma_win_t and returned 1387 * to leaf driver through the DDI interface. 1388 * 2. Each dma window can have several dma segments with each 1389 * segment representing a physically contiguous either memory 1390 * space (if we are doing an iommu bypass transfer) or pci address 1391 * space (if we are doing a peer-to-peer transfer). 1392 * 3. Each segment has a DMA cookie to program the DMA engine. 1393 * The cookies within each DMA window must be located in a 1394 * contiguous array per ddi_dma_nextcookie(9f). 1395 * 4. The number of DMA segments within each DMA window cannot exceed 1396 * mp->dmai_attr.dma_attr_sgllen. If the transfer size is 1397 * too large to fit in the sgllen, the rest needs to be 1398 * relocated to the next dma window. 1399 * 5. Peer-to-peer DMA segment follows device hi, lo, count_max, 1400 * and nocross restrictions while bypass DMA follows the set of 1401 * restrictions with system limits factored in. 1402 * 1403 * Return: 1404 * mp->dmai_winlst - points to a link list of pci_dma_win_t objects. 1405 * Each pci_dma_win_t object on the link list contains 1406 * infomation such as its window size (# of pages), 1407 * starting offset (also see Restriction), an array of 1408 * DMA cookies, and # of cookies in the array. 1409 * mp->dmai_pfnlst - NULL, the pfn list is freed to conserve memory. 1410 * mp->dmai_nwin - # of total DMA windows on mp->dmai_winlst. 1411 * mp->dmai_mapping - starting cookie address 1412 * mp->dmai_rflags - consistent, nosync, no redzone 1413 * mp->dmai_cookie - start of cookie table of the 1st DMA window 1414 * 1415 * Restriction: 1416 * Each pci_dma_win_t object can theoratically start from any offset 1417 * since the iommu is not involved. However, this implementation 1418 * always make windows start from page aligned offset (except 1419 * the 1st window, which follows the requested offset) due to the 1420 * fact that we are handed a pfn list. This does require device's 1421 * count_max and attr_seg to be at least IOMMU_PAGE_SIZE aligned. 1422 */ 1423 int 1424 pci_dma_physwin(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 1425 { 1426 uint_t npages = mp->dmai_ndvmapages; 1427 int ret, sgllen = mp->dmai_attr.dma_attr_sgllen; 1428 iopfn_t pfn_lo, pfn_hi, prev_pfn, bypass_pfn; 1429 iopfn_t pfn = PCI_GET_MP_PFN(mp, 0); 1430 uint32_t i, win_no = 0, pfn_no = 1, win_pfn0_index = 0, cookie_no = 0; 1431 uint64_t count_max, bypass = PCI_DMA_BYPASS_PREFIX(mp, pfn); 1432 pci_dma_win_t **win_pp = (pci_dma_win_t **)&mp->dmai_winlst; 1433 ddi_dma_cookie_t *cookie0_p; 1434 1435 if (PCI_DMA_ISPTP(mp)) { /* ignore sys limits for peer-to-peer */ 1436 ddi_dma_attr_t *dev_attr_p = DEV_ATTR(mp); 1437 iopfn_t pfn_base = pci_p->pci_pbm_p->pbm_base_pfn; 1438 iopfn_t pfn_last = pci_p->pci_pbm_p->pbm_last_pfn - pfn_base; 1439 uint64_t nocross = dev_attr_p->dma_attr_seg; 1440 if (nocross && (nocross < UINT32_MAX)) 1441 return (DDI_DMA_NOMAPPING); 1442 if (dev_attr_p->dma_attr_align > IOMMU_PAGE_SIZE) 1443 return (DDI_DMA_NOMAPPING); 1444 pfn_lo = IOMMU_BTOP(dev_attr_p->dma_attr_addr_lo); 1445 pfn_hi = IOMMU_BTOP(dev_attr_p->dma_attr_addr_hi); 1446 pfn_hi = MIN(pfn_hi, pfn_last); 1447 if ((pfn_lo > pfn_hi) || (pfn < pfn_lo)) 1448 return (DDI_DMA_NOMAPPING); 1449 count_max = dev_attr_p->dma_attr_count_max; 1450 count_max = MIN(count_max, nocross); 1451 /* 1452 * the following count_max trim is not done because we are 1453 * making sure pfn_lo <= pfn <= pfn_hi inside the loop 1454 * count_max=MIN(count_max, IOMMU_PTOB(pfn_hi - pfn_lo + 1)-1); 1455 */ 1456 } else { /* bypass hi/lo/count_max have been processed by attr2hdl() */ 1457 count_max = mp->dmai_attr.dma_attr_count_max; 1458 pfn_lo = IOMMU_BTOP(mp->dmai_attr.dma_attr_addr_lo); 1459 pfn_hi = IOMMU_BTOP(mp->dmai_attr.dma_attr_addr_hi); 1460 } 1461 1462 bypass_pfn = IOMMU_BTOP(bypass); 1463 1464 for (prev_pfn = (bypass_pfn | pfn), i = 1; i < npages; 1465 i++, prev_pfn = pfn, pfn_no++) { 1466 pfn = bypass_pfn | PCI_GET_MP_PFN1(mp, i); 1467 if ((pfn == prev_pfn + 1) && 1468 (IOMMU_PTOB(pfn_no + 1) - 1 <= count_max)) 1469 continue; 1470 if ((pfn < pfn_lo) || (prev_pfn > pfn_hi)) { 1471 ret = DDI_DMA_NOMAPPING; 1472 goto err; 1473 } 1474 cookie_no++; 1475 pfn_no = 0; 1476 if (cookie_no < sgllen) 1477 continue; 1478 1479 DEBUG3(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n", 1480 win_pfn0_index, i - 1, cookie_no); 1481 if (ret = pci_dma_newwin(dmareq, mp, cookie_no, 1482 win_pfn0_index, i - 1, win_pp, count_max, bypass)) 1483 goto err; 1484 1485 win_pp = &(*win_pp)->win_next; /* win_pp = *(win_pp) */ 1486 win_no++; 1487 win_pfn0_index = i; 1488 cookie_no = 0; 1489 } 1490 if (pfn > pfn_hi) { 1491 ret = DDI_DMA_NOMAPPING; 1492 goto err; 1493 } 1494 cookie_no++; 1495 DEBUG3(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n", 1496 win_pfn0_index, i - 1, cookie_no); 1497 if (ret = pci_dma_newwin(dmareq, mp, cookie_no, win_pfn0_index, 1498 i - 1, win_pp, count_max, bypass)) 1499 goto err; 1500 win_no++; 1501 pci_dma_adjust(dmareq, mp, mp->dmai_winlst); 1502 mp->dmai_nwin = win_no; 1503 mp->dmai_rflags |= DDI_DMA_CONSISTENT; 1504 if (!pci_p->pci_pbm_p->pbm_sync_reg_pa) { 1505 mp->dmai_rflags |= DMP_NOSYNC; 1506 mp->dmai_flags |= DMAI_FLAGS_NOSYNC; 1507 } 1508 mp->dmai_rflags &= ~DDI_DMA_REDZONE; 1509 cookie0_p = (ddi_dma_cookie_t *)(WINLST(mp) + 1); 1510 mp->dmai_cookie = WINLST(mp)->win_ncookies > 1 ? cookie0_p + 1 : 0; 1511 mp->dmai_mapping = cookie0_p->dmac_laddress; 1512 1513 pci_dma_freepfn(mp); 1514 return (DDI_DMA_MAPPED); 1515 err: 1516 pci_dma_freewin(mp); 1517 return (ret); 1518 } 1519 1520 /*ARGSUSED*/ 1521 int 1522 pci_dma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp, 1523 enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp, 1524 uint_t cache_flags) 1525 { 1526 switch (cmd) { 1527 case DDI_DMA_SYNC: /* XXX */ 1528 return (DDI_SUCCESS); 1529 1530 case DDI_DMA_HTOC: { 1531 off_t off = *offp; 1532 ddi_dma_cookie_t *loop_cp, *cp; 1533 pci_dma_win_t *win_p = mp->dmai_winlst; 1534 1535 if (off >= mp->dmai_object.dmao_size) 1536 return (DDI_FAILURE); 1537 1538 /* locate window */ 1539 while (win_p->win_offset + win_p->win_size <= off) 1540 win_p = win_p->win_next; 1541 1542 loop_cp = cp = (ddi_dma_cookie_t *)(win_p + 1); 1543 mp->dmai_offset = win_p->win_offset; 1544 mp->dmai_size = win_p->win_size; 1545 mp->dmai_mapping = cp->dmac_laddress; /* cookie0 start addr */ 1546 1547 /* adjust cookie addr/len if we are not on cookie boundary */ 1548 off -= win_p->win_offset; /* offset within window */ 1549 for (; off >= loop_cp->dmac_size; loop_cp++) 1550 off -= loop_cp->dmac_size; /* offset within cookie */ 1551 1552 mp->dmai_cookie = loop_cp + 1; 1553 win_p->win_curseg = loop_cp - cp; 1554 cp = (ddi_dma_cookie_t *)objp; 1555 MAKE_DMA_COOKIE(cp, loop_cp->dmac_laddress + off, 1556 loop_cp->dmac_size - off); 1557 1558 DEBUG2(DBG_DMA_CTL, dip, 1559 "HTOC: cookie - dmac_laddress=%p dmac_size=%x\n", 1560 cp->dmac_laddress, cp->dmac_size); 1561 } 1562 return (DDI_SUCCESS); 1563 1564 case DDI_DMA_REPWIN: 1565 *offp = mp->dmai_offset; 1566 *lenp = mp->dmai_size; 1567 return (DDI_SUCCESS); 1568 1569 case DDI_DMA_MOVWIN: { 1570 off_t off = *offp; 1571 ddi_dma_cookie_t *cp; 1572 pci_dma_win_t *win_p = mp->dmai_winlst; 1573 1574 if (off >= mp->dmai_object.dmao_size) 1575 return (DDI_FAILURE); 1576 1577 /* locate window */ 1578 while (win_p->win_offset + win_p->win_size <= off) 1579 win_p = win_p->win_next; 1580 1581 cp = (ddi_dma_cookie_t *)(win_p + 1); 1582 mp->dmai_offset = win_p->win_offset; 1583 mp->dmai_size = win_p->win_size; 1584 mp->dmai_mapping = cp->dmac_laddress; /* cookie0 star addr */ 1585 mp->dmai_cookie = cp + 1; 1586 win_p->win_curseg = 0; 1587 1588 *(ddi_dma_cookie_t *)objp = *cp; 1589 *offp = win_p->win_offset; 1590 *lenp = win_p->win_size; 1591 DEBUG2(DBG_DMA_CTL, dip, 1592 "HTOC: cookie - dmac_laddress=%p dmac_size=%x\n", 1593 cp->dmac_laddress, cp->dmac_size); 1594 } 1595 return (DDI_SUCCESS); 1596 1597 case DDI_DMA_NEXTWIN: { 1598 pci_dma_win_t *win_p = *(pci_dma_win_t **)offp; 1599 pci_dma_win_t **nw_pp = (pci_dma_win_t **)objp; 1600 ddi_dma_cookie_t *cp; 1601 if (!win_p) { 1602 *nw_pp = mp->dmai_winlst; 1603 return (DDI_SUCCESS); 1604 } 1605 1606 if (win_p->win_offset != mp->dmai_offset) 1607 return (DDI_DMA_STALE); 1608 if (!win_p->win_next) 1609 return (DDI_DMA_DONE); 1610 win_p = win_p->win_next; 1611 cp = (ddi_dma_cookie_t *)(win_p + 1); 1612 mp->dmai_offset = win_p->win_offset; 1613 mp->dmai_size = win_p->win_size; 1614 mp->dmai_mapping = cp->dmac_laddress; /* cookie0 star addr */ 1615 mp->dmai_cookie = cp + 1; 1616 win_p->win_curseg = 0; 1617 *nw_pp = win_p; 1618 } 1619 return (DDI_SUCCESS); 1620 1621 case DDI_DMA_NEXTSEG: { 1622 pci_dma_win_t *w_p = *(pci_dma_win_t **)offp; 1623 if (w_p->win_offset != mp->dmai_offset) 1624 return (DDI_DMA_STALE); 1625 if (w_p->win_curseg + 1 >= w_p->win_ncookies) 1626 return (DDI_DMA_DONE); 1627 w_p->win_curseg++; 1628 } 1629 *(ddi_dma_seg_t *)objp = (ddi_dma_seg_t)mp; 1630 return (DDI_SUCCESS); 1631 1632 case DDI_DMA_SEGTOC: { 1633 pci_dma_win_t *win_p = mp->dmai_winlst; 1634 off_t off = mp->dmai_offset; 1635 ddi_dma_cookie_t *cp; 1636 int i; 1637 1638 /* locate active window */ 1639 for (; win_p->win_offset != off; win_p = win_p->win_next) 1640 ; 1641 cp = (ddi_dma_cookie_t *)(win_p + 1); 1642 for (i = 0; i < win_p->win_curseg; i++, cp++) 1643 off += cp->dmac_size; 1644 *offp = off; 1645 *lenp = cp->dmac_size; 1646 *(ddi_dma_cookie_t *)objp = *cp; /* copy cookie */ 1647 } 1648 return (DDI_SUCCESS); 1649 1650 case DDI_DMA_COFF: { 1651 pci_dma_win_t *win_p; 1652 ddi_dma_cookie_t *cp; 1653 uint64_t addr, key = ((ddi_dma_cookie_t *)offp)->dmac_laddress; 1654 size_t win_off; 1655 1656 for (win_p = mp->dmai_winlst; win_p; win_p = win_p->win_next) { 1657 int i; 1658 win_off = 0; 1659 cp = (ddi_dma_cookie_t *)(win_p + 1); 1660 for (i = 0; i < win_p->win_ncookies; i++, cp++) { 1661 size_t sz = cp->dmac_size; 1662 1663 addr = cp->dmac_laddress; 1664 if ((addr <= key) && (addr + sz >= key)) 1665 goto found; 1666 win_off += sz; 1667 } 1668 } 1669 return (DDI_FAILURE); 1670 found: 1671 *objp = (caddr_t)(win_p->win_offset + win_off + (key - addr)); 1672 return (DDI_SUCCESS); 1673 } 1674 1675 case DDI_DMA_REMAP: 1676 return (DDI_FAILURE); 1677 1678 default: 1679 DEBUG3(DBG_DMA_CTL, dip, "unknown command (%x): rdip=%s%d\n", 1680 cmd, ddi_driver_name(rdip), ddi_get_instance(rdip)); 1681 break; 1682 } 1683 return (DDI_FAILURE); 1684 } 1685 1686 static void 1687 pci_dvma_debug_init(iommu_t *iommu_p) 1688 { 1689 size_t sz = sizeof (struct dvma_rec) * pci_dvma_debug_rec; 1690 ASSERT(MUTEX_HELD(&iommu_p->dvma_debug_lock)); 1691 cmn_err(CE_NOTE, "PCI DVMA %p stat ON", iommu_p); 1692 1693 iommu_p->dvma_alloc_rec = kmem_zalloc(sz, KM_SLEEP); 1694 iommu_p->dvma_free_rec = kmem_zalloc(sz, KM_SLEEP); 1695 1696 iommu_p->dvma_active_list = NULL; 1697 iommu_p->dvma_alloc_rec_index = 0; 1698 iommu_p->dvma_free_rec_index = 0; 1699 iommu_p->dvma_active_count = 0; 1700 } 1701 1702 void 1703 pci_dvma_debug_fini(iommu_t *iommu_p) 1704 { 1705 struct dvma_rec *prev, *ptr; 1706 size_t sz = sizeof (struct dvma_rec) * pci_dvma_debug_rec; 1707 uint64_t mask = ~(1ull << iommu_p->iommu_inst); 1708 cmn_err(CE_NOTE, "PCI DVMA %p stat OFF", iommu_p); 1709 1710 kmem_free(iommu_p->dvma_alloc_rec, sz); 1711 kmem_free(iommu_p->dvma_free_rec, sz); 1712 iommu_p->dvma_alloc_rec = iommu_p->dvma_free_rec = NULL; 1713 1714 prev = iommu_p->dvma_active_list; 1715 if (!prev) 1716 return; 1717 for (ptr = prev->next; ptr; prev = ptr, ptr = ptr->next) 1718 kmem_free(prev, sizeof (struct dvma_rec)); 1719 kmem_free(prev, sizeof (struct dvma_rec)); 1720 1721 iommu_p->dvma_active_list = NULL; 1722 iommu_p->dvma_alloc_rec_index = 0; 1723 iommu_p->dvma_free_rec_index = 0; 1724 iommu_p->dvma_active_count = 0; 1725 1726 pci_dvma_debug_on &= mask; 1727 pci_dvma_debug_off &= mask; 1728 } 1729 1730 void 1731 pci_dvma_alloc_debug(iommu_t *iommu_p, char *address, uint_t len, 1732 ddi_dma_impl_t *mp) 1733 { 1734 struct dvma_rec *ptr; 1735 mutex_enter(&iommu_p->dvma_debug_lock); 1736 1737 if (!iommu_p->dvma_alloc_rec) 1738 pci_dvma_debug_init(iommu_p); 1739 if (DVMA_DBG_OFF(iommu_p)) { 1740 pci_dvma_debug_fini(iommu_p); 1741 goto done; 1742 } 1743 1744 ptr = &iommu_p->dvma_alloc_rec[iommu_p->dvma_alloc_rec_index]; 1745 ptr->dvma_addr = address; 1746 ptr->len = len; 1747 ptr->mp = mp; 1748 if (++iommu_p->dvma_alloc_rec_index == pci_dvma_debug_rec) 1749 iommu_p->dvma_alloc_rec_index = 0; 1750 1751 ptr = kmem_alloc(sizeof (struct dvma_rec), KM_SLEEP); 1752 ptr->dvma_addr = address; 1753 ptr->len = len; 1754 ptr->mp = mp; 1755 1756 ptr->next = iommu_p->dvma_active_list; 1757 iommu_p->dvma_active_list = ptr; 1758 iommu_p->dvma_active_count++; 1759 done: 1760 mutex_exit(&iommu_p->dvma_debug_lock); 1761 } 1762 1763 void 1764 pci_dvma_free_debug(iommu_t *iommu_p, char *address, uint_t len, 1765 ddi_dma_impl_t *mp) 1766 { 1767 struct dvma_rec *ptr, *ptr_save; 1768 mutex_enter(&iommu_p->dvma_debug_lock); 1769 1770 if (!iommu_p->dvma_alloc_rec) 1771 pci_dvma_debug_init(iommu_p); 1772 if (DVMA_DBG_OFF(iommu_p)) { 1773 pci_dvma_debug_fini(iommu_p); 1774 goto done; 1775 } 1776 1777 ptr = &iommu_p->dvma_free_rec[iommu_p->dvma_free_rec_index]; 1778 ptr->dvma_addr = address; 1779 ptr->len = len; 1780 ptr->mp = mp; 1781 if (++iommu_p->dvma_free_rec_index == pci_dvma_debug_rec) 1782 iommu_p->dvma_free_rec_index = 0; 1783 1784 ptr_save = iommu_p->dvma_active_list; 1785 for (ptr = ptr_save; ptr; ptr = ptr->next) { 1786 if ((ptr->dvma_addr == address) && (ptr->len = len)) 1787 break; 1788 ptr_save = ptr; 1789 } 1790 if (!ptr) { 1791 cmn_err(CE_WARN, "bad dvma free addr=%lx len=%x", 1792 (long)address, len); 1793 goto done; 1794 } 1795 if (ptr == iommu_p->dvma_active_list) 1796 iommu_p->dvma_active_list = ptr->next; 1797 else 1798 ptr_save->next = ptr->next; 1799 kmem_free(ptr, sizeof (struct dvma_rec)); 1800 iommu_p->dvma_active_count--; 1801 done: 1802 mutex_exit(&iommu_p->dvma_debug_lock); 1803 } 1804 1805 #ifdef DEBUG 1806 void 1807 dump_dma_handle(uint64_t flag, dev_info_t *dip, ddi_dma_impl_t *hp) 1808 { 1809 DEBUG4(flag, dip, "mp(%p): flags=%x mapping=%lx xfer_size=%x\n", 1810 hp, hp->dmai_inuse, hp->dmai_mapping, hp->dmai_size); 1811 DEBUG4(flag|DBG_CONT, dip, "\tnpages=%x roffset=%x rflags=%x nwin=%x\n", 1812 hp->dmai_ndvmapages, hp->dmai_roffset, hp->dmai_rflags, 1813 hp->dmai_nwin); 1814 DEBUG4(flag|DBG_CONT, dip, "\twinsize=%x tte=%p pfnlst=%p pfn0=%p\n", 1815 hp->dmai_winsize, hp->dmai_tte, hp->dmai_pfnlst, hp->dmai_pfn0); 1816 DEBUG4(flag|DBG_CONT, dip, "\twinlst=%x obj=%p attr=%p ckp=%p\n", 1817 hp->dmai_winlst, &hp->dmai_object, &hp->dmai_attr, 1818 hp->dmai_cookie); 1819 } 1820 #endif 1821 1822 void 1823 pci_vmem_do_free(iommu_t *iommu_p, void *base_addr, size_t npages, 1824 int vmemcache) 1825 { 1826 vmem_t *map_p = iommu_p->iommu_dvma_map; 1827 1828 if (vmemcache) { 1829 vmem_free(map_p, base_addr, IOMMU_PAGE_SIZE); 1830 #ifdef PCI_DMA_PROF 1831 pci_dvma_vmem_free++; 1832 #endif 1833 return; 1834 } 1835 1836 vmem_xfree(map_p, base_addr, IOMMU_PTOB(npages)); 1837 #ifdef PCI_DMA_PROF 1838 pci_dvma_vmem_xfree++; 1839 #endif 1840 } 1841