1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 /* 29 * PCI nexus DVMA and DMA core routines: 30 * dma_map/dma_bind_handle implementation 31 * bypass and peer-to-peer support 32 * fast track DVMA space allocation 33 * runtime DVMA debug 34 */ 35 #include <sys/types.h> 36 #include <sys/kmem.h> 37 #include <sys/async.h> 38 #include <sys/sysmacros.h> 39 #include <sys/sunddi.h> 40 #include <sys/machsystm.h> /* lddphys() */ 41 #include <sys/ddi_impldefs.h> 42 #include <vm/hat.h> 43 #include <sys/pci/pci_obj.h> 44 45 /*LINTLIBRARY*/ 46 47 static void 48 pci_sc_pg_inv(dev_info_t *dip, sc_t *sc_p, ddi_dma_impl_t *mp, off_t off, 49 size_t len) 50 { 51 dvma_addr_t dvma_addr, pg_off; 52 volatile uint64_t *invl_va = sc_p->sc_invl_reg; 53 54 if (!len) 55 len = mp->dmai_size; 56 57 pg_off = mp->dmai_offset; /* start min */ 58 dvma_addr = MAX(off, pg_off); /* lo */ 59 pg_off += mp->dmai_size; /* end max */ 60 pg_off = MIN(off + len, pg_off); /* hi */ 61 if (dvma_addr >= pg_off) { /* lo >= hi ? */ 62 DEBUG4(DBG_SC, dip, "%x+%x out of window [%x,%x)\n", 63 off, len, mp->dmai_offset, 64 mp->dmai_offset + mp->dmai_size); 65 return; 66 } 67 68 len = pg_off - dvma_addr; /* sz = hi - lo */ 69 dvma_addr += mp->dmai_mapping; /* start addr */ 70 pg_off = dvma_addr & IOMMU_PAGE_OFFSET; /* offset in 1st pg */ 71 len = IOMMU_BTOPR(len + pg_off); /* # of pages */ 72 dvma_addr ^= pg_off; 73 74 DEBUG2(DBG_SC, dip, "addr=%x+%x pages: \n", dvma_addr, len); 75 for (; len; len--, dvma_addr += IOMMU_PAGE_SIZE) { 76 DEBUG1(DBG_SC|DBG_CONT, dip, " %x", dvma_addr); 77 *invl_va = (uint64_t)dvma_addr; 78 } 79 DEBUG0(DBG_SC|DBG_CONT, dip, "\n"); 80 } 81 82 static void 83 pci_dma_sync_flag_wait(ddi_dma_impl_t *mp, sc_t *sc_p, uint32_t onstack) 84 { 85 hrtime_t start_time; 86 uint64_t loops = 0; 87 uint64_t sync_flag_pa = SYNC_BUF_PA(mp); 88 uint64_t sync_reg_pa = sc_p->sc_sync_reg_pa; 89 uint8_t stack_buf[128]; 90 91 stack_buf[0] = DDI_SUCCESS; 92 93 /* check for handle specific sync flag */ 94 if (sync_flag_pa) 95 goto start; 96 97 sync_flag_pa = sc_p->sc_sync_flag_pa; 98 99 if (onstack) { 100 sync_flag_pa = va_to_pa(stack_buf); 101 sync_flag_pa += PCI_SYNC_FLAG_SIZE; 102 sync_flag_pa >>= PCI_SYNC_FLAG_SZSHIFT; 103 sync_flag_pa <<= PCI_SYNC_FLAG_SZSHIFT; 104 goto start; 105 } 106 stack_buf[0] |= PCI_SYNC_FLAG_LOCKED; 107 mutex_enter(&sc_p->sc_sync_mutex); 108 start: 109 ASSERT(!(sync_flag_pa & PCI_SYNC_FLAG_SIZE - 1)); 110 stdphys(sync_flag_pa, 0); /* reset sync flag to 0 */ 111 /* membar #LoadStore|#StoreStore */ 112 stdphysio(sync_reg_pa, sync_flag_pa); 113 start_time = gethrtime(); 114 115 for (; gethrtime() - start_time < pci_sync_buf_timeout; loops++) 116 if (lddphys(sync_flag_pa)) 117 goto done; 118 119 if (!lddphys(sync_flag_pa)) 120 stack_buf[0] |= PCI_SYNC_FLAG_FAILED; 121 done: 122 DEBUG3(DBG_SC|DBG_CONT, 0, "flag wait loops=%lu ticks=%lu status=%x\n", 123 loops, gethrtime() - start_time, stack_buf[0]); 124 125 if (stack_buf[0] & PCI_SYNC_FLAG_LOCKED) 126 mutex_exit(&sc_p->sc_sync_mutex); 127 128 if (stack_buf[0] & PCI_SYNC_FLAG_FAILED) 129 cmn_err(CE_PANIC, "%p pci dma sync %lx %lx timeout!", 130 mp, sync_flag_pa, loops); 131 } 132 133 /* 134 * Cache RW Before During After 135 * 136 * STREAMING read no/no pg/no ctx,pg/no 137 * STREAMING write no/no pg/yes ctx,pg/yes 138 * CONSISTENT read no/no yes,no/no yes,no/no 139 * CONSISTENT write no/no yes,yes/yes yes,yes/yes 140 * 141 * STREAMING read ctx,pg/no 142 * STREAMING write ctx,pg/yes 143 * CONSISTENT read yes,no/no 144 * CONSISTENT write yes,yes/yes 145 */ 146 int 147 pci_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 148 off_t off, size_t len, uint32_t sync_flag) 149 { 150 ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle; 151 int ret = ddi_get_instance(dip); 152 pci_t *pci_p = get_pci_soft_state(ret); 153 pbm_t *pbm_p = pci_p->pci_pbm_p; 154 uint32_t dev_flag = mp->dmai_rflags; 155 sc_t *sc_p; 156 157 DEBUG4(DBG_DMA_SYNC, dip, "%s%d flags=%x,%x\n", ddi_driver_name(rdip), 158 ddi_get_instance(rdip), dev_flag, sync_flag); 159 DEBUG4(DBG_SC, dip, "dmai_mapping=%x, dmai_sz=%x off=%x len=%x\n", 160 mp->dmai_mapping, mp->dmai_size, off, len); 161 DEBUG2(DBG_SC, dip, "mp=%p, ctx=%x\n", mp, MP2CTX(mp)); 162 163 if (!(mp->dmai_flags & DMAI_FLAGS_INUSE)) { 164 cmn_err(CE_WARN, "Unbound dma handle %p from %s%d", mp, 165 ddi_driver_name(rdip), ddi_get_instance(rdip)); 166 return (DDI_FAILURE); 167 } 168 169 if (mp->dmai_flags & DMAI_FLAGS_NOSYNC) 170 return (DDI_SUCCESS); 171 172 if (!(dev_flag & DDI_DMA_CONSISTENT)) 173 goto streaming; 174 175 if (sync_flag & PCI_DMA_SYNC_EXT) { 176 if (sync_flag & (PCI_DMA_SYNC_BEFORE | PCI_DMA_SYNC_POST) || 177 !(sync_flag & PCI_DMA_SYNC_WRITE)) 178 return (DDI_SUCCESS); 179 } else { 180 if (!(dev_flag & DDI_DMA_READ) || 181 ((sync_flag & PCI_DMA_SYNC_DDI_FLAGS) == 182 DDI_DMA_SYNC_FORDEV)) 183 return (DDI_SUCCESS); 184 } 185 186 pci_pbm_dma_sync(pbm_p, pbm_p->pbm_sync_ino); 187 return (DDI_SUCCESS); 188 189 streaming: 190 ASSERT(pci_stream_buf_exists && (pci_stream_buf_enable & 1 << ret)); 191 sc_p = pci_p->pci_sc_p; 192 ret = DDI_FAILURE; 193 194 if (sync_flag & PCI_DMA_SYNC_EXT) 195 goto ext; 196 197 if (mp->dmai_flags & DMAI_FLAGS_CONTEXT && pci_sc_use_contexts) 198 ret = pci_sc_ctx_inv(dip, sc_p, mp); 199 if (ret) 200 pci_sc_pg_inv(dip, sc_p, mp, off, len); 201 202 if ((dev_flag & DDI_DMA_READ) && 203 ((sync_flag & PCI_DMA_SYNC_DDI_FLAGS) != DDI_DMA_SYNC_FORDEV)) 204 goto wait; 205 206 return (DDI_SUCCESS); 207 ext: 208 if (sync_flag & PCI_DMA_SYNC_BEFORE) 209 return (DDI_SUCCESS); 210 if (sync_flag & PCI_DMA_SYNC_BAR) 211 goto wait_check; 212 if (sync_flag & PCI_DMA_SYNC_AFTER && 213 mp->dmai_flags & DMAI_FLAGS_CONTEXT && pci_sc_use_contexts) 214 ret = pci_sc_ctx_inv(dip, sc_p, mp); 215 if (ret) 216 pci_sc_pg_inv(dip, sc_p, mp, off, len); 217 wait_check: 218 if (sync_flag & PCI_DMA_SYNC_POST || !(sync_flag & PCI_DMA_SYNC_WRITE)) 219 return (DDI_SUCCESS); 220 wait: 221 pci_dma_sync_flag_wait(mp, sc_p, sync_flag & PCI_DMA_SYNC_PRIVATE); 222 return (DDI_SUCCESS); 223 } 224 225 int 226 pci_dma_handle_clean(dev_info_t *rdip, ddi_dma_handle_t h) 227 { 228 ddi_dma_impl_t *mp = (ddi_dma_impl_t *)h; 229 if ((mp->dmai_flags & DMAI_FLAGS_INUSE) == 0) 230 return (DDI_FAILURE); 231 mp->dmai_rflags |= DMP_NOSYNC; 232 mp->dmai_flags |= DMAI_FLAGS_NOSYNC; 233 return (DDI_SUCCESS); 234 } 235 236 /* 237 * pci_dma_allocmp - Allocate a pci dma implementation structure 238 * 239 * An extra ddi_dma_attr structure is bundled with the usual ddi_dma_impl 240 * to hold unmodified device limits. The ddi_dma_attr inside the 241 * ddi_dma_impl structure is augumented with system limits to enhance 242 * DVMA performance at runtime. The unaugumented device limits saved 243 * right after (accessed through the DEV_ATTR macro) is used 244 * strictly for peer-to-peer transfers which do not obey system limits. 245 * 246 * return: DDI_SUCCESS DDI_DMA_NORESOURCES 247 */ 248 ddi_dma_impl_t * 249 pci_dma_allocmp(dev_info_t *dip, dev_info_t *rdip, int (*waitfp)(caddr_t), 250 caddr_t arg) 251 { 252 ddi_dma_impl_t *mp; 253 int sleep = (waitfp == DDI_DMA_SLEEP) ? KM_SLEEP : KM_NOSLEEP; 254 255 /* Caution: we don't use zalloc to enhance performance! */ 256 if ((mp = kmem_alloc(sizeof (pci_dma_hdl_t), sleep)) == 0) { 257 DEBUG0(DBG_DMA_MAP, dip, "can't alloc dma_handle\n"); 258 if (waitfp != DDI_DMA_DONTWAIT) { 259 DEBUG0(DBG_DMA_MAP, dip, "alloc_mp kmem cb\n"); 260 ddi_set_callback(waitfp, arg, &pci_kmem_clid); 261 } 262 return (mp); 263 } 264 265 mp->dmai_rdip = rdip; 266 mp->dmai_flags = 0; 267 mp->dmai_pfnlst = NULL; 268 mp->dmai_winlst = NULL; 269 270 /* 271 * kmem_alloc debug: the following fields are not zero-ed 272 * mp->dmai_mapping = 0; 273 * mp->dmai_size = 0; 274 * mp->dmai_offset = 0; 275 * mp->dmai_minxfer = 0; 276 * mp->dmai_burstsizes = 0; 277 * mp->dmai_ndvmapages = 0; 278 * mp->dmai_pool/roffset = 0; 279 * mp->dmai_rflags = 0; 280 * mp->dmai_inuse/flags 281 * mp->dmai_nwin = 0; 282 * mp->dmai_winsize = 0; 283 * mp->dmai_nexus_private/tte = 0; 284 * mp->dmai_iopte/pfnlst 285 * mp->dmai_sbi/pfn0 = 0; 286 * mp->dmai_minfo/winlst/fdvma 287 * mp->dmai_rdip 288 * bzero(&mp->dmai_object, sizeof (ddi_dma_obj_t)); 289 * mp->dmai_cookie = 0; 290 */ 291 292 mp->dmai_attr.dma_attr_version = (uint_t)DMA_ATTR_VERSION; 293 mp->dmai_attr.dma_attr_flags = (uint_t)0; 294 mp->dmai_fault = 0; 295 mp->dmai_fault_check = NULL; 296 mp->dmai_fault_notify = NULL; 297 298 mp->dmai_error.err_ena = 0; 299 mp->dmai_error.err_status = DDI_FM_OK; 300 mp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 301 mp->dmai_error.err_ontrap = NULL; 302 mp->dmai_error.err_fep = NULL; 303 304 SYNC_BUF_PA(mp) = 0ull; 305 return (mp); 306 } 307 308 void 309 pci_dma_freemp(ddi_dma_impl_t *mp) 310 { 311 if (mp->dmai_ndvmapages > 1) 312 pci_dma_freepfn(mp); 313 if (mp->dmai_winlst) 314 pci_dma_freewin(mp); 315 kmem_free(mp, sizeof (pci_dma_hdl_t)); 316 } 317 318 void 319 pci_dma_freepfn(ddi_dma_impl_t *mp) 320 { 321 void *addr = mp->dmai_pfnlst; 322 ASSERT(!PCI_DMA_CANRELOC(mp)); 323 if (addr) { 324 size_t npages = mp->dmai_ndvmapages; 325 if (npages > 1) 326 kmem_free(addr, npages * sizeof (iopfn_t)); 327 mp->dmai_pfnlst = NULL; 328 } 329 mp->dmai_ndvmapages = 0; 330 } 331 332 /* 333 * pci_dma_lmts2hdl - alloate a ddi_dma_impl_t, validate practical limits 334 * and convert dmareq->dmar_limits to mp->dmai_attr 335 * 336 * ddi_dma_impl_t member modified input 337 * ------------------------------------------------------------------------ 338 * mp->dmai_minxfer - dev 339 * mp->dmai_burstsizes - dev 340 * mp->dmai_flags - no limit? peer-to-peer only? 341 * 342 * ddi_dma_attr member modified input 343 * ------------------------------------------------------------------------ 344 * mp->dmai_attr.dma_attr_addr_lo - dev lo, sys lo 345 * mp->dmai_attr.dma_attr_addr_hi - dev hi, sys hi 346 * mp->dmai_attr.dma_attr_count_max - dev count max, dev/sys lo/hi delta 347 * mp->dmai_attr.dma_attr_seg - 0 (no nocross restriction) 348 * mp->dmai_attr.dma_attr_align - 1 (no alignment restriction) 349 * 350 * The dlim_dmaspeed member of dmareq->dmar_limits is ignored. 351 */ 352 ddi_dma_impl_t * 353 pci_dma_lmts2hdl(dev_info_t *dip, dev_info_t *rdip, iommu_t *iommu_p, 354 ddi_dma_req_t *dmareq) 355 { 356 ddi_dma_impl_t *mp; 357 ddi_dma_attr_t *attr_p; 358 uint64_t syslo = iommu_p->iommu_dvma_base; 359 uint64_t syshi = iommu_p->iommu_dvma_end; 360 uint64_t fasthi = iommu_p->iommu_dvma_fast_end; 361 ddi_dma_lim_t *lim_p = dmareq->dmar_limits; 362 uint32_t count_max = lim_p->dlim_cntr_max; 363 uint64_t lo = lim_p->dlim_addr_lo; 364 uint64_t hi = lim_p->dlim_addr_hi; 365 if (hi <= lo) { 366 DEBUG0(DBG_DMA_MAP, dip, "Bad limits\n"); 367 return ((ddi_dma_impl_t *)DDI_DMA_NOMAPPING); 368 } 369 if (!count_max) 370 count_max--; 371 372 if (!(mp = pci_dma_allocmp(dip, rdip, dmareq->dmar_fp, 373 dmareq->dmar_arg))) 374 return (NULL); 375 376 /* store original dev input at the 2nd ddi_dma_attr */ 377 attr_p = DEV_ATTR(mp); 378 SET_DMAATTR(attr_p, lo, hi, -1, count_max); 379 SET_DMAALIGN(attr_p, 1); 380 381 lo = MAX(lo, syslo); 382 hi = MIN(hi, syshi); 383 if (hi <= lo) 384 mp->dmai_flags |= DMAI_FLAGS_PEER_ONLY; 385 count_max = MIN(count_max, hi - lo); 386 387 if (DEV_NOSYSLIMIT(lo, hi, syslo, fasthi, 1)) 388 mp->dmai_flags |= DMAI_FLAGS_NOFASTLIMIT | 389 DMAI_FLAGS_NOSYSLIMIT; 390 else { 391 if (DEV_NOFASTLIMIT(lo, hi, syslo, syshi, 1)) 392 mp->dmai_flags |= DMAI_FLAGS_NOFASTLIMIT; 393 } 394 if (PCI_DMA_NOCTX(rdip)) 395 mp->dmai_flags |= DMAI_FLAGS_NOCTX; 396 397 /* store augumented dev input to mp->dmai_attr */ 398 mp->dmai_minxfer = lim_p->dlim_minxfer; 399 mp->dmai_burstsizes = lim_p->dlim_burstsizes; 400 attr_p = &mp->dmai_attr; 401 SET_DMAATTR(attr_p, lo, hi, -1, count_max); 402 SET_DMAALIGN(attr_p, 1); 403 return (mp); 404 } 405 406 /* 407 * pci_dma_attr2hdl 408 * 409 * This routine is called from the alloc handle entry point to sanity check the 410 * dma attribute structure. 411 * 412 * use by: pci_dma_allochdl() 413 * 414 * return value: 415 * 416 * DDI_SUCCESS - on success 417 * DDI_DMA_BADATTR - attribute has invalid version number 418 * or address limits exclude dvma space 419 */ 420 int 421 pci_dma_attr2hdl(pci_t *pci_p, ddi_dma_impl_t *mp) 422 { 423 iommu_t *iommu_p = pci_p->pci_iommu_p; 424 uint64_t syslo, syshi; 425 ddi_dma_attr_t *attrp = DEV_ATTR(mp); 426 uint64_t hi = attrp->dma_attr_addr_hi; 427 uint64_t lo = attrp->dma_attr_addr_lo; 428 uint64_t align = attrp->dma_attr_align; 429 uint64_t nocross = attrp->dma_attr_seg; 430 uint64_t count_max = attrp->dma_attr_count_max; 431 432 DEBUG3(DBG_DMA_ALLOCH, pci_p->pci_dip, "attrp=%p cntr_max=%x.%08x\n", 433 attrp, HI32(count_max), LO32(count_max)); 434 DEBUG4(DBG_DMA_ALLOCH, pci_p->pci_dip, "hi=%x.%08x lo=%x.%08x\n", 435 HI32(hi), LO32(hi), HI32(lo), LO32(lo)); 436 DEBUG4(DBG_DMA_ALLOCH, pci_p->pci_dip, "seg=%x.%08x align=%x.%08x\n", 437 HI32(nocross), LO32(nocross), HI32(align), LO32(align)); 438 439 if (!nocross) 440 nocross--; 441 if (attrp->dma_attr_flags & DDI_DMA_FORCE_PHYSICAL) { /* BYPASS */ 442 443 DEBUG0(DBG_DMA_ALLOCH, pci_p->pci_dip, "bypass mode\n"); 444 /* if tomatillo ver <= 2.3 don't allow bypass */ 445 if (tomatillo_disallow_bypass) 446 return (DDI_DMA_BADATTR); 447 448 mp->dmai_flags |= DMAI_FLAGS_BYPASSREQ; 449 if (nocross != UINT64_MAX) 450 return (DDI_DMA_BADATTR); 451 if (align && (align > IOMMU_PAGE_SIZE)) 452 return (DDI_DMA_BADATTR); 453 align = 1; /* align on 1 page boundary */ 454 syslo = iommu_p->iommu_dma_bypass_base; 455 syshi = iommu_p->iommu_dma_bypass_end; 456 457 } else { /* IOMMU_XLATE or PEER_TO_PEER */ 458 align = MAX(align, IOMMU_PAGE_SIZE) - 1; 459 if ((align & nocross) != align) { 460 dev_info_t *rdip = mp->dmai_rdip; 461 cmn_err(CE_WARN, "%s%d dma_attr_seg not aligned", 462 NAMEINST(rdip)); 463 return (DDI_DMA_BADATTR); 464 } 465 align = IOMMU_BTOP(align + 1); 466 syslo = iommu_p->iommu_dvma_base; 467 syshi = iommu_p->iommu_dvma_end; 468 } 469 if (hi <= lo) { 470 dev_info_t *rdip = mp->dmai_rdip; 471 cmn_err(CE_WARN, "%s%d limits out of range", NAMEINST(rdip)); 472 return (DDI_DMA_BADATTR); 473 } 474 lo = MAX(lo, syslo); 475 hi = MIN(hi, syshi); 476 if (!count_max) 477 count_max--; 478 479 DEBUG4(DBG_DMA_ALLOCH, pci_p->pci_dip, "hi=%x.%08x, lo=%x.%08x\n", 480 HI32(hi), LO32(hi), HI32(lo), LO32(lo)); 481 if (hi <= lo) { /* peer transfers cannot have alignment & nocross */ 482 dev_info_t *rdip = mp->dmai_rdip; 483 cmn_err(CE_WARN, "%s%d peer only dev %p", NAMEINST(rdip), mp); 484 if ((nocross < UINT32_MAX) || (align > 1)) { 485 cmn_err(CE_WARN, "%s%d peer only device bad attr", 486 NAMEINST(rdip)); 487 return (DDI_DMA_BADATTR); 488 } 489 mp->dmai_flags |= DMAI_FLAGS_PEER_ONLY; 490 } else /* set practical counter_max value */ 491 count_max = MIN(count_max, hi - lo); 492 493 if (DEV_NOSYSLIMIT(lo, hi, syslo, syshi, align)) 494 mp->dmai_flags |= DMAI_FLAGS_NOSYSLIMIT | 495 DMAI_FLAGS_NOFASTLIMIT; 496 else { 497 syshi = iommu_p->iommu_dvma_fast_end; 498 if (DEV_NOFASTLIMIT(lo, hi, syslo, syshi, align)) 499 mp->dmai_flags |= DMAI_FLAGS_NOFASTLIMIT; 500 } 501 if (PCI_DMA_NOCTX(mp->dmai_rdip)) 502 mp->dmai_flags |= DMAI_FLAGS_NOCTX; 503 504 mp->dmai_minxfer = attrp->dma_attr_minxfer; 505 mp->dmai_burstsizes = attrp->dma_attr_burstsizes; 506 attrp = &mp->dmai_attr; 507 SET_DMAATTR(attrp, lo, hi, nocross, count_max); 508 return (DDI_SUCCESS); 509 } 510 511 /* 512 * set up consistent dma flags according to hardware capability 513 */ 514 uint32_t 515 pci_dma_consist_check(uint32_t req_flags, pbm_t *pbm_p) 516 { 517 if (!pci_stream_buf_enable || !pci_stream_buf_exists) 518 req_flags |= DDI_DMA_CONSISTENT; 519 if (req_flags & DDI_DMA_CONSISTENT && !pbm_p->pbm_sync_reg_pa) 520 req_flags |= DMP_NOSYNC; 521 return (req_flags); 522 } 523 524 #define TGT_PFN_INBETWEEN(pfn, bgn, end) ((pfn >= bgn) && (pfn <= end)) 525 526 /* 527 * pci_dma_type - determine which of the three types DMA (peer-to-peer, 528 * iommu bypass, or iommu translate) we are asked to do. 529 * Also checks pfn0 and rejects any non-peer-to-peer 530 * requests for peer-only devices. 531 * 532 * return values: 533 * DDI_DMA_NOMAPPING - can't get valid pfn0, or bad dma type 534 * DDI_SUCCESS 535 * 536 * dma handle members affected (set on exit): 537 * mp->dmai_object - dmareq->dmar_object 538 * mp->dmai_rflags - consistent?, nosync?, dmareq->dmar_flags 539 * mp->dmai_flags - DMA type 540 * mp->dmai_pfn0 - 1st page pfn (if va/size pair and not shadow) 541 * mp->dmai_roffset - initialized to starting IOMMU page offset 542 * mp->dmai_ndvmapages - # of total IOMMU pages of entire object 543 * mp->pdh_sync_buf_pa - dma sync buffer PA is DMA flow is supported 544 */ 545 int 546 pci_dma_type(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 547 { 548 dev_info_t *dip = pci_p->pci_dip; 549 ddi_dma_obj_t *dobj_p = &dmareq->dmar_object; 550 pbm_t *pbm_p = pci_p->pci_pbm_p; 551 page_t **pplist; 552 struct as *as_p; 553 uint32_t offset; 554 caddr_t vaddr; 555 pfn_t pfn0; 556 557 mp->dmai_rflags = pci_dma_consist_check(dmareq->dmar_flags, pbm_p); 558 mp->dmai_flags |= mp->dmai_rflags & DMP_NOSYNC ? DMAI_FLAGS_NOSYNC : 0; 559 560 switch (dobj_p->dmao_type) { 561 case DMA_OTYP_BUFVADDR: 562 case DMA_OTYP_VADDR: { 563 vaddr = dobj_p->dmao_obj.virt_obj.v_addr; 564 pplist = dobj_p->dmao_obj.virt_obj.v_priv; 565 as_p = dobj_p->dmao_obj.virt_obj.v_as; 566 if (as_p == NULL) 567 as_p = &kas; 568 569 DEBUG2(DBG_DMA_MAP, dip, "vaddr=%p pplist=%p\n", vaddr, pplist); 570 offset = (ulong_t)vaddr & IOMMU_PAGE_OFFSET; 571 572 if (pplist) { /* shadow list */ 573 mp->dmai_flags |= DMAI_FLAGS_PGPFN; 574 ASSERT(PAGE_LOCKED(*pplist)); 575 pfn0 = page_pptonum(*pplist); 576 } else if (pci_dvma_remap_enabled && as_p == &kas && 577 dobj_p->dmao_type != DMA_OTYP_BUFVADDR) { 578 int (*waitfp)(caddr_t) = dmareq->dmar_fp; 579 uint_t flags = ((waitfp == DDI_DMA_SLEEP)? 580 HAC_SLEEP : HAC_NOSLEEP) | HAC_PAGELOCK; 581 int ret; 582 583 ret = hat_add_callback(pci_dvma_cbid, vaddr, 584 IOMMU_PAGE_SIZE - offset, flags, mp, &pfn0); 585 586 if (pfn0 == PFN_INVALID && ret == ENOMEM) { 587 ASSERT(waitfp != DDI_DMA_SLEEP); 588 if (waitfp != DDI_DMA_DONTWAIT) { 589 ddi_set_callback(waitfp, 590 dmareq->dmar_arg, 591 &pci_kmem_clid); 592 return (DDI_DMA_NORESOURCES); 593 } 594 } 595 mp->dmai_flags |= DMAI_FLAGS_RELOC; 596 } else 597 pfn0 = hat_getpfnum(as_p->a_hat, vaddr); 598 } 599 break; 600 601 case DMA_OTYP_PAGES: 602 offset = dobj_p->dmao_obj.pp_obj.pp_offset; 603 mp->dmai_flags |= DMAI_FLAGS_PGPFN; 604 pfn0 = page_pptonum(dobj_p->dmao_obj.pp_obj.pp_pp); 605 ASSERT(PAGE_LOCKED(dobj_p->dmao_obj.pp_obj.pp_pp)); 606 break; 607 608 case DMA_OTYP_PADDR: 609 default: 610 cmn_err(CE_WARN, "%s%d requested unsupported dma type %x", 611 NAMEINST(mp->dmai_rdip), dobj_p->dmao_type); 612 return (DDI_DMA_NOMAPPING); 613 } 614 if (pfn0 == PFN_INVALID) { 615 cmn_err(CE_WARN, "%s%d: invalid pfn0 for DMA object %p", 616 NAMEINST(dip), dobj_p); 617 return (DDI_DMA_NOMAPPING); 618 } 619 if (TGT_PFN_INBETWEEN(pfn0, pbm_p->pbm_base_pfn, pbm_p->pbm_last_pfn)) { 620 mp->dmai_flags |= DMAI_FLAGS_PEER_TO_PEER; 621 goto done; /* leave bypass and dvma flag as 0 */ 622 } 623 if (PCI_DMA_ISPEERONLY(mp)) { 624 dev_info_t *rdip = mp->dmai_rdip; 625 cmn_err(CE_WARN, "Bad peer-to-peer req %s%d", NAMEINST(rdip)); 626 return (DDI_DMA_NOMAPPING); 627 } 628 mp->dmai_flags |= (mp->dmai_flags & DMAI_FLAGS_BYPASSREQ) ? 629 DMAI_FLAGS_BYPASS : DMAI_FLAGS_DVMA; 630 done: 631 mp->dmai_object = *dobj_p; /* whole object */ 632 mp->dmai_pfn0 = (void *)pfn0; /* cache pfn0 */ 633 mp->dmai_roffset = offset; /* win0 pg0 offset */ 634 mp->dmai_ndvmapages = IOMMU_BTOPR(offset + mp->dmai_object.dmao_size); 635 636 return (DDI_SUCCESS); 637 } 638 639 /* 640 * pci_dma_pgpfn - set up pfnlst array according to pages 641 * VA/size pair: <shadow IO, bypass, peer-to-peer>, or OTYP_PAGES 642 */ 643 /*ARGSUSED*/ 644 static int 645 pci_dma_pgpfn(pci_t *pci_p, ddi_dma_impl_t *mp, uint_t npages) 646 { 647 int i; 648 #ifdef DEBUG 649 dev_info_t *dip = pci_p->pci_dip; 650 #endif 651 switch (mp->dmai_object.dmao_type) { 652 case DMA_OTYP_BUFVADDR: 653 case DMA_OTYP_VADDR: { 654 page_t **pplist = mp->dmai_object.dmao_obj.virt_obj.v_priv; 655 DEBUG2(DBG_DMA_MAP, dip, "shadow pplist=%p, %x pages, pfns=", 656 pplist, npages); 657 for (i = 1; i < npages; i++) { 658 iopfn_t pfn = page_pptonum(pplist[i]); 659 ASSERT(PAGE_LOCKED(pplist[i])); 660 PCI_SET_MP_PFN1(mp, i, pfn); 661 DEBUG1(DBG_DMA_MAP|DBG_CONT, dip, "%x ", pfn); 662 } 663 DEBUG0(DBG_DMA_MAP|DBG_CONT, dip, "\n"); 664 } 665 break; 666 667 case DMA_OTYP_PAGES: { 668 page_t *pp = mp->dmai_object.dmao_obj.pp_obj.pp_pp->p_next; 669 DEBUG1(DBG_DMA_MAP, dip, "pp=%p pfns=", pp); 670 for (i = 1; i < npages; i++, pp = pp->p_next) { 671 iopfn_t pfn = page_pptonum(pp); 672 ASSERT(PAGE_LOCKED(pp)); 673 PCI_SET_MP_PFN1(mp, i, pfn); 674 DEBUG1(DBG_DMA_MAP|DBG_CONT, dip, "%x ", pfn); 675 } 676 DEBUG0(DBG_DMA_MAP|DBG_CONT, dip, "\n"); 677 } 678 break; 679 680 default: /* check is already done by pci_dma_type */ 681 ASSERT(0); 682 break; 683 } 684 return (DDI_SUCCESS); 685 } 686 687 /* 688 * pci_dma_vapfn - set up pfnlst array according to VA 689 * VA/size pair: <normal, bypass, peer-to-peer> 690 * pfn0 is skipped as it is already done. 691 * In this case, the cached pfn0 is used to fill pfnlst[0] 692 */ 693 static int 694 pci_dma_vapfn(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, 695 uint_t npages) 696 { 697 dev_info_t *dip = pci_p->pci_dip; 698 int i; 699 caddr_t vaddr = (caddr_t)mp->dmai_object.dmao_obj.virt_obj.v_as; 700 struct hat *hat_p = vaddr ? ((struct as *)vaddr)->a_hat : kas.a_hat; 701 caddr_t sva; 702 int needcb = 0; 703 704 sva = (caddr_t)(((uintptr_t)mp->dmai_object.dmao_obj.virt_obj.v_addr + 705 IOMMU_PAGE_SIZE) & IOMMU_PAGE_MASK); 706 707 if (pci_dvma_remap_enabled && hat_p == kas.a_hat && 708 mp->dmai_object.dmao_type != DMA_OTYP_BUFVADDR) 709 needcb = 1; 710 711 for (vaddr = sva, i = 1; i < npages; i++, vaddr += IOMMU_PAGE_SIZE) { 712 pfn_t pfn; 713 714 if (needcb) { 715 int (*waitfp)(caddr_t) = dmareq->dmar_fp; 716 uint_t flags = ((waitfp == DDI_DMA_SLEEP)? 717 HAC_SLEEP : HAC_NOSLEEP) | HAC_PAGELOCK; 718 int ret; 719 720 ret = hat_add_callback(pci_dvma_cbid, vaddr, 721 IOMMU_PAGE_SIZE, flags, mp, &pfn); 722 if (pfn == PFN_INVALID && ret == ENOMEM) { 723 ASSERT(waitfp != DDI_DMA_SLEEP); 724 if (waitfp != DDI_DMA_DONTWAIT) 725 ddi_set_callback(waitfp, 726 dmareq->dmar_arg, &pci_kmem_clid); 727 return (DDI_DMA_NORESOURCES); 728 } 729 } else 730 pfn = hat_getpfnum(hat_p, vaddr); 731 if (pfn == PFN_INVALID) 732 goto err_badpfn; 733 PCI_SET_MP_PFN1(mp, i, (iopfn_t)pfn); 734 DEBUG3(DBG_DMA_MAP, dip, "pci_dma_vapfn: mp=%p pfnlst[%x]=%x\n", 735 mp, i, (iopfn_t)pfn); 736 } 737 return (DDI_SUCCESS); 738 err_badpfn: 739 cmn_err(CE_WARN, "%s%d: bad page frame vaddr=%p", NAMEINST(dip), vaddr); 740 return (DDI_DMA_NOMAPPING); 741 } 742 743 /* 744 * pci_dma_pfn - Fills pfn list for all pages being DMA-ed. 745 * 746 * dependencies: 747 * mp->dmai_ndvmapages - set to total # of dma pages 748 * 749 * return value: 750 * DDI_SUCCESS 751 * DDI_DMA_NOMAPPING 752 */ 753 int 754 pci_dma_pfn(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 755 { 756 uint32_t npages = mp->dmai_ndvmapages; 757 int (*waitfp)(caddr_t) = dmareq->dmar_fp; 758 int i, ret, peer = PCI_DMA_ISPTP(mp); 759 760 pbm_t *pbm_p = pci_p->pci_pbm_p; 761 iopfn_t pfn_base = pbm_p->pbm_base_pfn; 762 iopfn_t pfn_last = pbm_p->pbm_last_pfn; 763 iopfn_t pfn_adj = peer ? pfn_base : 0; 764 765 DEBUG2(DBG_DMA_MAP, pci_p->pci_dip, "pci_dma_pfn: mp=%p pfn0=%x\n", 766 mp, MP_PFN0(mp) - pfn_adj); 767 /* 1 page: no array alloc/fill, no mixed mode check */ 768 if (npages == 1) { 769 PCI_SET_MP_PFN(mp, 0, MP_PFN0(mp) - pfn_adj); 770 return (DDI_SUCCESS); 771 } 772 /* allocate pfn array */ 773 if (!(mp->dmai_pfnlst = kmem_alloc(npages * sizeof (iopfn_t), 774 waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP))) { 775 if (waitfp != DDI_DMA_DONTWAIT) 776 ddi_set_callback(waitfp, dmareq->dmar_arg, 777 &pci_kmem_clid); 778 return (DDI_DMA_NORESOURCES); 779 } 780 /* fill pfn array */ 781 PCI_SET_MP_PFN(mp, 0, MP_PFN0(mp) - pfn_adj); /* pfnlst[0] */ 782 if ((ret = PCI_DMA_ISPGPFN(mp) ? pci_dma_pgpfn(pci_p, mp, npages) : 783 pci_dma_vapfn(pci_p, dmareq, mp, npages)) != DDI_SUCCESS) 784 goto err; 785 786 /* skip pfn0, check mixed mode and adjust peer to peer pfn */ 787 for (i = 1; i < npages; i++) { 788 iopfn_t pfn = PCI_GET_MP_PFN1(mp, i); 789 if (peer ^ TGT_PFN_INBETWEEN(pfn, pfn_base, pfn_last)) { 790 cmn_err(CE_WARN, "%s%d mixed mode DMA %lx %lx", 791 NAMEINST(mp->dmai_rdip), MP_PFN0(mp), pfn); 792 ret = DDI_DMA_NOMAPPING; /* mixed mode */ 793 goto err; 794 } 795 DEBUG3(DBG_DMA_MAP, pci_p->pci_dip, 796 "pci_dma_pfn: pfnlst[%x]=%x-%x\n", i, pfn, pfn_adj); 797 if (pfn_adj) 798 PCI_SET_MP_PFN1(mp, i, pfn - pfn_adj); 799 } 800 return (DDI_SUCCESS); 801 err: 802 pci_dvma_unregister_callbacks(pci_p, mp); 803 pci_dma_freepfn(mp); 804 return (ret); 805 } 806 807 /* 808 * pci_dvma_win() - trim requested DVMA size down to window size 809 * The 1st window starts from offset and ends at page-aligned boundary. 810 * From the 2nd window on, each window starts and ends at page-aligned 811 * boundary except the last window ends at wherever requested. 812 * 813 * accesses the following mp-> members: 814 * mp->dmai_attr.dma_attr_count_max 815 * mp->dmai_attr.dma_attr_seg 816 * mp->dmai_roffset - start offset of 1st window 817 * mp->dmai_rflags (redzone) 818 * mp->dmai_ndvmapages (for 1 page fast path) 819 * 820 * sets the following mp-> members: 821 * mp->dmai_size - xfer size, != winsize if 1st/last win (not fixed) 822 * mp->dmai_winsize - window size (no redzone), n * page size (fixed) 823 * mp->dmai_nwin - # of DMA windows of entire object (fixed) 824 * mp->dmai_rflags - remove partial flag if nwin == 1 (fixed) 825 * mp->dmai_winlst - NULL, window objects not used for DVMA (fixed) 826 * 827 * fixed - not changed across different DMA windows 828 */ 829 /*ARGSUSED*/ 830 int 831 pci_dvma_win(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 832 { 833 uint32_t redzone_sz = HAS_REDZONE(mp) ? IOMMU_PAGE_SIZE : 0; 834 size_t obj_sz = mp->dmai_object.dmao_size; 835 size_t xfer_sz; 836 ulong_t pg_off; 837 838 if ((mp->dmai_ndvmapages == 1) && !redzone_sz) { 839 mp->dmai_rflags &= ~DDI_DMA_PARTIAL; 840 mp->dmai_size = obj_sz; 841 mp->dmai_winsize = IOMMU_PAGE_SIZE; 842 mp->dmai_nwin = 1; 843 goto done; 844 } 845 846 pg_off = mp->dmai_roffset; 847 xfer_sz = obj_sz + redzone_sz; 848 849 /* include redzone in nocross check */ { 850 uint64_t nocross = mp->dmai_attr.dma_attr_seg; 851 if (xfer_sz + pg_off - 1 > nocross) 852 xfer_sz = nocross - pg_off + 1; 853 if (redzone_sz && (xfer_sz <= redzone_sz)) { 854 DEBUG5(DBG_DMA_MAP, pci_p->pci_dip, 855 "nocross too small %lx(%lx)+%lx+%x < %" PRIx64 "\n", 856 xfer_sz, obj_sz, pg_off, redzone_sz, nocross); 857 return (DDI_DMA_TOOBIG); 858 } 859 } 860 xfer_sz -= redzone_sz; /* restore transfer size */ 861 /* check counter max */ { 862 uint32_t count_max = mp->dmai_attr.dma_attr_count_max; 863 if (xfer_sz - 1 > count_max) 864 xfer_sz = count_max + 1; 865 } 866 if (xfer_sz >= obj_sz) { 867 mp->dmai_rflags &= ~DDI_DMA_PARTIAL; 868 mp->dmai_size = xfer_sz; 869 mp->dmai_winsize = P2ROUNDUP(xfer_sz + pg_off, IOMMU_PAGE_SIZE); 870 mp->dmai_nwin = 1; 871 goto done; 872 } 873 if (!(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 874 DEBUG4(DBG_DMA_MAP, pci_p->pci_dip, 875 "too big: %lx+%lx+%x > %lx\n", 876 obj_sz, pg_off, redzone_sz, xfer_sz); 877 return (DDI_DMA_TOOBIG); 878 } 879 880 xfer_sz = IOMMU_PTOB(IOMMU_BTOP(xfer_sz + pg_off)); /* page align */ 881 mp->dmai_size = xfer_sz - pg_off; /* 1st window xferrable size */ 882 mp->dmai_winsize = xfer_sz; /* redzone not in winsize */ 883 mp->dmai_nwin = (obj_sz + pg_off + xfer_sz - 1) / xfer_sz; 884 done: 885 mp->dmai_winlst = NULL; 886 dump_dma_handle(DBG_DMA_MAP, pci_p->pci_dip, mp); 887 return (DDI_SUCCESS); 888 } 889 890 /* 891 * fast track cache entry to iommu context, inserts 3 0 bits between 892 * upper 6-bits and lower 3-bits of the 9-bit cache entry 893 */ 894 #define IOMMU_FCE_TO_CTX(i) (((i) << 3) | ((i) & 0x7) | 0x38) 895 896 /* 897 * pci_dvma_map_fast - attempts to map fast trackable DVMA 898 */ 899 int 900 pci_dvma_map_fast(iommu_t *iommu_p, ddi_dma_impl_t *mp) 901 { 902 uint_t clustsz = pci_dvma_page_cache_clustsz; 903 uint_t entries = pci_dvma_page_cache_entries; 904 uint64_t *tte_addr; 905 uint64_t tte = GET_TTE_TEMPLATE(mp); 906 int i = iommu_p->iommu_dvma_addr_scan_start; 907 uint8_t *lock_addr = iommu_p->iommu_dvma_cache_locks + i; 908 iopfn_t *pfn_addr; 909 dvma_addr_t dvma_pg; 910 size_t npages = IOMMU_BTOP(mp->dmai_winsize); 911 #ifdef DEBUG 912 dev_info_t *dip = mp->dmai_rdip; 913 #endif 914 extern uint8_t ldstub(uint8_t *); 915 ASSERT(IOMMU_PTOB(npages) == mp->dmai_winsize); 916 ASSERT(npages + HAS_REDZONE(mp) <= clustsz); 917 918 for (; i < entries && ldstub(lock_addr); i++, lock_addr++); 919 if (i >= entries) { 920 lock_addr = iommu_p->iommu_dvma_cache_locks; 921 i = 0; 922 for (; i < entries && ldstub(lock_addr); i++, lock_addr++); 923 if (i >= entries) { 924 #ifdef PCI_DMA_PROF 925 pci_dvmaft_exhaust++; 926 #endif 927 return (DDI_DMA_NORESOURCES); 928 } 929 } 930 iommu_p->iommu_dvma_addr_scan_start = (i + 1) & (entries - 1); 931 if (PCI_DMA_USECTX(mp)) { 932 dvma_context_t ctx = IOMMU_FCE_TO_CTX(i); 933 tte |= IOMMU_CTX2TTE(ctx); 934 mp->dmai_flags |= DMAI_FLAGS_CONTEXT; 935 DEBUG1(DBG_DMA_MAP, dip, "fast: ctx=0x%x\n", ctx); 936 } 937 i *= clustsz; 938 tte_addr = iommu_p->iommu_tsb_vaddr + i; 939 dvma_pg = iommu_p->dvma_base_pg + i; 940 #ifdef DEBUG 941 for (i = 0; i < clustsz; i++) 942 ASSERT(TTE_IS_INVALID(tte_addr[i])); 943 #endif 944 *tte_addr = tte | IOMMU_PTOB(MP_PFN0(mp)); /* map page 0 */ 945 DEBUG5(DBG_DMA_MAP, dip, "fast %p:dvma_pg=%x tte0(%p)=%08x.%08x\n", mp, 946 dvma_pg, tte_addr, HI32(*tte_addr), LO32(*tte_addr)); 947 if (npages == 1) 948 goto tte_done; 949 pfn_addr = PCI_GET_MP_PFN1_ADDR(mp); /* short iommu_map_pages() */ 950 for (tte_addr++, i = 1; i < npages; i++, tte_addr++, pfn_addr++) { 951 *tte_addr = tte | IOMMU_PTOB(*pfn_addr); 952 DEBUG5(DBG_DMA_MAP, dip, "fast %p:tte(%p, %p)=%08x.%08x\n", mp, 953 tte_addr, pfn_addr, HI32(*tte_addr), LO32(*tte_addr)); 954 } 955 tte_done: 956 #ifdef PCI_DMA_PROF 957 pci_dvmaft_success++; 958 #endif 959 mp->dmai_mapping = mp->dmai_roffset | IOMMU_PTOB(dvma_pg); 960 mp->dmai_offset = 0; 961 mp->dmai_flags |= DMAI_FLAGS_FASTTRACK; 962 PCI_SAVE_MP_TTE(mp, tte); /* save TTE template for unmapping */ 963 if (DVMA_DBG_ON(iommu_p)) 964 pci_dvma_alloc_debug(iommu_p, (char *)mp->dmai_mapping, 965 mp->dmai_size, mp); 966 return (DDI_SUCCESS); 967 } 968 969 /* 970 * pci_dvma_map: map non-fasttrack DMA 971 * Use quantum cache if single page DMA. 972 */ 973 int 974 pci_dvma_map(ddi_dma_impl_t *mp, ddi_dma_req_t *dmareq, iommu_t *iommu_p) 975 { 976 uint_t npages = PCI_DMA_WINNPGS(mp); 977 dvma_addr_t dvma_pg, dvma_pg_index; 978 void *dvma_addr; 979 uint64_t tte = GET_TTE_TEMPLATE(mp); 980 int sleep = dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP; 981 #ifdef DEBUG 982 dev_info_t *dip = mp->dmai_rdip; 983 #endif 984 /* 985 * allocate dvma space resource and map in the first window. 986 * (vmem_t *vmp, size_t size, 987 * size_t align, size_t phase, size_t nocross, 988 * void *minaddr, void *maxaddr, int vmflag) 989 */ 990 if ((npages == 1) && !HAS_REDZONE(mp) && HAS_NOSYSLIMIT(mp)) { 991 dvma_addr = vmem_alloc(iommu_p->iommu_dvma_map, 992 IOMMU_PAGE_SIZE, sleep); 993 mp->dmai_flags |= DMAI_FLAGS_VMEMCACHE; 994 #ifdef PCI_DMA_PROF 995 pci_dvma_vmem_alloc++; 996 #endif 997 } else { 998 dvma_addr = vmem_xalloc(iommu_p->iommu_dvma_map, 999 IOMMU_PTOB(npages + HAS_REDZONE(mp)), 1000 MAX(mp->dmai_attr.dma_attr_align, IOMMU_PAGE_SIZE), 1001 0, 1002 mp->dmai_attr.dma_attr_seg + 1, 1003 (void *)mp->dmai_attr.dma_attr_addr_lo, 1004 (void *)(mp->dmai_attr.dma_attr_addr_hi + 1), 1005 sleep); 1006 #ifdef PCI_DMA_PROF 1007 pci_dvma_vmem_xalloc++; 1008 #endif 1009 } 1010 dvma_pg = IOMMU_BTOP((ulong_t)dvma_addr); 1011 dvma_pg_index = dvma_pg - iommu_p->dvma_base_pg; 1012 DEBUG2(DBG_DMA_MAP, dip, "fallback dvma_pages: dvma_pg=%x index=%x\n", 1013 dvma_pg, dvma_pg_index); 1014 if (dvma_pg == 0) 1015 goto noresource; 1016 1017 /* allocate DVMA context */ 1018 if ((npages >= pci_context_minpages) && PCI_DMA_USECTX(mp)) { 1019 dvma_context_t ctx; 1020 if (ctx = pci_iommu_get_dvma_context(iommu_p, dvma_pg_index)) { 1021 tte |= IOMMU_CTX2TTE(ctx); 1022 mp->dmai_flags |= DMAI_FLAGS_CONTEXT; 1023 } 1024 } 1025 mp->dmai_mapping = mp->dmai_roffset | IOMMU_PTOB(dvma_pg); 1026 mp->dmai_offset = 0; 1027 PCI_SAVE_MP_TTE(mp, tte); /* mp->dmai_tte = tte */ 1028 iommu_map_pages(iommu_p, mp, dvma_pg, npages, 0); 1029 return (DDI_SUCCESS); 1030 noresource: 1031 if (dmareq->dmar_fp != DDI_DMA_DONTWAIT) { 1032 DEBUG0(DBG_DMA_MAP, dip, "dvma_pg 0 - set callback\n"); 1033 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg, 1034 &iommu_p->iommu_dvma_clid); 1035 } 1036 DEBUG0(DBG_DMA_MAP, dip, "vmem_xalloc - DDI_DMA_NORESOURCES\n"); 1037 return (DDI_DMA_NORESOURCES); 1038 } 1039 1040 void 1041 pci_dvma_unmap(iommu_t *iommu_p, ddi_dma_impl_t *mp) 1042 { 1043 size_t npages; 1044 dvma_addr_t dvma_addr = (dvma_addr_t)mp->dmai_mapping; 1045 dvma_addr_t dvma_pg = IOMMU_BTOP(dvma_addr); 1046 dvma_addr = IOMMU_PTOB(dvma_pg); 1047 1048 if (mp->dmai_flags & DMAI_FLAGS_FASTTRACK) { 1049 iopfn_t index = dvma_pg - iommu_p->dvma_base_pg; 1050 ASSERT(index % pci_dvma_page_cache_clustsz == 0); 1051 index /= pci_dvma_page_cache_clustsz; 1052 ASSERT(index < pci_dvma_page_cache_entries); 1053 iommu_p->iommu_dvma_cache_locks[index] = 0; 1054 #ifdef PCI_DMA_PROF 1055 pci_dvmaft_free++; 1056 #endif 1057 return; 1058 } 1059 npages = IOMMU_BTOP(mp->dmai_winsize) + HAS_REDZONE(mp); 1060 pci_vmem_free(iommu_p, mp, (void *)dvma_addr, npages); 1061 1062 if (mp->dmai_flags & DMAI_FLAGS_CONTEXT) 1063 pci_iommu_free_dvma_context(iommu_p, MP2CTX(mp)); 1064 } 1065 1066 void 1067 pci_dma_sync_unmap(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp) 1068 { 1069 pci_t *pci_p = get_pci_soft_state(ddi_get_instance(dip)); 1070 iommu_t *iommu_p = pci_p->pci_iommu_p; 1071 uint64_t sync_buf_save = SYNC_BUF_PA(mp); 1072 uint32_t fast_track = mp->dmai_flags & DMAI_FLAGS_FASTTRACK; 1073 1074 if (fast_track) { 1075 dvma_addr_t dvma_pg = IOMMU_BTOP(mp->dmai_mapping); 1076 1077 SYNC_BUF_PA(mp) = IOMMU_PAGE_TTEPA(iommu_p, dvma_pg); 1078 ASSERT(!(SYNC_BUF_PA(mp) & PCI_SYNC_FLAG_SIZE - 1)); 1079 } 1080 1081 if (pci_dvma_sync_before_unmap) { 1082 pci_dma_sync(dip, rdip, (ddi_dma_handle_t)mp, 0, 0, 0); 1083 iommu_unmap_window(iommu_p, mp); 1084 } else { 1085 iommu_unmap_window(iommu_p, mp); 1086 pci_dma_sync(dip, rdip, (ddi_dma_handle_t)mp, 0, 0, 0); 1087 } 1088 1089 if (fast_track) 1090 SYNC_BUF_PA(mp) = sync_buf_save; 1091 } 1092 1093 /* 1094 * DVMA mappings may have multiple windows, but each window always have 1095 * one segment. 1096 */ 1097 int 1098 pci_dvma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp, 1099 enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp, 1100 uint_t cache_flags) 1101 { 1102 switch (cmd) { 1103 case DDI_DMA_SYNC: 1104 return (pci_dma_sync(dip, rdip, (ddi_dma_handle_t)mp, 1105 *offp, *lenp, cache_flags)); 1106 1107 case DDI_DMA_HTOC: { 1108 int ret; 1109 off_t wo_off, off = *offp; /* wo_off: wnd's obj offset */ 1110 uint_t win_size = mp->dmai_winsize; 1111 ddi_dma_cookie_t *cp = (ddi_dma_cookie_t *)objp; 1112 1113 if (off >= mp->dmai_object.dmao_size) { 1114 cmn_err(CE_WARN, "%s%d invalid dma_htoc offset %lx", 1115 NAMEINST(mp->dmai_rdip), off); 1116 return (DDI_FAILURE); 1117 } 1118 off += mp->dmai_roffset; 1119 ret = pci_dma_win(dip, rdip, (ddi_dma_handle_t)mp, 1120 off / win_size, &wo_off, NULL, cp, NULL); /* lenp == NULL */ 1121 if (ret) 1122 return (ret); 1123 DEBUG4(DBG_DMA_CTL, dip, "HTOC:cookie=%x+%lx off=%lx,%lx\n", 1124 cp->dmac_address, cp->dmac_size, off, *offp); 1125 1126 /* adjust cookie addr/len if we are not on window boundary */ 1127 ASSERT((off % win_size) == (off - 1128 (PCI_DMA_CURWIN(mp) ? mp->dmai_roffset : 0) - wo_off)); 1129 off = PCI_DMA_CURWIN(mp) ? off % win_size : *offp; 1130 ASSERT(cp->dmac_size > off); 1131 cp->dmac_laddress += off; 1132 cp->dmac_size -= off; 1133 DEBUG5(DBG_DMA_CTL, dip, 1134 "HTOC:mp=%p cookie=%x+%lx off=%lx,%lx\n", 1135 mp, cp->dmac_address, cp->dmac_size, off, wo_off); 1136 } 1137 return (DDI_SUCCESS); 1138 1139 case DDI_DMA_REPWIN: 1140 *offp = mp->dmai_offset; 1141 *lenp = mp->dmai_size; 1142 return (DDI_SUCCESS); 1143 1144 case DDI_DMA_MOVWIN: { 1145 off_t off = *offp; 1146 if (off >= mp->dmai_object.dmao_size) 1147 return (DDI_FAILURE); 1148 off += mp->dmai_roffset; 1149 return (pci_dma_win(dip, rdip, (ddi_dma_handle_t)mp, 1150 off / mp->dmai_winsize, offp, lenp, 1151 (ddi_dma_cookie_t *)objp, NULL)); 1152 } 1153 1154 case DDI_DMA_NEXTWIN: { 1155 window_t win = PCI_DMA_CURWIN(mp); 1156 if (offp) { 1157 if (*(window_t *)offp != win) { /* window not active */ 1158 *(window_t *)objp = win; /* return cur win */ 1159 return (DDI_DMA_STALE); 1160 } 1161 win++; 1162 } else /* map win 0 */ 1163 win = 0; 1164 if (win >= mp->dmai_nwin) { 1165 *(window_t *)objp = win - 1; 1166 return (DDI_DMA_DONE); 1167 } 1168 if (pci_dma_win(dip, rdip, (ddi_dma_handle_t)mp, 1169 win, 0, 0, 0, 0)) { 1170 *(window_t *)objp = win - 1; 1171 return (DDI_FAILURE); 1172 } 1173 *(window_t *)objp = win; 1174 } 1175 return (DDI_SUCCESS); 1176 1177 case DDI_DMA_NEXTSEG: 1178 if (*(window_t *)offp != PCI_DMA_CURWIN(mp)) 1179 return (DDI_DMA_STALE); 1180 if (lenp) /* only 1 seg allowed */ 1181 return (DDI_DMA_DONE); 1182 /* return mp as seg 0 */ 1183 *(ddi_dma_seg_t *)objp = (ddi_dma_seg_t)mp; 1184 return (DDI_SUCCESS); 1185 1186 case DDI_DMA_SEGTOC: 1187 MAKE_DMA_COOKIE((ddi_dma_cookie_t *)objp, mp->dmai_mapping, 1188 mp->dmai_size); 1189 *offp = mp->dmai_offset; 1190 *lenp = mp->dmai_size; 1191 return (DDI_SUCCESS); 1192 1193 case DDI_DMA_COFF: { 1194 ddi_dma_cookie_t *cp = (ddi_dma_cookie_t *)offp; 1195 if (cp->dmac_address < mp->dmai_mapping || 1196 (cp->dmac_address + cp->dmac_size) > 1197 (mp->dmai_mapping + mp->dmai_size)) 1198 return (DDI_FAILURE); 1199 *objp = (caddr_t)(cp->dmac_address - mp->dmai_mapping + 1200 mp->dmai_offset); 1201 } 1202 return (DDI_SUCCESS); 1203 1204 case DDI_DMA_REMAP: 1205 if (pci_dvma_remap_enabled) 1206 return (pci_dvma_remap(dip, rdip, mp, *offp, *lenp)); 1207 return (DDI_FAILURE); 1208 1209 default: 1210 DEBUG3(DBG_DMA_CTL, dip, "unknown command (%x): rdip=%s%d\n", 1211 cmd, ddi_driver_name(rdip), ddi_get_instance(rdip)); 1212 break; 1213 } 1214 return (DDI_FAILURE); 1215 } 1216 1217 void 1218 pci_dma_freewin(ddi_dma_impl_t *mp) 1219 { 1220 pci_dma_win_t *win_p = mp->dmai_winlst, *win2_p; 1221 for (win2_p = win_p; win_p; win2_p = win_p) { 1222 win_p = win2_p->win_next; 1223 kmem_free(win2_p, sizeof (pci_dma_win_t) + 1224 sizeof (ddi_dma_cookie_t) * win2_p->win_ncookies); 1225 } 1226 mp->dmai_nwin = 0; 1227 mp->dmai_winlst = NULL; 1228 } 1229 1230 /* 1231 * pci_dma_newwin - create a dma window object and cookies 1232 * 1233 * After the initial scan in pci_dma_physwin(), which identifies 1234 * a portion of the pfn array that belongs to a dma window, 1235 * we are called to allocate and initialize representing memory 1236 * resources. We know from the 1st scan the number of cookies 1237 * or dma segment in this window so we can allocate a contiguous 1238 * memory array for the dma cookies (The implementation of 1239 * ddi_dma_nextcookie(9f) dictates dma cookies be contiguous). 1240 * 1241 * A second round scan is done on the pfn array to identify 1242 * each dma segment and initialize its corresponding dma cookie. 1243 * We don't need to do all the safety checking and we know they 1244 * all belong to the same dma window. 1245 * 1246 * Input: cookie_no - # of cookies identified by the 1st scan 1247 * start_idx - subscript of the pfn array for the starting pfn 1248 * end_idx - subscript of the last pfn in dma window 1249 * win_pp - pointer to win_next member of previous window 1250 * Return: DDI_SUCCESS - with **win_pp as newly created window object 1251 * DDI_DMA_NORESROUCE - caller frees all previous window objs 1252 * Note: Each cookie and window size are all initialized on page 1253 * boundary. This is not true for the 1st cookie of the 1st 1254 * window and the last cookie of the last window. 1255 * We fix that later in upper layer which has access to size 1256 * and offset info. 1257 * 1258 */ 1259 static int 1260 pci_dma_newwin(ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, uint32_t cookie_no, 1261 uint32_t start_idx, uint32_t end_idx, pci_dma_win_t **win_pp, 1262 uint64_t count_max, uint64_t bypass_prefix) 1263 { 1264 int (*waitfp)(caddr_t) = dmareq->dmar_fp; 1265 ddi_dma_cookie_t *cookie_p; 1266 uint32_t pfn_no = 1; 1267 iopfn_t pfn = PCI_GET_MP_PFN(mp, start_idx); 1268 iopfn_t prev_pfn = pfn; 1269 uint64_t seg_pfn0 = pfn; 1270 size_t sz = cookie_no * sizeof (ddi_dma_cookie_t); 1271 pci_dma_win_t *win_p = kmem_alloc(sizeof (pci_dma_win_t) + sz, 1272 waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP); 1273 if (!win_p) 1274 goto noresource; 1275 1276 win_p->win_next = NULL; 1277 win_p->win_ncookies = cookie_no; 1278 win_p->win_curseg = 0; /* start from segment 0 */ 1279 win_p->win_size = IOMMU_PTOB(end_idx - start_idx + 1); 1280 /* win_p->win_offset is left uninitialized */ 1281 1282 cookie_p = (ddi_dma_cookie_t *)(win_p + 1); 1283 start_idx++; 1284 for (; start_idx <= end_idx; start_idx++, prev_pfn = pfn, pfn_no++) { 1285 pfn = PCI_GET_MP_PFN1(mp, start_idx); 1286 if ((pfn == prev_pfn + 1) && 1287 (IOMMU_PTOB(pfn_no + 1) - 1 <= count_max)) 1288 continue; 1289 1290 /* close up the cookie up to (including) prev_pfn */ 1291 MAKE_DMA_COOKIE(cookie_p, IOMMU_PTOB(seg_pfn0) | bypass_prefix, 1292 IOMMU_PTOB(pfn_no)); 1293 DEBUG2(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages)\n", 1294 IOMMU_PTOB(seg_pfn0) | bypass_prefix, pfn_no); 1295 1296 cookie_p++; /* advance to next available cookie cell */ 1297 pfn_no = 0; 1298 seg_pfn0 = pfn; /* start a new segment from current pfn */ 1299 } 1300 MAKE_DMA_COOKIE(cookie_p, IOMMU_PTOB(seg_pfn0) | bypass_prefix, 1301 IOMMU_PTOB(pfn_no)); 1302 DEBUG3(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages) of total %x\n", 1303 IOMMU_PTOB(seg_pfn0) | bypass_prefix, pfn_no, cookie_no); 1304 #ifdef DEBUG 1305 cookie_p++; 1306 ASSERT((cookie_p - (ddi_dma_cookie_t *)(win_p + 1)) == cookie_no); 1307 #endif 1308 *win_pp = win_p; 1309 return (DDI_SUCCESS); 1310 noresource: 1311 if (waitfp != DDI_DMA_DONTWAIT) 1312 ddi_set_callback(waitfp, dmareq->dmar_arg, &pci_kmem_clid); 1313 return (DDI_DMA_NORESOURCES); 1314 } 1315 1316 /* 1317 * pci_dma_adjust - adjust 1st and last cookie and window sizes 1318 * remove initial dma page offset from 1st cookie and window size 1319 * remove last dma page remainder from last cookie and window size 1320 * fill win_offset of each dma window according to just fixed up 1321 * each window sizes 1322 * pci_dma_win_t members modified: 1323 * win_p->win_offset - this window's offset within entire DMA object 1324 * win_p->win_size - xferrable size (in bytes) for this window 1325 * 1326 * ddi_dma_impl_t members modified: 1327 * mp->dmai_size - 1st window xferrable size 1328 * mp->dmai_offset - 0, which is the dma offset of the 1st window 1329 * 1330 * ddi_dma_cookie_t members modified: 1331 * cookie_p->dmac_size - 1st and last cookie remove offset or remainder 1332 * cookie_p->dmac_laddress - 1st cookie add page offset 1333 */ 1334 static void 1335 pci_dma_adjust(ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, pci_dma_win_t *win_p) 1336 { 1337 ddi_dma_cookie_t *cookie_p = (ddi_dma_cookie_t *)(win_p + 1); 1338 size_t pg_offset = mp->dmai_roffset; 1339 size_t win_offset = 0; 1340 1341 cookie_p->dmac_size -= pg_offset; 1342 cookie_p->dmac_laddress |= pg_offset; 1343 win_p->win_size -= pg_offset; 1344 DEBUG1(DBG_BYPASS, mp->dmai_rdip, "pg0 adjust %lx\n", pg_offset); 1345 1346 mp->dmai_size = win_p->win_size; 1347 mp->dmai_offset = 0; 1348 1349 pg_offset += mp->dmai_object.dmao_size; 1350 pg_offset &= IOMMU_PAGE_OFFSET; 1351 if (pg_offset) 1352 pg_offset = IOMMU_PAGE_SIZE - pg_offset; 1353 DEBUG1(DBG_BYPASS, mp->dmai_rdip, "last pg adjust %lx\n", pg_offset); 1354 1355 for (; win_p->win_next; win_p = win_p->win_next) { 1356 DEBUG1(DBG_BYPASS, mp->dmai_rdip, "win off %p\n", win_offset); 1357 win_p->win_offset = win_offset; 1358 win_offset += win_p->win_size; 1359 } 1360 /* last window */ 1361 win_p->win_offset = win_offset; 1362 cookie_p = (ddi_dma_cookie_t *)(win_p + 1); 1363 cookie_p[win_p->win_ncookies - 1].dmac_size -= pg_offset; 1364 win_p->win_size -= pg_offset; 1365 ASSERT((win_offset + win_p->win_size) == mp->dmai_object.dmao_size); 1366 } 1367 1368 /* 1369 * pci_dma_physwin() - carve up dma windows using physical addresses. 1370 * Called to handle iommu bypass and pci peer-to-peer transfers. 1371 * Calls pci_dma_newwin() to allocate window objects. 1372 * 1373 * Dependency: mp->dmai_pfnlst points to an array of pfns 1374 * 1375 * 1. Each dma window is represented by a pci_dma_win_t object. 1376 * The object will be casted to ddi_dma_win_t and returned 1377 * to leaf driver through the DDI interface. 1378 * 2. Each dma window can have several dma segments with each 1379 * segment representing a physically contiguous either memory 1380 * space (if we are doing an iommu bypass transfer) or pci address 1381 * space (if we are doing a peer-to-peer transfer). 1382 * 3. Each segment has a DMA cookie to program the DMA engine. 1383 * The cookies within each DMA window must be located in a 1384 * contiguous array per ddi_dma_nextcookie(9f). 1385 * 4. The number of DMA segments within each DMA window cannot exceed 1386 * mp->dmai_attr.dma_attr_sgllen. If the transfer size is 1387 * too large to fit in the sgllen, the rest needs to be 1388 * relocated to the next dma window. 1389 * 5. Peer-to-peer DMA segment follows device hi, lo, count_max, 1390 * and nocross restrictions while bypass DMA follows the set of 1391 * restrictions with system limits factored in. 1392 * 1393 * Return: 1394 * mp->dmai_winlst - points to a link list of pci_dma_win_t objects. 1395 * Each pci_dma_win_t object on the link list contains 1396 * infomation such as its window size (# of pages), 1397 * starting offset (also see Restriction), an array of 1398 * DMA cookies, and # of cookies in the array. 1399 * mp->dmai_pfnlst - NULL, the pfn list is freed to conserve memory. 1400 * mp->dmai_nwin - # of total DMA windows on mp->dmai_winlst. 1401 * mp->dmai_mapping - starting cookie address 1402 * mp->dmai_rflags - consistent, nosync, no redzone 1403 * mp->dmai_cookie - start of cookie table of the 1st DMA window 1404 * 1405 * Restriction: 1406 * Each pci_dma_win_t object can theoratically start from any offset 1407 * since the iommu is not involved. However, this implementation 1408 * always make windows start from page aligned offset (except 1409 * the 1st window, which follows the requested offset) due to the 1410 * fact that we are handed a pfn list. This does require device's 1411 * count_max and attr_seg to be at least IOMMU_PAGE_SIZE aligned. 1412 */ 1413 int 1414 pci_dma_physwin(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 1415 { 1416 uint_t npages = mp->dmai_ndvmapages; 1417 int ret, sgllen = mp->dmai_attr.dma_attr_sgllen; 1418 iopfn_t pfn_lo, pfn_hi, prev_pfn, bypass_pfn; 1419 iopfn_t pfn = PCI_GET_MP_PFN(mp, 0); 1420 uint32_t i, win_no = 0, pfn_no = 1, win_pfn0_index = 0, cookie_no = 0; 1421 uint64_t count_max, bypass = PCI_DMA_BYPASS_PREFIX(mp, pfn); 1422 pci_dma_win_t **win_pp = (pci_dma_win_t **)&mp->dmai_winlst; 1423 ddi_dma_cookie_t *cookie0_p; 1424 1425 if (PCI_DMA_ISPTP(mp)) { /* ignore sys limits for peer-to-peer */ 1426 ddi_dma_attr_t *dev_attr_p = DEV_ATTR(mp); 1427 iopfn_t pfn_base = pci_p->pci_pbm_p->pbm_base_pfn; 1428 iopfn_t pfn_last = pci_p->pci_pbm_p->pbm_last_pfn - pfn_base; 1429 uint64_t nocross = dev_attr_p->dma_attr_seg; 1430 if (nocross && (nocross < UINT32_MAX)) 1431 return (DDI_DMA_NOMAPPING); 1432 if (dev_attr_p->dma_attr_align > IOMMU_PAGE_SIZE) 1433 return (DDI_DMA_NOMAPPING); 1434 pfn_lo = IOMMU_BTOP(dev_attr_p->dma_attr_addr_lo); 1435 pfn_hi = IOMMU_BTOP(dev_attr_p->dma_attr_addr_hi); 1436 pfn_hi = MIN(pfn_hi, pfn_last); 1437 if ((pfn_lo > pfn_hi) || (pfn < pfn_lo)) 1438 return (DDI_DMA_NOMAPPING); 1439 count_max = dev_attr_p->dma_attr_count_max; 1440 count_max = MIN(count_max, nocross); 1441 /* 1442 * the following count_max trim is not done because we are 1443 * making sure pfn_lo <= pfn <= pfn_hi inside the loop 1444 * count_max=MIN(count_max, IOMMU_PTOB(pfn_hi - pfn_lo + 1)-1); 1445 */ 1446 } else { /* bypass hi/lo/count_max have been processed by attr2hdl() */ 1447 count_max = mp->dmai_attr.dma_attr_count_max; 1448 pfn_lo = IOMMU_BTOP(mp->dmai_attr.dma_attr_addr_lo); 1449 pfn_hi = IOMMU_BTOP(mp->dmai_attr.dma_attr_addr_hi); 1450 } 1451 1452 bypass_pfn = IOMMU_BTOP(bypass); 1453 1454 for (prev_pfn = (bypass_pfn | pfn), i = 1; i < npages; 1455 i++, prev_pfn = pfn, pfn_no++) { 1456 pfn = bypass_pfn | PCI_GET_MP_PFN1(mp, i); 1457 if ((pfn == prev_pfn + 1) && 1458 (IOMMU_PTOB(pfn_no + 1) - 1 <= count_max)) 1459 continue; 1460 if ((pfn < pfn_lo) || (prev_pfn > pfn_hi)) { 1461 ret = DDI_DMA_NOMAPPING; 1462 goto err; 1463 } 1464 cookie_no++; 1465 pfn_no = 0; 1466 if (cookie_no < sgllen) 1467 continue; 1468 1469 DEBUG3(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n", 1470 win_pfn0_index, i - 1, cookie_no); 1471 if (ret = pci_dma_newwin(dmareq, mp, cookie_no, 1472 win_pfn0_index, i - 1, win_pp, count_max, bypass)) 1473 goto err; 1474 1475 win_pp = &(*win_pp)->win_next; /* win_pp = *(win_pp) */ 1476 win_no++; 1477 win_pfn0_index = i; 1478 cookie_no = 0; 1479 } 1480 if (pfn > pfn_hi) { 1481 ret = DDI_DMA_NOMAPPING; 1482 goto err; 1483 } 1484 cookie_no++; 1485 DEBUG3(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n", 1486 win_pfn0_index, i - 1, cookie_no); 1487 if (ret = pci_dma_newwin(dmareq, mp, cookie_no, win_pfn0_index, 1488 i - 1, win_pp, count_max, bypass)) 1489 goto err; 1490 win_no++; 1491 pci_dma_adjust(dmareq, mp, mp->dmai_winlst); 1492 mp->dmai_nwin = win_no; 1493 mp->dmai_rflags |= DDI_DMA_CONSISTENT; 1494 if (!pci_p->pci_pbm_p->pbm_sync_reg_pa) { 1495 mp->dmai_rflags |= DMP_NOSYNC; 1496 mp->dmai_flags |= DMAI_FLAGS_NOSYNC; 1497 } 1498 mp->dmai_rflags &= ~DDI_DMA_REDZONE; 1499 cookie0_p = (ddi_dma_cookie_t *)(WINLST(mp) + 1); 1500 mp->dmai_cookie = WINLST(mp)->win_ncookies > 1 ? cookie0_p + 1 : 0; 1501 mp->dmai_mapping = cookie0_p->dmac_laddress; 1502 1503 pci_dma_freepfn(mp); 1504 return (DDI_DMA_MAPPED); 1505 err: 1506 pci_dma_freewin(mp); 1507 return (ret); 1508 } 1509 1510 /*ARGSUSED*/ 1511 int 1512 pci_dma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp, 1513 enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp, 1514 uint_t cache_flags) 1515 { 1516 switch (cmd) { 1517 case DDI_DMA_SYNC: /* XXX */ 1518 return (DDI_SUCCESS); 1519 1520 case DDI_DMA_HTOC: { 1521 off_t off = *offp; 1522 ddi_dma_cookie_t *loop_cp, *cp; 1523 pci_dma_win_t *win_p = mp->dmai_winlst; 1524 1525 if (off >= mp->dmai_object.dmao_size) 1526 return (DDI_FAILURE); 1527 1528 /* locate window */ 1529 while (win_p->win_offset + win_p->win_size <= off) 1530 win_p = win_p->win_next; 1531 1532 loop_cp = cp = (ddi_dma_cookie_t *)(win_p + 1); 1533 mp->dmai_offset = win_p->win_offset; 1534 mp->dmai_size = win_p->win_size; 1535 mp->dmai_mapping = cp->dmac_laddress; /* cookie0 start addr */ 1536 1537 /* adjust cookie addr/len if we are not on cookie boundary */ 1538 off -= win_p->win_offset; /* offset within window */ 1539 for (; off >= loop_cp->dmac_size; loop_cp++) 1540 off -= loop_cp->dmac_size; /* offset within cookie */ 1541 1542 mp->dmai_cookie = loop_cp + 1; 1543 win_p->win_curseg = loop_cp - cp; 1544 cp = (ddi_dma_cookie_t *)objp; 1545 MAKE_DMA_COOKIE(cp, loop_cp->dmac_laddress + off, 1546 loop_cp->dmac_size - off); 1547 1548 DEBUG2(DBG_DMA_CTL, dip, 1549 "HTOC: cookie - dmac_laddress=%p dmac_size=%x\n", 1550 cp->dmac_laddress, cp->dmac_size); 1551 } 1552 return (DDI_SUCCESS); 1553 1554 case DDI_DMA_REPWIN: 1555 *offp = mp->dmai_offset; 1556 *lenp = mp->dmai_size; 1557 return (DDI_SUCCESS); 1558 1559 case DDI_DMA_MOVWIN: { 1560 off_t off = *offp; 1561 ddi_dma_cookie_t *cp; 1562 pci_dma_win_t *win_p = mp->dmai_winlst; 1563 1564 if (off >= mp->dmai_object.dmao_size) 1565 return (DDI_FAILURE); 1566 1567 /* locate window */ 1568 while (win_p->win_offset + win_p->win_size <= off) 1569 win_p = win_p->win_next; 1570 1571 cp = (ddi_dma_cookie_t *)(win_p + 1); 1572 mp->dmai_offset = win_p->win_offset; 1573 mp->dmai_size = win_p->win_size; 1574 mp->dmai_mapping = cp->dmac_laddress; /* cookie0 star addr */ 1575 mp->dmai_cookie = cp + 1; 1576 win_p->win_curseg = 0; 1577 1578 *(ddi_dma_cookie_t *)objp = *cp; 1579 *offp = win_p->win_offset; 1580 *lenp = win_p->win_size; 1581 DEBUG2(DBG_DMA_CTL, dip, 1582 "HTOC: cookie - dmac_laddress=%p dmac_size=%x\n", 1583 cp->dmac_laddress, cp->dmac_size); 1584 } 1585 return (DDI_SUCCESS); 1586 1587 case DDI_DMA_NEXTWIN: { 1588 pci_dma_win_t *win_p = *(pci_dma_win_t **)offp; 1589 pci_dma_win_t **nw_pp = (pci_dma_win_t **)objp; 1590 ddi_dma_cookie_t *cp; 1591 if (!win_p) { 1592 *nw_pp = mp->dmai_winlst; 1593 return (DDI_SUCCESS); 1594 } 1595 1596 if (win_p->win_offset != mp->dmai_offset) 1597 return (DDI_DMA_STALE); 1598 if (!win_p->win_next) 1599 return (DDI_DMA_DONE); 1600 win_p = win_p->win_next; 1601 cp = (ddi_dma_cookie_t *)(win_p + 1); 1602 mp->dmai_offset = win_p->win_offset; 1603 mp->dmai_size = win_p->win_size; 1604 mp->dmai_mapping = cp->dmac_laddress; /* cookie0 star addr */ 1605 mp->dmai_cookie = cp + 1; 1606 win_p->win_curseg = 0; 1607 *nw_pp = win_p; 1608 } 1609 return (DDI_SUCCESS); 1610 1611 case DDI_DMA_NEXTSEG: { 1612 pci_dma_win_t *w_p = *(pci_dma_win_t **)offp; 1613 if (w_p->win_offset != mp->dmai_offset) 1614 return (DDI_DMA_STALE); 1615 if (w_p->win_curseg + 1 >= w_p->win_ncookies) 1616 return (DDI_DMA_DONE); 1617 w_p->win_curseg++; 1618 } 1619 *(ddi_dma_seg_t *)objp = (ddi_dma_seg_t)mp; 1620 return (DDI_SUCCESS); 1621 1622 case DDI_DMA_SEGTOC: { 1623 pci_dma_win_t *win_p = mp->dmai_winlst; 1624 off_t off = mp->dmai_offset; 1625 ddi_dma_cookie_t *cp; 1626 int i; 1627 1628 /* locate active window */ 1629 for (; win_p->win_offset != off; win_p = win_p->win_next); 1630 cp = (ddi_dma_cookie_t *)(win_p + 1); 1631 for (i = 0; i < win_p->win_curseg; i++, cp++) 1632 off += cp->dmac_size; 1633 *offp = off; 1634 *lenp = cp->dmac_size; 1635 *(ddi_dma_cookie_t *)objp = *cp; /* copy cookie */ 1636 } 1637 return (DDI_SUCCESS); 1638 1639 case DDI_DMA_COFF: { 1640 pci_dma_win_t *win_p; 1641 ddi_dma_cookie_t *cp; 1642 uint64_t addr, key = ((ddi_dma_cookie_t *)offp)->dmac_laddress; 1643 size_t win_off; 1644 1645 for (win_p = mp->dmai_winlst; win_p; win_p = win_p->win_next) { 1646 int i; 1647 win_off = 0; 1648 cp = (ddi_dma_cookie_t *)(win_p + 1); 1649 for (i = 0; i < win_p->win_ncookies; i++, cp++) { 1650 size_t sz = cp->dmac_size; 1651 1652 addr = cp->dmac_laddress; 1653 if ((addr <= key) && (addr + sz >= key)) 1654 goto found; 1655 win_off += sz; 1656 } 1657 } 1658 return (DDI_FAILURE); 1659 found: 1660 *objp = (caddr_t)(win_p->win_offset + win_off + (key - addr)); 1661 return (DDI_SUCCESS); 1662 } 1663 1664 case DDI_DMA_REMAP: 1665 return (DDI_FAILURE); 1666 1667 default: 1668 DEBUG3(DBG_DMA_CTL, dip, "unknown command (%x): rdip=%s%d\n", 1669 cmd, ddi_driver_name(rdip), ddi_get_instance(rdip)); 1670 break; 1671 } 1672 return (DDI_FAILURE); 1673 } 1674 1675 static void 1676 pci_dvma_debug_init(iommu_t *iommu_p) 1677 { 1678 size_t sz = sizeof (struct dvma_rec) * pci_dvma_debug_rec; 1679 ASSERT(MUTEX_HELD(&iommu_p->dvma_debug_lock)); 1680 cmn_err(CE_NOTE, "PCI DVMA %p stat ON", iommu_p); 1681 1682 iommu_p->dvma_alloc_rec = kmem_zalloc(sz, KM_SLEEP); 1683 iommu_p->dvma_free_rec = kmem_zalloc(sz, KM_SLEEP); 1684 1685 iommu_p->dvma_active_list = NULL; 1686 iommu_p->dvma_alloc_rec_index = 0; 1687 iommu_p->dvma_free_rec_index = 0; 1688 iommu_p->dvma_active_count = 0; 1689 } 1690 1691 void 1692 pci_dvma_debug_fini(iommu_t *iommu_p) 1693 { 1694 struct dvma_rec *prev, *ptr; 1695 size_t sz = sizeof (struct dvma_rec) * pci_dvma_debug_rec; 1696 uint64_t mask = ~(1ull << iommu_p->iommu_inst); 1697 cmn_err(CE_NOTE, "PCI DVMA %p stat OFF", iommu_p); 1698 1699 kmem_free(iommu_p->dvma_alloc_rec, sz); 1700 kmem_free(iommu_p->dvma_free_rec, sz); 1701 iommu_p->dvma_alloc_rec = iommu_p->dvma_free_rec = NULL; 1702 1703 prev = iommu_p->dvma_active_list; 1704 if (!prev) 1705 return; 1706 for (ptr = prev->next; ptr; prev = ptr, ptr = ptr->next) 1707 kmem_free(prev, sizeof (struct dvma_rec)); 1708 kmem_free(prev, sizeof (struct dvma_rec)); 1709 1710 iommu_p->dvma_active_list = NULL; 1711 iommu_p->dvma_alloc_rec_index = 0; 1712 iommu_p->dvma_free_rec_index = 0; 1713 iommu_p->dvma_active_count = 0; 1714 1715 pci_dvma_debug_on &= mask; 1716 pci_dvma_debug_off &= mask; 1717 } 1718 1719 void 1720 pci_dvma_alloc_debug(iommu_t *iommu_p, char *address, uint_t len, 1721 ddi_dma_impl_t *mp) 1722 { 1723 struct dvma_rec *ptr; 1724 mutex_enter(&iommu_p->dvma_debug_lock); 1725 1726 if (!iommu_p->dvma_alloc_rec) 1727 pci_dvma_debug_init(iommu_p); 1728 if (DVMA_DBG_OFF(iommu_p)) { 1729 pci_dvma_debug_fini(iommu_p); 1730 goto done; 1731 } 1732 1733 ptr = &iommu_p->dvma_alloc_rec[iommu_p->dvma_alloc_rec_index]; 1734 ptr->dvma_addr = address; 1735 ptr->len = len; 1736 ptr->mp = mp; 1737 if (++iommu_p->dvma_alloc_rec_index == pci_dvma_debug_rec) 1738 iommu_p->dvma_alloc_rec_index = 0; 1739 1740 ptr = kmem_alloc(sizeof (struct dvma_rec), KM_SLEEP); 1741 ptr->dvma_addr = address; 1742 ptr->len = len; 1743 ptr->mp = mp; 1744 1745 ptr->next = iommu_p->dvma_active_list; 1746 iommu_p->dvma_active_list = ptr; 1747 iommu_p->dvma_active_count++; 1748 done: 1749 mutex_exit(&iommu_p->dvma_debug_lock); 1750 } 1751 1752 void 1753 pci_dvma_free_debug(iommu_t *iommu_p, char *address, uint_t len, 1754 ddi_dma_impl_t *mp) 1755 { 1756 struct dvma_rec *ptr, *ptr_save; 1757 mutex_enter(&iommu_p->dvma_debug_lock); 1758 1759 if (!iommu_p->dvma_alloc_rec) 1760 pci_dvma_debug_init(iommu_p); 1761 if (DVMA_DBG_OFF(iommu_p)) { 1762 pci_dvma_debug_fini(iommu_p); 1763 goto done; 1764 } 1765 1766 ptr = &iommu_p->dvma_free_rec[iommu_p->dvma_free_rec_index]; 1767 ptr->dvma_addr = address; 1768 ptr->len = len; 1769 ptr->mp = mp; 1770 if (++iommu_p->dvma_free_rec_index == pci_dvma_debug_rec) 1771 iommu_p->dvma_free_rec_index = 0; 1772 1773 ptr_save = iommu_p->dvma_active_list; 1774 for (ptr = ptr_save; ptr; ptr = ptr->next) { 1775 if ((ptr->dvma_addr == address) && (ptr->len = len)) 1776 break; 1777 ptr_save = ptr; 1778 } 1779 if (!ptr) { 1780 cmn_err(CE_WARN, "bad dvma free addr=%lx len=%x", 1781 (long)address, len); 1782 goto done; 1783 } 1784 if (ptr == iommu_p->dvma_active_list) 1785 iommu_p->dvma_active_list = ptr->next; 1786 else 1787 ptr_save->next = ptr->next; 1788 kmem_free(ptr, sizeof (struct dvma_rec)); 1789 iommu_p->dvma_active_count--; 1790 done: 1791 mutex_exit(&iommu_p->dvma_debug_lock); 1792 } 1793 1794 #ifdef DEBUG 1795 void 1796 dump_dma_handle(uint64_t flag, dev_info_t *dip, ddi_dma_impl_t *hp) 1797 { 1798 DEBUG4(flag, dip, "mp(%p): flags=%x mapping=%lx xfer_size=%x\n", 1799 hp, hp->dmai_inuse, hp->dmai_mapping, hp->dmai_size); 1800 DEBUG4(flag|DBG_CONT, dip, "\tnpages=%x roffset=%x rflags=%x nwin=%x\n", 1801 hp->dmai_ndvmapages, hp->dmai_roffset, hp->dmai_rflags, 1802 hp->dmai_nwin); 1803 DEBUG4(flag|DBG_CONT, dip, "\twinsize=%x tte=%p pfnlst=%p pfn0=%p\n", 1804 hp->dmai_winsize, hp->dmai_tte, hp->dmai_pfnlst, hp->dmai_pfn0); 1805 DEBUG4(flag|DBG_CONT, dip, "\twinlst=%x obj=%p attr=%p ckp=%p\n", 1806 hp->dmai_winlst, &hp->dmai_object, &hp->dmai_attr, 1807 hp->dmai_cookie); 1808 } 1809 #endif 1810 1811 void 1812 pci_vmem_do_free(iommu_t *iommu_p, void *base_addr, size_t npages, 1813 int vmemcache) 1814 { 1815 vmem_t *map_p = iommu_p->iommu_dvma_map; 1816 1817 if (vmemcache) { 1818 vmem_free(map_p, base_addr, IOMMU_PAGE_SIZE); 1819 #ifdef PCI_DMA_PROF 1820 pci_dvma_vmem_free++; 1821 #endif 1822 return; 1823 } 1824 1825 vmem_xfree(map_p, base_addr, IOMMU_PTOB(npages)); 1826 #ifdef PCI_DMA_PROF 1827 pci_dvma_vmem_xfree++; 1828 #endif 1829 } 1830