1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 /* 29 * PCI Express nexus DVMA and DMA core routines: 30 * dma_map/dma_bind_handle implementation 31 * bypass and peer-to-peer support 32 * fast track DVMA space allocation 33 * runtime DVMA debug 34 */ 35 #include <sys/types.h> 36 #include <sys/kmem.h> 37 #include <sys/async.h> 38 #include <sys/sysmacros.h> 39 #include <sys/sunddi.h> 40 #include <sys/ddi_impldefs.h> 41 #include "px_obj.h" 42 43 /*LINTLIBRARY*/ 44 45 /* 46 * px_dma_allocmp - Allocate a pci dma implementation structure 47 * 48 * An extra ddi_dma_attr structure is bundled with the usual ddi_dma_impl 49 * to hold unmodified device limits. The ddi_dma_attr inside the 50 * ddi_dma_impl structure is augumented with system limits to enhance 51 * DVMA performance at runtime. The unaugumented device limits saved 52 * right after (accessed through (ddi_dma_attr_t *)(mp + 1)) is used 53 * strictly for peer-to-peer transfers which do not obey system limits. 54 * 55 * return: DDI_SUCCESS DDI_DMA_NORESOURCES 56 */ 57 ddi_dma_impl_t * 58 px_dma_allocmp(dev_info_t *dip, dev_info_t *rdip, int (*waitfp)(caddr_t), 59 caddr_t arg) 60 { 61 register ddi_dma_impl_t *mp; 62 int sleep = (waitfp == DDI_DMA_SLEEP) ? KM_SLEEP : KM_NOSLEEP; 63 64 /* Caution: we don't use zalloc to enhance performance! */ 65 if ((mp = kmem_alloc(sizeof (px_dma_hdl_t), sleep)) == 0) { 66 DBG(DBG_DMA_MAP, dip, "can't alloc dma_handle\n"); 67 if (waitfp != DDI_DMA_DONTWAIT) { 68 DBG(DBG_DMA_MAP, dip, "alloc_mp kmem cb\n"); 69 ddi_set_callback(waitfp, arg, &px_kmem_clid); 70 } 71 return (mp); 72 } 73 74 mp->dmai_rdip = rdip; 75 mp->dmai_flags = 0; 76 mp->dmai_pfnlst = NULL; 77 mp->dmai_winlst = NULL; 78 79 /* 80 * kmem_alloc debug: the following fields are not zero-ed 81 * mp->dmai_mapping = 0; 82 * mp->dmai_size = 0; 83 * mp->dmai_offset = 0; 84 * mp->dmai_minxfer = 0; 85 * mp->dmai_burstsizes = 0; 86 * mp->dmai_ndvmapages = 0; 87 * mp->dmai_pool/roffset = 0; 88 * mp->dmai_rflags = 0; 89 * mp->dmai_inuse/flags 90 * mp->dmai_nwin = 0; 91 * mp->dmai_winsize = 0; 92 * mp->dmai_nexus_private/tte = 0; 93 * mp->dmai_iopte/pfnlst 94 * mp->dmai_sbi/pfn0 = 0; 95 * mp->dmai_minfo/winlst/fdvma 96 * mp->dmai_rdip 97 * bzero(&mp->dmai_object, sizeof (ddi_dma_obj_t)); 98 * bzero(&mp->dmai_attr, sizeof (ddi_dma_attr_t)); 99 * mp->dmai_cookie = 0; 100 */ 101 102 mp->dmai_attr.dma_attr_version = (uint_t)DMA_ATTR_VERSION; 103 mp->dmai_attr.dma_attr_flags = (uint_t)0; 104 mp->dmai_fault = 0; 105 mp->dmai_fault_check = NULL; 106 mp->dmai_fault_notify = NULL; 107 108 mp->dmai_error.err_ena = 0; 109 mp->dmai_error.err_status = DDI_FM_OK; 110 mp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 111 mp->dmai_error.err_ontrap = NULL; 112 mp->dmai_error.err_fep = NULL; 113 mp->dmai_error.err_cf = NULL; 114 115 if (px_child_prefetch(mp->dmai_rdip)) 116 mp->dmai_flags |= (PX_DMAI_FLAGS_MAP_BUFZONE | 117 PX_DMAI_FLAGS_REDZONE); 118 119 return (mp); 120 } 121 122 void 123 px_dma_freemp(ddi_dma_impl_t *mp) 124 { 125 if (mp->dmai_ndvmapages > 1) 126 px_dma_freepfn(mp); 127 if (mp->dmai_winlst) 128 px_dma_freewin(mp); 129 kmem_free(mp, sizeof (px_dma_hdl_t)); 130 } 131 132 void 133 px_dma_freepfn(ddi_dma_impl_t *mp) 134 { 135 void *addr = mp->dmai_pfnlst; 136 if (addr) { 137 size_t npages = mp->dmai_ndvmapages; 138 if (npages > 1) 139 kmem_free(addr, npages * sizeof (px_iopfn_t)); 140 mp->dmai_pfnlst = NULL; 141 } 142 mp->dmai_ndvmapages = 0; 143 } 144 145 /* 146 * px_dma_lmts2hdl - alloate a ddi_dma_impl_t, validate practical limits 147 * and convert dmareq->dmar_limits to mp->dmai_attr 148 * 149 * ddi_dma_impl_t member modified input 150 * ------------------------------------------------------------------------ 151 * mp->dmai_minxfer - dev 152 * mp->dmai_burstsizes - dev 153 * mp->dmai_flags - no limit? peer-to-peer only? 154 * 155 * ddi_dma_attr member modified input 156 * ------------------------------------------------------------------------ 157 * mp->dmai_attr.dma_attr_addr_lo - dev lo, sys lo 158 * mp->dmai_attr.dma_attr_addr_hi - dev hi, sys hi 159 * mp->dmai_attr.dma_attr_count_max - dev count max, dev/sys lo/hi delta 160 * mp->dmai_attr.dma_attr_seg - 0 (no nocross restriction) 161 * mp->dmai_attr.dma_attr_align - 1 (no alignment restriction) 162 * 163 * The dlim_dmaspeed member of dmareq->dmar_limits is ignored. 164 */ 165 ddi_dma_impl_t * 166 px_dma_lmts2hdl(dev_info_t *dip, dev_info_t *rdip, px_mmu_t *mmu_p, 167 ddi_dma_req_t *dmareq) 168 { 169 ddi_dma_impl_t *mp; 170 ddi_dma_attr_t *attr_p; 171 uint64_t syslo = mmu_p->mmu_dvma_base; 172 uint64_t syshi = mmu_p->mmu_dvma_end; 173 uint64_t fasthi = mmu_p->mmu_dvma_fast_end; 174 ddi_dma_lim_t *lim_p = dmareq->dmar_limits; 175 uint32_t count_max = lim_p->dlim_cntr_max; 176 uint64_t lo = lim_p->dlim_addr_lo; 177 uint64_t hi = lim_p->dlim_addr_hi; 178 if (hi <= lo) { 179 DBG(DBG_DMA_MAP, dip, "Bad limits\n"); 180 return ((ddi_dma_impl_t *)DDI_DMA_NOMAPPING); 181 } 182 if (!count_max) 183 count_max--; 184 185 if (!(mp = px_dma_allocmp(dip, rdip, dmareq->dmar_fp, 186 dmareq->dmar_arg))) 187 return (NULL); 188 189 /* store original dev input at the 2nd ddi_dma_attr */ 190 attr_p = PX_DEV_ATTR(mp); 191 SET_DMAATTR(attr_p, lo, hi, -1, count_max); 192 SET_DMAALIGN(attr_p, 1); 193 194 lo = MAX(lo, syslo); 195 hi = MIN(hi, syshi); 196 if (hi <= lo) 197 mp->dmai_flags |= PX_DMAI_FLAGS_PEER_ONLY; 198 count_max = MIN(count_max, hi - lo); 199 200 if (PX_DEV_NOSYSLIMIT(lo, hi, syslo, fasthi, 1)) 201 mp->dmai_flags |= PX_DMAI_FLAGS_NOFASTLIMIT | 202 PX_DMAI_FLAGS_NOSYSLIMIT; 203 else { 204 if (PX_DEV_NOFASTLIMIT(lo, hi, syslo, syshi, 1)) 205 mp->dmai_flags |= PX_DMAI_FLAGS_NOFASTLIMIT; 206 } 207 if (PX_DMA_NOCTX(rdip)) 208 mp->dmai_flags |= PX_DMAI_FLAGS_NOCTX; 209 210 /* store augumented dev input to mp->dmai_attr */ 211 mp->dmai_minxfer = lim_p->dlim_minxfer; 212 mp->dmai_burstsizes = lim_p->dlim_burstsizes; 213 attr_p = &mp->dmai_attr; 214 SET_DMAATTR(attr_p, lo, hi, -1, count_max); 215 SET_DMAALIGN(attr_p, 1); 216 return (mp); 217 } 218 219 /* 220 * Called from px_attach to check for bypass dma support and set 221 * flags accordingly. 222 */ 223 int 224 px_dma_attach(px_t *px_p) 225 { 226 uint64_t baddr; 227 228 if (px_lib_iommu_getbypass(px_p->px_dip, 0ull, 229 PCI_MAP_ATTR_WRITE|PCI_MAP_ATTR_READ, 230 &baddr) != DDI_ENOTSUP) 231 /* ignore all other errors */ 232 px_p->px_dev_caps |= PX_BYPASS_DMA_ALLOWED; 233 234 return (DDI_SUCCESS); 235 } 236 237 /* 238 * px_dma_attr2hdl 239 * 240 * This routine is called from the alloc handle entry point to sanity check the 241 * dma attribute structure. 242 * 243 * use by: px_dma_allochdl() 244 * 245 * return value: 246 * 247 * DDI_SUCCESS - on success 248 * DDI_DMA_BADATTR - attribute has invalid version number 249 * or address limits exclude dvma space 250 */ 251 int 252 px_dma_attr2hdl(px_t *px_p, ddi_dma_impl_t *mp) 253 { 254 px_mmu_t *mmu_p = px_p->px_mmu_p; 255 uint64_t syslo, syshi; 256 int ret; 257 ddi_dma_attr_t *attrp = PX_DEV_ATTR(mp); 258 uint64_t hi = attrp->dma_attr_addr_hi; 259 uint64_t lo = attrp->dma_attr_addr_lo; 260 uint64_t align = attrp->dma_attr_align; 261 uint64_t nocross = attrp->dma_attr_seg; 262 uint64_t count_max = attrp->dma_attr_count_max; 263 264 DBG(DBG_DMA_ALLOCH, px_p->px_dip, "attrp=%p cntr_max=%x.%08x\n", 265 attrp, HI32(count_max), LO32(count_max)); 266 DBG(DBG_DMA_ALLOCH, px_p->px_dip, "hi=%x.%08x lo=%x.%08x\n", 267 HI32(hi), LO32(hi), HI32(lo), LO32(lo)); 268 DBG(DBG_DMA_ALLOCH, px_p->px_dip, "seg=%x.%08x align=%x.%08x\n", 269 HI32(nocross), LO32(nocross), HI32(align), LO32(align)); 270 271 if (!nocross) 272 nocross--; 273 if (attrp->dma_attr_flags & DDI_DMA_FORCE_PHYSICAL) { /* BYPASS */ 274 275 DBG(DBG_DMA_ALLOCH, px_p->px_dip, "bypass mode\n"); 276 /* 277 * If Bypass DMA is not supported, return error so that 278 * target driver can fall back to dvma mode of operation 279 */ 280 if (!(px_p->px_dev_caps & PX_BYPASS_DMA_ALLOWED)) 281 return (DDI_DMA_BADATTR); 282 mp->dmai_flags |= PX_DMAI_FLAGS_BYPASSREQ; 283 if (nocross != UINT64_MAX) 284 return (DDI_DMA_BADATTR); 285 if (align && (align > MMU_PAGE_SIZE)) 286 return (DDI_DMA_BADATTR); 287 align = 1; /* align on 1 page boundary */ 288 289 /* do a range check and get the limits */ 290 ret = px_lib_dma_bypass_rngchk(px_p->px_dip, attrp, 291 &syslo, &syshi); 292 if (ret != DDI_SUCCESS) 293 return (ret); 294 } else { /* MMU_XLATE or PEER_TO_PEER */ 295 align = MAX(align, MMU_PAGE_SIZE) - 1; 296 if ((align & nocross) != align) { 297 dev_info_t *rdip = mp->dmai_rdip; 298 cmn_err(CE_WARN, "%s%d dma_attr_seg not aligned", 299 NAMEINST(rdip)); 300 return (DDI_DMA_BADATTR); 301 } 302 align = MMU_BTOP(align + 1); 303 syslo = mmu_p->mmu_dvma_base; 304 syshi = mmu_p->mmu_dvma_end; 305 } 306 if (hi <= lo) { 307 dev_info_t *rdip = mp->dmai_rdip; 308 cmn_err(CE_WARN, "%s%d limits out of range", NAMEINST(rdip)); 309 return (DDI_DMA_BADATTR); 310 } 311 lo = MAX(lo, syslo); 312 hi = MIN(hi, syshi); 313 if (!count_max) 314 count_max--; 315 316 DBG(DBG_DMA_ALLOCH, px_p->px_dip, "hi=%x.%08x, lo=%x.%08x\n", 317 HI32(hi), LO32(hi), HI32(lo), LO32(lo)); 318 if (hi <= lo) { 319 /* 320 * If this is an IOMMU bypass access, the caller can't use 321 * the required addresses, so fail it. Otherwise, it's 322 * peer-to-peer; ensure that the caller has no alignment or 323 * segment size restrictions. 324 */ 325 if ((mp->dmai_flags & PX_DMAI_FLAGS_BYPASSREQ) || 326 (nocross < UINT32_MAX) || (align > 1)) 327 return (DDI_DMA_BADATTR); 328 329 mp->dmai_flags |= PX_DMAI_FLAGS_PEER_ONLY; 330 } else /* set practical counter_max value */ 331 count_max = MIN(count_max, hi - lo); 332 333 if (PX_DEV_NOSYSLIMIT(lo, hi, syslo, syshi, align)) 334 mp->dmai_flags |= PX_DMAI_FLAGS_NOSYSLIMIT | 335 PX_DMAI_FLAGS_NOFASTLIMIT; 336 else { 337 syshi = mmu_p->mmu_dvma_fast_end; 338 if (PX_DEV_NOFASTLIMIT(lo, hi, syslo, syshi, align)) 339 mp->dmai_flags |= PX_DMAI_FLAGS_NOFASTLIMIT; 340 } 341 if (PX_DMA_NOCTX(mp->dmai_rdip)) 342 mp->dmai_flags |= PX_DMAI_FLAGS_NOCTX; 343 344 mp->dmai_minxfer = attrp->dma_attr_minxfer; 345 mp->dmai_burstsizes = attrp->dma_attr_burstsizes; 346 attrp = &mp->dmai_attr; 347 SET_DMAATTR(attrp, lo, hi, nocross, count_max); 348 return (DDI_SUCCESS); 349 } 350 351 #define TGT_PFN_INBETWEEN(pfn, bgn, end) ((pfn >= bgn) && (pfn <= end)) 352 353 /* 354 * px_dma_type - determine which of the three types DMA (peer-to-peer, 355 * mmu bypass, or mmu translate) we are asked to do. 356 * Also checks pfn0 and rejects any non-peer-to-peer 357 * requests for peer-only devices. 358 * 359 * return values: 360 * DDI_DMA_NOMAPPING - can't get valid pfn0, or bad dma type 361 * DDI_SUCCESS 362 * 363 * dma handle members affected (set on exit): 364 * mp->dmai_object - dmareq->dmar_object 365 * mp->dmai_rflags - consistent?, nosync?, dmareq->dmar_flags 366 * mp->dmai_flags - DMA type 367 * mp->dmai_pfn0 - 1st page pfn (if va/size pair and not shadow) 368 * mp->dmai_roffset - initialized to starting MMU page offset 369 * mp->dmai_ndvmapages - # of total MMU pages of entire object 370 */ 371 int 372 px_dma_type(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 373 { 374 dev_info_t *dip = px_p->px_dip; 375 ddi_dma_obj_t *dobj_p = &dmareq->dmar_object; 376 px_pec_t *pec_p = px_p->px_pec_p; 377 uint32_t offset; 378 pfn_t pfn0; 379 380 mp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS | DMP_NOSYNC; 381 382 switch (dobj_p->dmao_type) { 383 case DMA_OTYP_BUFVADDR: 384 case DMA_OTYP_VADDR: { 385 page_t **pplist = dobj_p->dmao_obj.virt_obj.v_priv; 386 caddr_t vaddr = dobj_p->dmao_obj.virt_obj.v_addr; 387 388 DBG(DBG_DMA_MAP, dip, "vaddr=%p pplist=%p\n", vaddr, pplist); 389 offset = (ulong_t)vaddr & MMU_PAGE_OFFSET; 390 if (pplist) { /* shadow list */ 391 mp->dmai_flags |= PX_DMAI_FLAGS_PGPFN; 392 pfn0 = page_pptonum(*pplist); 393 } else { 394 struct as *as_p = dobj_p->dmao_obj.virt_obj.v_as; 395 struct hat *hat_p = as_p ? as_p->a_hat : kas.a_hat; 396 pfn0 = hat_getpfnum(hat_p, vaddr); 397 } 398 } 399 break; 400 401 case DMA_OTYP_PAGES: 402 offset = dobj_p->dmao_obj.pp_obj.pp_offset; 403 mp->dmai_flags |= PX_DMAI_FLAGS_PGPFN; 404 pfn0 = page_pptonum(dobj_p->dmao_obj.pp_obj.pp_pp); 405 break; 406 407 case DMA_OTYP_PADDR: 408 default: 409 cmn_err(CE_WARN, "%s%d requested unsupported dma type %x", 410 NAMEINST(mp->dmai_rdip), dobj_p->dmao_type); 411 return (DDI_DMA_NOMAPPING); 412 } 413 if (pfn0 == PFN_INVALID) { 414 cmn_err(CE_WARN, "%s%d: invalid pfn0 for DMA object %p", 415 NAMEINST(dip), dobj_p); 416 return (DDI_DMA_NOMAPPING); 417 } 418 if (TGT_PFN_INBETWEEN(pfn0, pec_p->pec_base32_pfn, 419 pec_p->pec_last32_pfn)) { 420 mp->dmai_flags |= PX_DMAI_FLAGS_PTP|PX_DMAI_FLAGS_PTP32; 421 goto done; /* leave bypass and dvma flag as 0 */ 422 } else if (TGT_PFN_INBETWEEN(pfn0, pec_p->pec_base64_pfn, 423 pec_p->pec_last64_pfn)) { 424 mp->dmai_flags |= PX_DMAI_FLAGS_PTP|PX_DMAI_FLAGS_PTP64; 425 goto done; /* leave bypass and dvma flag as 0 */ 426 } 427 if (PX_DMA_ISPEERONLY(mp)) { 428 dev_info_t *rdip = mp->dmai_rdip; 429 cmn_err(CE_WARN, "Bad peer-to-peer req %s%d", NAMEINST(rdip)); 430 return (DDI_DMA_NOMAPPING); 431 } 432 mp->dmai_flags |= (mp->dmai_flags & PX_DMAI_FLAGS_BYPASSREQ) ? 433 PX_DMAI_FLAGS_BYPASS : PX_DMAI_FLAGS_DVMA | 434 (mp->dmai_rflags & DDI_DMA_REDZONE ? PX_DMAI_FLAGS_REDZONE : 0); 435 done: 436 mp->dmai_object = *dobj_p; /* whole object */ 437 mp->dmai_pfn0 = (void *)pfn0; /* cache pfn0 */ 438 mp->dmai_roffset = offset; /* win0 pg0 offset */ 439 mp->dmai_ndvmapages = MMU_BTOPR(offset + mp->dmai_object.dmao_size); 440 return (DDI_SUCCESS); 441 } 442 443 /* 444 * px_dma_pgpfn - set up pfnlst array according to pages 445 * VA/size pair: <shadow IO, bypass, peer-to-peer>, or OTYP_PAGES 446 */ 447 /*ARGSUSED*/ 448 static int 449 px_dma_pgpfn(px_t *px_p, ddi_dma_impl_t *mp, uint_t npages) 450 { 451 int i; 452 dev_info_t *dip = px_p->px_dip; 453 454 switch (mp->dmai_object.dmao_type) { 455 case DMA_OTYP_BUFVADDR: 456 case DMA_OTYP_VADDR: { 457 page_t **pplist = mp->dmai_object.dmao_obj.virt_obj.v_priv; 458 DBG(DBG_DMA_MAP, dip, "shadow pplist=%p, %x pages, pfns=", 459 pplist, npages); 460 for (i = 1; i < npages; i++) { 461 px_iopfn_t pfn = page_pptonum(pplist[i]); 462 PX_SET_MP_PFN1(mp, i, pfn); 463 DBG(DBG_DMA_MAP|DBG_CONT, dip, "%x ", pfn); 464 } 465 DBG(DBG_DMA_MAP|DBG_CONT, dip, "\n"); 466 } 467 break; 468 469 case DMA_OTYP_PAGES: { 470 page_t *pp = mp->dmai_object.dmao_obj.pp_obj.pp_pp->p_next; 471 DBG(DBG_DMA_MAP, dip, "pp=%p pfns=", pp); 472 for (i = 1; i < npages; i++, pp = pp->p_next) { 473 px_iopfn_t pfn = page_pptonum(pp); 474 PX_SET_MP_PFN1(mp, i, pfn); 475 DBG(DBG_DMA_MAP|DBG_CONT, dip, "%x ", pfn); 476 } 477 DBG(DBG_DMA_MAP|DBG_CONT, dip, "\n"); 478 } 479 break; 480 481 default: /* check is already done by px_dma_type */ 482 ASSERT(0); 483 break; 484 } 485 return (DDI_SUCCESS); 486 } 487 488 /* 489 * px_dma_vapfn - set up pfnlst array according to VA 490 * VA/size pair: <normal, bypass, peer-to-peer> 491 * pfn0 is skipped as it is already done. 492 * In this case, the cached pfn0 is used to fill pfnlst[0] 493 */ 494 static int 495 px_dma_vapfn(px_t *px_p, ddi_dma_impl_t *mp, uint_t npages) 496 { 497 dev_info_t *dip = px_p->px_dip; 498 int i; 499 caddr_t vaddr = (caddr_t)mp->dmai_object.dmao_obj.virt_obj.v_as; 500 struct hat *hat_p = vaddr ? ((struct as *)vaddr)->a_hat : kas.a_hat; 501 502 vaddr = mp->dmai_object.dmao_obj.virt_obj.v_addr + MMU_PAGE_SIZE; 503 for (i = 1; i < npages; i++, vaddr += MMU_PAGE_SIZE) { 504 px_iopfn_t pfn = hat_getpfnum(hat_p, vaddr); 505 if (pfn == PFN_INVALID) 506 goto err_badpfn; 507 PX_SET_MP_PFN1(mp, i, pfn); 508 DBG(DBG_DMA_BINDH, dip, "px_dma_vapfn: mp=%p pfnlst[%x]=%x\n", 509 mp, i, pfn); 510 } 511 return (DDI_SUCCESS); 512 err_badpfn: 513 cmn_err(CE_WARN, "%s%d: bad page frame vaddr=%p", NAMEINST(dip), vaddr); 514 return (DDI_DMA_NOMAPPING); 515 } 516 517 /* 518 * px_dma_pfn - Fills pfn list for all pages being DMA-ed. 519 * 520 * dependencies: 521 * mp->dmai_ndvmapages - set to total # of dma pages 522 * 523 * return value: 524 * DDI_SUCCESS 525 * DDI_DMA_NOMAPPING 526 */ 527 int 528 px_dma_pfn(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 529 { 530 uint32_t npages = mp->dmai_ndvmapages; 531 int (*waitfp)(caddr_t) = dmareq->dmar_fp; 532 int i, ret, peer = PX_DMA_ISPTP(mp); 533 int peer32 = PX_DMA_ISPTP32(mp); 534 dev_info_t *dip = px_p->px_dip; 535 536 px_pec_t *pec_p = px_p->px_pec_p; 537 px_iopfn_t pfn_base = peer32 ? pec_p->pec_base32_pfn : 538 pec_p->pec_base64_pfn; 539 px_iopfn_t pfn_last = peer32 ? pec_p->pec_last32_pfn : 540 pec_p->pec_last64_pfn; 541 px_iopfn_t pfn_adj = peer ? pfn_base : 0; 542 543 DBG(DBG_DMA_BINDH, dip, "px_dma_pfn: mp=%p pfn0=%x\n", 544 mp, PX_MP_PFN0(mp) - pfn_adj); 545 /* 1 page: no array alloc/fill, no mixed mode check */ 546 if (npages == 1) { 547 PX_SET_MP_PFN(mp, 0, PX_MP_PFN0(mp) - pfn_adj); 548 return (DDI_SUCCESS); 549 } 550 /* allocate pfn array */ 551 if (!(mp->dmai_pfnlst = kmem_alloc(npages * sizeof (px_iopfn_t), 552 waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP))) { 553 if (waitfp != DDI_DMA_DONTWAIT) 554 ddi_set_callback(waitfp, dmareq->dmar_arg, 555 &px_kmem_clid); 556 return (DDI_DMA_NORESOURCES); 557 } 558 /* fill pfn array */ 559 PX_SET_MP_PFN(mp, 0, PX_MP_PFN0(mp) - pfn_adj); /* pfnlst[0] */ 560 if ((ret = PX_DMA_ISPGPFN(mp) ? px_dma_pgpfn(px_p, mp, npages) : 561 px_dma_vapfn(px_p, mp, npages)) != DDI_SUCCESS) 562 goto err; 563 564 /* skip pfn0, check mixed mode and adjust peer to peer pfn */ 565 for (i = 1; i < npages; i++) { 566 px_iopfn_t pfn = PX_GET_MP_PFN1(mp, i); 567 if (peer ^ TGT_PFN_INBETWEEN(pfn, pfn_base, pfn_last)) { 568 cmn_err(CE_WARN, "%s%d mixed mode DMA %lx %lx", 569 NAMEINST(mp->dmai_rdip), PX_MP_PFN0(mp), pfn); 570 ret = DDI_DMA_NOMAPPING; /* mixed mode */ 571 goto err; 572 } 573 DBG(DBG_DMA_MAP, dip, 574 "px_dma_pfn: pfnlst[%x]=%x-%x\n", i, pfn, pfn_adj); 575 if (pfn_adj) 576 PX_SET_MP_PFN1(mp, i, pfn - pfn_adj); 577 } 578 return (DDI_SUCCESS); 579 err: 580 px_dma_freepfn(mp); 581 return (ret); 582 } 583 584 /* 585 * px_dvma_win() - trim requested DVMA size down to window size 586 * The 1st window starts from offset and ends at page-aligned boundary. 587 * From the 2nd window on, each window starts and ends at page-aligned 588 * boundary except the last window ends at wherever requested. 589 * 590 * accesses the following mp-> members: 591 * mp->dmai_attr.dma_attr_count_max 592 * mp->dmai_attr.dma_attr_seg 593 * mp->dmai_roffset - start offset of 1st window 594 * mp->dmai_rflags (redzone) 595 * mp->dmai_ndvmapages (for 1 page fast path) 596 * 597 * sets the following mp-> members: 598 * mp->dmai_size - xfer size, != winsize if 1st/last win (not fixed) 599 * mp->dmai_winsize - window size (no redzone), n * page size (fixed) 600 * mp->dmai_nwin - # of DMA windows of entire object (fixed) 601 * mp->dmai_rflags - remove partial flag if nwin == 1 (fixed) 602 * mp->dmai_winlst - NULL, window objects not used for DVMA (fixed) 603 * 604 * fixed - not changed across different DMA windows 605 */ 606 /*ARGSUSED*/ 607 int 608 px_dvma_win(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 609 { 610 uint32_t redzone_sz = PX_HAS_REDZONE(mp) ? MMU_PAGE_SIZE : 0; 611 size_t obj_sz = mp->dmai_object.dmao_size; 612 size_t xfer_sz; 613 ulong_t pg_off; 614 615 if ((mp->dmai_ndvmapages == 1) && !redzone_sz) { 616 mp->dmai_rflags &= ~DDI_DMA_PARTIAL; 617 mp->dmai_size = obj_sz; 618 mp->dmai_winsize = MMU_PAGE_SIZE; 619 mp->dmai_nwin = 1; 620 goto done; 621 } 622 623 pg_off = mp->dmai_roffset; 624 xfer_sz = obj_sz + redzone_sz; 625 626 /* include redzone in nocross check */ { 627 uint64_t nocross = mp->dmai_attr.dma_attr_seg; 628 if (xfer_sz + pg_off - 1 > nocross) 629 xfer_sz = nocross - pg_off + 1; 630 if (redzone_sz && (xfer_sz <= redzone_sz)) { 631 DBG(DBG_DMA_MAP, px_p->px_dip, 632 "nocross too small: " 633 "%lx(%lx)+%lx+%lx < %llx\n", 634 xfer_sz, obj_sz, pg_off, redzone_sz, nocross); 635 return (DDI_DMA_TOOBIG); 636 } 637 } 638 xfer_sz -= redzone_sz; /* restore transfer size */ 639 /* check counter max */ { 640 uint32_t count_max = mp->dmai_attr.dma_attr_count_max; 641 if (xfer_sz - 1 > count_max) 642 xfer_sz = count_max + 1; 643 } 644 if (xfer_sz >= obj_sz) { 645 mp->dmai_rflags &= ~DDI_DMA_PARTIAL; 646 mp->dmai_size = xfer_sz; 647 mp->dmai_winsize = P2ROUNDUP(xfer_sz + pg_off, MMU_PAGE_SIZE); 648 mp->dmai_nwin = 1; 649 goto done; 650 } 651 if (!(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 652 DBG(DBG_DMA_MAP, px_p->px_dip, "too big: %lx+%lx+%lx > %lx\n", 653 obj_sz, pg_off, redzone_sz, xfer_sz); 654 return (DDI_DMA_TOOBIG); 655 } 656 657 xfer_sz = MMU_PTOB(MMU_BTOP(xfer_sz + pg_off)); /* page align */ 658 mp->dmai_size = xfer_sz - pg_off; /* 1st window xferrable size */ 659 mp->dmai_winsize = xfer_sz; /* redzone not in winsize */ 660 mp->dmai_nwin = (obj_sz + pg_off + xfer_sz - 1) / xfer_sz; 661 done: 662 mp->dmai_winlst = NULL; 663 px_dump_dma_handle(DBG_DMA_MAP, px_p->px_dip, mp); 664 return (DDI_SUCCESS); 665 } 666 667 /* 668 * fast track cache entry to mmu context, inserts 3 0 bits between 669 * upper 6-bits and lower 3-bits of the 9-bit cache entry 670 */ 671 #define MMU_FCE_TO_CTX(i) (((i) << 3) | ((i) & 0x7) | 0x38) 672 673 /* 674 * px_dvma_map_fast - attempts to map fast trackable DVMA 675 */ 676 /*ARGSUSED*/ 677 int 678 px_dvma_map_fast(px_mmu_t *mmu_p, ddi_dma_impl_t *mp) 679 { 680 uint_t clustsz = px_dvma_page_cache_clustsz; 681 uint_t entries = px_dvma_page_cache_entries; 682 io_attributes_t attr = PX_GET_TTE_ATTR(mp->dmai_rflags, 683 mp->dmai_attr.dma_attr_flags); 684 int i = mmu_p->mmu_dvma_addr_scan_start; 685 uint8_t *lock_addr = mmu_p->mmu_dvma_cache_locks + i; 686 px_dvma_addr_t dvma_pg; 687 size_t npages = MMU_BTOP(mp->dmai_winsize); 688 dev_info_t *dip = mmu_p->mmu_px_p->px_dip; 689 690 extern uint8_t ldstub(uint8_t *); 691 ASSERT(MMU_PTOB(npages) == mp->dmai_winsize); 692 ASSERT(npages + PX_HAS_REDZONE(mp) <= clustsz); 693 694 for (; i < entries && ldstub(lock_addr); i++, lock_addr++); 695 if (i >= entries) { 696 lock_addr = mmu_p->mmu_dvma_cache_locks; 697 i = 0; 698 for (; i < entries && ldstub(lock_addr); i++, lock_addr++); 699 if (i >= entries) { 700 #ifdef PX_DMA_PROF 701 px_dvmaft_exhaust++; 702 #endif /* PX_DMA_PROF */ 703 return (DDI_DMA_NORESOURCES); 704 } 705 } 706 mmu_p->mmu_dvma_addr_scan_start = (i + 1) & (entries - 1); 707 708 i *= clustsz; 709 dvma_pg = mmu_p->dvma_base_pg + i; 710 711 if (px_lib_iommu_map(dip, PCI_TSBID(0, i), npages, attr, 712 (void *)mp, 0, MMU_MAP_PFN) != DDI_SUCCESS) { 713 DBG(DBG_MAP_WIN, dip, "px_dvma_map_fast: " 714 "px_lib_iommu_map failed\n"); 715 716 return (DDI_FAILURE); 717 } 718 719 if (!PX_MAP_BUFZONE(mp)) 720 goto done; 721 722 DBG(DBG_MAP_WIN, dip, "px_dvma_map_fast: redzone pg=%x\n", i + npages); 723 724 ASSERT(PX_HAS_REDZONE(mp)); 725 726 if (px_lib_iommu_map(dip, PCI_TSBID(0, i + npages), 1, attr, 727 (void *)mp, npages - 1, MMU_MAP_PFN) != DDI_SUCCESS) { 728 DBG(DBG_MAP_WIN, dip, "px_dvma_map_fast: " 729 "mapping REDZONE page failed\n"); 730 731 (void) px_lib_iommu_demap(dip, PCI_TSBID(0, i), npages); 732 return (DDI_FAILURE); 733 } 734 735 done: 736 #ifdef PX_DMA_PROF 737 px_dvmaft_success++; 738 #endif 739 mp->dmai_mapping = mp->dmai_roffset | MMU_PTOB(dvma_pg); 740 mp->dmai_offset = 0; 741 mp->dmai_flags |= PX_DMAI_FLAGS_FASTTRACK; 742 PX_SAVE_MP_TTE(mp, attr); /* save TTE template for unmapping */ 743 if (PX_DVMA_DBG_ON(mmu_p)) 744 px_dvma_alloc_debug(mmu_p, (char *)mp->dmai_mapping, 745 mp->dmai_size, mp); 746 return (DDI_SUCCESS); 747 } 748 749 /* 750 * px_dvma_map: map non-fasttrack DMA 751 * Use quantum cache if single page DMA. 752 */ 753 int 754 px_dvma_map(ddi_dma_impl_t *mp, ddi_dma_req_t *dmareq, px_mmu_t *mmu_p) 755 { 756 uint_t npages = PX_DMA_WINNPGS(mp); 757 px_dvma_addr_t dvma_pg, dvma_pg_index; 758 void *dvma_addr; 759 uint64_t tte = PX_GET_TTE_ATTR(mp->dmai_rflags, 760 mp->dmai_attr.dma_attr_flags); 761 int sleep = dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP; 762 dev_info_t *dip = mp->dmai_rdip; 763 int ret = DDI_SUCCESS; 764 765 /* 766 * allocate dvma space resource and map in the first window. 767 * (vmem_t *vmp, size_t size, 768 * size_t align, size_t phase, size_t nocross, 769 * void *minaddr, void *maxaddr, int vmflag) 770 */ 771 if ((npages == 1) && !PX_HAS_REDZONE(mp) && PX_HAS_NOSYSLIMIT(mp)) { 772 dvma_addr = vmem_alloc(mmu_p->mmu_dvma_map, 773 MMU_PAGE_SIZE, sleep); 774 mp->dmai_flags |= PX_DMAI_FLAGS_VMEMCACHE; 775 #ifdef PX_DMA_PROF 776 px_dvma_vmem_alloc++; 777 #endif /* PX_DMA_PROF */ 778 } else { 779 dvma_addr = vmem_xalloc(mmu_p->mmu_dvma_map, 780 MMU_PTOB(npages + PX_HAS_REDZONE(mp)), 781 MAX(mp->dmai_attr.dma_attr_align, MMU_PAGE_SIZE), 782 0, 783 mp->dmai_attr.dma_attr_seg + 1, 784 (void *)mp->dmai_attr.dma_attr_addr_lo, 785 (void *)(mp->dmai_attr.dma_attr_addr_hi + 1), 786 sleep); 787 #ifdef PX_DMA_PROF 788 px_dvma_vmem_xalloc++; 789 #endif /* PX_DMA_PROF */ 790 } 791 dvma_pg = MMU_BTOP((ulong_t)dvma_addr); 792 dvma_pg_index = dvma_pg - mmu_p->dvma_base_pg; 793 DBG(DBG_DMA_MAP, dip, "fallback dvma_pages: dvma_pg=%x index=%x\n", 794 dvma_pg, dvma_pg_index); 795 if (dvma_pg == 0) 796 goto noresource; 797 798 mp->dmai_mapping = mp->dmai_roffset | MMU_PTOB(dvma_pg); 799 mp->dmai_offset = 0; 800 PX_SAVE_MP_TTE(mp, tte); /* mp->dmai_tte = tte */ 801 802 if ((ret = px_mmu_map_pages(mmu_p, 803 mp, dvma_pg, npages, 0)) != DDI_SUCCESS) { 804 if (mp->dmai_flags & PX_DMAI_FLAGS_VMEMCACHE) { 805 vmem_free(mmu_p->mmu_dvma_map, (void *)dvma_addr, 806 MMU_PAGE_SIZE); 807 #ifdef PX_DMA_PROF 808 px_dvma_vmem_free++; 809 #endif /* PX_DMA_PROF */ 810 } else { 811 vmem_xfree(mmu_p->mmu_dvma_map, (void *)dvma_addr, 812 MMU_PTOB(npages + PX_HAS_REDZONE(mp))); 813 #ifdef PX_DMA_PROF 814 px_dvma_vmem_xfree++; 815 #endif /* PX_DMA_PROF */ 816 } 817 } 818 819 return (ret); 820 noresource: 821 if (dmareq->dmar_fp != DDI_DMA_DONTWAIT) { 822 DBG(DBG_DMA_MAP, dip, "dvma_pg 0 - set callback\n"); 823 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg, 824 &mmu_p->mmu_dvma_clid); 825 } 826 DBG(DBG_DMA_MAP, dip, "vmem_xalloc - DDI_DMA_NORESOURCES\n"); 827 return (DDI_DMA_NORESOURCES); 828 } 829 830 void 831 px_dvma_unmap(px_mmu_t *mmu_p, ddi_dma_impl_t *mp) 832 { 833 px_dvma_addr_t dvma_addr = (px_dvma_addr_t)mp->dmai_mapping; 834 px_dvma_addr_t dvma_pg = MMU_BTOP(dvma_addr); 835 dvma_addr = MMU_PTOB(dvma_pg); 836 837 if (mp->dmai_flags & PX_DMAI_FLAGS_FASTTRACK) { 838 px_iopfn_t index = dvma_pg - mmu_p->dvma_base_pg; 839 ASSERT(index % px_dvma_page_cache_clustsz == 0); 840 index /= px_dvma_page_cache_clustsz; 841 ASSERT(index < px_dvma_page_cache_entries); 842 mmu_p->mmu_dvma_cache_locks[index] = 0; 843 #ifdef PX_DMA_PROF 844 px_dvmaft_free++; 845 #endif /* PX_DMA_PROF */ 846 return; 847 } 848 849 if (mp->dmai_flags & PX_DMAI_FLAGS_VMEMCACHE) { 850 vmem_free(mmu_p->mmu_dvma_map, (void *)dvma_addr, 851 MMU_PAGE_SIZE); 852 #ifdef PX_DMA_PROF 853 px_dvma_vmem_free++; 854 #endif /* PX_DMA_PROF */ 855 } else { 856 size_t npages = MMU_BTOP(mp->dmai_winsize) + PX_HAS_REDZONE(mp); 857 vmem_xfree(mmu_p->mmu_dvma_map, (void *)dvma_addr, 858 MMU_PTOB(npages)); 859 #ifdef PX_DMA_PROF 860 px_dvma_vmem_xfree++; 861 #endif /* PX_DMA_PROF */ 862 } 863 } 864 865 /* 866 * DVMA mappings may have multiple windows, but each window always have 867 * one segment. 868 */ 869 int 870 px_dvma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp, 871 enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp, 872 uint_t cache_flags) 873 { 874 switch (cmd) { 875 case DDI_DMA_SYNC: 876 return (px_lib_dma_sync(dip, rdip, (ddi_dma_handle_t)mp, 877 *offp, *lenp, cache_flags)); 878 879 case DDI_DMA_HTOC: { 880 int ret; 881 off_t wo_off, off = *offp; /* wo_off: wnd's obj offset */ 882 uint_t win_size = mp->dmai_winsize; 883 ddi_dma_cookie_t *cp = (ddi_dma_cookie_t *)objp; 884 885 if (off >= mp->dmai_object.dmao_size) { 886 cmn_err(CE_WARN, "%s%d invalid dma_htoc offset %lx", 887 NAMEINST(mp->dmai_rdip), off); 888 return (DDI_FAILURE); 889 } 890 off += mp->dmai_roffset; 891 ret = px_dma_win(dip, rdip, (ddi_dma_handle_t)mp, 892 off / win_size, &wo_off, NULL, cp, NULL); /* lenp == NULL */ 893 if (ret) 894 return (ret); 895 DBG(DBG_DMA_CTL, dip, "HTOC:cookie=%x+%lx off=%lx,%lx\n", 896 cp->dmac_address, cp->dmac_size, off, *offp); 897 898 /* adjust cookie addr/len if we are not on window boundary */ 899 ASSERT((off % win_size) == (off - 900 (PX_DMA_CURWIN(mp) ? mp->dmai_roffset : 0) - wo_off)); 901 off = PX_DMA_CURWIN(mp) ? off % win_size : *offp; 902 ASSERT(cp->dmac_size > off); 903 cp->dmac_laddress += off; 904 cp->dmac_size -= off; 905 DBG(DBG_DMA_CTL, dip, "HTOC:mp=%p cookie=%x+%lx off=%lx,%lx\n", 906 mp, cp->dmac_address, cp->dmac_size, off, wo_off); 907 } 908 return (DDI_SUCCESS); 909 910 case DDI_DMA_REPWIN: 911 *offp = mp->dmai_offset; 912 *lenp = mp->dmai_size; 913 return (DDI_SUCCESS); 914 915 case DDI_DMA_MOVWIN: { 916 off_t off = *offp; 917 if (off >= mp->dmai_object.dmao_size) 918 return (DDI_FAILURE); 919 off += mp->dmai_roffset; 920 return (px_dma_win(dip, rdip, (ddi_dma_handle_t)mp, 921 off / mp->dmai_winsize, offp, lenp, 922 (ddi_dma_cookie_t *)objp, NULL)); 923 } 924 925 case DDI_DMA_NEXTWIN: { 926 px_window_t win = PX_DMA_CURWIN(mp); 927 if (offp) { 928 if (*(px_window_t *)offp != win) { 929 /* window not active */ 930 *(px_window_t *)objp = win; /* return cur win */ 931 return (DDI_DMA_STALE); 932 } 933 win++; 934 } else /* map win 0 */ 935 win = 0; 936 if (win >= mp->dmai_nwin) { 937 *(px_window_t *)objp = win - 1; 938 return (DDI_DMA_DONE); 939 } 940 if (px_dma_win(dip, rdip, (ddi_dma_handle_t)mp, 941 win, 0, 0, 0, 0)) { 942 *(px_window_t *)objp = win - 1; 943 return (DDI_FAILURE); 944 } 945 *(px_window_t *)objp = win; 946 } 947 return (DDI_SUCCESS); 948 949 case DDI_DMA_NEXTSEG: 950 if (*(px_window_t *)offp != PX_DMA_CURWIN(mp)) 951 return (DDI_DMA_STALE); 952 if (lenp) /* only 1 seg allowed */ 953 return (DDI_DMA_DONE); 954 955 /* return mp as seg 0 */ 956 *(ddi_dma_seg_t *)objp = (ddi_dma_seg_t)mp; 957 return (DDI_SUCCESS); 958 959 case DDI_DMA_SEGTOC: 960 MAKE_DMA_COOKIE((ddi_dma_cookie_t *)objp, mp->dmai_mapping, 961 mp->dmai_size); 962 *offp = mp->dmai_offset; 963 *lenp = mp->dmai_size; 964 return (DDI_SUCCESS); 965 966 case DDI_DMA_COFF: { 967 ddi_dma_cookie_t *cp = (ddi_dma_cookie_t *)offp; 968 if (cp->dmac_address < mp->dmai_mapping || 969 (cp->dmac_address + cp->dmac_size) > 970 (mp->dmai_mapping + mp->dmai_size)) 971 return (DDI_FAILURE); 972 *objp = (caddr_t)(cp->dmac_address - mp->dmai_mapping + 973 mp->dmai_offset); 974 } 975 return (DDI_SUCCESS); 976 default: 977 DBG(DBG_DMA_CTL, dip, "unknown command (%x): rdip=%s%d\n", 978 cmd, ddi_driver_name(rdip), ddi_get_instance(rdip)); 979 break; 980 } 981 return (DDI_FAILURE); 982 } 983 984 void 985 px_dma_freewin(ddi_dma_impl_t *mp) 986 { 987 px_dma_win_t *win_p = mp->dmai_winlst, *win2_p; 988 for (win2_p = win_p; win_p; win2_p = win_p) { 989 win_p = win2_p->win_next; 990 kmem_free(win2_p, sizeof (px_dma_win_t) + 991 sizeof (ddi_dma_cookie_t) * win2_p->win_ncookies); 992 } 993 mp->dmai_nwin = 0; 994 mp->dmai_winlst = NULL; 995 } 996 997 /* 998 * px_dma_newwin - create a dma window object and cookies 999 * 1000 * After the initial scan in px_dma_physwin(), which identifies 1001 * a portion of the pfn array that belongs to a dma window, 1002 * we are called to allocate and initialize representing memory 1003 * resources. We know from the 1st scan the number of cookies 1004 * or dma segment in this window so we can allocate a contiguous 1005 * memory array for the dma cookies (The implementation of 1006 * ddi_dma_nextcookie(9f) dictates dma cookies be contiguous). 1007 * 1008 * A second round scan is done on the pfn array to identify 1009 * each dma segment and initialize its corresponding dma cookie. 1010 * We don't need to do all the safety checking and we know they 1011 * all belong to the same dma window. 1012 * 1013 * Input: cookie_no - # of cookies identified by the 1st scan 1014 * start_idx - subscript of the pfn array for the starting pfn 1015 * end_idx - subscript of the last pfn in dma window 1016 * win_pp - pointer to win_next member of previous window 1017 * Return: DDI_SUCCESS - with **win_pp as newly created window object 1018 * DDI_DMA_NORESROUCE - caller frees all previous window objs 1019 * Note: Each cookie and window size are all initialized on page 1020 * boundary. This is not true for the 1st cookie of the 1st 1021 * window and the last cookie of the last window. 1022 * We fix that later in upper layer which has access to size 1023 * and offset info. 1024 * 1025 */ 1026 /*ARGSUSED*/ 1027 static int 1028 px_dma_newwin(dev_info_t *dip, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, 1029 uint32_t cookie_no, uint32_t start_idx, uint32_t end_idx, 1030 px_dma_win_t **win_pp, uint64_t count_max, uint64_t bypass) 1031 { 1032 int (*waitfp)(caddr_t) = dmareq->dmar_fp; 1033 ddi_dma_cookie_t *cookie_p; 1034 uint32_t pfn_no = 1; 1035 px_iopfn_t pfn = PX_GET_MP_PFN(mp, start_idx); 1036 px_iopfn_t prev_pfn = pfn; 1037 uint64_t baddr, seg_pfn0 = pfn; 1038 size_t sz = cookie_no * sizeof (ddi_dma_cookie_t); 1039 px_dma_win_t *win_p = kmem_zalloc(sizeof (px_dma_win_t) + sz, 1040 waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP); 1041 io_attributes_t attr = PX_GET_TTE_ATTR(mp->dmai_rflags, 1042 mp->dmai_attr.dma_attr_flags); 1043 1044 if (!win_p) 1045 goto noresource; 1046 1047 win_p->win_next = NULL; 1048 win_p->win_ncookies = cookie_no; 1049 win_p->win_curseg = 0; /* start from segment 0 */ 1050 win_p->win_size = MMU_PTOB(end_idx - start_idx + 1); 1051 /* win_p->win_offset is left uninitialized */ 1052 1053 cookie_p = (ddi_dma_cookie_t *)(win_p + 1); 1054 start_idx++; 1055 for (; start_idx <= end_idx; start_idx++, prev_pfn = pfn, pfn_no++) { 1056 pfn = PX_GET_MP_PFN1(mp, start_idx); 1057 if ((pfn == prev_pfn + 1) && 1058 (MMU_PTOB(pfn_no + 1) - 1 <= count_max)) 1059 continue; 1060 1061 /* close up the cookie up to (including) prev_pfn */ 1062 baddr = MMU_PTOB(seg_pfn0); 1063 if (bypass && (px_lib_iommu_getbypass(dip, 1064 baddr, attr, &baddr) != DDI_SUCCESS)) 1065 return (DDI_FAILURE); 1066 1067 MAKE_DMA_COOKIE(cookie_p, baddr, MMU_PTOB(pfn_no)); 1068 DBG(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages)\n", 1069 MMU_PTOB(seg_pfn0), pfn_no); 1070 1071 cookie_p++; /* advance to next available cookie cell */ 1072 pfn_no = 0; 1073 seg_pfn0 = pfn; /* start a new segment from current pfn */ 1074 } 1075 1076 baddr = MMU_PTOB(seg_pfn0); 1077 if (bypass && (px_lib_iommu_getbypass(dip, 1078 baddr, attr, &baddr) != DDI_SUCCESS)) 1079 return (DDI_FAILURE); 1080 1081 MAKE_DMA_COOKIE(cookie_p, baddr, MMU_PTOB(pfn_no)); 1082 DBG(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages) of total %x\n", 1083 MMU_PTOB(seg_pfn0), pfn_no, cookie_no); 1084 #ifdef DEBUG 1085 cookie_p++; 1086 ASSERT((cookie_p - (ddi_dma_cookie_t *)(win_p + 1)) == cookie_no); 1087 #endif /* DEBUG */ 1088 *win_pp = win_p; 1089 return (DDI_SUCCESS); 1090 noresource: 1091 if (waitfp != DDI_DMA_DONTWAIT) 1092 ddi_set_callback(waitfp, dmareq->dmar_arg, &px_kmem_clid); 1093 return (DDI_DMA_NORESOURCES); 1094 } 1095 1096 /* 1097 * px_dma_adjust - adjust 1st and last cookie and window sizes 1098 * remove initial dma page offset from 1st cookie and window size 1099 * remove last dma page remainder from last cookie and window size 1100 * fill win_offset of each dma window according to just fixed up 1101 * each window sizes 1102 * px_dma_win_t members modified: 1103 * win_p->win_offset - this window's offset within entire DMA object 1104 * win_p->win_size - xferrable size (in bytes) for this window 1105 * 1106 * ddi_dma_impl_t members modified: 1107 * mp->dmai_size - 1st window xferrable size 1108 * mp->dmai_offset - 0, which is the dma offset of the 1st window 1109 * 1110 * ddi_dma_cookie_t members modified: 1111 * cookie_p->dmac_size - 1st and last cookie remove offset or remainder 1112 * cookie_p->dmac_laddress - 1st cookie add page offset 1113 */ 1114 static void 1115 px_dma_adjust(ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, px_dma_win_t *win_p) 1116 { 1117 ddi_dma_cookie_t *cookie_p = (ddi_dma_cookie_t *)(win_p + 1); 1118 size_t pg_offset = mp->dmai_roffset; 1119 size_t win_offset = 0; 1120 1121 cookie_p->dmac_size -= pg_offset; 1122 cookie_p->dmac_laddress |= pg_offset; 1123 win_p->win_size -= pg_offset; 1124 DBG(DBG_BYPASS, mp->dmai_rdip, "pg0 adjust %lx\n", pg_offset); 1125 1126 mp->dmai_size = win_p->win_size; 1127 mp->dmai_offset = 0; 1128 1129 pg_offset += mp->dmai_object.dmao_size; 1130 pg_offset &= MMU_PAGE_OFFSET; 1131 if (pg_offset) 1132 pg_offset = MMU_PAGE_SIZE - pg_offset; 1133 DBG(DBG_BYPASS, mp->dmai_rdip, "last pg adjust %lx\n", pg_offset); 1134 1135 for (; win_p->win_next; win_p = win_p->win_next) { 1136 DBG(DBG_BYPASS, mp->dmai_rdip, "win off %p\n", win_offset); 1137 win_p->win_offset = win_offset; 1138 win_offset += win_p->win_size; 1139 } 1140 /* last window */ 1141 win_p->win_offset = win_offset; 1142 cookie_p = (ddi_dma_cookie_t *)(win_p + 1); 1143 cookie_p[win_p->win_ncookies - 1].dmac_size -= pg_offset; 1144 win_p->win_size -= pg_offset; 1145 ASSERT((win_offset + win_p->win_size) == mp->dmai_object.dmao_size); 1146 } 1147 1148 /* 1149 * px_dma_physwin() - carve up dma windows using physical addresses. 1150 * Called to handle mmu bypass and pci peer-to-peer transfers. 1151 * Calls px_dma_newwin() to allocate window objects. 1152 * 1153 * Dependency: mp->dmai_pfnlst points to an array of pfns 1154 * 1155 * 1. Each dma window is represented by a px_dma_win_t object. 1156 * The object will be casted to ddi_dma_win_t and returned 1157 * to leaf driver through the DDI interface. 1158 * 2. Each dma window can have several dma segments with each 1159 * segment representing a physically contiguous either memory 1160 * space (if we are doing an mmu bypass transfer) or pci address 1161 * space (if we are doing a peer-to-peer transfer). 1162 * 3. Each segment has a DMA cookie to program the DMA engine. 1163 * The cookies within each DMA window must be located in a 1164 * contiguous array per ddi_dma_nextcookie(9f). 1165 * 4. The number of DMA segments within each DMA window cannot exceed 1166 * mp->dmai_attr.dma_attr_sgllen. If the transfer size is 1167 * too large to fit in the sgllen, the rest needs to be 1168 * relocated to the next dma window. 1169 * 5. Peer-to-peer DMA segment follows device hi, lo, count_max, 1170 * and nocross restrictions while bypass DMA follows the set of 1171 * restrictions with system limits factored in. 1172 * 1173 * Return: 1174 * mp->dmai_winlst - points to a link list of px_dma_win_t objects. 1175 * Each px_dma_win_t object on the link list contains 1176 * infomation such as its window size (# of pages), 1177 * starting offset (also see Restriction), an array of 1178 * DMA cookies, and # of cookies in the array. 1179 * mp->dmai_pfnlst - NULL, the pfn list is freed to conserve memory. 1180 * mp->dmai_nwin - # of total DMA windows on mp->dmai_winlst. 1181 * mp->dmai_mapping - starting cookie address 1182 * mp->dmai_rflags - consistent, nosync, no redzone 1183 * mp->dmai_cookie - start of cookie table of the 1st DMA window 1184 * 1185 * Restriction: 1186 * Each px_dma_win_t object can theoratically start from any offset 1187 * since the mmu is not involved. However, this implementation 1188 * always make windows start from page aligned offset (except 1189 * the 1st window, which follows the requested offset) due to the 1190 * fact that we are handed a pfn list. This does require device's 1191 * count_max and attr_seg to be at least MMU_PAGE_SIZE aligned. 1192 */ 1193 int 1194 px_dma_physwin(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 1195 { 1196 uint_t npages = mp->dmai_ndvmapages; 1197 int ret, sgllen = mp->dmai_attr.dma_attr_sgllen; 1198 px_iopfn_t pfn_lo, pfn_hi, prev_pfn; 1199 px_iopfn_t pfn = PX_GET_MP_PFN(mp, 0); 1200 uint32_t i, win_no = 0, pfn_no = 1, win_pfn0_index = 0, cookie_no = 0; 1201 uint64_t count_max, bypass_addr = 0; 1202 px_dma_win_t **win_pp = (px_dma_win_t **)&mp->dmai_winlst; 1203 ddi_dma_cookie_t *cookie0_p; 1204 io_attributes_t attr = PX_GET_TTE_ATTR(mp->dmai_rflags, 1205 mp->dmai_attr.dma_attr_flags); 1206 dev_info_t *dip = px_p->px_dip; 1207 1208 ASSERT(PX_DMA_ISPTP(mp) || PX_DMA_ISBYPASS(mp)); 1209 if (PX_DMA_ISPTP(mp)) { /* ignore sys limits for peer-to-peer */ 1210 ddi_dma_attr_t *dev_attr_p = PX_DEV_ATTR(mp); 1211 uint64_t nocross = dev_attr_p->dma_attr_seg; 1212 px_pec_t *pec_p = px_p->px_pec_p; 1213 px_iopfn_t pfn_last = PX_DMA_ISPTP32(mp) ? 1214 pec_p->pec_last32_pfn - pec_p->pec_base32_pfn : 1215 pec_p->pec_last64_pfn - pec_p->pec_base64_pfn; 1216 1217 if (nocross && (nocross < UINT32_MAX)) 1218 return (DDI_DMA_NOMAPPING); 1219 if (dev_attr_p->dma_attr_align > MMU_PAGE_SIZE) 1220 return (DDI_DMA_NOMAPPING); 1221 pfn_lo = MMU_BTOP(dev_attr_p->dma_attr_addr_lo); 1222 pfn_hi = MMU_BTOP(dev_attr_p->dma_attr_addr_hi); 1223 pfn_hi = MIN(pfn_hi, pfn_last); 1224 if ((pfn_lo > pfn_hi) || (pfn < pfn_lo)) 1225 return (DDI_DMA_NOMAPPING); 1226 1227 count_max = dev_attr_p->dma_attr_count_max; 1228 count_max = MIN(count_max, nocross); 1229 /* 1230 * the following count_max trim is not done because we are 1231 * making sure pfn_lo <= pfn <= pfn_hi inside the loop 1232 * count_max=MIN(count_max, MMU_PTOB(pfn_hi - pfn_lo + 1)-1); 1233 */ 1234 } else { /* bypass hi/lo/count_max have been processed by attr2hdl() */ 1235 count_max = mp->dmai_attr.dma_attr_count_max; 1236 pfn_lo = MMU_BTOP(mp->dmai_attr.dma_attr_addr_lo); 1237 pfn_hi = MMU_BTOP(mp->dmai_attr.dma_attr_addr_hi); 1238 1239 if (px_lib_iommu_getbypass(dip, MMU_PTOB(pfn), 1240 attr, &bypass_addr) != DDI_SUCCESS) { 1241 cmn_err(CE_WARN, "bypass cookie failure %lx\n", pfn); 1242 return (DDI_DMA_NOMAPPING); 1243 } 1244 pfn = MMU_BTOP(bypass_addr); 1245 } 1246 1247 /* pfn: absolute (bypass mode) or relative (p2p mode) */ 1248 for (prev_pfn = pfn, i = 1; i < npages; 1249 i++, prev_pfn = pfn, pfn_no++) { 1250 pfn = PX_GET_MP_PFN1(mp, i); 1251 if (bypass_addr) { 1252 if (px_lib_iommu_getbypass(dip, MMU_PTOB(pfn), attr, 1253 &bypass_addr) != DDI_SUCCESS) { 1254 ret = DDI_DMA_NOMAPPING; 1255 goto err; 1256 } 1257 pfn = MMU_BTOP(bypass_addr); 1258 } 1259 if ((pfn == prev_pfn + 1) && 1260 (MMU_PTOB(pfn_no + 1) - 1 <= count_max)) 1261 continue; 1262 if ((pfn < pfn_lo) || (prev_pfn > pfn_hi)) { 1263 ret = DDI_DMA_NOMAPPING; 1264 goto err; 1265 } 1266 cookie_no++; 1267 pfn_no = 0; 1268 if (cookie_no < sgllen) 1269 continue; 1270 1271 DBG(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n", 1272 win_pfn0_index, i - 1, cookie_no); 1273 if (ret = px_dma_newwin(dip, dmareq, mp, cookie_no, 1274 win_pfn0_index, i - 1, win_pp, count_max, bypass_addr)) 1275 goto err; 1276 1277 win_pp = &(*win_pp)->win_next; /* win_pp = *(win_pp) */ 1278 win_no++; 1279 win_pfn0_index = i; 1280 cookie_no = 0; 1281 } 1282 if (pfn > pfn_hi) { 1283 ret = DDI_DMA_NOMAPPING; 1284 goto err; 1285 } 1286 cookie_no++; 1287 DBG(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n", 1288 win_pfn0_index, i - 1, cookie_no); 1289 if (ret = px_dma_newwin(dip, dmareq, mp, cookie_no, win_pfn0_index, 1290 i - 1, win_pp, count_max, bypass_addr)) 1291 goto err; 1292 win_no++; 1293 px_dma_adjust(dmareq, mp, mp->dmai_winlst); 1294 mp->dmai_nwin = win_no; 1295 mp->dmai_rflags |= DDI_DMA_CONSISTENT | DMP_NOSYNC; 1296 mp->dmai_rflags &= ~DDI_DMA_REDZONE; 1297 mp->dmai_flags |= PX_DMAI_FLAGS_NOSYNC; 1298 cookie0_p = (ddi_dma_cookie_t *)(PX_WINLST(mp) + 1); 1299 mp->dmai_cookie = PX_WINLST(mp)->win_ncookies > 1 ? cookie0_p + 1 : 0; 1300 mp->dmai_mapping = cookie0_p->dmac_laddress; 1301 1302 px_dma_freepfn(mp); 1303 return (DDI_DMA_MAPPED); 1304 err: 1305 px_dma_freewin(mp); 1306 return (ret); 1307 } 1308 1309 int 1310 px_dma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp, 1311 enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp, 1312 uint_t cache_flags) 1313 { 1314 switch (cmd) { 1315 case DDI_DMA_SYNC: 1316 return (DDI_SUCCESS); 1317 1318 case DDI_DMA_HTOC: { 1319 off_t off = *offp; 1320 ddi_dma_cookie_t *loop_cp, *cp; 1321 px_dma_win_t *win_p = mp->dmai_winlst; 1322 1323 if (off >= mp->dmai_object.dmao_size) 1324 return (DDI_FAILURE); 1325 1326 /* locate window */ 1327 while (win_p->win_offset + win_p->win_size <= off) 1328 win_p = win_p->win_next; 1329 1330 loop_cp = cp = (ddi_dma_cookie_t *)(win_p + 1); 1331 mp->dmai_offset = win_p->win_offset; 1332 mp->dmai_size = win_p->win_size; 1333 mp->dmai_mapping = cp->dmac_laddress; /* cookie0 start addr */ 1334 1335 /* adjust cookie addr/len if we are not on cookie boundary */ 1336 off -= win_p->win_offset; /* offset within window */ 1337 for (; off >= loop_cp->dmac_size; loop_cp++) 1338 off -= loop_cp->dmac_size; /* offset within cookie */ 1339 1340 mp->dmai_cookie = loop_cp + 1; 1341 win_p->win_curseg = loop_cp - cp; 1342 cp = (ddi_dma_cookie_t *)objp; 1343 MAKE_DMA_COOKIE(cp, loop_cp->dmac_laddress + off, 1344 loop_cp->dmac_size - off); 1345 1346 DBG(DBG_DMA_CTL, dip, 1347 "HTOC: cookie - dmac_laddress=%p dmac_size=%x\n", 1348 cp->dmac_laddress, cp->dmac_size); 1349 } 1350 return (DDI_SUCCESS); 1351 1352 case DDI_DMA_REPWIN: 1353 *offp = mp->dmai_offset; 1354 *lenp = mp->dmai_size; 1355 return (DDI_SUCCESS); 1356 1357 case DDI_DMA_MOVWIN: { 1358 off_t off = *offp; 1359 ddi_dma_cookie_t *cp; 1360 px_dma_win_t *win_p = mp->dmai_winlst; 1361 1362 if (off >= mp->dmai_object.dmao_size) 1363 return (DDI_FAILURE); 1364 1365 /* locate window */ 1366 while (win_p->win_offset + win_p->win_size <= off) 1367 win_p = win_p->win_next; 1368 1369 cp = (ddi_dma_cookie_t *)(win_p + 1); 1370 mp->dmai_offset = win_p->win_offset; 1371 mp->dmai_size = win_p->win_size; 1372 mp->dmai_mapping = cp->dmac_laddress; /* cookie0 star addr */ 1373 mp->dmai_cookie = cp + 1; 1374 win_p->win_curseg = 0; 1375 1376 *(ddi_dma_cookie_t *)objp = *cp; 1377 *offp = win_p->win_offset; 1378 *lenp = win_p->win_size; 1379 DBG(DBG_DMA_CTL, dip, 1380 "HTOC: cookie - dmac_laddress=%p dmac_size=%x\n", 1381 cp->dmac_laddress, cp->dmac_size); 1382 } 1383 return (DDI_SUCCESS); 1384 1385 case DDI_DMA_NEXTWIN: { 1386 px_dma_win_t *win_p = *(px_dma_win_t **)offp; 1387 px_dma_win_t **nw_pp = (px_dma_win_t **)objp; 1388 ddi_dma_cookie_t *cp; 1389 if (!win_p) { 1390 *nw_pp = mp->dmai_winlst; 1391 return (DDI_SUCCESS); 1392 } 1393 1394 if (win_p->win_offset != mp->dmai_offset) 1395 return (DDI_DMA_STALE); 1396 if (!win_p->win_next) 1397 return (DDI_DMA_DONE); 1398 win_p = win_p->win_next; 1399 cp = (ddi_dma_cookie_t *)(win_p + 1); 1400 mp->dmai_offset = win_p->win_offset; 1401 mp->dmai_size = win_p->win_size; 1402 mp->dmai_mapping = cp->dmac_laddress; /* cookie0 star addr */ 1403 mp->dmai_cookie = cp + 1; 1404 win_p->win_curseg = 0; 1405 *nw_pp = win_p; 1406 } 1407 return (DDI_SUCCESS); 1408 1409 case DDI_DMA_NEXTSEG: { 1410 px_dma_win_t *w_p = *(px_dma_win_t **)offp; 1411 if (w_p->win_offset != mp->dmai_offset) 1412 return (DDI_DMA_STALE); 1413 if (w_p->win_curseg + 1 >= w_p->win_ncookies) 1414 return (DDI_DMA_DONE); 1415 w_p->win_curseg++; 1416 } 1417 *(ddi_dma_seg_t *)objp = (ddi_dma_seg_t)mp; 1418 return (DDI_SUCCESS); 1419 1420 case DDI_DMA_SEGTOC: { 1421 px_dma_win_t *win_p = mp->dmai_winlst; 1422 off_t off = mp->dmai_offset; 1423 ddi_dma_cookie_t *cp; 1424 int i; 1425 1426 /* locate active window */ 1427 for (; win_p->win_offset != off; win_p = win_p->win_next); 1428 cp = (ddi_dma_cookie_t *)(win_p + 1); 1429 for (i = 0; i < win_p->win_curseg; i++, cp++) 1430 off += cp->dmac_size; 1431 *offp = off; 1432 *lenp = cp->dmac_size; 1433 *(ddi_dma_cookie_t *)objp = *cp; /* copy cookie */ 1434 } 1435 return (DDI_SUCCESS); 1436 1437 case DDI_DMA_COFF: { 1438 px_dma_win_t *win_p; 1439 ddi_dma_cookie_t *cp; 1440 uint64_t addr, key = ((ddi_dma_cookie_t *)offp)->dmac_laddress; 1441 size_t win_off; 1442 1443 for (win_p = mp->dmai_winlst; win_p; win_p = win_p->win_next) { 1444 int i; 1445 win_off = 0; 1446 cp = (ddi_dma_cookie_t *)(win_p + 1); 1447 for (i = 0; i < win_p->win_ncookies; i++, cp++) { 1448 size_t sz = cp->dmac_size; 1449 1450 addr = cp->dmac_laddress; 1451 if ((addr <= key) && (addr + sz >= key)) 1452 goto found; 1453 win_off += sz; 1454 } 1455 } 1456 return (DDI_FAILURE); 1457 found: 1458 *objp = (caddr_t)(win_p->win_offset + win_off + (key - addr)); 1459 return (DDI_SUCCESS); 1460 } 1461 default: 1462 DBG(DBG_DMA_CTL, dip, "unknown command (%x): rdip=%s%d\n", 1463 cmd, ddi_driver_name(rdip), ddi_get_instance(rdip)); 1464 break; 1465 } 1466 return (DDI_FAILURE); 1467 } 1468 1469 static void 1470 px_dvma_debug_init(px_mmu_t *mmu_p) 1471 { 1472 size_t sz = sizeof (struct px_dvma_rec) * px_dvma_debug_rec; 1473 ASSERT(MUTEX_HELD(&mmu_p->dvma_debug_lock)); 1474 cmn_err(CE_NOTE, "PCI Express DVMA %p stat ON", mmu_p); 1475 1476 mmu_p->dvma_alloc_rec = kmem_alloc(sz, KM_SLEEP); 1477 mmu_p->dvma_free_rec = kmem_alloc(sz, KM_SLEEP); 1478 1479 mmu_p->dvma_active_list = NULL; 1480 mmu_p->dvma_alloc_rec_index = 0; 1481 mmu_p->dvma_free_rec_index = 0; 1482 mmu_p->dvma_active_count = 0; 1483 } 1484 1485 void 1486 px_dvma_debug_fini(px_mmu_t *mmu_p) 1487 { 1488 struct px_dvma_rec *prev, *ptr; 1489 size_t sz = sizeof (struct px_dvma_rec) * px_dvma_debug_rec; 1490 uint64_t mask = ~(1ull << mmu_p->mmu_inst); 1491 cmn_err(CE_NOTE, "PCI Express DVMA %p stat OFF", mmu_p); 1492 1493 kmem_free(mmu_p->dvma_alloc_rec, sz); 1494 kmem_free(mmu_p->dvma_free_rec, sz); 1495 mmu_p->dvma_alloc_rec = mmu_p->dvma_free_rec = NULL; 1496 1497 prev = mmu_p->dvma_active_list; 1498 if (!prev) 1499 return; 1500 for (ptr = prev->next; ptr; prev = ptr, ptr = ptr->next) 1501 kmem_free(prev, sizeof (struct px_dvma_rec)); 1502 kmem_free(prev, sizeof (struct px_dvma_rec)); 1503 1504 mmu_p->dvma_active_list = NULL; 1505 mmu_p->dvma_alloc_rec_index = 0; 1506 mmu_p->dvma_free_rec_index = 0; 1507 mmu_p->dvma_active_count = 0; 1508 1509 px_dvma_debug_off &= mask; 1510 px_dvma_debug_on &= mask; 1511 } 1512 1513 void 1514 px_dvma_alloc_debug(px_mmu_t *mmu_p, char *address, uint_t len, 1515 ddi_dma_impl_t *mp) 1516 { 1517 struct px_dvma_rec *ptr; 1518 mutex_enter(&mmu_p->dvma_debug_lock); 1519 1520 if (!mmu_p->dvma_alloc_rec) 1521 px_dvma_debug_init(mmu_p); 1522 if (PX_DVMA_DBG_OFF(mmu_p)) { 1523 px_dvma_debug_fini(mmu_p); 1524 goto done; 1525 } 1526 1527 ptr = &mmu_p->dvma_alloc_rec[mmu_p->dvma_alloc_rec_index]; 1528 ptr->dvma_addr = address; 1529 ptr->len = len; 1530 ptr->mp = mp; 1531 if (++mmu_p->dvma_alloc_rec_index == px_dvma_debug_rec) 1532 mmu_p->dvma_alloc_rec_index = 0; 1533 1534 ptr = kmem_alloc(sizeof (struct px_dvma_rec), KM_SLEEP); 1535 ptr->dvma_addr = address; 1536 ptr->len = len; 1537 ptr->mp = mp; 1538 1539 ptr->next = mmu_p->dvma_active_list; 1540 mmu_p->dvma_active_list = ptr; 1541 mmu_p->dvma_active_count++; 1542 done: 1543 mutex_exit(&mmu_p->dvma_debug_lock); 1544 } 1545 1546 void 1547 px_dvma_free_debug(px_mmu_t *mmu_p, char *address, uint_t len, 1548 ddi_dma_impl_t *mp) 1549 { 1550 struct px_dvma_rec *ptr, *ptr_save; 1551 mutex_enter(&mmu_p->dvma_debug_lock); 1552 1553 if (!mmu_p->dvma_alloc_rec) 1554 px_dvma_debug_init(mmu_p); 1555 if (PX_DVMA_DBG_OFF(mmu_p)) { 1556 px_dvma_debug_fini(mmu_p); 1557 goto done; 1558 } 1559 1560 ptr = &mmu_p->dvma_free_rec[mmu_p->dvma_free_rec_index]; 1561 ptr->dvma_addr = address; 1562 ptr->len = len; 1563 ptr->mp = mp; 1564 if (++mmu_p->dvma_free_rec_index == px_dvma_debug_rec) 1565 mmu_p->dvma_free_rec_index = 0; 1566 1567 ptr_save = mmu_p->dvma_active_list; 1568 for (ptr = ptr_save; ptr; ptr = ptr->next) { 1569 if ((ptr->dvma_addr == address) && (ptr->len = len)) 1570 break; 1571 ptr_save = ptr; 1572 } 1573 if (!ptr) { 1574 cmn_err(CE_WARN, "bad dvma free addr=%lx len=%x", 1575 (long)address, len); 1576 goto done; 1577 } 1578 if (ptr == mmu_p->dvma_active_list) 1579 mmu_p->dvma_active_list = ptr->next; 1580 else 1581 ptr_save->next = ptr->next; 1582 kmem_free(ptr, sizeof (struct px_dvma_rec)); 1583 mmu_p->dvma_active_count--; 1584 done: 1585 mutex_exit(&mmu_p->dvma_debug_lock); 1586 } 1587 1588 #ifdef DEBUG 1589 void 1590 px_dump_dma_handle(uint64_t flag, dev_info_t *dip, ddi_dma_impl_t *hp) 1591 { 1592 DBG(flag, dip, "mp(%p): flags=%x mapping=%lx xfer_size=%x\n", 1593 hp, hp->dmai_inuse, hp->dmai_mapping, hp->dmai_size); 1594 DBG(flag|DBG_CONT, dip, "\tnpages=%x roffset=%x rflags=%x nwin=%x\n", 1595 hp->dmai_ndvmapages, hp->dmai_roffset, hp->dmai_rflags, 1596 hp->dmai_nwin); 1597 DBG(flag|DBG_CONT, dip, "\twinsize=%x tte=%p pfnlst=%p pfn0=%p\n", 1598 hp->dmai_winsize, hp->dmai_tte, hp->dmai_pfnlst, hp->dmai_pfn0); 1599 DBG(flag|DBG_CONT, dip, "\twinlst=%x obj=%p attr=%p ckp=%p\n", 1600 hp->dmai_winlst, &hp->dmai_object, &hp->dmai_attr, 1601 hp->dmai_cookie); 1602 } 1603 #endif /* DEBUG */ 1604