1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 /* 30 * PCI Express nexus DVMA and DMA core routines: 31 * dma_map/dma_bind_handle implementation 32 * bypass and peer-to-peer support 33 * fast track DVMA space allocation 34 * runtime DVMA debug 35 */ 36 #include <sys/types.h> 37 #include <sys/kmem.h> 38 #include <sys/async.h> 39 #include <sys/sysmacros.h> 40 #include <sys/sunddi.h> 41 #include <sys/ddi_impldefs.h> 42 #include "px_obj.h" 43 44 /*LINTLIBRARY*/ 45 46 /* 47 * px_dma_allocmp - Allocate a pci dma implementation structure 48 * 49 * An extra ddi_dma_attr structure is bundled with the usual ddi_dma_impl 50 * to hold unmodified device limits. The ddi_dma_attr inside the 51 * ddi_dma_impl structure is augumented with system limits to enhance 52 * DVMA performance at runtime. The unaugumented device limits saved 53 * right after (accessed through (ddi_dma_attr_t *)(mp + 1)) is used 54 * strictly for peer-to-peer transfers which do not obey system limits. 55 * 56 * return: DDI_SUCCESS DDI_DMA_NORESOURCES 57 */ 58 ddi_dma_impl_t * 59 px_dma_allocmp(dev_info_t *dip, dev_info_t *rdip, int (*waitfp)(caddr_t), 60 caddr_t arg) 61 { 62 register ddi_dma_impl_t *mp; 63 int sleep = (waitfp == DDI_DMA_SLEEP) ? KM_SLEEP : KM_NOSLEEP; 64 65 /* Caution: we don't use zalloc to enhance performance! */ 66 if ((mp = kmem_alloc(sizeof (px_dma_hdl_t), sleep)) == 0) { 67 DBG(DBG_DMA_MAP, dip, "can't alloc dma_handle\n"); 68 if (waitfp != DDI_DMA_DONTWAIT) { 69 DBG(DBG_DMA_MAP, dip, "alloc_mp kmem cb\n"); 70 ddi_set_callback(waitfp, arg, &px_kmem_clid); 71 } 72 return (mp); 73 } 74 75 mp->dmai_rdip = rdip; 76 mp->dmai_flags = 0; 77 mp->dmai_pfnlst = NULL; 78 mp->dmai_winlst = NULL; 79 80 /* 81 * kmem_alloc debug: the following fields are not zero-ed 82 * mp->dmai_mapping = 0; 83 * mp->dmai_size = 0; 84 * mp->dmai_offset = 0; 85 * mp->dmai_minxfer = 0; 86 * mp->dmai_burstsizes = 0; 87 * mp->dmai_ndvmapages = 0; 88 * mp->dmai_pool/roffset = 0; 89 * mp->dmai_rflags = 0; 90 * mp->dmai_inuse/flags 91 * mp->dmai_nwin = 0; 92 * mp->dmai_winsize = 0; 93 * mp->dmai_nexus_private/tte = 0; 94 * mp->dmai_iopte/pfnlst 95 * mp->dmai_sbi/pfn0 = 0; 96 * mp->dmai_minfo/winlst/fdvma 97 * mp->dmai_rdip 98 * bzero(&mp->dmai_object, sizeof (ddi_dma_obj_t)); 99 * bzero(&mp->dmai_attr, sizeof (ddi_dma_attr_t)); 100 * mp->dmai_cookie = 0; 101 */ 102 103 mp->dmai_attr.dma_attr_version = (uint_t)DMA_ATTR_VERSION; 104 mp->dmai_attr.dma_attr_flags = (uint_t)0; 105 mp->dmai_fault = 0; 106 mp->dmai_fault_check = NULL; 107 mp->dmai_fault_notify = NULL; 108 109 mp->dmai_error.err_ena = 0; 110 mp->dmai_error.err_status = DDI_FM_OK; 111 mp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 112 mp->dmai_error.err_ontrap = NULL; 113 mp->dmai_error.err_fep = NULL; 114 115 return (mp); 116 } 117 118 void 119 px_dma_freemp(ddi_dma_impl_t *mp) 120 { 121 if (mp->dmai_ndvmapages > 1) 122 px_dma_freepfn(mp); 123 if (mp->dmai_winlst) 124 px_dma_freewin(mp); 125 kmem_free(mp, sizeof (px_dma_hdl_t)); 126 } 127 128 void 129 px_dma_freepfn(ddi_dma_impl_t *mp) 130 { 131 void *addr = mp->dmai_pfnlst; 132 if (addr) { 133 size_t npages = mp->dmai_ndvmapages; 134 if (npages > 1) 135 kmem_free(addr, npages * sizeof (px_iopfn_t)); 136 mp->dmai_pfnlst = NULL; 137 } 138 mp->dmai_ndvmapages = 0; 139 } 140 141 /* 142 * px_dma_lmts2hdl - alloate a ddi_dma_impl_t, validate practical limits 143 * and convert dmareq->dmar_limits to mp->dmai_attr 144 * 145 * ddi_dma_impl_t member modified input 146 * ------------------------------------------------------------------------ 147 * mp->dmai_minxfer - dev 148 * mp->dmai_burstsizes - dev 149 * mp->dmai_flags - no limit? peer-to-peer only? 150 * 151 * ddi_dma_attr member modified input 152 * ------------------------------------------------------------------------ 153 * mp->dmai_attr.dma_attr_addr_lo - dev lo, sys lo 154 * mp->dmai_attr.dma_attr_addr_hi - dev hi, sys hi 155 * mp->dmai_attr.dma_attr_count_max - dev count max, dev/sys lo/hi delta 156 * mp->dmai_attr.dma_attr_seg - 0 (no nocross restriction) 157 * mp->dmai_attr.dma_attr_align - 1 (no alignment restriction) 158 * 159 * The dlim_dmaspeed member of dmareq->dmar_limits is ignored. 160 */ 161 ddi_dma_impl_t * 162 px_dma_lmts2hdl(dev_info_t *dip, dev_info_t *rdip, px_mmu_t *mmu_p, 163 ddi_dma_req_t *dmareq) 164 { 165 ddi_dma_impl_t *mp; 166 ddi_dma_attr_t *attr_p; 167 uint64_t syslo = mmu_p->mmu_dvma_base; 168 uint64_t syshi = mmu_p->mmu_dvma_end; 169 uint64_t fasthi = mmu_p->mmu_dvma_fast_end; 170 ddi_dma_lim_t *lim_p = dmareq->dmar_limits; 171 uint32_t count_max = lim_p->dlim_cntr_max; 172 uint64_t lo = lim_p->dlim_addr_lo; 173 uint64_t hi = lim_p->dlim_addr_hi; 174 if (hi <= lo) { 175 DBG(DBG_DMA_MAP, dip, "Bad limits\n"); 176 return ((ddi_dma_impl_t *)DDI_DMA_NOMAPPING); 177 } 178 if (!count_max) 179 count_max--; 180 181 if (!(mp = px_dma_allocmp(dip, rdip, dmareq->dmar_fp, 182 dmareq->dmar_arg))) 183 return (NULL); 184 185 /* store original dev input at the 2nd ddi_dma_attr */ 186 attr_p = PX_DEV_ATTR(mp); 187 SET_DMAATTR(attr_p, lo, hi, -1, count_max); 188 SET_DMAALIGN(attr_p, 1); 189 190 lo = MAX(lo, syslo); 191 hi = MIN(hi, syshi); 192 if (hi <= lo) 193 mp->dmai_flags |= PX_DMAI_FLAGS_PEER_ONLY; 194 count_max = MIN(count_max, hi - lo); 195 196 if (PX_DEV_NOSYSLIMIT(lo, hi, syslo, fasthi, 1)) 197 mp->dmai_flags |= PX_DMAI_FLAGS_NOFASTLIMIT | 198 PX_DMAI_FLAGS_NOSYSLIMIT; 199 else { 200 if (PX_DEV_NOFASTLIMIT(lo, hi, syslo, syshi, 1)) 201 mp->dmai_flags |= PX_DMAI_FLAGS_NOFASTLIMIT; 202 } 203 if (PX_DMA_NOCTX(rdip)) 204 mp->dmai_flags |= PX_DMAI_FLAGS_NOCTX; 205 206 /* store augumented dev input to mp->dmai_attr */ 207 mp->dmai_minxfer = lim_p->dlim_minxfer; 208 mp->dmai_burstsizes = lim_p->dlim_burstsizes; 209 attr_p = &mp->dmai_attr; 210 SET_DMAATTR(attr_p, lo, hi, -1, count_max); 211 SET_DMAALIGN(attr_p, 1); 212 return (mp); 213 } 214 215 /* 216 * Called from px_attach to check for bypass dma support and set 217 * flags accordingly. 218 */ 219 int 220 px_dma_attach(px_t *px_p) 221 { 222 uint64_t baddr; 223 224 if (px_lib_iommu_getbypass(px_p->px_dip, 0ull, 225 PCI_MAP_ATTR_WRITE|PCI_MAP_ATTR_READ, 226 &baddr) != DDI_ENOTSUP) 227 /* ignore all other errors */ 228 px_p->px_soft_state |= PX_BYPASS_DMA_ALLOWED; 229 230 return (DDI_SUCCESS); 231 } 232 233 /* 234 * px_dma_attr2hdl 235 * 236 * This routine is called from the alloc handle entry point to sanity check the 237 * dma attribute structure. 238 * 239 * use by: px_dma_allochdl() 240 * 241 * return value: 242 * 243 * DDI_SUCCESS - on success 244 * DDI_DMA_BADATTR - attribute has invalid version number 245 * or address limits exclude dvma space 246 */ 247 int 248 px_dma_attr2hdl(px_t *px_p, ddi_dma_impl_t *mp) 249 { 250 px_mmu_t *mmu_p = px_p->px_mmu_p; 251 uint64_t syslo, syshi; 252 int ret; 253 ddi_dma_attr_t *attrp = PX_DEV_ATTR(mp); 254 uint64_t hi = attrp->dma_attr_addr_hi; 255 uint64_t lo = attrp->dma_attr_addr_lo; 256 uint64_t align = attrp->dma_attr_align; 257 uint64_t nocross = attrp->dma_attr_seg; 258 uint64_t count_max = attrp->dma_attr_count_max; 259 260 DBG(DBG_DMA_ALLOCH, px_p->px_dip, "attrp=%p cntr_max=%x.%08x\n", 261 attrp, HI32(count_max), LO32(count_max)); 262 DBG(DBG_DMA_ALLOCH, px_p->px_dip, "hi=%x.%08x lo=%x.%08x\n", 263 HI32(hi), LO32(hi), HI32(lo), LO32(lo)); 264 DBG(DBG_DMA_ALLOCH, px_p->px_dip, "seg=%x.%08x align=%x.%08x\n", 265 HI32(nocross), LO32(nocross), HI32(align), LO32(align)); 266 267 if (!nocross) 268 nocross--; 269 if (attrp->dma_attr_flags & DDI_DMA_FORCE_PHYSICAL) { /* BYPASS */ 270 271 DBG(DBG_DMA_ALLOCH, px_p->px_dip, "bypass mode\n"); 272 /* 273 * If Bypass DMA is not supported, return error so that 274 * target driver can fall back to dvma mode of operation 275 */ 276 if (!(px_p->px_soft_state & PX_BYPASS_DMA_ALLOWED)) 277 return (DDI_DMA_BADATTR); 278 mp->dmai_flags |= PX_DMAI_FLAGS_BYPASSREQ; 279 if (nocross != UINT64_MAX) 280 return (DDI_DMA_BADATTR); 281 if (align && (align > MMU_PAGE_SIZE)) 282 return (DDI_DMA_BADATTR); 283 align = 1; /* align on 1 page boundary */ 284 285 /* do a range check and get the limits */ 286 ret = px_lib_dma_bypass_rngchk(attrp, &syslo, &syshi); 287 if (ret != DDI_SUCCESS) 288 return (ret); 289 } else { /* MMU_XLATE or PEER_TO_PEER */ 290 align = MAX(align, MMU_PAGE_SIZE) - 1; 291 if ((align & nocross) != align) { 292 dev_info_t *rdip = mp->dmai_rdip; 293 cmn_err(CE_WARN, "%s%d dma_attr_seg not aligned", 294 NAMEINST(rdip)); 295 return (DDI_DMA_BADATTR); 296 } 297 align = MMU_BTOP(align + 1); 298 syslo = mmu_p->mmu_dvma_base; 299 syshi = mmu_p->mmu_dvma_end; 300 } 301 if (hi <= lo) { 302 dev_info_t *rdip = mp->dmai_rdip; 303 cmn_err(CE_WARN, "%s%d limits out of range", NAMEINST(rdip)); 304 return (DDI_DMA_BADATTR); 305 } 306 lo = MAX(lo, syslo); 307 hi = MIN(hi, syshi); 308 if (!count_max) 309 count_max--; 310 311 DBG(DBG_DMA_ALLOCH, px_p->px_dip, "hi=%x.%08x, lo=%x.%08x\n", 312 HI32(hi), LO32(hi), HI32(lo), LO32(lo)); 313 if (hi <= lo) { /* peer transfers cannot have alignment & nocross */ 314 dev_info_t *rdip = mp->dmai_rdip; 315 cmn_err(CE_WARN, "%s%d peer only dev %p", NAMEINST(rdip), mp); 316 if ((nocross < UINT32_MAX) || (align > 1)) { 317 cmn_err(CE_WARN, "%s%d peer only device bad attr", 318 NAMEINST(rdip)); 319 return (DDI_DMA_BADATTR); 320 } 321 mp->dmai_flags |= PX_DMAI_FLAGS_PEER_ONLY; 322 } else /* set practical counter_max value */ 323 count_max = MIN(count_max, hi - lo); 324 325 if (PX_DEV_NOSYSLIMIT(lo, hi, syslo, syshi, align)) 326 mp->dmai_flags |= PX_DMAI_FLAGS_NOSYSLIMIT | 327 PX_DMAI_FLAGS_NOFASTLIMIT; 328 else { 329 syshi = mmu_p->mmu_dvma_fast_end; 330 if (PX_DEV_NOFASTLIMIT(lo, hi, syslo, syshi, align)) 331 mp->dmai_flags |= PX_DMAI_FLAGS_NOFASTLIMIT; 332 } 333 if (PX_DMA_NOCTX(mp->dmai_rdip)) 334 mp->dmai_flags |= PX_DMAI_FLAGS_NOCTX; 335 336 mp->dmai_minxfer = attrp->dma_attr_minxfer; 337 mp->dmai_burstsizes = attrp->dma_attr_burstsizes; 338 attrp = &mp->dmai_attr; 339 SET_DMAATTR(attrp, lo, hi, nocross, count_max); 340 return (DDI_SUCCESS); 341 } 342 343 #define TGT_PFN_INBETWEEN(pfn, bgn, end) ((pfn >= bgn) && (pfn <= end)) 344 345 /* 346 * px_dma_type - determine which of the three types DMA (peer-to-peer, 347 * mmu bypass, or mmu translate) we are asked to do. 348 * Also checks pfn0 and rejects any non-peer-to-peer 349 * requests for peer-only devices. 350 * 351 * return values: 352 * DDI_DMA_NOMAPPING - can't get valid pfn0, or bad dma type 353 * DDI_SUCCESS 354 * 355 * dma handle members affected (set on exit): 356 * mp->dmai_object - dmareq->dmar_object 357 * mp->dmai_rflags - consistent?, nosync?, dmareq->dmar_flags 358 * mp->dmai_flags - DMA type 359 * mp->dmai_pfn0 - 1st page pfn (if va/size pair and not shadow) 360 * mp->dmai_roffset - initialized to starting MMU page offset 361 * mp->dmai_ndvmapages - # of total MMU pages of entire object 362 */ 363 int 364 px_dma_type(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 365 { 366 dev_info_t *dip = px_p->px_dip; 367 ddi_dma_obj_t *dobj_p = &dmareq->dmar_object; 368 px_pec_t *pec_p = px_p->px_pec_p; 369 uint32_t offset; 370 pfn_t pfn0; 371 372 mp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS | DMP_NOSYNC; 373 374 switch (dobj_p->dmao_type) { 375 case DMA_OTYP_BUFVADDR: 376 case DMA_OTYP_VADDR: { 377 page_t **pplist = dobj_p->dmao_obj.virt_obj.v_priv; 378 caddr_t vaddr = dobj_p->dmao_obj.virt_obj.v_addr; 379 380 DBG(DBG_DMA_MAP, dip, "vaddr=%p pplist=%p\n", vaddr, pplist); 381 offset = (ulong_t)vaddr & MMU_PAGE_OFFSET; 382 if (pplist) { /* shadow list */ 383 mp->dmai_flags |= PX_DMAI_FLAGS_PGPFN; 384 pfn0 = page_pptonum(*pplist); 385 } else { 386 struct as *as_p = dobj_p->dmao_obj.virt_obj.v_as; 387 struct hat *hat_p = as_p ? as_p->a_hat : kas.a_hat; 388 pfn0 = hat_getpfnum(hat_p, vaddr); 389 } 390 } 391 break; 392 393 case DMA_OTYP_PAGES: 394 offset = dobj_p->dmao_obj.pp_obj.pp_offset; 395 mp->dmai_flags |= PX_DMAI_FLAGS_PGPFN; 396 pfn0 = page_pptonum(dobj_p->dmao_obj.pp_obj.pp_pp); 397 break; 398 399 case DMA_OTYP_PADDR: 400 default: 401 cmn_err(CE_WARN, "%s%d requested unsupported dma type %x", 402 NAMEINST(mp->dmai_rdip), dobj_p->dmao_type); 403 return (DDI_DMA_NOMAPPING); 404 } 405 if (pfn0 == PFN_INVALID) { 406 cmn_err(CE_WARN, "%s%d: invalid pfn0 for DMA object %p", 407 NAMEINST(dip), dobj_p); 408 return (DDI_DMA_NOMAPPING); 409 } 410 if (TGT_PFN_INBETWEEN(pfn0, pec_p->pec_base32_pfn, 411 pec_p->pec_last32_pfn)) { 412 mp->dmai_flags |= PX_DMAI_FLAGS_PTP|PX_DMAI_FLAGS_PTP32; 413 goto done; /* leave bypass and dvma flag as 0 */ 414 } else if (TGT_PFN_INBETWEEN(pfn0, pec_p->pec_base64_pfn, 415 pec_p->pec_last64_pfn)) { 416 mp->dmai_flags |= PX_DMAI_FLAGS_PTP|PX_DMAI_FLAGS_PTP64; 417 goto done; /* leave bypass and dvma flag as 0 */ 418 } 419 if (PX_DMA_ISPEERONLY(mp)) { 420 dev_info_t *rdip = mp->dmai_rdip; 421 cmn_err(CE_WARN, "Bad peer-to-peer req %s%d", NAMEINST(rdip)); 422 return (DDI_DMA_NOMAPPING); 423 } 424 mp->dmai_flags |= (mp->dmai_flags & PX_DMAI_FLAGS_BYPASSREQ) ? 425 PX_DMAI_FLAGS_BYPASS : PX_DMAI_FLAGS_DVMA; 426 done: 427 mp->dmai_object = *dobj_p; /* whole object */ 428 mp->dmai_pfn0 = (void *)pfn0; /* cache pfn0 */ 429 mp->dmai_roffset = offset; /* win0 pg0 offset */ 430 mp->dmai_ndvmapages = MMU_BTOPR(offset + mp->dmai_object.dmao_size); 431 return (DDI_SUCCESS); 432 } 433 434 /* 435 * px_dma_pgpfn - set up pfnlst array according to pages 436 * VA/size pair: <shadow IO, bypass, peer-to-peer>, or OTYP_PAGES 437 */ 438 /*ARGSUSED*/ 439 static int 440 px_dma_pgpfn(px_t *px_p, ddi_dma_impl_t *mp, uint_t npages) 441 { 442 int i; 443 dev_info_t *dip = px_p->px_dip; 444 445 switch (mp->dmai_object.dmao_type) { 446 case DMA_OTYP_BUFVADDR: 447 case DMA_OTYP_VADDR: { 448 page_t **pplist = mp->dmai_object.dmao_obj.virt_obj.v_priv; 449 DBG(DBG_DMA_MAP, dip, "shadow pplist=%p, %x pages, pfns=", 450 pplist, npages); 451 for (i = 1; i < npages; i++) { 452 px_iopfn_t pfn = page_pptonum(pplist[i]); 453 PX_SET_MP_PFN1(mp, i, pfn); 454 DBG(DBG_DMA_MAP|DBG_CONT, dip, "%x ", pfn); 455 } 456 DBG(DBG_DMA_MAP|DBG_CONT, dip, "\n"); 457 } 458 break; 459 460 case DMA_OTYP_PAGES: { 461 page_t *pp = mp->dmai_object.dmao_obj.pp_obj.pp_pp->p_next; 462 DBG(DBG_DMA_MAP, dip, "pp=%p pfns=", pp); 463 for (i = 1; i < npages; i++, pp = pp->p_next) { 464 px_iopfn_t pfn = page_pptonum(pp); 465 PX_SET_MP_PFN1(mp, i, pfn); 466 DBG(DBG_DMA_MAP|DBG_CONT, dip, "%x ", pfn); 467 } 468 DBG(DBG_DMA_MAP|DBG_CONT, dip, "\n"); 469 } 470 break; 471 472 default: /* check is already done by px_dma_type */ 473 ASSERT(0); 474 break; 475 } 476 return (DDI_SUCCESS); 477 } 478 479 /* 480 * px_dma_vapfn - set up pfnlst array according to VA 481 * VA/size pair: <normal, bypass, peer-to-peer> 482 * pfn0 is skipped as it is already done. 483 * In this case, the cached pfn0 is used to fill pfnlst[0] 484 */ 485 static int 486 px_dma_vapfn(px_t *px_p, ddi_dma_impl_t *mp, uint_t npages) 487 { 488 dev_info_t *dip = px_p->px_dip; 489 int i; 490 caddr_t vaddr = (caddr_t)mp->dmai_object.dmao_obj.virt_obj.v_as; 491 struct hat *hat_p = vaddr ? ((struct as *)vaddr)->a_hat : kas.a_hat; 492 493 vaddr = mp->dmai_object.dmao_obj.virt_obj.v_addr + MMU_PAGE_SIZE; 494 for (i = 1; i < npages; i++, vaddr += MMU_PAGE_SIZE) { 495 px_iopfn_t pfn = hat_getpfnum(hat_p, vaddr); 496 if (pfn == PFN_INVALID) 497 goto err_badpfn; 498 PX_SET_MP_PFN1(mp, i, pfn); 499 DBG(DBG_DMA_BINDH, dip, "px_dma_vapfn: mp=%p pfnlst[%x]=%x\n", 500 mp, i, pfn); 501 } 502 return (DDI_SUCCESS); 503 err_badpfn: 504 cmn_err(CE_WARN, "%s%d: bad page frame vaddr=%p", NAMEINST(dip), vaddr); 505 return (DDI_DMA_NOMAPPING); 506 } 507 508 /* 509 * px_dma_pfn - Fills pfn list for all pages being DMA-ed. 510 * 511 * dependencies: 512 * mp->dmai_ndvmapages - set to total # of dma pages 513 * 514 * return value: 515 * DDI_SUCCESS 516 * DDI_DMA_NOMAPPING 517 */ 518 int 519 px_dma_pfn(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 520 { 521 uint32_t npages = mp->dmai_ndvmapages; 522 int (*waitfp)(caddr_t) = dmareq->dmar_fp; 523 int i, ret, peer = PX_DMA_ISPTP(mp); 524 int peer32 = PX_DMA_ISPTP32(mp); 525 dev_info_t *dip = px_p->px_dip; 526 527 px_pec_t *pec_p = px_p->px_pec_p; 528 px_iopfn_t pfn_base = peer32 ? pec_p->pec_base32_pfn : 529 pec_p->pec_base64_pfn; 530 px_iopfn_t pfn_last = peer32 ? pec_p->pec_last32_pfn : 531 pec_p->pec_last64_pfn; 532 px_iopfn_t pfn_adj = peer ? pfn_base : 0; 533 534 DBG(DBG_DMA_BINDH, dip, "px_dma_pfn: mp=%p pfn0=%x\n", 535 mp, PX_MP_PFN0(mp) - pfn_adj); 536 /* 1 page: no array alloc/fill, no mixed mode check */ 537 if (npages == 1) { 538 PX_SET_MP_PFN(mp, 0, PX_MP_PFN0(mp) - pfn_adj); 539 return (DDI_SUCCESS); 540 } 541 /* allocate pfn array */ 542 if (!(mp->dmai_pfnlst = kmem_alloc(npages * sizeof (px_iopfn_t), 543 waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP))) { 544 if (waitfp != DDI_DMA_DONTWAIT) 545 ddi_set_callback(waitfp, dmareq->dmar_arg, 546 &px_kmem_clid); 547 return (DDI_DMA_NORESOURCES); 548 } 549 /* fill pfn array */ 550 PX_SET_MP_PFN(mp, 0, PX_MP_PFN0(mp) - pfn_adj); /* pfnlst[0] */ 551 if ((ret = PX_DMA_ISPGPFN(mp) ? px_dma_pgpfn(px_p, mp, npages) : 552 px_dma_vapfn(px_p, mp, npages)) != DDI_SUCCESS) 553 goto err; 554 555 /* skip pfn0, check mixed mode and adjust peer to peer pfn */ 556 for (i = 1; i < npages; i++) { 557 px_iopfn_t pfn = PX_GET_MP_PFN1(mp, i); 558 if (peer ^ TGT_PFN_INBETWEEN(pfn, pfn_base, pfn_last)) { 559 cmn_err(CE_WARN, "%s%d mixed mode DMA %lx %lx", 560 NAMEINST(mp->dmai_rdip), PX_MP_PFN0(mp), pfn); 561 ret = DDI_DMA_NOMAPPING; /* mixed mode */ 562 goto err; 563 } 564 DBG(DBG_DMA_MAP, dip, 565 "px_dma_pfn: pfnlst[%x]=%x-%x\n", i, pfn, pfn_adj); 566 if (pfn_adj) 567 PX_SET_MP_PFN1(mp, i, pfn - pfn_adj); 568 } 569 return (DDI_SUCCESS); 570 err: 571 px_dma_freepfn(mp); 572 return (ret); 573 } 574 575 /* 576 * px_dvma_win() - trim requested DVMA size down to window size 577 * The 1st window starts from offset and ends at page-aligned boundary. 578 * From the 2nd window on, each window starts and ends at page-aligned 579 * boundary except the last window ends at wherever requested. 580 * 581 * accesses the following mp-> members: 582 * mp->dmai_attr.dma_attr_count_max 583 * mp->dmai_attr.dma_attr_seg 584 * mp->dmai_roffset - start offset of 1st window 585 * mp->dmai_rflags (redzone) 586 * mp->dmai_ndvmapages (for 1 page fast path) 587 * 588 * sets the following mp-> members: 589 * mp->dmai_size - xfer size, != winsize if 1st/last win (not fixed) 590 * mp->dmai_winsize - window size (no redzone), n * page size (fixed) 591 * mp->dmai_nwin - # of DMA windows of entire object (fixed) 592 * mp->dmai_rflags - remove partial flag if nwin == 1 (fixed) 593 * mp->dmai_winlst - NULL, window objects not used for DVMA (fixed) 594 * 595 * fixed - not changed across different DMA windows 596 */ 597 /*ARGSUSED*/ 598 int 599 px_dvma_win(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 600 { 601 uint32_t redzone_sz = HAS_REDZONE(mp) ? MMU_PAGE_SIZE : 0; 602 size_t obj_sz = mp->dmai_object.dmao_size; 603 size_t xfer_sz; 604 ulong_t pg_off; 605 606 if ((mp->dmai_ndvmapages == 1) && !redzone_sz) { 607 mp->dmai_rflags &= ~DDI_DMA_PARTIAL; 608 mp->dmai_size = obj_sz; 609 mp->dmai_winsize = MMU_PAGE_SIZE; 610 mp->dmai_nwin = 1; 611 goto done; 612 } 613 614 pg_off = mp->dmai_roffset; 615 xfer_sz = obj_sz + redzone_sz; 616 617 /* include redzone in nocross check */ { 618 uint64_t nocross = mp->dmai_attr.dma_attr_seg; 619 if (xfer_sz + pg_off - 1 > nocross) 620 xfer_sz = nocross - pg_off + 1; 621 if (redzone_sz && (xfer_sz <= redzone_sz)) { 622 DBG(DBG_DMA_MAP, px_p->px_dip, 623 "nocross too small: " 624 "%lx(%lx)+%lx+%lx < %llx\n", 625 xfer_sz, obj_sz, pg_off, redzone_sz, nocross); 626 return (DDI_DMA_TOOBIG); 627 } 628 } 629 xfer_sz -= redzone_sz; /* restore transfer size */ 630 /* check counter max */ { 631 uint32_t count_max = mp->dmai_attr.dma_attr_count_max; 632 if (xfer_sz - 1 > count_max) 633 xfer_sz = count_max + 1; 634 } 635 if (xfer_sz >= obj_sz) { 636 mp->dmai_rflags &= ~DDI_DMA_PARTIAL; 637 mp->dmai_size = xfer_sz; 638 mp->dmai_winsize = P2ROUNDUP(xfer_sz + pg_off, MMU_PAGE_SIZE); 639 mp->dmai_nwin = 1; 640 goto done; 641 } 642 if (!(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 643 DBG(DBG_DMA_MAP, px_p->px_dip, "too big: %lx+%lx+%lx > %lx\n", 644 obj_sz, pg_off, redzone_sz, xfer_sz); 645 return (DDI_DMA_TOOBIG); 646 } 647 648 xfer_sz = MMU_PTOB(MMU_BTOP(xfer_sz + pg_off)); /* page align */ 649 mp->dmai_size = xfer_sz - pg_off; /* 1st window xferrable size */ 650 mp->dmai_winsize = xfer_sz; /* redzone not in winsize */ 651 mp->dmai_nwin = (obj_sz + pg_off + xfer_sz - 1) / xfer_sz; 652 done: 653 mp->dmai_winlst = NULL; 654 px_dump_dma_handle(DBG_DMA_MAP, px_p->px_dip, mp); 655 return (DDI_SUCCESS); 656 } 657 658 /* 659 * fast track cache entry to mmu context, inserts 3 0 bits between 660 * upper 6-bits and lower 3-bits of the 9-bit cache entry 661 */ 662 #define MMU_FCE_TO_CTX(i) (((i) << 3) | ((i) & 0x7) | 0x38) 663 664 /* 665 * px_dvma_map_fast - attempts to map fast trackable DVMA 666 */ 667 /*ARGSUSED*/ 668 int 669 px_dvma_map_fast(px_mmu_t *mmu_p, ddi_dma_impl_t *mp) 670 { 671 uint_t clustsz = px_dvma_page_cache_clustsz; 672 uint_t entries = px_dvma_page_cache_entries; 673 io_attributes_t attr = PX_GET_TTE_ATTR(mp->dmai_rflags); 674 int i = mmu_p->mmu_dvma_addr_scan_start; 675 uint8_t *lock_addr = mmu_p->mmu_dvma_cache_locks + i; 676 px_dvma_addr_t dvma_pg; 677 size_t npages = MMU_BTOP(mp->dmai_winsize); 678 dev_info_t *px_dip = mmu_p->mmu_px_p->px_dip; 679 680 extern uint8_t ldstub(uint8_t *); 681 ASSERT(MMU_PTOB(npages) == mp->dmai_winsize); 682 ASSERT(npages + HAS_REDZONE(mp) <= clustsz); 683 684 for (; i < entries && ldstub(lock_addr); i++, lock_addr++); 685 if (i >= entries) { 686 lock_addr = mmu_p->mmu_dvma_cache_locks; 687 i = 0; 688 for (; i < entries && ldstub(lock_addr); i++, lock_addr++); 689 if (i >= entries) { 690 #ifdef PX_DMA_PROF 691 px_dvmaft_exhaust++; 692 #endif /* PX_DMA_PROF */ 693 return (DDI_DMA_NORESOURCES); 694 } 695 } 696 mmu_p->mmu_dvma_addr_scan_start = (i + 1) & (entries - 1); 697 698 i *= clustsz; 699 dvma_pg = mmu_p->dvma_base_pg + i; 700 701 if (px_lib_iommu_map(px_dip, PCI_TSBID(0, i), npages, attr, 702 (void *)mp, 0, MMU_MAP_MP) != DDI_SUCCESS) 703 return (DDI_FAILURE); 704 705 #ifdef PX_DMA_PROF 706 px_dvmaft_success++; 707 #endif 708 mp->dmai_mapping = mp->dmai_roffset | MMU_PTOB(dvma_pg); 709 mp->dmai_offset = 0; 710 mp->dmai_flags |= PX_DMAI_FLAGS_FASTTRACK; 711 PX_SAVE_MP_TTE(mp, attr); /* save TTE template for unmapping */ 712 if (PX_DVMA_DBG_ON(mmu_p)) 713 px_dvma_alloc_debug(mmu_p, (char *)mp->dmai_mapping, 714 mp->dmai_size, mp); 715 return (DDI_SUCCESS); 716 } 717 718 /* 719 * px_dvma_map: map non-fasttrack DMA 720 * Use quantum cache if single page DMA. 721 */ 722 int 723 px_dvma_map(ddi_dma_impl_t *mp, ddi_dma_req_t *dmareq, px_mmu_t *mmu_p) 724 { 725 uint_t npages = PX_DMA_WINNPGS(mp); 726 px_dvma_addr_t dvma_pg, dvma_pg_index; 727 void *dvma_addr; 728 uint64_t tte = PX_GET_TTE_ATTR(mp->dmai_rflags); 729 int sleep = dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP; 730 dev_info_t *dip = mp->dmai_rdip; 731 int ret = DDI_SUCCESS; 732 733 /* 734 * allocate dvma space resource and map in the first window. 735 * (vmem_t *vmp, size_t size, 736 * size_t align, size_t phase, size_t nocross, 737 * void *minaddr, void *maxaddr, int vmflag) 738 */ 739 if ((npages == 1) && PX_HAS_NOSYSLIMIT(mp)) { 740 dvma_addr = vmem_alloc(mmu_p->mmu_dvma_map, 741 MMU_PAGE_SIZE, sleep); 742 mp->dmai_flags |= PX_DMAI_FLAGS_VMEMCACHE; 743 #ifdef PX_DMA_PROF 744 px_dvma_vmem_alloc++; 745 #endif /* PX_DMA_PROF */ 746 } else { 747 dvma_addr = vmem_xalloc(mmu_p->mmu_dvma_map, 748 MMU_PTOB(npages + HAS_REDZONE(mp)), 749 MAX(mp->dmai_attr.dma_attr_align, MMU_PAGE_SIZE), 750 0, 751 mp->dmai_attr.dma_attr_seg + 1, 752 (void *)mp->dmai_attr.dma_attr_addr_lo, 753 (void *)(mp->dmai_attr.dma_attr_addr_hi + 1), 754 sleep); 755 #ifdef PX_DMA_PROF 756 px_dvma_vmem_xalloc++; 757 #endif /* PX_DMA_PROF */ 758 } 759 dvma_pg = MMU_BTOP((ulong_t)dvma_addr); 760 dvma_pg_index = dvma_pg - mmu_p->dvma_base_pg; 761 DBG(DBG_DMA_MAP, dip, "fallback dvma_pages: dvma_pg=%x index=%x\n", 762 dvma_pg, dvma_pg_index); 763 if (dvma_pg == 0) 764 goto noresource; 765 766 mp->dmai_mapping = mp->dmai_roffset | MMU_PTOB(dvma_pg); 767 mp->dmai_offset = 0; 768 PX_SAVE_MP_TTE(mp, tte); /* mp->dmai_tte = tte */ 769 770 if ((ret = px_mmu_map_pages(mmu_p, 771 mp, dvma_pg, npages, 0)) != DDI_SUCCESS) { 772 if (mp->dmai_flags & PX_DMAI_FLAGS_VMEMCACHE) { 773 vmem_free(mmu_p->mmu_dvma_map, (void *)dvma_addr, 774 MMU_PAGE_SIZE); 775 #ifdef PX_DMA_PROF 776 px_dvma_vmem_free++; 777 #endif /* PX_DMA_PROF */ 778 } else { 779 vmem_xfree(mmu_p->mmu_dvma_map, (void *)dvma_addr, 780 MMU_PTOB(npages + HAS_REDZONE(mp))); 781 #ifdef PX_DMA_PROF 782 px_dvma_vmem_xfree++; 783 #endif /* PX_DMA_PROF */ 784 } 785 } 786 787 return (ret); 788 noresource: 789 if (dmareq->dmar_fp != DDI_DMA_DONTWAIT) { 790 DBG(DBG_DMA_MAP, dip, "dvma_pg 0 - set callback\n"); 791 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg, 792 &mmu_p->mmu_dvma_clid); 793 } 794 DBG(DBG_DMA_MAP, dip, "vmem_xalloc - DDI_DMA_NORESOURCES\n"); 795 return (DDI_DMA_NORESOURCES); 796 } 797 798 void 799 px_dvma_unmap(px_mmu_t *mmu_p, ddi_dma_impl_t *mp) 800 { 801 px_dvma_addr_t dvma_addr = (px_dvma_addr_t)mp->dmai_mapping; 802 px_dvma_addr_t dvma_pg = MMU_BTOP(dvma_addr); 803 dvma_addr = MMU_PTOB(dvma_pg); 804 805 if (mp->dmai_flags & PX_DMAI_FLAGS_FASTTRACK) { 806 px_iopfn_t index = dvma_pg - mmu_p->dvma_base_pg; 807 ASSERT(index % px_dvma_page_cache_clustsz == 0); 808 index /= px_dvma_page_cache_clustsz; 809 ASSERT(index < px_dvma_page_cache_entries); 810 mmu_p->mmu_dvma_cache_locks[index] = 0; 811 #ifdef PX_DMA_PROF 812 px_dvmaft_free++; 813 #endif /* PX_DMA_PROF */ 814 return; 815 } 816 817 if (mp->dmai_flags & PX_DMAI_FLAGS_VMEMCACHE) { 818 vmem_free(mmu_p->mmu_dvma_map, (void *)dvma_addr, 819 MMU_PAGE_SIZE); 820 #ifdef PX_DMA_PROF 821 px_dvma_vmem_free++; 822 #endif /* PX_DMA_PROF */ 823 } else { 824 size_t npages = MMU_BTOP(mp->dmai_winsize) + HAS_REDZONE(mp); 825 vmem_xfree(mmu_p->mmu_dvma_map, (void *)dvma_addr, 826 MMU_PTOB(npages)); 827 #ifdef PX_DMA_PROF 828 px_dvma_vmem_xfree++; 829 #endif /* PX_DMA_PROF */ 830 } 831 } 832 833 /* 834 * DVMA mappings may have multiple windows, but each window always have 835 * one segment. 836 */ 837 int 838 px_dvma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp, 839 enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp, 840 uint_t cache_flags) 841 { 842 switch (cmd) { 843 case DDI_DMA_SYNC: 844 return (px_lib_dma_sync(dip, rdip, (ddi_dma_handle_t)mp, 845 *offp, *lenp, cache_flags)); 846 847 case DDI_DMA_HTOC: { 848 int ret; 849 off_t wo_off, off = *offp; /* wo_off: wnd's obj offset */ 850 uint_t win_size = mp->dmai_winsize; 851 ddi_dma_cookie_t *cp = (ddi_dma_cookie_t *)objp; 852 853 if (off >= mp->dmai_object.dmao_size) { 854 cmn_err(CE_WARN, "%s%d invalid dma_htoc offset %lx", 855 NAMEINST(mp->dmai_rdip), off); 856 return (DDI_FAILURE); 857 } 858 off += mp->dmai_roffset; 859 ret = px_dma_win(dip, rdip, (ddi_dma_handle_t)mp, 860 off / win_size, &wo_off, NULL, cp, NULL); /* lenp == NULL */ 861 if (ret) 862 return (ret); 863 DBG(DBG_DMA_CTL, dip, "HTOC:cookie=%x+%lx off=%lx,%lx\n", 864 cp->dmac_address, cp->dmac_size, off, *offp); 865 866 /* adjust cookie addr/len if we are not on window boundary */ 867 ASSERT((off % win_size) == (off - 868 (PX_DMA_CURWIN(mp) ? mp->dmai_roffset : 0) - wo_off)); 869 off = PX_DMA_CURWIN(mp) ? off % win_size : *offp; 870 ASSERT(cp->dmac_size > off); 871 cp->dmac_laddress += off; 872 cp->dmac_size -= off; 873 DBG(DBG_DMA_CTL, dip, "HTOC:mp=%p cookie=%x+%lx off=%lx,%lx\n", 874 mp, cp->dmac_address, cp->dmac_size, off, wo_off); 875 } 876 return (DDI_SUCCESS); 877 878 case DDI_DMA_REPWIN: 879 *offp = mp->dmai_offset; 880 *lenp = mp->dmai_size; 881 return (DDI_SUCCESS); 882 883 case DDI_DMA_MOVWIN: { 884 off_t off = *offp; 885 if (off >= mp->dmai_object.dmao_size) 886 return (DDI_FAILURE); 887 off += mp->dmai_roffset; 888 return (px_dma_win(dip, rdip, (ddi_dma_handle_t)mp, 889 off / mp->dmai_winsize, offp, lenp, 890 (ddi_dma_cookie_t *)objp, NULL)); 891 } 892 893 case DDI_DMA_NEXTWIN: { 894 px_window_t win = PX_DMA_CURWIN(mp); 895 if (offp) { 896 if (*(px_window_t *)offp != win) { 897 /* window not active */ 898 *(px_window_t *)objp = win; /* return cur win */ 899 return (DDI_DMA_STALE); 900 } 901 win++; 902 } else /* map win 0 */ 903 win = 0; 904 if (win >= mp->dmai_nwin) { 905 *(px_window_t *)objp = win - 1; 906 return (DDI_DMA_DONE); 907 } 908 if (px_dma_win(dip, rdip, (ddi_dma_handle_t)mp, 909 win, 0, 0, 0, 0)) { 910 *(px_window_t *)objp = win - 1; 911 return (DDI_FAILURE); 912 } 913 *(px_window_t *)objp = win; 914 } 915 return (DDI_SUCCESS); 916 917 case DDI_DMA_NEXTSEG: 918 if (*(px_window_t *)offp != PX_DMA_CURWIN(mp)) 919 return (DDI_DMA_STALE); 920 if (lenp) /* only 1 seg allowed */ 921 return (DDI_DMA_DONE); 922 923 /* return mp as seg 0 */ 924 *(ddi_dma_seg_t *)objp = (ddi_dma_seg_t)mp; 925 return (DDI_SUCCESS); 926 927 case DDI_DMA_SEGTOC: 928 MAKE_DMA_COOKIE((ddi_dma_cookie_t *)objp, mp->dmai_mapping, 929 mp->dmai_size); 930 *offp = mp->dmai_offset; 931 *lenp = mp->dmai_size; 932 return (DDI_SUCCESS); 933 934 case DDI_DMA_COFF: { 935 ddi_dma_cookie_t *cp = (ddi_dma_cookie_t *)offp; 936 if (cp->dmac_address < mp->dmai_mapping || 937 (cp->dmac_address + cp->dmac_size) > 938 (mp->dmai_mapping + mp->dmai_size)) 939 return (DDI_FAILURE); 940 *objp = (caddr_t)(cp->dmac_address - mp->dmai_mapping + 941 mp->dmai_offset); 942 } 943 return (DDI_SUCCESS); 944 default: 945 DBG(DBG_DMA_CTL, dip, "unknown command (%x): rdip=%s%d\n", 946 cmd, ddi_driver_name(rdip), ddi_get_instance(rdip)); 947 break; 948 } 949 return (DDI_FAILURE); 950 } 951 952 void 953 px_dma_freewin(ddi_dma_impl_t *mp) 954 { 955 px_dma_win_t *win_p = mp->dmai_winlst, *win2_p; 956 for (win2_p = win_p; win_p; win2_p = win_p) { 957 win_p = win2_p->win_next; 958 kmem_free(win2_p, sizeof (px_dma_win_t) + 959 sizeof (ddi_dma_cookie_t) * win2_p->win_ncookies); 960 } 961 mp->dmai_nwin = 0; 962 mp->dmai_winlst = NULL; 963 } 964 965 /* 966 * px_dma_newwin - create a dma window object and cookies 967 * 968 * After the initial scan in px_dma_physwin(), which identifies 969 * a portion of the pfn array that belongs to a dma window, 970 * we are called to allocate and initialize representing memory 971 * resources. We know from the 1st scan the number of cookies 972 * or dma segment in this window so we can allocate a contiguous 973 * memory array for the dma cookies (The implementation of 974 * ddi_dma_nextcookie(9f) dictates dma cookies be contiguous). 975 * 976 * A second round scan is done on the pfn array to identify 977 * each dma segment and initialize its corresponding dma cookie. 978 * We don't need to do all the safety checking and we know they 979 * all belong to the same dma window. 980 * 981 * Input: cookie_no - # of cookies identified by the 1st scan 982 * start_idx - subscript of the pfn array for the starting pfn 983 * end_idx - subscript of the last pfn in dma window 984 * win_pp - pointer to win_next member of previous window 985 * Return: DDI_SUCCESS - with **win_pp as newly created window object 986 * DDI_DMA_NORESROUCE - caller frees all previous window objs 987 * Note: Each cookie and window size are all initialized on page 988 * boundary. This is not true for the 1st cookie of the 1st 989 * window and the last cookie of the last window. 990 * We fix that later in upper layer which has access to size 991 * and offset info. 992 * 993 */ 994 /*ARGSUSED*/ 995 static int 996 px_dma_newwin(dev_info_t *dip, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, 997 uint32_t cookie_no, uint32_t start_idx, uint32_t end_idx, 998 px_dma_win_t **win_pp, uint64_t count_max, uint64_t bypass) 999 { 1000 int (*waitfp)(caddr_t) = dmareq->dmar_fp; 1001 ddi_dma_cookie_t *cookie_p; 1002 uint32_t pfn_no = 1; 1003 px_iopfn_t pfn = PX_GET_MP_PFN(mp, start_idx); 1004 px_iopfn_t prev_pfn = pfn; 1005 uint64_t baddr, seg_pfn0 = pfn; 1006 size_t sz = cookie_no * sizeof (ddi_dma_cookie_t); 1007 px_dma_win_t *win_p = kmem_zalloc(sizeof (px_dma_win_t) + sz, 1008 waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP); 1009 io_attributes_t attr = PX_GET_TTE_ATTR(mp->dmai_rflags); 1010 1011 if (!win_p) 1012 goto noresource; 1013 1014 win_p->win_next = NULL; 1015 win_p->win_ncookies = cookie_no; 1016 win_p->win_curseg = 0; /* start from segment 0 */ 1017 win_p->win_size = MMU_PTOB(end_idx - start_idx + 1); 1018 /* win_p->win_offset is left uninitialized */ 1019 1020 cookie_p = (ddi_dma_cookie_t *)(win_p + 1); 1021 start_idx++; 1022 for (; start_idx <= end_idx; start_idx++, prev_pfn = pfn, pfn_no++) { 1023 pfn = PX_GET_MP_PFN1(mp, start_idx); 1024 if ((pfn == prev_pfn + 1) && 1025 (MMU_PTOB(pfn_no + 1) - 1 <= count_max)) 1026 continue; 1027 1028 /* close up the cookie up to (including) prev_pfn */ 1029 baddr = MMU_PTOB(seg_pfn0); 1030 if (bypass && (px_lib_iommu_getbypass(dip, 1031 baddr, attr, &baddr) != DDI_SUCCESS)) 1032 return (DDI_FAILURE); 1033 1034 MAKE_DMA_COOKIE(cookie_p, baddr, MMU_PTOB(pfn_no)); 1035 DBG(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages)\n", 1036 MMU_PTOB(seg_pfn0), pfn_no); 1037 1038 cookie_p++; /* advance to next available cookie cell */ 1039 pfn_no = 0; 1040 seg_pfn0 = pfn; /* start a new segment from current pfn */ 1041 } 1042 1043 baddr = MMU_PTOB(seg_pfn0); 1044 if (bypass && (px_lib_iommu_getbypass(dip, 1045 baddr, attr, &baddr) != DDI_SUCCESS)) 1046 return (DDI_FAILURE); 1047 1048 MAKE_DMA_COOKIE(cookie_p, baddr, MMU_PTOB(pfn_no)); 1049 DBG(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages) of total %x\n", 1050 MMU_PTOB(seg_pfn0), pfn_no, cookie_no); 1051 #ifdef DEBUG 1052 cookie_p++; 1053 ASSERT((cookie_p - (ddi_dma_cookie_t *)(win_p + 1)) == cookie_no); 1054 #endif /* DEBUG */ 1055 *win_pp = win_p; 1056 return (DDI_SUCCESS); 1057 noresource: 1058 if (waitfp != DDI_DMA_DONTWAIT) 1059 ddi_set_callback(waitfp, dmareq->dmar_arg, &px_kmem_clid); 1060 return (DDI_DMA_NORESOURCES); 1061 } 1062 1063 /* 1064 * px_dma_adjust - adjust 1st and last cookie and window sizes 1065 * remove initial dma page offset from 1st cookie and window size 1066 * remove last dma page remainder from last cookie and window size 1067 * fill win_offset of each dma window according to just fixed up 1068 * each window sizes 1069 * px_dma_win_t members modified: 1070 * win_p->win_offset - this window's offset within entire DMA object 1071 * win_p->win_size - xferrable size (in bytes) for this window 1072 * 1073 * ddi_dma_impl_t members modified: 1074 * mp->dmai_size - 1st window xferrable size 1075 * mp->dmai_offset - 0, which is the dma offset of the 1st window 1076 * 1077 * ddi_dma_cookie_t members modified: 1078 * cookie_p->dmac_size - 1st and last cookie remove offset or remainder 1079 * cookie_p->dmac_laddress - 1st cookie add page offset 1080 */ 1081 static void 1082 px_dma_adjust(ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, px_dma_win_t *win_p) 1083 { 1084 ddi_dma_cookie_t *cookie_p = (ddi_dma_cookie_t *)(win_p + 1); 1085 size_t pg_offset = mp->dmai_roffset; 1086 size_t win_offset = 0; 1087 1088 cookie_p->dmac_size -= pg_offset; 1089 cookie_p->dmac_laddress |= pg_offset; 1090 win_p->win_size -= pg_offset; 1091 DBG(DBG_BYPASS, mp->dmai_rdip, "pg0 adjust %lx\n", pg_offset); 1092 1093 mp->dmai_size = win_p->win_size; 1094 mp->dmai_offset = 0; 1095 1096 pg_offset += mp->dmai_object.dmao_size; 1097 pg_offset &= MMU_PAGE_OFFSET; 1098 if (pg_offset) 1099 pg_offset = MMU_PAGE_SIZE - pg_offset; 1100 DBG(DBG_BYPASS, mp->dmai_rdip, "last pg adjust %lx\n", pg_offset); 1101 1102 for (; win_p->win_next; win_p = win_p->win_next) { 1103 DBG(DBG_BYPASS, mp->dmai_rdip, "win off %p\n", win_offset); 1104 win_p->win_offset = win_offset; 1105 win_offset += win_p->win_size; 1106 } 1107 /* last window */ 1108 win_p->win_offset = win_offset; 1109 cookie_p = (ddi_dma_cookie_t *)(win_p + 1); 1110 cookie_p[win_p->win_ncookies - 1].dmac_size -= pg_offset; 1111 win_p->win_size -= pg_offset; 1112 ASSERT((win_offset + win_p->win_size) == mp->dmai_object.dmao_size); 1113 } 1114 1115 /* 1116 * px_dma_physwin() - carve up dma windows using physical addresses. 1117 * Called to handle mmu bypass and pci peer-to-peer transfers. 1118 * Calls px_dma_newwin() to allocate window objects. 1119 * 1120 * Dependency: mp->dmai_pfnlst points to an array of pfns 1121 * 1122 * 1. Each dma window is represented by a px_dma_win_t object. 1123 * The object will be casted to ddi_dma_win_t and returned 1124 * to leaf driver through the DDI interface. 1125 * 2. Each dma window can have several dma segments with each 1126 * segment representing a physically contiguous either memory 1127 * space (if we are doing an mmu bypass transfer) or pci address 1128 * space (if we are doing a peer-to-peer transfer). 1129 * 3. Each segment has a DMA cookie to program the DMA engine. 1130 * The cookies within each DMA window must be located in a 1131 * contiguous array per ddi_dma_nextcookie(9f). 1132 * 4. The number of DMA segments within each DMA window cannot exceed 1133 * mp->dmai_attr.dma_attr_sgllen. If the transfer size is 1134 * too large to fit in the sgllen, the rest needs to be 1135 * relocated to the next dma window. 1136 * 5. Peer-to-peer DMA segment follows device hi, lo, count_max, 1137 * and nocross restrictions while bypass DMA follows the set of 1138 * restrictions with system limits factored in. 1139 * 1140 * Return: 1141 * mp->dmai_winlst - points to a link list of px_dma_win_t objects. 1142 * Each px_dma_win_t object on the link list contains 1143 * infomation such as its window size (# of pages), 1144 * starting offset (also see Restriction), an array of 1145 * DMA cookies, and # of cookies in the array. 1146 * mp->dmai_pfnlst - NULL, the pfn list is freed to conserve memory. 1147 * mp->dmai_nwin - # of total DMA windows on mp->dmai_winlst. 1148 * mp->dmai_mapping - starting cookie address 1149 * mp->dmai_rflags - consistent, nosync, no redzone 1150 * mp->dmai_cookie - start of cookie table of the 1st DMA window 1151 * 1152 * Restriction: 1153 * Each px_dma_win_t object can theoratically start from any offset 1154 * since the mmu is not involved. However, this implementation 1155 * always make windows start from page aligned offset (except 1156 * the 1st window, which follows the requested offset) due to the 1157 * fact that we are handed a pfn list. This does require device's 1158 * count_max and attr_seg to be at least MMU_PAGE_SIZE aligned. 1159 */ 1160 int 1161 px_dma_physwin(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 1162 { 1163 uint_t npages = mp->dmai_ndvmapages; 1164 int ret, sgllen = mp->dmai_attr.dma_attr_sgllen; 1165 px_iopfn_t pfn_lo, pfn_hi, prev_pfn; 1166 px_iopfn_t pfn = PX_GET_MP_PFN(mp, 0); 1167 uint32_t i, win_no = 0, pfn_no = 1, win_pfn0_index = 0, cookie_no = 0; 1168 uint64_t count_max, bypass_addr = 0; 1169 px_dma_win_t **win_pp = (px_dma_win_t **)&mp->dmai_winlst; 1170 ddi_dma_cookie_t *cookie0_p; 1171 io_attributes_t attr = PX_GET_TTE_ATTR(mp->dmai_rflags); 1172 dev_info_t *dip = px_p->px_dip; 1173 1174 ASSERT(PX_DMA_ISPTP(mp) || PX_DMA_ISBYPASS(mp)); 1175 if (PX_DMA_ISPTP(mp)) { /* ignore sys limits for peer-to-peer */ 1176 ddi_dma_attr_t *dev_attr_p = PX_DEV_ATTR(mp); 1177 uint64_t nocross = dev_attr_p->dma_attr_seg; 1178 px_pec_t *pec_p = px_p->px_pec_p; 1179 px_iopfn_t pfn_last = PX_DMA_ISPTP32(mp) ? 1180 pec_p->pec_last32_pfn - pec_p->pec_base32_pfn : 1181 pec_p->pec_last64_pfn - pec_p->pec_base64_pfn; 1182 1183 if (nocross && (nocross < UINT32_MAX)) 1184 return (DDI_DMA_NOMAPPING); 1185 if (dev_attr_p->dma_attr_align > MMU_PAGE_SIZE) 1186 return (DDI_DMA_NOMAPPING); 1187 pfn_lo = MMU_BTOP(dev_attr_p->dma_attr_addr_lo); 1188 pfn_hi = MMU_BTOP(dev_attr_p->dma_attr_addr_hi); 1189 pfn_hi = MIN(pfn_hi, pfn_last); 1190 if ((pfn_lo > pfn_hi) || (pfn < pfn_lo)) 1191 return (DDI_DMA_NOMAPPING); 1192 1193 count_max = dev_attr_p->dma_attr_count_max; 1194 count_max = MIN(count_max, nocross); 1195 /* 1196 * the following count_max trim is not done because we are 1197 * making sure pfn_lo <= pfn <= pfn_hi inside the loop 1198 * count_max=MIN(count_max, MMU_PTOB(pfn_hi - pfn_lo + 1)-1); 1199 */ 1200 } else { /* bypass hi/lo/count_max have been processed by attr2hdl() */ 1201 count_max = mp->dmai_attr.dma_attr_count_max; 1202 pfn_lo = MMU_BTOP(mp->dmai_attr.dma_attr_addr_lo); 1203 pfn_hi = MMU_BTOP(mp->dmai_attr.dma_attr_addr_hi); 1204 1205 if (px_lib_iommu_getbypass(dip, MMU_PTOB(pfn), 1206 attr, &bypass_addr) != DDI_SUCCESS) { 1207 cmn_err(CE_WARN, "bypass cookie failure %lx\n", pfn); 1208 return (DDI_DMA_NOMAPPING); 1209 } 1210 pfn = MMU_BTOP(bypass_addr); 1211 } 1212 1213 /* pfn: absolute (bypass mode) or relative (p2p mode) */ 1214 for (prev_pfn = pfn, i = 1; i < npages; 1215 i++, prev_pfn = pfn, pfn_no++) { 1216 pfn = PX_GET_MP_PFN1(mp, i); 1217 if (bypass_addr) { 1218 if (px_lib_iommu_getbypass(dip, MMU_PTOB(pfn), attr, 1219 &bypass_addr) != DDI_SUCCESS) { 1220 ret = DDI_DMA_NOMAPPING; 1221 goto err; 1222 } 1223 pfn = MMU_BTOP(bypass_addr); 1224 } 1225 if ((pfn == prev_pfn + 1) && 1226 (MMU_PTOB(pfn_no + 1) - 1 <= count_max)) 1227 continue; 1228 if ((pfn < pfn_lo) || (prev_pfn > pfn_hi)) { 1229 ret = DDI_DMA_NOMAPPING; 1230 goto err; 1231 } 1232 cookie_no++; 1233 pfn_no = 0; 1234 if (cookie_no < sgllen) 1235 continue; 1236 1237 DBG(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n", 1238 win_pfn0_index, i - 1, cookie_no); 1239 if (ret = px_dma_newwin(dip, dmareq, mp, cookie_no, 1240 win_pfn0_index, i - 1, win_pp, count_max, bypass_addr)) 1241 goto err; 1242 1243 win_pp = &(*win_pp)->win_next; /* win_pp = *(win_pp) */ 1244 win_no++; 1245 win_pfn0_index = i; 1246 cookie_no = 0; 1247 } 1248 if (pfn > pfn_hi) { 1249 ret = DDI_DMA_NOMAPPING; 1250 goto err; 1251 } 1252 cookie_no++; 1253 DBG(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n", 1254 win_pfn0_index, i - 1, cookie_no); 1255 if (ret = px_dma_newwin(dip, dmareq, mp, cookie_no, win_pfn0_index, 1256 i - 1, win_pp, count_max, bypass_addr)) 1257 goto err; 1258 win_no++; 1259 px_dma_adjust(dmareq, mp, mp->dmai_winlst); 1260 mp->dmai_nwin = win_no; 1261 mp->dmai_rflags |= DDI_DMA_CONSISTENT | DMP_NOSYNC; 1262 mp->dmai_rflags &= ~DDI_DMA_REDZONE; 1263 mp->dmai_flags |= PX_DMAI_FLAGS_NOSYNC; 1264 cookie0_p = (ddi_dma_cookie_t *)(PX_WINLST(mp) + 1); 1265 mp->dmai_cookie = PX_WINLST(mp)->win_ncookies > 1 ? cookie0_p + 1 : 0; 1266 mp->dmai_mapping = cookie0_p->dmac_laddress; 1267 1268 px_dma_freepfn(mp); 1269 return (DDI_DMA_MAPPED); 1270 err: 1271 px_dma_freewin(mp); 1272 return (ret); 1273 } 1274 1275 int 1276 px_dma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp, 1277 enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp, 1278 uint_t cache_flags) 1279 { 1280 switch (cmd) { 1281 case DDI_DMA_SYNC: 1282 return (DDI_SUCCESS); 1283 1284 case DDI_DMA_HTOC: { 1285 off_t off = *offp; 1286 ddi_dma_cookie_t *loop_cp, *cp; 1287 px_dma_win_t *win_p = mp->dmai_winlst; 1288 1289 if (off >= mp->dmai_object.dmao_size) 1290 return (DDI_FAILURE); 1291 1292 /* locate window */ 1293 while (win_p->win_offset + win_p->win_size <= off) 1294 win_p = win_p->win_next; 1295 1296 loop_cp = cp = (ddi_dma_cookie_t *)(win_p + 1); 1297 mp->dmai_offset = win_p->win_offset; 1298 mp->dmai_size = win_p->win_size; 1299 mp->dmai_mapping = cp->dmac_laddress; /* cookie0 start addr */ 1300 1301 /* adjust cookie addr/len if we are not on cookie boundary */ 1302 off -= win_p->win_offset; /* offset within window */ 1303 for (; off >= loop_cp->dmac_size; loop_cp++) 1304 off -= loop_cp->dmac_size; /* offset within cookie */ 1305 1306 mp->dmai_cookie = loop_cp + 1; 1307 win_p->win_curseg = loop_cp - cp; 1308 cp = (ddi_dma_cookie_t *)objp; 1309 MAKE_DMA_COOKIE(cp, loop_cp->dmac_laddress + off, 1310 loop_cp->dmac_size - off); 1311 1312 DBG(DBG_DMA_CTL, dip, 1313 "HTOC: cookie - dmac_laddress=%p dmac_size=%x\n", 1314 cp->dmac_laddress, cp->dmac_size); 1315 } 1316 return (DDI_SUCCESS); 1317 1318 case DDI_DMA_REPWIN: 1319 *offp = mp->dmai_offset; 1320 *lenp = mp->dmai_size; 1321 return (DDI_SUCCESS); 1322 1323 case DDI_DMA_MOVWIN: { 1324 off_t off = *offp; 1325 ddi_dma_cookie_t *cp; 1326 px_dma_win_t *win_p = mp->dmai_winlst; 1327 1328 if (off >= mp->dmai_object.dmao_size) 1329 return (DDI_FAILURE); 1330 1331 /* locate window */ 1332 while (win_p->win_offset + win_p->win_size <= off) 1333 win_p = win_p->win_next; 1334 1335 cp = (ddi_dma_cookie_t *)(win_p + 1); 1336 mp->dmai_offset = win_p->win_offset; 1337 mp->dmai_size = win_p->win_size; 1338 mp->dmai_mapping = cp->dmac_laddress; /* cookie0 star addr */ 1339 mp->dmai_cookie = cp + 1; 1340 win_p->win_curseg = 0; 1341 1342 *(ddi_dma_cookie_t *)objp = *cp; 1343 *offp = win_p->win_offset; 1344 *lenp = win_p->win_size; 1345 DBG(DBG_DMA_CTL, dip, 1346 "HTOC: cookie - dmac_laddress=%p dmac_size=%x\n", 1347 cp->dmac_laddress, cp->dmac_size); 1348 } 1349 return (DDI_SUCCESS); 1350 1351 case DDI_DMA_NEXTWIN: { 1352 px_dma_win_t *win_p = *(px_dma_win_t **)offp; 1353 px_dma_win_t **nw_pp = (px_dma_win_t **)objp; 1354 ddi_dma_cookie_t *cp; 1355 if (!win_p) { 1356 *nw_pp = mp->dmai_winlst; 1357 return (DDI_SUCCESS); 1358 } 1359 1360 if (win_p->win_offset != mp->dmai_offset) 1361 return (DDI_DMA_STALE); 1362 if (!win_p->win_next) 1363 return (DDI_DMA_DONE); 1364 win_p = win_p->win_next; 1365 cp = (ddi_dma_cookie_t *)(win_p + 1); 1366 mp->dmai_offset = win_p->win_offset; 1367 mp->dmai_size = win_p->win_size; 1368 mp->dmai_mapping = cp->dmac_laddress; /* cookie0 star addr */ 1369 mp->dmai_cookie = cp + 1; 1370 win_p->win_curseg = 0; 1371 *nw_pp = win_p; 1372 } 1373 return (DDI_SUCCESS); 1374 1375 case DDI_DMA_NEXTSEG: { 1376 px_dma_win_t *w_p = *(px_dma_win_t **)offp; 1377 if (w_p->win_offset != mp->dmai_offset) 1378 return (DDI_DMA_STALE); 1379 if (w_p->win_curseg + 1 >= w_p->win_ncookies) 1380 return (DDI_DMA_DONE); 1381 w_p->win_curseg++; 1382 } 1383 *(ddi_dma_seg_t *)objp = (ddi_dma_seg_t)mp; 1384 return (DDI_SUCCESS); 1385 1386 case DDI_DMA_SEGTOC: { 1387 px_dma_win_t *win_p = mp->dmai_winlst; 1388 off_t off = mp->dmai_offset; 1389 ddi_dma_cookie_t *cp; 1390 int i; 1391 1392 /* locate active window */ 1393 for (; win_p->win_offset != off; win_p = win_p->win_next); 1394 cp = (ddi_dma_cookie_t *)(win_p + 1); 1395 for (i = 0; i < win_p->win_curseg; i++, cp++) 1396 off += cp->dmac_size; 1397 *offp = off; 1398 *lenp = cp->dmac_size; 1399 *(ddi_dma_cookie_t *)objp = *cp; /* copy cookie */ 1400 } 1401 return (DDI_SUCCESS); 1402 1403 case DDI_DMA_COFF: { 1404 px_dma_win_t *win_p; 1405 ddi_dma_cookie_t *cp; 1406 uint64_t addr, key = ((ddi_dma_cookie_t *)offp)->dmac_laddress; 1407 size_t win_off; 1408 1409 for (win_p = mp->dmai_winlst; win_p; win_p = win_p->win_next) { 1410 int i; 1411 win_off = 0; 1412 cp = (ddi_dma_cookie_t *)(win_p + 1); 1413 for (i = 0; i < win_p->win_ncookies; i++, cp++) { 1414 size_t sz = cp->dmac_size; 1415 1416 addr = cp->dmac_laddress; 1417 if ((addr <= key) && (addr + sz >= key)) 1418 goto found; 1419 win_off += sz; 1420 } 1421 } 1422 return (DDI_FAILURE); 1423 found: 1424 *objp = (caddr_t)(win_p->win_offset + win_off + (key - addr)); 1425 return (DDI_SUCCESS); 1426 } 1427 default: 1428 DBG(DBG_DMA_CTL, dip, "unknown command (%x): rdip=%s%d\n", 1429 cmd, ddi_driver_name(rdip), ddi_get_instance(rdip)); 1430 break; 1431 } 1432 return (DDI_FAILURE); 1433 } 1434 1435 static void 1436 px_dvma_debug_init(px_mmu_t *mmu_p) 1437 { 1438 size_t sz = sizeof (struct px_dvma_rec) * px_dvma_debug_rec; 1439 ASSERT(MUTEX_HELD(&mmu_p->dvma_debug_lock)); 1440 cmn_err(CE_NOTE, "PCI Express DVMA %p stat ON", mmu_p); 1441 1442 mmu_p->dvma_alloc_rec = kmem_alloc(sz, KM_SLEEP); 1443 mmu_p->dvma_free_rec = kmem_alloc(sz, KM_SLEEP); 1444 1445 mmu_p->dvma_active_list = NULL; 1446 mmu_p->dvma_alloc_rec_index = 0; 1447 mmu_p->dvma_free_rec_index = 0; 1448 mmu_p->dvma_active_count = 0; 1449 } 1450 1451 void 1452 px_dvma_debug_fini(px_mmu_t *mmu_p) 1453 { 1454 struct px_dvma_rec *prev, *ptr; 1455 size_t sz = sizeof (struct px_dvma_rec) * px_dvma_debug_rec; 1456 uint64_t mask = ~(1ull << mmu_p->mmu_inst); 1457 cmn_err(CE_NOTE, "PCI Express DVMA %p stat OFF", mmu_p); 1458 1459 kmem_free(mmu_p->dvma_alloc_rec, sz); 1460 kmem_free(mmu_p->dvma_free_rec, sz); 1461 mmu_p->dvma_alloc_rec = mmu_p->dvma_free_rec = NULL; 1462 1463 prev = mmu_p->dvma_active_list; 1464 if (!prev) 1465 return; 1466 for (ptr = prev->next; ptr; prev = ptr, ptr = ptr->next) 1467 kmem_free(prev, sizeof (struct px_dvma_rec)); 1468 kmem_free(prev, sizeof (struct px_dvma_rec)); 1469 1470 mmu_p->dvma_active_list = NULL; 1471 mmu_p->dvma_alloc_rec_index = 0; 1472 mmu_p->dvma_free_rec_index = 0; 1473 mmu_p->dvma_active_count = 0; 1474 1475 px_dvma_debug_off &= mask; 1476 px_dvma_debug_on &= mask; 1477 } 1478 1479 void 1480 px_dvma_alloc_debug(px_mmu_t *mmu_p, char *address, uint_t len, 1481 ddi_dma_impl_t *mp) 1482 { 1483 struct px_dvma_rec *ptr; 1484 mutex_enter(&mmu_p->dvma_debug_lock); 1485 1486 if (!mmu_p->dvma_alloc_rec) 1487 px_dvma_debug_init(mmu_p); 1488 if (PX_DVMA_DBG_OFF(mmu_p)) { 1489 px_dvma_debug_fini(mmu_p); 1490 goto done; 1491 } 1492 1493 ptr = &mmu_p->dvma_alloc_rec[mmu_p->dvma_alloc_rec_index]; 1494 ptr->dvma_addr = address; 1495 ptr->len = len; 1496 ptr->mp = mp; 1497 if (++mmu_p->dvma_alloc_rec_index == px_dvma_debug_rec) 1498 mmu_p->dvma_alloc_rec_index = 0; 1499 1500 ptr = kmem_alloc(sizeof (struct px_dvma_rec), KM_SLEEP); 1501 ptr->dvma_addr = address; 1502 ptr->len = len; 1503 ptr->mp = mp; 1504 1505 ptr->next = mmu_p->dvma_active_list; 1506 mmu_p->dvma_active_list = ptr; 1507 mmu_p->dvma_active_count++; 1508 done: 1509 mutex_exit(&mmu_p->dvma_debug_lock); 1510 } 1511 1512 void 1513 px_dvma_free_debug(px_mmu_t *mmu_p, char *address, uint_t len, 1514 ddi_dma_impl_t *mp) 1515 { 1516 struct px_dvma_rec *ptr, *ptr_save; 1517 mutex_enter(&mmu_p->dvma_debug_lock); 1518 1519 if (!mmu_p->dvma_alloc_rec) 1520 px_dvma_debug_init(mmu_p); 1521 if (PX_DVMA_DBG_OFF(mmu_p)) { 1522 px_dvma_debug_fini(mmu_p); 1523 goto done; 1524 } 1525 1526 ptr = &mmu_p->dvma_free_rec[mmu_p->dvma_free_rec_index]; 1527 ptr->dvma_addr = address; 1528 ptr->len = len; 1529 ptr->mp = mp; 1530 if (++mmu_p->dvma_free_rec_index == px_dvma_debug_rec) 1531 mmu_p->dvma_free_rec_index = 0; 1532 1533 ptr_save = mmu_p->dvma_active_list; 1534 for (ptr = ptr_save; ptr; ptr = ptr->next) { 1535 if ((ptr->dvma_addr == address) && (ptr->len = len)) 1536 break; 1537 ptr_save = ptr; 1538 } 1539 if (!ptr) { 1540 cmn_err(CE_WARN, "bad dvma free addr=%lx len=%x", 1541 (long)address, len); 1542 goto done; 1543 } 1544 if (ptr == mmu_p->dvma_active_list) 1545 mmu_p->dvma_active_list = ptr->next; 1546 else 1547 ptr_save->next = ptr->next; 1548 kmem_free(ptr, sizeof (struct px_dvma_rec)); 1549 mmu_p->dvma_active_count--; 1550 done: 1551 mutex_exit(&mmu_p->dvma_debug_lock); 1552 } 1553 1554 #ifdef DEBUG 1555 void 1556 px_dump_dma_handle(uint64_t flag, dev_info_t *dip, ddi_dma_impl_t *hp) 1557 { 1558 DBG(flag, dip, "mp(%p): flags=%x mapping=%lx xfer_size=%x\n", 1559 hp, hp->dmai_inuse, hp->dmai_mapping, hp->dmai_size); 1560 DBG(flag|DBG_CONT, dip, "\tnpages=%x roffset=%x rflags=%x nwin=%x\n", 1561 hp->dmai_ndvmapages, hp->dmai_roffset, hp->dmai_rflags, 1562 hp->dmai_nwin); 1563 DBG(flag|DBG_CONT, dip, "\twinsize=%x tte=%p pfnlst=%p pfn0=%p\n", 1564 hp->dmai_winsize, hp->dmai_tte, hp->dmai_pfnlst, hp->dmai_pfn0); 1565 DBG(flag|DBG_CONT, dip, "\twinlst=%x obj=%p attr=%p ckp=%p\n", 1566 hp->dmai_winlst, &hp->dmai_object, &hp->dmai_attr, 1567 hp->dmai_cookie); 1568 } 1569 #endif /* DEBUG */ 1570