xref: /titanic_50/usr/src/uts/sun4/io/px/px_dma.c (revision 60c807700988885656502665e0cf8afd4b4346f7)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 /*
29  * PCI Express nexus DVMA and DMA core routines:
30  *	dma_map/dma_bind_handle implementation
31  *	bypass and peer-to-peer support
32  *	fast track DVMA space allocation
33  *	runtime DVMA debug
34  */
35 #include <sys/types.h>
36 #include <sys/kmem.h>
37 #include <sys/async.h>
38 #include <sys/sysmacros.h>
39 #include <sys/sunddi.h>
40 #include <sys/ddi_impldefs.h>
41 #include "px_obj.h"
42 
43 /*LINTLIBRARY*/
44 
45 /*
46  * px_dma_allocmp - Allocate a pci dma implementation structure
47  *
48  * An extra ddi_dma_attr structure is bundled with the usual ddi_dma_impl
49  * to hold unmodified device limits. The ddi_dma_attr inside the
50  * ddi_dma_impl structure is augumented with system limits to enhance
51  * DVMA performance at runtime. The unaugumented device limits saved
52  * right after (accessed through (ddi_dma_attr_t *)(mp + 1)) is used
53  * strictly for peer-to-peer transfers which do not obey system limits.
54  *
55  * return: DDI_SUCCESS DDI_DMA_NORESOURCES
56  */
57 ddi_dma_impl_t *
58 px_dma_allocmp(dev_info_t *dip, dev_info_t *rdip, int (*waitfp)(caddr_t),
59 	caddr_t arg)
60 {
61 	register ddi_dma_impl_t *mp;
62 	int sleep = (waitfp == DDI_DMA_SLEEP) ? KM_SLEEP : KM_NOSLEEP;
63 
64 	/* Caution: we don't use zalloc to enhance performance! */
65 	if ((mp = kmem_alloc(sizeof (px_dma_hdl_t), sleep)) == 0) {
66 		DBG(DBG_DMA_MAP, dip, "can't alloc dma_handle\n");
67 		if (waitfp != DDI_DMA_DONTWAIT) {
68 			DBG(DBG_DMA_MAP, dip, "alloc_mp kmem cb\n");
69 			ddi_set_callback(waitfp, arg, &px_kmem_clid);
70 		}
71 		return (mp);
72 	}
73 
74 	mp->dmai_rdip = rdip;
75 	mp->dmai_flags = 0;
76 	mp->dmai_pfnlst = NULL;
77 	mp->dmai_winlst = NULL;
78 
79 	/*
80 	 * kmem_alloc debug: the following fields are not zero-ed
81 	 * mp->dmai_mapping = 0;
82 	 * mp->dmai_size = 0;
83 	 * mp->dmai_offset = 0;
84 	 * mp->dmai_minxfer = 0;
85 	 * mp->dmai_burstsizes = 0;
86 	 * mp->dmai_ndvmapages = 0;
87 	 * mp->dmai_pool/roffset = 0;
88 	 * mp->dmai_rflags = 0;
89 	 * mp->dmai_inuse/flags
90 	 * mp->dmai_nwin = 0;
91 	 * mp->dmai_winsize = 0;
92 	 * mp->dmai_nexus_private/tte = 0;
93 	 * mp->dmai_iopte/pfnlst
94 	 * mp->dmai_sbi/pfn0 = 0;
95 	 * mp->dmai_minfo/winlst/fdvma
96 	 * mp->dmai_rdip
97 	 * bzero(&mp->dmai_object, sizeof (ddi_dma_obj_t));
98 	 * bzero(&mp->dmai_attr, sizeof (ddi_dma_attr_t));
99 	 * mp->dmai_cookie = 0;
100 	 */
101 
102 	mp->dmai_attr.dma_attr_version = (uint_t)DMA_ATTR_VERSION;
103 	mp->dmai_attr.dma_attr_flags = (uint_t)0;
104 	mp->dmai_fault = 0;
105 	mp->dmai_fault_check = NULL;
106 	mp->dmai_fault_notify = NULL;
107 
108 	mp->dmai_error.err_ena = 0;
109 	mp->dmai_error.err_status = DDI_FM_OK;
110 	mp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
111 	mp->dmai_error.err_ontrap = NULL;
112 	mp->dmai_error.err_fep = NULL;
113 	mp->dmai_error.err_cf = NULL;
114 
115 	if (px_child_prefetch(mp->dmai_rdip))
116 		mp->dmai_flags |= (PX_DMAI_FLAGS_MAP_BUFZONE |
117 		    PX_DMAI_FLAGS_REDZONE);
118 
119 	return (mp);
120 }
121 
122 void
123 px_dma_freemp(ddi_dma_impl_t *mp)
124 {
125 	if (mp->dmai_ndvmapages > 1)
126 		px_dma_freepfn(mp);
127 	if (mp->dmai_winlst)
128 		px_dma_freewin(mp);
129 	kmem_free(mp, sizeof (px_dma_hdl_t));
130 }
131 
132 void
133 px_dma_freepfn(ddi_dma_impl_t *mp)
134 {
135 	void *addr = mp->dmai_pfnlst;
136 	if (addr) {
137 		size_t npages = mp->dmai_ndvmapages;
138 		if (npages > 1)
139 			kmem_free(addr, npages * sizeof (px_iopfn_t));
140 		mp->dmai_pfnlst = NULL;
141 	}
142 	mp->dmai_ndvmapages = 0;
143 }
144 
145 /*
146  * px_dma_lmts2hdl - alloate a ddi_dma_impl_t, validate practical limits
147  *			and convert dmareq->dmar_limits to mp->dmai_attr
148  *
149  * ddi_dma_impl_t member modified     input
150  * ------------------------------------------------------------------------
151  * mp->dmai_minxfer		    - dev
152  * mp->dmai_burstsizes		    - dev
153  * mp->dmai_flags		    - no limit? peer-to-peer only?
154  *
155  * ddi_dma_attr member modified       input
156  * ------------------------------------------------------------------------
157  * mp->dmai_attr.dma_attr_addr_lo   - dev lo, sys lo
158  * mp->dmai_attr.dma_attr_addr_hi   - dev hi, sys hi
159  * mp->dmai_attr.dma_attr_count_max - dev count max, dev/sys lo/hi delta
160  * mp->dmai_attr.dma_attr_seg       - 0         (no nocross   restriction)
161  * mp->dmai_attr.dma_attr_align     - 1         (no alignment restriction)
162  *
163  * The dlim_dmaspeed member of dmareq->dmar_limits is ignored.
164  */
165 ddi_dma_impl_t *
166 px_dma_lmts2hdl(dev_info_t *dip, dev_info_t *rdip, px_mmu_t *mmu_p,
167 	ddi_dma_req_t *dmareq)
168 {
169 	ddi_dma_impl_t *mp;
170 	ddi_dma_attr_t *attr_p;
171 	uint64_t syslo		= mmu_p->mmu_dvma_base;
172 	uint64_t syshi		= mmu_p->mmu_dvma_end;
173 	uint64_t fasthi		= mmu_p->mmu_dvma_fast_end;
174 	ddi_dma_lim_t *lim_p	= dmareq->dmar_limits;
175 	uint32_t count_max	= lim_p->dlim_cntr_max;
176 	uint64_t lo		= lim_p->dlim_addr_lo;
177 	uint64_t hi		= lim_p->dlim_addr_hi;
178 	if (hi <= lo) {
179 		DBG(DBG_DMA_MAP, dip, "Bad limits\n");
180 		return ((ddi_dma_impl_t *)DDI_DMA_NOMAPPING);
181 	}
182 	if (!count_max)
183 		count_max--;
184 
185 	if (!(mp = px_dma_allocmp(dip, rdip, dmareq->dmar_fp,
186 		dmareq->dmar_arg)))
187 		return (NULL);
188 
189 	/* store original dev input at the 2nd ddi_dma_attr */
190 	attr_p = PX_DEV_ATTR(mp);
191 	SET_DMAATTR(attr_p, lo, hi, -1, count_max);
192 	SET_DMAALIGN(attr_p, 1);
193 
194 	lo = MAX(lo, syslo);
195 	hi = MIN(hi, syshi);
196 	if (hi <= lo)
197 		mp->dmai_flags |= PX_DMAI_FLAGS_PEER_ONLY;
198 	count_max = MIN(count_max, hi - lo);
199 
200 	if (PX_DEV_NOSYSLIMIT(lo, hi, syslo, fasthi, 1))
201 		mp->dmai_flags |= PX_DMAI_FLAGS_NOFASTLIMIT |
202 			PX_DMAI_FLAGS_NOSYSLIMIT;
203 	else {
204 		if (PX_DEV_NOFASTLIMIT(lo, hi, syslo, syshi, 1))
205 			mp->dmai_flags |= PX_DMAI_FLAGS_NOFASTLIMIT;
206 	}
207 	if (PX_DMA_NOCTX(rdip))
208 		mp->dmai_flags |= PX_DMAI_FLAGS_NOCTX;
209 
210 	/* store augumented dev input to mp->dmai_attr */
211 	mp->dmai_minxfer	= lim_p->dlim_minxfer;
212 	mp->dmai_burstsizes	= lim_p->dlim_burstsizes;
213 	attr_p = &mp->dmai_attr;
214 	SET_DMAATTR(attr_p, lo, hi, -1, count_max);
215 	SET_DMAALIGN(attr_p, 1);
216 	return (mp);
217 }
218 
219 /*
220  * Called from px_attach to check for bypass dma support and set
221  * flags accordingly.
222  */
223 int
224 px_dma_attach(px_t *px_p)
225 {
226 	uint64_t baddr;
227 
228 	if (px_lib_iommu_getbypass(px_p->px_dip, 0ull,
229 			PCI_MAP_ATTR_WRITE|PCI_MAP_ATTR_READ,
230 			&baddr) != DDI_ENOTSUP)
231 		/* ignore all other errors */
232 		px_p->px_dev_caps |= PX_BYPASS_DMA_ALLOWED;
233 
234 	return (DDI_SUCCESS);
235 }
236 
237 /*
238  * px_dma_attr2hdl
239  *
240  * This routine is called from the alloc handle entry point to sanity check the
241  * dma attribute structure.
242  *
243  * use by: px_dma_allochdl()
244  *
245  * return value:
246  *
247  *	DDI_SUCCESS		- on success
248  *	DDI_DMA_BADATTR		- attribute has invalid version number
249  *				  or address limits exclude dvma space
250  */
251 int
252 px_dma_attr2hdl(px_t *px_p, ddi_dma_impl_t *mp)
253 {
254 	px_mmu_t *mmu_p = px_p->px_mmu_p;
255 	uint64_t syslo, syshi;
256 	int	ret;
257 	ddi_dma_attr_t *attrp		= PX_DEV_ATTR(mp);
258 	uint64_t hi			= attrp->dma_attr_addr_hi;
259 	uint64_t lo			= attrp->dma_attr_addr_lo;
260 	uint64_t align			= attrp->dma_attr_align;
261 	uint64_t nocross		= attrp->dma_attr_seg;
262 	uint64_t count_max		= attrp->dma_attr_count_max;
263 
264 	DBG(DBG_DMA_ALLOCH, px_p->px_dip, "attrp=%p cntr_max=%x.%08x\n",
265 		attrp, HI32(count_max), LO32(count_max));
266 	DBG(DBG_DMA_ALLOCH, px_p->px_dip, "hi=%x.%08x lo=%x.%08x\n",
267 		HI32(hi), LO32(hi), HI32(lo), LO32(lo));
268 	DBG(DBG_DMA_ALLOCH, px_p->px_dip, "seg=%x.%08x align=%x.%08x\n",
269 		HI32(nocross), LO32(nocross), HI32(align), LO32(align));
270 
271 	if (!nocross)
272 		nocross--;
273 	if (attrp->dma_attr_flags & DDI_DMA_FORCE_PHYSICAL) { /* BYPASS */
274 
275 		DBG(DBG_DMA_ALLOCH, px_p->px_dip, "bypass mode\n");
276 		/*
277 		 * If Bypass DMA is not supported, return error so that
278 		 * target driver can fall back to dvma mode of operation
279 		 */
280 		if (!(px_p->px_dev_caps & PX_BYPASS_DMA_ALLOWED))
281 			return (DDI_DMA_BADATTR);
282 		mp->dmai_flags |= PX_DMAI_FLAGS_BYPASSREQ;
283 		if (nocross != UINT64_MAX)
284 			return (DDI_DMA_BADATTR);
285 		if (align && (align > MMU_PAGE_SIZE))
286 			return (DDI_DMA_BADATTR);
287 		align = 1; /* align on 1 page boundary */
288 
289 		/* do a range check and get the limits */
290 		ret = px_lib_dma_bypass_rngchk(px_p->px_dip, attrp,
291 				&syslo, &syshi);
292 		if (ret != DDI_SUCCESS)
293 			return (ret);
294 	} else { /* MMU_XLATE or PEER_TO_PEER */
295 		align = MAX(align, MMU_PAGE_SIZE) - 1;
296 		if ((align & nocross) != align) {
297 			dev_info_t *rdip = mp->dmai_rdip;
298 			cmn_err(CE_WARN, "%s%d dma_attr_seg not aligned",
299 				NAMEINST(rdip));
300 			return (DDI_DMA_BADATTR);
301 		}
302 		align = MMU_BTOP(align + 1);
303 		syslo = mmu_p->mmu_dvma_base;
304 		syshi = mmu_p->mmu_dvma_end;
305 	}
306 	if (hi <= lo) {
307 		dev_info_t *rdip = mp->dmai_rdip;
308 		cmn_err(CE_WARN, "%s%d limits out of range", NAMEINST(rdip));
309 		return (DDI_DMA_BADATTR);
310 	}
311 	lo = MAX(lo, syslo);
312 	hi = MIN(hi, syshi);
313 	if (!count_max)
314 		count_max--;
315 
316 	DBG(DBG_DMA_ALLOCH, px_p->px_dip, "hi=%x.%08x, lo=%x.%08x\n",
317 		HI32(hi), LO32(hi), HI32(lo), LO32(lo));
318 	if (hi <= lo) {
319 		/*
320 		 * If this is an IOMMU bypass access, the caller can't use
321 		 * the required addresses, so fail it.  Otherwise, it's
322 		 * peer-to-peer; ensure that the caller has no alignment or
323 		 * segment size restrictions.
324 		 */
325 		if ((mp->dmai_flags & PX_DMAI_FLAGS_BYPASSREQ) ||
326 		    (nocross < UINT32_MAX) || (align > 1))
327 			return (DDI_DMA_BADATTR);
328 
329 		mp->dmai_flags |= PX_DMAI_FLAGS_PEER_ONLY;
330 	} else /* set practical counter_max value */
331 		count_max = MIN(count_max, hi - lo);
332 
333 	if (PX_DEV_NOSYSLIMIT(lo, hi, syslo, syshi, align))
334 		mp->dmai_flags |= PX_DMAI_FLAGS_NOSYSLIMIT |
335 			PX_DMAI_FLAGS_NOFASTLIMIT;
336 	else {
337 		syshi = mmu_p->mmu_dvma_fast_end;
338 		if (PX_DEV_NOFASTLIMIT(lo, hi, syslo, syshi, align))
339 			mp->dmai_flags |= PX_DMAI_FLAGS_NOFASTLIMIT;
340 	}
341 	if (PX_DMA_NOCTX(mp->dmai_rdip))
342 		mp->dmai_flags |= PX_DMAI_FLAGS_NOCTX;
343 
344 	mp->dmai_minxfer	= attrp->dma_attr_minxfer;
345 	mp->dmai_burstsizes	= attrp->dma_attr_burstsizes;
346 	attrp = &mp->dmai_attr;
347 	SET_DMAATTR(attrp, lo, hi, nocross, count_max);
348 	return (DDI_SUCCESS);
349 }
350 
351 #define	TGT_PFN_INBETWEEN(pfn, bgn, end) ((pfn >= bgn) && (pfn <= end))
352 
353 /*
354  * px_dma_type - determine which of the three types DMA (peer-to-peer,
355  *		mmu bypass, or mmu translate) we are asked to do.
356  *		Also checks pfn0 and rejects any non-peer-to-peer
357  *		requests for peer-only devices.
358  *
359  *	return values:
360  *		DDI_DMA_NOMAPPING - can't get valid pfn0, or bad dma type
361  *		DDI_SUCCESS
362  *
363  *	dma handle members affected (set on exit):
364  *	mp->dmai_object		- dmareq->dmar_object
365  *	mp->dmai_rflags		- consistent?, nosync?, dmareq->dmar_flags
366  *	mp->dmai_flags   	- DMA type
367  *	mp->dmai_pfn0   	- 1st page pfn (if va/size pair and not shadow)
368  *	mp->dmai_roffset 	- initialized to starting MMU page offset
369  *	mp->dmai_ndvmapages	- # of total MMU pages of entire object
370  */
371 int
372 px_dma_type(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
373 {
374 	dev_info_t *dip = px_p->px_dip;
375 	ddi_dma_obj_t *dobj_p = &dmareq->dmar_object;
376 	px_pec_t *pec_p = px_p->px_pec_p;
377 	uint32_t offset;
378 	pfn_t pfn0;
379 
380 	mp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS | DMP_NOSYNC;
381 
382 	switch (dobj_p->dmao_type) {
383 	case DMA_OTYP_BUFVADDR:
384 	case DMA_OTYP_VADDR: {
385 		page_t **pplist = dobj_p->dmao_obj.virt_obj.v_priv;
386 		caddr_t vaddr = dobj_p->dmao_obj.virt_obj.v_addr;
387 
388 		DBG(DBG_DMA_MAP, dip, "vaddr=%p pplist=%p\n", vaddr, pplist);
389 		offset = (ulong_t)vaddr & MMU_PAGE_OFFSET;
390 		if (pplist) {				/* shadow list */
391 			mp->dmai_flags |= PX_DMAI_FLAGS_PGPFN;
392 			pfn0 = page_pptonum(*pplist);
393 		} else {
394 			struct as *as_p = dobj_p->dmao_obj.virt_obj.v_as;
395 			struct hat *hat_p = as_p ? as_p->a_hat : kas.a_hat;
396 			pfn0 = hat_getpfnum(hat_p, vaddr);
397 		}
398 		}
399 		break;
400 
401 	case DMA_OTYP_PAGES:
402 		offset = dobj_p->dmao_obj.pp_obj.pp_offset;
403 		mp->dmai_flags |= PX_DMAI_FLAGS_PGPFN;
404 		pfn0 = page_pptonum(dobj_p->dmao_obj.pp_obj.pp_pp);
405 		break;
406 
407 	case DMA_OTYP_PADDR:
408 	default:
409 		cmn_err(CE_WARN, "%s%d requested unsupported dma type %x",
410 			NAMEINST(mp->dmai_rdip), dobj_p->dmao_type);
411 		return (DDI_DMA_NOMAPPING);
412 	}
413 	if (pfn0 == PFN_INVALID) {
414 		cmn_err(CE_WARN, "%s%d: invalid pfn0 for DMA object %p",
415 			NAMEINST(dip), dobj_p);
416 		return (DDI_DMA_NOMAPPING);
417 	}
418 	if (TGT_PFN_INBETWEEN(pfn0, pec_p->pec_base32_pfn,
419 			pec_p->pec_last32_pfn)) {
420 		mp->dmai_flags |= PX_DMAI_FLAGS_PTP|PX_DMAI_FLAGS_PTP32;
421 		goto done;	/* leave bypass and dvma flag as 0 */
422 	} else if (TGT_PFN_INBETWEEN(pfn0, pec_p->pec_base64_pfn,
423 			pec_p->pec_last64_pfn)) {
424 		mp->dmai_flags |= PX_DMAI_FLAGS_PTP|PX_DMAI_FLAGS_PTP64;
425 		goto done;	/* leave bypass and dvma flag as 0 */
426 	}
427 	if (PX_DMA_ISPEERONLY(mp)) {
428 		dev_info_t *rdip = mp->dmai_rdip;
429 		cmn_err(CE_WARN, "Bad peer-to-peer req %s%d", NAMEINST(rdip));
430 		return (DDI_DMA_NOMAPPING);
431 	}
432 	mp->dmai_flags |= (mp->dmai_flags & PX_DMAI_FLAGS_BYPASSREQ) ?
433 	    PX_DMAI_FLAGS_BYPASS : PX_DMAI_FLAGS_DVMA |
434 	    (mp->dmai_rflags & DDI_DMA_REDZONE ? PX_DMAI_FLAGS_REDZONE : 0);
435 done:
436 	mp->dmai_object	 = *dobj_p;			/* whole object    */
437 	mp->dmai_pfn0	 = (void *)pfn0;		/* cache pfn0	   */
438 	mp->dmai_roffset = offset;			/* win0 pg0 offset */
439 	mp->dmai_ndvmapages = MMU_BTOPR(offset + mp->dmai_object.dmao_size);
440 	return (DDI_SUCCESS);
441 }
442 
443 /*
444  * px_dma_pgpfn - set up pfnlst array according to pages
445  *	VA/size pair: <shadow IO, bypass, peer-to-peer>, or OTYP_PAGES
446  */
447 /*ARGSUSED*/
448 static int
449 px_dma_pgpfn(px_t *px_p, ddi_dma_impl_t *mp, uint_t npages)
450 {
451 	int i;
452 	dev_info_t *dip = px_p->px_dip;
453 
454 	switch (mp->dmai_object.dmao_type) {
455 	case DMA_OTYP_BUFVADDR:
456 	case DMA_OTYP_VADDR: {
457 		page_t **pplist = mp->dmai_object.dmao_obj.virt_obj.v_priv;
458 		DBG(DBG_DMA_MAP, dip, "shadow pplist=%p, %x pages, pfns=",
459 			pplist, npages);
460 		for (i = 1; i < npages; i++) {
461 			px_iopfn_t pfn = page_pptonum(pplist[i]);
462 			PX_SET_MP_PFN1(mp, i, pfn);
463 			DBG(DBG_DMA_MAP|DBG_CONT, dip, "%x ", pfn);
464 		}
465 		DBG(DBG_DMA_MAP|DBG_CONT, dip, "\n");
466 		}
467 		break;
468 
469 	case DMA_OTYP_PAGES: {
470 		page_t *pp = mp->dmai_object.dmao_obj.pp_obj.pp_pp->p_next;
471 		DBG(DBG_DMA_MAP, dip, "pp=%p pfns=", pp);
472 		for (i = 1; i < npages; i++, pp = pp->p_next) {
473 			px_iopfn_t pfn = page_pptonum(pp);
474 			PX_SET_MP_PFN1(mp, i, pfn);
475 			DBG(DBG_DMA_MAP|DBG_CONT, dip, "%x ", pfn);
476 		}
477 		DBG(DBG_DMA_MAP|DBG_CONT, dip, "\n");
478 		}
479 		break;
480 
481 	default:	/* check is already done by px_dma_type */
482 		ASSERT(0);
483 		break;
484 	}
485 	return (DDI_SUCCESS);
486 }
487 
488 /*
489  * px_dma_vapfn - set up pfnlst array according to VA
490  *	VA/size pair: <normal, bypass, peer-to-peer>
491  *	pfn0 is skipped as it is already done.
492  *	In this case, the cached pfn0 is used to fill pfnlst[0]
493  */
494 static int
495 px_dma_vapfn(px_t *px_p, ddi_dma_impl_t *mp, uint_t npages)
496 {
497 	dev_info_t *dip = px_p->px_dip;
498 	int i;
499 	caddr_t vaddr = (caddr_t)mp->dmai_object.dmao_obj.virt_obj.v_as;
500 	struct hat *hat_p = vaddr ? ((struct as *)vaddr)->a_hat : kas.a_hat;
501 
502 	vaddr = mp->dmai_object.dmao_obj.virt_obj.v_addr + MMU_PAGE_SIZE;
503 	for (i = 1; i < npages; i++, vaddr += MMU_PAGE_SIZE) {
504 		px_iopfn_t pfn = hat_getpfnum(hat_p, vaddr);
505 		if (pfn == PFN_INVALID)
506 			goto err_badpfn;
507 		PX_SET_MP_PFN1(mp, i, pfn);
508 		DBG(DBG_DMA_BINDH, dip, "px_dma_vapfn: mp=%p pfnlst[%x]=%x\n",
509 			mp, i, pfn);
510 	}
511 	return (DDI_SUCCESS);
512 err_badpfn:
513 	cmn_err(CE_WARN, "%s%d: bad page frame vaddr=%p", NAMEINST(dip), vaddr);
514 	return (DDI_DMA_NOMAPPING);
515 }
516 
517 /*
518  * px_dma_pfn - Fills pfn list for all pages being DMA-ed.
519  *
520  * dependencies:
521  *	mp->dmai_ndvmapages	- set to total # of dma pages
522  *
523  * return value:
524  *	DDI_SUCCESS
525  *	DDI_DMA_NOMAPPING
526  */
527 int
528 px_dma_pfn(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
529 {
530 	uint32_t npages = mp->dmai_ndvmapages;
531 	int (*waitfp)(caddr_t) = dmareq->dmar_fp;
532 	int i, ret, peer = PX_DMA_ISPTP(mp);
533 	int peer32 = PX_DMA_ISPTP32(mp);
534 	dev_info_t *dip = px_p->px_dip;
535 
536 	px_pec_t *pec_p = px_p->px_pec_p;
537 	px_iopfn_t pfn_base = peer32 ? pec_p->pec_base32_pfn :
538 					pec_p->pec_base64_pfn;
539 	px_iopfn_t pfn_last = peer32 ? pec_p->pec_last32_pfn :
540 					pec_p->pec_last64_pfn;
541 	px_iopfn_t pfn_adj = peer ? pfn_base : 0;
542 
543 	DBG(DBG_DMA_BINDH, dip, "px_dma_pfn: mp=%p pfn0=%x\n",
544 		mp, PX_MP_PFN0(mp) - pfn_adj);
545 	/* 1 page: no array alloc/fill, no mixed mode check */
546 	if (npages == 1) {
547 		PX_SET_MP_PFN(mp, 0, PX_MP_PFN0(mp) - pfn_adj);
548 		return (DDI_SUCCESS);
549 	}
550 	/* allocate pfn array */
551 	if (!(mp->dmai_pfnlst = kmem_alloc(npages * sizeof (px_iopfn_t),
552 		waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP))) {
553 		if (waitfp != DDI_DMA_DONTWAIT)
554 			ddi_set_callback(waitfp, dmareq->dmar_arg,
555 				&px_kmem_clid);
556 		return (DDI_DMA_NORESOURCES);
557 	}
558 	/* fill pfn array */
559 	PX_SET_MP_PFN(mp, 0, PX_MP_PFN0(mp) - pfn_adj);	/* pfnlst[0] */
560 	if ((ret = PX_DMA_ISPGPFN(mp) ? px_dma_pgpfn(px_p, mp, npages) :
561 		px_dma_vapfn(px_p, mp, npages)) != DDI_SUCCESS)
562 		goto err;
563 
564 	/* skip pfn0, check mixed mode and adjust peer to peer pfn */
565 	for (i = 1; i < npages; i++) {
566 		px_iopfn_t pfn = PX_GET_MP_PFN1(mp, i);
567 		if (peer ^ TGT_PFN_INBETWEEN(pfn, pfn_base, pfn_last)) {
568 			cmn_err(CE_WARN, "%s%d mixed mode DMA %lx %lx",
569 				NAMEINST(mp->dmai_rdip), PX_MP_PFN0(mp), pfn);
570 			ret = DDI_DMA_NOMAPPING;	/* mixed mode */
571 			goto err;
572 		}
573 		DBG(DBG_DMA_MAP, dip,
574 			"px_dma_pfn: pfnlst[%x]=%x-%x\n", i, pfn, pfn_adj);
575 		if (pfn_adj)
576 			PX_SET_MP_PFN1(mp, i, pfn - pfn_adj);
577 	}
578 	return (DDI_SUCCESS);
579 err:
580 	px_dma_freepfn(mp);
581 	return (ret);
582 }
583 
584 /*
585  * px_dvma_win() - trim requested DVMA size down to window size
586  *	The 1st window starts from offset and ends at page-aligned boundary.
587  *	From the 2nd window on, each window starts and ends at page-aligned
588  *	boundary except the last window ends at wherever requested.
589  *
590  *	accesses the following mp-> members:
591  *	mp->dmai_attr.dma_attr_count_max
592  *	mp->dmai_attr.dma_attr_seg
593  *	mp->dmai_roffset   - start offset of 1st window
594  *	mp->dmai_rflags (redzone)
595  *	mp->dmai_ndvmapages (for 1 page fast path)
596  *
597  *	sets the following mp-> members:
598  *	mp->dmai_size	   - xfer size, != winsize if 1st/last win  (not fixed)
599  *	mp->dmai_winsize   - window size (no redzone), n * page size    (fixed)
600  *	mp->dmai_nwin	   - # of DMA windows of entire object		(fixed)
601  *	mp->dmai_rflags	   - remove partial flag if nwin == 1		(fixed)
602  *	mp->dmai_winlst	   - NULL, window objects not used for DVMA	(fixed)
603  *
604  *	fixed - not changed across different DMA windows
605  */
606 /*ARGSUSED*/
607 int
608 px_dvma_win(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
609 {
610 	uint32_t redzone_sz	= PX_HAS_REDZONE(mp) ? MMU_PAGE_SIZE : 0;
611 	size_t obj_sz		= mp->dmai_object.dmao_size;
612 	size_t xfer_sz;
613 	ulong_t pg_off;
614 
615 	if ((mp->dmai_ndvmapages == 1) && !redzone_sz) {
616 		mp->dmai_rflags &= ~DDI_DMA_PARTIAL;
617 		mp->dmai_size = obj_sz;
618 		mp->dmai_winsize = MMU_PAGE_SIZE;
619 		mp->dmai_nwin = 1;
620 		goto done;
621 	}
622 
623 	pg_off	= mp->dmai_roffset;
624 	xfer_sz	= obj_sz + redzone_sz;
625 
626 	/* include redzone in nocross check */ {
627 		uint64_t nocross = mp->dmai_attr.dma_attr_seg;
628 		if (xfer_sz + pg_off - 1 > nocross)
629 			xfer_sz = nocross - pg_off + 1;
630 		if (redzone_sz && (xfer_sz <= redzone_sz)) {
631 			DBG(DBG_DMA_MAP, px_p->px_dip,
632 			    "nocross too small: "
633 			    "%lx(%lx)+%lx+%lx < %llx\n",
634 			    xfer_sz, obj_sz, pg_off, redzone_sz, nocross);
635 			return (DDI_DMA_TOOBIG);
636 		}
637 	}
638 	xfer_sz -= redzone_sz;		/* restore transfer size  */
639 	/* check counter max */ {
640 		uint32_t count_max = mp->dmai_attr.dma_attr_count_max;
641 		if (xfer_sz - 1 > count_max)
642 			xfer_sz = count_max + 1;
643 	}
644 	if (xfer_sz >= obj_sz) {
645 		mp->dmai_rflags &= ~DDI_DMA_PARTIAL;
646 		mp->dmai_size = xfer_sz;
647 		mp->dmai_winsize = P2ROUNDUP(xfer_sz + pg_off, MMU_PAGE_SIZE);
648 		mp->dmai_nwin = 1;
649 		goto done;
650 	}
651 	if (!(dmareq->dmar_flags & DDI_DMA_PARTIAL)) {
652 		DBG(DBG_DMA_MAP, px_p->px_dip, "too big: %lx+%lx+%lx > %lx\n",
653 			obj_sz, pg_off, redzone_sz, xfer_sz);
654 		return (DDI_DMA_TOOBIG);
655 	}
656 
657 	xfer_sz = MMU_PTOB(MMU_BTOP(xfer_sz + pg_off)); /* page align */
658 	mp->dmai_size = xfer_sz - pg_off;	/* 1st window xferrable size */
659 	mp->dmai_winsize = xfer_sz;		/* redzone not in winsize */
660 	mp->dmai_nwin = (obj_sz + pg_off + xfer_sz - 1) / xfer_sz;
661 done:
662 	mp->dmai_winlst = NULL;
663 	px_dump_dma_handle(DBG_DMA_MAP, px_p->px_dip, mp);
664 	return (DDI_SUCCESS);
665 }
666 
667 /*
668  * fast track cache entry to mmu context, inserts 3 0 bits between
669  * upper 6-bits and lower 3-bits of the 9-bit cache entry
670  */
671 #define	MMU_FCE_TO_CTX(i)	(((i) << 3) | ((i) & 0x7) | 0x38)
672 
673 /*
674  * px_dvma_map_fast - attempts to map fast trackable DVMA
675  */
676 /*ARGSUSED*/
677 int
678 px_dvma_map_fast(px_mmu_t *mmu_p, ddi_dma_impl_t *mp)
679 {
680 	uint_t clustsz = px_dvma_page_cache_clustsz;
681 	uint_t entries = px_dvma_page_cache_entries;
682 	io_attributes_t attr = PX_GET_TTE_ATTR(mp->dmai_rflags,
683 	    mp->dmai_attr.dma_attr_flags);
684 	int i = mmu_p->mmu_dvma_addr_scan_start;
685 	uint8_t *lock_addr = mmu_p->mmu_dvma_cache_locks + i;
686 	px_dvma_addr_t dvma_pg;
687 	size_t npages = MMU_BTOP(mp->dmai_winsize);
688 	dev_info_t *dip = mmu_p->mmu_px_p->px_dip;
689 
690 	extern uint8_t ldstub(uint8_t *);
691 	ASSERT(MMU_PTOB(npages) == mp->dmai_winsize);
692 	ASSERT(npages + PX_HAS_REDZONE(mp) <= clustsz);
693 
694 	for (; i < entries && ldstub(lock_addr); i++, lock_addr++);
695 	if (i >= entries) {
696 		lock_addr = mmu_p->mmu_dvma_cache_locks;
697 		i = 0;
698 		for (; i < entries && ldstub(lock_addr); i++, lock_addr++);
699 		if (i >= entries) {
700 #ifdef	PX_DMA_PROF
701 			px_dvmaft_exhaust++;
702 #endif	/* PX_DMA_PROF */
703 			return (DDI_DMA_NORESOURCES);
704 		}
705 	}
706 	mmu_p->mmu_dvma_addr_scan_start = (i + 1) & (entries - 1);
707 
708 	i *= clustsz;
709 	dvma_pg = mmu_p->dvma_base_pg + i;
710 
711 	if (px_lib_iommu_map(dip, PCI_TSBID(0, i), npages, attr,
712 	    (void *)mp, 0, MMU_MAP_PFN) != DDI_SUCCESS) {
713 		DBG(DBG_MAP_WIN, dip, "px_dvma_map_fast: "
714 		    "px_lib_iommu_map failed\n");
715 
716 		return (DDI_FAILURE);
717 	}
718 
719 	if (!PX_MAP_BUFZONE(mp))
720 		goto done;
721 
722 	DBG(DBG_MAP_WIN, dip, "px_dvma_map_fast: redzone pg=%x\n", i + npages);
723 
724 	ASSERT(PX_HAS_REDZONE(mp));
725 
726 	if (px_lib_iommu_map(dip, PCI_TSBID(0, i + npages), 1, attr,
727 	    (void *)mp, npages - 1, MMU_MAP_PFN) != DDI_SUCCESS) {
728 		DBG(DBG_MAP_WIN, dip, "px_dvma_map_fast: "
729 		    "mapping REDZONE page failed\n");
730 
731 		(void) px_lib_iommu_demap(dip, PCI_TSBID(0, i), npages);
732 		return (DDI_FAILURE);
733 	}
734 
735 done:
736 #ifdef PX_DMA_PROF
737 	px_dvmaft_success++;
738 #endif
739 	mp->dmai_mapping = mp->dmai_roffset | MMU_PTOB(dvma_pg);
740 	mp->dmai_offset = 0;
741 	mp->dmai_flags |= PX_DMAI_FLAGS_FASTTRACK;
742 	PX_SAVE_MP_TTE(mp, attr);	/* save TTE template for unmapping */
743 	if (PX_DVMA_DBG_ON(mmu_p))
744 		px_dvma_alloc_debug(mmu_p, (char *)mp->dmai_mapping,
745 			mp->dmai_size, mp);
746 	return (DDI_SUCCESS);
747 }
748 
749 /*
750  * px_dvma_map: map non-fasttrack DMA
751  *		Use quantum cache if single page DMA.
752  */
753 int
754 px_dvma_map(ddi_dma_impl_t *mp, ddi_dma_req_t *dmareq, px_mmu_t *mmu_p)
755 {
756 	uint_t npages = PX_DMA_WINNPGS(mp);
757 	px_dvma_addr_t dvma_pg, dvma_pg_index;
758 	void *dvma_addr;
759 	uint64_t tte = PX_GET_TTE_ATTR(mp->dmai_rflags,
760 	    mp->dmai_attr.dma_attr_flags);
761 	int sleep = dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP;
762 	dev_info_t *dip = mp->dmai_rdip;
763 	int	ret = DDI_SUCCESS;
764 
765 	/*
766 	 * allocate dvma space resource and map in the first window.
767 	 * (vmem_t *vmp, size_t size,
768 	 *	size_t align, size_t phase, size_t nocross,
769 	 *	void *minaddr, void *maxaddr, int vmflag)
770 	 */
771 	if ((npages == 1) && !PX_HAS_REDZONE(mp) && PX_HAS_NOSYSLIMIT(mp)) {
772 		dvma_addr = vmem_alloc(mmu_p->mmu_dvma_map,
773 			MMU_PAGE_SIZE, sleep);
774 		mp->dmai_flags |= PX_DMAI_FLAGS_VMEMCACHE;
775 #ifdef	PX_DMA_PROF
776 		px_dvma_vmem_alloc++;
777 #endif	/* PX_DMA_PROF */
778 	} else {
779 		dvma_addr = vmem_xalloc(mmu_p->mmu_dvma_map,
780 			MMU_PTOB(npages + PX_HAS_REDZONE(mp)),
781 			MAX(mp->dmai_attr.dma_attr_align, MMU_PAGE_SIZE),
782 			0,
783 			mp->dmai_attr.dma_attr_seg + 1,
784 			(void *)mp->dmai_attr.dma_attr_addr_lo,
785 			(void *)(mp->dmai_attr.dma_attr_addr_hi + 1),
786 			sleep);
787 #ifdef	PX_DMA_PROF
788 		px_dvma_vmem_xalloc++;
789 #endif	/* PX_DMA_PROF */
790 	}
791 	dvma_pg = MMU_BTOP((ulong_t)dvma_addr);
792 	dvma_pg_index = dvma_pg - mmu_p->dvma_base_pg;
793 	DBG(DBG_DMA_MAP, dip, "fallback dvma_pages: dvma_pg=%x index=%x\n",
794 		dvma_pg, dvma_pg_index);
795 	if (dvma_pg == 0)
796 		goto noresource;
797 
798 	mp->dmai_mapping = mp->dmai_roffset | MMU_PTOB(dvma_pg);
799 	mp->dmai_offset = 0;
800 	PX_SAVE_MP_TTE(mp, tte);	/* mp->dmai_tte = tte */
801 
802 	if ((ret = px_mmu_map_pages(mmu_p,
803 	    mp, dvma_pg, npages, 0)) != DDI_SUCCESS) {
804 		if (mp->dmai_flags & PX_DMAI_FLAGS_VMEMCACHE) {
805 			vmem_free(mmu_p->mmu_dvma_map, (void *)dvma_addr,
806 			    MMU_PAGE_SIZE);
807 #ifdef PX_DMA_PROF
808 			px_dvma_vmem_free++;
809 #endif /* PX_DMA_PROF */
810 		} else {
811 			vmem_xfree(mmu_p->mmu_dvma_map, (void *)dvma_addr,
812 			    MMU_PTOB(npages + PX_HAS_REDZONE(mp)));
813 #ifdef PX_DMA_PROF
814 			px_dvma_vmem_xfree++;
815 #endif /* PX_DMA_PROF */
816 		}
817 	}
818 
819 	return (ret);
820 noresource:
821 	if (dmareq->dmar_fp != DDI_DMA_DONTWAIT) {
822 		DBG(DBG_DMA_MAP, dip, "dvma_pg 0 - set callback\n");
823 		ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg,
824 			&mmu_p->mmu_dvma_clid);
825 	}
826 	DBG(DBG_DMA_MAP, dip, "vmem_xalloc - DDI_DMA_NORESOURCES\n");
827 	return (DDI_DMA_NORESOURCES);
828 }
829 
830 void
831 px_dvma_unmap(px_mmu_t *mmu_p, ddi_dma_impl_t *mp)
832 {
833 	px_dvma_addr_t dvma_addr = (px_dvma_addr_t)mp->dmai_mapping;
834 	px_dvma_addr_t dvma_pg = MMU_BTOP(dvma_addr);
835 	dvma_addr = MMU_PTOB(dvma_pg);
836 
837 	if (mp->dmai_flags & PX_DMAI_FLAGS_FASTTRACK) {
838 		px_iopfn_t index = dvma_pg - mmu_p->dvma_base_pg;
839 		ASSERT(index % px_dvma_page_cache_clustsz == 0);
840 		index /= px_dvma_page_cache_clustsz;
841 		ASSERT(index < px_dvma_page_cache_entries);
842 		mmu_p->mmu_dvma_cache_locks[index] = 0;
843 #ifdef	PX_DMA_PROF
844 		px_dvmaft_free++;
845 #endif	/* PX_DMA_PROF */
846 		return;
847 	}
848 
849 	if (mp->dmai_flags & PX_DMAI_FLAGS_VMEMCACHE) {
850 		vmem_free(mmu_p->mmu_dvma_map, (void *)dvma_addr,
851 			MMU_PAGE_SIZE);
852 #ifdef PX_DMA_PROF
853 		px_dvma_vmem_free++;
854 #endif /* PX_DMA_PROF */
855 	} else {
856 		size_t npages = MMU_BTOP(mp->dmai_winsize) + PX_HAS_REDZONE(mp);
857 		vmem_xfree(mmu_p->mmu_dvma_map, (void *)dvma_addr,
858 			MMU_PTOB(npages));
859 #ifdef PX_DMA_PROF
860 		px_dvma_vmem_xfree++;
861 #endif /* PX_DMA_PROF */
862 	}
863 }
864 
865 /*
866  * DVMA mappings may have multiple windows, but each window always have
867  * one segment.
868  */
869 int
870 px_dvma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp,
871 	enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp,
872 	uint_t cache_flags)
873 {
874 	switch (cmd) {
875 	case DDI_DMA_SYNC:
876 		return (px_lib_dma_sync(dip, rdip, (ddi_dma_handle_t)mp,
877 		    *offp, *lenp, cache_flags));
878 
879 	case DDI_DMA_HTOC: {
880 		int ret;
881 		off_t wo_off, off = *offp;	/* wo_off: wnd's obj offset */
882 		uint_t win_size = mp->dmai_winsize;
883 		ddi_dma_cookie_t *cp = (ddi_dma_cookie_t *)objp;
884 
885 		if (off >= mp->dmai_object.dmao_size) {
886 			cmn_err(CE_WARN, "%s%d invalid dma_htoc offset %lx",
887 				NAMEINST(mp->dmai_rdip), off);
888 			return (DDI_FAILURE);
889 		}
890 		off += mp->dmai_roffset;
891 		ret = px_dma_win(dip, rdip, (ddi_dma_handle_t)mp,
892 		    off / win_size, &wo_off, NULL, cp, NULL); /* lenp == NULL */
893 		if (ret)
894 			return (ret);
895 		DBG(DBG_DMA_CTL, dip, "HTOC:cookie=%x+%lx off=%lx,%lx\n",
896 			cp->dmac_address, cp->dmac_size, off, *offp);
897 
898 		/* adjust cookie addr/len if we are not on window boundary */
899 		ASSERT((off % win_size) == (off -
900 			(PX_DMA_CURWIN(mp) ? mp->dmai_roffset : 0) - wo_off));
901 		off = PX_DMA_CURWIN(mp) ? off % win_size : *offp;
902 		ASSERT(cp->dmac_size > off);
903 		cp->dmac_laddress += off;
904 		cp->dmac_size -= off;
905 		DBG(DBG_DMA_CTL, dip, "HTOC:mp=%p cookie=%x+%lx off=%lx,%lx\n",
906 			mp, cp->dmac_address, cp->dmac_size, off, wo_off);
907 		}
908 		return (DDI_SUCCESS);
909 
910 	case DDI_DMA_REPWIN:
911 		*offp = mp->dmai_offset;
912 		*lenp = mp->dmai_size;
913 		return (DDI_SUCCESS);
914 
915 	case DDI_DMA_MOVWIN: {
916 		off_t off = *offp;
917 		if (off >= mp->dmai_object.dmao_size)
918 			return (DDI_FAILURE);
919 		off += mp->dmai_roffset;
920 		return (px_dma_win(dip, rdip, (ddi_dma_handle_t)mp,
921 		    off / mp->dmai_winsize, offp, lenp,
922 		    (ddi_dma_cookie_t *)objp, NULL));
923 		}
924 
925 	case DDI_DMA_NEXTWIN: {
926 		px_window_t win = PX_DMA_CURWIN(mp);
927 		if (offp) {
928 			if (*(px_window_t *)offp != win) {
929 				/* window not active */
930 				*(px_window_t *)objp = win; /* return cur win */
931 				return (DDI_DMA_STALE);
932 			}
933 			win++;
934 		} else	/* map win 0 */
935 			win = 0;
936 		if (win >= mp->dmai_nwin) {
937 			*(px_window_t *)objp = win - 1;
938 			return (DDI_DMA_DONE);
939 		}
940 		if (px_dma_win(dip, rdip, (ddi_dma_handle_t)mp,
941 		    win, 0, 0, 0, 0)) {
942 			*(px_window_t *)objp = win - 1;
943 			return (DDI_FAILURE);
944 		}
945 		*(px_window_t *)objp = win;
946 		}
947 		return (DDI_SUCCESS);
948 
949 	case DDI_DMA_NEXTSEG:
950 		if (*(px_window_t *)offp != PX_DMA_CURWIN(mp))
951 			return (DDI_DMA_STALE);
952 		if (lenp)				/* only 1 seg allowed */
953 			return (DDI_DMA_DONE);
954 
955 		/* return mp as seg 0 */
956 		*(ddi_dma_seg_t *)objp = (ddi_dma_seg_t)mp;
957 		return (DDI_SUCCESS);
958 
959 	case DDI_DMA_SEGTOC:
960 		MAKE_DMA_COOKIE((ddi_dma_cookie_t *)objp, mp->dmai_mapping,
961 			mp->dmai_size);
962 		*offp = mp->dmai_offset;
963 		*lenp = mp->dmai_size;
964 		return (DDI_SUCCESS);
965 
966 	case DDI_DMA_COFF: {
967 		ddi_dma_cookie_t *cp = (ddi_dma_cookie_t *)offp;
968 		if (cp->dmac_address < mp->dmai_mapping ||
969 			(cp->dmac_address + cp->dmac_size) >
970 			(mp->dmai_mapping + mp->dmai_size))
971 			return (DDI_FAILURE);
972 		*objp = (caddr_t)(cp->dmac_address - mp->dmai_mapping +
973 			mp->dmai_offset);
974 		}
975 		return (DDI_SUCCESS);
976 	default:
977 		DBG(DBG_DMA_CTL, dip, "unknown command (%x): rdip=%s%d\n",
978 			cmd, ddi_driver_name(rdip), ddi_get_instance(rdip));
979 		break;
980 	}
981 	return (DDI_FAILURE);
982 }
983 
984 void
985 px_dma_freewin(ddi_dma_impl_t *mp)
986 {
987 	px_dma_win_t *win_p = mp->dmai_winlst, *win2_p;
988 	for (win2_p = win_p; win_p; win2_p = win_p) {
989 		win_p = win2_p->win_next;
990 		kmem_free(win2_p, sizeof (px_dma_win_t) +
991 			sizeof (ddi_dma_cookie_t) * win2_p->win_ncookies);
992 	}
993 	mp->dmai_nwin = 0;
994 	mp->dmai_winlst = NULL;
995 }
996 
997 /*
998  * px_dma_newwin - create a dma window object and cookies
999  *
1000  *	After the initial scan in px_dma_physwin(), which identifies
1001  *	a portion of the pfn array that belongs to a dma window,
1002  *	we are called to allocate and initialize representing memory
1003  *	resources. We know from the 1st scan the number of cookies
1004  *	or dma segment in this window so we can allocate a contiguous
1005  *	memory array for the dma cookies (The implementation of
1006  *	ddi_dma_nextcookie(9f) dictates dma cookies be contiguous).
1007  *
1008  *	A second round scan is done on the pfn array to identify
1009  *	each dma segment and initialize its corresponding dma cookie.
1010  *	We don't need to do all the safety checking and we know they
1011  *	all belong to the same dma window.
1012  *
1013  *	Input:	cookie_no - # of cookies identified by the 1st scan
1014  *		start_idx - subscript of the pfn array for the starting pfn
1015  *		end_idx   - subscript of the last pfn in dma window
1016  *		win_pp    - pointer to win_next member of previous window
1017  *	Return:	DDI_SUCCESS - with **win_pp as newly created window object
1018  *		DDI_DMA_NORESROUCE - caller frees all previous window objs
1019  *	Note:	Each cookie and window size are all initialized on page
1020  *		boundary. This is not true for the 1st cookie of the 1st
1021  *		window and the last cookie of the last window.
1022  *		We fix that later in upper layer which has access to size
1023  *		and offset info.
1024  *
1025  */
1026 /*ARGSUSED*/
1027 static int
1028 px_dma_newwin(dev_info_t *dip, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp,
1029 	uint32_t cookie_no, uint32_t start_idx, uint32_t end_idx,
1030 	px_dma_win_t **win_pp, uint64_t count_max, uint64_t bypass)
1031 {
1032 	int (*waitfp)(caddr_t) = dmareq->dmar_fp;
1033 	ddi_dma_cookie_t *cookie_p;
1034 	uint32_t pfn_no = 1;
1035 	px_iopfn_t pfn = PX_GET_MP_PFN(mp, start_idx);
1036 	px_iopfn_t prev_pfn = pfn;
1037 	uint64_t baddr, seg_pfn0 = pfn;
1038 	size_t sz = cookie_no * sizeof (ddi_dma_cookie_t);
1039 	px_dma_win_t *win_p = kmem_zalloc(sizeof (px_dma_win_t) + sz,
1040 		waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP);
1041 	io_attributes_t	attr = PX_GET_TTE_ATTR(mp->dmai_rflags,
1042 	    mp->dmai_attr.dma_attr_flags);
1043 
1044 	if (!win_p)
1045 		goto noresource;
1046 
1047 	win_p->win_next = NULL;
1048 	win_p->win_ncookies = cookie_no;
1049 	win_p->win_curseg = 0;	/* start from segment 0 */
1050 	win_p->win_size = MMU_PTOB(end_idx - start_idx + 1);
1051 	/* win_p->win_offset is left uninitialized */
1052 
1053 	cookie_p = (ddi_dma_cookie_t *)(win_p + 1);
1054 	start_idx++;
1055 	for (; start_idx <= end_idx; start_idx++, prev_pfn = pfn, pfn_no++) {
1056 		pfn = PX_GET_MP_PFN1(mp, start_idx);
1057 		if ((pfn == prev_pfn + 1) &&
1058 			(MMU_PTOB(pfn_no + 1) - 1 <= count_max))
1059 			continue;
1060 
1061 		/* close up the cookie up to (including) prev_pfn */
1062 		baddr = MMU_PTOB(seg_pfn0);
1063 		if (bypass && (px_lib_iommu_getbypass(dip,
1064 				baddr, attr, &baddr) != DDI_SUCCESS))
1065 			return (DDI_FAILURE);
1066 
1067 		MAKE_DMA_COOKIE(cookie_p, baddr, MMU_PTOB(pfn_no));
1068 		DBG(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages)\n",
1069 			MMU_PTOB(seg_pfn0), pfn_no);
1070 
1071 		cookie_p++;	/* advance to next available cookie cell */
1072 		pfn_no = 0;
1073 		seg_pfn0 = pfn;	/* start a new segment from current pfn */
1074 	}
1075 
1076 	baddr = MMU_PTOB(seg_pfn0);
1077 	if (bypass && (px_lib_iommu_getbypass(dip,
1078 			baddr, attr, &baddr) != DDI_SUCCESS))
1079 		return (DDI_FAILURE);
1080 
1081 	MAKE_DMA_COOKIE(cookie_p, baddr, MMU_PTOB(pfn_no));
1082 	DBG(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages) of total %x\n",
1083 		MMU_PTOB(seg_pfn0), pfn_no, cookie_no);
1084 #ifdef	DEBUG
1085 	cookie_p++;
1086 	ASSERT((cookie_p - (ddi_dma_cookie_t *)(win_p + 1)) == cookie_no);
1087 #endif	/* DEBUG */
1088 	*win_pp = win_p;
1089 	return (DDI_SUCCESS);
1090 noresource:
1091 	if (waitfp != DDI_DMA_DONTWAIT)
1092 		ddi_set_callback(waitfp, dmareq->dmar_arg, &px_kmem_clid);
1093 	return (DDI_DMA_NORESOURCES);
1094 }
1095 
1096 /*
1097  * px_dma_adjust - adjust 1st and last cookie and window sizes
1098  *	remove initial dma page offset from 1st cookie and window size
1099  *	remove last dma page remainder from last cookie and window size
1100  *	fill win_offset of each dma window according to just fixed up
1101  *		each window sizes
1102  *	px_dma_win_t members modified:
1103  *	win_p->win_offset - this window's offset within entire DMA object
1104  *	win_p->win_size	  - xferrable size (in bytes) for this window
1105  *
1106  *	ddi_dma_impl_t members modified:
1107  *	mp->dmai_size	  - 1st window xferrable size
1108  *	mp->dmai_offset   - 0, which is the dma offset of the 1st window
1109  *
1110  *	ddi_dma_cookie_t members modified:
1111  *	cookie_p->dmac_size - 1st and last cookie remove offset or remainder
1112  *	cookie_p->dmac_laddress - 1st cookie add page offset
1113  */
1114 static void
1115 px_dma_adjust(ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, px_dma_win_t *win_p)
1116 {
1117 	ddi_dma_cookie_t *cookie_p = (ddi_dma_cookie_t *)(win_p + 1);
1118 	size_t pg_offset = mp->dmai_roffset;
1119 	size_t win_offset = 0;
1120 
1121 	cookie_p->dmac_size -= pg_offset;
1122 	cookie_p->dmac_laddress |= pg_offset;
1123 	win_p->win_size -= pg_offset;
1124 	DBG(DBG_BYPASS, mp->dmai_rdip, "pg0 adjust %lx\n", pg_offset);
1125 
1126 	mp->dmai_size = win_p->win_size;
1127 	mp->dmai_offset = 0;
1128 
1129 	pg_offset += mp->dmai_object.dmao_size;
1130 	pg_offset &= MMU_PAGE_OFFSET;
1131 	if (pg_offset)
1132 		pg_offset = MMU_PAGE_SIZE - pg_offset;
1133 	DBG(DBG_BYPASS, mp->dmai_rdip, "last pg adjust %lx\n", pg_offset);
1134 
1135 	for (; win_p->win_next; win_p = win_p->win_next) {
1136 		DBG(DBG_BYPASS, mp->dmai_rdip, "win off %p\n", win_offset);
1137 		win_p->win_offset = win_offset;
1138 		win_offset += win_p->win_size;
1139 	}
1140 	/* last window */
1141 	win_p->win_offset = win_offset;
1142 	cookie_p = (ddi_dma_cookie_t *)(win_p + 1);
1143 	cookie_p[win_p->win_ncookies - 1].dmac_size -= pg_offset;
1144 	win_p->win_size -= pg_offset;
1145 	ASSERT((win_offset + win_p->win_size) == mp->dmai_object.dmao_size);
1146 }
1147 
1148 /*
1149  * px_dma_physwin() - carve up dma windows using physical addresses.
1150  *	Called to handle mmu bypass and pci peer-to-peer transfers.
1151  *	Calls px_dma_newwin() to allocate window objects.
1152  *
1153  * Dependency: mp->dmai_pfnlst points to an array of pfns
1154  *
1155  * 1. Each dma window is represented by a px_dma_win_t object.
1156  *	The object will be casted to ddi_dma_win_t and returned
1157  *	to leaf driver through the DDI interface.
1158  * 2. Each dma window can have several dma segments with each
1159  *	segment representing a physically contiguous either memory
1160  *	space (if we are doing an mmu bypass transfer) or pci address
1161  *	space (if we are doing a peer-to-peer transfer).
1162  * 3. Each segment has a DMA cookie to program the DMA engine.
1163  *	The cookies within each DMA window must be located in a
1164  *	contiguous array per ddi_dma_nextcookie(9f).
1165  * 4. The number of DMA segments within each DMA window cannot exceed
1166  *	mp->dmai_attr.dma_attr_sgllen. If the transfer size is
1167  *	too large to fit in the sgllen, the rest needs to be
1168  *	relocated to the next dma window.
1169  * 5. Peer-to-peer DMA segment follows device hi, lo, count_max,
1170  *	and nocross restrictions while bypass DMA follows the set of
1171  *	restrictions with system limits factored in.
1172  *
1173  * Return:
1174  *	mp->dmai_winlst	 - points to a link list of px_dma_win_t objects.
1175  *		Each px_dma_win_t object on the link list contains
1176  *		infomation such as its window size (# of pages),
1177  *		starting offset (also see Restriction), an array of
1178  *		DMA cookies, and # of cookies in the array.
1179  *	mp->dmai_pfnlst	 - NULL, the pfn list is freed to conserve memory.
1180  *	mp->dmai_nwin	 - # of total DMA windows on mp->dmai_winlst.
1181  *	mp->dmai_mapping - starting cookie address
1182  *	mp->dmai_rflags	 - consistent, nosync, no redzone
1183  *	mp->dmai_cookie	 - start of cookie table of the 1st DMA window
1184  *
1185  * Restriction:
1186  *	Each px_dma_win_t object can theoratically start from any offset
1187  *	since the mmu is not involved. However, this implementation
1188  *	always make windows start from page aligned offset (except
1189  *	the 1st window, which follows the requested offset) due to the
1190  *	fact that we are handed a pfn list. This does require device's
1191  *	count_max and attr_seg to be at least MMU_PAGE_SIZE aligned.
1192  */
1193 int
1194 px_dma_physwin(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
1195 {
1196 	uint_t npages = mp->dmai_ndvmapages;
1197 	int ret, sgllen = mp->dmai_attr.dma_attr_sgllen;
1198 	px_iopfn_t pfn_lo, pfn_hi, prev_pfn;
1199 	px_iopfn_t pfn = PX_GET_MP_PFN(mp, 0);
1200 	uint32_t i, win_no = 0, pfn_no = 1, win_pfn0_index = 0, cookie_no = 0;
1201 	uint64_t count_max, bypass_addr = 0;
1202 	px_dma_win_t **win_pp = (px_dma_win_t **)&mp->dmai_winlst;
1203 	ddi_dma_cookie_t *cookie0_p;
1204 	io_attributes_t attr = PX_GET_TTE_ATTR(mp->dmai_rflags,
1205 	    mp->dmai_attr.dma_attr_flags);
1206 	dev_info_t *dip = px_p->px_dip;
1207 
1208 	ASSERT(PX_DMA_ISPTP(mp) || PX_DMA_ISBYPASS(mp));
1209 	if (PX_DMA_ISPTP(mp)) { /* ignore sys limits for peer-to-peer */
1210 		ddi_dma_attr_t *dev_attr_p = PX_DEV_ATTR(mp);
1211 		uint64_t nocross = dev_attr_p->dma_attr_seg;
1212 		px_pec_t *pec_p = px_p->px_pec_p;
1213 		px_iopfn_t pfn_last = PX_DMA_ISPTP32(mp) ?
1214 				pec_p->pec_last32_pfn - pec_p->pec_base32_pfn :
1215 				pec_p->pec_last64_pfn - pec_p->pec_base64_pfn;
1216 
1217 		if (nocross && (nocross < UINT32_MAX))
1218 			return (DDI_DMA_NOMAPPING);
1219 		if (dev_attr_p->dma_attr_align > MMU_PAGE_SIZE)
1220 			return (DDI_DMA_NOMAPPING);
1221 		pfn_lo = MMU_BTOP(dev_attr_p->dma_attr_addr_lo);
1222 		pfn_hi = MMU_BTOP(dev_attr_p->dma_attr_addr_hi);
1223 		pfn_hi = MIN(pfn_hi, pfn_last);
1224 		if ((pfn_lo > pfn_hi) || (pfn < pfn_lo))
1225 			return (DDI_DMA_NOMAPPING);
1226 
1227 		count_max = dev_attr_p->dma_attr_count_max;
1228 		count_max = MIN(count_max, nocross);
1229 		/*
1230 		 * the following count_max trim is not done because we are
1231 		 * making sure pfn_lo <= pfn <= pfn_hi inside the loop
1232 		 * count_max=MIN(count_max, MMU_PTOB(pfn_hi - pfn_lo + 1)-1);
1233 		 */
1234 	} else { /* bypass hi/lo/count_max have been processed by attr2hdl() */
1235 		count_max = mp->dmai_attr.dma_attr_count_max;
1236 		pfn_lo = MMU_BTOP(mp->dmai_attr.dma_attr_addr_lo);
1237 		pfn_hi = MMU_BTOP(mp->dmai_attr.dma_attr_addr_hi);
1238 
1239 		if (px_lib_iommu_getbypass(dip, MMU_PTOB(pfn),
1240 				attr, &bypass_addr) != DDI_SUCCESS) {
1241 			cmn_err(CE_WARN, "bypass cookie failure %lx\n", pfn);
1242 			return (DDI_DMA_NOMAPPING);
1243 		}
1244 		pfn = MMU_BTOP(bypass_addr);
1245 	}
1246 
1247 	/* pfn: absolute (bypass mode) or relative (p2p mode) */
1248 	for (prev_pfn = pfn, i = 1; i < npages;
1249 	    i++, prev_pfn = pfn, pfn_no++) {
1250 		pfn = PX_GET_MP_PFN1(mp, i);
1251 		if (bypass_addr) {
1252 			if (px_lib_iommu_getbypass(dip, MMU_PTOB(pfn), attr,
1253 					&bypass_addr) != DDI_SUCCESS) {
1254 				ret = DDI_DMA_NOMAPPING;
1255 				goto err;
1256 			}
1257 			pfn = MMU_BTOP(bypass_addr);
1258 		}
1259 		if ((pfn == prev_pfn + 1) &&
1260 				(MMU_PTOB(pfn_no + 1) - 1 <= count_max))
1261 			continue;
1262 		if ((pfn < pfn_lo) || (prev_pfn > pfn_hi)) {
1263 			ret = DDI_DMA_NOMAPPING;
1264 			goto err;
1265 		}
1266 		cookie_no++;
1267 		pfn_no = 0;
1268 		if (cookie_no < sgllen)
1269 			continue;
1270 
1271 		DBG(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n",
1272 			win_pfn0_index, i - 1, cookie_no);
1273 		if (ret = px_dma_newwin(dip, dmareq, mp, cookie_no,
1274 			win_pfn0_index, i - 1, win_pp, count_max, bypass_addr))
1275 			goto err;
1276 
1277 		win_pp = &(*win_pp)->win_next;	/* win_pp = *(win_pp) */
1278 		win_no++;
1279 		win_pfn0_index = i;
1280 		cookie_no = 0;
1281 	}
1282 	if (pfn > pfn_hi) {
1283 		ret = DDI_DMA_NOMAPPING;
1284 		goto err;
1285 	}
1286 	cookie_no++;
1287 	DBG(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n",
1288 		win_pfn0_index, i - 1, cookie_no);
1289 	if (ret = px_dma_newwin(dip, dmareq, mp, cookie_no, win_pfn0_index,
1290 		i - 1, win_pp, count_max, bypass_addr))
1291 		goto err;
1292 	win_no++;
1293 	px_dma_adjust(dmareq, mp, mp->dmai_winlst);
1294 	mp->dmai_nwin = win_no;
1295 	mp->dmai_rflags |= DDI_DMA_CONSISTENT | DMP_NOSYNC;
1296 	mp->dmai_rflags &= ~DDI_DMA_REDZONE;
1297 	mp->dmai_flags |= PX_DMAI_FLAGS_NOSYNC;
1298 	cookie0_p = (ddi_dma_cookie_t *)(PX_WINLST(mp) + 1);
1299 	mp->dmai_cookie = PX_WINLST(mp)->win_ncookies > 1 ? cookie0_p + 1 : 0;
1300 	mp->dmai_mapping = cookie0_p->dmac_laddress;
1301 
1302 	px_dma_freepfn(mp);
1303 	return (DDI_DMA_MAPPED);
1304 err:
1305 	px_dma_freewin(mp);
1306 	return (ret);
1307 }
1308 
1309 int
1310 px_dma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp,
1311 	enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp,
1312 	uint_t cache_flags)
1313 {
1314 	switch (cmd) {
1315 	case DDI_DMA_SYNC:
1316 		return (DDI_SUCCESS);
1317 
1318 	case DDI_DMA_HTOC: {
1319 		off_t off = *offp;
1320 		ddi_dma_cookie_t *loop_cp, *cp;
1321 		px_dma_win_t *win_p = mp->dmai_winlst;
1322 
1323 		if (off >= mp->dmai_object.dmao_size)
1324 			return (DDI_FAILURE);
1325 
1326 		/* locate window */
1327 		while (win_p->win_offset + win_p->win_size <= off)
1328 			win_p = win_p->win_next;
1329 
1330 		loop_cp = cp = (ddi_dma_cookie_t *)(win_p + 1);
1331 		mp->dmai_offset = win_p->win_offset;
1332 		mp->dmai_size   = win_p->win_size;
1333 		mp->dmai_mapping = cp->dmac_laddress; /* cookie0 start addr */
1334 
1335 		/* adjust cookie addr/len if we are not on cookie boundary */
1336 		off -= win_p->win_offset;	   /* offset within window */
1337 		for (; off >= loop_cp->dmac_size; loop_cp++)
1338 			off -= loop_cp->dmac_size; /* offset within cookie */
1339 
1340 		mp->dmai_cookie = loop_cp + 1;
1341 		win_p->win_curseg = loop_cp - cp;
1342 		cp = (ddi_dma_cookie_t *)objp;
1343 		MAKE_DMA_COOKIE(cp, loop_cp->dmac_laddress + off,
1344 			loop_cp->dmac_size - off);
1345 
1346 		DBG(DBG_DMA_CTL, dip,
1347 			"HTOC: cookie - dmac_laddress=%p dmac_size=%x\n",
1348 			cp->dmac_laddress, cp->dmac_size);
1349 		}
1350 		return (DDI_SUCCESS);
1351 
1352 	case DDI_DMA_REPWIN:
1353 		*offp = mp->dmai_offset;
1354 		*lenp = mp->dmai_size;
1355 		return (DDI_SUCCESS);
1356 
1357 	case DDI_DMA_MOVWIN: {
1358 		off_t off = *offp;
1359 		ddi_dma_cookie_t *cp;
1360 		px_dma_win_t *win_p = mp->dmai_winlst;
1361 
1362 		if (off >= mp->dmai_object.dmao_size)
1363 			return (DDI_FAILURE);
1364 
1365 		/* locate window */
1366 		while (win_p->win_offset + win_p->win_size <= off)
1367 			win_p = win_p->win_next;
1368 
1369 		cp = (ddi_dma_cookie_t *)(win_p + 1);
1370 		mp->dmai_offset = win_p->win_offset;
1371 		mp->dmai_size   = win_p->win_size;
1372 		mp->dmai_mapping = cp->dmac_laddress;	/* cookie0 star addr */
1373 		mp->dmai_cookie = cp + 1;
1374 		win_p->win_curseg = 0;
1375 
1376 		*(ddi_dma_cookie_t *)objp = *cp;
1377 		*offp = win_p->win_offset;
1378 		*lenp = win_p->win_size;
1379 		DBG(DBG_DMA_CTL, dip,
1380 			"HTOC: cookie - dmac_laddress=%p dmac_size=%x\n",
1381 			cp->dmac_laddress, cp->dmac_size);
1382 		}
1383 		return (DDI_SUCCESS);
1384 
1385 	case DDI_DMA_NEXTWIN: {
1386 		px_dma_win_t *win_p = *(px_dma_win_t **)offp;
1387 		px_dma_win_t **nw_pp = (px_dma_win_t **)objp;
1388 		ddi_dma_cookie_t *cp;
1389 		if (!win_p) {
1390 			*nw_pp = mp->dmai_winlst;
1391 			return (DDI_SUCCESS);
1392 		}
1393 
1394 		if (win_p->win_offset != mp->dmai_offset)
1395 			return (DDI_DMA_STALE);
1396 		if (!win_p->win_next)
1397 			return (DDI_DMA_DONE);
1398 		win_p = win_p->win_next;
1399 		cp = (ddi_dma_cookie_t *)(win_p + 1);
1400 		mp->dmai_offset = win_p->win_offset;
1401 		mp->dmai_size   = win_p->win_size;
1402 		mp->dmai_mapping = cp->dmac_laddress;   /* cookie0 star addr */
1403 		mp->dmai_cookie = cp + 1;
1404 		win_p->win_curseg = 0;
1405 		*nw_pp = win_p;
1406 		}
1407 		return (DDI_SUCCESS);
1408 
1409 	case DDI_DMA_NEXTSEG: {
1410 		px_dma_win_t *w_p = *(px_dma_win_t **)offp;
1411 		if (w_p->win_offset != mp->dmai_offset)
1412 			return (DDI_DMA_STALE);
1413 		if (w_p->win_curseg + 1 >= w_p->win_ncookies)
1414 			return (DDI_DMA_DONE);
1415 		w_p->win_curseg++;
1416 		}
1417 		*(ddi_dma_seg_t *)objp = (ddi_dma_seg_t)mp;
1418 		return (DDI_SUCCESS);
1419 
1420 	case DDI_DMA_SEGTOC: {
1421 		px_dma_win_t *win_p = mp->dmai_winlst;
1422 		off_t off = mp->dmai_offset;
1423 		ddi_dma_cookie_t *cp;
1424 		int i;
1425 
1426 		/* locate active window */
1427 		for (; win_p->win_offset != off; win_p = win_p->win_next);
1428 		cp = (ddi_dma_cookie_t *)(win_p + 1);
1429 		for (i = 0; i < win_p->win_curseg; i++, cp++)
1430 			off += cp->dmac_size;
1431 		*offp = off;
1432 		*lenp = cp->dmac_size;
1433 		*(ddi_dma_cookie_t *)objp = *cp;	/* copy cookie */
1434 		}
1435 		return (DDI_SUCCESS);
1436 
1437 	case DDI_DMA_COFF: {
1438 		px_dma_win_t *win_p;
1439 		ddi_dma_cookie_t *cp;
1440 		uint64_t addr, key = ((ddi_dma_cookie_t *)offp)->dmac_laddress;
1441 		size_t win_off;
1442 
1443 		for (win_p = mp->dmai_winlst; win_p; win_p = win_p->win_next) {
1444 			int i;
1445 			win_off = 0;
1446 			cp = (ddi_dma_cookie_t *)(win_p + 1);
1447 			for (i = 0; i < win_p->win_ncookies; i++, cp++) {
1448 				size_t sz = cp->dmac_size;
1449 
1450 				addr = cp->dmac_laddress;
1451 				if ((addr <= key) && (addr + sz >= key))
1452 					goto found;
1453 				win_off += sz;
1454 			}
1455 		}
1456 		return (DDI_FAILURE);
1457 found:
1458 		*objp = (caddr_t)(win_p->win_offset + win_off + (key - addr));
1459 		return (DDI_SUCCESS);
1460 		}
1461 	default:
1462 		DBG(DBG_DMA_CTL, dip, "unknown command (%x): rdip=%s%d\n",
1463 			cmd, ddi_driver_name(rdip), ddi_get_instance(rdip));
1464 		break;
1465 	}
1466 	return (DDI_FAILURE);
1467 }
1468 
1469 static void
1470 px_dvma_debug_init(px_mmu_t *mmu_p)
1471 {
1472 	size_t sz = sizeof (struct px_dvma_rec) * px_dvma_debug_rec;
1473 	ASSERT(MUTEX_HELD(&mmu_p->dvma_debug_lock));
1474 	cmn_err(CE_NOTE, "PCI Express DVMA %p stat ON", mmu_p);
1475 
1476 	mmu_p->dvma_alloc_rec = kmem_alloc(sz, KM_SLEEP);
1477 	mmu_p->dvma_free_rec = kmem_alloc(sz, KM_SLEEP);
1478 
1479 	mmu_p->dvma_active_list = NULL;
1480 	mmu_p->dvma_alloc_rec_index = 0;
1481 	mmu_p->dvma_free_rec_index = 0;
1482 	mmu_p->dvma_active_count = 0;
1483 }
1484 
1485 void
1486 px_dvma_debug_fini(px_mmu_t *mmu_p)
1487 {
1488 	struct px_dvma_rec *prev, *ptr;
1489 	size_t sz = sizeof (struct px_dvma_rec) * px_dvma_debug_rec;
1490 	uint64_t mask = ~(1ull << mmu_p->mmu_inst);
1491 	cmn_err(CE_NOTE, "PCI Express DVMA %p stat OFF", mmu_p);
1492 
1493 	kmem_free(mmu_p->dvma_alloc_rec, sz);
1494 	kmem_free(mmu_p->dvma_free_rec, sz);
1495 	mmu_p->dvma_alloc_rec = mmu_p->dvma_free_rec = NULL;
1496 
1497 	prev = mmu_p->dvma_active_list;
1498 	if (!prev)
1499 		return;
1500 	for (ptr = prev->next; ptr; prev = ptr, ptr = ptr->next)
1501 		kmem_free(prev, sizeof (struct px_dvma_rec));
1502 	kmem_free(prev, sizeof (struct px_dvma_rec));
1503 
1504 	mmu_p->dvma_active_list = NULL;
1505 	mmu_p->dvma_alloc_rec_index = 0;
1506 	mmu_p->dvma_free_rec_index = 0;
1507 	mmu_p->dvma_active_count = 0;
1508 
1509 	px_dvma_debug_off &= mask;
1510 	px_dvma_debug_on &= mask;
1511 }
1512 
1513 void
1514 px_dvma_alloc_debug(px_mmu_t *mmu_p, char *address, uint_t len,
1515 	ddi_dma_impl_t *mp)
1516 {
1517 	struct px_dvma_rec *ptr;
1518 	mutex_enter(&mmu_p->dvma_debug_lock);
1519 
1520 	if (!mmu_p->dvma_alloc_rec)
1521 		px_dvma_debug_init(mmu_p);
1522 	if (PX_DVMA_DBG_OFF(mmu_p)) {
1523 		px_dvma_debug_fini(mmu_p);
1524 		goto done;
1525 	}
1526 
1527 	ptr = &mmu_p->dvma_alloc_rec[mmu_p->dvma_alloc_rec_index];
1528 	ptr->dvma_addr = address;
1529 	ptr->len = len;
1530 	ptr->mp = mp;
1531 	if (++mmu_p->dvma_alloc_rec_index == px_dvma_debug_rec)
1532 		mmu_p->dvma_alloc_rec_index = 0;
1533 
1534 	ptr = kmem_alloc(sizeof (struct px_dvma_rec), KM_SLEEP);
1535 	ptr->dvma_addr = address;
1536 	ptr->len = len;
1537 	ptr->mp = mp;
1538 
1539 	ptr->next = mmu_p->dvma_active_list;
1540 	mmu_p->dvma_active_list = ptr;
1541 	mmu_p->dvma_active_count++;
1542 done:
1543 	mutex_exit(&mmu_p->dvma_debug_lock);
1544 }
1545 
1546 void
1547 px_dvma_free_debug(px_mmu_t *mmu_p, char *address, uint_t len,
1548     ddi_dma_impl_t *mp)
1549 {
1550 	struct px_dvma_rec *ptr, *ptr_save;
1551 	mutex_enter(&mmu_p->dvma_debug_lock);
1552 
1553 	if (!mmu_p->dvma_alloc_rec)
1554 		px_dvma_debug_init(mmu_p);
1555 	if (PX_DVMA_DBG_OFF(mmu_p)) {
1556 		px_dvma_debug_fini(mmu_p);
1557 		goto done;
1558 	}
1559 
1560 	ptr = &mmu_p->dvma_free_rec[mmu_p->dvma_free_rec_index];
1561 	ptr->dvma_addr = address;
1562 	ptr->len = len;
1563 	ptr->mp = mp;
1564 	if (++mmu_p->dvma_free_rec_index == px_dvma_debug_rec)
1565 		mmu_p->dvma_free_rec_index = 0;
1566 
1567 	ptr_save = mmu_p->dvma_active_list;
1568 	for (ptr = ptr_save; ptr; ptr = ptr->next) {
1569 		if ((ptr->dvma_addr == address) && (ptr->len = len))
1570 			break;
1571 		ptr_save = ptr;
1572 	}
1573 	if (!ptr) {
1574 		cmn_err(CE_WARN, "bad dvma free addr=%lx len=%x",
1575 			(long)address, len);
1576 		goto done;
1577 	}
1578 	if (ptr == mmu_p->dvma_active_list)
1579 		mmu_p->dvma_active_list = ptr->next;
1580 	else
1581 		ptr_save->next = ptr->next;
1582 	kmem_free(ptr, sizeof (struct px_dvma_rec));
1583 	mmu_p->dvma_active_count--;
1584 done:
1585 	mutex_exit(&mmu_p->dvma_debug_lock);
1586 }
1587 
1588 #ifdef	DEBUG
1589 void
1590 px_dump_dma_handle(uint64_t flag, dev_info_t *dip, ddi_dma_impl_t *hp)
1591 {
1592 	DBG(flag, dip, "mp(%p): flags=%x mapping=%lx xfer_size=%x\n",
1593 		hp, hp->dmai_inuse, hp->dmai_mapping, hp->dmai_size);
1594 	DBG(flag|DBG_CONT, dip, "\tnpages=%x roffset=%x rflags=%x nwin=%x\n",
1595 		hp->dmai_ndvmapages, hp->dmai_roffset, hp->dmai_rflags,
1596 		hp->dmai_nwin);
1597 	DBG(flag|DBG_CONT, dip, "\twinsize=%x tte=%p pfnlst=%p pfn0=%p\n",
1598 		hp->dmai_winsize, hp->dmai_tte, hp->dmai_pfnlst, hp->dmai_pfn0);
1599 	DBG(flag|DBG_CONT, dip, "\twinlst=%x obj=%p attr=%p ckp=%p\n",
1600 		hp->dmai_winlst, &hp->dmai_object, &hp->dmai_attr,
1601 		hp->dmai_cookie);
1602 }
1603 #endif	/* DEBUG */
1604