xref: /titanic_50/usr/src/uts/sun4u/io/iommu.c (revision 6d6fcbbb256ba9a8fe3e6076ae8a1f493b1f1a99)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 /*
26  * Copyright 2012 Garrett D'Amore <garrett@damore.org>.  All rights reserved.
27  */
28 
29 #include <sys/types.h>
30 #include <sys/param.h>
31 #include <sys/conf.h>
32 #include <sys/ddi.h>
33 #include <sys/sunddi.h>
34 #include <sys/ddi_impldefs.h>
35 #include <sys/cmn_err.h>
36 #include <sys/kmem.h>
37 #include <sys/vmem.h>
38 #include <sys/sysmacros.h>
39 
40 #include <sys/ddidmareq.h>
41 #include <sys/sysiosbus.h>
42 #include <sys/iommu.h>
43 #include <sys/iocache.h>
44 #include <sys/dvma.h>
45 
46 #include <vm/as.h>
47 #include <vm/hat.h>
48 #include <vm/page.h>
49 #include <vm/hat_sfmmu.h>
50 #include <sys/machparam.h>
51 #include <sys/machsystm.h>
52 #include <sys/vmsystm.h>
53 #include <sys/iommutsb.h>
54 
55 /* Useful debugging Stuff */
56 #include <sys/nexusdebug.h>
57 #include <sys/debug.h>
58 /* Bitfield debugging definitions for this file */
59 #define	IOMMU_GETDVMAPAGES_DEBUG	0x1
60 #define	IOMMU_DMAMAP_DEBUG		0x2
61 #define	IOMMU_DMAMCTL_DEBUG		0x4
62 #define	IOMMU_DMAMCTL_SYNC_DEBUG	0x8
63 #define	IOMMU_DMAMCTL_HTOC_DEBUG	0x10
64 #define	IOMMU_DMAMCTL_KVADDR_DEBUG	0x20
65 #define	IOMMU_DMAMCTL_GETERR_DEBUG	0x400
66 #define	IOMMU_DMAMCTL_DMA_FREE_DEBUG	0x1000
67 #define	IOMMU_REGISTERS_DEBUG		0x2000
68 #define	IOMMU_DMA_SETUP_DEBUG		0x4000
69 #define	IOMMU_DMA_UNBINDHDL_DEBUG	0x8000
70 #define	IOMMU_DMA_BINDHDL_DEBUG		0x10000
71 #define	IOMMU_DMA_WIN_DEBUG		0x20000
72 #define	IOMMU_DMA_ALLOCHDL_DEBUG	0x40000
73 #define	IOMMU_DMA_LIM_SETUP_DEBUG	0x80000
74 #define	IOMMU_FASTDMA_RESERVE		0x100000
75 #define	IOMMU_FASTDMA_LOAD		0x200000
76 #define	IOMMU_INTER_INTRA_XFER		0x400000
77 #define	IOMMU_TTE			0x800000
78 #define	IOMMU_TLB			0x1000000
79 #define	IOMMU_FASTDMA_SYNC		0x2000000
80 
81 /* Turn on if you need to keep track of outstanding IOMMU usage */
82 /* #define	IO_MEMUSAGE */
83 /* Turn on to debug IOMMU unmapping code */
84 /* #define	IO_MEMDEBUG */
85 
86 static struct dvma_ops iommu_dvma_ops = {
87 	DVMAO_REV,
88 	iommu_dvma_kaddr_load,
89 	iommu_dvma_unload,
90 	iommu_dvma_sync
91 };
92 
93 extern void *sbusp;		/* sbus soft state hook */
94 
95 #define	DVMA_MAX_CACHE	65536
96 
97 /*
98  * This is the number of pages that a mapping request needs before we force
99  * the TLB flush code to use diagnostic registers.  This value was determined
100  * through a series of test runs measuring dma mapping settup performance.
101  */
102 int tlb_flush_using_diag = 16;
103 
104 int sysio_iommu_tsb_sizes[] = {
105 	IOMMU_TSB_SIZE_8M,
106 	IOMMU_TSB_SIZE_16M,
107 	IOMMU_TSB_SIZE_32M,
108 	IOMMU_TSB_SIZE_64M,
109 	IOMMU_TSB_SIZE_128M,
110 	IOMMU_TSB_SIZE_256M,
111 	IOMMU_TSB_SIZE_512M,
112 	IOMMU_TSB_SIZE_1G
113 };
114 
115 static int iommu_map_window(ddi_dma_impl_t *, off_t, size_t);
116 
117 int
118 iommu_init(struct sbus_soft_state *softsp, caddr_t address)
119 {
120 	int i;
121 	char name[40];
122 
123 #ifdef DEBUG
124 	debug_info = 1;
125 #endif
126 
127 	/*
128 	 * Simply add each registers offset to the base address
129 	 * to calculate the already mapped virtual address of
130 	 * the device register...
131 	 *
132 	 * define a macro for the pointer arithmetic; all registers
133 	 * are 64 bits wide and are defined as uint64_t's.
134 	 */
135 
136 #define	REG_ADDR(b, o)	(uint64_t *)((caddr_t)(b) + (o))
137 
138 	softsp->iommu_ctrl_reg = REG_ADDR(address, OFF_IOMMU_CTRL_REG);
139 	softsp->tsb_base_addr = REG_ADDR(address, OFF_TSB_BASE_ADDR);
140 	softsp->iommu_flush_reg = REG_ADDR(address, OFF_IOMMU_FLUSH_REG);
141 	softsp->iommu_tlb_tag = REG_ADDR(address, OFF_IOMMU_TLB_TAG);
142 	softsp->iommu_tlb_data = REG_ADDR(address, OFF_IOMMU_TLB_DATA);
143 
144 #undef REG_ADDR
145 
146 	mutex_init(&softsp->dma_pool_lock, NULL, MUTEX_DEFAULT, NULL);
147 	mutex_init(&softsp->intr_poll_list_lock, NULL, MUTEX_DEFAULT, NULL);
148 
149 	/* Set up the DVMA resource sizes */
150 	if ((softsp->iommu_tsb_cookie = iommu_tsb_alloc(softsp->upa_id)) ==
151 	    IOMMU_TSB_COOKIE_NONE) {
152 		cmn_err(CE_WARN, "%s%d: Unable to retrieve IOMMU array.",
153 		    ddi_driver_name(softsp->dip),
154 		    ddi_get_instance(softsp->dip));
155 		return (DDI_FAILURE);
156 	}
157 	softsp->soft_tsb_base_addr =
158 	    iommu_tsb_cookie_to_va(softsp->iommu_tsb_cookie);
159 	softsp->iommu_dvma_size =
160 	    iommu_tsb_cookie_to_size(softsp->iommu_tsb_cookie) <<
161 	    IOMMU_TSB_TO_RNG;
162 	softsp->iommu_dvma_base = (ioaddr_t)
163 	    (0 - (ioaddr_t)softsp->iommu_dvma_size);
164 
165 	(void) snprintf(name, sizeof (name), "%s%d_dvma",
166 	    ddi_driver_name(softsp->dip), ddi_get_instance(softsp->dip));
167 
168 	/*
169 	 * Initialize the DVMA vmem arena.
170 	 */
171 	softsp->dvma_arena = vmem_create(name,
172 	    (void *)(uintptr_t)softsp->iommu_dvma_base,
173 	    softsp->iommu_dvma_size, PAGESIZE, NULL, NULL, NULL,
174 	    DVMA_MAX_CACHE, VM_SLEEP);
175 
176 	/* Set the limit for dvma_reserve() to 1/2 of the total dvma space */
177 	softsp->dma_reserve = iommu_btop(softsp->iommu_dvma_size >> 1);
178 
179 #if defined(DEBUG) && defined(IO_MEMUSAGE)
180 	mutex_init(&softsp->iomemlock, NULL, MUTEX_DEFAULT, NULL);
181 	softsp->iomem = (struct io_mem_list *)0;
182 #endif /* DEBUG && IO_MEMUSAGE */
183 	/*
184 	 * Get the base address of the TSB table and store it in the hardware
185 	 */
186 
187 	/*
188 	 * We plan on the PROM flushing all TLB entries.  If this is not the
189 	 * case, this is where we should flush the hardware TLB.
190 	 */
191 
192 	/* Set the IOMMU registers */
193 	(void) iommu_resume_init(softsp);
194 
195 	/* check the convenient copy of TSB base, and flush write buffers */
196 	if (*softsp->tsb_base_addr !=
197 	    va_to_pa((caddr_t)softsp->soft_tsb_base_addr)) {
198 		iommu_tsb_free(softsp->iommu_tsb_cookie);
199 		return (DDI_FAILURE);
200 	}
201 
202 	softsp->sbus_io_lo_pfn = UINT32_MAX;
203 	softsp->sbus_io_hi_pfn = 0;
204 	for (i = 0; i < sysio_pd_getnrng(softsp->dip); i++) {
205 		struct rangespec *rangep;
206 		uint64_t addr;
207 		pfn_t hipfn, lopfn;
208 
209 		rangep = sysio_pd_getrng(softsp->dip, i);
210 		addr = (uint64_t)((uint64_t)rangep->rng_bustype << 32);
211 		addr |= (uint64_t)rangep->rng_offset;
212 		lopfn = (pfn_t)(addr >> MMU_PAGESHIFT);
213 		addr += (uint64_t)(rangep->rng_size - 1);
214 		hipfn = (pfn_t)(addr >> MMU_PAGESHIFT);
215 
216 		softsp->sbus_io_lo_pfn = (lopfn < softsp->sbus_io_lo_pfn) ?
217 		    lopfn : softsp->sbus_io_lo_pfn;
218 
219 		softsp->sbus_io_hi_pfn = (hipfn > softsp->sbus_io_hi_pfn) ?
220 		    hipfn : softsp->sbus_io_hi_pfn;
221 	}
222 
223 	DPRINTF(IOMMU_REGISTERS_DEBUG, ("IOMMU Control reg: %p IOMMU TSB "
224 	    "base reg: %p IOMMU flush reg: %p TSB base addr %p\n",
225 	    (void *)softsp->iommu_ctrl_reg, (void *)softsp->tsb_base_addr,
226 	    (void *)softsp->iommu_flush_reg,
227 	    (void *)softsp->soft_tsb_base_addr));
228 
229 	return (DDI_SUCCESS);
230 }
231 
232 /*
233  * function to uninitialize the iommu and release the tsb back to
234  * the spare pool.  See startup.c for tsb spare management.
235  */
236 
237 int
238 iommu_uninit(struct sbus_soft_state *softsp)
239 {
240 	vmem_destroy(softsp->dvma_arena);
241 
242 	/* flip off the IOMMU enable switch */
243 	*softsp->iommu_ctrl_reg &=
244 	    (TSB_SIZE << TSB_SIZE_SHIFT | IOMMU_DISABLE);
245 
246 	iommu_tsb_free(softsp->iommu_tsb_cookie);
247 
248 	return (DDI_SUCCESS);
249 }
250 
251 /*
252  * Initialize iommu hardware registers when the system is being resumed.
253  * (Subset of iommu_init())
254  */
255 int
256 iommu_resume_init(struct sbus_soft_state *softsp)
257 {
258 	int i;
259 	uint_t tsb_size;
260 	uint_t tsb_bytes;
261 
262 	/*
263 	 * Reset the base address of the TSB table in the hardware
264 	 */
265 	*softsp->tsb_base_addr = va_to_pa((caddr_t)softsp->soft_tsb_base_addr);
266 
267 	/*
268 	 * Figure out the correct size of the IOMMU TSB entries.  If we
269 	 * end up with a size smaller than that needed for 8M of IOMMU
270 	 * space, default the size to 8M.  XXX We could probably panic here
271 	 */
272 	i = sizeof (sysio_iommu_tsb_sizes) / sizeof (sysio_iommu_tsb_sizes[0])
273 	    - 1;
274 
275 	tsb_bytes = iommu_tsb_cookie_to_size(softsp->iommu_tsb_cookie);
276 
277 	while (i > 0) {
278 		if (tsb_bytes >= sysio_iommu_tsb_sizes[i])
279 			break;
280 		i--;
281 	}
282 
283 	tsb_size = i;
284 
285 	/* OK, lets flip the "on" switch of the IOMMU */
286 	*softsp->iommu_ctrl_reg = (uint64_t)(tsb_size << TSB_SIZE_SHIFT
287 	    | IOMMU_ENABLE | IOMMU_DIAG_ENABLE);
288 
289 	return (DDI_SUCCESS);
290 }
291 
292 void
293 iommu_tlb_flush(struct sbus_soft_state *softsp, ioaddr_t addr, pgcnt_t npages)
294 {
295 	volatile uint64_t tmpreg;
296 	volatile uint64_t *vaddr_reg, *valid_bit_reg;
297 	ioaddr_t hiaddr, ioaddr;
298 	int i, do_flush = 0;
299 
300 	if (npages == 1) {
301 		*softsp->iommu_flush_reg = (uint64_t)addr;
302 		tmpreg = *softsp->sbus_ctrl_reg;
303 		return;
304 	}
305 
306 	hiaddr = addr + (ioaddr_t)(npages * IOMMU_PAGESIZE);
307 	for (i = 0, vaddr_reg = softsp->iommu_tlb_tag,
308 	    valid_bit_reg = softsp->iommu_tlb_data;
309 	    i < IOMMU_TLB_ENTRIES; i++, vaddr_reg++, valid_bit_reg++) {
310 		tmpreg = *vaddr_reg;
311 		ioaddr = (ioaddr_t)((tmpreg & IOMMU_TLBTAG_VA_MASK) <<
312 		    IOMMU_TLBTAG_VA_SHIFT);
313 
314 		DPRINTF(IOMMU_TLB, ("Vaddr reg 0x%p, "
315 		    "TLB vaddr reg %lx, IO addr 0x%x "
316 		    "Base addr 0x%x, Hi addr 0x%x\n",
317 		    (void *)vaddr_reg, tmpreg, ioaddr, addr, hiaddr));
318 
319 		if (ioaddr >= addr && ioaddr <= hiaddr) {
320 			tmpreg = *valid_bit_reg;
321 
322 			DPRINTF(IOMMU_TLB, ("Valid reg addr 0x%p, "
323 			    "TLB valid reg %lx\n",
324 			    (void *)valid_bit_reg, tmpreg));
325 
326 			if (tmpreg & IOMMU_TLB_VALID) {
327 				*softsp->iommu_flush_reg = (uint64_t)ioaddr;
328 				do_flush = 1;
329 			}
330 		}
331 	}
332 
333 	if (do_flush)
334 		tmpreg = *softsp->sbus_ctrl_reg;
335 }
336 
337 
338 /*
339  * Shorthand defines
340  */
341 
342 #define	ALO		dma_lim->dlim_addr_lo
343 #define	AHI		dma_lim->dlim_addr_hi
344 #define	OBJSIZE		dmareq->dmar_object.dmao_size
345 #define	IOTTE_NDX(vaddr, base) (base + \
346 		(int)(iommu_btop((vaddr & ~IOMMU_PAGEMASK) - \
347 		softsp->iommu_dvma_base)))
348 /*
349  * If DDI_DMA_PARTIAL flag is set and the request is for
350  * less than MIN_DVMA_WIN_SIZE, it's not worth the hassle so
351  * we turn off the DDI_DMA_PARTIAL flag
352  */
353 #define	MIN_DVMA_WIN_SIZE	(128)
354 
355 /* ARGSUSED */
356 void
357 iommu_remove_mappings(ddi_dma_impl_t *mp)
358 {
359 #if defined(DEBUG) && defined(IO_MEMDEBUG)
360 	pgcnt_t npages;
361 	ioaddr_t ioaddr;
362 	volatile uint64_t *iotte_ptr;
363 	ioaddr_t ioaddr = mp->dmai_mapping & ~IOMMU_PAGEOFFSET;
364 	pgcnt_t npages = mp->dmai_ndvmapages;
365 	struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp;
366 	struct sbus_soft_state *softsp = mppriv->softsp;
367 
368 #if defined(IO_MEMUSAGE)
369 	struct io_mem_list **prevp, *walk;
370 #endif /* DEBUG && IO_MEMUSAGE */
371 
372 	ASSERT(softsp != NULL);
373 	/*
374 	 * Run thru the mapped entries and free 'em
375 	 */
376 
377 	ioaddr = mp->dmai_mapping & ~IOMMU_PAGEOFFSET;
378 	npages = mp->dmai_ndvmapages;
379 
380 #if defined(IO_MEMUSAGE)
381 	mutex_enter(&softsp->iomemlock);
382 	prevp = &softsp->iomem;
383 	walk = softsp->iomem;
384 
385 	while (walk) {
386 		if (walk->ioaddr == ioaddr) {
387 			*prevp = walk->next;
388 			break;
389 		}
390 
391 		prevp = &walk->next;
392 		walk = walk->next;
393 	}
394 	mutex_exit(&softsp->iomemlock);
395 
396 	kmem_free(walk->pfn, sizeof (pfn_t) * (npages + 1));
397 	kmem_free(walk, sizeof (struct io_mem_list));
398 #endif /* IO_MEMUSAGE */
399 
400 	iotte_ptr = IOTTE_NDX(ioaddr, softsp->soft_tsb_base_addr);
401 
402 	while (npages) {
403 		DPRINTF(IOMMU_DMAMCTL_DEBUG,
404 		    ("dma_mctl: freeing ioaddr %x iotte %p\n",
405 		    ioaddr, iotte_ptr));
406 		*iotte_ptr = (uint64_t)0;	/* unload tte */
407 		iommu_tlb_flush(softsp, ioaddr, 1);
408 		npages--;
409 		ioaddr += IOMMU_PAGESIZE;
410 		iotte_ptr++;
411 	}
412 #endif /* DEBUG && IO_MEMDEBUG */
413 }
414 
415 
416 int
417 iommu_create_vaddr_mappings(ddi_dma_impl_t *mp, uintptr_t addr)
418 {
419 	pfn_t pfn;
420 	struct as *as = NULL;
421 	pgcnt_t npages;
422 	ioaddr_t ioaddr;
423 	uint_t offset;
424 	volatile uint64_t *iotte_ptr;
425 	uint64_t tmp_iotte_flag;
426 	int rval = DDI_DMA_MAPPED;
427 	struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp;
428 	struct sbus_soft_state *softsp = mppriv->softsp;
429 	int diag_tlb_flush;
430 #if defined(DEBUG) && defined(IO_MEMUSAGE)
431 	struct io_mem_list *iomemp;
432 	pfn_t *pfnp;
433 #endif /* DEBUG && IO_MEMUSAGE */
434 
435 	ASSERT(softsp != NULL);
436 
437 	/* Set Valid and Cache for mem xfer */
438 	tmp_iotte_flag = IOTTE_VALID | IOTTE_CACHE | IOTTE_WRITE | IOTTE_STREAM;
439 
440 	offset = (uint_t)(mp->dmai_mapping & IOMMU_PAGEOFFSET);
441 	npages = iommu_btopr(mp->dmai_size + offset);
442 	ioaddr = (ioaddr_t)(mp->dmai_mapping & ~IOMMU_PAGEOFFSET);
443 	iotte_ptr = IOTTE_NDX(ioaddr, softsp->soft_tsb_base_addr);
444 	diag_tlb_flush = npages > tlb_flush_using_diag ? 1 : 0;
445 
446 	as = mp->dmai_object.dmao_obj.virt_obj.v_as;
447 	if (as == NULL)
448 		as = &kas;
449 
450 	/*
451 	 * Set the per object bits of the TTE here. We optimize this for
452 	 * the memory case so that the while loop overhead is minimal.
453 	 */
454 	/* Turn on NOSYNC if we need consistent mem */
455 	if (mp->dmai_rflags & DDI_DMA_CONSISTENT) {
456 		mp->dmai_rflags |= DMP_NOSYNC;
457 		tmp_iotte_flag ^= IOTTE_STREAM;
458 	/* Set streaming mode if not consistent mem */
459 	} else if (softsp->stream_buf_off) {
460 		tmp_iotte_flag ^= IOTTE_STREAM;
461 	}
462 
463 #if defined(DEBUG) && defined(IO_MEMUSAGE)
464 	iomemp = kmem_alloc(sizeof (struct io_mem_list), KM_SLEEP);
465 	iomemp->rdip = mp->dmai_rdip;
466 	iomemp->ioaddr = ioaddr;
467 	iomemp->addr = addr;
468 	iomemp->npages = npages;
469 	pfnp = iomemp->pfn = kmem_zalloc(sizeof (*pfnp) * (npages + 1),
470 	    KM_SLEEP);
471 #endif /* DEBUG && IO_MEMUSAGE */
472 	/*
473 	 * Grab the mappings from the dmmu and stick 'em into the
474 	 * iommu.
475 	 */
476 	ASSERT(npages != 0);
477 
478 	/* If we're going to flush the TLB using diag mode, do it now. */
479 	if (diag_tlb_flush)
480 		iommu_tlb_flush(softsp, ioaddr, npages);
481 
482 	do {
483 		uint64_t iotte_flag = tmp_iotte_flag;
484 
485 		/*
486 		 * Fetch the pfn for the DMA object
487 		 */
488 
489 		ASSERT(as);
490 		pfn = hat_getpfnum(as->a_hat, (caddr_t)addr);
491 		ASSERT(pfn != PFN_INVALID);
492 
493 		if (!pf_is_memory(pfn)) {
494 			/* DVMA'ing to IO space */
495 
496 			/* Turn off cache bit if set */
497 			if (iotte_flag & IOTTE_CACHE)
498 				iotte_flag ^= IOTTE_CACHE;
499 
500 			/* Turn off stream bit if set */
501 			if (iotte_flag & IOTTE_STREAM)
502 				iotte_flag ^= IOTTE_STREAM;
503 
504 			if (IS_INTRA_SBUS(softsp, pfn)) {
505 				/* Intra sbus transfer */
506 
507 				/* Turn on intra flag */
508 				iotte_flag |= IOTTE_INTRA;
509 
510 				DPRINTF(IOMMU_INTER_INTRA_XFER, (
511 				    "Intra xfer pfnum %lx TTE %lx\n",
512 				    pfn, iotte_flag));
513 			} else {
514 				if (pf_is_dmacapable(pfn) == 1) {
515 					/*EMPTY*/
516 					DPRINTF(IOMMU_INTER_INTRA_XFER,
517 					    ("Inter xfer pfnum %lx "
518 					    "tte hi %lx\n",
519 					    pfn, iotte_flag));
520 				} else {
521 					rval = DDI_DMA_NOMAPPING;
522 #if defined(DEBUG) && defined(IO_MEMDEBUG)
523 					goto bad;
524 #endif /* DEBUG && IO_MEMDEBUG */
525 				}
526 			}
527 		}
528 		addr += IOMMU_PAGESIZE;
529 
530 		DPRINTF(IOMMU_TTE, ("vaddr mapping: tte index %p pfn %lx "
531 		    "tte flag %lx addr %lx ioaddr %x\n",
532 		    (void *)iotte_ptr, pfn, iotte_flag, addr, ioaddr));
533 
534 		/* Flush the IOMMU TLB before loading a new mapping */
535 		if (!diag_tlb_flush)
536 			iommu_tlb_flush(softsp, ioaddr, 1);
537 
538 		/* Set the hardware IO TTE */
539 		*iotte_ptr = ((uint64_t)pfn << IOMMU_PAGESHIFT) | iotte_flag;
540 
541 		ioaddr += IOMMU_PAGESIZE;
542 		npages--;
543 		iotte_ptr++;
544 #if defined(DEBUG) && defined(IO_MEMUSAGE)
545 		*pfnp = pfn;
546 		pfnp++;
547 #endif /* DEBUG && IO_MEMUSAGE */
548 	} while (npages != 0);
549 
550 #if defined(DEBUG) && defined(IO_MEMUSAGE)
551 	mutex_enter(&softsp->iomemlock);
552 	iomemp->next = softsp->iomem;
553 	softsp->iomem = iomemp;
554 	mutex_exit(&softsp->iomemlock);
555 #endif /* DEBUG && IO_MEMUSAGE */
556 
557 	return (rval);
558 
559 #if defined(DEBUG) && defined(IO_MEMDEBUG)
560 bad:
561 	/* If we fail a mapping, free up any mapping resources used */
562 	iommu_remove_mappings(mp);
563 	return (rval);
564 #endif /* DEBUG && IO_MEMDEBUG */
565 }
566 
567 
568 int
569 iommu_create_pp_mappings(ddi_dma_impl_t *mp, page_t *pp, page_t **pplist)
570 {
571 	pfn_t pfn;
572 	pgcnt_t npages;
573 	ioaddr_t ioaddr;
574 	uint_t offset;
575 	volatile uint64_t *iotte_ptr;
576 	uint64_t tmp_iotte_flag;
577 	struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp;
578 	struct sbus_soft_state *softsp = mppriv->softsp;
579 	int diag_tlb_flush;
580 #if defined(DEBUG) && defined(IO_MEMUSAGE)
581 	struct io_mem_list *iomemp;
582 	pfn_t *pfnp;
583 #endif /* DEBUG && IO_MEMUSAGE */
584 	int rval = DDI_DMA_MAPPED;
585 
586 	/* Set Valid and Cache for mem xfer */
587 	tmp_iotte_flag = IOTTE_VALID | IOTTE_CACHE | IOTTE_WRITE | IOTTE_STREAM;
588 
589 	ASSERT(softsp != NULL);
590 
591 	offset = (uint_t)(mp->dmai_mapping & IOMMU_PAGEOFFSET);
592 	npages = iommu_btopr(mp->dmai_size + offset);
593 	ioaddr = (ioaddr_t)(mp->dmai_mapping & ~IOMMU_PAGEOFFSET);
594 	iotte_ptr = IOTTE_NDX(ioaddr, softsp->soft_tsb_base_addr);
595 	diag_tlb_flush = npages > tlb_flush_using_diag ? 1 : 0;
596 
597 	/*
598 	 * Set the per object bits of the TTE here. We optimize this for
599 	 * the memory case so that the while loop overhead is minimal.
600 	 */
601 	if (mp->dmai_rflags & DDI_DMA_CONSISTENT) {
602 		/* Turn on NOSYNC if we need consistent mem */
603 		mp->dmai_rflags |= DMP_NOSYNC;
604 		tmp_iotte_flag ^= IOTTE_STREAM;
605 	} else if (softsp->stream_buf_off) {
606 		/* Set streaming mode if not consistent mem */
607 		tmp_iotte_flag ^= IOTTE_STREAM;
608 	}
609 
610 #if defined(DEBUG) && defined(IO_MEMUSAGE)
611 	iomemp = kmem_alloc(sizeof (struct io_mem_list), KM_SLEEP);
612 	iomemp->rdip = mp->dmai_rdip;
613 	iomemp->ioaddr = ioaddr;
614 	iomemp->npages = npages;
615 	pfnp = iomemp->pfn = kmem_zalloc(sizeof (*pfnp) * (npages + 1),
616 	    KM_SLEEP);
617 #endif /* DEBUG && IO_MEMUSAGE */
618 	/*
619 	 * Grab the mappings from the dmmu and stick 'em into the
620 	 * iommu.
621 	 */
622 	ASSERT(npages != 0);
623 
624 	/* If we're going to flush the TLB using diag mode, do it now. */
625 	if (diag_tlb_flush)
626 		iommu_tlb_flush(softsp, ioaddr, npages);
627 
628 	do {
629 		uint64_t iotte_flag;
630 
631 		iotte_flag = tmp_iotte_flag;
632 
633 		if (pp != NULL) {
634 			pfn = pp->p_pagenum;
635 			pp = pp->p_next;
636 		} else {
637 			pfn = (*pplist)->p_pagenum;
638 			pplist++;
639 		}
640 
641 		DPRINTF(IOMMU_TTE, ("pp mapping TTE index %p pfn %lx "
642 		    "tte flag %lx ioaddr %x\n", (void *)iotte_ptr,
643 		    pfn, iotte_flag, ioaddr));
644 
645 		/* Flush the IOMMU TLB before loading a new mapping */
646 		if (!diag_tlb_flush)
647 			iommu_tlb_flush(softsp, ioaddr, 1);
648 
649 		/* Set the hardware IO TTE */
650 		*iotte_ptr = ((uint64_t)pfn << IOMMU_PAGESHIFT) | iotte_flag;
651 
652 		ioaddr += IOMMU_PAGESIZE;
653 		npages--;
654 		iotte_ptr++;
655 
656 #if defined(DEBUG) && defined(IO_MEMUSAGE)
657 		*pfnp = pfn;
658 		pfnp++;
659 #endif /* DEBUG && IO_MEMUSAGE */
660 
661 	} while (npages != 0);
662 
663 #if defined(DEBUG) && defined(IO_MEMUSAGE)
664 	mutex_enter(&softsp->iomemlock);
665 	iomemp->next = softsp->iomem;
666 	softsp->iomem = iomemp;
667 	mutex_exit(&softsp->iomemlock);
668 #endif /* DEBUG && IO_MEMUSAGE */
669 
670 	return (rval);
671 }
672 
673 
674 int
675 iommu_dma_lim_setup(dev_info_t *dip, dev_info_t *rdip,
676     struct sbus_soft_state *softsp, uint_t *burstsizep, uint_t burstsize64,
677     uint_t *minxferp, uint_t dma_flags)
678 {
679 	struct regspec *rp;
680 
681 	/* Take care of 64 bit limits. */
682 	if (!(dma_flags & DDI_DMA_SBUS_64BIT)) {
683 		/*
684 		 * return burst size for 32-bit mode
685 		 */
686 		*burstsizep &= softsp->sbus_burst_sizes;
687 		return (DDI_FAILURE);
688 	}
689 
690 	/*
691 	 * check if SBus supports 64 bit and if caller
692 	 * is child of SBus. No support through bridges
693 	 */
694 	if (!softsp->sbus64_burst_sizes || (ddi_get_parent(rdip) != dip)) {
695 		/*
696 		 * SBus doesn't support it or bridge. Do 32-bit
697 		 * xfers
698 		 */
699 		*burstsizep &= softsp->sbus_burst_sizes;
700 		return (DDI_FAILURE);
701 	}
702 
703 	rp = ddi_rnumber_to_regspec(rdip, 0);
704 	if (rp == NULL) {
705 		*burstsizep &= softsp->sbus_burst_sizes;
706 		return (DDI_FAILURE);
707 	}
708 
709 	/* Check for old-style 64 bit burstsizes */
710 	if (burstsize64 & SYSIO64_BURST_MASK) {
711 		/* Scale back burstsizes if Necessary */
712 		*burstsizep &= (softsp->sbus64_burst_sizes |
713 		    softsp->sbus_burst_sizes);
714 	} else {
715 		/* Get the 64 bit burstsizes. */
716 		*burstsizep = burstsize64;
717 
718 		/* Scale back burstsizes if Necessary */
719 		*burstsizep &= (softsp->sbus64_burst_sizes >>
720 		    SYSIO64_BURST_SHIFT);
721 	}
722 
723 	/*
724 	 * Set the largest value of the smallest burstsize that the
725 	 * device or the bus can manage.
726 	 */
727 	*minxferp = MAX(*minxferp,
728 	    (1 << (ddi_ffs(softsp->sbus64_burst_sizes) - 1)));
729 
730 	return (DDI_SUCCESS);
731 }
732 
733 
734 int
735 iommu_dma_allochdl(dev_info_t *dip, dev_info_t *rdip,
736     ddi_dma_attr_t *dma_attr, int (*waitfp)(caddr_t), caddr_t arg,
737     ddi_dma_handle_t *handlep)
738 {
739 	ioaddr_t addrlow, addrhigh, segalign;
740 	ddi_dma_impl_t *mp;
741 	struct dma_impl_priv *mppriv;
742 	struct sbus_soft_state *softsp = (struct sbus_soft_state *)
743 	    ddi_get_soft_state(sbusp, ddi_get_instance(dip));
744 
745 	/*
746 	 * Setup dma burstsizes and min-xfer counts.
747 	 */
748 	(void) iommu_dma_lim_setup(dip, rdip, softsp,
749 	    &dma_attr->dma_attr_burstsizes,
750 	    dma_attr->dma_attr_burstsizes, &dma_attr->dma_attr_minxfer,
751 	    dma_attr->dma_attr_flags);
752 
753 	if (dma_attr->dma_attr_burstsizes == 0)
754 		return (DDI_DMA_BADATTR);
755 
756 	addrlow = (ioaddr_t)dma_attr->dma_attr_addr_lo;
757 	addrhigh = (ioaddr_t)dma_attr->dma_attr_addr_hi;
758 	segalign = (ioaddr_t)dma_attr->dma_attr_seg;
759 
760 	/*
761 	 * Check sanity for hi and lo address limits
762 	 */
763 	if ((addrhigh <= addrlow) ||
764 	    (addrhigh < (ioaddr_t)softsp->iommu_dvma_base)) {
765 		return (DDI_DMA_BADATTR);
766 	}
767 	if (dma_attr->dma_attr_flags & DDI_DMA_FORCE_PHYSICAL)
768 		return (DDI_DMA_BADATTR);
769 
770 	mppriv = kmem_zalloc(sizeof (*mppriv),
771 	    (waitfp == DDI_DMA_SLEEP) ? KM_SLEEP : KM_NOSLEEP);
772 
773 	if (mppriv == NULL) {
774 		if (waitfp != DDI_DMA_DONTWAIT) {
775 			ddi_set_callback(waitfp, arg,
776 			    &softsp->dvma_call_list_id);
777 		}
778 		return (DDI_DMA_NORESOURCES);
779 	}
780 	mp = (ddi_dma_impl_t *)mppriv;
781 
782 	DPRINTF(IOMMU_DMA_ALLOCHDL_DEBUG, ("dma_allochdl: (%s) handle %p "
783 	    "hi %x lo %x min %x burst %x\n",
784 	    ddi_get_name(dip), (void *)mp, addrhigh, addrlow,
785 	    dma_attr->dma_attr_minxfer, dma_attr->dma_attr_burstsizes));
786 
787 	mp->dmai_rdip = rdip;
788 	mp->dmai_minxfer = (uint_t)dma_attr->dma_attr_minxfer;
789 	mp->dmai_burstsizes = (uint_t)dma_attr->dma_attr_burstsizes;
790 	mp->dmai_attr = *dma_attr;
791 	/* See if the DMA engine has any limit restrictions. */
792 	if (segalign == (ioaddr_t)UINT32_MAX &&
793 	    addrhigh == (ioaddr_t)UINT32_MAX &&
794 	    (dma_attr->dma_attr_align <= IOMMU_PAGESIZE) && addrlow == 0) {
795 		mp->dmai_rflags |= DMP_NOLIMIT;
796 	}
797 	mppriv->softsp = softsp;
798 	mppriv->phys_sync_flag = va_to_pa((caddr_t)&mppriv->sync_flag);
799 
800 	*handlep = (ddi_dma_handle_t)mp;
801 	return (DDI_SUCCESS);
802 }
803 
804 /*ARGSUSED*/
805 int
806 iommu_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
807 {
808 	struct dma_impl_priv *mppriv = (struct dma_impl_priv *)handle;
809 	struct sbus_soft_state *softsp = mppriv->softsp;
810 	ASSERT(softsp != NULL);
811 
812 	kmem_free(mppriv, sizeof (*mppriv));
813 
814 	if (softsp->dvma_call_list_id != 0) {
815 		ddi_run_callback(&softsp->dvma_call_list_id);
816 	}
817 	return (DDI_SUCCESS);
818 }
819 
820 static int
821 check_dma_attr(struct ddi_dma_req *dmareq, ddi_dma_attr_t *dma_attr,
822     uint32_t *size)
823 {
824 	ioaddr_t addrlow;
825 	ioaddr_t addrhigh;
826 	uint32_t segalign;
827 	uint32_t smask;
828 
829 	smask = *size - 1;
830 	segalign = dma_attr->dma_attr_seg;
831 	if (smask > segalign) {
832 		if ((dmareq->dmar_flags & DDI_DMA_PARTIAL) == 0)
833 			return (DDI_DMA_TOOBIG);
834 		*size = segalign + 1;
835 	}
836 	addrlow = (ioaddr_t)dma_attr->dma_attr_addr_lo;
837 	addrhigh = (ioaddr_t)dma_attr->dma_attr_addr_hi;
838 	if (addrlow + smask > addrhigh || addrlow + smask < addrlow) {
839 		if (!((addrlow + dmareq->dmar_object.dmao_size == 0) &&
840 		    (addrhigh == (ioaddr_t)-1))) {
841 			if ((dmareq->dmar_flags & DDI_DMA_PARTIAL) == 0)
842 				return (DDI_DMA_TOOBIG);
843 			*size = MIN(addrhigh - addrlow + 1, *size);
844 		}
845 	}
846 	return (DDI_DMA_MAPOK);
847 }
848 
849 int
850 iommu_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
851     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
852     ddi_dma_cookie_t *cp, uint_t *ccountp)
853 {
854 	page_t *pp;
855 	uint32_t size;
856 	ioaddr_t ioaddr;
857 	uint_t offset;
858 	uintptr_t addr = 0;
859 	pgcnt_t npages;
860 	int rval;
861 	ddi_dma_attr_t *dma_attr;
862 	struct sbus_soft_state *softsp;
863 	struct page **pplist = NULL;
864 	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
865 	struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp;
866 
867 #ifdef lint
868 	dip = dip;
869 	rdip = rdip;
870 #endif
871 
872 	if (mp->dmai_inuse)
873 		return (DDI_DMA_INUSE);
874 
875 	dma_attr = &mp->dmai_attr;
876 	size = (uint32_t)dmareq->dmar_object.dmao_size;
877 	if (!(mp->dmai_rflags & DMP_NOLIMIT)) {
878 		rval = check_dma_attr(dmareq, dma_attr, &size);
879 		if (rval != DDI_DMA_MAPOK)
880 			return (rval);
881 	}
882 	mp->dmai_inuse = 1;
883 	mp->dmai_offset = 0;
884 	mp->dmai_rflags = (dmareq->dmar_flags & DMP_DDIFLAGS) |
885 	    (mp->dmai_rflags & DMP_NOLIMIT);
886 
887 	switch (dmareq->dmar_object.dmao_type) {
888 	case DMA_OTYP_VADDR:
889 	case DMA_OTYP_BUFVADDR:
890 		addr = (uintptr_t)dmareq->dmar_object.dmao_obj.virt_obj.v_addr;
891 		offset = addr & IOMMU_PAGEOFFSET;
892 		pplist = dmareq->dmar_object.dmao_obj.virt_obj.v_priv;
893 		npages = iommu_btopr(OBJSIZE + offset);
894 
895 		DPRINTF(IOMMU_DMAMAP_DEBUG, ("dma_map vaddr: %lx pages "
896 		    "req addr %lx off %x OBJSIZE %x\n",
897 		    npages, addr, offset, OBJSIZE));
898 
899 		/* We don't need the addr anymore if we have a shadow list */
900 		if (pplist != NULL)
901 			addr = NULL;
902 		pp = NULL;
903 		break;
904 
905 	case DMA_OTYP_PAGES:
906 		pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp;
907 		offset = dmareq->dmar_object.dmao_obj.pp_obj.pp_offset;
908 		npages = iommu_btopr(OBJSIZE + offset);
909 		break;
910 
911 	case DMA_OTYP_PADDR:
912 	default:
913 		/*
914 		 * Not a supported type for this implementation
915 		 */
916 		rval = DDI_DMA_NOMAPPING;
917 		goto bad;
918 	}
919 
920 	/* Get our soft state once we know we're mapping an object. */
921 	softsp = mppriv->softsp;
922 	ASSERT(softsp != NULL);
923 
924 	if (mp->dmai_rflags & DDI_DMA_PARTIAL) {
925 		if (size != OBJSIZE) {
926 			/*
927 			 * If the request is for partial mapping arrangement,
928 			 * the device has to be able to address at least the
929 			 * size of the window we are establishing.
930 			 */
931 			if (size < iommu_ptob(MIN_DVMA_WIN_SIZE)) {
932 				rval = DDI_DMA_NOMAPPING;
933 				goto bad;
934 			}
935 			npages = iommu_btopr(size + offset);
936 		}
937 		/*
938 		 * If the size requested is less than a moderate amt,
939 		 * skip the partial mapping stuff- it's not worth the
940 		 * effort.
941 		 */
942 		if (npages > MIN_DVMA_WIN_SIZE) {
943 			npages = MIN_DVMA_WIN_SIZE + iommu_btopr(offset);
944 			size = iommu_ptob(MIN_DVMA_WIN_SIZE);
945 			DPRINTF(IOMMU_DMA_SETUP_DEBUG, ("dma_setup: SZ %x pg "
946 			    "%lx sz %x\n", OBJSIZE, npages, size));
947 			if (pplist != NULL) {
948 				mp->dmai_minfo = (void *)pplist;
949 				mp->dmai_rflags |= DMP_SHADOW;
950 			}
951 		} else {
952 			mp->dmai_rflags ^= DDI_DMA_PARTIAL;
953 		}
954 	} else {
955 		if (npages >= iommu_btop(softsp->iommu_dvma_size) -
956 		    MIN_DVMA_WIN_SIZE) {
957 			rval = DDI_DMA_TOOBIG;
958 			goto bad;
959 		}
960 	}
961 
962 	/*
963 	 * save dmareq-object, size and npages into mp
964 	 */
965 	mp->dmai_object = dmareq->dmar_object;
966 	mp->dmai_size = size;
967 	mp->dmai_ndvmapages = npages;
968 
969 	if (mp->dmai_rflags & DMP_NOLIMIT) {
970 		ioaddr = (ioaddr_t)(uintptr_t)vmem_alloc(softsp->dvma_arena,
971 		    iommu_ptob(npages),
972 		    dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP);
973 		if (ioaddr == 0) {
974 			rval = DDI_DMA_NORESOURCES;
975 			goto bad;
976 		}
977 
978 		/*
979 		 * If we have a 1 page request and we're working with a page
980 		 * list, we're going to speed load an IOMMU entry.
981 		 */
982 		if (npages == 1 && !addr) {
983 			uint64_t iotte_flag = IOTTE_VALID | IOTTE_CACHE |
984 			    IOTTE_WRITE | IOTTE_STREAM;
985 			volatile uint64_t *iotte_ptr;
986 			pfn_t pfn;
987 #if defined(DEBUG) && defined(IO_MEMUSAGE)
988 			struct io_mem_list *iomemp;
989 			pfn_t *pfnp;
990 #endif /* DEBUG && IO_MEMUSAGE */
991 
992 			iotte_ptr = IOTTE_NDX(ioaddr,
993 			    softsp->soft_tsb_base_addr);
994 
995 			if (mp->dmai_rflags & DDI_DMA_CONSISTENT) {
996 				mp->dmai_rflags |= DMP_NOSYNC;
997 				iotte_flag ^= IOTTE_STREAM;
998 			} else if (softsp->stream_buf_off)
999 				iotte_flag ^= IOTTE_STREAM;
1000 
1001 			mp->dmai_rflags ^= DDI_DMA_PARTIAL;
1002 
1003 			if (pp != NULL)
1004 				pfn = pp->p_pagenum;
1005 			else
1006 				pfn = (*pplist)->p_pagenum;
1007 
1008 			iommu_tlb_flush(softsp, ioaddr, 1);
1009 
1010 			*iotte_ptr =
1011 			    ((uint64_t)pfn << IOMMU_PAGESHIFT) | iotte_flag;
1012 
1013 			mp->dmai_mapping = (ioaddr_t)(ioaddr + offset);
1014 			mp->dmai_nwin = 0;
1015 			if (cp != NULL) {
1016 				cp->dmac_notused = 0;
1017 				cp->dmac_address = (ioaddr_t)mp->dmai_mapping;
1018 				cp->dmac_size = mp->dmai_size;
1019 				cp->dmac_type = 0;
1020 				*ccountp = 1;
1021 			}
1022 
1023 			DPRINTF(IOMMU_TTE, ("speed loading: TTE index %p "
1024 			    "pfn %lx tte flag %lx addr %lx ioaddr %x\n",
1025 			    (void *)iotte_ptr, pfn, iotte_flag, addr, ioaddr));
1026 
1027 #if defined(DEBUG) && defined(IO_MEMUSAGE)
1028 			iomemp = kmem_alloc(sizeof (struct io_mem_list),
1029 			    KM_SLEEP);
1030 			iomemp->rdip = mp->dmai_rdip;
1031 			iomemp->ioaddr = ioaddr;
1032 			iomemp->addr = addr;
1033 			iomemp->npages = npages;
1034 			pfnp = iomemp->pfn = kmem_zalloc(sizeof (*pfnp) *
1035 			    (npages + 1), KM_SLEEP);
1036 			*pfnp = pfn;
1037 			mutex_enter(&softsp->iomemlock);
1038 			iomemp->next = softsp->iomem;
1039 			softsp->iomem = iomemp;
1040 			mutex_exit(&softsp->iomemlock);
1041 #endif /* DEBUG && IO_MEMUSAGE */
1042 
1043 			return (DDI_DMA_MAPPED);
1044 		}
1045 	} else {
1046 		ioaddr = (ioaddr_t)(uintptr_t)vmem_xalloc(softsp->dvma_arena,
1047 		    iommu_ptob(npages),
1048 		    MAX((uint_t)dma_attr->dma_attr_align, IOMMU_PAGESIZE), 0,
1049 		    (uint_t)dma_attr->dma_attr_seg + 1,
1050 		    (void *)(uintptr_t)(ioaddr_t)dma_attr->dma_attr_addr_lo,
1051 		    (void *)(uintptr_t)
1052 		    ((ioaddr_t)dma_attr->dma_attr_addr_hi + 1),
1053 		    dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP);
1054 	}
1055 
1056 	if (ioaddr == 0) {
1057 		if (dmareq->dmar_fp == DDI_DMA_SLEEP)
1058 			rval = DDI_DMA_NOMAPPING;
1059 		else
1060 			rval = DDI_DMA_NORESOURCES;
1061 		goto bad;
1062 	}
1063 
1064 	mp->dmai_mapping = ioaddr + offset;
1065 	ASSERT(mp->dmai_mapping >= softsp->iommu_dvma_base);
1066 
1067 	/*
1068 	 * At this point we have a range of virtual address allocated
1069 	 * with which we now have to map to the requested object.
1070 	 */
1071 	if (addr) {
1072 		rval = iommu_create_vaddr_mappings(mp,
1073 		    addr & ~IOMMU_PAGEOFFSET);
1074 		if (rval == DDI_DMA_NOMAPPING)
1075 			goto bad_nomap;
1076 	} else {
1077 		rval = iommu_create_pp_mappings(mp, pp, pplist);
1078 		if (rval == DDI_DMA_NOMAPPING)
1079 			goto bad_nomap;
1080 	}
1081 
1082 	if (cp) {
1083 		cp->dmac_notused = 0;
1084 		cp->dmac_address = (ioaddr_t)mp->dmai_mapping;
1085 		cp->dmac_size = mp->dmai_size;
1086 		cp->dmac_type = 0;
1087 		*ccountp = 1;
1088 	}
1089 	if (mp->dmai_rflags & DDI_DMA_PARTIAL) {
1090 		size = iommu_ptob(mp->dmai_ndvmapages - iommu_btopr(offset));
1091 		mp->dmai_nwin =
1092 		    (dmareq->dmar_object.dmao_size + (size - 1)) / size;
1093 		return (DDI_DMA_PARTIAL_MAP);
1094 	} else {
1095 		mp->dmai_nwin = 0;
1096 		return (DDI_DMA_MAPPED);
1097 	}
1098 
1099 bad_nomap:
1100 	/*
1101 	 * Could not create mmu mappings.
1102 	 */
1103 	if (mp->dmai_rflags & DMP_NOLIMIT) {
1104 		vmem_free(softsp->dvma_arena, (void *)(uintptr_t)ioaddr,
1105 		    iommu_ptob(npages));
1106 	} else {
1107 		vmem_xfree(softsp->dvma_arena, (void *)(uintptr_t)ioaddr,
1108 		    iommu_ptob(npages));
1109 	}
1110 
1111 bad:
1112 	if (rval == DDI_DMA_NORESOURCES &&
1113 	    dmareq->dmar_fp != DDI_DMA_DONTWAIT) {
1114 		ddi_set_callback(dmareq->dmar_fp,
1115 		    dmareq->dmar_arg, &softsp->dvma_call_list_id);
1116 	}
1117 	mp->dmai_inuse = 0;
1118 	return (rval);
1119 }
1120 
1121 /* ARGSUSED */
1122 int
1123 iommu_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
1124     ddi_dma_handle_t handle)
1125 {
1126 	ioaddr_t addr;
1127 	uint_t npages;
1128 	size_t size;
1129 	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
1130 	struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp;
1131 	struct sbus_soft_state *softsp = mppriv->softsp;
1132 	ASSERT(softsp != NULL);
1133 
1134 	addr = (ioaddr_t)(mp->dmai_mapping & ~IOMMU_PAGEOFFSET);
1135 	npages = mp->dmai_ndvmapages;
1136 	size = iommu_ptob(npages);
1137 
1138 	DPRINTF(IOMMU_DMA_UNBINDHDL_DEBUG, ("iommu_dma_unbindhdl: "
1139 	    "unbinding addr %x for %x pages\n", addr, mp->dmai_ndvmapages));
1140 
1141 	/* sync the entire object */
1142 	if (!(mp->dmai_rflags & DDI_DMA_CONSISTENT)) {
1143 		/* flush stream write buffers */
1144 		sync_stream_buf(softsp, addr, npages, (int *)&mppriv->sync_flag,
1145 		    mppriv->phys_sync_flag);
1146 	}
1147 
1148 #if defined(DEBUG) && defined(IO_MEMDEBUG)
1149 	/*
1150 	 * 'Free' the dma mappings.
1151 	 */
1152 	iommu_remove_mappings(mp);
1153 #endif /* DEBUG && IO_MEMDEBUG */
1154 
1155 	ASSERT(npages > (uint_t)0);
1156 	if (mp->dmai_rflags & DMP_NOLIMIT)
1157 		vmem_free(softsp->dvma_arena, (void *)(uintptr_t)addr, size);
1158 	else
1159 		vmem_xfree(softsp->dvma_arena, (void *)(uintptr_t)addr, size);
1160 
1161 	mp->dmai_ndvmapages = 0;
1162 	mp->dmai_inuse = 0;
1163 	mp->dmai_minfo = NULL;
1164 
1165 	if (softsp->dvma_call_list_id != 0)
1166 		ddi_run_callback(&softsp->dvma_call_list_id);
1167 
1168 	return (DDI_SUCCESS);
1169 }
1170 
1171 /*ARGSUSED*/
1172 int
1173 iommu_dma_flush(dev_info_t *dip, dev_info_t *rdip,
1174     ddi_dma_handle_t handle, off_t off, size_t len,
1175     uint_t cache_flags)
1176 {
1177 	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
1178 	struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp;
1179 
1180 	if (!(mp->dmai_rflags & DDI_DMA_CONSISTENT)) {
1181 		sync_stream_buf(mppriv->softsp, mp->dmai_mapping,
1182 		    mp->dmai_ndvmapages, (int *)&mppriv->sync_flag,
1183 		    mppriv->phys_sync_flag);
1184 	}
1185 	return (DDI_SUCCESS);
1186 }
1187 
1188 /*ARGSUSED*/
1189 int
1190 iommu_dma_win(dev_info_t *dip, dev_info_t *rdip,
1191     ddi_dma_handle_t handle, uint_t win, off_t *offp,
1192     size_t *lenp, ddi_dma_cookie_t *cookiep, uint_t *ccountp)
1193 {
1194 	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
1195 	off_t offset;
1196 	uint_t winsize;
1197 	uint_t newoff;
1198 	int rval;
1199 
1200 	offset = mp->dmai_mapping & IOMMU_PAGEOFFSET;
1201 	winsize = iommu_ptob(mp->dmai_ndvmapages - iommu_btopr(offset));
1202 
1203 	DPRINTF(IOMMU_DMA_WIN_DEBUG, ("getwin win %d winsize %x\n", win,
1204 	    winsize));
1205 
1206 	/*
1207 	 * win is in the range [0 .. dmai_nwin-1]
1208 	 */
1209 	if (win >= mp->dmai_nwin)
1210 		return (DDI_FAILURE);
1211 
1212 	newoff = win * winsize;
1213 	if (newoff > mp->dmai_object.dmao_size - mp->dmai_minxfer)
1214 		return (DDI_FAILURE);
1215 
1216 	ASSERT(cookiep);
1217 	cookiep->dmac_notused = 0;
1218 	cookiep->dmac_type = 0;
1219 	cookiep->dmac_address = (ioaddr_t)mp->dmai_mapping;
1220 	cookiep->dmac_size = mp->dmai_size;
1221 	*ccountp = 1;
1222 	*offp = (off_t)newoff;
1223 	*lenp = (uint_t)winsize;
1224 
1225 	if (newoff == mp->dmai_offset) {
1226 		/*
1227 		 * Nothing to do...
1228 		 */
1229 		return (DDI_SUCCESS);
1230 	}
1231 
1232 	if ((rval = iommu_map_window(mp, newoff, winsize)) != DDI_SUCCESS)
1233 		return (rval);
1234 
1235 	/*
1236 	 * Set this again in case iommu_map_window() has changed it
1237 	 */
1238 	cookiep->dmac_size = mp->dmai_size;
1239 
1240 	return (DDI_SUCCESS);
1241 }
1242 
1243 static int
1244 iommu_map_window(ddi_dma_impl_t *mp, off_t newoff, size_t winsize)
1245 {
1246 	uintptr_t addr = 0;
1247 	page_t *pp;
1248 	uint_t flags;
1249 	struct page **pplist = NULL;
1250 
1251 #if defined(DEBUG) && defined(IO_MEMDEBUG)
1252 	/* Free mappings for current window */
1253 	iommu_remove_mappings(mp);
1254 #endif /* DEBUG && IO_MEMDEBUG */
1255 
1256 	mp->dmai_offset = newoff;
1257 	mp->dmai_size = mp->dmai_object.dmao_size - newoff;
1258 	mp->dmai_size = MIN(mp->dmai_size, winsize);
1259 
1260 	if (mp->dmai_object.dmao_type == DMA_OTYP_VADDR ||
1261 	    mp->dmai_object.dmao_type == DMA_OTYP_BUFVADDR) {
1262 		if (mp->dmai_rflags & DMP_SHADOW) {
1263 			pplist = (struct page **)mp->dmai_minfo;
1264 			ASSERT(pplist != NULL);
1265 			pplist = pplist + (newoff >> MMU_PAGESHIFT);
1266 		} else {
1267 			addr = (uintptr_t)
1268 			    mp->dmai_object.dmao_obj.virt_obj.v_addr;
1269 			addr = (addr + newoff) & ~IOMMU_PAGEOFFSET;
1270 		}
1271 		pp = NULL;
1272 	} else {
1273 		pp = mp->dmai_object.dmao_obj.pp_obj.pp_pp;
1274 		flags = 0;
1275 		while (flags < newoff) {
1276 			pp = pp->p_next;
1277 			flags += MMU_PAGESIZE;
1278 		}
1279 	}
1280 
1281 	/* Set up mappings for next window */
1282 	if (addr) {
1283 		if (iommu_create_vaddr_mappings(mp, addr) < 0)
1284 			return (DDI_FAILURE);
1285 	} else {
1286 		if (iommu_create_pp_mappings(mp, pp, pplist) < 0)
1287 			return (DDI_FAILURE);
1288 	}
1289 
1290 	/*
1291 	 * also invalidate read stream buffer
1292 	 */
1293 	if (!(mp->dmai_rflags & DDI_DMA_CONSISTENT)) {
1294 		struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp;
1295 
1296 		sync_stream_buf(mppriv->softsp, mp->dmai_mapping,
1297 		    mp->dmai_ndvmapages, (int *)&mppriv->sync_flag,
1298 		    mppriv->phys_sync_flag);
1299 	}
1300 
1301 	return (DDI_SUCCESS);
1302 
1303 }
1304 
1305 
1306 /*ARGSUSED*/
1307 int
1308 iommu_dma_mctl(dev_info_t *dip, dev_info_t *rdip,
1309     ddi_dma_handle_t handle, enum ddi_dma_ctlops request,
1310     off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags)
1311 {
1312 	pgcnt_t npages;
1313 	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
1314 
1315 	DPRINTF(IOMMU_DMAMCTL_DEBUG, ("dma_mctl: handle %p ", (void *)mp));
1316 	switch (request) {
1317 
1318 	case DDI_DMA_SET_SBUS64:
1319 	{
1320 		struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp;
1321 
1322 		return (iommu_dma_lim_setup(dip, rdip, mppriv->softsp,
1323 		    &mp->dmai_burstsizes, (uint_t)*lenp, &mp->dmai_minxfer,
1324 		    DDI_DMA_SBUS_64BIT));
1325 	}
1326 
1327 	case DDI_DMA_RESERVE:
1328 	{
1329 		struct ddi_dma_req *dmareq = (struct ddi_dma_req *)offp;
1330 		ddi_dma_lim_t *dma_lim;
1331 		ddi_dma_handle_t *handlep;
1332 		uint_t np;
1333 		ioaddr_t ioaddr;
1334 		int i;
1335 		struct fast_dvma *iommu_fast_dvma;
1336 		struct sbus_soft_state *softsp =
1337 		    (struct sbus_soft_state *)ddi_get_soft_state(sbusp,
1338 		    ddi_get_instance(dip));
1339 
1340 		/* Some simple sanity checks */
1341 		dma_lim = dmareq->dmar_limits;
1342 		if (dma_lim->dlim_burstsizes == 0) {
1343 			DPRINTF(IOMMU_FASTDMA_RESERVE,
1344 			    ("Reserve: bad burstsizes\n"));
1345 			return (DDI_DMA_BADLIMITS);
1346 		}
1347 		if ((AHI <= ALO) || (AHI < softsp->iommu_dvma_base)) {
1348 			DPRINTF(IOMMU_FASTDMA_RESERVE,
1349 			    ("Reserve: bad limits\n"));
1350 			return (DDI_DMA_BADLIMITS);
1351 		}
1352 
1353 		np = dmareq->dmar_object.dmao_size;
1354 		mutex_enter(&softsp->dma_pool_lock);
1355 		if (np > softsp->dma_reserve) {
1356 			mutex_exit(&softsp->dma_pool_lock);
1357 			DPRINTF(IOMMU_FASTDMA_RESERVE,
1358 			    ("Reserve: dma_reserve is exhausted\n"));
1359 			return (DDI_DMA_NORESOURCES);
1360 		}
1361 
1362 		softsp->dma_reserve -= np;
1363 		mutex_exit(&softsp->dma_pool_lock);
1364 		mp = kmem_zalloc(sizeof (*mp), KM_SLEEP);
1365 		mp->dmai_rflags = DMP_BYPASSNEXUS;
1366 		mp->dmai_rdip = rdip;
1367 		mp->dmai_minxfer = dma_lim->dlim_minxfer;
1368 		mp->dmai_burstsizes = dma_lim->dlim_burstsizes;
1369 
1370 		ioaddr = (ioaddr_t)(uintptr_t)vmem_xalloc(softsp->dvma_arena,
1371 		    iommu_ptob(np), IOMMU_PAGESIZE, 0,
1372 		    dma_lim->dlim_cntr_max + 1,
1373 		    (void *)(uintptr_t)ALO, (void *)(uintptr_t)(AHI + 1),
1374 		    dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP);
1375 
1376 		if (ioaddr == 0) {
1377 			mutex_enter(&softsp->dma_pool_lock);
1378 			softsp->dma_reserve += np;
1379 			mutex_exit(&softsp->dma_pool_lock);
1380 			kmem_free(mp, sizeof (*mp));
1381 			DPRINTF(IOMMU_FASTDMA_RESERVE,
1382 			    ("Reserve: No dvma resources available\n"));
1383 			return (DDI_DMA_NOMAPPING);
1384 		}
1385 
1386 		/* create a per request structure */
1387 		iommu_fast_dvma = kmem_alloc(sizeof (struct fast_dvma),
1388 		    KM_SLEEP);
1389 
1390 		/*
1391 		 * We need to remember the size of the transfer so that
1392 		 * we can figure the virtual pages to sync when the transfer
1393 		 * is complete.
1394 		 */
1395 		iommu_fast_dvma->pagecnt = kmem_zalloc(np *
1396 		    sizeof (uint_t), KM_SLEEP);
1397 
1398 		/* Allocate a streaming cache sync flag for each index */
1399 		iommu_fast_dvma->sync_flag = kmem_zalloc(np *
1400 		    sizeof (int), KM_SLEEP);
1401 
1402 		/* Allocate a physical sync flag for each index */
1403 		iommu_fast_dvma->phys_sync_flag =
1404 		    kmem_zalloc(np * sizeof (uint64_t), KM_SLEEP);
1405 
1406 		for (i = 0; i < np; i++)
1407 			iommu_fast_dvma->phys_sync_flag[i] = va_to_pa((caddr_t)
1408 			    &iommu_fast_dvma->sync_flag[i]);
1409 
1410 		mp->dmai_mapping = ioaddr;
1411 		mp->dmai_ndvmapages = np;
1412 		iommu_fast_dvma->ops = &iommu_dvma_ops;
1413 		iommu_fast_dvma->softsp = (caddr_t)softsp;
1414 		mp->dmai_nexus_private = (caddr_t)iommu_fast_dvma;
1415 		handlep = (ddi_dma_handle_t *)objp;
1416 		*handlep = (ddi_dma_handle_t)mp;
1417 
1418 		DPRINTF(IOMMU_FASTDMA_RESERVE,
1419 		    ("Reserve: mapping object %p base addr %lx size %x\n",
1420 		    (void *)mp, mp->dmai_mapping, mp->dmai_ndvmapages));
1421 
1422 		break;
1423 	}
1424 
1425 	case DDI_DMA_RELEASE:
1426 	{
1427 		ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
1428 		uint_t np = npages = mp->dmai_ndvmapages;
1429 		ioaddr_t ioaddr = mp->dmai_mapping;
1430 		volatile uint64_t *iotte_ptr;
1431 		struct fast_dvma *iommu_fast_dvma = (struct fast_dvma *)
1432 		    mp->dmai_nexus_private;
1433 		struct sbus_soft_state *softsp = (struct sbus_soft_state *)
1434 		    iommu_fast_dvma->softsp;
1435 
1436 		ASSERT(softsp != NULL);
1437 
1438 		/* Unload stale mappings and flush stale tlb's */
1439 		iotte_ptr = IOTTE_NDX(ioaddr, softsp->soft_tsb_base_addr);
1440 
1441 		while (npages > (uint_t)0) {
1442 			*iotte_ptr = (uint64_t)0;	/* unload tte */
1443 			iommu_tlb_flush(softsp, ioaddr, 1);
1444 
1445 			npages--;
1446 			iotte_ptr++;
1447 			ioaddr += IOMMU_PAGESIZE;
1448 		}
1449 
1450 		ioaddr = (ioaddr_t)mp->dmai_mapping;
1451 		mutex_enter(&softsp->dma_pool_lock);
1452 		softsp->dma_reserve += np;
1453 		mutex_exit(&softsp->dma_pool_lock);
1454 
1455 		if (mp->dmai_rflags & DMP_NOLIMIT)
1456 			vmem_free(softsp->dvma_arena,
1457 			    (void *)(uintptr_t)ioaddr, iommu_ptob(np));
1458 		else
1459 			vmem_xfree(softsp->dvma_arena,
1460 			    (void *)(uintptr_t)ioaddr, iommu_ptob(np));
1461 
1462 		kmem_free(mp, sizeof (*mp));
1463 		kmem_free(iommu_fast_dvma->pagecnt, np * sizeof (uint_t));
1464 		kmem_free(iommu_fast_dvma->sync_flag, np * sizeof (int));
1465 		kmem_free(iommu_fast_dvma->phys_sync_flag, np *
1466 		    sizeof (uint64_t));
1467 		kmem_free(iommu_fast_dvma, sizeof (struct fast_dvma));
1468 
1469 
1470 		DPRINTF(IOMMU_FASTDMA_RESERVE,
1471 		    ("Release: Base addr %x size %x\n", ioaddr, np));
1472 		/*
1473 		 * Now that we've freed some resource,
1474 		 * if there is anybody waiting for it
1475 		 * try and get them going.
1476 		 */
1477 		if (softsp->dvma_call_list_id != 0)
1478 			ddi_run_callback(&softsp->dvma_call_list_id);
1479 
1480 		break;
1481 	}
1482 
1483 	default:
1484 		DPRINTF(IOMMU_DMAMCTL_DEBUG, ("iommu_dma_mctl: unknown option "
1485 		    "0%x\n", request));
1486 
1487 		return (DDI_FAILURE);
1488 	}
1489 	return (DDI_SUCCESS);
1490 }
1491 
1492 /*ARGSUSED*/
1493 void
1494 iommu_dvma_kaddr_load(ddi_dma_handle_t h, caddr_t a, uint_t len, uint_t index,
1495     ddi_dma_cookie_t *cp)
1496 {
1497 	uintptr_t addr;
1498 	ioaddr_t ioaddr;
1499 	uint_t offset;
1500 	pfn_t pfn;
1501 	int npages;
1502 	volatile uint64_t *iotte_ptr;
1503 	uint64_t iotte_flag = 0;
1504 	struct as *as = NULL;
1505 	extern struct as kas;
1506 	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)h;
1507 	struct fast_dvma *iommu_fast_dvma =
1508 	    (struct fast_dvma *)mp->dmai_nexus_private;
1509 	struct sbus_soft_state *softsp = (struct sbus_soft_state *)
1510 	    iommu_fast_dvma->softsp;
1511 #if defined(DEBUG) && defined(IO_MEMUSAGE)
1512 	struct io_mem_list *iomemp;
1513 	pfn_t *pfnp;
1514 #endif /* DEBUG && IO_MEMUSAGE */
1515 
1516 	ASSERT(softsp != NULL);
1517 
1518 	addr = (uintptr_t)a;
1519 	ioaddr = (ioaddr_t)(mp->dmai_mapping + iommu_ptob(index));
1520 	offset = (uint_t)(addr & IOMMU_PAGEOFFSET);
1521 	iommu_fast_dvma->pagecnt[index] = iommu_btopr(len + offset);
1522 	as = &kas;
1523 	addr &= ~IOMMU_PAGEOFFSET;
1524 	npages = iommu_btopr(len + offset);
1525 
1526 #if defined(DEBUG) && defined(IO_MEMUSAGE)
1527 	iomemp = kmem_alloc(sizeof (struct io_mem_list), KM_SLEEP);
1528 	iomemp->rdip = mp->dmai_rdip;
1529 	iomemp->ioaddr = ioaddr;
1530 	iomemp->addr = addr;
1531 	iomemp->npages = npages;
1532 	pfnp = iomemp->pfn = kmem_zalloc(sizeof (*pfnp) * (npages + 1),
1533 	    KM_SLEEP);
1534 #endif /* DEBUG && IO_MEMUSAGE */
1535 
1536 	cp->dmac_address = ioaddr | offset;
1537 	cp->dmac_size = len;
1538 
1539 	iotte_ptr = IOTTE_NDX(ioaddr, softsp->soft_tsb_base_addr);
1540 	/* read/write and streaming io on */
1541 	iotte_flag = IOTTE_VALID | IOTTE_WRITE | IOTTE_CACHE;
1542 
1543 	if (mp->dmai_rflags & DDI_DMA_CONSISTENT)
1544 		mp->dmai_rflags |= DMP_NOSYNC;
1545 	else if (!softsp->stream_buf_off)
1546 		iotte_flag |= IOTTE_STREAM;
1547 
1548 	DPRINTF(IOMMU_FASTDMA_LOAD, ("kaddr_load: ioaddr %x "
1549 	    "size %x offset %x index %x kaddr %lx\n",
1550 	    ioaddr, len, offset, index, addr));
1551 	ASSERT(npages > 0);
1552 	do {
1553 		pfn = hat_getpfnum(as->a_hat, (caddr_t)addr);
1554 		if (pfn == PFN_INVALID) {
1555 			DPRINTF(IOMMU_FASTDMA_LOAD, ("kaddr_load: invalid pfn "
1556 			    "from hat_getpfnum()\n"));
1557 		}
1558 
1559 		iommu_tlb_flush(softsp, ioaddr, 1);
1560 
1561 		/* load tte */
1562 		*iotte_ptr = ((uint64_t)pfn << IOMMU_PAGESHIFT) | iotte_flag;
1563 
1564 		npages--;
1565 		iotte_ptr++;
1566 
1567 		addr += IOMMU_PAGESIZE;
1568 		ioaddr += IOMMU_PAGESIZE;
1569 
1570 #if defined(DEBUG) && defined(IO_MEMUSAGE)
1571 		*pfnp = pfn;
1572 		pfnp++;
1573 #endif /* DEBUG && IO_MEMUSAGE */
1574 
1575 	} while (npages > 0);
1576 
1577 #if defined(DEBUG) && defined(IO_MEMUSAGE)
1578 	mutex_enter(&softsp->iomemlock);
1579 	iomemp->next = softsp->iomem;
1580 	softsp->iomem = iomemp;
1581 	mutex_exit(&softsp->iomemlock);
1582 #endif /* DEBUG && IO_MEMUSAGE */
1583 }
1584 
1585 /*ARGSUSED*/
1586 void
1587 iommu_dvma_unload(ddi_dma_handle_t h, uint_t index, uint_t view)
1588 {
1589 	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)h;
1590 	ioaddr_t ioaddr;
1591 	pgcnt_t npages;
1592 	struct fast_dvma *iommu_fast_dvma =
1593 	    (struct fast_dvma *)mp->dmai_nexus_private;
1594 	struct sbus_soft_state *softsp = (struct sbus_soft_state *)
1595 	    iommu_fast_dvma->softsp;
1596 #if defined(DEBUG) && defined(IO_MEMUSAGE)
1597 	struct io_mem_list **prevp, *walk;
1598 #endif /* DEBUG && IO_MEMUSAGE */
1599 
1600 	ASSERT(softsp != NULL);
1601 
1602 	ioaddr = (ioaddr_t)(mp->dmai_mapping + iommu_ptob(index));
1603 	npages = iommu_fast_dvma->pagecnt[index];
1604 
1605 #if defined(DEBUG) && defined(IO_MEMUSAGE)
1606 	mutex_enter(&softsp->iomemlock);
1607 	prevp = &softsp->iomem;
1608 	walk = softsp->iomem;
1609 
1610 	while (walk != NULL) {
1611 		if (walk->ioaddr == ioaddr) {
1612 			*prevp = walk->next;
1613 			break;
1614 		}
1615 		prevp = &walk->next;
1616 		walk = walk->next;
1617 	}
1618 	mutex_exit(&softsp->iomemlock);
1619 
1620 	kmem_free(walk->pfn, sizeof (pfn_t) * (npages + 1));
1621 	kmem_free(walk, sizeof (struct io_mem_list));
1622 #endif /* DEBUG && IO_MEMUSAGE */
1623 
1624 	DPRINTF(IOMMU_FASTDMA_SYNC, ("kaddr_unload: handle %p sync flag "
1625 	    "addr %p sync flag pfn %llx index %x page count %lx\n", (void *)mp,
1626 	    (void *)&iommu_fast_dvma->sync_flag[index],
1627 	    iommu_fast_dvma->phys_sync_flag[index],
1628 	    index, npages));
1629 
1630 	if ((mp->dmai_rflags & DMP_NOSYNC) != DMP_NOSYNC) {
1631 		sync_stream_buf(softsp, ioaddr, npages,
1632 		    (int *)&iommu_fast_dvma->sync_flag[index],
1633 		    iommu_fast_dvma->phys_sync_flag[index]);
1634 	}
1635 }
1636 
1637 /*ARGSUSED*/
1638 void
1639 iommu_dvma_sync(ddi_dma_handle_t h, uint_t index, uint_t view)
1640 {
1641 	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)h;
1642 	ioaddr_t ioaddr;
1643 	uint_t npages;
1644 	struct fast_dvma *iommu_fast_dvma =
1645 	    (struct fast_dvma *)mp->dmai_nexus_private;
1646 	struct sbus_soft_state *softsp = (struct sbus_soft_state *)
1647 	    iommu_fast_dvma->softsp;
1648 
1649 	if ((mp->dmai_rflags & DMP_NOSYNC) == DMP_NOSYNC)
1650 		return;
1651 
1652 	ASSERT(softsp != NULL);
1653 	ioaddr = (ioaddr_t)(mp->dmai_mapping + iommu_ptob(index));
1654 	npages = iommu_fast_dvma->pagecnt[index];
1655 
1656 	DPRINTF(IOMMU_FASTDMA_SYNC, ("kaddr_sync: handle %p, "
1657 	    "sync flag addr %p, sync flag pfn %llx\n", (void *)mp,
1658 	    (void *)&iommu_fast_dvma->sync_flag[index],
1659 	    iommu_fast_dvma->phys_sync_flag[index]));
1660 
1661 	sync_stream_buf(softsp, ioaddr, npages,
1662 	    (int *)&iommu_fast_dvma->sync_flag[index],
1663 	    iommu_fast_dvma->phys_sync_flag[index]);
1664 }
1665