xref: /titanic_44/usr/src/uts/i86pc/io/immu_dvma.c (revision 5253169e90b276216b53d82f9ba4c56334db5740)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Portions Copyright (c) 2010, Oracle and/or its affiliates.
23  * All rights reserved.
24  */
25 /*
26  * Copyright (c) 2009, Intel Corporation.
27  * All rights reserved.
28  */
29 
30 /*
31  * DVMA code
32  * This file contains Intel IOMMU code that deals with DVMA
33  * i.e. DMA remapping.
34  */
35 
36 #include <sys/sysmacros.h>
37 #include <sys/pcie.h>
38 #include <sys/pci_cfgspace.h>
39 #include <vm/hat_i86.h>
40 #include <sys/memlist.h>
41 #include <sys/acpi/acpi.h>
42 #include <sys/acpica.h>
43 #include <sys/modhash.h>
44 #include <sys/immu.h>
45 
46 #undef	TEST
47 
48 /*
49  * Macros based on PCI spec
50  */
51 #define	IMMU_PCI_REV2CLASS(r)   ((r) >> 8)  /* classcode from revid */
52 #define	IMMU_PCI_CLASS2BASE(c)  ((c) >> 16) /* baseclass from classcode */
53 #define	IMMU_PCI_CLASS2SUB(c)   (((c) >> 8) & 0xff); /* classcode */
54 
55 #define	IMMU_CONTIG_PADDR(d, p) \
56 	((d).dck_paddr && ((d).dck_paddr + IMMU_PAGESIZE) == (p))
57 
58 typedef struct dvma_arg {
59 	immu_t *dva_immu;
60 	dev_info_t *dva_rdip;
61 	dev_info_t *dva_ddip;
62 	domain_t *dva_domain;
63 	int dva_level;
64 	immu_flags_t dva_flags;
65 	list_t *dva_list;
66 	int dva_error;
67 } dvma_arg_t;
68 
69 static domain_t *domain_create(immu_t *immu, dev_info_t *ddip,
70     dev_info_t *rdip, immu_flags_t immu_flags);
71 static immu_devi_t *create_immu_devi(dev_info_t *rdip, int bus,
72     int dev, int func, immu_flags_t immu_flags);
73 static void destroy_immu_devi(immu_devi_t *immu_devi);
74 static boolean_t dvma_map(immu_t *immu, domain_t *domain, uint64_t sdvma,
75     uint64_t nvpages, dcookie_t *dcookies, int dcount, dev_info_t *rdip,
76     immu_flags_t immu_flags);
77 
78 /* Extern globals */
79 extern struct memlist  *phys_install;
80 
81 
82 /* static Globals */
83 
84 /*
85  * Used to setup DMA objects (memory regions)
86  * for DMA reads by IOMMU units
87  */
88 static ddi_dma_attr_t immu_dma_attr = {
89 	DMA_ATTR_V0,
90 	0U,
91 	0xffffffffU,
92 	0xffffffffU,
93 	MMU_PAGESIZE, /* MMU page aligned */
94 	0x1,
95 	0x1,
96 	0xffffffffU,
97 	0xffffffffU,
98 	1,
99 	4,
100 	0
101 };
102 
103 static ddi_device_acc_attr_t immu_acc_attr = {
104 	DDI_DEVICE_ATTR_V0,
105 	DDI_NEVERSWAP_ACC,
106 	DDI_STRICTORDER_ACC
107 };
108 
109 
110 /* globals private to this file */
111 static kmutex_t immu_domain_lock;
112 static list_t immu_unity_domain_list;
113 static list_t immu_xlate_domain_list;
114 
115 /* structure used to store idx into each level of the page tables */
116 typedef struct xlate {
117 	int xlt_level;
118 	uint_t xlt_idx;
119 	pgtable_t *xlt_pgtable;
120 } xlate_t;
121 
122 /* 0 is reserved by Vt-d spec. Solaris reserves 1 */
123 #define	IMMU_UNITY_DID   1
124 
125 static mod_hash_t *bdf_domain_hash;
126 
127 static domain_t *
128 bdf_domain_lookup(immu_devi_t *immu_devi)
129 {
130 	domain_t *domain;
131 	int16_t seg = immu_devi->imd_seg;
132 	int16_t bus = immu_devi->imd_bus;
133 	int16_t devfunc = immu_devi->imd_devfunc;
134 	uintptr_t bdf = (seg << 16 | bus << 8 | devfunc);
135 
136 	if (seg < 0 || bus < 0 || devfunc < 0) {
137 		return (NULL);
138 	}
139 
140 	domain = NULL;
141 	if (mod_hash_find(bdf_domain_hash,
142 	    (void *)bdf, (void *)&domain) == 0) {
143 		ASSERT(domain);
144 		ASSERT(domain->dom_did > 0);
145 		return (domain);
146 	} else {
147 		return (NULL);
148 	}
149 }
150 
151 static void
152 bdf_domain_insert(immu_devi_t *immu_devi, domain_t *domain)
153 {
154 	int16_t seg = immu_devi->imd_seg;
155 	int16_t bus = immu_devi->imd_bus;
156 	int16_t devfunc = immu_devi->imd_devfunc;
157 	uintptr_t bdf = (seg << 16 | bus << 8 | devfunc);
158 	int r;
159 
160 	if (seg < 0 || bus < 0 || devfunc < 0) {
161 		return;
162 	}
163 
164 	r = mod_hash_insert(bdf_domain_hash, (void *)bdf, (void *)domain);
165 	ASSERT(r != MH_ERR_DUPLICATE);
166 	ASSERT(r == 0);
167 }
168 
169 static int
170 match_lpc(dev_info_t *pdip, void *arg)
171 {
172 	immu_devi_t *immu_devi;
173 	dvma_arg_t *dvap = (dvma_arg_t *)arg;
174 
175 	ASSERT(dvap->dva_error == DDI_FAILURE);
176 	ASSERT(dvap->dva_ddip == NULL);
177 	ASSERT(dvap->dva_list);
178 
179 	if (list_is_empty(dvap->dva_list)) {
180 		return (DDI_WALK_TERMINATE);
181 	}
182 
183 	immu_devi = list_head(dvap->dva_list);
184 	for (; immu_devi; immu_devi = list_next(dvap->dva_list,
185 	    immu_devi)) {
186 		ASSERT(immu_devi->imd_dip);
187 		if (immu_devi->imd_dip == pdip) {
188 			dvap->dva_ddip = pdip;
189 			dvap->dva_error = DDI_SUCCESS;
190 			return (DDI_WALK_TERMINATE);
191 		}
192 	}
193 
194 	return (DDI_WALK_CONTINUE);
195 }
196 
197 static void
198 immu_devi_set_spclist(dev_info_t *dip, immu_t *immu)
199 {
200 	list_t *spclist = NULL;
201 	immu_devi_t *immu_devi;
202 
203 	ASSERT(MUTEX_HELD(&(DEVI(dip)->devi_lock)));
204 
205 	immu_devi = IMMU_DEVI(dip);
206 	if (immu_devi->imd_display == B_TRUE) {
207 		spclist = &(immu->immu_dvma_gfx_list);
208 	} else if (immu_devi->imd_lpc == B_TRUE) {
209 		spclist = &(immu->immu_dvma_lpc_list);
210 	}
211 
212 	if (spclist) {
213 		mutex_enter(&(immu->immu_lock));
214 		list_insert_head(spclist, immu_devi);
215 		mutex_exit(&(immu->immu_lock));
216 	}
217 }
218 
219 /*
220  * Set the immu_devi struct in the immu_devi field of a devinfo node
221  */
222 int
223 immu_devi_set(dev_info_t *dip, immu_flags_t immu_flags)
224 {
225 	int bus, dev, func;
226 	immu_devi_t *new_imd;
227 	immu_devi_t *immu_devi;
228 
229 	ASSERT(root_devinfo);
230 	ASSERT(dip);
231 	ASSERT(dip != root_devinfo);
232 
233 	immu_devi = immu_devi_get(dip);
234 	if (immu_devi != NULL) {
235 		return (DDI_SUCCESS);
236 	}
237 
238 	bus = dev = func = -1;
239 
240 	/*
241 	 * Assume a new immu_devi struct is needed
242 	 */
243 	if (!DEVI_IS_PCI(dip) || acpica_get_bdf(dip, &bus, &dev, &func) != 0) {
244 		/*
245 		 * No BDF. Set bus = -1 to indicate this.
246 		 * We still need to create a immu_devi struct
247 		 * though
248 		 */
249 		bus = -1;
250 		dev = 0;
251 		func = 0;
252 	}
253 
254 	new_imd = create_immu_devi(dip, bus, dev, func, immu_flags);
255 	if (new_imd  == NULL) {
256 		ddi_err(DER_WARN, dip, "Failed to create immu_devi "
257 		    "structure");
258 		return (DDI_FAILURE);
259 	}
260 
261 	/*
262 	 * Check if some other thread allocated a immu_devi while we
263 	 * didn't own the lock.
264 	 */
265 	mutex_enter(&(DEVI(dip)->devi_lock));
266 	if (IMMU_DEVI(dip) == NULL) {
267 		IMMU_DEVI_SET(dip, new_imd);
268 	} else {
269 		destroy_immu_devi(new_imd);
270 	}
271 	mutex_exit(&(DEVI(dip)->devi_lock));
272 
273 	return (DDI_SUCCESS);
274 }
275 
276 static dev_info_t *
277 get_lpc_devinfo(immu_t *immu, dev_info_t *rdip, immu_flags_t immu_flags)
278 {
279 	dvma_arg_t dvarg = {0};
280 	dvarg.dva_list = &(immu->immu_dvma_lpc_list);
281 	dvarg.dva_rdip = rdip;
282 	dvarg.dva_error = DDI_FAILURE;
283 
284 	if (immu_walk_ancestor(rdip, NULL, match_lpc,
285 	    &dvarg, NULL, immu_flags) != DDI_SUCCESS) {
286 		ddi_err(DER_MODE, rdip, "Could not walk ancestors to "
287 		    "find lpc_devinfo for ISA device");
288 		return (NULL);
289 	}
290 
291 	if (dvarg.dva_error != DDI_SUCCESS || dvarg.dva_ddip == NULL) {
292 		ddi_err(DER_MODE, rdip, "Could not find lpc_devinfo for "
293 		    "ISA device");
294 		return (NULL);
295 	}
296 
297 	return (dvarg.dva_ddip);
298 }
299 
300 static dev_info_t *
301 get_gfx_devinfo(dev_info_t *rdip)
302 {
303 	immu_t *immu;
304 	immu_devi_t *immu_devi;
305 	list_t *list_gfx;
306 
307 	/*
308 	 * The GFX device may not be on the same IMMU unit as "agpgart"
309 	 * so search globally
310 	 */
311 	immu_devi = NULL;
312 	immu = list_head(&immu_list);
313 	for (; immu; immu = list_next(&immu_list, immu)) {
314 		list_gfx = &(immu->immu_dvma_gfx_list);
315 		if (!list_is_empty(list_gfx)) {
316 			immu_devi = list_head(list_gfx);
317 			break;
318 		}
319 	}
320 
321 	if (immu_devi == NULL) {
322 		ddi_err(DER_WARN, rdip, "IMMU: No GFX device. "
323 		    "Cannot redirect agpgart");
324 		return (NULL);
325 	}
326 
327 	/* list is not empty we checked above */
328 	ASSERT(immu_devi);
329 	ASSERT(immu_devi->imd_dip);
330 
331 	ddi_err(DER_LOG, rdip, "IMMU: GFX redirect to %s",
332 	    ddi_node_name(immu_devi->imd_dip));
333 
334 	return (immu_devi->imd_dip);
335 }
336 
337 static immu_flags_t
338 dma_to_immu_flags(struct ddi_dma_req *dmareq)
339 {
340 	immu_flags_t flags = 0;
341 
342 	if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
343 		flags |= IMMU_FLAGS_SLEEP;
344 	} else {
345 		flags |= IMMU_FLAGS_NOSLEEP;
346 	}
347 
348 #ifdef BUGGY_DRIVERS
349 
350 	flags |= (IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);
351 
352 #else
353 	/*
354 	 * Read and write flags need to be reversed.
355 	 * DMA_READ means read from device and write
356 	 * to memory. So DMA read means DVMA write.
357 	 */
358 	if (dmareq->dmar_flags & DDI_DMA_READ)
359 		flags |= IMMU_FLAGS_WRITE;
360 
361 	if (dmareq->dmar_flags & DDI_DMA_WRITE)
362 		flags |= IMMU_FLAGS_READ;
363 
364 	/*
365 	 * Some buggy drivers specify neither READ or WRITE
366 	 * For such drivers set both read and write permissions
367 	 */
368 	if ((dmareq->dmar_flags & (DDI_DMA_READ | DDI_DMA_WRITE)) == 0) {
369 		flags |= (IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);
370 	}
371 #endif
372 
373 	return (flags);
374 }
375 
376 int
377 pgtable_ctor(void *buf, void *arg, int kmflag)
378 {
379 	size_t actual_size = 0;
380 	pgtable_t *pgtable;
381 	int (*dmafp)(caddr_t);
382 	caddr_t vaddr;
383 	void *next;
384 
385 	ASSERT(buf);
386 	ASSERT(arg == NULL);
387 
388 	pgtable = (pgtable_t *)buf;
389 
390 	dmafp = (kmflag & KM_NOSLEEP) ? DDI_DMA_DONTWAIT : DDI_DMA_SLEEP;
391 
392 	next = kmem_zalloc(IMMU_PAGESIZE, kmflag);
393 	if (next == NULL) {
394 		return (-1);
395 	}
396 
397 	ASSERT(root_devinfo);
398 	if (ddi_dma_alloc_handle(root_devinfo, &immu_dma_attr,
399 	    dmafp, NULL, &pgtable->hwpg_dmahdl) != DDI_SUCCESS) {
400 		kmem_free(next, IMMU_PAGESIZE);
401 		return (-1);
402 	}
403 
404 	if (ddi_dma_mem_alloc(pgtable->hwpg_dmahdl, IMMU_PAGESIZE,
405 	    &immu_acc_attr, DDI_DMA_CONSISTENT | IOMEM_DATA_UNCACHED,
406 	    dmafp, NULL, &vaddr, &actual_size,
407 	    &pgtable->hwpg_memhdl) != DDI_SUCCESS) {
408 		ddi_dma_free_handle(&pgtable->hwpg_dmahdl);
409 		kmem_free(next, IMMU_PAGESIZE);
410 		return (-1);
411 	}
412 
413 	/*
414 	 * Memory allocation failure. Maybe a temporary condition
415 	 * so return error rather than panic, so we can try again
416 	 */
417 	if (actual_size < IMMU_PAGESIZE) {
418 		ddi_dma_mem_free(&pgtable->hwpg_memhdl);
419 		ddi_dma_free_handle(&pgtable->hwpg_dmahdl);
420 		kmem_free(next, IMMU_PAGESIZE);
421 		return (-1);
422 	}
423 
424 	pgtable->hwpg_paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, vaddr));
425 	pgtable->hwpg_vaddr = vaddr;
426 	pgtable->swpg_next_array = next;
427 
428 	rw_init(&(pgtable->swpg_rwlock), NULL, RW_DEFAULT, NULL);
429 
430 	return (0);
431 }
432 
433 void
434 pgtable_dtor(void *buf, void *arg)
435 {
436 	pgtable_t *pgtable;
437 
438 	ASSERT(buf);
439 	ASSERT(arg == NULL);
440 
441 	pgtable = (pgtable_t *)buf;
442 	ASSERT(pgtable->swpg_next_array);
443 
444 	/* destroy will panic if lock is held. */
445 	rw_destroy(&(pgtable->swpg_rwlock));
446 
447 	ddi_dma_mem_free(&pgtable->hwpg_memhdl);
448 	ddi_dma_free_handle(&pgtable->hwpg_dmahdl);
449 	kmem_free(pgtable->swpg_next_array, IMMU_PAGESIZE);
450 
451 	/* don't zero out hwpg_vaddr and swpg_next_array for debugging */
452 }
453 
454 /*
455  * pgtable_alloc()
456  *	alloc a IOMMU pgtable structure.
457  *	This same struct is used for root and context tables as well.
458  *	This routine allocs the f/ollowing:
459  *	- a pgtable_t struct
460  *	- a HW page which holds PTEs/entries which is accesssed by HW
461  *        so we set up DMA for this page
462  *	- a SW page which is only for our bookeeping
463  *        (for example to  hold pointers to the next level pgtable).
464  *        So a simple kmem_alloc suffices
465  */
466 static pgtable_t *
467 pgtable_alloc(immu_t *immu, immu_flags_t immu_flags)
468 {
469 	pgtable_t *pgtable;
470 	int kmflags;
471 
472 	ASSERT(immu);
473 
474 	kmflags = (immu_flags & IMMU_FLAGS_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP;
475 
476 	pgtable = kmem_cache_alloc(immu_pgtable_cache, kmflags);
477 	if (pgtable == NULL) {
478 		return (NULL);
479 	}
480 	return (pgtable);
481 }
482 
483 static void
484 pgtable_zero(immu_t *immu, pgtable_t *pgtable)
485 {
486 	bzero(pgtable->hwpg_vaddr, IMMU_PAGESIZE);
487 	bzero(pgtable->swpg_next_array, IMMU_PAGESIZE);
488 
489 	/* Dont need to flush the write we will flush when we use the entry */
490 	immu_regs_cpu_flush(immu, pgtable->hwpg_vaddr, IMMU_PAGESIZE);
491 }
492 
493 static void
494 pgtable_free(immu_t *immu, pgtable_t *pgtable)
495 {
496 	ASSERT(immu);
497 	ASSERT(pgtable);
498 
499 	kmem_cache_free(immu_pgtable_cache, pgtable);
500 }
501 
502 /*
503  * Function to identify a display device from the PCI class code
504  */
505 static boolean_t
506 device_is_display(uint_t classcode)
507 {
508 	static uint_t disp_classes[] = {
509 		0x000100,
510 		0x030000,
511 		0x030001
512 	};
513 	int i, nclasses = sizeof (disp_classes) / sizeof (uint_t);
514 
515 	for (i = 0; i < nclasses; i++) {
516 		if (classcode == disp_classes[i])
517 			return (B_TRUE);
518 	}
519 	return (B_FALSE);
520 }
521 
522 /*
523  * Function that determines if device is PCIEX and/or PCIEX bridge
524  */
525 static boolean_t
526 device_is_pciex(
527 	uchar_t bus, uchar_t dev, uchar_t func, boolean_t *is_pcib)
528 {
529 	ushort_t cap;
530 	ushort_t capsp;
531 	ushort_t cap_count = PCI_CAP_MAX_PTR;
532 	ushort_t status;
533 	boolean_t is_pciex = B_FALSE;
534 
535 	*is_pcib = B_FALSE;
536 
537 	status = pci_getw_func(bus, dev, func, PCI_CONF_STAT);
538 	if (!(status & PCI_STAT_CAP))
539 		return (B_FALSE);
540 
541 	capsp = pci_getb_func(bus, dev, func, PCI_CONF_CAP_PTR);
542 	while (cap_count-- && capsp >= PCI_CAP_PTR_OFF) {
543 		capsp &= PCI_CAP_PTR_MASK;
544 		cap = pci_getb_func(bus, dev, func, capsp);
545 
546 		if (cap == PCI_CAP_ID_PCI_E) {
547 			status = pci_getw_func(bus, dev, func, capsp + 2);
548 			/*
549 			 * See section 7.8.2 of PCI-Express Base Spec v1.0a
550 			 * for Device/Port Type.
551 			 * PCIE_PCIECAP_DEV_TYPE_PCIE2PCI implies that the
552 			 * device is a PCIE2PCI bridge
553 			 */
554 			*is_pcib =
555 			    ((status & PCIE_PCIECAP_DEV_TYPE_MASK) ==
556 			    PCIE_PCIECAP_DEV_TYPE_PCIE2PCI) ? B_TRUE : B_FALSE;
557 			is_pciex = B_TRUE;
558 		}
559 
560 		capsp = (*pci_getb_func)(bus, dev, func,
561 		    capsp + PCI_CAP_NEXT_PTR);
562 	}
563 
564 	return (is_pciex);
565 }
566 
567 
568 /*
569  * immu_dvma_get_immu()
570  *   get the immu unit structure for a dev_info node
571  */
572 immu_t *
573 immu_dvma_get_immu(dev_info_t *dip, immu_flags_t immu_flags)
574 {
575 	immu_devi_t *immu_devi;
576 	immu_t *immu;
577 
578 	/*
579 	 * check if immu unit was already found earlier.
580 	 * If yes, then it will be stashed in immu_devi struct.
581 	 */
582 	immu_devi = immu_devi_get(dip);
583 	if (immu_devi == NULL) {
584 		if (immu_devi_set(dip, immu_flags) != DDI_SUCCESS) {
585 			/*
586 			 * May fail because of low memory. Return error rather
587 			 * than panic as we want driver to rey again later
588 			 */
589 			ddi_err(DER_PANIC, dip, "immu_dvma_get_immu: "
590 			    "No immu_devi structure");
591 			/*NOTREACHED*/
592 		}
593 		immu_devi = immu_devi_get(dip);
594 		ASSERT(immu_devi);
595 	}
596 
597 	mutex_enter(&(DEVI(dip)->devi_lock));
598 	if (immu_devi->imd_immu) {
599 		immu = immu_devi->imd_immu;
600 		mutex_exit(&(DEVI(dip)->devi_lock));
601 		return (immu);
602 	}
603 	mutex_exit(&(DEVI(dip)->devi_lock));
604 
605 	immu = immu_dmar_get_immu(dip);
606 	if (immu == NULL) {
607 		ddi_err(DER_PANIC, dip, "immu_dvma_get_immu: "
608 		    "Cannot find immu_t for device");
609 		/*NOTREACHED*/
610 	}
611 
612 	/*
613 	 * Check if some other thread found immu
614 	 * while lock was not held
615 	 */
616 	immu_devi = immu_devi_get(dip);
617 	/* immu_devi should be present as we found it earlier */
618 	if (immu_devi == NULL) {
619 		ddi_err(DER_PANIC, dip,
620 		    "immu_dvma_get_immu: No immu_devi structure");
621 		/*NOTREACHED*/
622 	}
623 
624 	mutex_enter(&(DEVI(dip)->devi_lock));
625 	if (immu_devi->imd_immu == NULL) {
626 		/* nobody else set it, so we should do it */
627 		immu_devi->imd_immu = immu;
628 		immu_devi_set_spclist(dip, immu);
629 	} else {
630 		/*
631 		 * if some other thread got immu before
632 		 * us, it should get the same results
633 		 */
634 		if (immu_devi->imd_immu != immu) {
635 			ddi_err(DER_PANIC, dip, "Multiple "
636 			    "immu units found for device. Expected (%p), "
637 			    "actual (%p)", (void *)immu,
638 			    (void *)immu_devi->imd_immu);
639 			mutex_exit(&(DEVI(dip)->devi_lock));
640 			/*NOTREACHED*/
641 		}
642 	}
643 	mutex_exit(&(DEVI(dip)->devi_lock));
644 
645 	return (immu);
646 }
647 
648 
649 /* ############################# IMMU_DEVI code ############################ */
650 
651 /*
652  * Allocate a immu_devi structure and initialize it
653  */
654 static immu_devi_t *
655 create_immu_devi(dev_info_t *rdip, int bus, int dev, int func,
656     immu_flags_t immu_flags)
657 {
658 	uchar_t baseclass, subclass;
659 	uint_t classcode, revclass;
660 	immu_devi_t *immu_devi;
661 	boolean_t pciex = B_FALSE;
662 	int kmflags;
663 	boolean_t is_pcib = B_FALSE;
664 
665 	/* bus ==  -1 indicate non-PCI device (no BDF) */
666 	ASSERT(bus == -1 || bus >= 0);
667 	ASSERT(dev >= 0);
668 	ASSERT(func >= 0);
669 
670 	kmflags = (immu_flags & IMMU_FLAGS_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP;
671 	immu_devi = kmem_zalloc(sizeof (immu_devi_t), kmflags);
672 	if (immu_devi == NULL) {
673 		ddi_err(DER_WARN, rdip, "Failed to allocate memory for "
674 		    "Intel IOMMU immu_devi structure");
675 		return (NULL);
676 	}
677 	immu_devi->imd_dip = rdip;
678 	immu_devi->imd_seg = 0; /* Currently seg can only be 0 */
679 	immu_devi->imd_bus = bus;
680 	immu_devi->imd_pcib_type = IMMU_PCIB_BAD;
681 
682 	if (bus == -1) {
683 		immu_devi->imd_pcib_type = IMMU_PCIB_NOBDF;
684 		return (immu_devi);
685 	}
686 
687 	immu_devi->imd_devfunc = IMMU_PCI_DEVFUNC(dev, func);
688 	immu_devi->imd_sec = 0;
689 	immu_devi->imd_sub = 0;
690 
691 	revclass = pci_getl_func(bus, dev, func, PCI_CONF_REVID);
692 
693 	classcode = IMMU_PCI_REV2CLASS(revclass);
694 	baseclass = IMMU_PCI_CLASS2BASE(classcode);
695 	subclass = IMMU_PCI_CLASS2SUB(classcode);
696 
697 	if (baseclass == PCI_CLASS_BRIDGE && subclass == PCI_BRIDGE_PCI) {
698 
699 		immu_devi->imd_sec = pci_getb_func(bus, dev, func,
700 		    PCI_BCNF_SECBUS);
701 		immu_devi->imd_sub = pci_getb_func(bus, dev, func,
702 		    PCI_BCNF_SUBBUS);
703 
704 		pciex = device_is_pciex(bus, dev, func, &is_pcib);
705 		if (pciex  == B_TRUE && is_pcib == B_TRUE) {
706 			immu_devi->imd_pcib_type = IMMU_PCIB_PCIE_PCI;
707 		} else if (pciex == B_TRUE) {
708 			immu_devi->imd_pcib_type = IMMU_PCIB_PCIE_PCIE;
709 		} else {
710 			immu_devi->imd_pcib_type = IMMU_PCIB_PCI_PCI;
711 		}
712 	} else {
713 		immu_devi->imd_pcib_type = IMMU_PCIB_ENDPOINT;
714 	}
715 
716 	/* check for certain special devices */
717 	immu_devi->imd_display = device_is_display(classcode);
718 
719 	immu_devi->imd_lpc = ((baseclass == PCI_CLASS_BRIDGE) &&
720 	    (subclass == PCI_BRIDGE_ISA)) ? B_TRUE : B_FALSE;
721 
722 	immu_devi->imd_domain = NULL;
723 
724 	immu_devi->imd_dvma_flags = immu_global_dvma_flags;
725 
726 	return (immu_devi);
727 }
728 
729 static void
730 destroy_immu_devi(immu_devi_t *immu_devi)
731 {
732 	kmem_free(immu_devi, sizeof (immu_devi_t));
733 }
734 
735 static domain_t *
736 immu_devi_domain(dev_info_t *rdip, dev_info_t **ddipp)
737 {
738 	immu_devi_t *immu_devi;
739 	domain_t *domain;
740 	dev_info_t *ddip;
741 
742 	ASSERT(rdip);
743 	ASSERT(ddipp);
744 
745 	*ddipp = NULL;
746 
747 	immu_devi = immu_devi_get(rdip);
748 	if (immu_devi == NULL) {
749 		return (NULL);
750 	}
751 
752 	mutex_enter(&(DEVI(rdip)->devi_lock));
753 	domain = immu_devi->imd_domain;
754 	ddip = immu_devi->imd_ddip;
755 	mutex_exit(&(DEVI(rdip)->devi_lock));
756 
757 	if (domain) {
758 		ASSERT(domain->dom_did > 0);
759 		ASSERT(ddip);
760 		*ddipp = ddip;
761 	}
762 
763 	return (domain);
764 
765 }
766 
767 /* ############################# END IMMU_DEVI code ######################## */
768 /* ############################# DOMAIN code ############################### */
769 
770 /*
771  * This routine always succeeds
772  */
773 static int
774 did_alloc(immu_t *immu, dev_info_t *rdip,
775     dev_info_t *ddip, immu_flags_t immu_flags)
776 {
777 	int did;
778 
779 	ASSERT(immu);
780 	ASSERT(rdip);
781 	ASSERT(rdip != root_devinfo);
782 
783 	did = (uintptr_t)vmem_alloc(immu->immu_did_arena, 1,
784 	    (immu_flags & IMMU_FLAGS_NOSLEEP) ? VM_NOSLEEP : VM_SLEEP);
785 
786 	if (did == 0) {
787 		ASSERT(immu->immu_unity_domain);
788 		ASSERT(immu->immu_unity_domain->dom_did > 0);
789 		ddi_err(DER_WARN, rdip, "device domain-id alloc error"
790 		    " domain-device: %s%d. immu unit is %s. Using "
791 		    "unity domain with domain-id (%d)",
792 		    ddi_driver_name(ddip), ddi_get_instance(ddip),
793 		    immu->immu_name, immu->immu_unity_domain->dom_did);
794 		did = immu->immu_unity_domain->dom_did;
795 	}
796 
797 	return (did);
798 }
799 
800 static int
801 get_branch_domain(dev_info_t *pdip, void *arg)
802 {
803 	immu_devi_t *immu_devi;
804 	domain_t *domain;
805 	dev_info_t *ddip;
806 	immu_t *immu;
807 	dvma_arg_t *dvp = (dvma_arg_t *)arg;
808 
809 	ASSERT(pdip);
810 	ASSERT(dvp);
811 	ASSERT(dvp->dva_rdip);
812 
813 	/*
814 	 * The field dvp->dva_rdip is a work-in-progress
815 	 * and gets updated as we walk up the ancestor
816 	 * tree. The final ddip is set only when we reach
817 	 * the top of the tree. So the dvp->dva_ddip field cannot
818 	 * be relied on until we reach the top of the field.
819 	 */
820 
821 	/* immu_devi may not be set. */
822 	immu_devi = immu_devi_get(pdip);
823 	if (immu_devi == NULL) {
824 		if (immu_devi_set(pdip, dvp->dva_flags) != DDI_SUCCESS) {
825 			dvp->dva_error = DDI_FAILURE;
826 			return (DDI_WALK_TERMINATE);
827 		}
828 	}
829 
830 	immu_devi = immu_devi_get(pdip);
831 	ASSERT(immu_devi);
832 	immu = immu_devi->imd_immu;
833 	if (immu == NULL) {
834 		immu = immu_dvma_get_immu(pdip, dvp->dva_flags);
835 		ASSERT(immu);
836 	}
837 
838 	/*
839 	 * If we encounter a PCIE_PCIE bridge *ANCESTOR* we need to
840 	 * terminate the walk (since the device under the PCIE bridge
841 	 * is a PCIE device and has an independent entry in the
842 	 * root/context table)
843 	 */
844 	if (dvp->dva_rdip != pdip &&
845 	    immu_devi->imd_pcib_type == IMMU_PCIB_PCIE_PCIE) {
846 		return (DDI_WALK_TERMINATE);
847 	}
848 
849 	/*
850 	 * In order to be a domain-dim, it must be a PCI device i.e.
851 	 * must have valid BDF. This also eliminates the root complex.
852 	 */
853 	if (immu_devi->imd_pcib_type != IMMU_PCIB_BAD &&
854 	    immu_devi->imd_pcib_type != IMMU_PCIB_NOBDF) {
855 		ASSERT(immu_devi->imd_bus >= 0);
856 		ASSERT(immu_devi->imd_devfunc >= 0);
857 		dvp->dva_ddip = pdip;
858 	}
859 
860 	if (immu_devi->imd_display == B_TRUE ||
861 	    (dvp->dva_flags & IMMU_FLAGS_UNITY)) {
862 		dvp->dva_domain = immu->immu_unity_domain;
863 		/* continue walking to find ddip */
864 		return (DDI_WALK_CONTINUE);
865 	}
866 
867 	mutex_enter(&(DEVI(pdip)->devi_lock));
868 	domain = immu_devi->imd_domain;
869 	ddip = immu_devi->imd_ddip;
870 	mutex_exit(&(DEVI(pdip)->devi_lock));
871 
872 	if (domain && ddip) {
873 		/* if domain is set, it must be the same */
874 		if (dvp->dva_domain) {
875 			ASSERT(domain == dvp->dva_domain);
876 		}
877 		dvp->dva_domain = domain;
878 		dvp->dva_ddip = ddip;
879 		return (DDI_WALK_TERMINATE);
880 	}
881 
882 	/* immu_devi either has both set or both clear */
883 	ASSERT(domain == NULL);
884 	ASSERT(ddip == NULL);
885 
886 	/* Domain may already be set, continue walking so that ddip gets set */
887 	if (dvp->dva_domain) {
888 		return (DDI_WALK_CONTINUE);
889 	}
890 
891 	/* domain is not set in either immu_devi or dvp */
892 	domain = bdf_domain_lookup(immu_devi);
893 	if (domain == NULL) {
894 		return (DDI_WALK_CONTINUE);
895 	}
896 
897 	/* ok, the BDF hash had a domain for this BDF. */
898 
899 	/* Grab lock again to check if something else set immu_devi fields */
900 	mutex_enter(&(DEVI(pdip)->devi_lock));
901 	if (immu_devi->imd_domain != NULL) {
902 		ASSERT(immu_devi->imd_domain == domain);
903 		dvp->dva_domain = domain;
904 	} else {
905 		dvp->dva_domain = domain;
906 	}
907 	mutex_exit(&(DEVI(pdip)->devi_lock));
908 
909 	/*
910 	 * walk upwards until the topmost PCI bridge is found
911 	 */
912 	return (DDI_WALK_CONTINUE);
913 
914 }
915 
916 static void
917 map_unity_domain(domain_t *domain)
918 {
919 	struct memlist *mp;
920 	uint64_t start;
921 	uint64_t npages;
922 	dcookie_t dcookies[1] = {0};
923 	int dcount = 0;
924 
925 	ASSERT(domain);
926 	ASSERT(domain->dom_did == IMMU_UNITY_DID);
927 
928 	/*
929 	 * We call into routines that grab the lock so we should
930 	 * not be called with the lock held. This does not matter
931 	 * much since, no else has a reference to this domain
932 	 */
933 	ASSERT(!rw_lock_held(&(domain->dom_pgtable_rwlock)));
934 
935 	/*
936 	 * UNITY arenas are a mirror of the physical memory
937 	 * installed on the system.
938 	 */
939 
940 #ifdef BUGGY_DRIVERS
941 	/*
942 	 * Dont skip page0. Some broken HW/FW access it.
943 	 */
944 	dcookies[0].dck_paddr = 0;
945 	dcookies[0].dck_npages = 1;
946 	dcount = 1;
947 	(void) dvma_map(domain->dom_immu, domain, 0, 1, dcookies, dcount, NULL,
948 	    IMMU_FLAGS_READ | IMMU_FLAGS_WRITE | IMMU_FLAGS_PAGE1);
949 #endif
950 
951 	memlist_read_lock();
952 
953 	mp = phys_install;
954 
955 	if (mp->ml_address == 0) {
956 		/* since we already mapped page1 above */
957 		start = IMMU_PAGESIZE;
958 	} else {
959 		start = mp->ml_address;
960 	}
961 	npages = mp->ml_size/IMMU_PAGESIZE + 1;
962 
963 	dcookies[0].dck_paddr = start;
964 	dcookies[0].dck_npages = npages;
965 	dcount = 1;
966 	(void) dvma_map(domain->dom_immu, domain, start, npages, dcookies,
967 	    dcount, NULL, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);
968 
969 	ddi_err(DER_LOG, NULL, "IMMU: mapping PHYS span [0x%" PRIx64
970 	    " - 0x%" PRIx64 "]", start, start + mp->ml_size);
971 
972 	mp = mp->ml_next;
973 	while (mp) {
974 		ddi_err(DER_LOG, NULL, "IMMU: mapping PHYS span [0x%" PRIx64
975 		    " - 0x%" PRIx64 "]", mp->ml_address,
976 		    mp->ml_address + mp->ml_size);
977 
978 		start = mp->ml_address;
979 		npages = mp->ml_size/IMMU_PAGESIZE + 1;
980 
981 		dcookies[0].dck_paddr = start;
982 		dcookies[0].dck_npages = npages;
983 		dcount = 1;
984 		(void) dvma_map(domain->dom_immu, domain, start, npages,
985 		    dcookies, dcount, NULL, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);
986 		mp = mp->ml_next;
987 	}
988 
989 	mp = bios_rsvd;
990 	while (mp) {
991 		ddi_err(DER_LOG, NULL, "IMMU: mapping PHYS span [0x%" PRIx64
992 		    " - 0x%" PRIx64 "]", mp->ml_address,
993 		    mp->ml_address + mp->ml_size);
994 
995 		start = mp->ml_address;
996 		npages = mp->ml_size/IMMU_PAGESIZE + 1;
997 
998 		dcookies[0].dck_paddr = start;
999 		dcookies[0].dck_npages = npages;
1000 		dcount = 1;
1001 		(void) dvma_map(domain->dom_immu, domain, start, npages,
1002 		    dcookies, dcount, NULL, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);
1003 
1004 		mp = mp->ml_next;
1005 	}
1006 
1007 	memlist_read_unlock();
1008 }
1009 
1010 /*
1011  * create_xlate_arena()
1012  * 	Create the dvma arena for a domain with translation
1013  *	mapping
1014  */
1015 static void
1016 create_xlate_arena(immu_t *immu, domain_t *domain,
1017     dev_info_t *rdip, immu_flags_t immu_flags)
1018 {
1019 	char *arena_name;
1020 	struct memlist *mp;
1021 	int vmem_flags;
1022 	uint64_t start;
1023 	uint_t mgaw;
1024 	uint64_t size;
1025 	uint64_t maxaddr;
1026 	void *vmem_ret;
1027 
1028 	arena_name = domain->dom_dvma_arena_name;
1029 
1030 	/* Note, don't do sizeof (arena_name) - it is just a pointer */
1031 	(void) snprintf(arena_name,
1032 	    sizeof (domain->dom_dvma_arena_name),
1033 	    "%s-domain-%d-xlate-DVMA-arena", immu->immu_name,
1034 	    domain->dom_did);
1035 
1036 	vmem_flags = (immu_flags & IMMU_FLAGS_NOSLEEP) ? VM_NOSLEEP : VM_SLEEP;
1037 
1038 	/*
1039 	 * No one else has access to this domain.
1040 	 * So no domain locks needed
1041 	 */
1042 	ASSERT(!rw_lock_held(&(domain->dom_pgtable_rwlock)));
1043 
1044 	/* Restrict mgaddr (max guest addr) to MGAW */
1045 	mgaw = IMMU_CAP_MGAW(immu->immu_regs_cap);
1046 
1047 	/*
1048 	 * To ensure we avoid ioapic and PCI MMIO ranges we just
1049 	 * use the physical memory address range of the system as the
1050 	 * range
1051 	 */
1052 	maxaddr = ((uint64_t)1 << mgaw);
1053 
1054 	memlist_read_lock();
1055 
1056 	mp = phys_install;
1057 
1058 	if (mp->ml_address == 0)
1059 		start = MMU_PAGESIZE;
1060 	else
1061 		start = mp->ml_address;
1062 
1063 	if (start + mp->ml_size > maxaddr)
1064 		size = maxaddr - start;
1065 	else
1066 		size = mp->ml_size;
1067 
1068 	ddi_err(DER_VERB, rdip,
1069 	    "%s: Creating dvma vmem arena [0x%" PRIx64
1070 	    " - 0x%" PRIx64 "]", arena_name, start, start + size);
1071 
1072 	ASSERT(domain->dom_dvma_arena == NULL);
1073 
1074 	/*
1075 	 * We always allocate in quanta of IMMU_PAGESIZE
1076 	 */
1077 	domain->dom_dvma_arena = vmem_create(arena_name,
1078 	    (void *)(uintptr_t)start,	/* start addr */
1079 	    size,			/* size */
1080 	    IMMU_PAGESIZE,		/* quantum */
1081 	    NULL,			/* afunc */
1082 	    NULL,			/* ffunc */
1083 	    NULL,			/* source */
1084 	    0,				/* qcache_max */
1085 	    vmem_flags);
1086 
1087 	if (domain->dom_dvma_arena == NULL) {
1088 		ddi_err(DER_PANIC, rdip,
1089 		    "Failed to allocate DVMA arena(%s) "
1090 		    "for domain ID (%d)", arena_name, domain->dom_did);
1091 		/*NOTREACHED*/
1092 	}
1093 
1094 	mp = mp->ml_next;
1095 	while (mp) {
1096 
1097 		if (mp->ml_address == 0)
1098 			start = MMU_PAGESIZE;
1099 		else
1100 			start = mp->ml_address;
1101 
1102 		if (start + mp->ml_size > maxaddr)
1103 			size = maxaddr - start;
1104 		else
1105 			size = mp->ml_size;
1106 
1107 		ddi_err(DER_VERB, rdip,
1108 		    "%s: Adding dvma vmem span [0x%" PRIx64
1109 		    " - 0x%" PRIx64 "]", arena_name, start,
1110 		    start + size);
1111 
1112 		vmem_ret = vmem_add(domain->dom_dvma_arena,
1113 		    (void *)(uintptr_t)start, size,  vmem_flags);
1114 
1115 		if (vmem_ret == NULL) {
1116 			ddi_err(DER_PANIC, rdip,
1117 			    "Failed to allocate DVMA arena(%s) "
1118 			    "for domain ID (%d)",
1119 			    arena_name, domain->dom_did);
1120 			/*NOTREACHED*/
1121 		}
1122 		mp = mp->ml_next;
1123 	}
1124 	memlist_read_unlock();
1125 }
1126 
1127 /* ################################### DOMAIN CODE ######################### */
1128 
1129 /*
1130  * Set the domain and domain-dip for a dip
1131  */
1132 static void
1133 set_domain(
1134 	dev_info_t *dip,
1135 	dev_info_t *ddip,
1136 	domain_t *domain)
1137 {
1138 	immu_devi_t *immu_devi;
1139 	domain_t *fdomain;
1140 	dev_info_t *fddip;
1141 
1142 	ASSERT(dip);
1143 	ASSERT(ddip);
1144 	ASSERT(domain);
1145 	ASSERT(domain->dom_did > 0); /* must be an initialized domain */
1146 
1147 	immu_devi = immu_devi_get(dip);
1148 	ASSERT(immu_devi);
1149 
1150 	mutex_enter(&(DEVI(dip)->devi_lock));
1151 	fddip = immu_devi->imd_ddip;
1152 	fdomain = immu_devi->imd_domain;
1153 
1154 	if (fddip) {
1155 		ASSERT(fddip == ddip);
1156 	} else {
1157 		immu_devi->imd_ddip = ddip;
1158 	}
1159 
1160 	if (fdomain) {
1161 		ASSERT(fdomain == domain);
1162 	} else {
1163 		immu_devi->imd_domain = domain;
1164 	}
1165 	mutex_exit(&(DEVI(dip)->devi_lock));
1166 }
1167 
1168 /*
1169  * device_domain()
1170  * 	Get domain for a device. The domain may be global in which case it
1171  *	is shared between all IOMMU units. Due to potential AGAW differences
1172  *      between IOMMU units, such global domains *have to be* UNITY mapping
1173  *      domains. Alternatively, the domain may be local to a IOMMU unit.
1174  *	Local domains may be shared or immu_devi, although the
1175  *      scope of sharing
1176  *	is restricted to devices controlled by the IOMMU unit to
1177  *      which the domain
1178  *	belongs. If shared, they (currently) have to be UNITY domains. If
1179  *      immu_devi a domain may be either UNITY or translation (XLATE) domain.
1180  */
1181 static domain_t *
1182 device_domain(dev_info_t *rdip, dev_info_t **ddipp, immu_flags_t immu_flags)
1183 {
1184 	dev_info_t *ddip; /* topmost dip in domain i.e. domain owner */
1185 	immu_t *immu;
1186 	domain_t *domain;
1187 	dvma_arg_t dvarg = {0};
1188 	int level;
1189 
1190 	ASSERT(rdip);
1191 
1192 	*ddipp = NULL;
1193 
1194 	/*
1195 	 * Check if the domain is already set. This is usually true
1196 	 * if this is not the first DVMA transaction.
1197 	 */
1198 	ddip = NULL;
1199 	domain = immu_devi_domain(rdip, &ddip);
1200 	if (domain) {
1201 		ASSERT(domain->dom_did > 0);
1202 		ASSERT(ddip);
1203 		*ddipp = ddip;
1204 		return (domain);
1205 	}
1206 
1207 	immu = immu_dvma_get_immu(rdip, immu_flags);
1208 	if (immu == NULL) {
1209 		/*
1210 		 * possible that there is no IOMMU unit for this device
1211 		 * - BIOS bugs are one example.
1212 		 */
1213 		ddi_err(DER_WARN, rdip, "No IMMU unit found for device");
1214 		return (NULL);
1215 	}
1216 
1217 	immu_flags |= immu_devi_get(rdip)->imd_dvma_flags;
1218 
1219 	dvarg.dva_rdip = rdip;
1220 	dvarg.dva_ddip = NULL;
1221 	dvarg.dva_domain = NULL;
1222 	dvarg.dva_flags = immu_flags;
1223 	level = 0;
1224 	if (immu_walk_ancestor(rdip, NULL, get_branch_domain,
1225 	    &dvarg, &level, immu_flags) != DDI_SUCCESS) {
1226 		/*
1227 		 * maybe low memory. return error,
1228 		 * so driver tries again later
1229 		 */
1230 		return (NULL);
1231 	}
1232 
1233 	/* should have walked at least 1 dip (i.e. edip) */
1234 	ASSERT(level > 0);
1235 
1236 	ddip = dvarg.dva_ddip;	/* must be present */
1237 	domain = dvarg.dva_domain;	/* may be NULL */
1238 
1239 	/*
1240 	 * We may find the domain during our ancestor walk on any one of our
1241 	 * ancestor dips, If the domain is found then the domain-dip
1242 	 * (i.e. ddip) will also be found in the same immu_devi struct.
1243 	 * The domain-dip is the highest ancestor dip which shares the
1244 	 * same domain with edip.
1245 	 * The domain may or may not be found, but the domain dip must
1246 	 * be found.
1247 	 */
1248 	if (ddip == NULL) {
1249 		ddi_err(DER_MODE, rdip, "Cannot find domain dip for device.");
1250 		return (NULL);
1251 	}
1252 
1253 	/*
1254 	 * Did we find a domain ?
1255 	 */
1256 	if (domain) {
1257 		goto found;
1258 	}
1259 
1260 	/* nope, so allocate */
1261 	domain = domain_create(immu, ddip, rdip, immu_flags);
1262 	if (domain == NULL) {
1263 		return (NULL);
1264 	}
1265 	ASSERT(domain->dom_did > 0);
1266 
1267 	/*FALLTHROUGH*/
1268 found:
1269 	/*
1270 	 * We know *domain *is* the right domain, so panic if
1271 	 * another domain is set for either the request-dip or
1272 	 * effective dip.
1273 	 */
1274 	set_domain(ddip, ddip, domain);
1275 	set_domain(rdip, ddip, domain);
1276 
1277 	*ddipp = ddip;
1278 	return (domain);
1279 }
1280 
1281 static void
1282 create_unity_domain(immu_t *immu)
1283 {
1284 	domain_t *domain;
1285 
1286 	/* 0 is reserved by Vt-d */
1287 	/*LINTED*/
1288 	ASSERT(IMMU_UNITY_DID > 0);
1289 
1290 	/* domain created during boot and always use sleep flag */
1291 	domain = kmem_zalloc(sizeof (domain_t), KM_SLEEP);
1292 
1293 	rw_init(&(domain->dom_pgtable_rwlock), NULL, RW_DEFAULT, NULL);
1294 
1295 	domain->dom_did = IMMU_UNITY_DID;
1296 	domain->dom_maptype = IMMU_MAPTYPE_UNITY;
1297 
1298 	domain->dom_immu = immu;
1299 	immu->immu_unity_domain = domain;
1300 
1301 	/*
1302 	 * Setup the domain's initial page table
1303 	 * should never fail.
1304 	 */
1305 	domain->dom_pgtable_root = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);
1306 	ASSERT(domain->dom_pgtable_root);
1307 	pgtable_zero(immu, domain->dom_pgtable_root);
1308 
1309 	map_unity_domain(domain);
1310 
1311 	/*
1312 	 * put it on the system-wide UNITY domain list
1313 	 */
1314 	mutex_enter(&(immu_domain_lock));
1315 	list_insert_tail(&immu_unity_domain_list, domain);
1316 	mutex_exit(&(immu_domain_lock));
1317 }
1318 
1319 /*
1320  * ddip is the domain-dip - the topmost dip in a domain
1321  * rdip is the requesting-dip - the device which is
1322  * requesting DVMA setup
1323  * if domain is a non-shared domain rdip == ddip
1324  */
1325 static domain_t *
1326 domain_create(immu_t *immu, dev_info_t *ddip, dev_info_t *rdip,
1327     immu_flags_t immu_flags)
1328 {
1329 	int kmflags;
1330 	domain_t *domain;
1331 	char mod_hash_name[128];
1332 	immu_devi_t *immu_devi;
1333 	int did;
1334 	dcookie_t dcookies[1] = {0};
1335 	int dcount = 0;
1336 
1337 	ASSERT(immu);
1338 	ASSERT(ddip);
1339 
1340 	immu_devi = immu_devi_get(rdip);
1341 
1342 	ASSERT(immu_devi);
1343 
1344 	/*
1345 	 * First allocate a domainid.
1346 	 * This routine will never fail, since if we run out
1347 	 * of domains the unity domain will be allocated.
1348 	 */
1349 	did = did_alloc(immu, rdip, ddip, immu_flags);
1350 	ASSERT(did > 0);
1351 	if (did == IMMU_UNITY_DID) {
1352 		/* domain overflow */
1353 		ASSERT(immu->immu_unity_domain);
1354 		return (immu->immu_unity_domain);
1355 	}
1356 
1357 	kmflags = (immu_flags & IMMU_FLAGS_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP;
1358 	domain = kmem_zalloc(sizeof (domain_t), kmflags);
1359 	if (domain == NULL) {
1360 		ddi_err(DER_PANIC, rdip, "Failed to alloc DVMA domain "
1361 		    "structure for device. IOMMU unit: %s", immu->immu_name);
1362 		/*NOTREACHED*/
1363 	}
1364 
1365 	rw_init(&(domain->dom_pgtable_rwlock), NULL, RW_DEFAULT, NULL);
1366 
1367 	(void) snprintf(mod_hash_name, sizeof (mod_hash_name),
1368 	    "immu%s-domain%d-pava-hash", immu->immu_name, did);
1369 
1370 	domain->dom_did = did;
1371 	domain->dom_immu = immu;
1372 	domain->dom_maptype = IMMU_MAPTYPE_XLATE;
1373 
1374 	/*
1375 	 * Create xlate DVMA arena for this domain.
1376 	 */
1377 	create_xlate_arena(immu, domain, rdip, immu_flags);
1378 
1379 	/*
1380 	 * Setup the domain's initial page table
1381 	 */
1382 	domain->dom_pgtable_root = pgtable_alloc(immu, immu_flags);
1383 	if (domain->dom_pgtable_root == NULL) {
1384 		ddi_err(DER_PANIC, rdip, "Failed to alloc root "
1385 		    "pgtable for domain (%d). IOMMU unit: %s",
1386 		    domain->dom_did, immu->immu_name);
1387 		/*NOTREACHED*/
1388 	}
1389 	pgtable_zero(immu, domain->dom_pgtable_root);
1390 
1391 	/*
1392 	 * Since this is a immu unit-specific domain, put it on
1393 	 * the per-immu domain list.
1394 	 */
1395 	mutex_enter(&(immu->immu_lock));
1396 	list_insert_head(&immu->immu_domain_list, domain);
1397 	mutex_exit(&(immu->immu_lock));
1398 
1399 	/*
1400 	 * Also put it on the system-wide xlate domain list
1401 	 */
1402 	mutex_enter(&(immu_domain_lock));
1403 	list_insert_head(&immu_xlate_domain_list, domain);
1404 	mutex_exit(&(immu_domain_lock));
1405 
1406 	bdf_domain_insert(immu_devi, domain);
1407 
1408 #ifdef BUGGY_DRIVERS
1409 	/*
1410 	 * Map page0. Some broken HW/FW access it.
1411 	 */
1412 	dcookies[0].dck_paddr = 0;
1413 	dcookies[0].dck_npages = 1;
1414 	dcount = 1;
1415 	(void) dvma_map(domain->dom_immu, domain, 0, 1, dcookies, dcount, NULL,
1416 	    IMMU_FLAGS_READ | IMMU_FLAGS_WRITE | IMMU_FLAGS_PAGE1);
1417 #endif
1418 	return (domain);
1419 }
1420 
1421 /*
1422  * Create domainid arena.
1423  * Domainid 0 is reserved by Vt-d spec and cannot be used by
1424  * system software.
1425  * Domainid 1 is reserved by solaris and used for *all* of the following:
1426  *	as the "uninitialized" domain - For devices not yet controlled
1427  *	by Solaris
1428  *	as the "unity" domain - For devices that will always belong
1429  *	to the unity domain
1430  *	as the "overflow" domain - Used for any new device after we
1431  *	run out of domains
1432  * All of the above domains map into a single domain with
1433  * domainid 1 and UNITY DVMA mapping
1434  * Each IMMU unity has its own unity/uninit/overflow domain
1435  */
1436 static void
1437 did_init(immu_t *immu)
1438 {
1439 	(void) snprintf(immu->immu_did_arena_name,
1440 	    sizeof (immu->immu_did_arena_name),
1441 	    "%s_domainid_arena", immu->immu_name);
1442 
1443 	ddi_err(DER_VERB, NULL, "%s: Creating domainid arena %s",
1444 	    immu->immu_name, immu->immu_did_arena_name);
1445 
1446 	immu->immu_did_arena = vmem_create(
1447 	    immu->immu_did_arena_name,
1448 	    (void *)(uintptr_t)(IMMU_UNITY_DID + 1),   /* start addr */
1449 	    immu->immu_max_domains - IMMU_UNITY_DID,
1450 	    1,				/* quantum */
1451 	    NULL,			/* afunc */
1452 	    NULL,			/* ffunc */
1453 	    NULL,			/* source */
1454 	    0,				/* qcache_max */
1455 	    VM_SLEEP);
1456 
1457 	/* Even with SLEEP flag, vmem_create() can fail */
1458 	if (immu->immu_did_arena == NULL) {
1459 		ddi_err(DER_PANIC, NULL, "%s: Failed to create Intel "
1460 		    "IOMMU domainid allocator: %s", immu->immu_name,
1461 		    immu->immu_did_arena_name);
1462 	}
1463 }
1464 
1465 /* #########################  CONTEXT CODE ################################# */
1466 
1467 static void
1468 context_set(immu_t *immu, domain_t *domain, pgtable_t *root_table,
1469     int bus, int devfunc)
1470 {
1471 	pgtable_t *context;
1472 	pgtable_t *pgtable_root;
1473 	pgtable_t *unity_pgtable_root;
1474 	hw_rce_t *hw_rent;
1475 	hw_rce_t *hw_cent;
1476 	hw_rce_t *ctxp;
1477 	int sid;
1478 	krw_t rwtype;
1479 	boolean_t fill_root;
1480 	boolean_t fill_ctx;
1481 
1482 	ASSERT(immu);
1483 	ASSERT(domain);
1484 	ASSERT(root_table);
1485 	ASSERT(bus >= 0);
1486 	ASSERT(devfunc >= 0);
1487 	ASSERT(domain->dom_pgtable_root);
1488 
1489 	pgtable_root = domain->dom_pgtable_root;
1490 
1491 	ctxp = (hw_rce_t *)(root_table->swpg_next_array);
1492 	context = *(pgtable_t **)(ctxp + bus);
1493 	hw_rent = (hw_rce_t *)(root_table->hwpg_vaddr) + bus;
1494 
1495 	fill_root = B_FALSE;
1496 	fill_ctx = B_FALSE;
1497 
1498 	/* Check the most common case first with reader lock */
1499 	rw_enter(&(immu->immu_ctx_rwlock), RW_READER);
1500 	rwtype = RW_READER;
1501 again:
1502 	if (ROOT_GET_P(hw_rent)) {
1503 		ASSERT(ROOT_GET_CONT(hw_rent) == context->hwpg_paddr);
1504 		hw_cent = (hw_rce_t *)(context->hwpg_vaddr) + devfunc;
1505 		if (CONT_GET_AVAIL(hw_cent) == IMMU_CONT_INITED) {
1506 			ASSERT(CONT_GET_P(hw_cent));
1507 			ASSERT(CONT_GET_DID(hw_cent) == domain->dom_did);
1508 			ASSERT(CONT_GET_AW(hw_cent) == immu->immu_dvma_agaw);
1509 			ASSERT(CONT_GET_TTYPE(hw_cent) == TTYPE_XLATE_ONLY);
1510 			ASSERT(CONT_GET_ASR(hw_cent) ==
1511 			    pgtable_root->hwpg_paddr);
1512 			rw_exit(&(immu->immu_ctx_rwlock));
1513 			return;
1514 		} else {
1515 			fill_ctx = B_TRUE;
1516 		}
1517 	} else {
1518 		fill_root = B_TRUE;
1519 		fill_ctx = B_TRUE;
1520 	}
1521 
1522 	if (rwtype == RW_READER &&
1523 	    rw_tryupgrade(&(immu->immu_ctx_rwlock)) == 0) {
1524 		rw_exit(&(immu->immu_ctx_rwlock));
1525 		rw_enter(&(immu->immu_ctx_rwlock), RW_WRITER);
1526 		rwtype = RW_WRITER;
1527 		goto again;
1528 	}
1529 	rwtype = RW_WRITER;
1530 
1531 	if (fill_root == B_TRUE) {
1532 		ROOT_SET_CONT(hw_rent, context->hwpg_paddr);
1533 		ROOT_SET_P(hw_rent);
1534 		immu_regs_cpu_flush(immu, (caddr_t)hw_rent, sizeof (hw_rce_t));
1535 	}
1536 
1537 	if (fill_ctx == B_TRUE) {
1538 		hw_cent = (hw_rce_t *)(context->hwpg_vaddr) + devfunc;
1539 		unity_pgtable_root = immu->immu_unity_domain->dom_pgtable_root;
1540 		ASSERT(CONT_GET_AVAIL(hw_cent) == IMMU_CONT_UNINITED);
1541 		ASSERT(CONT_GET_P(hw_cent));
1542 		ASSERT(CONT_GET_DID(hw_cent) ==
1543 		    immu->immu_unity_domain->dom_did);
1544 		ASSERT(CONT_GET_AW(hw_cent) == immu->immu_dvma_agaw);
1545 		ASSERT(CONT_GET_TTYPE(hw_cent) == TTYPE_XLATE_ONLY);
1546 		ASSERT(CONT_GET_ASR(hw_cent) ==
1547 		    unity_pgtable_root->hwpg_paddr);
1548 
1549 		/* need to disable context entry before reprogramming it */
1550 		bzero(hw_cent, sizeof (hw_rce_t));
1551 
1552 		/* flush caches */
1553 		immu_regs_cpu_flush(immu, (caddr_t)hw_cent, sizeof (hw_rce_t));
1554 		ASSERT(rw_write_held(&(immu->immu_ctx_rwlock)));
1555 
1556 		sid = ((bus << 8) | devfunc);
1557 		immu_regs_context_flush(immu, 0, sid, domain->dom_did,
1558 		    CONTEXT_FSI);
1559 
1560 		immu_regs_wbf_flush(immu);
1561 
1562 		CONT_SET_AVAIL(hw_cent, IMMU_CONT_INITED);
1563 		CONT_SET_DID(hw_cent, domain->dom_did);
1564 		CONT_SET_AW(hw_cent, immu->immu_dvma_agaw);
1565 		CONT_SET_ASR(hw_cent, pgtable_root->hwpg_paddr);
1566 		/*LINTED*/
1567 		CONT_SET_TTYPE(hw_cent, TTYPE_XLATE_ONLY);
1568 		CONT_SET_P(hw_cent);
1569 		immu_regs_cpu_flush(immu, (caddr_t)hw_cent, sizeof (hw_rce_t));
1570 	}
1571 	rw_exit(&(immu->immu_ctx_rwlock));
1572 }
1573 
1574 static pgtable_t *
1575 context_create(immu_t *immu)
1576 {
1577 	int	bus;
1578 	int	devfunc;
1579 	pgtable_t *root_table;
1580 	pgtable_t *context;
1581 	pgtable_t *pgtable_root;
1582 	hw_rce_t *ctxp;
1583 	hw_rce_t *hw_rent;
1584 	hw_rce_t *hw_cent;
1585 
1586 	/* Allocate a zeroed root table (4K 256b entries) */
1587 	root_table = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);
1588 	pgtable_zero(immu, root_table);
1589 
1590 	/*
1591 	 * Setup context tables for all possible root table entries.
1592 	 * Start out with unity domains for all entries.
1593 	 */
1594 	ctxp = (hw_rce_t *)(root_table->swpg_next_array);
1595 	hw_rent = (hw_rce_t *)(root_table->hwpg_vaddr);
1596 	for (bus = 0; bus < IMMU_ROOT_NUM; bus++, ctxp++, hw_rent++) {
1597 		context = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);
1598 		pgtable_zero(immu, context);
1599 		ASSERT(ROOT_GET_P(hw_rent) == 0);
1600 		ROOT_SET_P(hw_rent);
1601 		ROOT_SET_CONT(hw_rent, context->hwpg_paddr);
1602 		hw_cent = (hw_rce_t *)(context->hwpg_vaddr);
1603 		for (devfunc = 0; devfunc < IMMU_CONT_NUM;
1604 		    devfunc++, hw_cent++) {
1605 			ASSERT(CONT_GET_P(hw_cent) == 0);
1606 			pgtable_root =
1607 			    immu->immu_unity_domain->dom_pgtable_root;
1608 			CONT_SET_DID(hw_cent,
1609 			    immu->immu_unity_domain->dom_did);
1610 			CONT_SET_AW(hw_cent, immu->immu_dvma_agaw);
1611 			CONT_SET_ASR(hw_cent, pgtable_root->hwpg_paddr);
1612 			/*LINTED*/
1613 			CONT_SET_TTYPE(hw_cent, TTYPE_XLATE_ONLY);
1614 			CONT_SET_AVAIL(hw_cent, IMMU_CONT_UNINITED);
1615 			CONT_SET_P(hw_cent);
1616 		}
1617 		immu_regs_cpu_flush(immu, context->hwpg_vaddr, IMMU_PAGESIZE);
1618 		*((pgtable_t **)ctxp) = context;
1619 	}
1620 	immu_regs_cpu_flush(immu, root_table->hwpg_vaddr, IMMU_PAGESIZE);
1621 
1622 	return (root_table);
1623 }
1624 
1625 /*
1626  * Called during rootnex attach, so no locks needed
1627  */
1628 static void
1629 context_init(immu_t *immu)
1630 {
1631 	ASSERT(immu);
1632 	ASSERT(immu->immu_ctx_root == NULL);
1633 
1634 	rw_init(&(immu->immu_ctx_rwlock), NULL, RW_DEFAULT, NULL);
1635 
1636 	immu_regs_wbf_flush(immu);
1637 
1638 	immu->immu_ctx_root = context_create(immu);
1639 
1640 	immu_regs_set_root_table(immu);
1641 
1642 	rw_enter(&(immu->immu_ctx_rwlock), RW_WRITER);
1643 	immu_regs_context_flush(immu, 0, 0, 0, CONTEXT_GLOBAL);
1644 	rw_exit(&(immu->immu_ctx_rwlock));
1645 	immu_regs_iotlb_flush(immu, 0, 0, 0, 0, IOTLB_GLOBAL);
1646 	immu_regs_wbf_flush(immu);
1647 }
1648 
1649 
1650 /*
1651  * Find top pcib
1652  */
1653 static int
1654 find_top_pcib(dev_info_t *dip, void *arg)
1655 {
1656 	immu_devi_t *immu_devi;
1657 	dev_info_t **pcibdipp = (dev_info_t **)arg;
1658 
1659 	ASSERT(dip);
1660 
1661 	immu_devi = immu_devi_get(dip);
1662 	ASSERT(immu_devi);
1663 
1664 	if (immu_devi->imd_pcib_type == IMMU_PCIB_PCI_PCI) {
1665 		*pcibdipp = dip;
1666 	}
1667 
1668 	return (DDI_WALK_CONTINUE);
1669 }
1670 
1671 static int
1672 immu_context_update(immu_t *immu, domain_t *domain, dev_info_t *ddip,
1673     dev_info_t *rdip, immu_flags_t immu_flags)
1674 {
1675 	immu_devi_t *r_immu_devi;
1676 	immu_devi_t *d_immu_devi;
1677 	int r_bus;
1678 	int d_bus;
1679 	int r_devfunc;
1680 	int d_devfunc;
1681 	immu_pcib_t d_pcib_type;
1682 	immu_pcib_t r_pcib_type;
1683 	dev_info_t *pcibdip;
1684 
1685 	if (ddip == NULL || rdip == NULL ||
1686 	    ddip == root_devinfo || rdip == root_devinfo) {
1687 		ddi_err(DER_MODE, rdip, "immu_contexts_update: domain-dip or "
1688 		    "request-dip are NULL or are root devinfo");
1689 		return (DDI_FAILURE);
1690 	}
1691 
1692 	/*
1693 	 * We need to set the context fields
1694 	 * based on what type of device rdip and ddip are.
1695 	 * To do that we need the immu_devi field.
1696 	 * Set the immu_devi field (if not already set)
1697 	 */
1698 	if (immu_devi_set(ddip, immu_flags) == DDI_FAILURE) {
1699 		ddi_err(DER_MODE, rdip,
1700 		    "immu_context_update: failed to set immu_devi for ddip");
1701 		return (DDI_FAILURE);
1702 	}
1703 
1704 	if (immu_devi_set(rdip, immu_flags) == DDI_FAILURE) {
1705 		ddi_err(DER_MODE, rdip,
1706 		    "immu_context_update: failed to set immu_devi for rdip");
1707 		return (DDI_FAILURE);
1708 	}
1709 
1710 	d_immu_devi = immu_devi_get(ddip);
1711 	r_immu_devi = immu_devi_get(rdip);
1712 	ASSERT(r_immu_devi);
1713 	ASSERT(d_immu_devi);
1714 
1715 	d_bus = d_immu_devi->imd_bus;
1716 	d_devfunc = d_immu_devi->imd_devfunc;
1717 	d_pcib_type = d_immu_devi->imd_pcib_type;
1718 	r_bus = r_immu_devi->imd_bus;
1719 	r_devfunc = r_immu_devi->imd_devfunc;
1720 	r_pcib_type = r_immu_devi->imd_pcib_type;
1721 
1722 	ASSERT(d_bus >= 0);
1723 
1724 	if (rdip == ddip) {
1725 		ASSERT(d_pcib_type == IMMU_PCIB_ENDPOINT ||
1726 		    d_pcib_type == IMMU_PCIB_PCIE_PCIE);
1727 		ASSERT(r_bus >= 0);
1728 		ASSERT(r_devfunc >= 0);
1729 		/* rdip is a PCIE device. set context for it only */
1730 		context_set(immu, domain, immu->immu_ctx_root, r_bus,
1731 		    r_devfunc);
1732 #ifdef BUGGY_DRIVERS
1733 	} else if (r_immu_devi == d_immu_devi) {
1734 #ifdef TEST
1735 		ddi_err(DER_WARN, rdip, "Driver bug: Devices 0x%lx and "
1736 		    "0x%lx are identical", rdip, ddip);
1737 #endif
1738 		ASSERT(d_pcib_type == IMMU_PCIB_ENDPOINT);
1739 		ASSERT(r_bus >= 0);
1740 		ASSERT(r_devfunc >= 0);
1741 		/* rdip is a PCIE device. set context for it only */
1742 		context_set(immu, domain, immu->immu_ctx_root, r_bus,
1743 		    r_devfunc);
1744 #endif
1745 	} else if (d_pcib_type == IMMU_PCIB_PCIE_PCI) {
1746 		/*
1747 		 * ddip is a PCIE_PCI bridge. Set context for ddip's
1748 		 * secondary bus. If rdip is on ddip's secondary
1749 		 * bus, set context for rdip. Else, set context
1750 		 * for rdip's PCI bridge on ddip's secondary bus.
1751 		 */
1752 		context_set(immu, domain, immu->immu_ctx_root,
1753 		    d_immu_devi->imd_sec, 0);
1754 		if (d_immu_devi->imd_sec == r_bus) {
1755 			context_set(immu, domain, immu->immu_ctx_root,
1756 			    r_bus, r_devfunc);
1757 		} else {
1758 			pcibdip = NULL;
1759 			if (immu_walk_ancestor(rdip, ddip, find_top_pcib,
1760 			    &pcibdip, NULL, immu_flags) == DDI_SUCCESS &&
1761 			    pcibdip != NULL) {
1762 				ASSERT(pcibdip);
1763 				r_immu_devi = immu_devi_get(pcibdip);
1764 				ASSERT(d_immu_devi);
1765 				ASSERT(d_immu_devi->imd_pcib_type ==
1766 				    IMMU_PCIB_PCI_PCI);
1767 				r_bus = r_immu_devi->imd_bus;
1768 				r_devfunc = r_immu_devi->imd_devfunc;
1769 				context_set(immu, domain, immu->immu_ctx_root,
1770 				    r_bus, r_devfunc);
1771 			} else {
1772 				ddi_err(DER_PANIC, rdip, "Failed to find PCI "
1773 				    " bridge for PCI device");
1774 				/*NOTREACHED*/
1775 			}
1776 		}
1777 	} else if (d_pcib_type == IMMU_PCIB_PCI_PCI) {
1778 		context_set(immu, domain, immu->immu_ctx_root, d_bus,
1779 		    d_devfunc);
1780 	} else if (d_pcib_type == IMMU_PCIB_ENDPOINT) {
1781 		ASSERT(r_pcib_type == IMMU_PCIB_NOBDF);
1782 		/*
1783 		 * ddip is a PCIE device which has a non-PCI device under it
1784 		 * i.e. it is a PCI-nonPCI bridge. Example: pciicde-ata
1785 		 */
1786 		context_set(immu, domain, immu->immu_ctx_root, d_bus,
1787 		    d_devfunc);
1788 	} else {
1789 		ddi_err(DER_PANIC, rdip, "unknown device type. Cannot "
1790 		    "set IMMU context.");
1791 		/*NOTREACHED*/
1792 	}
1793 
1794 	/* XXX do we need a membar_producer() here */
1795 	return (DDI_SUCCESS);
1796 }
1797 
1798 /* ##################### END CONTEXT CODE ################################## */
1799 /* ##################### MAPPING CODE ################################## */
1800 
1801 
1802 static boolean_t
1803 PDTE_check(immu_t *immu, hw_pdte_t pdte, pgtable_t *next, paddr_t paddr,
1804     dev_info_t *rdip, immu_flags_t immu_flags)
1805 {
1806 	if (immu_flags & IMMU_FLAGS_PAGE1) {
1807 		ASSERT(paddr == 0);
1808 	} else {
1809 		ASSERT((next == NULL) ^ (paddr == 0));
1810 	}
1811 
1812 	/* The PDTE must be set i.e. present bit is set */
1813 	if (!PDTE_P(pdte)) {
1814 		ddi_err(DER_MODE, rdip, "No present flag");
1815 		return (B_FALSE);
1816 	}
1817 
1818 	/*
1819 	 * Just assert to check most significant system software field
1820 	 * (PDTE_SW4) as it is same as present bit and we
1821 	 * checked that above
1822 	 */
1823 	ASSERT(PDTE_SW4(pdte));
1824 
1825 	/*
1826 	 * TM field should be clear if not reserved.
1827 	 * non-leaf is always reserved
1828 	 */
1829 	if (next == NULL && immu->immu_TM_reserved == B_FALSE) {
1830 		if (PDTE_TM(pdte)) {
1831 			ddi_err(DER_MODE, rdip, "TM flag set");
1832 			return (B_FALSE);
1833 		}
1834 	}
1835 
1836 	/*
1837 	 * The SW3 field is not used and must be clear
1838 	 */
1839 	if (PDTE_SW3(pdte)) {
1840 		ddi_err(DER_MODE, rdip, "SW3 set");
1841 		return (B_FALSE);
1842 	}
1843 
1844 	/*
1845 	 * PFN (for PTE) or next level pgtable-paddr (for PDE) must be set
1846 	 */
1847 	if (next == NULL) {
1848 		ASSERT(paddr % IMMU_PAGESIZE == 0);
1849 		if (PDTE_PADDR(pdte) != paddr) {
1850 			ddi_err(DER_MODE, rdip,
1851 			    "PTE paddr mismatch: %lx != %lx",
1852 			    PDTE_PADDR(pdte), paddr);
1853 			return (B_FALSE);
1854 		}
1855 	} else {
1856 		if (PDTE_PADDR(pdte) != next->hwpg_paddr) {
1857 			ddi_err(DER_MODE, rdip,
1858 			    "PDE paddr mismatch: %lx != %lx",
1859 			    PDTE_PADDR(pdte), next->hwpg_paddr);
1860 			return (B_FALSE);
1861 		}
1862 	}
1863 
1864 	/*
1865 	 * SNP field should be clear if not reserved.
1866 	 * non-leaf is always reserved
1867 	 */
1868 	if (next == NULL && immu->immu_SNP_reserved == B_FALSE) {
1869 		if (PDTE_SNP(pdte)) {
1870 			ddi_err(DER_MODE, rdip, "SNP set");
1871 			return (B_FALSE);
1872 		}
1873 	}
1874 
1875 	/* second field available for system software should be clear */
1876 	if (PDTE_SW2(pdte)) {
1877 		ddi_err(DER_MODE, rdip, "SW2 set");
1878 		return (B_FALSE);
1879 	}
1880 
1881 	/* Super pages field should be clear */
1882 	if (PDTE_SP(pdte)) {
1883 		ddi_err(DER_MODE, rdip, "SP set");
1884 		return (B_FALSE);
1885 	}
1886 
1887 	/*
1888 	 * least significant field available for
1889 	 * system software should be clear
1890 	 */
1891 	if (PDTE_SW1(pdte)) {
1892 		ddi_err(DER_MODE, rdip, "SW1 set");
1893 		return (B_FALSE);
1894 	}
1895 
1896 	if ((immu_flags & IMMU_FLAGS_READ) && !PDTE_READ(pdte)) {
1897 		ddi_err(DER_MODE, rdip, "READ not set");
1898 		return (B_FALSE);
1899 	}
1900 
1901 	if ((immu_flags & IMMU_FLAGS_WRITE) && !PDTE_WRITE(pdte)) {
1902 		ddi_err(DER_MODE, rdip, "WRITE not set");
1903 		return (B_FALSE);
1904 	}
1905 
1906 	return (B_TRUE);
1907 }
1908 /*ARGSUSED*/
1909 static void
1910 PTE_clear_all(immu_t *immu, domain_t *domain, xlate_t *xlate,
1911     uint64_t *dvma_ptr, uint64_t *npages_ptr, dev_info_t *rdip)
1912 {
1913 	uint64_t npages;
1914 	uint64_t dvma;
1915 	pgtable_t *pgtable;
1916 	hw_pdte_t *hwp;
1917 	hw_pdte_t *shwp;
1918 	int idx;
1919 	hw_pdte_t pte;
1920 
1921 	ASSERT(xlate->xlt_level == 1);
1922 
1923 	pgtable = xlate->xlt_pgtable;
1924 	idx = xlate->xlt_idx;
1925 
1926 	ASSERT(pgtable);
1927 	ASSERT(idx <= IMMU_PGTABLE_MAXIDX);
1928 
1929 	dvma = *dvma_ptr;
1930 	npages = *npages_ptr;
1931 
1932 	ASSERT(dvma);
1933 	ASSERT(dvma % IMMU_PAGESIZE == 0);
1934 	ASSERT(npages);
1935 
1936 	/*
1937 	 * since a caller gets a unique dvma for a physical address,
1938 	 * no other concurrent thread will be writing to the same
1939 	 * PTE even if it has the same paddr. So no locks needed.
1940 	 */
1941 	shwp = (hw_pdte_t *)(pgtable->hwpg_vaddr) + idx;
1942 
1943 	hwp = shwp;
1944 	for (; npages > 0 && idx <= IMMU_PGTABLE_MAXIDX; idx++, hwp++) {
1945 
1946 		pte = *hwp;
1947 
1948 		/* Cannot clear a HW PTE that is aleady clear */
1949 		ASSERT(PDTE_P(pte));
1950 		PDTE_CLEAR_P(pte);
1951 		*hwp = pte;
1952 
1953 		dvma += IMMU_PAGESIZE;
1954 		npages--;
1955 	}
1956 
1957 
1958 #ifdef TEST
1959 	/* dont need to flush write during unmap */
1960 	immu_regs_cpu_flush(immu, (caddr_t)shwp,
1961 	    (hwp - shwp) * sizeof (hw_pdte_t));
1962 #endif
1963 
1964 	*dvma_ptr = dvma;
1965 	*npages_ptr = npages;
1966 
1967 	xlate->xlt_idx = idx;
1968 }
1969 
1970 /*ARGSUSED*/
1971 static void
1972 xlate_setup(immu_t *immu, uint64_t dvma, xlate_t *xlate,
1973     int nlevels, dev_info_t *rdip)
1974 {
1975 	int level;
1976 	uint64_t offbits;
1977 
1978 	/* level 0 is never used. Sanity check */
1979 	ASSERT(xlate->xlt_level == 0);
1980 	ASSERT(xlate->xlt_idx == 0);
1981 	ASSERT(xlate->xlt_pgtable == NULL);
1982 	ASSERT(dvma % IMMU_PAGESIZE == 0);
1983 
1984 	/*
1985 	 * Skip the first 12 bits which is the offset into
1986 	 * 4K PFN (phys page frame based on IMMU_PAGESIZE)
1987 	 */
1988 	offbits = dvma >> IMMU_PAGESHIFT;
1989 
1990 	/* skip to level 1 i.e. leaf PTE */
1991 	for (level = 1, xlate++; level <= nlevels; level++, xlate++) {
1992 		xlate->xlt_level = level;
1993 		xlate->xlt_idx = (offbits & IMMU_PGTABLE_LEVEL_MASK);
1994 		ASSERT(xlate->xlt_idx <= IMMU_PGTABLE_MAXIDX);
1995 		xlate->xlt_pgtable = NULL;
1996 		offbits >>= IMMU_PGTABLE_LEVEL_STRIDE;
1997 	}
1998 }
1999 
2000 /*
2001  * Read the pgtables
2002  */
2003 static void
2004 PDE_lookup(immu_t *immu, domain_t *domain, xlate_t *xlate, int nlevels,
2005     dev_info_t *rdip)
2006 {
2007 	pgtable_t *pgtable;
2008 	pgtable_t *next;
2009 	hw_pdte_t pde;
2010 	uint_t idx;
2011 
2012 	/* xlate should be at level 0 */
2013 	ASSERT(xlate->xlt_level == 0);
2014 	ASSERT(xlate->xlt_idx == 0);
2015 
2016 	/* start with highest level pgtable i.e. root */
2017 	xlate += nlevels;
2018 	ASSERT(xlate->xlt_level == nlevels);
2019 
2020 	if (xlate->xlt_pgtable == NULL) {
2021 		xlate->xlt_pgtable = domain->dom_pgtable_root;
2022 	}
2023 
2024 	for (; xlate->xlt_level > 1; xlate--) {
2025 
2026 		idx = xlate->xlt_idx;
2027 		pgtable = xlate->xlt_pgtable;
2028 
2029 		ASSERT(pgtable);
2030 		ASSERT(idx <= IMMU_PGTABLE_MAXIDX);
2031 
2032 		if ((xlate - 1)->xlt_pgtable) {
2033 			continue;
2034 		}
2035 
2036 		/* xlate's leafier level is not set, set it now */
2037 
2038 		/* Lock the pgtable in read mode */
2039 		rw_enter(&(pgtable->swpg_rwlock), RW_READER);
2040 
2041 		/*
2042 		 * since we are unmapping, the pgtable should
2043 		 * already point to a leafier pgtable.
2044 		 */
2045 		next = *(pgtable->swpg_next_array + idx);
2046 		ASSERT(next);
2047 
2048 		pde = *((hw_pdte_t *)(pgtable->hwpg_vaddr) + idx);
2049 
2050 		ASSERT(PDTE_check(immu, pde, next, 0, rdip, 0) == B_TRUE);
2051 
2052 		(xlate - 1)->xlt_pgtable = next;
2053 
2054 		rw_exit(&(pgtable->swpg_rwlock));
2055 	}
2056 }
2057 
2058 /*ARGSUSED*/
2059 static void
2060 PTE_set_one(immu_t *immu, hw_pdte_t *hwp, paddr_t paddr,
2061     dev_info_t *rdip, immu_flags_t immu_flags)
2062 {
2063 	hw_pdte_t pte;
2064 
2065 	pte = *hwp;
2066 
2067 #ifndef DEBUG
2068 	/* Set paddr */
2069 	ASSERT(paddr % IMMU_PAGESIZE == 0);
2070 	pte = 0;
2071 	PDTE_SET_PADDR(pte, paddr);
2072 	PDTE_SET_READ(pte);
2073 	PDTE_SET_WRITE(pte);
2074 	*hwp = pte;
2075 #else
2076 
2077 	if (PDTE_P(pte)) {
2078 		if (PDTE_PADDR(pte) != paddr) {
2079 			ddi_err(DER_MODE, rdip, "PTE paddr %lx != paddr %lx",
2080 			    PDTE_PADDR(pte), paddr);
2081 		}
2082 #ifdef BUGGY_DRIVERS
2083 		return;
2084 #else
2085 		goto out;
2086 #endif
2087 	}
2088 
2089 	/* Don't touch SW4. It is the present field */
2090 
2091 	/* clear TM field if not reserved */
2092 	if (immu->immu_TM_reserved == B_FALSE) {
2093 		PDTE_CLEAR_TM(pte);
2094 	}
2095 
2096 #ifdef DEBUG
2097 	/* Clear 3rd field for system software  - not used */
2098 	PDTE_CLEAR_SW3(pte);
2099 #endif
2100 
2101 	/* Set paddr */
2102 	ASSERT(paddr % IMMU_PAGESIZE == 0);
2103 	PDTE_CLEAR_PADDR(pte);
2104 	PDTE_SET_PADDR(pte, paddr);
2105 
2106 	/*  clear SNP field if not reserved. */
2107 	if (immu->immu_SNP_reserved == B_FALSE) {
2108 		PDTE_CLEAR_SNP(pte);
2109 	}
2110 
2111 #ifdef DEBUG
2112 	/* Clear SW2 field available for software */
2113 	PDTE_CLEAR_SW2(pte);
2114 #endif
2115 
2116 
2117 #ifdef DEBUG
2118 	/* SP is don't care for PTEs. Clear it for cleanliness */
2119 	PDTE_CLEAR_SP(pte);
2120 #endif
2121 
2122 #ifdef DEBUG
2123 	/* Clear SW1 field available for software */
2124 	PDTE_CLEAR_SW1(pte);
2125 #endif
2126 
2127 	/*
2128 	 * Now that we are done writing the PTE
2129 	 * set the "present" flag. Note this present
2130 	 * flag is a bit in the PDE/PTE that the
2131 	 * spec says is available for system software.
2132 	 * This is an implementation detail of Solaris
2133 	 * bare-metal Intel IOMMU.
2134 	 * The present field in a PDE/PTE is not defined
2135 	 * by the Vt-d spec
2136 	 */
2137 
2138 	PDTE_SET_P(pte);
2139 
2140 out:
2141 #ifdef BUGGY_DRIVERS
2142 	PDTE_SET_READ(pte);
2143 	PDTE_SET_WRITE(pte);
2144 #else
2145 	if (immu_flags & IMMU_FLAGS_READ)
2146 		PDTE_SET_READ(pte);
2147 	if (immu_flags & IMMU_FLAGS_WRITE)
2148 		PDTE_SET_WRITE(pte);
2149 #endif
2150 
2151 	*hwp = pte;
2152 #endif
2153 }
2154 
2155 /*ARGSUSED*/
2156 static void
2157 PTE_set_all(immu_t *immu, domain_t *domain, xlate_t *xlate,
2158     uint64_t *dvma_ptr, uint64_t *nvpages_ptr, dcookie_t *dcookies,
2159     int dcount, dev_info_t *rdip, immu_flags_t immu_flags)
2160 {
2161 	paddr_t paddr;
2162 	uint64_t nvpages;
2163 	uint64_t nppages;
2164 	uint64_t dvma;
2165 	pgtable_t *pgtable;
2166 	hw_pdte_t *hwp;
2167 	hw_pdte_t *shwp;
2168 	int idx;
2169 	int j;
2170 
2171 	ASSERT(xlate->xlt_level == 1);
2172 
2173 	pgtable = xlate->xlt_pgtable;
2174 	idx = xlate->xlt_idx;
2175 
2176 	ASSERT(idx <= IMMU_PGTABLE_MAXIDX);
2177 	ASSERT(pgtable);
2178 
2179 	dvma = *dvma_ptr;
2180 	nvpages = *nvpages_ptr;
2181 
2182 	ASSERT(dvma || (immu_flags & IMMU_FLAGS_PAGE1));
2183 	ASSERT(nvpages);
2184 
2185 	/*
2186 	 * since a caller gets a unique dvma for a physical address,
2187 	 * no other concurrent thread will be writing to the same
2188 	 * PTE even if it has the same paddr. So no locks needed.
2189 	 */
2190 	shwp = (hw_pdte_t *)(pgtable->hwpg_vaddr) + idx;
2191 
2192 	hwp = shwp;
2193 	for (j = dcount - 1; j >= 0; j--) {
2194 		if (nvpages <= dcookies[j].dck_npages)
2195 			break;
2196 		nvpages -= dcookies[j].dck_npages;
2197 	}
2198 
2199 	ASSERT(j >= 0);
2200 	ASSERT(nvpages);
2201 	ASSERT(nvpages <= dcookies[j].dck_npages);
2202 	nppages = nvpages;
2203 	paddr = dcookies[j].dck_paddr +
2204 	    (dcookies[j].dck_npages - nppages) * IMMU_PAGESIZE;
2205 
2206 	nvpages = *nvpages_ptr;
2207 	for (; nvpages > 0 && idx <= IMMU_PGTABLE_MAXIDX; idx++, hwp++) {
2208 
2209 		ASSERT(paddr || (immu_flags & IMMU_FLAGS_PAGE1));
2210 
2211 		PTE_set_one(immu, hwp, paddr, rdip, immu_flags);
2212 
2213 		ASSERT(PDTE_check(immu, *hwp, NULL, paddr, rdip, immu_flags)
2214 		    == B_TRUE);
2215 		nppages--;
2216 		nvpages--;
2217 		paddr += IMMU_PAGESIZE;
2218 		dvma += IMMU_PAGESIZE;
2219 
2220 		if (nppages == 0) {
2221 			j++;
2222 		}
2223 
2224 		if (j == dcount) {
2225 			ASSERT(nvpages == 0);
2226 			break;
2227 		}
2228 
2229 		ASSERT(nvpages);
2230 		if (nppages == 0) {
2231 			nppages = dcookies[j].dck_npages;
2232 			paddr = dcookies[j].dck_paddr;
2233 		}
2234 	}
2235 
2236 	/* flush writes to HW PTE table */
2237 	immu_regs_cpu_flush(immu, (caddr_t)shwp, (hwp - shwp) *
2238 	    sizeof (hw_pdte_t));
2239 
2240 	if (nvpages) {
2241 		*dvma_ptr = dvma;
2242 		*nvpages_ptr = nvpages;
2243 	} else {
2244 		*dvma_ptr = 0;
2245 		*nvpages_ptr = 0;
2246 	}
2247 
2248 	xlate->xlt_idx = idx;
2249 }
2250 
2251 /*ARGSUSED*/
2252 static void
2253 PDE_set_one(immu_t *immu, hw_pdte_t *hwp, pgtable_t *next,
2254     dev_info_t *rdip, immu_flags_t immu_flags)
2255 {
2256 	hw_pdte_t pde;
2257 
2258 	pde = *hwp;
2259 
2260 	/* if PDE is already set, make sure it is correct */
2261 	if (PDTE_P(pde)) {
2262 		ASSERT(PDTE_PADDR(pde) == next->hwpg_paddr);
2263 #ifdef BUGGY_DRIVERS
2264 		return;
2265 #else
2266 		goto out;
2267 #endif
2268 	}
2269 
2270 	/* Dont touch SW4, it is the present bit */
2271 
2272 	/* don't touch TM field it is reserved for PDEs */
2273 
2274 	/* 3rd field available for system software is not used */
2275 	PDTE_CLEAR_SW3(pde);
2276 
2277 	/* Set next level pgtable-paddr for PDE */
2278 	ASSERT(next->hwpg_paddr % IMMU_PAGESIZE == 0);
2279 	PDTE_CLEAR_PADDR(pde);
2280 	PDTE_SET_PADDR(pde, next->hwpg_paddr);
2281 
2282 	/* don't touch SNP field it is reserved for PDEs */
2283 
2284 	/* Clear second field available for system software */
2285 	PDTE_CLEAR_SW2(pde);
2286 
2287 	/* No super pages for PDEs */
2288 	PDTE_CLEAR_SP(pde);
2289 
2290 	/* Clear SW1 for software */
2291 	PDTE_CLEAR_SW1(pde);
2292 
2293 	/*
2294 	 * Now that we are done writing the PDE
2295 	 * set the "present" flag. Note this present
2296 	 * flag is a bit in the PDE/PTE that the
2297 	 * spec says is available for system software.
2298 	 * This is an implementation detail of Solaris
2299 	 * base-metal Intel IOMMU.
2300 	 * The present field in a PDE/PTE is not defined
2301 	 * by the Vt-d spec
2302 	 */
2303 
2304 out:
2305 #ifdef  BUGGY_DRIVERS
2306 	PDTE_SET_READ(pde);
2307 	PDTE_SET_WRITE(pde);
2308 #else
2309 	if (immu_flags & IMMU_FLAGS_READ)
2310 		PDTE_SET_READ(pde);
2311 	if (immu_flags & IMMU_FLAGS_WRITE)
2312 		PDTE_SET_WRITE(pde);
2313 #endif
2314 
2315 	PDTE_SET_P(pde);
2316 
2317 	*hwp = pde;
2318 
2319 	immu_regs_cpu_flush(immu, (caddr_t)hwp, sizeof (hw_pdte_t));
2320 }
2321 
2322 /*
2323  * Used to set PDEs
2324  */
2325 static boolean_t
2326 PDE_set_all(immu_t *immu, domain_t *domain, xlate_t *xlate, int nlevels,
2327     dev_info_t *rdip, immu_flags_t immu_flags)
2328 {
2329 	pgtable_t *pgtable;
2330 	pgtable_t *new;
2331 	pgtable_t *next;
2332 	hw_pdte_t *hwp;
2333 	int level;
2334 	uint_t idx;
2335 	krw_t rwtype;
2336 	boolean_t set = B_FALSE;
2337 
2338 	/* xlate should be at level 0 */
2339 	ASSERT(xlate->xlt_level == 0);
2340 	ASSERT(xlate->xlt_idx == 0);
2341 
2342 	/* start with highest level pgtable i.e. root */
2343 	xlate += nlevels;
2344 	ASSERT(xlate->xlt_level == nlevels);
2345 
2346 	new = NULL;
2347 	xlate->xlt_pgtable = domain->dom_pgtable_root;
2348 	for (level = nlevels; level > 1; level--, xlate--) {
2349 
2350 		ASSERT(xlate->xlt_level == level);
2351 
2352 		idx = xlate->xlt_idx;
2353 		pgtable = xlate->xlt_pgtable;
2354 
2355 		ASSERT(pgtable);
2356 		ASSERT(idx <= IMMU_PGTABLE_MAXIDX);
2357 
2358 		/* speculative alloc */
2359 		if (new == NULL) {
2360 			new = pgtable_alloc(immu, immu_flags);
2361 			if (new == NULL) {
2362 				ddi_err(DER_PANIC, rdip, "pgtable alloc err");
2363 			}
2364 		}
2365 
2366 		/* Lock the pgtable in READ mode first */
2367 		rw_enter(&(pgtable->swpg_rwlock), RW_READER);
2368 		rwtype = RW_READER;
2369 again:
2370 		hwp = (hw_pdte_t *)(pgtable->hwpg_vaddr) + idx;
2371 
2372 		ASSERT(pgtable->swpg_next_array);
2373 
2374 		next = (pgtable->swpg_next_array)[idx];
2375 
2376 		/*
2377 		 * check if leafier level already has a pgtable
2378 		 * if yes, verify
2379 		 */
2380 		if (next == NULL) {
2381 			/* Change to a write lock */
2382 			if (rwtype == RW_READER &&
2383 			    rw_tryupgrade(&(pgtable->swpg_rwlock)) == 0) {
2384 				rw_exit(&(pgtable->swpg_rwlock));
2385 				rw_enter(&(pgtable->swpg_rwlock), RW_WRITER);
2386 				rwtype = RW_WRITER;
2387 				goto again;
2388 			}
2389 			rwtype = RW_WRITER;
2390 			pgtable_zero(immu, new);
2391 			next = new;
2392 			new = NULL;
2393 			(pgtable->swpg_next_array)[idx] = next;
2394 			PDE_set_one(immu, hwp, next, rdip, immu_flags);
2395 			set = B_TRUE;
2396 			rw_downgrade(&(pgtable->swpg_rwlock));
2397 			rwtype = RW_READER;
2398 		} else {
2399 			hw_pdte_t pde = *hwp;
2400 
2401 #ifndef  BUGGY_DRIVERS
2402 			/*
2403 			 * If buggy driver we already set permission
2404 			 * READ+WRITE so nothing to do for that case
2405 			 * XXX Check that read writer perms change before
2406 			 * actually setting perms. Also need to hold lock
2407 			 */
2408 			if (immu_flags & IMMU_FLAGS_READ)
2409 				PDTE_SET_READ(pde);
2410 			if (immu_flags & IMMU_FLAGS_WRITE)
2411 				PDTE_SET_WRITE(pde);
2412 
2413 #endif
2414 
2415 			*hwp = pde;
2416 		}
2417 
2418 		ASSERT(PDTE_check(immu, *hwp, next, 0, rdip, immu_flags)
2419 		    == B_TRUE);
2420 
2421 		(xlate - 1)->xlt_pgtable = next;
2422 		ASSERT(rwtype == RW_READER);
2423 		rw_exit(&(pgtable->swpg_rwlock));
2424 	}
2425 
2426 	if (new) {
2427 		pgtable_free(immu, new);
2428 	}
2429 
2430 	return (set);
2431 }
2432 
2433 /*
2434  * dvma_map()
2435  *     map a contiguous range of DVMA pages
2436  *
2437  *     immu: IOMMU unit for which we are generating DVMA cookies
2438  *   domain: domain
2439  *    sdvma: Starting dvma
2440  *   spaddr: Starting paddr
2441  *   npages: Number of pages
2442  *     rdip: requesting device
2443  *     immu_flags: flags
2444  */
2445 static boolean_t
2446 dvma_map(immu_t *immu, domain_t *domain, uint64_t sdvma, uint64_t snvpages,
2447     dcookie_t *dcookies, int dcount, dev_info_t *rdip, immu_flags_t immu_flags)
2448 {
2449 	uint64_t dvma;
2450 	uint64_t n;
2451 	int nlevels = immu->immu_dvma_nlevels;
2452 	xlate_t xlate[IMMU_PGTABLE_MAX_LEVELS + 1] = {0};
2453 	boolean_t pde_set = B_FALSE;
2454 
2455 	ASSERT(nlevels <= IMMU_PGTABLE_MAX_LEVELS);
2456 	ASSERT(sdvma % IMMU_PAGESIZE == 0);
2457 	ASSERT(snvpages);
2458 
2459 	n = snvpages;
2460 	dvma = sdvma;
2461 
2462 	while (n > 0) {
2463 		xlate_setup(immu, dvma, xlate, nlevels, rdip);
2464 
2465 		/* Lookup or allocate PGDIRs and PGTABLEs if necessary */
2466 		if (PDE_set_all(immu, domain, xlate, nlevels, rdip, immu_flags)
2467 		    == B_TRUE) {
2468 			pde_set = B_TRUE;
2469 		}
2470 
2471 		/* set all matching ptes that fit into this leaf pgtable */
2472 		PTE_set_all(immu, domain, &xlate[1], &dvma, &n, dcookies,
2473 		    dcount, rdip, immu_flags);
2474 	}
2475 
2476 	return (pde_set);
2477 }
2478 
2479 /*
2480  * dvma_unmap()
2481  *   unmap a range of DVMAs
2482  *
2483  * immu: IOMMU unit state
2484  * domain: domain for requesting device
2485  * ddip: domain-dip
2486  * dvma: starting DVMA
2487  * npages: Number of IMMU pages to be unmapped
2488  * rdip: requesting device
2489  */
2490 static void
2491 dvma_unmap(immu_t *immu, domain_t *domain, uint64_t sdvma, uint64_t snpages,
2492     dev_info_t *rdip)
2493 {
2494 	int nlevels = immu->immu_dvma_nlevels;
2495 	xlate_t xlate[IMMU_PGTABLE_MAX_LEVELS + 1] = {0};
2496 	uint64_t n;
2497 	uint64_t dvma;
2498 
2499 	ASSERT(nlevels <= IMMU_PGTABLE_MAX_LEVELS);
2500 	ASSERT(sdvma != 0);
2501 	ASSERT(sdvma % IMMU_PAGESIZE == 0);
2502 	ASSERT(snpages);
2503 
2504 	dvma = sdvma;
2505 	n = snpages;
2506 
2507 	while (n > 0) {
2508 		/* setup the xlate array */
2509 		xlate_setup(immu, dvma, xlate, nlevels, rdip);
2510 
2511 		/* just lookup existing pgtables. Should never fail */
2512 		PDE_lookup(immu, domain, xlate, nlevels, rdip);
2513 
2514 		/* clear all matching ptes that fit into this leaf pgtable */
2515 		PTE_clear_all(immu, domain, &xlate[1], &dvma, &n, rdip);
2516 	}
2517 
2518 	/* No need to flush IOTLB after unmap */
2519 }
2520 
2521 static uint64_t
2522 dvma_alloc(ddi_dma_impl_t *hp, domain_t *domain, uint_t npages)
2523 {
2524 	ddi_dma_attr_t *dma_attr;
2525 	uint64_t dvma;
2526 	size_t xsize, align;
2527 	uint64_t minaddr, maxaddr;
2528 
2529 	ASSERT(domain->dom_maptype != IMMU_MAPTYPE_UNITY);
2530 
2531 	/* shotcuts */
2532 	dma_attr = &(hp->dmai_attr);
2533 
2534 	/* parameters */
2535 	xsize = npages * IMMU_PAGESIZE;
2536 	align = MAX((size_t)(dma_attr->dma_attr_align), IMMU_PAGESIZE);
2537 	minaddr = dma_attr->dma_attr_addr_lo;
2538 	maxaddr = dma_attr->dma_attr_addr_hi + 1;
2539 	/* nocross is checked in cookie_update() */
2540 
2541 	/* handle the rollover cases */
2542 	if (maxaddr < dma_attr->dma_attr_addr_hi) {
2543 		maxaddr = dma_attr->dma_attr_addr_hi;
2544 	}
2545 
2546 	/*
2547 	 * allocate from vmem arena.
2548 	 */
2549 	dvma = (uint64_t)(uintptr_t)vmem_xalloc(domain->dom_dvma_arena,
2550 	    xsize, align, 0, 0, (void *)(uintptr_t)minaddr,
2551 	    (void *)(uintptr_t)maxaddr, VM_NOSLEEP);
2552 
2553 	ASSERT(dvma);
2554 	ASSERT(dvma >= minaddr);
2555 	ASSERT(dvma + xsize - 1 < maxaddr);
2556 
2557 	return (dvma);
2558 }
2559 
2560 static void
2561 dvma_free(domain_t *domain, uint64_t dvma, uint64_t npages)
2562 {
2563 	uint64_t size = npages * IMMU_PAGESIZE;
2564 
2565 	ASSERT(domain);
2566 	ASSERT(domain->dom_did > 0);
2567 	ASSERT(dvma);
2568 	ASSERT(npages);
2569 
2570 	if (domain->dom_maptype != IMMU_MAPTYPE_XLATE) {
2571 		ASSERT(domain->dom_maptype == IMMU_MAPTYPE_UNITY);
2572 		return;
2573 	}
2574 
2575 	vmem_free(domain->dom_dvma_arena, (void *)(uintptr_t)dvma, size);
2576 }
2577 /*ARGSUSED*/
2578 static void
2579 cookie_free(rootnex_dma_t *dma, immu_t *immu, domain_t *domain,
2580     dev_info_t *rdip)
2581 {
2582 	int i;
2583 	uint64_t dvma;
2584 	uint64_t npages;
2585 	dvcookie_t  *dvcookies = dma->dp_dvcookies;
2586 
2587 	ASSERT(dma->dp_max_cookies);
2588 	ASSERT(dma->dp_max_dcookies);
2589 	ASSERT(dma->dp_dvmax < dma->dp_max_cookies);
2590 	ASSERT(dma->dp_dmax < dma->dp_max_dcookies);
2591 
2592 	/*
2593 	 * we allocated DVMA in a single chunk. Calculate total number
2594 	 * of pages
2595 	 */
2596 	for (i = 0, npages = 0; i <= dma->dp_dvmax; i++) {
2597 		npages += dvcookies[i].dvck_npages;
2598 	}
2599 	dvma = dvcookies[0].dvck_dvma;
2600 #ifdef DEBUG
2601 	/* Unmap only in DEBUG mode */
2602 	dvma_unmap(immu, domain, dvma, npages, rdip);
2603 #endif
2604 	dvma_free(domain, dvma, npages);
2605 
2606 	kmem_free(dma->dp_dvcookies, sizeof (dvcookie_t) * dma->dp_max_cookies);
2607 	dma->dp_dvcookies = NULL;
2608 	kmem_free(dma->dp_dcookies, sizeof (dcookie_t) * dma->dp_max_dcookies);
2609 	dma->dp_dcookies = NULL;
2610 	if (dma->dp_need_to_free_cookie == B_TRUE) {
2611 		kmem_free(dma->dp_cookies, sizeof (ddi_dma_cookie_t) *
2612 		    dma->dp_max_cookies);
2613 		dma->dp_dcookies = NULL;
2614 		dma->dp_need_to_free_cookie = B_FALSE;
2615 	}
2616 
2617 	dma->dp_max_cookies = 0;
2618 	dma->dp_max_dcookies = 0;
2619 	dma->dp_cookie_size = 0;
2620 	dma->dp_dvmax = 0;
2621 	dma->dp_dmax = 0;
2622 }
2623 
2624 /*
2625  * cookie_alloc()
2626  */
2627 static int
2628 cookie_alloc(rootnex_dma_t *dma, struct ddi_dma_req *dmareq,
2629     ddi_dma_attr_t *attr, uint_t prealloc)
2630 {
2631 	int kmflag;
2632 	rootnex_sglinfo_t *sinfo = &(dma->dp_sglinfo);
2633 	dvcookie_t *dvcookies = dma->dp_dvcookies;
2634 	dcookie_t *dcookies = dma->dp_dcookies;
2635 	ddi_dma_cookie_t *cookies = dma->dp_cookies;
2636 	uint64_t max_cookies;
2637 	uint64_t max_dcookies;
2638 	uint64_t cookie_size;
2639 
2640 	/* we need to allocate new array */
2641 	if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
2642 		kmflag =  KM_SLEEP;
2643 	} else {
2644 		kmflag =  KM_NOSLEEP;
2645 	}
2646 
2647 	/*
2648 	 * XXX make sure cookies size doen't exceed sinfo->si_max_cookie_size;
2649 	 */
2650 
2651 	/*
2652 	 * figure out the rough estimate of array size
2653 	 * At a minimum, each cookie must hold 1 page.
2654 	 * At a maximum, it cannot exceed dma_attr_sgllen
2655 	 */
2656 	max_dcookies = dmareq->dmar_object.dmao_size + IMMU_PAGEOFFSET;
2657 	max_dcookies /= IMMU_PAGESIZE;
2658 	max_dcookies++;
2659 	max_cookies = MIN(max_dcookies, attr->dma_attr_sgllen);
2660 
2661 	/* allocate the dvma cookie array */
2662 	dvcookies = kmem_zalloc(sizeof (dvcookie_t) * max_cookies, kmflag);
2663 	if (dvcookies == NULL) {
2664 		return (DDI_FAILURE);
2665 	}
2666 
2667 	/* allocate the "phys" cookie array */
2668 	dcookies = kmem_zalloc(sizeof (dcookie_t) * max_dcookies, kmflag);
2669 	if (dcookies == NULL) {
2670 		kmem_free(dvcookies, sizeof (dvcookie_t) * max_cookies);
2671 		dvcookies = NULL;
2672 		return (DDI_FAILURE);
2673 	}
2674 
2675 	/* allocate the "real" cookie array  - the one given to users */
2676 	cookie_size = sizeof (ddi_dma_cookie_t) * max_cookies;
2677 	if (max_cookies > prealloc) {
2678 		cookies = kmem_zalloc(cookie_size, kmflag);
2679 		if (cookies == NULL) {
2680 			kmem_free(dvcookies, sizeof (dvcookie_t) * max_cookies);
2681 			kmem_free(dcookies, sizeof (dcookie_t) * max_dcookies);
2682 			goto fail;
2683 		}
2684 		dma->dp_need_to_free_cookie = B_TRUE;
2685 	} else {
2686 		/* the preallocated buffer fits this size */
2687 		cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2688 		bzero(cookies, sizeof (ddi_dma_cookie_t)* max_cookies);
2689 		dma->dp_need_to_free_cookie = B_FALSE;
2690 	}
2691 
2692 	dma->dp_dvcookies = dvcookies;
2693 	dma->dp_dcookies = dcookies;
2694 	dma->dp_cookies = cookies;
2695 	dma->dp_cookie_size = cookie_size;
2696 	dma->dp_max_cookies = max_cookies;
2697 	dma->dp_max_dcookies = max_dcookies;
2698 	dma->dp_dvmax = 0;
2699 	dma->dp_dmax = 0;
2700 	sinfo->si_max_pages = dma->dp_max_cookies;
2701 
2702 	return (DDI_SUCCESS);
2703 
2704 fail:
2705 	dma->dp_dvcookies = NULL;
2706 	dma->dp_dcookies = NULL;
2707 	dma->dp_cookies = NULL;
2708 	dma->dp_cookie_size = 0;
2709 	dma->dp_max_cookies = 0;
2710 	dma->dp_max_dcookies = 0;
2711 	dma->dp_dvmax = 0;
2712 	dma->dp_dmax = 0;
2713 	dma->dp_need_to_free_cookie = B_FALSE;
2714 	sinfo->si_max_pages = 0;
2715 
2716 	return (DDI_FAILURE);
2717 }
2718 
2719 /*ARGSUSED*/
2720 static void
2721 cookie_update(domain_t *domain, rootnex_dma_t *dma, paddr_t paddr,
2722     int64_t psize, uint64_t maxseg, size_t nocross)
2723 {
2724 	dvcookie_t *dvcookies = dma->dp_dvcookies;
2725 	dcookie_t *dcookies = dma->dp_dcookies;
2726 	ddi_dma_cookie_t *cookies = dma->dp_cookies;
2727 	uint64_t dvmax = dma->dp_dvmax;
2728 	uint64_t dmax = dma->dp_dmax;
2729 
2730 	ASSERT(dvmax < dma->dp_max_cookies);
2731 	ASSERT(dmax < dma->dp_max_dcookies);
2732 
2733 	paddr &= IMMU_PAGEMASK;
2734 
2735 	ASSERT(paddr);
2736 	ASSERT(psize);
2737 	ASSERT(maxseg);
2738 
2739 	/*
2740 	 * check to see if this page would put us
2741 	 * over the max cookie size.
2742 	 */
2743 	if (cookies[dvmax].dmac_size + psize > maxseg) {
2744 		dvmax++;    /* use the next dvcookie */
2745 		dmax++;    /* also means we use the next dcookie */
2746 		ASSERT(dvmax < dma->dp_max_cookies);
2747 		ASSERT(dmax < dma->dp_max_dcookies);
2748 	}
2749 
2750 	/*
2751 	 * check to see if this page would make us larger than
2752 	 * the nocross boundary. If yes, create a new cookie
2753 	 * otherwise we will fail later with vmem_xalloc()
2754 	 * due to overconstrained alloc requests
2755 	 * nocross == 0 implies no nocross constraint.
2756 	 */
2757 	if (nocross > 0) {
2758 		ASSERT((dvcookies[dvmax].dvck_npages) * IMMU_PAGESIZE
2759 		    <= nocross);
2760 		if ((dvcookies[dvmax].dvck_npages + 1) * IMMU_PAGESIZE
2761 		    > nocross) {
2762 			dvmax++;    /* use the next dvcookie */
2763 			dmax++;    /* also means we use the next dcookie */
2764 			ASSERT(dvmax < dma->dp_max_cookies);
2765 			ASSERT(dmax < dma->dp_max_dcookies);
2766 		}
2767 		ASSERT((dvcookies[dvmax].dvck_npages) * IMMU_PAGESIZE
2768 		    <= nocross);
2769 	}
2770 
2771 	/*
2772 	 * If the cookie is empty
2773 	 */
2774 	if (dvcookies[dvmax].dvck_npages == 0) {
2775 		ASSERT(cookies[dvmax].dmac_size == 0);
2776 		ASSERT(dvcookies[dvmax].dvck_dvma == 0);
2777 		ASSERT(dvcookies[dvmax].dvck_npages
2778 		    == 0);
2779 		ASSERT(dcookies[dmax].dck_paddr == 0);
2780 		ASSERT(dcookies[dmax].dck_npages == 0);
2781 
2782 		dvcookies[dvmax].dvck_dvma = 0;
2783 		dvcookies[dvmax].dvck_npages = 1;
2784 		dcookies[dmax].dck_paddr = paddr;
2785 		dcookies[dmax].dck_npages = 1;
2786 		cookies[dvmax].dmac_size = psize;
2787 	} else {
2788 		/* Cookie not empty. Add to it */
2789 		cookies[dma->dp_dvmax].dmac_size += psize;
2790 		ASSERT(dvcookies[dma->dp_dvmax].dvck_dvma == 0);
2791 		dvcookies[dma->dp_dvmax].dvck_npages++;
2792 		ASSERT(dcookies[dmax].dck_paddr != 0);
2793 		ASSERT(dcookies[dmax].dck_npages != 0);
2794 
2795 		/* Check if this paddr is contiguous */
2796 		if (IMMU_CONTIG_PADDR(dcookies[dmax], paddr)) {
2797 			dcookies[dmax].dck_npages++;
2798 		} else {
2799 			/* No, we need a new dcookie */
2800 			dmax++;
2801 			ASSERT(dcookies[dmax].dck_paddr == 0);
2802 			ASSERT(dcookies[dmax].dck_npages == 0);
2803 			dcookies[dmax].dck_paddr = paddr;
2804 			dcookies[dmax].dck_npages = 1;
2805 		}
2806 	}
2807 
2808 	dma->dp_dvmax = dvmax;
2809 	dma->dp_dmax = dmax;
2810 }
2811 
2812 static void
2813 cookie_finalize(ddi_dma_impl_t *hp, immu_t *immu, domain_t *domain,
2814     dev_info_t *rdip, immu_flags_t immu_flags)
2815 {
2816 	int i;
2817 	rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2818 	dvcookie_t *dvcookies = dma->dp_dvcookies;
2819 	dcookie_t *dcookies = dma->dp_dcookies;
2820 	ddi_dma_cookie_t *cookies = dma->dp_cookies;
2821 	uint64_t npages;
2822 	uint64_t dvma;
2823 	boolean_t pde_set;
2824 
2825 	/* First calculate the total number of pages required */
2826 	for (i = 0, npages = 0; i <= dma->dp_dvmax; i++) {
2827 		npages += dvcookies[i].dvck_npages;
2828 	}
2829 
2830 	/* Now allocate dvma */
2831 	dvma = dvma_alloc(hp, domain, npages);
2832 
2833 	/* Now map the dvma */
2834 	pde_set = dvma_map(immu, domain, dvma, npages, dcookies,
2835 	    dma->dp_dmax + 1, rdip, immu_flags);
2836 
2837 	/* Invalidate the IOTLB */
2838 	immu_regs_iotlb_flush(immu, domain->dom_did, dvma, npages,
2839 	    pde_set == B_TRUE ? TLB_IVA_WHOLE : TLB_IVA_LEAF, IOTLB_PSI);
2840 
2841 	/* Now setup dvcookies and real cookie addresses */
2842 	for (i = 0; i <= dma->dp_dvmax; i++) {
2843 		dvcookies[i].dvck_dvma = dvma;
2844 		cookies[i].dmac_laddress = dvma;
2845 		ASSERT(cookies[i].dmac_size != 0);
2846 		cookies[i].dmac_type = 0;
2847 		dvma += (dvcookies[i].dvck_npages * IMMU_PAGESIZE);
2848 	}
2849 
2850 #ifdef TEST
2851 	immu_regs_iotlb_flush(immu, domain->dom_did, 0, 0, 0, IOTLB_DSI);
2852 #endif
2853 }
2854 
2855 /*
2856  * cookie_create()
2857  */
2858 static int
2859 cookie_create(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
2860     ddi_dma_attr_t *a, immu_t *immu, domain_t *domain, dev_info_t *rdip,
2861     uint_t prealloc_count, immu_flags_t immu_flags)
2862 {
2863 	ddi_dma_atyp_t buftype;
2864 	uint64_t offset;
2865 	page_t **pparray;
2866 	uint64_t paddr;
2867 	uint_t psize;
2868 	uint_t size;
2869 	uint64_t maxseg;
2870 	caddr_t vaddr;
2871 	uint_t pcnt;
2872 	page_t *page;
2873 	rootnex_sglinfo_t *sglinfo;
2874 	ddi_dma_obj_t *dmar_object;
2875 	rootnex_dma_t *dma;
2876 	size_t nocross;
2877 
2878 	dma = (rootnex_dma_t *)hp->dmai_private;
2879 	sglinfo = &(dma->dp_sglinfo);
2880 	dmar_object = &(dmareq->dmar_object);
2881 	maxseg = sglinfo->si_max_cookie_size;
2882 	pparray = dmar_object->dmao_obj.virt_obj.v_priv;
2883 	vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2884 	buftype = dmar_object->dmao_type;
2885 	size = dmar_object->dmao_size;
2886 	nocross = (size_t)(a->dma_attr_seg + 1);
2887 
2888 	/*
2889 	 * Allocate cookie, dvcookie and dcookie
2890 	 */
2891 	if (cookie_alloc(dma, dmareq, a, prealloc_count) != DDI_SUCCESS) {
2892 		return (DDI_FAILURE);
2893 	}
2894 	hp->dmai_cookie = dma->dp_cookies;
2895 
2896 	pcnt = 0;
2897 
2898 	/* retrieve paddr, psize, offset from dmareq */
2899 	if (buftype == DMA_OTYP_PAGES) {
2900 		page = dmar_object->dmao_obj.pp_obj.pp_pp;
2901 		ASSERT(!PP_ISFREE(page) && PAGE_LOCKED(page));
2902 		offset =  dmar_object->dmao_obj.pp_obj.pp_offset &
2903 		    MMU_PAGEOFFSET;
2904 		paddr = pfn_to_pa(page->p_pagenum) + offset;
2905 		psize = MIN((MMU_PAGESIZE - offset), size);
2906 		sglinfo->si_asp = NULL;
2907 		page = page->p_next;
2908 	} else {
2909 		ASSERT((buftype == DMA_OTYP_VADDR) ||
2910 		    (buftype == DMA_OTYP_BUFVADDR));
2911 		sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2912 		if (sglinfo->si_asp == NULL) {
2913 			sglinfo->si_asp = &kas;
2914 		}
2915 		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2916 		if (pparray != NULL) {
2917 			ASSERT(!PP_ISFREE(pparray[pcnt]));
2918 			paddr = pfn_to_pa(pparray[pcnt]->p_pagenum) + offset;
2919 			psize = MIN((MMU_PAGESIZE - offset), size);
2920 			pcnt++;
2921 		} else {
2922 			paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
2923 			    vaddr)) + offset;
2924 			psize = MIN(size, (MMU_PAGESIZE - offset));
2925 			vaddr += psize;
2926 		}
2927 	}
2928 
2929 	/* save the iommu page offset */
2930 	sglinfo->si_buf_offset = offset & IMMU_PAGEOFFSET;
2931 
2932 	/*
2933 	 * setup dvcookie and dcookie for [paddr, paddr+psize)
2934 	 */
2935 	cookie_update(domain, dma, paddr, psize, maxseg, nocross);
2936 
2937 	size -= psize;
2938 	while (size > 0) {
2939 		/* get the size for this page (i.e. partial or full page) */
2940 		psize = MIN(size, MMU_PAGESIZE);
2941 		if (buftype == DMA_OTYP_PAGES) {
2942 			/* get the paddr from the page_t */
2943 			ASSERT(!PP_ISFREE(page) && PAGE_LOCKED(page));
2944 			paddr = pfn_to_pa(page->p_pagenum);
2945 			page = page->p_next;
2946 		} else if (pparray != NULL) {
2947 			/* index into the array of page_t's to get the paddr */
2948 			ASSERT(!PP_ISFREE(pparray[pcnt]));
2949 			paddr = pfn_to_pa(pparray[pcnt]->p_pagenum);
2950 			pcnt++;
2951 		} else {
2952 			/* call into the VM to get the paddr */
2953 			paddr = pfn_to_pa(hat_getpfnum
2954 			    (sglinfo->si_asp->a_hat, vaddr));
2955 			vaddr += psize;
2956 		}
2957 		/*
2958 		 * set dvcookie and dcookie for [paddr, paddr+psize)
2959 		 */
2960 		cookie_update(domain, dma, paddr, psize, maxseg, nocross);
2961 		size -= psize;
2962 	}
2963 
2964 	cookie_finalize(hp, immu, domain, rdip, immu_flags);
2965 
2966 	/* take account in the offset into the first page */
2967 	dma->dp_cookies[0].dmac_laddress += sglinfo->si_buf_offset;
2968 
2969 	/* save away how many cookies we have */
2970 	sglinfo->si_sgl_size = dma->dp_dvmax + 1;
2971 
2972 	return (DDI_SUCCESS);
2973 }
2974 
2975 /* ############################# Functions exported ######################## */
2976 
2977 /*
2978  * setup the DVMA subsystem
2979  * this code runs only for the first IOMMU unit
2980  */
2981 void
2982 immu_dvma_setup(list_t *listp)
2983 {
2984 	immu_t *immu;
2985 	uint_t kval;
2986 	size_t nchains;
2987 
2988 	/* locks */
2989 	mutex_init(&immu_domain_lock, NULL, MUTEX_DEFAULT, NULL);
2990 
2991 	/* Create lists */
2992 	list_create(&immu_unity_domain_list, sizeof (domain_t),
2993 	    offsetof(domain_t, dom_maptype_node));
2994 	list_create(&immu_xlate_domain_list, sizeof (domain_t),
2995 	    offsetof(domain_t, dom_maptype_node));
2996 
2997 	/* Setup BDF domain hash */
2998 	nchains = 0xff;
2999 	kval = mod_hash_iddata_gen(nchains);
3000 
3001 	bdf_domain_hash = mod_hash_create_extended("BDF-DOMAIN_HASH",
3002 	    nchains, mod_hash_null_keydtor, mod_hash_null_valdtor,
3003 	    mod_hash_byid, (void *)(uintptr_t)kval, mod_hash_idkey_cmp,
3004 	    KM_NOSLEEP);
3005 	ASSERT(bdf_domain_hash);
3006 
3007 	immu = list_head(listp);
3008 	for (; immu; immu = list_next(listp, immu)) {
3009 		create_unity_domain(immu);
3010 		did_init(immu);
3011 		context_init(immu);
3012 		immu->immu_dvma_setup = B_TRUE;
3013 	}
3014 }
3015 
3016 /*
3017  * Startup up one DVMA unit
3018  */
3019 void
3020 immu_dvma_startup(immu_t *immu)
3021 {
3022 	ASSERT(immu);
3023 	ASSERT(immu->immu_dvma_running == B_FALSE);
3024 
3025 	if (immu_gfxdvma_enable == B_FALSE &&
3026 	    immu->immu_dvma_gfx_only == B_TRUE) {
3027 		return;
3028 	}
3029 
3030 	/*
3031 	 * DVMA will start once IOMMU is "running"
3032 	 */
3033 	ASSERT(immu->immu_dvma_running == B_FALSE);
3034 	immu->immu_dvma_running = B_TRUE;
3035 }
3036 
3037 /*
3038  * immu_dvma_physmem_update()
3039  *       called when the installed memory on a
3040  *       system increases, to expand domain DVMA
3041  *       for domains with UNITY mapping
3042  */
3043 void
3044 immu_dvma_physmem_update(uint64_t addr, uint64_t size)
3045 {
3046 	uint64_t start;
3047 	uint64_t npages;
3048 	int dcount;
3049 	dcookie_t dcookies[1] = {0};
3050 	domain_t *domain;
3051 
3052 	/*
3053 	 * Just walk the system-wide list of domains with
3054 	 * UNITY mapping. Both the list of *all* domains
3055 	 * and *UNITY* domains is protected by the same
3056 	 * single lock
3057 	 */
3058 	mutex_enter(&immu_domain_lock);
3059 	domain = list_head(&immu_unity_domain_list);
3060 	for (; domain; domain = list_next(&immu_unity_domain_list, domain)) {
3061 
3062 		/* There is no vmem_arena for unity domains. Just map it */
3063 		ddi_err(DER_LOG, NULL, "IMMU: unity-domain: Adding map "
3064 		    "[0x%" PRIx64 " - 0x%" PRIx64 "]", addr, addr + size);
3065 
3066 		start = IMMU_ROUNDOWN(addr);
3067 		npages = (IMMU_ROUNDUP(size) / IMMU_PAGESIZE) + 1;
3068 
3069 		dcookies[0].dck_paddr = start;
3070 		dcookies[0].dck_npages = npages;
3071 		dcount = 1;
3072 		(void) dvma_map(domain->dom_immu, domain, start, npages,
3073 		    dcookies, dcount, NULL, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);
3074 
3075 	}
3076 	mutex_exit(&immu_domain_lock);
3077 }
3078 
3079 
3080 int
3081 immu_dvma_map(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, memrng_t *mrng,
3082     uint_t prealloc_count, dev_info_t *rdip, immu_flags_t immu_flags)
3083 {
3084 	ddi_dma_attr_t *attr;
3085 	dev_info_t *ddip;
3086 	domain_t *domain;
3087 	immu_t *immu;
3088 	dcookie_t dcookies[1] = {0};
3089 	int dcount = 0;
3090 	boolean_t pde_set = B_TRUE;
3091 	int r = DDI_FAILURE;
3092 
3093 	ASSERT(immu_enable == B_TRUE);
3094 	ASSERT(immu_running == B_TRUE || !(immu_flags & IMMU_FLAGS_DMAHDL));
3095 	ASSERT(hp || !(immu_flags & IMMU_FLAGS_DMAHDL));
3096 
3097 	/*
3098 	 * Intel IOMMU will only be turned on if IOMMU
3099 	 * page size is a multiple of IOMMU page size
3100 	 */
3101 
3102 	/*LINTED*/
3103 	ASSERT(MMU_PAGESIZE % IMMU_PAGESIZE == 0);
3104 
3105 	/* Can only do DVMA if dip is attached */
3106 	if (rdip == NULL) {
3107 		ddi_err(DER_PANIC, rdip, "DVMA map: No device specified");
3108 		/*NOTREACHED*/
3109 	}
3110 
3111 	immu_flags |= dma_to_immu_flags(dmareq);
3112 
3113 	immu = immu_dvma_get_immu(rdip, immu_flags);
3114 	if (immu == NULL) {
3115 		/*
3116 		 * possible that there is no IOMMU unit for this device
3117 		 * - BIOS bugs are one example.
3118 		 */
3119 		ddi_err(DER_WARN, rdip, "No IMMU unit found for device");
3120 		return (DDI_DMA_NORESOURCES);
3121 	}
3122 
3123 	/*
3124 	 * redirect isa devices attached under lpc to lpc dip
3125 	 */
3126 	if (strcmp(ddi_node_name(ddi_get_parent(rdip)), "isa") == 0) {
3127 		rdip = get_lpc_devinfo(immu, rdip, immu_flags);
3128 		if (rdip == NULL) {
3129 			ddi_err(DER_PANIC, rdip, "IMMU redirect failed");
3130 			/*NOTREACHED*/
3131 		}
3132 	}
3133 
3134 	/* Reset immu, as redirection can change IMMU */
3135 	immu = NULL;
3136 
3137 	/*
3138 	 * for gart, redirect to the real graphic devinfo
3139 	 */
3140 	if (strcmp(ddi_node_name(rdip), "agpgart") == 0) {
3141 		rdip = get_gfx_devinfo(rdip);
3142 		if (rdip == NULL) {
3143 			ddi_err(DER_PANIC, rdip, "IMMU redirect failed");
3144 			/*NOTREACHED*/
3145 		}
3146 	}
3147 
3148 	/*
3149 	 * Setup DVMA domain for the device. This does
3150 	 * work only the first time we do DVMA for a
3151 	 * device.
3152 	 */
3153 	ddip = NULL;
3154 	domain = device_domain(rdip, &ddip, immu_flags);
3155 	if (domain == NULL) {
3156 		ASSERT(ddip == NULL);
3157 		ddi_err(DER_MODE, rdip, "Intel IOMMU setup failed for device");
3158 		return (DDI_DMA_NORESOURCES);
3159 	}
3160 
3161 	/*
3162 	 * If a domain is found, we must also have a domain dip
3163 	 * which is the topmost ancestor dip of rdip that shares
3164 	 * the same domain with rdip.
3165 	 */
3166 	if (domain->dom_did == 0 || ddip == NULL) {
3167 		ddi_err(DER_MODE, rdip, "domain did 0(%d) or ddip NULL(%p)",
3168 		    domain->dom_did, ddip);
3169 		return (DDI_DMA_NORESOURCES);
3170 	}
3171 
3172 	immu = domain->dom_immu;
3173 	ASSERT(immu);
3174 	if (domain->dom_did == IMMU_UNITY_DID) {
3175 		ASSERT(domain == immu->immu_unity_domain);
3176 		/* mapping already done. Let rootnex create cookies */
3177 		r = DDI_DMA_USE_PHYSICAL;
3178 	} else  if (immu_flags & IMMU_FLAGS_DMAHDL) {
3179 
3180 		/* if we have a DMA handle, the IOMMUs must be running */
3181 		ASSERT(immu->immu_regs_running == B_TRUE);
3182 		ASSERT(immu->immu_dvma_running == B_TRUE);
3183 
3184 		attr = &hp->dmai_attr;
3185 		if (attr == NULL) {
3186 			ddi_err(DER_PANIC, rdip,
3187 			    "DMA handle (%p): NULL attr", hp);
3188 			/*NOTREACHED*/
3189 		}
3190 
3191 		if (cookie_create(hp, dmareq, attr, immu, domain, rdip,
3192 		    prealloc_count, immu_flags) != DDI_SUCCESS) {
3193 			ddi_err(DER_MODE, rdip, "dvcookie_alloc: failed");
3194 			return (DDI_DMA_NORESOURCES);
3195 		}
3196 		r = DDI_DMA_MAPPED;
3197 	} else if (immu_flags & IMMU_FLAGS_MEMRNG) {
3198 		dcookies[0].dck_paddr = mrng->mrng_start;
3199 		dcookies[0].dck_npages = mrng->mrng_npages;
3200 		dcount = 1;
3201 		pde_set = dvma_map(immu, domain, mrng->mrng_start,
3202 		    mrng->mrng_npages, dcookies, dcount, rdip, immu_flags);
3203 		immu_regs_iotlb_flush(immu, domain->dom_did, mrng->mrng_start,
3204 		    mrng->mrng_npages, pde_set == B_TRUE ?
3205 		    TLB_IVA_WHOLE : TLB_IVA_LEAF, IOTLB_PSI);
3206 		r = DDI_DMA_MAPPED;
3207 	} else {
3208 		ddi_err(DER_PANIC, rdip, "invalid flags for immu_dvma_map()");
3209 		/*NOTREACHED*/
3210 	}
3211 
3212 	/*
3213 	 * Update the root and context entries
3214 	 */
3215 	if (immu_context_update(immu, domain, ddip, rdip, immu_flags)
3216 	    != DDI_SUCCESS) {
3217 		ddi_err(DER_MODE, rdip, "DVMA map: context update failed");
3218 		return (DDI_DMA_NORESOURCES);
3219 	}
3220 
3221 	immu_regs_wbf_flush(immu);
3222 
3223 	return (r);
3224 }
3225 
3226 int
3227 immu_dvma_unmap(ddi_dma_impl_t *hp, dev_info_t *rdip)
3228 {
3229 	ddi_dma_attr_t *attr;
3230 	rootnex_dma_t *dma;
3231 	domain_t *domain;
3232 	immu_t *immu;
3233 	dev_info_t *ddip;
3234 	immu_flags_t immu_flags;
3235 
3236 	ASSERT(immu_enable == B_TRUE);
3237 	ASSERT(immu_running == B_TRUE);
3238 	ASSERT(hp);
3239 
3240 	/*
3241 	 * Intel IOMMU will only be turned on if IOMMU
3242 	 * page size is same as MMU page size
3243 	 */
3244 	/*LINTED*/
3245 	ASSERT(MMU_PAGESIZE == IMMU_PAGESIZE);
3246 
3247 	/* rdip need not be attached */
3248 	if (rdip == NULL) {
3249 		ddi_err(DER_PANIC, rdip, "DVMA unmap: No device specified");
3250 		return (DDI_DMA_NORESOURCES);
3251 	}
3252 
3253 	/*
3254 	 * Get the device domain, this should always
3255 	 * succeed since there had to be a domain to
3256 	 * setup DVMA.
3257 	 */
3258 	dma = (rootnex_dma_t *)hp->dmai_private;
3259 	attr = &hp->dmai_attr;
3260 	if (attr == NULL) {
3261 		ddi_err(DER_PANIC, rdip, "DMA handle (%p) has NULL attr", hp);
3262 		/*NOTREACHED*/
3263 	}
3264 	immu_flags = dma->dp_sleep_flags;
3265 
3266 	immu = immu_dvma_get_immu(rdip, immu_flags);
3267 	if (immu == NULL) {
3268 		/*
3269 		 * possible that there is no IOMMU unit for this device
3270 		 * - BIOS bugs are one example.
3271 		 */
3272 		ddi_err(DER_WARN, rdip, "No IMMU unit found for device");
3273 		return (DDI_DMA_NORESOURCES);
3274 	}
3275 
3276 
3277 	/*
3278 	 * redirect isa devices attached under lpc to lpc dip
3279 	 */
3280 	if (strcmp(ddi_node_name(ddi_get_parent(rdip)), "isa") == 0) {
3281 		rdip = get_lpc_devinfo(immu, rdip, immu_flags);
3282 		if (rdip == NULL) {
3283 			ddi_err(DER_PANIC, rdip, "IMMU redirect failed");
3284 			/*NOTREACHED*/
3285 		}
3286 	}
3287 
3288 	/* Reset immu, as redirection can change IMMU */
3289 	immu = NULL;
3290 
3291 	/*
3292 	 * for gart, redirect to the real graphic devinfo
3293 	 */
3294 	if (strcmp(ddi_node_name(rdip), "agpgart") == 0) {
3295 		rdip = get_gfx_devinfo(rdip);
3296 		if (rdip == NULL) {
3297 			ddi_err(DER_PANIC, rdip, "IMMU redirect failed");
3298 			/*NOTREACHED*/
3299 		}
3300 	}
3301 
3302 	ddip = NULL;
3303 	domain = device_domain(rdip, &ddip, immu_flags);
3304 	if (domain == NULL || domain->dom_did == 0 || ddip == NULL) {
3305 		ddi_err(DER_MODE, rdip, "Attempt to unmap DVMA for "
3306 		    "a device without domain or with an uninitialized "
3307 		    "domain");
3308 		return (DDI_DMA_NORESOURCES);
3309 	}
3310 
3311 	/*
3312 	 * immu must be set in the domain.
3313 	 */
3314 	immu = domain->dom_immu;
3315 	ASSERT(immu);
3316 	if (domain->dom_did == IMMU_UNITY_DID) {
3317 		ASSERT(domain == immu->immu_unity_domain);
3318 		/*
3319 		 * domain is unity, nothing to do here, let the rootnex
3320 		 * code free the cookies.
3321 		 */
3322 		return (DDI_DMA_USE_PHYSICAL);
3323 	}
3324 
3325 	dma = hp->dmai_private;
3326 	if (dma == NULL) {
3327 		ddi_err(DER_PANIC, rdip, "DVMA unmap: DMA handle (%p) has "
3328 		    "no private dma structure", hp);
3329 		/*NOTREACHED*/
3330 	}
3331 
3332 	cookie_free(dma, immu, domain, rdip);
3333 
3334 	/* No invalidation needed for unmap */
3335 	immu_regs_wbf_flush(immu);
3336 
3337 	return (DDI_SUCCESS);
3338 }
3339 
3340 immu_devi_t *
3341 immu_devi_get(dev_info_t *rdip)
3342 {
3343 	immu_devi_t *immu_devi;
3344 	volatile uintptr_t *vptr = (uintptr_t *)&(DEVI(rdip)->devi_iommu);
3345 
3346 	/* Just want atomic reads. No need for lock */
3347 	immu_devi = (immu_devi_t *)(uintptr_t)atomic_or_64_nv((uint64_t *)vptr,
3348 	    0);
3349 	return (immu_devi);
3350 }
3351