77     dev_info_t *rdip, immu_flags_t immu_flags);
78 static immu_devi_t *create_immu_devi(dev_info_t *rdip, int bus,
82     uint64_t nvpages, immu_dcookie_t *dcookies, int dcount, dev_info_t *rdip,
93     dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
96     dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle);
98     dev_info_t *rdip, ddi_dma_handle_t dma_handle, struct ddi_dma_req *dma_req,
101     dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle);
103     dev_info_t *rdip, ddi_dma_handle_t dma_handle, off_t off, size_t len,
106     dev_info_t *rdip, ddi_dma_handle_t dma_handle, uint_t win,
109     dev_info_t *rdip, ddi_dma_handle_t dma_handle,
112     dev_info_t *rdip, ddi_dma_handle_t dma_handle, ddi_dma_obj_t *dmao);
323 get_lpc_devinfo(immu_t *immu, dev_info_t *rdip, immu_flags_t immu_flags)  in get_lpc_devinfo()  argument
327 	dvarg.dva_rdip = rdip;  in get_lpc_devinfo()
330 	if (immu_walk_ancestor(rdip, NULL, match_lpc,  in get_lpc_devinfo()
332 		ddi_err(DER_MODE, rdip, "Could not walk ancestors to "  in get_lpc_devinfo()
338 		ddi_err(DER_MODE, rdip, "Could not find lpc_devinfo for "  in get_lpc_devinfo()
347 get_gfx_devinfo(dev_info_t *rdip)  in get_gfx_devinfo()  argument
368 		ddi_err(DER_WARN, rdip, "iommu: No GFX device. "  in get_gfx_devinfo()
373 	ddi_err(DER_LOG, rdip, "iommu: GFX redirect to %s",  in get_gfx_devinfo()
693 create_immu_devi(dev_info_t *rdip, int bus, int dev, int func,  in create_immu_devi()  argument
711 		ddi_err(DER_WARN, rdip, "Failed to allocate memory for "  in create_immu_devi()
715 	immu_devi->imd_dip = rdip;  in create_immu_devi()
774 immu_devi_domain(dev_info_t *rdip, dev_info_t **ddipp)  in immu_devi_domain()  argument
782 	immu_devi = immu_devi_get(rdip);  in immu_devi_domain()
787 	mutex_enter(&(DEVI(rdip)->devi_lock));  in immu_devi_domain()
790 	mutex_exit(&(DEVI(rdip)->devi_lock));  in immu_devi_domain()
806 did_alloc(immu_t *immu, dev_info_t *rdip,  in did_alloc()  argument
815 		ddi_err(DER_WARN, rdip, "device domain-id alloc error"  in did_alloc()
1021     dev_info_t *rdip, immu_flags_t immu_flags)  in create_xlate_arena()  argument
1066 	ddi_err(DER_VERB, rdip,  in create_xlate_arena()
1084 		ddi_err(DER_PANIC, rdip,  in create_xlate_arena()
1103 		ddi_err(DER_VERB, rdip,  in create_xlate_arena()
1112 			ddi_err(DER_PANIC, rdip,  in create_xlate_arena()
1172 device_domain(dev_info_t *rdip, dev_info_t **ddipp, immu_flags_t immu_flags)  in device_domain()  argument
1187 	domain = immu_devi_domain(rdip, &ddip);  in device_domain()
1193 	immu = immu_dvma_get_immu(rdip, immu_flags);  in device_domain()
1199 		ddi_err(DER_WARN, rdip, "No iommu unit found for device");  in device_domain()
1203 	immu_flags |= immu_devi_get(rdip)->imd_dvma_flags;  in device_domain()
1205 	dvarg.dva_rdip = rdip;  in device_domain()
1210 	if (immu_walk_ancestor(rdip, NULL, get_branch_domain,  in device_domain()
1235 		ddi_err(DER_MODE, rdip, "Cannot find domain dip for device.");  in device_domain()
1247 	domain = domain_create(immu, ddip, rdip, immu_flags);  in device_domain()
1260 	set_domain(rdip, ddip, domain);  in device_domain()
1313 domain_create(immu_t *immu, dev_info_t *ddip, dev_info_t *rdip,  in domain_create()  argument
1324 	immu_devi = immu_devi_get(rdip);  in domain_create()
1331 	did = did_alloc(immu, rdip, ddip, immu_flags);  in domain_create()
1341 		ddi_err(DER_PANIC, rdip, "Failed to alloc DVMA domain "  in domain_create()
1359 	create_xlate_arena(immu, domain, rdip, immu_flags);  in domain_create()
1366 		ddi_err(DER_PANIC, rdip, "Failed to alloc root "  in domain_create()
1631     dev_info_t *rdip, immu_flags_t immu_flags)  in immu_context_update()  argument
1642 	if (ddip == NULL || rdip == NULL ||  in immu_context_update()
1643 	    ddip == root_devinfo || rdip == root_devinfo) {  in immu_context_update()
1644 		ddi_err(DER_MODE, rdip, "immu_contexts_update: domain-dip or "  in immu_context_update()
1656 		ddi_err(DER_MODE, rdip,  in immu_context_update()
1661 	if (immu_devi_set(rdip, immu_flags) == DDI_FAILURE) {  in immu_context_update()
1662 		ddi_err(DER_MODE, rdip,  in immu_context_update()
1668 	r_immu_devi = immu_devi_get(rdip);  in immu_context_update()
1676 	if (rdip == ddip) {  in immu_context_update()
1683 		ddi_err(DER_WARN, rdip, "Driver bug: Devices 0x%lx and "  in immu_context_update()
1684 		    "0x%lx are identical", rdip, ddip);  in immu_context_update()
1704 			if (immu_walk_ancestor(rdip, ddip, find_top_pcib,  in immu_context_update()
1713 				ddi_err(DER_PANIC, rdip, "Failed to find PCI "  in immu_context_update()
1729 		ddi_err(DER_PANIC, rdip, "unknown device type. Cannot "  in immu_context_update()
1745     dev_info_t *rdip, immu_flags_t immu_flags)  in PDTE_check()  argument
1749 		ddi_err(DER_MODE, rdip, "No present flag");  in PDTE_check()
1766 			ddi_err(DER_MODE, rdip, "TM flag set");  in PDTE_check()
1775 		ddi_err(DER_MODE, rdip, "SW3 set");  in PDTE_check()
1785 			ddi_err(DER_MODE, rdip,  in PDTE_check()
1792 			ddi_err(DER_MODE, rdip,  in PDTE_check()
1805 			ddi_err(DER_MODE, rdip, "SNP set");  in PDTE_check()
1812 		ddi_err(DER_MODE, rdip, "SW2 set");  in PDTE_check()
1818 		ddi_err(DER_MODE, rdip, "SP set");  in PDTE_check()
1827 		ddi_err(DER_MODE, rdip, "SW1 set");  in PDTE_check()
1832 		ddi_err(DER_MODE, rdip, "READ not set");  in PDTE_check()
1837 		ddi_err(DER_MODE, rdip, "WRITE not set");  in PDTE_check()
1848     uint64_t *dvma_ptr, uint64_t *npages_ptr, dev_info_t *rdip)  in PTE_clear_all()  argument
2013     dev_info_t *rdip, immu_flags_t immu_flags)  in PTE_set_one()  argument
2025 			ddi_err(DER_MODE, rdip, "PTE paddr %lx != paddr %lx",  in PTE_set_one()
2097     int dcount, dev_info_t *rdip, immu_flags_t immu_flags)  in PTE_set_all()  argument
2137 		PTE_set_one(immu, hwp, paddr, rdip, immu_flags);  in PTE_set_all()
2140 		ASSERT(PDTE_check(immu, *hwp, NULL, paddr, rdip, immu_flags)  in PTE_set_all()
2174     dev_info_t *rdip, immu_flags_t immu_flags)  in PDE_set_one()  argument
2244     dev_info_t *rdip, immu_flags_t immu_flags)  in PDE_set_all()  argument
2279 				    rdip, int, level);  in PDE_set_all()
2283 					ddi_err(DER_PANIC, rdip,  in PDE_set_all()
2301 			PDE_set_one(immu, hwp, next, rdip, immu_flags);  in PDE_set_all()
2325 		ASSERT(PDTE_check(immu, *hwp, next, 0, rdip, immu_flags)  in PDE_set_all()
2353     immu_dcookie_t *dcookies, int dcount, dev_info_t *rdip,  in dvma_map()  argument
2370 		if (PDE_set_all(immu, domain, xlate, nlevels, rdip, immu_flags)  in dvma_map()
2377 		    dcount, rdip, immu_flags);  in dvma_map()
2396     dev_info_t *rdip)  in dvma_unmap()  argument
2413 			ddi_err(DER_PANIC, rdip,  in dvma_unmap()
2418 		PTE_clear_all(immu, domain, &xlate[1], &dvma, &n, rdip);  in dvma_unmap()
2453 dvma_prealloc(dev_info_t *rdip, immu_hdl_priv_t *ihp, ddi_dma_attr_t *dma_attr)  in dvma_prealloc()  argument
2466 	domain = IMMU_DEVI(rdip)->imd_domain;  in dvma_prealloc()
2501 		(void) PDE_set_all(immu, domain, xlate, nlevels, rdip,  in dvma_prealloc()
2510 		    1, rdip, IMMU_FLAGS_READ);  in dvma_prealloc()
2526 dvma_prefree(dev_info_t *rdip, immu_hdl_priv_t *ihp)  in dvma_prefree()  argument
2530 	domain = IMMU_DEVI(rdip)->imd_domain;  in dvma_prefree()
2533 		dvma_unmap(domain, ihp->ihp_predvma, IMMU_NPREPTES, rdip);  in dvma_prefree()
2552 immu_map_dvmaseg(dev_info_t *rdip, ddi_dma_handle_t handle,  in immu_map_dvmaseg()  argument
2574 	domain = IMMU_DEVI(rdip)->imd_domain;  in immu_map_dvmaseg()
2586 	IMMU_DPROBE3(immu__map__dvma, dev_info_t *, rdip, ddi_dma_atyp_t,  in immu_map_dvmaseg()
2649 	IMMU_DPROBE3(immu__dvma__alloc, dev_info_t *, rdip, uint64_t, npgalloc,  in immu_map_dvmaseg()
2685 				    npages, dcookies, dmax + 1, rdip,  in immu_map_dvmaseg()
2690 				    dev_info_t *, rdip, uint64_t, dvma,  in immu_map_dvmaseg()
2712 		IMMU_DPROBE4(immu__dvmamap__late, dev_info_t *, rdip, \  in immu_map_dvmaseg()
2716 		    dmax + 1, rdip, immu_flags))  in immu_map_dvmaseg()
2739 immu_unmap_dvmaseg(dev_info_t *rdip, ddi_dma_obj_t *dmao)  in immu_unmap_dvmaseg()  argument
2745 	domain = IMMU_DEVI(rdip)->imd_domain;  in immu_unmap_dvmaseg()
2753 	dvma_unmap(domain, dvma, npages, rdip);  in immu_unmap_dvmaseg()
2757 	IMMU_DPROBE3(immu__dvma__free, dev_info_t *, rdip, uint_t, npages,  in immu_unmap_dvmaseg()
2878 immu_dvma_device_setup(dev_info_t *rdip, immu_flags_t immu_flags)  in immu_dvma_device_setup()  argument
2884 	odip = rdip;  in immu_dvma_device_setup()
2886 	immu = immu_dvma_get_immu(rdip, immu_flags);  in immu_dvma_device_setup()
2892 		ddi_err(DER_WARN, rdip, "No iommu unit found for device");  in immu_dvma_device_setup()
2899 	if (strcmp(ddi_node_name(ddi_get_parent(rdip)), "isa") == 0) {  in immu_dvma_device_setup()
2900 		rdip = get_lpc_devinfo(immu, rdip, immu_flags);  in immu_dvma_device_setup()
2901 		if (rdip == NULL) {  in immu_dvma_device_setup()
2902 			ddi_err(DER_PANIC, rdip, "iommu redirect failed");  in immu_dvma_device_setup()
2913 	if (strcmp(ddi_node_name(rdip), "agpgart") == 0) {  in immu_dvma_device_setup()
2914 		rdip = get_gfx_devinfo(rdip);  in immu_dvma_device_setup()
2915 		if (rdip == NULL) {  in immu_dvma_device_setup()
2916 			ddi_err(DER_PANIC, rdip, "iommu redirect failed");  in immu_dvma_device_setup()
2927 	domain = device_domain(rdip, &ddip, immu_flags);  in immu_dvma_device_setup()
2929 		ddi_err(DER_MODE, rdip, "Intel IOMMU setup failed for device");  in immu_dvma_device_setup()
2941 		ddi_err(DER_MODE, rdip, "domain did 0(%d) or ddip NULL(%p)",  in immu_dvma_device_setup()
2946 	if (odip != rdip)  in immu_dvma_device_setup()
2952 	if (immu_context_update(immu, domain, ddip, rdip, immu_flags)  in immu_dvma_device_setup()
2954 		ddi_err(DER_MODE, rdip, "DVMA map: context update failed");  in immu_dvma_device_setup()
2962 immu_map_memrange(dev_info_t *rdip, memrng_t *mrng)  in immu_map_memrange()  argument
2973 	domain = IMMU_DEVI(rdip)->imd_domain;  in immu_map_memrange()
2977 	    mrng->mrng_npages, dcookies, 1, rdip,  in immu_map_memrange()
2990 immu_devi_get(dev_info_t *rdip)  in immu_devi_get()  argument
2993 	volatile uintptr_t *vptr = (uintptr_t *)&(DEVI(rdip)->devi_iommu);  in immu_devi_get()
3060     dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,  in immu_allochdl()  argument
3067 	ret = iommulib_iommu_dma_allochdl(dip, rdip, attr, waitfp,  in immu_allochdl()
3070 		immu = IMMU_DEVI(rdip)->imd_immu;  in immu_allochdl()
3075 			(void) iommulib_iommu_dma_freehdl(dip, rdip,  in immu_allochdl()
3080 		if (IMMU_DEVI(rdip)->imd_use_premap)  in immu_allochdl()
3081 			dvma_prealloc(rdip, ihp, attr);  in immu_allochdl()
3086 		ret = iommulib_iommu_dmahdl_setprivate(dip, rdip, *dma_handlep,  in immu_allochdl()
3095     dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle)  in immu_freehdl()  argument
3099 	ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle);  in immu_freehdl()
3101 		if (IMMU_DEVI(rdip)->imd_use_premap)  in immu_freehdl()
3102 			dvma_prefree(rdip, ihp);  in immu_freehdl()
3103 		kmem_cache_free(IMMU_DEVI(rdip)->imd_immu->immu_hdl_cache, ihp);  in immu_freehdl()
3106 	return (iommulib_iommu_dma_freehdl(dip, rdip, dma_handle));  in immu_freehdl()
3113     dev_info_t *rdip, ddi_dma_handle_t dma_handle,  in immu_bindhdl()  argument
3120 	ret = iommulib_iommu_dma_bindhdl(dip, rdip, dma_handle,  in immu_bindhdl()
3124 		ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle);  in immu_bindhdl()
3125 		immu_flush_wait(IMMU_DEVI(rdip)->imd_immu, &ihp->ihp_inv_wait);  in immu_bindhdl()
3134     dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle)  in immu_unbindhdl()  argument
3136 	return (iommulib_iommu_dma_unbindhdl(dip, rdip, dma_handle));  in immu_unbindhdl()
3142     dev_info_t *rdip, ddi_dma_handle_t dma_handle, off_t off,  in immu_sync()  argument
3145 	return (iommulib_iommu_dma_sync(dip, rdip, dma_handle, off, len,  in immu_sync()
3152     dev_info_t *rdip, ddi_dma_handle_t dma_handle, uint_t win,  in immu_win()  argument
3156 	return (iommulib_iommu_dma_win(dip, rdip, dma_handle, win, offp,  in immu_win()
3163     dev_info_t *rdip, ddi_dma_handle_t dma_handle,  in immu_mapobject()  argument
3168 	ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle);  in immu_mapobject()
3170 	return (immu_map_dvmaseg(rdip, dma_handle, ihp, dmareq, dmao));  in immu_mapobject()
3176     dev_info_t *rdip, ddi_dma_handle_t dma_handle, ddi_dma_obj_t *dmao)  in immu_unmapobject()  argument
3180 	ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle);  in immu_unmapobject()
3183 	return (immu_unmap_dvmaseg(rdip, dmao));  in immu_unmapobject()