75     dev_info_t *rdip, immu_flags_t immu_flags);
76 static immu_devi_t *create_immu_devi(dev_info_t *rdip, int bus,
80     uint64_t nvpages, immu_dcookie_t *dcookies, int dcount, dev_info_t *rdip,
91     dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
94     dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle);
96     dev_info_t *rdip, ddi_dma_handle_t dma_handle, struct ddi_dma_req *dma_req,
99     dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle);
101     dev_info_t *rdip, ddi_dma_handle_t dma_handle, off_t off, size_t len,
104     dev_info_t *rdip, ddi_dma_handle_t dma_handle, uint_t win,
107     dev_info_t *rdip, ddi_dma_handle_t dma_handle,
110     dev_info_t *rdip, ddi_dma_handle_t dma_handle, ddi_dma_obj_t *dmao);
321 get_lpc_devinfo(immu_t *immu, dev_info_t *rdip, immu_flags_t immu_flags)  in get_lpc_devinfo()  argument
325 	dvarg.dva_rdip = rdip;  in get_lpc_devinfo()
328 	if (immu_walk_ancestor(rdip, NULL, match_lpc,  in get_lpc_devinfo()
330 		ddi_err(DER_MODE, rdip, "Could not walk ancestors to "  in get_lpc_devinfo()
336 		ddi_err(DER_MODE, rdip, "Could not find lpc_devinfo for "  in get_lpc_devinfo()
345 get_gfx_devinfo(dev_info_t *rdip)  in get_gfx_devinfo()  argument
366 		ddi_err(DER_WARN, rdip, "iommu: No GFX device. "  in get_gfx_devinfo()
371 	ddi_err(DER_LOG, rdip, "iommu: GFX redirect to %s",  in get_gfx_devinfo()
692 create_immu_devi(dev_info_t *rdip, int bus, int dev, int func,  in create_immu_devi()  argument
710 		ddi_err(DER_WARN, rdip, "Failed to allocate memory for "  in create_immu_devi()
714 	immu_devi->imd_dip = rdip;  in create_immu_devi()
773 immu_devi_domain(dev_info_t *rdip, dev_info_t **ddipp)  in immu_devi_domain()  argument
781 	immu_devi = immu_devi_get(rdip);  in immu_devi_domain()
786 	mutex_enter(&(DEVI(rdip)->devi_lock));  in immu_devi_domain()
789 	mutex_exit(&(DEVI(rdip)->devi_lock));  in immu_devi_domain()
805 did_alloc(immu_t *immu, dev_info_t *rdip,  in did_alloc()  argument
814 		ddi_err(DER_WARN, rdip, "device domain-id alloc error"  in did_alloc()
1020     dev_info_t *rdip, immu_flags_t immu_flags)  in create_xlate_arena()  argument
1065 	ddi_err(DER_VERB, rdip,  in create_xlate_arena()
1083 		ddi_err(DER_PANIC, rdip,  in create_xlate_arena()
1102 		ddi_err(DER_VERB, rdip,  in create_xlate_arena()
1111 			ddi_err(DER_PANIC, rdip,  in create_xlate_arena()
1171 device_domain(dev_info_t *rdip, dev_info_t **ddipp, immu_flags_t immu_flags)  in device_domain()  argument
1186 	domain = immu_devi_domain(rdip, &ddip);  in device_domain()
1192 	immu = immu_dvma_get_immu(rdip, immu_flags);  in device_domain()
1198 		ddi_err(DER_WARN, rdip, "No iommu unit found for device");  in device_domain()
1202 	immu_flags |= immu_devi_get(rdip)->imd_dvma_flags;  in device_domain()
1204 	dvarg.dva_rdip = rdip;  in device_domain()
1209 	if (immu_walk_ancestor(rdip, NULL, get_branch_domain,  in device_domain()
1234 		ddi_err(DER_MODE, rdip, "Cannot find domain dip for device.");  in device_domain()
1246 	domain = domain_create(immu, ddip, rdip, immu_flags);  in device_domain()
1259 	set_domain(rdip, ddip, domain);  in device_domain()
1312 domain_create(immu_t *immu, dev_info_t *ddip, dev_info_t *rdip,  in domain_create()  argument
1323 	immu_devi = immu_devi_get(rdip);  in domain_create()
1330 	did = did_alloc(immu, rdip, ddip, immu_flags);  in domain_create()
1340 		ddi_err(DER_PANIC, rdip, "Failed to alloc DVMA domain "  in domain_create()
1358 	create_xlate_arena(immu, domain, rdip, immu_flags);  in domain_create()
1365 		ddi_err(DER_PANIC, rdip, "Failed to alloc root "  in domain_create()
1630     dev_info_t *rdip, immu_flags_t immu_flags)  in immu_context_update()  argument
1641 	if (ddip == NULL || rdip == NULL ||  in immu_context_update()
1642 	    ddip == root_devinfo || rdip == root_devinfo) {  in immu_context_update()
1643 		ddi_err(DER_MODE, rdip, "immu_contexts_update: domain-dip or "  in immu_context_update()
1655 		ddi_err(DER_MODE, rdip,  in immu_context_update()
1660 	if (immu_devi_set(rdip, immu_flags) == DDI_FAILURE) {  in immu_context_update()
1661 		ddi_err(DER_MODE, rdip,  in immu_context_update()
1667 	r_immu_devi = immu_devi_get(rdip);  in immu_context_update()
1675 	if (rdip == ddip) {  in immu_context_update()
1682 		ddi_err(DER_WARN, rdip, "Driver bug: Devices 0x%lx and "  in immu_context_update()
1683 		    "0x%lx are identical", rdip, ddip);  in immu_context_update()
1703 			if (immu_walk_ancestor(rdip, ddip, find_top_pcib,  in immu_context_update()
1712 				ddi_err(DER_PANIC, rdip, "Failed to find PCI "  in immu_context_update()
1728 		ddi_err(DER_PANIC, rdip, "unknown device type. Cannot "  in immu_context_update()
1744     dev_info_t *rdip, immu_flags_t immu_flags)  in PDTE_check()  argument
1748 		ddi_err(DER_MODE, rdip, "No present flag");  in PDTE_check()
1765 			ddi_err(DER_MODE, rdip, "TM flag set");  in PDTE_check()
1774 		ddi_err(DER_MODE, rdip, "SW3 set");  in PDTE_check()
1784 			ddi_err(DER_MODE, rdip,  in PDTE_check()
1791 			ddi_err(DER_MODE, rdip,  in PDTE_check()
1804 			ddi_err(DER_MODE, rdip, "SNP set");  in PDTE_check()
1811 		ddi_err(DER_MODE, rdip, "SW2 set");  in PDTE_check()
1817 		ddi_err(DER_MODE, rdip, "SP set");  in PDTE_check()
1826 		ddi_err(DER_MODE, rdip, "SW1 set");  in PDTE_check()
1831 		ddi_err(DER_MODE, rdip, "READ not set");  in PDTE_check()
1836 		ddi_err(DER_MODE, rdip, "WRITE not set");  in PDTE_check()
1847     uint64_t *dvma_ptr, uint64_t *npages_ptr, dev_info_t *rdip)  in PTE_clear_all()  argument
2012     dev_info_t *rdip, immu_flags_t immu_flags)  in PTE_set_one()  argument
2024 			ddi_err(DER_MODE, rdip, "PTE paddr %lx != paddr %lx",  in PTE_set_one()
2096     int dcount, dev_info_t *rdip, immu_flags_t immu_flags)  in PTE_set_all()  argument
2135 		PTE_set_one(immu, hwp, paddr, rdip, immu_flags);  in PTE_set_all()
2138 		ASSERT(PDTE_check(immu, *hwp, NULL, paddr, rdip, immu_flags)  in PTE_set_all()
2172     dev_info_t *rdip, immu_flags_t immu_flags)  in PDE_set_one()  argument
2242     dev_info_t *rdip, immu_flags_t immu_flags)  in PDE_set_all()  argument
2277 				    rdip, int, level);  in PDE_set_all()
2281 					ddi_err(DER_PANIC, rdip,  in PDE_set_all()
2299 			PDE_set_one(immu, hwp, next, rdip, immu_flags);  in PDE_set_all()
2323 		ASSERT(PDTE_check(immu, *hwp, next, 0, rdip, immu_flags)  in PDE_set_all()
2351     immu_dcookie_t *dcookies, int dcount, dev_info_t *rdip,  in dvma_map()  argument
2368 		if (PDE_set_all(immu, domain, xlate, nlevels, rdip, immu_flags)  in dvma_map()
2375 		    dcount, rdip, immu_flags);  in dvma_map()
2394     dev_info_t *rdip)  in dvma_unmap()  argument
2411 			ddi_err(DER_PANIC, rdip,  in dvma_unmap()
2416 		PTE_clear_all(immu, domain, &xlate[1], &dvma, &n, rdip);  in dvma_unmap()
2451 dvma_prealloc(dev_info_t *rdip, immu_hdl_priv_t *ihp, ddi_dma_attr_t *dma_attr)  in dvma_prealloc()  argument
2464 	domain = IMMU_DEVI(rdip)->imd_domain;  in dvma_prealloc()
2499 		(void) PDE_set_all(immu, domain, xlate, nlevels, rdip,  in dvma_prealloc()
2508 		    1, rdip, IMMU_FLAGS_READ);  in dvma_prealloc()
2524 dvma_prefree(dev_info_t *rdip, immu_hdl_priv_t *ihp)  in dvma_prefree()  argument
2528 	domain = IMMU_DEVI(rdip)->imd_domain;  in dvma_prefree()
2531 		dvma_unmap(domain, ihp->ihp_predvma, IMMU_NPREPTES, rdip);  in dvma_prefree()
2550 immu_map_dvmaseg(dev_info_t *rdip, ddi_dma_handle_t handle,  in immu_map_dvmaseg()  argument
2570 	domain = IMMU_DEVI(rdip)->imd_domain;  in immu_map_dvmaseg()
2582 	IMMU_DPROBE3(immu__map__dvma, dev_info_t *, rdip, ddi_dma_atyp_t,  in immu_map_dvmaseg()
2645 	IMMU_DPROBE3(immu__dvma__alloc, dev_info_t *, rdip, uint64_t, npgalloc,  in immu_map_dvmaseg()
2683 				    npages, dcookies, dmax + 1, rdip,  in immu_map_dvmaseg()
2688 				    dev_info_t *, rdip, uint64_t, dvma,  in immu_map_dvmaseg()
2707 		IMMU_DPROBE4(immu__dvmamap__late, dev_info_t *, rdip, \  in immu_map_dvmaseg()
2711 		    dmax + 1, rdip, immu_flags))  in immu_map_dvmaseg()
2734 immu_unmap_dvmaseg(dev_info_t *rdip, ddi_dma_obj_t *dmao)  in immu_unmap_dvmaseg()  argument
2740 	domain = IMMU_DEVI(rdip)->imd_domain;  in immu_unmap_dvmaseg()
2748 	dvma_unmap(domain, dvma, npages, rdip);  in immu_unmap_dvmaseg()
2752 	IMMU_DPROBE3(immu__dvma__free, dev_info_t *, rdip, uint_t, npages,  in immu_unmap_dvmaseg()
2873 immu_dvma_device_setup(dev_info_t *rdip, immu_flags_t immu_flags)  in immu_dvma_device_setup()  argument
2879 	odip = rdip;  in immu_dvma_device_setup()
2881 	immu = immu_dvma_get_immu(rdip, immu_flags);  in immu_dvma_device_setup()
2887 		ddi_err(DER_WARN, rdip, "No iommu unit found for device");  in immu_dvma_device_setup()
2894 	if (strcmp(ddi_node_name(ddi_get_parent(rdip)), "isa") == 0) {  in immu_dvma_device_setup()
2895 		rdip = get_lpc_devinfo(immu, rdip, immu_flags);  in immu_dvma_device_setup()
2896 		if (rdip == NULL) {  in immu_dvma_device_setup()
2897 			ddi_err(DER_PANIC, rdip, "iommu redirect failed");  in immu_dvma_device_setup()
2908 	if (strcmp(ddi_node_name(rdip), "agpgart") == 0) {  in immu_dvma_device_setup()
2909 		rdip = get_gfx_devinfo(rdip);  in immu_dvma_device_setup()
2910 		if (rdip == NULL) {  in immu_dvma_device_setup()
2911 			ddi_err(DER_PANIC, rdip, "iommu redirect failed");  in immu_dvma_device_setup()
2922 	domain = device_domain(rdip, &ddip, immu_flags);  in immu_dvma_device_setup()
2924 		ddi_err(DER_MODE, rdip, "Intel IOMMU setup failed for device");  in immu_dvma_device_setup()
2936 		ddi_err(DER_MODE, rdip, "domain did 0(%d) or ddip NULL(%p)",  in immu_dvma_device_setup()
2941 	if (odip != rdip)  in immu_dvma_device_setup()
2947 	if (immu_context_update(immu, domain, ddip, rdip, immu_flags)  in immu_dvma_device_setup()
2949 		ddi_err(DER_MODE, rdip, "DVMA map: context update failed");  in immu_dvma_device_setup()
2957 immu_map_memrange(dev_info_t *rdip, memrng_t *mrng)  in immu_map_memrange()  argument
2968 	domain = IMMU_DEVI(rdip)->imd_domain;  in immu_map_memrange()
2972 	    mrng->mrng_npages, dcookies, 1, rdip,  in immu_map_memrange()
2985 immu_devi_get(dev_info_t *rdip)  in immu_devi_get()  argument
2988 	volatile uintptr_t *vptr = (uintptr_t *)&(DEVI(rdip)->devi_iommu);  in immu_devi_get()
3055     dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,  in immu_allochdl()  argument
3062 	ret = iommulib_iommu_dma_allochdl(dip, rdip, attr, waitfp,  in immu_allochdl()
3065 		immu = IMMU_DEVI(rdip)->imd_immu;  in immu_allochdl()
3070 			(void) iommulib_iommu_dma_freehdl(dip, rdip,  in immu_allochdl()
3075 		if (IMMU_DEVI(rdip)->imd_use_premap)  in immu_allochdl()
3076 			dvma_prealloc(rdip, ihp, attr);  in immu_allochdl()
3081 		ret = iommulib_iommu_dmahdl_setprivate(dip, rdip, *dma_handlep,  in immu_allochdl()
3090     dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle)  in immu_freehdl()  argument
3094 	ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle);  in immu_freehdl()
3096 		if (IMMU_DEVI(rdip)->imd_use_premap)  in immu_freehdl()
3097 			dvma_prefree(rdip, ihp);  in immu_freehdl()
3098 		kmem_cache_free(IMMU_DEVI(rdip)->imd_immu->immu_hdl_cache, ihp);  in immu_freehdl()
3101 	return (iommulib_iommu_dma_freehdl(dip, rdip, dma_handle));  in immu_freehdl()
3108     dev_info_t *rdip, ddi_dma_handle_t dma_handle,  in immu_bindhdl()  argument
3115 	ret = iommulib_iommu_dma_bindhdl(dip, rdip, dma_handle,  in immu_bindhdl()
3119 		ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle);  in immu_bindhdl()
3120 		immu_flush_wait(IMMU_DEVI(rdip)->imd_immu, &ihp->ihp_inv_wait);  in immu_bindhdl()
3129     dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle)  in immu_unbindhdl()  argument
3131 	return (iommulib_iommu_dma_unbindhdl(dip, rdip, dma_handle));  in immu_unbindhdl()
3137     dev_info_t *rdip, ddi_dma_handle_t dma_handle, off_t off,  in immu_sync()  argument
3140 	return (iommulib_iommu_dma_sync(dip, rdip, dma_handle, off, len,  in immu_sync()
3147     dev_info_t *rdip, ddi_dma_handle_t dma_handle, uint_t win,  in immu_win()  argument
3151 	return (iommulib_iommu_dma_win(dip, rdip, dma_handle, win, offp,  in immu_win()
3158     dev_info_t *rdip, ddi_dma_handle_t dma_handle,  in immu_mapobject()  argument
3163 	ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle);  in immu_mapobject()
3165 	return (immu_map_dvmaseg(rdip, dma_handle, ihp, dmareq, dmao));  in immu_mapobject()
3171     dev_info_t *rdip, ddi_dma_handle_t dma_handle, ddi_dma_obj_t *dmao)  in immu_unmapobject()  argument
3175 	ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle);  in immu_unmapobject()
3178 	return (immu_unmap_dvmaseg(rdip, dmao));  in immu_unmapobject()