81 static boolean_t dvma_map(domain_t *domain, uint64_t sdvma,
190 	domain_t *domain;  in bdf_domain_lookup()  local
200 	domain = NULL;  in bdf_domain_lookup()
202 	    (void *)bdf, (void *)&domain) == 0) {  in bdf_domain_lookup()
203 		ASSERT(domain);  in bdf_domain_lookup()
204 		ASSERT(domain->dom_did > 0);  in bdf_domain_lookup()
205 		return (domain);  in bdf_domain_lookup()
212 bdf_domain_insert(immu_devi_t *immu_devi, domain_t *domain)  in bdf_domain_insert()  argument
223 	(void) mod_hash_insert(bdf_domain_hash, (void *)bdf, (void *)domain);  in bdf_domain_insert()
777 	domain_t *domain;  in immu_devi_domain()  local
788 	domain = immu_devi->imd_domain;  in immu_devi_domain()
792 	if (domain)  in immu_devi_domain()
795 	return (domain);  in immu_devi_domain()
830 	domain_t *domain;  in get_branch_domain()  local
887 	domain = immu_devi->imd_domain;  in get_branch_domain()
891 	if (domain && ddip) {  in get_branch_domain()
894 			ASSERT(domain == dvp->dva_domain);  in get_branch_domain()
896 		dvp->dva_domain = domain;  in get_branch_domain()
907 	domain = bdf_domain_lookup(immu_devi);  in get_branch_domain()
908 	if (domain == NULL) {  in get_branch_domain()
917 		dvp->dva_domain = domain;  in get_branch_domain()
919 		dvp->dva_domain = domain;  in get_branch_domain()
931 map_unity_domain(domain_t *domain)  in map_unity_domain()  argument
951 	(void) dvma_map(domain, 0, 1, dcookies, dcount, NULL,  in map_unity_domain()
970 	(void) dvma_map(domain, start, npages, dcookies,  in map_unity_domain()
973 	ddi_err(DER_LOG, domain->dom_dip, "iommu: mapping PHYS span [0x%" PRIx64  in map_unity_domain()
978 		ddi_err(DER_LOG, domain->dom_dip,  in map_unity_domain()
988 		(void) dvma_map(domain, start, npages,  in map_unity_domain()
995 		ddi_err(DER_LOG, domain->dom_dip,  in map_unity_domain()
1005 		(void) dvma_map(domain, start, npages,  in map_unity_domain()
1020 create_xlate_arena(immu_t *immu, domain_t *domain,  in create_xlate_arena()  argument
1032 	arena_name = domain->dom_dvma_arena_name;  in create_xlate_arena()
1036 	    sizeof (domain->dom_dvma_arena_name),  in create_xlate_arena()
1038 	    domain->dom_did);  in create_xlate_arena()
1073 	domain->dom_dvma_arena = vmem_create(arena_name,  in create_xlate_arena()
1083 	if (domain->dom_dvma_arena == NULL) {  in create_xlate_arena()
1086 		    "for domain ID (%d)", arena_name, domain->dom_did);  in create_xlate_arena()
1108 		vmem_ret = vmem_add(domain->dom_dvma_arena,  in create_xlate_arena()
1115 			    arena_name, domain->dom_did);  in create_xlate_arena()
1132 	domain_t *domain)  in set_domain()  argument
1151 		ASSERT(fdomain == domain);  in set_domain()
1153 		immu_devi->imd_domain = domain;  in set_domain()
1176 	domain_t *domain;  in device_domain()  local
1187 	domain = immu_devi_domain(rdip, &ddip);  in device_domain()
1188 	if (domain) {  in device_domain()
1190 		return (domain);  in device_domain()
1223 	domain = dvarg.dva_domain;	/* may be NULL */  in device_domain()
1242 	if (domain) {  in device_domain()
1247 	domain = domain_create(immu, ddip, rdip, immu_flags);  in device_domain()
1248 	if (domain == NULL) {  in device_domain()
1259 	set_domain(ddip, ddip, domain);  in device_domain()
1260 	set_domain(rdip, ddip, domain);  in device_domain()
1263 	return (domain);  in device_domain()
1269 	domain_t *domain;  in create_unity_domain()  local
1272 	domain = kmem_zalloc(sizeof (domain_t), KM_SLEEP);  in create_unity_domain()
1274 	rw_init(&(domain->dom_pgtable_rwlock), NULL, RW_DEFAULT, NULL);  in create_unity_domain()
1276 	domain->dom_did = IMMU_UNITY_DID;  in create_unity_domain()
1277 	domain->dom_maptype = IMMU_MAPTYPE_UNITY;  in create_unity_domain()
1279 	domain->dom_immu = immu;  in create_unity_domain()
1280 	immu->immu_unity_domain = domain;  in create_unity_domain()
1286 	domain->dom_pgtable_root = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);  in create_unity_domain()
1287 	pgtable_zero(domain->dom_pgtable_root);  in create_unity_domain()
1295 		map_unity_domain(domain);  in create_unity_domain()
1302 	list_insert_tail(&immu_unity_domain_list, domain);  in create_unity_domain()
1317 	domain_t *domain;  in domain_create()  local
1339 	domain = kmem_zalloc(sizeof (domain_t), kmflags);  in domain_create()
1340 	if (domain == NULL) {  in domain_create()
1346 	rw_init(&(domain->dom_pgtable_rwlock), NULL, RW_DEFAULT, NULL);  in domain_create()
1351 	domain->dom_did = did;  in domain_create()
1352 	domain->dom_immu = immu;  in domain_create()
1353 	domain->dom_maptype = IMMU_MAPTYPE_XLATE;  in domain_create()
1354 	domain->dom_dip = ddip;  in domain_create()
1359 	create_xlate_arena(immu, domain, rdip, immu_flags);  in domain_create()
1364 	domain->dom_pgtable_root = pgtable_alloc(immu, immu_flags);  in domain_create()
1365 	if (domain->dom_pgtable_root == NULL) {  in domain_create()
1368 		    domain->dom_did, immu->immu_name);  in domain_create()
1371 	pgtable_zero(domain->dom_pgtable_root);  in domain_create()
1378 	list_insert_head(&immu->immu_domain_list, domain);  in domain_create()
1385 	list_insert_head(&immu_xlate_domain_list, domain);  in domain_create()
1388 	bdf_domain_insert(immu_devi, domain);  in domain_create()
1397 	(void) dvma_map(domain, 0, 1, dcookies, dcount, NULL,  in domain_create()
1400 	return (domain);  in domain_create()
1450 context_set(immu_t *immu, domain_t *domain, pgtable_t *root_table,  in context_set()  argument
1463 	pgtable_root = domain->dom_pgtable_root;  in context_set()
1513 		immu_flush_context_fsi(immu, 0, sid, domain->dom_did,  in context_set()
1517 		CONT_SET_DID(hw_cent, domain->dom_did);  in context_set()
1520 		if (domain->dom_did == IMMU_UNITY_DID &&  in context_set()
1630 immu_context_update(immu_t *immu, domain_t *domain, dev_info_t *ddip,  in immu_context_update()  argument
1678 		context_set(immu, domain, immu->immu_ctx_root, r_bus,  in immu_context_update()
1687 		context_set(immu, domain, immu->immu_ctx_root, r_bus,  in immu_context_update()
1697 		context_set(immu, domain, immu->immu_ctx_root,  in immu_context_update()
1700 			context_set(immu, domain, immu->immu_ctx_root,  in immu_context_update()
1710 				context_set(immu, domain, immu->immu_ctx_root,  in immu_context_update()
1719 		context_set(immu, domain, immu->immu_ctx_root, d_bus,  in immu_context_update()
1726 		context_set(immu, domain, immu->immu_ctx_root, d_bus,  in immu_context_update()
1847 PTE_clear_all(immu_t *immu, domain_t *domain, xlate_t *xlate,  in PTE_clear_all()  argument
1909 PDE_lookup(domain_t *domain, xlate_t *xlate, int nlevels)  in PDE_lookup()  argument
1919 		xlate->xlt_pgtable = domain->dom_pgtable_root;  in PDE_lookup()
1971 	domain_t *domain;  in immu_print_fault_info()  local
1976 	    (void *)(uintptr_t)sid, (void *)&domain) != 0) {  in immu_print_fault_info()
1982 	immu = domain->dom_immu;  in immu_print_fault_info()
1985 	vmem_walk(domain->dom_dvma_arena, VMEM_ALLOC, immu_fault_walk,  in immu_print_fault_info()
1988 		ddi_err(DER_WARN, domain->dom_dip,  in immu_print_fault_info()
1994 	if (!PDE_lookup(domain, xlate, nlevels)) {  in immu_print_fault_info()
1995 		ddi_err(DER_WARN, domain->dom_dip,  in immu_print_fault_info()
1997 		    domain->dom_did, dvma);  in immu_print_fault_info()
2005 	ddi_err(DER_WARN, domain->dom_dip,  in immu_print_fault_info()
2006 	    "domid %d pte: %" PRIx64 "(paddr %" PRIx64 ")", domain->dom_did,  in immu_print_fault_info()
2095 PTE_set_all(immu_t *immu, domain_t *domain, xlate_t *xlate,  in PTE_set_all()  argument
2243 PDE_set_all(immu_t *immu, domain_t *domain, xlate_t *xlate, int nlevels,  in PDE_set_all()  argument
2259 	xlate->xlt_pgtable = domain->dom_pgtable_root;  in PDE_set_all()
2352 dvma_map(domain_t *domain, uint64_t sdvma, uint64_t snvpages,  in dvma_map()  argument
2358 	immu_t *immu = domain->dom_immu;  in dvma_map()
2370 		if (PDE_set_all(immu, domain, xlate, nlevels, rdip, immu_flags)  in dvma_map()
2376 		PTE_set_all(immu, domain, &xlate[1], &dvma, &n, dcookies,  in dvma_map()
2395 dvma_unmap(domain_t *domain, uint64_t sdvma, uint64_t snpages,  in dvma_unmap()  argument
2398 	immu_t *immu = domain->dom_immu;  in dvma_unmap()
2412 		if (!PDE_lookup(domain, xlate, nlevels))  in dvma_unmap()
2418 		PTE_clear_all(immu, domain, &xlate[1], &dvma, &n, rdip);  in dvma_unmap()
2425 dvma_alloc(domain_t *domain, ddi_dma_attr_t *dma_attr, uint_t npages, int kmf)  in dvma_alloc()  argument
2445 	dvma = (uint64_t)(uintptr_t)vmem_xalloc(domain->dom_dvma_arena,  in dvma_alloc()
2463 	domain_t *domain;  in dvma_prealloc()  local
2466 	domain = IMMU_DEVI(rdip)->imd_domain;  in dvma_prealloc()
2467 	immu = domain->dom_immu;  in dvma_prealloc()
2481 	dvma = (uint64_t)(uintptr_t)vmem_xalloc(domain->dom_dvma_arena,  in dvma_prealloc()
2501 		(void) PDE_set_all(immu, domain, xlate, nlevels, rdip,  in dvma_prealloc()
2509 		PTE_set_all(immu, domain, xlp, &dvma, &n, &immu_precookie,  in dvma_prealloc()
2528 	domain_t *domain;  in dvma_prefree()  local
2530 	domain = IMMU_DEVI(rdip)->imd_domain;  in dvma_prefree()
2533 		dvma_unmap(domain, ihp->ihp_predvma, IMMU_NPREPTES, rdip);  in dvma_prefree()
2534 		vmem_free(domain->dom_dvma_arena,  in dvma_prefree()
2541 dvma_free(domain_t *domain, uint64_t dvma, uint64_t npages)  in dvma_free()  argument
2545 	if (domain->dom_maptype != IMMU_MAPTYPE_XLATE)  in dvma_free()
2548 	vmem_free(domain->dom_dvma_arena, (void *)(uintptr_t)dvma, size);  in dvma_free()
2556 	domain_t *domain;  in immu_map_dvmaseg()  local
2574 	domain = IMMU_DEVI(rdip)->imd_domain;  in immu_map_dvmaseg()
2575 	immu = domain->dom_immu;  in immu_map_dvmaseg()
2640 		sdvma = dvma_alloc(domain, attrp, npgalloc,  in immu_map_dvmaseg()
2684 				if (dvma_map(domain, dvma,  in immu_map_dvmaseg()
2715 		if (dvma_map(domain, dvma, npages, dcookies,  in immu_map_dvmaseg()
2721 	immu_flush_iotlb_psi(immu, domain->dom_did, sdvma, npgalloc,  in immu_map_dvmaseg()
2742 	domain_t *domain;  in immu_unmap_dvmaseg()  local
2745 	domain = IMMU_DEVI(rdip)->imd_domain;  in immu_unmap_dvmaseg()
2753 	dvma_unmap(domain, dvma, npages, rdip);  in immu_unmap_dvmaseg()
2755 	dvma_free(domain, dvma, npages);  in immu_unmap_dvmaseg()
2766 	immu_regs_wbf_flush(domain->dom_immu);  in immu_unmap_dvmaseg()
2842 	domain_t *domain;  in immu_dvma_physmem_update()  local
2851 	domain = list_head(&immu_unity_domain_list);  in immu_dvma_physmem_update()
2852 	for (; domain; domain = list_next(&immu_unity_domain_list, domain)) {  in immu_dvma_physmem_update()
2856 		if (IMMU_ECAP_GET_PT(domain->dom_immu->immu_regs_excap))  in immu_dvma_physmem_update()
2860 		ddi_err(DER_LOG, domain->dom_dip,  in immu_dvma_physmem_update()
2870 		(void) dvma_map(domain, start, npages,  in immu_dvma_physmem_update()
2882 	domain_t *domain;  in immu_dvma_device_setup()  local
2927 	domain = device_domain(rdip, &ddip, immu_flags);  in immu_dvma_device_setup()
2928 	if (domain == NULL) {  in immu_dvma_device_setup()
2933 	immu = domain->dom_immu;  in immu_dvma_device_setup()
2940 	if (domain->dom_did == 0 || ddip == NULL) {  in immu_dvma_device_setup()
2942 		    domain->dom_did, ddip);  in immu_dvma_device_setup()
2947 		set_domain(odip, ddip, domain);  in immu_dvma_device_setup()
2952 	if (immu_context_update(immu, domain, ddip, rdip, immu_flags)  in immu_dvma_device_setup()
2967 	domain_t *domain;  in immu_map_memrange()  local
2973 	domain = IMMU_DEVI(rdip)->imd_domain;  in immu_map_memrange()
2974 	immu = domain->dom_immu;  in immu_map_memrange()
2976 	pde_set = dvma_map(domain, mrng->mrng_start,  in immu_map_memrange()
2982 	immu_flush_iotlb_psi(immu, domain->dom_did, mrng->mrng_start,  in immu_map_memrange()