74 static domain_t *domain_create(immu_t *immu, dev_info_t *ddip,
248 immu_devi_set_spclist(dev_info_t *dip, immu_t *immu)  in immu_devi_set_spclist()  argument
255 		spclist = &(immu->immu_dvma_gfx_list);  in immu_devi_set_spclist()
257 		spclist = &(immu->immu_dvma_lpc_list);  in immu_devi_set_spclist()
261 		mutex_enter(&(immu->immu_lock));  in immu_devi_set_spclist()
263 		mutex_exit(&(immu->immu_lock));  in immu_devi_set_spclist()
321 get_lpc_devinfo(immu_t *immu, dev_info_t *rdip, immu_flags_t immu_flags)  in get_lpc_devinfo()  argument
324 	dvarg.dva_list = &(immu->immu_dvma_lpc_list);  in get_lpc_devinfo()
347 	immu_t *immu;  in get_gfx_devinfo()  local
356 	immu = list_head(&immu_list);  in get_gfx_devinfo()
357 	for (; immu; immu = list_next(&immu_list, immu)) {  in get_gfx_devinfo()
358 		list_gfx = &(immu->immu_dvma_gfx_list);  in get_gfx_devinfo()
426 	immu_t *immu = arg;  in pgtable_ctor()  local
444 	if (!immu->immu_dvma_coherent)  in pgtable_ctor()
505 pgtable_alloc(immu_t *immu, immu_flags_t immu_flags)  in pgtable_alloc()  argument
512 	pgtable = kmem_cache_alloc(immu->immu_pgtable_cache, kmflags);  in pgtable_alloc()
527 pgtable_free(immu_t *immu, pgtable_t *pgtable)  in pgtable_free()  argument
529 	kmem_cache_free(immu->immu_pgtable_cache, pgtable);  in pgtable_free()
614 	immu_t *immu;  in immu_dvma_get_immu()  local
636 		immu = immu_devi->imd_immu;  in immu_dvma_get_immu()
638 		return (immu);  in immu_dvma_get_immu()
642 	immu = immu_dmar_get_immu(dip);  in immu_dvma_get_immu()
643 	if (immu == NULL) {  in immu_dvma_get_immu()
664 		immu_devi->imd_immu = immu;  in immu_dvma_get_immu()
665 		immu_devi_set_spclist(dip, immu);  in immu_dvma_get_immu()
671 		if (immu_devi->imd_immu != immu) {  in immu_dvma_get_immu()
674 			    "actual (%p)", (void *)immu,  in immu_dvma_get_immu()
682 	return (immu);  in immu_dvma_get_immu()
805 did_alloc(immu_t *immu, dev_info_t *rdip,  in did_alloc()  argument
810 	did = (uintptr_t)vmem_alloc(immu->immu_did_arena, 1,  in did_alloc()
818 		    immu->immu_name, immu->immu_unity_domain->dom_did);  in did_alloc()
819 		did = immu->immu_unity_domain->dom_did;  in did_alloc()
831 	immu_t *immu;  in get_branch_domain()  local
852 	immu = immu_devi->imd_immu;  in get_branch_domain()
853 	if (immu == NULL)  in get_branch_domain()
854 		immu = immu_dvma_get_immu(pdip, dvp->dva_flags);  in get_branch_domain()
880 		dvp->dva_domain = immu->immu_unity_domain;  in get_branch_domain()
1019 create_xlate_arena(immu_t *immu, domain_t *domain,  in create_xlate_arena()  argument
1036 	    "%s-domain-%d-xlate-DVMA-arena", immu->immu_name,  in create_xlate_arena()
1042 	mgaw = IMMU_CAP_MGAW(immu->immu_regs_cap);  in create_xlate_arena()
1174 	immu_t *immu;  in device_domain()  local
1192 	immu = immu_dvma_get_immu(rdip, immu_flags);  in device_domain()
1193 	if (immu == NULL) {  in device_domain()
1246 	domain = domain_create(immu, ddip, rdip, immu_flags);  in device_domain()
1266 create_unity_domain(immu_t *immu)  in create_unity_domain()  argument
1278 	domain->dom_immu = immu;  in create_unity_domain()
1279 	immu->immu_unity_domain = domain;  in create_unity_domain()
1285 	domain->dom_pgtable_root = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);  in create_unity_domain()
1293 	if (!IMMU_ECAP_GET_PT(immu->immu_regs_excap))  in create_unity_domain()
1312 domain_create(immu_t *immu, dev_info_t *ddip, dev_info_t *rdip,  in domain_create()  argument
1330 	did = did_alloc(immu, rdip, ddip, immu_flags);  in domain_create()
1333 		ASSERT(immu->immu_unity_domain);  in domain_create()
1334 		return (immu->immu_unity_domain);  in domain_create()
1341 		    "structure for device. IOMMU unit: %s", immu->immu_name);  in domain_create()
1348 	    "immu%s-domain%d-pava-hash", immu->immu_name, did);  in domain_create()
1351 	domain->dom_immu = immu;  in domain_create()
1358 	create_xlate_arena(immu, domain, rdip, immu_flags);  in domain_create()
1363 	domain->dom_pgtable_root = pgtable_alloc(immu, immu_flags);  in domain_create()
1367 		    domain->dom_did, immu->immu_name);  in domain_create()
1376 	mutex_enter(&(immu->immu_lock));  in domain_create()
1377 	list_insert_head(&immu->immu_domain_list, domain);  in domain_create()
1378 	mutex_exit(&(immu->immu_lock));  in domain_create()
1418 did_init(immu_t *immu)  in did_init()  argument
1420 	(void) snprintf(immu->immu_did_arena_name,  in did_init()
1421 	    sizeof (immu->immu_did_arena_name),  in did_init()
1422 	    "%s_domainid_arena", immu->immu_name);  in did_init()
1424 	ddi_err(DER_VERB, immu->immu_dip, "creating domainid arena %s",  in did_init()
1425 	    immu->immu_did_arena_name);  in did_init()
1427 	immu->immu_did_arena = vmem_create(  in did_init()
1428 	    immu->immu_did_arena_name,  in did_init()
1430 	    immu->immu_max_domains - IMMU_UNITY_DID,  in did_init()
1439 	if (immu->immu_did_arena == NULL) {  in did_init()
1441 		    "IOMMU domainid allocator: %s", immu->immu_name,  in did_init()
1442 		    immu->immu_did_arena_name);  in did_init()
1449 context_set(immu_t *immu, domain_t *domain, pgtable_t *root_table,  in context_set()  argument
1472 	rw_enter(&(immu->immu_ctx_rwlock), RW_READER);  in context_set()
1478 			rw_exit(&(immu->immu_ctx_rwlock));  in context_set()
1489 	    rw_tryupgrade(&(immu->immu_ctx_rwlock)) == 0) {  in context_set()
1490 		rw_exit(&(immu->immu_ctx_rwlock));  in context_set()
1491 		rw_enter(&(immu->immu_ctx_rwlock), RW_WRITER);  in context_set()
1500 		immu_regs_cpu_flush(immu, (caddr_t)hw_rent, sizeof (hw_rce_t));  in context_set()
1509 		immu_regs_cpu_flush(immu, (caddr_t)hw_cent, sizeof (hw_rce_t));  in context_set()
1512 		immu_flush_context_fsi(immu, 0, sid, domain->dom_did,  in context_set()
1513 		    &immu->immu_ctx_inv_wait);  in context_set()
1517 		CONT_SET_AW(hw_cent, immu->immu_dvma_agaw);  in context_set()
1520 		    IMMU_ECAP_GET_PT(immu->immu_regs_excap))  in context_set()
1526 		if (IMMU_ECAP_GET_CH(immu->immu_regs_excap)) {  in context_set()
1531 		immu_regs_cpu_flush(immu, (caddr_t)hw_cent, sizeof (hw_rce_t));  in context_set()
1533 	rw_exit(&(immu->immu_ctx_rwlock));  in context_set()
1537 context_create(immu_t *immu)  in context_create()  argument
1549 	root_table = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);  in context_create()
1559 		context = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);  in context_create()
1567 			    immu->immu_unity_domain->dom_pgtable_root;  in context_create()
1569 			    immu->immu_unity_domain->dom_did);  in context_create()
1570 			CONT_SET_AW(hw_cent, immu->immu_dvma_agaw);  in context_create()
1572 			if (IMMU_ECAP_GET_PT(immu->immu_regs_excap))  in context_create()
1580 		immu_regs_cpu_flush(immu, context->hwpg_vaddr, IMMU_PAGESIZE);  in context_create()
1591 context_init(immu_t *immu)  in context_init()  argument
1593 	rw_init(&(immu->immu_ctx_rwlock), NULL, RW_DEFAULT, NULL);  in context_init()
1595 	immu_init_inv_wait(&immu->immu_ctx_inv_wait, "ctxglobal", B_TRUE);  in context_init()
1597 	immu_regs_wbf_flush(immu);  in context_init()
1599 	immu->immu_ctx_root = context_create(immu);  in context_init()
1601 	immu_regs_set_root_table(immu);  in context_init()
1603 	rw_enter(&(immu->immu_ctx_rwlock), RW_WRITER);  in context_init()
1604 	immu_flush_context_gbl(immu, &immu->immu_ctx_inv_wait);  in context_init()
1605 	immu_flush_iotlb_gbl(immu, &immu->immu_ctx_inv_wait);  in context_init()
1606 	rw_exit(&(immu->immu_ctx_rwlock));  in context_init()
1629 immu_context_update(immu_t *immu, domain_t *domain, dev_info_t *ddip,  in immu_context_update()  argument
1677 		context_set(immu, domain, immu->immu_ctx_root, r_bus,  in immu_context_update()
1686 		context_set(immu, domain, immu->immu_ctx_root, r_bus,  in immu_context_update()
1696 		context_set(immu, domain, immu->immu_ctx_root,  in immu_context_update()
1699 			context_set(immu, domain, immu->immu_ctx_root,  in immu_context_update()
1709 				context_set(immu, domain, immu->immu_ctx_root,  in immu_context_update()
1718 		context_set(immu, domain, immu->immu_ctx_root, d_bus,  in immu_context_update()
1725 		context_set(immu, domain, immu->immu_ctx_root, d_bus,  in immu_context_update()
1743 PDTE_check(immu_t *immu, hw_pdte_t pdte, pgtable_t *next, paddr_t paddr,  in PDTE_check()  argument
1763 	if (next == NULL && immu->immu_TM_reserved == B_FALSE) {  in PDTE_check()
1802 	if (next == NULL && immu->immu_SNP_reserved == B_FALSE) {  in PDTE_check()
1846 PTE_clear_all(immu_t *immu, domain_t *domain, xlate_t *xlate,  in PTE_clear_all()  argument
1971 	immu_t *immu;  in immu_print_fault_info()  local
1981 	immu = domain->dom_immu;  in immu_print_fault_info()
1990 	nlevels = immu->immu_dvma_nlevels;  in immu_print_fault_info()
2011 PTE_set_one(immu_t *immu, hw_pdte_t *hwp, paddr_t paddr,  in PTE_set_one()  argument
2017 	pte = immu->immu_ptemask;  in PTE_set_one()
2035 	if (immu->immu_TM_reserved == B_FALSE) {  in PTE_set_one()
2048 	if (immu->immu_SNP_reserved == B_FALSE) {  in PTE_set_one()
2075 	pte |= immu->immu_ptemask;  in PTE_set_one()
2094 PTE_set_all(immu_t *immu, domain_t *domain, xlate_t *xlate,  in PTE_set_all()  argument
2135 		PTE_set_one(immu, hwp, paddr, rdip, immu_flags);  in PTE_set_all()
2138 		ASSERT(PDTE_check(immu, *hwp, NULL, paddr, rdip, immu_flags)  in PTE_set_all()
2171 PDE_set_one(immu_t *immu, hw_pdte_t *hwp, pgtable_t *next,  in PDE_set_one()  argument
2241 PDE_set_all(immu_t *immu, domain_t *domain, xlate_t *xlate, int nlevels,  in PDE_set_all()  argument
2279 				new = pgtable_alloc(immu, immu_flags);  in PDE_set_all()
2299 			PDE_set_one(immu, hwp, next, rdip, immu_flags);  in PDE_set_all()
2323 		ASSERT(PDTE_check(immu, *hwp, next, 0, rdip, immu_flags)  in PDE_set_all()
2331 		pgtable_free(immu, new);  in PDE_set_all()
2356 	immu_t *immu = domain->dom_immu;  in dvma_map()  local
2357 	int nlevels = immu->immu_dvma_nlevels;  in dvma_map()
2368 		if (PDE_set_all(immu, domain, xlate, nlevels, rdip, immu_flags)  in dvma_map()
2374 		PTE_set_all(immu, domain, &xlate[1], &dvma, &n, dcookies,  in dvma_map()
2396 	immu_t *immu = domain->dom_immu;  in dvma_unmap()  local
2397 	int nlevels = immu->immu_dvma_nlevels;  in dvma_unmap()
2416 		PTE_clear_all(immu, domain, &xlate[1], &dvma, &n, rdip);  in dvma_unmap()
2460 	immu_t *immu;  in dvma_prealloc()  local
2465 	immu = domain->dom_immu;  in dvma_prealloc()
2466 	nlevels = immu->immu_dvma_nlevels;  in dvma_prealloc()
2499 		(void) PDE_set_all(immu, domain, xlate, nlevels, rdip,  in dvma_prealloc()
2507 		PTE_set_all(immu, domain, xlp, &dvma, &n, &immu_precookie,  in dvma_prealloc()
2555 	immu_t *immu;  in immu_map_dvmaseg()  local
2571 	immu = domain->dom_immu;  in immu_map_dvmaseg()
2619 		rwmask = PDTE_MASK_R | PDTE_MASK_W | immu->immu_ptemask;  in immu_map_dvmaseg()
2621 		rwmask = immu->immu_ptemask;  in immu_map_dvmaseg()
2716 	immu_flush_iotlb_psi(immu, domain->dom_did, sdvma, npgalloc,  in immu_map_dvmaseg()
2776 	immu_t *immu;  in immu_dvma_setup()  local
2798 	immu = list_head(listp);  in immu_dvma_setup()
2799 	for (; immu; immu = list_next(listp, immu)) {  in immu_dvma_setup()
2800 		create_unity_domain(immu);  in immu_dvma_setup()
2801 		did_init(immu);  in immu_dvma_setup()
2802 		context_init(immu);  in immu_dvma_setup()
2803 		immu->immu_dvma_setup = B_TRUE;  in immu_dvma_setup()
2811 immu_dvma_startup(immu_t *immu)  in immu_dvma_startup()  argument
2814 	    immu->immu_dvma_gfx_only == B_TRUE) {  in immu_dvma_startup()
2821 	immu->immu_dvma_running = B_TRUE;  in immu_dvma_startup()
2876 	immu_t *immu;  in immu_dvma_device_setup()  local
2881 	immu = immu_dvma_get_immu(rdip, immu_flags);  in immu_dvma_device_setup()
2882 	if (immu == NULL) {  in immu_dvma_device_setup()
2895 		rdip = get_lpc_devinfo(immu, rdip, immu_flags);  in immu_dvma_device_setup()
2903 	immu = NULL;  in immu_dvma_device_setup()
2928 	immu = domain->dom_immu;  in immu_dvma_device_setup()
2947 	if (immu_context_update(immu, domain, ddip, rdip, immu_flags)  in immu_dvma_device_setup()
2961 	immu_t *immu;  in immu_map_memrange()  local
2969 	immu = domain->dom_immu;  in immu_map_memrange()
2977 	immu_flush_iotlb_psi(immu, domain->dom_did, mrng->mrng_start,  in immu_map_memrange()
3060 	immu_t *immu;  in immu_allochdl()  local
3065 		immu = IMMU_DEVI(rdip)->imd_immu;  in immu_allochdl()
3067 		ihp = kmem_cache_alloc(immu->immu_hdl_cache,  in immu_allochdl()