76 static domain_t *domain_create(immu_t *immu, dev_info_t *ddip,
250 immu_devi_set_spclist(dev_info_t *dip, immu_t *immu)  in immu_devi_set_spclist()  argument
257 		spclist = &(immu->immu_dvma_gfx_list);  in immu_devi_set_spclist()
259 		spclist = &(immu->immu_dvma_lpc_list);  in immu_devi_set_spclist()
263 		mutex_enter(&(immu->immu_lock));  in immu_devi_set_spclist()
265 		mutex_exit(&(immu->immu_lock));  in immu_devi_set_spclist()
323 get_lpc_devinfo(immu_t *immu, dev_info_t *rdip, immu_flags_t immu_flags)  in get_lpc_devinfo()  argument
326 	dvarg.dva_list = &(immu->immu_dvma_lpc_list);  in get_lpc_devinfo()
349 	immu_t *immu;  in get_gfx_devinfo()  local
358 	immu = list_head(&immu_list);  in get_gfx_devinfo()
359 	for (; immu; immu = list_next(&immu_list, immu)) {  in get_gfx_devinfo()
360 		list_gfx = &(immu->immu_dvma_gfx_list);  in get_gfx_devinfo()
428 	immu_t *immu = arg;  in pgtable_ctor()  local
446 	if (!immu->immu_dvma_coherent)  in pgtable_ctor()
507 pgtable_alloc(immu_t *immu, immu_flags_t immu_flags)  in pgtable_alloc()  argument
514 	pgtable = kmem_cache_alloc(immu->immu_pgtable_cache, kmflags);  in pgtable_alloc()
529 pgtable_free(immu_t *immu, pgtable_t *pgtable)  in pgtable_free()  argument
531 	kmem_cache_free(immu->immu_pgtable_cache, pgtable);  in pgtable_free()
616 	immu_t *immu;  in immu_dvma_get_immu()  local
638 		immu = immu_devi->imd_immu;  in immu_dvma_get_immu()
640 		return (immu);  in immu_dvma_get_immu()
644 	immu = immu_dmar_get_immu(dip);  in immu_dvma_get_immu()
645 	if (immu == NULL) {  in immu_dvma_get_immu()
666 		immu_devi->imd_immu = immu;  in immu_dvma_get_immu()
667 		immu_devi_set_spclist(dip, immu);  in immu_dvma_get_immu()
673 		if (immu_devi->imd_immu != immu) {  in immu_dvma_get_immu()
676 			    "actual (%p)", (void *)immu,  in immu_dvma_get_immu()
683 	return (immu);  in immu_dvma_get_immu()
806 did_alloc(immu_t *immu, dev_info_t *rdip,  in did_alloc()  argument
811 	did = (uintptr_t)vmem_alloc(immu->immu_did_arena, 1,  in did_alloc()
819 		    immu->immu_name, immu->immu_unity_domain->dom_did);  in did_alloc()
820 		did = immu->immu_unity_domain->dom_did;  in did_alloc()
832 	immu_t *immu;  in get_branch_domain()  local
853 	immu = immu_devi->imd_immu;  in get_branch_domain()
854 	if (immu == NULL)  in get_branch_domain()
855 		immu = immu_dvma_get_immu(pdip, dvp->dva_flags);  in get_branch_domain()
881 		dvp->dva_domain = immu->immu_unity_domain;  in get_branch_domain()
1020 create_xlate_arena(immu_t *immu, domain_t *domain,  in create_xlate_arena()  argument
1037 	    "%s-domain-%d-xlate-DVMA-arena", immu->immu_name,  in create_xlate_arena()
1043 	mgaw = IMMU_CAP_MGAW(immu->immu_regs_cap);  in create_xlate_arena()
1175 	immu_t *immu;  in device_domain()  local
1193 	immu = immu_dvma_get_immu(rdip, immu_flags);  in device_domain()
1194 	if (immu == NULL) {  in device_domain()
1247 	domain = domain_create(immu, ddip, rdip, immu_flags);  in device_domain()
1267 create_unity_domain(immu_t *immu)  in create_unity_domain()  argument
1279 	domain->dom_immu = immu;  in create_unity_domain()
1280 	immu->immu_unity_domain = domain;  in create_unity_domain()
1286 	domain->dom_pgtable_root = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);  in create_unity_domain()
1294 	if (!IMMU_ECAP_GET_PT(immu->immu_regs_excap))  in create_unity_domain()
1313 domain_create(immu_t *immu, dev_info_t *ddip, dev_info_t *rdip,  in domain_create()  argument
1331 	did = did_alloc(immu, rdip, ddip, immu_flags);  in domain_create()
1334 		ASSERT(immu->immu_unity_domain);  in domain_create()
1335 		return (immu->immu_unity_domain);  in domain_create()
1342 		    "structure for device. IOMMU unit: %s", immu->immu_name);  in domain_create()
1349 	    "immu%s-domain%d-pava-hash", immu->immu_name, did);  in domain_create()
1352 	domain->dom_immu = immu;  in domain_create()
1359 	create_xlate_arena(immu, domain, rdip, immu_flags);  in domain_create()
1364 	domain->dom_pgtable_root = pgtable_alloc(immu, immu_flags);  in domain_create()
1368 		    domain->dom_did, immu->immu_name);  in domain_create()
1377 	mutex_enter(&(immu->immu_lock));  in domain_create()
1378 	list_insert_head(&immu->immu_domain_list, domain);  in domain_create()
1379 	mutex_exit(&(immu->immu_lock));  in domain_create()
1419 did_init(immu_t *immu)  in did_init()  argument
1421 	(void) snprintf(immu->immu_did_arena_name,  in did_init()
1422 	    sizeof (immu->immu_did_arena_name),  in did_init()
1423 	    "%s_domainid_arena", immu->immu_name);  in did_init()
1425 	ddi_err(DER_VERB, immu->immu_dip, "creating domainid arena %s",  in did_init()
1426 	    immu->immu_did_arena_name);  in did_init()
1428 	immu->immu_did_arena = vmem_create(  in did_init()
1429 	    immu->immu_did_arena_name,  in did_init()
1431 	    immu->immu_max_domains - IMMU_UNITY_DID,  in did_init()
1440 	if (immu->immu_did_arena == NULL) {  in did_init()
1442 		    "IOMMU domainid allocator: %s", immu->immu_name,  in did_init()
1443 		    immu->immu_did_arena_name);  in did_init()
1450 context_set(immu_t *immu, domain_t *domain, pgtable_t *root_table,  in context_set()  argument
1473 	rw_enter(&(immu->immu_ctx_rwlock), RW_READER);  in context_set()
1479 			rw_exit(&(immu->immu_ctx_rwlock));  in context_set()
1490 	    rw_tryupgrade(&(immu->immu_ctx_rwlock)) == 0) {  in context_set()
1491 		rw_exit(&(immu->immu_ctx_rwlock));  in context_set()
1492 		rw_enter(&(immu->immu_ctx_rwlock), RW_WRITER);  in context_set()
1501 		immu_regs_cpu_flush(immu, (caddr_t)hw_rent, sizeof (hw_rce_t));  in context_set()
1510 		immu_regs_cpu_flush(immu, (caddr_t)hw_cent, sizeof (hw_rce_t));  in context_set()
1513 		immu_flush_context_fsi(immu, 0, sid, domain->dom_did,  in context_set()
1514 		    &immu->immu_ctx_inv_wait);  in context_set()
1518 		CONT_SET_AW(hw_cent, immu->immu_dvma_agaw);  in context_set()
1521 		    IMMU_ECAP_GET_PT(immu->immu_regs_excap))  in context_set()
1527 		if (IMMU_ECAP_GET_CH(immu->immu_regs_excap)) {  in context_set()
1532 		immu_regs_cpu_flush(immu, (caddr_t)hw_cent, sizeof (hw_rce_t));  in context_set()
1534 	rw_exit(&(immu->immu_ctx_rwlock));  in context_set()
1538 context_create(immu_t *immu)  in context_create()  argument
1550 	root_table = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);  in context_create()
1560 		context = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);  in context_create()
1568 			    immu->immu_unity_domain->dom_pgtable_root;  in context_create()
1570 			    immu->immu_unity_domain->dom_did);  in context_create()
1571 			CONT_SET_AW(hw_cent, immu->immu_dvma_agaw);  in context_create()
1573 			if (IMMU_ECAP_GET_PT(immu->immu_regs_excap))  in context_create()
1581 		immu_regs_cpu_flush(immu, context->hwpg_vaddr, IMMU_PAGESIZE);  in context_create()
1592 context_init(immu_t *immu)  in context_init()  argument
1594 	rw_init(&(immu->immu_ctx_rwlock), NULL, RW_DEFAULT, NULL);  in context_init()
1596 	immu_init_inv_wait(&immu->immu_ctx_inv_wait, "ctxglobal", B_TRUE);  in context_init()
1598 	immu_regs_wbf_flush(immu);  in context_init()
1600 	immu->immu_ctx_root = context_create(immu);  in context_init()
1602 	immu_regs_set_root_table(immu);  in context_init()
1604 	rw_enter(&(immu->immu_ctx_rwlock), RW_WRITER);  in context_init()
1605 	immu_flush_context_gbl(immu, &immu->immu_ctx_inv_wait);  in context_init()
1606 	immu_flush_iotlb_gbl(immu, &immu->immu_ctx_inv_wait);  in context_init()
1607 	rw_exit(&(immu->immu_ctx_rwlock));  in context_init()
1630 immu_context_update(immu_t *immu, domain_t *domain, dev_info_t *ddip,  in immu_context_update()  argument
1678 		context_set(immu, domain, immu->immu_ctx_root, r_bus,  in immu_context_update()
1687 		context_set(immu, domain, immu->immu_ctx_root, r_bus,  in immu_context_update()
1697 		context_set(immu, domain, immu->immu_ctx_root,  in immu_context_update()
1700 			context_set(immu, domain, immu->immu_ctx_root,  in immu_context_update()
1710 				context_set(immu, domain, immu->immu_ctx_root,  in immu_context_update()
1719 		context_set(immu, domain, immu->immu_ctx_root, d_bus,  in immu_context_update()
1726 		context_set(immu, domain, immu->immu_ctx_root, d_bus,  in immu_context_update()
1744 PDTE_check(immu_t *immu, hw_pdte_t pdte, pgtable_t *next, paddr_t paddr,  in PDTE_check()  argument
1764 	if (next == NULL && immu->immu_TM_reserved == B_FALSE) {  in PDTE_check()
1803 	if (next == NULL && immu->immu_SNP_reserved == B_FALSE) {  in PDTE_check()
1847 PTE_clear_all(immu_t *immu, domain_t *domain, xlate_t *xlate,  in PTE_clear_all()  argument
1972 	immu_t *immu;  in immu_print_fault_info()  local
1982 	immu = domain->dom_immu;  in immu_print_fault_info()
1991 	nlevels = immu->immu_dvma_nlevels;  in immu_print_fault_info()
2012 PTE_set_one(immu_t *immu, hw_pdte_t *hwp, paddr_t paddr,  in PTE_set_one()  argument
2018 	pte = immu->immu_ptemask;  in PTE_set_one()
2036 	if (immu->immu_TM_reserved == B_FALSE) {  in PTE_set_one()
2049 	if (immu->immu_SNP_reserved == B_FALSE) {  in PTE_set_one()
2076 	pte |= immu->immu_ptemask;  in PTE_set_one()
2095 PTE_set_all(immu_t *immu, domain_t *domain, xlate_t *xlate,  in PTE_set_all()  argument
2137 		PTE_set_one(immu, hwp, paddr, rdip, immu_flags);  in PTE_set_all()
2140 		ASSERT(PDTE_check(immu, *hwp, NULL, paddr, rdip, immu_flags)  in PTE_set_all()
2173 PDE_set_one(immu_t *immu, hw_pdte_t *hwp, pgtable_t *next,  in PDE_set_one()  argument
2243 PDE_set_all(immu_t *immu, domain_t *domain, xlate_t *xlate, int nlevels,  in PDE_set_all()  argument
2281 				new = pgtable_alloc(immu, immu_flags);  in PDE_set_all()
2301 			PDE_set_one(immu, hwp, next, rdip, immu_flags);  in PDE_set_all()
2325 		ASSERT(PDTE_check(immu, *hwp, next, 0, rdip, immu_flags)  in PDE_set_all()
2333 		pgtable_free(immu, new);  in PDE_set_all()
2358 	immu_t *immu = domain->dom_immu;  in dvma_map()  local
2359 	int nlevels = immu->immu_dvma_nlevels;  in dvma_map()
2370 		if (PDE_set_all(immu, domain, xlate, nlevels, rdip, immu_flags)  in dvma_map()
2376 		PTE_set_all(immu, domain, &xlate[1], &dvma, &n, dcookies,  in dvma_map()
2398 	immu_t *immu = domain->dom_immu;  in dvma_unmap()  local
2399 	int nlevels = immu->immu_dvma_nlevels;  in dvma_unmap()
2418 		PTE_clear_all(immu, domain, &xlate[1], &dvma, &n, rdip);  in dvma_unmap()
2462 	immu_t *immu;  in dvma_prealloc()  local
2467 	immu = domain->dom_immu;  in dvma_prealloc()
2468 	nlevels = immu->immu_dvma_nlevels;  in dvma_prealloc()
2501 		(void) PDE_set_all(immu, domain, xlate, nlevels, rdip,  in dvma_prealloc()
2509 		PTE_set_all(immu, domain, xlp, &dvma, &n, &immu_precookie,  in dvma_prealloc()
2557 	immu_t *immu;  in immu_map_dvmaseg()  local
2575 	immu = domain->dom_immu;  in immu_map_dvmaseg()
2623 		rwmask = PDTE_MASK_R | PDTE_MASK_W | immu->immu_ptemask;  in immu_map_dvmaseg()
2625 		rwmask = immu->immu_ptemask;  in immu_map_dvmaseg()
2721 	immu_flush_iotlb_psi(immu, domain->dom_did, sdvma, npgalloc,  in immu_map_dvmaseg()
2781 	immu_t *immu;  in immu_dvma_setup()  local
2803 	immu = list_head(listp);  in immu_dvma_setup()
2804 	for (; immu; immu = list_next(listp, immu)) {  in immu_dvma_setup()
2805 		create_unity_domain(immu);  in immu_dvma_setup()
2806 		did_init(immu);  in immu_dvma_setup()
2807 		context_init(immu);  in immu_dvma_setup()
2808 		immu->immu_dvma_setup = B_TRUE;  in immu_dvma_setup()
2816 immu_dvma_startup(immu_t *immu)  in immu_dvma_startup()  argument
2819 	    immu->immu_dvma_gfx_only == B_TRUE) {  in immu_dvma_startup()
2826 	immu->immu_dvma_running = B_TRUE;  in immu_dvma_startup()
2881 	immu_t *immu;  in immu_dvma_device_setup()  local
2886 	immu = immu_dvma_get_immu(rdip, immu_flags);  in immu_dvma_device_setup()
2887 	if (immu == NULL) {  in immu_dvma_device_setup()
2900 		rdip = get_lpc_devinfo(immu, rdip, immu_flags);  in immu_dvma_device_setup()
2908 	immu = NULL;  in immu_dvma_device_setup()
2933 	immu = domain->dom_immu;  in immu_dvma_device_setup()
2952 	if (immu_context_update(immu, domain, ddip, rdip, immu_flags)  in immu_dvma_device_setup()
2966 	immu_t *immu;  in immu_map_memrange()  local
2974 	immu = domain->dom_immu;  in immu_map_memrange()
2982 	immu_flush_iotlb_psi(immu, domain->dom_did, mrng->mrng_start,  in immu_map_memrange()
3065 	immu_t *immu;  in immu_allochdl()  local
3070 		immu = IMMU_DEVI(rdip)->imd_immu;  in immu_allochdl()
3072 		ihp = kmem_cache_alloc(immu->immu_hdl_cache,  in immu_allochdl()