/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Portions Copyright (c) 2010, Oracle and/or its affiliates. * All rights reserved. */ /* * Copyright (c) 2008, Intel Corporation. * All rights reserved. */ #ifndef _SYS_INTEL_IOMMU_H #define _SYS_INTEL_IOMMU_H /* * Intel IOMMU implementation specific state */ #ifdef __cplusplus extern "C" { #endif #include #include #include #include #include #include #include #include /* * Some ON drivers have bugs. Keep this define until all such drivers * have been fixed */ #define BUGGY_DRIVERS 1 /* PD(T)E entries */ typedef uint64_t hw_pdte_t; #define IMMU_MAXNAMELEN (64) #define IMMU_MAXSEG (1) #define IMMU_REGSZ (1UL << 12) #define IMMU_PAGESIZE (4096) #define IMMU_PAGESHIFT (12) #define IMMU_PAGEOFFSET (IMMU_PAGESIZE - 1) #define IMMU_PAGEMASK (~IMMU_PAGEOFFSET) #define IMMU_BTOP(b) (((uint64_t)b) >> IMMU_PAGESHIFT) #define IMMU_PTOB(p) (((uint64_t)p) << IMMU_PAGESHIFT) #define IMMU_BTOPR(x) ((((x) + IMMU_PAGEOFFSET) >> IMMU_PAGESHIFT)) #define IMMU_PGTABLE_MAX_LEVELS (6) #define IMMU_ROUNDUP(size) (((size) + IMMU_PAGEOFFSET) & ~IMMU_PAGEOFFSET) #define IMMU_ROUNDOWN(addr) ((addr) & ~IMMU_PAGEOFFSET) #define IMMU_PGTABLE_LEVEL_STRIDE (9) #define IMMU_PGTABLE_LEVEL_MASK ((1<> 3) /* from devfunc */ #define IMMU_PCI_FUNC(devfunc) (devfunc & 7) /* get func from devfunc */ #define IMMU_PCI_DEVFUNC(d, f) (((d) << 3) | (f)) /* create devfunc */ typedef struct scope { uint8_t scp_type; uint8_t scp_enumid; uint8_t scp_bus; uint8_t scp_dev; uint8_t scp_func; list_node_t scp_node; } scope_t; /* * interrupt source id and drhd info for ioapic */ typedef struct ioapic_drhd { uchar_t ioapic_ioapicid; uint16_t ioapic_sid; /* ioapic source id */ drhd_t *ioapic_drhd; list_node_t ioapic_node; } ioapic_drhd_t; typedef struct memrng { uint64_t mrng_start; uint64_t mrng_npages; } memrng_t; typedef enum immu_flags { IMMU_FLAGS_NONE = 0x1, IMMU_FLAGS_SLEEP = 0x1, IMMU_FLAGS_NOSLEEP = 0x2, IMMU_FLAGS_READ = 0x4, IMMU_FLAGS_WRITE = 0x8, IMMU_FLAGS_DONTPASS = 0x10, IMMU_FLAGS_ALLOC = 0x20, IMMU_FLAGS_MUST_MATCH = 0x40, IMMU_FLAGS_PAGE1 = 0x80, IMMU_FLAGS_UNITY = 0x100, IMMU_FLAGS_DMAHDL = 0x200, IMMU_FLAGS_MEMRNG = 0x400 } immu_flags_t; typedef enum cont_avail { IMMU_CONT_BAD = 0x0, IMMU_CONT_UNINITED = 0x1, IMMU_CONT_INITED = 0x2 } cont_avail_t; /* Size of root and context tables and their entries */ #define IMMU_ROOT_TBLSZ (4096) #define IMMU_CONT_TBLSZ (4096) #define IMMU_ROOT_NUM (256) #define IMMU_CONT_NUM (256) /* register offset */ #define IMMU_REG_VERSION (0x00) /* Version Rigister, 32 bit */ #define IMMU_REG_CAP (0x08) /* Capability Register, 64 bit */ #define IMMU_REG_EXCAP (0x10) /* Extended Capability Reg, 64 bit */ #define IMMU_REG_GLOBAL_CMD (0x18) /* Global Command Register, 32 bit */ #define IMMU_REG_GLOBAL_STS (0x1C) /* Global Status Register, 32 bit */ #define IMMU_REG_ROOTENTRY (0x20) /* Root-Entry Table Addr Reg, 64 bit */ #define IMMU_REG_CONTEXT_CMD (0x28) /* Context Comand Register, 64 bit */ #define IMMU_REG_FAULT_STS (0x34) /* Fault Status Register, 32 bit */ #define IMMU_REG_FEVNT_CON (0x38) /* Fault Event Control Reg, 32 bit */ #define IMMU_REG_FEVNT_DATA (0x3C) /* Fault Event Data Register, 32 bit */ #define IMMU_REG_FEVNT_ADDR (0x40) /* Fault Event Address Reg, 32 bit */ #define IMMU_REG_FEVNT_UADDR (0x44) /* Fault Event Upper Addr Reg, 32 bit */ #define IMMU_REG_AFAULT_LOG (0x58) /* Advanced Fault Log Reg, 64 bit */ #define IMMU_REG_PMER (0x64) /* Protected Memory Enble Reg, 32 bit */ #define IMMU_REG_PLMBR (0x68) /* Protected Low Mem Base Reg, 32 bit */ #define IMMU_REG_PLMLR (0x6C) /* Protected Low Mem Lim Reg, 32 bit */ #define IMMU_REG_PHMBR (0X70) /* Protectd High Mem Base Reg, 64 bit */ #define IMMU_REG_PHMLR (0x78) /* Protected High Mem Lim Reg, 64 bit */ #define IMMU_REG_INVAL_QH (0x80) /* Invalidation Queue Head, 64 bit */ #define IMMU_REG_INVAL_QT (0x88) /* Invalidation Queue Tail, 64 bit */ #define IMMU_REG_INVAL_QAR (0x90) /* Invalidtion Queue Addr Reg, 64 bit */ #define IMMU_REG_INVAL_CSR (0x9C) /* Inval Compl Status Reg, 32 bit */ #define IMMU_REG_INVAL_CECR (0xA0) /* Inval Compl Evnt Ctrl Reg, 32 bit */ #define IMMU_REG_INVAL_CEDR (0xA4) /* Inval Compl Evnt Data Reg, 32 bit */ #define IMMU_REG_INVAL_CEAR (0xA8) /* Inval Compl Event Addr Reg, 32 bit */ #define IMMU_REG_INVAL_CEUAR (0xAC) /* Inval Comp Evnt Up Addr reg, 32bit */ #define IMMU_REG_IRTAR (0xB8) /* INTR Remap Tbl Addr Reg, 64 bit */ /* ioapic memory region */ #define IOAPIC_REGION_START (0xfee00000) #define IOAPIC_REGION_END (0xfeefffff) /* fault register */ #define IMMU_FAULT_STS_PPF (2) #define IMMU_FAULT_STS_PFO (1) #define IMMU_FAULT_STS_ITE (1 << 6) #define IMMU_FAULT_STS_ICE (1 << 5) #define IMMU_FAULT_STS_IQE (1 << 4) #define IMMU_FAULT_GET_INDEX(x) ((((uint64_t)x) >> 8) & 0xff) #define IMMU_FRR_GET_F(x) (((uint64_t)x) >> 63) #define IMMU_FRR_GET_FR(x) ((((uint64_t)x) >> 32) & 0xff) #define IMMU_FRR_GET_FT(x) ((((uint64_t)x) >> 62) & 0x1) #define IMMU_FRR_GET_SID(x) ((x) & 0xffff) /* (ex)capability register */ #define IMMU_CAP_GET_NFR(x) (((((uint64_t)x) >> 40) & 0xff) + 1) #define IMMU_CAP_GET_DWD(x) ((((uint64_t)x) >> 54) & 1) #define IMMU_CAP_GET_DRD(x) ((((uint64_t)x) >> 55) & 1) #define IMMU_CAP_GET_PSI(x) ((((uint64_t)x) >> 39) & 1) #define IMMU_CAP_GET_SPS(x) ((((uint64_t)x) >> 34) & 0xf) #define IMMU_CAP_GET_ISOCH(x) ((((uint64_t)x) >> 23) & 1) #define IMMU_CAP_GET_ZLR(x) ((((uint64_t)x) >> 22) & 1) #define IMMU_CAP_GET_MAMV(x) ((((uint64_t)x) >> 48) & 0x3f) #define IMMU_CAP_GET_CM(x) ((((uint64_t)x) >> 7) & 1) #define IMMU_CAP_GET_PHMR(x) ((((uint64_t)x) >> 6) & 1) #define IMMU_CAP_GET_PLMR(x) ((((uint64_t)x) >> 5) & 1) #define IMMU_CAP_GET_RWBF(x) ((((uint64_t)x) >> 4) & 1) #define IMMU_CAP_GET_AFL(x) ((((uint64_t)x) >> 3) & 1) #define IMMU_CAP_GET_FRO(x) (((((uint64_t)x) >> 24) & 0x3ff) * 16) #define IMMU_CAP_MGAW(x) (((((uint64_t)x) >> 16) & 0x3f) + 1) #define IMMU_CAP_SAGAW(x) ((((uint64_t)x) >> 8) & 0x1f) #define IMMU_CAP_ND(x) (1 << (((x) & 0x7) *2 + 4)) -1 #define IMMU_ECAP_GET_IRO(x) (((((uint64_t)x) >> 8) & 0x3ff) << 4) #define IMMU_ECAP_GET_MHMV(x) (((uint64_t)x >> 20) & 0xf) #define IMMU_ECAP_GET_SC(x) ((x) & 0x80) #define IMMU_ECAP_GET_PT(x) ((x) & 0x40) #define IMMU_ECAP_GET_CH(x) ((x) & 0x20) #define IMMU_ECAP_GET_EIM(x) ((x) & 0x10) #define IMMU_ECAP_GET_IR(x) ((x) & 0x8) #define IMMU_ECAP_GET_DI(x) ((x) & 0x4) #define IMMU_ECAP_GET_QI(x) ((x) & 0x2) #define IMMU_ECAP_GET_C(x) ((x) & 0x1) #define IMMU_CAP_SET_RWBF(x) ((x) |= (1 << 4)) /* iotlb invalidation */ #define TLB_INV_GLOBAL (((uint64_t)1) << 60) #define TLB_INV_DOMAIN (((uint64_t)2) << 60) #define TLB_INV_PAGE (((uint64_t)3) << 60) #define TLB_INV_GET_IAIG(x) ((((uint64_t)x) >> 57) & 7) #define TLB_INV_DRAIN_READ (((uint64_t)1) << 49) #define TLB_INV_DRAIN_WRITE (((uint64_t)1) << 48) #define TLB_INV_DID(x) (((uint64_t)((x) & 0xffff)) << 32) #define TLB_INV_IVT (((uint64_t)1) << 63) #define TLB_IVA_HINT(x) (((x) & 0x1) << 6) #define TLB_IVA_LEAF 1 #define TLB_IVA_WHOLE 0 /* dont use value 0 for enums - to catch unit 8 */ typedef enum iotlb_inv { IOTLB_PSI = 1, IOTLB_DSI, IOTLB_GLOBAL } immu_iotlb_inv_t; typedef enum context_inv { CONTEXT_FSI = 1, CONTEXT_DSI, CONTEXT_GLOBAL } immu_context_inv_t; /* context invalidation */ #define CCMD_INV_ICC (((uint64_t)1) << 63) #define CCMD_INV_GLOBAL (((uint64_t)1) << 61) #define CCMD_INV_DOMAIN (((uint64_t)2) << 61) #define CCMD_INV_DEVICE (((uint64_t)3) << 61) #define CCMD_INV_DID(x) ((uint64_t)((x) & 0xffff)) #define CCMD_INV_SID(x) (((uint64_t)((x) & 0xffff)) << 16) #define CCMD_INV_FM(x) (((uint64_t)((x) & 0x3)) << 32) /* global command register */ #define IMMU_GCMD_TE (((uint32_t)1) << 31) #define IMMU_GCMD_SRTP (((uint32_t)1) << 30) #define IMMU_GCMD_SFL (((uint32_t)1) << 29) #define IMMU_GCMD_EAFL (((uint32_t)1) << 28) #define IMMU_GCMD_WBF (((uint32_t)1) << 27) #define IMMU_GCMD_QIE (((uint32_t)1) << 26) #define IMMU_GCMD_IRE (((uint32_t)1) << 25) #define IMMU_GCMD_SIRTP (((uint32_t)1) << 24) #define IMMU_GCMD_CFI (((uint32_t)1) << 23) /* global status register */ #define IMMU_GSTS_TES (((uint32_t)1) << 31) #define IMMU_GSTS_RTPS (((uint32_t)1) << 30) #define IMMU_GSTS_FLS (((uint32_t)1) << 29) #define IMMU_GSTS_AFLS (((uint32_t)1) << 28) #define IMMU_GSTS_WBFS (((uint32_t)1) << 27) #define IMMU_GSTS_QIES (((uint32_t)1) << 26) #define IMMU_GSTS_IRES (((uint32_t)1) << 25) #define IMMU_GSTS_IRTPS (((uint32_t)1) << 24) #define IMMU_GSTS_CFIS (((uint32_t)1) << 23) /* psi address mask */ #define ADDR_AM_MAX(m) (((uint_t)1) << (m)) #define ADDR_AM_OFFSET(n, m) ((n) & (ADDR_AM_MAX(m) - 1)) /* dmar fault event */ #define IMMU_INTR_IPL (4) #define IMMU_REG_FEVNT_CON_IM_SHIFT (31) #define IMMU_ALLOC_RESOURCE_DELAY (drv_usectohz(5000)) /* max value of Size field of Interrupt Remapping Table Address Register */ #define INTRMAP_MAX_IRTA_SIZE 0xf /* interrupt remapping table entry size */ #define INTRMAP_RTE_SIZE 0x10 /* ioapic redirection table entry related shift of remappable interrupt */ #define INTRMAP_IOAPIC_IDX_SHIFT 17 #define INTRMAP_IOAPIC_FORMAT_SHIFT 16 #define INTRMAP_IOAPIC_TM_SHIFT 15 #define INTRMAP_IOAPIC_POL_SHIFT 13 #define INTRMAP_IOAPIC_IDX15_SHIFT 11 /* msi intr entry related shift of remappable interrupt */ #define INTRMAP_MSI_IDX_SHIFT 5 #define INTRMAP_MSI_FORMAT_SHIFT 4 #define INTRMAP_MSI_SHV_SHIFT 3 #define INTRMAP_MSI_IDX15_SHIFT 2 #define INTRMAP_IDX_FULL (uint_t)-1 #define RDT_DLM(rdt) BITX((rdt), 10, 8) #define RDT_DM(rdt) BT_TEST(&(rdt), 11) #define RDT_POL(rdt) BT_TEST(&(rdt), 13) #define RDT_TM(rdt) BT_TEST(&(rdt), 15) #define INTRMAP_DISABLE (void *)-1 /* * invalidation granularity */ typedef enum { TLB_INV_G_GLOBAL = 1, TLB_INV_G_DOMAIN, TLB_INV_G_PAGE } tlb_inv_g_t; typedef enum { CTT_INV_G_GLOBAL = 1, CTT_INV_G_DOMAIN, CTT_INV_G_DEVICE } ctt_inv_g_t; typedef enum { IEC_INV_GLOBAL = 0, IEC_INV_INDEX } iec_inv_g_t; struct inv_queue_state; struct intrmap_tbl_state; /* A software page table structure */ typedef struct pgtable { krwlock_t swpg_rwlock; caddr_t hwpg_vaddr; /* HW pgtable VA */ paddr_t hwpg_paddr; /* HW pgtable PA */ ddi_dma_handle_t hwpg_dmahdl; ddi_acc_handle_t hwpg_memhdl; struct pgtable **swpg_next_array; list_node_t swpg_domain_node; /* domain list of pgtables */ } pgtable_t; /* interrupt remapping table state info */ typedef struct intrmap { kmutex_t intrmap_lock; ddi_dma_handle_t intrmap_dma_hdl; ddi_acc_handle_t intrmap_acc_hdl; caddr_t intrmap_vaddr; paddr_t intrmap_paddr; uint_t intrmap_size; bitset_t intrmap_map; uint_t intrmap_free; } intrmap_t; typedef struct hw_rce { uint64_t lo; uint64_t hi; } hw_rce_t; #define ROOT_GET_P(hrent) ((hrent)->lo & 0x1) #define ROOT_SET_P(hrent) ((hrent)->lo |= 0x1) #define ROOT_GET_CONT(hrent) ((hrent)->lo & ~(0xFFF)) #define ROOT_SET_CONT(hrent, paddr) ((hrent)->lo |= (paddr & (~0xFFF))) #define TTYPE_XLATE_ONLY (0x0) #define TTYPE_XLATE_IOTLB (0x1) #define TTYPE_PASSTHRU (0x2) #define TTYPE_RESERVED (0x3) #define CONT_GET_DID(hcent) ((((uint64_t)(hcent)->hi) >> 8) & 0xFFFF) #define CONT_SET_DID(hcent, did) ((hcent)->hi |= ((0xFFFF & (did)) << 8)) #define CONT_GET_AVAIL(hcent) ((((uint64_t)((hcent)->hi)) >> 0x3) & 0xF) #define CONT_SET_AVAIL(hcent, av) ((hcent)->hi |= ((0xF & (av)) << 0x3)) #define CONT_GET_LO_AW(hcent) (30 + 9 *((hcent)->hi & 0x7)) #define CONT_GET_AW(hcent) \ ((CONT_GET_LO_AW(hcent) == 66) ? 64 : CONT_GET_LO_AW(hcent)) #define CONT_SET_AW(hcent, aw) \ ((hcent)->hi |= (((((aw) + 2) - 30) / 9) & 0x7)) #define CONT_GET_ASR(hcent) ((hcent)->lo & ~(0xFFF)) #define CONT_SET_ASR(hcent, paddr) ((hcent)->lo |= (paddr & (~0xFFF))) #define CONT_GET_TTYPE(hcent) ((((uint64_t)(hcent)->lo) >> 0x2) & 0x3) #define CONT_SET_TTYPE(hcent, ttype) ((hcent)->lo |= (((ttype) & 0x3) << 0x2)) #define CONT_GET_P(hcent) ((hcent)->lo & 0x1) #define CONT_SET_P(hcent) ((hcent)->lo |= 0x1) #define CONT_GET_ALH(hcent) ((hcent)->lo & 0x20) #define CONT_SET_ALH(hcent) ((hcent)->lo |= 0x20) #define CONT_GET_EH(hcent) ((hcent)->lo & 0x10) #define CONT_SET_EH(hcent) ((hcent)->lo |= 0x10) /* we use the bit 63 (available for system SW) as a present bit */ #define PDTE_SW4(hw_pdte) ((hw_pdte) & ((uint64_t)1<<63)) #define PDTE_CLEAR_SW4(hw_pdte) ((hw_pdte) &= ~((uint64_t)1<<63)) #define PDTE_P(hw_pdte) ((hw_pdte) & ((uint64_t)1<<63)) #define PDTE_CLEAR_P(hw_pdte) ((hw_pdte) &= ~((uint64_t)1<<63)) #define PDTE_SET_P(hw_pdte) ((hw_pdte) |= ((uint64_t)1<<63)) #define PDTE_TM(hw_pdte) ((hw_pdte) & ((uint64_t)1<<62)) #define PDTE_CLEAR_TM(hw_pdte) ((hw_pdte) &= ~((uint64_t)1<<62)) #define PDTE_SW3(hw_pdte) \ (((hw_pdte) & ~(((uint64_t)0x3<<62)|(((uint64_t)1<<52)-1))) >> 52) #define PDTE_SW3_OVERFLOW(hw_pdte) \ (PDTE_SW3(hw_pdte) == 0x3FF) #define PDTE_CLEAR_SW3(hw_pdte) \ ((hw_pdte) &= (((uint64_t)0x3<<62)|(((uint64_t)1<<52)-1))) #define PDTE_SET_SW3(hw_pdte, ref) \ ((hw_pdte) |= ((((uint64_t)(ref)) & 0x3FF) << 52)) #define PDTE_PADDR(hw_pdte) ((hw_pdte) & ~(((uint64_t)0xFFF<<52)|((1<<12)-1))) #define PDTE_CLEAR_PADDR(hw_pdte) \ ((hw_pdte) &= (((uint64_t)0xFFF<<52)|((1<<12)-1))) #define PDTE_SET_PADDR(hw_pdte, paddr) ((hw_pdte) |= PDTE_PADDR(paddr)) #define PDTE_SNP(hw_pdte) ((hw_pdte) & (1<<11)) #define PDTE_CLEAR_SNP(hw_pdte) ((hw_pdte) &= ~(1<<11)) #define PDTE_SET_SNP(hw_pdte) ((hw_pdte) |= (1<<11)) #define PDTE_SW2(hw_pdte) ((hw_pdte) & (0x700)) #define PDTE_CLEAR_SW2(hw_pdte) ((hw_pdte) &= ~(0x700)) #define PDTE_SP(hw_pdte) ((hw_pdte) & (0x80)) #define PDTE_CLEAR_SP(hw_pdte) ((hw_pdte) &= ~(0x80)) #define PDTE_SW1(hw_pdte) ((hw_pdte) & (0x7C)) #define PDTE_CLEAR_SW1(hw_pdte) ((hw_pdte) &= ~(0x7C)) #define PDTE_WRITE(hw_pdte) ((hw_pdte) & (0x2)) #define PDTE_CLEAR_WRITE(hw_pdte) ((hw_pdte) &= ~(0x2)) #define PDTE_SET_WRITE(hw_pdte) ((hw_pdte) |= (0x2)) #define PDTE_READ(hw_pdte) ((hw_pdte) & (0x1)) #define PDTE_CLEAR_READ(hw_pdte) ((hw_pdte) &= ~(0x1)) #define PDTE_SET_READ(hw_pdte) ((hw_pdte) |= (0x1)) #define PDTE_MASK_R ((uint64_t)1 << 0) #define PDTE_MASK_W ((uint64_t)1 << 1) #define PDTE_MASK_SNP ((uint64_t)1 << 11) #define PDTE_MASK_TM ((uint64_t)1 << 62) #define PDTE_MASK_P ((uint64_t)1 << 63) struct immu_flushops; /* * Used to wait for invalidation completion. * vstatus is the virtual address of the status word that will be written * pstatus is the physical addres * If sync is true, then the the operation will be waited on for * completion immediately. Else, the wait interface can be called * to wait for completion later. */ #define IMMU_INV_DATA_PENDING 1 #define IMMU_INV_DATA_DONE 2 typedef struct immu_inv_wait { volatile uint32_t iwp_vstatus; uint64_t iwp_pstatus; boolean_t iwp_sync; const char *iwp_name; /* ID for debugging/statistics */ } immu_inv_wait_t; /* * Used to batch IOMMU pagetable writes. */ typedef struct immu_dcookie { paddr_t dck_paddr; uint64_t dck_npages; } immu_dcookie_t; typedef struct immu { kmutex_t immu_lock; char *immu_name; /* lock grabbed by interrupt handler */ kmutex_t immu_intr_lock; /* ACPI/DMAR table related */ void *immu_dmar_unit; dev_info_t *immu_dip; struct domain *immu_unity_domain; /* IOMMU register related */ kmutex_t immu_regs_lock; kcondvar_t immu_regs_cv; boolean_t immu_regs_busy; boolean_t immu_regs_setup; boolean_t immu_regs_running; boolean_t immu_regs_quiesced; ddi_acc_handle_t immu_regs_handle; caddr_t immu_regs_addr; uint64_t immu_regs_cap; uint64_t immu_regs_excap; uint32_t immu_regs_cmdval; uint32_t immu_regs_intr_msi_addr; uint32_t immu_regs_intr_msi_data; uint32_t immu_regs_intr_uaddr; /* DVMA related */ kmutex_t immu_dvma_lock; boolean_t immu_dvma_setup; boolean_t immu_dvma_running; int immu_dvma_gaw; int immu_dvma_agaw; int immu_dvma_nlevels; boolean_t immu_dvma_coherent; boolean_t immu_TM_reserved; boolean_t immu_SNP_reserved; uint64_t immu_ptemask; /* DVMA context related */ krwlock_t immu_ctx_rwlock; pgtable_t *immu_ctx_root; immu_inv_wait_t immu_ctx_inv_wait; /* DVMA domain related */ int immu_max_domains; vmem_t *immu_did_arena; char immu_did_arena_name[IMMU_MAXNAMELEN]; list_t immu_domain_list; /* DVMA special devices */ boolean_t immu_dvma_gfx_only; list_t immu_dvma_lpc_list; list_t immu_dvma_gfx_list; /* interrupt remapping related */ kmutex_t immu_intrmap_lock; boolean_t immu_intrmap_setup; boolean_t immu_intrmap_running; intrmap_t *immu_intrmap; uint64_t immu_intrmap_irta_reg; immu_inv_wait_t immu_intrmap_inv_wait; /* queued invalidation related */ kmutex_t immu_qinv_lock; boolean_t immu_qinv_setup; boolean_t immu_qinv_running; boolean_t immu_qinv_enabled; void *immu_qinv; uint64_t immu_qinv_reg_value; /* list_node for system-wide list of DMAR units */ list_node_t immu_node; struct immu_flushops *immu_flushops; kmem_cache_t *immu_hdl_cache; kmem_cache_t *immu_pgtable_cache; iommulib_handle_t immu_iommulib_handle; } immu_t; /* * Enough space to hold the decimal number of any device instance. * Used for device/cache names. */ #define IMMU_ISTRLEN 11 /* log10(2^31) + 1 */ /* properties that control DVMA */ #define DDI_DVMA_MAPTYPE_ROOTNEX_PROP "immu-dvma-mapping" #define DDI_DVMA_MAPTYPE_UNITY "unity" #define DDI_DVMA_MAPTYPE_XLATE "xlate" typedef enum immu_maptype { IMMU_MAPTYPE_BAD = 0, /* 0 is always bad */ IMMU_MAPTYPE_UNITY = 1, IMMU_MAPTYPE_XLATE } immu_maptype_t; #define IMMU_COOKIE_HASHSZ (512) /* * domain_t * */ typedef struct domain { /* the basics */ uint_t dom_did; immu_t *dom_immu; /* mapping related */ immu_maptype_t dom_maptype; vmem_t *dom_dvma_arena; char dom_dvma_arena_name[IMMU_MAXNAMELEN]; /* pgtables */ pgtable_t *dom_pgtable_root; krwlock_t dom_pgtable_rwlock; /* list node for list of domains (unity or xlate) */ list_node_t dom_maptype_node; /* list node for list of domains off immu */ list_node_t dom_immu_node; mod_hash_t *dom_cookie_hash; /* topmost device in domain; usually the device itself (non-shared) */ dev_info_t *dom_dip; } domain_t; typedef enum immu_pcib { IMMU_PCIB_BAD = 0, IMMU_PCIB_NOBDF, IMMU_PCIB_PCIE_PCIE, IMMU_PCIB_PCIE_PCI, IMMU_PCIB_PCI_PCI, IMMU_PCIB_ENDPOINT } immu_pcib_t; /* * immu_devi_t * Intel IOMMU in devinfo node */ typedef struct immu_devi { /* pci seg, bus, dev, func */ int imd_seg; int imd_bus; int imd_devfunc; /* ppb information */ immu_pcib_t imd_pcib_type; int imd_sec; int imd_sub; /* identifier for special devices */ boolean_t imd_display; boolean_t imd_lpc; /* set if premapped DVMA space is used */ boolean_t imd_use_premap; /* dmar unit to which this dip belongs */ immu_t *imd_immu; immu_flags_t imd_dvma_flags; /* domain ptr */ domain_t *imd_domain; dev_info_t *imd_ddip; /* my devinfo */ dev_info_t *imd_dip; /* * if we are a "special" devinfo * the node for the special linked list * off the DMAR unit structure */ list_node_t imd_spc_node; } immu_devi_t; #define IMMU_DEVI(dip) ((immu_devi_t *)(DEVI(dip)->devi_iommu)) #define IMMU_DEVI_SET(dip, imd) (DEVI(dip)->devi_iommu = (void *)imd) /* * struct dmar_arg */ typedef struct immu_arg { int ima_seg; int ima_bus; int ima_devfunc; dev_info_t *ima_rdip; dev_info_t *ima_ddip; } immu_arg_t; #define IMMU_NDVSEG 8 #define IMMU_NDCK 64 #define IMMU_NPREPTES 8 typedef struct immu_hdl_private { immu_inv_wait_t ihp_inv_wait; size_t ihp_ndvseg; struct dvmaseg ihp_dvseg[IMMU_NDVSEG]; immu_dcookie_t ihp_dcookies[IMMU_NDCK]; hw_pdte_t *ihp_preptes[IMMU_NPREPTES]; uint64_t ihp_predvma; int ihp_npremapped; } immu_hdl_priv_t; /* * Invalidation operation function pointers for context and IOTLB. * These will be set to either the register or the queue invalidation * interface functions, since the hardware does not allow using them * both at the same time. */ struct immu_flushops { void (*imf_context_fsi)(immu_t *, uint8_t, uint16_t, uint_t, immu_inv_wait_t *); void (*imf_context_dsi)(immu_t *, uint_t, immu_inv_wait_t *); void (*imf_context_gbl)(immu_t *, immu_inv_wait_t *); void (*imf_iotlb_psi)(immu_t *, uint_t, uint64_t, uint_t, uint_t, immu_inv_wait_t *); void (*imf_iotlb_dsi)(immu_t *, uint_t, immu_inv_wait_t *); void (*imf_iotlb_gbl)(immu_t *, immu_inv_wait_t *); void (*imf_wait)(immu_inv_wait_t *); }; #define immu_flush_context_fsi(i, f, s, d, w) \ (i)->immu_flushops->imf_context_fsi(i, f, s, d, w) #define immu_flush_context_dsi(i, d, w) \ (i)->immu_flushops->imf_context_dsi(i, d, w) #define immu_flush_context_gbl(i, w) \ (i)->immu_flushops->imf_context_gbl(i, w) #define immu_flush_iotlb_psi(i, d, v, c, h, w) \ (i)->immu_flushops->imf_iotlb_psi(i, d, v, c, h, w) #define immu_flush_iotlb_dsi(i, d, w) \ (i)->immu_flushops->imf_iotlb_dsi(i, d, w) #define immu_flush_iotlb_gbl(i, w) \ (i)->immu_flushops->imf_iotlb_gbl(i, w) #define immu_flush_wait(i, w) \ (i)->immu_flushops->imf_wait(w) /* * Globals used by IOMMU code */ /* shared between IOMMU files */ extern dev_info_t *root_devinfo; extern kmutex_t immu_lock; extern list_t immu_list; extern boolean_t immu_setup; extern boolean_t immu_running; extern kmutex_t ioapic_drhd_lock; extern list_t ioapic_drhd_list; extern struct iommulib_ops immulib_ops; /* switches */ /* Various features */ extern boolean_t immu_enable; extern boolean_t immu_gfxdvma_enable; extern boolean_t immu_intrmap_enable; extern boolean_t immu_qinv_enable; /* various quirks that need working around */ extern boolean_t immu_quirk_usbpage0; extern boolean_t immu_quirk_usbfullpa; extern boolean_t immu_quirk_usbrmrr; extern boolean_t immu_quirk_mobile4; /* debug messages */ extern boolean_t immu_dmar_print; /* tunables */ extern int64_t immu_flush_gran; extern immu_flags_t immu_global_dvma_flags; extern int immu_use_tm; extern int immu_use_alh; /* ################### Interfaces exported outside IOMMU code ############## */ void immu_init(void); void immu_startup(void); void immu_shutdown(void); void immu_destroy(void); int immu_map_sgl(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, int prealloc_count, dev_info_t *rdip); int immu_unmap_sgl(ddi_dma_impl_t *hp, dev_info_t *rdip); void immu_device_tree_changed(void); void immu_physmem_update(uint64_t addr, uint64_t size); int immu_quiesce(void); int immu_unquiesce(void); /* ######################################################################### */ /* ################# Interfaces used within IOMMU code #################### */ /* immu_dmar.c interfaces */ int immu_dmar_setup(void); int immu_dmar_parse(void); void immu_dmar_startup(void); void immu_dmar_shutdown(void); void immu_dmar_destroy(void); boolean_t immu_dmar_blacklisted(char **strings_array, uint_t nstrings); immu_t *immu_dmar_get_immu(dev_info_t *rdip); dev_info_t *immu_dmar_unit_dip(void *dmar_unit); void immu_dmar_set_immu(void *dmar_unit, immu_t *immu); void *immu_dmar_walk_units(int seg, void *dmar_unit); boolean_t immu_dmar_intrmap_supported(void); uint16_t immu_dmar_ioapic_sid(int ioapicid); immu_t *immu_dmar_ioapic_immu(int ioapicid); void immu_dmar_rmrr_map(void); /* immu.c interfaces */ int immu_walk_ancestor(dev_info_t *rdip, dev_info_t *ddip, int (*func)(dev_info_t *, void *arg), void *arg, int *level, immu_flags_t immu_flags); void immu_init_inv_wait(immu_inv_wait_t *iwp, const char *s, boolean_t sync); /* immu_regs.c interfaces */ void immu_regs_setup(list_t *immu_list); void immu_regs_startup(immu_t *immu); int immu_regs_resume(immu_t *immu); void immu_regs_suspend(immu_t *immu); void immu_regs_shutdown(immu_t *immu); void immu_regs_destroy(list_t *immu_list); void immu_regs_intr(immu_t *immu, uint32_t msi_addr, uint32_t msi_data, uint32_t uaddr); boolean_t immu_regs_passthru_supported(immu_t *immu); boolean_t immu_regs_is_TM_reserved(immu_t *immu); boolean_t immu_regs_is_SNP_reserved(immu_t *immu); void immu_regs_wbf_flush(immu_t *immu); void immu_regs_cpu_flush(immu_t *immu, caddr_t addr, uint_t size); void immu_regs_context_fsi(immu_t *immu, uint8_t function_mask, uint16_t source_id, uint_t domain_id, immu_inv_wait_t *iwp); void immu_regs_context_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp); void immu_regs_context_gbl(immu_t *immu, immu_inv_wait_t *iwp); void immu_regs_iotlb_psi(immu_t *immu, uint_t domain_id, uint64_t dvma, uint_t count, uint_t hint, immu_inv_wait_t *iwp); void immu_regs_iotlb_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp); void immu_regs_iotlb_gbl(immu_t *immu, immu_inv_wait_t *iwp); void immu_regs_set_root_table(immu_t *immu); void immu_regs_qinv_enable(immu_t *immu, uint64_t qinv_reg_value); void immu_regs_intr_enable(immu_t *immu, uint32_t msi_addr, uint32_t msi_data, uint32_t uaddr); void immu_regs_intrmap_enable(immu_t *immu, uint64_t irta_reg); uint64_t immu_regs_get64(immu_t *immu, uint_t reg); void immu_regs_put64(immu_t *immu, uint_t reg, uint64_t val); uint32_t immu_regs_get32(immu_t *immu, uint_t reg); void immu_regs_put32(immu_t *immu, uint_t reg, uint32_t val); /* immu_dvma.c interfaces */ void immu_dvma_setup(list_t *immu_list); void immu_dvma_startup(immu_t *immu); void immu_dvma_shutdown(immu_t *immu); void immu_dvma_destroy(list_t *immu_list); void immu_dvma_physmem_update(uint64_t addr, uint64_t size); int immu_map_memrange(dev_info_t *, memrng_t *); int immu_dvma_map(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, uint_t prealloc_count, dev_info_t *rdip); int immu_dvma_unmap(ddi_dma_impl_t *hp, dev_info_t *rdip); int immu_devi_set(dev_info_t *dip, immu_flags_t immu_flags); immu_devi_t *immu_devi_get(dev_info_t *dip); immu_t *immu_dvma_get_immu(dev_info_t *dip, immu_flags_t immu_flags); int pgtable_ctor(void *buf, void *arg, int kmflag); void pgtable_dtor(void *buf, void *arg); int immu_hdl_priv_ctor(void *buf, void *arg, int kmf); int immu_dvma_device_setup(dev_info_t *rdip, immu_flags_t immu_flags); void immu_print_fault_info(uint_t sid, uint64_t dvma); /* immu_intrmap.c interfaces */ void immu_intrmap_setup(list_t *immu_list); void immu_intrmap_startup(immu_t *immu); void immu_intrmap_shutdown(immu_t *immu); void immu_intrmap_destroy(list_t *immu_list); /* registers interrupt handler for IOMMU unit */ void immu_intr_register(immu_t *immu); int immu_intr_handler(immu_t *immu); /* immu_qinv.c interfaces */ int immu_qinv_setup(list_t *immu_list); void immu_qinv_startup(immu_t *immu); void immu_qinv_shutdown(immu_t *immu); void immu_qinv_destroy(list_t *immu_list); void immu_qinv_context_fsi(immu_t *immu, uint8_t function_mask, uint16_t source_id, uint_t domain_id, immu_inv_wait_t *iwp); void immu_qinv_context_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp); void immu_qinv_context_gbl(immu_t *immu, immu_inv_wait_t *iwp); void immu_qinv_iotlb_psi(immu_t *immu, uint_t domain_id, uint64_t dvma, uint_t count, uint_t hint, immu_inv_wait_t *iwp); void immu_qinv_iotlb_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp); void immu_qinv_iotlb_gbl(immu_t *immu, immu_inv_wait_t *iwp); void immu_qinv_intr_global(immu_t *immu, immu_inv_wait_t *iwp); void immu_qinv_intr_one_cache(immu_t *immu, uint_t idx, immu_inv_wait_t *iwp); void immu_qinv_intr_caches(immu_t *immu, uint_t idx, uint_t cnt, immu_inv_wait_t *); void immu_qinv_report_fault(immu_t *immu); #ifdef DEBUG #define IMMU_DPROBE1(name, type1, arg1) \ DTRACE_PROBE1(name, type1, arg1) #define IMMU_DPROBE2(name, type1, arg1, type2, arg2) \ DTRACE_PROBE2(name, type1, arg1, type2, arg2) #define IMMU_DPROBE3(name, type1, arg1, type2, arg2, type3, arg3) \ DTRACE_PROBE3(name, type1, arg1, type2, arg2, type3, arg3) #define IMMU_DPROBE4(name, type1, arg1, type2, arg2, type3, arg3, type4, arg4) \ DTRACE_PROBE4(name, type1, arg1, type2, arg2, type3, arg3, type4, arg4) #else #define IMMU_DPROBE1(name, type1, arg1) #define IMMU_DPROBE2(name, type1, arg1, type2, arg2) #define IMMU_DPROBE3(name, type1, arg1, type2, arg2, type3, arg3) #define IMMU_DPROBE4(name, type1, arg1, type2, arg2, type3, arg3, type4, arg4) #endif #ifdef __cplusplus } #endif #endif /* _SYS_INTEL_IOMMU_H */