1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Portions Copyright (c) 2010, Oracle and/or its affiliates. 23 * All rights reserved. 24 */ 25 26 /* 27 * Copyright (c) 2008, Intel Corporation. 28 * All rights reserved. 29 */ 30 31 #ifndef _SYS_INTEL_IOMMU_H 32 #define _SYS_INTEL_IOMMU_H 33 34 /* 35 * Intel IOMMU implementation specific state 36 */ 37 38 #ifdef __cplusplus 39 extern "C" { 40 #endif 41 42 #include <sys/types.h> 43 #include <sys/bitset.h> 44 #include <sys/kstat.h> 45 #include <sys/vmem.h> 46 #include <sys/rootnex.h> 47 48 /* 49 * Some ON drivers have bugs. Keep this define until all such drivers 50 * have been fixed 51 */ 52 #define BUGGY_DRIVERS 1 53 54 /* PD(T)E entries */ 55 typedef uint64_t hw_pdte_t; 56 57 #define IMMU_MAXNAMELEN (64) 58 #define IMMU_MAXSEG (1) 59 #define IMMU_REGSZ (1UL << 12) 60 #define IMMU_PAGESIZE (4096) 61 #define IMMU_PAGESHIFT (12) 62 #define IMMU_PAGEOFFSET (IMMU_PAGESIZE - 1) 63 #define IMMU_PAGEMASK (~IMMU_PAGEOFFSET) 64 #define IMMU_BTOP(b) (((uint64_t)b) >> IMMU_PAGESHIFT) 65 #define IMMU_PTOB(p) (((uint64_t)p) << IMMU_PAGESHIFT) 66 #define IMMU_PGTABLE_MAX_LEVELS (6) 67 #define IMMU_ROUNDUP(size) (((size) + IMMU_PAGEOFFSET) & ~IMMU_PAGEOFFSET) 68 #define IMMU_ROUNDOWN(addr) ((addr) & ~IMMU_PAGEOFFSET) 69 #define IMMU_PGTABLE_LEVEL_STRIDE (9) 70 #define IMMU_PGTABLE_LEVEL_MASK ((1<<IMMU_PGTABLE_LEVEL_STRIDE) - 1) 71 #define IMMU_PGTABLE_OFFSHIFT (IMMU_PAGESHIFT - IMMU_PGTABLE_LEVEL_STRIDE) 72 #define IMMU_PGTABLE_MAXIDX ((IMMU_PAGESIZE / sizeof (hw_pdte_t)) - 1) 73 74 #define IMMU_ROUNDUP(size) (((size) + IMMU_PAGEOFFSET) & ~IMMU_PAGEOFFSET) 75 #define IMMU_ROUNDOWN(addr) ((addr) & ~IMMU_PAGEOFFSET) 76 77 /* 78 * DMAR global defines 79 */ 80 #define DMAR_TABLE "dmar-table" 81 #define DMAR_INTRMAP_SUPPORT (0x01) 82 83 /* DMAR unit types */ 84 #define DMAR_DRHD 0 85 #define DMAR_RMRR 1 86 #define DMAR_ATSR 2 87 #define DMAR_RHSA 3 88 89 /* DRHD flag values */ 90 #define DMAR_INCLUDE_ALL (0x01) 91 92 /* Device scope types */ 93 #define DMAR_ENDPOINT 1 94 #define DMAR_SUBTREE 2 95 #define DMAR_IOAPIC 3 96 #define DMAR_HPET 4 97 98 99 /* Forward declarations for IOMMU state structure and DVMA domain struct */ 100 struct immu; 101 struct domain; 102 103 /* 104 * The following structure describes the formate of DMAR ACPI table format. 105 * They are used to parse DMAR ACPI table. Read the spec for the meaning 106 * of each member. 107 */ 108 109 /* lengths of various strings */ 110 #define DMAR_SIG_LEN (4) /* table signature */ 111 #define DMAR_OEMID_LEN (6) /* OEM ID */ 112 #define DMAR_TBLID_LEN (8) /* OEM table ID */ 113 #define DMAR_ASL_LEN (4) /* ASL len */ 114 115 typedef struct dmar_table { 116 kmutex_t tbl_lock; 117 uint8_t tbl_haw; 118 boolean_t tbl_intrmap; 119 list_t tbl_drhd_list[IMMU_MAXSEG]; 120 list_t tbl_rmrr_list[IMMU_MAXSEG]; 121 char *tbl_oem_id; 122 char *tbl_oem_tblid; 123 uint32_t tbl_oem_rev; 124 caddr_t tbl_raw; 125 int tbl_rawlen; 126 } dmar_table_t; 127 128 typedef struct drhd { 129 kmutex_t dr_lock; /* protects the dmar field */ 130 struct immu *dr_immu; 131 dev_info_t *dr_dip; 132 uint16_t dr_seg; 133 uint64_t dr_regs; 134 boolean_t dr_include_all; 135 list_t dr_scope_list; 136 list_node_t dr_node; 137 } drhd_t; 138 139 typedef struct rmrr { 140 kmutex_t rm_lock; 141 uint16_t rm_seg; 142 uint64_t rm_base; 143 uint64_t rm_limit; 144 list_t rm_scope_list; 145 list_node_t rm_node; 146 } rmrr_t; 147 148 /* 149 * Macros based on PCI spec 150 */ 151 #define IMMU_PCI_DEV(devfunc) ((uint64_t)devfunc >> 3) /* from devfunc */ 152 #define IMMU_PCI_FUNC(devfunc) (devfunc & 7) /* get func from devfunc */ 153 #define IMMU_PCI_DEVFUNC(d, f) (((d) << 3) | (f)) /* create devfunc */ 154 155 typedef struct scope { 156 uint8_t scp_type; 157 uint8_t scp_enumid; 158 uint8_t scp_bus; 159 uint8_t scp_dev; 160 uint8_t scp_func; 161 list_node_t scp_node; 162 } scope_t; 163 164 /* 165 * interrupt source id and drhd info for ioapic 166 */ 167 typedef struct ioapic_drhd { 168 uchar_t ioapic_ioapicid; 169 uint16_t ioapic_sid; /* ioapic source id */ 170 drhd_t *ioapic_drhd; 171 list_node_t ioapic_node; 172 } ioapic_drhd_t; 173 174 typedef struct memrng { 175 uint64_t mrng_start; 176 uint64_t mrng_npages; 177 } memrng_t; 178 179 typedef enum immu_flags { 180 IMMU_FLAGS_NONE = 0x1, 181 IMMU_FLAGS_SLEEP = 0x1, 182 IMMU_FLAGS_NOSLEEP = 0x2, 183 IMMU_FLAGS_READ = 0x4, 184 IMMU_FLAGS_WRITE = 0x8, 185 IMMU_FLAGS_DONTPASS = 0x10, 186 IMMU_FLAGS_ALLOC = 0x20, 187 IMMU_FLAGS_MUST_MATCH = 0x40, 188 IMMU_FLAGS_PAGE1 = 0x80, 189 IMMU_FLAGS_UNITY = 0x100, 190 IMMU_FLAGS_DMAHDL = 0x200, 191 IMMU_FLAGS_MEMRNG = 0x400 192 } immu_flags_t; 193 194 typedef enum cont_avail { 195 IMMU_CONT_BAD = 0x0, 196 IMMU_CONT_UNINITED = 0x1, 197 IMMU_CONT_INITED = 0x2 198 } cont_avail_t; 199 200 /* Size of root and context tables and their entries */ 201 #define IMMU_ROOT_TBLSZ (4096) 202 #define IMMU_CONT_TBLSZ (4096) 203 #define IMMU_ROOT_NUM (256) 204 #define IMMU_CONT_NUM (256) 205 206 /* register offset */ 207 #define IMMU_REG_VERSION (0x00) /* Version Rigister, 32 bit */ 208 #define IMMU_REG_CAP (0x08) /* Capability Register, 64 bit */ 209 #define IMMU_REG_EXCAP (0x10) /* Extended Capability Reg, 64 bit */ 210 #define IMMU_REG_GLOBAL_CMD (0x18) /* Global Command Register, 32 bit */ 211 #define IMMU_REG_GLOBAL_STS (0x1C) /* Global Status Register, 32 bit */ 212 #define IMMU_REG_ROOTENTRY (0x20) /* Root-Entry Table Addr Reg, 64 bit */ 213 #define IMMU_REG_CONTEXT_CMD (0x28) /* Context Comand Register, 64 bit */ 214 #define IMMU_REG_FAULT_STS (0x34) /* Fault Status Register, 32 bit */ 215 #define IMMU_REG_FEVNT_CON (0x38) /* Fault Event Control Reg, 32 bit */ 216 #define IMMU_REG_FEVNT_DATA (0x3C) /* Fault Event Data Register, 32 bit */ 217 #define IMMU_REG_FEVNT_ADDR (0x40) /* Fault Event Address Reg, 32 bit */ 218 #define IMMU_REG_FEVNT_UADDR (0x44) /* Fault Event Upper Addr Reg, 32 bit */ 219 #define IMMU_REG_AFAULT_LOG (0x58) /* Advanced Fault Log Reg, 64 bit */ 220 #define IMMU_REG_PMER (0x64) /* Protected Memory Enble Reg, 32 bit */ 221 #define IMMU_REG_PLMBR (0x68) /* Protected Low Mem Base Reg, 32 bit */ 222 #define IMMU_REG_PLMLR (0x6C) /* Protected Low Mem Lim Reg, 32 bit */ 223 #define IMMU_REG_PHMBR (0X70) /* Protectd High Mem Base Reg, 64 bit */ 224 #define IMMU_REG_PHMLR (0x78) /* Protected High Mem Lim Reg, 64 bit */ 225 #define IMMU_REG_INVAL_QH (0x80) /* Invalidation Queue Head, 64 bit */ 226 #define IMMU_REG_INVAL_QT (0x88) /* Invalidation Queue Tail, 64 bit */ 227 #define IMMU_REG_INVAL_QAR (0x90) /* Invalidtion Queue Addr Reg, 64 bit */ 228 #define IMMU_REG_INVAL_CSR (0x9C) /* Inval Compl Status Reg, 32 bit */ 229 #define IMMU_REG_INVAL_CECR (0xA0) /* Inval Compl Evnt Ctrl Reg, 32 bit */ 230 #define IMMU_REG_INVAL_CEDR (0xA4) /* Inval Compl Evnt Data Reg, 32 bit */ 231 #define IMMU_REG_INVAL_CEAR (0xA8) /* Inval Compl Event Addr Reg, 32 bit */ 232 #define IMMU_REG_INVAL_CEUAR (0xAC) /* Inval Comp Evnt Up Addr reg, 32bit */ 233 #define IMMU_REG_IRTAR (0xB8) /* INTR Remap Tbl Addr Reg, 64 bit */ 234 235 /* ioapic memory region */ 236 #define IOAPIC_REGION_START (0xfee00000) 237 #define IOAPIC_REGION_END (0xfeefffff) 238 239 /* fault register */ 240 #define IMMU_FAULT_STS_PPF (2) 241 #define IMMU_FAULT_STS_PFO (1) 242 #define IMMU_FAULT_STS_ITE (1 << 6) 243 #define IMMU_FAULT_STS_ICE (1 << 5) 244 #define IMMU_FAULT_STS_IQE (1 << 4) 245 #define IMMU_FAULT_GET_INDEX(x) ((((uint64_t)x) >> 8) & 0xff) 246 #define IMMU_FRR_GET_F(x) (((uint64_t)x) >> 63) 247 #define IMMU_FRR_GET_FR(x) ((((uint64_t)x) >> 32) & 0xff) 248 #define IMMU_FRR_GET_FT(x) ((((uint64_t)x) >> 62) & 0x1) 249 #define IMMU_FRR_GET_SID(x) ((x) & 0xffff) 250 251 /* (ex)capability register */ 252 #define IMMU_CAP_GET_NFR(x) (((((uint64_t)x) >> 40) & 0xff) + 1) 253 #define IMMU_CAP_GET_DWD(x) ((((uint64_t)x) >> 54) & 1) 254 #define IMMU_CAP_GET_DRD(x) ((((uint64_t)x) >> 55) & 1) 255 #define IMMU_CAP_GET_PSI(x) ((((uint64_t)x) >> 39) & 1) 256 #define IMMU_CAP_GET_SPS(x) ((((uint64_t)x) >> 34) & 0xf) 257 #define IMMU_CAP_GET_ISOCH(x) ((((uint64_t)x) >> 23) & 1) 258 #define IMMU_CAP_GET_ZLR(x) ((((uint64_t)x) >> 22) & 1) 259 #define IMMU_CAP_GET_MAMV(x) ((((uint64_t)x) >> 48) & 0x3f) 260 #define IMMU_CAP_GET_CM(x) ((((uint64_t)x) >> 7) & 1) 261 #define IMMU_CAP_GET_PHMR(x) ((((uint64_t)x) >> 6) & 1) 262 #define IMMU_CAP_GET_PLMR(x) ((((uint64_t)x) >> 5) & 1) 263 #define IMMU_CAP_GET_RWBF(x) ((((uint64_t)x) >> 4) & 1) 264 #define IMMU_CAP_GET_AFL(x) ((((uint64_t)x) >> 3) & 1) 265 #define IMMU_CAP_GET_FRO(x) (((((uint64_t)x) >> 24) & 0x3ff) * 16) 266 #define IMMU_CAP_MGAW(x) (((((uint64_t)x) >> 16) & 0x3f) + 1) 267 #define IMMU_CAP_SAGAW(x) ((((uint64_t)x) >> 8) & 0x1f) 268 #define IMMU_CAP_ND(x) (1 << (((x) & 0x7) *2 + 4)) -1 269 #define IMMU_ECAP_GET_IRO(x) (((((uint64_t)x) >> 8) & 0x3ff) << 4) 270 #define IMMU_ECAP_GET_MHMV(x) (((uint64_t)x >> 20) & 0xf) 271 #define IMMU_ECAP_GET_SC(x) ((x) & 0x80) 272 #define IMMU_ECAP_GET_PT(x) ((x) & 0x40) 273 #define IMMU_ECAP_GET_CH(x) ((x) & 0x20) 274 #define IMMU_ECAP_GET_EIM(x) ((x) & 0x10) 275 #define IMMU_ECAP_GET_IR(x) ((x) & 0x8) 276 #define IMMU_ECAP_GET_DI(x) ((x) & 0x4) 277 #define IMMU_ECAP_GET_QI(x) ((x) & 0x2) 278 #define IMMU_ECAP_GET_C(x) ((x) & 0x1) 279 280 #define IMMU_CAP_SET_RWBF(x) ((x) |= (1 << 4)) 281 282 283 /* iotlb invalidation */ 284 #define TLB_INV_GLOBAL (((uint64_t)1) << 60) 285 #define TLB_INV_DOMAIN (((uint64_t)2) << 60) 286 #define TLB_INV_PAGE (((uint64_t)3) << 60) 287 #define TLB_INV_GET_IAIG(x) ((((uint64_t)x) >> 57) & 7) 288 #define TLB_INV_DRAIN_READ (((uint64_t)1) << 49) 289 #define TLB_INV_DRAIN_WRITE (((uint64_t)1) << 48) 290 #define TLB_INV_DID(x) (((uint64_t)((x) & 0xffff)) << 32) 291 #define TLB_INV_IVT (((uint64_t)1) << 63) 292 #define TLB_IVA_HINT(x) (((x) & 0x1) << 6) 293 #define TLB_IVA_LEAF 1 294 #define TLB_IVA_WHOLE 0 295 296 /* dont use value 0 for enums - to catch unit 8 */ 297 typedef enum iotlb_inv { 298 IOTLB_PSI = 1, 299 IOTLB_DSI, 300 IOTLB_GLOBAL 301 } immu_iotlb_inv_t; 302 303 typedef enum context_inv { 304 CONTEXT_FSI = 1, 305 CONTEXT_DSI, 306 CONTEXT_GLOBAL 307 } immu_context_inv_t; 308 309 /* context invalidation */ 310 #define CCMD_INV_ICC (((uint64_t)1) << 63) 311 #define CCMD_INV_GLOBAL (((uint64_t)1) << 61) 312 #define CCMD_INV_DOMAIN (((uint64_t)2) << 61) 313 #define CCMD_INV_DEVICE (((uint64_t)3) << 61) 314 #define CCMD_INV_DID(x) ((uint64_t)((x) & 0xffff)) 315 #define CCMD_INV_SID(x) (((uint64_t)((x) & 0xffff)) << 16) 316 #define CCMD_INV_FM(x) (((uint64_t)((x) & 0x3)) << 32) 317 318 /* global command register */ 319 #define IMMU_GCMD_TE (((uint32_t)1) << 31) 320 #define IMMU_GCMD_SRTP (((uint32_t)1) << 30) 321 #define IMMU_GCMD_SFL (((uint32_t)1) << 29) 322 #define IMMU_GCMD_EAFL (((uint32_t)1) << 28) 323 #define IMMU_GCMD_WBF (((uint32_t)1) << 27) 324 #define IMMU_GCMD_QIE (((uint32_t)1) << 26) 325 #define IMMU_GCMD_IRE (((uint32_t)1) << 25) 326 #define IMMU_GCMD_SIRTP (((uint32_t)1) << 24) 327 #define IMMU_GCMD_CFI (((uint32_t)1) << 23) 328 329 /* global status register */ 330 #define IMMU_GSTS_TES (((uint32_t)1) << 31) 331 #define IMMU_GSTS_RTPS (((uint32_t)1) << 30) 332 #define IMMU_GSTS_FLS (((uint32_t)1) << 29) 333 #define IMMU_GSTS_AFLS (((uint32_t)1) << 28) 334 #define IMMU_GSTS_WBFS (((uint32_t)1) << 27) 335 #define IMMU_GSTS_QIES (((uint32_t)1) << 26) 336 #define IMMU_GSTS_IRES (((uint32_t)1) << 25) 337 #define IMMU_GSTS_IRTPS (((uint32_t)1) << 24) 338 #define IMMU_GSTS_CFIS (((uint32_t)1) << 23) 339 340 /* psi address mask */ 341 #define ADDR_AM_MAX(m) (((uint_t)1) << (m)) 342 #define ADDR_AM_OFFSET(n, m) ((n) & (ADDR_AM_MAX(m) - 1)) 343 344 /* dmar fault event */ 345 #define IMMU_INTR_IPL (4) 346 #define IMMU_REG_FEVNT_CON_IM_SHIFT (31) 347 348 #define IMMU_ALLOC_RESOURCE_DELAY (drv_usectohz(5000)) 349 350 /* max value of Size field of Interrupt Remapping Table Address Register */ 351 #define INTRMAP_MAX_IRTA_SIZE 0xf 352 353 /* interrupt remapping table entry size */ 354 #define INTRMAP_RTE_SIZE 0x10 355 356 /* ioapic redirection table entry related shift of remappable interrupt */ 357 #define INTRMAP_IOAPIC_IDX_SHIFT 17 358 #define INTRMAP_IOAPIC_FORMAT_SHIFT 16 359 #define INTRMAP_IOAPIC_TM_SHIFT 15 360 #define INTRMAP_IOAPIC_POL_SHIFT 13 361 #define INTRMAP_IOAPIC_IDX15_SHIFT 11 362 363 /* msi intr entry related shift of remappable interrupt */ 364 #define INTRMAP_MSI_IDX_SHIFT 5 365 #define INTRMAP_MSI_FORMAT_SHIFT 4 366 #define INTRMAP_MSI_SHV_SHIFT 3 367 #define INTRMAP_MSI_IDX15_SHIFT 2 368 369 #define INTRMAP_IDX_FULL (uint_t)-1 370 371 #define RDT_DLM(rdt) BITX((rdt), 10, 8) 372 #define RDT_DM(rdt) BT_TEST(&(rdt), 11) 373 #define RDT_POL(rdt) BT_TEST(&(rdt), 13) 374 #define RDT_TM(rdt) BT_TEST(&(rdt), 15) 375 376 #define INTRMAP_DISABLE (void *)-1 377 378 /* 379 * invalidation granularity 380 */ 381 typedef enum { 382 TLB_INV_G_GLOBAL = 1, 383 TLB_INV_G_DOMAIN, 384 TLB_INV_G_PAGE 385 } tlb_inv_g_t; 386 387 typedef enum { 388 CTT_INV_G_GLOBAL = 1, 389 CTT_INV_G_DOMAIN, 390 CTT_INV_G_DEVICE 391 } ctt_inv_g_t; 392 393 typedef enum { 394 IEC_INV_GLOBAL = 0, 395 IEC_INV_INDEX 396 } iec_inv_g_t; 397 398 399 struct inv_queue_state; 400 struct intrmap_tbl_state; 401 402 /* A software page table structure */ 403 typedef struct pgtable { 404 krwlock_t swpg_rwlock; 405 caddr_t hwpg_vaddr; /* HW pgtable VA */ 406 paddr_t hwpg_paddr; /* HW pgtable PA */ 407 ddi_dma_handle_t hwpg_dmahdl; 408 ddi_acc_handle_t hwpg_memhdl; 409 struct pgtable **swpg_next_array; 410 list_node_t swpg_domain_node; /* domain list of pgtables */ 411 } pgtable_t; 412 413 /* interrupt remapping table state info */ 414 typedef struct intrmap { 415 kmutex_t intrmap_lock; 416 ddi_dma_handle_t intrmap_dma_hdl; 417 ddi_acc_handle_t intrmap_acc_hdl; 418 caddr_t intrmap_vaddr; 419 paddr_t intrmap_paddr; 420 uint_t intrmap_size; 421 bitset_t intrmap_map; 422 uint_t intrmap_free; 423 } intrmap_t; 424 425 typedef struct hw_rce { 426 uint64_t lo; 427 uint64_t hi; 428 } hw_rce_t; 429 430 431 #define ROOT_GET_P(hrent) ((hrent)->lo & 0x1) 432 #define ROOT_SET_P(hrent) ((hrent)->lo |= 0x1) 433 434 #define ROOT_GET_CONT(hrent) ((hrent)->lo & ~(0xFFF)) 435 #define ROOT_SET_CONT(hrent, paddr) ((hrent)->lo |= (paddr & (~0xFFF))) 436 437 #define TTYPE_XLATE_ONLY (0x0) 438 #define TTYPE_XLATE_IOTLB (0x1) 439 #define TTYPE_PASSTHRU (0x2) 440 #define TTYPE_RESERVED (0x3) 441 442 #define CONT_GET_DID(hcent) ((((uint64_t)(hcent)->hi) >> 8) & 0xFFFF) 443 #define CONT_SET_DID(hcent, did) ((hcent)->hi |= ((0xFFFF & (did)) << 8)) 444 445 #define CONT_GET_AVAIL(hcent) ((((uint64_t)((hcent)->hi)) >> 0x3) & 0xF) 446 #define CONT_SET_AVAIL(hcent, av) ((hcent)->hi |= ((0xF & (av)) << 0x3)) 447 448 #define CONT_GET_LO_AW(hcent) (30 + 9 *((hcent)->hi & 0x7)) 449 #define CONT_GET_AW(hcent) \ 450 ((CONT_GET_LO_AW(hcent) == 66) ? 64 : CONT_GET_LO_AW(hcent)) 451 #define CONT_SET_AW(hcent, aw) \ 452 ((hcent)->hi |= (((((aw) + 2) - 30) / 9) & 0x7)) 453 454 #define CONT_GET_ASR(hcent) ((hcent)->lo & ~(0xFFF)) 455 #define CONT_SET_ASR(hcent, paddr) ((hcent)->lo |= (paddr & (~0xFFF))) 456 457 #define CONT_GET_TTYPE(hcent) ((((uint64_t)(hcent)->lo) >> 0x2) & 0x3) 458 #define CONT_SET_TTYPE(hcent, ttype) ((hcent)->lo |= (((ttype) & 0x3) << 0x2)) 459 460 #define CONT_GET_P(hcent) ((hcent)->lo & 0x1) 461 #define CONT_SET_P(hcent) ((hcent)->lo |= 0x1) 462 463 464 /* we use the bit 63 (available for system SW) as a present bit */ 465 #define PDTE_SW4(hw_pdte) ((hw_pdte) & ((uint64_t)1<<63)) 466 #define PDTE_CLEAR_SW4(hw_pdte) ((hw_pdte) &= ~((uint64_t)1<<63)) 467 468 #define PDTE_P(hw_pdte) ((hw_pdte) & ((uint64_t)1<<63)) 469 #define PDTE_CLEAR_P(hw_pdte) ((hw_pdte) &= ~((uint64_t)1<<63)) 470 #define PDTE_SET_P(hw_pdte) ((hw_pdte) |= ((uint64_t)1<<63)) 471 472 #define PDTE_TM(hw_pdte) ((hw_pdte) & ((uint64_t)1<<62)) 473 #define PDTE_CLEAR_TM(hw_pdte) ((hw_pdte) &= ~((uint64_t)1<<62)) 474 475 #define PDTE_SW3(hw_pdte) \ 476 (((hw_pdte) & ~(((uint64_t)0x3<<62)|(((uint64_t)1<<52)-1))) >> 52) 477 #define PDTE_SW3_OVERFLOW(hw_pdte) \ 478 (PDTE_SW3(hw_pdte) == 0x3FF) 479 #define PDTE_CLEAR_SW3(hw_pdte) \ 480 ((hw_pdte) &= (((uint64_t)0x3<<62)|(((uint64_t)1<<52)-1))) 481 #define PDTE_SET_SW3(hw_pdte, ref) \ 482 ((hw_pdte) |= ((((uint64_t)(ref)) & 0x3FF) << 52)) 483 484 #define PDTE_PADDR(hw_pdte) ((hw_pdte) & ~(((uint64_t)0xFFF<<52)|((1<<12)-1))) 485 #define PDTE_CLEAR_PADDR(hw_pdte) \ 486 ((hw_pdte) &= (((uint64_t)0xFFF<<52)|((1<<12)-1))) 487 #define PDTE_SET_PADDR(hw_pdte, paddr) ((hw_pdte) |= PDTE_PADDR(paddr)) 488 489 #define PDTE_SNP(hw_pdte) ((hw_pdte) & (1<<11)) 490 #define PDTE_CLEAR_SNP(hw_pdte) ((hw_pdte) &= ~(1<<11)) 491 #define PDTE_SET_SNP(hw_pdte) ((hw_pdte) |= (1<<11)) 492 493 #define PDTE_SW2(hw_pdte) ((hw_pdte) & (0x700)) 494 #define PDTE_CLEAR_SW2(hw_pdte) ((hw_pdte) &= ~(0x700)) 495 496 #define PDTE_SP(hw_pdte) ((hw_pdte) & (0x80)) 497 #define PDTE_CLEAR_SP(hw_pdte) ((hw_pdte) &= ~(0x80)) 498 499 #define PDTE_SW1(hw_pdte) ((hw_pdte) & (0x7C)) 500 #define PDTE_CLEAR_SW1(hw_pdte) ((hw_pdte) &= ~(0x7C)) 501 502 #define PDTE_WRITE(hw_pdte) ((hw_pdte) & (0x2)) 503 #define PDTE_CLEAR_WRITE(hw_pdte) ((hw_pdte) &= ~(0x2)) 504 #define PDTE_SET_WRITE(hw_pdte) ((hw_pdte) |= (0x2)) 505 506 #define PDTE_READ(hw_pdte) ((hw_pdte) & (0x1)) 507 #define PDTE_CLEAR_READ(hw_pdte) ((hw_pdte) &= ~(0x1)) 508 #define PDTE_SET_READ(hw_pdte) ((hw_pdte) |= (0x1)) 509 510 struct immu_flushops; 511 512 typedef struct immu { 513 kmutex_t immu_lock; 514 char *immu_name; 515 516 /* lock grabbed by interrupt handler */ 517 kmutex_t immu_intr_lock; 518 519 /* ACPI/DMAR table related */ 520 void *immu_dmar_unit; 521 dev_info_t *immu_dip; 522 struct domain *immu_unity_domain; 523 524 /* IOMMU register related */ 525 kmutex_t immu_regs_lock; 526 kcondvar_t immu_regs_cv; 527 boolean_t immu_regs_busy; 528 boolean_t immu_regs_setup; 529 boolean_t immu_regs_running; 530 boolean_t immu_regs_quiesced; 531 ddi_acc_handle_t immu_regs_handle; 532 caddr_t immu_regs_addr; 533 uint64_t immu_regs_cap; 534 uint64_t immu_regs_excap; 535 uint32_t immu_regs_cmdval; 536 uint32_t immu_regs_intr_msi_addr; 537 uint32_t immu_regs_intr_msi_data; 538 uint32_t immu_regs_intr_uaddr; 539 540 /* DVMA related */ 541 kmutex_t immu_dvma_lock; 542 boolean_t immu_dvma_setup; 543 boolean_t immu_dvma_running; 544 int immu_dvma_gaw; 545 int immu_dvma_agaw; 546 int immu_dvma_nlevels; 547 boolean_t immu_dvma_coherent; 548 boolean_t immu_TM_reserved; 549 boolean_t immu_SNP_reserved; 550 551 /* DVMA context related */ 552 krwlock_t immu_ctx_rwlock; 553 pgtable_t *immu_ctx_root; 554 555 /* DVMA domain related */ 556 int immu_max_domains; 557 vmem_t *immu_did_arena; 558 char immu_did_arena_name[IMMU_MAXNAMELEN]; 559 list_t immu_domain_list; 560 561 /* DVMA special devices */ 562 boolean_t immu_dvma_gfx_only; 563 list_t immu_dvma_lpc_list; 564 list_t immu_dvma_gfx_list; 565 566 /* interrupt remapping related */ 567 kmutex_t immu_intrmap_lock; 568 boolean_t immu_intrmap_setup; 569 boolean_t immu_intrmap_running; 570 intrmap_t *immu_intrmap; 571 uint64_t immu_intrmap_irta_reg; 572 573 /* queued invalidation related */ 574 kmutex_t immu_qinv_lock; 575 boolean_t immu_qinv_setup; 576 boolean_t immu_qinv_running; 577 boolean_t immu_qinv_enabled; 578 void *immu_qinv; 579 uint64_t immu_qinv_reg_value; 580 581 /* list_node for system-wide list of DMAR units */ 582 list_node_t immu_node; 583 584 struct immu_flushops *immu_flushops; 585 } immu_t; 586 587 /* properties that control DVMA */ 588 #define DDI_DVMA_MAPTYPE_ROOTNEX_PROP "immu-dvma-mapping" 589 590 #define DDI_DVMA_MAPTYPE_UNITY "unity" 591 #define DDI_DVMA_MAPTYPE_XLATE "xlate" 592 593 typedef enum immu_maptype { 594 IMMU_MAPTYPE_BAD = 0, /* 0 is always bad */ 595 IMMU_MAPTYPE_UNITY = 1, 596 IMMU_MAPTYPE_XLATE 597 } immu_maptype_t; 598 599 #define IMMU_COOKIE_HASHSZ (512) 600 601 /* 602 * domain_t 603 * 604 */ 605 typedef struct domain { 606 /* the basics */ 607 uint_t dom_did; 608 immu_t *dom_immu; 609 610 /* mapping related */ 611 immu_maptype_t dom_maptype; 612 vmem_t *dom_dvma_arena; 613 char dom_dvma_arena_name[IMMU_MAXNAMELEN]; 614 615 /* pgtables */ 616 pgtable_t *dom_pgtable_root; 617 krwlock_t dom_pgtable_rwlock; 618 619 /* list node for list of domains (unity or xlate) */ 620 list_node_t dom_maptype_node; 621 /* list node for list of domains off immu */ 622 list_node_t dom_immu_node; 623 624 mod_hash_t *dom_cookie_hash; 625 } domain_t; 626 627 typedef enum immu_pcib { 628 IMMU_PCIB_BAD = 0, 629 IMMU_PCIB_NOBDF, 630 IMMU_PCIB_PCIE_PCIE, 631 IMMU_PCIB_PCIE_PCI, 632 IMMU_PCIB_PCI_PCI, 633 IMMU_PCIB_ENDPOINT 634 } immu_pcib_t; 635 636 /* 637 * immu_devi_t 638 * Intel IOMMU in devinfo node 639 */ 640 typedef struct immu_devi { 641 /* pci seg, bus, dev, func */ 642 int imd_seg; 643 int imd_bus; 644 int imd_devfunc; 645 646 /* ppb information */ 647 immu_pcib_t imd_pcib_type; 648 int imd_sec; 649 int imd_sub; 650 651 /* identifier for special devices */ 652 boolean_t imd_display; 653 boolean_t imd_lpc; 654 655 /* dmar unit to which this dip belongs */ 656 immu_t *imd_immu; 657 658 immu_flags_t imd_dvma_flags; 659 660 /* domain ptr */ 661 domain_t *imd_domain; 662 dev_info_t *imd_ddip; 663 664 /* my devinfo */ 665 dev_info_t *imd_dip; 666 667 /* 668 * if we are a "special" devinfo 669 * the node for the special linked list 670 * off the DMAR unit structure 671 */ 672 list_node_t imd_spc_node; 673 } immu_devi_t; 674 675 #define IMMU_DEVI(dip) ((immu_devi_t *)(DEVI(dip)->devi_iommu)) 676 #define IMMU_DEVI_SET(dip, imd) (DEVI(dip)->devi_iommu = (void *)imd) 677 678 /* 679 * struct dmar_arg 680 */ 681 typedef struct immu_arg { 682 int ima_seg; 683 int ima_bus; 684 int ima_devfunc; 685 dev_info_t *ima_rdip; 686 dev_info_t *ima_ddip; 687 } immu_arg_t; 688 689 /* 690 * Invalidation operation function pointers for context and IOTLB. 691 * These will be set to either the register or the queue invalidation 692 * interface functions, since the hardware does not allow using them 693 * both at the same time. 694 */ 695 struct immu_flushops { 696 void (*imf_context_fsi)(immu_t *, uint8_t, uint16_t, uint_t); 697 void (*imf_context_dsi)(immu_t *, uint_t); 698 void (*imf_context_gbl)(immu_t *); 699 700 void (*imf_iotlb_psi)(immu_t *, uint_t, uint64_t, uint_t, uint_t); 701 void (*imf_iotlb_dsi)(immu_t *, uint_t); 702 void (*imf_iotlb_gbl)(immu_t *); 703 }; 704 705 #define immu_flush_context_fsi(i, f, s, d) \ 706 (i)->immu_flushops->imf_context_fsi(i, f, s, d) 707 #define immu_flush_context_dsi(i, d) \ 708 (i)->immu_flushops->imf_context_dsi(i, d) 709 #define immu_flush_context_gbl(i) \ 710 (i)->immu_flushops->imf_context_gbl(i) 711 712 #define immu_flush_iotlb_psi(i, d, v, c, h) \ 713 (i)->immu_flushops->imf_iotlb_psi(i, d, v, c, h) 714 #define immu_flush_iotlb_dsi(i, d) \ 715 (i)->immu_flushops->imf_iotlb_dsi(i, d) 716 #define immu_flush_iotlb_gbl(i) \ 717 (i)->immu_flushops->imf_iotlb_gbl(i) 718 719 /* 720 * Globals used by IOMMU code 721 */ 722 /* shared between IOMMU files */ 723 extern dev_info_t *root_devinfo; 724 extern kmutex_t immu_lock; 725 extern list_t immu_list; 726 extern void *immu_pgtable_cache; 727 extern boolean_t immu_setup; 728 extern boolean_t immu_running; 729 extern kmutex_t ioapic_drhd_lock; 730 extern list_t ioapic_drhd_list; 731 732 /* switches */ 733 734 /* Various features */ 735 extern boolean_t immu_enable; 736 extern boolean_t immu_gfxdvma_enable; 737 extern boolean_t immu_intrmap_enable; 738 extern boolean_t immu_qinv_enable; 739 740 /* various quirks that need working around */ 741 extern boolean_t immu_quirk_usbpage0; 742 extern boolean_t immu_quirk_usbfullpa; 743 extern boolean_t immu_quirk_usbrmrr; 744 extern boolean_t immu_quirk_mobile4; 745 746 /* debug messages */ 747 extern boolean_t immu_dmar_print; 748 749 /* tunables */ 750 extern int64_t immu_flush_gran; 751 752 extern immu_flags_t immu_global_dvma_flags; 753 754 /* ################### Interfaces exported outside IOMMU code ############## */ 755 void immu_init(void); 756 void immu_startup(void); 757 void immu_shutdown(void); 758 void immu_destroy(void); 759 int immu_map_sgl(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 760 int prealloc_count, dev_info_t *rdip); 761 int immu_unmap_sgl(ddi_dma_impl_t *hp, dev_info_t *rdip); 762 void immu_device_tree_changed(void); 763 void immu_physmem_update(uint64_t addr, uint64_t size); 764 int immu_quiesce(void); 765 int immu_unquiesce(void); 766 /* ######################################################################### */ 767 768 /* ################# Interfaces used within IOMMU code #################### */ 769 770 /* functions in rootnex.c */ 771 int rootnex_dvcookies_alloc(ddi_dma_impl_t *hp, 772 struct ddi_dma_req *dmareq, dev_info_t *rdip, void *arg); 773 void rootnex_dvcookies_free(dvcookie_t *dvcookies, void *arg); 774 775 /* immu_dmar.c interfaces */ 776 int immu_dmar_setup(void); 777 int immu_dmar_parse(void); 778 void immu_dmar_startup(void); 779 void immu_dmar_shutdown(void); 780 void immu_dmar_destroy(void); 781 boolean_t immu_dmar_blacklisted(char **strings_array, uint_t nstrings); 782 immu_t *immu_dmar_get_immu(dev_info_t *rdip); 783 char *immu_dmar_unit_name(void *dmar_unit); 784 dev_info_t *immu_dmar_unit_dip(void *dmar_unit); 785 void immu_dmar_set_immu(void *dmar_unit, immu_t *immu); 786 void *immu_dmar_walk_units(int seg, void *dmar_unit); 787 boolean_t immu_dmar_intrmap_supported(void); 788 uint16_t immu_dmar_ioapic_sid(int ioapicid); 789 immu_t *immu_dmar_ioapic_immu(int ioapicid); 790 void immu_dmar_rmrr_map(void); 791 792 /* immu.c interfaces */ 793 int immu_walk_ancestor(dev_info_t *rdip, dev_info_t *ddip, 794 int (*func)(dev_info_t *, void *arg), void *arg, 795 int *level, immu_flags_t immu_flags); 796 797 /* immu_regs.c interfaces */ 798 void immu_regs_setup(list_t *immu_list); 799 void immu_regs_startup(immu_t *immu); 800 int immu_regs_resume(immu_t *immu); 801 void immu_regs_suspend(immu_t *immu); 802 void immu_regs_shutdown(immu_t *immu); 803 void immu_regs_destroy(list_t *immu_list); 804 805 void immu_regs_intr(immu_t *immu, uint32_t msi_addr, uint32_t msi_data, 806 uint32_t uaddr); 807 808 boolean_t immu_regs_passthru_supported(immu_t *immu); 809 boolean_t immu_regs_is_TM_reserved(immu_t *immu); 810 boolean_t immu_regs_is_SNP_reserved(immu_t *immu); 811 812 void immu_regs_wbf_flush(immu_t *immu); 813 void immu_regs_cpu_flush(immu_t *immu, caddr_t addr, uint_t size); 814 815 void immu_regs_context_fsi(immu_t *immu, uint8_t function_mask, 816 uint16_t source_id, uint_t domain_id); 817 void immu_regs_context_dsi(immu_t *immu, uint_t domain_id); 818 void immu_regs_context_gbl(immu_t *immu); 819 void immu_regs_iotlb_psi(immu_t *immu, uint_t domain_id, 820 uint64_t dvma, uint_t count, uint_t hint); 821 void immu_regs_iotlb_dsi(immu_t *immu, uint_t domain_id); 822 void immu_regs_iotlb_gbl(immu_t *immu); 823 824 void immu_regs_set_root_table(immu_t *immu); 825 void immu_regs_qinv_enable(immu_t *immu, uint64_t qinv_reg_value); 826 void immu_regs_intr_enable(immu_t *immu, uint32_t msi_addr, uint32_t msi_data, 827 uint32_t uaddr); 828 void immu_regs_intrmap_enable(immu_t *immu, uint64_t irta_reg); 829 uint64_t immu_regs_get64(immu_t *immu, uint_t reg); 830 void immu_regs_put64(immu_t *immu, uint_t reg, uint64_t val); 831 uint32_t immu_regs_get32(immu_t *immu, uint_t reg); 832 void immu_regs_put32(immu_t *immu, uint_t reg, uint32_t val); 833 834 /* immu_dvma.c interfaces */ 835 void immu_dvma_setup(list_t *immu_list); 836 void immu_dvma_startup(immu_t *immu); 837 void immu_dvma_shutdown(immu_t *immu); 838 void immu_dvma_destroy(list_t *immu_list); 839 840 void immu_dvma_physmem_update(uint64_t addr, uint64_t size); 841 int immu_dvma_map(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, memrng_t *, 842 uint_t prealloc_count, dev_info_t *rdip, immu_flags_t immu_flags); 843 int immu_dvma_unmap(ddi_dma_impl_t *hp, dev_info_t *rdip); 844 int immu_dvma_alloc(dvcookie_t *first_dvcookie, void *arg); 845 void immu_dvma_free(dvcookie_t *first_dvcookie, void *arg); 846 int immu_devi_set(dev_info_t *dip, immu_flags_t immu_flags); 847 immu_devi_t *immu_devi_get(dev_info_t *dip); 848 immu_t *immu_dvma_get_immu(dev_info_t *dip, immu_flags_t immu_flags); 849 int pgtable_ctor(void *buf, void *arg, int kmflag); 850 void pgtable_dtor(void *buf, void *arg); 851 852 /* immu_intrmap.c interfaces */ 853 void immu_intrmap_setup(list_t *immu_list); 854 void immu_intrmap_startup(immu_t *immu); 855 void immu_intrmap_shutdown(immu_t *immu); 856 void immu_intrmap_destroy(list_t *immu_list); 857 858 /* registers interrupt handler for IOMMU unit */ 859 void immu_intr_register(immu_t *immu); 860 int immu_intr_handler(immu_t *immu); 861 862 863 /* immu_qinv.c interfaces */ 864 int immu_qinv_setup(list_t *immu_list); 865 void immu_qinv_startup(immu_t *immu); 866 void immu_qinv_shutdown(immu_t *immu); 867 void immu_qinv_destroy(list_t *immu_list); 868 869 void immu_qinv_context_fsi(immu_t *immu, uint8_t function_mask, 870 uint16_t source_id, uint_t domain_id); 871 void immu_qinv_context_dsi(immu_t *immu, uint_t domain_id); 872 void immu_qinv_context_gbl(immu_t *immu); 873 void immu_qinv_iotlb_psi(immu_t *immu, uint_t domain_id, 874 uint64_t dvma, uint_t count, uint_t hint); 875 void immu_qinv_iotlb_dsi(immu_t *immu, uint_t domain_id); 876 void immu_qinv_iotlb_gbl(immu_t *immu); 877 878 void immu_qinv_intr_global(immu_t *immu); 879 void immu_qinv_intr_one_cache(immu_t *immu, uint_t idx); 880 void immu_qinv_intr_caches(immu_t *immu, uint_t idx, uint_t cnt); 881 void immu_qinv_report_fault(immu_t *immu); 882 883 884 #ifdef __cplusplus 885 } 886 #endif 887 888 #endif /* _SYS_INTEL_IOMMU_H */ 889