1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Portions Copyright (c) 2010, Oracle and/or its affiliates. 23 * All rights reserved. 24 */ 25 26 /* 27 * Copyright (c) 2008, Intel Corporation. 28 * All rights reserved. 29 */ 30 31 #ifndef _SYS_INTEL_IOMMU_H 32 #define _SYS_INTEL_IOMMU_H 33 34 /* 35 * Intel IOMMU implementation specific state 36 */ 37 38 #ifdef __cplusplus 39 extern "C" { 40 #endif 41 42 #include <sys/types.h> 43 #include <sys/bitset.h> 44 #include <sys/kstat.h> 45 #include <sys/kmem.h> 46 #include <sys/vmem.h> 47 #include <sys/rootnex.h> 48 #include <sys/iommulib.h> 49 #include <sys/sdt.h> 50 51 /* 52 * Some ON drivers have bugs. Keep this define until all such drivers 53 * have been fixed 54 */ 55 #define BUGGY_DRIVERS 1 56 57 /* PD(T)E entries */ 58 typedef uint64_t hw_pdte_t; 59 60 #define IMMU_MAXNAMELEN (64) 61 #define IMMU_MAXSEG (1) 62 #define IMMU_REGSZ (1UL << 12) 63 #define IMMU_PAGESIZE (4096) 64 #define IMMU_PAGESHIFT (12) 65 #define IMMU_PAGEOFFSET (IMMU_PAGESIZE - 1) 66 #define IMMU_PAGEMASK (~IMMU_PAGEOFFSET) 67 #define IMMU_BTOP(b) (((uint64_t)b) >> IMMU_PAGESHIFT) 68 #define IMMU_PTOB(p) (((uint64_t)p) << IMMU_PAGESHIFT) 69 #define IMMU_BTOPR(x) ((((x) + IMMU_PAGEOFFSET) >> IMMU_PAGESHIFT)) 70 #define IMMU_PGTABLE_MAX_LEVELS (6) 71 #define IMMU_ROUNDUP(size) (((size) + IMMU_PAGEOFFSET) & ~IMMU_PAGEOFFSET) 72 #define IMMU_ROUNDOWN(addr) ((addr) & ~IMMU_PAGEOFFSET) 73 #define IMMU_PGTABLE_LEVEL_STRIDE (9) 74 #define IMMU_PGTABLE_LEVEL_MASK ((1<<IMMU_PGTABLE_LEVEL_STRIDE) - 1) 75 #define IMMU_PGTABLE_OFFSHIFT (IMMU_PAGESHIFT - IMMU_PGTABLE_LEVEL_STRIDE) 76 #define IMMU_PGTABLE_MAXIDX ((IMMU_PAGESIZE / sizeof (hw_pdte_t)) - 1) 77 78 /* 79 * DMAR global defines 80 */ 81 #define DMAR_TABLE "dmar-table" 82 #define DMAR_INTRMAP_SUPPORT (0x01) 83 84 /* DMAR unit types */ 85 #define DMAR_DRHD 0 86 #define DMAR_RMRR 1 87 #define DMAR_ATSR 2 88 #define DMAR_RHSA 3 89 90 /* DRHD flag values */ 91 #define DMAR_INCLUDE_ALL (0x01) 92 93 /* Device scope types */ 94 #define DMAR_ENDPOINT 1 95 #define DMAR_SUBTREE 2 96 #define DMAR_IOAPIC 3 97 #define DMAR_HPET 4 98 99 100 /* Forward declarations for IOMMU state structure and DVMA domain struct */ 101 struct immu; 102 struct domain; 103 104 /* 105 * The following structure describes the formate of DMAR ACPI table format. 106 * They are used to parse DMAR ACPI table. Read the spec for the meaning 107 * of each member. 108 */ 109 110 /* lengths of various strings */ 111 #define DMAR_SIG_LEN (4) /* table signature */ 112 #define DMAR_OEMID_LEN (6) /* OEM ID */ 113 #define DMAR_TBLID_LEN (8) /* OEM table ID */ 114 #define DMAR_ASL_LEN (4) /* ASL len */ 115 116 typedef struct dmar_table { 117 kmutex_t tbl_lock; 118 uint8_t tbl_haw; 119 boolean_t tbl_intrmap; 120 list_t tbl_drhd_list[IMMU_MAXSEG]; 121 list_t tbl_rmrr_list[IMMU_MAXSEG]; 122 char *tbl_oem_id; 123 char *tbl_oem_tblid; 124 uint32_t tbl_oem_rev; 125 caddr_t tbl_raw; 126 int tbl_rawlen; 127 } dmar_table_t; 128 129 typedef struct drhd { 130 kmutex_t dr_lock; /* protects the dmar field */ 131 struct immu *dr_immu; 132 dev_info_t *dr_dip; 133 uint16_t dr_seg; 134 uint64_t dr_regs; 135 boolean_t dr_include_all; 136 list_t dr_scope_list; 137 list_node_t dr_node; 138 } drhd_t; 139 140 typedef struct rmrr { 141 kmutex_t rm_lock; 142 uint16_t rm_seg; 143 uint64_t rm_base; 144 uint64_t rm_limit; 145 list_t rm_scope_list; 146 list_node_t rm_node; 147 } rmrr_t; 148 149 #define IMMU_UNIT_NAME "iommu" 150 151 /* 152 * Macros based on PCI spec 153 */ 154 #define IMMU_PCI_DEV(devfunc) ((uint64_t)devfunc >> 3) /* from devfunc */ 155 #define IMMU_PCI_FUNC(devfunc) (devfunc & 7) /* get func from devfunc */ 156 #define IMMU_PCI_DEVFUNC(d, f) (((d) << 3) | (f)) /* create devfunc */ 157 158 typedef struct scope { 159 uint8_t scp_type; 160 uint8_t scp_enumid; 161 uint8_t scp_bus; 162 uint8_t scp_dev; 163 uint8_t scp_func; 164 list_node_t scp_node; 165 } scope_t; 166 167 /* 168 * interrupt source id and drhd info for ioapic 169 */ 170 typedef struct ioapic_drhd { 171 uchar_t ioapic_ioapicid; 172 uint16_t ioapic_sid; /* ioapic source id */ 173 drhd_t *ioapic_drhd; 174 list_node_t ioapic_node; 175 } ioapic_drhd_t; 176 177 typedef struct memrng { 178 uint64_t mrng_start; 179 uint64_t mrng_npages; 180 } memrng_t; 181 182 typedef enum immu_flags { 183 IMMU_FLAGS_NONE = 0x1, 184 IMMU_FLAGS_SLEEP = 0x1, 185 IMMU_FLAGS_NOSLEEP = 0x2, 186 IMMU_FLAGS_READ = 0x4, 187 IMMU_FLAGS_WRITE = 0x8, 188 IMMU_FLAGS_DONTPASS = 0x10, 189 IMMU_FLAGS_ALLOC = 0x20, 190 IMMU_FLAGS_MUST_MATCH = 0x40, 191 IMMU_FLAGS_PAGE1 = 0x80, 192 IMMU_FLAGS_UNITY = 0x100, 193 IMMU_FLAGS_DMAHDL = 0x200, 194 IMMU_FLAGS_MEMRNG = 0x400 195 } immu_flags_t; 196 197 typedef enum cont_avail { 198 IMMU_CONT_BAD = 0x0, 199 IMMU_CONT_UNINITED = 0x1, 200 IMMU_CONT_INITED = 0x2 201 } cont_avail_t; 202 203 /* Size of root and context tables and their entries */ 204 #define IMMU_ROOT_TBLSZ (4096) 205 #define IMMU_CONT_TBLSZ (4096) 206 #define IMMU_ROOT_NUM (256) 207 #define IMMU_CONT_NUM (256) 208 209 /* register offset */ 210 #define IMMU_REG_VERSION (0x00) /* Version Rigister, 32 bit */ 211 #define IMMU_REG_CAP (0x08) /* Capability Register, 64 bit */ 212 #define IMMU_REG_EXCAP (0x10) /* Extended Capability Reg, 64 bit */ 213 #define IMMU_REG_GLOBAL_CMD (0x18) /* Global Command Register, 32 bit */ 214 #define IMMU_REG_GLOBAL_STS (0x1C) /* Global Status Register, 32 bit */ 215 #define IMMU_REG_ROOTENTRY (0x20) /* Root-Entry Table Addr Reg, 64 bit */ 216 #define IMMU_REG_CONTEXT_CMD (0x28) /* Context Comand Register, 64 bit */ 217 #define IMMU_REG_FAULT_STS (0x34) /* Fault Status Register, 32 bit */ 218 #define IMMU_REG_FEVNT_CON (0x38) /* Fault Event Control Reg, 32 bit */ 219 #define IMMU_REG_FEVNT_DATA (0x3C) /* Fault Event Data Register, 32 bit */ 220 #define IMMU_REG_FEVNT_ADDR (0x40) /* Fault Event Address Reg, 32 bit */ 221 #define IMMU_REG_FEVNT_UADDR (0x44) /* Fault Event Upper Addr Reg, 32 bit */ 222 #define IMMU_REG_AFAULT_LOG (0x58) /* Advanced Fault Log Reg, 64 bit */ 223 #define IMMU_REG_PMER (0x64) /* Protected Memory Enble Reg, 32 bit */ 224 #define IMMU_REG_PLMBR (0x68) /* Protected Low Mem Base Reg, 32 bit */ 225 #define IMMU_REG_PLMLR (0x6C) /* Protected Low Mem Lim Reg, 32 bit */ 226 #define IMMU_REG_PHMBR (0X70) /* Protectd High Mem Base Reg, 64 bit */ 227 #define IMMU_REG_PHMLR (0x78) /* Protected High Mem Lim Reg, 64 bit */ 228 #define IMMU_REG_INVAL_QH (0x80) /* Invalidation Queue Head, 64 bit */ 229 #define IMMU_REG_INVAL_QT (0x88) /* Invalidation Queue Tail, 64 bit */ 230 #define IMMU_REG_INVAL_QAR (0x90) /* Invalidtion Queue Addr Reg, 64 bit */ 231 #define IMMU_REG_INVAL_CSR (0x9C) /* Inval Compl Status Reg, 32 bit */ 232 #define IMMU_REG_INVAL_CECR (0xA0) /* Inval Compl Evnt Ctrl Reg, 32 bit */ 233 #define IMMU_REG_INVAL_CEDR (0xA4) /* Inval Compl Evnt Data Reg, 32 bit */ 234 #define IMMU_REG_INVAL_CEAR (0xA8) /* Inval Compl Event Addr Reg, 32 bit */ 235 #define IMMU_REG_INVAL_CEUAR (0xAC) /* Inval Comp Evnt Up Addr reg, 32bit */ 236 #define IMMU_REG_IRTAR (0xB8) /* INTR Remap Tbl Addr Reg, 64 bit */ 237 238 /* ioapic memory region */ 239 #define IOAPIC_REGION_START (0xfee00000) 240 #define IOAPIC_REGION_END (0xfeefffff) 241 242 /* fault register */ 243 #define IMMU_FAULT_STS_PPF (2) 244 #define IMMU_FAULT_STS_PFO (1) 245 #define IMMU_FAULT_STS_ITE (1 << 6) 246 #define IMMU_FAULT_STS_ICE (1 << 5) 247 #define IMMU_FAULT_STS_IQE (1 << 4) 248 #define IMMU_FAULT_GET_INDEX(x) ((((uint64_t)x) >> 8) & 0xff) 249 #define IMMU_FRR_GET_F(x) (((uint64_t)x) >> 63) 250 #define IMMU_FRR_GET_FR(x) ((((uint64_t)x) >> 32) & 0xff) 251 #define IMMU_FRR_GET_FT(x) ((((uint64_t)x) >> 62) & 0x1) 252 #define IMMU_FRR_GET_SID(x) ((x) & 0xffff) 253 254 /* (ex)capability register */ 255 #define IMMU_CAP_GET_NFR(x) (((((uint64_t)x) >> 40) & 0xff) + 1) 256 #define IMMU_CAP_GET_DWD(x) ((((uint64_t)x) >> 54) & 1) 257 #define IMMU_CAP_GET_DRD(x) ((((uint64_t)x) >> 55) & 1) 258 #define IMMU_CAP_GET_PSI(x) ((((uint64_t)x) >> 39) & 1) 259 #define IMMU_CAP_GET_SPS(x) ((((uint64_t)x) >> 34) & 0xf) 260 #define IMMU_CAP_GET_ISOCH(x) ((((uint64_t)x) >> 23) & 1) 261 #define IMMU_CAP_GET_ZLR(x) ((((uint64_t)x) >> 22) & 1) 262 #define IMMU_CAP_GET_MAMV(x) ((((uint64_t)x) >> 48) & 0x3f) 263 #define IMMU_CAP_GET_CM(x) ((((uint64_t)x) >> 7) & 1) 264 #define IMMU_CAP_GET_PHMR(x) ((((uint64_t)x) >> 6) & 1) 265 #define IMMU_CAP_GET_PLMR(x) ((((uint64_t)x) >> 5) & 1) 266 #define IMMU_CAP_GET_RWBF(x) ((((uint64_t)x) >> 4) & 1) 267 #define IMMU_CAP_GET_AFL(x) ((((uint64_t)x) >> 3) & 1) 268 #define IMMU_CAP_GET_FRO(x) (((((uint64_t)x) >> 24) & 0x3ff) * 16) 269 #define IMMU_CAP_MGAW(x) (((((uint64_t)x) >> 16) & 0x3f) + 1) 270 #define IMMU_CAP_SAGAW(x) ((((uint64_t)x) >> 8) & 0x1f) 271 #define IMMU_CAP_ND(x) (1 << (((x) & 0x7) *2 + 4)) -1 272 #define IMMU_ECAP_GET_IRO(x) (((((uint64_t)x) >> 8) & 0x3ff) << 4) 273 #define IMMU_ECAP_GET_MHMV(x) (((uint64_t)x >> 20) & 0xf) 274 #define IMMU_ECAP_GET_SC(x) ((x) & 0x80) 275 #define IMMU_ECAP_GET_PT(x) ((x) & 0x40) 276 #define IMMU_ECAP_GET_CH(x) ((x) & 0x20) 277 #define IMMU_ECAP_GET_EIM(x) ((x) & 0x10) 278 #define IMMU_ECAP_GET_IR(x) ((x) & 0x8) 279 #define IMMU_ECAP_GET_DI(x) ((x) & 0x4) 280 #define IMMU_ECAP_GET_QI(x) ((x) & 0x2) 281 #define IMMU_ECAP_GET_C(x) ((x) & 0x1) 282 283 #define IMMU_CAP_SET_RWBF(x) ((x) |= (1 << 4)) 284 285 286 /* iotlb invalidation */ 287 #define TLB_INV_GLOBAL (((uint64_t)1) << 60) 288 #define TLB_INV_DOMAIN (((uint64_t)2) << 60) 289 #define TLB_INV_PAGE (((uint64_t)3) << 60) 290 #define TLB_INV_GET_IAIG(x) ((((uint64_t)x) >> 57) & 7) 291 #define TLB_INV_DRAIN_READ (((uint64_t)1) << 49) 292 #define TLB_INV_DRAIN_WRITE (((uint64_t)1) << 48) 293 #define TLB_INV_DID(x) (((uint64_t)((x) & 0xffff)) << 32) 294 #define TLB_INV_IVT (((uint64_t)1) << 63) 295 #define TLB_IVA_HINT(x) (((x) & 0x1) << 6) 296 #define TLB_IVA_LEAF 1 297 #define TLB_IVA_WHOLE 0 298 299 /* dont use value 0 for enums - to catch unit 8 */ 300 typedef enum iotlb_inv { 301 IOTLB_PSI = 1, 302 IOTLB_DSI, 303 IOTLB_GLOBAL 304 } immu_iotlb_inv_t; 305 306 typedef enum context_inv { 307 CONTEXT_FSI = 1, 308 CONTEXT_DSI, 309 CONTEXT_GLOBAL 310 } immu_context_inv_t; 311 312 /* context invalidation */ 313 #define CCMD_INV_ICC (((uint64_t)1) << 63) 314 #define CCMD_INV_GLOBAL (((uint64_t)1) << 61) 315 #define CCMD_INV_DOMAIN (((uint64_t)2) << 61) 316 #define CCMD_INV_DEVICE (((uint64_t)3) << 61) 317 #define CCMD_INV_DID(x) ((uint64_t)((x) & 0xffff)) 318 #define CCMD_INV_SID(x) (((uint64_t)((x) & 0xffff)) << 16) 319 #define CCMD_INV_FM(x) (((uint64_t)((x) & 0x3)) << 32) 320 321 /* global command register */ 322 #define IMMU_GCMD_TE (((uint32_t)1) << 31) 323 #define IMMU_GCMD_SRTP (((uint32_t)1) << 30) 324 #define IMMU_GCMD_SFL (((uint32_t)1) << 29) 325 #define IMMU_GCMD_EAFL (((uint32_t)1) << 28) 326 #define IMMU_GCMD_WBF (((uint32_t)1) << 27) 327 #define IMMU_GCMD_QIE (((uint32_t)1) << 26) 328 #define IMMU_GCMD_IRE (((uint32_t)1) << 25) 329 #define IMMU_GCMD_SIRTP (((uint32_t)1) << 24) 330 #define IMMU_GCMD_CFI (((uint32_t)1) << 23) 331 332 /* global status register */ 333 #define IMMU_GSTS_TES (((uint32_t)1) << 31) 334 #define IMMU_GSTS_RTPS (((uint32_t)1) << 30) 335 #define IMMU_GSTS_FLS (((uint32_t)1) << 29) 336 #define IMMU_GSTS_AFLS (((uint32_t)1) << 28) 337 #define IMMU_GSTS_WBFS (((uint32_t)1) << 27) 338 #define IMMU_GSTS_QIES (((uint32_t)1) << 26) 339 #define IMMU_GSTS_IRES (((uint32_t)1) << 25) 340 #define IMMU_GSTS_IRTPS (((uint32_t)1) << 24) 341 #define IMMU_GSTS_CFIS (((uint32_t)1) << 23) 342 343 /* psi address mask */ 344 #define ADDR_AM_MAX(m) (((uint_t)1) << (m)) 345 #define ADDR_AM_OFFSET(n, m) ((n) & (ADDR_AM_MAX(m) - 1)) 346 347 /* dmar fault event */ 348 #define IMMU_INTR_IPL (4) 349 #define IMMU_REG_FEVNT_CON_IM_SHIFT (31) 350 351 #define IMMU_ALLOC_RESOURCE_DELAY (drv_usectohz(5000)) 352 353 /* max value of Size field of Interrupt Remapping Table Address Register */ 354 #define INTRMAP_MAX_IRTA_SIZE 0xf 355 356 /* interrupt remapping table entry size */ 357 #define INTRMAP_RTE_SIZE 0x10 358 359 /* ioapic redirection table entry related shift of remappable interrupt */ 360 #define INTRMAP_IOAPIC_IDX_SHIFT 17 361 #define INTRMAP_IOAPIC_FORMAT_SHIFT 16 362 #define INTRMAP_IOAPIC_TM_SHIFT 15 363 #define INTRMAP_IOAPIC_POL_SHIFT 13 364 #define INTRMAP_IOAPIC_IDX15_SHIFT 11 365 366 /* msi intr entry related shift of remappable interrupt */ 367 #define INTRMAP_MSI_IDX_SHIFT 5 368 #define INTRMAP_MSI_FORMAT_SHIFT 4 369 #define INTRMAP_MSI_SHV_SHIFT 3 370 #define INTRMAP_MSI_IDX15_SHIFT 2 371 372 #define INTRMAP_IDX_FULL (uint_t)-1 373 374 #define RDT_DLM(rdt) BITX((rdt), 10, 8) 375 #define RDT_DM(rdt) BT_TEST(&(rdt), 11) 376 #define RDT_POL(rdt) BT_TEST(&(rdt), 13) 377 #define RDT_TM(rdt) BT_TEST(&(rdt), 15) 378 379 #define INTRMAP_DISABLE (void *)-1 380 381 /* 382 * invalidation granularity 383 */ 384 typedef enum { 385 TLB_INV_G_GLOBAL = 1, 386 TLB_INV_G_DOMAIN, 387 TLB_INV_G_PAGE 388 } tlb_inv_g_t; 389 390 typedef enum { 391 CTT_INV_G_GLOBAL = 1, 392 CTT_INV_G_DOMAIN, 393 CTT_INV_G_DEVICE 394 } ctt_inv_g_t; 395 396 typedef enum { 397 IEC_INV_GLOBAL = 0, 398 IEC_INV_INDEX 399 } iec_inv_g_t; 400 401 402 struct inv_queue_state; 403 struct intrmap_tbl_state; 404 405 /* A software page table structure */ 406 typedef struct pgtable { 407 krwlock_t swpg_rwlock; 408 caddr_t hwpg_vaddr; /* HW pgtable VA */ 409 paddr_t hwpg_paddr; /* HW pgtable PA */ 410 ddi_dma_handle_t hwpg_dmahdl; 411 ddi_acc_handle_t hwpg_memhdl; 412 struct pgtable **swpg_next_array; 413 list_node_t swpg_domain_node; /* domain list of pgtables */ 414 } pgtable_t; 415 416 /* interrupt remapping table state info */ 417 typedef struct intrmap { 418 kmutex_t intrmap_lock; 419 ddi_dma_handle_t intrmap_dma_hdl; 420 ddi_acc_handle_t intrmap_acc_hdl; 421 caddr_t intrmap_vaddr; 422 paddr_t intrmap_paddr; 423 uint_t intrmap_size; 424 bitset_t intrmap_map; 425 uint_t intrmap_free; 426 } intrmap_t; 427 428 typedef struct hw_rce { 429 uint64_t lo; 430 uint64_t hi; 431 } hw_rce_t; 432 433 434 #define ROOT_GET_P(hrent) ((hrent)->lo & 0x1) 435 #define ROOT_SET_P(hrent) ((hrent)->lo |= 0x1) 436 437 #define ROOT_GET_CONT(hrent) ((hrent)->lo & ~(0xFFF)) 438 #define ROOT_SET_CONT(hrent, paddr) ((hrent)->lo |= (paddr & (~0xFFF))) 439 440 #define TTYPE_XLATE_ONLY (0x0) 441 #define TTYPE_XLATE_IOTLB (0x1) 442 #define TTYPE_PASSTHRU (0x2) 443 #define TTYPE_RESERVED (0x3) 444 445 #define CONT_GET_DID(hcent) ((((uint64_t)(hcent)->hi) >> 8) & 0xFFFF) 446 #define CONT_SET_DID(hcent, did) ((hcent)->hi |= ((0xFFFF & (did)) << 8)) 447 448 #define CONT_GET_AVAIL(hcent) ((((uint64_t)((hcent)->hi)) >> 0x3) & 0xF) 449 #define CONT_SET_AVAIL(hcent, av) ((hcent)->hi |= ((0xF & (av)) << 0x3)) 450 451 #define CONT_GET_LO_AW(hcent) (30 + 9 *((hcent)->hi & 0x7)) 452 #define CONT_GET_AW(hcent) \ 453 ((CONT_GET_LO_AW(hcent) == 66) ? 64 : CONT_GET_LO_AW(hcent)) 454 #define CONT_SET_AW(hcent, aw) \ 455 ((hcent)->hi |= (((((aw) + 2) - 30) / 9) & 0x7)) 456 457 #define CONT_GET_ASR(hcent) ((hcent)->lo & ~(0xFFF)) 458 #define CONT_SET_ASR(hcent, paddr) ((hcent)->lo |= (paddr & (~0xFFF))) 459 460 #define CONT_GET_TTYPE(hcent) ((((uint64_t)(hcent)->lo) >> 0x2) & 0x3) 461 #define CONT_SET_TTYPE(hcent, ttype) ((hcent)->lo |= (((ttype) & 0x3) << 0x2)) 462 463 #define CONT_GET_P(hcent) ((hcent)->lo & 0x1) 464 #define CONT_SET_P(hcent) ((hcent)->lo |= 0x1) 465 466 #define CONT_GET_ALH(hcent) ((hcent)->lo & 0x20) 467 #define CONT_SET_ALH(hcent) ((hcent)->lo |= 0x20) 468 469 #define CONT_GET_EH(hcent) ((hcent)->lo & 0x10) 470 #define CONT_SET_EH(hcent) ((hcent)->lo |= 0x10) 471 472 473 /* we use the bit 63 (available for system SW) as a present bit */ 474 #define PDTE_SW4(hw_pdte) ((hw_pdte) & ((uint64_t)1<<63)) 475 #define PDTE_CLEAR_SW4(hw_pdte) ((hw_pdte) &= ~((uint64_t)1<<63)) 476 477 #define PDTE_P(hw_pdte) ((hw_pdte) & ((uint64_t)1<<63)) 478 #define PDTE_CLEAR_P(hw_pdte) ((hw_pdte) &= ~((uint64_t)1<<63)) 479 #define PDTE_SET_P(hw_pdte) ((hw_pdte) |= ((uint64_t)1<<63)) 480 481 #define PDTE_TM(hw_pdte) ((hw_pdte) & ((uint64_t)1<<62)) 482 #define PDTE_CLEAR_TM(hw_pdte) ((hw_pdte) &= ~((uint64_t)1<<62)) 483 484 #define PDTE_SW3(hw_pdte) \ 485 (((hw_pdte) & ~(((uint64_t)0x3<<62)|(((uint64_t)1<<52)-1))) >> 52) 486 #define PDTE_SW3_OVERFLOW(hw_pdte) \ 487 (PDTE_SW3(hw_pdte) == 0x3FF) 488 #define PDTE_CLEAR_SW3(hw_pdte) \ 489 ((hw_pdte) &= (((uint64_t)0x3<<62)|(((uint64_t)1<<52)-1))) 490 #define PDTE_SET_SW3(hw_pdte, ref) \ 491 ((hw_pdte) |= ((((uint64_t)(ref)) & 0x3FF) << 52)) 492 493 #define PDTE_PADDR(hw_pdte) ((hw_pdte) & ~(((uint64_t)0xFFF<<52)|((1<<12)-1))) 494 #define PDTE_CLEAR_PADDR(hw_pdte) \ 495 ((hw_pdte) &= (((uint64_t)0xFFF<<52)|((1<<12)-1))) 496 #define PDTE_SET_PADDR(hw_pdte, paddr) ((hw_pdte) |= PDTE_PADDR(paddr)) 497 498 #define PDTE_SNP(hw_pdte) ((hw_pdte) & (1<<11)) 499 #define PDTE_CLEAR_SNP(hw_pdte) ((hw_pdte) &= ~(1<<11)) 500 #define PDTE_SET_SNP(hw_pdte) ((hw_pdte) |= (1<<11)) 501 502 #define PDTE_SW2(hw_pdte) ((hw_pdte) & (0x700)) 503 #define PDTE_CLEAR_SW2(hw_pdte) ((hw_pdte) &= ~(0x700)) 504 505 #define PDTE_SP(hw_pdte) ((hw_pdte) & (0x80)) 506 #define PDTE_CLEAR_SP(hw_pdte) ((hw_pdte) &= ~(0x80)) 507 508 #define PDTE_SW1(hw_pdte) ((hw_pdte) & (0x7C)) 509 #define PDTE_CLEAR_SW1(hw_pdte) ((hw_pdte) &= ~(0x7C)) 510 511 #define PDTE_WRITE(hw_pdte) ((hw_pdte) & (0x2)) 512 #define PDTE_CLEAR_WRITE(hw_pdte) ((hw_pdte) &= ~(0x2)) 513 #define PDTE_SET_WRITE(hw_pdte) ((hw_pdte) |= (0x2)) 514 515 #define PDTE_READ(hw_pdte) ((hw_pdte) & (0x1)) 516 #define PDTE_CLEAR_READ(hw_pdte) ((hw_pdte) &= ~(0x1)) 517 #define PDTE_SET_READ(hw_pdte) ((hw_pdte) |= (0x1)) 518 519 #define PDTE_MASK_R ((uint64_t)1 << 0) 520 #define PDTE_MASK_W ((uint64_t)1 << 1) 521 #define PDTE_MASK_SNP ((uint64_t)1 << 11) 522 #define PDTE_MASK_TM ((uint64_t)1 << 62) 523 #define PDTE_MASK_P ((uint64_t)1 << 63) 524 525 struct immu_flushops; 526 527 /* 528 * Used to wait for invalidation completion. 529 * vstatus is the virtual address of the status word that will be written 530 * pstatus is the physical addres 531 * If sync is true, then the the operation will be waited on for 532 * completion immediately. Else, the wait interface can be called 533 * to wait for completion later. 534 */ 535 536 #define IMMU_INV_DATA_PENDING 1 537 #define IMMU_INV_DATA_DONE 2 538 539 typedef struct immu_inv_wait { 540 volatile uint32_t iwp_vstatus; 541 uint64_t iwp_pstatus; 542 boolean_t iwp_sync; 543 const char *iwp_name; /* ID for debugging/statistics */ 544 } immu_inv_wait_t; 545 546 /* 547 * Used to batch IOMMU pagetable writes. 548 */ 549 typedef struct immu_dcookie { 550 paddr_t dck_paddr; 551 uint64_t dck_npages; 552 } immu_dcookie_t; 553 554 typedef struct immu { 555 kmutex_t immu_lock; 556 char *immu_name; 557 558 /* lock grabbed by interrupt handler */ 559 kmutex_t immu_intr_lock; 560 561 /* ACPI/DMAR table related */ 562 void *immu_dmar_unit; 563 dev_info_t *immu_dip; 564 struct domain *immu_unity_domain; 565 566 /* IOMMU register related */ 567 kmutex_t immu_regs_lock; 568 kcondvar_t immu_regs_cv; 569 boolean_t immu_regs_busy; 570 boolean_t immu_regs_setup; 571 boolean_t immu_regs_running; 572 boolean_t immu_regs_quiesced; 573 ddi_acc_handle_t immu_regs_handle; 574 caddr_t immu_regs_addr; 575 uint64_t immu_regs_cap; 576 uint64_t immu_regs_excap; 577 uint32_t immu_regs_cmdval; 578 uint32_t immu_regs_intr_msi_addr; 579 uint32_t immu_regs_intr_msi_data; 580 uint32_t immu_regs_intr_uaddr; 581 582 /* DVMA related */ 583 kmutex_t immu_dvma_lock; 584 boolean_t immu_dvma_setup; 585 boolean_t immu_dvma_running; 586 int immu_dvma_gaw; 587 int immu_dvma_agaw; 588 int immu_dvma_nlevels; 589 boolean_t immu_dvma_coherent; 590 boolean_t immu_TM_reserved; 591 boolean_t immu_SNP_reserved; 592 uint64_t immu_ptemask; 593 594 /* DVMA context related */ 595 krwlock_t immu_ctx_rwlock; 596 pgtable_t *immu_ctx_root; 597 immu_inv_wait_t immu_ctx_inv_wait; 598 599 /* DVMA domain related */ 600 int immu_max_domains; 601 vmem_t *immu_did_arena; 602 char immu_did_arena_name[IMMU_MAXNAMELEN]; 603 list_t immu_domain_list; 604 605 /* DVMA special devices */ 606 boolean_t immu_dvma_gfx_only; 607 list_t immu_dvma_lpc_list; 608 list_t immu_dvma_gfx_list; 609 610 /* interrupt remapping related */ 611 kmutex_t immu_intrmap_lock; 612 boolean_t immu_intrmap_setup; 613 boolean_t immu_intrmap_running; 614 intrmap_t *immu_intrmap; 615 uint64_t immu_intrmap_irta_reg; 616 immu_inv_wait_t immu_intrmap_inv_wait; 617 618 /* queued invalidation related */ 619 kmutex_t immu_qinv_lock; 620 boolean_t immu_qinv_setup; 621 boolean_t immu_qinv_running; 622 boolean_t immu_qinv_enabled; 623 void *immu_qinv; 624 uint64_t immu_qinv_reg_value; 625 626 /* list_node for system-wide list of DMAR units */ 627 list_node_t immu_node; 628 629 struct immu_flushops *immu_flushops; 630 631 kmem_cache_t *immu_hdl_cache; 632 kmem_cache_t *immu_pgtable_cache; 633 634 iommulib_handle_t immu_iommulib_handle; 635 } immu_t; 636 637 /* 638 * Enough space to hold the decimal number of any device instance. 639 * Used for device/cache names. 640 */ 641 #define IMMU_ISTRLEN 11 /* log10(2^31) + 1 */ 642 643 /* properties that control DVMA */ 644 #define DDI_DVMA_MAPTYPE_ROOTNEX_PROP "immu-dvma-mapping" 645 646 #define DDI_DVMA_MAPTYPE_UNITY "unity" 647 #define DDI_DVMA_MAPTYPE_XLATE "xlate" 648 649 typedef enum immu_maptype { 650 IMMU_MAPTYPE_BAD = 0, /* 0 is always bad */ 651 IMMU_MAPTYPE_UNITY = 1, 652 IMMU_MAPTYPE_XLATE 653 } immu_maptype_t; 654 655 #define IMMU_COOKIE_HASHSZ (512) 656 657 /* 658 * domain_t 659 * 660 */ 661 typedef struct domain { 662 /* the basics */ 663 uint_t dom_did; 664 immu_t *dom_immu; 665 666 /* mapping related */ 667 immu_maptype_t dom_maptype; 668 vmem_t *dom_dvma_arena; 669 char dom_dvma_arena_name[IMMU_MAXNAMELEN]; 670 671 /* pgtables */ 672 pgtable_t *dom_pgtable_root; 673 krwlock_t dom_pgtable_rwlock; 674 675 /* list node for list of domains (unity or xlate) */ 676 list_node_t dom_maptype_node; 677 /* list node for list of domains off immu */ 678 list_node_t dom_immu_node; 679 680 mod_hash_t *dom_cookie_hash; 681 682 /* topmost device in domain; usually the device itself (non-shared) */ 683 dev_info_t *dom_dip; 684 } domain_t; 685 686 typedef enum immu_pcib { 687 IMMU_PCIB_BAD = 0, 688 IMMU_PCIB_NOBDF, 689 IMMU_PCIB_PCIE_PCIE, 690 IMMU_PCIB_PCIE_PCI, 691 IMMU_PCIB_PCI_PCI, 692 IMMU_PCIB_ENDPOINT 693 } immu_pcib_t; 694 695 /* 696 * immu_devi_t 697 * Intel IOMMU in devinfo node 698 */ 699 typedef struct immu_devi { 700 /* pci seg, bus, dev, func */ 701 int imd_seg; 702 int imd_bus; 703 int imd_devfunc; 704 705 /* ppb information */ 706 immu_pcib_t imd_pcib_type; 707 int imd_sec; 708 int imd_sub; 709 710 /* identifier for special devices */ 711 boolean_t imd_display; 712 boolean_t imd_lpc; 713 714 /* set if premapped DVMA space is used */ 715 boolean_t imd_use_premap; 716 717 /* dmar unit to which this dip belongs */ 718 immu_t *imd_immu; 719 720 immu_flags_t imd_dvma_flags; 721 722 /* domain ptr */ 723 domain_t *imd_domain; 724 dev_info_t *imd_ddip; 725 726 /* my devinfo */ 727 dev_info_t *imd_dip; 728 729 /* 730 * if we are a "special" devinfo 731 * the node for the special linked list 732 * off the DMAR unit structure 733 */ 734 list_node_t imd_spc_node; 735 } immu_devi_t; 736 737 #define IMMU_DEVI(dip) ((immu_devi_t *)(DEVI(dip)->devi_iommu)) 738 #define IMMU_DEVI_SET(dip, imd) (DEVI(dip)->devi_iommu = (void *)imd) 739 740 /* 741 * struct dmar_arg 742 */ 743 typedef struct immu_arg { 744 int ima_seg; 745 int ima_bus; 746 int ima_devfunc; 747 dev_info_t *ima_rdip; 748 dev_info_t *ima_ddip; 749 } immu_arg_t; 750 751 #define IMMU_NDVSEG 8 752 #define IMMU_NDCK 64 753 #define IMMU_NPREPTES 8 754 755 typedef struct immu_hdl_private { 756 immu_inv_wait_t ihp_inv_wait; 757 size_t ihp_ndvseg; 758 struct dvmaseg ihp_dvseg[IMMU_NDVSEG]; 759 immu_dcookie_t ihp_dcookies[IMMU_NDCK]; 760 761 hw_pdte_t *ihp_preptes[IMMU_NPREPTES]; 762 uint64_t ihp_predvma; 763 int ihp_npremapped; 764 } immu_hdl_priv_t; 765 766 /* 767 * Invalidation operation function pointers for context and IOTLB. 768 * These will be set to either the register or the queue invalidation 769 * interface functions, since the hardware does not allow using them 770 * both at the same time. 771 */ 772 struct immu_flushops { 773 void (*imf_context_fsi)(immu_t *, uint8_t, uint16_t, uint_t, 774 immu_inv_wait_t *); 775 void (*imf_context_dsi)(immu_t *, uint_t, immu_inv_wait_t *); 776 void (*imf_context_gbl)(immu_t *, immu_inv_wait_t *); 777 778 void (*imf_iotlb_psi)(immu_t *, uint_t, uint64_t, uint_t, uint_t, 779 immu_inv_wait_t *); 780 void (*imf_iotlb_dsi)(immu_t *, uint_t, immu_inv_wait_t *); 781 void (*imf_iotlb_gbl)(immu_t *, immu_inv_wait_t *); 782 783 void (*imf_wait)(immu_inv_wait_t *); 784 }; 785 786 #define immu_flush_context_fsi(i, f, s, d, w) \ 787 (i)->immu_flushops->imf_context_fsi(i, f, s, d, w) 788 #define immu_flush_context_dsi(i, d, w) \ 789 (i)->immu_flushops->imf_context_dsi(i, d, w) 790 #define immu_flush_context_gbl(i, w) \ 791 (i)->immu_flushops->imf_context_gbl(i, w) 792 793 #define immu_flush_iotlb_psi(i, d, v, c, h, w) \ 794 (i)->immu_flushops->imf_iotlb_psi(i, d, v, c, h, w) 795 #define immu_flush_iotlb_dsi(i, d, w) \ 796 (i)->immu_flushops->imf_iotlb_dsi(i, d, w) 797 #define immu_flush_iotlb_gbl(i, w) \ 798 (i)->immu_flushops->imf_iotlb_gbl(i, w) 799 800 #define immu_flush_wait(i, w) \ 801 (i)->immu_flushops->imf_wait(w) 802 803 /* 804 * Globals used by IOMMU code 805 */ 806 /* shared between IOMMU files */ 807 extern dev_info_t *root_devinfo; 808 extern kmutex_t immu_lock; 809 extern list_t immu_list; 810 extern boolean_t immu_setup; 811 extern boolean_t immu_running; 812 extern kmutex_t ioapic_drhd_lock; 813 extern list_t ioapic_drhd_list; 814 extern struct iommulib_ops immulib_ops; 815 816 /* switches */ 817 818 /* Various features */ 819 extern boolean_t immu_enable; 820 extern boolean_t immu_gfxdvma_enable; 821 extern boolean_t immu_intrmap_enable; 822 extern boolean_t immu_qinv_enable; 823 824 /* various quirks that need working around */ 825 extern boolean_t immu_quirk_usbpage0; 826 extern boolean_t immu_quirk_usbfullpa; 827 extern boolean_t immu_quirk_usbrmrr; 828 extern boolean_t immu_quirk_mobile4; 829 830 /* debug messages */ 831 extern boolean_t immu_dmar_print; 832 833 /* tunables */ 834 extern int64_t immu_flush_gran; 835 836 extern immu_flags_t immu_global_dvma_flags; 837 838 extern int immu_use_tm; 839 extern int immu_use_alh; 840 841 /* ################### Interfaces exported outside IOMMU code ############## */ 842 void immu_init(void); 843 void immu_startup(void); 844 void immu_shutdown(void); 845 void immu_destroy(void); 846 int immu_map_sgl(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 847 int prealloc_count, dev_info_t *rdip); 848 int immu_unmap_sgl(ddi_dma_impl_t *hp, dev_info_t *rdip); 849 void immu_device_tree_changed(void); 850 void immu_physmem_update(uint64_t addr, uint64_t size); 851 int immu_quiesce(void); 852 int immu_unquiesce(void); 853 /* ######################################################################### */ 854 855 /* ################# Interfaces used within IOMMU code #################### */ 856 /* immu_dmar.c interfaces */ 857 int immu_dmar_setup(void); 858 int immu_dmar_parse(void); 859 void immu_dmar_startup(void); 860 void immu_dmar_shutdown(void); 861 void immu_dmar_destroy(void); 862 boolean_t immu_dmar_blacklisted(char **strings_array, uint_t nstrings); 863 immu_t *immu_dmar_get_immu(dev_info_t *rdip); 864 dev_info_t *immu_dmar_unit_dip(void *dmar_unit); 865 void immu_dmar_set_immu(void *dmar_unit, immu_t *immu); 866 void *immu_dmar_walk_units(int seg, void *dmar_unit); 867 boolean_t immu_dmar_intrmap_supported(void); 868 uint16_t immu_dmar_ioapic_sid(int ioapicid); 869 immu_t *immu_dmar_ioapic_immu(int ioapicid); 870 void immu_dmar_rmrr_map(void); 871 872 /* immu.c interfaces */ 873 int immu_walk_ancestor(dev_info_t *rdip, dev_info_t *ddip, 874 int (*func)(dev_info_t *, void *arg), void *arg, 875 int *level, immu_flags_t immu_flags); 876 void immu_init_inv_wait(immu_inv_wait_t *iwp, const char *s, boolean_t sync); 877 878 /* immu_regs.c interfaces */ 879 void immu_regs_setup(list_t *immu_list); 880 void immu_regs_startup(immu_t *immu); 881 int immu_regs_resume(immu_t *immu); 882 void immu_regs_suspend(immu_t *immu); 883 void immu_regs_shutdown(immu_t *immu); 884 void immu_regs_destroy(list_t *immu_list); 885 886 void immu_regs_intr(immu_t *immu, uint32_t msi_addr, uint32_t msi_data, 887 uint32_t uaddr); 888 889 boolean_t immu_regs_passthru_supported(immu_t *immu); 890 boolean_t immu_regs_is_TM_reserved(immu_t *immu); 891 boolean_t immu_regs_is_SNP_reserved(immu_t *immu); 892 893 void immu_regs_wbf_flush(immu_t *immu); 894 void immu_regs_cpu_flush(immu_t *immu, caddr_t addr, uint_t size); 895 896 void immu_regs_context_fsi(immu_t *immu, uint8_t function_mask, 897 uint16_t source_id, uint_t domain_id, immu_inv_wait_t *iwp); 898 void immu_regs_context_dsi(immu_t *immu, uint_t domain_id, 899 immu_inv_wait_t *iwp); 900 void immu_regs_context_gbl(immu_t *immu, immu_inv_wait_t *iwp); 901 void immu_regs_iotlb_psi(immu_t *immu, uint_t domain_id, 902 uint64_t dvma, uint_t count, uint_t hint, immu_inv_wait_t *iwp); 903 void immu_regs_iotlb_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp); 904 void immu_regs_iotlb_gbl(immu_t *immu, immu_inv_wait_t *iwp); 905 906 void immu_regs_set_root_table(immu_t *immu); 907 void immu_regs_qinv_enable(immu_t *immu, uint64_t qinv_reg_value); 908 void immu_regs_intr_enable(immu_t *immu, uint32_t msi_addr, uint32_t msi_data, 909 uint32_t uaddr); 910 void immu_regs_intrmap_enable(immu_t *immu, uint64_t irta_reg); 911 uint64_t immu_regs_get64(immu_t *immu, uint_t reg); 912 void immu_regs_put64(immu_t *immu, uint_t reg, uint64_t val); 913 uint32_t immu_regs_get32(immu_t *immu, uint_t reg); 914 void immu_regs_put32(immu_t *immu, uint_t reg, uint32_t val); 915 916 /* immu_dvma.c interfaces */ 917 void immu_dvma_setup(list_t *immu_list); 918 void immu_dvma_startup(immu_t *immu); 919 void immu_dvma_shutdown(immu_t *immu); 920 void immu_dvma_destroy(list_t *immu_list); 921 922 void immu_dvma_physmem_update(uint64_t addr, uint64_t size); 923 int immu_map_memrange(dev_info_t *, memrng_t *); 924 int immu_dvma_map(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 925 uint_t prealloc_count, dev_info_t *rdip); 926 int immu_dvma_unmap(ddi_dma_impl_t *hp, dev_info_t *rdip); 927 int immu_devi_set(dev_info_t *dip, immu_flags_t immu_flags); 928 immu_devi_t *immu_devi_get(dev_info_t *dip); 929 immu_t *immu_dvma_get_immu(dev_info_t *dip, immu_flags_t immu_flags); 930 int pgtable_ctor(void *buf, void *arg, int kmflag); 931 void pgtable_dtor(void *buf, void *arg); 932 933 int immu_hdl_priv_ctor(void *buf, void *arg, int kmf); 934 935 int immu_dvma_device_setup(dev_info_t *rdip, immu_flags_t immu_flags); 936 937 void immu_print_fault_info(uint_t sid, uint64_t dvma); 938 939 /* immu_intrmap.c interfaces */ 940 void immu_intrmap_setup(list_t *immu_list); 941 void immu_intrmap_startup(immu_t *immu); 942 void immu_intrmap_shutdown(immu_t *immu); 943 void immu_intrmap_destroy(list_t *immu_list); 944 945 /* registers interrupt handler for IOMMU unit */ 946 void immu_intr_register(immu_t *immu); 947 int immu_intr_handler(immu_t *immu); 948 949 950 /* immu_qinv.c interfaces */ 951 int immu_qinv_setup(list_t *immu_list); 952 void immu_qinv_startup(immu_t *immu); 953 void immu_qinv_shutdown(immu_t *immu); 954 void immu_qinv_destroy(list_t *immu_list); 955 956 void immu_qinv_context_fsi(immu_t *immu, uint8_t function_mask, 957 uint16_t source_id, uint_t domain_id, immu_inv_wait_t *iwp); 958 void immu_qinv_context_dsi(immu_t *immu, uint_t domain_id, 959 immu_inv_wait_t *iwp); 960 void immu_qinv_context_gbl(immu_t *immu, immu_inv_wait_t *iwp); 961 void immu_qinv_iotlb_psi(immu_t *immu, uint_t domain_id, 962 uint64_t dvma, uint_t count, uint_t hint, immu_inv_wait_t *iwp); 963 void immu_qinv_iotlb_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp); 964 void immu_qinv_iotlb_gbl(immu_t *immu, immu_inv_wait_t *iwp); 965 966 void immu_qinv_intr_global(immu_t *immu, immu_inv_wait_t *iwp); 967 void immu_qinv_intr_one_cache(immu_t *immu, uint_t idx, immu_inv_wait_t *iwp); 968 void immu_qinv_intr_caches(immu_t *immu, uint_t idx, uint_t cnt, 969 immu_inv_wait_t *); 970 void immu_qinv_report_fault(immu_t *immu); 971 972 #ifdef DEBUG 973 #define IMMU_DPROBE1(name, type1, arg1) \ 974 DTRACE_PROBE1(name, type1, arg1) 975 #define IMMU_DPROBE2(name, type1, arg1, type2, arg2) \ 976 DTRACE_PROBE2(name, type1, arg1, type2, arg2) 977 #define IMMU_DPROBE3(name, type1, arg1, type2, arg2, type3, arg3) \ 978 DTRACE_PROBE3(name, type1, arg1, type2, arg2, type3, arg3) 979 #define IMMU_DPROBE4(name, type1, arg1, type2, arg2, type3, arg3, type4, arg4) \ 980 DTRACE_PROBE4(name, type1, arg1, type2, arg2, type3, arg3, type4, arg4) 981 #else 982 #define IMMU_DPROBE1(name, type1, arg1) 983 #define IMMU_DPROBE2(name, type1, arg1, type2, arg2) 984 #define IMMU_DPROBE3(name, type1, arg1, type2, arg2, type3, arg3) 985 #define IMMU_DPROBE4(name, type1, arg1, type2, arg2, type3, arg3, type4, arg4) 986 #endif 987 988 989 #ifdef __cplusplus 990 } 991 #endif 992 993 #endif /* _SYS_INTEL_IOMMU_H */ 994