1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc. 4 * Author: Joerg Roedel <jroedel@suse.de> 5 * Leo Duran <leo.duran@amd.com> 6 */ 7 8 #ifndef _ASM_X86_AMD_IOMMU_TYPES_H 9 #define _ASM_X86_AMD_IOMMU_TYPES_H 10 11 #include <linux/bitfield.h> 12 #include <linux/iommu.h> 13 #include <linux/types.h> 14 #include <linux/mmu_notifier.h> 15 #include <linux/mutex.h> 16 #include <linux/msi.h> 17 #include <linux/list.h> 18 #include <linux/sizes.h> 19 #include <linux/spinlock.h> 20 #include <linux/pci.h> 21 #include <linux/iommufd.h> 22 #include <linux/irqreturn.h> 23 #include <linux/generic_pt/iommu.h> 24 25 #include <uapi/linux/iommufd.h> 26 27 /* 28 * Maximum number of IOMMUs supported 29 */ 30 #define MAX_IOMMUS 32 31 32 /* 33 * some size calculation constants 34 */ 35 #define DEV_TABLE_ENTRY_SIZE 32 36 37 /* Capability offsets used by the driver */ 38 #define MMIO_CAP_HDR_OFFSET 0x00 39 #define MMIO_RANGE_OFFSET 0x0c 40 #define MMIO_MISC_OFFSET 0x10 41 42 /* Used offsets into the MMIO space */ 43 #define MMIO_DEV_TABLE_OFFSET 0x0000 44 #define MMIO_CMD_BUF_OFFSET 0x0008 45 #define MMIO_EVT_BUF_OFFSET 0x0010 46 #define MMIO_CONTROL_OFFSET 0x0018 47 #define MMIO_EXCL_BASE_OFFSET 0x0020 48 #define MMIO_EXCL_LIMIT_OFFSET 0x0028 49 #define MMIO_EXT_FEATURES 0x0030 50 #define MMIO_PPR_LOG_OFFSET 0x0038 51 #define MMIO_GA_LOG_BASE_OFFSET 0x00e0 52 #define MMIO_GA_LOG_TAIL_OFFSET 0x00e8 53 #define MMIO_MSI_ADDR_LO_OFFSET 0x015C 54 #define MMIO_MSI_ADDR_HI_OFFSET 0x0160 55 #define MMIO_MSI_DATA_OFFSET 0x0164 56 #define MMIO_PERF_OPT_OFFSET 0x016C 57 #define MMIO_INTCAPXT_EVT_OFFSET 0x0170 58 #define MMIO_INTCAPXT_PPR_OFFSET 0x0178 59 #define MMIO_INTCAPXT_GALOG_OFFSET 0x0180 60 #define MMIO_EXT_FEATURES2 0x01A0 61 #define MMIO_CMD_HEAD_OFFSET 0x2000 62 #define MMIO_CMD_TAIL_OFFSET 0x2008 63 #define MMIO_EVT_HEAD_OFFSET 0x2010 64 #define MMIO_EVT_TAIL_OFFSET 0x2018 65 #define MMIO_STATUS_OFFSET 0x2020 66 #define MMIO_PPR_HEAD_OFFSET 0x2030 67 #define MMIO_PPR_TAIL_OFFSET 0x2038 68 #define MMIO_GA_HEAD_OFFSET 0x2040 69 #define MMIO_GA_TAIL_OFFSET 0x2048 70 #define MMIO_CNTR_CONF_OFFSET 0x4000 71 #define MMIO_CNTR_REG_OFFSET 0x40000 72 #define MMIO_REG_END_OFFSET 0x80000 73 74 75 76 /* Extended Feature Bits */ 77 #define FEATURE_PREFETCH BIT_ULL(0) 78 #define FEATURE_PPR BIT_ULL(1) 79 #define FEATURE_X2APIC BIT_ULL(2) 80 #define FEATURE_NX BIT_ULL(3) 81 #define FEATURE_GT BIT_ULL(4) 82 #define FEATURE_IA BIT_ULL(6) 83 #define FEATURE_GA BIT_ULL(7) 84 #define FEATURE_HE BIT_ULL(8) 85 #define FEATURE_PC BIT_ULL(9) 86 #define FEATURE_HATS GENMASK_ULL(11, 10) 87 #define FEATURE_GATS GENMASK_ULL(13, 12) 88 #define FEATURE_GLX GENMASK_ULL(15, 14) 89 #define FEATURE_GAM_VAPIC BIT_ULL(21) 90 #define FEATURE_PASMAX GENMASK_ULL(36, 32) 91 #define FEATURE_PERF_OPT BIT_ULL(45) 92 #define PERF_OPT_EN BIT(13) 93 #define FEATURE_GIOSUP BIT_ULL(48) 94 #define FEATURE_HASUP BIT_ULL(49) 95 #define FEATURE_EPHSUP BIT_ULL(50) 96 #define FEATURE_HDSUP BIT_ULL(52) 97 #define FEATURE_SNP BIT_ULL(63) 98 99 100 /* Extended Feature 2 Bits */ 101 #define FEATURE_SEVSNPIO_SUP BIT_ULL(1) 102 #define FEATURE_GCR3TRPMODE BIT_ULL(3) 103 #define FEATURE_SNPAVICSUP GENMASK_ULL(7, 5) 104 #define FEATURE_SNPAVICSUP_GAM(x) \ 105 (FIELD_GET(FEATURE_SNPAVICSUP, x) == 0x1) 106 #define FEATURE_HT_RANGE_IGNORE BIT_ULL(11) 107 #define FEATURE_SNP_PAGE_MODE0_SUP BIT_ULL(13) 108 109 #define FEATURE_NUM_INT_REMAP_SUP GENMASK_ULL(9, 8) 110 #define FEATURE_NUM_INT_REMAP_SUP_2K(x) \ 111 (FIELD_GET(FEATURE_NUM_INT_REMAP_SUP, x) == 0x1) 112 113 /* Note: 114 * The current driver only support 16-bit PASID. 115 * Currently, hardware only implement upto 16-bit PASID 116 * even though the spec says it could have upto 20 bits. 117 */ 118 #define PASID_MASK 0x0000ffff 119 120 /* MMIO status bits */ 121 #define MMIO_STATUS_EVT_OVERFLOW_MASK BIT(0) 122 #define MMIO_STATUS_EVT_INT_MASK BIT(1) 123 #define MMIO_STATUS_COM_WAIT_INT_MASK BIT(2) 124 #define MMIO_STATUS_EVT_RUN_MASK BIT(3) 125 #define MMIO_STATUS_PPR_OVERFLOW_MASK BIT(5) 126 #define MMIO_STATUS_PPR_INT_MASK BIT(6) 127 #define MMIO_STATUS_PPR_RUN_MASK BIT(7) 128 #define MMIO_STATUS_GALOG_RUN_MASK BIT(8) 129 #define MMIO_STATUS_GALOG_OVERFLOW_MASK BIT(9) 130 #define MMIO_STATUS_GALOG_INT_MASK BIT(10) 131 132 /* event logging constants */ 133 #define EVENT_TYPE_SHIFT 28 134 #define EVENT_TYPE_MASK 0xf 135 #define EVENT_TYPE_ILL_DEV 0x1 136 #define EVENT_TYPE_IO_FAULT 0x2 137 #define EVENT_TYPE_DEV_TAB_ERR 0x3 138 #define EVENT_TYPE_PAGE_TAB_ERR 0x4 139 #define EVENT_TYPE_ILL_CMD 0x5 140 #define EVENT_TYPE_CMD_HARD_ERR 0x6 141 #define EVENT_TYPE_IOTLB_INV_TO 0x7 142 #define EVENT_TYPE_INV_DEV_REQ 0x8 143 #define EVENT_TYPE_INV_PPR_REQ 0x9 144 #define EVENT_TYPE_RMP_FAULT 0xd 145 #define EVENT_TYPE_RMP_HW_ERR 0xe 146 #define EVENT_DEVID_MASK 0xffff 147 #define EVENT_DEVID_SHIFT 0 148 #define EVENT_DOMID_MASK_LO 0xffff 149 #define EVENT_DOMID_MASK_HI 0xf0000 150 #define EVENT_FLAGS_MASK 0xfff 151 #define EVENT_FLAGS_SHIFT 0x10 152 #define EVENT_FLAG_RW 0x020 153 #define EVENT_FLAG_I 0x008 154 #define EVENT_FLAG_PPR_RX 0x001 155 #define EVENT_FLAG_PPR_GN 0x200 156 157 /* feature control bits */ 158 #define CONTROL_IOMMU_EN 0 159 #define CONTROL_HT_TUN_EN 1 160 #define CONTROL_EVT_LOG_EN 2 161 #define CONTROL_EVT_INT_EN 3 162 #define CONTROL_COMWAIT_EN 4 163 #define CONTROL_INV_TIMEOUT 5 164 #define CONTROL_PASSPW_EN 8 165 #define CONTROL_RESPASSPW_EN 9 166 #define CONTROL_COHERENT_EN 10 167 #define CONTROL_ISOC_EN 11 168 #define CONTROL_CMDBUF_EN 12 169 #define CONTROL_PPRLOG_EN 13 170 #define CONTROL_PPRINT_EN 14 171 #define CONTROL_PPR_EN 15 172 #define CONTROL_GT_EN 16 173 #define CONTROL_GA_EN 17 174 #define CONTROL_GAM_EN 25 175 #define CONTROL_GALOG_EN 28 176 #define CONTROL_GAINT_EN 29 177 #define CONTROL_NUM_INT_REMAP_MODE 43 178 #define CONTROL_NUM_INT_REMAP_MODE_MASK 0x03 179 #define CONTROL_NUM_INT_REMAP_MODE_2K 0x01 180 #define CONTROL_EPH_EN 45 181 #define CONTROL_XT_EN 50 182 #define CONTROL_INTCAPXT_EN 51 183 #define CONTROL_GCR3TRPMODE 58 184 #define CONTROL_IRTCACHEDIS 59 185 #define CONTROL_SNPAVIC_EN 61 186 187 #define CTRL_INV_TO_MASK 7 188 #define CTRL_INV_TO_NONE 0 189 #define CTRL_INV_TO_1MS 1 190 #define CTRL_INV_TO_10MS 2 191 #define CTRL_INV_TO_100MS 3 192 #define CTRL_INV_TO_1S 4 193 #define CTRL_INV_TO_10S 5 194 #define CTRL_INV_TO_100S 6 195 196 /* command specific defines */ 197 #define CMD_COMPL_WAIT 0x01 198 #define CMD_INV_DEV_ENTRY 0x02 199 #define CMD_INV_IOMMU_PAGES 0x03 200 #define CMD_INV_IOTLB_PAGES 0x04 201 #define CMD_INV_IRT 0x05 202 #define CMD_COMPLETE_PPR 0x07 203 #define CMD_INV_ALL 0x08 204 205 #define CMD_COMPL_WAIT_STORE_MASK 0x01 206 #define CMD_COMPL_WAIT_INT_MASK 0x02 207 #define CMD_INV_IOMMU_PAGES_SIZE_MASK 0x01 208 #define CMD_INV_IOMMU_PAGES_PDE_MASK 0x02 209 #define CMD_INV_IOMMU_PAGES_GN_MASK 0x04 210 211 #define PPR_STATUS_MASK 0xf 212 #define PPR_STATUS_SHIFT 12 213 214 #define CMD_INV_IOMMU_ALL_PAGES_ADDRESS 0x7ffffffffffff000ULL 215 216 /* macros and definitions for device table entries */ 217 #define DEV_ENTRY_VALID 0x00 218 #define DEV_ENTRY_TRANSLATION 0x01 219 #define DEV_ENTRY_HAD 0x07 220 #define DEV_ENTRY_PPR 0x34 221 #define DEV_ENTRY_IR 0x3d 222 #define DEV_ENTRY_IW 0x3e 223 #define DEV_ENTRY_NO_PAGE_FAULT 0x62 224 #define DEV_ENTRY_SYSMGT1 0x68 225 #define DEV_ENTRY_SYSMGT2 0x69 226 #define DTE_DATA1_SYSMGT_MASK GENMASK_ULL(41, 40) 227 228 #define DEV_ENTRY_IRQ_TBL_EN 0x80 229 #define DEV_ENTRY_INIT_PASS 0xb8 230 #define DEV_ENTRY_EINT_PASS 0xb9 231 #define DEV_ENTRY_NMI_PASS 0xba 232 #define DEV_ENTRY_LINT0_PASS 0xbe 233 #define DEV_ENTRY_LINT1_PASS 0xbf 234 #define DEV_ENTRY_MODE_MASK 0x07 235 #define DEV_ENTRY_MODE_SHIFT 0x09 236 237 #define MAX_DEV_TABLE_ENTRIES 0xffff 238 239 /* constants to configure the command buffer */ 240 #define CMD_BUFFER_SIZE 8192 241 #define CMD_BUFFER_ENTRIES 512 242 #define MMIO_CMD_SIZE_SHIFT 56 243 #define MMIO_CMD_SIZE_512 (0x9ULL << MMIO_CMD_SIZE_SHIFT) 244 #define MMIO_CMD_HEAD_MASK GENMASK_ULL(18, 4) /* Command buffer head ptr field [18:4] */ 245 #define MMIO_CMD_BUFFER_HEAD(x) FIELD_GET(MMIO_CMD_HEAD_MASK, (x)) 246 #define MMIO_CMD_TAIL_MASK GENMASK_ULL(18, 4) /* Command buffer tail ptr field [18:4] */ 247 #define MMIO_CMD_BUFFER_TAIL(x) FIELD_GET(MMIO_CMD_TAIL_MASK, (x)) 248 249 /* constants for event buffer handling */ 250 #define EVTLOG_ENTRY_SIZE 0x10 251 #define EVTLOG_SIZE_SHIFT 56 252 #define EVTLOG_SIZE_DEF SZ_8K /* 512 entries */ 253 #define EVTLOG_LEN_MASK_DEF (0x9ULL << EVTLOG_SIZE_SHIFT) 254 #define EVTLOG_SIZE_MAX SZ_512K /* 32K entries */ 255 #define EVTLOG_LEN_MASK_MAX (0xFULL << EVTLOG_SIZE_SHIFT) 256 257 /* Constants for PPR Log handling */ 258 #define PPRLOG_ENTRY_SIZE 0x10 259 #define PPRLOG_SIZE_SHIFT 56 260 #define PPRLOG_SIZE_DEF SZ_8K /* 512 entries */ 261 #define PPRLOG_LEN_MASK_DEF (0x9ULL << PPRLOG_SIZE_SHIFT) 262 #define PPRLOG_SIZE_MAX SZ_512K /* 32K entries */ 263 #define PPRLOG_LEN_MASK_MAX (0xFULL << PPRLOG_SIZE_SHIFT) 264 265 /* PAGE_SERVICE_REQUEST PPR Log Buffer Entry flags */ 266 #define PPR_FLAG_EXEC 0x002 /* Execute permission requested */ 267 #define PPR_FLAG_READ 0x004 /* Read permission requested */ 268 #define PPR_FLAG_WRITE 0x020 /* Write permission requested */ 269 #define PPR_FLAG_US 0x040 /* 1: User, 0: Supervisor */ 270 #define PPR_FLAG_RVSD 0x080 /* Reserved bit not zero */ 271 #define PPR_FLAG_GN 0x100 /* GVA and PASID is valid */ 272 273 #define PPR_REQ_TYPE(x) (((x) >> 60) & 0xfULL) 274 #define PPR_FLAGS(x) (((x) >> 48) & 0xfffULL) 275 #define PPR_DEVID(x) ((x) & 0xffffULL) 276 #define PPR_TAG(x) (((x) >> 32) & 0x1ffULL) 277 #define PPR_TAG_LAST_PAGE(x) (((x) >> 32) & 0x200ULL) 278 #define PPR_PASID1(x) (((x) >> 16) & 0xffffULL) 279 #define PPR_PASID2(x) (((x) >> 42) & 0xfULL) 280 #define PPR_PASID(x) ((PPR_PASID2(x) << 16) | PPR_PASID1(x)) 281 282 #define PPR_REQ_FAULT 0x01 283 284 /* Constants for GA Log handling */ 285 #define GA_LOG_ENTRIES 512 286 #define GA_LOG_SIZE_SHIFT 56 287 #define GA_LOG_SIZE_512 (0x8ULL << GA_LOG_SIZE_SHIFT) 288 #define GA_ENTRY_SIZE 8 289 #define GA_LOG_SIZE (GA_ENTRY_SIZE * GA_LOG_ENTRIES) 290 291 #define GA_TAG(x) (u32)(x & 0xffffffffULL) 292 #define GA_DEVID(x) (u16)(((x) >> 32) & 0xffffULL) 293 #define GA_REQ_TYPE(x) (((x) >> 60) & 0xfULL) 294 295 #define GA_GUEST_NR 0x1 296 297 /* 298 * This bitmap is used to advertise the page sizes our hardware support 299 * to the IOMMU core, which will then use this information to split 300 * physically contiguous memory regions it is mapping into page sizes 301 * that we support. 302 * 303 * 512GB Pages are not supported due to a hardware bug 304 * Page sizes >= the 52 bit max physical address of the CPU are not supported. 305 */ 306 #define AMD_IOMMU_PGSIZES (GENMASK_ULL(51, 12) ^ SZ_512G) 307 308 /* Special mode where page-sizes are limited to 4 KiB */ 309 #define AMD_IOMMU_PGSIZES_4K (PAGE_SIZE) 310 311 /* 4K, 2MB, 1G page sizes are supported */ 312 #define AMD_IOMMU_PGSIZES_V2 (PAGE_SIZE | (1ULL << 21) | (1ULL << 30)) 313 314 /* Bit value definition for dte irq remapping fields*/ 315 #define DTE_IRQ_PHYS_ADDR_MASK GENMASK_ULL(51, 6) 316 #define DTE_IRQ_REMAP_INTCTL_MASK (0x3ULL << 60) 317 #define DTE_IRQ_REMAP_INTCTL (2ULL << 60) 318 #define DTE_IRQ_REMAP_ENABLE 1ULL 319 320 #define DTE_INTTAB_ALIGNMENT 128 321 #define DTE_INTTABLEN_MASK (0xfULL << 1) 322 #define DTE_INTTABLEN_VALUE_512 9ULL 323 #define DTE_INTTABLEN_512 (DTE_INTTABLEN_VALUE_512 << 1) 324 #define MAX_IRQS_PER_TABLE_512 BIT(DTE_INTTABLEN_VALUE_512) 325 #define DTE_INTTABLEN_VALUE_2K 11ULL 326 #define DTE_INTTABLEN_2K (DTE_INTTABLEN_VALUE_2K << 1) 327 #define MAX_IRQS_PER_TABLE_2K BIT(DTE_INTTABLEN_VALUE_2K) 328 329 #define PAGE_MODE_NONE 0x00 330 #define PAGE_MODE_1_LEVEL 0x01 331 #define PAGE_MODE_2_LEVEL 0x02 332 #define PAGE_MODE_3_LEVEL 0x03 333 #define PAGE_MODE_4_LEVEL 0x04 334 #define PAGE_MODE_5_LEVEL 0x05 335 #define PAGE_MODE_6_LEVEL 0x06 336 #define PAGE_MODE_7_LEVEL 0x07 337 338 #define GUEST_PGTABLE_4_LEVEL 0x00 339 #define GUEST_PGTABLE_5_LEVEL 0x01 340 341 #define PM_ADDR_MASK 0x000ffffffffff000ULL 342 343 /* 344 * Bit value definition for DTE fields 345 */ 346 #define DTE_FLAG_V BIT_ULL(0) 347 #define DTE_FLAG_TV BIT_ULL(1) 348 #define DTE_FLAG_HAD (3ULL << 7) 349 #define DTE_MODE_MASK GENMASK_ULL(11, 9) 350 #define DTE_HOST_TRP GENMASK_ULL(51, 12) 351 #define DTE_FLAG_PPR BIT_ULL(52) 352 #define DTE_FLAG_GIOV BIT_ULL(54) 353 #define DTE_FLAG_GV BIT_ULL(55) 354 #define DTE_GLX GENMASK_ULL(57, 56) 355 #define DTE_FLAG_IR BIT_ULL(61) 356 #define DTE_FLAG_IW BIT_ULL(62) 357 358 #define DTE_FLAG_IOTLB BIT_ULL(32) 359 #define DTE_FLAG_MASK (0x3ffULL << 32) 360 #define DTE_DOMID_MASK GENMASK_ULL(15, 0) 361 362 #define DTE_GCR3_14_12 GENMASK_ULL(60, 58) 363 #define DTE_GCR3_30_15 GENMASK_ULL(31, 16) 364 #define DTE_GCR3_51_31 GENMASK_ULL(63, 43) 365 366 #define DTE_GPT_LEVEL_SHIFT 54 367 #define DTE_GPT_LEVEL_MASK GENMASK_ULL(55, 54) 368 369 #define GCR3_VALID 0x01ULL 370 371 /* DTE[128:179] | DTE[184:191] */ 372 #define DTE_DATA2_INTR_MASK ~GENMASK_ULL(55, 52) 373 374 #define IOMMU_PROT_MASK 0x03 375 #define IOMMU_PROT_IR 0x01 376 #define IOMMU_PROT_IW 0x02 377 378 /* IOMMU capabilities */ 379 #define IOMMU_CAP_IOTLB 24 380 #define IOMMU_CAP_NPCACHE 26 381 #define IOMMU_CAP_EFR 27 382 383 /* IOMMU IVINFO */ 384 #define IOMMU_IVINFO_OFFSET 36 385 #define IOMMU_IVINFO_EFRSUP BIT(0) 386 #define IOMMU_IVINFO_DMA_REMAP BIT(1) 387 #define IOMMU_IVINFO_VASIZE GENMASK_ULL(21, 15) 388 389 /* IOMMU Feature Reporting Field (for IVHD type 10h */ 390 #define IOMMU_FEAT_GASUP_SHIFT 6 391 392 /* IOMMU HATDIS for IVHD type 11h and 40h */ 393 #define IOMMU_IVHD_ATTR_HATDIS_SHIFT 0 394 395 /* IOMMU Extended Feature Register (EFR) */ 396 #define IOMMU_EFR_XTSUP_SHIFT 2 397 #define IOMMU_EFR_GASUP_SHIFT 7 398 #define IOMMU_EFR_MSICAPMMIOSUP_SHIFT 46 399 400 #define MAX_DOMAIN_ID 65536 401 402 /* Timeout stuff */ 403 #define LOOP_TIMEOUT 100000 404 #define MMIO_STATUS_TIMEOUT 2000000 405 406 extern bool amd_iommu_dump; 407 #define DUMP_printk(format, arg...) \ 408 do { \ 409 if (amd_iommu_dump) \ 410 pr_info(format, ## arg); \ 411 } while(0); 412 413 /* SNP page mode 0 support */ 414 extern bool amd_iommu_snp_mode0_sup; 415 416 /* global flag if IOMMUs cache non-present entries */ 417 extern bool amd_iommu_np_cache; 418 /* Only true if all IOMMUs support device IOTLBs */ 419 extern bool amd_iommu_iotlb_sup; 420 421 struct irq_remap_table { 422 raw_spinlock_t lock; 423 unsigned min_index; 424 u32 *table; 425 }; 426 427 /* Interrupt remapping feature used? */ 428 extern bool amd_iommu_irq_remap; 429 430 extern const struct iommu_ops amd_iommu_ops; 431 432 /* IVRS indicates that pre-boot remapping was enabled */ 433 extern bool amdr_ivrs_remap_support; 434 435 #define PCI_SBDF_TO_SEGID(sbdf) (((sbdf) >> 16) & 0xffff) 436 #define PCI_SBDF_TO_DEVID(sbdf) ((sbdf) & 0xffff) 437 #define PCI_SEG_DEVID_TO_SBDF(seg, devid) ((((u32)(seg) & 0xffff) << 16) | \ 438 ((devid) & 0xffff)) 439 440 /* Make iterating over all pci segment easier */ 441 #define for_each_pci_segment(pci_seg) \ 442 list_for_each_entry((pci_seg), &amd_iommu_pci_seg_list, list) 443 #define for_each_pci_segment_safe(pci_seg, next) \ 444 list_for_each_entry_safe((pci_seg), (next), &amd_iommu_pci_seg_list, list) 445 /* 446 * Make iterating over all IOMMUs easier 447 */ 448 #define for_each_iommu(iommu) \ 449 list_for_each_entry((iommu), &amd_iommu_list, list) 450 #define for_each_iommu_safe(iommu, next) \ 451 list_for_each_entry_safe((iommu), (next), &amd_iommu_list, list) 452 /* Making iterating over protection_domain->dev_data_list easier */ 453 #define for_each_pdom_dev_data(pdom_dev_data, pdom) \ 454 list_for_each_entry(pdom_dev_data, &pdom->dev_data_list, list) 455 #define for_each_pdom_dev_data_safe(pdom_dev_data, next, pdom) \ 456 list_for_each_entry_safe((pdom_dev_data), (next), &pdom->dev_data_list, list) 457 458 #define for_each_ivhd_dte_flags(entry) \ 459 list_for_each_entry((entry), &amd_ivhd_dev_flags_list, list) 460 461 struct amd_iommu; 462 struct iommu_domain; 463 struct irq_domain; 464 struct amd_irte_ops; 465 466 #define AMD_IOMMU_FLAG_TRANS_PRE_ENABLED (1 << 0) 467 468 struct gcr3_tbl_info { 469 u64 *gcr3_tbl; /* Guest CR3 table */ 470 int glx; /* Number of levels for GCR3 table */ 471 u32 pasid_cnt; /* Track attached PASIDs */ 472 u16 domid; /* Per device domain ID */ 473 }; 474 475 enum protection_domain_mode { 476 PD_MODE_NONE, 477 PD_MODE_V1, 478 PD_MODE_V2, 479 }; 480 481 /* Track dev_data/PASID list for the protection domain */ 482 struct pdom_dev_data { 483 /* Points to attached device data */ 484 struct iommu_dev_data *dev_data; 485 /* PASID attached to the protection domain */ 486 ioasid_t pasid; 487 /* For protection_domain->dev_data_list */ 488 struct list_head list; 489 }; 490 491 /* Keeps track of the IOMMUs attached to protection domain */ 492 struct pdom_iommu_info { 493 struct amd_iommu *iommu; /* IOMMUs attach to protection domain */ 494 u32 refcnt; /* Count of attached dev/pasid per domain/IOMMU */ 495 }; 496 497 struct amd_iommu_viommu { 498 struct iommufd_viommu core; 499 struct protection_domain *parent; /* nest parent domain for this viommu */ 500 struct list_head pdom_list; /* For protection_domain->viommu_list */ 501 502 /* 503 * Per-vIOMMU guest domain ID to host domain ID mapping. 504 * Indexed by guest domain ID. 505 */ 506 struct xarray gdomid_array; 507 }; 508 509 /* 510 * Contains guest domain ID mapping info, 511 * which is stored in the struct xarray gdomid_array. 512 */ 513 struct guest_domain_mapping_info { 514 refcount_t users; 515 u32 hdom_id; /* Host domain ID */ 516 }; 517 518 /* 519 * Nested domain is specifically used for nested translation 520 */ 521 struct nested_domain { 522 struct iommu_domain domain; /* generic domain handle used by iommu core code */ 523 u16 gdom_id; /* domain ID from gDTE */ 524 struct guest_domain_mapping_info *gdom_info; 525 struct iommu_hwpt_amd_guest gdte; /* Guest vIOMMU DTE */ 526 struct amd_iommu_viommu *viommu; /* AMD hw-viommu this nested domain belong to */ 527 }; 528 529 /* 530 * This structure contains generic data for IOMMU protection domains 531 * independent of their use. 532 */ 533 struct protection_domain { 534 union { 535 struct iommu_domain domain; 536 struct pt_iommu iommu; 537 struct pt_iommu_amdv1 amdv1; 538 struct pt_iommu_x86_64 amdv2; 539 }; 540 struct list_head dev_list; /* List of all devices in this domain */ 541 spinlock_t lock; /* mostly used to lock the page table*/ 542 u16 id; /* the domain id written to the device table */ 543 enum protection_domain_mode pd_mode; /* Track page table type */ 544 bool dirty_tracking; /* dirty tracking is enabled in the domain */ 545 struct xarray iommu_array; /* per-IOMMU reference count */ 546 547 struct mmu_notifier mn; /* mmu notifier for the SVA domain */ 548 struct list_head dev_data_list; /* List of pdom_dev_data */ 549 550 /* 551 * Store reference to list of vIOMMUs, which use this protection domain. 552 * This will be used to look up host domain ID when flushing this domain. 553 */ 554 struct list_head viommu_list; 555 }; 556 PT_IOMMU_CHECK_DOMAIN(struct protection_domain, iommu, domain); 557 PT_IOMMU_CHECK_DOMAIN(struct protection_domain, amdv1.iommu, domain); 558 PT_IOMMU_CHECK_DOMAIN(struct protection_domain, amdv2.iommu, domain); 559 560 /* 561 * This structure contains information about one PCI segment in the system. 562 */ 563 struct amd_iommu_pci_seg { 564 /* List with all PCI segments in the system */ 565 struct list_head list; 566 567 /* List of all available dev_data structures */ 568 struct llist_head dev_data_list; 569 570 /* PCI segment number */ 571 u16 id; 572 573 /* Largest PCI device id we expect translation requests for */ 574 u16 last_bdf; 575 576 /* Size of the device table */ 577 u32 dev_table_size; 578 579 /* 580 * device table virtual address 581 * 582 * Pointer to the per PCI segment device table. 583 * It is indexed by the PCI device id or the HT unit id and contains 584 * information about the domain the device belongs to as well as the 585 * page table root pointer. 586 */ 587 struct dev_table_entry *dev_table; 588 589 /* 590 * The rlookup iommu table is used to find the IOMMU which is 591 * responsible for a specific device. It is indexed by the PCI 592 * device id. 593 */ 594 struct amd_iommu **rlookup_table; 595 596 /* 597 * This table is used to find the irq remapping table for a given 598 * device id quickly. 599 */ 600 struct irq_remap_table **irq_lookup_table; 601 602 /* 603 * Pointer to a device table which the content of old device table 604 * will be copied to. It's only be used in kdump kernel. 605 */ 606 struct dev_table_entry *old_dev_tbl_cpy; 607 608 /* 609 * The alias table is a driver specific data structure which contains the 610 * mappings of the PCI device ids to the actual requestor ids on the IOMMU. 611 * More than one device can share the same requestor id. 612 */ 613 u16 *alias_table; 614 615 /* 616 * A list of required unity mappings we find in ACPI. It is not locked 617 * because as runtime it is only read. It is created at ACPI table 618 * parsing time. 619 */ 620 struct list_head unity_map; 621 }; 622 623 /* 624 * Structure where we save information about one hardware AMD IOMMU in the 625 * system. 626 */ 627 struct amd_iommu { 628 struct list_head list; 629 630 /* Index within the IOMMU array */ 631 int index; 632 633 /* locks the accesses to the hardware */ 634 raw_spinlock_t lock; 635 636 /* Pointer to PCI device of this IOMMU */ 637 struct pci_dev *dev; 638 639 /* Cache pdev to root device for resume quirks */ 640 struct pci_dev *root_pdev; 641 642 /* physical address of MMIO space */ 643 u64 mmio_phys; 644 645 /* physical end address of MMIO space */ 646 u64 mmio_phys_end; 647 648 /* virtual address of MMIO space */ 649 u8 __iomem *mmio_base; 650 651 /* capabilities of that IOMMU read from ACPI */ 652 u32 cap; 653 654 /* flags read from acpi table */ 655 u8 acpi_flags; 656 657 /* Extended features */ 658 u64 features; 659 660 /* Extended features 2 */ 661 u64 features2; 662 663 /* Devices requesting PerfOpt; the shared PERF_OPT_EN bit is on while >0. Protected by @lock. */ 664 int perfopt_refcount; 665 666 /* PCI device id of the IOMMU device */ 667 u16 devid; 668 669 /* 670 * Capability pointer. There could be more than one IOMMU per PCI 671 * device function if there are more than one AMD IOMMU capability 672 * pointers. 673 */ 674 u16 cap_ptr; 675 676 /* pci domain of this IOMMU */ 677 struct amd_iommu_pci_seg *pci_seg; 678 679 /* command buffer virtual address */ 680 u8 *cmd_buf; 681 u32 cmd_buf_head; 682 u32 cmd_buf_tail; 683 684 /* event buffer virtual address */ 685 u8 *evt_buf; 686 687 /* Name for event log interrupt */ 688 unsigned char evt_irq_name[16]; 689 690 /* Base of the PPR log, if present */ 691 u8 *ppr_log; 692 693 /* Name for PPR log interrupt */ 694 unsigned char ppr_irq_name[16]; 695 696 /* Base of the GA log, if present */ 697 u8 *ga_log; 698 699 /* Name for GA log interrupt */ 700 unsigned char ga_irq_name[16]; 701 702 /* Tail of the GA log, if present */ 703 u8 *ga_log_tail; 704 705 /* true if interrupts for this IOMMU are already enabled */ 706 bool int_enabled; 707 708 /* if one, we need to send a completion wait command */ 709 bool need_sync; 710 711 /* true if disable irte caching */ 712 bool irtcachedis_enabled; 713 714 /* Handle for IOMMU core code */ 715 struct iommu_device iommu; 716 717 /* 718 * We can't rely on the BIOS to restore all values on reinit, so we 719 * need to stash them 720 */ 721 722 /* The iommu BAR */ 723 u32 stored_addr_lo; 724 u32 stored_addr_hi; 725 726 /* 727 * Each iommu has 6 l1s, each of which is documented as having 0x12 728 * registers 729 */ 730 u32 stored_l1[6][0x12]; 731 732 /* The l2 indirect registers */ 733 u32 stored_l2[0x83]; 734 735 /* The maximum PC banks and counters/bank (PCSup=1) */ 736 u8 max_banks; 737 u8 max_counters; 738 #ifdef CONFIG_IRQ_REMAP 739 struct irq_domain *ir_domain; 740 741 struct amd_irte_ops *irte_ops; 742 #endif 743 744 u32 flags; 745 volatile u64 *cmd_sem; 746 u64 cmd_sem_val; 747 /* 748 * Track physical address to directly use it in build_completion_wait() 749 * and avoid adding any special checks and handling for kdump. 750 */ 751 u64 cmd_sem_paddr; 752 753 #ifdef CONFIG_AMD_IOMMU_DEBUGFS 754 /* DebugFS Info */ 755 struct dentry *debugfs; 756 int dbg_mmio_offset; 757 int dbg_cap_offset; 758 #endif 759 760 /* IOPF support */ 761 struct iopf_queue *iopf_queue; 762 unsigned char iopfq_name[32]; 763 }; 764 765 static inline struct amd_iommu *dev_to_amd_iommu(struct device *dev) 766 { 767 struct iommu_device *iommu = dev_to_iommu_device(dev); 768 769 return container_of(iommu, struct amd_iommu, iommu); 770 } 771 772 #define ACPIHID_UID_LEN 256 773 #define ACPIHID_HID_LEN 9 774 775 struct acpihid_map_entry { 776 struct list_head list; 777 u8 uid[ACPIHID_UID_LEN]; 778 u8 hid[ACPIHID_HID_LEN]; 779 u32 devid; 780 u32 root_devid; 781 bool cmd_line; 782 struct iommu_group *group; 783 }; 784 785 struct devid_map { 786 struct list_head list; 787 u8 id; 788 u32 devid; 789 bool cmd_line; 790 }; 791 792 #define AMD_IOMMU_DEVICE_FLAG_ATS_SUP 0x1 /* ATS feature supported */ 793 #define AMD_IOMMU_DEVICE_FLAG_PRI_SUP 0x2 /* PRI feature supported */ 794 #define AMD_IOMMU_DEVICE_FLAG_PASID_SUP 0x4 /* PASID context supported */ 795 /* Device may request execution on memory pages */ 796 #define AMD_IOMMU_DEVICE_FLAG_EXEC_SUP 0x8 797 /* Device may request super-user privileges */ 798 #define AMD_IOMMU_DEVICE_FLAG_PRIV_SUP 0x10 799 800 /* 801 * This struct contains device specific data for the IOMMU 802 */ 803 struct iommu_dev_data { 804 /*Protect against attach/detach races */ 805 struct mutex mutex; 806 spinlock_t dte_lock; /* DTE lock for 256-bit access */ 807 808 struct list_head list; /* For domain->dev_list */ 809 struct llist_node dev_data_list; /* For global dev_data_list */ 810 struct protection_domain *domain; /* Domain the device is bound to */ 811 struct gcr3_tbl_info gcr3_info; /* Per-device GCR3 table */ 812 struct device *dev; 813 u16 devid; /* PCI Device ID */ 814 815 unsigned int max_irqs; /* Maximum IRQs supported by device */ 816 u32 max_pasids; /* Max supported PASIDs */ 817 u32 flags; /* Holds AMD_IOMMU_DEVICE_FLAG_<*> */ 818 int ats_qdep; 819 u8 ats_enabled :1; /* ATS state */ 820 u8 pri_enabled :1; /* PRI state */ 821 u8 pasid_enabled:1; /* PASID state */ 822 u8 pri_tlp :1; /* PASID TLB required for 823 PPR completions */ 824 u8 ppr :1; /* Enable device PPR support */ 825 bool use_vapic; /* Enable device to use vapic mode */ 826 bool defer_attach; 827 bool perfopt; 828 829 struct ratelimit_state rs; /* Ratelimit IOPF messages */ 830 }; 831 832 /* Map HPET and IOAPIC ids to the devid used by the IOMMU */ 833 extern struct list_head ioapic_map; 834 extern struct list_head hpet_map; 835 extern struct list_head acpihid_map; 836 837 /* 838 * List with all PCI segments in the system. This list is not locked because 839 * it is only written at driver initialization time 840 */ 841 extern struct list_head amd_iommu_pci_seg_list; 842 843 /* 844 * List with all IOMMUs in the system. This list is not locked because it is 845 * only written and read at driver initialization or suspend time 846 */ 847 extern struct list_head amd_iommu_list; 848 849 /* 850 * Structure defining one entry in the device table 851 */ 852 struct dev_table_entry { 853 union { 854 u64 data[4]; 855 u128 data128[2]; 856 }; 857 }; 858 859 /* 860 * Structure defining one entry in the command buffer 861 */ 862 struct iommu_cmd { 863 u32 data[4]; 864 }; 865 866 /* 867 * Structure to sture persistent DTE flags from IVHD 868 */ 869 struct ivhd_dte_flags { 870 struct list_head list; 871 u16 segid; 872 u16 devid_first; 873 u16 devid_last; 874 struct dev_table_entry dte; 875 }; 876 877 /* 878 * One entry for unity mappings parsed out of the ACPI table. 879 */ 880 struct unity_map_entry { 881 struct list_head list; 882 883 /* starting device id this entry is used for (including) */ 884 u16 devid_start; 885 /* end device id this entry is used for (including) */ 886 u16 devid_end; 887 888 /* start address to unity map (including) */ 889 u64 address_start; 890 /* end address to unity map (including) */ 891 u64 address_end; 892 893 /* required protection */ 894 int prot; 895 }; 896 897 /* 898 * Data structures for device handling 899 */ 900 901 extern bool amd_iommu_force_isolation; 902 903 /* Max levels of glxval supported */ 904 extern int amd_iommu_max_glx_val; 905 906 /* IDA to track protection domain IDs */ 907 extern struct ida pdom_ids; 908 909 /* Global EFR and EFR2 registers */ 910 extern u64 amd_iommu_efr; 911 extern u64 amd_iommu_efr2; 912 913 static inline int get_ioapic_devid(int id) 914 { 915 struct devid_map *entry; 916 917 list_for_each_entry(entry, &ioapic_map, list) { 918 if (entry->id == id) 919 return entry->devid; 920 } 921 922 return -EINVAL; 923 } 924 925 static inline int get_hpet_devid(int id) 926 { 927 struct devid_map *entry; 928 929 list_for_each_entry(entry, &hpet_map, list) { 930 if (entry->id == id) 931 return entry->devid; 932 } 933 934 return -EINVAL; 935 } 936 937 enum amd_iommu_intr_mode_type { 938 /* 939 * The legacy format mode is not visible to users to prevent the user 940 * from crashing x2APIC systems, which for all intents and purposes 941 * require 128-bit IRTEs. The legacy format will be forced as needed 942 * when hardware doesn't support 128-bit IRTEs. 943 */ 944 AMD_IOMMU_GUEST_IR_LEGACY, 945 AMD_IOMMU_GUEST_IR_LEGACY_GA, 946 AMD_IOMMU_GUEST_IR_VAPIC, 947 }; 948 949 #define AMD_IOMMU_GUEST_IR_GA(x) (x == AMD_IOMMU_GUEST_IR_VAPIC || \ 950 x == AMD_IOMMU_GUEST_IR_LEGACY_GA) 951 952 #define AMD_IOMMU_GUEST_IR_VAPIC(x) (x == AMD_IOMMU_GUEST_IR_VAPIC) 953 954 union irte { 955 u32 val; 956 struct { 957 u32 valid : 1, 958 no_fault : 1, 959 int_type : 3, 960 rq_eoi : 1, 961 dm : 1, 962 rsvd_1 : 1, 963 destination : 8, 964 vector : 8, 965 rsvd_2 : 8; 966 } fields; 967 }; 968 969 #define APICID_TO_IRTE_DEST_LO(x) (x & 0xffffff) 970 #define APICID_TO_IRTE_DEST_HI(x) ((x >> 24) & 0xff) 971 972 union irte_ga_lo { 973 u64 val; 974 975 /* For int remapping */ 976 struct { 977 u64 valid : 1, 978 no_fault : 1, 979 /* ------ */ 980 int_type : 3, 981 rq_eoi : 1, 982 dm : 1, 983 /* ------ */ 984 guest_mode : 1, 985 destination : 24, 986 ga_tag : 32; 987 } fields_remap; 988 989 /* For guest vAPIC */ 990 struct { 991 u64 valid : 1, 992 no_fault : 1, 993 /* ------ */ 994 ga_log_intr : 1, 995 rsvd1 : 3, 996 is_run : 1, 997 /* ------ */ 998 guest_mode : 1, 999 destination : 24, 1000 ga_tag : 32; 1001 } fields_vapic; 1002 }; 1003 1004 union irte_ga_hi { 1005 u64 val; 1006 struct { 1007 u64 vector : 8, 1008 rsvd_1 : 4, 1009 ga_root_ptr : 40, 1010 rsvd_2 : 4, 1011 destination : 8; 1012 } fields; 1013 }; 1014 1015 struct irte_ga { 1016 union { 1017 struct { 1018 union irte_ga_lo lo; 1019 union irte_ga_hi hi; 1020 }; 1021 u128 irte; 1022 }; 1023 }; 1024 1025 struct irq_2_irte { 1026 u16 devid; /* Device ID for IRTE table */ 1027 u16 index; /* Index into IRTE table*/ 1028 }; 1029 1030 struct amd_ir_data { 1031 struct amd_iommu *iommu; 1032 struct irq_2_irte irq_2_irte; 1033 struct msi_msg msi_entry; 1034 void *entry; /* Pointer to union irte or struct irte_ga */ 1035 1036 /** 1037 * Store information for activate/de-activate 1038 * Guest virtual APIC mode during runtime. 1039 */ 1040 struct irq_cfg *cfg; 1041 int ga_vector; 1042 u64 ga_root_ptr; 1043 u32 ga_tag; 1044 }; 1045 1046 struct amd_irte_ops { 1047 void (*prepare)(void *, u32, bool, u8, u32, int); 1048 void (*activate)(struct amd_iommu *iommu, void *, u16, u16); 1049 void (*deactivate)(struct amd_iommu *iommu, void *, u16, u16); 1050 void (*set_affinity)(struct amd_iommu *iommu, void *, u16, u16, u8, u32); 1051 void *(*get)(struct irq_remap_table *, int); 1052 void (*set_allocated)(struct irq_remap_table *, int); 1053 bool (*is_allocated)(struct irq_remap_table *, int); 1054 void (*clear_allocated)(struct irq_remap_table *, int); 1055 }; 1056 1057 #ifdef CONFIG_IRQ_REMAP 1058 extern struct amd_irte_ops irte_32_ops; 1059 extern struct amd_irte_ops irte_128_ops; 1060 #endif 1061 1062 #endif /* _ASM_X86_AMD_IOMMU_TYPES_H */ 1063