1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc. 4 * Author: Joerg Roedel <jroedel@suse.de> 5 * Leo Duran <leo.duran@amd.com> 6 */ 7 8 #define pr_fmt(fmt) "AMD-Vi: " fmt 9 #define dev_fmt(fmt) pr_fmt(fmt) 10 11 #include <linux/pci.h> 12 #include <linux/acpi.h> 13 #include <linux/list.h> 14 #include <linux/bitmap.h> 15 #include <linux/syscore_ops.h> 16 #include <linux/interrupt.h> 17 #include <linux/msi.h> 18 #include <linux/irq.h> 19 #include <linux/amd-iommu.h> 20 #include <linux/export.h> 21 #include <linux/kmemleak.h> 22 #include <linux/cc_platform.h> 23 #include <linux/iopoll.h> 24 #include <asm/pci-direct.h> 25 #include <asm/iommu.h> 26 #include <asm/apic.h> 27 #include <asm/gart.h> 28 #include <asm/x86_init.h> 29 #include <asm/io_apic.h> 30 #include <asm/irq_remapping.h> 31 #include <asm/set_memory.h> 32 #include <asm/sev.h> 33 34 #include <linux/crash_dump.h> 35 36 #include "amd_iommu.h" 37 #include "../irq_remapping.h" 38 #include "../iommu-pages.h" 39 40 /* 41 * definitions for the ACPI scanning code 42 */ 43 #define IVRS_HEADER_LENGTH 48 44 45 #define ACPI_IVHD_TYPE_MAX_SUPPORTED 0x40 46 #define ACPI_IVMD_TYPE_ALL 0x20 47 #define ACPI_IVMD_TYPE 0x21 48 #define ACPI_IVMD_TYPE_RANGE 0x22 49 50 #define IVHD_DEV_ALL 0x01 51 #define IVHD_DEV_SELECT 0x02 52 #define IVHD_DEV_SELECT_RANGE_START 0x03 53 #define IVHD_DEV_RANGE_END 0x04 54 #define IVHD_DEV_ALIAS 0x42 55 #define IVHD_DEV_ALIAS_RANGE 0x43 56 #define IVHD_DEV_EXT_SELECT 0x46 57 #define IVHD_DEV_EXT_SELECT_RANGE 0x47 58 #define IVHD_DEV_SPECIAL 0x48 59 #define IVHD_DEV_ACPI_HID 0xf0 60 61 #define UID_NOT_PRESENT 0 62 #define UID_IS_INTEGER 1 63 #define UID_IS_CHARACTER 2 64 65 #define IVHD_SPECIAL_IOAPIC 1 66 #define IVHD_SPECIAL_HPET 2 67 68 #define IVHD_FLAG_HT_TUN_EN_MASK 0x01 69 #define IVHD_FLAG_PASSPW_EN_MASK 0x02 70 #define IVHD_FLAG_RESPASSPW_EN_MASK 0x04 71 #define IVHD_FLAG_ISOC_EN_MASK 0x08 72 73 #define IVMD_FLAG_EXCL_RANGE 0x08 74 #define IVMD_FLAG_IW 0x04 75 #define IVMD_FLAG_IR 0x02 76 #define IVMD_FLAG_UNITY_MAP 0x01 77 78 #define ACPI_DEVFLAG_INITPASS 0x01 79 #define ACPI_DEVFLAG_EXTINT 0x02 80 #define ACPI_DEVFLAG_NMI 0x04 81 #define ACPI_DEVFLAG_SYSMGT1 0x10 82 #define ACPI_DEVFLAG_SYSMGT2 0x20 83 #define ACPI_DEVFLAG_LINT0 0x40 84 #define ACPI_DEVFLAG_LINT1 0x80 85 #define ACPI_DEVFLAG_ATSDIS 0x10000000 86 87 #define IVRS_GET_SBDF_ID(seg, bus, dev, fn) (((seg & 0xffff) << 16) | ((bus & 0xff) << 8) \ 88 | ((dev & 0x1f) << 3) | (fn & 0x7)) 89 90 /* 91 * ACPI table definitions 92 * 93 * These data structures are laid over the table to parse the important values 94 * out of it. 95 */ 96 97 /* 98 * structure describing one IOMMU in the ACPI table. Typically followed by one 99 * or more ivhd_entrys. 100 */ 101 struct ivhd_header { 102 u8 type; 103 u8 flags; 104 u16 length; 105 u16 devid; 106 u16 cap_ptr; 107 u64 mmio_phys; 108 u16 pci_seg; 109 u16 info; 110 u32 efr_attr; 111 112 /* Following only valid on IVHD type 11h and 40h */ 113 u64 efr_reg; /* Exact copy of MMIO_EXT_FEATURES */ 114 u64 efr_reg2; 115 } __attribute__((packed)); 116 117 /* 118 * A device entry describing which devices a specific IOMMU translates and 119 * which requestor ids they use. 120 */ 121 struct ivhd_entry { 122 u8 type; 123 u16 devid; 124 u8 flags; 125 struct_group(ext_hid, 126 u32 ext; 127 u32 hidh; 128 ); 129 u64 cid; 130 u8 uidf; 131 u8 uidl; 132 u8 uid; 133 } __attribute__((packed)); 134 135 /* 136 * An AMD IOMMU memory definition structure. It defines things like exclusion 137 * ranges for devices and regions that should be unity mapped. 138 */ 139 struct ivmd_header { 140 u8 type; 141 u8 flags; 142 u16 length; 143 u16 devid; 144 u16 aux; 145 u16 pci_seg; 146 u8 resv[6]; 147 u64 range_start; 148 u64 range_length; 149 } __attribute__((packed)); 150 151 bool amd_iommu_dump; 152 bool amd_iommu_irq_remap __read_mostly; 153 154 enum protection_domain_mode amd_iommu_pgtable = PD_MODE_V1; 155 /* Guest page table level */ 156 int amd_iommu_gpt_level = PAGE_MODE_4_LEVEL; 157 158 int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC; 159 static int amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE; 160 161 static bool amd_iommu_detected; 162 static bool amd_iommu_disabled __initdata; 163 static bool amd_iommu_force_enable __initdata; 164 static bool amd_iommu_irtcachedis; 165 static int amd_iommu_target_ivhd_type; 166 167 /* Global EFR and EFR2 registers */ 168 u64 amd_iommu_efr; 169 u64 amd_iommu_efr2; 170 171 /* SNP is enabled on the system? */ 172 bool amd_iommu_snp_en; 173 EXPORT_SYMBOL(amd_iommu_snp_en); 174 175 LIST_HEAD(amd_iommu_pci_seg_list); /* list of all PCI segments */ 176 LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the system */ 177 LIST_HEAD(amd_ivhd_dev_flags_list); /* list of all IVHD device entry settings */ 178 179 /* Number of IOMMUs present in the system */ 180 static int amd_iommus_present; 181 182 /* IOMMUs have a non-present cache? */ 183 bool amd_iommu_np_cache __read_mostly; 184 bool amd_iommu_iotlb_sup __read_mostly = true; 185 186 static bool amd_iommu_pc_present __read_mostly; 187 bool amdr_ivrs_remap_support __read_mostly; 188 189 bool amd_iommu_force_isolation __read_mostly; 190 191 unsigned long amd_iommu_pgsize_bitmap __ro_after_init = AMD_IOMMU_PGSIZES; 192 193 enum iommu_init_state { 194 IOMMU_START_STATE, 195 IOMMU_IVRS_DETECTED, 196 IOMMU_ACPI_FINISHED, 197 IOMMU_ENABLED, 198 IOMMU_PCI_INIT, 199 IOMMU_INTERRUPTS_EN, 200 IOMMU_INITIALIZED, 201 IOMMU_NOT_FOUND, 202 IOMMU_INIT_ERROR, 203 IOMMU_CMDLINE_DISABLED, 204 }; 205 206 /* Early ioapic and hpet maps from kernel command line */ 207 #define EARLY_MAP_SIZE 4 208 static struct devid_map __initdata early_ioapic_map[EARLY_MAP_SIZE]; 209 static struct devid_map __initdata early_hpet_map[EARLY_MAP_SIZE]; 210 static struct acpihid_map_entry __initdata early_acpihid_map[EARLY_MAP_SIZE]; 211 212 static int __initdata early_ioapic_map_size; 213 static int __initdata early_hpet_map_size; 214 static int __initdata early_acpihid_map_size; 215 216 static bool __initdata cmdline_maps; 217 218 static enum iommu_init_state init_state = IOMMU_START_STATE; 219 220 static int amd_iommu_enable_interrupts(void); 221 static void init_device_table_dma(struct amd_iommu_pci_seg *pci_seg); 222 223 static bool amd_iommu_pre_enabled = true; 224 225 static u32 amd_iommu_ivinfo __initdata; 226 227 bool translation_pre_enabled(struct amd_iommu *iommu) 228 { 229 return (iommu->flags & AMD_IOMMU_FLAG_TRANS_PRE_ENABLED); 230 } 231 232 static void clear_translation_pre_enabled(struct amd_iommu *iommu) 233 { 234 iommu->flags &= ~AMD_IOMMU_FLAG_TRANS_PRE_ENABLED; 235 } 236 237 static void init_translation_status(struct amd_iommu *iommu) 238 { 239 u64 ctrl; 240 241 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); 242 if (ctrl & (1<<CONTROL_IOMMU_EN)) 243 iommu->flags |= AMD_IOMMU_FLAG_TRANS_PRE_ENABLED; 244 } 245 246 int amd_iommu_get_num_iommus(void) 247 { 248 return amd_iommus_present; 249 } 250 251 /* 252 * Iterate through all the IOMMUs to get common EFR 253 * masks among all IOMMUs and warn if found inconsistency. 254 */ 255 static __init void get_global_efr(void) 256 { 257 struct amd_iommu *iommu; 258 259 for_each_iommu(iommu) { 260 u64 tmp = iommu->features; 261 u64 tmp2 = iommu->features2; 262 263 if (list_is_first(&iommu->list, &amd_iommu_list)) { 264 amd_iommu_efr = tmp; 265 amd_iommu_efr2 = tmp2; 266 continue; 267 } 268 269 if (amd_iommu_efr == tmp && 270 amd_iommu_efr2 == tmp2) 271 continue; 272 273 pr_err(FW_BUG 274 "Found inconsistent EFR/EFR2 %#llx,%#llx (global %#llx,%#llx) on iommu%d (%04x:%02x:%02x.%01x).\n", 275 tmp, tmp2, amd_iommu_efr, amd_iommu_efr2, 276 iommu->index, iommu->pci_seg->id, 277 PCI_BUS_NUM(iommu->devid), PCI_SLOT(iommu->devid), 278 PCI_FUNC(iommu->devid)); 279 280 amd_iommu_efr &= tmp; 281 amd_iommu_efr2 &= tmp2; 282 } 283 284 pr_info("Using global IVHD EFR:%#llx, EFR2:%#llx\n", amd_iommu_efr, amd_iommu_efr2); 285 } 286 287 /* 288 * For IVHD type 0x11/0x40, EFR is also available via IVHD. 289 * Default to IVHD EFR since it is available sooner 290 * (i.e. before PCI init). 291 */ 292 static void __init early_iommu_features_init(struct amd_iommu *iommu, 293 struct ivhd_header *h) 294 { 295 if (amd_iommu_ivinfo & IOMMU_IVINFO_EFRSUP) { 296 iommu->features = h->efr_reg; 297 iommu->features2 = h->efr_reg2; 298 } 299 if (amd_iommu_ivinfo & IOMMU_IVINFO_DMA_REMAP) 300 amdr_ivrs_remap_support = true; 301 } 302 303 /* Access to l1 and l2 indexed register spaces */ 304 305 static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address) 306 { 307 u32 val; 308 309 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16)); 310 pci_read_config_dword(iommu->dev, 0xfc, &val); 311 return val; 312 } 313 314 static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val) 315 { 316 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31)); 317 pci_write_config_dword(iommu->dev, 0xfc, val); 318 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16)); 319 } 320 321 static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address) 322 { 323 u32 val; 324 325 pci_write_config_dword(iommu->dev, 0xf0, address); 326 pci_read_config_dword(iommu->dev, 0xf4, &val); 327 return val; 328 } 329 330 static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val) 331 { 332 pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8)); 333 pci_write_config_dword(iommu->dev, 0xf4, val); 334 } 335 336 /**************************************************************************** 337 * 338 * AMD IOMMU MMIO register space handling functions 339 * 340 * These functions are used to program the IOMMU device registers in 341 * MMIO space required for that driver. 342 * 343 ****************************************************************************/ 344 345 /* 346 * This function set the exclusion range in the IOMMU. DMA accesses to the 347 * exclusion range are passed through untranslated 348 */ 349 static void iommu_set_exclusion_range(struct amd_iommu *iommu) 350 { 351 u64 start = iommu->exclusion_start & PAGE_MASK; 352 u64 limit = (start + iommu->exclusion_length - 1) & PAGE_MASK; 353 u64 entry; 354 355 if (!iommu->exclusion_start) 356 return; 357 358 entry = start | MMIO_EXCL_ENABLE_MASK; 359 memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET, 360 &entry, sizeof(entry)); 361 362 entry = limit; 363 memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET, 364 &entry, sizeof(entry)); 365 } 366 367 static void iommu_set_cwwb_range(struct amd_iommu *iommu) 368 { 369 u64 start = iommu_virt_to_phys((void *)iommu->cmd_sem); 370 u64 entry = start & PM_ADDR_MASK; 371 372 if (!check_feature(FEATURE_SNP)) 373 return; 374 375 /* Note: 376 * Re-purpose Exclusion base/limit registers for Completion wait 377 * write-back base/limit. 378 */ 379 memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET, 380 &entry, sizeof(entry)); 381 382 /* Note: 383 * Default to 4 Kbytes, which can be specified by setting base 384 * address equal to the limit address. 385 */ 386 memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET, 387 &entry, sizeof(entry)); 388 } 389 390 /* Programs the physical address of the device table into the IOMMU hardware */ 391 static void iommu_set_device_table(struct amd_iommu *iommu) 392 { 393 u64 entry; 394 u32 dev_table_size = iommu->pci_seg->dev_table_size; 395 void *dev_table = (void *)get_dev_table(iommu); 396 397 BUG_ON(iommu->mmio_base == NULL); 398 399 entry = iommu_virt_to_phys(dev_table); 400 entry |= (dev_table_size >> 12) - 1; 401 memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET, 402 &entry, sizeof(entry)); 403 } 404 405 static void iommu_feature_set(struct amd_iommu *iommu, u64 val, u64 mask, u8 shift) 406 { 407 u64 ctrl; 408 409 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); 410 mask <<= shift; 411 ctrl &= ~mask; 412 ctrl |= (val << shift) & mask; 413 writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET); 414 } 415 416 /* Generic functions to enable/disable certain features of the IOMMU. */ 417 void iommu_feature_enable(struct amd_iommu *iommu, u8 bit) 418 { 419 iommu_feature_set(iommu, 1ULL, 1ULL, bit); 420 } 421 422 static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit) 423 { 424 iommu_feature_set(iommu, 0ULL, 1ULL, bit); 425 } 426 427 /* Function to enable the hardware */ 428 static void iommu_enable(struct amd_iommu *iommu) 429 { 430 iommu_feature_enable(iommu, CONTROL_IOMMU_EN); 431 } 432 433 static void iommu_disable(struct amd_iommu *iommu) 434 { 435 if (!iommu->mmio_base) 436 return; 437 438 /* Disable command buffer */ 439 iommu_feature_disable(iommu, CONTROL_CMDBUF_EN); 440 441 /* Disable event logging and event interrupts */ 442 iommu_feature_disable(iommu, CONTROL_EVT_INT_EN); 443 iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN); 444 445 /* Disable IOMMU GA_LOG */ 446 iommu_feature_disable(iommu, CONTROL_GALOG_EN); 447 iommu_feature_disable(iommu, CONTROL_GAINT_EN); 448 449 /* Disable IOMMU PPR logging */ 450 iommu_feature_disable(iommu, CONTROL_PPRLOG_EN); 451 iommu_feature_disable(iommu, CONTROL_PPRINT_EN); 452 453 /* Disable IOMMU hardware itself */ 454 iommu_feature_disable(iommu, CONTROL_IOMMU_EN); 455 456 /* Clear IRTE cache disabling bit */ 457 iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS); 458 } 459 460 /* 461 * mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in 462 * the system has one. 463 */ 464 static u8 __iomem * __init iommu_map_mmio_space(u64 address, u64 end) 465 { 466 if (!request_mem_region(address, end, "amd_iommu")) { 467 pr_err("Can not reserve memory region %llx-%llx for mmio\n", 468 address, end); 469 pr_err("This is a BIOS bug. Please contact your hardware vendor\n"); 470 return NULL; 471 } 472 473 return (u8 __iomem *)ioremap(address, end); 474 } 475 476 static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu) 477 { 478 if (iommu->mmio_base) 479 iounmap(iommu->mmio_base); 480 release_mem_region(iommu->mmio_phys, iommu->mmio_phys_end); 481 } 482 483 static inline u32 get_ivhd_header_size(struct ivhd_header *h) 484 { 485 u32 size = 0; 486 487 switch (h->type) { 488 case 0x10: 489 size = 24; 490 break; 491 case 0x11: 492 case 0x40: 493 size = 40; 494 break; 495 } 496 return size; 497 } 498 499 /**************************************************************************** 500 * 501 * The functions below belong to the first pass of AMD IOMMU ACPI table 502 * parsing. In this pass we try to find out the highest device id this 503 * code has to handle. Upon this information the size of the shared data 504 * structures is determined later. 505 * 506 ****************************************************************************/ 507 508 /* 509 * This function calculates the length of a given IVHD entry 510 */ 511 static inline int ivhd_entry_length(u8 *ivhd) 512 { 513 u32 type = ((struct ivhd_entry *)ivhd)->type; 514 515 if (type < 0x80) { 516 return 0x04 << (*ivhd >> 6); 517 } else if (type == IVHD_DEV_ACPI_HID) { 518 /* For ACPI_HID, offset 21 is uid len */ 519 return *((u8 *)ivhd + 21) + 22; 520 } 521 return 0; 522 } 523 524 /* 525 * After reading the highest device id from the IOMMU PCI capability header 526 * this function looks if there is a higher device id defined in the ACPI table 527 */ 528 static int __init find_last_devid_from_ivhd(struct ivhd_header *h) 529 { 530 u8 *p = (void *)h, *end = (void *)h; 531 struct ivhd_entry *dev; 532 int last_devid = -EINVAL; 533 534 u32 ivhd_size = get_ivhd_header_size(h); 535 536 if (!ivhd_size) { 537 pr_err("Unsupported IVHD type %#x\n", h->type); 538 return -EINVAL; 539 } 540 541 p += ivhd_size; 542 end += h->length; 543 544 while (p < end) { 545 dev = (struct ivhd_entry *)p; 546 switch (dev->type) { 547 case IVHD_DEV_ALL: 548 /* Use maximum BDF value for DEV_ALL */ 549 return 0xffff; 550 case IVHD_DEV_SELECT: 551 case IVHD_DEV_RANGE_END: 552 case IVHD_DEV_ALIAS: 553 case IVHD_DEV_EXT_SELECT: 554 /* all the above subfield types refer to device ids */ 555 if (dev->devid > last_devid) 556 last_devid = dev->devid; 557 break; 558 default: 559 break; 560 } 561 p += ivhd_entry_length(p); 562 } 563 564 WARN_ON(p != end); 565 566 return last_devid; 567 } 568 569 static int __init check_ivrs_checksum(struct acpi_table_header *table) 570 { 571 int i; 572 u8 checksum = 0, *p = (u8 *)table; 573 574 for (i = 0; i < table->length; ++i) 575 checksum += p[i]; 576 if (checksum != 0) { 577 /* ACPI table corrupt */ 578 pr_err(FW_BUG "IVRS invalid checksum\n"); 579 return -ENODEV; 580 } 581 582 return 0; 583 } 584 585 /* 586 * Iterate over all IVHD entries in the ACPI table and find the highest device 587 * id which we need to handle. This is the first of three functions which parse 588 * the ACPI table. So we check the checksum here. 589 */ 590 static int __init find_last_devid_acpi(struct acpi_table_header *table, u16 pci_seg) 591 { 592 u8 *p = (u8 *)table, *end = (u8 *)table; 593 struct ivhd_header *h; 594 int last_devid, last_bdf = 0; 595 596 p += IVRS_HEADER_LENGTH; 597 598 end += table->length; 599 while (p < end) { 600 h = (struct ivhd_header *)p; 601 if (h->pci_seg == pci_seg && 602 h->type == amd_iommu_target_ivhd_type) { 603 last_devid = find_last_devid_from_ivhd(h); 604 605 if (last_devid < 0) 606 return -EINVAL; 607 if (last_devid > last_bdf) 608 last_bdf = last_devid; 609 } 610 p += h->length; 611 } 612 WARN_ON(p != end); 613 614 return last_bdf; 615 } 616 617 /**************************************************************************** 618 * 619 * The following functions belong to the code path which parses the ACPI table 620 * the second time. In this ACPI parsing iteration we allocate IOMMU specific 621 * data structures, initialize the per PCI segment device/alias/rlookup table 622 * and also basically initialize the hardware. 623 * 624 ****************************************************************************/ 625 626 /* Allocate per PCI segment device table */ 627 static inline int __init alloc_dev_table(struct amd_iommu_pci_seg *pci_seg) 628 { 629 pci_seg->dev_table = iommu_alloc_pages_sz(GFP_KERNEL | GFP_DMA32, 630 pci_seg->dev_table_size); 631 if (!pci_seg->dev_table) 632 return -ENOMEM; 633 634 return 0; 635 } 636 637 static inline void free_dev_table(struct amd_iommu_pci_seg *pci_seg) 638 { 639 iommu_free_pages(pci_seg->dev_table); 640 pci_seg->dev_table = NULL; 641 } 642 643 /* Allocate per PCI segment IOMMU rlookup table. */ 644 static inline int __init alloc_rlookup_table(struct amd_iommu_pci_seg *pci_seg) 645 { 646 pci_seg->rlookup_table = kvcalloc(pci_seg->last_bdf + 1, 647 sizeof(*pci_seg->rlookup_table), 648 GFP_KERNEL); 649 if (pci_seg->rlookup_table == NULL) 650 return -ENOMEM; 651 652 return 0; 653 } 654 655 static inline void free_rlookup_table(struct amd_iommu_pci_seg *pci_seg) 656 { 657 kvfree(pci_seg->rlookup_table); 658 pci_seg->rlookup_table = NULL; 659 } 660 661 static inline int __init alloc_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg) 662 { 663 pci_seg->irq_lookup_table = kvcalloc(pci_seg->last_bdf + 1, 664 sizeof(*pci_seg->irq_lookup_table), 665 GFP_KERNEL); 666 if (pci_seg->irq_lookup_table == NULL) 667 return -ENOMEM; 668 669 return 0; 670 } 671 672 static inline void free_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg) 673 { 674 kvfree(pci_seg->irq_lookup_table); 675 pci_seg->irq_lookup_table = NULL; 676 } 677 678 static int __init alloc_alias_table(struct amd_iommu_pci_seg *pci_seg) 679 { 680 int i; 681 682 pci_seg->alias_table = kvmalloc_array(pci_seg->last_bdf + 1, 683 sizeof(*pci_seg->alias_table), 684 GFP_KERNEL); 685 if (!pci_seg->alias_table) 686 return -ENOMEM; 687 688 /* 689 * let all alias entries point to itself 690 */ 691 for (i = 0; i <= pci_seg->last_bdf; ++i) 692 pci_seg->alias_table[i] = i; 693 694 return 0; 695 } 696 697 static void __init free_alias_table(struct amd_iommu_pci_seg *pci_seg) 698 { 699 kvfree(pci_seg->alias_table); 700 pci_seg->alias_table = NULL; 701 } 702 703 /* 704 * Allocates the command buffer. This buffer is per AMD IOMMU. We can 705 * write commands to that buffer later and the IOMMU will execute them 706 * asynchronously 707 */ 708 static int __init alloc_command_buffer(struct amd_iommu *iommu) 709 { 710 iommu->cmd_buf = iommu_alloc_pages_sz(GFP_KERNEL, CMD_BUFFER_SIZE); 711 712 return iommu->cmd_buf ? 0 : -ENOMEM; 713 } 714 715 /* 716 * Interrupt handler has processed all pending events and adjusted head 717 * and tail pointer. Reset overflow mask and restart logging again. 718 */ 719 void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type, 720 u8 cntrl_intr, u8 cntrl_log, 721 u32 status_run_mask, u32 status_overflow_mask) 722 { 723 u32 status; 724 725 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); 726 if (status & status_run_mask) 727 return; 728 729 pr_info_ratelimited("IOMMU %s log restarting\n", evt_type); 730 731 iommu_feature_disable(iommu, cntrl_log); 732 iommu_feature_disable(iommu, cntrl_intr); 733 734 writel(status_overflow_mask, iommu->mmio_base + MMIO_STATUS_OFFSET); 735 736 iommu_feature_enable(iommu, cntrl_intr); 737 iommu_feature_enable(iommu, cntrl_log); 738 } 739 740 /* 741 * This function restarts event logging in case the IOMMU experienced 742 * an event log buffer overflow. 743 */ 744 void amd_iommu_restart_event_logging(struct amd_iommu *iommu) 745 { 746 amd_iommu_restart_log(iommu, "Event", CONTROL_EVT_INT_EN, 747 CONTROL_EVT_LOG_EN, MMIO_STATUS_EVT_RUN_MASK, 748 MMIO_STATUS_EVT_OVERFLOW_MASK); 749 } 750 751 /* 752 * This function restarts event logging in case the IOMMU experienced 753 * GA log overflow. 754 */ 755 void amd_iommu_restart_ga_log(struct amd_iommu *iommu) 756 { 757 amd_iommu_restart_log(iommu, "GA", CONTROL_GAINT_EN, 758 CONTROL_GALOG_EN, MMIO_STATUS_GALOG_RUN_MASK, 759 MMIO_STATUS_GALOG_OVERFLOW_MASK); 760 } 761 762 /* 763 * This function resets the command buffer if the IOMMU stopped fetching 764 * commands from it. 765 */ 766 static void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu) 767 { 768 iommu_feature_disable(iommu, CONTROL_CMDBUF_EN); 769 770 writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET); 771 writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET); 772 iommu->cmd_buf_head = 0; 773 iommu->cmd_buf_tail = 0; 774 775 iommu_feature_enable(iommu, CONTROL_CMDBUF_EN); 776 } 777 778 /* 779 * This function writes the command buffer address to the hardware and 780 * enables it. 781 */ 782 static void iommu_enable_command_buffer(struct amd_iommu *iommu) 783 { 784 u64 entry; 785 786 BUG_ON(iommu->cmd_buf == NULL); 787 788 entry = iommu_virt_to_phys(iommu->cmd_buf); 789 entry |= MMIO_CMD_SIZE_512; 790 791 memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET, 792 &entry, sizeof(entry)); 793 794 amd_iommu_reset_cmd_buffer(iommu); 795 } 796 797 /* 798 * This function disables the command buffer 799 */ 800 static void iommu_disable_command_buffer(struct amd_iommu *iommu) 801 { 802 iommu_feature_disable(iommu, CONTROL_CMDBUF_EN); 803 } 804 805 static void __init free_command_buffer(struct amd_iommu *iommu) 806 { 807 iommu_free_pages(iommu->cmd_buf); 808 } 809 810 void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, gfp_t gfp, 811 size_t size) 812 { 813 void *buf; 814 815 size = PAGE_ALIGN(size); 816 buf = iommu_alloc_pages_sz(gfp, size); 817 if (!buf) 818 return NULL; 819 if (check_feature(FEATURE_SNP) && 820 set_memory_4k((unsigned long)buf, size / PAGE_SIZE)) { 821 iommu_free_pages(buf); 822 return NULL; 823 } 824 825 return buf; 826 } 827 828 /* allocates the memory where the IOMMU will log its events to */ 829 static int __init alloc_event_buffer(struct amd_iommu *iommu) 830 { 831 iommu->evt_buf = iommu_alloc_4k_pages(iommu, GFP_KERNEL, 832 EVT_BUFFER_SIZE); 833 834 return iommu->evt_buf ? 0 : -ENOMEM; 835 } 836 837 static void iommu_enable_event_buffer(struct amd_iommu *iommu) 838 { 839 u64 entry; 840 841 BUG_ON(iommu->evt_buf == NULL); 842 843 entry = iommu_virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK; 844 845 memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET, 846 &entry, sizeof(entry)); 847 848 /* set head and tail to zero manually */ 849 writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); 850 writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET); 851 852 iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN); 853 } 854 855 /* 856 * This function disables the event log buffer 857 */ 858 static void iommu_disable_event_buffer(struct amd_iommu *iommu) 859 { 860 iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN); 861 } 862 863 static void __init free_event_buffer(struct amd_iommu *iommu) 864 { 865 iommu_free_pages(iommu->evt_buf); 866 } 867 868 static void free_ga_log(struct amd_iommu *iommu) 869 { 870 #ifdef CONFIG_IRQ_REMAP 871 iommu_free_pages(iommu->ga_log); 872 iommu_free_pages(iommu->ga_log_tail); 873 #endif 874 } 875 876 #ifdef CONFIG_IRQ_REMAP 877 static int iommu_ga_log_enable(struct amd_iommu *iommu) 878 { 879 u32 status, i; 880 u64 entry; 881 882 if (!iommu->ga_log) 883 return -EINVAL; 884 885 entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512; 886 memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET, 887 &entry, sizeof(entry)); 888 entry = (iommu_virt_to_phys(iommu->ga_log_tail) & 889 (BIT_ULL(52)-1)) & ~7ULL; 890 memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET, 891 &entry, sizeof(entry)); 892 writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET); 893 writel(0x00, iommu->mmio_base + MMIO_GA_TAIL_OFFSET); 894 895 896 iommu_feature_enable(iommu, CONTROL_GAINT_EN); 897 iommu_feature_enable(iommu, CONTROL_GALOG_EN); 898 899 for (i = 0; i < MMIO_STATUS_TIMEOUT; ++i) { 900 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); 901 if (status & (MMIO_STATUS_GALOG_RUN_MASK)) 902 break; 903 udelay(10); 904 } 905 906 if (WARN_ON(i >= MMIO_STATUS_TIMEOUT)) 907 return -EINVAL; 908 909 return 0; 910 } 911 912 static int iommu_init_ga_log(struct amd_iommu *iommu) 913 { 914 if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) 915 return 0; 916 917 iommu->ga_log = iommu_alloc_pages_sz(GFP_KERNEL, GA_LOG_SIZE); 918 if (!iommu->ga_log) 919 goto err_out; 920 921 iommu->ga_log_tail = iommu_alloc_pages_sz(GFP_KERNEL, 8); 922 if (!iommu->ga_log_tail) 923 goto err_out; 924 925 return 0; 926 err_out: 927 free_ga_log(iommu); 928 return -EINVAL; 929 } 930 #endif /* CONFIG_IRQ_REMAP */ 931 932 static int __init alloc_cwwb_sem(struct amd_iommu *iommu) 933 { 934 iommu->cmd_sem = iommu_alloc_4k_pages(iommu, GFP_KERNEL, 1); 935 936 return iommu->cmd_sem ? 0 : -ENOMEM; 937 } 938 939 static void __init free_cwwb_sem(struct amd_iommu *iommu) 940 { 941 if (iommu->cmd_sem) 942 iommu_free_pages((void *)iommu->cmd_sem); 943 } 944 945 static void iommu_enable_xt(struct amd_iommu *iommu) 946 { 947 #ifdef CONFIG_IRQ_REMAP 948 /* 949 * XT mode (32-bit APIC destination ID) requires 950 * GA mode (128-bit IRTE support) as a prerequisite. 951 */ 952 if (AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir) && 953 amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE) 954 iommu_feature_enable(iommu, CONTROL_XT_EN); 955 #endif /* CONFIG_IRQ_REMAP */ 956 } 957 958 static void iommu_enable_gt(struct amd_iommu *iommu) 959 { 960 if (!check_feature(FEATURE_GT)) 961 return; 962 963 iommu_feature_enable(iommu, CONTROL_GT_EN); 964 } 965 966 /* sets a specific bit in the device table entry. */ 967 static void set_dte_bit(struct dev_table_entry *dte, u8 bit) 968 { 969 int i = (bit >> 6) & 0x03; 970 int _bit = bit & 0x3f; 971 972 dte->data[i] |= (1UL << _bit); 973 } 974 975 static bool __copy_device_table(struct amd_iommu *iommu) 976 { 977 u64 int_ctl, int_tab_len, entry = 0; 978 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 979 struct dev_table_entry *old_devtb = NULL; 980 u32 lo, hi, devid, old_devtb_size; 981 phys_addr_t old_devtb_phys; 982 u16 dom_id, dte_v, irq_v; 983 u64 tmp; 984 985 /* Each IOMMU use separate device table with the same size */ 986 lo = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET); 987 hi = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET + 4); 988 entry = (((u64) hi) << 32) + lo; 989 990 old_devtb_size = ((entry & ~PAGE_MASK) + 1) << 12; 991 if (old_devtb_size != pci_seg->dev_table_size) { 992 pr_err("The device table size of IOMMU:%d is not expected!\n", 993 iommu->index); 994 return false; 995 } 996 997 /* 998 * When SME is enabled in the first kernel, the entry includes the 999 * memory encryption mask(sme_me_mask), we must remove the memory 1000 * encryption mask to obtain the true physical address in kdump kernel. 1001 */ 1002 old_devtb_phys = __sme_clr(entry) & PAGE_MASK; 1003 1004 if (old_devtb_phys >= 0x100000000ULL) { 1005 pr_err("The address of old device table is above 4G, not trustworthy!\n"); 1006 return false; 1007 } 1008 old_devtb = (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT) && is_kdump_kernel()) 1009 ? (__force void *)ioremap_encrypted(old_devtb_phys, 1010 pci_seg->dev_table_size) 1011 : memremap(old_devtb_phys, pci_seg->dev_table_size, MEMREMAP_WB); 1012 1013 if (!old_devtb) 1014 return false; 1015 1016 pci_seg->old_dev_tbl_cpy = iommu_alloc_pages_sz( 1017 GFP_KERNEL | GFP_DMA32, pci_seg->dev_table_size); 1018 if (pci_seg->old_dev_tbl_cpy == NULL) { 1019 pr_err("Failed to allocate memory for copying old device table!\n"); 1020 memunmap(old_devtb); 1021 return false; 1022 } 1023 1024 for (devid = 0; devid <= pci_seg->last_bdf; ++devid) { 1025 pci_seg->old_dev_tbl_cpy[devid] = old_devtb[devid]; 1026 dom_id = old_devtb[devid].data[1] & DEV_DOMID_MASK; 1027 dte_v = old_devtb[devid].data[0] & DTE_FLAG_V; 1028 1029 if (dte_v && dom_id) { 1030 pci_seg->old_dev_tbl_cpy[devid].data[0] = old_devtb[devid].data[0]; 1031 pci_seg->old_dev_tbl_cpy[devid].data[1] = old_devtb[devid].data[1]; 1032 /* Reserve the Domain IDs used by previous kernel */ 1033 if (ida_alloc_range(&pdom_ids, dom_id, dom_id, GFP_ATOMIC) != dom_id) { 1034 pr_err("Failed to reserve domain ID 0x%x\n", dom_id); 1035 memunmap(old_devtb); 1036 return false; 1037 } 1038 /* If gcr3 table existed, mask it out */ 1039 if (old_devtb[devid].data[0] & DTE_FLAG_GV) { 1040 tmp = (DTE_GCR3_30_15 | DTE_GCR3_51_31); 1041 pci_seg->old_dev_tbl_cpy[devid].data[1] &= ~tmp; 1042 tmp = (DTE_GCR3_14_12 | DTE_FLAG_GV); 1043 pci_seg->old_dev_tbl_cpy[devid].data[0] &= ~tmp; 1044 } 1045 } 1046 1047 irq_v = old_devtb[devid].data[2] & DTE_IRQ_REMAP_ENABLE; 1048 int_ctl = old_devtb[devid].data[2] & DTE_IRQ_REMAP_INTCTL_MASK; 1049 int_tab_len = old_devtb[devid].data[2] & DTE_INTTABLEN_MASK; 1050 if (irq_v && (int_ctl || int_tab_len)) { 1051 if ((int_ctl != DTE_IRQ_REMAP_INTCTL) || 1052 (int_tab_len != DTE_INTTABLEN_512 && 1053 int_tab_len != DTE_INTTABLEN_2K)) { 1054 pr_err("Wrong old irq remapping flag: %#x\n", devid); 1055 memunmap(old_devtb); 1056 return false; 1057 } 1058 1059 pci_seg->old_dev_tbl_cpy[devid].data[2] = old_devtb[devid].data[2]; 1060 } 1061 } 1062 memunmap(old_devtb); 1063 1064 return true; 1065 } 1066 1067 static bool copy_device_table(void) 1068 { 1069 struct amd_iommu *iommu; 1070 struct amd_iommu_pci_seg *pci_seg; 1071 1072 if (!amd_iommu_pre_enabled) 1073 return false; 1074 1075 pr_warn("Translation is already enabled - trying to copy translation structures\n"); 1076 1077 /* 1078 * All IOMMUs within PCI segment shares common device table. 1079 * Hence copy device table only once per PCI segment. 1080 */ 1081 for_each_pci_segment(pci_seg) { 1082 for_each_iommu(iommu) { 1083 if (pci_seg->id != iommu->pci_seg->id) 1084 continue; 1085 if (!__copy_device_table(iommu)) 1086 return false; 1087 break; 1088 } 1089 } 1090 1091 return true; 1092 } 1093 1094 struct dev_table_entry *amd_iommu_get_ivhd_dte_flags(u16 segid, u16 devid) 1095 { 1096 struct ivhd_dte_flags *e; 1097 unsigned int best_len = UINT_MAX; 1098 struct dev_table_entry *dte = NULL; 1099 1100 for_each_ivhd_dte_flags(e) { 1101 /* 1102 * Need to go through the whole list to find the smallest range, 1103 * which contains the devid. 1104 */ 1105 if ((e->segid == segid) && 1106 (e->devid_first <= devid) && (devid <= e->devid_last)) { 1107 unsigned int len = e->devid_last - e->devid_first; 1108 1109 if (len < best_len) { 1110 dte = &(e->dte); 1111 best_len = len; 1112 } 1113 } 1114 } 1115 return dte; 1116 } 1117 1118 static bool search_ivhd_dte_flags(u16 segid, u16 first, u16 last) 1119 { 1120 struct ivhd_dte_flags *e; 1121 1122 for_each_ivhd_dte_flags(e) { 1123 if ((e->segid == segid) && 1124 (e->devid_first == first) && 1125 (e->devid_last == last)) 1126 return true; 1127 } 1128 return false; 1129 } 1130 1131 /* 1132 * This function takes the device specific flags read from the ACPI 1133 * table and sets up the device table entry with that information 1134 */ 1135 static void __init 1136 set_dev_entry_from_acpi_range(struct amd_iommu *iommu, u16 first, u16 last, 1137 u32 flags, u32 ext_flags) 1138 { 1139 int i; 1140 struct dev_table_entry dte = {}; 1141 1142 /* Parse IVHD DTE setting flags and store information */ 1143 if (flags) { 1144 struct ivhd_dte_flags *d; 1145 1146 if (search_ivhd_dte_flags(iommu->pci_seg->id, first, last)) 1147 return; 1148 1149 d = kzalloc(sizeof(struct ivhd_dte_flags), GFP_KERNEL); 1150 if (!d) 1151 return; 1152 1153 pr_debug("%s: devid range %#x:%#x\n", __func__, first, last); 1154 1155 if (flags & ACPI_DEVFLAG_INITPASS) 1156 set_dte_bit(&dte, DEV_ENTRY_INIT_PASS); 1157 if (flags & ACPI_DEVFLAG_EXTINT) 1158 set_dte_bit(&dte, DEV_ENTRY_EINT_PASS); 1159 if (flags & ACPI_DEVFLAG_NMI) 1160 set_dte_bit(&dte, DEV_ENTRY_NMI_PASS); 1161 if (flags & ACPI_DEVFLAG_SYSMGT1) 1162 set_dte_bit(&dte, DEV_ENTRY_SYSMGT1); 1163 if (flags & ACPI_DEVFLAG_SYSMGT2) 1164 set_dte_bit(&dte, DEV_ENTRY_SYSMGT2); 1165 if (flags & ACPI_DEVFLAG_LINT0) 1166 set_dte_bit(&dte, DEV_ENTRY_LINT0_PASS); 1167 if (flags & ACPI_DEVFLAG_LINT1) 1168 set_dte_bit(&dte, DEV_ENTRY_LINT1_PASS); 1169 1170 /* Apply erratum 63, which needs info in initial_dte */ 1171 if (FIELD_GET(DTE_DATA1_SYSMGT_MASK, dte.data[1]) == 0x1) 1172 dte.data[0] |= DTE_FLAG_IW; 1173 1174 memcpy(&d->dte, &dte, sizeof(dte)); 1175 d->segid = iommu->pci_seg->id; 1176 d->devid_first = first; 1177 d->devid_last = last; 1178 list_add_tail(&d->list, &amd_ivhd_dev_flags_list); 1179 } 1180 1181 for (i = first; i <= last; i++) { 1182 if (flags) { 1183 struct dev_table_entry *dev_table = get_dev_table(iommu); 1184 1185 memcpy(&dev_table[i], &dte, sizeof(dte)); 1186 } 1187 amd_iommu_set_rlookup_table(iommu, i); 1188 } 1189 } 1190 1191 static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu, 1192 u16 devid, u32 flags, u32 ext_flags) 1193 { 1194 set_dev_entry_from_acpi_range(iommu, devid, devid, flags, ext_flags); 1195 } 1196 1197 int __init add_special_device(u8 type, u8 id, u32 *devid, bool cmd_line) 1198 { 1199 struct devid_map *entry; 1200 struct list_head *list; 1201 1202 if (type == IVHD_SPECIAL_IOAPIC) 1203 list = &ioapic_map; 1204 else if (type == IVHD_SPECIAL_HPET) 1205 list = &hpet_map; 1206 else 1207 return -EINVAL; 1208 1209 list_for_each_entry(entry, list, list) { 1210 if (!(entry->id == id && entry->cmd_line)) 1211 continue; 1212 1213 pr_info("Command-line override present for %s id %d - ignoring\n", 1214 type == IVHD_SPECIAL_IOAPIC ? "IOAPIC" : "HPET", id); 1215 1216 *devid = entry->devid; 1217 1218 return 0; 1219 } 1220 1221 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 1222 if (!entry) 1223 return -ENOMEM; 1224 1225 entry->id = id; 1226 entry->devid = *devid; 1227 entry->cmd_line = cmd_line; 1228 1229 list_add_tail(&entry->list, list); 1230 1231 return 0; 1232 } 1233 1234 static int __init add_acpi_hid_device(u8 *hid, u8 *uid, u32 *devid, 1235 bool cmd_line) 1236 { 1237 struct acpihid_map_entry *entry; 1238 struct list_head *list = &acpihid_map; 1239 1240 list_for_each_entry(entry, list, list) { 1241 if (strcmp(entry->hid, hid) || 1242 (*uid && *entry->uid && strcmp(entry->uid, uid)) || 1243 !entry->cmd_line) 1244 continue; 1245 1246 pr_info("Command-line override for hid:%s uid:%s\n", 1247 hid, uid); 1248 *devid = entry->devid; 1249 return 0; 1250 } 1251 1252 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 1253 if (!entry) 1254 return -ENOMEM; 1255 1256 memcpy(entry->uid, uid, strlen(uid)); 1257 memcpy(entry->hid, hid, strlen(hid)); 1258 entry->devid = *devid; 1259 entry->cmd_line = cmd_line; 1260 entry->root_devid = (entry->devid & (~0x7)); 1261 1262 pr_info("%s, add hid:%s, uid:%s, rdevid:%#x\n", 1263 entry->cmd_line ? "cmd" : "ivrs", 1264 entry->hid, entry->uid, entry->root_devid); 1265 1266 list_add_tail(&entry->list, list); 1267 return 0; 1268 } 1269 1270 static int __init add_early_maps(void) 1271 { 1272 int i, ret; 1273 1274 for (i = 0; i < early_ioapic_map_size; ++i) { 1275 ret = add_special_device(IVHD_SPECIAL_IOAPIC, 1276 early_ioapic_map[i].id, 1277 &early_ioapic_map[i].devid, 1278 early_ioapic_map[i].cmd_line); 1279 if (ret) 1280 return ret; 1281 } 1282 1283 for (i = 0; i < early_hpet_map_size; ++i) { 1284 ret = add_special_device(IVHD_SPECIAL_HPET, 1285 early_hpet_map[i].id, 1286 &early_hpet_map[i].devid, 1287 early_hpet_map[i].cmd_line); 1288 if (ret) 1289 return ret; 1290 } 1291 1292 for (i = 0; i < early_acpihid_map_size; ++i) { 1293 ret = add_acpi_hid_device(early_acpihid_map[i].hid, 1294 early_acpihid_map[i].uid, 1295 &early_acpihid_map[i].devid, 1296 early_acpihid_map[i].cmd_line); 1297 if (ret) 1298 return ret; 1299 } 1300 1301 return 0; 1302 } 1303 1304 /* 1305 * Takes a pointer to an AMD IOMMU entry in the ACPI table and 1306 * initializes the hardware and our data structures with it. 1307 */ 1308 static int __init init_iommu_from_acpi(struct amd_iommu *iommu, 1309 struct ivhd_header *h) 1310 { 1311 u8 *p = (u8 *)h; 1312 u8 *end = p, flags = 0; 1313 u16 devid = 0, devid_start = 0, devid_to = 0, seg_id; 1314 u32 dev_i, ext_flags = 0; 1315 bool alias = false; 1316 struct ivhd_entry *e; 1317 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 1318 u32 ivhd_size; 1319 int ret; 1320 1321 1322 ret = add_early_maps(); 1323 if (ret) 1324 return ret; 1325 1326 amd_iommu_apply_ivrs_quirks(); 1327 1328 /* 1329 * First save the recommended feature enable bits from ACPI 1330 */ 1331 iommu->acpi_flags = h->flags; 1332 1333 /* 1334 * Done. Now parse the device entries 1335 */ 1336 ivhd_size = get_ivhd_header_size(h); 1337 if (!ivhd_size) { 1338 pr_err("Unsupported IVHD type %#x\n", h->type); 1339 return -EINVAL; 1340 } 1341 1342 p += ivhd_size; 1343 1344 end += h->length; 1345 1346 1347 while (p < end) { 1348 e = (struct ivhd_entry *)p; 1349 seg_id = pci_seg->id; 1350 1351 switch (e->type) { 1352 case IVHD_DEV_ALL: 1353 1354 DUMP_printk(" DEV_ALL\t\t\tsetting: %#02x\n", e->flags); 1355 set_dev_entry_from_acpi_range(iommu, 0, pci_seg->last_bdf, e->flags, 0); 1356 break; 1357 case IVHD_DEV_SELECT: 1358 1359 DUMP_printk(" DEV_SELECT\t\t\tdevid: %04x:%02x:%02x.%x flags: %#02x\n", 1360 seg_id, PCI_BUS_NUM(e->devid), 1361 PCI_SLOT(e->devid), 1362 PCI_FUNC(e->devid), 1363 e->flags); 1364 1365 devid = e->devid; 1366 set_dev_entry_from_acpi(iommu, devid, e->flags, 0); 1367 break; 1368 case IVHD_DEV_SELECT_RANGE_START: 1369 1370 DUMP_printk(" DEV_SELECT_RANGE_START\tdevid: %04x:%02x:%02x.%x flags: %#02x\n", 1371 seg_id, PCI_BUS_NUM(e->devid), 1372 PCI_SLOT(e->devid), 1373 PCI_FUNC(e->devid), 1374 e->flags); 1375 1376 devid_start = e->devid; 1377 flags = e->flags; 1378 ext_flags = 0; 1379 alias = false; 1380 break; 1381 case IVHD_DEV_ALIAS: 1382 1383 DUMP_printk(" DEV_ALIAS\t\t\tdevid: %04x:%02x:%02x.%x flags: %#02x devid_to: %02x:%02x.%x\n", 1384 seg_id, PCI_BUS_NUM(e->devid), 1385 PCI_SLOT(e->devid), 1386 PCI_FUNC(e->devid), 1387 e->flags, 1388 PCI_BUS_NUM(e->ext >> 8), 1389 PCI_SLOT(e->ext >> 8), 1390 PCI_FUNC(e->ext >> 8)); 1391 1392 devid = e->devid; 1393 devid_to = e->ext >> 8; 1394 set_dev_entry_from_acpi(iommu, devid , e->flags, 0); 1395 set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0); 1396 pci_seg->alias_table[devid] = devid_to; 1397 break; 1398 case IVHD_DEV_ALIAS_RANGE: 1399 1400 DUMP_printk(" DEV_ALIAS_RANGE\t\tdevid: %04x:%02x:%02x.%x flags: %#02x devid_to: %04x:%02x:%02x.%x\n", 1401 seg_id, PCI_BUS_NUM(e->devid), 1402 PCI_SLOT(e->devid), 1403 PCI_FUNC(e->devid), 1404 e->flags, 1405 seg_id, PCI_BUS_NUM(e->ext >> 8), 1406 PCI_SLOT(e->ext >> 8), 1407 PCI_FUNC(e->ext >> 8)); 1408 1409 devid_start = e->devid; 1410 flags = e->flags; 1411 devid_to = e->ext >> 8; 1412 ext_flags = 0; 1413 alias = true; 1414 break; 1415 case IVHD_DEV_EXT_SELECT: 1416 1417 DUMP_printk(" DEV_EXT_SELECT\t\tdevid: %04x:%02x:%02x.%x flags: %#02x ext: %08x\n", 1418 seg_id, PCI_BUS_NUM(e->devid), 1419 PCI_SLOT(e->devid), 1420 PCI_FUNC(e->devid), 1421 e->flags, e->ext); 1422 1423 devid = e->devid; 1424 set_dev_entry_from_acpi(iommu, devid, e->flags, 1425 e->ext); 1426 break; 1427 case IVHD_DEV_EXT_SELECT_RANGE: 1428 1429 DUMP_printk(" DEV_EXT_SELECT_RANGE\tdevid: %04x:%02x:%02x.%x flags: %#02x ext: %08x\n", 1430 seg_id, PCI_BUS_NUM(e->devid), 1431 PCI_SLOT(e->devid), 1432 PCI_FUNC(e->devid), 1433 e->flags, e->ext); 1434 1435 devid_start = e->devid; 1436 flags = e->flags; 1437 ext_flags = e->ext; 1438 alias = false; 1439 break; 1440 case IVHD_DEV_RANGE_END: 1441 1442 DUMP_printk(" DEV_RANGE_END\t\tdevid: %04x:%02x:%02x.%x\n", 1443 seg_id, PCI_BUS_NUM(e->devid), 1444 PCI_SLOT(e->devid), 1445 PCI_FUNC(e->devid)); 1446 1447 devid = e->devid; 1448 for (dev_i = devid_start; dev_i <= devid; ++dev_i) { 1449 if (alias) 1450 pci_seg->alias_table[dev_i] = devid_to; 1451 } 1452 set_dev_entry_from_acpi_range(iommu, devid_start, devid, flags, ext_flags); 1453 set_dev_entry_from_acpi(iommu, devid_to, flags, ext_flags); 1454 break; 1455 case IVHD_DEV_SPECIAL: { 1456 u8 handle, type; 1457 const char *var; 1458 u32 devid; 1459 int ret; 1460 1461 handle = e->ext & 0xff; 1462 devid = PCI_SEG_DEVID_TO_SBDF(seg_id, (e->ext >> 8)); 1463 type = (e->ext >> 24) & 0xff; 1464 1465 if (type == IVHD_SPECIAL_IOAPIC) 1466 var = "IOAPIC"; 1467 else if (type == IVHD_SPECIAL_HPET) 1468 var = "HPET"; 1469 else 1470 var = "UNKNOWN"; 1471 1472 DUMP_printk(" DEV_SPECIAL(%s[%d])\t\tdevid: %04x:%02x:%02x.%x, flags: %#02x\n", 1473 var, (int)handle, 1474 seg_id, PCI_BUS_NUM(devid), 1475 PCI_SLOT(devid), 1476 PCI_FUNC(devid), 1477 e->flags); 1478 1479 ret = add_special_device(type, handle, &devid, false); 1480 if (ret) 1481 return ret; 1482 1483 /* 1484 * add_special_device might update the devid in case a 1485 * command-line override is present. So call 1486 * set_dev_entry_from_acpi after add_special_device. 1487 */ 1488 set_dev_entry_from_acpi(iommu, devid, e->flags, 0); 1489 1490 break; 1491 } 1492 case IVHD_DEV_ACPI_HID: { 1493 u32 devid; 1494 u8 hid[ACPIHID_HID_LEN]; 1495 u8 uid[ACPIHID_UID_LEN]; 1496 int ret; 1497 1498 if (h->type != 0x40) { 1499 pr_err(FW_BUG "Invalid IVHD device type %#x\n", 1500 e->type); 1501 break; 1502 } 1503 1504 BUILD_BUG_ON(sizeof(e->ext_hid) != ACPIHID_HID_LEN - 1); 1505 memcpy(hid, &e->ext_hid, ACPIHID_HID_LEN - 1); 1506 hid[ACPIHID_HID_LEN - 1] = '\0'; 1507 1508 if (!(*hid)) { 1509 pr_err(FW_BUG "Invalid HID.\n"); 1510 break; 1511 } 1512 1513 uid[0] = '\0'; 1514 switch (e->uidf) { 1515 case UID_NOT_PRESENT: 1516 1517 if (e->uidl != 0) 1518 pr_warn(FW_BUG "Invalid UID length.\n"); 1519 1520 break; 1521 case UID_IS_INTEGER: 1522 1523 sprintf(uid, "%d", e->uid); 1524 1525 break; 1526 case UID_IS_CHARACTER: 1527 1528 memcpy(uid, &e->uid, e->uidl); 1529 uid[e->uidl] = '\0'; 1530 1531 break; 1532 default: 1533 break; 1534 } 1535 1536 devid = PCI_SEG_DEVID_TO_SBDF(seg_id, e->devid); 1537 DUMP_printk(" DEV_ACPI_HID(%s[%s])\t\tdevid: %04x:%02x:%02x.%x, flags: %#02x\n", 1538 hid, uid, seg_id, 1539 PCI_BUS_NUM(devid), 1540 PCI_SLOT(devid), 1541 PCI_FUNC(devid), 1542 e->flags); 1543 1544 flags = e->flags; 1545 1546 ret = add_acpi_hid_device(hid, uid, &devid, false); 1547 if (ret) 1548 return ret; 1549 1550 /* 1551 * add_special_device might update the devid in case a 1552 * command-line override is present. So call 1553 * set_dev_entry_from_acpi after add_special_device. 1554 */ 1555 set_dev_entry_from_acpi(iommu, devid, e->flags, 0); 1556 1557 break; 1558 } 1559 default: 1560 break; 1561 } 1562 1563 p += ivhd_entry_length(p); 1564 } 1565 1566 return 0; 1567 } 1568 1569 /* Allocate PCI segment data structure */ 1570 static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id, 1571 struct acpi_table_header *ivrs_base) 1572 { 1573 struct amd_iommu_pci_seg *pci_seg; 1574 int last_bdf; 1575 1576 /* 1577 * First parse ACPI tables to find the largest Bus/Dev/Func we need to 1578 * handle in this PCI segment. Upon this information the shared data 1579 * structures for the PCI segments in the system will be allocated. 1580 */ 1581 last_bdf = find_last_devid_acpi(ivrs_base, id); 1582 if (last_bdf < 0) 1583 return NULL; 1584 1585 pci_seg = kzalloc(sizeof(struct amd_iommu_pci_seg), GFP_KERNEL); 1586 if (pci_seg == NULL) 1587 return NULL; 1588 1589 pci_seg->last_bdf = last_bdf; 1590 DUMP_printk("PCI segment : 0x%0x, last bdf : 0x%04x\n", id, last_bdf); 1591 pci_seg->dev_table_size = 1592 max(roundup_pow_of_two((last_bdf + 1) * DEV_TABLE_ENTRY_SIZE), 1593 SZ_4K); 1594 1595 pci_seg->id = id; 1596 init_llist_head(&pci_seg->dev_data_list); 1597 INIT_LIST_HEAD(&pci_seg->unity_map); 1598 list_add_tail(&pci_seg->list, &amd_iommu_pci_seg_list); 1599 1600 if (alloc_dev_table(pci_seg)) 1601 return NULL; 1602 if (alloc_alias_table(pci_seg)) 1603 return NULL; 1604 if (alloc_rlookup_table(pci_seg)) 1605 return NULL; 1606 1607 return pci_seg; 1608 } 1609 1610 static struct amd_iommu_pci_seg *__init get_pci_segment(u16 id, 1611 struct acpi_table_header *ivrs_base) 1612 { 1613 struct amd_iommu_pci_seg *pci_seg; 1614 1615 for_each_pci_segment(pci_seg) { 1616 if (pci_seg->id == id) 1617 return pci_seg; 1618 } 1619 1620 return alloc_pci_segment(id, ivrs_base); 1621 } 1622 1623 static void __init free_pci_segments(void) 1624 { 1625 struct amd_iommu_pci_seg *pci_seg, *next; 1626 1627 for_each_pci_segment_safe(pci_seg, next) { 1628 list_del(&pci_seg->list); 1629 free_irq_lookup_table(pci_seg); 1630 free_rlookup_table(pci_seg); 1631 free_alias_table(pci_seg); 1632 free_dev_table(pci_seg); 1633 kfree(pci_seg); 1634 } 1635 } 1636 1637 static void __init free_sysfs(struct amd_iommu *iommu) 1638 { 1639 if (iommu->iommu.dev) { 1640 iommu_device_unregister(&iommu->iommu); 1641 iommu_device_sysfs_remove(&iommu->iommu); 1642 } 1643 } 1644 1645 static void __init free_iommu_one(struct amd_iommu *iommu) 1646 { 1647 free_sysfs(iommu); 1648 free_cwwb_sem(iommu); 1649 free_command_buffer(iommu); 1650 free_event_buffer(iommu); 1651 amd_iommu_free_ppr_log(iommu); 1652 free_ga_log(iommu); 1653 iommu_unmap_mmio_space(iommu); 1654 amd_iommu_iopf_uninit(iommu); 1655 } 1656 1657 static void __init free_iommu_all(void) 1658 { 1659 struct amd_iommu *iommu, *next; 1660 1661 for_each_iommu_safe(iommu, next) { 1662 list_del(&iommu->list); 1663 free_iommu_one(iommu); 1664 kfree(iommu); 1665 } 1666 } 1667 1668 /* 1669 * Family15h Model 10h-1fh erratum 746 (IOMMU Logging May Stall Translations) 1670 * Workaround: 1671 * BIOS should disable L2B micellaneous clock gating by setting 1672 * L2_L2B_CK_GATE_CONTROL[CKGateL2BMiscDisable](D0F2xF4_x90[2]) = 1b 1673 */ 1674 static void amd_iommu_erratum_746_workaround(struct amd_iommu *iommu) 1675 { 1676 u32 value; 1677 1678 if ((boot_cpu_data.x86 != 0x15) || 1679 (boot_cpu_data.x86_model < 0x10) || 1680 (boot_cpu_data.x86_model > 0x1f)) 1681 return; 1682 1683 pci_write_config_dword(iommu->dev, 0xf0, 0x90); 1684 pci_read_config_dword(iommu->dev, 0xf4, &value); 1685 1686 if (value & BIT(2)) 1687 return; 1688 1689 /* Select NB indirect register 0x90 and enable writing */ 1690 pci_write_config_dword(iommu->dev, 0xf0, 0x90 | (1 << 8)); 1691 1692 pci_write_config_dword(iommu->dev, 0xf4, value | 0x4); 1693 pci_info(iommu->dev, "Applying erratum 746 workaround\n"); 1694 1695 /* Clear the enable writing bit */ 1696 pci_write_config_dword(iommu->dev, 0xf0, 0x90); 1697 } 1698 1699 /* 1700 * Family15h Model 30h-3fh (IOMMU Mishandles ATS Write Permission) 1701 * Workaround: 1702 * BIOS should enable ATS write permission check by setting 1703 * L2_DEBUG_3[AtsIgnoreIWDis](D0F2xF4_x47[0]) = 1b 1704 */ 1705 static void amd_iommu_ats_write_check_workaround(struct amd_iommu *iommu) 1706 { 1707 u32 value; 1708 1709 if ((boot_cpu_data.x86 != 0x15) || 1710 (boot_cpu_data.x86_model < 0x30) || 1711 (boot_cpu_data.x86_model > 0x3f)) 1712 return; 1713 1714 /* Test L2_DEBUG_3[AtsIgnoreIWDis] == 1 */ 1715 value = iommu_read_l2(iommu, 0x47); 1716 1717 if (value & BIT(0)) 1718 return; 1719 1720 /* Set L2_DEBUG_3[AtsIgnoreIWDis] = 1 */ 1721 iommu_write_l2(iommu, 0x47, value | BIT(0)); 1722 1723 pci_info(iommu->dev, "Applying ATS write check workaround\n"); 1724 } 1725 1726 /* 1727 * This function glues the initialization function for one IOMMU 1728 * together and also allocates the command buffer and programs the 1729 * hardware. It does NOT enable the IOMMU. This is done afterwards. 1730 */ 1731 static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, 1732 struct acpi_table_header *ivrs_base) 1733 { 1734 struct amd_iommu_pci_seg *pci_seg; 1735 1736 pci_seg = get_pci_segment(h->pci_seg, ivrs_base); 1737 if (pci_seg == NULL) 1738 return -ENOMEM; 1739 iommu->pci_seg = pci_seg; 1740 1741 raw_spin_lock_init(&iommu->lock); 1742 atomic64_set(&iommu->cmd_sem_val, 0); 1743 1744 /* Add IOMMU to internal data structures */ 1745 list_add_tail(&iommu->list, &amd_iommu_list); 1746 iommu->index = amd_iommus_present++; 1747 1748 if (unlikely(iommu->index >= MAX_IOMMUS)) { 1749 WARN(1, "System has more IOMMUs than supported by this driver\n"); 1750 return -ENOSYS; 1751 } 1752 1753 /* 1754 * Copy data from ACPI table entry to the iommu struct 1755 */ 1756 iommu->devid = h->devid; 1757 iommu->cap_ptr = h->cap_ptr; 1758 iommu->mmio_phys = h->mmio_phys; 1759 1760 switch (h->type) { 1761 case 0x10: 1762 /* Check if IVHD EFR contains proper max banks/counters */ 1763 if ((h->efr_attr != 0) && 1764 ((h->efr_attr & (0xF << 13)) != 0) && 1765 ((h->efr_attr & (0x3F << 17)) != 0)) 1766 iommu->mmio_phys_end = MMIO_REG_END_OFFSET; 1767 else 1768 iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET; 1769 1770 /* GAM requires GA mode. */ 1771 if ((h->efr_attr & (0x1 << IOMMU_FEAT_GASUP_SHIFT)) == 0) 1772 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; 1773 break; 1774 case 0x11: 1775 case 0x40: 1776 if (h->efr_reg & (1 << 9)) 1777 iommu->mmio_phys_end = MMIO_REG_END_OFFSET; 1778 else 1779 iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET; 1780 1781 /* XT and GAM require GA mode. */ 1782 if ((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0) { 1783 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; 1784 break; 1785 } 1786 1787 if (h->efr_reg & BIT(IOMMU_EFR_XTSUP_SHIFT)) 1788 amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE; 1789 1790 early_iommu_features_init(iommu, h); 1791 1792 break; 1793 default: 1794 return -EINVAL; 1795 } 1796 1797 iommu->mmio_base = iommu_map_mmio_space(iommu->mmio_phys, 1798 iommu->mmio_phys_end); 1799 if (!iommu->mmio_base) 1800 return -ENOMEM; 1801 1802 return init_iommu_from_acpi(iommu, h); 1803 } 1804 1805 static int __init init_iommu_one_late(struct amd_iommu *iommu) 1806 { 1807 int ret; 1808 1809 if (alloc_cwwb_sem(iommu)) 1810 return -ENOMEM; 1811 1812 if (alloc_command_buffer(iommu)) 1813 return -ENOMEM; 1814 1815 if (alloc_event_buffer(iommu)) 1816 return -ENOMEM; 1817 1818 iommu->int_enabled = false; 1819 1820 init_translation_status(iommu); 1821 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) { 1822 iommu_disable(iommu); 1823 clear_translation_pre_enabled(iommu); 1824 pr_warn("Translation was enabled for IOMMU:%d but we are not in kdump mode\n", 1825 iommu->index); 1826 } 1827 if (amd_iommu_pre_enabled) 1828 amd_iommu_pre_enabled = translation_pre_enabled(iommu); 1829 1830 if (amd_iommu_irq_remap) { 1831 ret = amd_iommu_create_irq_domain(iommu); 1832 if (ret) 1833 return ret; 1834 } 1835 1836 /* 1837 * Make sure IOMMU is not considered to translate itself. The IVRS 1838 * table tells us so, but this is a lie! 1839 */ 1840 iommu->pci_seg->rlookup_table[iommu->devid] = NULL; 1841 1842 return 0; 1843 } 1844 1845 /** 1846 * get_highest_supported_ivhd_type - Look up the appropriate IVHD type 1847 * @ivrs: Pointer to the IVRS header 1848 * 1849 * This function search through all IVDB of the maximum supported IVHD 1850 */ 1851 static u8 get_highest_supported_ivhd_type(struct acpi_table_header *ivrs) 1852 { 1853 u8 *base = (u8 *)ivrs; 1854 struct ivhd_header *ivhd = (struct ivhd_header *) 1855 (base + IVRS_HEADER_LENGTH); 1856 u8 last_type = ivhd->type; 1857 u16 devid = ivhd->devid; 1858 1859 while (((u8 *)ivhd - base < ivrs->length) && 1860 (ivhd->type <= ACPI_IVHD_TYPE_MAX_SUPPORTED)) { 1861 u8 *p = (u8 *) ivhd; 1862 1863 if (ivhd->devid == devid) 1864 last_type = ivhd->type; 1865 ivhd = (struct ivhd_header *)(p + ivhd->length); 1866 } 1867 1868 return last_type; 1869 } 1870 1871 /* 1872 * Iterates over all IOMMU entries in the ACPI table, allocates the 1873 * IOMMU structure and initializes it with init_iommu_one() 1874 */ 1875 static int __init init_iommu_all(struct acpi_table_header *table) 1876 { 1877 u8 *p = (u8 *)table, *end = (u8 *)table; 1878 struct ivhd_header *h; 1879 struct amd_iommu *iommu; 1880 int ret; 1881 1882 end += table->length; 1883 p += IVRS_HEADER_LENGTH; 1884 1885 /* Phase 1: Process all IVHD blocks */ 1886 while (p < end) { 1887 h = (struct ivhd_header *)p; 1888 if (*p == amd_iommu_target_ivhd_type) { 1889 1890 DUMP_printk("device: %04x:%02x:%02x.%01x cap: %04x " 1891 "flags: %01x info %04x\n", 1892 h->pci_seg, PCI_BUS_NUM(h->devid), 1893 PCI_SLOT(h->devid), PCI_FUNC(h->devid), 1894 h->cap_ptr, h->flags, h->info); 1895 DUMP_printk(" mmio-addr: %016llx\n", 1896 h->mmio_phys); 1897 1898 iommu = kzalloc(sizeof(struct amd_iommu), GFP_KERNEL); 1899 if (iommu == NULL) 1900 return -ENOMEM; 1901 1902 ret = init_iommu_one(iommu, h, table); 1903 if (ret) 1904 return ret; 1905 } 1906 p += h->length; 1907 1908 } 1909 WARN_ON(p != end); 1910 1911 /* Phase 2 : Early feature support check */ 1912 get_global_efr(); 1913 1914 /* Phase 3 : Enabling IOMMU features */ 1915 for_each_iommu(iommu) { 1916 ret = init_iommu_one_late(iommu); 1917 if (ret) 1918 return ret; 1919 } 1920 1921 return 0; 1922 } 1923 1924 static void init_iommu_perf_ctr(struct amd_iommu *iommu) 1925 { 1926 u64 val; 1927 struct pci_dev *pdev = iommu->dev; 1928 1929 if (!check_feature(FEATURE_PC)) 1930 return; 1931 1932 amd_iommu_pc_present = true; 1933 1934 pci_info(pdev, "IOMMU performance counters supported\n"); 1935 1936 val = readl(iommu->mmio_base + MMIO_CNTR_CONF_OFFSET); 1937 iommu->max_banks = (u8) ((val >> 12) & 0x3f); 1938 iommu->max_counters = (u8) ((val >> 7) & 0xf); 1939 1940 return; 1941 } 1942 1943 static ssize_t amd_iommu_show_cap(struct device *dev, 1944 struct device_attribute *attr, 1945 char *buf) 1946 { 1947 struct amd_iommu *iommu = dev_to_amd_iommu(dev); 1948 return sysfs_emit(buf, "%x\n", iommu->cap); 1949 } 1950 static DEVICE_ATTR(cap, S_IRUGO, amd_iommu_show_cap, NULL); 1951 1952 static ssize_t amd_iommu_show_features(struct device *dev, 1953 struct device_attribute *attr, 1954 char *buf) 1955 { 1956 return sysfs_emit(buf, "%llx:%llx\n", amd_iommu_efr, amd_iommu_efr2); 1957 } 1958 static DEVICE_ATTR(features, S_IRUGO, amd_iommu_show_features, NULL); 1959 1960 static struct attribute *amd_iommu_attrs[] = { 1961 &dev_attr_cap.attr, 1962 &dev_attr_features.attr, 1963 NULL, 1964 }; 1965 1966 static struct attribute_group amd_iommu_group = { 1967 .name = "amd-iommu", 1968 .attrs = amd_iommu_attrs, 1969 }; 1970 1971 static const struct attribute_group *amd_iommu_groups[] = { 1972 &amd_iommu_group, 1973 NULL, 1974 }; 1975 1976 /* 1977 * Note: IVHD 0x11 and 0x40 also contains exact copy 1978 * of the IOMMU Extended Feature Register [MMIO Offset 0030h]. 1979 * Default to EFR in IVHD since it is available sooner (i.e. before PCI init). 1980 */ 1981 static void __init late_iommu_features_init(struct amd_iommu *iommu) 1982 { 1983 u64 features, features2; 1984 1985 if (!(iommu->cap & (1 << IOMMU_CAP_EFR))) 1986 return; 1987 1988 /* read extended feature bits */ 1989 features = readq(iommu->mmio_base + MMIO_EXT_FEATURES); 1990 features2 = readq(iommu->mmio_base + MMIO_EXT_FEATURES2); 1991 1992 if (!amd_iommu_efr) { 1993 amd_iommu_efr = features; 1994 amd_iommu_efr2 = features2; 1995 return; 1996 } 1997 1998 /* 1999 * Sanity check and warn if EFR values from 2000 * IVHD and MMIO conflict. 2001 */ 2002 if (features != amd_iommu_efr || 2003 features2 != amd_iommu_efr2) { 2004 pr_warn(FW_WARN 2005 "EFR mismatch. Use IVHD EFR (%#llx : %#llx), EFR2 (%#llx : %#llx).\n", 2006 features, amd_iommu_efr, 2007 features2, amd_iommu_efr2); 2008 } 2009 } 2010 2011 static int __init iommu_init_pci(struct amd_iommu *iommu) 2012 { 2013 int cap_ptr = iommu->cap_ptr; 2014 int ret; 2015 2016 iommu->dev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, 2017 PCI_BUS_NUM(iommu->devid), 2018 iommu->devid & 0xff); 2019 if (!iommu->dev) 2020 return -ENODEV; 2021 2022 /* Prevent binding other PCI device drivers to IOMMU devices */ 2023 iommu->dev->match_driver = false; 2024 2025 /* ACPI _PRT won't have an IRQ for IOMMU */ 2026 iommu->dev->irq_managed = 1; 2027 2028 pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET, 2029 &iommu->cap); 2030 2031 if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB))) 2032 amd_iommu_iotlb_sup = false; 2033 2034 late_iommu_features_init(iommu); 2035 2036 if (check_feature(FEATURE_GT)) { 2037 int glxval; 2038 u64 pasmax; 2039 2040 pasmax = FIELD_GET(FEATURE_PASMAX, amd_iommu_efr); 2041 iommu->iommu.max_pasids = (1 << (pasmax + 1)) - 1; 2042 2043 BUG_ON(iommu->iommu.max_pasids & ~PASID_MASK); 2044 2045 glxval = FIELD_GET(FEATURE_GLX, amd_iommu_efr); 2046 2047 if (amd_iommu_max_glx_val == -1) 2048 amd_iommu_max_glx_val = glxval; 2049 else 2050 amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval); 2051 2052 iommu_enable_gt(iommu); 2053 } 2054 2055 if (check_feature(FEATURE_PPR) && amd_iommu_alloc_ppr_log(iommu)) 2056 return -ENOMEM; 2057 2058 if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) { 2059 pr_info("Using strict mode due to virtualization\n"); 2060 iommu_set_dma_strict(); 2061 amd_iommu_np_cache = true; 2062 } 2063 2064 init_iommu_perf_ctr(iommu); 2065 2066 if (is_rd890_iommu(iommu->dev)) { 2067 int i, j; 2068 2069 iommu->root_pdev = 2070 pci_get_domain_bus_and_slot(iommu->pci_seg->id, 2071 iommu->dev->bus->number, 2072 PCI_DEVFN(0, 0)); 2073 2074 /* 2075 * Some rd890 systems may not be fully reconfigured by the 2076 * BIOS, so it's necessary for us to store this information so 2077 * it can be reprogrammed on resume 2078 */ 2079 pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4, 2080 &iommu->stored_addr_lo); 2081 pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8, 2082 &iommu->stored_addr_hi); 2083 2084 /* Low bit locks writes to configuration space */ 2085 iommu->stored_addr_lo &= ~1; 2086 2087 for (i = 0; i < 6; i++) 2088 for (j = 0; j < 0x12; j++) 2089 iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j); 2090 2091 for (i = 0; i < 0x83; i++) 2092 iommu->stored_l2[i] = iommu_read_l2(iommu, i); 2093 } 2094 2095 amd_iommu_erratum_746_workaround(iommu); 2096 amd_iommu_ats_write_check_workaround(iommu); 2097 2098 ret = iommu_device_sysfs_add(&iommu->iommu, &iommu->dev->dev, 2099 amd_iommu_groups, "ivhd%d", iommu->index); 2100 if (ret) 2101 return ret; 2102 2103 /* 2104 * Allocate per IOMMU IOPF queue here so that in attach device path, 2105 * PRI capable device can be added to IOPF queue 2106 */ 2107 if (amd_iommu_gt_ppr_supported()) { 2108 ret = amd_iommu_iopf_init(iommu); 2109 if (ret) 2110 return ret; 2111 } 2112 2113 iommu_device_register(&iommu->iommu, &amd_iommu_ops, NULL); 2114 2115 return pci_enable_device(iommu->dev); 2116 } 2117 2118 static void print_iommu_info(void) 2119 { 2120 int i; 2121 static const char * const feat_str[] = { 2122 "PreF", "PPR", "X2APIC", "NX", "GT", "[5]", 2123 "IA", "GA", "HE", "PC" 2124 }; 2125 2126 if (amd_iommu_efr) { 2127 pr_info("Extended features (%#llx, %#llx):", amd_iommu_efr, amd_iommu_efr2); 2128 2129 for (i = 0; i < ARRAY_SIZE(feat_str); ++i) { 2130 if (check_feature(1ULL << i)) 2131 pr_cont(" %s", feat_str[i]); 2132 } 2133 2134 if (check_feature(FEATURE_GAM_VAPIC)) 2135 pr_cont(" GA_vAPIC"); 2136 2137 if (check_feature(FEATURE_SNP)) 2138 pr_cont(" SNP"); 2139 2140 pr_cont("\n"); 2141 } 2142 2143 if (irq_remapping_enabled) { 2144 pr_info("Interrupt remapping enabled\n"); 2145 if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE) 2146 pr_info("X2APIC enabled\n"); 2147 } 2148 if (amd_iommu_pgtable == PD_MODE_V2) { 2149 pr_info("V2 page table enabled (Paging mode : %d level)\n", 2150 amd_iommu_gpt_level); 2151 } 2152 } 2153 2154 static int __init amd_iommu_init_pci(void) 2155 { 2156 struct amd_iommu *iommu; 2157 struct amd_iommu_pci_seg *pci_seg; 2158 int ret; 2159 2160 /* Init global identity domain before registering IOMMU */ 2161 amd_iommu_init_identity_domain(); 2162 2163 for_each_iommu(iommu) { 2164 ret = iommu_init_pci(iommu); 2165 if (ret) { 2166 pr_err("IOMMU%d: Failed to initialize IOMMU Hardware (error=%d)!\n", 2167 iommu->index, ret); 2168 goto out; 2169 } 2170 /* Need to setup range after PCI init */ 2171 iommu_set_cwwb_range(iommu); 2172 } 2173 2174 /* 2175 * Order is important here to make sure any unity map requirements are 2176 * fulfilled. The unity mappings are created and written to the device 2177 * table during the iommu_init_pci() call. 2178 * 2179 * After that we call init_device_table_dma() to make sure any 2180 * uninitialized DTE will block DMA, and in the end we flush the caches 2181 * of all IOMMUs to make sure the changes to the device table are 2182 * active. 2183 */ 2184 for_each_pci_segment(pci_seg) 2185 init_device_table_dma(pci_seg); 2186 2187 for_each_iommu(iommu) 2188 amd_iommu_flush_all_caches(iommu); 2189 2190 print_iommu_info(); 2191 2192 out: 2193 return ret; 2194 } 2195 2196 /**************************************************************************** 2197 * 2198 * The following functions initialize the MSI interrupts for all IOMMUs 2199 * in the system. It's a bit challenging because there could be multiple 2200 * IOMMUs per PCI BDF but we can call pci_enable_msi(x) only once per 2201 * pci_dev. 2202 * 2203 ****************************************************************************/ 2204 2205 static int iommu_setup_msi(struct amd_iommu *iommu) 2206 { 2207 int r; 2208 2209 r = pci_enable_msi(iommu->dev); 2210 if (r) 2211 return r; 2212 2213 r = request_threaded_irq(iommu->dev->irq, 2214 amd_iommu_int_handler, 2215 amd_iommu_int_thread, 2216 0, "AMD-Vi", 2217 iommu); 2218 2219 if (r) { 2220 pci_disable_msi(iommu->dev); 2221 return r; 2222 } 2223 2224 return 0; 2225 } 2226 2227 union intcapxt { 2228 u64 capxt; 2229 struct { 2230 u64 reserved_0 : 2, 2231 dest_mode_logical : 1, 2232 reserved_1 : 5, 2233 destid_0_23 : 24, 2234 vector : 8, 2235 reserved_2 : 16, 2236 destid_24_31 : 8; 2237 }; 2238 } __attribute__ ((packed)); 2239 2240 2241 static struct irq_chip intcapxt_controller; 2242 2243 static int intcapxt_irqdomain_activate(struct irq_domain *domain, 2244 struct irq_data *irqd, bool reserve) 2245 { 2246 return 0; 2247 } 2248 2249 static void intcapxt_irqdomain_deactivate(struct irq_domain *domain, 2250 struct irq_data *irqd) 2251 { 2252 } 2253 2254 2255 static int intcapxt_irqdomain_alloc(struct irq_domain *domain, unsigned int virq, 2256 unsigned int nr_irqs, void *arg) 2257 { 2258 struct irq_alloc_info *info = arg; 2259 int i, ret; 2260 2261 if (!info || info->type != X86_IRQ_ALLOC_TYPE_AMDVI) 2262 return -EINVAL; 2263 2264 ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg); 2265 if (ret < 0) 2266 return ret; 2267 2268 for (i = virq; i < virq + nr_irqs; i++) { 2269 struct irq_data *irqd = irq_domain_get_irq_data(domain, i); 2270 2271 irqd->chip = &intcapxt_controller; 2272 irqd->hwirq = info->hwirq; 2273 irqd->chip_data = info->data; 2274 __irq_set_handler(i, handle_edge_irq, 0, "edge"); 2275 } 2276 2277 return ret; 2278 } 2279 2280 static void intcapxt_irqdomain_free(struct irq_domain *domain, unsigned int virq, 2281 unsigned int nr_irqs) 2282 { 2283 irq_domain_free_irqs_top(domain, virq, nr_irqs); 2284 } 2285 2286 2287 static void intcapxt_unmask_irq(struct irq_data *irqd) 2288 { 2289 struct amd_iommu *iommu = irqd->chip_data; 2290 struct irq_cfg *cfg = irqd_cfg(irqd); 2291 union intcapxt xt; 2292 2293 xt.capxt = 0ULL; 2294 xt.dest_mode_logical = apic->dest_mode_logical; 2295 xt.vector = cfg->vector; 2296 xt.destid_0_23 = cfg->dest_apicid & GENMASK(23, 0); 2297 xt.destid_24_31 = cfg->dest_apicid >> 24; 2298 2299 writeq(xt.capxt, iommu->mmio_base + irqd->hwirq); 2300 } 2301 2302 static void intcapxt_mask_irq(struct irq_data *irqd) 2303 { 2304 struct amd_iommu *iommu = irqd->chip_data; 2305 2306 writeq(0, iommu->mmio_base + irqd->hwirq); 2307 } 2308 2309 2310 static int intcapxt_set_affinity(struct irq_data *irqd, 2311 const struct cpumask *mask, bool force) 2312 { 2313 struct irq_data *parent = irqd->parent_data; 2314 int ret; 2315 2316 ret = parent->chip->irq_set_affinity(parent, mask, force); 2317 if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE) 2318 return ret; 2319 return 0; 2320 } 2321 2322 static int intcapxt_set_wake(struct irq_data *irqd, unsigned int on) 2323 { 2324 return on ? -EOPNOTSUPP : 0; 2325 } 2326 2327 static struct irq_chip intcapxt_controller = { 2328 .name = "IOMMU-MSI", 2329 .irq_unmask = intcapxt_unmask_irq, 2330 .irq_mask = intcapxt_mask_irq, 2331 .irq_ack = irq_chip_ack_parent, 2332 .irq_retrigger = irq_chip_retrigger_hierarchy, 2333 .irq_set_affinity = intcapxt_set_affinity, 2334 .irq_set_wake = intcapxt_set_wake, 2335 .flags = IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_MOVE_DEFERRED, 2336 }; 2337 2338 static const struct irq_domain_ops intcapxt_domain_ops = { 2339 .alloc = intcapxt_irqdomain_alloc, 2340 .free = intcapxt_irqdomain_free, 2341 .activate = intcapxt_irqdomain_activate, 2342 .deactivate = intcapxt_irqdomain_deactivate, 2343 }; 2344 2345 2346 static struct irq_domain *iommu_irqdomain; 2347 2348 static struct irq_domain *iommu_get_irqdomain(void) 2349 { 2350 struct fwnode_handle *fn; 2351 2352 /* No need for locking here (yet) as the init is single-threaded */ 2353 if (iommu_irqdomain) 2354 return iommu_irqdomain; 2355 2356 fn = irq_domain_alloc_named_fwnode("AMD-Vi-MSI"); 2357 if (!fn) 2358 return NULL; 2359 2360 iommu_irqdomain = irq_domain_create_hierarchy(x86_vector_domain, 0, 0, 2361 fn, &intcapxt_domain_ops, 2362 NULL); 2363 if (!iommu_irqdomain) 2364 irq_domain_free_fwnode(fn); 2365 2366 return iommu_irqdomain; 2367 } 2368 2369 static int __iommu_setup_intcapxt(struct amd_iommu *iommu, const char *devname, 2370 int hwirq, irq_handler_t thread_fn) 2371 { 2372 struct irq_domain *domain; 2373 struct irq_alloc_info info; 2374 int irq, ret; 2375 int node = dev_to_node(&iommu->dev->dev); 2376 2377 domain = iommu_get_irqdomain(); 2378 if (!domain) 2379 return -ENXIO; 2380 2381 init_irq_alloc_info(&info, NULL); 2382 info.type = X86_IRQ_ALLOC_TYPE_AMDVI; 2383 info.data = iommu; 2384 info.hwirq = hwirq; 2385 2386 irq = irq_domain_alloc_irqs(domain, 1, node, &info); 2387 if (irq < 0) { 2388 irq_domain_remove(domain); 2389 return irq; 2390 } 2391 2392 ret = request_threaded_irq(irq, amd_iommu_int_handler, 2393 thread_fn, 0, devname, iommu); 2394 if (ret) { 2395 irq_domain_free_irqs(irq, 1); 2396 irq_domain_remove(domain); 2397 return ret; 2398 } 2399 2400 return 0; 2401 } 2402 2403 static int iommu_setup_intcapxt(struct amd_iommu *iommu) 2404 { 2405 int ret; 2406 2407 snprintf(iommu->evt_irq_name, sizeof(iommu->evt_irq_name), 2408 "AMD-Vi%d-Evt", iommu->index); 2409 ret = __iommu_setup_intcapxt(iommu, iommu->evt_irq_name, 2410 MMIO_INTCAPXT_EVT_OFFSET, 2411 amd_iommu_int_thread_evtlog); 2412 if (ret) 2413 return ret; 2414 2415 snprintf(iommu->ppr_irq_name, sizeof(iommu->ppr_irq_name), 2416 "AMD-Vi%d-PPR", iommu->index); 2417 ret = __iommu_setup_intcapxt(iommu, iommu->ppr_irq_name, 2418 MMIO_INTCAPXT_PPR_OFFSET, 2419 amd_iommu_int_thread_pprlog); 2420 if (ret) 2421 return ret; 2422 2423 #ifdef CONFIG_IRQ_REMAP 2424 snprintf(iommu->ga_irq_name, sizeof(iommu->ga_irq_name), 2425 "AMD-Vi%d-GA", iommu->index); 2426 ret = __iommu_setup_intcapxt(iommu, iommu->ga_irq_name, 2427 MMIO_INTCAPXT_GALOG_OFFSET, 2428 amd_iommu_int_thread_galog); 2429 #endif 2430 2431 return ret; 2432 } 2433 2434 static int iommu_init_irq(struct amd_iommu *iommu) 2435 { 2436 int ret; 2437 2438 if (iommu->int_enabled) 2439 goto enable_faults; 2440 2441 if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE) 2442 ret = iommu_setup_intcapxt(iommu); 2443 else if (iommu->dev->msi_cap) 2444 ret = iommu_setup_msi(iommu); 2445 else 2446 ret = -ENODEV; 2447 2448 if (ret) 2449 return ret; 2450 2451 iommu->int_enabled = true; 2452 enable_faults: 2453 2454 if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE) 2455 iommu_feature_enable(iommu, CONTROL_INTCAPXT_EN); 2456 2457 iommu_feature_enable(iommu, CONTROL_EVT_INT_EN); 2458 2459 return 0; 2460 } 2461 2462 /**************************************************************************** 2463 * 2464 * The next functions belong to the third pass of parsing the ACPI 2465 * table. In this last pass the memory mapping requirements are 2466 * gathered (like exclusion and unity mapping ranges). 2467 * 2468 ****************************************************************************/ 2469 2470 static void __init free_unity_maps(void) 2471 { 2472 struct unity_map_entry *entry, *next; 2473 struct amd_iommu_pci_seg *p, *pci_seg; 2474 2475 for_each_pci_segment_safe(pci_seg, p) { 2476 list_for_each_entry_safe(entry, next, &pci_seg->unity_map, list) { 2477 list_del(&entry->list); 2478 kfree(entry); 2479 } 2480 } 2481 } 2482 2483 /* called for unity map ACPI definition */ 2484 static int __init init_unity_map_range(struct ivmd_header *m, 2485 struct acpi_table_header *ivrs_base) 2486 { 2487 struct unity_map_entry *e = NULL; 2488 struct amd_iommu_pci_seg *pci_seg; 2489 char *s; 2490 2491 pci_seg = get_pci_segment(m->pci_seg, ivrs_base); 2492 if (pci_seg == NULL) 2493 return -ENOMEM; 2494 2495 e = kzalloc(sizeof(*e), GFP_KERNEL); 2496 if (e == NULL) 2497 return -ENOMEM; 2498 2499 switch (m->type) { 2500 default: 2501 kfree(e); 2502 return 0; 2503 case ACPI_IVMD_TYPE: 2504 s = "IVMD_TYPEi\t\t\t"; 2505 e->devid_start = e->devid_end = m->devid; 2506 break; 2507 case ACPI_IVMD_TYPE_ALL: 2508 s = "IVMD_TYPE_ALL\t\t"; 2509 e->devid_start = 0; 2510 e->devid_end = pci_seg->last_bdf; 2511 break; 2512 case ACPI_IVMD_TYPE_RANGE: 2513 s = "IVMD_TYPE_RANGE\t\t"; 2514 e->devid_start = m->devid; 2515 e->devid_end = m->aux; 2516 break; 2517 } 2518 e->address_start = PAGE_ALIGN(m->range_start); 2519 e->address_end = e->address_start + PAGE_ALIGN(m->range_length); 2520 e->prot = m->flags >> 1; 2521 2522 /* 2523 * Treat per-device exclusion ranges as r/w unity-mapped regions 2524 * since some buggy BIOSes might lead to the overwritten exclusion 2525 * range (exclusion_start and exclusion_length members). This 2526 * happens when there are multiple exclusion ranges (IVMD entries) 2527 * defined in ACPI table. 2528 */ 2529 if (m->flags & IVMD_FLAG_EXCL_RANGE) 2530 e->prot = (IVMD_FLAG_IW | IVMD_FLAG_IR) >> 1; 2531 2532 DUMP_printk("%s devid_start: %04x:%02x:%02x.%x devid_end: " 2533 "%04x:%02x:%02x.%x range_start: %016llx range_end: %016llx" 2534 " flags: %x\n", s, m->pci_seg, 2535 PCI_BUS_NUM(e->devid_start), PCI_SLOT(e->devid_start), 2536 PCI_FUNC(e->devid_start), m->pci_seg, 2537 PCI_BUS_NUM(e->devid_end), 2538 PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end), 2539 e->address_start, e->address_end, m->flags); 2540 2541 list_add_tail(&e->list, &pci_seg->unity_map); 2542 2543 return 0; 2544 } 2545 2546 /* iterates over all memory definitions we find in the ACPI table */ 2547 static int __init init_memory_definitions(struct acpi_table_header *table) 2548 { 2549 u8 *p = (u8 *)table, *end = (u8 *)table; 2550 struct ivmd_header *m; 2551 2552 end += table->length; 2553 p += IVRS_HEADER_LENGTH; 2554 2555 while (p < end) { 2556 m = (struct ivmd_header *)p; 2557 if (m->flags & (IVMD_FLAG_UNITY_MAP | IVMD_FLAG_EXCL_RANGE)) 2558 init_unity_map_range(m, table); 2559 2560 p += m->length; 2561 } 2562 2563 return 0; 2564 } 2565 2566 /* 2567 * Init the device table to not allow DMA access for devices 2568 */ 2569 static void init_device_table_dma(struct amd_iommu_pci_seg *pci_seg) 2570 { 2571 u32 devid; 2572 struct dev_table_entry *dev_table = pci_seg->dev_table; 2573 2574 if (dev_table == NULL) 2575 return; 2576 2577 for (devid = 0; devid <= pci_seg->last_bdf; ++devid) { 2578 set_dte_bit(&dev_table[devid], DEV_ENTRY_VALID); 2579 if (!amd_iommu_snp_en) 2580 set_dte_bit(&dev_table[devid], DEV_ENTRY_TRANSLATION); 2581 } 2582 } 2583 2584 static void __init uninit_device_table_dma(struct amd_iommu_pci_seg *pci_seg) 2585 { 2586 u32 devid; 2587 struct dev_table_entry *dev_table = pci_seg->dev_table; 2588 2589 if (dev_table == NULL) 2590 return; 2591 2592 for (devid = 0; devid <= pci_seg->last_bdf; ++devid) { 2593 dev_table[devid].data[0] = 0ULL; 2594 dev_table[devid].data[1] = 0ULL; 2595 } 2596 } 2597 2598 static void init_device_table(void) 2599 { 2600 struct amd_iommu_pci_seg *pci_seg; 2601 u32 devid; 2602 2603 if (!amd_iommu_irq_remap) 2604 return; 2605 2606 for_each_pci_segment(pci_seg) { 2607 for (devid = 0; devid <= pci_seg->last_bdf; ++devid) 2608 set_dte_bit(&pci_seg->dev_table[devid], DEV_ENTRY_IRQ_TBL_EN); 2609 } 2610 } 2611 2612 static void iommu_init_flags(struct amd_iommu *iommu) 2613 { 2614 iommu->acpi_flags & IVHD_FLAG_HT_TUN_EN_MASK ? 2615 iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) : 2616 iommu_feature_disable(iommu, CONTROL_HT_TUN_EN); 2617 2618 iommu->acpi_flags & IVHD_FLAG_PASSPW_EN_MASK ? 2619 iommu_feature_enable(iommu, CONTROL_PASSPW_EN) : 2620 iommu_feature_disable(iommu, CONTROL_PASSPW_EN); 2621 2622 iommu->acpi_flags & IVHD_FLAG_RESPASSPW_EN_MASK ? 2623 iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) : 2624 iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN); 2625 2626 iommu->acpi_flags & IVHD_FLAG_ISOC_EN_MASK ? 2627 iommu_feature_enable(iommu, CONTROL_ISOC_EN) : 2628 iommu_feature_disable(iommu, CONTROL_ISOC_EN); 2629 2630 /* 2631 * make IOMMU memory accesses cache coherent 2632 */ 2633 iommu_feature_enable(iommu, CONTROL_COHERENT_EN); 2634 2635 /* Set IOTLB invalidation timeout to 1s */ 2636 iommu_feature_set(iommu, CTRL_INV_TO_1S, CTRL_INV_TO_MASK, CONTROL_INV_TIMEOUT); 2637 2638 /* Enable Enhanced Peripheral Page Request Handling */ 2639 if (check_feature(FEATURE_EPHSUP)) 2640 iommu_feature_enable(iommu, CONTROL_EPH_EN); 2641 } 2642 2643 static void iommu_apply_resume_quirks(struct amd_iommu *iommu) 2644 { 2645 int i, j; 2646 u32 ioc_feature_control; 2647 struct pci_dev *pdev = iommu->root_pdev; 2648 2649 /* RD890 BIOSes may not have completely reconfigured the iommu */ 2650 if (!is_rd890_iommu(iommu->dev) || !pdev) 2651 return; 2652 2653 /* 2654 * First, we need to ensure that the iommu is enabled. This is 2655 * controlled by a register in the northbridge 2656 */ 2657 2658 /* Select Northbridge indirect register 0x75 and enable writing */ 2659 pci_write_config_dword(pdev, 0x60, 0x75 | (1 << 7)); 2660 pci_read_config_dword(pdev, 0x64, &ioc_feature_control); 2661 2662 /* Enable the iommu */ 2663 if (!(ioc_feature_control & 0x1)) 2664 pci_write_config_dword(pdev, 0x64, ioc_feature_control | 1); 2665 2666 /* Restore the iommu BAR */ 2667 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4, 2668 iommu->stored_addr_lo); 2669 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8, 2670 iommu->stored_addr_hi); 2671 2672 /* Restore the l1 indirect regs for each of the 6 l1s */ 2673 for (i = 0; i < 6; i++) 2674 for (j = 0; j < 0x12; j++) 2675 iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]); 2676 2677 /* Restore the l2 indirect regs */ 2678 for (i = 0; i < 0x83; i++) 2679 iommu_write_l2(iommu, i, iommu->stored_l2[i]); 2680 2681 /* Lock PCI setup registers */ 2682 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4, 2683 iommu->stored_addr_lo | 1); 2684 } 2685 2686 static void iommu_enable_ga(struct amd_iommu *iommu) 2687 { 2688 #ifdef CONFIG_IRQ_REMAP 2689 switch (amd_iommu_guest_ir) { 2690 case AMD_IOMMU_GUEST_IR_VAPIC: 2691 case AMD_IOMMU_GUEST_IR_LEGACY_GA: 2692 iommu_feature_enable(iommu, CONTROL_GA_EN); 2693 iommu->irte_ops = &irte_128_ops; 2694 break; 2695 default: 2696 iommu->irte_ops = &irte_32_ops; 2697 break; 2698 } 2699 #endif 2700 } 2701 2702 static void iommu_disable_irtcachedis(struct amd_iommu *iommu) 2703 { 2704 iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS); 2705 } 2706 2707 static void iommu_enable_irtcachedis(struct amd_iommu *iommu) 2708 { 2709 u64 ctrl; 2710 2711 if (!amd_iommu_irtcachedis) 2712 return; 2713 2714 /* 2715 * Note: 2716 * The support for IRTCacheDis feature is dertermined by 2717 * checking if the bit is writable. 2718 */ 2719 iommu_feature_enable(iommu, CONTROL_IRTCACHEDIS); 2720 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); 2721 ctrl &= (1ULL << CONTROL_IRTCACHEDIS); 2722 if (ctrl) 2723 iommu->irtcachedis_enabled = true; 2724 pr_info("iommu%d (%#06x) : IRT cache is %s\n", 2725 iommu->index, iommu->devid, 2726 iommu->irtcachedis_enabled ? "disabled" : "enabled"); 2727 } 2728 2729 static void iommu_enable_2k_int(struct amd_iommu *iommu) 2730 { 2731 if (!FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2)) 2732 return; 2733 2734 iommu_feature_set(iommu, 2735 CONTROL_NUM_INT_REMAP_MODE_2K, 2736 CONTROL_NUM_INT_REMAP_MODE_MASK, 2737 CONTROL_NUM_INT_REMAP_MODE); 2738 } 2739 2740 static void early_enable_iommu(struct amd_iommu *iommu) 2741 { 2742 iommu_disable(iommu); 2743 iommu_init_flags(iommu); 2744 iommu_set_device_table(iommu); 2745 iommu_enable_command_buffer(iommu); 2746 iommu_enable_event_buffer(iommu); 2747 iommu_set_exclusion_range(iommu); 2748 iommu_enable_gt(iommu); 2749 iommu_enable_ga(iommu); 2750 iommu_enable_xt(iommu); 2751 iommu_enable_irtcachedis(iommu); 2752 iommu_enable_2k_int(iommu); 2753 iommu_enable(iommu); 2754 amd_iommu_flush_all_caches(iommu); 2755 } 2756 2757 /* 2758 * This function finally enables all IOMMUs found in the system after 2759 * they have been initialized. 2760 * 2761 * Or if in kdump kernel and IOMMUs are all pre-enabled, try to copy 2762 * the old content of device table entries. Not this case or copy failed, 2763 * just continue as normal kernel does. 2764 */ 2765 static void early_enable_iommus(void) 2766 { 2767 struct amd_iommu *iommu; 2768 struct amd_iommu_pci_seg *pci_seg; 2769 2770 if (!copy_device_table()) { 2771 /* 2772 * If come here because of failure in copying device table from old 2773 * kernel with all IOMMUs enabled, print error message and try to 2774 * free allocated old_dev_tbl_cpy. 2775 */ 2776 if (amd_iommu_pre_enabled) 2777 pr_err("Failed to copy DEV table from previous kernel.\n"); 2778 2779 for_each_pci_segment(pci_seg) { 2780 if (pci_seg->old_dev_tbl_cpy != NULL) { 2781 iommu_free_pages(pci_seg->old_dev_tbl_cpy); 2782 pci_seg->old_dev_tbl_cpy = NULL; 2783 } 2784 } 2785 2786 for_each_iommu(iommu) { 2787 clear_translation_pre_enabled(iommu); 2788 early_enable_iommu(iommu); 2789 } 2790 } else { 2791 pr_info("Copied DEV table from previous kernel.\n"); 2792 2793 for_each_pci_segment(pci_seg) { 2794 iommu_free_pages(pci_seg->dev_table); 2795 pci_seg->dev_table = pci_seg->old_dev_tbl_cpy; 2796 } 2797 2798 for_each_iommu(iommu) { 2799 iommu_disable_command_buffer(iommu); 2800 iommu_disable_event_buffer(iommu); 2801 iommu_disable_irtcachedis(iommu); 2802 iommu_enable_command_buffer(iommu); 2803 iommu_enable_event_buffer(iommu); 2804 iommu_enable_ga(iommu); 2805 iommu_enable_xt(iommu); 2806 iommu_enable_irtcachedis(iommu); 2807 iommu_enable_2k_int(iommu); 2808 iommu_set_device_table(iommu); 2809 amd_iommu_flush_all_caches(iommu); 2810 } 2811 } 2812 } 2813 2814 static void enable_iommus_ppr(void) 2815 { 2816 struct amd_iommu *iommu; 2817 2818 if (!amd_iommu_gt_ppr_supported()) 2819 return; 2820 2821 for_each_iommu(iommu) 2822 amd_iommu_enable_ppr_log(iommu); 2823 } 2824 2825 static void enable_iommus_vapic(void) 2826 { 2827 #ifdef CONFIG_IRQ_REMAP 2828 u32 status, i; 2829 struct amd_iommu *iommu; 2830 2831 for_each_iommu(iommu) { 2832 /* 2833 * Disable GALog if already running. It could have been enabled 2834 * in the previous boot before kdump. 2835 */ 2836 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); 2837 if (!(status & MMIO_STATUS_GALOG_RUN_MASK)) 2838 continue; 2839 2840 iommu_feature_disable(iommu, CONTROL_GALOG_EN); 2841 iommu_feature_disable(iommu, CONTROL_GAINT_EN); 2842 2843 /* 2844 * Need to set and poll check the GALOGRun bit to zero before 2845 * we can set/ modify GA Log registers safely. 2846 */ 2847 for (i = 0; i < MMIO_STATUS_TIMEOUT; ++i) { 2848 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); 2849 if (!(status & MMIO_STATUS_GALOG_RUN_MASK)) 2850 break; 2851 udelay(10); 2852 } 2853 2854 if (WARN_ON(i >= MMIO_STATUS_TIMEOUT)) 2855 return; 2856 } 2857 2858 if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) && 2859 !check_feature(FEATURE_GAM_VAPIC)) { 2860 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA; 2861 return; 2862 } 2863 2864 if (amd_iommu_snp_en && 2865 !FEATURE_SNPAVICSUP_GAM(amd_iommu_efr2)) { 2866 pr_warn("Force to disable Virtual APIC due to SNP\n"); 2867 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA; 2868 return; 2869 } 2870 2871 /* Enabling GAM and SNPAVIC support */ 2872 for_each_iommu(iommu) { 2873 if (iommu_init_ga_log(iommu) || 2874 iommu_ga_log_enable(iommu)) 2875 return; 2876 2877 iommu_feature_enable(iommu, CONTROL_GAM_EN); 2878 if (amd_iommu_snp_en) 2879 iommu_feature_enable(iommu, CONTROL_SNPAVIC_EN); 2880 } 2881 2882 amd_iommu_irq_ops.capability |= (1 << IRQ_POSTING_CAP); 2883 pr_info("Virtual APIC enabled\n"); 2884 #endif 2885 } 2886 2887 static void disable_iommus(void) 2888 { 2889 struct amd_iommu *iommu; 2890 2891 for_each_iommu(iommu) 2892 iommu_disable(iommu); 2893 2894 #ifdef CONFIG_IRQ_REMAP 2895 if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) 2896 amd_iommu_irq_ops.capability &= ~(1 << IRQ_POSTING_CAP); 2897 #endif 2898 } 2899 2900 /* 2901 * Suspend/Resume support 2902 * disable suspend until real resume implemented 2903 */ 2904 2905 static void amd_iommu_resume(void) 2906 { 2907 struct amd_iommu *iommu; 2908 2909 for_each_iommu(iommu) 2910 iommu_apply_resume_quirks(iommu); 2911 2912 /* re-load the hardware */ 2913 for_each_iommu(iommu) 2914 early_enable_iommu(iommu); 2915 2916 amd_iommu_enable_interrupts(); 2917 } 2918 2919 static int amd_iommu_suspend(void) 2920 { 2921 /* disable IOMMUs to go out of the way for BIOS */ 2922 disable_iommus(); 2923 2924 return 0; 2925 } 2926 2927 static struct syscore_ops amd_iommu_syscore_ops = { 2928 .suspend = amd_iommu_suspend, 2929 .resume = amd_iommu_resume, 2930 }; 2931 2932 static void __init free_iommu_resources(void) 2933 { 2934 free_iommu_all(); 2935 free_pci_segments(); 2936 } 2937 2938 /* SB IOAPIC is always on this device in AMD systems */ 2939 #define IOAPIC_SB_DEVID ((0x00 << 8) | PCI_DEVFN(0x14, 0)) 2940 2941 static bool __init check_ioapic_information(void) 2942 { 2943 const char *fw_bug = FW_BUG; 2944 bool ret, has_sb_ioapic; 2945 int idx; 2946 2947 has_sb_ioapic = false; 2948 ret = false; 2949 2950 /* 2951 * If we have map overrides on the kernel command line the 2952 * messages in this function might not describe firmware bugs 2953 * anymore - so be careful 2954 */ 2955 if (cmdline_maps) 2956 fw_bug = ""; 2957 2958 for (idx = 0; idx < nr_ioapics; idx++) { 2959 int devid, id = mpc_ioapic_id(idx); 2960 2961 devid = get_ioapic_devid(id); 2962 if (devid < 0) { 2963 pr_err("%s: IOAPIC[%d] not in IVRS table\n", 2964 fw_bug, id); 2965 ret = false; 2966 } else if (devid == IOAPIC_SB_DEVID) { 2967 has_sb_ioapic = true; 2968 ret = true; 2969 } 2970 } 2971 2972 if (!has_sb_ioapic) { 2973 /* 2974 * We expect the SB IOAPIC to be listed in the IVRS 2975 * table. The system timer is connected to the SB IOAPIC 2976 * and if we don't have it in the list the system will 2977 * panic at boot time. This situation usually happens 2978 * when the BIOS is buggy and provides us the wrong 2979 * device id for the IOAPIC in the system. 2980 */ 2981 pr_err("%s: No southbridge IOAPIC found\n", fw_bug); 2982 } 2983 2984 if (!ret) 2985 pr_err("Disabling interrupt remapping\n"); 2986 2987 return ret; 2988 } 2989 2990 static void __init free_dma_resources(void) 2991 { 2992 ida_destroy(&pdom_ids); 2993 2994 free_unity_maps(); 2995 } 2996 2997 static void __init ivinfo_init(void *ivrs) 2998 { 2999 amd_iommu_ivinfo = *((u32 *)(ivrs + IOMMU_IVINFO_OFFSET)); 3000 } 3001 3002 /* 3003 * This is the hardware init function for AMD IOMMU in the system. 3004 * This function is called either from amd_iommu_init or from the interrupt 3005 * remapping setup code. 3006 * 3007 * This function basically parses the ACPI table for AMD IOMMU (IVRS) 3008 * four times: 3009 * 3010 * 1 pass) Discover the most comprehensive IVHD type to use. 3011 * 3012 * 2 pass) Find the highest PCI device id the driver has to handle. 3013 * Upon this information the size of the data structures is 3014 * determined that needs to be allocated. 3015 * 3016 * 3 pass) Initialize the data structures just allocated with the 3017 * information in the ACPI table about available AMD IOMMUs 3018 * in the system. It also maps the PCI devices in the 3019 * system to specific IOMMUs 3020 * 3021 * 4 pass) After the basic data structures are allocated and 3022 * initialized we update them with information about memory 3023 * remapping requirements parsed out of the ACPI table in 3024 * this last pass. 3025 * 3026 * After everything is set up the IOMMUs are enabled and the necessary 3027 * hotplug and suspend notifiers are registered. 3028 */ 3029 static int __init early_amd_iommu_init(void) 3030 { 3031 struct acpi_table_header *ivrs_base; 3032 int ret; 3033 acpi_status status; 3034 3035 if (!amd_iommu_detected) 3036 return -ENODEV; 3037 3038 status = acpi_get_table("IVRS", 0, &ivrs_base); 3039 if (status == AE_NOT_FOUND) 3040 return -ENODEV; 3041 else if (ACPI_FAILURE(status)) { 3042 const char *err = acpi_format_exception(status); 3043 pr_err("IVRS table error: %s\n", err); 3044 return -EINVAL; 3045 } 3046 3047 if (!boot_cpu_has(X86_FEATURE_CX16)) { 3048 pr_err("Failed to initialize. The CMPXCHG16B feature is required.\n"); 3049 return -EINVAL; 3050 } 3051 3052 /* 3053 * Validate checksum here so we don't need to do it when 3054 * we actually parse the table 3055 */ 3056 ret = check_ivrs_checksum(ivrs_base); 3057 if (ret) 3058 goto out; 3059 3060 ivinfo_init(ivrs_base); 3061 3062 amd_iommu_target_ivhd_type = get_highest_supported_ivhd_type(ivrs_base); 3063 DUMP_printk("Using IVHD type %#x\n", amd_iommu_target_ivhd_type); 3064 3065 /* 3066 * now the data structures are allocated and basically initialized 3067 * start the real acpi table scan 3068 */ 3069 ret = init_iommu_all(ivrs_base); 3070 if (ret) 3071 goto out; 3072 3073 /* 5 level guest page table */ 3074 if (cpu_feature_enabled(X86_FEATURE_LA57) && 3075 FIELD_GET(FEATURE_GATS, amd_iommu_efr) == GUEST_PGTABLE_5_LEVEL) 3076 amd_iommu_gpt_level = PAGE_MODE_5_LEVEL; 3077 3078 if (amd_iommu_pgtable == PD_MODE_V2) { 3079 if (!amd_iommu_v2_pgtbl_supported()) { 3080 pr_warn("Cannot enable v2 page table for DMA-API. Fallback to v1.\n"); 3081 amd_iommu_pgtable = PD_MODE_V1; 3082 } 3083 } 3084 3085 /* Disable any previously enabled IOMMUs */ 3086 if (!is_kdump_kernel() || amd_iommu_disabled) 3087 disable_iommus(); 3088 3089 if (amd_iommu_irq_remap) 3090 amd_iommu_irq_remap = check_ioapic_information(); 3091 3092 if (amd_iommu_irq_remap) { 3093 struct amd_iommu_pci_seg *pci_seg; 3094 ret = -ENOMEM; 3095 for_each_pci_segment(pci_seg) { 3096 if (alloc_irq_lookup_table(pci_seg)) 3097 goto out; 3098 } 3099 } 3100 3101 ret = init_memory_definitions(ivrs_base); 3102 if (ret) 3103 goto out; 3104 3105 /* init the device table */ 3106 init_device_table(); 3107 3108 out: 3109 /* Don't leak any ACPI memory */ 3110 acpi_put_table(ivrs_base); 3111 3112 return ret; 3113 } 3114 3115 static int amd_iommu_enable_interrupts(void) 3116 { 3117 struct amd_iommu *iommu; 3118 int ret = 0; 3119 3120 for_each_iommu(iommu) { 3121 ret = iommu_init_irq(iommu); 3122 if (ret) 3123 goto out; 3124 } 3125 3126 /* 3127 * Interrupt handler is ready to process interrupts. Enable 3128 * PPR and GA log interrupt for all IOMMUs. 3129 */ 3130 enable_iommus_vapic(); 3131 enable_iommus_ppr(); 3132 3133 out: 3134 return ret; 3135 } 3136 3137 static bool __init detect_ivrs(void) 3138 { 3139 struct acpi_table_header *ivrs_base; 3140 acpi_status status; 3141 int i; 3142 3143 status = acpi_get_table("IVRS", 0, &ivrs_base); 3144 if (status == AE_NOT_FOUND) 3145 return false; 3146 else if (ACPI_FAILURE(status)) { 3147 const char *err = acpi_format_exception(status); 3148 pr_err("IVRS table error: %s\n", err); 3149 return false; 3150 } 3151 3152 acpi_put_table(ivrs_base); 3153 3154 if (amd_iommu_force_enable) 3155 goto out; 3156 3157 /* Don't use IOMMU if there is Stoney Ridge graphics */ 3158 for (i = 0; i < 32; i++) { 3159 u32 pci_id; 3160 3161 pci_id = read_pci_config(0, i, 0, 0); 3162 if ((pci_id & 0xffff) == 0x1002 && (pci_id >> 16) == 0x98e4) { 3163 pr_info("Disable IOMMU on Stoney Ridge\n"); 3164 return false; 3165 } 3166 } 3167 3168 out: 3169 /* Make sure ACS will be enabled during PCI probe */ 3170 pci_request_acs(); 3171 3172 return true; 3173 } 3174 3175 static __init void iommu_snp_enable(void) 3176 { 3177 #ifdef CONFIG_KVM_AMD_SEV 3178 if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) 3179 return; 3180 /* 3181 * The SNP support requires that IOMMU must be enabled, and is 3182 * configured with V1 page table (DTE[Mode] = 0 is not supported). 3183 */ 3184 if (no_iommu || iommu_default_passthrough()) { 3185 pr_warn("SNP: IOMMU disabled or configured in passthrough mode, SNP cannot be supported.\n"); 3186 goto disable_snp; 3187 } 3188 3189 if (amd_iommu_pgtable != PD_MODE_V1) { 3190 pr_warn("SNP: IOMMU is configured with V2 page table mode, SNP cannot be supported.\n"); 3191 goto disable_snp; 3192 } 3193 3194 amd_iommu_snp_en = check_feature(FEATURE_SNP); 3195 if (!amd_iommu_snp_en) { 3196 pr_warn("SNP: IOMMU SNP feature not enabled, SNP cannot be supported.\n"); 3197 goto disable_snp; 3198 } 3199 3200 /* 3201 * Enable host SNP support once SNP support is checked on IOMMU. 3202 */ 3203 if (snp_rmptable_init()) { 3204 pr_warn("SNP: RMP initialization failed, SNP cannot be supported.\n"); 3205 goto disable_snp; 3206 } 3207 3208 pr_info("IOMMU SNP support enabled.\n"); 3209 return; 3210 3211 disable_snp: 3212 cc_platform_clear(CC_ATTR_HOST_SEV_SNP); 3213 #endif 3214 } 3215 3216 /**************************************************************************** 3217 * 3218 * AMD IOMMU Initialization State Machine 3219 * 3220 ****************************************************************************/ 3221 3222 static int __init state_next(void) 3223 { 3224 int ret = 0; 3225 3226 switch (init_state) { 3227 case IOMMU_START_STATE: 3228 if (!detect_ivrs()) { 3229 init_state = IOMMU_NOT_FOUND; 3230 ret = -ENODEV; 3231 } else { 3232 init_state = IOMMU_IVRS_DETECTED; 3233 } 3234 break; 3235 case IOMMU_IVRS_DETECTED: 3236 if (amd_iommu_disabled) { 3237 init_state = IOMMU_CMDLINE_DISABLED; 3238 ret = -EINVAL; 3239 } else { 3240 ret = early_amd_iommu_init(); 3241 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED; 3242 } 3243 break; 3244 case IOMMU_ACPI_FINISHED: 3245 early_enable_iommus(); 3246 x86_platform.iommu_shutdown = disable_iommus; 3247 init_state = IOMMU_ENABLED; 3248 break; 3249 case IOMMU_ENABLED: 3250 register_syscore_ops(&amd_iommu_syscore_ops); 3251 iommu_snp_enable(); 3252 ret = amd_iommu_init_pci(); 3253 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_PCI_INIT; 3254 break; 3255 case IOMMU_PCI_INIT: 3256 ret = amd_iommu_enable_interrupts(); 3257 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_INTERRUPTS_EN; 3258 break; 3259 case IOMMU_INTERRUPTS_EN: 3260 init_state = IOMMU_INITIALIZED; 3261 break; 3262 case IOMMU_INITIALIZED: 3263 /* Nothing to do */ 3264 break; 3265 case IOMMU_NOT_FOUND: 3266 case IOMMU_INIT_ERROR: 3267 case IOMMU_CMDLINE_DISABLED: 3268 /* Error states => do nothing */ 3269 ret = -EINVAL; 3270 break; 3271 default: 3272 /* Unknown state */ 3273 BUG(); 3274 } 3275 3276 if (ret) { 3277 free_dma_resources(); 3278 if (!irq_remapping_enabled) { 3279 disable_iommus(); 3280 free_iommu_resources(); 3281 } else { 3282 struct amd_iommu *iommu; 3283 struct amd_iommu_pci_seg *pci_seg; 3284 3285 for_each_pci_segment(pci_seg) 3286 uninit_device_table_dma(pci_seg); 3287 3288 for_each_iommu(iommu) 3289 amd_iommu_flush_all_caches(iommu); 3290 } 3291 } 3292 return ret; 3293 } 3294 3295 static int __init iommu_go_to_state(enum iommu_init_state state) 3296 { 3297 int ret = -EINVAL; 3298 3299 while (init_state != state) { 3300 if (init_state == IOMMU_NOT_FOUND || 3301 init_state == IOMMU_INIT_ERROR || 3302 init_state == IOMMU_CMDLINE_DISABLED) 3303 break; 3304 ret = state_next(); 3305 } 3306 3307 /* 3308 * SNP platform initilazation requires IOMMUs to be fully configured. 3309 * If the SNP support on IOMMUs has NOT been checked, simply mark SNP 3310 * as unsupported. If the SNP support on IOMMUs has been checked and 3311 * host SNP support enabled but RMP enforcement has not been enabled 3312 * in IOMMUs, then the system is in a half-baked state, but can limp 3313 * along as all memory should be Hypervisor-Owned in the RMP. WARN, 3314 * but leave SNP as "supported" to avoid confusing the kernel. 3315 */ 3316 if (ret && cc_platform_has(CC_ATTR_HOST_SEV_SNP) && 3317 !WARN_ON_ONCE(amd_iommu_snp_en)) 3318 cc_platform_clear(CC_ATTR_HOST_SEV_SNP); 3319 3320 return ret; 3321 } 3322 3323 #ifdef CONFIG_IRQ_REMAP 3324 int __init amd_iommu_prepare(void) 3325 { 3326 int ret; 3327 3328 amd_iommu_irq_remap = true; 3329 3330 ret = iommu_go_to_state(IOMMU_ACPI_FINISHED); 3331 if (ret) { 3332 amd_iommu_irq_remap = false; 3333 return ret; 3334 } 3335 3336 return amd_iommu_irq_remap ? 0 : -ENODEV; 3337 } 3338 3339 int __init amd_iommu_enable(void) 3340 { 3341 int ret; 3342 3343 ret = iommu_go_to_state(IOMMU_ENABLED); 3344 if (ret) 3345 return ret; 3346 3347 irq_remapping_enabled = 1; 3348 return amd_iommu_xt_mode; 3349 } 3350 3351 void amd_iommu_disable(void) 3352 { 3353 amd_iommu_suspend(); 3354 } 3355 3356 int amd_iommu_reenable(int mode) 3357 { 3358 amd_iommu_resume(); 3359 3360 return 0; 3361 } 3362 3363 int amd_iommu_enable_faulting(unsigned int cpu) 3364 { 3365 /* We enable MSI later when PCI is initialized */ 3366 return 0; 3367 } 3368 #endif 3369 3370 /* 3371 * This is the core init function for AMD IOMMU hardware in the system. 3372 * This function is called from the generic x86 DMA layer initialization 3373 * code. 3374 */ 3375 static int __init amd_iommu_init(void) 3376 { 3377 struct amd_iommu *iommu; 3378 int ret; 3379 3380 ret = iommu_go_to_state(IOMMU_INITIALIZED); 3381 #ifdef CONFIG_GART_IOMMU 3382 if (ret && list_empty(&amd_iommu_list)) { 3383 /* 3384 * We failed to initialize the AMD IOMMU - try fallback 3385 * to GART if possible. 3386 */ 3387 gart_iommu_init(); 3388 } 3389 #endif 3390 3391 for_each_iommu(iommu) 3392 amd_iommu_debugfs_setup(iommu); 3393 3394 return ret; 3395 } 3396 3397 static bool amd_iommu_sme_check(void) 3398 { 3399 if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT) || 3400 (boot_cpu_data.x86 != 0x17)) 3401 return true; 3402 3403 /* For Fam17h, a specific level of support is required */ 3404 if (boot_cpu_data.microcode >= 0x08001205) 3405 return true; 3406 3407 if ((boot_cpu_data.microcode >= 0x08001126) && 3408 (boot_cpu_data.microcode <= 0x080011ff)) 3409 return true; 3410 3411 pr_notice("IOMMU not currently supported when SME is active\n"); 3412 3413 return false; 3414 } 3415 3416 /**************************************************************************** 3417 * 3418 * Early detect code. This code runs at IOMMU detection time in the DMA 3419 * layer. It just looks if there is an IVRS ACPI table to detect AMD 3420 * IOMMUs 3421 * 3422 ****************************************************************************/ 3423 void __init amd_iommu_detect(void) 3424 { 3425 int ret; 3426 3427 if (no_iommu || (iommu_detected && !gart_iommu_aperture)) 3428 goto disable_snp; 3429 3430 if (!amd_iommu_sme_check()) 3431 goto disable_snp; 3432 3433 ret = iommu_go_to_state(IOMMU_IVRS_DETECTED); 3434 if (ret) 3435 goto disable_snp; 3436 3437 amd_iommu_detected = true; 3438 iommu_detected = 1; 3439 x86_init.iommu.iommu_init = amd_iommu_init; 3440 return; 3441 3442 disable_snp: 3443 if (cc_platform_has(CC_ATTR_HOST_SEV_SNP)) 3444 cc_platform_clear(CC_ATTR_HOST_SEV_SNP); 3445 } 3446 3447 /**************************************************************************** 3448 * 3449 * Parsing functions for the AMD IOMMU specific kernel command line 3450 * options. 3451 * 3452 ****************************************************************************/ 3453 3454 static int __init parse_amd_iommu_dump(char *str) 3455 { 3456 amd_iommu_dump = true; 3457 3458 return 1; 3459 } 3460 3461 static int __init parse_amd_iommu_intr(char *str) 3462 { 3463 for (; *str; ++str) { 3464 if (strncmp(str, "legacy", 6) == 0) { 3465 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA; 3466 break; 3467 } 3468 if (strncmp(str, "vapic", 5) == 0) { 3469 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC; 3470 break; 3471 } 3472 } 3473 return 1; 3474 } 3475 3476 static int __init parse_amd_iommu_options(char *str) 3477 { 3478 if (!str) 3479 return -EINVAL; 3480 3481 while (*str) { 3482 if (strncmp(str, "fullflush", 9) == 0) { 3483 pr_warn("amd_iommu=fullflush deprecated; use iommu.strict=1 instead\n"); 3484 iommu_set_dma_strict(); 3485 } else if (strncmp(str, "force_enable", 12) == 0) { 3486 amd_iommu_force_enable = true; 3487 } else if (strncmp(str, "off", 3) == 0) { 3488 amd_iommu_disabled = true; 3489 } else if (strncmp(str, "force_isolation", 15) == 0) { 3490 amd_iommu_force_isolation = true; 3491 } else if (strncmp(str, "pgtbl_v1", 8) == 0) { 3492 amd_iommu_pgtable = PD_MODE_V1; 3493 } else if (strncmp(str, "pgtbl_v2", 8) == 0) { 3494 amd_iommu_pgtable = PD_MODE_V2; 3495 } else if (strncmp(str, "irtcachedis", 11) == 0) { 3496 amd_iommu_irtcachedis = true; 3497 } else if (strncmp(str, "nohugepages", 11) == 0) { 3498 pr_info("Restricting V1 page-sizes to 4KiB"); 3499 amd_iommu_pgsize_bitmap = AMD_IOMMU_PGSIZES_4K; 3500 } else if (strncmp(str, "v2_pgsizes_only", 15) == 0) { 3501 pr_info("Restricting V1 page-sizes to 4KiB/2MiB/1GiB"); 3502 amd_iommu_pgsize_bitmap = AMD_IOMMU_PGSIZES_V2; 3503 } else { 3504 pr_notice("Unknown option - '%s'\n", str); 3505 } 3506 3507 str += strcspn(str, ","); 3508 while (*str == ',') 3509 str++; 3510 } 3511 3512 return 1; 3513 } 3514 3515 static int __init parse_ivrs_ioapic(char *str) 3516 { 3517 u32 seg = 0, bus, dev, fn; 3518 int id, i; 3519 u32 devid; 3520 3521 if (sscanf(str, "=%d@%x:%x.%x", &id, &bus, &dev, &fn) == 4 || 3522 sscanf(str, "=%d@%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5) 3523 goto found; 3524 3525 if (sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn) == 4 || 3526 sscanf(str, "[%d]=%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5) { 3527 pr_warn("ivrs_ioapic%s option format deprecated; use ivrs_ioapic=%d@%04x:%02x:%02x.%d instead\n", 3528 str, id, seg, bus, dev, fn); 3529 goto found; 3530 } 3531 3532 pr_err("Invalid command line: ivrs_ioapic%s\n", str); 3533 return 1; 3534 3535 found: 3536 if (early_ioapic_map_size == EARLY_MAP_SIZE) { 3537 pr_err("Early IOAPIC map overflow - ignoring ivrs_ioapic%s\n", 3538 str); 3539 return 1; 3540 } 3541 3542 devid = IVRS_GET_SBDF_ID(seg, bus, dev, fn); 3543 3544 cmdline_maps = true; 3545 i = early_ioapic_map_size++; 3546 early_ioapic_map[i].id = id; 3547 early_ioapic_map[i].devid = devid; 3548 early_ioapic_map[i].cmd_line = true; 3549 3550 return 1; 3551 } 3552 3553 static int __init parse_ivrs_hpet(char *str) 3554 { 3555 u32 seg = 0, bus, dev, fn; 3556 int id, i; 3557 u32 devid; 3558 3559 if (sscanf(str, "=%d@%x:%x.%x", &id, &bus, &dev, &fn) == 4 || 3560 sscanf(str, "=%d@%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5) 3561 goto found; 3562 3563 if (sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn) == 4 || 3564 sscanf(str, "[%d]=%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5) { 3565 pr_warn("ivrs_hpet%s option format deprecated; use ivrs_hpet=%d@%04x:%02x:%02x.%d instead\n", 3566 str, id, seg, bus, dev, fn); 3567 goto found; 3568 } 3569 3570 pr_err("Invalid command line: ivrs_hpet%s\n", str); 3571 return 1; 3572 3573 found: 3574 if (early_hpet_map_size == EARLY_MAP_SIZE) { 3575 pr_err("Early HPET map overflow - ignoring ivrs_hpet%s\n", 3576 str); 3577 return 1; 3578 } 3579 3580 devid = IVRS_GET_SBDF_ID(seg, bus, dev, fn); 3581 3582 cmdline_maps = true; 3583 i = early_hpet_map_size++; 3584 early_hpet_map[i].id = id; 3585 early_hpet_map[i].devid = devid; 3586 early_hpet_map[i].cmd_line = true; 3587 3588 return 1; 3589 } 3590 3591 #define ACPIID_LEN (ACPIHID_UID_LEN + ACPIHID_HID_LEN) 3592 3593 static int __init parse_ivrs_acpihid(char *str) 3594 { 3595 u32 seg = 0, bus, dev, fn; 3596 char *hid, *uid, *p, *addr; 3597 char acpiid[ACPIID_LEN] = {0}; 3598 int i; 3599 3600 addr = strchr(str, '@'); 3601 if (!addr) { 3602 addr = strchr(str, '='); 3603 if (!addr) 3604 goto not_found; 3605 3606 ++addr; 3607 3608 if (strlen(addr) > ACPIID_LEN) 3609 goto not_found; 3610 3611 if (sscanf(str, "[%x:%x.%x]=%s", &bus, &dev, &fn, acpiid) == 4 || 3612 sscanf(str, "[%x:%x:%x.%x]=%s", &seg, &bus, &dev, &fn, acpiid) == 5) { 3613 pr_warn("ivrs_acpihid%s option format deprecated; use ivrs_acpihid=%s@%04x:%02x:%02x.%d instead\n", 3614 str, acpiid, seg, bus, dev, fn); 3615 goto found; 3616 } 3617 goto not_found; 3618 } 3619 3620 /* We have the '@', make it the terminator to get just the acpiid */ 3621 *addr++ = 0; 3622 3623 if (strlen(str) > ACPIID_LEN + 1) 3624 goto not_found; 3625 3626 if (sscanf(str, "=%s", acpiid) != 1) 3627 goto not_found; 3628 3629 if (sscanf(addr, "%x:%x.%x", &bus, &dev, &fn) == 3 || 3630 sscanf(addr, "%x:%x:%x.%x", &seg, &bus, &dev, &fn) == 4) 3631 goto found; 3632 3633 not_found: 3634 pr_err("Invalid command line: ivrs_acpihid%s\n", str); 3635 return 1; 3636 3637 found: 3638 p = acpiid; 3639 hid = strsep(&p, ":"); 3640 uid = p; 3641 3642 if (!hid || !(*hid) || !uid) { 3643 pr_err("Invalid command line: hid or uid\n"); 3644 return 1; 3645 } 3646 3647 /* 3648 * Ignore leading zeroes after ':', so e.g., AMDI0095:00 3649 * will match AMDI0095:0 in the second strcmp in acpi_dev_hid_uid_match 3650 */ 3651 while (*uid == '0' && *(uid + 1)) 3652 uid++; 3653 3654 i = early_acpihid_map_size++; 3655 memcpy(early_acpihid_map[i].hid, hid, strlen(hid)); 3656 memcpy(early_acpihid_map[i].uid, uid, strlen(uid)); 3657 early_acpihid_map[i].devid = IVRS_GET_SBDF_ID(seg, bus, dev, fn); 3658 early_acpihid_map[i].cmd_line = true; 3659 3660 return 1; 3661 } 3662 3663 __setup("amd_iommu_dump", parse_amd_iommu_dump); 3664 __setup("amd_iommu=", parse_amd_iommu_options); 3665 __setup("amd_iommu_intr=", parse_amd_iommu_intr); 3666 __setup("ivrs_ioapic", parse_ivrs_ioapic); 3667 __setup("ivrs_hpet", parse_ivrs_hpet); 3668 __setup("ivrs_acpihid", parse_ivrs_acpihid); 3669 3670 bool amd_iommu_pasid_supported(void) 3671 { 3672 /* CPU page table size should match IOMMU guest page table size */ 3673 if (cpu_feature_enabled(X86_FEATURE_LA57) && 3674 amd_iommu_gpt_level != PAGE_MODE_5_LEVEL) 3675 return false; 3676 3677 /* 3678 * Since DTE[Mode]=0 is prohibited on SNP-enabled system 3679 * (i.e. EFR[SNPSup]=1), IOMMUv2 page table cannot be used without 3680 * setting up IOMMUv1 page table. 3681 */ 3682 return amd_iommu_gt_ppr_supported() && !amd_iommu_snp_en; 3683 } 3684 3685 struct amd_iommu *get_amd_iommu(unsigned int idx) 3686 { 3687 unsigned int i = 0; 3688 struct amd_iommu *iommu; 3689 3690 for_each_iommu(iommu) 3691 if (i++ == idx) 3692 return iommu; 3693 return NULL; 3694 } 3695 3696 /**************************************************************************** 3697 * 3698 * IOMMU EFR Performance Counter support functionality. This code allows 3699 * access to the IOMMU PC functionality. 3700 * 3701 ****************************************************************************/ 3702 3703 u8 amd_iommu_pc_get_max_banks(unsigned int idx) 3704 { 3705 struct amd_iommu *iommu = get_amd_iommu(idx); 3706 3707 if (iommu) 3708 return iommu->max_banks; 3709 3710 return 0; 3711 } 3712 3713 bool amd_iommu_pc_supported(void) 3714 { 3715 return amd_iommu_pc_present; 3716 } 3717 3718 u8 amd_iommu_pc_get_max_counters(unsigned int idx) 3719 { 3720 struct amd_iommu *iommu = get_amd_iommu(idx); 3721 3722 if (iommu) 3723 return iommu->max_counters; 3724 3725 return 0; 3726 } 3727 3728 static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, 3729 u8 fxn, u64 *value, bool is_write) 3730 { 3731 u32 offset; 3732 u32 max_offset_lim; 3733 3734 /* Make sure the IOMMU PC resource is available */ 3735 if (!amd_iommu_pc_present) 3736 return -ENODEV; 3737 3738 /* Check for valid iommu and pc register indexing */ 3739 if (WARN_ON(!iommu || (fxn > 0x28) || (fxn & 7))) 3740 return -ENODEV; 3741 3742 offset = (u32)(((0x40 | bank) << 12) | (cntr << 8) | fxn); 3743 3744 /* Limit the offset to the hw defined mmio region aperture */ 3745 max_offset_lim = (u32)(((0x40 | iommu->max_banks) << 12) | 3746 (iommu->max_counters << 8) | 0x28); 3747 if ((offset < MMIO_CNTR_REG_OFFSET) || 3748 (offset > max_offset_lim)) 3749 return -EINVAL; 3750 3751 if (is_write) { 3752 u64 val = *value & GENMASK_ULL(47, 0); 3753 3754 writel((u32)val, iommu->mmio_base + offset); 3755 writel((val >> 32), iommu->mmio_base + offset + 4); 3756 } else { 3757 *value = readl(iommu->mmio_base + offset + 4); 3758 *value <<= 32; 3759 *value |= readl(iommu->mmio_base + offset); 3760 *value &= GENMASK_ULL(47, 0); 3761 } 3762 3763 return 0; 3764 } 3765 3766 int amd_iommu_pc_get_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value) 3767 { 3768 if (!iommu) 3769 return -EINVAL; 3770 3771 return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, false); 3772 } 3773 3774 int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value) 3775 { 3776 if (!iommu) 3777 return -EINVAL; 3778 3779 return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, true); 3780 } 3781 3782 #ifdef CONFIG_KVM_AMD_SEV 3783 static int iommu_page_make_shared(void *page) 3784 { 3785 unsigned long paddr, pfn; 3786 3787 paddr = iommu_virt_to_phys(page); 3788 /* Cbit maybe set in the paddr */ 3789 pfn = __sme_clr(paddr) >> PAGE_SHIFT; 3790 3791 if (!(pfn % PTRS_PER_PMD)) { 3792 int ret, level; 3793 bool assigned; 3794 3795 ret = snp_lookup_rmpentry(pfn, &assigned, &level); 3796 if (ret) { 3797 pr_warn("IOMMU PFN %lx RMP lookup failed, ret %d\n", pfn, ret); 3798 return ret; 3799 } 3800 3801 if (!assigned) { 3802 pr_warn("IOMMU PFN %lx not assigned in RMP table\n", pfn); 3803 return -EINVAL; 3804 } 3805 3806 if (level > PG_LEVEL_4K) { 3807 ret = psmash(pfn); 3808 if (!ret) 3809 goto done; 3810 3811 pr_warn("PSMASH failed for IOMMU PFN %lx huge RMP entry, ret: %d, level: %d\n", 3812 pfn, ret, level); 3813 return ret; 3814 } 3815 } 3816 3817 done: 3818 return rmp_make_shared(pfn, PG_LEVEL_4K); 3819 } 3820 3821 static int iommu_make_shared(void *va, size_t size) 3822 { 3823 void *page; 3824 int ret; 3825 3826 if (!va) 3827 return 0; 3828 3829 for (page = va; page < (va + size); page += PAGE_SIZE) { 3830 ret = iommu_page_make_shared(page); 3831 if (ret) 3832 return ret; 3833 } 3834 3835 return 0; 3836 } 3837 3838 int amd_iommu_snp_disable(void) 3839 { 3840 struct amd_iommu *iommu; 3841 int ret; 3842 3843 if (!amd_iommu_snp_en) 3844 return 0; 3845 3846 for_each_iommu(iommu) { 3847 ret = iommu_make_shared(iommu->evt_buf, EVT_BUFFER_SIZE); 3848 if (ret) 3849 return ret; 3850 3851 ret = iommu_make_shared(iommu->ppr_log, PPR_LOG_SIZE); 3852 if (ret) 3853 return ret; 3854 3855 ret = iommu_make_shared((void *)iommu->cmd_sem, PAGE_SIZE); 3856 if (ret) 3857 return ret; 3858 } 3859 3860 return 0; 3861 } 3862 EXPORT_SYMBOL_GPL(amd_iommu_snp_disable); 3863 #endif 3864