1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc. 4 * Author: Joerg Roedel <jroedel@suse.de> 5 * Leo Duran <leo.duran@amd.com> 6 */ 7 8 #define pr_fmt(fmt) "AMD-Vi: " fmt 9 #define dev_fmt(fmt) pr_fmt(fmt) 10 11 #include <linux/pci.h> 12 #include <linux/acpi.h> 13 #include <linux/list.h> 14 #include <linux/bitmap.h> 15 #include <linux/syscore_ops.h> 16 #include <linux/interrupt.h> 17 #include <linux/msi.h> 18 #include <linux/irq.h> 19 #include <linux/amd-iommu.h> 20 #include <linux/export.h> 21 #include <linux/kmemleak.h> 22 #include <linux/cc_platform.h> 23 #include <linux/iopoll.h> 24 #include <asm/pci-direct.h> 25 #include <asm/iommu.h> 26 #include <asm/apic.h> 27 #include <asm/gart.h> 28 #include <asm/x86_init.h> 29 #include <asm/io_apic.h> 30 #include <asm/irq_remapping.h> 31 #include <asm/set_memory.h> 32 #include <asm/sev.h> 33 34 #include <linux/crash_dump.h> 35 36 #include "amd_iommu.h" 37 #include "../irq_remapping.h" 38 #include "../iommu-pages.h" 39 40 /* 41 * definitions for the ACPI scanning code 42 */ 43 #define IVRS_HEADER_LENGTH 48 44 45 #define ACPI_IVHD_TYPE_MAX_SUPPORTED 0x40 46 #define ACPI_IVMD_TYPE_ALL 0x20 47 #define ACPI_IVMD_TYPE 0x21 48 #define ACPI_IVMD_TYPE_RANGE 0x22 49 50 #define IVHD_DEV_ALL 0x01 51 #define IVHD_DEV_SELECT 0x02 52 #define IVHD_DEV_SELECT_RANGE_START 0x03 53 #define IVHD_DEV_RANGE_END 0x04 54 #define IVHD_DEV_ALIAS 0x42 55 #define IVHD_DEV_ALIAS_RANGE 0x43 56 #define IVHD_DEV_EXT_SELECT 0x46 57 #define IVHD_DEV_EXT_SELECT_RANGE 0x47 58 #define IVHD_DEV_SPECIAL 0x48 59 #define IVHD_DEV_ACPI_HID 0xf0 60 61 #define UID_NOT_PRESENT 0 62 #define UID_IS_INTEGER 1 63 #define UID_IS_CHARACTER 2 64 65 #define IVHD_SPECIAL_IOAPIC 1 66 #define IVHD_SPECIAL_HPET 2 67 68 #define IVHD_FLAG_HT_TUN_EN_MASK 0x01 69 #define IVHD_FLAG_PASSPW_EN_MASK 0x02 70 #define IVHD_FLAG_RESPASSPW_EN_MASK 0x04 71 #define IVHD_FLAG_ISOC_EN_MASK 0x08 72 73 #define IVMD_FLAG_EXCL_RANGE 0x08 74 #define IVMD_FLAG_IW 0x04 75 #define IVMD_FLAG_IR 0x02 76 #define IVMD_FLAG_UNITY_MAP 0x01 77 78 #define ACPI_DEVFLAG_INITPASS 0x01 79 #define ACPI_DEVFLAG_EXTINT 0x02 80 #define ACPI_DEVFLAG_NMI 0x04 81 #define ACPI_DEVFLAG_SYSMGT1 0x10 82 #define ACPI_DEVFLAG_SYSMGT2 0x20 83 #define ACPI_DEVFLAG_LINT0 0x40 84 #define ACPI_DEVFLAG_LINT1 0x80 85 #define ACPI_DEVFLAG_ATSDIS 0x10000000 86 87 #define IVRS_GET_SBDF_ID(seg, bus, dev, fn) (((seg & 0xffff) << 16) | ((bus & 0xff) << 8) \ 88 | ((dev & 0x1f) << 3) | (fn & 0x7)) 89 90 /* 91 * ACPI table definitions 92 * 93 * These data structures are laid over the table to parse the important values 94 * out of it. 95 */ 96 97 /* 98 * structure describing one IOMMU in the ACPI table. Typically followed by one 99 * or more ivhd_entrys. 100 */ 101 struct ivhd_header { 102 u8 type; 103 u8 flags; 104 u16 length; 105 u16 devid; 106 u16 cap_ptr; 107 u64 mmio_phys; 108 u16 pci_seg; 109 u16 info; 110 u32 efr_attr; 111 112 /* Following only valid on IVHD type 11h and 40h */ 113 u64 efr_reg; /* Exact copy of MMIO_EXT_FEATURES */ 114 u64 efr_reg2; 115 } __attribute__((packed)); 116 117 /* 118 * A device entry describing which devices a specific IOMMU translates and 119 * which requestor ids they use. 120 */ 121 struct ivhd_entry { 122 u8 type; 123 u16 devid; 124 u8 flags; 125 struct_group(ext_hid, 126 u32 ext; 127 u32 hidh; 128 ); 129 u64 cid; 130 u8 uidf; 131 u8 uidl; 132 u8 uid; 133 } __attribute__((packed)); 134 135 /* 136 * An AMD IOMMU memory definition structure. It defines things like exclusion 137 * ranges for devices and regions that should be unity mapped. 138 */ 139 struct ivmd_header { 140 u8 type; 141 u8 flags; 142 u16 length; 143 u16 devid; 144 u16 aux; 145 u16 pci_seg; 146 u8 resv[6]; 147 u64 range_start; 148 u64 range_length; 149 } __attribute__((packed)); 150 151 bool amd_iommu_dump; 152 bool amd_iommu_irq_remap __read_mostly; 153 154 enum protection_domain_mode amd_iommu_pgtable = PD_MODE_V1; 155 /* Host page table level */ 156 u8 amd_iommu_hpt_level; 157 /* Guest page table level */ 158 int amd_iommu_gpt_level = PAGE_MODE_4_LEVEL; 159 160 int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC; 161 static int amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE; 162 163 static bool amd_iommu_detected; 164 static bool amd_iommu_disabled __initdata; 165 static bool amd_iommu_force_enable __initdata; 166 static bool amd_iommu_irtcachedis; 167 static int amd_iommu_target_ivhd_type; 168 169 /* Global EFR and EFR2 registers */ 170 u64 amd_iommu_efr; 171 u64 amd_iommu_efr2; 172 173 /* Host (v1) page table is not supported*/ 174 bool amd_iommu_hatdis; 175 176 /* SNP is enabled on the system? */ 177 bool amd_iommu_snp_en; 178 EXPORT_SYMBOL(amd_iommu_snp_en); 179 180 LIST_HEAD(amd_iommu_pci_seg_list); /* list of all PCI segments */ 181 LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the system */ 182 LIST_HEAD(amd_ivhd_dev_flags_list); /* list of all IVHD device entry settings */ 183 184 /* Number of IOMMUs present in the system */ 185 static int amd_iommus_present; 186 187 /* IOMMUs have a non-present cache? */ 188 bool amd_iommu_np_cache __read_mostly; 189 bool amd_iommu_iotlb_sup __read_mostly = true; 190 191 static bool amd_iommu_pc_present __read_mostly; 192 bool amdr_ivrs_remap_support __read_mostly; 193 194 bool amd_iommu_force_isolation __read_mostly; 195 196 unsigned long amd_iommu_pgsize_bitmap __ro_after_init = AMD_IOMMU_PGSIZES; 197 198 enum iommu_init_state { 199 IOMMU_START_STATE, 200 IOMMU_IVRS_DETECTED, 201 IOMMU_ACPI_FINISHED, 202 IOMMU_ENABLED, 203 IOMMU_PCI_INIT, 204 IOMMU_INTERRUPTS_EN, 205 IOMMU_INITIALIZED, 206 IOMMU_NOT_FOUND, 207 IOMMU_INIT_ERROR, 208 IOMMU_CMDLINE_DISABLED, 209 }; 210 211 /* Early ioapic and hpet maps from kernel command line */ 212 #define EARLY_MAP_SIZE 4 213 static struct devid_map __initdata early_ioapic_map[EARLY_MAP_SIZE]; 214 static struct devid_map __initdata early_hpet_map[EARLY_MAP_SIZE]; 215 static struct acpihid_map_entry __initdata early_acpihid_map[EARLY_MAP_SIZE]; 216 217 static int __initdata early_ioapic_map_size; 218 static int __initdata early_hpet_map_size; 219 static int __initdata early_acpihid_map_size; 220 221 static bool __initdata cmdline_maps; 222 223 static enum iommu_init_state init_state = IOMMU_START_STATE; 224 225 static int amd_iommu_enable_interrupts(void); 226 static void init_device_table_dma(struct amd_iommu_pci_seg *pci_seg); 227 228 static bool amd_iommu_pre_enabled = true; 229 230 static u32 amd_iommu_ivinfo __initdata; 231 232 bool translation_pre_enabled(struct amd_iommu *iommu) 233 { 234 return (iommu->flags & AMD_IOMMU_FLAG_TRANS_PRE_ENABLED); 235 } 236 237 static void clear_translation_pre_enabled(struct amd_iommu *iommu) 238 { 239 iommu->flags &= ~AMD_IOMMU_FLAG_TRANS_PRE_ENABLED; 240 } 241 242 static void init_translation_status(struct amd_iommu *iommu) 243 { 244 u64 ctrl; 245 246 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); 247 if (ctrl & (1<<CONTROL_IOMMU_EN)) 248 iommu->flags |= AMD_IOMMU_FLAG_TRANS_PRE_ENABLED; 249 } 250 251 int amd_iommu_get_num_iommus(void) 252 { 253 return amd_iommus_present; 254 } 255 256 bool amd_iommu_ht_range_ignore(void) 257 { 258 return check_feature2(FEATURE_HT_RANGE_IGNORE); 259 } 260 261 /* 262 * Iterate through all the IOMMUs to get common EFR 263 * masks among all IOMMUs and warn if found inconsistency. 264 */ 265 static __init void get_global_efr(void) 266 { 267 struct amd_iommu *iommu; 268 269 for_each_iommu(iommu) { 270 u64 tmp = iommu->features; 271 u64 tmp2 = iommu->features2; 272 273 if (list_is_first(&iommu->list, &amd_iommu_list)) { 274 amd_iommu_efr = tmp; 275 amd_iommu_efr2 = tmp2; 276 continue; 277 } 278 279 if (amd_iommu_efr == tmp && 280 amd_iommu_efr2 == tmp2) 281 continue; 282 283 pr_err(FW_BUG 284 "Found inconsistent EFR/EFR2 %#llx,%#llx (global %#llx,%#llx) on iommu%d (%04x:%02x:%02x.%01x).\n", 285 tmp, tmp2, amd_iommu_efr, amd_iommu_efr2, 286 iommu->index, iommu->pci_seg->id, 287 PCI_BUS_NUM(iommu->devid), PCI_SLOT(iommu->devid), 288 PCI_FUNC(iommu->devid)); 289 290 amd_iommu_efr &= tmp; 291 amd_iommu_efr2 &= tmp2; 292 } 293 294 pr_info("Using global IVHD EFR:%#llx, EFR2:%#llx\n", amd_iommu_efr, amd_iommu_efr2); 295 } 296 297 /* 298 * For IVHD type 0x11/0x40, EFR is also available via IVHD. 299 * Default to IVHD EFR since it is available sooner 300 * (i.e. before PCI init). 301 */ 302 static void __init early_iommu_features_init(struct amd_iommu *iommu, 303 struct ivhd_header *h) 304 { 305 if (amd_iommu_ivinfo & IOMMU_IVINFO_EFRSUP) { 306 iommu->features = h->efr_reg; 307 iommu->features2 = h->efr_reg2; 308 } 309 if (amd_iommu_ivinfo & IOMMU_IVINFO_DMA_REMAP) 310 amdr_ivrs_remap_support = true; 311 } 312 313 /* Access to l1 and l2 indexed register spaces */ 314 315 static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address) 316 { 317 u32 val; 318 319 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16)); 320 pci_read_config_dword(iommu->dev, 0xfc, &val); 321 return val; 322 } 323 324 static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val) 325 { 326 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31)); 327 pci_write_config_dword(iommu->dev, 0xfc, val); 328 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16)); 329 } 330 331 static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address) 332 { 333 u32 val; 334 335 pci_write_config_dword(iommu->dev, 0xf0, address); 336 pci_read_config_dword(iommu->dev, 0xf4, &val); 337 return val; 338 } 339 340 static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val) 341 { 342 pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8)); 343 pci_write_config_dword(iommu->dev, 0xf4, val); 344 } 345 346 /**************************************************************************** 347 * 348 * AMD IOMMU MMIO register space handling functions 349 * 350 * These functions are used to program the IOMMU device registers in 351 * MMIO space required for that driver. 352 * 353 ****************************************************************************/ 354 355 /* 356 * This function set the exclusion range in the IOMMU. DMA accesses to the 357 * exclusion range are passed through untranslated 358 */ 359 static void iommu_set_exclusion_range(struct amd_iommu *iommu) 360 { 361 u64 start = iommu->exclusion_start & PAGE_MASK; 362 u64 limit = (start + iommu->exclusion_length - 1) & PAGE_MASK; 363 u64 entry; 364 365 if (!iommu->exclusion_start) 366 return; 367 368 entry = start | MMIO_EXCL_ENABLE_MASK; 369 memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET, 370 &entry, sizeof(entry)); 371 372 entry = limit; 373 memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET, 374 &entry, sizeof(entry)); 375 } 376 377 static void iommu_set_cwwb_range(struct amd_iommu *iommu) 378 { 379 u64 start = iommu_virt_to_phys((void *)iommu->cmd_sem); 380 u64 entry = start & PM_ADDR_MASK; 381 382 if (!check_feature(FEATURE_SNP)) 383 return; 384 385 /* Note: 386 * Re-purpose Exclusion base/limit registers for Completion wait 387 * write-back base/limit. 388 */ 389 memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET, 390 &entry, sizeof(entry)); 391 392 /* Note: 393 * Default to 4 Kbytes, which can be specified by setting base 394 * address equal to the limit address. 395 */ 396 memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET, 397 &entry, sizeof(entry)); 398 } 399 400 /* Programs the physical address of the device table into the IOMMU hardware */ 401 static void iommu_set_device_table(struct amd_iommu *iommu) 402 { 403 u64 entry; 404 u32 dev_table_size = iommu->pci_seg->dev_table_size; 405 void *dev_table = (void *)get_dev_table(iommu); 406 407 BUG_ON(iommu->mmio_base == NULL); 408 409 if (is_kdump_kernel()) 410 return; 411 412 entry = iommu_virt_to_phys(dev_table); 413 entry |= (dev_table_size >> 12) - 1; 414 memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET, 415 &entry, sizeof(entry)); 416 } 417 418 static void iommu_feature_set(struct amd_iommu *iommu, u64 val, u64 mask, u8 shift) 419 { 420 u64 ctrl; 421 422 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); 423 mask <<= shift; 424 ctrl &= ~mask; 425 ctrl |= (val << shift) & mask; 426 writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET); 427 } 428 429 /* Generic functions to enable/disable certain features of the IOMMU. */ 430 void iommu_feature_enable(struct amd_iommu *iommu, u8 bit) 431 { 432 iommu_feature_set(iommu, 1ULL, 1ULL, bit); 433 } 434 435 static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit) 436 { 437 iommu_feature_set(iommu, 0ULL, 1ULL, bit); 438 } 439 440 /* Function to enable the hardware */ 441 static void iommu_enable(struct amd_iommu *iommu) 442 { 443 iommu_feature_enable(iommu, CONTROL_IOMMU_EN); 444 } 445 446 static void iommu_disable(struct amd_iommu *iommu) 447 { 448 if (!iommu->mmio_base) 449 return; 450 451 /* Disable command buffer */ 452 iommu_feature_disable(iommu, CONTROL_CMDBUF_EN); 453 454 /* Disable event logging and event interrupts */ 455 iommu_feature_disable(iommu, CONTROL_EVT_INT_EN); 456 iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN); 457 458 /* Disable IOMMU GA_LOG */ 459 iommu_feature_disable(iommu, CONTROL_GALOG_EN); 460 iommu_feature_disable(iommu, CONTROL_GAINT_EN); 461 462 /* Disable IOMMU PPR logging */ 463 iommu_feature_disable(iommu, CONTROL_PPRLOG_EN); 464 iommu_feature_disable(iommu, CONTROL_PPRINT_EN); 465 466 /* Disable IOMMU hardware itself */ 467 iommu_feature_disable(iommu, CONTROL_IOMMU_EN); 468 469 /* Clear IRTE cache disabling bit */ 470 iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS); 471 } 472 473 /* 474 * mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in 475 * the system has one. 476 */ 477 static u8 __iomem * __init iommu_map_mmio_space(u64 address, u64 end) 478 { 479 if (!request_mem_region(address, end, "amd_iommu")) { 480 pr_err("Can not reserve memory region %llx-%llx for mmio\n", 481 address, end); 482 pr_err("This is a BIOS bug. Please contact your hardware vendor\n"); 483 return NULL; 484 } 485 486 return (u8 __iomem *)ioremap(address, end); 487 } 488 489 static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu) 490 { 491 if (iommu->mmio_base) 492 iounmap(iommu->mmio_base); 493 release_mem_region(iommu->mmio_phys, iommu->mmio_phys_end); 494 } 495 496 static inline u32 get_ivhd_header_size(struct ivhd_header *h) 497 { 498 u32 size = 0; 499 500 switch (h->type) { 501 case 0x10: 502 size = 24; 503 break; 504 case 0x11: 505 case 0x40: 506 size = 40; 507 break; 508 } 509 return size; 510 } 511 512 /**************************************************************************** 513 * 514 * The functions below belong to the first pass of AMD IOMMU ACPI table 515 * parsing. In this pass we try to find out the highest device id this 516 * code has to handle. Upon this information the size of the shared data 517 * structures is determined later. 518 * 519 ****************************************************************************/ 520 521 /* 522 * This function calculates the length of a given IVHD entry 523 */ 524 static inline int ivhd_entry_length(u8 *ivhd) 525 { 526 u32 type = ((struct ivhd_entry *)ivhd)->type; 527 528 if (type < 0x80) { 529 return 0x04 << (*ivhd >> 6); 530 } else if (type == IVHD_DEV_ACPI_HID) { 531 /* For ACPI_HID, offset 21 is uid len */ 532 return *((u8 *)ivhd + 21) + 22; 533 } 534 return 0; 535 } 536 537 /* 538 * After reading the highest device id from the IOMMU PCI capability header 539 * this function looks if there is a higher device id defined in the ACPI table 540 */ 541 static int __init find_last_devid_from_ivhd(struct ivhd_header *h) 542 { 543 u8 *p = (void *)h, *end = (void *)h; 544 struct ivhd_entry *dev; 545 int last_devid = -EINVAL; 546 547 u32 ivhd_size = get_ivhd_header_size(h); 548 549 if (!ivhd_size) { 550 pr_err("Unsupported IVHD type %#x\n", h->type); 551 return -EINVAL; 552 } 553 554 p += ivhd_size; 555 end += h->length; 556 557 while (p < end) { 558 dev = (struct ivhd_entry *)p; 559 switch (dev->type) { 560 case IVHD_DEV_ALL: 561 /* Use maximum BDF value for DEV_ALL */ 562 return 0xffff; 563 case IVHD_DEV_SELECT: 564 case IVHD_DEV_RANGE_END: 565 case IVHD_DEV_ALIAS: 566 case IVHD_DEV_EXT_SELECT: 567 /* all the above subfield types refer to device ids */ 568 if (dev->devid > last_devid) 569 last_devid = dev->devid; 570 break; 571 default: 572 break; 573 } 574 p += ivhd_entry_length(p); 575 } 576 577 WARN_ON(p != end); 578 579 return last_devid; 580 } 581 582 static int __init check_ivrs_checksum(struct acpi_table_header *table) 583 { 584 int i; 585 u8 checksum = 0, *p = (u8 *)table; 586 587 for (i = 0; i < table->length; ++i) 588 checksum += p[i]; 589 if (checksum != 0) { 590 /* ACPI table corrupt */ 591 pr_err(FW_BUG "IVRS invalid checksum\n"); 592 return -ENODEV; 593 } 594 595 return 0; 596 } 597 598 /* 599 * Iterate over all IVHD entries in the ACPI table and find the highest device 600 * id which we need to handle. This is the first of three functions which parse 601 * the ACPI table. So we check the checksum here. 602 */ 603 static int __init find_last_devid_acpi(struct acpi_table_header *table, u16 pci_seg) 604 { 605 u8 *p = (u8 *)table, *end = (u8 *)table; 606 struct ivhd_header *h; 607 int last_devid, last_bdf = 0; 608 609 p += IVRS_HEADER_LENGTH; 610 611 end += table->length; 612 while (p < end) { 613 h = (struct ivhd_header *)p; 614 if (h->pci_seg == pci_seg && 615 h->type == amd_iommu_target_ivhd_type) { 616 last_devid = find_last_devid_from_ivhd(h); 617 618 if (last_devid < 0) 619 return -EINVAL; 620 if (last_devid > last_bdf) 621 last_bdf = last_devid; 622 } 623 p += h->length; 624 } 625 WARN_ON(p != end); 626 627 return last_bdf; 628 } 629 630 /**************************************************************************** 631 * 632 * The following functions belong to the code path which parses the ACPI table 633 * the second time. In this ACPI parsing iteration we allocate IOMMU specific 634 * data structures, initialize the per PCI segment device/alias/rlookup table 635 * and also basically initialize the hardware. 636 * 637 ****************************************************************************/ 638 639 /* Allocate per PCI segment device table */ 640 static inline int __init alloc_dev_table(struct amd_iommu_pci_seg *pci_seg) 641 { 642 pci_seg->dev_table = iommu_alloc_pages_sz(GFP_KERNEL | GFP_DMA32, 643 pci_seg->dev_table_size); 644 if (!pci_seg->dev_table) 645 return -ENOMEM; 646 647 return 0; 648 } 649 650 static inline void free_dev_table(struct amd_iommu_pci_seg *pci_seg) 651 { 652 if (is_kdump_kernel()) 653 memunmap((void *)pci_seg->dev_table); 654 else 655 iommu_free_pages(pci_seg->dev_table); 656 pci_seg->dev_table = NULL; 657 } 658 659 /* Allocate per PCI segment IOMMU rlookup table. */ 660 static inline int __init alloc_rlookup_table(struct amd_iommu_pci_seg *pci_seg) 661 { 662 pci_seg->rlookup_table = kvzalloc_objs(*pci_seg->rlookup_table, 663 pci_seg->last_bdf + 1, 664 GFP_KERNEL); 665 if (pci_seg->rlookup_table == NULL) 666 return -ENOMEM; 667 668 return 0; 669 } 670 671 static inline void free_rlookup_table(struct amd_iommu_pci_seg *pci_seg) 672 { 673 kvfree(pci_seg->rlookup_table); 674 pci_seg->rlookup_table = NULL; 675 } 676 677 static inline int __init alloc_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg) 678 { 679 pci_seg->irq_lookup_table = kvzalloc_objs(*pci_seg->irq_lookup_table, 680 pci_seg->last_bdf + 1, 681 GFP_KERNEL); 682 if (pci_seg->irq_lookup_table == NULL) 683 return -ENOMEM; 684 685 return 0; 686 } 687 688 static inline void free_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg) 689 { 690 kvfree(pci_seg->irq_lookup_table); 691 pci_seg->irq_lookup_table = NULL; 692 } 693 694 static int __init alloc_alias_table(struct amd_iommu_pci_seg *pci_seg) 695 { 696 int i; 697 698 pci_seg->alias_table = kvmalloc_objs(*pci_seg->alias_table, 699 pci_seg->last_bdf + 1, GFP_KERNEL); 700 if (!pci_seg->alias_table) 701 return -ENOMEM; 702 703 /* 704 * let all alias entries point to itself 705 */ 706 for (i = 0; i <= pci_seg->last_bdf; ++i) 707 pci_seg->alias_table[i] = i; 708 709 return 0; 710 } 711 712 static void __init free_alias_table(struct amd_iommu_pci_seg *pci_seg) 713 { 714 kvfree(pci_seg->alias_table); 715 pci_seg->alias_table = NULL; 716 } 717 718 static inline void *iommu_memremap(unsigned long paddr, size_t size) 719 { 720 phys_addr_t phys; 721 722 if (!paddr) 723 return NULL; 724 725 /* 726 * Obtain true physical address in kdump kernel when SME is enabled. 727 * Currently, previous kernel with SME enabled and kdump kernel 728 * with SME support disabled is not supported. 729 */ 730 phys = __sme_clr(paddr); 731 732 if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) 733 return (__force void *)ioremap_encrypted(phys, size); 734 else 735 return memremap(phys, size, MEMREMAP_WB); 736 } 737 738 /* 739 * Allocates the command buffer. This buffer is per AMD IOMMU. We can 740 * write commands to that buffer later and the IOMMU will execute them 741 * asynchronously 742 */ 743 static int __init alloc_command_buffer(struct amd_iommu *iommu) 744 { 745 iommu->cmd_buf = iommu_alloc_pages_sz(GFP_KERNEL, CMD_BUFFER_SIZE); 746 747 return iommu->cmd_buf ? 0 : -ENOMEM; 748 } 749 750 /* 751 * Interrupt handler has processed all pending events and adjusted head 752 * and tail pointer. Reset overflow mask and restart logging again. 753 */ 754 void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type, 755 u8 cntrl_intr, u8 cntrl_log, 756 u32 status_run_mask, u32 status_overflow_mask) 757 { 758 u32 status; 759 760 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); 761 if (status & status_run_mask) 762 return; 763 764 pr_info_ratelimited("IOMMU %s log restarting\n", evt_type); 765 766 iommu_feature_disable(iommu, cntrl_log); 767 iommu_feature_disable(iommu, cntrl_intr); 768 769 writel(status_overflow_mask, iommu->mmio_base + MMIO_STATUS_OFFSET); 770 771 iommu_feature_enable(iommu, cntrl_intr); 772 iommu_feature_enable(iommu, cntrl_log); 773 } 774 775 /* 776 * This function restarts event logging in case the IOMMU experienced 777 * an event log buffer overflow. 778 */ 779 void amd_iommu_restart_event_logging(struct amd_iommu *iommu) 780 { 781 amd_iommu_restart_log(iommu, "Event", CONTROL_EVT_INT_EN, 782 CONTROL_EVT_LOG_EN, MMIO_STATUS_EVT_RUN_MASK, 783 MMIO_STATUS_EVT_OVERFLOW_MASK); 784 } 785 786 /* 787 * This function restarts event logging in case the IOMMU experienced 788 * GA log overflow. 789 */ 790 void amd_iommu_restart_ga_log(struct amd_iommu *iommu) 791 { 792 amd_iommu_restart_log(iommu, "GA", CONTROL_GAINT_EN, 793 CONTROL_GALOG_EN, MMIO_STATUS_GALOG_RUN_MASK, 794 MMIO_STATUS_GALOG_OVERFLOW_MASK); 795 } 796 797 /* 798 * This function resets the command buffer if the IOMMU stopped fetching 799 * commands from it. 800 */ 801 static void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu) 802 { 803 iommu_feature_disable(iommu, CONTROL_CMDBUF_EN); 804 805 writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET); 806 writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET); 807 iommu->cmd_buf_head = 0; 808 iommu->cmd_buf_tail = 0; 809 810 iommu_feature_enable(iommu, CONTROL_CMDBUF_EN); 811 } 812 813 /* 814 * This function writes the command buffer address to the hardware and 815 * enables it. 816 */ 817 static void iommu_enable_command_buffer(struct amd_iommu *iommu) 818 { 819 u64 entry; 820 821 BUG_ON(iommu->cmd_buf == NULL); 822 823 if (!is_kdump_kernel()) { 824 /* 825 * Command buffer is re-used for kdump kernel and setting 826 * of MMIO register is not required. 827 */ 828 entry = iommu_virt_to_phys(iommu->cmd_buf); 829 entry |= MMIO_CMD_SIZE_512; 830 memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET, 831 &entry, sizeof(entry)); 832 } 833 834 amd_iommu_reset_cmd_buffer(iommu); 835 } 836 837 /* 838 * This function disables the command buffer 839 */ 840 static void iommu_disable_command_buffer(struct amd_iommu *iommu) 841 { 842 iommu_feature_disable(iommu, CONTROL_CMDBUF_EN); 843 } 844 845 static void __init free_command_buffer(struct amd_iommu *iommu) 846 { 847 iommu_free_pages(iommu->cmd_buf); 848 } 849 850 void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, gfp_t gfp, 851 size_t size) 852 { 853 void *buf; 854 855 size = PAGE_ALIGN(size); 856 buf = iommu_alloc_pages_sz(gfp, size); 857 if (!buf) 858 return NULL; 859 if (check_feature(FEATURE_SNP) && 860 set_memory_4k((unsigned long)buf, size / PAGE_SIZE)) { 861 iommu_free_pages(buf); 862 return NULL; 863 } 864 865 return buf; 866 } 867 868 /* allocates the memory where the IOMMU will log its events to */ 869 static int __init alloc_event_buffer(struct amd_iommu *iommu) 870 { 871 iommu->evt_buf = iommu_alloc_4k_pages(iommu, GFP_KERNEL, 872 EVT_BUFFER_SIZE); 873 874 return iommu->evt_buf ? 0 : -ENOMEM; 875 } 876 877 static void iommu_enable_event_buffer(struct amd_iommu *iommu) 878 { 879 u64 entry; 880 881 BUG_ON(iommu->evt_buf == NULL); 882 883 if (!is_kdump_kernel()) { 884 /* 885 * Event buffer is re-used for kdump kernel and setting 886 * of MMIO register is not required. 887 */ 888 entry = iommu_virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK; 889 memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET, 890 &entry, sizeof(entry)); 891 } 892 893 /* set head and tail to zero manually */ 894 writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); 895 writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET); 896 897 iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN); 898 } 899 900 /* 901 * This function disables the event log buffer 902 */ 903 static void iommu_disable_event_buffer(struct amd_iommu *iommu) 904 { 905 iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN); 906 } 907 908 static void __init free_event_buffer(struct amd_iommu *iommu) 909 { 910 iommu_free_pages(iommu->evt_buf); 911 } 912 913 static void free_ga_log(struct amd_iommu *iommu) 914 { 915 #ifdef CONFIG_IRQ_REMAP 916 iommu_free_pages(iommu->ga_log); 917 iommu_free_pages(iommu->ga_log_tail); 918 #endif 919 } 920 921 #ifdef CONFIG_IRQ_REMAP 922 static int iommu_ga_log_enable(struct amd_iommu *iommu) 923 { 924 u32 status, i; 925 u64 entry; 926 927 if (!iommu->ga_log) 928 return -EINVAL; 929 930 entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512; 931 memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET, 932 &entry, sizeof(entry)); 933 entry = (iommu_virt_to_phys(iommu->ga_log_tail) & 934 (BIT_ULL(52)-1)) & ~7ULL; 935 memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET, 936 &entry, sizeof(entry)); 937 writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET); 938 writel(0x00, iommu->mmio_base + MMIO_GA_TAIL_OFFSET); 939 940 941 iommu_feature_enable(iommu, CONTROL_GAINT_EN); 942 iommu_feature_enable(iommu, CONTROL_GALOG_EN); 943 944 for (i = 0; i < MMIO_STATUS_TIMEOUT; ++i) { 945 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); 946 if (status & (MMIO_STATUS_GALOG_RUN_MASK)) 947 break; 948 udelay(10); 949 } 950 951 if (WARN_ON(i >= MMIO_STATUS_TIMEOUT)) 952 return -EINVAL; 953 954 return 0; 955 } 956 957 static int iommu_init_ga_log(struct amd_iommu *iommu) 958 { 959 if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) 960 return 0; 961 962 iommu->ga_log = iommu_alloc_pages_sz(GFP_KERNEL, GA_LOG_SIZE); 963 if (!iommu->ga_log) 964 goto err_out; 965 966 iommu->ga_log_tail = iommu_alloc_pages_sz(GFP_KERNEL, 8); 967 if (!iommu->ga_log_tail) 968 goto err_out; 969 970 return 0; 971 err_out: 972 free_ga_log(iommu); 973 return -EINVAL; 974 } 975 #endif /* CONFIG_IRQ_REMAP */ 976 977 static int __init alloc_cwwb_sem(struct amd_iommu *iommu) 978 { 979 iommu->cmd_sem = iommu_alloc_4k_pages(iommu, GFP_KERNEL, 1); 980 if (!iommu->cmd_sem) 981 return -ENOMEM; 982 iommu->cmd_sem_paddr = iommu_virt_to_phys((void *)iommu->cmd_sem); 983 return 0; 984 } 985 986 static int __init remap_event_buffer(struct amd_iommu *iommu) 987 { 988 u64 paddr; 989 990 pr_info_once("Re-using event buffer from the previous kernel\n"); 991 paddr = readq(iommu->mmio_base + MMIO_EVT_BUF_OFFSET) & PM_ADDR_MASK; 992 iommu->evt_buf = iommu_memremap(paddr, EVT_BUFFER_SIZE); 993 994 return iommu->evt_buf ? 0 : -ENOMEM; 995 } 996 997 static int __init remap_command_buffer(struct amd_iommu *iommu) 998 { 999 u64 paddr; 1000 1001 pr_info_once("Re-using command buffer from the previous kernel\n"); 1002 paddr = readq(iommu->mmio_base + MMIO_CMD_BUF_OFFSET) & PM_ADDR_MASK; 1003 iommu->cmd_buf = iommu_memremap(paddr, CMD_BUFFER_SIZE); 1004 1005 return iommu->cmd_buf ? 0 : -ENOMEM; 1006 } 1007 1008 static int __init remap_or_alloc_cwwb_sem(struct amd_iommu *iommu) 1009 { 1010 u64 paddr; 1011 1012 if (check_feature(FEATURE_SNP)) { 1013 /* 1014 * When SNP is enabled, the exclusion base register is used for the 1015 * completion wait buffer (CWB) address. Read and re-use it. 1016 */ 1017 pr_info_once("Re-using CWB buffers from the previous kernel\n"); 1018 paddr = readq(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET) & PM_ADDR_MASK; 1019 iommu->cmd_sem = iommu_memremap(paddr, PAGE_SIZE); 1020 if (!iommu->cmd_sem) 1021 return -ENOMEM; 1022 iommu->cmd_sem_paddr = paddr; 1023 } else { 1024 return alloc_cwwb_sem(iommu); 1025 } 1026 1027 return 0; 1028 } 1029 1030 static int __init alloc_iommu_buffers(struct amd_iommu *iommu) 1031 { 1032 int ret; 1033 1034 /* 1035 * Reuse/Remap the previous kernel's allocated completion wait 1036 * command and event buffers for kdump boot. 1037 */ 1038 if (is_kdump_kernel()) { 1039 ret = remap_or_alloc_cwwb_sem(iommu); 1040 if (ret) 1041 return ret; 1042 1043 ret = remap_command_buffer(iommu); 1044 if (ret) 1045 return ret; 1046 1047 ret = remap_event_buffer(iommu); 1048 if (ret) 1049 return ret; 1050 } else { 1051 ret = alloc_cwwb_sem(iommu); 1052 if (ret) 1053 return ret; 1054 1055 ret = alloc_command_buffer(iommu); 1056 if (ret) 1057 return ret; 1058 1059 ret = alloc_event_buffer(iommu); 1060 if (ret) 1061 return ret; 1062 } 1063 1064 return 0; 1065 } 1066 1067 static void __init free_cwwb_sem(struct amd_iommu *iommu) 1068 { 1069 if (iommu->cmd_sem) 1070 iommu_free_pages((void *)iommu->cmd_sem); 1071 } 1072 static void __init unmap_cwwb_sem(struct amd_iommu *iommu) 1073 { 1074 if (iommu->cmd_sem) { 1075 if (check_feature(FEATURE_SNP)) 1076 memunmap((void *)iommu->cmd_sem); 1077 else 1078 iommu_free_pages((void *)iommu->cmd_sem); 1079 } 1080 } 1081 1082 static void __init unmap_command_buffer(struct amd_iommu *iommu) 1083 { 1084 memunmap((void *)iommu->cmd_buf); 1085 } 1086 1087 static void __init unmap_event_buffer(struct amd_iommu *iommu) 1088 { 1089 memunmap(iommu->evt_buf); 1090 } 1091 1092 static void __init free_iommu_buffers(struct amd_iommu *iommu) 1093 { 1094 if (is_kdump_kernel()) { 1095 unmap_cwwb_sem(iommu); 1096 unmap_command_buffer(iommu); 1097 unmap_event_buffer(iommu); 1098 } else { 1099 free_cwwb_sem(iommu); 1100 free_command_buffer(iommu); 1101 free_event_buffer(iommu); 1102 } 1103 } 1104 1105 static void iommu_enable_xt(struct amd_iommu *iommu) 1106 { 1107 #ifdef CONFIG_IRQ_REMAP 1108 /* 1109 * XT mode (32-bit APIC destination ID) requires 1110 * GA mode (128-bit IRTE support) as a prerequisite. 1111 */ 1112 if (AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir) && 1113 amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE) 1114 iommu_feature_enable(iommu, CONTROL_XT_EN); 1115 #endif /* CONFIG_IRQ_REMAP */ 1116 } 1117 1118 static void iommu_enable_gt(struct amd_iommu *iommu) 1119 { 1120 if (!check_feature(FEATURE_GT)) 1121 return; 1122 1123 iommu_feature_enable(iommu, CONTROL_GT_EN); 1124 1125 /* 1126 * This feature needs to be enabled prior to a call 1127 * to iommu_snp_enable(). Since this function is called 1128 * in early_enable_iommu(), it is safe to enable here. 1129 */ 1130 if (check_feature2(FEATURE_GCR3TRPMODE)) 1131 iommu_feature_enable(iommu, CONTROL_GCR3TRPMODE); 1132 } 1133 1134 /* sets a specific bit in the device table entry. */ 1135 static void set_dte_bit(struct dev_table_entry *dte, u8 bit) 1136 { 1137 int i = (bit >> 6) & 0x03; 1138 int _bit = bit & 0x3f; 1139 1140 dte->data[i] |= (1UL << _bit); 1141 } 1142 1143 static bool __reuse_device_table(struct amd_iommu *iommu) 1144 { 1145 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 1146 struct dev_table_entry *old_dev_tbl_entry; 1147 u32 lo, hi, old_devtb_size, devid; 1148 phys_addr_t old_devtb_phys; 1149 u16 dom_id; 1150 bool dte_v; 1151 u64 entry; 1152 1153 /* Each IOMMU use separate device table with the same size */ 1154 lo = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET); 1155 hi = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET + 4); 1156 entry = (((u64) hi) << 32) + lo; 1157 1158 old_devtb_size = ((entry & ~PAGE_MASK) + 1) << 12; 1159 if (old_devtb_size != pci_seg->dev_table_size) { 1160 pr_err("The device table size of IOMMU:%d is not expected!\n", 1161 iommu->index); 1162 return false; 1163 } 1164 1165 /* 1166 * When SME is enabled in the first kernel, the entry includes the 1167 * memory encryption mask(sme_me_mask), we must remove the memory 1168 * encryption mask to obtain the true physical address in kdump kernel. 1169 */ 1170 old_devtb_phys = __sme_clr(entry) & PAGE_MASK; 1171 1172 if (old_devtb_phys >= 0x100000000ULL) { 1173 pr_err("The address of old device table is above 4G, not trustworthy!\n"); 1174 return false; 1175 } 1176 1177 /* 1178 * Re-use the previous kernel's device table for kdump. 1179 */ 1180 pci_seg->old_dev_tbl_cpy = iommu_memremap(old_devtb_phys, pci_seg->dev_table_size); 1181 if (pci_seg->old_dev_tbl_cpy == NULL) { 1182 pr_err("Failed to remap memory for reusing old device table!\n"); 1183 return false; 1184 } 1185 1186 for (devid = 0; devid <= pci_seg->last_bdf; devid++) { 1187 old_dev_tbl_entry = &pci_seg->old_dev_tbl_cpy[devid]; 1188 dte_v = FIELD_GET(DTE_FLAG_V, old_dev_tbl_entry->data[0]); 1189 dom_id = FIELD_GET(DTE_DOMID_MASK, old_dev_tbl_entry->data[1]); 1190 1191 if (!dte_v || !dom_id) 1192 continue; 1193 /* 1194 * ID reservation can fail with -ENOSPC when there 1195 * are multiple devices present in the same domain, 1196 * hence check only for -ENOMEM. 1197 */ 1198 if (amd_iommu_pdom_id_reserve(dom_id, GFP_KERNEL) == -ENOMEM) 1199 return false; 1200 } 1201 1202 return true; 1203 } 1204 1205 static bool reuse_device_table(void) 1206 { 1207 struct amd_iommu *iommu; 1208 struct amd_iommu_pci_seg *pci_seg; 1209 1210 if (!amd_iommu_pre_enabled) 1211 return false; 1212 1213 pr_warn("Translation is already enabled - trying to reuse translation structures\n"); 1214 1215 /* 1216 * All IOMMUs within PCI segment shares common device table. 1217 * Hence reuse device table only once per PCI segment. 1218 */ 1219 for_each_pci_segment(pci_seg) { 1220 for_each_iommu(iommu) { 1221 if (pci_seg->id != iommu->pci_seg->id) 1222 continue; 1223 if (!__reuse_device_table(iommu)) 1224 return false; 1225 break; 1226 } 1227 } 1228 1229 return true; 1230 } 1231 1232 struct dev_table_entry *amd_iommu_get_ivhd_dte_flags(u16 segid, u16 devid) 1233 { 1234 struct ivhd_dte_flags *e; 1235 unsigned int best_len = UINT_MAX; 1236 struct dev_table_entry *dte = NULL; 1237 1238 for_each_ivhd_dte_flags(e) { 1239 /* 1240 * Need to go through the whole list to find the smallest range, 1241 * which contains the devid. 1242 */ 1243 if ((e->segid == segid) && 1244 (e->devid_first <= devid) && (devid <= e->devid_last)) { 1245 unsigned int len = e->devid_last - e->devid_first; 1246 1247 if (len < best_len) { 1248 dte = &(e->dte); 1249 best_len = len; 1250 } 1251 } 1252 } 1253 return dte; 1254 } 1255 1256 static bool search_ivhd_dte_flags(u16 segid, u16 first, u16 last) 1257 { 1258 struct ivhd_dte_flags *e; 1259 1260 for_each_ivhd_dte_flags(e) { 1261 if ((e->segid == segid) && 1262 (e->devid_first == first) && 1263 (e->devid_last == last)) 1264 return true; 1265 } 1266 return false; 1267 } 1268 1269 /* 1270 * This function takes the device specific flags read from the ACPI 1271 * table and sets up the device table entry with that information 1272 */ 1273 static void __init 1274 set_dev_entry_from_acpi_range(struct amd_iommu *iommu, u16 first, u16 last, 1275 u32 flags, u32 ext_flags) 1276 { 1277 int i; 1278 struct dev_table_entry dte = {}; 1279 1280 /* Parse IVHD DTE setting flags and store information */ 1281 if (flags) { 1282 struct ivhd_dte_flags *d; 1283 1284 if (search_ivhd_dte_flags(iommu->pci_seg->id, first, last)) 1285 return; 1286 1287 d = kzalloc_obj(struct ivhd_dte_flags, GFP_KERNEL); 1288 if (!d) 1289 return; 1290 1291 pr_debug("%s: devid range %#x:%#x\n", __func__, first, last); 1292 1293 if (flags & ACPI_DEVFLAG_INITPASS) 1294 set_dte_bit(&dte, DEV_ENTRY_INIT_PASS); 1295 if (flags & ACPI_DEVFLAG_EXTINT) 1296 set_dte_bit(&dte, DEV_ENTRY_EINT_PASS); 1297 if (flags & ACPI_DEVFLAG_NMI) 1298 set_dte_bit(&dte, DEV_ENTRY_NMI_PASS); 1299 if (flags & ACPI_DEVFLAG_SYSMGT1) 1300 set_dte_bit(&dte, DEV_ENTRY_SYSMGT1); 1301 if (flags & ACPI_DEVFLAG_SYSMGT2) 1302 set_dte_bit(&dte, DEV_ENTRY_SYSMGT2); 1303 if (flags & ACPI_DEVFLAG_LINT0) 1304 set_dte_bit(&dte, DEV_ENTRY_LINT0_PASS); 1305 if (flags & ACPI_DEVFLAG_LINT1) 1306 set_dte_bit(&dte, DEV_ENTRY_LINT1_PASS); 1307 1308 /* Apply erratum 63, which needs info in initial_dte */ 1309 if (FIELD_GET(DTE_DATA1_SYSMGT_MASK, dte.data[1]) == 0x1) 1310 dte.data[0] |= DTE_FLAG_IW; 1311 1312 memcpy(&d->dte, &dte, sizeof(dte)); 1313 d->segid = iommu->pci_seg->id; 1314 d->devid_first = first; 1315 d->devid_last = last; 1316 list_add_tail(&d->list, &amd_ivhd_dev_flags_list); 1317 } 1318 1319 for (i = first; i <= last; i++) { 1320 if (flags) { 1321 struct dev_table_entry *dev_table = get_dev_table(iommu); 1322 1323 memcpy(&dev_table[i], &dte, sizeof(dte)); 1324 } 1325 amd_iommu_set_rlookup_table(iommu, i); 1326 } 1327 } 1328 1329 static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu, 1330 u16 devid, u32 flags, u32 ext_flags) 1331 { 1332 set_dev_entry_from_acpi_range(iommu, devid, devid, flags, ext_flags); 1333 } 1334 1335 int __init add_special_device(u8 type, u8 id, u32 *devid, bool cmd_line) 1336 { 1337 struct devid_map *entry; 1338 struct list_head *list; 1339 1340 if (type == IVHD_SPECIAL_IOAPIC) 1341 list = &ioapic_map; 1342 else if (type == IVHD_SPECIAL_HPET) 1343 list = &hpet_map; 1344 else 1345 return -EINVAL; 1346 1347 list_for_each_entry(entry, list, list) { 1348 if (!(entry->id == id && entry->cmd_line)) 1349 continue; 1350 1351 pr_info("Command-line override present for %s id %d - ignoring\n", 1352 type == IVHD_SPECIAL_IOAPIC ? "IOAPIC" : "HPET", id); 1353 1354 *devid = entry->devid; 1355 1356 return 0; 1357 } 1358 1359 entry = kzalloc_obj(*entry, GFP_KERNEL); 1360 if (!entry) 1361 return -ENOMEM; 1362 1363 entry->id = id; 1364 entry->devid = *devid; 1365 entry->cmd_line = cmd_line; 1366 1367 list_add_tail(&entry->list, list); 1368 1369 return 0; 1370 } 1371 1372 static int __init add_acpi_hid_device(u8 *hid, u8 *uid, u32 *devid, 1373 bool cmd_line) 1374 { 1375 struct acpihid_map_entry *entry; 1376 struct list_head *list = &acpihid_map; 1377 1378 list_for_each_entry(entry, list, list) { 1379 if (strcmp(entry->hid, hid) || 1380 (*uid && *entry->uid && strcmp(entry->uid, uid)) || 1381 !entry->cmd_line) 1382 continue; 1383 1384 pr_info("Command-line override for hid:%s uid:%s\n", 1385 hid, uid); 1386 *devid = entry->devid; 1387 return 0; 1388 } 1389 1390 entry = kzalloc_obj(*entry, GFP_KERNEL); 1391 if (!entry) 1392 return -ENOMEM; 1393 1394 memcpy(entry->uid, uid, strlen(uid)); 1395 memcpy(entry->hid, hid, strlen(hid)); 1396 entry->devid = *devid; 1397 entry->cmd_line = cmd_line; 1398 entry->root_devid = (entry->devid & (~0x7)); 1399 1400 pr_info("%s, add hid:%s, uid:%s, rdevid:%#x\n", 1401 entry->cmd_line ? "cmd" : "ivrs", 1402 entry->hid, entry->uid, entry->root_devid); 1403 1404 list_add_tail(&entry->list, list); 1405 return 0; 1406 } 1407 1408 static int __init add_early_maps(void) 1409 { 1410 int i, ret; 1411 1412 for (i = 0; i < early_ioapic_map_size; ++i) { 1413 ret = add_special_device(IVHD_SPECIAL_IOAPIC, 1414 early_ioapic_map[i].id, 1415 &early_ioapic_map[i].devid, 1416 early_ioapic_map[i].cmd_line); 1417 if (ret) 1418 return ret; 1419 } 1420 1421 for (i = 0; i < early_hpet_map_size; ++i) { 1422 ret = add_special_device(IVHD_SPECIAL_HPET, 1423 early_hpet_map[i].id, 1424 &early_hpet_map[i].devid, 1425 early_hpet_map[i].cmd_line); 1426 if (ret) 1427 return ret; 1428 } 1429 1430 for (i = 0; i < early_acpihid_map_size; ++i) { 1431 ret = add_acpi_hid_device(early_acpihid_map[i].hid, 1432 early_acpihid_map[i].uid, 1433 &early_acpihid_map[i].devid, 1434 early_acpihid_map[i].cmd_line); 1435 if (ret) 1436 return ret; 1437 } 1438 1439 return 0; 1440 } 1441 1442 /* 1443 * Takes a pointer to an AMD IOMMU entry in the ACPI table and 1444 * initializes the hardware and our data structures with it. 1445 */ 1446 static int __init init_iommu_from_acpi(struct amd_iommu *iommu, 1447 struct ivhd_header *h) 1448 { 1449 u8 *p = (u8 *)h; 1450 u8 *end = p, flags = 0; 1451 u16 devid = 0, devid_start = 0, devid_to = 0, seg_id; 1452 u32 dev_i, ext_flags = 0; 1453 bool alias = false; 1454 struct ivhd_entry *e; 1455 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 1456 u32 ivhd_size; 1457 int ret; 1458 1459 1460 ret = add_early_maps(); 1461 if (ret) 1462 return ret; 1463 1464 amd_iommu_apply_ivrs_quirks(); 1465 1466 /* 1467 * First save the recommended feature enable bits from ACPI 1468 */ 1469 iommu->acpi_flags = h->flags; 1470 1471 /* 1472 * Done. Now parse the device entries 1473 */ 1474 ivhd_size = get_ivhd_header_size(h); 1475 if (!ivhd_size) { 1476 pr_err("Unsupported IVHD type %#x\n", h->type); 1477 return -EINVAL; 1478 } 1479 1480 p += ivhd_size; 1481 1482 end += h->length; 1483 1484 1485 while (p < end) { 1486 e = (struct ivhd_entry *)p; 1487 seg_id = pci_seg->id; 1488 1489 switch (e->type) { 1490 case IVHD_DEV_ALL: 1491 1492 DUMP_printk(" DEV_ALL\t\t\tsetting: %#02x\n", e->flags); 1493 set_dev_entry_from_acpi_range(iommu, 0, pci_seg->last_bdf, e->flags, 0); 1494 break; 1495 case IVHD_DEV_SELECT: 1496 1497 DUMP_printk(" DEV_SELECT\t\t\tdevid: %04x:%02x:%02x.%x flags: %#02x\n", 1498 seg_id, PCI_BUS_NUM(e->devid), 1499 PCI_SLOT(e->devid), 1500 PCI_FUNC(e->devid), 1501 e->flags); 1502 1503 devid = e->devid; 1504 set_dev_entry_from_acpi(iommu, devid, e->flags, 0); 1505 break; 1506 case IVHD_DEV_SELECT_RANGE_START: 1507 1508 DUMP_printk(" DEV_SELECT_RANGE_START\tdevid: %04x:%02x:%02x.%x flags: %#02x\n", 1509 seg_id, PCI_BUS_NUM(e->devid), 1510 PCI_SLOT(e->devid), 1511 PCI_FUNC(e->devid), 1512 e->flags); 1513 1514 devid_start = e->devid; 1515 flags = e->flags; 1516 ext_flags = 0; 1517 alias = false; 1518 break; 1519 case IVHD_DEV_ALIAS: 1520 1521 DUMP_printk(" DEV_ALIAS\t\t\tdevid: %04x:%02x:%02x.%x flags: %#02x devid_to: %02x:%02x.%x\n", 1522 seg_id, PCI_BUS_NUM(e->devid), 1523 PCI_SLOT(e->devid), 1524 PCI_FUNC(e->devid), 1525 e->flags, 1526 PCI_BUS_NUM(e->ext >> 8), 1527 PCI_SLOT(e->ext >> 8), 1528 PCI_FUNC(e->ext >> 8)); 1529 1530 devid = e->devid; 1531 devid_to = e->ext >> 8; 1532 set_dev_entry_from_acpi(iommu, devid , e->flags, 0); 1533 set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0); 1534 pci_seg->alias_table[devid] = devid_to; 1535 break; 1536 case IVHD_DEV_ALIAS_RANGE: 1537 1538 DUMP_printk(" DEV_ALIAS_RANGE\t\tdevid: %04x:%02x:%02x.%x flags: %#02x devid_to: %04x:%02x:%02x.%x\n", 1539 seg_id, PCI_BUS_NUM(e->devid), 1540 PCI_SLOT(e->devid), 1541 PCI_FUNC(e->devid), 1542 e->flags, 1543 seg_id, PCI_BUS_NUM(e->ext >> 8), 1544 PCI_SLOT(e->ext >> 8), 1545 PCI_FUNC(e->ext >> 8)); 1546 1547 devid_start = e->devid; 1548 flags = e->flags; 1549 devid_to = e->ext >> 8; 1550 ext_flags = 0; 1551 alias = true; 1552 break; 1553 case IVHD_DEV_EXT_SELECT: 1554 1555 DUMP_printk(" DEV_EXT_SELECT\t\tdevid: %04x:%02x:%02x.%x flags: %#02x ext: %08x\n", 1556 seg_id, PCI_BUS_NUM(e->devid), 1557 PCI_SLOT(e->devid), 1558 PCI_FUNC(e->devid), 1559 e->flags, e->ext); 1560 1561 devid = e->devid; 1562 set_dev_entry_from_acpi(iommu, devid, e->flags, 1563 e->ext); 1564 break; 1565 case IVHD_DEV_EXT_SELECT_RANGE: 1566 1567 DUMP_printk(" DEV_EXT_SELECT_RANGE\tdevid: %04x:%02x:%02x.%x flags: %#02x ext: %08x\n", 1568 seg_id, PCI_BUS_NUM(e->devid), 1569 PCI_SLOT(e->devid), 1570 PCI_FUNC(e->devid), 1571 e->flags, e->ext); 1572 1573 devid_start = e->devid; 1574 flags = e->flags; 1575 ext_flags = e->ext; 1576 alias = false; 1577 break; 1578 case IVHD_DEV_RANGE_END: 1579 1580 DUMP_printk(" DEV_RANGE_END\t\tdevid: %04x:%02x:%02x.%x\n", 1581 seg_id, PCI_BUS_NUM(e->devid), 1582 PCI_SLOT(e->devid), 1583 PCI_FUNC(e->devid)); 1584 1585 devid = e->devid; 1586 if (alias) { 1587 for (dev_i = devid_start; dev_i <= devid; ++dev_i) 1588 pci_seg->alias_table[dev_i] = devid_to; 1589 set_dev_entry_from_acpi(iommu, devid_to, flags, ext_flags); 1590 } 1591 set_dev_entry_from_acpi_range(iommu, devid_start, devid, flags, ext_flags); 1592 break; 1593 case IVHD_DEV_SPECIAL: { 1594 u8 handle, type; 1595 const char *var; 1596 u32 devid; 1597 int ret; 1598 1599 handle = e->ext & 0xff; 1600 devid = PCI_SEG_DEVID_TO_SBDF(seg_id, (e->ext >> 8)); 1601 type = (e->ext >> 24) & 0xff; 1602 1603 if (type == IVHD_SPECIAL_IOAPIC) 1604 var = "IOAPIC"; 1605 else if (type == IVHD_SPECIAL_HPET) 1606 var = "HPET"; 1607 else 1608 var = "UNKNOWN"; 1609 1610 DUMP_printk(" DEV_SPECIAL(%s[%d])\t\tdevid: %04x:%02x:%02x.%x, flags: %#02x\n", 1611 var, (int)handle, 1612 seg_id, PCI_BUS_NUM(devid), 1613 PCI_SLOT(devid), 1614 PCI_FUNC(devid), 1615 e->flags); 1616 1617 ret = add_special_device(type, handle, &devid, false); 1618 if (ret) 1619 return ret; 1620 1621 /* 1622 * add_special_device might update the devid in case a 1623 * command-line override is present. So call 1624 * set_dev_entry_from_acpi after add_special_device. 1625 */ 1626 set_dev_entry_from_acpi(iommu, devid, e->flags, 0); 1627 1628 break; 1629 } 1630 case IVHD_DEV_ACPI_HID: { 1631 u32 devid; 1632 u8 hid[ACPIHID_HID_LEN]; 1633 u8 uid[ACPIHID_UID_LEN]; 1634 int ret; 1635 1636 if (h->type != 0x40) { 1637 pr_err(FW_BUG "Invalid IVHD device type %#x\n", 1638 e->type); 1639 break; 1640 } 1641 1642 BUILD_BUG_ON(sizeof(e->ext_hid) != ACPIHID_HID_LEN - 1); 1643 memcpy(hid, &e->ext_hid, ACPIHID_HID_LEN - 1); 1644 hid[ACPIHID_HID_LEN - 1] = '\0'; 1645 1646 if (!(*hid)) { 1647 pr_err(FW_BUG "Invalid HID.\n"); 1648 break; 1649 } 1650 1651 uid[0] = '\0'; 1652 switch (e->uidf) { 1653 case UID_NOT_PRESENT: 1654 1655 if (e->uidl != 0) 1656 pr_warn(FW_BUG "Invalid UID length.\n"); 1657 1658 break; 1659 case UID_IS_INTEGER: 1660 1661 sprintf(uid, "%d", e->uid); 1662 1663 break; 1664 case UID_IS_CHARACTER: 1665 1666 memcpy(uid, &e->uid, e->uidl); 1667 uid[e->uidl] = '\0'; 1668 1669 break; 1670 default: 1671 break; 1672 } 1673 1674 devid = PCI_SEG_DEVID_TO_SBDF(seg_id, e->devid); 1675 DUMP_printk(" DEV_ACPI_HID(%s[%s])\t\tdevid: %04x:%02x:%02x.%x, flags: %#02x\n", 1676 hid, uid, seg_id, 1677 PCI_BUS_NUM(devid), 1678 PCI_SLOT(devid), 1679 PCI_FUNC(devid), 1680 e->flags); 1681 1682 flags = e->flags; 1683 1684 ret = add_acpi_hid_device(hid, uid, &devid, false); 1685 if (ret) 1686 return ret; 1687 1688 /* 1689 * add_special_device might update the devid in case a 1690 * command-line override is present. So call 1691 * set_dev_entry_from_acpi after add_special_device. 1692 */ 1693 set_dev_entry_from_acpi(iommu, devid, e->flags, 0); 1694 1695 break; 1696 } 1697 default: 1698 break; 1699 } 1700 1701 p += ivhd_entry_length(p); 1702 } 1703 1704 return 0; 1705 } 1706 1707 /* Allocate PCI segment data structure */ 1708 static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id, 1709 struct acpi_table_header *ivrs_base) 1710 { 1711 struct amd_iommu_pci_seg *pci_seg; 1712 int last_bdf; 1713 1714 /* 1715 * First parse ACPI tables to find the largest Bus/Dev/Func we need to 1716 * handle in this PCI segment. Upon this information the shared data 1717 * structures for the PCI segments in the system will be allocated. 1718 */ 1719 last_bdf = find_last_devid_acpi(ivrs_base, id); 1720 if (last_bdf < 0) 1721 return NULL; 1722 1723 pci_seg = kzalloc_obj(struct amd_iommu_pci_seg, GFP_KERNEL); 1724 if (pci_seg == NULL) 1725 return NULL; 1726 1727 pci_seg->last_bdf = last_bdf; 1728 DUMP_printk("PCI segment : 0x%0x, last bdf : 0x%04x\n", id, last_bdf); 1729 pci_seg->dev_table_size = 1730 max(roundup_pow_of_two((last_bdf + 1) * DEV_TABLE_ENTRY_SIZE), 1731 SZ_4K); 1732 1733 pci_seg->id = id; 1734 init_llist_head(&pci_seg->dev_data_list); 1735 INIT_LIST_HEAD(&pci_seg->unity_map); 1736 list_add_tail(&pci_seg->list, &amd_iommu_pci_seg_list); 1737 1738 if (alloc_dev_table(pci_seg)) 1739 goto err_free_pci_seg; 1740 if (alloc_alias_table(pci_seg)) 1741 goto err_free_dev_table; 1742 if (alloc_rlookup_table(pci_seg)) 1743 goto err_free_alias_table; 1744 1745 return pci_seg; 1746 1747 err_free_alias_table: 1748 free_alias_table(pci_seg); 1749 err_free_dev_table: 1750 free_dev_table(pci_seg); 1751 err_free_pci_seg: 1752 list_del(&pci_seg->list); 1753 kfree(pci_seg); 1754 return NULL; 1755 } 1756 1757 static struct amd_iommu_pci_seg *__init get_pci_segment(u16 id, 1758 struct acpi_table_header *ivrs_base) 1759 { 1760 struct amd_iommu_pci_seg *pci_seg; 1761 1762 for_each_pci_segment(pci_seg) { 1763 if (pci_seg->id == id) 1764 return pci_seg; 1765 } 1766 1767 return alloc_pci_segment(id, ivrs_base); 1768 } 1769 1770 static void __init free_pci_segments(void) 1771 { 1772 struct amd_iommu_pci_seg *pci_seg, *next; 1773 1774 for_each_pci_segment_safe(pci_seg, next) { 1775 list_del(&pci_seg->list); 1776 free_irq_lookup_table(pci_seg); 1777 free_rlookup_table(pci_seg); 1778 free_alias_table(pci_seg); 1779 free_dev_table(pci_seg); 1780 kfree(pci_seg); 1781 } 1782 } 1783 1784 static void __init free_sysfs(struct amd_iommu *iommu) 1785 { 1786 if (iommu->iommu.dev) { 1787 iommu_device_unregister(&iommu->iommu); 1788 iommu_device_sysfs_remove(&iommu->iommu); 1789 } 1790 } 1791 1792 static void __init free_iommu_one(struct amd_iommu *iommu) 1793 { 1794 free_sysfs(iommu); 1795 free_iommu_buffers(iommu); 1796 amd_iommu_free_ppr_log(iommu); 1797 free_ga_log(iommu); 1798 iommu_unmap_mmio_space(iommu); 1799 amd_iommu_iopf_uninit(iommu); 1800 } 1801 1802 static void __init free_iommu_all(void) 1803 { 1804 struct amd_iommu *iommu, *next; 1805 1806 for_each_iommu_safe(iommu, next) { 1807 list_del(&iommu->list); 1808 free_iommu_one(iommu); 1809 kfree(iommu); 1810 } 1811 } 1812 1813 /* 1814 * Family15h Model 10h-1fh erratum 746 (IOMMU Logging May Stall Translations) 1815 * Workaround: 1816 * BIOS should disable L2B micellaneous clock gating by setting 1817 * L2_L2B_CK_GATE_CONTROL[CKGateL2BMiscDisable](D0F2xF4_x90[2]) = 1b 1818 */ 1819 static void amd_iommu_erratum_746_workaround(struct amd_iommu *iommu) 1820 { 1821 u32 value; 1822 1823 if ((boot_cpu_data.x86 != 0x15) || 1824 (boot_cpu_data.x86_model < 0x10) || 1825 (boot_cpu_data.x86_model > 0x1f)) 1826 return; 1827 1828 pci_write_config_dword(iommu->dev, 0xf0, 0x90); 1829 pci_read_config_dword(iommu->dev, 0xf4, &value); 1830 1831 if (value & BIT(2)) 1832 return; 1833 1834 /* Select NB indirect register 0x90 and enable writing */ 1835 pci_write_config_dword(iommu->dev, 0xf0, 0x90 | (1 << 8)); 1836 1837 pci_write_config_dword(iommu->dev, 0xf4, value | 0x4); 1838 pci_info(iommu->dev, "Applying erratum 746 workaround\n"); 1839 1840 /* Clear the enable writing bit */ 1841 pci_write_config_dword(iommu->dev, 0xf0, 0x90); 1842 } 1843 1844 /* 1845 * Family15h Model 30h-3fh (IOMMU Mishandles ATS Write Permission) 1846 * Workaround: 1847 * BIOS should enable ATS write permission check by setting 1848 * L2_DEBUG_3[AtsIgnoreIWDis](D0F2xF4_x47[0]) = 1b 1849 */ 1850 static void amd_iommu_ats_write_check_workaround(struct amd_iommu *iommu) 1851 { 1852 u32 value; 1853 1854 if ((boot_cpu_data.x86 != 0x15) || 1855 (boot_cpu_data.x86_model < 0x30) || 1856 (boot_cpu_data.x86_model > 0x3f)) 1857 return; 1858 1859 /* Test L2_DEBUG_3[AtsIgnoreIWDis] == 1 */ 1860 value = iommu_read_l2(iommu, 0x47); 1861 1862 if (value & BIT(0)) 1863 return; 1864 1865 /* Set L2_DEBUG_3[AtsIgnoreIWDis] = 1 */ 1866 iommu_write_l2(iommu, 0x47, value | BIT(0)); 1867 1868 pci_info(iommu->dev, "Applying ATS write check workaround\n"); 1869 } 1870 1871 /* 1872 * This function glues the initialization function for one IOMMU 1873 * together and also allocates the command buffer and programs the 1874 * hardware. It does NOT enable the IOMMU. This is done afterwards. 1875 */ 1876 static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, 1877 struct acpi_table_header *ivrs_base) 1878 { 1879 struct amd_iommu_pci_seg *pci_seg; 1880 1881 pci_seg = get_pci_segment(h->pci_seg, ivrs_base); 1882 if (pci_seg == NULL) 1883 return -ENOMEM; 1884 iommu->pci_seg = pci_seg; 1885 1886 raw_spin_lock_init(&iommu->lock); 1887 iommu->cmd_sem_val = 0; 1888 1889 /* Add IOMMU to internal data structures */ 1890 list_add_tail(&iommu->list, &amd_iommu_list); 1891 iommu->index = amd_iommus_present++; 1892 1893 if (unlikely(iommu->index >= MAX_IOMMUS)) { 1894 WARN(1, "System has more IOMMUs than supported by this driver\n"); 1895 return -ENOSYS; 1896 } 1897 1898 /* 1899 * Copy data from ACPI table entry to the iommu struct 1900 */ 1901 iommu->devid = h->devid; 1902 iommu->cap_ptr = h->cap_ptr; 1903 iommu->mmio_phys = h->mmio_phys; 1904 1905 switch (h->type) { 1906 case 0x10: 1907 /* Check if IVHD EFR contains proper max banks/counters */ 1908 if ((h->efr_attr != 0) && 1909 ((h->efr_attr & (0xF << 13)) != 0) && 1910 ((h->efr_attr & (0x3F << 17)) != 0)) 1911 iommu->mmio_phys_end = MMIO_REG_END_OFFSET; 1912 else 1913 iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET; 1914 1915 /* GAM requires GA mode. */ 1916 if ((h->efr_attr & (0x1 << IOMMU_FEAT_GASUP_SHIFT)) == 0) 1917 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; 1918 break; 1919 case 0x11: 1920 case 0x40: 1921 if (h->efr_reg & (1 << 9)) 1922 iommu->mmio_phys_end = MMIO_REG_END_OFFSET; 1923 else 1924 iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET; 1925 1926 /* XT and GAM require GA mode. */ 1927 if ((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0) { 1928 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; 1929 break; 1930 } 1931 1932 if (h->efr_reg & BIT(IOMMU_EFR_XTSUP_SHIFT)) 1933 amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE; 1934 1935 if (h->efr_attr & BIT(IOMMU_IVHD_ATTR_HATDIS_SHIFT)) { 1936 pr_warn_once("Host Address Translation is not supported.\n"); 1937 amd_iommu_hatdis = true; 1938 } 1939 1940 early_iommu_features_init(iommu, h); 1941 1942 break; 1943 default: 1944 return -EINVAL; 1945 } 1946 1947 iommu->mmio_base = iommu_map_mmio_space(iommu->mmio_phys, 1948 iommu->mmio_phys_end); 1949 if (!iommu->mmio_base) 1950 return -ENOMEM; 1951 1952 return init_iommu_from_acpi(iommu, h); 1953 } 1954 1955 static int __init init_iommu_one_late(struct amd_iommu *iommu) 1956 { 1957 int ret; 1958 1959 ret = alloc_iommu_buffers(iommu); 1960 if (ret) 1961 return ret; 1962 1963 iommu->int_enabled = false; 1964 1965 init_translation_status(iommu); 1966 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) { 1967 iommu_disable(iommu); 1968 clear_translation_pre_enabled(iommu); 1969 pr_warn("Translation was enabled for IOMMU:%d but we are not in kdump mode\n", 1970 iommu->index); 1971 } 1972 if (amd_iommu_pre_enabled) 1973 amd_iommu_pre_enabled = translation_pre_enabled(iommu); 1974 1975 if (amd_iommu_irq_remap) { 1976 ret = amd_iommu_create_irq_domain(iommu); 1977 if (ret) 1978 return ret; 1979 } 1980 1981 /* 1982 * Make sure IOMMU is not considered to translate itself. The IVRS 1983 * table tells us so, but this is a lie! 1984 */ 1985 iommu->pci_seg->rlookup_table[iommu->devid] = NULL; 1986 1987 return 0; 1988 } 1989 1990 /** 1991 * get_highest_supported_ivhd_type - Look up the appropriate IVHD type 1992 * @ivrs: Pointer to the IVRS header 1993 * 1994 * This function search through all IVDB of the maximum supported IVHD 1995 */ 1996 static u8 get_highest_supported_ivhd_type(struct acpi_table_header *ivrs) 1997 { 1998 u8 *base = (u8 *)ivrs; 1999 struct ivhd_header *ivhd = (struct ivhd_header *) 2000 (base + IVRS_HEADER_LENGTH); 2001 u8 last_type = ivhd->type; 2002 u16 devid = ivhd->devid; 2003 2004 while (((u8 *)ivhd - base < ivrs->length) && 2005 (ivhd->type <= ACPI_IVHD_TYPE_MAX_SUPPORTED)) { 2006 u8 *p = (u8 *) ivhd; 2007 2008 if (ivhd->devid == devid) 2009 last_type = ivhd->type; 2010 ivhd = (struct ivhd_header *)(p + ivhd->length); 2011 } 2012 2013 return last_type; 2014 } 2015 2016 /* 2017 * Iterates over all IOMMU entries in the ACPI table, allocates the 2018 * IOMMU structure and initializes it with init_iommu_one() 2019 */ 2020 static int __init init_iommu_all(struct acpi_table_header *table) 2021 { 2022 u8 *p = (u8 *)table, *end = (u8 *)table; 2023 struct ivhd_header *h; 2024 struct amd_iommu *iommu; 2025 int ret; 2026 2027 end += table->length; 2028 p += IVRS_HEADER_LENGTH; 2029 2030 /* Phase 1: Process all IVHD blocks */ 2031 while (p < end) { 2032 h = (struct ivhd_header *)p; 2033 if (*p == amd_iommu_target_ivhd_type) { 2034 2035 DUMP_printk("device: %04x:%02x:%02x.%01x cap: %04x " 2036 "flags: %01x info %04x\n", 2037 h->pci_seg, PCI_BUS_NUM(h->devid), 2038 PCI_SLOT(h->devid), PCI_FUNC(h->devid), 2039 h->cap_ptr, h->flags, h->info); 2040 DUMP_printk(" mmio-addr: %016llx\n", 2041 h->mmio_phys); 2042 2043 iommu = kzalloc_obj(struct amd_iommu, GFP_KERNEL); 2044 if (iommu == NULL) 2045 return -ENOMEM; 2046 2047 ret = init_iommu_one(iommu, h, table); 2048 if (ret) 2049 return ret; 2050 } 2051 p += h->length; 2052 2053 } 2054 WARN_ON(p != end); 2055 2056 /* Phase 2 : Early feature support check */ 2057 get_global_efr(); 2058 2059 /* Phase 3 : Enabling IOMMU features */ 2060 for_each_iommu(iommu) { 2061 ret = init_iommu_one_late(iommu); 2062 if (ret) 2063 return ret; 2064 } 2065 2066 return 0; 2067 } 2068 2069 static void init_iommu_perf_ctr(struct amd_iommu *iommu) 2070 { 2071 u64 val; 2072 struct pci_dev *pdev = iommu->dev; 2073 2074 if (!check_feature(FEATURE_PC)) 2075 return; 2076 2077 amd_iommu_pc_present = true; 2078 2079 pci_info(pdev, "IOMMU performance counters supported\n"); 2080 2081 val = readl(iommu->mmio_base + MMIO_CNTR_CONF_OFFSET); 2082 iommu->max_banks = (u8) ((val >> 12) & 0x3f); 2083 iommu->max_counters = (u8) ((val >> 7) & 0xf); 2084 2085 return; 2086 } 2087 2088 static ssize_t amd_iommu_show_cap(struct device *dev, 2089 struct device_attribute *attr, 2090 char *buf) 2091 { 2092 struct amd_iommu *iommu = dev_to_amd_iommu(dev); 2093 return sysfs_emit(buf, "%x\n", iommu->cap); 2094 } 2095 static DEVICE_ATTR(cap, S_IRUGO, amd_iommu_show_cap, NULL); 2096 2097 static ssize_t amd_iommu_show_features(struct device *dev, 2098 struct device_attribute *attr, 2099 char *buf) 2100 { 2101 return sysfs_emit(buf, "%llx:%llx\n", amd_iommu_efr, amd_iommu_efr2); 2102 } 2103 static DEVICE_ATTR(features, S_IRUGO, amd_iommu_show_features, NULL); 2104 2105 static struct attribute *amd_iommu_attrs[] = { 2106 &dev_attr_cap.attr, 2107 &dev_attr_features.attr, 2108 NULL, 2109 }; 2110 2111 static struct attribute_group amd_iommu_group = { 2112 .name = "amd-iommu", 2113 .attrs = amd_iommu_attrs, 2114 }; 2115 2116 static const struct attribute_group *amd_iommu_groups[] = { 2117 &amd_iommu_group, 2118 NULL, 2119 }; 2120 2121 /* 2122 * Note: IVHD 0x11 and 0x40 also contains exact copy 2123 * of the IOMMU Extended Feature Register [MMIO Offset 0030h]. 2124 * Default to EFR in IVHD since it is available sooner (i.e. before PCI init). 2125 */ 2126 static void __init late_iommu_features_init(struct amd_iommu *iommu) 2127 { 2128 u64 features, features2; 2129 2130 if (!(iommu->cap & (1 << IOMMU_CAP_EFR))) 2131 return; 2132 2133 /* read extended feature bits */ 2134 features = readq(iommu->mmio_base + MMIO_EXT_FEATURES); 2135 features2 = readq(iommu->mmio_base + MMIO_EXT_FEATURES2); 2136 2137 if (!amd_iommu_efr) { 2138 amd_iommu_efr = features; 2139 amd_iommu_efr2 = features2; 2140 return; 2141 } 2142 2143 /* 2144 * Sanity check and warn if EFR values from 2145 * IVHD and MMIO conflict. 2146 */ 2147 if (features != amd_iommu_efr || 2148 features2 != amd_iommu_efr2) { 2149 pr_warn(FW_WARN 2150 "EFR mismatch. Use IVHD EFR (%#llx : %#llx), EFR2 (%#llx : %#llx).\n", 2151 features, amd_iommu_efr, 2152 features2, amd_iommu_efr2); 2153 } 2154 } 2155 2156 static int __init iommu_init_pci(struct amd_iommu *iommu) 2157 { 2158 int cap_ptr = iommu->cap_ptr; 2159 int ret; 2160 2161 iommu->dev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, 2162 PCI_BUS_NUM(iommu->devid), 2163 iommu->devid & 0xff); 2164 if (!iommu->dev) 2165 return -ENODEV; 2166 2167 /* ACPI _PRT won't have an IRQ for IOMMU */ 2168 iommu->dev->irq_managed = 1; 2169 2170 pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET, 2171 &iommu->cap); 2172 2173 if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB))) 2174 amd_iommu_iotlb_sup = false; 2175 2176 late_iommu_features_init(iommu); 2177 2178 if (check_feature(FEATURE_GT)) { 2179 int glxval; 2180 u64 pasmax; 2181 2182 pasmax = FIELD_GET(FEATURE_PASMAX, amd_iommu_efr); 2183 iommu->iommu.max_pasids = (1 << (pasmax + 1)) - 1; 2184 2185 BUG_ON(iommu->iommu.max_pasids & ~PASID_MASK); 2186 2187 glxval = FIELD_GET(FEATURE_GLX, amd_iommu_efr); 2188 2189 if (amd_iommu_max_glx_val == -1) 2190 amd_iommu_max_glx_val = glxval; 2191 else 2192 amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval); 2193 2194 iommu_enable_gt(iommu); 2195 } 2196 2197 if (check_feature(FEATURE_PPR) && amd_iommu_alloc_ppr_log(iommu)) 2198 return -ENOMEM; 2199 2200 if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) { 2201 pr_info("Using strict mode due to virtualization\n"); 2202 iommu_set_dma_strict(); 2203 amd_iommu_np_cache = true; 2204 } 2205 2206 init_iommu_perf_ctr(iommu); 2207 2208 if (is_rd890_iommu(iommu->dev)) { 2209 int i, j; 2210 2211 iommu->root_pdev = 2212 pci_get_domain_bus_and_slot(iommu->pci_seg->id, 2213 iommu->dev->bus->number, 2214 PCI_DEVFN(0, 0)); 2215 2216 /* 2217 * Some rd890 systems may not be fully reconfigured by the 2218 * BIOS, so it's necessary for us to store this information so 2219 * it can be reprogrammed on resume 2220 */ 2221 pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4, 2222 &iommu->stored_addr_lo); 2223 pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8, 2224 &iommu->stored_addr_hi); 2225 2226 /* Low bit locks writes to configuration space */ 2227 iommu->stored_addr_lo &= ~1; 2228 2229 for (i = 0; i < 6; i++) 2230 for (j = 0; j < 0x12; j++) 2231 iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j); 2232 2233 for (i = 0; i < 0x83; i++) 2234 iommu->stored_l2[i] = iommu_read_l2(iommu, i); 2235 } 2236 2237 amd_iommu_erratum_746_workaround(iommu); 2238 amd_iommu_ats_write_check_workaround(iommu); 2239 2240 ret = iommu_device_sysfs_add(&iommu->iommu, &iommu->dev->dev, 2241 amd_iommu_groups, "ivhd%d", iommu->index); 2242 if (ret) 2243 return ret; 2244 2245 /* 2246 * Allocate per IOMMU IOPF queue here so that in attach device path, 2247 * PRI capable device can be added to IOPF queue 2248 */ 2249 if (amd_iommu_gt_ppr_supported()) { 2250 ret = amd_iommu_iopf_init(iommu); 2251 if (ret) 2252 return ret; 2253 } 2254 2255 ret = iommu_device_register(&iommu->iommu, &amd_iommu_ops, NULL); 2256 if (ret || amd_iommu_pgtable == PD_MODE_NONE) { 2257 /* 2258 * Remove sysfs if DMA translation is not supported by the 2259 * IOMMU. Do not return an error to enable IRQ remapping 2260 * in state_next(), DTE[V, TV] must eventually be set to 0. 2261 */ 2262 iommu_device_sysfs_remove(&iommu->iommu); 2263 } 2264 2265 return pci_enable_device(iommu->dev); 2266 } 2267 2268 static void print_iommu_info(void) 2269 { 2270 int i; 2271 static const char * const feat_str[] = { 2272 "PreF", "PPR", "X2APIC", "NX", "GT", "[5]", 2273 "IA", "GA", "HE", "PC" 2274 }; 2275 2276 if (amd_iommu_efr) { 2277 pr_info("Extended features (%#llx, %#llx):", amd_iommu_efr, amd_iommu_efr2); 2278 2279 for (i = 0; i < ARRAY_SIZE(feat_str); ++i) { 2280 if (check_feature(1ULL << i)) 2281 pr_cont(" %s", feat_str[i]); 2282 } 2283 2284 if (check_feature(FEATURE_GAM_VAPIC)) 2285 pr_cont(" GA_vAPIC"); 2286 2287 if (check_feature(FEATURE_SNP)) 2288 pr_cont(" SNP"); 2289 2290 if (check_feature2(FEATURE_SEVSNPIO_SUP)) 2291 pr_cont(" SEV-TIO"); 2292 2293 pr_cont("\n"); 2294 } 2295 2296 if (irq_remapping_enabled) { 2297 pr_info("Interrupt remapping enabled\n"); 2298 if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE) 2299 pr_info("X2APIC enabled\n"); 2300 } 2301 if (amd_iommu_pgtable == PD_MODE_V2) { 2302 pr_info("V2 page table enabled (Paging mode : %d level)\n", 2303 amd_iommu_gpt_level); 2304 } 2305 } 2306 2307 static int __init amd_iommu_init_pci(void) 2308 { 2309 struct amd_iommu *iommu; 2310 struct amd_iommu_pci_seg *pci_seg; 2311 int ret; 2312 2313 /* Init global identity domain before registering IOMMU */ 2314 amd_iommu_init_identity_domain(); 2315 2316 for_each_iommu(iommu) { 2317 ret = iommu_init_pci(iommu); 2318 if (ret) { 2319 pr_err("IOMMU%d: Failed to initialize IOMMU Hardware (error=%d)!\n", 2320 iommu->index, ret); 2321 goto out; 2322 } 2323 /* Need to setup range after PCI init */ 2324 iommu_set_cwwb_range(iommu); 2325 } 2326 2327 /* 2328 * Order is important here to make sure any unity map requirements are 2329 * fulfilled. The unity mappings are created and written to the device 2330 * table during the iommu_init_pci() call. 2331 * 2332 * After that we call init_device_table_dma() to make sure any 2333 * uninitialized DTE will block DMA, and in the end we flush the caches 2334 * of all IOMMUs to make sure the changes to the device table are 2335 * active. 2336 */ 2337 for_each_pci_segment(pci_seg) 2338 init_device_table_dma(pci_seg); 2339 2340 for_each_iommu(iommu) 2341 amd_iommu_flush_all_caches(iommu); 2342 2343 print_iommu_info(); 2344 2345 out: 2346 return ret; 2347 } 2348 2349 /**************************************************************************** 2350 * 2351 * The following functions initialize the MSI interrupts for all IOMMUs 2352 * in the system. It's a bit challenging because there could be multiple 2353 * IOMMUs per PCI BDF but we can call pci_enable_msi(x) only once per 2354 * pci_dev. 2355 * 2356 ****************************************************************************/ 2357 2358 static int iommu_setup_msi(struct amd_iommu *iommu) 2359 { 2360 int r; 2361 2362 r = pci_enable_msi(iommu->dev); 2363 if (r) 2364 return r; 2365 2366 r = request_threaded_irq(iommu->dev->irq, NULL, amd_iommu_int_thread, 2367 IRQF_ONESHOT, "AMD-Vi", iommu); 2368 if (r) { 2369 pci_disable_msi(iommu->dev); 2370 return r; 2371 } 2372 2373 return 0; 2374 } 2375 2376 union intcapxt { 2377 u64 capxt; 2378 struct { 2379 u64 reserved_0 : 2, 2380 dest_mode_logical : 1, 2381 reserved_1 : 5, 2382 destid_0_23 : 24, 2383 vector : 8, 2384 reserved_2 : 16, 2385 destid_24_31 : 8; 2386 }; 2387 } __attribute__ ((packed)); 2388 2389 2390 static struct irq_chip intcapxt_controller; 2391 2392 static int intcapxt_irqdomain_activate(struct irq_domain *domain, 2393 struct irq_data *irqd, bool reserve) 2394 { 2395 return 0; 2396 } 2397 2398 static void intcapxt_irqdomain_deactivate(struct irq_domain *domain, 2399 struct irq_data *irqd) 2400 { 2401 } 2402 2403 2404 static int intcapxt_irqdomain_alloc(struct irq_domain *domain, unsigned int virq, 2405 unsigned int nr_irqs, void *arg) 2406 { 2407 struct irq_alloc_info *info = arg; 2408 int i, ret; 2409 2410 if (!info || info->type != X86_IRQ_ALLOC_TYPE_AMDVI) 2411 return -EINVAL; 2412 2413 ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg); 2414 if (ret < 0) 2415 return ret; 2416 2417 for (i = virq; i < virq + nr_irqs; i++) { 2418 struct irq_data *irqd = irq_domain_get_irq_data(domain, i); 2419 2420 irqd->chip = &intcapxt_controller; 2421 irqd->hwirq = info->hwirq; 2422 irqd->chip_data = info->data; 2423 __irq_set_handler(i, handle_edge_irq, 0, "edge"); 2424 } 2425 2426 return ret; 2427 } 2428 2429 static void intcapxt_irqdomain_free(struct irq_domain *domain, unsigned int virq, 2430 unsigned int nr_irqs) 2431 { 2432 irq_domain_free_irqs_top(domain, virq, nr_irqs); 2433 } 2434 2435 2436 static void intcapxt_unmask_irq(struct irq_data *irqd) 2437 { 2438 struct amd_iommu *iommu = irqd->chip_data; 2439 struct irq_cfg *cfg = irqd_cfg(irqd); 2440 union intcapxt xt; 2441 2442 xt.capxt = 0ULL; 2443 xt.dest_mode_logical = apic->dest_mode_logical; 2444 xt.vector = cfg->vector; 2445 xt.destid_0_23 = cfg->dest_apicid & GENMASK(23, 0); 2446 xt.destid_24_31 = cfg->dest_apicid >> 24; 2447 2448 writeq(xt.capxt, iommu->mmio_base + irqd->hwirq); 2449 } 2450 2451 static void intcapxt_mask_irq(struct irq_data *irqd) 2452 { 2453 struct amd_iommu *iommu = irqd->chip_data; 2454 2455 writeq(0, iommu->mmio_base + irqd->hwirq); 2456 } 2457 2458 2459 static int intcapxt_set_affinity(struct irq_data *irqd, 2460 const struct cpumask *mask, bool force) 2461 { 2462 struct irq_data *parent = irqd->parent_data; 2463 int ret; 2464 2465 ret = parent->chip->irq_set_affinity(parent, mask, force); 2466 if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE) 2467 return ret; 2468 return 0; 2469 } 2470 2471 static int intcapxt_set_wake(struct irq_data *irqd, unsigned int on) 2472 { 2473 return on ? -EOPNOTSUPP : 0; 2474 } 2475 2476 static struct irq_chip intcapxt_controller = { 2477 .name = "IOMMU-MSI", 2478 .irq_unmask = intcapxt_unmask_irq, 2479 .irq_mask = intcapxt_mask_irq, 2480 .irq_ack = irq_chip_ack_parent, 2481 .irq_retrigger = irq_chip_retrigger_hierarchy, 2482 .irq_set_affinity = intcapxt_set_affinity, 2483 .irq_set_wake = intcapxt_set_wake, 2484 .flags = IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_MOVE_DEFERRED, 2485 }; 2486 2487 static const struct irq_domain_ops intcapxt_domain_ops = { 2488 .alloc = intcapxt_irqdomain_alloc, 2489 .free = intcapxt_irqdomain_free, 2490 .activate = intcapxt_irqdomain_activate, 2491 .deactivate = intcapxt_irqdomain_deactivate, 2492 }; 2493 2494 2495 static struct irq_domain *iommu_irqdomain; 2496 2497 static struct irq_domain *iommu_get_irqdomain(void) 2498 { 2499 struct fwnode_handle *fn; 2500 2501 /* No need for locking here (yet) as the init is single-threaded */ 2502 if (iommu_irqdomain) 2503 return iommu_irqdomain; 2504 2505 fn = irq_domain_alloc_named_fwnode("AMD-Vi-MSI"); 2506 if (!fn) 2507 return NULL; 2508 2509 iommu_irqdomain = irq_domain_create_hierarchy(x86_vector_domain, 0, 0, 2510 fn, &intcapxt_domain_ops, 2511 NULL); 2512 if (!iommu_irqdomain) 2513 irq_domain_free_fwnode(fn); 2514 2515 return iommu_irqdomain; 2516 } 2517 2518 static int __iommu_setup_intcapxt(struct amd_iommu *iommu, const char *devname, 2519 int hwirq, irq_handler_t thread_fn) 2520 { 2521 struct irq_domain *domain; 2522 struct irq_alloc_info info; 2523 int irq, ret; 2524 int node = dev_to_node(&iommu->dev->dev); 2525 2526 domain = iommu_get_irqdomain(); 2527 if (!domain) 2528 return -ENXIO; 2529 2530 init_irq_alloc_info(&info, NULL); 2531 info.type = X86_IRQ_ALLOC_TYPE_AMDVI; 2532 info.data = iommu; 2533 info.hwirq = hwirq; 2534 2535 irq = irq_domain_alloc_irqs(domain, 1, node, &info); 2536 if (irq < 0) { 2537 irq_domain_remove(domain); 2538 return irq; 2539 } 2540 2541 ret = request_threaded_irq(irq, NULL, thread_fn, IRQF_ONESHOT, devname, 2542 iommu); 2543 if (ret) { 2544 irq_domain_free_irqs(irq, 1); 2545 irq_domain_remove(domain); 2546 return ret; 2547 } 2548 2549 return 0; 2550 } 2551 2552 static int iommu_setup_intcapxt(struct amd_iommu *iommu) 2553 { 2554 int ret; 2555 2556 snprintf(iommu->evt_irq_name, sizeof(iommu->evt_irq_name), 2557 "AMD-Vi%d-Evt", iommu->index); 2558 ret = __iommu_setup_intcapxt(iommu, iommu->evt_irq_name, 2559 MMIO_INTCAPXT_EVT_OFFSET, 2560 amd_iommu_int_thread_evtlog); 2561 if (ret) 2562 return ret; 2563 2564 snprintf(iommu->ppr_irq_name, sizeof(iommu->ppr_irq_name), 2565 "AMD-Vi%d-PPR", iommu->index); 2566 ret = __iommu_setup_intcapxt(iommu, iommu->ppr_irq_name, 2567 MMIO_INTCAPXT_PPR_OFFSET, 2568 amd_iommu_int_thread_pprlog); 2569 if (ret) 2570 return ret; 2571 2572 #ifdef CONFIG_IRQ_REMAP 2573 snprintf(iommu->ga_irq_name, sizeof(iommu->ga_irq_name), 2574 "AMD-Vi%d-GA", iommu->index); 2575 ret = __iommu_setup_intcapxt(iommu, iommu->ga_irq_name, 2576 MMIO_INTCAPXT_GALOG_OFFSET, 2577 amd_iommu_int_thread_galog); 2578 #endif 2579 2580 return ret; 2581 } 2582 2583 static int iommu_init_irq(struct amd_iommu *iommu) 2584 { 2585 int ret; 2586 2587 if (iommu->int_enabled) 2588 goto enable_faults; 2589 2590 if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE) 2591 ret = iommu_setup_intcapxt(iommu); 2592 else if (iommu->dev->msi_cap) 2593 ret = iommu_setup_msi(iommu); 2594 else 2595 ret = -ENODEV; 2596 2597 if (ret) 2598 return ret; 2599 2600 iommu->int_enabled = true; 2601 enable_faults: 2602 2603 if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE) 2604 iommu_feature_enable(iommu, CONTROL_INTCAPXT_EN); 2605 2606 iommu_feature_enable(iommu, CONTROL_EVT_INT_EN); 2607 2608 return 0; 2609 } 2610 2611 /**************************************************************************** 2612 * 2613 * The next functions belong to the third pass of parsing the ACPI 2614 * table. In this last pass the memory mapping requirements are 2615 * gathered (like exclusion and unity mapping ranges). 2616 * 2617 ****************************************************************************/ 2618 2619 static void __init free_unity_maps(void) 2620 { 2621 struct unity_map_entry *entry, *next; 2622 struct amd_iommu_pci_seg *p, *pci_seg; 2623 2624 for_each_pci_segment_safe(pci_seg, p) { 2625 list_for_each_entry_safe(entry, next, &pci_seg->unity_map, list) { 2626 list_del(&entry->list); 2627 kfree(entry); 2628 } 2629 } 2630 } 2631 2632 /* called for unity map ACPI definition */ 2633 static int __init init_unity_map_range(struct ivmd_header *m, 2634 struct acpi_table_header *ivrs_base) 2635 { 2636 struct unity_map_entry *e = NULL; 2637 struct amd_iommu_pci_seg *pci_seg; 2638 char *s; 2639 2640 pci_seg = get_pci_segment(m->pci_seg, ivrs_base); 2641 if (pci_seg == NULL) 2642 return -ENOMEM; 2643 2644 e = kzalloc_obj(*e, GFP_KERNEL); 2645 if (e == NULL) 2646 return -ENOMEM; 2647 2648 switch (m->type) { 2649 default: 2650 kfree(e); 2651 return 0; 2652 case ACPI_IVMD_TYPE: 2653 s = "IVMD_TYPEi\t\t\t"; 2654 e->devid_start = e->devid_end = m->devid; 2655 break; 2656 case ACPI_IVMD_TYPE_ALL: 2657 s = "IVMD_TYPE_ALL\t\t"; 2658 e->devid_start = 0; 2659 e->devid_end = pci_seg->last_bdf; 2660 break; 2661 case ACPI_IVMD_TYPE_RANGE: 2662 s = "IVMD_TYPE_RANGE\t\t"; 2663 e->devid_start = m->devid; 2664 e->devid_end = m->aux; 2665 break; 2666 } 2667 e->address_start = PAGE_ALIGN(m->range_start); 2668 e->address_end = e->address_start + PAGE_ALIGN(m->range_length); 2669 e->prot = m->flags >> 1; 2670 2671 /* 2672 * Treat per-device exclusion ranges as r/w unity-mapped regions 2673 * since some buggy BIOSes might lead to the overwritten exclusion 2674 * range (exclusion_start and exclusion_length members). This 2675 * happens when there are multiple exclusion ranges (IVMD entries) 2676 * defined in ACPI table. 2677 */ 2678 if (m->flags & IVMD_FLAG_EXCL_RANGE) 2679 e->prot = (IVMD_FLAG_IW | IVMD_FLAG_IR) >> 1; 2680 2681 DUMP_printk("%s devid_start: %04x:%02x:%02x.%x devid_end: " 2682 "%04x:%02x:%02x.%x range_start: %016llx range_end: %016llx" 2683 " flags: %x\n", s, m->pci_seg, 2684 PCI_BUS_NUM(e->devid_start), PCI_SLOT(e->devid_start), 2685 PCI_FUNC(e->devid_start), m->pci_seg, 2686 PCI_BUS_NUM(e->devid_end), 2687 PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end), 2688 e->address_start, e->address_end, m->flags); 2689 2690 list_add_tail(&e->list, &pci_seg->unity_map); 2691 2692 return 0; 2693 } 2694 2695 /* iterates over all memory definitions we find in the ACPI table */ 2696 static int __init init_memory_definitions(struct acpi_table_header *table) 2697 { 2698 u8 *p = (u8 *)table, *end = (u8 *)table; 2699 struct ivmd_header *m; 2700 2701 end += table->length; 2702 p += IVRS_HEADER_LENGTH; 2703 2704 while (p < end) { 2705 m = (struct ivmd_header *)p; 2706 if (m->flags & (IVMD_FLAG_UNITY_MAP | IVMD_FLAG_EXCL_RANGE)) 2707 init_unity_map_range(m, table); 2708 2709 p += m->length; 2710 } 2711 2712 return 0; 2713 } 2714 2715 /* 2716 * Init the device table to not allow DMA access for devices 2717 */ 2718 static void init_device_table_dma(struct amd_iommu_pci_seg *pci_seg) 2719 { 2720 u32 devid; 2721 struct dev_table_entry *dev_table = pci_seg->dev_table; 2722 2723 if (!dev_table || amd_iommu_pgtable == PD_MODE_NONE) 2724 return; 2725 2726 for (devid = 0; devid <= pci_seg->last_bdf; ++devid) { 2727 set_dte_bit(&dev_table[devid], DEV_ENTRY_VALID); 2728 if (!amd_iommu_snp_en) 2729 set_dte_bit(&dev_table[devid], DEV_ENTRY_TRANSLATION); 2730 } 2731 } 2732 2733 static void __init uninit_device_table_dma(struct amd_iommu_pci_seg *pci_seg) 2734 { 2735 u32 devid; 2736 struct dev_table_entry *dev_table = pci_seg->dev_table; 2737 2738 if (dev_table == NULL) 2739 return; 2740 2741 for (devid = 0; devid <= pci_seg->last_bdf; ++devid) { 2742 dev_table[devid].data[0] = 0ULL; 2743 dev_table[devid].data[1] = 0ULL; 2744 } 2745 } 2746 2747 static void init_device_table(void) 2748 { 2749 struct amd_iommu_pci_seg *pci_seg; 2750 u32 devid; 2751 2752 if (!amd_iommu_irq_remap) 2753 return; 2754 2755 for_each_pci_segment(pci_seg) { 2756 for (devid = 0; devid <= pci_seg->last_bdf; ++devid) 2757 set_dte_bit(&pci_seg->dev_table[devid], DEV_ENTRY_IRQ_TBL_EN); 2758 } 2759 } 2760 2761 static void iommu_init_flags(struct amd_iommu *iommu) 2762 { 2763 iommu->acpi_flags & IVHD_FLAG_HT_TUN_EN_MASK ? 2764 iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) : 2765 iommu_feature_disable(iommu, CONTROL_HT_TUN_EN); 2766 2767 iommu->acpi_flags & IVHD_FLAG_PASSPW_EN_MASK ? 2768 iommu_feature_enable(iommu, CONTROL_PASSPW_EN) : 2769 iommu_feature_disable(iommu, CONTROL_PASSPW_EN); 2770 2771 iommu->acpi_flags & IVHD_FLAG_RESPASSPW_EN_MASK ? 2772 iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) : 2773 iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN); 2774 2775 iommu->acpi_flags & IVHD_FLAG_ISOC_EN_MASK ? 2776 iommu_feature_enable(iommu, CONTROL_ISOC_EN) : 2777 iommu_feature_disable(iommu, CONTROL_ISOC_EN); 2778 2779 /* 2780 * make IOMMU memory accesses cache coherent 2781 */ 2782 iommu_feature_enable(iommu, CONTROL_COHERENT_EN); 2783 2784 /* Set IOTLB invalidation timeout to 1s */ 2785 iommu_feature_set(iommu, CTRL_INV_TO_1S, CTRL_INV_TO_MASK, CONTROL_INV_TIMEOUT); 2786 2787 /* Enable Enhanced Peripheral Page Request Handling */ 2788 if (check_feature(FEATURE_EPHSUP)) 2789 iommu_feature_enable(iommu, CONTROL_EPH_EN); 2790 } 2791 2792 static void iommu_apply_resume_quirks(struct amd_iommu *iommu) 2793 { 2794 int i, j; 2795 u32 ioc_feature_control; 2796 struct pci_dev *pdev = iommu->root_pdev; 2797 2798 /* RD890 BIOSes may not have completely reconfigured the iommu */ 2799 if (!is_rd890_iommu(iommu->dev) || !pdev) 2800 return; 2801 2802 /* 2803 * First, we need to ensure that the iommu is enabled. This is 2804 * controlled by a register in the northbridge 2805 */ 2806 2807 /* Select Northbridge indirect register 0x75 and enable writing */ 2808 pci_write_config_dword(pdev, 0x60, 0x75 | (1 << 7)); 2809 pci_read_config_dword(pdev, 0x64, &ioc_feature_control); 2810 2811 /* Enable the iommu */ 2812 if (!(ioc_feature_control & 0x1)) 2813 pci_write_config_dword(pdev, 0x64, ioc_feature_control | 1); 2814 2815 /* Restore the iommu BAR */ 2816 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4, 2817 iommu->stored_addr_lo); 2818 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8, 2819 iommu->stored_addr_hi); 2820 2821 /* Restore the l1 indirect regs for each of the 6 l1s */ 2822 for (i = 0; i < 6; i++) 2823 for (j = 0; j < 0x12; j++) 2824 iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]); 2825 2826 /* Restore the l2 indirect regs */ 2827 for (i = 0; i < 0x83; i++) 2828 iommu_write_l2(iommu, i, iommu->stored_l2[i]); 2829 2830 /* Lock PCI setup registers */ 2831 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4, 2832 iommu->stored_addr_lo | 1); 2833 } 2834 2835 static void iommu_enable_ga(struct amd_iommu *iommu) 2836 { 2837 #ifdef CONFIG_IRQ_REMAP 2838 switch (amd_iommu_guest_ir) { 2839 case AMD_IOMMU_GUEST_IR_VAPIC: 2840 case AMD_IOMMU_GUEST_IR_LEGACY_GA: 2841 iommu_feature_enable(iommu, CONTROL_GA_EN); 2842 iommu->irte_ops = &irte_128_ops; 2843 break; 2844 default: 2845 iommu->irte_ops = &irte_32_ops; 2846 break; 2847 } 2848 #endif 2849 } 2850 2851 static void iommu_disable_irtcachedis(struct amd_iommu *iommu) 2852 { 2853 iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS); 2854 } 2855 2856 static void iommu_enable_irtcachedis(struct amd_iommu *iommu) 2857 { 2858 u64 ctrl; 2859 2860 if (!amd_iommu_irtcachedis) 2861 return; 2862 2863 /* 2864 * Note: 2865 * The support for IRTCacheDis feature is dertermined by 2866 * checking if the bit is writable. 2867 */ 2868 iommu_feature_enable(iommu, CONTROL_IRTCACHEDIS); 2869 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); 2870 ctrl &= (1ULL << CONTROL_IRTCACHEDIS); 2871 if (ctrl) 2872 iommu->irtcachedis_enabled = true; 2873 pr_info("iommu%d (%#06x) : IRT cache is %s\n", 2874 iommu->index, iommu->devid, 2875 iommu->irtcachedis_enabled ? "disabled" : "enabled"); 2876 } 2877 2878 static void iommu_enable_2k_int(struct amd_iommu *iommu) 2879 { 2880 if (!FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2)) 2881 return; 2882 2883 iommu_feature_set(iommu, 2884 CONTROL_NUM_INT_REMAP_MODE_2K, 2885 CONTROL_NUM_INT_REMAP_MODE_MASK, 2886 CONTROL_NUM_INT_REMAP_MODE); 2887 } 2888 2889 static void early_enable_iommu(struct amd_iommu *iommu) 2890 { 2891 iommu_disable(iommu); 2892 iommu_init_flags(iommu); 2893 iommu_set_device_table(iommu); 2894 iommu_enable_command_buffer(iommu); 2895 iommu_enable_event_buffer(iommu); 2896 iommu_set_exclusion_range(iommu); 2897 iommu_enable_gt(iommu); 2898 iommu_enable_ga(iommu); 2899 iommu_enable_xt(iommu); 2900 iommu_enable_irtcachedis(iommu); 2901 iommu_enable_2k_int(iommu); 2902 iommu_enable(iommu); 2903 amd_iommu_flush_all_caches(iommu); 2904 } 2905 2906 /* 2907 * This function finally enables all IOMMUs found in the system after 2908 * they have been initialized. 2909 * 2910 * Or if in kdump kernel and IOMMUs are all pre-enabled, try to reuse 2911 * the old content of device table entries. Not this case or reuse failed, 2912 * just continue as normal kernel does. 2913 */ 2914 static void early_enable_iommus(void) 2915 { 2916 struct amd_iommu *iommu; 2917 struct amd_iommu_pci_seg *pci_seg; 2918 2919 if (!reuse_device_table()) { 2920 /* 2921 * If come here because of failure in reusing device table from old 2922 * kernel with all IOMMUs enabled, print error message and try to 2923 * free allocated old_dev_tbl_cpy. 2924 */ 2925 if (amd_iommu_pre_enabled) { 2926 pr_err("Failed to reuse DEV table from previous kernel.\n"); 2927 /* 2928 * Bail out early if unable to remap/reuse DEV table from 2929 * previous kernel if SNP enabled as IOMMU commands will 2930 * time out without DEV table and cause kdump boot panic. 2931 */ 2932 BUG_ON(check_feature(FEATURE_SNP)); 2933 } 2934 2935 for_each_pci_segment(pci_seg) { 2936 if (pci_seg->old_dev_tbl_cpy != NULL) { 2937 memunmap((void *)pci_seg->old_dev_tbl_cpy); 2938 pci_seg->old_dev_tbl_cpy = NULL; 2939 } 2940 } 2941 2942 for_each_iommu(iommu) { 2943 clear_translation_pre_enabled(iommu); 2944 early_enable_iommu(iommu); 2945 } 2946 } else { 2947 pr_info("Reused DEV table from previous kernel.\n"); 2948 2949 for_each_pci_segment(pci_seg) { 2950 iommu_free_pages(pci_seg->dev_table); 2951 pci_seg->dev_table = pci_seg->old_dev_tbl_cpy; 2952 } 2953 2954 for_each_iommu(iommu) { 2955 iommu_disable_command_buffer(iommu); 2956 iommu_disable_event_buffer(iommu); 2957 iommu_disable_irtcachedis(iommu); 2958 iommu_enable_command_buffer(iommu); 2959 iommu_enable_event_buffer(iommu); 2960 iommu_enable_ga(iommu); 2961 iommu_enable_xt(iommu); 2962 iommu_enable_irtcachedis(iommu); 2963 iommu_enable_2k_int(iommu); 2964 iommu_set_device_table(iommu); 2965 amd_iommu_flush_all_caches(iommu); 2966 } 2967 } 2968 } 2969 2970 static void enable_iommus_ppr(void) 2971 { 2972 struct amd_iommu *iommu; 2973 2974 if (!amd_iommu_gt_ppr_supported()) 2975 return; 2976 2977 for_each_iommu(iommu) 2978 amd_iommu_enable_ppr_log(iommu); 2979 } 2980 2981 static void enable_iommus_vapic(void) 2982 { 2983 #ifdef CONFIG_IRQ_REMAP 2984 u32 status, i; 2985 struct amd_iommu *iommu; 2986 2987 for_each_iommu(iommu) { 2988 /* 2989 * Disable GALog if already running. It could have been enabled 2990 * in the previous boot before kdump. 2991 */ 2992 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); 2993 if (!(status & MMIO_STATUS_GALOG_RUN_MASK)) 2994 continue; 2995 2996 iommu_feature_disable(iommu, CONTROL_GALOG_EN); 2997 iommu_feature_disable(iommu, CONTROL_GAINT_EN); 2998 2999 /* 3000 * Need to set and poll check the GALOGRun bit to zero before 3001 * we can set/ modify GA Log registers safely. 3002 */ 3003 for (i = 0; i < MMIO_STATUS_TIMEOUT; ++i) { 3004 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); 3005 if (!(status & MMIO_STATUS_GALOG_RUN_MASK)) 3006 break; 3007 udelay(10); 3008 } 3009 3010 if (WARN_ON(i >= MMIO_STATUS_TIMEOUT)) 3011 return; 3012 } 3013 3014 if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) && 3015 !check_feature(FEATURE_GAM_VAPIC)) { 3016 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA; 3017 return; 3018 } 3019 3020 if (amd_iommu_snp_en && 3021 !FEATURE_SNPAVICSUP_GAM(amd_iommu_efr2)) { 3022 pr_warn("Force to disable Virtual APIC due to SNP\n"); 3023 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA; 3024 return; 3025 } 3026 3027 /* Enabling GAM and SNPAVIC support */ 3028 for_each_iommu(iommu) { 3029 if (iommu_init_ga_log(iommu) || 3030 iommu_ga_log_enable(iommu)) 3031 return; 3032 3033 iommu_feature_enable(iommu, CONTROL_GAM_EN); 3034 if (amd_iommu_snp_en) 3035 iommu_feature_enable(iommu, CONTROL_SNPAVIC_EN); 3036 } 3037 3038 amd_iommu_irq_ops.capability |= (1 << IRQ_POSTING_CAP); 3039 pr_info("Virtual APIC enabled\n"); 3040 #endif 3041 } 3042 3043 static void disable_iommus(void) 3044 { 3045 struct amd_iommu *iommu; 3046 3047 for_each_iommu(iommu) 3048 iommu_disable(iommu); 3049 3050 #ifdef CONFIG_IRQ_REMAP 3051 if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) 3052 amd_iommu_irq_ops.capability &= ~(1 << IRQ_POSTING_CAP); 3053 #endif 3054 } 3055 3056 /* 3057 * Suspend/Resume support 3058 * disable suspend until real resume implemented 3059 */ 3060 3061 static void amd_iommu_resume(void *data) 3062 { 3063 struct amd_iommu *iommu; 3064 3065 for_each_iommu(iommu) 3066 iommu_apply_resume_quirks(iommu); 3067 3068 /* re-load the hardware */ 3069 for_each_iommu(iommu) 3070 early_enable_iommu(iommu); 3071 3072 amd_iommu_enable_interrupts(); 3073 } 3074 3075 static int amd_iommu_suspend(void *data) 3076 { 3077 /* disable IOMMUs to go out of the way for BIOS */ 3078 disable_iommus(); 3079 3080 return 0; 3081 } 3082 3083 static const struct syscore_ops amd_iommu_syscore_ops = { 3084 .suspend = amd_iommu_suspend, 3085 .resume = amd_iommu_resume, 3086 }; 3087 3088 static struct syscore amd_iommu_syscore = { 3089 .ops = &amd_iommu_syscore_ops, 3090 }; 3091 3092 static void __init free_iommu_resources(void) 3093 { 3094 free_iommu_all(); 3095 free_pci_segments(); 3096 } 3097 3098 /* SB IOAPIC is always on this device in AMD systems */ 3099 #define IOAPIC_SB_DEVID ((0x00 << 8) | PCI_DEVFN(0x14, 0)) 3100 3101 static bool __init check_ioapic_information(void) 3102 { 3103 const char *fw_bug = FW_BUG; 3104 bool ret, has_sb_ioapic; 3105 int idx; 3106 3107 has_sb_ioapic = false; 3108 ret = false; 3109 3110 /* 3111 * If we have map overrides on the kernel command line the 3112 * messages in this function might not describe firmware bugs 3113 * anymore - so be careful 3114 */ 3115 if (cmdline_maps) 3116 fw_bug = ""; 3117 3118 for (idx = 0; idx < nr_ioapics; idx++) { 3119 int devid, id = mpc_ioapic_id(idx); 3120 3121 devid = get_ioapic_devid(id); 3122 if (devid < 0) { 3123 pr_err("%s: IOAPIC[%d] not in IVRS table\n", 3124 fw_bug, id); 3125 ret = false; 3126 } else if (devid == IOAPIC_SB_DEVID) { 3127 has_sb_ioapic = true; 3128 ret = true; 3129 } 3130 } 3131 3132 if (!has_sb_ioapic) { 3133 /* 3134 * We expect the SB IOAPIC to be listed in the IVRS 3135 * table. The system timer is connected to the SB IOAPIC 3136 * and if we don't have it in the list the system will 3137 * panic at boot time. This situation usually happens 3138 * when the BIOS is buggy and provides us the wrong 3139 * device id for the IOAPIC in the system. 3140 */ 3141 pr_err("%s: No southbridge IOAPIC found\n", fw_bug); 3142 } 3143 3144 if (!ret) 3145 pr_err("Disabling interrupt remapping\n"); 3146 3147 return ret; 3148 } 3149 3150 static void __init free_dma_resources(void) 3151 { 3152 amd_iommu_pdom_id_destroy(); 3153 free_unity_maps(); 3154 } 3155 3156 static void __init ivinfo_init(void *ivrs) 3157 { 3158 amd_iommu_ivinfo = *((u32 *)(ivrs + IOMMU_IVINFO_OFFSET)); 3159 } 3160 3161 /* 3162 * This is the hardware init function for AMD IOMMU in the system. 3163 * This function is called either from amd_iommu_init or from the interrupt 3164 * remapping setup code. 3165 * 3166 * This function basically parses the ACPI table for AMD IOMMU (IVRS) 3167 * four times: 3168 * 3169 * 1 pass) Discover the most comprehensive IVHD type to use. 3170 * 3171 * 2 pass) Find the highest PCI device id the driver has to handle. 3172 * Upon this information the size of the data structures is 3173 * determined that needs to be allocated. 3174 * 3175 * 3 pass) Initialize the data structures just allocated with the 3176 * information in the ACPI table about available AMD IOMMUs 3177 * in the system. It also maps the PCI devices in the 3178 * system to specific IOMMUs 3179 * 3180 * 4 pass) After the basic data structures are allocated and 3181 * initialized we update them with information about memory 3182 * remapping requirements parsed out of the ACPI table in 3183 * this last pass. 3184 * 3185 * After everything is set up the IOMMUs are enabled and the necessary 3186 * hotplug and suspend notifiers are registered. 3187 */ 3188 static int __init early_amd_iommu_init(void) 3189 { 3190 struct acpi_table_header *ivrs_base; 3191 int ret; 3192 acpi_status status; 3193 u8 efr_hats; 3194 3195 if (!amd_iommu_detected) 3196 return -ENODEV; 3197 3198 status = acpi_get_table("IVRS", 0, &ivrs_base); 3199 if (status == AE_NOT_FOUND) 3200 return -ENODEV; 3201 else if (ACPI_FAILURE(status)) { 3202 const char *err = acpi_format_exception(status); 3203 pr_err("IVRS table error: %s\n", err); 3204 return -EINVAL; 3205 } 3206 3207 if (!boot_cpu_has(X86_FEATURE_CX16)) { 3208 pr_err("Failed to initialize. The CMPXCHG16B feature is required.\n"); 3209 ret = -EINVAL; 3210 goto out; 3211 } 3212 3213 /* 3214 * Validate checksum here so we don't need to do it when 3215 * we actually parse the table 3216 */ 3217 ret = check_ivrs_checksum(ivrs_base); 3218 if (ret) 3219 goto out; 3220 3221 ivinfo_init(ivrs_base); 3222 3223 amd_iommu_target_ivhd_type = get_highest_supported_ivhd_type(ivrs_base); 3224 DUMP_printk("Using IVHD type %#x\n", amd_iommu_target_ivhd_type); 3225 3226 /* 3227 * now the data structures are allocated and basically initialized 3228 * start the real acpi table scan 3229 */ 3230 ret = init_iommu_all(ivrs_base); 3231 if (ret) 3232 goto out; 3233 3234 /* 5 level guest page table */ 3235 if (cpu_feature_enabled(X86_FEATURE_LA57) && 3236 FIELD_GET(FEATURE_GATS, amd_iommu_efr) == GUEST_PGTABLE_5_LEVEL) 3237 amd_iommu_gpt_level = PAGE_MODE_5_LEVEL; 3238 3239 efr_hats = FIELD_GET(FEATURE_HATS, amd_iommu_efr); 3240 if (efr_hats != 0x3) { 3241 /* 3242 * efr[HATS] bits specify the maximum host translation level 3243 * supported, with LEVEL 4 being initial max level. 3244 */ 3245 amd_iommu_hpt_level = efr_hats + PAGE_MODE_4_LEVEL; 3246 } else { 3247 pr_warn_once(FW_BUG "Disable host address translation due to invalid translation level (%#x).\n", 3248 efr_hats); 3249 amd_iommu_hatdis = true; 3250 } 3251 3252 if (amd_iommu_pgtable == PD_MODE_V2) { 3253 if (!amd_iommu_v2_pgtbl_supported()) { 3254 pr_warn("Cannot enable v2 page table for DMA-API. Fallback to v1.\n"); 3255 amd_iommu_pgtable = PD_MODE_V1; 3256 } 3257 } 3258 3259 if (amd_iommu_hatdis) { 3260 /* 3261 * Host (v1) page table is not available. Attempt to use 3262 * Guest (v2) page table. 3263 */ 3264 if (amd_iommu_v2_pgtbl_supported()) 3265 amd_iommu_pgtable = PD_MODE_V2; 3266 else 3267 amd_iommu_pgtable = PD_MODE_NONE; 3268 } 3269 3270 /* Disable any previously enabled IOMMUs */ 3271 if (!is_kdump_kernel() || amd_iommu_disabled) 3272 disable_iommus(); 3273 3274 if (amd_iommu_irq_remap) 3275 amd_iommu_irq_remap = check_ioapic_information(); 3276 3277 if (amd_iommu_irq_remap) { 3278 struct amd_iommu_pci_seg *pci_seg; 3279 ret = -ENOMEM; 3280 for_each_pci_segment(pci_seg) { 3281 if (alloc_irq_lookup_table(pci_seg)) 3282 goto out; 3283 } 3284 } 3285 3286 ret = init_memory_definitions(ivrs_base); 3287 if (ret) 3288 goto out; 3289 3290 /* init the device table */ 3291 init_device_table(); 3292 3293 out: 3294 /* Don't leak any ACPI memory */ 3295 acpi_put_table(ivrs_base); 3296 3297 return ret; 3298 } 3299 3300 static int amd_iommu_enable_interrupts(void) 3301 { 3302 struct amd_iommu *iommu; 3303 int ret = 0; 3304 3305 for_each_iommu(iommu) { 3306 ret = iommu_init_irq(iommu); 3307 if (ret) 3308 goto out; 3309 } 3310 3311 /* 3312 * Interrupt handler is ready to process interrupts. Enable 3313 * PPR and GA log interrupt for all IOMMUs. 3314 */ 3315 enable_iommus_vapic(); 3316 enable_iommus_ppr(); 3317 3318 out: 3319 return ret; 3320 } 3321 3322 static bool __init detect_ivrs(void) 3323 { 3324 struct acpi_table_header *ivrs_base; 3325 acpi_status status; 3326 int i; 3327 3328 status = acpi_get_table("IVRS", 0, &ivrs_base); 3329 if (status == AE_NOT_FOUND) 3330 return false; 3331 else if (ACPI_FAILURE(status)) { 3332 const char *err = acpi_format_exception(status); 3333 pr_err("IVRS table error: %s\n", err); 3334 return false; 3335 } 3336 3337 acpi_put_table(ivrs_base); 3338 3339 if (amd_iommu_force_enable) 3340 goto out; 3341 3342 /* Don't use IOMMU if there is Stoney Ridge graphics */ 3343 for (i = 0; i < 32; i++) { 3344 u32 pci_id; 3345 3346 pci_id = read_pci_config(0, i, 0, 0); 3347 if ((pci_id & 0xffff) == 0x1002 && (pci_id >> 16) == 0x98e4) { 3348 pr_info("Disable IOMMU on Stoney Ridge\n"); 3349 return false; 3350 } 3351 } 3352 3353 out: 3354 /* Make sure ACS will be enabled during PCI probe */ 3355 pci_request_acs(); 3356 3357 return true; 3358 } 3359 3360 static __init void iommu_snp_enable(void) 3361 { 3362 #ifdef CONFIG_KVM_AMD_SEV 3363 if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) 3364 return; 3365 /* 3366 * The SNP support requires that IOMMU must be enabled, and is 3367 * configured with V1 page table (DTE[Mode] = 0 is not supported). 3368 */ 3369 if (no_iommu || iommu_default_passthrough()) { 3370 pr_warn("SNP: IOMMU disabled or configured in passthrough mode, SNP cannot be supported.\n"); 3371 goto disable_snp; 3372 } 3373 3374 if (amd_iommu_pgtable != PD_MODE_V1) { 3375 pr_warn("SNP: IOMMU is configured with V2 page table mode, SNP cannot be supported.\n"); 3376 goto disable_snp; 3377 } 3378 3379 amd_iommu_snp_en = check_feature(FEATURE_SNP); 3380 if (!amd_iommu_snp_en) { 3381 pr_warn("SNP: IOMMU SNP feature not enabled, SNP cannot be supported.\n"); 3382 goto disable_snp; 3383 } 3384 3385 /* 3386 * Enable host SNP support once SNP support is checked on IOMMU. 3387 */ 3388 if (snp_rmptable_init()) { 3389 pr_warn("SNP: RMP initialization failed, SNP cannot be supported.\n"); 3390 goto disable_snp; 3391 } 3392 3393 pr_info("IOMMU SNP support enabled.\n"); 3394 return; 3395 3396 disable_snp: 3397 cc_platform_clear(CC_ATTR_HOST_SEV_SNP); 3398 #endif 3399 } 3400 3401 /**************************************************************************** 3402 * 3403 * AMD IOMMU Initialization State Machine 3404 * 3405 ****************************************************************************/ 3406 3407 static int __init state_next(void) 3408 { 3409 int ret = 0; 3410 3411 switch (init_state) { 3412 case IOMMU_START_STATE: 3413 if (!detect_ivrs()) { 3414 init_state = IOMMU_NOT_FOUND; 3415 ret = -ENODEV; 3416 } else { 3417 init_state = IOMMU_IVRS_DETECTED; 3418 } 3419 break; 3420 case IOMMU_IVRS_DETECTED: 3421 if (amd_iommu_disabled) { 3422 init_state = IOMMU_CMDLINE_DISABLED; 3423 ret = -EINVAL; 3424 } else { 3425 ret = early_amd_iommu_init(); 3426 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED; 3427 } 3428 break; 3429 case IOMMU_ACPI_FINISHED: 3430 early_enable_iommus(); 3431 x86_platform.iommu_shutdown = disable_iommus; 3432 init_state = IOMMU_ENABLED; 3433 break; 3434 case IOMMU_ENABLED: 3435 register_syscore(&amd_iommu_syscore); 3436 iommu_snp_enable(); 3437 ret = amd_iommu_init_pci(); 3438 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_PCI_INIT; 3439 break; 3440 case IOMMU_PCI_INIT: 3441 ret = amd_iommu_enable_interrupts(); 3442 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_INTERRUPTS_EN; 3443 break; 3444 case IOMMU_INTERRUPTS_EN: 3445 init_state = IOMMU_INITIALIZED; 3446 break; 3447 case IOMMU_INITIALIZED: 3448 /* Nothing to do */ 3449 break; 3450 case IOMMU_NOT_FOUND: 3451 case IOMMU_INIT_ERROR: 3452 case IOMMU_CMDLINE_DISABLED: 3453 /* Error states => do nothing */ 3454 ret = -EINVAL; 3455 break; 3456 default: 3457 /* Unknown state */ 3458 BUG(); 3459 } 3460 3461 if (ret) { 3462 free_dma_resources(); 3463 if (!irq_remapping_enabled) { 3464 disable_iommus(); 3465 free_iommu_resources(); 3466 } else { 3467 struct amd_iommu *iommu; 3468 struct amd_iommu_pci_seg *pci_seg; 3469 3470 for_each_pci_segment(pci_seg) 3471 uninit_device_table_dma(pci_seg); 3472 3473 for_each_iommu(iommu) 3474 amd_iommu_flush_all_caches(iommu); 3475 } 3476 } 3477 return ret; 3478 } 3479 3480 static int __init iommu_go_to_state(enum iommu_init_state state) 3481 { 3482 int ret = -EINVAL; 3483 3484 while (init_state != state) { 3485 if (init_state == IOMMU_NOT_FOUND || 3486 init_state == IOMMU_INIT_ERROR || 3487 init_state == IOMMU_CMDLINE_DISABLED) 3488 break; 3489 ret = state_next(); 3490 } 3491 3492 /* 3493 * SNP platform initilazation requires IOMMUs to be fully configured. 3494 * If the SNP support on IOMMUs has NOT been checked, simply mark SNP 3495 * as unsupported. If the SNP support on IOMMUs has been checked and 3496 * host SNP support enabled but RMP enforcement has not been enabled 3497 * in IOMMUs, then the system is in a half-baked state, but can limp 3498 * along as all memory should be Hypervisor-Owned in the RMP. WARN, 3499 * but leave SNP as "supported" to avoid confusing the kernel. 3500 */ 3501 if (ret && cc_platform_has(CC_ATTR_HOST_SEV_SNP) && 3502 !WARN_ON_ONCE(amd_iommu_snp_en)) 3503 cc_platform_clear(CC_ATTR_HOST_SEV_SNP); 3504 3505 return ret; 3506 } 3507 3508 #ifdef CONFIG_IRQ_REMAP 3509 int __init amd_iommu_prepare(void) 3510 { 3511 int ret; 3512 3513 amd_iommu_irq_remap = true; 3514 3515 ret = iommu_go_to_state(IOMMU_ACPI_FINISHED); 3516 if (ret) { 3517 amd_iommu_irq_remap = false; 3518 return ret; 3519 } 3520 3521 return amd_iommu_irq_remap ? 0 : -ENODEV; 3522 } 3523 3524 int __init amd_iommu_enable(void) 3525 { 3526 int ret; 3527 3528 ret = iommu_go_to_state(IOMMU_ENABLED); 3529 if (ret) 3530 return ret; 3531 3532 irq_remapping_enabled = 1; 3533 return amd_iommu_xt_mode; 3534 } 3535 3536 void amd_iommu_disable(void) 3537 { 3538 amd_iommu_suspend(NULL); 3539 } 3540 3541 int amd_iommu_reenable(int mode) 3542 { 3543 amd_iommu_resume(NULL); 3544 3545 return 0; 3546 } 3547 3548 int amd_iommu_enable_faulting(unsigned int cpu) 3549 { 3550 /* We enable MSI later when PCI is initialized */ 3551 return 0; 3552 } 3553 #endif 3554 3555 /* 3556 * This is the core init function for AMD IOMMU hardware in the system. 3557 * This function is called from the generic x86 DMA layer initialization 3558 * code. 3559 */ 3560 static int __init amd_iommu_init(void) 3561 { 3562 int ret; 3563 3564 ret = iommu_go_to_state(IOMMU_INITIALIZED); 3565 #ifdef CONFIG_GART_IOMMU 3566 if (ret && list_empty(&amd_iommu_list)) { 3567 /* 3568 * We failed to initialize the AMD IOMMU - try fallback 3569 * to GART if possible. 3570 */ 3571 gart_iommu_init(); 3572 } 3573 #endif 3574 3575 if (!ret) 3576 amd_iommu_debugfs_setup(); 3577 3578 return ret; 3579 } 3580 3581 static bool amd_iommu_sme_check(void) 3582 { 3583 if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT) || 3584 (boot_cpu_data.x86 != 0x17)) 3585 return true; 3586 3587 /* For Fam17h, a specific level of support is required */ 3588 if (boot_cpu_data.microcode >= 0x08001205) 3589 return true; 3590 3591 if ((boot_cpu_data.microcode >= 0x08001126) && 3592 (boot_cpu_data.microcode <= 0x080011ff)) 3593 return true; 3594 3595 pr_notice("IOMMU not currently supported when SME is active\n"); 3596 3597 return false; 3598 } 3599 3600 /**************************************************************************** 3601 * 3602 * Early detect code. This code runs at IOMMU detection time in the DMA 3603 * layer. It just looks if there is an IVRS ACPI table to detect AMD 3604 * IOMMUs 3605 * 3606 ****************************************************************************/ 3607 void __init amd_iommu_detect(void) 3608 { 3609 int ret; 3610 3611 if (no_iommu || (iommu_detected && !gart_iommu_aperture)) 3612 goto disable_snp; 3613 3614 if (!amd_iommu_sme_check()) 3615 goto disable_snp; 3616 3617 ret = iommu_go_to_state(IOMMU_IVRS_DETECTED); 3618 if (ret) 3619 goto disable_snp; 3620 3621 amd_iommu_detected = true; 3622 iommu_detected = 1; 3623 x86_init.iommu.iommu_init = amd_iommu_init; 3624 return; 3625 3626 disable_snp: 3627 if (cc_platform_has(CC_ATTR_HOST_SEV_SNP)) 3628 cc_platform_clear(CC_ATTR_HOST_SEV_SNP); 3629 } 3630 3631 /**************************************************************************** 3632 * 3633 * Parsing functions for the AMD IOMMU specific kernel command line 3634 * options. 3635 * 3636 ****************************************************************************/ 3637 3638 static int __init parse_amd_iommu_dump(char *str) 3639 { 3640 amd_iommu_dump = true; 3641 3642 return 1; 3643 } 3644 3645 static int __init parse_amd_iommu_intr(char *str) 3646 { 3647 for (; *str; ++str) { 3648 if (strncmp(str, "legacy", 6) == 0) { 3649 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA; 3650 break; 3651 } 3652 if (strncmp(str, "vapic", 5) == 0) { 3653 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC; 3654 break; 3655 } 3656 } 3657 return 1; 3658 } 3659 3660 static int __init parse_amd_iommu_options(char *str) 3661 { 3662 if (!str) 3663 return -EINVAL; 3664 3665 while (*str) { 3666 if (strncmp(str, "fullflush", 9) == 0) { 3667 pr_warn("amd_iommu=fullflush deprecated; use iommu.strict=1 instead\n"); 3668 iommu_set_dma_strict(); 3669 } else if (strncmp(str, "force_enable", 12) == 0) { 3670 amd_iommu_force_enable = true; 3671 } else if (strncmp(str, "off", 3) == 0) { 3672 amd_iommu_disabled = true; 3673 } else if (strncmp(str, "force_isolation", 15) == 0) { 3674 amd_iommu_force_isolation = true; 3675 } else if (strncmp(str, "pgtbl_v1", 8) == 0) { 3676 amd_iommu_pgtable = PD_MODE_V1; 3677 } else if (strncmp(str, "pgtbl_v2", 8) == 0) { 3678 amd_iommu_pgtable = PD_MODE_V2; 3679 } else if (strncmp(str, "irtcachedis", 11) == 0) { 3680 amd_iommu_irtcachedis = true; 3681 } else if (strncmp(str, "nohugepages", 11) == 0) { 3682 pr_info("Restricting V1 page-sizes to 4KiB"); 3683 amd_iommu_pgsize_bitmap = AMD_IOMMU_PGSIZES_4K; 3684 } else if (strncmp(str, "v2_pgsizes_only", 15) == 0) { 3685 pr_info("Restricting V1 page-sizes to 4KiB/2MiB/1GiB"); 3686 amd_iommu_pgsize_bitmap = AMD_IOMMU_PGSIZES_V2; 3687 } else { 3688 pr_notice("Unknown option - '%s'\n", str); 3689 } 3690 3691 str += strcspn(str, ","); 3692 while (*str == ',') 3693 str++; 3694 } 3695 3696 return 1; 3697 } 3698 3699 static int __init parse_ivrs_ioapic(char *str) 3700 { 3701 u32 seg = 0, bus, dev, fn; 3702 int id, i; 3703 u32 devid; 3704 3705 if (sscanf(str, "=%d@%x:%x.%x", &id, &bus, &dev, &fn) == 4 || 3706 sscanf(str, "=%d@%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5) 3707 goto found; 3708 3709 if (sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn) == 4 || 3710 sscanf(str, "[%d]=%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5) { 3711 pr_warn("ivrs_ioapic%s option format deprecated; use ivrs_ioapic=%d@%04x:%02x:%02x.%d instead\n", 3712 str, id, seg, bus, dev, fn); 3713 goto found; 3714 } 3715 3716 pr_err("Invalid command line: ivrs_ioapic%s\n", str); 3717 return 1; 3718 3719 found: 3720 if (early_ioapic_map_size == EARLY_MAP_SIZE) { 3721 pr_err("Early IOAPIC map overflow - ignoring ivrs_ioapic%s\n", 3722 str); 3723 return 1; 3724 } 3725 3726 devid = IVRS_GET_SBDF_ID(seg, bus, dev, fn); 3727 3728 cmdline_maps = true; 3729 i = early_ioapic_map_size++; 3730 early_ioapic_map[i].id = id; 3731 early_ioapic_map[i].devid = devid; 3732 early_ioapic_map[i].cmd_line = true; 3733 3734 return 1; 3735 } 3736 3737 static int __init parse_ivrs_hpet(char *str) 3738 { 3739 u32 seg = 0, bus, dev, fn; 3740 int id, i; 3741 u32 devid; 3742 3743 if (sscanf(str, "=%d@%x:%x.%x", &id, &bus, &dev, &fn) == 4 || 3744 sscanf(str, "=%d@%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5) 3745 goto found; 3746 3747 if (sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn) == 4 || 3748 sscanf(str, "[%d]=%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5) { 3749 pr_warn("ivrs_hpet%s option format deprecated; use ivrs_hpet=%d@%04x:%02x:%02x.%d instead\n", 3750 str, id, seg, bus, dev, fn); 3751 goto found; 3752 } 3753 3754 pr_err("Invalid command line: ivrs_hpet%s\n", str); 3755 return 1; 3756 3757 found: 3758 if (early_hpet_map_size == EARLY_MAP_SIZE) { 3759 pr_err("Early HPET map overflow - ignoring ivrs_hpet%s\n", 3760 str); 3761 return 1; 3762 } 3763 3764 devid = IVRS_GET_SBDF_ID(seg, bus, dev, fn); 3765 3766 cmdline_maps = true; 3767 i = early_hpet_map_size++; 3768 early_hpet_map[i].id = id; 3769 early_hpet_map[i].devid = devid; 3770 early_hpet_map[i].cmd_line = true; 3771 3772 return 1; 3773 } 3774 3775 #define ACPIID_LEN (ACPIHID_UID_LEN + ACPIHID_HID_LEN) 3776 3777 static int __init parse_ivrs_acpihid(char *str) 3778 { 3779 u32 seg = 0, bus, dev, fn; 3780 char *hid, *uid, *p, *addr; 3781 char acpiid[ACPIID_LEN + 1] = { }; /* size with NULL terminator */ 3782 int i; 3783 3784 addr = strchr(str, '@'); 3785 if (!addr) { 3786 addr = strchr(str, '='); 3787 if (!addr) 3788 goto not_found; 3789 3790 ++addr; 3791 3792 if (strlen(addr) > ACPIID_LEN) 3793 goto not_found; 3794 3795 if (sscanf(str, "[%x:%x.%x]=%s", &bus, &dev, &fn, acpiid) == 4 || 3796 sscanf(str, "[%x:%x:%x.%x]=%s", &seg, &bus, &dev, &fn, acpiid) == 5) { 3797 pr_warn("ivrs_acpihid%s option format deprecated; use ivrs_acpihid=%s@%04x:%02x:%02x.%d instead\n", 3798 str, acpiid, seg, bus, dev, fn); 3799 goto found; 3800 } 3801 goto not_found; 3802 } 3803 3804 /* We have the '@', make it the terminator to get just the acpiid */ 3805 *addr++ = 0; 3806 3807 if (strlen(str) > ACPIID_LEN) 3808 goto not_found; 3809 3810 if (sscanf(str, "=%s", acpiid) != 1) 3811 goto not_found; 3812 3813 if (sscanf(addr, "%x:%x.%x", &bus, &dev, &fn) == 3 || 3814 sscanf(addr, "%x:%x:%x.%x", &seg, &bus, &dev, &fn) == 4) 3815 goto found; 3816 3817 not_found: 3818 pr_err("Invalid command line: ivrs_acpihid%s\n", str); 3819 return 1; 3820 3821 found: 3822 p = acpiid; 3823 hid = strsep(&p, ":"); 3824 uid = p; 3825 3826 if (!hid || !(*hid) || !uid) { 3827 pr_err("Invalid command line: hid or uid\n"); 3828 return 1; 3829 } 3830 3831 /* 3832 * Ignore leading zeroes after ':', so e.g., AMDI0095:00 3833 * will match AMDI0095:0 in the second strcmp in acpi_dev_hid_uid_match 3834 */ 3835 while (*uid == '0' && *(uid + 1)) 3836 uid++; 3837 3838 if (strlen(hid) >= ACPIHID_HID_LEN) { 3839 pr_err("Invalid command line: hid is too long\n"); 3840 return 1; 3841 } else if (strlen(uid) >= ACPIHID_UID_LEN) { 3842 pr_err("Invalid command line: uid is too long\n"); 3843 return 1; 3844 } 3845 3846 i = early_acpihid_map_size++; 3847 memcpy(early_acpihid_map[i].hid, hid, strlen(hid)); 3848 memcpy(early_acpihid_map[i].uid, uid, strlen(uid)); 3849 early_acpihid_map[i].devid = IVRS_GET_SBDF_ID(seg, bus, dev, fn); 3850 early_acpihid_map[i].cmd_line = true; 3851 3852 return 1; 3853 } 3854 3855 __setup("amd_iommu_dump", parse_amd_iommu_dump); 3856 __setup("amd_iommu=", parse_amd_iommu_options); 3857 __setup("amd_iommu_intr=", parse_amd_iommu_intr); 3858 __setup("ivrs_ioapic", parse_ivrs_ioapic); 3859 __setup("ivrs_hpet", parse_ivrs_hpet); 3860 __setup("ivrs_acpihid", parse_ivrs_acpihid); 3861 3862 bool amd_iommu_pasid_supported(void) 3863 { 3864 /* CPU page table size should match IOMMU guest page table size */ 3865 if (cpu_feature_enabled(X86_FEATURE_LA57) && 3866 amd_iommu_gpt_level != PAGE_MODE_5_LEVEL) 3867 return false; 3868 3869 /* 3870 * Since DTE[Mode]=0 is prohibited on SNP-enabled system 3871 * (i.e. EFR[SNPSup]=1), IOMMUv2 page table cannot be used without 3872 * setting up IOMMUv1 page table. 3873 */ 3874 return amd_iommu_gt_ppr_supported() && !amd_iommu_snp_en; 3875 } 3876 3877 struct amd_iommu *get_amd_iommu(unsigned int idx) 3878 { 3879 unsigned int i = 0; 3880 struct amd_iommu *iommu; 3881 3882 for_each_iommu(iommu) 3883 if (i++ == idx) 3884 return iommu; 3885 return NULL; 3886 } 3887 3888 /**************************************************************************** 3889 * 3890 * IOMMU EFR Performance Counter support functionality. This code allows 3891 * access to the IOMMU PC functionality. 3892 * 3893 ****************************************************************************/ 3894 3895 u8 amd_iommu_pc_get_max_banks(unsigned int idx) 3896 { 3897 struct amd_iommu *iommu = get_amd_iommu(idx); 3898 3899 if (iommu) 3900 return iommu->max_banks; 3901 3902 return 0; 3903 } 3904 3905 bool amd_iommu_pc_supported(void) 3906 { 3907 return amd_iommu_pc_present; 3908 } 3909 3910 u8 amd_iommu_pc_get_max_counters(unsigned int idx) 3911 { 3912 struct amd_iommu *iommu = get_amd_iommu(idx); 3913 3914 if (iommu) 3915 return iommu->max_counters; 3916 3917 return 0; 3918 } 3919 3920 static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, 3921 u8 fxn, u64 *value, bool is_write) 3922 { 3923 u32 offset; 3924 u32 max_offset_lim; 3925 3926 /* Make sure the IOMMU PC resource is available */ 3927 if (!amd_iommu_pc_present) 3928 return -ENODEV; 3929 3930 /* Check for valid iommu and pc register indexing */ 3931 if (WARN_ON(!iommu || (fxn > 0x28) || (fxn & 7))) 3932 return -ENODEV; 3933 3934 offset = (u32)(((0x40 | bank) << 12) | (cntr << 8) | fxn); 3935 3936 /* Limit the offset to the hw defined mmio region aperture */ 3937 max_offset_lim = (u32)(((0x40 | iommu->max_banks) << 12) | 3938 (iommu->max_counters << 8) | 0x28); 3939 if ((offset < MMIO_CNTR_REG_OFFSET) || 3940 (offset > max_offset_lim)) 3941 return -EINVAL; 3942 3943 if (is_write) { 3944 u64 val = *value & GENMASK_ULL(47, 0); 3945 3946 writel((u32)val, iommu->mmio_base + offset); 3947 writel((val >> 32), iommu->mmio_base + offset + 4); 3948 } else { 3949 *value = readl(iommu->mmio_base + offset + 4); 3950 *value <<= 32; 3951 *value |= readl(iommu->mmio_base + offset); 3952 *value &= GENMASK_ULL(47, 0); 3953 } 3954 3955 return 0; 3956 } 3957 3958 int amd_iommu_pc_get_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value) 3959 { 3960 if (!iommu) 3961 return -EINVAL; 3962 3963 return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, false); 3964 } 3965 3966 int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value) 3967 { 3968 if (!iommu) 3969 return -EINVAL; 3970 3971 return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, true); 3972 } 3973 3974 #ifdef CONFIG_KVM_AMD_SEV 3975 static int iommu_page_make_shared(void *page) 3976 { 3977 unsigned long paddr, pfn; 3978 3979 paddr = iommu_virt_to_phys(page); 3980 /* Cbit maybe set in the paddr */ 3981 pfn = __sme_clr(paddr) >> PAGE_SHIFT; 3982 3983 if (!(pfn % PTRS_PER_PMD)) { 3984 int ret, level; 3985 bool assigned; 3986 3987 ret = snp_lookup_rmpentry(pfn, &assigned, &level); 3988 if (ret) { 3989 pr_warn("IOMMU PFN %lx RMP lookup failed, ret %d\n", pfn, ret); 3990 return ret; 3991 } 3992 3993 if (!assigned) { 3994 pr_warn("IOMMU PFN %lx not assigned in RMP table\n", pfn); 3995 return -EINVAL; 3996 } 3997 3998 if (level > PG_LEVEL_4K) { 3999 ret = psmash(pfn); 4000 if (!ret) 4001 goto done; 4002 4003 pr_warn("PSMASH failed for IOMMU PFN %lx huge RMP entry, ret: %d, level: %d\n", 4004 pfn, ret, level); 4005 return ret; 4006 } 4007 } 4008 4009 done: 4010 return rmp_make_shared(pfn, PG_LEVEL_4K); 4011 } 4012 4013 static int iommu_make_shared(void *va, size_t size) 4014 { 4015 void *page; 4016 int ret; 4017 4018 if (!va) 4019 return 0; 4020 4021 for (page = va; page < (va + size); page += PAGE_SIZE) { 4022 ret = iommu_page_make_shared(page); 4023 if (ret) 4024 return ret; 4025 } 4026 4027 return 0; 4028 } 4029 4030 int amd_iommu_snp_disable(void) 4031 { 4032 struct amd_iommu *iommu; 4033 int ret; 4034 4035 if (!amd_iommu_snp_en) 4036 return 0; 4037 4038 for_each_iommu(iommu) { 4039 ret = iommu_make_shared(iommu->evt_buf, EVT_BUFFER_SIZE); 4040 if (ret) 4041 return ret; 4042 4043 ret = iommu_make_shared(iommu->ppr_log, PPR_LOG_SIZE); 4044 if (ret) 4045 return ret; 4046 4047 ret = iommu_make_shared((void *)iommu->cmd_sem, PAGE_SIZE); 4048 if (ret) 4049 return ret; 4050 } 4051 4052 return 0; 4053 } 4054 EXPORT_SYMBOL_GPL(amd_iommu_snp_disable); 4055 4056 bool amd_iommu_sev_tio_supported(void) 4057 { 4058 return check_feature2(FEATURE_SEVSNPIO_SUP); 4059 } 4060 EXPORT_SYMBOL_GPL(amd_iommu_sev_tio_supported); 4061 #endif 4062