1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * AMD SVM-SEV Host Support. 4 * 5 * Copyright (C) 2023 Advanced Micro Devices, Inc. 6 * 7 * Author: Ashish Kalra <ashish.kalra@amd.com> 8 * 9 */ 10 11 #include <linux/cc_platform.h> 12 #include <linux/printk.h> 13 #include <linux/mm_types.h> 14 #include <linux/set_memory.h> 15 #include <linux/memblock.h> 16 #include <linux/kernel.h> 17 #include <linux/mm.h> 18 #include <linux/cpumask.h> 19 #include <linux/iommu.h> 20 #include <linux/amd-iommu.h> 21 22 #include <asm/sev.h> 23 #include <asm/processor.h> 24 #include <asm/setup.h> 25 #include <asm/svm.h> 26 #include <asm/smp.h> 27 #include <asm/cpu.h> 28 #include <asm/apic.h> 29 #include <asm/cpuid.h> 30 #include <asm/cmdline.h> 31 #include <asm/iommu.h> 32 33 /* 34 * The RMP entry format is not architectural. The format is defined in PPR 35 * Family 19h Model 01h, Rev B1 processor. 36 */ 37 struct rmpentry { 38 union { 39 struct { 40 u64 assigned : 1, 41 pagesize : 1, 42 immutable : 1, 43 rsvd1 : 9, 44 gpa : 39, 45 asid : 10, 46 vmsa : 1, 47 validated : 1, 48 rsvd2 : 1; 49 }; 50 u64 lo; 51 }; 52 u64 hi; 53 } __packed; 54 55 /* 56 * The first 16KB from the RMP_BASE is used by the processor for the 57 * bookkeeping, the range needs to be added during the RMP entry lookup. 58 */ 59 #define RMPTABLE_CPU_BOOKKEEPING_SZ 0x4000 60 61 /* Mask to apply to a PFN to get the first PFN of a 2MB page */ 62 #define PFN_PMD_MASK GENMASK_ULL(63, PMD_SHIFT - PAGE_SHIFT) 63 64 static u64 probed_rmp_base, probed_rmp_size; 65 static struct rmpentry *rmptable __ro_after_init; 66 static u64 rmptable_max_pfn __ro_after_init; 67 68 static LIST_HEAD(snp_leaked_pages_list); 69 static DEFINE_SPINLOCK(snp_leaked_pages_list_lock); 70 71 static unsigned long snp_nr_leaked_pages; 72 73 #undef pr_fmt 74 #define pr_fmt(fmt) "SEV-SNP: " fmt 75 76 static int __mfd_enable(unsigned int cpu) 77 { 78 u64 val; 79 80 if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) 81 return 0; 82 83 rdmsrl(MSR_AMD64_SYSCFG, val); 84 85 val |= MSR_AMD64_SYSCFG_MFDM; 86 87 wrmsrl(MSR_AMD64_SYSCFG, val); 88 89 return 0; 90 } 91 92 static __init void mfd_enable(void *arg) 93 { 94 __mfd_enable(smp_processor_id()); 95 } 96 97 static int __snp_enable(unsigned int cpu) 98 { 99 u64 val; 100 101 if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) 102 return 0; 103 104 rdmsrl(MSR_AMD64_SYSCFG, val); 105 106 val |= MSR_AMD64_SYSCFG_SNP_EN; 107 val |= MSR_AMD64_SYSCFG_SNP_VMPL_EN; 108 109 wrmsrl(MSR_AMD64_SYSCFG, val); 110 111 return 0; 112 } 113 114 static __init void snp_enable(void *arg) 115 { 116 __snp_enable(smp_processor_id()); 117 } 118 119 #define RMP_ADDR_MASK GENMASK_ULL(51, 13) 120 121 bool snp_probe_rmptable_info(void) 122 { 123 u64 rmp_sz, rmp_base, rmp_end; 124 125 rdmsrl(MSR_AMD64_RMP_BASE, rmp_base); 126 rdmsrl(MSR_AMD64_RMP_END, rmp_end); 127 128 if (!(rmp_base & RMP_ADDR_MASK) || !(rmp_end & RMP_ADDR_MASK)) { 129 pr_err("Memory for the RMP table has not been reserved by BIOS\n"); 130 return false; 131 } 132 133 if (rmp_base > rmp_end) { 134 pr_err("RMP configuration not valid: base=%#llx, end=%#llx\n", rmp_base, rmp_end); 135 return false; 136 } 137 138 rmp_sz = rmp_end - rmp_base + 1; 139 140 probed_rmp_base = rmp_base; 141 probed_rmp_size = rmp_sz; 142 143 pr_info("RMP table physical range [0x%016llx - 0x%016llx]\n", 144 rmp_base, rmp_end); 145 146 return true; 147 } 148 149 static void __init __snp_fixup_e820_tables(u64 pa) 150 { 151 if (IS_ALIGNED(pa, PMD_SIZE)) 152 return; 153 154 /* 155 * Handle cases where the RMP table placement by the BIOS is not 156 * 2M aligned and the kexec kernel could try to allocate 157 * from within that chunk which then causes a fatal RMP fault. 158 * 159 * The e820_table needs to be updated as it is converted to 160 * kernel memory resources and used by KEXEC_FILE_LOAD syscall 161 * to load kexec segments. 162 * 163 * The e820_table_firmware needs to be updated as it is exposed 164 * to sysfs and used by the KEXEC_LOAD syscall to load kexec 165 * segments. 166 * 167 * The e820_table_kexec needs to be updated as it passed to 168 * the kexec-ed kernel. 169 */ 170 pa = ALIGN_DOWN(pa, PMD_SIZE); 171 if (e820__mapped_any(pa, pa + PMD_SIZE, E820_TYPE_RAM)) { 172 pr_info("Reserving start/end of RMP table on a 2MB boundary [0x%016llx]\n", pa); 173 e820__range_update(pa, PMD_SIZE, E820_TYPE_RAM, E820_TYPE_RESERVED); 174 e820__range_update_table(e820_table_kexec, pa, PMD_SIZE, E820_TYPE_RAM, E820_TYPE_RESERVED); 175 e820__range_update_table(e820_table_firmware, pa, PMD_SIZE, E820_TYPE_RAM, E820_TYPE_RESERVED); 176 } 177 } 178 179 void __init snp_fixup_e820_tables(void) 180 { 181 __snp_fixup_e820_tables(probed_rmp_base); 182 __snp_fixup_e820_tables(probed_rmp_base + probed_rmp_size); 183 } 184 185 /* 186 * Do the necessary preparations which are verified by the firmware as 187 * described in the SNP_INIT_EX firmware command description in the SNP 188 * firmware ABI spec. 189 */ 190 static int __init snp_rmptable_init(void) 191 { 192 u64 max_rmp_pfn, calc_rmp_sz, rmptable_size, rmp_end, val; 193 void *rmptable_start; 194 195 if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) 196 return 0; 197 198 if (!amd_iommu_snp_en) 199 goto nosnp; 200 201 if (!probed_rmp_size) 202 goto nosnp; 203 204 rmp_end = probed_rmp_base + probed_rmp_size - 1; 205 206 /* 207 * Calculate the amount the memory that must be reserved by the BIOS to 208 * address the whole RAM, including the bookkeeping area. The RMP itself 209 * must also be covered. 210 */ 211 max_rmp_pfn = max_pfn; 212 if (PFN_UP(rmp_end) > max_pfn) 213 max_rmp_pfn = PFN_UP(rmp_end); 214 215 calc_rmp_sz = (max_rmp_pfn << 4) + RMPTABLE_CPU_BOOKKEEPING_SZ; 216 if (calc_rmp_sz > probed_rmp_size) { 217 pr_err("Memory reserved for the RMP table does not cover full system RAM (expected 0x%llx got 0x%llx)\n", 218 calc_rmp_sz, probed_rmp_size); 219 goto nosnp; 220 } 221 222 rmptable_start = memremap(probed_rmp_base, probed_rmp_size, MEMREMAP_WB); 223 if (!rmptable_start) { 224 pr_err("Failed to map RMP table\n"); 225 goto nosnp; 226 } 227 228 /* 229 * Check if SEV-SNP is already enabled, this can happen in case of 230 * kexec boot. 231 */ 232 rdmsrl(MSR_AMD64_SYSCFG, val); 233 if (val & MSR_AMD64_SYSCFG_SNP_EN) 234 goto skip_enable; 235 236 memset(rmptable_start, 0, probed_rmp_size); 237 238 /* Flush the caches to ensure that data is written before SNP is enabled. */ 239 wbinvd_on_all_cpus(); 240 241 /* MtrrFixDramModEn must be enabled on all the CPUs prior to enabling SNP. */ 242 on_each_cpu(mfd_enable, NULL, 1); 243 244 on_each_cpu(snp_enable, NULL, 1); 245 246 skip_enable: 247 rmptable_start += RMPTABLE_CPU_BOOKKEEPING_SZ; 248 rmptable_size = probed_rmp_size - RMPTABLE_CPU_BOOKKEEPING_SZ; 249 250 rmptable = (struct rmpentry *)rmptable_start; 251 rmptable_max_pfn = rmptable_size / sizeof(struct rmpentry) - 1; 252 253 cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/rmptable_init:online", __snp_enable, NULL); 254 255 /* 256 * Setting crash_kexec_post_notifiers to 'true' to ensure that SNP panic 257 * notifier is invoked to do SNP IOMMU shutdown before kdump. 258 */ 259 crash_kexec_post_notifiers = true; 260 261 return 0; 262 263 nosnp: 264 cc_platform_clear(CC_ATTR_HOST_SEV_SNP); 265 return -ENOSYS; 266 } 267 268 /* 269 * This must be called after the IOMMU has been initialized. 270 */ 271 device_initcall(snp_rmptable_init); 272 273 static struct rmpentry *get_rmpentry(u64 pfn) 274 { 275 if (WARN_ON_ONCE(pfn > rmptable_max_pfn)) 276 return ERR_PTR(-EFAULT); 277 278 return &rmptable[pfn]; 279 } 280 281 static struct rmpentry *__snp_lookup_rmpentry(u64 pfn, int *level) 282 { 283 struct rmpentry *large_entry, *entry; 284 285 if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) 286 return ERR_PTR(-ENODEV); 287 288 entry = get_rmpentry(pfn); 289 if (IS_ERR(entry)) 290 return entry; 291 292 /* 293 * Find the authoritative RMP entry for a PFN. This can be either a 4K 294 * RMP entry or a special large RMP entry that is authoritative for a 295 * whole 2M area. 296 */ 297 large_entry = get_rmpentry(pfn & PFN_PMD_MASK); 298 if (IS_ERR(large_entry)) 299 return large_entry; 300 301 *level = RMP_TO_PG_LEVEL(large_entry->pagesize); 302 303 return entry; 304 } 305 306 int snp_lookup_rmpentry(u64 pfn, bool *assigned, int *level) 307 { 308 struct rmpentry *e; 309 310 e = __snp_lookup_rmpentry(pfn, level); 311 if (IS_ERR(e)) 312 return PTR_ERR(e); 313 314 *assigned = !!e->assigned; 315 return 0; 316 } 317 EXPORT_SYMBOL_GPL(snp_lookup_rmpentry); 318 319 /* 320 * Dump the raw RMP entry for a particular PFN. These bits are documented in the 321 * PPR for a particular CPU model and provide useful information about how a 322 * particular PFN is being utilized by the kernel/firmware at the time certain 323 * unexpected events occur, such as RMP faults. 324 */ 325 static void dump_rmpentry(u64 pfn) 326 { 327 u64 pfn_i, pfn_end; 328 struct rmpentry *e; 329 int level; 330 331 e = __snp_lookup_rmpentry(pfn, &level); 332 if (IS_ERR(e)) { 333 pr_err("Failed to read RMP entry for PFN 0x%llx, error %ld\n", 334 pfn, PTR_ERR(e)); 335 return; 336 } 337 338 if (e->assigned) { 339 pr_info("PFN 0x%llx, RMP entry: [0x%016llx - 0x%016llx]\n", 340 pfn, e->lo, e->hi); 341 return; 342 } 343 344 /* 345 * If the RMP entry for a particular PFN is not in an assigned state, 346 * then it is sometimes useful to get an idea of whether or not any RMP 347 * entries for other PFNs within the same 2MB region are assigned, since 348 * those too can affect the ability to access a particular PFN in 349 * certain situations, such as when the PFN is being accessed via a 2MB 350 * mapping in the host page table. 351 */ 352 pfn_i = ALIGN_DOWN(pfn, PTRS_PER_PMD); 353 pfn_end = pfn_i + PTRS_PER_PMD; 354 355 pr_info("PFN 0x%llx unassigned, dumping non-zero entries in 2M PFN region: [0x%llx - 0x%llx]\n", 356 pfn, pfn_i, pfn_end); 357 358 while (pfn_i < pfn_end) { 359 e = __snp_lookup_rmpentry(pfn_i, &level); 360 if (IS_ERR(e)) { 361 pr_err("Error %ld reading RMP entry for PFN 0x%llx\n", 362 PTR_ERR(e), pfn_i); 363 pfn_i++; 364 continue; 365 } 366 367 if (e->lo || e->hi) 368 pr_info("PFN: 0x%llx, [0x%016llx - 0x%016llx]\n", pfn_i, e->lo, e->hi); 369 pfn_i++; 370 } 371 } 372 373 void snp_dump_hva_rmpentry(unsigned long hva) 374 { 375 unsigned long paddr; 376 unsigned int level; 377 pgd_t *pgd; 378 pte_t *pte; 379 380 pgd = __va(read_cr3_pa()); 381 pgd += pgd_index(hva); 382 pte = lookup_address_in_pgd(pgd, hva, &level); 383 384 if (!pte) { 385 pr_err("Can't dump RMP entry for HVA %lx: no PTE/PFN found\n", hva); 386 return; 387 } 388 389 paddr = PFN_PHYS(pte_pfn(*pte)) | (hva & ~page_level_mask(level)); 390 dump_rmpentry(PHYS_PFN(paddr)); 391 } 392 393 /* 394 * PSMASH a 2MB aligned page into 4K pages in the RMP table while preserving the 395 * Validated bit. 396 */ 397 int psmash(u64 pfn) 398 { 399 unsigned long paddr = pfn << PAGE_SHIFT; 400 int ret; 401 402 if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) 403 return -ENODEV; 404 405 if (!pfn_valid(pfn)) 406 return -EINVAL; 407 408 /* Binutils version 2.36 supports the PSMASH mnemonic. */ 409 asm volatile(".byte 0xF3, 0x0F, 0x01, 0xFF" 410 : "=a" (ret) 411 : "a" (paddr) 412 : "memory", "cc"); 413 414 return ret; 415 } 416 EXPORT_SYMBOL_GPL(psmash); 417 418 /* 419 * If the kernel uses a 2MB or larger directmap mapping to write to an address, 420 * and that mapping contains any 4KB pages that are set to private in the RMP 421 * table, an RMP #PF will trigger and cause a host crash. Hypervisor code that 422 * owns the PFNs being transitioned will never attempt such a write, but other 423 * kernel tasks writing to other PFNs in the range may trigger these checks 424 * inadvertently due a large directmap mapping that happens to overlap such a 425 * PFN. 426 * 427 * Prevent this by splitting any 2MB+ mappings that might end up containing a 428 * mix of private/shared PFNs as a result of a subsequent RMPUPDATE for the 429 * PFN/rmp_level passed in. 430 * 431 * Note that there is no attempt here to scan all the RMP entries for the 2MB 432 * physical range, since it would only be worthwhile in determining if a 433 * subsequent RMPUPDATE for a 4KB PFN would result in all the entries being of 434 * the same shared/private state, thus avoiding the need to split the mapping. 435 * But that would mean the entries are currently in a mixed state, and so the 436 * mapping would have already been split as a result of prior transitions. 437 * And since the 4K split is only done if the mapping is 2MB+, and there isn't 438 * currently a mechanism in place to restore 2MB+ mappings, such a check would 439 * not provide any usable benefit. 440 * 441 * More specifics on how these checks are carried out can be found in APM 442 * Volume 2, "RMP and VMPL Access Checks". 443 */ 444 static int adjust_direct_map(u64 pfn, int rmp_level) 445 { 446 unsigned long vaddr; 447 unsigned int level; 448 int npages, ret; 449 pte_t *pte; 450 451 /* 452 * pfn_to_kaddr() will return a vaddr only within the direct 453 * map range. 454 */ 455 vaddr = (unsigned long)pfn_to_kaddr(pfn); 456 457 /* Only 4KB/2MB RMP entries are supported by current hardware. */ 458 if (WARN_ON_ONCE(rmp_level > PG_LEVEL_2M)) 459 return -EINVAL; 460 461 if (!pfn_valid(pfn)) 462 return -EINVAL; 463 464 if (rmp_level == PG_LEVEL_2M && 465 (!IS_ALIGNED(pfn, PTRS_PER_PMD) || !pfn_valid(pfn + PTRS_PER_PMD - 1))) 466 return -EINVAL; 467 468 /* 469 * If an entire 2MB physical range is being transitioned, then there is 470 * no risk of RMP #PFs due to write accesses from overlapping mappings, 471 * since even accesses from 1GB mappings will be treated as 2MB accesses 472 * as far as RMP table checks are concerned. 473 */ 474 if (rmp_level == PG_LEVEL_2M) 475 return 0; 476 477 pte = lookup_address(vaddr, &level); 478 if (!pte || pte_none(*pte)) 479 return 0; 480 481 if (level == PG_LEVEL_4K) 482 return 0; 483 484 npages = page_level_size(rmp_level) / PAGE_SIZE; 485 ret = set_memory_4k(vaddr, npages); 486 if (ret) 487 pr_warn("Failed to split direct map for PFN 0x%llx, ret: %d\n", 488 pfn, ret); 489 490 return ret; 491 } 492 493 /* 494 * It is expected that those operations are seldom enough so that no mutual 495 * exclusion of updaters is needed and thus the overlap error condition below 496 * should happen very rarely and would get resolved relatively quickly by 497 * the firmware. 498 * 499 * If not, one could consider introducing a mutex or so here to sync concurrent 500 * RMP updates and thus diminish the amount of cases where firmware needs to 501 * lock 2M ranges to protect against concurrent updates. 502 * 503 * The optimal solution would be range locking to avoid locking disjoint 504 * regions unnecessarily but there's no support for that yet. 505 */ 506 static int rmpupdate(u64 pfn, struct rmp_state *state) 507 { 508 unsigned long paddr = pfn << PAGE_SHIFT; 509 int ret, level; 510 511 if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) 512 return -ENODEV; 513 514 level = RMP_TO_PG_LEVEL(state->pagesize); 515 516 if (adjust_direct_map(pfn, level)) 517 return -EFAULT; 518 519 do { 520 /* Binutils version 2.36 supports the RMPUPDATE mnemonic. */ 521 asm volatile(".byte 0xF2, 0x0F, 0x01, 0xFE" 522 : "=a" (ret) 523 : "a" (paddr), "c" ((unsigned long)state) 524 : "memory", "cc"); 525 } while (ret == RMPUPDATE_FAIL_OVERLAP); 526 527 if (ret) { 528 pr_err("RMPUPDATE failed for PFN %llx, pg_level: %d, ret: %d\n", 529 pfn, level, ret); 530 dump_rmpentry(pfn); 531 dump_stack(); 532 return -EFAULT; 533 } 534 535 return 0; 536 } 537 538 /* Transition a page to guest-owned/private state in the RMP table. */ 539 int rmp_make_private(u64 pfn, u64 gpa, enum pg_level level, u32 asid, bool immutable) 540 { 541 struct rmp_state state; 542 543 memset(&state, 0, sizeof(state)); 544 state.assigned = 1; 545 state.asid = asid; 546 state.immutable = immutable; 547 state.gpa = gpa; 548 state.pagesize = PG_LEVEL_TO_RMP(level); 549 550 return rmpupdate(pfn, &state); 551 } 552 EXPORT_SYMBOL_GPL(rmp_make_private); 553 554 /* Transition a page to hypervisor-owned/shared state in the RMP table. */ 555 int rmp_make_shared(u64 pfn, enum pg_level level) 556 { 557 struct rmp_state state; 558 559 memset(&state, 0, sizeof(state)); 560 state.pagesize = PG_LEVEL_TO_RMP(level); 561 562 return rmpupdate(pfn, &state); 563 } 564 EXPORT_SYMBOL_GPL(rmp_make_shared); 565 566 void snp_leak_pages(u64 pfn, unsigned int npages) 567 { 568 struct page *page = pfn_to_page(pfn); 569 570 pr_warn("Leaking PFN range 0x%llx-0x%llx\n", pfn, pfn + npages); 571 572 spin_lock(&snp_leaked_pages_list_lock); 573 while (npages--) { 574 575 /* 576 * Reuse the page's buddy list for chaining into the leaked 577 * pages list. This page should not be on a free list currently 578 * and is also unsafe to be added to a free list. 579 */ 580 if (likely(!PageCompound(page)) || 581 582 /* 583 * Skip inserting tail pages of compound page as 584 * page->buddy_list of tail pages is not usable. 585 */ 586 (PageHead(page) && compound_nr(page) <= npages)) 587 list_add_tail(&page->buddy_list, &snp_leaked_pages_list); 588 589 dump_rmpentry(pfn); 590 snp_nr_leaked_pages++; 591 pfn++; 592 page++; 593 } 594 spin_unlock(&snp_leaked_pages_list_lock); 595 } 596 EXPORT_SYMBOL_GPL(snp_leak_pages); 597 598 void kdump_sev_callback(void) 599 { 600 /* 601 * Do wbinvd() on remote CPUs when SNP is enabled in order to 602 * safely do SNP_SHUTDOWN on the local CPU. 603 */ 604 if (cc_platform_has(CC_ATTR_HOST_SEV_SNP)) 605 wbinvd(); 606 } 607