1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * AMD SVM-SEV Host Support.
4 *
5 * Copyright (C) 2023 Advanced Micro Devices, Inc.
6 *
7 * Author: Ashish Kalra <ashish.kalra@amd.com>
8 *
9 */
10
11 #include <linux/cc_platform.h>
12 #include <linux/printk.h>
13 #include <linux/mm_types.h>
14 #include <linux/set_memory.h>
15 #include <linux/memblock.h>
16 #include <linux/kernel.h>
17 #include <linux/mm.h>
18 #include <linux/cpumask.h>
19 #include <linux/iommu.h>
20 #include <linux/amd-iommu.h>
21 #include <linux/nospec.h>
22 #include <linux/workqueue.h>
23
24 #include <asm/sev.h>
25 #include <asm/processor.h>
26 #include <asm/setup.h>
27 #include <asm/svm.h>
28 #include <asm/smp.h>
29 #include <asm/cpu.h>
30 #include <asm/apic.h>
31 #include <asm/cpuid/api.h>
32 #include <asm/cmdline.h>
33 #include <asm/iommu.h>
34 #include <asm/msr.h>
35
36 /*
37 * The RMP entry information as returned by the RMPREAD instruction.
38 */
39 struct rmpentry {
40 u64 gpa;
41 u8 assigned :1,
42 rsvd1 :7;
43 u8 pagesize :1,
44 hpage_region_status :1,
45 rsvd2 :6;
46 u8 immutable :1,
47 rsvd3 :7;
48 u8 rsvd4;
49 u32 asid;
50 } __packed;
51
52 /*
53 * The raw RMP entry format is not architectural. The format is defined in PPR
54 * Family 19h Model 01h, Rev B1 processor. This format represents the actual
55 * entry in the RMP table memory. The bitfield definitions are used for machines
56 * without the RMPREAD instruction (Zen3 and Zen4), otherwise the "hi" and "lo"
57 * fields are only used for dumping the raw data.
58 */
59 struct rmpentry_raw {
60 union {
61 struct {
62 u64 assigned : 1,
63 pagesize : 1,
64 immutable : 1,
65 rsvd1 : 9,
66 gpa : 39,
67 asid : 10,
68 vmsa : 1,
69 validated : 1,
70 rsvd2 : 1;
71 };
72 u64 lo;
73 };
74 u64 hi;
75 } __packed;
76
77 /*
78 * The first 16KB from the RMP_BASE is used by the processor for the
79 * bookkeeping, the range needs to be added during the RMP entry lookup.
80 */
81 #define RMPTABLE_CPU_BOOKKEEPING_SZ 0x4000
82
83 /*
84 * For a non-segmented RMP table, use the maximum physical addressing as the
85 * segment size in order to always arrive at index 0 in the table.
86 */
87 #define RMPTABLE_NON_SEGMENTED_SHIFT 52
88
89 struct rmp_segment_desc {
90 struct rmpentry_raw *rmp_entry;
91 u64 max_index;
92 u64 size;
93 };
94
95 /*
96 * Segmented RMP Table support.
97 * - The segment size is used for two purposes:
98 * - Identify the amount of memory covered by an RMP segment
99 * - Quickly locate an RMP segment table entry for a physical address
100 *
101 * - The RMP segment table contains pointers to an RMP table that covers
102 * a specific portion of memory. There can be up to 512 8-byte entries,
103 * one pages worth.
104 */
105 #define RST_ENTRY_MAPPED_SIZE(x) ((x) & GENMASK_ULL(19, 0))
106 #define RST_ENTRY_SEGMENT_BASE(x) ((x) & GENMASK_ULL(51, 20))
107
108 #define RST_SIZE SZ_4K
109 static struct rmp_segment_desc **rmp_segment_table __ro_after_init;
110 static unsigned int rst_max_index __ro_after_init = 512;
111
112 static unsigned int rmp_segment_shift;
113 static u64 rmp_segment_size;
114 static u64 rmp_segment_mask;
115
116 #define RST_ENTRY_INDEX(x) ((x) >> rmp_segment_shift)
117 #define RMP_ENTRY_INDEX(x) ((u64)(PHYS_PFN((x) & rmp_segment_mask)))
118
119 static u64 rmp_cfg;
120
121 static void *rmp_bookkeeping __ro_after_init;
122
123 /* Mask to apply to a PFN to get the first PFN of a 2MB page */
124 #define PFN_PMD_MASK GENMASK_ULL(63, PMD_SHIFT - PAGE_SHIFT)
125
126 static u64 probed_rmp_base, probed_rmp_size;
127
128 static u64 rmpopt_pa_start, rmpopt_pa_end;
129
130 enum rmpopt_op_type {
131 RMPOPT_OP_VERIFY_AND_REPORT_STATUS,
132 RMPOPT_OP_REPORT_STATUS
133 };
134
135 static struct workqueue_struct *rmpopt_wq;
136 static struct delayed_work rmpopt_delayed_work;
137
138 /* Software RMPOPT facilities initialized */
139 static bool rmpopt_soft_init;
140
141 /*
142 * Delay, in milliseconds, before the RMP re-optimization pass runs after an
143 * SNP guest is torn down. This coalesces a burst of teardowns into a single
144 * scan and gives each guest's pages time to be converted back to the shared,
145 * hypervisor-owned state. The 10 second value is a heuristic trading
146 * re-optimization latency against scanning too eagerly.
147 */
148 #define RMPOPT_WORK_TIMEOUT (10 * MSEC_PER_SEC)
149
150 static LIST_HEAD(snp_leaked_pages_list);
151 static DEFINE_SPINLOCK(snp_leaked_pages_list_lock);
152
153 static unsigned long snp_nr_leaked_pages;
154
155 #undef pr_fmt
156 #define pr_fmt(fmt) "SEV-SNP: " fmt
157
mfd_reconfigure(void * arg)158 static void mfd_reconfigure(void *arg)
159 {
160 if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
161 return;
162
163 if (arg)
164 msr_set_bit(MSR_AMD64_SYSCFG, MSR_AMD64_SYSCFG_MFDM_BIT);
165 else
166 msr_clear_bit(MSR_AMD64_SYSCFG, MSR_AMD64_SYSCFG_MFDM_BIT);
167 }
168
snp_enable(void * arg)169 static void snp_enable(void *arg)
170 {
171 u64 val;
172
173 if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
174 return;
175
176 rdmsrq(MSR_AMD64_SYSCFG, val);
177
178 val |= MSR_AMD64_SYSCFG_SNP_EN;
179 val |= MSR_AMD64_SYSCFG_SNP_VMPL_EN;
180
181 wrmsrq(MSR_AMD64_SYSCFG, val);
182 }
183
__snp_fixup_e820_tables(u64 pa)184 static void __init __snp_fixup_e820_tables(u64 pa)
185 {
186 if (IS_ALIGNED(pa, PMD_SIZE))
187 return;
188
189 /*
190 * Handle cases where the RMP table placement by the BIOS is not
191 * 2M aligned and the kexec kernel could try to allocate
192 * from within that chunk which then causes a fatal RMP fault.
193 *
194 * The e820_table needs to be updated as it is converted to
195 * kernel memory resources and used by KEXEC_FILE_LOAD syscall
196 * to load kexec segments.
197 *
198 * The e820_table_firmware needs to be updated as it is exposed
199 * to sysfs and used by the KEXEC_LOAD syscall to load kexec
200 * segments.
201 *
202 * The e820_table_kexec needs to be updated as it passed to
203 * the kexec-ed kernel.
204 */
205 pa = ALIGN_DOWN(pa, PMD_SIZE);
206 if (e820__mapped_any(pa, pa + PMD_SIZE, E820_TYPE_RAM)) {
207 pr_info("Reserving start/end of RMP table on a 2MB boundary [0x%016llx]\n", pa);
208 e820__range_update(pa, PMD_SIZE, E820_TYPE_RAM, E820_TYPE_RESERVED);
209 e820__range_update_table(e820_table_kexec, pa, PMD_SIZE, E820_TYPE_RAM, E820_TYPE_RESERVED);
210 if (!memblock_is_region_reserved(pa, PMD_SIZE))
211 memblock_reserve(pa, PMD_SIZE);
212 }
213 }
214
fixup_e820_tables_for_segmented_rmp(void)215 static void __init fixup_e820_tables_for_segmented_rmp(void)
216 {
217 u64 pa, *rst, size, mapped_size;
218 unsigned int i;
219
220 __snp_fixup_e820_tables(probed_rmp_base);
221
222 pa = probed_rmp_base + RMPTABLE_CPU_BOOKKEEPING_SZ;
223
224 __snp_fixup_e820_tables(pa + RST_SIZE);
225
226 rst = early_memremap(pa, RST_SIZE);
227 if (!rst)
228 return;
229
230 for (i = 0; i < rst_max_index; i++) {
231 pa = RST_ENTRY_SEGMENT_BASE(rst[i]);
232 mapped_size = RST_ENTRY_MAPPED_SIZE(rst[i]);
233 if (!mapped_size)
234 continue;
235
236 __snp_fixup_e820_tables(pa);
237
238 /*
239 * Mapped size in GB. Mapped size is allowed to exceed
240 * the segment coverage size, but gets reduced to the
241 * segment coverage size.
242 */
243 mapped_size <<= 30;
244 if (mapped_size > rmp_segment_size)
245 mapped_size = rmp_segment_size;
246
247 /* Calculate the RMP segment size (16 bytes/page mapped) */
248 size = PHYS_PFN(mapped_size) << 4;
249
250 __snp_fixup_e820_tables(pa + size);
251 }
252
253 early_memunmap(rst, RST_SIZE);
254 }
255
fixup_e820_tables_for_contiguous_rmp(void)256 static void __init fixup_e820_tables_for_contiguous_rmp(void)
257 {
258 __snp_fixup_e820_tables(probed_rmp_base);
259 __snp_fixup_e820_tables(probed_rmp_base + probed_rmp_size);
260 }
261
snp_fixup_e820_tables(void)262 void __init snp_fixup_e820_tables(void)
263 {
264 if (rmp_cfg & MSR_AMD64_SEG_RMP_ENABLED) {
265 fixup_e820_tables_for_segmented_rmp();
266 } else {
267 fixup_e820_tables_for_contiguous_rmp();
268 }
269 }
270
clear_rmp(void)271 static void clear_rmp(void)
272 {
273 unsigned int i;
274 u64 val;
275
276 if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
277 return;
278
279 /* Clearing the RMP while SNP is enabled will cause an exception */
280 rdmsrq(MSR_AMD64_SYSCFG, val);
281 if (WARN_ON_ONCE(val & MSR_AMD64_SYSCFG_SNP_EN))
282 return;
283
284 memset(rmp_bookkeeping, 0, RMPTABLE_CPU_BOOKKEEPING_SZ);
285
286 for (i = 0; i < rst_max_index; i++) {
287 struct rmp_segment_desc *desc;
288
289 desc = rmp_segment_table[i];
290 if (!desc)
291 continue;
292
293 memset(desc->rmp_entry, 0, desc->size);
294 }
295 }
296
alloc_rmp_segment_desc(u64 segment_pa,u64 segment_size,u64 pa)297 static bool __init alloc_rmp_segment_desc(u64 segment_pa, u64 segment_size, u64 pa)
298 {
299 u64 rst_index, rmp_segment_size_max;
300 struct rmp_segment_desc *desc;
301 void *rmp_segment;
302
303 /* Calculate the maximum size an RMP can be (16 bytes/page mapped) */
304 rmp_segment_size_max = PHYS_PFN(rmp_segment_size) << 4;
305
306 /* Validate the RMP segment size */
307 if (segment_size > rmp_segment_size_max) {
308 pr_err("Invalid RMP size 0x%llx for configured segment size 0x%llx\n",
309 segment_size, rmp_segment_size_max);
310 return false;
311 }
312
313 /* Validate the RMP segment table index */
314 rst_index = RST_ENTRY_INDEX(pa);
315 if (rst_index >= rst_max_index) {
316 pr_err("Invalid RMP segment base address 0x%llx for configured segment size 0x%llx\n",
317 pa, rmp_segment_size);
318 return false;
319 }
320
321 if (rmp_segment_table[rst_index]) {
322 pr_err("RMP segment descriptor already exists at index %llu\n", rst_index);
323 return false;
324 }
325
326 rmp_segment = memremap(segment_pa, segment_size, MEMREMAP_WB);
327 if (!rmp_segment) {
328 pr_err("Failed to map RMP segment addr 0x%llx size 0x%llx\n",
329 segment_pa, segment_size);
330 return false;
331 }
332
333 desc = kzalloc_obj(*desc);
334 if (!desc) {
335 memunmap(rmp_segment);
336 return false;
337 }
338
339 desc->rmp_entry = rmp_segment;
340 desc->max_index = segment_size / sizeof(*desc->rmp_entry);
341 desc->size = segment_size;
342
343 rmp_segment_table[rst_index] = desc;
344
345 return true;
346 }
347
free_rmp_segment_table(void)348 static void __init free_rmp_segment_table(void)
349 {
350 unsigned int i;
351
352 for (i = 0; i < rst_max_index; i++) {
353 struct rmp_segment_desc *desc;
354
355 desc = rmp_segment_table[i];
356 if (!desc)
357 continue;
358
359 memunmap(desc->rmp_entry);
360
361 kfree(desc);
362 }
363
364 free_page((unsigned long)rmp_segment_table);
365
366 rmp_segment_table = NULL;
367 }
368
369 /* Allocate the table used to index into the RMP segments */
alloc_rmp_segment_table(void)370 static bool __init alloc_rmp_segment_table(void)
371 {
372 struct page *page;
373
374 page = alloc_page(__GFP_ZERO);
375 if (!page)
376 return false;
377
378 rmp_segment_table = page_address(page);
379
380 return true;
381 }
382
setup_contiguous_rmptable(void)383 static bool __init setup_contiguous_rmptable(void)
384 {
385 u64 max_rmp_pfn, calc_rmp_sz, rmptable_segment, rmptable_size, rmp_end;
386
387 if (!probed_rmp_size)
388 return false;
389
390 rmp_end = probed_rmp_base + probed_rmp_size - 1;
391
392 /*
393 * Calculate the amount of memory that must be reserved by the BIOS to
394 * address the whole RAM, including the bookkeeping area. The RMP itself
395 * must also be covered.
396 */
397 max_rmp_pfn = max_pfn;
398 if (PFN_UP(rmp_end) > max_pfn)
399 max_rmp_pfn = PFN_UP(rmp_end);
400
401 calc_rmp_sz = (max_rmp_pfn << 4) + RMPTABLE_CPU_BOOKKEEPING_SZ;
402 if (calc_rmp_sz > probed_rmp_size) {
403 pr_err("Memory reserved for the RMP table does not cover full system RAM (expected 0x%llx got 0x%llx)\n",
404 calc_rmp_sz, probed_rmp_size);
405 return false;
406 }
407
408 if (!alloc_rmp_segment_table())
409 return false;
410
411 /* Map only the RMP entries */
412 rmptable_segment = probed_rmp_base + RMPTABLE_CPU_BOOKKEEPING_SZ;
413 rmptable_size = probed_rmp_size - RMPTABLE_CPU_BOOKKEEPING_SZ;
414
415 if (!alloc_rmp_segment_desc(rmptable_segment, rmptable_size, 0)) {
416 free_rmp_segment_table();
417 return false;
418 }
419
420 return true;
421 }
422
setup_segmented_rmptable(void)423 static bool __init setup_segmented_rmptable(void)
424 {
425 u64 rst_pa, *rst, pa, ram_pa_end, ram_pa_max;
426 unsigned int i, max_index;
427
428 if (!probed_rmp_base)
429 return false;
430
431 if (!alloc_rmp_segment_table())
432 return false;
433
434 rst_pa = probed_rmp_base + RMPTABLE_CPU_BOOKKEEPING_SZ;
435 rst = memremap(rst_pa, RST_SIZE, MEMREMAP_WB);
436 if (!rst) {
437 pr_err("Failed to map RMP segment table addr 0x%llx\n", rst_pa);
438 goto e_free;
439 }
440
441 pr_info("Segmented RMP using %lluGB segments\n", rmp_segment_size >> 30);
442
443 ram_pa_max = max_pfn << PAGE_SHIFT;
444
445 max_index = 0;
446 ram_pa_end = 0;
447 for (i = 0; i < rst_max_index; i++) {
448 u64 rmp_segment, rmp_size, mapped_size;
449
450 mapped_size = RST_ENTRY_MAPPED_SIZE(rst[i]);
451 if (!mapped_size)
452 continue;
453
454 max_index = i;
455
456 /*
457 * Mapped size in GB. Mapped size is allowed to exceed the
458 * segment coverage size, but gets reduced to the segment
459 * coverage size.
460 */
461 mapped_size <<= 30;
462 if (mapped_size > rmp_segment_size) {
463 pr_info("RMP segment %u mapped size (0x%llx) reduced to 0x%llx\n",
464 i, mapped_size, rmp_segment_size);
465 mapped_size = rmp_segment_size;
466 }
467
468 rmp_segment = RST_ENTRY_SEGMENT_BASE(rst[i]);
469
470 /* Calculate the RMP segment size (16 bytes/page mapped) */
471 rmp_size = PHYS_PFN(mapped_size) << 4;
472
473 pa = (u64)i << rmp_segment_shift;
474
475 /*
476 * Some segments may be for MMIO mapped above system RAM. These
477 * segments are used for Trusted I/O.
478 */
479 if (pa < ram_pa_max)
480 ram_pa_end = pa + mapped_size;
481
482 if (!alloc_rmp_segment_desc(rmp_segment, rmp_size, pa))
483 goto e_unmap;
484
485 pr_info("RMP segment %u physical address [0x%llx - 0x%llx] covering [0x%llx - 0x%llx]\n",
486 i, rmp_segment, rmp_segment + rmp_size - 1, pa, pa + mapped_size - 1);
487 }
488
489 if (ram_pa_max > ram_pa_end) {
490 pr_err("Segmented RMP does not cover full system RAM (expected 0x%llx got 0x%llx)\n",
491 ram_pa_max, ram_pa_end);
492 goto e_unmap;
493 }
494
495 /* Adjust the maximum index based on the found segments */
496 rst_max_index = max_index + 1;
497
498 memunmap(rst);
499
500 return true;
501
502 e_unmap:
503 memunmap(rst);
504
505 e_free:
506 free_rmp_segment_table();
507
508 return false;
509 }
510
setup_rmptable(void)511 static bool __init setup_rmptable(void)
512 {
513 if (rmp_cfg & MSR_AMD64_SEG_RMP_ENABLED) {
514 if (!setup_segmented_rmptable())
515 return false;
516 } else {
517 if (!setup_contiguous_rmptable())
518 return false;
519 }
520
521 rmp_bookkeeping = memremap(probed_rmp_base, RMPTABLE_CPU_BOOKKEEPING_SZ, MEMREMAP_WB);
522 if (!rmp_bookkeeping) {
523 pr_err("Failed to map RMP bookkeeping area\n");
524 free_rmp_segment_table();
525
526 return false;
527 }
528
529 return true;
530 }
531
clear_hsave_pa(void * arg)532 static void clear_hsave_pa(void *arg)
533 {
534 wrmsrq(MSR_VM_HSAVE_PA, 0);
535 }
536
snp_prepare(void)537 int snp_prepare(void)
538 {
539 u64 val;
540
541 /*
542 * Check if SEV-SNP is already enabled, this can happen in case of
543 * kexec boot.
544 */
545 rdmsrq(MSR_AMD64_SYSCFG, val);
546 if (val & MSR_AMD64_SYSCFG_SNP_EN)
547 return 0;
548
549 clear_rmp();
550
551 /*
552 * No CPU may come online without SnpEn while SNP is active; disable
553 * hotplug here and re-enable it in snp_shutdown().
554 */
555 cpu_hotplug_disable();
556
557 if (!cpumask_equal(cpu_online_mask, cpu_present_mask)) {
558 cpu_hotplug_enable();
559 pr_warn("SNP init failed: not all CPUs online. (%*pbl online <-> %*pbl present masks).\n",
560 cpumask_pr_args(cpu_online_mask),
561 cpumask_pr_args(cpu_present_mask));
562 return -EOPNOTSUPP;
563 }
564
565 wbinvd_on_all_cpus();
566
567 /*
568 * MtrrFixDramModEn is not shared between threads on a core,
569 * therefore it must be set on all CPUs prior to enabling SNP.
570 */
571 on_each_cpu(mfd_reconfigure, (void *)1, 1);
572 on_each_cpu(snp_enable, NULL, 1);
573
574 /* SNP_INIT requires MSR_VM_HSAVE_PA to be cleared on all CPUs. */
575 on_each_cpu(clear_hsave_pa, NULL, 1);
576
577 return 0;
578 }
579 EXPORT_SYMBOL_FOR_MODULES(snp_prepare, "ccp");
580
snp_shutdown(void)581 void snp_shutdown(void)
582 {
583 u64 syscfg;
584
585 rdmsrq(MSR_AMD64_SYSCFG, syscfg);
586 if (syscfg & MSR_AMD64_SYSCFG_SNP_EN)
587 return;
588
589 if (rmpopt_soft_init)
590 cancel_delayed_work_sync(&rmpopt_delayed_work);
591
592 clear_rmp();
593 on_each_cpu(mfd_reconfigure, NULL, 1);
594
595 /*
596 * The firmware has disabled SNP (SnpEn is clear), so re-enable CPU
597 * hotplug. A legacy SNP shutdown returns above with SnpEn still set and
598 * leaves hotplug disabled.
599 */
600 cpu_hotplug_enable();
601 }
602 EXPORT_SYMBOL_FOR_MODULES(snp_shutdown, "ccp");
603
604 /*
605 * RMPOPT optimizations skip RMP checks at 1GB granularity if this range of
606 * memory does not contain any SNP guest memory.
607 *
608 * @pa is a system physical address; RMPOPT operates on the containing 1GB.
609 */
rmpopt(u64 pa)610 static void rmpopt(u64 pa)
611 {
612 enum rmpopt_op_type op = RMPOPT_OP_VERIFY_AND_REPORT_STATUS;
613 u64 pa_start = ALIGN_DOWN(pa, SZ_1G);
614
615 /* Supported by binutils 2.48+ */
616 asm volatile(".byte 0xf2, 0x0f, 0x01, 0xfc"
617 :: "a" (pa_start), "c" (op)
618 : "memory", "cc");
619 }
620
rmpopt_scan_range(void * arg)621 static void rmpopt_scan_range(void *arg)
622 {
623 u64 pa;
624
625 for (pa = rmpopt_pa_start; pa < rmpopt_pa_end; pa += SZ_1G)
626 rmpopt(pa);
627 }
628
do_rmpopt_work(struct work_struct * work)629 static void do_rmpopt_work(struct work_struct *work)
630 {
631 /*
632 * Warm up the RMPOPT cache on this pinned per-CPU worker with interrupts
633 * enabled, so the IRQ-disabled fan-out below only issues cache-hit RMPOPTs.
634 */
635 rmpopt_scan_range(NULL);
636
637 on_each_cpu_mask(cpu_primary_thread_mask, rmpopt_scan_range, NULL, true);
638 }
639
rmpopt_init(void)640 static int __init rmpopt_init(void)
641 {
642 if (!cpu_feature_enabled(X86_FEATURE_RMPOPT))
643 return -ENODEV;
644
645 rmpopt_wq = alloc_workqueue("rmpopt_wq", WQ_PERCPU, 1);
646 if (!rmpopt_wq) {
647 pr_err("Failed to allocate RMPOPT workqueue\n");
648 return -ENOMEM;
649 }
650
651 INIT_DELAYED_WORK(&rmpopt_delayed_work, do_rmpopt_work);
652
653 /* The optimization range is fixed at boot; compute it once. */
654 rmpopt_pa_start = ALIGN_DOWN(PFN_PHYS(min_low_pfn), SZ_1G);
655 rmpopt_pa_end = ALIGN(PFN_PHYS(max_pfn), SZ_1G);
656 if ((rmpopt_pa_end - rmpopt_pa_start) > SZ_2T)
657 rmpopt_pa_end = rmpopt_pa_start + SZ_2T;
658
659 pr_info("RMPOPT optimizations enabled\n");
660
661 rmpopt_soft_init = true;
662
663 return 0;
664 }
665 device_initcall(rmpopt_init);
666
snp_enable_rmpopt(void)667 void snp_enable_rmpopt(void)
668 {
669 u64 base;
670 int cpu;
671
672 if (!cpu_feature_enabled(X86_FEATURE_RMPOPT))
673 return;
674
675 if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
676 return;
677
678 if (!rmpopt_soft_init)
679 return;
680
681 /*
682 * Per-CPU RMPOPT tables cover at most 2 TB. Program each core's
683 * RMPOPT_BASE with the start of RAM to optimize up to 2 TB.
684 */
685 rdmsrq(MSR_AMD64_RMPOPT_BASE, base);
686 if (!(base & MSR_AMD64_RMPOPT_ENABLE))
687 for_each_cpu(cpu, cpu_primary_thread_mask)
688 wrmsrq_on_cpu(cpu, MSR_AMD64_RMPOPT_BASE,
689 rmpopt_pa_start | MSR_AMD64_RMPOPT_ENABLE);
690
691 mod_delayed_work(rmpopt_wq, &rmpopt_delayed_work,
692 msecs_to_jiffies(RMPOPT_WORK_TIMEOUT));
693 }
694 EXPORT_SYMBOL_FOR_MODULES(snp_enable_rmpopt, "ccp,kvm-amd");
695
696 /*
697 * Do the necessary preparations which are verified by the firmware as
698 * described in the SNP_INIT_EX firmware command description in the SNP
699 * firmware ABI spec.
700 */
snp_rmptable_init(void)701 int __init snp_rmptable_init(void)
702 {
703 u64 val;
704
705 if (WARN_ON_ONCE(!cc_platform_has(CC_ATTR_HOST_SEV_SNP)))
706 return -ENOSYS;
707
708 if (WARN_ON_ONCE(!amd_iommu_snp_en))
709 return -ENOSYS;
710
711 if (!setup_rmptable())
712 return -ENOSYS;
713
714 /*
715 * On a kexec boot SNP may already be enabled (legacy firmware leaves
716 * SnpEn set across shutdown), in which case snp_prepare() bails without
717 * disabling CPU hotplug, so disable it here.
718 */
719 rdmsrq(MSR_AMD64_SYSCFG, val);
720 if (val & MSR_AMD64_SYSCFG_SNP_EN)
721 cpu_hotplug_disable();
722
723 /*
724 * Setting crash_kexec_post_notifiers to 'true' to ensure that SNP panic
725 * notifier is invoked to do SNP IOMMU shutdown before kdump.
726 */
727 crash_kexec_post_notifiers = true;
728
729 return 0;
730 }
731
set_rmp_segment_info(unsigned int segment_shift)732 static void set_rmp_segment_info(unsigned int segment_shift)
733 {
734 rmp_segment_shift = segment_shift;
735 rmp_segment_size = 1ULL << rmp_segment_shift;
736 rmp_segment_mask = rmp_segment_size - 1;
737 }
738
739 #define RMP_ADDR_MASK GENMASK_ULL(51, 13)
740
probe_contiguous_rmptable_info(void)741 static bool probe_contiguous_rmptable_info(void)
742 {
743 u64 rmp_sz, rmp_base, rmp_end;
744
745 rdmsrq(MSR_AMD64_RMP_BASE, rmp_base);
746 rdmsrq(MSR_AMD64_RMP_END, rmp_end);
747
748 if (!(rmp_base & RMP_ADDR_MASK) || !(rmp_end & RMP_ADDR_MASK)) {
749 pr_err("Memory for the RMP table has not been reserved by BIOS\n");
750 return false;
751 }
752
753 if (rmp_base > rmp_end) {
754 pr_err("RMP configuration not valid: base=%#llx, end=%#llx\n", rmp_base, rmp_end);
755 return false;
756 }
757
758 rmp_sz = rmp_end - rmp_base + 1;
759
760 /* Treat the contiguous RMP table as a single segment */
761 rst_max_index = 1;
762
763 set_rmp_segment_info(RMPTABLE_NON_SEGMENTED_SHIFT);
764
765 probed_rmp_base = rmp_base;
766 probed_rmp_size = rmp_sz;
767
768 pr_info("RMP table physical range [0x%016llx - 0x%016llx]\n",
769 rmp_base, rmp_end);
770
771 return true;
772 }
773
probe_segmented_rmptable_info(void)774 static bool probe_segmented_rmptable_info(void)
775 {
776 unsigned int eax, ebx, segment_shift, segment_shift_min, segment_shift_max;
777 u64 rmp_base, rmp_end;
778
779 rdmsrq(MSR_AMD64_RMP_BASE, rmp_base);
780 if (!(rmp_base & RMP_ADDR_MASK)) {
781 pr_err("Memory for the RMP table has not been reserved by BIOS\n");
782 return false;
783 }
784
785 rdmsrq(MSR_AMD64_RMP_END, rmp_end);
786 WARN_ONCE(rmp_end & RMP_ADDR_MASK,
787 "Segmented RMP enabled but RMP_END MSR is non-zero\n");
788
789 /* Obtain the min and max supported RMP segment size */
790 eax = cpuid_eax(0x80000025);
791 segment_shift_min = eax & GENMASK(5, 0);
792 segment_shift_max = (eax & GENMASK(11, 6)) >> 6;
793
794 /* Verify the segment size is within the supported limits */
795 segment_shift = MSR_AMD64_RMP_SEGMENT_SHIFT(rmp_cfg);
796 if (segment_shift > segment_shift_max || segment_shift < segment_shift_min) {
797 pr_err("RMP segment size (%u) is not within advertised bounds (min=%u, max=%u)\n",
798 segment_shift, segment_shift_min, segment_shift_max);
799 return false;
800 }
801
802 /* Override the max supported RST index if a hardware limit exists */
803 ebx = cpuid_ebx(0x80000025);
804 if (ebx & BIT(10))
805 rst_max_index = ebx & GENMASK(9, 0);
806
807 set_rmp_segment_info(segment_shift);
808
809 probed_rmp_base = rmp_base;
810 probed_rmp_size = 0;
811
812 pr_info("Segmented RMP base table physical range [0x%016llx - 0x%016llx]\n",
813 rmp_base, rmp_base + RMPTABLE_CPU_BOOKKEEPING_SZ + RST_SIZE);
814
815 return true;
816 }
817
snp_probe_rmptable_info(void)818 bool snp_probe_rmptable_info(void)
819 {
820 if (cpu_feature_enabled(X86_FEATURE_SEGMENTED_RMP)) {
821 rdmsrq(MSR_AMD64_RMP_CFG, rmp_cfg);
822
823 if (rmp_cfg & MSR_AMD64_SEG_RMP_ENABLED) {
824 if (probe_segmented_rmptable_info())
825 return true;
826
827 setup_clear_cpu_cap(X86_FEATURE_RMPOPT);
828 return false;
829 }
830 } else {
831 setup_clear_cpu_cap(X86_FEATURE_RMPOPT);
832 }
833
834 return probe_contiguous_rmptable_info();
835 }
836
837 /*
838 * About the array_index_nospec() usage below:
839 *
840 * This function can get called by exported functions like
841 * snp_lookup_rmpentry(), which is used by the KVM #PF handler, among
842 * others, and since the @pfn passed in cannot always be trusted,
843 * speculation should be stopped as a protective measure.
844 */
get_raw_rmpentry(u64 pfn)845 static struct rmpentry_raw *get_raw_rmpentry(u64 pfn)
846 {
847 u64 paddr, rst_index, segment_index;
848 struct rmp_segment_desc *desc;
849
850 if (!rmp_segment_table)
851 return ERR_PTR(-ENODEV);
852
853 paddr = pfn << PAGE_SHIFT;
854
855 rst_index = RST_ENTRY_INDEX(paddr);
856 if (unlikely(rst_index >= rst_max_index))
857 return ERR_PTR(-EFAULT);
858
859 rst_index = array_index_nospec(rst_index, rst_max_index);
860
861 desc = rmp_segment_table[rst_index];
862 if (unlikely(!desc))
863 return ERR_PTR(-EFAULT);
864
865 segment_index = RMP_ENTRY_INDEX(paddr);
866 if (unlikely(segment_index >= desc->max_index))
867 return ERR_PTR(-EFAULT);
868
869 segment_index = array_index_nospec(segment_index, desc->max_index);
870
871 return desc->rmp_entry + segment_index;
872 }
873
get_rmpentry(u64 pfn,struct rmpentry * e)874 static int get_rmpentry(u64 pfn, struct rmpentry *e)
875 {
876 struct rmpentry_raw *e_raw;
877
878 if (cpu_feature_enabled(X86_FEATURE_RMPREAD)) {
879 int ret;
880
881 /* Binutils version 2.44 supports the RMPREAD mnemonic. */
882 asm volatile(".byte 0xf2, 0x0f, 0x01, 0xfd"
883 : "=a" (ret)
884 : "a" (pfn << PAGE_SHIFT), "c" (e)
885 : "memory", "cc");
886
887 return ret;
888 }
889
890 e_raw = get_raw_rmpentry(pfn);
891 if (IS_ERR(e_raw))
892 return PTR_ERR(e_raw);
893
894 /*
895 * Map the raw RMP table entry onto the RMPREAD output format.
896 * The 2MB region status indicator (hpage_region_status field) is not
897 * calculated, since the overhead could be significant and the field
898 * is not used.
899 */
900 memset(e, 0, sizeof(*e));
901 e->gpa = e_raw->gpa << PAGE_SHIFT;
902 e->asid = e_raw->asid;
903 e->assigned = e_raw->assigned;
904 e->pagesize = e_raw->pagesize;
905 e->immutable = e_raw->immutable;
906
907 return 0;
908 }
909
__snp_lookup_rmpentry(u64 pfn,struct rmpentry * e,int * level)910 static int __snp_lookup_rmpentry(u64 pfn, struct rmpentry *e, int *level)
911 {
912 struct rmpentry e_large;
913 int ret;
914
915 if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
916 return -ENODEV;
917
918 ret = get_rmpentry(pfn, e);
919 if (ret)
920 return ret;
921
922 /*
923 * Find the authoritative RMP entry for a PFN. This can be either a 4K
924 * RMP entry or a special large RMP entry that is authoritative for a
925 * whole 2M area.
926 */
927 ret = get_rmpentry(pfn & PFN_PMD_MASK, &e_large);
928 if (ret)
929 return ret;
930
931 *level = RMP_TO_PG_LEVEL(e_large.pagesize);
932
933 return 0;
934 }
935
snp_lookup_rmpentry(u64 pfn,bool * assigned,int * level)936 int snp_lookup_rmpentry(u64 pfn, bool *assigned, int *level)
937 {
938 struct rmpentry e;
939 int ret;
940
941 ret = __snp_lookup_rmpentry(pfn, &e, level);
942 if (ret)
943 return ret;
944
945 *assigned = !!e.assigned;
946 return 0;
947 }
948 EXPORT_SYMBOL_GPL(snp_lookup_rmpentry);
949
950 /*
951 * Dump the raw RMP entry for a particular PFN. These bits are documented in the
952 * PPR for a particular CPU model and provide useful information about how a
953 * particular PFN is being utilized by the kernel/firmware at the time certain
954 * unexpected events occur, such as RMP faults.
955 */
dump_rmpentry(u64 pfn)956 static void dump_rmpentry(u64 pfn)
957 {
958 struct rmpentry_raw *e_raw;
959 u64 pfn_i, pfn_end;
960 struct rmpentry e;
961 int level, ret;
962
963 ret = __snp_lookup_rmpentry(pfn, &e, &level);
964 if (ret) {
965 pr_err("Failed to read RMP entry for PFN 0x%llx, error %d\n",
966 pfn, ret);
967 return;
968 }
969
970 if (e.assigned) {
971 e_raw = get_raw_rmpentry(pfn);
972 if (IS_ERR(e_raw)) {
973 pr_err("Failed to read RMP contents for PFN 0x%llx, error %ld\n",
974 pfn, PTR_ERR(e_raw));
975 return;
976 }
977
978 pr_info("PFN 0x%llx, RMP entry: [0x%016llx - 0x%016llx]\n",
979 pfn, e_raw->lo, e_raw->hi);
980 return;
981 }
982
983 /*
984 * If the RMP entry for a particular PFN is not in an assigned state,
985 * then it is sometimes useful to get an idea of whether or not any RMP
986 * entries for other PFNs within the same 2MB region are assigned, since
987 * those too can affect the ability to access a particular PFN in
988 * certain situations, such as when the PFN is being accessed via a 2MB
989 * mapping in the host page table.
990 */
991 pfn_i = ALIGN_DOWN(pfn, PTRS_PER_PMD);
992 pfn_end = pfn_i + PTRS_PER_PMD;
993
994 pr_info("PFN 0x%llx unassigned, dumping non-zero entries in 2M PFN region: [0x%llx - 0x%llx]\n",
995 pfn, pfn_i, pfn_end);
996
997 while (pfn_i < pfn_end) {
998 e_raw = get_raw_rmpentry(pfn_i);
999 if (IS_ERR(e_raw)) {
1000 pr_err("Error %ld reading RMP contents for PFN 0x%llx\n",
1001 PTR_ERR(e_raw), pfn_i);
1002 pfn_i++;
1003 continue;
1004 }
1005
1006 if (e_raw->lo || e_raw->hi)
1007 pr_info("PFN: 0x%llx, [0x%016llx - 0x%016llx]\n", pfn_i, e_raw->lo, e_raw->hi);
1008 pfn_i++;
1009 }
1010 }
1011
snp_dump_hva_rmpentry(unsigned long hva)1012 void snp_dump_hva_rmpentry(unsigned long hva)
1013 {
1014 unsigned long paddr;
1015 unsigned int level;
1016 pgd_t *pgd;
1017 pte_t *pte;
1018
1019 pgd = __va(read_cr3_pa());
1020 pgd += pgd_index(hva);
1021 pte = lookup_address_in_pgd(pgd, hva, &level);
1022
1023 if (!pte) {
1024 pr_err("Can't dump RMP entry for HVA %lx: no PTE/PFN found\n", hva);
1025 return;
1026 }
1027
1028 paddr = PFN_PHYS(pte_pfn(*pte)) | (hva & ~page_level_mask(level));
1029 dump_rmpentry(PHYS_PFN(paddr));
1030 }
1031
1032 /*
1033 * PSMASH a 2MB aligned page into 4K pages in the RMP table while preserving the
1034 * Validated bit.
1035 */
psmash(u64 pfn)1036 int psmash(u64 pfn)
1037 {
1038 unsigned long paddr = pfn << PAGE_SHIFT;
1039 int ret;
1040
1041 if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
1042 return -ENODEV;
1043
1044 if (!pfn_valid(pfn))
1045 return -EINVAL;
1046
1047 /* Binutils version 2.36 supports the PSMASH mnemonic. */
1048 asm volatile(".byte 0xF3, 0x0F, 0x01, 0xFF"
1049 : "=a" (ret)
1050 : "a" (paddr)
1051 : "memory", "cc");
1052
1053 return ret;
1054 }
1055 EXPORT_SYMBOL_GPL(psmash);
1056
1057 /*
1058 * If the kernel uses a 2MB or larger directmap mapping to write to an address,
1059 * and that mapping contains any 4KB pages that are set to private in the RMP
1060 * table, an RMP #PF will trigger and cause a host crash. Hypervisor code that
1061 * owns the PFNs being transitioned will never attempt such a write, but other
1062 * kernel tasks writing to other PFNs in the range may trigger these checks
1063 * inadvertently due a large directmap mapping that happens to overlap such a
1064 * PFN.
1065 *
1066 * Prevent this by splitting any 2MB+ mappings that might end up containing a
1067 * mix of private/shared PFNs as a result of a subsequent RMPUPDATE for the
1068 * PFN/rmp_level passed in.
1069 *
1070 * Note that there is no attempt here to scan all the RMP entries for the 2MB
1071 * physical range, since it would only be worthwhile in determining if a
1072 * subsequent RMPUPDATE for a 4KB PFN would result in all the entries being of
1073 * the same shared/private state, thus avoiding the need to split the mapping.
1074 * But that would mean the entries are currently in a mixed state, and so the
1075 * mapping would have already been split as a result of prior transitions.
1076 * And since the 4K split is only done if the mapping is 2MB+, and there isn't
1077 * currently a mechanism in place to restore 2MB+ mappings, such a check would
1078 * not provide any usable benefit.
1079 *
1080 * More specifics on how these checks are carried out can be found in APM
1081 * Volume 2, "RMP and VMPL Access Checks".
1082 */
adjust_direct_map(u64 pfn,int rmp_level)1083 static int adjust_direct_map(u64 pfn, int rmp_level)
1084 {
1085 unsigned long vaddr;
1086 unsigned int level;
1087 int npages, ret;
1088 pte_t *pte;
1089
1090 /*
1091 * pfn_to_kaddr() will return a vaddr only within the direct
1092 * map range.
1093 */
1094 vaddr = (unsigned long)pfn_to_kaddr(pfn);
1095
1096 /* Only 4KB/2MB RMP entries are supported by current hardware. */
1097 if (WARN_ON_ONCE(rmp_level > PG_LEVEL_2M))
1098 return -EINVAL;
1099
1100 if (!pfn_valid(pfn))
1101 return -EINVAL;
1102
1103 if (rmp_level == PG_LEVEL_2M &&
1104 (!IS_ALIGNED(pfn, PTRS_PER_PMD) || !pfn_valid(pfn + PTRS_PER_PMD - 1)))
1105 return -EINVAL;
1106
1107 /*
1108 * If an entire 2MB physical range is being transitioned, then there is
1109 * no risk of RMP #PFs due to write accesses from overlapping mappings,
1110 * since even accesses from 1GB mappings will be treated as 2MB accesses
1111 * as far as RMP table checks are concerned.
1112 */
1113 if (rmp_level == PG_LEVEL_2M)
1114 return 0;
1115
1116 pte = lookup_address(vaddr, &level);
1117 if (!pte || pte_none(*pte))
1118 return 0;
1119
1120 if (level == PG_LEVEL_4K)
1121 return 0;
1122
1123 npages = page_level_size(rmp_level) / PAGE_SIZE;
1124 ret = set_memory_4k(vaddr, npages);
1125 if (ret)
1126 pr_warn("Failed to split direct map for PFN 0x%llx, ret: %d\n",
1127 pfn, ret);
1128
1129 return ret;
1130 }
1131
1132 /*
1133 * It is expected that those operations are seldom enough so that no mutual
1134 * exclusion of updaters is needed and thus the overlap error condition below
1135 * should happen very rarely and would get resolved relatively quickly by
1136 * the firmware.
1137 *
1138 * If not, one could consider introducing a mutex or so here to sync concurrent
1139 * RMP updates and thus diminish the amount of cases where firmware needs to
1140 * lock 2M ranges to protect against concurrent updates.
1141 *
1142 * The optimal solution would be range locking to avoid locking disjoint
1143 * regions unnecessarily but there's no support for that yet.
1144 */
rmpupdate(u64 pfn,struct rmp_state * state)1145 static int rmpupdate(u64 pfn, struct rmp_state *state)
1146 {
1147 unsigned long paddr = pfn << PAGE_SHIFT;
1148 int ret, level;
1149
1150 if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
1151 return -ENODEV;
1152
1153 level = RMP_TO_PG_LEVEL(state->pagesize);
1154
1155 if (adjust_direct_map(pfn, level))
1156 return -EFAULT;
1157
1158 do {
1159 /* Binutils version 2.36 supports the RMPUPDATE mnemonic. */
1160 asm volatile(".byte 0xF2, 0x0F, 0x01, 0xFE"
1161 : "=a" (ret)
1162 : "a" (paddr), "c" ((unsigned long)state)
1163 : "memory", "cc");
1164 } while (ret == RMPUPDATE_FAIL_OVERLAP);
1165
1166 if (ret) {
1167 pr_err("RMPUPDATE failed for PFN %llx, pg_level: %d, ret: %d\n",
1168 pfn, level, ret);
1169 dump_rmpentry(pfn);
1170 dump_stack();
1171 return -EFAULT;
1172 }
1173
1174 return 0;
1175 }
1176
1177 /* Transition a page to guest-owned/private state in the RMP table. */
rmp_make_private(u64 pfn,u64 gpa,enum pg_level level,u32 asid,bool immutable)1178 int rmp_make_private(u64 pfn, u64 gpa, enum pg_level level, u32 asid, bool immutable)
1179 {
1180 struct rmp_state state;
1181
1182 memset(&state, 0, sizeof(state));
1183 state.assigned = 1;
1184 state.asid = asid;
1185 state.immutable = immutable;
1186 state.gpa = gpa;
1187 state.pagesize = PG_LEVEL_TO_RMP(level);
1188
1189 return rmpupdate(pfn, &state);
1190 }
1191 EXPORT_SYMBOL_GPL(rmp_make_private);
1192
1193 /* Transition a page to hypervisor-owned/shared state in the RMP table. */
rmp_make_shared(u64 pfn,enum pg_level level)1194 int rmp_make_shared(u64 pfn, enum pg_level level)
1195 {
1196 struct rmp_state state;
1197
1198 memset(&state, 0, sizeof(state));
1199 state.pagesize = PG_LEVEL_TO_RMP(level);
1200
1201 return rmpupdate(pfn, &state);
1202 }
1203 EXPORT_SYMBOL_GPL(rmp_make_shared);
1204
__snp_leak_pages(u64 pfn,unsigned int npages,bool dump_rmp)1205 void __snp_leak_pages(u64 pfn, unsigned int npages, bool dump_rmp)
1206 {
1207 struct page *page = pfn_to_page(pfn);
1208
1209 pr_warn("Leaking PFN range 0x%llx-0x%llx\n", pfn, pfn + npages);
1210
1211 spin_lock(&snp_leaked_pages_list_lock);
1212 while (npages--) {
1213
1214 /*
1215 * Reuse the page's buddy list for chaining into the leaked
1216 * pages list. This page should not be on a free list currently
1217 * and is also unsafe to be added to a free list.
1218 */
1219 if (likely(!PageCompound(page)) ||
1220
1221 /*
1222 * Skip inserting tail pages of compound page as
1223 * page->buddy_list of tail pages is not usable.
1224 */
1225 (PageHead(page) && compound_nr(page) <= npages))
1226 list_add_tail(&page->buddy_list, &snp_leaked_pages_list);
1227
1228 if (dump_rmp)
1229 dump_rmpentry(pfn);
1230 snp_nr_leaked_pages++;
1231 pfn++;
1232 page++;
1233 }
1234 spin_unlock(&snp_leaked_pages_list_lock);
1235 }
1236 EXPORT_SYMBOL_GPL(__snp_leak_pages);
1237
kdump_sev_callback(void)1238 void kdump_sev_callback(void)
1239 {
1240 /*
1241 * Do wbinvd() on remote CPUs when SNP is enabled in order to
1242 * safely do SNP_SHUTDOWN on the local CPU.
1243 */
1244 if (cc_platform_has(CC_ATTR_HOST_SEV_SNP))
1245 wbinvd();
1246 }
1247