1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * AMD Encrypted Register State Support
4 *
5 * Author: Joerg Roedel <jroedel@suse.de>
6 */
7
8 /*
9 * misc.h needs to be first because it knows how to include the other kernel
10 * headers in the pre-decompression code in a way that does not break
11 * compilation.
12 */
13 #include "misc.h"
14
15 #include <asm/bootparam.h>
16 #include <asm/pgtable_types.h>
17 #include <asm/shared/msr.h>
18 #include <asm/sev.h>
19 #include <asm/trapnr.h>
20 #include <asm/trap_pf.h>
21 #include <asm/msr-index.h>
22 #include <asm/fpu/xcr.h>
23 #include <asm/ptrace.h>
24 #include <asm/svm.h>
25 #include <asm/cpuid/api.h>
26
27 #include "error.h"
28 #include "sev.h"
29
30 static struct ghcb boot_ghcb_page __aligned(PAGE_SIZE);
31 struct ghcb *boot_ghcb __section(".data");
32
33 #undef __init
34 #define __init
35
36 #define __BOOT_COMPRESSED
37
38 u8 snp_vmpl __section(".data");
39 u16 ghcb_version __section(".data");
40
41 u64 boot_svsm_caa_pa __section(".data");
42
43 /* Include code for early handlers */
44 #include "../../boot/startup/sev-shared.c"
45
sev_snp_enabled(void)46 static bool sev_snp_enabled(void)
47 {
48 return sev_status & MSR_AMD64_SEV_SNP_ENABLED;
49 }
50
snp_set_page_private(unsigned long paddr)51 void snp_set_page_private(unsigned long paddr)
52 {
53 struct psc_desc d = {
54 SNP_PAGE_STATE_PRIVATE,
55 (struct svsm_ca *)boot_svsm_caa_pa,
56 boot_svsm_caa_pa
57 };
58
59 if (!sev_snp_enabled())
60 return;
61
62 __page_state_change(paddr, paddr, &d);
63 }
64
snp_set_page_shared(unsigned long paddr)65 void snp_set_page_shared(unsigned long paddr)
66 {
67 struct psc_desc d = {
68 SNP_PAGE_STATE_SHARED,
69 (struct svsm_ca *)boot_svsm_caa_pa,
70 boot_svsm_caa_pa
71 };
72
73 if (!sev_snp_enabled())
74 return;
75
76 __page_state_change(paddr, paddr, &d);
77 }
78
early_setup_ghcb(void)79 bool early_setup_ghcb(void)
80 {
81 if (set_page_decrypted((unsigned long)&boot_ghcb_page))
82 return false;
83
84 /* Page is now mapped decrypted, clear it */
85 memset(&boot_ghcb_page, 0, sizeof(boot_ghcb_page));
86
87 boot_ghcb = &boot_ghcb_page;
88
89 /* Initialize lookup tables for the instruction decoder */
90 sev_insn_decode_init();
91
92 /* SNP guest requires the GHCB GPA must be registered */
93 if (sev_snp_enabled())
94 snp_register_ghcb_early(__pa(&boot_ghcb_page));
95
96 return true;
97 }
98
snp_accept_memory(phys_addr_t start,phys_addr_t end)99 void snp_accept_memory(phys_addr_t start, phys_addr_t end)
100 {
101 struct psc_desc d = {
102 SNP_PAGE_STATE_PRIVATE,
103 (struct svsm_ca *)boot_svsm_caa_pa,
104 boot_svsm_caa_pa
105 };
106
107 for (phys_addr_t pa = start; pa < end; pa += PAGE_SIZE)
108 __page_state_change(pa, pa, &d);
109 }
110
sev_es_shutdown_ghcb(void)111 void sev_es_shutdown_ghcb(void)
112 {
113 if (!boot_ghcb)
114 return;
115
116 if (!sev_es_check_cpu_features())
117 error("SEV-ES CPU Features missing.");
118
119 /*
120 * This denotes whether to use the GHCB MSR protocol or the GHCB
121 * shared page to perform a GHCB request. Since the GHCB page is
122 * being changed to encrypted, it can't be used to perform GHCB
123 * requests. Clear the boot_ghcb variable so that the GHCB MSR
124 * protocol is used to change the GHCB page over to an encrypted
125 * page.
126 */
127 boot_ghcb = NULL;
128
129 /*
130 * GHCB Page must be flushed from the cache and mapped encrypted again.
131 * Otherwise the running kernel will see strange cache effects when
132 * trying to use that page.
133 */
134 if (set_page_encrypted((unsigned long)&boot_ghcb_page))
135 error("Can't map GHCB page encrypted");
136
137 /*
138 * GHCB page is mapped encrypted again and flushed from the cache.
139 * Mark it non-present now to catch bugs when #VC exceptions trigger
140 * after this point.
141 */
142 if (set_page_non_present((unsigned long)&boot_ghcb_page))
143 error("Can't unmap GHCB page");
144 }
145
sev_es_ghcb_terminate(struct ghcb * ghcb,unsigned int set,unsigned int reason,u64 exit_info_2)146 static void __noreturn sev_es_ghcb_terminate(struct ghcb *ghcb, unsigned int set,
147 unsigned int reason, u64 exit_info_2)
148 {
149 u64 exit_info_1 = SVM_VMGEXIT_TERM_REASON(set, reason);
150
151 vc_ghcb_invalidate(ghcb);
152 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_TERM_REQUEST);
153 ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
154 ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
155
156 sev_es_wr_ghcb_msr(__pa(ghcb));
157 VMGEXIT();
158
159 while (true)
160 asm volatile("hlt\n" : : : "memory");
161 }
162
sev_es_check_ghcb_fault(unsigned long address)163 bool sev_es_check_ghcb_fault(unsigned long address)
164 {
165 /* Check whether the fault was on the GHCB page */
166 return ((address & PAGE_MASK) == (unsigned long)&boot_ghcb_page);
167 }
168
169 /*
170 * SNP_FEATURES_IMPL_REQ is the mask of SNP features that will need
171 * guest side implementation for proper functioning of the guest. If any
172 * of these features are enabled in the hypervisor but are lacking guest
173 * side implementation, the behavior of the guest will be undefined. The
174 * guest could fail in non-obvious way making it difficult to debug.
175 *
176 * As the behavior of reserved feature bits is unknown to be on the
177 * safe side add them to the required features mask.
178 */
179 #define SNP_FEATURES_IMPL_REQ (MSR_AMD64_SNP_VTOM | \
180 MSR_AMD64_SNP_REFLECT_VC | \
181 MSR_AMD64_SNP_RESTRICTED_INJ | \
182 MSR_AMD64_SNP_ALT_INJ | \
183 MSR_AMD64_SNP_DEBUG_SWAP | \
184 MSR_AMD64_SNP_VMPL_SSS | \
185 MSR_AMD64_SNP_SECURE_TSC | \
186 MSR_AMD64_SNP_VMGEXIT_PARAM | \
187 MSR_AMD64_SNP_VMSA_REG_PROT | \
188 MSR_AMD64_SNP_RESERVED_BIT13 | \
189 MSR_AMD64_SNP_RESERVED_BIT15 | \
190 MSR_AMD64_SNP_SECURE_AVIC | \
191 MSR_AMD64_SNP_RESERVED_BITS19_22 | \
192 MSR_AMD64_SNP_RESERVED_MASK)
193
194 #ifdef CONFIG_AMD_SECURE_AVIC
195 #define SNP_FEATURE_SECURE_AVIC MSR_AMD64_SNP_SECURE_AVIC
196 #else
197 #define SNP_FEATURE_SECURE_AVIC 0
198 #endif
199
200 /*
201 * SNP_FEATURES_PRESENT is the mask of SNP features that are implemented
202 * by the guest kernel. As and when a new feature is implemented in the
203 * guest kernel, a corresponding bit should be added to the mask.
204 */
205 #define SNP_FEATURES_PRESENT (MSR_AMD64_SNP_DEBUG_SWAP | \
206 MSR_AMD64_SNP_SECURE_TSC | \
207 SNP_FEATURE_SECURE_AVIC)
208
snp_get_unsupported_features(u64 status)209 u64 snp_get_unsupported_features(u64 status)
210 {
211 if (!(status & MSR_AMD64_SEV_SNP_ENABLED))
212 return 0;
213
214 return status & SNP_FEATURES_IMPL_REQ & ~SNP_FEATURES_PRESENT;
215 }
216
snp_check_features(void)217 void snp_check_features(void)
218 {
219 u64 unsupported;
220
221 /*
222 * Terminate the boot if hypervisor has enabled any feature lacking
223 * guest side implementation. Pass on the unsupported features mask through
224 * EXIT_INFO_2 of the GHCB protocol so that those features can be reported
225 * as part of the guest boot failure.
226 */
227 unsupported = snp_get_unsupported_features(sev_status);
228 if (unsupported) {
229 if (ghcb_version < 2 || (!boot_ghcb && !early_setup_ghcb()))
230 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
231
232 sev_es_ghcb_terminate(boot_ghcb, SEV_TERM_SET_GEN,
233 GHCB_SNP_UNSUPPORTED, unsupported);
234 }
235 }
236
237 /* Search for Confidential Computing blob in the EFI config table. */
find_cc_blob_efi(struct boot_params * bp)238 static struct cc_blob_sev_info *find_cc_blob_efi(struct boot_params *bp)
239 {
240 unsigned long cfg_table_pa;
241 unsigned int cfg_table_len;
242 int ret;
243
244 ret = efi_get_conf_table(bp, &cfg_table_pa, &cfg_table_len);
245 if (ret)
246 return NULL;
247
248 return (struct cc_blob_sev_info *)efi_find_vendor_table(bp, cfg_table_pa,
249 cfg_table_len,
250 EFI_CC_BLOB_GUID);
251 }
252
253 /*
254 * Initial set up of SNP relies on information provided by the
255 * Confidential Computing blob, which can be passed to the boot kernel
256 * by firmware/bootloader in the following ways:
257 *
258 * - via an entry in the EFI config table
259 * - via a setup_data structure, as defined by the Linux Boot Protocol
260 *
261 * Scan for the blob in that order.
262 */
find_cc_blob(struct boot_params * bp)263 static struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp)
264 {
265 struct cc_blob_sev_info *cc_info;
266
267 cc_info = find_cc_blob_efi(bp);
268 if (cc_info)
269 goto found_cc_info;
270
271 cc_info = find_cc_blob_setup_data(bp);
272 if (!cc_info)
273 return NULL;
274
275 found_cc_info:
276 if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC)
277 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
278
279 return cc_info;
280 }
281
282 /*
283 * Indicate SNP based on presence of SNP-specific CC blob. Subsequent checks
284 * will verify the SNP CPUID/MSR bits.
285 */
early_snp_init(struct boot_params * bp)286 static bool early_snp_init(struct boot_params *bp)
287 {
288 struct cc_blob_sev_info *cc_info;
289
290 if (!bp)
291 return false;
292
293 cc_info = find_cc_blob(bp);
294 if (!cc_info)
295 return false;
296
297 /*
298 * If a SNP-specific Confidential Computing blob is present, then
299 * firmware/bootloader have indicated SNP support. Verifying this
300 * involves CPUID checks which will be more reliable if the SNP
301 * CPUID table is used. See comments over snp_setup_cpuid_table() for
302 * more details.
303 */
304 setup_cpuid_table(cc_info);
305
306 /*
307 * Record the SVSM Calling Area (CA) address if the guest is not
308 * running at VMPL0. The CA will be used to communicate with the
309 * SVSM and request its services.
310 */
311 svsm_setup_ca(cc_info, rip_rel_ptr(&boot_ghcb_page));
312
313 /*
314 * Pass run-time kernel a pointer to CC info via boot_params so EFI
315 * config table doesn't need to be searched again during early startup
316 * phase.
317 */
318 bp->cc_blob_address = (u32)(unsigned long)cc_info;
319
320 return true;
321 }
322
323 /*
324 * sev_check_cpu_support - Check for SEV support in the CPU capabilities
325 *
326 * Returns < 0 if SEV is not supported, otherwise the position of the
327 * encryption bit in the page table descriptors.
328 */
sev_check_cpu_support(void)329 static int sev_check_cpu_support(void)
330 {
331 unsigned int eax, ebx, ecx, edx;
332
333 /* Check for the SME/SEV support leaf */
334 eax = 0x80000000;
335 ecx = 0;
336 native_cpuid(&eax, &ebx, &ecx, &edx);
337 if (eax < 0x8000001f)
338 return -ENODEV;
339
340 /*
341 * Check for the SME/SEV feature:
342 * CPUID Fn8000_001F[EAX]
343 * - Bit 0 - Secure Memory Encryption support
344 * - Bit 1 - Secure Encrypted Virtualization support
345 * CPUID Fn8000_001F[EBX]
346 * - Bits 5:0 - Pagetable bit position used to indicate encryption
347 */
348 eax = 0x8000001f;
349 ecx = 0;
350 native_cpuid(&eax, &ebx, &ecx, &edx);
351 /* Check whether SEV is supported */
352 if (!(eax & BIT(1)))
353 return -ENODEV;
354
355 sev_snp_needs_sfw = !(ebx & BIT(31));
356
357 return ebx & 0x3f;
358 }
359
sev_enable(struct boot_params * bp)360 void sev_enable(struct boot_params *bp)
361 {
362 struct msr m;
363 int bitpos;
364 bool snp;
365
366 /*
367 * bp->cc_blob_address should only be set by boot/compressed kernel.
368 * Initialize it to 0 to ensure that uninitialized values from
369 * buggy bootloaders aren't propagated.
370 */
371 if (bp)
372 bp->cc_blob_address = 0;
373
374 /*
375 * Do an initial SEV capability check before early_snp_init() which
376 * loads the CPUID page and the same checks afterwards are done
377 * without the hypervisor and are trustworthy.
378 *
379 * If the HV fakes SEV support, the guest will crash'n'burn
380 * which is good enough.
381 */
382
383 if (sev_check_cpu_support() < 0)
384 return;
385
386 /*
387 * Setup/preliminary detection of SNP. This will be sanity-checked
388 * against CPUID/MSR values later.
389 */
390 snp = early_snp_init(bp);
391
392 /* Now repeat the checks with the SNP CPUID table. */
393
394 bitpos = sev_check_cpu_support();
395 if (bitpos < 0) {
396 if (snp)
397 error("SEV-SNP support indicated by CC blob, but not CPUID.");
398 return;
399 }
400
401 /* Set the SME mask if this is an SEV guest. */
402 raw_rdmsr(MSR_AMD64_SEV, &m);
403 sev_status = m.q;
404 if (!(sev_status & MSR_AMD64_SEV_ENABLED))
405 return;
406
407 /* Negotiate the GHCB protocol version. */
408 if (sev_status & MSR_AMD64_SEV_ES_ENABLED) {
409 if (!sev_es_negotiate_protocol())
410 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_PROT_UNSUPPORTED);
411 }
412
413 /*
414 * SNP is supported in v2 of the GHCB spec which mandates support for HV
415 * features.
416 */
417 if (sev_status & MSR_AMD64_SEV_SNP_ENABLED) {
418 u64 hv_features;
419
420 hv_features = get_hv_features();
421 if (!(hv_features & GHCB_HV_FT_SNP))
422 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
423
424 /*
425 * Running at VMPL0 is required unless an SVSM is present and
426 * the hypervisor supports the required SVSM GHCB events.
427 */
428 if (snp_vmpl && !(hv_features & GHCB_HV_FT_SNP_MULTI_VMPL))
429 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_NOT_VMPL0);
430 }
431
432 if (snp && !(sev_status & MSR_AMD64_SEV_SNP_ENABLED))
433 error("SEV-SNP supported indicated by CC blob, but not SEV status MSR.");
434
435 sme_me_mask = BIT_ULL(bitpos);
436 }
437
438 /*
439 * sev_get_status - Retrieve the SEV status mask
440 *
441 * Returns 0 if the CPU is not SEV capable, otherwise the value of the
442 * AMD64_SEV MSR.
443 */
sev_get_status(void)444 u64 sev_get_status(void)
445 {
446 struct msr m;
447
448 if (sev_check_cpu_support() < 0)
449 return 0;
450
451 raw_rdmsr(MSR_AMD64_SEV, &m);
452 return m.q;
453 }
454
sev_prep_identity_maps(unsigned long top_level_pgt)455 void sev_prep_identity_maps(unsigned long top_level_pgt)
456 {
457 /*
458 * The Confidential Computing blob is used very early in uncompressed
459 * kernel to find the in-memory CPUID table to handle CPUID
460 * instructions. Make sure an identity-mapping exists so it can be
461 * accessed after switchover.
462 */
463 if (sev_snp_enabled()) {
464 unsigned long cc_info_pa = boot_params_ptr->cc_blob_address;
465 struct cc_blob_sev_info *cc_info;
466
467 kernel_add_identity_map(cc_info_pa, cc_info_pa + sizeof(*cc_info));
468
469 cc_info = (struct cc_blob_sev_info *)cc_info_pa;
470 kernel_add_identity_map(cc_info->cpuid_phys, cc_info->cpuid_phys + cc_info->cpuid_len);
471 }
472
473 sev_verify_cbit(top_level_pgt);
474 }
475
early_is_sevsnp_guest(void)476 bool early_is_sevsnp_guest(void)
477 {
478 static bool sevsnp;
479
480 if (sevsnp)
481 return true;
482
483 if (!(sev_get_status() & MSR_AMD64_SEV_SNP_ENABLED))
484 return false;
485
486 sevsnp = true;
487
488 if (!snp_vmpl) {
489 unsigned int eax, ebx, ecx, edx;
490
491 /*
492 * CPUID Fn8000_001F_EAX[28] - SVSM support
493 */
494 eax = 0x8000001f;
495 ecx = 0;
496 native_cpuid(&eax, &ebx, &ecx, &edx);
497 if (eax & BIT(28)) {
498 struct msr m;
499
500 /* Obtain the address of the calling area to use */
501 raw_rdmsr(MSR_SVSM_CAA, &m);
502 boot_svsm_caa_pa = m.q;
503
504 /*
505 * The real VMPL level cannot be discovered, but the
506 * memory acceptance routines make no use of that so
507 * any non-zero value suffices here.
508 */
509 snp_vmpl = U8_MAX;
510 }
511 }
512 return true;
513 }
514