1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * AMD Memory Encryption Support
4 *
5 * Copyright (C) 2019 SUSE
6 *
7 * Author: Joerg Roedel <jroedel@suse.de>
8 */
9
10 #define pr_fmt(fmt) "SEV: " fmt
11
12 #include <linux/sched/debug.h> /* For show_regs() */
13 #include <linux/percpu-defs.h>
14 #include <linux/cc_platform.h>
15 #include <linux/printk.h>
16 #include <linux/mm_types.h>
17 #include <linux/set_memory.h>
18 #include <linux/memblock.h>
19 #include <linux/kernel.h>
20 #include <linux/mm.h>
21 #include <linux/cpumask.h>
22 #include <linux/efi.h>
23 #include <linux/platform_device.h>
24 #include <linux/io.h>
25 #include <linux/psp-sev.h>
26 #include <linux/dmi.h>
27 #include <uapi/linux/sev-guest.h>
28 #include <crypto/aes-gcm.h>
29
30 #include <asm/init.h>
31 #include <asm/cpu_entry_area.h>
32 #include <asm/stacktrace.h>
33 #include <asm/sev.h>
34 #include <asm/insn-eval.h>
35 #include <asm/fpu/xcr.h>
36 #include <asm/processor.h>
37 #include <asm/realmode.h>
38 #include <asm/setup.h>
39 #include <asm/traps.h>
40 #include <asm/svm.h>
41 #include <asm/smp.h>
42 #include <asm/cpu.h>
43 #include <asm/apic.h>
44 #include <asm/cpuid/api.h>
45 #include <asm/cmdline.h>
46 #include <asm/msr.h>
47
48 #include "internal.h"
49
50 /* Bitmap of SEV features supported by the hypervisor */
51 u64 sev_hv_features __ro_after_init;
52 SYM_PIC_ALIAS(sev_hv_features);
53
54 /* Secrets page physical address from the CC blob */
55 u64 sev_secrets_pa __ro_after_init;
56 SYM_PIC_ALIAS(sev_secrets_pa);
57
58 /* AP INIT values as documented in the APM2 section "Processor Initialization State" */
59 #define AP_INIT_CS_LIMIT 0xffff
60 #define AP_INIT_DS_LIMIT 0xffff
61 #define AP_INIT_LDTR_LIMIT 0xffff
62 #define AP_INIT_GDTR_LIMIT 0xffff
63 #define AP_INIT_IDTR_LIMIT 0xffff
64 #define AP_INIT_TR_LIMIT 0xffff
65 #define AP_INIT_RFLAGS_DEFAULT 0x2
66 #define AP_INIT_DR6_DEFAULT 0xffff0ff0
67 #define AP_INIT_GPAT_DEFAULT 0x0007040600070406ULL
68 #define AP_INIT_XCR0_DEFAULT 0x1
69 #define AP_INIT_X87_FTW_DEFAULT 0x5555
70 #define AP_INIT_X87_FCW_DEFAULT 0x0040
71 #define AP_INIT_CR0_DEFAULT 0x60000010
72 #define AP_INIT_MXCSR_DEFAULT 0x1f80
73
74 static const char * const sev_status_feat_names[] = {
75 [MSR_AMD64_SEV_ENABLED_BIT] = "SEV",
76 [MSR_AMD64_SEV_ES_ENABLED_BIT] = "SEV-ES",
77 [MSR_AMD64_SEV_SNP_ENABLED_BIT] = "SEV-SNP",
78 [MSR_AMD64_SNP_VTOM_BIT] = "vTom",
79 [MSR_AMD64_SNP_REFLECT_VC_BIT] = "ReflectVC",
80 [MSR_AMD64_SNP_RESTRICTED_INJ_BIT] = "RI",
81 [MSR_AMD64_SNP_ALT_INJ_BIT] = "AI",
82 [MSR_AMD64_SNP_DEBUG_SWAP_BIT] = "DebugSwap",
83 [MSR_AMD64_SNP_PREVENT_HOST_IBS_BIT] = "NoHostIBS",
84 [MSR_AMD64_SNP_BTB_ISOLATION_BIT] = "BTBIsol",
85 [MSR_AMD64_SNP_VMPL_SSS_BIT] = "VmplSSS",
86 [MSR_AMD64_SNP_SECURE_TSC_BIT] = "SecureTSC",
87 [MSR_AMD64_SNP_VMGEXIT_PARAM_BIT] = "VMGExitParam",
88 [MSR_AMD64_SNP_IBS_VIRT_BIT] = "IBSVirt",
89 [MSR_AMD64_SNP_VMSA_REG_PROT_BIT] = "VMSARegProt",
90 [MSR_AMD64_SNP_SMT_PROT_BIT] = "SMTProt",
91 [MSR_AMD64_SNP_SECURE_AVIC_BIT] = "SecureAVIC",
92 [MSR_AMD64_SNP_IBPB_ON_ENTRY_BIT] = "IBPBOnEntry",
93 };
94
95 /*
96 * For Secure TSC guests, the BSP fetches TSC_INFO using SNP guest messaging and
97 * initializes snp_tsc_scale and snp_tsc_offset. These values are replicated
98 * across the APs VMSA fields (TSC_SCALE and TSC_OFFSET).
99 */
100 static u64 snp_tsc_scale __ro_after_init;
101 static u64 snp_tsc_offset __ro_after_init;
102 static unsigned long snp_tsc_freq_khz __ro_after_init;
103
104 DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data);
105 DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa);
106
107 /*
108 * SVSM related information:
109 * When running under an SVSM, the VMPL that Linux is executing at must be
110 * non-zero. The VMPL is therefore used to indicate the presence of an SVSM.
111 */
112 u8 snp_vmpl __ro_after_init;
113 EXPORT_SYMBOL_GPL(snp_vmpl);
114 SYM_PIC_ALIAS(snp_vmpl);
115
116 /*
117 * Since feature negotiation related variables are set early in the boot
118 * process they must reside in the .data section so as not to be zeroed
119 * out when the .bss section is later cleared.
120 *
121 * GHCB protocol version negotiated with the hypervisor.
122 */
123 u16 ghcb_version __ro_after_init;
124 SYM_PIC_ALIAS(ghcb_version);
125
126 /* For early boot hypervisor communication in SEV-ES enabled guests */
127 static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE);
128
129 /*
130 * Needs to be in the .data section because we need it NULL before bss is
131 * cleared
132 */
133 struct ghcb *boot_ghcb __section(".data");
134
get_snp_jump_table_addr(void)135 static u64 __init get_snp_jump_table_addr(void)
136 {
137 struct snp_secrets_page *secrets;
138 void __iomem *mem;
139 u64 addr;
140
141 mem = ioremap_encrypted(sev_secrets_pa, PAGE_SIZE);
142 if (!mem) {
143 pr_err("Unable to locate AP jump table address: failed to map the SNP secrets page.\n");
144 return 0;
145 }
146
147 secrets = (__force struct snp_secrets_page *)mem;
148
149 addr = secrets->os_area.ap_jump_table_pa;
150 iounmap(mem);
151
152 return addr;
153 }
154
get_jump_table_addr(void)155 static u64 __init get_jump_table_addr(void)
156 {
157 struct ghcb_state state;
158 unsigned long flags;
159 struct ghcb *ghcb;
160 u64 ret = 0;
161
162 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
163 return get_snp_jump_table_addr();
164
165 local_irq_save(flags);
166
167 ghcb = __sev_get_ghcb(&state);
168
169 vc_ghcb_invalidate(ghcb);
170 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_JUMP_TABLE);
171 ghcb_set_sw_exit_info_1(ghcb, SVM_VMGEXIT_GET_AP_JUMP_TABLE);
172 ghcb_set_sw_exit_info_2(ghcb, 0);
173
174 sev_es_wr_ghcb_msr(__pa(ghcb));
175 VMGEXIT();
176
177 if (ghcb_sw_exit_info_1_is_valid(ghcb) &&
178 ghcb_sw_exit_info_2_is_valid(ghcb))
179 ret = ghcb->save.sw_exit_info_2;
180
181 __sev_put_ghcb(&state);
182
183 local_irq_restore(flags);
184
185 return ret;
186 }
187
pval_pages(struct snp_psc_desc * desc)188 static void pval_pages(struct snp_psc_desc *desc)
189 {
190 struct psc_entry *e;
191 unsigned long vaddr;
192 unsigned int size;
193 unsigned int i;
194 bool validate;
195 u64 pfn;
196 int rc;
197
198 for (i = 0; i <= desc->hdr.end_entry; i++) {
199 e = &desc->entries[i];
200
201 pfn = e->gfn;
202 vaddr = (unsigned long)pfn_to_kaddr(pfn);
203 size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K;
204 validate = e->operation == SNP_PAGE_STATE_PRIVATE;
205
206 rc = pvalidate(vaddr, size, validate);
207 if (!rc)
208 continue;
209
210 if (rc == PVALIDATE_FAIL_SIZEMISMATCH && size == RMP_PG_SIZE_2M) {
211 unsigned long vaddr_end = vaddr + PMD_SIZE;
212
213 for (; vaddr < vaddr_end; vaddr += PAGE_SIZE, pfn++) {
214 rc = pvalidate(vaddr, RMP_PG_SIZE_4K, validate);
215 if (rc)
216 __pval_terminate(pfn, validate, RMP_PG_SIZE_4K, rc, 0);
217 }
218 } else {
219 __pval_terminate(pfn, validate, size, rc, 0);
220 }
221 }
222 }
223
pvalidate_pages(struct snp_psc_desc * desc)224 static void pvalidate_pages(struct snp_psc_desc *desc)
225 {
226 struct psc_entry *e;
227 unsigned int i;
228
229 if (snp_vmpl)
230 svsm_pval_pages(desc);
231 else
232 pval_pages(desc);
233
234 /*
235 * If not affected by the cache-coherency vulnerability there is no need
236 * to perform the cache eviction mitigation.
237 */
238 if (cpu_feature_enabled(X86_FEATURE_COHERENCY_SFW_NO))
239 return;
240
241 for (i = 0; i <= desc->hdr.end_entry; i++) {
242 e = &desc->entries[i];
243
244 /*
245 * If validating memory (making it private) perform the cache
246 * eviction mitigation.
247 */
248 if (e->operation == SNP_PAGE_STATE_PRIVATE)
249 sev_evict_cache(pfn_to_kaddr(e->gfn), e->pagesize ? 512 : 1);
250 }
251 }
252
vmgexit_psc(struct ghcb * ghcb,struct snp_psc_desc * desc)253 static int vmgexit_psc(struct ghcb *ghcb, struct snp_psc_desc *desc)
254 {
255 int cur_entry, end_entry, ret = 0;
256 struct snp_psc_desc *data;
257 struct es_em_ctxt ctxt;
258
259 vc_ghcb_invalidate(ghcb);
260
261 /* Copy the input desc into GHCB shared buffer */
262 data = (struct snp_psc_desc *)ghcb->shared_buffer;
263 memcpy(ghcb->shared_buffer, desc, min_t(int, GHCB_SHARED_BUF_SIZE, sizeof(*desc)));
264
265 /*
266 * As per the GHCB specification, the hypervisor can resume the guest
267 * before processing all the entries. Check whether all the entries
268 * are processed. If not, then keep retrying. Note, the hypervisor
269 * will update the data memory directly to indicate the status, so
270 * reference the data->hdr everywhere.
271 *
272 * The strategy here is to wait for the hypervisor to change the page
273 * state in the RMP table before guest accesses the memory pages. If the
274 * page state change was not successful, then later memory access will
275 * result in a crash.
276 */
277 cur_entry = data->hdr.cur_entry;
278 end_entry = data->hdr.end_entry;
279
280 while (data->hdr.cur_entry <= data->hdr.end_entry) {
281 ghcb_set_sw_scratch(ghcb, (u64)__pa(data));
282
283 /* This will advance the shared buffer data points to. */
284 ret = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_PSC, 0, 0);
285
286 /*
287 * Page State Change VMGEXIT can pass error code through
288 * exit_info_2.
289 */
290 if (WARN(ret || ghcb->save.sw_exit_info_2,
291 "SNP: PSC failed ret=%d exit_info_2=%llx\n",
292 ret, ghcb->save.sw_exit_info_2)) {
293 ret = 1;
294 goto out;
295 }
296
297 /* Verify that reserved bit is not set */
298 if (WARN(data->hdr.reserved, "Reserved bit is set in the PSC header\n")) {
299 ret = 1;
300 goto out;
301 }
302
303 /*
304 * Sanity check that entry processing is not going backwards.
305 * This will happen only if hypervisor is tricking us.
306 */
307 if (WARN(data->hdr.end_entry > end_entry || cur_entry > data->hdr.cur_entry,
308 "SNP: PSC processing going backward, end_entry %d (got %d) cur_entry %d (got %d)\n",
309 end_entry, data->hdr.end_entry, cur_entry, data->hdr.cur_entry)) {
310 ret = 1;
311 goto out;
312 }
313 }
314
315 out:
316 return ret;
317 }
318
__set_pages_state(struct snp_psc_desc * data,unsigned long vaddr,unsigned long vaddr_end,int op)319 static unsigned long __set_pages_state(struct snp_psc_desc *data, unsigned long vaddr,
320 unsigned long vaddr_end, int op)
321 {
322 struct ghcb_state state;
323 bool use_large_entry;
324 struct psc_hdr *hdr;
325 struct psc_entry *e;
326 unsigned long flags;
327 unsigned long pfn;
328 struct ghcb *ghcb;
329 int i;
330
331 hdr = &data->hdr;
332 e = data->entries;
333
334 memset(data, 0, sizeof(*data));
335 i = 0;
336
337 while (vaddr < vaddr_end && i < ARRAY_SIZE(data->entries)) {
338 hdr->end_entry = i;
339
340 if (is_vmalloc_addr((void *)vaddr)) {
341 pfn = vmalloc_to_pfn((void *)vaddr);
342 use_large_entry = false;
343 } else {
344 pfn = __pa(vaddr) >> PAGE_SHIFT;
345 use_large_entry = true;
346 }
347
348 e->gfn = pfn;
349 e->operation = op;
350
351 if (use_large_entry && IS_ALIGNED(vaddr, PMD_SIZE) &&
352 (vaddr_end - vaddr) >= PMD_SIZE) {
353 e->pagesize = RMP_PG_SIZE_2M;
354 vaddr += PMD_SIZE;
355 } else {
356 e->pagesize = RMP_PG_SIZE_4K;
357 vaddr += PAGE_SIZE;
358 }
359
360 e++;
361 i++;
362 }
363
364 /* Page validation must be rescinded before changing to shared */
365 if (op == SNP_PAGE_STATE_SHARED)
366 pvalidate_pages(data);
367
368 local_irq_save(flags);
369
370 ghcb = __sev_get_ghcb(&state);
371
372 /* Invoke the hypervisor to perform the page state changes */
373 if (!ghcb || vmgexit_psc(ghcb, data))
374 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
375
376 __sev_put_ghcb(&state);
377
378 local_irq_restore(flags);
379
380 /* Page validation must be performed after changing to private */
381 if (op == SNP_PAGE_STATE_PRIVATE)
382 pvalidate_pages(data);
383
384 return vaddr;
385 }
386
set_pages_state(unsigned long vaddr,unsigned long npages,int op)387 static void set_pages_state(unsigned long vaddr, unsigned long npages, int op)
388 {
389 struct snp_psc_desc desc;
390 unsigned long vaddr_end;
391
392 /* Use the MSR protocol when a GHCB is not available. */
393 if (!boot_ghcb) {
394 struct psc_desc d = { op, svsm_get_caa(), svsm_get_caa_pa() };
395
396 return early_set_pages_state(vaddr, __pa(vaddr), npages, &d);
397 }
398
399 vaddr = vaddr & PAGE_MASK;
400 vaddr_end = vaddr + (npages << PAGE_SHIFT);
401
402 while (vaddr < vaddr_end)
403 vaddr = __set_pages_state(&desc, vaddr, vaddr_end, op);
404 }
405
snp_set_memory_shared(unsigned long vaddr,unsigned long npages)406 void snp_set_memory_shared(unsigned long vaddr, unsigned long npages)
407 {
408 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
409 return;
410
411 set_pages_state(vaddr, npages, SNP_PAGE_STATE_SHARED);
412 }
413
snp_set_memory_private(unsigned long vaddr,unsigned long npages)414 void snp_set_memory_private(unsigned long vaddr, unsigned long npages)
415 {
416 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
417 return;
418
419 set_pages_state(vaddr, npages, SNP_PAGE_STATE_PRIVATE);
420 }
421
snp_accept_memory(phys_addr_t start,phys_addr_t end)422 void snp_accept_memory(phys_addr_t start, phys_addr_t end)
423 {
424 unsigned long vaddr, npages;
425
426 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
427 return;
428
429 vaddr = (unsigned long)__va(start);
430 npages = (end - start) >> PAGE_SHIFT;
431
432 set_pages_state(vaddr, npages, SNP_PAGE_STATE_PRIVATE);
433 }
434
vmgexit_ap_control(u64 event,struct sev_es_save_area * vmsa,u32 apic_id)435 static int vmgexit_ap_control(u64 event, struct sev_es_save_area *vmsa, u32 apic_id)
436 {
437 bool create = event != SVM_VMGEXIT_AP_DESTROY;
438 struct ghcb_state state;
439 unsigned long flags;
440 struct ghcb *ghcb;
441 int ret = 0;
442
443 local_irq_save(flags);
444
445 ghcb = __sev_get_ghcb(&state);
446
447 vc_ghcb_invalidate(ghcb);
448
449 if (create)
450 ghcb_set_rax(ghcb, vmsa->sev_features);
451
452 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_CREATION);
453 ghcb_set_sw_exit_info_1(ghcb,
454 ((u64)apic_id << 32) |
455 ((u64)snp_vmpl << 16) |
456 event);
457 ghcb_set_sw_exit_info_2(ghcb, __pa(vmsa));
458
459 sev_es_wr_ghcb_msr(__pa(ghcb));
460 VMGEXIT();
461
462 if (!ghcb_sw_exit_info_1_is_valid(ghcb) ||
463 lower_32_bits(ghcb->save.sw_exit_info_1)) {
464 pr_err("SNP AP %s error\n", (create ? "CREATE" : "DESTROY"));
465 ret = -EINVAL;
466 }
467
468 __sev_put_ghcb(&state);
469
470 local_irq_restore(flags);
471
472 return ret;
473 }
474
snp_set_vmsa(void * va,void * caa,int apic_id,bool make_vmsa)475 static int snp_set_vmsa(void *va, void *caa, int apic_id, bool make_vmsa)
476 {
477 int ret;
478
479 if (snp_vmpl) {
480 struct svsm_call call = {};
481 unsigned long flags;
482
483 local_irq_save(flags);
484
485 call.caa = this_cpu_read(svsm_caa);
486 call.rcx = __pa(va);
487
488 if (make_vmsa) {
489 /* Protocol 0, Call ID 2 */
490 call.rax = SVSM_CORE_CALL(SVSM_CORE_CREATE_VCPU);
491 call.rdx = __pa(caa);
492 call.r8 = apic_id;
493 } else {
494 /* Protocol 0, Call ID 3 */
495 call.rax = SVSM_CORE_CALL(SVSM_CORE_DELETE_VCPU);
496 }
497
498 ret = svsm_perform_call_protocol(&call);
499
500 local_irq_restore(flags);
501 } else {
502 /*
503 * If the kernel runs at VMPL0, it can change the VMSA
504 * bit for a page using the RMPADJUST instruction.
505 * However, for the instruction to succeed it must
506 * target the permissions of a lesser privileged (higher
507 * numbered) VMPL level, so use VMPL1.
508 */
509 u64 attrs = 1;
510
511 if (make_vmsa)
512 attrs |= RMPADJUST_VMSA_PAGE_BIT;
513
514 ret = rmpadjust((unsigned long)va, RMP_PG_SIZE_4K, attrs);
515 }
516
517 return ret;
518 }
519
snp_cleanup_vmsa(struct sev_es_save_area * vmsa,int apic_id)520 static void snp_cleanup_vmsa(struct sev_es_save_area *vmsa, int apic_id)
521 {
522 int err;
523
524 err = snp_set_vmsa(vmsa, NULL, apic_id, false);
525 if (err)
526 pr_err("clear VMSA page failed (%u), leaking page\n", err);
527 else
528 free_page((unsigned long)vmsa);
529 }
530
set_pte_enc(pte_t * kpte,int level,void * va)531 static void set_pte_enc(pte_t *kpte, int level, void *va)
532 {
533 struct pte_enc_desc d = {
534 .kpte = kpte,
535 .pte_level = level,
536 .va = va,
537 .encrypt = true
538 };
539
540 prepare_pte_enc(&d);
541 set_pte_enc_mask(kpte, d.pfn, d.new_pgprot);
542 }
543
unshare_all_memory(void)544 static void unshare_all_memory(void)
545 {
546 unsigned long addr, end, size, ghcb;
547 struct sev_es_runtime_data *data;
548 unsigned int npages, level;
549 bool skipped_addr;
550 pte_t *pte;
551 int cpu;
552
553 /* Unshare the direct mapping. */
554 addr = PAGE_OFFSET;
555 end = PAGE_OFFSET + get_max_mapped();
556
557 while (addr < end) {
558 pte = lookup_address(addr, &level);
559 size = page_level_size(level);
560 npages = size / PAGE_SIZE;
561 skipped_addr = false;
562
563 if (!pte || !pte_decrypted(*pte) || pte_none(*pte)) {
564 addr += size;
565 continue;
566 }
567
568 /*
569 * Ensure that all the per-CPU GHCBs are made private at the
570 * end of the unsharing loop so that the switch to the slower
571 * MSR protocol happens last.
572 */
573 for_each_possible_cpu(cpu) {
574 data = per_cpu(runtime_data, cpu);
575 ghcb = (unsigned long)&data->ghcb_page;
576
577 /* Handle the case of a huge page containing the GHCB page */
578 if (addr <= ghcb && ghcb < addr + size) {
579 skipped_addr = true;
580 break;
581 }
582 }
583
584 if (!skipped_addr) {
585 set_pte_enc(pte, level, (void *)addr);
586 snp_set_memory_private(addr, npages);
587 }
588 addr += size;
589 }
590
591 /* Unshare all bss decrypted memory. */
592 addr = (unsigned long)__start_bss_decrypted;
593 end = (unsigned long)__start_bss_decrypted_unused;
594 npages = (end - addr) >> PAGE_SHIFT;
595
596 for (; addr < end; addr += PAGE_SIZE) {
597 pte = lookup_address(addr, &level);
598 if (!pte || !pte_decrypted(*pte) || pte_none(*pte))
599 continue;
600
601 set_pte_enc(pte, level, (void *)addr);
602 }
603 addr = (unsigned long)__start_bss_decrypted;
604 snp_set_memory_private(addr, npages);
605
606 __flush_tlb_all();
607 }
608
609 /* Stop new private<->shared conversions */
snp_kexec_begin(void)610 void snp_kexec_begin(void)
611 {
612 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
613 return;
614
615 if (!IS_ENABLED(CONFIG_KEXEC_CORE))
616 return;
617
618 /*
619 * Crash kernel ends up here with interrupts disabled: can't wait for
620 * conversions to finish.
621 *
622 * If race happened, just report and proceed.
623 */
624 if (!set_memory_enc_stop_conversion())
625 pr_warn("Failed to stop shared<->private conversions\n");
626 }
627
628 /*
629 * Shutdown all APs except the one handling kexec/kdump and clearing
630 * the VMSA tag on AP's VMSA pages as they are not being used as
631 * VMSA page anymore.
632 */
shutdown_all_aps(void)633 static void shutdown_all_aps(void)
634 {
635 struct sev_es_save_area *vmsa;
636 int apic_id, this_cpu, cpu;
637
638 this_cpu = get_cpu();
639
640 /*
641 * APs are already in HLT loop when enc_kexec_finish() callback
642 * is invoked.
643 */
644 for_each_present_cpu(cpu) {
645 vmsa = per_cpu(sev_vmsa, cpu);
646
647 /*
648 * The BSP or offlined APs do not have guest allocated VMSA
649 * and there is no need to clear the VMSA tag for this page.
650 */
651 if (!vmsa)
652 continue;
653
654 /*
655 * Cannot clear the VMSA tag for the currently running vCPU.
656 */
657 if (this_cpu == cpu) {
658 unsigned long pa;
659 struct page *p;
660
661 pa = __pa(vmsa);
662 /*
663 * Mark the VMSA page of the running vCPU as offline
664 * so that is excluded and not touched by makedumpfile
665 * while generating vmcore during kdump.
666 */
667 p = pfn_to_online_page(pa >> PAGE_SHIFT);
668 if (p)
669 __SetPageOffline(p);
670 continue;
671 }
672
673 apic_id = cpuid_to_apicid[cpu];
674
675 /*
676 * Issue AP destroy to ensure AP gets kicked out of guest mode
677 * to allow using RMPADJUST to remove the VMSA tag on it's
678 * VMSA page.
679 */
680 vmgexit_ap_control(SVM_VMGEXIT_AP_DESTROY, vmsa, apic_id);
681 snp_cleanup_vmsa(vmsa, apic_id);
682 }
683
684 put_cpu();
685 }
686
snp_kexec_finish(void)687 void snp_kexec_finish(void)
688 {
689 struct sev_es_runtime_data *data;
690 unsigned long size, addr;
691 unsigned int level, cpu;
692 struct ghcb *ghcb;
693 pte_t *pte;
694
695 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
696 return;
697
698 if (!IS_ENABLED(CONFIG_KEXEC_CORE))
699 return;
700
701 shutdown_all_aps();
702
703 unshare_all_memory();
704
705 /*
706 * Switch to using the MSR protocol to change per-CPU GHCBs to
707 * private. All the per-CPU GHCBs have been switched back to private,
708 * so can't do any more GHCB calls to the hypervisor beyond this point
709 * until the kexec'ed kernel starts running.
710 */
711 boot_ghcb = NULL;
712 sev_cfg.ghcbs_initialized = false;
713
714 for_each_possible_cpu(cpu) {
715 data = per_cpu(runtime_data, cpu);
716 ghcb = &data->ghcb_page;
717 pte = lookup_address((unsigned long)ghcb, &level);
718 size = page_level_size(level);
719 /* Handle the case of a huge page containing the GHCB page */
720 addr = (unsigned long)ghcb & page_level_mask(level);
721 set_pte_enc(pte, level, (void *)addr);
722 snp_set_memory_private(addr, (size / PAGE_SIZE));
723 }
724 }
725
726 #define __ATTR_BASE (SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK)
727 #define INIT_CS_ATTRIBS (__ATTR_BASE | SVM_SELECTOR_READ_MASK | SVM_SELECTOR_CODE_MASK)
728 #define INIT_DS_ATTRIBS (__ATTR_BASE | SVM_SELECTOR_WRITE_MASK)
729
730 #define INIT_LDTR_ATTRIBS (SVM_SELECTOR_P_MASK | 2)
731 #define INIT_TR_ATTRIBS (SVM_SELECTOR_P_MASK | 3)
732
snp_alloc_vmsa_page(int cpu)733 static void *snp_alloc_vmsa_page(int cpu)
734 {
735 struct page *p;
736
737 /*
738 * Allocate VMSA page to work around the SNP erratum where the CPU will
739 * incorrectly signal an RMP violation #PF if a large page (2MB or 1GB)
740 * collides with the RMP entry of VMSA page. The recommended workaround
741 * is to not use a large page.
742 *
743 * Allocate an 8k page which is also 8k-aligned.
744 */
745 p = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL_ACCOUNT | __GFP_ZERO, 1);
746 if (!p)
747 return NULL;
748
749 split_page(p, 1);
750
751 /* Free the first 4k. This page may be 2M/1G aligned and cannot be used. */
752 __free_page(p);
753
754 return page_address(p + 1);
755 }
756
wakeup_cpu_via_vmgexit(u32 apic_id,unsigned long start_ip,unsigned int cpu)757 static int wakeup_cpu_via_vmgexit(u32 apic_id, unsigned long start_ip, unsigned int cpu)
758 {
759 struct sev_es_save_area *cur_vmsa, *vmsa;
760 struct svsm_ca *caa;
761 u8 sipi_vector;
762 int ret;
763 u64 cr4;
764
765 /*
766 * The hypervisor SNP feature support check has happened earlier, just check
767 * the AP_CREATION one here.
768 */
769 if (!(sev_hv_features & GHCB_HV_FT_SNP_AP_CREATION))
770 return -EOPNOTSUPP;
771
772 /*
773 * Verify the desired start IP against the known trampoline start IP
774 * to catch any future new trampolines that may be introduced that
775 * would require a new protected guest entry point.
776 */
777 if (WARN_ONCE(start_ip != real_mode_header->trampoline_start,
778 "Unsupported SNP start_ip: %lx\n", start_ip))
779 return -EINVAL;
780
781 /* Override start_ip with known protected guest start IP */
782 start_ip = real_mode_header->sev_es_trampoline_start;
783 cur_vmsa = per_cpu(sev_vmsa, cpu);
784
785 /*
786 * A new VMSA is created each time because there is no guarantee that
787 * the current VMSA is the kernels or that the vCPU is not running. If
788 * an attempt was done to use the current VMSA with a running vCPU, a
789 * #VMEXIT of that vCPU would wipe out all of the settings being done
790 * here.
791 */
792 vmsa = (struct sev_es_save_area *)snp_alloc_vmsa_page(cpu);
793 if (!vmsa)
794 return -ENOMEM;
795
796 /* If an SVSM is present, the SVSM per-CPU CAA will be !NULL */
797 caa = per_cpu(svsm_caa, cpu);
798
799 /* CR4 should maintain the MCE value */
800 cr4 = native_read_cr4() & X86_CR4_MCE;
801
802 /* Set the CS value based on the start_ip converted to a SIPI vector */
803 sipi_vector = (start_ip >> 12);
804 vmsa->cs.base = sipi_vector << 12;
805 vmsa->cs.limit = AP_INIT_CS_LIMIT;
806 vmsa->cs.attrib = INIT_CS_ATTRIBS;
807 vmsa->cs.selector = sipi_vector << 8;
808
809 /* Set the RIP value based on start_ip */
810 vmsa->rip = start_ip & 0xfff;
811
812 /* Set AP INIT defaults as documented in the APM */
813 vmsa->ds.limit = AP_INIT_DS_LIMIT;
814 vmsa->ds.attrib = INIT_DS_ATTRIBS;
815 vmsa->es = vmsa->ds;
816 vmsa->fs = vmsa->ds;
817 vmsa->gs = vmsa->ds;
818 vmsa->ss = vmsa->ds;
819
820 vmsa->gdtr.limit = AP_INIT_GDTR_LIMIT;
821 vmsa->ldtr.limit = AP_INIT_LDTR_LIMIT;
822 vmsa->ldtr.attrib = INIT_LDTR_ATTRIBS;
823 vmsa->idtr.limit = AP_INIT_IDTR_LIMIT;
824 vmsa->tr.limit = AP_INIT_TR_LIMIT;
825 vmsa->tr.attrib = INIT_TR_ATTRIBS;
826
827 vmsa->cr4 = cr4;
828 vmsa->cr0 = AP_INIT_CR0_DEFAULT;
829 vmsa->dr7 = DR7_RESET_VALUE;
830 vmsa->dr6 = AP_INIT_DR6_DEFAULT;
831 vmsa->rflags = AP_INIT_RFLAGS_DEFAULT;
832 vmsa->g_pat = AP_INIT_GPAT_DEFAULT;
833 vmsa->xcr0 = AP_INIT_XCR0_DEFAULT;
834 vmsa->mxcsr = AP_INIT_MXCSR_DEFAULT;
835 vmsa->x87_ftw = AP_INIT_X87_FTW_DEFAULT;
836 vmsa->x87_fcw = AP_INIT_X87_FCW_DEFAULT;
837
838 if (cc_platform_has(CC_ATTR_SNP_SECURE_AVIC))
839 vmsa->vintr_ctrl |= V_GIF_MASK | V_NMI_ENABLE_MASK;
840
841 /* SVME must be set. */
842 vmsa->efer = EFER_SVME;
843
844 /*
845 * Set the SNP-specific fields for this VMSA:
846 * VMPL level
847 * SEV_FEATURES (matches the SEV STATUS MSR right shifted 2 bits)
848 */
849 vmsa->vmpl = snp_vmpl;
850 vmsa->sev_features = sev_status >> 2;
851
852 /* Populate AP's TSC scale/offset to get accurate TSC values. */
853 if (cc_platform_has(CC_ATTR_GUEST_SNP_SECURE_TSC)) {
854 vmsa->tsc_scale = snp_tsc_scale;
855 vmsa->tsc_offset = snp_tsc_offset;
856 }
857
858 /* Switch the page over to a VMSA page now that it is initialized */
859 ret = snp_set_vmsa(vmsa, caa, apic_id, true);
860 if (ret) {
861 pr_err("set VMSA page failed (%u)\n", ret);
862 free_page((unsigned long)vmsa);
863
864 return -EINVAL;
865 }
866
867 /* Issue VMGEXIT AP Creation NAE event */
868 ret = vmgexit_ap_control(SVM_VMGEXIT_AP_CREATE, vmsa, apic_id);
869 if (ret) {
870 snp_cleanup_vmsa(vmsa, apic_id);
871 vmsa = NULL;
872 }
873
874 /* Free up any previous VMSA page */
875 if (cur_vmsa)
876 snp_cleanup_vmsa(cur_vmsa, apic_id);
877
878 /* Record the current VMSA page */
879 per_cpu(sev_vmsa, cpu) = vmsa;
880
881 return ret;
882 }
883
snp_set_wakeup_secondary_cpu(void)884 void __init snp_set_wakeup_secondary_cpu(void)
885 {
886 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
887 return;
888
889 /*
890 * Always set this override if SNP is enabled. This makes it the
891 * required method to start APs under SNP. If the hypervisor does
892 * not support AP creation, then no APs will be started.
893 */
894 apic_update_callback(wakeup_secondary_cpu, wakeup_cpu_via_vmgexit);
895 }
896
sev_es_setup_ap_jump_table(struct real_mode_header * rmh)897 int __init sev_es_setup_ap_jump_table(struct real_mode_header *rmh)
898 {
899 u16 startup_cs, startup_ip;
900 phys_addr_t jump_table_pa;
901 u64 jump_table_addr;
902 u16 __iomem *jump_table;
903
904 jump_table_addr = get_jump_table_addr();
905
906 /* On UP guests there is no jump table so this is not a failure */
907 if (!jump_table_addr)
908 return 0;
909
910 /* Check if AP Jump Table is page-aligned */
911 if (jump_table_addr & ~PAGE_MASK)
912 return -EINVAL;
913
914 jump_table_pa = jump_table_addr & PAGE_MASK;
915
916 startup_cs = (u16)(rmh->trampoline_start >> 4);
917 startup_ip = (u16)(rmh->sev_es_trampoline_start -
918 rmh->trampoline_start);
919
920 jump_table = ioremap_encrypted(jump_table_pa, PAGE_SIZE);
921 if (!jump_table)
922 return -EIO;
923
924 writew(startup_ip, &jump_table[0]);
925 writew(startup_cs, &jump_table[1]);
926
927 iounmap(jump_table);
928
929 return 0;
930 }
931
932 /*
933 * This is needed by the OVMF UEFI firmware which will use whatever it finds in
934 * the GHCB MSR as its GHCB to talk to the hypervisor. So make sure the per-cpu
935 * runtime GHCBs used by the kernel are also mapped in the EFI page-table.
936 *
937 * When running under SVSM the CA page is needed too, so map it as well.
938 */
sev_es_efi_map_ghcbs_cas(pgd_t * pgd)939 int __init sev_es_efi_map_ghcbs_cas(pgd_t *pgd)
940 {
941 unsigned long address, pflags, pflags_enc;
942 struct sev_es_runtime_data *data;
943 int cpu;
944 u64 pfn;
945
946 if (!cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT))
947 return 0;
948
949 pflags = _PAGE_NX | _PAGE_RW;
950 pflags_enc = cc_mkenc(pflags);
951
952 for_each_possible_cpu(cpu) {
953 data = per_cpu(runtime_data, cpu);
954
955 address = __pa(&data->ghcb_page);
956 pfn = address >> PAGE_SHIFT;
957
958 if (kernel_map_pages_in_pgd(pgd, pfn, address, 1, pflags))
959 return 1;
960
961 if (snp_vmpl) {
962 address = per_cpu(svsm_caa_pa, cpu);
963 if (!address)
964 return 1;
965
966 pfn = address >> PAGE_SHIFT;
967 if (kernel_map_pages_in_pgd(pgd, pfn, address, 1, pflags_enc))
968 return 1;
969 }
970 }
971
972 return 0;
973 }
974
savic_ghcb_msr_read(u32 reg)975 u64 savic_ghcb_msr_read(u32 reg)
976 {
977 u64 msr = APIC_BASE_MSR + (reg >> 4);
978 struct pt_regs regs = { .cx = msr };
979 struct es_em_ctxt ctxt = { .regs = ®s };
980 struct ghcb_state state;
981 enum es_result res;
982 struct ghcb *ghcb;
983
984 guard(irqsave)();
985
986 ghcb = __sev_get_ghcb(&state);
987 vc_ghcb_invalidate(ghcb);
988
989 res = __vc_handle_msr(ghcb, &ctxt, false);
990 if (res != ES_OK) {
991 pr_err("Secure AVIC MSR (0x%llx) read returned error (%d)\n", msr, res);
992 /* MSR read failures are treated as fatal errors */
993 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SAVIC_FAIL);
994 }
995
996 __sev_put_ghcb(&state);
997
998 return regs.ax | regs.dx << 32;
999 }
1000
savic_ghcb_msr_write(u32 reg,u64 value)1001 void savic_ghcb_msr_write(u32 reg, u64 value)
1002 {
1003 u64 msr = APIC_BASE_MSR + (reg >> 4);
1004 struct pt_regs regs = {
1005 .cx = msr,
1006 .ax = lower_32_bits(value),
1007 .dx = upper_32_bits(value)
1008 };
1009 struct es_em_ctxt ctxt = { .regs = ®s };
1010 struct ghcb_state state;
1011 enum es_result res;
1012 struct ghcb *ghcb;
1013
1014 guard(irqsave)();
1015
1016 ghcb = __sev_get_ghcb(&state);
1017 vc_ghcb_invalidate(ghcb);
1018
1019 res = __vc_handle_msr(ghcb, &ctxt, true);
1020 if (res != ES_OK) {
1021 pr_err("Secure AVIC MSR (0x%llx) write returned error (%d)\n", msr, res);
1022 /* MSR writes should never fail. Any failure is fatal error for SNP guest */
1023 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SAVIC_FAIL);
1024 }
1025
1026 __sev_put_ghcb(&state);
1027 }
1028
savic_register_gpa(u64 gpa)1029 enum es_result savic_register_gpa(u64 gpa)
1030 {
1031 struct ghcb_state state;
1032 struct es_em_ctxt ctxt;
1033 enum es_result res;
1034 struct ghcb *ghcb;
1035
1036 guard(irqsave)();
1037
1038 ghcb = __sev_get_ghcb(&state);
1039 vc_ghcb_invalidate(ghcb);
1040
1041 ghcb_set_rax(ghcb, SVM_VMGEXIT_SAVIC_SELF_GPA);
1042 ghcb_set_rbx(ghcb, gpa);
1043 res = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_SAVIC,
1044 SVM_VMGEXIT_SAVIC_REGISTER_GPA, 0);
1045
1046 __sev_put_ghcb(&state);
1047
1048 return res;
1049 }
1050
savic_unregister_gpa(u64 * gpa)1051 enum es_result savic_unregister_gpa(u64 *gpa)
1052 {
1053 struct ghcb_state state;
1054 struct es_em_ctxt ctxt;
1055 enum es_result res;
1056 struct ghcb *ghcb;
1057
1058 guard(irqsave)();
1059
1060 ghcb = __sev_get_ghcb(&state);
1061 vc_ghcb_invalidate(ghcb);
1062
1063 ghcb_set_rax(ghcb, SVM_VMGEXIT_SAVIC_SELF_GPA);
1064 res = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_SAVIC,
1065 SVM_VMGEXIT_SAVIC_UNREGISTER_GPA, 0);
1066 if (gpa && res == ES_OK)
1067 *gpa = ghcb->save.rbx;
1068
1069 __sev_put_ghcb(&state);
1070
1071 return res;
1072 }
1073
snp_register_per_cpu_ghcb(void)1074 static void snp_register_per_cpu_ghcb(void)
1075 {
1076 struct sev_es_runtime_data *data;
1077 struct ghcb *ghcb;
1078
1079 data = this_cpu_read(runtime_data);
1080 ghcb = &data->ghcb_page;
1081
1082 snp_register_ghcb_early(__pa(ghcb));
1083 }
1084
setup_ghcb(void)1085 void setup_ghcb(void)
1086 {
1087 if (!cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT))
1088 return;
1089
1090 /*
1091 * Check whether the runtime #VC exception handler is active. It uses
1092 * the per-CPU GHCB page which is set up by sev_es_init_vc_handling().
1093 *
1094 * If SNP is active, register the per-CPU GHCB page so that the runtime
1095 * exception handler can use it.
1096 */
1097 if (initial_vc_handler == (unsigned long)kernel_exc_vmm_communication) {
1098 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
1099 snp_register_per_cpu_ghcb();
1100
1101 sev_cfg.ghcbs_initialized = true;
1102
1103 return;
1104 }
1105
1106 /*
1107 * Make sure the hypervisor talks a supported protocol.
1108 * This gets called only in the BSP boot phase.
1109 */
1110 if (!sev_es_negotiate_protocol())
1111 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
1112
1113 /*
1114 * Clear the boot_ghcb. The first exception comes in before the bss
1115 * section is cleared.
1116 */
1117 memset(&boot_ghcb_page, 0, PAGE_SIZE);
1118
1119 /* Alright - Make the boot-ghcb public */
1120 boot_ghcb = &boot_ghcb_page;
1121
1122 /* SNP guest requires that GHCB GPA must be registered. */
1123 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
1124 snp_register_ghcb_early(__pa(&boot_ghcb_page));
1125 }
1126
1127 #ifdef CONFIG_HOTPLUG_CPU
sev_es_ap_hlt_loop(void)1128 static void sev_es_ap_hlt_loop(void)
1129 {
1130 struct ghcb_state state;
1131 struct ghcb *ghcb;
1132
1133 ghcb = __sev_get_ghcb(&state);
1134
1135 while (true) {
1136 vc_ghcb_invalidate(ghcb);
1137 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_HLT_LOOP);
1138 ghcb_set_sw_exit_info_1(ghcb, 0);
1139 ghcb_set_sw_exit_info_2(ghcb, 0);
1140
1141 sev_es_wr_ghcb_msr(__pa(ghcb));
1142 VMGEXIT();
1143
1144 /* Wakeup signal? */
1145 if (ghcb_sw_exit_info_2_is_valid(ghcb) &&
1146 ghcb->save.sw_exit_info_2)
1147 break;
1148 }
1149
1150 __sev_put_ghcb(&state);
1151 }
1152
1153 /*
1154 * Play_dead handler when running under SEV-ES. This is needed because
1155 * the hypervisor can't deliver an SIPI request to restart the AP.
1156 * Instead the kernel has to issue a VMGEXIT to halt the VCPU until the
1157 * hypervisor wakes it up again.
1158 */
sev_es_play_dead(void)1159 static void sev_es_play_dead(void)
1160 {
1161 play_dead_common();
1162
1163 /* IRQs now disabled */
1164
1165 sev_es_ap_hlt_loop();
1166
1167 /*
1168 * If we get here, the VCPU was woken up again. Jump to CPU
1169 * startup code to get it back online.
1170 */
1171 soft_restart_cpu();
1172 }
1173 #else /* CONFIG_HOTPLUG_CPU */
1174 #define sev_es_play_dead native_play_dead
1175 #endif /* CONFIG_HOTPLUG_CPU */
1176
1177 #ifdef CONFIG_SMP
sev_es_setup_play_dead(void)1178 static void __init sev_es_setup_play_dead(void)
1179 {
1180 smp_ops.play_dead = sev_es_play_dead;
1181 }
1182 #else
sev_es_setup_play_dead(void)1183 static inline void sev_es_setup_play_dead(void) { }
1184 #endif
1185
alloc_runtime_data(int cpu)1186 static void __init alloc_runtime_data(int cpu)
1187 {
1188 struct sev_es_runtime_data *data;
1189
1190 data = memblock_alloc_node(sizeof(*data), PAGE_SIZE, cpu_to_node(cpu));
1191 if (!data)
1192 panic("Can't allocate SEV-ES runtime data");
1193
1194 per_cpu(runtime_data, cpu) = data;
1195
1196 if (snp_vmpl) {
1197 struct svsm_ca *caa;
1198
1199 /* Allocate the SVSM CA page if an SVSM is present */
1200 caa = cpu ? memblock_alloc_or_panic(sizeof(*caa), PAGE_SIZE)
1201 : &boot_svsm_ca_page;
1202
1203 per_cpu(svsm_caa, cpu) = caa;
1204 per_cpu(svsm_caa_pa, cpu) = __pa(caa);
1205 }
1206 }
1207
init_ghcb(int cpu)1208 static void __init init_ghcb(int cpu)
1209 {
1210 struct sev_es_runtime_data *data;
1211 int err;
1212
1213 data = per_cpu(runtime_data, cpu);
1214
1215 err = early_set_memory_decrypted((unsigned long)&data->ghcb_page,
1216 sizeof(data->ghcb_page));
1217 if (err)
1218 panic("Can't map GHCBs unencrypted");
1219
1220 memset(&data->ghcb_page, 0, sizeof(data->ghcb_page));
1221
1222 data->ghcb_active = false;
1223 data->backup_ghcb_active = false;
1224 }
1225
sev_es_init_vc_handling(void)1226 void __init sev_es_init_vc_handling(void)
1227 {
1228 int cpu;
1229
1230 BUILD_BUG_ON(offsetof(struct sev_es_runtime_data, ghcb_page) % PAGE_SIZE);
1231
1232 if (!cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT))
1233 return;
1234
1235 if (!sev_es_check_cpu_features())
1236 panic("SEV-ES CPU Features missing");
1237
1238 /*
1239 * SNP is supported in v2 of the GHCB spec which mandates support for HV
1240 * features.
1241 */
1242 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) {
1243 sev_hv_features = get_hv_features();
1244
1245 if (!(sev_hv_features & GHCB_HV_FT_SNP))
1246 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
1247 }
1248
1249 /* Initialize per-cpu GHCB pages */
1250 for_each_possible_cpu(cpu) {
1251 alloc_runtime_data(cpu);
1252 init_ghcb(cpu);
1253 }
1254
1255 if (snp_vmpl)
1256 sev_cfg.use_cas = true;
1257
1258 sev_es_setup_play_dead();
1259
1260 /* Secondary CPUs use the runtime #VC handler */
1261 initial_vc_handler = (unsigned long)kernel_exc_vmm_communication;
1262 }
1263
1264 /*
1265 * SEV-SNP guests should only execute dmi_setup() if EFI_CONFIG_TABLES are
1266 * enabled, as the alternative (fallback) logic for DMI probing in the legacy
1267 * ROM region can cause a crash since this region is not pre-validated.
1268 */
snp_dmi_setup(void)1269 void __init snp_dmi_setup(void)
1270 {
1271 if (efi_enabled(EFI_CONFIG_TABLES))
1272 dmi_setup();
1273 }
1274
dump_cpuid_table(void)1275 static void dump_cpuid_table(void)
1276 {
1277 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
1278 int i = 0;
1279
1280 pr_info("count=%d reserved=0x%x reserved2=0x%llx\n",
1281 cpuid_table->count, cpuid_table->__reserved1, cpuid_table->__reserved2);
1282
1283 for (i = 0; i < SNP_CPUID_COUNT_MAX; i++) {
1284 const struct snp_cpuid_fn *fn = &cpuid_table->fn[i];
1285
1286 pr_info("index=%3d fn=0x%08x subfn=0x%08x: eax=0x%08x ebx=0x%08x ecx=0x%08x edx=0x%08x xcr0_in=0x%016llx xss_in=0x%016llx reserved=0x%016llx\n",
1287 i, fn->eax_in, fn->ecx_in, fn->eax, fn->ebx, fn->ecx,
1288 fn->edx, fn->xcr0_in, fn->xss_in, fn->__reserved);
1289 }
1290 }
1291
1292 /*
1293 * It is useful from an auditing/testing perspective to provide an easy way
1294 * for the guest owner to know that the CPUID table has been initialized as
1295 * expected, but that initialization happens too early in boot to print any
1296 * sort of indicator, and there's not really any other good place to do it,
1297 * so do it here.
1298 *
1299 * If running as an SNP guest, report the current VM privilege level (VMPL).
1300 */
report_snp_info(void)1301 static int __init report_snp_info(void)
1302 {
1303 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
1304
1305 if (cpuid_table->count) {
1306 pr_info("Using SNP CPUID table, %d entries present.\n",
1307 cpuid_table->count);
1308
1309 if (sev_cfg.debug)
1310 dump_cpuid_table();
1311 }
1312
1313 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
1314 pr_info("SNP running at VMPL%u.\n", snp_vmpl);
1315
1316 return 0;
1317 }
1318 arch_initcall(report_snp_info);
1319
snp_issue_guest_request(struct snp_guest_req * req)1320 static int snp_issue_guest_request(struct snp_guest_req *req)
1321 {
1322 struct snp_req_data *input = &req->input;
1323 struct ghcb_state state;
1324 struct es_em_ctxt ctxt;
1325 unsigned long flags;
1326 struct ghcb *ghcb;
1327 int ret;
1328
1329 req->exitinfo2 = SEV_RET_NO_FW_CALL;
1330
1331 /*
1332 * __sev_get_ghcb() needs to run with IRQs disabled because it is using
1333 * a per-CPU GHCB.
1334 */
1335 local_irq_save(flags);
1336
1337 ghcb = __sev_get_ghcb(&state);
1338 if (!ghcb) {
1339 ret = -EIO;
1340 goto e_restore_irq;
1341 }
1342
1343 vc_ghcb_invalidate(ghcb);
1344
1345 if (req->exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST) {
1346 ghcb_set_rax(ghcb, input->data_gpa);
1347 ghcb_set_rbx(ghcb, input->data_npages);
1348 }
1349
1350 ret = sev_es_ghcb_hv_call(ghcb, &ctxt, req->exit_code, input->req_gpa, input->resp_gpa);
1351 if (ret)
1352 goto e_put;
1353
1354 req->exitinfo2 = ghcb->save.sw_exit_info_2;
1355 switch (req->exitinfo2) {
1356 case 0:
1357 break;
1358
1359 case SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_BUSY):
1360 ret = -EAGAIN;
1361 break;
1362
1363 case SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN):
1364 /* Number of expected pages are returned in RBX */
1365 if (req->exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST) {
1366 input->data_npages = ghcb_get_rbx(ghcb);
1367 ret = -ENOSPC;
1368 break;
1369 }
1370 fallthrough;
1371 default:
1372 ret = -EIO;
1373 break;
1374 }
1375
1376 e_put:
1377 __sev_put_ghcb(&state);
1378 e_restore_irq:
1379 local_irq_restore(flags);
1380
1381 return ret;
1382 }
1383
1384 static struct platform_device sev_guest_device = {
1385 .name = "sev-guest",
1386 .id = -1,
1387 };
1388
1389 static struct platform_device tpm_svsm_device = {
1390 .name = "tpm-svsm",
1391 .id = -1,
1392 };
1393
snp_init_platform_device(void)1394 static int __init snp_init_platform_device(void)
1395 {
1396 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
1397 return -ENODEV;
1398
1399 if (platform_device_register(&sev_guest_device))
1400 return -ENODEV;
1401
1402 if (snp_svsm_vtpm_probe() &&
1403 platform_device_register(&tpm_svsm_device))
1404 return -ENODEV;
1405
1406 pr_info("SNP guest platform devices initialized.\n");
1407 return 0;
1408 }
1409 device_initcall(snp_init_platform_device);
1410
sev_show_status(void)1411 void sev_show_status(void)
1412 {
1413 int i;
1414
1415 pr_info("Status: ");
1416 for (i = 0; i < MSR_AMD64_SNP_RESV_BIT; i++) {
1417 if (sev_status & BIT_ULL(i)) {
1418 if (!sev_status_feat_names[i])
1419 continue;
1420
1421 pr_cont("%s ", sev_status_feat_names[i]);
1422 }
1423 }
1424 pr_cont("\n");
1425 }
1426
1427 #ifdef CONFIG_SYSFS
vmpl_show(struct kobject * kobj,struct kobj_attribute * attr,char * buf)1428 static ssize_t vmpl_show(struct kobject *kobj,
1429 struct kobj_attribute *attr, char *buf)
1430 {
1431 return sysfs_emit(buf, "%d\n", snp_vmpl);
1432 }
1433
sev_status_show(struct kobject * kobj,struct kobj_attribute * attr,char * buf)1434 static ssize_t sev_status_show(struct kobject *kobj,
1435 struct kobj_attribute *attr, char *buf)
1436 {
1437 return sysfs_emit(buf, "0x%llx\n", sev_status);
1438 }
1439
1440 static struct kobj_attribute vmpl_attr = __ATTR_RO(vmpl);
1441 static struct kobj_attribute sev_status_attr = __ATTR_RO(sev_status);
1442
1443 static struct attribute *sev_status_attrs[] = {
1444 &sev_status_attr.attr,
1445 NULL
1446 };
1447
1448 static struct attribute_group sev_attr_group = {
1449 .attrs = sev_status_attrs,
1450 };
1451
sev_sysfs_init(void)1452 static int __init sev_sysfs_init(void)
1453 {
1454 struct kobject *sev_kobj;
1455 struct device *dev_root;
1456 int ret;
1457
1458 if (!(sev_status & MSR_AMD64_SEV_ENABLED))
1459 return -ENODEV;
1460
1461 dev_root = bus_get_dev_root(&cpu_subsys);
1462 if (!dev_root)
1463 return -ENODEV;
1464
1465 sev_kobj = kobject_create_and_add("sev", &dev_root->kobj);
1466 put_device(dev_root);
1467
1468 if (!sev_kobj)
1469 return -ENOMEM;
1470
1471 ret = sysfs_create_group(sev_kobj, &sev_attr_group);
1472 if (ret)
1473 goto drop_kobj;
1474
1475 if (sev_status & MSR_AMD64_SEV_SNP_ENABLED) {
1476 ret = sysfs_add_file_to_group(sev_kobj, &vmpl_attr.attr, NULL);
1477 if (ret)
1478 goto drop_sysfs;
1479 }
1480
1481 return 0;
1482
1483 drop_sysfs:
1484 sysfs_remove_group(sev_kobj, &sev_attr_group);
1485 drop_kobj:
1486 kobject_put(sev_kobj);
1487
1488 return ret;
1489 }
1490 arch_initcall(sev_sysfs_init);
1491 #endif // CONFIG_SYSFS
1492
free_shared_pages(void * buf,size_t sz)1493 static void free_shared_pages(void *buf, size_t sz)
1494 {
1495 unsigned int npages = PAGE_ALIGN(sz) >> PAGE_SHIFT;
1496 int ret;
1497
1498 if (!buf)
1499 return;
1500
1501 ret = set_memory_encrypted((unsigned long)buf, npages);
1502 if (ret) {
1503 WARN_ONCE(ret, "failed to restore encryption mask (leak it)\n");
1504 return;
1505 }
1506
1507 __free_pages(virt_to_page(buf), get_order(sz));
1508 }
1509
alloc_shared_pages(size_t sz)1510 static void *alloc_shared_pages(size_t sz)
1511 {
1512 unsigned int npages = PAGE_ALIGN(sz) >> PAGE_SHIFT;
1513 struct page *page;
1514 int ret;
1515
1516 page = alloc_pages(GFP_KERNEL_ACCOUNT, get_order(sz));
1517 if (!page)
1518 return NULL;
1519
1520 ret = set_memory_decrypted((unsigned long)page_address(page), npages);
1521 if (ret) {
1522 pr_err("failed to mark page shared, ret=%d\n", ret);
1523 __free_pages(page, get_order(sz));
1524 return NULL;
1525 }
1526
1527 return page_address(page);
1528 }
1529
get_vmpck(int id,struct snp_secrets_page * secrets,u32 ** seqno)1530 static u8 *get_vmpck(int id, struct snp_secrets_page *secrets, u32 **seqno)
1531 {
1532 u8 *key = NULL;
1533
1534 switch (id) {
1535 case 0:
1536 *seqno = &secrets->os_area.msg_seqno_0;
1537 key = secrets->vmpck0;
1538 break;
1539 case 1:
1540 *seqno = &secrets->os_area.msg_seqno_1;
1541 key = secrets->vmpck1;
1542 break;
1543 case 2:
1544 *seqno = &secrets->os_area.msg_seqno_2;
1545 key = secrets->vmpck2;
1546 break;
1547 case 3:
1548 *seqno = &secrets->os_area.msg_seqno_3;
1549 key = secrets->vmpck3;
1550 break;
1551 default:
1552 break;
1553 }
1554
1555 return key;
1556 }
1557
snp_init_crypto(const u8 * key,size_t keylen)1558 static struct aes_gcm_key *snp_init_crypto(const u8 *key, size_t keylen)
1559 {
1560 struct aes_gcm_key *gcm_key;
1561
1562 gcm_key = kzalloc_obj(*gcm_key);
1563 if (!gcm_key)
1564 return NULL;
1565
1566 if (aes_gcm_preparekey(gcm_key, key, keylen, AUTHTAG_LEN)) {
1567 pr_err("AES-GCM key preparation failed\n");
1568 kfree_sensitive(gcm_key);
1569 return NULL;
1570 }
1571
1572 return gcm_key;
1573 }
1574
snp_msg_init(struct snp_msg_desc * mdesc,int vmpck_id)1575 int snp_msg_init(struct snp_msg_desc *mdesc, int vmpck_id)
1576 {
1577 /* Adjust the default VMPCK key based on the executing VMPL level */
1578 if (vmpck_id == -1)
1579 vmpck_id = snp_vmpl;
1580
1581 mdesc->vmpck = get_vmpck(vmpck_id, mdesc->secrets, &mdesc->os_area_msg_seqno);
1582 if (!mdesc->vmpck) {
1583 pr_err("Invalid VMPCK%d communication key\n", vmpck_id);
1584 return -EINVAL;
1585 }
1586
1587 /* Verify that VMPCK is not zero. */
1588 if (!memchr_inv(mdesc->vmpck, 0, VMPCK_KEY_LEN)) {
1589 pr_err("Empty VMPCK%d communication key\n", vmpck_id);
1590 return -EINVAL;
1591 }
1592
1593 mdesc->vmpck_id = vmpck_id;
1594
1595 mdesc->gcm_key = snp_init_crypto(mdesc->vmpck, VMPCK_KEY_LEN);
1596 if (!mdesc->gcm_key)
1597 return -ENOMEM;
1598
1599 return 0;
1600 }
1601 EXPORT_SYMBOL_GPL(snp_msg_init);
1602
snp_msg_alloc(void)1603 struct snp_msg_desc *snp_msg_alloc(void)
1604 {
1605 struct snp_msg_desc *mdesc;
1606 void __iomem *mem;
1607
1608 BUILD_BUG_ON(sizeof(struct snp_guest_msg) > PAGE_SIZE);
1609
1610 mdesc = kzalloc_obj(struct snp_msg_desc);
1611 if (!mdesc)
1612 return ERR_PTR(-ENOMEM);
1613
1614 mem = ioremap_encrypted(sev_secrets_pa, PAGE_SIZE);
1615 if (!mem)
1616 goto e_free_mdesc;
1617
1618 mdesc->secrets = (__force struct snp_secrets_page *)mem;
1619
1620 /* Allocate the shared page used for the request and response message. */
1621 mdesc->request = alloc_shared_pages(sizeof(struct snp_guest_msg));
1622 if (!mdesc->request)
1623 goto e_unmap;
1624
1625 mdesc->response = alloc_shared_pages(sizeof(struct snp_guest_msg));
1626 if (!mdesc->response)
1627 goto e_free_request;
1628
1629 return mdesc;
1630
1631 e_free_request:
1632 free_shared_pages(mdesc->request, sizeof(struct snp_guest_msg));
1633 e_unmap:
1634 iounmap(mem);
1635 e_free_mdesc:
1636 kfree(mdesc);
1637
1638 return ERR_PTR(-ENOMEM);
1639 }
1640 EXPORT_SYMBOL_GPL(snp_msg_alloc);
1641
snp_msg_free(struct snp_msg_desc * mdesc)1642 void snp_msg_free(struct snp_msg_desc *mdesc)
1643 {
1644 if (!mdesc)
1645 return;
1646
1647 kfree_sensitive(mdesc->gcm_key);
1648 free_shared_pages(mdesc->response, sizeof(struct snp_guest_msg));
1649 free_shared_pages(mdesc->request, sizeof(struct snp_guest_msg));
1650 iounmap((__force void __iomem *)mdesc->secrets);
1651
1652 kfree_sensitive(mdesc);
1653 }
1654 EXPORT_SYMBOL_GPL(snp_msg_free);
1655
1656 /* Mutex to serialize the shared buffer access and command handling. */
1657 static DEFINE_MUTEX(snp_cmd_mutex);
1658
1659 /*
1660 * If an error is received from the host or AMD Secure Processor (ASP) there
1661 * are two options. Either retry the exact same encrypted request or discontinue
1662 * using the VMPCK.
1663 *
1664 * This is because in the current encryption scheme GHCB v2 uses AES-GCM to
1665 * encrypt the requests. The IV for this scheme is the sequence number. GCM
1666 * cannot tolerate IV reuse.
1667 *
1668 * The ASP FW v1.51 only increments the sequence numbers on a successful
1669 * guest<->ASP back and forth and only accepts messages at its exact sequence
1670 * number.
1671 *
1672 * So if the sequence number were to be reused the encryption scheme is
1673 * vulnerable. If the sequence number were incremented for a fresh IV the ASP
1674 * will reject the request.
1675 */
snp_disable_vmpck(struct snp_msg_desc * mdesc)1676 static void snp_disable_vmpck(struct snp_msg_desc *mdesc)
1677 {
1678 pr_alert("Disabling VMPCK%d communication key to prevent IV reuse.\n",
1679 mdesc->vmpck_id);
1680 memzero_explicit(mdesc->vmpck, VMPCK_KEY_LEN);
1681 mdesc->vmpck = NULL;
1682 }
1683
__snp_get_msg_seqno(struct snp_msg_desc * mdesc)1684 static inline u64 __snp_get_msg_seqno(struct snp_msg_desc *mdesc)
1685 {
1686 u64 count;
1687
1688 lockdep_assert_held(&snp_cmd_mutex);
1689
1690 /* Read the current message sequence counter from secrets pages */
1691 count = *mdesc->os_area_msg_seqno;
1692
1693 return count + 1;
1694 }
1695
1696 /* Return a non-zero on success */
snp_get_msg_seqno(struct snp_msg_desc * mdesc)1697 static u64 snp_get_msg_seqno(struct snp_msg_desc *mdesc)
1698 {
1699 u64 count = __snp_get_msg_seqno(mdesc);
1700
1701 /*
1702 * The message sequence counter for the SNP guest request is a 64-bit
1703 * value but the version 2 of GHCB specification defines a 32-bit storage
1704 * for it. If the counter exceeds the 32-bit value then return zero.
1705 * The caller should check the return value, but if the caller happens to
1706 * not check the value and use it, then the firmware treats zero as an
1707 * invalid number and will fail the message request.
1708 */
1709 if (count >= UINT_MAX) {
1710 pr_err("request message sequence counter overflow\n");
1711 return 0;
1712 }
1713
1714 return count;
1715 }
1716
snp_inc_msg_seqno(struct snp_msg_desc * mdesc)1717 static void snp_inc_msg_seqno(struct snp_msg_desc *mdesc)
1718 {
1719 /*
1720 * The counter is also incremented by the PSP, so increment it by 2
1721 * and save in secrets page.
1722 */
1723 *mdesc->os_area_msg_seqno += 2;
1724 }
1725
verify_and_dec_payload(struct snp_msg_desc * mdesc,struct snp_guest_req * req)1726 static int verify_and_dec_payload(struct snp_msg_desc *mdesc, struct snp_guest_req *req)
1727 {
1728 struct snp_guest_msg *resp_msg = &mdesc->secret_response;
1729 struct snp_guest_msg *req_msg = &mdesc->secret_request;
1730 struct snp_guest_msg_hdr *req_msg_hdr = &req_msg->hdr;
1731 struct snp_guest_msg_hdr *resp_msg_hdr = &resp_msg->hdr;
1732 struct aes_gcm_key *gcm_key = mdesc->gcm_key;
1733 u8 iv[GCM_AES_IV_SIZE] = {};
1734
1735 pr_debug("response [seqno %lld type %d version %d sz %d]\n",
1736 resp_msg_hdr->msg_seqno, resp_msg_hdr->msg_type, resp_msg_hdr->msg_version,
1737 resp_msg_hdr->msg_sz);
1738
1739 /* Copy response from shared memory to encrypted memory. */
1740 memcpy(resp_msg, mdesc->response, sizeof(*resp_msg));
1741
1742 /* Verify that the sequence counter is incremented by 1 */
1743 if (unlikely(resp_msg_hdr->msg_seqno != (req_msg_hdr->msg_seqno + 1)))
1744 return -EBADMSG;
1745
1746 /* Verify response message type and version number. */
1747 if (resp_msg_hdr->msg_type != (req_msg_hdr->msg_type + 1) ||
1748 resp_msg_hdr->msg_version != req_msg_hdr->msg_version)
1749 return -EBADMSG;
1750
1751 /*
1752 * If the message size is greater than our buffer length then return
1753 * an error.
1754 */
1755 if (unlikely(resp_msg_hdr->msg_sz + AUTHTAG_LEN > req->resp_sz))
1756 return -EBADMSG;
1757
1758 /* Decrypt the payload */
1759 memcpy(iv, &resp_msg_hdr->msg_seqno, min(sizeof(iv), sizeof(resp_msg_hdr->msg_seqno)));
1760 return aes_gcm_decrypt(req->resp_buf, resp_msg->payload,
1761 resp_msg_hdr->msg_sz, resp_msg_hdr->authtag,
1762 &resp_msg_hdr->algo, AAD_LEN, iv, gcm_key);
1763 }
1764
enc_payload(struct snp_msg_desc * mdesc,u64 seqno,struct snp_guest_req * req)1765 static int enc_payload(struct snp_msg_desc *mdesc, u64 seqno, struct snp_guest_req *req)
1766 {
1767 struct snp_guest_msg *msg = &mdesc->secret_request;
1768 struct snp_guest_msg_hdr *hdr = &msg->hdr;
1769 struct aes_gcm_key *gcm_key = mdesc->gcm_key;
1770 u8 iv[GCM_AES_IV_SIZE] = {};
1771
1772 memset(msg, 0, sizeof(*msg));
1773
1774 hdr->algo = SNP_AEAD_AES_256_GCM;
1775 hdr->hdr_version = MSG_HDR_VER;
1776 hdr->hdr_sz = sizeof(*hdr);
1777 hdr->msg_type = req->msg_type;
1778 hdr->msg_version = req->msg_version;
1779 hdr->msg_seqno = seqno;
1780 hdr->msg_vmpck = req->vmpck_id;
1781 hdr->msg_sz = req->req_sz;
1782
1783 /* Verify the sequence number is non-zero */
1784 if (!hdr->msg_seqno)
1785 return -ENOSR;
1786
1787 pr_debug("request [seqno %lld type %d version %d sz %d]\n",
1788 hdr->msg_seqno, hdr->msg_type, hdr->msg_version, hdr->msg_sz);
1789
1790 if (WARN_ON(req->req_sz + AUTHTAG_LEN > sizeof(msg->payload)))
1791 return -EBADMSG;
1792
1793 memcpy(iv, &hdr->msg_seqno, min(sizeof(iv), sizeof(hdr->msg_seqno)));
1794 aes_gcm_encrypt(msg->payload, req->req_buf, req->req_sz, hdr->authtag,
1795 &hdr->algo, AAD_LEN, iv, gcm_key);
1796
1797 return 0;
1798 }
1799
__handle_guest_request(struct snp_msg_desc * mdesc,struct snp_guest_req * req)1800 static int __handle_guest_request(struct snp_msg_desc *mdesc, struct snp_guest_req *req)
1801 {
1802 unsigned long req_start = jiffies;
1803 unsigned int override_npages = 0;
1804 u64 override_err = 0;
1805 int rc;
1806
1807 retry_request:
1808 /*
1809 * Call firmware to process the request. In this function the encrypted
1810 * message enters shared memory with the host. So after this call the
1811 * sequence number must be incremented or the VMPCK must be deleted to
1812 * prevent reuse of the IV.
1813 */
1814 rc = snp_issue_guest_request(req);
1815 switch (rc) {
1816 case -ENOSPC:
1817 /*
1818 * If the extended guest request fails due to having too
1819 * small of a certificate data buffer, retry the same
1820 * guest request without the extended data request in
1821 * order to increment the sequence number and thus avoid
1822 * IV reuse.
1823 */
1824 override_npages = req->input.data_npages;
1825 req->exit_code = SVM_VMGEXIT_GUEST_REQUEST;
1826
1827 /*
1828 * Override the error to inform callers the given extended
1829 * request buffer size was too small and give the caller the
1830 * required buffer size.
1831 */
1832 override_err = SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN);
1833
1834 /*
1835 * If this call to the firmware succeeds, the sequence number can
1836 * be incremented allowing for continued use of the VMPCK. If
1837 * there is an error reflected in the return value, this value
1838 * is checked further down and the result will be the deletion
1839 * of the VMPCK and the error code being propagated back to the
1840 * user as an ioctl() return code.
1841 */
1842 goto retry_request;
1843
1844 /*
1845 * The host may return SNP_GUEST_VMM_ERR_BUSY if the request has been
1846 * throttled. Retry in the driver to avoid returning and reusing the
1847 * message sequence number on a different message.
1848 */
1849 case -EAGAIN:
1850 if (jiffies - req_start > SNP_REQ_MAX_RETRY_DURATION) {
1851 rc = -ETIMEDOUT;
1852 break;
1853 }
1854 schedule_timeout_killable(SNP_REQ_RETRY_DELAY);
1855 goto retry_request;
1856 }
1857
1858 /*
1859 * Increment the message sequence number. There is no harm in doing
1860 * this now because decryption uses the value stored in the response
1861 * structure and any failure will wipe the VMPCK, preventing further
1862 * use anyway.
1863 */
1864 snp_inc_msg_seqno(mdesc);
1865
1866 if (override_err) {
1867 req->exitinfo2 = override_err;
1868
1869 /*
1870 * If an extended guest request was issued and the supplied certificate
1871 * buffer was not large enough, a standard guest request was issued to
1872 * prevent IV reuse. If the standard request was successful, return -EIO
1873 * back to the caller as would have originally been returned.
1874 */
1875 if (!rc && override_err == SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN))
1876 rc = -EIO;
1877 }
1878
1879 if (override_npages)
1880 req->input.data_npages = override_npages;
1881
1882 return rc;
1883 }
1884
snp_send_guest_request(struct snp_msg_desc * mdesc,struct snp_guest_req * req)1885 int snp_send_guest_request(struct snp_msg_desc *mdesc, struct snp_guest_req *req)
1886 {
1887 u64 seqno;
1888 int rc;
1889
1890 guard(mutex)(&snp_cmd_mutex);
1891
1892 /* Check if the VMPCK is not empty */
1893 if (!mdesc->vmpck || !memchr_inv(mdesc->vmpck, 0, VMPCK_KEY_LEN)) {
1894 pr_err_ratelimited("VMPCK is disabled\n");
1895 return -ENOTTY;
1896 }
1897
1898 /* Get message sequence and verify that its a non-zero */
1899 seqno = snp_get_msg_seqno(mdesc);
1900 if (!seqno)
1901 return -EIO;
1902
1903 /* Clear shared memory's response for the host to populate. */
1904 memset(mdesc->response, 0, sizeof(struct snp_guest_msg));
1905
1906 /* Encrypt the userspace provided payload in mdesc->secret_request. */
1907 rc = enc_payload(mdesc, seqno, req);
1908 if (rc)
1909 return rc;
1910
1911 /*
1912 * Write the fully encrypted request to the shared unencrypted
1913 * request page.
1914 */
1915 memcpy(mdesc->request, &mdesc->secret_request, sizeof(mdesc->secret_request));
1916
1917 /* Initialize the input address for guest request */
1918 req->input.req_gpa = __pa(mdesc->request);
1919 req->input.resp_gpa = __pa(mdesc->response);
1920 req->input.data_gpa = req->certs_data ? __pa(req->certs_data) : 0;
1921
1922 rc = __handle_guest_request(mdesc, req);
1923 if (rc) {
1924 if (rc == -EIO &&
1925 req->exitinfo2 == SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN))
1926 return rc;
1927
1928 pr_alert("Detected error from ASP request. rc: %d, exitinfo2: 0x%llx\n",
1929 rc, req->exitinfo2);
1930
1931 snp_disable_vmpck(mdesc);
1932 return rc;
1933 }
1934
1935 rc = verify_and_dec_payload(mdesc, req);
1936 if (rc) {
1937 pr_alert("Detected unexpected decode failure from ASP. rc: %d\n", rc);
1938 snp_disable_vmpck(mdesc);
1939 return rc;
1940 }
1941
1942 return 0;
1943 }
1944 EXPORT_SYMBOL_GPL(snp_send_guest_request);
1945
snp_get_tsc_info(void)1946 static int __init snp_get_tsc_info(void)
1947 {
1948 struct snp_tsc_info_resp *tsc_resp;
1949 struct snp_tsc_info_req *tsc_req;
1950 struct snp_msg_desc *mdesc;
1951 struct snp_guest_req req = {};
1952 int rc = -ENOMEM;
1953
1954 tsc_req = kzalloc_obj(*tsc_req);
1955 if (!tsc_req)
1956 return rc;
1957
1958 /*
1959 * The intermediate response buffer is used while decrypting the
1960 * response payload. Make sure that it has enough space to cover
1961 * the authtag.
1962 */
1963 tsc_resp = kzalloc(sizeof(*tsc_resp) + AUTHTAG_LEN, GFP_KERNEL);
1964 if (!tsc_resp)
1965 goto e_free_tsc_req;
1966
1967 mdesc = snp_msg_alloc();
1968 if (IS_ERR_OR_NULL(mdesc))
1969 goto e_free_tsc_resp;
1970
1971 rc = snp_msg_init(mdesc, snp_vmpl);
1972 if (rc)
1973 goto e_free_mdesc;
1974
1975 req.msg_version = MSG_HDR_VER;
1976 req.msg_type = SNP_MSG_TSC_INFO_REQ;
1977 req.vmpck_id = snp_vmpl;
1978 req.req_buf = tsc_req;
1979 req.req_sz = sizeof(*tsc_req);
1980 req.resp_buf = (void *)tsc_resp;
1981 req.resp_sz = sizeof(*tsc_resp) + AUTHTAG_LEN;
1982 req.exit_code = SVM_VMGEXIT_GUEST_REQUEST;
1983
1984 rc = snp_send_guest_request(mdesc, &req);
1985 if (rc)
1986 goto e_request;
1987
1988 pr_debug("%s: response status 0x%x scale 0x%llx offset 0x%llx factor 0x%x\n",
1989 __func__, tsc_resp->status, tsc_resp->tsc_scale, tsc_resp->tsc_offset,
1990 tsc_resp->tsc_factor);
1991
1992 if (!tsc_resp->status) {
1993 snp_tsc_scale = tsc_resp->tsc_scale;
1994 snp_tsc_offset = tsc_resp->tsc_offset;
1995 } else {
1996 pr_err("Failed to get TSC info, response status 0x%x\n", tsc_resp->status);
1997 rc = -EIO;
1998 }
1999
2000 e_request:
2001 /* The response buffer contains sensitive data, explicitly clear it. */
2002 memzero_explicit(tsc_resp, sizeof(*tsc_resp) + AUTHTAG_LEN);
2003 e_free_mdesc:
2004 snp_msg_free(mdesc);
2005 e_free_tsc_resp:
2006 kfree(tsc_resp);
2007 e_free_tsc_req:
2008 kfree(tsc_req);
2009
2010 return rc;
2011 }
2012
snp_secure_tsc_prepare(void)2013 void __init snp_secure_tsc_prepare(void)
2014 {
2015 if (!cc_platform_has(CC_ATTR_GUEST_SNP_SECURE_TSC))
2016 return;
2017
2018 if (snp_get_tsc_info()) {
2019 pr_alert("Unable to retrieve Secure TSC info from ASP\n");
2020 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SECURE_TSC);
2021 }
2022
2023 pr_debug("SecureTSC enabled");
2024 }
2025
securetsc_get_tsc_khz(void)2026 static unsigned long securetsc_get_tsc_khz(void)
2027 {
2028 return snp_tsc_freq_khz;
2029 }
2030
snp_secure_tsc_init(void)2031 void __init snp_secure_tsc_init(void)
2032 {
2033 struct snp_secrets_page *secrets;
2034 unsigned long tsc_freq_mhz;
2035 void *mem;
2036
2037 if (!cc_platform_has(CC_ATTR_GUEST_SNP_SECURE_TSC))
2038 return;
2039
2040 mem = early_memremap_encrypted(sev_secrets_pa, PAGE_SIZE);
2041 if (!mem) {
2042 pr_err("Unable to get TSC_FACTOR: failed to map the SNP secrets page.\n");
2043 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SECURE_TSC);
2044 }
2045
2046 secrets = (__force struct snp_secrets_page *)mem;
2047
2048 setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ);
2049 rdmsrq(MSR_AMD64_GUEST_TSC_FREQ, tsc_freq_mhz);
2050
2051 /* Extract the GUEST TSC MHZ from BIT[17:0], rest is reserved space */
2052 tsc_freq_mhz &= GENMASK_ULL(17, 0);
2053
2054 snp_tsc_freq_khz = SNP_SCALE_TSC_FREQ(tsc_freq_mhz * 1000, secrets->tsc_factor);
2055
2056 x86_platform.calibrate_cpu = securetsc_get_tsc_khz;
2057 x86_platform.calibrate_tsc = securetsc_get_tsc_khz;
2058
2059 early_memunmap(mem, PAGE_SIZE);
2060 }
2061