1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * AMD Memory Encryption Support
4 *
5 * Copyright (C) 2019 SUSE
6 *
7 * Author: Joerg Roedel <jroedel@suse.de>
8 */
9
10 #define pr_fmt(fmt) "SEV: " fmt
11
12 #include <linux/sched/debug.h> /* For show_regs() */
13 #include <linux/percpu-defs.h>
14 #include <linux/cc_platform.h>
15 #include <linux/printk.h>
16 #include <linux/mm_types.h>
17 #include <linux/set_memory.h>
18 #include <linux/memblock.h>
19 #include <linux/kernel.h>
20 #include <linux/mm.h>
21 #include <linux/cpumask.h>
22 #include <linux/efi.h>
23 #include <linux/platform_device.h>
24 #include <linux/io.h>
25 #include <linux/psp-sev.h>
26 #include <linux/dmi.h>
27 #include <uapi/linux/sev-guest.h>
28
29 #include <asm/init.h>
30 #include <asm/cpu_entry_area.h>
31 #include <asm/stacktrace.h>
32 #include <asm/sev.h>
33 #include <asm/insn-eval.h>
34 #include <asm/fpu/xcr.h>
35 #include <asm/processor.h>
36 #include <asm/realmode.h>
37 #include <asm/setup.h>
38 #include <asm/traps.h>
39 #include <asm/svm.h>
40 #include <asm/smp.h>
41 #include <asm/cpu.h>
42 #include <asm/apic.h>
43 #include <asm/cpuid.h>
44 #include <asm/cmdline.h>
45
46 #define DR7_RESET_VALUE 0x400
47
48 /* AP INIT values as documented in the APM2 section "Processor Initialization State" */
49 #define AP_INIT_CS_LIMIT 0xffff
50 #define AP_INIT_DS_LIMIT 0xffff
51 #define AP_INIT_LDTR_LIMIT 0xffff
52 #define AP_INIT_GDTR_LIMIT 0xffff
53 #define AP_INIT_IDTR_LIMIT 0xffff
54 #define AP_INIT_TR_LIMIT 0xffff
55 #define AP_INIT_RFLAGS_DEFAULT 0x2
56 #define AP_INIT_DR6_DEFAULT 0xffff0ff0
57 #define AP_INIT_GPAT_DEFAULT 0x0007040600070406ULL
58 #define AP_INIT_XCR0_DEFAULT 0x1
59 #define AP_INIT_X87_FTW_DEFAULT 0x5555
60 #define AP_INIT_X87_FCW_DEFAULT 0x0040
61 #define AP_INIT_CR0_DEFAULT 0x60000010
62 #define AP_INIT_MXCSR_DEFAULT 0x1f80
63
64 static const char * const sev_status_feat_names[] = {
65 [MSR_AMD64_SEV_ENABLED_BIT] = "SEV",
66 [MSR_AMD64_SEV_ES_ENABLED_BIT] = "SEV-ES",
67 [MSR_AMD64_SEV_SNP_ENABLED_BIT] = "SEV-SNP",
68 [MSR_AMD64_SNP_VTOM_BIT] = "vTom",
69 [MSR_AMD64_SNP_REFLECT_VC_BIT] = "ReflectVC",
70 [MSR_AMD64_SNP_RESTRICTED_INJ_BIT] = "RI",
71 [MSR_AMD64_SNP_ALT_INJ_BIT] = "AI",
72 [MSR_AMD64_SNP_DEBUG_SWAP_BIT] = "DebugSwap",
73 [MSR_AMD64_SNP_PREVENT_HOST_IBS_BIT] = "NoHostIBS",
74 [MSR_AMD64_SNP_BTB_ISOLATION_BIT] = "BTBIsol",
75 [MSR_AMD64_SNP_VMPL_SSS_BIT] = "VmplSSS",
76 [MSR_AMD64_SNP_SECURE_TSC_BIT] = "SecureTSC",
77 [MSR_AMD64_SNP_VMGEXIT_PARAM_BIT] = "VMGExitParam",
78 [MSR_AMD64_SNP_IBS_VIRT_BIT] = "IBSVirt",
79 [MSR_AMD64_SNP_VMSA_REG_PROT_BIT] = "VMSARegProt",
80 [MSR_AMD64_SNP_SMT_PROT_BIT] = "SMTProt",
81 };
82
83 /* For early boot hypervisor communication in SEV-ES enabled guests */
84 static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE);
85
86 /*
87 * Needs to be in the .data section because we need it NULL before bss is
88 * cleared
89 */
90 static struct ghcb *boot_ghcb __section(".data");
91
92 /* Bitmap of SEV features supported by the hypervisor */
93 static u64 sev_hv_features __ro_after_init;
94
95 /* Secrets page physical address from the CC blob */
96 static u64 secrets_pa __ro_after_init;
97
98 /* #VC handler runtime per-CPU data */
99 struct sev_es_runtime_data {
100 struct ghcb ghcb_page;
101
102 /*
103 * Reserve one page per CPU as backup storage for the unencrypted GHCB.
104 * It is needed when an NMI happens while the #VC handler uses the real
105 * GHCB, and the NMI handler itself is causing another #VC exception. In
106 * that case the GHCB content of the first handler needs to be backed up
107 * and restored.
108 */
109 struct ghcb backup_ghcb;
110
111 /*
112 * Mark the per-cpu GHCBs as in-use to detect nested #VC exceptions.
113 * There is no need for it to be atomic, because nothing is written to
114 * the GHCB between the read and the write of ghcb_active. So it is safe
115 * to use it when a nested #VC exception happens before the write.
116 *
117 * This is necessary for example in the #VC->NMI->#VC case when the NMI
118 * happens while the first #VC handler uses the GHCB. When the NMI code
119 * raises a second #VC handler it might overwrite the contents of the
120 * GHCB written by the first handler. To avoid this the content of the
121 * GHCB is saved and restored when the GHCB is detected to be in use
122 * already.
123 */
124 bool ghcb_active;
125 bool backup_ghcb_active;
126
127 /*
128 * Cached DR7 value - write it on DR7 writes and return it on reads.
129 * That value will never make it to the real hardware DR7 as debugging
130 * is currently unsupported in SEV-ES guests.
131 */
132 unsigned long dr7;
133 };
134
135 struct ghcb_state {
136 struct ghcb *ghcb;
137 };
138
139 /* For early boot SVSM communication */
140 static struct svsm_ca boot_svsm_ca_page __aligned(PAGE_SIZE);
141
142 static DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data);
143 static DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa);
144 static DEFINE_PER_CPU(struct svsm_ca *, svsm_caa);
145 static DEFINE_PER_CPU(u64, svsm_caa_pa);
146
on_vc_stack(struct pt_regs * regs)147 static __always_inline bool on_vc_stack(struct pt_regs *regs)
148 {
149 unsigned long sp = regs->sp;
150
151 /* User-mode RSP is not trusted */
152 if (user_mode(regs))
153 return false;
154
155 /* SYSCALL gap still has user-mode RSP */
156 if (ip_within_syscall_gap(regs))
157 return false;
158
159 return ((sp >= __this_cpu_ist_bottom_va(VC)) && (sp < __this_cpu_ist_top_va(VC)));
160 }
161
162 /*
163 * This function handles the case when an NMI is raised in the #VC
164 * exception handler entry code, before the #VC handler has switched off
165 * its IST stack. In this case, the IST entry for #VC must be adjusted,
166 * so that any nested #VC exception will not overwrite the stack
167 * contents of the interrupted #VC handler.
168 *
169 * The IST entry is adjusted unconditionally so that it can be also be
170 * unconditionally adjusted back in __sev_es_ist_exit(). Otherwise a
171 * nested sev_es_ist_exit() call may adjust back the IST entry too
172 * early.
173 *
174 * The __sev_es_ist_enter() and __sev_es_ist_exit() functions always run
175 * on the NMI IST stack, as they are only called from NMI handling code
176 * right now.
177 */
__sev_es_ist_enter(struct pt_regs * regs)178 void noinstr __sev_es_ist_enter(struct pt_regs *regs)
179 {
180 unsigned long old_ist, new_ist;
181
182 /* Read old IST entry */
183 new_ist = old_ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]);
184
185 /*
186 * If NMI happened while on the #VC IST stack, set the new IST
187 * value below regs->sp, so that the interrupted stack frame is
188 * not overwritten by subsequent #VC exceptions.
189 */
190 if (on_vc_stack(regs))
191 new_ist = regs->sp;
192
193 /*
194 * Reserve additional 8 bytes and store old IST value so this
195 * adjustment can be unrolled in __sev_es_ist_exit().
196 */
197 new_ist -= sizeof(old_ist);
198 *(unsigned long *)new_ist = old_ist;
199
200 /* Set new IST entry */
201 this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], new_ist);
202 }
203
__sev_es_ist_exit(void)204 void noinstr __sev_es_ist_exit(void)
205 {
206 unsigned long ist;
207
208 /* Read IST entry */
209 ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]);
210
211 if (WARN_ON(ist == __this_cpu_ist_top_va(VC)))
212 return;
213
214 /* Read back old IST entry and write it to the TSS */
215 this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], *(unsigned long *)ist);
216 }
217
218 /*
219 * Nothing shall interrupt this code path while holding the per-CPU
220 * GHCB. The backup GHCB is only for NMIs interrupting this path.
221 *
222 * Callers must disable local interrupts around it.
223 */
__sev_get_ghcb(struct ghcb_state * state)224 static noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)
225 {
226 struct sev_es_runtime_data *data;
227 struct ghcb *ghcb;
228
229 WARN_ON(!irqs_disabled());
230
231 data = this_cpu_read(runtime_data);
232 ghcb = &data->ghcb_page;
233
234 if (unlikely(data->ghcb_active)) {
235 /* GHCB is already in use - save its contents */
236
237 if (unlikely(data->backup_ghcb_active)) {
238 /*
239 * Backup-GHCB is also already in use. There is no way
240 * to continue here so just kill the machine. To make
241 * panic() work, mark GHCBs inactive so that messages
242 * can be printed out.
243 */
244 data->ghcb_active = false;
245 data->backup_ghcb_active = false;
246
247 instrumentation_begin();
248 panic("Unable to handle #VC exception! GHCB and Backup GHCB are already in use");
249 instrumentation_end();
250 }
251
252 /* Mark backup_ghcb active before writing to it */
253 data->backup_ghcb_active = true;
254
255 state->ghcb = &data->backup_ghcb;
256
257 /* Backup GHCB content */
258 *state->ghcb = *ghcb;
259 } else {
260 state->ghcb = NULL;
261 data->ghcb_active = true;
262 }
263
264 return ghcb;
265 }
266
sev_es_rd_ghcb_msr(void)267 static inline u64 sev_es_rd_ghcb_msr(void)
268 {
269 return __rdmsr(MSR_AMD64_SEV_ES_GHCB);
270 }
271
sev_es_wr_ghcb_msr(u64 val)272 static __always_inline void sev_es_wr_ghcb_msr(u64 val)
273 {
274 u32 low, high;
275
276 low = (u32)(val);
277 high = (u32)(val >> 32);
278
279 native_wrmsr(MSR_AMD64_SEV_ES_GHCB, low, high);
280 }
281
vc_fetch_insn_kernel(struct es_em_ctxt * ctxt,unsigned char * buffer)282 static int vc_fetch_insn_kernel(struct es_em_ctxt *ctxt,
283 unsigned char *buffer)
284 {
285 return copy_from_kernel_nofault(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE);
286 }
287
__vc_decode_user_insn(struct es_em_ctxt * ctxt)288 static enum es_result __vc_decode_user_insn(struct es_em_ctxt *ctxt)
289 {
290 char buffer[MAX_INSN_SIZE];
291 int insn_bytes;
292
293 insn_bytes = insn_fetch_from_user_inatomic(ctxt->regs, buffer);
294 if (insn_bytes == 0) {
295 /* Nothing could be copied */
296 ctxt->fi.vector = X86_TRAP_PF;
297 ctxt->fi.error_code = X86_PF_INSTR | X86_PF_USER;
298 ctxt->fi.cr2 = ctxt->regs->ip;
299 return ES_EXCEPTION;
300 } else if (insn_bytes == -EINVAL) {
301 /* Effective RIP could not be calculated */
302 ctxt->fi.vector = X86_TRAP_GP;
303 ctxt->fi.error_code = 0;
304 ctxt->fi.cr2 = 0;
305 return ES_EXCEPTION;
306 }
307
308 if (!insn_decode_from_regs(&ctxt->insn, ctxt->regs, buffer, insn_bytes))
309 return ES_DECODE_FAILED;
310
311 if (ctxt->insn.immediate.got)
312 return ES_OK;
313 else
314 return ES_DECODE_FAILED;
315 }
316
__vc_decode_kern_insn(struct es_em_ctxt * ctxt)317 static enum es_result __vc_decode_kern_insn(struct es_em_ctxt *ctxt)
318 {
319 char buffer[MAX_INSN_SIZE];
320 int res, ret;
321
322 res = vc_fetch_insn_kernel(ctxt, buffer);
323 if (res) {
324 ctxt->fi.vector = X86_TRAP_PF;
325 ctxt->fi.error_code = X86_PF_INSTR;
326 ctxt->fi.cr2 = ctxt->regs->ip;
327 return ES_EXCEPTION;
328 }
329
330 ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64);
331 if (ret < 0)
332 return ES_DECODE_FAILED;
333 else
334 return ES_OK;
335 }
336
vc_decode_insn(struct es_em_ctxt * ctxt)337 static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)
338 {
339 if (user_mode(ctxt->regs))
340 return __vc_decode_user_insn(ctxt);
341 else
342 return __vc_decode_kern_insn(ctxt);
343 }
344
vc_write_mem(struct es_em_ctxt * ctxt,char * dst,char * buf,size_t size)345 static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,
346 char *dst, char *buf, size_t size)
347 {
348 unsigned long error_code = X86_PF_PROT | X86_PF_WRITE;
349
350 /*
351 * This function uses __put_user() independent of whether kernel or user
352 * memory is accessed. This works fine because __put_user() does no
353 * sanity checks of the pointer being accessed. All that it does is
354 * to report when the access failed.
355 *
356 * Also, this function runs in atomic context, so __put_user() is not
357 * allowed to sleep. The page-fault handler detects that it is running
358 * in atomic context and will not try to take mmap_sem and handle the
359 * fault, so additional pagefault_enable()/disable() calls are not
360 * needed.
361 *
362 * The access can't be done via copy_to_user() here because
363 * vc_write_mem() must not use string instructions to access unsafe
364 * memory. The reason is that MOVS is emulated by the #VC handler by
365 * splitting the move up into a read and a write and taking a nested #VC
366 * exception on whatever of them is the MMIO access. Using string
367 * instructions here would cause infinite nesting.
368 */
369 switch (size) {
370 case 1: {
371 u8 d1;
372 u8 __user *target = (u8 __user *)dst;
373
374 memcpy(&d1, buf, 1);
375 if (__put_user(d1, target))
376 goto fault;
377 break;
378 }
379 case 2: {
380 u16 d2;
381 u16 __user *target = (u16 __user *)dst;
382
383 memcpy(&d2, buf, 2);
384 if (__put_user(d2, target))
385 goto fault;
386 break;
387 }
388 case 4: {
389 u32 d4;
390 u32 __user *target = (u32 __user *)dst;
391
392 memcpy(&d4, buf, 4);
393 if (__put_user(d4, target))
394 goto fault;
395 break;
396 }
397 case 8: {
398 u64 d8;
399 u64 __user *target = (u64 __user *)dst;
400
401 memcpy(&d8, buf, 8);
402 if (__put_user(d8, target))
403 goto fault;
404 break;
405 }
406 default:
407 WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size);
408 return ES_UNSUPPORTED;
409 }
410
411 return ES_OK;
412
413 fault:
414 if (user_mode(ctxt->regs))
415 error_code |= X86_PF_USER;
416
417 ctxt->fi.vector = X86_TRAP_PF;
418 ctxt->fi.error_code = error_code;
419 ctxt->fi.cr2 = (unsigned long)dst;
420
421 return ES_EXCEPTION;
422 }
423
vc_read_mem(struct es_em_ctxt * ctxt,char * src,char * buf,size_t size)424 static enum es_result vc_read_mem(struct es_em_ctxt *ctxt,
425 char *src, char *buf, size_t size)
426 {
427 unsigned long error_code = X86_PF_PROT;
428
429 /*
430 * This function uses __get_user() independent of whether kernel or user
431 * memory is accessed. This works fine because __get_user() does no
432 * sanity checks of the pointer being accessed. All that it does is
433 * to report when the access failed.
434 *
435 * Also, this function runs in atomic context, so __get_user() is not
436 * allowed to sleep. The page-fault handler detects that it is running
437 * in atomic context and will not try to take mmap_sem and handle the
438 * fault, so additional pagefault_enable()/disable() calls are not
439 * needed.
440 *
441 * The access can't be done via copy_from_user() here because
442 * vc_read_mem() must not use string instructions to access unsafe
443 * memory. The reason is that MOVS is emulated by the #VC handler by
444 * splitting the move up into a read and a write and taking a nested #VC
445 * exception on whatever of them is the MMIO access. Using string
446 * instructions here would cause infinite nesting.
447 */
448 switch (size) {
449 case 1: {
450 u8 d1;
451 u8 __user *s = (u8 __user *)src;
452
453 if (__get_user(d1, s))
454 goto fault;
455 memcpy(buf, &d1, 1);
456 break;
457 }
458 case 2: {
459 u16 d2;
460 u16 __user *s = (u16 __user *)src;
461
462 if (__get_user(d2, s))
463 goto fault;
464 memcpy(buf, &d2, 2);
465 break;
466 }
467 case 4: {
468 u32 d4;
469 u32 __user *s = (u32 __user *)src;
470
471 if (__get_user(d4, s))
472 goto fault;
473 memcpy(buf, &d4, 4);
474 break;
475 }
476 case 8: {
477 u64 d8;
478 u64 __user *s = (u64 __user *)src;
479 if (__get_user(d8, s))
480 goto fault;
481 memcpy(buf, &d8, 8);
482 break;
483 }
484 default:
485 WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size);
486 return ES_UNSUPPORTED;
487 }
488
489 return ES_OK;
490
491 fault:
492 if (user_mode(ctxt->regs))
493 error_code |= X86_PF_USER;
494
495 ctxt->fi.vector = X86_TRAP_PF;
496 ctxt->fi.error_code = error_code;
497 ctxt->fi.cr2 = (unsigned long)src;
498
499 return ES_EXCEPTION;
500 }
501
vc_slow_virt_to_phys(struct ghcb * ghcb,struct es_em_ctxt * ctxt,unsigned long vaddr,phys_addr_t * paddr)502 static enum es_result vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt,
503 unsigned long vaddr, phys_addr_t *paddr)
504 {
505 unsigned long va = (unsigned long)vaddr;
506 unsigned int level;
507 phys_addr_t pa;
508 pgd_t *pgd;
509 pte_t *pte;
510
511 pgd = __va(read_cr3_pa());
512 pgd = &pgd[pgd_index(va)];
513 pte = lookup_address_in_pgd(pgd, va, &level);
514 if (!pte) {
515 ctxt->fi.vector = X86_TRAP_PF;
516 ctxt->fi.cr2 = vaddr;
517 ctxt->fi.error_code = 0;
518
519 if (user_mode(ctxt->regs))
520 ctxt->fi.error_code |= X86_PF_USER;
521
522 return ES_EXCEPTION;
523 }
524
525 if (WARN_ON_ONCE(pte_val(*pte) & _PAGE_ENC))
526 /* Emulated MMIO to/from encrypted memory not supported */
527 return ES_UNSUPPORTED;
528
529 pa = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT;
530 pa |= va & ~page_level_mask(level);
531
532 *paddr = pa;
533
534 return ES_OK;
535 }
536
vc_ioio_check(struct es_em_ctxt * ctxt,u16 port,size_t size)537 static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size)
538 {
539 BUG_ON(size > 4);
540
541 if (user_mode(ctxt->regs)) {
542 struct thread_struct *t = ¤t->thread;
543 struct io_bitmap *iobm = t->io_bitmap;
544 size_t idx;
545
546 if (!iobm)
547 goto fault;
548
549 for (idx = port; idx < port + size; ++idx) {
550 if (test_bit(idx, iobm->bitmap))
551 goto fault;
552 }
553 }
554
555 return ES_OK;
556
557 fault:
558 ctxt->fi.vector = X86_TRAP_GP;
559 ctxt->fi.error_code = 0;
560
561 return ES_EXCEPTION;
562 }
563
vc_forward_exception(struct es_em_ctxt * ctxt)564 static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt)
565 {
566 long error_code = ctxt->fi.error_code;
567 int trapnr = ctxt->fi.vector;
568
569 ctxt->regs->orig_ax = ctxt->fi.error_code;
570
571 switch (trapnr) {
572 case X86_TRAP_GP:
573 exc_general_protection(ctxt->regs, error_code);
574 break;
575 case X86_TRAP_UD:
576 exc_invalid_op(ctxt->regs);
577 break;
578 case X86_TRAP_PF:
579 write_cr2(ctxt->fi.cr2);
580 exc_page_fault(ctxt->regs, error_code);
581 break;
582 case X86_TRAP_AC:
583 exc_alignment_check(ctxt->regs, error_code);
584 break;
585 default:
586 pr_emerg("Unsupported exception in #VC instruction emulation - can't continue\n");
587 BUG();
588 }
589 }
590
591 /* Include code shared with pre-decompression boot stage */
592 #include "shared.c"
593
svsm_get_caa(void)594 static inline struct svsm_ca *svsm_get_caa(void)
595 {
596 /*
597 * Use rIP-relative references when called early in the boot. If
598 * ->use_cas is set, then it is late in the boot and no need
599 * to worry about rIP-relative references.
600 */
601 if (RIP_REL_REF(sev_cfg).use_cas)
602 return this_cpu_read(svsm_caa);
603 else
604 return RIP_REL_REF(boot_svsm_caa);
605 }
606
svsm_get_caa_pa(void)607 static u64 svsm_get_caa_pa(void)
608 {
609 /*
610 * Use rIP-relative references when called early in the boot. If
611 * ->use_cas is set, then it is late in the boot and no need
612 * to worry about rIP-relative references.
613 */
614 if (RIP_REL_REF(sev_cfg).use_cas)
615 return this_cpu_read(svsm_caa_pa);
616 else
617 return RIP_REL_REF(boot_svsm_caa_pa);
618 }
619
__sev_put_ghcb(struct ghcb_state * state)620 static noinstr void __sev_put_ghcb(struct ghcb_state *state)
621 {
622 struct sev_es_runtime_data *data;
623 struct ghcb *ghcb;
624
625 WARN_ON(!irqs_disabled());
626
627 data = this_cpu_read(runtime_data);
628 ghcb = &data->ghcb_page;
629
630 if (state->ghcb) {
631 /* Restore GHCB from Backup */
632 *ghcb = *state->ghcb;
633 data->backup_ghcb_active = false;
634 state->ghcb = NULL;
635 } else {
636 /*
637 * Invalidate the GHCB so a VMGEXIT instruction issued
638 * from userspace won't appear to be valid.
639 */
640 vc_ghcb_invalidate(ghcb);
641 data->ghcb_active = false;
642 }
643 }
644
svsm_perform_call_protocol(struct svsm_call * call)645 static int svsm_perform_call_protocol(struct svsm_call *call)
646 {
647 struct ghcb_state state;
648 unsigned long flags;
649 struct ghcb *ghcb;
650 int ret;
651
652 /*
653 * This can be called very early in the boot, use native functions in
654 * order to avoid paravirt issues.
655 */
656 flags = native_local_irq_save();
657
658 /*
659 * Use rip-relative references when called early in the boot. If
660 * ghcbs_initialized is set, then it is late in the boot and no need
661 * to worry about rip-relative references in called functions.
662 */
663 if (RIP_REL_REF(sev_cfg).ghcbs_initialized)
664 ghcb = __sev_get_ghcb(&state);
665 else if (RIP_REL_REF(boot_ghcb))
666 ghcb = RIP_REL_REF(boot_ghcb);
667 else
668 ghcb = NULL;
669
670 do {
671 ret = ghcb ? svsm_perform_ghcb_protocol(ghcb, call)
672 : svsm_perform_msr_protocol(call);
673 } while (ret == -EAGAIN);
674
675 if (RIP_REL_REF(sev_cfg).ghcbs_initialized)
676 __sev_put_ghcb(&state);
677
678 native_local_irq_restore(flags);
679
680 return ret;
681 }
682
__sev_es_nmi_complete(void)683 void noinstr __sev_es_nmi_complete(void)
684 {
685 struct ghcb_state state;
686 struct ghcb *ghcb;
687
688 ghcb = __sev_get_ghcb(&state);
689
690 vc_ghcb_invalidate(ghcb);
691 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_NMI_COMPLETE);
692 ghcb_set_sw_exit_info_1(ghcb, 0);
693 ghcb_set_sw_exit_info_2(ghcb, 0);
694
695 sev_es_wr_ghcb_msr(__pa_nodebug(ghcb));
696 VMGEXIT();
697
698 __sev_put_ghcb(&state);
699 }
700
get_snp_jump_table_addr(void)701 static u64 __init get_snp_jump_table_addr(void)
702 {
703 struct snp_secrets_page *secrets;
704 void __iomem *mem;
705 u64 addr;
706
707 mem = ioremap_encrypted(secrets_pa, PAGE_SIZE);
708 if (!mem) {
709 pr_err("Unable to locate AP jump table address: failed to map the SNP secrets page.\n");
710 return 0;
711 }
712
713 secrets = (__force struct snp_secrets_page *)mem;
714
715 addr = secrets->os_area.ap_jump_table_pa;
716 iounmap(mem);
717
718 return addr;
719 }
720
get_jump_table_addr(void)721 static u64 __init get_jump_table_addr(void)
722 {
723 struct ghcb_state state;
724 unsigned long flags;
725 struct ghcb *ghcb;
726 u64 ret = 0;
727
728 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
729 return get_snp_jump_table_addr();
730
731 local_irq_save(flags);
732
733 ghcb = __sev_get_ghcb(&state);
734
735 vc_ghcb_invalidate(ghcb);
736 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_JUMP_TABLE);
737 ghcb_set_sw_exit_info_1(ghcb, SVM_VMGEXIT_GET_AP_JUMP_TABLE);
738 ghcb_set_sw_exit_info_2(ghcb, 0);
739
740 sev_es_wr_ghcb_msr(__pa(ghcb));
741 VMGEXIT();
742
743 if (ghcb_sw_exit_info_1_is_valid(ghcb) &&
744 ghcb_sw_exit_info_2_is_valid(ghcb))
745 ret = ghcb->save.sw_exit_info_2;
746
747 __sev_put_ghcb(&state);
748
749 local_irq_restore(flags);
750
751 return ret;
752 }
753
754 static void __head
early_set_pages_state(unsigned long vaddr,unsigned long paddr,unsigned long npages,enum psc_op op)755 early_set_pages_state(unsigned long vaddr, unsigned long paddr,
756 unsigned long npages, enum psc_op op)
757 {
758 unsigned long paddr_end;
759 u64 val;
760
761 vaddr = vaddr & PAGE_MASK;
762
763 paddr = paddr & PAGE_MASK;
764 paddr_end = paddr + (npages << PAGE_SHIFT);
765
766 while (paddr < paddr_end) {
767 /* Page validation must be rescinded before changing to shared */
768 if (op == SNP_PAGE_STATE_SHARED)
769 pvalidate_4k_page(vaddr, paddr, false);
770
771 /*
772 * Use the MSR protocol because this function can be called before
773 * the GHCB is established.
774 */
775 sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op));
776 VMGEXIT();
777
778 val = sev_es_rd_ghcb_msr();
779
780 if (WARN(GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP,
781 "Wrong PSC response code: 0x%x\n",
782 (unsigned int)GHCB_RESP_CODE(val)))
783 goto e_term;
784
785 if (WARN(GHCB_MSR_PSC_RESP_VAL(val),
786 "Failed to change page state to '%s' paddr 0x%lx error 0x%llx\n",
787 op == SNP_PAGE_STATE_PRIVATE ? "private" : "shared",
788 paddr, GHCB_MSR_PSC_RESP_VAL(val)))
789 goto e_term;
790
791 /* Page validation must be performed after changing to private */
792 if (op == SNP_PAGE_STATE_PRIVATE)
793 pvalidate_4k_page(vaddr, paddr, true);
794
795 vaddr += PAGE_SIZE;
796 paddr += PAGE_SIZE;
797 }
798
799 return;
800
801 e_term:
802 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
803 }
804
early_snp_set_memory_private(unsigned long vaddr,unsigned long paddr,unsigned long npages)805 void __head early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr,
806 unsigned long npages)
807 {
808 /*
809 * This can be invoked in early boot while running identity mapped, so
810 * use an open coded check for SNP instead of using cc_platform_has().
811 * This eliminates worries about jump tables or checking boot_cpu_data
812 * in the cc_platform_has() function.
813 */
814 if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED))
815 return;
816
817 /*
818 * Ask the hypervisor to mark the memory pages as private in the RMP
819 * table.
820 */
821 early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_PRIVATE);
822 }
823
early_snp_set_memory_shared(unsigned long vaddr,unsigned long paddr,unsigned long npages)824 void __init early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr,
825 unsigned long npages)
826 {
827 /*
828 * This can be invoked in early boot while running identity mapped, so
829 * use an open coded check for SNP instead of using cc_platform_has().
830 * This eliminates worries about jump tables or checking boot_cpu_data
831 * in the cc_platform_has() function.
832 */
833 if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED))
834 return;
835
836 /* Ask hypervisor to mark the memory pages shared in the RMP table. */
837 early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_SHARED);
838 }
839
__set_pages_state(struct snp_psc_desc * data,unsigned long vaddr,unsigned long vaddr_end,int op)840 static unsigned long __set_pages_state(struct snp_psc_desc *data, unsigned long vaddr,
841 unsigned long vaddr_end, int op)
842 {
843 struct ghcb_state state;
844 bool use_large_entry;
845 struct psc_hdr *hdr;
846 struct psc_entry *e;
847 unsigned long flags;
848 unsigned long pfn;
849 struct ghcb *ghcb;
850 int i;
851
852 hdr = &data->hdr;
853 e = data->entries;
854
855 memset(data, 0, sizeof(*data));
856 i = 0;
857
858 while (vaddr < vaddr_end && i < ARRAY_SIZE(data->entries)) {
859 hdr->end_entry = i;
860
861 if (is_vmalloc_addr((void *)vaddr)) {
862 pfn = vmalloc_to_pfn((void *)vaddr);
863 use_large_entry = false;
864 } else {
865 pfn = __pa(vaddr) >> PAGE_SHIFT;
866 use_large_entry = true;
867 }
868
869 e->gfn = pfn;
870 e->operation = op;
871
872 if (use_large_entry && IS_ALIGNED(vaddr, PMD_SIZE) &&
873 (vaddr_end - vaddr) >= PMD_SIZE) {
874 e->pagesize = RMP_PG_SIZE_2M;
875 vaddr += PMD_SIZE;
876 } else {
877 e->pagesize = RMP_PG_SIZE_4K;
878 vaddr += PAGE_SIZE;
879 }
880
881 e++;
882 i++;
883 }
884
885 /* Page validation must be rescinded before changing to shared */
886 if (op == SNP_PAGE_STATE_SHARED)
887 pvalidate_pages(data);
888
889 local_irq_save(flags);
890
891 if (sev_cfg.ghcbs_initialized)
892 ghcb = __sev_get_ghcb(&state);
893 else
894 ghcb = boot_ghcb;
895
896 /* Invoke the hypervisor to perform the page state changes */
897 if (!ghcb || vmgexit_psc(ghcb, data))
898 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
899
900 if (sev_cfg.ghcbs_initialized)
901 __sev_put_ghcb(&state);
902
903 local_irq_restore(flags);
904
905 /* Page validation must be performed after changing to private */
906 if (op == SNP_PAGE_STATE_PRIVATE)
907 pvalidate_pages(data);
908
909 return vaddr;
910 }
911
set_pages_state(unsigned long vaddr,unsigned long npages,int op)912 static void set_pages_state(unsigned long vaddr, unsigned long npages, int op)
913 {
914 struct snp_psc_desc desc;
915 unsigned long vaddr_end;
916
917 /* Use the MSR protocol when a GHCB is not available. */
918 if (!boot_ghcb)
919 return early_set_pages_state(vaddr, __pa(vaddr), npages, op);
920
921 vaddr = vaddr & PAGE_MASK;
922 vaddr_end = vaddr + (npages << PAGE_SHIFT);
923
924 while (vaddr < vaddr_end)
925 vaddr = __set_pages_state(&desc, vaddr, vaddr_end, op);
926 }
927
snp_set_memory_shared(unsigned long vaddr,unsigned long npages)928 void snp_set_memory_shared(unsigned long vaddr, unsigned long npages)
929 {
930 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
931 return;
932
933 set_pages_state(vaddr, npages, SNP_PAGE_STATE_SHARED);
934 }
935
snp_set_memory_private(unsigned long vaddr,unsigned long npages)936 void snp_set_memory_private(unsigned long vaddr, unsigned long npages)
937 {
938 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
939 return;
940
941 set_pages_state(vaddr, npages, SNP_PAGE_STATE_PRIVATE);
942 }
943
snp_accept_memory(phys_addr_t start,phys_addr_t end)944 void snp_accept_memory(phys_addr_t start, phys_addr_t end)
945 {
946 unsigned long vaddr, npages;
947
948 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
949 return;
950
951 vaddr = (unsigned long)__va(start);
952 npages = (end - start) >> PAGE_SHIFT;
953
954 set_pages_state(vaddr, npages, SNP_PAGE_STATE_PRIVATE);
955 }
956
set_pte_enc(pte_t * kpte,int level,void * va)957 static void set_pte_enc(pte_t *kpte, int level, void *va)
958 {
959 struct pte_enc_desc d = {
960 .kpte = kpte,
961 .pte_level = level,
962 .va = va,
963 .encrypt = true
964 };
965
966 prepare_pte_enc(&d);
967 set_pte_enc_mask(kpte, d.pfn, d.new_pgprot);
968 }
969
unshare_all_memory(void)970 static void unshare_all_memory(void)
971 {
972 unsigned long addr, end, size, ghcb;
973 struct sev_es_runtime_data *data;
974 unsigned int npages, level;
975 bool skipped_addr;
976 pte_t *pte;
977 int cpu;
978
979 /* Unshare the direct mapping. */
980 addr = PAGE_OFFSET;
981 end = PAGE_OFFSET + get_max_mapped();
982
983 while (addr < end) {
984 pte = lookup_address(addr, &level);
985 size = page_level_size(level);
986 npages = size / PAGE_SIZE;
987 skipped_addr = false;
988
989 if (!pte || !pte_decrypted(*pte) || pte_none(*pte)) {
990 addr += size;
991 continue;
992 }
993
994 /*
995 * Ensure that all the per-CPU GHCBs are made private at the
996 * end of the unsharing loop so that the switch to the slower
997 * MSR protocol happens last.
998 */
999 for_each_possible_cpu(cpu) {
1000 data = per_cpu(runtime_data, cpu);
1001 ghcb = (unsigned long)&data->ghcb_page;
1002
1003 if (addr <= ghcb && ghcb <= addr + size) {
1004 skipped_addr = true;
1005 break;
1006 }
1007 }
1008
1009 if (!skipped_addr) {
1010 set_pte_enc(pte, level, (void *)addr);
1011 snp_set_memory_private(addr, npages);
1012 }
1013 addr += size;
1014 }
1015
1016 /* Unshare all bss decrypted memory. */
1017 addr = (unsigned long)__start_bss_decrypted;
1018 end = (unsigned long)__start_bss_decrypted_unused;
1019 npages = (end - addr) >> PAGE_SHIFT;
1020
1021 for (; addr < end; addr += PAGE_SIZE) {
1022 pte = lookup_address(addr, &level);
1023 if (!pte || !pte_decrypted(*pte) || pte_none(*pte))
1024 continue;
1025
1026 set_pte_enc(pte, level, (void *)addr);
1027 }
1028 addr = (unsigned long)__start_bss_decrypted;
1029 snp_set_memory_private(addr, npages);
1030
1031 __flush_tlb_all();
1032 }
1033
1034 /* Stop new private<->shared conversions */
snp_kexec_begin(void)1035 void snp_kexec_begin(void)
1036 {
1037 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
1038 return;
1039
1040 if (!IS_ENABLED(CONFIG_KEXEC_CORE))
1041 return;
1042
1043 /*
1044 * Crash kernel ends up here with interrupts disabled: can't wait for
1045 * conversions to finish.
1046 *
1047 * If race happened, just report and proceed.
1048 */
1049 if (!set_memory_enc_stop_conversion())
1050 pr_warn("Failed to stop shared<->private conversions\n");
1051 }
1052
snp_kexec_finish(void)1053 void snp_kexec_finish(void)
1054 {
1055 struct sev_es_runtime_data *data;
1056 unsigned int level, cpu;
1057 unsigned long size;
1058 struct ghcb *ghcb;
1059 pte_t *pte;
1060
1061 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
1062 return;
1063
1064 if (!IS_ENABLED(CONFIG_KEXEC_CORE))
1065 return;
1066
1067 unshare_all_memory();
1068
1069 /*
1070 * Switch to using the MSR protocol to change per-CPU GHCBs to
1071 * private. All the per-CPU GHCBs have been switched back to private,
1072 * so can't do any more GHCB calls to the hypervisor beyond this point
1073 * until the kexec'ed kernel starts running.
1074 */
1075 boot_ghcb = NULL;
1076 sev_cfg.ghcbs_initialized = false;
1077
1078 for_each_possible_cpu(cpu) {
1079 data = per_cpu(runtime_data, cpu);
1080 ghcb = &data->ghcb_page;
1081 pte = lookup_address((unsigned long)ghcb, &level);
1082 size = page_level_size(level);
1083 set_pte_enc(pte, level, (void *)ghcb);
1084 snp_set_memory_private((unsigned long)ghcb, (size / PAGE_SIZE));
1085 }
1086 }
1087
snp_set_vmsa(void * va,void * caa,int apic_id,bool make_vmsa)1088 static int snp_set_vmsa(void *va, void *caa, int apic_id, bool make_vmsa)
1089 {
1090 int ret;
1091
1092 if (snp_vmpl) {
1093 struct svsm_call call = {};
1094 unsigned long flags;
1095
1096 local_irq_save(flags);
1097
1098 call.caa = this_cpu_read(svsm_caa);
1099 call.rcx = __pa(va);
1100
1101 if (make_vmsa) {
1102 /* Protocol 0, Call ID 2 */
1103 call.rax = SVSM_CORE_CALL(SVSM_CORE_CREATE_VCPU);
1104 call.rdx = __pa(caa);
1105 call.r8 = apic_id;
1106 } else {
1107 /* Protocol 0, Call ID 3 */
1108 call.rax = SVSM_CORE_CALL(SVSM_CORE_DELETE_VCPU);
1109 }
1110
1111 ret = svsm_perform_call_protocol(&call);
1112
1113 local_irq_restore(flags);
1114 } else {
1115 /*
1116 * If the kernel runs at VMPL0, it can change the VMSA
1117 * bit for a page using the RMPADJUST instruction.
1118 * However, for the instruction to succeed it must
1119 * target the permissions of a lesser privileged (higher
1120 * numbered) VMPL level, so use VMPL1.
1121 */
1122 u64 attrs = 1;
1123
1124 if (make_vmsa)
1125 attrs |= RMPADJUST_VMSA_PAGE_BIT;
1126
1127 ret = rmpadjust((unsigned long)va, RMP_PG_SIZE_4K, attrs);
1128 }
1129
1130 return ret;
1131 }
1132
1133 #define __ATTR_BASE (SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK)
1134 #define INIT_CS_ATTRIBS (__ATTR_BASE | SVM_SELECTOR_READ_MASK | SVM_SELECTOR_CODE_MASK)
1135 #define INIT_DS_ATTRIBS (__ATTR_BASE | SVM_SELECTOR_WRITE_MASK)
1136
1137 #define INIT_LDTR_ATTRIBS (SVM_SELECTOR_P_MASK | 2)
1138 #define INIT_TR_ATTRIBS (SVM_SELECTOR_P_MASK | 3)
1139
snp_alloc_vmsa_page(int cpu)1140 static void *snp_alloc_vmsa_page(int cpu)
1141 {
1142 struct page *p;
1143
1144 /*
1145 * Allocate VMSA page to work around the SNP erratum where the CPU will
1146 * incorrectly signal an RMP violation #PF if a large page (2MB or 1GB)
1147 * collides with the RMP entry of VMSA page. The recommended workaround
1148 * is to not use a large page.
1149 *
1150 * Allocate an 8k page which is also 8k-aligned.
1151 */
1152 p = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL_ACCOUNT | __GFP_ZERO, 1);
1153 if (!p)
1154 return NULL;
1155
1156 split_page(p, 1);
1157
1158 /* Free the first 4k. This page may be 2M/1G aligned and cannot be used. */
1159 __free_page(p);
1160
1161 return page_address(p + 1);
1162 }
1163
snp_cleanup_vmsa(struct sev_es_save_area * vmsa,int apic_id)1164 static void snp_cleanup_vmsa(struct sev_es_save_area *vmsa, int apic_id)
1165 {
1166 int err;
1167
1168 err = snp_set_vmsa(vmsa, NULL, apic_id, false);
1169 if (err)
1170 pr_err("clear VMSA page failed (%u), leaking page\n", err);
1171 else
1172 free_page((unsigned long)vmsa);
1173 }
1174
wakeup_cpu_via_vmgexit(u32 apic_id,unsigned long start_ip)1175 static int wakeup_cpu_via_vmgexit(u32 apic_id, unsigned long start_ip)
1176 {
1177 struct sev_es_save_area *cur_vmsa, *vmsa;
1178 struct ghcb_state state;
1179 struct svsm_ca *caa;
1180 unsigned long flags;
1181 struct ghcb *ghcb;
1182 u8 sipi_vector;
1183 int cpu, ret;
1184 u64 cr4;
1185
1186 /*
1187 * The hypervisor SNP feature support check has happened earlier, just check
1188 * the AP_CREATION one here.
1189 */
1190 if (!(sev_hv_features & GHCB_HV_FT_SNP_AP_CREATION))
1191 return -EOPNOTSUPP;
1192
1193 /*
1194 * Verify the desired start IP against the known trampoline start IP
1195 * to catch any future new trampolines that may be introduced that
1196 * would require a new protected guest entry point.
1197 */
1198 if (WARN_ONCE(start_ip != real_mode_header->trampoline_start,
1199 "Unsupported SNP start_ip: %lx\n", start_ip))
1200 return -EINVAL;
1201
1202 /* Override start_ip with known protected guest start IP */
1203 start_ip = real_mode_header->sev_es_trampoline_start;
1204
1205 /* Find the logical CPU for the APIC ID */
1206 for_each_present_cpu(cpu) {
1207 if (arch_match_cpu_phys_id(cpu, apic_id))
1208 break;
1209 }
1210 if (cpu >= nr_cpu_ids)
1211 return -EINVAL;
1212
1213 cur_vmsa = per_cpu(sev_vmsa, cpu);
1214
1215 /*
1216 * A new VMSA is created each time because there is no guarantee that
1217 * the current VMSA is the kernels or that the vCPU is not running. If
1218 * an attempt was done to use the current VMSA with a running vCPU, a
1219 * #VMEXIT of that vCPU would wipe out all of the settings being done
1220 * here.
1221 */
1222 vmsa = (struct sev_es_save_area *)snp_alloc_vmsa_page(cpu);
1223 if (!vmsa)
1224 return -ENOMEM;
1225
1226 /* If an SVSM is present, the SVSM per-CPU CAA will be !NULL */
1227 caa = per_cpu(svsm_caa, cpu);
1228
1229 /* CR4 should maintain the MCE value */
1230 cr4 = native_read_cr4() & X86_CR4_MCE;
1231
1232 /* Set the CS value based on the start_ip converted to a SIPI vector */
1233 sipi_vector = (start_ip >> 12);
1234 vmsa->cs.base = sipi_vector << 12;
1235 vmsa->cs.limit = AP_INIT_CS_LIMIT;
1236 vmsa->cs.attrib = INIT_CS_ATTRIBS;
1237 vmsa->cs.selector = sipi_vector << 8;
1238
1239 /* Set the RIP value based on start_ip */
1240 vmsa->rip = start_ip & 0xfff;
1241
1242 /* Set AP INIT defaults as documented in the APM */
1243 vmsa->ds.limit = AP_INIT_DS_LIMIT;
1244 vmsa->ds.attrib = INIT_DS_ATTRIBS;
1245 vmsa->es = vmsa->ds;
1246 vmsa->fs = vmsa->ds;
1247 vmsa->gs = vmsa->ds;
1248 vmsa->ss = vmsa->ds;
1249
1250 vmsa->gdtr.limit = AP_INIT_GDTR_LIMIT;
1251 vmsa->ldtr.limit = AP_INIT_LDTR_LIMIT;
1252 vmsa->ldtr.attrib = INIT_LDTR_ATTRIBS;
1253 vmsa->idtr.limit = AP_INIT_IDTR_LIMIT;
1254 vmsa->tr.limit = AP_INIT_TR_LIMIT;
1255 vmsa->tr.attrib = INIT_TR_ATTRIBS;
1256
1257 vmsa->cr4 = cr4;
1258 vmsa->cr0 = AP_INIT_CR0_DEFAULT;
1259 vmsa->dr7 = DR7_RESET_VALUE;
1260 vmsa->dr6 = AP_INIT_DR6_DEFAULT;
1261 vmsa->rflags = AP_INIT_RFLAGS_DEFAULT;
1262 vmsa->g_pat = AP_INIT_GPAT_DEFAULT;
1263 vmsa->xcr0 = AP_INIT_XCR0_DEFAULT;
1264 vmsa->mxcsr = AP_INIT_MXCSR_DEFAULT;
1265 vmsa->x87_ftw = AP_INIT_X87_FTW_DEFAULT;
1266 vmsa->x87_fcw = AP_INIT_X87_FCW_DEFAULT;
1267
1268 /* SVME must be set. */
1269 vmsa->efer = EFER_SVME;
1270
1271 /*
1272 * Set the SNP-specific fields for this VMSA:
1273 * VMPL level
1274 * SEV_FEATURES (matches the SEV STATUS MSR right shifted 2 bits)
1275 */
1276 vmsa->vmpl = snp_vmpl;
1277 vmsa->sev_features = sev_status >> 2;
1278
1279 /* Switch the page over to a VMSA page now that it is initialized */
1280 ret = snp_set_vmsa(vmsa, caa, apic_id, true);
1281 if (ret) {
1282 pr_err("set VMSA page failed (%u)\n", ret);
1283 free_page((unsigned long)vmsa);
1284
1285 return -EINVAL;
1286 }
1287
1288 /* Issue VMGEXIT AP Creation NAE event */
1289 local_irq_save(flags);
1290
1291 ghcb = __sev_get_ghcb(&state);
1292
1293 vc_ghcb_invalidate(ghcb);
1294 ghcb_set_rax(ghcb, vmsa->sev_features);
1295 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_CREATION);
1296 ghcb_set_sw_exit_info_1(ghcb,
1297 ((u64)apic_id << 32) |
1298 ((u64)snp_vmpl << 16) |
1299 SVM_VMGEXIT_AP_CREATE);
1300 ghcb_set_sw_exit_info_2(ghcb, __pa(vmsa));
1301
1302 sev_es_wr_ghcb_msr(__pa(ghcb));
1303 VMGEXIT();
1304
1305 if (!ghcb_sw_exit_info_1_is_valid(ghcb) ||
1306 lower_32_bits(ghcb->save.sw_exit_info_1)) {
1307 pr_err("SNP AP Creation error\n");
1308 ret = -EINVAL;
1309 }
1310
1311 __sev_put_ghcb(&state);
1312
1313 local_irq_restore(flags);
1314
1315 /* Perform cleanup if there was an error */
1316 if (ret) {
1317 snp_cleanup_vmsa(vmsa, apic_id);
1318 vmsa = NULL;
1319 }
1320
1321 /* Free up any previous VMSA page */
1322 if (cur_vmsa)
1323 snp_cleanup_vmsa(cur_vmsa, apic_id);
1324
1325 /* Record the current VMSA page */
1326 per_cpu(sev_vmsa, cpu) = vmsa;
1327
1328 return ret;
1329 }
1330
snp_set_wakeup_secondary_cpu(void)1331 void __init snp_set_wakeup_secondary_cpu(void)
1332 {
1333 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
1334 return;
1335
1336 /*
1337 * Always set this override if SNP is enabled. This makes it the
1338 * required method to start APs under SNP. If the hypervisor does
1339 * not support AP creation, then no APs will be started.
1340 */
1341 apic_update_callback(wakeup_secondary_cpu, wakeup_cpu_via_vmgexit);
1342 }
1343
sev_es_setup_ap_jump_table(struct real_mode_header * rmh)1344 int __init sev_es_setup_ap_jump_table(struct real_mode_header *rmh)
1345 {
1346 u16 startup_cs, startup_ip;
1347 phys_addr_t jump_table_pa;
1348 u64 jump_table_addr;
1349 u16 __iomem *jump_table;
1350
1351 jump_table_addr = get_jump_table_addr();
1352
1353 /* On UP guests there is no jump table so this is not a failure */
1354 if (!jump_table_addr)
1355 return 0;
1356
1357 /* Check if AP Jump Table is page-aligned */
1358 if (jump_table_addr & ~PAGE_MASK)
1359 return -EINVAL;
1360
1361 jump_table_pa = jump_table_addr & PAGE_MASK;
1362
1363 startup_cs = (u16)(rmh->trampoline_start >> 4);
1364 startup_ip = (u16)(rmh->sev_es_trampoline_start -
1365 rmh->trampoline_start);
1366
1367 jump_table = ioremap_encrypted(jump_table_pa, PAGE_SIZE);
1368 if (!jump_table)
1369 return -EIO;
1370
1371 writew(startup_ip, &jump_table[0]);
1372 writew(startup_cs, &jump_table[1]);
1373
1374 iounmap(jump_table);
1375
1376 return 0;
1377 }
1378
1379 /*
1380 * This is needed by the OVMF UEFI firmware which will use whatever it finds in
1381 * the GHCB MSR as its GHCB to talk to the hypervisor. So make sure the per-cpu
1382 * runtime GHCBs used by the kernel are also mapped in the EFI page-table.
1383 */
sev_es_efi_map_ghcbs(pgd_t * pgd)1384 int __init sev_es_efi_map_ghcbs(pgd_t *pgd)
1385 {
1386 struct sev_es_runtime_data *data;
1387 unsigned long address, pflags;
1388 int cpu;
1389 u64 pfn;
1390
1391 if (!cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT))
1392 return 0;
1393
1394 pflags = _PAGE_NX | _PAGE_RW;
1395
1396 for_each_possible_cpu(cpu) {
1397 data = per_cpu(runtime_data, cpu);
1398
1399 address = __pa(&data->ghcb_page);
1400 pfn = address >> PAGE_SHIFT;
1401
1402 if (kernel_map_pages_in_pgd(pgd, pfn, address, 1, pflags))
1403 return 1;
1404 }
1405
1406 return 0;
1407 }
1408
1409 /* Writes to the SVSM CAA MSR are ignored */
__vc_handle_msr_caa(struct pt_regs * regs,bool write)1410 static enum es_result __vc_handle_msr_caa(struct pt_regs *regs, bool write)
1411 {
1412 if (write)
1413 return ES_OK;
1414
1415 regs->ax = lower_32_bits(this_cpu_read(svsm_caa_pa));
1416 regs->dx = upper_32_bits(this_cpu_read(svsm_caa_pa));
1417
1418 return ES_OK;
1419 }
1420
vc_handle_msr(struct ghcb * ghcb,struct es_em_ctxt * ctxt)1421 static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
1422 {
1423 struct pt_regs *regs = ctxt->regs;
1424 enum es_result ret;
1425 bool write;
1426
1427 /* Is it a WRMSR? */
1428 write = ctxt->insn.opcode.bytes[1] == 0x30;
1429
1430 if (regs->cx == MSR_SVSM_CAA)
1431 return __vc_handle_msr_caa(regs, write);
1432
1433 ghcb_set_rcx(ghcb, regs->cx);
1434 if (write) {
1435 ghcb_set_rax(ghcb, regs->ax);
1436 ghcb_set_rdx(ghcb, regs->dx);
1437 }
1438
1439 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, write, 0);
1440
1441 if ((ret == ES_OK) && !write) {
1442 regs->ax = ghcb->save.rax;
1443 regs->dx = ghcb->save.rdx;
1444 }
1445
1446 return ret;
1447 }
1448
snp_register_per_cpu_ghcb(void)1449 static void snp_register_per_cpu_ghcb(void)
1450 {
1451 struct sev_es_runtime_data *data;
1452 struct ghcb *ghcb;
1453
1454 data = this_cpu_read(runtime_data);
1455 ghcb = &data->ghcb_page;
1456
1457 snp_register_ghcb_early(__pa(ghcb));
1458 }
1459
setup_ghcb(void)1460 void setup_ghcb(void)
1461 {
1462 if (!cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT))
1463 return;
1464
1465 /*
1466 * Check whether the runtime #VC exception handler is active. It uses
1467 * the per-CPU GHCB page which is set up by sev_es_init_vc_handling().
1468 *
1469 * If SNP is active, register the per-CPU GHCB page so that the runtime
1470 * exception handler can use it.
1471 */
1472 if (initial_vc_handler == (unsigned long)kernel_exc_vmm_communication) {
1473 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
1474 snp_register_per_cpu_ghcb();
1475
1476 sev_cfg.ghcbs_initialized = true;
1477
1478 return;
1479 }
1480
1481 /*
1482 * Make sure the hypervisor talks a supported protocol.
1483 * This gets called only in the BSP boot phase.
1484 */
1485 if (!sev_es_negotiate_protocol())
1486 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
1487
1488 /*
1489 * Clear the boot_ghcb. The first exception comes in before the bss
1490 * section is cleared.
1491 */
1492 memset(&boot_ghcb_page, 0, PAGE_SIZE);
1493
1494 /* Alright - Make the boot-ghcb public */
1495 boot_ghcb = &boot_ghcb_page;
1496
1497 /* SNP guest requires that GHCB GPA must be registered. */
1498 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
1499 snp_register_ghcb_early(__pa(&boot_ghcb_page));
1500 }
1501
1502 #ifdef CONFIG_HOTPLUG_CPU
sev_es_ap_hlt_loop(void)1503 static void sev_es_ap_hlt_loop(void)
1504 {
1505 struct ghcb_state state;
1506 struct ghcb *ghcb;
1507
1508 ghcb = __sev_get_ghcb(&state);
1509
1510 while (true) {
1511 vc_ghcb_invalidate(ghcb);
1512 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_HLT_LOOP);
1513 ghcb_set_sw_exit_info_1(ghcb, 0);
1514 ghcb_set_sw_exit_info_2(ghcb, 0);
1515
1516 sev_es_wr_ghcb_msr(__pa(ghcb));
1517 VMGEXIT();
1518
1519 /* Wakeup signal? */
1520 if (ghcb_sw_exit_info_2_is_valid(ghcb) &&
1521 ghcb->save.sw_exit_info_2)
1522 break;
1523 }
1524
1525 __sev_put_ghcb(&state);
1526 }
1527
1528 /*
1529 * Play_dead handler when running under SEV-ES. This is needed because
1530 * the hypervisor can't deliver an SIPI request to restart the AP.
1531 * Instead the kernel has to issue a VMGEXIT to halt the VCPU until the
1532 * hypervisor wakes it up again.
1533 */
sev_es_play_dead(void)1534 static void sev_es_play_dead(void)
1535 {
1536 play_dead_common();
1537
1538 /* IRQs now disabled */
1539
1540 sev_es_ap_hlt_loop();
1541
1542 /*
1543 * If we get here, the VCPU was woken up again. Jump to CPU
1544 * startup code to get it back online.
1545 */
1546 soft_restart_cpu();
1547 }
1548 #else /* CONFIG_HOTPLUG_CPU */
1549 #define sev_es_play_dead native_play_dead
1550 #endif /* CONFIG_HOTPLUG_CPU */
1551
1552 #ifdef CONFIG_SMP
sev_es_setup_play_dead(void)1553 static void __init sev_es_setup_play_dead(void)
1554 {
1555 smp_ops.play_dead = sev_es_play_dead;
1556 }
1557 #else
sev_es_setup_play_dead(void)1558 static inline void sev_es_setup_play_dead(void) { }
1559 #endif
1560
alloc_runtime_data(int cpu)1561 static void __init alloc_runtime_data(int cpu)
1562 {
1563 struct sev_es_runtime_data *data;
1564
1565 data = memblock_alloc_node(sizeof(*data), PAGE_SIZE, cpu_to_node(cpu));
1566 if (!data)
1567 panic("Can't allocate SEV-ES runtime data");
1568
1569 per_cpu(runtime_data, cpu) = data;
1570
1571 if (snp_vmpl) {
1572 struct svsm_ca *caa;
1573
1574 /* Allocate the SVSM CA page if an SVSM is present */
1575 caa = memblock_alloc(sizeof(*caa), PAGE_SIZE);
1576 if (!caa)
1577 panic("Can't allocate SVSM CA page\n");
1578
1579 per_cpu(svsm_caa, cpu) = caa;
1580 per_cpu(svsm_caa_pa, cpu) = __pa(caa);
1581 }
1582 }
1583
init_ghcb(int cpu)1584 static void __init init_ghcb(int cpu)
1585 {
1586 struct sev_es_runtime_data *data;
1587 int err;
1588
1589 data = per_cpu(runtime_data, cpu);
1590
1591 err = early_set_memory_decrypted((unsigned long)&data->ghcb_page,
1592 sizeof(data->ghcb_page));
1593 if (err)
1594 panic("Can't map GHCBs unencrypted");
1595
1596 memset(&data->ghcb_page, 0, sizeof(data->ghcb_page));
1597
1598 data->ghcb_active = false;
1599 data->backup_ghcb_active = false;
1600 }
1601
sev_es_init_vc_handling(void)1602 void __init sev_es_init_vc_handling(void)
1603 {
1604 int cpu;
1605
1606 BUILD_BUG_ON(offsetof(struct sev_es_runtime_data, ghcb_page) % PAGE_SIZE);
1607
1608 if (!cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT))
1609 return;
1610
1611 if (!sev_es_check_cpu_features())
1612 panic("SEV-ES CPU Features missing");
1613
1614 /*
1615 * SNP is supported in v2 of the GHCB spec which mandates support for HV
1616 * features.
1617 */
1618 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) {
1619 sev_hv_features = get_hv_features();
1620
1621 if (!(sev_hv_features & GHCB_HV_FT_SNP))
1622 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
1623 }
1624
1625 /* Initialize per-cpu GHCB pages */
1626 for_each_possible_cpu(cpu) {
1627 alloc_runtime_data(cpu);
1628 init_ghcb(cpu);
1629 }
1630
1631 /* If running under an SVSM, switch to the per-cpu CA */
1632 if (snp_vmpl) {
1633 struct svsm_call call = {};
1634 unsigned long flags;
1635 int ret;
1636
1637 local_irq_save(flags);
1638
1639 /*
1640 * SVSM_CORE_REMAP_CA call:
1641 * RAX = 0 (Protocol=0, CallID=0)
1642 * RCX = New CA GPA
1643 */
1644 call.caa = svsm_get_caa();
1645 call.rax = SVSM_CORE_CALL(SVSM_CORE_REMAP_CA);
1646 call.rcx = this_cpu_read(svsm_caa_pa);
1647 ret = svsm_perform_call_protocol(&call);
1648 if (ret)
1649 panic("Can't remap the SVSM CA, ret=%d, rax_out=0x%llx\n",
1650 ret, call.rax_out);
1651
1652 sev_cfg.use_cas = true;
1653
1654 local_irq_restore(flags);
1655 }
1656
1657 sev_es_setup_play_dead();
1658
1659 /* Secondary CPUs use the runtime #VC handler */
1660 initial_vc_handler = (unsigned long)kernel_exc_vmm_communication;
1661 }
1662
vc_early_forward_exception(struct es_em_ctxt * ctxt)1663 static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt)
1664 {
1665 int trapnr = ctxt->fi.vector;
1666
1667 if (trapnr == X86_TRAP_PF)
1668 native_write_cr2(ctxt->fi.cr2);
1669
1670 ctxt->regs->orig_ax = ctxt->fi.error_code;
1671 do_early_exception(ctxt->regs, trapnr);
1672 }
1673
vc_insn_get_rm(struct es_em_ctxt * ctxt)1674 static long *vc_insn_get_rm(struct es_em_ctxt *ctxt)
1675 {
1676 long *reg_array;
1677 int offset;
1678
1679 reg_array = (long *)ctxt->regs;
1680 offset = insn_get_modrm_rm_off(&ctxt->insn, ctxt->regs);
1681
1682 if (offset < 0)
1683 return NULL;
1684
1685 offset /= sizeof(long);
1686
1687 return reg_array + offset;
1688 }
vc_do_mmio(struct ghcb * ghcb,struct es_em_ctxt * ctxt,unsigned int bytes,bool read)1689 static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt,
1690 unsigned int bytes, bool read)
1691 {
1692 u64 exit_code, exit_info_1, exit_info_2;
1693 unsigned long ghcb_pa = __pa(ghcb);
1694 enum es_result res;
1695 phys_addr_t paddr;
1696 void __user *ref;
1697
1698 ref = insn_get_addr_ref(&ctxt->insn, ctxt->regs);
1699 if (ref == (void __user *)-1L)
1700 return ES_UNSUPPORTED;
1701
1702 exit_code = read ? SVM_VMGEXIT_MMIO_READ : SVM_VMGEXIT_MMIO_WRITE;
1703
1704 res = vc_slow_virt_to_phys(ghcb, ctxt, (unsigned long)ref, &paddr);
1705 if (res != ES_OK) {
1706 if (res == ES_EXCEPTION && !read)
1707 ctxt->fi.error_code |= X86_PF_WRITE;
1708
1709 return res;
1710 }
1711
1712 exit_info_1 = paddr;
1713 /* Can never be greater than 8 */
1714 exit_info_2 = bytes;
1715
1716 ghcb_set_sw_scratch(ghcb, ghcb_pa + offsetof(struct ghcb, shared_buffer));
1717
1718 return sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, exit_info_1, exit_info_2);
1719 }
1720
1721 /*
1722 * The MOVS instruction has two memory operands, which raises the
1723 * problem that it is not known whether the access to the source or the
1724 * destination caused the #VC exception (and hence whether an MMIO read
1725 * or write operation needs to be emulated).
1726 *
1727 * Instead of playing games with walking page-tables and trying to guess
1728 * whether the source or destination is an MMIO range, split the move
1729 * into two operations, a read and a write with only one memory operand.
1730 * This will cause a nested #VC exception on the MMIO address which can
1731 * then be handled.
1732 *
1733 * This implementation has the benefit that it also supports MOVS where
1734 * source _and_ destination are MMIO regions.
1735 *
1736 * It will slow MOVS on MMIO down a lot, but in SEV-ES guests it is a
1737 * rare operation. If it turns out to be a performance problem the split
1738 * operations can be moved to memcpy_fromio() and memcpy_toio().
1739 */
vc_handle_mmio_movs(struct es_em_ctxt * ctxt,unsigned int bytes)1740 static enum es_result vc_handle_mmio_movs(struct es_em_ctxt *ctxt,
1741 unsigned int bytes)
1742 {
1743 unsigned long ds_base, es_base;
1744 unsigned char *src, *dst;
1745 unsigned char buffer[8];
1746 enum es_result ret;
1747 bool rep;
1748 int off;
1749
1750 ds_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_DS);
1751 es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
1752
1753 if (ds_base == -1L || es_base == -1L) {
1754 ctxt->fi.vector = X86_TRAP_GP;
1755 ctxt->fi.error_code = 0;
1756 return ES_EXCEPTION;
1757 }
1758
1759 src = ds_base + (unsigned char *)ctxt->regs->si;
1760 dst = es_base + (unsigned char *)ctxt->regs->di;
1761
1762 ret = vc_read_mem(ctxt, src, buffer, bytes);
1763 if (ret != ES_OK)
1764 return ret;
1765
1766 ret = vc_write_mem(ctxt, dst, buffer, bytes);
1767 if (ret != ES_OK)
1768 return ret;
1769
1770 if (ctxt->regs->flags & X86_EFLAGS_DF)
1771 off = -bytes;
1772 else
1773 off = bytes;
1774
1775 ctxt->regs->si += off;
1776 ctxt->regs->di += off;
1777
1778 rep = insn_has_rep_prefix(&ctxt->insn);
1779 if (rep)
1780 ctxt->regs->cx -= 1;
1781
1782 if (!rep || ctxt->regs->cx == 0)
1783 return ES_OK;
1784 else
1785 return ES_RETRY;
1786 }
1787
vc_handle_mmio(struct ghcb * ghcb,struct es_em_ctxt * ctxt)1788 static enum es_result vc_handle_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
1789 {
1790 struct insn *insn = &ctxt->insn;
1791 enum insn_mmio_type mmio;
1792 unsigned int bytes = 0;
1793 enum es_result ret;
1794 u8 sign_byte;
1795 long *reg_data;
1796
1797 mmio = insn_decode_mmio(insn, &bytes);
1798 if (mmio == INSN_MMIO_DECODE_FAILED)
1799 return ES_DECODE_FAILED;
1800
1801 if (mmio != INSN_MMIO_WRITE_IMM && mmio != INSN_MMIO_MOVS) {
1802 reg_data = insn_get_modrm_reg_ptr(insn, ctxt->regs);
1803 if (!reg_data)
1804 return ES_DECODE_FAILED;
1805 }
1806
1807 if (user_mode(ctxt->regs))
1808 return ES_UNSUPPORTED;
1809
1810 switch (mmio) {
1811 case INSN_MMIO_WRITE:
1812 memcpy(ghcb->shared_buffer, reg_data, bytes);
1813 ret = vc_do_mmio(ghcb, ctxt, bytes, false);
1814 break;
1815 case INSN_MMIO_WRITE_IMM:
1816 memcpy(ghcb->shared_buffer, insn->immediate1.bytes, bytes);
1817 ret = vc_do_mmio(ghcb, ctxt, bytes, false);
1818 break;
1819 case INSN_MMIO_READ:
1820 ret = vc_do_mmio(ghcb, ctxt, bytes, true);
1821 if (ret)
1822 break;
1823
1824 /* Zero-extend for 32-bit operation */
1825 if (bytes == 4)
1826 *reg_data = 0;
1827
1828 memcpy(reg_data, ghcb->shared_buffer, bytes);
1829 break;
1830 case INSN_MMIO_READ_ZERO_EXTEND:
1831 ret = vc_do_mmio(ghcb, ctxt, bytes, true);
1832 if (ret)
1833 break;
1834
1835 /* Zero extend based on operand size */
1836 memset(reg_data, 0, insn->opnd_bytes);
1837 memcpy(reg_data, ghcb->shared_buffer, bytes);
1838 break;
1839 case INSN_MMIO_READ_SIGN_EXTEND:
1840 ret = vc_do_mmio(ghcb, ctxt, bytes, true);
1841 if (ret)
1842 break;
1843
1844 if (bytes == 1) {
1845 u8 *val = (u8 *)ghcb->shared_buffer;
1846
1847 sign_byte = (*val & 0x80) ? 0xff : 0x00;
1848 } else {
1849 u16 *val = (u16 *)ghcb->shared_buffer;
1850
1851 sign_byte = (*val & 0x8000) ? 0xff : 0x00;
1852 }
1853
1854 /* Sign extend based on operand size */
1855 memset(reg_data, sign_byte, insn->opnd_bytes);
1856 memcpy(reg_data, ghcb->shared_buffer, bytes);
1857 break;
1858 case INSN_MMIO_MOVS:
1859 ret = vc_handle_mmio_movs(ctxt, bytes);
1860 break;
1861 default:
1862 ret = ES_UNSUPPORTED;
1863 break;
1864 }
1865
1866 return ret;
1867 }
1868
vc_handle_dr7_write(struct ghcb * ghcb,struct es_em_ctxt * ctxt)1869 static enum es_result vc_handle_dr7_write(struct ghcb *ghcb,
1870 struct es_em_ctxt *ctxt)
1871 {
1872 struct sev_es_runtime_data *data = this_cpu_read(runtime_data);
1873 long val, *reg = vc_insn_get_rm(ctxt);
1874 enum es_result ret;
1875
1876 if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP)
1877 return ES_VMM_ERROR;
1878
1879 if (!reg)
1880 return ES_DECODE_FAILED;
1881
1882 val = *reg;
1883
1884 /* Upper 32 bits must be written as zeroes */
1885 if (val >> 32) {
1886 ctxt->fi.vector = X86_TRAP_GP;
1887 ctxt->fi.error_code = 0;
1888 return ES_EXCEPTION;
1889 }
1890
1891 /* Clear out other reserved bits and set bit 10 */
1892 val = (val & 0xffff23ffL) | BIT(10);
1893
1894 /* Early non-zero writes to DR7 are not supported */
1895 if (!data && (val & ~DR7_RESET_VALUE))
1896 return ES_UNSUPPORTED;
1897
1898 /* Using a value of 0 for ExitInfo1 means RAX holds the value */
1899 ghcb_set_rax(ghcb, val);
1900 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WRITE_DR7, 0, 0);
1901 if (ret != ES_OK)
1902 return ret;
1903
1904 if (data)
1905 data->dr7 = val;
1906
1907 return ES_OK;
1908 }
1909
vc_handle_dr7_read(struct ghcb * ghcb,struct es_em_ctxt * ctxt)1910 static enum es_result vc_handle_dr7_read(struct ghcb *ghcb,
1911 struct es_em_ctxt *ctxt)
1912 {
1913 struct sev_es_runtime_data *data = this_cpu_read(runtime_data);
1914 long *reg = vc_insn_get_rm(ctxt);
1915
1916 if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP)
1917 return ES_VMM_ERROR;
1918
1919 if (!reg)
1920 return ES_DECODE_FAILED;
1921
1922 if (data)
1923 *reg = data->dr7;
1924 else
1925 *reg = DR7_RESET_VALUE;
1926
1927 return ES_OK;
1928 }
1929
vc_handle_wbinvd(struct ghcb * ghcb,struct es_em_ctxt * ctxt)1930 static enum es_result vc_handle_wbinvd(struct ghcb *ghcb,
1931 struct es_em_ctxt *ctxt)
1932 {
1933 return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0);
1934 }
1935
vc_handle_rdpmc(struct ghcb * ghcb,struct es_em_ctxt * ctxt)1936 static enum es_result vc_handle_rdpmc(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
1937 {
1938 enum es_result ret;
1939
1940 ghcb_set_rcx(ghcb, ctxt->regs->cx);
1941
1942 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_RDPMC, 0, 0);
1943 if (ret != ES_OK)
1944 return ret;
1945
1946 if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb)))
1947 return ES_VMM_ERROR;
1948
1949 ctxt->regs->ax = ghcb->save.rax;
1950 ctxt->regs->dx = ghcb->save.rdx;
1951
1952 return ES_OK;
1953 }
1954
vc_handle_monitor(struct ghcb * ghcb,struct es_em_ctxt * ctxt)1955 static enum es_result vc_handle_monitor(struct ghcb *ghcb,
1956 struct es_em_ctxt *ctxt)
1957 {
1958 /*
1959 * Treat it as a NOP and do not leak a physical address to the
1960 * hypervisor.
1961 */
1962 return ES_OK;
1963 }
1964
vc_handle_mwait(struct ghcb * ghcb,struct es_em_ctxt * ctxt)1965 static enum es_result vc_handle_mwait(struct ghcb *ghcb,
1966 struct es_em_ctxt *ctxt)
1967 {
1968 /* Treat the same as MONITOR/MONITORX */
1969 return ES_OK;
1970 }
1971
vc_handle_vmmcall(struct ghcb * ghcb,struct es_em_ctxt * ctxt)1972 static enum es_result vc_handle_vmmcall(struct ghcb *ghcb,
1973 struct es_em_ctxt *ctxt)
1974 {
1975 enum es_result ret;
1976
1977 ghcb_set_rax(ghcb, ctxt->regs->ax);
1978 ghcb_set_cpl(ghcb, user_mode(ctxt->regs) ? 3 : 0);
1979
1980 if (x86_platform.hyper.sev_es_hcall_prepare)
1981 x86_platform.hyper.sev_es_hcall_prepare(ghcb, ctxt->regs);
1982
1983 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_VMMCALL, 0, 0);
1984 if (ret != ES_OK)
1985 return ret;
1986
1987 if (!ghcb_rax_is_valid(ghcb))
1988 return ES_VMM_ERROR;
1989
1990 ctxt->regs->ax = ghcb->save.rax;
1991
1992 /*
1993 * Call sev_es_hcall_finish() after regs->ax is already set.
1994 * This allows the hypervisor handler to overwrite it again if
1995 * necessary.
1996 */
1997 if (x86_platform.hyper.sev_es_hcall_finish &&
1998 !x86_platform.hyper.sev_es_hcall_finish(ghcb, ctxt->regs))
1999 return ES_VMM_ERROR;
2000
2001 return ES_OK;
2002 }
2003
vc_handle_trap_ac(struct ghcb * ghcb,struct es_em_ctxt * ctxt)2004 static enum es_result vc_handle_trap_ac(struct ghcb *ghcb,
2005 struct es_em_ctxt *ctxt)
2006 {
2007 /*
2008 * Calling ecx_alignment_check() directly does not work, because it
2009 * enables IRQs and the GHCB is active. Forward the exception and call
2010 * it later from vc_forward_exception().
2011 */
2012 ctxt->fi.vector = X86_TRAP_AC;
2013 ctxt->fi.error_code = 0;
2014 return ES_EXCEPTION;
2015 }
2016
vc_handle_exitcode(struct es_em_ctxt * ctxt,struct ghcb * ghcb,unsigned long exit_code)2017 static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt,
2018 struct ghcb *ghcb,
2019 unsigned long exit_code)
2020 {
2021 enum es_result result = vc_check_opcode_bytes(ctxt, exit_code);
2022
2023 if (result != ES_OK)
2024 return result;
2025
2026 switch (exit_code) {
2027 case SVM_EXIT_READ_DR7:
2028 result = vc_handle_dr7_read(ghcb, ctxt);
2029 break;
2030 case SVM_EXIT_WRITE_DR7:
2031 result = vc_handle_dr7_write(ghcb, ctxt);
2032 break;
2033 case SVM_EXIT_EXCP_BASE + X86_TRAP_AC:
2034 result = vc_handle_trap_ac(ghcb, ctxt);
2035 break;
2036 case SVM_EXIT_RDTSC:
2037 case SVM_EXIT_RDTSCP:
2038 result = vc_handle_rdtsc(ghcb, ctxt, exit_code);
2039 break;
2040 case SVM_EXIT_RDPMC:
2041 result = vc_handle_rdpmc(ghcb, ctxt);
2042 break;
2043 case SVM_EXIT_INVD:
2044 pr_err_ratelimited("#VC exception for INVD??? Seriously???\n");
2045 result = ES_UNSUPPORTED;
2046 break;
2047 case SVM_EXIT_CPUID:
2048 result = vc_handle_cpuid(ghcb, ctxt);
2049 break;
2050 case SVM_EXIT_IOIO:
2051 result = vc_handle_ioio(ghcb, ctxt);
2052 break;
2053 case SVM_EXIT_MSR:
2054 result = vc_handle_msr(ghcb, ctxt);
2055 break;
2056 case SVM_EXIT_VMMCALL:
2057 result = vc_handle_vmmcall(ghcb, ctxt);
2058 break;
2059 case SVM_EXIT_WBINVD:
2060 result = vc_handle_wbinvd(ghcb, ctxt);
2061 break;
2062 case SVM_EXIT_MONITOR:
2063 result = vc_handle_monitor(ghcb, ctxt);
2064 break;
2065 case SVM_EXIT_MWAIT:
2066 result = vc_handle_mwait(ghcb, ctxt);
2067 break;
2068 case SVM_EXIT_NPF:
2069 result = vc_handle_mmio(ghcb, ctxt);
2070 break;
2071 default:
2072 /*
2073 * Unexpected #VC exception
2074 */
2075 result = ES_UNSUPPORTED;
2076 }
2077
2078 return result;
2079 }
2080
is_vc2_stack(unsigned long sp)2081 static __always_inline bool is_vc2_stack(unsigned long sp)
2082 {
2083 return (sp >= __this_cpu_ist_bottom_va(VC2) && sp < __this_cpu_ist_top_va(VC2));
2084 }
2085
vc_from_invalid_context(struct pt_regs * regs)2086 static __always_inline bool vc_from_invalid_context(struct pt_regs *regs)
2087 {
2088 unsigned long sp, prev_sp;
2089
2090 sp = (unsigned long)regs;
2091 prev_sp = regs->sp;
2092
2093 /*
2094 * If the code was already executing on the VC2 stack when the #VC
2095 * happened, let it proceed to the normal handling routine. This way the
2096 * code executing on the VC2 stack can cause #VC exceptions to get handled.
2097 */
2098 return is_vc2_stack(sp) && !is_vc2_stack(prev_sp);
2099 }
2100
vc_raw_handle_exception(struct pt_regs * regs,unsigned long error_code)2101 static bool vc_raw_handle_exception(struct pt_regs *regs, unsigned long error_code)
2102 {
2103 struct ghcb_state state;
2104 struct es_em_ctxt ctxt;
2105 enum es_result result;
2106 struct ghcb *ghcb;
2107 bool ret = true;
2108
2109 ghcb = __sev_get_ghcb(&state);
2110
2111 vc_ghcb_invalidate(ghcb);
2112 result = vc_init_em_ctxt(&ctxt, regs, error_code);
2113
2114 if (result == ES_OK)
2115 result = vc_handle_exitcode(&ctxt, ghcb, error_code);
2116
2117 __sev_put_ghcb(&state);
2118
2119 /* Done - now check the result */
2120 switch (result) {
2121 case ES_OK:
2122 vc_finish_insn(&ctxt);
2123 break;
2124 case ES_UNSUPPORTED:
2125 pr_err_ratelimited("Unsupported exit-code 0x%02lx in #VC exception (IP: 0x%lx)\n",
2126 error_code, regs->ip);
2127 ret = false;
2128 break;
2129 case ES_VMM_ERROR:
2130 pr_err_ratelimited("Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n",
2131 error_code, regs->ip);
2132 ret = false;
2133 break;
2134 case ES_DECODE_FAILED:
2135 pr_err_ratelimited("Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n",
2136 error_code, regs->ip);
2137 ret = false;
2138 break;
2139 case ES_EXCEPTION:
2140 vc_forward_exception(&ctxt);
2141 break;
2142 case ES_RETRY:
2143 /* Nothing to do */
2144 break;
2145 default:
2146 pr_emerg("Unknown result in %s():%d\n", __func__, result);
2147 /*
2148 * Emulating the instruction which caused the #VC exception
2149 * failed - can't continue so print debug information
2150 */
2151 BUG();
2152 }
2153
2154 return ret;
2155 }
2156
vc_is_db(unsigned long error_code)2157 static __always_inline bool vc_is_db(unsigned long error_code)
2158 {
2159 return error_code == SVM_EXIT_EXCP_BASE + X86_TRAP_DB;
2160 }
2161
2162 /*
2163 * Runtime #VC exception handler when raised from kernel mode. Runs in NMI mode
2164 * and will panic when an error happens.
2165 */
DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication)2166 DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication)
2167 {
2168 irqentry_state_t irq_state;
2169
2170 /*
2171 * With the current implementation it is always possible to switch to a
2172 * safe stack because #VC exceptions only happen at known places, like
2173 * intercepted instructions or accesses to MMIO areas/IO ports. They can
2174 * also happen with code instrumentation when the hypervisor intercepts
2175 * #DB, but the critical paths are forbidden to be instrumented, so #DB
2176 * exceptions currently also only happen in safe places.
2177 *
2178 * But keep this here in case the noinstr annotations are violated due
2179 * to bug elsewhere.
2180 */
2181 if (unlikely(vc_from_invalid_context(regs))) {
2182 instrumentation_begin();
2183 panic("Can't handle #VC exception from unsupported context\n");
2184 instrumentation_end();
2185 }
2186
2187 /*
2188 * Handle #DB before calling into !noinstr code to avoid recursive #DB.
2189 */
2190 if (vc_is_db(error_code)) {
2191 exc_debug(regs);
2192 return;
2193 }
2194
2195 irq_state = irqentry_nmi_enter(regs);
2196
2197 instrumentation_begin();
2198
2199 if (!vc_raw_handle_exception(regs, error_code)) {
2200 /* Show some debug info */
2201 show_regs(regs);
2202
2203 /* Ask hypervisor to sev_es_terminate */
2204 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
2205
2206 /* If that fails and we get here - just panic */
2207 panic("Returned from Terminate-Request to Hypervisor\n");
2208 }
2209
2210 instrumentation_end();
2211 irqentry_nmi_exit(regs, irq_state);
2212 }
2213
2214 /*
2215 * Runtime #VC exception handler when raised from user mode. Runs in IRQ mode
2216 * and will kill the current task with SIGBUS when an error happens.
2217 */
DEFINE_IDTENTRY_VC_USER(exc_vmm_communication)2218 DEFINE_IDTENTRY_VC_USER(exc_vmm_communication)
2219 {
2220 /*
2221 * Handle #DB before calling into !noinstr code to avoid recursive #DB.
2222 */
2223 if (vc_is_db(error_code)) {
2224 noist_exc_debug(regs);
2225 return;
2226 }
2227
2228 irqentry_enter_from_user_mode(regs);
2229 instrumentation_begin();
2230
2231 if (!vc_raw_handle_exception(regs, error_code)) {
2232 /*
2233 * Do not kill the machine if user-space triggered the
2234 * exception. Send SIGBUS instead and let user-space deal with
2235 * it.
2236 */
2237 force_sig_fault(SIGBUS, BUS_OBJERR, (void __user *)0);
2238 }
2239
2240 instrumentation_end();
2241 irqentry_exit_to_user_mode(regs);
2242 }
2243
handle_vc_boot_ghcb(struct pt_regs * regs)2244 bool __init handle_vc_boot_ghcb(struct pt_regs *regs)
2245 {
2246 unsigned long exit_code = regs->orig_ax;
2247 struct es_em_ctxt ctxt;
2248 enum es_result result;
2249
2250 vc_ghcb_invalidate(boot_ghcb);
2251
2252 result = vc_init_em_ctxt(&ctxt, regs, exit_code);
2253 if (result == ES_OK)
2254 result = vc_handle_exitcode(&ctxt, boot_ghcb, exit_code);
2255
2256 /* Done - now check the result */
2257 switch (result) {
2258 case ES_OK:
2259 vc_finish_insn(&ctxt);
2260 break;
2261 case ES_UNSUPPORTED:
2262 early_printk("PANIC: Unsupported exit-code 0x%02lx in early #VC exception (IP: 0x%lx)\n",
2263 exit_code, regs->ip);
2264 goto fail;
2265 case ES_VMM_ERROR:
2266 early_printk("PANIC: Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n",
2267 exit_code, regs->ip);
2268 goto fail;
2269 case ES_DECODE_FAILED:
2270 early_printk("PANIC: Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n",
2271 exit_code, regs->ip);
2272 goto fail;
2273 case ES_EXCEPTION:
2274 vc_early_forward_exception(&ctxt);
2275 break;
2276 case ES_RETRY:
2277 /* Nothing to do */
2278 break;
2279 default:
2280 BUG();
2281 }
2282
2283 return true;
2284
2285 fail:
2286 show_regs(regs);
2287
2288 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
2289 }
2290
2291 /*
2292 * Initial set up of SNP relies on information provided by the
2293 * Confidential Computing blob, which can be passed to the kernel
2294 * in the following ways, depending on how it is booted:
2295 *
2296 * - when booted via the boot/decompress kernel:
2297 * - via boot_params
2298 *
2299 * - when booted directly by firmware/bootloader (e.g. CONFIG_PVH):
2300 * - via a setup_data entry, as defined by the Linux Boot Protocol
2301 *
2302 * Scan for the blob in that order.
2303 */
find_cc_blob(struct boot_params * bp)2304 static __head struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp)
2305 {
2306 struct cc_blob_sev_info *cc_info;
2307
2308 /* Boot kernel would have passed the CC blob via boot_params. */
2309 if (bp->cc_blob_address) {
2310 cc_info = (struct cc_blob_sev_info *)(unsigned long)bp->cc_blob_address;
2311 goto found_cc_info;
2312 }
2313
2314 /*
2315 * If kernel was booted directly, without the use of the
2316 * boot/decompression kernel, the CC blob may have been passed via
2317 * setup_data instead.
2318 */
2319 cc_info = find_cc_blob_setup_data(bp);
2320 if (!cc_info)
2321 return NULL;
2322
2323 found_cc_info:
2324 if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC)
2325 snp_abort();
2326
2327 return cc_info;
2328 }
2329
svsm_setup(struct cc_blob_sev_info * cc_info)2330 static __head void svsm_setup(struct cc_blob_sev_info *cc_info)
2331 {
2332 struct svsm_call call = {};
2333 int ret;
2334 u64 pa;
2335
2336 /*
2337 * Record the SVSM Calling Area address (CAA) if the guest is not
2338 * running at VMPL0. The CA will be used to communicate with the
2339 * SVSM to perform the SVSM services.
2340 */
2341 if (!svsm_setup_ca(cc_info))
2342 return;
2343
2344 /*
2345 * It is very early in the boot and the kernel is running identity
2346 * mapped but without having adjusted the pagetables to where the
2347 * kernel was loaded (physbase), so the get the CA address using
2348 * RIP-relative addressing.
2349 */
2350 pa = (u64)&RIP_REL_REF(boot_svsm_ca_page);
2351
2352 /*
2353 * Switch over to the boot SVSM CA while the current CA is still
2354 * addressable. There is no GHCB at this point so use the MSR protocol.
2355 *
2356 * SVSM_CORE_REMAP_CA call:
2357 * RAX = 0 (Protocol=0, CallID=0)
2358 * RCX = New CA GPA
2359 */
2360 call.caa = svsm_get_caa();
2361 call.rax = SVSM_CORE_CALL(SVSM_CORE_REMAP_CA);
2362 call.rcx = pa;
2363 ret = svsm_perform_call_protocol(&call);
2364 if (ret)
2365 panic("Can't remap the SVSM CA, ret=%d, rax_out=0x%llx\n", ret, call.rax_out);
2366
2367 RIP_REL_REF(boot_svsm_caa) = (struct svsm_ca *)pa;
2368 RIP_REL_REF(boot_svsm_caa_pa) = pa;
2369 }
2370
snp_init(struct boot_params * bp)2371 bool __head snp_init(struct boot_params *bp)
2372 {
2373 struct cc_blob_sev_info *cc_info;
2374
2375 if (!bp)
2376 return false;
2377
2378 cc_info = find_cc_blob(bp);
2379 if (!cc_info)
2380 return false;
2381
2382 if (cc_info->secrets_phys && cc_info->secrets_len == PAGE_SIZE)
2383 secrets_pa = cc_info->secrets_phys;
2384 else
2385 return false;
2386
2387 setup_cpuid_table(cc_info);
2388
2389 svsm_setup(cc_info);
2390
2391 /*
2392 * The CC blob will be used later to access the secrets page. Cache
2393 * it here like the boot kernel does.
2394 */
2395 bp->cc_blob_address = (u32)(unsigned long)cc_info;
2396
2397 return true;
2398 }
2399
snp_abort(void)2400 void __head __noreturn snp_abort(void)
2401 {
2402 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
2403 }
2404
2405 /*
2406 * SEV-SNP guests should only execute dmi_setup() if EFI_CONFIG_TABLES are
2407 * enabled, as the alternative (fallback) logic for DMI probing in the legacy
2408 * ROM region can cause a crash since this region is not pre-validated.
2409 */
snp_dmi_setup(void)2410 void __init snp_dmi_setup(void)
2411 {
2412 if (efi_enabled(EFI_CONFIG_TABLES))
2413 dmi_setup();
2414 }
2415
dump_cpuid_table(void)2416 static void dump_cpuid_table(void)
2417 {
2418 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
2419 int i = 0;
2420
2421 pr_info("count=%d reserved=0x%x reserved2=0x%llx\n",
2422 cpuid_table->count, cpuid_table->__reserved1, cpuid_table->__reserved2);
2423
2424 for (i = 0; i < SNP_CPUID_COUNT_MAX; i++) {
2425 const struct snp_cpuid_fn *fn = &cpuid_table->fn[i];
2426
2427 pr_info("index=%3d fn=0x%08x subfn=0x%08x: eax=0x%08x ebx=0x%08x ecx=0x%08x edx=0x%08x xcr0_in=0x%016llx xss_in=0x%016llx reserved=0x%016llx\n",
2428 i, fn->eax_in, fn->ecx_in, fn->eax, fn->ebx, fn->ecx,
2429 fn->edx, fn->xcr0_in, fn->xss_in, fn->__reserved);
2430 }
2431 }
2432
2433 /*
2434 * It is useful from an auditing/testing perspective to provide an easy way
2435 * for the guest owner to know that the CPUID table has been initialized as
2436 * expected, but that initialization happens too early in boot to print any
2437 * sort of indicator, and there's not really any other good place to do it,
2438 * so do it here.
2439 *
2440 * If running as an SNP guest, report the current VM privilege level (VMPL).
2441 */
report_snp_info(void)2442 static int __init report_snp_info(void)
2443 {
2444 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
2445
2446 if (cpuid_table->count) {
2447 pr_info("Using SNP CPUID table, %d entries present.\n",
2448 cpuid_table->count);
2449
2450 if (sev_cfg.debug)
2451 dump_cpuid_table();
2452 }
2453
2454 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
2455 pr_info("SNP running at VMPL%u.\n", snp_vmpl);
2456
2457 return 0;
2458 }
2459 arch_initcall(report_snp_info);
2460
update_attest_input(struct svsm_call * call,struct svsm_attest_call * input)2461 static void update_attest_input(struct svsm_call *call, struct svsm_attest_call *input)
2462 {
2463 /* If (new) lengths have been returned, propagate them up */
2464 if (call->rcx_out != call->rcx)
2465 input->manifest_buf.len = call->rcx_out;
2466
2467 if (call->rdx_out != call->rdx)
2468 input->certificates_buf.len = call->rdx_out;
2469
2470 if (call->r8_out != call->r8)
2471 input->report_buf.len = call->r8_out;
2472 }
2473
snp_issue_svsm_attest_req(u64 call_id,struct svsm_call * call,struct svsm_attest_call * input)2474 int snp_issue_svsm_attest_req(u64 call_id, struct svsm_call *call,
2475 struct svsm_attest_call *input)
2476 {
2477 struct svsm_attest_call *ac;
2478 unsigned long flags;
2479 u64 attest_call_pa;
2480 int ret;
2481
2482 if (!snp_vmpl)
2483 return -EINVAL;
2484
2485 local_irq_save(flags);
2486
2487 call->caa = svsm_get_caa();
2488
2489 ac = (struct svsm_attest_call *)call->caa->svsm_buffer;
2490 attest_call_pa = svsm_get_caa_pa() + offsetof(struct svsm_ca, svsm_buffer);
2491
2492 *ac = *input;
2493
2494 /*
2495 * Set input registers for the request and set RDX and R8 to known
2496 * values in order to detect length values being returned in them.
2497 */
2498 call->rax = call_id;
2499 call->rcx = attest_call_pa;
2500 call->rdx = -1;
2501 call->r8 = -1;
2502 ret = svsm_perform_call_protocol(call);
2503 update_attest_input(call, input);
2504
2505 local_irq_restore(flags);
2506
2507 return ret;
2508 }
2509 EXPORT_SYMBOL_GPL(snp_issue_svsm_attest_req);
2510
snp_issue_guest_request(struct snp_guest_req * req,struct snp_req_data * input,struct snp_guest_request_ioctl * rio)2511 int snp_issue_guest_request(struct snp_guest_req *req, struct snp_req_data *input,
2512 struct snp_guest_request_ioctl *rio)
2513 {
2514 struct ghcb_state state;
2515 struct es_em_ctxt ctxt;
2516 unsigned long flags;
2517 struct ghcb *ghcb;
2518 int ret;
2519
2520 rio->exitinfo2 = SEV_RET_NO_FW_CALL;
2521
2522 /*
2523 * __sev_get_ghcb() needs to run with IRQs disabled because it is using
2524 * a per-CPU GHCB.
2525 */
2526 local_irq_save(flags);
2527
2528 ghcb = __sev_get_ghcb(&state);
2529 if (!ghcb) {
2530 ret = -EIO;
2531 goto e_restore_irq;
2532 }
2533
2534 vc_ghcb_invalidate(ghcb);
2535
2536 if (req->exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST) {
2537 ghcb_set_rax(ghcb, input->data_gpa);
2538 ghcb_set_rbx(ghcb, input->data_npages);
2539 }
2540
2541 ret = sev_es_ghcb_hv_call(ghcb, &ctxt, req->exit_code, input->req_gpa, input->resp_gpa);
2542 if (ret)
2543 goto e_put;
2544
2545 rio->exitinfo2 = ghcb->save.sw_exit_info_2;
2546 switch (rio->exitinfo2) {
2547 case 0:
2548 break;
2549
2550 case SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_BUSY):
2551 ret = -EAGAIN;
2552 break;
2553
2554 case SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN):
2555 /* Number of expected pages are returned in RBX */
2556 if (req->exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST) {
2557 input->data_npages = ghcb_get_rbx(ghcb);
2558 ret = -ENOSPC;
2559 break;
2560 }
2561 fallthrough;
2562 default:
2563 ret = -EIO;
2564 break;
2565 }
2566
2567 e_put:
2568 __sev_put_ghcb(&state);
2569 e_restore_irq:
2570 local_irq_restore(flags);
2571
2572 return ret;
2573 }
2574 EXPORT_SYMBOL_GPL(snp_issue_guest_request);
2575
2576 static struct platform_device sev_guest_device = {
2577 .name = "sev-guest",
2578 .id = -1,
2579 };
2580
snp_init_platform_device(void)2581 static int __init snp_init_platform_device(void)
2582 {
2583 struct sev_guest_platform_data data;
2584
2585 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
2586 return -ENODEV;
2587
2588 data.secrets_gpa = secrets_pa;
2589 if (platform_device_add_data(&sev_guest_device, &data, sizeof(data)))
2590 return -ENODEV;
2591
2592 if (platform_device_register(&sev_guest_device))
2593 return -ENODEV;
2594
2595 pr_info("SNP guest platform device initialized.\n");
2596 return 0;
2597 }
2598 device_initcall(snp_init_platform_device);
2599
sev_show_status(void)2600 void sev_show_status(void)
2601 {
2602 int i;
2603
2604 pr_info("Status: ");
2605 for (i = 0; i < MSR_AMD64_SNP_RESV_BIT; i++) {
2606 if (sev_status & BIT_ULL(i)) {
2607 if (!sev_status_feat_names[i])
2608 continue;
2609
2610 pr_cont("%s ", sev_status_feat_names[i]);
2611 }
2612 }
2613 pr_cont("\n");
2614 }
2615
snp_update_svsm_ca(void)2616 void __init snp_update_svsm_ca(void)
2617 {
2618 if (!snp_vmpl)
2619 return;
2620
2621 /* Update the CAA to a proper kernel address */
2622 boot_svsm_caa = &boot_svsm_ca_page;
2623 }
2624
2625 #ifdef CONFIG_SYSFS
vmpl_show(struct kobject * kobj,struct kobj_attribute * attr,char * buf)2626 static ssize_t vmpl_show(struct kobject *kobj,
2627 struct kobj_attribute *attr, char *buf)
2628 {
2629 return sysfs_emit(buf, "%d\n", snp_vmpl);
2630 }
2631
2632 static struct kobj_attribute vmpl_attr = __ATTR_RO(vmpl);
2633
2634 static struct attribute *vmpl_attrs[] = {
2635 &vmpl_attr.attr,
2636 NULL
2637 };
2638
2639 static struct attribute_group sev_attr_group = {
2640 .attrs = vmpl_attrs,
2641 };
2642
sev_sysfs_init(void)2643 static int __init sev_sysfs_init(void)
2644 {
2645 struct kobject *sev_kobj;
2646 struct device *dev_root;
2647 int ret;
2648
2649 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
2650 return -ENODEV;
2651
2652 dev_root = bus_get_dev_root(&cpu_subsys);
2653 if (!dev_root)
2654 return -ENODEV;
2655
2656 sev_kobj = kobject_create_and_add("sev", &dev_root->kobj);
2657 put_device(dev_root);
2658
2659 if (!sev_kobj)
2660 return -ENOMEM;
2661
2662 ret = sysfs_create_group(sev_kobj, &sev_attr_group);
2663 if (ret)
2664 kobject_put(sev_kobj);
2665
2666 return ret;
2667 }
2668 arch_initcall(sev_sysfs_init);
2669 #endif // CONFIG_SYSFS
2670