1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * AMD Memory Encryption Support 4 * 5 * Copyright (C) 2019 SUSE 6 * 7 * Author: Joerg Roedel <jroedel@suse.de> 8 */ 9 10 #define pr_fmt(fmt) "SEV: " fmt 11 12 #include <linux/sched/debug.h> /* For show_regs() */ 13 #include <linux/percpu-defs.h> 14 #include <linux/cc_platform.h> 15 #include <linux/printk.h> 16 #include <linux/mm_types.h> 17 #include <linux/set_memory.h> 18 #include <linux/memblock.h> 19 #include <linux/kernel.h> 20 #include <linux/mm.h> 21 #include <linux/cpumask.h> 22 #include <linux/efi.h> 23 #include <linux/platform_device.h> 24 #include <linux/io.h> 25 #include <linux/psp-sev.h> 26 #include <linux/dmi.h> 27 #include <uapi/linux/sev-guest.h> 28 29 #include <asm/init.h> 30 #include <asm/cpu_entry_area.h> 31 #include <asm/stacktrace.h> 32 #include <asm/sev.h> 33 #include <asm/insn-eval.h> 34 #include <asm/fpu/xcr.h> 35 #include <asm/processor.h> 36 #include <asm/realmode.h> 37 #include <asm/setup.h> 38 #include <asm/traps.h> 39 #include <asm/svm.h> 40 #include <asm/smp.h> 41 #include <asm/cpu.h> 42 #include <asm/apic.h> 43 #include <asm/cpuid.h> 44 #include <asm/cmdline.h> 45 46 #define DR7_RESET_VALUE 0x400 47 48 /* AP INIT values as documented in the APM2 section "Processor Initialization State" */ 49 #define AP_INIT_CS_LIMIT 0xffff 50 #define AP_INIT_DS_LIMIT 0xffff 51 #define AP_INIT_LDTR_LIMIT 0xffff 52 #define AP_INIT_GDTR_LIMIT 0xffff 53 #define AP_INIT_IDTR_LIMIT 0xffff 54 #define AP_INIT_TR_LIMIT 0xffff 55 #define AP_INIT_RFLAGS_DEFAULT 0x2 56 #define AP_INIT_DR6_DEFAULT 0xffff0ff0 57 #define AP_INIT_GPAT_DEFAULT 0x0007040600070406ULL 58 #define AP_INIT_XCR0_DEFAULT 0x1 59 #define AP_INIT_X87_FTW_DEFAULT 0x5555 60 #define AP_INIT_X87_FCW_DEFAULT 0x0040 61 #define AP_INIT_CR0_DEFAULT 0x60000010 62 #define AP_INIT_MXCSR_DEFAULT 0x1f80 63 64 static const char * const sev_status_feat_names[] = { 65 [MSR_AMD64_SEV_ENABLED_BIT] = "SEV", 66 [MSR_AMD64_SEV_ES_ENABLED_BIT] = "SEV-ES", 67 [MSR_AMD64_SEV_SNP_ENABLED_BIT] = "SEV-SNP", 68 [MSR_AMD64_SNP_VTOM_BIT] = "vTom", 69 [MSR_AMD64_SNP_REFLECT_VC_BIT] = "ReflectVC", 70 [MSR_AMD64_SNP_RESTRICTED_INJ_BIT] = "RI", 71 [MSR_AMD64_SNP_ALT_INJ_BIT] = "AI", 72 [MSR_AMD64_SNP_DEBUG_SWAP_BIT] = "DebugSwap", 73 [MSR_AMD64_SNP_PREVENT_HOST_IBS_BIT] = "NoHostIBS", 74 [MSR_AMD64_SNP_BTB_ISOLATION_BIT] = "BTBIsol", 75 [MSR_AMD64_SNP_VMPL_SSS_BIT] = "VmplSSS", 76 [MSR_AMD64_SNP_SECURE_TSC_BIT] = "SecureTSC", 77 [MSR_AMD64_SNP_VMGEXIT_PARAM_BIT] = "VMGExitParam", 78 [MSR_AMD64_SNP_IBS_VIRT_BIT] = "IBSVirt", 79 [MSR_AMD64_SNP_VMSA_REG_PROT_BIT] = "VMSARegProt", 80 [MSR_AMD64_SNP_SMT_PROT_BIT] = "SMTProt", 81 }; 82 83 /* For early boot hypervisor communication in SEV-ES enabled guests */ 84 static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE); 85 86 /* 87 * Needs to be in the .data section because we need it NULL before bss is 88 * cleared 89 */ 90 static struct ghcb *boot_ghcb __section(".data"); 91 92 /* Bitmap of SEV features supported by the hypervisor */ 93 static u64 sev_hv_features __ro_after_init; 94 95 /* Secrets page physical address from the CC blob */ 96 static u64 secrets_pa __ro_after_init; 97 98 /* #VC handler runtime per-CPU data */ 99 struct sev_es_runtime_data { 100 struct ghcb ghcb_page; 101 102 /* 103 * Reserve one page per CPU as backup storage for the unencrypted GHCB. 104 * It is needed when an NMI happens while the #VC handler uses the real 105 * GHCB, and the NMI handler itself is causing another #VC exception. In 106 * that case the GHCB content of the first handler needs to be backed up 107 * and restored. 108 */ 109 struct ghcb backup_ghcb; 110 111 /* 112 * Mark the per-cpu GHCBs as in-use to detect nested #VC exceptions. 113 * There is no need for it to be atomic, because nothing is written to 114 * the GHCB between the read and the write of ghcb_active. So it is safe 115 * to use it when a nested #VC exception happens before the write. 116 * 117 * This is necessary for example in the #VC->NMI->#VC case when the NMI 118 * happens while the first #VC handler uses the GHCB. When the NMI code 119 * raises a second #VC handler it might overwrite the contents of the 120 * GHCB written by the first handler. To avoid this the content of the 121 * GHCB is saved and restored when the GHCB is detected to be in use 122 * already. 123 */ 124 bool ghcb_active; 125 bool backup_ghcb_active; 126 127 /* 128 * Cached DR7 value - write it on DR7 writes and return it on reads. 129 * That value will never make it to the real hardware DR7 as debugging 130 * is currently unsupported in SEV-ES guests. 131 */ 132 unsigned long dr7; 133 }; 134 135 struct ghcb_state { 136 struct ghcb *ghcb; 137 }; 138 139 /* For early boot SVSM communication */ 140 static struct svsm_ca boot_svsm_ca_page __aligned(PAGE_SIZE); 141 142 static DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data); 143 static DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa); 144 static DEFINE_PER_CPU(struct svsm_ca *, svsm_caa); 145 static DEFINE_PER_CPU(u64, svsm_caa_pa); 146 147 static __always_inline bool on_vc_stack(struct pt_regs *regs) 148 { 149 unsigned long sp = regs->sp; 150 151 /* User-mode RSP is not trusted */ 152 if (user_mode(regs)) 153 return false; 154 155 /* SYSCALL gap still has user-mode RSP */ 156 if (ip_within_syscall_gap(regs)) 157 return false; 158 159 return ((sp >= __this_cpu_ist_bottom_va(VC)) && (sp < __this_cpu_ist_top_va(VC))); 160 } 161 162 /* 163 * This function handles the case when an NMI is raised in the #VC 164 * exception handler entry code, before the #VC handler has switched off 165 * its IST stack. In this case, the IST entry for #VC must be adjusted, 166 * so that any nested #VC exception will not overwrite the stack 167 * contents of the interrupted #VC handler. 168 * 169 * The IST entry is adjusted unconditionally so that it can be also be 170 * unconditionally adjusted back in __sev_es_ist_exit(). Otherwise a 171 * nested sev_es_ist_exit() call may adjust back the IST entry too 172 * early. 173 * 174 * The __sev_es_ist_enter() and __sev_es_ist_exit() functions always run 175 * on the NMI IST stack, as they are only called from NMI handling code 176 * right now. 177 */ 178 void noinstr __sev_es_ist_enter(struct pt_regs *regs) 179 { 180 unsigned long old_ist, new_ist; 181 182 /* Read old IST entry */ 183 new_ist = old_ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]); 184 185 /* 186 * If NMI happened while on the #VC IST stack, set the new IST 187 * value below regs->sp, so that the interrupted stack frame is 188 * not overwritten by subsequent #VC exceptions. 189 */ 190 if (on_vc_stack(regs)) 191 new_ist = regs->sp; 192 193 /* 194 * Reserve additional 8 bytes and store old IST value so this 195 * adjustment can be unrolled in __sev_es_ist_exit(). 196 */ 197 new_ist -= sizeof(old_ist); 198 *(unsigned long *)new_ist = old_ist; 199 200 /* Set new IST entry */ 201 this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], new_ist); 202 } 203 204 void noinstr __sev_es_ist_exit(void) 205 { 206 unsigned long ist; 207 208 /* Read IST entry */ 209 ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]); 210 211 if (WARN_ON(ist == __this_cpu_ist_top_va(VC))) 212 return; 213 214 /* Read back old IST entry and write it to the TSS */ 215 this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], *(unsigned long *)ist); 216 } 217 218 /* 219 * Nothing shall interrupt this code path while holding the per-CPU 220 * GHCB. The backup GHCB is only for NMIs interrupting this path. 221 * 222 * Callers must disable local interrupts around it. 223 */ 224 static noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state) 225 { 226 struct sev_es_runtime_data *data; 227 struct ghcb *ghcb; 228 229 WARN_ON(!irqs_disabled()); 230 231 data = this_cpu_read(runtime_data); 232 ghcb = &data->ghcb_page; 233 234 if (unlikely(data->ghcb_active)) { 235 /* GHCB is already in use - save its contents */ 236 237 if (unlikely(data->backup_ghcb_active)) { 238 /* 239 * Backup-GHCB is also already in use. There is no way 240 * to continue here so just kill the machine. To make 241 * panic() work, mark GHCBs inactive so that messages 242 * can be printed out. 243 */ 244 data->ghcb_active = false; 245 data->backup_ghcb_active = false; 246 247 instrumentation_begin(); 248 panic("Unable to handle #VC exception! GHCB and Backup GHCB are already in use"); 249 instrumentation_end(); 250 } 251 252 /* Mark backup_ghcb active before writing to it */ 253 data->backup_ghcb_active = true; 254 255 state->ghcb = &data->backup_ghcb; 256 257 /* Backup GHCB content */ 258 *state->ghcb = *ghcb; 259 } else { 260 state->ghcb = NULL; 261 data->ghcb_active = true; 262 } 263 264 return ghcb; 265 } 266 267 static inline u64 sev_es_rd_ghcb_msr(void) 268 { 269 return __rdmsr(MSR_AMD64_SEV_ES_GHCB); 270 } 271 272 static __always_inline void sev_es_wr_ghcb_msr(u64 val) 273 { 274 u32 low, high; 275 276 low = (u32)(val); 277 high = (u32)(val >> 32); 278 279 native_wrmsr(MSR_AMD64_SEV_ES_GHCB, low, high); 280 } 281 282 static int vc_fetch_insn_kernel(struct es_em_ctxt *ctxt, 283 unsigned char *buffer) 284 { 285 return copy_from_kernel_nofault(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE); 286 } 287 288 static enum es_result __vc_decode_user_insn(struct es_em_ctxt *ctxt) 289 { 290 char buffer[MAX_INSN_SIZE]; 291 int insn_bytes; 292 293 insn_bytes = insn_fetch_from_user_inatomic(ctxt->regs, buffer); 294 if (insn_bytes == 0) { 295 /* Nothing could be copied */ 296 ctxt->fi.vector = X86_TRAP_PF; 297 ctxt->fi.error_code = X86_PF_INSTR | X86_PF_USER; 298 ctxt->fi.cr2 = ctxt->regs->ip; 299 return ES_EXCEPTION; 300 } else if (insn_bytes == -EINVAL) { 301 /* Effective RIP could not be calculated */ 302 ctxt->fi.vector = X86_TRAP_GP; 303 ctxt->fi.error_code = 0; 304 ctxt->fi.cr2 = 0; 305 return ES_EXCEPTION; 306 } 307 308 if (!insn_decode_from_regs(&ctxt->insn, ctxt->regs, buffer, insn_bytes)) 309 return ES_DECODE_FAILED; 310 311 if (ctxt->insn.immediate.got) 312 return ES_OK; 313 else 314 return ES_DECODE_FAILED; 315 } 316 317 static enum es_result __vc_decode_kern_insn(struct es_em_ctxt *ctxt) 318 { 319 char buffer[MAX_INSN_SIZE]; 320 int res, ret; 321 322 res = vc_fetch_insn_kernel(ctxt, buffer); 323 if (res) { 324 ctxt->fi.vector = X86_TRAP_PF; 325 ctxt->fi.error_code = X86_PF_INSTR; 326 ctxt->fi.cr2 = ctxt->regs->ip; 327 return ES_EXCEPTION; 328 } 329 330 ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64); 331 if (ret < 0) 332 return ES_DECODE_FAILED; 333 else 334 return ES_OK; 335 } 336 337 static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt) 338 { 339 if (user_mode(ctxt->regs)) 340 return __vc_decode_user_insn(ctxt); 341 else 342 return __vc_decode_kern_insn(ctxt); 343 } 344 345 static enum es_result vc_write_mem(struct es_em_ctxt *ctxt, 346 char *dst, char *buf, size_t size) 347 { 348 unsigned long error_code = X86_PF_PROT | X86_PF_WRITE; 349 350 /* 351 * This function uses __put_user() independent of whether kernel or user 352 * memory is accessed. This works fine because __put_user() does no 353 * sanity checks of the pointer being accessed. All that it does is 354 * to report when the access failed. 355 * 356 * Also, this function runs in atomic context, so __put_user() is not 357 * allowed to sleep. The page-fault handler detects that it is running 358 * in atomic context and will not try to take mmap_sem and handle the 359 * fault, so additional pagefault_enable()/disable() calls are not 360 * needed. 361 * 362 * The access can't be done via copy_to_user() here because 363 * vc_write_mem() must not use string instructions to access unsafe 364 * memory. The reason is that MOVS is emulated by the #VC handler by 365 * splitting the move up into a read and a write and taking a nested #VC 366 * exception on whatever of them is the MMIO access. Using string 367 * instructions here would cause infinite nesting. 368 */ 369 switch (size) { 370 case 1: { 371 u8 d1; 372 u8 __user *target = (u8 __user *)dst; 373 374 memcpy(&d1, buf, 1); 375 if (__put_user(d1, target)) 376 goto fault; 377 break; 378 } 379 case 2: { 380 u16 d2; 381 u16 __user *target = (u16 __user *)dst; 382 383 memcpy(&d2, buf, 2); 384 if (__put_user(d2, target)) 385 goto fault; 386 break; 387 } 388 case 4: { 389 u32 d4; 390 u32 __user *target = (u32 __user *)dst; 391 392 memcpy(&d4, buf, 4); 393 if (__put_user(d4, target)) 394 goto fault; 395 break; 396 } 397 case 8: { 398 u64 d8; 399 u64 __user *target = (u64 __user *)dst; 400 401 memcpy(&d8, buf, 8); 402 if (__put_user(d8, target)) 403 goto fault; 404 break; 405 } 406 default: 407 WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size); 408 return ES_UNSUPPORTED; 409 } 410 411 return ES_OK; 412 413 fault: 414 if (user_mode(ctxt->regs)) 415 error_code |= X86_PF_USER; 416 417 ctxt->fi.vector = X86_TRAP_PF; 418 ctxt->fi.error_code = error_code; 419 ctxt->fi.cr2 = (unsigned long)dst; 420 421 return ES_EXCEPTION; 422 } 423 424 static enum es_result vc_read_mem(struct es_em_ctxt *ctxt, 425 char *src, char *buf, size_t size) 426 { 427 unsigned long error_code = X86_PF_PROT; 428 429 /* 430 * This function uses __get_user() independent of whether kernel or user 431 * memory is accessed. This works fine because __get_user() does no 432 * sanity checks of the pointer being accessed. All that it does is 433 * to report when the access failed. 434 * 435 * Also, this function runs in atomic context, so __get_user() is not 436 * allowed to sleep. The page-fault handler detects that it is running 437 * in atomic context and will not try to take mmap_sem and handle the 438 * fault, so additional pagefault_enable()/disable() calls are not 439 * needed. 440 * 441 * The access can't be done via copy_from_user() here because 442 * vc_read_mem() must not use string instructions to access unsafe 443 * memory. The reason is that MOVS is emulated by the #VC handler by 444 * splitting the move up into a read and a write and taking a nested #VC 445 * exception on whatever of them is the MMIO access. Using string 446 * instructions here would cause infinite nesting. 447 */ 448 switch (size) { 449 case 1: { 450 u8 d1; 451 u8 __user *s = (u8 __user *)src; 452 453 if (__get_user(d1, s)) 454 goto fault; 455 memcpy(buf, &d1, 1); 456 break; 457 } 458 case 2: { 459 u16 d2; 460 u16 __user *s = (u16 __user *)src; 461 462 if (__get_user(d2, s)) 463 goto fault; 464 memcpy(buf, &d2, 2); 465 break; 466 } 467 case 4: { 468 u32 d4; 469 u32 __user *s = (u32 __user *)src; 470 471 if (__get_user(d4, s)) 472 goto fault; 473 memcpy(buf, &d4, 4); 474 break; 475 } 476 case 8: { 477 u64 d8; 478 u64 __user *s = (u64 __user *)src; 479 if (__get_user(d8, s)) 480 goto fault; 481 memcpy(buf, &d8, 8); 482 break; 483 } 484 default: 485 WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size); 486 return ES_UNSUPPORTED; 487 } 488 489 return ES_OK; 490 491 fault: 492 if (user_mode(ctxt->regs)) 493 error_code |= X86_PF_USER; 494 495 ctxt->fi.vector = X86_TRAP_PF; 496 ctxt->fi.error_code = error_code; 497 ctxt->fi.cr2 = (unsigned long)src; 498 499 return ES_EXCEPTION; 500 } 501 502 static enum es_result vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt, 503 unsigned long vaddr, phys_addr_t *paddr) 504 { 505 unsigned long va = (unsigned long)vaddr; 506 unsigned int level; 507 phys_addr_t pa; 508 pgd_t *pgd; 509 pte_t *pte; 510 511 pgd = __va(read_cr3_pa()); 512 pgd = &pgd[pgd_index(va)]; 513 pte = lookup_address_in_pgd(pgd, va, &level); 514 if (!pte) { 515 ctxt->fi.vector = X86_TRAP_PF; 516 ctxt->fi.cr2 = vaddr; 517 ctxt->fi.error_code = 0; 518 519 if (user_mode(ctxt->regs)) 520 ctxt->fi.error_code |= X86_PF_USER; 521 522 return ES_EXCEPTION; 523 } 524 525 if (WARN_ON_ONCE(pte_val(*pte) & _PAGE_ENC)) 526 /* Emulated MMIO to/from encrypted memory not supported */ 527 return ES_UNSUPPORTED; 528 529 pa = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT; 530 pa |= va & ~page_level_mask(level); 531 532 *paddr = pa; 533 534 return ES_OK; 535 } 536 537 static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size) 538 { 539 BUG_ON(size > 4); 540 541 if (user_mode(ctxt->regs)) { 542 struct thread_struct *t = ¤t->thread; 543 struct io_bitmap *iobm = t->io_bitmap; 544 size_t idx; 545 546 if (!iobm) 547 goto fault; 548 549 for (idx = port; idx < port + size; ++idx) { 550 if (test_bit(idx, iobm->bitmap)) 551 goto fault; 552 } 553 } 554 555 return ES_OK; 556 557 fault: 558 ctxt->fi.vector = X86_TRAP_GP; 559 ctxt->fi.error_code = 0; 560 561 return ES_EXCEPTION; 562 } 563 564 static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt) 565 { 566 long error_code = ctxt->fi.error_code; 567 int trapnr = ctxt->fi.vector; 568 569 ctxt->regs->orig_ax = ctxt->fi.error_code; 570 571 switch (trapnr) { 572 case X86_TRAP_GP: 573 exc_general_protection(ctxt->regs, error_code); 574 break; 575 case X86_TRAP_UD: 576 exc_invalid_op(ctxt->regs); 577 break; 578 case X86_TRAP_PF: 579 write_cr2(ctxt->fi.cr2); 580 exc_page_fault(ctxt->regs, error_code); 581 break; 582 case X86_TRAP_AC: 583 exc_alignment_check(ctxt->regs, error_code); 584 break; 585 default: 586 pr_emerg("Unsupported exception in #VC instruction emulation - can't continue\n"); 587 BUG(); 588 } 589 } 590 591 /* Include code shared with pre-decompression boot stage */ 592 #include "shared.c" 593 594 static inline struct svsm_ca *svsm_get_caa(void) 595 { 596 /* 597 * Use rIP-relative references when called early in the boot. If 598 * ->use_cas is set, then it is late in the boot and no need 599 * to worry about rIP-relative references. 600 */ 601 if (RIP_REL_REF(sev_cfg).use_cas) 602 return this_cpu_read(svsm_caa); 603 else 604 return RIP_REL_REF(boot_svsm_caa); 605 } 606 607 static u64 svsm_get_caa_pa(void) 608 { 609 /* 610 * Use rIP-relative references when called early in the boot. If 611 * ->use_cas is set, then it is late in the boot and no need 612 * to worry about rIP-relative references. 613 */ 614 if (RIP_REL_REF(sev_cfg).use_cas) 615 return this_cpu_read(svsm_caa_pa); 616 else 617 return RIP_REL_REF(boot_svsm_caa_pa); 618 } 619 620 static noinstr void __sev_put_ghcb(struct ghcb_state *state) 621 { 622 struct sev_es_runtime_data *data; 623 struct ghcb *ghcb; 624 625 WARN_ON(!irqs_disabled()); 626 627 data = this_cpu_read(runtime_data); 628 ghcb = &data->ghcb_page; 629 630 if (state->ghcb) { 631 /* Restore GHCB from Backup */ 632 *ghcb = *state->ghcb; 633 data->backup_ghcb_active = false; 634 state->ghcb = NULL; 635 } else { 636 /* 637 * Invalidate the GHCB so a VMGEXIT instruction issued 638 * from userspace won't appear to be valid. 639 */ 640 vc_ghcb_invalidate(ghcb); 641 data->ghcb_active = false; 642 } 643 } 644 645 static int svsm_perform_call_protocol(struct svsm_call *call) 646 { 647 struct ghcb_state state; 648 unsigned long flags; 649 struct ghcb *ghcb; 650 int ret; 651 652 /* 653 * This can be called very early in the boot, use native functions in 654 * order to avoid paravirt issues. 655 */ 656 flags = native_local_irq_save(); 657 658 /* 659 * Use rip-relative references when called early in the boot. If 660 * ghcbs_initialized is set, then it is late in the boot and no need 661 * to worry about rip-relative references in called functions. 662 */ 663 if (RIP_REL_REF(sev_cfg).ghcbs_initialized) 664 ghcb = __sev_get_ghcb(&state); 665 else if (RIP_REL_REF(boot_ghcb)) 666 ghcb = RIP_REL_REF(boot_ghcb); 667 else 668 ghcb = NULL; 669 670 do { 671 ret = ghcb ? svsm_perform_ghcb_protocol(ghcb, call) 672 : svsm_perform_msr_protocol(call); 673 } while (ret == -EAGAIN); 674 675 if (RIP_REL_REF(sev_cfg).ghcbs_initialized) 676 __sev_put_ghcb(&state); 677 678 native_local_irq_restore(flags); 679 680 return ret; 681 } 682 683 void noinstr __sev_es_nmi_complete(void) 684 { 685 struct ghcb_state state; 686 struct ghcb *ghcb; 687 688 ghcb = __sev_get_ghcb(&state); 689 690 vc_ghcb_invalidate(ghcb); 691 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_NMI_COMPLETE); 692 ghcb_set_sw_exit_info_1(ghcb, 0); 693 ghcb_set_sw_exit_info_2(ghcb, 0); 694 695 sev_es_wr_ghcb_msr(__pa_nodebug(ghcb)); 696 VMGEXIT(); 697 698 __sev_put_ghcb(&state); 699 } 700 701 static u64 __init get_snp_jump_table_addr(void) 702 { 703 struct snp_secrets_page *secrets; 704 void __iomem *mem; 705 u64 addr; 706 707 mem = ioremap_encrypted(secrets_pa, PAGE_SIZE); 708 if (!mem) { 709 pr_err("Unable to locate AP jump table address: failed to map the SNP secrets page.\n"); 710 return 0; 711 } 712 713 secrets = (__force struct snp_secrets_page *)mem; 714 715 addr = secrets->os_area.ap_jump_table_pa; 716 iounmap(mem); 717 718 return addr; 719 } 720 721 static u64 __init get_jump_table_addr(void) 722 { 723 struct ghcb_state state; 724 unsigned long flags; 725 struct ghcb *ghcb; 726 u64 ret = 0; 727 728 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 729 return get_snp_jump_table_addr(); 730 731 local_irq_save(flags); 732 733 ghcb = __sev_get_ghcb(&state); 734 735 vc_ghcb_invalidate(ghcb); 736 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_JUMP_TABLE); 737 ghcb_set_sw_exit_info_1(ghcb, SVM_VMGEXIT_GET_AP_JUMP_TABLE); 738 ghcb_set_sw_exit_info_2(ghcb, 0); 739 740 sev_es_wr_ghcb_msr(__pa(ghcb)); 741 VMGEXIT(); 742 743 if (ghcb_sw_exit_info_1_is_valid(ghcb) && 744 ghcb_sw_exit_info_2_is_valid(ghcb)) 745 ret = ghcb->save.sw_exit_info_2; 746 747 __sev_put_ghcb(&state); 748 749 local_irq_restore(flags); 750 751 return ret; 752 } 753 754 static void __head 755 early_set_pages_state(unsigned long vaddr, unsigned long paddr, 756 unsigned long npages, enum psc_op op) 757 { 758 unsigned long paddr_end; 759 u64 val; 760 761 vaddr = vaddr & PAGE_MASK; 762 763 paddr = paddr & PAGE_MASK; 764 paddr_end = paddr + (npages << PAGE_SHIFT); 765 766 while (paddr < paddr_end) { 767 /* Page validation must be rescinded before changing to shared */ 768 if (op == SNP_PAGE_STATE_SHARED) 769 pvalidate_4k_page(vaddr, paddr, false); 770 771 /* 772 * Use the MSR protocol because this function can be called before 773 * the GHCB is established. 774 */ 775 sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op)); 776 VMGEXIT(); 777 778 val = sev_es_rd_ghcb_msr(); 779 780 if (WARN(GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP, 781 "Wrong PSC response code: 0x%x\n", 782 (unsigned int)GHCB_RESP_CODE(val))) 783 goto e_term; 784 785 if (WARN(GHCB_MSR_PSC_RESP_VAL(val), 786 "Failed to change page state to '%s' paddr 0x%lx error 0x%llx\n", 787 op == SNP_PAGE_STATE_PRIVATE ? "private" : "shared", 788 paddr, GHCB_MSR_PSC_RESP_VAL(val))) 789 goto e_term; 790 791 /* Page validation must be performed after changing to private */ 792 if (op == SNP_PAGE_STATE_PRIVATE) 793 pvalidate_4k_page(vaddr, paddr, true); 794 795 vaddr += PAGE_SIZE; 796 paddr += PAGE_SIZE; 797 } 798 799 return; 800 801 e_term: 802 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC); 803 } 804 805 void __head early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr, 806 unsigned long npages) 807 { 808 /* 809 * This can be invoked in early boot while running identity mapped, so 810 * use an open coded check for SNP instead of using cc_platform_has(). 811 * This eliminates worries about jump tables or checking boot_cpu_data 812 * in the cc_platform_has() function. 813 */ 814 if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED)) 815 return; 816 817 /* 818 * Ask the hypervisor to mark the memory pages as private in the RMP 819 * table. 820 */ 821 early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_PRIVATE); 822 } 823 824 void __init early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr, 825 unsigned long npages) 826 { 827 /* 828 * This can be invoked in early boot while running identity mapped, so 829 * use an open coded check for SNP instead of using cc_platform_has(). 830 * This eliminates worries about jump tables or checking boot_cpu_data 831 * in the cc_platform_has() function. 832 */ 833 if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED)) 834 return; 835 836 /* Ask hypervisor to mark the memory pages shared in the RMP table. */ 837 early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_SHARED); 838 } 839 840 static unsigned long __set_pages_state(struct snp_psc_desc *data, unsigned long vaddr, 841 unsigned long vaddr_end, int op) 842 { 843 struct ghcb_state state; 844 bool use_large_entry; 845 struct psc_hdr *hdr; 846 struct psc_entry *e; 847 unsigned long flags; 848 unsigned long pfn; 849 struct ghcb *ghcb; 850 int i; 851 852 hdr = &data->hdr; 853 e = data->entries; 854 855 memset(data, 0, sizeof(*data)); 856 i = 0; 857 858 while (vaddr < vaddr_end && i < ARRAY_SIZE(data->entries)) { 859 hdr->end_entry = i; 860 861 if (is_vmalloc_addr((void *)vaddr)) { 862 pfn = vmalloc_to_pfn((void *)vaddr); 863 use_large_entry = false; 864 } else { 865 pfn = __pa(vaddr) >> PAGE_SHIFT; 866 use_large_entry = true; 867 } 868 869 e->gfn = pfn; 870 e->operation = op; 871 872 if (use_large_entry && IS_ALIGNED(vaddr, PMD_SIZE) && 873 (vaddr_end - vaddr) >= PMD_SIZE) { 874 e->pagesize = RMP_PG_SIZE_2M; 875 vaddr += PMD_SIZE; 876 } else { 877 e->pagesize = RMP_PG_SIZE_4K; 878 vaddr += PAGE_SIZE; 879 } 880 881 e++; 882 i++; 883 } 884 885 /* Page validation must be rescinded before changing to shared */ 886 if (op == SNP_PAGE_STATE_SHARED) 887 pvalidate_pages(data); 888 889 local_irq_save(flags); 890 891 if (sev_cfg.ghcbs_initialized) 892 ghcb = __sev_get_ghcb(&state); 893 else 894 ghcb = boot_ghcb; 895 896 /* Invoke the hypervisor to perform the page state changes */ 897 if (!ghcb || vmgexit_psc(ghcb, data)) 898 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC); 899 900 if (sev_cfg.ghcbs_initialized) 901 __sev_put_ghcb(&state); 902 903 local_irq_restore(flags); 904 905 /* Page validation must be performed after changing to private */ 906 if (op == SNP_PAGE_STATE_PRIVATE) 907 pvalidate_pages(data); 908 909 return vaddr; 910 } 911 912 static void set_pages_state(unsigned long vaddr, unsigned long npages, int op) 913 { 914 struct snp_psc_desc desc; 915 unsigned long vaddr_end; 916 917 /* Use the MSR protocol when a GHCB is not available. */ 918 if (!boot_ghcb) 919 return early_set_pages_state(vaddr, __pa(vaddr), npages, op); 920 921 vaddr = vaddr & PAGE_MASK; 922 vaddr_end = vaddr + (npages << PAGE_SHIFT); 923 924 while (vaddr < vaddr_end) 925 vaddr = __set_pages_state(&desc, vaddr, vaddr_end, op); 926 } 927 928 void snp_set_memory_shared(unsigned long vaddr, unsigned long npages) 929 { 930 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 931 return; 932 933 set_pages_state(vaddr, npages, SNP_PAGE_STATE_SHARED); 934 } 935 936 void snp_set_memory_private(unsigned long vaddr, unsigned long npages) 937 { 938 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 939 return; 940 941 set_pages_state(vaddr, npages, SNP_PAGE_STATE_PRIVATE); 942 } 943 944 void snp_accept_memory(phys_addr_t start, phys_addr_t end) 945 { 946 unsigned long vaddr, npages; 947 948 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 949 return; 950 951 vaddr = (unsigned long)__va(start); 952 npages = (end - start) >> PAGE_SHIFT; 953 954 set_pages_state(vaddr, npages, SNP_PAGE_STATE_PRIVATE); 955 } 956 957 static void set_pte_enc(pte_t *kpte, int level, void *va) 958 { 959 struct pte_enc_desc d = { 960 .kpte = kpte, 961 .pte_level = level, 962 .va = va, 963 .encrypt = true 964 }; 965 966 prepare_pte_enc(&d); 967 set_pte_enc_mask(kpte, d.pfn, d.new_pgprot); 968 } 969 970 static void unshare_all_memory(void) 971 { 972 unsigned long addr, end, size, ghcb; 973 struct sev_es_runtime_data *data; 974 unsigned int npages, level; 975 bool skipped_addr; 976 pte_t *pte; 977 int cpu; 978 979 /* Unshare the direct mapping. */ 980 addr = PAGE_OFFSET; 981 end = PAGE_OFFSET + get_max_mapped(); 982 983 while (addr < end) { 984 pte = lookup_address(addr, &level); 985 size = page_level_size(level); 986 npages = size / PAGE_SIZE; 987 skipped_addr = false; 988 989 if (!pte || !pte_decrypted(*pte) || pte_none(*pte)) { 990 addr += size; 991 continue; 992 } 993 994 /* 995 * Ensure that all the per-CPU GHCBs are made private at the 996 * end of the unsharing loop so that the switch to the slower 997 * MSR protocol happens last. 998 */ 999 for_each_possible_cpu(cpu) { 1000 data = per_cpu(runtime_data, cpu); 1001 ghcb = (unsigned long)&data->ghcb_page; 1002 1003 if (addr <= ghcb && ghcb <= addr + size) { 1004 skipped_addr = true; 1005 break; 1006 } 1007 } 1008 1009 if (!skipped_addr) { 1010 set_pte_enc(pte, level, (void *)addr); 1011 snp_set_memory_private(addr, npages); 1012 } 1013 addr += size; 1014 } 1015 1016 /* Unshare all bss decrypted memory. */ 1017 addr = (unsigned long)__start_bss_decrypted; 1018 end = (unsigned long)__start_bss_decrypted_unused; 1019 npages = (end - addr) >> PAGE_SHIFT; 1020 1021 for (; addr < end; addr += PAGE_SIZE) { 1022 pte = lookup_address(addr, &level); 1023 if (!pte || !pte_decrypted(*pte) || pte_none(*pte)) 1024 continue; 1025 1026 set_pte_enc(pte, level, (void *)addr); 1027 } 1028 addr = (unsigned long)__start_bss_decrypted; 1029 snp_set_memory_private(addr, npages); 1030 1031 __flush_tlb_all(); 1032 } 1033 1034 /* Stop new private<->shared conversions */ 1035 void snp_kexec_begin(void) 1036 { 1037 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 1038 return; 1039 1040 if (!IS_ENABLED(CONFIG_KEXEC_CORE)) 1041 return; 1042 1043 /* 1044 * Crash kernel ends up here with interrupts disabled: can't wait for 1045 * conversions to finish. 1046 * 1047 * If race happened, just report and proceed. 1048 */ 1049 if (!set_memory_enc_stop_conversion()) 1050 pr_warn("Failed to stop shared<->private conversions\n"); 1051 } 1052 1053 void snp_kexec_finish(void) 1054 { 1055 struct sev_es_runtime_data *data; 1056 unsigned int level, cpu; 1057 unsigned long size; 1058 struct ghcb *ghcb; 1059 pte_t *pte; 1060 1061 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 1062 return; 1063 1064 if (!IS_ENABLED(CONFIG_KEXEC_CORE)) 1065 return; 1066 1067 unshare_all_memory(); 1068 1069 /* 1070 * Switch to using the MSR protocol to change per-CPU GHCBs to 1071 * private. All the per-CPU GHCBs have been switched back to private, 1072 * so can't do any more GHCB calls to the hypervisor beyond this point 1073 * until the kexec'ed kernel starts running. 1074 */ 1075 boot_ghcb = NULL; 1076 sev_cfg.ghcbs_initialized = false; 1077 1078 for_each_possible_cpu(cpu) { 1079 data = per_cpu(runtime_data, cpu); 1080 ghcb = &data->ghcb_page; 1081 pte = lookup_address((unsigned long)ghcb, &level); 1082 size = page_level_size(level); 1083 set_pte_enc(pte, level, (void *)ghcb); 1084 snp_set_memory_private((unsigned long)ghcb, (size / PAGE_SIZE)); 1085 } 1086 } 1087 1088 static int snp_set_vmsa(void *va, void *caa, int apic_id, bool make_vmsa) 1089 { 1090 int ret; 1091 1092 if (snp_vmpl) { 1093 struct svsm_call call = {}; 1094 unsigned long flags; 1095 1096 local_irq_save(flags); 1097 1098 call.caa = this_cpu_read(svsm_caa); 1099 call.rcx = __pa(va); 1100 1101 if (make_vmsa) { 1102 /* Protocol 0, Call ID 2 */ 1103 call.rax = SVSM_CORE_CALL(SVSM_CORE_CREATE_VCPU); 1104 call.rdx = __pa(caa); 1105 call.r8 = apic_id; 1106 } else { 1107 /* Protocol 0, Call ID 3 */ 1108 call.rax = SVSM_CORE_CALL(SVSM_CORE_DELETE_VCPU); 1109 } 1110 1111 ret = svsm_perform_call_protocol(&call); 1112 1113 local_irq_restore(flags); 1114 } else { 1115 /* 1116 * If the kernel runs at VMPL0, it can change the VMSA 1117 * bit for a page using the RMPADJUST instruction. 1118 * However, for the instruction to succeed it must 1119 * target the permissions of a lesser privileged (higher 1120 * numbered) VMPL level, so use VMPL1. 1121 */ 1122 u64 attrs = 1; 1123 1124 if (make_vmsa) 1125 attrs |= RMPADJUST_VMSA_PAGE_BIT; 1126 1127 ret = rmpadjust((unsigned long)va, RMP_PG_SIZE_4K, attrs); 1128 } 1129 1130 return ret; 1131 } 1132 1133 #define __ATTR_BASE (SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK) 1134 #define INIT_CS_ATTRIBS (__ATTR_BASE | SVM_SELECTOR_READ_MASK | SVM_SELECTOR_CODE_MASK) 1135 #define INIT_DS_ATTRIBS (__ATTR_BASE | SVM_SELECTOR_WRITE_MASK) 1136 1137 #define INIT_LDTR_ATTRIBS (SVM_SELECTOR_P_MASK | 2) 1138 #define INIT_TR_ATTRIBS (SVM_SELECTOR_P_MASK | 3) 1139 1140 static void *snp_alloc_vmsa_page(int cpu) 1141 { 1142 struct page *p; 1143 1144 /* 1145 * Allocate VMSA page to work around the SNP erratum where the CPU will 1146 * incorrectly signal an RMP violation #PF if a large page (2MB or 1GB) 1147 * collides with the RMP entry of VMSA page. The recommended workaround 1148 * is to not use a large page. 1149 * 1150 * Allocate an 8k page which is also 8k-aligned. 1151 */ 1152 p = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL_ACCOUNT | __GFP_ZERO, 1); 1153 if (!p) 1154 return NULL; 1155 1156 split_page(p, 1); 1157 1158 /* Free the first 4k. This page may be 2M/1G aligned and cannot be used. */ 1159 __free_page(p); 1160 1161 return page_address(p + 1); 1162 } 1163 1164 static void snp_cleanup_vmsa(struct sev_es_save_area *vmsa, int apic_id) 1165 { 1166 int err; 1167 1168 err = snp_set_vmsa(vmsa, NULL, apic_id, false); 1169 if (err) 1170 pr_err("clear VMSA page failed (%u), leaking page\n", err); 1171 else 1172 free_page((unsigned long)vmsa); 1173 } 1174 1175 static int wakeup_cpu_via_vmgexit(u32 apic_id, unsigned long start_ip) 1176 { 1177 struct sev_es_save_area *cur_vmsa, *vmsa; 1178 struct ghcb_state state; 1179 struct svsm_ca *caa; 1180 unsigned long flags; 1181 struct ghcb *ghcb; 1182 u8 sipi_vector; 1183 int cpu, ret; 1184 u64 cr4; 1185 1186 /* 1187 * The hypervisor SNP feature support check has happened earlier, just check 1188 * the AP_CREATION one here. 1189 */ 1190 if (!(sev_hv_features & GHCB_HV_FT_SNP_AP_CREATION)) 1191 return -EOPNOTSUPP; 1192 1193 /* 1194 * Verify the desired start IP against the known trampoline start IP 1195 * to catch any future new trampolines that may be introduced that 1196 * would require a new protected guest entry point. 1197 */ 1198 if (WARN_ONCE(start_ip != real_mode_header->trampoline_start, 1199 "Unsupported SNP start_ip: %lx\n", start_ip)) 1200 return -EINVAL; 1201 1202 /* Override start_ip with known protected guest start IP */ 1203 start_ip = real_mode_header->sev_es_trampoline_start; 1204 1205 /* Find the logical CPU for the APIC ID */ 1206 for_each_present_cpu(cpu) { 1207 if (arch_match_cpu_phys_id(cpu, apic_id)) 1208 break; 1209 } 1210 if (cpu >= nr_cpu_ids) 1211 return -EINVAL; 1212 1213 cur_vmsa = per_cpu(sev_vmsa, cpu); 1214 1215 /* 1216 * A new VMSA is created each time because there is no guarantee that 1217 * the current VMSA is the kernels or that the vCPU is not running. If 1218 * an attempt was done to use the current VMSA with a running vCPU, a 1219 * #VMEXIT of that vCPU would wipe out all of the settings being done 1220 * here. 1221 */ 1222 vmsa = (struct sev_es_save_area *)snp_alloc_vmsa_page(cpu); 1223 if (!vmsa) 1224 return -ENOMEM; 1225 1226 /* If an SVSM is present, the SVSM per-CPU CAA will be !NULL */ 1227 caa = per_cpu(svsm_caa, cpu); 1228 1229 /* CR4 should maintain the MCE value */ 1230 cr4 = native_read_cr4() & X86_CR4_MCE; 1231 1232 /* Set the CS value based on the start_ip converted to a SIPI vector */ 1233 sipi_vector = (start_ip >> 12); 1234 vmsa->cs.base = sipi_vector << 12; 1235 vmsa->cs.limit = AP_INIT_CS_LIMIT; 1236 vmsa->cs.attrib = INIT_CS_ATTRIBS; 1237 vmsa->cs.selector = sipi_vector << 8; 1238 1239 /* Set the RIP value based on start_ip */ 1240 vmsa->rip = start_ip & 0xfff; 1241 1242 /* Set AP INIT defaults as documented in the APM */ 1243 vmsa->ds.limit = AP_INIT_DS_LIMIT; 1244 vmsa->ds.attrib = INIT_DS_ATTRIBS; 1245 vmsa->es = vmsa->ds; 1246 vmsa->fs = vmsa->ds; 1247 vmsa->gs = vmsa->ds; 1248 vmsa->ss = vmsa->ds; 1249 1250 vmsa->gdtr.limit = AP_INIT_GDTR_LIMIT; 1251 vmsa->ldtr.limit = AP_INIT_LDTR_LIMIT; 1252 vmsa->ldtr.attrib = INIT_LDTR_ATTRIBS; 1253 vmsa->idtr.limit = AP_INIT_IDTR_LIMIT; 1254 vmsa->tr.limit = AP_INIT_TR_LIMIT; 1255 vmsa->tr.attrib = INIT_TR_ATTRIBS; 1256 1257 vmsa->cr4 = cr4; 1258 vmsa->cr0 = AP_INIT_CR0_DEFAULT; 1259 vmsa->dr7 = DR7_RESET_VALUE; 1260 vmsa->dr6 = AP_INIT_DR6_DEFAULT; 1261 vmsa->rflags = AP_INIT_RFLAGS_DEFAULT; 1262 vmsa->g_pat = AP_INIT_GPAT_DEFAULT; 1263 vmsa->xcr0 = AP_INIT_XCR0_DEFAULT; 1264 vmsa->mxcsr = AP_INIT_MXCSR_DEFAULT; 1265 vmsa->x87_ftw = AP_INIT_X87_FTW_DEFAULT; 1266 vmsa->x87_fcw = AP_INIT_X87_FCW_DEFAULT; 1267 1268 /* SVME must be set. */ 1269 vmsa->efer = EFER_SVME; 1270 1271 /* 1272 * Set the SNP-specific fields for this VMSA: 1273 * VMPL level 1274 * SEV_FEATURES (matches the SEV STATUS MSR right shifted 2 bits) 1275 */ 1276 vmsa->vmpl = snp_vmpl; 1277 vmsa->sev_features = sev_status >> 2; 1278 1279 /* Switch the page over to a VMSA page now that it is initialized */ 1280 ret = snp_set_vmsa(vmsa, caa, apic_id, true); 1281 if (ret) { 1282 pr_err("set VMSA page failed (%u)\n", ret); 1283 free_page((unsigned long)vmsa); 1284 1285 return -EINVAL; 1286 } 1287 1288 /* Issue VMGEXIT AP Creation NAE event */ 1289 local_irq_save(flags); 1290 1291 ghcb = __sev_get_ghcb(&state); 1292 1293 vc_ghcb_invalidate(ghcb); 1294 ghcb_set_rax(ghcb, vmsa->sev_features); 1295 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_CREATION); 1296 ghcb_set_sw_exit_info_1(ghcb, 1297 ((u64)apic_id << 32) | 1298 ((u64)snp_vmpl << 16) | 1299 SVM_VMGEXIT_AP_CREATE); 1300 ghcb_set_sw_exit_info_2(ghcb, __pa(vmsa)); 1301 1302 sev_es_wr_ghcb_msr(__pa(ghcb)); 1303 VMGEXIT(); 1304 1305 if (!ghcb_sw_exit_info_1_is_valid(ghcb) || 1306 lower_32_bits(ghcb->save.sw_exit_info_1)) { 1307 pr_err("SNP AP Creation error\n"); 1308 ret = -EINVAL; 1309 } 1310 1311 __sev_put_ghcb(&state); 1312 1313 local_irq_restore(flags); 1314 1315 /* Perform cleanup if there was an error */ 1316 if (ret) { 1317 snp_cleanup_vmsa(vmsa, apic_id); 1318 vmsa = NULL; 1319 } 1320 1321 /* Free up any previous VMSA page */ 1322 if (cur_vmsa) 1323 snp_cleanup_vmsa(cur_vmsa, apic_id); 1324 1325 /* Record the current VMSA page */ 1326 per_cpu(sev_vmsa, cpu) = vmsa; 1327 1328 return ret; 1329 } 1330 1331 void __init snp_set_wakeup_secondary_cpu(void) 1332 { 1333 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 1334 return; 1335 1336 /* 1337 * Always set this override if SNP is enabled. This makes it the 1338 * required method to start APs under SNP. If the hypervisor does 1339 * not support AP creation, then no APs will be started. 1340 */ 1341 apic_update_callback(wakeup_secondary_cpu, wakeup_cpu_via_vmgexit); 1342 } 1343 1344 int __init sev_es_setup_ap_jump_table(struct real_mode_header *rmh) 1345 { 1346 u16 startup_cs, startup_ip; 1347 phys_addr_t jump_table_pa; 1348 u64 jump_table_addr; 1349 u16 __iomem *jump_table; 1350 1351 jump_table_addr = get_jump_table_addr(); 1352 1353 /* On UP guests there is no jump table so this is not a failure */ 1354 if (!jump_table_addr) 1355 return 0; 1356 1357 /* Check if AP Jump Table is page-aligned */ 1358 if (jump_table_addr & ~PAGE_MASK) 1359 return -EINVAL; 1360 1361 jump_table_pa = jump_table_addr & PAGE_MASK; 1362 1363 startup_cs = (u16)(rmh->trampoline_start >> 4); 1364 startup_ip = (u16)(rmh->sev_es_trampoline_start - 1365 rmh->trampoline_start); 1366 1367 jump_table = ioremap_encrypted(jump_table_pa, PAGE_SIZE); 1368 if (!jump_table) 1369 return -EIO; 1370 1371 writew(startup_ip, &jump_table[0]); 1372 writew(startup_cs, &jump_table[1]); 1373 1374 iounmap(jump_table); 1375 1376 return 0; 1377 } 1378 1379 /* 1380 * This is needed by the OVMF UEFI firmware which will use whatever it finds in 1381 * the GHCB MSR as its GHCB to talk to the hypervisor. So make sure the per-cpu 1382 * runtime GHCBs used by the kernel are also mapped in the EFI page-table. 1383 */ 1384 int __init sev_es_efi_map_ghcbs(pgd_t *pgd) 1385 { 1386 struct sev_es_runtime_data *data; 1387 unsigned long address, pflags; 1388 int cpu; 1389 u64 pfn; 1390 1391 if (!cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) 1392 return 0; 1393 1394 pflags = _PAGE_NX | _PAGE_RW; 1395 1396 for_each_possible_cpu(cpu) { 1397 data = per_cpu(runtime_data, cpu); 1398 1399 address = __pa(&data->ghcb_page); 1400 pfn = address >> PAGE_SHIFT; 1401 1402 if (kernel_map_pages_in_pgd(pgd, pfn, address, 1, pflags)) 1403 return 1; 1404 } 1405 1406 return 0; 1407 } 1408 1409 /* Writes to the SVSM CAA MSR are ignored */ 1410 static enum es_result __vc_handle_msr_caa(struct pt_regs *regs, bool write) 1411 { 1412 if (write) 1413 return ES_OK; 1414 1415 regs->ax = lower_32_bits(this_cpu_read(svsm_caa_pa)); 1416 regs->dx = upper_32_bits(this_cpu_read(svsm_caa_pa)); 1417 1418 return ES_OK; 1419 } 1420 1421 static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 1422 { 1423 struct pt_regs *regs = ctxt->regs; 1424 enum es_result ret; 1425 bool write; 1426 1427 /* Is it a WRMSR? */ 1428 write = ctxt->insn.opcode.bytes[1] == 0x30; 1429 1430 if (regs->cx == MSR_SVSM_CAA) 1431 return __vc_handle_msr_caa(regs, write); 1432 1433 ghcb_set_rcx(ghcb, regs->cx); 1434 if (write) { 1435 ghcb_set_rax(ghcb, regs->ax); 1436 ghcb_set_rdx(ghcb, regs->dx); 1437 } 1438 1439 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, write, 0); 1440 1441 if ((ret == ES_OK) && !write) { 1442 regs->ax = ghcb->save.rax; 1443 regs->dx = ghcb->save.rdx; 1444 } 1445 1446 return ret; 1447 } 1448 1449 static void snp_register_per_cpu_ghcb(void) 1450 { 1451 struct sev_es_runtime_data *data; 1452 struct ghcb *ghcb; 1453 1454 data = this_cpu_read(runtime_data); 1455 ghcb = &data->ghcb_page; 1456 1457 snp_register_ghcb_early(__pa(ghcb)); 1458 } 1459 1460 void setup_ghcb(void) 1461 { 1462 if (!cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) 1463 return; 1464 1465 /* 1466 * Check whether the runtime #VC exception handler is active. It uses 1467 * the per-CPU GHCB page which is set up by sev_es_init_vc_handling(). 1468 * 1469 * If SNP is active, register the per-CPU GHCB page so that the runtime 1470 * exception handler can use it. 1471 */ 1472 if (initial_vc_handler == (unsigned long)kernel_exc_vmm_communication) { 1473 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 1474 snp_register_per_cpu_ghcb(); 1475 1476 sev_cfg.ghcbs_initialized = true; 1477 1478 return; 1479 } 1480 1481 /* 1482 * Make sure the hypervisor talks a supported protocol. 1483 * This gets called only in the BSP boot phase. 1484 */ 1485 if (!sev_es_negotiate_protocol()) 1486 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); 1487 1488 /* 1489 * Clear the boot_ghcb. The first exception comes in before the bss 1490 * section is cleared. 1491 */ 1492 memset(&boot_ghcb_page, 0, PAGE_SIZE); 1493 1494 /* Alright - Make the boot-ghcb public */ 1495 boot_ghcb = &boot_ghcb_page; 1496 1497 /* SNP guest requires that GHCB GPA must be registered. */ 1498 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 1499 snp_register_ghcb_early(__pa(&boot_ghcb_page)); 1500 } 1501 1502 #ifdef CONFIG_HOTPLUG_CPU 1503 static void sev_es_ap_hlt_loop(void) 1504 { 1505 struct ghcb_state state; 1506 struct ghcb *ghcb; 1507 1508 ghcb = __sev_get_ghcb(&state); 1509 1510 while (true) { 1511 vc_ghcb_invalidate(ghcb); 1512 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_HLT_LOOP); 1513 ghcb_set_sw_exit_info_1(ghcb, 0); 1514 ghcb_set_sw_exit_info_2(ghcb, 0); 1515 1516 sev_es_wr_ghcb_msr(__pa(ghcb)); 1517 VMGEXIT(); 1518 1519 /* Wakeup signal? */ 1520 if (ghcb_sw_exit_info_2_is_valid(ghcb) && 1521 ghcb->save.sw_exit_info_2) 1522 break; 1523 } 1524 1525 __sev_put_ghcb(&state); 1526 } 1527 1528 /* 1529 * Play_dead handler when running under SEV-ES. This is needed because 1530 * the hypervisor can't deliver an SIPI request to restart the AP. 1531 * Instead the kernel has to issue a VMGEXIT to halt the VCPU until the 1532 * hypervisor wakes it up again. 1533 */ 1534 static void sev_es_play_dead(void) 1535 { 1536 play_dead_common(); 1537 1538 /* IRQs now disabled */ 1539 1540 sev_es_ap_hlt_loop(); 1541 1542 /* 1543 * If we get here, the VCPU was woken up again. Jump to CPU 1544 * startup code to get it back online. 1545 */ 1546 soft_restart_cpu(); 1547 } 1548 #else /* CONFIG_HOTPLUG_CPU */ 1549 #define sev_es_play_dead native_play_dead 1550 #endif /* CONFIG_HOTPLUG_CPU */ 1551 1552 #ifdef CONFIG_SMP 1553 static void __init sev_es_setup_play_dead(void) 1554 { 1555 smp_ops.play_dead = sev_es_play_dead; 1556 } 1557 #else 1558 static inline void sev_es_setup_play_dead(void) { } 1559 #endif 1560 1561 static void __init alloc_runtime_data(int cpu) 1562 { 1563 struct sev_es_runtime_data *data; 1564 1565 data = memblock_alloc_node(sizeof(*data), PAGE_SIZE, cpu_to_node(cpu)); 1566 if (!data) 1567 panic("Can't allocate SEV-ES runtime data"); 1568 1569 per_cpu(runtime_data, cpu) = data; 1570 1571 if (snp_vmpl) { 1572 struct svsm_ca *caa; 1573 1574 /* Allocate the SVSM CA page if an SVSM is present */ 1575 caa = memblock_alloc(sizeof(*caa), PAGE_SIZE); 1576 if (!caa) 1577 panic("Can't allocate SVSM CA page\n"); 1578 1579 per_cpu(svsm_caa, cpu) = caa; 1580 per_cpu(svsm_caa_pa, cpu) = __pa(caa); 1581 } 1582 } 1583 1584 static void __init init_ghcb(int cpu) 1585 { 1586 struct sev_es_runtime_data *data; 1587 int err; 1588 1589 data = per_cpu(runtime_data, cpu); 1590 1591 err = early_set_memory_decrypted((unsigned long)&data->ghcb_page, 1592 sizeof(data->ghcb_page)); 1593 if (err) 1594 panic("Can't map GHCBs unencrypted"); 1595 1596 memset(&data->ghcb_page, 0, sizeof(data->ghcb_page)); 1597 1598 data->ghcb_active = false; 1599 data->backup_ghcb_active = false; 1600 } 1601 1602 void __init sev_es_init_vc_handling(void) 1603 { 1604 int cpu; 1605 1606 BUILD_BUG_ON(offsetof(struct sev_es_runtime_data, ghcb_page) % PAGE_SIZE); 1607 1608 if (!cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) 1609 return; 1610 1611 if (!sev_es_check_cpu_features()) 1612 panic("SEV-ES CPU Features missing"); 1613 1614 /* 1615 * SNP is supported in v2 of the GHCB spec which mandates support for HV 1616 * features. 1617 */ 1618 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) { 1619 sev_hv_features = get_hv_features(); 1620 1621 if (!(sev_hv_features & GHCB_HV_FT_SNP)) 1622 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED); 1623 } 1624 1625 /* Initialize per-cpu GHCB pages */ 1626 for_each_possible_cpu(cpu) { 1627 alloc_runtime_data(cpu); 1628 init_ghcb(cpu); 1629 } 1630 1631 /* If running under an SVSM, switch to the per-cpu CA */ 1632 if (snp_vmpl) { 1633 struct svsm_call call = {}; 1634 unsigned long flags; 1635 int ret; 1636 1637 local_irq_save(flags); 1638 1639 /* 1640 * SVSM_CORE_REMAP_CA call: 1641 * RAX = 0 (Protocol=0, CallID=0) 1642 * RCX = New CA GPA 1643 */ 1644 call.caa = svsm_get_caa(); 1645 call.rax = SVSM_CORE_CALL(SVSM_CORE_REMAP_CA); 1646 call.rcx = this_cpu_read(svsm_caa_pa); 1647 ret = svsm_perform_call_protocol(&call); 1648 if (ret) 1649 panic("Can't remap the SVSM CA, ret=%d, rax_out=0x%llx\n", 1650 ret, call.rax_out); 1651 1652 sev_cfg.use_cas = true; 1653 1654 local_irq_restore(flags); 1655 } 1656 1657 sev_es_setup_play_dead(); 1658 1659 /* Secondary CPUs use the runtime #VC handler */ 1660 initial_vc_handler = (unsigned long)kernel_exc_vmm_communication; 1661 } 1662 1663 static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt) 1664 { 1665 int trapnr = ctxt->fi.vector; 1666 1667 if (trapnr == X86_TRAP_PF) 1668 native_write_cr2(ctxt->fi.cr2); 1669 1670 ctxt->regs->orig_ax = ctxt->fi.error_code; 1671 do_early_exception(ctxt->regs, trapnr); 1672 } 1673 1674 static long *vc_insn_get_rm(struct es_em_ctxt *ctxt) 1675 { 1676 long *reg_array; 1677 int offset; 1678 1679 reg_array = (long *)ctxt->regs; 1680 offset = insn_get_modrm_rm_off(&ctxt->insn, ctxt->regs); 1681 1682 if (offset < 0) 1683 return NULL; 1684 1685 offset /= sizeof(long); 1686 1687 return reg_array + offset; 1688 } 1689 static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt, 1690 unsigned int bytes, bool read) 1691 { 1692 u64 exit_code, exit_info_1, exit_info_2; 1693 unsigned long ghcb_pa = __pa(ghcb); 1694 enum es_result res; 1695 phys_addr_t paddr; 1696 void __user *ref; 1697 1698 ref = insn_get_addr_ref(&ctxt->insn, ctxt->regs); 1699 if (ref == (void __user *)-1L) 1700 return ES_UNSUPPORTED; 1701 1702 exit_code = read ? SVM_VMGEXIT_MMIO_READ : SVM_VMGEXIT_MMIO_WRITE; 1703 1704 res = vc_slow_virt_to_phys(ghcb, ctxt, (unsigned long)ref, &paddr); 1705 if (res != ES_OK) { 1706 if (res == ES_EXCEPTION && !read) 1707 ctxt->fi.error_code |= X86_PF_WRITE; 1708 1709 return res; 1710 } 1711 1712 exit_info_1 = paddr; 1713 /* Can never be greater than 8 */ 1714 exit_info_2 = bytes; 1715 1716 ghcb_set_sw_scratch(ghcb, ghcb_pa + offsetof(struct ghcb, shared_buffer)); 1717 1718 return sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, exit_info_1, exit_info_2); 1719 } 1720 1721 /* 1722 * The MOVS instruction has two memory operands, which raises the 1723 * problem that it is not known whether the access to the source or the 1724 * destination caused the #VC exception (and hence whether an MMIO read 1725 * or write operation needs to be emulated). 1726 * 1727 * Instead of playing games with walking page-tables and trying to guess 1728 * whether the source or destination is an MMIO range, split the move 1729 * into two operations, a read and a write with only one memory operand. 1730 * This will cause a nested #VC exception on the MMIO address which can 1731 * then be handled. 1732 * 1733 * This implementation has the benefit that it also supports MOVS where 1734 * source _and_ destination are MMIO regions. 1735 * 1736 * It will slow MOVS on MMIO down a lot, but in SEV-ES guests it is a 1737 * rare operation. If it turns out to be a performance problem the split 1738 * operations can be moved to memcpy_fromio() and memcpy_toio(). 1739 */ 1740 static enum es_result vc_handle_mmio_movs(struct es_em_ctxt *ctxt, 1741 unsigned int bytes) 1742 { 1743 unsigned long ds_base, es_base; 1744 unsigned char *src, *dst; 1745 unsigned char buffer[8]; 1746 enum es_result ret; 1747 bool rep; 1748 int off; 1749 1750 ds_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_DS); 1751 es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES); 1752 1753 if (ds_base == -1L || es_base == -1L) { 1754 ctxt->fi.vector = X86_TRAP_GP; 1755 ctxt->fi.error_code = 0; 1756 return ES_EXCEPTION; 1757 } 1758 1759 src = ds_base + (unsigned char *)ctxt->regs->si; 1760 dst = es_base + (unsigned char *)ctxt->regs->di; 1761 1762 ret = vc_read_mem(ctxt, src, buffer, bytes); 1763 if (ret != ES_OK) 1764 return ret; 1765 1766 ret = vc_write_mem(ctxt, dst, buffer, bytes); 1767 if (ret != ES_OK) 1768 return ret; 1769 1770 if (ctxt->regs->flags & X86_EFLAGS_DF) 1771 off = -bytes; 1772 else 1773 off = bytes; 1774 1775 ctxt->regs->si += off; 1776 ctxt->regs->di += off; 1777 1778 rep = insn_has_rep_prefix(&ctxt->insn); 1779 if (rep) 1780 ctxt->regs->cx -= 1; 1781 1782 if (!rep || ctxt->regs->cx == 0) 1783 return ES_OK; 1784 else 1785 return ES_RETRY; 1786 } 1787 1788 static enum es_result vc_handle_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 1789 { 1790 struct insn *insn = &ctxt->insn; 1791 enum insn_mmio_type mmio; 1792 unsigned int bytes = 0; 1793 enum es_result ret; 1794 u8 sign_byte; 1795 long *reg_data; 1796 1797 mmio = insn_decode_mmio(insn, &bytes); 1798 if (mmio == INSN_MMIO_DECODE_FAILED) 1799 return ES_DECODE_FAILED; 1800 1801 if (mmio != INSN_MMIO_WRITE_IMM && mmio != INSN_MMIO_MOVS) { 1802 reg_data = insn_get_modrm_reg_ptr(insn, ctxt->regs); 1803 if (!reg_data) 1804 return ES_DECODE_FAILED; 1805 } 1806 1807 if (user_mode(ctxt->regs)) 1808 return ES_UNSUPPORTED; 1809 1810 switch (mmio) { 1811 case INSN_MMIO_WRITE: 1812 memcpy(ghcb->shared_buffer, reg_data, bytes); 1813 ret = vc_do_mmio(ghcb, ctxt, bytes, false); 1814 break; 1815 case INSN_MMIO_WRITE_IMM: 1816 memcpy(ghcb->shared_buffer, insn->immediate1.bytes, bytes); 1817 ret = vc_do_mmio(ghcb, ctxt, bytes, false); 1818 break; 1819 case INSN_MMIO_READ: 1820 ret = vc_do_mmio(ghcb, ctxt, bytes, true); 1821 if (ret) 1822 break; 1823 1824 /* Zero-extend for 32-bit operation */ 1825 if (bytes == 4) 1826 *reg_data = 0; 1827 1828 memcpy(reg_data, ghcb->shared_buffer, bytes); 1829 break; 1830 case INSN_MMIO_READ_ZERO_EXTEND: 1831 ret = vc_do_mmio(ghcb, ctxt, bytes, true); 1832 if (ret) 1833 break; 1834 1835 /* Zero extend based on operand size */ 1836 memset(reg_data, 0, insn->opnd_bytes); 1837 memcpy(reg_data, ghcb->shared_buffer, bytes); 1838 break; 1839 case INSN_MMIO_READ_SIGN_EXTEND: 1840 ret = vc_do_mmio(ghcb, ctxt, bytes, true); 1841 if (ret) 1842 break; 1843 1844 if (bytes == 1) { 1845 u8 *val = (u8 *)ghcb->shared_buffer; 1846 1847 sign_byte = (*val & 0x80) ? 0xff : 0x00; 1848 } else { 1849 u16 *val = (u16 *)ghcb->shared_buffer; 1850 1851 sign_byte = (*val & 0x8000) ? 0xff : 0x00; 1852 } 1853 1854 /* Sign extend based on operand size */ 1855 memset(reg_data, sign_byte, insn->opnd_bytes); 1856 memcpy(reg_data, ghcb->shared_buffer, bytes); 1857 break; 1858 case INSN_MMIO_MOVS: 1859 ret = vc_handle_mmio_movs(ctxt, bytes); 1860 break; 1861 default: 1862 ret = ES_UNSUPPORTED; 1863 break; 1864 } 1865 1866 return ret; 1867 } 1868 1869 static enum es_result vc_handle_dr7_write(struct ghcb *ghcb, 1870 struct es_em_ctxt *ctxt) 1871 { 1872 struct sev_es_runtime_data *data = this_cpu_read(runtime_data); 1873 long val, *reg = vc_insn_get_rm(ctxt); 1874 enum es_result ret; 1875 1876 if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP) 1877 return ES_VMM_ERROR; 1878 1879 if (!reg) 1880 return ES_DECODE_FAILED; 1881 1882 val = *reg; 1883 1884 /* Upper 32 bits must be written as zeroes */ 1885 if (val >> 32) { 1886 ctxt->fi.vector = X86_TRAP_GP; 1887 ctxt->fi.error_code = 0; 1888 return ES_EXCEPTION; 1889 } 1890 1891 /* Clear out other reserved bits and set bit 10 */ 1892 val = (val & 0xffff23ffL) | BIT(10); 1893 1894 /* Early non-zero writes to DR7 are not supported */ 1895 if (!data && (val & ~DR7_RESET_VALUE)) 1896 return ES_UNSUPPORTED; 1897 1898 /* Using a value of 0 for ExitInfo1 means RAX holds the value */ 1899 ghcb_set_rax(ghcb, val); 1900 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WRITE_DR7, 0, 0); 1901 if (ret != ES_OK) 1902 return ret; 1903 1904 if (data) 1905 data->dr7 = val; 1906 1907 return ES_OK; 1908 } 1909 1910 static enum es_result vc_handle_dr7_read(struct ghcb *ghcb, 1911 struct es_em_ctxt *ctxt) 1912 { 1913 struct sev_es_runtime_data *data = this_cpu_read(runtime_data); 1914 long *reg = vc_insn_get_rm(ctxt); 1915 1916 if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP) 1917 return ES_VMM_ERROR; 1918 1919 if (!reg) 1920 return ES_DECODE_FAILED; 1921 1922 if (data) 1923 *reg = data->dr7; 1924 else 1925 *reg = DR7_RESET_VALUE; 1926 1927 return ES_OK; 1928 } 1929 1930 static enum es_result vc_handle_wbinvd(struct ghcb *ghcb, 1931 struct es_em_ctxt *ctxt) 1932 { 1933 return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0); 1934 } 1935 1936 static enum es_result vc_handle_rdpmc(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 1937 { 1938 enum es_result ret; 1939 1940 ghcb_set_rcx(ghcb, ctxt->regs->cx); 1941 1942 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_RDPMC, 0, 0); 1943 if (ret != ES_OK) 1944 return ret; 1945 1946 if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb))) 1947 return ES_VMM_ERROR; 1948 1949 ctxt->regs->ax = ghcb->save.rax; 1950 ctxt->regs->dx = ghcb->save.rdx; 1951 1952 return ES_OK; 1953 } 1954 1955 static enum es_result vc_handle_monitor(struct ghcb *ghcb, 1956 struct es_em_ctxt *ctxt) 1957 { 1958 /* 1959 * Treat it as a NOP and do not leak a physical address to the 1960 * hypervisor. 1961 */ 1962 return ES_OK; 1963 } 1964 1965 static enum es_result vc_handle_mwait(struct ghcb *ghcb, 1966 struct es_em_ctxt *ctxt) 1967 { 1968 /* Treat the same as MONITOR/MONITORX */ 1969 return ES_OK; 1970 } 1971 1972 static enum es_result vc_handle_vmmcall(struct ghcb *ghcb, 1973 struct es_em_ctxt *ctxt) 1974 { 1975 enum es_result ret; 1976 1977 ghcb_set_rax(ghcb, ctxt->regs->ax); 1978 ghcb_set_cpl(ghcb, user_mode(ctxt->regs) ? 3 : 0); 1979 1980 if (x86_platform.hyper.sev_es_hcall_prepare) 1981 x86_platform.hyper.sev_es_hcall_prepare(ghcb, ctxt->regs); 1982 1983 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_VMMCALL, 0, 0); 1984 if (ret != ES_OK) 1985 return ret; 1986 1987 if (!ghcb_rax_is_valid(ghcb)) 1988 return ES_VMM_ERROR; 1989 1990 ctxt->regs->ax = ghcb->save.rax; 1991 1992 /* 1993 * Call sev_es_hcall_finish() after regs->ax is already set. 1994 * This allows the hypervisor handler to overwrite it again if 1995 * necessary. 1996 */ 1997 if (x86_platform.hyper.sev_es_hcall_finish && 1998 !x86_platform.hyper.sev_es_hcall_finish(ghcb, ctxt->regs)) 1999 return ES_VMM_ERROR; 2000 2001 return ES_OK; 2002 } 2003 2004 static enum es_result vc_handle_trap_ac(struct ghcb *ghcb, 2005 struct es_em_ctxt *ctxt) 2006 { 2007 /* 2008 * Calling ecx_alignment_check() directly does not work, because it 2009 * enables IRQs and the GHCB is active. Forward the exception and call 2010 * it later from vc_forward_exception(). 2011 */ 2012 ctxt->fi.vector = X86_TRAP_AC; 2013 ctxt->fi.error_code = 0; 2014 return ES_EXCEPTION; 2015 } 2016 2017 static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, 2018 struct ghcb *ghcb, 2019 unsigned long exit_code) 2020 { 2021 enum es_result result = vc_check_opcode_bytes(ctxt, exit_code); 2022 2023 if (result != ES_OK) 2024 return result; 2025 2026 switch (exit_code) { 2027 case SVM_EXIT_READ_DR7: 2028 result = vc_handle_dr7_read(ghcb, ctxt); 2029 break; 2030 case SVM_EXIT_WRITE_DR7: 2031 result = vc_handle_dr7_write(ghcb, ctxt); 2032 break; 2033 case SVM_EXIT_EXCP_BASE + X86_TRAP_AC: 2034 result = vc_handle_trap_ac(ghcb, ctxt); 2035 break; 2036 case SVM_EXIT_RDTSC: 2037 case SVM_EXIT_RDTSCP: 2038 result = vc_handle_rdtsc(ghcb, ctxt, exit_code); 2039 break; 2040 case SVM_EXIT_RDPMC: 2041 result = vc_handle_rdpmc(ghcb, ctxt); 2042 break; 2043 case SVM_EXIT_INVD: 2044 pr_err_ratelimited("#VC exception for INVD??? Seriously???\n"); 2045 result = ES_UNSUPPORTED; 2046 break; 2047 case SVM_EXIT_CPUID: 2048 result = vc_handle_cpuid(ghcb, ctxt); 2049 break; 2050 case SVM_EXIT_IOIO: 2051 result = vc_handle_ioio(ghcb, ctxt); 2052 break; 2053 case SVM_EXIT_MSR: 2054 result = vc_handle_msr(ghcb, ctxt); 2055 break; 2056 case SVM_EXIT_VMMCALL: 2057 result = vc_handle_vmmcall(ghcb, ctxt); 2058 break; 2059 case SVM_EXIT_WBINVD: 2060 result = vc_handle_wbinvd(ghcb, ctxt); 2061 break; 2062 case SVM_EXIT_MONITOR: 2063 result = vc_handle_monitor(ghcb, ctxt); 2064 break; 2065 case SVM_EXIT_MWAIT: 2066 result = vc_handle_mwait(ghcb, ctxt); 2067 break; 2068 case SVM_EXIT_NPF: 2069 result = vc_handle_mmio(ghcb, ctxt); 2070 break; 2071 default: 2072 /* 2073 * Unexpected #VC exception 2074 */ 2075 result = ES_UNSUPPORTED; 2076 } 2077 2078 return result; 2079 } 2080 2081 static __always_inline bool is_vc2_stack(unsigned long sp) 2082 { 2083 return (sp >= __this_cpu_ist_bottom_va(VC2) && sp < __this_cpu_ist_top_va(VC2)); 2084 } 2085 2086 static __always_inline bool vc_from_invalid_context(struct pt_regs *regs) 2087 { 2088 unsigned long sp, prev_sp; 2089 2090 sp = (unsigned long)regs; 2091 prev_sp = regs->sp; 2092 2093 /* 2094 * If the code was already executing on the VC2 stack when the #VC 2095 * happened, let it proceed to the normal handling routine. This way the 2096 * code executing on the VC2 stack can cause #VC exceptions to get handled. 2097 */ 2098 return is_vc2_stack(sp) && !is_vc2_stack(prev_sp); 2099 } 2100 2101 static bool vc_raw_handle_exception(struct pt_regs *regs, unsigned long error_code) 2102 { 2103 struct ghcb_state state; 2104 struct es_em_ctxt ctxt; 2105 enum es_result result; 2106 struct ghcb *ghcb; 2107 bool ret = true; 2108 2109 ghcb = __sev_get_ghcb(&state); 2110 2111 vc_ghcb_invalidate(ghcb); 2112 result = vc_init_em_ctxt(&ctxt, regs, error_code); 2113 2114 if (result == ES_OK) 2115 result = vc_handle_exitcode(&ctxt, ghcb, error_code); 2116 2117 __sev_put_ghcb(&state); 2118 2119 /* Done - now check the result */ 2120 switch (result) { 2121 case ES_OK: 2122 vc_finish_insn(&ctxt); 2123 break; 2124 case ES_UNSUPPORTED: 2125 pr_err_ratelimited("Unsupported exit-code 0x%02lx in #VC exception (IP: 0x%lx)\n", 2126 error_code, regs->ip); 2127 ret = false; 2128 break; 2129 case ES_VMM_ERROR: 2130 pr_err_ratelimited("Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n", 2131 error_code, regs->ip); 2132 ret = false; 2133 break; 2134 case ES_DECODE_FAILED: 2135 pr_err_ratelimited("Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n", 2136 error_code, regs->ip); 2137 ret = false; 2138 break; 2139 case ES_EXCEPTION: 2140 vc_forward_exception(&ctxt); 2141 break; 2142 case ES_RETRY: 2143 /* Nothing to do */ 2144 break; 2145 default: 2146 pr_emerg("Unknown result in %s():%d\n", __func__, result); 2147 /* 2148 * Emulating the instruction which caused the #VC exception 2149 * failed - can't continue so print debug information 2150 */ 2151 BUG(); 2152 } 2153 2154 return ret; 2155 } 2156 2157 static __always_inline bool vc_is_db(unsigned long error_code) 2158 { 2159 return error_code == SVM_EXIT_EXCP_BASE + X86_TRAP_DB; 2160 } 2161 2162 /* 2163 * Runtime #VC exception handler when raised from kernel mode. Runs in NMI mode 2164 * and will panic when an error happens. 2165 */ 2166 DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication) 2167 { 2168 irqentry_state_t irq_state; 2169 2170 /* 2171 * With the current implementation it is always possible to switch to a 2172 * safe stack because #VC exceptions only happen at known places, like 2173 * intercepted instructions or accesses to MMIO areas/IO ports. They can 2174 * also happen with code instrumentation when the hypervisor intercepts 2175 * #DB, but the critical paths are forbidden to be instrumented, so #DB 2176 * exceptions currently also only happen in safe places. 2177 * 2178 * But keep this here in case the noinstr annotations are violated due 2179 * to bug elsewhere. 2180 */ 2181 if (unlikely(vc_from_invalid_context(regs))) { 2182 instrumentation_begin(); 2183 panic("Can't handle #VC exception from unsupported context\n"); 2184 instrumentation_end(); 2185 } 2186 2187 /* 2188 * Handle #DB before calling into !noinstr code to avoid recursive #DB. 2189 */ 2190 if (vc_is_db(error_code)) { 2191 exc_debug(regs); 2192 return; 2193 } 2194 2195 irq_state = irqentry_nmi_enter(regs); 2196 2197 instrumentation_begin(); 2198 2199 if (!vc_raw_handle_exception(regs, error_code)) { 2200 /* Show some debug info */ 2201 show_regs(regs); 2202 2203 /* Ask hypervisor to sev_es_terminate */ 2204 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); 2205 2206 /* If that fails and we get here - just panic */ 2207 panic("Returned from Terminate-Request to Hypervisor\n"); 2208 } 2209 2210 instrumentation_end(); 2211 irqentry_nmi_exit(regs, irq_state); 2212 } 2213 2214 /* 2215 * Runtime #VC exception handler when raised from user mode. Runs in IRQ mode 2216 * and will kill the current task with SIGBUS when an error happens. 2217 */ 2218 DEFINE_IDTENTRY_VC_USER(exc_vmm_communication) 2219 { 2220 /* 2221 * Handle #DB before calling into !noinstr code to avoid recursive #DB. 2222 */ 2223 if (vc_is_db(error_code)) { 2224 noist_exc_debug(regs); 2225 return; 2226 } 2227 2228 irqentry_enter_from_user_mode(regs); 2229 instrumentation_begin(); 2230 2231 if (!vc_raw_handle_exception(regs, error_code)) { 2232 /* 2233 * Do not kill the machine if user-space triggered the 2234 * exception. Send SIGBUS instead and let user-space deal with 2235 * it. 2236 */ 2237 force_sig_fault(SIGBUS, BUS_OBJERR, (void __user *)0); 2238 } 2239 2240 instrumentation_end(); 2241 irqentry_exit_to_user_mode(regs); 2242 } 2243 2244 bool __init handle_vc_boot_ghcb(struct pt_regs *regs) 2245 { 2246 unsigned long exit_code = regs->orig_ax; 2247 struct es_em_ctxt ctxt; 2248 enum es_result result; 2249 2250 vc_ghcb_invalidate(boot_ghcb); 2251 2252 result = vc_init_em_ctxt(&ctxt, regs, exit_code); 2253 if (result == ES_OK) 2254 result = vc_handle_exitcode(&ctxt, boot_ghcb, exit_code); 2255 2256 /* Done - now check the result */ 2257 switch (result) { 2258 case ES_OK: 2259 vc_finish_insn(&ctxt); 2260 break; 2261 case ES_UNSUPPORTED: 2262 early_printk("PANIC: Unsupported exit-code 0x%02lx in early #VC exception (IP: 0x%lx)\n", 2263 exit_code, regs->ip); 2264 goto fail; 2265 case ES_VMM_ERROR: 2266 early_printk("PANIC: Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n", 2267 exit_code, regs->ip); 2268 goto fail; 2269 case ES_DECODE_FAILED: 2270 early_printk("PANIC: Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n", 2271 exit_code, regs->ip); 2272 goto fail; 2273 case ES_EXCEPTION: 2274 vc_early_forward_exception(&ctxt); 2275 break; 2276 case ES_RETRY: 2277 /* Nothing to do */ 2278 break; 2279 default: 2280 BUG(); 2281 } 2282 2283 return true; 2284 2285 fail: 2286 show_regs(regs); 2287 2288 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); 2289 } 2290 2291 /* 2292 * Initial set up of SNP relies on information provided by the 2293 * Confidential Computing blob, which can be passed to the kernel 2294 * in the following ways, depending on how it is booted: 2295 * 2296 * - when booted via the boot/decompress kernel: 2297 * - via boot_params 2298 * 2299 * - when booted directly by firmware/bootloader (e.g. CONFIG_PVH): 2300 * - via a setup_data entry, as defined by the Linux Boot Protocol 2301 * 2302 * Scan for the blob in that order. 2303 */ 2304 static __head struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp) 2305 { 2306 struct cc_blob_sev_info *cc_info; 2307 2308 /* Boot kernel would have passed the CC blob via boot_params. */ 2309 if (bp->cc_blob_address) { 2310 cc_info = (struct cc_blob_sev_info *)(unsigned long)bp->cc_blob_address; 2311 goto found_cc_info; 2312 } 2313 2314 /* 2315 * If kernel was booted directly, without the use of the 2316 * boot/decompression kernel, the CC blob may have been passed via 2317 * setup_data instead. 2318 */ 2319 cc_info = find_cc_blob_setup_data(bp); 2320 if (!cc_info) 2321 return NULL; 2322 2323 found_cc_info: 2324 if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC) 2325 snp_abort(); 2326 2327 return cc_info; 2328 } 2329 2330 static __head void svsm_setup(struct cc_blob_sev_info *cc_info) 2331 { 2332 struct svsm_call call = {}; 2333 int ret; 2334 u64 pa; 2335 2336 /* 2337 * Record the SVSM Calling Area address (CAA) if the guest is not 2338 * running at VMPL0. The CA will be used to communicate with the 2339 * SVSM to perform the SVSM services. 2340 */ 2341 if (!svsm_setup_ca(cc_info)) 2342 return; 2343 2344 /* 2345 * It is very early in the boot and the kernel is running identity 2346 * mapped but without having adjusted the pagetables to where the 2347 * kernel was loaded (physbase), so the get the CA address using 2348 * RIP-relative addressing. 2349 */ 2350 pa = (u64)&RIP_REL_REF(boot_svsm_ca_page); 2351 2352 /* 2353 * Switch over to the boot SVSM CA while the current CA is still 2354 * addressable. There is no GHCB at this point so use the MSR protocol. 2355 * 2356 * SVSM_CORE_REMAP_CA call: 2357 * RAX = 0 (Protocol=0, CallID=0) 2358 * RCX = New CA GPA 2359 */ 2360 call.caa = svsm_get_caa(); 2361 call.rax = SVSM_CORE_CALL(SVSM_CORE_REMAP_CA); 2362 call.rcx = pa; 2363 ret = svsm_perform_call_protocol(&call); 2364 if (ret) 2365 panic("Can't remap the SVSM CA, ret=%d, rax_out=0x%llx\n", ret, call.rax_out); 2366 2367 RIP_REL_REF(boot_svsm_caa) = (struct svsm_ca *)pa; 2368 RIP_REL_REF(boot_svsm_caa_pa) = pa; 2369 } 2370 2371 bool __head snp_init(struct boot_params *bp) 2372 { 2373 struct cc_blob_sev_info *cc_info; 2374 2375 if (!bp) 2376 return false; 2377 2378 cc_info = find_cc_blob(bp); 2379 if (!cc_info) 2380 return false; 2381 2382 if (cc_info->secrets_phys && cc_info->secrets_len == PAGE_SIZE) 2383 secrets_pa = cc_info->secrets_phys; 2384 else 2385 return false; 2386 2387 setup_cpuid_table(cc_info); 2388 2389 svsm_setup(cc_info); 2390 2391 /* 2392 * The CC blob will be used later to access the secrets page. Cache 2393 * it here like the boot kernel does. 2394 */ 2395 bp->cc_blob_address = (u32)(unsigned long)cc_info; 2396 2397 return true; 2398 } 2399 2400 void __head __noreturn snp_abort(void) 2401 { 2402 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED); 2403 } 2404 2405 /* 2406 * SEV-SNP guests should only execute dmi_setup() if EFI_CONFIG_TABLES are 2407 * enabled, as the alternative (fallback) logic for DMI probing in the legacy 2408 * ROM region can cause a crash since this region is not pre-validated. 2409 */ 2410 void __init snp_dmi_setup(void) 2411 { 2412 if (efi_enabled(EFI_CONFIG_TABLES)) 2413 dmi_setup(); 2414 } 2415 2416 static void dump_cpuid_table(void) 2417 { 2418 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); 2419 int i = 0; 2420 2421 pr_info("count=%d reserved=0x%x reserved2=0x%llx\n", 2422 cpuid_table->count, cpuid_table->__reserved1, cpuid_table->__reserved2); 2423 2424 for (i = 0; i < SNP_CPUID_COUNT_MAX; i++) { 2425 const struct snp_cpuid_fn *fn = &cpuid_table->fn[i]; 2426 2427 pr_info("index=%3d fn=0x%08x subfn=0x%08x: eax=0x%08x ebx=0x%08x ecx=0x%08x edx=0x%08x xcr0_in=0x%016llx xss_in=0x%016llx reserved=0x%016llx\n", 2428 i, fn->eax_in, fn->ecx_in, fn->eax, fn->ebx, fn->ecx, 2429 fn->edx, fn->xcr0_in, fn->xss_in, fn->__reserved); 2430 } 2431 } 2432 2433 /* 2434 * It is useful from an auditing/testing perspective to provide an easy way 2435 * for the guest owner to know that the CPUID table has been initialized as 2436 * expected, but that initialization happens too early in boot to print any 2437 * sort of indicator, and there's not really any other good place to do it, 2438 * so do it here. 2439 * 2440 * If running as an SNP guest, report the current VM privilege level (VMPL). 2441 */ 2442 static int __init report_snp_info(void) 2443 { 2444 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); 2445 2446 if (cpuid_table->count) { 2447 pr_info("Using SNP CPUID table, %d entries present.\n", 2448 cpuid_table->count); 2449 2450 if (sev_cfg.debug) 2451 dump_cpuid_table(); 2452 } 2453 2454 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 2455 pr_info("SNP running at VMPL%u.\n", snp_vmpl); 2456 2457 return 0; 2458 } 2459 arch_initcall(report_snp_info); 2460 2461 static void update_attest_input(struct svsm_call *call, struct svsm_attest_call *input) 2462 { 2463 /* If (new) lengths have been returned, propagate them up */ 2464 if (call->rcx_out != call->rcx) 2465 input->manifest_buf.len = call->rcx_out; 2466 2467 if (call->rdx_out != call->rdx) 2468 input->certificates_buf.len = call->rdx_out; 2469 2470 if (call->r8_out != call->r8) 2471 input->report_buf.len = call->r8_out; 2472 } 2473 2474 int snp_issue_svsm_attest_req(u64 call_id, struct svsm_call *call, 2475 struct svsm_attest_call *input) 2476 { 2477 struct svsm_attest_call *ac; 2478 unsigned long flags; 2479 u64 attest_call_pa; 2480 int ret; 2481 2482 if (!snp_vmpl) 2483 return -EINVAL; 2484 2485 local_irq_save(flags); 2486 2487 call->caa = svsm_get_caa(); 2488 2489 ac = (struct svsm_attest_call *)call->caa->svsm_buffer; 2490 attest_call_pa = svsm_get_caa_pa() + offsetof(struct svsm_ca, svsm_buffer); 2491 2492 *ac = *input; 2493 2494 /* 2495 * Set input registers for the request and set RDX and R8 to known 2496 * values in order to detect length values being returned in them. 2497 */ 2498 call->rax = call_id; 2499 call->rcx = attest_call_pa; 2500 call->rdx = -1; 2501 call->r8 = -1; 2502 ret = svsm_perform_call_protocol(call); 2503 update_attest_input(call, input); 2504 2505 local_irq_restore(flags); 2506 2507 return ret; 2508 } 2509 EXPORT_SYMBOL_GPL(snp_issue_svsm_attest_req); 2510 2511 int snp_issue_guest_request(struct snp_guest_req *req, struct snp_req_data *input, 2512 struct snp_guest_request_ioctl *rio) 2513 { 2514 struct ghcb_state state; 2515 struct es_em_ctxt ctxt; 2516 unsigned long flags; 2517 struct ghcb *ghcb; 2518 int ret; 2519 2520 rio->exitinfo2 = SEV_RET_NO_FW_CALL; 2521 2522 /* 2523 * __sev_get_ghcb() needs to run with IRQs disabled because it is using 2524 * a per-CPU GHCB. 2525 */ 2526 local_irq_save(flags); 2527 2528 ghcb = __sev_get_ghcb(&state); 2529 if (!ghcb) { 2530 ret = -EIO; 2531 goto e_restore_irq; 2532 } 2533 2534 vc_ghcb_invalidate(ghcb); 2535 2536 if (req->exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST) { 2537 ghcb_set_rax(ghcb, input->data_gpa); 2538 ghcb_set_rbx(ghcb, input->data_npages); 2539 } 2540 2541 ret = sev_es_ghcb_hv_call(ghcb, &ctxt, req->exit_code, input->req_gpa, input->resp_gpa); 2542 if (ret) 2543 goto e_put; 2544 2545 rio->exitinfo2 = ghcb->save.sw_exit_info_2; 2546 switch (rio->exitinfo2) { 2547 case 0: 2548 break; 2549 2550 case SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_BUSY): 2551 ret = -EAGAIN; 2552 break; 2553 2554 case SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN): 2555 /* Number of expected pages are returned in RBX */ 2556 if (req->exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST) { 2557 input->data_npages = ghcb_get_rbx(ghcb); 2558 ret = -ENOSPC; 2559 break; 2560 } 2561 fallthrough; 2562 default: 2563 ret = -EIO; 2564 break; 2565 } 2566 2567 e_put: 2568 __sev_put_ghcb(&state); 2569 e_restore_irq: 2570 local_irq_restore(flags); 2571 2572 return ret; 2573 } 2574 EXPORT_SYMBOL_GPL(snp_issue_guest_request); 2575 2576 static struct platform_device sev_guest_device = { 2577 .name = "sev-guest", 2578 .id = -1, 2579 }; 2580 2581 static int __init snp_init_platform_device(void) 2582 { 2583 struct sev_guest_platform_data data; 2584 2585 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 2586 return -ENODEV; 2587 2588 data.secrets_gpa = secrets_pa; 2589 if (platform_device_add_data(&sev_guest_device, &data, sizeof(data))) 2590 return -ENODEV; 2591 2592 if (platform_device_register(&sev_guest_device)) 2593 return -ENODEV; 2594 2595 pr_info("SNP guest platform device initialized.\n"); 2596 return 0; 2597 } 2598 device_initcall(snp_init_platform_device); 2599 2600 void sev_show_status(void) 2601 { 2602 int i; 2603 2604 pr_info("Status: "); 2605 for (i = 0; i < MSR_AMD64_SNP_RESV_BIT; i++) { 2606 if (sev_status & BIT_ULL(i)) { 2607 if (!sev_status_feat_names[i]) 2608 continue; 2609 2610 pr_cont("%s ", sev_status_feat_names[i]); 2611 } 2612 } 2613 pr_cont("\n"); 2614 } 2615 2616 void __init snp_update_svsm_ca(void) 2617 { 2618 if (!snp_vmpl) 2619 return; 2620 2621 /* Update the CAA to a proper kernel address */ 2622 boot_svsm_caa = &boot_svsm_ca_page; 2623 } 2624 2625 #ifdef CONFIG_SYSFS 2626 static ssize_t vmpl_show(struct kobject *kobj, 2627 struct kobj_attribute *attr, char *buf) 2628 { 2629 return sysfs_emit(buf, "%d\n", snp_vmpl); 2630 } 2631 2632 static struct kobj_attribute vmpl_attr = __ATTR_RO(vmpl); 2633 2634 static struct attribute *vmpl_attrs[] = { 2635 &vmpl_attr.attr, 2636 NULL 2637 }; 2638 2639 static struct attribute_group sev_attr_group = { 2640 .attrs = vmpl_attrs, 2641 }; 2642 2643 static int __init sev_sysfs_init(void) 2644 { 2645 struct kobject *sev_kobj; 2646 struct device *dev_root; 2647 int ret; 2648 2649 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 2650 return -ENODEV; 2651 2652 dev_root = bus_get_dev_root(&cpu_subsys); 2653 if (!dev_root) 2654 return -ENODEV; 2655 2656 sev_kobj = kobject_create_and_add("sev", &dev_root->kobj); 2657 put_device(dev_root); 2658 2659 if (!sev_kobj) 2660 return -ENOMEM; 2661 2662 ret = sysfs_create_group(sev_kobj, &sev_attr_group); 2663 if (ret) 2664 kobject_put(sev_kobj); 2665 2666 return ret; 2667 } 2668 arch_initcall(sev_sysfs_init); 2669 #endif // CONFIG_SYSFS 2670