1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * AMD Memory Encryption Support 4 * 5 * Copyright (C) 2019 SUSE 6 * 7 * Author: Joerg Roedel <jroedel@suse.de> 8 */ 9 10 #define pr_fmt(fmt) "SEV: " fmt 11 12 #include <linux/sched/debug.h> /* For show_regs() */ 13 #include <linux/percpu-defs.h> 14 #include <linux/cc_platform.h> 15 #include <linux/printk.h> 16 #include <linux/mm_types.h> 17 #include <linux/set_memory.h> 18 #include <linux/memblock.h> 19 #include <linux/kernel.h> 20 #include <linux/mm.h> 21 #include <linux/cpumask.h> 22 #include <linux/efi.h> 23 #include <linux/platform_device.h> 24 #include <linux/io.h> 25 #include <linux/psp-sev.h> 26 #include <linux/dmi.h> 27 #include <uapi/linux/sev-guest.h> 28 29 #include <asm/init.h> 30 #include <asm/cpu_entry_area.h> 31 #include <asm/stacktrace.h> 32 #include <asm/sev.h> 33 #include <asm/insn-eval.h> 34 #include <asm/fpu/xcr.h> 35 #include <asm/processor.h> 36 #include <asm/realmode.h> 37 #include <asm/setup.h> 38 #include <asm/traps.h> 39 #include <asm/svm.h> 40 #include <asm/smp.h> 41 #include <asm/cpu.h> 42 #include <asm/apic.h> 43 #include <asm/cpuid.h> 44 #include <asm/cmdline.h> 45 46 #define DR7_RESET_VALUE 0x400 47 48 /* AP INIT values as documented in the APM2 section "Processor Initialization State" */ 49 #define AP_INIT_CS_LIMIT 0xffff 50 #define AP_INIT_DS_LIMIT 0xffff 51 #define AP_INIT_LDTR_LIMIT 0xffff 52 #define AP_INIT_GDTR_LIMIT 0xffff 53 #define AP_INIT_IDTR_LIMIT 0xffff 54 #define AP_INIT_TR_LIMIT 0xffff 55 #define AP_INIT_RFLAGS_DEFAULT 0x2 56 #define AP_INIT_DR6_DEFAULT 0xffff0ff0 57 #define AP_INIT_GPAT_DEFAULT 0x0007040600070406ULL 58 #define AP_INIT_XCR0_DEFAULT 0x1 59 #define AP_INIT_X87_FTW_DEFAULT 0x5555 60 #define AP_INIT_X87_FCW_DEFAULT 0x0040 61 #define AP_INIT_CR0_DEFAULT 0x60000010 62 #define AP_INIT_MXCSR_DEFAULT 0x1f80 63 64 static const char * const sev_status_feat_names[] = { 65 [MSR_AMD64_SEV_ENABLED_BIT] = "SEV", 66 [MSR_AMD64_SEV_ES_ENABLED_BIT] = "SEV-ES", 67 [MSR_AMD64_SEV_SNP_ENABLED_BIT] = "SEV-SNP", 68 [MSR_AMD64_SNP_VTOM_BIT] = "vTom", 69 [MSR_AMD64_SNP_REFLECT_VC_BIT] = "ReflectVC", 70 [MSR_AMD64_SNP_RESTRICTED_INJ_BIT] = "RI", 71 [MSR_AMD64_SNP_ALT_INJ_BIT] = "AI", 72 [MSR_AMD64_SNP_DEBUG_SWAP_BIT] = "DebugSwap", 73 [MSR_AMD64_SNP_PREVENT_HOST_IBS_BIT] = "NoHostIBS", 74 [MSR_AMD64_SNP_BTB_ISOLATION_BIT] = "BTBIsol", 75 [MSR_AMD64_SNP_VMPL_SSS_BIT] = "VmplSSS", 76 [MSR_AMD64_SNP_SECURE_TSC_BIT] = "SecureTSC", 77 [MSR_AMD64_SNP_VMGEXIT_PARAM_BIT] = "VMGExitParam", 78 [MSR_AMD64_SNP_IBS_VIRT_BIT] = "IBSVirt", 79 [MSR_AMD64_SNP_VMSA_REG_PROT_BIT] = "VMSARegProt", 80 [MSR_AMD64_SNP_SMT_PROT_BIT] = "SMTProt", 81 }; 82 83 /* For early boot hypervisor communication in SEV-ES enabled guests */ 84 static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE); 85 86 /* 87 * Needs to be in the .data section because we need it NULL before bss is 88 * cleared 89 */ 90 static struct ghcb *boot_ghcb __section(".data"); 91 92 /* Bitmap of SEV features supported by the hypervisor */ 93 static u64 sev_hv_features __ro_after_init; 94 95 /* #VC handler runtime per-CPU data */ 96 struct sev_es_runtime_data { 97 struct ghcb ghcb_page; 98 99 /* 100 * Reserve one page per CPU as backup storage for the unencrypted GHCB. 101 * It is needed when an NMI happens while the #VC handler uses the real 102 * GHCB, and the NMI handler itself is causing another #VC exception. In 103 * that case the GHCB content of the first handler needs to be backed up 104 * and restored. 105 */ 106 struct ghcb backup_ghcb; 107 108 /* 109 * Mark the per-cpu GHCBs as in-use to detect nested #VC exceptions. 110 * There is no need for it to be atomic, because nothing is written to 111 * the GHCB between the read and the write of ghcb_active. So it is safe 112 * to use it when a nested #VC exception happens before the write. 113 * 114 * This is necessary for example in the #VC->NMI->#VC case when the NMI 115 * happens while the first #VC handler uses the GHCB. When the NMI code 116 * raises a second #VC handler it might overwrite the contents of the 117 * GHCB written by the first handler. To avoid this the content of the 118 * GHCB is saved and restored when the GHCB is detected to be in use 119 * already. 120 */ 121 bool ghcb_active; 122 bool backup_ghcb_active; 123 124 /* 125 * Cached DR7 value - write it on DR7 writes and return it on reads. 126 * That value will never make it to the real hardware DR7 as debugging 127 * is currently unsupported in SEV-ES guests. 128 */ 129 unsigned long dr7; 130 }; 131 132 struct ghcb_state { 133 struct ghcb *ghcb; 134 }; 135 136 /* For early boot SVSM communication */ 137 static struct svsm_ca boot_svsm_ca_page __aligned(PAGE_SIZE); 138 139 static DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data); 140 static DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa); 141 static DEFINE_PER_CPU(struct svsm_ca *, svsm_caa); 142 static DEFINE_PER_CPU(u64, svsm_caa_pa); 143 144 struct sev_config { 145 __u64 debug : 1, 146 147 /* 148 * Indicates when the per-CPU GHCB has been created and registered 149 * and thus can be used by the BSP instead of the early boot GHCB. 150 * 151 * For APs, the per-CPU GHCB is created before they are started 152 * and registered upon startup, so this flag can be used globally 153 * for the BSP and APs. 154 */ 155 ghcbs_initialized : 1, 156 157 /* 158 * Indicates when the per-CPU SVSM CA is to be used instead of the 159 * boot SVSM CA. 160 * 161 * For APs, the per-CPU SVSM CA is created as part of the AP 162 * bringup, so this flag can be used globally for the BSP and APs. 163 */ 164 use_cas : 1, 165 166 __reserved : 61; 167 }; 168 169 static struct sev_config sev_cfg __read_mostly; 170 171 static __always_inline bool on_vc_stack(struct pt_regs *regs) 172 { 173 unsigned long sp = regs->sp; 174 175 /* User-mode RSP is not trusted */ 176 if (user_mode(regs)) 177 return false; 178 179 /* SYSCALL gap still has user-mode RSP */ 180 if (ip_within_syscall_gap(regs)) 181 return false; 182 183 return ((sp >= __this_cpu_ist_bottom_va(VC)) && (sp < __this_cpu_ist_top_va(VC))); 184 } 185 186 /* 187 * This function handles the case when an NMI is raised in the #VC 188 * exception handler entry code, before the #VC handler has switched off 189 * its IST stack. In this case, the IST entry for #VC must be adjusted, 190 * so that any nested #VC exception will not overwrite the stack 191 * contents of the interrupted #VC handler. 192 * 193 * The IST entry is adjusted unconditionally so that it can be also be 194 * unconditionally adjusted back in __sev_es_ist_exit(). Otherwise a 195 * nested sev_es_ist_exit() call may adjust back the IST entry too 196 * early. 197 * 198 * The __sev_es_ist_enter() and __sev_es_ist_exit() functions always run 199 * on the NMI IST stack, as they are only called from NMI handling code 200 * right now. 201 */ 202 void noinstr __sev_es_ist_enter(struct pt_regs *regs) 203 { 204 unsigned long old_ist, new_ist; 205 206 /* Read old IST entry */ 207 new_ist = old_ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]); 208 209 /* 210 * If NMI happened while on the #VC IST stack, set the new IST 211 * value below regs->sp, so that the interrupted stack frame is 212 * not overwritten by subsequent #VC exceptions. 213 */ 214 if (on_vc_stack(regs)) 215 new_ist = regs->sp; 216 217 /* 218 * Reserve additional 8 bytes and store old IST value so this 219 * adjustment can be unrolled in __sev_es_ist_exit(). 220 */ 221 new_ist -= sizeof(old_ist); 222 *(unsigned long *)new_ist = old_ist; 223 224 /* Set new IST entry */ 225 this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], new_ist); 226 } 227 228 void noinstr __sev_es_ist_exit(void) 229 { 230 unsigned long ist; 231 232 /* Read IST entry */ 233 ist = __this_cpu_read(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC]); 234 235 if (WARN_ON(ist == __this_cpu_ist_top_va(VC))) 236 return; 237 238 /* Read back old IST entry and write it to the TSS */ 239 this_cpu_write(cpu_tss_rw.x86_tss.ist[IST_INDEX_VC], *(unsigned long *)ist); 240 } 241 242 /* 243 * Nothing shall interrupt this code path while holding the per-CPU 244 * GHCB. The backup GHCB is only for NMIs interrupting this path. 245 * 246 * Callers must disable local interrupts around it. 247 */ 248 static noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state) 249 { 250 struct sev_es_runtime_data *data; 251 struct ghcb *ghcb; 252 253 WARN_ON(!irqs_disabled()); 254 255 data = this_cpu_read(runtime_data); 256 ghcb = &data->ghcb_page; 257 258 if (unlikely(data->ghcb_active)) { 259 /* GHCB is already in use - save its contents */ 260 261 if (unlikely(data->backup_ghcb_active)) { 262 /* 263 * Backup-GHCB is also already in use. There is no way 264 * to continue here so just kill the machine. To make 265 * panic() work, mark GHCBs inactive so that messages 266 * can be printed out. 267 */ 268 data->ghcb_active = false; 269 data->backup_ghcb_active = false; 270 271 instrumentation_begin(); 272 panic("Unable to handle #VC exception! GHCB and Backup GHCB are already in use"); 273 instrumentation_end(); 274 } 275 276 /* Mark backup_ghcb active before writing to it */ 277 data->backup_ghcb_active = true; 278 279 state->ghcb = &data->backup_ghcb; 280 281 /* Backup GHCB content */ 282 *state->ghcb = *ghcb; 283 } else { 284 state->ghcb = NULL; 285 data->ghcb_active = true; 286 } 287 288 return ghcb; 289 } 290 291 static inline u64 sev_es_rd_ghcb_msr(void) 292 { 293 return __rdmsr(MSR_AMD64_SEV_ES_GHCB); 294 } 295 296 static __always_inline void sev_es_wr_ghcb_msr(u64 val) 297 { 298 u32 low, high; 299 300 low = (u32)(val); 301 high = (u32)(val >> 32); 302 303 native_wrmsr(MSR_AMD64_SEV_ES_GHCB, low, high); 304 } 305 306 static int vc_fetch_insn_kernel(struct es_em_ctxt *ctxt, 307 unsigned char *buffer) 308 { 309 return copy_from_kernel_nofault(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE); 310 } 311 312 static enum es_result __vc_decode_user_insn(struct es_em_ctxt *ctxt) 313 { 314 char buffer[MAX_INSN_SIZE]; 315 int insn_bytes; 316 317 insn_bytes = insn_fetch_from_user_inatomic(ctxt->regs, buffer); 318 if (insn_bytes == 0) { 319 /* Nothing could be copied */ 320 ctxt->fi.vector = X86_TRAP_PF; 321 ctxt->fi.error_code = X86_PF_INSTR | X86_PF_USER; 322 ctxt->fi.cr2 = ctxt->regs->ip; 323 return ES_EXCEPTION; 324 } else if (insn_bytes == -EINVAL) { 325 /* Effective RIP could not be calculated */ 326 ctxt->fi.vector = X86_TRAP_GP; 327 ctxt->fi.error_code = 0; 328 ctxt->fi.cr2 = 0; 329 return ES_EXCEPTION; 330 } 331 332 if (!insn_decode_from_regs(&ctxt->insn, ctxt->regs, buffer, insn_bytes)) 333 return ES_DECODE_FAILED; 334 335 if (ctxt->insn.immediate.got) 336 return ES_OK; 337 else 338 return ES_DECODE_FAILED; 339 } 340 341 static enum es_result __vc_decode_kern_insn(struct es_em_ctxt *ctxt) 342 { 343 char buffer[MAX_INSN_SIZE]; 344 int res, ret; 345 346 res = vc_fetch_insn_kernel(ctxt, buffer); 347 if (res) { 348 ctxt->fi.vector = X86_TRAP_PF; 349 ctxt->fi.error_code = X86_PF_INSTR; 350 ctxt->fi.cr2 = ctxt->regs->ip; 351 return ES_EXCEPTION; 352 } 353 354 ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64); 355 if (ret < 0) 356 return ES_DECODE_FAILED; 357 else 358 return ES_OK; 359 } 360 361 static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt) 362 { 363 if (user_mode(ctxt->regs)) 364 return __vc_decode_user_insn(ctxt); 365 else 366 return __vc_decode_kern_insn(ctxt); 367 } 368 369 static enum es_result vc_write_mem(struct es_em_ctxt *ctxt, 370 char *dst, char *buf, size_t size) 371 { 372 unsigned long error_code = X86_PF_PROT | X86_PF_WRITE; 373 374 /* 375 * This function uses __put_user() independent of whether kernel or user 376 * memory is accessed. This works fine because __put_user() does no 377 * sanity checks of the pointer being accessed. All that it does is 378 * to report when the access failed. 379 * 380 * Also, this function runs in atomic context, so __put_user() is not 381 * allowed to sleep. The page-fault handler detects that it is running 382 * in atomic context and will not try to take mmap_sem and handle the 383 * fault, so additional pagefault_enable()/disable() calls are not 384 * needed. 385 * 386 * The access can't be done via copy_to_user() here because 387 * vc_write_mem() must not use string instructions to access unsafe 388 * memory. The reason is that MOVS is emulated by the #VC handler by 389 * splitting the move up into a read and a write and taking a nested #VC 390 * exception on whatever of them is the MMIO access. Using string 391 * instructions here would cause infinite nesting. 392 */ 393 switch (size) { 394 case 1: { 395 u8 d1; 396 u8 __user *target = (u8 __user *)dst; 397 398 memcpy(&d1, buf, 1); 399 if (__put_user(d1, target)) 400 goto fault; 401 break; 402 } 403 case 2: { 404 u16 d2; 405 u16 __user *target = (u16 __user *)dst; 406 407 memcpy(&d2, buf, 2); 408 if (__put_user(d2, target)) 409 goto fault; 410 break; 411 } 412 case 4: { 413 u32 d4; 414 u32 __user *target = (u32 __user *)dst; 415 416 memcpy(&d4, buf, 4); 417 if (__put_user(d4, target)) 418 goto fault; 419 break; 420 } 421 case 8: { 422 u64 d8; 423 u64 __user *target = (u64 __user *)dst; 424 425 memcpy(&d8, buf, 8); 426 if (__put_user(d8, target)) 427 goto fault; 428 break; 429 } 430 default: 431 WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size); 432 return ES_UNSUPPORTED; 433 } 434 435 return ES_OK; 436 437 fault: 438 if (user_mode(ctxt->regs)) 439 error_code |= X86_PF_USER; 440 441 ctxt->fi.vector = X86_TRAP_PF; 442 ctxt->fi.error_code = error_code; 443 ctxt->fi.cr2 = (unsigned long)dst; 444 445 return ES_EXCEPTION; 446 } 447 448 static enum es_result vc_read_mem(struct es_em_ctxt *ctxt, 449 char *src, char *buf, size_t size) 450 { 451 unsigned long error_code = X86_PF_PROT; 452 453 /* 454 * This function uses __get_user() independent of whether kernel or user 455 * memory is accessed. This works fine because __get_user() does no 456 * sanity checks of the pointer being accessed. All that it does is 457 * to report when the access failed. 458 * 459 * Also, this function runs in atomic context, so __get_user() is not 460 * allowed to sleep. The page-fault handler detects that it is running 461 * in atomic context and will not try to take mmap_sem and handle the 462 * fault, so additional pagefault_enable()/disable() calls are not 463 * needed. 464 * 465 * The access can't be done via copy_from_user() here because 466 * vc_read_mem() must not use string instructions to access unsafe 467 * memory. The reason is that MOVS is emulated by the #VC handler by 468 * splitting the move up into a read and a write and taking a nested #VC 469 * exception on whatever of them is the MMIO access. Using string 470 * instructions here would cause infinite nesting. 471 */ 472 switch (size) { 473 case 1: { 474 u8 d1; 475 u8 __user *s = (u8 __user *)src; 476 477 if (__get_user(d1, s)) 478 goto fault; 479 memcpy(buf, &d1, 1); 480 break; 481 } 482 case 2: { 483 u16 d2; 484 u16 __user *s = (u16 __user *)src; 485 486 if (__get_user(d2, s)) 487 goto fault; 488 memcpy(buf, &d2, 2); 489 break; 490 } 491 case 4: { 492 u32 d4; 493 u32 __user *s = (u32 __user *)src; 494 495 if (__get_user(d4, s)) 496 goto fault; 497 memcpy(buf, &d4, 4); 498 break; 499 } 500 case 8: { 501 u64 d8; 502 u64 __user *s = (u64 __user *)src; 503 if (__get_user(d8, s)) 504 goto fault; 505 memcpy(buf, &d8, 8); 506 break; 507 } 508 default: 509 WARN_ONCE(1, "%s: Invalid size: %zu\n", __func__, size); 510 return ES_UNSUPPORTED; 511 } 512 513 return ES_OK; 514 515 fault: 516 if (user_mode(ctxt->regs)) 517 error_code |= X86_PF_USER; 518 519 ctxt->fi.vector = X86_TRAP_PF; 520 ctxt->fi.error_code = error_code; 521 ctxt->fi.cr2 = (unsigned long)src; 522 523 return ES_EXCEPTION; 524 } 525 526 static enum es_result vc_slow_virt_to_phys(struct ghcb *ghcb, struct es_em_ctxt *ctxt, 527 unsigned long vaddr, phys_addr_t *paddr) 528 { 529 unsigned long va = (unsigned long)vaddr; 530 unsigned int level; 531 phys_addr_t pa; 532 pgd_t *pgd; 533 pte_t *pte; 534 535 pgd = __va(read_cr3_pa()); 536 pgd = &pgd[pgd_index(va)]; 537 pte = lookup_address_in_pgd(pgd, va, &level); 538 if (!pte) { 539 ctxt->fi.vector = X86_TRAP_PF; 540 ctxt->fi.cr2 = vaddr; 541 ctxt->fi.error_code = 0; 542 543 if (user_mode(ctxt->regs)) 544 ctxt->fi.error_code |= X86_PF_USER; 545 546 return ES_EXCEPTION; 547 } 548 549 if (WARN_ON_ONCE(pte_val(*pte) & _PAGE_ENC)) 550 /* Emulated MMIO to/from encrypted memory not supported */ 551 return ES_UNSUPPORTED; 552 553 pa = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT; 554 pa |= va & ~page_level_mask(level); 555 556 *paddr = pa; 557 558 return ES_OK; 559 } 560 561 static enum es_result vc_ioio_check(struct es_em_ctxt *ctxt, u16 port, size_t size) 562 { 563 BUG_ON(size > 4); 564 565 if (user_mode(ctxt->regs)) { 566 struct thread_struct *t = ¤t->thread; 567 struct io_bitmap *iobm = t->io_bitmap; 568 size_t idx; 569 570 if (!iobm) 571 goto fault; 572 573 for (idx = port; idx < port + size; ++idx) { 574 if (test_bit(idx, iobm->bitmap)) 575 goto fault; 576 } 577 } 578 579 return ES_OK; 580 581 fault: 582 ctxt->fi.vector = X86_TRAP_GP; 583 ctxt->fi.error_code = 0; 584 585 return ES_EXCEPTION; 586 } 587 588 static __always_inline void vc_forward_exception(struct es_em_ctxt *ctxt) 589 { 590 long error_code = ctxt->fi.error_code; 591 int trapnr = ctxt->fi.vector; 592 593 ctxt->regs->orig_ax = ctxt->fi.error_code; 594 595 switch (trapnr) { 596 case X86_TRAP_GP: 597 exc_general_protection(ctxt->regs, error_code); 598 break; 599 case X86_TRAP_UD: 600 exc_invalid_op(ctxt->regs); 601 break; 602 case X86_TRAP_PF: 603 write_cr2(ctxt->fi.cr2); 604 exc_page_fault(ctxt->regs, error_code); 605 break; 606 case X86_TRAP_AC: 607 exc_alignment_check(ctxt->regs, error_code); 608 break; 609 default: 610 pr_emerg("Unsupported exception in #VC instruction emulation - can't continue\n"); 611 BUG(); 612 } 613 } 614 615 /* Include code shared with pre-decompression boot stage */ 616 #include "shared.c" 617 618 static inline struct svsm_ca *svsm_get_caa(void) 619 { 620 /* 621 * Use rIP-relative references when called early in the boot. If 622 * ->use_cas is set, then it is late in the boot and no need 623 * to worry about rIP-relative references. 624 */ 625 if (RIP_REL_REF(sev_cfg).use_cas) 626 return this_cpu_read(svsm_caa); 627 else 628 return RIP_REL_REF(boot_svsm_caa); 629 } 630 631 static u64 svsm_get_caa_pa(void) 632 { 633 /* 634 * Use rIP-relative references when called early in the boot. If 635 * ->use_cas is set, then it is late in the boot and no need 636 * to worry about rIP-relative references. 637 */ 638 if (RIP_REL_REF(sev_cfg).use_cas) 639 return this_cpu_read(svsm_caa_pa); 640 else 641 return RIP_REL_REF(boot_svsm_caa_pa); 642 } 643 644 static noinstr void __sev_put_ghcb(struct ghcb_state *state) 645 { 646 struct sev_es_runtime_data *data; 647 struct ghcb *ghcb; 648 649 WARN_ON(!irqs_disabled()); 650 651 data = this_cpu_read(runtime_data); 652 ghcb = &data->ghcb_page; 653 654 if (state->ghcb) { 655 /* Restore GHCB from Backup */ 656 *ghcb = *state->ghcb; 657 data->backup_ghcb_active = false; 658 state->ghcb = NULL; 659 } else { 660 /* 661 * Invalidate the GHCB so a VMGEXIT instruction issued 662 * from userspace won't appear to be valid. 663 */ 664 vc_ghcb_invalidate(ghcb); 665 data->ghcb_active = false; 666 } 667 } 668 669 static int svsm_perform_call_protocol(struct svsm_call *call) 670 { 671 struct ghcb_state state; 672 unsigned long flags; 673 struct ghcb *ghcb; 674 int ret; 675 676 /* 677 * This can be called very early in the boot, use native functions in 678 * order to avoid paravirt issues. 679 */ 680 flags = native_local_irq_save(); 681 682 /* 683 * Use rip-relative references when called early in the boot. If 684 * ghcbs_initialized is set, then it is late in the boot and no need 685 * to worry about rip-relative references in called functions. 686 */ 687 if (RIP_REL_REF(sev_cfg).ghcbs_initialized) 688 ghcb = __sev_get_ghcb(&state); 689 else if (RIP_REL_REF(boot_ghcb)) 690 ghcb = RIP_REL_REF(boot_ghcb); 691 else 692 ghcb = NULL; 693 694 do { 695 ret = ghcb ? svsm_perform_ghcb_protocol(ghcb, call) 696 : svsm_perform_msr_protocol(call); 697 } while (ret == -EAGAIN); 698 699 if (RIP_REL_REF(sev_cfg).ghcbs_initialized) 700 __sev_put_ghcb(&state); 701 702 native_local_irq_restore(flags); 703 704 return ret; 705 } 706 707 void noinstr __sev_es_nmi_complete(void) 708 { 709 struct ghcb_state state; 710 struct ghcb *ghcb; 711 712 ghcb = __sev_get_ghcb(&state); 713 714 vc_ghcb_invalidate(ghcb); 715 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_NMI_COMPLETE); 716 ghcb_set_sw_exit_info_1(ghcb, 0); 717 ghcb_set_sw_exit_info_2(ghcb, 0); 718 719 sev_es_wr_ghcb_msr(__pa_nodebug(ghcb)); 720 VMGEXIT(); 721 722 __sev_put_ghcb(&state); 723 } 724 725 static u64 __init get_secrets_page(void) 726 { 727 u64 pa_data = boot_params.cc_blob_address; 728 struct cc_blob_sev_info info; 729 void *map; 730 731 /* 732 * The CC blob contains the address of the secrets page, check if the 733 * blob is present. 734 */ 735 if (!pa_data) 736 return 0; 737 738 map = early_memremap(pa_data, sizeof(info)); 739 if (!map) { 740 pr_err("Unable to locate SNP secrets page: failed to map the Confidential Computing blob.\n"); 741 return 0; 742 } 743 memcpy(&info, map, sizeof(info)); 744 early_memunmap(map, sizeof(info)); 745 746 /* smoke-test the secrets page passed */ 747 if (!info.secrets_phys || info.secrets_len != PAGE_SIZE) 748 return 0; 749 750 return info.secrets_phys; 751 } 752 753 static u64 __init get_snp_jump_table_addr(void) 754 { 755 struct snp_secrets_page *secrets; 756 void __iomem *mem; 757 u64 pa, addr; 758 759 pa = get_secrets_page(); 760 if (!pa) 761 return 0; 762 763 mem = ioremap_encrypted(pa, PAGE_SIZE); 764 if (!mem) { 765 pr_err("Unable to locate AP jump table address: failed to map the SNP secrets page.\n"); 766 return 0; 767 } 768 769 secrets = (__force struct snp_secrets_page *)mem; 770 771 addr = secrets->os_area.ap_jump_table_pa; 772 iounmap(mem); 773 774 return addr; 775 } 776 777 static u64 __init get_jump_table_addr(void) 778 { 779 struct ghcb_state state; 780 unsigned long flags; 781 struct ghcb *ghcb; 782 u64 ret = 0; 783 784 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 785 return get_snp_jump_table_addr(); 786 787 local_irq_save(flags); 788 789 ghcb = __sev_get_ghcb(&state); 790 791 vc_ghcb_invalidate(ghcb); 792 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_JUMP_TABLE); 793 ghcb_set_sw_exit_info_1(ghcb, SVM_VMGEXIT_GET_AP_JUMP_TABLE); 794 ghcb_set_sw_exit_info_2(ghcb, 0); 795 796 sev_es_wr_ghcb_msr(__pa(ghcb)); 797 VMGEXIT(); 798 799 if (ghcb_sw_exit_info_1_is_valid(ghcb) && 800 ghcb_sw_exit_info_2_is_valid(ghcb)) 801 ret = ghcb->save.sw_exit_info_2; 802 803 __sev_put_ghcb(&state); 804 805 local_irq_restore(flags); 806 807 return ret; 808 } 809 810 static void __head 811 early_set_pages_state(unsigned long vaddr, unsigned long paddr, 812 unsigned long npages, enum psc_op op) 813 { 814 unsigned long paddr_end; 815 u64 val; 816 817 vaddr = vaddr & PAGE_MASK; 818 819 paddr = paddr & PAGE_MASK; 820 paddr_end = paddr + (npages << PAGE_SHIFT); 821 822 while (paddr < paddr_end) { 823 /* Page validation must be rescinded before changing to shared */ 824 if (op == SNP_PAGE_STATE_SHARED) 825 pvalidate_4k_page(vaddr, paddr, false); 826 827 /* 828 * Use the MSR protocol because this function can be called before 829 * the GHCB is established. 830 */ 831 sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op)); 832 VMGEXIT(); 833 834 val = sev_es_rd_ghcb_msr(); 835 836 if (WARN(GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP, 837 "Wrong PSC response code: 0x%x\n", 838 (unsigned int)GHCB_RESP_CODE(val))) 839 goto e_term; 840 841 if (WARN(GHCB_MSR_PSC_RESP_VAL(val), 842 "Failed to change page state to '%s' paddr 0x%lx error 0x%llx\n", 843 op == SNP_PAGE_STATE_PRIVATE ? "private" : "shared", 844 paddr, GHCB_MSR_PSC_RESP_VAL(val))) 845 goto e_term; 846 847 /* Page validation must be performed after changing to private */ 848 if (op == SNP_PAGE_STATE_PRIVATE) 849 pvalidate_4k_page(vaddr, paddr, true); 850 851 vaddr += PAGE_SIZE; 852 paddr += PAGE_SIZE; 853 } 854 855 return; 856 857 e_term: 858 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC); 859 } 860 861 void __head early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr, 862 unsigned long npages) 863 { 864 /* 865 * This can be invoked in early boot while running identity mapped, so 866 * use an open coded check for SNP instead of using cc_platform_has(). 867 * This eliminates worries about jump tables or checking boot_cpu_data 868 * in the cc_platform_has() function. 869 */ 870 if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED)) 871 return; 872 873 /* 874 * Ask the hypervisor to mark the memory pages as private in the RMP 875 * table. 876 */ 877 early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_PRIVATE); 878 } 879 880 void __init early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr, 881 unsigned long npages) 882 { 883 /* 884 * This can be invoked in early boot while running identity mapped, so 885 * use an open coded check for SNP instead of using cc_platform_has(). 886 * This eliminates worries about jump tables or checking boot_cpu_data 887 * in the cc_platform_has() function. 888 */ 889 if (!(RIP_REL_REF(sev_status) & MSR_AMD64_SEV_SNP_ENABLED)) 890 return; 891 892 /* Ask hypervisor to mark the memory pages shared in the RMP table. */ 893 early_set_pages_state(vaddr, paddr, npages, SNP_PAGE_STATE_SHARED); 894 } 895 896 static unsigned long __set_pages_state(struct snp_psc_desc *data, unsigned long vaddr, 897 unsigned long vaddr_end, int op) 898 { 899 struct ghcb_state state; 900 bool use_large_entry; 901 struct psc_hdr *hdr; 902 struct psc_entry *e; 903 unsigned long flags; 904 unsigned long pfn; 905 struct ghcb *ghcb; 906 int i; 907 908 hdr = &data->hdr; 909 e = data->entries; 910 911 memset(data, 0, sizeof(*data)); 912 i = 0; 913 914 while (vaddr < vaddr_end && i < ARRAY_SIZE(data->entries)) { 915 hdr->end_entry = i; 916 917 if (is_vmalloc_addr((void *)vaddr)) { 918 pfn = vmalloc_to_pfn((void *)vaddr); 919 use_large_entry = false; 920 } else { 921 pfn = __pa(vaddr) >> PAGE_SHIFT; 922 use_large_entry = true; 923 } 924 925 e->gfn = pfn; 926 e->operation = op; 927 928 if (use_large_entry && IS_ALIGNED(vaddr, PMD_SIZE) && 929 (vaddr_end - vaddr) >= PMD_SIZE) { 930 e->pagesize = RMP_PG_SIZE_2M; 931 vaddr += PMD_SIZE; 932 } else { 933 e->pagesize = RMP_PG_SIZE_4K; 934 vaddr += PAGE_SIZE; 935 } 936 937 e++; 938 i++; 939 } 940 941 /* Page validation must be rescinded before changing to shared */ 942 if (op == SNP_PAGE_STATE_SHARED) 943 pvalidate_pages(data); 944 945 local_irq_save(flags); 946 947 if (sev_cfg.ghcbs_initialized) 948 ghcb = __sev_get_ghcb(&state); 949 else 950 ghcb = boot_ghcb; 951 952 /* Invoke the hypervisor to perform the page state changes */ 953 if (!ghcb || vmgexit_psc(ghcb, data)) 954 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC); 955 956 if (sev_cfg.ghcbs_initialized) 957 __sev_put_ghcb(&state); 958 959 local_irq_restore(flags); 960 961 /* Page validation must be performed after changing to private */ 962 if (op == SNP_PAGE_STATE_PRIVATE) 963 pvalidate_pages(data); 964 965 return vaddr; 966 } 967 968 static void set_pages_state(unsigned long vaddr, unsigned long npages, int op) 969 { 970 struct snp_psc_desc desc; 971 unsigned long vaddr_end; 972 973 /* Use the MSR protocol when a GHCB is not available. */ 974 if (!boot_ghcb) 975 return early_set_pages_state(vaddr, __pa(vaddr), npages, op); 976 977 vaddr = vaddr & PAGE_MASK; 978 vaddr_end = vaddr + (npages << PAGE_SHIFT); 979 980 while (vaddr < vaddr_end) 981 vaddr = __set_pages_state(&desc, vaddr, vaddr_end, op); 982 } 983 984 void snp_set_memory_shared(unsigned long vaddr, unsigned long npages) 985 { 986 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 987 return; 988 989 set_pages_state(vaddr, npages, SNP_PAGE_STATE_SHARED); 990 } 991 992 void snp_set_memory_private(unsigned long vaddr, unsigned long npages) 993 { 994 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 995 return; 996 997 set_pages_state(vaddr, npages, SNP_PAGE_STATE_PRIVATE); 998 } 999 1000 void snp_accept_memory(phys_addr_t start, phys_addr_t end) 1001 { 1002 unsigned long vaddr, npages; 1003 1004 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 1005 return; 1006 1007 vaddr = (unsigned long)__va(start); 1008 npages = (end - start) >> PAGE_SHIFT; 1009 1010 set_pages_state(vaddr, npages, SNP_PAGE_STATE_PRIVATE); 1011 } 1012 1013 static int snp_set_vmsa(void *va, void *caa, int apic_id, bool make_vmsa) 1014 { 1015 int ret; 1016 1017 if (snp_vmpl) { 1018 struct svsm_call call = {}; 1019 unsigned long flags; 1020 1021 local_irq_save(flags); 1022 1023 call.caa = this_cpu_read(svsm_caa); 1024 call.rcx = __pa(va); 1025 1026 if (make_vmsa) { 1027 /* Protocol 0, Call ID 2 */ 1028 call.rax = SVSM_CORE_CALL(SVSM_CORE_CREATE_VCPU); 1029 call.rdx = __pa(caa); 1030 call.r8 = apic_id; 1031 } else { 1032 /* Protocol 0, Call ID 3 */ 1033 call.rax = SVSM_CORE_CALL(SVSM_CORE_DELETE_VCPU); 1034 } 1035 1036 ret = svsm_perform_call_protocol(&call); 1037 1038 local_irq_restore(flags); 1039 } else { 1040 /* 1041 * If the kernel runs at VMPL0, it can change the VMSA 1042 * bit for a page using the RMPADJUST instruction. 1043 * However, for the instruction to succeed it must 1044 * target the permissions of a lesser privileged (higher 1045 * numbered) VMPL level, so use VMPL1. 1046 */ 1047 u64 attrs = 1; 1048 1049 if (make_vmsa) 1050 attrs |= RMPADJUST_VMSA_PAGE_BIT; 1051 1052 ret = rmpadjust((unsigned long)va, RMP_PG_SIZE_4K, attrs); 1053 } 1054 1055 return ret; 1056 } 1057 1058 #define __ATTR_BASE (SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK) 1059 #define INIT_CS_ATTRIBS (__ATTR_BASE | SVM_SELECTOR_READ_MASK | SVM_SELECTOR_CODE_MASK) 1060 #define INIT_DS_ATTRIBS (__ATTR_BASE | SVM_SELECTOR_WRITE_MASK) 1061 1062 #define INIT_LDTR_ATTRIBS (SVM_SELECTOR_P_MASK | 2) 1063 #define INIT_TR_ATTRIBS (SVM_SELECTOR_P_MASK | 3) 1064 1065 static void *snp_alloc_vmsa_page(int cpu) 1066 { 1067 struct page *p; 1068 1069 /* 1070 * Allocate VMSA page to work around the SNP erratum where the CPU will 1071 * incorrectly signal an RMP violation #PF if a large page (2MB or 1GB) 1072 * collides with the RMP entry of VMSA page. The recommended workaround 1073 * is to not use a large page. 1074 * 1075 * Allocate an 8k page which is also 8k-aligned. 1076 */ 1077 p = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL_ACCOUNT | __GFP_ZERO, 1); 1078 if (!p) 1079 return NULL; 1080 1081 split_page(p, 1); 1082 1083 /* Free the first 4k. This page may be 2M/1G aligned and cannot be used. */ 1084 __free_page(p); 1085 1086 return page_address(p + 1); 1087 } 1088 1089 static void snp_cleanup_vmsa(struct sev_es_save_area *vmsa, int apic_id) 1090 { 1091 int err; 1092 1093 err = snp_set_vmsa(vmsa, NULL, apic_id, false); 1094 if (err) 1095 pr_err("clear VMSA page failed (%u), leaking page\n", err); 1096 else 1097 free_page((unsigned long)vmsa); 1098 } 1099 1100 static int wakeup_cpu_via_vmgexit(u32 apic_id, unsigned long start_ip) 1101 { 1102 struct sev_es_save_area *cur_vmsa, *vmsa; 1103 struct ghcb_state state; 1104 struct svsm_ca *caa; 1105 unsigned long flags; 1106 struct ghcb *ghcb; 1107 u8 sipi_vector; 1108 int cpu, ret; 1109 u64 cr4; 1110 1111 /* 1112 * The hypervisor SNP feature support check has happened earlier, just check 1113 * the AP_CREATION one here. 1114 */ 1115 if (!(sev_hv_features & GHCB_HV_FT_SNP_AP_CREATION)) 1116 return -EOPNOTSUPP; 1117 1118 /* 1119 * Verify the desired start IP against the known trampoline start IP 1120 * to catch any future new trampolines that may be introduced that 1121 * would require a new protected guest entry point. 1122 */ 1123 if (WARN_ONCE(start_ip != real_mode_header->trampoline_start, 1124 "Unsupported SNP start_ip: %lx\n", start_ip)) 1125 return -EINVAL; 1126 1127 /* Override start_ip with known protected guest start IP */ 1128 start_ip = real_mode_header->sev_es_trampoline_start; 1129 1130 /* Find the logical CPU for the APIC ID */ 1131 for_each_present_cpu(cpu) { 1132 if (arch_match_cpu_phys_id(cpu, apic_id)) 1133 break; 1134 } 1135 if (cpu >= nr_cpu_ids) 1136 return -EINVAL; 1137 1138 cur_vmsa = per_cpu(sev_vmsa, cpu); 1139 1140 /* 1141 * A new VMSA is created each time because there is no guarantee that 1142 * the current VMSA is the kernels or that the vCPU is not running. If 1143 * an attempt was done to use the current VMSA with a running vCPU, a 1144 * #VMEXIT of that vCPU would wipe out all of the settings being done 1145 * here. 1146 */ 1147 vmsa = (struct sev_es_save_area *)snp_alloc_vmsa_page(cpu); 1148 if (!vmsa) 1149 return -ENOMEM; 1150 1151 /* If an SVSM is present, the SVSM per-CPU CAA will be !NULL */ 1152 caa = per_cpu(svsm_caa, cpu); 1153 1154 /* CR4 should maintain the MCE value */ 1155 cr4 = native_read_cr4() & X86_CR4_MCE; 1156 1157 /* Set the CS value based on the start_ip converted to a SIPI vector */ 1158 sipi_vector = (start_ip >> 12); 1159 vmsa->cs.base = sipi_vector << 12; 1160 vmsa->cs.limit = AP_INIT_CS_LIMIT; 1161 vmsa->cs.attrib = INIT_CS_ATTRIBS; 1162 vmsa->cs.selector = sipi_vector << 8; 1163 1164 /* Set the RIP value based on start_ip */ 1165 vmsa->rip = start_ip & 0xfff; 1166 1167 /* Set AP INIT defaults as documented in the APM */ 1168 vmsa->ds.limit = AP_INIT_DS_LIMIT; 1169 vmsa->ds.attrib = INIT_DS_ATTRIBS; 1170 vmsa->es = vmsa->ds; 1171 vmsa->fs = vmsa->ds; 1172 vmsa->gs = vmsa->ds; 1173 vmsa->ss = vmsa->ds; 1174 1175 vmsa->gdtr.limit = AP_INIT_GDTR_LIMIT; 1176 vmsa->ldtr.limit = AP_INIT_LDTR_LIMIT; 1177 vmsa->ldtr.attrib = INIT_LDTR_ATTRIBS; 1178 vmsa->idtr.limit = AP_INIT_IDTR_LIMIT; 1179 vmsa->tr.limit = AP_INIT_TR_LIMIT; 1180 vmsa->tr.attrib = INIT_TR_ATTRIBS; 1181 1182 vmsa->cr4 = cr4; 1183 vmsa->cr0 = AP_INIT_CR0_DEFAULT; 1184 vmsa->dr7 = DR7_RESET_VALUE; 1185 vmsa->dr6 = AP_INIT_DR6_DEFAULT; 1186 vmsa->rflags = AP_INIT_RFLAGS_DEFAULT; 1187 vmsa->g_pat = AP_INIT_GPAT_DEFAULT; 1188 vmsa->xcr0 = AP_INIT_XCR0_DEFAULT; 1189 vmsa->mxcsr = AP_INIT_MXCSR_DEFAULT; 1190 vmsa->x87_ftw = AP_INIT_X87_FTW_DEFAULT; 1191 vmsa->x87_fcw = AP_INIT_X87_FCW_DEFAULT; 1192 1193 /* SVME must be set. */ 1194 vmsa->efer = EFER_SVME; 1195 1196 /* 1197 * Set the SNP-specific fields for this VMSA: 1198 * VMPL level 1199 * SEV_FEATURES (matches the SEV STATUS MSR right shifted 2 bits) 1200 */ 1201 vmsa->vmpl = snp_vmpl; 1202 vmsa->sev_features = sev_status >> 2; 1203 1204 /* Switch the page over to a VMSA page now that it is initialized */ 1205 ret = snp_set_vmsa(vmsa, caa, apic_id, true); 1206 if (ret) { 1207 pr_err("set VMSA page failed (%u)\n", ret); 1208 free_page((unsigned long)vmsa); 1209 1210 return -EINVAL; 1211 } 1212 1213 /* Issue VMGEXIT AP Creation NAE event */ 1214 local_irq_save(flags); 1215 1216 ghcb = __sev_get_ghcb(&state); 1217 1218 vc_ghcb_invalidate(ghcb); 1219 ghcb_set_rax(ghcb, vmsa->sev_features); 1220 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_CREATION); 1221 ghcb_set_sw_exit_info_1(ghcb, 1222 ((u64)apic_id << 32) | 1223 ((u64)snp_vmpl << 16) | 1224 SVM_VMGEXIT_AP_CREATE); 1225 ghcb_set_sw_exit_info_2(ghcb, __pa(vmsa)); 1226 1227 sev_es_wr_ghcb_msr(__pa(ghcb)); 1228 VMGEXIT(); 1229 1230 if (!ghcb_sw_exit_info_1_is_valid(ghcb) || 1231 lower_32_bits(ghcb->save.sw_exit_info_1)) { 1232 pr_err("SNP AP Creation error\n"); 1233 ret = -EINVAL; 1234 } 1235 1236 __sev_put_ghcb(&state); 1237 1238 local_irq_restore(flags); 1239 1240 /* Perform cleanup if there was an error */ 1241 if (ret) { 1242 snp_cleanup_vmsa(vmsa, apic_id); 1243 vmsa = NULL; 1244 } 1245 1246 /* Free up any previous VMSA page */ 1247 if (cur_vmsa) 1248 snp_cleanup_vmsa(cur_vmsa, apic_id); 1249 1250 /* Record the current VMSA page */ 1251 per_cpu(sev_vmsa, cpu) = vmsa; 1252 1253 return ret; 1254 } 1255 1256 void __init snp_set_wakeup_secondary_cpu(void) 1257 { 1258 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 1259 return; 1260 1261 /* 1262 * Always set this override if SNP is enabled. This makes it the 1263 * required method to start APs under SNP. If the hypervisor does 1264 * not support AP creation, then no APs will be started. 1265 */ 1266 apic_update_callback(wakeup_secondary_cpu, wakeup_cpu_via_vmgexit); 1267 } 1268 1269 int __init sev_es_setup_ap_jump_table(struct real_mode_header *rmh) 1270 { 1271 u16 startup_cs, startup_ip; 1272 phys_addr_t jump_table_pa; 1273 u64 jump_table_addr; 1274 u16 __iomem *jump_table; 1275 1276 jump_table_addr = get_jump_table_addr(); 1277 1278 /* On UP guests there is no jump table so this is not a failure */ 1279 if (!jump_table_addr) 1280 return 0; 1281 1282 /* Check if AP Jump Table is page-aligned */ 1283 if (jump_table_addr & ~PAGE_MASK) 1284 return -EINVAL; 1285 1286 jump_table_pa = jump_table_addr & PAGE_MASK; 1287 1288 startup_cs = (u16)(rmh->trampoline_start >> 4); 1289 startup_ip = (u16)(rmh->sev_es_trampoline_start - 1290 rmh->trampoline_start); 1291 1292 jump_table = ioremap_encrypted(jump_table_pa, PAGE_SIZE); 1293 if (!jump_table) 1294 return -EIO; 1295 1296 writew(startup_ip, &jump_table[0]); 1297 writew(startup_cs, &jump_table[1]); 1298 1299 iounmap(jump_table); 1300 1301 return 0; 1302 } 1303 1304 /* 1305 * This is needed by the OVMF UEFI firmware which will use whatever it finds in 1306 * the GHCB MSR as its GHCB to talk to the hypervisor. So make sure the per-cpu 1307 * runtime GHCBs used by the kernel are also mapped in the EFI page-table. 1308 */ 1309 int __init sev_es_efi_map_ghcbs(pgd_t *pgd) 1310 { 1311 struct sev_es_runtime_data *data; 1312 unsigned long address, pflags; 1313 int cpu; 1314 u64 pfn; 1315 1316 if (!cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) 1317 return 0; 1318 1319 pflags = _PAGE_NX | _PAGE_RW; 1320 1321 for_each_possible_cpu(cpu) { 1322 data = per_cpu(runtime_data, cpu); 1323 1324 address = __pa(&data->ghcb_page); 1325 pfn = address >> PAGE_SHIFT; 1326 1327 if (kernel_map_pages_in_pgd(pgd, pfn, address, 1, pflags)) 1328 return 1; 1329 } 1330 1331 return 0; 1332 } 1333 1334 static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 1335 { 1336 struct pt_regs *regs = ctxt->regs; 1337 enum es_result ret; 1338 u64 exit_info_1; 1339 1340 /* Is it a WRMSR? */ 1341 exit_info_1 = (ctxt->insn.opcode.bytes[1] == 0x30) ? 1 : 0; 1342 1343 if (regs->cx == MSR_SVSM_CAA) { 1344 /* Writes to the SVSM CAA msr are ignored */ 1345 if (exit_info_1) 1346 return ES_OK; 1347 1348 regs->ax = lower_32_bits(this_cpu_read(svsm_caa_pa)); 1349 regs->dx = upper_32_bits(this_cpu_read(svsm_caa_pa)); 1350 1351 return ES_OK; 1352 } 1353 1354 ghcb_set_rcx(ghcb, regs->cx); 1355 if (exit_info_1) { 1356 ghcb_set_rax(ghcb, regs->ax); 1357 ghcb_set_rdx(ghcb, regs->dx); 1358 } 1359 1360 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, exit_info_1, 0); 1361 1362 if ((ret == ES_OK) && (!exit_info_1)) { 1363 regs->ax = ghcb->save.rax; 1364 regs->dx = ghcb->save.rdx; 1365 } 1366 1367 return ret; 1368 } 1369 1370 static void snp_register_per_cpu_ghcb(void) 1371 { 1372 struct sev_es_runtime_data *data; 1373 struct ghcb *ghcb; 1374 1375 data = this_cpu_read(runtime_data); 1376 ghcb = &data->ghcb_page; 1377 1378 snp_register_ghcb_early(__pa(ghcb)); 1379 } 1380 1381 void setup_ghcb(void) 1382 { 1383 if (!cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) 1384 return; 1385 1386 /* 1387 * Check whether the runtime #VC exception handler is active. It uses 1388 * the per-CPU GHCB page which is set up by sev_es_init_vc_handling(). 1389 * 1390 * If SNP is active, register the per-CPU GHCB page so that the runtime 1391 * exception handler can use it. 1392 */ 1393 if (initial_vc_handler == (unsigned long)kernel_exc_vmm_communication) { 1394 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 1395 snp_register_per_cpu_ghcb(); 1396 1397 sev_cfg.ghcbs_initialized = true; 1398 1399 return; 1400 } 1401 1402 /* 1403 * Make sure the hypervisor talks a supported protocol. 1404 * This gets called only in the BSP boot phase. 1405 */ 1406 if (!sev_es_negotiate_protocol()) 1407 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); 1408 1409 /* 1410 * Clear the boot_ghcb. The first exception comes in before the bss 1411 * section is cleared. 1412 */ 1413 memset(&boot_ghcb_page, 0, PAGE_SIZE); 1414 1415 /* Alright - Make the boot-ghcb public */ 1416 boot_ghcb = &boot_ghcb_page; 1417 1418 /* SNP guest requires that GHCB GPA must be registered. */ 1419 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 1420 snp_register_ghcb_early(__pa(&boot_ghcb_page)); 1421 } 1422 1423 #ifdef CONFIG_HOTPLUG_CPU 1424 static void sev_es_ap_hlt_loop(void) 1425 { 1426 struct ghcb_state state; 1427 struct ghcb *ghcb; 1428 1429 ghcb = __sev_get_ghcb(&state); 1430 1431 while (true) { 1432 vc_ghcb_invalidate(ghcb); 1433 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_HLT_LOOP); 1434 ghcb_set_sw_exit_info_1(ghcb, 0); 1435 ghcb_set_sw_exit_info_2(ghcb, 0); 1436 1437 sev_es_wr_ghcb_msr(__pa(ghcb)); 1438 VMGEXIT(); 1439 1440 /* Wakeup signal? */ 1441 if (ghcb_sw_exit_info_2_is_valid(ghcb) && 1442 ghcb->save.sw_exit_info_2) 1443 break; 1444 } 1445 1446 __sev_put_ghcb(&state); 1447 } 1448 1449 /* 1450 * Play_dead handler when running under SEV-ES. This is needed because 1451 * the hypervisor can't deliver an SIPI request to restart the AP. 1452 * Instead the kernel has to issue a VMGEXIT to halt the VCPU until the 1453 * hypervisor wakes it up again. 1454 */ 1455 static void sev_es_play_dead(void) 1456 { 1457 play_dead_common(); 1458 1459 /* IRQs now disabled */ 1460 1461 sev_es_ap_hlt_loop(); 1462 1463 /* 1464 * If we get here, the VCPU was woken up again. Jump to CPU 1465 * startup code to get it back online. 1466 */ 1467 soft_restart_cpu(); 1468 } 1469 #else /* CONFIG_HOTPLUG_CPU */ 1470 #define sev_es_play_dead native_play_dead 1471 #endif /* CONFIG_HOTPLUG_CPU */ 1472 1473 #ifdef CONFIG_SMP 1474 static void __init sev_es_setup_play_dead(void) 1475 { 1476 smp_ops.play_dead = sev_es_play_dead; 1477 } 1478 #else 1479 static inline void sev_es_setup_play_dead(void) { } 1480 #endif 1481 1482 static void __init alloc_runtime_data(int cpu) 1483 { 1484 struct sev_es_runtime_data *data; 1485 1486 data = memblock_alloc_node(sizeof(*data), PAGE_SIZE, cpu_to_node(cpu)); 1487 if (!data) 1488 panic("Can't allocate SEV-ES runtime data"); 1489 1490 per_cpu(runtime_data, cpu) = data; 1491 1492 if (snp_vmpl) { 1493 struct svsm_ca *caa; 1494 1495 /* Allocate the SVSM CA page if an SVSM is present */ 1496 caa = memblock_alloc(sizeof(*caa), PAGE_SIZE); 1497 if (!caa) 1498 panic("Can't allocate SVSM CA page\n"); 1499 1500 per_cpu(svsm_caa, cpu) = caa; 1501 per_cpu(svsm_caa_pa, cpu) = __pa(caa); 1502 } 1503 } 1504 1505 static void __init init_ghcb(int cpu) 1506 { 1507 struct sev_es_runtime_data *data; 1508 int err; 1509 1510 data = per_cpu(runtime_data, cpu); 1511 1512 err = early_set_memory_decrypted((unsigned long)&data->ghcb_page, 1513 sizeof(data->ghcb_page)); 1514 if (err) 1515 panic("Can't map GHCBs unencrypted"); 1516 1517 memset(&data->ghcb_page, 0, sizeof(data->ghcb_page)); 1518 1519 data->ghcb_active = false; 1520 data->backup_ghcb_active = false; 1521 } 1522 1523 void __init sev_es_init_vc_handling(void) 1524 { 1525 int cpu; 1526 1527 BUILD_BUG_ON(offsetof(struct sev_es_runtime_data, ghcb_page) % PAGE_SIZE); 1528 1529 if (!cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) 1530 return; 1531 1532 if (!sev_es_check_cpu_features()) 1533 panic("SEV-ES CPU Features missing"); 1534 1535 /* 1536 * SNP is supported in v2 of the GHCB spec which mandates support for HV 1537 * features. 1538 */ 1539 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) { 1540 sev_hv_features = get_hv_features(); 1541 1542 if (!(sev_hv_features & GHCB_HV_FT_SNP)) 1543 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED); 1544 } 1545 1546 /* Initialize per-cpu GHCB pages */ 1547 for_each_possible_cpu(cpu) { 1548 alloc_runtime_data(cpu); 1549 init_ghcb(cpu); 1550 } 1551 1552 /* If running under an SVSM, switch to the per-cpu CA */ 1553 if (snp_vmpl) { 1554 struct svsm_call call = {}; 1555 unsigned long flags; 1556 int ret; 1557 1558 local_irq_save(flags); 1559 1560 /* 1561 * SVSM_CORE_REMAP_CA call: 1562 * RAX = 0 (Protocol=0, CallID=0) 1563 * RCX = New CA GPA 1564 */ 1565 call.caa = svsm_get_caa(); 1566 call.rax = SVSM_CORE_CALL(SVSM_CORE_REMAP_CA); 1567 call.rcx = this_cpu_read(svsm_caa_pa); 1568 ret = svsm_perform_call_protocol(&call); 1569 if (ret) 1570 panic("Can't remap the SVSM CA, ret=%d, rax_out=0x%llx\n", 1571 ret, call.rax_out); 1572 1573 sev_cfg.use_cas = true; 1574 1575 local_irq_restore(flags); 1576 } 1577 1578 sev_es_setup_play_dead(); 1579 1580 /* Secondary CPUs use the runtime #VC handler */ 1581 initial_vc_handler = (unsigned long)kernel_exc_vmm_communication; 1582 } 1583 1584 static void __init vc_early_forward_exception(struct es_em_ctxt *ctxt) 1585 { 1586 int trapnr = ctxt->fi.vector; 1587 1588 if (trapnr == X86_TRAP_PF) 1589 native_write_cr2(ctxt->fi.cr2); 1590 1591 ctxt->regs->orig_ax = ctxt->fi.error_code; 1592 do_early_exception(ctxt->regs, trapnr); 1593 } 1594 1595 static long *vc_insn_get_rm(struct es_em_ctxt *ctxt) 1596 { 1597 long *reg_array; 1598 int offset; 1599 1600 reg_array = (long *)ctxt->regs; 1601 offset = insn_get_modrm_rm_off(&ctxt->insn, ctxt->regs); 1602 1603 if (offset < 0) 1604 return NULL; 1605 1606 offset /= sizeof(long); 1607 1608 return reg_array + offset; 1609 } 1610 static enum es_result vc_do_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt, 1611 unsigned int bytes, bool read) 1612 { 1613 u64 exit_code, exit_info_1, exit_info_2; 1614 unsigned long ghcb_pa = __pa(ghcb); 1615 enum es_result res; 1616 phys_addr_t paddr; 1617 void __user *ref; 1618 1619 ref = insn_get_addr_ref(&ctxt->insn, ctxt->regs); 1620 if (ref == (void __user *)-1L) 1621 return ES_UNSUPPORTED; 1622 1623 exit_code = read ? SVM_VMGEXIT_MMIO_READ : SVM_VMGEXIT_MMIO_WRITE; 1624 1625 res = vc_slow_virt_to_phys(ghcb, ctxt, (unsigned long)ref, &paddr); 1626 if (res != ES_OK) { 1627 if (res == ES_EXCEPTION && !read) 1628 ctxt->fi.error_code |= X86_PF_WRITE; 1629 1630 return res; 1631 } 1632 1633 exit_info_1 = paddr; 1634 /* Can never be greater than 8 */ 1635 exit_info_2 = bytes; 1636 1637 ghcb_set_sw_scratch(ghcb, ghcb_pa + offsetof(struct ghcb, shared_buffer)); 1638 1639 return sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, exit_info_1, exit_info_2); 1640 } 1641 1642 /* 1643 * The MOVS instruction has two memory operands, which raises the 1644 * problem that it is not known whether the access to the source or the 1645 * destination caused the #VC exception (and hence whether an MMIO read 1646 * or write operation needs to be emulated). 1647 * 1648 * Instead of playing games with walking page-tables and trying to guess 1649 * whether the source or destination is an MMIO range, split the move 1650 * into two operations, a read and a write with only one memory operand. 1651 * This will cause a nested #VC exception on the MMIO address which can 1652 * then be handled. 1653 * 1654 * This implementation has the benefit that it also supports MOVS where 1655 * source _and_ destination are MMIO regions. 1656 * 1657 * It will slow MOVS on MMIO down a lot, but in SEV-ES guests it is a 1658 * rare operation. If it turns out to be a performance problem the split 1659 * operations can be moved to memcpy_fromio() and memcpy_toio(). 1660 */ 1661 static enum es_result vc_handle_mmio_movs(struct es_em_ctxt *ctxt, 1662 unsigned int bytes) 1663 { 1664 unsigned long ds_base, es_base; 1665 unsigned char *src, *dst; 1666 unsigned char buffer[8]; 1667 enum es_result ret; 1668 bool rep; 1669 int off; 1670 1671 ds_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_DS); 1672 es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES); 1673 1674 if (ds_base == -1L || es_base == -1L) { 1675 ctxt->fi.vector = X86_TRAP_GP; 1676 ctxt->fi.error_code = 0; 1677 return ES_EXCEPTION; 1678 } 1679 1680 src = ds_base + (unsigned char *)ctxt->regs->si; 1681 dst = es_base + (unsigned char *)ctxt->regs->di; 1682 1683 ret = vc_read_mem(ctxt, src, buffer, bytes); 1684 if (ret != ES_OK) 1685 return ret; 1686 1687 ret = vc_write_mem(ctxt, dst, buffer, bytes); 1688 if (ret != ES_OK) 1689 return ret; 1690 1691 if (ctxt->regs->flags & X86_EFLAGS_DF) 1692 off = -bytes; 1693 else 1694 off = bytes; 1695 1696 ctxt->regs->si += off; 1697 ctxt->regs->di += off; 1698 1699 rep = insn_has_rep_prefix(&ctxt->insn); 1700 if (rep) 1701 ctxt->regs->cx -= 1; 1702 1703 if (!rep || ctxt->regs->cx == 0) 1704 return ES_OK; 1705 else 1706 return ES_RETRY; 1707 } 1708 1709 static enum es_result vc_handle_mmio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 1710 { 1711 struct insn *insn = &ctxt->insn; 1712 enum insn_mmio_type mmio; 1713 unsigned int bytes = 0; 1714 enum es_result ret; 1715 u8 sign_byte; 1716 long *reg_data; 1717 1718 mmio = insn_decode_mmio(insn, &bytes); 1719 if (mmio == INSN_MMIO_DECODE_FAILED) 1720 return ES_DECODE_FAILED; 1721 1722 if (mmio != INSN_MMIO_WRITE_IMM && mmio != INSN_MMIO_MOVS) { 1723 reg_data = insn_get_modrm_reg_ptr(insn, ctxt->regs); 1724 if (!reg_data) 1725 return ES_DECODE_FAILED; 1726 } 1727 1728 if (user_mode(ctxt->regs)) 1729 return ES_UNSUPPORTED; 1730 1731 switch (mmio) { 1732 case INSN_MMIO_WRITE: 1733 memcpy(ghcb->shared_buffer, reg_data, bytes); 1734 ret = vc_do_mmio(ghcb, ctxt, bytes, false); 1735 break; 1736 case INSN_MMIO_WRITE_IMM: 1737 memcpy(ghcb->shared_buffer, insn->immediate1.bytes, bytes); 1738 ret = vc_do_mmio(ghcb, ctxt, bytes, false); 1739 break; 1740 case INSN_MMIO_READ: 1741 ret = vc_do_mmio(ghcb, ctxt, bytes, true); 1742 if (ret) 1743 break; 1744 1745 /* Zero-extend for 32-bit operation */ 1746 if (bytes == 4) 1747 *reg_data = 0; 1748 1749 memcpy(reg_data, ghcb->shared_buffer, bytes); 1750 break; 1751 case INSN_MMIO_READ_ZERO_EXTEND: 1752 ret = vc_do_mmio(ghcb, ctxt, bytes, true); 1753 if (ret) 1754 break; 1755 1756 /* Zero extend based on operand size */ 1757 memset(reg_data, 0, insn->opnd_bytes); 1758 memcpy(reg_data, ghcb->shared_buffer, bytes); 1759 break; 1760 case INSN_MMIO_READ_SIGN_EXTEND: 1761 ret = vc_do_mmio(ghcb, ctxt, bytes, true); 1762 if (ret) 1763 break; 1764 1765 if (bytes == 1) { 1766 u8 *val = (u8 *)ghcb->shared_buffer; 1767 1768 sign_byte = (*val & 0x80) ? 0xff : 0x00; 1769 } else { 1770 u16 *val = (u16 *)ghcb->shared_buffer; 1771 1772 sign_byte = (*val & 0x8000) ? 0xff : 0x00; 1773 } 1774 1775 /* Sign extend based on operand size */ 1776 memset(reg_data, sign_byte, insn->opnd_bytes); 1777 memcpy(reg_data, ghcb->shared_buffer, bytes); 1778 break; 1779 case INSN_MMIO_MOVS: 1780 ret = vc_handle_mmio_movs(ctxt, bytes); 1781 break; 1782 default: 1783 ret = ES_UNSUPPORTED; 1784 break; 1785 } 1786 1787 return ret; 1788 } 1789 1790 static enum es_result vc_handle_dr7_write(struct ghcb *ghcb, 1791 struct es_em_ctxt *ctxt) 1792 { 1793 struct sev_es_runtime_data *data = this_cpu_read(runtime_data); 1794 long val, *reg = vc_insn_get_rm(ctxt); 1795 enum es_result ret; 1796 1797 if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP) 1798 return ES_VMM_ERROR; 1799 1800 if (!reg) 1801 return ES_DECODE_FAILED; 1802 1803 val = *reg; 1804 1805 /* Upper 32 bits must be written as zeroes */ 1806 if (val >> 32) { 1807 ctxt->fi.vector = X86_TRAP_GP; 1808 ctxt->fi.error_code = 0; 1809 return ES_EXCEPTION; 1810 } 1811 1812 /* Clear out other reserved bits and set bit 10 */ 1813 val = (val & 0xffff23ffL) | BIT(10); 1814 1815 /* Early non-zero writes to DR7 are not supported */ 1816 if (!data && (val & ~DR7_RESET_VALUE)) 1817 return ES_UNSUPPORTED; 1818 1819 /* Using a value of 0 for ExitInfo1 means RAX holds the value */ 1820 ghcb_set_rax(ghcb, val); 1821 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WRITE_DR7, 0, 0); 1822 if (ret != ES_OK) 1823 return ret; 1824 1825 if (data) 1826 data->dr7 = val; 1827 1828 return ES_OK; 1829 } 1830 1831 static enum es_result vc_handle_dr7_read(struct ghcb *ghcb, 1832 struct es_em_ctxt *ctxt) 1833 { 1834 struct sev_es_runtime_data *data = this_cpu_read(runtime_data); 1835 long *reg = vc_insn_get_rm(ctxt); 1836 1837 if (sev_status & MSR_AMD64_SNP_DEBUG_SWAP) 1838 return ES_VMM_ERROR; 1839 1840 if (!reg) 1841 return ES_DECODE_FAILED; 1842 1843 if (data) 1844 *reg = data->dr7; 1845 else 1846 *reg = DR7_RESET_VALUE; 1847 1848 return ES_OK; 1849 } 1850 1851 static enum es_result vc_handle_wbinvd(struct ghcb *ghcb, 1852 struct es_em_ctxt *ctxt) 1853 { 1854 return sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_WBINVD, 0, 0); 1855 } 1856 1857 static enum es_result vc_handle_rdpmc(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 1858 { 1859 enum es_result ret; 1860 1861 ghcb_set_rcx(ghcb, ctxt->regs->cx); 1862 1863 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_RDPMC, 0, 0); 1864 if (ret != ES_OK) 1865 return ret; 1866 1867 if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb))) 1868 return ES_VMM_ERROR; 1869 1870 ctxt->regs->ax = ghcb->save.rax; 1871 ctxt->regs->dx = ghcb->save.rdx; 1872 1873 return ES_OK; 1874 } 1875 1876 static enum es_result vc_handle_monitor(struct ghcb *ghcb, 1877 struct es_em_ctxt *ctxt) 1878 { 1879 /* 1880 * Treat it as a NOP and do not leak a physical address to the 1881 * hypervisor. 1882 */ 1883 return ES_OK; 1884 } 1885 1886 static enum es_result vc_handle_mwait(struct ghcb *ghcb, 1887 struct es_em_ctxt *ctxt) 1888 { 1889 /* Treat the same as MONITOR/MONITORX */ 1890 return ES_OK; 1891 } 1892 1893 static enum es_result vc_handle_vmmcall(struct ghcb *ghcb, 1894 struct es_em_ctxt *ctxt) 1895 { 1896 enum es_result ret; 1897 1898 ghcb_set_rax(ghcb, ctxt->regs->ax); 1899 ghcb_set_cpl(ghcb, user_mode(ctxt->regs) ? 3 : 0); 1900 1901 if (x86_platform.hyper.sev_es_hcall_prepare) 1902 x86_platform.hyper.sev_es_hcall_prepare(ghcb, ctxt->regs); 1903 1904 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_VMMCALL, 0, 0); 1905 if (ret != ES_OK) 1906 return ret; 1907 1908 if (!ghcb_rax_is_valid(ghcb)) 1909 return ES_VMM_ERROR; 1910 1911 ctxt->regs->ax = ghcb->save.rax; 1912 1913 /* 1914 * Call sev_es_hcall_finish() after regs->ax is already set. 1915 * This allows the hypervisor handler to overwrite it again if 1916 * necessary. 1917 */ 1918 if (x86_platform.hyper.sev_es_hcall_finish && 1919 !x86_platform.hyper.sev_es_hcall_finish(ghcb, ctxt->regs)) 1920 return ES_VMM_ERROR; 1921 1922 return ES_OK; 1923 } 1924 1925 static enum es_result vc_handle_trap_ac(struct ghcb *ghcb, 1926 struct es_em_ctxt *ctxt) 1927 { 1928 /* 1929 * Calling ecx_alignment_check() directly does not work, because it 1930 * enables IRQs and the GHCB is active. Forward the exception and call 1931 * it later from vc_forward_exception(). 1932 */ 1933 ctxt->fi.vector = X86_TRAP_AC; 1934 ctxt->fi.error_code = 0; 1935 return ES_EXCEPTION; 1936 } 1937 1938 static enum es_result vc_handle_exitcode(struct es_em_ctxt *ctxt, 1939 struct ghcb *ghcb, 1940 unsigned long exit_code) 1941 { 1942 enum es_result result = vc_check_opcode_bytes(ctxt, exit_code); 1943 1944 if (result != ES_OK) 1945 return result; 1946 1947 switch (exit_code) { 1948 case SVM_EXIT_READ_DR7: 1949 result = vc_handle_dr7_read(ghcb, ctxt); 1950 break; 1951 case SVM_EXIT_WRITE_DR7: 1952 result = vc_handle_dr7_write(ghcb, ctxt); 1953 break; 1954 case SVM_EXIT_EXCP_BASE + X86_TRAP_AC: 1955 result = vc_handle_trap_ac(ghcb, ctxt); 1956 break; 1957 case SVM_EXIT_RDTSC: 1958 case SVM_EXIT_RDTSCP: 1959 result = vc_handle_rdtsc(ghcb, ctxt, exit_code); 1960 break; 1961 case SVM_EXIT_RDPMC: 1962 result = vc_handle_rdpmc(ghcb, ctxt); 1963 break; 1964 case SVM_EXIT_INVD: 1965 pr_err_ratelimited("#VC exception for INVD??? Seriously???\n"); 1966 result = ES_UNSUPPORTED; 1967 break; 1968 case SVM_EXIT_CPUID: 1969 result = vc_handle_cpuid(ghcb, ctxt); 1970 break; 1971 case SVM_EXIT_IOIO: 1972 result = vc_handle_ioio(ghcb, ctxt); 1973 break; 1974 case SVM_EXIT_MSR: 1975 result = vc_handle_msr(ghcb, ctxt); 1976 break; 1977 case SVM_EXIT_VMMCALL: 1978 result = vc_handle_vmmcall(ghcb, ctxt); 1979 break; 1980 case SVM_EXIT_WBINVD: 1981 result = vc_handle_wbinvd(ghcb, ctxt); 1982 break; 1983 case SVM_EXIT_MONITOR: 1984 result = vc_handle_monitor(ghcb, ctxt); 1985 break; 1986 case SVM_EXIT_MWAIT: 1987 result = vc_handle_mwait(ghcb, ctxt); 1988 break; 1989 case SVM_EXIT_NPF: 1990 result = vc_handle_mmio(ghcb, ctxt); 1991 break; 1992 default: 1993 /* 1994 * Unexpected #VC exception 1995 */ 1996 result = ES_UNSUPPORTED; 1997 } 1998 1999 return result; 2000 } 2001 2002 static __always_inline bool is_vc2_stack(unsigned long sp) 2003 { 2004 return (sp >= __this_cpu_ist_bottom_va(VC2) && sp < __this_cpu_ist_top_va(VC2)); 2005 } 2006 2007 static __always_inline bool vc_from_invalid_context(struct pt_regs *regs) 2008 { 2009 unsigned long sp, prev_sp; 2010 2011 sp = (unsigned long)regs; 2012 prev_sp = regs->sp; 2013 2014 /* 2015 * If the code was already executing on the VC2 stack when the #VC 2016 * happened, let it proceed to the normal handling routine. This way the 2017 * code executing on the VC2 stack can cause #VC exceptions to get handled. 2018 */ 2019 return is_vc2_stack(sp) && !is_vc2_stack(prev_sp); 2020 } 2021 2022 static bool vc_raw_handle_exception(struct pt_regs *regs, unsigned long error_code) 2023 { 2024 struct ghcb_state state; 2025 struct es_em_ctxt ctxt; 2026 enum es_result result; 2027 struct ghcb *ghcb; 2028 bool ret = true; 2029 2030 ghcb = __sev_get_ghcb(&state); 2031 2032 vc_ghcb_invalidate(ghcb); 2033 result = vc_init_em_ctxt(&ctxt, regs, error_code); 2034 2035 if (result == ES_OK) 2036 result = vc_handle_exitcode(&ctxt, ghcb, error_code); 2037 2038 __sev_put_ghcb(&state); 2039 2040 /* Done - now check the result */ 2041 switch (result) { 2042 case ES_OK: 2043 vc_finish_insn(&ctxt); 2044 break; 2045 case ES_UNSUPPORTED: 2046 pr_err_ratelimited("Unsupported exit-code 0x%02lx in #VC exception (IP: 0x%lx)\n", 2047 error_code, regs->ip); 2048 ret = false; 2049 break; 2050 case ES_VMM_ERROR: 2051 pr_err_ratelimited("Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n", 2052 error_code, regs->ip); 2053 ret = false; 2054 break; 2055 case ES_DECODE_FAILED: 2056 pr_err_ratelimited("Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n", 2057 error_code, regs->ip); 2058 ret = false; 2059 break; 2060 case ES_EXCEPTION: 2061 vc_forward_exception(&ctxt); 2062 break; 2063 case ES_RETRY: 2064 /* Nothing to do */ 2065 break; 2066 default: 2067 pr_emerg("Unknown result in %s():%d\n", __func__, result); 2068 /* 2069 * Emulating the instruction which caused the #VC exception 2070 * failed - can't continue so print debug information 2071 */ 2072 BUG(); 2073 } 2074 2075 return ret; 2076 } 2077 2078 static __always_inline bool vc_is_db(unsigned long error_code) 2079 { 2080 return error_code == SVM_EXIT_EXCP_BASE + X86_TRAP_DB; 2081 } 2082 2083 /* 2084 * Runtime #VC exception handler when raised from kernel mode. Runs in NMI mode 2085 * and will panic when an error happens. 2086 */ 2087 DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication) 2088 { 2089 irqentry_state_t irq_state; 2090 2091 /* 2092 * With the current implementation it is always possible to switch to a 2093 * safe stack because #VC exceptions only happen at known places, like 2094 * intercepted instructions or accesses to MMIO areas/IO ports. They can 2095 * also happen with code instrumentation when the hypervisor intercepts 2096 * #DB, but the critical paths are forbidden to be instrumented, so #DB 2097 * exceptions currently also only happen in safe places. 2098 * 2099 * But keep this here in case the noinstr annotations are violated due 2100 * to bug elsewhere. 2101 */ 2102 if (unlikely(vc_from_invalid_context(regs))) { 2103 instrumentation_begin(); 2104 panic("Can't handle #VC exception from unsupported context\n"); 2105 instrumentation_end(); 2106 } 2107 2108 /* 2109 * Handle #DB before calling into !noinstr code to avoid recursive #DB. 2110 */ 2111 if (vc_is_db(error_code)) { 2112 exc_debug(regs); 2113 return; 2114 } 2115 2116 irq_state = irqentry_nmi_enter(regs); 2117 2118 instrumentation_begin(); 2119 2120 if (!vc_raw_handle_exception(regs, error_code)) { 2121 /* Show some debug info */ 2122 show_regs(regs); 2123 2124 /* Ask hypervisor to sev_es_terminate */ 2125 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); 2126 2127 /* If that fails and we get here - just panic */ 2128 panic("Returned from Terminate-Request to Hypervisor\n"); 2129 } 2130 2131 instrumentation_end(); 2132 irqentry_nmi_exit(regs, irq_state); 2133 } 2134 2135 /* 2136 * Runtime #VC exception handler when raised from user mode. Runs in IRQ mode 2137 * and will kill the current task with SIGBUS when an error happens. 2138 */ 2139 DEFINE_IDTENTRY_VC_USER(exc_vmm_communication) 2140 { 2141 /* 2142 * Handle #DB before calling into !noinstr code to avoid recursive #DB. 2143 */ 2144 if (vc_is_db(error_code)) { 2145 noist_exc_debug(regs); 2146 return; 2147 } 2148 2149 irqentry_enter_from_user_mode(regs); 2150 instrumentation_begin(); 2151 2152 if (!vc_raw_handle_exception(regs, error_code)) { 2153 /* 2154 * Do not kill the machine if user-space triggered the 2155 * exception. Send SIGBUS instead and let user-space deal with 2156 * it. 2157 */ 2158 force_sig_fault(SIGBUS, BUS_OBJERR, (void __user *)0); 2159 } 2160 2161 instrumentation_end(); 2162 irqentry_exit_to_user_mode(regs); 2163 } 2164 2165 bool __init handle_vc_boot_ghcb(struct pt_regs *regs) 2166 { 2167 unsigned long exit_code = regs->orig_ax; 2168 struct es_em_ctxt ctxt; 2169 enum es_result result; 2170 2171 vc_ghcb_invalidate(boot_ghcb); 2172 2173 result = vc_init_em_ctxt(&ctxt, regs, exit_code); 2174 if (result == ES_OK) 2175 result = vc_handle_exitcode(&ctxt, boot_ghcb, exit_code); 2176 2177 /* Done - now check the result */ 2178 switch (result) { 2179 case ES_OK: 2180 vc_finish_insn(&ctxt); 2181 break; 2182 case ES_UNSUPPORTED: 2183 early_printk("PANIC: Unsupported exit-code 0x%02lx in early #VC exception (IP: 0x%lx)\n", 2184 exit_code, regs->ip); 2185 goto fail; 2186 case ES_VMM_ERROR: 2187 early_printk("PANIC: Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n", 2188 exit_code, regs->ip); 2189 goto fail; 2190 case ES_DECODE_FAILED: 2191 early_printk("PANIC: Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n", 2192 exit_code, regs->ip); 2193 goto fail; 2194 case ES_EXCEPTION: 2195 vc_early_forward_exception(&ctxt); 2196 break; 2197 case ES_RETRY: 2198 /* Nothing to do */ 2199 break; 2200 default: 2201 BUG(); 2202 } 2203 2204 return true; 2205 2206 fail: 2207 show_regs(regs); 2208 2209 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); 2210 } 2211 2212 /* 2213 * Initial set up of SNP relies on information provided by the 2214 * Confidential Computing blob, which can be passed to the kernel 2215 * in the following ways, depending on how it is booted: 2216 * 2217 * - when booted via the boot/decompress kernel: 2218 * - via boot_params 2219 * 2220 * - when booted directly by firmware/bootloader (e.g. CONFIG_PVH): 2221 * - via a setup_data entry, as defined by the Linux Boot Protocol 2222 * 2223 * Scan for the blob in that order. 2224 */ 2225 static __head struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp) 2226 { 2227 struct cc_blob_sev_info *cc_info; 2228 2229 /* Boot kernel would have passed the CC blob via boot_params. */ 2230 if (bp->cc_blob_address) { 2231 cc_info = (struct cc_blob_sev_info *)(unsigned long)bp->cc_blob_address; 2232 goto found_cc_info; 2233 } 2234 2235 /* 2236 * If kernel was booted directly, without the use of the 2237 * boot/decompression kernel, the CC blob may have been passed via 2238 * setup_data instead. 2239 */ 2240 cc_info = find_cc_blob_setup_data(bp); 2241 if (!cc_info) 2242 return NULL; 2243 2244 found_cc_info: 2245 if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC) 2246 snp_abort(); 2247 2248 return cc_info; 2249 } 2250 2251 static __head void svsm_setup(struct cc_blob_sev_info *cc_info) 2252 { 2253 struct svsm_call call = {}; 2254 int ret; 2255 u64 pa; 2256 2257 /* 2258 * Record the SVSM Calling Area address (CAA) if the guest is not 2259 * running at VMPL0. The CA will be used to communicate with the 2260 * SVSM to perform the SVSM services. 2261 */ 2262 if (!svsm_setup_ca(cc_info)) 2263 return; 2264 2265 /* 2266 * It is very early in the boot and the kernel is running identity 2267 * mapped but without having adjusted the pagetables to where the 2268 * kernel was loaded (physbase), so the get the CA address using 2269 * RIP-relative addressing. 2270 */ 2271 pa = (u64)&RIP_REL_REF(boot_svsm_ca_page); 2272 2273 /* 2274 * Switch over to the boot SVSM CA while the current CA is still 2275 * addressable. There is no GHCB at this point so use the MSR protocol. 2276 * 2277 * SVSM_CORE_REMAP_CA call: 2278 * RAX = 0 (Protocol=0, CallID=0) 2279 * RCX = New CA GPA 2280 */ 2281 call.caa = svsm_get_caa(); 2282 call.rax = SVSM_CORE_CALL(SVSM_CORE_REMAP_CA); 2283 call.rcx = pa; 2284 ret = svsm_perform_call_protocol(&call); 2285 if (ret) 2286 panic("Can't remap the SVSM CA, ret=%d, rax_out=0x%llx\n", ret, call.rax_out); 2287 2288 RIP_REL_REF(boot_svsm_caa) = (struct svsm_ca *)pa; 2289 RIP_REL_REF(boot_svsm_caa_pa) = pa; 2290 } 2291 2292 bool __head snp_init(struct boot_params *bp) 2293 { 2294 struct cc_blob_sev_info *cc_info; 2295 2296 if (!bp) 2297 return false; 2298 2299 cc_info = find_cc_blob(bp); 2300 if (!cc_info) 2301 return false; 2302 2303 setup_cpuid_table(cc_info); 2304 2305 svsm_setup(cc_info); 2306 2307 /* 2308 * The CC blob will be used later to access the secrets page. Cache 2309 * it here like the boot kernel does. 2310 */ 2311 bp->cc_blob_address = (u32)(unsigned long)cc_info; 2312 2313 return true; 2314 } 2315 2316 void __head __noreturn snp_abort(void) 2317 { 2318 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED); 2319 } 2320 2321 /* 2322 * SEV-SNP guests should only execute dmi_setup() if EFI_CONFIG_TABLES are 2323 * enabled, as the alternative (fallback) logic for DMI probing in the legacy 2324 * ROM region can cause a crash since this region is not pre-validated. 2325 */ 2326 void __init snp_dmi_setup(void) 2327 { 2328 if (efi_enabled(EFI_CONFIG_TABLES)) 2329 dmi_setup(); 2330 } 2331 2332 static void dump_cpuid_table(void) 2333 { 2334 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); 2335 int i = 0; 2336 2337 pr_info("count=%d reserved=0x%x reserved2=0x%llx\n", 2338 cpuid_table->count, cpuid_table->__reserved1, cpuid_table->__reserved2); 2339 2340 for (i = 0; i < SNP_CPUID_COUNT_MAX; i++) { 2341 const struct snp_cpuid_fn *fn = &cpuid_table->fn[i]; 2342 2343 pr_info("index=%3d fn=0x%08x subfn=0x%08x: eax=0x%08x ebx=0x%08x ecx=0x%08x edx=0x%08x xcr0_in=0x%016llx xss_in=0x%016llx reserved=0x%016llx\n", 2344 i, fn->eax_in, fn->ecx_in, fn->eax, fn->ebx, fn->ecx, 2345 fn->edx, fn->xcr0_in, fn->xss_in, fn->__reserved); 2346 } 2347 } 2348 2349 /* 2350 * It is useful from an auditing/testing perspective to provide an easy way 2351 * for the guest owner to know that the CPUID table has been initialized as 2352 * expected, but that initialization happens too early in boot to print any 2353 * sort of indicator, and there's not really any other good place to do it, 2354 * so do it here. 2355 * 2356 * If running as an SNP guest, report the current VM privilege level (VMPL). 2357 */ 2358 static int __init report_snp_info(void) 2359 { 2360 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); 2361 2362 if (cpuid_table->count) { 2363 pr_info("Using SNP CPUID table, %d entries present.\n", 2364 cpuid_table->count); 2365 2366 if (sev_cfg.debug) 2367 dump_cpuid_table(); 2368 } 2369 2370 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 2371 pr_info("SNP running at VMPL%u.\n", snp_vmpl); 2372 2373 return 0; 2374 } 2375 arch_initcall(report_snp_info); 2376 2377 static int __init init_sev_config(char *str) 2378 { 2379 char *s; 2380 2381 while ((s = strsep(&str, ","))) { 2382 if (!strcmp(s, "debug")) { 2383 sev_cfg.debug = true; 2384 continue; 2385 } 2386 2387 pr_info("SEV command-line option '%s' was not recognized\n", s); 2388 } 2389 2390 return 1; 2391 } 2392 __setup("sev=", init_sev_config); 2393 2394 static void update_attest_input(struct svsm_call *call, struct svsm_attest_call *input) 2395 { 2396 /* If (new) lengths have been returned, propagate them up */ 2397 if (call->rcx_out != call->rcx) 2398 input->manifest_buf.len = call->rcx_out; 2399 2400 if (call->rdx_out != call->rdx) 2401 input->certificates_buf.len = call->rdx_out; 2402 2403 if (call->r8_out != call->r8) 2404 input->report_buf.len = call->r8_out; 2405 } 2406 2407 int snp_issue_svsm_attest_req(u64 call_id, struct svsm_call *call, 2408 struct svsm_attest_call *input) 2409 { 2410 struct svsm_attest_call *ac; 2411 unsigned long flags; 2412 u64 attest_call_pa; 2413 int ret; 2414 2415 if (!snp_vmpl) 2416 return -EINVAL; 2417 2418 local_irq_save(flags); 2419 2420 call->caa = svsm_get_caa(); 2421 2422 ac = (struct svsm_attest_call *)call->caa->svsm_buffer; 2423 attest_call_pa = svsm_get_caa_pa() + offsetof(struct svsm_ca, svsm_buffer); 2424 2425 *ac = *input; 2426 2427 /* 2428 * Set input registers for the request and set RDX and R8 to known 2429 * values in order to detect length values being returned in them. 2430 */ 2431 call->rax = call_id; 2432 call->rcx = attest_call_pa; 2433 call->rdx = -1; 2434 call->r8 = -1; 2435 ret = svsm_perform_call_protocol(call); 2436 update_attest_input(call, input); 2437 2438 local_irq_restore(flags); 2439 2440 return ret; 2441 } 2442 EXPORT_SYMBOL_GPL(snp_issue_svsm_attest_req); 2443 2444 int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, struct snp_guest_request_ioctl *rio) 2445 { 2446 struct ghcb_state state; 2447 struct es_em_ctxt ctxt; 2448 unsigned long flags; 2449 struct ghcb *ghcb; 2450 int ret; 2451 2452 rio->exitinfo2 = SEV_RET_NO_FW_CALL; 2453 2454 /* 2455 * __sev_get_ghcb() needs to run with IRQs disabled because it is using 2456 * a per-CPU GHCB. 2457 */ 2458 local_irq_save(flags); 2459 2460 ghcb = __sev_get_ghcb(&state); 2461 if (!ghcb) { 2462 ret = -EIO; 2463 goto e_restore_irq; 2464 } 2465 2466 vc_ghcb_invalidate(ghcb); 2467 2468 if (exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST) { 2469 ghcb_set_rax(ghcb, input->data_gpa); 2470 ghcb_set_rbx(ghcb, input->data_npages); 2471 } 2472 2473 ret = sev_es_ghcb_hv_call(ghcb, &ctxt, exit_code, input->req_gpa, input->resp_gpa); 2474 if (ret) 2475 goto e_put; 2476 2477 rio->exitinfo2 = ghcb->save.sw_exit_info_2; 2478 switch (rio->exitinfo2) { 2479 case 0: 2480 break; 2481 2482 case SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_BUSY): 2483 ret = -EAGAIN; 2484 break; 2485 2486 case SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN): 2487 /* Number of expected pages are returned in RBX */ 2488 if (exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST) { 2489 input->data_npages = ghcb_get_rbx(ghcb); 2490 ret = -ENOSPC; 2491 break; 2492 } 2493 fallthrough; 2494 default: 2495 ret = -EIO; 2496 break; 2497 } 2498 2499 e_put: 2500 __sev_put_ghcb(&state); 2501 e_restore_irq: 2502 local_irq_restore(flags); 2503 2504 return ret; 2505 } 2506 EXPORT_SYMBOL_GPL(snp_issue_guest_request); 2507 2508 static struct platform_device sev_guest_device = { 2509 .name = "sev-guest", 2510 .id = -1, 2511 }; 2512 2513 static int __init snp_init_platform_device(void) 2514 { 2515 struct sev_guest_platform_data data; 2516 u64 gpa; 2517 2518 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 2519 return -ENODEV; 2520 2521 gpa = get_secrets_page(); 2522 if (!gpa) 2523 return -ENODEV; 2524 2525 data.secrets_gpa = gpa; 2526 if (platform_device_add_data(&sev_guest_device, &data, sizeof(data))) 2527 return -ENODEV; 2528 2529 if (platform_device_register(&sev_guest_device)) 2530 return -ENODEV; 2531 2532 pr_info("SNP guest platform device initialized.\n"); 2533 return 0; 2534 } 2535 device_initcall(snp_init_platform_device); 2536 2537 void sev_show_status(void) 2538 { 2539 int i; 2540 2541 pr_info("Status: "); 2542 for (i = 0; i < MSR_AMD64_SNP_RESV_BIT; i++) { 2543 if (sev_status & BIT_ULL(i)) { 2544 if (!sev_status_feat_names[i]) 2545 continue; 2546 2547 pr_cont("%s ", sev_status_feat_names[i]); 2548 } 2549 } 2550 pr_cont("\n"); 2551 } 2552 2553 void __init snp_update_svsm_ca(void) 2554 { 2555 if (!snp_vmpl) 2556 return; 2557 2558 /* Update the CAA to a proper kernel address */ 2559 boot_svsm_caa = &boot_svsm_ca_page; 2560 } 2561 2562 #ifdef CONFIG_SYSFS 2563 static ssize_t vmpl_show(struct kobject *kobj, 2564 struct kobj_attribute *attr, char *buf) 2565 { 2566 return sysfs_emit(buf, "%d\n", snp_vmpl); 2567 } 2568 2569 static struct kobj_attribute vmpl_attr = __ATTR_RO(vmpl); 2570 2571 static struct attribute *vmpl_attrs[] = { 2572 &vmpl_attr.attr, 2573 NULL 2574 }; 2575 2576 static struct attribute_group sev_attr_group = { 2577 .attrs = vmpl_attrs, 2578 }; 2579 2580 static int __init sev_sysfs_init(void) 2581 { 2582 struct kobject *sev_kobj; 2583 struct device *dev_root; 2584 int ret; 2585 2586 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 2587 return -ENODEV; 2588 2589 dev_root = bus_get_dev_root(&cpu_subsys); 2590 if (!dev_root) 2591 return -ENODEV; 2592 2593 sev_kobj = kobject_create_and_add("sev", &dev_root->kobj); 2594 put_device(dev_root); 2595 2596 if (!sev_kobj) 2597 return -ENOMEM; 2598 2599 ret = sysfs_create_group(sev_kobj, &sev_attr_group); 2600 if (ret) 2601 kobject_put(sev_kobj); 2602 2603 return ret; 2604 } 2605 arch_initcall(sev_sysfs_init); 2606 #endif // CONFIG_SYSFS 2607