1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * AMD Memory Encryption Support 4 * 5 * Copyright (C) 2019 SUSE 6 * 7 * Author: Joerg Roedel <jroedel@suse.de> 8 */ 9 10 #define pr_fmt(fmt) "SEV: " fmt 11 12 #include <linux/sched/debug.h> /* For show_regs() */ 13 #include <linux/percpu-defs.h> 14 #include <linux/cc_platform.h> 15 #include <linux/printk.h> 16 #include <linux/mm_types.h> 17 #include <linux/set_memory.h> 18 #include <linux/memblock.h> 19 #include <linux/kernel.h> 20 #include <linux/mm.h> 21 #include <linux/cpumask.h> 22 #include <linux/efi.h> 23 #include <linux/platform_device.h> 24 #include <linux/io.h> 25 #include <linux/psp-sev.h> 26 #include <linux/dmi.h> 27 #include <uapi/linux/sev-guest.h> 28 #include <crypto/aes-gcm.h> 29 30 #include <asm/init.h> 31 #include <asm/cpu_entry_area.h> 32 #include <asm/stacktrace.h> 33 #include <asm/sev.h> 34 #include <asm/insn-eval.h> 35 #include <asm/fpu/xcr.h> 36 #include <asm/processor.h> 37 #include <asm/realmode.h> 38 #include <asm/setup.h> 39 #include <asm/traps.h> 40 #include <asm/svm.h> 41 #include <asm/smp.h> 42 #include <asm/cpu.h> 43 #include <asm/apic.h> 44 #include <asm/cpuid/api.h> 45 #include <asm/cmdline.h> 46 #include <asm/msr.h> 47 48 #include "internal.h" 49 50 /* Bitmap of SEV features supported by the hypervisor */ 51 u64 sev_hv_features __ro_after_init; 52 SYM_PIC_ALIAS(sev_hv_features); 53 54 /* Secrets page physical address from the CC blob */ 55 u64 sev_secrets_pa __ro_after_init; 56 SYM_PIC_ALIAS(sev_secrets_pa); 57 58 /* AP INIT values as documented in the APM2 section "Processor Initialization State" */ 59 #define AP_INIT_CS_LIMIT 0xffff 60 #define AP_INIT_DS_LIMIT 0xffff 61 #define AP_INIT_LDTR_LIMIT 0xffff 62 #define AP_INIT_GDTR_LIMIT 0xffff 63 #define AP_INIT_IDTR_LIMIT 0xffff 64 #define AP_INIT_TR_LIMIT 0xffff 65 #define AP_INIT_RFLAGS_DEFAULT 0x2 66 #define AP_INIT_DR6_DEFAULT 0xffff0ff0 67 #define AP_INIT_GPAT_DEFAULT 0x0007040600070406ULL 68 #define AP_INIT_XCR0_DEFAULT 0x1 69 #define AP_INIT_X87_FTW_DEFAULT 0x5555 70 #define AP_INIT_X87_FCW_DEFAULT 0x0040 71 #define AP_INIT_CR0_DEFAULT 0x60000010 72 #define AP_INIT_MXCSR_DEFAULT 0x1f80 73 74 static const char * const sev_status_feat_names[] = { 75 [MSR_AMD64_SEV_ENABLED_BIT] = "SEV", 76 [MSR_AMD64_SEV_ES_ENABLED_BIT] = "SEV-ES", 77 [MSR_AMD64_SEV_SNP_ENABLED_BIT] = "SEV-SNP", 78 [MSR_AMD64_SNP_VTOM_BIT] = "vTom", 79 [MSR_AMD64_SNP_REFLECT_VC_BIT] = "ReflectVC", 80 [MSR_AMD64_SNP_RESTRICTED_INJ_BIT] = "RI", 81 [MSR_AMD64_SNP_ALT_INJ_BIT] = "AI", 82 [MSR_AMD64_SNP_DEBUG_SWAP_BIT] = "DebugSwap", 83 [MSR_AMD64_SNP_PREVENT_HOST_IBS_BIT] = "NoHostIBS", 84 [MSR_AMD64_SNP_BTB_ISOLATION_BIT] = "BTBIsol", 85 [MSR_AMD64_SNP_VMPL_SSS_BIT] = "VmplSSS", 86 [MSR_AMD64_SNP_SECURE_TSC_BIT] = "SecureTSC", 87 [MSR_AMD64_SNP_VMGEXIT_PARAM_BIT] = "VMGExitParam", 88 [MSR_AMD64_SNP_IBS_VIRT_BIT] = "IBSVirt", 89 [MSR_AMD64_SNP_VMSA_REG_PROT_BIT] = "VMSARegProt", 90 [MSR_AMD64_SNP_SMT_PROT_BIT] = "SMTProt", 91 [MSR_AMD64_SNP_SECURE_AVIC_BIT] = "SecureAVIC", 92 [MSR_AMD64_SNP_IBPB_ON_ENTRY_BIT] = "IBPBOnEntry", 93 }; 94 95 /* 96 * For Secure TSC guests, the BSP fetches TSC_INFO using SNP guest messaging and 97 * initializes snp_tsc_scale and snp_tsc_offset. These values are replicated 98 * across the APs VMSA fields (TSC_SCALE and TSC_OFFSET). 99 */ 100 static u64 snp_tsc_scale __ro_after_init; 101 static u64 snp_tsc_offset __ro_after_init; 102 static unsigned long snp_tsc_freq_khz __ro_after_init; 103 104 DEFINE_PER_CPU(struct sev_es_runtime_data*, runtime_data); 105 DEFINE_PER_CPU(struct sev_es_save_area *, sev_vmsa); 106 107 /* 108 * SVSM related information: 109 * When running under an SVSM, the VMPL that Linux is executing at must be 110 * non-zero. The VMPL is therefore used to indicate the presence of an SVSM. 111 */ 112 u8 snp_vmpl __ro_after_init; 113 EXPORT_SYMBOL_GPL(snp_vmpl); 114 SYM_PIC_ALIAS(snp_vmpl); 115 116 /* 117 * Since feature negotiation related variables are set early in the boot 118 * process they must reside in the .data section so as not to be zeroed 119 * out when the .bss section is later cleared. 120 * 121 * GHCB protocol version negotiated with the hypervisor. 122 */ 123 u16 ghcb_version __ro_after_init; 124 SYM_PIC_ALIAS(ghcb_version); 125 126 /* For early boot hypervisor communication in SEV-ES enabled guests */ 127 static struct ghcb boot_ghcb_page __bss_decrypted __aligned(PAGE_SIZE); 128 129 /* 130 * Needs to be in the .data section because we need it NULL before bss is 131 * cleared 132 */ 133 struct ghcb *boot_ghcb __section(".data"); 134 135 static u64 __init get_snp_jump_table_addr(void) 136 { 137 struct snp_secrets_page *secrets; 138 void __iomem *mem; 139 u64 addr; 140 141 mem = ioremap_encrypted(sev_secrets_pa, PAGE_SIZE); 142 if (!mem) { 143 pr_err("Unable to locate AP jump table address: failed to map the SNP secrets page.\n"); 144 return 0; 145 } 146 147 secrets = (__force struct snp_secrets_page *)mem; 148 149 addr = secrets->os_area.ap_jump_table_pa; 150 iounmap(mem); 151 152 return addr; 153 } 154 155 static u64 __init get_jump_table_addr(void) 156 { 157 struct ghcb_state state; 158 unsigned long flags; 159 struct ghcb *ghcb; 160 u64 ret = 0; 161 162 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 163 return get_snp_jump_table_addr(); 164 165 local_irq_save(flags); 166 167 ghcb = __sev_get_ghcb(&state); 168 169 vc_ghcb_invalidate(ghcb); 170 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_JUMP_TABLE); 171 ghcb_set_sw_exit_info_1(ghcb, SVM_VMGEXIT_GET_AP_JUMP_TABLE); 172 ghcb_set_sw_exit_info_2(ghcb, 0); 173 174 sev_es_wr_ghcb_msr(__pa(ghcb)); 175 VMGEXIT(); 176 177 if (ghcb_sw_exit_info_1_is_valid(ghcb) && 178 ghcb_sw_exit_info_2_is_valid(ghcb)) 179 ret = ghcb->save.sw_exit_info_2; 180 181 __sev_put_ghcb(&state); 182 183 local_irq_restore(flags); 184 185 return ret; 186 } 187 188 static void pval_pages(struct snp_psc_desc *desc) 189 { 190 struct psc_entry *e; 191 unsigned long vaddr; 192 unsigned int size; 193 unsigned int i; 194 bool validate; 195 u64 pfn; 196 int rc; 197 198 for (i = 0; i <= desc->hdr.end_entry; i++) { 199 e = &desc->entries[i]; 200 201 pfn = e->gfn; 202 vaddr = (unsigned long)pfn_to_kaddr(pfn); 203 size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K; 204 validate = e->operation == SNP_PAGE_STATE_PRIVATE; 205 206 rc = pvalidate(vaddr, size, validate); 207 if (!rc) 208 continue; 209 210 if (rc == PVALIDATE_FAIL_SIZEMISMATCH && size == RMP_PG_SIZE_2M) { 211 unsigned long vaddr_end = vaddr + PMD_SIZE; 212 213 for (; vaddr < vaddr_end; vaddr += PAGE_SIZE, pfn++) { 214 rc = pvalidate(vaddr, RMP_PG_SIZE_4K, validate); 215 if (rc) 216 __pval_terminate(pfn, validate, RMP_PG_SIZE_4K, rc, 0); 217 } 218 } else { 219 __pval_terminate(pfn, validate, size, rc, 0); 220 } 221 } 222 } 223 224 static void pvalidate_pages(struct snp_psc_desc *desc) 225 { 226 struct psc_entry *e; 227 unsigned int i; 228 229 if (snp_vmpl) 230 svsm_pval_pages(desc); 231 else 232 pval_pages(desc); 233 234 /* 235 * If not affected by the cache-coherency vulnerability there is no need 236 * to perform the cache eviction mitigation. 237 */ 238 if (cpu_feature_enabled(X86_FEATURE_COHERENCY_SFW_NO)) 239 return; 240 241 for (i = 0; i <= desc->hdr.end_entry; i++) { 242 e = &desc->entries[i]; 243 244 /* 245 * If validating memory (making it private) perform the cache 246 * eviction mitigation. 247 */ 248 if (e->operation == SNP_PAGE_STATE_PRIVATE) 249 sev_evict_cache(pfn_to_kaddr(e->gfn), e->pagesize ? 512 : 1); 250 } 251 } 252 253 static int vmgexit_psc(struct ghcb *ghcb, struct snp_psc_desc *desc) 254 { 255 int cur_entry, end_entry, ret = 0; 256 struct snp_psc_desc *data; 257 struct es_em_ctxt ctxt; 258 259 vc_ghcb_invalidate(ghcb); 260 261 /* Copy the input desc into GHCB shared buffer */ 262 data = (struct snp_psc_desc *)ghcb->shared_buffer; 263 memcpy(ghcb->shared_buffer, desc, min_t(int, GHCB_SHARED_BUF_SIZE, sizeof(*desc))); 264 265 /* 266 * As per the GHCB specification, the hypervisor can resume the guest 267 * before processing all the entries. Check whether all the entries 268 * are processed. If not, then keep retrying. Note, the hypervisor 269 * will update the data memory directly to indicate the status, so 270 * reference the data->hdr everywhere. 271 * 272 * The strategy here is to wait for the hypervisor to change the page 273 * state in the RMP table before guest accesses the memory pages. If the 274 * page state change was not successful, then later memory access will 275 * result in a crash. 276 */ 277 cur_entry = data->hdr.cur_entry; 278 end_entry = data->hdr.end_entry; 279 280 while (data->hdr.cur_entry <= data->hdr.end_entry) { 281 ghcb_set_sw_scratch(ghcb, (u64)__pa(data)); 282 283 /* This will advance the shared buffer data points to. */ 284 ret = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_PSC, 0, 0); 285 286 /* 287 * Page State Change VMGEXIT can pass error code through 288 * exit_info_2. 289 */ 290 if (WARN(ret || ghcb->save.sw_exit_info_2, 291 "SNP: PSC failed ret=%d exit_info_2=%llx\n", 292 ret, ghcb->save.sw_exit_info_2)) { 293 ret = 1; 294 goto out; 295 } 296 297 /* Verify that reserved bit is not set */ 298 if (WARN(data->hdr.reserved, "Reserved bit is set in the PSC header\n")) { 299 ret = 1; 300 goto out; 301 } 302 303 /* 304 * Sanity check that entry processing is not going backwards. 305 * This will happen only if hypervisor is tricking us. 306 */ 307 if (WARN(data->hdr.end_entry > end_entry || cur_entry > data->hdr.cur_entry, 308 "SNP: PSC processing going backward, end_entry %d (got %d) cur_entry %d (got %d)\n", 309 end_entry, data->hdr.end_entry, cur_entry, data->hdr.cur_entry)) { 310 ret = 1; 311 goto out; 312 } 313 } 314 315 out: 316 return ret; 317 } 318 319 static unsigned long __set_pages_state(struct snp_psc_desc *data, unsigned long vaddr, 320 unsigned long vaddr_end, int op) 321 { 322 struct ghcb_state state; 323 bool use_large_entry; 324 struct psc_hdr *hdr; 325 struct psc_entry *e; 326 unsigned long flags; 327 unsigned long pfn; 328 struct ghcb *ghcb; 329 int i; 330 331 hdr = &data->hdr; 332 e = data->entries; 333 334 memset(data, 0, sizeof(*data)); 335 i = 0; 336 337 while (vaddr < vaddr_end && i < ARRAY_SIZE(data->entries)) { 338 hdr->end_entry = i; 339 340 if (is_vmalloc_addr((void *)vaddr)) { 341 pfn = vmalloc_to_pfn((void *)vaddr); 342 use_large_entry = false; 343 } else { 344 pfn = __pa(vaddr) >> PAGE_SHIFT; 345 use_large_entry = true; 346 } 347 348 e->gfn = pfn; 349 e->operation = op; 350 351 if (use_large_entry && IS_ALIGNED(vaddr, PMD_SIZE) && 352 (vaddr_end - vaddr) >= PMD_SIZE) { 353 e->pagesize = RMP_PG_SIZE_2M; 354 vaddr += PMD_SIZE; 355 } else { 356 e->pagesize = RMP_PG_SIZE_4K; 357 vaddr += PAGE_SIZE; 358 } 359 360 e++; 361 i++; 362 } 363 364 /* Page validation must be rescinded before changing to shared */ 365 if (op == SNP_PAGE_STATE_SHARED) 366 pvalidate_pages(data); 367 368 local_irq_save(flags); 369 370 ghcb = __sev_get_ghcb(&state); 371 372 /* Invoke the hypervisor to perform the page state changes */ 373 if (!ghcb || vmgexit_psc(ghcb, data)) 374 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC); 375 376 __sev_put_ghcb(&state); 377 378 local_irq_restore(flags); 379 380 /* Page validation must be performed after changing to private */ 381 if (op == SNP_PAGE_STATE_PRIVATE) 382 pvalidate_pages(data); 383 384 return vaddr; 385 } 386 387 static void set_pages_state(unsigned long vaddr, unsigned long npages, int op) 388 { 389 struct snp_psc_desc desc; 390 unsigned long vaddr_end; 391 392 /* Use the MSR protocol when a GHCB is not available. */ 393 if (!boot_ghcb) { 394 struct psc_desc d = { op, svsm_get_caa(), svsm_get_caa_pa() }; 395 396 return early_set_pages_state(vaddr, __pa(vaddr), npages, &d); 397 } 398 399 vaddr = vaddr & PAGE_MASK; 400 vaddr_end = vaddr + (npages << PAGE_SHIFT); 401 402 while (vaddr < vaddr_end) 403 vaddr = __set_pages_state(&desc, vaddr, vaddr_end, op); 404 } 405 406 void snp_set_memory_shared(unsigned long vaddr, unsigned long npages) 407 { 408 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 409 return; 410 411 set_pages_state(vaddr, npages, SNP_PAGE_STATE_SHARED); 412 } 413 414 void snp_set_memory_private(unsigned long vaddr, unsigned long npages) 415 { 416 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 417 return; 418 419 set_pages_state(vaddr, npages, SNP_PAGE_STATE_PRIVATE); 420 } 421 422 void snp_accept_memory(phys_addr_t start, phys_addr_t end) 423 { 424 unsigned long vaddr, npages; 425 426 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 427 return; 428 429 vaddr = (unsigned long)__va(start); 430 npages = (end - start) >> PAGE_SHIFT; 431 432 set_pages_state(vaddr, npages, SNP_PAGE_STATE_PRIVATE); 433 } 434 435 static int vmgexit_ap_control(u64 event, struct sev_es_save_area *vmsa, u32 apic_id) 436 { 437 bool create = event != SVM_VMGEXIT_AP_DESTROY; 438 struct ghcb_state state; 439 unsigned long flags; 440 struct ghcb *ghcb; 441 int ret = 0; 442 443 local_irq_save(flags); 444 445 ghcb = __sev_get_ghcb(&state); 446 447 vc_ghcb_invalidate(ghcb); 448 449 if (create) 450 ghcb_set_rax(ghcb, vmsa->sev_features); 451 452 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_CREATION); 453 ghcb_set_sw_exit_info_1(ghcb, 454 ((u64)apic_id << 32) | 455 ((u64)snp_vmpl << 16) | 456 event); 457 ghcb_set_sw_exit_info_2(ghcb, __pa(vmsa)); 458 459 sev_es_wr_ghcb_msr(__pa(ghcb)); 460 VMGEXIT(); 461 462 if (!ghcb_sw_exit_info_1_is_valid(ghcb) || 463 lower_32_bits(ghcb->save.sw_exit_info_1)) { 464 pr_err("SNP AP %s error\n", (create ? "CREATE" : "DESTROY")); 465 ret = -EINVAL; 466 } 467 468 __sev_put_ghcb(&state); 469 470 local_irq_restore(flags); 471 472 return ret; 473 } 474 475 static int snp_set_vmsa(void *va, void *caa, int apic_id, bool make_vmsa) 476 { 477 int ret; 478 479 if (snp_vmpl) { 480 struct svsm_call call = {}; 481 unsigned long flags; 482 483 local_irq_save(flags); 484 485 call.caa = this_cpu_read(svsm_caa); 486 call.rcx = __pa(va); 487 488 if (make_vmsa) { 489 /* Protocol 0, Call ID 2 */ 490 call.rax = SVSM_CORE_CALL(SVSM_CORE_CREATE_VCPU); 491 call.rdx = __pa(caa); 492 call.r8 = apic_id; 493 } else { 494 /* Protocol 0, Call ID 3 */ 495 call.rax = SVSM_CORE_CALL(SVSM_CORE_DELETE_VCPU); 496 } 497 498 ret = svsm_perform_call_protocol(&call); 499 500 local_irq_restore(flags); 501 } else { 502 /* 503 * If the kernel runs at VMPL0, it can change the VMSA 504 * bit for a page using the RMPADJUST instruction. 505 * However, for the instruction to succeed it must 506 * target the permissions of a lesser privileged (higher 507 * numbered) VMPL level, so use VMPL1. 508 */ 509 u64 attrs = 1; 510 511 if (make_vmsa) 512 attrs |= RMPADJUST_VMSA_PAGE_BIT; 513 514 ret = rmpadjust((unsigned long)va, RMP_PG_SIZE_4K, attrs); 515 } 516 517 return ret; 518 } 519 520 static void snp_cleanup_vmsa(struct sev_es_save_area *vmsa, int apic_id) 521 { 522 int err; 523 524 err = snp_set_vmsa(vmsa, NULL, apic_id, false); 525 if (err) 526 pr_err("clear VMSA page failed (%u), leaking page\n", err); 527 else 528 free_page((unsigned long)vmsa); 529 } 530 531 static void set_pte_enc(pte_t *kpte, int level, void *va) 532 { 533 struct pte_enc_desc d = { 534 .kpte = kpte, 535 .pte_level = level, 536 .va = va, 537 .encrypt = true 538 }; 539 540 prepare_pte_enc(&d); 541 set_pte_enc_mask(kpte, d.pfn, d.new_pgprot); 542 } 543 544 static void unshare_all_memory(void) 545 { 546 unsigned long addr, end, size, ghcb; 547 struct sev_es_runtime_data *data; 548 unsigned int npages, level; 549 bool skipped_addr; 550 pte_t *pte; 551 int cpu; 552 553 /* Unshare the direct mapping. */ 554 addr = PAGE_OFFSET; 555 end = PAGE_OFFSET + get_max_mapped(); 556 557 while (addr < end) { 558 pte = lookup_address(addr, &level); 559 size = page_level_size(level); 560 npages = size / PAGE_SIZE; 561 skipped_addr = false; 562 563 if (!pte || !pte_decrypted(*pte) || pte_none(*pte)) { 564 addr += size; 565 continue; 566 } 567 568 /* 569 * Ensure that all the per-CPU GHCBs are made private at the 570 * end of the unsharing loop so that the switch to the slower 571 * MSR protocol happens last. 572 */ 573 for_each_possible_cpu(cpu) { 574 data = per_cpu(runtime_data, cpu); 575 ghcb = (unsigned long)&data->ghcb_page; 576 577 /* Handle the case of a huge page containing the GHCB page */ 578 if (addr <= ghcb && ghcb < addr + size) { 579 skipped_addr = true; 580 break; 581 } 582 } 583 584 if (!skipped_addr) { 585 set_pte_enc(pte, level, (void *)addr); 586 snp_set_memory_private(addr, npages); 587 } 588 addr += size; 589 } 590 591 /* Unshare all bss decrypted memory. */ 592 addr = (unsigned long)__start_bss_decrypted; 593 end = (unsigned long)__start_bss_decrypted_unused; 594 npages = (end - addr) >> PAGE_SHIFT; 595 596 for (; addr < end; addr += PAGE_SIZE) { 597 pte = lookup_address(addr, &level); 598 if (!pte || !pte_decrypted(*pte) || pte_none(*pte)) 599 continue; 600 601 set_pte_enc(pte, level, (void *)addr); 602 } 603 addr = (unsigned long)__start_bss_decrypted; 604 snp_set_memory_private(addr, npages); 605 606 __flush_tlb_all(); 607 } 608 609 /* Stop new private<->shared conversions */ 610 void snp_kexec_begin(void) 611 { 612 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 613 return; 614 615 if (!IS_ENABLED(CONFIG_KEXEC_CORE)) 616 return; 617 618 /* 619 * Crash kernel ends up here with interrupts disabled: can't wait for 620 * conversions to finish. 621 * 622 * If race happened, just report and proceed. 623 */ 624 if (!set_memory_enc_stop_conversion()) 625 pr_warn("Failed to stop shared<->private conversions\n"); 626 } 627 628 /* 629 * Shutdown all APs except the one handling kexec/kdump and clearing 630 * the VMSA tag on AP's VMSA pages as they are not being used as 631 * VMSA page anymore. 632 */ 633 static void shutdown_all_aps(void) 634 { 635 struct sev_es_save_area *vmsa; 636 int apic_id, this_cpu, cpu; 637 638 this_cpu = get_cpu(); 639 640 /* 641 * APs are already in HLT loop when enc_kexec_finish() callback 642 * is invoked. 643 */ 644 for_each_present_cpu(cpu) { 645 vmsa = per_cpu(sev_vmsa, cpu); 646 647 /* 648 * The BSP or offlined APs do not have guest allocated VMSA 649 * and there is no need to clear the VMSA tag for this page. 650 */ 651 if (!vmsa) 652 continue; 653 654 /* 655 * Cannot clear the VMSA tag for the currently running vCPU. 656 */ 657 if (this_cpu == cpu) { 658 unsigned long pa; 659 struct page *p; 660 661 pa = __pa(vmsa); 662 /* 663 * Mark the VMSA page of the running vCPU as offline 664 * so that is excluded and not touched by makedumpfile 665 * while generating vmcore during kdump. 666 */ 667 p = pfn_to_online_page(pa >> PAGE_SHIFT); 668 if (p) 669 __SetPageOffline(p); 670 continue; 671 } 672 673 apic_id = cpuid_to_apicid[cpu]; 674 675 /* 676 * Issue AP destroy to ensure AP gets kicked out of guest mode 677 * to allow using RMPADJUST to remove the VMSA tag on it's 678 * VMSA page. 679 */ 680 vmgexit_ap_control(SVM_VMGEXIT_AP_DESTROY, vmsa, apic_id); 681 snp_cleanup_vmsa(vmsa, apic_id); 682 } 683 684 put_cpu(); 685 } 686 687 void snp_kexec_finish(void) 688 { 689 struct sev_es_runtime_data *data; 690 unsigned long size, addr; 691 unsigned int level, cpu; 692 struct ghcb *ghcb; 693 pte_t *pte; 694 695 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 696 return; 697 698 if (!IS_ENABLED(CONFIG_KEXEC_CORE)) 699 return; 700 701 shutdown_all_aps(); 702 703 unshare_all_memory(); 704 705 /* 706 * Switch to using the MSR protocol to change per-CPU GHCBs to 707 * private. All the per-CPU GHCBs have been switched back to private, 708 * so can't do any more GHCB calls to the hypervisor beyond this point 709 * until the kexec'ed kernel starts running. 710 */ 711 boot_ghcb = NULL; 712 sev_cfg.ghcbs_initialized = false; 713 714 for_each_possible_cpu(cpu) { 715 data = per_cpu(runtime_data, cpu); 716 ghcb = &data->ghcb_page; 717 pte = lookup_address((unsigned long)ghcb, &level); 718 size = page_level_size(level); 719 /* Handle the case of a huge page containing the GHCB page */ 720 addr = (unsigned long)ghcb & page_level_mask(level); 721 set_pte_enc(pte, level, (void *)addr); 722 snp_set_memory_private(addr, (size / PAGE_SIZE)); 723 } 724 } 725 726 #define __ATTR_BASE (SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK) 727 #define INIT_CS_ATTRIBS (__ATTR_BASE | SVM_SELECTOR_READ_MASK | SVM_SELECTOR_CODE_MASK) 728 #define INIT_DS_ATTRIBS (__ATTR_BASE | SVM_SELECTOR_WRITE_MASK) 729 730 #define INIT_LDTR_ATTRIBS (SVM_SELECTOR_P_MASK | 2) 731 #define INIT_TR_ATTRIBS (SVM_SELECTOR_P_MASK | 3) 732 733 static void *snp_alloc_vmsa_page(int cpu) 734 { 735 struct page *p; 736 737 /* 738 * Allocate VMSA page to work around the SNP erratum where the CPU will 739 * incorrectly signal an RMP violation #PF if a large page (2MB or 1GB) 740 * collides with the RMP entry of VMSA page. The recommended workaround 741 * is to not use a large page. 742 * 743 * Allocate an 8k page which is also 8k-aligned. 744 */ 745 p = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL_ACCOUNT | __GFP_ZERO, 1); 746 if (!p) 747 return NULL; 748 749 split_page(p, 1); 750 751 /* Free the first 4k. This page may be 2M/1G aligned and cannot be used. */ 752 __free_page(p); 753 754 return page_address(p + 1); 755 } 756 757 static int wakeup_cpu_via_vmgexit(u32 apic_id, unsigned long start_ip, unsigned int cpu) 758 { 759 struct sev_es_save_area *cur_vmsa, *vmsa; 760 struct svsm_ca *caa; 761 u8 sipi_vector; 762 int ret; 763 u64 cr4; 764 765 /* 766 * The hypervisor SNP feature support check has happened earlier, just check 767 * the AP_CREATION one here. 768 */ 769 if (!(sev_hv_features & GHCB_HV_FT_SNP_AP_CREATION)) 770 return -EOPNOTSUPP; 771 772 /* 773 * Verify the desired start IP against the known trampoline start IP 774 * to catch any future new trampolines that may be introduced that 775 * would require a new protected guest entry point. 776 */ 777 if (WARN_ONCE(start_ip != real_mode_header->trampoline_start, 778 "Unsupported SNP start_ip: %lx\n", start_ip)) 779 return -EINVAL; 780 781 /* Override start_ip with known protected guest start IP */ 782 start_ip = real_mode_header->sev_es_trampoline_start; 783 cur_vmsa = per_cpu(sev_vmsa, cpu); 784 785 /* 786 * A new VMSA is created each time because there is no guarantee that 787 * the current VMSA is the kernels or that the vCPU is not running. If 788 * an attempt was done to use the current VMSA with a running vCPU, a 789 * #VMEXIT of that vCPU would wipe out all of the settings being done 790 * here. 791 */ 792 vmsa = (struct sev_es_save_area *)snp_alloc_vmsa_page(cpu); 793 if (!vmsa) 794 return -ENOMEM; 795 796 /* If an SVSM is present, the SVSM per-CPU CAA will be !NULL */ 797 caa = per_cpu(svsm_caa, cpu); 798 799 /* CR4 should maintain the MCE value */ 800 cr4 = native_read_cr4() & X86_CR4_MCE; 801 802 /* Set the CS value based on the start_ip converted to a SIPI vector */ 803 sipi_vector = (start_ip >> 12); 804 vmsa->cs.base = sipi_vector << 12; 805 vmsa->cs.limit = AP_INIT_CS_LIMIT; 806 vmsa->cs.attrib = INIT_CS_ATTRIBS; 807 vmsa->cs.selector = sipi_vector << 8; 808 809 /* Set the RIP value based on start_ip */ 810 vmsa->rip = start_ip & 0xfff; 811 812 /* Set AP INIT defaults as documented in the APM */ 813 vmsa->ds.limit = AP_INIT_DS_LIMIT; 814 vmsa->ds.attrib = INIT_DS_ATTRIBS; 815 vmsa->es = vmsa->ds; 816 vmsa->fs = vmsa->ds; 817 vmsa->gs = vmsa->ds; 818 vmsa->ss = vmsa->ds; 819 820 vmsa->gdtr.limit = AP_INIT_GDTR_LIMIT; 821 vmsa->ldtr.limit = AP_INIT_LDTR_LIMIT; 822 vmsa->ldtr.attrib = INIT_LDTR_ATTRIBS; 823 vmsa->idtr.limit = AP_INIT_IDTR_LIMIT; 824 vmsa->tr.limit = AP_INIT_TR_LIMIT; 825 vmsa->tr.attrib = INIT_TR_ATTRIBS; 826 827 vmsa->cr4 = cr4; 828 vmsa->cr0 = AP_INIT_CR0_DEFAULT; 829 vmsa->dr7 = DR7_RESET_VALUE; 830 vmsa->dr6 = AP_INIT_DR6_DEFAULT; 831 vmsa->rflags = AP_INIT_RFLAGS_DEFAULT; 832 vmsa->g_pat = AP_INIT_GPAT_DEFAULT; 833 vmsa->xcr0 = AP_INIT_XCR0_DEFAULT; 834 vmsa->mxcsr = AP_INIT_MXCSR_DEFAULT; 835 vmsa->x87_ftw = AP_INIT_X87_FTW_DEFAULT; 836 vmsa->x87_fcw = AP_INIT_X87_FCW_DEFAULT; 837 838 if (cc_platform_has(CC_ATTR_SNP_SECURE_AVIC)) 839 vmsa->vintr_ctrl |= V_GIF_MASK | V_NMI_ENABLE_MASK; 840 841 /* SVME must be set. */ 842 vmsa->efer = EFER_SVME; 843 844 /* 845 * Set the SNP-specific fields for this VMSA: 846 * VMPL level 847 * SEV_FEATURES (matches the SEV STATUS MSR right shifted 2 bits) 848 */ 849 vmsa->vmpl = snp_vmpl; 850 vmsa->sev_features = sev_status >> 2; 851 852 /* Populate AP's TSC scale/offset to get accurate TSC values. */ 853 if (cc_platform_has(CC_ATTR_GUEST_SNP_SECURE_TSC)) { 854 vmsa->tsc_scale = snp_tsc_scale; 855 vmsa->tsc_offset = snp_tsc_offset; 856 } 857 858 /* Switch the page over to a VMSA page now that it is initialized */ 859 ret = snp_set_vmsa(vmsa, caa, apic_id, true); 860 if (ret) { 861 pr_err("set VMSA page failed (%u)\n", ret); 862 free_page((unsigned long)vmsa); 863 864 return -EINVAL; 865 } 866 867 /* Issue VMGEXIT AP Creation NAE event */ 868 ret = vmgexit_ap_control(SVM_VMGEXIT_AP_CREATE, vmsa, apic_id); 869 if (ret) { 870 snp_cleanup_vmsa(vmsa, apic_id); 871 vmsa = NULL; 872 } 873 874 /* Free up any previous VMSA page */ 875 if (cur_vmsa) 876 snp_cleanup_vmsa(cur_vmsa, apic_id); 877 878 /* Record the current VMSA page */ 879 per_cpu(sev_vmsa, cpu) = vmsa; 880 881 return ret; 882 } 883 884 void __init snp_set_wakeup_secondary_cpu(void) 885 { 886 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 887 return; 888 889 /* 890 * Always set this override if SNP is enabled. This makes it the 891 * required method to start APs under SNP. If the hypervisor does 892 * not support AP creation, then no APs will be started. 893 */ 894 apic_update_callback(wakeup_secondary_cpu, wakeup_cpu_via_vmgexit); 895 } 896 897 int __init sev_es_setup_ap_jump_table(struct real_mode_header *rmh) 898 { 899 u16 startup_cs, startup_ip; 900 phys_addr_t jump_table_pa; 901 u64 jump_table_addr; 902 u16 __iomem *jump_table; 903 904 jump_table_addr = get_jump_table_addr(); 905 906 /* On UP guests there is no jump table so this is not a failure */ 907 if (!jump_table_addr) 908 return 0; 909 910 /* Check if AP Jump Table is page-aligned */ 911 if (jump_table_addr & ~PAGE_MASK) 912 return -EINVAL; 913 914 jump_table_pa = jump_table_addr & PAGE_MASK; 915 916 startup_cs = (u16)(rmh->trampoline_start >> 4); 917 startup_ip = (u16)(rmh->sev_es_trampoline_start - 918 rmh->trampoline_start); 919 920 jump_table = ioremap_encrypted(jump_table_pa, PAGE_SIZE); 921 if (!jump_table) 922 return -EIO; 923 924 writew(startup_ip, &jump_table[0]); 925 writew(startup_cs, &jump_table[1]); 926 927 iounmap(jump_table); 928 929 return 0; 930 } 931 932 /* 933 * This is needed by the OVMF UEFI firmware which will use whatever it finds in 934 * the GHCB MSR as its GHCB to talk to the hypervisor. So make sure the per-cpu 935 * runtime GHCBs used by the kernel are also mapped in the EFI page-table. 936 * 937 * When running under SVSM the CA page is needed too, so map it as well. 938 */ 939 int __init sev_es_efi_map_ghcbs_cas(pgd_t *pgd) 940 { 941 unsigned long address, pflags, pflags_enc; 942 struct sev_es_runtime_data *data; 943 int cpu; 944 u64 pfn; 945 946 if (!cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) 947 return 0; 948 949 pflags = _PAGE_NX | _PAGE_RW; 950 pflags_enc = cc_mkenc(pflags); 951 952 for_each_possible_cpu(cpu) { 953 data = per_cpu(runtime_data, cpu); 954 955 address = __pa(&data->ghcb_page); 956 pfn = address >> PAGE_SHIFT; 957 958 if (kernel_map_pages_in_pgd(pgd, pfn, address, 1, pflags)) 959 return 1; 960 961 if (snp_vmpl) { 962 address = per_cpu(svsm_caa_pa, cpu); 963 if (!address) 964 return 1; 965 966 pfn = address >> PAGE_SHIFT; 967 if (kernel_map_pages_in_pgd(pgd, pfn, address, 1, pflags_enc)) 968 return 1; 969 } 970 } 971 972 return 0; 973 } 974 975 u64 savic_ghcb_msr_read(u32 reg) 976 { 977 u64 msr = APIC_BASE_MSR + (reg >> 4); 978 struct pt_regs regs = { .cx = msr }; 979 struct es_em_ctxt ctxt = { .regs = ®s }; 980 struct ghcb_state state; 981 enum es_result res; 982 struct ghcb *ghcb; 983 984 guard(irqsave)(); 985 986 ghcb = __sev_get_ghcb(&state); 987 vc_ghcb_invalidate(ghcb); 988 989 res = __vc_handle_msr(ghcb, &ctxt, false); 990 if (res != ES_OK) { 991 pr_err("Secure AVIC MSR (0x%llx) read returned error (%d)\n", msr, res); 992 /* MSR read failures are treated as fatal errors */ 993 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SAVIC_FAIL); 994 } 995 996 __sev_put_ghcb(&state); 997 998 return regs.ax | regs.dx << 32; 999 } 1000 1001 void savic_ghcb_msr_write(u32 reg, u64 value) 1002 { 1003 u64 msr = APIC_BASE_MSR + (reg >> 4); 1004 struct pt_regs regs = { 1005 .cx = msr, 1006 .ax = lower_32_bits(value), 1007 .dx = upper_32_bits(value) 1008 }; 1009 struct es_em_ctxt ctxt = { .regs = ®s }; 1010 struct ghcb_state state; 1011 enum es_result res; 1012 struct ghcb *ghcb; 1013 1014 guard(irqsave)(); 1015 1016 ghcb = __sev_get_ghcb(&state); 1017 vc_ghcb_invalidate(ghcb); 1018 1019 res = __vc_handle_msr(ghcb, &ctxt, true); 1020 if (res != ES_OK) { 1021 pr_err("Secure AVIC MSR (0x%llx) write returned error (%d)\n", msr, res); 1022 /* MSR writes should never fail. Any failure is fatal error for SNP guest */ 1023 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SAVIC_FAIL); 1024 } 1025 1026 __sev_put_ghcb(&state); 1027 } 1028 1029 enum es_result savic_register_gpa(u64 gpa) 1030 { 1031 struct ghcb_state state; 1032 struct es_em_ctxt ctxt; 1033 enum es_result res; 1034 struct ghcb *ghcb; 1035 1036 guard(irqsave)(); 1037 1038 ghcb = __sev_get_ghcb(&state); 1039 vc_ghcb_invalidate(ghcb); 1040 1041 ghcb_set_rax(ghcb, SVM_VMGEXIT_SAVIC_SELF_GPA); 1042 ghcb_set_rbx(ghcb, gpa); 1043 res = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_SAVIC, 1044 SVM_VMGEXIT_SAVIC_REGISTER_GPA, 0); 1045 1046 __sev_put_ghcb(&state); 1047 1048 return res; 1049 } 1050 1051 enum es_result savic_unregister_gpa(u64 *gpa) 1052 { 1053 struct ghcb_state state; 1054 struct es_em_ctxt ctxt; 1055 enum es_result res; 1056 struct ghcb *ghcb; 1057 1058 guard(irqsave)(); 1059 1060 ghcb = __sev_get_ghcb(&state); 1061 vc_ghcb_invalidate(ghcb); 1062 1063 ghcb_set_rax(ghcb, SVM_VMGEXIT_SAVIC_SELF_GPA); 1064 res = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_SAVIC, 1065 SVM_VMGEXIT_SAVIC_UNREGISTER_GPA, 0); 1066 if (gpa && res == ES_OK) 1067 *gpa = ghcb->save.rbx; 1068 1069 __sev_put_ghcb(&state); 1070 1071 return res; 1072 } 1073 1074 static void snp_register_per_cpu_ghcb(void) 1075 { 1076 struct sev_es_runtime_data *data; 1077 struct ghcb *ghcb; 1078 1079 data = this_cpu_read(runtime_data); 1080 ghcb = &data->ghcb_page; 1081 1082 snp_register_ghcb_early(__pa(ghcb)); 1083 } 1084 1085 void setup_ghcb(void) 1086 { 1087 if (!cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) 1088 return; 1089 1090 /* 1091 * Check whether the runtime #VC exception handler is active. It uses 1092 * the per-CPU GHCB page which is set up by sev_es_init_vc_handling(). 1093 * 1094 * If SNP is active, register the per-CPU GHCB page so that the runtime 1095 * exception handler can use it. 1096 */ 1097 if (initial_vc_handler == (unsigned long)kernel_exc_vmm_communication) { 1098 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 1099 snp_register_per_cpu_ghcb(); 1100 1101 sev_cfg.ghcbs_initialized = true; 1102 1103 return; 1104 } 1105 1106 /* 1107 * Make sure the hypervisor talks a supported protocol. 1108 * This gets called only in the BSP boot phase. 1109 */ 1110 if (!sev_es_negotiate_protocol()) 1111 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); 1112 1113 /* 1114 * Clear the boot_ghcb. The first exception comes in before the bss 1115 * section is cleared. 1116 */ 1117 memset(&boot_ghcb_page, 0, PAGE_SIZE); 1118 1119 /* Alright - Make the boot-ghcb public */ 1120 boot_ghcb = &boot_ghcb_page; 1121 1122 /* SNP guest requires that GHCB GPA must be registered. */ 1123 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 1124 snp_register_ghcb_early(__pa(&boot_ghcb_page)); 1125 } 1126 1127 #ifdef CONFIG_HOTPLUG_CPU 1128 static void sev_es_ap_hlt_loop(void) 1129 { 1130 struct ghcb_state state; 1131 struct ghcb *ghcb; 1132 1133 ghcb = __sev_get_ghcb(&state); 1134 1135 while (true) { 1136 vc_ghcb_invalidate(ghcb); 1137 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_AP_HLT_LOOP); 1138 ghcb_set_sw_exit_info_1(ghcb, 0); 1139 ghcb_set_sw_exit_info_2(ghcb, 0); 1140 1141 sev_es_wr_ghcb_msr(__pa(ghcb)); 1142 VMGEXIT(); 1143 1144 /* Wakeup signal? */ 1145 if (ghcb_sw_exit_info_2_is_valid(ghcb) && 1146 ghcb->save.sw_exit_info_2) 1147 break; 1148 } 1149 1150 __sev_put_ghcb(&state); 1151 } 1152 1153 /* 1154 * Play_dead handler when running under SEV-ES. This is needed because 1155 * the hypervisor can't deliver an SIPI request to restart the AP. 1156 * Instead the kernel has to issue a VMGEXIT to halt the VCPU until the 1157 * hypervisor wakes it up again. 1158 */ 1159 static void sev_es_play_dead(void) 1160 { 1161 play_dead_common(); 1162 1163 /* IRQs now disabled */ 1164 1165 sev_es_ap_hlt_loop(); 1166 1167 /* 1168 * If we get here, the VCPU was woken up again. Jump to CPU 1169 * startup code to get it back online. 1170 */ 1171 soft_restart_cpu(); 1172 } 1173 #else /* CONFIG_HOTPLUG_CPU */ 1174 #define sev_es_play_dead native_play_dead 1175 #endif /* CONFIG_HOTPLUG_CPU */ 1176 1177 #ifdef CONFIG_SMP 1178 static void __init sev_es_setup_play_dead(void) 1179 { 1180 smp_ops.play_dead = sev_es_play_dead; 1181 } 1182 #else 1183 static inline void sev_es_setup_play_dead(void) { } 1184 #endif 1185 1186 static void __init alloc_runtime_data(int cpu) 1187 { 1188 struct sev_es_runtime_data *data; 1189 1190 data = memblock_alloc_node(sizeof(*data), PAGE_SIZE, cpu_to_node(cpu)); 1191 if (!data) 1192 panic("Can't allocate SEV-ES runtime data"); 1193 1194 per_cpu(runtime_data, cpu) = data; 1195 1196 if (snp_vmpl) { 1197 struct svsm_ca *caa; 1198 1199 /* Allocate the SVSM CA page if an SVSM is present */ 1200 caa = cpu ? memblock_alloc_or_panic(sizeof(*caa), PAGE_SIZE) 1201 : &boot_svsm_ca_page; 1202 1203 per_cpu(svsm_caa, cpu) = caa; 1204 per_cpu(svsm_caa_pa, cpu) = __pa(caa); 1205 } 1206 } 1207 1208 static void __init init_ghcb(int cpu) 1209 { 1210 struct sev_es_runtime_data *data; 1211 int err; 1212 1213 data = per_cpu(runtime_data, cpu); 1214 1215 err = early_set_memory_decrypted((unsigned long)&data->ghcb_page, 1216 sizeof(data->ghcb_page)); 1217 if (err) 1218 panic("Can't map GHCBs unencrypted"); 1219 1220 memset(&data->ghcb_page, 0, sizeof(data->ghcb_page)); 1221 1222 data->ghcb_active = false; 1223 data->backup_ghcb_active = false; 1224 } 1225 1226 void __init sev_es_init_vc_handling(void) 1227 { 1228 int cpu; 1229 1230 BUILD_BUG_ON(offsetof(struct sev_es_runtime_data, ghcb_page) % PAGE_SIZE); 1231 1232 if (!cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) 1233 return; 1234 1235 if (!sev_es_check_cpu_features()) 1236 panic("SEV-ES CPU Features missing"); 1237 1238 /* 1239 * SNP is supported in v2 of the GHCB spec which mandates support for HV 1240 * features. 1241 */ 1242 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) { 1243 sev_hv_features = get_hv_features(); 1244 1245 if (!(sev_hv_features & GHCB_HV_FT_SNP)) 1246 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED); 1247 } 1248 1249 /* Initialize per-cpu GHCB pages */ 1250 for_each_possible_cpu(cpu) { 1251 alloc_runtime_data(cpu); 1252 init_ghcb(cpu); 1253 } 1254 1255 if (snp_vmpl) 1256 sev_cfg.use_cas = true; 1257 1258 sev_es_setup_play_dead(); 1259 1260 /* Secondary CPUs use the runtime #VC handler */ 1261 initial_vc_handler = (unsigned long)kernel_exc_vmm_communication; 1262 } 1263 1264 /* 1265 * SEV-SNP guests should only execute dmi_setup() if EFI_CONFIG_TABLES are 1266 * enabled, as the alternative (fallback) logic for DMI probing in the legacy 1267 * ROM region can cause a crash since this region is not pre-validated. 1268 */ 1269 void __init snp_dmi_setup(void) 1270 { 1271 if (efi_enabled(EFI_CONFIG_TABLES)) 1272 dmi_setup(); 1273 } 1274 1275 static void dump_cpuid_table(void) 1276 { 1277 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); 1278 int i = 0; 1279 1280 pr_info("count=%d reserved=0x%x reserved2=0x%llx\n", 1281 cpuid_table->count, cpuid_table->__reserved1, cpuid_table->__reserved2); 1282 1283 for (i = 0; i < SNP_CPUID_COUNT_MAX; i++) { 1284 const struct snp_cpuid_fn *fn = &cpuid_table->fn[i]; 1285 1286 pr_info("index=%3d fn=0x%08x subfn=0x%08x: eax=0x%08x ebx=0x%08x ecx=0x%08x edx=0x%08x xcr0_in=0x%016llx xss_in=0x%016llx reserved=0x%016llx\n", 1287 i, fn->eax_in, fn->ecx_in, fn->eax, fn->ebx, fn->ecx, 1288 fn->edx, fn->xcr0_in, fn->xss_in, fn->__reserved); 1289 } 1290 } 1291 1292 /* 1293 * It is useful from an auditing/testing perspective to provide an easy way 1294 * for the guest owner to know that the CPUID table has been initialized as 1295 * expected, but that initialization happens too early in boot to print any 1296 * sort of indicator, and there's not really any other good place to do it, 1297 * so do it here. 1298 * 1299 * If running as an SNP guest, report the current VM privilege level (VMPL). 1300 */ 1301 static int __init report_snp_info(void) 1302 { 1303 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); 1304 1305 if (cpuid_table->count) { 1306 pr_info("Using SNP CPUID table, %d entries present.\n", 1307 cpuid_table->count); 1308 1309 if (sev_cfg.debug) 1310 dump_cpuid_table(); 1311 } 1312 1313 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 1314 pr_info("SNP running at VMPL%u.\n", snp_vmpl); 1315 1316 return 0; 1317 } 1318 arch_initcall(report_snp_info); 1319 1320 static int snp_issue_guest_request(struct snp_guest_req *req) 1321 { 1322 struct snp_req_data *input = &req->input; 1323 struct ghcb_state state; 1324 struct es_em_ctxt ctxt; 1325 unsigned long flags; 1326 struct ghcb *ghcb; 1327 int ret; 1328 1329 req->exitinfo2 = SEV_RET_NO_FW_CALL; 1330 1331 /* 1332 * __sev_get_ghcb() needs to run with IRQs disabled because it is using 1333 * a per-CPU GHCB. 1334 */ 1335 local_irq_save(flags); 1336 1337 ghcb = __sev_get_ghcb(&state); 1338 if (!ghcb) { 1339 ret = -EIO; 1340 goto e_restore_irq; 1341 } 1342 1343 vc_ghcb_invalidate(ghcb); 1344 1345 if (req->exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST) { 1346 ghcb_set_rax(ghcb, input->data_gpa); 1347 ghcb_set_rbx(ghcb, input->data_npages); 1348 } 1349 1350 ret = sev_es_ghcb_hv_call(ghcb, &ctxt, req->exit_code, input->req_gpa, input->resp_gpa); 1351 if (ret) 1352 goto e_put; 1353 1354 req->exitinfo2 = ghcb->save.sw_exit_info_2; 1355 switch (req->exitinfo2) { 1356 case 0: 1357 break; 1358 1359 case SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_BUSY): 1360 ret = -EAGAIN; 1361 break; 1362 1363 case SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN): 1364 /* Number of expected pages are returned in RBX */ 1365 if (req->exit_code == SVM_VMGEXIT_EXT_GUEST_REQUEST) { 1366 input->data_npages = ghcb_get_rbx(ghcb); 1367 ret = -ENOSPC; 1368 break; 1369 } 1370 fallthrough; 1371 default: 1372 ret = -EIO; 1373 break; 1374 } 1375 1376 e_put: 1377 __sev_put_ghcb(&state); 1378 e_restore_irq: 1379 local_irq_restore(flags); 1380 1381 return ret; 1382 } 1383 1384 static struct platform_device sev_guest_device = { 1385 .name = "sev-guest", 1386 .id = -1, 1387 }; 1388 1389 static struct platform_device tpm_svsm_device = { 1390 .name = "tpm-svsm", 1391 .id = -1, 1392 }; 1393 1394 static int __init snp_init_platform_device(void) 1395 { 1396 if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) 1397 return -ENODEV; 1398 1399 if (platform_device_register(&sev_guest_device)) 1400 return -ENODEV; 1401 1402 if (snp_svsm_vtpm_probe() && 1403 platform_device_register(&tpm_svsm_device)) 1404 return -ENODEV; 1405 1406 pr_info("SNP guest platform devices initialized.\n"); 1407 return 0; 1408 } 1409 device_initcall(snp_init_platform_device); 1410 1411 void sev_show_status(void) 1412 { 1413 int i; 1414 1415 pr_info("Status: "); 1416 for (i = 0; i < MSR_AMD64_SNP_RESV_BIT; i++) { 1417 if (sev_status & BIT_ULL(i)) { 1418 if (!sev_status_feat_names[i]) 1419 continue; 1420 1421 pr_cont("%s ", sev_status_feat_names[i]); 1422 } 1423 } 1424 pr_cont("\n"); 1425 } 1426 1427 #ifdef CONFIG_SYSFS 1428 static ssize_t vmpl_show(struct kobject *kobj, 1429 struct kobj_attribute *attr, char *buf) 1430 { 1431 return sysfs_emit(buf, "%d\n", snp_vmpl); 1432 } 1433 1434 static ssize_t sev_status_show(struct kobject *kobj, 1435 struct kobj_attribute *attr, char *buf) 1436 { 1437 return sysfs_emit(buf, "0x%llx\n", sev_status); 1438 } 1439 1440 static struct kobj_attribute vmpl_attr = __ATTR_RO(vmpl); 1441 static struct kobj_attribute sev_status_attr = __ATTR_RO(sev_status); 1442 1443 static struct attribute *sev_status_attrs[] = { 1444 &sev_status_attr.attr, 1445 NULL 1446 }; 1447 1448 static struct attribute_group sev_attr_group = { 1449 .attrs = sev_status_attrs, 1450 }; 1451 1452 static int __init sev_sysfs_init(void) 1453 { 1454 struct kobject *sev_kobj; 1455 struct device *dev_root; 1456 int ret; 1457 1458 if (!(sev_status & MSR_AMD64_SEV_ENABLED)) 1459 return -ENODEV; 1460 1461 dev_root = bus_get_dev_root(&cpu_subsys); 1462 if (!dev_root) 1463 return -ENODEV; 1464 1465 sev_kobj = kobject_create_and_add("sev", &dev_root->kobj); 1466 put_device(dev_root); 1467 1468 if (!sev_kobj) 1469 return -ENOMEM; 1470 1471 ret = sysfs_create_group(sev_kobj, &sev_attr_group); 1472 if (ret) 1473 goto drop_kobj; 1474 1475 if (sev_status & MSR_AMD64_SEV_SNP_ENABLED) { 1476 ret = sysfs_add_file_to_group(sev_kobj, &vmpl_attr.attr, NULL); 1477 if (ret) 1478 goto drop_sysfs; 1479 } 1480 1481 return 0; 1482 1483 drop_sysfs: 1484 sysfs_remove_group(sev_kobj, &sev_attr_group); 1485 drop_kobj: 1486 kobject_put(sev_kobj); 1487 1488 return ret; 1489 } 1490 arch_initcall(sev_sysfs_init); 1491 #endif // CONFIG_SYSFS 1492 1493 static void free_shared_pages(void *buf, size_t sz) 1494 { 1495 unsigned int npages = PAGE_ALIGN(sz) >> PAGE_SHIFT; 1496 int ret; 1497 1498 if (!buf) 1499 return; 1500 1501 ret = set_memory_encrypted((unsigned long)buf, npages); 1502 if (ret) { 1503 WARN_ONCE(ret, "failed to restore encryption mask (leak it)\n"); 1504 return; 1505 } 1506 1507 __free_pages(virt_to_page(buf), get_order(sz)); 1508 } 1509 1510 static void *alloc_shared_pages(size_t sz) 1511 { 1512 unsigned int npages = PAGE_ALIGN(sz) >> PAGE_SHIFT; 1513 struct page *page; 1514 int ret; 1515 1516 page = alloc_pages(GFP_KERNEL_ACCOUNT, get_order(sz)); 1517 if (!page) 1518 return NULL; 1519 1520 ret = set_memory_decrypted((unsigned long)page_address(page), npages); 1521 if (ret) { 1522 pr_err("failed to mark page shared, ret=%d\n", ret); 1523 __free_pages(page, get_order(sz)); 1524 return NULL; 1525 } 1526 1527 return page_address(page); 1528 } 1529 1530 static u8 *get_vmpck(int id, struct snp_secrets_page *secrets, u32 **seqno) 1531 { 1532 u8 *key = NULL; 1533 1534 switch (id) { 1535 case 0: 1536 *seqno = &secrets->os_area.msg_seqno_0; 1537 key = secrets->vmpck0; 1538 break; 1539 case 1: 1540 *seqno = &secrets->os_area.msg_seqno_1; 1541 key = secrets->vmpck1; 1542 break; 1543 case 2: 1544 *seqno = &secrets->os_area.msg_seqno_2; 1545 key = secrets->vmpck2; 1546 break; 1547 case 3: 1548 *seqno = &secrets->os_area.msg_seqno_3; 1549 key = secrets->vmpck3; 1550 break; 1551 default: 1552 break; 1553 } 1554 1555 return key; 1556 } 1557 1558 static struct aes_gcm_key *snp_init_crypto(const u8 *key, size_t keylen) 1559 { 1560 struct aes_gcm_key *gcm_key; 1561 1562 gcm_key = kzalloc_obj(*gcm_key); 1563 if (!gcm_key) 1564 return NULL; 1565 1566 if (aes_gcm_preparekey(gcm_key, key, keylen, AUTHTAG_LEN)) { 1567 pr_err("AES-GCM key preparation failed\n"); 1568 kfree_sensitive(gcm_key); 1569 return NULL; 1570 } 1571 1572 return gcm_key; 1573 } 1574 1575 int snp_msg_init(struct snp_msg_desc *mdesc, int vmpck_id) 1576 { 1577 /* Adjust the default VMPCK key based on the executing VMPL level */ 1578 if (vmpck_id == -1) 1579 vmpck_id = snp_vmpl; 1580 1581 mdesc->vmpck = get_vmpck(vmpck_id, mdesc->secrets, &mdesc->os_area_msg_seqno); 1582 if (!mdesc->vmpck) { 1583 pr_err("Invalid VMPCK%d communication key\n", vmpck_id); 1584 return -EINVAL; 1585 } 1586 1587 /* Verify that VMPCK is not zero. */ 1588 if (!memchr_inv(mdesc->vmpck, 0, VMPCK_KEY_LEN)) { 1589 pr_err("Empty VMPCK%d communication key\n", vmpck_id); 1590 return -EINVAL; 1591 } 1592 1593 mdesc->vmpck_id = vmpck_id; 1594 1595 mdesc->gcm_key = snp_init_crypto(mdesc->vmpck, VMPCK_KEY_LEN); 1596 if (!mdesc->gcm_key) 1597 return -ENOMEM; 1598 1599 return 0; 1600 } 1601 EXPORT_SYMBOL_GPL(snp_msg_init); 1602 1603 struct snp_msg_desc *snp_msg_alloc(void) 1604 { 1605 struct snp_msg_desc *mdesc; 1606 void __iomem *mem; 1607 1608 BUILD_BUG_ON(sizeof(struct snp_guest_msg) > PAGE_SIZE); 1609 1610 mdesc = kzalloc_obj(struct snp_msg_desc); 1611 if (!mdesc) 1612 return ERR_PTR(-ENOMEM); 1613 1614 mem = ioremap_encrypted(sev_secrets_pa, PAGE_SIZE); 1615 if (!mem) 1616 goto e_free_mdesc; 1617 1618 mdesc->secrets = (__force struct snp_secrets_page *)mem; 1619 1620 /* Allocate the shared page used for the request and response message. */ 1621 mdesc->request = alloc_shared_pages(sizeof(struct snp_guest_msg)); 1622 if (!mdesc->request) 1623 goto e_unmap; 1624 1625 mdesc->response = alloc_shared_pages(sizeof(struct snp_guest_msg)); 1626 if (!mdesc->response) 1627 goto e_free_request; 1628 1629 return mdesc; 1630 1631 e_free_request: 1632 free_shared_pages(mdesc->request, sizeof(struct snp_guest_msg)); 1633 e_unmap: 1634 iounmap(mem); 1635 e_free_mdesc: 1636 kfree(mdesc); 1637 1638 return ERR_PTR(-ENOMEM); 1639 } 1640 EXPORT_SYMBOL_GPL(snp_msg_alloc); 1641 1642 void snp_msg_free(struct snp_msg_desc *mdesc) 1643 { 1644 if (!mdesc) 1645 return; 1646 1647 kfree_sensitive(mdesc->gcm_key); 1648 free_shared_pages(mdesc->response, sizeof(struct snp_guest_msg)); 1649 free_shared_pages(mdesc->request, sizeof(struct snp_guest_msg)); 1650 iounmap((__force void __iomem *)mdesc->secrets); 1651 1652 kfree_sensitive(mdesc); 1653 } 1654 EXPORT_SYMBOL_GPL(snp_msg_free); 1655 1656 /* Mutex to serialize the shared buffer access and command handling. */ 1657 static DEFINE_MUTEX(snp_cmd_mutex); 1658 1659 /* 1660 * If an error is received from the host or AMD Secure Processor (ASP) there 1661 * are two options. Either retry the exact same encrypted request or discontinue 1662 * using the VMPCK. 1663 * 1664 * This is because in the current encryption scheme GHCB v2 uses AES-GCM to 1665 * encrypt the requests. The IV for this scheme is the sequence number. GCM 1666 * cannot tolerate IV reuse. 1667 * 1668 * The ASP FW v1.51 only increments the sequence numbers on a successful 1669 * guest<->ASP back and forth and only accepts messages at its exact sequence 1670 * number. 1671 * 1672 * So if the sequence number were to be reused the encryption scheme is 1673 * vulnerable. If the sequence number were incremented for a fresh IV the ASP 1674 * will reject the request. 1675 */ 1676 static void snp_disable_vmpck(struct snp_msg_desc *mdesc) 1677 { 1678 pr_alert("Disabling VMPCK%d communication key to prevent IV reuse.\n", 1679 mdesc->vmpck_id); 1680 memzero_explicit(mdesc->vmpck, VMPCK_KEY_LEN); 1681 mdesc->vmpck = NULL; 1682 } 1683 1684 static inline u64 __snp_get_msg_seqno(struct snp_msg_desc *mdesc) 1685 { 1686 u64 count; 1687 1688 lockdep_assert_held(&snp_cmd_mutex); 1689 1690 /* Read the current message sequence counter from secrets pages */ 1691 count = *mdesc->os_area_msg_seqno; 1692 1693 return count + 1; 1694 } 1695 1696 /* Return a non-zero on success */ 1697 static u64 snp_get_msg_seqno(struct snp_msg_desc *mdesc) 1698 { 1699 u64 count = __snp_get_msg_seqno(mdesc); 1700 1701 /* 1702 * The message sequence counter for the SNP guest request is a 64-bit 1703 * value but the version 2 of GHCB specification defines a 32-bit storage 1704 * for it. If the counter exceeds the 32-bit value then return zero. 1705 * The caller should check the return value, but if the caller happens to 1706 * not check the value and use it, then the firmware treats zero as an 1707 * invalid number and will fail the message request. 1708 */ 1709 if (count >= UINT_MAX) { 1710 pr_err("request message sequence counter overflow\n"); 1711 return 0; 1712 } 1713 1714 return count; 1715 } 1716 1717 static void snp_inc_msg_seqno(struct snp_msg_desc *mdesc) 1718 { 1719 /* 1720 * The counter is also incremented by the PSP, so increment it by 2 1721 * and save in secrets page. 1722 */ 1723 *mdesc->os_area_msg_seqno += 2; 1724 } 1725 1726 static int verify_and_dec_payload(struct snp_msg_desc *mdesc, struct snp_guest_req *req) 1727 { 1728 struct snp_guest_msg *resp_msg = &mdesc->secret_response; 1729 struct snp_guest_msg *req_msg = &mdesc->secret_request; 1730 struct snp_guest_msg_hdr *req_msg_hdr = &req_msg->hdr; 1731 struct snp_guest_msg_hdr *resp_msg_hdr = &resp_msg->hdr; 1732 struct aes_gcm_key *gcm_key = mdesc->gcm_key; 1733 u8 iv[GCM_AES_IV_SIZE] = {}; 1734 1735 pr_debug("response [seqno %lld type %d version %d sz %d]\n", 1736 resp_msg_hdr->msg_seqno, resp_msg_hdr->msg_type, resp_msg_hdr->msg_version, 1737 resp_msg_hdr->msg_sz); 1738 1739 /* Copy response from shared memory to encrypted memory. */ 1740 memcpy(resp_msg, mdesc->response, sizeof(*resp_msg)); 1741 1742 /* Verify that the sequence counter is incremented by 1 */ 1743 if (unlikely(resp_msg_hdr->msg_seqno != (req_msg_hdr->msg_seqno + 1))) 1744 return -EBADMSG; 1745 1746 /* Verify response message type and version number. */ 1747 if (resp_msg_hdr->msg_type != (req_msg_hdr->msg_type + 1) || 1748 resp_msg_hdr->msg_version != req_msg_hdr->msg_version) 1749 return -EBADMSG; 1750 1751 /* 1752 * If the message size is greater than our buffer length then return 1753 * an error. 1754 */ 1755 if (unlikely(resp_msg_hdr->msg_sz + AUTHTAG_LEN > req->resp_sz)) 1756 return -EBADMSG; 1757 1758 /* Decrypt the payload */ 1759 memcpy(iv, &resp_msg_hdr->msg_seqno, min(sizeof(iv), sizeof(resp_msg_hdr->msg_seqno))); 1760 return aes_gcm_decrypt(req->resp_buf, resp_msg->payload, 1761 resp_msg_hdr->msg_sz, resp_msg_hdr->authtag, 1762 &resp_msg_hdr->algo, AAD_LEN, iv, gcm_key); 1763 } 1764 1765 static int enc_payload(struct snp_msg_desc *mdesc, u64 seqno, struct snp_guest_req *req) 1766 { 1767 struct snp_guest_msg *msg = &mdesc->secret_request; 1768 struct snp_guest_msg_hdr *hdr = &msg->hdr; 1769 struct aes_gcm_key *gcm_key = mdesc->gcm_key; 1770 u8 iv[GCM_AES_IV_SIZE] = {}; 1771 1772 memset(msg, 0, sizeof(*msg)); 1773 1774 hdr->algo = SNP_AEAD_AES_256_GCM; 1775 hdr->hdr_version = MSG_HDR_VER; 1776 hdr->hdr_sz = sizeof(*hdr); 1777 hdr->msg_type = req->msg_type; 1778 hdr->msg_version = req->msg_version; 1779 hdr->msg_seqno = seqno; 1780 hdr->msg_vmpck = req->vmpck_id; 1781 hdr->msg_sz = req->req_sz; 1782 1783 /* Verify the sequence number is non-zero */ 1784 if (!hdr->msg_seqno) 1785 return -ENOSR; 1786 1787 pr_debug("request [seqno %lld type %d version %d sz %d]\n", 1788 hdr->msg_seqno, hdr->msg_type, hdr->msg_version, hdr->msg_sz); 1789 1790 if (WARN_ON(req->req_sz + AUTHTAG_LEN > sizeof(msg->payload))) 1791 return -EBADMSG; 1792 1793 memcpy(iv, &hdr->msg_seqno, min(sizeof(iv), sizeof(hdr->msg_seqno))); 1794 aes_gcm_encrypt(msg->payload, req->req_buf, req->req_sz, hdr->authtag, 1795 &hdr->algo, AAD_LEN, iv, gcm_key); 1796 1797 return 0; 1798 } 1799 1800 static int __handle_guest_request(struct snp_msg_desc *mdesc, struct snp_guest_req *req) 1801 { 1802 unsigned long req_start = jiffies; 1803 unsigned int override_npages = 0; 1804 u64 override_err = 0; 1805 int rc; 1806 1807 retry_request: 1808 /* 1809 * Call firmware to process the request. In this function the encrypted 1810 * message enters shared memory with the host. So after this call the 1811 * sequence number must be incremented or the VMPCK must be deleted to 1812 * prevent reuse of the IV. 1813 */ 1814 rc = snp_issue_guest_request(req); 1815 switch (rc) { 1816 case -ENOSPC: 1817 /* 1818 * If the extended guest request fails due to having too 1819 * small of a certificate data buffer, retry the same 1820 * guest request without the extended data request in 1821 * order to increment the sequence number and thus avoid 1822 * IV reuse. 1823 */ 1824 override_npages = req->input.data_npages; 1825 req->exit_code = SVM_VMGEXIT_GUEST_REQUEST; 1826 1827 /* 1828 * Override the error to inform callers the given extended 1829 * request buffer size was too small and give the caller the 1830 * required buffer size. 1831 */ 1832 override_err = SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN); 1833 1834 /* 1835 * If this call to the firmware succeeds, the sequence number can 1836 * be incremented allowing for continued use of the VMPCK. If 1837 * there is an error reflected in the return value, this value 1838 * is checked further down and the result will be the deletion 1839 * of the VMPCK and the error code being propagated back to the 1840 * user as an ioctl() return code. 1841 */ 1842 goto retry_request; 1843 1844 /* 1845 * The host may return SNP_GUEST_VMM_ERR_BUSY if the request has been 1846 * throttled. Retry in the driver to avoid returning and reusing the 1847 * message sequence number on a different message. 1848 */ 1849 case -EAGAIN: 1850 if (jiffies - req_start > SNP_REQ_MAX_RETRY_DURATION) { 1851 rc = -ETIMEDOUT; 1852 break; 1853 } 1854 schedule_timeout_killable(SNP_REQ_RETRY_DELAY); 1855 goto retry_request; 1856 } 1857 1858 /* 1859 * Increment the message sequence number. There is no harm in doing 1860 * this now because decryption uses the value stored in the response 1861 * structure and any failure will wipe the VMPCK, preventing further 1862 * use anyway. 1863 */ 1864 snp_inc_msg_seqno(mdesc); 1865 1866 if (override_err) { 1867 req->exitinfo2 = override_err; 1868 1869 /* 1870 * If an extended guest request was issued and the supplied certificate 1871 * buffer was not large enough, a standard guest request was issued to 1872 * prevent IV reuse. If the standard request was successful, return -EIO 1873 * back to the caller as would have originally been returned. 1874 */ 1875 if (!rc && override_err == SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN)) 1876 rc = -EIO; 1877 } 1878 1879 if (override_npages) 1880 req->input.data_npages = override_npages; 1881 1882 return rc; 1883 } 1884 1885 int snp_send_guest_request(struct snp_msg_desc *mdesc, struct snp_guest_req *req) 1886 { 1887 u64 seqno; 1888 int rc; 1889 1890 guard(mutex)(&snp_cmd_mutex); 1891 1892 /* Check if the VMPCK is not empty */ 1893 if (!mdesc->vmpck || !memchr_inv(mdesc->vmpck, 0, VMPCK_KEY_LEN)) { 1894 pr_err_ratelimited("VMPCK is disabled\n"); 1895 return -ENOTTY; 1896 } 1897 1898 /* Get message sequence and verify that its a non-zero */ 1899 seqno = snp_get_msg_seqno(mdesc); 1900 if (!seqno) 1901 return -EIO; 1902 1903 /* Clear shared memory's response for the host to populate. */ 1904 memset(mdesc->response, 0, sizeof(struct snp_guest_msg)); 1905 1906 /* Encrypt the userspace provided payload in mdesc->secret_request. */ 1907 rc = enc_payload(mdesc, seqno, req); 1908 if (rc) 1909 return rc; 1910 1911 /* 1912 * Write the fully encrypted request to the shared unencrypted 1913 * request page. 1914 */ 1915 memcpy(mdesc->request, &mdesc->secret_request, sizeof(mdesc->secret_request)); 1916 1917 /* Initialize the input address for guest request */ 1918 req->input.req_gpa = __pa(mdesc->request); 1919 req->input.resp_gpa = __pa(mdesc->response); 1920 req->input.data_gpa = req->certs_data ? __pa(req->certs_data) : 0; 1921 1922 rc = __handle_guest_request(mdesc, req); 1923 if (rc) { 1924 if (rc == -EIO && 1925 req->exitinfo2 == SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN)) 1926 return rc; 1927 1928 pr_alert("Detected error from ASP request. rc: %d, exitinfo2: 0x%llx\n", 1929 rc, req->exitinfo2); 1930 1931 snp_disable_vmpck(mdesc); 1932 return rc; 1933 } 1934 1935 rc = verify_and_dec_payload(mdesc, req); 1936 if (rc) { 1937 pr_alert("Detected unexpected decode failure from ASP. rc: %d\n", rc); 1938 snp_disable_vmpck(mdesc); 1939 return rc; 1940 } 1941 1942 return 0; 1943 } 1944 EXPORT_SYMBOL_GPL(snp_send_guest_request); 1945 1946 static int __init snp_get_tsc_info(void) 1947 { 1948 struct snp_tsc_info_resp *tsc_resp; 1949 struct snp_tsc_info_req *tsc_req; 1950 struct snp_msg_desc *mdesc; 1951 struct snp_guest_req req = {}; 1952 int rc = -ENOMEM; 1953 1954 tsc_req = kzalloc_obj(*tsc_req); 1955 if (!tsc_req) 1956 return rc; 1957 1958 /* 1959 * The intermediate response buffer is used while decrypting the 1960 * response payload. Make sure that it has enough space to cover 1961 * the authtag. 1962 */ 1963 tsc_resp = kzalloc(sizeof(*tsc_resp) + AUTHTAG_LEN, GFP_KERNEL); 1964 if (!tsc_resp) 1965 goto e_free_tsc_req; 1966 1967 mdesc = snp_msg_alloc(); 1968 if (IS_ERR_OR_NULL(mdesc)) 1969 goto e_free_tsc_resp; 1970 1971 rc = snp_msg_init(mdesc, snp_vmpl); 1972 if (rc) 1973 goto e_free_mdesc; 1974 1975 req.msg_version = MSG_HDR_VER; 1976 req.msg_type = SNP_MSG_TSC_INFO_REQ; 1977 req.vmpck_id = snp_vmpl; 1978 req.req_buf = tsc_req; 1979 req.req_sz = sizeof(*tsc_req); 1980 req.resp_buf = (void *)tsc_resp; 1981 req.resp_sz = sizeof(*tsc_resp) + AUTHTAG_LEN; 1982 req.exit_code = SVM_VMGEXIT_GUEST_REQUEST; 1983 1984 rc = snp_send_guest_request(mdesc, &req); 1985 if (rc) 1986 goto e_request; 1987 1988 pr_debug("%s: response status 0x%x scale 0x%llx offset 0x%llx factor 0x%x\n", 1989 __func__, tsc_resp->status, tsc_resp->tsc_scale, tsc_resp->tsc_offset, 1990 tsc_resp->tsc_factor); 1991 1992 if (!tsc_resp->status) { 1993 snp_tsc_scale = tsc_resp->tsc_scale; 1994 snp_tsc_offset = tsc_resp->tsc_offset; 1995 } else { 1996 pr_err("Failed to get TSC info, response status 0x%x\n", tsc_resp->status); 1997 rc = -EIO; 1998 } 1999 2000 e_request: 2001 /* The response buffer contains sensitive data, explicitly clear it. */ 2002 memzero_explicit(tsc_resp, sizeof(*tsc_resp) + AUTHTAG_LEN); 2003 e_free_mdesc: 2004 snp_msg_free(mdesc); 2005 e_free_tsc_resp: 2006 kfree(tsc_resp); 2007 e_free_tsc_req: 2008 kfree(tsc_req); 2009 2010 return rc; 2011 } 2012 2013 void __init snp_secure_tsc_prepare(void) 2014 { 2015 if (!cc_platform_has(CC_ATTR_GUEST_SNP_SECURE_TSC)) 2016 return; 2017 2018 if (snp_get_tsc_info()) { 2019 pr_alert("Unable to retrieve Secure TSC info from ASP\n"); 2020 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SECURE_TSC); 2021 } 2022 2023 pr_debug("SecureTSC enabled"); 2024 } 2025 2026 static unsigned long securetsc_get_tsc_khz(void) 2027 { 2028 return snp_tsc_freq_khz; 2029 } 2030 2031 void __init snp_secure_tsc_init(void) 2032 { 2033 struct snp_secrets_page *secrets; 2034 unsigned long tsc_freq_mhz; 2035 void *mem; 2036 2037 if (!cc_platform_has(CC_ATTR_GUEST_SNP_SECURE_TSC)) 2038 return; 2039 2040 mem = early_memremap_encrypted(sev_secrets_pa, PAGE_SIZE); 2041 if (!mem) { 2042 pr_err("Unable to get TSC_FACTOR: failed to map the SNP secrets page.\n"); 2043 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SECURE_TSC); 2044 } 2045 2046 secrets = (__force struct snp_secrets_page *)mem; 2047 2048 setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ); 2049 rdmsrq(MSR_AMD64_GUEST_TSC_FREQ, tsc_freq_mhz); 2050 2051 /* Extract the GUEST TSC MHZ from BIT[17:0], rest is reserved space */ 2052 tsc_freq_mhz &= GENMASK_ULL(17, 0); 2053 2054 snp_tsc_freq_khz = SNP_SCALE_TSC_FREQ(tsc_freq_mhz * 1000, secrets->tsc_factor); 2055 2056 x86_platform.calibrate_cpu = securetsc_get_tsc_khz; 2057 x86_platform.calibrate_tsc = securetsc_get_tsc_khz; 2058 2059 early_memunmap(mem, PAGE_SIZE); 2060 } 2061