1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * AMD Encrypted Register State Support 4 * 5 * Author: Joerg Roedel <jroedel@suse.de> 6 * 7 * This file is not compiled stand-alone. It contains code shared 8 * between the pre-decompression boot code and the running Linux kernel 9 * and is included directly into both code-bases. 10 */ 11 12 #include <asm/setup_data.h> 13 14 #ifndef __BOOT_COMPRESSED 15 #define error(v) pr_err(v) 16 #define has_cpuflag(f) boot_cpu_has(f) 17 #define sev_printk(fmt, ...) printk(fmt, ##__VA_ARGS__) 18 #define sev_printk_rtl(fmt, ...) printk_ratelimited(fmt, ##__VA_ARGS__) 19 #else 20 #undef WARN 21 #define WARN(condition, format...) (!!(condition)) 22 #define sev_printk(fmt, ...) 23 #define sev_printk_rtl(fmt, ...) 24 #undef vc_forward_exception 25 #define vc_forward_exception(c) panic("SNP: Hypervisor requested exception\n") 26 #endif 27 28 /* 29 * SVSM related information: 30 * When running under an SVSM, the VMPL that Linux is executing at must be 31 * non-zero. The VMPL is therefore used to indicate the presence of an SVSM. 32 * 33 * During boot, the page tables are set up as identity mapped and later 34 * changed to use kernel virtual addresses. Maintain separate virtual and 35 * physical addresses for the CAA to allow SVSM functions to be used during 36 * early boot, both with identity mapped virtual addresses and proper kernel 37 * virtual addresses. 38 */ 39 u8 snp_vmpl __ro_after_init; 40 EXPORT_SYMBOL_GPL(snp_vmpl); 41 static struct svsm_ca *boot_svsm_caa __ro_after_init; 42 static u64 boot_svsm_caa_pa __ro_after_init; 43 44 static struct svsm_ca *svsm_get_caa(void); 45 static u64 svsm_get_caa_pa(void); 46 static int svsm_perform_call_protocol(struct svsm_call *call); 47 48 /* I/O parameters for CPUID-related helpers */ 49 struct cpuid_leaf { 50 u32 fn; 51 u32 subfn; 52 u32 eax; 53 u32 ebx; 54 u32 ecx; 55 u32 edx; 56 }; 57 58 /* 59 * Individual entries of the SNP CPUID table, as defined by the SNP 60 * Firmware ABI, Revision 0.9, Section 7.1, Table 14. 61 */ 62 struct snp_cpuid_fn { 63 u32 eax_in; 64 u32 ecx_in; 65 u64 xcr0_in; 66 u64 xss_in; 67 u32 eax; 68 u32 ebx; 69 u32 ecx; 70 u32 edx; 71 u64 __reserved; 72 } __packed; 73 74 /* 75 * SNP CPUID table, as defined by the SNP Firmware ABI, Revision 0.9, 76 * Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit 77 * of 64 entries per CPUID table. 78 */ 79 #define SNP_CPUID_COUNT_MAX 64 80 81 struct snp_cpuid_table { 82 u32 count; 83 u32 __reserved1; 84 u64 __reserved2; 85 struct snp_cpuid_fn fn[SNP_CPUID_COUNT_MAX]; 86 } __packed; 87 88 /* 89 * Since feature negotiation related variables are set early in the boot 90 * process they must reside in the .data section so as not to be zeroed 91 * out when the .bss section is later cleared. 92 * 93 * GHCB protocol version negotiated with the hypervisor. 94 */ 95 static u16 ghcb_version __ro_after_init; 96 97 /* Copy of the SNP firmware's CPUID page. */ 98 static struct snp_cpuid_table cpuid_table_copy __ro_after_init; 99 100 /* 101 * These will be initialized based on CPUID table so that non-present 102 * all-zero leaves (for sparse tables) can be differentiated from 103 * invalid/out-of-range leaves. This is needed since all-zero leaves 104 * still need to be post-processed. 105 */ 106 static u32 cpuid_std_range_max __ro_after_init; 107 static u32 cpuid_hyp_range_max __ro_after_init; 108 static u32 cpuid_ext_range_max __ro_after_init; 109 110 static bool __init sev_es_check_cpu_features(void) 111 { 112 if (!has_cpuflag(X86_FEATURE_RDRAND)) { 113 error("RDRAND instruction not supported - no trusted source of randomness available\n"); 114 return false; 115 } 116 117 return true; 118 } 119 120 static void __head __noreturn 121 sev_es_terminate(unsigned int set, unsigned int reason) 122 { 123 u64 val = GHCB_MSR_TERM_REQ; 124 125 /* Tell the hypervisor what went wrong. */ 126 val |= GHCB_SEV_TERM_REASON(set, reason); 127 128 /* Request Guest Termination from Hypervisor */ 129 sev_es_wr_ghcb_msr(val); 130 VMGEXIT(); 131 132 while (true) 133 asm volatile("hlt\n" : : : "memory"); 134 } 135 136 /* 137 * The hypervisor features are available from GHCB version 2 onward. 138 */ 139 static u64 get_hv_features(void) 140 { 141 u64 val; 142 143 if (ghcb_version < 2) 144 return 0; 145 146 sev_es_wr_ghcb_msr(GHCB_MSR_HV_FT_REQ); 147 VMGEXIT(); 148 149 val = sev_es_rd_ghcb_msr(); 150 if (GHCB_RESP_CODE(val) != GHCB_MSR_HV_FT_RESP) 151 return 0; 152 153 return GHCB_MSR_HV_FT_RESP_VAL(val); 154 } 155 156 static void snp_register_ghcb_early(unsigned long paddr) 157 { 158 unsigned long pfn = paddr >> PAGE_SHIFT; 159 u64 val; 160 161 sev_es_wr_ghcb_msr(GHCB_MSR_REG_GPA_REQ_VAL(pfn)); 162 VMGEXIT(); 163 164 val = sev_es_rd_ghcb_msr(); 165 166 /* If the response GPA is not ours then abort the guest */ 167 if ((GHCB_RESP_CODE(val) != GHCB_MSR_REG_GPA_RESP) || 168 (GHCB_MSR_REG_GPA_RESP_VAL(val) != pfn)) 169 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER); 170 } 171 172 static bool sev_es_negotiate_protocol(void) 173 { 174 u64 val; 175 176 /* Do the GHCB protocol version negotiation */ 177 sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ); 178 VMGEXIT(); 179 val = sev_es_rd_ghcb_msr(); 180 181 if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP) 182 return false; 183 184 if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN || 185 GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX) 186 return false; 187 188 ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val), GHCB_PROTOCOL_MAX); 189 190 return true; 191 } 192 193 static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb) 194 { 195 ghcb->save.sw_exit_code = 0; 196 __builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); 197 } 198 199 static bool vc_decoding_needed(unsigned long exit_code) 200 { 201 /* Exceptions don't require to decode the instruction */ 202 return !(exit_code >= SVM_EXIT_EXCP_BASE && 203 exit_code <= SVM_EXIT_LAST_EXCP); 204 } 205 206 static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt, 207 struct pt_regs *regs, 208 unsigned long exit_code) 209 { 210 enum es_result ret = ES_OK; 211 212 memset(ctxt, 0, sizeof(*ctxt)); 213 ctxt->regs = regs; 214 215 if (vc_decoding_needed(exit_code)) 216 ret = vc_decode_insn(ctxt); 217 218 return ret; 219 } 220 221 static void vc_finish_insn(struct es_em_ctxt *ctxt) 222 { 223 ctxt->regs->ip += ctxt->insn.length; 224 } 225 226 static enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 227 { 228 u32 ret; 229 230 ret = ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0); 231 if (!ret) 232 return ES_OK; 233 234 if (ret == 1) { 235 u64 info = ghcb->save.sw_exit_info_2; 236 unsigned long v = info & SVM_EVTINJ_VEC_MASK; 237 238 /* Check if exception information from hypervisor is sane. */ 239 if ((info & SVM_EVTINJ_VALID) && 240 ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) && 241 ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) { 242 ctxt->fi.vector = v; 243 244 if (info & SVM_EVTINJ_VALID_ERR) 245 ctxt->fi.error_code = info >> 32; 246 247 return ES_EXCEPTION; 248 } 249 } 250 251 return ES_VMM_ERROR; 252 } 253 254 static inline int svsm_process_result_codes(struct svsm_call *call) 255 { 256 switch (call->rax_out) { 257 case SVSM_SUCCESS: 258 return 0; 259 case SVSM_ERR_INCOMPLETE: 260 case SVSM_ERR_BUSY: 261 return -EAGAIN; 262 default: 263 return -EINVAL; 264 } 265 } 266 267 /* 268 * Issue a VMGEXIT to call the SVSM: 269 * - Load the SVSM register state (RAX, RCX, RDX, R8 and R9) 270 * - Set the CA call pending field to 1 271 * - Issue VMGEXIT 272 * - Save the SVSM return register state (RAX, RCX, RDX, R8 and R9) 273 * - Perform atomic exchange of the CA call pending field 274 * 275 * - See the "Secure VM Service Module for SEV-SNP Guests" specification for 276 * details on the calling convention. 277 * - The calling convention loosely follows the Microsoft X64 calling 278 * convention by putting arguments in RCX, RDX, R8 and R9. 279 * - RAX specifies the SVSM protocol/callid as input and the return code 280 * as output. 281 */ 282 static __always_inline void svsm_issue_call(struct svsm_call *call, u8 *pending) 283 { 284 register unsigned long rax asm("rax") = call->rax; 285 register unsigned long rcx asm("rcx") = call->rcx; 286 register unsigned long rdx asm("rdx") = call->rdx; 287 register unsigned long r8 asm("r8") = call->r8; 288 register unsigned long r9 asm("r9") = call->r9; 289 290 call->caa->call_pending = 1; 291 292 asm volatile("rep; vmmcall\n\t" 293 : "+r" (rax), "+r" (rcx), "+r" (rdx), "+r" (r8), "+r" (r9) 294 : : "memory"); 295 296 *pending = xchg(&call->caa->call_pending, *pending); 297 298 call->rax_out = rax; 299 call->rcx_out = rcx; 300 call->rdx_out = rdx; 301 call->r8_out = r8; 302 call->r9_out = r9; 303 } 304 305 static int svsm_perform_msr_protocol(struct svsm_call *call) 306 { 307 u8 pending = 0; 308 u64 val, resp; 309 310 /* 311 * When using the MSR protocol, be sure to save and restore 312 * the current MSR value. 313 */ 314 val = sev_es_rd_ghcb_msr(); 315 316 sev_es_wr_ghcb_msr(GHCB_MSR_VMPL_REQ_LEVEL(0)); 317 318 svsm_issue_call(call, &pending); 319 320 resp = sev_es_rd_ghcb_msr(); 321 322 sev_es_wr_ghcb_msr(val); 323 324 if (pending) 325 return -EINVAL; 326 327 if (GHCB_RESP_CODE(resp) != GHCB_MSR_VMPL_RESP) 328 return -EINVAL; 329 330 if (GHCB_MSR_VMPL_RESP_VAL(resp)) 331 return -EINVAL; 332 333 return svsm_process_result_codes(call); 334 } 335 336 static int svsm_perform_ghcb_protocol(struct ghcb *ghcb, struct svsm_call *call) 337 { 338 struct es_em_ctxt ctxt; 339 u8 pending = 0; 340 341 vc_ghcb_invalidate(ghcb); 342 343 /* 344 * Fill in protocol and format specifiers. This can be called very early 345 * in the boot, so use rip-relative references as needed. 346 */ 347 ghcb->protocol_version = RIP_REL_REF(ghcb_version); 348 ghcb->ghcb_usage = GHCB_DEFAULT_USAGE; 349 350 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_SNP_RUN_VMPL); 351 ghcb_set_sw_exit_info_1(ghcb, 0); 352 ghcb_set_sw_exit_info_2(ghcb, 0); 353 354 sev_es_wr_ghcb_msr(__pa(ghcb)); 355 356 svsm_issue_call(call, &pending); 357 358 if (pending) 359 return -EINVAL; 360 361 switch (verify_exception_info(ghcb, &ctxt)) { 362 case ES_OK: 363 break; 364 case ES_EXCEPTION: 365 vc_forward_exception(&ctxt); 366 fallthrough; 367 default: 368 return -EINVAL; 369 } 370 371 return svsm_process_result_codes(call); 372 } 373 374 static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, 375 struct es_em_ctxt *ctxt, 376 u64 exit_code, u64 exit_info_1, 377 u64 exit_info_2) 378 { 379 /* Fill in protocol and format specifiers */ 380 ghcb->protocol_version = ghcb_version; 381 ghcb->ghcb_usage = GHCB_DEFAULT_USAGE; 382 383 ghcb_set_sw_exit_code(ghcb, exit_code); 384 ghcb_set_sw_exit_info_1(ghcb, exit_info_1); 385 ghcb_set_sw_exit_info_2(ghcb, exit_info_2); 386 387 sev_es_wr_ghcb_msr(__pa(ghcb)); 388 VMGEXIT(); 389 390 return verify_exception_info(ghcb, ctxt); 391 } 392 393 static int __sev_cpuid_hv(u32 fn, int reg_idx, u32 *reg) 394 { 395 u64 val; 396 397 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, reg_idx)); 398 VMGEXIT(); 399 val = sev_es_rd_ghcb_msr(); 400 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP) 401 return -EIO; 402 403 *reg = (val >> 32); 404 405 return 0; 406 } 407 408 static int __sev_cpuid_hv_msr(struct cpuid_leaf *leaf) 409 { 410 int ret; 411 412 /* 413 * MSR protocol does not support fetching non-zero subfunctions, but is 414 * sufficient to handle current early-boot cases. Should that change, 415 * make sure to report an error rather than ignoring the index and 416 * grabbing random values. If this issue arises in the future, handling 417 * can be added here to use GHCB-page protocol for cases that occur late 418 * enough in boot that GHCB page is available. 419 */ 420 if (cpuid_function_is_indexed(leaf->fn) && leaf->subfn) 421 return -EINVAL; 422 423 ret = __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EAX, &leaf->eax); 424 ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EBX, &leaf->ebx); 425 ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_ECX, &leaf->ecx); 426 ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EDX, &leaf->edx); 427 428 return ret; 429 } 430 431 static int __sev_cpuid_hv_ghcb(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf) 432 { 433 u32 cr4 = native_read_cr4(); 434 int ret; 435 436 ghcb_set_rax(ghcb, leaf->fn); 437 ghcb_set_rcx(ghcb, leaf->subfn); 438 439 if (cr4 & X86_CR4_OSXSAVE) 440 /* Safe to read xcr0 */ 441 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK)); 442 else 443 /* xgetbv will cause #UD - use reset value for xcr0 */ 444 ghcb_set_xcr0(ghcb, 1); 445 446 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0); 447 if (ret != ES_OK) 448 return ret; 449 450 if (!(ghcb_rax_is_valid(ghcb) && 451 ghcb_rbx_is_valid(ghcb) && 452 ghcb_rcx_is_valid(ghcb) && 453 ghcb_rdx_is_valid(ghcb))) 454 return ES_VMM_ERROR; 455 456 leaf->eax = ghcb->save.rax; 457 leaf->ebx = ghcb->save.rbx; 458 leaf->ecx = ghcb->save.rcx; 459 leaf->edx = ghcb->save.rdx; 460 461 return ES_OK; 462 } 463 464 static int sev_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf) 465 { 466 return ghcb ? __sev_cpuid_hv_ghcb(ghcb, ctxt, leaf) 467 : __sev_cpuid_hv_msr(leaf); 468 } 469 470 /* 471 * This may be called early while still running on the initial identity 472 * mapping. Use RIP-relative addressing to obtain the correct address 473 * while running with the initial identity mapping as well as the 474 * switch-over to kernel virtual addresses later. 475 */ 476 static const struct snp_cpuid_table *snp_cpuid_get_table(void) 477 { 478 return &RIP_REL_REF(cpuid_table_copy); 479 } 480 481 /* 482 * The SNP Firmware ABI, Revision 0.9, Section 7.1, details the use of 483 * XCR0_IN and XSS_IN to encode multiple versions of 0xD subfunctions 0 484 * and 1 based on the corresponding features enabled by a particular 485 * combination of XCR0 and XSS registers so that a guest can look up the 486 * version corresponding to the features currently enabled in its XCR0/XSS 487 * registers. The only values that differ between these versions/table 488 * entries is the enabled XSAVE area size advertised via EBX. 489 * 490 * While hypervisors may choose to make use of this support, it is more 491 * robust/secure for a guest to simply find the entry corresponding to the 492 * base/legacy XSAVE area size (XCR0=1 or XCR0=3), and then calculate the 493 * XSAVE area size using subfunctions 2 through 64, as documented in APM 494 * Volume 3, Rev 3.31, Appendix E.3.8, which is what is done here. 495 * 496 * Since base/legacy XSAVE area size is documented as 0x240, use that value 497 * directly rather than relying on the base size in the CPUID table. 498 * 499 * Return: XSAVE area size on success, 0 otherwise. 500 */ 501 static u32 snp_cpuid_calc_xsave_size(u64 xfeatures_en, bool compacted) 502 { 503 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); 504 u64 xfeatures_found = 0; 505 u32 xsave_size = 0x240; 506 int i; 507 508 for (i = 0; i < cpuid_table->count; i++) { 509 const struct snp_cpuid_fn *e = &cpuid_table->fn[i]; 510 511 if (!(e->eax_in == 0xD && e->ecx_in > 1 && e->ecx_in < 64)) 512 continue; 513 if (!(xfeatures_en & (BIT_ULL(e->ecx_in)))) 514 continue; 515 if (xfeatures_found & (BIT_ULL(e->ecx_in))) 516 continue; 517 518 xfeatures_found |= (BIT_ULL(e->ecx_in)); 519 520 if (compacted) 521 xsave_size += e->eax; 522 else 523 xsave_size = max(xsave_size, e->eax + e->ebx); 524 } 525 526 /* 527 * Either the guest set unsupported XCR0/XSS bits, or the corresponding 528 * entries in the CPUID table were not present. This is not a valid 529 * state to be in. 530 */ 531 if (xfeatures_found != (xfeatures_en & GENMASK_ULL(63, 2))) 532 return 0; 533 534 return xsave_size; 535 } 536 537 static bool __head 538 snp_cpuid_get_validated_func(struct cpuid_leaf *leaf) 539 { 540 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); 541 int i; 542 543 for (i = 0; i < cpuid_table->count; i++) { 544 const struct snp_cpuid_fn *e = &cpuid_table->fn[i]; 545 546 if (e->eax_in != leaf->fn) 547 continue; 548 549 if (cpuid_function_is_indexed(leaf->fn) && e->ecx_in != leaf->subfn) 550 continue; 551 552 /* 553 * For 0xD subfunctions 0 and 1, only use the entry corresponding 554 * to the base/legacy XSAVE area size (XCR0=1 or XCR0=3, XSS=0). 555 * See the comments above snp_cpuid_calc_xsave_size() for more 556 * details. 557 */ 558 if (e->eax_in == 0xD && (e->ecx_in == 0 || e->ecx_in == 1)) 559 if (!(e->xcr0_in == 1 || e->xcr0_in == 3) || e->xss_in) 560 continue; 561 562 leaf->eax = e->eax; 563 leaf->ebx = e->ebx; 564 leaf->ecx = e->ecx; 565 leaf->edx = e->edx; 566 567 return true; 568 } 569 570 return false; 571 } 572 573 static void snp_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf) 574 { 575 if (sev_cpuid_hv(ghcb, ctxt, leaf)) 576 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID_HV); 577 } 578 579 static int snp_cpuid_postprocess(struct ghcb *ghcb, struct es_em_ctxt *ctxt, 580 struct cpuid_leaf *leaf) 581 { 582 struct cpuid_leaf leaf_hv = *leaf; 583 584 switch (leaf->fn) { 585 case 0x1: 586 snp_cpuid_hv(ghcb, ctxt, &leaf_hv); 587 588 /* initial APIC ID */ 589 leaf->ebx = (leaf_hv.ebx & GENMASK(31, 24)) | (leaf->ebx & GENMASK(23, 0)); 590 /* APIC enabled bit */ 591 leaf->edx = (leaf_hv.edx & BIT(9)) | (leaf->edx & ~BIT(9)); 592 593 /* OSXSAVE enabled bit */ 594 if (native_read_cr4() & X86_CR4_OSXSAVE) 595 leaf->ecx |= BIT(27); 596 break; 597 case 0x7: 598 /* OSPKE enabled bit */ 599 leaf->ecx &= ~BIT(4); 600 if (native_read_cr4() & X86_CR4_PKE) 601 leaf->ecx |= BIT(4); 602 break; 603 case 0xB: 604 leaf_hv.subfn = 0; 605 snp_cpuid_hv(ghcb, ctxt, &leaf_hv); 606 607 /* extended APIC ID */ 608 leaf->edx = leaf_hv.edx; 609 break; 610 case 0xD: { 611 bool compacted = false; 612 u64 xcr0 = 1, xss = 0; 613 u32 xsave_size; 614 615 if (leaf->subfn != 0 && leaf->subfn != 1) 616 return 0; 617 618 if (native_read_cr4() & X86_CR4_OSXSAVE) 619 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); 620 if (leaf->subfn == 1) { 621 /* Get XSS value if XSAVES is enabled. */ 622 if (leaf->eax & BIT(3)) { 623 unsigned long lo, hi; 624 625 asm volatile("rdmsr" : "=a" (lo), "=d" (hi) 626 : "c" (MSR_IA32_XSS)); 627 xss = (hi << 32) | lo; 628 } 629 630 /* 631 * The PPR and APM aren't clear on what size should be 632 * encoded in 0xD:0x1:EBX when compaction is not enabled 633 * by either XSAVEC (feature bit 1) or XSAVES (feature 634 * bit 3) since SNP-capable hardware has these feature 635 * bits fixed as 1. KVM sets it to 0 in this case, but 636 * to avoid this becoming an issue it's safer to simply 637 * treat this as unsupported for SNP guests. 638 */ 639 if (!(leaf->eax & (BIT(1) | BIT(3)))) 640 return -EINVAL; 641 642 compacted = true; 643 } 644 645 xsave_size = snp_cpuid_calc_xsave_size(xcr0 | xss, compacted); 646 if (!xsave_size) 647 return -EINVAL; 648 649 leaf->ebx = xsave_size; 650 } 651 break; 652 case 0x8000001E: 653 snp_cpuid_hv(ghcb, ctxt, &leaf_hv); 654 655 /* extended APIC ID */ 656 leaf->eax = leaf_hv.eax; 657 /* compute ID */ 658 leaf->ebx = (leaf->ebx & GENMASK(31, 8)) | (leaf_hv.ebx & GENMASK(7, 0)); 659 /* node ID */ 660 leaf->ecx = (leaf->ecx & GENMASK(31, 8)) | (leaf_hv.ecx & GENMASK(7, 0)); 661 break; 662 default: 663 /* No fix-ups needed, use values as-is. */ 664 break; 665 } 666 667 return 0; 668 } 669 670 /* 671 * Returns -EOPNOTSUPP if feature not enabled. Any other non-zero return value 672 * should be treated as fatal by caller. 673 */ 674 static int __head 675 snp_cpuid(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf) 676 { 677 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); 678 679 if (!cpuid_table->count) 680 return -EOPNOTSUPP; 681 682 if (!snp_cpuid_get_validated_func(leaf)) { 683 /* 684 * Some hypervisors will avoid keeping track of CPUID entries 685 * where all values are zero, since they can be handled the 686 * same as out-of-range values (all-zero). This is useful here 687 * as well as it allows virtually all guest configurations to 688 * work using a single SNP CPUID table. 689 * 690 * To allow for this, there is a need to distinguish between 691 * out-of-range entries and in-range zero entries, since the 692 * CPUID table entries are only a template that may need to be 693 * augmented with additional values for things like 694 * CPU-specific information during post-processing. So if it's 695 * not in the table, set the values to zero. Then, if they are 696 * within a valid CPUID range, proceed with post-processing 697 * using zeros as the initial values. Otherwise, skip 698 * post-processing and just return zeros immediately. 699 */ 700 leaf->eax = leaf->ebx = leaf->ecx = leaf->edx = 0; 701 702 /* Skip post-processing for out-of-range zero leafs. */ 703 if (!(leaf->fn <= RIP_REL_REF(cpuid_std_range_max) || 704 (leaf->fn >= 0x40000000 && leaf->fn <= RIP_REL_REF(cpuid_hyp_range_max)) || 705 (leaf->fn >= 0x80000000 && leaf->fn <= RIP_REL_REF(cpuid_ext_range_max)))) 706 return 0; 707 } 708 709 return snp_cpuid_postprocess(ghcb, ctxt, leaf); 710 } 711 712 /* 713 * Boot VC Handler - This is the first VC handler during boot, there is no GHCB 714 * page yet, so it only supports the MSR based communication with the 715 * hypervisor and only the CPUID exit-code. 716 */ 717 void __head do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code) 718 { 719 unsigned int subfn = lower_bits(regs->cx, 32); 720 unsigned int fn = lower_bits(regs->ax, 32); 721 u16 opcode = *(unsigned short *)regs->ip; 722 struct cpuid_leaf leaf; 723 int ret; 724 725 /* Only CPUID is supported via MSR protocol */ 726 if (exit_code != SVM_EXIT_CPUID) 727 goto fail; 728 729 /* Is it really a CPUID insn? */ 730 if (opcode != 0xa20f) 731 goto fail; 732 733 leaf.fn = fn; 734 leaf.subfn = subfn; 735 736 ret = snp_cpuid(NULL, NULL, &leaf); 737 if (!ret) 738 goto cpuid_done; 739 740 if (ret != -EOPNOTSUPP) 741 goto fail; 742 743 if (__sev_cpuid_hv_msr(&leaf)) 744 goto fail; 745 746 cpuid_done: 747 regs->ax = leaf.eax; 748 regs->bx = leaf.ebx; 749 regs->cx = leaf.ecx; 750 regs->dx = leaf.edx; 751 752 /* 753 * This is a VC handler and the #VC is only raised when SEV-ES is 754 * active, which means SEV must be active too. Do sanity checks on the 755 * CPUID results to make sure the hypervisor does not trick the kernel 756 * into the no-sev path. This could map sensitive data unencrypted and 757 * make it accessible to the hypervisor. 758 * 759 * In particular, check for: 760 * - Availability of CPUID leaf 0x8000001f 761 * - SEV CPUID bit. 762 * 763 * The hypervisor might still report the wrong C-bit position, but this 764 * can't be checked here. 765 */ 766 767 if (fn == 0x80000000 && (regs->ax < 0x8000001f)) 768 /* SEV leaf check */ 769 goto fail; 770 else if ((fn == 0x8000001f && !(regs->ax & BIT(1)))) 771 /* SEV bit */ 772 goto fail; 773 774 /* Skip over the CPUID two-byte opcode */ 775 regs->ip += 2; 776 777 return; 778 779 fail: 780 /* Terminate the guest */ 781 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ); 782 } 783 784 static enum es_result vc_insn_string_check(struct es_em_ctxt *ctxt, 785 unsigned long address, 786 bool write) 787 { 788 if (user_mode(ctxt->regs) && fault_in_kernel_space(address)) { 789 ctxt->fi.vector = X86_TRAP_PF; 790 ctxt->fi.error_code = X86_PF_USER; 791 ctxt->fi.cr2 = address; 792 if (write) 793 ctxt->fi.error_code |= X86_PF_WRITE; 794 795 return ES_EXCEPTION; 796 } 797 798 return ES_OK; 799 } 800 801 static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt, 802 void *src, char *buf, 803 unsigned int data_size, 804 unsigned int count, 805 bool backwards) 806 { 807 int i, b = backwards ? -1 : 1; 808 unsigned long address = (unsigned long)src; 809 enum es_result ret; 810 811 ret = vc_insn_string_check(ctxt, address, false); 812 if (ret != ES_OK) 813 return ret; 814 815 for (i = 0; i < count; i++) { 816 void *s = src + (i * data_size * b); 817 char *d = buf + (i * data_size); 818 819 ret = vc_read_mem(ctxt, s, d, data_size); 820 if (ret != ES_OK) 821 break; 822 } 823 824 return ret; 825 } 826 827 static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt, 828 void *dst, char *buf, 829 unsigned int data_size, 830 unsigned int count, 831 bool backwards) 832 { 833 int i, s = backwards ? -1 : 1; 834 unsigned long address = (unsigned long)dst; 835 enum es_result ret; 836 837 ret = vc_insn_string_check(ctxt, address, true); 838 if (ret != ES_OK) 839 return ret; 840 841 for (i = 0; i < count; i++) { 842 void *d = dst + (i * data_size * s); 843 char *b = buf + (i * data_size); 844 845 ret = vc_write_mem(ctxt, d, b, data_size); 846 if (ret != ES_OK) 847 break; 848 } 849 850 return ret; 851 } 852 853 #define IOIO_TYPE_STR BIT(2) 854 #define IOIO_TYPE_IN 1 855 #define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR) 856 #define IOIO_TYPE_OUT 0 857 #define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR) 858 859 #define IOIO_REP BIT(3) 860 861 #define IOIO_ADDR_64 BIT(9) 862 #define IOIO_ADDR_32 BIT(8) 863 #define IOIO_ADDR_16 BIT(7) 864 865 #define IOIO_DATA_32 BIT(6) 866 #define IOIO_DATA_16 BIT(5) 867 #define IOIO_DATA_8 BIT(4) 868 869 #define IOIO_SEG_ES (0 << 10) 870 #define IOIO_SEG_DS (3 << 10) 871 872 static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo) 873 { 874 struct insn *insn = &ctxt->insn; 875 size_t size; 876 u64 port; 877 878 *exitinfo = 0; 879 880 switch (insn->opcode.bytes[0]) { 881 /* INS opcodes */ 882 case 0x6c: 883 case 0x6d: 884 *exitinfo |= IOIO_TYPE_INS; 885 *exitinfo |= IOIO_SEG_ES; 886 port = ctxt->regs->dx & 0xffff; 887 break; 888 889 /* OUTS opcodes */ 890 case 0x6e: 891 case 0x6f: 892 *exitinfo |= IOIO_TYPE_OUTS; 893 *exitinfo |= IOIO_SEG_DS; 894 port = ctxt->regs->dx & 0xffff; 895 break; 896 897 /* IN immediate opcodes */ 898 case 0xe4: 899 case 0xe5: 900 *exitinfo |= IOIO_TYPE_IN; 901 port = (u8)insn->immediate.value & 0xffff; 902 break; 903 904 /* OUT immediate opcodes */ 905 case 0xe6: 906 case 0xe7: 907 *exitinfo |= IOIO_TYPE_OUT; 908 port = (u8)insn->immediate.value & 0xffff; 909 break; 910 911 /* IN register opcodes */ 912 case 0xec: 913 case 0xed: 914 *exitinfo |= IOIO_TYPE_IN; 915 port = ctxt->regs->dx & 0xffff; 916 break; 917 918 /* OUT register opcodes */ 919 case 0xee: 920 case 0xef: 921 *exitinfo |= IOIO_TYPE_OUT; 922 port = ctxt->regs->dx & 0xffff; 923 break; 924 925 default: 926 return ES_DECODE_FAILED; 927 } 928 929 *exitinfo |= port << 16; 930 931 switch (insn->opcode.bytes[0]) { 932 case 0x6c: 933 case 0x6e: 934 case 0xe4: 935 case 0xe6: 936 case 0xec: 937 case 0xee: 938 /* Single byte opcodes */ 939 *exitinfo |= IOIO_DATA_8; 940 size = 1; 941 break; 942 default: 943 /* Length determined by instruction parsing */ 944 *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16 945 : IOIO_DATA_32; 946 size = (insn->opnd_bytes == 2) ? 2 : 4; 947 } 948 949 switch (insn->addr_bytes) { 950 case 2: 951 *exitinfo |= IOIO_ADDR_16; 952 break; 953 case 4: 954 *exitinfo |= IOIO_ADDR_32; 955 break; 956 case 8: 957 *exitinfo |= IOIO_ADDR_64; 958 break; 959 } 960 961 if (insn_has_rep_prefix(insn)) 962 *exitinfo |= IOIO_REP; 963 964 return vc_ioio_check(ctxt, (u16)port, size); 965 } 966 967 static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 968 { 969 struct pt_regs *regs = ctxt->regs; 970 u64 exit_info_1, exit_info_2; 971 enum es_result ret; 972 973 ret = vc_ioio_exitinfo(ctxt, &exit_info_1); 974 if (ret != ES_OK) 975 return ret; 976 977 if (exit_info_1 & IOIO_TYPE_STR) { 978 979 /* (REP) INS/OUTS */ 980 981 bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF); 982 unsigned int io_bytes, exit_bytes; 983 unsigned int ghcb_count, op_count; 984 unsigned long es_base; 985 u64 sw_scratch; 986 987 /* 988 * For the string variants with rep prefix the amount of in/out 989 * operations per #VC exception is limited so that the kernel 990 * has a chance to take interrupts and re-schedule while the 991 * instruction is emulated. 992 */ 993 io_bytes = (exit_info_1 >> 4) & 0x7; 994 ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes; 995 996 op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1; 997 exit_info_2 = min(op_count, ghcb_count); 998 exit_bytes = exit_info_2 * io_bytes; 999 1000 es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES); 1001 1002 /* Read bytes of OUTS into the shared buffer */ 1003 if (!(exit_info_1 & IOIO_TYPE_IN)) { 1004 ret = vc_insn_string_read(ctxt, 1005 (void *)(es_base + regs->si), 1006 ghcb->shared_buffer, io_bytes, 1007 exit_info_2, df); 1008 if (ret) 1009 return ret; 1010 } 1011 1012 /* 1013 * Issue an VMGEXIT to the HV to consume the bytes from the 1014 * shared buffer or to have it write them into the shared buffer 1015 * depending on the instruction: OUTS or INS. 1016 */ 1017 sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer); 1018 ghcb_set_sw_scratch(ghcb, sw_scratch); 1019 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, 1020 exit_info_1, exit_info_2); 1021 if (ret != ES_OK) 1022 return ret; 1023 1024 /* Read bytes from shared buffer into the guest's destination. */ 1025 if (exit_info_1 & IOIO_TYPE_IN) { 1026 ret = vc_insn_string_write(ctxt, 1027 (void *)(es_base + regs->di), 1028 ghcb->shared_buffer, io_bytes, 1029 exit_info_2, df); 1030 if (ret) 1031 return ret; 1032 1033 if (df) 1034 regs->di -= exit_bytes; 1035 else 1036 regs->di += exit_bytes; 1037 } else { 1038 if (df) 1039 regs->si -= exit_bytes; 1040 else 1041 regs->si += exit_bytes; 1042 } 1043 1044 if (exit_info_1 & IOIO_REP) 1045 regs->cx -= exit_info_2; 1046 1047 ret = regs->cx ? ES_RETRY : ES_OK; 1048 1049 } else { 1050 1051 /* IN/OUT into/from rAX */ 1052 1053 int bits = (exit_info_1 & 0x70) >> 1; 1054 u64 rax = 0; 1055 1056 if (!(exit_info_1 & IOIO_TYPE_IN)) 1057 rax = lower_bits(regs->ax, bits); 1058 1059 ghcb_set_rax(ghcb, rax); 1060 1061 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0); 1062 if (ret != ES_OK) 1063 return ret; 1064 1065 if (exit_info_1 & IOIO_TYPE_IN) { 1066 if (!ghcb_rax_is_valid(ghcb)) 1067 return ES_VMM_ERROR; 1068 regs->ax = lower_bits(ghcb->save.rax, bits); 1069 } 1070 } 1071 1072 return ret; 1073 } 1074 1075 static int vc_handle_cpuid_snp(struct ghcb *ghcb, struct es_em_ctxt *ctxt) 1076 { 1077 struct pt_regs *regs = ctxt->regs; 1078 struct cpuid_leaf leaf; 1079 int ret; 1080 1081 leaf.fn = regs->ax; 1082 leaf.subfn = regs->cx; 1083 ret = snp_cpuid(ghcb, ctxt, &leaf); 1084 if (!ret) { 1085 regs->ax = leaf.eax; 1086 regs->bx = leaf.ebx; 1087 regs->cx = leaf.ecx; 1088 regs->dx = leaf.edx; 1089 } 1090 1091 return ret; 1092 } 1093 1094 static enum es_result vc_handle_cpuid(struct ghcb *ghcb, 1095 struct es_em_ctxt *ctxt) 1096 { 1097 struct pt_regs *regs = ctxt->regs; 1098 u32 cr4 = native_read_cr4(); 1099 enum es_result ret; 1100 int snp_cpuid_ret; 1101 1102 snp_cpuid_ret = vc_handle_cpuid_snp(ghcb, ctxt); 1103 if (!snp_cpuid_ret) 1104 return ES_OK; 1105 if (snp_cpuid_ret != -EOPNOTSUPP) 1106 return ES_VMM_ERROR; 1107 1108 ghcb_set_rax(ghcb, regs->ax); 1109 ghcb_set_rcx(ghcb, regs->cx); 1110 1111 if (cr4 & X86_CR4_OSXSAVE) 1112 /* Safe to read xcr0 */ 1113 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK)); 1114 else 1115 /* xgetbv will cause #GP - use reset value for xcr0 */ 1116 ghcb_set_xcr0(ghcb, 1); 1117 1118 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0); 1119 if (ret != ES_OK) 1120 return ret; 1121 1122 if (!(ghcb_rax_is_valid(ghcb) && 1123 ghcb_rbx_is_valid(ghcb) && 1124 ghcb_rcx_is_valid(ghcb) && 1125 ghcb_rdx_is_valid(ghcb))) 1126 return ES_VMM_ERROR; 1127 1128 regs->ax = ghcb->save.rax; 1129 regs->bx = ghcb->save.rbx; 1130 regs->cx = ghcb->save.rcx; 1131 regs->dx = ghcb->save.rdx; 1132 1133 return ES_OK; 1134 } 1135 1136 static enum es_result vc_handle_rdtsc(struct ghcb *ghcb, 1137 struct es_em_ctxt *ctxt, 1138 unsigned long exit_code) 1139 { 1140 bool rdtscp = (exit_code == SVM_EXIT_RDTSCP); 1141 enum es_result ret; 1142 1143 /* 1144 * The hypervisor should not be intercepting RDTSC/RDTSCP when Secure 1145 * TSC is enabled. A #VC exception will be generated if the RDTSC/RDTSCP 1146 * instructions are being intercepted. If this should occur and Secure 1147 * TSC is enabled, guest execution should be terminated as the guest 1148 * cannot rely on the TSC value provided by the hypervisor. 1149 */ 1150 if (sev_status & MSR_AMD64_SNP_SECURE_TSC) 1151 return ES_VMM_ERROR; 1152 1153 ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0); 1154 if (ret != ES_OK) 1155 return ret; 1156 1157 if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) && 1158 (!rdtscp || ghcb_rcx_is_valid(ghcb)))) 1159 return ES_VMM_ERROR; 1160 1161 ctxt->regs->ax = ghcb->save.rax; 1162 ctxt->regs->dx = ghcb->save.rdx; 1163 if (rdtscp) 1164 ctxt->regs->cx = ghcb->save.rcx; 1165 1166 return ES_OK; 1167 } 1168 1169 struct cc_setup_data { 1170 struct setup_data header; 1171 u32 cc_blob_address; 1172 }; 1173 1174 /* 1175 * Search for a Confidential Computing blob passed in as a setup_data entry 1176 * via the Linux Boot Protocol. 1177 */ 1178 static __head 1179 struct cc_blob_sev_info *find_cc_blob_setup_data(struct boot_params *bp) 1180 { 1181 struct cc_setup_data *sd = NULL; 1182 struct setup_data *hdr; 1183 1184 hdr = (struct setup_data *)bp->hdr.setup_data; 1185 1186 while (hdr) { 1187 if (hdr->type == SETUP_CC_BLOB) { 1188 sd = (struct cc_setup_data *)hdr; 1189 return (struct cc_blob_sev_info *)(unsigned long)sd->cc_blob_address; 1190 } 1191 hdr = (struct setup_data *)hdr->next; 1192 } 1193 1194 return NULL; 1195 } 1196 1197 /* 1198 * Initialize the kernel's copy of the SNP CPUID table, and set up the 1199 * pointer that will be used to access it. 1200 * 1201 * Maintaining a direct mapping of the SNP CPUID table used by firmware would 1202 * be possible as an alternative, but the approach is brittle since the 1203 * mapping needs to be updated in sync with all the changes to virtual memory 1204 * layout and related mapping facilities throughout the boot process. 1205 */ 1206 static void __head setup_cpuid_table(const struct cc_blob_sev_info *cc_info) 1207 { 1208 const struct snp_cpuid_table *cpuid_table_fw, *cpuid_table; 1209 int i; 1210 1211 if (!cc_info || !cc_info->cpuid_phys || cc_info->cpuid_len < PAGE_SIZE) 1212 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID); 1213 1214 cpuid_table_fw = (const struct snp_cpuid_table *)cc_info->cpuid_phys; 1215 if (!cpuid_table_fw->count || cpuid_table_fw->count > SNP_CPUID_COUNT_MAX) 1216 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID); 1217 1218 cpuid_table = snp_cpuid_get_table(); 1219 memcpy((void *)cpuid_table, cpuid_table_fw, sizeof(*cpuid_table)); 1220 1221 /* Initialize CPUID ranges for range-checking. */ 1222 for (i = 0; i < cpuid_table->count; i++) { 1223 const struct snp_cpuid_fn *fn = &cpuid_table->fn[i]; 1224 1225 if (fn->eax_in == 0x0) 1226 RIP_REL_REF(cpuid_std_range_max) = fn->eax; 1227 else if (fn->eax_in == 0x40000000) 1228 RIP_REL_REF(cpuid_hyp_range_max) = fn->eax; 1229 else if (fn->eax_in == 0x80000000) 1230 RIP_REL_REF(cpuid_ext_range_max) = fn->eax; 1231 } 1232 } 1233 1234 static inline void __pval_terminate(u64 pfn, bool action, unsigned int page_size, 1235 int ret, u64 svsm_ret) 1236 { 1237 WARN(1, "PVALIDATE failure: pfn: 0x%llx, action: %u, size: %u, ret: %d, svsm_ret: 0x%llx\n", 1238 pfn, action, page_size, ret, svsm_ret); 1239 1240 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE); 1241 } 1242 1243 static void svsm_pval_terminate(struct svsm_pvalidate_call *pc, int ret, u64 svsm_ret) 1244 { 1245 unsigned int page_size; 1246 bool action; 1247 u64 pfn; 1248 1249 pfn = pc->entry[pc->cur_index].pfn; 1250 action = pc->entry[pc->cur_index].action; 1251 page_size = pc->entry[pc->cur_index].page_size; 1252 1253 __pval_terminate(pfn, action, page_size, ret, svsm_ret); 1254 } 1255 1256 static void svsm_pval_4k_page(unsigned long paddr, bool validate) 1257 { 1258 struct svsm_pvalidate_call *pc; 1259 struct svsm_call call = {}; 1260 unsigned long flags; 1261 u64 pc_pa; 1262 int ret; 1263 1264 /* 1265 * This can be called very early in the boot, use native functions in 1266 * order to avoid paravirt issues. 1267 */ 1268 flags = native_local_irq_save(); 1269 1270 call.caa = svsm_get_caa(); 1271 1272 pc = (struct svsm_pvalidate_call *)call.caa->svsm_buffer; 1273 pc_pa = svsm_get_caa_pa() + offsetof(struct svsm_ca, svsm_buffer); 1274 1275 pc->num_entries = 1; 1276 pc->cur_index = 0; 1277 pc->entry[0].page_size = RMP_PG_SIZE_4K; 1278 pc->entry[0].action = validate; 1279 pc->entry[0].ignore_cf = 0; 1280 pc->entry[0].pfn = paddr >> PAGE_SHIFT; 1281 1282 /* Protocol 0, Call ID 1 */ 1283 call.rax = SVSM_CORE_CALL(SVSM_CORE_PVALIDATE); 1284 call.rcx = pc_pa; 1285 1286 ret = svsm_perform_call_protocol(&call); 1287 if (ret) 1288 svsm_pval_terminate(pc, ret, call.rax_out); 1289 1290 native_local_irq_restore(flags); 1291 } 1292 1293 static void pvalidate_4k_page(unsigned long vaddr, unsigned long paddr, bool validate) 1294 { 1295 int ret; 1296 1297 /* 1298 * This can be called very early during boot, so use rIP-relative 1299 * references as needed. 1300 */ 1301 if (RIP_REL_REF(snp_vmpl)) { 1302 svsm_pval_4k_page(paddr, validate); 1303 } else { 1304 ret = pvalidate(vaddr, RMP_PG_SIZE_4K, validate); 1305 if (ret) 1306 __pval_terminate(PHYS_PFN(paddr), validate, RMP_PG_SIZE_4K, ret, 0); 1307 } 1308 } 1309 1310 static void pval_pages(struct snp_psc_desc *desc) 1311 { 1312 struct psc_entry *e; 1313 unsigned long vaddr; 1314 unsigned int size; 1315 unsigned int i; 1316 bool validate; 1317 u64 pfn; 1318 int rc; 1319 1320 for (i = 0; i <= desc->hdr.end_entry; i++) { 1321 e = &desc->entries[i]; 1322 1323 pfn = e->gfn; 1324 vaddr = (unsigned long)pfn_to_kaddr(pfn); 1325 size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K; 1326 validate = e->operation == SNP_PAGE_STATE_PRIVATE; 1327 1328 rc = pvalidate(vaddr, size, validate); 1329 if (!rc) 1330 continue; 1331 1332 if (rc == PVALIDATE_FAIL_SIZEMISMATCH && size == RMP_PG_SIZE_2M) { 1333 unsigned long vaddr_end = vaddr + PMD_SIZE; 1334 1335 for (; vaddr < vaddr_end; vaddr += PAGE_SIZE, pfn++) { 1336 rc = pvalidate(vaddr, RMP_PG_SIZE_4K, validate); 1337 if (rc) 1338 __pval_terminate(pfn, validate, RMP_PG_SIZE_4K, rc, 0); 1339 } 1340 } else { 1341 __pval_terminate(pfn, validate, size, rc, 0); 1342 } 1343 } 1344 } 1345 1346 static u64 svsm_build_ca_from_pfn_range(u64 pfn, u64 pfn_end, bool action, 1347 struct svsm_pvalidate_call *pc) 1348 { 1349 struct svsm_pvalidate_entry *pe; 1350 1351 /* Nothing in the CA yet */ 1352 pc->num_entries = 0; 1353 pc->cur_index = 0; 1354 1355 pe = &pc->entry[0]; 1356 1357 while (pfn < pfn_end) { 1358 pe->page_size = RMP_PG_SIZE_4K; 1359 pe->action = action; 1360 pe->ignore_cf = 0; 1361 pe->pfn = pfn; 1362 1363 pe++; 1364 pfn++; 1365 1366 pc->num_entries++; 1367 if (pc->num_entries == SVSM_PVALIDATE_MAX_COUNT) 1368 break; 1369 } 1370 1371 return pfn; 1372 } 1373 1374 static int svsm_build_ca_from_psc_desc(struct snp_psc_desc *desc, unsigned int desc_entry, 1375 struct svsm_pvalidate_call *pc) 1376 { 1377 struct svsm_pvalidate_entry *pe; 1378 struct psc_entry *e; 1379 1380 /* Nothing in the CA yet */ 1381 pc->num_entries = 0; 1382 pc->cur_index = 0; 1383 1384 pe = &pc->entry[0]; 1385 e = &desc->entries[desc_entry]; 1386 1387 while (desc_entry <= desc->hdr.end_entry) { 1388 pe->page_size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K; 1389 pe->action = e->operation == SNP_PAGE_STATE_PRIVATE; 1390 pe->ignore_cf = 0; 1391 pe->pfn = e->gfn; 1392 1393 pe++; 1394 e++; 1395 1396 desc_entry++; 1397 pc->num_entries++; 1398 if (pc->num_entries == SVSM_PVALIDATE_MAX_COUNT) 1399 break; 1400 } 1401 1402 return desc_entry; 1403 } 1404 1405 static void svsm_pval_pages(struct snp_psc_desc *desc) 1406 { 1407 struct svsm_pvalidate_entry pv_4k[VMGEXIT_PSC_MAX_ENTRY]; 1408 unsigned int i, pv_4k_count = 0; 1409 struct svsm_pvalidate_call *pc; 1410 struct svsm_call call = {}; 1411 unsigned long flags; 1412 bool action; 1413 u64 pc_pa; 1414 int ret; 1415 1416 /* 1417 * This can be called very early in the boot, use native functions in 1418 * order to avoid paravirt issues. 1419 */ 1420 flags = native_local_irq_save(); 1421 1422 /* 1423 * The SVSM calling area (CA) can support processing 510 entries at a 1424 * time. Loop through the Page State Change descriptor until the CA is 1425 * full or the last entry in the descriptor is reached, at which time 1426 * the SVSM is invoked. This repeats until all entries in the descriptor 1427 * are processed. 1428 */ 1429 call.caa = svsm_get_caa(); 1430 1431 pc = (struct svsm_pvalidate_call *)call.caa->svsm_buffer; 1432 pc_pa = svsm_get_caa_pa() + offsetof(struct svsm_ca, svsm_buffer); 1433 1434 /* Protocol 0, Call ID 1 */ 1435 call.rax = SVSM_CORE_CALL(SVSM_CORE_PVALIDATE); 1436 call.rcx = pc_pa; 1437 1438 for (i = 0; i <= desc->hdr.end_entry;) { 1439 i = svsm_build_ca_from_psc_desc(desc, i, pc); 1440 1441 do { 1442 ret = svsm_perform_call_protocol(&call); 1443 if (!ret) 1444 continue; 1445 1446 /* 1447 * Check if the entry failed because of an RMP mismatch (a 1448 * PVALIDATE at 2M was requested, but the page is mapped in 1449 * the RMP as 4K). 1450 */ 1451 1452 if (call.rax_out == SVSM_PVALIDATE_FAIL_SIZEMISMATCH && 1453 pc->entry[pc->cur_index].page_size == RMP_PG_SIZE_2M) { 1454 /* Save this entry for post-processing at 4K */ 1455 pv_4k[pv_4k_count++] = pc->entry[pc->cur_index]; 1456 1457 /* Skip to the next one unless at the end of the list */ 1458 pc->cur_index++; 1459 if (pc->cur_index < pc->num_entries) 1460 ret = -EAGAIN; 1461 else 1462 ret = 0; 1463 } 1464 } while (ret == -EAGAIN); 1465 1466 if (ret) 1467 svsm_pval_terminate(pc, ret, call.rax_out); 1468 } 1469 1470 /* Process any entries that failed to be validated at 2M and validate them at 4K */ 1471 for (i = 0; i < pv_4k_count; i++) { 1472 u64 pfn, pfn_end; 1473 1474 action = pv_4k[i].action; 1475 pfn = pv_4k[i].pfn; 1476 pfn_end = pfn + 512; 1477 1478 while (pfn < pfn_end) { 1479 pfn = svsm_build_ca_from_pfn_range(pfn, pfn_end, action, pc); 1480 1481 ret = svsm_perform_call_protocol(&call); 1482 if (ret) 1483 svsm_pval_terminate(pc, ret, call.rax_out); 1484 } 1485 } 1486 1487 native_local_irq_restore(flags); 1488 } 1489 1490 static void pvalidate_pages(struct snp_psc_desc *desc) 1491 { 1492 if (snp_vmpl) 1493 svsm_pval_pages(desc); 1494 else 1495 pval_pages(desc); 1496 } 1497 1498 static int vmgexit_psc(struct ghcb *ghcb, struct snp_psc_desc *desc) 1499 { 1500 int cur_entry, end_entry, ret = 0; 1501 struct snp_psc_desc *data; 1502 struct es_em_ctxt ctxt; 1503 1504 vc_ghcb_invalidate(ghcb); 1505 1506 /* Copy the input desc into GHCB shared buffer */ 1507 data = (struct snp_psc_desc *)ghcb->shared_buffer; 1508 memcpy(ghcb->shared_buffer, desc, min_t(int, GHCB_SHARED_BUF_SIZE, sizeof(*desc))); 1509 1510 /* 1511 * As per the GHCB specification, the hypervisor can resume the guest 1512 * before processing all the entries. Check whether all the entries 1513 * are processed. If not, then keep retrying. Note, the hypervisor 1514 * will update the data memory directly to indicate the status, so 1515 * reference the data->hdr everywhere. 1516 * 1517 * The strategy here is to wait for the hypervisor to change the page 1518 * state in the RMP table before guest accesses the memory pages. If the 1519 * page state change was not successful, then later memory access will 1520 * result in a crash. 1521 */ 1522 cur_entry = data->hdr.cur_entry; 1523 end_entry = data->hdr.end_entry; 1524 1525 while (data->hdr.cur_entry <= data->hdr.end_entry) { 1526 ghcb_set_sw_scratch(ghcb, (u64)__pa(data)); 1527 1528 /* This will advance the shared buffer data points to. */ 1529 ret = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_PSC, 0, 0); 1530 1531 /* 1532 * Page State Change VMGEXIT can pass error code through 1533 * exit_info_2. 1534 */ 1535 if (WARN(ret || ghcb->save.sw_exit_info_2, 1536 "SNP: PSC failed ret=%d exit_info_2=%llx\n", 1537 ret, ghcb->save.sw_exit_info_2)) { 1538 ret = 1; 1539 goto out; 1540 } 1541 1542 /* Verify that reserved bit is not set */ 1543 if (WARN(data->hdr.reserved, "Reserved bit is set in the PSC header\n")) { 1544 ret = 1; 1545 goto out; 1546 } 1547 1548 /* 1549 * Sanity check that entry processing is not going backwards. 1550 * This will happen only if hypervisor is tricking us. 1551 */ 1552 if (WARN(data->hdr.end_entry > end_entry || cur_entry > data->hdr.cur_entry, 1553 "SNP: PSC processing going backward, end_entry %d (got %d) cur_entry %d (got %d)\n", 1554 end_entry, data->hdr.end_entry, cur_entry, data->hdr.cur_entry)) { 1555 ret = 1; 1556 goto out; 1557 } 1558 } 1559 1560 out: 1561 return ret; 1562 } 1563 1564 static enum es_result vc_check_opcode_bytes(struct es_em_ctxt *ctxt, 1565 unsigned long exit_code) 1566 { 1567 unsigned int opcode = (unsigned int)ctxt->insn.opcode.value; 1568 u8 modrm = ctxt->insn.modrm.value; 1569 1570 switch (exit_code) { 1571 1572 case SVM_EXIT_IOIO: 1573 case SVM_EXIT_NPF: 1574 /* handled separately */ 1575 return ES_OK; 1576 1577 case SVM_EXIT_CPUID: 1578 if (opcode == 0xa20f) 1579 return ES_OK; 1580 break; 1581 1582 case SVM_EXIT_INVD: 1583 if (opcode == 0x080f) 1584 return ES_OK; 1585 break; 1586 1587 case SVM_EXIT_MONITOR: 1588 /* MONITOR and MONITORX instructions generate the same error code */ 1589 if (opcode == 0x010f && (modrm == 0xc8 || modrm == 0xfa)) 1590 return ES_OK; 1591 break; 1592 1593 case SVM_EXIT_MWAIT: 1594 /* MWAIT and MWAITX instructions generate the same error code */ 1595 if (opcode == 0x010f && (modrm == 0xc9 || modrm == 0xfb)) 1596 return ES_OK; 1597 break; 1598 1599 case SVM_EXIT_MSR: 1600 /* RDMSR */ 1601 if (opcode == 0x320f || 1602 /* WRMSR */ 1603 opcode == 0x300f) 1604 return ES_OK; 1605 break; 1606 1607 case SVM_EXIT_RDPMC: 1608 if (opcode == 0x330f) 1609 return ES_OK; 1610 break; 1611 1612 case SVM_EXIT_RDTSC: 1613 if (opcode == 0x310f) 1614 return ES_OK; 1615 break; 1616 1617 case SVM_EXIT_RDTSCP: 1618 if (opcode == 0x010f && modrm == 0xf9) 1619 return ES_OK; 1620 break; 1621 1622 case SVM_EXIT_READ_DR7: 1623 if (opcode == 0x210f && 1624 X86_MODRM_REG(ctxt->insn.modrm.value) == 7) 1625 return ES_OK; 1626 break; 1627 1628 case SVM_EXIT_VMMCALL: 1629 if (opcode == 0x010f && modrm == 0xd9) 1630 return ES_OK; 1631 1632 break; 1633 1634 case SVM_EXIT_WRITE_DR7: 1635 if (opcode == 0x230f && 1636 X86_MODRM_REG(ctxt->insn.modrm.value) == 7) 1637 return ES_OK; 1638 break; 1639 1640 case SVM_EXIT_WBINVD: 1641 if (opcode == 0x90f) 1642 return ES_OK; 1643 break; 1644 1645 default: 1646 break; 1647 } 1648 1649 sev_printk(KERN_ERR "Wrong/unhandled opcode bytes: 0x%x, exit_code: 0x%lx, rIP: 0x%lx\n", 1650 opcode, exit_code, ctxt->regs->ip); 1651 1652 return ES_UNSUPPORTED; 1653 } 1654 1655 /* 1656 * Maintain the GPA of the SVSM Calling Area (CA) in order to utilize the SVSM 1657 * services needed when not running in VMPL0. 1658 */ 1659 static bool __head svsm_setup_ca(const struct cc_blob_sev_info *cc_info) 1660 { 1661 struct snp_secrets_page *secrets_page; 1662 struct snp_cpuid_table *cpuid_table; 1663 unsigned int i; 1664 u64 caa; 1665 1666 BUILD_BUG_ON(sizeof(*secrets_page) != PAGE_SIZE); 1667 1668 /* 1669 * Check if running at VMPL0. 1670 * 1671 * Use RMPADJUST (see the rmpadjust() function for a description of what 1672 * the instruction does) to update the VMPL1 permissions of a page. If 1673 * the guest is running at VMPL0, this will succeed and implies there is 1674 * no SVSM. If the guest is running at any other VMPL, this will fail. 1675 * Linux SNP guests only ever run at a single VMPL level so permission mask 1676 * changes of a lesser-privileged VMPL are a don't-care. 1677 * 1678 * Use a rip-relative reference to obtain the proper address, since this 1679 * routine is running identity mapped when called, both by the decompressor 1680 * code and the early kernel code. 1681 */ 1682 if (!rmpadjust((unsigned long)&RIP_REL_REF(boot_ghcb_page), RMP_PG_SIZE_4K, 1)) 1683 return false; 1684 1685 /* 1686 * Not running at VMPL0, ensure everything has been properly supplied 1687 * for running under an SVSM. 1688 */ 1689 if (!cc_info || !cc_info->secrets_phys || cc_info->secrets_len != PAGE_SIZE) 1690 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SECRETS_PAGE); 1691 1692 secrets_page = (struct snp_secrets_page *)cc_info->secrets_phys; 1693 if (!secrets_page->svsm_size) 1694 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_NO_SVSM); 1695 1696 if (!secrets_page->svsm_guest_vmpl) 1697 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SVSM_VMPL0); 1698 1699 RIP_REL_REF(snp_vmpl) = secrets_page->svsm_guest_vmpl; 1700 1701 caa = secrets_page->svsm_caa; 1702 1703 /* 1704 * An open-coded PAGE_ALIGNED() in order to avoid including 1705 * kernel-proper headers into the decompressor. 1706 */ 1707 if (caa & (PAGE_SIZE - 1)) 1708 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SVSM_CAA); 1709 1710 /* 1711 * The CA is identity mapped when this routine is called, both by the 1712 * decompressor code and the early kernel code. 1713 */ 1714 RIP_REL_REF(boot_svsm_caa) = (struct svsm_ca *)caa; 1715 RIP_REL_REF(boot_svsm_caa_pa) = caa; 1716 1717 /* Advertise the SVSM presence via CPUID. */ 1718 cpuid_table = (struct snp_cpuid_table *)snp_cpuid_get_table(); 1719 for (i = 0; i < cpuid_table->count; i++) { 1720 struct snp_cpuid_fn *fn = &cpuid_table->fn[i]; 1721 1722 if (fn->eax_in == 0x8000001f) 1723 fn->eax |= BIT(28); 1724 } 1725 1726 return true; 1727 } 1728