1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * AMD Encrypted Register State Support 4 * 5 * Author: Joerg Roedel <jroedel@suse.de> 6 */ 7 8 #ifndef __ASM_ENCRYPTED_STATE_H 9 #define __ASM_ENCRYPTED_STATE_H 10 11 #include <linux/types.h> 12 #include <linux/sev-guest.h> 13 14 #include <asm/insn.h> 15 #include <asm/sev-common.h> 16 #include <asm/coco.h> 17 #include <asm/set_memory.h> 18 #include <asm/svm.h> 19 20 #define GHCB_PROTOCOL_MIN 1ULL 21 #define GHCB_PROTOCOL_MAX 2ULL 22 #define GHCB_DEFAULT_USAGE 0ULL 23 24 #define VMGEXIT() { asm volatile("rep; vmmcall\n\r"); } 25 26 struct boot_params; 27 28 enum es_result { 29 ES_OK, /* All good */ 30 ES_UNSUPPORTED, /* Requested operation not supported */ 31 ES_VMM_ERROR, /* Unexpected state from the VMM */ 32 ES_DECODE_FAILED, /* Instruction decoding failed */ 33 ES_EXCEPTION, /* Instruction caused exception */ 34 ES_RETRY, /* Retry instruction emulation */ 35 }; 36 37 struct es_fault_info { 38 unsigned long vector; 39 unsigned long error_code; 40 unsigned long cr2; 41 }; 42 43 struct pt_regs; 44 45 /* ES instruction emulation context */ 46 struct es_em_ctxt { 47 struct pt_regs *regs; 48 struct insn insn; 49 struct es_fault_info fi; 50 }; 51 52 /* 53 * AMD SEV Confidential computing blob structure. The structure is 54 * defined in OVMF UEFI firmware header: 55 * https://github.com/tianocore/edk2/blob/master/OvmfPkg/Include/Guid/ConfidentialComputingSevSnpBlob.h 56 */ 57 #define CC_BLOB_SEV_HDR_MAGIC 0x45444d41 58 struct cc_blob_sev_info { 59 u32 magic; 60 u16 version; 61 u16 reserved; 62 u64 secrets_phys; 63 u32 secrets_len; 64 u32 rsvd1; 65 u64 cpuid_phys; 66 u32 cpuid_len; 67 u32 rsvd2; 68 } __packed; 69 70 void do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code); 71 72 static inline u64 lower_bits(u64 val, unsigned int bits) 73 { 74 u64 mask = (1ULL << bits) - 1; 75 76 return (val & mask); 77 } 78 79 struct real_mode_header; 80 enum stack_type; 81 82 /* Early IDT entry points for #VC handler */ 83 extern void vc_no_ghcb(void); 84 extern void vc_boot_ghcb(void); 85 extern bool handle_vc_boot_ghcb(struct pt_regs *regs); 86 87 /* 88 * Individual entries of the SNP CPUID table, as defined by the SNP 89 * Firmware ABI, Revision 0.9, Section 7.1, Table 14. 90 */ 91 struct snp_cpuid_fn { 92 u32 eax_in; 93 u32 ecx_in; 94 u64 xcr0_in; 95 u64 xss_in; 96 u32 eax; 97 u32 ebx; 98 u32 ecx; 99 u32 edx; 100 u64 __reserved; 101 } __packed; 102 103 /* 104 * SNP CPUID table, as defined by the SNP Firmware ABI, Revision 0.9, 105 * Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit 106 * of 64 entries per CPUID table. 107 */ 108 #define SNP_CPUID_COUNT_MAX 64 109 110 struct snp_cpuid_table { 111 u32 count; 112 u32 __reserved1; 113 u64 __reserved2; 114 struct snp_cpuid_fn fn[SNP_CPUID_COUNT_MAX]; 115 } __packed; 116 117 /* PVALIDATE return codes */ 118 #define PVALIDATE_FAIL_SIZEMISMATCH 6 119 120 /* Software defined (when rFlags.CF = 1) */ 121 #define PVALIDATE_FAIL_NOUPDATE 255 122 123 /* RMUPDATE detected 4K page and 2MB page overlap. */ 124 #define RMPUPDATE_FAIL_OVERLAP 4 125 126 /* PSMASH failed due to concurrent access by another CPU */ 127 #define PSMASH_FAIL_INUSE 3 128 129 /* RMP page size */ 130 #define RMP_PG_SIZE_4K 0 131 #define RMP_PG_SIZE_2M 1 132 #define RMP_TO_PG_LEVEL(level) (((level) == RMP_PG_SIZE_4K) ? PG_LEVEL_4K : PG_LEVEL_2M) 133 #define PG_LEVEL_TO_RMP(level) (((level) == PG_LEVEL_4K) ? RMP_PG_SIZE_4K : RMP_PG_SIZE_2M) 134 135 struct rmp_state { 136 u64 gpa; 137 u8 assigned; 138 u8 pagesize; 139 u8 immutable; 140 u8 rsvd; 141 u32 asid; 142 } __packed; 143 144 #define RMPADJUST_VMSA_PAGE_BIT BIT(16) 145 146 /* SNP Guest message request */ 147 struct snp_req_data { 148 unsigned long req_gpa; 149 unsigned long resp_gpa; 150 unsigned long data_gpa; 151 unsigned int data_npages; 152 }; 153 154 #define MAX_AUTHTAG_LEN 32 155 #define AUTHTAG_LEN 16 156 #define AAD_LEN 48 157 #define MSG_HDR_VER 1 158 159 #define SNP_REQ_MAX_RETRY_DURATION (60*HZ) 160 #define SNP_REQ_RETRY_DELAY (2*HZ) 161 162 /* See SNP spec SNP_GUEST_REQUEST section for the structure */ 163 enum msg_type { 164 SNP_MSG_TYPE_INVALID = 0, 165 SNP_MSG_CPUID_REQ, 166 SNP_MSG_CPUID_RSP, 167 SNP_MSG_KEY_REQ, 168 SNP_MSG_KEY_RSP, 169 SNP_MSG_REPORT_REQ, 170 SNP_MSG_REPORT_RSP, 171 SNP_MSG_EXPORT_REQ, 172 SNP_MSG_EXPORT_RSP, 173 SNP_MSG_IMPORT_REQ, 174 SNP_MSG_IMPORT_RSP, 175 SNP_MSG_ABSORB_REQ, 176 SNP_MSG_ABSORB_RSP, 177 SNP_MSG_VMRK_REQ, 178 SNP_MSG_VMRK_RSP, 179 180 SNP_MSG_TSC_INFO_REQ = 17, 181 SNP_MSG_TSC_INFO_RSP, 182 183 SNP_MSG_TYPE_MAX 184 }; 185 186 enum aead_algo { 187 SNP_AEAD_INVALID, 188 SNP_AEAD_AES_256_GCM, 189 }; 190 191 struct snp_guest_msg_hdr { 192 u8 authtag[MAX_AUTHTAG_LEN]; 193 u64 msg_seqno; 194 u8 rsvd1[8]; 195 u8 algo; 196 u8 hdr_version; 197 u16 hdr_sz; 198 u8 msg_type; 199 u8 msg_version; 200 u16 msg_sz; 201 u32 rsvd2; 202 u8 msg_vmpck; 203 u8 rsvd3[35]; 204 } __packed; 205 206 struct snp_guest_msg { 207 struct snp_guest_msg_hdr hdr; 208 u8 payload[PAGE_SIZE - sizeof(struct snp_guest_msg_hdr)]; 209 } __packed; 210 211 #define SNP_TSC_INFO_REQ_SZ 128 212 213 struct snp_tsc_info_req { 214 u8 rsvd[SNP_TSC_INFO_REQ_SZ]; 215 } __packed; 216 217 struct snp_tsc_info_resp { 218 u32 status; 219 u32 rsvd1; 220 u64 tsc_scale; 221 u64 tsc_offset; 222 u32 tsc_factor; 223 u8 rsvd2[100]; 224 } __packed; 225 226 struct snp_guest_req { 227 void *req_buf; 228 size_t req_sz; 229 230 void *resp_buf; 231 size_t resp_sz; 232 233 u64 exit_code; 234 unsigned int vmpck_id; 235 u8 msg_version; 236 u8 msg_type; 237 238 struct snp_req_data input; 239 void *certs_data; 240 }; 241 242 /* 243 * The secrets page contains 96-bytes of reserved field that can be used by 244 * the guest OS. The guest OS uses the area to save the message sequence 245 * number for each VMPCK. 246 * 247 * See the GHCB spec section Secret page layout for the format for this area. 248 */ 249 struct secrets_os_area { 250 u32 msg_seqno_0; 251 u32 msg_seqno_1; 252 u32 msg_seqno_2; 253 u32 msg_seqno_3; 254 u64 ap_jump_table_pa; 255 u8 rsvd[40]; 256 u8 guest_usage[32]; 257 } __packed; 258 259 #define VMPCK_KEY_LEN 32 260 261 /* See the SNP spec version 0.9 for secrets page format */ 262 struct snp_secrets_page { 263 u32 version; 264 u32 imien : 1, 265 rsvd1 : 31; 266 u32 fms; 267 u32 rsvd2; 268 u8 gosvw[16]; 269 u8 vmpck0[VMPCK_KEY_LEN]; 270 u8 vmpck1[VMPCK_KEY_LEN]; 271 u8 vmpck2[VMPCK_KEY_LEN]; 272 u8 vmpck3[VMPCK_KEY_LEN]; 273 struct secrets_os_area os_area; 274 275 u8 vmsa_tweak_bitmap[64]; 276 277 /* SVSM fields */ 278 u64 svsm_base; 279 u64 svsm_size; 280 u64 svsm_caa; 281 u32 svsm_max_version; 282 u8 svsm_guest_vmpl; 283 u8 rsvd3[3]; 284 285 /* Remainder of page */ 286 u8 rsvd4[3744]; 287 } __packed; 288 289 struct snp_msg_desc { 290 /* request and response are in unencrypted memory */ 291 struct snp_guest_msg *request, *response; 292 293 /* 294 * Avoid information leakage by double-buffering shared messages 295 * in fields that are in regular encrypted memory. 296 */ 297 struct snp_guest_msg secret_request, secret_response; 298 299 struct snp_secrets_page *secrets; 300 301 struct aesgcm_ctx *ctx; 302 303 u32 *os_area_msg_seqno; 304 u8 *vmpck; 305 int vmpck_id; 306 }; 307 308 /* 309 * The SVSM Calling Area (CA) related structures. 310 */ 311 struct svsm_ca { 312 u8 call_pending; 313 u8 mem_available; 314 u8 rsvd1[6]; 315 316 u8 svsm_buffer[PAGE_SIZE - 8]; 317 }; 318 319 #define SVSM_SUCCESS 0 320 #define SVSM_ERR_INCOMPLETE 0x80000000 321 #define SVSM_ERR_UNSUPPORTED_PROTOCOL 0x80000001 322 #define SVSM_ERR_UNSUPPORTED_CALL 0x80000002 323 #define SVSM_ERR_INVALID_ADDRESS 0x80000003 324 #define SVSM_ERR_INVALID_FORMAT 0x80000004 325 #define SVSM_ERR_INVALID_PARAMETER 0x80000005 326 #define SVSM_ERR_INVALID_REQUEST 0x80000006 327 #define SVSM_ERR_BUSY 0x80000007 328 #define SVSM_PVALIDATE_FAIL_SIZEMISMATCH 0x80001006 329 330 /* 331 * The SVSM PVALIDATE related structures 332 */ 333 struct svsm_pvalidate_entry { 334 u64 page_size : 2, 335 action : 1, 336 ignore_cf : 1, 337 rsvd : 8, 338 pfn : 52; 339 }; 340 341 struct svsm_pvalidate_call { 342 u16 num_entries; 343 u16 cur_index; 344 345 u8 rsvd1[4]; 346 347 struct svsm_pvalidate_entry entry[]; 348 }; 349 350 #define SVSM_PVALIDATE_MAX_COUNT ((sizeof_field(struct svsm_ca, svsm_buffer) - \ 351 offsetof(struct svsm_pvalidate_call, entry)) / \ 352 sizeof(struct svsm_pvalidate_entry)) 353 354 /* 355 * The SVSM Attestation related structures 356 */ 357 struct svsm_loc_entry { 358 u64 pa; 359 u32 len; 360 u8 rsvd[4]; 361 }; 362 363 struct svsm_attest_call { 364 struct svsm_loc_entry report_buf; 365 struct svsm_loc_entry nonce; 366 struct svsm_loc_entry manifest_buf; 367 struct svsm_loc_entry certificates_buf; 368 369 /* For attesting a single service */ 370 u8 service_guid[16]; 371 u32 service_manifest_ver; 372 u8 rsvd[4]; 373 }; 374 375 /* PTE descriptor used for the prepare_pte_enc() operations. */ 376 struct pte_enc_desc { 377 pte_t *kpte; 378 int pte_level; 379 bool encrypt; 380 /* pfn of the kpte above */ 381 unsigned long pfn; 382 /* physical address of @pfn */ 383 unsigned long pa; 384 /* virtual address of @pfn */ 385 void *va; 386 /* memory covered by the pte */ 387 unsigned long size; 388 pgprot_t new_pgprot; 389 }; 390 391 /* 392 * SVSM protocol structure 393 */ 394 struct svsm_call { 395 struct svsm_ca *caa; 396 u64 rax; 397 u64 rcx; 398 u64 rdx; 399 u64 r8; 400 u64 r9; 401 u64 rax_out; 402 u64 rcx_out; 403 u64 rdx_out; 404 u64 r8_out; 405 u64 r9_out; 406 }; 407 408 #define SVSM_CORE_CALL(x) ((0ULL << 32) | (x)) 409 #define SVSM_CORE_REMAP_CA 0 410 #define SVSM_CORE_PVALIDATE 1 411 #define SVSM_CORE_CREATE_VCPU 2 412 #define SVSM_CORE_DELETE_VCPU 3 413 414 #define SVSM_ATTEST_CALL(x) ((1ULL << 32) | (x)) 415 #define SVSM_ATTEST_SERVICES 0 416 #define SVSM_ATTEST_SINGLE_SERVICE 1 417 418 #ifdef CONFIG_AMD_MEM_ENCRYPT 419 420 extern u8 snp_vmpl; 421 422 extern void __sev_es_ist_enter(struct pt_regs *regs); 423 extern void __sev_es_ist_exit(void); 424 static __always_inline void sev_es_ist_enter(struct pt_regs *regs) 425 { 426 if (cc_vendor == CC_VENDOR_AMD && 427 cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) 428 __sev_es_ist_enter(regs); 429 } 430 static __always_inline void sev_es_ist_exit(void) 431 { 432 if (cc_vendor == CC_VENDOR_AMD && 433 cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) 434 __sev_es_ist_exit(); 435 } 436 extern int sev_es_setup_ap_jump_table(struct real_mode_header *rmh); 437 extern void __sev_es_nmi_complete(void); 438 static __always_inline void sev_es_nmi_complete(void) 439 { 440 if (cc_vendor == CC_VENDOR_AMD && 441 cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) 442 __sev_es_nmi_complete(); 443 } 444 extern int __init sev_es_efi_map_ghcbs(pgd_t *pgd); 445 extern void sev_enable(struct boot_params *bp); 446 447 /* 448 * RMPADJUST modifies the RMP permissions of a page of a lesser- 449 * privileged (numerically higher) VMPL. 450 * 451 * If the guest is running at a higher-privilege than the privilege 452 * level the instruction is targeting, the instruction will succeed, 453 * otherwise, it will fail. 454 */ 455 static inline int rmpadjust(unsigned long vaddr, bool rmp_psize, unsigned long attrs) 456 { 457 int rc; 458 459 /* "rmpadjust" mnemonic support in binutils 2.36 and newer */ 460 asm volatile(".byte 0xF3,0x0F,0x01,0xFE\n\t" 461 : "=a"(rc) 462 : "a"(vaddr), "c"(rmp_psize), "d"(attrs) 463 : "memory", "cc"); 464 465 return rc; 466 } 467 static inline int pvalidate(unsigned long vaddr, bool rmp_psize, bool validate) 468 { 469 bool no_rmpupdate; 470 int rc; 471 472 /* "pvalidate" mnemonic support in binutils 2.36 and newer */ 473 asm volatile(".byte 0xF2, 0x0F, 0x01, 0xFF\n\t" 474 CC_SET(c) 475 : CC_OUT(c) (no_rmpupdate), "=a"(rc) 476 : "a"(vaddr), "c"(rmp_psize), "d"(validate) 477 : "memory", "cc"); 478 479 if (no_rmpupdate) 480 return PVALIDATE_FAIL_NOUPDATE; 481 482 return rc; 483 } 484 485 struct snp_guest_request_ioctl; 486 487 void setup_ghcb(void); 488 void early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr, 489 unsigned long npages); 490 void early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr, 491 unsigned long npages); 492 void snp_set_memory_shared(unsigned long vaddr, unsigned long npages); 493 void snp_set_memory_private(unsigned long vaddr, unsigned long npages); 494 void snp_set_wakeup_secondary_cpu(void); 495 bool snp_init(struct boot_params *bp); 496 void __noreturn snp_abort(void); 497 void snp_dmi_setup(void); 498 int snp_issue_svsm_attest_req(u64 call_id, struct svsm_call *call, struct svsm_attest_call *input); 499 void snp_accept_memory(phys_addr_t start, phys_addr_t end); 500 u64 snp_get_unsupported_features(u64 status); 501 u64 sev_get_status(void); 502 void sev_show_status(void); 503 void snp_update_svsm_ca(void); 504 int prepare_pte_enc(struct pte_enc_desc *d); 505 void set_pte_enc_mask(pte_t *kpte, unsigned long pfn, pgprot_t new_prot); 506 void snp_kexec_finish(void); 507 void snp_kexec_begin(void); 508 509 int snp_msg_init(struct snp_msg_desc *mdesc, int vmpck_id); 510 struct snp_msg_desc *snp_msg_alloc(void); 511 void snp_msg_free(struct snp_msg_desc *mdesc); 512 int snp_send_guest_request(struct snp_msg_desc *mdesc, struct snp_guest_req *req, 513 struct snp_guest_request_ioctl *rio); 514 515 void __init snp_secure_tsc_prepare(void); 516 void __init snp_secure_tsc_init(void); 517 518 static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb) 519 { 520 ghcb->save.sw_exit_code = 0; 521 __builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); 522 } 523 524 void vc_forward_exception(struct es_em_ctxt *ctxt); 525 526 /* I/O parameters for CPUID-related helpers */ 527 struct cpuid_leaf { 528 u32 fn; 529 u32 subfn; 530 u32 eax; 531 u32 ebx; 532 u32 ecx; 533 u32 edx; 534 }; 535 536 int snp_cpuid(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf); 537 538 void __noreturn sev_es_terminate(unsigned int set, unsigned int reason); 539 enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb, 540 struct es_em_ctxt *ctxt, 541 u64 exit_code, u64 exit_info_1, 542 u64 exit_info_2); 543 544 extern struct ghcb *boot_ghcb; 545 546 #else /* !CONFIG_AMD_MEM_ENCRYPT */ 547 548 #define snp_vmpl 0 549 static inline void sev_es_ist_enter(struct pt_regs *regs) { } 550 static inline void sev_es_ist_exit(void) { } 551 static inline int sev_es_setup_ap_jump_table(struct real_mode_header *rmh) { return 0; } 552 static inline void sev_es_nmi_complete(void) { } 553 static inline int sev_es_efi_map_ghcbs(pgd_t *pgd) { return 0; } 554 static inline void sev_enable(struct boot_params *bp) { } 555 static inline int pvalidate(unsigned long vaddr, bool rmp_psize, bool validate) { return 0; } 556 static inline int rmpadjust(unsigned long vaddr, bool rmp_psize, unsigned long attrs) { return 0; } 557 static inline void setup_ghcb(void) { } 558 static inline void __init 559 early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr, unsigned long npages) { } 560 static inline void __init 561 early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr, unsigned long npages) { } 562 static inline void snp_set_memory_shared(unsigned long vaddr, unsigned long npages) { } 563 static inline void snp_set_memory_private(unsigned long vaddr, unsigned long npages) { } 564 static inline void snp_set_wakeup_secondary_cpu(void) { } 565 static inline bool snp_init(struct boot_params *bp) { return false; } 566 static inline void snp_abort(void) { } 567 static inline void snp_dmi_setup(void) { } 568 static inline int snp_issue_svsm_attest_req(u64 call_id, struct svsm_call *call, struct svsm_attest_call *input) 569 { 570 return -ENOTTY; 571 } 572 static inline void snp_accept_memory(phys_addr_t start, phys_addr_t end) { } 573 static inline u64 snp_get_unsupported_features(u64 status) { return 0; } 574 static inline u64 sev_get_status(void) { return 0; } 575 static inline void sev_show_status(void) { } 576 static inline void snp_update_svsm_ca(void) { } 577 static inline int prepare_pte_enc(struct pte_enc_desc *d) { return 0; } 578 static inline void set_pte_enc_mask(pte_t *kpte, unsigned long pfn, pgprot_t new_prot) { } 579 static inline void snp_kexec_finish(void) { } 580 static inline void snp_kexec_begin(void) { } 581 static inline int snp_msg_init(struct snp_msg_desc *mdesc, int vmpck_id) { return -1; } 582 static inline struct snp_msg_desc *snp_msg_alloc(void) { return NULL; } 583 static inline void snp_msg_free(struct snp_msg_desc *mdesc) { } 584 static inline int snp_send_guest_request(struct snp_msg_desc *mdesc, struct snp_guest_req *req, 585 struct snp_guest_request_ioctl *rio) { return -ENODEV; } 586 static inline void __init snp_secure_tsc_prepare(void) { } 587 static inline void __init snp_secure_tsc_init(void) { } 588 589 #endif /* CONFIG_AMD_MEM_ENCRYPT */ 590 591 #ifdef CONFIG_KVM_AMD_SEV 592 bool snp_probe_rmptable_info(void); 593 int snp_rmptable_init(void); 594 int snp_lookup_rmpentry(u64 pfn, bool *assigned, int *level); 595 void snp_dump_hva_rmpentry(unsigned long address); 596 int psmash(u64 pfn); 597 int rmp_make_private(u64 pfn, u64 gpa, enum pg_level level, u32 asid, bool immutable); 598 int rmp_make_shared(u64 pfn, enum pg_level level); 599 void snp_leak_pages(u64 pfn, unsigned int npages); 600 void kdump_sev_callback(void); 601 void snp_fixup_e820_tables(void); 602 #else 603 static inline bool snp_probe_rmptable_info(void) { return false; } 604 static inline int snp_rmptable_init(void) { return -ENOSYS; } 605 static inline int snp_lookup_rmpentry(u64 pfn, bool *assigned, int *level) { return -ENODEV; } 606 static inline void snp_dump_hva_rmpentry(unsigned long address) {} 607 static inline int psmash(u64 pfn) { return -ENODEV; } 608 static inline int rmp_make_private(u64 pfn, u64 gpa, enum pg_level level, u32 asid, 609 bool immutable) 610 { 611 return -ENODEV; 612 } 613 static inline int rmp_make_shared(u64 pfn, enum pg_level level) { return -ENODEV; } 614 static inline void snp_leak_pages(u64 pfn, unsigned int npages) {} 615 static inline void kdump_sev_callback(void) { } 616 static inline void snp_fixup_e820_tables(void) {} 617 #endif 618 619 #endif 620