1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * AMD SVM-SEV support 6 * 7 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 8 */ 9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 10 11 #include <linux/kvm_types.h> 12 #include <linux/kvm_host.h> 13 #include <linux/kernel.h> 14 #include <linux/highmem.h> 15 #include <linux/psp.h> 16 #include <linux/psp-sev.h> 17 #include <linux/pagemap.h> 18 #include <linux/swap.h> 19 #include <linux/misc_cgroup.h> 20 #include <linux/processor.h> 21 #include <linux/trace_events.h> 22 #include <uapi/linux/sev-guest.h> 23 24 #include <asm/pkru.h> 25 #include <asm/trapnr.h> 26 #include <asm/cpuid/api.h> 27 #include <asm/fpu/xcr.h> 28 #include <asm/fpu/xstate.h> 29 #include <asm/debugreg.h> 30 #include <asm/msr.h> 31 #include <asm/sev.h> 32 33 #include "mmu.h" 34 #include "x86.h" 35 #include "svm.h" 36 #include "svm_ops.h" 37 #include "cpuid.h" 38 #include "trace.h" 39 40 #define GHCB_VERSION_MAX 2ULL 41 #define GHCB_VERSION_MIN 1ULL 42 43 #define GHCB_HV_FT_SUPPORTED (GHCB_HV_FT_SNP | GHCB_HV_FT_SNP_AP_CREATION) 44 45 /* 46 * The GHCB spec essentially states that all non-zero error codes other than 47 * those explicitly defined above should be treated as an error by the guest. 48 * Define a generic error to cover that case, and choose a value that is not 49 * likely to overlap with new explicit error codes should more be added to 50 * the GHCB spec later. KVM will use this to report generic errors when 51 * handling SNP guest requests. 52 */ 53 #define SNP_GUEST_VMM_ERR_GENERIC (~0U) 54 55 /* enable/disable SEV support */ 56 static bool __ro_after_init sev_enabled = true; 57 module_param_named(sev, sev_enabled, bool, 0444); 58 59 /* enable/disable SEV-ES support */ 60 static bool __ro_after_init sev_es_enabled = true; 61 module_param_named(sev_es, sev_es_enabled, bool, 0444); 62 63 /* enable/disable SEV-SNP support */ 64 static bool __ro_after_init sev_snp_enabled = true; 65 module_param_named(sev_snp, sev_snp_enabled, bool, 0444); 66 67 static unsigned int __ro_after_init nr_ciphertext_hiding_asids; 68 module_param_named(ciphertext_hiding_asids, nr_ciphertext_hiding_asids, uint, 0444); 69 70 #define AP_RESET_HOLD_NONE 0 71 #define AP_RESET_HOLD_NAE_EVENT 1 72 #define AP_RESET_HOLD_MSR_PROTO 2 73 74 /* 75 * SEV-SNP policy bits that can be supported by KVM. These include policy bits 76 * that have implementation support within KVM or policy bits that do not 77 * require implementation support within KVM to enforce the policy. 78 */ 79 #define KVM_SNP_POLICY_MASK_VALID (SNP_POLICY_MASK_API_MINOR | \ 80 SNP_POLICY_MASK_API_MAJOR | \ 81 SNP_POLICY_MASK_SMT | \ 82 SNP_POLICY_MASK_RSVD_MBO | \ 83 SNP_POLICY_MASK_DEBUG | \ 84 SNP_POLICY_MASK_SINGLE_SOCKET | \ 85 SNP_POLICY_MASK_CXL_ALLOW | \ 86 SNP_POLICY_MASK_MEM_AES_256_XTS | \ 87 SNP_POLICY_MASK_RAPL_DIS | \ 88 SNP_POLICY_MASK_CIPHERTEXT_HIDING_DRAM | \ 89 SNP_POLICY_MASK_PAGE_SWAP_DISABLE) 90 91 static u64 snp_supported_policy_bits __ro_after_init; 92 93 static u64 sev_supported_vmsa_features __ro_after_init; 94 95 #define INITIAL_VMSA_GPA 0xFFFFFFFFF000 96 97 static u8 sev_enc_bit; 98 static DECLARE_RWSEM(sev_deactivate_lock); 99 static DEFINE_MUTEX(sev_bitmap_lock); 100 /* Protects kvm_sev_info's enc_context_owner, mirror_vms and mirror_entry. */ 101 static DEFINE_MUTEX(sev_mirror_lock); 102 unsigned int max_sev_asid; 103 static unsigned int min_sev_asid; 104 static unsigned int max_sev_es_asid; 105 static unsigned int min_sev_es_asid; 106 static unsigned int max_snp_asid; 107 static unsigned int min_snp_asid; 108 static unsigned long sev_me_mask; 109 static unsigned int nr_asids; 110 static unsigned long *sev_asid_bitmap; 111 static unsigned long *sev_reclaim_asid_bitmap; 112 113 static __always_inline void kvm_lockdep_assert_sev_lock_held(struct kvm *kvm) 114 { 115 #ifdef CONFIG_PROVE_LOCKING 116 /* 117 * Querying SEV+ support is safe if there are no other references, i.e. 118 * if concurrent initialization of SEV+ is impossible. 119 */ 120 if (!refcount_read(&kvm->users_count)) 121 return; 122 123 /* 124 * Querying SEV+ support from vCPU context is always safe, as vCPUs can 125 * only be created after SEV+ is initialized (and KVM disallows all SEV 126 * sub-ioctls while vCPU creation is in-progress). 127 */ 128 if (kvm_get_running_vcpu()) 129 return; 130 131 lockdep_assert_held(&kvm->lock); 132 #endif 133 } 134 135 static bool sev_guest(struct kvm *kvm) 136 { 137 kvm_lockdep_assert_sev_lock_held(kvm); 138 return ____sev_guest(kvm); 139 } 140 static bool sev_es_guest(struct kvm *kvm) 141 { 142 kvm_lockdep_assert_sev_lock_held(kvm); 143 return ____sev_es_guest(kvm); 144 } 145 146 static bool sev_snp_guest(struct kvm *kvm) 147 { 148 kvm_lockdep_assert_sev_lock_held(kvm); 149 return ____sev_snp_guest(kvm); 150 } 151 152 static int snp_decommission_context(struct kvm *kvm); 153 154 struct enc_region { 155 struct list_head list; 156 unsigned long npages; 157 struct page **pages; 158 unsigned long uaddr; 159 unsigned long size; 160 }; 161 162 /* Called with the sev_bitmap_lock held, or on shutdown */ 163 static int sev_flush_asids(unsigned int min_asid, unsigned int max_asid) 164 { 165 int ret, error = 0; 166 unsigned int asid; 167 168 /* Check if there are any ASIDs to reclaim before performing a flush */ 169 asid = find_next_bit(sev_reclaim_asid_bitmap, nr_asids, min_asid); 170 if (asid > max_asid) 171 return -EBUSY; 172 173 /* 174 * DEACTIVATE will clear the WBINVD indicator causing DF_FLUSH to fail, 175 * so it must be guarded. 176 */ 177 down_write(&sev_deactivate_lock); 178 179 /* SNP firmware requires use of WBINVD for ASID recycling. */ 180 wbinvd_on_all_cpus(); 181 182 if (sev_snp_enabled) 183 ret = sev_do_cmd(SEV_CMD_SNP_DF_FLUSH, NULL, &error); 184 else 185 ret = sev_guest_df_flush(&error); 186 187 up_write(&sev_deactivate_lock); 188 189 if (ret) 190 pr_err("SEV%s: DF_FLUSH failed, ret=%d, error=%#x\n", 191 sev_snp_enabled ? "-SNP" : "", ret, error); 192 193 return ret; 194 } 195 196 static inline bool is_mirroring_enc_context(struct kvm *kvm) 197 { 198 return !!to_kvm_sev_info(kvm)->enc_context_owner; 199 } 200 201 static bool sev_vcpu_has_debug_swap(struct vcpu_svm *svm) 202 { 203 struct kvm_vcpu *vcpu = &svm->vcpu; 204 struct kvm_sev_info *sev = to_kvm_sev_info(vcpu->kvm); 205 206 return sev->vmsa_features & SVM_SEV_FEAT_DEBUG_SWAP; 207 } 208 209 static bool snp_is_secure_tsc_enabled(struct kvm *kvm) 210 { 211 struct kvm_sev_info *sev = to_kvm_sev_info(kvm); 212 213 return (sev->vmsa_features & SVM_SEV_FEAT_SECURE_TSC) && 214 !WARN_ON_ONCE(!sev_snp_guest(kvm)); 215 } 216 217 /* Must be called with the sev_bitmap_lock held */ 218 static bool __sev_recycle_asids(unsigned int min_asid, unsigned int max_asid) 219 { 220 if (sev_flush_asids(min_asid, max_asid)) 221 return false; 222 223 /* The flush process will flush all reclaimable SEV and SEV-ES ASIDs */ 224 bitmap_xor(sev_asid_bitmap, sev_asid_bitmap, sev_reclaim_asid_bitmap, 225 nr_asids); 226 bitmap_zero(sev_reclaim_asid_bitmap, nr_asids); 227 228 return true; 229 } 230 231 static int sev_misc_cg_try_charge(struct kvm_sev_info *sev) 232 { 233 enum misc_res_type type = sev->es_active ? MISC_CG_RES_SEV_ES : MISC_CG_RES_SEV; 234 return misc_cg_try_charge(type, sev->misc_cg, 1); 235 } 236 237 static void sev_misc_cg_uncharge(struct kvm_sev_info *sev) 238 { 239 enum misc_res_type type = sev->es_active ? MISC_CG_RES_SEV_ES : MISC_CG_RES_SEV; 240 misc_cg_uncharge(type, sev->misc_cg, 1); 241 } 242 243 static unsigned int sev_alloc_asid(unsigned int min_asid, unsigned int max_asid) 244 { 245 unsigned int asid; 246 bool retry = true; 247 248 guard(mutex)(&sev_bitmap_lock); 249 250 again: 251 asid = find_next_zero_bit(sev_asid_bitmap, max_asid + 1, min_asid); 252 if (asid > max_asid) { 253 if (retry && __sev_recycle_asids(min_asid, max_asid)) { 254 retry = false; 255 goto again; 256 } 257 258 return asid; 259 } 260 261 __set_bit(asid, sev_asid_bitmap); 262 return asid; 263 } 264 265 static int sev_asid_new(struct kvm_sev_info *sev, unsigned long vm_type) 266 { 267 /* 268 * SEV-enabled guests must use asid from min_sev_asid to max_sev_asid. 269 * SEV-ES-enabled guest can use from 1 to min_sev_asid - 1. 270 */ 271 unsigned int min_asid, max_asid, asid; 272 int ret; 273 274 if (vm_type == KVM_X86_SNP_VM) { 275 min_asid = min_snp_asid; 276 max_asid = max_snp_asid; 277 } else if (sev->es_active) { 278 min_asid = min_sev_es_asid; 279 max_asid = max_sev_es_asid; 280 } else { 281 min_asid = min_sev_asid; 282 max_asid = max_sev_asid; 283 } 284 285 /* 286 * The min ASID can end up larger than the max if basic SEV support is 287 * effectively disabled by disallowing use of ASIDs for SEV guests. 288 * Similarly for SEV-ES guests the min ASID can end up larger than the 289 * max when ciphertext hiding is enabled, effectively disabling SEV-ES 290 * support. 291 */ 292 if (min_asid > max_asid) 293 return -ENOTTY; 294 295 WARN_ON_ONCE(sev->misc_cg); 296 sev->misc_cg = get_current_misc_cg(); 297 ret = sev_misc_cg_try_charge(sev); 298 if (ret) 299 goto e_put_cg; 300 301 asid = sev_alloc_asid(min_asid, max_asid); 302 if (asid > max_asid) { 303 ret = -EBUSY; 304 goto e_uncharge; 305 } 306 307 sev->asid = asid; 308 return 0; 309 310 e_uncharge: 311 sev_misc_cg_uncharge(sev); 312 e_put_cg: 313 put_misc_cg(sev->misc_cg); 314 sev->misc_cg = NULL; 315 return ret; 316 } 317 318 static unsigned int sev_get_asid(struct kvm *kvm) 319 { 320 return to_kvm_sev_info(kvm)->asid; 321 } 322 323 static void sev_asid_free(struct kvm_sev_info *sev) 324 { 325 struct svm_cpu_data *sd; 326 int cpu; 327 328 mutex_lock(&sev_bitmap_lock); 329 330 __set_bit(sev->asid, sev_reclaim_asid_bitmap); 331 332 for_each_possible_cpu(cpu) { 333 sd = per_cpu_ptr(&svm_data, cpu); 334 sd->sev_vmcbs[sev->asid] = NULL; 335 } 336 337 mutex_unlock(&sev_bitmap_lock); 338 339 sev_misc_cg_uncharge(sev); 340 put_misc_cg(sev->misc_cg); 341 sev->misc_cg = NULL; 342 } 343 344 static void sev_decommission(unsigned int handle) 345 { 346 struct sev_data_decommission decommission; 347 348 if (!handle) 349 return; 350 351 decommission.handle = handle; 352 sev_guest_decommission(&decommission, NULL); 353 } 354 355 /* 356 * Transition a page to hypervisor-owned/shared state in the RMP table. This 357 * should not fail under normal conditions, but leak the page should that 358 * happen since it will no longer be usable by the host due to RMP protections. 359 */ 360 static int kvm_rmp_make_shared(struct kvm *kvm, u64 pfn, enum pg_level level) 361 { 362 if (KVM_BUG_ON(rmp_make_shared(pfn, level), kvm)) { 363 snp_leak_pages(pfn, page_level_size(level) >> PAGE_SHIFT); 364 return -EIO; 365 } 366 367 return 0; 368 } 369 370 /* 371 * Certain page-states, such as Pre-Guest and Firmware pages (as documented 372 * in Chapter 5 of the SEV-SNP Firmware ABI under "Page States") cannot be 373 * directly transitioned back to normal/hypervisor-owned state via RMPUPDATE 374 * unless they are reclaimed first. 375 * 376 * Until they are reclaimed and subsequently transitioned via RMPUPDATE, they 377 * might not be usable by the host due to being set as immutable or still 378 * being associated with a guest ASID. 379 * 380 * Bug the VM and leak the page if reclaim fails, or if the RMP entry can't be 381 * converted back to shared, as the page is no longer usable due to RMP 382 * protections, and it's infeasible for the guest to continue on. 383 */ 384 static int snp_page_reclaim(struct kvm *kvm, u64 pfn) 385 { 386 struct sev_data_snp_page_reclaim data = {0}; 387 int fw_err, rc; 388 389 data.paddr = __sme_set(pfn << PAGE_SHIFT); 390 rc = sev_do_cmd(SEV_CMD_SNP_PAGE_RECLAIM, &data, &fw_err); 391 if (KVM_BUG(rc, kvm, "Failed to reclaim PFN %llx, rc %d fw_err %d", pfn, rc, fw_err)) { 392 snp_leak_pages(pfn, 1); 393 return -EIO; 394 } 395 396 if (kvm_rmp_make_shared(kvm, pfn, PG_LEVEL_4K)) 397 return -EIO; 398 399 return rc; 400 } 401 402 static void sev_unbind_asid(struct kvm *kvm, unsigned int handle) 403 { 404 struct sev_data_deactivate deactivate; 405 406 if (!handle) 407 return; 408 409 deactivate.handle = handle; 410 411 /* Guard DEACTIVATE against WBINVD/DF_FLUSH used in ASID recycling */ 412 down_read(&sev_deactivate_lock); 413 sev_guest_deactivate(&deactivate, NULL); 414 up_read(&sev_deactivate_lock); 415 416 sev_decommission(handle); 417 } 418 419 /* 420 * This sets up bounce buffers/firmware pages to handle SNP Guest Request 421 * messages (e.g. attestation requests). See "SNP Guest Request" in the GHCB 422 * 2.0 specification for more details. 423 * 424 * Technically, when an SNP Guest Request is issued, the guest will provide its 425 * own request/response pages, which could in theory be passed along directly 426 * to firmware rather than using bounce pages. However, these pages would need 427 * special care: 428 * 429 * - Both pages are from shared guest memory, so they need to be protected 430 * from migration/etc. occurring while firmware reads/writes to them. At a 431 * minimum, this requires elevating the ref counts and potentially needing 432 * an explicit pinning of the memory. This places additional restrictions 433 * on what type of memory backends userspace can use for shared guest 434 * memory since there is some reliance on using refcounted pages. 435 * 436 * - The response page needs to be switched to Firmware-owned[1] state 437 * before the firmware can write to it, which can lead to potential 438 * host RMP #PFs if the guest is misbehaved and hands the host a 439 * guest page that KVM might write to for other reasons (e.g. virtio 440 * buffers/etc.). 441 * 442 * Both of these issues can be avoided completely by using separately-allocated 443 * bounce pages for both the request/response pages and passing those to 444 * firmware instead. So that's what is being set up here. 445 * 446 * Guest requests rely on message sequence numbers to ensure requests are 447 * issued to firmware in the order the guest issues them, so concurrent guest 448 * requests generally shouldn't happen. But a misbehaved guest could issue 449 * concurrent guest requests in theory, so a mutex is used to serialize 450 * access to the bounce buffers. 451 * 452 * [1] See the "Page States" section of the SEV-SNP Firmware ABI for more 453 * details on Firmware-owned pages, along with "RMP and VMPL Access Checks" 454 * in the APM for details on the related RMP restrictions. 455 */ 456 static int snp_guest_req_init(struct kvm *kvm) 457 { 458 struct kvm_sev_info *sev = to_kvm_sev_info(kvm); 459 struct page *req_page; 460 461 req_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 462 if (!req_page) 463 return -ENOMEM; 464 465 sev->guest_resp_buf = snp_alloc_firmware_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 466 if (!sev->guest_resp_buf) { 467 __free_page(req_page); 468 return -EIO; 469 } 470 471 sev->guest_req_buf = page_address(req_page); 472 mutex_init(&sev->guest_req_mutex); 473 474 return 0; 475 } 476 477 static void snp_guest_req_cleanup(struct kvm *kvm) 478 { 479 struct kvm_sev_info *sev = to_kvm_sev_info(kvm); 480 481 if (sev->guest_resp_buf) 482 snp_free_firmware_page(sev->guest_resp_buf); 483 484 if (sev->guest_req_buf) 485 __free_page(virt_to_page(sev->guest_req_buf)); 486 487 sev->guest_req_buf = NULL; 488 sev->guest_resp_buf = NULL; 489 } 490 491 static int sev_alloc_have_run_cpus(struct kvm_sev_info *sev) 492 { 493 if (!zalloc_cpumask_var(&sev->have_run_cpus, GFP_KERNEL_ACCOUNT)) 494 return -ENOMEM; 495 496 return 0; 497 } 498 499 static void sev_free_have_run_cpus(struct kvm_sev_info *sev) 500 { 501 free_cpumask_var(sev->have_run_cpus); 502 memset(&sev->have_run_cpus, 0, sizeof(sev->have_run_cpus)); 503 } 504 505 static int __sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp, 506 struct kvm_sev_init *data, 507 unsigned long vm_type) 508 { 509 struct kvm_sev_info *sev = to_kvm_sev_info(kvm); 510 struct sev_platform_init_args init_args = {0}; 511 bool es_active = vm_type != KVM_X86_SEV_VM; 512 bool snp_active = vm_type == KVM_X86_SNP_VM; 513 u64 valid_vmsa_features = es_active ? sev_supported_vmsa_features : 0; 514 int ret; 515 516 if (kvm->created_vcpus) 517 return -EINVAL; 518 519 if (data->flags) 520 return -EINVAL; 521 522 if (!snp_active) 523 valid_vmsa_features &= ~SVM_SEV_FEAT_SECURE_TSC; 524 525 if (data->vmsa_features & ~valid_vmsa_features) 526 return -EINVAL; 527 528 if (data->ghcb_version > GHCB_VERSION_MAX || (!es_active && data->ghcb_version)) 529 return -EINVAL; 530 531 /* 532 * KVM supports the full range of mandatory features defined by version 533 * 2 of the GHCB protocol, so default to that for SEV-ES guests created 534 * via KVM_SEV_INIT2 (KVM_SEV_INIT forces version 1). 535 */ 536 if (es_active && !data->ghcb_version) 537 data->ghcb_version = 2; 538 539 if (snp_active && data->ghcb_version < 2) 540 return -EINVAL; 541 542 if (unlikely(sev->active)) 543 return -EINVAL; 544 545 sev->active = true; 546 sev->es_active = es_active; 547 sev->vmsa_features = data->vmsa_features; 548 sev->ghcb_version = data->ghcb_version; 549 550 if (snp_active) 551 sev->vmsa_features |= SVM_SEV_FEAT_SNP_ACTIVE; 552 553 ret = sev_asid_new(sev, vm_type); 554 if (ret) 555 goto e_no_asid; 556 557 init_args.probe = false; 558 ret = sev_platform_init(&init_args); 559 if (ret) 560 goto e_free_asid; 561 562 ret = sev_alloc_have_run_cpus(sev); 563 if (ret) 564 goto e_free_asid; 565 566 /* This needs to happen after SEV/SNP firmware initialization. */ 567 if (snp_active) { 568 ret = snp_guest_req_init(kvm); 569 if (ret) 570 goto e_free; 571 } 572 573 INIT_LIST_HEAD(&sev->regions_list); 574 INIT_LIST_HEAD(&sev->mirror_vms); 575 sev->need_init = false; 576 kvm->arch.has_protected_page_tables = true; 577 578 kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_SEV); 579 580 return 0; 581 582 e_free: 583 sev_free_have_run_cpus(sev); 584 e_free_asid: 585 argp->error = init_args.error; 586 sev_asid_free(sev); 587 sev->asid = 0; 588 e_no_asid: 589 sev->vmsa_features = 0; 590 sev->es_active = false; 591 sev->active = false; 592 return ret; 593 } 594 595 static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp) 596 { 597 struct kvm_sev_init data = { 598 .vmsa_features = 0, 599 .ghcb_version = 0, 600 }; 601 unsigned long vm_type; 602 603 if (kvm->arch.vm_type != KVM_X86_DEFAULT_VM) 604 return -EINVAL; 605 606 vm_type = (argp->id == KVM_SEV_INIT ? KVM_X86_SEV_VM : KVM_X86_SEV_ES_VM); 607 608 /* 609 * KVM_SEV_ES_INIT has been deprecated by KVM_SEV_INIT2, so it will 610 * continue to only ever support the minimal GHCB protocol version. 611 */ 612 if (vm_type == KVM_X86_SEV_ES_VM) 613 data.ghcb_version = GHCB_VERSION_MIN; 614 615 return __sev_guest_init(kvm, argp, &data, vm_type); 616 } 617 618 static int sev_guest_init2(struct kvm *kvm, struct kvm_sev_cmd *argp) 619 { 620 struct kvm_sev_init data; 621 622 if (!to_kvm_sev_info(kvm)->need_init) 623 return -EINVAL; 624 625 if (kvm->arch.vm_type != KVM_X86_SEV_VM && 626 kvm->arch.vm_type != KVM_X86_SEV_ES_VM && 627 kvm->arch.vm_type != KVM_X86_SNP_VM) 628 return -EINVAL; 629 630 if (copy_from_user(&data, u64_to_user_ptr(argp->data), sizeof(data))) 631 return -EFAULT; 632 633 return __sev_guest_init(kvm, argp, &data, kvm->arch.vm_type); 634 } 635 636 static int sev_bind_asid(struct kvm *kvm, unsigned int handle, int *error) 637 { 638 unsigned int asid = sev_get_asid(kvm); 639 struct sev_data_activate activate; 640 int ret; 641 642 /* activate ASID on the given handle */ 643 activate.handle = handle; 644 activate.asid = asid; 645 ret = sev_guest_activate(&activate, error); 646 647 return ret; 648 } 649 650 static int __sev_issue_cmd(int fd, int id, void *data, int *error) 651 { 652 CLASS(fd, f)(fd); 653 654 if (fd_empty(f)) 655 return -EBADF; 656 657 return sev_issue_cmd_external_user(fd_file(f), id, data, error); 658 } 659 660 static int sev_issue_cmd(struct kvm *kvm, int id, void *data, int *error) 661 { 662 struct kvm_sev_info *sev = to_kvm_sev_info(kvm); 663 664 return __sev_issue_cmd(sev->fd, id, data, error); 665 } 666 667 static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp) 668 { 669 struct kvm_sev_info *sev = to_kvm_sev_info(kvm); 670 struct sev_data_launch_start start; 671 struct kvm_sev_launch_start params; 672 void *dh_blob, *session_blob; 673 int *error = &argp->error; 674 int ret; 675 676 if (!sev_guest(kvm)) 677 return -ENOTTY; 678 679 if (copy_from_user(¶ms, u64_to_user_ptr(argp->data), sizeof(params))) 680 return -EFAULT; 681 682 memset(&start, 0, sizeof(start)); 683 684 dh_blob = NULL; 685 if (params.dh_uaddr) { 686 dh_blob = psp_copy_user_blob(params.dh_uaddr, params.dh_len); 687 if (IS_ERR(dh_blob)) 688 return PTR_ERR(dh_blob); 689 690 start.dh_cert_address = __sme_set(__pa(dh_blob)); 691 start.dh_cert_len = params.dh_len; 692 } 693 694 session_blob = NULL; 695 if (params.session_uaddr) { 696 session_blob = psp_copy_user_blob(params.session_uaddr, params.session_len); 697 if (IS_ERR(session_blob)) { 698 ret = PTR_ERR(session_blob); 699 goto e_free_dh; 700 } 701 702 start.session_address = __sme_set(__pa(session_blob)); 703 start.session_len = params.session_len; 704 } 705 706 start.handle = params.handle; 707 start.policy = params.policy; 708 709 /* create memory encryption context */ 710 ret = __sev_issue_cmd(argp->sev_fd, SEV_CMD_LAUNCH_START, &start, error); 711 if (ret) 712 goto e_free_session; 713 714 /* Bind ASID to this guest */ 715 ret = sev_bind_asid(kvm, start.handle, error); 716 if (ret) { 717 sev_decommission(start.handle); 718 goto e_free_session; 719 } 720 721 /* return handle to userspace */ 722 params.handle = start.handle; 723 if (copy_to_user(u64_to_user_ptr(argp->data), ¶ms, sizeof(params))) { 724 sev_unbind_asid(kvm, start.handle); 725 ret = -EFAULT; 726 goto e_free_session; 727 } 728 729 sev->policy = params.policy; 730 sev->handle = start.handle; 731 sev->fd = argp->sev_fd; 732 733 e_free_session: 734 kfree(session_blob); 735 e_free_dh: 736 kfree(dh_blob); 737 return ret; 738 } 739 740 static int sev_check_pin_count(struct kvm *kvm, unsigned long npages) 741 { 742 unsigned long total_npages, lock_limit; 743 744 total_npages = to_kvm_sev_info(kvm)->pages_locked + npages; 745 if (total_npages > totalram_pages()) 746 return -EINVAL; 747 748 lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 749 if (total_npages > lock_limit && !capable(CAP_IPC_LOCK)) { 750 pr_err_ratelimited("SEV: %lu total pages would exceed the lock limit of %lu.\n", 751 total_npages, lock_limit); 752 return -ENOMEM; 753 } 754 755 return 0; 756 } 757 758 static int sev_pin_user_pages(struct kvm *kvm, unsigned long addr, int npages, 759 unsigned int gup_flags, struct page **pages) 760 { 761 int npinned; 762 763 lockdep_assert_held(&kvm->lock); 764 765 npinned = pin_user_pages_fast(addr, npages, gup_flags, pages); 766 if (npinned != npages) { 767 if (npinned > 0) 768 unpin_user_pages(pages, npinned); 769 pr_err_ratelimited("SEV: Failure locking %u pages.\n", npages); 770 return -ENOMEM; 771 } 772 773 to_kvm_sev_info(kvm)->pages_locked += npages; 774 return 0; 775 } 776 777 static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr, 778 unsigned long ulen, unsigned long *n, 779 unsigned int flags) 780 { 781 unsigned long npages; 782 struct page **pages; 783 int ret; 784 785 lockdep_assert_held(&kvm->lock); 786 787 if (ulen == 0 || uaddr + ulen < uaddr) 788 return ERR_PTR(-EINVAL); 789 790 /* 791 * Calculate the number of pages that need to be pinned to cover the 792 * entire range. Note! This isn't simply PFN_DOWN(ulen), as KVM 793 * doesn't require the incoming address+size to be page aligned! 794 */ 795 npages = PFN_DOWN(uaddr + ulen - 1) - PFN_DOWN(uaddr) + 1; 796 if (npages > INT_MAX) 797 return ERR_PTR(-EINVAL); 798 799 ret = sev_check_pin_count(kvm, npages); 800 if (ret) 801 return ERR_PTR(ret); 802 803 /* 804 * Don't WARN if the kernel (rightly) thinks the total size is absurd, 805 * i.e. rely on the kernel to reject outrageous range sizes. The above 806 * check on the number of pages is purely to avoid truncation as 807 * pin_user_pages_fast() takes the number of pages as a 32-bit int. 808 */ 809 pages = kvzalloc_objs(*pages, npages, GFP_KERNEL_ACCOUNT | __GFP_NOWARN); 810 if (!pages) 811 return ERR_PTR(-ENOMEM); 812 813 ret = sev_pin_user_pages(kvm, uaddr, npages, flags, pages); 814 if (ret) { 815 kvfree(pages); 816 return ERR_PTR(ret); 817 } 818 819 *n = npages; 820 return pages; 821 } 822 823 static void sev_unpin_memory(struct kvm *kvm, struct page **pages, 824 unsigned long npages) 825 { 826 unpin_user_pages(pages, npages); 827 kvfree(pages); 828 to_kvm_sev_info(kvm)->pages_locked -= npages; 829 } 830 831 static struct page *sev_pin_page(struct kvm *kvm, unsigned long addr, 832 unsigned int flags) 833 { 834 struct page *page; 835 int r; 836 837 r = sev_check_pin_count(kvm, 1); 838 if (r) 839 return ERR_PTR(r); 840 841 r = sev_pin_user_pages(kvm, addr, 1, flags, &page); 842 if (r) 843 return ERR_PTR(r); 844 845 return page; 846 } 847 848 static void sev_unpin_page(struct kvm *kvm, struct page *page) 849 { 850 unpin_user_pages(&page, 1); 851 to_kvm_sev_info(kvm)->pages_locked -= 1; 852 } 853 854 static void sev_clflush_pages(struct page *pages[], unsigned long npages) 855 { 856 uint8_t *page_virtual; 857 unsigned long i; 858 859 if (this_cpu_has(X86_FEATURE_SME_COHERENT) || npages == 0 || 860 pages == NULL) 861 return; 862 863 for (i = 0; i < npages; i++) { 864 page_virtual = kmap_local_page(pages[i]); 865 clflush_cache_range(page_virtual, PAGE_SIZE); 866 kunmap_local(page_virtual); 867 cond_resched(); 868 } 869 } 870 871 static void sev_writeback_caches(struct kvm *kvm) 872 { 873 /* 874 * Ensure that all dirty guest tagged cache entries are written back 875 * before releasing the pages back to the system for use. CLFLUSH will 876 * not do this without SME_COHERENT, and flushing many cache lines 877 * individually is slower than blasting WBINVD for large VMs, so issue 878 * WBNOINVD (or WBINVD if the "no invalidate" variant is unsupported) 879 * on CPUs that have done VMRUN, i.e. may have dirtied data using the 880 * VM's ASID. 881 * 882 * For simplicity, never remove CPUs from the bitmap. Ideally, KVM 883 * would clear the mask when flushing caches, but doing so requires 884 * serializing multiple calls and having responding CPUs (to the IPI) 885 * mark themselves as still running if they are running (or about to 886 * run) a vCPU for the VM. 887 * 888 * Note, the caller is responsible for ensuring correctness if the mask 889 * can be modified, e.g. if a CPU could be doing VMRUN. 890 */ 891 wbnoinvd_on_cpus_mask(to_kvm_sev_info(kvm)->have_run_cpus); 892 } 893 894 static unsigned long get_num_contig_pages(unsigned long idx, 895 struct page **inpages, unsigned long npages) 896 { 897 unsigned long paddr, next_paddr; 898 unsigned long i = idx + 1, pages = 1; 899 900 /* find the number of contiguous pages starting from idx */ 901 paddr = __sme_page_pa(inpages[idx]); 902 while (i < npages) { 903 next_paddr = __sme_page_pa(inpages[i++]); 904 if ((paddr + PAGE_SIZE) == next_paddr) { 905 pages++; 906 paddr = next_paddr; 907 continue; 908 } 909 break; 910 } 911 912 return pages; 913 } 914 915 static int sev_launch_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp) 916 { 917 unsigned long vaddr, vaddr_end, next_vaddr, npages, pages, size, i; 918 struct kvm_sev_launch_update_data params; 919 struct sev_data_launch_update_data data; 920 struct page **inpages; 921 int ret; 922 923 if (!sev_guest(kvm)) 924 return -ENOTTY; 925 926 if (copy_from_user(¶ms, u64_to_user_ptr(argp->data), sizeof(params))) 927 return -EFAULT; 928 929 vaddr = params.uaddr; 930 size = params.len; 931 vaddr_end = vaddr + size; 932 933 /* Lock the user memory. */ 934 inpages = sev_pin_memory(kvm, vaddr, size, &npages, FOLL_WRITE); 935 if (IS_ERR(inpages)) 936 return PTR_ERR(inpages); 937 938 /* 939 * Flush (on non-coherent CPUs) before LAUNCH_UPDATE encrypts pages in 940 * place; the cache may contain the data that was written unencrypted. 941 */ 942 sev_clflush_pages(inpages, npages); 943 944 data.reserved = 0; 945 data.handle = to_kvm_sev_info(kvm)->handle; 946 947 for (i = 0; vaddr < vaddr_end; vaddr = next_vaddr, i += pages) { 948 int offset, len; 949 950 /* 951 * If the user buffer is not page-aligned, calculate the offset 952 * within the page. 953 */ 954 offset = vaddr & (PAGE_SIZE - 1); 955 956 /* Calculate the number of pages that can be encrypted in one go. */ 957 pages = get_num_contig_pages(i, inpages, npages); 958 959 len = min_t(size_t, ((pages * PAGE_SIZE) - offset), size); 960 961 data.len = len; 962 data.address = __sme_page_pa(inpages[i]) + offset; 963 ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_DATA, &data, &argp->error); 964 if (ret) 965 goto e_unpin; 966 967 size -= len; 968 next_vaddr = vaddr + len; 969 } 970 971 e_unpin: 972 /* content of memory is updated, mark pages dirty */ 973 for (i = 0; i < npages; i++) { 974 set_page_dirty_lock(inpages[i]); 975 mark_page_accessed(inpages[i]); 976 } 977 /* unlock the user pages */ 978 sev_unpin_memory(kvm, inpages, npages); 979 return ret; 980 } 981 982 static int sev_es_sync_vmsa(struct vcpu_svm *svm) 983 { 984 struct kvm_vcpu *vcpu = &svm->vcpu; 985 struct kvm_sev_info *sev = to_kvm_sev_info(vcpu->kvm); 986 struct sev_es_save_area *save = svm->sev_es.vmsa; 987 struct xregs_state *xsave; 988 const u8 *s; 989 u8 *d; 990 int i; 991 992 lockdep_assert_held(&vcpu->mutex); 993 994 if (vcpu->arch.guest_state_protected) 995 return -EINVAL; 996 997 /* Check some debug related fields before encrypting the VMSA */ 998 if (svm->vcpu.guest_debug || (svm->vmcb->save.dr7 & ~DR7_FIXED_1)) 999 return -EINVAL; 1000 1001 /* 1002 * SEV-ES will use a VMSA that is pointed to by the VMCB, not 1003 * the traditional VMSA that is part of the VMCB. Copy the 1004 * traditional VMSA as it has been built so far (in prep 1005 * for LAUNCH_UPDATE_VMSA) to be the initial SEV-ES state. 1006 */ 1007 memcpy(save, &svm->vmcb->save, sizeof(svm->vmcb->save)); 1008 1009 /* Sync registgers */ 1010 save->rax = svm->vcpu.arch.regs[VCPU_REGS_RAX]; 1011 save->rbx = svm->vcpu.arch.regs[VCPU_REGS_RBX]; 1012 save->rcx = svm->vcpu.arch.regs[VCPU_REGS_RCX]; 1013 save->rdx = svm->vcpu.arch.regs[VCPU_REGS_RDX]; 1014 save->rsp = svm->vcpu.arch.regs[VCPU_REGS_RSP]; 1015 save->rbp = svm->vcpu.arch.regs[VCPU_REGS_RBP]; 1016 save->rsi = svm->vcpu.arch.regs[VCPU_REGS_RSI]; 1017 save->rdi = svm->vcpu.arch.regs[VCPU_REGS_RDI]; 1018 #ifdef CONFIG_X86_64 1019 save->r8 = svm->vcpu.arch.regs[VCPU_REGS_R8]; 1020 save->r9 = svm->vcpu.arch.regs[VCPU_REGS_R9]; 1021 save->r10 = svm->vcpu.arch.regs[VCPU_REGS_R10]; 1022 save->r11 = svm->vcpu.arch.regs[VCPU_REGS_R11]; 1023 save->r12 = svm->vcpu.arch.regs[VCPU_REGS_R12]; 1024 save->r13 = svm->vcpu.arch.regs[VCPU_REGS_R13]; 1025 save->r14 = svm->vcpu.arch.regs[VCPU_REGS_R14]; 1026 save->r15 = svm->vcpu.arch.regs[VCPU_REGS_R15]; 1027 #endif 1028 save->rip = svm->vcpu.arch.rip; 1029 1030 /* Sync some non-GPR registers before encrypting */ 1031 save->xcr0 = svm->vcpu.arch.xcr0; 1032 save->pkru = svm->vcpu.arch.pkru; 1033 save->xss = svm->vcpu.arch.ia32_xss; 1034 save->dr6 = svm->vcpu.arch.dr6; 1035 1036 save->sev_features = sev->vmsa_features; 1037 1038 /* 1039 * Skip FPU and AVX setup with KVM_SEV_ES_INIT to avoid 1040 * breaking older measurements. 1041 */ 1042 if (vcpu->kvm->arch.vm_type != KVM_X86_DEFAULT_VM) { 1043 xsave = &vcpu->arch.guest_fpu.fpstate->regs.xsave; 1044 save->x87_dp = xsave->i387.rdp; 1045 save->mxcsr = xsave->i387.mxcsr; 1046 save->x87_ftw = xsave->i387.twd; 1047 save->x87_fsw = xsave->i387.swd; 1048 save->x87_fcw = xsave->i387.cwd; 1049 save->x87_fop = xsave->i387.fop; 1050 save->x87_ds = 0; 1051 save->x87_cs = 0; 1052 save->x87_rip = xsave->i387.rip; 1053 1054 for (i = 0; i < 8; i++) { 1055 /* 1056 * The format of the x87 save area is undocumented and 1057 * definitely not what you would expect. It consists of 1058 * an 8*8 bytes area with bytes 0-7, and an 8*2 bytes 1059 * area with bytes 8-9 of each register. 1060 */ 1061 d = save->fpreg_x87 + i * 8; 1062 s = ((u8 *)xsave->i387.st_space) + i * 16; 1063 memcpy(d, s, 8); 1064 save->fpreg_x87[64 + i * 2] = s[8]; 1065 save->fpreg_x87[64 + i * 2 + 1] = s[9]; 1066 } 1067 memcpy(save->fpreg_xmm, xsave->i387.xmm_space, 256); 1068 1069 s = get_xsave_addr(xsave, XFEATURE_YMM); 1070 if (s) 1071 memcpy(save->fpreg_ymm, s, 256); 1072 else 1073 memset(save->fpreg_ymm, 0, 256); 1074 } 1075 1076 pr_debug("Virtual Machine Save Area (VMSA):\n"); 1077 print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, save, sizeof(*save), false); 1078 1079 return 0; 1080 } 1081 1082 static int __sev_launch_update_vmsa(struct kvm *kvm, struct kvm_vcpu *vcpu, 1083 int *error) 1084 { 1085 struct sev_data_launch_update_vmsa vmsa; 1086 struct vcpu_svm *svm = to_svm(vcpu); 1087 int ret; 1088 1089 if (vcpu->guest_debug) { 1090 pr_warn_once("KVM_SET_GUEST_DEBUG for SEV-ES guest is not supported"); 1091 return -EINVAL; 1092 } 1093 1094 /* Perform some pre-encryption checks against the VMSA */ 1095 ret = sev_es_sync_vmsa(svm); 1096 if (ret) 1097 return ret; 1098 1099 /* 1100 * The LAUNCH_UPDATE_VMSA command will perform in-place encryption of 1101 * the VMSA memory content (i.e it will write the same memory region 1102 * with the guest's key), so invalidate it first. 1103 */ 1104 clflush_cache_range(svm->sev_es.vmsa, PAGE_SIZE); 1105 1106 vmsa.reserved = 0; 1107 vmsa.handle = to_kvm_sev_info(kvm)->handle; 1108 vmsa.address = __sme_pa(svm->sev_es.vmsa); 1109 vmsa.len = PAGE_SIZE; 1110 ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_VMSA, &vmsa, error); 1111 if (ret) 1112 return ret; 1113 1114 /* 1115 * SEV-ES guests maintain an encrypted version of their FPU 1116 * state which is restored and saved on VMRUN and VMEXIT. 1117 * Mark vcpu->arch.guest_fpu->fpstate as scratch so it won't 1118 * do xsave/xrstor on it. 1119 */ 1120 fpstate_set_confidential(&vcpu->arch.guest_fpu); 1121 vcpu->arch.guest_state_protected = true; 1122 1123 /* 1124 * SEV-ES guest mandates LBR Virtualization to be _always_ ON. Enable it 1125 * only after setting guest_state_protected because KVM_SET_MSRS allows 1126 * dynamic toggling of LBRV (for performance reason) on write access to 1127 * MSR_IA32_DEBUGCTLMSR when guest_state_protected is not set. 1128 */ 1129 svm_enable_lbrv(vcpu); 1130 return 0; 1131 } 1132 1133 static int sev_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) 1134 { 1135 struct kvm_vcpu *vcpu; 1136 unsigned long i; 1137 int ret; 1138 1139 if (!sev_es_guest(kvm)) 1140 return -ENOTTY; 1141 1142 ret = kvm_lock_all_vcpus(kvm); 1143 if (ret) 1144 return ret; 1145 1146 kvm_for_each_vcpu(i, vcpu, kvm) { 1147 ret = __sev_launch_update_vmsa(kvm, vcpu, &argp->error); 1148 if (ret) 1149 break; 1150 } 1151 1152 kvm_unlock_all_vcpus(kvm); 1153 return ret; 1154 } 1155 1156 static int sev_launch_measure(struct kvm *kvm, struct kvm_sev_cmd *argp) 1157 { 1158 void __user *measure = u64_to_user_ptr(argp->data); 1159 struct sev_data_launch_measure data; 1160 struct kvm_sev_launch_measure params; 1161 void __user *p = NULL; 1162 void *blob = NULL; 1163 int ret; 1164 1165 if (!sev_guest(kvm)) 1166 return -ENOTTY; 1167 1168 if (copy_from_user(¶ms, measure, sizeof(params))) 1169 return -EFAULT; 1170 1171 memset(&data, 0, sizeof(data)); 1172 1173 /* User wants to query the blob length */ 1174 if (!params.len) 1175 goto cmd; 1176 1177 p = u64_to_user_ptr(params.uaddr); 1178 if (p) { 1179 if (params.len > SEV_FW_BLOB_MAX_SIZE) 1180 return -EINVAL; 1181 1182 blob = kzalloc(params.len, GFP_KERNEL_ACCOUNT); 1183 if (!blob) 1184 return -ENOMEM; 1185 1186 data.address = __psp_pa(blob); 1187 data.len = params.len; 1188 } 1189 1190 cmd: 1191 data.handle = to_kvm_sev_info(kvm)->handle; 1192 ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_MEASURE, &data, &argp->error); 1193 1194 /* 1195 * If we query the session length, FW responded with expected data. 1196 */ 1197 if (!params.len) 1198 goto done; 1199 1200 if (ret) 1201 goto e_free_blob; 1202 1203 if (blob) { 1204 if (copy_to_user(p, blob, params.len)) 1205 ret = -EFAULT; 1206 } 1207 1208 done: 1209 params.len = data.len; 1210 if (copy_to_user(measure, ¶ms, sizeof(params))) 1211 ret = -EFAULT; 1212 e_free_blob: 1213 kfree(blob); 1214 return ret; 1215 } 1216 1217 static int sev_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp) 1218 { 1219 struct sev_data_launch_finish data; 1220 1221 if (!sev_guest(kvm)) 1222 return -ENOTTY; 1223 1224 data.handle = to_kvm_sev_info(kvm)->handle; 1225 return sev_issue_cmd(kvm, SEV_CMD_LAUNCH_FINISH, &data, &argp->error); 1226 } 1227 1228 static int sev_guest_status(struct kvm *kvm, struct kvm_sev_cmd *argp) 1229 { 1230 struct kvm_sev_guest_status params; 1231 struct sev_data_guest_status data; 1232 int ret; 1233 1234 if (!sev_guest(kvm)) 1235 return -ENOTTY; 1236 1237 memset(&data, 0, sizeof(data)); 1238 1239 data.handle = to_kvm_sev_info(kvm)->handle; 1240 ret = sev_issue_cmd(kvm, SEV_CMD_GUEST_STATUS, &data, &argp->error); 1241 if (ret) 1242 return ret; 1243 1244 params.policy = data.policy; 1245 params.state = data.state; 1246 params.handle = data.handle; 1247 1248 if (copy_to_user(u64_to_user_ptr(argp->data), ¶ms, sizeof(params))) 1249 ret = -EFAULT; 1250 1251 return ret; 1252 } 1253 1254 static int sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src_pa, 1255 unsigned long dst_pa, unsigned int size, 1256 unsigned int ioctl, int *error) 1257 { 1258 int cmd = ioctl == KVM_SEV_DBG_DECRYPT ? SEV_CMD_DBG_DECRYPT : 1259 SEV_CMD_DBG_ENCRYPT; 1260 struct sev_data_dbg data = { 1261 .handle = to_kvm_sev_info(kvm)->handle, 1262 .dst_addr = dst_pa, 1263 .src_addr = src_pa, 1264 .len = size, 1265 }; 1266 1267 return sev_issue_cmd(kvm, cmd, &data, error); 1268 } 1269 1270 static void *sev_dbg_crypt_slow_alloc(struct page *page, unsigned long __va, 1271 unsigned int len, unsigned long *pa, 1272 unsigned int *nr_bytes) 1273 { 1274 unsigned long va = ALIGN_DOWN(__va, 16); 1275 1276 /* The number of bytes to {de,en}crypt must be 16-byte aligned. */ 1277 *nr_bytes = round_up(len, 16); 1278 1279 /* 1280 * Increase the number of bytes to {de,en}crypt by one chunk (16 bytes) 1281 * if the aligned address and length doesn't cover the unaligned range, 1282 * e.g. if the address is unaligned _and_ the access will split a chunk 1283 * at the tail. 1284 */ 1285 if (va + *nr_bytes < __va + len) 1286 *nr_bytes += 16; 1287 1288 *pa = __sme_page_pa(page) + (va & ~PAGE_MASK); 1289 1290 /* 1291 * Sanity check that the new access won't split a page. This should 1292 * never happen; just pretend the allocation failed. 1293 */ 1294 if (WARN_ON_ONCE((*pa & PAGE_MASK) != ((*pa + *nr_bytes - 1) & PAGE_MASK))) 1295 return NULL; 1296 1297 /* 1298 * If SNP is enabled, i.e. the RMP is active, allocate a full page to 1299 * prevent concurrent accesses to the page. As required by firmware, 1300 * the PSP driver updates the RMP to temporarily transfer ownership of 1301 * the page to Firmware while the {DE,EN}CRYPT operation is in-progress, 1302 * and so concurrent software accesses to the page will encounter 1303 * seemingly spurious RMP #PF violations 1304 */ 1305 if (cc_platform_has(CC_ATTR_HOST_SEV_SNP)) 1306 return (void *)__get_free_page(GFP_KERNEL); 1307 1308 return kmalloc(*nr_bytes, GFP_KERNEL); 1309 } 1310 1311 static void sev_dbg_crypt_slow_free(void *buf) 1312 { 1313 if (cc_platform_has(CC_ATTR_HOST_SEV_SNP)) 1314 free_page((unsigned long)buf); 1315 else 1316 kfree(buf); 1317 } 1318 1319 static int sev_dbg_decrypt_slow(struct kvm *kvm, unsigned long src, 1320 struct page *src_p, unsigned long dst, 1321 unsigned int len, int *err) 1322 { 1323 unsigned int nr_bytes; 1324 unsigned long src_pa; 1325 void *buf; 1326 int r; 1327 1328 buf = sev_dbg_crypt_slow_alloc(src_p, src, len, &src_pa, &nr_bytes); 1329 if (!buf) 1330 return -ENOMEM; 1331 1332 r = sev_issue_dbg_cmd(kvm, src_pa, __sme_set(__pa(buf)), 1333 nr_bytes, KVM_SEV_DBG_DECRYPT, err); 1334 if (r) 1335 goto out; 1336 1337 if (copy_to_user((void __user *)dst, buf + (src & 15), len)) 1338 r = -EFAULT; 1339 out: 1340 sev_dbg_crypt_slow_free(buf); 1341 return r; 1342 } 1343 1344 static int sev_dbg_encrypt_slow(struct kvm *kvm, unsigned long src, 1345 unsigned long dst, struct page *dst_p, 1346 unsigned int len, int *err) 1347 { 1348 unsigned int nr_bytes; 1349 unsigned long dst_pa; 1350 void *buf; 1351 int r; 1352 1353 /* Decrypt the _destination_ to do a RMW on plaintext. */ 1354 buf = sev_dbg_crypt_slow_alloc(dst_p, dst, len, &dst_pa, &nr_bytes); 1355 if (!buf) 1356 return -ENOMEM; 1357 1358 r = sev_issue_dbg_cmd(kvm, dst_pa, __sme_set(__pa(buf)), 1359 nr_bytes, KVM_SEV_DBG_DECRYPT, err); 1360 if (r) 1361 goto out; 1362 1363 /* 1364 * Copy from the source into the intermediate buffer, and then 1365 * re-encrypt the buffer into the destination. 1366 */ 1367 if (copy_from_user(buf + (dst & 15), (void __user *)src, len)) 1368 r = -EFAULT; 1369 else 1370 r = sev_issue_dbg_cmd(kvm, __sme_set(__pa(buf)), dst_pa, 1371 nr_bytes, KVM_SEV_DBG_ENCRYPT, err); 1372 out: 1373 sev_dbg_crypt_slow_free(buf); 1374 return r; 1375 } 1376 1377 static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, 1378 unsigned int cmd) 1379 { 1380 struct kvm_sev_dbg debug; 1381 unsigned int i, len; 1382 1383 if (!sev_guest(kvm)) 1384 return -ENOTTY; 1385 1386 if (copy_from_user(&debug, u64_to_user_ptr(argp->data), sizeof(debug))) 1387 return -EFAULT; 1388 1389 if (!debug.len || !debug.src_uaddr || !debug.dst_uaddr) 1390 return -EINVAL; 1391 1392 if (debug.src_uaddr + debug.len < debug.src_uaddr || 1393 debug.dst_uaddr + debug.len < debug.dst_uaddr) 1394 return -EINVAL; 1395 1396 for (i = 0; i < debug.len; i += len) { 1397 unsigned long src = debug.src_uaddr + i; 1398 unsigned long dst = debug.dst_uaddr + i; 1399 unsigned long s_off = src & ~PAGE_MASK; 1400 unsigned long d_off = dst & ~PAGE_MASK; 1401 struct page *src_p, *dst_p; 1402 int ret; 1403 1404 /* 1405 * Copy as many remaining bytes as possible while staying in a 1406 * single page for both the source and destination. 1407 */ 1408 len = min3(debug.len - i, PAGE_SIZE - s_off, PAGE_SIZE - d_off); 1409 1410 /* 1411 * Pin the source and destination pages; firmware operates on 1412 * physical addresses. 1413 */ 1414 src_p = sev_pin_page(kvm, src & PAGE_MASK, 0); 1415 if (IS_ERR(src_p)) 1416 return PTR_ERR(src_p); 1417 1418 dst_p = sev_pin_page(kvm, dst & PAGE_MASK, FOLL_WRITE); 1419 if (IS_ERR(dst_p)) { 1420 sev_unpin_page(kvm, src_p); 1421 return PTR_ERR(dst_p); 1422 } 1423 1424 /* 1425 * Flush (on non-coherent CPUs) before DBG_{DE,EN}CRYPT read or modify 1426 * the pages; flush the destination too so that future accesses do not 1427 * see stale data. 1428 */ 1429 sev_clflush_pages(&src_p, 1); 1430 sev_clflush_pages(&dst_p, 1); 1431 1432 if (IS_ALIGNED(src, 16) && IS_ALIGNED(dst, 16) && IS_ALIGNED(len, 16)) 1433 ret = sev_issue_dbg_cmd(kvm, 1434 __sme_page_pa(src_p) + s_off, 1435 __sme_page_pa(dst_p) + d_off, 1436 len, cmd, &argp->error); 1437 else if (cmd == KVM_SEV_DBG_DECRYPT) 1438 ret = sev_dbg_decrypt_slow(kvm, src, src_p, dst, 1439 len, &argp->error); 1440 else 1441 ret = sev_dbg_encrypt_slow(kvm, src, dst, dst_p, 1442 len, &argp->error); 1443 1444 sev_unpin_page(kvm, src_p); 1445 sev_unpin_page(kvm, dst_p); 1446 1447 if (ret) 1448 return ret; 1449 } 1450 return 0; 1451 } 1452 1453 static int sev_launch_secret(struct kvm *kvm, struct kvm_sev_cmd *argp) 1454 { 1455 struct sev_data_launch_secret data; 1456 struct kvm_sev_launch_secret params; 1457 struct page **pages; 1458 void *blob, *hdr; 1459 unsigned long n, i; 1460 int ret, offset; 1461 1462 if (!sev_guest(kvm)) 1463 return -ENOTTY; 1464 1465 if (copy_from_user(¶ms, u64_to_user_ptr(argp->data), sizeof(params))) 1466 return -EFAULT; 1467 1468 pages = sev_pin_memory(kvm, params.guest_uaddr, params.guest_len, &n, FOLL_WRITE); 1469 if (IS_ERR(pages)) 1470 return PTR_ERR(pages); 1471 1472 /* 1473 * Flush (on non-coherent CPUs) before LAUNCH_SECRET encrypts pages in 1474 * place; the cache may contain the data that was written unencrypted. 1475 */ 1476 sev_clflush_pages(pages, n); 1477 1478 /* 1479 * The secret must be copied into contiguous memory region, lets verify 1480 * that userspace memory pages are contiguous before we issue command. 1481 */ 1482 if (get_num_contig_pages(0, pages, n) != n) { 1483 ret = -EINVAL; 1484 goto e_unpin_memory; 1485 } 1486 1487 memset(&data, 0, sizeof(data)); 1488 1489 offset = params.guest_uaddr & (PAGE_SIZE - 1); 1490 data.guest_address = __sme_page_pa(pages[0]) + offset; 1491 data.guest_len = params.guest_len; 1492 1493 blob = psp_copy_user_blob(params.trans_uaddr, params.trans_len); 1494 if (IS_ERR(blob)) { 1495 ret = PTR_ERR(blob); 1496 goto e_unpin_memory; 1497 } 1498 1499 data.trans_address = __psp_pa(blob); 1500 data.trans_len = params.trans_len; 1501 1502 hdr = psp_copy_user_blob(params.hdr_uaddr, params.hdr_len); 1503 if (IS_ERR(hdr)) { 1504 ret = PTR_ERR(hdr); 1505 goto e_free_blob; 1506 } 1507 data.hdr_address = __psp_pa(hdr); 1508 data.hdr_len = params.hdr_len; 1509 1510 data.handle = to_kvm_sev_info(kvm)->handle; 1511 ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_SECRET, &data, &argp->error); 1512 1513 kfree(hdr); 1514 1515 e_free_blob: 1516 kfree(blob); 1517 e_unpin_memory: 1518 /* content of memory is updated, mark pages dirty */ 1519 for (i = 0; i < n; i++) { 1520 set_page_dirty_lock(pages[i]); 1521 mark_page_accessed(pages[i]); 1522 } 1523 sev_unpin_memory(kvm, pages, n); 1524 return ret; 1525 } 1526 1527 static int sev_get_attestation_report(struct kvm *kvm, struct kvm_sev_cmd *argp) 1528 { 1529 void __user *report = u64_to_user_ptr(argp->data); 1530 struct sev_data_attestation_report data; 1531 struct kvm_sev_attestation_report params; 1532 void __user *p; 1533 void *blob = NULL; 1534 int ret; 1535 1536 if (!sev_guest(kvm)) 1537 return -ENOTTY; 1538 1539 if (copy_from_user(¶ms, u64_to_user_ptr(argp->data), sizeof(params))) 1540 return -EFAULT; 1541 1542 memset(&data, 0, sizeof(data)); 1543 1544 /* User wants to query the blob length */ 1545 if (!params.len) 1546 goto cmd; 1547 1548 p = u64_to_user_ptr(params.uaddr); 1549 if (p) { 1550 if (params.len > SEV_FW_BLOB_MAX_SIZE) 1551 return -EINVAL; 1552 1553 blob = kzalloc(params.len, GFP_KERNEL_ACCOUNT); 1554 if (!blob) 1555 return -ENOMEM; 1556 1557 data.address = __psp_pa(blob); 1558 data.len = params.len; 1559 memcpy(data.mnonce, params.mnonce, sizeof(params.mnonce)); 1560 } 1561 cmd: 1562 data.handle = to_kvm_sev_info(kvm)->handle; 1563 ret = sev_issue_cmd(kvm, SEV_CMD_ATTESTATION_REPORT, &data, &argp->error); 1564 /* 1565 * If we query the session length, FW responded with expected data. 1566 */ 1567 if (!params.len) 1568 goto done; 1569 1570 if (ret) 1571 goto e_free_blob; 1572 1573 if (blob) { 1574 if (copy_to_user(p, blob, params.len)) 1575 ret = -EFAULT; 1576 } 1577 1578 done: 1579 params.len = data.len; 1580 if (copy_to_user(report, ¶ms, sizeof(params))) 1581 ret = -EFAULT; 1582 e_free_blob: 1583 kfree(blob); 1584 return ret; 1585 } 1586 1587 /* Userspace wants to query session length. */ 1588 static int 1589 __sev_send_start_query_session_length(struct kvm *kvm, struct kvm_sev_cmd *argp, 1590 struct kvm_sev_send_start *params) 1591 { 1592 struct sev_data_send_start data; 1593 int ret; 1594 1595 memset(&data, 0, sizeof(data)); 1596 data.handle = to_kvm_sev_info(kvm)->handle; 1597 ret = sev_issue_cmd(kvm, SEV_CMD_SEND_START, &data, &argp->error); 1598 1599 params->session_len = data.session_len; 1600 if (copy_to_user(u64_to_user_ptr(argp->data), params, 1601 sizeof(struct kvm_sev_send_start))) 1602 ret = -EFAULT; 1603 1604 return ret; 1605 } 1606 1607 static int sev_send_start(struct kvm *kvm, struct kvm_sev_cmd *argp) 1608 { 1609 struct sev_data_send_start data; 1610 struct kvm_sev_send_start params; 1611 void *amd_certs, *session_data; 1612 void *pdh_cert, *plat_certs; 1613 int ret; 1614 1615 if (!sev_guest(kvm)) 1616 return -ENOTTY; 1617 1618 if (copy_from_user(¶ms, u64_to_user_ptr(argp->data), 1619 sizeof(struct kvm_sev_send_start))) 1620 return -EFAULT; 1621 1622 /* if session_len is zero, userspace wants to query the session length */ 1623 if (!params.session_len) 1624 return __sev_send_start_query_session_length(kvm, argp, 1625 ¶ms); 1626 1627 /* some sanity checks */ 1628 if (!params.pdh_cert_uaddr || !params.pdh_cert_len || 1629 !params.session_uaddr || params.session_len > SEV_FW_BLOB_MAX_SIZE) 1630 return -EINVAL; 1631 1632 /* allocate the memory to hold the session data blob */ 1633 session_data = kzalloc(params.session_len, GFP_KERNEL_ACCOUNT); 1634 if (!session_data) 1635 return -ENOMEM; 1636 1637 /* copy the certificate blobs from userspace */ 1638 pdh_cert = psp_copy_user_blob(params.pdh_cert_uaddr, 1639 params.pdh_cert_len); 1640 if (IS_ERR(pdh_cert)) { 1641 ret = PTR_ERR(pdh_cert); 1642 goto e_free_session; 1643 } 1644 1645 plat_certs = psp_copy_user_blob(params.plat_certs_uaddr, 1646 params.plat_certs_len); 1647 if (IS_ERR(plat_certs)) { 1648 ret = PTR_ERR(plat_certs); 1649 goto e_free_pdh; 1650 } 1651 1652 amd_certs = psp_copy_user_blob(params.amd_certs_uaddr, 1653 params.amd_certs_len); 1654 if (IS_ERR(amd_certs)) { 1655 ret = PTR_ERR(amd_certs); 1656 goto e_free_plat_cert; 1657 } 1658 1659 /* populate the FW SEND_START field with system physical address */ 1660 memset(&data, 0, sizeof(data)); 1661 data.pdh_cert_address = __psp_pa(pdh_cert); 1662 data.pdh_cert_len = params.pdh_cert_len; 1663 data.plat_certs_address = __psp_pa(plat_certs); 1664 data.plat_certs_len = params.plat_certs_len; 1665 data.amd_certs_address = __psp_pa(amd_certs); 1666 data.amd_certs_len = params.amd_certs_len; 1667 data.session_address = __psp_pa(session_data); 1668 data.session_len = params.session_len; 1669 data.handle = to_kvm_sev_info(kvm)->handle; 1670 1671 ret = sev_issue_cmd(kvm, SEV_CMD_SEND_START, &data, &argp->error); 1672 1673 if (!ret && copy_to_user(u64_to_user_ptr(params.session_uaddr), 1674 session_data, params.session_len)) { 1675 ret = -EFAULT; 1676 goto e_free_amd_cert; 1677 } 1678 1679 params.policy = data.policy; 1680 params.session_len = data.session_len; 1681 if (copy_to_user(u64_to_user_ptr(argp->data), ¶ms, 1682 sizeof(struct kvm_sev_send_start))) 1683 ret = -EFAULT; 1684 1685 e_free_amd_cert: 1686 kfree(amd_certs); 1687 e_free_plat_cert: 1688 kfree(plat_certs); 1689 e_free_pdh: 1690 kfree(pdh_cert); 1691 e_free_session: 1692 kfree(session_data); 1693 return ret; 1694 } 1695 1696 /* Userspace wants to query either header or trans length. */ 1697 static int 1698 __sev_send_update_data_query_lengths(struct kvm *kvm, struct kvm_sev_cmd *argp, 1699 struct kvm_sev_send_update_data *params) 1700 { 1701 struct sev_data_send_update_data data; 1702 int ret; 1703 1704 memset(&data, 0, sizeof(data)); 1705 data.handle = to_kvm_sev_info(kvm)->handle; 1706 ret = sev_issue_cmd(kvm, SEV_CMD_SEND_UPDATE_DATA, &data, &argp->error); 1707 1708 params->hdr_len = data.hdr_len; 1709 params->trans_len = data.trans_len; 1710 1711 if (copy_to_user(u64_to_user_ptr(argp->data), params, 1712 sizeof(struct kvm_sev_send_update_data))) 1713 ret = -EFAULT; 1714 1715 return ret; 1716 } 1717 1718 static int sev_send_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp) 1719 { 1720 struct sev_data_send_update_data data; 1721 struct kvm_sev_send_update_data params; 1722 void *hdr, *trans_data; 1723 struct page *guest_page; 1724 int ret, offset; 1725 1726 if (!sev_guest(kvm)) 1727 return -ENOTTY; 1728 1729 if (copy_from_user(¶ms, u64_to_user_ptr(argp->data), 1730 sizeof(struct kvm_sev_send_update_data))) 1731 return -EFAULT; 1732 1733 /* userspace wants to query either header or trans length */ 1734 if (!params.trans_len || !params.hdr_len) 1735 return __sev_send_update_data_query_lengths(kvm, argp, ¶ms); 1736 1737 if (!params.trans_uaddr || !params.guest_uaddr || 1738 !params.guest_len || !params.hdr_uaddr) 1739 return -EINVAL; 1740 1741 /* Check if we are crossing the page boundary */ 1742 offset = params.guest_uaddr & (PAGE_SIZE - 1); 1743 if (params.guest_len > PAGE_SIZE || (params.guest_len + offset) > PAGE_SIZE) 1744 return -EINVAL; 1745 1746 /* Pin guest memory */ 1747 guest_page = sev_pin_page(kvm, params.guest_uaddr & PAGE_MASK, 0); 1748 if (IS_ERR(guest_page)) 1749 return PTR_ERR(guest_page); 1750 1751 /* allocate memory for header and transport buffer */ 1752 ret = -ENOMEM; 1753 hdr = kzalloc(params.hdr_len, GFP_KERNEL); 1754 if (!hdr) 1755 goto e_unpin; 1756 1757 trans_data = kzalloc(params.trans_len, GFP_KERNEL); 1758 if (!trans_data) 1759 goto e_free_hdr; 1760 1761 memset(&data, 0, sizeof(data)); 1762 data.hdr_address = __psp_pa(hdr); 1763 data.hdr_len = params.hdr_len; 1764 data.trans_address = __psp_pa(trans_data); 1765 data.trans_len = params.trans_len; 1766 1767 /* The SEND_UPDATE_DATA command requires C-bit to be always set. */ 1768 data.guest_address = page_to_phys(guest_page) + offset; 1769 data.guest_address |= sev_me_mask; 1770 data.guest_len = params.guest_len; 1771 data.handle = to_kvm_sev_info(kvm)->handle; 1772 1773 ret = sev_issue_cmd(kvm, SEV_CMD_SEND_UPDATE_DATA, &data, &argp->error); 1774 1775 if (ret) 1776 goto e_free_trans_data; 1777 1778 /* copy transport buffer to user space */ 1779 if (copy_to_user(u64_to_user_ptr(params.trans_uaddr), 1780 trans_data, params.trans_len)) { 1781 ret = -EFAULT; 1782 goto e_free_trans_data; 1783 } 1784 1785 /* Copy packet header to userspace. */ 1786 if (copy_to_user(u64_to_user_ptr(params.hdr_uaddr), hdr, 1787 params.hdr_len)) 1788 ret = -EFAULT; 1789 1790 e_free_trans_data: 1791 kfree(trans_data); 1792 e_free_hdr: 1793 kfree(hdr); 1794 e_unpin: 1795 sev_unpin_page(kvm, guest_page); 1796 return ret; 1797 } 1798 1799 static int sev_send_finish(struct kvm *kvm, struct kvm_sev_cmd *argp) 1800 { 1801 struct sev_data_send_finish data; 1802 1803 if (!sev_guest(kvm)) 1804 return -ENOTTY; 1805 1806 data.handle = to_kvm_sev_info(kvm)->handle; 1807 return sev_issue_cmd(kvm, SEV_CMD_SEND_FINISH, &data, &argp->error); 1808 } 1809 1810 static int sev_send_cancel(struct kvm *kvm, struct kvm_sev_cmd *argp) 1811 { 1812 struct sev_data_send_cancel data; 1813 1814 if (!sev_guest(kvm)) 1815 return -ENOTTY; 1816 1817 data.handle = to_kvm_sev_info(kvm)->handle; 1818 return sev_issue_cmd(kvm, SEV_CMD_SEND_CANCEL, &data, &argp->error); 1819 } 1820 1821 static int sev_receive_start(struct kvm *kvm, struct kvm_sev_cmd *argp) 1822 { 1823 struct kvm_sev_info *sev = to_kvm_sev_info(kvm); 1824 struct sev_data_receive_start start; 1825 struct kvm_sev_receive_start params; 1826 int *error = &argp->error; 1827 void *session_data; 1828 void *pdh_data; 1829 int ret; 1830 1831 if (!sev_guest(kvm)) 1832 return -ENOTTY; 1833 1834 /* Get parameter from the userspace */ 1835 if (copy_from_user(¶ms, u64_to_user_ptr(argp->data), 1836 sizeof(struct kvm_sev_receive_start))) 1837 return -EFAULT; 1838 1839 /* some sanity checks */ 1840 if (!params.pdh_uaddr || !params.pdh_len || 1841 !params.session_uaddr || !params.session_len) 1842 return -EINVAL; 1843 1844 pdh_data = psp_copy_user_blob(params.pdh_uaddr, params.pdh_len); 1845 if (IS_ERR(pdh_data)) 1846 return PTR_ERR(pdh_data); 1847 1848 session_data = psp_copy_user_blob(params.session_uaddr, 1849 params.session_len); 1850 if (IS_ERR(session_data)) { 1851 ret = PTR_ERR(session_data); 1852 goto e_free_pdh; 1853 } 1854 1855 memset(&start, 0, sizeof(start)); 1856 start.handle = params.handle; 1857 start.policy = params.policy; 1858 start.pdh_cert_address = __psp_pa(pdh_data); 1859 start.pdh_cert_len = params.pdh_len; 1860 start.session_address = __psp_pa(session_data); 1861 start.session_len = params.session_len; 1862 1863 /* create memory encryption context */ 1864 ret = __sev_issue_cmd(argp->sev_fd, SEV_CMD_RECEIVE_START, &start, 1865 error); 1866 if (ret) 1867 goto e_free_session; 1868 1869 /* Bind ASID to this guest */ 1870 ret = sev_bind_asid(kvm, start.handle, error); 1871 if (ret) { 1872 sev_decommission(start.handle); 1873 goto e_free_session; 1874 } 1875 1876 params.handle = start.handle; 1877 if (copy_to_user(u64_to_user_ptr(argp->data), 1878 ¶ms, sizeof(struct kvm_sev_receive_start))) { 1879 ret = -EFAULT; 1880 sev_unbind_asid(kvm, start.handle); 1881 goto e_free_session; 1882 } 1883 1884 sev->handle = start.handle; 1885 sev->fd = argp->sev_fd; 1886 1887 e_free_session: 1888 kfree(session_data); 1889 e_free_pdh: 1890 kfree(pdh_data); 1891 1892 return ret; 1893 } 1894 1895 static int sev_receive_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp) 1896 { 1897 struct kvm_sev_receive_update_data params; 1898 struct sev_data_receive_update_data data; 1899 void *hdr = NULL, *trans = NULL; 1900 struct page *guest_page; 1901 int ret, offset; 1902 1903 if (!sev_guest(kvm)) 1904 return -EINVAL; 1905 1906 if (copy_from_user(¶ms, u64_to_user_ptr(argp->data), 1907 sizeof(struct kvm_sev_receive_update_data))) 1908 return -EFAULT; 1909 1910 if (!params.hdr_uaddr || !params.hdr_len || 1911 !params.guest_uaddr || !params.guest_len || 1912 !params.trans_uaddr || !params.trans_len) 1913 return -EINVAL; 1914 1915 /* Check if we are crossing the page boundary */ 1916 offset = params.guest_uaddr & (PAGE_SIZE - 1); 1917 if (params.guest_len > PAGE_SIZE || (params.guest_len + offset) > PAGE_SIZE) 1918 return -EINVAL; 1919 1920 hdr = psp_copy_user_blob(params.hdr_uaddr, params.hdr_len); 1921 if (IS_ERR(hdr)) 1922 return PTR_ERR(hdr); 1923 1924 trans = psp_copy_user_blob(params.trans_uaddr, params.trans_len); 1925 if (IS_ERR(trans)) { 1926 ret = PTR_ERR(trans); 1927 goto e_free_hdr; 1928 } 1929 1930 memset(&data, 0, sizeof(data)); 1931 data.hdr_address = __psp_pa(hdr); 1932 data.hdr_len = params.hdr_len; 1933 data.trans_address = __psp_pa(trans); 1934 data.trans_len = params.trans_len; 1935 1936 /* Pin guest memory */ 1937 guest_page = sev_pin_page(kvm, params.guest_uaddr & PAGE_MASK, FOLL_WRITE); 1938 if (IS_ERR(guest_page)) { 1939 ret = PTR_ERR(guest_page); 1940 goto e_free_trans; 1941 } 1942 1943 /* 1944 * Flush (on non-coherent CPUs) before RECEIVE_UPDATE_DATA, the PSP 1945 * encrypts the written data with the guest's key, and the cache may 1946 * contain dirty, unencrypted data. 1947 */ 1948 sev_clflush_pages(&guest_page, 1); 1949 1950 /* The RECEIVE_UPDATE_DATA command requires C-bit to be always set. */ 1951 data.guest_address = page_to_phys(guest_page) + offset; 1952 data.guest_address |= sev_me_mask; 1953 data.guest_len = params.guest_len; 1954 data.handle = to_kvm_sev_info(kvm)->handle; 1955 1956 ret = sev_issue_cmd(kvm, SEV_CMD_RECEIVE_UPDATE_DATA, &data, 1957 &argp->error); 1958 1959 sev_unpin_page(kvm, guest_page); 1960 1961 e_free_trans: 1962 kfree(trans); 1963 e_free_hdr: 1964 kfree(hdr); 1965 1966 return ret; 1967 } 1968 1969 static int sev_receive_finish(struct kvm *kvm, struct kvm_sev_cmd *argp) 1970 { 1971 struct sev_data_receive_finish data; 1972 1973 if (!sev_guest(kvm)) 1974 return -ENOTTY; 1975 1976 data.handle = to_kvm_sev_info(kvm)->handle; 1977 return sev_issue_cmd(kvm, SEV_CMD_RECEIVE_FINISH, &data, &argp->error); 1978 } 1979 1980 static bool is_cmd_allowed_from_mirror(u32 cmd_id) 1981 { 1982 /* 1983 * Allow mirrors VM to call KVM_SEV_LAUNCH_UPDATE_VMSA to enable SEV-ES 1984 * active mirror VMs. Also allow the debugging and status commands. 1985 */ 1986 if (cmd_id == KVM_SEV_LAUNCH_UPDATE_VMSA || 1987 cmd_id == KVM_SEV_GUEST_STATUS || cmd_id == KVM_SEV_DBG_DECRYPT || 1988 cmd_id == KVM_SEV_DBG_ENCRYPT) 1989 return true; 1990 1991 return false; 1992 } 1993 1994 static int sev_lock_two_vms(struct kvm *dst_kvm, struct kvm *src_kvm) 1995 { 1996 struct kvm_sev_info *dst_sev = to_kvm_sev_info(dst_kvm); 1997 struct kvm_sev_info *src_sev = to_kvm_sev_info(src_kvm); 1998 int r = -EBUSY; 1999 2000 if (dst_kvm == src_kvm) 2001 return -EINVAL; 2002 2003 /* 2004 * Bail if these VMs are already involved in a migration to avoid 2005 * deadlock between two VMs trying to migrate to/from each other. 2006 */ 2007 if (atomic_cmpxchg_acquire(&dst_sev->migration_in_progress, 0, 1)) 2008 return -EBUSY; 2009 2010 if (atomic_cmpxchg_acquire(&src_sev->migration_in_progress, 0, 1)) 2011 goto release_dst; 2012 2013 r = -EINTR; 2014 if (mutex_lock_killable(&dst_kvm->lock)) 2015 goto release_src; 2016 if (mutex_lock_killable_nested(&src_kvm->lock, SINGLE_DEPTH_NESTING)) 2017 goto unlock_dst; 2018 return 0; 2019 2020 unlock_dst: 2021 mutex_unlock(&dst_kvm->lock); 2022 release_src: 2023 atomic_set_release(&src_sev->migration_in_progress, 0); 2024 release_dst: 2025 atomic_set_release(&dst_sev->migration_in_progress, 0); 2026 return r; 2027 } 2028 2029 static void sev_unlock_two_vms(struct kvm *dst_kvm, struct kvm *src_kvm) 2030 { 2031 struct kvm_sev_info *dst_sev = to_kvm_sev_info(dst_kvm); 2032 struct kvm_sev_info *src_sev = to_kvm_sev_info(src_kvm); 2033 2034 mutex_unlock(&dst_kvm->lock); 2035 mutex_unlock(&src_kvm->lock); 2036 atomic_set_release(&dst_sev->migration_in_progress, 0); 2037 atomic_set_release(&src_sev->migration_in_progress, 0); 2038 } 2039 2040 static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm) 2041 { 2042 struct kvm_sev_info *dst = to_kvm_sev_info(dst_kvm); 2043 struct kvm_sev_info *src = to_kvm_sev_info(src_kvm); 2044 struct kvm_vcpu *dst_vcpu, *src_vcpu; 2045 struct vcpu_svm *dst_svm, *src_svm; 2046 struct kvm_sev_info *mirror; 2047 unsigned long i; 2048 2049 /* 2050 * Do cache maintenance on the source VM *before* clearing "SEV active", 2051 * as memory reclaim flows won't trigger cache maintenance on the VM 2052 * once it's no longer an SEV VM. 2053 */ 2054 sev_writeback_caches(src_kvm); 2055 2056 dst->active = true; 2057 dst->asid = src->asid; 2058 dst->handle = src->handle; 2059 dst->pages_locked = src->pages_locked; 2060 dst->es_active = src->es_active; 2061 dst->vmsa_features = src->vmsa_features; 2062 dst_kvm->arch.has_protected_page_tables = true; 2063 2064 src->asid = 0; 2065 src->active = false; 2066 src->handle = 0; 2067 src->pages_locked = 0; 2068 src->es_active = false; 2069 src_kvm->arch.has_protected_page_tables = false; 2070 2071 list_cut_before(&dst->regions_list, &src->regions_list, &src->regions_list); 2072 2073 mutex_lock(&sev_mirror_lock); 2074 2075 /* 2076 * If this VM has mirrors, "transfer" each mirror's refcount of the 2077 * source to the destination (this KVM). The caller holds a reference 2078 * to the source, so there's no danger of use-after-free. 2079 */ 2080 list_cut_before(&dst->mirror_vms, &src->mirror_vms, &src->mirror_vms); 2081 list_for_each_entry(mirror, &dst->mirror_vms, mirror_entry) { 2082 kvm_get_kvm(dst_kvm); 2083 kvm_put_kvm(src_kvm); 2084 mirror->enc_context_owner = dst_kvm; 2085 } 2086 2087 /* 2088 * If this VM is a mirror, remove the old mirror from the owners list 2089 * and add the new mirror to the list. 2090 */ 2091 if (is_mirroring_enc_context(src_kvm)) { 2092 struct kvm_sev_info *owner_sev_info = to_kvm_sev_info(src->enc_context_owner); 2093 2094 dst->enc_context_owner = src->enc_context_owner; 2095 src->enc_context_owner = NULL; 2096 list_del(&src->mirror_entry); 2097 list_add_tail(&dst->mirror_entry, &owner_sev_info->mirror_vms); 2098 } 2099 mutex_unlock(&sev_mirror_lock); 2100 2101 kvm_for_each_vcpu(i, dst_vcpu, dst_kvm) { 2102 dst_svm = to_svm(dst_vcpu); 2103 2104 sev_init_vmcb(dst_svm, false); 2105 2106 if (!dst->es_active) 2107 continue; 2108 2109 /* 2110 * Note, the source is not required to have the same number of 2111 * vCPUs as the destination when migrating a vanilla SEV VM. 2112 */ 2113 src_vcpu = kvm_get_vcpu(src_kvm, i); 2114 src_svm = to_svm(src_vcpu); 2115 2116 /* 2117 * Transfer VMSA and GHCB state to the destination. Nullify and 2118 * clear source fields as appropriate, the state now belongs to 2119 * the destination. 2120 */ 2121 memcpy(&dst_svm->sev_es, &src_svm->sev_es, sizeof(src_svm->sev_es)); 2122 dst_svm->vmcb->control.ghcb_gpa = src_svm->vmcb->control.ghcb_gpa; 2123 dst_svm->vmcb->control.vmsa_pa = src_svm->vmcb->control.vmsa_pa; 2124 dst_vcpu->arch.guest_state_protected = true; 2125 2126 memset(&src_svm->sev_es, 0, sizeof(src_svm->sev_es)); 2127 src_svm->vmcb->control.ghcb_gpa = INVALID_PAGE; 2128 src_svm->vmcb->control.vmsa_pa = INVALID_PAGE; 2129 src_vcpu->arch.guest_state_protected = false; 2130 } 2131 } 2132 2133 static int sev_check_source_vcpus(struct kvm *dst, struct kvm *src) 2134 { 2135 struct kvm_vcpu *src_vcpu; 2136 unsigned long i; 2137 2138 if (!sev_es_guest(src)) 2139 return 0; 2140 2141 if (atomic_read(&src->online_vcpus) != atomic_read(&dst->online_vcpus)) 2142 return -EINVAL; 2143 2144 kvm_for_each_vcpu(i, src_vcpu, src) { 2145 if (!src_vcpu->arch.guest_state_protected) 2146 return -EINVAL; 2147 } 2148 2149 return 0; 2150 } 2151 2152 int sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd) 2153 { 2154 struct kvm_sev_info *dst_sev = to_kvm_sev_info(kvm); 2155 struct kvm_sev_info *src_sev, *cg_cleanup_sev; 2156 CLASS(fd, f)(source_fd); 2157 struct kvm *source_kvm; 2158 bool charged = false; 2159 int ret; 2160 2161 if (fd_empty(f)) 2162 return -EBADF; 2163 2164 source_kvm = file_to_kvm_x86(fd_file(f)); 2165 if (!source_kvm) 2166 return -EBADF; 2167 2168 ret = sev_lock_two_vms(kvm, source_kvm); 2169 if (ret) 2170 return ret; 2171 2172 /* Do not allow SNP VM migration until additional state transfer is implemented */ 2173 if (kvm->arch.vm_type != source_kvm->arch.vm_type || 2174 sev_guest(kvm) || !sev_guest(source_kvm) || sev_snp_guest(source_kvm)) { 2175 ret = -EINVAL; 2176 goto out_unlock; 2177 } 2178 2179 src_sev = to_kvm_sev_info(source_kvm); 2180 2181 dst_sev->misc_cg = get_current_misc_cg(); 2182 cg_cleanup_sev = dst_sev; 2183 if (dst_sev->misc_cg != src_sev->misc_cg) { 2184 ret = sev_misc_cg_try_charge(dst_sev); 2185 if (ret) 2186 goto out_dst_cgroup; 2187 charged = true; 2188 } 2189 2190 ret = kvm_lock_all_vcpus(kvm); 2191 if (ret) 2192 goto out_dst_cgroup; 2193 ret = kvm_lock_all_vcpus(source_kvm); 2194 if (ret) 2195 goto out_dst_vcpu; 2196 2197 ret = sev_check_source_vcpus(kvm, source_kvm); 2198 if (ret) 2199 goto out_source_vcpu; 2200 2201 /* 2202 * Allocate a new have_run_cpus for the destination, i.e. don't copy 2203 * the set of CPUs from the source. If a CPU was used to run a vCPU in 2204 * the source VM but is never used for the destination VM, then the CPU 2205 * can only have cached memory that was accessible to the source VM. 2206 * Furthermore, KVM *must* perform cache maintenance on the source VM, 2207 * as the source VM may have access to memory that the destination VM 2208 * does not, i.e. KVM could skip flushes if memory is reclaimed from 2209 * the old VM but not the new VM. 2210 */ 2211 ret = sev_alloc_have_run_cpus(dst_sev); 2212 if (ret) 2213 goto out_source_vcpu; 2214 2215 sev_migrate_from(kvm, source_kvm); 2216 kvm_vm_dead(source_kvm); 2217 cg_cleanup_sev = src_sev; 2218 ret = 0; 2219 2220 out_source_vcpu: 2221 kvm_unlock_all_vcpus(source_kvm); 2222 out_dst_vcpu: 2223 kvm_unlock_all_vcpus(kvm); 2224 out_dst_cgroup: 2225 /* Operates on the source on success, on the destination on failure. */ 2226 if (charged) 2227 sev_misc_cg_uncharge(cg_cleanup_sev); 2228 put_misc_cg(cg_cleanup_sev->misc_cg); 2229 cg_cleanup_sev->misc_cg = NULL; 2230 out_unlock: 2231 sev_unlock_two_vms(kvm, source_kvm); 2232 return ret; 2233 } 2234 2235 int sev_dev_get_attr(u32 group, u64 attr, u64 *val) 2236 { 2237 if (group != KVM_X86_GRP_SEV) 2238 return -ENXIO; 2239 2240 switch (attr) { 2241 case KVM_X86_SEV_VMSA_FEATURES: 2242 *val = sev_supported_vmsa_features; 2243 return 0; 2244 2245 case KVM_X86_SNP_POLICY_BITS: 2246 *val = snp_supported_policy_bits; 2247 return 0; 2248 2249 case KVM_X86_SEV_SNP_REQ_CERTS: 2250 *val = sev_snp_enabled ? 1 : 0; 2251 return 0; 2252 default: 2253 return -ENXIO; 2254 } 2255 } 2256 2257 /* 2258 * The guest context contains all the information, keys and metadata 2259 * associated with the guest that the firmware tracks to implement SEV 2260 * and SNP features. The firmware stores the guest context in hypervisor 2261 * provide page via the SNP_GCTX_CREATE command. 2262 */ 2263 static void *snp_context_create(struct kvm *kvm, struct kvm_sev_cmd *argp) 2264 { 2265 struct sev_data_snp_addr data = {}; 2266 void *context; 2267 int rc; 2268 2269 /* Allocate memory for context page */ 2270 context = snp_alloc_firmware_page(GFP_KERNEL_ACCOUNT); 2271 if (!context) 2272 return NULL; 2273 2274 data.address = __psp_pa(context); 2275 rc = __sev_issue_cmd(argp->sev_fd, SEV_CMD_SNP_GCTX_CREATE, &data, &argp->error); 2276 if (rc) { 2277 pr_warn("Failed to create SEV-SNP context, rc %d fw_error %d", 2278 rc, argp->error); 2279 snp_free_firmware_page(context); 2280 return NULL; 2281 } 2282 2283 return context; 2284 } 2285 2286 static int snp_bind_asid(struct kvm *kvm, int *error) 2287 { 2288 struct kvm_sev_info *sev = to_kvm_sev_info(kvm); 2289 struct sev_data_snp_activate data = {0}; 2290 2291 data.gctx_paddr = __psp_pa(sev->snp_context); 2292 data.asid = sev_get_asid(kvm); 2293 return sev_issue_cmd(kvm, SEV_CMD_SNP_ACTIVATE, &data, error); 2294 } 2295 2296 static int snp_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp) 2297 { 2298 struct kvm_sev_info *sev = to_kvm_sev_info(kvm); 2299 struct sev_data_snp_launch_start start = {0}; 2300 struct kvm_sev_snp_launch_start params; 2301 int rc; 2302 2303 if (!sev_snp_guest(kvm)) 2304 return -ENOTTY; 2305 2306 if (copy_from_user(¶ms, u64_to_user_ptr(argp->data), sizeof(params))) 2307 return -EFAULT; 2308 2309 /* Don't allow userspace to allocate memory for more than 1 SNP context. */ 2310 if (sev->snp_context) 2311 return -EINVAL; 2312 2313 if (params.flags) 2314 return -EINVAL; 2315 2316 if (params.policy & ~snp_supported_policy_bits) 2317 return -EINVAL; 2318 2319 /* Check for policy bits that must be set */ 2320 if (!(params.policy & SNP_POLICY_MASK_RSVD_MBO)) 2321 return -EINVAL; 2322 2323 if (snp_is_secure_tsc_enabled(kvm)) { 2324 if (WARN_ON_ONCE(!kvm->arch.default_tsc_khz)) 2325 return -EINVAL; 2326 2327 start.desired_tsc_khz = kvm->arch.default_tsc_khz; 2328 } 2329 2330 sev->snp_context = snp_context_create(kvm, argp); 2331 if (!sev->snp_context) 2332 return -ENOTTY; 2333 2334 start.gctx_paddr = __psp_pa(sev->snp_context); 2335 start.policy = params.policy; 2336 2337 memcpy(start.gosvw, params.gosvw, sizeof(params.gosvw)); 2338 rc = __sev_issue_cmd(argp->sev_fd, SEV_CMD_SNP_LAUNCH_START, &start, &argp->error); 2339 if (rc) { 2340 pr_debug("%s: SEV_CMD_SNP_LAUNCH_START firmware command failed, rc %d\n", 2341 __func__, rc); 2342 goto e_free_context; 2343 } 2344 2345 sev->policy = params.policy; 2346 sev->fd = argp->sev_fd; 2347 rc = snp_bind_asid(kvm, &argp->error); 2348 if (rc) { 2349 pr_debug("%s: Failed to bind ASID to SEV-SNP context, rc %d\n", 2350 __func__, rc); 2351 goto e_free_context; 2352 } 2353 2354 return 0; 2355 2356 e_free_context: 2357 snp_decommission_context(kvm); 2358 2359 return rc; 2360 } 2361 2362 struct sev_gmem_populate_args { 2363 __u8 type; 2364 int sev_fd; 2365 int fw_error; 2366 }; 2367 2368 static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, 2369 struct page *src_page, void *opaque) 2370 { 2371 struct sev_gmem_populate_args *sev_populate_args = opaque; 2372 struct sev_data_snp_launch_update fw_args = {0}; 2373 struct kvm_sev_info *sev = to_kvm_sev_info(kvm); 2374 bool assigned = false; 2375 int level; 2376 int ret; 2377 2378 ret = snp_lookup_rmpentry((u64)pfn, &assigned, &level); 2379 if (ret || assigned) { 2380 pr_debug("%s: Failed to ensure GFN 0x%llx RMP entry is initial shared state, ret: %d assigned: %d\n", 2381 __func__, gfn, ret, assigned); 2382 ret = ret ? -EINVAL : -EEXIST; 2383 goto out; 2384 } 2385 2386 if (src_page) { 2387 void *src_vaddr = kmap_local_page(src_page); 2388 void *dst_vaddr = kmap_local_pfn(pfn); 2389 2390 memcpy(dst_vaddr, src_vaddr, PAGE_SIZE); 2391 2392 kunmap_local(dst_vaddr); 2393 kunmap_local(src_vaddr); 2394 } 2395 2396 ret = rmp_make_private(pfn, gfn << PAGE_SHIFT, PG_LEVEL_4K, 2397 sev_get_asid(kvm), true); 2398 if (ret) 2399 goto out; 2400 2401 fw_args.gctx_paddr = __psp_pa(sev->snp_context); 2402 fw_args.address = __sme_set(pfn_to_hpa(pfn)); 2403 fw_args.page_size = PG_LEVEL_TO_RMP(PG_LEVEL_4K); 2404 fw_args.page_type = sev_populate_args->type; 2405 2406 ret = __sev_issue_cmd(sev_populate_args->sev_fd, SEV_CMD_SNP_LAUNCH_UPDATE, 2407 &fw_args, &sev_populate_args->fw_error); 2408 /* 2409 * If the firmware command failed handle the reclaim and cleanup of that 2410 * PFN before reporting an error. 2411 * 2412 * Additionally, when invalid CPUID function entries are detected, 2413 * firmware writes the expected values into the page and leaves it 2414 * unencrypted so it can be used for debugging and error-reporting. 2415 * 2416 * Copy this page back into the source buffer so userspace can use this 2417 * information to provide information on which CPUID leaves/fields 2418 * failed CPUID validation. 2419 */ 2420 if (ret && !snp_page_reclaim(kvm, pfn) && 2421 sev_populate_args->type == KVM_SEV_SNP_PAGE_TYPE_CPUID && 2422 sev_populate_args->fw_error == SEV_RET_INVALID_PARAM && src_page) { 2423 void *src_vaddr = kmap_local_page(src_page); 2424 void *dst_vaddr = kmap_local_pfn(pfn); 2425 2426 memcpy(src_vaddr, dst_vaddr, PAGE_SIZE); 2427 set_page_dirty_lock(src_page); 2428 2429 kunmap_local(dst_vaddr); 2430 kunmap_local(src_vaddr); 2431 } 2432 2433 out: 2434 if (ret) 2435 pr_debug("%s: error updating GFN %llx, return code %d (fw_error %d)\n", 2436 __func__, gfn, ret, sev_populate_args->fw_error); 2437 return ret; 2438 } 2439 2440 static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp) 2441 { 2442 struct kvm_sev_info *sev = to_kvm_sev_info(kvm); 2443 struct sev_gmem_populate_args sev_populate_args = {0}; 2444 struct kvm_sev_snp_launch_update params; 2445 struct kvm_memory_slot *memslot; 2446 long npages, count; 2447 void __user *src; 2448 2449 if (!sev_snp_guest(kvm) || !sev->snp_context) 2450 return -EINVAL; 2451 2452 if (copy_from_user(¶ms, u64_to_user_ptr(argp->data), sizeof(params))) 2453 return -EFAULT; 2454 2455 pr_debug("%s: GFN start 0x%llx length 0x%llx type %d flags %d src %llx\n", __func__, 2456 params.gfn_start, params.len, params.type, params.flags, params.uaddr); 2457 2458 if (!params.len || !PAGE_ALIGNED(params.len) || params.flags || 2459 (params.type != KVM_SEV_SNP_PAGE_TYPE_NORMAL && 2460 params.type != KVM_SEV_SNP_PAGE_TYPE_ZERO && 2461 params.type != KVM_SEV_SNP_PAGE_TYPE_UNMEASURED && 2462 params.type != KVM_SEV_SNP_PAGE_TYPE_SECRETS && 2463 params.type != KVM_SEV_SNP_PAGE_TYPE_CPUID)) 2464 return -EINVAL; 2465 2466 if (params.type == KVM_SEV_SNP_PAGE_TYPE_ZERO) 2467 src = NULL; 2468 else if ((!gmem_in_place_conversion && !params.uaddr) || 2469 !PAGE_ALIGNED(params.uaddr)) 2470 return -EINVAL; 2471 else 2472 src = u64_to_user_ptr(params.uaddr); 2473 2474 npages = params.len / PAGE_SIZE; 2475 2476 /* 2477 * For each GFN that's being prepared as part of the initial guest 2478 * state, the following pre-conditions are verified: 2479 * 2480 * 1) The backing memslot is a valid private memslot. 2481 * 2) The GFN has been set to private via KVM_SET_MEMORY_ATTRIBUTES 2482 * beforehand. 2483 * 3) The PFN of the guest_memfd has not already been set to private 2484 * in the RMP table. 2485 * 2486 * The KVM MMU relies on kvm->mmu_invalidate_seq to retry nested page 2487 * faults if there's a race between a fault and an attribute update via 2488 * KVM_SET_MEMORY_ATTRIBUTES, and a similar approach could be utilized 2489 * here. However, kvm->slots_lock guards against both this as well as 2490 * concurrent memslot updates occurring while these checks are being 2491 * performed, so use that here to make it easier to reason about the 2492 * initial expected state and better guard against unexpected 2493 * situations. 2494 */ 2495 guard(mutex)(&kvm->slots_lock); 2496 2497 memslot = gfn_to_memslot(kvm, params.gfn_start); 2498 if (!kvm_slot_has_gmem(memslot)) 2499 return -EINVAL; 2500 2501 sev_populate_args.sev_fd = argp->sev_fd; 2502 sev_populate_args.type = params.type; 2503 2504 count = kvm_gmem_populate(kvm, params.gfn_start, src, npages, 2505 params.type == KVM_SEV_SNP_PAGE_TYPE_CPUID, 2506 sev_gmem_post_populate, &sev_populate_args); 2507 if (count < 0) { 2508 argp->error = sev_populate_args.fw_error; 2509 pr_debug("%s: kvm_gmem_populate failed, ret %ld (fw_error %d)\n", 2510 __func__, count, argp->error); 2511 return -EIO; 2512 } 2513 2514 params.gfn_start += count; 2515 params.len -= count * PAGE_SIZE; 2516 if (src && params.type != KVM_SEV_SNP_PAGE_TYPE_ZERO) 2517 params.uaddr += count * PAGE_SIZE; 2518 2519 if (copy_to_user(u64_to_user_ptr(argp->data), ¶ms, sizeof(params))) 2520 return -EFAULT; 2521 2522 return 0; 2523 } 2524 2525 static int snp_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) 2526 { 2527 struct kvm_sev_info *sev = to_kvm_sev_info(kvm); 2528 struct sev_data_snp_launch_update data = {}; 2529 struct kvm_vcpu *vcpu; 2530 unsigned long i; 2531 int ret; 2532 2533 ret = kvm_lock_all_vcpus(kvm); 2534 if (ret) 2535 return ret; 2536 2537 data.gctx_paddr = __psp_pa(sev->snp_context); 2538 data.page_type = SNP_PAGE_TYPE_VMSA; 2539 2540 kvm_for_each_vcpu(i, vcpu, kvm) { 2541 struct vcpu_svm *svm = to_svm(vcpu); 2542 u64 pfn = __pa(svm->sev_es.vmsa) >> PAGE_SHIFT; 2543 2544 ret = sev_es_sync_vmsa(svm); 2545 if (ret) 2546 goto out; 2547 2548 /* Transition the VMSA page to a firmware state. */ 2549 ret = rmp_make_private(pfn, INITIAL_VMSA_GPA, PG_LEVEL_4K, sev->asid, true); 2550 if (ret) 2551 goto out; 2552 2553 /* Issue the SNP command to encrypt the VMSA */ 2554 data.address = __sme_pa(svm->sev_es.vmsa); 2555 ret = __sev_issue_cmd(argp->sev_fd, SEV_CMD_SNP_LAUNCH_UPDATE, 2556 &data, &argp->error); 2557 if (ret) { 2558 snp_page_reclaim(kvm, pfn); 2559 2560 goto out; 2561 } 2562 2563 svm->vcpu.arch.guest_state_protected = true; 2564 /* 2565 * SEV-ES (and thus SNP) guest mandates LBR Virtualization to 2566 * be _always_ ON. Enable it only after setting 2567 * guest_state_protected because KVM_SET_MSRS allows dynamic 2568 * toggling of LBRV (for performance reason) on write access to 2569 * MSR_IA32_DEBUGCTLMSR when guest_state_protected is not set. 2570 */ 2571 svm_enable_lbrv(vcpu); 2572 } 2573 2574 out: 2575 kvm_unlock_all_vcpus(kvm); 2576 return ret; 2577 } 2578 2579 static int snp_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp) 2580 { 2581 struct kvm_sev_info *sev = to_kvm_sev_info(kvm); 2582 struct kvm_sev_snp_launch_finish params; 2583 struct sev_data_snp_launch_finish *data; 2584 void *id_block = NULL, *id_auth = NULL; 2585 int ret; 2586 2587 if (!sev_snp_guest(kvm)) 2588 return -ENOTTY; 2589 2590 if (!sev->snp_context) 2591 return -EINVAL; 2592 2593 if (copy_from_user(¶ms, u64_to_user_ptr(argp->data), sizeof(params))) 2594 return -EFAULT; 2595 2596 if (params.flags) 2597 return -EINVAL; 2598 2599 /* Measure all vCPUs using LAUNCH_UPDATE before finalizing the launch flow. */ 2600 ret = snp_launch_update_vmsa(kvm, argp); 2601 if (ret) 2602 return ret; 2603 2604 data = kzalloc_obj(*data, GFP_KERNEL_ACCOUNT); 2605 if (!data) 2606 return -ENOMEM; 2607 2608 if (params.id_block_en) { 2609 id_block = psp_copy_user_blob(params.id_block_uaddr, KVM_SEV_SNP_ID_BLOCK_SIZE); 2610 if (IS_ERR(id_block)) { 2611 ret = PTR_ERR(id_block); 2612 goto e_free; 2613 } 2614 2615 data->id_block_en = 1; 2616 data->id_block_paddr = __sme_pa(id_block); 2617 2618 id_auth = psp_copy_user_blob(params.id_auth_uaddr, KVM_SEV_SNP_ID_AUTH_SIZE); 2619 if (IS_ERR(id_auth)) { 2620 ret = PTR_ERR(id_auth); 2621 goto e_free_id_block; 2622 } 2623 2624 data->id_auth_paddr = __sme_pa(id_auth); 2625 2626 if (params.auth_key_en) 2627 data->auth_key_en = 1; 2628 } 2629 2630 data->vcek_disabled = params.vcek_disabled; 2631 2632 memcpy(data->host_data, params.host_data, KVM_SEV_SNP_FINISH_DATA_SIZE); 2633 data->gctx_paddr = __psp_pa(sev->snp_context); 2634 ret = sev_issue_cmd(kvm, SEV_CMD_SNP_LAUNCH_FINISH, data, &argp->error); 2635 2636 /* 2637 * Now that there will be no more SNP_LAUNCH_UPDATE ioctls, private pages 2638 * can be given to the guest simply by marking the RMP entry as private. 2639 * This can happen on first access and also with KVM_PRE_FAULT_MEMORY. 2640 */ 2641 if (!ret) 2642 kvm->arch.pre_fault_allowed = true; 2643 2644 kfree(id_auth); 2645 2646 e_free_id_block: 2647 kfree(id_block); 2648 2649 e_free: 2650 kfree(data); 2651 2652 return ret; 2653 } 2654 2655 static int snp_enable_certs(struct kvm *kvm) 2656 { 2657 if (kvm->created_vcpus || !sev_snp_guest(kvm)) 2658 return -EINVAL; 2659 2660 to_kvm_sev_info(kvm)->snp_certs_enabled = true; 2661 2662 return 0; 2663 } 2664 2665 int sev_mem_enc_ioctl(struct kvm *kvm, void __user *argp) 2666 { 2667 struct kvm_sev_cmd sev_cmd; 2668 int r; 2669 2670 if (!sev_enabled) 2671 return -ENOTTY; 2672 2673 if (!argp) 2674 return 0; 2675 2676 if (copy_from_user(&sev_cmd, argp, sizeof(struct kvm_sev_cmd))) 2677 return -EFAULT; 2678 2679 guard(mutex)(&kvm->lock); 2680 2681 /* Only the enc_context_owner handles some memory enc operations. */ 2682 if (is_mirroring_enc_context(kvm) && 2683 !is_cmd_allowed_from_mirror(sev_cmd.id)) 2684 return -EINVAL; 2685 2686 /* 2687 * Once KVM_SEV_INIT2 initializes a KVM instance as an SNP guest, only 2688 * allow the use of SNP-specific commands. 2689 */ 2690 if (sev_snp_guest(kvm) && sev_cmd.id < KVM_SEV_SNP_LAUNCH_START) 2691 return -EPERM; 2692 2693 switch (sev_cmd.id) { 2694 case KVM_SEV_ES_INIT: 2695 if (!sev_es_enabled) 2696 return -ENOTTY; 2697 fallthrough; 2698 case KVM_SEV_INIT: 2699 r = sev_guest_init(kvm, &sev_cmd); 2700 break; 2701 case KVM_SEV_INIT2: 2702 r = sev_guest_init2(kvm, &sev_cmd); 2703 break; 2704 case KVM_SEV_LAUNCH_START: 2705 r = sev_launch_start(kvm, &sev_cmd); 2706 break; 2707 case KVM_SEV_LAUNCH_UPDATE_DATA: 2708 r = sev_launch_update_data(kvm, &sev_cmd); 2709 break; 2710 case KVM_SEV_LAUNCH_UPDATE_VMSA: 2711 r = sev_launch_update_vmsa(kvm, &sev_cmd); 2712 break; 2713 case KVM_SEV_LAUNCH_MEASURE: 2714 r = sev_launch_measure(kvm, &sev_cmd); 2715 break; 2716 case KVM_SEV_LAUNCH_FINISH: 2717 r = sev_launch_finish(kvm, &sev_cmd); 2718 break; 2719 case KVM_SEV_GUEST_STATUS: 2720 r = sev_guest_status(kvm, &sev_cmd); 2721 break; 2722 case KVM_SEV_DBG_DECRYPT: 2723 case KVM_SEV_DBG_ENCRYPT: 2724 r = sev_dbg_crypt(kvm, &sev_cmd, sev_cmd.id); 2725 break; 2726 case KVM_SEV_LAUNCH_SECRET: 2727 r = sev_launch_secret(kvm, &sev_cmd); 2728 break; 2729 case KVM_SEV_GET_ATTESTATION_REPORT: 2730 r = sev_get_attestation_report(kvm, &sev_cmd); 2731 break; 2732 case KVM_SEV_SEND_START: 2733 r = sev_send_start(kvm, &sev_cmd); 2734 break; 2735 case KVM_SEV_SEND_UPDATE_DATA: 2736 r = sev_send_update_data(kvm, &sev_cmd); 2737 break; 2738 case KVM_SEV_SEND_FINISH: 2739 r = sev_send_finish(kvm, &sev_cmd); 2740 break; 2741 case KVM_SEV_SEND_CANCEL: 2742 r = sev_send_cancel(kvm, &sev_cmd); 2743 break; 2744 case KVM_SEV_RECEIVE_START: 2745 r = sev_receive_start(kvm, &sev_cmd); 2746 break; 2747 case KVM_SEV_RECEIVE_UPDATE_DATA: 2748 r = sev_receive_update_data(kvm, &sev_cmd); 2749 break; 2750 case KVM_SEV_RECEIVE_FINISH: 2751 r = sev_receive_finish(kvm, &sev_cmd); 2752 break; 2753 case KVM_SEV_SNP_LAUNCH_START: 2754 r = snp_launch_start(kvm, &sev_cmd); 2755 break; 2756 case KVM_SEV_SNP_LAUNCH_UPDATE: 2757 r = snp_launch_update(kvm, &sev_cmd); 2758 break; 2759 case KVM_SEV_SNP_LAUNCH_FINISH: 2760 r = snp_launch_finish(kvm, &sev_cmd); 2761 break; 2762 case KVM_SEV_SNP_ENABLE_REQ_CERTS: 2763 r = snp_enable_certs(kvm); 2764 break; 2765 default: 2766 return -EINVAL; 2767 } 2768 2769 if (copy_to_user(argp, &sev_cmd, sizeof(struct kvm_sev_cmd))) 2770 r = -EFAULT; 2771 2772 return r; 2773 } 2774 2775 int sev_mem_enc_register_region(struct kvm *kvm, 2776 struct kvm_enc_region *range) 2777 { 2778 struct kvm_sev_info *sev = to_kvm_sev_info(kvm); 2779 struct enc_region *region; 2780 int ret = 0; 2781 2782 guard(mutex)(&kvm->lock); 2783 2784 if (!sev_guest(kvm)) 2785 return -ENOTTY; 2786 2787 /* If kvm is mirroring encryption context it isn't responsible for it */ 2788 if (is_mirroring_enc_context(kvm)) 2789 return -EINVAL; 2790 2791 region = kzalloc_obj(*region, GFP_KERNEL_ACCOUNT); 2792 if (!region) 2793 return -ENOMEM; 2794 2795 /* 2796 * Do NOT specify FOLL_WRITE, as KVM isn't using the pinned pages to 2797 * write memory, and FOLL_LONGTERM itself triggers CoW unshare. 2798 */ 2799 region->pages = sev_pin_memory(kvm, range->addr, range->size, ®ion->npages, 2800 FOLL_LONGTERM); 2801 if (IS_ERR(region->pages)) { 2802 ret = PTR_ERR(region->pages); 2803 goto e_free; 2804 } 2805 2806 /* 2807 * The guest may change the memory encryption attribute from C=0 -> C=1 2808 * or vice versa for this memory range. Lets make sure caches are 2809 * flushed to ensure that guest data gets written into memory with 2810 * correct C-bit. Note, this must be done before dropping kvm->lock, 2811 * as region and its array of pages can be freed by a different task 2812 * once kvm->lock is released. 2813 */ 2814 sev_clflush_pages(region->pages, region->npages); 2815 2816 region->uaddr = range->addr; 2817 region->size = range->size; 2818 2819 list_add_tail(®ion->list, &sev->regions_list); 2820 return ret; 2821 2822 e_free: 2823 kfree(region); 2824 return ret; 2825 } 2826 2827 static struct enc_region * 2828 find_enc_region(struct kvm *kvm, struct kvm_enc_region *range) 2829 { 2830 struct kvm_sev_info *sev = to_kvm_sev_info(kvm); 2831 struct list_head *head = &sev->regions_list; 2832 struct enc_region *i; 2833 2834 list_for_each_entry(i, head, list) { 2835 if (i->uaddr == range->addr && 2836 i->size == range->size) 2837 return i; 2838 } 2839 2840 return NULL; 2841 } 2842 2843 static void __unregister_enc_region_locked(struct kvm *kvm, 2844 struct enc_region *region) 2845 { 2846 sev_unpin_memory(kvm, region->pages, region->npages); 2847 list_del(®ion->list); 2848 kfree(region); 2849 } 2850 2851 int sev_mem_enc_unregister_region(struct kvm *kvm, 2852 struct kvm_enc_region *range) 2853 { 2854 struct enc_region *region; 2855 2856 /* If kvm is mirroring encryption context it isn't responsible for it */ 2857 if (is_mirroring_enc_context(kvm)) 2858 return -EINVAL; 2859 2860 guard(mutex)(&kvm->lock); 2861 2862 if (!sev_guest(kvm)) 2863 return -ENOTTY; 2864 2865 region = find_enc_region(kvm, range); 2866 if (!region) 2867 return -EINVAL; 2868 2869 sev_writeback_caches(kvm); 2870 2871 __unregister_enc_region_locked(kvm, region); 2872 2873 return 0; 2874 } 2875 2876 int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd) 2877 { 2878 CLASS(fd, f)(source_fd); 2879 struct kvm *source_kvm; 2880 struct kvm_sev_info *source_sev, *mirror_sev; 2881 int ret; 2882 2883 if (fd_empty(f)) 2884 return -EBADF; 2885 2886 source_kvm = file_to_kvm_x86(fd_file(f)); 2887 if (!source_kvm) 2888 return -EBADF; 2889 2890 ret = sev_lock_two_vms(kvm, source_kvm); 2891 if (ret) 2892 return ret; 2893 2894 /* 2895 * Mirrors of mirrors should work, but let's not get silly. Also 2896 * disallow out-of-band SEV/SEV-ES init if the target is already an 2897 * SEV guest, or if vCPUs have been created. KVM relies on vCPUs being 2898 * created after SEV/SEV-ES initialization, e.g. to init intercepts. 2899 * Also do not allow SNP VM mirroring until additional state transfer is implemented. 2900 */ 2901 if (sev_guest(kvm) || !sev_guest(source_kvm) || sev_snp_guest(source_kvm) || 2902 is_mirroring_enc_context(source_kvm) || kvm->created_vcpus) { 2903 ret = -EINVAL; 2904 goto e_unlock; 2905 } 2906 2907 mirror_sev = to_kvm_sev_info(kvm); 2908 ret = sev_alloc_have_run_cpus(mirror_sev); 2909 if (ret) 2910 goto e_unlock; 2911 2912 /* 2913 * The mirror kvm holds an enc_context_owner ref so its asid can't 2914 * disappear until we're done with it 2915 */ 2916 source_sev = to_kvm_sev_info(source_kvm); 2917 2918 /* Set enc_context_owner and copy its encryption context over */ 2919 mutex_lock(&sev_mirror_lock); 2920 kvm_get_kvm(source_kvm); 2921 list_add_tail(&mirror_sev->mirror_entry, &source_sev->mirror_vms); 2922 mirror_sev->enc_context_owner = source_kvm; 2923 mutex_unlock(&sev_mirror_lock); 2924 2925 mirror_sev->active = true; 2926 kvm->arch.has_protected_page_tables = true; 2927 mirror_sev->asid = source_sev->asid; 2928 mirror_sev->fd = source_sev->fd; 2929 mirror_sev->es_active = source_sev->es_active; 2930 mirror_sev->need_init = false; 2931 mirror_sev->handle = source_sev->handle; 2932 INIT_LIST_HEAD(&mirror_sev->regions_list); 2933 INIT_LIST_HEAD(&mirror_sev->mirror_vms); 2934 ret = 0; 2935 2936 /* 2937 * Do not copy ap_jump_table. Since the mirror does not share the same 2938 * KVM contexts as the original, and they may have different 2939 * memory-views. 2940 */ 2941 2942 e_unlock: 2943 sev_unlock_two_vms(kvm, source_kvm); 2944 return ret; 2945 } 2946 2947 static int snp_decommission_context(struct kvm *kvm) 2948 { 2949 struct kvm_sev_info *sev = to_kvm_sev_info(kvm); 2950 struct sev_data_snp_addr data = {}; 2951 int ret; 2952 2953 /* If context is not created then do nothing */ 2954 if (!sev->snp_context) 2955 return 0; 2956 2957 /* Do the decommision, which will unbind the ASID from the SNP context */ 2958 data.address = __sme_pa(sev->snp_context); 2959 down_write(&sev_deactivate_lock); 2960 ret = sev_do_cmd(SEV_CMD_SNP_DECOMMISSION, &data, NULL); 2961 up_write(&sev_deactivate_lock); 2962 2963 if (WARN_ONCE(ret, "Failed to release guest context, ret %d", ret)) 2964 return ret; 2965 2966 snp_free_firmware_page(sev->snp_context); 2967 sev->snp_context = NULL; 2968 2969 return 0; 2970 } 2971 2972 void sev_vm_init(struct kvm *kvm) 2973 { 2974 switch (kvm->arch.vm_type) { 2975 case KVM_X86_DEFAULT_VM: 2976 case KVM_X86_SW_PROTECTED_VM: 2977 break; 2978 case KVM_X86_SNP_VM: 2979 kvm->arch.has_private_mem = true; 2980 fallthrough; 2981 case KVM_X86_SEV_ES_VM: 2982 kvm->arch.has_protected_state = true; 2983 fallthrough; 2984 case KVM_X86_SEV_VM: 2985 kvm->arch.has_protected_page_tables = true; 2986 kvm->arch.pre_fault_allowed = !kvm->arch.has_private_mem; 2987 to_kvm_sev_info(kvm)->need_init = true; 2988 break; 2989 default: 2990 WARN_ONCE(1, "Unsupported VM type %u", kvm->arch.vm_type); 2991 break; 2992 } 2993 } 2994 2995 void sev_vm_destroy(struct kvm *kvm) 2996 { 2997 struct kvm_sev_info *sev = to_kvm_sev_info(kvm); 2998 struct list_head *head = &sev->regions_list; 2999 struct list_head *pos, *q; 3000 3001 /* 3002 * Free the mask even if the VM is not *currently* an SEV VM, as it may 3003 * have been an SEV VM prior to intra-host migration. 3004 */ 3005 sev_free_have_run_cpus(sev); 3006 3007 if (!sev_guest(kvm)) 3008 return; 3009 3010 WARN_ON(!list_empty(&sev->mirror_vms)); 3011 3012 /* 3013 * If this is a mirror VM, remove it from the owner's list of a mirrors 3014 * and skip ASID cleanup (the ASID is tied to the lifetime of the owner). 3015 * Note, mirror VMs don't support registering encrypted regions. 3016 */ 3017 if (is_mirroring_enc_context(kvm)) { 3018 struct kvm *owner_kvm; 3019 3020 mutex_lock(&sev_mirror_lock); 3021 owner_kvm = sev->enc_context_owner; 3022 list_del(&sev->mirror_entry); 3023 sev->enc_context_owner = NULL; 3024 3025 /* 3026 * The reference to owner_kvm cannot move after sev_mirror_lock is 3027 * released. Release it before kvm_put_kvm() so that owner_kvm is 3028 * never destroyed inside sev_mirror_lock. 3029 */ 3030 mutex_unlock(&sev_mirror_lock); 3031 kvm_put_kvm(owner_kvm); 3032 return; 3033 } 3034 3035 3036 /* 3037 * if userspace was terminated before unregistering the memory regions 3038 * then lets unpin all the registered memory. 3039 */ 3040 if (!list_empty(head)) { 3041 list_for_each_safe(pos, q, head) { 3042 __unregister_enc_region_locked(kvm, 3043 list_entry(pos, struct enc_region, list)); 3044 cond_resched(); 3045 } 3046 } 3047 3048 if (sev_snp_guest(kvm)) { 3049 snp_guest_req_cleanup(kvm); 3050 3051 /* 3052 * Decomission handles unbinding of the ASID. If it fails for 3053 * some unexpected reason, just leak the ASID. 3054 */ 3055 if (snp_decommission_context(kvm)) 3056 return; 3057 3058 snp_enable_rmpopt(); 3059 } else { 3060 sev_unbind_asid(kvm, sev->handle); 3061 } 3062 3063 sev_asid_free(sev); 3064 } 3065 3066 void __init sev_set_cpu_caps(void) 3067 { 3068 if (sev_enabled) 3069 kvm_cpu_cap_set(X86_FEATURE_SEV); 3070 3071 if (sev_es_enabled) 3072 kvm_cpu_cap_set(X86_FEATURE_SEV_ES); 3073 3074 if (sev_snp_enabled) 3075 kvm_cpu_cap_set(X86_FEATURE_SEV_SNP); 3076 } 3077 3078 static bool is_sev_snp_initialized(void) 3079 { 3080 struct sev_user_data_snp_status *status; 3081 struct sev_data_snp_addr buf; 3082 bool initialized = false; 3083 int ret, error = 0; 3084 3085 status = snp_alloc_firmware_page(GFP_KERNEL | __GFP_ZERO); 3086 if (!status) 3087 return false; 3088 3089 buf.address = __psp_pa(status); 3090 ret = sev_do_cmd(SEV_CMD_SNP_PLATFORM_STATUS, &buf, &error); 3091 if (ret) { 3092 pr_err("SEV: SNP_PLATFORM_STATUS failed ret=%d, fw_error=%d (%#x)\n", 3093 ret, error, error); 3094 goto out; 3095 } 3096 3097 initialized = !!status->state; 3098 3099 out: 3100 snp_free_firmware_page(status); 3101 3102 return initialized; 3103 } 3104 3105 static const char * __init sev_str_feature_state(bool is_supported, bool is_usable) 3106 { 3107 return is_supported ? is_usable ? "enabled" : "unusable" : "disabled"; 3108 } 3109 3110 void __init sev_hardware_setup(void) 3111 { 3112 unsigned int eax, ebx, ecx, edx, sev_asid_count, sev_es_asid_count; 3113 struct sev_platform_init_args init_args = {0}; 3114 bool sev_snp_supported = false; 3115 bool sev_es_supported = false; 3116 bool sev_supported = false; 3117 u32 vm_types = 0; 3118 3119 if (!sev_enabled || !npt_enabled || !nrips) 3120 goto out; 3121 3122 /* 3123 * SEV must obviously be supported in hardware. Sanity check that the 3124 * CPU supports decode assists, which is mandatory for SEV guests to 3125 * support instruction emulation. Ditto for flushing by ASID, as SEV 3126 * guests are bound to a single ASID, i.e. KVM can't rotate to a new 3127 * ASID to effect a TLB flush. 3128 */ 3129 if (!boot_cpu_has(X86_FEATURE_SEV) || 3130 WARN_ON_ONCE(!boot_cpu_has(X86_FEATURE_DECODEASSISTS)) || 3131 WARN_ON_ONCE(!boot_cpu_has(X86_FEATURE_FLUSHBYASID))) 3132 goto out; 3133 3134 /* 3135 * The kernel's initcall infrastructure lacks the ability to express 3136 * dependencies between initcalls, whereas the modules infrastructure 3137 * automatically handles dependencies via symbol loading. Ensure the 3138 * PSP SEV driver is initialized before proceeding if KVM is built-in, 3139 * as the dependency isn't handled by the initcall infrastructure. 3140 */ 3141 if (IS_BUILTIN(CONFIG_KVM_AMD) && sev_module_init()) 3142 goto out; 3143 3144 /* Retrieve SEV CPUID information */ 3145 cpuid(0x8000001f, &eax, &ebx, &ecx, &edx); 3146 3147 /* Set encryption bit location for SEV-ES guests */ 3148 sev_enc_bit = ebx & 0x3f; 3149 3150 /* Maximum number of encrypted guests supported simultaneously */ 3151 max_sev_asid = ecx; 3152 if (!max_sev_asid) 3153 goto out; 3154 3155 /* Minimum ASID value that should be used for SEV guest */ 3156 min_sev_asid = edx; 3157 sev_me_mask = 1UL << (ebx & 0x3f); 3158 3159 /* 3160 * Initialize SEV ASID bitmaps. Allocate space for ASID 0 in the bitmap, 3161 * even though it's never used, so that the bitmap is indexed by the 3162 * actual ASID. 3163 */ 3164 nr_asids = max_sev_asid + 1; 3165 sev_asid_bitmap = bitmap_zalloc(nr_asids, GFP_KERNEL); 3166 if (!sev_asid_bitmap) 3167 goto out; 3168 3169 sev_reclaim_asid_bitmap = bitmap_zalloc(nr_asids, GFP_KERNEL); 3170 if (!sev_reclaim_asid_bitmap) { 3171 bitmap_free(sev_asid_bitmap); 3172 sev_asid_bitmap = NULL; 3173 goto out; 3174 } 3175 3176 if (min_sev_asid <= max_sev_asid) { 3177 sev_asid_count = max_sev_asid - min_sev_asid + 1; 3178 WARN_ON_ONCE(misc_cg_set_capacity(MISC_CG_RES_SEV, sev_asid_count)); 3179 } 3180 sev_supported = true; 3181 3182 /* SEV-ES support requested? */ 3183 if (!sev_es_enabled) 3184 goto out; 3185 3186 /* 3187 * SEV-ES requires MMIO caching as KVM doesn't have access to the guest 3188 * instruction stream, i.e. can't emulate in response to a #NPF and 3189 * instead relies on #NPF(RSVD) being reflected into the guest as #VC 3190 * (the guest can then do a #VMGEXIT to request MMIO emulation). 3191 */ 3192 if (!enable_mmio_caching) 3193 goto out; 3194 3195 /* Does the CPU support SEV-ES? */ 3196 if (!boot_cpu_has(X86_FEATURE_SEV_ES)) 3197 goto out; 3198 3199 if (!lbrv) { 3200 WARN_ONCE(!boot_cpu_has(X86_FEATURE_LBRV), 3201 "LBRV must be present for SEV-ES support"); 3202 goto out; 3203 } 3204 3205 /* Has the system been allocated ASIDs for SEV-ES? */ 3206 if (min_sev_asid == 1) 3207 goto out; 3208 3209 min_sev_es_asid = min_snp_asid = 1; 3210 max_sev_es_asid = max_snp_asid = min_sev_asid - 1; 3211 3212 sev_es_asid_count = min_sev_asid - 1; 3213 WARN_ON_ONCE(misc_cg_set_capacity(MISC_CG_RES_SEV_ES, sev_es_asid_count)); 3214 sev_es_supported = true; 3215 sev_snp_supported = sev_snp_enabled && cc_platform_has(CC_ATTR_HOST_SEV_SNP); 3216 3217 out: 3218 if (sev_enabled) { 3219 init_args.probe = true; 3220 3221 if (sev_is_snp_ciphertext_hiding_supported()) 3222 init_args.max_snp_asid = min(nr_ciphertext_hiding_asids, 3223 min_sev_asid - 1); 3224 3225 if (sev_platform_init(&init_args)) 3226 sev_supported = sev_es_supported = sev_snp_supported = false; 3227 else if (sev_snp_supported) 3228 sev_snp_supported = is_sev_snp_initialized(); 3229 3230 if (sev_snp_supported) { 3231 snp_supported_policy_bits = sev_get_snp_policy_bits() & 3232 KVM_SNP_POLICY_MASK_VALID; 3233 nr_ciphertext_hiding_asids = init_args.max_snp_asid; 3234 } 3235 3236 /* 3237 * If ciphertext hiding is enabled, the joint SEV-ES/SEV-SNP 3238 * ASID range is partitioned into separate SEV-ES and SEV-SNP 3239 * ASID ranges, with the SEV-SNP range being [1..max_snp_asid] 3240 * and the SEV-ES range being (max_snp_asid..max_sev_es_asid]. 3241 * Note, SEV-ES may effectively be disabled if all ASIDs from 3242 * the joint range are assigned to SEV-SNP. 3243 */ 3244 if (nr_ciphertext_hiding_asids) { 3245 max_snp_asid = nr_ciphertext_hiding_asids; 3246 min_sev_es_asid = max_snp_asid + 1; 3247 pr_info("SEV-SNP ciphertext hiding enabled\n"); 3248 } 3249 } 3250 3251 if (sev_supported && min_sev_asid <= max_sev_asid) 3252 vm_types |= BIT(KVM_X86_SEV_VM); 3253 if (sev_es_supported && min_sev_es_asid <= max_sev_es_asid) 3254 vm_types |= BIT(KVM_X86_SEV_ES_VM); 3255 if (sev_snp_supported) 3256 vm_types |= BIT(KVM_X86_SNP_VM); 3257 vm_types &= sev_firmware_supported_vm_types(); 3258 3259 kvm_caps.supported_vm_types |= vm_types; 3260 3261 if (boot_cpu_has(X86_FEATURE_SEV)) 3262 pr_info("SEV %s (ASIDs %u - %u)\n", 3263 sev_str_feature_state(sev_supported, vm_types & BIT(KVM_X86_SEV_VM)), 3264 min_sev_asid, max_sev_asid); 3265 if (boot_cpu_has(X86_FEATURE_SEV_ES)) 3266 pr_info("SEV-ES %s (ASIDs %u - %u)\n", 3267 sev_str_feature_state(sev_es_supported, vm_types & BIT(KVM_X86_SEV_ES_VM)), 3268 min_sev_es_asid, max_sev_es_asid); 3269 if (boot_cpu_has(X86_FEATURE_SEV_SNP)) 3270 pr_info("SEV-SNP %s (ASIDs %u - %u)\n", 3271 sev_str_feature_state(sev_snp_supported, vm_types & BIT(KVM_X86_SNP_VM)), 3272 min_snp_asid, max_snp_asid); 3273 3274 sev_enabled = sev_supported; 3275 sev_es_enabled = sev_es_supported; 3276 sev_snp_enabled = sev_snp_supported; 3277 3278 sev_supported_vmsa_features = 0; 3279 3280 if (sev_es_enabled && cpu_feature_enabled(X86_FEATURE_DEBUG_SWAP) && 3281 cpu_feature_enabled(X86_FEATURE_NO_NESTED_DATA_BP)) 3282 sev_supported_vmsa_features |= SVM_SEV_FEAT_DEBUG_SWAP; 3283 3284 if (sev_snp_enabled && tsc_khz && cpu_feature_enabled(X86_FEATURE_SNP_SECURE_TSC)) 3285 sev_supported_vmsa_features |= SVM_SEV_FEAT_SECURE_TSC; 3286 } 3287 3288 void sev_hardware_unsetup(void) 3289 { 3290 if (!sev_enabled) 3291 return; 3292 3293 /* No need to take sev_bitmap_lock, all VMs have been destroyed. */ 3294 sev_flush_asids(1, max_sev_asid); 3295 3296 bitmap_free(sev_asid_bitmap); 3297 bitmap_free(sev_reclaim_asid_bitmap); 3298 3299 misc_cg_set_capacity(MISC_CG_RES_SEV, 0); 3300 misc_cg_set_capacity(MISC_CG_RES_SEV_ES, 0); 3301 3302 sev_platform_shutdown(); 3303 } 3304 3305 int sev_cpu_init(struct svm_cpu_data *sd) 3306 { 3307 if (!sev_enabled) 3308 return 0; 3309 3310 sd->sev_vmcbs = kcalloc(nr_asids, sizeof(void *), GFP_KERNEL); 3311 if (!sd->sev_vmcbs) 3312 return -ENOMEM; 3313 3314 return 0; 3315 } 3316 3317 /* 3318 * Pages used by hardware to hold guest encrypted state must be flushed before 3319 * returning them to the system. 3320 */ 3321 static void sev_flush_encrypted_page(struct kvm_vcpu *vcpu, void *va) 3322 { 3323 unsigned int asid = sev_get_asid(vcpu->kvm); 3324 3325 /* 3326 * Note! The address must be a kernel address, as regular page walk 3327 * checks are performed by VM_PAGE_FLUSH, i.e. operating on a user 3328 * address is non-deterministic and unsafe. This function deliberately 3329 * takes a pointer to deter passing in a user address. 3330 */ 3331 unsigned long addr = (unsigned long)va; 3332 3333 /* 3334 * If CPU enforced cache coherency for encrypted mappings of the 3335 * same physical page is supported, use CLFLUSHOPT instead. NOTE: cache 3336 * flush is still needed in order to work properly with DMA devices. 3337 */ 3338 if (boot_cpu_has(X86_FEATURE_SME_COHERENT)) { 3339 clflush_cache_range(va, PAGE_SIZE); 3340 return; 3341 } 3342 3343 /* 3344 * VM Page Flush takes a host virtual address and a guest ASID. Fall 3345 * back to full writeback of caches if this faults so as not to make 3346 * any problems worse by leaving stale encrypted data in the cache. 3347 */ 3348 if (WARN_ON_ONCE(wrmsrq_safe(MSR_AMD64_VM_PAGE_FLUSH, addr | asid))) 3349 goto do_sev_writeback_caches; 3350 3351 return; 3352 3353 do_sev_writeback_caches: 3354 sev_writeback_caches(vcpu->kvm); 3355 } 3356 3357 void sev_guest_memory_reclaimed(struct kvm *kvm) 3358 { 3359 /* 3360 * With SNP+gmem, private/encrypted memory is unreachable via the 3361 * hva-based mmu notifiers, i.e. these events are explicitly scoped to 3362 * shared pages, where there's no need to flush caches. 3363 * 3364 * Checking for SEV+ outside of kvm->lock is safe as __sev_guest_init() 3365 * can only be done before vCPUs are created, caches can be incoherent 3366 * if and only if a vCPU was run, and either this task will see the VM 3367 * as being SEV+ or the vCPU won't be to access the memory (because of 3368 * the in-progress invalidation). 3369 */ 3370 if (!____sev_guest(kvm) || ____sev_snp_guest(kvm)) 3371 return; 3372 3373 sev_writeback_caches(kvm); 3374 } 3375 3376 static void dump_ghcb(struct vcpu_svm *svm) 3377 { 3378 struct vmcb_control_area *control = &svm->vmcb->control; 3379 unsigned int nbits; 3380 3381 /* Re-use the dump_invalid_vmcb module parameter */ 3382 if (!dump_invalid_vmcb) { 3383 pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n"); 3384 return; 3385 } 3386 3387 nbits = sizeof(svm->sev_es.valid_bitmap) * 8; 3388 3389 /* 3390 * Print KVM's snapshot of the GHCB values that were (unsuccessfully) 3391 * used to handle the exit. If the guest has since modified the GHCB 3392 * itself, dumping the raw GHCB won't help debug why KVM was unable to 3393 * handle the VMGEXIT that KVM observed. 3394 */ 3395 pr_err("GHCB (GPA=%016llx) snapshot:\n", svm->vmcb->control.ghcb_gpa); 3396 pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_code", 3397 control->exit_code, kvm_ghcb_sw_exit_code_is_valid(svm)); 3398 pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_info_1", 3399 control->exit_info_1, kvm_ghcb_sw_exit_info_1_is_valid(svm)); 3400 pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_info_2", 3401 control->exit_info_2, kvm_ghcb_sw_exit_info_2_is_valid(svm)); 3402 pr_err("%-20s%016llx is_valid: %u\n", "sw_scratch", 3403 svm->sev_es.sw_scratch, kvm_ghcb_sw_scratch_is_valid(svm)); 3404 pr_err("%-20s%*pb\n", "valid_bitmap", nbits, svm->sev_es.valid_bitmap); 3405 } 3406 3407 static void sev_es_sync_to_ghcb(struct vcpu_svm *svm) 3408 { 3409 struct kvm_vcpu *vcpu = &svm->vcpu; 3410 struct ghcb *ghcb = svm->sev_es.ghcb; 3411 3412 /* 3413 * The GHCB protocol so far allows for the following data 3414 * to be returned: 3415 * GPRs RAX, RBX, RCX, RDX 3416 * 3417 * Copy their values, even if they may not have been written during the 3418 * VM-Exit. It's the guest's responsibility to not consume random data. 3419 */ 3420 ghcb_set_rax(ghcb, vcpu->arch.regs[VCPU_REGS_RAX]); 3421 ghcb_set_rbx(ghcb, vcpu->arch.regs[VCPU_REGS_RBX]); 3422 ghcb_set_rcx(ghcb, vcpu->arch.regs[VCPU_REGS_RCX]); 3423 ghcb_set_rdx(ghcb, vcpu->arch.regs[VCPU_REGS_RDX]); 3424 } 3425 3426 static void sev_es_sync_from_ghcb(struct vcpu_svm *svm) 3427 { 3428 struct vmcb_control_area *control = &svm->vmcb->control; 3429 struct kvm_vcpu *vcpu = &svm->vcpu; 3430 struct ghcb *ghcb = svm->sev_es.ghcb; 3431 3432 /* 3433 * The GHCB protocol so far allows for the following data 3434 * to be supplied: 3435 * GPRs RAX, RBX, RCX, RDX 3436 * XCR0 3437 * CPL 3438 * 3439 * VMMCALL allows the guest to provide extra registers. KVM also 3440 * expects RSI for hypercalls, so include that, too. 3441 * 3442 * Copy their values to the appropriate location if supplied. 3443 */ 3444 memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs)); 3445 3446 BUILD_BUG_ON(sizeof(svm->sev_es.valid_bitmap) != sizeof(ghcb->save.valid_bitmap)); 3447 memcpy(&svm->sev_es.valid_bitmap, &ghcb->save.valid_bitmap, sizeof(ghcb->save.valid_bitmap)); 3448 3449 vcpu->arch.regs[VCPU_REGS_RAX] = kvm_ghcb_get_rax_if_valid(svm); 3450 vcpu->arch.regs[VCPU_REGS_RBX] = kvm_ghcb_get_rbx_if_valid(svm); 3451 vcpu->arch.regs[VCPU_REGS_RCX] = kvm_ghcb_get_rcx_if_valid(svm); 3452 vcpu->arch.regs[VCPU_REGS_RDX] = kvm_ghcb_get_rdx_if_valid(svm); 3453 vcpu->arch.regs[VCPU_REGS_RSI] = kvm_ghcb_get_rsi_if_valid(svm); 3454 3455 svm->vmcb->save.cpl = kvm_ghcb_get_cpl_if_valid(svm); 3456 3457 if (kvm_ghcb_xcr0_is_valid(svm)) 3458 __kvm_set_xcr(vcpu, 0, kvm_ghcb_get_xcr0(svm)); 3459 3460 if (kvm_ghcb_xss_is_valid(svm)) 3461 __kvm_emulate_msr_write(vcpu, MSR_IA32_XSS, kvm_ghcb_get_xss(svm)); 3462 3463 /* Copy the GHCB exit information into the VMCB fields */ 3464 control->exit_code = kvm_ghcb_get_sw_exit_code(svm); 3465 control->exit_info_1 = kvm_ghcb_get_sw_exit_info_1(svm); 3466 control->exit_info_2 = kvm_ghcb_get_sw_exit_info_2(svm); 3467 svm->sev_es.sw_scratch = kvm_ghcb_get_sw_scratch_if_valid(svm); 3468 3469 /* Clear the valid entries fields */ 3470 memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); 3471 } 3472 3473 static bool sev_es_are_required_ghcb_fields_valid(struct vcpu_svm *svm) 3474 { 3475 struct vmcb_control_area *control = &svm->vmcb->control; 3476 struct kvm_vcpu *vcpu = &svm->vcpu; 3477 3478 if (!kvm_ghcb_sw_exit_code_is_valid(svm) || 3479 !kvm_ghcb_sw_exit_info_1_is_valid(svm) || 3480 !kvm_ghcb_sw_exit_info_2_is_valid(svm)) 3481 return false; 3482 3483 switch (control->exit_code) { 3484 case SVM_EXIT_WRITE_DR7: 3485 return kvm_ghcb_rax_is_valid(svm); 3486 case SVM_EXIT_RDPMC: 3487 return kvm_ghcb_rcx_is_valid(svm); 3488 case SVM_EXIT_CPUID: 3489 if (!kvm_ghcb_rax_is_valid(svm) || 3490 !kvm_ghcb_rcx_is_valid(svm)) 3491 return false; 3492 3493 return vcpu->arch.regs[VCPU_REGS_RAX] != 0xd || 3494 kvm_ghcb_xcr0_is_valid(svm); 3495 case SVM_EXIT_IOIO: 3496 if (control->exit_info_1 & SVM_IOIO_STR_MASK) 3497 return kvm_ghcb_sw_scratch_is_valid(svm); 3498 3499 if (!(control->exit_info_1 & SVM_IOIO_TYPE_MASK)) 3500 return kvm_ghcb_rax_is_valid(svm); 3501 3502 return true; 3503 case SVM_EXIT_MSR: 3504 if (!kvm_ghcb_rcx_is_valid(svm)) 3505 return false; 3506 3507 return !control->exit_info_1 || 3508 (kvm_ghcb_rax_is_valid(svm) && kvm_ghcb_rdx_is_valid(svm)); 3509 case SVM_EXIT_VMMCALL: 3510 return kvm_ghcb_rax_is_valid(svm) && kvm_ghcb_cpl_is_valid(svm); 3511 case SVM_EXIT_MONITOR: 3512 return kvm_ghcb_rax_is_valid(svm) && 3513 kvm_ghcb_rcx_is_valid(svm) && 3514 kvm_ghcb_rdx_is_valid(svm); 3515 case SVM_EXIT_MWAIT: 3516 return kvm_ghcb_rax_is_valid(svm) && kvm_ghcb_rcx_is_valid(svm); 3517 case SVM_VMGEXIT_AP_CREATION: 3518 return kvm_ghcb_rax_is_valid(svm) || 3519 lower_32_bits(control->exit_info_1) == SVM_VMGEXIT_AP_DESTROY; 3520 break; 3521 case SVM_VMGEXIT_MMIO_READ: 3522 case SVM_VMGEXIT_MMIO_WRITE: 3523 case SVM_VMGEXIT_PSC: 3524 return kvm_ghcb_sw_scratch_is_valid(svm); 3525 default: 3526 return true; 3527 } 3528 } 3529 3530 static void __sev_es_unmap_ghcb(struct vcpu_svm *svm) 3531 { 3532 if (svm->sev_es.ghcb_sa_free) { 3533 kvfree(svm->sev_es.ghcb_sa); 3534 svm->sev_es.ghcb_sa = NULL; 3535 svm->sev_es.ghcb_sa_free = false; 3536 } 3537 3538 if (svm->sev_es.ghcb) { 3539 kvm_vcpu_unmap(&svm->vcpu, &svm->sev_es.ghcb_map); 3540 svm->sev_es.ghcb = NULL; 3541 } 3542 } 3543 3544 void sev_es_unmap_ghcb(struct vcpu_svm *svm) 3545 { 3546 /* Clear any indication that the vCPU is in a type of AP Reset Hold */ 3547 svm->sev_es.ap_reset_hold_type = AP_RESET_HOLD_NONE; 3548 3549 if (!svm->sev_es.ghcb) 3550 return; 3551 3552 /* 3553 * If the scratch area lives outside the GHCB, there's a buffer that, 3554 * depending on the operation performed, may need to be synced. 3555 */ 3556 if (svm->sev_es.ghcb_sa_sync) { 3557 kvm_write_guest(svm->vcpu.kvm, svm->sev_es.sw_scratch, 3558 svm->sev_es.ghcb_sa, svm->sev_es.ghcb_sa_len); 3559 svm->sev_es.ghcb_sa_sync = false; 3560 } 3561 3562 trace_kvm_vmgexit_exit(svm->vcpu.vcpu_id, svm->sev_es.ghcb); 3563 3564 sev_es_sync_to_ghcb(svm); 3565 3566 __sev_es_unmap_ghcb(svm); 3567 } 3568 3569 void sev_free_vcpu(struct kvm_vcpu *vcpu) 3570 { 3571 struct vcpu_svm *svm; 3572 3573 if (!is_sev_es_guest(vcpu)) 3574 return; 3575 3576 svm = to_svm(vcpu); 3577 3578 /* 3579 * If it's an SNP guest, then the VMSA was marked in the RMP table as 3580 * a guest-owned page. Transition the page to hypervisor state before 3581 * releasing it back to the system. 3582 */ 3583 if (is_sev_snp_guest(vcpu)) { 3584 u64 pfn = __pa(svm->sev_es.vmsa) >> PAGE_SHIFT; 3585 3586 if (kvm_rmp_make_shared(vcpu->kvm, pfn, PG_LEVEL_4K)) 3587 goto skip_vmsa_free; 3588 } 3589 3590 if (vcpu->arch.guest_state_protected) 3591 sev_flush_encrypted_page(vcpu, svm->sev_es.vmsa); 3592 3593 __free_page(virt_to_page(svm->sev_es.vmsa)); 3594 3595 skip_vmsa_free: 3596 __sev_es_unmap_ghcb(svm); 3597 } 3598 3599 bool sev_vcpu_needs_initialization(struct kvm_vcpu *vcpu) 3600 { 3601 return to_kvm_sev_info(vcpu->kvm)->need_init; 3602 } 3603 3604 int pre_sev_run(struct vcpu_svm *svm, int cpu) 3605 { 3606 struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu); 3607 struct kvm_vcpu *vcpu = &svm->vcpu; 3608 struct kvm *kvm = vcpu->kvm; 3609 unsigned int asid = sev_get_asid(kvm); 3610 3611 /* 3612 * Reject KVM_RUN if userspace attempts to run the vCPU with an invalid 3613 * VMSA, e.g. if userspace forces the vCPU to be RUNNABLE after an SNP 3614 * AP Destroy event. 3615 */ 3616 if (is_sev_es_guest(vcpu) && !VALID_PAGE(svm->vmcb->control.vmsa_pa)) 3617 return -EINVAL; 3618 3619 /* 3620 * To optimize cache flushes when memory is reclaimed from an SEV VM, 3621 * track physical CPUs that enter the guest for SEV VMs and thus can 3622 * have encrypted, dirty data in the cache, and flush caches only for 3623 * CPUs that have entered the guest. 3624 */ 3625 if (!cpumask_test_cpu(cpu, to_kvm_sev_info(kvm)->have_run_cpus)) 3626 cpumask_set_cpu(cpu, to_kvm_sev_info(kvm)->have_run_cpus); 3627 3628 /* Assign the asid allocated with this SEV guest */ 3629 svm->asid = asid; 3630 3631 /* 3632 * Flush guest TLB: 3633 * 3634 * 1) when different VMCB for the same ASID is to be run on the same host CPU. 3635 * 2) or this VMCB was executed on different host CPU in previous VMRUNs. 3636 */ 3637 if (sd->sev_vmcbs[asid] == svm->vmcb && 3638 svm->vcpu.arch.last_vmentry_cpu == cpu) 3639 return 0; 3640 3641 sd->sev_vmcbs[asid] = svm->vmcb; 3642 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID; 3643 return 0; 3644 } 3645 3646 #define GHCB_SCRATCH_AREA_LIMIT (16ULL * PAGE_SIZE) 3647 static int setup_vmgexit_scratch(struct vcpu_svm *svm, bool sync, u64 min_len) 3648 { 3649 struct vmcb_control_area *control = &svm->vmcb->control; 3650 u64 ghcb_scratch_beg, ghcb_scratch_end; 3651 u64 scratch_gpa_beg, scratch_gpa_end; 3652 void *scratch_va; 3653 3654 if (WARN_ON_ONCE(!min_len)) 3655 goto e_scratch; 3656 3657 scratch_gpa_beg = svm->sev_es.sw_scratch; 3658 if (!scratch_gpa_beg) { 3659 pr_err("vmgexit: scratch gpa not provided\n"); 3660 goto e_scratch; 3661 } 3662 3663 scratch_gpa_end = scratch_gpa_beg + min_len; 3664 if (scratch_gpa_end < scratch_gpa_beg) { 3665 pr_err("vmgexit: scratch length (%#llx) not valid for scratch address (%#llx)\n", 3666 min_len, scratch_gpa_beg); 3667 goto e_scratch; 3668 } 3669 3670 WARN_ON_ONCE(svm->sev_es.ghcb_sa_sync || svm->sev_es.ghcb_sa_free); 3671 3672 if ((scratch_gpa_beg & PAGE_MASK) == control->ghcb_gpa) { 3673 /* Scratch area begins within GHCB */ 3674 ghcb_scratch_beg = control->ghcb_gpa + 3675 offsetof(struct ghcb, shared_buffer); 3676 ghcb_scratch_end = control->ghcb_gpa + 3677 offsetof(struct ghcb, reserved_0xff0); 3678 3679 /* 3680 * If the scratch area begins within the GHCB, it must be 3681 * completely contained in the GHCB shared buffer area. 3682 */ 3683 if (scratch_gpa_beg < ghcb_scratch_beg || 3684 scratch_gpa_end > ghcb_scratch_end) { 3685 pr_err("vmgexit: scratch area is outside of GHCB shared buffer area (%#llx - %#llx)\n", 3686 scratch_gpa_beg, scratch_gpa_end); 3687 goto e_scratch; 3688 } 3689 3690 scratch_va = (void *)svm->sev_es.ghcb; 3691 scratch_va += (scratch_gpa_beg - control->ghcb_gpa); 3692 3693 svm->sev_es.ghcb_sa_sync = false; 3694 svm->sev_es.ghcb_sa_free = false; 3695 svm->sev_es.ghcb_sa_len = ghcb_scratch_end - scratch_gpa_beg; 3696 } else { 3697 /* GHCB v2 requires the scratch area to be within the GHCB. */ 3698 if (to_kvm_sev_info(svm->vcpu.kvm)->ghcb_version >= 2) 3699 goto e_scratch; 3700 3701 /* 3702 * The guest memory must be read into a kernel buffer, so 3703 * limit the size 3704 */ 3705 if (min_len > GHCB_SCRATCH_AREA_LIMIT) { 3706 pr_err("vmgexit: scratch area exceeds KVM limits (%#llx requested, %#llx limit)\n", 3707 min_len, GHCB_SCRATCH_AREA_LIMIT); 3708 goto e_scratch; 3709 } 3710 scratch_va = kvzalloc(min_len, GFP_KERNEL_ACCOUNT); 3711 if (!scratch_va) 3712 return -ENOMEM; 3713 3714 if (kvm_read_guest(svm->vcpu.kvm, scratch_gpa_beg, scratch_va, min_len)) { 3715 /* Unable to copy scratch area from guest */ 3716 pr_err("vmgexit: kvm_read_guest for scratch area failed\n"); 3717 3718 kvfree(scratch_va); 3719 return -EFAULT; 3720 } 3721 3722 /* 3723 * The scratch area is outside the GHCB. The operation will 3724 * dictate whether the buffer needs to be synced before running 3725 * the vCPU next time (i.e. a read was requested so the data 3726 * must be written back to the guest memory). 3727 */ 3728 svm->sev_es.ghcb_sa_sync = sync; 3729 svm->sev_es.ghcb_sa_free = true; 3730 svm->sev_es.ghcb_sa_len = min_len; 3731 } 3732 3733 svm->sev_es.ghcb_sa = scratch_va; 3734 return 0; 3735 3736 e_scratch: 3737 svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_SCRATCH_AREA); 3738 3739 return 1; 3740 } 3741 3742 static void set_ghcb_msr_bits(struct vcpu_svm *svm, u64 value, u64 mask, 3743 unsigned int pos) 3744 { 3745 svm->vmcb->control.ghcb_gpa &= ~(mask << pos); 3746 svm->vmcb->control.ghcb_gpa |= (value & mask) << pos; 3747 } 3748 3749 static u64 get_ghcb_msr_bits(struct vcpu_svm *svm, u64 mask, unsigned int pos) 3750 { 3751 return (svm->vmcb->control.ghcb_gpa >> pos) & mask; 3752 } 3753 3754 static void set_ghcb_msr(struct vcpu_svm *svm, u64 value) 3755 { 3756 svm->vmcb->control.ghcb_gpa = value; 3757 } 3758 3759 static int snp_rmptable_psmash(kvm_pfn_t pfn) 3760 { 3761 int ret; 3762 3763 pfn = pfn & ~(KVM_PAGES_PER_HPAGE(PG_LEVEL_2M) - 1); 3764 3765 /* 3766 * PSMASH_FAIL_INUSE indicates another processor is modifying the 3767 * entry, so retry until that's no longer the case. 3768 */ 3769 do { 3770 ret = psmash(pfn); 3771 } while (ret == PSMASH_FAIL_INUSE); 3772 3773 return ret; 3774 } 3775 3776 static int snp_complete_psc_msr(struct kvm_vcpu *vcpu) 3777 { 3778 u64 hypercall_ret = READ_ONCE(vcpu->run->hypercall.ret); 3779 struct vcpu_svm *svm = to_svm(vcpu); 3780 3781 if (!kvm_is_valid_map_gpa_range_ret(hypercall_ret)) 3782 return -EINVAL; 3783 3784 if (hypercall_ret) 3785 set_ghcb_msr(svm, GHCB_MSR_PSC_RESP_ERROR); 3786 else 3787 set_ghcb_msr(svm, GHCB_MSR_PSC_RESP); 3788 3789 return 1; /* resume guest */ 3790 } 3791 3792 static int snp_begin_psc_msr(struct vcpu_svm *svm, u64 ghcb_msr) 3793 { 3794 u64 gpa = gfn_to_gpa(GHCB_MSR_PSC_REQ_TO_GFN(ghcb_msr)); 3795 u8 op = GHCB_MSR_PSC_REQ_TO_OP(ghcb_msr); 3796 struct kvm_vcpu *vcpu = &svm->vcpu; 3797 3798 if (op != SNP_PAGE_STATE_PRIVATE && op != SNP_PAGE_STATE_SHARED) { 3799 set_ghcb_msr(svm, GHCB_MSR_PSC_RESP_ERROR); 3800 return 1; /* resume guest */ 3801 } 3802 3803 if (!user_exit_on_hypercall(vcpu->kvm, KVM_HC_MAP_GPA_RANGE)) { 3804 set_ghcb_msr(svm, GHCB_MSR_PSC_RESP_ERROR); 3805 return 1; /* resume guest */ 3806 } 3807 3808 vcpu->run->exit_reason = KVM_EXIT_HYPERCALL; 3809 vcpu->run->hypercall.nr = KVM_HC_MAP_GPA_RANGE; 3810 /* 3811 * In principle this should have been -KVM_ENOSYS, but userspace (QEMU <=9.2) 3812 * assumed that vcpu->run->hypercall.ret is never changed by KVM and thus that 3813 * it was always zero on KVM_EXIT_HYPERCALL. Since KVM is now overwriting 3814 * vcpu->run->hypercall.ret, ensuring that it is zero to not break QEMU. 3815 */ 3816 vcpu->run->hypercall.ret = 0; 3817 vcpu->run->hypercall.args[0] = gpa; 3818 vcpu->run->hypercall.args[1] = 1; 3819 vcpu->run->hypercall.args[2] = (op == SNP_PAGE_STATE_PRIVATE) 3820 ? KVM_MAP_GPA_RANGE_ENCRYPTED 3821 : KVM_MAP_GPA_RANGE_DECRYPTED; 3822 vcpu->run->hypercall.args[2] |= KVM_MAP_GPA_RANGE_PAGE_SZ_4K; 3823 3824 vcpu->arch.complete_userspace_io = snp_complete_psc_msr; 3825 3826 return 0; /* forward request to userspace */ 3827 } 3828 3829 struct psc_buffer { 3830 struct psc_hdr hdr; 3831 struct psc_entry entries[]; 3832 } __packed; 3833 3834 static int snp_do_psc(struct vcpu_svm *svm); 3835 3836 static void snp_complete_psc(struct vcpu_svm *svm, u64 psc_ret) 3837 { 3838 memset(&svm->sev_es.psc, 0, sizeof(svm->sev_es.psc)); 3839 3840 /* 3841 * PSC requests always get a "no action" response in SW_EXITINFO1, with 3842 * a PSC-specific return code in SW_EXITINFO2 that provides the "real" 3843 * return code. E.g. if the PSC request was interrupted, the need to 3844 * retry is communicated via SW_EXITINFO2, not SW_EXITINFO1. 3845 */ 3846 svm_vmgexit_no_action(svm, psc_ret); 3847 } 3848 3849 static void __snp_complete_one_psc(struct vcpu_svm *svm) 3850 { 3851 struct vcpu_sev_es_state *sev_es = &svm->sev_es; 3852 struct psc_buffer *guest_psc = sev_es->ghcb_sa; 3853 __u16 idx; 3854 3855 /* 3856 * Everything in-flight has been processed successfully. Update the 3857 * corresponding entries in the guest's PSC buffer and zero out the 3858 * count of in-flight PSC entries. 3859 */ 3860 for (idx = sev_es->psc.cur_idx; sev_es->psc.batch_size; 3861 sev_es->psc.batch_size--, idx++) { 3862 struct psc_entry entry = READ_ONCE(guest_psc->entries[idx]); 3863 3864 guest_psc->entries[idx].cur_page = entry.pagesize ? 512 : 1; 3865 } 3866 3867 sev_es->psc.cur_idx = idx; 3868 guest_psc->hdr.cur_entry = idx; 3869 } 3870 3871 static int snp_complete_one_psc(struct kvm_vcpu *vcpu) 3872 { 3873 u64 hypercall_ret = READ_ONCE(vcpu->run->hypercall.ret); 3874 struct vcpu_svm *svm = to_svm(vcpu); 3875 3876 if (!kvm_is_valid_map_gpa_range_ret(hypercall_ret)) 3877 return -EINVAL; 3878 3879 if (hypercall_ret) { 3880 snp_complete_psc(svm, VMGEXIT_PSC_ERROR_GENERIC); 3881 return 1; /* resume guest */ 3882 } 3883 3884 __snp_complete_one_psc(svm); 3885 3886 /* Handle the next range (if any). */ 3887 return snp_do_psc(svm); 3888 } 3889 3890 static int snp_do_psc(struct vcpu_svm *svm) 3891 { 3892 struct vcpu_sev_es_state *sev_es = &svm->sev_es; 3893 struct psc_buffer *guest_psc = sev_es->ghcb_sa; 3894 struct kvm_vcpu *vcpu = &svm->vcpu; 3895 struct psc_entry entry_start; 3896 int npages; 3897 bool huge; 3898 u64 gfn; 3899 u16 idx; 3900 3901 next_range: 3902 /* There should be no other PSCs in-flight at this point. */ 3903 if (WARN_ON_ONCE(svm->sev_es.psc.batch_size)) { 3904 snp_complete_psc(svm, VMGEXIT_PSC_ERROR_GENERIC); 3905 return 1; 3906 } 3907 3908 /* Find the start of the next range which needs processing. */ 3909 for (idx = sev_es->psc.cur_idx; idx <= sev_es->psc.end_idx; idx++) { 3910 entry_start = READ_ONCE(guest_psc->entries[idx]); 3911 3912 gfn = entry_start.gfn; 3913 huge = entry_start.pagesize; 3914 npages = huge ? 512 : 1; 3915 3916 if (entry_start.cur_page > npages || !IS_ALIGNED(gfn, npages)) { 3917 snp_complete_psc(svm, VMGEXIT_PSC_ERROR_INVALID_ENTRY); 3918 return 1; 3919 } 3920 3921 if (entry_start.cur_page) { 3922 /* 3923 * If this is a partially-completed 2M range, force 4K handling 3924 * for the remaining pages since they're effectively split at 3925 * this point. Subsequent code should ensure this doesn't get 3926 * combined with adjacent PSC entries where 2M handling is still 3927 * possible. 3928 */ 3929 npages -= entry_start.cur_page; 3930 gfn += entry_start.cur_page; 3931 huge = false; 3932 } 3933 3934 if (npages) 3935 break; 3936 3937 /* 3938 * Increment the guest-visible index to communicate the current 3939 * entry back to the guest, e.g. in case of failure. No need 3940 * for READ_ONCE() as KVM doesn't consume the field, i.e. a 3941 * misbehaving guest can only break itself. 3942 */ 3943 guest_psc->hdr.cur_entry++; 3944 } 3945 3946 if (idx > sev_es->psc.end_idx) { 3947 /* Nothing more to process. */ 3948 snp_complete_psc(svm, 0); 3949 return 1; 3950 } 3951 3952 sev_es->psc.is_2m = huge; 3953 sev_es->psc.cur_idx = idx; 3954 sev_es->psc.batch_size = 1; 3955 3956 /* 3957 * Find all subsequent PSC entries that contain adjacent GPA 3958 * ranges/operations and can be combined into a single 3959 * KVM_HC_MAP_GPA_RANGE exit. 3960 */ 3961 while (++idx <= sev_es->psc.end_idx) { 3962 struct psc_entry entry = READ_ONCE(guest_psc->entries[idx]); 3963 3964 if (entry.operation != entry_start.operation || 3965 entry.gfn != entry_start.gfn + npages || 3966 entry.cur_page || !!entry.pagesize != huge) 3967 break; 3968 3969 sev_es->psc.batch_size++; 3970 npages += huge ? 512 : 1; 3971 } 3972 3973 switch (entry_start.operation) { 3974 case VMGEXIT_PSC_OP_PRIVATE: 3975 case VMGEXIT_PSC_OP_SHARED: 3976 vcpu->run->exit_reason = KVM_EXIT_HYPERCALL; 3977 vcpu->run->hypercall.nr = KVM_HC_MAP_GPA_RANGE; 3978 /* 3979 * In principle this should have been -KVM_ENOSYS, but userspace (QEMU <=9.2) 3980 * assumed that vcpu->run->hypercall.ret is never changed by KVM and thus that 3981 * it was always zero on KVM_EXIT_HYPERCALL. Since KVM is now overwriting 3982 * vcpu->run->hypercall.ret, ensuring that it is zero to not break QEMU. 3983 */ 3984 vcpu->run->hypercall.ret = 0; 3985 vcpu->run->hypercall.args[0] = gfn_to_gpa(gfn); 3986 vcpu->run->hypercall.args[1] = npages; 3987 vcpu->run->hypercall.args[2] = entry_start.operation == VMGEXIT_PSC_OP_PRIVATE 3988 ? KVM_MAP_GPA_RANGE_ENCRYPTED 3989 : KVM_MAP_GPA_RANGE_DECRYPTED; 3990 vcpu->run->hypercall.args[2] |= entry_start.pagesize 3991 ? KVM_MAP_GPA_RANGE_PAGE_SZ_2M 3992 : KVM_MAP_GPA_RANGE_PAGE_SZ_4K; 3993 vcpu->arch.complete_userspace_io = snp_complete_one_psc; 3994 return 0; /* forward request to userspace */ 3995 default: 3996 /* 3997 * Only shared/private PSC operations are currently supported, so if the 3998 * entire range consists of unsupported operations (e.g. SMASH/UNSMASH), 3999 * then consider the entire range completed and avoid exiting to 4000 * userspace. In theory snp_complete_psc() can always be called directly 4001 * at this point to complete the current range and start the next one, 4002 * but that could lead to unexpected levels of recursion. 4003 */ 4004 __snp_complete_one_psc(svm); 4005 goto next_range; 4006 } 4007 4008 BUG(); 4009 } 4010 4011 static int snp_begin_psc(struct vcpu_svm *svm) 4012 { 4013 struct vcpu_sev_es_state *sev_es = &svm->sev_es; 4014 struct psc_buffer *guest_psc = sev_es->ghcb_sa; 4015 u16 max_nr_entries; 4016 4017 if (!user_exit_on_hypercall(svm->vcpu.kvm, KVM_HC_MAP_GPA_RANGE)) { 4018 snp_complete_psc(svm, VMGEXIT_PSC_ERROR_GENERIC); 4019 return 1; 4020 } 4021 4022 /* 4023 * GHCB v2 requires the scratch area to reside within the GHCB itself, 4024 * and PSC requests are only supported for GHCB v2+. Thus it should be 4025 * impossible to exceed the max PSC entry count (which is derived from 4026 * the size of the shared GHCB buffer). 4027 */ 4028 max_nr_entries = (sev_es->ghcb_sa_len - sizeof(struct psc_hdr)) / 4029 sizeof(struct psc_entry); 4030 if (WARN_ON_ONCE(max_nr_entries > VMGEXIT_PSC_MAX_COUNT)) { 4031 snp_complete_psc(svm, VMGEXIT_PSC_ERROR_GENERIC); 4032 return 1; 4033 } 4034 4035 /* 4036 * The PSC descriptor buffer can be modified by a misbehaved guest after 4037 * validation, so take care to only use validated copies of values used 4038 * for things like array indexing. 4039 */ 4040 sev_es->psc.cur_idx = READ_ONCE(guest_psc->hdr.cur_entry); 4041 sev_es->psc.end_idx = READ_ONCE(guest_psc->hdr.end_entry); 4042 4043 if (sev_es->psc.end_idx >= max_nr_entries) { 4044 snp_complete_psc(svm, VMGEXIT_PSC_ERROR_INVALID_HDR); 4045 return 1; 4046 } 4047 4048 return snp_do_psc(svm); 4049 } 4050 4051 static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) 4052 { 4053 struct vcpu_svm *svm = to_svm(vcpu); 4054 struct kvm_memory_slot *slot; 4055 struct kvm *kvm = vcpu->kvm; 4056 gfn_t gfn = gpa_to_gfn(gpa); 4057 unsigned long mmu_seq; 4058 kvm_pfn_t pfn; 4059 4060 lockdep_assert_held(&svm->sev_es.snp_vmsa_mutex); 4061 4062 /* 4063 * Clear use of the VMSA. Ensure snp_guest_vmsa_gpa is written exactly 4064 * once, as it is read locklessly when responding to gfn invalidations. 4065 * Pairs with the READ_ONCE() in sev_gmem_invalidate_range(). 4066 */ 4067 svm->vmcb->control.vmsa_pa = INVALID_PAGE; 4068 WRITE_ONCE(svm->sev_es.snp_guest_vmsa_gpa, INVALID_PAGE); 4069 4070 /* 4071 * When replacing the VMSA during SEV-SNP AP creation, 4072 * mark the VMCB dirty so that full state is always reloaded. 4073 */ 4074 vmcb_mark_all_dirty(svm->vmcb); 4075 4076 /* 4077 * From this point forward, the VMSA will always be a guest-mapped page 4078 * rather than the initial one allocated by KVM in svm->sev_es.vmsa. In 4079 * theory, svm->sev_es.vmsa could be free'd and cleaned up here, but 4080 * that involves cleanups like flushing caches, which would ideally be 4081 * handled during teardown rather than guest boot. Deferring that also 4082 * allows the existing logic for SEV-ES VMSAs to be re-used with 4083 * minimal SNP-specific changes. 4084 */ 4085 svm->sev_es.snp_has_guest_vmsa = true; 4086 4087 if (!VALID_PAGE(gpa)) 4088 return; 4089 4090 slot = gfn_to_memslot(vcpu->kvm, gfn); 4091 if (!slot) 4092 return; 4093 4094 mmu_seq = kvm->mmu_invalidate_seq; 4095 smp_rmb(); 4096 4097 /* 4098 * The new VMSA will be private memory guest memory, so retrieve the 4099 * PFN from the gmem backend. 4100 */ 4101 if (kvm_gmem_get_pfn(vcpu->kvm, slot, gfn, &pfn, NULL)) 4102 return; 4103 4104 read_lock(&kvm->mmu_lock); 4105 /* 4106 * Save the guest-provided GPA. If retry is needed, then KVM will try 4107 * again with the same GPA. If the VMSA is usable, then KVM needs to 4108 * track the GPA so that the VMSA can be reloaded if the backing page 4109 * for the GPA is invalidated. 4110 */ 4111 svm->sev_es.snp_guest_vmsa_gpa = gpa; 4112 if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) 4113 kvm_make_request(KVM_REQ_VMSA_PAGE_RELOAD, vcpu); 4114 else 4115 svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn); 4116 read_unlock(&kvm->mmu_lock); 4117 } 4118 4119 /* 4120 * Invoked as part of svm_vcpu_reset() processing of an init event. 4121 */ 4122 static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) 4123 { 4124 struct vcpu_svm *svm = to_svm(vcpu); 4125 gpa_t gpa; 4126 4127 guard(mutex)(&svm->sev_es.snp_vmsa_mutex); 4128 4129 if (!svm->sev_es.snp_ap_waiting_for_reset) 4130 return; 4131 4132 svm->sev_es.snp_ap_waiting_for_reset = false; 4133 4134 /* Mark the vCPU as offline and not runnable */ 4135 vcpu->arch.pv.pv_unhalted = false; 4136 kvm_set_mp_state(vcpu, KVM_MP_STATE_HALTED); 4137 4138 gpa = svm->sev_es.snp_pending_vmsa_gpa; 4139 svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; 4140 4141 __sev_snp_reload_vmsa(vcpu, gpa); 4142 4143 /* 4144 * Mark the vCPU as runnable for CREATE requests, indicated by a valid 4145 * VMSA GPA, even if installing the VMSA failed, so that KVM_RUN will 4146 * fail instead of blocking indefinitely and hanging the vCPU, e.g. if 4147 * the backing guest_memfd page is unavailable. 4148 */ 4149 if (VALID_PAGE(gpa)) 4150 kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); 4151 } 4152 4153 void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu) 4154 { 4155 struct vcpu_sev_es_state *sev_es = &to_svm(vcpu)->sev_es; 4156 4157 guard(mutex)(&sev_es->snp_vmsa_mutex); 4158 4159 __sev_snp_reload_vmsa(vcpu, sev_es->snp_guest_vmsa_gpa); 4160 } 4161 4162 static int sev_snp_ap_creation(struct vcpu_svm *svm) 4163 { 4164 struct kvm_sev_info *sev = to_kvm_sev_info(svm->vcpu.kvm); 4165 struct kvm_vcpu *vcpu = &svm->vcpu; 4166 struct kvm_vcpu *target_vcpu; 4167 struct vcpu_svm *target_svm; 4168 unsigned int request; 4169 unsigned int apic_id; 4170 4171 request = lower_32_bits(svm->vmcb->control.exit_info_1); 4172 apic_id = upper_32_bits(svm->vmcb->control.exit_info_1); 4173 4174 /* Validate the APIC ID */ 4175 target_vcpu = kvm_get_vcpu_by_id(vcpu->kvm, apic_id); 4176 if (!target_vcpu) { 4177 vcpu_unimpl(vcpu, "vmgexit: invalid AP APIC ID [%#x] from guest\n", 4178 apic_id); 4179 return -EINVAL; 4180 } 4181 4182 target_svm = to_svm(target_vcpu); 4183 4184 guard(mutex)(&target_svm->sev_es.snp_vmsa_mutex); 4185 4186 switch (request) { 4187 case SVM_VMGEXIT_AP_CREATE_ON_INIT: 4188 case SVM_VMGEXIT_AP_CREATE: 4189 if (vcpu->arch.regs[VCPU_REGS_RAX] != sev->vmsa_features) { 4190 vcpu_unimpl(vcpu, "vmgexit: mismatched AP sev_features [%#lx] != [%#llx] from guest\n", 4191 vcpu->arch.regs[VCPU_REGS_RAX], sev->vmsa_features); 4192 return -EINVAL; 4193 } 4194 4195 if (!page_address_valid(vcpu, svm->vmcb->control.exit_info_2)) { 4196 vcpu_unimpl(vcpu, "vmgexit: invalid AP VMSA address [%#llx] from guest\n", 4197 svm->vmcb->control.exit_info_2); 4198 return -EINVAL; 4199 } 4200 4201 /* 4202 * Malicious guest can RMPADJUST a large page into VMSA which 4203 * will hit the SNP erratum where the CPU will incorrectly signal 4204 * an RMP violation #PF if a hugepage collides with the RMP entry 4205 * of VMSA page, reject the AP CREATE request if VMSA address from 4206 * guest is 2M aligned. 4207 */ 4208 if (IS_ALIGNED(svm->vmcb->control.exit_info_2, PMD_SIZE)) { 4209 vcpu_unimpl(vcpu, 4210 "vmgexit: AP VMSA address [%llx] from guest is unsafe as it is 2M aligned\n", 4211 svm->vmcb->control.exit_info_2); 4212 return -EINVAL; 4213 } 4214 4215 target_svm->sev_es.snp_pending_vmsa_gpa = svm->vmcb->control.exit_info_2; 4216 break; 4217 case SVM_VMGEXIT_AP_DESTROY: 4218 target_svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; 4219 break; 4220 default: 4221 vcpu_unimpl(vcpu, "vmgexit: invalid AP creation request [%#x] from guest\n", 4222 request); 4223 return -EINVAL; 4224 } 4225 4226 target_svm->sev_es.snp_ap_waiting_for_reset = true; 4227 4228 /* 4229 * Unless Creation is deferred until INIT, signal the vCPU to update 4230 * its state. 4231 */ 4232 if (request != SVM_VMGEXIT_AP_CREATE_ON_INIT) 4233 kvm_make_request_and_kick(KVM_REQ_UPDATE_PROTECTED_GUEST_STATE, target_vcpu); 4234 4235 return 0; 4236 } 4237 4238 static int snp_handle_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t resp_gpa) 4239 { 4240 struct sev_data_snp_guest_request data = {0}; 4241 struct kvm *kvm = svm->vcpu.kvm; 4242 struct kvm_sev_info *sev = to_kvm_sev_info(kvm); 4243 sev_ret_code fw_err = 0; 4244 u8 tickle = 0; 4245 int ret; 4246 4247 if (!is_sev_snp_guest(&svm->vcpu)) 4248 return -EINVAL; 4249 4250 guard(mutex)(&sev->guest_req_mutex); 4251 4252 if (kvm_read_guest(kvm, req_gpa, sev->guest_req_buf, PAGE_SIZE)) { 4253 svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT); 4254 return 1; 4255 } 4256 4257 /* 4258 * "Tickle" the response buffer to verify it's writable before sending 4259 * the request to firmware, which will modify the VMPCK sequence number. 4260 * Note, this is a best effort check and doesn't guard against TOCTOU 4261 * issues. See below for more information. 4262 */ 4263 if (kvm_write_guest(kvm, resp_gpa, &tickle, sizeof(tickle))) { 4264 svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT); 4265 return 1; 4266 } 4267 4268 data.gctx_paddr = __psp_pa(sev->snp_context); 4269 data.req_paddr = __psp_pa(sev->guest_req_buf); 4270 data.res_paddr = __psp_pa(sev->guest_resp_buf); 4271 4272 /* 4273 * Firmware failures are propagated on to guest, but any other failure 4274 * condition along the way should be reported to userspace. E.g. if 4275 * the PSP is dead and commands are timing out. 4276 */ 4277 ret = sev_issue_cmd(kvm, SEV_CMD_SNP_GUEST_REQUEST, &data, &fw_err); 4278 if (ret && !fw_err) 4279 return ret; 4280 4281 /* 4282 * Exit to userspace if writing the response fails, e.g. if the mapping 4283 * changed between the initial tickle and the actual write, to avoid 4284 * creating an ambiguous failure ABI with the guest. Don't return an 4285 * error to the guest because firmware already incremented the VMPCK 4286 * sequence number, but there's no way to communicate that to the guest. 4287 */ 4288 if (kvm_write_guest(kvm, resp_gpa, sev->guest_resp_buf, PAGE_SIZE)) 4289 return -EIO; 4290 4291 /* No action is requested *from KVM* if there was a firmware error. */ 4292 svm_vmgexit_no_action(svm, SNP_GUEST_ERR(0, fw_err)); 4293 4294 /* resume guest */ 4295 return 1; 4296 } 4297 4298 static int snp_req_certs_err(struct vcpu_svm *svm, u32 vmm_error) 4299 { 4300 ghcb_set_sw_exit_info_2(svm->sev_es.ghcb, SNP_GUEST_ERR(vmm_error, 0)); 4301 4302 return 1; /* resume guest */ 4303 } 4304 4305 static int snp_complete_req_certs(struct kvm_vcpu *vcpu) 4306 { 4307 struct vcpu_svm *svm = to_svm(vcpu); 4308 struct vmcb_control_area *control = &svm->vmcb->control; 4309 4310 switch (READ_ONCE(vcpu->run->snp_req_certs.ret)) { 4311 case 0: 4312 return snp_handle_guest_req(svm, control->exit_info_1, 4313 control->exit_info_2); 4314 case ENOSPC: 4315 vcpu->arch.regs[VCPU_REGS_RBX] = vcpu->run->snp_req_certs.npages; 4316 return snp_req_certs_err(svm, SNP_GUEST_VMM_ERR_INVALID_LEN); 4317 case EAGAIN: 4318 return snp_req_certs_err(svm, SNP_GUEST_VMM_ERR_BUSY); 4319 case EIO: 4320 return snp_req_certs_err(svm, SNP_GUEST_VMM_ERR_GENERIC); 4321 default: 4322 break; 4323 } 4324 4325 return -EINVAL; 4326 } 4327 4328 static int snp_handle_ext_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t resp_gpa) 4329 { 4330 struct kvm_vcpu *vcpu = &svm->vcpu; 4331 struct kvm *kvm = vcpu->kvm; 4332 4333 u8 msg_type; 4334 4335 if (!is_sev_snp_guest(vcpu)) 4336 return -EINVAL; 4337 4338 if (kvm_read_guest(kvm, req_gpa + offsetof(struct snp_guest_msg_hdr, msg_type), 4339 &msg_type, 1)) { 4340 svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT); 4341 return 1; 4342 } 4343 4344 /* 4345 * As per GHCB spec, requests of type MSG_REPORT_REQ also allow for 4346 * additional certificate data to be provided alongside the attestation 4347 * report via the guest-provided data pages indicated by RAX/RBX. If 4348 * userspace enables KVM_EXIT_SNP_REQ_CERTS, then exit to userspace 4349 * to give userspace an opportunity to provide the certificate data 4350 * before issuing/completing the attestation request. Otherwise, return 4351 * an empty certificate table in the guest-provided data pages and 4352 * handle the attestation request immediately. 4353 */ 4354 if (msg_type == SNP_MSG_REPORT_REQ) { 4355 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 4356 u64 data_npages; 4357 gpa_t data_gpa; 4358 4359 if (!kvm_ghcb_rax_is_valid(svm) || !kvm_ghcb_rbx_is_valid(svm)) 4360 goto request_invalid; 4361 4362 data_gpa = vcpu->arch.regs[VCPU_REGS_RAX]; 4363 data_npages = vcpu->arch.regs[VCPU_REGS_RBX]; 4364 4365 if (!PAGE_ALIGNED(data_gpa)) 4366 goto request_invalid; 4367 4368 if (sev->snp_certs_enabled) { 4369 vcpu->run->exit_reason = KVM_EXIT_SNP_REQ_CERTS; 4370 vcpu->run->snp_req_certs.gpa = data_gpa; 4371 vcpu->run->snp_req_certs.npages = data_npages; 4372 vcpu->run->snp_req_certs.ret = 0; 4373 vcpu->arch.complete_userspace_io = snp_complete_req_certs; 4374 return 0; 4375 } 4376 4377 /* 4378 * As per GHCB spec (see "SNP Extended Guest Request"), the 4379 * certificate table is terminated by 24-bytes of zeroes. 4380 */ 4381 if (data_npages && kvm_clear_guest(kvm, data_gpa, 24)) { 4382 svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT); 4383 return 1; 4384 } 4385 } 4386 4387 return snp_handle_guest_req(svm, req_gpa, resp_gpa); 4388 4389 request_invalid: 4390 svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT); 4391 return 1; /* resume guest */ 4392 } 4393 4394 static int sev_handle_vmgexit_msr_protocol(struct vcpu_svm *svm) 4395 { 4396 struct vmcb_control_area *control = &svm->vmcb->control; 4397 struct kvm_vcpu *vcpu = &svm->vcpu; 4398 struct kvm_sev_info *sev = to_kvm_sev_info(vcpu->kvm); 4399 u64 ghcb_info; 4400 int ret = 1; 4401 4402 ghcb_info = control->ghcb_gpa & GHCB_MSR_INFO_MASK; 4403 4404 trace_kvm_vmgexit_msr_protocol_enter(svm->vcpu.vcpu_id, 4405 control->ghcb_gpa); 4406 4407 switch (ghcb_info) { 4408 case GHCB_MSR_SEV_INFO_REQ: 4409 set_ghcb_msr(svm, GHCB_MSR_SEV_INFO((__u64)sev->ghcb_version, 4410 GHCB_VERSION_MIN, 4411 sev_enc_bit)); 4412 break; 4413 case GHCB_MSR_CPUID_REQ: { 4414 u64 cpuid_fn, cpuid_reg, cpuid_value; 4415 4416 cpuid_fn = get_ghcb_msr_bits(svm, 4417 GHCB_MSR_CPUID_FUNC_MASK, 4418 GHCB_MSR_CPUID_FUNC_POS); 4419 4420 /* Initialize the registers needed by the CPUID intercept */ 4421 vcpu->arch.regs[VCPU_REGS_RAX] = cpuid_fn; 4422 vcpu->arch.regs[VCPU_REGS_RCX] = 0; 4423 4424 ret = svm_invoke_exit_handler(vcpu, SVM_EXIT_CPUID); 4425 if (!ret) { 4426 /* Error, keep GHCB MSR value as-is */ 4427 break; 4428 } 4429 4430 cpuid_reg = get_ghcb_msr_bits(svm, 4431 GHCB_MSR_CPUID_REG_MASK, 4432 GHCB_MSR_CPUID_REG_POS); 4433 if (cpuid_reg == 0) 4434 cpuid_value = vcpu->arch.regs[VCPU_REGS_RAX]; 4435 else if (cpuid_reg == 1) 4436 cpuid_value = vcpu->arch.regs[VCPU_REGS_RBX]; 4437 else if (cpuid_reg == 2) 4438 cpuid_value = vcpu->arch.regs[VCPU_REGS_RCX]; 4439 else 4440 cpuid_value = vcpu->arch.regs[VCPU_REGS_RDX]; 4441 4442 set_ghcb_msr_bits(svm, cpuid_value, 4443 GHCB_MSR_CPUID_VALUE_MASK, 4444 GHCB_MSR_CPUID_VALUE_POS); 4445 4446 set_ghcb_msr_bits(svm, GHCB_MSR_CPUID_RESP, 4447 GHCB_MSR_INFO_MASK, 4448 GHCB_MSR_INFO_POS); 4449 break; 4450 } 4451 case GHCB_MSR_AP_RESET_HOLD_REQ: 4452 svm->sev_es.ap_reset_hold_type = AP_RESET_HOLD_MSR_PROTO; 4453 ret = kvm_emulate_ap_reset_hold(&svm->vcpu); 4454 4455 /* 4456 * Preset the result to a non-SIPI return and then only set 4457 * the result to non-zero when delivering a SIPI. 4458 */ 4459 set_ghcb_msr_bits(svm, 0, 4460 GHCB_MSR_AP_RESET_HOLD_RESULT_MASK, 4461 GHCB_MSR_AP_RESET_HOLD_RESULT_POS); 4462 4463 set_ghcb_msr_bits(svm, GHCB_MSR_AP_RESET_HOLD_RESP, 4464 GHCB_MSR_INFO_MASK, 4465 GHCB_MSR_INFO_POS); 4466 break; 4467 case GHCB_MSR_HV_FT_REQ: 4468 set_ghcb_msr_bits(svm, GHCB_HV_FT_SUPPORTED, 4469 GHCB_MSR_HV_FT_MASK, GHCB_MSR_HV_FT_POS); 4470 set_ghcb_msr_bits(svm, GHCB_MSR_HV_FT_RESP, 4471 GHCB_MSR_INFO_MASK, GHCB_MSR_INFO_POS); 4472 break; 4473 case GHCB_MSR_PREF_GPA_REQ: 4474 if (!is_sev_snp_guest(vcpu)) 4475 goto out_terminate; 4476 4477 set_ghcb_msr_bits(svm, GHCB_MSR_PREF_GPA_NONE, GHCB_MSR_GPA_VALUE_MASK, 4478 GHCB_MSR_GPA_VALUE_POS); 4479 set_ghcb_msr_bits(svm, GHCB_MSR_PREF_GPA_RESP, GHCB_MSR_INFO_MASK, 4480 GHCB_MSR_INFO_POS); 4481 break; 4482 case GHCB_MSR_REG_GPA_REQ: { 4483 u64 gfn; 4484 4485 if (!is_sev_snp_guest(vcpu)) 4486 goto out_terminate; 4487 4488 gfn = get_ghcb_msr_bits(svm, GHCB_MSR_GPA_VALUE_MASK, 4489 GHCB_MSR_GPA_VALUE_POS); 4490 4491 svm->sev_es.ghcb_registered_gpa = gfn_to_gpa(gfn); 4492 4493 set_ghcb_msr_bits(svm, gfn, GHCB_MSR_GPA_VALUE_MASK, 4494 GHCB_MSR_GPA_VALUE_POS); 4495 set_ghcb_msr_bits(svm, GHCB_MSR_REG_GPA_RESP, GHCB_MSR_INFO_MASK, 4496 GHCB_MSR_INFO_POS); 4497 break; 4498 } 4499 case GHCB_MSR_PSC_REQ: 4500 if (!is_sev_snp_guest(vcpu)) 4501 goto out_terminate; 4502 4503 ret = snp_begin_psc_msr(svm, control->ghcb_gpa); 4504 break; 4505 case GHCB_MSR_TERM_REQ: { 4506 u64 reason_set, reason_code; 4507 4508 reason_set = get_ghcb_msr_bits(svm, 4509 GHCB_MSR_TERM_REASON_SET_MASK, 4510 GHCB_MSR_TERM_REASON_SET_POS); 4511 reason_code = get_ghcb_msr_bits(svm, 4512 GHCB_MSR_TERM_REASON_MASK, 4513 GHCB_MSR_TERM_REASON_POS); 4514 pr_info("SEV-ES guest requested termination: %#llx:%#llx\n", 4515 reason_set, reason_code); 4516 4517 goto out_terminate; 4518 } 4519 default: 4520 /* Error, keep GHCB MSR value as-is */ 4521 break; 4522 } 4523 4524 trace_kvm_vmgexit_msr_protocol_exit(svm->vcpu.vcpu_id, 4525 control->ghcb_gpa, ret); 4526 4527 return ret; 4528 4529 out_terminate: 4530 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; 4531 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_SEV_TERM; 4532 vcpu->run->system_event.ndata = 1; 4533 vcpu->run->system_event.data[0] = control->ghcb_gpa; 4534 4535 return 0; 4536 } 4537 4538 static bool is_snp_only_vmgexit(u64 exit_code) 4539 { 4540 switch (exit_code) { 4541 case SVM_VMGEXIT_AP_CREATION: 4542 case SVM_VMGEXIT_GUEST_REQUEST: 4543 case SVM_VMGEXIT_EXT_GUEST_REQUEST: 4544 case SVM_VMGEXIT_PSC: 4545 return true; 4546 default: 4547 return false; 4548 } 4549 } 4550 4551 int sev_handle_vmgexit(struct kvm_vcpu *vcpu) 4552 { 4553 struct vcpu_svm *svm = to_svm(vcpu); 4554 struct vmcb_control_area *control = &svm->vmcb->control; 4555 u64 ghcb_gpa; 4556 4557 /* Validate the GHCB */ 4558 ghcb_gpa = control->ghcb_gpa; 4559 if (ghcb_gpa & GHCB_MSR_INFO_MASK) 4560 return sev_handle_vmgexit_msr_protocol(svm); 4561 4562 if (!ghcb_gpa) { 4563 vcpu_unimpl(vcpu, "vmgexit: GHCB gpa is not set\n"); 4564 4565 /* Without a GHCB, just return right back to the guest */ 4566 return 1; 4567 } 4568 4569 if (kvm_vcpu_map(vcpu, ghcb_gpa >> PAGE_SHIFT, &svm->sev_es.ghcb_map)) { 4570 /* Unable to map GHCB from guest */ 4571 vcpu_unimpl(vcpu, "vmgexit: error mapping GHCB [%#llx] from guest\n", 4572 ghcb_gpa); 4573 4574 /* Without a GHCB, just return right back to the guest */ 4575 return 1; 4576 } 4577 4578 svm->sev_es.ghcb = svm->sev_es.ghcb_map.hva; 4579 4580 trace_kvm_vmgexit_enter(vcpu->vcpu_id, svm->sev_es.ghcb); 4581 4582 sev_es_sync_from_ghcb(svm); 4583 4584 /* SEV-SNP guest requires that the GHCB GPA must be registered */ 4585 if (is_sev_snp_guest(vcpu) && 4586 !ghcb_gpa_is_registered(svm, control->ghcb_gpa)) { 4587 vcpu_unimpl(vcpu, "vmgexit: GHCB GPA [%#llx] is not registered.\n", 4588 control->ghcb_gpa); 4589 svm_vmgexit_bad_input(svm, GHCB_ERR_NOT_REGISTERED); 4590 return 1; 4591 } 4592 4593 /* Only GHCB Usage code 0 is supported */ 4594 if (svm->sev_es.ghcb->ghcb_usage) { 4595 vcpu_unimpl(vcpu, "vmgexit: ghcb usage %#x is not valid\n", 4596 svm->sev_es.ghcb->ghcb_usage); 4597 svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_USAGE); 4598 return 1; 4599 } 4600 4601 if (is_snp_only_vmgexit(control->exit_code) && !is_sev_snp_guest(vcpu)) { 4602 vcpu_unimpl(vcpu, "vmgexit: exit code %#llx is SNP-only\n", 4603 control->exit_code); 4604 svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_EVENT); 4605 return 1; 4606 } 4607 4608 if (!sev_es_are_required_ghcb_fields_valid(svm)) { 4609 /* 4610 * Print the exit code even though it may not be marked valid 4611 * as it could help with debugging. 4612 */ 4613 vcpu_unimpl(vcpu, "vmgexit: exit code %#llx input is not valid\n", 4614 control->exit_code); 4615 dump_ghcb(svm); 4616 svm_vmgexit_bad_input(svm, GHCB_ERR_MISSING_INPUT); 4617 return 1; 4618 } 4619 4620 svm_vmgexit_success(svm, 0); 4621 4622 switch (control->exit_code) { 4623 case SVM_EXIT_IOIO: 4624 if (!((control->exit_info_1 & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT)) 4625 return 1; 4626 4627 fallthrough; 4628 case SVM_EXIT_READ_DR7: 4629 case SVM_EXIT_WRITE_DR7: 4630 case SVM_EXIT_RDTSC: 4631 case SVM_EXIT_RDTSCP: 4632 case SVM_EXIT_RDPMC: 4633 case SVM_EXIT_CPUID: 4634 case SVM_EXIT_INVD: 4635 case SVM_EXIT_MSR: 4636 case SVM_EXIT_VMMCALL: 4637 case SVM_EXIT_WBINVD: 4638 case SVM_EXIT_MONITOR: 4639 case SVM_EXIT_MWAIT: 4640 return svm_invoke_exit_handler(vcpu, control->exit_code); 4641 case SVM_VMGEXIT_MMIO_READ: 4642 case SVM_VMGEXIT_MMIO_WRITE: { 4643 bool is_write = control->exit_code == SVM_VMGEXIT_MMIO_WRITE; 4644 u64 len = control->exit_info_2; 4645 int r; 4646 4647 if (!len) 4648 return 1; 4649 4650 if (to_kvm_sev_info(vcpu->kvm)->ghcb_version >= 2 && len > 8) { 4651 svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT); 4652 return 1; 4653 } 4654 4655 r = setup_vmgexit_scratch(svm, !is_write, len); 4656 if (r) 4657 return r; 4658 4659 return kvm_sev_es_mmio(vcpu, is_write, control->exit_info_1, len, 4660 svm->sev_es.ghcb_sa); 4661 } 4662 case SVM_VMGEXIT_NMI_COMPLETE: 4663 ++vcpu->stat.nmi_window_exits; 4664 svm->nmi_masked = false; 4665 kvm_make_request(KVM_REQ_EVENT, vcpu); 4666 return 1; 4667 case SVM_VMGEXIT_AP_HLT_LOOP: 4668 svm->sev_es.ap_reset_hold_type = AP_RESET_HOLD_NAE_EVENT; 4669 return kvm_emulate_ap_reset_hold(vcpu); 4670 case SVM_VMGEXIT_AP_JUMP_TABLE: { 4671 struct kvm_sev_info *sev = to_kvm_sev_info(vcpu->kvm); 4672 4673 switch (control->exit_info_1) { 4674 case 0: 4675 /* Set AP jump table address */ 4676 sev->ap_jump_table = control->exit_info_2; 4677 break; 4678 case 1: 4679 /* Get AP jump table address */ 4680 svm_vmgexit_success(svm, sev->ap_jump_table); 4681 break; 4682 default: 4683 pr_err("svm: vmgexit: unsupported AP jump table request - exit_info_1=%#llx\n", 4684 control->exit_info_1); 4685 svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT); 4686 } 4687 return 1; 4688 } 4689 case SVM_VMGEXIT_HV_FEATURES: 4690 svm_vmgexit_success(svm, GHCB_HV_FT_SUPPORTED); 4691 return 1; 4692 case SVM_VMGEXIT_TERM_REQUEST: 4693 pr_info("SEV-ES guest requested termination: reason %#llx info %#llx\n", 4694 control->exit_info_1, control->exit_info_2); 4695 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; 4696 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_SEV_TERM; 4697 vcpu->run->system_event.ndata = 1; 4698 vcpu->run->system_event.data[0] = control->ghcb_gpa; 4699 return 0; 4700 case SVM_VMGEXIT_PSC: { 4701 int r; 4702 4703 r = setup_vmgexit_scratch(svm, true, sizeof(struct psc_hdr)); 4704 if (r) 4705 return r; 4706 4707 return snp_begin_psc(svm); 4708 } 4709 case SVM_VMGEXIT_AP_CREATION: 4710 if (sev_snp_ap_creation(svm)) 4711 svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT); 4712 return 1; 4713 case SVM_VMGEXIT_GUEST_REQUEST: 4714 case SVM_VMGEXIT_EXT_GUEST_REQUEST: 4715 if (!PAGE_ALIGNED(control->exit_info_1) || 4716 !PAGE_ALIGNED(control->exit_info_2) || 4717 control->exit_info_1 == control->exit_info_2) { 4718 svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT); 4719 return 1; 4720 } 4721 4722 if (control->exit_code == SVM_VMGEXIT_GUEST_REQUEST) 4723 return snp_handle_guest_req(svm, control->exit_info_1, 4724 control->exit_info_2); 4725 4726 return snp_handle_ext_guest_req(svm, control->exit_info_1, 4727 control->exit_info_2); 4728 case SVM_VMGEXIT_UNSUPPORTED_EVENT: 4729 /* 4730 * Note, the _guest_ is reporting an unsupported #VC, i.e. this 4731 * isn't the same thing as KVM getting an unsupported #VMGEXIT. 4732 */ 4733 vcpu_unimpl(vcpu, 4734 "vmgexit: unsupported event - exit_info_1=%#llx, exit_info_2=%#llx\n", 4735 control->exit_info_1, control->exit_info_2); 4736 return -EINVAL; 4737 default: 4738 vcpu_unimpl(vcpu, "vmgexit: exit code %#llx is not valid\n", 4739 control->exit_code); 4740 svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_EVENT); 4741 return 1; 4742 } 4743 4744 KVM_BUG_ON(1, vcpu->kvm); 4745 return -EIO; 4746 } 4747 4748 int sev_es_string_io(struct vcpu_svm *svm, int size, unsigned int port, int in) 4749 { 4750 int count; 4751 int bytes; 4752 int r; 4753 4754 if (svm->vmcb->control.exit_info_2 > INT_MAX) 4755 return -EINVAL; 4756 4757 count = svm->vmcb->control.exit_info_2; 4758 if (unlikely(check_mul_overflow(count, size, &bytes))) 4759 return -EINVAL; 4760 4761 if (!bytes) 4762 return 1; 4763 4764 r = setup_vmgexit_scratch(svm, in, bytes); 4765 if (r) 4766 return r; 4767 4768 return kvm_sev_es_string_io(&svm->vcpu, size, port, svm->sev_es.ghcb_sa, 4769 count, in); 4770 } 4771 4772 void sev_es_recalc_msr_intercepts(struct kvm_vcpu *vcpu) 4773 { 4774 /* Clear intercepts on MSRs that are context switched by hardware. */ 4775 svm_disable_intercept_for_msr(vcpu, MSR_AMD64_SEV_ES_GHCB, MSR_TYPE_RW); 4776 svm_disable_intercept_for_msr(vcpu, MSR_EFER, MSR_TYPE_RW); 4777 svm_disable_intercept_for_msr(vcpu, MSR_IA32_CR_PAT, MSR_TYPE_RW); 4778 4779 if (boot_cpu_has(X86_FEATURE_V_TSC_AUX)) 4780 svm_set_intercept_for_msr(vcpu, MSR_TSC_AUX, MSR_TYPE_RW, 4781 !guest_cpu_cap_has(vcpu, X86_FEATURE_RDTSCP) && 4782 !guest_cpu_cap_has(vcpu, X86_FEATURE_RDPID)); 4783 4784 svm_set_intercept_for_msr(vcpu, MSR_AMD64_GUEST_TSC_FREQ, MSR_TYPE_R, 4785 !snp_is_secure_tsc_enabled(vcpu->kvm)); 4786 4787 /* 4788 * For SEV-ES, accesses to MSR_IA32_XSS should not be intercepted if 4789 * the host/guest supports its use. 4790 * 4791 * KVM treats the guest as being capable of using XSAVES even if XSAVES 4792 * isn't enabled in guest CPUID as there is no intercept for XSAVES, 4793 * i.e. the guest can use XSAVES/XRSTOR to read/write XSS if XSAVE is 4794 * exposed to the guest and XSAVES is supported in hardware. Condition 4795 * full XSS passthrough on the guest being able to use XSAVES *and* 4796 * XSAVES being exposed to the guest so that KVM can at least honor 4797 * guest CPUID for RDMSR and WRMSR. 4798 */ 4799 svm_set_intercept_for_msr(vcpu, MSR_IA32_XSS, MSR_TYPE_RW, 4800 !guest_cpu_cap_has(vcpu, X86_FEATURE_XSAVES) || 4801 !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES)); 4802 } 4803 4804 void sev_vcpu_after_set_cpuid(struct vcpu_svm *svm) 4805 { 4806 struct kvm_vcpu *vcpu = &svm->vcpu; 4807 struct kvm_cpuid_entry2 *best; 4808 4809 /* For sev guests, the memory encryption bit is not reserved in CR3. */ 4810 best = kvm_find_cpuid_entry(vcpu, 0x8000001F); 4811 if (best) 4812 vcpu->arch.reserved_gpa_bits &= ~(1UL << (best->ebx & 0x3f)); 4813 } 4814 4815 static void sev_es_init_vmcb(struct vcpu_svm *svm, bool init_event) 4816 { 4817 struct kvm_sev_info *sev = to_kvm_sev_info(svm->vcpu.kvm); 4818 struct vmcb *vmcb = svm->vmcb01.ptr; 4819 4820 svm->vmcb->control.misc_ctl |= SVM_MISC_ENABLE_SEV_ES; 4821 4822 /* 4823 * An SEV-ES guest requires a VMSA area that is a separate from the 4824 * VMCB page. Do not include the encryption mask on the VMSA physical 4825 * address since hardware will access it using the guest key. Note, 4826 * the VMSA will be NULL if this vCPU is the destination for intrahost 4827 * migration, and will be copied later. 4828 */ 4829 if (!svm->sev_es.snp_has_guest_vmsa) { 4830 if (svm->sev_es.vmsa) 4831 svm->vmcb->control.vmsa_pa = __pa(svm->sev_es.vmsa); 4832 else 4833 svm->vmcb->control.vmsa_pa = INVALID_PAGE; 4834 } 4835 4836 if (cpu_feature_enabled(X86_FEATURE_ALLOWED_SEV_FEATURES)) 4837 svm->vmcb->control.allowed_sev_features = sev->vmsa_features | 4838 VMCB_ALLOWED_SEV_FEATURES_VALID; 4839 4840 /* Can't intercept CR register access, HV can't modify CR registers */ 4841 svm_clr_intercept(svm, INTERCEPT_CR0_READ); 4842 svm_clr_intercept(svm, INTERCEPT_CR4_READ); 4843 svm_clr_intercept(svm, INTERCEPT_CR8_READ); 4844 svm_clr_intercept(svm, INTERCEPT_CR0_WRITE); 4845 svm_clr_intercept(svm, INTERCEPT_CR4_WRITE); 4846 svm_clr_intercept(svm, INTERCEPT_CR8_WRITE); 4847 4848 svm_clr_intercept(svm, INTERCEPT_SELECTIVE_CR0); 4849 4850 /* Track EFER/CR register changes */ 4851 svm_set_intercept(svm, TRAP_EFER_WRITE); 4852 svm_set_intercept(svm, TRAP_CR0_WRITE); 4853 svm_set_intercept(svm, TRAP_CR4_WRITE); 4854 svm_set_intercept(svm, TRAP_CR8_WRITE); 4855 4856 vmcb->control.intercepts[INTERCEPT_DR] = 0; 4857 if (!sev_vcpu_has_debug_swap(svm)) { 4858 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_READ); 4859 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_WRITE); 4860 svm_mark_intercepts_dirty(svm); 4861 } else { 4862 /* 4863 * Disable #DB intercept iff DebugSwap is enabled. KVM doesn't 4864 * allow debugging SEV-ES guests, and enables DebugSwap iff 4865 * NO_NESTED_DATA_BP is supported, so there's no reason to 4866 * intercept #DB when DebugSwap is enabled. For simplicity 4867 * with respect to guest debug, intercept #DB for other VMs 4868 * even if NO_NESTED_DATA_BP is supported, i.e. even if the 4869 * guest can't DoS the CPU with infinite #DB vectoring. 4870 */ 4871 clr_exception_intercept(svm, DB_VECTOR); 4872 } 4873 4874 /* Can't intercept XSETBV, HV can't modify XCR0 directly */ 4875 svm_clr_intercept(svm, INTERCEPT_XSETBV); 4876 4877 /* 4878 * Set the GHCB MSR value as per the GHCB specification when emulating 4879 * vCPU RESET for an SEV-ES guest. 4880 */ 4881 if (!init_event) 4882 set_ghcb_msr(svm, GHCB_MSR_SEV_INFO((__u64)sev->ghcb_version, 4883 GHCB_VERSION_MIN, 4884 sev_enc_bit)); 4885 } 4886 4887 void sev_init_vmcb(struct vcpu_svm *svm, bool init_event) 4888 { 4889 struct kvm_vcpu *vcpu = &svm->vcpu; 4890 4891 svm->vmcb->control.misc_ctl |= SVM_MISC_ENABLE_SEV; 4892 clr_exception_intercept(svm, UD_VECTOR); 4893 4894 /* 4895 * Don't intercept #GP for SEV guests, e.g. for the VMware backdoor, as 4896 * KVM can't decrypt guest memory to decode the faulting instruction. 4897 */ 4898 clr_exception_intercept(svm, GP_VECTOR); 4899 4900 if (init_event && is_sev_snp_guest(vcpu)) 4901 sev_snp_init_protected_guest_state(vcpu); 4902 4903 if (is_sev_es_guest(vcpu)) 4904 sev_es_init_vmcb(svm, init_event); 4905 } 4906 4907 int sev_vcpu_create(struct kvm_vcpu *vcpu) 4908 { 4909 struct vcpu_svm *svm = to_svm(vcpu); 4910 struct page *vmsa_page; 4911 4912 mutex_init(&svm->sev_es.snp_vmsa_mutex); 4913 4914 if (!is_sev_es_guest(vcpu)) 4915 return 0; 4916 4917 /* 4918 * SEV-ES guests require a separate (from the VMCB) VMSA page used to 4919 * contain the encrypted register state of the guest. 4920 */ 4921 vmsa_page = snp_safe_alloc_page(); 4922 if (!vmsa_page) 4923 return -ENOMEM; 4924 4925 svm->sev_es.vmsa = page_address(vmsa_page); 4926 svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; 4927 svm->sev_es.snp_guest_vmsa_gpa = INVALID_PAGE; 4928 4929 vcpu->arch.guest_tsc_protected = snp_is_secure_tsc_enabled(vcpu->kvm); 4930 4931 return 0; 4932 } 4933 4934 void sev_es_prepare_switch_to_guest(struct vcpu_svm *svm, struct sev_es_save_area *hostsa) 4935 { 4936 /* 4937 * All host state for SEV-ES guests is categorized into three swap types 4938 * based on how it is handled by hardware during a world switch: 4939 * 4940 * A: VMRUN: Host state saved in host save area 4941 * VMEXIT: Host state loaded from host save area 4942 * 4943 * B: VMRUN: Host state _NOT_ saved in host save area 4944 * VMEXIT: Host state loaded from host save area 4945 * 4946 * C: VMRUN: Host state _NOT_ saved in host save area 4947 * VMEXIT: Host state initialized to default(reset) values 4948 * 4949 * Manually save type-B state, i.e. state that is loaded by VMEXIT but 4950 * isn't saved by VMRUN, that isn't already saved by VMSAVE (performed 4951 * by common SVM code). 4952 */ 4953 hostsa->xcr0 = kvm_host.xcr0; 4954 hostsa->pkru = read_pkru(); 4955 hostsa->xss = kvm_host.xss; 4956 4957 /* 4958 * If DebugSwap is enabled, debug registers are loaded but NOT saved by 4959 * the CPU (Type-B). If DebugSwap is disabled/unsupported, the CPU does 4960 * not save or load debug registers. Sadly, KVM can't prevent SNP 4961 * guests from lying about DebugSwap on secondary vCPUs, i.e. the 4962 * SEV_FEATURES provided at "AP Create" isn't guaranteed to match what 4963 * the guest has actually enabled (or not!) in the VMSA. 4964 * 4965 * If DebugSwap is *possible*, save the masks so that they're restored 4966 * if the guest enables DebugSwap. But for the DRs themselves, do NOT 4967 * rely on the CPU to restore the host values; KVM will restore them as 4968 * needed in common code, via hw_breakpoint_restore(). Note, KVM does 4969 * NOT support virtualizing Breakpoint Extensions, i.e. the mask MSRs 4970 * don't need to be restored per se, KVM just needs to ensure they are 4971 * loaded with the correct values *if* the CPU writes the MSRs. 4972 */ 4973 if (sev_vcpu_has_debug_swap(svm) || 4974 (cpu_feature_enabled(X86_FEATURE_DEBUG_SWAP) && 4975 is_sev_snp_guest(&svm->vcpu))) { 4976 hostsa->dr0_addr_mask = amd_get_dr_addr_mask(0); 4977 hostsa->dr1_addr_mask = amd_get_dr_addr_mask(1); 4978 hostsa->dr2_addr_mask = amd_get_dr_addr_mask(2); 4979 hostsa->dr3_addr_mask = amd_get_dr_addr_mask(3); 4980 } 4981 4982 /* 4983 * TSC_AUX is always virtualized for SEV-ES guests when the feature is 4984 * available, i.e. TSC_AUX is loaded on #VMEXIT from the host save area. 4985 * Set the save area to the current hardware value, i.e. the current 4986 * user return value, so that the correct value is restored on #VMEXIT. 4987 */ 4988 if (cpu_feature_enabled(X86_FEATURE_V_TSC_AUX) && 4989 !WARN_ON_ONCE(tsc_aux_uret_slot < 0)) 4990 hostsa->tsc_aux = kvm_get_user_return_msr(tsc_aux_uret_slot); 4991 } 4992 4993 void sev_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector) 4994 { 4995 struct vcpu_svm *svm = to_svm(vcpu); 4996 4997 /* First SIPI: Use the values as initially set by the VMM */ 4998 if (!svm->sev_es.received_first_sipi) { 4999 svm->sev_es.received_first_sipi = true; 5000 return; 5001 } 5002 5003 /* Subsequent SIPI */ 5004 switch (svm->sev_es.ap_reset_hold_type) { 5005 case AP_RESET_HOLD_NAE_EVENT: 5006 /* 5007 * Return from an AP Reset Hold VMGEXIT, where the guest will 5008 * set the CS and RIP. Set SW_EXIT_INFO_2 to a non-zero value. 5009 */ 5010 svm_vmgexit_success(svm, 1); 5011 break; 5012 case AP_RESET_HOLD_MSR_PROTO: 5013 /* 5014 * Return from an AP Reset Hold VMGEXIT, where the guest will 5015 * set the CS and RIP. Set GHCB data field to a non-zero value. 5016 */ 5017 set_ghcb_msr_bits(svm, 1, 5018 GHCB_MSR_AP_RESET_HOLD_RESULT_MASK, 5019 GHCB_MSR_AP_RESET_HOLD_RESULT_POS); 5020 5021 set_ghcb_msr_bits(svm, GHCB_MSR_AP_RESET_HOLD_RESP, 5022 GHCB_MSR_INFO_MASK, 5023 GHCB_MSR_INFO_POS); 5024 break; 5025 default: 5026 break; 5027 } 5028 } 5029 5030 struct page *snp_safe_alloc_page_node(int node, gfp_t gfp) 5031 { 5032 unsigned long pfn; 5033 struct page *p; 5034 5035 if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) 5036 return alloc_pages_node(node, gfp | __GFP_ZERO, 0); 5037 5038 /* 5039 * Allocate an SNP-safe page to workaround the SNP erratum where 5040 * the CPU will incorrectly signal an RMP violation #PF if a 5041 * hugepage (2MB or 1GB) collides with the RMP entry of a 5042 * 2MB-aligned VMCB, VMSA, PML or AVIC backing page. 5043 * 5044 * Allocate one extra page, choose a page which is not 5045 * 2MB-aligned, and free the other. 5046 */ 5047 p = alloc_pages_node(node, gfp | __GFP_ZERO, 1); 5048 if (!p) 5049 return NULL; 5050 5051 split_page(p, 1); 5052 5053 pfn = page_to_pfn(p); 5054 if (IS_ALIGNED(pfn, PTRS_PER_PMD)) 5055 __free_page(p++); 5056 else 5057 __free_page(p + 1); 5058 5059 return p; 5060 } 5061 5062 void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) 5063 { 5064 struct kvm_memory_slot *slot; 5065 struct kvm *kvm = vcpu->kvm; 5066 int order, rmp_level, ret; 5067 unsigned long mmu_seq; 5068 bool assigned; 5069 kvm_pfn_t pfn; 5070 gfn_t gfn; 5071 5072 gfn = gpa >> PAGE_SHIFT; 5073 5074 /* 5075 * The only time RMP faults occur for shared pages is when the guest is 5076 * triggering an RMP fault for an implicit page-state change from 5077 * shared->private. Implicit page-state changes are forwarded to 5078 * userspace via KVM_EXIT_MEMORY_FAULT events, however, so RMP faults 5079 * for shared pages should not end up here. 5080 */ 5081 if (!kvm_is_private_gfn(kvm, gfn)) { 5082 pr_warn_ratelimited("SEV: Unexpected RMP fault for non-private GPA 0x%llx\n", 5083 gpa); 5084 return; 5085 } 5086 5087 slot = gfn_to_memslot(kvm, gfn); 5088 if (!kvm_slot_has_gmem(slot)) { 5089 pr_warn_ratelimited("SEV: Unexpected RMP fault, non-private slot for GPA 0x%llx\n", 5090 gpa); 5091 return; 5092 } 5093 5094 mmu_seq = kvm->mmu_invalidate_seq; 5095 smp_rmb(); 5096 5097 ret = kvm_gmem_get_pfn(kvm, slot, gfn, &pfn, &order); 5098 if (ret) { 5099 pr_warn_ratelimited("SEV: Unexpected RMP fault, no backing page for private GPA 0x%llx\n", 5100 gpa); 5101 return; 5102 } 5103 5104 ret = snp_lookup_rmpentry(pfn, &assigned, &rmp_level); 5105 if (ret || !assigned) { 5106 guard(read_lock)(&kvm->mmu_lock); 5107 5108 if (!mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) 5109 pr_warn_ratelimited("SEV: Unexpected RMP fault, no assigned RMP entry found for GPA 0x%llx PFN 0x%llx error %d\n", 5110 gpa, pfn, ret); 5111 5112 return; 5113 } 5114 5115 /* 5116 * There are 2 cases where a PSMASH may be needed to resolve an #NPF 5117 * with PFERR_GUEST_RMP_BIT set: 5118 * 5119 * 1) RMPADJUST/PVALIDATE can trigger an #NPF with PFERR_GUEST_SIZEM 5120 * bit set if the guest issues them with a smaller granularity than 5121 * what is indicated by the page-size bit in the 2MB RMP entry for 5122 * the PFN that backs the GPA. 5123 * 5124 * 2) Guest access via NPT can trigger an #NPF if the NPT mapping is 5125 * smaller than what is indicated by the 2MB RMP entry for the PFN 5126 * that backs the GPA. 5127 * 5128 * In both these cases, the corresponding 2M RMP entry needs to 5129 * be PSMASH'd to 512 4K RMP entries. If the RMP entry is already 5130 * split into 4K RMP entries, then this is likely a spurious case which 5131 * can occur when there are concurrent accesses by the guest to a 2MB 5132 * GPA range that is backed by a 2MB-aligned PFN who's RMP entry is in 5133 * the process of being PMASH'd into 4K entries. These cases should 5134 * resolve automatically on subsequent accesses, so just ignore them 5135 * here. 5136 */ 5137 if (rmp_level == PG_LEVEL_4K) 5138 goto out; 5139 5140 scoped_guard(read_lock, &kvm->mmu_lock) { 5141 if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) 5142 goto out; 5143 5144 ret = snp_rmptable_psmash(pfn); 5145 if (ret) { 5146 /* 5147 * Look it up again. If it's 4K now then the PSMASH may 5148 * have raced with another process and the issue has 5149 * already resolved itself. If it's not assigned, then 5150 * this must have raced with another process that made 5151 * this page shared. 5152 */ 5153 if (!snp_lookup_rmpentry(pfn, &assigned, &rmp_level) && 5154 ((assigned && rmp_level == PG_LEVEL_4K) || !assigned)) 5155 goto out; 5156 5157 pr_warn_ratelimited("SEV: Unable to split RMP entry for GPA 0x%llx PFN 0x%llx ret %d\n", 5158 gpa, pfn, ret); 5159 } 5160 } 5161 5162 kvm_zap_gfn_range(kvm, gfn, gfn + PTRS_PER_PMD); 5163 out: 5164 trace_kvm_rmp_fault(vcpu, gpa, pfn, error_code, rmp_level, ret); 5165 } 5166 5167 static bool is_pfn_range_shared(kvm_pfn_t start, kvm_pfn_t end) 5168 { 5169 kvm_pfn_t pfn = start; 5170 5171 while (pfn < end) { 5172 int ret, rmp_level; 5173 bool assigned; 5174 5175 ret = snp_lookup_rmpentry(pfn, &assigned, &rmp_level); 5176 if (ret) { 5177 pr_warn_ratelimited("SEV: Failed to retrieve RMP entry: PFN 0x%llx GFN start 0x%llx GFN end 0x%llx RMP level %d error %d\n", 5178 pfn, start, end, rmp_level, ret); 5179 return false; 5180 } 5181 5182 if (assigned) { 5183 pr_debug("%s: overlap detected, PFN 0x%llx start 0x%llx end 0x%llx RMP level %d\n", 5184 __func__, pfn, start, end, rmp_level); 5185 return false; 5186 } 5187 5188 pfn++; 5189 } 5190 5191 return true; 5192 } 5193 5194 static bool is_large_rmp_possible(kvm_pfn_t pfn, kvm_pfn_t nr_pages) 5195 { 5196 kvm_pfn_t pfn_aligned = ALIGN_DOWN(pfn, PTRS_PER_PMD); 5197 5198 /* 5199 * If this is a large folio, and the entire 2M range containing the 5200 * PFN is currently shared, then the entire 2M-aligned range can be 5201 * set to private via a single 2M RMP entry. 5202 */ 5203 if (nr_pages >= KVM_PAGES_PER_HPAGE(PG_LEVEL_2M) && 5204 is_pfn_range_shared(pfn_aligned, pfn_aligned + PTRS_PER_PMD)) 5205 return true; 5206 5207 return false; 5208 } 5209 5210 int sev_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, kvm_pfn_t nr_pages) 5211 { 5212 struct kvm_sev_info *sev = to_kvm_sev_info(kvm); 5213 kvm_pfn_t pfn_aligned; 5214 gfn_t gfn_aligned; 5215 int level, rc; 5216 bool assigned; 5217 5218 if (!sev_snp_guest(kvm)) 5219 return 0; 5220 5221 if (WARN_ON_ONCE(nr_pages != 1)) 5222 return -EIO; 5223 5224 rc = snp_lookup_rmpentry(pfn, &assigned, &level); 5225 if (rc) { 5226 pr_err_ratelimited("SEV: Failed to look up RMP entry: GFN %llx PFN %llx error %d\n", 5227 gfn, pfn, rc); 5228 return -ENOENT; 5229 } 5230 5231 if (assigned) { 5232 pr_debug("%s: already assigned: gfn %llx pfn %llx nr_pages %llx level %d\n", 5233 __func__, gfn, pfn, nr_pages, level); 5234 return 0; 5235 } 5236 5237 if (is_large_rmp_possible(pfn, nr_pages)) { 5238 level = PG_LEVEL_2M; 5239 pfn_aligned = ALIGN_DOWN(pfn, PTRS_PER_PMD); 5240 gfn_aligned = ALIGN_DOWN(gfn, PTRS_PER_PMD); 5241 } else { 5242 level = PG_LEVEL_4K; 5243 pfn_aligned = pfn; 5244 gfn_aligned = gfn; 5245 } 5246 5247 rc = rmp_make_private(pfn_aligned, gfn_to_gpa(gfn_aligned), level, sev->asid, false); 5248 if (rc) { 5249 pr_err_ratelimited("SEV: Failed to update RMP entry: GFN %llx PFN %llx level %d error %d\n", 5250 gfn, pfn, level, rc); 5251 return -EINVAL; 5252 } 5253 5254 pr_debug("%s: updated: gfn %llx pfn %llx pfn_aligned %llx nr_pages %llx level %d\n", 5255 __func__, gfn, pfn, pfn_aligned, nr_pages, level); 5256 5257 return 0; 5258 } 5259 5260 void sev_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages) 5261 { 5262 kvm_pfn_t end = pfn + nr_pages; 5263 5264 if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) 5265 return; 5266 5267 pr_debug("%s: PFN start 0x%llx PFN end 0x%llx\n", __func__, pfn, end); 5268 5269 while (pfn < end) { 5270 bool use_2m_update = false; 5271 int rc, rmp_level; 5272 bool assigned; 5273 5274 rc = snp_lookup_rmpentry(pfn, &assigned, &rmp_level); 5275 if (rc || !assigned) 5276 goto next_pfn; 5277 5278 use_2m_update = IS_ALIGNED(pfn, PTRS_PER_PMD) && 5279 end >= (pfn + PTRS_PER_PMD) && 5280 rmp_level > PG_LEVEL_4K; 5281 5282 /* 5283 * If an unaligned PFN corresponds to a 2M region assigned as a 5284 * large page in the RMP table, PSMASH the region into individual 5285 * 4K RMP entries before attempting to convert a 4K sub-page. 5286 */ 5287 if (!use_2m_update && rmp_level > PG_LEVEL_4K) { 5288 /* 5289 * This shouldn't fail, but if it does, report it, but 5290 * still try to update RMP entry to shared and pray this 5291 * was a spurious error that can be addressed later. 5292 */ 5293 rc = snp_rmptable_psmash(pfn); 5294 WARN_ONCE(rc, "SEV: Failed to PSMASH RMP entry for PFN 0x%llx error %d\n", 5295 pfn, rc); 5296 } 5297 5298 rc = rmp_make_shared(pfn, use_2m_update ? PG_LEVEL_2M : PG_LEVEL_4K); 5299 if (WARN_ONCE(rc, "SEV: Failed to update RMP entry for PFN 0x%llx error %d\n", 5300 pfn, rc)) 5301 goto next_pfn; 5302 5303 /* 5304 * SEV-ES avoids host/guest cache coherency issues through 5305 * WBNOINVD hooks issued via MMU notifiers during run-time, and 5306 * KVM's VM destroy path at shutdown. Those MMU notifier events 5307 * don't cover gmem since there is no requirement to map pages 5308 * to a HVA in order to use them for a running guest. While the 5309 * shutdown path would still likely cover things for SNP guests, 5310 * userspace may also free gmem pages during run-time via 5311 * hole-punching operations on the guest_memfd, so flush the 5312 * cache entries for these pages before free'ing them back to 5313 * the host. 5314 */ 5315 clflush_cache_range(__va(pfn_to_hpa(pfn)), 5316 use_2m_update ? PMD_SIZE : PAGE_SIZE); 5317 next_pfn: 5318 pfn += use_2m_update ? PTRS_PER_PMD : 1; 5319 cond_resched(); 5320 } 5321 } 5322 5323 void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range) 5324 { 5325 struct kvm_vcpu *vcpu; 5326 unsigned long i; 5327 5328 lockdep_assert_held_write(&kvm->mmu_lock); 5329 5330 /* 5331 * An unstable result for "is SNP" is a-ok here, thanks to mmu_lock. 5332 * The vCPU's VMSA GPA is invalidated before the vCPU is made visible 5333 * to other tasks, and can only become valid while holding mmu_lock, 5334 * after the VM is fully committed to being an SNP VM. 5335 */ 5336 if (!____sev_snp_guest(kvm)) 5337 return; 5338 5339 kvm_for_each_vcpu(i, vcpu, kvm) { 5340 /* 5341 * Read snp_guest_vmsa_gpa without taking the vCPU's VMSA mutex 5342 * (or its generic mutex) as mmu_lock is held, i.e. this task 5343 * can't sleep. The VMSA is invalidated outside of mmu_lock, 5344 * but can only become valid inside of mmu_lock, i.e. the below 5345 * can get false positives, but not false negatives. A false 5346 * positive is benign, as a spurious request simply forces the 5347 * vCPU to re-establish its VMSA. 5348 */ 5349 gpa_t gpa = READ_ONCE(to_svm(vcpu)->sev_es.snp_guest_vmsa_gpa); 5350 5351 if (VALID_PAGE(gpa) && 5352 gpa_to_gfn(gpa) >= range->start && 5353 gpa_to_gfn(gpa) < range->end) 5354 kvm_make_request_and_kick(KVM_REQ_VMSA_PAGE_RELOAD, vcpu); 5355 } 5356 } 5357 5358 int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private) 5359 { 5360 int level, rc; 5361 bool assigned; 5362 5363 if (!sev_snp_guest(kvm)) 5364 return 0; 5365 5366 rc = snp_lookup_rmpentry(pfn, &assigned, &level); 5367 if (rc || !assigned) 5368 return PG_LEVEL_4K; 5369 5370 return level; 5371 } 5372 5373 struct vmcb_save_area *sev_decrypt_vmsa(struct kvm_vcpu *vcpu) 5374 { 5375 struct vcpu_svm *svm = to_svm(vcpu); 5376 struct vmcb_save_area *vmsa; 5377 struct kvm_sev_info *sev; 5378 int error = 0; 5379 int ret; 5380 5381 if (!is_sev_es_guest(vcpu)) 5382 return NULL; 5383 5384 /* 5385 * If the VMSA has not yet been encrypted, return a pointer to the 5386 * current un-encrypted VMSA. 5387 */ 5388 if (!vcpu->arch.guest_state_protected) 5389 return (struct vmcb_save_area *)svm->sev_es.vmsa; 5390 5391 sev = to_kvm_sev_info(vcpu->kvm); 5392 5393 /* Check if the SEV policy allows debugging */ 5394 if (is_sev_snp_guest(vcpu)) { 5395 if (!(sev->policy & SNP_POLICY_MASK_DEBUG)) 5396 return NULL; 5397 } else { 5398 if (sev->policy & SEV_POLICY_MASK_NODBG) 5399 return NULL; 5400 } 5401 5402 if (is_sev_snp_guest(vcpu)) { 5403 struct sev_data_snp_dbg dbg = {0}; 5404 5405 vmsa = snp_alloc_firmware_page(__GFP_ZERO); 5406 if (!vmsa) 5407 return NULL; 5408 5409 dbg.gctx_paddr = __psp_pa(sev->snp_context); 5410 dbg.src_addr = svm->vmcb->control.vmsa_pa; 5411 dbg.dst_addr = __psp_pa(vmsa); 5412 5413 ret = sev_do_cmd(SEV_CMD_SNP_DBG_DECRYPT, &dbg, &error); 5414 5415 /* 5416 * Return the target page to a hypervisor page no matter what. 5417 * If this fails, the page can't be used, so leak it and don't 5418 * try to use it. 5419 */ 5420 if (snp_page_reclaim(vcpu->kvm, PHYS_PFN(__pa(vmsa)))) 5421 return NULL; 5422 5423 if (ret) { 5424 pr_err("SEV: SNP_DBG_DECRYPT failed ret=%d, fw_error=%d (%#x)\n", 5425 ret, error, error); 5426 free_page((unsigned long)vmsa); 5427 5428 return NULL; 5429 } 5430 } else { 5431 struct sev_data_dbg dbg = {0}; 5432 struct page *vmsa_page; 5433 5434 vmsa_page = alloc_page(GFP_KERNEL); 5435 if (!vmsa_page) 5436 return NULL; 5437 5438 vmsa = page_address(vmsa_page); 5439 5440 dbg.handle = sev->handle; 5441 dbg.src_addr = svm->vmcb->control.vmsa_pa; 5442 dbg.dst_addr = __psp_pa(vmsa); 5443 dbg.len = PAGE_SIZE; 5444 5445 ret = sev_do_cmd(SEV_CMD_DBG_DECRYPT, &dbg, &error); 5446 if (ret) { 5447 pr_err("SEV: SEV_CMD_DBG_DECRYPT failed ret=%d, fw_error=%d (0x%x)\n", 5448 ret, error, error); 5449 __free_page(vmsa_page); 5450 5451 return NULL; 5452 } 5453 } 5454 5455 return vmsa; 5456 } 5457 5458 void sev_free_decrypted_vmsa(struct kvm_vcpu *vcpu, struct vmcb_save_area *vmsa) 5459 { 5460 /* If the VMSA has not yet been encrypted, nothing was allocated */ 5461 if (!vcpu->arch.guest_state_protected || !vmsa) 5462 return; 5463 5464 free_page((unsigned long)vmsa); 5465 } 5466