1 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 2 3 #include <linux/kvm_host.h> 4 5 #include "irq.h" 6 #include "mmu.h" 7 #include "regs.h" 8 #include "x86.h" 9 #include "smm.h" 10 #include "cpuid.h" 11 #include "pmu.h" 12 13 #include <linux/module.h> 14 #include <linux/mod_devicetable.h> 15 #include <linux/kernel.h> 16 #include <linux/vmalloc.h> 17 #include <linux/highmem.h> 18 #include <linux/amd-iommu.h> 19 #include <linux/sched.h> 20 #include <linux/trace_events.h> 21 #include <linux/slab.h> 22 #include <linux/hashtable.h> 23 #include <linux/objtool.h> 24 #include <linux/psp-sev.h> 25 #include <linux/file.h> 26 #include <linux/pagemap.h> 27 #include <linux/swap.h> 28 #include <linux/rwsem.h> 29 #include <linux/cc_platform.h> 30 #include <linux/smp.h> 31 #include <linux/string_choices.h> 32 #include <linux/mutex.h> 33 34 #include <asm/apic.h> 35 #include <asm/msr.h> 36 #include <asm/perf_event.h> 37 #include <asm/tlbflush.h> 38 #include <asm/desc.h> 39 #include <asm/debugreg.h> 40 #include <asm/kvm_para.h> 41 #include <asm/irq_remapping.h> 42 #include <asm/spec-ctrl.h> 43 #include <asm/cpu_device_id.h> 44 #include <asm/cpuid/api.h> 45 #include <asm/traps.h> 46 #include <asm/reboot.h> 47 #include <asm/fpu/api.h> 48 #include <asm/virt.h> 49 50 #include <trace/events/ipi.h> 51 52 #include "trace.h" 53 54 #include "vmenter.h" 55 #include "svm.h" 56 #include "svm_ops.h" 57 58 #include "hyperv.h" 59 #include "kvm_onhyperv.h" 60 #include "svm_onhyperv.h" 61 62 MODULE_AUTHOR("Qumranet"); 63 MODULE_DESCRIPTION("KVM support for SVM (AMD-V) extensions"); 64 MODULE_LICENSE("GPL"); 65 66 #ifdef MODULE 67 static const struct x86_cpu_id svm_cpu_id[] = { 68 X86_MATCH_FEATURE(X86_FEATURE_SVM, NULL), 69 {} 70 }; 71 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id); 72 #endif 73 74 #define SEG_TYPE_LDT 2 75 #define SEG_TYPE_BUSY_TSS16 3 76 77 static bool erratum_383_found __read_mostly; 78 79 /* 80 * Set osvw_len to higher value when updated Revision Guides 81 * are published and we know what the new status bits are 82 */ 83 static uint64_t osvw_len = 4, osvw_status; 84 static DEFINE_SPINLOCK(osvw_lock); 85 86 static DEFINE_PER_CPU(u64, current_tsc_ratio); 87 88 /* 89 * These 2 parameters are used to config the controls for Pause-Loop Exiting: 90 * pause_filter_count: On processors that support Pause filtering(indicated 91 * by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter 92 * count value. On VMRUN this value is loaded into an internal counter. 93 * Each time a pause instruction is executed, this counter is decremented 94 * until it reaches zero at which time a #VMEXIT is generated if pause 95 * intercept is enabled. Refer to AMD APM Vol 2 Section 15.14.4 Pause 96 * Intercept Filtering for more details. 97 * This also indicate if ple logic enabled. 98 * 99 * pause_filter_thresh: In addition, some processor families support advanced 100 * pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on 101 * the amount of time a guest is allowed to execute in a pause loop. 102 * In this mode, a 16-bit pause filter threshold field is added in the 103 * VMCB. The threshold value is a cycle count that is used to reset the 104 * pause counter. As with simple pause filtering, VMRUN loads the pause 105 * count value from VMCB into an internal counter. Then, on each pause 106 * instruction the hardware checks the elapsed number of cycles since 107 * the most recent pause instruction against the pause filter threshold. 108 * If the elapsed cycle count is greater than the pause filter threshold, 109 * then the internal pause count is reloaded from the VMCB and execution 110 * continues. If the elapsed cycle count is less than the pause filter 111 * threshold, then the internal pause count is decremented. If the count 112 * value is less than zero and PAUSE intercept is enabled, a #VMEXIT is 113 * triggered. If advanced pause filtering is supported and pause filter 114 * threshold field is set to zero, the filter will operate in the simpler, 115 * count only mode. 116 */ 117 118 static unsigned short __ro_after_init pause_filter_thresh = KVM_DEFAULT_PLE_GAP; 119 module_param(pause_filter_thresh, ushort, 0444); 120 121 static unsigned short __ro_after_init pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW; 122 module_param(pause_filter_count, ushort, 0444); 123 124 /* Default doubles per-vcpu window every exit. */ 125 static unsigned short __ro_after_init pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW; 126 module_param(pause_filter_count_grow, ushort, 0444); 127 128 /* Default resets per-vcpu window every exit to pause_filter_count. */ 129 static unsigned short __ro_after_init pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK; 130 module_param(pause_filter_count_shrink, ushort, 0444); 131 132 /* Default is to compute the maximum so we can never overflow. */ 133 static unsigned short __ro_after_init pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX; 134 module_param(pause_filter_count_max, ushort, 0444); 135 136 /* 137 * Use nested page tables by default. Note, NPT may get forced off by 138 * svm_hardware_setup() if it's unsupported by hardware or the host kernel. 139 */ 140 bool __ro_after_init npt_enabled = true; 141 module_param_named(npt, npt_enabled, bool, 0444); 142 143 bool gmet_enabled = true; 144 module_param_named(gmet, gmet_enabled, bool, 0444); 145 146 /* allow nested virtualization in KVM/SVM */ 147 static int __ro_after_init nested = true; 148 module_param(nested, int, 0444); 149 150 /* enable/disable Next RIP Save */ 151 int __ro_after_init nrips = true; 152 module_param(nrips, int, 0444); 153 154 /* enable/disable Virtual VMLOAD VMSAVE */ 155 static int __ro_after_init vls = true; 156 module_param(vls, int, 0444); 157 158 /* enable/disable Virtual GIF */ 159 int __ro_after_init vgif = true; 160 module_param(vgif, int, 0444); 161 162 /* enable/disable LBR virtualization */ 163 int __ro_after_init lbrv = true; 164 module_param(lbrv, int, 0444); 165 166 static int __ro_after_init tsc_scaling = true; 167 module_param(tsc_scaling, int, 0444); 168 169 module_param(enable_device_posted_irqs, bool, 0444); 170 171 bool __read_mostly dump_invalid_vmcb; 172 module_param(dump_invalid_vmcb, bool, 0644); 173 174 175 bool __ro_after_init intercept_smi = true; 176 module_param(intercept_smi, bool, 0444); 177 178 bool __ro_after_init vnmi = true; 179 module_param(vnmi, bool, 0444); 180 181 module_param(enable_mediated_pmu, bool, 0444); 182 183 static bool __ro_after_init svm_gp_erratum_intercept = true; 184 185 static u8 rsm_ins_bytes[] = "\x0f\xaa"; 186 187 static unsigned long __read_mostly iopm_base; 188 189 DEFINE_PER_CPU(struct svm_cpu_data, svm_data); 190 191 static DEFINE_MUTEX(vmcb_dump_mutex); 192 193 /* 194 * Only MSR_TSC_AUX is switched via the user return hook. EFER is switched via 195 * the VMCB, and the SYSCALL/SYSENTER MSRs are handled by VMLOAD/VMSAVE. 196 * 197 * RDTSCP and RDPID are not used in the kernel, specifically to allow KVM to 198 * defer the restoration of TSC_AUX until the CPU returns to userspace. 199 */ 200 int tsc_aux_uret_slot __ro_after_init = -1; 201 202 static int get_npt_level(void) 203 { 204 #ifdef CONFIG_X86_64 205 return pgtable_l5_enabled() ? PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL; 206 #else 207 return PT32E_ROOT_LEVEL; 208 #endif 209 } 210 211 int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) 212 { 213 struct vcpu_svm *svm = to_svm(vcpu); 214 u64 old_efer = vcpu->arch.efer; 215 vcpu->arch.efer = efer; 216 217 if (!npt_enabled) { 218 /* Shadow paging assumes NX to be available. */ 219 efer |= EFER_NX; 220 221 if (!(efer & EFER_LMA)) 222 efer &= ~EFER_LME; 223 } 224 225 if ((old_efer & EFER_SVME) != (efer & EFER_SVME)) { 226 if (!(efer & EFER_SVME)) { 227 /* 228 * Architecturally, clearing EFER.SVME while a guest is 229 * running yields undefined behavior, i.e. KVM can do 230 * literally anything. Force the vCPU back into L1 as 231 * that is the safest option for KVM, but synthesize a 232 * triple fault (for L1!) so that KVM at least doesn't 233 * run random L2 code in the context of L1. Do so if 234 * and only if the vCPU is actively running, e.g. to 235 * avoid positives if userspace is stuffing state. 236 */ 237 if (is_guest_mode(vcpu) && vcpu->wants_to_run) 238 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 239 240 svm_leave_nested(vcpu); 241 /* #GP intercept is still needed for vmware backdoor */ 242 if (!enable_vmware_backdoor) 243 clr_exception_intercept(svm, GP_VECTOR); 244 245 /* 246 * Free the nested guest state, unless we are in SMM. 247 * In this case we will return to the nested guest 248 * as soon as we leave SMM. 249 */ 250 if (!is_smm(vcpu)) 251 svm_free_nested(svm); 252 253 } else { 254 int ret = svm_allocate_nested(svm); 255 256 if (ret) { 257 vcpu->arch.efer = old_efer; 258 return ret; 259 } 260 261 /* 262 * Never intercept #GP for SEV guests, KVM can't 263 * decrypt guest memory to workaround the erratum. 264 */ 265 if (svm_gp_erratum_intercept && !is_sev_guest(vcpu)) 266 set_exception_intercept(svm, GP_VECTOR); 267 } 268 269 svm_pmu_handle_nested_transition(svm); 270 kvm_make_request(KVM_REQ_RECALC_INTERCEPTS, vcpu); 271 } 272 273 svm->vmcb->save.efer = efer | EFER_SVME; 274 vmcb_mark_dirty(svm->vmcb, VMCB_CR); 275 return 0; 276 } 277 278 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu) 279 { 280 struct vcpu_svm *svm = to_svm(vcpu); 281 u32 ret = 0; 282 283 if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) 284 ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS; 285 return ret; 286 } 287 288 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask) 289 { 290 struct vcpu_svm *svm = to_svm(vcpu); 291 292 if (mask == 0) 293 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK; 294 else 295 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK; 296 297 } 298 299 static int __svm_skip_emulated_instruction(struct kvm_vcpu *vcpu, 300 int emul_type, 301 bool commit_side_effects) 302 { 303 struct vcpu_svm *svm = to_svm(vcpu); 304 unsigned long old_rflags; 305 306 /* 307 * SEV-ES does not expose the next RIP. The RIP update is controlled by 308 * the type of exit and the #VC handler in the guest. 309 */ 310 if (is_sev_es_guest(vcpu)) 311 goto done; 312 313 if (nrips && svm->vmcb->control.next_rip != 0) { 314 WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS)); 315 svm->next_rip = svm->vmcb->control.next_rip; 316 } 317 318 if (!svm->next_rip) { 319 if (unlikely(!commit_side_effects)) 320 old_rflags = svm->vmcb->save.rflags; 321 322 if (!kvm_emulate_instruction(vcpu, emul_type)) 323 return 0; 324 325 if (unlikely(!commit_side_effects)) 326 svm->vmcb->save.rflags = old_rflags; 327 } else { 328 kvm_rip_write(vcpu, svm->next_rip); 329 } 330 331 done: 332 if (likely(commit_side_effects)) 333 svm_set_interrupt_shadow(vcpu, 0); 334 335 return 1; 336 } 337 338 int svm_skip_emulated_instruction(struct kvm_vcpu *vcpu) 339 { 340 return __svm_skip_emulated_instruction(vcpu, EMULTYPE_SKIP, true); 341 } 342 343 static int svm_update_soft_interrupt_rip(struct kvm_vcpu *vcpu, u8 vector) 344 { 345 const int emul_type = EMULTYPE_SKIP | EMULTYPE_SKIP_SOFT_INT | 346 EMULTYPE_SET_SOFT_INT_VECTOR(vector); 347 unsigned long rip, old_rip = kvm_rip_read(vcpu); 348 struct vcpu_svm *svm = to_svm(vcpu); 349 350 /* 351 * Due to architectural shortcomings, the CPU doesn't always provide 352 * NextRIP, e.g. if KVM intercepted an exception that occurred while 353 * the CPU was vectoring an INTO/INT3 in the guest. Temporarily skip 354 * the instruction even if NextRIP is supported to acquire the next 355 * RIP so that it can be shoved into the NextRIP field, otherwise 356 * hardware will fail to advance guest RIP during event injection. 357 * Drop the exception/interrupt if emulation fails and effectively 358 * retry the instruction, it's the least awful option. If NRIPS is 359 * in use, the skip must not commit any side effects such as clearing 360 * the interrupt shadow or RFLAGS.RF. 361 */ 362 if (!__svm_skip_emulated_instruction(vcpu, emul_type, !nrips)) 363 return -EIO; 364 365 rip = kvm_rip_read(vcpu); 366 367 /* 368 * Save the injection information, even when using next_rip, as the 369 * VMCB's next_rip will be lost (cleared on VM-Exit) if the injection 370 * doesn't complete due to a VM-Exit occurring while the CPU is 371 * vectoring the event. Decoding the instruction isn't guaranteed to 372 * work as there may be no backing instruction, e.g. if the event is 373 * being injected by L1 for L2, or if the guest is patching INT3 into 374 * a different instruction. 375 */ 376 svm->soft_int_injected = true; 377 svm->soft_int_csbase = svm->vmcb->save.cs.base; 378 svm->soft_int_old_rip = old_rip; 379 svm->soft_int_next_rip = rip; 380 381 if (nrips) 382 kvm_rip_write(vcpu, old_rip); 383 384 if (static_cpu_has(X86_FEATURE_NRIPS)) 385 svm->vmcb->control.next_rip = rip; 386 387 return 0; 388 } 389 390 static void svm_inject_exception(struct kvm_vcpu *vcpu) 391 { 392 struct kvm_queued_exception *ex = &vcpu->arch.exception; 393 struct vcpu_svm *svm = to_svm(vcpu); 394 395 kvm_deliver_exception_payload(vcpu, ex); 396 397 if (kvm_exception_is_soft(ex->vector) && 398 svm_update_soft_interrupt_rip(vcpu, ex->vector)) 399 return; 400 401 svm->vmcb->control.event_inj = ex->vector 402 | SVM_EVTINJ_VALID 403 | (ex->has_error_code ? SVM_EVTINJ_VALID_ERR : 0) 404 | SVM_EVTINJ_TYPE_EXEPT; 405 svm->vmcb->control.event_inj_err = ex->error_code; 406 } 407 408 static void svm_init_erratum_383(void) 409 { 410 u64 val; 411 412 if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH)) 413 return; 414 415 /* Use _safe variants to not break nested virtualization */ 416 if (native_read_msr_safe(MSR_AMD64_DC_CFG, &val)) 417 return; 418 419 val |= (1ULL << 47); 420 421 native_write_msr_safe(MSR_AMD64_DC_CFG, val); 422 423 erratum_383_found = true; 424 } 425 426 static void svm_init_osvw(struct kvm_vcpu *vcpu) 427 { 428 /* 429 * Guests should see errata 400 and 415 as fixed (assuming that 430 * HLT and IO instructions are intercepted). 431 */ 432 vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3; 433 vcpu->arch.osvw.status = osvw_status & ~(6ULL); 434 435 /* 436 * By increasing VCPU's osvw.length to 3 we are telling the guest that 437 * all osvw.status bits inside that length, including bit 0 (which is 438 * reserved for erratum 298), are valid. However, if host processor's 439 * osvw_len is 0 then osvw_status[0] carries no information. We need to 440 * be conservative here and therefore we tell the guest that erratum 298 441 * is present (because we really don't know). 442 */ 443 if (osvw_len == 0 && boot_cpu_data.x86 == 0x10) 444 vcpu->arch.osvw.status |= 1; 445 } 446 447 static void svm_init_os_visible_workarounds(void) 448 { 449 u64 len, status; 450 451 /* 452 * Get OS-Visible Workarounds (OSVW) bits. 453 * 454 * Note that it is possible to have a system with mixed processor 455 * revisions and therefore different OSVW bits. If bits are not the same 456 * on different processors then choose the worst case (i.e. if erratum 457 * is present on one processor and not on another then assume that the 458 * erratum is present everywhere). 459 * 460 * Note #2! The OSVW MSRs are used to communciate that an erratum is 461 * NOT present! Software must assume erratum as present if its bit is 462 * set in OSVW_STATUS *or* the bit number exceeds OSVW_ID_LENGTH. If 463 * either RDMSR fails, simply zero out the length to treat all errata 464 * as being present. Similarly, use the *minimum* length across all 465 * CPUs, not the maximum length. 466 * 467 * If the length is zero, then is KVM already treating all errata as 468 * being present and there's nothing left to do. 469 */ 470 if (!osvw_len) 471 return; 472 473 if (!this_cpu_has(X86_FEATURE_OSVW) || 474 native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &len) || 475 native_read_msr_safe(MSR_AMD64_OSVW_STATUS, &status)) 476 len = status = 0; 477 478 if (status == READ_ONCE(osvw_status) && len >= READ_ONCE(osvw_len)) 479 return; 480 481 guard(spinlock)(&osvw_lock); 482 483 if (len < osvw_len) 484 osvw_len = len; 485 osvw_status |= status; 486 osvw_status &= (1ULL << osvw_len) - 1; 487 } 488 489 static bool __kvm_is_svm_supported(void) 490 { 491 int cpu = smp_processor_id(); 492 struct cpuinfo_x86 *c = &cpu_data(cpu); 493 494 if (c->x86_vendor != X86_VENDOR_AMD && 495 c->x86_vendor != X86_VENDOR_HYGON) { 496 pr_err("CPU %d isn't AMD or Hygon\n", cpu); 497 return false; 498 } 499 500 if (!cpu_has(c, X86_FEATURE_SVM)) { 501 pr_err("SVM not supported by CPU %d\n", cpu); 502 return false; 503 } 504 505 if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) { 506 pr_info("KVM is unsupported when running as an SEV guest\n"); 507 return false; 508 } 509 510 return true; 511 } 512 513 static bool kvm_is_svm_supported(void) 514 { 515 bool supported; 516 517 migrate_disable(); 518 supported = __kvm_is_svm_supported(); 519 migrate_enable(); 520 521 return supported; 522 } 523 524 static int svm_check_processor_compat(void) 525 { 526 if (!__kvm_is_svm_supported()) 527 return -EIO; 528 529 return 0; 530 } 531 532 static void __svm_write_tsc_multiplier(u64 multiplier) 533 { 534 if (multiplier == __this_cpu_read(current_tsc_ratio)) 535 return; 536 537 wrmsrq(MSR_AMD64_TSC_RATIO, multiplier); 538 __this_cpu_write(current_tsc_ratio, multiplier); 539 } 540 541 static __always_inline struct sev_es_save_area *sev_es_host_save_area(struct svm_cpu_data *sd) 542 { 543 return &sd->save_area->host_sev_es_save; 544 } 545 546 static void svm_emergency_disable_virtualization_cpu(void) 547 { 548 wrmsrq(MSR_VM_HSAVE_PA, 0); 549 } 550 551 static void svm_disable_virtualization_cpu(void) 552 { 553 /* Make sure we clean up behind us */ 554 if (tsc_scaling) 555 __svm_write_tsc_multiplier(SVM_TSC_RATIO_DEFAULT); 556 557 x86_virt_put_ref(X86_FEATURE_SVM); 558 wrmsrq(MSR_VM_HSAVE_PA, 0); 559 560 amd_pmu_disable_virt(); 561 } 562 563 static int svm_enable_virtualization_cpu(void) 564 { 565 566 struct svm_cpu_data *sd; 567 int me = raw_smp_processor_id(); 568 int r; 569 570 r = x86_virt_get_ref(X86_FEATURE_SVM); 571 if (r) 572 return r; 573 574 sd = per_cpu_ptr(&svm_data, me); 575 sd->asid_generation = 1; 576 sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1; 577 sd->next_asid = sd->max_asid + 1; 578 sd->min_asid = max_sev_asid + 1; 579 580 wrmsrq(MSR_VM_HSAVE_PA, sd->save_area_pa); 581 582 if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) { 583 /* 584 * Set the default value, even if we don't use TSC scaling 585 * to avoid having stale value in the msr 586 */ 587 __svm_write_tsc_multiplier(SVM_TSC_RATIO_DEFAULT); 588 } 589 590 svm_init_os_visible_workarounds(); 591 592 svm_init_erratum_383(); 593 594 amd_pmu_enable_virt(); 595 596 return 0; 597 } 598 599 static void svm_cpu_uninit(int cpu) 600 { 601 struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu); 602 603 if (!sd->save_area) 604 return; 605 606 kfree(sd->sev_vmcbs); 607 __free_page(__sme_pa_to_page(sd->save_area_pa)); 608 sd->save_area_pa = 0; 609 sd->save_area = NULL; 610 } 611 612 static int svm_cpu_init(int cpu) 613 { 614 struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu); 615 struct page *save_area_page; 616 int ret = -ENOMEM; 617 618 memset(sd, 0, sizeof(struct svm_cpu_data)); 619 save_area_page = snp_safe_alloc_page_node(cpu_to_node(cpu), GFP_KERNEL); 620 if (!save_area_page) 621 return ret; 622 623 ret = sev_cpu_init(sd); 624 if (ret) 625 goto free_save_area; 626 627 sd->save_area = page_address(save_area_page); 628 sd->save_area_pa = __sme_page_pa(save_area_page); 629 return 0; 630 631 free_save_area: 632 __free_page(save_area_page); 633 return ret; 634 635 } 636 637 static void set_dr_intercepts(struct vcpu_svm *svm) 638 { 639 struct vmcb *vmcb = svm->vmcb01.ptr; 640 641 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR0_READ); 642 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR1_READ); 643 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR2_READ); 644 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR3_READ); 645 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR4_READ); 646 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR5_READ); 647 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR6_READ); 648 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR0_WRITE); 649 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR1_WRITE); 650 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR2_WRITE); 651 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR3_WRITE); 652 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR4_WRITE); 653 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR5_WRITE); 654 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR6_WRITE); 655 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_READ); 656 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_WRITE); 657 658 svm_mark_intercepts_dirty(svm); 659 } 660 661 static void clr_dr_intercepts(struct vcpu_svm *svm) 662 { 663 struct vmcb *vmcb = svm->vmcb01.ptr; 664 665 vmcb->control.intercepts[INTERCEPT_DR] = 0; 666 667 svm_mark_intercepts_dirty(svm); 668 } 669 670 static bool msr_write_intercepted(struct vcpu_svm *svm, u32 msr) 671 { 672 /* 673 * For non-nested case: 674 * If the L01 MSR bitmap does not intercept the MSR, then we need to 675 * save it. 676 * 677 * For nested case: 678 * If the L02 MSR bitmap does not intercept the MSR, then we need to 679 * save it. 680 */ 681 void *msrpm = is_guest_mode(&svm->vcpu) ? svm->nested.msrpm : svm->msrpm; 682 683 return svm_test_msr_bitmap_write(msrpm, msr); 684 } 685 686 void svm_set_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type, bool set) 687 { 688 struct vcpu_svm *svm = to_svm(vcpu); 689 void *msrpm = svm->msrpm; 690 691 /* Don't disable interception for MSRs userspace wants to handle. */ 692 if (type & MSR_TYPE_R) { 693 if (!set && kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_READ)) 694 svm_clear_msr_bitmap_read(msrpm, msr); 695 else 696 svm_set_msr_bitmap_read(msrpm, msr); 697 } 698 699 if (type & MSR_TYPE_W) { 700 if (!set && kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_WRITE)) 701 svm_clear_msr_bitmap_write(msrpm, msr); 702 else 703 svm_set_msr_bitmap_write(msrpm, msr); 704 } 705 706 svm_hv_vmcb_dirty_nested_enlightenments(vcpu); 707 svm->nested.force_msr_bitmap_recalc = true; 708 } 709 710 void *svm_alloc_permissions_map(unsigned long size, gfp_t gfp_mask) 711 { 712 unsigned int order = get_order(size); 713 struct page *pages = alloc_pages(gfp_mask, order); 714 void *pm; 715 716 if (!pages) 717 return NULL; 718 719 /* 720 * Set all bits in the permissions map so that all MSR and I/O accesses 721 * are intercepted by default. 722 */ 723 pm = page_address(pages); 724 memset(pm, 0xff, PAGE_SIZE * (1 << order)); 725 726 return pm; 727 } 728 729 static void svm_recalc_lbr_msr_intercepts(struct kvm_vcpu *vcpu) 730 { 731 struct vcpu_svm *svm = to_svm(vcpu); 732 bool intercept = !(svm->vmcb->control.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR); 733 734 if (intercept == svm->lbr_msrs_intercepted) 735 return; 736 737 svm_set_intercept_for_msr(vcpu, MSR_IA32_LASTBRANCHFROMIP, MSR_TYPE_RW, intercept); 738 svm_set_intercept_for_msr(vcpu, MSR_IA32_LASTBRANCHTOIP, MSR_TYPE_RW, intercept); 739 svm_set_intercept_for_msr(vcpu, MSR_IA32_LASTINTFROMIP, MSR_TYPE_RW, intercept); 740 svm_set_intercept_for_msr(vcpu, MSR_IA32_LASTINTTOIP, MSR_TYPE_RW, intercept); 741 742 if (is_sev_es_guest(vcpu)) 743 svm_set_intercept_for_msr(vcpu, MSR_IA32_DEBUGCTLMSR, MSR_TYPE_RW, intercept); 744 745 svm->lbr_msrs_intercepted = intercept; 746 } 747 748 void svm_vcpu_free_msrpm(void *msrpm) 749 { 750 __free_pages(virt_to_page(msrpm), get_order(MSRPM_SIZE)); 751 } 752 753 static void svm_recalc_pmu_msr_intercepts(struct kvm_vcpu *vcpu) 754 { 755 bool intercept = !kvm_vcpu_has_mediated_pmu(vcpu); 756 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 757 int i; 758 759 if (!enable_mediated_pmu) 760 return; 761 762 /* Legacy counters are always available for AMD CPUs with a PMU. */ 763 for (i = 0; i < min(pmu->nr_arch_gp_counters, AMD64_NUM_COUNTERS); i++) 764 svm_set_intercept_for_msr(vcpu, MSR_K7_PERFCTR0 + i, 765 MSR_TYPE_RW, intercept); 766 767 intercept |= !guest_cpu_cap_has(vcpu, X86_FEATURE_PERFCTR_CORE); 768 for (i = 0; i < pmu->nr_arch_gp_counters; i++) 769 svm_set_intercept_for_msr(vcpu, MSR_F15H_PERF_CTR + 2 * i, 770 MSR_TYPE_RW, intercept); 771 772 for ( ; i < kvm_pmu_cap.num_counters_gp; i++) 773 svm_enable_intercept_for_msr(vcpu, MSR_F15H_PERF_CTR + 2 * i, 774 MSR_TYPE_RW); 775 776 intercept = kvm_need_perf_global_ctrl_intercept(vcpu); 777 svm_set_intercept_for_msr(vcpu, MSR_AMD64_PERF_CNTR_GLOBAL_CTL, 778 MSR_TYPE_RW, intercept); 779 svm_set_intercept_for_msr(vcpu, MSR_AMD64_PERF_CNTR_GLOBAL_STATUS, 780 MSR_TYPE_RW, intercept); 781 svm_set_intercept_for_msr(vcpu, MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR, 782 MSR_TYPE_RW, intercept); 783 svm_set_intercept_for_msr(vcpu, MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET, 784 MSR_TYPE_RW, intercept); 785 } 786 787 static void svm_recalc_msr_intercepts(struct kvm_vcpu *vcpu) 788 { 789 struct vcpu_svm *svm = to_svm(vcpu); 790 791 svm_disable_intercept_for_msr(vcpu, MSR_STAR, MSR_TYPE_RW); 792 svm_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_CS, MSR_TYPE_RW); 793 794 #ifdef CONFIG_X86_64 795 svm_disable_intercept_for_msr(vcpu, MSR_GS_BASE, MSR_TYPE_RW); 796 svm_disable_intercept_for_msr(vcpu, MSR_FS_BASE, MSR_TYPE_RW); 797 svm_disable_intercept_for_msr(vcpu, MSR_KERNEL_GS_BASE, MSR_TYPE_RW); 798 svm_disable_intercept_for_msr(vcpu, MSR_LSTAR, MSR_TYPE_RW); 799 svm_disable_intercept_for_msr(vcpu, MSR_CSTAR, MSR_TYPE_RW); 800 svm_disable_intercept_for_msr(vcpu, MSR_SYSCALL_MASK, MSR_TYPE_RW); 801 #endif 802 803 if (lbrv) 804 svm_recalc_lbr_msr_intercepts(vcpu); 805 806 if (cpu_feature_enabled(X86_FEATURE_IBPB)) 807 svm_set_intercept_for_msr(vcpu, MSR_IA32_PRED_CMD, MSR_TYPE_W, 808 !guest_has_pred_cmd_msr(vcpu)); 809 810 if (cpu_feature_enabled(X86_FEATURE_FLUSH_L1D)) 811 svm_set_intercept_for_msr(vcpu, MSR_IA32_FLUSH_CMD, MSR_TYPE_W, 812 !guest_cpu_cap_has(vcpu, X86_FEATURE_FLUSH_L1D)); 813 814 /* 815 * Disable interception of SPEC_CTRL if KVM doesn't need to manually 816 * context switch the MSR (SPEC_CTRL is virtualized by the CPU), or if 817 * the guest has a non-zero SPEC_CTRL value, i.e. is likely actively 818 * using SPEC_CTRL. 819 */ 820 if (cpu_feature_enabled(X86_FEATURE_V_SPEC_CTRL)) 821 svm_set_intercept_for_msr(vcpu, MSR_IA32_SPEC_CTRL, MSR_TYPE_RW, 822 !guest_has_spec_ctrl_msr(vcpu)); 823 else 824 svm_set_intercept_for_msr(vcpu, MSR_IA32_SPEC_CTRL, MSR_TYPE_RW, 825 !svm->spec_ctrl); 826 827 /* 828 * Intercept SYSENTER_EIP and SYSENTER_ESP when emulating an Intel CPU, 829 * as AMD hardware only store 32 bits, whereas Intel CPUs track 64 bits. 830 */ 831 svm_set_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_EIP, MSR_TYPE_RW, 832 guest_cpuid_is_intel_compatible(vcpu)); 833 svm_set_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_ESP, MSR_TYPE_RW, 834 guest_cpuid_is_intel_compatible(vcpu)); 835 836 if (kvm_aperfmperf_in_guest(vcpu->kvm)) { 837 svm_disable_intercept_for_msr(vcpu, MSR_IA32_APERF, MSR_TYPE_R); 838 svm_disable_intercept_for_msr(vcpu, MSR_IA32_MPERF, MSR_TYPE_R); 839 } 840 841 if (kvm_cpu_cap_has(X86_FEATURE_SHSTK)) { 842 bool shstk_enabled = guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK); 843 844 svm_set_intercept_for_msr(vcpu, MSR_IA32_U_CET, MSR_TYPE_RW, !shstk_enabled); 845 svm_set_intercept_for_msr(vcpu, MSR_IA32_S_CET, MSR_TYPE_RW, !shstk_enabled); 846 svm_set_intercept_for_msr(vcpu, MSR_IA32_PL0_SSP, MSR_TYPE_RW, !shstk_enabled); 847 svm_set_intercept_for_msr(vcpu, MSR_IA32_PL1_SSP, MSR_TYPE_RW, !shstk_enabled); 848 svm_set_intercept_for_msr(vcpu, MSR_IA32_PL2_SSP, MSR_TYPE_RW, !shstk_enabled); 849 svm_set_intercept_for_msr(vcpu, MSR_IA32_PL3_SSP, MSR_TYPE_RW, !shstk_enabled); 850 } 851 852 if (is_sev_es_guest(vcpu)) 853 sev_es_recalc_msr_intercepts(vcpu); 854 855 svm_recalc_pmu_msr_intercepts(vcpu); 856 857 /* 858 * x2APIC intercepts are modified on-demand and cannot be filtered by 859 * userspace. 860 */ 861 } 862 863 static void __svm_enable_lbrv(struct kvm_vcpu *vcpu) 864 { 865 to_svm(vcpu)->vmcb->control.misc_ctl2 |= SVM_MISC2_ENABLE_V_LBR; 866 } 867 868 void svm_enable_lbrv(struct kvm_vcpu *vcpu) 869 { 870 __svm_enable_lbrv(vcpu); 871 svm_recalc_lbr_msr_intercepts(vcpu); 872 } 873 874 static void __svm_disable_lbrv(struct kvm_vcpu *vcpu) 875 { 876 KVM_BUG_ON(is_sev_es_guest(vcpu), vcpu->kvm); 877 to_svm(vcpu)->vmcb->control.misc_ctl2 &= ~SVM_MISC2_ENABLE_V_LBR; 878 } 879 880 void svm_update_lbrv(struct kvm_vcpu *vcpu) 881 { 882 struct vcpu_svm *svm = to_svm(vcpu); 883 bool current_enable_lbrv = svm->vmcb->control.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR; 884 bool enable_lbrv = (svm->vmcb->save.dbgctl & DEBUGCTLMSR_LBR) || 885 (is_guest_mode(vcpu) && guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) && 886 (svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR)); 887 888 if (enable_lbrv && !current_enable_lbrv) 889 __svm_enable_lbrv(vcpu); 890 else if (!enable_lbrv && current_enable_lbrv) 891 __svm_disable_lbrv(vcpu); 892 893 /* 894 * During nested transitions, it is possible that the current VMCB has 895 * LBR_CTL set, but the previous LBR_CTL had it cleared (or vice versa). 896 * In this case, even though LBR_CTL does not need an update, intercepts 897 * do, so always recalculate the intercepts here. 898 */ 899 svm_recalc_lbr_msr_intercepts(vcpu); 900 } 901 902 void disable_nmi_singlestep(struct vcpu_svm *svm) 903 { 904 svm->nmi_singlestep = false; 905 906 if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) { 907 /* Clear our flags if they were not set by the guest */ 908 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF)) 909 svm->vmcb->save.rflags &= ~X86_EFLAGS_TF; 910 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF)) 911 svm->vmcb->save.rflags &= ~X86_EFLAGS_RF; 912 } 913 } 914 915 static void grow_ple_window(struct kvm_vcpu *vcpu) 916 { 917 struct vcpu_svm *svm = to_svm(vcpu); 918 struct vmcb_control_area *control = &svm->vmcb->control; 919 int old = control->pause_filter_count; 920 921 /* Adjusting pause_filter_count makes no sense if PLE is disabled. */ 922 WARN_ON_ONCE(kvm_pause_in_guest(vcpu->kvm)); 923 924 /* 925 * While running L2, KVM should intercept PAUSE if and only if L1 wants 926 * to intercept PAUSE, and L1's intercept should take priority, i.e. 927 * KVM should never handle a PAUSE intercept from L2. 928 */ 929 if (WARN_ON_ONCE(is_guest_mode(vcpu))) 930 return; 931 932 control->pause_filter_count = __grow_ple_window(old, 933 pause_filter_count, 934 pause_filter_count_grow, 935 pause_filter_count_max); 936 937 if (control->pause_filter_count != old) { 938 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 939 trace_kvm_ple_window_update(vcpu->vcpu_id, 940 control->pause_filter_count, old); 941 } 942 } 943 944 static void shrink_ple_window(struct kvm_vcpu *vcpu) 945 { 946 struct vcpu_svm *svm = to_svm(vcpu); 947 struct vmcb_control_area *control = &svm->vmcb->control; 948 int old = control->pause_filter_count; 949 950 /* Adjusting pause_filter_count makes no sense if PLE is disabled. */ 951 WARN_ON_ONCE(kvm_pause_in_guest(vcpu->kvm)); 952 953 if (is_guest_mode(vcpu)) 954 return; 955 956 control->pause_filter_count = 957 __shrink_ple_window(old, 958 pause_filter_count, 959 pause_filter_count_shrink, 960 pause_filter_count); 961 if (control->pause_filter_count != old) { 962 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 963 trace_kvm_ple_window_update(vcpu->vcpu_id, 964 control->pause_filter_count, old); 965 } 966 } 967 968 static void svm_hardware_unsetup(void) 969 { 970 int cpu; 971 972 avic_hardware_unsetup(); 973 974 sev_hardware_unsetup(); 975 976 for_each_possible_cpu(cpu) 977 svm_cpu_uninit(cpu); 978 979 __free_pages(__sme_pa_to_page(iopm_base), get_order(IOPM_SIZE)); 980 iopm_base = 0; 981 } 982 983 static void init_seg(struct vmcb_seg *seg) 984 { 985 seg->selector = 0; 986 seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK | 987 SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */ 988 seg->limit = 0xffff; 989 seg->base = 0; 990 } 991 992 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type) 993 { 994 seg->selector = 0; 995 seg->attrib = SVM_SELECTOR_P_MASK | type; 996 seg->limit = 0xffff; 997 seg->base = 0; 998 } 999 1000 static u64 svm_get_l2_tsc_offset(struct kvm_vcpu *vcpu) 1001 { 1002 struct vcpu_svm *svm = to_svm(vcpu); 1003 1004 return svm->nested.ctl.tsc_offset; 1005 } 1006 1007 static u64 svm_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu) 1008 { 1009 struct vcpu_svm *svm = to_svm(vcpu); 1010 1011 return svm->tsc_ratio_msr; 1012 } 1013 1014 static void svm_write_tsc_offset(struct kvm_vcpu *vcpu) 1015 { 1016 struct vcpu_svm *svm = to_svm(vcpu); 1017 1018 svm->vmcb01.ptr->control.tsc_offset = vcpu->arch.l1_tsc_offset; 1019 svm->vmcb->control.tsc_offset = vcpu->arch.tsc_offset; 1020 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 1021 } 1022 1023 void svm_write_tsc_multiplier(struct kvm_vcpu *vcpu) 1024 { 1025 preempt_disable(); 1026 if (to_svm(vcpu)->guest_state_loaded) 1027 __svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio); 1028 preempt_enable(); 1029 } 1030 1031 static bool svm_has_pending_gif_event(struct vcpu_svm *svm) 1032 { 1033 return svm->vcpu.arch.smi_pending || 1034 svm->vcpu.arch.nmi_pending || 1035 kvm_cpu_has_injectable_intr(&svm->vcpu) || 1036 kvm_apic_has_pending_init_or_sipi(&svm->vcpu); 1037 } 1038 1039 /* Evaluate instruction intercepts that depend on guest CPUID features. */ 1040 static void svm_recalc_instruction_intercepts(struct kvm_vcpu *vcpu) 1041 { 1042 struct vcpu_svm *svm = to_svm(vcpu); 1043 1044 /* 1045 * Intercept INVPCID if shadow paging is enabled to sync/free shadow 1046 * roots, or if INVPCID is disabled in the guest to inject #UD. 1047 */ 1048 if (kvm_cpu_cap_has(X86_FEATURE_INVPCID)) { 1049 if (!npt_enabled || 1050 !guest_cpu_cap_has(&svm->vcpu, X86_FEATURE_INVPCID)) 1051 svm_set_intercept(svm, INTERCEPT_INVPCID); 1052 else 1053 svm_clr_intercept(svm, INTERCEPT_INVPCID); 1054 } 1055 1056 if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP)) { 1057 if (guest_cpu_cap_has(vcpu, X86_FEATURE_RDTSCP)) 1058 svm_clr_intercept(svm, INTERCEPT_RDTSCP); 1059 else 1060 svm_set_intercept(svm, INTERCEPT_RDTSCP); 1061 } 1062 1063 /* 1064 * Intercept instructions that #UD if EFER.SVME=0, as SVME must be set 1065 * even when running the guest, i.e. hardware will only ever see 1066 * EFER.SVME=1. 1067 * 1068 * No need to toggle any of the vgif/vls/etc. enable bits here, as they 1069 * are set when the VMCB is initialized and never cleared (if the 1070 * relevant intercepts are set, the enablements are meaningless anyway). 1071 * 1072 * FIXME: When #GP is not intercepted, a #GP on these instructions (e.g. 1073 * due to CPL > 0) could be injected by hardware before the instruction 1074 * is intercepted, leading to #GP taking precedence over #UD from the 1075 * guest's perspective. 1076 */ 1077 if (!(vcpu->arch.efer & EFER_SVME)) { 1078 svm_set_intercept(svm, INTERCEPT_VMLOAD); 1079 svm_set_intercept(svm, INTERCEPT_VMSAVE); 1080 svm_set_intercept(svm, INTERCEPT_CLGI); 1081 svm_set_intercept(svm, INTERCEPT_STGI); 1082 } else { 1083 /* 1084 * If hardware supports Virtual VMLOAD VMSAVE then enable it 1085 * in VMCB and clear intercepts to avoid #VMEXIT. 1086 */ 1087 if (guest_cpuid_is_intel_compatible(vcpu)) { 1088 svm_set_intercept(svm, INTERCEPT_VMLOAD); 1089 svm_set_intercept(svm, INTERCEPT_VMSAVE); 1090 } else if (vls) { 1091 svm_clr_intercept(svm, INTERCEPT_VMLOAD); 1092 svm_clr_intercept(svm, INTERCEPT_VMSAVE); 1093 } 1094 1095 /* 1096 * Process pending events when clearing STGI/CLGI intercepts if 1097 * there's at least one pending event that is masked by GIF, so 1098 * that KVM re-evaluates if the intercept needs to be set again 1099 * to track when GIF is re-enabled (e.g. for NMI injection). 1100 */ 1101 if (vgif) { 1102 svm_clr_intercept(svm, INTERCEPT_CLGI); 1103 svm_clr_intercept(svm, INTERCEPT_STGI); 1104 1105 if (svm_has_pending_gif_event(svm)) 1106 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); 1107 } 1108 } 1109 1110 if (kvm_need_rdpmc_intercept(vcpu)) 1111 svm_set_intercept(svm, INTERCEPT_RDPMC); 1112 else 1113 svm_clr_intercept(svm, INTERCEPT_RDPMC); 1114 } 1115 1116 static void svm_recalc_intercepts(struct kvm_vcpu *vcpu) 1117 { 1118 svm_recalc_instruction_intercepts(vcpu); 1119 svm_recalc_msr_intercepts(vcpu); 1120 } 1121 1122 static void init_vmcb(struct kvm_vcpu *vcpu, bool init_event) 1123 { 1124 struct vcpu_svm *svm = to_svm(vcpu); 1125 struct vmcb *vmcb = svm->vmcb01.ptr; 1126 struct vmcb_control_area *control = &vmcb->control; 1127 struct vmcb_save_area *save = &vmcb->save; 1128 1129 svm_set_intercept(svm, INTERCEPT_CR0_READ); 1130 svm_set_intercept(svm, INTERCEPT_CR3_READ); 1131 svm_set_intercept(svm, INTERCEPT_CR4_READ); 1132 svm_set_intercept(svm, INTERCEPT_CR0_WRITE); 1133 svm_set_intercept(svm, INTERCEPT_CR3_WRITE); 1134 svm_set_intercept(svm, INTERCEPT_CR4_WRITE); 1135 svm_set_intercept(svm, INTERCEPT_CR8_WRITE); 1136 1137 set_dr_intercepts(svm); 1138 1139 set_exception_intercept(svm, PF_VECTOR); 1140 set_exception_intercept(svm, UD_VECTOR); 1141 set_exception_intercept(svm, MC_VECTOR); 1142 set_exception_intercept(svm, AC_VECTOR); 1143 set_exception_intercept(svm, DB_VECTOR); 1144 /* 1145 * Guest access to VMware backdoor ports could legitimately 1146 * trigger #GP because of TSS I/O permission bitmap. 1147 * We intercept those #GP and allow access to them anyway 1148 * as VMware does. 1149 */ 1150 if (enable_vmware_backdoor) 1151 set_exception_intercept(svm, GP_VECTOR); 1152 1153 svm_set_intercept(svm, INTERCEPT_INTR); 1154 svm_set_intercept(svm, INTERCEPT_NMI); 1155 1156 if (intercept_smi) 1157 svm_set_intercept(svm, INTERCEPT_SMI); 1158 1159 svm_set_intercept(svm, INTERCEPT_SELECTIVE_CR0); 1160 svm_set_intercept(svm, INTERCEPT_RDPMC); 1161 svm_set_intercept(svm, INTERCEPT_CPUID); 1162 svm_set_intercept(svm, INTERCEPT_INVD); 1163 svm_set_intercept(svm, INTERCEPT_INVLPG); 1164 svm_set_intercept(svm, INTERCEPT_INVLPGA); 1165 svm_set_intercept(svm, INTERCEPT_IOIO_PROT); 1166 svm_set_intercept(svm, INTERCEPT_MSR_PROT); 1167 svm_set_intercept(svm, INTERCEPT_TASK_SWITCH); 1168 svm_set_intercept(svm, INTERCEPT_SHUTDOWN); 1169 svm_set_intercept(svm, INTERCEPT_VMRUN); 1170 svm_set_intercept(svm, INTERCEPT_VMMCALL); 1171 svm_set_intercept(svm, INTERCEPT_VMLOAD); 1172 svm_set_intercept(svm, INTERCEPT_VMSAVE); 1173 svm_set_intercept(svm, INTERCEPT_STGI); 1174 svm_set_intercept(svm, INTERCEPT_CLGI); 1175 svm_set_intercept(svm, INTERCEPT_SKINIT); 1176 svm_set_intercept(svm, INTERCEPT_WBINVD); 1177 svm_set_intercept(svm, INTERCEPT_XSETBV); 1178 svm_set_intercept(svm, INTERCEPT_RDPRU); 1179 svm_set_intercept(svm, INTERCEPT_RSM); 1180 1181 if (!kvm_mwait_in_guest(vcpu->kvm)) { 1182 svm_set_intercept(svm, INTERCEPT_MONITOR); 1183 svm_set_intercept(svm, INTERCEPT_MWAIT); 1184 } 1185 1186 if (!kvm_hlt_in_guest(vcpu->kvm)) { 1187 if (cpu_feature_enabled(X86_FEATURE_IDLE_HLT)) 1188 svm_set_intercept(svm, INTERCEPT_IDLE_HLT); 1189 else 1190 svm_set_intercept(svm, INTERCEPT_HLT); 1191 } 1192 1193 control->iopm_base_pa = iopm_base; 1194 control->msrpm_base_pa = __sme_set(__pa(svm->msrpm)); 1195 control->int_ctl = V_INTR_MASKING_MASK; 1196 1197 init_seg(&save->es); 1198 init_seg(&save->ss); 1199 init_seg(&save->ds); 1200 init_seg(&save->fs); 1201 init_seg(&save->gs); 1202 1203 save->cs.selector = 0xf000; 1204 save->cs.base = 0xffff0000; 1205 /* Executable/Readable Code Segment */ 1206 save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK | 1207 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK; 1208 save->cs.limit = 0xffff; 1209 1210 save->gdtr.base = 0; 1211 save->gdtr.limit = 0xffff; 1212 save->idtr.base = 0; 1213 save->idtr.limit = 0xffff; 1214 1215 init_sys_seg(&save->ldtr, SEG_TYPE_LDT); 1216 init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16); 1217 1218 if (npt_enabled) { 1219 /* Setup VMCB for Nested Paging */ 1220 control->misc_ctl |= SVM_MISC_ENABLE_NP; 1221 svm_clr_intercept(svm, INTERCEPT_INVLPG); 1222 clr_exception_intercept(svm, PF_VECTOR); 1223 svm_clr_intercept(svm, INTERCEPT_CR3_READ); 1224 svm_clr_intercept(svm, INTERCEPT_CR3_WRITE); 1225 save->g_pat = vcpu->arch.pat; 1226 save->cr3 = 0; 1227 } 1228 1229 if (gmet_enabled) 1230 control->misc_ctl |= SVM_MISC_ENABLE_GMET; 1231 1232 svm->current_vmcb->asid_generation = 0; 1233 svm->asid = 0; 1234 1235 svm->nested.vmcb12_gpa = INVALID_GPA; 1236 svm->nested.last_vmcb12_gpa = INVALID_GPA; 1237 1238 if (!kvm_pause_in_guest(vcpu->kvm)) { 1239 control->pause_filter_count = pause_filter_count; 1240 if (pause_filter_thresh) 1241 control->pause_filter_thresh = pause_filter_thresh; 1242 svm_set_intercept(svm, INTERCEPT_PAUSE); 1243 } else { 1244 svm_clr_intercept(svm, INTERCEPT_PAUSE); 1245 } 1246 1247 if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS)) 1248 svm->vmcb->control.erap_ctl |= ERAP_CONTROL_ALLOW_LARGER_RAP; 1249 1250 if (enable_apicv && irqchip_in_kernel(vcpu->kvm)) 1251 avic_init_vmcb(svm, vmcb); 1252 1253 if (vnmi) 1254 svm->vmcb->control.int_ctl |= V_NMI_ENABLE_MASK; 1255 1256 if (vgif) 1257 svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK; 1258 1259 if (vls) 1260 svm->vmcb->control.misc_ctl2 |= SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE; 1261 1262 if (vcpu->kvm->arch.bus_lock_detection_enabled) 1263 svm_set_intercept(svm, INTERCEPT_BUSLOCK); 1264 1265 if (is_sev_guest(vcpu)) 1266 sev_init_vmcb(svm, init_event); 1267 1268 svm_hv_init_vmcb(vmcb); 1269 1270 kvm_make_request(KVM_REQ_RECALC_INTERCEPTS, vcpu); 1271 1272 vmcb_mark_all_dirty(vmcb); 1273 1274 enable_gif(svm); 1275 } 1276 1277 static void __svm_vcpu_reset(struct kvm_vcpu *vcpu) 1278 { 1279 struct vcpu_svm *svm = to_svm(vcpu); 1280 1281 svm_init_osvw(vcpu); 1282 1283 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS)) 1284 vcpu->arch.microcode_version = 0x01000065; 1285 svm->tsc_ratio_msr = kvm_caps.default_tsc_scaling_ratio; 1286 1287 svm->nmi_masked = false; 1288 svm->awaiting_iret_completion = false; 1289 } 1290 1291 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) 1292 { 1293 struct vcpu_svm *svm = to_svm(vcpu); 1294 1295 svm->spec_ctrl = 0; 1296 svm->virt_spec_ctrl = 0; 1297 1298 init_vmcb(vcpu, init_event); 1299 1300 if (!init_event) 1301 __svm_vcpu_reset(vcpu); 1302 } 1303 1304 void svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb) 1305 { 1306 svm->current_vmcb = target_vmcb; 1307 svm->vmcb = target_vmcb->ptr; 1308 } 1309 1310 static int svm_vcpu_precreate(struct kvm *kvm) 1311 { 1312 return avic_alloc_physical_id_table(kvm); 1313 } 1314 1315 static int svm_vcpu_create(struct kvm_vcpu *vcpu) 1316 { 1317 struct vcpu_svm *svm; 1318 struct page *vmcb01_page; 1319 int err; 1320 1321 BUILD_BUG_ON(offsetof(struct vcpu_svm, vcpu) != 0); 1322 svm = to_svm(vcpu); 1323 1324 err = -ENOMEM; 1325 vmcb01_page = snp_safe_alloc_page(); 1326 if (!vmcb01_page) 1327 goto out; 1328 1329 err = sev_vcpu_create(vcpu); 1330 if (err) 1331 goto error_free_vmcb_page; 1332 1333 err = avic_init_vcpu(svm); 1334 if (err) 1335 goto error_free_sev; 1336 1337 svm->msrpm = svm_vcpu_alloc_msrpm(); 1338 if (!svm->msrpm) { 1339 err = -ENOMEM; 1340 goto error_free_sev; 1341 } 1342 1343 svm->x2avic_msrs_intercepted = true; 1344 svm->lbr_msrs_intercepted = true; 1345 1346 svm->vmcb01.ptr = page_address(vmcb01_page); 1347 svm->vmcb01.pa = __sme_set(page_to_pfn(vmcb01_page) << PAGE_SHIFT); 1348 svm_switch_vmcb(svm, &svm->vmcb01); 1349 1350 svm->guest_state_loaded = false; 1351 1352 return 0; 1353 1354 error_free_sev: 1355 sev_free_vcpu(vcpu); 1356 error_free_vmcb_page: 1357 __free_page(vmcb01_page); 1358 out: 1359 return err; 1360 } 1361 1362 static void svm_vcpu_free(struct kvm_vcpu *vcpu) 1363 { 1364 struct vcpu_svm *svm = to_svm(vcpu); 1365 1366 WARN_ON_ONCE(!list_empty(&svm->ir_list)); 1367 1368 svm_leave_nested(vcpu); 1369 svm_free_nested(svm); 1370 1371 sev_free_vcpu(vcpu); 1372 1373 __free_page(__sme_pa_to_page(svm->vmcb01.pa)); 1374 svm_vcpu_free_msrpm(svm->msrpm); 1375 } 1376 1377 #ifdef CONFIG_CPU_MITIGATIONS 1378 static DEFINE_SPINLOCK(srso_lock); 1379 static atomic_t srso_nr_vms; 1380 1381 static void svm_srso_clear_bp_spec_reduce(void *ign) 1382 { 1383 struct svm_cpu_data *sd = this_cpu_ptr(&svm_data); 1384 1385 if (!sd->bp_spec_reduce_set) 1386 return; 1387 1388 msr_clear_bit(MSR_ZEN4_BP_CFG, MSR_ZEN4_BP_CFG_BP_SPEC_REDUCE_BIT); 1389 sd->bp_spec_reduce_set = false; 1390 } 1391 1392 static void svm_srso_vm_destroy(void) 1393 { 1394 if (!cpu_feature_enabled(X86_FEATURE_SRSO_BP_SPEC_REDUCE)) 1395 return; 1396 1397 if (atomic_dec_return(&srso_nr_vms)) 1398 return; 1399 1400 guard(spinlock)(&srso_lock); 1401 1402 /* 1403 * Verify a new VM didn't come along, acquire the lock, and increment 1404 * the count before this task acquired the lock. 1405 */ 1406 if (atomic_read(&srso_nr_vms)) 1407 return; 1408 1409 on_each_cpu(svm_srso_clear_bp_spec_reduce, NULL, 1); 1410 } 1411 1412 static void svm_srso_vm_init(void) 1413 { 1414 if (!cpu_feature_enabled(X86_FEATURE_SRSO_BP_SPEC_REDUCE)) 1415 return; 1416 1417 /* 1418 * Acquire the lock on 0 => 1 transitions to ensure a potential 1 => 0 1419 * transition, i.e. destroying the last VM, is fully complete, e.g. so 1420 * that a delayed IPI doesn't clear BP_SPEC_REDUCE after a vCPU runs. 1421 */ 1422 if (atomic_inc_not_zero(&srso_nr_vms)) 1423 return; 1424 1425 guard(spinlock)(&srso_lock); 1426 1427 atomic_inc(&srso_nr_vms); 1428 } 1429 #else 1430 static void svm_srso_vm_init(void) { } 1431 static void svm_srso_vm_destroy(void) { } 1432 #endif 1433 1434 static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu) 1435 { 1436 struct vcpu_svm *svm = to_svm(vcpu); 1437 struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu); 1438 1439 if (is_sev_es_guest(vcpu)) 1440 sev_es_unmap_ghcb(svm); 1441 1442 if (svm->guest_state_loaded) 1443 return; 1444 1445 /* 1446 * Save additional host state that will be restored on VMEXIT (sev-es) 1447 * or subsequent vmload of host save area. 1448 */ 1449 vmsave(sd->save_area_pa); 1450 if (is_sev_es_guest(vcpu)) 1451 sev_es_prepare_switch_to_guest(svm, sev_es_host_save_area(sd)); 1452 1453 if (tsc_scaling) 1454 __svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio); 1455 1456 /* 1457 * TSC_AUX is always virtualized (context switched by hardware) for 1458 * SEV-ES guests when the feature is available. For non-SEV-ES guests, 1459 * context switch TSC_AUX via the user_return MSR infrastructure (not 1460 * all CPUs support TSC_AUX virtualization). 1461 */ 1462 if (likely(tsc_aux_uret_slot >= 0) && 1463 (!boot_cpu_has(X86_FEATURE_V_TSC_AUX) || !is_sev_es_guest(vcpu))) 1464 kvm_set_user_return_msr(tsc_aux_uret_slot, svm->tsc_aux, -1ull); 1465 1466 if (cpu_feature_enabled(X86_FEATURE_SRSO_BP_SPEC_REDUCE) && 1467 !sd->bp_spec_reduce_set) { 1468 sd->bp_spec_reduce_set = true; 1469 msr_set_bit(MSR_ZEN4_BP_CFG, MSR_ZEN4_BP_CFG_BP_SPEC_REDUCE_BIT); 1470 } 1471 svm->guest_state_loaded = true; 1472 } 1473 1474 static void svm_prepare_host_switch(struct kvm_vcpu *vcpu) 1475 { 1476 to_svm(vcpu)->guest_state_loaded = false; 1477 } 1478 1479 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1480 { 1481 if (vcpu->scheduled_out && !kvm_pause_in_guest(vcpu->kvm)) 1482 shrink_ple_window(vcpu); 1483 1484 if (kvm_vcpu_apicv_active(vcpu)) 1485 avic_vcpu_load(vcpu, cpu); 1486 } 1487 1488 static void svm_vcpu_put(struct kvm_vcpu *vcpu) 1489 { 1490 if (kvm_vcpu_apicv_active(vcpu)) 1491 avic_vcpu_put(vcpu); 1492 1493 svm_prepare_host_switch(vcpu); 1494 1495 ++vcpu->stat.host_state_reload; 1496 } 1497 1498 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu) 1499 { 1500 struct vcpu_svm *svm = to_svm(vcpu); 1501 unsigned long rflags = svm->vmcb->save.rflags; 1502 1503 if (svm->nmi_singlestep) { 1504 /* Hide our flags if they were not set by the guest */ 1505 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF)) 1506 rflags &= ~X86_EFLAGS_TF; 1507 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF)) 1508 rflags &= ~X86_EFLAGS_RF; 1509 } 1510 return rflags; 1511 } 1512 1513 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) 1514 { 1515 if (to_svm(vcpu)->nmi_singlestep) 1516 rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF); 1517 1518 /* 1519 * Any change of EFLAGS.VM is accompanied by a reload of SS 1520 * (caused by either a task switch or an inter-privilege IRET), 1521 * so we do not need to update the CPL here. 1522 */ 1523 to_svm(vcpu)->vmcb->save.rflags = rflags; 1524 } 1525 1526 static bool svm_get_if_flag(struct kvm_vcpu *vcpu) 1527 { 1528 struct vmcb *vmcb = to_svm(vcpu)->vmcb; 1529 1530 return is_sev_es_guest(vcpu) 1531 ? vmcb->control.int_state & SVM_GUEST_INTERRUPT_MASK 1532 : kvm_get_rflags(vcpu) & X86_EFLAGS_IF; 1533 } 1534 1535 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) 1536 { 1537 kvm_register_mark_available(vcpu, reg); 1538 1539 switch (reg) { 1540 case VCPU_REG_PDPTR: 1541 /* 1542 * When !npt_enabled, vcpu->pdptrs[] is already available since 1543 * it is always updated per SDM when moving to CRs. 1544 */ 1545 if (npt_enabled) 1546 load_pdptrs(vcpu, kvm_read_cr3(vcpu)); 1547 break; 1548 default: 1549 KVM_BUG_ON(1, vcpu->kvm); 1550 } 1551 } 1552 1553 static void svm_set_vintr(struct vcpu_svm *svm) 1554 { 1555 struct vmcb_control_area *control; 1556 1557 /* 1558 * The following fields are ignored when AVIC is enabled 1559 */ 1560 WARN_ON(kvm_vcpu_apicv_activated(&svm->vcpu)); 1561 1562 svm_set_intercept(svm, INTERCEPT_VINTR); 1563 1564 /* 1565 * Recalculating intercepts may have cleared the VINTR intercept. If 1566 * V_INTR_MASKING is enabled in vmcb12, then the effective RFLAGS.IF 1567 * for L1 physical interrupts is L1's RFLAGS.IF at the time of VMRUN. 1568 * Requesting an interrupt window if save.RFLAGS.IF=0 is pointless as 1569 * interrupts will never be unblocked while L2 is running. 1570 */ 1571 if (!svm_is_intercept(svm, INTERCEPT_VINTR)) 1572 return; 1573 1574 /* 1575 * This is just a dummy VINTR to actually cause a vmexit to happen. 1576 * Actual injection of virtual interrupts happens through EVENTINJ. 1577 */ 1578 control = &svm->vmcb->control; 1579 control->int_vector = 0x0; 1580 control->int_ctl &= ~V_INTR_PRIO_MASK; 1581 control->int_ctl |= V_IRQ_MASK | 1582 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT); 1583 vmcb_mark_dirty(svm->vmcb, VMCB_INTR); 1584 } 1585 1586 static void svm_clear_vintr(struct vcpu_svm *svm) 1587 { 1588 svm_clr_intercept(svm, INTERCEPT_VINTR); 1589 1590 /* Drop int_ctl fields related to VINTR injection. */ 1591 svm->vmcb->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK; 1592 if (is_guest_mode(&svm->vcpu)) { 1593 svm->vmcb01.ptr->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK; 1594 1595 WARN_ON((svm->vmcb->control.int_ctl & V_TPR_MASK) != 1596 (svm->nested.ctl.int_ctl & V_TPR_MASK)); 1597 1598 svm->vmcb->control.int_ctl |= svm->nested.ctl.int_ctl & 1599 V_IRQ_INJECTION_BITS_MASK; 1600 1601 svm->vmcb->control.int_vector = svm->nested.ctl.int_vector; 1602 } 1603 1604 vmcb_mark_dirty(svm->vmcb, VMCB_INTR); 1605 } 1606 1607 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg) 1608 { 1609 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save; 1610 struct vmcb_save_area *save01 = &to_svm(vcpu)->vmcb01.ptr->save; 1611 1612 switch (seg) { 1613 case VCPU_SREG_CS: return &save->cs; 1614 case VCPU_SREG_DS: return &save->ds; 1615 case VCPU_SREG_ES: return &save->es; 1616 case VCPU_SREG_FS: return &save01->fs; 1617 case VCPU_SREG_GS: return &save01->gs; 1618 case VCPU_SREG_SS: return &save->ss; 1619 case VCPU_SREG_TR: return &save01->tr; 1620 case VCPU_SREG_LDTR: return &save01->ldtr; 1621 } 1622 BUG(); 1623 return NULL; 1624 } 1625 1626 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg) 1627 { 1628 struct vmcb_seg *s = svm_seg(vcpu, seg); 1629 1630 return s->base; 1631 } 1632 1633 static void svm_get_segment(struct kvm_vcpu *vcpu, 1634 struct kvm_segment *var, int seg) 1635 { 1636 struct vmcb_seg *s = svm_seg(vcpu, seg); 1637 1638 var->base = s->base; 1639 var->limit = s->limit; 1640 var->selector = s->selector; 1641 var->type = s->attrib & SVM_SELECTOR_TYPE_MASK; 1642 var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1; 1643 var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3; 1644 var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1; 1645 var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1; 1646 var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1; 1647 var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1; 1648 1649 /* 1650 * AMD CPUs circa 2014 track the G bit for all segments except CS. 1651 * However, the SVM spec states that the G bit is not observed by the 1652 * CPU, and some VMware virtual CPUs drop the G bit for all segments. 1653 * So let's synthesize a legal G bit for all segments, this helps 1654 * running KVM nested. It also helps cross-vendor migration, because 1655 * Intel's vmentry has a check on the 'G' bit. 1656 */ 1657 var->g = s->limit > 0xfffff; 1658 1659 /* 1660 * AMD's VMCB does not have an explicit unusable field, so emulate it 1661 * for cross vendor migration purposes by "not present" 1662 */ 1663 var->unusable = !var->present; 1664 1665 switch (seg) { 1666 case VCPU_SREG_TR: 1667 /* 1668 * Work around a bug where the busy flag in the tr selector 1669 * isn't exposed 1670 */ 1671 var->type |= 0x2; 1672 break; 1673 case VCPU_SREG_DS: 1674 case VCPU_SREG_ES: 1675 case VCPU_SREG_FS: 1676 case VCPU_SREG_GS: 1677 /* 1678 * The accessed bit must always be set in the segment 1679 * descriptor cache, although it can be cleared in the 1680 * descriptor, the cached bit always remains at 1. Since 1681 * Intel has a check on this, set it here to support 1682 * cross-vendor migration. 1683 */ 1684 if (!var->unusable) 1685 var->type |= 0x1; 1686 break; 1687 case VCPU_SREG_SS: 1688 /* 1689 * On AMD CPUs sometimes the DB bit in the segment 1690 * descriptor is left as 1, although the whole segment has 1691 * been made unusable. Clear it here to pass an Intel VMX 1692 * entry check when cross vendor migrating. 1693 */ 1694 if (var->unusable) 1695 var->db = 0; 1696 /* This is symmetric with svm_set_segment() */ 1697 var->dpl = to_svm(vcpu)->vmcb->save.cpl; 1698 break; 1699 } 1700 } 1701 1702 static int svm_get_cpl(struct kvm_vcpu *vcpu) 1703 { 1704 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save; 1705 1706 return save->cpl; 1707 } 1708 1709 static void svm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) 1710 { 1711 struct kvm_segment cs; 1712 1713 svm_get_segment(vcpu, &cs, VCPU_SREG_CS); 1714 *db = cs.db; 1715 *l = cs.l; 1716 } 1717 1718 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 1719 { 1720 struct vcpu_svm *svm = to_svm(vcpu); 1721 1722 dt->size = svm->vmcb->save.idtr.limit; 1723 dt->address = svm->vmcb->save.idtr.base; 1724 } 1725 1726 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 1727 { 1728 struct vcpu_svm *svm = to_svm(vcpu); 1729 1730 svm->vmcb->save.idtr.limit = dt->size; 1731 svm->vmcb->save.idtr.base = dt->address ; 1732 vmcb_mark_dirty(svm->vmcb, VMCB_DT); 1733 } 1734 1735 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 1736 { 1737 struct vcpu_svm *svm = to_svm(vcpu); 1738 1739 dt->size = svm->vmcb->save.gdtr.limit; 1740 dt->address = svm->vmcb->save.gdtr.base; 1741 } 1742 1743 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 1744 { 1745 struct vcpu_svm *svm = to_svm(vcpu); 1746 1747 svm->vmcb->save.gdtr.limit = dt->size; 1748 svm->vmcb->save.gdtr.base = dt->address ; 1749 vmcb_mark_dirty(svm->vmcb, VMCB_DT); 1750 } 1751 1752 static void sev_post_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) 1753 { 1754 struct vcpu_svm *svm = to_svm(vcpu); 1755 1756 /* 1757 * For guests that don't set guest_state_protected, the cr3 update is 1758 * handled via kvm_mmu_load() while entering the guest. For guests 1759 * that do (SEV-ES/SEV-SNP), the cr3 update needs to be written to 1760 * VMCB save area now, since the save area will become the initial 1761 * contents of the VMSA, and future VMCB save area updates won't be 1762 * seen. 1763 */ 1764 if (is_sev_es_guest(vcpu)) { 1765 svm->vmcb->save.cr3 = cr3; 1766 vmcb_mark_dirty(svm->vmcb, VMCB_CR); 1767 } 1768 } 1769 1770 static bool svm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 1771 { 1772 return true; 1773 } 1774 1775 void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 1776 { 1777 struct vcpu_svm *svm = to_svm(vcpu); 1778 u64 hcr0 = cr0; 1779 bool old_paging = is_paging(vcpu); 1780 1781 #ifdef CONFIG_X86_64 1782 if (vcpu->arch.efer & EFER_LME) { 1783 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) { 1784 vcpu->arch.efer |= EFER_LMA; 1785 if (!vcpu->arch.guest_state_protected) 1786 svm->vmcb->save.efer |= EFER_LMA | EFER_LME; 1787 } 1788 1789 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) { 1790 vcpu->arch.efer &= ~EFER_LMA; 1791 if (!vcpu->arch.guest_state_protected) 1792 svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME); 1793 } 1794 } 1795 #endif 1796 vcpu->arch.cr0 = cr0; 1797 1798 if (!npt_enabled) { 1799 hcr0 |= X86_CR0_PG | X86_CR0_WP; 1800 if (old_paging != is_paging(vcpu)) 1801 svm_set_cr4(vcpu, kvm_read_cr4(vcpu)); 1802 } 1803 1804 /* 1805 * re-enable caching here because the QEMU bios 1806 * does not do it - this results in some delay at 1807 * reboot 1808 */ 1809 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED)) 1810 hcr0 &= ~(X86_CR0_CD | X86_CR0_NW); 1811 1812 svm->vmcb->save.cr0 = hcr0; 1813 vmcb_mark_dirty(svm->vmcb, VMCB_CR); 1814 1815 /* 1816 * SEV-ES guests must always keep the CR intercepts cleared. CR 1817 * tracking is done using the CR write traps. 1818 */ 1819 if (is_sev_es_guest(vcpu)) 1820 return; 1821 1822 if (hcr0 == cr0) { 1823 /* Selective CR0 write remains on. */ 1824 svm_clr_intercept(svm, INTERCEPT_CR0_READ); 1825 svm_clr_intercept(svm, INTERCEPT_CR0_WRITE); 1826 } else { 1827 svm_set_intercept(svm, INTERCEPT_CR0_READ); 1828 svm_set_intercept(svm, INTERCEPT_CR0_WRITE); 1829 } 1830 } 1831 1832 static bool svm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 1833 { 1834 return true; 1835 } 1836 1837 void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 1838 { 1839 unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE; 1840 unsigned long old_cr4 = vcpu->arch.cr4; 1841 1842 vcpu->arch.cr4 = cr4; 1843 if (!npt_enabled) { 1844 cr4 |= X86_CR4_PAE; 1845 1846 if (!is_paging(vcpu)) 1847 cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE); 1848 } 1849 cr4 |= host_cr4_mce; 1850 to_svm(vcpu)->vmcb->save.cr4 = cr4; 1851 vmcb_mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR); 1852 1853 if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE)) 1854 vcpu->arch.cpuid_dynamic_bits_dirty = true; 1855 } 1856 1857 static void svm_set_segment(struct kvm_vcpu *vcpu, 1858 struct kvm_segment *var, int seg) 1859 { 1860 struct vcpu_svm *svm = to_svm(vcpu); 1861 struct vmcb_seg *s = svm_seg(vcpu, seg); 1862 1863 s->base = var->base; 1864 s->limit = var->limit; 1865 s->selector = var->selector; 1866 s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK); 1867 s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT; 1868 s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT; 1869 s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT; 1870 s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT; 1871 s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT; 1872 s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT; 1873 s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT; 1874 1875 /* 1876 * This is always accurate, except if SYSRET returned to a segment 1877 * with SS.DPL != 3. Intel does not have this quirk, and always 1878 * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it 1879 * would entail passing the CPL to userspace and back. 1880 */ 1881 if (seg == VCPU_SREG_SS) 1882 /* This is symmetric with svm_get_segment() */ 1883 svm->vmcb->save.cpl = (var->dpl & 3); 1884 1885 vmcb_mark_dirty(svm->vmcb, VMCB_SEG); 1886 } 1887 1888 static void svm_update_exception_bitmap(struct kvm_vcpu *vcpu) 1889 { 1890 struct vcpu_svm *svm = to_svm(vcpu); 1891 1892 clr_exception_intercept(svm, BP_VECTOR); 1893 1894 if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) { 1895 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) 1896 set_exception_intercept(svm, BP_VECTOR); 1897 } 1898 } 1899 1900 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd) 1901 { 1902 if (sd->next_asid > sd->max_asid) { 1903 ++sd->asid_generation; 1904 sd->next_asid = sd->min_asid; 1905 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID; 1906 vmcb_mark_dirty(svm->vmcb, VMCB_ASID); 1907 } 1908 1909 svm->current_vmcb->asid_generation = sd->asid_generation; 1910 svm->asid = sd->next_asid++; 1911 } 1912 1913 static void svm_set_dr6(struct kvm_vcpu *vcpu, unsigned long value) 1914 { 1915 struct vmcb *vmcb = to_svm(vcpu)->vmcb; 1916 1917 if (vcpu->arch.guest_state_protected) 1918 return; 1919 1920 if (unlikely(value != vmcb->save.dr6)) { 1921 vmcb->save.dr6 = value; 1922 vmcb_mark_dirty(vmcb, VMCB_DR); 1923 } 1924 } 1925 1926 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu) 1927 { 1928 struct vcpu_svm *svm = to_svm(vcpu); 1929 1930 if (WARN_ON_ONCE(is_sev_es_guest(vcpu))) 1931 return; 1932 1933 get_debugreg(vcpu->arch.db[0], 0); 1934 get_debugreg(vcpu->arch.db[1], 1); 1935 get_debugreg(vcpu->arch.db[2], 2); 1936 get_debugreg(vcpu->arch.db[3], 3); 1937 /* 1938 * We cannot reset svm->vmcb->save.dr6 to DR6_ACTIVE_LOW here, 1939 * because db_interception might need it. We can do it before vmentry. 1940 */ 1941 vcpu->arch.dr6 = svm->vmcb->save.dr6; 1942 vcpu->arch.dr7 = svm->vmcb->save.dr7; 1943 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT; 1944 set_dr_intercepts(svm); 1945 } 1946 1947 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value) 1948 { 1949 struct vcpu_svm *svm = to_svm(vcpu); 1950 1951 if (vcpu->arch.guest_state_protected) 1952 return; 1953 1954 svm->vmcb->save.dr7 = value; 1955 vmcb_mark_dirty(svm->vmcb, VMCB_DR); 1956 } 1957 1958 static int pf_interception(struct kvm_vcpu *vcpu) 1959 { 1960 struct vcpu_svm *svm = to_svm(vcpu); 1961 1962 u64 fault_address = svm->vmcb->control.exit_info_2; 1963 u64 error_code = svm->vmcb->control.exit_info_1; 1964 1965 return kvm_handle_page_fault(vcpu, error_code, fault_address, 1966 static_cpu_has(X86_FEATURE_DECODEASSISTS) ? 1967 svm->vmcb->control.insn_bytes : NULL, 1968 svm->vmcb->control.insn_len); 1969 } 1970 1971 static int svm_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, 1972 void *insn, int insn_len); 1973 1974 static int npf_interception(struct kvm_vcpu *vcpu) 1975 { 1976 struct vcpu_svm *svm = to_svm(vcpu); 1977 int rc; 1978 1979 u64 error_code = svm->vmcb->control.exit_info_1; 1980 gpa_t gpa = svm->vmcb->control.exit_info_2; 1981 1982 /* 1983 * WARN if hardware generates a fault with an error code that collides 1984 * with KVM-defined sythentic flags. Clear the flags and continue on, 1985 * i.e. don't terminate the VM, as KVM can't possibly be relying on a 1986 * flag that KVM doesn't know about. 1987 */ 1988 if (WARN_ON_ONCE(error_code & PFERR_SYNTHETIC_MASK)) 1989 error_code &= ~PFERR_SYNTHETIC_MASK; 1990 1991 /* 1992 * Expedite fast MMIO kicks if the next RIP is known and KVM is allowed 1993 * emulate a page fault, e.g. skipping the current instruction is wrong 1994 * if the #NPF occurred while vectoring an event. 1995 */ 1996 if ((error_code & PFERR_RSVD_MASK) && !is_guest_mode(vcpu)) { 1997 const int emul_type = EMULTYPE_PF | EMULTYPE_NO_DECODE; 1998 1999 if (svm_check_emulate_instruction(vcpu, emul_type, NULL, 0)) 2000 return 1; 2001 2002 if (nrips && svm->vmcb->control.next_rip && 2003 !kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) { 2004 trace_kvm_fast_mmio(gpa); 2005 return kvm_skip_emulated_instruction(vcpu); 2006 } 2007 } 2008 2009 if (!is_sev_es_guest(vcpu) && 2010 (svm->vmcb->control.misc_ctl & SVM_MISC_ENABLE_GMET) && 2011 (error_code & PFERR_FETCH_MASK)) { 2012 /* 2013 * Work around errata 1218: EXITINFO1[2] May Be Incorrectly Set 2014 * When GMET (Guest Mode Execute Trap extension) is Enabled 2015 */ 2016 error_code |= PFERR_USER_MASK; 2017 if (svm_get_cpl(vcpu) != 3) 2018 error_code &= ~PFERR_USER_MASK; 2019 } 2020 2021 if (is_sev_snp_guest(vcpu) && (error_code & PFERR_GUEST_ENC_MASK)) 2022 error_code |= PFERR_PRIVATE_ACCESS; 2023 2024 trace_kvm_page_fault(vcpu, gpa, error_code); 2025 rc = kvm_mmu_page_fault(vcpu, gpa, error_code, 2026 static_cpu_has(X86_FEATURE_DECODEASSISTS) ? 2027 svm->vmcb->control.insn_bytes : NULL, 2028 svm->vmcb->control.insn_len); 2029 2030 if (rc > 0 && error_code & PFERR_GUEST_RMP_MASK) 2031 sev_handle_rmp_fault(vcpu, gpa, error_code); 2032 2033 return rc; 2034 } 2035 2036 static int db_interception(struct kvm_vcpu *vcpu) 2037 { 2038 struct kvm_run *kvm_run = vcpu->run; 2039 struct vcpu_svm *svm = to_svm(vcpu); 2040 2041 if (!(vcpu->guest_debug & 2042 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) && 2043 !svm->nmi_singlestep) { 2044 u32 payload = svm->vmcb->save.dr6 ^ DR6_ACTIVE_LOW; 2045 kvm_queue_exception_p(vcpu, DB_VECTOR, payload); 2046 return 1; 2047 } 2048 2049 if (svm->nmi_singlestep) { 2050 disable_nmi_singlestep(svm); 2051 /* Make sure we check for pending NMIs upon entry */ 2052 kvm_make_request(KVM_REQ_EVENT, vcpu); 2053 } 2054 2055 if (vcpu->guest_debug & 2056 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) { 2057 kvm_run->exit_reason = KVM_EXIT_DEBUG; 2058 kvm_run->debug.arch.dr6 = svm->vmcb->save.dr6; 2059 kvm_run->debug.arch.dr7 = svm->vmcb->save.dr7; 2060 kvm_run->debug.arch.pc = 2061 svm->vmcb->save.cs.base + svm->vmcb->save.rip; 2062 kvm_run->debug.arch.exception = DB_VECTOR; 2063 return 0; 2064 } 2065 2066 return 1; 2067 } 2068 2069 static int bp_interception(struct kvm_vcpu *vcpu) 2070 { 2071 struct vcpu_svm *svm = to_svm(vcpu); 2072 struct kvm_run *kvm_run = vcpu->run; 2073 2074 kvm_run->exit_reason = KVM_EXIT_DEBUG; 2075 kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip; 2076 kvm_run->debug.arch.exception = BP_VECTOR; 2077 return 0; 2078 } 2079 2080 static int ud_interception(struct kvm_vcpu *vcpu) 2081 { 2082 return handle_ud(vcpu); 2083 } 2084 2085 static int ac_interception(struct kvm_vcpu *vcpu) 2086 { 2087 kvm_queue_exception_e(vcpu, AC_VECTOR, 0); 2088 return 1; 2089 } 2090 2091 static bool is_erratum_383(void) 2092 { 2093 int i; 2094 u64 value; 2095 2096 if (!erratum_383_found) 2097 return false; 2098 2099 if (native_read_msr_safe(MSR_IA32_MC0_STATUS, &value)) 2100 return false; 2101 2102 /* Bit 62 may or may not be set for this mce */ 2103 value &= ~(1ULL << 62); 2104 2105 if (value != 0xb600000000010015ULL) 2106 return false; 2107 2108 /* Clear MCi_STATUS registers */ 2109 for (i = 0; i < 6; ++i) 2110 native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0); 2111 2112 if (!native_read_msr_safe(MSR_IA32_MCG_STATUS, &value)) { 2113 value &= ~(1ULL << 2); 2114 native_write_msr_safe(MSR_IA32_MCG_STATUS, value); 2115 } 2116 2117 /* Flush tlb to evict multi-match entries */ 2118 __flush_tlb_all(); 2119 2120 return true; 2121 } 2122 2123 static void svm_handle_mce(struct kvm_vcpu *vcpu) 2124 { 2125 if (is_erratum_383()) { 2126 /* 2127 * Erratum 383 triggered. Guest state is corrupt so kill the 2128 * guest. 2129 */ 2130 pr_err("Guest triggered AMD Erratum 383\n"); 2131 2132 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 2133 2134 return; 2135 } 2136 2137 /* 2138 * On an #MC intercept the MCE handler is not called automatically in 2139 * the host. So do it by hand here. 2140 */ 2141 kvm_machine_check(); 2142 } 2143 2144 static int mc_interception(struct kvm_vcpu *vcpu) 2145 { 2146 return 1; 2147 } 2148 2149 static int shutdown_interception(struct kvm_vcpu *vcpu) 2150 { 2151 struct kvm_run *kvm_run = vcpu->run; 2152 struct vcpu_svm *svm = to_svm(vcpu); 2153 2154 2155 /* 2156 * VMCB is undefined after a SHUTDOWN intercept. INIT the vCPU to put 2157 * the VMCB in a known good state. Unfortuately, KVM doesn't have 2158 * KVM_MP_STATE_SHUTDOWN and can't add it without potentially breaking 2159 * userspace. At a platform view, INIT is acceptable behavior as 2160 * there exist bare metal platforms that automatically INIT the CPU 2161 * in response to shutdown. 2162 * 2163 * The VM save area for SEV-ES guests has already been encrypted so it 2164 * cannot be reinitialized, i.e. synthesizing INIT is futile. 2165 */ 2166 if (!is_sev_es_guest(vcpu)) { 2167 clear_page(svm->vmcb); 2168 #ifdef CONFIG_KVM_SMM 2169 if (is_smm(vcpu)) 2170 kvm_smm_changed(vcpu, false); 2171 #endif 2172 kvm_vcpu_reset(vcpu, true); 2173 } 2174 2175 kvm_run->exit_reason = KVM_EXIT_SHUTDOWN; 2176 return 0; 2177 } 2178 2179 static int io_interception(struct kvm_vcpu *vcpu) 2180 { 2181 struct vcpu_svm *svm = to_svm(vcpu); 2182 u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */ 2183 int size, in, string; 2184 unsigned port; 2185 2186 ++vcpu->stat.io_exits; 2187 string = (io_info & SVM_IOIO_STR_MASK) != 0; 2188 in = (io_info & SVM_IOIO_TYPE_MASK) != 0; 2189 port = io_info >> 16; 2190 size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT; 2191 2192 if (string) { 2193 if (is_sev_es_guest(vcpu)) 2194 return sev_es_string_io(svm, size, port, in); 2195 else 2196 return kvm_emulate_instruction(vcpu, 0); 2197 } 2198 2199 svm->next_rip = svm->vmcb->control.exit_info_2; 2200 2201 return kvm_fast_pio(vcpu, size, port, in); 2202 } 2203 2204 static int nmi_interception(struct kvm_vcpu *vcpu) 2205 { 2206 return 1; 2207 } 2208 2209 static int smi_interception(struct kvm_vcpu *vcpu) 2210 { 2211 return 1; 2212 } 2213 2214 static int intr_interception(struct kvm_vcpu *vcpu) 2215 { 2216 ++vcpu->stat.irq_exits; 2217 return 1; 2218 } 2219 2220 static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload) 2221 { 2222 u64 vmcb12_gpa = kvm_rax_read(vcpu); 2223 struct vcpu_svm *svm = to_svm(vcpu); 2224 struct vmcb *vmcb12; 2225 struct kvm_host_map map; 2226 int ret; 2227 2228 if (nested_svm_check_permissions(vcpu)) 2229 return 1; 2230 2231 if (!page_address_valid(vcpu, vmcb12_gpa)) { 2232 kvm_inject_gp(vcpu, 0); 2233 return 1; 2234 } 2235 2236 if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcb12_gpa), &map)) 2237 return kvm_handle_memory_failure(vcpu, X86EMUL_IO_NEEDED, NULL); 2238 2239 vmcb12 = map.hva; 2240 2241 ret = kvm_skip_emulated_instruction(vcpu); 2242 2243 /* KVM always performs VMLOAD/VMSAVE on VMCB01 (see __svm_vcpu_run()) */ 2244 if (vmload) { 2245 svm_copy_vmloadsave_state(svm->vmcb01.ptr, vmcb12); 2246 svm->sysenter_eip_hi = 0; 2247 svm->sysenter_esp_hi = 0; 2248 } else { 2249 svm_copy_vmloadsave_state(vmcb12, svm->vmcb01.ptr); 2250 } 2251 2252 kvm_vcpu_unmap(vcpu, &map); 2253 2254 return ret; 2255 } 2256 2257 static int vmload_interception(struct kvm_vcpu *vcpu) 2258 { 2259 return vmload_vmsave_interception(vcpu, true); 2260 } 2261 2262 static int vmsave_interception(struct kvm_vcpu *vcpu) 2263 { 2264 return vmload_vmsave_interception(vcpu, false); 2265 } 2266 2267 static int vmrun_interception(struct kvm_vcpu *vcpu) 2268 { 2269 if (nested_svm_check_permissions(vcpu)) 2270 return 1; 2271 2272 return nested_svm_vmrun(vcpu); 2273 } 2274 2275 /* Return 0 if not SVM instr, otherwise return associated exit_code */ 2276 static u64 svm_get_decoded_instr_exit_code(struct kvm_vcpu *vcpu) 2277 { 2278 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 2279 2280 if (ctxt->b != 0x1 || ctxt->opcode_len != 2) 2281 return 0; 2282 2283 BUILD_BUG_ON(!SVM_EXIT_VMRUN || !SVM_EXIT_VMLOAD || !SVM_EXIT_VMSAVE); 2284 2285 switch (ctxt->modrm) { 2286 case 0xd8: /* VMRUN */ 2287 return SVM_EXIT_VMRUN; 2288 case 0xda: /* VMLOAD */ 2289 return SVM_EXIT_VMLOAD; 2290 case 0xdb: /* VMSAVE */ 2291 return SVM_EXIT_VMSAVE; 2292 default: 2293 break; 2294 } 2295 2296 return 0; 2297 } 2298 2299 /* 2300 * #GP handling code. Note that #GP can be triggered under the following two 2301 * cases: 2302 * 1) SVM VM-related instructions (VMRUN/VMSAVE/VMLOAD) that trigger #GP on 2303 * some AMD CPUs when EAX of these instructions are in the reserved memory 2304 * regions (e.g. SMM memory on host). 2305 * 2) VMware backdoor 2306 */ 2307 static int gp_interception(struct kvm_vcpu *vcpu) 2308 { 2309 struct vcpu_svm *svm = to_svm(vcpu); 2310 u32 error_code = svm->vmcb->control.exit_info_1; 2311 u64 svm_exit_code; 2312 2313 /* Both #GP cases have zero error_code */ 2314 if (error_code) 2315 goto reinject; 2316 2317 /* Decode the instruction for usage later */ 2318 if (x86_decode_emulated_instruction(vcpu, 0, NULL, 0) != EMULATION_OK) 2319 goto reinject; 2320 2321 /* FIXME: Handle SVM instructions through the emulator */ 2322 svm_exit_code = svm_get_decoded_instr_exit_code(vcpu); 2323 if (svm_exit_code) { 2324 if (!is_guest_mode(vcpu)) 2325 return svm_invoke_exit_handler(vcpu, svm_exit_code); 2326 2327 if (nested_svm_check_permissions(vcpu)) 2328 return 1; 2329 2330 if (!page_address_valid(vcpu, kvm_rax_read(vcpu))) 2331 goto reinject; 2332 2333 /* 2334 * FIXME: Only synthesize a #VMEXIT if L1 sets the intercept, 2335 * but only after the VMLOAD/VMSAVE exit handlers can properly 2336 * handle VMLOAD/VMSAVE from L2 with VLS enabled in L1 (i.e. 2337 * RAX is an L2 GPA that needs translation through L1's NPT). 2338 */ 2339 nested_svm_simple_vmexit(svm, svm_exit_code); 2340 return 1; 2341 } 2342 2343 /* 2344 * VMware backdoor emulation on #GP interception only handles 2345 * IN{S}, OUT{S}, and RDPMC, and only for L1. 2346 */ 2347 if (!enable_vmware_backdoor || is_guest_mode(vcpu)) 2348 goto reinject; 2349 2350 return kvm_emulate_instruction(vcpu, EMULTYPE_VMWARE_GP | EMULTYPE_NO_DECODE); 2351 2352 reinject: 2353 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code); 2354 return 1; 2355 } 2356 2357 void svm_set_gif(struct vcpu_svm *svm, bool value) 2358 { 2359 if (value) { 2360 /* 2361 * If VGIF is enabled, the STGI intercept is only added to 2362 * detect the opening of the SMI/NMI window; remove it now. 2363 * Likewise, clear the VINTR intercept, we will set it 2364 * again while processing KVM_REQ_EVENT if needed. 2365 */ 2366 if (vgif) 2367 svm_clr_intercept(svm, INTERCEPT_STGI); 2368 if (svm_is_intercept(svm, INTERCEPT_VINTR)) 2369 svm_clear_vintr(svm); 2370 2371 enable_gif(svm); 2372 if (svm_has_pending_gif_event(svm)) 2373 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); 2374 } else { 2375 disable_gif(svm); 2376 2377 /* 2378 * After a CLGI no interrupts should come. But if vGIF is 2379 * in use, we still rely on the VINTR intercept (rather than 2380 * STGI) to detect an open interrupt window. 2381 */ 2382 if (!vgif) 2383 svm_clear_vintr(svm); 2384 } 2385 } 2386 2387 static int stgi_interception(struct kvm_vcpu *vcpu) 2388 { 2389 int ret; 2390 2391 if (nested_svm_check_permissions(vcpu)) 2392 return 1; 2393 2394 ret = kvm_skip_emulated_instruction(vcpu); 2395 svm_set_gif(to_svm(vcpu), true); 2396 return ret; 2397 } 2398 2399 static int clgi_interception(struct kvm_vcpu *vcpu) 2400 { 2401 int ret; 2402 2403 if (nested_svm_check_permissions(vcpu)) 2404 return 1; 2405 2406 ret = kvm_skip_emulated_instruction(vcpu); 2407 svm_set_gif(to_svm(vcpu), false); 2408 return ret; 2409 } 2410 2411 static int invlpga_interception(struct kvm_vcpu *vcpu) 2412 { 2413 /* FIXME: Handle an address size prefix. */ 2414 gva_t gva = kvm_rax_read(vcpu); 2415 u32 asid = kvm_ecx_read(vcpu); 2416 2417 if (nested_svm_check_permissions(vcpu)) 2418 return 1; 2419 2420 trace_kvm_invlpga(to_svm(vcpu)->vmcb->save.rip, asid, gva); 2421 2422 /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */ 2423 kvm_mmu_invlpg(vcpu, gva); 2424 2425 return kvm_skip_emulated_instruction(vcpu); 2426 } 2427 2428 static int skinit_interception(struct kvm_vcpu *vcpu) 2429 { 2430 trace_kvm_skinit(to_svm(vcpu)->vmcb->save.rip, kvm_rax_read(vcpu)); 2431 2432 kvm_queue_exception(vcpu, UD_VECTOR); 2433 return 1; 2434 } 2435 2436 static int task_switch_interception(struct kvm_vcpu *vcpu) 2437 { 2438 struct vcpu_svm *svm = to_svm(vcpu); 2439 u16 tss_selector; 2440 int reason; 2441 int int_type = svm->vmcb->control.exit_int_info & 2442 SVM_EXITINTINFO_TYPE_MASK; 2443 int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK; 2444 uint32_t type = 2445 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK; 2446 uint32_t idt_v = 2447 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID; 2448 bool has_error_code = false; 2449 u32 error_code = 0; 2450 2451 tss_selector = (u16)svm->vmcb->control.exit_info_1; 2452 2453 if (svm->vmcb->control.exit_info_2 & 2454 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET)) 2455 reason = TASK_SWITCH_IRET; 2456 else if (svm->vmcb->control.exit_info_2 & 2457 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP)) 2458 reason = TASK_SWITCH_JMP; 2459 else if (idt_v) 2460 reason = TASK_SWITCH_GATE; 2461 else 2462 reason = TASK_SWITCH_CALL; 2463 2464 if (reason == TASK_SWITCH_GATE) { 2465 switch (type) { 2466 case SVM_EXITINTINFO_TYPE_NMI: 2467 vcpu->arch.nmi_injected = false; 2468 break; 2469 case SVM_EXITINTINFO_TYPE_EXEPT: 2470 if (svm->vmcb->control.exit_info_2 & 2471 (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) { 2472 has_error_code = true; 2473 error_code = 2474 (u32)svm->vmcb->control.exit_info_2; 2475 } 2476 kvm_clear_exception_queue(vcpu); 2477 break; 2478 case SVM_EXITINTINFO_TYPE_INTR: 2479 case SVM_EXITINTINFO_TYPE_SOFT: 2480 kvm_clear_interrupt_queue(vcpu); 2481 break; 2482 default: 2483 break; 2484 } 2485 } 2486 2487 if (reason != TASK_SWITCH_GATE || 2488 int_type == SVM_EXITINTINFO_TYPE_SOFT || 2489 (int_type == SVM_EXITINTINFO_TYPE_EXEPT && 2490 (int_vec == OF_VECTOR || int_vec == BP_VECTOR))) { 2491 if (!svm_skip_emulated_instruction(vcpu)) 2492 return 0; 2493 } 2494 2495 if (int_type != SVM_EXITINTINFO_TYPE_SOFT) 2496 int_vec = -1; 2497 2498 return kvm_task_switch(vcpu, tss_selector, int_vec, reason, 2499 has_error_code, error_code); 2500 } 2501 2502 static void svm_clr_iret_intercept(struct vcpu_svm *svm) 2503 { 2504 if (!is_sev_es_guest(&svm->vcpu)) 2505 svm_clr_intercept(svm, INTERCEPT_IRET); 2506 } 2507 2508 static void svm_set_iret_intercept(struct vcpu_svm *svm) 2509 { 2510 if (!is_sev_es_guest(&svm->vcpu)) 2511 svm_set_intercept(svm, INTERCEPT_IRET); 2512 } 2513 2514 static int iret_interception(struct kvm_vcpu *vcpu) 2515 { 2516 struct vcpu_svm *svm = to_svm(vcpu); 2517 2518 WARN_ON_ONCE(is_sev_es_guest(vcpu)); 2519 2520 ++vcpu->stat.nmi_window_exits; 2521 svm->awaiting_iret_completion = true; 2522 2523 svm_clr_iret_intercept(svm); 2524 svm->nmi_iret_rip = kvm_rip_read(vcpu); 2525 2526 kvm_make_request(KVM_REQ_EVENT, vcpu); 2527 return 1; 2528 } 2529 2530 static int invlpg_interception(struct kvm_vcpu *vcpu) 2531 { 2532 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS)) 2533 return kvm_emulate_instruction(vcpu, 0); 2534 2535 kvm_mmu_invlpg(vcpu, to_svm(vcpu)->vmcb->control.exit_info_1); 2536 return kvm_skip_emulated_instruction(vcpu); 2537 } 2538 2539 static int emulate_on_interception(struct kvm_vcpu *vcpu) 2540 { 2541 return kvm_emulate_instruction(vcpu, 0); 2542 } 2543 2544 static int rsm_interception(struct kvm_vcpu *vcpu) 2545 { 2546 return kvm_emulate_instruction_from_buffer(vcpu, rsm_ins_bytes, 2); 2547 } 2548 2549 static bool check_selective_cr0_intercepted(struct kvm_vcpu *vcpu, 2550 unsigned long val) 2551 { 2552 struct vcpu_svm *svm = to_svm(vcpu); 2553 unsigned long cr0 = vcpu->arch.cr0; 2554 bool ret = false; 2555 2556 if (!is_guest_mode(vcpu) || 2557 (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0)))) 2558 return false; 2559 2560 cr0 &= ~SVM_CR0_SELECTIVE_MASK; 2561 val &= ~SVM_CR0_SELECTIVE_MASK; 2562 2563 if (cr0 ^ val) { 2564 svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE; 2565 ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE); 2566 } 2567 2568 return ret; 2569 } 2570 2571 #define CR_VALID (1ULL << 63) 2572 2573 static int cr_interception(struct kvm_vcpu *vcpu) 2574 { 2575 struct vcpu_svm *svm = to_svm(vcpu); 2576 int reg, cr; 2577 unsigned long val; 2578 int err; 2579 2580 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS)) 2581 return emulate_on_interception(vcpu); 2582 2583 if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0)) 2584 return emulate_on_interception(vcpu); 2585 2586 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK; 2587 if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE) 2588 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0; 2589 else 2590 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0; 2591 2592 err = 0; 2593 if (cr >= 16) { /* mov to cr */ 2594 cr -= 16; 2595 val = kvm_register_read(vcpu, reg); 2596 trace_kvm_cr_write(cr, val); 2597 switch (cr) { 2598 case 0: 2599 if (!check_selective_cr0_intercepted(vcpu, val)) 2600 err = kvm_set_cr0(vcpu, val); 2601 else 2602 return 1; 2603 2604 break; 2605 case 3: 2606 err = kvm_set_cr3(vcpu, val); 2607 break; 2608 case 4: 2609 err = kvm_set_cr4(vcpu, val); 2610 break; 2611 case 8: 2612 err = kvm_set_cr8(vcpu, val); 2613 break; 2614 default: 2615 WARN(1, "unhandled write to CR%d", cr); 2616 kvm_queue_exception(vcpu, UD_VECTOR); 2617 return 1; 2618 } 2619 } else { /* mov from cr */ 2620 switch (cr) { 2621 case 0: 2622 val = kvm_read_cr0(vcpu); 2623 break; 2624 case 2: 2625 val = vcpu->arch.cr2; 2626 break; 2627 case 3: 2628 val = kvm_read_cr3(vcpu); 2629 break; 2630 case 4: 2631 val = kvm_read_cr4(vcpu); 2632 break; 2633 case 8: 2634 val = kvm_get_cr8(vcpu); 2635 break; 2636 default: 2637 WARN(1, "unhandled read from CR%d", cr); 2638 kvm_queue_exception(vcpu, UD_VECTOR); 2639 return 1; 2640 } 2641 kvm_register_write(vcpu, reg, val); 2642 trace_kvm_cr_read(cr, val); 2643 } 2644 return kvm_complete_insn_gp(vcpu, err); 2645 } 2646 2647 static int cr_trap(struct kvm_vcpu *vcpu) 2648 { 2649 struct vcpu_svm *svm = to_svm(vcpu); 2650 unsigned long old_value, new_value; 2651 unsigned int cr; 2652 int ret = 0; 2653 2654 new_value = (unsigned long)svm->vmcb->control.exit_info_1; 2655 2656 cr = svm->vmcb->control.exit_code - SVM_EXIT_CR0_WRITE_TRAP; 2657 switch (cr) { 2658 case 0: 2659 old_value = kvm_read_cr0(vcpu); 2660 svm_set_cr0(vcpu, new_value); 2661 2662 kvm_post_set_cr0(vcpu, old_value, new_value); 2663 break; 2664 case 4: 2665 old_value = kvm_read_cr4(vcpu); 2666 svm_set_cr4(vcpu, new_value); 2667 2668 kvm_post_set_cr4(vcpu, old_value, new_value); 2669 break; 2670 case 8: 2671 ret = kvm_set_cr8(vcpu, new_value); 2672 break; 2673 default: 2674 WARN(1, "unhandled CR%d write trap", cr); 2675 kvm_queue_exception(vcpu, UD_VECTOR); 2676 return 1; 2677 } 2678 2679 return kvm_complete_insn_gp(vcpu, ret); 2680 } 2681 2682 static int dr_interception(struct kvm_vcpu *vcpu) 2683 { 2684 struct vcpu_svm *svm = to_svm(vcpu); 2685 int reg, dr; 2686 int err = 0; 2687 2688 /* 2689 * SEV-ES intercepts DR7 only to disable guest debugging and the guest issues a VMGEXIT 2690 * for DR7 write only. KVM cannot change DR7 (always swapped as type 'A') so return early. 2691 */ 2692 if (is_sev_es_guest(vcpu)) 2693 return 1; 2694 2695 if (vcpu->guest_debug == 0) { 2696 /* 2697 * No more DR vmexits; force a reload of the debug registers 2698 * and reenter on this instruction. The next vmexit will 2699 * retrieve the full state of the debug registers. 2700 */ 2701 clr_dr_intercepts(svm); 2702 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT; 2703 return 1; 2704 } 2705 2706 if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS)) 2707 return emulate_on_interception(vcpu); 2708 2709 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK; 2710 dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0; 2711 if (dr >= 16) { /* mov to DRn */ 2712 dr -= 16; 2713 err = kvm_set_dr(vcpu, dr, kvm_register_read(vcpu, reg)); 2714 } else { 2715 kvm_register_write(vcpu, reg, kvm_get_dr(vcpu, dr)); 2716 } 2717 2718 return kvm_complete_insn_gp(vcpu, err); 2719 } 2720 2721 static int cr8_write_interception(struct kvm_vcpu *vcpu) 2722 { 2723 u8 cr8_prev = kvm_get_cr8(vcpu); 2724 int r; 2725 2726 WARN_ON_ONCE(kvm_vcpu_apicv_active(vcpu)); 2727 2728 /* instruction emulation calls kvm_set_cr8() */ 2729 r = cr_interception(vcpu); 2730 if (lapic_in_kernel(vcpu)) 2731 return r; 2732 if (cr8_prev <= kvm_get_cr8(vcpu)) 2733 return r; 2734 vcpu->run->exit_reason = KVM_EXIT_SET_TPR; 2735 return 0; 2736 } 2737 2738 static int efer_trap(struct kvm_vcpu *vcpu) 2739 { 2740 struct msr_data msr_info; 2741 int ret; 2742 2743 /* 2744 * Clear the EFER_SVME bit from EFER. The SVM code always sets this 2745 * bit in svm_set_efer(), but __kvm_valid_efer() checks it against 2746 * whether the guest has X86_FEATURE_SVM - this avoids a failure if 2747 * the guest doesn't have X86_FEATURE_SVM. 2748 */ 2749 msr_info.host_initiated = false; 2750 msr_info.index = MSR_EFER; 2751 msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & ~EFER_SVME; 2752 ret = kvm_set_msr_common(vcpu, &msr_info); 2753 2754 return kvm_complete_insn_gp(vcpu, ret); 2755 } 2756 2757 static int svm_get_feature_msr(u32 msr, u64 *data) 2758 { 2759 *data = 0; 2760 2761 switch (msr) { 2762 case MSR_AMD64_DE_CFG: 2763 if (cpu_feature_enabled(X86_FEATURE_LFENCE_RDTSC)) 2764 *data |= MSR_AMD64_DE_CFG_LFENCE_SERIALIZE; 2765 break; 2766 default: 2767 return KVM_MSR_RET_UNSUPPORTED; 2768 } 2769 2770 return 0; 2771 } 2772 2773 static u64 *svm_vmcb_lbr(struct vcpu_svm *svm, u32 msr) 2774 { 2775 switch (msr) { 2776 case MSR_IA32_LASTBRANCHFROMIP: 2777 return &svm->vmcb->save.br_from; 2778 case MSR_IA32_LASTBRANCHTOIP: 2779 return &svm->vmcb->save.br_to; 2780 case MSR_IA32_LASTINTFROMIP: 2781 return &svm->vmcb->save.last_excp_from; 2782 case MSR_IA32_LASTINTTOIP: 2783 return &svm->vmcb->save.last_excp_to; 2784 default: 2785 break; 2786 } 2787 KVM_BUG_ON(1, svm->vcpu.kvm); 2788 return &svm->vmcb->save.br_from; 2789 } 2790 2791 static bool sev_es_prevent_msr_access(struct kvm_vcpu *vcpu, 2792 struct msr_data *msr_info) 2793 { 2794 return is_sev_es_guest(vcpu) && vcpu->arch.guest_state_protected && 2795 msr_info->index != MSR_IA32_XSS && 2796 !msr_write_intercepted(to_svm(vcpu), msr_info->index); 2797 } 2798 2799 static bool svm_pat_accesses_gpat(struct kvm_vcpu *vcpu, bool from_host) 2800 { 2801 /* 2802 * When KVM_X86_QUIRK_NESTED_SVM_SHARED_PAT is disabled and nested 2803 * NPT is enabled, L2 has a separate PAT from L1. Guest accesses 2804 * to IA32_PAT while running L2 target L2's gPAT; host-initiated 2805 * accesses always target L1's hPAT so that KVM_GET/SET_MSRS and 2806 * KVM_GET/SET_NESTED_STATE are independent of each other and can 2807 * be ordered arbitrarily during save and restore. 2808 */ 2809 WARN_ON_ONCE(from_host && vcpu->wants_to_run); 2810 return !from_host && is_guest_mode(vcpu) && l2_has_separate_pat(vcpu); 2811 } 2812 2813 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 2814 { 2815 struct vcpu_svm *svm = to_svm(vcpu); 2816 2817 if (sev_es_prevent_msr_access(vcpu, msr_info)) { 2818 msr_info->data = 0; 2819 return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0; 2820 } 2821 2822 switch (msr_info->index) { 2823 case MSR_AMD64_TSC_RATIO: 2824 if (!msr_info->host_initiated && 2825 !guest_cpu_cap_has(vcpu, X86_FEATURE_TSCRATEMSR)) 2826 return 1; 2827 msr_info->data = svm->tsc_ratio_msr; 2828 break; 2829 case MSR_STAR: 2830 msr_info->data = svm->vmcb01.ptr->save.star; 2831 break; 2832 #ifdef CONFIG_X86_64 2833 case MSR_LSTAR: 2834 msr_info->data = svm->vmcb01.ptr->save.lstar; 2835 break; 2836 case MSR_CSTAR: 2837 msr_info->data = svm->vmcb01.ptr->save.cstar; 2838 break; 2839 case MSR_GS_BASE: 2840 msr_info->data = svm->vmcb01.ptr->save.gs.base; 2841 break; 2842 case MSR_FS_BASE: 2843 msr_info->data = svm->vmcb01.ptr->save.fs.base; 2844 break; 2845 case MSR_KERNEL_GS_BASE: 2846 msr_info->data = svm->vmcb01.ptr->save.kernel_gs_base; 2847 break; 2848 case MSR_SYSCALL_MASK: 2849 msr_info->data = svm->vmcb01.ptr->save.sfmask; 2850 break; 2851 #endif 2852 case MSR_IA32_SYSENTER_CS: 2853 msr_info->data = svm->vmcb01.ptr->save.sysenter_cs; 2854 break; 2855 case MSR_IA32_SYSENTER_EIP: 2856 msr_info->data = (u32)svm->vmcb01.ptr->save.sysenter_eip; 2857 if (guest_cpuid_is_intel_compatible(vcpu)) 2858 msr_info->data |= (u64)svm->sysenter_eip_hi << 32; 2859 break; 2860 case MSR_IA32_SYSENTER_ESP: 2861 msr_info->data = svm->vmcb01.ptr->save.sysenter_esp; 2862 if (guest_cpuid_is_intel_compatible(vcpu)) 2863 msr_info->data |= (u64)svm->sysenter_esp_hi << 32; 2864 break; 2865 case MSR_IA32_S_CET: 2866 msr_info->data = svm->vmcb->save.s_cet; 2867 break; 2868 case MSR_IA32_INT_SSP_TAB: 2869 msr_info->data = svm->vmcb->save.isst_addr; 2870 break; 2871 case MSR_KVM_INTERNAL_GUEST_SSP: 2872 msr_info->data = svm->vmcb->save.ssp; 2873 break; 2874 case MSR_TSC_AUX: 2875 msr_info->data = svm->tsc_aux; 2876 break; 2877 case MSR_IA32_DEBUGCTLMSR: 2878 msr_info->data = lbrv ? svm->vmcb->save.dbgctl : 0; 2879 break; 2880 case MSR_IA32_LASTBRANCHFROMIP: 2881 case MSR_IA32_LASTBRANCHTOIP: 2882 case MSR_IA32_LASTINTFROMIP: 2883 case MSR_IA32_LASTINTTOIP: 2884 msr_info->data = lbrv ? *svm_vmcb_lbr(svm, msr_info->index) : 0; 2885 break; 2886 case MSR_VM_HSAVE_PA: 2887 msr_info->data = svm->nested.hsave_msr; 2888 break; 2889 case MSR_VM_CR: 2890 msr_info->data = svm->nested.vm_cr_msr; 2891 break; 2892 case MSR_IA32_SPEC_CTRL: 2893 if (!msr_info->host_initiated && 2894 !guest_has_spec_ctrl_msr(vcpu)) 2895 return 1; 2896 2897 if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 2898 msr_info->data = svm->vmcb->save.spec_ctrl; 2899 else 2900 msr_info->data = svm->spec_ctrl; 2901 break; 2902 case MSR_AMD64_VIRT_SPEC_CTRL: 2903 if (!msr_info->host_initiated && 2904 !guest_cpu_cap_has(vcpu, X86_FEATURE_VIRT_SSBD)) 2905 return 1; 2906 2907 msr_info->data = svm->virt_spec_ctrl; 2908 break; 2909 case MSR_F15H_IC_CFG: { 2910 2911 int family, model; 2912 2913 family = guest_cpuid_family(vcpu); 2914 model = guest_cpuid_model(vcpu); 2915 2916 if (family < 0 || model < 0) 2917 return kvm_get_msr_common(vcpu, msr_info); 2918 2919 msr_info->data = 0; 2920 2921 if (family == 0x15 && 2922 (model >= 0x2 && model < 0x20)) 2923 msr_info->data = 0x1E; 2924 } 2925 break; 2926 case MSR_AMD64_DE_CFG: 2927 msr_info->data = svm->msr_decfg; 2928 break; 2929 case MSR_IA32_CR_PAT: 2930 if (svm_pat_accesses_gpat(vcpu, msr_info->host_initiated)) { 2931 msr_info->data = svm->vmcb->save.g_pat; 2932 break; 2933 } 2934 return kvm_get_msr_common(vcpu, msr_info); 2935 default: 2936 return kvm_get_msr_common(vcpu, msr_info); 2937 } 2938 return 0; 2939 } 2940 2941 static int svm_complete_emulated_msr(struct kvm_vcpu *vcpu, int err) 2942 { 2943 struct vcpu_svm *svm = to_svm(vcpu); 2944 if (!err || !is_sev_es_guest(vcpu) || WARN_ON_ONCE(!svm->sev_es.ghcb)) 2945 return kvm_complete_insn_gp(vcpu, err); 2946 2947 svm_vmgexit_inject_exception(svm, X86_TRAP_GP); 2948 return 1; 2949 } 2950 2951 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data) 2952 { 2953 struct vcpu_svm *svm = to_svm(vcpu); 2954 int svm_dis, chg_mask; 2955 2956 if (data & ~SVM_VM_CR_VALID_MASK) 2957 return 1; 2958 2959 chg_mask = SVM_VM_CR_VALID_MASK; 2960 2961 if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK) 2962 chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK); 2963 2964 svm->nested.vm_cr_msr &= ~chg_mask; 2965 svm->nested.vm_cr_msr |= (data & chg_mask); 2966 2967 svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK; 2968 2969 /* check for svm_disable while efer.svme is set */ 2970 if (svm_dis && (vcpu->arch.efer & EFER_SVME)) 2971 return 1; 2972 2973 return 0; 2974 } 2975 2976 static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) 2977 { 2978 struct vcpu_svm *svm = to_svm(vcpu); 2979 int ret = 0; 2980 2981 u32 ecx = msr->index; 2982 u64 data = msr->data; 2983 2984 if (sev_es_prevent_msr_access(vcpu, msr)) 2985 return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0; 2986 2987 switch (ecx) { 2988 case MSR_AMD64_TSC_RATIO: 2989 2990 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_TSCRATEMSR)) { 2991 2992 if (!msr->host_initiated) 2993 return 1; 2994 /* 2995 * In case TSC scaling is not enabled, always 2996 * leave this MSR at the default value. 2997 * 2998 * Due to bug in qemu 6.2.0, it would try to set 2999 * this msr to 0 if tsc scaling is not enabled. 3000 * Ignore this value as well. 3001 */ 3002 if (data != 0 && data != svm->tsc_ratio_msr) 3003 return 1; 3004 break; 3005 } 3006 3007 if (data & SVM_TSC_RATIO_RSVD) 3008 return 1; 3009 3010 svm->tsc_ratio_msr = data; 3011 3012 if (guest_cpu_cap_has(vcpu, X86_FEATURE_TSCRATEMSR) && 3013 is_guest_mode(vcpu)) 3014 nested_svm_update_tsc_ratio_msr(vcpu); 3015 3016 break; 3017 case MSR_IA32_CR_PAT: 3018 if (svm_pat_accesses_gpat(vcpu, msr->host_initiated)) { 3019 if (!kvm_pat_valid(data)) 3020 return 1; 3021 3022 vmcb_set_gpat(svm->vmcb, data); 3023 break; 3024 } 3025 3026 ret = kvm_set_msr_common(vcpu, msr); 3027 if (ret) 3028 break; 3029 3030 if (npt_enabled) { 3031 vmcb_set_gpat(svm->vmcb01.ptr, data); 3032 if (is_guest_mode(vcpu) && !l2_has_separate_pat(vcpu)) 3033 vmcb_set_gpat(svm->vmcb, data); 3034 } 3035 break; 3036 case MSR_IA32_SPEC_CTRL: 3037 if (!msr->host_initiated && 3038 !guest_has_spec_ctrl_msr(vcpu)) 3039 return 1; 3040 3041 if (kvm_spec_ctrl_test_value(data)) 3042 return 1; 3043 3044 if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 3045 svm->vmcb->save.spec_ctrl = data; 3046 else 3047 svm->spec_ctrl = data; 3048 if (!data) 3049 break; 3050 3051 /* 3052 * For non-nested: 3053 * When it's written (to non-zero) for the first time, pass 3054 * it through. 3055 * 3056 * For nested: 3057 * The handling of the MSR bitmap for L2 guests is done in 3058 * nested_svm_merge_msrpm(). 3059 * We update the L1 MSR bit as well since it will end up 3060 * touching the MSR anyway now. 3061 */ 3062 svm_disable_intercept_for_msr(vcpu, MSR_IA32_SPEC_CTRL, MSR_TYPE_RW); 3063 break; 3064 case MSR_AMD64_VIRT_SPEC_CTRL: 3065 if (!msr->host_initiated && 3066 !guest_cpu_cap_has(vcpu, X86_FEATURE_VIRT_SSBD)) 3067 return 1; 3068 3069 if (data & ~SPEC_CTRL_SSBD) 3070 return 1; 3071 3072 svm->virt_spec_ctrl = data; 3073 break; 3074 case MSR_STAR: 3075 svm->vmcb01.ptr->save.star = data; 3076 break; 3077 #ifdef CONFIG_X86_64 3078 case MSR_LSTAR: 3079 svm->vmcb01.ptr->save.lstar = data; 3080 break; 3081 case MSR_CSTAR: 3082 svm->vmcb01.ptr->save.cstar = data; 3083 break; 3084 case MSR_GS_BASE: 3085 svm->vmcb01.ptr->save.gs.base = data; 3086 break; 3087 case MSR_FS_BASE: 3088 svm->vmcb01.ptr->save.fs.base = data; 3089 break; 3090 case MSR_KERNEL_GS_BASE: 3091 svm->vmcb01.ptr->save.kernel_gs_base = data; 3092 break; 3093 case MSR_SYSCALL_MASK: 3094 svm->vmcb01.ptr->save.sfmask = data; 3095 break; 3096 #endif 3097 case MSR_IA32_SYSENTER_CS: 3098 svm->vmcb01.ptr->save.sysenter_cs = data; 3099 break; 3100 case MSR_IA32_SYSENTER_EIP: 3101 svm->vmcb01.ptr->save.sysenter_eip = (u32)data; 3102 /* 3103 * We only intercept the MSR_IA32_SYSENTER_{EIP|ESP} msrs 3104 * when we spoof an Intel vendor ID (for cross vendor migration). 3105 * In this case we use this intercept to track the high 3106 * 32 bit part of these msrs to support Intel's 3107 * implementation of SYSENTER/SYSEXIT. 3108 */ 3109 svm->sysenter_eip_hi = guest_cpuid_is_intel_compatible(vcpu) ? (data >> 32) : 0; 3110 break; 3111 case MSR_IA32_SYSENTER_ESP: 3112 svm->vmcb01.ptr->save.sysenter_esp = (u32)data; 3113 svm->sysenter_esp_hi = guest_cpuid_is_intel_compatible(vcpu) ? (data >> 32) : 0; 3114 break; 3115 case MSR_IA32_S_CET: 3116 svm->vmcb->save.s_cet = data; 3117 vmcb_mark_dirty(svm->vmcb01.ptr, VMCB_CET); 3118 break; 3119 case MSR_IA32_INT_SSP_TAB: 3120 svm->vmcb->save.isst_addr = data; 3121 vmcb_mark_dirty(svm->vmcb01.ptr, VMCB_CET); 3122 break; 3123 case MSR_KVM_INTERNAL_GUEST_SSP: 3124 svm->vmcb->save.ssp = data; 3125 vmcb_mark_dirty(svm->vmcb01.ptr, VMCB_CET); 3126 break; 3127 case MSR_TSC_AUX: 3128 /* 3129 * TSC_AUX is always virtualized for SEV-ES guests when the 3130 * feature is available. The user return MSR support is not 3131 * required in this case because TSC_AUX is restored on #VMEXIT 3132 * from the host save area. 3133 */ 3134 if (boot_cpu_has(X86_FEATURE_V_TSC_AUX) && is_sev_es_guest(vcpu)) 3135 break; 3136 3137 /* 3138 * TSC_AUX is usually changed only during boot and never read 3139 * directly. Intercept TSC_AUX and switch it via user return. 3140 */ 3141 preempt_disable(); 3142 ret = kvm_set_user_return_msr(tsc_aux_uret_slot, data, -1ull); 3143 preempt_enable(); 3144 if (ret) 3145 break; 3146 3147 svm->tsc_aux = data; 3148 break; 3149 case MSR_IA32_DEBUGCTLMSR: 3150 if (!lbrv) { 3151 kvm_pr_unimpl_wrmsr(vcpu, ecx, data); 3152 break; 3153 } 3154 3155 /* 3156 * Suppress BTF as KVM doesn't virtualize BTF, but there's no 3157 * way to communicate lack of support to the guest. 3158 */ 3159 if (data & DEBUGCTLMSR_BTF) { 3160 kvm_pr_unimpl_wrmsr(vcpu, MSR_IA32_DEBUGCTLMSR, data); 3161 data &= ~DEBUGCTLMSR_BTF; 3162 } 3163 3164 if (data & DEBUGCTL_RESERVED_BITS) 3165 return 1; 3166 3167 if (svm->vmcb->save.dbgctl == data) 3168 break; 3169 3170 svm->vmcb->save.dbgctl = data; 3171 vmcb_mark_dirty(svm->vmcb, VMCB_LBR); 3172 svm_update_lbrv(vcpu); 3173 break; 3174 case MSR_IA32_LASTBRANCHFROMIP: 3175 case MSR_IA32_LASTBRANCHTOIP: 3176 case MSR_IA32_LASTINTFROMIP: 3177 case MSR_IA32_LASTINTTOIP: 3178 if (!lbrv) 3179 return KVM_MSR_RET_UNSUPPORTED; 3180 if (!msr->host_initiated) 3181 return 1; 3182 *svm_vmcb_lbr(svm, ecx) = data; 3183 vmcb_mark_dirty(svm->vmcb, VMCB_LBR); 3184 break; 3185 case MSR_VM_HSAVE_PA: 3186 /* 3187 * Old kernels did not validate the value written to 3188 * MSR_VM_HSAVE_PA. Allow KVM_SET_MSR to set an invalid 3189 * value to allow live migrating buggy or malicious guests 3190 * originating from those kernels. 3191 */ 3192 if (!msr->host_initiated && !page_address_valid(vcpu, data)) 3193 return 1; 3194 3195 svm->nested.hsave_msr = data & PAGE_MASK; 3196 break; 3197 case MSR_VM_CR: 3198 return svm_set_vm_cr(vcpu, data); 3199 case MSR_VM_IGNNE: 3200 kvm_pr_unimpl_wrmsr(vcpu, ecx, data); 3201 break; 3202 case MSR_AMD64_DE_CFG: { 3203 u64 supported_de_cfg; 3204 3205 if (svm_get_feature_msr(ecx, &supported_de_cfg)) 3206 return 1; 3207 3208 if (data & ~supported_de_cfg) 3209 return 1; 3210 3211 svm->msr_decfg = data; 3212 break; 3213 } 3214 default: 3215 return kvm_set_msr_common(vcpu, msr); 3216 } 3217 return ret; 3218 } 3219 3220 static int msr_interception(struct kvm_vcpu *vcpu) 3221 { 3222 if (to_svm(vcpu)->vmcb->control.exit_info_1) 3223 return kvm_emulate_wrmsr(vcpu); 3224 else 3225 return kvm_emulate_rdmsr(vcpu); 3226 } 3227 3228 static int interrupt_window_interception(struct kvm_vcpu *vcpu) 3229 { 3230 kvm_make_request(KVM_REQ_EVENT, vcpu); 3231 svm_clear_vintr(to_svm(vcpu)); 3232 3233 ++vcpu->stat.irq_window_exits; 3234 return 1; 3235 } 3236 3237 static int pause_interception(struct kvm_vcpu *vcpu) 3238 { 3239 bool in_kernel; 3240 /* 3241 * CPL is not made available for an SEV-ES guest, therefore 3242 * vcpu->arch.preempted_in_kernel can never be true. Just 3243 * set in_kernel to false as well. 3244 */ 3245 in_kernel = !is_sev_es_guest(vcpu) && svm_get_cpl(vcpu) == 0; 3246 3247 grow_ple_window(vcpu); 3248 3249 kvm_vcpu_on_spin(vcpu, in_kernel); 3250 return kvm_skip_emulated_instruction(vcpu); 3251 } 3252 3253 static int invpcid_interception(struct kvm_vcpu *vcpu) 3254 { 3255 struct vcpu_svm *svm = to_svm(vcpu); 3256 unsigned long type; 3257 gva_t gva; 3258 3259 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_INVPCID)) { 3260 kvm_queue_exception(vcpu, UD_VECTOR); 3261 return 1; 3262 } 3263 3264 /* 3265 * For an INVPCID intercept: 3266 * EXITINFO1 provides the linear address of the memory operand. 3267 * EXITINFO2 provides the contents of the register operand. 3268 */ 3269 type = svm->vmcb->control.exit_info_2; 3270 gva = svm->vmcb->control.exit_info_1; 3271 3272 /* 3273 * FIXME: Perform segment checks for 32-bit mode, and inject #SS if the 3274 * stack segment is used. The intercept takes priority over all 3275 * #GP checks except CPL>0, but somehow still generates a linear 3276 * address? The APM is sorely lacking. 3277 */ 3278 if (is_noncanonical_address(gva, vcpu, 0)) { 3279 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); 3280 return 1; 3281 } 3282 3283 return kvm_handle_invpcid(vcpu, type, gva); 3284 } 3285 3286 static inline int complete_userspace_buslock(struct kvm_vcpu *vcpu) 3287 { 3288 struct vcpu_svm *svm = to_svm(vcpu); 3289 3290 /* 3291 * If userspace has NOT changed RIP, then KVM's ABI is to let the guest 3292 * execute the bus-locking instruction. Set the bus lock counter to '1' 3293 * to effectively step past the bus lock. 3294 */ 3295 if (kvm_is_linear_rip(vcpu, vcpu->arch.cui_linear_rip)) 3296 svm->vmcb->control.bus_lock_counter = 1; 3297 3298 return 1; 3299 } 3300 3301 static int bus_lock_exit(struct kvm_vcpu *vcpu) 3302 { 3303 struct vcpu_svm *svm = to_svm(vcpu); 3304 3305 vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; 3306 vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK; 3307 3308 vcpu->arch.cui_linear_rip = kvm_get_linear_rip(vcpu); 3309 vcpu->arch.complete_userspace_io = complete_userspace_buslock; 3310 3311 if (is_guest_mode(vcpu)) 3312 svm->nested.last_bus_lock_rip = vcpu->arch.cui_linear_rip; 3313 3314 return 0; 3315 } 3316 3317 static int vmmcall_interception(struct kvm_vcpu *vcpu) 3318 { 3319 /* 3320 * Inject a #UD if L2 is active and the VMMCALL isn't a Hyper-V TLB 3321 * hypercall, as VMMCALL #UDs if it's not intercepted, and this path is 3322 * reachable if and only if L1 doesn't want to intercept VMMCALL or has 3323 * enabled L0 (KVM) handling of Hyper-V L2 TLB flush hypercalls. 3324 */ 3325 if (is_guest_mode(vcpu) && !nested_svm_is_l2_tlb_flush_hcall(vcpu)) { 3326 kvm_queue_exception(vcpu, UD_VECTOR); 3327 return 1; 3328 } 3329 3330 return kvm_emulate_hypercall(vcpu); 3331 } 3332 3333 static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = { 3334 [SVM_EXIT_READ_CR0] = cr_interception, 3335 [SVM_EXIT_READ_CR3] = cr_interception, 3336 [SVM_EXIT_READ_CR4] = cr_interception, 3337 [SVM_EXIT_READ_CR8] = cr_interception, 3338 [SVM_EXIT_CR0_SEL_WRITE] = cr_interception, 3339 [SVM_EXIT_WRITE_CR0] = cr_interception, 3340 [SVM_EXIT_WRITE_CR3] = cr_interception, 3341 [SVM_EXIT_WRITE_CR4] = cr_interception, 3342 [SVM_EXIT_WRITE_CR8] = cr8_write_interception, 3343 [SVM_EXIT_READ_DR0] = dr_interception, 3344 [SVM_EXIT_READ_DR1] = dr_interception, 3345 [SVM_EXIT_READ_DR2] = dr_interception, 3346 [SVM_EXIT_READ_DR3] = dr_interception, 3347 [SVM_EXIT_READ_DR4] = dr_interception, 3348 [SVM_EXIT_READ_DR5] = dr_interception, 3349 [SVM_EXIT_READ_DR6] = dr_interception, 3350 [SVM_EXIT_READ_DR7] = dr_interception, 3351 [SVM_EXIT_WRITE_DR0] = dr_interception, 3352 [SVM_EXIT_WRITE_DR1] = dr_interception, 3353 [SVM_EXIT_WRITE_DR2] = dr_interception, 3354 [SVM_EXIT_WRITE_DR3] = dr_interception, 3355 [SVM_EXIT_WRITE_DR4] = dr_interception, 3356 [SVM_EXIT_WRITE_DR5] = dr_interception, 3357 [SVM_EXIT_WRITE_DR6] = dr_interception, 3358 [SVM_EXIT_WRITE_DR7] = dr_interception, 3359 [SVM_EXIT_EXCP_BASE + DB_VECTOR] = db_interception, 3360 [SVM_EXIT_EXCP_BASE + BP_VECTOR] = bp_interception, 3361 [SVM_EXIT_EXCP_BASE + UD_VECTOR] = ud_interception, 3362 [SVM_EXIT_EXCP_BASE + PF_VECTOR] = pf_interception, 3363 [SVM_EXIT_EXCP_BASE + MC_VECTOR] = mc_interception, 3364 [SVM_EXIT_EXCP_BASE + AC_VECTOR] = ac_interception, 3365 [SVM_EXIT_EXCP_BASE + GP_VECTOR] = gp_interception, 3366 [SVM_EXIT_INTR] = intr_interception, 3367 [SVM_EXIT_NMI] = nmi_interception, 3368 [SVM_EXIT_SMI] = smi_interception, 3369 [SVM_EXIT_VINTR] = interrupt_window_interception, 3370 [SVM_EXIT_RDPMC] = kvm_emulate_rdpmc, 3371 [SVM_EXIT_CPUID] = kvm_emulate_cpuid, 3372 [SVM_EXIT_IRET] = iret_interception, 3373 [SVM_EXIT_INVD] = kvm_emulate_invd, 3374 [SVM_EXIT_PAUSE] = pause_interception, 3375 [SVM_EXIT_HLT] = kvm_emulate_halt, 3376 [SVM_EXIT_INVLPG] = invlpg_interception, 3377 [SVM_EXIT_INVLPGA] = invlpga_interception, 3378 [SVM_EXIT_IOIO] = io_interception, 3379 [SVM_EXIT_MSR] = msr_interception, 3380 [SVM_EXIT_TASK_SWITCH] = task_switch_interception, 3381 [SVM_EXIT_SHUTDOWN] = shutdown_interception, 3382 [SVM_EXIT_VMRUN] = vmrun_interception, 3383 [SVM_EXIT_VMMCALL] = vmmcall_interception, 3384 [SVM_EXIT_VMLOAD] = vmload_interception, 3385 [SVM_EXIT_VMSAVE] = vmsave_interception, 3386 [SVM_EXIT_STGI] = stgi_interception, 3387 [SVM_EXIT_CLGI] = clgi_interception, 3388 [SVM_EXIT_SKINIT] = skinit_interception, 3389 [SVM_EXIT_RDTSCP] = kvm_handle_invalid_op, 3390 [SVM_EXIT_WBINVD] = kvm_emulate_wbinvd, 3391 [SVM_EXIT_MONITOR] = kvm_emulate_monitor, 3392 [SVM_EXIT_MWAIT] = kvm_emulate_mwait, 3393 [SVM_EXIT_XSETBV] = kvm_emulate_xsetbv, 3394 [SVM_EXIT_RDPRU] = kvm_handle_invalid_op, 3395 [SVM_EXIT_EFER_WRITE_TRAP] = efer_trap, 3396 [SVM_EXIT_CR0_WRITE_TRAP] = cr_trap, 3397 [SVM_EXIT_CR4_WRITE_TRAP] = cr_trap, 3398 [SVM_EXIT_CR8_WRITE_TRAP] = cr_trap, 3399 [SVM_EXIT_INVPCID] = invpcid_interception, 3400 [SVM_EXIT_IDLE_HLT] = kvm_emulate_halt, 3401 [SVM_EXIT_NPF] = npf_interception, 3402 [SVM_EXIT_BUS_LOCK] = bus_lock_exit, 3403 [SVM_EXIT_RSM] = rsm_interception, 3404 [SVM_EXIT_AVIC_INCOMPLETE_IPI] = avic_incomplete_ipi_interception, 3405 [SVM_EXIT_AVIC_UNACCELERATED_ACCESS] = avic_unaccelerated_access_interception, 3406 #ifdef CONFIG_KVM_AMD_SEV 3407 [SVM_EXIT_VMGEXIT] = sev_handle_vmgexit, 3408 #endif 3409 }; 3410 3411 static void dump_vmcb(struct kvm_vcpu *vcpu) 3412 { 3413 struct vcpu_svm *svm = to_svm(vcpu); 3414 struct vmcb_control_area *control = &svm->vmcb->control; 3415 struct vmcb_save_area *save = &svm->vmcb->save; 3416 struct vmcb_save_area *save01 = &svm->vmcb01.ptr->save; 3417 char *vm_type; 3418 3419 if (!dump_invalid_vmcb) { 3420 pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n"); 3421 return; 3422 } 3423 3424 guard(mutex)(&vmcb_dump_mutex); 3425 3426 vm_type = is_sev_snp_guest(vcpu) ? "SEV-SNP" : 3427 is_sev_es_guest(vcpu) ? "SEV-ES" : 3428 is_sev_guest(vcpu) ? "SEV" : "SVM"; 3429 3430 pr_err("%s vCPU%u VMCB %p, last attempted VMRUN on CPU %d\n", 3431 vm_type, vcpu->vcpu_id, svm->current_vmcb->ptr, vcpu->arch.last_vmentry_cpu); 3432 pr_err("VMCB Control Area:\n"); 3433 pr_err("%-20s%04x\n", "cr_read:", control->intercepts[INTERCEPT_CR] & 0xffff); 3434 pr_err("%-20s%04x\n", "cr_write:", control->intercepts[INTERCEPT_CR] >> 16); 3435 pr_err("%-20s%04x\n", "dr_read:", control->intercepts[INTERCEPT_DR] & 0xffff); 3436 pr_err("%-20s%04x\n", "dr_write:", control->intercepts[INTERCEPT_DR] >> 16); 3437 pr_err("%-20s%08x\n", "exceptions:", control->intercepts[INTERCEPT_EXCEPTION]); 3438 pr_err("%-20s%08x %08x\n", "intercepts:", 3439 control->intercepts[INTERCEPT_WORD3], 3440 control->intercepts[INTERCEPT_WORD4]); 3441 pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count); 3442 pr_err("%-20s%d\n", "pause filter threshold:", 3443 control->pause_filter_thresh); 3444 pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa); 3445 pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa); 3446 pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset); 3447 pr_err("%-20s%d\n", "asid:", control->asid); 3448 pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl); 3449 pr_err("%-20s%d\n", "erap_ctl:", control->erap_ctl); 3450 pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl); 3451 pr_err("%-20s%08x\n", "int_vector:", control->int_vector); 3452 pr_err("%-20s%08x\n", "int_state:", control->int_state); 3453 pr_err("%-20s%016llx\n", "exit_code:", control->exit_code); 3454 pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1); 3455 pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2); 3456 pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info); 3457 pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err); 3458 pr_err("%-20s%lld\n", "misc_ctl:", control->misc_ctl); 3459 pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3); 3460 pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar); 3461 pr_err("%-20s%016llx\n", "ghcb:", control->ghcb_gpa); 3462 pr_err("%-20s%08x\n", "event_inj:", control->event_inj); 3463 pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err); 3464 pr_err("%-20s%lld\n", "misc_ctl2:", control->misc_ctl2); 3465 pr_err("%-20s%016llx\n", "next_rip:", control->next_rip); 3466 pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page); 3467 pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id); 3468 pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id); 3469 pr_err("%-20s%016llx\n", "vmsa_pa:", control->vmsa_pa); 3470 pr_err("%-20s%016llx\n", "allowed_sev_features:", control->allowed_sev_features); 3471 pr_err("%-20s%016llx\n", "guest_sev_features:", control->guest_sev_features); 3472 3473 if (is_sev_es_guest(vcpu)) { 3474 save = sev_decrypt_vmsa(vcpu); 3475 if (!save) 3476 goto no_vmsa; 3477 3478 save01 = save; 3479 } 3480 3481 pr_err("VMCB State Save Area:\n"); 3482 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3483 "es:", 3484 save->es.selector, save->es.attrib, 3485 save->es.limit, save->es.base); 3486 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3487 "cs:", 3488 save->cs.selector, save->cs.attrib, 3489 save->cs.limit, save->cs.base); 3490 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3491 "ss:", 3492 save->ss.selector, save->ss.attrib, 3493 save->ss.limit, save->ss.base); 3494 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3495 "ds:", 3496 save->ds.selector, save->ds.attrib, 3497 save->ds.limit, save->ds.base); 3498 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3499 "fs:", 3500 save01->fs.selector, save01->fs.attrib, 3501 save01->fs.limit, save01->fs.base); 3502 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3503 "gs:", 3504 save01->gs.selector, save01->gs.attrib, 3505 save01->gs.limit, save01->gs.base); 3506 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3507 "gdtr:", 3508 save->gdtr.selector, save->gdtr.attrib, 3509 save->gdtr.limit, save->gdtr.base); 3510 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3511 "ldtr:", 3512 save01->ldtr.selector, save01->ldtr.attrib, 3513 save01->ldtr.limit, save01->ldtr.base); 3514 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3515 "idtr:", 3516 save->idtr.selector, save->idtr.attrib, 3517 save->idtr.limit, save->idtr.base); 3518 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", 3519 "tr:", 3520 save01->tr.selector, save01->tr.attrib, 3521 save01->tr.limit, save01->tr.base); 3522 pr_err("vmpl: %d cpl: %d efer: %016llx\n", 3523 save->vmpl, save->cpl, save->efer); 3524 pr_err("%-15s %016llx %-13s %016llx\n", 3525 "cr0:", save->cr0, "cr2:", save->cr2); 3526 pr_err("%-15s %016llx %-13s %016llx\n", 3527 "cr3:", save->cr3, "cr4:", save->cr4); 3528 pr_err("%-15s %016llx %-13s %016llx\n", 3529 "dr6:", save->dr6, "dr7:", save->dr7); 3530 pr_err("%-15s %016llx %-13s %016llx\n", 3531 "rip:", save->rip, "rflags:", save->rflags); 3532 pr_err("%-15s %016llx %-13s %016llx\n", 3533 "rsp:", save->rsp, "rax:", save->rax); 3534 pr_err("%-15s %016llx %-13s %016llx\n", 3535 "s_cet:", save->s_cet, "ssp:", save->ssp); 3536 pr_err("%-15s %016llx\n", 3537 "isst_addr:", save->isst_addr); 3538 pr_err("%-15s %016llx %-13s %016llx\n", 3539 "star:", save01->star, "lstar:", save01->lstar); 3540 pr_err("%-15s %016llx %-13s %016llx\n", 3541 "cstar:", save01->cstar, "sfmask:", save01->sfmask); 3542 pr_err("%-15s %016llx %-13s %016llx\n", 3543 "kernel_gs_base:", save01->kernel_gs_base, 3544 "sysenter_cs:", save01->sysenter_cs); 3545 pr_err("%-15s %016llx %-13s %016llx\n", 3546 "sysenter_esp:", save01->sysenter_esp, 3547 "sysenter_eip:", save01->sysenter_eip); 3548 pr_err("%-15s %016llx %-13s %016llx\n", 3549 "gpat:", save->g_pat, "dbgctl:", save->dbgctl); 3550 pr_err("%-15s %016llx %-13s %016llx\n", 3551 "br_from:", save->br_from, "br_to:", save->br_to); 3552 pr_err("%-15s %016llx %-13s %016llx\n", 3553 "excp_from:", save->last_excp_from, 3554 "excp_to:", save->last_excp_to); 3555 3556 if (is_sev_es_guest(vcpu)) { 3557 struct sev_es_save_area *vmsa = (struct sev_es_save_area *)save; 3558 3559 pr_err("%-15s %016llx\n", 3560 "sev_features", vmsa->sev_features); 3561 3562 pr_err("%-15s %016llx %-13s %016llx\n", 3563 "pl0_ssp:", vmsa->pl0_ssp, "pl1_ssp:", vmsa->pl1_ssp); 3564 pr_err("%-15s %016llx %-13s %016llx\n", 3565 "pl2_ssp:", vmsa->pl2_ssp, "pl3_ssp:", vmsa->pl3_ssp); 3566 pr_err("%-15s %016llx\n", 3567 "u_cet:", vmsa->u_cet); 3568 3569 pr_err("%-15s %016llx %-13s %016llx\n", 3570 "rax:", vmsa->rax, "rbx:", vmsa->rbx); 3571 pr_err("%-15s %016llx %-13s %016llx\n", 3572 "rcx:", vmsa->rcx, "rdx:", vmsa->rdx); 3573 pr_err("%-15s %016llx %-13s %016llx\n", 3574 "rsi:", vmsa->rsi, "rdi:", vmsa->rdi); 3575 pr_err("%-15s %016llx %-13s %016llx\n", 3576 "rbp:", vmsa->rbp, "rsp:", vmsa->rsp); 3577 pr_err("%-15s %016llx %-13s %016llx\n", 3578 "r8:", vmsa->r8, "r9:", vmsa->r9); 3579 pr_err("%-15s %016llx %-13s %016llx\n", 3580 "r10:", vmsa->r10, "r11:", vmsa->r11); 3581 pr_err("%-15s %016llx %-13s %016llx\n", 3582 "r12:", vmsa->r12, "r13:", vmsa->r13); 3583 pr_err("%-15s %016llx %-13s %016llx\n", 3584 "r14:", vmsa->r14, "r15:", vmsa->r15); 3585 pr_err("%-15s %016llx %-13s %016llx\n", 3586 "xcr0:", vmsa->xcr0, "xss:", vmsa->xss); 3587 } else { 3588 pr_err("%-15s %016llx %-13s %016lx\n", 3589 "rax:", save->rax, "rbx:", 3590 vcpu->arch.regs[VCPU_REGS_RBX]); 3591 pr_err("%-15s %016lx %-13s %016lx\n", 3592 "rcx:", vcpu->arch.regs[VCPU_REGS_RCX], 3593 "rdx:", vcpu->arch.regs[VCPU_REGS_RDX]); 3594 pr_err("%-15s %016lx %-13s %016lx\n", 3595 "rsi:", vcpu->arch.regs[VCPU_REGS_RSI], 3596 "rdi:", vcpu->arch.regs[VCPU_REGS_RDI]); 3597 pr_err("%-15s %016lx %-13s %016llx\n", 3598 "rbp:", vcpu->arch.regs[VCPU_REGS_RBP], 3599 "rsp:", save->rsp); 3600 #ifdef CONFIG_X86_64 3601 pr_err("%-15s %016lx %-13s %016lx\n", 3602 "r8:", vcpu->arch.regs[VCPU_REGS_R8], 3603 "r9:", vcpu->arch.regs[VCPU_REGS_R9]); 3604 pr_err("%-15s %016lx %-13s %016lx\n", 3605 "r10:", vcpu->arch.regs[VCPU_REGS_R10], 3606 "r11:", vcpu->arch.regs[VCPU_REGS_R11]); 3607 pr_err("%-15s %016lx %-13s %016lx\n", 3608 "r12:", vcpu->arch.regs[VCPU_REGS_R12], 3609 "r13:", vcpu->arch.regs[VCPU_REGS_R13]); 3610 pr_err("%-15s %016lx %-13s %016lx\n", 3611 "r14:", vcpu->arch.regs[VCPU_REGS_R14], 3612 "r15:", vcpu->arch.regs[VCPU_REGS_R15]); 3613 #endif 3614 } 3615 3616 no_vmsa: 3617 if (is_sev_es_guest(vcpu)) 3618 sev_free_decrypted_vmsa(vcpu, save); 3619 } 3620 3621 int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 __exit_code) 3622 { 3623 u32 exit_code = __exit_code; 3624 3625 /* 3626 * SVM uses negative values, i.e. 64-bit values, to indicate that VMRUN 3627 * failed. Report all such errors to userspace (note, VMEXIT_INVALID, 3628 * a.k.a. SVM_EXIT_ERR, is special cased by svm_handle_exit()). Skip 3629 * the check when running as a VM, as KVM has historically left garbage 3630 * in bits 63:32, i.e. running KVM-on-KVM would hit false positives if 3631 * the underlying kernel is buggy. 3632 */ 3633 if (!cpu_feature_enabled(X86_FEATURE_HYPERVISOR) && 3634 (u64)exit_code != __exit_code) 3635 goto unexpected_vmexit; 3636 3637 #ifdef CONFIG_MITIGATION_RETPOLINE 3638 if (exit_code == SVM_EXIT_MSR) 3639 return msr_interception(vcpu); 3640 else if (exit_code == SVM_EXIT_VINTR) 3641 return interrupt_window_interception(vcpu); 3642 else if (exit_code == SVM_EXIT_INTR) 3643 return intr_interception(vcpu); 3644 else if (exit_code == SVM_EXIT_HLT || exit_code == SVM_EXIT_IDLE_HLT) 3645 return kvm_emulate_halt(vcpu); 3646 else if (exit_code == SVM_EXIT_NPF) 3647 return npf_interception(vcpu); 3648 #ifdef CONFIG_KVM_AMD_SEV 3649 else if (exit_code == SVM_EXIT_VMGEXIT) 3650 return sev_handle_vmgexit(vcpu); 3651 #endif 3652 #endif 3653 if (exit_code >= ARRAY_SIZE(svm_exit_handlers)) 3654 goto unexpected_vmexit; 3655 3656 exit_code = array_index_nospec(exit_code, ARRAY_SIZE(svm_exit_handlers)); 3657 if (!svm_exit_handlers[exit_code]) 3658 goto unexpected_vmexit; 3659 3660 return svm_exit_handlers[exit_code](vcpu); 3661 3662 unexpected_vmexit: 3663 dump_vmcb(vcpu); 3664 kvm_prepare_unexpected_reason_exit(vcpu, __exit_code); 3665 return 0; 3666 } 3667 3668 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, 3669 u64 *info1, u64 *info2, 3670 u32 *intr_info, u32 *error_code) 3671 { 3672 struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control; 3673 3674 *reason = control->exit_code; 3675 *info1 = control->exit_info_1; 3676 *info2 = control->exit_info_2; 3677 *intr_info = control->exit_int_info; 3678 if ((*intr_info & SVM_EXITINTINFO_VALID) && 3679 (*intr_info & SVM_EXITINTINFO_VALID_ERR)) 3680 *error_code = control->exit_int_info_err; 3681 else 3682 *error_code = 0; 3683 } 3684 3685 static void svm_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info, 3686 u32 *error_code) 3687 { 3688 struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control; 3689 3690 *intr_info = control->event_inj; 3691 3692 if ((*intr_info & SVM_EXITINTINFO_VALID) && 3693 (*intr_info & SVM_EXITINTINFO_VALID_ERR)) 3694 *error_code = control->event_inj_err; 3695 else 3696 *error_code = 0; 3697 3698 } 3699 3700 static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) 3701 { 3702 struct vcpu_svm *svm = to_svm(vcpu); 3703 struct kvm_run *kvm_run = vcpu->run; 3704 3705 if (unlikely(exit_fastpath == EXIT_FASTPATH_EXIT_USERSPACE)) 3706 return 0; 3707 3708 if (is_guest_mode(vcpu)) { 3709 int vmexit; 3710 3711 trace_kvm_nested_vmexit(vcpu, KVM_ISA_SVM); 3712 3713 vmexit = nested_svm_exit_special(svm); 3714 3715 if (vmexit == NESTED_EXIT_CONTINUE) 3716 vmexit = nested_svm_exit_handled(svm); 3717 3718 if (vmexit == NESTED_EXIT_DONE) 3719 return 1; 3720 } 3721 3722 if (svm_is_vmrun_failure(svm->vmcb->control.exit_code)) { 3723 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY; 3724 kvm_run->fail_entry.hardware_entry_failure_reason 3725 = svm->vmcb->control.exit_code; 3726 kvm_run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu; 3727 dump_vmcb(vcpu); 3728 return 0; 3729 } 3730 3731 if (exit_fastpath != EXIT_FASTPATH_NONE) 3732 return 1; 3733 3734 return svm_invoke_exit_handler(vcpu, svm->vmcb->control.exit_code); 3735 } 3736 3737 static void svm_set_nested_run_soft_int_state(struct kvm_vcpu *vcpu) 3738 { 3739 struct vcpu_svm *svm = to_svm(vcpu); 3740 3741 svm->soft_int_csbase = svm->vmcb->save.cs.base; 3742 svm->soft_int_old_rip = kvm_rip_read(vcpu); 3743 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS)) 3744 svm->soft_int_next_rip = kvm_rip_read(vcpu); 3745 } 3746 3747 static int pre_svm_run(struct kvm_vcpu *vcpu) 3748 { 3749 struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu); 3750 struct vcpu_svm *svm = to_svm(vcpu); 3751 3752 /* 3753 * If the previous vmrun of the vmcb occurred on a different physical 3754 * cpu, then mark the vmcb dirty and assign a new asid. Hardware's 3755 * vmcb clean bits are per logical CPU, as are KVM's asid assignments. 3756 */ 3757 if (unlikely(svm->current_vmcb->cpu != vcpu->cpu)) { 3758 svm->current_vmcb->asid_generation = 0; 3759 vmcb_mark_all_dirty(svm->vmcb); 3760 svm->current_vmcb->cpu = vcpu->cpu; 3761 } 3762 3763 if (is_sev_guest(vcpu)) 3764 return pre_sev_run(svm, vcpu->cpu); 3765 3766 /* FIXME: handle wraparound of asid_generation */ 3767 if (svm->current_vmcb->asid_generation != sd->asid_generation) 3768 new_asid(svm, sd); 3769 3770 return 0; 3771 } 3772 3773 static void svm_inject_nmi(struct kvm_vcpu *vcpu) 3774 { 3775 struct vcpu_svm *svm = to_svm(vcpu); 3776 3777 svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI; 3778 3779 if (svm->nmi_l1_to_l2) 3780 return; 3781 3782 /* 3783 * No need to manually track NMI masking when vNMI is enabled, hardware 3784 * automatically sets V_NMI_BLOCKING_MASK as appropriate, including the 3785 * case where software directly injects an NMI. 3786 */ 3787 if (!is_vnmi_enabled(svm)) { 3788 svm->nmi_masked = true; 3789 svm_set_iret_intercept(svm); 3790 } 3791 ++vcpu->stat.nmi_injections; 3792 } 3793 3794 static bool svm_is_vnmi_pending(struct kvm_vcpu *vcpu) 3795 { 3796 struct vcpu_svm *svm = to_svm(vcpu); 3797 3798 if (!is_vnmi_enabled(svm)) 3799 return false; 3800 3801 return !!(svm->vmcb->control.int_ctl & V_NMI_PENDING_MASK); 3802 } 3803 3804 static bool svm_set_vnmi_pending(struct kvm_vcpu *vcpu) 3805 { 3806 struct vcpu_svm *svm = to_svm(vcpu); 3807 3808 if (!is_vnmi_enabled(svm)) 3809 return false; 3810 3811 if (svm->vmcb->control.int_ctl & V_NMI_PENDING_MASK) 3812 return false; 3813 3814 svm->vmcb->control.int_ctl |= V_NMI_PENDING_MASK; 3815 vmcb_mark_dirty(svm->vmcb, VMCB_INTR); 3816 3817 /* 3818 * Because the pending NMI is serviced by hardware, KVM can't know when 3819 * the NMI is "injected", but for all intents and purposes, passing the 3820 * NMI off to hardware counts as injection. 3821 */ 3822 ++vcpu->stat.nmi_injections; 3823 3824 return true; 3825 } 3826 3827 static void svm_inject_irq(struct kvm_vcpu *vcpu, bool reinjected) 3828 { 3829 struct kvm_queued_interrupt *intr = &vcpu->arch.interrupt; 3830 struct vcpu_svm *svm = to_svm(vcpu); 3831 u32 type; 3832 3833 if (intr->soft) { 3834 if (svm_update_soft_interrupt_rip(vcpu, intr->nr)) 3835 return; 3836 3837 type = SVM_EVTINJ_TYPE_SOFT; 3838 } else { 3839 type = SVM_EVTINJ_TYPE_INTR; 3840 } 3841 3842 /* 3843 * If AVIC was inhibited in order to detect an IRQ window, and there's 3844 * no other injectable interrupts pending or L2 is active (see below), 3845 * then drop the inhibit as the window has served its purpose. 3846 * 3847 * If L2 is active, this path is reachable if L1 is not intercepting 3848 * IRQs, i.e. if KVM is injecting L1 IRQs into L2. AVIC is locally 3849 * inhibited while L2 is active; drop the VM-wide inhibit to optimize 3850 * the case in which the interrupt window was requested while L1 was 3851 * active (the vCPU was not running nested). 3852 */ 3853 if (svm->avic_irq_window && 3854 (!kvm_cpu_has_injectable_intr(vcpu) || is_guest_mode(vcpu))) { 3855 svm->avic_irq_window = false; 3856 kvm_dec_apicv_irq_window_req(svm->vcpu.kvm); 3857 } 3858 3859 trace_kvm_inj_virq(intr->nr, intr->soft, reinjected); 3860 ++vcpu->stat.irq_injections; 3861 3862 svm->vmcb->control.event_inj = intr->nr | SVM_EVTINJ_VALID | type; 3863 } 3864 3865 static void svm_fixup_nested_rips(struct kvm_vcpu *vcpu) 3866 { 3867 struct vcpu_svm *svm = to_svm(vcpu); 3868 3869 if (!is_guest_mode(vcpu) || !vcpu->arch.nested_run_pending) 3870 return; 3871 3872 /* 3873 * If nrips is supported in hardware but not exposed to L1, stuff the 3874 * actual L2 RIP to emulate what a nrips=0 CPU would do (L1 is 3875 * responsible for advancing RIP prior to injecting the event). Once L2 3876 * runs after L1 executes VMRUN, NextRIP is updated by the CPU and/or 3877 * KVM, and this is no longer needed. 3878 * 3879 * This is done here (as opposed to when preparing vmcb02) to use the 3880 * most up-to-date value of RIP regardless of the order of restoring 3881 * registers and nested state in the vCPU save+restore path. 3882 */ 3883 if (boot_cpu_has(X86_FEATURE_NRIPS) && 3884 !guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS)) 3885 svm->vmcb->control.next_rip = kvm_rip_read(vcpu); 3886 3887 /* 3888 * Simiarly, initialize the soft int metadata here to use the most 3889 * up-to-date values of RIP and CS base, regardless of restore order. 3890 */ 3891 if (svm->soft_int_injected) 3892 svm_set_nested_run_soft_int_state(vcpu); 3893 } 3894 3895 void svm_complete_interrupt_delivery(struct kvm_vcpu *vcpu, int delivery_mode, 3896 int trig_mode, int vector) 3897 { 3898 /* 3899 * apic->apicv_active must be read after vcpu->mode. 3900 * Pairs with smp_store_release in vcpu_enter_guest. 3901 */ 3902 bool in_guest_mode = (smp_load_acquire(&vcpu->mode) == IN_GUEST_MODE); 3903 3904 /* Note, this is called iff the local APIC is in-kernel. */ 3905 if (!READ_ONCE(vcpu->arch.apic->apicv_active)) { 3906 /* Process the interrupt via kvm_check_and_inject_events(). */ 3907 kvm_make_request(KVM_REQ_EVENT, vcpu); 3908 kvm_vcpu_kick(vcpu); 3909 return; 3910 } 3911 3912 trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, trig_mode, vector); 3913 if (in_guest_mode) { 3914 /* 3915 * Signal the doorbell to tell hardware to inject the IRQ. If 3916 * the vCPU exits the guest before the doorbell chimes, hardware 3917 * will automatically process AVIC interrupts at the next VMRUN. 3918 */ 3919 avic_ring_doorbell(vcpu); 3920 } else { 3921 /* 3922 * Wake the vCPU if it was blocking. KVM will then detect the 3923 * pending IRQ when checking if the vCPU has a wake event. 3924 */ 3925 kvm_vcpu_wake_up(vcpu); 3926 } 3927 } 3928 3929 static void svm_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, 3930 int trig_mode, int vector) 3931 { 3932 kvm_lapic_set_irr(vector, apic); 3933 3934 /* 3935 * Pairs with the smp_mb_*() after setting vcpu->guest_mode in 3936 * vcpu_enter_guest() to ensure the write to the vIRR is ordered before 3937 * the read of guest_mode. This guarantees that either VMRUN will see 3938 * and process the new vIRR entry, or that svm_complete_interrupt_delivery 3939 * will signal the doorbell if the CPU has already entered the guest. 3940 */ 3941 smp_mb__after_atomic(); 3942 svm_complete_interrupt_delivery(apic->vcpu, delivery_mode, trig_mode, vector); 3943 } 3944 3945 static void svm_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) 3946 { 3947 struct vcpu_svm *svm = to_svm(vcpu); 3948 3949 /* 3950 * SEV-ES guests must always keep the CR intercepts cleared. CR 3951 * tracking is done using the CR write traps. 3952 */ 3953 if (is_sev_es_guest(vcpu)) 3954 return; 3955 3956 if (nested_svm_virtualize_tpr(vcpu)) 3957 return; 3958 3959 svm_clr_intercept(svm, INTERCEPT_CR8_WRITE); 3960 3961 if (irr == -1) 3962 return; 3963 3964 if (tpr >= irr) 3965 svm_set_intercept(svm, INTERCEPT_CR8_WRITE); 3966 } 3967 3968 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu) 3969 { 3970 struct vcpu_svm *svm = to_svm(vcpu); 3971 3972 if (is_vnmi_enabled(svm)) 3973 return svm->vmcb->control.int_ctl & V_NMI_BLOCKING_MASK; 3974 else 3975 return svm->nmi_masked; 3976 } 3977 3978 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) 3979 { 3980 struct vcpu_svm *svm = to_svm(vcpu); 3981 3982 if (is_vnmi_enabled(svm)) { 3983 if (masked) 3984 svm->vmcb->control.int_ctl |= V_NMI_BLOCKING_MASK; 3985 else 3986 svm->vmcb->control.int_ctl &= ~V_NMI_BLOCKING_MASK; 3987 3988 } else { 3989 svm->nmi_masked = masked; 3990 if (masked) 3991 svm_set_iret_intercept(svm); 3992 else 3993 svm_clr_iret_intercept(svm); 3994 } 3995 } 3996 3997 bool svm_nmi_blocked(struct kvm_vcpu *vcpu) 3998 { 3999 struct vcpu_svm *svm = to_svm(vcpu); 4000 struct vmcb *vmcb = svm->vmcb; 4001 4002 if (!gif_set(svm)) 4003 return true; 4004 4005 if (is_guest_mode(vcpu) && nested_exit_on_nmi(svm)) 4006 return false; 4007 4008 if (svm_get_nmi_mask(vcpu)) 4009 return true; 4010 4011 return vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK; 4012 } 4013 4014 static int svm_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 4015 { 4016 struct vcpu_svm *svm = to_svm(vcpu); 4017 if (vcpu->arch.nested_run_pending) 4018 return -EBUSY; 4019 4020 if (svm_nmi_blocked(vcpu)) 4021 return 0; 4022 4023 /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */ 4024 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(svm)) 4025 return -EBUSY; 4026 return 1; 4027 } 4028 4029 bool svm_interrupt_blocked(struct kvm_vcpu *vcpu) 4030 { 4031 struct vcpu_svm *svm = to_svm(vcpu); 4032 struct vmcb *vmcb = svm->vmcb; 4033 4034 if (!gif_set(svm)) 4035 return true; 4036 4037 if (is_guest_mode(vcpu)) { 4038 /* As long as interrupts are being delivered... */ 4039 if ((svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK) 4040 ? !(svm->vmcb01.ptr->save.rflags & X86_EFLAGS_IF) 4041 : !(kvm_get_rflags(vcpu) & X86_EFLAGS_IF)) 4042 return true; 4043 4044 /* ... vmexits aren't blocked by the interrupt shadow */ 4045 if (nested_exit_on_intr(svm)) 4046 return false; 4047 } else { 4048 if (!svm_get_if_flag(vcpu)) 4049 return true; 4050 } 4051 4052 return (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK); 4053 } 4054 4055 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) 4056 { 4057 struct vcpu_svm *svm = to_svm(vcpu); 4058 4059 if (vcpu->arch.nested_run_pending) 4060 return -EBUSY; 4061 4062 if (svm_interrupt_blocked(vcpu)) 4063 return 0; 4064 4065 /* 4066 * An IRQ must not be injected into L2 if it's supposed to VM-Exit, 4067 * e.g. if the IRQ arrived asynchronously after checking nested events. 4068 */ 4069 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(svm)) 4070 return -EBUSY; 4071 4072 return 1; 4073 } 4074 4075 static void svm_enable_irq_window(struct kvm_vcpu *vcpu) 4076 { 4077 struct vcpu_svm *svm = to_svm(vcpu); 4078 4079 /* 4080 * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes 4081 * 1, because that's a separate STGI/VMRUN intercept. The next time we 4082 * get that intercept, this function will be called again though and 4083 * we'll get the vintr intercept. However, if the vGIF feature is 4084 * enabled, the STGI interception will not occur. Enable the irq 4085 * window under the assumption that the hardware will set the GIF. 4086 */ 4087 if (vgif || gif_set(svm)) { 4088 /* 4089 * KVM only enables IRQ windows when AVIC is enabled if there's 4090 * pending ExtINT since it cannot be injected via AVIC (ExtINT 4091 * bypasses the local APIC). V_IRQ is ignored by hardware when 4092 * AVIC is enabled, and so KVM needs to temporarily disable 4093 * AVIC in order to detect when it's ok to inject the ExtINT. 4094 * 4095 * If running nested, AVIC is already locally inhibited on this 4096 * vCPU (L2 vCPUs use a different MMU that never maps the AVIC 4097 * backing page), therefore there is no need to increment the 4098 * VM-wide AVIC inhibit. KVM will re-evaluate events when the 4099 * vCPU exits to L1 and enable an IRQ window if the ExtINT is 4100 * still pending. 4101 * 4102 * Note, the IRQ window inhibit needs to be updated even if 4103 * AVIC is inhibited for a different reason, as KVM needs to 4104 * keep AVIC inhibited if the other reason is cleared and there 4105 * is still an injectable interrupt pending. 4106 */ 4107 if (enable_apicv && !svm->avic_irq_window && !is_guest_mode(vcpu)) { 4108 svm->avic_irq_window = true; 4109 kvm_inc_apicv_irq_window_req(vcpu->kvm); 4110 } 4111 4112 svm_set_vintr(svm); 4113 } 4114 } 4115 4116 static void svm_enable_nmi_window(struct kvm_vcpu *vcpu) 4117 { 4118 struct vcpu_svm *svm = to_svm(vcpu); 4119 4120 /* 4121 * If NMIs are outright masked, i.e. the vCPU is already handling an 4122 * NMI, and KVM has not yet intercepted an IRET, then there is nothing 4123 * more to do at this time as KVM has already enabled IRET intercepts. 4124 * If KVM has already intercepted IRET, then single-step over the IRET, 4125 * as NMIs aren't architecturally unmasked until the IRET completes. 4126 * 4127 * If vNMI is enabled, KVM should never request an NMI window if NMIs 4128 * are masked, as KVM allows at most one to-be-injected NMI and one 4129 * pending NMI. If two NMIs arrive simultaneously, KVM will inject one 4130 * NMI and set V_NMI_PENDING for the other, but if and only if NMIs are 4131 * unmasked. KVM _will_ request an NMI window in some situations, e.g. 4132 * if the vCPU is in an STI shadow or if GIF=0, KVM can't immediately 4133 * inject the NMI. In those situations, KVM needs to single-step over 4134 * the STI shadow or intercept STGI. 4135 */ 4136 if (svm_get_nmi_mask(vcpu)) { 4137 WARN_ON_ONCE(is_vnmi_enabled(svm)); 4138 4139 if (!svm->awaiting_iret_completion) 4140 return; /* IRET will cause a vm exit */ 4141 } 4142 4143 /* 4144 * SEV-ES guests are responsible for signaling when a vCPU is ready to 4145 * receive a new NMI, as SEV-ES guests can't be single-stepped, i.e. 4146 * KVM can't intercept and single-step IRET to detect when NMIs are 4147 * unblocked (architecturally speaking). See SVM_VMGEXIT_NMI_COMPLETE. 4148 * 4149 * Note, GIF is guaranteed to be '1' for SEV-ES guests as hardware 4150 * ignores SEV-ES guest writes to EFER.SVME *and* CLGI/STGI are not 4151 * supported NAEs in the GHCB protocol. 4152 */ 4153 if (is_sev_es_guest(vcpu)) 4154 return; 4155 4156 if (!gif_set(svm)) { 4157 if (vgif) 4158 svm_set_intercept(svm, INTERCEPT_STGI); 4159 return; /* STGI will cause a vm exit */ 4160 } 4161 4162 /* 4163 * Something prevents NMI from been injected. Single step over possible 4164 * problem (IRET or exception injection or interrupt shadow) 4165 */ 4166 svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu); 4167 svm->nmi_singlestep = true; 4168 svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF); 4169 } 4170 4171 static void svm_flush_tlb_asid(struct kvm_vcpu *vcpu) 4172 { 4173 struct vcpu_svm *svm = to_svm(vcpu); 4174 4175 /* 4176 * Unlike VMX, SVM doesn't provide a way to flush only NPT TLB entries. 4177 * A TLB flush for the current ASID flushes both "host" and "guest" TLB 4178 * entries, and thus is a superset of Hyper-V's fine grained flushing. 4179 */ 4180 kvm_hv_vcpu_purge_flush_tlb(vcpu); 4181 4182 /* 4183 * Flush only the current ASID even if the TLB flush was invoked via 4184 * kvm_flush_remote_tlbs(). Although flushing remote TLBs requires all 4185 * ASIDs to be flushed, KVM uses a single ASID for L1 and L2, and 4186 * unconditionally does a TLB flush on both nested VM-Enter and nested 4187 * VM-Exit (via kvm_mmu_reset_context()). 4188 */ 4189 if (static_cpu_has(X86_FEATURE_FLUSHBYASID)) 4190 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID; 4191 else 4192 svm->current_vmcb->asid_generation--; 4193 } 4194 4195 static void svm_flush_tlb_current(struct kvm_vcpu *vcpu) 4196 { 4197 hpa_t root_tdp = vcpu->arch.mmu->root.hpa; 4198 4199 /* 4200 * When running on Hyper-V with EnlightenedNptTlb enabled, explicitly 4201 * flush the NPT mappings via hypercall as flushing the ASID only 4202 * affects virtual to physical mappings, it does not invalidate guest 4203 * physical to host physical mappings. 4204 */ 4205 if (svm_hv_is_enlightened_tlb_enabled(vcpu) && VALID_PAGE(root_tdp)) 4206 hyperv_flush_guest_mapping(root_tdp); 4207 4208 svm_flush_tlb_asid(vcpu); 4209 } 4210 4211 static void svm_flush_tlb_all(struct kvm_vcpu *vcpu) 4212 { 4213 /* 4214 * When running on Hyper-V with EnlightenedNptTlb enabled, remote TLB 4215 * flushes should be routed to hv_flush_remote_tlbs() without requesting 4216 * a "regular" remote flush. Reaching this point means either there's 4217 * a KVM bug or a prior hv_flush_remote_tlbs() call failed, both of 4218 * which might be fatal to the guest. Yell, but try to recover. 4219 */ 4220 if (WARN_ON_ONCE(svm_hv_is_enlightened_tlb_enabled(vcpu))) 4221 hv_flush_remote_tlbs(vcpu->kvm); 4222 4223 svm_flush_tlb_asid(vcpu); 4224 } 4225 4226 static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva) 4227 { 4228 struct vcpu_svm *svm = to_svm(vcpu); 4229 4230 invlpga(gva, svm->vmcb->control.asid); 4231 } 4232 4233 static void svm_flush_tlb_guest(struct kvm_vcpu *vcpu) 4234 { 4235 kvm_register_mark_dirty(vcpu, VCPU_REG_ERAPS); 4236 4237 svm_flush_tlb_asid(vcpu); 4238 } 4239 4240 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu) 4241 { 4242 struct vcpu_svm *svm = to_svm(vcpu); 4243 4244 if (nested_svm_virtualize_tpr(vcpu)) 4245 return; 4246 4247 if (!svm_is_intercept(svm, INTERCEPT_CR8_WRITE)) { 4248 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK; 4249 kvm_set_cr8(vcpu, cr8); 4250 } 4251 } 4252 4253 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu) 4254 { 4255 struct vcpu_svm *svm = to_svm(vcpu); 4256 u64 cr8; 4257 4258 if (nested_svm_virtualize_tpr(vcpu)) 4259 return; 4260 4261 cr8 = kvm_get_cr8(vcpu); 4262 svm->vmcb->control.int_ctl &= ~V_TPR_MASK; 4263 svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK; 4264 } 4265 4266 static void svm_complete_soft_interrupt(struct kvm_vcpu *vcpu, u8 vector, 4267 int type) 4268 { 4269 bool is_exception = (type == SVM_EXITINTINFO_TYPE_EXEPT); 4270 bool is_soft = (type == SVM_EXITINTINFO_TYPE_SOFT); 4271 struct vcpu_svm *svm = to_svm(vcpu); 4272 4273 /* 4274 * Initialize the soft int fields *before* reading them below if KVM 4275 * aborted entry to the guest with a nested VMRUN pending. To ensure 4276 * KVM uses up-to-date values for RIP and CS base across save/restore, 4277 * regardless of restore order, KVM waits to set the soft int fields 4278 * until VMRUN is imminent. But when canceling injection, KVM requeues 4279 * the soft int and will reinject it via the standard injection flow, 4280 * and so KVM needs to grab the state from the pending nested VMRUN. 4281 */ 4282 if (is_guest_mode(vcpu) && vcpu->arch.nested_run_pending) 4283 svm_set_nested_run_soft_int_state(vcpu); 4284 4285 /* 4286 * If NRIPS is enabled, KVM must snapshot the pre-VMRUN next_rip that's 4287 * associated with the original soft exception/interrupt. next_rip is 4288 * cleared on all exits that can occur while vectoring an event, so KVM 4289 * needs to manually set next_rip for re-injection. Unlike the !nrips 4290 * case below, this needs to be done if and only if KVM is re-injecting 4291 * the same event, i.e. if the event is a soft exception/interrupt, 4292 * otherwise next_rip is unused on VMRUN. 4293 */ 4294 if (nrips && (is_soft || (is_exception && kvm_exception_is_soft(vector))) && 4295 kvm_is_linear_rip(vcpu, svm->soft_int_old_rip + svm->soft_int_csbase)) 4296 svm->vmcb->control.next_rip = svm->soft_int_next_rip; 4297 /* 4298 * If NRIPS isn't enabled, KVM must manually advance RIP prior to 4299 * injecting the soft exception/interrupt. That advancement needs to 4300 * be unwound if vectoring didn't complete. Note, the new event may 4301 * not be the injected event, e.g. if KVM injected an INTn, the INTn 4302 * hit a #NP in the guest, and the #NP encountered a #PF, the #NP will 4303 * be the reported vectored event, but RIP still needs to be unwound. 4304 */ 4305 else if (!nrips && (is_soft || is_exception) && 4306 kvm_is_linear_rip(vcpu, svm->soft_int_next_rip + svm->soft_int_csbase)) 4307 kvm_rip_write(vcpu, svm->soft_int_old_rip); 4308 } 4309 4310 static void svm_complete_interrupts(struct kvm_vcpu *vcpu) 4311 { 4312 struct vcpu_svm *svm = to_svm(vcpu); 4313 u8 vector; 4314 int type; 4315 u32 exitintinfo = svm->vmcb->control.exit_int_info; 4316 bool nmi_l1_to_l2 = svm->nmi_l1_to_l2; 4317 bool soft_int_injected = svm->soft_int_injected; 4318 4319 svm->nmi_l1_to_l2 = false; 4320 svm->soft_int_injected = false; 4321 4322 /* 4323 * If we've made progress since setting awaiting_iret_completion, we've 4324 * executed an IRET and can allow NMI injection. 4325 */ 4326 if (svm->awaiting_iret_completion && 4327 kvm_rip_read(vcpu) != svm->nmi_iret_rip) { 4328 svm->awaiting_iret_completion = false; 4329 svm->nmi_masked = false; 4330 kvm_make_request(KVM_REQ_EVENT, vcpu); 4331 } 4332 4333 vcpu->arch.nmi_injected = false; 4334 kvm_clear_exception_queue(vcpu); 4335 kvm_clear_interrupt_queue(vcpu); 4336 4337 if (!(exitintinfo & SVM_EXITINTINFO_VALID)) 4338 return; 4339 4340 kvm_make_request(KVM_REQ_EVENT, vcpu); 4341 4342 vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK; 4343 type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK; 4344 4345 if (soft_int_injected) 4346 svm_complete_soft_interrupt(vcpu, vector, type); 4347 4348 switch (type) { 4349 case SVM_EXITINTINFO_TYPE_NMI: 4350 vcpu->arch.nmi_injected = true; 4351 svm->nmi_l1_to_l2 = nmi_l1_to_l2; 4352 break; 4353 case SVM_EXITINTINFO_TYPE_EXEPT: { 4354 u32 error_code = 0; 4355 4356 /* 4357 * Never re-inject a #VC exception. 4358 */ 4359 if (vector == X86_TRAP_VC) 4360 break; 4361 4362 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) 4363 error_code = svm->vmcb->control.exit_int_info_err; 4364 4365 kvm_requeue_exception(vcpu, vector, 4366 exitintinfo & SVM_EXITINTINFO_VALID_ERR, 4367 error_code); 4368 break; 4369 } 4370 case SVM_EXITINTINFO_TYPE_INTR: 4371 kvm_queue_interrupt(vcpu, vector, false); 4372 break; 4373 case SVM_EXITINTINFO_TYPE_SOFT: 4374 kvm_queue_interrupt(vcpu, vector, true); 4375 break; 4376 default: 4377 break; 4378 } 4379 4380 } 4381 4382 static void svm_cancel_injection(struct kvm_vcpu *vcpu) 4383 { 4384 struct vcpu_svm *svm = to_svm(vcpu); 4385 struct vmcb_control_area *control = &svm->vmcb->control; 4386 4387 control->exit_int_info = control->event_inj; 4388 control->exit_int_info_err = control->event_inj_err; 4389 control->event_inj = 0; 4390 svm_complete_interrupts(vcpu); 4391 } 4392 4393 static int svm_vcpu_pre_run(struct kvm_vcpu *vcpu) 4394 { 4395 #ifdef CONFIG_KVM_AMD_SEV 4396 if (to_kvm_sev_info(vcpu->kvm)->need_init) 4397 return -EINVAL; 4398 #endif 4399 4400 return 1; 4401 } 4402 4403 static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu) 4404 { 4405 struct vcpu_svm *svm = to_svm(vcpu); 4406 struct vmcb_control_area *control = &svm->vmcb->control; 4407 4408 /* 4409 * Next RIP must be provided as IRQs are disabled, and accessing guest 4410 * memory to decode the instruction might fault, i.e. might sleep. 4411 */ 4412 if (!nrips || !control->next_rip) 4413 return EXIT_FASTPATH_NONE; 4414 4415 if (is_guest_mode(vcpu)) 4416 return EXIT_FASTPATH_NONE; 4417 4418 switch (control->exit_code) { 4419 case SVM_EXIT_MSR: 4420 if (!control->exit_info_1) 4421 break; 4422 return handle_fastpath_wrmsr(vcpu); 4423 case SVM_EXIT_HLT: 4424 return handle_fastpath_hlt(vcpu); 4425 case SVM_EXIT_INVD: 4426 return handle_fastpath_invd(vcpu); 4427 default: 4428 break; 4429 } 4430 4431 return EXIT_FASTPATH_NONE; 4432 } 4433 4434 static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, unsigned enter_flags) 4435 { 4436 struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu); 4437 struct vcpu_svm *svm = to_svm(vcpu); 4438 4439 guest_state_enter_irqoff(); 4440 4441 /* 4442 * Set RFLAGS.IF prior to VMRUN, as the host's RFLAGS.IF at the time of 4443 * VMRUN controls whether or not physical IRQs are masked (KVM always 4444 * runs with V_INTR_MASKING_MASK). Toggle RFLAGS.IF here to avoid the 4445 * temptation to do STI+VMRUN+CLI, as AMD CPUs bleed the STI shadow 4446 * into guest state if delivery of an event during VMRUN triggers a 4447 * #VMEXIT, and the guest_state transitions already tell lockdep that 4448 * IRQs are being enabled/disabled. Note! GIF=0 for the entirety of 4449 * this path, so IRQs aren't actually unmasked while running host code. 4450 */ 4451 raw_local_irq_enable(); 4452 4453 amd_clear_divider(); 4454 4455 if (is_sev_es_guest(vcpu)) 4456 __svm_sev_es_vcpu_run(svm, enter_flags, 4457 sev_es_host_save_area(sd)); 4458 else 4459 __svm_vcpu_run(svm, enter_flags); 4460 4461 raw_local_irq_disable(); 4462 4463 guest_state_exit_irqoff(); 4464 } 4465 4466 static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) 4467 { 4468 bool force_immediate_exit = run_flags & KVM_RUN_FORCE_IMMEDIATE_EXIT; 4469 struct vcpu_svm *svm = to_svm(vcpu); 4470 unsigned enter_flags = 0; 4471 4472 if (!msr_write_intercepted(svm, MSR_IA32_SPEC_CTRL)) 4473 enter_flags |= KVM_ENTER_SAVE_SPEC_CTRL; 4474 4475 trace_kvm_entry(vcpu, force_immediate_exit); 4476 4477 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; 4478 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; 4479 svm->vmcb->save.rip = vcpu->arch.rip; 4480 4481 /* 4482 * Disable singlestep if we're injecting an interrupt/exception. 4483 * We don't want our modified rflags to be pushed on the stack where 4484 * we might not be able to easily reset them if we disabled NMI 4485 * singlestep later. 4486 */ 4487 if (svm->nmi_singlestep && svm->vmcb->control.event_inj) { 4488 /* 4489 * Event injection happens before external interrupts cause a 4490 * vmexit and interrupts are disabled here, so smp_send_reschedule 4491 * is enough to force an immediate vmexit. 4492 */ 4493 disable_nmi_singlestep(svm); 4494 force_immediate_exit = true; 4495 } 4496 4497 if (force_immediate_exit) 4498 smp_send_reschedule(vcpu->cpu); 4499 4500 if (pre_svm_run(vcpu)) { 4501 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; 4502 vcpu->run->fail_entry.hardware_entry_failure_reason = SVM_EXIT_ERR; 4503 vcpu->run->fail_entry.cpu = vcpu->cpu; 4504 return EXIT_FASTPATH_EXIT_USERSPACE; 4505 } 4506 4507 sync_lapic_to_cr8(vcpu); 4508 4509 if (unlikely(svm->asid != svm->vmcb->control.asid)) { 4510 svm->vmcb->control.asid = svm->asid; 4511 vmcb_mark_dirty(svm->vmcb, VMCB_ASID); 4512 } 4513 svm->vmcb->save.cr2 = vcpu->arch.cr2; 4514 4515 if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS) && 4516 kvm_register_is_dirty(vcpu, VCPU_REG_ERAPS)) 4517 svm->vmcb->control.erap_ctl |= ERAP_CONTROL_CLEAR_RAP; 4518 4519 svm_fixup_nested_rips(vcpu); 4520 4521 svm_hv_update_vp_id(svm->vmcb, vcpu); 4522 4523 /* 4524 * Run with all-zero DR6 unless the guest can write DR6 freely, so that 4525 * KVM can get the exact cause of a #DB. Note, loading guest DR6 from 4526 * KVM's snapshot is only necessary when DR accesses won't exit. 4527 */ 4528 if (unlikely(run_flags & KVM_RUN_LOAD_GUEST_DR6)) 4529 svm_set_dr6(vcpu, vcpu->arch.dr6); 4530 else if (likely(!(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT))) 4531 svm_set_dr6(vcpu, DR6_ACTIVE_LOW); 4532 4533 clgi(); 4534 4535 /* 4536 * Hardware only context switches DEBUGCTL if LBR virtualization is 4537 * enabled. Manually load DEBUGCTL if necessary (and restore it after 4538 * VM-Exit), as running with the host's DEBUGCTL can negatively affect 4539 * guest state and can even be fatal, e.g. due to Bus Lock Detect. 4540 */ 4541 if (!(svm->vmcb->control.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR) && 4542 vcpu->arch.host_debugctl != svm->vmcb->save.dbgctl) 4543 update_debugctlmsr(svm->vmcb->save.dbgctl); 4544 4545 kvm_wait_lapic_expire(vcpu); 4546 4547 /* 4548 * If this vCPU has touched SPEC_CTRL, restore the guest's value if 4549 * it's non-zero. Since vmentry is serialising on affected CPUs, there 4550 * is no need to worry about the conditional branch over the wrmsr 4551 * being speculatively taken. 4552 */ 4553 if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 4554 x86_spec_ctrl_set_guest(svm->virt_spec_ctrl); 4555 4556 svm_vcpu_enter_exit(vcpu, enter_flags); 4557 4558 if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL)) 4559 x86_spec_ctrl_restore_host(svm->virt_spec_ctrl); 4560 4561 /* SEV-ES guests must use the CR write traps to track CR registers. */ 4562 if (!is_sev_es_guest(vcpu)) { 4563 vcpu->arch.cr2 = svm->vmcb->save.cr2; 4564 vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax; 4565 vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp; 4566 vcpu->arch.rip = svm->vmcb->save.rip; 4567 4568 if (!svm_is_intercept(svm, INTERCEPT_CR0_WRITE)) 4569 vcpu->arch.cr0 = svm->vmcb->save.cr0; 4570 if (npt_enabled) 4571 vcpu->arch.cr3 = svm->vmcb->save.cr3; 4572 } 4573 kvm_reset_dirty_registers(vcpu); 4574 4575 if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI)) 4576 kvm_before_interrupt(vcpu, KVM_HANDLING_NMI); 4577 4578 if (!(svm->vmcb->control.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR) && 4579 vcpu->arch.host_debugctl != svm->vmcb->save.dbgctl) 4580 update_debugctlmsr(vcpu->arch.host_debugctl); 4581 4582 stgi(); 4583 4584 /* Any pending NMI will happen here */ 4585 4586 if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI)) 4587 kvm_after_interrupt(vcpu); 4588 4589 sync_cr8_to_lapic(vcpu); 4590 4591 svm->next_rip = 0; 4592 if (is_guest_mode(vcpu)) { 4593 nested_sync_control_from_vmcb02(svm); 4594 4595 /* Track VMRUNs that have made past consistency checking */ 4596 if (vcpu->arch.nested_run_pending && 4597 !svm_is_vmrun_failure(svm->vmcb->control.exit_code)) 4598 ++vcpu->stat.nested_run; 4599 4600 vcpu->arch.nested_run_pending = 0; 4601 } 4602 4603 svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; 4604 4605 /* 4606 * Unconditionally mask off the CLEAR_RAP bit, the AND is just as cheap 4607 * as the TEST+Jcc to avoid it. 4608 */ 4609 if (cpu_feature_enabled(X86_FEATURE_ERAPS)) 4610 svm->vmcb->control.erap_ctl &= ~ERAP_CONTROL_CLEAR_RAP; 4611 4612 vmcb_mark_all_clean(svm->vmcb); 4613 4614 /* if exit due to PF check for async PF */ 4615 if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) 4616 vcpu->arch.apf.host_apf_flags = 4617 kvm_read_and_reset_apf_flags(); 4618 4619 kvm_clear_available_registers(vcpu, SVM_REGS_LAZY_LOAD_SET); 4620 4621 if (!msr_write_intercepted(svm, MSR_AMD64_PERF_CNTR_GLOBAL_CTL)) 4622 rdmsrq(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, vcpu_to_pmu(vcpu)->global_ctrl); 4623 4624 trace_kvm_exit(vcpu, KVM_ISA_SVM); 4625 4626 svm_complete_interrupts(vcpu); 4627 4628 /* 4629 * Update the cache after completing interrupts to get an accurate 4630 * NextRIP, e.g. when re-injecting a soft interrupt. 4631 * 4632 * FIXME: Rework svm_get_nested_state() to not pull data from the 4633 * cache (except for maybe int_ctl). 4634 */ 4635 if (is_guest_mode(vcpu)) 4636 svm->nested.ctl.next_rip = svm->vmcb->control.next_rip; 4637 4638 return svm_exit_handlers_fastpath(vcpu); 4639 } 4640 4641 static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, 4642 int root_level) 4643 { 4644 struct vcpu_svm *svm = to_svm(vcpu); 4645 unsigned long cr3; 4646 4647 if (npt_enabled) { 4648 svm->vmcb->control.nested_cr3 = __sme_set(root_hpa); 4649 vmcb_mark_dirty(svm->vmcb, VMCB_NPT); 4650 4651 hv_track_root_tdp(vcpu, root_hpa); 4652 4653 cr3 = vcpu->arch.cr3; 4654 } else if (root_level >= PT64_ROOT_4LEVEL) { 4655 cr3 = __sme_set(root_hpa) | kvm_get_active_pcid(vcpu); 4656 } else { 4657 /* PCID in the guest should be impossible with a 32-bit MMU. */ 4658 WARN_ON_ONCE(kvm_get_active_pcid(vcpu)); 4659 cr3 = root_hpa; 4660 } 4661 4662 svm->vmcb->save.cr3 = cr3; 4663 vmcb_mark_dirty(svm->vmcb, VMCB_CR); 4664 } 4665 4666 static void 4667 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) 4668 { 4669 /* 4670 * Patch in the VMMCALL instruction: 4671 */ 4672 hypercall[0] = 0x0f; 4673 hypercall[1] = 0x01; 4674 hypercall[2] = 0xd9; 4675 } 4676 4677 static bool svm_tdp_has_smep(struct kvm *kvm) 4678 { 4679 return gmet_enabled; 4680 } 4681 4682 /* 4683 * The kvm parameter can be NULL (module initialization, or invocation before 4684 * VM creation). Be sure to check the kvm parameter before using it. 4685 */ 4686 static bool svm_has_emulated_msr(struct kvm *kvm, u32 index) 4687 { 4688 switch (index) { 4689 case MSR_IA32_MCG_EXT_CTL: 4690 case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR: 4691 return false; 4692 case MSR_IA32_SMBASE: 4693 if (!IS_ENABLED(CONFIG_KVM_SMM)) 4694 return false; 4695 4696 #ifdef CONFIG_KVM_AMD_SEV 4697 /* 4698 * KVM can't access register state to emulate SMM for SEV-ES 4699 * guests. Conusming stale data here is "fine", as KVM only 4700 * checks for MSR_IA32_SMBASE support without a vCPU when 4701 * userspace is querying KVM_CAP_X86_SMM. 4702 */ 4703 if (kvm && ____sev_es_guest(kvm)) 4704 return false; 4705 #endif 4706 break; 4707 default: 4708 break; 4709 } 4710 4711 return true; 4712 } 4713 4714 static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) 4715 { 4716 struct vcpu_svm *svm = to_svm(vcpu); 4717 4718 /* 4719 * SVM doesn't provide a way to disable just XSAVES in the guest, KVM 4720 * can only disable all variants of by disallowing CR4.OSXSAVE from 4721 * being set. As a result, if the host has XSAVE and XSAVES, and the 4722 * guest has XSAVE enabled, the guest can execute XSAVES without 4723 * faulting. Treat XSAVES as enabled in this case regardless of 4724 * whether it's advertised to the guest so that KVM context switches 4725 * XSS on VM-Enter/VM-Exit. Failure to do so would effectively give 4726 * the guest read/write access to the host's XSS. 4727 */ 4728 guest_cpu_cap_change(vcpu, X86_FEATURE_XSAVES, 4729 boot_cpu_has(X86_FEATURE_XSAVES) && 4730 guest_cpu_cap_has(vcpu, X86_FEATURE_XSAVE)); 4731 4732 /* 4733 * Intercept VMLOAD if the vCPU model is Intel in order to emulate that 4734 * VMLOAD drops bits 63:32 of SYSENTER (ignoring the fact that exposing 4735 * SVM on Intel is bonkers and extremely unlikely to work). 4736 */ 4737 if (guest_cpuid_is_intel_compatible(vcpu)) 4738 guest_cpu_cap_clear(vcpu, X86_FEATURE_V_VMSAVE_VMLOAD); 4739 4740 if (is_sev_guest(vcpu)) 4741 sev_vcpu_after_set_cpuid(svm); 4742 } 4743 4744 static bool svm_has_wbinvd_exit(void) 4745 { 4746 return true; 4747 } 4748 4749 #define PRE_EX(exit) { .exit_code = (exit), \ 4750 .stage = X86_ICPT_PRE_EXCEPT, } 4751 #define POST_EX(exit) { .exit_code = (exit), \ 4752 .stage = X86_ICPT_POST_EXCEPT, } 4753 #define POST_MEM(exit) { .exit_code = (exit), \ 4754 .stage = X86_ICPT_POST_MEMACCESS, } 4755 4756 static const struct __x86_intercept { 4757 u32 exit_code; 4758 enum x86_intercept_stage stage; 4759 } x86_intercept_map[] = { 4760 [x86_intercept_cr_read] = POST_EX(SVM_EXIT_READ_CR0), 4761 [x86_intercept_cr_write] = POST_EX(SVM_EXIT_WRITE_CR0), 4762 [x86_intercept_clts] = POST_EX(SVM_EXIT_WRITE_CR0), 4763 [x86_intercept_lmsw] = POST_EX(SVM_EXIT_WRITE_CR0), 4764 [x86_intercept_smsw] = POST_EX(SVM_EXIT_READ_CR0), 4765 [x86_intercept_dr_read] = POST_EX(SVM_EXIT_READ_DR0), 4766 [x86_intercept_dr_write] = POST_EX(SVM_EXIT_WRITE_DR0), 4767 [x86_intercept_sldt] = POST_EX(SVM_EXIT_LDTR_READ), 4768 [x86_intercept_str] = POST_EX(SVM_EXIT_TR_READ), 4769 [x86_intercept_lldt] = POST_EX(SVM_EXIT_LDTR_WRITE), 4770 [x86_intercept_ltr] = POST_EX(SVM_EXIT_TR_WRITE), 4771 [x86_intercept_sgdt] = POST_EX(SVM_EXIT_GDTR_READ), 4772 [x86_intercept_sidt] = POST_EX(SVM_EXIT_IDTR_READ), 4773 [x86_intercept_lgdt] = POST_EX(SVM_EXIT_GDTR_WRITE), 4774 [x86_intercept_lidt] = POST_EX(SVM_EXIT_IDTR_WRITE), 4775 [x86_intercept_vmrun] = POST_EX(SVM_EXIT_VMRUN), 4776 [x86_intercept_vmmcall] = POST_EX(SVM_EXIT_VMMCALL), 4777 [x86_intercept_vmload] = POST_EX(SVM_EXIT_VMLOAD), 4778 [x86_intercept_vmsave] = POST_EX(SVM_EXIT_VMSAVE), 4779 [x86_intercept_stgi] = POST_EX(SVM_EXIT_STGI), 4780 [x86_intercept_clgi] = POST_EX(SVM_EXIT_CLGI), 4781 [x86_intercept_skinit] = POST_EX(SVM_EXIT_SKINIT), 4782 [x86_intercept_invlpga] = POST_EX(SVM_EXIT_INVLPGA), 4783 [x86_intercept_rdtscp] = POST_EX(SVM_EXIT_RDTSCP), 4784 [x86_intercept_monitor] = POST_MEM(SVM_EXIT_MONITOR), 4785 [x86_intercept_mwait] = POST_EX(SVM_EXIT_MWAIT), 4786 [x86_intercept_invlpg] = POST_EX(SVM_EXIT_INVLPG), 4787 [x86_intercept_invd] = POST_EX(SVM_EXIT_INVD), 4788 [x86_intercept_wbinvd] = POST_EX(SVM_EXIT_WBINVD), 4789 [x86_intercept_wrmsr] = POST_EX(SVM_EXIT_MSR), 4790 [x86_intercept_rdtsc] = POST_EX(SVM_EXIT_RDTSC), 4791 [x86_intercept_rdmsr] = POST_EX(SVM_EXIT_MSR), 4792 [x86_intercept_rdpmc] = POST_EX(SVM_EXIT_RDPMC), 4793 [x86_intercept_cpuid] = PRE_EX(SVM_EXIT_CPUID), 4794 [x86_intercept_rsm] = PRE_EX(SVM_EXIT_RSM), 4795 [x86_intercept_pause] = PRE_EX(SVM_EXIT_PAUSE), 4796 [x86_intercept_pushf] = PRE_EX(SVM_EXIT_PUSHF), 4797 [x86_intercept_popf] = PRE_EX(SVM_EXIT_POPF), 4798 [x86_intercept_intn] = PRE_EX(SVM_EXIT_SWINT), 4799 [x86_intercept_iret] = PRE_EX(SVM_EXIT_IRET), 4800 [x86_intercept_icebp] = PRE_EX(SVM_EXIT_ICEBP), 4801 [x86_intercept_hlt] = POST_EX(SVM_EXIT_HLT), 4802 [x86_intercept_in] = POST_EX(SVM_EXIT_IOIO), 4803 [x86_intercept_ins] = POST_EX(SVM_EXIT_IOIO), 4804 [x86_intercept_out] = POST_EX(SVM_EXIT_IOIO), 4805 [x86_intercept_outs] = POST_EX(SVM_EXIT_IOIO), 4806 [x86_intercept_xsetbv] = PRE_EX(SVM_EXIT_XSETBV), 4807 }; 4808 4809 #undef PRE_EX 4810 #undef POST_EX 4811 #undef POST_MEM 4812 4813 static int svm_check_intercept(struct kvm_vcpu *vcpu, 4814 struct x86_instruction_info *info, 4815 enum x86_intercept_stage stage, 4816 struct x86_exception *exception) 4817 { 4818 struct vcpu_svm *svm = to_svm(vcpu); 4819 int vmexit, ret = X86EMUL_CONTINUE; 4820 struct __x86_intercept icpt_info; 4821 struct vmcb *vmcb = svm->vmcb; 4822 4823 if (info->intercept >= ARRAY_SIZE(x86_intercept_map)) 4824 goto out; 4825 4826 icpt_info = x86_intercept_map[info->intercept]; 4827 4828 if (stage != icpt_info.stage) 4829 goto out; 4830 4831 switch (icpt_info.exit_code) { 4832 case SVM_EXIT_READ_CR0: 4833 if (info->intercept == x86_intercept_cr_read) 4834 icpt_info.exit_code += info->modrm_reg; 4835 break; 4836 case SVM_EXIT_WRITE_CR0: { 4837 unsigned long cr0, val; 4838 4839 /* 4840 * Adjust the exit code accordingly if a CR other than CR0 is 4841 * being written, and skip straight to the common handling as 4842 * only CR0 has an additional selective intercept. 4843 */ 4844 if (info->intercept == x86_intercept_cr_write && info->modrm_reg) { 4845 icpt_info.exit_code += info->modrm_reg; 4846 break; 4847 } 4848 4849 /* 4850 * Convert the exit_code to SVM_EXIT_CR0_SEL_WRITE if a 4851 * selective CR0 intercept is triggered (the common logic will 4852 * treat the selective intercept as being enabled). Note, the 4853 * unconditional intercept has higher priority, i.e. this is 4854 * only relevant if *only* the selective intercept is enabled. 4855 */ 4856 if (vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_CR0_WRITE) || 4857 !(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0))) 4858 break; 4859 4860 /* CLTS never triggers INTERCEPT_SELECTIVE_CR0 */ 4861 if (info->intercept == x86_intercept_clts) 4862 break; 4863 4864 /* LMSW always triggers INTERCEPT_SELECTIVE_CR0 */ 4865 if (info->intercept == x86_intercept_lmsw) { 4866 icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE; 4867 break; 4868 } 4869 4870 /* 4871 * MOV-to-CR0 only triggers INTERCEPT_SELECTIVE_CR0 if any bit 4872 * other than SVM_CR0_SELECTIVE_MASK is changed. 4873 */ 4874 cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK; 4875 val = info->src_val & ~SVM_CR0_SELECTIVE_MASK; 4876 if (cr0 ^ val) 4877 icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE; 4878 break; 4879 } 4880 case SVM_EXIT_READ_DR0: 4881 case SVM_EXIT_WRITE_DR0: 4882 icpt_info.exit_code += info->modrm_reg; 4883 break; 4884 case SVM_EXIT_MSR: 4885 if (info->intercept == x86_intercept_wrmsr) 4886 vmcb->control.exit_info_1 = 1; 4887 else 4888 vmcb->control.exit_info_1 = 0; 4889 break; 4890 case SVM_EXIT_PAUSE: 4891 /* 4892 * We get this for NOP only, but pause 4893 * is rep not, check this here 4894 */ 4895 if (info->rep_prefix != REPE_PREFIX) 4896 goto out; 4897 break; 4898 case SVM_EXIT_IOIO: { 4899 u64 exit_info; 4900 u32 bytes; 4901 4902 if (info->intercept == x86_intercept_in || 4903 info->intercept == x86_intercept_ins) { 4904 exit_info = ((info->src_val & 0xffff) << 16) | 4905 SVM_IOIO_TYPE_MASK; 4906 bytes = info->dst_bytes; 4907 } else { 4908 exit_info = (info->dst_val & 0xffff) << 16; 4909 bytes = info->src_bytes; 4910 } 4911 4912 if (info->intercept == x86_intercept_outs || 4913 info->intercept == x86_intercept_ins) 4914 exit_info |= SVM_IOIO_STR_MASK; 4915 4916 if (info->rep_prefix) 4917 exit_info |= SVM_IOIO_REP_MASK; 4918 4919 bytes = min(bytes, 4u); 4920 4921 exit_info |= bytes << SVM_IOIO_SIZE_SHIFT; 4922 4923 exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1); 4924 4925 vmcb->control.exit_info_1 = exit_info; 4926 vmcb->control.exit_info_2 = info->next_rip; 4927 4928 break; 4929 } 4930 default: 4931 break; 4932 } 4933 4934 /* TODO: Advertise NRIPS to guest hypervisor unconditionally */ 4935 if (static_cpu_has(X86_FEATURE_NRIPS)) 4936 vmcb->control.next_rip = info->next_rip; 4937 vmcb->control.exit_code = icpt_info.exit_code; 4938 vmexit = nested_svm_exit_handled(svm); 4939 4940 ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED 4941 : X86EMUL_CONTINUE; 4942 4943 out: 4944 return ret; 4945 } 4946 4947 static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu) 4948 { 4949 switch (to_svm(vcpu)->vmcb->control.exit_code) { 4950 case SVM_EXIT_EXCP_BASE + MC_VECTOR: 4951 svm_handle_mce(vcpu); 4952 break; 4953 case SVM_EXIT_INTR: 4954 vcpu->arch.at_instruction_boundary = true; 4955 break; 4956 default: 4957 break; 4958 } 4959 } 4960 4961 static void svm_setup_mce(struct kvm_vcpu *vcpu) 4962 { 4963 /* [63:9] are reserved. */ 4964 vcpu->arch.mcg_cap &= 0x1ff; 4965 } 4966 4967 #ifdef CONFIG_KVM_SMM 4968 bool svm_smi_blocked(struct kvm_vcpu *vcpu) 4969 { 4970 struct vcpu_svm *svm = to_svm(vcpu); 4971 4972 /* Per APM Vol.2 15.22.2 "Response to SMI" */ 4973 if (!gif_set(svm)) 4974 return true; 4975 4976 return is_smm(vcpu); 4977 } 4978 4979 static int svm_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 4980 { 4981 struct vcpu_svm *svm = to_svm(vcpu); 4982 if (vcpu->arch.nested_run_pending) 4983 return -EBUSY; 4984 4985 if (svm_smi_blocked(vcpu)) 4986 return 0; 4987 4988 /* An SMI must not be injected into L2 if it's supposed to VM-Exit. */ 4989 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_smi(svm)) 4990 return -EBUSY; 4991 4992 return 1; 4993 } 4994 4995 static int svm_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram) 4996 { 4997 struct vcpu_svm *svm = to_svm(vcpu); 4998 struct kvm_host_map map_save; 4999 5000 if (!is_guest_mode(vcpu)) 5001 return 0; 5002 5003 /* 5004 * 32-bit SMRAM format doesn't preserve EFER and SVM state. Userspace is 5005 * responsible for ensuring nested SVM and SMIs are mutually exclusive. 5006 */ 5007 5008 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_LM)) 5009 return 1; 5010 5011 smram->smram64.svm_guest_flag = 1; 5012 smram->smram64.svm_guest_vmcb_gpa = svm->nested.vmcb12_gpa; 5013 5014 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; 5015 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; 5016 svm->vmcb->save.rip = vcpu->arch.rip; 5017 5018 nested_svm_simple_vmexit(svm, SVM_EXIT_SW); 5019 5020 /* 5021 * KVM uses VMCB01 to store L1 host state while L2 runs but 5022 * VMCB01 is going to be used during SMM and thus the state will 5023 * be lost. Temporary save non-VMLOAD/VMSAVE state to the host save 5024 * area pointed to by MSR_VM_HSAVE_PA. APM guarantees that the 5025 * format of the area is identical to guest save area offsetted 5026 * by 0x400 (matches the offset of 'struct vmcb_save_area' 5027 * within 'struct vmcb'). Note: HSAVE area may also be used by 5028 * L1 hypervisor to save additional host context (e.g. KVM does 5029 * that, see svm_prepare_switch_to_guest()) which must be 5030 * preserved. 5031 */ 5032 if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save)) 5033 return 1; 5034 5035 BUILD_BUG_ON(offsetof(struct vmcb, save) != 0x400); 5036 5037 svm_copy_vmrun_state(map_save.hva + 0x400, 5038 &svm->vmcb01.ptr->save); 5039 5040 kvm_vcpu_unmap(vcpu, &map_save); 5041 return 0; 5042 } 5043 5044 static int svm_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram) 5045 { 5046 struct vcpu_svm *svm = to_svm(vcpu); 5047 struct kvm_host_map map, map_save; 5048 struct vmcb *vmcb12; 5049 int ret; 5050 5051 const struct kvm_smram_state_64 *smram64 = &smram->smram64; 5052 5053 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_LM)) 5054 return 0; 5055 5056 /* Non-zero if SMI arrived while vCPU was in guest mode. */ 5057 if (!smram64->svm_guest_flag) 5058 return 0; 5059 5060 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SVM)) 5061 return 1; 5062 5063 if (!(smram64->efer & EFER_SVME)) 5064 return 1; 5065 5066 if (kvm_vcpu_map(vcpu, gpa_to_gfn(smram64->svm_guest_vmcb_gpa), &map)) 5067 return 1; 5068 5069 ret = 1; 5070 if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save)) 5071 goto unmap_map; 5072 5073 if (svm_allocate_nested(svm)) 5074 goto unmap_save; 5075 5076 /* 5077 * Restore L1 host state from L1 HSAVE area as VMCB01 was 5078 * used during SMM (see svm_enter_smm()) 5079 */ 5080 5081 svm_copy_vmrun_state(&svm->vmcb01.ptr->save, map_save.hva + 0x400); 5082 5083 /* 5084 * Enter the nested guest now 5085 */ 5086 5087 vmcb_mark_all_dirty(svm->vmcb01.ptr); 5088 5089 vmcb12 = map.hva; 5090 nested_copy_vmcb_control_to_cache(svm, &vmcb12->control); 5091 nested_copy_vmcb_save_to_cache(svm, &vmcb12->save); 5092 5093 if (nested_svm_check_cached_vmcb12(vcpu) < 0) 5094 goto unmap_save; 5095 5096 if (enter_svm_guest_mode(vcpu, smram64->svm_guest_vmcb_gpa, false) != 0) 5097 goto unmap_save; 5098 5099 ret = 0; 5100 vcpu->arch.nested_run_pending = KVM_NESTED_RUN_PENDING; 5101 5102 unmap_save: 5103 kvm_vcpu_unmap(vcpu, &map_save); 5104 unmap_map: 5105 kvm_vcpu_unmap(vcpu, &map); 5106 return ret; 5107 } 5108 5109 static void svm_enable_smi_window(struct kvm_vcpu *vcpu) 5110 { 5111 struct vcpu_svm *svm = to_svm(vcpu); 5112 5113 if (!gif_set(svm)) { 5114 if (vgif) 5115 svm_set_intercept(svm, INTERCEPT_STGI); 5116 /* STGI will cause a vm exit */ 5117 } else { 5118 /* We must be in SMM; RSM will cause a vmexit anyway. */ 5119 } 5120 } 5121 #endif 5122 5123 static int svm_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, 5124 void *insn, int insn_len) 5125 { 5126 struct vcpu_svm *svm = to_svm(vcpu); 5127 bool smep, smap, is_user; 5128 u64 error_code; 5129 5130 /* Check that emulation is possible during event vectoring */ 5131 if ((svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK) && 5132 !kvm_can_emulate_event_vectoring(emul_type)) 5133 return X86EMUL_UNHANDLEABLE_VECTORING; 5134 5135 /* Emulation is always possible when KVM has access to all guest state. */ 5136 if (!is_sev_guest(vcpu)) 5137 return X86EMUL_CONTINUE; 5138 5139 /* #UD and #GP should never be intercepted for SEV guests. */ 5140 WARN_ON_ONCE(emul_type & (EMULTYPE_TRAP_UD | 5141 EMULTYPE_TRAP_UD_FORCED | 5142 EMULTYPE_VMWARE_GP)); 5143 5144 /* 5145 * Emulation is impossible for SEV-ES guests as KVM doesn't have access 5146 * to guest register state. 5147 */ 5148 if (is_sev_es_guest(vcpu)) 5149 return X86EMUL_RETRY_INSTR; 5150 5151 /* 5152 * Emulation is possible if the instruction is already decoded, e.g. 5153 * when completing I/O after returning from userspace. 5154 */ 5155 if (emul_type & EMULTYPE_NO_DECODE) 5156 return X86EMUL_CONTINUE; 5157 5158 /* 5159 * Emulation is possible for SEV guests if and only if a prefilled 5160 * buffer containing the bytes of the intercepted instruction is 5161 * available. SEV guest memory is encrypted with a guest specific key 5162 * and cannot be decrypted by KVM, i.e. KVM would read ciphertext and 5163 * decode garbage. 5164 * 5165 * If KVM is NOT trying to simply skip an instruction, inject #UD if 5166 * KVM reached this point without an instruction buffer. In practice, 5167 * this path should never be hit by a well-behaved guest, e.g. KVM 5168 * doesn't intercept #UD or #GP for SEV guests, but this path is still 5169 * theoretically reachable, e.g. via unaccelerated fault-like AVIC 5170 * access, and needs to be handled by KVM to avoid putting the guest 5171 * into an infinite loop. Injecting #UD is somewhat arbitrary, but 5172 * its the least awful option given lack of insight into the guest. 5173 * 5174 * If KVM is trying to skip an instruction, simply resume the guest. 5175 * If a #NPF occurs while the guest is vectoring an INT3/INTO, then KVM 5176 * will attempt to re-inject the INT3/INTO and skip the instruction. 5177 * In that scenario, retrying the INT3/INTO and hoping the guest will 5178 * make forward progress is the only option that has a chance of 5179 * success (and in practice it will work the vast majority of the time). 5180 */ 5181 if (unlikely(!insn)) { 5182 if (emul_type & EMULTYPE_SKIP) 5183 return X86EMUL_UNHANDLEABLE; 5184 5185 kvm_queue_exception(vcpu, UD_VECTOR); 5186 return X86EMUL_PROPAGATE_FAULT; 5187 } 5188 5189 /* 5190 * Emulate for SEV guests if the insn buffer is not empty. The buffer 5191 * will be empty if the DecodeAssist microcode cannot fetch bytes for 5192 * the faulting instruction because the code fetch itself faulted, e.g. 5193 * the guest attempted to fetch from emulated MMIO or a guest page 5194 * table used to translate CS:RIP resides in emulated MMIO. 5195 */ 5196 if (likely(insn_len)) 5197 return X86EMUL_CONTINUE; 5198 5199 /* 5200 * Detect and workaround Errata 1096 Fam_17h_00_0Fh. 5201 * 5202 * Errata: 5203 * When CPU raises #NPF on guest data access and vCPU CR4.SMAP=1, it is 5204 * possible that CPU microcode implementing DecodeAssist will fail to 5205 * read guest memory at CS:RIP and vmcb.GuestIntrBytes will incorrectly 5206 * be '0'. This happens because microcode reads CS:RIP using a _data_ 5207 * loap uop with CPL=0 privileges. If the load hits a SMAP #PF, ucode 5208 * gives up and does not fill the instruction bytes buffer. 5209 * 5210 * As above, KVM reaches this point iff the VM is an SEV guest, the CPU 5211 * supports DecodeAssist, a #NPF was raised, KVM's page fault handler 5212 * triggered emulation (e.g. for MMIO), and the CPU returned 0 in the 5213 * GuestIntrBytes field of the VMCB. 5214 * 5215 * This does _not_ mean that the erratum has been encountered, as the 5216 * DecodeAssist will also fail if the load for CS:RIP hits a legitimate 5217 * #PF, e.g. if the guest attempt to execute from emulated MMIO and 5218 * encountered a reserved/not-present #PF. 5219 * 5220 * To hit the erratum, the following conditions must be true: 5221 * 1. CR4.SMAP=1 (obviously). 5222 * 2. CR4.SMEP=0 || CPL=3. If SMEP=1 and CPL<3, the erratum cannot 5223 * have been hit as the guest would have encountered a SMEP 5224 * violation #PF, not a #NPF. 5225 * 3. The #NPF is not due to a code fetch, in which case failure to 5226 * retrieve the instruction bytes is legitimate (see abvoe). 5227 * 5228 * In addition, don't apply the erratum workaround if the #NPF occurred 5229 * while translating guest page tables (see below). 5230 */ 5231 error_code = svm->vmcb->control.exit_info_1; 5232 if (error_code & (PFERR_GUEST_PAGE_MASK | PFERR_FETCH_MASK)) 5233 goto resume_guest; 5234 5235 smep = kvm_is_cr4_bit_set(vcpu, X86_CR4_SMEP); 5236 smap = kvm_is_cr4_bit_set(vcpu, X86_CR4_SMAP); 5237 is_user = svm_get_cpl(vcpu) == 3; 5238 if (smap && (!smep || is_user)) { 5239 pr_err_ratelimited("SEV Guest triggered AMD Erratum 1096\n"); 5240 5241 /* 5242 * If the fault occurred in userspace, arbitrarily inject #GP 5243 * to avoid killing the guest and to hopefully avoid confusing 5244 * the guest kernel too much, e.g. injecting #PF would not be 5245 * coherent with respect to the guest's page tables. Request 5246 * triple fault if the fault occurred in the kernel as there's 5247 * no fault that KVM can inject without confusing the guest. 5248 * In practice, the triple fault is moot as no sane SEV kernel 5249 * will execute from user memory while also running with SMAP=1. 5250 */ 5251 if (is_user) 5252 kvm_inject_gp(vcpu, 0); 5253 else 5254 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 5255 return X86EMUL_PROPAGATE_FAULT; 5256 } 5257 5258 resume_guest: 5259 /* 5260 * If the erratum was not hit, simply resume the guest and let it fault 5261 * again. While awful, e.g. the vCPU may get stuck in an infinite loop 5262 * if the fault is at CPL=0, it's the lesser of all evils. Exiting to 5263 * userspace will kill the guest, and letting the emulator read garbage 5264 * will yield random behavior and potentially corrupt the guest. 5265 * 5266 * Simply resuming the guest is technically not a violation of the SEV 5267 * architecture. AMD's APM states that all code fetches and page table 5268 * accesses for SEV guest are encrypted, regardless of the C-Bit. The 5269 * APM also states that encrypted accesses to MMIO are "ignored", but 5270 * doesn't explicitly define "ignored", i.e. doing nothing and letting 5271 * the guest spin is technically "ignoring" the access. 5272 */ 5273 return X86EMUL_RETRY_INSTR; 5274 } 5275 5276 static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu) 5277 { 5278 struct vcpu_svm *svm = to_svm(vcpu); 5279 5280 return !gif_set(svm); 5281 } 5282 5283 static void svm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector) 5284 { 5285 if (!is_sev_es_guest(vcpu)) 5286 return kvm_vcpu_deliver_sipi_vector(vcpu, vector); 5287 5288 sev_vcpu_deliver_sipi_vector(vcpu, vector); 5289 } 5290 5291 static void svm_vm_destroy(struct kvm *kvm) 5292 { 5293 avic_vm_destroy(kvm); 5294 sev_vm_destroy(kvm); 5295 5296 svm_srso_vm_destroy(); 5297 } 5298 5299 static int svm_vm_init(struct kvm *kvm) 5300 { 5301 sev_vm_init(kvm); 5302 5303 if (!pause_filter_count || !pause_filter_thresh) 5304 kvm_disable_exits(kvm, KVM_X86_DISABLE_EXITS_PAUSE); 5305 5306 if (enable_apicv) { 5307 int ret = avic_vm_init(kvm); 5308 if (ret) 5309 return ret; 5310 } 5311 5312 svm_srso_vm_init(); 5313 return 0; 5314 } 5315 5316 static void *svm_alloc_apic_backing_page(struct kvm_vcpu *vcpu) 5317 { 5318 struct page *page = snp_safe_alloc_page(); 5319 5320 if (!page) 5321 return NULL; 5322 5323 return page_address(page); 5324 } 5325 5326 struct kvm_x86_ops svm_x86_ops __initdata = { 5327 .name = KBUILD_MODNAME, 5328 5329 .check_processor_compatibility = svm_check_processor_compat, 5330 5331 .hardware_unsetup = svm_hardware_unsetup, 5332 .enable_virtualization_cpu = svm_enable_virtualization_cpu, 5333 .disable_virtualization_cpu = svm_disable_virtualization_cpu, 5334 .emergency_disable_virtualization_cpu = svm_emergency_disable_virtualization_cpu, 5335 .has_emulated_msr = svm_has_emulated_msr, 5336 5337 .vcpu_precreate = svm_vcpu_precreate, 5338 .vcpu_create = svm_vcpu_create, 5339 .vcpu_free = svm_vcpu_free, 5340 .vcpu_reset = svm_vcpu_reset, 5341 5342 .vm_size = sizeof(struct kvm_svm), 5343 .vm_init = svm_vm_init, 5344 .vm_destroy = svm_vm_destroy, 5345 5346 .prepare_switch_to_guest = svm_prepare_switch_to_guest, 5347 .vcpu_load = svm_vcpu_load, 5348 .vcpu_put = svm_vcpu_put, 5349 .vcpu_blocking = avic_vcpu_blocking, 5350 .vcpu_unblocking = avic_vcpu_unblocking, 5351 5352 .update_exception_bitmap = svm_update_exception_bitmap, 5353 .get_feature_msr = svm_get_feature_msr, 5354 .get_msr = svm_get_msr, 5355 .set_msr = svm_set_msr, 5356 .get_segment_base = svm_get_segment_base, 5357 .get_segment = svm_get_segment, 5358 .set_segment = svm_set_segment, 5359 .get_cpl = svm_get_cpl, 5360 .get_cpl_no_cache = svm_get_cpl, 5361 .get_cs_db_l_bits = svm_get_cs_db_l_bits, 5362 .is_valid_cr0 = svm_is_valid_cr0, 5363 .set_cr0 = svm_set_cr0, 5364 .post_set_cr3 = sev_post_set_cr3, 5365 .is_valid_cr4 = svm_is_valid_cr4, 5366 .set_cr4 = svm_set_cr4, 5367 .set_efer = svm_set_efer, 5368 .get_idt = svm_get_idt, 5369 .set_idt = svm_set_idt, 5370 .get_gdt = svm_get_gdt, 5371 .set_gdt = svm_set_gdt, 5372 .set_dr7 = svm_set_dr7, 5373 .sync_dirty_debug_regs = svm_sync_dirty_debug_regs, 5374 .cache_reg = svm_cache_reg, 5375 .get_rflags = svm_get_rflags, 5376 .set_rflags = svm_set_rflags, 5377 .get_if_flag = svm_get_if_flag, 5378 5379 .flush_tlb_all = svm_flush_tlb_all, 5380 .flush_tlb_current = svm_flush_tlb_current, 5381 .flush_tlb_gva = svm_flush_tlb_gva, 5382 .flush_tlb_guest = svm_flush_tlb_guest, 5383 5384 .vcpu_pre_run = svm_vcpu_pre_run, 5385 .vcpu_run = svm_vcpu_run, 5386 .handle_exit = svm_handle_exit, 5387 .skip_emulated_instruction = svm_skip_emulated_instruction, 5388 .update_emulated_instruction = NULL, 5389 .set_interrupt_shadow = svm_set_interrupt_shadow, 5390 .get_interrupt_shadow = svm_get_interrupt_shadow, 5391 .patch_hypercall = svm_patch_hypercall, 5392 .inject_irq = svm_inject_irq, 5393 .inject_nmi = svm_inject_nmi, 5394 .is_vnmi_pending = svm_is_vnmi_pending, 5395 .set_vnmi_pending = svm_set_vnmi_pending, 5396 .inject_exception = svm_inject_exception, 5397 .cancel_injection = svm_cancel_injection, 5398 .interrupt_allowed = svm_interrupt_allowed, 5399 .nmi_allowed = svm_nmi_allowed, 5400 .get_nmi_mask = svm_get_nmi_mask, 5401 .set_nmi_mask = svm_set_nmi_mask, 5402 .enable_nmi_window = svm_enable_nmi_window, 5403 .enable_irq_window = svm_enable_irq_window, 5404 .update_cr8_intercept = svm_update_cr8_intercept, 5405 5406 .x2apic_icr_is_split = true, 5407 .set_virtual_apic_mode = avic_refresh_virtual_apic_mode, 5408 .refresh_apicv_exec_ctrl = avic_refresh_apicv_exec_ctrl, 5409 .apicv_post_state_restore = avic_apicv_post_state_restore, 5410 .required_apicv_inhibits = AVIC_REQUIRED_APICV_INHIBITS, 5411 5412 .get_exit_info = svm_get_exit_info, 5413 .get_entry_info = svm_get_entry_info, 5414 5415 .vcpu_after_set_cpuid = svm_vcpu_after_set_cpuid, 5416 5417 .has_wbinvd_exit = svm_has_wbinvd_exit, 5418 5419 .get_l2_tsc_offset = svm_get_l2_tsc_offset, 5420 .get_l2_tsc_multiplier = svm_get_l2_tsc_multiplier, 5421 .write_tsc_offset = svm_write_tsc_offset, 5422 .write_tsc_multiplier = svm_write_tsc_multiplier, 5423 5424 .load_mmu_pgd = svm_load_mmu_pgd, 5425 .tdp_has_smep = svm_tdp_has_smep, 5426 5427 .check_intercept = svm_check_intercept, 5428 .handle_exit_irqoff = svm_handle_exit_irqoff, 5429 5430 .nested_ops = &svm_nested_ops, 5431 5432 .deliver_interrupt = svm_deliver_interrupt, 5433 .pi_update_irte = avic_pi_update_irte, 5434 .setup_mce = svm_setup_mce, 5435 5436 #ifdef CONFIG_KVM_SMM 5437 .smi_allowed = svm_smi_allowed, 5438 .enter_smm = svm_enter_smm, 5439 .leave_smm = svm_leave_smm, 5440 .enable_smi_window = svm_enable_smi_window, 5441 #endif 5442 5443 #ifdef CONFIG_KVM_AMD_SEV 5444 .dev_get_attr = sev_dev_get_attr, 5445 .mem_enc_ioctl = sev_mem_enc_ioctl, 5446 .mem_enc_register_region = sev_mem_enc_register_region, 5447 .mem_enc_unregister_region = sev_mem_enc_unregister_region, 5448 .guest_memory_reclaimed = sev_guest_memory_reclaimed, 5449 5450 .vm_copy_enc_context_from = sev_vm_copy_enc_context_from, 5451 .vm_move_enc_context_from = sev_vm_move_enc_context_from, 5452 #endif 5453 .check_emulate_instruction = svm_check_emulate_instruction, 5454 5455 .apic_init_signal_blocked = svm_apic_init_signal_blocked, 5456 5457 .recalc_intercepts = svm_recalc_intercepts, 5458 .complete_emulated_msr = svm_complete_emulated_msr, 5459 5460 .vcpu_deliver_sipi_vector = svm_vcpu_deliver_sipi_vector, 5461 .vcpu_get_apicv_inhibit_reasons = avic_vcpu_get_apicv_inhibit_reasons, 5462 .alloc_apic_backing_page = svm_alloc_apic_backing_page, 5463 5464 .gmem_prepare = sev_gmem_prepare, 5465 .gmem_invalidate = sev_gmem_invalidate, 5466 .gmem_max_mapping_level = sev_gmem_max_mapping_level, 5467 }; 5468 5469 /* 5470 * The default MMIO mask is a single bit (excluding the present bit), 5471 * which could conflict with the memory encryption bit. Check for 5472 * memory encryption support and override the default MMIO mask if 5473 * memory encryption is enabled. 5474 */ 5475 static __init void svm_adjust_mmio_mask(void) 5476 { 5477 unsigned int enc_bit, mask_bit; 5478 u64 msr, mask; 5479 5480 /* If there is no memory encryption support, use existing mask */ 5481 if (cpuid_eax(0x80000000) < 0x8000001f) 5482 return; 5483 5484 /* If memory encryption is not enabled, use existing mask */ 5485 rdmsrq(MSR_AMD64_SYSCFG, msr); 5486 if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT)) 5487 return; 5488 5489 enc_bit = cpuid_ebx(0x8000001f) & 0x3f; 5490 mask_bit = boot_cpu_data.x86_phys_bits; 5491 5492 /* Increment the mask bit if it is the same as the encryption bit */ 5493 if (enc_bit == mask_bit) 5494 mask_bit++; 5495 5496 /* 5497 * If the mask bit location is below 52, then some bits above the 5498 * physical addressing limit will always be reserved, so use the 5499 * rsvd_bits() function to generate the mask. This mask, along with 5500 * the present bit, will be used to generate a page fault with 5501 * PFER.RSV = 1. 5502 * 5503 * If the mask bit location is 52 (or above), then clear the mask. 5504 */ 5505 mask = (mask_bit < 52) ? rsvd_bits(mask_bit, 51) | PT_PRESENT_MASK : 0; 5506 5507 kvm_mmu_set_mmio_spte_mask(mask, mask, PT_WRITABLE_MASK | PT_USER_MASK); 5508 } 5509 5510 static __init void svm_set_cpu_caps(void) 5511 { 5512 kvm_initialize_cpu_caps(); 5513 5514 kvm_caps.supported_perf_cap = 0; 5515 5516 kvm_cpu_cap_clear(X86_FEATURE_IBT); 5517 5518 /* CPUID 0x80000001 and 0x8000000A (SVM features) */ 5519 if (nested) { 5520 kvm_cpu_cap_set(X86_FEATURE_SVM); 5521 kvm_cpu_cap_set(X86_FEATURE_VMCBCLEAN); 5522 5523 /* 5524 * KVM currently flushes TLBs on *every* nested SVM transition, 5525 * and so for all intents and purposes KVM supports flushing by 5526 * ASID, i.e. KVM is guaranteed to honor every L1 ASID flush. 5527 */ 5528 kvm_cpu_cap_set(X86_FEATURE_FLUSHBYASID); 5529 5530 if (nrips) 5531 kvm_cpu_cap_set(X86_FEATURE_NRIPS); 5532 5533 if (npt_enabled) 5534 kvm_cpu_cap_set(X86_FEATURE_NPT); 5535 5536 if (tsc_scaling) 5537 kvm_cpu_cap_set(X86_FEATURE_TSCRATEMSR); 5538 5539 if (vls) 5540 kvm_cpu_cap_set(X86_FEATURE_V_VMSAVE_VMLOAD); 5541 if (lbrv) 5542 kvm_cpu_cap_set(X86_FEATURE_LBRV); 5543 5544 if (boot_cpu_has(X86_FEATURE_PAUSEFILTER)) 5545 kvm_cpu_cap_set(X86_FEATURE_PAUSEFILTER); 5546 5547 if (boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) 5548 kvm_cpu_cap_set(X86_FEATURE_PFTHRESHOLD); 5549 5550 if (gmet_enabled) 5551 kvm_cpu_cap_set(X86_FEATURE_GMET); 5552 5553 if (vgif) 5554 kvm_cpu_cap_set(X86_FEATURE_VGIF); 5555 5556 if (vnmi) 5557 kvm_cpu_cap_set(X86_FEATURE_VNMI); 5558 5559 /* Nested VM can receive #VMEXIT instead of triggering #GP */ 5560 kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK); 5561 } 5562 5563 if (cpu_feature_enabled(X86_FEATURE_BUS_LOCK_THRESHOLD)) 5564 kvm_caps.has_bus_lock_exit = true; 5565 5566 /* CPUID 0x80000008 */ 5567 if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) || 5568 boot_cpu_has(X86_FEATURE_AMD_SSBD)) 5569 kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD); 5570 5571 if (enable_pmu) { 5572 /* 5573 * Enumerate support for PERFCTR_CORE if and only if KVM has 5574 * access to enough counters to virtualize "core" support, 5575 * otherwise limit vPMU support to the legacy number of counters. 5576 */ 5577 if (kvm_pmu_cap.num_counters_gp < AMD64_NUM_COUNTERS_CORE) 5578 kvm_pmu_cap.num_counters_gp = min(AMD64_NUM_COUNTERS, 5579 kvm_pmu_cap.num_counters_gp); 5580 else 5581 kvm_cpu_cap_check_and_set(X86_FEATURE_PERFCTR_CORE); 5582 5583 if (kvm_pmu_cap.version != 2 || 5584 !kvm_cpu_cap_has(X86_FEATURE_PERFCTR_CORE)) 5585 kvm_cpu_cap_clear(X86_FEATURE_PERFMON_V2); 5586 } 5587 5588 /* CPUID 0x8000001F (SME/SEV features) */ 5589 sev_set_cpu_caps(); 5590 5591 /* 5592 * Clear capabilities that are automatically configured by common code, 5593 * but that require explicit SVM support (that isn't yet implemented). 5594 */ 5595 kvm_cpu_cap_clear(X86_FEATURE_BUS_LOCK_DETECT); 5596 kvm_cpu_cap_clear(X86_FEATURE_MSR_IMM); 5597 5598 kvm_setup_xss_caps(); 5599 kvm_finalize_cpu_caps(); 5600 } 5601 5602 static __init int svm_hardware_setup(void) 5603 { 5604 void *iopm_va; 5605 int cpu, r; 5606 5607 /* 5608 * NX is required for shadow paging and for NPT if the NX huge pages 5609 * mitigation is enabled. 5610 */ 5611 if (!boot_cpu_has(X86_FEATURE_NX)) { 5612 pr_err_ratelimited("NX (Execute Disable) not supported\n"); 5613 return -EOPNOTSUPP; 5614 } 5615 5616 kvm_caps.supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS | 5617 XFEATURE_MASK_BNDCSR); 5618 5619 if (tsc_scaling) { 5620 if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR)) { 5621 tsc_scaling = false; 5622 } else { 5623 pr_info("TSC scaling supported\n"); 5624 kvm_caps.has_tsc_control = true; 5625 } 5626 } 5627 kvm_caps.max_tsc_scaling_ratio = SVM_TSC_RATIO_MAX; 5628 kvm_caps.tsc_scaling_ratio_frac_bits = 32; 5629 5630 tsc_aux_uret_slot = kvm_add_user_return_msr(MSR_TSC_AUX); 5631 5632 /* Check for pause filtering support */ 5633 if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) { 5634 pause_filter_count = 0; 5635 pause_filter_thresh = 0; 5636 } else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) { 5637 pause_filter_thresh = 0; 5638 } 5639 5640 if (nested) { 5641 pr_info("Nested Virtualization enabled\n"); 5642 kvm_enable_efer_bits(EFER_SVME); 5643 if (!boot_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ)) 5644 kvm_enable_efer_bits(EFER_LMSLE); 5645 5646 r = nested_svm_init_msrpm_merge_offsets(); 5647 if (r) 5648 return r; 5649 } 5650 5651 /* 5652 * KVM's MMU doesn't support using 2-level paging for itself, and thus 5653 * NPT isn't supported if the host is using 2-level paging since host 5654 * CR4 is unchanged on VMRUN. 5655 */ 5656 if (!IS_ENABLED(CONFIG_X86_64) && !IS_ENABLED(CONFIG_X86_PAE)) 5657 npt_enabled = false; 5658 5659 if (!boot_cpu_has(X86_FEATURE_NPT)) 5660 npt_enabled = false; 5661 5662 if (!npt_enabled || !boot_cpu_has(X86_FEATURE_GMET)) 5663 gmet_enabled = false; 5664 5665 /* Force VM NPT level equal to the host's paging level */ 5666 kvm_configure_mmu(npt_enabled, get_npt_level(), 5667 get_npt_level(), PG_LEVEL_1G); 5668 pr_info("Nested Paging %s\n", str_enabled_disabled(npt_enabled)); 5669 5670 /* 5671 * It seems that on AMD processors PTE's accessed bit is 5672 * being set by the CPU hardware before the NPF vmexit. 5673 * This is not expected behaviour and our tests fail because 5674 * of it. 5675 * A workaround here is to disable support for 5676 * GUEST_MAXPHYADDR < HOST_MAXPHYADDR if NPT is enabled. 5677 * In this case userspace can know if there is support using 5678 * KVM_CAP_SMALLER_MAXPHYADDR extension and decide how to handle 5679 * it 5680 * If future AMD CPU models change the behaviour described above, 5681 * this variable can be changed accordingly 5682 */ 5683 allow_smaller_maxphyaddr = !npt_enabled; 5684 5685 /* Setup shadow_me_value and shadow_me_mask */ 5686 kvm_mmu_set_me_spte_mask(sme_me_mask, sme_me_mask); 5687 5688 svm_adjust_mmio_mask(); 5689 5690 nrips = nrips && boot_cpu_has(X86_FEATURE_NRIPS); 5691 5692 if (lbrv) { 5693 if (!boot_cpu_has(X86_FEATURE_LBRV)) 5694 lbrv = false; 5695 else 5696 pr_info("LBR virtualization supported\n"); 5697 } 5698 5699 iopm_va = svm_alloc_permissions_map(IOPM_SIZE, GFP_KERNEL); 5700 if (!iopm_va) 5701 return -ENOMEM; 5702 5703 iopm_base = __sme_set(__pa(iopm_va)); 5704 5705 /* 5706 * Note, SEV setup consumes npt_enabled and enable_mmio_caching (which 5707 * may be modified by svm_adjust_mmio_mask()), as well as nrips. 5708 */ 5709 sev_hardware_setup(); 5710 5711 svm_hv_hardware_setup(); 5712 5713 enable_apicv = avic_hardware_setup(); 5714 if (!enable_apicv) { 5715 enable_ipiv = false; 5716 svm_x86_ops.vcpu_blocking = NULL; 5717 svm_x86_ops.vcpu_unblocking = NULL; 5718 svm_x86_ops.vcpu_get_apicv_inhibit_reasons = NULL; 5719 } 5720 5721 if (vls) { 5722 if (!npt_enabled || 5723 !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) || 5724 !IS_ENABLED(CONFIG_X86_64)) { 5725 vls = false; 5726 } else { 5727 pr_info("Virtual VMLOAD VMSAVE supported\n"); 5728 } 5729 } 5730 5731 if (boot_cpu_has(X86_FEATURE_SVME_ADDR_CHK)) 5732 svm_gp_erratum_intercept = false; 5733 5734 if (vgif) { 5735 if (!boot_cpu_has(X86_FEATURE_VGIF)) 5736 vgif = false; 5737 else 5738 pr_info("Virtual GIF supported\n"); 5739 } 5740 5741 vnmi = vgif && vnmi && boot_cpu_has(X86_FEATURE_VNMI); 5742 if (vnmi) 5743 pr_info("Virtual NMI enabled\n"); 5744 5745 if (!vnmi) { 5746 svm_x86_ops.is_vnmi_pending = NULL; 5747 svm_x86_ops.set_vnmi_pending = NULL; 5748 } 5749 5750 if (!enable_pmu) 5751 pr_info("PMU virtualization is disabled\n"); 5752 5753 svm_set_cpu_caps(); 5754 5755 kvm_caps.inapplicable_quirks &= ~KVM_X86_QUIRK_CD_NW_CLEARED; 5756 5757 for_each_possible_cpu(cpu) { 5758 r = svm_cpu_init(cpu); 5759 if (r) 5760 goto err; 5761 } 5762 5763 return 0; 5764 5765 err: 5766 svm_hardware_unsetup(); 5767 return r; 5768 } 5769 5770 5771 static struct kvm_x86_init_ops svm_init_ops __initdata = { 5772 .hardware_setup = svm_hardware_setup, 5773 5774 .runtime_ops = &svm_x86_ops, 5775 .pmu_ops = &amd_pmu_ops, 5776 }; 5777 5778 static void __svm_exit(void) 5779 { 5780 kvm_x86_vendor_exit(); 5781 } 5782 5783 static int __init svm_init(void) 5784 { 5785 int r; 5786 5787 KVM_SANITY_CHECK_VM_STRUCT_SIZE(kvm_svm); 5788 5789 __unused_size_checks(); 5790 5791 if (!kvm_is_svm_supported()) 5792 return -EOPNOTSUPP; 5793 5794 r = kvm_x86_vendor_init(&svm_init_ops); 5795 if (r) 5796 return r; 5797 5798 /* 5799 * Common KVM initialization _must_ come last, after this, /dev/kvm is 5800 * exposed to userspace! 5801 */ 5802 r = kvm_init(sizeof(struct vcpu_svm), __alignof__(struct vcpu_svm), 5803 THIS_MODULE); 5804 if (r) 5805 goto err_kvm_init; 5806 5807 return 0; 5808 5809 err_kvm_init: 5810 __svm_exit(); 5811 return r; 5812 } 5813 5814 static void __exit svm_exit(void) 5815 { 5816 kvm_exit(); 5817 __svm_exit(); 5818 } 5819 5820 module_init(svm_init) 5821 module_exit(svm_exit) 5822