1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * This module enables machines with Intel VT-x extensions to run virtual 6 * machines without emulation or binary translation. 7 * 8 * Copyright (C) 2006 Qumranet, Inc. 9 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 10 * 11 * Authors: 12 * Avi Kivity <avi@qumranet.com> 13 * Yaniv Kamay <yaniv@qumranet.com> 14 */ 15 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 16 17 #include <linux/highmem.h> 18 #include <linux/hrtimer.h> 19 #include <linux/kernel.h> 20 #include <linux/kvm_host.h> 21 #include <linux/module.h> 22 #include <linux/moduleparam.h> 23 #include <linux/mod_devicetable.h> 24 #include <linux/mm.h> 25 #include <linux/objtool.h> 26 #include <linux/sched.h> 27 #include <linux/sched/smt.h> 28 #include <linux/slab.h> 29 #include <linux/tboot.h> 30 #include <linux/trace_events.h> 31 32 #include <asm/apic.h> 33 #include <asm/asm.h> 34 #include <asm/cpu.h> 35 #include <asm/cpu_device_id.h> 36 #include <asm/debugreg.h> 37 #include <asm/desc.h> 38 #include <asm/fpu/api.h> 39 #include <asm/fpu/xstate.h> 40 #include <asm/fred.h> 41 #include <asm/idtentry.h> 42 #include <asm/io.h> 43 #include <asm/irq_remapping.h> 44 #include <asm/reboot.h> 45 #include <asm/perf_event.h> 46 #include <asm/mmu_context.h> 47 #include <asm/mshyperv.h> 48 #include <asm/msr.h> 49 #include <asm/mwait.h> 50 #include <asm/spec-ctrl.h> 51 #include <asm/vmx.h> 52 53 #include <trace/events/ipi.h> 54 55 #include "capabilities.h" 56 #include "common.h" 57 #include "cpuid.h" 58 #include "hyperv.h" 59 #include "kvm_onhyperv.h" 60 #include "irq.h" 61 #include "kvm_cache_regs.h" 62 #include "lapic.h" 63 #include "mmu.h" 64 #include "nested.h" 65 #include "pmu.h" 66 #include "sgx.h" 67 #include "trace.h" 68 #include "vmcs.h" 69 #include "vmcs12.h" 70 #include "vmx.h" 71 #include "x86.h" 72 #include "x86_ops.h" 73 #include "smm.h" 74 #include "vmx_onhyperv.h" 75 #include "posted_intr.h" 76 77 #include "mmu/spte.h" 78 79 MODULE_AUTHOR("Qumranet"); 80 MODULE_DESCRIPTION("KVM support for VMX (Intel VT-x) extensions"); 81 MODULE_LICENSE("GPL"); 82 83 #ifdef MODULE 84 static const struct x86_cpu_id vmx_cpu_id[] = { 85 X86_MATCH_FEATURE(X86_FEATURE_VMX, NULL), 86 {} 87 }; 88 MODULE_DEVICE_TABLE(x86cpu, vmx_cpu_id); 89 #endif 90 91 bool __read_mostly enable_vpid = 1; 92 module_param_named(vpid, enable_vpid, bool, 0444); 93 94 static bool __read_mostly enable_vnmi = 1; 95 module_param_named(vnmi, enable_vnmi, bool, 0444); 96 97 bool __read_mostly flexpriority_enabled = 1; 98 module_param_named(flexpriority, flexpriority_enabled, bool, 0444); 99 100 bool __read_mostly enable_ept = 1; 101 module_param_named(ept, enable_ept, bool, 0444); 102 103 bool __read_mostly enable_unrestricted_guest = 1; 104 module_param_named(unrestricted_guest, 105 enable_unrestricted_guest, bool, 0444); 106 107 bool __read_mostly enable_ept_ad_bits = 1; 108 module_param_named(eptad, enable_ept_ad_bits, bool, 0444); 109 110 static bool __read_mostly emulate_invalid_guest_state = true; 111 module_param(emulate_invalid_guest_state, bool, 0444); 112 113 static bool __read_mostly fasteoi = 1; 114 module_param(fasteoi, bool, 0444); 115 116 module_param(enable_apicv, bool, 0444); 117 module_param(enable_ipiv, bool, 0444); 118 119 module_param(enable_device_posted_irqs, bool, 0444); 120 121 /* 122 * If nested=1, nested virtualization is supported, i.e., guests may use 123 * VMX and be a hypervisor for its own guests. If nested=0, guests may not 124 * use VMX instructions. 125 */ 126 static bool __read_mostly nested = 1; 127 module_param(nested, bool, 0444); 128 129 bool __read_mostly enable_pml = 1; 130 module_param_named(pml, enable_pml, bool, 0444); 131 132 static bool __read_mostly error_on_inconsistent_vmcs_config = true; 133 module_param(error_on_inconsistent_vmcs_config, bool, 0444); 134 135 static bool __read_mostly dump_invalid_vmcs = 0; 136 module_param(dump_invalid_vmcs, bool, 0644); 137 138 #define MSR_BITMAP_MODE_X2APIC 1 139 #define MSR_BITMAP_MODE_X2APIC_APICV 2 140 141 #define KVM_VMX_TSC_MULTIPLIER_MAX 0xffffffffffffffffULL 142 143 /* Guest_tsc -> host_tsc conversion requires 64-bit division. */ 144 static int __read_mostly cpu_preemption_timer_multi; 145 static bool __read_mostly enable_preemption_timer = 1; 146 #ifdef CONFIG_X86_64 147 module_param_named(preemption_timer, enable_preemption_timer, bool, S_IRUGO); 148 #endif 149 150 extern bool __read_mostly allow_smaller_maxphyaddr; 151 module_param(allow_smaller_maxphyaddr, bool, S_IRUGO); 152 153 #define KVM_VM_CR0_ALWAYS_OFF (X86_CR0_NW | X86_CR0_CD) 154 #define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR0_NE 155 #define KVM_VM_CR0_ALWAYS_ON \ 156 (KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE) 157 158 #define KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR4_VMXE 159 #define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE) 160 #define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE) 161 162 #define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM)) 163 164 #define MSR_IA32_RTIT_STATUS_MASK (~(RTIT_STATUS_FILTEREN | \ 165 RTIT_STATUS_CONTEXTEN | RTIT_STATUS_TRIGGEREN | \ 166 RTIT_STATUS_ERROR | RTIT_STATUS_STOPPED | \ 167 RTIT_STATUS_BYTECNT)) 168 169 /* 170 * These 2 parameters are used to config the controls for Pause-Loop Exiting: 171 * ple_gap: upper bound on the amount of time between two successive 172 * executions of PAUSE in a loop. Also indicate if ple enabled. 173 * According to test, this time is usually smaller than 128 cycles. 174 * ple_window: upper bound on the amount of time a guest is allowed to execute 175 * in a PAUSE loop. Tests indicate that most spinlocks are held for 176 * less than 2^12 cycles 177 * Time is measured based on a counter that runs at the same rate as the TSC, 178 * refer SDM volume 3b section 21.6.13 & 22.1.3. 179 */ 180 static unsigned int ple_gap = KVM_DEFAULT_PLE_GAP; 181 module_param(ple_gap, uint, 0444); 182 183 static unsigned int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW; 184 module_param(ple_window, uint, 0444); 185 186 /* Default doubles per-vcpu window every exit. */ 187 static unsigned int ple_window_grow = KVM_DEFAULT_PLE_WINDOW_GROW; 188 module_param(ple_window_grow, uint, 0444); 189 190 /* Default resets per-vcpu window every exit to ple_window. */ 191 static unsigned int ple_window_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK; 192 module_param(ple_window_shrink, uint, 0444); 193 194 /* Default is to compute the maximum so we can never overflow. */ 195 static unsigned int ple_window_max = KVM_VMX_DEFAULT_PLE_WINDOW_MAX; 196 module_param(ple_window_max, uint, 0444); 197 198 /* Default is SYSTEM mode, 1 for host-guest mode (which is BROKEN) */ 199 int __read_mostly pt_mode = PT_MODE_SYSTEM; 200 #ifdef CONFIG_BROKEN 201 module_param(pt_mode, int, S_IRUGO); 202 #endif 203 204 struct x86_pmu_lbr __ro_after_init vmx_lbr_caps; 205 206 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_should_flush); 207 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_flush_cond); 208 static DEFINE_MUTEX(vmx_l1d_flush_mutex); 209 210 /* Storage for pre module init parameter parsing */ 211 static enum vmx_l1d_flush_state __read_mostly vmentry_l1d_flush_param = VMENTER_L1D_FLUSH_AUTO; 212 213 static const struct { 214 const char *option; 215 bool for_parse; 216 } vmentry_l1d_param[] = { 217 [VMENTER_L1D_FLUSH_AUTO] = {"auto", true}, 218 [VMENTER_L1D_FLUSH_NEVER] = {"never", true}, 219 [VMENTER_L1D_FLUSH_COND] = {"cond", true}, 220 [VMENTER_L1D_FLUSH_ALWAYS] = {"always", true}, 221 [VMENTER_L1D_FLUSH_EPT_DISABLED] = {"EPT disabled", false}, 222 [VMENTER_L1D_FLUSH_NOT_REQUIRED] = {"not required", false}, 223 }; 224 225 #define L1D_CACHE_ORDER 4 226 static void *vmx_l1d_flush_pages; 227 228 static int vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf) 229 { 230 struct page *page; 231 unsigned int i; 232 233 if (!boot_cpu_has_bug(X86_BUG_L1TF)) { 234 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED; 235 return 0; 236 } 237 238 if (!enable_ept) { 239 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_EPT_DISABLED; 240 return 0; 241 } 242 243 if (kvm_host.arch_capabilities & ARCH_CAP_SKIP_VMENTRY_L1DFLUSH) { 244 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED; 245 return 0; 246 } 247 248 /* If set to auto use the default l1tf mitigation method */ 249 if (l1tf == VMENTER_L1D_FLUSH_AUTO) { 250 switch (l1tf_mitigation) { 251 case L1TF_MITIGATION_OFF: 252 l1tf = VMENTER_L1D_FLUSH_NEVER; 253 break; 254 case L1TF_MITIGATION_AUTO: 255 case L1TF_MITIGATION_FLUSH_NOWARN: 256 case L1TF_MITIGATION_FLUSH: 257 case L1TF_MITIGATION_FLUSH_NOSMT: 258 l1tf = VMENTER_L1D_FLUSH_COND; 259 break; 260 case L1TF_MITIGATION_FULL: 261 case L1TF_MITIGATION_FULL_FORCE: 262 l1tf = VMENTER_L1D_FLUSH_ALWAYS; 263 break; 264 } 265 } else if (l1tf_mitigation == L1TF_MITIGATION_FULL_FORCE) { 266 l1tf = VMENTER_L1D_FLUSH_ALWAYS; 267 } 268 269 if (l1tf != VMENTER_L1D_FLUSH_NEVER && !vmx_l1d_flush_pages && 270 !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) { 271 /* 272 * This allocation for vmx_l1d_flush_pages is not tied to a VM 273 * lifetime and so should not be charged to a memcg. 274 */ 275 page = alloc_pages(GFP_KERNEL, L1D_CACHE_ORDER); 276 if (!page) 277 return -ENOMEM; 278 vmx_l1d_flush_pages = page_address(page); 279 280 /* 281 * Initialize each page with a different pattern in 282 * order to protect against KSM in the nested 283 * virtualization case. 284 */ 285 for (i = 0; i < 1u << L1D_CACHE_ORDER; ++i) { 286 memset(vmx_l1d_flush_pages + i * PAGE_SIZE, i + 1, 287 PAGE_SIZE); 288 } 289 } 290 291 l1tf_vmx_mitigation = l1tf; 292 293 if (l1tf != VMENTER_L1D_FLUSH_NEVER) 294 static_branch_enable(&vmx_l1d_should_flush); 295 else 296 static_branch_disable(&vmx_l1d_should_flush); 297 298 if (l1tf == VMENTER_L1D_FLUSH_COND) 299 static_branch_enable(&vmx_l1d_flush_cond); 300 else 301 static_branch_disable(&vmx_l1d_flush_cond); 302 return 0; 303 } 304 305 static int vmentry_l1d_flush_parse(const char *s) 306 { 307 unsigned int i; 308 309 if (s) { 310 for (i = 0; i < ARRAY_SIZE(vmentry_l1d_param); i++) { 311 if (vmentry_l1d_param[i].for_parse && 312 sysfs_streq(s, vmentry_l1d_param[i].option)) 313 return i; 314 } 315 } 316 return -EINVAL; 317 } 318 319 static int vmentry_l1d_flush_set(const char *s, const struct kernel_param *kp) 320 { 321 int l1tf, ret; 322 323 l1tf = vmentry_l1d_flush_parse(s); 324 if (l1tf < 0) 325 return l1tf; 326 327 if (!boot_cpu_has(X86_BUG_L1TF)) 328 return 0; 329 330 /* 331 * Has vmx_init() run already? If not then this is the pre init 332 * parameter parsing. In that case just store the value and let 333 * vmx_init() do the proper setup after enable_ept has been 334 * established. 335 */ 336 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) { 337 vmentry_l1d_flush_param = l1tf; 338 return 0; 339 } 340 341 mutex_lock(&vmx_l1d_flush_mutex); 342 ret = vmx_setup_l1d_flush(l1tf); 343 mutex_unlock(&vmx_l1d_flush_mutex); 344 return ret; 345 } 346 347 static int vmentry_l1d_flush_get(char *s, const struct kernel_param *kp) 348 { 349 if (WARN_ON_ONCE(l1tf_vmx_mitigation >= ARRAY_SIZE(vmentry_l1d_param))) 350 return sysfs_emit(s, "???\n"); 351 352 return sysfs_emit(s, "%s\n", vmentry_l1d_param[l1tf_vmx_mitigation].option); 353 } 354 355 static __always_inline void vmx_disable_fb_clear(struct vcpu_vmx *vmx) 356 { 357 u64 msr; 358 359 if (!vmx->disable_fb_clear) 360 return; 361 362 msr = native_rdmsrq(MSR_IA32_MCU_OPT_CTRL); 363 msr |= FB_CLEAR_DIS; 364 native_wrmsrq(MSR_IA32_MCU_OPT_CTRL, msr); 365 /* Cache the MSR value to avoid reading it later */ 366 vmx->msr_ia32_mcu_opt_ctrl = msr; 367 } 368 369 static __always_inline void vmx_enable_fb_clear(struct vcpu_vmx *vmx) 370 { 371 if (!vmx->disable_fb_clear) 372 return; 373 374 vmx->msr_ia32_mcu_opt_ctrl &= ~FB_CLEAR_DIS; 375 native_wrmsrq(MSR_IA32_MCU_OPT_CTRL, vmx->msr_ia32_mcu_opt_ctrl); 376 } 377 378 static void vmx_update_fb_clear_dis(struct kvm_vcpu *vcpu, struct vcpu_vmx *vmx) 379 { 380 /* 381 * Disable VERW's behavior of clearing CPU buffers for the guest if the 382 * CPU isn't affected by MDS/TAA, and the host hasn't forcefully enabled 383 * the mitigation. Disabling the clearing behavior provides a 384 * performance boost for guests that aren't aware that manually clearing 385 * CPU buffers is unnecessary, at the cost of MSR accesses on VM-Entry 386 * and VM-Exit. 387 */ 388 vmx->disable_fb_clear = !cpu_feature_enabled(X86_FEATURE_CLEAR_CPU_BUF) && 389 (kvm_host.arch_capabilities & ARCH_CAP_FB_CLEAR_CTRL) && 390 !boot_cpu_has_bug(X86_BUG_MDS) && 391 !boot_cpu_has_bug(X86_BUG_TAA); 392 393 /* 394 * If guest will not execute VERW, there is no need to set FB_CLEAR_DIS 395 * at VMEntry. Skip the MSR read/write when a guest has no use case to 396 * execute VERW. 397 */ 398 if ((vcpu->arch.arch_capabilities & ARCH_CAP_FB_CLEAR) || 399 ((vcpu->arch.arch_capabilities & ARCH_CAP_MDS_NO) && 400 (vcpu->arch.arch_capabilities & ARCH_CAP_TAA_NO) && 401 (vcpu->arch.arch_capabilities & ARCH_CAP_PSDP_NO) && 402 (vcpu->arch.arch_capabilities & ARCH_CAP_FBSDP_NO) && 403 (vcpu->arch.arch_capabilities & ARCH_CAP_SBDR_SSDP_NO))) 404 vmx->disable_fb_clear = false; 405 } 406 407 static const struct kernel_param_ops vmentry_l1d_flush_ops = { 408 .set = vmentry_l1d_flush_set, 409 .get = vmentry_l1d_flush_get, 410 }; 411 module_param_cb(vmentry_l1d_flush, &vmentry_l1d_flush_ops, NULL, 0644); 412 413 static u32 vmx_segment_access_rights(struct kvm_segment *var); 414 415 void vmx_vmexit(void); 416 417 #define vmx_insn_failed(fmt...) \ 418 do { \ 419 WARN_ONCE(1, fmt); \ 420 pr_warn_ratelimited(fmt); \ 421 } while (0) 422 423 noinline void vmread_error(unsigned long field) 424 { 425 vmx_insn_failed("vmread failed: field=%lx\n", field); 426 } 427 428 #ifndef CONFIG_CC_HAS_ASM_GOTO_OUTPUT 429 noinstr void vmread_error_trampoline2(unsigned long field, bool fault) 430 { 431 if (fault) { 432 kvm_spurious_fault(); 433 } else { 434 instrumentation_begin(); 435 vmread_error(field); 436 instrumentation_end(); 437 } 438 } 439 #endif 440 441 noinline void vmwrite_error(unsigned long field, unsigned long value) 442 { 443 vmx_insn_failed("vmwrite failed: field=%lx val=%lx err=%u\n", 444 field, value, vmcs_read32(VM_INSTRUCTION_ERROR)); 445 } 446 447 noinline void vmclear_error(struct vmcs *vmcs, u64 phys_addr) 448 { 449 vmx_insn_failed("vmclear failed: %p/%llx err=%u\n", 450 vmcs, phys_addr, vmcs_read32(VM_INSTRUCTION_ERROR)); 451 } 452 453 noinline void vmptrld_error(struct vmcs *vmcs, u64 phys_addr) 454 { 455 vmx_insn_failed("vmptrld failed: %p/%llx err=%u\n", 456 vmcs, phys_addr, vmcs_read32(VM_INSTRUCTION_ERROR)); 457 } 458 459 noinline void invvpid_error(unsigned long ext, u16 vpid, gva_t gva) 460 { 461 vmx_insn_failed("invvpid failed: ext=0x%lx vpid=%u gva=0x%lx\n", 462 ext, vpid, gva); 463 } 464 465 noinline void invept_error(unsigned long ext, u64 eptp) 466 { 467 vmx_insn_failed("invept failed: ext=0x%lx eptp=%llx\n", ext, eptp); 468 } 469 470 static DEFINE_PER_CPU(struct vmcs *, vmxarea); 471 DEFINE_PER_CPU(struct vmcs *, current_vmcs); 472 /* 473 * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed 474 * when a CPU is brought down, and we need to VMCLEAR all VMCSs loaded on it. 475 */ 476 static DEFINE_PER_CPU(struct list_head, loaded_vmcss_on_cpu); 477 478 static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS); 479 static DEFINE_SPINLOCK(vmx_vpid_lock); 480 481 struct vmcs_config vmcs_config __ro_after_init; 482 struct vmx_capability vmx_capability __ro_after_init; 483 484 #define VMX_SEGMENT_FIELD(seg) \ 485 [VCPU_SREG_##seg] = { \ 486 .selector = GUEST_##seg##_SELECTOR, \ 487 .base = GUEST_##seg##_BASE, \ 488 .limit = GUEST_##seg##_LIMIT, \ 489 .ar_bytes = GUEST_##seg##_AR_BYTES, \ 490 } 491 492 static const struct kvm_vmx_segment_field { 493 unsigned selector; 494 unsigned base; 495 unsigned limit; 496 unsigned ar_bytes; 497 } kvm_vmx_segment_fields[] = { 498 VMX_SEGMENT_FIELD(CS), 499 VMX_SEGMENT_FIELD(DS), 500 VMX_SEGMENT_FIELD(ES), 501 VMX_SEGMENT_FIELD(FS), 502 VMX_SEGMENT_FIELD(GS), 503 VMX_SEGMENT_FIELD(SS), 504 VMX_SEGMENT_FIELD(TR), 505 VMX_SEGMENT_FIELD(LDTR), 506 }; 507 508 509 static unsigned long host_idt_base; 510 511 #if IS_ENABLED(CONFIG_HYPERV) 512 static bool __read_mostly enlightened_vmcs = true; 513 module_param(enlightened_vmcs, bool, 0444); 514 515 static int hv_enable_l2_tlb_flush(struct kvm_vcpu *vcpu) 516 { 517 struct hv_enlightened_vmcs *evmcs; 518 hpa_t partition_assist_page = hv_get_partition_assist_page(vcpu); 519 520 if (partition_assist_page == INVALID_PAGE) 521 return -ENOMEM; 522 523 evmcs = (struct hv_enlightened_vmcs *)to_vmx(vcpu)->loaded_vmcs->vmcs; 524 525 evmcs->partition_assist_page = partition_assist_page; 526 evmcs->hv_vm_id = (unsigned long)vcpu->kvm; 527 evmcs->hv_enlightenments_control.nested_flush_hypercall = 1; 528 529 return 0; 530 } 531 532 static __init void hv_init_evmcs(void) 533 { 534 int cpu; 535 536 if (!enlightened_vmcs) 537 return; 538 539 /* 540 * Enlightened VMCS usage should be recommended and the host needs 541 * to support eVMCS v1 or above. 542 */ 543 if (ms_hyperv.hints & HV_X64_ENLIGHTENED_VMCS_RECOMMENDED && 544 (ms_hyperv.nested_features & HV_X64_ENLIGHTENED_VMCS_VERSION) >= 545 KVM_EVMCS_VERSION) { 546 547 /* Check that we have assist pages on all online CPUs */ 548 for_each_online_cpu(cpu) { 549 if (!hv_get_vp_assist_page(cpu)) { 550 enlightened_vmcs = false; 551 break; 552 } 553 } 554 555 if (enlightened_vmcs) { 556 pr_info("Using Hyper-V Enlightened VMCS\n"); 557 static_branch_enable(&__kvm_is_using_evmcs); 558 } 559 560 if (ms_hyperv.nested_features & HV_X64_NESTED_DIRECT_FLUSH) 561 vt_x86_ops.enable_l2_tlb_flush 562 = hv_enable_l2_tlb_flush; 563 } else { 564 enlightened_vmcs = false; 565 } 566 } 567 568 static void hv_reset_evmcs(void) 569 { 570 struct hv_vp_assist_page *vp_ap; 571 572 if (!kvm_is_using_evmcs()) 573 return; 574 575 /* 576 * KVM should enable eVMCS if and only if all CPUs have a VP assist 577 * page, and should reject CPU onlining if eVMCS is enabled the CPU 578 * doesn't have a VP assist page allocated. 579 */ 580 vp_ap = hv_get_vp_assist_page(smp_processor_id()); 581 if (WARN_ON_ONCE(!vp_ap)) 582 return; 583 584 /* 585 * Reset everything to support using non-enlightened VMCS access later 586 * (e.g. when we reload the module with enlightened_vmcs=0) 587 */ 588 vp_ap->nested_control.features.directhypercall = 0; 589 vp_ap->current_nested_vmcs = 0; 590 vp_ap->enlighten_vmentry = 0; 591 } 592 593 #else /* IS_ENABLED(CONFIG_HYPERV) */ 594 static void hv_init_evmcs(void) {} 595 static void hv_reset_evmcs(void) {} 596 #endif /* IS_ENABLED(CONFIG_HYPERV) */ 597 598 /* 599 * Comment's format: document - errata name - stepping - processor name. 600 * Refer from 601 * https://www.virtualbox.org/svn/vbox/trunk/src/VBox/VMM/VMMR0/HMR0.cpp 602 */ 603 static u32 vmx_preemption_cpu_tfms[] = { 604 /* 323344.pdf - BA86 - D0 - Xeon 7500 Series */ 605 0x000206E6, 606 /* 323056.pdf - AAX65 - C2 - Xeon L3406 */ 607 /* 322814.pdf - AAT59 - C2 - i7-600, i5-500, i5-400 and i3-300 Mobile */ 608 /* 322911.pdf - AAU65 - C2 - i5-600, i3-500 Desktop and Pentium G6950 */ 609 0x00020652, 610 /* 322911.pdf - AAU65 - K0 - i5-600, i3-500 Desktop and Pentium G6950 */ 611 0x00020655, 612 /* 322373.pdf - AAO95 - B1 - Xeon 3400 Series */ 613 /* 322166.pdf - AAN92 - B1 - i7-800 and i5-700 Desktop */ 614 /* 615 * 320767.pdf - AAP86 - B1 - 616 * i7-900 Mobile Extreme, i7-800 and i7-700 Mobile 617 */ 618 0x000106E5, 619 /* 321333.pdf - AAM126 - C0 - Xeon 3500 */ 620 0x000106A0, 621 /* 321333.pdf - AAM126 - C1 - Xeon 3500 */ 622 0x000106A1, 623 /* 320836.pdf - AAJ124 - C0 - i7-900 Desktop Extreme and i7-900 Desktop */ 624 0x000106A4, 625 /* 321333.pdf - AAM126 - D0 - Xeon 3500 */ 626 /* 321324.pdf - AAK139 - D0 - Xeon 5500 */ 627 /* 320836.pdf - AAJ124 - D0 - i7-900 Extreme and i7-900 Desktop */ 628 0x000106A5, 629 /* Xeon E3-1220 V2 */ 630 0x000306A8, 631 }; 632 633 static inline bool cpu_has_broken_vmx_preemption_timer(void) 634 { 635 u32 eax = cpuid_eax(0x00000001), i; 636 637 /* Clear the reserved bits */ 638 eax &= ~(0x3U << 14 | 0xfU << 28); 639 for (i = 0; i < ARRAY_SIZE(vmx_preemption_cpu_tfms); i++) 640 if (eax == vmx_preemption_cpu_tfms[i]) 641 return true; 642 643 return false; 644 } 645 646 static inline bool cpu_need_virtualize_apic_accesses(struct kvm_vcpu *vcpu) 647 { 648 return flexpriority_enabled && lapic_in_kernel(vcpu); 649 } 650 651 struct vmx_uret_msr *vmx_find_uret_msr(struct vcpu_vmx *vmx, u32 msr) 652 { 653 int i; 654 655 i = kvm_find_user_return_msr(msr); 656 if (i >= 0) 657 return &vmx->guest_uret_msrs[i]; 658 return NULL; 659 } 660 661 static int vmx_set_guest_uret_msr(struct vcpu_vmx *vmx, 662 struct vmx_uret_msr *msr, u64 data) 663 { 664 unsigned int slot = msr - vmx->guest_uret_msrs; 665 int ret = 0; 666 667 if (msr->load_into_hardware) { 668 preempt_disable(); 669 ret = kvm_set_user_return_msr(slot, data, msr->mask); 670 preempt_enable(); 671 } 672 if (!ret) 673 msr->data = data; 674 return ret; 675 } 676 677 /* 678 * Disable VMX and clear CR4.VMXE (even if VMXOFF faults) 679 * 680 * Note, VMXOFF causes a #UD if the CPU is !post-VMXON, but it's impossible to 681 * atomically track post-VMXON state, e.g. this may be called in NMI context. 682 * Eat all faults as all other faults on VMXOFF faults are mode related, i.e. 683 * faults are guaranteed to be due to the !post-VMXON check unless the CPU is 684 * magically in RM, VM86, compat mode, or at CPL>0. 685 */ 686 static int kvm_cpu_vmxoff(void) 687 { 688 asm goto("1: vmxoff\n\t" 689 _ASM_EXTABLE(1b, %l[fault]) 690 ::: "cc", "memory" : fault); 691 692 cr4_clear_bits(X86_CR4_VMXE); 693 return 0; 694 695 fault: 696 cr4_clear_bits(X86_CR4_VMXE); 697 return -EIO; 698 } 699 700 void vmx_emergency_disable_virtualization_cpu(void) 701 { 702 int cpu = raw_smp_processor_id(); 703 struct loaded_vmcs *v; 704 705 kvm_rebooting = true; 706 707 /* 708 * Note, CR4.VMXE can be _cleared_ in NMI context, but it can only be 709 * set in task context. If this races with VMX is disabled by an NMI, 710 * VMCLEAR and VMXOFF may #UD, but KVM will eat those faults due to 711 * kvm_rebooting set. 712 */ 713 if (!(__read_cr4() & X86_CR4_VMXE)) 714 return; 715 716 list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu), 717 loaded_vmcss_on_cpu_link) { 718 vmcs_clear(v->vmcs); 719 if (v->shadow_vmcs) 720 vmcs_clear(v->shadow_vmcs); 721 } 722 723 kvm_cpu_vmxoff(); 724 } 725 726 static void __loaded_vmcs_clear(void *arg) 727 { 728 struct loaded_vmcs *loaded_vmcs = arg; 729 int cpu = raw_smp_processor_id(); 730 731 if (loaded_vmcs->cpu != cpu) 732 return; /* vcpu migration can race with cpu offline */ 733 if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs) 734 per_cpu(current_vmcs, cpu) = NULL; 735 736 vmcs_clear(loaded_vmcs->vmcs); 737 if (loaded_vmcs->shadow_vmcs && loaded_vmcs->launched) 738 vmcs_clear(loaded_vmcs->shadow_vmcs); 739 740 list_del(&loaded_vmcs->loaded_vmcss_on_cpu_link); 741 742 /* 743 * Ensure all writes to loaded_vmcs, including deleting it from its 744 * current percpu list, complete before setting loaded_vmcs->cpu to 745 * -1, otherwise a different cpu can see loaded_vmcs->cpu == -1 first 746 * and add loaded_vmcs to its percpu list before it's deleted from this 747 * cpu's list. Pairs with the smp_rmb() in vmx_vcpu_load_vmcs(). 748 */ 749 smp_wmb(); 750 751 loaded_vmcs->cpu = -1; 752 loaded_vmcs->launched = 0; 753 } 754 755 void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs) 756 { 757 int cpu = loaded_vmcs->cpu; 758 759 if (cpu != -1) 760 smp_call_function_single(cpu, 761 __loaded_vmcs_clear, loaded_vmcs, 1); 762 } 763 764 static bool vmx_segment_cache_test_set(struct vcpu_vmx *vmx, unsigned seg, 765 unsigned field) 766 { 767 bool ret; 768 u32 mask = 1 << (seg * SEG_FIELD_NR + field); 769 770 if (!kvm_register_is_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS)) { 771 kvm_register_mark_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS); 772 vmx->segment_cache.bitmask = 0; 773 } 774 ret = vmx->segment_cache.bitmask & mask; 775 vmx->segment_cache.bitmask |= mask; 776 return ret; 777 } 778 779 static u16 vmx_read_guest_seg_selector(struct vcpu_vmx *vmx, unsigned seg) 780 { 781 u16 *p = &vmx->segment_cache.seg[seg].selector; 782 783 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_SEL)) 784 *p = vmcs_read16(kvm_vmx_segment_fields[seg].selector); 785 return *p; 786 } 787 788 static ulong vmx_read_guest_seg_base(struct vcpu_vmx *vmx, unsigned seg) 789 { 790 ulong *p = &vmx->segment_cache.seg[seg].base; 791 792 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_BASE)) 793 *p = vmcs_readl(kvm_vmx_segment_fields[seg].base); 794 return *p; 795 } 796 797 static u32 vmx_read_guest_seg_limit(struct vcpu_vmx *vmx, unsigned seg) 798 { 799 u32 *p = &vmx->segment_cache.seg[seg].limit; 800 801 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_LIMIT)) 802 *p = vmcs_read32(kvm_vmx_segment_fields[seg].limit); 803 return *p; 804 } 805 806 static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg) 807 { 808 u32 *p = &vmx->segment_cache.seg[seg].ar; 809 810 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_AR)) 811 *p = vmcs_read32(kvm_vmx_segment_fields[seg].ar_bytes); 812 return *p; 813 } 814 815 void vmx_update_exception_bitmap(struct kvm_vcpu *vcpu) 816 { 817 u32 eb; 818 819 eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) | 820 (1u << DB_VECTOR) | (1u << AC_VECTOR); 821 /* 822 * #VE isn't used for VMX. To test against unexpected changes 823 * related to #VE for VMX, intercept unexpected #VE and warn on it. 824 */ 825 if (IS_ENABLED(CONFIG_KVM_INTEL_PROVE_VE)) 826 eb |= 1u << VE_VECTOR; 827 /* 828 * Guest access to VMware backdoor ports could legitimately 829 * trigger #GP because of TSS I/O permission bitmap. 830 * We intercept those #GP and allow access to them anyway 831 * as VMware does. 832 */ 833 if (enable_vmware_backdoor) 834 eb |= (1u << GP_VECTOR); 835 if ((vcpu->guest_debug & 836 (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) == 837 (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) 838 eb |= 1u << BP_VECTOR; 839 if (to_vmx(vcpu)->rmode.vm86_active) 840 eb = ~0; 841 if (!vmx_need_pf_intercept(vcpu)) 842 eb &= ~(1u << PF_VECTOR); 843 844 /* When we are running a nested L2 guest and L1 specified for it a 845 * certain exception bitmap, we must trap the same exceptions and pass 846 * them to L1. When running L2, we will only handle the exceptions 847 * specified above if L1 did not want them. 848 */ 849 if (is_guest_mode(vcpu)) 850 eb |= get_vmcs12(vcpu)->exception_bitmap; 851 else { 852 int mask = 0, match = 0; 853 854 if (enable_ept && (eb & (1u << PF_VECTOR))) { 855 /* 856 * If EPT is enabled, #PF is currently only intercepted 857 * if MAXPHYADDR is smaller on the guest than on the 858 * host. In that case we only care about present, 859 * non-reserved faults. For vmcs02, however, PFEC_MASK 860 * and PFEC_MATCH are set in prepare_vmcs02_rare. 861 */ 862 mask = PFERR_PRESENT_MASK | PFERR_RSVD_MASK; 863 match = PFERR_PRESENT_MASK; 864 } 865 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, mask); 866 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, match); 867 } 868 869 /* 870 * Disabling xfd interception indicates that dynamic xfeatures 871 * might be used in the guest. Always trap #NM in this case 872 * to save guest xfd_err timely. 873 */ 874 if (vcpu->arch.xfd_no_write_intercept) 875 eb |= (1u << NM_VECTOR); 876 877 vmcs_write32(EXCEPTION_BITMAP, eb); 878 } 879 880 /* 881 * Check if MSR is intercepted for currently loaded MSR bitmap. 882 */ 883 static bool msr_write_intercepted(struct vcpu_vmx *vmx, u32 msr) 884 { 885 if (!(exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS)) 886 return true; 887 888 return vmx_test_msr_bitmap_write(vmx->loaded_vmcs->msr_bitmap, msr); 889 } 890 891 unsigned int __vmx_vcpu_run_flags(struct vcpu_vmx *vmx) 892 { 893 unsigned int flags = 0; 894 895 if (vmx->loaded_vmcs->launched) 896 flags |= VMX_RUN_VMRESUME; 897 898 /* 899 * If writes to the SPEC_CTRL MSR aren't intercepted, the guest is free 900 * to change it directly without causing a vmexit. In that case read 901 * it after vmexit and store it in vmx->spec_ctrl. 902 */ 903 if (!msr_write_intercepted(vmx, MSR_IA32_SPEC_CTRL)) 904 flags |= VMX_RUN_SAVE_SPEC_CTRL; 905 906 if (static_branch_unlikely(&cpu_buf_vm_clear) && 907 kvm_vcpu_can_access_host_mmio(&vmx->vcpu)) 908 flags |= VMX_RUN_CLEAR_CPU_BUFFERS_FOR_MMIO; 909 910 return flags; 911 } 912 913 static __always_inline void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx, 914 unsigned long entry, unsigned long exit) 915 { 916 vm_entry_controls_clearbit(vmx, entry); 917 vm_exit_controls_clearbit(vmx, exit); 918 } 919 920 int vmx_find_loadstore_msr_slot(struct vmx_msrs *m, u32 msr) 921 { 922 unsigned int i; 923 924 for (i = 0; i < m->nr; ++i) { 925 if (m->val[i].index == msr) 926 return i; 927 } 928 return -ENOENT; 929 } 930 931 static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr) 932 { 933 int i; 934 struct msr_autoload *m = &vmx->msr_autoload; 935 936 switch (msr) { 937 case MSR_EFER: 938 if (cpu_has_load_ia32_efer()) { 939 clear_atomic_switch_msr_special(vmx, 940 VM_ENTRY_LOAD_IA32_EFER, 941 VM_EXIT_LOAD_IA32_EFER); 942 return; 943 } 944 break; 945 case MSR_CORE_PERF_GLOBAL_CTRL: 946 if (cpu_has_load_perf_global_ctrl()) { 947 clear_atomic_switch_msr_special(vmx, 948 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL, 949 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL); 950 return; 951 } 952 break; 953 } 954 i = vmx_find_loadstore_msr_slot(&m->guest, msr); 955 if (i < 0) 956 goto skip_guest; 957 --m->guest.nr; 958 m->guest.val[i] = m->guest.val[m->guest.nr]; 959 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr); 960 961 skip_guest: 962 i = vmx_find_loadstore_msr_slot(&m->host, msr); 963 if (i < 0) 964 return; 965 966 --m->host.nr; 967 m->host.val[i] = m->host.val[m->host.nr]; 968 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr); 969 } 970 971 static __always_inline void add_atomic_switch_msr_special(struct vcpu_vmx *vmx, 972 unsigned long entry, unsigned long exit, 973 unsigned long guest_val_vmcs, unsigned long host_val_vmcs, 974 u64 guest_val, u64 host_val) 975 { 976 vmcs_write64(guest_val_vmcs, guest_val); 977 if (host_val_vmcs != HOST_IA32_EFER) 978 vmcs_write64(host_val_vmcs, host_val); 979 vm_entry_controls_setbit(vmx, entry); 980 vm_exit_controls_setbit(vmx, exit); 981 } 982 983 static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr, 984 u64 guest_val, u64 host_val, bool entry_only) 985 { 986 int i, j = 0; 987 struct msr_autoload *m = &vmx->msr_autoload; 988 989 switch (msr) { 990 case MSR_EFER: 991 if (cpu_has_load_ia32_efer()) { 992 add_atomic_switch_msr_special(vmx, 993 VM_ENTRY_LOAD_IA32_EFER, 994 VM_EXIT_LOAD_IA32_EFER, 995 GUEST_IA32_EFER, 996 HOST_IA32_EFER, 997 guest_val, host_val); 998 return; 999 } 1000 break; 1001 case MSR_CORE_PERF_GLOBAL_CTRL: 1002 if (cpu_has_load_perf_global_ctrl()) { 1003 add_atomic_switch_msr_special(vmx, 1004 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL, 1005 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL, 1006 GUEST_IA32_PERF_GLOBAL_CTRL, 1007 HOST_IA32_PERF_GLOBAL_CTRL, 1008 guest_val, host_val); 1009 return; 1010 } 1011 break; 1012 case MSR_IA32_PEBS_ENABLE: 1013 /* PEBS needs a quiescent period after being disabled (to write 1014 * a record). Disabling PEBS through VMX MSR swapping doesn't 1015 * provide that period, so a CPU could write host's record into 1016 * guest's memory. 1017 */ 1018 wrmsrq(MSR_IA32_PEBS_ENABLE, 0); 1019 } 1020 1021 i = vmx_find_loadstore_msr_slot(&m->guest, msr); 1022 if (!entry_only) 1023 j = vmx_find_loadstore_msr_slot(&m->host, msr); 1024 1025 if ((i < 0 && m->guest.nr == MAX_NR_LOADSTORE_MSRS) || 1026 (j < 0 && m->host.nr == MAX_NR_LOADSTORE_MSRS)) { 1027 printk_once(KERN_WARNING "Not enough msr switch entries. " 1028 "Can't add msr %x\n", msr); 1029 return; 1030 } 1031 if (i < 0) { 1032 i = m->guest.nr++; 1033 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr); 1034 } 1035 m->guest.val[i].index = msr; 1036 m->guest.val[i].value = guest_val; 1037 1038 if (entry_only) 1039 return; 1040 1041 if (j < 0) { 1042 j = m->host.nr++; 1043 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr); 1044 } 1045 m->host.val[j].index = msr; 1046 m->host.val[j].value = host_val; 1047 } 1048 1049 static bool update_transition_efer(struct vcpu_vmx *vmx) 1050 { 1051 u64 guest_efer = vmx->vcpu.arch.efer; 1052 u64 ignore_bits = 0; 1053 int i; 1054 1055 /* Shadow paging assumes NX to be available. */ 1056 if (!enable_ept) 1057 guest_efer |= EFER_NX; 1058 1059 /* 1060 * LMA and LME handled by hardware; SCE meaningless outside long mode. 1061 */ 1062 ignore_bits |= EFER_SCE; 1063 #ifdef CONFIG_X86_64 1064 ignore_bits |= EFER_LMA | EFER_LME; 1065 /* SCE is meaningful only in long mode on Intel */ 1066 if (guest_efer & EFER_LMA) 1067 ignore_bits &= ~(u64)EFER_SCE; 1068 #endif 1069 1070 /* 1071 * On EPT, we can't emulate NX, so we must switch EFER atomically. 1072 * On CPUs that support "load IA32_EFER", always switch EFER 1073 * atomically, since it's faster than switching it manually. 1074 */ 1075 if (cpu_has_load_ia32_efer() || 1076 (enable_ept && ((vmx->vcpu.arch.efer ^ kvm_host.efer) & EFER_NX))) { 1077 if (!(guest_efer & EFER_LMA)) 1078 guest_efer &= ~EFER_LME; 1079 if (guest_efer != kvm_host.efer) 1080 add_atomic_switch_msr(vmx, MSR_EFER, 1081 guest_efer, kvm_host.efer, false); 1082 else 1083 clear_atomic_switch_msr(vmx, MSR_EFER); 1084 return false; 1085 } 1086 1087 i = kvm_find_user_return_msr(MSR_EFER); 1088 if (i < 0) 1089 return false; 1090 1091 clear_atomic_switch_msr(vmx, MSR_EFER); 1092 1093 guest_efer &= ~ignore_bits; 1094 guest_efer |= kvm_host.efer & ignore_bits; 1095 1096 vmx->guest_uret_msrs[i].data = guest_efer; 1097 vmx->guest_uret_msrs[i].mask = ~ignore_bits; 1098 1099 return true; 1100 } 1101 1102 #ifdef CONFIG_X86_32 1103 /* 1104 * On 32-bit kernels, VM exits still load the FS and GS bases from the 1105 * VMCS rather than the segment table. KVM uses this helper to figure 1106 * out the current bases to poke them into the VMCS before entry. 1107 */ 1108 static unsigned long segment_base(u16 selector) 1109 { 1110 struct desc_struct *table; 1111 unsigned long v; 1112 1113 if (!(selector & ~SEGMENT_RPL_MASK)) 1114 return 0; 1115 1116 table = get_current_gdt_ro(); 1117 1118 if ((selector & SEGMENT_TI_MASK) == SEGMENT_LDT) { 1119 u16 ldt_selector = kvm_read_ldt(); 1120 1121 if (!(ldt_selector & ~SEGMENT_RPL_MASK)) 1122 return 0; 1123 1124 table = (struct desc_struct *)segment_base(ldt_selector); 1125 } 1126 v = get_desc_base(&table[selector >> 3]); 1127 return v; 1128 } 1129 #endif 1130 1131 static inline bool pt_can_write_msr(struct vcpu_vmx *vmx) 1132 { 1133 return vmx_pt_mode_is_host_guest() && 1134 !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN); 1135 } 1136 1137 static inline bool pt_output_base_valid(struct kvm_vcpu *vcpu, u64 base) 1138 { 1139 /* The base must be 128-byte aligned and a legal physical address. */ 1140 return kvm_vcpu_is_legal_aligned_gpa(vcpu, base, 128); 1141 } 1142 1143 static inline void pt_load_msr(struct pt_ctx *ctx, u32 addr_range) 1144 { 1145 u32 i; 1146 1147 wrmsrq(MSR_IA32_RTIT_STATUS, ctx->status); 1148 wrmsrq(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base); 1149 wrmsrq(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask); 1150 wrmsrq(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match); 1151 for (i = 0; i < addr_range; i++) { 1152 wrmsrq(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]); 1153 wrmsrq(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]); 1154 } 1155 } 1156 1157 static inline void pt_save_msr(struct pt_ctx *ctx, u32 addr_range) 1158 { 1159 u32 i; 1160 1161 rdmsrq(MSR_IA32_RTIT_STATUS, ctx->status); 1162 rdmsrq(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base); 1163 rdmsrq(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask); 1164 rdmsrq(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match); 1165 for (i = 0; i < addr_range; i++) { 1166 rdmsrq(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]); 1167 rdmsrq(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]); 1168 } 1169 } 1170 1171 static void pt_guest_enter(struct vcpu_vmx *vmx) 1172 { 1173 if (vmx_pt_mode_is_system()) 1174 return; 1175 1176 /* 1177 * GUEST_IA32_RTIT_CTL is already set in the VMCS. 1178 * Save host state before VM entry. 1179 */ 1180 rdmsrq(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); 1181 if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) { 1182 wrmsrq(MSR_IA32_RTIT_CTL, 0); 1183 pt_save_msr(&vmx->pt_desc.host, vmx->pt_desc.num_address_ranges); 1184 pt_load_msr(&vmx->pt_desc.guest, vmx->pt_desc.num_address_ranges); 1185 } 1186 } 1187 1188 static void pt_guest_exit(struct vcpu_vmx *vmx) 1189 { 1190 if (vmx_pt_mode_is_system()) 1191 return; 1192 1193 if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) { 1194 pt_save_msr(&vmx->pt_desc.guest, vmx->pt_desc.num_address_ranges); 1195 pt_load_msr(&vmx->pt_desc.host, vmx->pt_desc.num_address_ranges); 1196 } 1197 1198 /* 1199 * KVM requires VM_EXIT_CLEAR_IA32_RTIT_CTL to expose PT to the guest, 1200 * i.e. RTIT_CTL is always cleared on VM-Exit. Restore it if necessary. 1201 */ 1202 if (vmx->pt_desc.host.ctl) 1203 wrmsrq(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); 1204 } 1205 1206 void vmx_set_host_fs_gs(struct vmcs_host_state *host, u16 fs_sel, u16 gs_sel, 1207 unsigned long fs_base, unsigned long gs_base) 1208 { 1209 if (unlikely(fs_sel != host->fs_sel)) { 1210 if (!(fs_sel & 7)) 1211 vmcs_write16(HOST_FS_SELECTOR, fs_sel); 1212 else 1213 vmcs_write16(HOST_FS_SELECTOR, 0); 1214 host->fs_sel = fs_sel; 1215 } 1216 if (unlikely(gs_sel != host->gs_sel)) { 1217 if (!(gs_sel & 7)) 1218 vmcs_write16(HOST_GS_SELECTOR, gs_sel); 1219 else 1220 vmcs_write16(HOST_GS_SELECTOR, 0); 1221 host->gs_sel = gs_sel; 1222 } 1223 if (unlikely(fs_base != host->fs_base)) { 1224 vmcs_writel(HOST_FS_BASE, fs_base); 1225 host->fs_base = fs_base; 1226 } 1227 if (unlikely(gs_base != host->gs_base)) { 1228 vmcs_writel(HOST_GS_BASE, gs_base); 1229 host->gs_base = gs_base; 1230 } 1231 } 1232 1233 void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu) 1234 { 1235 struct vcpu_vmx *vmx = to_vmx(vcpu); 1236 struct vcpu_vt *vt = to_vt(vcpu); 1237 struct vmcs_host_state *host_state; 1238 #ifdef CONFIG_X86_64 1239 int cpu = raw_smp_processor_id(); 1240 #endif 1241 unsigned long fs_base, gs_base; 1242 u16 fs_sel, gs_sel; 1243 int i; 1244 1245 /* 1246 * Note that guest MSRs to be saved/restored can also be changed 1247 * when guest state is loaded. This happens when guest transitions 1248 * to/from long-mode by setting MSR_EFER.LMA. 1249 */ 1250 if (!vmx->guest_uret_msrs_loaded) { 1251 vmx->guest_uret_msrs_loaded = true; 1252 for (i = 0; i < kvm_nr_uret_msrs; ++i) { 1253 if (!vmx->guest_uret_msrs[i].load_into_hardware) 1254 continue; 1255 1256 kvm_set_user_return_msr(i, 1257 vmx->guest_uret_msrs[i].data, 1258 vmx->guest_uret_msrs[i].mask); 1259 } 1260 } 1261 1262 if (vmx->nested.need_vmcs12_to_shadow_sync) 1263 nested_sync_vmcs12_to_shadow(vcpu); 1264 1265 if (vt->guest_state_loaded) 1266 return; 1267 1268 host_state = &vmx->loaded_vmcs->host_state; 1269 1270 /* 1271 * Set host fs and gs selectors. Unfortunately, 22.2.3 does not 1272 * allow segment selectors with cpl > 0 or ti == 1. 1273 */ 1274 host_state->ldt_sel = kvm_read_ldt(); 1275 1276 #ifdef CONFIG_X86_64 1277 savesegment(ds, host_state->ds_sel); 1278 savesegment(es, host_state->es_sel); 1279 1280 gs_base = cpu_kernelmode_gs_base(cpu); 1281 if (likely(is_64bit_mm(current->mm))) { 1282 current_save_fsgs(); 1283 fs_sel = current->thread.fsindex; 1284 gs_sel = current->thread.gsindex; 1285 fs_base = current->thread.fsbase; 1286 vt->msr_host_kernel_gs_base = current->thread.gsbase; 1287 } else { 1288 savesegment(fs, fs_sel); 1289 savesegment(gs, gs_sel); 1290 fs_base = read_msr(MSR_FS_BASE); 1291 vt->msr_host_kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE); 1292 } 1293 1294 wrmsrq(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); 1295 #else 1296 savesegment(fs, fs_sel); 1297 savesegment(gs, gs_sel); 1298 fs_base = segment_base(fs_sel); 1299 gs_base = segment_base(gs_sel); 1300 #endif 1301 1302 vmx_set_host_fs_gs(host_state, fs_sel, gs_sel, fs_base, gs_base); 1303 vt->guest_state_loaded = true; 1304 } 1305 1306 static void vmx_prepare_switch_to_host(struct vcpu_vmx *vmx) 1307 { 1308 struct vmcs_host_state *host_state; 1309 1310 if (!vmx->vt.guest_state_loaded) 1311 return; 1312 1313 host_state = &vmx->loaded_vmcs->host_state; 1314 1315 ++vmx->vcpu.stat.host_state_reload; 1316 1317 #ifdef CONFIG_X86_64 1318 rdmsrq(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); 1319 #endif 1320 if (host_state->ldt_sel || (host_state->gs_sel & 7)) { 1321 kvm_load_ldt(host_state->ldt_sel); 1322 #ifdef CONFIG_X86_64 1323 load_gs_index(host_state->gs_sel); 1324 #else 1325 loadsegment(gs, host_state->gs_sel); 1326 #endif 1327 } 1328 if (host_state->fs_sel & 7) 1329 loadsegment(fs, host_state->fs_sel); 1330 #ifdef CONFIG_X86_64 1331 if (unlikely(host_state->ds_sel | host_state->es_sel)) { 1332 loadsegment(ds, host_state->ds_sel); 1333 loadsegment(es, host_state->es_sel); 1334 } 1335 #endif 1336 invalidate_tss_limit(); 1337 #ifdef CONFIG_X86_64 1338 wrmsrq(MSR_KERNEL_GS_BASE, vmx->vt.msr_host_kernel_gs_base); 1339 #endif 1340 load_fixmap_gdt(raw_smp_processor_id()); 1341 vmx->vt.guest_state_loaded = false; 1342 vmx->guest_uret_msrs_loaded = false; 1343 } 1344 1345 #ifdef CONFIG_X86_64 1346 static u64 vmx_read_guest_host_msr(struct vcpu_vmx *vmx, u32 msr, u64 *cache) 1347 { 1348 preempt_disable(); 1349 if (vmx->vt.guest_state_loaded) 1350 *cache = read_msr(msr); 1351 preempt_enable(); 1352 return *cache; 1353 } 1354 1355 static void vmx_write_guest_host_msr(struct vcpu_vmx *vmx, u32 msr, u64 data, 1356 u64 *cache) 1357 { 1358 preempt_disable(); 1359 if (vmx->vt.guest_state_loaded) 1360 wrmsrns(msr, data); 1361 preempt_enable(); 1362 *cache = data; 1363 } 1364 1365 static u64 vmx_read_guest_kernel_gs_base(struct vcpu_vmx *vmx) 1366 { 1367 return vmx_read_guest_host_msr(vmx, MSR_KERNEL_GS_BASE, 1368 &vmx->msr_guest_kernel_gs_base); 1369 } 1370 1371 static void vmx_write_guest_kernel_gs_base(struct vcpu_vmx *vmx, u64 data) 1372 { 1373 vmx_write_guest_host_msr(vmx, MSR_KERNEL_GS_BASE, data, 1374 &vmx->msr_guest_kernel_gs_base); 1375 } 1376 #endif 1377 1378 static void grow_ple_window(struct kvm_vcpu *vcpu) 1379 { 1380 struct vcpu_vmx *vmx = to_vmx(vcpu); 1381 unsigned int old = vmx->ple_window; 1382 1383 vmx->ple_window = __grow_ple_window(old, ple_window, 1384 ple_window_grow, 1385 ple_window_max); 1386 1387 if (vmx->ple_window != old) { 1388 vmx->ple_window_dirty = true; 1389 trace_kvm_ple_window_update(vcpu->vcpu_id, 1390 vmx->ple_window, old); 1391 } 1392 } 1393 1394 static void shrink_ple_window(struct kvm_vcpu *vcpu) 1395 { 1396 struct vcpu_vmx *vmx = to_vmx(vcpu); 1397 unsigned int old = vmx->ple_window; 1398 1399 vmx->ple_window = __shrink_ple_window(old, ple_window, 1400 ple_window_shrink, 1401 ple_window); 1402 1403 if (vmx->ple_window != old) { 1404 vmx->ple_window_dirty = true; 1405 trace_kvm_ple_window_update(vcpu->vcpu_id, 1406 vmx->ple_window, old); 1407 } 1408 } 1409 1410 void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu) 1411 { 1412 struct vcpu_vmx *vmx = to_vmx(vcpu); 1413 bool already_loaded = vmx->loaded_vmcs->cpu == cpu; 1414 struct vmcs *prev; 1415 1416 if (!already_loaded) { 1417 loaded_vmcs_clear(vmx->loaded_vmcs); 1418 local_irq_disable(); 1419 1420 /* 1421 * Ensure loaded_vmcs->cpu is read before adding loaded_vmcs to 1422 * this cpu's percpu list, otherwise it may not yet be deleted 1423 * from its previous cpu's percpu list. Pairs with the 1424 * smb_wmb() in __loaded_vmcs_clear(). 1425 */ 1426 smp_rmb(); 1427 1428 list_add(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link, 1429 &per_cpu(loaded_vmcss_on_cpu, cpu)); 1430 local_irq_enable(); 1431 } 1432 1433 prev = per_cpu(current_vmcs, cpu); 1434 if (prev != vmx->loaded_vmcs->vmcs) { 1435 per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs; 1436 vmcs_load(vmx->loaded_vmcs->vmcs); 1437 } 1438 1439 if (!already_loaded) { 1440 void *gdt = get_current_gdt_ro(); 1441 1442 /* 1443 * Flush all EPTP/VPID contexts, the new pCPU may have stale 1444 * TLB entries from its previous association with the vCPU. 1445 */ 1446 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 1447 1448 /* 1449 * Linux uses per-cpu TSS and GDT, so set these when switching 1450 * processors. See 22.2.4. 1451 */ 1452 vmcs_writel(HOST_TR_BASE, 1453 (unsigned long)&get_cpu_entry_area(cpu)->tss.x86_tss); 1454 vmcs_writel(HOST_GDTR_BASE, (unsigned long)gdt); /* 22.2.4 */ 1455 1456 if (IS_ENABLED(CONFIG_IA32_EMULATION) || IS_ENABLED(CONFIG_X86_32)) { 1457 /* 22.2.3 */ 1458 vmcs_writel(HOST_IA32_SYSENTER_ESP, 1459 (unsigned long)(cpu_entry_stack(cpu) + 1)); 1460 } 1461 1462 vmx->loaded_vmcs->cpu = cpu; 1463 } 1464 } 1465 1466 /* 1467 * Switches to specified vcpu, until a matching vcpu_put(), but assumes 1468 * vcpu mutex is already taken. 1469 */ 1470 void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1471 { 1472 if (vcpu->scheduled_out && !kvm_pause_in_guest(vcpu->kvm)) 1473 shrink_ple_window(vcpu); 1474 1475 vmx_vcpu_load_vmcs(vcpu, cpu); 1476 1477 vmx_vcpu_pi_load(vcpu, cpu); 1478 } 1479 1480 void vmx_vcpu_put(struct kvm_vcpu *vcpu) 1481 { 1482 vmx_vcpu_pi_put(vcpu); 1483 1484 vmx_prepare_switch_to_host(to_vmx(vcpu)); 1485 } 1486 1487 bool vmx_emulation_required(struct kvm_vcpu *vcpu) 1488 { 1489 return emulate_invalid_guest_state && !vmx_guest_state_valid(vcpu); 1490 } 1491 1492 unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu) 1493 { 1494 struct vcpu_vmx *vmx = to_vmx(vcpu); 1495 unsigned long rflags, save_rflags; 1496 1497 if (!kvm_register_is_available(vcpu, VCPU_EXREG_RFLAGS)) { 1498 kvm_register_mark_available(vcpu, VCPU_EXREG_RFLAGS); 1499 rflags = vmcs_readl(GUEST_RFLAGS); 1500 if (vmx->rmode.vm86_active) { 1501 rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS; 1502 save_rflags = vmx->rmode.save_rflags; 1503 rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS; 1504 } 1505 vmx->rflags = rflags; 1506 } 1507 return vmx->rflags; 1508 } 1509 1510 void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) 1511 { 1512 struct vcpu_vmx *vmx = to_vmx(vcpu); 1513 unsigned long old_rflags; 1514 1515 /* 1516 * Unlike CR0 and CR4, RFLAGS handling requires checking if the vCPU 1517 * is an unrestricted guest in order to mark L2 as needing emulation 1518 * if L1 runs L2 as a restricted guest. 1519 */ 1520 if (is_unrestricted_guest(vcpu)) { 1521 kvm_register_mark_available(vcpu, VCPU_EXREG_RFLAGS); 1522 vmx->rflags = rflags; 1523 vmcs_writel(GUEST_RFLAGS, rflags); 1524 return; 1525 } 1526 1527 old_rflags = vmx_get_rflags(vcpu); 1528 vmx->rflags = rflags; 1529 if (vmx->rmode.vm86_active) { 1530 vmx->rmode.save_rflags = rflags; 1531 rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM; 1532 } 1533 vmcs_writel(GUEST_RFLAGS, rflags); 1534 1535 if ((old_rflags ^ vmx->rflags) & X86_EFLAGS_VM) 1536 vmx->vt.emulation_required = vmx_emulation_required(vcpu); 1537 } 1538 1539 bool vmx_get_if_flag(struct kvm_vcpu *vcpu) 1540 { 1541 return vmx_get_rflags(vcpu) & X86_EFLAGS_IF; 1542 } 1543 1544 u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu) 1545 { 1546 u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 1547 int ret = 0; 1548 1549 if (interruptibility & GUEST_INTR_STATE_STI) 1550 ret |= KVM_X86_SHADOW_INT_STI; 1551 if (interruptibility & GUEST_INTR_STATE_MOV_SS) 1552 ret |= KVM_X86_SHADOW_INT_MOV_SS; 1553 1554 return ret; 1555 } 1556 1557 void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask) 1558 { 1559 u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 1560 u32 interruptibility = interruptibility_old; 1561 1562 interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS); 1563 1564 if (mask & KVM_X86_SHADOW_INT_MOV_SS) 1565 interruptibility |= GUEST_INTR_STATE_MOV_SS; 1566 else if (mask & KVM_X86_SHADOW_INT_STI) 1567 interruptibility |= GUEST_INTR_STATE_STI; 1568 1569 if ((interruptibility != interruptibility_old)) 1570 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility); 1571 } 1572 1573 static int vmx_rtit_ctl_check(struct kvm_vcpu *vcpu, u64 data) 1574 { 1575 struct vcpu_vmx *vmx = to_vmx(vcpu); 1576 unsigned long value; 1577 1578 /* 1579 * Any MSR write that attempts to change bits marked reserved will 1580 * case a #GP fault. 1581 */ 1582 if (data & vmx->pt_desc.ctl_bitmask) 1583 return 1; 1584 1585 /* 1586 * Any attempt to modify IA32_RTIT_CTL while TraceEn is set will 1587 * result in a #GP unless the same write also clears TraceEn. 1588 */ 1589 if ((vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) && 1590 (data & RTIT_CTL_TRACEEN) && 1591 data != vmx->pt_desc.guest.ctl) 1592 return 1; 1593 1594 /* 1595 * WRMSR to IA32_RTIT_CTL that sets TraceEn but clears this bit 1596 * and FabricEn would cause #GP, if 1597 * CPUID.(EAX=14H, ECX=0):ECX.SNGLRGNOUT[bit 2] = 0 1598 */ 1599 if ((data & RTIT_CTL_TRACEEN) && !(data & RTIT_CTL_TOPA) && 1600 !(data & RTIT_CTL_FABRIC_EN) && 1601 !intel_pt_validate_cap(vmx->pt_desc.caps, 1602 PT_CAP_single_range_output)) 1603 return 1; 1604 1605 /* 1606 * MTCFreq, CycThresh and PSBFreq encodings check, any MSR write that 1607 * utilize encodings marked reserved will cause a #GP fault. 1608 */ 1609 value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc_periods); 1610 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc) && 1611 !test_bit((data & RTIT_CTL_MTC_RANGE) >> 1612 RTIT_CTL_MTC_RANGE_OFFSET, &value)) 1613 return 1; 1614 value = intel_pt_validate_cap(vmx->pt_desc.caps, 1615 PT_CAP_cycle_thresholds); 1616 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) && 1617 !test_bit((data & RTIT_CTL_CYC_THRESH) >> 1618 RTIT_CTL_CYC_THRESH_OFFSET, &value)) 1619 return 1; 1620 value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_periods); 1621 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) && 1622 !test_bit((data & RTIT_CTL_PSB_FREQ) >> 1623 RTIT_CTL_PSB_FREQ_OFFSET, &value)) 1624 return 1; 1625 1626 /* 1627 * If ADDRx_CFG is reserved or the encodings is >2 will 1628 * cause a #GP fault. 1629 */ 1630 value = (data & RTIT_CTL_ADDR0) >> RTIT_CTL_ADDR0_OFFSET; 1631 if ((value && (vmx->pt_desc.num_address_ranges < 1)) || (value > 2)) 1632 return 1; 1633 value = (data & RTIT_CTL_ADDR1) >> RTIT_CTL_ADDR1_OFFSET; 1634 if ((value && (vmx->pt_desc.num_address_ranges < 2)) || (value > 2)) 1635 return 1; 1636 value = (data & RTIT_CTL_ADDR2) >> RTIT_CTL_ADDR2_OFFSET; 1637 if ((value && (vmx->pt_desc.num_address_ranges < 3)) || (value > 2)) 1638 return 1; 1639 value = (data & RTIT_CTL_ADDR3) >> RTIT_CTL_ADDR3_OFFSET; 1640 if ((value && (vmx->pt_desc.num_address_ranges < 4)) || (value > 2)) 1641 return 1; 1642 1643 return 0; 1644 } 1645 1646 int vmx_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, 1647 void *insn, int insn_len) 1648 { 1649 /* 1650 * Emulation of instructions in SGX enclaves is impossible as RIP does 1651 * not point at the failing instruction, and even if it did, the code 1652 * stream is inaccessible. Inject #UD instead of exiting to userspace 1653 * so that guest userspace can't DoS the guest simply by triggering 1654 * emulation (enclaves are CPL3 only). 1655 */ 1656 if (vmx_get_exit_reason(vcpu).enclave_mode) { 1657 kvm_queue_exception(vcpu, UD_VECTOR); 1658 return X86EMUL_PROPAGATE_FAULT; 1659 } 1660 1661 /* Check that emulation is possible during event vectoring */ 1662 if ((to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) && 1663 !kvm_can_emulate_event_vectoring(emul_type)) 1664 return X86EMUL_UNHANDLEABLE_VECTORING; 1665 1666 return X86EMUL_CONTINUE; 1667 } 1668 1669 static int skip_emulated_instruction(struct kvm_vcpu *vcpu) 1670 { 1671 union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu); 1672 unsigned long rip, orig_rip; 1673 u32 instr_len; 1674 1675 /* 1676 * Using VMCS.VM_EXIT_INSTRUCTION_LEN on EPT misconfig depends on 1677 * undefined behavior: Intel's SDM doesn't mandate the VMCS field be 1678 * set when EPT misconfig occurs. In practice, real hardware updates 1679 * VM_EXIT_INSTRUCTION_LEN on EPT misconfig, but other hypervisors 1680 * (namely Hyper-V) don't set it due to it being undefined behavior, 1681 * i.e. we end up advancing IP with some random value. 1682 */ 1683 if (!static_cpu_has(X86_FEATURE_HYPERVISOR) || 1684 exit_reason.basic != EXIT_REASON_EPT_MISCONFIG) { 1685 instr_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 1686 1687 /* 1688 * Emulating an enclave's instructions isn't supported as KVM 1689 * cannot access the enclave's memory or its true RIP, e.g. the 1690 * vmcs.GUEST_RIP points at the exit point of the enclave, not 1691 * the RIP that actually triggered the VM-Exit. But, because 1692 * most instructions that cause VM-Exit will #UD in an enclave, 1693 * most instruction-based VM-Exits simply do not occur. 1694 * 1695 * There are a few exceptions, notably the debug instructions 1696 * INT1ICEBRK and INT3, as they are allowed in debug enclaves 1697 * and generate #DB/#BP as expected, which KVM might intercept. 1698 * But again, the CPU does the dirty work and saves an instr 1699 * length of zero so VMMs don't shoot themselves in the foot. 1700 * WARN if KVM tries to skip a non-zero length instruction on 1701 * a VM-Exit from an enclave. 1702 */ 1703 if (!instr_len) 1704 goto rip_updated; 1705 1706 WARN_ONCE(exit_reason.enclave_mode, 1707 "skipping instruction after SGX enclave VM-Exit"); 1708 1709 orig_rip = kvm_rip_read(vcpu); 1710 rip = orig_rip + instr_len; 1711 #ifdef CONFIG_X86_64 1712 /* 1713 * We need to mask out the high 32 bits of RIP if not in 64-bit 1714 * mode, but just finding out that we are in 64-bit mode is 1715 * quite expensive. Only do it if there was a carry. 1716 */ 1717 if (unlikely(((rip ^ orig_rip) >> 31) == 3) && !is_64_bit_mode(vcpu)) 1718 rip = (u32)rip; 1719 #endif 1720 kvm_rip_write(vcpu, rip); 1721 } else { 1722 if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP)) 1723 return 0; 1724 } 1725 1726 rip_updated: 1727 /* skipping an emulated instruction also counts */ 1728 vmx_set_interrupt_shadow(vcpu, 0); 1729 1730 return 1; 1731 } 1732 1733 /* 1734 * Recognizes a pending MTF VM-exit and records the nested state for later 1735 * delivery. 1736 */ 1737 void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu) 1738 { 1739 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1740 struct vcpu_vmx *vmx = to_vmx(vcpu); 1741 1742 if (!is_guest_mode(vcpu)) 1743 return; 1744 1745 /* 1746 * Per the SDM, MTF takes priority over debug-trap exceptions besides 1747 * TSS T-bit traps and ICEBP (INT1). KVM doesn't emulate T-bit traps 1748 * or ICEBP (in the emulator proper), and skipping of ICEBP after an 1749 * intercepted #DB deliberately avoids single-step #DB and MTF updates 1750 * as ICEBP is higher priority than both. As instruction emulation is 1751 * completed at this point (i.e. KVM is at the instruction boundary), 1752 * any #DB exception pending delivery must be a debug-trap of lower 1753 * priority than MTF. Record the pending MTF state to be delivered in 1754 * vmx_check_nested_events(). 1755 */ 1756 if (nested_cpu_has_mtf(vmcs12) && 1757 (!vcpu->arch.exception.pending || 1758 vcpu->arch.exception.vector == DB_VECTOR) && 1759 (!vcpu->arch.exception_vmexit.pending || 1760 vcpu->arch.exception_vmexit.vector == DB_VECTOR)) { 1761 vmx->nested.mtf_pending = true; 1762 kvm_make_request(KVM_REQ_EVENT, vcpu); 1763 } else { 1764 vmx->nested.mtf_pending = false; 1765 } 1766 } 1767 1768 int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu) 1769 { 1770 vmx_update_emulated_instruction(vcpu); 1771 return skip_emulated_instruction(vcpu); 1772 } 1773 1774 static void vmx_clear_hlt(struct kvm_vcpu *vcpu) 1775 { 1776 /* 1777 * Ensure that we clear the HLT state in the VMCS. We don't need to 1778 * explicitly skip the instruction because if the HLT state is set, 1779 * then the instruction is already executing and RIP has already been 1780 * advanced. 1781 */ 1782 if (kvm_hlt_in_guest(vcpu->kvm) && 1783 vmcs_read32(GUEST_ACTIVITY_STATE) == GUEST_ACTIVITY_HLT) 1784 vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE); 1785 } 1786 1787 void vmx_inject_exception(struct kvm_vcpu *vcpu) 1788 { 1789 struct kvm_queued_exception *ex = &vcpu->arch.exception; 1790 u32 intr_info = ex->vector | INTR_INFO_VALID_MASK; 1791 struct vcpu_vmx *vmx = to_vmx(vcpu); 1792 1793 kvm_deliver_exception_payload(vcpu, ex); 1794 1795 if (ex->has_error_code) { 1796 /* 1797 * Despite the error code being architecturally defined as 32 1798 * bits, and the VMCS field being 32 bits, Intel CPUs and thus 1799 * VMX don't actually supporting setting bits 31:16. Hardware 1800 * will (should) never provide a bogus error code, but AMD CPUs 1801 * do generate error codes with bits 31:16 set, and so KVM's 1802 * ABI lets userspace shove in arbitrary 32-bit values. Drop 1803 * the upper bits to avoid VM-Fail, losing information that 1804 * doesn't really exist is preferable to killing the VM. 1805 */ 1806 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, (u16)ex->error_code); 1807 intr_info |= INTR_INFO_DELIVER_CODE_MASK; 1808 } 1809 1810 if (vmx->rmode.vm86_active) { 1811 int inc_eip = 0; 1812 if (kvm_exception_is_soft(ex->vector)) 1813 inc_eip = vcpu->arch.event_exit_inst_len; 1814 kvm_inject_realmode_interrupt(vcpu, ex->vector, inc_eip); 1815 return; 1816 } 1817 1818 WARN_ON_ONCE(vmx->vt.emulation_required); 1819 1820 if (kvm_exception_is_soft(ex->vector)) { 1821 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1822 vmx->vcpu.arch.event_exit_inst_len); 1823 intr_info |= INTR_TYPE_SOFT_EXCEPTION; 1824 } else 1825 intr_info |= INTR_TYPE_HARD_EXCEPTION; 1826 1827 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info); 1828 1829 vmx_clear_hlt(vcpu); 1830 } 1831 1832 static void vmx_setup_uret_msr(struct vcpu_vmx *vmx, unsigned int msr, 1833 bool load_into_hardware) 1834 { 1835 struct vmx_uret_msr *uret_msr; 1836 1837 uret_msr = vmx_find_uret_msr(vmx, msr); 1838 if (!uret_msr) 1839 return; 1840 1841 uret_msr->load_into_hardware = load_into_hardware; 1842 } 1843 1844 /* 1845 * Configuring user return MSRs to automatically save, load, and restore MSRs 1846 * that need to be shoved into hardware when running the guest. Note, omitting 1847 * an MSR here does _NOT_ mean it's not emulated, only that it will not be 1848 * loaded into hardware when running the guest. 1849 */ 1850 static void vmx_setup_uret_msrs(struct vcpu_vmx *vmx) 1851 { 1852 #ifdef CONFIG_X86_64 1853 bool load_syscall_msrs; 1854 1855 /* 1856 * The SYSCALL MSRs are only needed on long mode guests, and only 1857 * when EFER.SCE is set. 1858 */ 1859 load_syscall_msrs = is_long_mode(&vmx->vcpu) && 1860 (vmx->vcpu.arch.efer & EFER_SCE); 1861 1862 vmx_setup_uret_msr(vmx, MSR_STAR, load_syscall_msrs); 1863 vmx_setup_uret_msr(vmx, MSR_LSTAR, load_syscall_msrs); 1864 vmx_setup_uret_msr(vmx, MSR_SYSCALL_MASK, load_syscall_msrs); 1865 #endif 1866 vmx_setup_uret_msr(vmx, MSR_EFER, update_transition_efer(vmx)); 1867 1868 vmx_setup_uret_msr(vmx, MSR_TSC_AUX, 1869 guest_cpu_cap_has(&vmx->vcpu, X86_FEATURE_RDTSCP) || 1870 guest_cpu_cap_has(&vmx->vcpu, X86_FEATURE_RDPID)); 1871 1872 /* 1873 * hle=0, rtm=0, tsx_ctrl=1 can be found with some combinations of new 1874 * kernel and old userspace. If those guests run on a tsx=off host, do 1875 * allow guests to use TSX_CTRL, but don't change the value in hardware 1876 * so that TSX remains always disabled. 1877 */ 1878 vmx_setup_uret_msr(vmx, MSR_IA32_TSX_CTRL, boot_cpu_has(X86_FEATURE_RTM)); 1879 1880 /* 1881 * The set of MSRs to load may have changed, reload MSRs before the 1882 * next VM-Enter. 1883 */ 1884 vmx->guest_uret_msrs_loaded = false; 1885 } 1886 1887 u64 vmx_get_l2_tsc_offset(struct kvm_vcpu *vcpu) 1888 { 1889 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1890 1891 if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING)) 1892 return vmcs12->tsc_offset; 1893 1894 return 0; 1895 } 1896 1897 u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu) 1898 { 1899 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1900 1901 if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING) && 1902 nested_cpu_has2(vmcs12, SECONDARY_EXEC_TSC_SCALING)) 1903 return vmcs12->tsc_multiplier; 1904 1905 return kvm_caps.default_tsc_scaling_ratio; 1906 } 1907 1908 void vmx_write_tsc_offset(struct kvm_vcpu *vcpu) 1909 { 1910 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 1911 } 1912 1913 void vmx_write_tsc_multiplier(struct kvm_vcpu *vcpu) 1914 { 1915 vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio); 1916 } 1917 1918 /* 1919 * Userspace is allowed to set any supported IA32_FEATURE_CONTROL regardless of 1920 * guest CPUID. Note, KVM allows userspace to set "VMX in SMX" to maintain 1921 * backwards compatibility even though KVM doesn't support emulating SMX. And 1922 * because userspace set "VMX in SMX", the guest must also be allowed to set it, 1923 * e.g. if the MSR is left unlocked and the guest does a RMW operation. 1924 */ 1925 #define KVM_SUPPORTED_FEATURE_CONTROL (FEAT_CTL_LOCKED | \ 1926 FEAT_CTL_VMX_ENABLED_INSIDE_SMX | \ 1927 FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX | \ 1928 FEAT_CTL_SGX_LC_ENABLED | \ 1929 FEAT_CTL_SGX_ENABLED | \ 1930 FEAT_CTL_LMCE_ENABLED) 1931 1932 static inline bool is_vmx_feature_control_msr_valid(struct vcpu_vmx *vmx, 1933 struct msr_data *msr) 1934 { 1935 uint64_t valid_bits; 1936 1937 /* 1938 * Ensure KVM_SUPPORTED_FEATURE_CONTROL is updated when new bits are 1939 * exposed to the guest. 1940 */ 1941 WARN_ON_ONCE(vmx->msr_ia32_feature_control_valid_bits & 1942 ~KVM_SUPPORTED_FEATURE_CONTROL); 1943 1944 if (!msr->host_initiated && 1945 (vmx->msr_ia32_feature_control & FEAT_CTL_LOCKED)) 1946 return false; 1947 1948 if (msr->host_initiated) 1949 valid_bits = KVM_SUPPORTED_FEATURE_CONTROL; 1950 else 1951 valid_bits = vmx->msr_ia32_feature_control_valid_bits; 1952 1953 return !(msr->data & ~valid_bits); 1954 } 1955 1956 int vmx_get_feature_msr(u32 msr, u64 *data) 1957 { 1958 switch (msr) { 1959 case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR: 1960 if (!nested) 1961 return 1; 1962 return vmx_get_vmx_msr(&vmcs_config.nested, msr, data); 1963 default: 1964 return KVM_MSR_RET_UNSUPPORTED; 1965 } 1966 } 1967 1968 /* 1969 * Reads an msr value (of 'msr_info->index') into 'msr_info->data'. 1970 * Returns 0 on success, non-0 otherwise. 1971 * Assumes vcpu_load() was already called. 1972 */ 1973 int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 1974 { 1975 struct vcpu_vmx *vmx = to_vmx(vcpu); 1976 struct vmx_uret_msr *msr; 1977 u32 index; 1978 1979 switch (msr_info->index) { 1980 #ifdef CONFIG_X86_64 1981 case MSR_FS_BASE: 1982 msr_info->data = vmcs_readl(GUEST_FS_BASE); 1983 break; 1984 case MSR_GS_BASE: 1985 msr_info->data = vmcs_readl(GUEST_GS_BASE); 1986 break; 1987 case MSR_KERNEL_GS_BASE: 1988 msr_info->data = vmx_read_guest_kernel_gs_base(vmx); 1989 break; 1990 #endif 1991 case MSR_EFER: 1992 return kvm_get_msr_common(vcpu, msr_info); 1993 case MSR_IA32_TSX_CTRL: 1994 if (!msr_info->host_initiated && 1995 !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR)) 1996 return 1; 1997 goto find_uret_msr; 1998 case MSR_IA32_UMWAIT_CONTROL: 1999 if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx)) 2000 return 1; 2001 2002 msr_info->data = vmx->msr_ia32_umwait_control; 2003 break; 2004 case MSR_IA32_SPEC_CTRL: 2005 if (!msr_info->host_initiated && 2006 !guest_has_spec_ctrl_msr(vcpu)) 2007 return 1; 2008 2009 msr_info->data = to_vmx(vcpu)->spec_ctrl; 2010 break; 2011 case MSR_IA32_SYSENTER_CS: 2012 msr_info->data = vmcs_read32(GUEST_SYSENTER_CS); 2013 break; 2014 case MSR_IA32_SYSENTER_EIP: 2015 msr_info->data = vmcs_readl(GUEST_SYSENTER_EIP); 2016 break; 2017 case MSR_IA32_SYSENTER_ESP: 2018 msr_info->data = vmcs_readl(GUEST_SYSENTER_ESP); 2019 break; 2020 case MSR_IA32_BNDCFGS: 2021 if (!kvm_mpx_supported() || 2022 (!msr_info->host_initiated && 2023 !guest_cpu_cap_has(vcpu, X86_FEATURE_MPX))) 2024 return 1; 2025 msr_info->data = vmcs_read64(GUEST_BNDCFGS); 2026 break; 2027 case MSR_IA32_MCG_EXT_CTL: 2028 if (!msr_info->host_initiated && 2029 !(vmx->msr_ia32_feature_control & 2030 FEAT_CTL_LMCE_ENABLED)) 2031 return 1; 2032 msr_info->data = vcpu->arch.mcg_ext_ctl; 2033 break; 2034 case MSR_IA32_FEAT_CTL: 2035 msr_info->data = vmx->msr_ia32_feature_control; 2036 break; 2037 case MSR_IA32_SGXLEPUBKEYHASH0 ... MSR_IA32_SGXLEPUBKEYHASH3: 2038 if (!msr_info->host_initiated && 2039 !guest_cpu_cap_has(vcpu, X86_FEATURE_SGX_LC)) 2040 return 1; 2041 msr_info->data = to_vmx(vcpu)->msr_ia32_sgxlepubkeyhash 2042 [msr_info->index - MSR_IA32_SGXLEPUBKEYHASH0]; 2043 break; 2044 case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR: 2045 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_VMX)) 2046 return 1; 2047 if (vmx_get_vmx_msr(&vmx->nested.msrs, msr_info->index, 2048 &msr_info->data)) 2049 return 1; 2050 #ifdef CONFIG_KVM_HYPERV 2051 /* 2052 * Enlightened VMCS v1 doesn't have certain VMCS fields but 2053 * instead of just ignoring the features, different Hyper-V 2054 * versions are either trying to use them and fail or do some 2055 * sanity checking and refuse to boot. Filter all unsupported 2056 * features out. 2057 */ 2058 if (!msr_info->host_initiated && guest_cpu_cap_has_evmcs(vcpu)) 2059 nested_evmcs_filter_control_msr(vcpu, msr_info->index, 2060 &msr_info->data); 2061 #endif 2062 break; 2063 case MSR_IA32_RTIT_CTL: 2064 if (!vmx_pt_mode_is_host_guest()) 2065 return 1; 2066 msr_info->data = vmx->pt_desc.guest.ctl; 2067 break; 2068 case MSR_IA32_RTIT_STATUS: 2069 if (!vmx_pt_mode_is_host_guest()) 2070 return 1; 2071 msr_info->data = vmx->pt_desc.guest.status; 2072 break; 2073 case MSR_IA32_RTIT_CR3_MATCH: 2074 if (!vmx_pt_mode_is_host_guest() || 2075 !intel_pt_validate_cap(vmx->pt_desc.caps, 2076 PT_CAP_cr3_filtering)) 2077 return 1; 2078 msr_info->data = vmx->pt_desc.guest.cr3_match; 2079 break; 2080 case MSR_IA32_RTIT_OUTPUT_BASE: 2081 if (!vmx_pt_mode_is_host_guest() || 2082 (!intel_pt_validate_cap(vmx->pt_desc.caps, 2083 PT_CAP_topa_output) && 2084 !intel_pt_validate_cap(vmx->pt_desc.caps, 2085 PT_CAP_single_range_output))) 2086 return 1; 2087 msr_info->data = vmx->pt_desc.guest.output_base; 2088 break; 2089 case MSR_IA32_RTIT_OUTPUT_MASK: 2090 if (!vmx_pt_mode_is_host_guest() || 2091 (!intel_pt_validate_cap(vmx->pt_desc.caps, 2092 PT_CAP_topa_output) && 2093 !intel_pt_validate_cap(vmx->pt_desc.caps, 2094 PT_CAP_single_range_output))) 2095 return 1; 2096 msr_info->data = vmx->pt_desc.guest.output_mask; 2097 break; 2098 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: 2099 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A; 2100 if (!vmx_pt_mode_is_host_guest() || 2101 (index >= 2 * vmx->pt_desc.num_address_ranges)) 2102 return 1; 2103 if (index % 2) 2104 msr_info->data = vmx->pt_desc.guest.addr_b[index / 2]; 2105 else 2106 msr_info->data = vmx->pt_desc.guest.addr_a[index / 2]; 2107 break; 2108 case MSR_IA32_S_CET: 2109 msr_info->data = vmcs_readl(GUEST_S_CET); 2110 break; 2111 case MSR_KVM_INTERNAL_GUEST_SSP: 2112 msr_info->data = vmcs_readl(GUEST_SSP); 2113 break; 2114 case MSR_IA32_INT_SSP_TAB: 2115 msr_info->data = vmcs_readl(GUEST_INTR_SSP_TABLE); 2116 break; 2117 case MSR_IA32_DEBUGCTLMSR: 2118 msr_info->data = vmx_guest_debugctl_read(); 2119 break; 2120 default: 2121 find_uret_msr: 2122 msr = vmx_find_uret_msr(vmx, msr_info->index); 2123 if (msr) { 2124 msr_info->data = msr->data; 2125 break; 2126 } 2127 return kvm_get_msr_common(vcpu, msr_info); 2128 } 2129 2130 return 0; 2131 } 2132 2133 static u64 nested_vmx_truncate_sysenter_addr(struct kvm_vcpu *vcpu, 2134 u64 data) 2135 { 2136 #ifdef CONFIG_X86_64 2137 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_LM)) 2138 return (u32)data; 2139 #endif 2140 return (unsigned long)data; 2141 } 2142 2143 u64 vmx_get_supported_debugctl(struct kvm_vcpu *vcpu, bool host_initiated) 2144 { 2145 u64 debugctl = 0; 2146 2147 if (boot_cpu_has(X86_FEATURE_BUS_LOCK_DETECT) && 2148 (host_initiated || guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT))) 2149 debugctl |= DEBUGCTLMSR_BUS_LOCK_DETECT; 2150 2151 if ((kvm_caps.supported_perf_cap & PERF_CAP_LBR_FMT) && 2152 (host_initiated || intel_pmu_lbr_is_enabled(vcpu))) 2153 debugctl |= DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI; 2154 2155 if (boot_cpu_has(X86_FEATURE_RTM) && 2156 (host_initiated || guest_cpu_cap_has(vcpu, X86_FEATURE_RTM))) 2157 debugctl |= DEBUGCTLMSR_RTM_DEBUG; 2158 2159 return debugctl; 2160 } 2161 2162 bool vmx_is_valid_debugctl(struct kvm_vcpu *vcpu, u64 data, bool host_initiated) 2163 { 2164 u64 invalid; 2165 2166 invalid = data & ~vmx_get_supported_debugctl(vcpu, host_initiated); 2167 if (invalid & (DEBUGCTLMSR_BTF | DEBUGCTLMSR_LBR)) { 2168 kvm_pr_unimpl_wrmsr(vcpu, MSR_IA32_DEBUGCTLMSR, data); 2169 invalid &= ~(DEBUGCTLMSR_BTF | DEBUGCTLMSR_LBR); 2170 } 2171 return !invalid; 2172 } 2173 2174 /* 2175 * Writes msr value into the appropriate "register". 2176 * Returns 0 on success, non-0 otherwise. 2177 * Assumes vcpu_load() was already called. 2178 */ 2179 int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 2180 { 2181 struct vcpu_vmx *vmx = to_vmx(vcpu); 2182 struct vmx_uret_msr *msr; 2183 int ret = 0; 2184 u32 msr_index = msr_info->index; 2185 u64 data = msr_info->data; 2186 u32 index; 2187 2188 switch (msr_index) { 2189 case MSR_EFER: 2190 ret = kvm_set_msr_common(vcpu, msr_info); 2191 break; 2192 #ifdef CONFIG_X86_64 2193 case MSR_FS_BASE: 2194 vmx_segment_cache_clear(vmx); 2195 vmcs_writel(GUEST_FS_BASE, data); 2196 break; 2197 case MSR_GS_BASE: 2198 vmx_segment_cache_clear(vmx); 2199 vmcs_writel(GUEST_GS_BASE, data); 2200 break; 2201 case MSR_KERNEL_GS_BASE: 2202 vmx_write_guest_kernel_gs_base(vmx, data); 2203 break; 2204 case MSR_IA32_XFD: 2205 ret = kvm_set_msr_common(vcpu, msr_info); 2206 /* 2207 * Always intercepting WRMSR could incur non-negligible 2208 * overhead given xfd might be changed frequently in 2209 * guest context switch. Disable write interception 2210 * upon the first write with a non-zero value (indicating 2211 * potential usage on dynamic xfeatures). Also update 2212 * exception bitmap to trap #NM for proper virtualization 2213 * of guest xfd_err. 2214 */ 2215 if (!ret && data) { 2216 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_XFD, 2217 MSR_TYPE_RW); 2218 vcpu->arch.xfd_no_write_intercept = true; 2219 vmx_update_exception_bitmap(vcpu); 2220 } 2221 break; 2222 #endif 2223 case MSR_IA32_SYSENTER_CS: 2224 if (is_guest_mode(vcpu)) 2225 get_vmcs12(vcpu)->guest_sysenter_cs = data; 2226 vmcs_write32(GUEST_SYSENTER_CS, data); 2227 break; 2228 case MSR_IA32_SYSENTER_EIP: 2229 if (is_guest_mode(vcpu)) { 2230 data = nested_vmx_truncate_sysenter_addr(vcpu, data); 2231 get_vmcs12(vcpu)->guest_sysenter_eip = data; 2232 } 2233 vmcs_writel(GUEST_SYSENTER_EIP, data); 2234 break; 2235 case MSR_IA32_SYSENTER_ESP: 2236 if (is_guest_mode(vcpu)) { 2237 data = nested_vmx_truncate_sysenter_addr(vcpu, data); 2238 get_vmcs12(vcpu)->guest_sysenter_esp = data; 2239 } 2240 vmcs_writel(GUEST_SYSENTER_ESP, data); 2241 break; 2242 case MSR_IA32_DEBUGCTLMSR: 2243 if (!vmx_is_valid_debugctl(vcpu, data, msr_info->host_initiated)) 2244 return 1; 2245 2246 data &= vmx_get_supported_debugctl(vcpu, msr_info->host_initiated); 2247 2248 if (is_guest_mode(vcpu) && get_vmcs12(vcpu)->vm_exit_controls & 2249 VM_EXIT_SAVE_DEBUG_CONTROLS) 2250 get_vmcs12(vcpu)->guest_ia32_debugctl = data; 2251 2252 vmx_guest_debugctl_write(vcpu, data); 2253 2254 if (intel_pmu_lbr_is_enabled(vcpu) && !to_vmx(vcpu)->lbr_desc.event && 2255 (data & DEBUGCTLMSR_LBR)) 2256 intel_pmu_create_guest_lbr_event(vcpu); 2257 return 0; 2258 case MSR_IA32_BNDCFGS: 2259 if (!kvm_mpx_supported() || 2260 (!msr_info->host_initiated && 2261 !guest_cpu_cap_has(vcpu, X86_FEATURE_MPX))) 2262 return 1; 2263 if (is_noncanonical_msr_address(data & PAGE_MASK, vcpu) || 2264 (data & MSR_IA32_BNDCFGS_RSVD)) 2265 return 1; 2266 2267 if (is_guest_mode(vcpu) && 2268 ((vmx->nested.msrs.entry_ctls_high & VM_ENTRY_LOAD_BNDCFGS) || 2269 (vmx->nested.msrs.exit_ctls_high & VM_EXIT_CLEAR_BNDCFGS))) 2270 get_vmcs12(vcpu)->guest_bndcfgs = data; 2271 2272 vmcs_write64(GUEST_BNDCFGS, data); 2273 break; 2274 case MSR_IA32_UMWAIT_CONTROL: 2275 if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx)) 2276 return 1; 2277 2278 /* The reserved bit 1 and non-32 bit [63:32] should be zero */ 2279 if (data & (BIT_ULL(1) | GENMASK_ULL(63, 32))) 2280 return 1; 2281 2282 vmx->msr_ia32_umwait_control = data; 2283 break; 2284 case MSR_IA32_SPEC_CTRL: 2285 if (!msr_info->host_initiated && 2286 !guest_has_spec_ctrl_msr(vcpu)) 2287 return 1; 2288 2289 if (kvm_spec_ctrl_test_value(data)) 2290 return 1; 2291 2292 vmx->spec_ctrl = data; 2293 if (!data) 2294 break; 2295 2296 /* 2297 * For non-nested: 2298 * When it's written (to non-zero) for the first time, pass 2299 * it through. 2300 * 2301 * For nested: 2302 * The handling of the MSR bitmap for L2 guests is done in 2303 * nested_vmx_prepare_msr_bitmap. We should not touch the 2304 * vmcs02.msr_bitmap here since it gets completely overwritten 2305 * in the merging. We update the vmcs01 here for L1 as well 2306 * since it will end up touching the MSR anyway now. 2307 */ 2308 vmx_disable_intercept_for_msr(vcpu, 2309 MSR_IA32_SPEC_CTRL, 2310 MSR_TYPE_RW); 2311 break; 2312 case MSR_IA32_TSX_CTRL: 2313 if (!msr_info->host_initiated && 2314 !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR)) 2315 return 1; 2316 if (data & ~(TSX_CTRL_RTM_DISABLE | TSX_CTRL_CPUID_CLEAR)) 2317 return 1; 2318 goto find_uret_msr; 2319 case MSR_IA32_CR_PAT: 2320 ret = kvm_set_msr_common(vcpu, msr_info); 2321 if (ret) 2322 break; 2323 2324 if (is_guest_mode(vcpu) && 2325 get_vmcs12(vcpu)->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT) 2326 get_vmcs12(vcpu)->guest_ia32_pat = data; 2327 2328 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) 2329 vmcs_write64(GUEST_IA32_PAT, data); 2330 break; 2331 case MSR_IA32_MCG_EXT_CTL: 2332 if ((!msr_info->host_initiated && 2333 !(to_vmx(vcpu)->msr_ia32_feature_control & 2334 FEAT_CTL_LMCE_ENABLED)) || 2335 (data & ~MCG_EXT_CTL_LMCE_EN)) 2336 return 1; 2337 vcpu->arch.mcg_ext_ctl = data; 2338 break; 2339 case MSR_IA32_FEAT_CTL: 2340 if (!is_vmx_feature_control_msr_valid(vmx, msr_info)) 2341 return 1; 2342 2343 vmx->msr_ia32_feature_control = data; 2344 if (msr_info->host_initiated && data == 0) 2345 vmx_leave_nested(vcpu); 2346 2347 /* SGX may be enabled/disabled by guest's firmware */ 2348 vmx_write_encls_bitmap(vcpu, NULL); 2349 break; 2350 case MSR_IA32_SGXLEPUBKEYHASH0 ... MSR_IA32_SGXLEPUBKEYHASH3: 2351 /* 2352 * On real hardware, the LE hash MSRs are writable before 2353 * the firmware sets bit 0 in MSR 0x7a ("activating" SGX), 2354 * at which point SGX related bits in IA32_FEATURE_CONTROL 2355 * become writable. 2356 * 2357 * KVM does not emulate SGX activation for simplicity, so 2358 * allow writes to the LE hash MSRs if IA32_FEATURE_CONTROL 2359 * is unlocked. This is technically not architectural 2360 * behavior, but it's close enough. 2361 */ 2362 if (!msr_info->host_initiated && 2363 (!guest_cpu_cap_has(vcpu, X86_FEATURE_SGX_LC) || 2364 ((vmx->msr_ia32_feature_control & FEAT_CTL_LOCKED) && 2365 !(vmx->msr_ia32_feature_control & FEAT_CTL_SGX_LC_ENABLED)))) 2366 return 1; 2367 vmx->msr_ia32_sgxlepubkeyhash 2368 [msr_index - MSR_IA32_SGXLEPUBKEYHASH0] = data; 2369 break; 2370 case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR: 2371 if (!msr_info->host_initiated) 2372 return 1; /* they are read-only */ 2373 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_VMX)) 2374 return 1; 2375 return vmx_set_vmx_msr(vcpu, msr_index, data); 2376 case MSR_IA32_RTIT_CTL: 2377 if (!vmx_pt_mode_is_host_guest() || 2378 vmx_rtit_ctl_check(vcpu, data) || 2379 vmx->nested.vmxon) 2380 return 1; 2381 vmcs_write64(GUEST_IA32_RTIT_CTL, data); 2382 vmx->pt_desc.guest.ctl = data; 2383 pt_update_intercept_for_msr(vcpu); 2384 break; 2385 case MSR_IA32_RTIT_STATUS: 2386 if (!pt_can_write_msr(vmx)) 2387 return 1; 2388 if (data & MSR_IA32_RTIT_STATUS_MASK) 2389 return 1; 2390 vmx->pt_desc.guest.status = data; 2391 break; 2392 case MSR_IA32_RTIT_CR3_MATCH: 2393 if (!pt_can_write_msr(vmx)) 2394 return 1; 2395 if (!intel_pt_validate_cap(vmx->pt_desc.caps, 2396 PT_CAP_cr3_filtering)) 2397 return 1; 2398 vmx->pt_desc.guest.cr3_match = data; 2399 break; 2400 case MSR_IA32_RTIT_OUTPUT_BASE: 2401 if (!pt_can_write_msr(vmx)) 2402 return 1; 2403 if (!intel_pt_validate_cap(vmx->pt_desc.caps, 2404 PT_CAP_topa_output) && 2405 !intel_pt_validate_cap(vmx->pt_desc.caps, 2406 PT_CAP_single_range_output)) 2407 return 1; 2408 if (!pt_output_base_valid(vcpu, data)) 2409 return 1; 2410 vmx->pt_desc.guest.output_base = data; 2411 break; 2412 case MSR_IA32_RTIT_OUTPUT_MASK: 2413 if (!pt_can_write_msr(vmx)) 2414 return 1; 2415 if (!intel_pt_validate_cap(vmx->pt_desc.caps, 2416 PT_CAP_topa_output) && 2417 !intel_pt_validate_cap(vmx->pt_desc.caps, 2418 PT_CAP_single_range_output)) 2419 return 1; 2420 vmx->pt_desc.guest.output_mask = data; 2421 break; 2422 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: 2423 if (!pt_can_write_msr(vmx)) 2424 return 1; 2425 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A; 2426 if (index >= 2 * vmx->pt_desc.num_address_ranges) 2427 return 1; 2428 if (is_noncanonical_msr_address(data, vcpu)) 2429 return 1; 2430 if (index % 2) 2431 vmx->pt_desc.guest.addr_b[index / 2] = data; 2432 else 2433 vmx->pt_desc.guest.addr_a[index / 2] = data; 2434 break; 2435 case MSR_IA32_S_CET: 2436 vmcs_writel(GUEST_S_CET, data); 2437 break; 2438 case MSR_KVM_INTERNAL_GUEST_SSP: 2439 vmcs_writel(GUEST_SSP, data); 2440 break; 2441 case MSR_IA32_INT_SSP_TAB: 2442 vmcs_writel(GUEST_INTR_SSP_TABLE, data); 2443 break; 2444 case MSR_IA32_PERF_CAPABILITIES: 2445 if (data & PERF_CAP_LBR_FMT) { 2446 if ((data & PERF_CAP_LBR_FMT) != 2447 (kvm_caps.supported_perf_cap & PERF_CAP_LBR_FMT)) 2448 return 1; 2449 if (!cpuid_model_is_consistent(vcpu)) 2450 return 1; 2451 } 2452 if (data & PERF_CAP_PEBS_FORMAT) { 2453 if ((data & PERF_CAP_PEBS_MASK) != 2454 (kvm_caps.supported_perf_cap & PERF_CAP_PEBS_MASK)) 2455 return 1; 2456 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_DS)) 2457 return 1; 2458 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_DTES64)) 2459 return 1; 2460 if (!cpuid_model_is_consistent(vcpu)) 2461 return 1; 2462 } 2463 ret = kvm_set_msr_common(vcpu, msr_info); 2464 break; 2465 2466 default: 2467 find_uret_msr: 2468 msr = vmx_find_uret_msr(vmx, msr_index); 2469 if (msr) 2470 ret = vmx_set_guest_uret_msr(vmx, msr, data); 2471 else 2472 ret = kvm_set_msr_common(vcpu, msr_info); 2473 } 2474 2475 /* FB_CLEAR may have changed, also update the FB_CLEAR_DIS behavior */ 2476 if (msr_index == MSR_IA32_ARCH_CAPABILITIES) 2477 vmx_update_fb_clear_dis(vcpu, vmx); 2478 2479 return ret; 2480 } 2481 2482 void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) 2483 { 2484 unsigned long guest_owned_bits; 2485 2486 kvm_register_mark_available(vcpu, reg); 2487 2488 switch (reg) { 2489 case VCPU_REGS_RSP: 2490 vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP); 2491 break; 2492 case VCPU_REGS_RIP: 2493 vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP); 2494 break; 2495 case VCPU_EXREG_PDPTR: 2496 if (enable_ept) 2497 ept_save_pdptrs(vcpu); 2498 break; 2499 case VCPU_EXREG_CR0: 2500 guest_owned_bits = vcpu->arch.cr0_guest_owned_bits; 2501 2502 vcpu->arch.cr0 &= ~guest_owned_bits; 2503 vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & guest_owned_bits; 2504 break; 2505 case VCPU_EXREG_CR3: 2506 /* 2507 * When intercepting CR3 loads, e.g. for shadowing paging, KVM's 2508 * CR3 is loaded into hardware, not the guest's CR3. 2509 */ 2510 if (!(exec_controls_get(to_vmx(vcpu)) & CPU_BASED_CR3_LOAD_EXITING)) 2511 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); 2512 break; 2513 case VCPU_EXREG_CR4: 2514 guest_owned_bits = vcpu->arch.cr4_guest_owned_bits; 2515 2516 vcpu->arch.cr4 &= ~guest_owned_bits; 2517 vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & guest_owned_bits; 2518 break; 2519 default: 2520 KVM_BUG_ON(1, vcpu->kvm); 2521 break; 2522 } 2523 } 2524 2525 /* 2526 * There is no X86_FEATURE for SGX yet, but anyway we need to query CPUID 2527 * directly instead of going through cpu_has(), to ensure KVM is trapping 2528 * ENCLS whenever it's supported in hardware. It does not matter whether 2529 * the host OS supports or has enabled SGX. 2530 */ 2531 static bool cpu_has_sgx(void) 2532 { 2533 return cpuid_eax(0) >= 0x12 && (cpuid_eax(0x12) & BIT(0)); 2534 } 2535 2536 static int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt, u32 msr, u32 *result) 2537 { 2538 u32 vmx_msr_low, vmx_msr_high; 2539 u32 ctl = ctl_min | ctl_opt; 2540 2541 rdmsr(msr, vmx_msr_low, vmx_msr_high); 2542 2543 ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */ 2544 ctl |= vmx_msr_low; /* bit == 1 in low word ==> must be one */ 2545 2546 /* Ensure minimum (required) set of control bits are supported. */ 2547 if (ctl_min & ~ctl) 2548 return -EIO; 2549 2550 *result = ctl; 2551 return 0; 2552 } 2553 2554 static u64 adjust_vmx_controls64(u64 ctl_opt, u32 msr) 2555 { 2556 u64 allowed; 2557 2558 rdmsrq(msr, allowed); 2559 2560 return ctl_opt & allowed; 2561 } 2562 2563 #define vmx_check_entry_exit_pairs(pairs, entry_controls, exit_controls) \ 2564 ({ \ 2565 int i, r = 0; \ 2566 \ 2567 BUILD_BUG_ON(sizeof(pairs[0].entry_control) != sizeof(entry_controls)); \ 2568 BUILD_BUG_ON(sizeof(pairs[0].exit_control) != sizeof(exit_controls)); \ 2569 \ 2570 for (i = 0; i < ARRAY_SIZE(pairs); i++) { \ 2571 typeof(entry_controls) n_ctrl = pairs[i].entry_control; \ 2572 typeof(exit_controls) x_ctrl = pairs[i].exit_control; \ 2573 \ 2574 if (!(entry_controls & n_ctrl) == !(exit_controls & x_ctrl)) \ 2575 continue; \ 2576 \ 2577 pr_warn_once("Inconsistent VM-Entry/VM-Exit pair, " \ 2578 "entry = %llx (%llx), exit = %llx (%llx)\n", \ 2579 (u64)(entry_controls & n_ctrl), (u64)n_ctrl, \ 2580 (u64)(exit_controls & x_ctrl), (u64)x_ctrl); \ 2581 \ 2582 if (error_on_inconsistent_vmcs_config) \ 2583 r = -EIO; \ 2584 \ 2585 entry_controls &= ~n_ctrl; \ 2586 exit_controls &= ~x_ctrl; \ 2587 } \ 2588 r; \ 2589 }) 2590 2591 static int setup_vmcs_config(struct vmcs_config *vmcs_conf, 2592 struct vmx_capability *vmx_cap) 2593 { 2594 u32 _pin_based_exec_control = 0; 2595 u32 _cpu_based_exec_control = 0; 2596 u32 _cpu_based_2nd_exec_control = 0; 2597 u64 _cpu_based_3rd_exec_control = 0; 2598 u32 _vmexit_control = 0; 2599 u32 _vmentry_control = 0; 2600 u64 basic_msr; 2601 u64 misc_msr; 2602 2603 /* 2604 * LOAD/SAVE_DEBUG_CONTROLS are absent because both are mandatory. 2605 * SAVE_IA32_PAT and SAVE_IA32_EFER are absent because KVM always 2606 * intercepts writes to PAT and EFER, i.e. never enables those controls. 2607 */ 2608 struct { 2609 u32 entry_control; 2610 u32 exit_control; 2611 } const vmcs_entry_exit_pairs[] = { 2612 { VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL, VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL }, 2613 { VM_ENTRY_LOAD_IA32_PAT, VM_EXIT_LOAD_IA32_PAT }, 2614 { VM_ENTRY_LOAD_IA32_EFER, VM_EXIT_LOAD_IA32_EFER }, 2615 { VM_ENTRY_LOAD_BNDCFGS, VM_EXIT_CLEAR_BNDCFGS }, 2616 { VM_ENTRY_LOAD_IA32_RTIT_CTL, VM_EXIT_CLEAR_IA32_RTIT_CTL }, 2617 { VM_ENTRY_LOAD_CET_STATE, VM_EXIT_LOAD_CET_STATE }, 2618 }; 2619 2620 memset(vmcs_conf, 0, sizeof(*vmcs_conf)); 2621 2622 if (adjust_vmx_controls(KVM_REQUIRED_VMX_CPU_BASED_VM_EXEC_CONTROL, 2623 KVM_OPTIONAL_VMX_CPU_BASED_VM_EXEC_CONTROL, 2624 MSR_IA32_VMX_PROCBASED_CTLS, 2625 &_cpu_based_exec_control)) 2626 return -EIO; 2627 if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) { 2628 if (adjust_vmx_controls(KVM_REQUIRED_VMX_SECONDARY_VM_EXEC_CONTROL, 2629 KVM_OPTIONAL_VMX_SECONDARY_VM_EXEC_CONTROL, 2630 MSR_IA32_VMX_PROCBASED_CTLS2, 2631 &_cpu_based_2nd_exec_control)) 2632 return -EIO; 2633 } 2634 if (!IS_ENABLED(CONFIG_KVM_INTEL_PROVE_VE)) 2635 _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_EPT_VIOLATION_VE; 2636 2637 #ifndef CONFIG_X86_64 2638 if (!(_cpu_based_2nd_exec_control & 2639 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) 2640 _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW; 2641 #endif 2642 2643 if (!(_cpu_based_exec_control & CPU_BASED_TPR_SHADOW)) 2644 _cpu_based_2nd_exec_control &= ~( 2645 SECONDARY_EXEC_APIC_REGISTER_VIRT | 2646 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 2647 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 2648 2649 rdmsr_safe(MSR_IA32_VMX_EPT_VPID_CAP, 2650 &vmx_cap->ept, &vmx_cap->vpid); 2651 2652 if (!(_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) && 2653 vmx_cap->ept) { 2654 pr_warn_once("EPT CAP should not exist if not support " 2655 "1-setting enable EPT VM-execution control\n"); 2656 2657 if (error_on_inconsistent_vmcs_config) 2658 return -EIO; 2659 2660 vmx_cap->ept = 0; 2661 _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_EPT_VIOLATION_VE; 2662 } 2663 if (!(_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_VPID) && 2664 vmx_cap->vpid) { 2665 pr_warn_once("VPID CAP should not exist if not support " 2666 "1-setting enable VPID VM-execution control\n"); 2667 2668 if (error_on_inconsistent_vmcs_config) 2669 return -EIO; 2670 2671 vmx_cap->vpid = 0; 2672 } 2673 2674 if (!cpu_has_sgx()) 2675 _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_ENCLS_EXITING; 2676 2677 if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_TERTIARY_CONTROLS) 2678 _cpu_based_3rd_exec_control = 2679 adjust_vmx_controls64(KVM_OPTIONAL_VMX_TERTIARY_VM_EXEC_CONTROL, 2680 MSR_IA32_VMX_PROCBASED_CTLS3); 2681 2682 if (adjust_vmx_controls(KVM_REQUIRED_VMX_VM_EXIT_CONTROLS, 2683 KVM_OPTIONAL_VMX_VM_EXIT_CONTROLS, 2684 MSR_IA32_VMX_EXIT_CTLS, 2685 &_vmexit_control)) 2686 return -EIO; 2687 2688 if (adjust_vmx_controls(KVM_REQUIRED_VMX_PIN_BASED_VM_EXEC_CONTROL, 2689 KVM_OPTIONAL_VMX_PIN_BASED_VM_EXEC_CONTROL, 2690 MSR_IA32_VMX_PINBASED_CTLS, 2691 &_pin_based_exec_control)) 2692 return -EIO; 2693 2694 if (cpu_has_broken_vmx_preemption_timer()) 2695 _pin_based_exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER; 2696 if (!(_cpu_based_2nd_exec_control & 2697 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)) 2698 _pin_based_exec_control &= ~PIN_BASED_POSTED_INTR; 2699 2700 if (adjust_vmx_controls(KVM_REQUIRED_VMX_VM_ENTRY_CONTROLS, 2701 KVM_OPTIONAL_VMX_VM_ENTRY_CONTROLS, 2702 MSR_IA32_VMX_ENTRY_CTLS, 2703 &_vmentry_control)) 2704 return -EIO; 2705 2706 if (vmx_check_entry_exit_pairs(vmcs_entry_exit_pairs, 2707 _vmentry_control, _vmexit_control)) 2708 return -EIO; 2709 2710 /* 2711 * Some cpus support VM_{ENTRY,EXIT}_IA32_PERF_GLOBAL_CTRL but they 2712 * can't be used due to an errata where VM Exit may incorrectly clear 2713 * IA32_PERF_GLOBAL_CTRL[34:32]. Workaround the errata by using the 2714 * MSR load mechanism to switch IA32_PERF_GLOBAL_CTRL. 2715 */ 2716 switch (boot_cpu_data.x86_vfm) { 2717 case INTEL_NEHALEM_EP: /* AAK155 */ 2718 case INTEL_NEHALEM: /* AAP115 */ 2719 case INTEL_WESTMERE: /* AAT100 */ 2720 case INTEL_WESTMERE_EP: /* BC86,AAY89,BD102 */ 2721 case INTEL_NEHALEM_EX: /* BA97 */ 2722 _vmentry_control &= ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL; 2723 _vmexit_control &= ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; 2724 pr_warn_once("VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL " 2725 "does not work properly. Using workaround\n"); 2726 break; 2727 default: 2728 break; 2729 } 2730 2731 rdmsrq(MSR_IA32_VMX_BASIC, basic_msr); 2732 2733 /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */ 2734 if (vmx_basic_vmcs_size(basic_msr) > PAGE_SIZE) 2735 return -EIO; 2736 2737 #ifdef CONFIG_X86_64 2738 /* 2739 * KVM expects to be able to shove all legal physical addresses into 2740 * VMCS fields for 64-bit kernels, and per the SDM, "This bit is always 2741 * 0 for processors that support Intel 64 architecture". 2742 */ 2743 if (basic_msr & VMX_BASIC_32BIT_PHYS_ADDR_ONLY) 2744 return -EIO; 2745 #endif 2746 2747 /* Require Write-Back (WB) memory type for VMCS accesses. */ 2748 if (vmx_basic_vmcs_mem_type(basic_msr) != X86_MEMTYPE_WB) 2749 return -EIO; 2750 2751 rdmsrq(MSR_IA32_VMX_MISC, misc_msr); 2752 2753 vmcs_conf->basic = basic_msr; 2754 vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control; 2755 vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control; 2756 vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control; 2757 vmcs_conf->cpu_based_3rd_exec_ctrl = _cpu_based_3rd_exec_control; 2758 vmcs_conf->vmexit_ctrl = _vmexit_control; 2759 vmcs_conf->vmentry_ctrl = _vmentry_control; 2760 vmcs_conf->misc = misc_msr; 2761 2762 #if IS_ENABLED(CONFIG_HYPERV) 2763 if (enlightened_vmcs) 2764 evmcs_sanitize_exec_ctrls(vmcs_conf); 2765 #endif 2766 2767 return 0; 2768 } 2769 2770 static bool __kvm_is_vmx_supported(void) 2771 { 2772 int cpu = smp_processor_id(); 2773 2774 if (!(cpuid_ecx(1) & feature_bit(VMX))) { 2775 pr_err("VMX not supported by CPU %d\n", cpu); 2776 return false; 2777 } 2778 2779 if (!this_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) || 2780 !this_cpu_has(X86_FEATURE_VMX)) { 2781 pr_err("VMX not enabled (by BIOS) in MSR_IA32_FEAT_CTL on CPU %d\n", cpu); 2782 return false; 2783 } 2784 2785 return true; 2786 } 2787 2788 static bool kvm_is_vmx_supported(void) 2789 { 2790 bool supported; 2791 2792 migrate_disable(); 2793 supported = __kvm_is_vmx_supported(); 2794 migrate_enable(); 2795 2796 return supported; 2797 } 2798 2799 int vmx_check_processor_compat(void) 2800 { 2801 int cpu = raw_smp_processor_id(); 2802 struct vmcs_config vmcs_conf; 2803 struct vmx_capability vmx_cap; 2804 2805 if (!__kvm_is_vmx_supported()) 2806 return -EIO; 2807 2808 if (setup_vmcs_config(&vmcs_conf, &vmx_cap) < 0) { 2809 pr_err("Failed to setup VMCS config on CPU %d\n", cpu); 2810 return -EIO; 2811 } 2812 if (nested) 2813 nested_vmx_setup_ctls_msrs(&vmcs_conf, vmx_cap.ept); 2814 if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config))) { 2815 pr_err("Inconsistent VMCS config on CPU %d\n", cpu); 2816 return -EIO; 2817 } 2818 return 0; 2819 } 2820 2821 static int kvm_cpu_vmxon(u64 vmxon_pointer) 2822 { 2823 u64 msr; 2824 2825 cr4_set_bits(X86_CR4_VMXE); 2826 2827 asm goto("1: vmxon %[vmxon_pointer]\n\t" 2828 _ASM_EXTABLE(1b, %l[fault]) 2829 : : [vmxon_pointer] "m"(vmxon_pointer) 2830 : : fault); 2831 return 0; 2832 2833 fault: 2834 WARN_ONCE(1, "VMXON faulted, MSR_IA32_FEAT_CTL (0x3a) = 0x%llx\n", 2835 rdmsrq_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr); 2836 cr4_clear_bits(X86_CR4_VMXE); 2837 2838 return -EFAULT; 2839 } 2840 2841 int vmx_enable_virtualization_cpu(void) 2842 { 2843 int cpu = raw_smp_processor_id(); 2844 u64 phys_addr = __pa(per_cpu(vmxarea, cpu)); 2845 int r; 2846 2847 if (cr4_read_shadow() & X86_CR4_VMXE) 2848 return -EBUSY; 2849 2850 /* 2851 * This can happen if we hot-added a CPU but failed to allocate 2852 * VP assist page for it. 2853 */ 2854 if (kvm_is_using_evmcs() && !hv_get_vp_assist_page(cpu)) 2855 return -EFAULT; 2856 2857 intel_pt_handle_vmx(1); 2858 2859 r = kvm_cpu_vmxon(phys_addr); 2860 if (r) { 2861 intel_pt_handle_vmx(0); 2862 return r; 2863 } 2864 2865 return 0; 2866 } 2867 2868 static void vmclear_local_loaded_vmcss(void) 2869 { 2870 int cpu = raw_smp_processor_id(); 2871 struct loaded_vmcs *v, *n; 2872 2873 list_for_each_entry_safe(v, n, &per_cpu(loaded_vmcss_on_cpu, cpu), 2874 loaded_vmcss_on_cpu_link) 2875 __loaded_vmcs_clear(v); 2876 } 2877 2878 void vmx_disable_virtualization_cpu(void) 2879 { 2880 vmclear_local_loaded_vmcss(); 2881 2882 if (kvm_cpu_vmxoff()) 2883 kvm_spurious_fault(); 2884 2885 hv_reset_evmcs(); 2886 2887 intel_pt_handle_vmx(0); 2888 } 2889 2890 struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags) 2891 { 2892 int node = cpu_to_node(cpu); 2893 struct page *pages; 2894 struct vmcs *vmcs; 2895 2896 pages = __alloc_pages_node(node, flags, 0); 2897 if (!pages) 2898 return NULL; 2899 vmcs = page_address(pages); 2900 memset(vmcs, 0, vmx_basic_vmcs_size(vmcs_config.basic)); 2901 2902 /* KVM supports Enlightened VMCS v1 only */ 2903 if (kvm_is_using_evmcs()) 2904 vmcs->hdr.revision_id = KVM_EVMCS_VERSION; 2905 else 2906 vmcs->hdr.revision_id = vmx_basic_vmcs_revision_id(vmcs_config.basic); 2907 2908 if (shadow) 2909 vmcs->hdr.shadow_vmcs = 1; 2910 return vmcs; 2911 } 2912 2913 void free_vmcs(struct vmcs *vmcs) 2914 { 2915 free_page((unsigned long)vmcs); 2916 } 2917 2918 /* 2919 * Free a VMCS, but before that VMCLEAR it on the CPU where it was last loaded 2920 */ 2921 void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs) 2922 { 2923 if (!loaded_vmcs->vmcs) 2924 return; 2925 loaded_vmcs_clear(loaded_vmcs); 2926 free_vmcs(loaded_vmcs->vmcs); 2927 loaded_vmcs->vmcs = NULL; 2928 if (loaded_vmcs->msr_bitmap) 2929 free_page((unsigned long)loaded_vmcs->msr_bitmap); 2930 WARN_ON(loaded_vmcs->shadow_vmcs != NULL); 2931 } 2932 2933 int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs) 2934 { 2935 loaded_vmcs->vmcs = alloc_vmcs(false); 2936 if (!loaded_vmcs->vmcs) 2937 return -ENOMEM; 2938 2939 vmcs_clear(loaded_vmcs->vmcs); 2940 2941 loaded_vmcs->shadow_vmcs = NULL; 2942 loaded_vmcs->hv_timer_soft_disabled = false; 2943 loaded_vmcs->cpu = -1; 2944 loaded_vmcs->launched = 0; 2945 2946 if (cpu_has_vmx_msr_bitmap()) { 2947 loaded_vmcs->msr_bitmap = (unsigned long *) 2948 __get_free_page(GFP_KERNEL_ACCOUNT); 2949 if (!loaded_vmcs->msr_bitmap) 2950 goto out_vmcs; 2951 memset(loaded_vmcs->msr_bitmap, 0xff, PAGE_SIZE); 2952 } 2953 2954 memset(&loaded_vmcs->host_state, 0, sizeof(struct vmcs_host_state)); 2955 memset(&loaded_vmcs->controls_shadow, 0, 2956 sizeof(struct vmcs_controls_shadow)); 2957 2958 return 0; 2959 2960 out_vmcs: 2961 free_loaded_vmcs(loaded_vmcs); 2962 return -ENOMEM; 2963 } 2964 2965 static void free_kvm_area(void) 2966 { 2967 int cpu; 2968 2969 for_each_possible_cpu(cpu) { 2970 free_vmcs(per_cpu(vmxarea, cpu)); 2971 per_cpu(vmxarea, cpu) = NULL; 2972 } 2973 } 2974 2975 static __init int alloc_kvm_area(void) 2976 { 2977 int cpu; 2978 2979 for_each_possible_cpu(cpu) { 2980 struct vmcs *vmcs; 2981 2982 vmcs = alloc_vmcs_cpu(false, cpu, GFP_KERNEL); 2983 if (!vmcs) { 2984 free_kvm_area(); 2985 return -ENOMEM; 2986 } 2987 2988 /* 2989 * When eVMCS is enabled, alloc_vmcs_cpu() sets 2990 * vmcs->revision_id to KVM_EVMCS_VERSION instead of 2991 * revision_id reported by MSR_IA32_VMX_BASIC. 2992 * 2993 * However, even though not explicitly documented by 2994 * TLFS, VMXArea passed as VMXON argument should 2995 * still be marked with revision_id reported by 2996 * physical CPU. 2997 */ 2998 if (kvm_is_using_evmcs()) 2999 vmcs->hdr.revision_id = vmx_basic_vmcs_revision_id(vmcs_config.basic); 3000 3001 per_cpu(vmxarea, cpu) = vmcs; 3002 } 3003 return 0; 3004 } 3005 3006 static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg, 3007 struct kvm_segment *save) 3008 { 3009 if (!emulate_invalid_guest_state) { 3010 /* 3011 * CS and SS RPL should be equal during guest entry according 3012 * to VMX spec, but in reality it is not always so. Since vcpu 3013 * is in the middle of the transition from real mode to 3014 * protected mode it is safe to assume that RPL 0 is a good 3015 * default value. 3016 */ 3017 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS) 3018 save->selector &= ~SEGMENT_RPL_MASK; 3019 save->dpl = save->selector & SEGMENT_RPL_MASK; 3020 save->s = 1; 3021 } 3022 __vmx_set_segment(vcpu, save, seg); 3023 } 3024 3025 static void enter_pmode(struct kvm_vcpu *vcpu) 3026 { 3027 unsigned long flags; 3028 struct vcpu_vmx *vmx = to_vmx(vcpu); 3029 3030 /* 3031 * Update real mode segment cache. It may be not up-to-date if segment 3032 * register was written while vcpu was in a guest mode. 3033 */ 3034 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES); 3035 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS); 3036 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS); 3037 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS); 3038 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS); 3039 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS); 3040 3041 vmx->rmode.vm86_active = 0; 3042 3043 __vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR); 3044 3045 flags = vmcs_readl(GUEST_RFLAGS); 3046 flags &= RMODE_GUEST_OWNED_EFLAGS_BITS; 3047 flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS; 3048 vmcs_writel(GUEST_RFLAGS, flags); 3049 3050 vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) | 3051 (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME)); 3052 3053 vmx_update_exception_bitmap(vcpu); 3054 3055 fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]); 3056 fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]); 3057 fix_pmode_seg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]); 3058 fix_pmode_seg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]); 3059 fix_pmode_seg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]); 3060 fix_pmode_seg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]); 3061 } 3062 3063 static void fix_rmode_seg(int seg, struct kvm_segment *save) 3064 { 3065 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 3066 struct kvm_segment var = *save; 3067 3068 var.dpl = 0x3; 3069 if (seg == VCPU_SREG_CS) 3070 var.type = 0x3; 3071 3072 if (!emulate_invalid_guest_state) { 3073 var.selector = var.base >> 4; 3074 var.base = var.base & 0xffff0; 3075 var.limit = 0xffff; 3076 var.g = 0; 3077 var.db = 0; 3078 var.present = 1; 3079 var.s = 1; 3080 var.l = 0; 3081 var.unusable = 0; 3082 var.type = 0x3; 3083 var.avl = 0; 3084 if (save->base & 0xf) 3085 pr_warn_once("segment base is not paragraph aligned " 3086 "when entering protected mode (seg=%d)", seg); 3087 } 3088 3089 vmcs_write16(sf->selector, var.selector); 3090 vmcs_writel(sf->base, var.base); 3091 vmcs_write32(sf->limit, var.limit); 3092 vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(&var)); 3093 } 3094 3095 static void enter_rmode(struct kvm_vcpu *vcpu) 3096 { 3097 unsigned long flags; 3098 struct vcpu_vmx *vmx = to_vmx(vcpu); 3099 struct kvm_vmx *kvm_vmx = to_kvm_vmx(vcpu->kvm); 3100 3101 /* 3102 * KVM should never use VM86 to virtualize Real Mode when L2 is active, 3103 * as using VM86 is unnecessary if unrestricted guest is enabled, and 3104 * if unrestricted guest is disabled, VM-Enter (from L1) with CR0.PG=0 3105 * should VM-Fail and KVM should reject userspace attempts to stuff 3106 * CR0.PG=0 when L2 is active. 3107 */ 3108 WARN_ON_ONCE(is_guest_mode(vcpu)); 3109 3110 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR); 3111 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES); 3112 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS); 3113 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS); 3114 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS); 3115 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS); 3116 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS); 3117 3118 vmx->rmode.vm86_active = 1; 3119 3120 vmx_segment_cache_clear(vmx); 3121 3122 vmcs_writel(GUEST_TR_BASE, kvm_vmx->tss_addr); 3123 vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1); 3124 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); 3125 3126 flags = vmcs_readl(GUEST_RFLAGS); 3127 vmx->rmode.save_rflags = flags; 3128 3129 flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM; 3130 3131 vmcs_writel(GUEST_RFLAGS, flags); 3132 vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME); 3133 vmx_update_exception_bitmap(vcpu); 3134 3135 fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]); 3136 fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]); 3137 fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]); 3138 fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]); 3139 fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]); 3140 fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]); 3141 } 3142 3143 int vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer) 3144 { 3145 struct vcpu_vmx *vmx = to_vmx(vcpu); 3146 3147 /* Nothing to do if hardware doesn't support EFER. */ 3148 if (!vmx_find_uret_msr(vmx, MSR_EFER)) 3149 return 0; 3150 3151 vcpu->arch.efer = efer; 3152 #ifdef CONFIG_X86_64 3153 if (efer & EFER_LMA) 3154 vm_entry_controls_setbit(vmx, VM_ENTRY_IA32E_MODE); 3155 else 3156 vm_entry_controls_clearbit(vmx, VM_ENTRY_IA32E_MODE); 3157 #else 3158 if (KVM_BUG_ON(efer & EFER_LMA, vcpu->kvm)) 3159 return 1; 3160 #endif 3161 3162 vmx_setup_uret_msrs(vmx); 3163 return 0; 3164 } 3165 3166 #ifdef CONFIG_X86_64 3167 3168 static void enter_lmode(struct kvm_vcpu *vcpu) 3169 { 3170 u32 guest_tr_ar; 3171 3172 vmx_segment_cache_clear(to_vmx(vcpu)); 3173 3174 guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES); 3175 if ((guest_tr_ar & VMX_AR_TYPE_MASK) != VMX_AR_TYPE_BUSY_64_TSS) { 3176 pr_debug_ratelimited("%s: tss fixup for long mode. \n", 3177 __func__); 3178 vmcs_write32(GUEST_TR_AR_BYTES, 3179 (guest_tr_ar & ~VMX_AR_TYPE_MASK) 3180 | VMX_AR_TYPE_BUSY_64_TSS); 3181 } 3182 vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA); 3183 } 3184 3185 static void exit_lmode(struct kvm_vcpu *vcpu) 3186 { 3187 vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA); 3188 } 3189 3190 #endif 3191 3192 void vmx_flush_tlb_all(struct kvm_vcpu *vcpu) 3193 { 3194 struct vcpu_vmx *vmx = to_vmx(vcpu); 3195 3196 /* 3197 * INVEPT must be issued when EPT is enabled, irrespective of VPID, as 3198 * the CPU is not required to invalidate guest-physical mappings on 3199 * VM-Entry, even if VPID is disabled. Guest-physical mappings are 3200 * associated with the root EPT structure and not any particular VPID 3201 * (INVVPID also isn't required to invalidate guest-physical mappings). 3202 */ 3203 if (enable_ept) { 3204 ept_sync_global(); 3205 } else if (enable_vpid) { 3206 if (cpu_has_vmx_invvpid_global()) { 3207 vpid_sync_vcpu_global(); 3208 } else { 3209 vpid_sync_vcpu_single(vmx->vpid); 3210 vpid_sync_vcpu_single(vmx->nested.vpid02); 3211 } 3212 } 3213 } 3214 3215 static inline int vmx_get_current_vpid(struct kvm_vcpu *vcpu) 3216 { 3217 if (is_guest_mode(vcpu) && nested_cpu_has_vpid(get_vmcs12(vcpu))) 3218 return nested_get_vpid02(vcpu); 3219 return to_vmx(vcpu)->vpid; 3220 } 3221 3222 void vmx_flush_tlb_current(struct kvm_vcpu *vcpu) 3223 { 3224 struct kvm_mmu *mmu = vcpu->arch.mmu; 3225 u64 root_hpa = mmu->root.hpa; 3226 3227 /* No flush required if the current context is invalid. */ 3228 if (!VALID_PAGE(root_hpa)) 3229 return; 3230 3231 if (enable_ept) 3232 ept_sync_context(construct_eptp(vcpu, root_hpa, 3233 mmu->root_role.level)); 3234 else 3235 vpid_sync_context(vmx_get_current_vpid(vcpu)); 3236 } 3237 3238 void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) 3239 { 3240 /* 3241 * vpid_sync_vcpu_addr() is a nop if vpid==0, see the comment in 3242 * vmx_flush_tlb_guest() for an explanation of why this is ok. 3243 */ 3244 vpid_sync_vcpu_addr(vmx_get_current_vpid(vcpu), addr); 3245 } 3246 3247 void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu) 3248 { 3249 /* 3250 * vpid_sync_context() is a nop if vpid==0, e.g. if enable_vpid==0 or a 3251 * vpid couldn't be allocated for this vCPU. VM-Enter and VM-Exit are 3252 * required to flush GVA->{G,H}PA mappings from the TLB if vpid is 3253 * disabled (VM-Enter with vpid enabled and vpid==0 is disallowed), 3254 * i.e. no explicit INVVPID is necessary. 3255 */ 3256 vpid_sync_context(vmx_get_current_vpid(vcpu)); 3257 } 3258 3259 void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu) 3260 { 3261 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 3262 3263 if (!kvm_register_is_dirty(vcpu, VCPU_EXREG_PDPTR)) 3264 return; 3265 3266 if (is_pae_paging(vcpu)) { 3267 vmcs_write64(GUEST_PDPTR0, mmu->pdptrs[0]); 3268 vmcs_write64(GUEST_PDPTR1, mmu->pdptrs[1]); 3269 vmcs_write64(GUEST_PDPTR2, mmu->pdptrs[2]); 3270 vmcs_write64(GUEST_PDPTR3, mmu->pdptrs[3]); 3271 } 3272 } 3273 3274 void ept_save_pdptrs(struct kvm_vcpu *vcpu) 3275 { 3276 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 3277 3278 if (WARN_ON_ONCE(!is_pae_paging(vcpu))) 3279 return; 3280 3281 mmu->pdptrs[0] = vmcs_read64(GUEST_PDPTR0); 3282 mmu->pdptrs[1] = vmcs_read64(GUEST_PDPTR1); 3283 mmu->pdptrs[2] = vmcs_read64(GUEST_PDPTR2); 3284 mmu->pdptrs[3] = vmcs_read64(GUEST_PDPTR3); 3285 3286 kvm_register_mark_available(vcpu, VCPU_EXREG_PDPTR); 3287 } 3288 3289 #define CR3_EXITING_BITS (CPU_BASED_CR3_LOAD_EXITING | \ 3290 CPU_BASED_CR3_STORE_EXITING) 3291 3292 bool vmx_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 3293 { 3294 if (is_guest_mode(vcpu)) 3295 return nested_guest_cr0_valid(vcpu, cr0); 3296 3297 if (to_vmx(vcpu)->nested.vmxon) 3298 return nested_host_cr0_valid(vcpu, cr0); 3299 3300 return true; 3301 } 3302 3303 void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 3304 { 3305 struct vcpu_vmx *vmx = to_vmx(vcpu); 3306 unsigned long hw_cr0, old_cr0_pg; 3307 u32 tmp; 3308 3309 old_cr0_pg = kvm_read_cr0_bits(vcpu, X86_CR0_PG); 3310 3311 hw_cr0 = (cr0 & ~KVM_VM_CR0_ALWAYS_OFF); 3312 if (enable_unrestricted_guest) 3313 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST; 3314 else { 3315 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON; 3316 if (!enable_ept) 3317 hw_cr0 |= X86_CR0_WP; 3318 3319 if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE)) 3320 enter_pmode(vcpu); 3321 3322 if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE)) 3323 enter_rmode(vcpu); 3324 } 3325 3326 vmcs_writel(CR0_READ_SHADOW, cr0); 3327 vmcs_writel(GUEST_CR0, hw_cr0); 3328 vcpu->arch.cr0 = cr0; 3329 kvm_register_mark_available(vcpu, VCPU_EXREG_CR0); 3330 3331 #ifdef CONFIG_X86_64 3332 if (vcpu->arch.efer & EFER_LME) { 3333 if (!old_cr0_pg && (cr0 & X86_CR0_PG)) 3334 enter_lmode(vcpu); 3335 else if (old_cr0_pg && !(cr0 & X86_CR0_PG)) 3336 exit_lmode(vcpu); 3337 } 3338 #endif 3339 3340 if (enable_ept && !enable_unrestricted_guest) { 3341 /* 3342 * Ensure KVM has an up-to-date snapshot of the guest's CR3. If 3343 * the below code _enables_ CR3 exiting, vmx_cache_reg() will 3344 * (correctly) stop reading vmcs.GUEST_CR3 because it thinks 3345 * KVM's CR3 is installed. 3346 */ 3347 if (!kvm_register_is_available(vcpu, VCPU_EXREG_CR3)) 3348 vmx_cache_reg(vcpu, VCPU_EXREG_CR3); 3349 3350 /* 3351 * When running with EPT but not unrestricted guest, KVM must 3352 * intercept CR3 accesses when paging is _disabled_. This is 3353 * necessary because restricted guests can't actually run with 3354 * paging disabled, and so KVM stuffs its own CR3 in order to 3355 * run the guest when identity mapped page tables. 3356 * 3357 * Do _NOT_ check the old CR0.PG, e.g. to optimize away the 3358 * update, it may be stale with respect to CR3 interception, 3359 * e.g. after nested VM-Enter. 3360 * 3361 * Lastly, honor L1's desires, i.e. intercept CR3 loads and/or 3362 * stores to forward them to L1, even if KVM does not need to 3363 * intercept them to preserve its identity mapped page tables. 3364 */ 3365 if (!(cr0 & X86_CR0_PG)) { 3366 exec_controls_setbit(vmx, CR3_EXITING_BITS); 3367 } else if (!is_guest_mode(vcpu)) { 3368 exec_controls_clearbit(vmx, CR3_EXITING_BITS); 3369 } else { 3370 tmp = exec_controls_get(vmx); 3371 tmp &= ~CR3_EXITING_BITS; 3372 tmp |= get_vmcs12(vcpu)->cpu_based_vm_exec_control & CR3_EXITING_BITS; 3373 exec_controls_set(vmx, tmp); 3374 } 3375 3376 /* Note, vmx_set_cr4() consumes the new vcpu->arch.cr0. */ 3377 if ((old_cr0_pg ^ cr0) & X86_CR0_PG) 3378 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu)); 3379 3380 /* 3381 * When !CR0_PG -> CR0_PG, vcpu->arch.cr3 becomes active, but 3382 * GUEST_CR3 is still vmx->ept_identity_map_addr if EPT + !URG. 3383 */ 3384 if (!(old_cr0_pg & X86_CR0_PG) && (cr0 & X86_CR0_PG)) 3385 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); 3386 } 3387 3388 /* depends on vcpu->arch.cr0 to be set to a new value */ 3389 vmx->vt.emulation_required = vmx_emulation_required(vcpu); 3390 } 3391 3392 static int vmx_get_max_ept_level(void) 3393 { 3394 if (cpu_has_vmx_ept_5levels()) 3395 return 5; 3396 return 4; 3397 } 3398 3399 u64 construct_eptp(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level) 3400 { 3401 u64 eptp = VMX_EPTP_MT_WB; 3402 3403 eptp |= (root_level == 5) ? VMX_EPTP_PWL_5 : VMX_EPTP_PWL_4; 3404 3405 if (enable_ept_ad_bits && 3406 (!is_guest_mode(vcpu) || nested_ept_ad_enabled(vcpu))) 3407 eptp |= VMX_EPTP_AD_ENABLE_BIT; 3408 eptp |= root_hpa; 3409 3410 return eptp; 3411 } 3412 3413 void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level) 3414 { 3415 struct kvm *kvm = vcpu->kvm; 3416 bool update_guest_cr3 = true; 3417 unsigned long guest_cr3; 3418 u64 eptp; 3419 3420 if (enable_ept) { 3421 eptp = construct_eptp(vcpu, root_hpa, root_level); 3422 vmcs_write64(EPT_POINTER, eptp); 3423 3424 hv_track_root_tdp(vcpu, root_hpa); 3425 3426 if (!enable_unrestricted_guest && !is_paging(vcpu)) 3427 guest_cr3 = to_kvm_vmx(kvm)->ept_identity_map_addr; 3428 else if (kvm_register_is_dirty(vcpu, VCPU_EXREG_CR3)) 3429 guest_cr3 = vcpu->arch.cr3; 3430 else /* vmcs.GUEST_CR3 is already up-to-date. */ 3431 update_guest_cr3 = false; 3432 vmx_ept_load_pdptrs(vcpu); 3433 } else { 3434 guest_cr3 = root_hpa | kvm_get_active_pcid(vcpu) | 3435 kvm_get_active_cr3_lam_bits(vcpu); 3436 } 3437 3438 if (update_guest_cr3) 3439 vmcs_writel(GUEST_CR3, guest_cr3); 3440 } 3441 3442 bool vmx_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 3443 { 3444 /* 3445 * We operate under the default treatment of SMM, so VMX cannot be 3446 * enabled under SMM. Note, whether or not VMXE is allowed at all, 3447 * i.e. is a reserved bit, is handled by common x86 code. 3448 */ 3449 if ((cr4 & X86_CR4_VMXE) && is_smm(vcpu)) 3450 return false; 3451 3452 if (to_vmx(vcpu)->nested.vmxon && !nested_cr4_valid(vcpu, cr4)) 3453 return false; 3454 3455 return true; 3456 } 3457 3458 void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 3459 { 3460 unsigned long old_cr4 = kvm_read_cr4(vcpu); 3461 struct vcpu_vmx *vmx = to_vmx(vcpu); 3462 unsigned long hw_cr4; 3463 3464 /* 3465 * Pass through host's Machine Check Enable value to hw_cr4, which 3466 * is in force while we are in guest mode. Do not let guests control 3467 * this bit, even if host CR4.MCE == 0. 3468 */ 3469 hw_cr4 = (cr4_read_shadow() & X86_CR4_MCE) | (cr4 & ~X86_CR4_MCE); 3470 if (enable_unrestricted_guest) 3471 hw_cr4 |= KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST; 3472 else if (vmx->rmode.vm86_active) 3473 hw_cr4 |= KVM_RMODE_VM_CR4_ALWAYS_ON; 3474 else 3475 hw_cr4 |= KVM_PMODE_VM_CR4_ALWAYS_ON; 3476 3477 if (vmx_umip_emulated()) { 3478 if (cr4 & X86_CR4_UMIP) { 3479 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_DESC); 3480 hw_cr4 &= ~X86_CR4_UMIP; 3481 } else if (!is_guest_mode(vcpu) || 3482 !nested_cpu_has2(get_vmcs12(vcpu), SECONDARY_EXEC_DESC)) { 3483 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_DESC); 3484 } 3485 } 3486 3487 vcpu->arch.cr4 = cr4; 3488 kvm_register_mark_available(vcpu, VCPU_EXREG_CR4); 3489 3490 if (!enable_unrestricted_guest) { 3491 if (enable_ept) { 3492 if (!is_paging(vcpu)) { 3493 hw_cr4 &= ~X86_CR4_PAE; 3494 hw_cr4 |= X86_CR4_PSE; 3495 } else if (!(cr4 & X86_CR4_PAE)) { 3496 hw_cr4 &= ~X86_CR4_PAE; 3497 } 3498 } 3499 3500 /* 3501 * SMEP/SMAP/PKU is disabled if CPU is in non-paging mode in 3502 * hardware. To emulate this behavior, SMEP/SMAP/PKU needs 3503 * to be manually disabled when guest switches to non-paging 3504 * mode. 3505 * 3506 * If !enable_unrestricted_guest, the CPU is always running 3507 * with CR0.PG=1 and CR4 needs to be modified. 3508 * If enable_unrestricted_guest, the CPU automatically 3509 * disables SMEP/SMAP/PKU when the guest sets CR0.PG=0. 3510 */ 3511 if (!is_paging(vcpu)) 3512 hw_cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE); 3513 } 3514 3515 vmcs_writel(CR4_READ_SHADOW, cr4); 3516 vmcs_writel(GUEST_CR4, hw_cr4); 3517 3518 if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE)) 3519 vcpu->arch.cpuid_dynamic_bits_dirty = true; 3520 } 3521 3522 void vmx_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) 3523 { 3524 struct vcpu_vmx *vmx = to_vmx(vcpu); 3525 u32 ar; 3526 3527 if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) { 3528 *var = vmx->rmode.segs[seg]; 3529 if (seg == VCPU_SREG_TR 3530 || var->selector == vmx_read_guest_seg_selector(vmx, seg)) 3531 return; 3532 var->base = vmx_read_guest_seg_base(vmx, seg); 3533 var->selector = vmx_read_guest_seg_selector(vmx, seg); 3534 return; 3535 } 3536 var->base = vmx_read_guest_seg_base(vmx, seg); 3537 var->limit = vmx_read_guest_seg_limit(vmx, seg); 3538 var->selector = vmx_read_guest_seg_selector(vmx, seg); 3539 ar = vmx_read_guest_seg_ar(vmx, seg); 3540 var->unusable = (ar >> 16) & 1; 3541 var->type = ar & 15; 3542 var->s = (ar >> 4) & 1; 3543 var->dpl = (ar >> 5) & 3; 3544 /* 3545 * Some userspaces do not preserve unusable property. Since usable 3546 * segment has to be present according to VMX spec we can use present 3547 * property to amend userspace bug by making unusable segment always 3548 * nonpresent. vmx_segment_access_rights() already marks nonpresent 3549 * segment as unusable. 3550 */ 3551 var->present = !var->unusable; 3552 var->avl = (ar >> 12) & 1; 3553 var->l = (ar >> 13) & 1; 3554 var->db = (ar >> 14) & 1; 3555 var->g = (ar >> 15) & 1; 3556 } 3557 3558 u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg) 3559 { 3560 struct kvm_segment s; 3561 3562 if (to_vmx(vcpu)->rmode.vm86_active) { 3563 vmx_get_segment(vcpu, &s, seg); 3564 return s.base; 3565 } 3566 return vmx_read_guest_seg_base(to_vmx(vcpu), seg); 3567 } 3568 3569 static int __vmx_get_cpl(struct kvm_vcpu *vcpu, bool no_cache) 3570 { 3571 struct vcpu_vmx *vmx = to_vmx(vcpu); 3572 int ar; 3573 3574 if (unlikely(vmx->rmode.vm86_active)) 3575 return 0; 3576 3577 if (no_cache) 3578 ar = vmcs_read32(GUEST_SS_AR_BYTES); 3579 else 3580 ar = vmx_read_guest_seg_ar(vmx, VCPU_SREG_SS); 3581 return VMX_AR_DPL(ar); 3582 } 3583 3584 int vmx_get_cpl(struct kvm_vcpu *vcpu) 3585 { 3586 return __vmx_get_cpl(vcpu, false); 3587 } 3588 3589 int vmx_get_cpl_no_cache(struct kvm_vcpu *vcpu) 3590 { 3591 return __vmx_get_cpl(vcpu, true); 3592 } 3593 3594 static u32 vmx_segment_access_rights(struct kvm_segment *var) 3595 { 3596 u32 ar; 3597 3598 ar = var->type & 15; 3599 ar |= (var->s & 1) << 4; 3600 ar |= (var->dpl & 3) << 5; 3601 ar |= (var->present & 1) << 7; 3602 ar |= (var->avl & 1) << 12; 3603 ar |= (var->l & 1) << 13; 3604 ar |= (var->db & 1) << 14; 3605 ar |= (var->g & 1) << 15; 3606 ar |= (var->unusable || !var->present) << 16; 3607 3608 return ar; 3609 } 3610 3611 void __vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) 3612 { 3613 struct vcpu_vmx *vmx = to_vmx(vcpu); 3614 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 3615 3616 vmx_segment_cache_clear(vmx); 3617 3618 if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) { 3619 vmx->rmode.segs[seg] = *var; 3620 if (seg == VCPU_SREG_TR) 3621 vmcs_write16(sf->selector, var->selector); 3622 else if (var->s) 3623 fix_rmode_seg(seg, &vmx->rmode.segs[seg]); 3624 return; 3625 } 3626 3627 vmcs_writel(sf->base, var->base); 3628 vmcs_write32(sf->limit, var->limit); 3629 vmcs_write16(sf->selector, var->selector); 3630 3631 /* 3632 * Fix the "Accessed" bit in AR field of segment registers for older 3633 * qemu binaries. 3634 * IA32 arch specifies that at the time of processor reset the 3635 * "Accessed" bit in the AR field of segment registers is 1. And qemu 3636 * is setting it to 0 in the userland code. This causes invalid guest 3637 * state vmexit when "unrestricted guest" mode is turned on. 3638 * Fix for this setup issue in cpu_reset is being pushed in the qemu 3639 * tree. Newer qemu binaries with that qemu fix would not need this 3640 * kvm hack. 3641 */ 3642 if (is_unrestricted_guest(vcpu) && (seg != VCPU_SREG_LDTR)) 3643 var->type |= 0x1; /* Accessed */ 3644 3645 vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(var)); 3646 } 3647 3648 void vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) 3649 { 3650 __vmx_set_segment(vcpu, var, seg); 3651 3652 to_vmx(vcpu)->vt.emulation_required = vmx_emulation_required(vcpu); 3653 } 3654 3655 void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) 3656 { 3657 u32 ar = vmx_read_guest_seg_ar(to_vmx(vcpu), VCPU_SREG_CS); 3658 3659 *db = (ar >> 14) & 1; 3660 *l = (ar >> 13) & 1; 3661 } 3662 3663 void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3664 { 3665 dt->size = vmcs_read32(GUEST_IDTR_LIMIT); 3666 dt->address = vmcs_readl(GUEST_IDTR_BASE); 3667 } 3668 3669 void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3670 { 3671 vmcs_write32(GUEST_IDTR_LIMIT, dt->size); 3672 vmcs_writel(GUEST_IDTR_BASE, dt->address); 3673 } 3674 3675 void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3676 { 3677 dt->size = vmcs_read32(GUEST_GDTR_LIMIT); 3678 dt->address = vmcs_readl(GUEST_GDTR_BASE); 3679 } 3680 3681 void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3682 { 3683 vmcs_write32(GUEST_GDTR_LIMIT, dt->size); 3684 vmcs_writel(GUEST_GDTR_BASE, dt->address); 3685 } 3686 3687 static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg) 3688 { 3689 struct kvm_segment var; 3690 u32 ar; 3691 3692 vmx_get_segment(vcpu, &var, seg); 3693 var.dpl = 0x3; 3694 if (seg == VCPU_SREG_CS) 3695 var.type = 0x3; 3696 ar = vmx_segment_access_rights(&var); 3697 3698 if (var.base != (var.selector << 4)) 3699 return false; 3700 if (var.limit != 0xffff) 3701 return false; 3702 if (ar != 0xf3) 3703 return false; 3704 3705 return true; 3706 } 3707 3708 static bool code_segment_valid(struct kvm_vcpu *vcpu) 3709 { 3710 struct kvm_segment cs; 3711 unsigned int cs_rpl; 3712 3713 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS); 3714 cs_rpl = cs.selector & SEGMENT_RPL_MASK; 3715 3716 if (cs.unusable) 3717 return false; 3718 if (~cs.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_ACCESSES_MASK)) 3719 return false; 3720 if (!cs.s) 3721 return false; 3722 if (cs.type & VMX_AR_TYPE_WRITEABLE_MASK) { 3723 if (cs.dpl > cs_rpl) 3724 return false; 3725 } else { 3726 if (cs.dpl != cs_rpl) 3727 return false; 3728 } 3729 if (!cs.present) 3730 return false; 3731 3732 /* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */ 3733 return true; 3734 } 3735 3736 static bool stack_segment_valid(struct kvm_vcpu *vcpu) 3737 { 3738 struct kvm_segment ss; 3739 unsigned int ss_rpl; 3740 3741 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS); 3742 ss_rpl = ss.selector & SEGMENT_RPL_MASK; 3743 3744 if (ss.unusable) 3745 return true; 3746 if (ss.type != 3 && ss.type != 7) 3747 return false; 3748 if (!ss.s) 3749 return false; 3750 if (ss.dpl != ss_rpl) /* DPL != RPL */ 3751 return false; 3752 if (!ss.present) 3753 return false; 3754 3755 return true; 3756 } 3757 3758 static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg) 3759 { 3760 struct kvm_segment var; 3761 unsigned int rpl; 3762 3763 vmx_get_segment(vcpu, &var, seg); 3764 rpl = var.selector & SEGMENT_RPL_MASK; 3765 3766 if (var.unusable) 3767 return true; 3768 if (!var.s) 3769 return false; 3770 if (!var.present) 3771 return false; 3772 if (~var.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_WRITEABLE_MASK)) { 3773 if (var.dpl < rpl) /* DPL < RPL */ 3774 return false; 3775 } 3776 3777 /* TODO: Add other members to kvm_segment_field to allow checking for other access 3778 * rights flags 3779 */ 3780 return true; 3781 } 3782 3783 static bool tr_valid(struct kvm_vcpu *vcpu) 3784 { 3785 struct kvm_segment tr; 3786 3787 vmx_get_segment(vcpu, &tr, VCPU_SREG_TR); 3788 3789 if (tr.unusable) 3790 return false; 3791 if (tr.selector & SEGMENT_TI_MASK) /* TI = 1 */ 3792 return false; 3793 if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */ 3794 return false; 3795 if (!tr.present) 3796 return false; 3797 3798 return true; 3799 } 3800 3801 static bool ldtr_valid(struct kvm_vcpu *vcpu) 3802 { 3803 struct kvm_segment ldtr; 3804 3805 vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR); 3806 3807 if (ldtr.unusable) 3808 return true; 3809 if (ldtr.selector & SEGMENT_TI_MASK) /* TI = 1 */ 3810 return false; 3811 if (ldtr.type != 2) 3812 return false; 3813 if (!ldtr.present) 3814 return false; 3815 3816 return true; 3817 } 3818 3819 static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu) 3820 { 3821 struct kvm_segment cs, ss; 3822 3823 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS); 3824 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS); 3825 3826 return ((cs.selector & SEGMENT_RPL_MASK) == 3827 (ss.selector & SEGMENT_RPL_MASK)); 3828 } 3829 3830 /* 3831 * Check if guest state is valid. Returns true if valid, false if 3832 * not. 3833 * We assume that registers are always usable 3834 */ 3835 bool __vmx_guest_state_valid(struct kvm_vcpu *vcpu) 3836 { 3837 /* real mode guest state checks */ 3838 if (!is_protmode(vcpu) || (vmx_get_rflags(vcpu) & X86_EFLAGS_VM)) { 3839 if (!rmode_segment_valid(vcpu, VCPU_SREG_CS)) 3840 return false; 3841 if (!rmode_segment_valid(vcpu, VCPU_SREG_SS)) 3842 return false; 3843 if (!rmode_segment_valid(vcpu, VCPU_SREG_DS)) 3844 return false; 3845 if (!rmode_segment_valid(vcpu, VCPU_SREG_ES)) 3846 return false; 3847 if (!rmode_segment_valid(vcpu, VCPU_SREG_FS)) 3848 return false; 3849 if (!rmode_segment_valid(vcpu, VCPU_SREG_GS)) 3850 return false; 3851 } else { 3852 /* protected mode guest state checks */ 3853 if (!cs_ss_rpl_check(vcpu)) 3854 return false; 3855 if (!code_segment_valid(vcpu)) 3856 return false; 3857 if (!stack_segment_valid(vcpu)) 3858 return false; 3859 if (!data_segment_valid(vcpu, VCPU_SREG_DS)) 3860 return false; 3861 if (!data_segment_valid(vcpu, VCPU_SREG_ES)) 3862 return false; 3863 if (!data_segment_valid(vcpu, VCPU_SREG_FS)) 3864 return false; 3865 if (!data_segment_valid(vcpu, VCPU_SREG_GS)) 3866 return false; 3867 if (!tr_valid(vcpu)) 3868 return false; 3869 if (!ldtr_valid(vcpu)) 3870 return false; 3871 } 3872 /* TODO: 3873 * - Add checks on RIP 3874 * - Add checks on RFLAGS 3875 */ 3876 3877 return true; 3878 } 3879 3880 static int init_rmode_tss(struct kvm *kvm, void __user *ua) 3881 { 3882 const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0))); 3883 u16 data; 3884 int i; 3885 3886 for (i = 0; i < 3; i++) { 3887 if (__copy_to_user(ua + PAGE_SIZE * i, zero_page, PAGE_SIZE)) 3888 return -EFAULT; 3889 } 3890 3891 data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE; 3892 if (__copy_to_user(ua + TSS_IOPB_BASE_OFFSET, &data, sizeof(u16))) 3893 return -EFAULT; 3894 3895 data = ~0; 3896 if (__copy_to_user(ua + RMODE_TSS_SIZE - 1, &data, sizeof(u8))) 3897 return -EFAULT; 3898 3899 return 0; 3900 } 3901 3902 static int init_rmode_identity_map(struct kvm *kvm) 3903 { 3904 struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); 3905 int i, r = 0; 3906 void __user *uaddr; 3907 u32 tmp; 3908 3909 /* Protect kvm_vmx->ept_identity_pagetable_done. */ 3910 mutex_lock(&kvm->slots_lock); 3911 3912 if (likely(kvm_vmx->ept_identity_pagetable_done)) 3913 goto out; 3914 3915 if (!kvm_vmx->ept_identity_map_addr) 3916 kvm_vmx->ept_identity_map_addr = VMX_EPT_IDENTITY_PAGETABLE_ADDR; 3917 3918 uaddr = __x86_set_memory_region(kvm, 3919 IDENTITY_PAGETABLE_PRIVATE_MEMSLOT, 3920 kvm_vmx->ept_identity_map_addr, 3921 PAGE_SIZE); 3922 if (IS_ERR(uaddr)) { 3923 r = PTR_ERR(uaddr); 3924 goto out; 3925 } 3926 3927 /* Set up identity-mapping pagetable for EPT in real mode */ 3928 for (i = 0; i < (PAGE_SIZE / sizeof(tmp)); i++) { 3929 tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | 3930 _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE); 3931 if (__copy_to_user(uaddr + i * sizeof(tmp), &tmp, sizeof(tmp))) { 3932 r = -EFAULT; 3933 goto out; 3934 } 3935 } 3936 kvm_vmx->ept_identity_pagetable_done = true; 3937 3938 out: 3939 mutex_unlock(&kvm->slots_lock); 3940 return r; 3941 } 3942 3943 static void seg_setup(int seg) 3944 { 3945 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 3946 unsigned int ar; 3947 3948 vmcs_write16(sf->selector, 0); 3949 vmcs_writel(sf->base, 0); 3950 vmcs_write32(sf->limit, 0xffff); 3951 ar = 0x93; 3952 if (seg == VCPU_SREG_CS) 3953 ar |= 0x08; /* code segment */ 3954 3955 vmcs_write32(sf->ar_bytes, ar); 3956 } 3957 3958 int allocate_vpid(void) 3959 { 3960 int vpid; 3961 3962 if (!enable_vpid) 3963 return 0; 3964 spin_lock(&vmx_vpid_lock); 3965 vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS); 3966 if (vpid < VMX_NR_VPIDS) 3967 __set_bit(vpid, vmx_vpid_bitmap); 3968 else 3969 vpid = 0; 3970 spin_unlock(&vmx_vpid_lock); 3971 return vpid; 3972 } 3973 3974 void free_vpid(int vpid) 3975 { 3976 if (!enable_vpid || vpid == 0) 3977 return; 3978 spin_lock(&vmx_vpid_lock); 3979 __clear_bit(vpid, vmx_vpid_bitmap); 3980 spin_unlock(&vmx_vpid_lock); 3981 } 3982 3983 static void vmx_msr_bitmap_l01_changed(struct vcpu_vmx *vmx) 3984 { 3985 /* 3986 * When KVM is a nested hypervisor on top of Hyper-V and uses 3987 * 'Enlightened MSR Bitmap' feature L0 needs to know that MSR 3988 * bitmap has changed. 3989 */ 3990 if (kvm_is_using_evmcs()) { 3991 struct hv_enlightened_vmcs *evmcs = (void *)vmx->vmcs01.vmcs; 3992 3993 if (evmcs->hv_enlightenments_control.msr_bitmap) 3994 evmcs->hv_clean_fields &= 3995 ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP; 3996 } 3997 3998 vmx->nested.force_msr_bitmap_recalc = true; 3999 } 4000 4001 void vmx_set_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type, bool set) 4002 { 4003 struct vcpu_vmx *vmx = to_vmx(vcpu); 4004 unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap; 4005 4006 if (!cpu_has_vmx_msr_bitmap()) 4007 return; 4008 4009 vmx_msr_bitmap_l01_changed(vmx); 4010 4011 if (type & MSR_TYPE_R) { 4012 if (!set && kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_READ)) 4013 vmx_clear_msr_bitmap_read(msr_bitmap, msr); 4014 else 4015 vmx_set_msr_bitmap_read(msr_bitmap, msr); 4016 } 4017 4018 if (type & MSR_TYPE_W) { 4019 if (!set && kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_WRITE)) 4020 vmx_clear_msr_bitmap_write(msr_bitmap, msr); 4021 else 4022 vmx_set_msr_bitmap_write(msr_bitmap, msr); 4023 } 4024 } 4025 4026 static void vmx_update_msr_bitmap_x2apic(struct kvm_vcpu *vcpu) 4027 { 4028 /* 4029 * x2APIC indices for 64-bit accesses into the RDMSR and WRMSR halves 4030 * of the MSR bitmap. KVM emulates APIC registers up through 0x3f0, 4031 * i.e. MSR 0x83f, and so only needs to dynamically manipulate 64 bits. 4032 */ 4033 const int read_idx = APIC_BASE_MSR / BITS_PER_LONG_LONG; 4034 const int write_idx = read_idx + (0x800 / sizeof(u64)); 4035 struct vcpu_vmx *vmx = to_vmx(vcpu); 4036 u64 *msr_bitmap = (u64 *)vmx->vmcs01.msr_bitmap; 4037 u8 mode; 4038 4039 if (!cpu_has_vmx_msr_bitmap() || WARN_ON_ONCE(!lapic_in_kernel(vcpu))) 4040 return; 4041 4042 if (cpu_has_secondary_exec_ctrls() && 4043 (secondary_exec_controls_get(vmx) & 4044 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE)) { 4045 mode = MSR_BITMAP_MODE_X2APIC; 4046 if (enable_apicv && kvm_vcpu_apicv_active(vcpu)) 4047 mode |= MSR_BITMAP_MODE_X2APIC_APICV; 4048 } else { 4049 mode = 0; 4050 } 4051 4052 if (mode == vmx->x2apic_msr_bitmap_mode) 4053 return; 4054 4055 vmx->x2apic_msr_bitmap_mode = mode; 4056 4057 /* 4058 * Reset the bitmap for MSRs 0x800 - 0x83f. Leave AMD's uber-extended 4059 * registers (0x840 and above) intercepted, KVM doesn't support them. 4060 * Intercept all writes by default and poke holes as needed. Pass 4061 * through reads for all valid registers by default in x2APIC+APICv 4062 * mode, only the current timer count needs on-demand emulation by KVM. 4063 */ 4064 if (mode & MSR_BITMAP_MODE_X2APIC_APICV) 4065 msr_bitmap[read_idx] = ~kvm_lapic_readable_reg_mask(vcpu->arch.apic); 4066 else 4067 msr_bitmap[read_idx] = ~0ull; 4068 msr_bitmap[write_idx] = ~0ull; 4069 4070 /* 4071 * TPR reads and writes can be virtualized even if virtual interrupt 4072 * delivery is not in use. 4073 */ 4074 vmx_set_intercept_for_msr(vcpu, X2APIC_MSR(APIC_TASKPRI), MSR_TYPE_RW, 4075 !(mode & MSR_BITMAP_MODE_X2APIC)); 4076 4077 if (mode & MSR_BITMAP_MODE_X2APIC_APICV) { 4078 vmx_enable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_TMCCT), MSR_TYPE_RW); 4079 vmx_disable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_EOI), MSR_TYPE_W); 4080 vmx_disable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_SELF_IPI), MSR_TYPE_W); 4081 if (enable_ipiv) 4082 vmx_disable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_ICR), MSR_TYPE_RW); 4083 } 4084 } 4085 4086 void pt_update_intercept_for_msr(struct kvm_vcpu *vcpu) 4087 { 4088 struct vcpu_vmx *vmx = to_vmx(vcpu); 4089 bool flag = !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN); 4090 u32 i; 4091 4092 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_STATUS, MSR_TYPE_RW, flag); 4093 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_OUTPUT_BASE, MSR_TYPE_RW, flag); 4094 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_OUTPUT_MASK, MSR_TYPE_RW, flag); 4095 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_CR3_MATCH, MSR_TYPE_RW, flag); 4096 for (i = 0; i < vmx->pt_desc.num_address_ranges; i++) { 4097 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_ADDR0_A + i * 2, MSR_TYPE_RW, flag); 4098 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_ADDR0_B + i * 2, MSR_TYPE_RW, flag); 4099 } 4100 } 4101 4102 static void vmx_recalc_msr_intercepts(struct kvm_vcpu *vcpu) 4103 { 4104 bool intercept; 4105 4106 if (!cpu_has_vmx_msr_bitmap()) 4107 return; 4108 4109 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_TSC, MSR_TYPE_R); 4110 #ifdef CONFIG_X86_64 4111 vmx_disable_intercept_for_msr(vcpu, MSR_FS_BASE, MSR_TYPE_RW); 4112 vmx_disable_intercept_for_msr(vcpu, MSR_GS_BASE, MSR_TYPE_RW); 4113 vmx_disable_intercept_for_msr(vcpu, MSR_KERNEL_GS_BASE, MSR_TYPE_RW); 4114 #endif 4115 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_CS, MSR_TYPE_RW); 4116 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_ESP, MSR_TYPE_RW); 4117 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_EIP, MSR_TYPE_RW); 4118 if (kvm_cstate_in_guest(vcpu->kvm)) { 4119 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C1_RES, MSR_TYPE_R); 4120 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C3_RESIDENCY, MSR_TYPE_R); 4121 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C6_RESIDENCY, MSR_TYPE_R); 4122 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C7_RESIDENCY, MSR_TYPE_R); 4123 } 4124 if (kvm_aperfmperf_in_guest(vcpu->kvm)) { 4125 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_APERF, MSR_TYPE_R); 4126 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_MPERF, MSR_TYPE_R); 4127 } 4128 4129 /* PT MSRs can be passed through iff PT is exposed to the guest. */ 4130 if (vmx_pt_mode_is_host_guest()) 4131 pt_update_intercept_for_msr(vcpu); 4132 4133 if (vcpu->arch.xfd_no_write_intercept) 4134 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_XFD, MSR_TYPE_RW); 4135 4136 vmx_set_intercept_for_msr(vcpu, MSR_IA32_SPEC_CTRL, MSR_TYPE_RW, 4137 !to_vmx(vcpu)->spec_ctrl); 4138 4139 if (kvm_cpu_cap_has(X86_FEATURE_XFD)) 4140 vmx_set_intercept_for_msr(vcpu, MSR_IA32_XFD_ERR, MSR_TYPE_R, 4141 !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD)); 4142 4143 if (cpu_feature_enabled(X86_FEATURE_IBPB)) 4144 vmx_set_intercept_for_msr(vcpu, MSR_IA32_PRED_CMD, MSR_TYPE_W, 4145 !guest_has_pred_cmd_msr(vcpu)); 4146 4147 if (cpu_feature_enabled(X86_FEATURE_FLUSH_L1D)) 4148 vmx_set_intercept_for_msr(vcpu, MSR_IA32_FLUSH_CMD, MSR_TYPE_W, 4149 !guest_cpu_cap_has(vcpu, X86_FEATURE_FLUSH_L1D)); 4150 4151 if (kvm_cpu_cap_has(X86_FEATURE_SHSTK)) { 4152 intercept = !guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK); 4153 4154 vmx_set_intercept_for_msr(vcpu, MSR_IA32_PL0_SSP, MSR_TYPE_RW, intercept); 4155 vmx_set_intercept_for_msr(vcpu, MSR_IA32_PL1_SSP, MSR_TYPE_RW, intercept); 4156 vmx_set_intercept_for_msr(vcpu, MSR_IA32_PL2_SSP, MSR_TYPE_RW, intercept); 4157 vmx_set_intercept_for_msr(vcpu, MSR_IA32_PL3_SSP, MSR_TYPE_RW, intercept); 4158 } 4159 4160 if (kvm_cpu_cap_has(X86_FEATURE_SHSTK) || kvm_cpu_cap_has(X86_FEATURE_IBT)) { 4161 intercept = !guest_cpu_cap_has(vcpu, X86_FEATURE_IBT) && 4162 !guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK); 4163 4164 vmx_set_intercept_for_msr(vcpu, MSR_IA32_U_CET, MSR_TYPE_RW, intercept); 4165 vmx_set_intercept_for_msr(vcpu, MSR_IA32_S_CET, MSR_TYPE_RW, intercept); 4166 } 4167 4168 /* 4169 * x2APIC and LBR MSR intercepts are modified on-demand and cannot be 4170 * filtered by userspace. 4171 */ 4172 } 4173 4174 void vmx_recalc_intercepts(struct kvm_vcpu *vcpu) 4175 { 4176 vmx_recalc_msr_intercepts(vcpu); 4177 } 4178 4179 static int vmx_deliver_nested_posted_interrupt(struct kvm_vcpu *vcpu, 4180 int vector) 4181 { 4182 struct vcpu_vmx *vmx = to_vmx(vcpu); 4183 4184 /* 4185 * DO NOT query the vCPU's vmcs12, as vmcs12 is dynamically allocated 4186 * and freed, and must not be accessed outside of vcpu->mutex. The 4187 * vCPU's cached PI NV is valid if and only if posted interrupts 4188 * enabled in its vmcs12, i.e. checking the vector also checks that 4189 * L1 has enabled posted interrupts for L2. 4190 */ 4191 if (is_guest_mode(vcpu) && 4192 vector == vmx->nested.posted_intr_nv) { 4193 /* 4194 * If a posted intr is not recognized by hardware, 4195 * we will accomplish it in the next vmentry. 4196 */ 4197 vmx->nested.pi_pending = true; 4198 kvm_make_request(KVM_REQ_EVENT, vcpu); 4199 4200 /* 4201 * This pairs with the smp_mb_*() after setting vcpu->mode in 4202 * vcpu_enter_guest() to guarantee the vCPU sees the event 4203 * request if triggering a posted interrupt "fails" because 4204 * vcpu->mode != IN_GUEST_MODE. The extra barrier is needed as 4205 * the smb_wmb() in kvm_make_request() only ensures everything 4206 * done before making the request is visible when the request 4207 * is visible, it doesn't ensure ordering between the store to 4208 * vcpu->requests and the load from vcpu->mode. 4209 */ 4210 smp_mb__after_atomic(); 4211 4212 /* the PIR and ON have been set by L1. */ 4213 kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_NESTED_VECTOR); 4214 return 0; 4215 } 4216 return -1; 4217 } 4218 /* 4219 * Send interrupt to vcpu via posted interrupt way. 4220 * 1. If target vcpu is running(non-root mode), send posted interrupt 4221 * notification to vcpu and hardware will sync PIR to vIRR atomically. 4222 * 2. If target vcpu isn't running(root mode), kick it to pick up the 4223 * interrupt from PIR in next vmentry. 4224 */ 4225 static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector) 4226 { 4227 struct vcpu_vt *vt = to_vt(vcpu); 4228 int r; 4229 4230 r = vmx_deliver_nested_posted_interrupt(vcpu, vector); 4231 if (!r) 4232 return 0; 4233 4234 /* Note, this is called iff the local APIC is in-kernel. */ 4235 if (!vcpu->arch.apic->apicv_active) 4236 return -1; 4237 4238 __vmx_deliver_posted_interrupt(vcpu, &vt->pi_desc, vector); 4239 return 0; 4240 } 4241 4242 void vmx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, 4243 int trig_mode, int vector) 4244 { 4245 struct kvm_vcpu *vcpu = apic->vcpu; 4246 4247 if (vmx_deliver_posted_interrupt(vcpu, vector)) { 4248 kvm_lapic_set_irr(vector, apic); 4249 kvm_make_request(KVM_REQ_EVENT, vcpu); 4250 kvm_vcpu_kick(vcpu); 4251 } else { 4252 trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, 4253 trig_mode, vector); 4254 } 4255 } 4256 4257 /* 4258 * Set up the vmcs's constant host-state fields, i.e., host-state fields that 4259 * will not change in the lifetime of the guest. 4260 * Note that host-state that does change is set elsewhere. E.g., host-state 4261 * that is set differently for each CPU is set in vmx_vcpu_load(), not here. 4262 */ 4263 void vmx_set_constant_host_state(struct vcpu_vmx *vmx) 4264 { 4265 u32 low32, high32; 4266 unsigned long tmpl; 4267 unsigned long cr0, cr3, cr4; 4268 4269 cr0 = read_cr0(); 4270 WARN_ON(cr0 & X86_CR0_TS); 4271 vmcs_writel(HOST_CR0, cr0); /* 22.2.3 */ 4272 4273 /* 4274 * Save the most likely value for this task's CR3 in the VMCS. 4275 * We can't use __get_current_cr3_fast() because we're not atomic. 4276 */ 4277 cr3 = __read_cr3(); 4278 vmcs_writel(HOST_CR3, cr3); /* 22.2.3 FIXME: shadow tables */ 4279 vmx->loaded_vmcs->host_state.cr3 = cr3; 4280 4281 /* Save the most likely value for this task's CR4 in the VMCS. */ 4282 cr4 = cr4_read_shadow(); 4283 vmcs_writel(HOST_CR4, cr4); /* 22.2.3, 22.2.5 */ 4284 vmx->loaded_vmcs->host_state.cr4 = cr4; 4285 4286 vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS); /* 22.2.4 */ 4287 #ifdef CONFIG_X86_64 4288 /* 4289 * Load null selectors, so we can avoid reloading them in 4290 * vmx_prepare_switch_to_host(), in case userspace uses 4291 * the null selectors too (the expected case). 4292 */ 4293 vmcs_write16(HOST_DS_SELECTOR, 0); 4294 vmcs_write16(HOST_ES_SELECTOR, 0); 4295 #else 4296 vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS); /* 22.2.4 */ 4297 vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS); /* 22.2.4 */ 4298 #endif 4299 vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS); /* 22.2.4 */ 4300 vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8); /* 22.2.4 */ 4301 4302 vmcs_writel(HOST_IDTR_BASE, host_idt_base); /* 22.2.4 */ 4303 4304 vmcs_writel(HOST_RIP, (unsigned long)vmx_vmexit); /* 22.2.5 */ 4305 4306 rdmsr(MSR_IA32_SYSENTER_CS, low32, high32); 4307 vmcs_write32(HOST_IA32_SYSENTER_CS, low32); 4308 4309 /* 4310 * SYSENTER is used for 32-bit system calls on either 32-bit or 4311 * 64-bit kernels. It is always zero If neither is allowed, otherwise 4312 * vmx_vcpu_load_vmcs loads it with the per-CPU entry stack (and may 4313 * have already done so!). 4314 */ 4315 if (!IS_ENABLED(CONFIG_IA32_EMULATION) && !IS_ENABLED(CONFIG_X86_32)) 4316 vmcs_writel(HOST_IA32_SYSENTER_ESP, 0); 4317 4318 rdmsrq(MSR_IA32_SYSENTER_EIP, tmpl); 4319 vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl); /* 22.2.3 */ 4320 4321 if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) { 4322 rdmsr(MSR_IA32_CR_PAT, low32, high32); 4323 vmcs_write64(HOST_IA32_PAT, low32 | ((u64) high32 << 32)); 4324 } 4325 4326 if (cpu_has_load_ia32_efer()) 4327 vmcs_write64(HOST_IA32_EFER, kvm_host.efer); 4328 4329 /* 4330 * Supervisor shadow stack is not enabled on host side, i.e., 4331 * host IA32_S_CET.SHSTK_EN bit is guaranteed to 0 now, per SDM 4332 * description(RDSSP instruction), SSP is not readable in CPL0, 4333 * so resetting the two registers to 0s at VM-Exit does no harm 4334 * to kernel execution. When execution flow exits to userspace, 4335 * SSP is reloaded from IA32_PL3_SSP. Check SDM Vol.2A/B Chapter 4336 * 3 and 4 for details. 4337 */ 4338 if (cpu_has_load_cet_ctrl()) { 4339 vmcs_writel(HOST_S_CET, kvm_host.s_cet); 4340 vmcs_writel(HOST_SSP, 0); 4341 vmcs_writel(HOST_INTR_SSP_TABLE, 0); 4342 } 4343 } 4344 4345 void set_cr4_guest_host_mask(struct vcpu_vmx *vmx) 4346 { 4347 struct kvm_vcpu *vcpu = &vmx->vcpu; 4348 4349 vcpu->arch.cr4_guest_owned_bits = KVM_POSSIBLE_CR4_GUEST_BITS & 4350 ~vcpu->arch.cr4_guest_rsvd_bits; 4351 if (!enable_ept) { 4352 vcpu->arch.cr4_guest_owned_bits &= ~X86_CR4_TLBFLUSH_BITS; 4353 vcpu->arch.cr4_guest_owned_bits &= ~X86_CR4_PDPTR_BITS; 4354 } 4355 if (is_guest_mode(&vmx->vcpu)) 4356 vcpu->arch.cr4_guest_owned_bits &= 4357 ~get_vmcs12(vcpu)->cr4_guest_host_mask; 4358 vmcs_writel(CR4_GUEST_HOST_MASK, ~vcpu->arch.cr4_guest_owned_bits); 4359 } 4360 4361 static u32 vmx_pin_based_exec_ctrl(struct vcpu_vmx *vmx) 4362 { 4363 u32 pin_based_exec_ctrl = vmcs_config.pin_based_exec_ctrl; 4364 4365 if (!kvm_vcpu_apicv_active(&vmx->vcpu)) 4366 pin_based_exec_ctrl &= ~PIN_BASED_POSTED_INTR; 4367 4368 if (!enable_vnmi) 4369 pin_based_exec_ctrl &= ~PIN_BASED_VIRTUAL_NMIS; 4370 4371 if (!enable_preemption_timer) 4372 pin_based_exec_ctrl &= ~PIN_BASED_VMX_PREEMPTION_TIMER; 4373 4374 return pin_based_exec_ctrl; 4375 } 4376 4377 static u32 vmx_get_initial_vmentry_ctrl(void) 4378 { 4379 u32 vmentry_ctrl = vmcs_config.vmentry_ctrl; 4380 4381 if (vmx_pt_mode_is_system()) 4382 vmentry_ctrl &= ~(VM_ENTRY_PT_CONCEAL_PIP | 4383 VM_ENTRY_LOAD_IA32_RTIT_CTL); 4384 /* 4385 * IA32e mode, and loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically. 4386 */ 4387 vmentry_ctrl &= ~(VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL | 4388 VM_ENTRY_LOAD_IA32_EFER | 4389 VM_ENTRY_IA32E_MODE); 4390 4391 return vmentry_ctrl; 4392 } 4393 4394 static u32 vmx_get_initial_vmexit_ctrl(void) 4395 { 4396 u32 vmexit_ctrl = vmcs_config.vmexit_ctrl; 4397 4398 /* 4399 * Not used by KVM and never set in vmcs01 or vmcs02, but emulated for 4400 * nested virtualization and thus allowed to be set in vmcs12. 4401 */ 4402 vmexit_ctrl &= ~(VM_EXIT_SAVE_IA32_PAT | VM_EXIT_SAVE_IA32_EFER | 4403 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER); 4404 4405 if (vmx_pt_mode_is_system()) 4406 vmexit_ctrl &= ~(VM_EXIT_PT_CONCEAL_PIP | 4407 VM_EXIT_CLEAR_IA32_RTIT_CTL); 4408 /* Loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically */ 4409 return vmexit_ctrl & 4410 ~(VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | VM_EXIT_LOAD_IA32_EFER); 4411 } 4412 4413 void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) 4414 { 4415 struct vcpu_vmx *vmx = to_vmx(vcpu); 4416 4417 if (is_guest_mode(vcpu)) { 4418 vmx->nested.update_vmcs01_apicv_status = true; 4419 return; 4420 } 4421 4422 pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx)); 4423 4424 secondary_exec_controls_changebit(vmx, 4425 SECONDARY_EXEC_APIC_REGISTER_VIRT | 4426 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY, 4427 kvm_vcpu_apicv_active(vcpu)); 4428 if (enable_ipiv) 4429 tertiary_exec_controls_changebit(vmx, TERTIARY_EXEC_IPI_VIRT, 4430 kvm_vcpu_apicv_active(vcpu)); 4431 4432 vmx_update_msr_bitmap_x2apic(vcpu); 4433 } 4434 4435 static u32 vmx_exec_control(struct vcpu_vmx *vmx) 4436 { 4437 u32 exec_control = vmcs_config.cpu_based_exec_ctrl; 4438 4439 /* 4440 * Not used by KVM, but fully supported for nesting, i.e. are allowed in 4441 * vmcs12 and propagated to vmcs02 when set in vmcs12. 4442 */ 4443 exec_control &= ~(CPU_BASED_RDTSC_EXITING | 4444 CPU_BASED_USE_IO_BITMAPS | 4445 CPU_BASED_MONITOR_TRAP_FLAG | 4446 CPU_BASED_PAUSE_EXITING); 4447 4448 /* INTR_WINDOW_EXITING and NMI_WINDOW_EXITING are toggled dynamically */ 4449 exec_control &= ~(CPU_BASED_INTR_WINDOW_EXITING | 4450 CPU_BASED_NMI_WINDOW_EXITING); 4451 4452 if (vmx->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT) 4453 exec_control &= ~CPU_BASED_MOV_DR_EXITING; 4454 4455 if (!cpu_need_tpr_shadow(&vmx->vcpu)) 4456 exec_control &= ~CPU_BASED_TPR_SHADOW; 4457 4458 #ifdef CONFIG_X86_64 4459 if (exec_control & CPU_BASED_TPR_SHADOW) 4460 exec_control &= ~(CPU_BASED_CR8_LOAD_EXITING | 4461 CPU_BASED_CR8_STORE_EXITING); 4462 else 4463 exec_control |= CPU_BASED_CR8_STORE_EXITING | 4464 CPU_BASED_CR8_LOAD_EXITING; 4465 #endif 4466 /* No need to intercept CR3 access or INVPLG when using EPT. */ 4467 if (enable_ept) 4468 exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING | 4469 CPU_BASED_CR3_STORE_EXITING | 4470 CPU_BASED_INVLPG_EXITING); 4471 if (kvm_mwait_in_guest(vmx->vcpu.kvm)) 4472 exec_control &= ~(CPU_BASED_MWAIT_EXITING | 4473 CPU_BASED_MONITOR_EXITING); 4474 if (kvm_hlt_in_guest(vmx->vcpu.kvm)) 4475 exec_control &= ~CPU_BASED_HLT_EXITING; 4476 return exec_control; 4477 } 4478 4479 static u64 vmx_tertiary_exec_control(struct vcpu_vmx *vmx) 4480 { 4481 u64 exec_control = vmcs_config.cpu_based_3rd_exec_ctrl; 4482 4483 /* 4484 * IPI virtualization relies on APICv. Disable IPI virtualization if 4485 * APICv is inhibited. 4486 */ 4487 if (!enable_ipiv || !kvm_vcpu_apicv_active(&vmx->vcpu)) 4488 exec_control &= ~TERTIARY_EXEC_IPI_VIRT; 4489 4490 return exec_control; 4491 } 4492 4493 /* 4494 * Adjust a single secondary execution control bit to intercept/allow an 4495 * instruction in the guest. This is usually done based on whether or not a 4496 * feature has been exposed to the guest in order to correctly emulate faults. 4497 */ 4498 static inline void 4499 vmx_adjust_secondary_exec_control(struct vcpu_vmx *vmx, u32 *exec_control, 4500 u32 control, bool enabled, bool exiting) 4501 { 4502 /* 4503 * If the control is for an opt-in feature, clear the control if the 4504 * feature is not exposed to the guest, i.e. not enabled. If the 4505 * control is opt-out, i.e. an exiting control, clear the control if 4506 * the feature _is_ exposed to the guest, i.e. exiting/interception is 4507 * disabled for the associated instruction. Note, the caller is 4508 * responsible presetting exec_control to set all supported bits. 4509 */ 4510 if (enabled == exiting) 4511 *exec_control &= ~control; 4512 4513 /* 4514 * Update the nested MSR settings so that a nested VMM can/can't set 4515 * controls for features that are/aren't exposed to the guest. 4516 */ 4517 if (nested && 4518 kvm_check_has_quirk(vmx->vcpu.kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS)) { 4519 /* 4520 * All features that can be added or removed to VMX MSRs must 4521 * be supported in the first place for nested virtualization. 4522 */ 4523 if (WARN_ON_ONCE(!(vmcs_config.nested.secondary_ctls_high & control))) 4524 enabled = false; 4525 4526 if (enabled) 4527 vmx->nested.msrs.secondary_ctls_high |= control; 4528 else 4529 vmx->nested.msrs.secondary_ctls_high &= ~control; 4530 } 4531 } 4532 4533 /* 4534 * Wrapper macro for the common case of adjusting a secondary execution control 4535 * based on a single guest CPUID bit, with a dedicated feature bit. This also 4536 * verifies that the control is actually supported by KVM and hardware. 4537 */ 4538 #define vmx_adjust_sec_exec_control(vmx, exec_control, name, feat_name, ctrl_name, exiting) \ 4539 ({ \ 4540 struct kvm_vcpu *__vcpu = &(vmx)->vcpu; \ 4541 bool __enabled; \ 4542 \ 4543 if (cpu_has_vmx_##name()) { \ 4544 __enabled = guest_cpu_cap_has(__vcpu, X86_FEATURE_##feat_name); \ 4545 vmx_adjust_secondary_exec_control(vmx, exec_control, SECONDARY_EXEC_##ctrl_name,\ 4546 __enabled, exiting); \ 4547 } \ 4548 }) 4549 4550 /* More macro magic for ENABLE_/opt-in versus _EXITING/opt-out controls. */ 4551 #define vmx_adjust_sec_exec_feature(vmx, exec_control, lname, uname) \ 4552 vmx_adjust_sec_exec_control(vmx, exec_control, lname, uname, ENABLE_##uname, false) 4553 4554 #define vmx_adjust_sec_exec_exiting(vmx, exec_control, lname, uname) \ 4555 vmx_adjust_sec_exec_control(vmx, exec_control, lname, uname, uname##_EXITING, true) 4556 4557 static u32 vmx_secondary_exec_control(struct vcpu_vmx *vmx) 4558 { 4559 struct kvm_vcpu *vcpu = &vmx->vcpu; 4560 4561 u32 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl; 4562 4563 if (vmx_pt_mode_is_system()) 4564 exec_control &= ~(SECONDARY_EXEC_PT_USE_GPA | SECONDARY_EXEC_PT_CONCEAL_VMX); 4565 if (!cpu_need_virtualize_apic_accesses(vcpu)) 4566 exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 4567 if (vmx->vpid == 0) 4568 exec_control &= ~SECONDARY_EXEC_ENABLE_VPID; 4569 if (!enable_ept) { 4570 exec_control &= ~SECONDARY_EXEC_ENABLE_EPT; 4571 exec_control &= ~SECONDARY_EXEC_EPT_VIOLATION_VE; 4572 enable_unrestricted_guest = 0; 4573 } 4574 if (!enable_unrestricted_guest) 4575 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST; 4576 if (kvm_pause_in_guest(vmx->vcpu.kvm)) 4577 exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING; 4578 if (!kvm_vcpu_apicv_active(vcpu)) 4579 exec_control &= ~(SECONDARY_EXEC_APIC_REGISTER_VIRT | 4580 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 4581 exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE; 4582 4583 /* 4584 * KVM doesn't support VMFUNC for L1, but the control is set in KVM's 4585 * base configuration as KVM emulates VMFUNC[EPTP_SWITCHING] for L2. 4586 */ 4587 exec_control &= ~SECONDARY_EXEC_ENABLE_VMFUNC; 4588 4589 /* SECONDARY_EXEC_DESC is enabled/disabled on writes to CR4.UMIP, 4590 * in vmx_set_cr4. */ 4591 exec_control &= ~SECONDARY_EXEC_DESC; 4592 4593 /* SECONDARY_EXEC_SHADOW_VMCS is enabled when L1 executes VMPTRLD 4594 (handle_vmptrld). 4595 We can NOT enable shadow_vmcs here because we don't have yet 4596 a current VMCS12 4597 */ 4598 exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS; 4599 4600 /* 4601 * PML is enabled/disabled when dirty logging of memsmlots changes, but 4602 * it needs to be set here when dirty logging is already active, e.g. 4603 * if this vCPU was created after dirty logging was enabled. 4604 */ 4605 if (!enable_pml || !atomic_read(&vcpu->kvm->nr_memslots_dirty_logging)) 4606 exec_control &= ~SECONDARY_EXEC_ENABLE_PML; 4607 4608 vmx_adjust_sec_exec_feature(vmx, &exec_control, xsaves, XSAVES); 4609 4610 /* 4611 * RDPID is also gated by ENABLE_RDTSCP, turn on the control if either 4612 * feature is exposed to the guest. This creates a virtualization hole 4613 * if both are supported in hardware but only one is exposed to the 4614 * guest, but letting the guest execute RDTSCP or RDPID when either one 4615 * is advertised is preferable to emulating the advertised instruction 4616 * in KVM on #UD, and obviously better than incorrectly injecting #UD. 4617 */ 4618 if (cpu_has_vmx_rdtscp()) { 4619 bool rdpid_or_rdtscp_enabled = 4620 guest_cpu_cap_has(vcpu, X86_FEATURE_RDTSCP) || 4621 guest_cpu_cap_has(vcpu, X86_FEATURE_RDPID); 4622 4623 vmx_adjust_secondary_exec_control(vmx, &exec_control, 4624 SECONDARY_EXEC_ENABLE_RDTSCP, 4625 rdpid_or_rdtscp_enabled, false); 4626 } 4627 4628 vmx_adjust_sec_exec_feature(vmx, &exec_control, invpcid, INVPCID); 4629 4630 vmx_adjust_sec_exec_exiting(vmx, &exec_control, rdrand, RDRAND); 4631 vmx_adjust_sec_exec_exiting(vmx, &exec_control, rdseed, RDSEED); 4632 4633 vmx_adjust_sec_exec_control(vmx, &exec_control, waitpkg, WAITPKG, 4634 ENABLE_USR_WAIT_PAUSE, false); 4635 4636 if (!vcpu->kvm->arch.bus_lock_detection_enabled) 4637 exec_control &= ~SECONDARY_EXEC_BUS_LOCK_DETECTION; 4638 4639 if (!kvm_notify_vmexit_enabled(vcpu->kvm)) 4640 exec_control &= ~SECONDARY_EXEC_NOTIFY_VM_EXITING; 4641 4642 return exec_control; 4643 } 4644 4645 static inline int vmx_get_pid_table_order(struct kvm *kvm) 4646 { 4647 return get_order(kvm->arch.max_vcpu_ids * sizeof(*to_kvm_vmx(kvm)->pid_table)); 4648 } 4649 4650 static int vmx_alloc_ipiv_pid_table(struct kvm *kvm) 4651 { 4652 struct page *pages; 4653 struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); 4654 4655 if (!irqchip_in_kernel(kvm) || !enable_ipiv) 4656 return 0; 4657 4658 if (kvm_vmx->pid_table) 4659 return 0; 4660 4661 pages = alloc_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO, 4662 vmx_get_pid_table_order(kvm)); 4663 if (!pages) 4664 return -ENOMEM; 4665 4666 kvm_vmx->pid_table = (void *)page_address(pages); 4667 return 0; 4668 } 4669 4670 int vmx_vcpu_precreate(struct kvm *kvm) 4671 { 4672 return vmx_alloc_ipiv_pid_table(kvm); 4673 } 4674 4675 #define VMX_XSS_EXIT_BITMAP 0 4676 4677 static void init_vmcs(struct vcpu_vmx *vmx) 4678 { 4679 struct kvm *kvm = vmx->vcpu.kvm; 4680 struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); 4681 4682 if (nested) 4683 nested_vmx_set_vmcs_shadowing_bitmap(); 4684 4685 if (cpu_has_vmx_msr_bitmap()) 4686 vmcs_write64(MSR_BITMAP, __pa(vmx->vmcs01.msr_bitmap)); 4687 4688 vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); /* 22.3.1.5 */ 4689 4690 /* Control */ 4691 pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx)); 4692 4693 exec_controls_set(vmx, vmx_exec_control(vmx)); 4694 4695 if (cpu_has_secondary_exec_ctrls()) { 4696 secondary_exec_controls_set(vmx, vmx_secondary_exec_control(vmx)); 4697 if (vmx->ve_info) 4698 vmcs_write64(VE_INFORMATION_ADDRESS, 4699 __pa(vmx->ve_info)); 4700 } 4701 4702 if (cpu_has_tertiary_exec_ctrls()) 4703 tertiary_exec_controls_set(vmx, vmx_tertiary_exec_control(vmx)); 4704 4705 if (enable_apicv && lapic_in_kernel(&vmx->vcpu)) { 4706 vmcs_write64(EOI_EXIT_BITMAP0, 0); 4707 vmcs_write64(EOI_EXIT_BITMAP1, 0); 4708 vmcs_write64(EOI_EXIT_BITMAP2, 0); 4709 vmcs_write64(EOI_EXIT_BITMAP3, 0); 4710 4711 vmcs_write16(GUEST_INTR_STATUS, 0); 4712 4713 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR); 4714 vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->vt.pi_desc))); 4715 } 4716 4717 if (vmx_can_use_ipiv(&vmx->vcpu)) { 4718 vmcs_write64(PID_POINTER_TABLE, __pa(kvm_vmx->pid_table)); 4719 vmcs_write16(LAST_PID_POINTER_INDEX, kvm->arch.max_vcpu_ids - 1); 4720 } 4721 4722 if (!kvm_pause_in_guest(kvm)) { 4723 vmcs_write32(PLE_GAP, ple_gap); 4724 vmx->ple_window = ple_window; 4725 vmx->ple_window_dirty = true; 4726 } 4727 4728 if (kvm_notify_vmexit_enabled(kvm)) 4729 vmcs_write32(NOTIFY_WINDOW, kvm->arch.notify_window); 4730 4731 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0); 4732 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0); 4733 vmcs_write32(CR3_TARGET_COUNT, 0); /* 22.2.1 */ 4734 4735 vmcs_write16(HOST_FS_SELECTOR, 0); /* 22.2.4 */ 4736 vmcs_write16(HOST_GS_SELECTOR, 0); /* 22.2.4 */ 4737 vmx_set_constant_host_state(vmx); 4738 vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */ 4739 vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */ 4740 4741 if (cpu_has_vmx_vmfunc()) 4742 vmcs_write64(VM_FUNCTION_CONTROL, 0); 4743 4744 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0); 4745 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0); 4746 vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val)); 4747 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0); 4748 vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); 4749 4750 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) 4751 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); 4752 4753 vm_exit_controls_set(vmx, vmx_get_initial_vmexit_ctrl()); 4754 4755 /* 22.2.1, 20.8.1 */ 4756 vm_entry_controls_set(vmx, vmx_get_initial_vmentry_ctrl()); 4757 4758 vmx->vcpu.arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits(); 4759 vmcs_writel(CR0_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr0_guest_owned_bits); 4760 4761 set_cr4_guest_host_mask(vmx); 4762 4763 if (vmx->vpid != 0) 4764 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid); 4765 4766 if (cpu_has_vmx_xsaves()) 4767 vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP); 4768 4769 if (enable_pml) { 4770 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg)); 4771 vmcs_write16(GUEST_PML_INDEX, PML_HEAD_INDEX); 4772 } 4773 4774 vmx_write_encls_bitmap(&vmx->vcpu, NULL); 4775 4776 if (vmx_pt_mode_is_host_guest()) { 4777 memset(&vmx->pt_desc, 0, sizeof(vmx->pt_desc)); 4778 /* Bit[6~0] are forced to 1, writes are ignored. */ 4779 vmx->pt_desc.guest.output_mask = 0x7F; 4780 vmcs_write64(GUEST_IA32_RTIT_CTL, 0); 4781 } 4782 4783 vmcs_write32(GUEST_SYSENTER_CS, 0); 4784 vmcs_writel(GUEST_SYSENTER_ESP, 0); 4785 vmcs_writel(GUEST_SYSENTER_EIP, 0); 4786 4787 vmx_guest_debugctl_write(&vmx->vcpu, 0); 4788 4789 if (cpu_has_vmx_tpr_shadow()) { 4790 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0); 4791 if (cpu_need_tpr_shadow(&vmx->vcpu)) 4792 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 4793 __pa(vmx->vcpu.arch.apic->regs)); 4794 vmcs_write32(TPR_THRESHOLD, 0); 4795 } 4796 4797 vmx_setup_uret_msrs(vmx); 4798 } 4799 4800 static void __vmx_vcpu_reset(struct kvm_vcpu *vcpu) 4801 { 4802 struct vcpu_vmx *vmx = to_vmx(vcpu); 4803 4804 init_vmcs(vmx); 4805 4806 if (nested && 4807 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS)) 4808 memcpy(&vmx->nested.msrs, &vmcs_config.nested, sizeof(vmx->nested.msrs)); 4809 4810 vcpu_setup_sgx_lepubkeyhash(vcpu); 4811 4812 vmx->nested.posted_intr_nv = -1; 4813 vmx->nested.vmxon_ptr = INVALID_GPA; 4814 vmx->nested.current_vmptr = INVALID_GPA; 4815 4816 #ifdef CONFIG_KVM_HYPERV 4817 vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID; 4818 #endif 4819 4820 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS)) 4821 vcpu->arch.microcode_version = 0x100000000ULL; 4822 vmx->msr_ia32_feature_control_valid_bits = FEAT_CTL_LOCKED; 4823 4824 /* 4825 * Enforce invariant: pi_desc.nv is always either POSTED_INTR_VECTOR 4826 * or POSTED_INTR_WAKEUP_VECTOR. 4827 */ 4828 vmx->vt.pi_desc.nv = POSTED_INTR_VECTOR; 4829 __pi_set_sn(&vmx->vt.pi_desc); 4830 } 4831 4832 void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) 4833 { 4834 struct vcpu_vmx *vmx = to_vmx(vcpu); 4835 4836 if (!init_event) 4837 __vmx_vcpu_reset(vcpu); 4838 4839 vmx->rmode.vm86_active = 0; 4840 vmx->spec_ctrl = 0; 4841 4842 vmx->msr_ia32_umwait_control = 0; 4843 4844 vmx->hv_deadline_tsc = -1; 4845 kvm_set_cr8(vcpu, 0); 4846 4847 seg_setup(VCPU_SREG_CS); 4848 vmcs_write16(GUEST_CS_SELECTOR, 0xf000); 4849 vmcs_writel(GUEST_CS_BASE, 0xffff0000ul); 4850 4851 seg_setup(VCPU_SREG_DS); 4852 seg_setup(VCPU_SREG_ES); 4853 seg_setup(VCPU_SREG_FS); 4854 seg_setup(VCPU_SREG_GS); 4855 seg_setup(VCPU_SREG_SS); 4856 4857 vmcs_write16(GUEST_TR_SELECTOR, 0); 4858 vmcs_writel(GUEST_TR_BASE, 0); 4859 vmcs_write32(GUEST_TR_LIMIT, 0xffff); 4860 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); 4861 4862 vmcs_write16(GUEST_LDTR_SELECTOR, 0); 4863 vmcs_writel(GUEST_LDTR_BASE, 0); 4864 vmcs_write32(GUEST_LDTR_LIMIT, 0xffff); 4865 vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082); 4866 4867 vmcs_writel(GUEST_GDTR_BASE, 0); 4868 vmcs_write32(GUEST_GDTR_LIMIT, 0xffff); 4869 4870 vmcs_writel(GUEST_IDTR_BASE, 0); 4871 vmcs_write32(GUEST_IDTR_LIMIT, 0xffff); 4872 4873 vmx_segment_cache_clear(vmx); 4874 kvm_register_mark_available(vcpu, VCPU_EXREG_SEGMENTS); 4875 4876 vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE); 4877 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0); 4878 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 0); 4879 if (kvm_mpx_supported()) 4880 vmcs_write64(GUEST_BNDCFGS, 0); 4881 4882 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); /* 22.2.1 */ 4883 4884 if (kvm_cpu_cap_has(X86_FEATURE_SHSTK)) { 4885 vmcs_writel(GUEST_SSP, 0); 4886 vmcs_writel(GUEST_INTR_SSP_TABLE, 0); 4887 } 4888 if (kvm_cpu_cap_has(X86_FEATURE_IBT) || 4889 kvm_cpu_cap_has(X86_FEATURE_SHSTK)) 4890 vmcs_writel(GUEST_S_CET, 0); 4891 4892 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 4893 4894 vpid_sync_context(vmx->vpid); 4895 4896 vmx_update_fb_clear_dis(vcpu, vmx); 4897 } 4898 4899 void vmx_enable_irq_window(struct kvm_vcpu *vcpu) 4900 { 4901 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING); 4902 } 4903 4904 void vmx_enable_nmi_window(struct kvm_vcpu *vcpu) 4905 { 4906 if (!enable_vnmi || 4907 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) { 4908 vmx_enable_irq_window(vcpu); 4909 return; 4910 } 4911 4912 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_NMI_WINDOW_EXITING); 4913 } 4914 4915 void vmx_inject_irq(struct kvm_vcpu *vcpu, bool reinjected) 4916 { 4917 struct vcpu_vmx *vmx = to_vmx(vcpu); 4918 uint32_t intr; 4919 int irq = vcpu->arch.interrupt.nr; 4920 4921 trace_kvm_inj_virq(irq, vcpu->arch.interrupt.soft, reinjected); 4922 4923 ++vcpu->stat.irq_injections; 4924 if (vmx->rmode.vm86_active) { 4925 int inc_eip = 0; 4926 if (vcpu->arch.interrupt.soft) 4927 inc_eip = vcpu->arch.event_exit_inst_len; 4928 kvm_inject_realmode_interrupt(vcpu, irq, inc_eip); 4929 return; 4930 } 4931 intr = irq | INTR_INFO_VALID_MASK; 4932 if (vcpu->arch.interrupt.soft) { 4933 intr |= INTR_TYPE_SOFT_INTR; 4934 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 4935 vmx->vcpu.arch.event_exit_inst_len); 4936 } else 4937 intr |= INTR_TYPE_EXT_INTR; 4938 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr); 4939 4940 vmx_clear_hlt(vcpu); 4941 } 4942 4943 void vmx_inject_nmi(struct kvm_vcpu *vcpu) 4944 { 4945 struct vcpu_vmx *vmx = to_vmx(vcpu); 4946 4947 if (!enable_vnmi) { 4948 /* 4949 * Tracking the NMI-blocked state in software is built upon 4950 * finding the next open IRQ window. This, in turn, depends on 4951 * well-behaving guests: They have to keep IRQs disabled at 4952 * least as long as the NMI handler runs. Otherwise we may 4953 * cause NMI nesting, maybe breaking the guest. But as this is 4954 * highly unlikely, we can live with the residual risk. 4955 */ 4956 vmx->loaded_vmcs->soft_vnmi_blocked = 1; 4957 vmx->loaded_vmcs->vnmi_blocked_time = 0; 4958 } 4959 4960 ++vcpu->stat.nmi_injections; 4961 vmx->loaded_vmcs->nmi_known_unmasked = false; 4962 4963 if (vmx->rmode.vm86_active) { 4964 kvm_inject_realmode_interrupt(vcpu, NMI_VECTOR, 0); 4965 return; 4966 } 4967 4968 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 4969 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR); 4970 4971 vmx_clear_hlt(vcpu); 4972 } 4973 4974 bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu) 4975 { 4976 struct vcpu_vmx *vmx = to_vmx(vcpu); 4977 bool masked; 4978 4979 if (!enable_vnmi) 4980 return vmx->loaded_vmcs->soft_vnmi_blocked; 4981 if (vmx->loaded_vmcs->nmi_known_unmasked) 4982 return false; 4983 masked = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_NMI; 4984 vmx->loaded_vmcs->nmi_known_unmasked = !masked; 4985 return masked; 4986 } 4987 4988 void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) 4989 { 4990 struct vcpu_vmx *vmx = to_vmx(vcpu); 4991 4992 if (!enable_vnmi) { 4993 if (vmx->loaded_vmcs->soft_vnmi_blocked != masked) { 4994 vmx->loaded_vmcs->soft_vnmi_blocked = masked; 4995 vmx->loaded_vmcs->vnmi_blocked_time = 0; 4996 } 4997 } else { 4998 vmx->loaded_vmcs->nmi_known_unmasked = !masked; 4999 if (masked) 5000 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 5001 GUEST_INTR_STATE_NMI); 5002 else 5003 vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO, 5004 GUEST_INTR_STATE_NMI); 5005 } 5006 } 5007 5008 bool vmx_nmi_blocked(struct kvm_vcpu *vcpu) 5009 { 5010 if (is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu)) 5011 return false; 5012 5013 if (!enable_vnmi && to_vmx(vcpu)->loaded_vmcs->soft_vnmi_blocked) 5014 return true; 5015 5016 return (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 5017 (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI | 5018 GUEST_INTR_STATE_NMI)); 5019 } 5020 5021 int vmx_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 5022 { 5023 if (to_vmx(vcpu)->nested.nested_run_pending) 5024 return -EBUSY; 5025 5026 /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */ 5027 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu)) 5028 return -EBUSY; 5029 5030 return !vmx_nmi_blocked(vcpu); 5031 } 5032 5033 bool __vmx_interrupt_blocked(struct kvm_vcpu *vcpu) 5034 { 5035 return !(vmx_get_rflags(vcpu) & X86_EFLAGS_IF) || 5036 (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 5037 (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS)); 5038 } 5039 5040 bool vmx_interrupt_blocked(struct kvm_vcpu *vcpu) 5041 { 5042 if (is_guest_mode(vcpu) && nested_exit_on_intr(vcpu)) 5043 return false; 5044 5045 return __vmx_interrupt_blocked(vcpu); 5046 } 5047 5048 int vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) 5049 { 5050 if (to_vmx(vcpu)->nested.nested_run_pending) 5051 return -EBUSY; 5052 5053 /* 5054 * An IRQ must not be injected into L2 if it's supposed to VM-Exit, 5055 * e.g. if the IRQ arrived asynchronously after checking nested events. 5056 */ 5057 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(vcpu)) 5058 return -EBUSY; 5059 5060 return !vmx_interrupt_blocked(vcpu); 5061 } 5062 5063 int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr) 5064 { 5065 void __user *ret; 5066 5067 if (enable_unrestricted_guest) 5068 return 0; 5069 5070 mutex_lock(&kvm->slots_lock); 5071 ret = __x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, addr, 5072 PAGE_SIZE * 3); 5073 mutex_unlock(&kvm->slots_lock); 5074 5075 if (IS_ERR(ret)) 5076 return PTR_ERR(ret); 5077 5078 to_kvm_vmx(kvm)->tss_addr = addr; 5079 5080 return init_rmode_tss(kvm, ret); 5081 } 5082 5083 int vmx_set_identity_map_addr(struct kvm *kvm, u64 ident_addr) 5084 { 5085 to_kvm_vmx(kvm)->ept_identity_map_addr = ident_addr; 5086 return 0; 5087 } 5088 5089 static bool rmode_exception(struct kvm_vcpu *vcpu, int vec) 5090 { 5091 switch (vec) { 5092 case BP_VECTOR: 5093 /* 5094 * Update instruction length as we may reinject the exception 5095 * from user space while in guest debugging mode. 5096 */ 5097 to_vmx(vcpu)->vcpu.arch.event_exit_inst_len = 5098 vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 5099 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) 5100 return false; 5101 fallthrough; 5102 case DB_VECTOR: 5103 return !(vcpu->guest_debug & 5104 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)); 5105 case DE_VECTOR: 5106 case OF_VECTOR: 5107 case BR_VECTOR: 5108 case UD_VECTOR: 5109 case DF_VECTOR: 5110 case SS_VECTOR: 5111 case GP_VECTOR: 5112 case MF_VECTOR: 5113 return true; 5114 } 5115 return false; 5116 } 5117 5118 static int handle_rmode_exception(struct kvm_vcpu *vcpu, 5119 int vec, u32 err_code) 5120 { 5121 /* 5122 * Instruction with address size override prefix opcode 0x67 5123 * Cause the #SS fault with 0 error code in VM86 mode. 5124 */ 5125 if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) { 5126 if (kvm_emulate_instruction(vcpu, 0)) { 5127 if (vcpu->arch.halt_request) { 5128 vcpu->arch.halt_request = 0; 5129 return kvm_emulate_halt_noskip(vcpu); 5130 } 5131 return 1; 5132 } 5133 return 0; 5134 } 5135 5136 /* 5137 * Forward all other exceptions that are valid in real mode. 5138 * FIXME: Breaks guest debugging in real mode, needs to be fixed with 5139 * the required debugging infrastructure rework. 5140 */ 5141 kvm_queue_exception(vcpu, vec); 5142 return 1; 5143 } 5144 5145 static int handle_machine_check(struct kvm_vcpu *vcpu) 5146 { 5147 /* handled by vmx_vcpu_run() */ 5148 return 1; 5149 } 5150 5151 /* 5152 * If the host has split lock detection disabled, then #AC is 5153 * unconditionally injected into the guest, which is the pre split lock 5154 * detection behaviour. 5155 * 5156 * If the host has split lock detection enabled then #AC is 5157 * only injected into the guest when: 5158 * - Guest CPL == 3 (user mode) 5159 * - Guest has #AC detection enabled in CR0 5160 * - Guest EFLAGS has AC bit set 5161 */ 5162 bool vmx_guest_inject_ac(struct kvm_vcpu *vcpu) 5163 { 5164 if (!boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT)) 5165 return true; 5166 5167 return vmx_get_cpl(vcpu) == 3 && kvm_is_cr0_bit_set(vcpu, X86_CR0_AM) && 5168 (kvm_get_rflags(vcpu) & X86_EFLAGS_AC); 5169 } 5170 5171 static bool is_xfd_nm_fault(struct kvm_vcpu *vcpu) 5172 { 5173 return vcpu->arch.guest_fpu.fpstate->xfd && 5174 !kvm_is_cr0_bit_set(vcpu, X86_CR0_TS); 5175 } 5176 5177 static int handle_exception_nmi(struct kvm_vcpu *vcpu) 5178 { 5179 struct vcpu_vmx *vmx = to_vmx(vcpu); 5180 struct kvm_run *kvm_run = vcpu->run; 5181 u32 intr_info, ex_no, error_code; 5182 unsigned long cr2, dr6; 5183 u32 vect_info; 5184 5185 vect_info = vmx->idt_vectoring_info; 5186 intr_info = vmx_get_intr_info(vcpu); 5187 5188 /* 5189 * Machine checks are handled by handle_exception_irqoff(), or by 5190 * vmx_vcpu_run() if a #MC occurs on VM-Entry. NMIs are handled by 5191 * vmx_vcpu_enter_exit(). 5192 */ 5193 if (is_machine_check(intr_info) || is_nmi(intr_info)) 5194 return 1; 5195 5196 /* 5197 * Queue the exception here instead of in handle_nm_fault_irqoff(). 5198 * This ensures the nested_vmx check is not skipped so vmexit can 5199 * be reflected to L1 (when it intercepts #NM) before reaching this 5200 * point. 5201 */ 5202 if (is_nm_fault(intr_info)) { 5203 kvm_queue_exception_p(vcpu, NM_VECTOR, 5204 is_xfd_nm_fault(vcpu) ? vcpu->arch.guest_fpu.xfd_err : 0); 5205 return 1; 5206 } 5207 5208 if (is_invalid_opcode(intr_info)) 5209 return handle_ud(vcpu); 5210 5211 if (WARN_ON_ONCE(is_ve_fault(intr_info))) { 5212 struct vmx_ve_information *ve_info = vmx->ve_info; 5213 5214 WARN_ONCE(ve_info->exit_reason != EXIT_REASON_EPT_VIOLATION, 5215 "Unexpected #VE on VM-Exit reason 0x%x", ve_info->exit_reason); 5216 dump_vmcs(vcpu); 5217 kvm_mmu_print_sptes(vcpu, ve_info->guest_physical_address, "#VE"); 5218 return 1; 5219 } 5220 5221 error_code = 0; 5222 if (intr_info & INTR_INFO_DELIVER_CODE_MASK) 5223 error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); 5224 5225 if (!vmx->rmode.vm86_active && is_gp_fault(intr_info)) { 5226 WARN_ON_ONCE(!enable_vmware_backdoor); 5227 5228 /* 5229 * VMware backdoor emulation on #GP interception only handles 5230 * IN{S}, OUT{S}, and RDPMC, none of which generate a non-zero 5231 * error code on #GP. 5232 */ 5233 if (error_code) { 5234 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code); 5235 return 1; 5236 } 5237 return kvm_emulate_instruction(vcpu, EMULTYPE_VMWARE_GP); 5238 } 5239 5240 /* 5241 * The #PF with PFEC.RSVD = 1 indicates the guest is accessing 5242 * MMIO, it is better to report an internal error. 5243 * See the comments in vmx_handle_exit. 5244 */ 5245 if ((vect_info & VECTORING_INFO_VALID_MASK) && 5246 !(is_page_fault(intr_info) && !(error_code & PFERR_RSVD_MASK))) { 5247 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 5248 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX; 5249 vcpu->run->internal.ndata = 4; 5250 vcpu->run->internal.data[0] = vect_info; 5251 vcpu->run->internal.data[1] = intr_info; 5252 vcpu->run->internal.data[2] = error_code; 5253 vcpu->run->internal.data[3] = vcpu->arch.last_vmentry_cpu; 5254 return 0; 5255 } 5256 5257 if (is_page_fault(intr_info)) { 5258 cr2 = vmx_get_exit_qual(vcpu); 5259 if (enable_ept && !vcpu->arch.apf.host_apf_flags) { 5260 /* 5261 * EPT will cause page fault only if we need to 5262 * detect illegal GPAs. 5263 */ 5264 WARN_ON_ONCE(!allow_smaller_maxphyaddr); 5265 kvm_fixup_and_inject_pf_error(vcpu, cr2, error_code); 5266 return 1; 5267 } else 5268 return kvm_handle_page_fault(vcpu, error_code, cr2, NULL, 0); 5269 } 5270 5271 ex_no = intr_info & INTR_INFO_VECTOR_MASK; 5272 5273 if (vmx->rmode.vm86_active && rmode_exception(vcpu, ex_no)) 5274 return handle_rmode_exception(vcpu, ex_no, error_code); 5275 5276 switch (ex_no) { 5277 case DB_VECTOR: 5278 dr6 = vmx_get_exit_qual(vcpu); 5279 if (!(vcpu->guest_debug & 5280 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) { 5281 /* 5282 * If the #DB was due to ICEBP, a.k.a. INT1, skip the 5283 * instruction. ICEBP generates a trap-like #DB, but 5284 * despite its interception control being tied to #DB, 5285 * is an instruction intercept, i.e. the VM-Exit occurs 5286 * on the ICEBP itself. Use the inner "skip" helper to 5287 * avoid single-step #DB and MTF updates, as ICEBP is 5288 * higher priority. Note, skipping ICEBP still clears 5289 * STI and MOVSS blocking. 5290 * 5291 * For all other #DBs, set vmcs.PENDING_DBG_EXCEPTIONS.BS 5292 * if single-step is enabled in RFLAGS and STI or MOVSS 5293 * blocking is active, as the CPU doesn't set the bit 5294 * on VM-Exit due to #DB interception. VM-Entry has a 5295 * consistency check that a single-step #DB is pending 5296 * in this scenario as the previous instruction cannot 5297 * have toggled RFLAGS.TF 0=>1 (because STI and POP/MOV 5298 * don't modify RFLAGS), therefore the one instruction 5299 * delay when activating single-step breakpoints must 5300 * have already expired. Note, the CPU sets/clears BS 5301 * as appropriate for all other VM-Exits types. 5302 */ 5303 if (is_icebp(intr_info)) 5304 WARN_ON(!skip_emulated_instruction(vcpu)); 5305 else if ((vmx_get_rflags(vcpu) & X86_EFLAGS_TF) && 5306 (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 5307 (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS))) 5308 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 5309 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS) | DR6_BS); 5310 5311 kvm_queue_exception_p(vcpu, DB_VECTOR, dr6); 5312 return 1; 5313 } 5314 kvm_run->debug.arch.dr6 = dr6 | DR6_ACTIVE_LOW; 5315 kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7); 5316 fallthrough; 5317 case BP_VECTOR: 5318 /* 5319 * Update instruction length as we may reinject #BP from 5320 * user space while in guest debugging mode. Reading it for 5321 * #DB as well causes no harm, it is not used in that case. 5322 */ 5323 vmx->vcpu.arch.event_exit_inst_len = 5324 vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 5325 kvm_run->exit_reason = KVM_EXIT_DEBUG; 5326 kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu); 5327 kvm_run->debug.arch.exception = ex_no; 5328 break; 5329 case AC_VECTOR: 5330 if (vmx_guest_inject_ac(vcpu)) { 5331 kvm_queue_exception_e(vcpu, AC_VECTOR, error_code); 5332 return 1; 5333 } 5334 5335 /* 5336 * Handle split lock. Depending on detection mode this will 5337 * either warn and disable split lock detection for this 5338 * task or force SIGBUS on it. 5339 */ 5340 if (handle_guest_split_lock(kvm_rip_read(vcpu))) 5341 return 1; 5342 fallthrough; 5343 default: 5344 kvm_run->exit_reason = KVM_EXIT_EXCEPTION; 5345 kvm_run->ex.exception = ex_no; 5346 kvm_run->ex.error_code = error_code; 5347 break; 5348 } 5349 return 0; 5350 } 5351 5352 static __always_inline int handle_external_interrupt(struct kvm_vcpu *vcpu) 5353 { 5354 ++vcpu->stat.irq_exits; 5355 return 1; 5356 } 5357 5358 static int handle_triple_fault(struct kvm_vcpu *vcpu) 5359 { 5360 vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN; 5361 vcpu->mmio_needed = 0; 5362 return 0; 5363 } 5364 5365 static int handle_io(struct kvm_vcpu *vcpu) 5366 { 5367 unsigned long exit_qualification; 5368 int size, in, string; 5369 unsigned port; 5370 5371 exit_qualification = vmx_get_exit_qual(vcpu); 5372 string = (exit_qualification & 16) != 0; 5373 5374 ++vcpu->stat.io_exits; 5375 5376 if (string) 5377 return kvm_emulate_instruction(vcpu, 0); 5378 5379 port = exit_qualification >> 16; 5380 size = (exit_qualification & 7) + 1; 5381 in = (exit_qualification & 8) != 0; 5382 5383 return kvm_fast_pio(vcpu, size, port, in); 5384 } 5385 5386 void vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) 5387 { 5388 /* 5389 * Patch in the VMCALL instruction: 5390 */ 5391 hypercall[0] = 0x0f; 5392 hypercall[1] = 0x01; 5393 hypercall[2] = 0xc1; 5394 } 5395 5396 /* called to set cr0 as appropriate for a mov-to-cr0 exit. */ 5397 static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val) 5398 { 5399 if (is_guest_mode(vcpu)) { 5400 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5401 unsigned long orig_val = val; 5402 5403 /* 5404 * We get here when L2 changed cr0 in a way that did not change 5405 * any of L1's shadowed bits (see nested_vmx_exit_handled_cr), 5406 * but did change L0 shadowed bits. So we first calculate the 5407 * effective cr0 value that L1 would like to write into the 5408 * hardware. It consists of the L2-owned bits from the new 5409 * value combined with the L1-owned bits from L1's guest_cr0. 5410 */ 5411 val = (val & ~vmcs12->cr0_guest_host_mask) | 5412 (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask); 5413 5414 if (kvm_set_cr0(vcpu, val)) 5415 return 1; 5416 vmcs_writel(CR0_READ_SHADOW, orig_val); 5417 return 0; 5418 } else { 5419 return kvm_set_cr0(vcpu, val); 5420 } 5421 } 5422 5423 static int handle_set_cr4(struct kvm_vcpu *vcpu, unsigned long val) 5424 { 5425 if (is_guest_mode(vcpu)) { 5426 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5427 unsigned long orig_val = val; 5428 5429 /* analogously to handle_set_cr0 */ 5430 val = (val & ~vmcs12->cr4_guest_host_mask) | 5431 (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask); 5432 if (kvm_set_cr4(vcpu, val)) 5433 return 1; 5434 vmcs_writel(CR4_READ_SHADOW, orig_val); 5435 return 0; 5436 } else 5437 return kvm_set_cr4(vcpu, val); 5438 } 5439 5440 static int handle_desc(struct kvm_vcpu *vcpu) 5441 { 5442 /* 5443 * UMIP emulation relies on intercepting writes to CR4.UMIP, i.e. this 5444 * and other code needs to be updated if UMIP can be guest owned. 5445 */ 5446 BUILD_BUG_ON(KVM_POSSIBLE_CR4_GUEST_BITS & X86_CR4_UMIP); 5447 5448 WARN_ON_ONCE(!kvm_is_cr4_bit_set(vcpu, X86_CR4_UMIP)); 5449 return kvm_emulate_instruction(vcpu, 0); 5450 } 5451 5452 static int handle_cr(struct kvm_vcpu *vcpu) 5453 { 5454 unsigned long exit_qualification, val; 5455 int cr; 5456 int reg; 5457 int err; 5458 int ret; 5459 5460 exit_qualification = vmx_get_exit_qual(vcpu); 5461 cr = exit_qualification & 15; 5462 reg = (exit_qualification >> 8) & 15; 5463 switch ((exit_qualification >> 4) & 3) { 5464 case 0: /* mov to cr */ 5465 val = kvm_register_read(vcpu, reg); 5466 trace_kvm_cr_write(cr, val); 5467 switch (cr) { 5468 case 0: 5469 err = handle_set_cr0(vcpu, val); 5470 return kvm_complete_insn_gp(vcpu, err); 5471 case 3: 5472 WARN_ON_ONCE(enable_unrestricted_guest); 5473 5474 err = kvm_set_cr3(vcpu, val); 5475 return kvm_complete_insn_gp(vcpu, err); 5476 case 4: 5477 err = handle_set_cr4(vcpu, val); 5478 return kvm_complete_insn_gp(vcpu, err); 5479 case 8: { 5480 u8 cr8_prev = kvm_get_cr8(vcpu); 5481 u8 cr8 = (u8)val; 5482 err = kvm_set_cr8(vcpu, cr8); 5483 ret = kvm_complete_insn_gp(vcpu, err); 5484 if (lapic_in_kernel(vcpu)) 5485 return ret; 5486 if (cr8_prev <= cr8) 5487 return ret; 5488 /* 5489 * TODO: we might be squashing a 5490 * KVM_GUESTDBG_SINGLESTEP-triggered 5491 * KVM_EXIT_DEBUG here. 5492 */ 5493 vcpu->run->exit_reason = KVM_EXIT_SET_TPR; 5494 return 0; 5495 } 5496 } 5497 break; 5498 case 2: /* clts */ 5499 KVM_BUG(1, vcpu->kvm, "Guest always owns CR0.TS"); 5500 return -EIO; 5501 case 1: /*mov from cr*/ 5502 switch (cr) { 5503 case 3: 5504 WARN_ON_ONCE(enable_unrestricted_guest); 5505 5506 val = kvm_read_cr3(vcpu); 5507 kvm_register_write(vcpu, reg, val); 5508 trace_kvm_cr_read(cr, val); 5509 return kvm_skip_emulated_instruction(vcpu); 5510 case 8: 5511 val = kvm_get_cr8(vcpu); 5512 kvm_register_write(vcpu, reg, val); 5513 trace_kvm_cr_read(cr, val); 5514 return kvm_skip_emulated_instruction(vcpu); 5515 } 5516 break; 5517 case 3: /* lmsw */ 5518 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f; 5519 trace_kvm_cr_write(0, (kvm_read_cr0_bits(vcpu, ~0xful) | val)); 5520 kvm_lmsw(vcpu, val); 5521 5522 return kvm_skip_emulated_instruction(vcpu); 5523 default: 5524 break; 5525 } 5526 vcpu->run->exit_reason = 0; 5527 vcpu_unimpl(vcpu, "unhandled control register: op %d cr %d\n", 5528 (int)(exit_qualification >> 4) & 3, cr); 5529 return 0; 5530 } 5531 5532 static int handle_dr(struct kvm_vcpu *vcpu) 5533 { 5534 unsigned long exit_qualification; 5535 int dr, dr7, reg; 5536 int err = 1; 5537 5538 exit_qualification = vmx_get_exit_qual(vcpu); 5539 dr = exit_qualification & DEBUG_REG_ACCESS_NUM; 5540 5541 /* First, if DR does not exist, trigger UD */ 5542 if (!kvm_require_dr(vcpu, dr)) 5543 return 1; 5544 5545 if (vmx_get_cpl(vcpu) > 0) 5546 goto out; 5547 5548 dr7 = vmcs_readl(GUEST_DR7); 5549 if (dr7 & DR7_GD) { 5550 /* 5551 * As the vm-exit takes precedence over the debug trap, we 5552 * need to emulate the latter, either for the host or the 5553 * guest debugging itself. 5554 */ 5555 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) { 5556 vcpu->run->debug.arch.dr6 = DR6_BD | DR6_ACTIVE_LOW; 5557 vcpu->run->debug.arch.dr7 = dr7; 5558 vcpu->run->debug.arch.pc = kvm_get_linear_rip(vcpu); 5559 vcpu->run->debug.arch.exception = DB_VECTOR; 5560 vcpu->run->exit_reason = KVM_EXIT_DEBUG; 5561 return 0; 5562 } else { 5563 kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BD); 5564 return 1; 5565 } 5566 } 5567 5568 if (vcpu->guest_debug == 0) { 5569 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_MOV_DR_EXITING); 5570 5571 /* 5572 * No more DR vmexits; force a reload of the debug registers 5573 * and reenter on this instruction. The next vmexit will 5574 * retrieve the full state of the debug registers. 5575 */ 5576 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT; 5577 return 1; 5578 } 5579 5580 reg = DEBUG_REG_ACCESS_REG(exit_qualification); 5581 if (exit_qualification & TYPE_MOV_FROM_DR) { 5582 kvm_register_write(vcpu, reg, kvm_get_dr(vcpu, dr)); 5583 err = 0; 5584 } else { 5585 err = kvm_set_dr(vcpu, dr, kvm_register_read(vcpu, reg)); 5586 } 5587 5588 out: 5589 return kvm_complete_insn_gp(vcpu, err); 5590 } 5591 5592 void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu) 5593 { 5594 get_debugreg(vcpu->arch.db[0], 0); 5595 get_debugreg(vcpu->arch.db[1], 1); 5596 get_debugreg(vcpu->arch.db[2], 2); 5597 get_debugreg(vcpu->arch.db[3], 3); 5598 get_debugreg(vcpu->arch.dr6, 6); 5599 vcpu->arch.dr7 = vmcs_readl(GUEST_DR7); 5600 5601 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT; 5602 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_MOV_DR_EXITING); 5603 5604 /* 5605 * exc_debug expects dr6 to be cleared after it runs, avoid that it sees 5606 * a stale dr6 from the guest. 5607 */ 5608 set_debugreg(DR6_RESERVED, 6); 5609 } 5610 5611 void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val) 5612 { 5613 vmcs_writel(GUEST_DR7, val); 5614 } 5615 5616 static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu) 5617 { 5618 kvm_apic_update_ppr(vcpu); 5619 return 1; 5620 } 5621 5622 static int handle_interrupt_window(struct kvm_vcpu *vcpu) 5623 { 5624 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING); 5625 5626 kvm_make_request(KVM_REQ_EVENT, vcpu); 5627 5628 ++vcpu->stat.irq_window_exits; 5629 return 1; 5630 } 5631 5632 static int handle_invlpg(struct kvm_vcpu *vcpu) 5633 { 5634 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5635 5636 kvm_mmu_invlpg(vcpu, exit_qualification); 5637 return kvm_skip_emulated_instruction(vcpu); 5638 } 5639 5640 static int handle_apic_access(struct kvm_vcpu *vcpu) 5641 { 5642 if (likely(fasteoi)) { 5643 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5644 int access_type, offset; 5645 5646 access_type = exit_qualification & APIC_ACCESS_TYPE; 5647 offset = exit_qualification & APIC_ACCESS_OFFSET; 5648 /* 5649 * Sane guest uses MOV to write EOI, with written value 5650 * not cared. So make a short-circuit here by avoiding 5651 * heavy instruction emulation. 5652 */ 5653 if ((access_type == TYPE_LINEAR_APIC_INST_WRITE) && 5654 (offset == APIC_EOI)) { 5655 kvm_lapic_set_eoi(vcpu); 5656 return kvm_skip_emulated_instruction(vcpu); 5657 } 5658 } 5659 return kvm_emulate_instruction(vcpu, 0); 5660 } 5661 5662 static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu) 5663 { 5664 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5665 int vector = exit_qualification & 0xff; 5666 5667 /* EOI-induced VM exit is trap-like and thus no need to adjust IP */ 5668 kvm_apic_set_eoi_accelerated(vcpu, vector); 5669 return 1; 5670 } 5671 5672 static int handle_apic_write(struct kvm_vcpu *vcpu) 5673 { 5674 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5675 5676 /* 5677 * APIC-write VM-Exit is trap-like, KVM doesn't need to advance RIP and 5678 * hardware has done any necessary aliasing, offset adjustments, etc... 5679 * for the access. I.e. the correct value has already been written to 5680 * the vAPIC page for the correct 16-byte chunk. KVM needs only to 5681 * retrieve the register value and emulate the access. 5682 */ 5683 u32 offset = exit_qualification & 0xff0; 5684 5685 kvm_apic_write_nodecode(vcpu, offset); 5686 return 1; 5687 } 5688 5689 static int handle_task_switch(struct kvm_vcpu *vcpu) 5690 { 5691 struct vcpu_vmx *vmx = to_vmx(vcpu); 5692 unsigned long exit_qualification; 5693 bool has_error_code = false; 5694 u32 error_code = 0; 5695 u16 tss_selector; 5696 int reason, type, idt_v, idt_index; 5697 5698 idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK); 5699 idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK); 5700 type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK); 5701 5702 exit_qualification = vmx_get_exit_qual(vcpu); 5703 5704 reason = (u32)exit_qualification >> 30; 5705 if (reason == TASK_SWITCH_GATE && idt_v) { 5706 switch (type) { 5707 case INTR_TYPE_NMI_INTR: 5708 vcpu->arch.nmi_injected = false; 5709 vmx_set_nmi_mask(vcpu, true); 5710 break; 5711 case INTR_TYPE_EXT_INTR: 5712 case INTR_TYPE_SOFT_INTR: 5713 kvm_clear_interrupt_queue(vcpu); 5714 break; 5715 case INTR_TYPE_HARD_EXCEPTION: 5716 if (vmx->idt_vectoring_info & 5717 VECTORING_INFO_DELIVER_CODE_MASK) { 5718 has_error_code = true; 5719 error_code = 5720 vmcs_read32(IDT_VECTORING_ERROR_CODE); 5721 } 5722 fallthrough; 5723 case INTR_TYPE_SOFT_EXCEPTION: 5724 kvm_clear_exception_queue(vcpu); 5725 break; 5726 default: 5727 break; 5728 } 5729 } 5730 tss_selector = exit_qualification; 5731 5732 if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION && 5733 type != INTR_TYPE_EXT_INTR && 5734 type != INTR_TYPE_NMI_INTR)) 5735 WARN_ON(!skip_emulated_instruction(vcpu)); 5736 5737 /* 5738 * TODO: What about debug traps on tss switch? 5739 * Are we supposed to inject them and update dr6? 5740 */ 5741 return kvm_task_switch(vcpu, tss_selector, 5742 type == INTR_TYPE_SOFT_INTR ? idt_index : -1, 5743 reason, has_error_code, error_code); 5744 } 5745 5746 static int handle_ept_violation(struct kvm_vcpu *vcpu) 5747 { 5748 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5749 gpa_t gpa; 5750 5751 /* 5752 * EPT violation happened while executing iret from NMI, 5753 * "blocked by NMI" bit has to be set before next VM entry. 5754 * There are errata that may cause this bit to not be set: 5755 * AAK134, BY25. 5756 */ 5757 if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) && 5758 enable_vnmi && 5759 (exit_qualification & INTR_INFO_UNBLOCK_NMI)) 5760 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI); 5761 5762 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS); 5763 trace_kvm_page_fault(vcpu, gpa, exit_qualification); 5764 5765 /* 5766 * Check that the GPA doesn't exceed physical memory limits, as that is 5767 * a guest page fault. We have to emulate the instruction here, because 5768 * if the illegal address is that of a paging structure, then 5769 * EPT_VIOLATION_ACC_WRITE bit is set. Alternatively, if supported we 5770 * would also use advanced VM-exit information for EPT violations to 5771 * reconstruct the page fault error code. 5772 */ 5773 if (unlikely(allow_smaller_maxphyaddr && !kvm_vcpu_is_legal_gpa(vcpu, gpa))) 5774 return kvm_emulate_instruction(vcpu, 0); 5775 5776 return __vmx_handle_ept_violation(vcpu, gpa, exit_qualification); 5777 } 5778 5779 static int handle_ept_misconfig(struct kvm_vcpu *vcpu) 5780 { 5781 gpa_t gpa; 5782 5783 if (vmx_check_emulate_instruction(vcpu, EMULTYPE_PF, NULL, 0)) 5784 return 1; 5785 5786 /* 5787 * A nested guest cannot optimize MMIO vmexits, because we have an 5788 * nGPA here instead of the required GPA. 5789 */ 5790 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS); 5791 if (!is_guest_mode(vcpu) && 5792 !kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) { 5793 trace_kvm_fast_mmio(gpa); 5794 return kvm_skip_emulated_instruction(vcpu); 5795 } 5796 5797 return kvm_mmu_page_fault(vcpu, gpa, PFERR_RSVD_MASK, NULL, 0); 5798 } 5799 5800 static int handle_nmi_window(struct kvm_vcpu *vcpu) 5801 { 5802 if (KVM_BUG_ON(!enable_vnmi, vcpu->kvm)) 5803 return -EIO; 5804 5805 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_NMI_WINDOW_EXITING); 5806 ++vcpu->stat.nmi_window_exits; 5807 kvm_make_request(KVM_REQ_EVENT, vcpu); 5808 5809 return 1; 5810 } 5811 5812 /* 5813 * Returns true if emulation is required (due to the vCPU having invalid state 5814 * with unsrestricted guest mode disabled) and KVM can't faithfully emulate the 5815 * current vCPU state. 5816 */ 5817 static bool vmx_unhandleable_emulation_required(struct kvm_vcpu *vcpu) 5818 { 5819 struct vcpu_vmx *vmx = to_vmx(vcpu); 5820 5821 if (!vmx->vt.emulation_required) 5822 return false; 5823 5824 /* 5825 * It is architecturally impossible for emulation to be required when a 5826 * nested VM-Enter is pending completion, as VM-Enter will VM-Fail if 5827 * guest state is invalid and unrestricted guest is disabled, i.e. KVM 5828 * should synthesize VM-Fail instead emulation L2 code. This path is 5829 * only reachable if userspace modifies L2 guest state after KVM has 5830 * performed the nested VM-Enter consistency checks. 5831 */ 5832 if (vmx->nested.nested_run_pending) 5833 return true; 5834 5835 /* 5836 * KVM only supports emulating exceptions if the vCPU is in Real Mode. 5837 * If emulation is required, KVM can't perform a successful VM-Enter to 5838 * inject the exception. 5839 */ 5840 return !vmx->rmode.vm86_active && 5841 (kvm_is_exception_pending(vcpu) || vcpu->arch.exception.injected); 5842 } 5843 5844 static int handle_invalid_guest_state(struct kvm_vcpu *vcpu) 5845 { 5846 struct vcpu_vmx *vmx = to_vmx(vcpu); 5847 bool intr_window_requested; 5848 unsigned count = 130; 5849 5850 intr_window_requested = exec_controls_get(vmx) & 5851 CPU_BASED_INTR_WINDOW_EXITING; 5852 5853 while (vmx->vt.emulation_required && count-- != 0) { 5854 if (intr_window_requested && !vmx_interrupt_blocked(vcpu)) 5855 return handle_interrupt_window(&vmx->vcpu); 5856 5857 if (kvm_test_request(KVM_REQ_EVENT, vcpu)) 5858 return 1; 5859 5860 /* 5861 * Ensure that any updates to kvm->buses[] observed by the 5862 * previous instruction (emulated or otherwise) are also 5863 * visible to the instruction KVM is about to emulate. 5864 */ 5865 smp_rmb(); 5866 5867 if (!kvm_emulate_instruction(vcpu, 0)) 5868 return 0; 5869 5870 if (vmx_unhandleable_emulation_required(vcpu)) { 5871 kvm_prepare_emulation_failure_exit(vcpu); 5872 return 0; 5873 } 5874 5875 if (vcpu->arch.halt_request) { 5876 vcpu->arch.halt_request = 0; 5877 return kvm_emulate_halt_noskip(vcpu); 5878 } 5879 5880 /* 5881 * Note, return 1 and not 0, vcpu_run() will invoke 5882 * xfer_to_guest_mode() which will create a proper return 5883 * code. 5884 */ 5885 if (__xfer_to_guest_mode_work_pending()) 5886 return 1; 5887 } 5888 5889 return 1; 5890 } 5891 5892 int vmx_vcpu_pre_run(struct kvm_vcpu *vcpu) 5893 { 5894 if (vmx_unhandleable_emulation_required(vcpu)) { 5895 kvm_prepare_emulation_failure_exit(vcpu); 5896 return 0; 5897 } 5898 5899 return 1; 5900 } 5901 5902 /* 5903 * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE 5904 * exiting, so only get here on cpu with PAUSE-Loop-Exiting. 5905 */ 5906 static int handle_pause(struct kvm_vcpu *vcpu) 5907 { 5908 if (!kvm_pause_in_guest(vcpu->kvm)) 5909 grow_ple_window(vcpu); 5910 5911 /* 5912 * Intel sdm vol3 ch-25.1.3 says: The "PAUSE-loop exiting" 5913 * VM-execution control is ignored if CPL > 0. OTOH, KVM 5914 * never set PAUSE_EXITING and just set PLE if supported, 5915 * so the vcpu must be CPL=0 if it gets a PAUSE exit. 5916 */ 5917 kvm_vcpu_on_spin(vcpu, true); 5918 return kvm_skip_emulated_instruction(vcpu); 5919 } 5920 5921 static int handle_monitor_trap(struct kvm_vcpu *vcpu) 5922 { 5923 return 1; 5924 } 5925 5926 static int handle_invpcid(struct kvm_vcpu *vcpu) 5927 { 5928 u32 vmx_instruction_info; 5929 unsigned long type; 5930 gva_t gva; 5931 struct { 5932 u64 pcid; 5933 u64 gla; 5934 } operand; 5935 int gpr_index; 5936 5937 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_INVPCID)) { 5938 kvm_queue_exception(vcpu, UD_VECTOR); 5939 return 1; 5940 } 5941 5942 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5943 gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info); 5944 type = kvm_register_read(vcpu, gpr_index); 5945 5946 /* According to the Intel instruction reference, the memory operand 5947 * is read even if it isn't needed (e.g., for type==all) 5948 */ 5949 if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), 5950 vmx_instruction_info, false, 5951 sizeof(operand), &gva)) 5952 return 1; 5953 5954 return kvm_handle_invpcid(vcpu, type, gva); 5955 } 5956 5957 static int handle_pml_full(struct kvm_vcpu *vcpu) 5958 { 5959 unsigned long exit_qualification; 5960 5961 trace_kvm_pml_full(vcpu->vcpu_id); 5962 5963 exit_qualification = vmx_get_exit_qual(vcpu); 5964 5965 /* 5966 * PML buffer FULL happened while executing iret from NMI, 5967 * "blocked by NMI" bit has to be set before next VM entry. 5968 */ 5969 if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) && 5970 enable_vnmi && 5971 (exit_qualification & INTR_INFO_UNBLOCK_NMI)) 5972 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 5973 GUEST_INTR_STATE_NMI); 5974 5975 /* 5976 * PML buffer already flushed at beginning of VMEXIT. Nothing to do 5977 * here.., and there's no userspace involvement needed for PML. 5978 */ 5979 return 1; 5980 } 5981 5982 static fastpath_t handle_fastpath_preemption_timer(struct kvm_vcpu *vcpu, 5983 bool force_immediate_exit) 5984 { 5985 struct vcpu_vmx *vmx = to_vmx(vcpu); 5986 5987 /* 5988 * In the *extremely* unlikely scenario that this is a spurious VM-Exit 5989 * due to the timer expiring while it was "soft" disabled, just eat the 5990 * exit and re-enter the guest. 5991 */ 5992 if (unlikely(vmx->loaded_vmcs->hv_timer_soft_disabled)) 5993 return EXIT_FASTPATH_REENTER_GUEST; 5994 5995 /* 5996 * If the timer expired because KVM used it to force an immediate exit, 5997 * then mission accomplished. 5998 */ 5999 if (force_immediate_exit) 6000 return EXIT_FASTPATH_EXIT_HANDLED; 6001 6002 /* 6003 * If L2 is active, go down the slow path as emulating the guest timer 6004 * expiration likely requires synthesizing a nested VM-Exit. 6005 */ 6006 if (is_guest_mode(vcpu)) 6007 return EXIT_FASTPATH_NONE; 6008 6009 kvm_lapic_expired_hv_timer(vcpu); 6010 return EXIT_FASTPATH_REENTER_GUEST; 6011 } 6012 6013 static int handle_preemption_timer(struct kvm_vcpu *vcpu) 6014 { 6015 /* 6016 * This non-fastpath handler is reached if and only if the preemption 6017 * timer was being used to emulate a guest timer while L2 is active. 6018 * All other scenarios are supposed to be handled in the fastpath. 6019 */ 6020 WARN_ON_ONCE(!is_guest_mode(vcpu)); 6021 kvm_lapic_expired_hv_timer(vcpu); 6022 return 1; 6023 } 6024 6025 /* 6026 * When nested=0, all VMX instruction VM Exits filter here. The handlers 6027 * are overwritten by nested_vmx_hardware_setup() when nested=1. 6028 */ 6029 static int handle_vmx_instruction(struct kvm_vcpu *vcpu) 6030 { 6031 kvm_queue_exception(vcpu, UD_VECTOR); 6032 return 1; 6033 } 6034 6035 #ifndef CONFIG_X86_SGX_KVM 6036 static int handle_encls(struct kvm_vcpu *vcpu) 6037 { 6038 /* 6039 * SGX virtualization is disabled. There is no software enable bit for 6040 * SGX, so KVM intercepts all ENCLS leafs and injects a #UD to prevent 6041 * the guest from executing ENCLS (when SGX is supported by hardware). 6042 */ 6043 kvm_queue_exception(vcpu, UD_VECTOR); 6044 return 1; 6045 } 6046 #endif /* CONFIG_X86_SGX_KVM */ 6047 6048 static int handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) 6049 { 6050 /* 6051 * Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK 6052 * VM-Exits. Unconditionally set the flag here and leave the handling to 6053 * vmx_handle_exit(). 6054 */ 6055 to_vt(vcpu)->exit_reason.bus_lock_detected = true; 6056 return 1; 6057 } 6058 6059 static int handle_notify(struct kvm_vcpu *vcpu) 6060 { 6061 unsigned long exit_qual = vmx_get_exit_qual(vcpu); 6062 bool context_invalid = exit_qual & NOTIFY_VM_CONTEXT_INVALID; 6063 6064 ++vcpu->stat.notify_window_exits; 6065 6066 /* 6067 * Notify VM exit happened while executing iret from NMI, 6068 * "blocked by NMI" bit has to be set before next VM entry. 6069 */ 6070 if (enable_vnmi && (exit_qual & INTR_INFO_UNBLOCK_NMI)) 6071 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 6072 GUEST_INTR_STATE_NMI); 6073 6074 if (vcpu->kvm->arch.notify_vmexit_flags & KVM_X86_NOTIFY_VMEXIT_USER || 6075 context_invalid) { 6076 vcpu->run->exit_reason = KVM_EXIT_NOTIFY; 6077 vcpu->run->notify.flags = context_invalid ? 6078 KVM_NOTIFY_CONTEXT_INVALID : 0; 6079 return 0; 6080 } 6081 6082 return 1; 6083 } 6084 6085 static int vmx_get_msr_imm_reg(struct kvm_vcpu *vcpu) 6086 { 6087 return vmx_get_instr_info_reg(vmcs_read32(VMX_INSTRUCTION_INFO)); 6088 } 6089 6090 static int handle_rdmsr_imm(struct kvm_vcpu *vcpu) 6091 { 6092 return kvm_emulate_rdmsr_imm(vcpu, vmx_get_exit_qual(vcpu), 6093 vmx_get_msr_imm_reg(vcpu)); 6094 } 6095 6096 static int handle_wrmsr_imm(struct kvm_vcpu *vcpu) 6097 { 6098 return kvm_emulate_wrmsr_imm(vcpu, vmx_get_exit_qual(vcpu), 6099 vmx_get_msr_imm_reg(vcpu)); 6100 } 6101 6102 /* 6103 * The exit handlers return 1 if the exit was handled fully and guest execution 6104 * may resume. Otherwise they set the kvm_run parameter to indicate what needs 6105 * to be done to userspace and return 0. 6106 */ 6107 static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { 6108 [EXIT_REASON_EXCEPTION_NMI] = handle_exception_nmi, 6109 [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt, 6110 [EXIT_REASON_TRIPLE_FAULT] = handle_triple_fault, 6111 [EXIT_REASON_NMI_WINDOW] = handle_nmi_window, 6112 [EXIT_REASON_IO_INSTRUCTION] = handle_io, 6113 [EXIT_REASON_CR_ACCESS] = handle_cr, 6114 [EXIT_REASON_DR_ACCESS] = handle_dr, 6115 [EXIT_REASON_CPUID] = kvm_emulate_cpuid, 6116 [EXIT_REASON_MSR_READ] = kvm_emulate_rdmsr, 6117 [EXIT_REASON_MSR_WRITE] = kvm_emulate_wrmsr, 6118 [EXIT_REASON_INTERRUPT_WINDOW] = handle_interrupt_window, 6119 [EXIT_REASON_HLT] = kvm_emulate_halt, 6120 [EXIT_REASON_INVD] = kvm_emulate_invd, 6121 [EXIT_REASON_INVLPG] = handle_invlpg, 6122 [EXIT_REASON_RDPMC] = kvm_emulate_rdpmc, 6123 [EXIT_REASON_VMCALL] = kvm_emulate_hypercall, 6124 [EXIT_REASON_VMCLEAR] = handle_vmx_instruction, 6125 [EXIT_REASON_VMLAUNCH] = handle_vmx_instruction, 6126 [EXIT_REASON_VMPTRLD] = handle_vmx_instruction, 6127 [EXIT_REASON_VMPTRST] = handle_vmx_instruction, 6128 [EXIT_REASON_VMREAD] = handle_vmx_instruction, 6129 [EXIT_REASON_VMRESUME] = handle_vmx_instruction, 6130 [EXIT_REASON_VMWRITE] = handle_vmx_instruction, 6131 [EXIT_REASON_VMOFF] = handle_vmx_instruction, 6132 [EXIT_REASON_VMON] = handle_vmx_instruction, 6133 [EXIT_REASON_TPR_BELOW_THRESHOLD] = handle_tpr_below_threshold, 6134 [EXIT_REASON_APIC_ACCESS] = handle_apic_access, 6135 [EXIT_REASON_APIC_WRITE] = handle_apic_write, 6136 [EXIT_REASON_EOI_INDUCED] = handle_apic_eoi_induced, 6137 [EXIT_REASON_WBINVD] = kvm_emulate_wbinvd, 6138 [EXIT_REASON_XSETBV] = kvm_emulate_xsetbv, 6139 [EXIT_REASON_TASK_SWITCH] = handle_task_switch, 6140 [EXIT_REASON_MCE_DURING_VMENTRY] = handle_machine_check, 6141 [EXIT_REASON_GDTR_IDTR] = handle_desc, 6142 [EXIT_REASON_LDTR_TR] = handle_desc, 6143 [EXIT_REASON_EPT_VIOLATION] = handle_ept_violation, 6144 [EXIT_REASON_EPT_MISCONFIG] = handle_ept_misconfig, 6145 [EXIT_REASON_PAUSE_INSTRUCTION] = handle_pause, 6146 [EXIT_REASON_MWAIT_INSTRUCTION] = kvm_emulate_mwait, 6147 [EXIT_REASON_MONITOR_TRAP_FLAG] = handle_monitor_trap, 6148 [EXIT_REASON_MONITOR_INSTRUCTION] = kvm_emulate_monitor, 6149 [EXIT_REASON_INVEPT] = handle_vmx_instruction, 6150 [EXIT_REASON_INVVPID] = handle_vmx_instruction, 6151 [EXIT_REASON_RDRAND] = kvm_handle_invalid_op, 6152 [EXIT_REASON_RDSEED] = kvm_handle_invalid_op, 6153 [EXIT_REASON_PML_FULL] = handle_pml_full, 6154 [EXIT_REASON_INVPCID] = handle_invpcid, 6155 [EXIT_REASON_VMFUNC] = handle_vmx_instruction, 6156 [EXIT_REASON_PREEMPTION_TIMER] = handle_preemption_timer, 6157 [EXIT_REASON_ENCLS] = handle_encls, 6158 [EXIT_REASON_BUS_LOCK] = handle_bus_lock_vmexit, 6159 [EXIT_REASON_NOTIFY] = handle_notify, 6160 [EXIT_REASON_MSR_READ_IMM] = handle_rdmsr_imm, 6161 [EXIT_REASON_MSR_WRITE_IMM] = handle_wrmsr_imm, 6162 }; 6163 6164 static const int kvm_vmx_max_exit_handlers = 6165 ARRAY_SIZE(kvm_vmx_exit_handlers); 6166 6167 void vmx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, 6168 u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code) 6169 { 6170 struct vcpu_vmx *vmx = to_vmx(vcpu); 6171 6172 *reason = vmx->vt.exit_reason.full; 6173 *info1 = vmx_get_exit_qual(vcpu); 6174 if (!(vmx->vt.exit_reason.failed_vmentry)) { 6175 *info2 = vmx->idt_vectoring_info; 6176 *intr_info = vmx_get_intr_info(vcpu); 6177 if (is_exception_with_error_code(*intr_info)) 6178 *error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); 6179 else 6180 *error_code = 0; 6181 } else { 6182 *info2 = 0; 6183 *intr_info = 0; 6184 *error_code = 0; 6185 } 6186 } 6187 6188 void vmx_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info, u32 *error_code) 6189 { 6190 *intr_info = vmcs_read32(VM_ENTRY_INTR_INFO_FIELD); 6191 if (is_exception_with_error_code(*intr_info)) 6192 *error_code = vmcs_read32(VM_ENTRY_EXCEPTION_ERROR_CODE); 6193 else 6194 *error_code = 0; 6195 } 6196 6197 static void vmx_destroy_pml_buffer(struct vcpu_vmx *vmx) 6198 { 6199 if (vmx->pml_pg) { 6200 __free_page(vmx->pml_pg); 6201 vmx->pml_pg = NULL; 6202 } 6203 } 6204 6205 static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu) 6206 { 6207 struct vcpu_vmx *vmx = to_vmx(vcpu); 6208 u16 pml_idx, pml_tail_index; 6209 u64 *pml_buf; 6210 int i; 6211 6212 pml_idx = vmcs_read16(GUEST_PML_INDEX); 6213 6214 /* Do nothing if PML buffer is empty */ 6215 if (pml_idx == PML_HEAD_INDEX) 6216 return; 6217 /* 6218 * PML index always points to the next available PML buffer entity 6219 * unless PML log has just overflowed. 6220 */ 6221 pml_tail_index = (pml_idx >= PML_LOG_NR_ENTRIES) ? 0 : pml_idx + 1; 6222 6223 /* 6224 * PML log is written backwards: the CPU first writes the entry 511 6225 * then the entry 510, and so on. 6226 * 6227 * Read the entries in the same order they were written, to ensure that 6228 * the dirty ring is filled in the same order the CPU wrote them. 6229 */ 6230 pml_buf = page_address(vmx->pml_pg); 6231 6232 for (i = PML_HEAD_INDEX; i >= pml_tail_index; i--) { 6233 u64 gpa; 6234 6235 gpa = pml_buf[i]; 6236 WARN_ON(gpa & (PAGE_SIZE - 1)); 6237 kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT); 6238 } 6239 6240 /* reset PML index */ 6241 vmcs_write16(GUEST_PML_INDEX, PML_HEAD_INDEX); 6242 } 6243 6244 static void vmx_dump_sel(char *name, uint32_t sel) 6245 { 6246 pr_err("%s sel=0x%04x, attr=0x%05x, limit=0x%08x, base=0x%016lx\n", 6247 name, vmcs_read16(sel), 6248 vmcs_read32(sel + GUEST_ES_AR_BYTES - GUEST_ES_SELECTOR), 6249 vmcs_read32(sel + GUEST_ES_LIMIT - GUEST_ES_SELECTOR), 6250 vmcs_readl(sel + GUEST_ES_BASE - GUEST_ES_SELECTOR)); 6251 } 6252 6253 static void vmx_dump_dtsel(char *name, uint32_t limit) 6254 { 6255 pr_err("%s limit=0x%08x, base=0x%016lx\n", 6256 name, vmcs_read32(limit), 6257 vmcs_readl(limit + GUEST_GDTR_BASE - GUEST_GDTR_LIMIT)); 6258 } 6259 6260 static void vmx_dump_msrs(char *name, struct vmx_msrs *m) 6261 { 6262 unsigned int i; 6263 struct vmx_msr_entry *e; 6264 6265 pr_err("MSR %s:\n", name); 6266 for (i = 0, e = m->val; i < m->nr; ++i, ++e) 6267 pr_err(" %2d: msr=0x%08x value=0x%016llx\n", i, e->index, e->value); 6268 } 6269 6270 void dump_vmcs(struct kvm_vcpu *vcpu) 6271 { 6272 struct vcpu_vmx *vmx = to_vmx(vcpu); 6273 u32 vmentry_ctl, vmexit_ctl; 6274 u32 cpu_based_exec_ctrl, pin_based_exec_ctrl, secondary_exec_control; 6275 u64 tertiary_exec_control; 6276 unsigned long cr4; 6277 int efer_slot; 6278 6279 if (!dump_invalid_vmcs) { 6280 pr_warn_ratelimited("set kvm_intel.dump_invalid_vmcs=1 to dump internal KVM state.\n"); 6281 return; 6282 } 6283 6284 vmentry_ctl = vmcs_read32(VM_ENTRY_CONTROLS); 6285 vmexit_ctl = vmcs_read32(VM_EXIT_CONTROLS); 6286 cpu_based_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL); 6287 pin_based_exec_ctrl = vmcs_read32(PIN_BASED_VM_EXEC_CONTROL); 6288 cr4 = vmcs_readl(GUEST_CR4); 6289 6290 if (cpu_has_secondary_exec_ctrls()) 6291 secondary_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL); 6292 else 6293 secondary_exec_control = 0; 6294 6295 if (cpu_has_tertiary_exec_ctrls()) 6296 tertiary_exec_control = vmcs_read64(TERTIARY_VM_EXEC_CONTROL); 6297 else 6298 tertiary_exec_control = 0; 6299 6300 pr_err("VMCS %p, last attempted VM-entry on CPU %d\n", 6301 vmx->loaded_vmcs->vmcs, vcpu->arch.last_vmentry_cpu); 6302 pr_err("*** Guest State ***\n"); 6303 pr_err("CR0: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n", 6304 vmcs_readl(GUEST_CR0), vmcs_readl(CR0_READ_SHADOW), 6305 vmcs_readl(CR0_GUEST_HOST_MASK)); 6306 pr_err("CR4: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n", 6307 cr4, vmcs_readl(CR4_READ_SHADOW), vmcs_readl(CR4_GUEST_HOST_MASK)); 6308 pr_err("CR3 = 0x%016lx\n", vmcs_readl(GUEST_CR3)); 6309 if (cpu_has_vmx_ept()) { 6310 pr_err("PDPTR0 = 0x%016llx PDPTR1 = 0x%016llx\n", 6311 vmcs_read64(GUEST_PDPTR0), vmcs_read64(GUEST_PDPTR1)); 6312 pr_err("PDPTR2 = 0x%016llx PDPTR3 = 0x%016llx\n", 6313 vmcs_read64(GUEST_PDPTR2), vmcs_read64(GUEST_PDPTR3)); 6314 } 6315 pr_err("RSP = 0x%016lx RIP = 0x%016lx\n", 6316 vmcs_readl(GUEST_RSP), vmcs_readl(GUEST_RIP)); 6317 pr_err("RFLAGS=0x%08lx DR7 = 0x%016lx\n", 6318 vmcs_readl(GUEST_RFLAGS), vmcs_readl(GUEST_DR7)); 6319 pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n", 6320 vmcs_readl(GUEST_SYSENTER_ESP), 6321 vmcs_read32(GUEST_SYSENTER_CS), vmcs_readl(GUEST_SYSENTER_EIP)); 6322 vmx_dump_sel("CS: ", GUEST_CS_SELECTOR); 6323 vmx_dump_sel("DS: ", GUEST_DS_SELECTOR); 6324 vmx_dump_sel("SS: ", GUEST_SS_SELECTOR); 6325 vmx_dump_sel("ES: ", GUEST_ES_SELECTOR); 6326 vmx_dump_sel("FS: ", GUEST_FS_SELECTOR); 6327 vmx_dump_sel("GS: ", GUEST_GS_SELECTOR); 6328 vmx_dump_dtsel("GDTR:", GUEST_GDTR_LIMIT); 6329 vmx_dump_sel("LDTR:", GUEST_LDTR_SELECTOR); 6330 vmx_dump_dtsel("IDTR:", GUEST_IDTR_LIMIT); 6331 vmx_dump_sel("TR: ", GUEST_TR_SELECTOR); 6332 efer_slot = vmx_find_loadstore_msr_slot(&vmx->msr_autoload.guest, MSR_EFER); 6333 if (vmentry_ctl & VM_ENTRY_LOAD_IA32_EFER) 6334 pr_err("EFER= 0x%016llx\n", vmcs_read64(GUEST_IA32_EFER)); 6335 else if (efer_slot >= 0) 6336 pr_err("EFER= 0x%016llx (autoload)\n", 6337 vmx->msr_autoload.guest.val[efer_slot].value); 6338 else if (vmentry_ctl & VM_ENTRY_IA32E_MODE) 6339 pr_err("EFER= 0x%016llx (effective)\n", 6340 vcpu->arch.efer | (EFER_LMA | EFER_LME)); 6341 else 6342 pr_err("EFER= 0x%016llx (effective)\n", 6343 vcpu->arch.efer & ~(EFER_LMA | EFER_LME)); 6344 if (vmentry_ctl & VM_ENTRY_LOAD_IA32_PAT) 6345 pr_err("PAT = 0x%016llx\n", vmcs_read64(GUEST_IA32_PAT)); 6346 pr_err("DebugCtl = 0x%016llx DebugExceptions = 0x%016lx\n", 6347 vmcs_read64(GUEST_IA32_DEBUGCTL), 6348 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS)); 6349 if (cpu_has_load_perf_global_ctrl() && 6350 vmentry_ctl & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) 6351 pr_err("PerfGlobCtl = 0x%016llx\n", 6352 vmcs_read64(GUEST_IA32_PERF_GLOBAL_CTRL)); 6353 if (vmentry_ctl & VM_ENTRY_LOAD_BNDCFGS) 6354 pr_err("BndCfgS = 0x%016llx\n", vmcs_read64(GUEST_BNDCFGS)); 6355 pr_err("Interruptibility = %08x ActivityState = %08x\n", 6356 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO), 6357 vmcs_read32(GUEST_ACTIVITY_STATE)); 6358 if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) 6359 pr_err("InterruptStatus = %04x\n", 6360 vmcs_read16(GUEST_INTR_STATUS)); 6361 if (vmcs_read32(VM_ENTRY_MSR_LOAD_COUNT) > 0) 6362 vmx_dump_msrs("guest autoload", &vmx->msr_autoload.guest); 6363 if (vmcs_read32(VM_EXIT_MSR_STORE_COUNT) > 0) 6364 vmx_dump_msrs("guest autostore", &vmx->msr_autostore.guest); 6365 6366 if (vmentry_ctl & VM_ENTRY_LOAD_CET_STATE) 6367 pr_err("S_CET = 0x%016lx, SSP = 0x%016lx, SSP TABLE = 0x%016lx\n", 6368 vmcs_readl(GUEST_S_CET), vmcs_readl(GUEST_SSP), 6369 vmcs_readl(GUEST_INTR_SSP_TABLE)); 6370 pr_err("*** Host State ***\n"); 6371 pr_err("RIP = 0x%016lx RSP = 0x%016lx\n", 6372 vmcs_readl(HOST_RIP), vmcs_readl(HOST_RSP)); 6373 pr_err("CS=%04x SS=%04x DS=%04x ES=%04x FS=%04x GS=%04x TR=%04x\n", 6374 vmcs_read16(HOST_CS_SELECTOR), vmcs_read16(HOST_SS_SELECTOR), 6375 vmcs_read16(HOST_DS_SELECTOR), vmcs_read16(HOST_ES_SELECTOR), 6376 vmcs_read16(HOST_FS_SELECTOR), vmcs_read16(HOST_GS_SELECTOR), 6377 vmcs_read16(HOST_TR_SELECTOR)); 6378 pr_err("FSBase=%016lx GSBase=%016lx TRBase=%016lx\n", 6379 vmcs_readl(HOST_FS_BASE), vmcs_readl(HOST_GS_BASE), 6380 vmcs_readl(HOST_TR_BASE)); 6381 pr_err("GDTBase=%016lx IDTBase=%016lx\n", 6382 vmcs_readl(HOST_GDTR_BASE), vmcs_readl(HOST_IDTR_BASE)); 6383 pr_err("CR0=%016lx CR3=%016lx CR4=%016lx\n", 6384 vmcs_readl(HOST_CR0), vmcs_readl(HOST_CR3), 6385 vmcs_readl(HOST_CR4)); 6386 pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n", 6387 vmcs_readl(HOST_IA32_SYSENTER_ESP), 6388 vmcs_read32(HOST_IA32_SYSENTER_CS), 6389 vmcs_readl(HOST_IA32_SYSENTER_EIP)); 6390 if (vmexit_ctl & VM_EXIT_LOAD_IA32_EFER) 6391 pr_err("EFER= 0x%016llx\n", vmcs_read64(HOST_IA32_EFER)); 6392 if (vmexit_ctl & VM_EXIT_LOAD_IA32_PAT) 6393 pr_err("PAT = 0x%016llx\n", vmcs_read64(HOST_IA32_PAT)); 6394 if (cpu_has_load_perf_global_ctrl() && 6395 vmexit_ctl & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) 6396 pr_err("PerfGlobCtl = 0x%016llx\n", 6397 vmcs_read64(HOST_IA32_PERF_GLOBAL_CTRL)); 6398 if (vmcs_read32(VM_EXIT_MSR_LOAD_COUNT) > 0) 6399 vmx_dump_msrs("host autoload", &vmx->msr_autoload.host); 6400 if (vmexit_ctl & VM_EXIT_LOAD_CET_STATE) 6401 pr_err("S_CET = 0x%016lx, SSP = 0x%016lx, SSP TABLE = 0x%016lx\n", 6402 vmcs_readl(HOST_S_CET), vmcs_readl(HOST_SSP), 6403 vmcs_readl(HOST_INTR_SSP_TABLE)); 6404 6405 pr_err("*** Control State ***\n"); 6406 pr_err("CPUBased=0x%08x SecondaryExec=0x%08x TertiaryExec=0x%016llx\n", 6407 cpu_based_exec_ctrl, secondary_exec_control, tertiary_exec_control); 6408 pr_err("PinBased=0x%08x EntryControls=%08x ExitControls=%08x\n", 6409 pin_based_exec_ctrl, vmentry_ctl, vmexit_ctl); 6410 pr_err("ExceptionBitmap=%08x PFECmask=%08x PFECmatch=%08x\n", 6411 vmcs_read32(EXCEPTION_BITMAP), 6412 vmcs_read32(PAGE_FAULT_ERROR_CODE_MASK), 6413 vmcs_read32(PAGE_FAULT_ERROR_CODE_MATCH)); 6414 pr_err("VMEntry: intr_info=%08x errcode=%08x ilen=%08x\n", 6415 vmcs_read32(VM_ENTRY_INTR_INFO_FIELD), 6416 vmcs_read32(VM_ENTRY_EXCEPTION_ERROR_CODE), 6417 vmcs_read32(VM_ENTRY_INSTRUCTION_LEN)); 6418 pr_err("VMExit: intr_info=%08x errcode=%08x ilen=%08x\n", 6419 vmcs_read32(VM_EXIT_INTR_INFO), 6420 vmcs_read32(VM_EXIT_INTR_ERROR_CODE), 6421 vmcs_read32(VM_EXIT_INSTRUCTION_LEN)); 6422 pr_err(" reason=%08x qualification=%016lx\n", 6423 vmcs_read32(VM_EXIT_REASON), vmcs_readl(EXIT_QUALIFICATION)); 6424 pr_err("IDTVectoring: info=%08x errcode=%08x\n", 6425 vmcs_read32(IDT_VECTORING_INFO_FIELD), 6426 vmcs_read32(IDT_VECTORING_ERROR_CODE)); 6427 pr_err("TSC Offset = 0x%016llx\n", vmcs_read64(TSC_OFFSET)); 6428 if (secondary_exec_control & SECONDARY_EXEC_TSC_SCALING) 6429 pr_err("TSC Multiplier = 0x%016llx\n", 6430 vmcs_read64(TSC_MULTIPLIER)); 6431 if (cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW) { 6432 if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) { 6433 u16 status = vmcs_read16(GUEST_INTR_STATUS); 6434 pr_err("SVI|RVI = %02x|%02x ", status >> 8, status & 0xff); 6435 } 6436 pr_cont("TPR Threshold = 0x%02x\n", vmcs_read32(TPR_THRESHOLD)); 6437 if (secondary_exec_control & SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) 6438 pr_err("APIC-access addr = 0x%016llx ", vmcs_read64(APIC_ACCESS_ADDR)); 6439 pr_cont("virt-APIC addr = 0x%016llx\n", vmcs_read64(VIRTUAL_APIC_PAGE_ADDR)); 6440 } 6441 if (pin_based_exec_ctrl & PIN_BASED_POSTED_INTR) 6442 pr_err("PostedIntrVec = 0x%02x\n", vmcs_read16(POSTED_INTR_NV)); 6443 if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT)) 6444 pr_err("EPT pointer = 0x%016llx\n", vmcs_read64(EPT_POINTER)); 6445 if (secondary_exec_control & SECONDARY_EXEC_PAUSE_LOOP_EXITING) 6446 pr_err("PLE Gap=%08x Window=%08x\n", 6447 vmcs_read32(PLE_GAP), vmcs_read32(PLE_WINDOW)); 6448 if (secondary_exec_control & SECONDARY_EXEC_ENABLE_VPID) 6449 pr_err("Virtual processor ID = 0x%04x\n", 6450 vmcs_read16(VIRTUAL_PROCESSOR_ID)); 6451 if (secondary_exec_control & SECONDARY_EXEC_EPT_VIOLATION_VE) { 6452 struct vmx_ve_information *ve_info = vmx->ve_info; 6453 u64 ve_info_pa = vmcs_read64(VE_INFORMATION_ADDRESS); 6454 6455 /* 6456 * If KVM is dumping the VMCS, then something has gone wrong 6457 * already. Derefencing an address from the VMCS, which could 6458 * very well be corrupted, is a terrible idea. The virtual 6459 * address is known so use it. 6460 */ 6461 pr_err("VE info address = 0x%016llx%s\n", ve_info_pa, 6462 ve_info_pa == __pa(ve_info) ? "" : "(corrupted!)"); 6463 pr_err("ve_info: 0x%08x 0x%08x 0x%016llx 0x%016llx 0x%016llx 0x%04x\n", 6464 ve_info->exit_reason, ve_info->delivery, 6465 ve_info->exit_qualification, 6466 ve_info->guest_linear_address, 6467 ve_info->guest_physical_address, ve_info->eptp_index); 6468 } 6469 } 6470 6471 /* 6472 * The guest has exited. See if we can fix it or if we need userspace 6473 * assistance. 6474 */ 6475 static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) 6476 { 6477 struct vcpu_vmx *vmx = to_vmx(vcpu); 6478 union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu); 6479 u32 vectoring_info = vmx->idt_vectoring_info; 6480 u16 exit_handler_index; 6481 6482 /* 6483 * Flush logged GPAs PML buffer, this will make dirty_bitmap more 6484 * updated. Another good is, in kvm_vm_ioctl_get_dirty_log, before 6485 * querying dirty_bitmap, we only need to kick all vcpus out of guest 6486 * mode as if vcpus is in root mode, the PML buffer must has been 6487 * flushed already. Note, PML is never enabled in hardware while 6488 * running L2. 6489 */ 6490 if (enable_pml && !is_guest_mode(vcpu)) 6491 vmx_flush_pml_buffer(vcpu); 6492 6493 /* 6494 * KVM should never reach this point with a pending nested VM-Enter. 6495 * More specifically, short-circuiting VM-Entry to emulate L2 due to 6496 * invalid guest state should never happen as that means KVM knowingly 6497 * allowed a nested VM-Enter with an invalid vmcs12. More below. 6498 */ 6499 if (KVM_BUG_ON(vmx->nested.nested_run_pending, vcpu->kvm)) 6500 return -EIO; 6501 6502 if (is_guest_mode(vcpu)) { 6503 /* 6504 * PML is never enabled when running L2, bail immediately if a 6505 * PML full exit occurs as something is horribly wrong. 6506 */ 6507 if (exit_reason.basic == EXIT_REASON_PML_FULL) 6508 goto unexpected_vmexit; 6509 6510 /* 6511 * The host physical addresses of some pages of guest memory 6512 * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC 6513 * Page). The CPU may write to these pages via their host 6514 * physical address while L2 is running, bypassing any 6515 * address-translation-based dirty tracking (e.g. EPT write 6516 * protection). 6517 * 6518 * Mark them dirty on every exit from L2 to prevent them from 6519 * getting out of sync with dirty tracking. 6520 */ 6521 nested_mark_vmcs12_pages_dirty(vcpu); 6522 6523 /* 6524 * Synthesize a triple fault if L2 state is invalid. In normal 6525 * operation, nested VM-Enter rejects any attempt to enter L2 6526 * with invalid state. However, those checks are skipped if 6527 * state is being stuffed via RSM or KVM_SET_NESTED_STATE. If 6528 * L2 state is invalid, it means either L1 modified SMRAM state 6529 * or userspace provided bad state. Synthesize TRIPLE_FAULT as 6530 * doing so is architecturally allowed in the RSM case, and is 6531 * the least awful solution for the userspace case without 6532 * risking false positives. 6533 */ 6534 if (vmx->vt.emulation_required) { 6535 nested_vmx_vmexit(vcpu, EXIT_REASON_TRIPLE_FAULT, 0, 0); 6536 return 1; 6537 } 6538 6539 if (nested_vmx_reflect_vmexit(vcpu)) 6540 return 1; 6541 } 6542 6543 /* If guest state is invalid, start emulating. L2 is handled above. */ 6544 if (vmx->vt.emulation_required) 6545 return handle_invalid_guest_state(vcpu); 6546 6547 if (exit_reason.failed_vmentry) { 6548 dump_vmcs(vcpu); 6549 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; 6550 vcpu->run->fail_entry.hardware_entry_failure_reason 6551 = exit_reason.full; 6552 vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu; 6553 return 0; 6554 } 6555 6556 if (unlikely(vmx->fail)) { 6557 dump_vmcs(vcpu); 6558 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; 6559 vcpu->run->fail_entry.hardware_entry_failure_reason 6560 = vmcs_read32(VM_INSTRUCTION_ERROR); 6561 vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu; 6562 return 0; 6563 } 6564 6565 if ((vectoring_info & VECTORING_INFO_VALID_MASK) && 6566 (exit_reason.basic != EXIT_REASON_EXCEPTION_NMI && 6567 exit_reason.basic != EXIT_REASON_EPT_VIOLATION && 6568 exit_reason.basic != EXIT_REASON_PML_FULL && 6569 exit_reason.basic != EXIT_REASON_APIC_ACCESS && 6570 exit_reason.basic != EXIT_REASON_TASK_SWITCH && 6571 exit_reason.basic != EXIT_REASON_NOTIFY && 6572 exit_reason.basic != EXIT_REASON_EPT_MISCONFIG)) { 6573 kvm_prepare_event_vectoring_exit(vcpu, INVALID_GPA); 6574 return 0; 6575 } 6576 6577 if (unlikely(!enable_vnmi && 6578 vmx->loaded_vmcs->soft_vnmi_blocked)) { 6579 if (!vmx_interrupt_blocked(vcpu)) { 6580 vmx->loaded_vmcs->soft_vnmi_blocked = 0; 6581 } else if (vmx->loaded_vmcs->vnmi_blocked_time > 1000000000LL && 6582 vcpu->arch.nmi_pending) { 6583 /* 6584 * This CPU don't support us in finding the end of an 6585 * NMI-blocked window if the guest runs with IRQs 6586 * disabled. So we pull the trigger after 1 s of 6587 * futile waiting, but inform the user about this. 6588 */ 6589 printk(KERN_WARNING "%s: Breaking out of NMI-blocked " 6590 "state on VCPU %d after 1 s timeout\n", 6591 __func__, vcpu->vcpu_id); 6592 vmx->loaded_vmcs->soft_vnmi_blocked = 0; 6593 } 6594 } 6595 6596 if (exit_fastpath != EXIT_FASTPATH_NONE) 6597 return 1; 6598 6599 if (exit_reason.basic >= kvm_vmx_max_exit_handlers) 6600 goto unexpected_vmexit; 6601 #ifdef CONFIG_MITIGATION_RETPOLINE 6602 if (exit_reason.basic == EXIT_REASON_MSR_WRITE) 6603 return kvm_emulate_wrmsr(vcpu); 6604 else if (exit_reason.basic == EXIT_REASON_MSR_WRITE_IMM) 6605 return handle_wrmsr_imm(vcpu); 6606 else if (exit_reason.basic == EXIT_REASON_PREEMPTION_TIMER) 6607 return handle_preemption_timer(vcpu); 6608 else if (exit_reason.basic == EXIT_REASON_INTERRUPT_WINDOW) 6609 return handle_interrupt_window(vcpu); 6610 else if (exit_reason.basic == EXIT_REASON_EXTERNAL_INTERRUPT) 6611 return handle_external_interrupt(vcpu); 6612 else if (exit_reason.basic == EXIT_REASON_HLT) 6613 return kvm_emulate_halt(vcpu); 6614 else if (exit_reason.basic == EXIT_REASON_EPT_MISCONFIG) 6615 return handle_ept_misconfig(vcpu); 6616 #endif 6617 6618 exit_handler_index = array_index_nospec((u16)exit_reason.basic, 6619 kvm_vmx_max_exit_handlers); 6620 if (!kvm_vmx_exit_handlers[exit_handler_index]) 6621 goto unexpected_vmexit; 6622 6623 return kvm_vmx_exit_handlers[exit_handler_index](vcpu); 6624 6625 unexpected_vmexit: 6626 vcpu_unimpl(vcpu, "vmx: unexpected exit reason 0x%x\n", 6627 exit_reason.full); 6628 dump_vmcs(vcpu); 6629 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 6630 vcpu->run->internal.suberror = 6631 KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON; 6632 vcpu->run->internal.ndata = 2; 6633 vcpu->run->internal.data[0] = exit_reason.full; 6634 vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu; 6635 return 0; 6636 } 6637 6638 int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) 6639 { 6640 int ret = __vmx_handle_exit(vcpu, exit_fastpath); 6641 6642 /* 6643 * Exit to user space when bus lock detected to inform that there is 6644 * a bus lock in guest. 6645 */ 6646 if (vmx_get_exit_reason(vcpu).bus_lock_detected) { 6647 if (ret > 0) 6648 vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; 6649 6650 vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK; 6651 return 0; 6652 } 6653 return ret; 6654 } 6655 6656 /* 6657 * Software based L1D cache flush which is used when microcode providing 6658 * the cache control MSR is not loaded. 6659 * 6660 * The L1D cache is 32 KiB on Nehalem and later microarchitectures, but to 6661 * flush it is required to read in 64 KiB because the replacement algorithm 6662 * is not exactly LRU. This could be sized at runtime via topology 6663 * information but as all relevant affected CPUs have 32KiB L1D cache size 6664 * there is no point in doing so. 6665 */ 6666 static noinstr void vmx_l1d_flush(struct kvm_vcpu *vcpu) 6667 { 6668 int size = PAGE_SIZE << L1D_CACHE_ORDER; 6669 6670 /* 6671 * This code is only executed when the flush mode is 'cond' or 6672 * 'always' 6673 */ 6674 if (static_branch_likely(&vmx_l1d_flush_cond)) { 6675 bool flush_l1d; 6676 6677 /* 6678 * Clear the per-vcpu flush bit, it gets set again if the vCPU 6679 * is reloaded, i.e. if the vCPU is scheduled out or if KVM 6680 * exits to userspace, or if KVM reaches one of the unsafe 6681 * VMEXIT handlers, e.g. if KVM calls into the emulator. 6682 */ 6683 flush_l1d = vcpu->arch.l1tf_flush_l1d; 6684 vcpu->arch.l1tf_flush_l1d = false; 6685 6686 /* 6687 * Clear the per-cpu flush bit, it gets set again from 6688 * the interrupt handlers. 6689 */ 6690 flush_l1d |= kvm_get_cpu_l1tf_flush_l1d(); 6691 kvm_clear_cpu_l1tf_flush_l1d(); 6692 6693 if (!flush_l1d) 6694 return; 6695 } 6696 6697 vcpu->stat.l1d_flush++; 6698 6699 if (static_cpu_has(X86_FEATURE_FLUSH_L1D)) { 6700 native_wrmsrq(MSR_IA32_FLUSH_CMD, L1D_FLUSH); 6701 return; 6702 } 6703 6704 asm volatile( 6705 /* First ensure the pages are in the TLB */ 6706 "xorl %%eax, %%eax\n" 6707 ".Lpopulate_tlb:\n\t" 6708 "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t" 6709 "addl $4096, %%eax\n\t" 6710 "cmpl %%eax, %[size]\n\t" 6711 "jne .Lpopulate_tlb\n\t" 6712 "xorl %%eax, %%eax\n\t" 6713 "cpuid\n\t" 6714 /* Now fill the cache */ 6715 "xorl %%eax, %%eax\n" 6716 ".Lfill_cache:\n" 6717 "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t" 6718 "addl $64, %%eax\n\t" 6719 "cmpl %%eax, %[size]\n\t" 6720 "jne .Lfill_cache\n\t" 6721 "lfence\n" 6722 :: [flush_pages] "r" (vmx_l1d_flush_pages), 6723 [size] "r" (size) 6724 : "eax", "ebx", "ecx", "edx"); 6725 } 6726 6727 void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) 6728 { 6729 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 6730 int tpr_threshold; 6731 6732 if (is_guest_mode(vcpu) && 6733 nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) 6734 return; 6735 6736 tpr_threshold = (irr == -1 || tpr < irr) ? 0 : irr; 6737 if (is_guest_mode(vcpu)) 6738 to_vmx(vcpu)->nested.l1_tpr_threshold = tpr_threshold; 6739 else 6740 vmcs_write32(TPR_THRESHOLD, tpr_threshold); 6741 } 6742 6743 void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu) 6744 { 6745 struct vcpu_vmx *vmx = to_vmx(vcpu); 6746 u32 sec_exec_control; 6747 6748 if (!lapic_in_kernel(vcpu)) 6749 return; 6750 6751 if (!flexpriority_enabled && 6752 !cpu_has_vmx_virtualize_x2apic_mode()) 6753 return; 6754 6755 /* Postpone execution until vmcs01 is the current VMCS. */ 6756 if (is_guest_mode(vcpu)) { 6757 vmx->nested.change_vmcs01_virtual_apic_mode = true; 6758 return; 6759 } 6760 6761 sec_exec_control = secondary_exec_controls_get(vmx); 6762 sec_exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 6763 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE); 6764 6765 switch (kvm_get_apic_mode(vcpu)) { 6766 case LAPIC_MODE_INVALID: 6767 WARN_ONCE(true, "Invalid local APIC state"); 6768 break; 6769 case LAPIC_MODE_DISABLED: 6770 break; 6771 case LAPIC_MODE_XAPIC: 6772 if (flexpriority_enabled) { 6773 sec_exec_control |= 6774 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 6775 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 6776 6777 /* 6778 * Flush the TLB, reloading the APIC access page will 6779 * only do so if its physical address has changed, but 6780 * the guest may have inserted a non-APIC mapping into 6781 * the TLB while the APIC access page was disabled. 6782 */ 6783 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 6784 } 6785 break; 6786 case LAPIC_MODE_X2APIC: 6787 if (cpu_has_vmx_virtualize_x2apic_mode()) 6788 sec_exec_control |= 6789 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE; 6790 break; 6791 } 6792 secondary_exec_controls_set(vmx, sec_exec_control); 6793 6794 vmx_update_msr_bitmap_x2apic(vcpu); 6795 } 6796 6797 void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu) 6798 { 6799 const gfn_t gfn = APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT; 6800 struct kvm *kvm = vcpu->kvm; 6801 struct kvm_memslots *slots = kvm_memslots(kvm); 6802 struct kvm_memory_slot *slot; 6803 struct page *refcounted_page; 6804 unsigned long mmu_seq; 6805 kvm_pfn_t pfn; 6806 bool writable; 6807 6808 /* Defer reload until vmcs01 is the current VMCS. */ 6809 if (is_guest_mode(vcpu)) { 6810 to_vmx(vcpu)->nested.reload_vmcs01_apic_access_page = true; 6811 return; 6812 } 6813 6814 if (!(secondary_exec_controls_get(to_vmx(vcpu)) & 6815 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) 6816 return; 6817 6818 /* 6819 * Explicitly grab the memslot using KVM's internal slot ID to ensure 6820 * KVM doesn't unintentionally grab a userspace memslot. It _should_ 6821 * be impossible for userspace to create a memslot for the APIC when 6822 * APICv is enabled, but paranoia won't hurt in this case. 6823 */ 6824 slot = id_to_memslot(slots, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT); 6825 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) 6826 return; 6827 6828 /* 6829 * Ensure that the mmu_notifier sequence count is read before KVM 6830 * retrieves the pfn from the primary MMU. Note, the memslot is 6831 * protected by SRCU, not the mmu_notifier. Pairs with the smp_wmb() 6832 * in kvm_mmu_invalidate_end(). 6833 */ 6834 mmu_seq = kvm->mmu_invalidate_seq; 6835 smp_rmb(); 6836 6837 /* 6838 * No need to retry if the memslot does not exist or is invalid. KVM 6839 * controls the APIC-access page memslot, and only deletes the memslot 6840 * if APICv is permanently inhibited, i.e. the memslot won't reappear. 6841 */ 6842 pfn = __kvm_faultin_pfn(slot, gfn, FOLL_WRITE, &writable, &refcounted_page); 6843 if (is_error_noslot_pfn(pfn)) 6844 return; 6845 6846 read_lock(&vcpu->kvm->mmu_lock); 6847 if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) 6848 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 6849 else 6850 vmcs_write64(APIC_ACCESS_ADDR, pfn_to_hpa(pfn)); 6851 6852 /* 6853 * Do not pin the APIC access page in memory so that it can be freely 6854 * migrated, the MMU notifier will call us again if it is migrated or 6855 * swapped out. KVM backs the memslot with anonymous memory, the pfn 6856 * should always point at a refcounted page (if the pfn is valid). 6857 */ 6858 if (!WARN_ON_ONCE(!refcounted_page)) 6859 kvm_release_page_clean(refcounted_page); 6860 6861 /* 6862 * No need for a manual TLB flush at this point, KVM has already done a 6863 * flush if there were SPTEs pointing at the previous page. 6864 */ 6865 read_unlock(&vcpu->kvm->mmu_lock); 6866 } 6867 6868 void vmx_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr) 6869 { 6870 u16 status; 6871 u8 old; 6872 6873 /* 6874 * If L2 is active, defer the SVI update until vmcs01 is loaded, as SVI 6875 * is only relevant for if and only if Virtual Interrupt Delivery is 6876 * enabled in vmcs12, and if VID is enabled then L2 EOIs affect L2's 6877 * vAPIC, not L1's vAPIC. KVM must update vmcs01 on the next nested 6878 * VM-Exit, otherwise L1 with run with a stale SVI. 6879 */ 6880 if (is_guest_mode(vcpu)) { 6881 /* 6882 * KVM is supposed to forward intercepted L2 EOIs to L1 if VID 6883 * is enabled in vmcs12; as above, the EOIs affect L2's vAPIC. 6884 * Note, userspace can stuff state while L2 is active; assert 6885 * that VID is disabled if and only if the vCPU is in KVM_RUN 6886 * to avoid false positives if userspace is setting APIC state. 6887 */ 6888 WARN_ON_ONCE(vcpu->wants_to_run && 6889 nested_cpu_has_vid(get_vmcs12(vcpu))); 6890 to_vmx(vcpu)->nested.update_vmcs01_hwapic_isr = true; 6891 return; 6892 } 6893 6894 if (max_isr == -1) 6895 max_isr = 0; 6896 6897 status = vmcs_read16(GUEST_INTR_STATUS); 6898 old = status >> 8; 6899 if (max_isr != old) { 6900 status &= 0xff; 6901 status |= max_isr << 8; 6902 vmcs_write16(GUEST_INTR_STATUS, status); 6903 } 6904 } 6905 6906 static void vmx_set_rvi(int vector) 6907 { 6908 u16 status; 6909 u8 old; 6910 6911 if (vector == -1) 6912 vector = 0; 6913 6914 status = vmcs_read16(GUEST_INTR_STATUS); 6915 old = (u8)status & 0xff; 6916 if ((u8)vector != old) { 6917 status &= ~0xff; 6918 status |= (u8)vector; 6919 vmcs_write16(GUEST_INTR_STATUS, status); 6920 } 6921 } 6922 6923 int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu) 6924 { 6925 struct vcpu_vt *vt = to_vt(vcpu); 6926 int max_irr; 6927 bool got_posted_interrupt; 6928 6929 if (KVM_BUG_ON(!enable_apicv, vcpu->kvm)) 6930 return -EIO; 6931 6932 if (pi_test_on(&vt->pi_desc)) { 6933 pi_clear_on(&vt->pi_desc); 6934 /* 6935 * IOMMU can write to PID.ON, so the barrier matters even on UP. 6936 * But on x86 this is just a compiler barrier anyway. 6937 */ 6938 smp_mb__after_atomic(); 6939 got_posted_interrupt = 6940 kvm_apic_update_irr(vcpu, vt->pi_desc.pir, &max_irr); 6941 } else { 6942 max_irr = kvm_lapic_find_highest_irr(vcpu); 6943 got_posted_interrupt = false; 6944 } 6945 6946 /* 6947 * Newly recognized interrupts are injected via either virtual interrupt 6948 * delivery (RVI) or KVM_REQ_EVENT. Virtual interrupt delivery is 6949 * disabled in two cases: 6950 * 6951 * 1) If L2 is running and the vCPU has a new pending interrupt. If L1 6952 * wants to exit on interrupts, KVM_REQ_EVENT is needed to synthesize a 6953 * VM-Exit to L1. If L1 doesn't want to exit, the interrupt is injected 6954 * into L2, but KVM doesn't use virtual interrupt delivery to inject 6955 * interrupts into L2, and so KVM_REQ_EVENT is again needed. 6956 * 6957 * 2) If APICv is disabled for this vCPU, assigned devices may still 6958 * attempt to post interrupts. The posted interrupt vector will cause 6959 * a VM-Exit and the subsequent entry will call sync_pir_to_irr. 6960 */ 6961 if (!is_guest_mode(vcpu) && kvm_vcpu_apicv_active(vcpu)) 6962 vmx_set_rvi(max_irr); 6963 else if (got_posted_interrupt) 6964 kvm_make_request(KVM_REQ_EVENT, vcpu); 6965 6966 return max_irr; 6967 } 6968 6969 void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) 6970 { 6971 if (!kvm_vcpu_apicv_active(vcpu)) 6972 return; 6973 6974 vmcs_write64(EOI_EXIT_BITMAP0, eoi_exit_bitmap[0]); 6975 vmcs_write64(EOI_EXIT_BITMAP1, eoi_exit_bitmap[1]); 6976 vmcs_write64(EOI_EXIT_BITMAP2, eoi_exit_bitmap[2]); 6977 vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]); 6978 } 6979 6980 void vmx_do_interrupt_irqoff(unsigned long entry); 6981 void vmx_do_nmi_irqoff(void); 6982 6983 static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu) 6984 { 6985 /* 6986 * Save xfd_err to guest_fpu before interrupt is enabled, so the 6987 * MSR value is not clobbered by the host activity before the guest 6988 * has chance to consume it. 6989 * 6990 * Update the guest's XFD_ERR if and only if XFD is enabled, as the #NM 6991 * interception may have been caused by L1 interception. Per the SDM, 6992 * XFD_ERR is not modified for non-XFD #NM, i.e. if CR0.TS=1. 6993 * 6994 * Note, XFD_ERR is updated _before_ the #NM interception check, i.e. 6995 * unlike CR2 and DR6, the value is not a payload that is attached to 6996 * the #NM exception. 6997 */ 6998 if (is_xfd_nm_fault(vcpu)) 6999 rdmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); 7000 } 7001 7002 static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info) 7003 { 7004 /* if exit due to PF check for async PF */ 7005 if (is_page_fault(intr_info)) 7006 vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags(); 7007 /* if exit due to NM, handle before interrupts are enabled */ 7008 else if (is_nm_fault(intr_info)) 7009 handle_nm_fault_irqoff(vcpu); 7010 /* Handle machine checks before interrupts are enabled */ 7011 else if (is_machine_check(intr_info)) 7012 kvm_machine_check(); 7013 } 7014 7015 static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu, 7016 u32 intr_info) 7017 { 7018 unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK; 7019 7020 if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm, 7021 "unexpected VM-Exit interrupt info: 0x%x", intr_info)) 7022 return; 7023 7024 /* 7025 * Invoke the kernel's IRQ handler for the vector. Use the FRED path 7026 * when it's available even if FRED isn't fully enabled, e.g. even if 7027 * FRED isn't supported in hardware, in order to avoid the indirect 7028 * CALL in the non-FRED path. 7029 */ 7030 kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ); 7031 if (IS_ENABLED(CONFIG_X86_FRED)) 7032 fred_entry_from_kvm(EVENT_TYPE_EXTINT, vector); 7033 else 7034 vmx_do_interrupt_irqoff(gate_offset((gate_desc *)host_idt_base + vector)); 7035 kvm_after_interrupt(vcpu); 7036 7037 vcpu->arch.at_instruction_boundary = true; 7038 } 7039 7040 void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu) 7041 { 7042 if (to_vt(vcpu)->emulation_required) 7043 return; 7044 7045 if (vmx_get_exit_reason(vcpu).basic == EXIT_REASON_EXTERNAL_INTERRUPT) 7046 handle_external_interrupt_irqoff(vcpu, vmx_get_intr_info(vcpu)); 7047 else if (vmx_get_exit_reason(vcpu).basic == EXIT_REASON_EXCEPTION_NMI) 7048 handle_exception_irqoff(vcpu, vmx_get_intr_info(vcpu)); 7049 } 7050 7051 /* 7052 * The kvm parameter can be NULL (module initialization, or invocation before 7053 * VM creation). Be sure to check the kvm parameter before using it. 7054 */ 7055 bool vmx_has_emulated_msr(struct kvm *kvm, u32 index) 7056 { 7057 switch (index) { 7058 case MSR_IA32_SMBASE: 7059 if (!IS_ENABLED(CONFIG_KVM_SMM)) 7060 return false; 7061 /* 7062 * We cannot do SMM unless we can run the guest in big 7063 * real mode. 7064 */ 7065 return enable_unrestricted_guest || emulate_invalid_guest_state; 7066 case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR: 7067 return nested; 7068 case MSR_AMD64_VIRT_SPEC_CTRL: 7069 case MSR_AMD64_TSC_RATIO: 7070 /* This is AMD only. */ 7071 return false; 7072 default: 7073 return true; 7074 } 7075 } 7076 7077 static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx) 7078 { 7079 u32 exit_intr_info; 7080 bool unblock_nmi; 7081 u8 vector; 7082 bool idtv_info_valid; 7083 7084 idtv_info_valid = vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK; 7085 7086 if (enable_vnmi) { 7087 if (vmx->loaded_vmcs->nmi_known_unmasked) 7088 return; 7089 7090 exit_intr_info = vmx_get_intr_info(&vmx->vcpu); 7091 unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0; 7092 vector = exit_intr_info & INTR_INFO_VECTOR_MASK; 7093 /* 7094 * SDM 3: 27.7.1.2 (September 2008) 7095 * Re-set bit "block by NMI" before VM entry if vmexit caused by 7096 * a guest IRET fault. 7097 * SDM 3: 23.2.2 (September 2008) 7098 * Bit 12 is undefined in any of the following cases: 7099 * If the VM exit sets the valid bit in the IDT-vectoring 7100 * information field. 7101 * If the VM exit is due to a double fault. 7102 */ 7103 if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi && 7104 vector != DF_VECTOR && !idtv_info_valid) 7105 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 7106 GUEST_INTR_STATE_NMI); 7107 else 7108 vmx->loaded_vmcs->nmi_known_unmasked = 7109 !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) 7110 & GUEST_INTR_STATE_NMI); 7111 } else if (unlikely(vmx->loaded_vmcs->soft_vnmi_blocked)) 7112 vmx->loaded_vmcs->vnmi_blocked_time += 7113 ktime_to_ns(ktime_sub(ktime_get(), 7114 vmx->loaded_vmcs->entry_time)); 7115 } 7116 7117 static void __vmx_complete_interrupts(struct kvm_vcpu *vcpu, 7118 u32 idt_vectoring_info, 7119 int instr_len_field, 7120 int error_code_field) 7121 { 7122 u8 vector; 7123 int type; 7124 bool idtv_info_valid; 7125 7126 idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK; 7127 7128 vcpu->arch.nmi_injected = false; 7129 kvm_clear_exception_queue(vcpu); 7130 kvm_clear_interrupt_queue(vcpu); 7131 7132 if (!idtv_info_valid) 7133 return; 7134 7135 kvm_make_request(KVM_REQ_EVENT, vcpu); 7136 7137 vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK; 7138 type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK; 7139 7140 switch (type) { 7141 case INTR_TYPE_NMI_INTR: 7142 vcpu->arch.nmi_injected = true; 7143 /* 7144 * SDM 3: 27.7.1.2 (September 2008) 7145 * Clear bit "block by NMI" before VM entry if a NMI 7146 * delivery faulted. 7147 */ 7148 vmx_set_nmi_mask(vcpu, false); 7149 break; 7150 case INTR_TYPE_SOFT_EXCEPTION: 7151 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field); 7152 fallthrough; 7153 case INTR_TYPE_HARD_EXCEPTION: { 7154 u32 error_code = 0; 7155 7156 if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) 7157 error_code = vmcs_read32(error_code_field); 7158 7159 kvm_requeue_exception(vcpu, vector, 7160 idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK, 7161 error_code); 7162 break; 7163 } 7164 case INTR_TYPE_SOFT_INTR: 7165 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field); 7166 fallthrough; 7167 case INTR_TYPE_EXT_INTR: 7168 kvm_queue_interrupt(vcpu, vector, type == INTR_TYPE_SOFT_INTR); 7169 break; 7170 default: 7171 break; 7172 } 7173 } 7174 7175 static void vmx_complete_interrupts(struct vcpu_vmx *vmx) 7176 { 7177 __vmx_complete_interrupts(&vmx->vcpu, vmx->idt_vectoring_info, 7178 VM_EXIT_INSTRUCTION_LEN, 7179 IDT_VECTORING_ERROR_CODE); 7180 } 7181 7182 void vmx_cancel_injection(struct kvm_vcpu *vcpu) 7183 { 7184 __vmx_complete_interrupts(vcpu, 7185 vmcs_read32(VM_ENTRY_INTR_INFO_FIELD), 7186 VM_ENTRY_INSTRUCTION_LEN, 7187 VM_ENTRY_EXCEPTION_ERROR_CODE); 7188 7189 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); 7190 } 7191 7192 static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx) 7193 { 7194 int i, nr_msrs; 7195 struct perf_guest_switch_msr *msrs; 7196 struct kvm_pmu *pmu = vcpu_to_pmu(&vmx->vcpu); 7197 7198 pmu->host_cross_mapped_mask = 0; 7199 if (pmu->pebs_enable & pmu->global_ctrl) 7200 intel_pmu_cross_mapped_check(pmu); 7201 7202 /* Note, nr_msrs may be garbage if perf_guest_get_msrs() returns NULL. */ 7203 msrs = perf_guest_get_msrs(&nr_msrs, (void *)pmu); 7204 if (!msrs) 7205 return; 7206 7207 for (i = 0; i < nr_msrs; i++) 7208 if (msrs[i].host == msrs[i].guest) 7209 clear_atomic_switch_msr(vmx, msrs[i].msr); 7210 else 7211 add_atomic_switch_msr(vmx, msrs[i].msr, msrs[i].guest, 7212 msrs[i].host, false); 7213 } 7214 7215 static void vmx_update_hv_timer(struct kvm_vcpu *vcpu, bool force_immediate_exit) 7216 { 7217 struct vcpu_vmx *vmx = to_vmx(vcpu); 7218 u64 tscl; 7219 u32 delta_tsc; 7220 7221 if (force_immediate_exit) { 7222 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, 0); 7223 vmx->loaded_vmcs->hv_timer_soft_disabled = false; 7224 } else if (vmx->hv_deadline_tsc != -1) { 7225 tscl = rdtsc(); 7226 if (vmx->hv_deadline_tsc > tscl) 7227 /* set_hv_timer ensures the delta fits in 32-bits */ 7228 delta_tsc = (u32)((vmx->hv_deadline_tsc - tscl) >> 7229 cpu_preemption_timer_multi); 7230 else 7231 delta_tsc = 0; 7232 7233 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, delta_tsc); 7234 vmx->loaded_vmcs->hv_timer_soft_disabled = false; 7235 } else if (!vmx->loaded_vmcs->hv_timer_soft_disabled) { 7236 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, -1); 7237 vmx->loaded_vmcs->hv_timer_soft_disabled = true; 7238 } 7239 } 7240 7241 void noinstr vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp) 7242 { 7243 if (unlikely(host_rsp != vmx->loaded_vmcs->host_state.rsp)) { 7244 vmx->loaded_vmcs->host_state.rsp = host_rsp; 7245 vmcs_writel(HOST_RSP, host_rsp); 7246 } 7247 } 7248 7249 void noinstr vmx_spec_ctrl_restore_host(struct vcpu_vmx *vmx, 7250 unsigned int flags) 7251 { 7252 u64 hostval = this_cpu_read(x86_spec_ctrl_current); 7253 7254 if (!cpu_feature_enabled(X86_FEATURE_MSR_SPEC_CTRL)) 7255 return; 7256 7257 if (flags & VMX_RUN_SAVE_SPEC_CTRL) 7258 vmx->spec_ctrl = native_rdmsrq(MSR_IA32_SPEC_CTRL); 7259 7260 /* 7261 * If the guest/host SPEC_CTRL values differ, restore the host value. 7262 * 7263 * For legacy IBRS, the IBRS bit always needs to be written after 7264 * transitioning from a less privileged predictor mode, regardless of 7265 * whether the guest/host values differ. 7266 */ 7267 if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) || 7268 vmx->spec_ctrl != hostval) 7269 native_wrmsrq(MSR_IA32_SPEC_CTRL, hostval); 7270 7271 barrier_nospec(); 7272 } 7273 7274 static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu, 7275 bool force_immediate_exit) 7276 { 7277 /* 7278 * If L2 is active, some VMX preemption timer exits can be handled in 7279 * the fastpath even, all other exits must use the slow path. 7280 */ 7281 if (is_guest_mode(vcpu) && 7282 vmx_get_exit_reason(vcpu).basic != EXIT_REASON_PREEMPTION_TIMER) 7283 return EXIT_FASTPATH_NONE; 7284 7285 switch (vmx_get_exit_reason(vcpu).basic) { 7286 case EXIT_REASON_MSR_WRITE: 7287 return handle_fastpath_wrmsr(vcpu); 7288 case EXIT_REASON_MSR_WRITE_IMM: 7289 return handle_fastpath_wrmsr_imm(vcpu, vmx_get_exit_qual(vcpu), 7290 vmx_get_msr_imm_reg(vcpu)); 7291 case EXIT_REASON_PREEMPTION_TIMER: 7292 return handle_fastpath_preemption_timer(vcpu, force_immediate_exit); 7293 case EXIT_REASON_HLT: 7294 return handle_fastpath_hlt(vcpu); 7295 case EXIT_REASON_INVD: 7296 return handle_fastpath_invd(vcpu); 7297 default: 7298 return EXIT_FASTPATH_NONE; 7299 } 7300 } 7301 7302 noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu) 7303 { 7304 if ((u16)vmx_get_exit_reason(vcpu).basic != EXIT_REASON_EXCEPTION_NMI || 7305 !is_nmi(vmx_get_intr_info(vcpu))) 7306 return; 7307 7308 kvm_before_interrupt(vcpu, KVM_HANDLING_NMI); 7309 if (cpu_feature_enabled(X86_FEATURE_FRED)) 7310 fred_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR); 7311 else 7312 vmx_do_nmi_irqoff(); 7313 kvm_after_interrupt(vcpu); 7314 } 7315 7316 static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, 7317 unsigned int flags) 7318 { 7319 struct vcpu_vmx *vmx = to_vmx(vcpu); 7320 7321 guest_state_enter_irqoff(); 7322 7323 /* 7324 * L1D Flush includes CPU buffer clear to mitigate MDS, but VERW 7325 * mitigation for MDS is done late in VMentry and is still 7326 * executed in spite of L1D Flush. This is because an extra VERW 7327 * should not matter much after the big hammer L1D Flush. 7328 * 7329 * cpu_buf_vm_clear is used when system is not vulnerable to MDS/TAA, 7330 * and is affected by MMIO Stale Data. In such cases mitigation in only 7331 * needed against an MMIO capable guest. 7332 */ 7333 if (static_branch_unlikely(&vmx_l1d_should_flush)) 7334 vmx_l1d_flush(vcpu); 7335 else if (static_branch_unlikely(&cpu_buf_vm_clear) && 7336 (flags & VMX_RUN_CLEAR_CPU_BUFFERS_FOR_MMIO)) 7337 x86_clear_cpu_buffers(); 7338 7339 vmx_disable_fb_clear(vmx); 7340 7341 if (vcpu->arch.cr2 != native_read_cr2()) 7342 native_write_cr2(vcpu->arch.cr2); 7343 7344 vmx->fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs, 7345 flags); 7346 7347 vcpu->arch.cr2 = native_read_cr2(); 7348 vcpu->arch.regs_avail &= ~VMX_REGS_LAZY_LOAD_SET; 7349 7350 vmx->idt_vectoring_info = 0; 7351 7352 vmx_enable_fb_clear(vmx); 7353 7354 if (unlikely(vmx->fail)) { 7355 vmx->vt.exit_reason.full = 0xdead; 7356 goto out; 7357 } 7358 7359 vmx->vt.exit_reason.full = vmcs_read32(VM_EXIT_REASON); 7360 if (likely(!vmx_get_exit_reason(vcpu).failed_vmentry)) 7361 vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); 7362 7363 vmx_handle_nmi(vcpu); 7364 7365 out: 7366 guest_state_exit_irqoff(); 7367 } 7368 7369 fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) 7370 { 7371 bool force_immediate_exit = run_flags & KVM_RUN_FORCE_IMMEDIATE_EXIT; 7372 struct vcpu_vmx *vmx = to_vmx(vcpu); 7373 unsigned long cr3, cr4; 7374 7375 /* Record the guest's net vcpu time for enforced NMI injections. */ 7376 if (unlikely(!enable_vnmi && 7377 vmx->loaded_vmcs->soft_vnmi_blocked)) 7378 vmx->loaded_vmcs->entry_time = ktime_get(); 7379 7380 /* 7381 * Don't enter VMX if guest state is invalid, let the exit handler 7382 * start emulation until we arrive back to a valid state. Synthesize a 7383 * consistency check VM-Exit due to invalid guest state and bail. 7384 */ 7385 if (unlikely(vmx->vt.emulation_required)) { 7386 vmx->fail = 0; 7387 7388 vmx->vt.exit_reason.full = EXIT_REASON_INVALID_STATE; 7389 vmx->vt.exit_reason.failed_vmentry = 1; 7390 kvm_register_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_1); 7391 vmx->vt.exit_qualification = ENTRY_FAIL_DEFAULT; 7392 kvm_register_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_2); 7393 vmx->vt.exit_intr_info = 0; 7394 return EXIT_FASTPATH_NONE; 7395 } 7396 7397 trace_kvm_entry(vcpu, force_immediate_exit); 7398 7399 if (vmx->ple_window_dirty) { 7400 vmx->ple_window_dirty = false; 7401 vmcs_write32(PLE_WINDOW, vmx->ple_window); 7402 } 7403 7404 /* 7405 * We did this in prepare_switch_to_guest, because it needs to 7406 * be within srcu_read_lock. 7407 */ 7408 WARN_ON_ONCE(vmx->nested.need_vmcs12_to_shadow_sync); 7409 7410 if (kvm_register_is_dirty(vcpu, VCPU_REGS_RSP)) 7411 vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]); 7412 if (kvm_register_is_dirty(vcpu, VCPU_REGS_RIP)) 7413 vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]); 7414 vcpu->arch.regs_dirty = 0; 7415 7416 if (run_flags & KVM_RUN_LOAD_GUEST_DR6) 7417 set_debugreg(vcpu->arch.dr6, 6); 7418 7419 if (run_flags & KVM_RUN_LOAD_DEBUGCTL) 7420 vmx_reload_guest_debugctl(vcpu); 7421 7422 /* 7423 * Refresh vmcs.HOST_CR3 if necessary. This must be done immediately 7424 * prior to VM-Enter, as the kernel may load a new ASID (PCID) any time 7425 * it switches back to the current->mm, which can occur in KVM context 7426 * when switching to a temporary mm to patch kernel code, e.g. if KVM 7427 * toggles a static key while handling a VM-Exit. 7428 */ 7429 cr3 = __get_current_cr3_fast(); 7430 if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) { 7431 vmcs_writel(HOST_CR3, cr3); 7432 vmx->loaded_vmcs->host_state.cr3 = cr3; 7433 } 7434 7435 cr4 = cr4_read_shadow(); 7436 if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) { 7437 vmcs_writel(HOST_CR4, cr4); 7438 vmx->loaded_vmcs->host_state.cr4 = cr4; 7439 } 7440 7441 /* When single-stepping over STI and MOV SS, we must clear the 7442 * corresponding interruptibility bits in the guest state. Otherwise 7443 * vmentry fails as it then expects bit 14 (BS) in pending debug 7444 * exceptions being set, but that's not correct for the guest debugging 7445 * case. */ 7446 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) 7447 vmx_set_interrupt_shadow(vcpu, 0); 7448 7449 kvm_load_guest_xsave_state(vcpu); 7450 7451 pt_guest_enter(vmx); 7452 7453 atomic_switch_perf_msrs(vmx); 7454 if (intel_pmu_lbr_is_enabled(vcpu)) 7455 vmx_passthrough_lbr_msrs(vcpu); 7456 7457 if (enable_preemption_timer) 7458 vmx_update_hv_timer(vcpu, force_immediate_exit); 7459 else if (force_immediate_exit) 7460 smp_send_reschedule(vcpu->cpu); 7461 7462 kvm_wait_lapic_expire(vcpu); 7463 7464 /* The actual VMENTER/EXIT is in the .noinstr.text section. */ 7465 vmx_vcpu_enter_exit(vcpu, __vmx_vcpu_run_flags(vmx)); 7466 7467 /* All fields are clean at this point */ 7468 if (kvm_is_using_evmcs()) { 7469 current_evmcs->hv_clean_fields |= 7470 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 7471 7472 current_evmcs->hv_vp_id = kvm_hv_get_vpindex(vcpu); 7473 } 7474 7475 /* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */ 7476 if (vcpu->arch.host_debugctl) 7477 update_debugctlmsr(vcpu->arch.host_debugctl); 7478 7479 #ifndef CONFIG_X86_64 7480 /* 7481 * The sysexit path does not restore ds/es, so we must set them to 7482 * a reasonable value ourselves. 7483 * 7484 * We can't defer this to vmx_prepare_switch_to_host() since that 7485 * function may be executed in interrupt context, which saves and 7486 * restore segments around it, nullifying its effect. 7487 */ 7488 loadsegment(ds, __USER_DS); 7489 loadsegment(es, __USER_DS); 7490 #endif 7491 7492 pt_guest_exit(vmx); 7493 7494 kvm_load_host_xsave_state(vcpu); 7495 7496 if (is_guest_mode(vcpu)) { 7497 /* 7498 * Track VMLAUNCH/VMRESUME that have made past guest state 7499 * checking. 7500 */ 7501 if (vmx->nested.nested_run_pending && 7502 !vmx_get_exit_reason(vcpu).failed_vmentry) 7503 ++vcpu->stat.nested_run; 7504 7505 vmx->nested.nested_run_pending = 0; 7506 } 7507 7508 if (unlikely(vmx->fail)) 7509 return EXIT_FASTPATH_NONE; 7510 7511 if (unlikely((u16)vmx_get_exit_reason(vcpu).basic == EXIT_REASON_MCE_DURING_VMENTRY)) 7512 kvm_machine_check(); 7513 7514 trace_kvm_exit(vcpu, KVM_ISA_VMX); 7515 7516 if (unlikely(vmx_get_exit_reason(vcpu).failed_vmentry)) 7517 return EXIT_FASTPATH_NONE; 7518 7519 vmx->loaded_vmcs->launched = 1; 7520 7521 vmx_recover_nmi_blocking(vmx); 7522 vmx_complete_interrupts(vmx); 7523 7524 return vmx_exit_handlers_fastpath(vcpu, force_immediate_exit); 7525 } 7526 7527 void vmx_vcpu_free(struct kvm_vcpu *vcpu) 7528 { 7529 struct vcpu_vmx *vmx = to_vmx(vcpu); 7530 7531 if (enable_pml) 7532 vmx_destroy_pml_buffer(vmx); 7533 free_vpid(vmx->vpid); 7534 nested_vmx_free_vcpu(vcpu); 7535 free_loaded_vmcs(vmx->loaded_vmcs); 7536 free_page((unsigned long)vmx->ve_info); 7537 } 7538 7539 int vmx_vcpu_create(struct kvm_vcpu *vcpu) 7540 { 7541 struct vmx_uret_msr *tsx_ctrl; 7542 struct vcpu_vmx *vmx; 7543 int i, err; 7544 7545 BUILD_BUG_ON(offsetof(struct vcpu_vmx, vcpu) != 0); 7546 vmx = to_vmx(vcpu); 7547 7548 INIT_LIST_HEAD(&vmx->vt.pi_wakeup_list); 7549 7550 err = -ENOMEM; 7551 7552 vmx->vpid = allocate_vpid(); 7553 7554 /* 7555 * If PML is turned on, failure on enabling PML just results in failure 7556 * of creating the vcpu, therefore we can simplify PML logic (by 7557 * avoiding dealing with cases, such as enabling PML partially on vcpus 7558 * for the guest), etc. 7559 */ 7560 if (enable_pml) { 7561 vmx->pml_pg = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 7562 if (!vmx->pml_pg) 7563 goto free_vpid; 7564 } 7565 7566 for (i = 0; i < kvm_nr_uret_msrs; ++i) 7567 vmx->guest_uret_msrs[i].mask = -1ull; 7568 if (boot_cpu_has(X86_FEATURE_RTM)) { 7569 /* 7570 * TSX_CTRL_CPUID_CLEAR is handled in the CPUID interception. 7571 * Keep the host value unchanged to avoid changing CPUID bits 7572 * under the host kernel's feet. 7573 */ 7574 tsx_ctrl = vmx_find_uret_msr(vmx, MSR_IA32_TSX_CTRL); 7575 if (tsx_ctrl) 7576 tsx_ctrl->mask = ~(u64)TSX_CTRL_CPUID_CLEAR; 7577 } 7578 7579 err = alloc_loaded_vmcs(&vmx->vmcs01); 7580 if (err < 0) 7581 goto free_pml; 7582 7583 /* 7584 * Use Hyper-V 'Enlightened MSR Bitmap' feature when KVM runs as a 7585 * nested (L1) hypervisor and Hyper-V in L0 supports it. Enable the 7586 * feature only for vmcs01, KVM currently isn't equipped to realize any 7587 * performance benefits from enabling it for vmcs02. 7588 */ 7589 if (kvm_is_using_evmcs() && 7590 (ms_hyperv.nested_features & HV_X64_NESTED_MSR_BITMAP)) { 7591 struct hv_enlightened_vmcs *evmcs = (void *)vmx->vmcs01.vmcs; 7592 7593 evmcs->hv_enlightenments_control.msr_bitmap = 1; 7594 } 7595 7596 vmx->loaded_vmcs = &vmx->vmcs01; 7597 7598 if (cpu_need_virtualize_apic_accesses(vcpu)) { 7599 err = kvm_alloc_apic_access_page(vcpu->kvm); 7600 if (err) 7601 goto free_vmcs; 7602 } 7603 7604 if (enable_ept && !enable_unrestricted_guest) { 7605 err = init_rmode_identity_map(vcpu->kvm); 7606 if (err) 7607 goto free_vmcs; 7608 } 7609 7610 err = -ENOMEM; 7611 if (vmcs_config.cpu_based_2nd_exec_ctrl & SECONDARY_EXEC_EPT_VIOLATION_VE) { 7612 struct page *page; 7613 7614 BUILD_BUG_ON(sizeof(*vmx->ve_info) > PAGE_SIZE); 7615 7616 /* ve_info must be page aligned. */ 7617 page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 7618 if (!page) 7619 goto free_vmcs; 7620 7621 vmx->ve_info = page_to_virt(page); 7622 } 7623 7624 if (vmx_can_use_ipiv(vcpu)) 7625 WRITE_ONCE(to_kvm_vmx(vcpu->kvm)->pid_table[vcpu->vcpu_id], 7626 __pa(&vmx->vt.pi_desc) | PID_TABLE_ENTRY_VALID); 7627 7628 return 0; 7629 7630 free_vmcs: 7631 free_loaded_vmcs(vmx->loaded_vmcs); 7632 free_pml: 7633 vmx_destroy_pml_buffer(vmx); 7634 free_vpid: 7635 free_vpid(vmx->vpid); 7636 return err; 7637 } 7638 7639 #define L1TF_MSG_SMT "L1TF CPU bug present and SMT on, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n" 7640 #define L1TF_MSG_L1D "L1TF CPU bug present and virtualization mitigation disabled, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n" 7641 7642 int vmx_vm_init(struct kvm *kvm) 7643 { 7644 if (!ple_gap) 7645 kvm_disable_exits(kvm, KVM_X86_DISABLE_EXITS_PAUSE); 7646 7647 if (boot_cpu_has(X86_BUG_L1TF) && enable_ept) { 7648 switch (l1tf_mitigation) { 7649 case L1TF_MITIGATION_OFF: 7650 case L1TF_MITIGATION_FLUSH_NOWARN: 7651 /* 'I explicitly don't care' is set */ 7652 break; 7653 case L1TF_MITIGATION_AUTO: 7654 case L1TF_MITIGATION_FLUSH: 7655 case L1TF_MITIGATION_FLUSH_NOSMT: 7656 case L1TF_MITIGATION_FULL: 7657 /* 7658 * Warn upon starting the first VM in a potentially 7659 * insecure environment. 7660 */ 7661 if (sched_smt_active()) 7662 pr_warn_once(L1TF_MSG_SMT); 7663 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER) 7664 pr_warn_once(L1TF_MSG_L1D); 7665 break; 7666 case L1TF_MITIGATION_FULL_FORCE: 7667 /* Flush is enforced */ 7668 break; 7669 } 7670 } 7671 7672 if (enable_pml) 7673 kvm->arch.cpu_dirty_log_size = PML_LOG_NR_ENTRIES; 7674 return 0; 7675 } 7676 7677 static inline bool vmx_ignore_guest_pat(struct kvm *kvm) 7678 { 7679 /* 7680 * Non-coherent DMA devices need the guest to flush CPU properly. 7681 * In that case it is not possible to map all guest RAM as WB, so 7682 * always trust guest PAT. 7683 */ 7684 return !kvm_arch_has_noncoherent_dma(kvm) && 7685 kvm_check_has_quirk(kvm, KVM_X86_QUIRK_IGNORE_GUEST_PAT); 7686 } 7687 7688 u8 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio) 7689 { 7690 /* 7691 * Force UC for host MMIO regions, as allowing the guest to access MMIO 7692 * with cacheable accesses will result in Machine Checks. 7693 */ 7694 if (is_mmio) 7695 return MTRR_TYPE_UNCACHABLE << VMX_EPT_MT_EPTE_SHIFT; 7696 7697 /* Force WB if ignoring guest PAT */ 7698 if (vmx_ignore_guest_pat(vcpu->kvm)) 7699 return (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT) | VMX_EPT_IPAT_BIT; 7700 7701 return (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT); 7702 } 7703 7704 static void vmcs_set_secondary_exec_control(struct vcpu_vmx *vmx, u32 new_ctl) 7705 { 7706 /* 7707 * These bits in the secondary execution controls field 7708 * are dynamic, the others are mostly based on the hypervisor 7709 * architecture and the guest's CPUID. Do not touch the 7710 * dynamic bits. 7711 */ 7712 u32 mask = 7713 SECONDARY_EXEC_SHADOW_VMCS | 7714 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 7715 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 7716 SECONDARY_EXEC_DESC; 7717 7718 u32 cur_ctl = secondary_exec_controls_get(vmx); 7719 7720 secondary_exec_controls_set(vmx, (new_ctl & ~mask) | (cur_ctl & mask)); 7721 } 7722 7723 /* 7724 * Generate MSR_IA32_VMX_CR{0,4}_FIXED1 according to CPUID. Only set bits 7725 * (indicating "allowed-1") if they are supported in the guest's CPUID. 7726 */ 7727 static void nested_vmx_cr_fixed1_bits_update(struct kvm_vcpu *vcpu) 7728 { 7729 struct vcpu_vmx *vmx = to_vmx(vcpu); 7730 struct kvm_cpuid_entry2 *entry; 7731 7732 vmx->nested.msrs.cr0_fixed1 = 0xffffffff; 7733 vmx->nested.msrs.cr4_fixed1 = X86_CR4_PCE; 7734 7735 #define cr4_fixed1_update(_cr4_mask, _reg, _cpuid_mask) do { \ 7736 if (entry && (entry->_reg & (_cpuid_mask))) \ 7737 vmx->nested.msrs.cr4_fixed1 |= (_cr4_mask); \ 7738 } while (0) 7739 7740 entry = kvm_find_cpuid_entry(vcpu, 0x1); 7741 cr4_fixed1_update(X86_CR4_VME, edx, feature_bit(VME)); 7742 cr4_fixed1_update(X86_CR4_PVI, edx, feature_bit(VME)); 7743 cr4_fixed1_update(X86_CR4_TSD, edx, feature_bit(TSC)); 7744 cr4_fixed1_update(X86_CR4_DE, edx, feature_bit(DE)); 7745 cr4_fixed1_update(X86_CR4_PSE, edx, feature_bit(PSE)); 7746 cr4_fixed1_update(X86_CR4_PAE, edx, feature_bit(PAE)); 7747 cr4_fixed1_update(X86_CR4_MCE, edx, feature_bit(MCE)); 7748 cr4_fixed1_update(X86_CR4_PGE, edx, feature_bit(PGE)); 7749 cr4_fixed1_update(X86_CR4_OSFXSR, edx, feature_bit(FXSR)); 7750 cr4_fixed1_update(X86_CR4_OSXMMEXCPT, edx, feature_bit(XMM)); 7751 cr4_fixed1_update(X86_CR4_VMXE, ecx, feature_bit(VMX)); 7752 cr4_fixed1_update(X86_CR4_SMXE, ecx, feature_bit(SMX)); 7753 cr4_fixed1_update(X86_CR4_PCIDE, ecx, feature_bit(PCID)); 7754 cr4_fixed1_update(X86_CR4_OSXSAVE, ecx, feature_bit(XSAVE)); 7755 7756 entry = kvm_find_cpuid_entry_index(vcpu, 0x7, 0); 7757 cr4_fixed1_update(X86_CR4_FSGSBASE, ebx, feature_bit(FSGSBASE)); 7758 cr4_fixed1_update(X86_CR4_SMEP, ebx, feature_bit(SMEP)); 7759 cr4_fixed1_update(X86_CR4_SMAP, ebx, feature_bit(SMAP)); 7760 cr4_fixed1_update(X86_CR4_PKE, ecx, feature_bit(PKU)); 7761 cr4_fixed1_update(X86_CR4_UMIP, ecx, feature_bit(UMIP)); 7762 cr4_fixed1_update(X86_CR4_LA57, ecx, feature_bit(LA57)); 7763 cr4_fixed1_update(X86_CR4_CET, ecx, feature_bit(SHSTK)); 7764 cr4_fixed1_update(X86_CR4_CET, edx, feature_bit(IBT)); 7765 7766 entry = kvm_find_cpuid_entry_index(vcpu, 0x7, 1); 7767 cr4_fixed1_update(X86_CR4_LAM_SUP, eax, feature_bit(LAM)); 7768 7769 #undef cr4_fixed1_update 7770 } 7771 7772 static void update_intel_pt_cfg(struct kvm_vcpu *vcpu) 7773 { 7774 struct vcpu_vmx *vmx = to_vmx(vcpu); 7775 struct kvm_cpuid_entry2 *best = NULL; 7776 int i; 7777 7778 for (i = 0; i < PT_CPUID_LEAVES; i++) { 7779 best = kvm_find_cpuid_entry_index(vcpu, 0x14, i); 7780 if (!best) 7781 return; 7782 vmx->pt_desc.caps[CPUID_EAX + i*PT_CPUID_REGS_NUM] = best->eax; 7783 vmx->pt_desc.caps[CPUID_EBX + i*PT_CPUID_REGS_NUM] = best->ebx; 7784 vmx->pt_desc.caps[CPUID_ECX + i*PT_CPUID_REGS_NUM] = best->ecx; 7785 vmx->pt_desc.caps[CPUID_EDX + i*PT_CPUID_REGS_NUM] = best->edx; 7786 } 7787 7788 /* Get the number of configurable Address Ranges for filtering */ 7789 vmx->pt_desc.num_address_ranges = intel_pt_validate_cap(vmx->pt_desc.caps, 7790 PT_CAP_num_address_ranges); 7791 7792 /* Initialize and clear the no dependency bits */ 7793 vmx->pt_desc.ctl_bitmask = ~(RTIT_CTL_TRACEEN | RTIT_CTL_OS | 7794 RTIT_CTL_USR | RTIT_CTL_TSC_EN | RTIT_CTL_DISRETC | 7795 RTIT_CTL_BRANCH_EN); 7796 7797 /* 7798 * If CPUID.(EAX=14H,ECX=0):EBX[0]=1 CR3Filter can be set otherwise 7799 * will inject an #GP 7800 */ 7801 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_cr3_filtering)) 7802 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_CR3EN; 7803 7804 /* 7805 * If CPUID.(EAX=14H,ECX=0):EBX[1]=1 CYCEn, CycThresh and 7806 * PSBFreq can be set 7807 */ 7808 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc)) 7809 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_CYCLEACC | 7810 RTIT_CTL_CYC_THRESH | RTIT_CTL_PSB_FREQ); 7811 7812 /* 7813 * If CPUID.(EAX=14H,ECX=0):EBX[3]=1 MTCEn and MTCFreq can be set 7814 */ 7815 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc)) 7816 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_MTC_EN | 7817 RTIT_CTL_MTC_RANGE); 7818 7819 /* If CPUID.(EAX=14H,ECX=0):EBX[4]=1 FUPonPTW and PTWEn can be set */ 7820 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_ptwrite)) 7821 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_FUP_ON_PTW | 7822 RTIT_CTL_PTW_EN); 7823 7824 /* If CPUID.(EAX=14H,ECX=0):EBX[5]=1 PwrEvEn can be set */ 7825 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_power_event_trace)) 7826 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_PWR_EVT_EN; 7827 7828 /* If CPUID.(EAX=14H,ECX=0):ECX[0]=1 ToPA can be set */ 7829 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_topa_output)) 7830 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_TOPA; 7831 7832 /* If CPUID.(EAX=14H,ECX=0):ECX[3]=1 FabricEn can be set */ 7833 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_output_subsys)) 7834 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_FABRIC_EN; 7835 7836 /* unmask address range configure area */ 7837 for (i = 0; i < vmx->pt_desc.num_address_ranges; i++) 7838 vmx->pt_desc.ctl_bitmask &= ~(0xfULL << (32 + i * 4)); 7839 } 7840 7841 void vmx_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) 7842 { 7843 struct vcpu_vmx *vmx = to_vmx(vcpu); 7844 7845 /* 7846 * XSAVES is effectively enabled if and only if XSAVE is also exposed 7847 * to the guest. XSAVES depends on CR4.OSXSAVE, and CR4.OSXSAVE can be 7848 * set if and only if XSAVE is supported. 7849 */ 7850 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_XSAVE)) 7851 guest_cpu_cap_clear(vcpu, X86_FEATURE_XSAVES); 7852 7853 vmx_setup_uret_msrs(vmx); 7854 7855 if (cpu_has_secondary_exec_ctrls()) 7856 vmcs_set_secondary_exec_control(vmx, 7857 vmx_secondary_exec_control(vmx)); 7858 7859 if (guest_cpu_cap_has(vcpu, X86_FEATURE_VMX)) 7860 vmx->msr_ia32_feature_control_valid_bits |= 7861 FEAT_CTL_VMX_ENABLED_INSIDE_SMX | 7862 FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX; 7863 else 7864 vmx->msr_ia32_feature_control_valid_bits &= 7865 ~(FEAT_CTL_VMX_ENABLED_INSIDE_SMX | 7866 FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX); 7867 7868 if (guest_cpu_cap_has(vcpu, X86_FEATURE_VMX)) 7869 nested_vmx_cr_fixed1_bits_update(vcpu); 7870 7871 if (boot_cpu_has(X86_FEATURE_INTEL_PT) && 7872 guest_cpu_cap_has(vcpu, X86_FEATURE_INTEL_PT)) 7873 update_intel_pt_cfg(vcpu); 7874 7875 if (boot_cpu_has(X86_FEATURE_RTM)) { 7876 struct vmx_uret_msr *msr; 7877 msr = vmx_find_uret_msr(vmx, MSR_IA32_TSX_CTRL); 7878 if (msr) { 7879 bool enabled = guest_cpu_cap_has(vcpu, X86_FEATURE_RTM); 7880 vmx_set_guest_uret_msr(vmx, msr, enabled ? 0 : TSX_CTRL_RTM_DISABLE); 7881 } 7882 } 7883 7884 set_cr4_guest_host_mask(vmx); 7885 7886 vmx_write_encls_bitmap(vcpu, NULL); 7887 if (guest_cpu_cap_has(vcpu, X86_FEATURE_SGX)) 7888 vmx->msr_ia32_feature_control_valid_bits |= FEAT_CTL_SGX_ENABLED; 7889 else 7890 vmx->msr_ia32_feature_control_valid_bits &= ~FEAT_CTL_SGX_ENABLED; 7891 7892 if (guest_cpu_cap_has(vcpu, X86_FEATURE_SGX_LC)) 7893 vmx->msr_ia32_feature_control_valid_bits |= 7894 FEAT_CTL_SGX_LC_ENABLED; 7895 else 7896 vmx->msr_ia32_feature_control_valid_bits &= 7897 ~FEAT_CTL_SGX_LC_ENABLED; 7898 7899 /* Refresh #PF interception to account for MAXPHYADDR changes. */ 7900 vmx_update_exception_bitmap(vcpu); 7901 } 7902 7903 static __init u64 vmx_get_perf_capabilities(void) 7904 { 7905 u64 perf_cap = PERF_CAP_FW_WRITES; 7906 u64 host_perf_cap = 0; 7907 7908 if (!enable_pmu) 7909 return 0; 7910 7911 if (boot_cpu_has(X86_FEATURE_PDCM)) 7912 rdmsrq(MSR_IA32_PERF_CAPABILITIES, host_perf_cap); 7913 7914 if (!cpu_feature_enabled(X86_FEATURE_ARCH_LBR)) { 7915 x86_perf_get_lbr(&vmx_lbr_caps); 7916 7917 /* 7918 * KVM requires LBR callstack support, as the overhead due to 7919 * context switching LBRs without said support is too high. 7920 * See intel_pmu_create_guest_lbr_event() for more info. 7921 */ 7922 if (!vmx_lbr_caps.has_callstack) 7923 memset(&vmx_lbr_caps, 0, sizeof(vmx_lbr_caps)); 7924 else if (vmx_lbr_caps.nr) 7925 perf_cap |= host_perf_cap & PERF_CAP_LBR_FMT; 7926 } 7927 7928 if (vmx_pebs_supported()) { 7929 perf_cap |= host_perf_cap & PERF_CAP_PEBS_MASK; 7930 7931 /* 7932 * Disallow adaptive PEBS as it is functionally broken, can be 7933 * used by the guest to read *host* LBRs, and can be used to 7934 * bypass userspace event filters. To correctly and safely 7935 * support adaptive PEBS, KVM needs to: 7936 * 7937 * 1. Account for the ADAPTIVE flag when (re)programming fixed 7938 * counters. 7939 * 7940 * 2. Gain support from perf (or take direct control of counter 7941 * programming) to support events without adaptive PEBS 7942 * enabled for the hardware counter. 7943 * 7944 * 3. Ensure LBR MSRs cannot hold host data on VM-Entry with 7945 * adaptive PEBS enabled and MSR_PEBS_DATA_CFG.LBRS=1. 7946 * 7947 * 4. Document which PMU events are effectively exposed to the 7948 * guest via adaptive PEBS, and make adaptive PEBS mutually 7949 * exclusive with KVM_SET_PMU_EVENT_FILTER if necessary. 7950 */ 7951 perf_cap &= ~PERF_CAP_PEBS_BASELINE; 7952 } 7953 7954 return perf_cap; 7955 } 7956 7957 static __init void vmx_set_cpu_caps(void) 7958 { 7959 kvm_set_cpu_caps(); 7960 7961 /* CPUID 0x1 */ 7962 if (nested) 7963 kvm_cpu_cap_set(X86_FEATURE_VMX); 7964 7965 /* CPUID 0x7 */ 7966 if (kvm_mpx_supported()) 7967 kvm_cpu_cap_check_and_set(X86_FEATURE_MPX); 7968 if (!cpu_has_vmx_invpcid()) 7969 kvm_cpu_cap_clear(X86_FEATURE_INVPCID); 7970 if (vmx_pt_mode_is_host_guest()) 7971 kvm_cpu_cap_check_and_set(X86_FEATURE_INTEL_PT); 7972 if (vmx_pebs_supported()) { 7973 kvm_cpu_cap_check_and_set(X86_FEATURE_DS); 7974 kvm_cpu_cap_check_and_set(X86_FEATURE_DTES64); 7975 } 7976 7977 if (!enable_pmu) 7978 kvm_cpu_cap_clear(X86_FEATURE_PDCM); 7979 kvm_caps.supported_perf_cap = vmx_get_perf_capabilities(); 7980 7981 if (!enable_sgx) { 7982 kvm_cpu_cap_clear(X86_FEATURE_SGX); 7983 kvm_cpu_cap_clear(X86_FEATURE_SGX_LC); 7984 kvm_cpu_cap_clear(X86_FEATURE_SGX1); 7985 kvm_cpu_cap_clear(X86_FEATURE_SGX2); 7986 kvm_cpu_cap_clear(X86_FEATURE_SGX_EDECCSSA); 7987 } 7988 7989 if (vmx_umip_emulated()) 7990 kvm_cpu_cap_set(X86_FEATURE_UMIP); 7991 7992 /* CPUID 0xD.1 */ 7993 if (!cpu_has_vmx_xsaves()) 7994 kvm_cpu_cap_clear(X86_FEATURE_XSAVES); 7995 7996 /* CPUID 0x80000001 and 0x7 (RDPID) */ 7997 if (!cpu_has_vmx_rdtscp()) { 7998 kvm_cpu_cap_clear(X86_FEATURE_RDTSCP); 7999 kvm_cpu_cap_clear(X86_FEATURE_RDPID); 8000 } 8001 8002 if (cpu_has_vmx_waitpkg()) 8003 kvm_cpu_cap_check_and_set(X86_FEATURE_WAITPKG); 8004 8005 /* 8006 * Disable CET if unrestricted_guest is unsupported as KVM doesn't 8007 * enforce CET HW behaviors in emulator. On platforms with 8008 * VMX_BASIC[bit56] == 0, inject #CP at VMX entry with error code 8009 * fails, so disable CET in this case too. 8010 */ 8011 if (!cpu_has_load_cet_ctrl() || !enable_unrestricted_guest || 8012 !cpu_has_vmx_basic_no_hw_errcode_cc()) { 8013 kvm_cpu_cap_clear(X86_FEATURE_SHSTK); 8014 kvm_cpu_cap_clear(X86_FEATURE_IBT); 8015 } 8016 } 8017 8018 static bool vmx_is_io_intercepted(struct kvm_vcpu *vcpu, 8019 struct x86_instruction_info *info, 8020 unsigned long *exit_qualification) 8021 { 8022 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 8023 unsigned short port; 8024 int size; 8025 bool imm; 8026 8027 /* 8028 * If the 'use IO bitmaps' VM-execution control is 0, IO instruction 8029 * VM-exits depend on the 'unconditional IO exiting' VM-execution 8030 * control. 8031 * 8032 * Otherwise, IO instruction VM-exits are controlled by the IO bitmaps. 8033 */ 8034 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 8035 return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING); 8036 8037 if (info->intercept == x86_intercept_in || 8038 info->intercept == x86_intercept_ins) { 8039 port = info->src_val; 8040 size = info->dst_bytes; 8041 imm = info->src_type == OP_IMM; 8042 } else { 8043 port = info->dst_val; 8044 size = info->src_bytes; 8045 imm = info->dst_type == OP_IMM; 8046 } 8047 8048 8049 *exit_qualification = ((unsigned long)port << 16) | (size - 1); 8050 8051 if (info->intercept == x86_intercept_ins || 8052 info->intercept == x86_intercept_outs) 8053 *exit_qualification |= BIT(4); 8054 8055 if (info->rep_prefix) 8056 *exit_qualification |= BIT(5); 8057 8058 if (imm) 8059 *exit_qualification |= BIT(6); 8060 8061 return nested_vmx_check_io_bitmaps(vcpu, port, size); 8062 } 8063 8064 int vmx_check_intercept(struct kvm_vcpu *vcpu, 8065 struct x86_instruction_info *info, 8066 enum x86_intercept_stage stage, 8067 struct x86_exception *exception) 8068 { 8069 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 8070 unsigned long exit_qualification = 0; 8071 u32 vm_exit_reason; 8072 u64 exit_insn_len; 8073 8074 switch (info->intercept) { 8075 case x86_intercept_rdpid: 8076 /* 8077 * RDPID causes #UD if not enabled through secondary execution 8078 * controls (ENABLE_RDTSCP). Note, the implicit MSR access to 8079 * TSC_AUX is NOT subject to interception, i.e. checking only 8080 * the dedicated execution control is architecturally correct. 8081 */ 8082 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_RDTSCP)) { 8083 exception->vector = UD_VECTOR; 8084 exception->error_code_valid = false; 8085 return X86EMUL_PROPAGATE_FAULT; 8086 } 8087 return X86EMUL_CONTINUE; 8088 8089 case x86_intercept_in: 8090 case x86_intercept_ins: 8091 case x86_intercept_out: 8092 case x86_intercept_outs: 8093 if (!vmx_is_io_intercepted(vcpu, info, &exit_qualification)) 8094 return X86EMUL_CONTINUE; 8095 8096 vm_exit_reason = EXIT_REASON_IO_INSTRUCTION; 8097 break; 8098 8099 case x86_intercept_lgdt: 8100 case x86_intercept_lidt: 8101 case x86_intercept_lldt: 8102 case x86_intercept_ltr: 8103 case x86_intercept_sgdt: 8104 case x86_intercept_sidt: 8105 case x86_intercept_sldt: 8106 case x86_intercept_str: 8107 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC)) 8108 return X86EMUL_CONTINUE; 8109 8110 if (info->intercept == x86_intercept_lldt || 8111 info->intercept == x86_intercept_ltr || 8112 info->intercept == x86_intercept_sldt || 8113 info->intercept == x86_intercept_str) 8114 vm_exit_reason = EXIT_REASON_LDTR_TR; 8115 else 8116 vm_exit_reason = EXIT_REASON_GDTR_IDTR; 8117 /* 8118 * FIXME: Decode the ModR/M to generate the correct exit 8119 * qualification for memory operands. 8120 */ 8121 break; 8122 8123 case x86_intercept_hlt: 8124 if (!nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING)) 8125 return X86EMUL_CONTINUE; 8126 8127 vm_exit_reason = EXIT_REASON_HLT; 8128 break; 8129 8130 case x86_intercept_pause: 8131 /* 8132 * PAUSE is a single-byte NOP with a REPE prefix, i.e. collides 8133 * with vanilla NOPs in the emulator. Apply the interception 8134 * check only to actual PAUSE instructions. Don't check 8135 * PAUSE-loop-exiting, software can't expect a given PAUSE to 8136 * exit, i.e. KVM is within its rights to allow L2 to execute 8137 * the PAUSE. 8138 */ 8139 if ((info->rep_prefix != REPE_PREFIX) || 8140 !nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING)) 8141 return X86EMUL_CONTINUE; 8142 8143 vm_exit_reason = EXIT_REASON_PAUSE_INSTRUCTION; 8144 break; 8145 8146 /* TODO: check more intercepts... */ 8147 default: 8148 return X86EMUL_UNHANDLEABLE; 8149 } 8150 8151 exit_insn_len = abs_diff((s64)info->next_rip, (s64)info->rip); 8152 if (!exit_insn_len || exit_insn_len > X86_MAX_INSTRUCTION_LENGTH) 8153 return X86EMUL_UNHANDLEABLE; 8154 8155 __nested_vmx_vmexit(vcpu, vm_exit_reason, 0, exit_qualification, 8156 exit_insn_len); 8157 return X86EMUL_INTERCEPTED; 8158 } 8159 8160 #ifdef CONFIG_X86_64 8161 /* (a << shift) / divisor, return 1 if overflow otherwise 0 */ 8162 static inline int u64_shl_div_u64(u64 a, unsigned int shift, 8163 u64 divisor, u64 *result) 8164 { 8165 u64 low = a << shift, high = a >> (64 - shift); 8166 8167 /* To avoid the overflow on divq */ 8168 if (high >= divisor) 8169 return 1; 8170 8171 /* Low hold the result, high hold rem which is discarded */ 8172 asm("divq %2\n\t" : "=a" (low), "=d" (high) : 8173 "rm" (divisor), "0" (low), "1" (high)); 8174 *result = low; 8175 8176 return 0; 8177 } 8178 8179 int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, 8180 bool *expired) 8181 { 8182 struct vcpu_vmx *vmx; 8183 u64 tscl, guest_tscl, delta_tsc, lapic_timer_advance_cycles; 8184 struct kvm_timer *ktimer = &vcpu->arch.apic->lapic_timer; 8185 8186 vmx = to_vmx(vcpu); 8187 tscl = rdtsc(); 8188 guest_tscl = kvm_read_l1_tsc(vcpu, tscl); 8189 delta_tsc = max(guest_deadline_tsc, guest_tscl) - guest_tscl; 8190 lapic_timer_advance_cycles = nsec_to_cycles(vcpu, 8191 ktimer->timer_advance_ns); 8192 8193 if (delta_tsc > lapic_timer_advance_cycles) 8194 delta_tsc -= lapic_timer_advance_cycles; 8195 else 8196 delta_tsc = 0; 8197 8198 /* Convert to host delta tsc if tsc scaling is enabled */ 8199 if (vcpu->arch.l1_tsc_scaling_ratio != kvm_caps.default_tsc_scaling_ratio && 8200 delta_tsc && u64_shl_div_u64(delta_tsc, 8201 kvm_caps.tsc_scaling_ratio_frac_bits, 8202 vcpu->arch.l1_tsc_scaling_ratio, &delta_tsc)) 8203 return -ERANGE; 8204 8205 /* 8206 * If the delta tsc can't fit in the 32 bit after the multi shift, 8207 * we can't use the preemption timer. 8208 * It's possible that it fits on later vmentries, but checking 8209 * on every vmentry is costly so we just use an hrtimer. 8210 */ 8211 if (delta_tsc >> (cpu_preemption_timer_multi + 32)) 8212 return -ERANGE; 8213 8214 vmx->hv_deadline_tsc = tscl + delta_tsc; 8215 *expired = !delta_tsc; 8216 return 0; 8217 } 8218 8219 void vmx_cancel_hv_timer(struct kvm_vcpu *vcpu) 8220 { 8221 to_vmx(vcpu)->hv_deadline_tsc = -1; 8222 } 8223 #endif 8224 8225 void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) 8226 { 8227 struct vcpu_vmx *vmx = to_vmx(vcpu); 8228 8229 if (WARN_ON_ONCE(!enable_pml)) 8230 return; 8231 8232 if (is_guest_mode(vcpu)) { 8233 vmx->nested.update_vmcs01_cpu_dirty_logging = true; 8234 return; 8235 } 8236 8237 /* 8238 * Note, nr_memslots_dirty_logging can be changed concurrent with this 8239 * code, but in that case another update request will be made and so 8240 * the guest will never run with a stale PML value. 8241 */ 8242 if (atomic_read(&vcpu->kvm->nr_memslots_dirty_logging)) 8243 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_ENABLE_PML); 8244 else 8245 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_ENABLE_PML); 8246 } 8247 8248 void vmx_setup_mce(struct kvm_vcpu *vcpu) 8249 { 8250 if (vcpu->arch.mcg_cap & MCG_LMCE_P) 8251 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |= 8252 FEAT_CTL_LMCE_ENABLED; 8253 else 8254 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &= 8255 ~FEAT_CTL_LMCE_ENABLED; 8256 } 8257 8258 #ifdef CONFIG_KVM_SMM 8259 int vmx_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 8260 { 8261 /* we need a nested vmexit to enter SMM, postpone if run is pending */ 8262 if (to_vmx(vcpu)->nested.nested_run_pending) 8263 return -EBUSY; 8264 return !is_smm(vcpu); 8265 } 8266 8267 int vmx_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram) 8268 { 8269 struct vcpu_vmx *vmx = to_vmx(vcpu); 8270 8271 /* 8272 * TODO: Implement custom flows for forcing the vCPU out/in of L2 on 8273 * SMI and RSM. Using the common VM-Exit + VM-Enter routines is wrong 8274 * SMI and RSM only modify state that is saved and restored via SMRAM. 8275 * E.g. most MSRs are left untouched, but many are modified by VM-Exit 8276 * and VM-Enter, and thus L2's values may be corrupted on SMI+RSM. 8277 */ 8278 vmx->nested.smm.guest_mode = is_guest_mode(vcpu); 8279 if (vmx->nested.smm.guest_mode) 8280 nested_vmx_vmexit(vcpu, -1, 0, 0); 8281 8282 vmx->nested.smm.vmxon = vmx->nested.vmxon; 8283 vmx->nested.vmxon = false; 8284 vmx_clear_hlt(vcpu); 8285 return 0; 8286 } 8287 8288 int vmx_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram) 8289 { 8290 struct vcpu_vmx *vmx = to_vmx(vcpu); 8291 int ret; 8292 8293 if (vmx->nested.smm.vmxon) { 8294 vmx->nested.vmxon = true; 8295 vmx->nested.smm.vmxon = false; 8296 } 8297 8298 if (vmx->nested.smm.guest_mode) { 8299 ret = nested_vmx_enter_non_root_mode(vcpu, false); 8300 if (ret) 8301 return ret; 8302 8303 vmx->nested.nested_run_pending = 1; 8304 vmx->nested.smm.guest_mode = false; 8305 } 8306 return 0; 8307 } 8308 8309 void vmx_enable_smi_window(struct kvm_vcpu *vcpu) 8310 { 8311 /* RSM will cause a vmexit anyway. */ 8312 } 8313 #endif 8314 8315 bool vmx_apic_init_signal_blocked(struct kvm_vcpu *vcpu) 8316 { 8317 return to_vmx(vcpu)->nested.vmxon && !is_guest_mode(vcpu); 8318 } 8319 8320 void vmx_migrate_timers(struct kvm_vcpu *vcpu) 8321 { 8322 if (is_guest_mode(vcpu)) { 8323 struct hrtimer *timer = &to_vmx(vcpu)->nested.preemption_timer; 8324 8325 if (hrtimer_try_to_cancel(timer) == 1) 8326 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED); 8327 } 8328 } 8329 8330 void vmx_hardware_unsetup(void) 8331 { 8332 kvm_set_posted_intr_wakeup_handler(NULL); 8333 8334 if (nested) 8335 nested_vmx_hardware_unsetup(); 8336 8337 free_kvm_area(); 8338 } 8339 8340 void vmx_vm_destroy(struct kvm *kvm) 8341 { 8342 struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); 8343 8344 free_pages((unsigned long)kvm_vmx->pid_table, vmx_get_pid_table_order(kvm)); 8345 } 8346 8347 /* 8348 * Note, the SDM states that the linear address is masked *after* the modified 8349 * canonicality check, whereas KVM masks (untags) the address and then performs 8350 * a "normal" canonicality check. Functionally, the two methods are identical, 8351 * and when the masking occurs relative to the canonicality check isn't visible 8352 * to software, i.e. KVM's behavior doesn't violate the SDM. 8353 */ 8354 gva_t vmx_get_untagged_addr(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags) 8355 { 8356 int lam_bit; 8357 unsigned long cr3_bits; 8358 8359 if (flags & (X86EMUL_F_FETCH | X86EMUL_F_IMPLICIT | X86EMUL_F_INVLPG)) 8360 return gva; 8361 8362 if (!is_64_bit_mode(vcpu)) 8363 return gva; 8364 8365 /* 8366 * Bit 63 determines if the address should be treated as user address 8367 * or a supervisor address. 8368 */ 8369 if (!(gva & BIT_ULL(63))) { 8370 cr3_bits = kvm_get_active_cr3_lam_bits(vcpu); 8371 if (!(cr3_bits & (X86_CR3_LAM_U57 | X86_CR3_LAM_U48))) 8372 return gva; 8373 8374 /* LAM_U48 is ignored if LAM_U57 is set. */ 8375 lam_bit = cr3_bits & X86_CR3_LAM_U57 ? 56 : 47; 8376 } else { 8377 if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_LAM_SUP)) 8378 return gva; 8379 8380 lam_bit = kvm_is_cr4_bit_set(vcpu, X86_CR4_LA57) ? 56 : 47; 8381 } 8382 8383 /* 8384 * Untag the address by sign-extending the lam_bit, but NOT to bit 63. 8385 * Bit 63 is retained from the raw virtual address so that untagging 8386 * doesn't change a user access to a supervisor access, and vice versa. 8387 */ 8388 return (sign_extend64(gva, lam_bit) & ~BIT_ULL(63)) | (gva & BIT_ULL(63)); 8389 } 8390 8391 static unsigned int vmx_handle_intel_pt_intr(void) 8392 { 8393 struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); 8394 8395 /* '0' on failure so that the !PT case can use a RET0 static call. */ 8396 if (!vcpu || !kvm_handling_nmi_from_guest(vcpu)) 8397 return 0; 8398 8399 kvm_make_request(KVM_REQ_PMI, vcpu); 8400 __set_bit(MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT, 8401 (unsigned long *)&vcpu->arch.pmu.global_status); 8402 return 1; 8403 } 8404 8405 static __init void vmx_setup_user_return_msrs(void) 8406 { 8407 8408 /* 8409 * Though SYSCALL is only supported in 64-bit mode on Intel CPUs, kvm 8410 * will emulate SYSCALL in legacy mode if the vendor string in guest 8411 * CPUID.0:{EBX,ECX,EDX} is "AuthenticAMD" or "AMDisbetter!" To 8412 * support this emulation, MSR_STAR is included in the list for i386, 8413 * but is never loaded into hardware. MSR_CSTAR is also never loaded 8414 * into hardware and is here purely for emulation purposes. 8415 */ 8416 const u32 vmx_uret_msrs_list[] = { 8417 #ifdef CONFIG_X86_64 8418 MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR, 8419 #endif 8420 MSR_EFER, MSR_TSC_AUX, MSR_STAR, 8421 MSR_IA32_TSX_CTRL, 8422 }; 8423 int i; 8424 8425 BUILD_BUG_ON(ARRAY_SIZE(vmx_uret_msrs_list) != MAX_NR_USER_RETURN_MSRS); 8426 8427 for (i = 0; i < ARRAY_SIZE(vmx_uret_msrs_list); ++i) 8428 kvm_add_user_return_msr(vmx_uret_msrs_list[i]); 8429 } 8430 8431 static void __init vmx_setup_me_spte_mask(void) 8432 { 8433 u64 me_mask = 0; 8434 8435 /* 8436 * On pre-MKTME system, boot_cpu_data.x86_phys_bits equals to 8437 * kvm_host.maxphyaddr. On MKTME and/or TDX capable systems, 8438 * boot_cpu_data.x86_phys_bits holds the actual physical address 8439 * w/o the KeyID bits, and kvm_host.maxphyaddr equals to 8440 * MAXPHYADDR reported by CPUID. Those bits between are KeyID bits. 8441 */ 8442 if (boot_cpu_data.x86_phys_bits != kvm_host.maxphyaddr) 8443 me_mask = rsvd_bits(boot_cpu_data.x86_phys_bits, 8444 kvm_host.maxphyaddr - 1); 8445 8446 /* 8447 * Unlike SME, host kernel doesn't support setting up any 8448 * MKTME KeyID on Intel platforms. No memory encryption 8449 * bits should be included into the SPTE. 8450 */ 8451 kvm_mmu_set_me_spte_mask(0, me_mask); 8452 } 8453 8454 __init int vmx_hardware_setup(void) 8455 { 8456 unsigned long host_bndcfgs; 8457 struct desc_ptr dt; 8458 int r; 8459 8460 store_idt(&dt); 8461 host_idt_base = dt.address; 8462 8463 vmx_setup_user_return_msrs(); 8464 8465 8466 if (boot_cpu_has(X86_FEATURE_NX)) 8467 kvm_enable_efer_bits(EFER_NX); 8468 8469 if (boot_cpu_has(X86_FEATURE_MPX)) { 8470 rdmsrq(MSR_IA32_BNDCFGS, host_bndcfgs); 8471 WARN_ONCE(host_bndcfgs, "BNDCFGS in host will be lost"); 8472 } 8473 8474 if (!cpu_has_vmx_mpx()) 8475 kvm_caps.supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS | 8476 XFEATURE_MASK_BNDCSR); 8477 8478 if (!cpu_has_vmx_vpid() || !cpu_has_vmx_invvpid() || 8479 !(cpu_has_vmx_invvpid_single() || cpu_has_vmx_invvpid_global())) 8480 enable_vpid = 0; 8481 8482 if (!cpu_has_vmx_ept() || 8483 !cpu_has_vmx_ept_4levels() || 8484 !cpu_has_vmx_ept_mt_wb() || 8485 !cpu_has_vmx_invept_global()) 8486 enable_ept = 0; 8487 8488 /* NX support is required for shadow paging. */ 8489 if (!enable_ept && !boot_cpu_has(X86_FEATURE_NX)) { 8490 pr_err_ratelimited("NX (Execute Disable) not supported\n"); 8491 return -EOPNOTSUPP; 8492 } 8493 8494 /* 8495 * Shadow paging doesn't have a (further) performance penalty 8496 * from GUEST_MAXPHYADDR < HOST_MAXPHYADDR so enable it 8497 * by default 8498 */ 8499 if (!enable_ept) 8500 allow_smaller_maxphyaddr = true; 8501 8502 if (!cpu_has_vmx_ept_ad_bits() || !enable_ept) 8503 enable_ept_ad_bits = 0; 8504 8505 if (!cpu_has_vmx_unrestricted_guest() || !enable_ept) 8506 enable_unrestricted_guest = 0; 8507 8508 if (!cpu_has_vmx_flexpriority()) 8509 flexpriority_enabled = 0; 8510 8511 if (!cpu_has_virtual_nmis()) 8512 enable_vnmi = 0; 8513 8514 #ifdef CONFIG_X86_SGX_KVM 8515 if (!cpu_has_vmx_encls_vmexit()) 8516 enable_sgx = false; 8517 #endif 8518 8519 /* 8520 * set_apic_access_page_addr() is used to reload apic access 8521 * page upon invalidation. No need to do anything if not 8522 * using the APIC_ACCESS_ADDR VMCS field. 8523 */ 8524 if (!flexpriority_enabled) 8525 vt_x86_ops.set_apic_access_page_addr = NULL; 8526 8527 if (!cpu_has_vmx_tpr_shadow()) 8528 vt_x86_ops.update_cr8_intercept = NULL; 8529 8530 #if IS_ENABLED(CONFIG_HYPERV) 8531 if (ms_hyperv.nested_features & HV_X64_NESTED_GUEST_MAPPING_FLUSH 8532 && enable_ept) { 8533 vt_x86_ops.flush_remote_tlbs = hv_flush_remote_tlbs; 8534 vt_x86_ops.flush_remote_tlbs_range = hv_flush_remote_tlbs_range; 8535 } 8536 #endif 8537 8538 if (!cpu_has_vmx_ple()) { 8539 ple_gap = 0; 8540 ple_window = 0; 8541 ple_window_grow = 0; 8542 ple_window_max = 0; 8543 ple_window_shrink = 0; 8544 } 8545 8546 if (!cpu_has_vmx_apicv()) 8547 enable_apicv = 0; 8548 if (!enable_apicv) 8549 vt_x86_ops.sync_pir_to_irr = NULL; 8550 8551 if (!enable_apicv || !cpu_has_vmx_ipiv()) 8552 enable_ipiv = false; 8553 8554 if (cpu_has_vmx_tsc_scaling()) 8555 kvm_caps.has_tsc_control = true; 8556 8557 kvm_caps.max_tsc_scaling_ratio = KVM_VMX_TSC_MULTIPLIER_MAX; 8558 kvm_caps.tsc_scaling_ratio_frac_bits = 48; 8559 kvm_caps.has_bus_lock_exit = cpu_has_vmx_bus_lock_detection(); 8560 kvm_caps.has_notify_vmexit = cpu_has_notify_vmexit(); 8561 8562 set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */ 8563 8564 if (enable_ept) 8565 kvm_mmu_set_ept_masks(enable_ept_ad_bits, 8566 cpu_has_vmx_ept_execute_only()); 8567 else 8568 vt_x86_ops.get_mt_mask = NULL; 8569 8570 /* 8571 * Setup shadow_me_value/shadow_me_mask to include MKTME KeyID 8572 * bits to shadow_zero_check. 8573 */ 8574 vmx_setup_me_spte_mask(); 8575 8576 kvm_configure_mmu(enable_ept, 0, vmx_get_max_ept_level(), 8577 ept_caps_to_lpage_level(vmx_capability.ept)); 8578 8579 /* 8580 * Only enable PML when hardware supports PML feature, and both EPT 8581 * and EPT A/D bit features are enabled -- PML depends on them to work. 8582 */ 8583 if (!enable_ept || !enable_ept_ad_bits || !cpu_has_vmx_pml()) 8584 enable_pml = 0; 8585 8586 if (!cpu_has_vmx_preemption_timer()) 8587 enable_preemption_timer = false; 8588 8589 if (enable_preemption_timer) { 8590 u64 use_timer_freq = 5000ULL * 1000 * 1000; 8591 8592 cpu_preemption_timer_multi = 8593 vmx_misc_preemption_timer_rate(vmcs_config.misc); 8594 8595 if (tsc_khz) 8596 use_timer_freq = (u64)tsc_khz * 1000; 8597 use_timer_freq >>= cpu_preemption_timer_multi; 8598 8599 /* 8600 * KVM "disables" the preemption timer by setting it to its max 8601 * value. Don't use the timer if it might cause spurious exits 8602 * at a rate faster than 0.1 Hz (of uninterrupted guest time). 8603 */ 8604 if (use_timer_freq > 0xffffffffu / 10) 8605 enable_preemption_timer = false; 8606 } 8607 8608 if (!enable_preemption_timer) { 8609 vt_x86_ops.set_hv_timer = NULL; 8610 vt_x86_ops.cancel_hv_timer = NULL; 8611 } 8612 8613 kvm_caps.supported_mce_cap |= MCG_LMCE_P; 8614 kvm_caps.supported_mce_cap |= MCG_CMCI_P; 8615 8616 if (pt_mode != PT_MODE_SYSTEM && pt_mode != PT_MODE_HOST_GUEST) 8617 return -EINVAL; 8618 if (!enable_ept || !enable_pmu || !cpu_has_vmx_intel_pt()) 8619 pt_mode = PT_MODE_SYSTEM; 8620 if (pt_mode == PT_MODE_HOST_GUEST) 8621 vt_init_ops.handle_intel_pt_intr = vmx_handle_intel_pt_intr; 8622 else 8623 vt_init_ops.handle_intel_pt_intr = NULL; 8624 8625 setup_default_sgx_lepubkeyhash(); 8626 8627 vmx_set_cpu_caps(); 8628 8629 /* 8630 * Configure nested capabilities after core CPU capabilities so that 8631 * nested support can be conditional on base support, e.g. so that KVM 8632 * can hide/show features based on kvm_cpu_cap_has(). 8633 */ 8634 if (nested) { 8635 nested_vmx_setup_ctls_msrs(&vmcs_config, vmx_capability.ept); 8636 8637 r = nested_vmx_hardware_setup(kvm_vmx_exit_handlers); 8638 if (r) 8639 return r; 8640 } 8641 8642 r = alloc_kvm_area(); 8643 if (r && nested) 8644 nested_vmx_hardware_unsetup(); 8645 8646 kvm_set_posted_intr_wakeup_handler(pi_wakeup_handler); 8647 8648 /* 8649 * On Intel CPUs that lack self-snoop feature, letting the guest control 8650 * memory types may result in unexpected behavior. So always ignore guest 8651 * PAT on those CPUs and map VM as writeback, not allowing userspace to 8652 * disable the quirk. 8653 * 8654 * On certain Intel CPUs (e.g. SPR, ICX), though self-snoop feature is 8655 * supported, UC is slow enough to cause issues with some older guests (e.g. 8656 * an old version of bochs driver uses ioremap() instead of ioremap_wc() to 8657 * map the video RAM, causing wayland desktop to fail to get started 8658 * correctly). To avoid breaking those older guests that rely on KVM to force 8659 * memory type to WB, provide KVM_X86_QUIRK_IGNORE_GUEST_PAT to preserve the 8660 * safer (for performance) default behavior. 8661 * 8662 * On top of this, non-coherent DMA devices need the guest to flush CPU 8663 * caches properly. This also requires honoring guest PAT, and is forced 8664 * independent of the quirk in vmx_ignore_guest_pat(). 8665 */ 8666 if (!static_cpu_has(X86_FEATURE_SELFSNOOP)) 8667 kvm_caps.supported_quirks &= ~KVM_X86_QUIRK_IGNORE_GUEST_PAT; 8668 8669 kvm_caps.inapplicable_quirks &= ~KVM_X86_QUIRK_IGNORE_GUEST_PAT; 8670 8671 return r; 8672 } 8673 8674 static void vmx_cleanup_l1d_flush(void) 8675 { 8676 if (vmx_l1d_flush_pages) { 8677 free_pages((unsigned long)vmx_l1d_flush_pages, L1D_CACHE_ORDER); 8678 vmx_l1d_flush_pages = NULL; 8679 } 8680 /* Restore state so sysfs ignores VMX */ 8681 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; 8682 } 8683 8684 void vmx_exit(void) 8685 { 8686 allow_smaller_maxphyaddr = false; 8687 8688 vmx_cleanup_l1d_flush(); 8689 8690 kvm_x86_vendor_exit(); 8691 } 8692 8693 int __init vmx_init(void) 8694 { 8695 int r, cpu; 8696 8697 KVM_SANITY_CHECK_VM_STRUCT_SIZE(kvm_vmx); 8698 8699 if (!kvm_is_vmx_supported()) 8700 return -EOPNOTSUPP; 8701 8702 /* 8703 * Note, VMCS and eVMCS configuration only touch VMX knobs/variables, 8704 * i.e. there's nothing to unwind if a later step fails. 8705 */ 8706 hv_init_evmcs(); 8707 8708 /* 8709 * Parse the VMCS config and VMX capabilities before anything else, so 8710 * that the information is available to all setup flows. 8711 */ 8712 if (setup_vmcs_config(&vmcs_config, &vmx_capability) < 0) 8713 return -EIO; 8714 8715 r = kvm_x86_vendor_init(&vt_init_ops); 8716 if (r) 8717 return r; 8718 8719 /* 8720 * Must be called after common x86 init so enable_ept is properly set 8721 * up. Hand the parameter mitigation value in which was stored in 8722 * the pre module init parser. If no parameter was given, it will 8723 * contain 'auto' which will be turned into the default 'cond' 8724 * mitigation mode. 8725 */ 8726 r = vmx_setup_l1d_flush(vmentry_l1d_flush_param); 8727 if (r) 8728 goto err_l1d_flush; 8729 8730 for_each_possible_cpu(cpu) { 8731 INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu)); 8732 8733 pi_init_cpu(cpu); 8734 } 8735 8736 vmx_check_vmcs12_offsets(); 8737 8738 return 0; 8739 8740 err_l1d_flush: 8741 kvm_x86_vendor_exit(); 8742 return r; 8743 } 8744