1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * This module enables machines with Intel VT-x extensions to run virtual 6 * machines without emulation or binary translation. 7 * 8 * Copyright (C) 2006 Qumranet, Inc. 9 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 10 * 11 * Authors: 12 * Avi Kivity <avi@qumranet.com> 13 * Yaniv Kamay <yaniv@qumranet.com> 14 */ 15 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 16 17 #include <linux/highmem.h> 18 #include <linux/hrtimer.h> 19 #include <linux/kernel.h> 20 #include <linux/kvm_host.h> 21 #include <linux/module.h> 22 #include <linux/moduleparam.h> 23 #include <linux/mod_devicetable.h> 24 #include <linux/mm.h> 25 #include <linux/objtool.h> 26 #include <linux/sched.h> 27 #include <linux/sched/smt.h> 28 #include <linux/slab.h> 29 #include <linux/tboot.h> 30 #include <linux/trace_events.h> 31 #include <linux/entry-kvm.h> 32 33 #include <asm/apic.h> 34 #include <asm/asm.h> 35 #include <asm/cpu.h> 36 #include <asm/cpu_device_id.h> 37 #include <asm/debugreg.h> 38 #include <asm/desc.h> 39 #include <asm/fpu/api.h> 40 #include <asm/fpu/xstate.h> 41 #include <asm/fred.h> 42 #include <asm/idtentry.h> 43 #include <asm/io.h> 44 #include <asm/irq_remapping.h> 45 #include <asm/reboot.h> 46 #include <asm/perf_event.h> 47 #include <asm/mmu_context.h> 48 #include <asm/mshyperv.h> 49 #include <asm/mwait.h> 50 #include <asm/spec-ctrl.h> 51 #include <asm/vmx.h> 52 53 #include <trace/events/ipi.h> 54 55 #include "capabilities.h" 56 #include "cpuid.h" 57 #include "hyperv.h" 58 #include "kvm_onhyperv.h" 59 #include "irq.h" 60 #include "kvm_cache_regs.h" 61 #include "lapic.h" 62 #include "mmu.h" 63 #include "nested.h" 64 #include "pmu.h" 65 #include "sgx.h" 66 #include "trace.h" 67 #include "vmcs.h" 68 #include "vmcs12.h" 69 #include "vmx.h" 70 #include "x86.h" 71 #include "x86_ops.h" 72 #include "smm.h" 73 #include "vmx_onhyperv.h" 74 #include "posted_intr.h" 75 76 MODULE_AUTHOR("Qumranet"); 77 MODULE_DESCRIPTION("KVM support for VMX (Intel VT-x) extensions"); 78 MODULE_LICENSE("GPL"); 79 80 #ifdef MODULE 81 static const struct x86_cpu_id vmx_cpu_id[] = { 82 X86_MATCH_FEATURE(X86_FEATURE_VMX, NULL), 83 {} 84 }; 85 MODULE_DEVICE_TABLE(x86cpu, vmx_cpu_id); 86 #endif 87 88 bool __read_mostly enable_vpid = 1; 89 module_param_named(vpid, enable_vpid, bool, 0444); 90 91 static bool __read_mostly enable_vnmi = 1; 92 module_param_named(vnmi, enable_vnmi, bool, 0444); 93 94 bool __read_mostly flexpriority_enabled = 1; 95 module_param_named(flexpriority, flexpriority_enabled, bool, 0444); 96 97 bool __read_mostly enable_ept = 1; 98 module_param_named(ept, enable_ept, bool, 0444); 99 100 bool __read_mostly enable_unrestricted_guest = 1; 101 module_param_named(unrestricted_guest, 102 enable_unrestricted_guest, bool, 0444); 103 104 bool __read_mostly enable_ept_ad_bits = 1; 105 module_param_named(eptad, enable_ept_ad_bits, bool, 0444); 106 107 static bool __read_mostly emulate_invalid_guest_state = true; 108 module_param(emulate_invalid_guest_state, bool, 0444); 109 110 static bool __read_mostly fasteoi = 1; 111 module_param(fasteoi, bool, 0444); 112 113 module_param(enable_apicv, bool, 0444); 114 115 bool __read_mostly enable_ipiv = true; 116 module_param(enable_ipiv, bool, 0444); 117 118 /* 119 * If nested=1, nested virtualization is supported, i.e., guests may use 120 * VMX and be a hypervisor for its own guests. If nested=0, guests may not 121 * use VMX instructions. 122 */ 123 static bool __read_mostly nested = 1; 124 module_param(nested, bool, 0444); 125 126 bool __read_mostly enable_pml = 1; 127 module_param_named(pml, enable_pml, bool, 0444); 128 129 static bool __read_mostly error_on_inconsistent_vmcs_config = true; 130 module_param(error_on_inconsistent_vmcs_config, bool, 0444); 131 132 static bool __read_mostly dump_invalid_vmcs = 0; 133 module_param(dump_invalid_vmcs, bool, 0644); 134 135 #define MSR_BITMAP_MODE_X2APIC 1 136 #define MSR_BITMAP_MODE_X2APIC_APICV 2 137 138 #define KVM_VMX_TSC_MULTIPLIER_MAX 0xffffffffffffffffULL 139 140 /* Guest_tsc -> host_tsc conversion requires 64-bit division. */ 141 static int __read_mostly cpu_preemption_timer_multi; 142 static bool __read_mostly enable_preemption_timer = 1; 143 #ifdef CONFIG_X86_64 144 module_param_named(preemption_timer, enable_preemption_timer, bool, S_IRUGO); 145 #endif 146 147 extern bool __read_mostly allow_smaller_maxphyaddr; 148 module_param(allow_smaller_maxphyaddr, bool, S_IRUGO); 149 150 #define KVM_VM_CR0_ALWAYS_OFF (X86_CR0_NW | X86_CR0_CD) 151 #define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR0_NE 152 #define KVM_VM_CR0_ALWAYS_ON \ 153 (KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE) 154 155 #define KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR4_VMXE 156 #define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE) 157 #define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE) 158 159 #define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM)) 160 161 #define MSR_IA32_RTIT_STATUS_MASK (~(RTIT_STATUS_FILTEREN | \ 162 RTIT_STATUS_CONTEXTEN | RTIT_STATUS_TRIGGEREN | \ 163 RTIT_STATUS_ERROR | RTIT_STATUS_STOPPED | \ 164 RTIT_STATUS_BYTECNT)) 165 166 /* 167 * List of MSRs that can be directly passed to the guest. 168 * In addition to these x2apic, PT and LBR MSRs are handled specially. 169 */ 170 static u32 vmx_possible_passthrough_msrs[MAX_POSSIBLE_PASSTHROUGH_MSRS] = { 171 MSR_IA32_SPEC_CTRL, 172 MSR_IA32_PRED_CMD, 173 MSR_IA32_FLUSH_CMD, 174 MSR_IA32_TSC, 175 #ifdef CONFIG_X86_64 176 MSR_FS_BASE, 177 MSR_GS_BASE, 178 MSR_KERNEL_GS_BASE, 179 MSR_IA32_XFD, 180 MSR_IA32_XFD_ERR, 181 #endif 182 MSR_IA32_SYSENTER_CS, 183 MSR_IA32_SYSENTER_ESP, 184 MSR_IA32_SYSENTER_EIP, 185 MSR_CORE_C1_RES, 186 MSR_CORE_C3_RESIDENCY, 187 MSR_CORE_C6_RESIDENCY, 188 MSR_CORE_C7_RESIDENCY, 189 }; 190 191 /* 192 * These 2 parameters are used to config the controls for Pause-Loop Exiting: 193 * ple_gap: upper bound on the amount of time between two successive 194 * executions of PAUSE in a loop. Also indicate if ple enabled. 195 * According to test, this time is usually smaller than 128 cycles. 196 * ple_window: upper bound on the amount of time a guest is allowed to execute 197 * in a PAUSE loop. Tests indicate that most spinlocks are held for 198 * less than 2^12 cycles 199 * Time is measured based on a counter that runs at the same rate as the TSC, 200 * refer SDM volume 3b section 21.6.13 & 22.1.3. 201 */ 202 static unsigned int ple_gap = KVM_DEFAULT_PLE_GAP; 203 module_param(ple_gap, uint, 0444); 204 205 static unsigned int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW; 206 module_param(ple_window, uint, 0444); 207 208 /* Default doubles per-vcpu window every exit. */ 209 static unsigned int ple_window_grow = KVM_DEFAULT_PLE_WINDOW_GROW; 210 module_param(ple_window_grow, uint, 0444); 211 212 /* Default resets per-vcpu window every exit to ple_window. */ 213 static unsigned int ple_window_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK; 214 module_param(ple_window_shrink, uint, 0444); 215 216 /* Default is to compute the maximum so we can never overflow. */ 217 static unsigned int ple_window_max = KVM_VMX_DEFAULT_PLE_WINDOW_MAX; 218 module_param(ple_window_max, uint, 0444); 219 220 /* Default is SYSTEM mode, 1 for host-guest mode (which is BROKEN) */ 221 int __read_mostly pt_mode = PT_MODE_SYSTEM; 222 #ifdef CONFIG_BROKEN 223 module_param(pt_mode, int, S_IRUGO); 224 #endif 225 226 struct x86_pmu_lbr __ro_after_init vmx_lbr_caps; 227 228 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_should_flush); 229 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_flush_cond); 230 static DEFINE_MUTEX(vmx_l1d_flush_mutex); 231 232 /* Storage for pre module init parameter parsing */ 233 static enum vmx_l1d_flush_state __read_mostly vmentry_l1d_flush_param = VMENTER_L1D_FLUSH_AUTO; 234 235 static const struct { 236 const char *option; 237 bool for_parse; 238 } vmentry_l1d_param[] = { 239 [VMENTER_L1D_FLUSH_AUTO] = {"auto", true}, 240 [VMENTER_L1D_FLUSH_NEVER] = {"never", true}, 241 [VMENTER_L1D_FLUSH_COND] = {"cond", true}, 242 [VMENTER_L1D_FLUSH_ALWAYS] = {"always", true}, 243 [VMENTER_L1D_FLUSH_EPT_DISABLED] = {"EPT disabled", false}, 244 [VMENTER_L1D_FLUSH_NOT_REQUIRED] = {"not required", false}, 245 }; 246 247 #define L1D_CACHE_ORDER 4 248 static void *vmx_l1d_flush_pages; 249 250 static int vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf) 251 { 252 struct page *page; 253 unsigned int i; 254 255 if (!boot_cpu_has_bug(X86_BUG_L1TF)) { 256 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED; 257 return 0; 258 } 259 260 if (!enable_ept) { 261 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_EPT_DISABLED; 262 return 0; 263 } 264 265 if (kvm_host.arch_capabilities & ARCH_CAP_SKIP_VMENTRY_L1DFLUSH) { 266 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED; 267 return 0; 268 } 269 270 /* If set to auto use the default l1tf mitigation method */ 271 if (l1tf == VMENTER_L1D_FLUSH_AUTO) { 272 switch (l1tf_mitigation) { 273 case L1TF_MITIGATION_OFF: 274 l1tf = VMENTER_L1D_FLUSH_NEVER; 275 break; 276 case L1TF_MITIGATION_FLUSH_NOWARN: 277 case L1TF_MITIGATION_FLUSH: 278 case L1TF_MITIGATION_FLUSH_NOSMT: 279 l1tf = VMENTER_L1D_FLUSH_COND; 280 break; 281 case L1TF_MITIGATION_FULL: 282 case L1TF_MITIGATION_FULL_FORCE: 283 l1tf = VMENTER_L1D_FLUSH_ALWAYS; 284 break; 285 } 286 } else if (l1tf_mitigation == L1TF_MITIGATION_FULL_FORCE) { 287 l1tf = VMENTER_L1D_FLUSH_ALWAYS; 288 } 289 290 if (l1tf != VMENTER_L1D_FLUSH_NEVER && !vmx_l1d_flush_pages && 291 !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) { 292 /* 293 * This allocation for vmx_l1d_flush_pages is not tied to a VM 294 * lifetime and so should not be charged to a memcg. 295 */ 296 page = alloc_pages(GFP_KERNEL, L1D_CACHE_ORDER); 297 if (!page) 298 return -ENOMEM; 299 vmx_l1d_flush_pages = page_address(page); 300 301 /* 302 * Initialize each page with a different pattern in 303 * order to protect against KSM in the nested 304 * virtualization case. 305 */ 306 for (i = 0; i < 1u << L1D_CACHE_ORDER; ++i) { 307 memset(vmx_l1d_flush_pages + i * PAGE_SIZE, i + 1, 308 PAGE_SIZE); 309 } 310 } 311 312 l1tf_vmx_mitigation = l1tf; 313 314 if (l1tf != VMENTER_L1D_FLUSH_NEVER) 315 static_branch_enable(&vmx_l1d_should_flush); 316 else 317 static_branch_disable(&vmx_l1d_should_flush); 318 319 if (l1tf == VMENTER_L1D_FLUSH_COND) 320 static_branch_enable(&vmx_l1d_flush_cond); 321 else 322 static_branch_disable(&vmx_l1d_flush_cond); 323 return 0; 324 } 325 326 static int vmentry_l1d_flush_parse(const char *s) 327 { 328 unsigned int i; 329 330 if (s) { 331 for (i = 0; i < ARRAY_SIZE(vmentry_l1d_param); i++) { 332 if (vmentry_l1d_param[i].for_parse && 333 sysfs_streq(s, vmentry_l1d_param[i].option)) 334 return i; 335 } 336 } 337 return -EINVAL; 338 } 339 340 static int vmentry_l1d_flush_set(const char *s, const struct kernel_param *kp) 341 { 342 int l1tf, ret; 343 344 l1tf = vmentry_l1d_flush_parse(s); 345 if (l1tf < 0) 346 return l1tf; 347 348 if (!boot_cpu_has(X86_BUG_L1TF)) 349 return 0; 350 351 /* 352 * Has vmx_init() run already? If not then this is the pre init 353 * parameter parsing. In that case just store the value and let 354 * vmx_init() do the proper setup after enable_ept has been 355 * established. 356 */ 357 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) { 358 vmentry_l1d_flush_param = l1tf; 359 return 0; 360 } 361 362 mutex_lock(&vmx_l1d_flush_mutex); 363 ret = vmx_setup_l1d_flush(l1tf); 364 mutex_unlock(&vmx_l1d_flush_mutex); 365 return ret; 366 } 367 368 static int vmentry_l1d_flush_get(char *s, const struct kernel_param *kp) 369 { 370 if (WARN_ON_ONCE(l1tf_vmx_mitigation >= ARRAY_SIZE(vmentry_l1d_param))) 371 return sysfs_emit(s, "???\n"); 372 373 return sysfs_emit(s, "%s\n", vmentry_l1d_param[l1tf_vmx_mitigation].option); 374 } 375 376 static __always_inline void vmx_disable_fb_clear(struct vcpu_vmx *vmx) 377 { 378 u64 msr; 379 380 if (!vmx->disable_fb_clear) 381 return; 382 383 msr = __rdmsr(MSR_IA32_MCU_OPT_CTRL); 384 msr |= FB_CLEAR_DIS; 385 native_wrmsrl(MSR_IA32_MCU_OPT_CTRL, msr); 386 /* Cache the MSR value to avoid reading it later */ 387 vmx->msr_ia32_mcu_opt_ctrl = msr; 388 } 389 390 static __always_inline void vmx_enable_fb_clear(struct vcpu_vmx *vmx) 391 { 392 if (!vmx->disable_fb_clear) 393 return; 394 395 vmx->msr_ia32_mcu_opt_ctrl &= ~FB_CLEAR_DIS; 396 native_wrmsrl(MSR_IA32_MCU_OPT_CTRL, vmx->msr_ia32_mcu_opt_ctrl); 397 } 398 399 static void vmx_update_fb_clear_dis(struct kvm_vcpu *vcpu, struct vcpu_vmx *vmx) 400 { 401 /* 402 * Disable VERW's behavior of clearing CPU buffers for the guest if the 403 * CPU isn't affected by MDS/TAA, and the host hasn't forcefully enabled 404 * the mitigation. Disabling the clearing behavior provides a 405 * performance boost for guests that aren't aware that manually clearing 406 * CPU buffers is unnecessary, at the cost of MSR accesses on VM-Entry 407 * and VM-Exit. 408 */ 409 vmx->disable_fb_clear = !cpu_feature_enabled(X86_FEATURE_CLEAR_CPU_BUF) && 410 (kvm_host.arch_capabilities & ARCH_CAP_FB_CLEAR_CTRL) && 411 !boot_cpu_has_bug(X86_BUG_MDS) && 412 !boot_cpu_has_bug(X86_BUG_TAA); 413 414 /* 415 * If guest will not execute VERW, there is no need to set FB_CLEAR_DIS 416 * at VMEntry. Skip the MSR read/write when a guest has no use case to 417 * execute VERW. 418 */ 419 if ((vcpu->arch.arch_capabilities & ARCH_CAP_FB_CLEAR) || 420 ((vcpu->arch.arch_capabilities & ARCH_CAP_MDS_NO) && 421 (vcpu->arch.arch_capabilities & ARCH_CAP_TAA_NO) && 422 (vcpu->arch.arch_capabilities & ARCH_CAP_PSDP_NO) && 423 (vcpu->arch.arch_capabilities & ARCH_CAP_FBSDP_NO) && 424 (vcpu->arch.arch_capabilities & ARCH_CAP_SBDR_SSDP_NO))) 425 vmx->disable_fb_clear = false; 426 } 427 428 static const struct kernel_param_ops vmentry_l1d_flush_ops = { 429 .set = vmentry_l1d_flush_set, 430 .get = vmentry_l1d_flush_get, 431 }; 432 module_param_cb(vmentry_l1d_flush, &vmentry_l1d_flush_ops, NULL, 0644); 433 434 static u32 vmx_segment_access_rights(struct kvm_segment *var); 435 436 void vmx_vmexit(void); 437 438 #define vmx_insn_failed(fmt...) \ 439 do { \ 440 WARN_ONCE(1, fmt); \ 441 pr_warn_ratelimited(fmt); \ 442 } while (0) 443 444 noinline void vmread_error(unsigned long field) 445 { 446 vmx_insn_failed("vmread failed: field=%lx\n", field); 447 } 448 449 #ifndef CONFIG_CC_HAS_ASM_GOTO_OUTPUT 450 noinstr void vmread_error_trampoline2(unsigned long field, bool fault) 451 { 452 if (fault) { 453 kvm_spurious_fault(); 454 } else { 455 instrumentation_begin(); 456 vmread_error(field); 457 instrumentation_end(); 458 } 459 } 460 #endif 461 462 noinline void vmwrite_error(unsigned long field, unsigned long value) 463 { 464 vmx_insn_failed("vmwrite failed: field=%lx val=%lx err=%u\n", 465 field, value, vmcs_read32(VM_INSTRUCTION_ERROR)); 466 } 467 468 noinline void vmclear_error(struct vmcs *vmcs, u64 phys_addr) 469 { 470 vmx_insn_failed("vmclear failed: %p/%llx err=%u\n", 471 vmcs, phys_addr, vmcs_read32(VM_INSTRUCTION_ERROR)); 472 } 473 474 noinline void vmptrld_error(struct vmcs *vmcs, u64 phys_addr) 475 { 476 vmx_insn_failed("vmptrld failed: %p/%llx err=%u\n", 477 vmcs, phys_addr, vmcs_read32(VM_INSTRUCTION_ERROR)); 478 } 479 480 noinline void invvpid_error(unsigned long ext, u16 vpid, gva_t gva) 481 { 482 vmx_insn_failed("invvpid failed: ext=0x%lx vpid=%u gva=0x%lx\n", 483 ext, vpid, gva); 484 } 485 486 noinline void invept_error(unsigned long ext, u64 eptp) 487 { 488 vmx_insn_failed("invept failed: ext=0x%lx eptp=%llx\n", ext, eptp); 489 } 490 491 static DEFINE_PER_CPU(struct vmcs *, vmxarea); 492 DEFINE_PER_CPU(struct vmcs *, current_vmcs); 493 /* 494 * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed 495 * when a CPU is brought down, and we need to VMCLEAR all VMCSs loaded on it. 496 */ 497 static DEFINE_PER_CPU(struct list_head, loaded_vmcss_on_cpu); 498 499 static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS); 500 static DEFINE_SPINLOCK(vmx_vpid_lock); 501 502 struct vmcs_config vmcs_config __ro_after_init; 503 struct vmx_capability vmx_capability __ro_after_init; 504 505 #define VMX_SEGMENT_FIELD(seg) \ 506 [VCPU_SREG_##seg] = { \ 507 .selector = GUEST_##seg##_SELECTOR, \ 508 .base = GUEST_##seg##_BASE, \ 509 .limit = GUEST_##seg##_LIMIT, \ 510 .ar_bytes = GUEST_##seg##_AR_BYTES, \ 511 } 512 513 static const struct kvm_vmx_segment_field { 514 unsigned selector; 515 unsigned base; 516 unsigned limit; 517 unsigned ar_bytes; 518 } kvm_vmx_segment_fields[] = { 519 VMX_SEGMENT_FIELD(CS), 520 VMX_SEGMENT_FIELD(DS), 521 VMX_SEGMENT_FIELD(ES), 522 VMX_SEGMENT_FIELD(FS), 523 VMX_SEGMENT_FIELD(GS), 524 VMX_SEGMENT_FIELD(SS), 525 VMX_SEGMENT_FIELD(TR), 526 VMX_SEGMENT_FIELD(LDTR), 527 }; 528 529 530 static unsigned long host_idt_base; 531 532 #if IS_ENABLED(CONFIG_HYPERV) 533 static bool __read_mostly enlightened_vmcs = true; 534 module_param(enlightened_vmcs, bool, 0444); 535 536 static int hv_enable_l2_tlb_flush(struct kvm_vcpu *vcpu) 537 { 538 struct hv_enlightened_vmcs *evmcs; 539 hpa_t partition_assist_page = hv_get_partition_assist_page(vcpu); 540 541 if (partition_assist_page == INVALID_PAGE) 542 return -ENOMEM; 543 544 evmcs = (struct hv_enlightened_vmcs *)to_vmx(vcpu)->loaded_vmcs->vmcs; 545 546 evmcs->partition_assist_page = partition_assist_page; 547 evmcs->hv_vm_id = (unsigned long)vcpu->kvm; 548 evmcs->hv_enlightenments_control.nested_flush_hypercall = 1; 549 550 return 0; 551 } 552 553 static __init void hv_init_evmcs(void) 554 { 555 int cpu; 556 557 if (!enlightened_vmcs) 558 return; 559 560 /* 561 * Enlightened VMCS usage should be recommended and the host needs 562 * to support eVMCS v1 or above. 563 */ 564 if (ms_hyperv.hints & HV_X64_ENLIGHTENED_VMCS_RECOMMENDED && 565 (ms_hyperv.nested_features & HV_X64_ENLIGHTENED_VMCS_VERSION) >= 566 KVM_EVMCS_VERSION) { 567 568 /* Check that we have assist pages on all online CPUs */ 569 for_each_online_cpu(cpu) { 570 if (!hv_get_vp_assist_page(cpu)) { 571 enlightened_vmcs = false; 572 break; 573 } 574 } 575 576 if (enlightened_vmcs) { 577 pr_info("Using Hyper-V Enlightened VMCS\n"); 578 static_branch_enable(&__kvm_is_using_evmcs); 579 } 580 581 if (ms_hyperv.nested_features & HV_X64_NESTED_DIRECT_FLUSH) 582 vt_x86_ops.enable_l2_tlb_flush 583 = hv_enable_l2_tlb_flush; 584 } else { 585 enlightened_vmcs = false; 586 } 587 } 588 589 static void hv_reset_evmcs(void) 590 { 591 struct hv_vp_assist_page *vp_ap; 592 593 if (!kvm_is_using_evmcs()) 594 return; 595 596 /* 597 * KVM should enable eVMCS if and only if all CPUs have a VP assist 598 * page, and should reject CPU onlining if eVMCS is enabled the CPU 599 * doesn't have a VP assist page allocated. 600 */ 601 vp_ap = hv_get_vp_assist_page(smp_processor_id()); 602 if (WARN_ON_ONCE(!vp_ap)) 603 return; 604 605 /* 606 * Reset everything to support using non-enlightened VMCS access later 607 * (e.g. when we reload the module with enlightened_vmcs=0) 608 */ 609 vp_ap->nested_control.features.directhypercall = 0; 610 vp_ap->current_nested_vmcs = 0; 611 vp_ap->enlighten_vmentry = 0; 612 } 613 614 #else /* IS_ENABLED(CONFIG_HYPERV) */ 615 static void hv_init_evmcs(void) {} 616 static void hv_reset_evmcs(void) {} 617 #endif /* IS_ENABLED(CONFIG_HYPERV) */ 618 619 /* 620 * Comment's format: document - errata name - stepping - processor name. 621 * Refer from 622 * https://www.virtualbox.org/svn/vbox/trunk/src/VBox/VMM/VMMR0/HMR0.cpp 623 */ 624 static u32 vmx_preemption_cpu_tfms[] = { 625 /* 323344.pdf - BA86 - D0 - Xeon 7500 Series */ 626 0x000206E6, 627 /* 323056.pdf - AAX65 - C2 - Xeon L3406 */ 628 /* 322814.pdf - AAT59 - C2 - i7-600, i5-500, i5-400 and i3-300 Mobile */ 629 /* 322911.pdf - AAU65 - C2 - i5-600, i3-500 Desktop and Pentium G6950 */ 630 0x00020652, 631 /* 322911.pdf - AAU65 - K0 - i5-600, i3-500 Desktop and Pentium G6950 */ 632 0x00020655, 633 /* 322373.pdf - AAO95 - B1 - Xeon 3400 Series */ 634 /* 322166.pdf - AAN92 - B1 - i7-800 and i5-700 Desktop */ 635 /* 636 * 320767.pdf - AAP86 - B1 - 637 * i7-900 Mobile Extreme, i7-800 and i7-700 Mobile 638 */ 639 0x000106E5, 640 /* 321333.pdf - AAM126 - C0 - Xeon 3500 */ 641 0x000106A0, 642 /* 321333.pdf - AAM126 - C1 - Xeon 3500 */ 643 0x000106A1, 644 /* 320836.pdf - AAJ124 - C0 - i7-900 Desktop Extreme and i7-900 Desktop */ 645 0x000106A4, 646 /* 321333.pdf - AAM126 - D0 - Xeon 3500 */ 647 /* 321324.pdf - AAK139 - D0 - Xeon 5500 */ 648 /* 320836.pdf - AAJ124 - D0 - i7-900 Extreme and i7-900 Desktop */ 649 0x000106A5, 650 /* Xeon E3-1220 V2 */ 651 0x000306A8, 652 }; 653 654 static inline bool cpu_has_broken_vmx_preemption_timer(void) 655 { 656 u32 eax = cpuid_eax(0x00000001), i; 657 658 /* Clear the reserved bits */ 659 eax &= ~(0x3U << 14 | 0xfU << 28); 660 for (i = 0; i < ARRAY_SIZE(vmx_preemption_cpu_tfms); i++) 661 if (eax == vmx_preemption_cpu_tfms[i]) 662 return true; 663 664 return false; 665 } 666 667 static inline bool cpu_need_virtualize_apic_accesses(struct kvm_vcpu *vcpu) 668 { 669 return flexpriority_enabled && lapic_in_kernel(vcpu); 670 } 671 672 static int vmx_get_passthrough_msr_slot(u32 msr) 673 { 674 int i; 675 676 switch (msr) { 677 case 0x800 ... 0x8ff: 678 /* x2APIC MSRs. These are handled in vmx_update_msr_bitmap_x2apic() */ 679 return -ENOENT; 680 case MSR_IA32_RTIT_STATUS: 681 case MSR_IA32_RTIT_OUTPUT_BASE: 682 case MSR_IA32_RTIT_OUTPUT_MASK: 683 case MSR_IA32_RTIT_CR3_MATCH: 684 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: 685 /* PT MSRs. These are handled in pt_update_intercept_for_msr() */ 686 case MSR_LBR_SELECT: 687 case MSR_LBR_TOS: 688 case MSR_LBR_INFO_0 ... MSR_LBR_INFO_0 + 31: 689 case MSR_LBR_NHM_FROM ... MSR_LBR_NHM_FROM + 31: 690 case MSR_LBR_NHM_TO ... MSR_LBR_NHM_TO + 31: 691 case MSR_LBR_CORE_FROM ... MSR_LBR_CORE_FROM + 8: 692 case MSR_LBR_CORE_TO ... MSR_LBR_CORE_TO + 8: 693 /* LBR MSRs. These are handled in vmx_update_intercept_for_lbr_msrs() */ 694 return -ENOENT; 695 } 696 697 for (i = 0; i < ARRAY_SIZE(vmx_possible_passthrough_msrs); i++) { 698 if (vmx_possible_passthrough_msrs[i] == msr) 699 return i; 700 } 701 702 WARN(1, "Invalid MSR %x, please adapt vmx_possible_passthrough_msrs[]", msr); 703 return -ENOENT; 704 } 705 706 struct vmx_uret_msr *vmx_find_uret_msr(struct vcpu_vmx *vmx, u32 msr) 707 { 708 int i; 709 710 i = kvm_find_user_return_msr(msr); 711 if (i >= 0) 712 return &vmx->guest_uret_msrs[i]; 713 return NULL; 714 } 715 716 static int vmx_set_guest_uret_msr(struct vcpu_vmx *vmx, 717 struct vmx_uret_msr *msr, u64 data) 718 { 719 unsigned int slot = msr - vmx->guest_uret_msrs; 720 int ret = 0; 721 722 if (msr->load_into_hardware) { 723 preempt_disable(); 724 ret = kvm_set_user_return_msr(slot, data, msr->mask); 725 preempt_enable(); 726 } 727 if (!ret) 728 msr->data = data; 729 return ret; 730 } 731 732 /* 733 * Disable VMX and clear CR4.VMXE (even if VMXOFF faults) 734 * 735 * Note, VMXOFF causes a #UD if the CPU is !post-VMXON, but it's impossible to 736 * atomically track post-VMXON state, e.g. this may be called in NMI context. 737 * Eat all faults as all other faults on VMXOFF faults are mode related, i.e. 738 * faults are guaranteed to be due to the !post-VMXON check unless the CPU is 739 * magically in RM, VM86, compat mode, or at CPL>0. 740 */ 741 static int kvm_cpu_vmxoff(void) 742 { 743 asm goto("1: vmxoff\n\t" 744 _ASM_EXTABLE(1b, %l[fault]) 745 ::: "cc", "memory" : fault); 746 747 cr4_clear_bits(X86_CR4_VMXE); 748 return 0; 749 750 fault: 751 cr4_clear_bits(X86_CR4_VMXE); 752 return -EIO; 753 } 754 755 void vmx_emergency_disable_virtualization_cpu(void) 756 { 757 int cpu = raw_smp_processor_id(); 758 struct loaded_vmcs *v; 759 760 kvm_rebooting = true; 761 762 /* 763 * Note, CR4.VMXE can be _cleared_ in NMI context, but it can only be 764 * set in task context. If this races with VMX is disabled by an NMI, 765 * VMCLEAR and VMXOFF may #UD, but KVM will eat those faults due to 766 * kvm_rebooting set. 767 */ 768 if (!(__read_cr4() & X86_CR4_VMXE)) 769 return; 770 771 list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu), 772 loaded_vmcss_on_cpu_link) 773 vmcs_clear(v->vmcs); 774 775 kvm_cpu_vmxoff(); 776 } 777 778 static void __loaded_vmcs_clear(void *arg) 779 { 780 struct loaded_vmcs *loaded_vmcs = arg; 781 int cpu = raw_smp_processor_id(); 782 783 if (loaded_vmcs->cpu != cpu) 784 return; /* vcpu migration can race with cpu offline */ 785 if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs) 786 per_cpu(current_vmcs, cpu) = NULL; 787 788 vmcs_clear(loaded_vmcs->vmcs); 789 if (loaded_vmcs->shadow_vmcs && loaded_vmcs->launched) 790 vmcs_clear(loaded_vmcs->shadow_vmcs); 791 792 list_del(&loaded_vmcs->loaded_vmcss_on_cpu_link); 793 794 /* 795 * Ensure all writes to loaded_vmcs, including deleting it from its 796 * current percpu list, complete before setting loaded_vmcs->cpu to 797 * -1, otherwise a different cpu can see loaded_vmcs->cpu == -1 first 798 * and add loaded_vmcs to its percpu list before it's deleted from this 799 * cpu's list. Pairs with the smp_rmb() in vmx_vcpu_load_vmcs(). 800 */ 801 smp_wmb(); 802 803 loaded_vmcs->cpu = -1; 804 loaded_vmcs->launched = 0; 805 } 806 807 void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs) 808 { 809 int cpu = loaded_vmcs->cpu; 810 811 if (cpu != -1) 812 smp_call_function_single(cpu, 813 __loaded_vmcs_clear, loaded_vmcs, 1); 814 } 815 816 static bool vmx_segment_cache_test_set(struct vcpu_vmx *vmx, unsigned seg, 817 unsigned field) 818 { 819 bool ret; 820 u32 mask = 1 << (seg * SEG_FIELD_NR + field); 821 822 if (!kvm_register_is_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS)) { 823 kvm_register_mark_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS); 824 vmx->segment_cache.bitmask = 0; 825 } 826 ret = vmx->segment_cache.bitmask & mask; 827 vmx->segment_cache.bitmask |= mask; 828 return ret; 829 } 830 831 static u16 vmx_read_guest_seg_selector(struct vcpu_vmx *vmx, unsigned seg) 832 { 833 u16 *p = &vmx->segment_cache.seg[seg].selector; 834 835 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_SEL)) 836 *p = vmcs_read16(kvm_vmx_segment_fields[seg].selector); 837 return *p; 838 } 839 840 static ulong vmx_read_guest_seg_base(struct vcpu_vmx *vmx, unsigned seg) 841 { 842 ulong *p = &vmx->segment_cache.seg[seg].base; 843 844 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_BASE)) 845 *p = vmcs_readl(kvm_vmx_segment_fields[seg].base); 846 return *p; 847 } 848 849 static u32 vmx_read_guest_seg_limit(struct vcpu_vmx *vmx, unsigned seg) 850 { 851 u32 *p = &vmx->segment_cache.seg[seg].limit; 852 853 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_LIMIT)) 854 *p = vmcs_read32(kvm_vmx_segment_fields[seg].limit); 855 return *p; 856 } 857 858 static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg) 859 { 860 u32 *p = &vmx->segment_cache.seg[seg].ar; 861 862 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_AR)) 863 *p = vmcs_read32(kvm_vmx_segment_fields[seg].ar_bytes); 864 return *p; 865 } 866 867 void vmx_update_exception_bitmap(struct kvm_vcpu *vcpu) 868 { 869 u32 eb; 870 871 eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) | 872 (1u << DB_VECTOR) | (1u << AC_VECTOR); 873 /* 874 * #VE isn't used for VMX. To test against unexpected changes 875 * related to #VE for VMX, intercept unexpected #VE and warn on it. 876 */ 877 if (IS_ENABLED(CONFIG_KVM_INTEL_PROVE_VE)) 878 eb |= 1u << VE_VECTOR; 879 /* 880 * Guest access to VMware backdoor ports could legitimately 881 * trigger #GP because of TSS I/O permission bitmap. 882 * We intercept those #GP and allow access to them anyway 883 * as VMware does. 884 */ 885 if (enable_vmware_backdoor) 886 eb |= (1u << GP_VECTOR); 887 if ((vcpu->guest_debug & 888 (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) == 889 (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) 890 eb |= 1u << BP_VECTOR; 891 if (to_vmx(vcpu)->rmode.vm86_active) 892 eb = ~0; 893 if (!vmx_need_pf_intercept(vcpu)) 894 eb &= ~(1u << PF_VECTOR); 895 896 /* When we are running a nested L2 guest and L1 specified for it a 897 * certain exception bitmap, we must trap the same exceptions and pass 898 * them to L1. When running L2, we will only handle the exceptions 899 * specified above if L1 did not want them. 900 */ 901 if (is_guest_mode(vcpu)) 902 eb |= get_vmcs12(vcpu)->exception_bitmap; 903 else { 904 int mask = 0, match = 0; 905 906 if (enable_ept && (eb & (1u << PF_VECTOR))) { 907 /* 908 * If EPT is enabled, #PF is currently only intercepted 909 * if MAXPHYADDR is smaller on the guest than on the 910 * host. In that case we only care about present, 911 * non-reserved faults. For vmcs02, however, PFEC_MASK 912 * and PFEC_MATCH are set in prepare_vmcs02_rare. 913 */ 914 mask = PFERR_PRESENT_MASK | PFERR_RSVD_MASK; 915 match = PFERR_PRESENT_MASK; 916 } 917 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, mask); 918 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, match); 919 } 920 921 /* 922 * Disabling xfd interception indicates that dynamic xfeatures 923 * might be used in the guest. Always trap #NM in this case 924 * to save guest xfd_err timely. 925 */ 926 if (vcpu->arch.xfd_no_write_intercept) 927 eb |= (1u << NM_VECTOR); 928 929 vmcs_write32(EXCEPTION_BITMAP, eb); 930 } 931 932 /* 933 * Check if MSR is intercepted for currently loaded MSR bitmap. 934 */ 935 static bool msr_write_intercepted(struct vcpu_vmx *vmx, u32 msr) 936 { 937 if (!(exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS)) 938 return true; 939 940 return vmx_test_msr_bitmap_write(vmx->loaded_vmcs->msr_bitmap, msr); 941 } 942 943 unsigned int __vmx_vcpu_run_flags(struct vcpu_vmx *vmx) 944 { 945 unsigned int flags = 0; 946 947 if (vmx->loaded_vmcs->launched) 948 flags |= VMX_RUN_VMRESUME; 949 950 /* 951 * If writes to the SPEC_CTRL MSR aren't intercepted, the guest is free 952 * to change it directly without causing a vmexit. In that case read 953 * it after vmexit and store it in vmx->spec_ctrl. 954 */ 955 if (!msr_write_intercepted(vmx, MSR_IA32_SPEC_CTRL)) 956 flags |= VMX_RUN_SAVE_SPEC_CTRL; 957 958 return flags; 959 } 960 961 static __always_inline void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx, 962 unsigned long entry, unsigned long exit) 963 { 964 vm_entry_controls_clearbit(vmx, entry); 965 vm_exit_controls_clearbit(vmx, exit); 966 } 967 968 int vmx_find_loadstore_msr_slot(struct vmx_msrs *m, u32 msr) 969 { 970 unsigned int i; 971 972 for (i = 0; i < m->nr; ++i) { 973 if (m->val[i].index == msr) 974 return i; 975 } 976 return -ENOENT; 977 } 978 979 static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr) 980 { 981 int i; 982 struct msr_autoload *m = &vmx->msr_autoload; 983 984 switch (msr) { 985 case MSR_EFER: 986 if (cpu_has_load_ia32_efer()) { 987 clear_atomic_switch_msr_special(vmx, 988 VM_ENTRY_LOAD_IA32_EFER, 989 VM_EXIT_LOAD_IA32_EFER); 990 return; 991 } 992 break; 993 case MSR_CORE_PERF_GLOBAL_CTRL: 994 if (cpu_has_load_perf_global_ctrl()) { 995 clear_atomic_switch_msr_special(vmx, 996 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL, 997 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL); 998 return; 999 } 1000 break; 1001 } 1002 i = vmx_find_loadstore_msr_slot(&m->guest, msr); 1003 if (i < 0) 1004 goto skip_guest; 1005 --m->guest.nr; 1006 m->guest.val[i] = m->guest.val[m->guest.nr]; 1007 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr); 1008 1009 skip_guest: 1010 i = vmx_find_loadstore_msr_slot(&m->host, msr); 1011 if (i < 0) 1012 return; 1013 1014 --m->host.nr; 1015 m->host.val[i] = m->host.val[m->host.nr]; 1016 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr); 1017 } 1018 1019 static __always_inline void add_atomic_switch_msr_special(struct vcpu_vmx *vmx, 1020 unsigned long entry, unsigned long exit, 1021 unsigned long guest_val_vmcs, unsigned long host_val_vmcs, 1022 u64 guest_val, u64 host_val) 1023 { 1024 vmcs_write64(guest_val_vmcs, guest_val); 1025 if (host_val_vmcs != HOST_IA32_EFER) 1026 vmcs_write64(host_val_vmcs, host_val); 1027 vm_entry_controls_setbit(vmx, entry); 1028 vm_exit_controls_setbit(vmx, exit); 1029 } 1030 1031 static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr, 1032 u64 guest_val, u64 host_val, bool entry_only) 1033 { 1034 int i, j = 0; 1035 struct msr_autoload *m = &vmx->msr_autoload; 1036 1037 switch (msr) { 1038 case MSR_EFER: 1039 if (cpu_has_load_ia32_efer()) { 1040 add_atomic_switch_msr_special(vmx, 1041 VM_ENTRY_LOAD_IA32_EFER, 1042 VM_EXIT_LOAD_IA32_EFER, 1043 GUEST_IA32_EFER, 1044 HOST_IA32_EFER, 1045 guest_val, host_val); 1046 return; 1047 } 1048 break; 1049 case MSR_CORE_PERF_GLOBAL_CTRL: 1050 if (cpu_has_load_perf_global_ctrl()) { 1051 add_atomic_switch_msr_special(vmx, 1052 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL, 1053 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL, 1054 GUEST_IA32_PERF_GLOBAL_CTRL, 1055 HOST_IA32_PERF_GLOBAL_CTRL, 1056 guest_val, host_val); 1057 return; 1058 } 1059 break; 1060 case MSR_IA32_PEBS_ENABLE: 1061 /* PEBS needs a quiescent period after being disabled (to write 1062 * a record). Disabling PEBS through VMX MSR swapping doesn't 1063 * provide that period, so a CPU could write host's record into 1064 * guest's memory. 1065 */ 1066 wrmsrl(MSR_IA32_PEBS_ENABLE, 0); 1067 } 1068 1069 i = vmx_find_loadstore_msr_slot(&m->guest, msr); 1070 if (!entry_only) 1071 j = vmx_find_loadstore_msr_slot(&m->host, msr); 1072 1073 if ((i < 0 && m->guest.nr == MAX_NR_LOADSTORE_MSRS) || 1074 (j < 0 && m->host.nr == MAX_NR_LOADSTORE_MSRS)) { 1075 printk_once(KERN_WARNING "Not enough msr switch entries. " 1076 "Can't add msr %x\n", msr); 1077 return; 1078 } 1079 if (i < 0) { 1080 i = m->guest.nr++; 1081 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr); 1082 } 1083 m->guest.val[i].index = msr; 1084 m->guest.val[i].value = guest_val; 1085 1086 if (entry_only) 1087 return; 1088 1089 if (j < 0) { 1090 j = m->host.nr++; 1091 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr); 1092 } 1093 m->host.val[j].index = msr; 1094 m->host.val[j].value = host_val; 1095 } 1096 1097 static bool update_transition_efer(struct vcpu_vmx *vmx) 1098 { 1099 u64 guest_efer = vmx->vcpu.arch.efer; 1100 u64 ignore_bits = 0; 1101 int i; 1102 1103 /* Shadow paging assumes NX to be available. */ 1104 if (!enable_ept) 1105 guest_efer |= EFER_NX; 1106 1107 /* 1108 * LMA and LME handled by hardware; SCE meaningless outside long mode. 1109 */ 1110 ignore_bits |= EFER_SCE; 1111 #ifdef CONFIG_X86_64 1112 ignore_bits |= EFER_LMA | EFER_LME; 1113 /* SCE is meaningful only in long mode on Intel */ 1114 if (guest_efer & EFER_LMA) 1115 ignore_bits &= ~(u64)EFER_SCE; 1116 #endif 1117 1118 /* 1119 * On EPT, we can't emulate NX, so we must switch EFER atomically. 1120 * On CPUs that support "load IA32_EFER", always switch EFER 1121 * atomically, since it's faster than switching it manually. 1122 */ 1123 if (cpu_has_load_ia32_efer() || 1124 (enable_ept && ((vmx->vcpu.arch.efer ^ kvm_host.efer) & EFER_NX))) { 1125 if (!(guest_efer & EFER_LMA)) 1126 guest_efer &= ~EFER_LME; 1127 if (guest_efer != kvm_host.efer) 1128 add_atomic_switch_msr(vmx, MSR_EFER, 1129 guest_efer, kvm_host.efer, false); 1130 else 1131 clear_atomic_switch_msr(vmx, MSR_EFER); 1132 return false; 1133 } 1134 1135 i = kvm_find_user_return_msr(MSR_EFER); 1136 if (i < 0) 1137 return false; 1138 1139 clear_atomic_switch_msr(vmx, MSR_EFER); 1140 1141 guest_efer &= ~ignore_bits; 1142 guest_efer |= kvm_host.efer & ignore_bits; 1143 1144 vmx->guest_uret_msrs[i].data = guest_efer; 1145 vmx->guest_uret_msrs[i].mask = ~ignore_bits; 1146 1147 return true; 1148 } 1149 1150 #ifdef CONFIG_X86_32 1151 /* 1152 * On 32-bit kernels, VM exits still load the FS and GS bases from the 1153 * VMCS rather than the segment table. KVM uses this helper to figure 1154 * out the current bases to poke them into the VMCS before entry. 1155 */ 1156 static unsigned long segment_base(u16 selector) 1157 { 1158 struct desc_struct *table; 1159 unsigned long v; 1160 1161 if (!(selector & ~SEGMENT_RPL_MASK)) 1162 return 0; 1163 1164 table = get_current_gdt_ro(); 1165 1166 if ((selector & SEGMENT_TI_MASK) == SEGMENT_LDT) { 1167 u16 ldt_selector = kvm_read_ldt(); 1168 1169 if (!(ldt_selector & ~SEGMENT_RPL_MASK)) 1170 return 0; 1171 1172 table = (struct desc_struct *)segment_base(ldt_selector); 1173 } 1174 v = get_desc_base(&table[selector >> 3]); 1175 return v; 1176 } 1177 #endif 1178 1179 static inline bool pt_can_write_msr(struct vcpu_vmx *vmx) 1180 { 1181 return vmx_pt_mode_is_host_guest() && 1182 !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN); 1183 } 1184 1185 static inline bool pt_output_base_valid(struct kvm_vcpu *vcpu, u64 base) 1186 { 1187 /* The base must be 128-byte aligned and a legal physical address. */ 1188 return kvm_vcpu_is_legal_aligned_gpa(vcpu, base, 128); 1189 } 1190 1191 static inline void pt_load_msr(struct pt_ctx *ctx, u32 addr_range) 1192 { 1193 u32 i; 1194 1195 wrmsrl(MSR_IA32_RTIT_STATUS, ctx->status); 1196 wrmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base); 1197 wrmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask); 1198 wrmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match); 1199 for (i = 0; i < addr_range; i++) { 1200 wrmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]); 1201 wrmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]); 1202 } 1203 } 1204 1205 static inline void pt_save_msr(struct pt_ctx *ctx, u32 addr_range) 1206 { 1207 u32 i; 1208 1209 rdmsrl(MSR_IA32_RTIT_STATUS, ctx->status); 1210 rdmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base); 1211 rdmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask); 1212 rdmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match); 1213 for (i = 0; i < addr_range; i++) { 1214 rdmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]); 1215 rdmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]); 1216 } 1217 } 1218 1219 static void pt_guest_enter(struct vcpu_vmx *vmx) 1220 { 1221 if (vmx_pt_mode_is_system()) 1222 return; 1223 1224 /* 1225 * GUEST_IA32_RTIT_CTL is already set in the VMCS. 1226 * Save host state before VM entry. 1227 */ 1228 rdmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); 1229 if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) { 1230 wrmsrl(MSR_IA32_RTIT_CTL, 0); 1231 pt_save_msr(&vmx->pt_desc.host, vmx->pt_desc.num_address_ranges); 1232 pt_load_msr(&vmx->pt_desc.guest, vmx->pt_desc.num_address_ranges); 1233 } 1234 } 1235 1236 static void pt_guest_exit(struct vcpu_vmx *vmx) 1237 { 1238 if (vmx_pt_mode_is_system()) 1239 return; 1240 1241 if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) { 1242 pt_save_msr(&vmx->pt_desc.guest, vmx->pt_desc.num_address_ranges); 1243 pt_load_msr(&vmx->pt_desc.host, vmx->pt_desc.num_address_ranges); 1244 } 1245 1246 /* 1247 * KVM requires VM_EXIT_CLEAR_IA32_RTIT_CTL to expose PT to the guest, 1248 * i.e. RTIT_CTL is always cleared on VM-Exit. Restore it if necessary. 1249 */ 1250 if (vmx->pt_desc.host.ctl) 1251 wrmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); 1252 } 1253 1254 void vmx_set_host_fs_gs(struct vmcs_host_state *host, u16 fs_sel, u16 gs_sel, 1255 unsigned long fs_base, unsigned long gs_base) 1256 { 1257 if (unlikely(fs_sel != host->fs_sel)) { 1258 if (!(fs_sel & 7)) 1259 vmcs_write16(HOST_FS_SELECTOR, fs_sel); 1260 else 1261 vmcs_write16(HOST_FS_SELECTOR, 0); 1262 host->fs_sel = fs_sel; 1263 } 1264 if (unlikely(gs_sel != host->gs_sel)) { 1265 if (!(gs_sel & 7)) 1266 vmcs_write16(HOST_GS_SELECTOR, gs_sel); 1267 else 1268 vmcs_write16(HOST_GS_SELECTOR, 0); 1269 host->gs_sel = gs_sel; 1270 } 1271 if (unlikely(fs_base != host->fs_base)) { 1272 vmcs_writel(HOST_FS_BASE, fs_base); 1273 host->fs_base = fs_base; 1274 } 1275 if (unlikely(gs_base != host->gs_base)) { 1276 vmcs_writel(HOST_GS_BASE, gs_base); 1277 host->gs_base = gs_base; 1278 } 1279 } 1280 1281 void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu) 1282 { 1283 struct vcpu_vmx *vmx = to_vmx(vcpu); 1284 struct vmcs_host_state *host_state; 1285 #ifdef CONFIG_X86_64 1286 int cpu = raw_smp_processor_id(); 1287 #endif 1288 unsigned long fs_base, gs_base; 1289 u16 fs_sel, gs_sel; 1290 int i; 1291 1292 /* 1293 * Note that guest MSRs to be saved/restored can also be changed 1294 * when guest state is loaded. This happens when guest transitions 1295 * to/from long-mode by setting MSR_EFER.LMA. 1296 */ 1297 if (!vmx->guest_uret_msrs_loaded) { 1298 vmx->guest_uret_msrs_loaded = true; 1299 for (i = 0; i < kvm_nr_uret_msrs; ++i) { 1300 if (!vmx->guest_uret_msrs[i].load_into_hardware) 1301 continue; 1302 1303 kvm_set_user_return_msr(i, 1304 vmx->guest_uret_msrs[i].data, 1305 vmx->guest_uret_msrs[i].mask); 1306 } 1307 } 1308 1309 if (vmx->nested.need_vmcs12_to_shadow_sync) 1310 nested_sync_vmcs12_to_shadow(vcpu); 1311 1312 if (vmx->guest_state_loaded) 1313 return; 1314 1315 host_state = &vmx->loaded_vmcs->host_state; 1316 1317 /* 1318 * Set host fs and gs selectors. Unfortunately, 22.2.3 does not 1319 * allow segment selectors with cpl > 0 or ti == 1. 1320 */ 1321 host_state->ldt_sel = kvm_read_ldt(); 1322 1323 #ifdef CONFIG_X86_64 1324 savesegment(ds, host_state->ds_sel); 1325 savesegment(es, host_state->es_sel); 1326 1327 gs_base = cpu_kernelmode_gs_base(cpu); 1328 if (likely(is_64bit_mm(current->mm))) { 1329 current_save_fsgs(); 1330 fs_sel = current->thread.fsindex; 1331 gs_sel = current->thread.gsindex; 1332 fs_base = current->thread.fsbase; 1333 vmx->msr_host_kernel_gs_base = current->thread.gsbase; 1334 } else { 1335 savesegment(fs, fs_sel); 1336 savesegment(gs, gs_sel); 1337 fs_base = read_msr(MSR_FS_BASE); 1338 vmx->msr_host_kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE); 1339 } 1340 1341 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); 1342 #else 1343 savesegment(fs, fs_sel); 1344 savesegment(gs, gs_sel); 1345 fs_base = segment_base(fs_sel); 1346 gs_base = segment_base(gs_sel); 1347 #endif 1348 1349 vmx_set_host_fs_gs(host_state, fs_sel, gs_sel, fs_base, gs_base); 1350 vmx->guest_state_loaded = true; 1351 } 1352 1353 static void vmx_prepare_switch_to_host(struct vcpu_vmx *vmx) 1354 { 1355 struct vmcs_host_state *host_state; 1356 1357 if (!vmx->guest_state_loaded) 1358 return; 1359 1360 host_state = &vmx->loaded_vmcs->host_state; 1361 1362 ++vmx->vcpu.stat.host_state_reload; 1363 1364 #ifdef CONFIG_X86_64 1365 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); 1366 #endif 1367 if (host_state->ldt_sel || (host_state->gs_sel & 7)) { 1368 kvm_load_ldt(host_state->ldt_sel); 1369 #ifdef CONFIG_X86_64 1370 load_gs_index(host_state->gs_sel); 1371 #else 1372 loadsegment(gs, host_state->gs_sel); 1373 #endif 1374 } 1375 if (host_state->fs_sel & 7) 1376 loadsegment(fs, host_state->fs_sel); 1377 #ifdef CONFIG_X86_64 1378 if (unlikely(host_state->ds_sel | host_state->es_sel)) { 1379 loadsegment(ds, host_state->ds_sel); 1380 loadsegment(es, host_state->es_sel); 1381 } 1382 #endif 1383 invalidate_tss_limit(); 1384 #ifdef CONFIG_X86_64 1385 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base); 1386 #endif 1387 load_fixmap_gdt(raw_smp_processor_id()); 1388 vmx->guest_state_loaded = false; 1389 vmx->guest_uret_msrs_loaded = false; 1390 } 1391 1392 #ifdef CONFIG_X86_64 1393 static u64 vmx_read_guest_kernel_gs_base(struct vcpu_vmx *vmx) 1394 { 1395 preempt_disable(); 1396 if (vmx->guest_state_loaded) 1397 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); 1398 preempt_enable(); 1399 return vmx->msr_guest_kernel_gs_base; 1400 } 1401 1402 static void vmx_write_guest_kernel_gs_base(struct vcpu_vmx *vmx, u64 data) 1403 { 1404 preempt_disable(); 1405 if (vmx->guest_state_loaded) 1406 wrmsrl(MSR_KERNEL_GS_BASE, data); 1407 preempt_enable(); 1408 vmx->msr_guest_kernel_gs_base = data; 1409 } 1410 #endif 1411 1412 static void grow_ple_window(struct kvm_vcpu *vcpu) 1413 { 1414 struct vcpu_vmx *vmx = to_vmx(vcpu); 1415 unsigned int old = vmx->ple_window; 1416 1417 vmx->ple_window = __grow_ple_window(old, ple_window, 1418 ple_window_grow, 1419 ple_window_max); 1420 1421 if (vmx->ple_window != old) { 1422 vmx->ple_window_dirty = true; 1423 trace_kvm_ple_window_update(vcpu->vcpu_id, 1424 vmx->ple_window, old); 1425 } 1426 } 1427 1428 static void shrink_ple_window(struct kvm_vcpu *vcpu) 1429 { 1430 struct vcpu_vmx *vmx = to_vmx(vcpu); 1431 unsigned int old = vmx->ple_window; 1432 1433 vmx->ple_window = __shrink_ple_window(old, ple_window, 1434 ple_window_shrink, 1435 ple_window); 1436 1437 if (vmx->ple_window != old) { 1438 vmx->ple_window_dirty = true; 1439 trace_kvm_ple_window_update(vcpu->vcpu_id, 1440 vmx->ple_window, old); 1441 } 1442 } 1443 1444 void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu, 1445 struct loaded_vmcs *buddy) 1446 { 1447 struct vcpu_vmx *vmx = to_vmx(vcpu); 1448 bool already_loaded = vmx->loaded_vmcs->cpu == cpu; 1449 struct vmcs *prev; 1450 1451 if (!already_loaded) { 1452 loaded_vmcs_clear(vmx->loaded_vmcs); 1453 local_irq_disable(); 1454 1455 /* 1456 * Ensure loaded_vmcs->cpu is read before adding loaded_vmcs to 1457 * this cpu's percpu list, otherwise it may not yet be deleted 1458 * from its previous cpu's percpu list. Pairs with the 1459 * smb_wmb() in __loaded_vmcs_clear(). 1460 */ 1461 smp_rmb(); 1462 1463 list_add(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link, 1464 &per_cpu(loaded_vmcss_on_cpu, cpu)); 1465 local_irq_enable(); 1466 } 1467 1468 prev = per_cpu(current_vmcs, cpu); 1469 if (prev != vmx->loaded_vmcs->vmcs) { 1470 per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs; 1471 vmcs_load(vmx->loaded_vmcs->vmcs); 1472 1473 /* 1474 * No indirect branch prediction barrier needed when switching 1475 * the active VMCS within a vCPU, unless IBRS is advertised to 1476 * the vCPU. To minimize the number of IBPBs executed, KVM 1477 * performs IBPB on nested VM-Exit (a single nested transition 1478 * may switch the active VMCS multiple times). 1479 */ 1480 if (!buddy || WARN_ON_ONCE(buddy->vmcs != prev)) 1481 indirect_branch_prediction_barrier(); 1482 } 1483 1484 if (!already_loaded) { 1485 void *gdt = get_current_gdt_ro(); 1486 1487 /* 1488 * Flush all EPTP/VPID contexts, the new pCPU may have stale 1489 * TLB entries from its previous association with the vCPU. 1490 */ 1491 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 1492 1493 /* 1494 * Linux uses per-cpu TSS and GDT, so set these when switching 1495 * processors. See 22.2.4. 1496 */ 1497 vmcs_writel(HOST_TR_BASE, 1498 (unsigned long)&get_cpu_entry_area(cpu)->tss.x86_tss); 1499 vmcs_writel(HOST_GDTR_BASE, (unsigned long)gdt); /* 22.2.4 */ 1500 1501 if (IS_ENABLED(CONFIG_IA32_EMULATION) || IS_ENABLED(CONFIG_X86_32)) { 1502 /* 22.2.3 */ 1503 vmcs_writel(HOST_IA32_SYSENTER_ESP, 1504 (unsigned long)(cpu_entry_stack(cpu) + 1)); 1505 } 1506 1507 vmx->loaded_vmcs->cpu = cpu; 1508 } 1509 } 1510 1511 /* 1512 * Switches to specified vcpu, until a matching vcpu_put(), but assumes 1513 * vcpu mutex is already taken. 1514 */ 1515 void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1516 { 1517 struct vcpu_vmx *vmx = to_vmx(vcpu); 1518 1519 if (vcpu->scheduled_out && !kvm_pause_in_guest(vcpu->kvm)) 1520 shrink_ple_window(vcpu); 1521 1522 vmx_vcpu_load_vmcs(vcpu, cpu, NULL); 1523 1524 vmx_vcpu_pi_load(vcpu, cpu); 1525 1526 vmx->host_debugctlmsr = get_debugctlmsr(); 1527 } 1528 1529 void vmx_vcpu_put(struct kvm_vcpu *vcpu) 1530 { 1531 vmx_vcpu_pi_put(vcpu); 1532 1533 vmx_prepare_switch_to_host(to_vmx(vcpu)); 1534 } 1535 1536 bool vmx_emulation_required(struct kvm_vcpu *vcpu) 1537 { 1538 return emulate_invalid_guest_state && !vmx_guest_state_valid(vcpu); 1539 } 1540 1541 unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu) 1542 { 1543 struct vcpu_vmx *vmx = to_vmx(vcpu); 1544 unsigned long rflags, save_rflags; 1545 1546 if (!kvm_register_is_available(vcpu, VCPU_EXREG_RFLAGS)) { 1547 kvm_register_mark_available(vcpu, VCPU_EXREG_RFLAGS); 1548 rflags = vmcs_readl(GUEST_RFLAGS); 1549 if (vmx->rmode.vm86_active) { 1550 rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS; 1551 save_rflags = vmx->rmode.save_rflags; 1552 rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS; 1553 } 1554 vmx->rflags = rflags; 1555 } 1556 return vmx->rflags; 1557 } 1558 1559 void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) 1560 { 1561 struct vcpu_vmx *vmx = to_vmx(vcpu); 1562 unsigned long old_rflags; 1563 1564 /* 1565 * Unlike CR0 and CR4, RFLAGS handling requires checking if the vCPU 1566 * is an unrestricted guest in order to mark L2 as needing emulation 1567 * if L1 runs L2 as a restricted guest. 1568 */ 1569 if (is_unrestricted_guest(vcpu)) { 1570 kvm_register_mark_available(vcpu, VCPU_EXREG_RFLAGS); 1571 vmx->rflags = rflags; 1572 vmcs_writel(GUEST_RFLAGS, rflags); 1573 return; 1574 } 1575 1576 old_rflags = vmx_get_rflags(vcpu); 1577 vmx->rflags = rflags; 1578 if (vmx->rmode.vm86_active) { 1579 vmx->rmode.save_rflags = rflags; 1580 rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM; 1581 } 1582 vmcs_writel(GUEST_RFLAGS, rflags); 1583 1584 if ((old_rflags ^ vmx->rflags) & X86_EFLAGS_VM) 1585 vmx->emulation_required = vmx_emulation_required(vcpu); 1586 } 1587 1588 bool vmx_get_if_flag(struct kvm_vcpu *vcpu) 1589 { 1590 return vmx_get_rflags(vcpu) & X86_EFLAGS_IF; 1591 } 1592 1593 u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu) 1594 { 1595 u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 1596 int ret = 0; 1597 1598 if (interruptibility & GUEST_INTR_STATE_STI) 1599 ret |= KVM_X86_SHADOW_INT_STI; 1600 if (interruptibility & GUEST_INTR_STATE_MOV_SS) 1601 ret |= KVM_X86_SHADOW_INT_MOV_SS; 1602 1603 return ret; 1604 } 1605 1606 void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask) 1607 { 1608 u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 1609 u32 interruptibility = interruptibility_old; 1610 1611 interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS); 1612 1613 if (mask & KVM_X86_SHADOW_INT_MOV_SS) 1614 interruptibility |= GUEST_INTR_STATE_MOV_SS; 1615 else if (mask & KVM_X86_SHADOW_INT_STI) 1616 interruptibility |= GUEST_INTR_STATE_STI; 1617 1618 if ((interruptibility != interruptibility_old)) 1619 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility); 1620 } 1621 1622 static int vmx_rtit_ctl_check(struct kvm_vcpu *vcpu, u64 data) 1623 { 1624 struct vcpu_vmx *vmx = to_vmx(vcpu); 1625 unsigned long value; 1626 1627 /* 1628 * Any MSR write that attempts to change bits marked reserved will 1629 * case a #GP fault. 1630 */ 1631 if (data & vmx->pt_desc.ctl_bitmask) 1632 return 1; 1633 1634 /* 1635 * Any attempt to modify IA32_RTIT_CTL while TraceEn is set will 1636 * result in a #GP unless the same write also clears TraceEn. 1637 */ 1638 if ((vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) && 1639 (data & RTIT_CTL_TRACEEN) && 1640 data != vmx->pt_desc.guest.ctl) 1641 return 1; 1642 1643 /* 1644 * WRMSR to IA32_RTIT_CTL that sets TraceEn but clears this bit 1645 * and FabricEn would cause #GP, if 1646 * CPUID.(EAX=14H, ECX=0):ECX.SNGLRGNOUT[bit 2] = 0 1647 */ 1648 if ((data & RTIT_CTL_TRACEEN) && !(data & RTIT_CTL_TOPA) && 1649 !(data & RTIT_CTL_FABRIC_EN) && 1650 !intel_pt_validate_cap(vmx->pt_desc.caps, 1651 PT_CAP_single_range_output)) 1652 return 1; 1653 1654 /* 1655 * MTCFreq, CycThresh and PSBFreq encodings check, any MSR write that 1656 * utilize encodings marked reserved will cause a #GP fault. 1657 */ 1658 value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc_periods); 1659 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc) && 1660 !test_bit((data & RTIT_CTL_MTC_RANGE) >> 1661 RTIT_CTL_MTC_RANGE_OFFSET, &value)) 1662 return 1; 1663 value = intel_pt_validate_cap(vmx->pt_desc.caps, 1664 PT_CAP_cycle_thresholds); 1665 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) && 1666 !test_bit((data & RTIT_CTL_CYC_THRESH) >> 1667 RTIT_CTL_CYC_THRESH_OFFSET, &value)) 1668 return 1; 1669 value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_periods); 1670 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) && 1671 !test_bit((data & RTIT_CTL_PSB_FREQ) >> 1672 RTIT_CTL_PSB_FREQ_OFFSET, &value)) 1673 return 1; 1674 1675 /* 1676 * If ADDRx_CFG is reserved or the encodings is >2 will 1677 * cause a #GP fault. 1678 */ 1679 value = (data & RTIT_CTL_ADDR0) >> RTIT_CTL_ADDR0_OFFSET; 1680 if ((value && (vmx->pt_desc.num_address_ranges < 1)) || (value > 2)) 1681 return 1; 1682 value = (data & RTIT_CTL_ADDR1) >> RTIT_CTL_ADDR1_OFFSET; 1683 if ((value && (vmx->pt_desc.num_address_ranges < 2)) || (value > 2)) 1684 return 1; 1685 value = (data & RTIT_CTL_ADDR2) >> RTIT_CTL_ADDR2_OFFSET; 1686 if ((value && (vmx->pt_desc.num_address_ranges < 3)) || (value > 2)) 1687 return 1; 1688 value = (data & RTIT_CTL_ADDR3) >> RTIT_CTL_ADDR3_OFFSET; 1689 if ((value && (vmx->pt_desc.num_address_ranges < 4)) || (value > 2)) 1690 return 1; 1691 1692 return 0; 1693 } 1694 1695 int vmx_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, 1696 void *insn, int insn_len) 1697 { 1698 /* 1699 * Emulation of instructions in SGX enclaves is impossible as RIP does 1700 * not point at the failing instruction, and even if it did, the code 1701 * stream is inaccessible. Inject #UD instead of exiting to userspace 1702 * so that guest userspace can't DoS the guest simply by triggering 1703 * emulation (enclaves are CPL3 only). 1704 */ 1705 if (to_vmx(vcpu)->exit_reason.enclave_mode) { 1706 kvm_queue_exception(vcpu, UD_VECTOR); 1707 return X86EMUL_PROPAGATE_FAULT; 1708 } 1709 1710 /* Check that emulation is possible during event vectoring */ 1711 if ((to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) && 1712 !kvm_can_emulate_event_vectoring(emul_type)) 1713 return X86EMUL_UNHANDLEABLE_VECTORING; 1714 1715 return X86EMUL_CONTINUE; 1716 } 1717 1718 static int skip_emulated_instruction(struct kvm_vcpu *vcpu) 1719 { 1720 union vmx_exit_reason exit_reason = to_vmx(vcpu)->exit_reason; 1721 unsigned long rip, orig_rip; 1722 u32 instr_len; 1723 1724 /* 1725 * Using VMCS.VM_EXIT_INSTRUCTION_LEN on EPT misconfig depends on 1726 * undefined behavior: Intel's SDM doesn't mandate the VMCS field be 1727 * set when EPT misconfig occurs. In practice, real hardware updates 1728 * VM_EXIT_INSTRUCTION_LEN on EPT misconfig, but other hypervisors 1729 * (namely Hyper-V) don't set it due to it being undefined behavior, 1730 * i.e. we end up advancing IP with some random value. 1731 */ 1732 if (!static_cpu_has(X86_FEATURE_HYPERVISOR) || 1733 exit_reason.basic != EXIT_REASON_EPT_MISCONFIG) { 1734 instr_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 1735 1736 /* 1737 * Emulating an enclave's instructions isn't supported as KVM 1738 * cannot access the enclave's memory or its true RIP, e.g. the 1739 * vmcs.GUEST_RIP points at the exit point of the enclave, not 1740 * the RIP that actually triggered the VM-Exit. But, because 1741 * most instructions that cause VM-Exit will #UD in an enclave, 1742 * most instruction-based VM-Exits simply do not occur. 1743 * 1744 * There are a few exceptions, notably the debug instructions 1745 * INT1ICEBRK and INT3, as they are allowed in debug enclaves 1746 * and generate #DB/#BP as expected, which KVM might intercept. 1747 * But again, the CPU does the dirty work and saves an instr 1748 * length of zero so VMMs don't shoot themselves in the foot. 1749 * WARN if KVM tries to skip a non-zero length instruction on 1750 * a VM-Exit from an enclave. 1751 */ 1752 if (!instr_len) 1753 goto rip_updated; 1754 1755 WARN_ONCE(exit_reason.enclave_mode, 1756 "skipping instruction after SGX enclave VM-Exit"); 1757 1758 orig_rip = kvm_rip_read(vcpu); 1759 rip = orig_rip + instr_len; 1760 #ifdef CONFIG_X86_64 1761 /* 1762 * We need to mask out the high 32 bits of RIP if not in 64-bit 1763 * mode, but just finding out that we are in 64-bit mode is 1764 * quite expensive. Only do it if there was a carry. 1765 */ 1766 if (unlikely(((rip ^ orig_rip) >> 31) == 3) && !is_64_bit_mode(vcpu)) 1767 rip = (u32)rip; 1768 #endif 1769 kvm_rip_write(vcpu, rip); 1770 } else { 1771 if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP)) 1772 return 0; 1773 } 1774 1775 rip_updated: 1776 /* skipping an emulated instruction also counts */ 1777 vmx_set_interrupt_shadow(vcpu, 0); 1778 1779 return 1; 1780 } 1781 1782 /* 1783 * Recognizes a pending MTF VM-exit and records the nested state for later 1784 * delivery. 1785 */ 1786 void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu) 1787 { 1788 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1789 struct vcpu_vmx *vmx = to_vmx(vcpu); 1790 1791 if (!is_guest_mode(vcpu)) 1792 return; 1793 1794 /* 1795 * Per the SDM, MTF takes priority over debug-trap exceptions besides 1796 * TSS T-bit traps and ICEBP (INT1). KVM doesn't emulate T-bit traps 1797 * or ICEBP (in the emulator proper), and skipping of ICEBP after an 1798 * intercepted #DB deliberately avoids single-step #DB and MTF updates 1799 * as ICEBP is higher priority than both. As instruction emulation is 1800 * completed at this point (i.e. KVM is at the instruction boundary), 1801 * any #DB exception pending delivery must be a debug-trap of lower 1802 * priority than MTF. Record the pending MTF state to be delivered in 1803 * vmx_check_nested_events(). 1804 */ 1805 if (nested_cpu_has_mtf(vmcs12) && 1806 (!vcpu->arch.exception.pending || 1807 vcpu->arch.exception.vector == DB_VECTOR) && 1808 (!vcpu->arch.exception_vmexit.pending || 1809 vcpu->arch.exception_vmexit.vector == DB_VECTOR)) { 1810 vmx->nested.mtf_pending = true; 1811 kvm_make_request(KVM_REQ_EVENT, vcpu); 1812 } else { 1813 vmx->nested.mtf_pending = false; 1814 } 1815 } 1816 1817 int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu) 1818 { 1819 vmx_update_emulated_instruction(vcpu); 1820 return skip_emulated_instruction(vcpu); 1821 } 1822 1823 static void vmx_clear_hlt(struct kvm_vcpu *vcpu) 1824 { 1825 /* 1826 * Ensure that we clear the HLT state in the VMCS. We don't need to 1827 * explicitly skip the instruction because if the HLT state is set, 1828 * then the instruction is already executing and RIP has already been 1829 * advanced. 1830 */ 1831 if (kvm_hlt_in_guest(vcpu->kvm) && 1832 vmcs_read32(GUEST_ACTIVITY_STATE) == GUEST_ACTIVITY_HLT) 1833 vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE); 1834 } 1835 1836 void vmx_inject_exception(struct kvm_vcpu *vcpu) 1837 { 1838 struct kvm_queued_exception *ex = &vcpu->arch.exception; 1839 u32 intr_info = ex->vector | INTR_INFO_VALID_MASK; 1840 struct vcpu_vmx *vmx = to_vmx(vcpu); 1841 1842 kvm_deliver_exception_payload(vcpu, ex); 1843 1844 if (ex->has_error_code) { 1845 /* 1846 * Despite the error code being architecturally defined as 32 1847 * bits, and the VMCS field being 32 bits, Intel CPUs and thus 1848 * VMX don't actually supporting setting bits 31:16. Hardware 1849 * will (should) never provide a bogus error code, but AMD CPUs 1850 * do generate error codes with bits 31:16 set, and so KVM's 1851 * ABI lets userspace shove in arbitrary 32-bit values. Drop 1852 * the upper bits to avoid VM-Fail, losing information that 1853 * doesn't really exist is preferable to killing the VM. 1854 */ 1855 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, (u16)ex->error_code); 1856 intr_info |= INTR_INFO_DELIVER_CODE_MASK; 1857 } 1858 1859 if (vmx->rmode.vm86_active) { 1860 int inc_eip = 0; 1861 if (kvm_exception_is_soft(ex->vector)) 1862 inc_eip = vcpu->arch.event_exit_inst_len; 1863 kvm_inject_realmode_interrupt(vcpu, ex->vector, inc_eip); 1864 return; 1865 } 1866 1867 WARN_ON_ONCE(vmx->emulation_required); 1868 1869 if (kvm_exception_is_soft(ex->vector)) { 1870 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1871 vmx->vcpu.arch.event_exit_inst_len); 1872 intr_info |= INTR_TYPE_SOFT_EXCEPTION; 1873 } else 1874 intr_info |= INTR_TYPE_HARD_EXCEPTION; 1875 1876 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info); 1877 1878 vmx_clear_hlt(vcpu); 1879 } 1880 1881 static void vmx_setup_uret_msr(struct vcpu_vmx *vmx, unsigned int msr, 1882 bool load_into_hardware) 1883 { 1884 struct vmx_uret_msr *uret_msr; 1885 1886 uret_msr = vmx_find_uret_msr(vmx, msr); 1887 if (!uret_msr) 1888 return; 1889 1890 uret_msr->load_into_hardware = load_into_hardware; 1891 } 1892 1893 /* 1894 * Configuring user return MSRs to automatically save, load, and restore MSRs 1895 * that need to be shoved into hardware when running the guest. Note, omitting 1896 * an MSR here does _NOT_ mean it's not emulated, only that it will not be 1897 * loaded into hardware when running the guest. 1898 */ 1899 static void vmx_setup_uret_msrs(struct vcpu_vmx *vmx) 1900 { 1901 #ifdef CONFIG_X86_64 1902 bool load_syscall_msrs; 1903 1904 /* 1905 * The SYSCALL MSRs are only needed on long mode guests, and only 1906 * when EFER.SCE is set. 1907 */ 1908 load_syscall_msrs = is_long_mode(&vmx->vcpu) && 1909 (vmx->vcpu.arch.efer & EFER_SCE); 1910 1911 vmx_setup_uret_msr(vmx, MSR_STAR, load_syscall_msrs); 1912 vmx_setup_uret_msr(vmx, MSR_LSTAR, load_syscall_msrs); 1913 vmx_setup_uret_msr(vmx, MSR_SYSCALL_MASK, load_syscall_msrs); 1914 #endif 1915 vmx_setup_uret_msr(vmx, MSR_EFER, update_transition_efer(vmx)); 1916 1917 vmx_setup_uret_msr(vmx, MSR_TSC_AUX, 1918 guest_cpu_cap_has(&vmx->vcpu, X86_FEATURE_RDTSCP) || 1919 guest_cpu_cap_has(&vmx->vcpu, X86_FEATURE_RDPID)); 1920 1921 /* 1922 * hle=0, rtm=0, tsx_ctrl=1 can be found with some combinations of new 1923 * kernel and old userspace. If those guests run on a tsx=off host, do 1924 * allow guests to use TSX_CTRL, but don't change the value in hardware 1925 * so that TSX remains always disabled. 1926 */ 1927 vmx_setup_uret_msr(vmx, MSR_IA32_TSX_CTRL, boot_cpu_has(X86_FEATURE_RTM)); 1928 1929 /* 1930 * The set of MSRs to load may have changed, reload MSRs before the 1931 * next VM-Enter. 1932 */ 1933 vmx->guest_uret_msrs_loaded = false; 1934 } 1935 1936 u64 vmx_get_l2_tsc_offset(struct kvm_vcpu *vcpu) 1937 { 1938 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1939 1940 if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING)) 1941 return vmcs12->tsc_offset; 1942 1943 return 0; 1944 } 1945 1946 u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu) 1947 { 1948 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1949 1950 if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING) && 1951 nested_cpu_has2(vmcs12, SECONDARY_EXEC_TSC_SCALING)) 1952 return vmcs12->tsc_multiplier; 1953 1954 return kvm_caps.default_tsc_scaling_ratio; 1955 } 1956 1957 void vmx_write_tsc_offset(struct kvm_vcpu *vcpu) 1958 { 1959 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 1960 } 1961 1962 void vmx_write_tsc_multiplier(struct kvm_vcpu *vcpu) 1963 { 1964 vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio); 1965 } 1966 1967 /* 1968 * Userspace is allowed to set any supported IA32_FEATURE_CONTROL regardless of 1969 * guest CPUID. Note, KVM allows userspace to set "VMX in SMX" to maintain 1970 * backwards compatibility even though KVM doesn't support emulating SMX. And 1971 * because userspace set "VMX in SMX", the guest must also be allowed to set it, 1972 * e.g. if the MSR is left unlocked and the guest does a RMW operation. 1973 */ 1974 #define KVM_SUPPORTED_FEATURE_CONTROL (FEAT_CTL_LOCKED | \ 1975 FEAT_CTL_VMX_ENABLED_INSIDE_SMX | \ 1976 FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX | \ 1977 FEAT_CTL_SGX_LC_ENABLED | \ 1978 FEAT_CTL_SGX_ENABLED | \ 1979 FEAT_CTL_LMCE_ENABLED) 1980 1981 static inline bool is_vmx_feature_control_msr_valid(struct vcpu_vmx *vmx, 1982 struct msr_data *msr) 1983 { 1984 uint64_t valid_bits; 1985 1986 /* 1987 * Ensure KVM_SUPPORTED_FEATURE_CONTROL is updated when new bits are 1988 * exposed to the guest. 1989 */ 1990 WARN_ON_ONCE(vmx->msr_ia32_feature_control_valid_bits & 1991 ~KVM_SUPPORTED_FEATURE_CONTROL); 1992 1993 if (!msr->host_initiated && 1994 (vmx->msr_ia32_feature_control & FEAT_CTL_LOCKED)) 1995 return false; 1996 1997 if (msr->host_initiated) 1998 valid_bits = KVM_SUPPORTED_FEATURE_CONTROL; 1999 else 2000 valid_bits = vmx->msr_ia32_feature_control_valid_bits; 2001 2002 return !(msr->data & ~valid_bits); 2003 } 2004 2005 int vmx_get_feature_msr(u32 msr, u64 *data) 2006 { 2007 switch (msr) { 2008 case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR: 2009 if (!nested) 2010 return 1; 2011 return vmx_get_vmx_msr(&vmcs_config.nested, msr, data); 2012 default: 2013 return KVM_MSR_RET_UNSUPPORTED; 2014 } 2015 } 2016 2017 /* 2018 * Reads an msr value (of 'msr_info->index') into 'msr_info->data'. 2019 * Returns 0 on success, non-0 otherwise. 2020 * Assumes vcpu_load() was already called. 2021 */ 2022 int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 2023 { 2024 struct vcpu_vmx *vmx = to_vmx(vcpu); 2025 struct vmx_uret_msr *msr; 2026 u32 index; 2027 2028 switch (msr_info->index) { 2029 #ifdef CONFIG_X86_64 2030 case MSR_FS_BASE: 2031 msr_info->data = vmcs_readl(GUEST_FS_BASE); 2032 break; 2033 case MSR_GS_BASE: 2034 msr_info->data = vmcs_readl(GUEST_GS_BASE); 2035 break; 2036 case MSR_KERNEL_GS_BASE: 2037 msr_info->data = vmx_read_guest_kernel_gs_base(vmx); 2038 break; 2039 #endif 2040 case MSR_EFER: 2041 return kvm_get_msr_common(vcpu, msr_info); 2042 case MSR_IA32_TSX_CTRL: 2043 if (!msr_info->host_initiated && 2044 !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR)) 2045 return 1; 2046 goto find_uret_msr; 2047 case MSR_IA32_UMWAIT_CONTROL: 2048 if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx)) 2049 return 1; 2050 2051 msr_info->data = vmx->msr_ia32_umwait_control; 2052 break; 2053 case MSR_IA32_SPEC_CTRL: 2054 if (!msr_info->host_initiated && 2055 !guest_has_spec_ctrl_msr(vcpu)) 2056 return 1; 2057 2058 msr_info->data = to_vmx(vcpu)->spec_ctrl; 2059 break; 2060 case MSR_IA32_SYSENTER_CS: 2061 msr_info->data = vmcs_read32(GUEST_SYSENTER_CS); 2062 break; 2063 case MSR_IA32_SYSENTER_EIP: 2064 msr_info->data = vmcs_readl(GUEST_SYSENTER_EIP); 2065 break; 2066 case MSR_IA32_SYSENTER_ESP: 2067 msr_info->data = vmcs_readl(GUEST_SYSENTER_ESP); 2068 break; 2069 case MSR_IA32_BNDCFGS: 2070 if (!kvm_mpx_supported() || 2071 (!msr_info->host_initiated && 2072 !guest_cpu_cap_has(vcpu, X86_FEATURE_MPX))) 2073 return 1; 2074 msr_info->data = vmcs_read64(GUEST_BNDCFGS); 2075 break; 2076 case MSR_IA32_MCG_EXT_CTL: 2077 if (!msr_info->host_initiated && 2078 !(vmx->msr_ia32_feature_control & 2079 FEAT_CTL_LMCE_ENABLED)) 2080 return 1; 2081 msr_info->data = vcpu->arch.mcg_ext_ctl; 2082 break; 2083 case MSR_IA32_FEAT_CTL: 2084 msr_info->data = vmx->msr_ia32_feature_control; 2085 break; 2086 case MSR_IA32_SGXLEPUBKEYHASH0 ... MSR_IA32_SGXLEPUBKEYHASH3: 2087 if (!msr_info->host_initiated && 2088 !guest_cpu_cap_has(vcpu, X86_FEATURE_SGX_LC)) 2089 return 1; 2090 msr_info->data = to_vmx(vcpu)->msr_ia32_sgxlepubkeyhash 2091 [msr_info->index - MSR_IA32_SGXLEPUBKEYHASH0]; 2092 break; 2093 case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR: 2094 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_VMX)) 2095 return 1; 2096 if (vmx_get_vmx_msr(&vmx->nested.msrs, msr_info->index, 2097 &msr_info->data)) 2098 return 1; 2099 #ifdef CONFIG_KVM_HYPERV 2100 /* 2101 * Enlightened VMCS v1 doesn't have certain VMCS fields but 2102 * instead of just ignoring the features, different Hyper-V 2103 * versions are either trying to use them and fail or do some 2104 * sanity checking and refuse to boot. Filter all unsupported 2105 * features out. 2106 */ 2107 if (!msr_info->host_initiated && guest_cpu_cap_has_evmcs(vcpu)) 2108 nested_evmcs_filter_control_msr(vcpu, msr_info->index, 2109 &msr_info->data); 2110 #endif 2111 break; 2112 case MSR_IA32_RTIT_CTL: 2113 if (!vmx_pt_mode_is_host_guest()) 2114 return 1; 2115 msr_info->data = vmx->pt_desc.guest.ctl; 2116 break; 2117 case MSR_IA32_RTIT_STATUS: 2118 if (!vmx_pt_mode_is_host_guest()) 2119 return 1; 2120 msr_info->data = vmx->pt_desc.guest.status; 2121 break; 2122 case MSR_IA32_RTIT_CR3_MATCH: 2123 if (!vmx_pt_mode_is_host_guest() || 2124 !intel_pt_validate_cap(vmx->pt_desc.caps, 2125 PT_CAP_cr3_filtering)) 2126 return 1; 2127 msr_info->data = vmx->pt_desc.guest.cr3_match; 2128 break; 2129 case MSR_IA32_RTIT_OUTPUT_BASE: 2130 if (!vmx_pt_mode_is_host_guest() || 2131 (!intel_pt_validate_cap(vmx->pt_desc.caps, 2132 PT_CAP_topa_output) && 2133 !intel_pt_validate_cap(vmx->pt_desc.caps, 2134 PT_CAP_single_range_output))) 2135 return 1; 2136 msr_info->data = vmx->pt_desc.guest.output_base; 2137 break; 2138 case MSR_IA32_RTIT_OUTPUT_MASK: 2139 if (!vmx_pt_mode_is_host_guest() || 2140 (!intel_pt_validate_cap(vmx->pt_desc.caps, 2141 PT_CAP_topa_output) && 2142 !intel_pt_validate_cap(vmx->pt_desc.caps, 2143 PT_CAP_single_range_output))) 2144 return 1; 2145 msr_info->data = vmx->pt_desc.guest.output_mask; 2146 break; 2147 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: 2148 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A; 2149 if (!vmx_pt_mode_is_host_guest() || 2150 (index >= 2 * vmx->pt_desc.num_address_ranges)) 2151 return 1; 2152 if (index % 2) 2153 msr_info->data = vmx->pt_desc.guest.addr_b[index / 2]; 2154 else 2155 msr_info->data = vmx->pt_desc.guest.addr_a[index / 2]; 2156 break; 2157 case MSR_IA32_DEBUGCTLMSR: 2158 msr_info->data = vmcs_read64(GUEST_IA32_DEBUGCTL); 2159 break; 2160 default: 2161 find_uret_msr: 2162 msr = vmx_find_uret_msr(vmx, msr_info->index); 2163 if (msr) { 2164 msr_info->data = msr->data; 2165 break; 2166 } 2167 return kvm_get_msr_common(vcpu, msr_info); 2168 } 2169 2170 return 0; 2171 } 2172 2173 static u64 nested_vmx_truncate_sysenter_addr(struct kvm_vcpu *vcpu, 2174 u64 data) 2175 { 2176 #ifdef CONFIG_X86_64 2177 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_LM)) 2178 return (u32)data; 2179 #endif 2180 return (unsigned long)data; 2181 } 2182 2183 static u64 vmx_get_supported_debugctl(struct kvm_vcpu *vcpu, bool host_initiated) 2184 { 2185 u64 debugctl = 0; 2186 2187 if (boot_cpu_has(X86_FEATURE_BUS_LOCK_DETECT) && 2188 (host_initiated || guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT))) 2189 debugctl |= DEBUGCTLMSR_BUS_LOCK_DETECT; 2190 2191 if ((kvm_caps.supported_perf_cap & PMU_CAP_LBR_FMT) && 2192 (host_initiated || intel_pmu_lbr_is_enabled(vcpu))) 2193 debugctl |= DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI; 2194 2195 return debugctl; 2196 } 2197 2198 /* 2199 * Writes msr value into the appropriate "register". 2200 * Returns 0 on success, non-0 otherwise. 2201 * Assumes vcpu_load() was already called. 2202 */ 2203 int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 2204 { 2205 struct vcpu_vmx *vmx = to_vmx(vcpu); 2206 struct vmx_uret_msr *msr; 2207 int ret = 0; 2208 u32 msr_index = msr_info->index; 2209 u64 data = msr_info->data; 2210 u32 index; 2211 2212 switch (msr_index) { 2213 case MSR_EFER: 2214 ret = kvm_set_msr_common(vcpu, msr_info); 2215 break; 2216 #ifdef CONFIG_X86_64 2217 case MSR_FS_BASE: 2218 vmx_segment_cache_clear(vmx); 2219 vmcs_writel(GUEST_FS_BASE, data); 2220 break; 2221 case MSR_GS_BASE: 2222 vmx_segment_cache_clear(vmx); 2223 vmcs_writel(GUEST_GS_BASE, data); 2224 break; 2225 case MSR_KERNEL_GS_BASE: 2226 vmx_write_guest_kernel_gs_base(vmx, data); 2227 break; 2228 case MSR_IA32_XFD: 2229 ret = kvm_set_msr_common(vcpu, msr_info); 2230 /* 2231 * Always intercepting WRMSR could incur non-negligible 2232 * overhead given xfd might be changed frequently in 2233 * guest context switch. Disable write interception 2234 * upon the first write with a non-zero value (indicating 2235 * potential usage on dynamic xfeatures). Also update 2236 * exception bitmap to trap #NM for proper virtualization 2237 * of guest xfd_err. 2238 */ 2239 if (!ret && data) { 2240 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_XFD, 2241 MSR_TYPE_RW); 2242 vcpu->arch.xfd_no_write_intercept = true; 2243 vmx_update_exception_bitmap(vcpu); 2244 } 2245 break; 2246 #endif 2247 case MSR_IA32_SYSENTER_CS: 2248 if (is_guest_mode(vcpu)) 2249 get_vmcs12(vcpu)->guest_sysenter_cs = data; 2250 vmcs_write32(GUEST_SYSENTER_CS, data); 2251 break; 2252 case MSR_IA32_SYSENTER_EIP: 2253 if (is_guest_mode(vcpu)) { 2254 data = nested_vmx_truncate_sysenter_addr(vcpu, data); 2255 get_vmcs12(vcpu)->guest_sysenter_eip = data; 2256 } 2257 vmcs_writel(GUEST_SYSENTER_EIP, data); 2258 break; 2259 case MSR_IA32_SYSENTER_ESP: 2260 if (is_guest_mode(vcpu)) { 2261 data = nested_vmx_truncate_sysenter_addr(vcpu, data); 2262 get_vmcs12(vcpu)->guest_sysenter_esp = data; 2263 } 2264 vmcs_writel(GUEST_SYSENTER_ESP, data); 2265 break; 2266 case MSR_IA32_DEBUGCTLMSR: { 2267 u64 invalid; 2268 2269 invalid = data & ~vmx_get_supported_debugctl(vcpu, msr_info->host_initiated); 2270 if (invalid & (DEBUGCTLMSR_BTF|DEBUGCTLMSR_LBR)) { 2271 kvm_pr_unimpl_wrmsr(vcpu, msr_index, data); 2272 data &= ~(DEBUGCTLMSR_BTF|DEBUGCTLMSR_LBR); 2273 invalid &= ~(DEBUGCTLMSR_BTF|DEBUGCTLMSR_LBR); 2274 } 2275 2276 if (invalid) 2277 return 1; 2278 2279 if (is_guest_mode(vcpu) && get_vmcs12(vcpu)->vm_exit_controls & 2280 VM_EXIT_SAVE_DEBUG_CONTROLS) 2281 get_vmcs12(vcpu)->guest_ia32_debugctl = data; 2282 2283 vmcs_write64(GUEST_IA32_DEBUGCTL, data); 2284 if (intel_pmu_lbr_is_enabled(vcpu) && !to_vmx(vcpu)->lbr_desc.event && 2285 (data & DEBUGCTLMSR_LBR)) 2286 intel_pmu_create_guest_lbr_event(vcpu); 2287 return 0; 2288 } 2289 case MSR_IA32_BNDCFGS: 2290 if (!kvm_mpx_supported() || 2291 (!msr_info->host_initiated && 2292 !guest_cpu_cap_has(vcpu, X86_FEATURE_MPX))) 2293 return 1; 2294 if (is_noncanonical_msr_address(data & PAGE_MASK, vcpu) || 2295 (data & MSR_IA32_BNDCFGS_RSVD)) 2296 return 1; 2297 2298 if (is_guest_mode(vcpu) && 2299 ((vmx->nested.msrs.entry_ctls_high & VM_ENTRY_LOAD_BNDCFGS) || 2300 (vmx->nested.msrs.exit_ctls_high & VM_EXIT_CLEAR_BNDCFGS))) 2301 get_vmcs12(vcpu)->guest_bndcfgs = data; 2302 2303 vmcs_write64(GUEST_BNDCFGS, data); 2304 break; 2305 case MSR_IA32_UMWAIT_CONTROL: 2306 if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx)) 2307 return 1; 2308 2309 /* The reserved bit 1 and non-32 bit [63:32] should be zero */ 2310 if (data & (BIT_ULL(1) | GENMASK_ULL(63, 32))) 2311 return 1; 2312 2313 vmx->msr_ia32_umwait_control = data; 2314 break; 2315 case MSR_IA32_SPEC_CTRL: 2316 if (!msr_info->host_initiated && 2317 !guest_has_spec_ctrl_msr(vcpu)) 2318 return 1; 2319 2320 if (kvm_spec_ctrl_test_value(data)) 2321 return 1; 2322 2323 vmx->spec_ctrl = data; 2324 if (!data) 2325 break; 2326 2327 /* 2328 * For non-nested: 2329 * When it's written (to non-zero) for the first time, pass 2330 * it through. 2331 * 2332 * For nested: 2333 * The handling of the MSR bitmap for L2 guests is done in 2334 * nested_vmx_prepare_msr_bitmap. We should not touch the 2335 * vmcs02.msr_bitmap here since it gets completely overwritten 2336 * in the merging. We update the vmcs01 here for L1 as well 2337 * since it will end up touching the MSR anyway now. 2338 */ 2339 vmx_disable_intercept_for_msr(vcpu, 2340 MSR_IA32_SPEC_CTRL, 2341 MSR_TYPE_RW); 2342 break; 2343 case MSR_IA32_TSX_CTRL: 2344 if (!msr_info->host_initiated && 2345 !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR)) 2346 return 1; 2347 if (data & ~(TSX_CTRL_RTM_DISABLE | TSX_CTRL_CPUID_CLEAR)) 2348 return 1; 2349 goto find_uret_msr; 2350 case MSR_IA32_CR_PAT: 2351 ret = kvm_set_msr_common(vcpu, msr_info); 2352 if (ret) 2353 break; 2354 2355 if (is_guest_mode(vcpu) && 2356 get_vmcs12(vcpu)->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT) 2357 get_vmcs12(vcpu)->guest_ia32_pat = data; 2358 2359 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) 2360 vmcs_write64(GUEST_IA32_PAT, data); 2361 break; 2362 case MSR_IA32_MCG_EXT_CTL: 2363 if ((!msr_info->host_initiated && 2364 !(to_vmx(vcpu)->msr_ia32_feature_control & 2365 FEAT_CTL_LMCE_ENABLED)) || 2366 (data & ~MCG_EXT_CTL_LMCE_EN)) 2367 return 1; 2368 vcpu->arch.mcg_ext_ctl = data; 2369 break; 2370 case MSR_IA32_FEAT_CTL: 2371 if (!is_vmx_feature_control_msr_valid(vmx, msr_info)) 2372 return 1; 2373 2374 vmx->msr_ia32_feature_control = data; 2375 if (msr_info->host_initiated && data == 0) 2376 vmx_leave_nested(vcpu); 2377 2378 /* SGX may be enabled/disabled by guest's firmware */ 2379 vmx_write_encls_bitmap(vcpu, NULL); 2380 break; 2381 case MSR_IA32_SGXLEPUBKEYHASH0 ... MSR_IA32_SGXLEPUBKEYHASH3: 2382 /* 2383 * On real hardware, the LE hash MSRs are writable before 2384 * the firmware sets bit 0 in MSR 0x7a ("activating" SGX), 2385 * at which point SGX related bits in IA32_FEATURE_CONTROL 2386 * become writable. 2387 * 2388 * KVM does not emulate SGX activation for simplicity, so 2389 * allow writes to the LE hash MSRs if IA32_FEATURE_CONTROL 2390 * is unlocked. This is technically not architectural 2391 * behavior, but it's close enough. 2392 */ 2393 if (!msr_info->host_initiated && 2394 (!guest_cpu_cap_has(vcpu, X86_FEATURE_SGX_LC) || 2395 ((vmx->msr_ia32_feature_control & FEAT_CTL_LOCKED) && 2396 !(vmx->msr_ia32_feature_control & FEAT_CTL_SGX_LC_ENABLED)))) 2397 return 1; 2398 vmx->msr_ia32_sgxlepubkeyhash 2399 [msr_index - MSR_IA32_SGXLEPUBKEYHASH0] = data; 2400 break; 2401 case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR: 2402 if (!msr_info->host_initiated) 2403 return 1; /* they are read-only */ 2404 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_VMX)) 2405 return 1; 2406 return vmx_set_vmx_msr(vcpu, msr_index, data); 2407 case MSR_IA32_RTIT_CTL: 2408 if (!vmx_pt_mode_is_host_guest() || 2409 vmx_rtit_ctl_check(vcpu, data) || 2410 vmx->nested.vmxon) 2411 return 1; 2412 vmcs_write64(GUEST_IA32_RTIT_CTL, data); 2413 vmx->pt_desc.guest.ctl = data; 2414 pt_update_intercept_for_msr(vcpu); 2415 break; 2416 case MSR_IA32_RTIT_STATUS: 2417 if (!pt_can_write_msr(vmx)) 2418 return 1; 2419 if (data & MSR_IA32_RTIT_STATUS_MASK) 2420 return 1; 2421 vmx->pt_desc.guest.status = data; 2422 break; 2423 case MSR_IA32_RTIT_CR3_MATCH: 2424 if (!pt_can_write_msr(vmx)) 2425 return 1; 2426 if (!intel_pt_validate_cap(vmx->pt_desc.caps, 2427 PT_CAP_cr3_filtering)) 2428 return 1; 2429 vmx->pt_desc.guest.cr3_match = data; 2430 break; 2431 case MSR_IA32_RTIT_OUTPUT_BASE: 2432 if (!pt_can_write_msr(vmx)) 2433 return 1; 2434 if (!intel_pt_validate_cap(vmx->pt_desc.caps, 2435 PT_CAP_topa_output) && 2436 !intel_pt_validate_cap(vmx->pt_desc.caps, 2437 PT_CAP_single_range_output)) 2438 return 1; 2439 if (!pt_output_base_valid(vcpu, data)) 2440 return 1; 2441 vmx->pt_desc.guest.output_base = data; 2442 break; 2443 case MSR_IA32_RTIT_OUTPUT_MASK: 2444 if (!pt_can_write_msr(vmx)) 2445 return 1; 2446 if (!intel_pt_validate_cap(vmx->pt_desc.caps, 2447 PT_CAP_topa_output) && 2448 !intel_pt_validate_cap(vmx->pt_desc.caps, 2449 PT_CAP_single_range_output)) 2450 return 1; 2451 vmx->pt_desc.guest.output_mask = data; 2452 break; 2453 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: 2454 if (!pt_can_write_msr(vmx)) 2455 return 1; 2456 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A; 2457 if (index >= 2 * vmx->pt_desc.num_address_ranges) 2458 return 1; 2459 if (is_noncanonical_msr_address(data, vcpu)) 2460 return 1; 2461 if (index % 2) 2462 vmx->pt_desc.guest.addr_b[index / 2] = data; 2463 else 2464 vmx->pt_desc.guest.addr_a[index / 2] = data; 2465 break; 2466 case MSR_IA32_PERF_CAPABILITIES: 2467 if (data & PMU_CAP_LBR_FMT) { 2468 if ((data & PMU_CAP_LBR_FMT) != 2469 (kvm_caps.supported_perf_cap & PMU_CAP_LBR_FMT)) 2470 return 1; 2471 if (!cpuid_model_is_consistent(vcpu)) 2472 return 1; 2473 } 2474 if (data & PERF_CAP_PEBS_FORMAT) { 2475 if ((data & PERF_CAP_PEBS_MASK) != 2476 (kvm_caps.supported_perf_cap & PERF_CAP_PEBS_MASK)) 2477 return 1; 2478 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_DS)) 2479 return 1; 2480 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_DTES64)) 2481 return 1; 2482 if (!cpuid_model_is_consistent(vcpu)) 2483 return 1; 2484 } 2485 ret = kvm_set_msr_common(vcpu, msr_info); 2486 break; 2487 2488 default: 2489 find_uret_msr: 2490 msr = vmx_find_uret_msr(vmx, msr_index); 2491 if (msr) 2492 ret = vmx_set_guest_uret_msr(vmx, msr, data); 2493 else 2494 ret = kvm_set_msr_common(vcpu, msr_info); 2495 } 2496 2497 /* FB_CLEAR may have changed, also update the FB_CLEAR_DIS behavior */ 2498 if (msr_index == MSR_IA32_ARCH_CAPABILITIES) 2499 vmx_update_fb_clear_dis(vcpu, vmx); 2500 2501 return ret; 2502 } 2503 2504 void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) 2505 { 2506 unsigned long guest_owned_bits; 2507 2508 kvm_register_mark_available(vcpu, reg); 2509 2510 switch (reg) { 2511 case VCPU_REGS_RSP: 2512 vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP); 2513 break; 2514 case VCPU_REGS_RIP: 2515 vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP); 2516 break; 2517 case VCPU_EXREG_PDPTR: 2518 if (enable_ept) 2519 ept_save_pdptrs(vcpu); 2520 break; 2521 case VCPU_EXREG_CR0: 2522 guest_owned_bits = vcpu->arch.cr0_guest_owned_bits; 2523 2524 vcpu->arch.cr0 &= ~guest_owned_bits; 2525 vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & guest_owned_bits; 2526 break; 2527 case VCPU_EXREG_CR3: 2528 /* 2529 * When intercepting CR3 loads, e.g. for shadowing paging, KVM's 2530 * CR3 is loaded into hardware, not the guest's CR3. 2531 */ 2532 if (!(exec_controls_get(to_vmx(vcpu)) & CPU_BASED_CR3_LOAD_EXITING)) 2533 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); 2534 break; 2535 case VCPU_EXREG_CR4: 2536 guest_owned_bits = vcpu->arch.cr4_guest_owned_bits; 2537 2538 vcpu->arch.cr4 &= ~guest_owned_bits; 2539 vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & guest_owned_bits; 2540 break; 2541 default: 2542 KVM_BUG_ON(1, vcpu->kvm); 2543 break; 2544 } 2545 } 2546 2547 /* 2548 * There is no X86_FEATURE for SGX yet, but anyway we need to query CPUID 2549 * directly instead of going through cpu_has(), to ensure KVM is trapping 2550 * ENCLS whenever it's supported in hardware. It does not matter whether 2551 * the host OS supports or has enabled SGX. 2552 */ 2553 static bool cpu_has_sgx(void) 2554 { 2555 return cpuid_eax(0) >= 0x12 && (cpuid_eax(0x12) & BIT(0)); 2556 } 2557 2558 static int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt, u32 msr, u32 *result) 2559 { 2560 u32 vmx_msr_low, vmx_msr_high; 2561 u32 ctl = ctl_min | ctl_opt; 2562 2563 rdmsr(msr, vmx_msr_low, vmx_msr_high); 2564 2565 ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */ 2566 ctl |= vmx_msr_low; /* bit == 1 in low word ==> must be one */ 2567 2568 /* Ensure minimum (required) set of control bits are supported. */ 2569 if (ctl_min & ~ctl) 2570 return -EIO; 2571 2572 *result = ctl; 2573 return 0; 2574 } 2575 2576 static u64 adjust_vmx_controls64(u64 ctl_opt, u32 msr) 2577 { 2578 u64 allowed; 2579 2580 rdmsrl(msr, allowed); 2581 2582 return ctl_opt & allowed; 2583 } 2584 2585 static int setup_vmcs_config(struct vmcs_config *vmcs_conf, 2586 struct vmx_capability *vmx_cap) 2587 { 2588 u32 _pin_based_exec_control = 0; 2589 u32 _cpu_based_exec_control = 0; 2590 u32 _cpu_based_2nd_exec_control = 0; 2591 u64 _cpu_based_3rd_exec_control = 0; 2592 u32 _vmexit_control = 0; 2593 u32 _vmentry_control = 0; 2594 u64 basic_msr; 2595 u64 misc_msr; 2596 int i; 2597 2598 /* 2599 * LOAD/SAVE_DEBUG_CONTROLS are absent because both are mandatory. 2600 * SAVE_IA32_PAT and SAVE_IA32_EFER are absent because KVM always 2601 * intercepts writes to PAT and EFER, i.e. never enables those controls. 2602 */ 2603 struct { 2604 u32 entry_control; 2605 u32 exit_control; 2606 } const vmcs_entry_exit_pairs[] = { 2607 { VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL, VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL }, 2608 { VM_ENTRY_LOAD_IA32_PAT, VM_EXIT_LOAD_IA32_PAT }, 2609 { VM_ENTRY_LOAD_IA32_EFER, VM_EXIT_LOAD_IA32_EFER }, 2610 { VM_ENTRY_LOAD_BNDCFGS, VM_EXIT_CLEAR_BNDCFGS }, 2611 { VM_ENTRY_LOAD_IA32_RTIT_CTL, VM_EXIT_CLEAR_IA32_RTIT_CTL }, 2612 }; 2613 2614 memset(vmcs_conf, 0, sizeof(*vmcs_conf)); 2615 2616 if (adjust_vmx_controls(KVM_REQUIRED_VMX_CPU_BASED_VM_EXEC_CONTROL, 2617 KVM_OPTIONAL_VMX_CPU_BASED_VM_EXEC_CONTROL, 2618 MSR_IA32_VMX_PROCBASED_CTLS, 2619 &_cpu_based_exec_control)) 2620 return -EIO; 2621 if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) { 2622 if (adjust_vmx_controls(KVM_REQUIRED_VMX_SECONDARY_VM_EXEC_CONTROL, 2623 KVM_OPTIONAL_VMX_SECONDARY_VM_EXEC_CONTROL, 2624 MSR_IA32_VMX_PROCBASED_CTLS2, 2625 &_cpu_based_2nd_exec_control)) 2626 return -EIO; 2627 } 2628 if (!IS_ENABLED(CONFIG_KVM_INTEL_PROVE_VE)) 2629 _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_EPT_VIOLATION_VE; 2630 2631 #ifndef CONFIG_X86_64 2632 if (!(_cpu_based_2nd_exec_control & 2633 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) 2634 _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW; 2635 #endif 2636 2637 if (!(_cpu_based_exec_control & CPU_BASED_TPR_SHADOW)) 2638 _cpu_based_2nd_exec_control &= ~( 2639 SECONDARY_EXEC_APIC_REGISTER_VIRT | 2640 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 2641 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 2642 2643 rdmsr_safe(MSR_IA32_VMX_EPT_VPID_CAP, 2644 &vmx_cap->ept, &vmx_cap->vpid); 2645 2646 if (!(_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) && 2647 vmx_cap->ept) { 2648 pr_warn_once("EPT CAP should not exist if not support " 2649 "1-setting enable EPT VM-execution control\n"); 2650 2651 if (error_on_inconsistent_vmcs_config) 2652 return -EIO; 2653 2654 vmx_cap->ept = 0; 2655 _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_EPT_VIOLATION_VE; 2656 } 2657 if (!(_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_VPID) && 2658 vmx_cap->vpid) { 2659 pr_warn_once("VPID CAP should not exist if not support " 2660 "1-setting enable VPID VM-execution control\n"); 2661 2662 if (error_on_inconsistent_vmcs_config) 2663 return -EIO; 2664 2665 vmx_cap->vpid = 0; 2666 } 2667 2668 if (!cpu_has_sgx()) 2669 _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_ENCLS_EXITING; 2670 2671 if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_TERTIARY_CONTROLS) 2672 _cpu_based_3rd_exec_control = 2673 adjust_vmx_controls64(KVM_OPTIONAL_VMX_TERTIARY_VM_EXEC_CONTROL, 2674 MSR_IA32_VMX_PROCBASED_CTLS3); 2675 2676 if (adjust_vmx_controls(KVM_REQUIRED_VMX_VM_EXIT_CONTROLS, 2677 KVM_OPTIONAL_VMX_VM_EXIT_CONTROLS, 2678 MSR_IA32_VMX_EXIT_CTLS, 2679 &_vmexit_control)) 2680 return -EIO; 2681 2682 if (adjust_vmx_controls(KVM_REQUIRED_VMX_PIN_BASED_VM_EXEC_CONTROL, 2683 KVM_OPTIONAL_VMX_PIN_BASED_VM_EXEC_CONTROL, 2684 MSR_IA32_VMX_PINBASED_CTLS, 2685 &_pin_based_exec_control)) 2686 return -EIO; 2687 2688 if (cpu_has_broken_vmx_preemption_timer()) 2689 _pin_based_exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER; 2690 if (!(_cpu_based_2nd_exec_control & 2691 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)) 2692 _pin_based_exec_control &= ~PIN_BASED_POSTED_INTR; 2693 2694 if (adjust_vmx_controls(KVM_REQUIRED_VMX_VM_ENTRY_CONTROLS, 2695 KVM_OPTIONAL_VMX_VM_ENTRY_CONTROLS, 2696 MSR_IA32_VMX_ENTRY_CTLS, 2697 &_vmentry_control)) 2698 return -EIO; 2699 2700 for (i = 0; i < ARRAY_SIZE(vmcs_entry_exit_pairs); i++) { 2701 u32 n_ctrl = vmcs_entry_exit_pairs[i].entry_control; 2702 u32 x_ctrl = vmcs_entry_exit_pairs[i].exit_control; 2703 2704 if (!(_vmentry_control & n_ctrl) == !(_vmexit_control & x_ctrl)) 2705 continue; 2706 2707 pr_warn_once("Inconsistent VM-Entry/VM-Exit pair, entry = %x, exit = %x\n", 2708 _vmentry_control & n_ctrl, _vmexit_control & x_ctrl); 2709 2710 if (error_on_inconsistent_vmcs_config) 2711 return -EIO; 2712 2713 _vmentry_control &= ~n_ctrl; 2714 _vmexit_control &= ~x_ctrl; 2715 } 2716 2717 /* 2718 * Some cpus support VM_{ENTRY,EXIT}_IA32_PERF_GLOBAL_CTRL but they 2719 * can't be used due to an errata where VM Exit may incorrectly clear 2720 * IA32_PERF_GLOBAL_CTRL[34:32]. Workaround the errata by using the 2721 * MSR load mechanism to switch IA32_PERF_GLOBAL_CTRL. 2722 */ 2723 switch (boot_cpu_data.x86_vfm) { 2724 case INTEL_NEHALEM_EP: /* AAK155 */ 2725 case INTEL_NEHALEM: /* AAP115 */ 2726 case INTEL_WESTMERE: /* AAT100 */ 2727 case INTEL_WESTMERE_EP: /* BC86,AAY89,BD102 */ 2728 case INTEL_NEHALEM_EX: /* BA97 */ 2729 _vmentry_control &= ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL; 2730 _vmexit_control &= ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; 2731 pr_warn_once("VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL " 2732 "does not work properly. Using workaround\n"); 2733 break; 2734 default: 2735 break; 2736 } 2737 2738 rdmsrl(MSR_IA32_VMX_BASIC, basic_msr); 2739 2740 /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */ 2741 if (vmx_basic_vmcs_size(basic_msr) > PAGE_SIZE) 2742 return -EIO; 2743 2744 #ifdef CONFIG_X86_64 2745 /* 2746 * KVM expects to be able to shove all legal physical addresses into 2747 * VMCS fields for 64-bit kernels, and per the SDM, "This bit is always 2748 * 0 for processors that support Intel 64 architecture". 2749 */ 2750 if (basic_msr & VMX_BASIC_32BIT_PHYS_ADDR_ONLY) 2751 return -EIO; 2752 #endif 2753 2754 /* Require Write-Back (WB) memory type for VMCS accesses. */ 2755 if (vmx_basic_vmcs_mem_type(basic_msr) != X86_MEMTYPE_WB) 2756 return -EIO; 2757 2758 rdmsrl(MSR_IA32_VMX_MISC, misc_msr); 2759 2760 vmcs_conf->basic = basic_msr; 2761 vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control; 2762 vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control; 2763 vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control; 2764 vmcs_conf->cpu_based_3rd_exec_ctrl = _cpu_based_3rd_exec_control; 2765 vmcs_conf->vmexit_ctrl = _vmexit_control; 2766 vmcs_conf->vmentry_ctrl = _vmentry_control; 2767 vmcs_conf->misc = misc_msr; 2768 2769 #if IS_ENABLED(CONFIG_HYPERV) 2770 if (enlightened_vmcs) 2771 evmcs_sanitize_exec_ctrls(vmcs_conf); 2772 #endif 2773 2774 return 0; 2775 } 2776 2777 static bool __kvm_is_vmx_supported(void) 2778 { 2779 int cpu = smp_processor_id(); 2780 2781 if (!(cpuid_ecx(1) & feature_bit(VMX))) { 2782 pr_err("VMX not supported by CPU %d\n", cpu); 2783 return false; 2784 } 2785 2786 if (!this_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) || 2787 !this_cpu_has(X86_FEATURE_VMX)) { 2788 pr_err("VMX not enabled (by BIOS) in MSR_IA32_FEAT_CTL on CPU %d\n", cpu); 2789 return false; 2790 } 2791 2792 return true; 2793 } 2794 2795 static bool kvm_is_vmx_supported(void) 2796 { 2797 bool supported; 2798 2799 migrate_disable(); 2800 supported = __kvm_is_vmx_supported(); 2801 migrate_enable(); 2802 2803 return supported; 2804 } 2805 2806 int vmx_check_processor_compat(void) 2807 { 2808 int cpu = raw_smp_processor_id(); 2809 struct vmcs_config vmcs_conf; 2810 struct vmx_capability vmx_cap; 2811 2812 if (!__kvm_is_vmx_supported()) 2813 return -EIO; 2814 2815 if (setup_vmcs_config(&vmcs_conf, &vmx_cap) < 0) { 2816 pr_err("Failed to setup VMCS config on CPU %d\n", cpu); 2817 return -EIO; 2818 } 2819 if (nested) 2820 nested_vmx_setup_ctls_msrs(&vmcs_conf, vmx_cap.ept); 2821 if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config))) { 2822 pr_err("Inconsistent VMCS config on CPU %d\n", cpu); 2823 return -EIO; 2824 } 2825 return 0; 2826 } 2827 2828 static int kvm_cpu_vmxon(u64 vmxon_pointer) 2829 { 2830 u64 msr; 2831 2832 cr4_set_bits(X86_CR4_VMXE); 2833 2834 asm goto("1: vmxon %[vmxon_pointer]\n\t" 2835 _ASM_EXTABLE(1b, %l[fault]) 2836 : : [vmxon_pointer] "m"(vmxon_pointer) 2837 : : fault); 2838 return 0; 2839 2840 fault: 2841 WARN_ONCE(1, "VMXON faulted, MSR_IA32_FEAT_CTL (0x3a) = 0x%llx\n", 2842 rdmsrl_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr); 2843 cr4_clear_bits(X86_CR4_VMXE); 2844 2845 return -EFAULT; 2846 } 2847 2848 int vmx_enable_virtualization_cpu(void) 2849 { 2850 int cpu = raw_smp_processor_id(); 2851 u64 phys_addr = __pa(per_cpu(vmxarea, cpu)); 2852 int r; 2853 2854 if (cr4_read_shadow() & X86_CR4_VMXE) 2855 return -EBUSY; 2856 2857 /* 2858 * This can happen if we hot-added a CPU but failed to allocate 2859 * VP assist page for it. 2860 */ 2861 if (kvm_is_using_evmcs() && !hv_get_vp_assist_page(cpu)) 2862 return -EFAULT; 2863 2864 intel_pt_handle_vmx(1); 2865 2866 r = kvm_cpu_vmxon(phys_addr); 2867 if (r) { 2868 intel_pt_handle_vmx(0); 2869 return r; 2870 } 2871 2872 return 0; 2873 } 2874 2875 static void vmclear_local_loaded_vmcss(void) 2876 { 2877 int cpu = raw_smp_processor_id(); 2878 struct loaded_vmcs *v, *n; 2879 2880 list_for_each_entry_safe(v, n, &per_cpu(loaded_vmcss_on_cpu, cpu), 2881 loaded_vmcss_on_cpu_link) 2882 __loaded_vmcs_clear(v); 2883 } 2884 2885 void vmx_disable_virtualization_cpu(void) 2886 { 2887 vmclear_local_loaded_vmcss(); 2888 2889 if (kvm_cpu_vmxoff()) 2890 kvm_spurious_fault(); 2891 2892 hv_reset_evmcs(); 2893 2894 intel_pt_handle_vmx(0); 2895 } 2896 2897 struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags) 2898 { 2899 int node = cpu_to_node(cpu); 2900 struct page *pages; 2901 struct vmcs *vmcs; 2902 2903 pages = __alloc_pages_node(node, flags, 0); 2904 if (!pages) 2905 return NULL; 2906 vmcs = page_address(pages); 2907 memset(vmcs, 0, vmx_basic_vmcs_size(vmcs_config.basic)); 2908 2909 /* KVM supports Enlightened VMCS v1 only */ 2910 if (kvm_is_using_evmcs()) 2911 vmcs->hdr.revision_id = KVM_EVMCS_VERSION; 2912 else 2913 vmcs->hdr.revision_id = vmx_basic_vmcs_revision_id(vmcs_config.basic); 2914 2915 if (shadow) 2916 vmcs->hdr.shadow_vmcs = 1; 2917 return vmcs; 2918 } 2919 2920 void free_vmcs(struct vmcs *vmcs) 2921 { 2922 free_page((unsigned long)vmcs); 2923 } 2924 2925 /* 2926 * Free a VMCS, but before that VMCLEAR it on the CPU where it was last loaded 2927 */ 2928 void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs) 2929 { 2930 if (!loaded_vmcs->vmcs) 2931 return; 2932 loaded_vmcs_clear(loaded_vmcs); 2933 free_vmcs(loaded_vmcs->vmcs); 2934 loaded_vmcs->vmcs = NULL; 2935 if (loaded_vmcs->msr_bitmap) 2936 free_page((unsigned long)loaded_vmcs->msr_bitmap); 2937 WARN_ON(loaded_vmcs->shadow_vmcs != NULL); 2938 } 2939 2940 int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs) 2941 { 2942 loaded_vmcs->vmcs = alloc_vmcs(false); 2943 if (!loaded_vmcs->vmcs) 2944 return -ENOMEM; 2945 2946 vmcs_clear(loaded_vmcs->vmcs); 2947 2948 loaded_vmcs->shadow_vmcs = NULL; 2949 loaded_vmcs->hv_timer_soft_disabled = false; 2950 loaded_vmcs->cpu = -1; 2951 loaded_vmcs->launched = 0; 2952 2953 if (cpu_has_vmx_msr_bitmap()) { 2954 loaded_vmcs->msr_bitmap = (unsigned long *) 2955 __get_free_page(GFP_KERNEL_ACCOUNT); 2956 if (!loaded_vmcs->msr_bitmap) 2957 goto out_vmcs; 2958 memset(loaded_vmcs->msr_bitmap, 0xff, PAGE_SIZE); 2959 } 2960 2961 memset(&loaded_vmcs->host_state, 0, sizeof(struct vmcs_host_state)); 2962 memset(&loaded_vmcs->controls_shadow, 0, 2963 sizeof(struct vmcs_controls_shadow)); 2964 2965 return 0; 2966 2967 out_vmcs: 2968 free_loaded_vmcs(loaded_vmcs); 2969 return -ENOMEM; 2970 } 2971 2972 static void free_kvm_area(void) 2973 { 2974 int cpu; 2975 2976 for_each_possible_cpu(cpu) { 2977 free_vmcs(per_cpu(vmxarea, cpu)); 2978 per_cpu(vmxarea, cpu) = NULL; 2979 } 2980 } 2981 2982 static __init int alloc_kvm_area(void) 2983 { 2984 int cpu; 2985 2986 for_each_possible_cpu(cpu) { 2987 struct vmcs *vmcs; 2988 2989 vmcs = alloc_vmcs_cpu(false, cpu, GFP_KERNEL); 2990 if (!vmcs) { 2991 free_kvm_area(); 2992 return -ENOMEM; 2993 } 2994 2995 /* 2996 * When eVMCS is enabled, alloc_vmcs_cpu() sets 2997 * vmcs->revision_id to KVM_EVMCS_VERSION instead of 2998 * revision_id reported by MSR_IA32_VMX_BASIC. 2999 * 3000 * However, even though not explicitly documented by 3001 * TLFS, VMXArea passed as VMXON argument should 3002 * still be marked with revision_id reported by 3003 * physical CPU. 3004 */ 3005 if (kvm_is_using_evmcs()) 3006 vmcs->hdr.revision_id = vmx_basic_vmcs_revision_id(vmcs_config.basic); 3007 3008 per_cpu(vmxarea, cpu) = vmcs; 3009 } 3010 return 0; 3011 } 3012 3013 static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg, 3014 struct kvm_segment *save) 3015 { 3016 if (!emulate_invalid_guest_state) { 3017 /* 3018 * CS and SS RPL should be equal during guest entry according 3019 * to VMX spec, but in reality it is not always so. Since vcpu 3020 * is in the middle of the transition from real mode to 3021 * protected mode it is safe to assume that RPL 0 is a good 3022 * default value. 3023 */ 3024 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS) 3025 save->selector &= ~SEGMENT_RPL_MASK; 3026 save->dpl = save->selector & SEGMENT_RPL_MASK; 3027 save->s = 1; 3028 } 3029 __vmx_set_segment(vcpu, save, seg); 3030 } 3031 3032 static void enter_pmode(struct kvm_vcpu *vcpu) 3033 { 3034 unsigned long flags; 3035 struct vcpu_vmx *vmx = to_vmx(vcpu); 3036 3037 /* 3038 * Update real mode segment cache. It may be not up-to-date if segment 3039 * register was written while vcpu was in a guest mode. 3040 */ 3041 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES); 3042 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS); 3043 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS); 3044 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS); 3045 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS); 3046 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS); 3047 3048 vmx->rmode.vm86_active = 0; 3049 3050 __vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR); 3051 3052 flags = vmcs_readl(GUEST_RFLAGS); 3053 flags &= RMODE_GUEST_OWNED_EFLAGS_BITS; 3054 flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS; 3055 vmcs_writel(GUEST_RFLAGS, flags); 3056 3057 vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) | 3058 (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME)); 3059 3060 vmx_update_exception_bitmap(vcpu); 3061 3062 fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]); 3063 fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]); 3064 fix_pmode_seg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]); 3065 fix_pmode_seg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]); 3066 fix_pmode_seg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]); 3067 fix_pmode_seg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]); 3068 } 3069 3070 static void fix_rmode_seg(int seg, struct kvm_segment *save) 3071 { 3072 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 3073 struct kvm_segment var = *save; 3074 3075 var.dpl = 0x3; 3076 if (seg == VCPU_SREG_CS) 3077 var.type = 0x3; 3078 3079 if (!emulate_invalid_guest_state) { 3080 var.selector = var.base >> 4; 3081 var.base = var.base & 0xffff0; 3082 var.limit = 0xffff; 3083 var.g = 0; 3084 var.db = 0; 3085 var.present = 1; 3086 var.s = 1; 3087 var.l = 0; 3088 var.unusable = 0; 3089 var.type = 0x3; 3090 var.avl = 0; 3091 if (save->base & 0xf) 3092 pr_warn_once("segment base is not paragraph aligned " 3093 "when entering protected mode (seg=%d)", seg); 3094 } 3095 3096 vmcs_write16(sf->selector, var.selector); 3097 vmcs_writel(sf->base, var.base); 3098 vmcs_write32(sf->limit, var.limit); 3099 vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(&var)); 3100 } 3101 3102 static void enter_rmode(struct kvm_vcpu *vcpu) 3103 { 3104 unsigned long flags; 3105 struct vcpu_vmx *vmx = to_vmx(vcpu); 3106 struct kvm_vmx *kvm_vmx = to_kvm_vmx(vcpu->kvm); 3107 3108 /* 3109 * KVM should never use VM86 to virtualize Real Mode when L2 is active, 3110 * as using VM86 is unnecessary if unrestricted guest is enabled, and 3111 * if unrestricted guest is disabled, VM-Enter (from L1) with CR0.PG=0 3112 * should VM-Fail and KVM should reject userspace attempts to stuff 3113 * CR0.PG=0 when L2 is active. 3114 */ 3115 WARN_ON_ONCE(is_guest_mode(vcpu)); 3116 3117 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR); 3118 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES); 3119 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS); 3120 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS); 3121 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS); 3122 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS); 3123 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS); 3124 3125 vmx->rmode.vm86_active = 1; 3126 3127 vmx_segment_cache_clear(vmx); 3128 3129 vmcs_writel(GUEST_TR_BASE, kvm_vmx->tss_addr); 3130 vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1); 3131 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); 3132 3133 flags = vmcs_readl(GUEST_RFLAGS); 3134 vmx->rmode.save_rflags = flags; 3135 3136 flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM; 3137 3138 vmcs_writel(GUEST_RFLAGS, flags); 3139 vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME); 3140 vmx_update_exception_bitmap(vcpu); 3141 3142 fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]); 3143 fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]); 3144 fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]); 3145 fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]); 3146 fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]); 3147 fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]); 3148 } 3149 3150 int vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer) 3151 { 3152 struct vcpu_vmx *vmx = to_vmx(vcpu); 3153 3154 /* Nothing to do if hardware doesn't support EFER. */ 3155 if (!vmx_find_uret_msr(vmx, MSR_EFER)) 3156 return 0; 3157 3158 vcpu->arch.efer = efer; 3159 #ifdef CONFIG_X86_64 3160 if (efer & EFER_LMA) 3161 vm_entry_controls_setbit(vmx, VM_ENTRY_IA32E_MODE); 3162 else 3163 vm_entry_controls_clearbit(vmx, VM_ENTRY_IA32E_MODE); 3164 #else 3165 if (KVM_BUG_ON(efer & EFER_LMA, vcpu->kvm)) 3166 return 1; 3167 #endif 3168 3169 vmx_setup_uret_msrs(vmx); 3170 return 0; 3171 } 3172 3173 #ifdef CONFIG_X86_64 3174 3175 static void enter_lmode(struct kvm_vcpu *vcpu) 3176 { 3177 u32 guest_tr_ar; 3178 3179 vmx_segment_cache_clear(to_vmx(vcpu)); 3180 3181 guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES); 3182 if ((guest_tr_ar & VMX_AR_TYPE_MASK) != VMX_AR_TYPE_BUSY_64_TSS) { 3183 pr_debug_ratelimited("%s: tss fixup for long mode. \n", 3184 __func__); 3185 vmcs_write32(GUEST_TR_AR_BYTES, 3186 (guest_tr_ar & ~VMX_AR_TYPE_MASK) 3187 | VMX_AR_TYPE_BUSY_64_TSS); 3188 } 3189 vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA); 3190 } 3191 3192 static void exit_lmode(struct kvm_vcpu *vcpu) 3193 { 3194 vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA); 3195 } 3196 3197 #endif 3198 3199 void vmx_flush_tlb_all(struct kvm_vcpu *vcpu) 3200 { 3201 struct vcpu_vmx *vmx = to_vmx(vcpu); 3202 3203 /* 3204 * INVEPT must be issued when EPT is enabled, irrespective of VPID, as 3205 * the CPU is not required to invalidate guest-physical mappings on 3206 * VM-Entry, even if VPID is disabled. Guest-physical mappings are 3207 * associated with the root EPT structure and not any particular VPID 3208 * (INVVPID also isn't required to invalidate guest-physical mappings). 3209 */ 3210 if (enable_ept) { 3211 ept_sync_global(); 3212 } else if (enable_vpid) { 3213 if (cpu_has_vmx_invvpid_global()) { 3214 vpid_sync_vcpu_global(); 3215 } else { 3216 vpid_sync_vcpu_single(vmx->vpid); 3217 vpid_sync_vcpu_single(vmx->nested.vpid02); 3218 } 3219 } 3220 } 3221 3222 static inline int vmx_get_current_vpid(struct kvm_vcpu *vcpu) 3223 { 3224 if (is_guest_mode(vcpu) && nested_cpu_has_vpid(get_vmcs12(vcpu))) 3225 return nested_get_vpid02(vcpu); 3226 return to_vmx(vcpu)->vpid; 3227 } 3228 3229 void vmx_flush_tlb_current(struct kvm_vcpu *vcpu) 3230 { 3231 struct kvm_mmu *mmu = vcpu->arch.mmu; 3232 u64 root_hpa = mmu->root.hpa; 3233 3234 /* No flush required if the current context is invalid. */ 3235 if (!VALID_PAGE(root_hpa)) 3236 return; 3237 3238 if (enable_ept) 3239 ept_sync_context(construct_eptp(vcpu, root_hpa, 3240 mmu->root_role.level)); 3241 else 3242 vpid_sync_context(vmx_get_current_vpid(vcpu)); 3243 } 3244 3245 void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) 3246 { 3247 /* 3248 * vpid_sync_vcpu_addr() is a nop if vpid==0, see the comment in 3249 * vmx_flush_tlb_guest() for an explanation of why this is ok. 3250 */ 3251 vpid_sync_vcpu_addr(vmx_get_current_vpid(vcpu), addr); 3252 } 3253 3254 void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu) 3255 { 3256 /* 3257 * vpid_sync_context() is a nop if vpid==0, e.g. if enable_vpid==0 or a 3258 * vpid couldn't be allocated for this vCPU. VM-Enter and VM-Exit are 3259 * required to flush GVA->{G,H}PA mappings from the TLB if vpid is 3260 * disabled (VM-Enter with vpid enabled and vpid==0 is disallowed), 3261 * i.e. no explicit INVVPID is necessary. 3262 */ 3263 vpid_sync_context(vmx_get_current_vpid(vcpu)); 3264 } 3265 3266 void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu) 3267 { 3268 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 3269 3270 if (!kvm_register_is_dirty(vcpu, VCPU_EXREG_PDPTR)) 3271 return; 3272 3273 if (is_pae_paging(vcpu)) { 3274 vmcs_write64(GUEST_PDPTR0, mmu->pdptrs[0]); 3275 vmcs_write64(GUEST_PDPTR1, mmu->pdptrs[1]); 3276 vmcs_write64(GUEST_PDPTR2, mmu->pdptrs[2]); 3277 vmcs_write64(GUEST_PDPTR3, mmu->pdptrs[3]); 3278 } 3279 } 3280 3281 void ept_save_pdptrs(struct kvm_vcpu *vcpu) 3282 { 3283 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 3284 3285 if (WARN_ON_ONCE(!is_pae_paging(vcpu))) 3286 return; 3287 3288 mmu->pdptrs[0] = vmcs_read64(GUEST_PDPTR0); 3289 mmu->pdptrs[1] = vmcs_read64(GUEST_PDPTR1); 3290 mmu->pdptrs[2] = vmcs_read64(GUEST_PDPTR2); 3291 mmu->pdptrs[3] = vmcs_read64(GUEST_PDPTR3); 3292 3293 kvm_register_mark_available(vcpu, VCPU_EXREG_PDPTR); 3294 } 3295 3296 #define CR3_EXITING_BITS (CPU_BASED_CR3_LOAD_EXITING | \ 3297 CPU_BASED_CR3_STORE_EXITING) 3298 3299 bool vmx_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 3300 { 3301 if (is_guest_mode(vcpu)) 3302 return nested_guest_cr0_valid(vcpu, cr0); 3303 3304 if (to_vmx(vcpu)->nested.vmxon) 3305 return nested_host_cr0_valid(vcpu, cr0); 3306 3307 return true; 3308 } 3309 3310 void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 3311 { 3312 struct vcpu_vmx *vmx = to_vmx(vcpu); 3313 unsigned long hw_cr0, old_cr0_pg; 3314 u32 tmp; 3315 3316 old_cr0_pg = kvm_read_cr0_bits(vcpu, X86_CR0_PG); 3317 3318 hw_cr0 = (cr0 & ~KVM_VM_CR0_ALWAYS_OFF); 3319 if (enable_unrestricted_guest) 3320 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST; 3321 else { 3322 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON; 3323 if (!enable_ept) 3324 hw_cr0 |= X86_CR0_WP; 3325 3326 if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE)) 3327 enter_pmode(vcpu); 3328 3329 if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE)) 3330 enter_rmode(vcpu); 3331 } 3332 3333 vmcs_writel(CR0_READ_SHADOW, cr0); 3334 vmcs_writel(GUEST_CR0, hw_cr0); 3335 vcpu->arch.cr0 = cr0; 3336 kvm_register_mark_available(vcpu, VCPU_EXREG_CR0); 3337 3338 #ifdef CONFIG_X86_64 3339 if (vcpu->arch.efer & EFER_LME) { 3340 if (!old_cr0_pg && (cr0 & X86_CR0_PG)) 3341 enter_lmode(vcpu); 3342 else if (old_cr0_pg && !(cr0 & X86_CR0_PG)) 3343 exit_lmode(vcpu); 3344 } 3345 #endif 3346 3347 if (enable_ept && !enable_unrestricted_guest) { 3348 /* 3349 * Ensure KVM has an up-to-date snapshot of the guest's CR3. If 3350 * the below code _enables_ CR3 exiting, vmx_cache_reg() will 3351 * (correctly) stop reading vmcs.GUEST_CR3 because it thinks 3352 * KVM's CR3 is installed. 3353 */ 3354 if (!kvm_register_is_available(vcpu, VCPU_EXREG_CR3)) 3355 vmx_cache_reg(vcpu, VCPU_EXREG_CR3); 3356 3357 /* 3358 * When running with EPT but not unrestricted guest, KVM must 3359 * intercept CR3 accesses when paging is _disabled_. This is 3360 * necessary because restricted guests can't actually run with 3361 * paging disabled, and so KVM stuffs its own CR3 in order to 3362 * run the guest when identity mapped page tables. 3363 * 3364 * Do _NOT_ check the old CR0.PG, e.g. to optimize away the 3365 * update, it may be stale with respect to CR3 interception, 3366 * e.g. after nested VM-Enter. 3367 * 3368 * Lastly, honor L1's desires, i.e. intercept CR3 loads and/or 3369 * stores to forward them to L1, even if KVM does not need to 3370 * intercept them to preserve its identity mapped page tables. 3371 */ 3372 if (!(cr0 & X86_CR0_PG)) { 3373 exec_controls_setbit(vmx, CR3_EXITING_BITS); 3374 } else if (!is_guest_mode(vcpu)) { 3375 exec_controls_clearbit(vmx, CR3_EXITING_BITS); 3376 } else { 3377 tmp = exec_controls_get(vmx); 3378 tmp &= ~CR3_EXITING_BITS; 3379 tmp |= get_vmcs12(vcpu)->cpu_based_vm_exec_control & CR3_EXITING_BITS; 3380 exec_controls_set(vmx, tmp); 3381 } 3382 3383 /* Note, vmx_set_cr4() consumes the new vcpu->arch.cr0. */ 3384 if ((old_cr0_pg ^ cr0) & X86_CR0_PG) 3385 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu)); 3386 3387 /* 3388 * When !CR0_PG -> CR0_PG, vcpu->arch.cr3 becomes active, but 3389 * GUEST_CR3 is still vmx->ept_identity_map_addr if EPT + !URG. 3390 */ 3391 if (!(old_cr0_pg & X86_CR0_PG) && (cr0 & X86_CR0_PG)) 3392 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); 3393 } 3394 3395 /* depends on vcpu->arch.cr0 to be set to a new value */ 3396 vmx->emulation_required = vmx_emulation_required(vcpu); 3397 } 3398 3399 static int vmx_get_max_ept_level(void) 3400 { 3401 if (cpu_has_vmx_ept_5levels()) 3402 return 5; 3403 return 4; 3404 } 3405 3406 u64 construct_eptp(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level) 3407 { 3408 u64 eptp = VMX_EPTP_MT_WB; 3409 3410 eptp |= (root_level == 5) ? VMX_EPTP_PWL_5 : VMX_EPTP_PWL_4; 3411 3412 if (enable_ept_ad_bits && 3413 (!is_guest_mode(vcpu) || nested_ept_ad_enabled(vcpu))) 3414 eptp |= VMX_EPTP_AD_ENABLE_BIT; 3415 eptp |= root_hpa; 3416 3417 return eptp; 3418 } 3419 3420 void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level) 3421 { 3422 struct kvm *kvm = vcpu->kvm; 3423 bool update_guest_cr3 = true; 3424 unsigned long guest_cr3; 3425 u64 eptp; 3426 3427 if (enable_ept) { 3428 eptp = construct_eptp(vcpu, root_hpa, root_level); 3429 vmcs_write64(EPT_POINTER, eptp); 3430 3431 hv_track_root_tdp(vcpu, root_hpa); 3432 3433 if (!enable_unrestricted_guest && !is_paging(vcpu)) 3434 guest_cr3 = to_kvm_vmx(kvm)->ept_identity_map_addr; 3435 else if (kvm_register_is_dirty(vcpu, VCPU_EXREG_CR3)) 3436 guest_cr3 = vcpu->arch.cr3; 3437 else /* vmcs.GUEST_CR3 is already up-to-date. */ 3438 update_guest_cr3 = false; 3439 vmx_ept_load_pdptrs(vcpu); 3440 } else { 3441 guest_cr3 = root_hpa | kvm_get_active_pcid(vcpu) | 3442 kvm_get_active_cr3_lam_bits(vcpu); 3443 } 3444 3445 if (update_guest_cr3) 3446 vmcs_writel(GUEST_CR3, guest_cr3); 3447 } 3448 3449 bool vmx_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 3450 { 3451 /* 3452 * We operate under the default treatment of SMM, so VMX cannot be 3453 * enabled under SMM. Note, whether or not VMXE is allowed at all, 3454 * i.e. is a reserved bit, is handled by common x86 code. 3455 */ 3456 if ((cr4 & X86_CR4_VMXE) && is_smm(vcpu)) 3457 return false; 3458 3459 if (to_vmx(vcpu)->nested.vmxon && !nested_cr4_valid(vcpu, cr4)) 3460 return false; 3461 3462 return true; 3463 } 3464 3465 void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 3466 { 3467 unsigned long old_cr4 = kvm_read_cr4(vcpu); 3468 struct vcpu_vmx *vmx = to_vmx(vcpu); 3469 unsigned long hw_cr4; 3470 3471 /* 3472 * Pass through host's Machine Check Enable value to hw_cr4, which 3473 * is in force while we are in guest mode. Do not let guests control 3474 * this bit, even if host CR4.MCE == 0. 3475 */ 3476 hw_cr4 = (cr4_read_shadow() & X86_CR4_MCE) | (cr4 & ~X86_CR4_MCE); 3477 if (enable_unrestricted_guest) 3478 hw_cr4 |= KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST; 3479 else if (vmx->rmode.vm86_active) 3480 hw_cr4 |= KVM_RMODE_VM_CR4_ALWAYS_ON; 3481 else 3482 hw_cr4 |= KVM_PMODE_VM_CR4_ALWAYS_ON; 3483 3484 if (vmx_umip_emulated()) { 3485 if (cr4 & X86_CR4_UMIP) { 3486 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_DESC); 3487 hw_cr4 &= ~X86_CR4_UMIP; 3488 } else if (!is_guest_mode(vcpu) || 3489 !nested_cpu_has2(get_vmcs12(vcpu), SECONDARY_EXEC_DESC)) { 3490 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_DESC); 3491 } 3492 } 3493 3494 vcpu->arch.cr4 = cr4; 3495 kvm_register_mark_available(vcpu, VCPU_EXREG_CR4); 3496 3497 if (!enable_unrestricted_guest) { 3498 if (enable_ept) { 3499 if (!is_paging(vcpu)) { 3500 hw_cr4 &= ~X86_CR4_PAE; 3501 hw_cr4 |= X86_CR4_PSE; 3502 } else if (!(cr4 & X86_CR4_PAE)) { 3503 hw_cr4 &= ~X86_CR4_PAE; 3504 } 3505 } 3506 3507 /* 3508 * SMEP/SMAP/PKU is disabled if CPU is in non-paging mode in 3509 * hardware. To emulate this behavior, SMEP/SMAP/PKU needs 3510 * to be manually disabled when guest switches to non-paging 3511 * mode. 3512 * 3513 * If !enable_unrestricted_guest, the CPU is always running 3514 * with CR0.PG=1 and CR4 needs to be modified. 3515 * If enable_unrestricted_guest, the CPU automatically 3516 * disables SMEP/SMAP/PKU when the guest sets CR0.PG=0. 3517 */ 3518 if (!is_paging(vcpu)) 3519 hw_cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE); 3520 } 3521 3522 vmcs_writel(CR4_READ_SHADOW, cr4); 3523 vmcs_writel(GUEST_CR4, hw_cr4); 3524 3525 if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE)) 3526 kvm_update_cpuid_runtime(vcpu); 3527 } 3528 3529 void vmx_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) 3530 { 3531 struct vcpu_vmx *vmx = to_vmx(vcpu); 3532 u32 ar; 3533 3534 if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) { 3535 *var = vmx->rmode.segs[seg]; 3536 if (seg == VCPU_SREG_TR 3537 || var->selector == vmx_read_guest_seg_selector(vmx, seg)) 3538 return; 3539 var->base = vmx_read_guest_seg_base(vmx, seg); 3540 var->selector = vmx_read_guest_seg_selector(vmx, seg); 3541 return; 3542 } 3543 var->base = vmx_read_guest_seg_base(vmx, seg); 3544 var->limit = vmx_read_guest_seg_limit(vmx, seg); 3545 var->selector = vmx_read_guest_seg_selector(vmx, seg); 3546 ar = vmx_read_guest_seg_ar(vmx, seg); 3547 var->unusable = (ar >> 16) & 1; 3548 var->type = ar & 15; 3549 var->s = (ar >> 4) & 1; 3550 var->dpl = (ar >> 5) & 3; 3551 /* 3552 * Some userspaces do not preserve unusable property. Since usable 3553 * segment has to be present according to VMX spec we can use present 3554 * property to amend userspace bug by making unusable segment always 3555 * nonpresent. vmx_segment_access_rights() already marks nonpresent 3556 * segment as unusable. 3557 */ 3558 var->present = !var->unusable; 3559 var->avl = (ar >> 12) & 1; 3560 var->l = (ar >> 13) & 1; 3561 var->db = (ar >> 14) & 1; 3562 var->g = (ar >> 15) & 1; 3563 } 3564 3565 u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg) 3566 { 3567 struct kvm_segment s; 3568 3569 if (to_vmx(vcpu)->rmode.vm86_active) { 3570 vmx_get_segment(vcpu, &s, seg); 3571 return s.base; 3572 } 3573 return vmx_read_guest_seg_base(to_vmx(vcpu), seg); 3574 } 3575 3576 static int __vmx_get_cpl(struct kvm_vcpu *vcpu, bool no_cache) 3577 { 3578 struct vcpu_vmx *vmx = to_vmx(vcpu); 3579 int ar; 3580 3581 if (unlikely(vmx->rmode.vm86_active)) 3582 return 0; 3583 3584 if (no_cache) 3585 ar = vmcs_read32(GUEST_SS_AR_BYTES); 3586 else 3587 ar = vmx_read_guest_seg_ar(vmx, VCPU_SREG_SS); 3588 return VMX_AR_DPL(ar); 3589 } 3590 3591 int vmx_get_cpl(struct kvm_vcpu *vcpu) 3592 { 3593 return __vmx_get_cpl(vcpu, false); 3594 } 3595 3596 int vmx_get_cpl_no_cache(struct kvm_vcpu *vcpu) 3597 { 3598 return __vmx_get_cpl(vcpu, true); 3599 } 3600 3601 static u32 vmx_segment_access_rights(struct kvm_segment *var) 3602 { 3603 u32 ar; 3604 3605 ar = var->type & 15; 3606 ar |= (var->s & 1) << 4; 3607 ar |= (var->dpl & 3) << 5; 3608 ar |= (var->present & 1) << 7; 3609 ar |= (var->avl & 1) << 12; 3610 ar |= (var->l & 1) << 13; 3611 ar |= (var->db & 1) << 14; 3612 ar |= (var->g & 1) << 15; 3613 ar |= (var->unusable || !var->present) << 16; 3614 3615 return ar; 3616 } 3617 3618 void __vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) 3619 { 3620 struct vcpu_vmx *vmx = to_vmx(vcpu); 3621 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 3622 3623 vmx_segment_cache_clear(vmx); 3624 3625 if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) { 3626 vmx->rmode.segs[seg] = *var; 3627 if (seg == VCPU_SREG_TR) 3628 vmcs_write16(sf->selector, var->selector); 3629 else if (var->s) 3630 fix_rmode_seg(seg, &vmx->rmode.segs[seg]); 3631 return; 3632 } 3633 3634 vmcs_writel(sf->base, var->base); 3635 vmcs_write32(sf->limit, var->limit); 3636 vmcs_write16(sf->selector, var->selector); 3637 3638 /* 3639 * Fix the "Accessed" bit in AR field of segment registers for older 3640 * qemu binaries. 3641 * IA32 arch specifies that at the time of processor reset the 3642 * "Accessed" bit in the AR field of segment registers is 1. And qemu 3643 * is setting it to 0 in the userland code. This causes invalid guest 3644 * state vmexit when "unrestricted guest" mode is turned on. 3645 * Fix for this setup issue in cpu_reset is being pushed in the qemu 3646 * tree. Newer qemu binaries with that qemu fix would not need this 3647 * kvm hack. 3648 */ 3649 if (is_unrestricted_guest(vcpu) && (seg != VCPU_SREG_LDTR)) 3650 var->type |= 0x1; /* Accessed */ 3651 3652 vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(var)); 3653 } 3654 3655 void vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) 3656 { 3657 __vmx_set_segment(vcpu, var, seg); 3658 3659 to_vmx(vcpu)->emulation_required = vmx_emulation_required(vcpu); 3660 } 3661 3662 void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) 3663 { 3664 u32 ar = vmx_read_guest_seg_ar(to_vmx(vcpu), VCPU_SREG_CS); 3665 3666 *db = (ar >> 14) & 1; 3667 *l = (ar >> 13) & 1; 3668 } 3669 3670 void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3671 { 3672 dt->size = vmcs_read32(GUEST_IDTR_LIMIT); 3673 dt->address = vmcs_readl(GUEST_IDTR_BASE); 3674 } 3675 3676 void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3677 { 3678 vmcs_write32(GUEST_IDTR_LIMIT, dt->size); 3679 vmcs_writel(GUEST_IDTR_BASE, dt->address); 3680 } 3681 3682 void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3683 { 3684 dt->size = vmcs_read32(GUEST_GDTR_LIMIT); 3685 dt->address = vmcs_readl(GUEST_GDTR_BASE); 3686 } 3687 3688 void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3689 { 3690 vmcs_write32(GUEST_GDTR_LIMIT, dt->size); 3691 vmcs_writel(GUEST_GDTR_BASE, dt->address); 3692 } 3693 3694 static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg) 3695 { 3696 struct kvm_segment var; 3697 u32 ar; 3698 3699 vmx_get_segment(vcpu, &var, seg); 3700 var.dpl = 0x3; 3701 if (seg == VCPU_SREG_CS) 3702 var.type = 0x3; 3703 ar = vmx_segment_access_rights(&var); 3704 3705 if (var.base != (var.selector << 4)) 3706 return false; 3707 if (var.limit != 0xffff) 3708 return false; 3709 if (ar != 0xf3) 3710 return false; 3711 3712 return true; 3713 } 3714 3715 static bool code_segment_valid(struct kvm_vcpu *vcpu) 3716 { 3717 struct kvm_segment cs; 3718 unsigned int cs_rpl; 3719 3720 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS); 3721 cs_rpl = cs.selector & SEGMENT_RPL_MASK; 3722 3723 if (cs.unusable) 3724 return false; 3725 if (~cs.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_ACCESSES_MASK)) 3726 return false; 3727 if (!cs.s) 3728 return false; 3729 if (cs.type & VMX_AR_TYPE_WRITEABLE_MASK) { 3730 if (cs.dpl > cs_rpl) 3731 return false; 3732 } else { 3733 if (cs.dpl != cs_rpl) 3734 return false; 3735 } 3736 if (!cs.present) 3737 return false; 3738 3739 /* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */ 3740 return true; 3741 } 3742 3743 static bool stack_segment_valid(struct kvm_vcpu *vcpu) 3744 { 3745 struct kvm_segment ss; 3746 unsigned int ss_rpl; 3747 3748 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS); 3749 ss_rpl = ss.selector & SEGMENT_RPL_MASK; 3750 3751 if (ss.unusable) 3752 return true; 3753 if (ss.type != 3 && ss.type != 7) 3754 return false; 3755 if (!ss.s) 3756 return false; 3757 if (ss.dpl != ss_rpl) /* DPL != RPL */ 3758 return false; 3759 if (!ss.present) 3760 return false; 3761 3762 return true; 3763 } 3764 3765 static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg) 3766 { 3767 struct kvm_segment var; 3768 unsigned int rpl; 3769 3770 vmx_get_segment(vcpu, &var, seg); 3771 rpl = var.selector & SEGMENT_RPL_MASK; 3772 3773 if (var.unusable) 3774 return true; 3775 if (!var.s) 3776 return false; 3777 if (!var.present) 3778 return false; 3779 if (~var.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_WRITEABLE_MASK)) { 3780 if (var.dpl < rpl) /* DPL < RPL */ 3781 return false; 3782 } 3783 3784 /* TODO: Add other members to kvm_segment_field to allow checking for other access 3785 * rights flags 3786 */ 3787 return true; 3788 } 3789 3790 static bool tr_valid(struct kvm_vcpu *vcpu) 3791 { 3792 struct kvm_segment tr; 3793 3794 vmx_get_segment(vcpu, &tr, VCPU_SREG_TR); 3795 3796 if (tr.unusable) 3797 return false; 3798 if (tr.selector & SEGMENT_TI_MASK) /* TI = 1 */ 3799 return false; 3800 if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */ 3801 return false; 3802 if (!tr.present) 3803 return false; 3804 3805 return true; 3806 } 3807 3808 static bool ldtr_valid(struct kvm_vcpu *vcpu) 3809 { 3810 struct kvm_segment ldtr; 3811 3812 vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR); 3813 3814 if (ldtr.unusable) 3815 return true; 3816 if (ldtr.selector & SEGMENT_TI_MASK) /* TI = 1 */ 3817 return false; 3818 if (ldtr.type != 2) 3819 return false; 3820 if (!ldtr.present) 3821 return false; 3822 3823 return true; 3824 } 3825 3826 static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu) 3827 { 3828 struct kvm_segment cs, ss; 3829 3830 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS); 3831 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS); 3832 3833 return ((cs.selector & SEGMENT_RPL_MASK) == 3834 (ss.selector & SEGMENT_RPL_MASK)); 3835 } 3836 3837 /* 3838 * Check if guest state is valid. Returns true if valid, false if 3839 * not. 3840 * We assume that registers are always usable 3841 */ 3842 bool __vmx_guest_state_valid(struct kvm_vcpu *vcpu) 3843 { 3844 /* real mode guest state checks */ 3845 if (!is_protmode(vcpu) || (vmx_get_rflags(vcpu) & X86_EFLAGS_VM)) { 3846 if (!rmode_segment_valid(vcpu, VCPU_SREG_CS)) 3847 return false; 3848 if (!rmode_segment_valid(vcpu, VCPU_SREG_SS)) 3849 return false; 3850 if (!rmode_segment_valid(vcpu, VCPU_SREG_DS)) 3851 return false; 3852 if (!rmode_segment_valid(vcpu, VCPU_SREG_ES)) 3853 return false; 3854 if (!rmode_segment_valid(vcpu, VCPU_SREG_FS)) 3855 return false; 3856 if (!rmode_segment_valid(vcpu, VCPU_SREG_GS)) 3857 return false; 3858 } else { 3859 /* protected mode guest state checks */ 3860 if (!cs_ss_rpl_check(vcpu)) 3861 return false; 3862 if (!code_segment_valid(vcpu)) 3863 return false; 3864 if (!stack_segment_valid(vcpu)) 3865 return false; 3866 if (!data_segment_valid(vcpu, VCPU_SREG_DS)) 3867 return false; 3868 if (!data_segment_valid(vcpu, VCPU_SREG_ES)) 3869 return false; 3870 if (!data_segment_valid(vcpu, VCPU_SREG_FS)) 3871 return false; 3872 if (!data_segment_valid(vcpu, VCPU_SREG_GS)) 3873 return false; 3874 if (!tr_valid(vcpu)) 3875 return false; 3876 if (!ldtr_valid(vcpu)) 3877 return false; 3878 } 3879 /* TODO: 3880 * - Add checks on RIP 3881 * - Add checks on RFLAGS 3882 */ 3883 3884 return true; 3885 } 3886 3887 static int init_rmode_tss(struct kvm *kvm, void __user *ua) 3888 { 3889 const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0))); 3890 u16 data; 3891 int i; 3892 3893 for (i = 0; i < 3; i++) { 3894 if (__copy_to_user(ua + PAGE_SIZE * i, zero_page, PAGE_SIZE)) 3895 return -EFAULT; 3896 } 3897 3898 data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE; 3899 if (__copy_to_user(ua + TSS_IOPB_BASE_OFFSET, &data, sizeof(u16))) 3900 return -EFAULT; 3901 3902 data = ~0; 3903 if (__copy_to_user(ua + RMODE_TSS_SIZE - 1, &data, sizeof(u8))) 3904 return -EFAULT; 3905 3906 return 0; 3907 } 3908 3909 static int init_rmode_identity_map(struct kvm *kvm) 3910 { 3911 struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); 3912 int i, r = 0; 3913 void __user *uaddr; 3914 u32 tmp; 3915 3916 /* Protect kvm_vmx->ept_identity_pagetable_done. */ 3917 mutex_lock(&kvm->slots_lock); 3918 3919 if (likely(kvm_vmx->ept_identity_pagetable_done)) 3920 goto out; 3921 3922 if (!kvm_vmx->ept_identity_map_addr) 3923 kvm_vmx->ept_identity_map_addr = VMX_EPT_IDENTITY_PAGETABLE_ADDR; 3924 3925 uaddr = __x86_set_memory_region(kvm, 3926 IDENTITY_PAGETABLE_PRIVATE_MEMSLOT, 3927 kvm_vmx->ept_identity_map_addr, 3928 PAGE_SIZE); 3929 if (IS_ERR(uaddr)) { 3930 r = PTR_ERR(uaddr); 3931 goto out; 3932 } 3933 3934 /* Set up identity-mapping pagetable for EPT in real mode */ 3935 for (i = 0; i < (PAGE_SIZE / sizeof(tmp)); i++) { 3936 tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | 3937 _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE); 3938 if (__copy_to_user(uaddr + i * sizeof(tmp), &tmp, sizeof(tmp))) { 3939 r = -EFAULT; 3940 goto out; 3941 } 3942 } 3943 kvm_vmx->ept_identity_pagetable_done = true; 3944 3945 out: 3946 mutex_unlock(&kvm->slots_lock); 3947 return r; 3948 } 3949 3950 static void seg_setup(int seg) 3951 { 3952 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 3953 unsigned int ar; 3954 3955 vmcs_write16(sf->selector, 0); 3956 vmcs_writel(sf->base, 0); 3957 vmcs_write32(sf->limit, 0xffff); 3958 ar = 0x93; 3959 if (seg == VCPU_SREG_CS) 3960 ar |= 0x08; /* code segment */ 3961 3962 vmcs_write32(sf->ar_bytes, ar); 3963 } 3964 3965 int allocate_vpid(void) 3966 { 3967 int vpid; 3968 3969 if (!enable_vpid) 3970 return 0; 3971 spin_lock(&vmx_vpid_lock); 3972 vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS); 3973 if (vpid < VMX_NR_VPIDS) 3974 __set_bit(vpid, vmx_vpid_bitmap); 3975 else 3976 vpid = 0; 3977 spin_unlock(&vmx_vpid_lock); 3978 return vpid; 3979 } 3980 3981 void free_vpid(int vpid) 3982 { 3983 if (!enable_vpid || vpid == 0) 3984 return; 3985 spin_lock(&vmx_vpid_lock); 3986 __clear_bit(vpid, vmx_vpid_bitmap); 3987 spin_unlock(&vmx_vpid_lock); 3988 } 3989 3990 static void vmx_msr_bitmap_l01_changed(struct vcpu_vmx *vmx) 3991 { 3992 /* 3993 * When KVM is a nested hypervisor on top of Hyper-V and uses 3994 * 'Enlightened MSR Bitmap' feature L0 needs to know that MSR 3995 * bitmap has changed. 3996 */ 3997 if (kvm_is_using_evmcs()) { 3998 struct hv_enlightened_vmcs *evmcs = (void *)vmx->vmcs01.vmcs; 3999 4000 if (evmcs->hv_enlightenments_control.msr_bitmap) 4001 evmcs->hv_clean_fields &= 4002 ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP; 4003 } 4004 4005 vmx->nested.force_msr_bitmap_recalc = true; 4006 } 4007 4008 void vmx_disable_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type) 4009 { 4010 struct vcpu_vmx *vmx = to_vmx(vcpu); 4011 unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap; 4012 int idx; 4013 4014 if (!cpu_has_vmx_msr_bitmap()) 4015 return; 4016 4017 vmx_msr_bitmap_l01_changed(vmx); 4018 4019 /* 4020 * Mark the desired intercept state in shadow bitmap, this is needed 4021 * for resync when the MSR filters change. 4022 */ 4023 idx = vmx_get_passthrough_msr_slot(msr); 4024 if (idx >= 0) { 4025 if (type & MSR_TYPE_R) 4026 clear_bit(idx, vmx->shadow_msr_intercept.read); 4027 if (type & MSR_TYPE_W) 4028 clear_bit(idx, vmx->shadow_msr_intercept.write); 4029 } 4030 4031 if ((type & MSR_TYPE_R) && 4032 !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_READ)) { 4033 vmx_set_msr_bitmap_read(msr_bitmap, msr); 4034 type &= ~MSR_TYPE_R; 4035 } 4036 4037 if ((type & MSR_TYPE_W) && 4038 !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_WRITE)) { 4039 vmx_set_msr_bitmap_write(msr_bitmap, msr); 4040 type &= ~MSR_TYPE_W; 4041 } 4042 4043 if (type & MSR_TYPE_R) 4044 vmx_clear_msr_bitmap_read(msr_bitmap, msr); 4045 4046 if (type & MSR_TYPE_W) 4047 vmx_clear_msr_bitmap_write(msr_bitmap, msr); 4048 } 4049 4050 void vmx_enable_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type) 4051 { 4052 struct vcpu_vmx *vmx = to_vmx(vcpu); 4053 unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap; 4054 int idx; 4055 4056 if (!cpu_has_vmx_msr_bitmap()) 4057 return; 4058 4059 vmx_msr_bitmap_l01_changed(vmx); 4060 4061 /* 4062 * Mark the desired intercept state in shadow bitmap, this is needed 4063 * for resync when the MSR filter changes. 4064 */ 4065 idx = vmx_get_passthrough_msr_slot(msr); 4066 if (idx >= 0) { 4067 if (type & MSR_TYPE_R) 4068 set_bit(idx, vmx->shadow_msr_intercept.read); 4069 if (type & MSR_TYPE_W) 4070 set_bit(idx, vmx->shadow_msr_intercept.write); 4071 } 4072 4073 if (type & MSR_TYPE_R) 4074 vmx_set_msr_bitmap_read(msr_bitmap, msr); 4075 4076 if (type & MSR_TYPE_W) 4077 vmx_set_msr_bitmap_write(msr_bitmap, msr); 4078 } 4079 4080 static void vmx_update_msr_bitmap_x2apic(struct kvm_vcpu *vcpu) 4081 { 4082 /* 4083 * x2APIC indices for 64-bit accesses into the RDMSR and WRMSR halves 4084 * of the MSR bitmap. KVM emulates APIC registers up through 0x3f0, 4085 * i.e. MSR 0x83f, and so only needs to dynamically manipulate 64 bits. 4086 */ 4087 const int read_idx = APIC_BASE_MSR / BITS_PER_LONG_LONG; 4088 const int write_idx = read_idx + (0x800 / sizeof(u64)); 4089 struct vcpu_vmx *vmx = to_vmx(vcpu); 4090 u64 *msr_bitmap = (u64 *)vmx->vmcs01.msr_bitmap; 4091 u8 mode; 4092 4093 if (!cpu_has_vmx_msr_bitmap() || WARN_ON_ONCE(!lapic_in_kernel(vcpu))) 4094 return; 4095 4096 if (cpu_has_secondary_exec_ctrls() && 4097 (secondary_exec_controls_get(vmx) & 4098 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE)) { 4099 mode = MSR_BITMAP_MODE_X2APIC; 4100 if (enable_apicv && kvm_vcpu_apicv_active(vcpu)) 4101 mode |= MSR_BITMAP_MODE_X2APIC_APICV; 4102 } else { 4103 mode = 0; 4104 } 4105 4106 if (mode == vmx->x2apic_msr_bitmap_mode) 4107 return; 4108 4109 vmx->x2apic_msr_bitmap_mode = mode; 4110 4111 /* 4112 * Reset the bitmap for MSRs 0x800 - 0x83f. Leave AMD's uber-extended 4113 * registers (0x840 and above) intercepted, KVM doesn't support them. 4114 * Intercept all writes by default and poke holes as needed. Pass 4115 * through reads for all valid registers by default in x2APIC+APICv 4116 * mode, only the current timer count needs on-demand emulation by KVM. 4117 */ 4118 if (mode & MSR_BITMAP_MODE_X2APIC_APICV) 4119 msr_bitmap[read_idx] = ~kvm_lapic_readable_reg_mask(vcpu->arch.apic); 4120 else 4121 msr_bitmap[read_idx] = ~0ull; 4122 msr_bitmap[write_idx] = ~0ull; 4123 4124 /* 4125 * TPR reads and writes can be virtualized even if virtual interrupt 4126 * delivery is not in use. 4127 */ 4128 vmx_set_intercept_for_msr(vcpu, X2APIC_MSR(APIC_TASKPRI), MSR_TYPE_RW, 4129 !(mode & MSR_BITMAP_MODE_X2APIC)); 4130 4131 if (mode & MSR_BITMAP_MODE_X2APIC_APICV) { 4132 vmx_enable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_TMCCT), MSR_TYPE_RW); 4133 vmx_disable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_EOI), MSR_TYPE_W); 4134 vmx_disable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_SELF_IPI), MSR_TYPE_W); 4135 if (enable_ipiv) 4136 vmx_disable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_ICR), MSR_TYPE_RW); 4137 } 4138 } 4139 4140 void pt_update_intercept_for_msr(struct kvm_vcpu *vcpu) 4141 { 4142 struct vcpu_vmx *vmx = to_vmx(vcpu); 4143 bool flag = !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN); 4144 u32 i; 4145 4146 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_STATUS, MSR_TYPE_RW, flag); 4147 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_OUTPUT_BASE, MSR_TYPE_RW, flag); 4148 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_OUTPUT_MASK, MSR_TYPE_RW, flag); 4149 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_CR3_MATCH, MSR_TYPE_RW, flag); 4150 for (i = 0; i < vmx->pt_desc.num_address_ranges; i++) { 4151 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_ADDR0_A + i * 2, MSR_TYPE_RW, flag); 4152 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_ADDR0_B + i * 2, MSR_TYPE_RW, flag); 4153 } 4154 } 4155 4156 void vmx_msr_filter_changed(struct kvm_vcpu *vcpu) 4157 { 4158 struct vcpu_vmx *vmx = to_vmx(vcpu); 4159 u32 i; 4160 4161 if (!cpu_has_vmx_msr_bitmap()) 4162 return; 4163 4164 /* 4165 * Redo intercept permissions for MSRs that KVM is passing through to 4166 * the guest. Disabling interception will check the new MSR filter and 4167 * ensure that KVM enables interception if usersepace wants to filter 4168 * the MSR. MSRs that KVM is already intercepting don't need to be 4169 * refreshed since KVM is going to intercept them regardless of what 4170 * userspace wants. 4171 */ 4172 for (i = 0; i < ARRAY_SIZE(vmx_possible_passthrough_msrs); i++) { 4173 u32 msr = vmx_possible_passthrough_msrs[i]; 4174 4175 if (!test_bit(i, vmx->shadow_msr_intercept.read)) 4176 vmx_disable_intercept_for_msr(vcpu, msr, MSR_TYPE_R); 4177 4178 if (!test_bit(i, vmx->shadow_msr_intercept.write)) 4179 vmx_disable_intercept_for_msr(vcpu, msr, MSR_TYPE_W); 4180 } 4181 4182 /* PT MSRs can be passed through iff PT is exposed to the guest. */ 4183 if (vmx_pt_mode_is_host_guest()) 4184 pt_update_intercept_for_msr(vcpu); 4185 } 4186 4187 static inline void kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu, 4188 int pi_vec) 4189 { 4190 #ifdef CONFIG_SMP 4191 if (vcpu->mode == IN_GUEST_MODE) { 4192 /* 4193 * The vector of the virtual has already been set in the PIR. 4194 * Send a notification event to deliver the virtual interrupt 4195 * unless the vCPU is the currently running vCPU, i.e. the 4196 * event is being sent from a fastpath VM-Exit handler, in 4197 * which case the PIR will be synced to the vIRR before 4198 * re-entering the guest. 4199 * 4200 * When the target is not the running vCPU, the following 4201 * possibilities emerge: 4202 * 4203 * Case 1: vCPU stays in non-root mode. Sending a notification 4204 * event posts the interrupt to the vCPU. 4205 * 4206 * Case 2: vCPU exits to root mode and is still runnable. The 4207 * PIR will be synced to the vIRR before re-entering the guest. 4208 * Sending a notification event is ok as the host IRQ handler 4209 * will ignore the spurious event. 4210 * 4211 * Case 3: vCPU exits to root mode and is blocked. vcpu_block() 4212 * has already synced PIR to vIRR and never blocks the vCPU if 4213 * the vIRR is not empty. Therefore, a blocked vCPU here does 4214 * not wait for any requested interrupts in PIR, and sending a 4215 * notification event also results in a benign, spurious event. 4216 */ 4217 4218 if (vcpu != kvm_get_running_vcpu()) 4219 __apic_send_IPI_mask(get_cpu_mask(vcpu->cpu), pi_vec); 4220 return; 4221 } 4222 #endif 4223 /* 4224 * The vCPU isn't in the guest; wake the vCPU in case it is blocking, 4225 * otherwise do nothing as KVM will grab the highest priority pending 4226 * IRQ via ->sync_pir_to_irr() in vcpu_enter_guest(). 4227 */ 4228 kvm_vcpu_wake_up(vcpu); 4229 } 4230 4231 static int vmx_deliver_nested_posted_interrupt(struct kvm_vcpu *vcpu, 4232 int vector) 4233 { 4234 struct vcpu_vmx *vmx = to_vmx(vcpu); 4235 4236 /* 4237 * DO NOT query the vCPU's vmcs12, as vmcs12 is dynamically allocated 4238 * and freed, and must not be accessed outside of vcpu->mutex. The 4239 * vCPU's cached PI NV is valid if and only if posted interrupts 4240 * enabled in its vmcs12, i.e. checking the vector also checks that 4241 * L1 has enabled posted interrupts for L2. 4242 */ 4243 if (is_guest_mode(vcpu) && 4244 vector == vmx->nested.posted_intr_nv) { 4245 /* 4246 * If a posted intr is not recognized by hardware, 4247 * we will accomplish it in the next vmentry. 4248 */ 4249 vmx->nested.pi_pending = true; 4250 kvm_make_request(KVM_REQ_EVENT, vcpu); 4251 4252 /* 4253 * This pairs with the smp_mb_*() after setting vcpu->mode in 4254 * vcpu_enter_guest() to guarantee the vCPU sees the event 4255 * request if triggering a posted interrupt "fails" because 4256 * vcpu->mode != IN_GUEST_MODE. The extra barrier is needed as 4257 * the smb_wmb() in kvm_make_request() only ensures everything 4258 * done before making the request is visible when the request 4259 * is visible, it doesn't ensure ordering between the store to 4260 * vcpu->requests and the load from vcpu->mode. 4261 */ 4262 smp_mb__after_atomic(); 4263 4264 /* the PIR and ON have been set by L1. */ 4265 kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_NESTED_VECTOR); 4266 return 0; 4267 } 4268 return -1; 4269 } 4270 /* 4271 * Send interrupt to vcpu via posted interrupt way. 4272 * 1. If target vcpu is running(non-root mode), send posted interrupt 4273 * notification to vcpu and hardware will sync PIR to vIRR atomically. 4274 * 2. If target vcpu isn't running(root mode), kick it to pick up the 4275 * interrupt from PIR in next vmentry. 4276 */ 4277 static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector) 4278 { 4279 struct vcpu_vmx *vmx = to_vmx(vcpu); 4280 int r; 4281 4282 r = vmx_deliver_nested_posted_interrupt(vcpu, vector); 4283 if (!r) 4284 return 0; 4285 4286 /* Note, this is called iff the local APIC is in-kernel. */ 4287 if (!vcpu->arch.apic->apicv_active) 4288 return -1; 4289 4290 if (pi_test_and_set_pir(vector, &vmx->pi_desc)) 4291 return 0; 4292 4293 /* If a previous notification has sent the IPI, nothing to do. */ 4294 if (pi_test_and_set_on(&vmx->pi_desc)) 4295 return 0; 4296 4297 /* 4298 * The implied barrier in pi_test_and_set_on() pairs with the smp_mb_*() 4299 * after setting vcpu->mode in vcpu_enter_guest(), thus the vCPU is 4300 * guaranteed to see PID.ON=1 and sync the PIR to IRR if triggering a 4301 * posted interrupt "fails" because vcpu->mode != IN_GUEST_MODE. 4302 */ 4303 kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_VECTOR); 4304 return 0; 4305 } 4306 4307 void vmx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, 4308 int trig_mode, int vector) 4309 { 4310 struct kvm_vcpu *vcpu = apic->vcpu; 4311 4312 if (vmx_deliver_posted_interrupt(vcpu, vector)) { 4313 kvm_lapic_set_irr(vector, apic); 4314 kvm_make_request(KVM_REQ_EVENT, vcpu); 4315 kvm_vcpu_kick(vcpu); 4316 } else { 4317 trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, 4318 trig_mode, vector); 4319 } 4320 } 4321 4322 /* 4323 * Set up the vmcs's constant host-state fields, i.e., host-state fields that 4324 * will not change in the lifetime of the guest. 4325 * Note that host-state that does change is set elsewhere. E.g., host-state 4326 * that is set differently for each CPU is set in vmx_vcpu_load(), not here. 4327 */ 4328 void vmx_set_constant_host_state(struct vcpu_vmx *vmx) 4329 { 4330 u32 low32, high32; 4331 unsigned long tmpl; 4332 unsigned long cr0, cr3, cr4; 4333 4334 cr0 = read_cr0(); 4335 WARN_ON(cr0 & X86_CR0_TS); 4336 vmcs_writel(HOST_CR0, cr0); /* 22.2.3 */ 4337 4338 /* 4339 * Save the most likely value for this task's CR3 in the VMCS. 4340 * We can't use __get_current_cr3_fast() because we're not atomic. 4341 */ 4342 cr3 = __read_cr3(); 4343 vmcs_writel(HOST_CR3, cr3); /* 22.2.3 FIXME: shadow tables */ 4344 vmx->loaded_vmcs->host_state.cr3 = cr3; 4345 4346 /* Save the most likely value for this task's CR4 in the VMCS. */ 4347 cr4 = cr4_read_shadow(); 4348 vmcs_writel(HOST_CR4, cr4); /* 22.2.3, 22.2.5 */ 4349 vmx->loaded_vmcs->host_state.cr4 = cr4; 4350 4351 vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS); /* 22.2.4 */ 4352 #ifdef CONFIG_X86_64 4353 /* 4354 * Load null selectors, so we can avoid reloading them in 4355 * vmx_prepare_switch_to_host(), in case userspace uses 4356 * the null selectors too (the expected case). 4357 */ 4358 vmcs_write16(HOST_DS_SELECTOR, 0); 4359 vmcs_write16(HOST_ES_SELECTOR, 0); 4360 #else 4361 vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS); /* 22.2.4 */ 4362 vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS); /* 22.2.4 */ 4363 #endif 4364 vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS); /* 22.2.4 */ 4365 vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8); /* 22.2.4 */ 4366 4367 vmcs_writel(HOST_IDTR_BASE, host_idt_base); /* 22.2.4 */ 4368 4369 vmcs_writel(HOST_RIP, (unsigned long)vmx_vmexit); /* 22.2.5 */ 4370 4371 rdmsr(MSR_IA32_SYSENTER_CS, low32, high32); 4372 vmcs_write32(HOST_IA32_SYSENTER_CS, low32); 4373 4374 /* 4375 * SYSENTER is used for 32-bit system calls on either 32-bit or 4376 * 64-bit kernels. It is always zero If neither is allowed, otherwise 4377 * vmx_vcpu_load_vmcs loads it with the per-CPU entry stack (and may 4378 * have already done so!). 4379 */ 4380 if (!IS_ENABLED(CONFIG_IA32_EMULATION) && !IS_ENABLED(CONFIG_X86_32)) 4381 vmcs_writel(HOST_IA32_SYSENTER_ESP, 0); 4382 4383 rdmsrl(MSR_IA32_SYSENTER_EIP, tmpl); 4384 vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl); /* 22.2.3 */ 4385 4386 if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) { 4387 rdmsr(MSR_IA32_CR_PAT, low32, high32); 4388 vmcs_write64(HOST_IA32_PAT, low32 | ((u64) high32 << 32)); 4389 } 4390 4391 if (cpu_has_load_ia32_efer()) 4392 vmcs_write64(HOST_IA32_EFER, kvm_host.efer); 4393 } 4394 4395 void set_cr4_guest_host_mask(struct vcpu_vmx *vmx) 4396 { 4397 struct kvm_vcpu *vcpu = &vmx->vcpu; 4398 4399 vcpu->arch.cr4_guest_owned_bits = KVM_POSSIBLE_CR4_GUEST_BITS & 4400 ~vcpu->arch.cr4_guest_rsvd_bits; 4401 if (!enable_ept) { 4402 vcpu->arch.cr4_guest_owned_bits &= ~X86_CR4_TLBFLUSH_BITS; 4403 vcpu->arch.cr4_guest_owned_bits &= ~X86_CR4_PDPTR_BITS; 4404 } 4405 if (is_guest_mode(&vmx->vcpu)) 4406 vcpu->arch.cr4_guest_owned_bits &= 4407 ~get_vmcs12(vcpu)->cr4_guest_host_mask; 4408 vmcs_writel(CR4_GUEST_HOST_MASK, ~vcpu->arch.cr4_guest_owned_bits); 4409 } 4410 4411 static u32 vmx_pin_based_exec_ctrl(struct vcpu_vmx *vmx) 4412 { 4413 u32 pin_based_exec_ctrl = vmcs_config.pin_based_exec_ctrl; 4414 4415 if (!kvm_vcpu_apicv_active(&vmx->vcpu)) 4416 pin_based_exec_ctrl &= ~PIN_BASED_POSTED_INTR; 4417 4418 if (!enable_vnmi) 4419 pin_based_exec_ctrl &= ~PIN_BASED_VIRTUAL_NMIS; 4420 4421 if (!enable_preemption_timer) 4422 pin_based_exec_ctrl &= ~PIN_BASED_VMX_PREEMPTION_TIMER; 4423 4424 return pin_based_exec_ctrl; 4425 } 4426 4427 static u32 vmx_vmentry_ctrl(void) 4428 { 4429 u32 vmentry_ctrl = vmcs_config.vmentry_ctrl; 4430 4431 if (vmx_pt_mode_is_system()) 4432 vmentry_ctrl &= ~(VM_ENTRY_PT_CONCEAL_PIP | 4433 VM_ENTRY_LOAD_IA32_RTIT_CTL); 4434 /* 4435 * IA32e mode, and loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically. 4436 */ 4437 vmentry_ctrl &= ~(VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL | 4438 VM_ENTRY_LOAD_IA32_EFER | 4439 VM_ENTRY_IA32E_MODE); 4440 4441 return vmentry_ctrl; 4442 } 4443 4444 static u32 vmx_vmexit_ctrl(void) 4445 { 4446 u32 vmexit_ctrl = vmcs_config.vmexit_ctrl; 4447 4448 /* 4449 * Not used by KVM and never set in vmcs01 or vmcs02, but emulated for 4450 * nested virtualization and thus allowed to be set in vmcs12. 4451 */ 4452 vmexit_ctrl &= ~(VM_EXIT_SAVE_IA32_PAT | VM_EXIT_SAVE_IA32_EFER | 4453 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER); 4454 4455 if (vmx_pt_mode_is_system()) 4456 vmexit_ctrl &= ~(VM_EXIT_PT_CONCEAL_PIP | 4457 VM_EXIT_CLEAR_IA32_RTIT_CTL); 4458 /* Loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically */ 4459 return vmexit_ctrl & 4460 ~(VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | VM_EXIT_LOAD_IA32_EFER); 4461 } 4462 4463 void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) 4464 { 4465 struct vcpu_vmx *vmx = to_vmx(vcpu); 4466 4467 if (is_guest_mode(vcpu)) { 4468 vmx->nested.update_vmcs01_apicv_status = true; 4469 return; 4470 } 4471 4472 pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx)); 4473 4474 if (kvm_vcpu_apicv_active(vcpu)) { 4475 secondary_exec_controls_setbit(vmx, 4476 SECONDARY_EXEC_APIC_REGISTER_VIRT | 4477 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 4478 if (enable_ipiv) 4479 tertiary_exec_controls_setbit(vmx, TERTIARY_EXEC_IPI_VIRT); 4480 } else { 4481 secondary_exec_controls_clearbit(vmx, 4482 SECONDARY_EXEC_APIC_REGISTER_VIRT | 4483 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 4484 if (enable_ipiv) 4485 tertiary_exec_controls_clearbit(vmx, TERTIARY_EXEC_IPI_VIRT); 4486 } 4487 4488 vmx_update_msr_bitmap_x2apic(vcpu); 4489 } 4490 4491 static u32 vmx_exec_control(struct vcpu_vmx *vmx) 4492 { 4493 u32 exec_control = vmcs_config.cpu_based_exec_ctrl; 4494 4495 /* 4496 * Not used by KVM, but fully supported for nesting, i.e. are allowed in 4497 * vmcs12 and propagated to vmcs02 when set in vmcs12. 4498 */ 4499 exec_control &= ~(CPU_BASED_RDTSC_EXITING | 4500 CPU_BASED_USE_IO_BITMAPS | 4501 CPU_BASED_MONITOR_TRAP_FLAG | 4502 CPU_BASED_PAUSE_EXITING); 4503 4504 /* INTR_WINDOW_EXITING and NMI_WINDOW_EXITING are toggled dynamically */ 4505 exec_control &= ~(CPU_BASED_INTR_WINDOW_EXITING | 4506 CPU_BASED_NMI_WINDOW_EXITING); 4507 4508 if (vmx->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT) 4509 exec_control &= ~CPU_BASED_MOV_DR_EXITING; 4510 4511 if (!cpu_need_tpr_shadow(&vmx->vcpu)) 4512 exec_control &= ~CPU_BASED_TPR_SHADOW; 4513 4514 #ifdef CONFIG_X86_64 4515 if (exec_control & CPU_BASED_TPR_SHADOW) 4516 exec_control &= ~(CPU_BASED_CR8_LOAD_EXITING | 4517 CPU_BASED_CR8_STORE_EXITING); 4518 else 4519 exec_control |= CPU_BASED_CR8_STORE_EXITING | 4520 CPU_BASED_CR8_LOAD_EXITING; 4521 #endif 4522 /* No need to intercept CR3 access or INVPLG when using EPT. */ 4523 if (enable_ept) 4524 exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING | 4525 CPU_BASED_CR3_STORE_EXITING | 4526 CPU_BASED_INVLPG_EXITING); 4527 if (kvm_mwait_in_guest(vmx->vcpu.kvm)) 4528 exec_control &= ~(CPU_BASED_MWAIT_EXITING | 4529 CPU_BASED_MONITOR_EXITING); 4530 if (kvm_hlt_in_guest(vmx->vcpu.kvm)) 4531 exec_control &= ~CPU_BASED_HLT_EXITING; 4532 return exec_control; 4533 } 4534 4535 static u64 vmx_tertiary_exec_control(struct vcpu_vmx *vmx) 4536 { 4537 u64 exec_control = vmcs_config.cpu_based_3rd_exec_ctrl; 4538 4539 /* 4540 * IPI virtualization relies on APICv. Disable IPI virtualization if 4541 * APICv is inhibited. 4542 */ 4543 if (!enable_ipiv || !kvm_vcpu_apicv_active(&vmx->vcpu)) 4544 exec_control &= ~TERTIARY_EXEC_IPI_VIRT; 4545 4546 return exec_control; 4547 } 4548 4549 /* 4550 * Adjust a single secondary execution control bit to intercept/allow an 4551 * instruction in the guest. This is usually done based on whether or not a 4552 * feature has been exposed to the guest in order to correctly emulate faults. 4553 */ 4554 static inline void 4555 vmx_adjust_secondary_exec_control(struct vcpu_vmx *vmx, u32 *exec_control, 4556 u32 control, bool enabled, bool exiting) 4557 { 4558 /* 4559 * If the control is for an opt-in feature, clear the control if the 4560 * feature is not exposed to the guest, i.e. not enabled. If the 4561 * control is opt-out, i.e. an exiting control, clear the control if 4562 * the feature _is_ exposed to the guest, i.e. exiting/interception is 4563 * disabled for the associated instruction. Note, the caller is 4564 * responsible presetting exec_control to set all supported bits. 4565 */ 4566 if (enabled == exiting) 4567 *exec_control &= ~control; 4568 4569 /* 4570 * Update the nested MSR settings so that a nested VMM can/can't set 4571 * controls for features that are/aren't exposed to the guest. 4572 */ 4573 if (nested && 4574 kvm_check_has_quirk(vmx->vcpu.kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS)) { 4575 /* 4576 * All features that can be added or removed to VMX MSRs must 4577 * be supported in the first place for nested virtualization. 4578 */ 4579 if (WARN_ON_ONCE(!(vmcs_config.nested.secondary_ctls_high & control))) 4580 enabled = false; 4581 4582 if (enabled) 4583 vmx->nested.msrs.secondary_ctls_high |= control; 4584 else 4585 vmx->nested.msrs.secondary_ctls_high &= ~control; 4586 } 4587 } 4588 4589 /* 4590 * Wrapper macro for the common case of adjusting a secondary execution control 4591 * based on a single guest CPUID bit, with a dedicated feature bit. This also 4592 * verifies that the control is actually supported by KVM and hardware. 4593 */ 4594 #define vmx_adjust_sec_exec_control(vmx, exec_control, name, feat_name, ctrl_name, exiting) \ 4595 ({ \ 4596 struct kvm_vcpu *__vcpu = &(vmx)->vcpu; \ 4597 bool __enabled; \ 4598 \ 4599 if (cpu_has_vmx_##name()) { \ 4600 __enabled = guest_cpu_cap_has(__vcpu, X86_FEATURE_##feat_name); \ 4601 vmx_adjust_secondary_exec_control(vmx, exec_control, SECONDARY_EXEC_##ctrl_name,\ 4602 __enabled, exiting); \ 4603 } \ 4604 }) 4605 4606 /* More macro magic for ENABLE_/opt-in versus _EXITING/opt-out controls. */ 4607 #define vmx_adjust_sec_exec_feature(vmx, exec_control, lname, uname) \ 4608 vmx_adjust_sec_exec_control(vmx, exec_control, lname, uname, ENABLE_##uname, false) 4609 4610 #define vmx_adjust_sec_exec_exiting(vmx, exec_control, lname, uname) \ 4611 vmx_adjust_sec_exec_control(vmx, exec_control, lname, uname, uname##_EXITING, true) 4612 4613 static u32 vmx_secondary_exec_control(struct vcpu_vmx *vmx) 4614 { 4615 struct kvm_vcpu *vcpu = &vmx->vcpu; 4616 4617 u32 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl; 4618 4619 if (vmx_pt_mode_is_system()) 4620 exec_control &= ~(SECONDARY_EXEC_PT_USE_GPA | SECONDARY_EXEC_PT_CONCEAL_VMX); 4621 if (!cpu_need_virtualize_apic_accesses(vcpu)) 4622 exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 4623 if (vmx->vpid == 0) 4624 exec_control &= ~SECONDARY_EXEC_ENABLE_VPID; 4625 if (!enable_ept) { 4626 exec_control &= ~SECONDARY_EXEC_ENABLE_EPT; 4627 exec_control &= ~SECONDARY_EXEC_EPT_VIOLATION_VE; 4628 enable_unrestricted_guest = 0; 4629 } 4630 if (!enable_unrestricted_guest) 4631 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST; 4632 if (kvm_pause_in_guest(vmx->vcpu.kvm)) 4633 exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING; 4634 if (!kvm_vcpu_apicv_active(vcpu)) 4635 exec_control &= ~(SECONDARY_EXEC_APIC_REGISTER_VIRT | 4636 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 4637 exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE; 4638 4639 /* 4640 * KVM doesn't support VMFUNC for L1, but the control is set in KVM's 4641 * base configuration as KVM emulates VMFUNC[EPTP_SWITCHING] for L2. 4642 */ 4643 exec_control &= ~SECONDARY_EXEC_ENABLE_VMFUNC; 4644 4645 /* SECONDARY_EXEC_DESC is enabled/disabled on writes to CR4.UMIP, 4646 * in vmx_set_cr4. */ 4647 exec_control &= ~SECONDARY_EXEC_DESC; 4648 4649 /* SECONDARY_EXEC_SHADOW_VMCS is enabled when L1 executes VMPTRLD 4650 (handle_vmptrld). 4651 We can NOT enable shadow_vmcs here because we don't have yet 4652 a current VMCS12 4653 */ 4654 exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS; 4655 4656 /* 4657 * PML is enabled/disabled when dirty logging of memsmlots changes, but 4658 * it needs to be set here when dirty logging is already active, e.g. 4659 * if this vCPU was created after dirty logging was enabled. 4660 */ 4661 if (!enable_pml || !atomic_read(&vcpu->kvm->nr_memslots_dirty_logging)) 4662 exec_control &= ~SECONDARY_EXEC_ENABLE_PML; 4663 4664 vmx_adjust_sec_exec_feature(vmx, &exec_control, xsaves, XSAVES); 4665 4666 /* 4667 * RDPID is also gated by ENABLE_RDTSCP, turn on the control if either 4668 * feature is exposed to the guest. This creates a virtualization hole 4669 * if both are supported in hardware but only one is exposed to the 4670 * guest, but letting the guest execute RDTSCP or RDPID when either one 4671 * is advertised is preferable to emulating the advertised instruction 4672 * in KVM on #UD, and obviously better than incorrectly injecting #UD. 4673 */ 4674 if (cpu_has_vmx_rdtscp()) { 4675 bool rdpid_or_rdtscp_enabled = 4676 guest_cpu_cap_has(vcpu, X86_FEATURE_RDTSCP) || 4677 guest_cpu_cap_has(vcpu, X86_FEATURE_RDPID); 4678 4679 vmx_adjust_secondary_exec_control(vmx, &exec_control, 4680 SECONDARY_EXEC_ENABLE_RDTSCP, 4681 rdpid_or_rdtscp_enabled, false); 4682 } 4683 4684 vmx_adjust_sec_exec_feature(vmx, &exec_control, invpcid, INVPCID); 4685 4686 vmx_adjust_sec_exec_exiting(vmx, &exec_control, rdrand, RDRAND); 4687 vmx_adjust_sec_exec_exiting(vmx, &exec_control, rdseed, RDSEED); 4688 4689 vmx_adjust_sec_exec_control(vmx, &exec_control, waitpkg, WAITPKG, 4690 ENABLE_USR_WAIT_PAUSE, false); 4691 4692 if (!vcpu->kvm->arch.bus_lock_detection_enabled) 4693 exec_control &= ~SECONDARY_EXEC_BUS_LOCK_DETECTION; 4694 4695 if (!kvm_notify_vmexit_enabled(vcpu->kvm)) 4696 exec_control &= ~SECONDARY_EXEC_NOTIFY_VM_EXITING; 4697 4698 return exec_control; 4699 } 4700 4701 static inline int vmx_get_pid_table_order(struct kvm *kvm) 4702 { 4703 return get_order(kvm->arch.max_vcpu_ids * sizeof(*to_kvm_vmx(kvm)->pid_table)); 4704 } 4705 4706 static int vmx_alloc_ipiv_pid_table(struct kvm *kvm) 4707 { 4708 struct page *pages; 4709 struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); 4710 4711 if (!irqchip_in_kernel(kvm) || !enable_ipiv) 4712 return 0; 4713 4714 if (kvm_vmx->pid_table) 4715 return 0; 4716 4717 pages = alloc_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO, 4718 vmx_get_pid_table_order(kvm)); 4719 if (!pages) 4720 return -ENOMEM; 4721 4722 kvm_vmx->pid_table = (void *)page_address(pages); 4723 return 0; 4724 } 4725 4726 int vmx_vcpu_precreate(struct kvm *kvm) 4727 { 4728 return vmx_alloc_ipiv_pid_table(kvm); 4729 } 4730 4731 #define VMX_XSS_EXIT_BITMAP 0 4732 4733 static void init_vmcs(struct vcpu_vmx *vmx) 4734 { 4735 struct kvm *kvm = vmx->vcpu.kvm; 4736 struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); 4737 4738 if (nested) 4739 nested_vmx_set_vmcs_shadowing_bitmap(); 4740 4741 if (cpu_has_vmx_msr_bitmap()) 4742 vmcs_write64(MSR_BITMAP, __pa(vmx->vmcs01.msr_bitmap)); 4743 4744 vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); /* 22.3.1.5 */ 4745 4746 /* Control */ 4747 pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx)); 4748 4749 exec_controls_set(vmx, vmx_exec_control(vmx)); 4750 4751 if (cpu_has_secondary_exec_ctrls()) { 4752 secondary_exec_controls_set(vmx, vmx_secondary_exec_control(vmx)); 4753 if (vmx->ve_info) 4754 vmcs_write64(VE_INFORMATION_ADDRESS, 4755 __pa(vmx->ve_info)); 4756 } 4757 4758 if (cpu_has_tertiary_exec_ctrls()) 4759 tertiary_exec_controls_set(vmx, vmx_tertiary_exec_control(vmx)); 4760 4761 if (enable_apicv && lapic_in_kernel(&vmx->vcpu)) { 4762 vmcs_write64(EOI_EXIT_BITMAP0, 0); 4763 vmcs_write64(EOI_EXIT_BITMAP1, 0); 4764 vmcs_write64(EOI_EXIT_BITMAP2, 0); 4765 vmcs_write64(EOI_EXIT_BITMAP3, 0); 4766 4767 vmcs_write16(GUEST_INTR_STATUS, 0); 4768 4769 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR); 4770 vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->pi_desc))); 4771 } 4772 4773 if (vmx_can_use_ipiv(&vmx->vcpu)) { 4774 vmcs_write64(PID_POINTER_TABLE, __pa(kvm_vmx->pid_table)); 4775 vmcs_write16(LAST_PID_POINTER_INDEX, kvm->arch.max_vcpu_ids - 1); 4776 } 4777 4778 if (!kvm_pause_in_guest(kvm)) { 4779 vmcs_write32(PLE_GAP, ple_gap); 4780 vmx->ple_window = ple_window; 4781 vmx->ple_window_dirty = true; 4782 } 4783 4784 if (kvm_notify_vmexit_enabled(kvm)) 4785 vmcs_write32(NOTIFY_WINDOW, kvm->arch.notify_window); 4786 4787 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0); 4788 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0); 4789 vmcs_write32(CR3_TARGET_COUNT, 0); /* 22.2.1 */ 4790 4791 vmcs_write16(HOST_FS_SELECTOR, 0); /* 22.2.4 */ 4792 vmcs_write16(HOST_GS_SELECTOR, 0); /* 22.2.4 */ 4793 vmx_set_constant_host_state(vmx); 4794 vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */ 4795 vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */ 4796 4797 if (cpu_has_vmx_vmfunc()) 4798 vmcs_write64(VM_FUNCTION_CONTROL, 0); 4799 4800 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0); 4801 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0); 4802 vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val)); 4803 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0); 4804 vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); 4805 4806 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) 4807 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); 4808 4809 vm_exit_controls_set(vmx, vmx_vmexit_ctrl()); 4810 4811 /* 22.2.1, 20.8.1 */ 4812 vm_entry_controls_set(vmx, vmx_vmentry_ctrl()); 4813 4814 vmx->vcpu.arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits(); 4815 vmcs_writel(CR0_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr0_guest_owned_bits); 4816 4817 set_cr4_guest_host_mask(vmx); 4818 4819 if (vmx->vpid != 0) 4820 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid); 4821 4822 if (cpu_has_vmx_xsaves()) 4823 vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP); 4824 4825 if (enable_pml) { 4826 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg)); 4827 vmcs_write16(GUEST_PML_INDEX, PML_HEAD_INDEX); 4828 } 4829 4830 vmx_write_encls_bitmap(&vmx->vcpu, NULL); 4831 4832 if (vmx_pt_mode_is_host_guest()) { 4833 memset(&vmx->pt_desc, 0, sizeof(vmx->pt_desc)); 4834 /* Bit[6~0] are forced to 1, writes are ignored. */ 4835 vmx->pt_desc.guest.output_mask = 0x7F; 4836 vmcs_write64(GUEST_IA32_RTIT_CTL, 0); 4837 } 4838 4839 vmcs_write32(GUEST_SYSENTER_CS, 0); 4840 vmcs_writel(GUEST_SYSENTER_ESP, 0); 4841 vmcs_writel(GUEST_SYSENTER_EIP, 0); 4842 vmcs_write64(GUEST_IA32_DEBUGCTL, 0); 4843 4844 if (cpu_has_vmx_tpr_shadow()) { 4845 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0); 4846 if (cpu_need_tpr_shadow(&vmx->vcpu)) 4847 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 4848 __pa(vmx->vcpu.arch.apic->regs)); 4849 vmcs_write32(TPR_THRESHOLD, 0); 4850 } 4851 4852 vmx_setup_uret_msrs(vmx); 4853 } 4854 4855 static void __vmx_vcpu_reset(struct kvm_vcpu *vcpu) 4856 { 4857 struct vcpu_vmx *vmx = to_vmx(vcpu); 4858 4859 init_vmcs(vmx); 4860 4861 if (nested && 4862 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS)) 4863 memcpy(&vmx->nested.msrs, &vmcs_config.nested, sizeof(vmx->nested.msrs)); 4864 4865 vcpu_setup_sgx_lepubkeyhash(vcpu); 4866 4867 vmx->nested.posted_intr_nv = -1; 4868 vmx->nested.vmxon_ptr = INVALID_GPA; 4869 vmx->nested.current_vmptr = INVALID_GPA; 4870 4871 #ifdef CONFIG_KVM_HYPERV 4872 vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID; 4873 #endif 4874 4875 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS)) 4876 vcpu->arch.microcode_version = 0x100000000ULL; 4877 vmx->msr_ia32_feature_control_valid_bits = FEAT_CTL_LOCKED; 4878 4879 /* 4880 * Enforce invariant: pi_desc.nv is always either POSTED_INTR_VECTOR 4881 * or POSTED_INTR_WAKEUP_VECTOR. 4882 */ 4883 vmx->pi_desc.nv = POSTED_INTR_VECTOR; 4884 __pi_set_sn(&vmx->pi_desc); 4885 } 4886 4887 void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) 4888 { 4889 struct vcpu_vmx *vmx = to_vmx(vcpu); 4890 4891 if (!init_event) 4892 __vmx_vcpu_reset(vcpu); 4893 4894 vmx->rmode.vm86_active = 0; 4895 vmx->spec_ctrl = 0; 4896 4897 vmx->msr_ia32_umwait_control = 0; 4898 4899 vmx->hv_deadline_tsc = -1; 4900 kvm_set_cr8(vcpu, 0); 4901 4902 seg_setup(VCPU_SREG_CS); 4903 vmcs_write16(GUEST_CS_SELECTOR, 0xf000); 4904 vmcs_writel(GUEST_CS_BASE, 0xffff0000ul); 4905 4906 seg_setup(VCPU_SREG_DS); 4907 seg_setup(VCPU_SREG_ES); 4908 seg_setup(VCPU_SREG_FS); 4909 seg_setup(VCPU_SREG_GS); 4910 seg_setup(VCPU_SREG_SS); 4911 4912 vmcs_write16(GUEST_TR_SELECTOR, 0); 4913 vmcs_writel(GUEST_TR_BASE, 0); 4914 vmcs_write32(GUEST_TR_LIMIT, 0xffff); 4915 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); 4916 4917 vmcs_write16(GUEST_LDTR_SELECTOR, 0); 4918 vmcs_writel(GUEST_LDTR_BASE, 0); 4919 vmcs_write32(GUEST_LDTR_LIMIT, 0xffff); 4920 vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082); 4921 4922 vmcs_writel(GUEST_GDTR_BASE, 0); 4923 vmcs_write32(GUEST_GDTR_LIMIT, 0xffff); 4924 4925 vmcs_writel(GUEST_IDTR_BASE, 0); 4926 vmcs_write32(GUEST_IDTR_LIMIT, 0xffff); 4927 4928 vmx_segment_cache_clear(vmx); 4929 kvm_register_mark_available(vcpu, VCPU_EXREG_SEGMENTS); 4930 4931 vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE); 4932 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0); 4933 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 0); 4934 if (kvm_mpx_supported()) 4935 vmcs_write64(GUEST_BNDCFGS, 0); 4936 4937 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); /* 22.2.1 */ 4938 4939 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 4940 4941 vpid_sync_context(vmx->vpid); 4942 4943 vmx_update_fb_clear_dis(vcpu, vmx); 4944 } 4945 4946 void vmx_enable_irq_window(struct kvm_vcpu *vcpu) 4947 { 4948 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING); 4949 } 4950 4951 void vmx_enable_nmi_window(struct kvm_vcpu *vcpu) 4952 { 4953 if (!enable_vnmi || 4954 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) { 4955 vmx_enable_irq_window(vcpu); 4956 return; 4957 } 4958 4959 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_NMI_WINDOW_EXITING); 4960 } 4961 4962 void vmx_inject_irq(struct kvm_vcpu *vcpu, bool reinjected) 4963 { 4964 struct vcpu_vmx *vmx = to_vmx(vcpu); 4965 uint32_t intr; 4966 int irq = vcpu->arch.interrupt.nr; 4967 4968 trace_kvm_inj_virq(irq, vcpu->arch.interrupt.soft, reinjected); 4969 4970 ++vcpu->stat.irq_injections; 4971 if (vmx->rmode.vm86_active) { 4972 int inc_eip = 0; 4973 if (vcpu->arch.interrupt.soft) 4974 inc_eip = vcpu->arch.event_exit_inst_len; 4975 kvm_inject_realmode_interrupt(vcpu, irq, inc_eip); 4976 return; 4977 } 4978 intr = irq | INTR_INFO_VALID_MASK; 4979 if (vcpu->arch.interrupt.soft) { 4980 intr |= INTR_TYPE_SOFT_INTR; 4981 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 4982 vmx->vcpu.arch.event_exit_inst_len); 4983 } else 4984 intr |= INTR_TYPE_EXT_INTR; 4985 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr); 4986 4987 vmx_clear_hlt(vcpu); 4988 } 4989 4990 void vmx_inject_nmi(struct kvm_vcpu *vcpu) 4991 { 4992 struct vcpu_vmx *vmx = to_vmx(vcpu); 4993 4994 if (!enable_vnmi) { 4995 /* 4996 * Tracking the NMI-blocked state in software is built upon 4997 * finding the next open IRQ window. This, in turn, depends on 4998 * well-behaving guests: They have to keep IRQs disabled at 4999 * least as long as the NMI handler runs. Otherwise we may 5000 * cause NMI nesting, maybe breaking the guest. But as this is 5001 * highly unlikely, we can live with the residual risk. 5002 */ 5003 vmx->loaded_vmcs->soft_vnmi_blocked = 1; 5004 vmx->loaded_vmcs->vnmi_blocked_time = 0; 5005 } 5006 5007 ++vcpu->stat.nmi_injections; 5008 vmx->loaded_vmcs->nmi_known_unmasked = false; 5009 5010 if (vmx->rmode.vm86_active) { 5011 kvm_inject_realmode_interrupt(vcpu, NMI_VECTOR, 0); 5012 return; 5013 } 5014 5015 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 5016 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR); 5017 5018 vmx_clear_hlt(vcpu); 5019 } 5020 5021 bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu) 5022 { 5023 struct vcpu_vmx *vmx = to_vmx(vcpu); 5024 bool masked; 5025 5026 if (!enable_vnmi) 5027 return vmx->loaded_vmcs->soft_vnmi_blocked; 5028 if (vmx->loaded_vmcs->nmi_known_unmasked) 5029 return false; 5030 masked = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_NMI; 5031 vmx->loaded_vmcs->nmi_known_unmasked = !masked; 5032 return masked; 5033 } 5034 5035 void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) 5036 { 5037 struct vcpu_vmx *vmx = to_vmx(vcpu); 5038 5039 if (!enable_vnmi) { 5040 if (vmx->loaded_vmcs->soft_vnmi_blocked != masked) { 5041 vmx->loaded_vmcs->soft_vnmi_blocked = masked; 5042 vmx->loaded_vmcs->vnmi_blocked_time = 0; 5043 } 5044 } else { 5045 vmx->loaded_vmcs->nmi_known_unmasked = !masked; 5046 if (masked) 5047 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 5048 GUEST_INTR_STATE_NMI); 5049 else 5050 vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO, 5051 GUEST_INTR_STATE_NMI); 5052 } 5053 } 5054 5055 bool vmx_nmi_blocked(struct kvm_vcpu *vcpu) 5056 { 5057 if (is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu)) 5058 return false; 5059 5060 if (!enable_vnmi && to_vmx(vcpu)->loaded_vmcs->soft_vnmi_blocked) 5061 return true; 5062 5063 return (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 5064 (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI | 5065 GUEST_INTR_STATE_NMI)); 5066 } 5067 5068 int vmx_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 5069 { 5070 if (to_vmx(vcpu)->nested.nested_run_pending) 5071 return -EBUSY; 5072 5073 /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */ 5074 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu)) 5075 return -EBUSY; 5076 5077 return !vmx_nmi_blocked(vcpu); 5078 } 5079 5080 bool __vmx_interrupt_blocked(struct kvm_vcpu *vcpu) 5081 { 5082 return !(vmx_get_rflags(vcpu) & X86_EFLAGS_IF) || 5083 (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 5084 (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS)); 5085 } 5086 5087 bool vmx_interrupt_blocked(struct kvm_vcpu *vcpu) 5088 { 5089 if (is_guest_mode(vcpu) && nested_exit_on_intr(vcpu)) 5090 return false; 5091 5092 return __vmx_interrupt_blocked(vcpu); 5093 } 5094 5095 int vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) 5096 { 5097 if (to_vmx(vcpu)->nested.nested_run_pending) 5098 return -EBUSY; 5099 5100 /* 5101 * An IRQ must not be injected into L2 if it's supposed to VM-Exit, 5102 * e.g. if the IRQ arrived asynchronously after checking nested events. 5103 */ 5104 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(vcpu)) 5105 return -EBUSY; 5106 5107 return !vmx_interrupt_blocked(vcpu); 5108 } 5109 5110 int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr) 5111 { 5112 void __user *ret; 5113 5114 if (enable_unrestricted_guest) 5115 return 0; 5116 5117 mutex_lock(&kvm->slots_lock); 5118 ret = __x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, addr, 5119 PAGE_SIZE * 3); 5120 mutex_unlock(&kvm->slots_lock); 5121 5122 if (IS_ERR(ret)) 5123 return PTR_ERR(ret); 5124 5125 to_kvm_vmx(kvm)->tss_addr = addr; 5126 5127 return init_rmode_tss(kvm, ret); 5128 } 5129 5130 int vmx_set_identity_map_addr(struct kvm *kvm, u64 ident_addr) 5131 { 5132 to_kvm_vmx(kvm)->ept_identity_map_addr = ident_addr; 5133 return 0; 5134 } 5135 5136 static bool rmode_exception(struct kvm_vcpu *vcpu, int vec) 5137 { 5138 switch (vec) { 5139 case BP_VECTOR: 5140 /* 5141 * Update instruction length as we may reinject the exception 5142 * from user space while in guest debugging mode. 5143 */ 5144 to_vmx(vcpu)->vcpu.arch.event_exit_inst_len = 5145 vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 5146 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) 5147 return false; 5148 fallthrough; 5149 case DB_VECTOR: 5150 return !(vcpu->guest_debug & 5151 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)); 5152 case DE_VECTOR: 5153 case OF_VECTOR: 5154 case BR_VECTOR: 5155 case UD_VECTOR: 5156 case DF_VECTOR: 5157 case SS_VECTOR: 5158 case GP_VECTOR: 5159 case MF_VECTOR: 5160 return true; 5161 } 5162 return false; 5163 } 5164 5165 static int handle_rmode_exception(struct kvm_vcpu *vcpu, 5166 int vec, u32 err_code) 5167 { 5168 /* 5169 * Instruction with address size override prefix opcode 0x67 5170 * Cause the #SS fault with 0 error code in VM86 mode. 5171 */ 5172 if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) { 5173 if (kvm_emulate_instruction(vcpu, 0)) { 5174 if (vcpu->arch.halt_request) { 5175 vcpu->arch.halt_request = 0; 5176 return kvm_emulate_halt_noskip(vcpu); 5177 } 5178 return 1; 5179 } 5180 return 0; 5181 } 5182 5183 /* 5184 * Forward all other exceptions that are valid in real mode. 5185 * FIXME: Breaks guest debugging in real mode, needs to be fixed with 5186 * the required debugging infrastructure rework. 5187 */ 5188 kvm_queue_exception(vcpu, vec); 5189 return 1; 5190 } 5191 5192 static int handle_machine_check(struct kvm_vcpu *vcpu) 5193 { 5194 /* handled by vmx_vcpu_run() */ 5195 return 1; 5196 } 5197 5198 /* 5199 * If the host has split lock detection disabled, then #AC is 5200 * unconditionally injected into the guest, which is the pre split lock 5201 * detection behaviour. 5202 * 5203 * If the host has split lock detection enabled then #AC is 5204 * only injected into the guest when: 5205 * - Guest CPL == 3 (user mode) 5206 * - Guest has #AC detection enabled in CR0 5207 * - Guest EFLAGS has AC bit set 5208 */ 5209 bool vmx_guest_inject_ac(struct kvm_vcpu *vcpu) 5210 { 5211 if (!boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT)) 5212 return true; 5213 5214 return vmx_get_cpl(vcpu) == 3 && kvm_is_cr0_bit_set(vcpu, X86_CR0_AM) && 5215 (kvm_get_rflags(vcpu) & X86_EFLAGS_AC); 5216 } 5217 5218 static int handle_exception_nmi(struct kvm_vcpu *vcpu) 5219 { 5220 struct vcpu_vmx *vmx = to_vmx(vcpu); 5221 struct kvm_run *kvm_run = vcpu->run; 5222 u32 intr_info, ex_no, error_code; 5223 unsigned long cr2, dr6; 5224 u32 vect_info; 5225 5226 vect_info = vmx->idt_vectoring_info; 5227 intr_info = vmx_get_intr_info(vcpu); 5228 5229 /* 5230 * Machine checks are handled by handle_exception_irqoff(), or by 5231 * vmx_vcpu_run() if a #MC occurs on VM-Entry. NMIs are handled by 5232 * vmx_vcpu_enter_exit(). 5233 */ 5234 if (is_machine_check(intr_info) || is_nmi(intr_info)) 5235 return 1; 5236 5237 /* 5238 * Queue the exception here instead of in handle_nm_fault_irqoff(). 5239 * This ensures the nested_vmx check is not skipped so vmexit can 5240 * be reflected to L1 (when it intercepts #NM) before reaching this 5241 * point. 5242 */ 5243 if (is_nm_fault(intr_info)) { 5244 kvm_queue_exception(vcpu, NM_VECTOR); 5245 return 1; 5246 } 5247 5248 if (is_invalid_opcode(intr_info)) 5249 return handle_ud(vcpu); 5250 5251 if (WARN_ON_ONCE(is_ve_fault(intr_info))) { 5252 struct vmx_ve_information *ve_info = vmx->ve_info; 5253 5254 WARN_ONCE(ve_info->exit_reason != EXIT_REASON_EPT_VIOLATION, 5255 "Unexpected #VE on VM-Exit reason 0x%x", ve_info->exit_reason); 5256 dump_vmcs(vcpu); 5257 kvm_mmu_print_sptes(vcpu, ve_info->guest_physical_address, "#VE"); 5258 return 1; 5259 } 5260 5261 error_code = 0; 5262 if (intr_info & INTR_INFO_DELIVER_CODE_MASK) 5263 error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); 5264 5265 if (!vmx->rmode.vm86_active && is_gp_fault(intr_info)) { 5266 WARN_ON_ONCE(!enable_vmware_backdoor); 5267 5268 /* 5269 * VMware backdoor emulation on #GP interception only handles 5270 * IN{S}, OUT{S}, and RDPMC, none of which generate a non-zero 5271 * error code on #GP. 5272 */ 5273 if (error_code) { 5274 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code); 5275 return 1; 5276 } 5277 return kvm_emulate_instruction(vcpu, EMULTYPE_VMWARE_GP); 5278 } 5279 5280 /* 5281 * The #PF with PFEC.RSVD = 1 indicates the guest is accessing 5282 * MMIO, it is better to report an internal error. 5283 * See the comments in vmx_handle_exit. 5284 */ 5285 if ((vect_info & VECTORING_INFO_VALID_MASK) && 5286 !(is_page_fault(intr_info) && !(error_code & PFERR_RSVD_MASK))) { 5287 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 5288 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX; 5289 vcpu->run->internal.ndata = 4; 5290 vcpu->run->internal.data[0] = vect_info; 5291 vcpu->run->internal.data[1] = intr_info; 5292 vcpu->run->internal.data[2] = error_code; 5293 vcpu->run->internal.data[3] = vcpu->arch.last_vmentry_cpu; 5294 return 0; 5295 } 5296 5297 if (is_page_fault(intr_info)) { 5298 cr2 = vmx_get_exit_qual(vcpu); 5299 if (enable_ept && !vcpu->arch.apf.host_apf_flags) { 5300 /* 5301 * EPT will cause page fault only if we need to 5302 * detect illegal GPAs. 5303 */ 5304 WARN_ON_ONCE(!allow_smaller_maxphyaddr); 5305 kvm_fixup_and_inject_pf_error(vcpu, cr2, error_code); 5306 return 1; 5307 } else 5308 return kvm_handle_page_fault(vcpu, error_code, cr2, NULL, 0); 5309 } 5310 5311 ex_no = intr_info & INTR_INFO_VECTOR_MASK; 5312 5313 if (vmx->rmode.vm86_active && rmode_exception(vcpu, ex_no)) 5314 return handle_rmode_exception(vcpu, ex_no, error_code); 5315 5316 switch (ex_no) { 5317 case DB_VECTOR: 5318 dr6 = vmx_get_exit_qual(vcpu); 5319 if (!(vcpu->guest_debug & 5320 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) { 5321 /* 5322 * If the #DB was due to ICEBP, a.k.a. INT1, skip the 5323 * instruction. ICEBP generates a trap-like #DB, but 5324 * despite its interception control being tied to #DB, 5325 * is an instruction intercept, i.e. the VM-Exit occurs 5326 * on the ICEBP itself. Use the inner "skip" helper to 5327 * avoid single-step #DB and MTF updates, as ICEBP is 5328 * higher priority. Note, skipping ICEBP still clears 5329 * STI and MOVSS blocking. 5330 * 5331 * For all other #DBs, set vmcs.PENDING_DBG_EXCEPTIONS.BS 5332 * if single-step is enabled in RFLAGS and STI or MOVSS 5333 * blocking is active, as the CPU doesn't set the bit 5334 * on VM-Exit due to #DB interception. VM-Entry has a 5335 * consistency check that a single-step #DB is pending 5336 * in this scenario as the previous instruction cannot 5337 * have toggled RFLAGS.TF 0=>1 (because STI and POP/MOV 5338 * don't modify RFLAGS), therefore the one instruction 5339 * delay when activating single-step breakpoints must 5340 * have already expired. Note, the CPU sets/clears BS 5341 * as appropriate for all other VM-Exits types. 5342 */ 5343 if (is_icebp(intr_info)) 5344 WARN_ON(!skip_emulated_instruction(vcpu)); 5345 else if ((vmx_get_rflags(vcpu) & X86_EFLAGS_TF) && 5346 (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 5347 (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS))) 5348 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 5349 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS) | DR6_BS); 5350 5351 kvm_queue_exception_p(vcpu, DB_VECTOR, dr6); 5352 return 1; 5353 } 5354 kvm_run->debug.arch.dr6 = dr6 | DR6_ACTIVE_LOW; 5355 kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7); 5356 fallthrough; 5357 case BP_VECTOR: 5358 /* 5359 * Update instruction length as we may reinject #BP from 5360 * user space while in guest debugging mode. Reading it for 5361 * #DB as well causes no harm, it is not used in that case. 5362 */ 5363 vmx->vcpu.arch.event_exit_inst_len = 5364 vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 5365 kvm_run->exit_reason = KVM_EXIT_DEBUG; 5366 kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu); 5367 kvm_run->debug.arch.exception = ex_no; 5368 break; 5369 case AC_VECTOR: 5370 if (vmx_guest_inject_ac(vcpu)) { 5371 kvm_queue_exception_e(vcpu, AC_VECTOR, error_code); 5372 return 1; 5373 } 5374 5375 /* 5376 * Handle split lock. Depending on detection mode this will 5377 * either warn and disable split lock detection for this 5378 * task or force SIGBUS on it. 5379 */ 5380 if (handle_guest_split_lock(kvm_rip_read(vcpu))) 5381 return 1; 5382 fallthrough; 5383 default: 5384 kvm_run->exit_reason = KVM_EXIT_EXCEPTION; 5385 kvm_run->ex.exception = ex_no; 5386 kvm_run->ex.error_code = error_code; 5387 break; 5388 } 5389 return 0; 5390 } 5391 5392 static __always_inline int handle_external_interrupt(struct kvm_vcpu *vcpu) 5393 { 5394 ++vcpu->stat.irq_exits; 5395 return 1; 5396 } 5397 5398 static int handle_triple_fault(struct kvm_vcpu *vcpu) 5399 { 5400 vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN; 5401 vcpu->mmio_needed = 0; 5402 return 0; 5403 } 5404 5405 static int handle_io(struct kvm_vcpu *vcpu) 5406 { 5407 unsigned long exit_qualification; 5408 int size, in, string; 5409 unsigned port; 5410 5411 exit_qualification = vmx_get_exit_qual(vcpu); 5412 string = (exit_qualification & 16) != 0; 5413 5414 ++vcpu->stat.io_exits; 5415 5416 if (string) 5417 return kvm_emulate_instruction(vcpu, 0); 5418 5419 port = exit_qualification >> 16; 5420 size = (exit_qualification & 7) + 1; 5421 in = (exit_qualification & 8) != 0; 5422 5423 return kvm_fast_pio(vcpu, size, port, in); 5424 } 5425 5426 void vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) 5427 { 5428 /* 5429 * Patch in the VMCALL instruction: 5430 */ 5431 hypercall[0] = 0x0f; 5432 hypercall[1] = 0x01; 5433 hypercall[2] = 0xc1; 5434 } 5435 5436 /* called to set cr0 as appropriate for a mov-to-cr0 exit. */ 5437 static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val) 5438 { 5439 if (is_guest_mode(vcpu)) { 5440 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5441 unsigned long orig_val = val; 5442 5443 /* 5444 * We get here when L2 changed cr0 in a way that did not change 5445 * any of L1's shadowed bits (see nested_vmx_exit_handled_cr), 5446 * but did change L0 shadowed bits. So we first calculate the 5447 * effective cr0 value that L1 would like to write into the 5448 * hardware. It consists of the L2-owned bits from the new 5449 * value combined with the L1-owned bits from L1's guest_cr0. 5450 */ 5451 val = (val & ~vmcs12->cr0_guest_host_mask) | 5452 (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask); 5453 5454 if (kvm_set_cr0(vcpu, val)) 5455 return 1; 5456 vmcs_writel(CR0_READ_SHADOW, orig_val); 5457 return 0; 5458 } else { 5459 return kvm_set_cr0(vcpu, val); 5460 } 5461 } 5462 5463 static int handle_set_cr4(struct kvm_vcpu *vcpu, unsigned long val) 5464 { 5465 if (is_guest_mode(vcpu)) { 5466 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5467 unsigned long orig_val = val; 5468 5469 /* analogously to handle_set_cr0 */ 5470 val = (val & ~vmcs12->cr4_guest_host_mask) | 5471 (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask); 5472 if (kvm_set_cr4(vcpu, val)) 5473 return 1; 5474 vmcs_writel(CR4_READ_SHADOW, orig_val); 5475 return 0; 5476 } else 5477 return kvm_set_cr4(vcpu, val); 5478 } 5479 5480 static int handle_desc(struct kvm_vcpu *vcpu) 5481 { 5482 /* 5483 * UMIP emulation relies on intercepting writes to CR4.UMIP, i.e. this 5484 * and other code needs to be updated if UMIP can be guest owned. 5485 */ 5486 BUILD_BUG_ON(KVM_POSSIBLE_CR4_GUEST_BITS & X86_CR4_UMIP); 5487 5488 WARN_ON_ONCE(!kvm_is_cr4_bit_set(vcpu, X86_CR4_UMIP)); 5489 return kvm_emulate_instruction(vcpu, 0); 5490 } 5491 5492 static int handle_cr(struct kvm_vcpu *vcpu) 5493 { 5494 unsigned long exit_qualification, val; 5495 int cr; 5496 int reg; 5497 int err; 5498 int ret; 5499 5500 exit_qualification = vmx_get_exit_qual(vcpu); 5501 cr = exit_qualification & 15; 5502 reg = (exit_qualification >> 8) & 15; 5503 switch ((exit_qualification >> 4) & 3) { 5504 case 0: /* mov to cr */ 5505 val = kvm_register_read(vcpu, reg); 5506 trace_kvm_cr_write(cr, val); 5507 switch (cr) { 5508 case 0: 5509 err = handle_set_cr0(vcpu, val); 5510 return kvm_complete_insn_gp(vcpu, err); 5511 case 3: 5512 WARN_ON_ONCE(enable_unrestricted_guest); 5513 5514 err = kvm_set_cr3(vcpu, val); 5515 return kvm_complete_insn_gp(vcpu, err); 5516 case 4: 5517 err = handle_set_cr4(vcpu, val); 5518 return kvm_complete_insn_gp(vcpu, err); 5519 case 8: { 5520 u8 cr8_prev = kvm_get_cr8(vcpu); 5521 u8 cr8 = (u8)val; 5522 err = kvm_set_cr8(vcpu, cr8); 5523 ret = kvm_complete_insn_gp(vcpu, err); 5524 if (lapic_in_kernel(vcpu)) 5525 return ret; 5526 if (cr8_prev <= cr8) 5527 return ret; 5528 /* 5529 * TODO: we might be squashing a 5530 * KVM_GUESTDBG_SINGLESTEP-triggered 5531 * KVM_EXIT_DEBUG here. 5532 */ 5533 vcpu->run->exit_reason = KVM_EXIT_SET_TPR; 5534 return 0; 5535 } 5536 } 5537 break; 5538 case 2: /* clts */ 5539 KVM_BUG(1, vcpu->kvm, "Guest always owns CR0.TS"); 5540 return -EIO; 5541 case 1: /*mov from cr*/ 5542 switch (cr) { 5543 case 3: 5544 WARN_ON_ONCE(enable_unrestricted_guest); 5545 5546 val = kvm_read_cr3(vcpu); 5547 kvm_register_write(vcpu, reg, val); 5548 trace_kvm_cr_read(cr, val); 5549 return kvm_skip_emulated_instruction(vcpu); 5550 case 8: 5551 val = kvm_get_cr8(vcpu); 5552 kvm_register_write(vcpu, reg, val); 5553 trace_kvm_cr_read(cr, val); 5554 return kvm_skip_emulated_instruction(vcpu); 5555 } 5556 break; 5557 case 3: /* lmsw */ 5558 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f; 5559 trace_kvm_cr_write(0, (kvm_read_cr0_bits(vcpu, ~0xful) | val)); 5560 kvm_lmsw(vcpu, val); 5561 5562 return kvm_skip_emulated_instruction(vcpu); 5563 default: 5564 break; 5565 } 5566 vcpu->run->exit_reason = 0; 5567 vcpu_unimpl(vcpu, "unhandled control register: op %d cr %d\n", 5568 (int)(exit_qualification >> 4) & 3, cr); 5569 return 0; 5570 } 5571 5572 static int handle_dr(struct kvm_vcpu *vcpu) 5573 { 5574 unsigned long exit_qualification; 5575 int dr, dr7, reg; 5576 int err = 1; 5577 5578 exit_qualification = vmx_get_exit_qual(vcpu); 5579 dr = exit_qualification & DEBUG_REG_ACCESS_NUM; 5580 5581 /* First, if DR does not exist, trigger UD */ 5582 if (!kvm_require_dr(vcpu, dr)) 5583 return 1; 5584 5585 if (vmx_get_cpl(vcpu) > 0) 5586 goto out; 5587 5588 dr7 = vmcs_readl(GUEST_DR7); 5589 if (dr7 & DR7_GD) { 5590 /* 5591 * As the vm-exit takes precedence over the debug trap, we 5592 * need to emulate the latter, either for the host or the 5593 * guest debugging itself. 5594 */ 5595 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) { 5596 vcpu->run->debug.arch.dr6 = DR6_BD | DR6_ACTIVE_LOW; 5597 vcpu->run->debug.arch.dr7 = dr7; 5598 vcpu->run->debug.arch.pc = kvm_get_linear_rip(vcpu); 5599 vcpu->run->debug.arch.exception = DB_VECTOR; 5600 vcpu->run->exit_reason = KVM_EXIT_DEBUG; 5601 return 0; 5602 } else { 5603 kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BD); 5604 return 1; 5605 } 5606 } 5607 5608 if (vcpu->guest_debug == 0) { 5609 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_MOV_DR_EXITING); 5610 5611 /* 5612 * No more DR vmexits; force a reload of the debug registers 5613 * and reenter on this instruction. The next vmexit will 5614 * retrieve the full state of the debug registers. 5615 */ 5616 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT; 5617 return 1; 5618 } 5619 5620 reg = DEBUG_REG_ACCESS_REG(exit_qualification); 5621 if (exit_qualification & TYPE_MOV_FROM_DR) { 5622 kvm_register_write(vcpu, reg, kvm_get_dr(vcpu, dr)); 5623 err = 0; 5624 } else { 5625 err = kvm_set_dr(vcpu, dr, kvm_register_read(vcpu, reg)); 5626 } 5627 5628 out: 5629 return kvm_complete_insn_gp(vcpu, err); 5630 } 5631 5632 void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu) 5633 { 5634 get_debugreg(vcpu->arch.db[0], 0); 5635 get_debugreg(vcpu->arch.db[1], 1); 5636 get_debugreg(vcpu->arch.db[2], 2); 5637 get_debugreg(vcpu->arch.db[3], 3); 5638 get_debugreg(vcpu->arch.dr6, 6); 5639 vcpu->arch.dr7 = vmcs_readl(GUEST_DR7); 5640 5641 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT; 5642 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_MOV_DR_EXITING); 5643 5644 /* 5645 * exc_debug expects dr6 to be cleared after it runs, avoid that it sees 5646 * a stale dr6 from the guest. 5647 */ 5648 set_debugreg(DR6_RESERVED, 6); 5649 } 5650 5651 void vmx_set_dr6(struct kvm_vcpu *vcpu, unsigned long val) 5652 { 5653 lockdep_assert_irqs_disabled(); 5654 set_debugreg(vcpu->arch.dr6, 6); 5655 } 5656 5657 void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val) 5658 { 5659 vmcs_writel(GUEST_DR7, val); 5660 } 5661 5662 static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu) 5663 { 5664 kvm_apic_update_ppr(vcpu); 5665 return 1; 5666 } 5667 5668 static int handle_interrupt_window(struct kvm_vcpu *vcpu) 5669 { 5670 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING); 5671 5672 kvm_make_request(KVM_REQ_EVENT, vcpu); 5673 5674 ++vcpu->stat.irq_window_exits; 5675 return 1; 5676 } 5677 5678 static int handle_invlpg(struct kvm_vcpu *vcpu) 5679 { 5680 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5681 5682 kvm_mmu_invlpg(vcpu, exit_qualification); 5683 return kvm_skip_emulated_instruction(vcpu); 5684 } 5685 5686 static int handle_apic_access(struct kvm_vcpu *vcpu) 5687 { 5688 if (likely(fasteoi)) { 5689 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5690 int access_type, offset; 5691 5692 access_type = exit_qualification & APIC_ACCESS_TYPE; 5693 offset = exit_qualification & APIC_ACCESS_OFFSET; 5694 /* 5695 * Sane guest uses MOV to write EOI, with written value 5696 * not cared. So make a short-circuit here by avoiding 5697 * heavy instruction emulation. 5698 */ 5699 if ((access_type == TYPE_LINEAR_APIC_INST_WRITE) && 5700 (offset == APIC_EOI)) { 5701 kvm_lapic_set_eoi(vcpu); 5702 return kvm_skip_emulated_instruction(vcpu); 5703 } 5704 } 5705 return kvm_emulate_instruction(vcpu, 0); 5706 } 5707 5708 static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu) 5709 { 5710 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5711 int vector = exit_qualification & 0xff; 5712 5713 /* EOI-induced VM exit is trap-like and thus no need to adjust IP */ 5714 kvm_apic_set_eoi_accelerated(vcpu, vector); 5715 return 1; 5716 } 5717 5718 static int handle_apic_write(struct kvm_vcpu *vcpu) 5719 { 5720 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5721 5722 /* 5723 * APIC-write VM-Exit is trap-like, KVM doesn't need to advance RIP and 5724 * hardware has done any necessary aliasing, offset adjustments, etc... 5725 * for the access. I.e. the correct value has already been written to 5726 * the vAPIC page for the correct 16-byte chunk. KVM needs only to 5727 * retrieve the register value and emulate the access. 5728 */ 5729 u32 offset = exit_qualification & 0xff0; 5730 5731 kvm_apic_write_nodecode(vcpu, offset); 5732 return 1; 5733 } 5734 5735 static int handle_task_switch(struct kvm_vcpu *vcpu) 5736 { 5737 struct vcpu_vmx *vmx = to_vmx(vcpu); 5738 unsigned long exit_qualification; 5739 bool has_error_code = false; 5740 u32 error_code = 0; 5741 u16 tss_selector; 5742 int reason, type, idt_v, idt_index; 5743 5744 idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK); 5745 idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK); 5746 type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK); 5747 5748 exit_qualification = vmx_get_exit_qual(vcpu); 5749 5750 reason = (u32)exit_qualification >> 30; 5751 if (reason == TASK_SWITCH_GATE && idt_v) { 5752 switch (type) { 5753 case INTR_TYPE_NMI_INTR: 5754 vcpu->arch.nmi_injected = false; 5755 vmx_set_nmi_mask(vcpu, true); 5756 break; 5757 case INTR_TYPE_EXT_INTR: 5758 case INTR_TYPE_SOFT_INTR: 5759 kvm_clear_interrupt_queue(vcpu); 5760 break; 5761 case INTR_TYPE_HARD_EXCEPTION: 5762 if (vmx->idt_vectoring_info & 5763 VECTORING_INFO_DELIVER_CODE_MASK) { 5764 has_error_code = true; 5765 error_code = 5766 vmcs_read32(IDT_VECTORING_ERROR_CODE); 5767 } 5768 fallthrough; 5769 case INTR_TYPE_SOFT_EXCEPTION: 5770 kvm_clear_exception_queue(vcpu); 5771 break; 5772 default: 5773 break; 5774 } 5775 } 5776 tss_selector = exit_qualification; 5777 5778 if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION && 5779 type != INTR_TYPE_EXT_INTR && 5780 type != INTR_TYPE_NMI_INTR)) 5781 WARN_ON(!skip_emulated_instruction(vcpu)); 5782 5783 /* 5784 * TODO: What about debug traps on tss switch? 5785 * Are we supposed to inject them and update dr6? 5786 */ 5787 return kvm_task_switch(vcpu, tss_selector, 5788 type == INTR_TYPE_SOFT_INTR ? idt_index : -1, 5789 reason, has_error_code, error_code); 5790 } 5791 5792 static int handle_ept_violation(struct kvm_vcpu *vcpu) 5793 { 5794 unsigned long exit_qualification; 5795 gpa_t gpa; 5796 u64 error_code; 5797 5798 exit_qualification = vmx_get_exit_qual(vcpu); 5799 5800 /* 5801 * EPT violation happened while executing iret from NMI, 5802 * "blocked by NMI" bit has to be set before next VM entry. 5803 * There are errata that may cause this bit to not be set: 5804 * AAK134, BY25. 5805 */ 5806 if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) && 5807 enable_vnmi && 5808 (exit_qualification & INTR_INFO_UNBLOCK_NMI)) 5809 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI); 5810 5811 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS); 5812 trace_kvm_page_fault(vcpu, gpa, exit_qualification); 5813 5814 /* Is it a read fault? */ 5815 error_code = (exit_qualification & EPT_VIOLATION_ACC_READ) 5816 ? PFERR_USER_MASK : 0; 5817 /* Is it a write fault? */ 5818 error_code |= (exit_qualification & EPT_VIOLATION_ACC_WRITE) 5819 ? PFERR_WRITE_MASK : 0; 5820 /* Is it a fetch fault? */ 5821 error_code |= (exit_qualification & EPT_VIOLATION_ACC_INSTR) 5822 ? PFERR_FETCH_MASK : 0; 5823 /* ept page table entry is present? */ 5824 error_code |= (exit_qualification & EPT_VIOLATION_RWX_MASK) 5825 ? PFERR_PRESENT_MASK : 0; 5826 5827 if (error_code & EPT_VIOLATION_GVA_IS_VALID) 5828 error_code |= (exit_qualification & EPT_VIOLATION_GVA_TRANSLATED) ? 5829 PFERR_GUEST_FINAL_MASK : PFERR_GUEST_PAGE_MASK; 5830 5831 /* 5832 * Check that the GPA doesn't exceed physical memory limits, as that is 5833 * a guest page fault. We have to emulate the instruction here, because 5834 * if the illegal address is that of a paging structure, then 5835 * EPT_VIOLATION_ACC_WRITE bit is set. Alternatively, if supported we 5836 * would also use advanced VM-exit information for EPT violations to 5837 * reconstruct the page fault error code. 5838 */ 5839 if (unlikely(allow_smaller_maxphyaddr && !kvm_vcpu_is_legal_gpa(vcpu, gpa))) 5840 return kvm_emulate_instruction(vcpu, 0); 5841 5842 return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0); 5843 } 5844 5845 static int handle_ept_misconfig(struct kvm_vcpu *vcpu) 5846 { 5847 gpa_t gpa; 5848 5849 if (vmx_check_emulate_instruction(vcpu, EMULTYPE_PF, NULL, 0)) 5850 return 1; 5851 5852 /* 5853 * A nested guest cannot optimize MMIO vmexits, because we have an 5854 * nGPA here instead of the required GPA. 5855 */ 5856 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS); 5857 if (!is_guest_mode(vcpu) && 5858 !kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) { 5859 trace_kvm_fast_mmio(gpa); 5860 return kvm_skip_emulated_instruction(vcpu); 5861 } 5862 5863 return kvm_mmu_page_fault(vcpu, gpa, PFERR_RSVD_MASK, NULL, 0); 5864 } 5865 5866 static int handle_nmi_window(struct kvm_vcpu *vcpu) 5867 { 5868 if (KVM_BUG_ON(!enable_vnmi, vcpu->kvm)) 5869 return -EIO; 5870 5871 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_NMI_WINDOW_EXITING); 5872 ++vcpu->stat.nmi_window_exits; 5873 kvm_make_request(KVM_REQ_EVENT, vcpu); 5874 5875 return 1; 5876 } 5877 5878 static bool vmx_emulation_required_with_pending_exception(struct kvm_vcpu *vcpu) 5879 { 5880 struct vcpu_vmx *vmx = to_vmx(vcpu); 5881 5882 return vmx->emulation_required && !vmx->rmode.vm86_active && 5883 (kvm_is_exception_pending(vcpu) || vcpu->arch.exception.injected); 5884 } 5885 5886 static int handle_invalid_guest_state(struct kvm_vcpu *vcpu) 5887 { 5888 struct vcpu_vmx *vmx = to_vmx(vcpu); 5889 bool intr_window_requested; 5890 unsigned count = 130; 5891 5892 intr_window_requested = exec_controls_get(vmx) & 5893 CPU_BASED_INTR_WINDOW_EXITING; 5894 5895 while (vmx->emulation_required && count-- != 0) { 5896 if (intr_window_requested && !vmx_interrupt_blocked(vcpu)) 5897 return handle_interrupt_window(&vmx->vcpu); 5898 5899 if (kvm_test_request(KVM_REQ_EVENT, vcpu)) 5900 return 1; 5901 5902 if (!kvm_emulate_instruction(vcpu, 0)) 5903 return 0; 5904 5905 if (vmx_emulation_required_with_pending_exception(vcpu)) { 5906 kvm_prepare_emulation_failure_exit(vcpu); 5907 return 0; 5908 } 5909 5910 if (vcpu->arch.halt_request) { 5911 vcpu->arch.halt_request = 0; 5912 return kvm_emulate_halt_noskip(vcpu); 5913 } 5914 5915 /* 5916 * Note, return 1 and not 0, vcpu_run() will invoke 5917 * xfer_to_guest_mode() which will create a proper return 5918 * code. 5919 */ 5920 if (__xfer_to_guest_mode_work_pending()) 5921 return 1; 5922 } 5923 5924 return 1; 5925 } 5926 5927 int vmx_vcpu_pre_run(struct kvm_vcpu *vcpu) 5928 { 5929 if (vmx_emulation_required_with_pending_exception(vcpu)) { 5930 kvm_prepare_emulation_failure_exit(vcpu); 5931 return 0; 5932 } 5933 5934 return 1; 5935 } 5936 5937 /* 5938 * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE 5939 * exiting, so only get here on cpu with PAUSE-Loop-Exiting. 5940 */ 5941 static int handle_pause(struct kvm_vcpu *vcpu) 5942 { 5943 if (!kvm_pause_in_guest(vcpu->kvm)) 5944 grow_ple_window(vcpu); 5945 5946 /* 5947 * Intel sdm vol3 ch-25.1.3 says: The "PAUSE-loop exiting" 5948 * VM-execution control is ignored if CPL > 0. OTOH, KVM 5949 * never set PAUSE_EXITING and just set PLE if supported, 5950 * so the vcpu must be CPL=0 if it gets a PAUSE exit. 5951 */ 5952 kvm_vcpu_on_spin(vcpu, true); 5953 return kvm_skip_emulated_instruction(vcpu); 5954 } 5955 5956 static int handle_monitor_trap(struct kvm_vcpu *vcpu) 5957 { 5958 return 1; 5959 } 5960 5961 static int handle_invpcid(struct kvm_vcpu *vcpu) 5962 { 5963 u32 vmx_instruction_info; 5964 unsigned long type; 5965 gva_t gva; 5966 struct { 5967 u64 pcid; 5968 u64 gla; 5969 } operand; 5970 int gpr_index; 5971 5972 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_INVPCID)) { 5973 kvm_queue_exception(vcpu, UD_VECTOR); 5974 return 1; 5975 } 5976 5977 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5978 gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info); 5979 type = kvm_register_read(vcpu, gpr_index); 5980 5981 /* According to the Intel instruction reference, the memory operand 5982 * is read even if it isn't needed (e.g., for type==all) 5983 */ 5984 if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), 5985 vmx_instruction_info, false, 5986 sizeof(operand), &gva)) 5987 return 1; 5988 5989 return kvm_handle_invpcid(vcpu, type, gva); 5990 } 5991 5992 static int handle_pml_full(struct kvm_vcpu *vcpu) 5993 { 5994 unsigned long exit_qualification; 5995 5996 trace_kvm_pml_full(vcpu->vcpu_id); 5997 5998 exit_qualification = vmx_get_exit_qual(vcpu); 5999 6000 /* 6001 * PML buffer FULL happened while executing iret from NMI, 6002 * "blocked by NMI" bit has to be set before next VM entry. 6003 */ 6004 if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) && 6005 enable_vnmi && 6006 (exit_qualification & INTR_INFO_UNBLOCK_NMI)) 6007 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 6008 GUEST_INTR_STATE_NMI); 6009 6010 /* 6011 * PML buffer already flushed at beginning of VMEXIT. Nothing to do 6012 * here.., and there's no userspace involvement needed for PML. 6013 */ 6014 return 1; 6015 } 6016 6017 static fastpath_t handle_fastpath_preemption_timer(struct kvm_vcpu *vcpu, 6018 bool force_immediate_exit) 6019 { 6020 struct vcpu_vmx *vmx = to_vmx(vcpu); 6021 6022 /* 6023 * In the *extremely* unlikely scenario that this is a spurious VM-Exit 6024 * due to the timer expiring while it was "soft" disabled, just eat the 6025 * exit and re-enter the guest. 6026 */ 6027 if (unlikely(vmx->loaded_vmcs->hv_timer_soft_disabled)) 6028 return EXIT_FASTPATH_REENTER_GUEST; 6029 6030 /* 6031 * If the timer expired because KVM used it to force an immediate exit, 6032 * then mission accomplished. 6033 */ 6034 if (force_immediate_exit) 6035 return EXIT_FASTPATH_EXIT_HANDLED; 6036 6037 /* 6038 * If L2 is active, go down the slow path as emulating the guest timer 6039 * expiration likely requires synthesizing a nested VM-Exit. 6040 */ 6041 if (is_guest_mode(vcpu)) 6042 return EXIT_FASTPATH_NONE; 6043 6044 kvm_lapic_expired_hv_timer(vcpu); 6045 return EXIT_FASTPATH_REENTER_GUEST; 6046 } 6047 6048 static int handle_preemption_timer(struct kvm_vcpu *vcpu) 6049 { 6050 /* 6051 * This non-fastpath handler is reached if and only if the preemption 6052 * timer was being used to emulate a guest timer while L2 is active. 6053 * All other scenarios are supposed to be handled in the fastpath. 6054 */ 6055 WARN_ON_ONCE(!is_guest_mode(vcpu)); 6056 kvm_lapic_expired_hv_timer(vcpu); 6057 return 1; 6058 } 6059 6060 /* 6061 * When nested=0, all VMX instruction VM Exits filter here. The handlers 6062 * are overwritten by nested_vmx_hardware_setup() when nested=1. 6063 */ 6064 static int handle_vmx_instruction(struct kvm_vcpu *vcpu) 6065 { 6066 kvm_queue_exception(vcpu, UD_VECTOR); 6067 return 1; 6068 } 6069 6070 #ifndef CONFIG_X86_SGX_KVM 6071 static int handle_encls(struct kvm_vcpu *vcpu) 6072 { 6073 /* 6074 * SGX virtualization is disabled. There is no software enable bit for 6075 * SGX, so KVM intercepts all ENCLS leafs and injects a #UD to prevent 6076 * the guest from executing ENCLS (when SGX is supported by hardware). 6077 */ 6078 kvm_queue_exception(vcpu, UD_VECTOR); 6079 return 1; 6080 } 6081 #endif /* CONFIG_X86_SGX_KVM */ 6082 6083 static int handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) 6084 { 6085 /* 6086 * Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK 6087 * VM-Exits. Unconditionally set the flag here and leave the handling to 6088 * vmx_handle_exit(). 6089 */ 6090 to_vmx(vcpu)->exit_reason.bus_lock_detected = true; 6091 return 1; 6092 } 6093 6094 static int handle_notify(struct kvm_vcpu *vcpu) 6095 { 6096 unsigned long exit_qual = vmx_get_exit_qual(vcpu); 6097 bool context_invalid = exit_qual & NOTIFY_VM_CONTEXT_INVALID; 6098 6099 ++vcpu->stat.notify_window_exits; 6100 6101 /* 6102 * Notify VM exit happened while executing iret from NMI, 6103 * "blocked by NMI" bit has to be set before next VM entry. 6104 */ 6105 if (enable_vnmi && (exit_qual & INTR_INFO_UNBLOCK_NMI)) 6106 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 6107 GUEST_INTR_STATE_NMI); 6108 6109 if (vcpu->kvm->arch.notify_vmexit_flags & KVM_X86_NOTIFY_VMEXIT_USER || 6110 context_invalid) { 6111 vcpu->run->exit_reason = KVM_EXIT_NOTIFY; 6112 vcpu->run->notify.flags = context_invalid ? 6113 KVM_NOTIFY_CONTEXT_INVALID : 0; 6114 return 0; 6115 } 6116 6117 return 1; 6118 } 6119 6120 /* 6121 * The exit handlers return 1 if the exit was handled fully and guest execution 6122 * may resume. Otherwise they set the kvm_run parameter to indicate what needs 6123 * to be done to userspace and return 0. 6124 */ 6125 static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { 6126 [EXIT_REASON_EXCEPTION_NMI] = handle_exception_nmi, 6127 [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt, 6128 [EXIT_REASON_TRIPLE_FAULT] = handle_triple_fault, 6129 [EXIT_REASON_NMI_WINDOW] = handle_nmi_window, 6130 [EXIT_REASON_IO_INSTRUCTION] = handle_io, 6131 [EXIT_REASON_CR_ACCESS] = handle_cr, 6132 [EXIT_REASON_DR_ACCESS] = handle_dr, 6133 [EXIT_REASON_CPUID] = kvm_emulate_cpuid, 6134 [EXIT_REASON_MSR_READ] = kvm_emulate_rdmsr, 6135 [EXIT_REASON_MSR_WRITE] = kvm_emulate_wrmsr, 6136 [EXIT_REASON_INTERRUPT_WINDOW] = handle_interrupt_window, 6137 [EXIT_REASON_HLT] = kvm_emulate_halt, 6138 [EXIT_REASON_INVD] = kvm_emulate_invd, 6139 [EXIT_REASON_INVLPG] = handle_invlpg, 6140 [EXIT_REASON_RDPMC] = kvm_emulate_rdpmc, 6141 [EXIT_REASON_VMCALL] = kvm_emulate_hypercall, 6142 [EXIT_REASON_VMCLEAR] = handle_vmx_instruction, 6143 [EXIT_REASON_VMLAUNCH] = handle_vmx_instruction, 6144 [EXIT_REASON_VMPTRLD] = handle_vmx_instruction, 6145 [EXIT_REASON_VMPTRST] = handle_vmx_instruction, 6146 [EXIT_REASON_VMREAD] = handle_vmx_instruction, 6147 [EXIT_REASON_VMRESUME] = handle_vmx_instruction, 6148 [EXIT_REASON_VMWRITE] = handle_vmx_instruction, 6149 [EXIT_REASON_VMOFF] = handle_vmx_instruction, 6150 [EXIT_REASON_VMON] = handle_vmx_instruction, 6151 [EXIT_REASON_TPR_BELOW_THRESHOLD] = handle_tpr_below_threshold, 6152 [EXIT_REASON_APIC_ACCESS] = handle_apic_access, 6153 [EXIT_REASON_APIC_WRITE] = handle_apic_write, 6154 [EXIT_REASON_EOI_INDUCED] = handle_apic_eoi_induced, 6155 [EXIT_REASON_WBINVD] = kvm_emulate_wbinvd, 6156 [EXIT_REASON_XSETBV] = kvm_emulate_xsetbv, 6157 [EXIT_REASON_TASK_SWITCH] = handle_task_switch, 6158 [EXIT_REASON_MCE_DURING_VMENTRY] = handle_machine_check, 6159 [EXIT_REASON_GDTR_IDTR] = handle_desc, 6160 [EXIT_REASON_LDTR_TR] = handle_desc, 6161 [EXIT_REASON_EPT_VIOLATION] = handle_ept_violation, 6162 [EXIT_REASON_EPT_MISCONFIG] = handle_ept_misconfig, 6163 [EXIT_REASON_PAUSE_INSTRUCTION] = handle_pause, 6164 [EXIT_REASON_MWAIT_INSTRUCTION] = kvm_emulate_mwait, 6165 [EXIT_REASON_MONITOR_TRAP_FLAG] = handle_monitor_trap, 6166 [EXIT_REASON_MONITOR_INSTRUCTION] = kvm_emulate_monitor, 6167 [EXIT_REASON_INVEPT] = handle_vmx_instruction, 6168 [EXIT_REASON_INVVPID] = handle_vmx_instruction, 6169 [EXIT_REASON_RDRAND] = kvm_handle_invalid_op, 6170 [EXIT_REASON_RDSEED] = kvm_handle_invalid_op, 6171 [EXIT_REASON_PML_FULL] = handle_pml_full, 6172 [EXIT_REASON_INVPCID] = handle_invpcid, 6173 [EXIT_REASON_VMFUNC] = handle_vmx_instruction, 6174 [EXIT_REASON_PREEMPTION_TIMER] = handle_preemption_timer, 6175 [EXIT_REASON_ENCLS] = handle_encls, 6176 [EXIT_REASON_BUS_LOCK] = handle_bus_lock_vmexit, 6177 [EXIT_REASON_NOTIFY] = handle_notify, 6178 }; 6179 6180 static const int kvm_vmx_max_exit_handlers = 6181 ARRAY_SIZE(kvm_vmx_exit_handlers); 6182 6183 void vmx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, 6184 u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code) 6185 { 6186 struct vcpu_vmx *vmx = to_vmx(vcpu); 6187 6188 *reason = vmx->exit_reason.full; 6189 *info1 = vmx_get_exit_qual(vcpu); 6190 if (!(vmx->exit_reason.failed_vmentry)) { 6191 *info2 = vmx->idt_vectoring_info; 6192 *intr_info = vmx_get_intr_info(vcpu); 6193 if (is_exception_with_error_code(*intr_info)) 6194 *error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); 6195 else 6196 *error_code = 0; 6197 } else { 6198 *info2 = 0; 6199 *intr_info = 0; 6200 *error_code = 0; 6201 } 6202 } 6203 6204 void vmx_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info, u32 *error_code) 6205 { 6206 *intr_info = vmcs_read32(VM_ENTRY_INTR_INFO_FIELD); 6207 if (is_exception_with_error_code(*intr_info)) 6208 *error_code = vmcs_read32(VM_ENTRY_EXCEPTION_ERROR_CODE); 6209 else 6210 *error_code = 0; 6211 } 6212 6213 static void vmx_destroy_pml_buffer(struct vcpu_vmx *vmx) 6214 { 6215 if (vmx->pml_pg) { 6216 __free_page(vmx->pml_pg); 6217 vmx->pml_pg = NULL; 6218 } 6219 } 6220 6221 static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu) 6222 { 6223 struct vcpu_vmx *vmx = to_vmx(vcpu); 6224 u16 pml_idx, pml_tail_index; 6225 u64 *pml_buf; 6226 int i; 6227 6228 pml_idx = vmcs_read16(GUEST_PML_INDEX); 6229 6230 /* Do nothing if PML buffer is empty */ 6231 if (pml_idx == PML_HEAD_INDEX) 6232 return; 6233 /* 6234 * PML index always points to the next available PML buffer entity 6235 * unless PML log has just overflowed. 6236 */ 6237 pml_tail_index = (pml_idx >= PML_LOG_NR_ENTRIES) ? 0 : pml_idx + 1; 6238 6239 /* 6240 * PML log is written backwards: the CPU first writes the entry 511 6241 * then the entry 510, and so on. 6242 * 6243 * Read the entries in the same order they were written, to ensure that 6244 * the dirty ring is filled in the same order the CPU wrote them. 6245 */ 6246 pml_buf = page_address(vmx->pml_pg); 6247 6248 for (i = PML_HEAD_INDEX; i >= pml_tail_index; i--) { 6249 u64 gpa; 6250 6251 gpa = pml_buf[i]; 6252 WARN_ON(gpa & (PAGE_SIZE - 1)); 6253 kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT); 6254 } 6255 6256 /* reset PML index */ 6257 vmcs_write16(GUEST_PML_INDEX, PML_HEAD_INDEX); 6258 } 6259 6260 static void vmx_dump_sel(char *name, uint32_t sel) 6261 { 6262 pr_err("%s sel=0x%04x, attr=0x%05x, limit=0x%08x, base=0x%016lx\n", 6263 name, vmcs_read16(sel), 6264 vmcs_read32(sel + GUEST_ES_AR_BYTES - GUEST_ES_SELECTOR), 6265 vmcs_read32(sel + GUEST_ES_LIMIT - GUEST_ES_SELECTOR), 6266 vmcs_readl(sel + GUEST_ES_BASE - GUEST_ES_SELECTOR)); 6267 } 6268 6269 static void vmx_dump_dtsel(char *name, uint32_t limit) 6270 { 6271 pr_err("%s limit=0x%08x, base=0x%016lx\n", 6272 name, vmcs_read32(limit), 6273 vmcs_readl(limit + GUEST_GDTR_BASE - GUEST_GDTR_LIMIT)); 6274 } 6275 6276 static void vmx_dump_msrs(char *name, struct vmx_msrs *m) 6277 { 6278 unsigned int i; 6279 struct vmx_msr_entry *e; 6280 6281 pr_err("MSR %s:\n", name); 6282 for (i = 0, e = m->val; i < m->nr; ++i, ++e) 6283 pr_err(" %2d: msr=0x%08x value=0x%016llx\n", i, e->index, e->value); 6284 } 6285 6286 void dump_vmcs(struct kvm_vcpu *vcpu) 6287 { 6288 struct vcpu_vmx *vmx = to_vmx(vcpu); 6289 u32 vmentry_ctl, vmexit_ctl; 6290 u32 cpu_based_exec_ctrl, pin_based_exec_ctrl, secondary_exec_control; 6291 u64 tertiary_exec_control; 6292 unsigned long cr4; 6293 int efer_slot; 6294 6295 if (!dump_invalid_vmcs) { 6296 pr_warn_ratelimited("set kvm_intel.dump_invalid_vmcs=1 to dump internal KVM state.\n"); 6297 return; 6298 } 6299 6300 vmentry_ctl = vmcs_read32(VM_ENTRY_CONTROLS); 6301 vmexit_ctl = vmcs_read32(VM_EXIT_CONTROLS); 6302 cpu_based_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL); 6303 pin_based_exec_ctrl = vmcs_read32(PIN_BASED_VM_EXEC_CONTROL); 6304 cr4 = vmcs_readl(GUEST_CR4); 6305 6306 if (cpu_has_secondary_exec_ctrls()) 6307 secondary_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL); 6308 else 6309 secondary_exec_control = 0; 6310 6311 if (cpu_has_tertiary_exec_ctrls()) 6312 tertiary_exec_control = vmcs_read64(TERTIARY_VM_EXEC_CONTROL); 6313 else 6314 tertiary_exec_control = 0; 6315 6316 pr_err("VMCS %p, last attempted VM-entry on CPU %d\n", 6317 vmx->loaded_vmcs->vmcs, vcpu->arch.last_vmentry_cpu); 6318 pr_err("*** Guest State ***\n"); 6319 pr_err("CR0: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n", 6320 vmcs_readl(GUEST_CR0), vmcs_readl(CR0_READ_SHADOW), 6321 vmcs_readl(CR0_GUEST_HOST_MASK)); 6322 pr_err("CR4: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n", 6323 cr4, vmcs_readl(CR4_READ_SHADOW), vmcs_readl(CR4_GUEST_HOST_MASK)); 6324 pr_err("CR3 = 0x%016lx\n", vmcs_readl(GUEST_CR3)); 6325 if (cpu_has_vmx_ept()) { 6326 pr_err("PDPTR0 = 0x%016llx PDPTR1 = 0x%016llx\n", 6327 vmcs_read64(GUEST_PDPTR0), vmcs_read64(GUEST_PDPTR1)); 6328 pr_err("PDPTR2 = 0x%016llx PDPTR3 = 0x%016llx\n", 6329 vmcs_read64(GUEST_PDPTR2), vmcs_read64(GUEST_PDPTR3)); 6330 } 6331 pr_err("RSP = 0x%016lx RIP = 0x%016lx\n", 6332 vmcs_readl(GUEST_RSP), vmcs_readl(GUEST_RIP)); 6333 pr_err("RFLAGS=0x%08lx DR7 = 0x%016lx\n", 6334 vmcs_readl(GUEST_RFLAGS), vmcs_readl(GUEST_DR7)); 6335 pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n", 6336 vmcs_readl(GUEST_SYSENTER_ESP), 6337 vmcs_read32(GUEST_SYSENTER_CS), vmcs_readl(GUEST_SYSENTER_EIP)); 6338 vmx_dump_sel("CS: ", GUEST_CS_SELECTOR); 6339 vmx_dump_sel("DS: ", GUEST_DS_SELECTOR); 6340 vmx_dump_sel("SS: ", GUEST_SS_SELECTOR); 6341 vmx_dump_sel("ES: ", GUEST_ES_SELECTOR); 6342 vmx_dump_sel("FS: ", GUEST_FS_SELECTOR); 6343 vmx_dump_sel("GS: ", GUEST_GS_SELECTOR); 6344 vmx_dump_dtsel("GDTR:", GUEST_GDTR_LIMIT); 6345 vmx_dump_sel("LDTR:", GUEST_LDTR_SELECTOR); 6346 vmx_dump_dtsel("IDTR:", GUEST_IDTR_LIMIT); 6347 vmx_dump_sel("TR: ", GUEST_TR_SELECTOR); 6348 efer_slot = vmx_find_loadstore_msr_slot(&vmx->msr_autoload.guest, MSR_EFER); 6349 if (vmentry_ctl & VM_ENTRY_LOAD_IA32_EFER) 6350 pr_err("EFER= 0x%016llx\n", vmcs_read64(GUEST_IA32_EFER)); 6351 else if (efer_slot >= 0) 6352 pr_err("EFER= 0x%016llx (autoload)\n", 6353 vmx->msr_autoload.guest.val[efer_slot].value); 6354 else if (vmentry_ctl & VM_ENTRY_IA32E_MODE) 6355 pr_err("EFER= 0x%016llx (effective)\n", 6356 vcpu->arch.efer | (EFER_LMA | EFER_LME)); 6357 else 6358 pr_err("EFER= 0x%016llx (effective)\n", 6359 vcpu->arch.efer & ~(EFER_LMA | EFER_LME)); 6360 if (vmentry_ctl & VM_ENTRY_LOAD_IA32_PAT) 6361 pr_err("PAT = 0x%016llx\n", vmcs_read64(GUEST_IA32_PAT)); 6362 pr_err("DebugCtl = 0x%016llx DebugExceptions = 0x%016lx\n", 6363 vmcs_read64(GUEST_IA32_DEBUGCTL), 6364 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS)); 6365 if (cpu_has_load_perf_global_ctrl() && 6366 vmentry_ctl & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) 6367 pr_err("PerfGlobCtl = 0x%016llx\n", 6368 vmcs_read64(GUEST_IA32_PERF_GLOBAL_CTRL)); 6369 if (vmentry_ctl & VM_ENTRY_LOAD_BNDCFGS) 6370 pr_err("BndCfgS = 0x%016llx\n", vmcs_read64(GUEST_BNDCFGS)); 6371 pr_err("Interruptibility = %08x ActivityState = %08x\n", 6372 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO), 6373 vmcs_read32(GUEST_ACTIVITY_STATE)); 6374 if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) 6375 pr_err("InterruptStatus = %04x\n", 6376 vmcs_read16(GUEST_INTR_STATUS)); 6377 if (vmcs_read32(VM_ENTRY_MSR_LOAD_COUNT) > 0) 6378 vmx_dump_msrs("guest autoload", &vmx->msr_autoload.guest); 6379 if (vmcs_read32(VM_EXIT_MSR_STORE_COUNT) > 0) 6380 vmx_dump_msrs("guest autostore", &vmx->msr_autostore.guest); 6381 6382 pr_err("*** Host State ***\n"); 6383 pr_err("RIP = 0x%016lx RSP = 0x%016lx\n", 6384 vmcs_readl(HOST_RIP), vmcs_readl(HOST_RSP)); 6385 pr_err("CS=%04x SS=%04x DS=%04x ES=%04x FS=%04x GS=%04x TR=%04x\n", 6386 vmcs_read16(HOST_CS_SELECTOR), vmcs_read16(HOST_SS_SELECTOR), 6387 vmcs_read16(HOST_DS_SELECTOR), vmcs_read16(HOST_ES_SELECTOR), 6388 vmcs_read16(HOST_FS_SELECTOR), vmcs_read16(HOST_GS_SELECTOR), 6389 vmcs_read16(HOST_TR_SELECTOR)); 6390 pr_err("FSBase=%016lx GSBase=%016lx TRBase=%016lx\n", 6391 vmcs_readl(HOST_FS_BASE), vmcs_readl(HOST_GS_BASE), 6392 vmcs_readl(HOST_TR_BASE)); 6393 pr_err("GDTBase=%016lx IDTBase=%016lx\n", 6394 vmcs_readl(HOST_GDTR_BASE), vmcs_readl(HOST_IDTR_BASE)); 6395 pr_err("CR0=%016lx CR3=%016lx CR4=%016lx\n", 6396 vmcs_readl(HOST_CR0), vmcs_readl(HOST_CR3), 6397 vmcs_readl(HOST_CR4)); 6398 pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n", 6399 vmcs_readl(HOST_IA32_SYSENTER_ESP), 6400 vmcs_read32(HOST_IA32_SYSENTER_CS), 6401 vmcs_readl(HOST_IA32_SYSENTER_EIP)); 6402 if (vmexit_ctl & VM_EXIT_LOAD_IA32_EFER) 6403 pr_err("EFER= 0x%016llx\n", vmcs_read64(HOST_IA32_EFER)); 6404 if (vmexit_ctl & VM_EXIT_LOAD_IA32_PAT) 6405 pr_err("PAT = 0x%016llx\n", vmcs_read64(HOST_IA32_PAT)); 6406 if (cpu_has_load_perf_global_ctrl() && 6407 vmexit_ctl & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) 6408 pr_err("PerfGlobCtl = 0x%016llx\n", 6409 vmcs_read64(HOST_IA32_PERF_GLOBAL_CTRL)); 6410 if (vmcs_read32(VM_EXIT_MSR_LOAD_COUNT) > 0) 6411 vmx_dump_msrs("host autoload", &vmx->msr_autoload.host); 6412 6413 pr_err("*** Control State ***\n"); 6414 pr_err("CPUBased=0x%08x SecondaryExec=0x%08x TertiaryExec=0x%016llx\n", 6415 cpu_based_exec_ctrl, secondary_exec_control, tertiary_exec_control); 6416 pr_err("PinBased=0x%08x EntryControls=%08x ExitControls=%08x\n", 6417 pin_based_exec_ctrl, vmentry_ctl, vmexit_ctl); 6418 pr_err("ExceptionBitmap=%08x PFECmask=%08x PFECmatch=%08x\n", 6419 vmcs_read32(EXCEPTION_BITMAP), 6420 vmcs_read32(PAGE_FAULT_ERROR_CODE_MASK), 6421 vmcs_read32(PAGE_FAULT_ERROR_CODE_MATCH)); 6422 pr_err("VMEntry: intr_info=%08x errcode=%08x ilen=%08x\n", 6423 vmcs_read32(VM_ENTRY_INTR_INFO_FIELD), 6424 vmcs_read32(VM_ENTRY_EXCEPTION_ERROR_CODE), 6425 vmcs_read32(VM_ENTRY_INSTRUCTION_LEN)); 6426 pr_err("VMExit: intr_info=%08x errcode=%08x ilen=%08x\n", 6427 vmcs_read32(VM_EXIT_INTR_INFO), 6428 vmcs_read32(VM_EXIT_INTR_ERROR_CODE), 6429 vmcs_read32(VM_EXIT_INSTRUCTION_LEN)); 6430 pr_err(" reason=%08x qualification=%016lx\n", 6431 vmcs_read32(VM_EXIT_REASON), vmcs_readl(EXIT_QUALIFICATION)); 6432 pr_err("IDTVectoring: info=%08x errcode=%08x\n", 6433 vmcs_read32(IDT_VECTORING_INFO_FIELD), 6434 vmcs_read32(IDT_VECTORING_ERROR_CODE)); 6435 pr_err("TSC Offset = 0x%016llx\n", vmcs_read64(TSC_OFFSET)); 6436 if (secondary_exec_control & SECONDARY_EXEC_TSC_SCALING) 6437 pr_err("TSC Multiplier = 0x%016llx\n", 6438 vmcs_read64(TSC_MULTIPLIER)); 6439 if (cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW) { 6440 if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) { 6441 u16 status = vmcs_read16(GUEST_INTR_STATUS); 6442 pr_err("SVI|RVI = %02x|%02x ", status >> 8, status & 0xff); 6443 } 6444 pr_cont("TPR Threshold = 0x%02x\n", vmcs_read32(TPR_THRESHOLD)); 6445 if (secondary_exec_control & SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) 6446 pr_err("APIC-access addr = 0x%016llx ", vmcs_read64(APIC_ACCESS_ADDR)); 6447 pr_cont("virt-APIC addr = 0x%016llx\n", vmcs_read64(VIRTUAL_APIC_PAGE_ADDR)); 6448 } 6449 if (pin_based_exec_ctrl & PIN_BASED_POSTED_INTR) 6450 pr_err("PostedIntrVec = 0x%02x\n", vmcs_read16(POSTED_INTR_NV)); 6451 if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT)) 6452 pr_err("EPT pointer = 0x%016llx\n", vmcs_read64(EPT_POINTER)); 6453 if (secondary_exec_control & SECONDARY_EXEC_PAUSE_LOOP_EXITING) 6454 pr_err("PLE Gap=%08x Window=%08x\n", 6455 vmcs_read32(PLE_GAP), vmcs_read32(PLE_WINDOW)); 6456 if (secondary_exec_control & SECONDARY_EXEC_ENABLE_VPID) 6457 pr_err("Virtual processor ID = 0x%04x\n", 6458 vmcs_read16(VIRTUAL_PROCESSOR_ID)); 6459 if (secondary_exec_control & SECONDARY_EXEC_EPT_VIOLATION_VE) { 6460 struct vmx_ve_information *ve_info = vmx->ve_info; 6461 u64 ve_info_pa = vmcs_read64(VE_INFORMATION_ADDRESS); 6462 6463 /* 6464 * If KVM is dumping the VMCS, then something has gone wrong 6465 * already. Derefencing an address from the VMCS, which could 6466 * very well be corrupted, is a terrible idea. The virtual 6467 * address is known so use it. 6468 */ 6469 pr_err("VE info address = 0x%016llx%s\n", ve_info_pa, 6470 ve_info_pa == __pa(ve_info) ? "" : "(corrupted!)"); 6471 pr_err("ve_info: 0x%08x 0x%08x 0x%016llx 0x%016llx 0x%016llx 0x%04x\n", 6472 ve_info->exit_reason, ve_info->delivery, 6473 ve_info->exit_qualification, 6474 ve_info->guest_linear_address, 6475 ve_info->guest_physical_address, ve_info->eptp_index); 6476 } 6477 } 6478 6479 /* 6480 * The guest has exited. See if we can fix it or if we need userspace 6481 * assistance. 6482 */ 6483 static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) 6484 { 6485 struct vcpu_vmx *vmx = to_vmx(vcpu); 6486 union vmx_exit_reason exit_reason = vmx->exit_reason; 6487 u32 vectoring_info = vmx->idt_vectoring_info; 6488 u16 exit_handler_index; 6489 6490 /* 6491 * Flush logged GPAs PML buffer, this will make dirty_bitmap more 6492 * updated. Another good is, in kvm_vm_ioctl_get_dirty_log, before 6493 * querying dirty_bitmap, we only need to kick all vcpus out of guest 6494 * mode as if vcpus is in root mode, the PML buffer must has been 6495 * flushed already. Note, PML is never enabled in hardware while 6496 * running L2. 6497 */ 6498 if (enable_pml && !is_guest_mode(vcpu)) 6499 vmx_flush_pml_buffer(vcpu); 6500 6501 /* 6502 * KVM should never reach this point with a pending nested VM-Enter. 6503 * More specifically, short-circuiting VM-Entry to emulate L2 due to 6504 * invalid guest state should never happen as that means KVM knowingly 6505 * allowed a nested VM-Enter with an invalid vmcs12. More below. 6506 */ 6507 if (KVM_BUG_ON(vmx->nested.nested_run_pending, vcpu->kvm)) 6508 return -EIO; 6509 6510 if (is_guest_mode(vcpu)) { 6511 /* 6512 * PML is never enabled when running L2, bail immediately if a 6513 * PML full exit occurs as something is horribly wrong. 6514 */ 6515 if (exit_reason.basic == EXIT_REASON_PML_FULL) 6516 goto unexpected_vmexit; 6517 6518 /* 6519 * The host physical addresses of some pages of guest memory 6520 * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC 6521 * Page). The CPU may write to these pages via their host 6522 * physical address while L2 is running, bypassing any 6523 * address-translation-based dirty tracking (e.g. EPT write 6524 * protection). 6525 * 6526 * Mark them dirty on every exit from L2 to prevent them from 6527 * getting out of sync with dirty tracking. 6528 */ 6529 nested_mark_vmcs12_pages_dirty(vcpu); 6530 6531 /* 6532 * Synthesize a triple fault if L2 state is invalid. In normal 6533 * operation, nested VM-Enter rejects any attempt to enter L2 6534 * with invalid state. However, those checks are skipped if 6535 * state is being stuffed via RSM or KVM_SET_NESTED_STATE. If 6536 * L2 state is invalid, it means either L1 modified SMRAM state 6537 * or userspace provided bad state. Synthesize TRIPLE_FAULT as 6538 * doing so is architecturally allowed in the RSM case, and is 6539 * the least awful solution for the userspace case without 6540 * risking false positives. 6541 */ 6542 if (vmx->emulation_required) { 6543 nested_vmx_vmexit(vcpu, EXIT_REASON_TRIPLE_FAULT, 0, 0); 6544 return 1; 6545 } 6546 6547 if (nested_vmx_reflect_vmexit(vcpu)) 6548 return 1; 6549 } 6550 6551 /* If guest state is invalid, start emulating. L2 is handled above. */ 6552 if (vmx->emulation_required) 6553 return handle_invalid_guest_state(vcpu); 6554 6555 if (exit_reason.failed_vmentry) { 6556 dump_vmcs(vcpu); 6557 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; 6558 vcpu->run->fail_entry.hardware_entry_failure_reason 6559 = exit_reason.full; 6560 vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu; 6561 return 0; 6562 } 6563 6564 if (unlikely(vmx->fail)) { 6565 dump_vmcs(vcpu); 6566 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; 6567 vcpu->run->fail_entry.hardware_entry_failure_reason 6568 = vmcs_read32(VM_INSTRUCTION_ERROR); 6569 vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu; 6570 return 0; 6571 } 6572 6573 if ((vectoring_info & VECTORING_INFO_VALID_MASK) && 6574 (exit_reason.basic != EXIT_REASON_EXCEPTION_NMI && 6575 exit_reason.basic != EXIT_REASON_EPT_VIOLATION && 6576 exit_reason.basic != EXIT_REASON_PML_FULL && 6577 exit_reason.basic != EXIT_REASON_APIC_ACCESS && 6578 exit_reason.basic != EXIT_REASON_TASK_SWITCH && 6579 exit_reason.basic != EXIT_REASON_NOTIFY && 6580 exit_reason.basic != EXIT_REASON_EPT_MISCONFIG)) { 6581 kvm_prepare_event_vectoring_exit(vcpu, INVALID_GPA); 6582 return 0; 6583 } 6584 6585 if (unlikely(!enable_vnmi && 6586 vmx->loaded_vmcs->soft_vnmi_blocked)) { 6587 if (!vmx_interrupt_blocked(vcpu)) { 6588 vmx->loaded_vmcs->soft_vnmi_blocked = 0; 6589 } else if (vmx->loaded_vmcs->vnmi_blocked_time > 1000000000LL && 6590 vcpu->arch.nmi_pending) { 6591 /* 6592 * This CPU don't support us in finding the end of an 6593 * NMI-blocked window if the guest runs with IRQs 6594 * disabled. So we pull the trigger after 1 s of 6595 * futile waiting, but inform the user about this. 6596 */ 6597 printk(KERN_WARNING "%s: Breaking out of NMI-blocked " 6598 "state on VCPU %d after 1 s timeout\n", 6599 __func__, vcpu->vcpu_id); 6600 vmx->loaded_vmcs->soft_vnmi_blocked = 0; 6601 } 6602 } 6603 6604 if (exit_fastpath != EXIT_FASTPATH_NONE) 6605 return 1; 6606 6607 if (exit_reason.basic >= kvm_vmx_max_exit_handlers) 6608 goto unexpected_vmexit; 6609 #ifdef CONFIG_MITIGATION_RETPOLINE 6610 if (exit_reason.basic == EXIT_REASON_MSR_WRITE) 6611 return kvm_emulate_wrmsr(vcpu); 6612 else if (exit_reason.basic == EXIT_REASON_PREEMPTION_TIMER) 6613 return handle_preemption_timer(vcpu); 6614 else if (exit_reason.basic == EXIT_REASON_INTERRUPT_WINDOW) 6615 return handle_interrupt_window(vcpu); 6616 else if (exit_reason.basic == EXIT_REASON_EXTERNAL_INTERRUPT) 6617 return handle_external_interrupt(vcpu); 6618 else if (exit_reason.basic == EXIT_REASON_HLT) 6619 return kvm_emulate_halt(vcpu); 6620 else if (exit_reason.basic == EXIT_REASON_EPT_MISCONFIG) 6621 return handle_ept_misconfig(vcpu); 6622 #endif 6623 6624 exit_handler_index = array_index_nospec((u16)exit_reason.basic, 6625 kvm_vmx_max_exit_handlers); 6626 if (!kvm_vmx_exit_handlers[exit_handler_index]) 6627 goto unexpected_vmexit; 6628 6629 return kvm_vmx_exit_handlers[exit_handler_index](vcpu); 6630 6631 unexpected_vmexit: 6632 vcpu_unimpl(vcpu, "vmx: unexpected exit reason 0x%x\n", 6633 exit_reason.full); 6634 dump_vmcs(vcpu); 6635 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 6636 vcpu->run->internal.suberror = 6637 KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON; 6638 vcpu->run->internal.ndata = 2; 6639 vcpu->run->internal.data[0] = exit_reason.full; 6640 vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu; 6641 return 0; 6642 } 6643 6644 int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) 6645 { 6646 int ret = __vmx_handle_exit(vcpu, exit_fastpath); 6647 6648 /* 6649 * Exit to user space when bus lock detected to inform that there is 6650 * a bus lock in guest. 6651 */ 6652 if (to_vmx(vcpu)->exit_reason.bus_lock_detected) { 6653 if (ret > 0) 6654 vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; 6655 6656 vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK; 6657 return 0; 6658 } 6659 return ret; 6660 } 6661 6662 /* 6663 * Software based L1D cache flush which is used when microcode providing 6664 * the cache control MSR is not loaded. 6665 * 6666 * The L1D cache is 32 KiB on Nehalem and later microarchitectures, but to 6667 * flush it is required to read in 64 KiB because the replacement algorithm 6668 * is not exactly LRU. This could be sized at runtime via topology 6669 * information but as all relevant affected CPUs have 32KiB L1D cache size 6670 * there is no point in doing so. 6671 */ 6672 static noinstr void vmx_l1d_flush(struct kvm_vcpu *vcpu) 6673 { 6674 int size = PAGE_SIZE << L1D_CACHE_ORDER; 6675 6676 /* 6677 * This code is only executed when the flush mode is 'cond' or 6678 * 'always' 6679 */ 6680 if (static_branch_likely(&vmx_l1d_flush_cond)) { 6681 bool flush_l1d; 6682 6683 /* 6684 * Clear the per-vcpu flush bit, it gets set again if the vCPU 6685 * is reloaded, i.e. if the vCPU is scheduled out or if KVM 6686 * exits to userspace, or if KVM reaches one of the unsafe 6687 * VMEXIT handlers, e.g. if KVM calls into the emulator. 6688 */ 6689 flush_l1d = vcpu->arch.l1tf_flush_l1d; 6690 vcpu->arch.l1tf_flush_l1d = false; 6691 6692 /* 6693 * Clear the per-cpu flush bit, it gets set again from 6694 * the interrupt handlers. 6695 */ 6696 flush_l1d |= kvm_get_cpu_l1tf_flush_l1d(); 6697 kvm_clear_cpu_l1tf_flush_l1d(); 6698 6699 if (!flush_l1d) 6700 return; 6701 } 6702 6703 vcpu->stat.l1d_flush++; 6704 6705 if (static_cpu_has(X86_FEATURE_FLUSH_L1D)) { 6706 native_wrmsrl(MSR_IA32_FLUSH_CMD, L1D_FLUSH); 6707 return; 6708 } 6709 6710 asm volatile( 6711 /* First ensure the pages are in the TLB */ 6712 "xorl %%eax, %%eax\n" 6713 ".Lpopulate_tlb:\n\t" 6714 "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t" 6715 "addl $4096, %%eax\n\t" 6716 "cmpl %%eax, %[size]\n\t" 6717 "jne .Lpopulate_tlb\n\t" 6718 "xorl %%eax, %%eax\n\t" 6719 "cpuid\n\t" 6720 /* Now fill the cache */ 6721 "xorl %%eax, %%eax\n" 6722 ".Lfill_cache:\n" 6723 "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t" 6724 "addl $64, %%eax\n\t" 6725 "cmpl %%eax, %[size]\n\t" 6726 "jne .Lfill_cache\n\t" 6727 "lfence\n" 6728 :: [flush_pages] "r" (vmx_l1d_flush_pages), 6729 [size] "r" (size) 6730 : "eax", "ebx", "ecx", "edx"); 6731 } 6732 6733 void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) 6734 { 6735 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 6736 int tpr_threshold; 6737 6738 if (is_guest_mode(vcpu) && 6739 nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) 6740 return; 6741 6742 tpr_threshold = (irr == -1 || tpr < irr) ? 0 : irr; 6743 if (is_guest_mode(vcpu)) 6744 to_vmx(vcpu)->nested.l1_tpr_threshold = tpr_threshold; 6745 else 6746 vmcs_write32(TPR_THRESHOLD, tpr_threshold); 6747 } 6748 6749 void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu) 6750 { 6751 struct vcpu_vmx *vmx = to_vmx(vcpu); 6752 u32 sec_exec_control; 6753 6754 if (!lapic_in_kernel(vcpu)) 6755 return; 6756 6757 if (!flexpriority_enabled && 6758 !cpu_has_vmx_virtualize_x2apic_mode()) 6759 return; 6760 6761 /* Postpone execution until vmcs01 is the current VMCS. */ 6762 if (is_guest_mode(vcpu)) { 6763 vmx->nested.change_vmcs01_virtual_apic_mode = true; 6764 return; 6765 } 6766 6767 sec_exec_control = secondary_exec_controls_get(vmx); 6768 sec_exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 6769 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE); 6770 6771 switch (kvm_get_apic_mode(vcpu)) { 6772 case LAPIC_MODE_INVALID: 6773 WARN_ONCE(true, "Invalid local APIC state"); 6774 break; 6775 case LAPIC_MODE_DISABLED: 6776 break; 6777 case LAPIC_MODE_XAPIC: 6778 if (flexpriority_enabled) { 6779 sec_exec_control |= 6780 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 6781 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 6782 6783 /* 6784 * Flush the TLB, reloading the APIC access page will 6785 * only do so if its physical address has changed, but 6786 * the guest may have inserted a non-APIC mapping into 6787 * the TLB while the APIC access page was disabled. 6788 */ 6789 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 6790 } 6791 break; 6792 case LAPIC_MODE_X2APIC: 6793 if (cpu_has_vmx_virtualize_x2apic_mode()) 6794 sec_exec_control |= 6795 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE; 6796 break; 6797 } 6798 secondary_exec_controls_set(vmx, sec_exec_control); 6799 6800 vmx_update_msr_bitmap_x2apic(vcpu); 6801 } 6802 6803 void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu) 6804 { 6805 const gfn_t gfn = APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT; 6806 struct kvm *kvm = vcpu->kvm; 6807 struct kvm_memslots *slots = kvm_memslots(kvm); 6808 struct kvm_memory_slot *slot; 6809 struct page *refcounted_page; 6810 unsigned long mmu_seq; 6811 kvm_pfn_t pfn; 6812 bool writable; 6813 6814 /* Defer reload until vmcs01 is the current VMCS. */ 6815 if (is_guest_mode(vcpu)) { 6816 to_vmx(vcpu)->nested.reload_vmcs01_apic_access_page = true; 6817 return; 6818 } 6819 6820 if (!(secondary_exec_controls_get(to_vmx(vcpu)) & 6821 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) 6822 return; 6823 6824 /* 6825 * Explicitly grab the memslot using KVM's internal slot ID to ensure 6826 * KVM doesn't unintentionally grab a userspace memslot. It _should_ 6827 * be impossible for userspace to create a memslot for the APIC when 6828 * APICv is enabled, but paranoia won't hurt in this case. 6829 */ 6830 slot = id_to_memslot(slots, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT); 6831 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) 6832 return; 6833 6834 /* 6835 * Ensure that the mmu_notifier sequence count is read before KVM 6836 * retrieves the pfn from the primary MMU. Note, the memslot is 6837 * protected by SRCU, not the mmu_notifier. Pairs with the smp_wmb() 6838 * in kvm_mmu_invalidate_end(). 6839 */ 6840 mmu_seq = kvm->mmu_invalidate_seq; 6841 smp_rmb(); 6842 6843 /* 6844 * No need to retry if the memslot does not exist or is invalid. KVM 6845 * controls the APIC-access page memslot, and only deletes the memslot 6846 * if APICv is permanently inhibited, i.e. the memslot won't reappear. 6847 */ 6848 pfn = __kvm_faultin_pfn(slot, gfn, FOLL_WRITE, &writable, &refcounted_page); 6849 if (is_error_noslot_pfn(pfn)) 6850 return; 6851 6852 read_lock(&vcpu->kvm->mmu_lock); 6853 if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) 6854 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 6855 else 6856 vmcs_write64(APIC_ACCESS_ADDR, pfn_to_hpa(pfn)); 6857 6858 /* 6859 * Do not pin the APIC access page in memory so that it can be freely 6860 * migrated, the MMU notifier will call us again if it is migrated or 6861 * swapped out. KVM backs the memslot with anonymous memory, the pfn 6862 * should always point at a refcounted page (if the pfn is valid). 6863 */ 6864 if (!WARN_ON_ONCE(!refcounted_page)) 6865 kvm_release_page_clean(refcounted_page); 6866 6867 /* 6868 * No need for a manual TLB flush at this point, KVM has already done a 6869 * flush if there were SPTEs pointing at the previous page. 6870 */ 6871 read_unlock(&vcpu->kvm->mmu_lock); 6872 } 6873 6874 void vmx_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr) 6875 { 6876 u16 status; 6877 u8 old; 6878 6879 /* 6880 * If L2 is active, defer the SVI update until vmcs01 is loaded, as SVI 6881 * is only relevant for if and only if Virtual Interrupt Delivery is 6882 * enabled in vmcs12, and if VID is enabled then L2 EOIs affect L2's 6883 * vAPIC, not L1's vAPIC. KVM must update vmcs01 on the next nested 6884 * VM-Exit, otherwise L1 with run with a stale SVI. 6885 */ 6886 if (is_guest_mode(vcpu)) { 6887 /* 6888 * KVM is supposed to forward intercepted L2 EOIs to L1 if VID 6889 * is enabled in vmcs12; as above, the EOIs affect L2's vAPIC. 6890 * Note, userspace can stuff state while L2 is active; assert 6891 * that VID is disabled if and only if the vCPU is in KVM_RUN 6892 * to avoid false positives if userspace is setting APIC state. 6893 */ 6894 WARN_ON_ONCE(vcpu->wants_to_run && 6895 nested_cpu_has_vid(get_vmcs12(vcpu))); 6896 to_vmx(vcpu)->nested.update_vmcs01_hwapic_isr = true; 6897 return; 6898 } 6899 6900 if (max_isr == -1) 6901 max_isr = 0; 6902 6903 status = vmcs_read16(GUEST_INTR_STATUS); 6904 old = status >> 8; 6905 if (max_isr != old) { 6906 status &= 0xff; 6907 status |= max_isr << 8; 6908 vmcs_write16(GUEST_INTR_STATUS, status); 6909 } 6910 } 6911 6912 static void vmx_set_rvi(int vector) 6913 { 6914 u16 status; 6915 u8 old; 6916 6917 if (vector == -1) 6918 vector = 0; 6919 6920 status = vmcs_read16(GUEST_INTR_STATUS); 6921 old = (u8)status & 0xff; 6922 if ((u8)vector != old) { 6923 status &= ~0xff; 6924 status |= (u8)vector; 6925 vmcs_write16(GUEST_INTR_STATUS, status); 6926 } 6927 } 6928 6929 int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu) 6930 { 6931 struct vcpu_vmx *vmx = to_vmx(vcpu); 6932 int max_irr; 6933 bool got_posted_interrupt; 6934 6935 if (KVM_BUG_ON(!enable_apicv, vcpu->kvm)) 6936 return -EIO; 6937 6938 if (pi_test_on(&vmx->pi_desc)) { 6939 pi_clear_on(&vmx->pi_desc); 6940 /* 6941 * IOMMU can write to PID.ON, so the barrier matters even on UP. 6942 * But on x86 this is just a compiler barrier anyway. 6943 */ 6944 smp_mb__after_atomic(); 6945 got_posted_interrupt = 6946 kvm_apic_update_irr(vcpu, vmx->pi_desc.pir, &max_irr); 6947 } else { 6948 max_irr = kvm_lapic_find_highest_irr(vcpu); 6949 got_posted_interrupt = false; 6950 } 6951 6952 /* 6953 * Newly recognized interrupts are injected via either virtual interrupt 6954 * delivery (RVI) or KVM_REQ_EVENT. Virtual interrupt delivery is 6955 * disabled in two cases: 6956 * 6957 * 1) If L2 is running and the vCPU has a new pending interrupt. If L1 6958 * wants to exit on interrupts, KVM_REQ_EVENT is needed to synthesize a 6959 * VM-Exit to L1. If L1 doesn't want to exit, the interrupt is injected 6960 * into L2, but KVM doesn't use virtual interrupt delivery to inject 6961 * interrupts into L2, and so KVM_REQ_EVENT is again needed. 6962 * 6963 * 2) If APICv is disabled for this vCPU, assigned devices may still 6964 * attempt to post interrupts. The posted interrupt vector will cause 6965 * a VM-Exit and the subsequent entry will call sync_pir_to_irr. 6966 */ 6967 if (!is_guest_mode(vcpu) && kvm_vcpu_apicv_active(vcpu)) 6968 vmx_set_rvi(max_irr); 6969 else if (got_posted_interrupt) 6970 kvm_make_request(KVM_REQ_EVENT, vcpu); 6971 6972 return max_irr; 6973 } 6974 6975 void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) 6976 { 6977 if (!kvm_vcpu_apicv_active(vcpu)) 6978 return; 6979 6980 vmcs_write64(EOI_EXIT_BITMAP0, eoi_exit_bitmap[0]); 6981 vmcs_write64(EOI_EXIT_BITMAP1, eoi_exit_bitmap[1]); 6982 vmcs_write64(EOI_EXIT_BITMAP2, eoi_exit_bitmap[2]); 6983 vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]); 6984 } 6985 6986 void vmx_apicv_pre_state_restore(struct kvm_vcpu *vcpu) 6987 { 6988 struct vcpu_vmx *vmx = to_vmx(vcpu); 6989 6990 pi_clear_on(&vmx->pi_desc); 6991 memset(vmx->pi_desc.pir, 0, sizeof(vmx->pi_desc.pir)); 6992 } 6993 6994 void vmx_do_interrupt_irqoff(unsigned long entry); 6995 void vmx_do_nmi_irqoff(void); 6996 6997 static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu) 6998 { 6999 /* 7000 * Save xfd_err to guest_fpu before interrupt is enabled, so the 7001 * MSR value is not clobbered by the host activity before the guest 7002 * has chance to consume it. 7003 * 7004 * Do not blindly read xfd_err here, since this exception might 7005 * be caused by L1 interception on a platform which doesn't 7006 * support xfd at all. 7007 * 7008 * Do it conditionally upon guest_fpu::xfd. xfd_err matters 7009 * only when xfd contains a non-zero value. 7010 * 7011 * Queuing exception is done in vmx_handle_exit. See comment there. 7012 */ 7013 if (vcpu->arch.guest_fpu.fpstate->xfd) 7014 rdmsrl(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); 7015 } 7016 7017 static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info) 7018 { 7019 /* if exit due to PF check for async PF */ 7020 if (is_page_fault(intr_info)) 7021 vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags(); 7022 /* if exit due to NM, handle before interrupts are enabled */ 7023 else if (is_nm_fault(intr_info)) 7024 handle_nm_fault_irqoff(vcpu); 7025 /* Handle machine checks before interrupts are enabled */ 7026 else if (is_machine_check(intr_info)) 7027 kvm_machine_check(); 7028 } 7029 7030 static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu, 7031 u32 intr_info) 7032 { 7033 unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK; 7034 7035 if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm, 7036 "unexpected VM-Exit interrupt info: 0x%x", intr_info)) 7037 return; 7038 7039 kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ); 7040 if (cpu_feature_enabled(X86_FEATURE_FRED)) 7041 fred_entry_from_kvm(EVENT_TYPE_EXTINT, vector); 7042 else 7043 vmx_do_interrupt_irqoff(gate_offset((gate_desc *)host_idt_base + vector)); 7044 kvm_after_interrupt(vcpu); 7045 7046 vcpu->arch.at_instruction_boundary = true; 7047 } 7048 7049 void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu) 7050 { 7051 struct vcpu_vmx *vmx = to_vmx(vcpu); 7052 7053 if (vmx->emulation_required) 7054 return; 7055 7056 if (vmx->exit_reason.basic == EXIT_REASON_EXTERNAL_INTERRUPT) 7057 handle_external_interrupt_irqoff(vcpu, vmx_get_intr_info(vcpu)); 7058 else if (vmx->exit_reason.basic == EXIT_REASON_EXCEPTION_NMI) 7059 handle_exception_irqoff(vcpu, vmx_get_intr_info(vcpu)); 7060 } 7061 7062 /* 7063 * The kvm parameter can be NULL (module initialization, or invocation before 7064 * VM creation). Be sure to check the kvm parameter before using it. 7065 */ 7066 bool vmx_has_emulated_msr(struct kvm *kvm, u32 index) 7067 { 7068 switch (index) { 7069 case MSR_IA32_SMBASE: 7070 if (!IS_ENABLED(CONFIG_KVM_SMM)) 7071 return false; 7072 /* 7073 * We cannot do SMM unless we can run the guest in big 7074 * real mode. 7075 */ 7076 return enable_unrestricted_guest || emulate_invalid_guest_state; 7077 case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR: 7078 return nested; 7079 case MSR_AMD64_VIRT_SPEC_CTRL: 7080 case MSR_AMD64_TSC_RATIO: 7081 /* This is AMD only. */ 7082 return false; 7083 default: 7084 return true; 7085 } 7086 } 7087 7088 static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx) 7089 { 7090 u32 exit_intr_info; 7091 bool unblock_nmi; 7092 u8 vector; 7093 bool idtv_info_valid; 7094 7095 idtv_info_valid = vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK; 7096 7097 if (enable_vnmi) { 7098 if (vmx->loaded_vmcs->nmi_known_unmasked) 7099 return; 7100 7101 exit_intr_info = vmx_get_intr_info(&vmx->vcpu); 7102 unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0; 7103 vector = exit_intr_info & INTR_INFO_VECTOR_MASK; 7104 /* 7105 * SDM 3: 27.7.1.2 (September 2008) 7106 * Re-set bit "block by NMI" before VM entry if vmexit caused by 7107 * a guest IRET fault. 7108 * SDM 3: 23.2.2 (September 2008) 7109 * Bit 12 is undefined in any of the following cases: 7110 * If the VM exit sets the valid bit in the IDT-vectoring 7111 * information field. 7112 * If the VM exit is due to a double fault. 7113 */ 7114 if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi && 7115 vector != DF_VECTOR && !idtv_info_valid) 7116 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 7117 GUEST_INTR_STATE_NMI); 7118 else 7119 vmx->loaded_vmcs->nmi_known_unmasked = 7120 !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) 7121 & GUEST_INTR_STATE_NMI); 7122 } else if (unlikely(vmx->loaded_vmcs->soft_vnmi_blocked)) 7123 vmx->loaded_vmcs->vnmi_blocked_time += 7124 ktime_to_ns(ktime_sub(ktime_get(), 7125 vmx->loaded_vmcs->entry_time)); 7126 } 7127 7128 static void __vmx_complete_interrupts(struct kvm_vcpu *vcpu, 7129 u32 idt_vectoring_info, 7130 int instr_len_field, 7131 int error_code_field) 7132 { 7133 u8 vector; 7134 int type; 7135 bool idtv_info_valid; 7136 7137 idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK; 7138 7139 vcpu->arch.nmi_injected = false; 7140 kvm_clear_exception_queue(vcpu); 7141 kvm_clear_interrupt_queue(vcpu); 7142 7143 if (!idtv_info_valid) 7144 return; 7145 7146 kvm_make_request(KVM_REQ_EVENT, vcpu); 7147 7148 vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK; 7149 type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK; 7150 7151 switch (type) { 7152 case INTR_TYPE_NMI_INTR: 7153 vcpu->arch.nmi_injected = true; 7154 /* 7155 * SDM 3: 27.7.1.2 (September 2008) 7156 * Clear bit "block by NMI" before VM entry if a NMI 7157 * delivery faulted. 7158 */ 7159 vmx_set_nmi_mask(vcpu, false); 7160 break; 7161 case INTR_TYPE_SOFT_EXCEPTION: 7162 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field); 7163 fallthrough; 7164 case INTR_TYPE_HARD_EXCEPTION: 7165 if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) { 7166 u32 err = vmcs_read32(error_code_field); 7167 kvm_requeue_exception_e(vcpu, vector, err); 7168 } else 7169 kvm_requeue_exception(vcpu, vector); 7170 break; 7171 case INTR_TYPE_SOFT_INTR: 7172 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field); 7173 fallthrough; 7174 case INTR_TYPE_EXT_INTR: 7175 kvm_queue_interrupt(vcpu, vector, type == INTR_TYPE_SOFT_INTR); 7176 break; 7177 default: 7178 break; 7179 } 7180 } 7181 7182 static void vmx_complete_interrupts(struct vcpu_vmx *vmx) 7183 { 7184 __vmx_complete_interrupts(&vmx->vcpu, vmx->idt_vectoring_info, 7185 VM_EXIT_INSTRUCTION_LEN, 7186 IDT_VECTORING_ERROR_CODE); 7187 } 7188 7189 void vmx_cancel_injection(struct kvm_vcpu *vcpu) 7190 { 7191 __vmx_complete_interrupts(vcpu, 7192 vmcs_read32(VM_ENTRY_INTR_INFO_FIELD), 7193 VM_ENTRY_INSTRUCTION_LEN, 7194 VM_ENTRY_EXCEPTION_ERROR_CODE); 7195 7196 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); 7197 } 7198 7199 static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx) 7200 { 7201 int i, nr_msrs; 7202 struct perf_guest_switch_msr *msrs; 7203 struct kvm_pmu *pmu = vcpu_to_pmu(&vmx->vcpu); 7204 7205 pmu->host_cross_mapped_mask = 0; 7206 if (pmu->pebs_enable & pmu->global_ctrl) 7207 intel_pmu_cross_mapped_check(pmu); 7208 7209 /* Note, nr_msrs may be garbage if perf_guest_get_msrs() returns NULL. */ 7210 msrs = perf_guest_get_msrs(&nr_msrs, (void *)pmu); 7211 if (!msrs) 7212 return; 7213 7214 for (i = 0; i < nr_msrs; i++) 7215 if (msrs[i].host == msrs[i].guest) 7216 clear_atomic_switch_msr(vmx, msrs[i].msr); 7217 else 7218 add_atomic_switch_msr(vmx, msrs[i].msr, msrs[i].guest, 7219 msrs[i].host, false); 7220 } 7221 7222 static void vmx_update_hv_timer(struct kvm_vcpu *vcpu, bool force_immediate_exit) 7223 { 7224 struct vcpu_vmx *vmx = to_vmx(vcpu); 7225 u64 tscl; 7226 u32 delta_tsc; 7227 7228 if (force_immediate_exit) { 7229 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, 0); 7230 vmx->loaded_vmcs->hv_timer_soft_disabled = false; 7231 } else if (vmx->hv_deadline_tsc != -1) { 7232 tscl = rdtsc(); 7233 if (vmx->hv_deadline_tsc > tscl) 7234 /* set_hv_timer ensures the delta fits in 32-bits */ 7235 delta_tsc = (u32)((vmx->hv_deadline_tsc - tscl) >> 7236 cpu_preemption_timer_multi); 7237 else 7238 delta_tsc = 0; 7239 7240 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, delta_tsc); 7241 vmx->loaded_vmcs->hv_timer_soft_disabled = false; 7242 } else if (!vmx->loaded_vmcs->hv_timer_soft_disabled) { 7243 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, -1); 7244 vmx->loaded_vmcs->hv_timer_soft_disabled = true; 7245 } 7246 } 7247 7248 void noinstr vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp) 7249 { 7250 if (unlikely(host_rsp != vmx->loaded_vmcs->host_state.rsp)) { 7251 vmx->loaded_vmcs->host_state.rsp = host_rsp; 7252 vmcs_writel(HOST_RSP, host_rsp); 7253 } 7254 } 7255 7256 void noinstr vmx_spec_ctrl_restore_host(struct vcpu_vmx *vmx, 7257 unsigned int flags) 7258 { 7259 u64 hostval = this_cpu_read(x86_spec_ctrl_current); 7260 7261 if (!cpu_feature_enabled(X86_FEATURE_MSR_SPEC_CTRL)) 7262 return; 7263 7264 if (flags & VMX_RUN_SAVE_SPEC_CTRL) 7265 vmx->spec_ctrl = __rdmsr(MSR_IA32_SPEC_CTRL); 7266 7267 /* 7268 * If the guest/host SPEC_CTRL values differ, restore the host value. 7269 * 7270 * For legacy IBRS, the IBRS bit always needs to be written after 7271 * transitioning from a less privileged predictor mode, regardless of 7272 * whether the guest/host values differ. 7273 */ 7274 if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) || 7275 vmx->spec_ctrl != hostval) 7276 native_wrmsrl(MSR_IA32_SPEC_CTRL, hostval); 7277 7278 barrier_nospec(); 7279 } 7280 7281 static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu, 7282 bool force_immediate_exit) 7283 { 7284 /* 7285 * If L2 is active, some VMX preemption timer exits can be handled in 7286 * the fastpath even, all other exits must use the slow path. 7287 */ 7288 if (is_guest_mode(vcpu) && 7289 to_vmx(vcpu)->exit_reason.basic != EXIT_REASON_PREEMPTION_TIMER) 7290 return EXIT_FASTPATH_NONE; 7291 7292 switch (to_vmx(vcpu)->exit_reason.basic) { 7293 case EXIT_REASON_MSR_WRITE: 7294 return handle_fastpath_set_msr_irqoff(vcpu); 7295 case EXIT_REASON_PREEMPTION_TIMER: 7296 return handle_fastpath_preemption_timer(vcpu, force_immediate_exit); 7297 case EXIT_REASON_HLT: 7298 return handle_fastpath_hlt(vcpu); 7299 default: 7300 return EXIT_FASTPATH_NONE; 7301 } 7302 } 7303 7304 static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, 7305 unsigned int flags) 7306 { 7307 struct vcpu_vmx *vmx = to_vmx(vcpu); 7308 7309 guest_state_enter_irqoff(); 7310 7311 /* 7312 * L1D Flush includes CPU buffer clear to mitigate MDS, but VERW 7313 * mitigation for MDS is done late in VMentry and is still 7314 * executed in spite of L1D Flush. This is because an extra VERW 7315 * should not matter much after the big hammer L1D Flush. 7316 */ 7317 if (static_branch_unlikely(&vmx_l1d_should_flush)) 7318 vmx_l1d_flush(vcpu); 7319 else if (static_branch_unlikely(&mmio_stale_data_clear) && 7320 kvm_arch_has_assigned_device(vcpu->kvm)) 7321 mds_clear_cpu_buffers(); 7322 7323 vmx_disable_fb_clear(vmx); 7324 7325 if (vcpu->arch.cr2 != native_read_cr2()) 7326 native_write_cr2(vcpu->arch.cr2); 7327 7328 vmx->fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs, 7329 flags); 7330 7331 vcpu->arch.cr2 = native_read_cr2(); 7332 vcpu->arch.regs_avail &= ~VMX_REGS_LAZY_LOAD_SET; 7333 7334 vmx->idt_vectoring_info = 0; 7335 7336 vmx_enable_fb_clear(vmx); 7337 7338 if (unlikely(vmx->fail)) { 7339 vmx->exit_reason.full = 0xdead; 7340 goto out; 7341 } 7342 7343 vmx->exit_reason.full = vmcs_read32(VM_EXIT_REASON); 7344 if (likely(!vmx->exit_reason.failed_vmentry)) 7345 vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); 7346 7347 if ((u16)vmx->exit_reason.basic == EXIT_REASON_EXCEPTION_NMI && 7348 is_nmi(vmx_get_intr_info(vcpu))) { 7349 kvm_before_interrupt(vcpu, KVM_HANDLING_NMI); 7350 if (cpu_feature_enabled(X86_FEATURE_FRED)) 7351 fred_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR); 7352 else 7353 vmx_do_nmi_irqoff(); 7354 kvm_after_interrupt(vcpu); 7355 } 7356 7357 out: 7358 guest_state_exit_irqoff(); 7359 } 7360 7361 fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, bool force_immediate_exit) 7362 { 7363 struct vcpu_vmx *vmx = to_vmx(vcpu); 7364 unsigned long cr3, cr4; 7365 7366 /* Record the guest's net vcpu time for enforced NMI injections. */ 7367 if (unlikely(!enable_vnmi && 7368 vmx->loaded_vmcs->soft_vnmi_blocked)) 7369 vmx->loaded_vmcs->entry_time = ktime_get(); 7370 7371 /* 7372 * Don't enter VMX if guest state is invalid, let the exit handler 7373 * start emulation until we arrive back to a valid state. Synthesize a 7374 * consistency check VM-Exit due to invalid guest state and bail. 7375 */ 7376 if (unlikely(vmx->emulation_required)) { 7377 vmx->fail = 0; 7378 7379 vmx->exit_reason.full = EXIT_REASON_INVALID_STATE; 7380 vmx->exit_reason.failed_vmentry = 1; 7381 kvm_register_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_1); 7382 vmx->exit_qualification = ENTRY_FAIL_DEFAULT; 7383 kvm_register_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_2); 7384 vmx->exit_intr_info = 0; 7385 return EXIT_FASTPATH_NONE; 7386 } 7387 7388 trace_kvm_entry(vcpu, force_immediate_exit); 7389 7390 if (vmx->ple_window_dirty) { 7391 vmx->ple_window_dirty = false; 7392 vmcs_write32(PLE_WINDOW, vmx->ple_window); 7393 } 7394 7395 /* 7396 * We did this in prepare_switch_to_guest, because it needs to 7397 * be within srcu_read_lock. 7398 */ 7399 WARN_ON_ONCE(vmx->nested.need_vmcs12_to_shadow_sync); 7400 7401 if (kvm_register_is_dirty(vcpu, VCPU_REGS_RSP)) 7402 vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]); 7403 if (kvm_register_is_dirty(vcpu, VCPU_REGS_RIP)) 7404 vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]); 7405 vcpu->arch.regs_dirty = 0; 7406 7407 /* 7408 * Refresh vmcs.HOST_CR3 if necessary. This must be done immediately 7409 * prior to VM-Enter, as the kernel may load a new ASID (PCID) any time 7410 * it switches back to the current->mm, which can occur in KVM context 7411 * when switching to a temporary mm to patch kernel code, e.g. if KVM 7412 * toggles a static key while handling a VM-Exit. 7413 */ 7414 cr3 = __get_current_cr3_fast(); 7415 if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) { 7416 vmcs_writel(HOST_CR3, cr3); 7417 vmx->loaded_vmcs->host_state.cr3 = cr3; 7418 } 7419 7420 cr4 = cr4_read_shadow(); 7421 if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) { 7422 vmcs_writel(HOST_CR4, cr4); 7423 vmx->loaded_vmcs->host_state.cr4 = cr4; 7424 } 7425 7426 /* When single-stepping over STI and MOV SS, we must clear the 7427 * corresponding interruptibility bits in the guest state. Otherwise 7428 * vmentry fails as it then expects bit 14 (BS) in pending debug 7429 * exceptions being set, but that's not correct for the guest debugging 7430 * case. */ 7431 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) 7432 vmx_set_interrupt_shadow(vcpu, 0); 7433 7434 kvm_load_guest_xsave_state(vcpu); 7435 7436 pt_guest_enter(vmx); 7437 7438 atomic_switch_perf_msrs(vmx); 7439 if (intel_pmu_lbr_is_enabled(vcpu)) 7440 vmx_passthrough_lbr_msrs(vcpu); 7441 7442 if (enable_preemption_timer) 7443 vmx_update_hv_timer(vcpu, force_immediate_exit); 7444 else if (force_immediate_exit) 7445 smp_send_reschedule(vcpu->cpu); 7446 7447 kvm_wait_lapic_expire(vcpu); 7448 7449 /* The actual VMENTER/EXIT is in the .noinstr.text section. */ 7450 vmx_vcpu_enter_exit(vcpu, __vmx_vcpu_run_flags(vmx)); 7451 7452 /* All fields are clean at this point */ 7453 if (kvm_is_using_evmcs()) { 7454 current_evmcs->hv_clean_fields |= 7455 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 7456 7457 current_evmcs->hv_vp_id = kvm_hv_get_vpindex(vcpu); 7458 } 7459 7460 /* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */ 7461 if (vmx->host_debugctlmsr) 7462 update_debugctlmsr(vmx->host_debugctlmsr); 7463 7464 #ifndef CONFIG_X86_64 7465 /* 7466 * The sysexit path does not restore ds/es, so we must set them to 7467 * a reasonable value ourselves. 7468 * 7469 * We can't defer this to vmx_prepare_switch_to_host() since that 7470 * function may be executed in interrupt context, which saves and 7471 * restore segments around it, nullifying its effect. 7472 */ 7473 loadsegment(ds, __USER_DS); 7474 loadsegment(es, __USER_DS); 7475 #endif 7476 7477 pt_guest_exit(vmx); 7478 7479 kvm_load_host_xsave_state(vcpu); 7480 7481 if (is_guest_mode(vcpu)) { 7482 /* 7483 * Track VMLAUNCH/VMRESUME that have made past guest state 7484 * checking. 7485 */ 7486 if (vmx->nested.nested_run_pending && 7487 !vmx->exit_reason.failed_vmentry) 7488 ++vcpu->stat.nested_run; 7489 7490 vmx->nested.nested_run_pending = 0; 7491 } 7492 7493 if (unlikely(vmx->fail)) 7494 return EXIT_FASTPATH_NONE; 7495 7496 if (unlikely((u16)vmx->exit_reason.basic == EXIT_REASON_MCE_DURING_VMENTRY)) 7497 kvm_machine_check(); 7498 7499 trace_kvm_exit(vcpu, KVM_ISA_VMX); 7500 7501 if (unlikely(vmx->exit_reason.failed_vmentry)) 7502 return EXIT_FASTPATH_NONE; 7503 7504 vmx->loaded_vmcs->launched = 1; 7505 7506 vmx_recover_nmi_blocking(vmx); 7507 vmx_complete_interrupts(vmx); 7508 7509 return vmx_exit_handlers_fastpath(vcpu, force_immediate_exit); 7510 } 7511 7512 void vmx_vcpu_free(struct kvm_vcpu *vcpu) 7513 { 7514 struct vcpu_vmx *vmx = to_vmx(vcpu); 7515 7516 if (enable_pml) 7517 vmx_destroy_pml_buffer(vmx); 7518 free_vpid(vmx->vpid); 7519 nested_vmx_free_vcpu(vcpu); 7520 free_loaded_vmcs(vmx->loaded_vmcs); 7521 free_page((unsigned long)vmx->ve_info); 7522 } 7523 7524 int vmx_vcpu_create(struct kvm_vcpu *vcpu) 7525 { 7526 struct vmx_uret_msr *tsx_ctrl; 7527 struct vcpu_vmx *vmx; 7528 int i, err; 7529 7530 BUILD_BUG_ON(offsetof(struct vcpu_vmx, vcpu) != 0); 7531 vmx = to_vmx(vcpu); 7532 7533 INIT_LIST_HEAD(&vmx->pi_wakeup_list); 7534 7535 err = -ENOMEM; 7536 7537 vmx->vpid = allocate_vpid(); 7538 7539 /* 7540 * If PML is turned on, failure on enabling PML just results in failure 7541 * of creating the vcpu, therefore we can simplify PML logic (by 7542 * avoiding dealing with cases, such as enabling PML partially on vcpus 7543 * for the guest), etc. 7544 */ 7545 if (enable_pml) { 7546 vmx->pml_pg = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 7547 if (!vmx->pml_pg) 7548 goto free_vpid; 7549 } 7550 7551 for (i = 0; i < kvm_nr_uret_msrs; ++i) 7552 vmx->guest_uret_msrs[i].mask = -1ull; 7553 if (boot_cpu_has(X86_FEATURE_RTM)) { 7554 /* 7555 * TSX_CTRL_CPUID_CLEAR is handled in the CPUID interception. 7556 * Keep the host value unchanged to avoid changing CPUID bits 7557 * under the host kernel's feet. 7558 */ 7559 tsx_ctrl = vmx_find_uret_msr(vmx, MSR_IA32_TSX_CTRL); 7560 if (tsx_ctrl) 7561 tsx_ctrl->mask = ~(u64)TSX_CTRL_CPUID_CLEAR; 7562 } 7563 7564 err = alloc_loaded_vmcs(&vmx->vmcs01); 7565 if (err < 0) 7566 goto free_pml; 7567 7568 /* 7569 * Use Hyper-V 'Enlightened MSR Bitmap' feature when KVM runs as a 7570 * nested (L1) hypervisor and Hyper-V in L0 supports it. Enable the 7571 * feature only for vmcs01, KVM currently isn't equipped to realize any 7572 * performance benefits from enabling it for vmcs02. 7573 */ 7574 if (kvm_is_using_evmcs() && 7575 (ms_hyperv.nested_features & HV_X64_NESTED_MSR_BITMAP)) { 7576 struct hv_enlightened_vmcs *evmcs = (void *)vmx->vmcs01.vmcs; 7577 7578 evmcs->hv_enlightenments_control.msr_bitmap = 1; 7579 } 7580 7581 /* The MSR bitmap starts with all ones */ 7582 bitmap_fill(vmx->shadow_msr_intercept.read, MAX_POSSIBLE_PASSTHROUGH_MSRS); 7583 bitmap_fill(vmx->shadow_msr_intercept.write, MAX_POSSIBLE_PASSTHROUGH_MSRS); 7584 7585 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_TSC, MSR_TYPE_R); 7586 #ifdef CONFIG_X86_64 7587 vmx_disable_intercept_for_msr(vcpu, MSR_FS_BASE, MSR_TYPE_RW); 7588 vmx_disable_intercept_for_msr(vcpu, MSR_GS_BASE, MSR_TYPE_RW); 7589 vmx_disable_intercept_for_msr(vcpu, MSR_KERNEL_GS_BASE, MSR_TYPE_RW); 7590 #endif 7591 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_CS, MSR_TYPE_RW); 7592 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_ESP, MSR_TYPE_RW); 7593 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_EIP, MSR_TYPE_RW); 7594 if (kvm_cstate_in_guest(vcpu->kvm)) { 7595 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C1_RES, MSR_TYPE_R); 7596 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C3_RESIDENCY, MSR_TYPE_R); 7597 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C6_RESIDENCY, MSR_TYPE_R); 7598 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C7_RESIDENCY, MSR_TYPE_R); 7599 } 7600 7601 vmx->loaded_vmcs = &vmx->vmcs01; 7602 7603 if (cpu_need_virtualize_apic_accesses(vcpu)) { 7604 err = kvm_alloc_apic_access_page(vcpu->kvm); 7605 if (err) 7606 goto free_vmcs; 7607 } 7608 7609 if (enable_ept && !enable_unrestricted_guest) { 7610 err = init_rmode_identity_map(vcpu->kvm); 7611 if (err) 7612 goto free_vmcs; 7613 } 7614 7615 err = -ENOMEM; 7616 if (vmcs_config.cpu_based_2nd_exec_ctrl & SECONDARY_EXEC_EPT_VIOLATION_VE) { 7617 struct page *page; 7618 7619 BUILD_BUG_ON(sizeof(*vmx->ve_info) > PAGE_SIZE); 7620 7621 /* ve_info must be page aligned. */ 7622 page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 7623 if (!page) 7624 goto free_vmcs; 7625 7626 vmx->ve_info = page_to_virt(page); 7627 } 7628 7629 if (vmx_can_use_ipiv(vcpu)) 7630 WRITE_ONCE(to_kvm_vmx(vcpu->kvm)->pid_table[vcpu->vcpu_id], 7631 __pa(&vmx->pi_desc) | PID_TABLE_ENTRY_VALID); 7632 7633 return 0; 7634 7635 free_vmcs: 7636 free_loaded_vmcs(vmx->loaded_vmcs); 7637 free_pml: 7638 vmx_destroy_pml_buffer(vmx); 7639 free_vpid: 7640 free_vpid(vmx->vpid); 7641 return err; 7642 } 7643 7644 #define L1TF_MSG_SMT "L1TF CPU bug present and SMT on, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n" 7645 #define L1TF_MSG_L1D "L1TF CPU bug present and virtualization mitigation disabled, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n" 7646 7647 int vmx_vm_init(struct kvm *kvm) 7648 { 7649 if (!ple_gap) 7650 kvm->arch.pause_in_guest = true; 7651 7652 if (boot_cpu_has(X86_BUG_L1TF) && enable_ept) { 7653 switch (l1tf_mitigation) { 7654 case L1TF_MITIGATION_OFF: 7655 case L1TF_MITIGATION_FLUSH_NOWARN: 7656 /* 'I explicitly don't care' is set */ 7657 break; 7658 case L1TF_MITIGATION_FLUSH: 7659 case L1TF_MITIGATION_FLUSH_NOSMT: 7660 case L1TF_MITIGATION_FULL: 7661 /* 7662 * Warn upon starting the first VM in a potentially 7663 * insecure environment. 7664 */ 7665 if (sched_smt_active()) 7666 pr_warn_once(L1TF_MSG_SMT); 7667 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER) 7668 pr_warn_once(L1TF_MSG_L1D); 7669 break; 7670 case L1TF_MITIGATION_FULL_FORCE: 7671 /* Flush is enforced */ 7672 break; 7673 } 7674 } 7675 return 0; 7676 } 7677 7678 u8 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio) 7679 { 7680 /* 7681 * Force UC for host MMIO regions, as allowing the guest to access MMIO 7682 * with cacheable accesses will result in Machine Checks. 7683 */ 7684 if (is_mmio) 7685 return MTRR_TYPE_UNCACHABLE << VMX_EPT_MT_EPTE_SHIFT; 7686 7687 /* 7688 * Force WB and ignore guest PAT if the VM does NOT have a non-coherent 7689 * device attached. Letting the guest control memory types on Intel 7690 * CPUs may result in unexpected behavior, and so KVM's ABI is to trust 7691 * the guest to behave only as a last resort. 7692 */ 7693 if (!kvm_arch_has_noncoherent_dma(vcpu->kvm)) 7694 return (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT) | VMX_EPT_IPAT_BIT; 7695 7696 return (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT); 7697 } 7698 7699 static void vmcs_set_secondary_exec_control(struct vcpu_vmx *vmx, u32 new_ctl) 7700 { 7701 /* 7702 * These bits in the secondary execution controls field 7703 * are dynamic, the others are mostly based on the hypervisor 7704 * architecture and the guest's CPUID. Do not touch the 7705 * dynamic bits. 7706 */ 7707 u32 mask = 7708 SECONDARY_EXEC_SHADOW_VMCS | 7709 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 7710 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 7711 SECONDARY_EXEC_DESC; 7712 7713 u32 cur_ctl = secondary_exec_controls_get(vmx); 7714 7715 secondary_exec_controls_set(vmx, (new_ctl & ~mask) | (cur_ctl & mask)); 7716 } 7717 7718 /* 7719 * Generate MSR_IA32_VMX_CR{0,4}_FIXED1 according to CPUID. Only set bits 7720 * (indicating "allowed-1") if they are supported in the guest's CPUID. 7721 */ 7722 static void nested_vmx_cr_fixed1_bits_update(struct kvm_vcpu *vcpu) 7723 { 7724 struct vcpu_vmx *vmx = to_vmx(vcpu); 7725 struct kvm_cpuid_entry2 *entry; 7726 7727 vmx->nested.msrs.cr0_fixed1 = 0xffffffff; 7728 vmx->nested.msrs.cr4_fixed1 = X86_CR4_PCE; 7729 7730 #define cr4_fixed1_update(_cr4_mask, _reg, _cpuid_mask) do { \ 7731 if (entry && (entry->_reg & (_cpuid_mask))) \ 7732 vmx->nested.msrs.cr4_fixed1 |= (_cr4_mask); \ 7733 } while (0) 7734 7735 entry = kvm_find_cpuid_entry(vcpu, 0x1); 7736 cr4_fixed1_update(X86_CR4_VME, edx, feature_bit(VME)); 7737 cr4_fixed1_update(X86_CR4_PVI, edx, feature_bit(VME)); 7738 cr4_fixed1_update(X86_CR4_TSD, edx, feature_bit(TSC)); 7739 cr4_fixed1_update(X86_CR4_DE, edx, feature_bit(DE)); 7740 cr4_fixed1_update(X86_CR4_PSE, edx, feature_bit(PSE)); 7741 cr4_fixed1_update(X86_CR4_PAE, edx, feature_bit(PAE)); 7742 cr4_fixed1_update(X86_CR4_MCE, edx, feature_bit(MCE)); 7743 cr4_fixed1_update(X86_CR4_PGE, edx, feature_bit(PGE)); 7744 cr4_fixed1_update(X86_CR4_OSFXSR, edx, feature_bit(FXSR)); 7745 cr4_fixed1_update(X86_CR4_OSXMMEXCPT, edx, feature_bit(XMM)); 7746 cr4_fixed1_update(X86_CR4_VMXE, ecx, feature_bit(VMX)); 7747 cr4_fixed1_update(X86_CR4_SMXE, ecx, feature_bit(SMX)); 7748 cr4_fixed1_update(X86_CR4_PCIDE, ecx, feature_bit(PCID)); 7749 cr4_fixed1_update(X86_CR4_OSXSAVE, ecx, feature_bit(XSAVE)); 7750 7751 entry = kvm_find_cpuid_entry_index(vcpu, 0x7, 0); 7752 cr4_fixed1_update(X86_CR4_FSGSBASE, ebx, feature_bit(FSGSBASE)); 7753 cr4_fixed1_update(X86_CR4_SMEP, ebx, feature_bit(SMEP)); 7754 cr4_fixed1_update(X86_CR4_SMAP, ebx, feature_bit(SMAP)); 7755 cr4_fixed1_update(X86_CR4_PKE, ecx, feature_bit(PKU)); 7756 cr4_fixed1_update(X86_CR4_UMIP, ecx, feature_bit(UMIP)); 7757 cr4_fixed1_update(X86_CR4_LA57, ecx, feature_bit(LA57)); 7758 7759 entry = kvm_find_cpuid_entry_index(vcpu, 0x7, 1); 7760 cr4_fixed1_update(X86_CR4_LAM_SUP, eax, feature_bit(LAM)); 7761 7762 #undef cr4_fixed1_update 7763 } 7764 7765 static void update_intel_pt_cfg(struct kvm_vcpu *vcpu) 7766 { 7767 struct vcpu_vmx *vmx = to_vmx(vcpu); 7768 struct kvm_cpuid_entry2 *best = NULL; 7769 int i; 7770 7771 for (i = 0; i < PT_CPUID_LEAVES; i++) { 7772 best = kvm_find_cpuid_entry_index(vcpu, 0x14, i); 7773 if (!best) 7774 return; 7775 vmx->pt_desc.caps[CPUID_EAX + i*PT_CPUID_REGS_NUM] = best->eax; 7776 vmx->pt_desc.caps[CPUID_EBX + i*PT_CPUID_REGS_NUM] = best->ebx; 7777 vmx->pt_desc.caps[CPUID_ECX + i*PT_CPUID_REGS_NUM] = best->ecx; 7778 vmx->pt_desc.caps[CPUID_EDX + i*PT_CPUID_REGS_NUM] = best->edx; 7779 } 7780 7781 /* Get the number of configurable Address Ranges for filtering */ 7782 vmx->pt_desc.num_address_ranges = intel_pt_validate_cap(vmx->pt_desc.caps, 7783 PT_CAP_num_address_ranges); 7784 7785 /* Initialize and clear the no dependency bits */ 7786 vmx->pt_desc.ctl_bitmask = ~(RTIT_CTL_TRACEEN | RTIT_CTL_OS | 7787 RTIT_CTL_USR | RTIT_CTL_TSC_EN | RTIT_CTL_DISRETC | 7788 RTIT_CTL_BRANCH_EN); 7789 7790 /* 7791 * If CPUID.(EAX=14H,ECX=0):EBX[0]=1 CR3Filter can be set otherwise 7792 * will inject an #GP 7793 */ 7794 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_cr3_filtering)) 7795 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_CR3EN; 7796 7797 /* 7798 * If CPUID.(EAX=14H,ECX=0):EBX[1]=1 CYCEn, CycThresh and 7799 * PSBFreq can be set 7800 */ 7801 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc)) 7802 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_CYCLEACC | 7803 RTIT_CTL_CYC_THRESH | RTIT_CTL_PSB_FREQ); 7804 7805 /* 7806 * If CPUID.(EAX=14H,ECX=0):EBX[3]=1 MTCEn and MTCFreq can be set 7807 */ 7808 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc)) 7809 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_MTC_EN | 7810 RTIT_CTL_MTC_RANGE); 7811 7812 /* If CPUID.(EAX=14H,ECX=0):EBX[4]=1 FUPonPTW and PTWEn can be set */ 7813 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_ptwrite)) 7814 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_FUP_ON_PTW | 7815 RTIT_CTL_PTW_EN); 7816 7817 /* If CPUID.(EAX=14H,ECX=0):EBX[5]=1 PwrEvEn can be set */ 7818 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_power_event_trace)) 7819 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_PWR_EVT_EN; 7820 7821 /* If CPUID.(EAX=14H,ECX=0):ECX[0]=1 ToPA can be set */ 7822 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_topa_output)) 7823 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_TOPA; 7824 7825 /* If CPUID.(EAX=14H,ECX=0):ECX[3]=1 FabricEn can be set */ 7826 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_output_subsys)) 7827 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_FABRIC_EN; 7828 7829 /* unmask address range configure area */ 7830 for (i = 0; i < vmx->pt_desc.num_address_ranges; i++) 7831 vmx->pt_desc.ctl_bitmask &= ~(0xfULL << (32 + i * 4)); 7832 } 7833 7834 void vmx_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) 7835 { 7836 struct vcpu_vmx *vmx = to_vmx(vcpu); 7837 7838 /* 7839 * XSAVES is effectively enabled if and only if XSAVE is also exposed 7840 * to the guest. XSAVES depends on CR4.OSXSAVE, and CR4.OSXSAVE can be 7841 * set if and only if XSAVE is supported. 7842 */ 7843 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_XSAVE)) 7844 guest_cpu_cap_clear(vcpu, X86_FEATURE_XSAVES); 7845 7846 vmx_setup_uret_msrs(vmx); 7847 7848 if (cpu_has_secondary_exec_ctrls()) 7849 vmcs_set_secondary_exec_control(vmx, 7850 vmx_secondary_exec_control(vmx)); 7851 7852 if (guest_cpu_cap_has(vcpu, X86_FEATURE_VMX)) 7853 vmx->msr_ia32_feature_control_valid_bits |= 7854 FEAT_CTL_VMX_ENABLED_INSIDE_SMX | 7855 FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX; 7856 else 7857 vmx->msr_ia32_feature_control_valid_bits &= 7858 ~(FEAT_CTL_VMX_ENABLED_INSIDE_SMX | 7859 FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX); 7860 7861 if (guest_cpu_cap_has(vcpu, X86_FEATURE_VMX)) 7862 nested_vmx_cr_fixed1_bits_update(vcpu); 7863 7864 if (boot_cpu_has(X86_FEATURE_INTEL_PT) && 7865 guest_cpu_cap_has(vcpu, X86_FEATURE_INTEL_PT)) 7866 update_intel_pt_cfg(vcpu); 7867 7868 if (boot_cpu_has(X86_FEATURE_RTM)) { 7869 struct vmx_uret_msr *msr; 7870 msr = vmx_find_uret_msr(vmx, MSR_IA32_TSX_CTRL); 7871 if (msr) { 7872 bool enabled = guest_cpu_cap_has(vcpu, X86_FEATURE_RTM); 7873 vmx_set_guest_uret_msr(vmx, msr, enabled ? 0 : TSX_CTRL_RTM_DISABLE); 7874 } 7875 } 7876 7877 if (kvm_cpu_cap_has(X86_FEATURE_XFD)) 7878 vmx_set_intercept_for_msr(vcpu, MSR_IA32_XFD_ERR, MSR_TYPE_R, 7879 !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD)); 7880 7881 if (boot_cpu_has(X86_FEATURE_IBPB)) 7882 vmx_set_intercept_for_msr(vcpu, MSR_IA32_PRED_CMD, MSR_TYPE_W, 7883 !guest_has_pred_cmd_msr(vcpu)); 7884 7885 if (boot_cpu_has(X86_FEATURE_FLUSH_L1D)) 7886 vmx_set_intercept_for_msr(vcpu, MSR_IA32_FLUSH_CMD, MSR_TYPE_W, 7887 !guest_cpu_cap_has(vcpu, X86_FEATURE_FLUSH_L1D)); 7888 7889 set_cr4_guest_host_mask(vmx); 7890 7891 vmx_write_encls_bitmap(vcpu, NULL); 7892 if (guest_cpu_cap_has(vcpu, X86_FEATURE_SGX)) 7893 vmx->msr_ia32_feature_control_valid_bits |= FEAT_CTL_SGX_ENABLED; 7894 else 7895 vmx->msr_ia32_feature_control_valid_bits &= ~FEAT_CTL_SGX_ENABLED; 7896 7897 if (guest_cpu_cap_has(vcpu, X86_FEATURE_SGX_LC)) 7898 vmx->msr_ia32_feature_control_valid_bits |= 7899 FEAT_CTL_SGX_LC_ENABLED; 7900 else 7901 vmx->msr_ia32_feature_control_valid_bits &= 7902 ~FEAT_CTL_SGX_LC_ENABLED; 7903 7904 /* Refresh #PF interception to account for MAXPHYADDR changes. */ 7905 vmx_update_exception_bitmap(vcpu); 7906 } 7907 7908 static __init u64 vmx_get_perf_capabilities(void) 7909 { 7910 u64 perf_cap = PMU_CAP_FW_WRITES; 7911 u64 host_perf_cap = 0; 7912 7913 if (!enable_pmu) 7914 return 0; 7915 7916 if (boot_cpu_has(X86_FEATURE_PDCM)) 7917 rdmsrl(MSR_IA32_PERF_CAPABILITIES, host_perf_cap); 7918 7919 if (!cpu_feature_enabled(X86_FEATURE_ARCH_LBR)) { 7920 x86_perf_get_lbr(&vmx_lbr_caps); 7921 7922 /* 7923 * KVM requires LBR callstack support, as the overhead due to 7924 * context switching LBRs without said support is too high. 7925 * See intel_pmu_create_guest_lbr_event() for more info. 7926 */ 7927 if (!vmx_lbr_caps.has_callstack) 7928 memset(&vmx_lbr_caps, 0, sizeof(vmx_lbr_caps)); 7929 else if (vmx_lbr_caps.nr) 7930 perf_cap |= host_perf_cap & PMU_CAP_LBR_FMT; 7931 } 7932 7933 if (vmx_pebs_supported()) { 7934 perf_cap |= host_perf_cap & PERF_CAP_PEBS_MASK; 7935 7936 /* 7937 * Disallow adaptive PEBS as it is functionally broken, can be 7938 * used by the guest to read *host* LBRs, and can be used to 7939 * bypass userspace event filters. To correctly and safely 7940 * support adaptive PEBS, KVM needs to: 7941 * 7942 * 1. Account for the ADAPTIVE flag when (re)programming fixed 7943 * counters. 7944 * 7945 * 2. Gain support from perf (or take direct control of counter 7946 * programming) to support events without adaptive PEBS 7947 * enabled for the hardware counter. 7948 * 7949 * 3. Ensure LBR MSRs cannot hold host data on VM-Entry with 7950 * adaptive PEBS enabled and MSR_PEBS_DATA_CFG.LBRS=1. 7951 * 7952 * 4. Document which PMU events are effectively exposed to the 7953 * guest via adaptive PEBS, and make adaptive PEBS mutually 7954 * exclusive with KVM_SET_PMU_EVENT_FILTER if necessary. 7955 */ 7956 perf_cap &= ~PERF_CAP_PEBS_BASELINE; 7957 } 7958 7959 return perf_cap; 7960 } 7961 7962 static __init void vmx_set_cpu_caps(void) 7963 { 7964 kvm_set_cpu_caps(); 7965 7966 /* CPUID 0x1 */ 7967 if (nested) 7968 kvm_cpu_cap_set(X86_FEATURE_VMX); 7969 7970 /* CPUID 0x7 */ 7971 if (kvm_mpx_supported()) 7972 kvm_cpu_cap_check_and_set(X86_FEATURE_MPX); 7973 if (!cpu_has_vmx_invpcid()) 7974 kvm_cpu_cap_clear(X86_FEATURE_INVPCID); 7975 if (vmx_pt_mode_is_host_guest()) 7976 kvm_cpu_cap_check_and_set(X86_FEATURE_INTEL_PT); 7977 if (vmx_pebs_supported()) { 7978 kvm_cpu_cap_check_and_set(X86_FEATURE_DS); 7979 kvm_cpu_cap_check_and_set(X86_FEATURE_DTES64); 7980 } 7981 7982 if (!enable_pmu) 7983 kvm_cpu_cap_clear(X86_FEATURE_PDCM); 7984 kvm_caps.supported_perf_cap = vmx_get_perf_capabilities(); 7985 7986 if (!enable_sgx) { 7987 kvm_cpu_cap_clear(X86_FEATURE_SGX); 7988 kvm_cpu_cap_clear(X86_FEATURE_SGX_LC); 7989 kvm_cpu_cap_clear(X86_FEATURE_SGX1); 7990 kvm_cpu_cap_clear(X86_FEATURE_SGX2); 7991 kvm_cpu_cap_clear(X86_FEATURE_SGX_EDECCSSA); 7992 } 7993 7994 if (vmx_umip_emulated()) 7995 kvm_cpu_cap_set(X86_FEATURE_UMIP); 7996 7997 /* CPUID 0xD.1 */ 7998 kvm_caps.supported_xss = 0; 7999 if (!cpu_has_vmx_xsaves()) 8000 kvm_cpu_cap_clear(X86_FEATURE_XSAVES); 8001 8002 /* CPUID 0x80000001 and 0x7 (RDPID) */ 8003 if (!cpu_has_vmx_rdtscp()) { 8004 kvm_cpu_cap_clear(X86_FEATURE_RDTSCP); 8005 kvm_cpu_cap_clear(X86_FEATURE_RDPID); 8006 } 8007 8008 if (cpu_has_vmx_waitpkg()) 8009 kvm_cpu_cap_check_and_set(X86_FEATURE_WAITPKG); 8010 } 8011 8012 static int vmx_check_intercept_io(struct kvm_vcpu *vcpu, 8013 struct x86_instruction_info *info) 8014 { 8015 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 8016 unsigned short port; 8017 bool intercept; 8018 int size; 8019 8020 if (info->intercept == x86_intercept_in || 8021 info->intercept == x86_intercept_ins) { 8022 port = info->src_val; 8023 size = info->dst_bytes; 8024 } else { 8025 port = info->dst_val; 8026 size = info->src_bytes; 8027 } 8028 8029 /* 8030 * If the 'use IO bitmaps' VM-execution control is 0, IO instruction 8031 * VM-exits depend on the 'unconditional IO exiting' VM-execution 8032 * control. 8033 * 8034 * Otherwise, IO instruction VM-exits are controlled by the IO bitmaps. 8035 */ 8036 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 8037 intercept = nested_cpu_has(vmcs12, 8038 CPU_BASED_UNCOND_IO_EXITING); 8039 else 8040 intercept = nested_vmx_check_io_bitmaps(vcpu, port, size); 8041 8042 /* FIXME: produce nested vmexit and return X86EMUL_INTERCEPTED. */ 8043 return intercept ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; 8044 } 8045 8046 int vmx_check_intercept(struct kvm_vcpu *vcpu, 8047 struct x86_instruction_info *info, 8048 enum x86_intercept_stage stage, 8049 struct x86_exception *exception) 8050 { 8051 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 8052 8053 switch (info->intercept) { 8054 /* 8055 * RDPID causes #UD if disabled through secondary execution controls. 8056 * Because it is marked as EmulateOnUD, we need to intercept it here. 8057 * Note, RDPID is hidden behind ENABLE_RDTSCP. 8058 */ 8059 case x86_intercept_rdpid: 8060 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_RDTSCP)) { 8061 exception->vector = UD_VECTOR; 8062 exception->error_code_valid = false; 8063 return X86EMUL_PROPAGATE_FAULT; 8064 } 8065 break; 8066 8067 case x86_intercept_in: 8068 case x86_intercept_ins: 8069 case x86_intercept_out: 8070 case x86_intercept_outs: 8071 return vmx_check_intercept_io(vcpu, info); 8072 8073 case x86_intercept_lgdt: 8074 case x86_intercept_lidt: 8075 case x86_intercept_lldt: 8076 case x86_intercept_ltr: 8077 case x86_intercept_sgdt: 8078 case x86_intercept_sidt: 8079 case x86_intercept_sldt: 8080 case x86_intercept_str: 8081 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC)) 8082 return X86EMUL_CONTINUE; 8083 8084 /* FIXME: produce nested vmexit and return X86EMUL_INTERCEPTED. */ 8085 break; 8086 8087 case x86_intercept_pause: 8088 /* 8089 * PAUSE is a single-byte NOP with a REPE prefix, i.e. collides 8090 * with vanilla NOPs in the emulator. Apply the interception 8091 * check only to actual PAUSE instructions. Don't check 8092 * PAUSE-loop-exiting, software can't expect a given PAUSE to 8093 * exit, i.e. KVM is within its rights to allow L2 to execute 8094 * the PAUSE. 8095 */ 8096 if ((info->rep_prefix != REPE_PREFIX) || 8097 !nested_cpu_has2(vmcs12, CPU_BASED_PAUSE_EXITING)) 8098 return X86EMUL_CONTINUE; 8099 8100 break; 8101 8102 /* TODO: check more intercepts... */ 8103 default: 8104 break; 8105 } 8106 8107 return X86EMUL_UNHANDLEABLE; 8108 } 8109 8110 #ifdef CONFIG_X86_64 8111 /* (a << shift) / divisor, return 1 if overflow otherwise 0 */ 8112 static inline int u64_shl_div_u64(u64 a, unsigned int shift, 8113 u64 divisor, u64 *result) 8114 { 8115 u64 low = a << shift, high = a >> (64 - shift); 8116 8117 /* To avoid the overflow on divq */ 8118 if (high >= divisor) 8119 return 1; 8120 8121 /* Low hold the result, high hold rem which is discarded */ 8122 asm("divq %2\n\t" : "=a" (low), "=d" (high) : 8123 "rm" (divisor), "0" (low), "1" (high)); 8124 *result = low; 8125 8126 return 0; 8127 } 8128 8129 int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, 8130 bool *expired) 8131 { 8132 struct vcpu_vmx *vmx; 8133 u64 tscl, guest_tscl, delta_tsc, lapic_timer_advance_cycles; 8134 struct kvm_timer *ktimer = &vcpu->arch.apic->lapic_timer; 8135 8136 vmx = to_vmx(vcpu); 8137 tscl = rdtsc(); 8138 guest_tscl = kvm_read_l1_tsc(vcpu, tscl); 8139 delta_tsc = max(guest_deadline_tsc, guest_tscl) - guest_tscl; 8140 lapic_timer_advance_cycles = nsec_to_cycles(vcpu, 8141 ktimer->timer_advance_ns); 8142 8143 if (delta_tsc > lapic_timer_advance_cycles) 8144 delta_tsc -= lapic_timer_advance_cycles; 8145 else 8146 delta_tsc = 0; 8147 8148 /* Convert to host delta tsc if tsc scaling is enabled */ 8149 if (vcpu->arch.l1_tsc_scaling_ratio != kvm_caps.default_tsc_scaling_ratio && 8150 delta_tsc && u64_shl_div_u64(delta_tsc, 8151 kvm_caps.tsc_scaling_ratio_frac_bits, 8152 vcpu->arch.l1_tsc_scaling_ratio, &delta_tsc)) 8153 return -ERANGE; 8154 8155 /* 8156 * If the delta tsc can't fit in the 32 bit after the multi shift, 8157 * we can't use the preemption timer. 8158 * It's possible that it fits on later vmentries, but checking 8159 * on every vmentry is costly so we just use an hrtimer. 8160 */ 8161 if (delta_tsc >> (cpu_preemption_timer_multi + 32)) 8162 return -ERANGE; 8163 8164 vmx->hv_deadline_tsc = tscl + delta_tsc; 8165 *expired = !delta_tsc; 8166 return 0; 8167 } 8168 8169 void vmx_cancel_hv_timer(struct kvm_vcpu *vcpu) 8170 { 8171 to_vmx(vcpu)->hv_deadline_tsc = -1; 8172 } 8173 #endif 8174 8175 void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) 8176 { 8177 struct vcpu_vmx *vmx = to_vmx(vcpu); 8178 8179 if (WARN_ON_ONCE(!enable_pml)) 8180 return; 8181 8182 if (is_guest_mode(vcpu)) { 8183 vmx->nested.update_vmcs01_cpu_dirty_logging = true; 8184 return; 8185 } 8186 8187 /* 8188 * Note, nr_memslots_dirty_logging can be changed concurrent with this 8189 * code, but in that case another update request will be made and so 8190 * the guest will never run with a stale PML value. 8191 */ 8192 if (atomic_read(&vcpu->kvm->nr_memslots_dirty_logging)) 8193 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_ENABLE_PML); 8194 else 8195 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_ENABLE_PML); 8196 } 8197 8198 void vmx_setup_mce(struct kvm_vcpu *vcpu) 8199 { 8200 if (vcpu->arch.mcg_cap & MCG_LMCE_P) 8201 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |= 8202 FEAT_CTL_LMCE_ENABLED; 8203 else 8204 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &= 8205 ~FEAT_CTL_LMCE_ENABLED; 8206 } 8207 8208 #ifdef CONFIG_KVM_SMM 8209 int vmx_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 8210 { 8211 /* we need a nested vmexit to enter SMM, postpone if run is pending */ 8212 if (to_vmx(vcpu)->nested.nested_run_pending) 8213 return -EBUSY; 8214 return !is_smm(vcpu); 8215 } 8216 8217 int vmx_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram) 8218 { 8219 struct vcpu_vmx *vmx = to_vmx(vcpu); 8220 8221 /* 8222 * TODO: Implement custom flows for forcing the vCPU out/in of L2 on 8223 * SMI and RSM. Using the common VM-Exit + VM-Enter routines is wrong 8224 * SMI and RSM only modify state that is saved and restored via SMRAM. 8225 * E.g. most MSRs are left untouched, but many are modified by VM-Exit 8226 * and VM-Enter, and thus L2's values may be corrupted on SMI+RSM. 8227 */ 8228 vmx->nested.smm.guest_mode = is_guest_mode(vcpu); 8229 if (vmx->nested.smm.guest_mode) 8230 nested_vmx_vmexit(vcpu, -1, 0, 0); 8231 8232 vmx->nested.smm.vmxon = vmx->nested.vmxon; 8233 vmx->nested.vmxon = false; 8234 vmx_clear_hlt(vcpu); 8235 return 0; 8236 } 8237 8238 int vmx_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram) 8239 { 8240 struct vcpu_vmx *vmx = to_vmx(vcpu); 8241 int ret; 8242 8243 if (vmx->nested.smm.vmxon) { 8244 vmx->nested.vmxon = true; 8245 vmx->nested.smm.vmxon = false; 8246 } 8247 8248 if (vmx->nested.smm.guest_mode) { 8249 ret = nested_vmx_enter_non_root_mode(vcpu, false); 8250 if (ret) 8251 return ret; 8252 8253 vmx->nested.nested_run_pending = 1; 8254 vmx->nested.smm.guest_mode = false; 8255 } 8256 return 0; 8257 } 8258 8259 void vmx_enable_smi_window(struct kvm_vcpu *vcpu) 8260 { 8261 /* RSM will cause a vmexit anyway. */ 8262 } 8263 #endif 8264 8265 bool vmx_apic_init_signal_blocked(struct kvm_vcpu *vcpu) 8266 { 8267 return to_vmx(vcpu)->nested.vmxon && !is_guest_mode(vcpu); 8268 } 8269 8270 void vmx_migrate_timers(struct kvm_vcpu *vcpu) 8271 { 8272 if (is_guest_mode(vcpu)) { 8273 struct hrtimer *timer = &to_vmx(vcpu)->nested.preemption_timer; 8274 8275 if (hrtimer_try_to_cancel(timer) == 1) 8276 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED); 8277 } 8278 } 8279 8280 void vmx_hardware_unsetup(void) 8281 { 8282 kvm_set_posted_intr_wakeup_handler(NULL); 8283 8284 if (nested) 8285 nested_vmx_hardware_unsetup(); 8286 8287 free_kvm_area(); 8288 } 8289 8290 void vmx_vm_destroy(struct kvm *kvm) 8291 { 8292 struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); 8293 8294 free_pages((unsigned long)kvm_vmx->pid_table, vmx_get_pid_table_order(kvm)); 8295 } 8296 8297 /* 8298 * Note, the SDM states that the linear address is masked *after* the modified 8299 * canonicality check, whereas KVM masks (untags) the address and then performs 8300 * a "normal" canonicality check. Functionally, the two methods are identical, 8301 * and when the masking occurs relative to the canonicality check isn't visible 8302 * to software, i.e. KVM's behavior doesn't violate the SDM. 8303 */ 8304 gva_t vmx_get_untagged_addr(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags) 8305 { 8306 int lam_bit; 8307 unsigned long cr3_bits; 8308 8309 if (flags & (X86EMUL_F_FETCH | X86EMUL_F_IMPLICIT | X86EMUL_F_INVLPG)) 8310 return gva; 8311 8312 if (!is_64_bit_mode(vcpu)) 8313 return gva; 8314 8315 /* 8316 * Bit 63 determines if the address should be treated as user address 8317 * or a supervisor address. 8318 */ 8319 if (!(gva & BIT_ULL(63))) { 8320 cr3_bits = kvm_get_active_cr3_lam_bits(vcpu); 8321 if (!(cr3_bits & (X86_CR3_LAM_U57 | X86_CR3_LAM_U48))) 8322 return gva; 8323 8324 /* LAM_U48 is ignored if LAM_U57 is set. */ 8325 lam_bit = cr3_bits & X86_CR3_LAM_U57 ? 56 : 47; 8326 } else { 8327 if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_LAM_SUP)) 8328 return gva; 8329 8330 lam_bit = kvm_is_cr4_bit_set(vcpu, X86_CR4_LA57) ? 56 : 47; 8331 } 8332 8333 /* 8334 * Untag the address by sign-extending the lam_bit, but NOT to bit 63. 8335 * Bit 63 is retained from the raw virtual address so that untagging 8336 * doesn't change a user access to a supervisor access, and vice versa. 8337 */ 8338 return (sign_extend64(gva, lam_bit) & ~BIT_ULL(63)) | (gva & BIT_ULL(63)); 8339 } 8340 8341 static unsigned int vmx_handle_intel_pt_intr(void) 8342 { 8343 struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); 8344 8345 /* '0' on failure so that the !PT case can use a RET0 static call. */ 8346 if (!vcpu || !kvm_handling_nmi_from_guest(vcpu)) 8347 return 0; 8348 8349 kvm_make_request(KVM_REQ_PMI, vcpu); 8350 __set_bit(MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT, 8351 (unsigned long *)&vcpu->arch.pmu.global_status); 8352 return 1; 8353 } 8354 8355 static __init void vmx_setup_user_return_msrs(void) 8356 { 8357 8358 /* 8359 * Though SYSCALL is only supported in 64-bit mode on Intel CPUs, kvm 8360 * will emulate SYSCALL in legacy mode if the vendor string in guest 8361 * CPUID.0:{EBX,ECX,EDX} is "AuthenticAMD" or "AMDisbetter!" To 8362 * support this emulation, MSR_STAR is included in the list for i386, 8363 * but is never loaded into hardware. MSR_CSTAR is also never loaded 8364 * into hardware and is here purely for emulation purposes. 8365 */ 8366 const u32 vmx_uret_msrs_list[] = { 8367 #ifdef CONFIG_X86_64 8368 MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR, 8369 #endif 8370 MSR_EFER, MSR_TSC_AUX, MSR_STAR, 8371 MSR_IA32_TSX_CTRL, 8372 }; 8373 int i; 8374 8375 BUILD_BUG_ON(ARRAY_SIZE(vmx_uret_msrs_list) != MAX_NR_USER_RETURN_MSRS); 8376 8377 for (i = 0; i < ARRAY_SIZE(vmx_uret_msrs_list); ++i) 8378 kvm_add_user_return_msr(vmx_uret_msrs_list[i]); 8379 } 8380 8381 static void __init vmx_setup_me_spte_mask(void) 8382 { 8383 u64 me_mask = 0; 8384 8385 /* 8386 * On pre-MKTME system, boot_cpu_data.x86_phys_bits equals to 8387 * kvm_host.maxphyaddr. On MKTME and/or TDX capable systems, 8388 * boot_cpu_data.x86_phys_bits holds the actual physical address 8389 * w/o the KeyID bits, and kvm_host.maxphyaddr equals to 8390 * MAXPHYADDR reported by CPUID. Those bits between are KeyID bits. 8391 */ 8392 if (boot_cpu_data.x86_phys_bits != kvm_host.maxphyaddr) 8393 me_mask = rsvd_bits(boot_cpu_data.x86_phys_bits, 8394 kvm_host.maxphyaddr - 1); 8395 8396 /* 8397 * Unlike SME, host kernel doesn't support setting up any 8398 * MKTME KeyID on Intel platforms. No memory encryption 8399 * bits should be included into the SPTE. 8400 */ 8401 kvm_mmu_set_me_spte_mask(0, me_mask); 8402 } 8403 8404 __init int vmx_hardware_setup(void) 8405 { 8406 unsigned long host_bndcfgs; 8407 struct desc_ptr dt; 8408 int r; 8409 8410 store_idt(&dt); 8411 host_idt_base = dt.address; 8412 8413 vmx_setup_user_return_msrs(); 8414 8415 if (setup_vmcs_config(&vmcs_config, &vmx_capability) < 0) 8416 return -EIO; 8417 8418 if (boot_cpu_has(X86_FEATURE_NX)) 8419 kvm_enable_efer_bits(EFER_NX); 8420 8421 if (boot_cpu_has(X86_FEATURE_MPX)) { 8422 rdmsrl(MSR_IA32_BNDCFGS, host_bndcfgs); 8423 WARN_ONCE(host_bndcfgs, "BNDCFGS in host will be lost"); 8424 } 8425 8426 if (!cpu_has_vmx_mpx()) 8427 kvm_caps.supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS | 8428 XFEATURE_MASK_BNDCSR); 8429 8430 if (!cpu_has_vmx_vpid() || !cpu_has_vmx_invvpid() || 8431 !(cpu_has_vmx_invvpid_single() || cpu_has_vmx_invvpid_global())) 8432 enable_vpid = 0; 8433 8434 if (!cpu_has_vmx_ept() || 8435 !cpu_has_vmx_ept_4levels() || 8436 !cpu_has_vmx_ept_mt_wb() || 8437 !cpu_has_vmx_invept_global()) 8438 enable_ept = 0; 8439 8440 /* NX support is required for shadow paging. */ 8441 if (!enable_ept && !boot_cpu_has(X86_FEATURE_NX)) { 8442 pr_err_ratelimited("NX (Execute Disable) not supported\n"); 8443 return -EOPNOTSUPP; 8444 } 8445 8446 if (!cpu_has_vmx_ept_ad_bits() || !enable_ept) 8447 enable_ept_ad_bits = 0; 8448 8449 if (!cpu_has_vmx_unrestricted_guest() || !enable_ept) 8450 enable_unrestricted_guest = 0; 8451 8452 if (!cpu_has_vmx_flexpriority()) 8453 flexpriority_enabled = 0; 8454 8455 if (!cpu_has_virtual_nmis()) 8456 enable_vnmi = 0; 8457 8458 #ifdef CONFIG_X86_SGX_KVM 8459 if (!cpu_has_vmx_encls_vmexit()) 8460 enable_sgx = false; 8461 #endif 8462 8463 /* 8464 * set_apic_access_page_addr() is used to reload apic access 8465 * page upon invalidation. No need to do anything if not 8466 * using the APIC_ACCESS_ADDR VMCS field. 8467 */ 8468 if (!flexpriority_enabled) 8469 vt_x86_ops.set_apic_access_page_addr = NULL; 8470 8471 if (!cpu_has_vmx_tpr_shadow()) 8472 vt_x86_ops.update_cr8_intercept = NULL; 8473 8474 #if IS_ENABLED(CONFIG_HYPERV) 8475 if (ms_hyperv.nested_features & HV_X64_NESTED_GUEST_MAPPING_FLUSH 8476 && enable_ept) { 8477 vt_x86_ops.flush_remote_tlbs = hv_flush_remote_tlbs; 8478 vt_x86_ops.flush_remote_tlbs_range = hv_flush_remote_tlbs_range; 8479 } 8480 #endif 8481 8482 if (!cpu_has_vmx_ple()) { 8483 ple_gap = 0; 8484 ple_window = 0; 8485 ple_window_grow = 0; 8486 ple_window_max = 0; 8487 ple_window_shrink = 0; 8488 } 8489 8490 if (!cpu_has_vmx_apicv()) 8491 enable_apicv = 0; 8492 if (!enable_apicv) 8493 vt_x86_ops.sync_pir_to_irr = NULL; 8494 8495 if (!enable_apicv || !cpu_has_vmx_ipiv()) 8496 enable_ipiv = false; 8497 8498 if (cpu_has_vmx_tsc_scaling()) 8499 kvm_caps.has_tsc_control = true; 8500 8501 kvm_caps.max_tsc_scaling_ratio = KVM_VMX_TSC_MULTIPLIER_MAX; 8502 kvm_caps.tsc_scaling_ratio_frac_bits = 48; 8503 kvm_caps.has_bus_lock_exit = cpu_has_vmx_bus_lock_detection(); 8504 kvm_caps.has_notify_vmexit = cpu_has_notify_vmexit(); 8505 8506 set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */ 8507 8508 if (enable_ept) 8509 kvm_mmu_set_ept_masks(enable_ept_ad_bits, 8510 cpu_has_vmx_ept_execute_only()); 8511 8512 /* 8513 * Setup shadow_me_value/shadow_me_mask to include MKTME KeyID 8514 * bits to shadow_zero_check. 8515 */ 8516 vmx_setup_me_spte_mask(); 8517 8518 kvm_configure_mmu(enable_ept, 0, vmx_get_max_ept_level(), 8519 ept_caps_to_lpage_level(vmx_capability.ept)); 8520 8521 /* 8522 * Only enable PML when hardware supports PML feature, and both EPT 8523 * and EPT A/D bit features are enabled -- PML depends on them to work. 8524 */ 8525 if (!enable_ept || !enable_ept_ad_bits || !cpu_has_vmx_pml()) 8526 enable_pml = 0; 8527 8528 if (!enable_pml) 8529 vt_x86_ops.cpu_dirty_log_size = 0; 8530 8531 if (!cpu_has_vmx_preemption_timer()) 8532 enable_preemption_timer = false; 8533 8534 if (enable_preemption_timer) { 8535 u64 use_timer_freq = 5000ULL * 1000 * 1000; 8536 8537 cpu_preemption_timer_multi = 8538 vmx_misc_preemption_timer_rate(vmcs_config.misc); 8539 8540 if (tsc_khz) 8541 use_timer_freq = (u64)tsc_khz * 1000; 8542 use_timer_freq >>= cpu_preemption_timer_multi; 8543 8544 /* 8545 * KVM "disables" the preemption timer by setting it to its max 8546 * value. Don't use the timer if it might cause spurious exits 8547 * at a rate faster than 0.1 Hz (of uninterrupted guest time). 8548 */ 8549 if (use_timer_freq > 0xffffffffu / 10) 8550 enable_preemption_timer = false; 8551 } 8552 8553 if (!enable_preemption_timer) { 8554 vt_x86_ops.set_hv_timer = NULL; 8555 vt_x86_ops.cancel_hv_timer = NULL; 8556 } 8557 8558 kvm_caps.supported_mce_cap |= MCG_LMCE_P; 8559 kvm_caps.supported_mce_cap |= MCG_CMCI_P; 8560 8561 if (pt_mode != PT_MODE_SYSTEM && pt_mode != PT_MODE_HOST_GUEST) 8562 return -EINVAL; 8563 if (!enable_ept || !enable_pmu || !cpu_has_vmx_intel_pt()) 8564 pt_mode = PT_MODE_SYSTEM; 8565 if (pt_mode == PT_MODE_HOST_GUEST) 8566 vt_init_ops.handle_intel_pt_intr = vmx_handle_intel_pt_intr; 8567 else 8568 vt_init_ops.handle_intel_pt_intr = NULL; 8569 8570 setup_default_sgx_lepubkeyhash(); 8571 8572 if (nested) { 8573 nested_vmx_setup_ctls_msrs(&vmcs_config, vmx_capability.ept); 8574 8575 r = nested_vmx_hardware_setup(kvm_vmx_exit_handlers); 8576 if (r) 8577 return r; 8578 } 8579 8580 vmx_set_cpu_caps(); 8581 8582 r = alloc_kvm_area(); 8583 if (r && nested) 8584 nested_vmx_hardware_unsetup(); 8585 8586 kvm_set_posted_intr_wakeup_handler(pi_wakeup_handler); 8587 8588 return r; 8589 } 8590 8591 static void vmx_cleanup_l1d_flush(void) 8592 { 8593 if (vmx_l1d_flush_pages) { 8594 free_pages((unsigned long)vmx_l1d_flush_pages, L1D_CACHE_ORDER); 8595 vmx_l1d_flush_pages = NULL; 8596 } 8597 /* Restore state so sysfs ignores VMX */ 8598 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; 8599 } 8600 8601 static void __vmx_exit(void) 8602 { 8603 allow_smaller_maxphyaddr = false; 8604 8605 vmx_cleanup_l1d_flush(); 8606 } 8607 8608 static void __exit vmx_exit(void) 8609 { 8610 kvm_exit(); 8611 __vmx_exit(); 8612 kvm_x86_vendor_exit(); 8613 8614 } 8615 module_exit(vmx_exit); 8616 8617 static int __init vmx_init(void) 8618 { 8619 int r, cpu; 8620 8621 if (!kvm_is_vmx_supported()) 8622 return -EOPNOTSUPP; 8623 8624 /* 8625 * Note, hv_init_evmcs() touches only VMX knobs, i.e. there's nothing 8626 * to unwind if a later step fails. 8627 */ 8628 hv_init_evmcs(); 8629 8630 r = kvm_x86_vendor_init(&vt_init_ops); 8631 if (r) 8632 return r; 8633 8634 /* 8635 * Must be called after common x86 init so enable_ept is properly set 8636 * up. Hand the parameter mitigation value in which was stored in 8637 * the pre module init parser. If no parameter was given, it will 8638 * contain 'auto' which will be turned into the default 'cond' 8639 * mitigation mode. 8640 */ 8641 r = vmx_setup_l1d_flush(vmentry_l1d_flush_param); 8642 if (r) 8643 goto err_l1d_flush; 8644 8645 for_each_possible_cpu(cpu) { 8646 INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu)); 8647 8648 pi_init_cpu(cpu); 8649 } 8650 8651 vmx_check_vmcs12_offsets(); 8652 8653 /* 8654 * Shadow paging doesn't have a (further) performance penalty 8655 * from GUEST_MAXPHYADDR < HOST_MAXPHYADDR so enable it 8656 * by default 8657 */ 8658 if (!enable_ept) 8659 allow_smaller_maxphyaddr = true; 8660 8661 /* 8662 * Common KVM initialization _must_ come last, after this, /dev/kvm is 8663 * exposed to userspace! 8664 */ 8665 r = kvm_init(sizeof(struct vcpu_vmx), __alignof__(struct vcpu_vmx), 8666 THIS_MODULE); 8667 if (r) 8668 goto err_kvm_init; 8669 8670 return 0; 8671 8672 err_kvm_init: 8673 __vmx_exit(); 8674 err_l1d_flush: 8675 kvm_x86_vendor_exit(); 8676 return r; 8677 } 8678 module_init(vmx_init); 8679