1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * This module enables machines with Intel VT-x extensions to run virtual 6 * machines without emulation or binary translation. 7 * 8 * Copyright (C) 2006 Qumranet, Inc. 9 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 10 * 11 * Authors: 12 * Avi Kivity <avi@qumranet.com> 13 * Yaniv Kamay <yaniv@qumranet.com> 14 */ 15 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 16 17 #include <linux/highmem.h> 18 #include <linux/hrtimer.h> 19 #include <linux/kernel.h> 20 #include <linux/kvm_host.h> 21 #include <linux/module.h> 22 #include <linux/moduleparam.h> 23 #include <linux/mod_devicetable.h> 24 #include <linux/mm.h> 25 #include <linux/objtool.h> 26 #include <linux/sched.h> 27 #include <linux/sched/smt.h> 28 #include <linux/slab.h> 29 #include <linux/tboot.h> 30 #include <linux/trace_events.h> 31 #include <linux/entry-kvm.h> 32 33 #include <asm/apic.h> 34 #include <asm/asm.h> 35 #include <asm/cpu.h> 36 #include <asm/cpu_device_id.h> 37 #include <asm/debugreg.h> 38 #include <asm/desc.h> 39 #include <asm/fpu/api.h> 40 #include <asm/fpu/xstate.h> 41 #include <asm/fred.h> 42 #include <asm/idtentry.h> 43 #include <asm/io.h> 44 #include <asm/irq_remapping.h> 45 #include <asm/reboot.h> 46 #include <asm/perf_event.h> 47 #include <asm/mmu_context.h> 48 #include <asm/mshyperv.h> 49 #include <asm/mwait.h> 50 #include <asm/spec-ctrl.h> 51 #include <asm/vmx.h> 52 53 #include <trace/events/ipi.h> 54 55 #include "capabilities.h" 56 #include "cpuid.h" 57 #include "hyperv.h" 58 #include "kvm_onhyperv.h" 59 #include "irq.h" 60 #include "kvm_cache_regs.h" 61 #include "lapic.h" 62 #include "mmu.h" 63 #include "nested.h" 64 #include "pmu.h" 65 #include "sgx.h" 66 #include "trace.h" 67 #include "vmcs.h" 68 #include "vmcs12.h" 69 #include "vmx.h" 70 #include "x86.h" 71 #include "x86_ops.h" 72 #include "smm.h" 73 #include "vmx_onhyperv.h" 74 #include "posted_intr.h" 75 76 MODULE_AUTHOR("Qumranet"); 77 MODULE_LICENSE("GPL"); 78 79 #ifdef MODULE 80 static const struct x86_cpu_id vmx_cpu_id[] = { 81 X86_MATCH_FEATURE(X86_FEATURE_VMX, NULL), 82 {} 83 }; 84 MODULE_DEVICE_TABLE(x86cpu, vmx_cpu_id); 85 #endif 86 87 bool __read_mostly enable_vpid = 1; 88 module_param_named(vpid, enable_vpid, bool, 0444); 89 90 static bool __read_mostly enable_vnmi = 1; 91 module_param_named(vnmi, enable_vnmi, bool, 0444); 92 93 bool __read_mostly flexpriority_enabled = 1; 94 module_param_named(flexpriority, flexpriority_enabled, bool, 0444); 95 96 bool __read_mostly enable_ept = 1; 97 module_param_named(ept, enable_ept, bool, 0444); 98 99 bool __read_mostly enable_unrestricted_guest = 1; 100 module_param_named(unrestricted_guest, 101 enable_unrestricted_guest, bool, 0444); 102 103 bool __read_mostly enable_ept_ad_bits = 1; 104 module_param_named(eptad, enable_ept_ad_bits, bool, 0444); 105 106 static bool __read_mostly emulate_invalid_guest_state = true; 107 module_param(emulate_invalid_guest_state, bool, 0444); 108 109 static bool __read_mostly fasteoi = 1; 110 module_param(fasteoi, bool, 0444); 111 112 module_param(enable_apicv, bool, 0444); 113 114 bool __read_mostly enable_ipiv = true; 115 module_param(enable_ipiv, bool, 0444); 116 117 /* 118 * If nested=1, nested virtualization is supported, i.e., guests may use 119 * VMX and be a hypervisor for its own guests. If nested=0, guests may not 120 * use VMX instructions. 121 */ 122 static bool __read_mostly nested = 1; 123 module_param(nested, bool, 0444); 124 125 bool __read_mostly enable_pml = 1; 126 module_param_named(pml, enable_pml, bool, 0444); 127 128 static bool __read_mostly error_on_inconsistent_vmcs_config = true; 129 module_param(error_on_inconsistent_vmcs_config, bool, 0444); 130 131 static bool __read_mostly dump_invalid_vmcs = 0; 132 module_param(dump_invalid_vmcs, bool, 0644); 133 134 #define MSR_BITMAP_MODE_X2APIC 1 135 #define MSR_BITMAP_MODE_X2APIC_APICV 2 136 137 #define KVM_VMX_TSC_MULTIPLIER_MAX 0xffffffffffffffffULL 138 139 /* Guest_tsc -> host_tsc conversion requires 64-bit division. */ 140 static int __read_mostly cpu_preemption_timer_multi; 141 static bool __read_mostly enable_preemption_timer = 1; 142 #ifdef CONFIG_X86_64 143 module_param_named(preemption_timer, enable_preemption_timer, bool, S_IRUGO); 144 #endif 145 146 extern bool __read_mostly allow_smaller_maxphyaddr; 147 module_param(allow_smaller_maxphyaddr, bool, S_IRUGO); 148 149 #define KVM_VM_CR0_ALWAYS_OFF (X86_CR0_NW | X86_CR0_CD) 150 #define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR0_NE 151 #define KVM_VM_CR0_ALWAYS_ON \ 152 (KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE) 153 154 #define KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR4_VMXE 155 #define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE) 156 #define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE) 157 158 #define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM)) 159 160 #define MSR_IA32_RTIT_STATUS_MASK (~(RTIT_STATUS_FILTEREN | \ 161 RTIT_STATUS_CONTEXTEN | RTIT_STATUS_TRIGGEREN | \ 162 RTIT_STATUS_ERROR | RTIT_STATUS_STOPPED | \ 163 RTIT_STATUS_BYTECNT)) 164 165 /* 166 * List of MSRs that can be directly passed to the guest. 167 * In addition to these x2apic, PT and LBR MSRs are handled specially. 168 */ 169 static u32 vmx_possible_passthrough_msrs[MAX_POSSIBLE_PASSTHROUGH_MSRS] = { 170 MSR_IA32_SPEC_CTRL, 171 MSR_IA32_PRED_CMD, 172 MSR_IA32_FLUSH_CMD, 173 MSR_IA32_TSC, 174 #ifdef CONFIG_X86_64 175 MSR_FS_BASE, 176 MSR_GS_BASE, 177 MSR_KERNEL_GS_BASE, 178 MSR_IA32_XFD, 179 MSR_IA32_XFD_ERR, 180 #endif 181 MSR_IA32_SYSENTER_CS, 182 MSR_IA32_SYSENTER_ESP, 183 MSR_IA32_SYSENTER_EIP, 184 MSR_CORE_C1_RES, 185 MSR_CORE_C3_RESIDENCY, 186 MSR_CORE_C6_RESIDENCY, 187 MSR_CORE_C7_RESIDENCY, 188 }; 189 190 /* 191 * These 2 parameters are used to config the controls for Pause-Loop Exiting: 192 * ple_gap: upper bound on the amount of time between two successive 193 * executions of PAUSE in a loop. Also indicate if ple enabled. 194 * According to test, this time is usually smaller than 128 cycles. 195 * ple_window: upper bound on the amount of time a guest is allowed to execute 196 * in a PAUSE loop. Tests indicate that most spinlocks are held for 197 * less than 2^12 cycles 198 * Time is measured based on a counter that runs at the same rate as the TSC, 199 * refer SDM volume 3b section 21.6.13 & 22.1.3. 200 */ 201 static unsigned int ple_gap = KVM_DEFAULT_PLE_GAP; 202 module_param(ple_gap, uint, 0444); 203 204 static unsigned int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW; 205 module_param(ple_window, uint, 0444); 206 207 /* Default doubles per-vcpu window every exit. */ 208 static unsigned int ple_window_grow = KVM_DEFAULT_PLE_WINDOW_GROW; 209 module_param(ple_window_grow, uint, 0444); 210 211 /* Default resets per-vcpu window every exit to ple_window. */ 212 static unsigned int ple_window_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK; 213 module_param(ple_window_shrink, uint, 0444); 214 215 /* Default is to compute the maximum so we can never overflow. */ 216 static unsigned int ple_window_max = KVM_VMX_DEFAULT_PLE_WINDOW_MAX; 217 module_param(ple_window_max, uint, 0444); 218 219 /* Default is SYSTEM mode, 1 for host-guest mode */ 220 int __read_mostly pt_mode = PT_MODE_SYSTEM; 221 module_param(pt_mode, int, S_IRUGO); 222 223 struct x86_pmu_lbr __ro_after_init vmx_lbr_caps; 224 225 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_should_flush); 226 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_flush_cond); 227 static DEFINE_MUTEX(vmx_l1d_flush_mutex); 228 229 /* Storage for pre module init parameter parsing */ 230 static enum vmx_l1d_flush_state __read_mostly vmentry_l1d_flush_param = VMENTER_L1D_FLUSH_AUTO; 231 232 static const struct { 233 const char *option; 234 bool for_parse; 235 } vmentry_l1d_param[] = { 236 [VMENTER_L1D_FLUSH_AUTO] = {"auto", true}, 237 [VMENTER_L1D_FLUSH_NEVER] = {"never", true}, 238 [VMENTER_L1D_FLUSH_COND] = {"cond", true}, 239 [VMENTER_L1D_FLUSH_ALWAYS] = {"always", true}, 240 [VMENTER_L1D_FLUSH_EPT_DISABLED] = {"EPT disabled", false}, 241 [VMENTER_L1D_FLUSH_NOT_REQUIRED] = {"not required", false}, 242 }; 243 244 #define L1D_CACHE_ORDER 4 245 static void *vmx_l1d_flush_pages; 246 247 static int vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf) 248 { 249 struct page *page; 250 unsigned int i; 251 252 if (!boot_cpu_has_bug(X86_BUG_L1TF)) { 253 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED; 254 return 0; 255 } 256 257 if (!enable_ept) { 258 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_EPT_DISABLED; 259 return 0; 260 } 261 262 if (host_arch_capabilities & ARCH_CAP_SKIP_VMENTRY_L1DFLUSH) { 263 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED; 264 return 0; 265 } 266 267 /* If set to auto use the default l1tf mitigation method */ 268 if (l1tf == VMENTER_L1D_FLUSH_AUTO) { 269 switch (l1tf_mitigation) { 270 case L1TF_MITIGATION_OFF: 271 l1tf = VMENTER_L1D_FLUSH_NEVER; 272 break; 273 case L1TF_MITIGATION_FLUSH_NOWARN: 274 case L1TF_MITIGATION_FLUSH: 275 case L1TF_MITIGATION_FLUSH_NOSMT: 276 l1tf = VMENTER_L1D_FLUSH_COND; 277 break; 278 case L1TF_MITIGATION_FULL: 279 case L1TF_MITIGATION_FULL_FORCE: 280 l1tf = VMENTER_L1D_FLUSH_ALWAYS; 281 break; 282 } 283 } else if (l1tf_mitigation == L1TF_MITIGATION_FULL_FORCE) { 284 l1tf = VMENTER_L1D_FLUSH_ALWAYS; 285 } 286 287 if (l1tf != VMENTER_L1D_FLUSH_NEVER && !vmx_l1d_flush_pages && 288 !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) { 289 /* 290 * This allocation for vmx_l1d_flush_pages is not tied to a VM 291 * lifetime and so should not be charged to a memcg. 292 */ 293 page = alloc_pages(GFP_KERNEL, L1D_CACHE_ORDER); 294 if (!page) 295 return -ENOMEM; 296 vmx_l1d_flush_pages = page_address(page); 297 298 /* 299 * Initialize each page with a different pattern in 300 * order to protect against KSM in the nested 301 * virtualization case. 302 */ 303 for (i = 0; i < 1u << L1D_CACHE_ORDER; ++i) { 304 memset(vmx_l1d_flush_pages + i * PAGE_SIZE, i + 1, 305 PAGE_SIZE); 306 } 307 } 308 309 l1tf_vmx_mitigation = l1tf; 310 311 if (l1tf != VMENTER_L1D_FLUSH_NEVER) 312 static_branch_enable(&vmx_l1d_should_flush); 313 else 314 static_branch_disable(&vmx_l1d_should_flush); 315 316 if (l1tf == VMENTER_L1D_FLUSH_COND) 317 static_branch_enable(&vmx_l1d_flush_cond); 318 else 319 static_branch_disable(&vmx_l1d_flush_cond); 320 return 0; 321 } 322 323 static int vmentry_l1d_flush_parse(const char *s) 324 { 325 unsigned int i; 326 327 if (s) { 328 for (i = 0; i < ARRAY_SIZE(vmentry_l1d_param); i++) { 329 if (vmentry_l1d_param[i].for_parse && 330 sysfs_streq(s, vmentry_l1d_param[i].option)) 331 return i; 332 } 333 } 334 return -EINVAL; 335 } 336 337 static int vmentry_l1d_flush_set(const char *s, const struct kernel_param *kp) 338 { 339 int l1tf, ret; 340 341 l1tf = vmentry_l1d_flush_parse(s); 342 if (l1tf < 0) 343 return l1tf; 344 345 if (!boot_cpu_has(X86_BUG_L1TF)) 346 return 0; 347 348 /* 349 * Has vmx_init() run already? If not then this is the pre init 350 * parameter parsing. In that case just store the value and let 351 * vmx_init() do the proper setup after enable_ept has been 352 * established. 353 */ 354 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) { 355 vmentry_l1d_flush_param = l1tf; 356 return 0; 357 } 358 359 mutex_lock(&vmx_l1d_flush_mutex); 360 ret = vmx_setup_l1d_flush(l1tf); 361 mutex_unlock(&vmx_l1d_flush_mutex); 362 return ret; 363 } 364 365 static int vmentry_l1d_flush_get(char *s, const struct kernel_param *kp) 366 { 367 if (WARN_ON_ONCE(l1tf_vmx_mitigation >= ARRAY_SIZE(vmentry_l1d_param))) 368 return sysfs_emit(s, "???\n"); 369 370 return sysfs_emit(s, "%s\n", vmentry_l1d_param[l1tf_vmx_mitigation].option); 371 } 372 373 static __always_inline void vmx_disable_fb_clear(struct vcpu_vmx *vmx) 374 { 375 u64 msr; 376 377 if (!vmx->disable_fb_clear) 378 return; 379 380 msr = __rdmsr(MSR_IA32_MCU_OPT_CTRL); 381 msr |= FB_CLEAR_DIS; 382 native_wrmsrl(MSR_IA32_MCU_OPT_CTRL, msr); 383 /* Cache the MSR value to avoid reading it later */ 384 vmx->msr_ia32_mcu_opt_ctrl = msr; 385 } 386 387 static __always_inline void vmx_enable_fb_clear(struct vcpu_vmx *vmx) 388 { 389 if (!vmx->disable_fb_clear) 390 return; 391 392 vmx->msr_ia32_mcu_opt_ctrl &= ~FB_CLEAR_DIS; 393 native_wrmsrl(MSR_IA32_MCU_OPT_CTRL, vmx->msr_ia32_mcu_opt_ctrl); 394 } 395 396 static void vmx_update_fb_clear_dis(struct kvm_vcpu *vcpu, struct vcpu_vmx *vmx) 397 { 398 /* 399 * Disable VERW's behavior of clearing CPU buffers for the guest if the 400 * CPU isn't affected by MDS/TAA, and the host hasn't forcefully enabled 401 * the mitigation. Disabling the clearing behavior provides a 402 * performance boost for guests that aren't aware that manually clearing 403 * CPU buffers is unnecessary, at the cost of MSR accesses on VM-Entry 404 * and VM-Exit. 405 */ 406 vmx->disable_fb_clear = !cpu_feature_enabled(X86_FEATURE_CLEAR_CPU_BUF) && 407 (host_arch_capabilities & ARCH_CAP_FB_CLEAR_CTRL) && 408 !boot_cpu_has_bug(X86_BUG_MDS) && 409 !boot_cpu_has_bug(X86_BUG_TAA); 410 411 /* 412 * If guest will not execute VERW, there is no need to set FB_CLEAR_DIS 413 * at VMEntry. Skip the MSR read/write when a guest has no use case to 414 * execute VERW. 415 */ 416 if ((vcpu->arch.arch_capabilities & ARCH_CAP_FB_CLEAR) || 417 ((vcpu->arch.arch_capabilities & ARCH_CAP_MDS_NO) && 418 (vcpu->arch.arch_capabilities & ARCH_CAP_TAA_NO) && 419 (vcpu->arch.arch_capabilities & ARCH_CAP_PSDP_NO) && 420 (vcpu->arch.arch_capabilities & ARCH_CAP_FBSDP_NO) && 421 (vcpu->arch.arch_capabilities & ARCH_CAP_SBDR_SSDP_NO))) 422 vmx->disable_fb_clear = false; 423 } 424 425 static const struct kernel_param_ops vmentry_l1d_flush_ops = { 426 .set = vmentry_l1d_flush_set, 427 .get = vmentry_l1d_flush_get, 428 }; 429 module_param_cb(vmentry_l1d_flush, &vmentry_l1d_flush_ops, NULL, 0644); 430 431 static u32 vmx_segment_access_rights(struct kvm_segment *var); 432 433 void vmx_vmexit(void); 434 435 #define vmx_insn_failed(fmt...) \ 436 do { \ 437 WARN_ONCE(1, fmt); \ 438 pr_warn_ratelimited(fmt); \ 439 } while (0) 440 441 noinline void vmread_error(unsigned long field) 442 { 443 vmx_insn_failed("vmread failed: field=%lx\n", field); 444 } 445 446 #ifndef CONFIG_CC_HAS_ASM_GOTO_OUTPUT 447 noinstr void vmread_error_trampoline2(unsigned long field, bool fault) 448 { 449 if (fault) { 450 kvm_spurious_fault(); 451 } else { 452 instrumentation_begin(); 453 vmread_error(field); 454 instrumentation_end(); 455 } 456 } 457 #endif 458 459 noinline void vmwrite_error(unsigned long field, unsigned long value) 460 { 461 vmx_insn_failed("vmwrite failed: field=%lx val=%lx err=%u\n", 462 field, value, vmcs_read32(VM_INSTRUCTION_ERROR)); 463 } 464 465 noinline void vmclear_error(struct vmcs *vmcs, u64 phys_addr) 466 { 467 vmx_insn_failed("vmclear failed: %p/%llx err=%u\n", 468 vmcs, phys_addr, vmcs_read32(VM_INSTRUCTION_ERROR)); 469 } 470 471 noinline void vmptrld_error(struct vmcs *vmcs, u64 phys_addr) 472 { 473 vmx_insn_failed("vmptrld failed: %p/%llx err=%u\n", 474 vmcs, phys_addr, vmcs_read32(VM_INSTRUCTION_ERROR)); 475 } 476 477 noinline void invvpid_error(unsigned long ext, u16 vpid, gva_t gva) 478 { 479 vmx_insn_failed("invvpid failed: ext=0x%lx vpid=%u gva=0x%lx\n", 480 ext, vpid, gva); 481 } 482 483 noinline void invept_error(unsigned long ext, u64 eptp, gpa_t gpa) 484 { 485 vmx_insn_failed("invept failed: ext=0x%lx eptp=%llx gpa=0x%llx\n", 486 ext, eptp, gpa); 487 } 488 489 static DEFINE_PER_CPU(struct vmcs *, vmxarea); 490 DEFINE_PER_CPU(struct vmcs *, current_vmcs); 491 /* 492 * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed 493 * when a CPU is brought down, and we need to VMCLEAR all VMCSs loaded on it. 494 */ 495 static DEFINE_PER_CPU(struct list_head, loaded_vmcss_on_cpu); 496 497 static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS); 498 static DEFINE_SPINLOCK(vmx_vpid_lock); 499 500 struct vmcs_config vmcs_config __ro_after_init; 501 struct vmx_capability vmx_capability __ro_after_init; 502 503 #define VMX_SEGMENT_FIELD(seg) \ 504 [VCPU_SREG_##seg] = { \ 505 .selector = GUEST_##seg##_SELECTOR, \ 506 .base = GUEST_##seg##_BASE, \ 507 .limit = GUEST_##seg##_LIMIT, \ 508 .ar_bytes = GUEST_##seg##_AR_BYTES, \ 509 } 510 511 static const struct kvm_vmx_segment_field { 512 unsigned selector; 513 unsigned base; 514 unsigned limit; 515 unsigned ar_bytes; 516 } kvm_vmx_segment_fields[] = { 517 VMX_SEGMENT_FIELD(CS), 518 VMX_SEGMENT_FIELD(DS), 519 VMX_SEGMENT_FIELD(ES), 520 VMX_SEGMENT_FIELD(FS), 521 VMX_SEGMENT_FIELD(GS), 522 VMX_SEGMENT_FIELD(SS), 523 VMX_SEGMENT_FIELD(TR), 524 VMX_SEGMENT_FIELD(LDTR), 525 }; 526 527 static inline void vmx_segment_cache_clear(struct vcpu_vmx *vmx) 528 { 529 vmx->segment_cache.bitmask = 0; 530 } 531 532 static unsigned long host_idt_base; 533 534 #if IS_ENABLED(CONFIG_HYPERV) 535 static bool __read_mostly enlightened_vmcs = true; 536 module_param(enlightened_vmcs, bool, 0444); 537 538 static int hv_enable_l2_tlb_flush(struct kvm_vcpu *vcpu) 539 { 540 struct hv_enlightened_vmcs *evmcs; 541 hpa_t partition_assist_page = hv_get_partition_assist_page(vcpu); 542 543 if (partition_assist_page == INVALID_PAGE) 544 return -ENOMEM; 545 546 evmcs = (struct hv_enlightened_vmcs *)to_vmx(vcpu)->loaded_vmcs->vmcs; 547 548 evmcs->partition_assist_page = partition_assist_page; 549 evmcs->hv_vm_id = (unsigned long)vcpu->kvm; 550 evmcs->hv_enlightenments_control.nested_flush_hypercall = 1; 551 552 return 0; 553 } 554 555 static __init void hv_init_evmcs(void) 556 { 557 int cpu; 558 559 if (!enlightened_vmcs) 560 return; 561 562 /* 563 * Enlightened VMCS usage should be recommended and the host needs 564 * to support eVMCS v1 or above. 565 */ 566 if (ms_hyperv.hints & HV_X64_ENLIGHTENED_VMCS_RECOMMENDED && 567 (ms_hyperv.nested_features & HV_X64_ENLIGHTENED_VMCS_VERSION) >= 568 KVM_EVMCS_VERSION) { 569 570 /* Check that we have assist pages on all online CPUs */ 571 for_each_online_cpu(cpu) { 572 if (!hv_get_vp_assist_page(cpu)) { 573 enlightened_vmcs = false; 574 break; 575 } 576 } 577 578 if (enlightened_vmcs) { 579 pr_info("Using Hyper-V Enlightened VMCS\n"); 580 static_branch_enable(&__kvm_is_using_evmcs); 581 } 582 583 if (ms_hyperv.nested_features & HV_X64_NESTED_DIRECT_FLUSH) 584 vt_x86_ops.enable_l2_tlb_flush 585 = hv_enable_l2_tlb_flush; 586 } else { 587 enlightened_vmcs = false; 588 } 589 } 590 591 static void hv_reset_evmcs(void) 592 { 593 struct hv_vp_assist_page *vp_ap; 594 595 if (!kvm_is_using_evmcs()) 596 return; 597 598 /* 599 * KVM should enable eVMCS if and only if all CPUs have a VP assist 600 * page, and should reject CPU onlining if eVMCS is enabled the CPU 601 * doesn't have a VP assist page allocated. 602 */ 603 vp_ap = hv_get_vp_assist_page(smp_processor_id()); 604 if (WARN_ON_ONCE(!vp_ap)) 605 return; 606 607 /* 608 * Reset everything to support using non-enlightened VMCS access later 609 * (e.g. when we reload the module with enlightened_vmcs=0) 610 */ 611 vp_ap->nested_control.features.directhypercall = 0; 612 vp_ap->current_nested_vmcs = 0; 613 vp_ap->enlighten_vmentry = 0; 614 } 615 616 #else /* IS_ENABLED(CONFIG_HYPERV) */ 617 static void hv_init_evmcs(void) {} 618 static void hv_reset_evmcs(void) {} 619 #endif /* IS_ENABLED(CONFIG_HYPERV) */ 620 621 /* 622 * Comment's format: document - errata name - stepping - processor name. 623 * Refer from 624 * https://www.virtualbox.org/svn/vbox/trunk/src/VBox/VMM/VMMR0/HMR0.cpp 625 */ 626 static u32 vmx_preemption_cpu_tfms[] = { 627 /* 323344.pdf - BA86 - D0 - Xeon 7500 Series */ 628 0x000206E6, 629 /* 323056.pdf - AAX65 - C2 - Xeon L3406 */ 630 /* 322814.pdf - AAT59 - C2 - i7-600, i5-500, i5-400 and i3-300 Mobile */ 631 /* 322911.pdf - AAU65 - C2 - i5-600, i3-500 Desktop and Pentium G6950 */ 632 0x00020652, 633 /* 322911.pdf - AAU65 - K0 - i5-600, i3-500 Desktop and Pentium G6950 */ 634 0x00020655, 635 /* 322373.pdf - AAO95 - B1 - Xeon 3400 Series */ 636 /* 322166.pdf - AAN92 - B1 - i7-800 and i5-700 Desktop */ 637 /* 638 * 320767.pdf - AAP86 - B1 - 639 * i7-900 Mobile Extreme, i7-800 and i7-700 Mobile 640 */ 641 0x000106E5, 642 /* 321333.pdf - AAM126 - C0 - Xeon 3500 */ 643 0x000106A0, 644 /* 321333.pdf - AAM126 - C1 - Xeon 3500 */ 645 0x000106A1, 646 /* 320836.pdf - AAJ124 - C0 - i7-900 Desktop Extreme and i7-900 Desktop */ 647 0x000106A4, 648 /* 321333.pdf - AAM126 - D0 - Xeon 3500 */ 649 /* 321324.pdf - AAK139 - D0 - Xeon 5500 */ 650 /* 320836.pdf - AAJ124 - D0 - i7-900 Extreme and i7-900 Desktop */ 651 0x000106A5, 652 /* Xeon E3-1220 V2 */ 653 0x000306A8, 654 }; 655 656 static inline bool cpu_has_broken_vmx_preemption_timer(void) 657 { 658 u32 eax = cpuid_eax(0x00000001), i; 659 660 /* Clear the reserved bits */ 661 eax &= ~(0x3U << 14 | 0xfU << 28); 662 for (i = 0; i < ARRAY_SIZE(vmx_preemption_cpu_tfms); i++) 663 if (eax == vmx_preemption_cpu_tfms[i]) 664 return true; 665 666 return false; 667 } 668 669 static inline bool cpu_need_virtualize_apic_accesses(struct kvm_vcpu *vcpu) 670 { 671 return flexpriority_enabled && lapic_in_kernel(vcpu); 672 } 673 674 static int vmx_get_passthrough_msr_slot(u32 msr) 675 { 676 int i; 677 678 switch (msr) { 679 case 0x800 ... 0x8ff: 680 /* x2APIC MSRs. These are handled in vmx_update_msr_bitmap_x2apic() */ 681 return -ENOENT; 682 case MSR_IA32_RTIT_STATUS: 683 case MSR_IA32_RTIT_OUTPUT_BASE: 684 case MSR_IA32_RTIT_OUTPUT_MASK: 685 case MSR_IA32_RTIT_CR3_MATCH: 686 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: 687 /* PT MSRs. These are handled in pt_update_intercept_for_msr() */ 688 case MSR_LBR_SELECT: 689 case MSR_LBR_TOS: 690 case MSR_LBR_INFO_0 ... MSR_LBR_INFO_0 + 31: 691 case MSR_LBR_NHM_FROM ... MSR_LBR_NHM_FROM + 31: 692 case MSR_LBR_NHM_TO ... MSR_LBR_NHM_TO + 31: 693 case MSR_LBR_CORE_FROM ... MSR_LBR_CORE_FROM + 8: 694 case MSR_LBR_CORE_TO ... MSR_LBR_CORE_TO + 8: 695 /* LBR MSRs. These are handled in vmx_update_intercept_for_lbr_msrs() */ 696 return -ENOENT; 697 } 698 699 for (i = 0; i < ARRAY_SIZE(vmx_possible_passthrough_msrs); i++) { 700 if (vmx_possible_passthrough_msrs[i] == msr) 701 return i; 702 } 703 704 WARN(1, "Invalid MSR %x, please adapt vmx_possible_passthrough_msrs[]", msr); 705 return -ENOENT; 706 } 707 708 struct vmx_uret_msr *vmx_find_uret_msr(struct vcpu_vmx *vmx, u32 msr) 709 { 710 int i; 711 712 i = kvm_find_user_return_msr(msr); 713 if (i >= 0) 714 return &vmx->guest_uret_msrs[i]; 715 return NULL; 716 } 717 718 static int vmx_set_guest_uret_msr(struct vcpu_vmx *vmx, 719 struct vmx_uret_msr *msr, u64 data) 720 { 721 unsigned int slot = msr - vmx->guest_uret_msrs; 722 int ret = 0; 723 724 if (msr->load_into_hardware) { 725 preempt_disable(); 726 ret = kvm_set_user_return_msr(slot, data, msr->mask); 727 preempt_enable(); 728 } 729 if (!ret) 730 msr->data = data; 731 return ret; 732 } 733 734 /* 735 * Disable VMX and clear CR4.VMXE (even if VMXOFF faults) 736 * 737 * Note, VMXOFF causes a #UD if the CPU is !post-VMXON, but it's impossible to 738 * atomically track post-VMXON state, e.g. this may be called in NMI context. 739 * Eat all faults as all other faults on VMXOFF faults are mode related, i.e. 740 * faults are guaranteed to be due to the !post-VMXON check unless the CPU is 741 * magically in RM, VM86, compat mode, or at CPL>0. 742 */ 743 static int kvm_cpu_vmxoff(void) 744 { 745 asm goto("1: vmxoff\n\t" 746 _ASM_EXTABLE(1b, %l[fault]) 747 ::: "cc", "memory" : fault); 748 749 cr4_clear_bits(X86_CR4_VMXE); 750 return 0; 751 752 fault: 753 cr4_clear_bits(X86_CR4_VMXE); 754 return -EIO; 755 } 756 757 static void vmx_emergency_disable(void) 758 { 759 int cpu = raw_smp_processor_id(); 760 struct loaded_vmcs *v; 761 762 kvm_rebooting = true; 763 764 /* 765 * Note, CR4.VMXE can be _cleared_ in NMI context, but it can only be 766 * set in task context. If this races with VMX is disabled by an NMI, 767 * VMCLEAR and VMXOFF may #UD, but KVM will eat those faults due to 768 * kvm_rebooting set. 769 */ 770 if (!(__read_cr4() & X86_CR4_VMXE)) 771 return; 772 773 list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu), 774 loaded_vmcss_on_cpu_link) 775 vmcs_clear(v->vmcs); 776 777 kvm_cpu_vmxoff(); 778 } 779 780 static void __loaded_vmcs_clear(void *arg) 781 { 782 struct loaded_vmcs *loaded_vmcs = arg; 783 int cpu = raw_smp_processor_id(); 784 785 if (loaded_vmcs->cpu != cpu) 786 return; /* vcpu migration can race with cpu offline */ 787 if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs) 788 per_cpu(current_vmcs, cpu) = NULL; 789 790 vmcs_clear(loaded_vmcs->vmcs); 791 if (loaded_vmcs->shadow_vmcs && loaded_vmcs->launched) 792 vmcs_clear(loaded_vmcs->shadow_vmcs); 793 794 list_del(&loaded_vmcs->loaded_vmcss_on_cpu_link); 795 796 /* 797 * Ensure all writes to loaded_vmcs, including deleting it from its 798 * current percpu list, complete before setting loaded_vmcs->cpu to 799 * -1, otherwise a different cpu can see loaded_vmcs->cpu == -1 first 800 * and add loaded_vmcs to its percpu list before it's deleted from this 801 * cpu's list. Pairs with the smp_rmb() in vmx_vcpu_load_vmcs(). 802 */ 803 smp_wmb(); 804 805 loaded_vmcs->cpu = -1; 806 loaded_vmcs->launched = 0; 807 } 808 809 void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs) 810 { 811 int cpu = loaded_vmcs->cpu; 812 813 if (cpu != -1) 814 smp_call_function_single(cpu, 815 __loaded_vmcs_clear, loaded_vmcs, 1); 816 } 817 818 static bool vmx_segment_cache_test_set(struct vcpu_vmx *vmx, unsigned seg, 819 unsigned field) 820 { 821 bool ret; 822 u32 mask = 1 << (seg * SEG_FIELD_NR + field); 823 824 if (!kvm_register_is_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS)) { 825 kvm_register_mark_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS); 826 vmx->segment_cache.bitmask = 0; 827 } 828 ret = vmx->segment_cache.bitmask & mask; 829 vmx->segment_cache.bitmask |= mask; 830 return ret; 831 } 832 833 static u16 vmx_read_guest_seg_selector(struct vcpu_vmx *vmx, unsigned seg) 834 { 835 u16 *p = &vmx->segment_cache.seg[seg].selector; 836 837 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_SEL)) 838 *p = vmcs_read16(kvm_vmx_segment_fields[seg].selector); 839 return *p; 840 } 841 842 static ulong vmx_read_guest_seg_base(struct vcpu_vmx *vmx, unsigned seg) 843 { 844 ulong *p = &vmx->segment_cache.seg[seg].base; 845 846 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_BASE)) 847 *p = vmcs_readl(kvm_vmx_segment_fields[seg].base); 848 return *p; 849 } 850 851 static u32 vmx_read_guest_seg_limit(struct vcpu_vmx *vmx, unsigned seg) 852 { 853 u32 *p = &vmx->segment_cache.seg[seg].limit; 854 855 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_LIMIT)) 856 *p = vmcs_read32(kvm_vmx_segment_fields[seg].limit); 857 return *p; 858 } 859 860 static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg) 861 { 862 u32 *p = &vmx->segment_cache.seg[seg].ar; 863 864 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_AR)) 865 *p = vmcs_read32(kvm_vmx_segment_fields[seg].ar_bytes); 866 return *p; 867 } 868 869 void vmx_update_exception_bitmap(struct kvm_vcpu *vcpu) 870 { 871 u32 eb; 872 873 eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) | 874 (1u << DB_VECTOR) | (1u << AC_VECTOR); 875 /* 876 * #VE isn't used for VMX. To test against unexpected changes 877 * related to #VE for VMX, intercept unexpected #VE and warn on it. 878 */ 879 if (IS_ENABLED(CONFIG_KVM_INTEL_PROVE_VE)) 880 eb |= 1u << VE_VECTOR; 881 /* 882 * Guest access to VMware backdoor ports could legitimately 883 * trigger #GP because of TSS I/O permission bitmap. 884 * We intercept those #GP and allow access to them anyway 885 * as VMware does. 886 */ 887 if (enable_vmware_backdoor) 888 eb |= (1u << GP_VECTOR); 889 if ((vcpu->guest_debug & 890 (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) == 891 (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) 892 eb |= 1u << BP_VECTOR; 893 if (to_vmx(vcpu)->rmode.vm86_active) 894 eb = ~0; 895 if (!vmx_need_pf_intercept(vcpu)) 896 eb &= ~(1u << PF_VECTOR); 897 898 /* When we are running a nested L2 guest and L1 specified for it a 899 * certain exception bitmap, we must trap the same exceptions and pass 900 * them to L1. When running L2, we will only handle the exceptions 901 * specified above if L1 did not want them. 902 */ 903 if (is_guest_mode(vcpu)) 904 eb |= get_vmcs12(vcpu)->exception_bitmap; 905 else { 906 int mask = 0, match = 0; 907 908 if (enable_ept && (eb & (1u << PF_VECTOR))) { 909 /* 910 * If EPT is enabled, #PF is currently only intercepted 911 * if MAXPHYADDR is smaller on the guest than on the 912 * host. In that case we only care about present, 913 * non-reserved faults. For vmcs02, however, PFEC_MASK 914 * and PFEC_MATCH are set in prepare_vmcs02_rare. 915 */ 916 mask = PFERR_PRESENT_MASK | PFERR_RSVD_MASK; 917 match = PFERR_PRESENT_MASK; 918 } 919 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, mask); 920 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, match); 921 } 922 923 /* 924 * Disabling xfd interception indicates that dynamic xfeatures 925 * might be used in the guest. Always trap #NM in this case 926 * to save guest xfd_err timely. 927 */ 928 if (vcpu->arch.xfd_no_write_intercept) 929 eb |= (1u << NM_VECTOR); 930 931 vmcs_write32(EXCEPTION_BITMAP, eb); 932 } 933 934 /* 935 * Check if MSR is intercepted for currently loaded MSR bitmap. 936 */ 937 static bool msr_write_intercepted(struct vcpu_vmx *vmx, u32 msr) 938 { 939 if (!(exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS)) 940 return true; 941 942 return vmx_test_msr_bitmap_write(vmx->loaded_vmcs->msr_bitmap, msr); 943 } 944 945 unsigned int __vmx_vcpu_run_flags(struct vcpu_vmx *vmx) 946 { 947 unsigned int flags = 0; 948 949 if (vmx->loaded_vmcs->launched) 950 flags |= VMX_RUN_VMRESUME; 951 952 /* 953 * If writes to the SPEC_CTRL MSR aren't intercepted, the guest is free 954 * to change it directly without causing a vmexit. In that case read 955 * it after vmexit and store it in vmx->spec_ctrl. 956 */ 957 if (!msr_write_intercepted(vmx, MSR_IA32_SPEC_CTRL)) 958 flags |= VMX_RUN_SAVE_SPEC_CTRL; 959 960 return flags; 961 } 962 963 static __always_inline void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx, 964 unsigned long entry, unsigned long exit) 965 { 966 vm_entry_controls_clearbit(vmx, entry); 967 vm_exit_controls_clearbit(vmx, exit); 968 } 969 970 int vmx_find_loadstore_msr_slot(struct vmx_msrs *m, u32 msr) 971 { 972 unsigned int i; 973 974 for (i = 0; i < m->nr; ++i) { 975 if (m->val[i].index == msr) 976 return i; 977 } 978 return -ENOENT; 979 } 980 981 static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr) 982 { 983 int i; 984 struct msr_autoload *m = &vmx->msr_autoload; 985 986 switch (msr) { 987 case MSR_EFER: 988 if (cpu_has_load_ia32_efer()) { 989 clear_atomic_switch_msr_special(vmx, 990 VM_ENTRY_LOAD_IA32_EFER, 991 VM_EXIT_LOAD_IA32_EFER); 992 return; 993 } 994 break; 995 case MSR_CORE_PERF_GLOBAL_CTRL: 996 if (cpu_has_load_perf_global_ctrl()) { 997 clear_atomic_switch_msr_special(vmx, 998 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL, 999 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL); 1000 return; 1001 } 1002 break; 1003 } 1004 i = vmx_find_loadstore_msr_slot(&m->guest, msr); 1005 if (i < 0) 1006 goto skip_guest; 1007 --m->guest.nr; 1008 m->guest.val[i] = m->guest.val[m->guest.nr]; 1009 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr); 1010 1011 skip_guest: 1012 i = vmx_find_loadstore_msr_slot(&m->host, msr); 1013 if (i < 0) 1014 return; 1015 1016 --m->host.nr; 1017 m->host.val[i] = m->host.val[m->host.nr]; 1018 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr); 1019 } 1020 1021 static __always_inline void add_atomic_switch_msr_special(struct vcpu_vmx *vmx, 1022 unsigned long entry, unsigned long exit, 1023 unsigned long guest_val_vmcs, unsigned long host_val_vmcs, 1024 u64 guest_val, u64 host_val) 1025 { 1026 vmcs_write64(guest_val_vmcs, guest_val); 1027 if (host_val_vmcs != HOST_IA32_EFER) 1028 vmcs_write64(host_val_vmcs, host_val); 1029 vm_entry_controls_setbit(vmx, entry); 1030 vm_exit_controls_setbit(vmx, exit); 1031 } 1032 1033 static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr, 1034 u64 guest_val, u64 host_val, bool entry_only) 1035 { 1036 int i, j = 0; 1037 struct msr_autoload *m = &vmx->msr_autoload; 1038 1039 switch (msr) { 1040 case MSR_EFER: 1041 if (cpu_has_load_ia32_efer()) { 1042 add_atomic_switch_msr_special(vmx, 1043 VM_ENTRY_LOAD_IA32_EFER, 1044 VM_EXIT_LOAD_IA32_EFER, 1045 GUEST_IA32_EFER, 1046 HOST_IA32_EFER, 1047 guest_val, host_val); 1048 return; 1049 } 1050 break; 1051 case MSR_CORE_PERF_GLOBAL_CTRL: 1052 if (cpu_has_load_perf_global_ctrl()) { 1053 add_atomic_switch_msr_special(vmx, 1054 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL, 1055 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL, 1056 GUEST_IA32_PERF_GLOBAL_CTRL, 1057 HOST_IA32_PERF_GLOBAL_CTRL, 1058 guest_val, host_val); 1059 return; 1060 } 1061 break; 1062 case MSR_IA32_PEBS_ENABLE: 1063 /* PEBS needs a quiescent period after being disabled (to write 1064 * a record). Disabling PEBS through VMX MSR swapping doesn't 1065 * provide that period, so a CPU could write host's record into 1066 * guest's memory. 1067 */ 1068 wrmsrl(MSR_IA32_PEBS_ENABLE, 0); 1069 } 1070 1071 i = vmx_find_loadstore_msr_slot(&m->guest, msr); 1072 if (!entry_only) 1073 j = vmx_find_loadstore_msr_slot(&m->host, msr); 1074 1075 if ((i < 0 && m->guest.nr == MAX_NR_LOADSTORE_MSRS) || 1076 (j < 0 && m->host.nr == MAX_NR_LOADSTORE_MSRS)) { 1077 printk_once(KERN_WARNING "Not enough msr switch entries. " 1078 "Can't add msr %x\n", msr); 1079 return; 1080 } 1081 if (i < 0) { 1082 i = m->guest.nr++; 1083 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr); 1084 } 1085 m->guest.val[i].index = msr; 1086 m->guest.val[i].value = guest_val; 1087 1088 if (entry_only) 1089 return; 1090 1091 if (j < 0) { 1092 j = m->host.nr++; 1093 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr); 1094 } 1095 m->host.val[j].index = msr; 1096 m->host.val[j].value = host_val; 1097 } 1098 1099 static bool update_transition_efer(struct vcpu_vmx *vmx) 1100 { 1101 u64 guest_efer = vmx->vcpu.arch.efer; 1102 u64 ignore_bits = 0; 1103 int i; 1104 1105 /* Shadow paging assumes NX to be available. */ 1106 if (!enable_ept) 1107 guest_efer |= EFER_NX; 1108 1109 /* 1110 * LMA and LME handled by hardware; SCE meaningless outside long mode. 1111 */ 1112 ignore_bits |= EFER_SCE; 1113 #ifdef CONFIG_X86_64 1114 ignore_bits |= EFER_LMA | EFER_LME; 1115 /* SCE is meaningful only in long mode on Intel */ 1116 if (guest_efer & EFER_LMA) 1117 ignore_bits &= ~(u64)EFER_SCE; 1118 #endif 1119 1120 /* 1121 * On EPT, we can't emulate NX, so we must switch EFER atomically. 1122 * On CPUs that support "load IA32_EFER", always switch EFER 1123 * atomically, since it's faster than switching it manually. 1124 */ 1125 if (cpu_has_load_ia32_efer() || 1126 (enable_ept && ((vmx->vcpu.arch.efer ^ host_efer) & EFER_NX))) { 1127 if (!(guest_efer & EFER_LMA)) 1128 guest_efer &= ~EFER_LME; 1129 if (guest_efer != host_efer) 1130 add_atomic_switch_msr(vmx, MSR_EFER, 1131 guest_efer, host_efer, false); 1132 else 1133 clear_atomic_switch_msr(vmx, MSR_EFER); 1134 return false; 1135 } 1136 1137 i = kvm_find_user_return_msr(MSR_EFER); 1138 if (i < 0) 1139 return false; 1140 1141 clear_atomic_switch_msr(vmx, MSR_EFER); 1142 1143 guest_efer &= ~ignore_bits; 1144 guest_efer |= host_efer & ignore_bits; 1145 1146 vmx->guest_uret_msrs[i].data = guest_efer; 1147 vmx->guest_uret_msrs[i].mask = ~ignore_bits; 1148 1149 return true; 1150 } 1151 1152 #ifdef CONFIG_X86_32 1153 /* 1154 * On 32-bit kernels, VM exits still load the FS and GS bases from the 1155 * VMCS rather than the segment table. KVM uses this helper to figure 1156 * out the current bases to poke them into the VMCS before entry. 1157 */ 1158 static unsigned long segment_base(u16 selector) 1159 { 1160 struct desc_struct *table; 1161 unsigned long v; 1162 1163 if (!(selector & ~SEGMENT_RPL_MASK)) 1164 return 0; 1165 1166 table = get_current_gdt_ro(); 1167 1168 if ((selector & SEGMENT_TI_MASK) == SEGMENT_LDT) { 1169 u16 ldt_selector = kvm_read_ldt(); 1170 1171 if (!(ldt_selector & ~SEGMENT_RPL_MASK)) 1172 return 0; 1173 1174 table = (struct desc_struct *)segment_base(ldt_selector); 1175 } 1176 v = get_desc_base(&table[selector >> 3]); 1177 return v; 1178 } 1179 #endif 1180 1181 static inline bool pt_can_write_msr(struct vcpu_vmx *vmx) 1182 { 1183 return vmx_pt_mode_is_host_guest() && 1184 !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN); 1185 } 1186 1187 static inline bool pt_output_base_valid(struct kvm_vcpu *vcpu, u64 base) 1188 { 1189 /* The base must be 128-byte aligned and a legal physical address. */ 1190 return kvm_vcpu_is_legal_aligned_gpa(vcpu, base, 128); 1191 } 1192 1193 static inline void pt_load_msr(struct pt_ctx *ctx, u32 addr_range) 1194 { 1195 u32 i; 1196 1197 wrmsrl(MSR_IA32_RTIT_STATUS, ctx->status); 1198 wrmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base); 1199 wrmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask); 1200 wrmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match); 1201 for (i = 0; i < addr_range; i++) { 1202 wrmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]); 1203 wrmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]); 1204 } 1205 } 1206 1207 static inline void pt_save_msr(struct pt_ctx *ctx, u32 addr_range) 1208 { 1209 u32 i; 1210 1211 rdmsrl(MSR_IA32_RTIT_STATUS, ctx->status); 1212 rdmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base); 1213 rdmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask); 1214 rdmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match); 1215 for (i = 0; i < addr_range; i++) { 1216 rdmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]); 1217 rdmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]); 1218 } 1219 } 1220 1221 static void pt_guest_enter(struct vcpu_vmx *vmx) 1222 { 1223 if (vmx_pt_mode_is_system()) 1224 return; 1225 1226 /* 1227 * GUEST_IA32_RTIT_CTL is already set in the VMCS. 1228 * Save host state before VM entry. 1229 */ 1230 rdmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); 1231 if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) { 1232 wrmsrl(MSR_IA32_RTIT_CTL, 0); 1233 pt_save_msr(&vmx->pt_desc.host, vmx->pt_desc.num_address_ranges); 1234 pt_load_msr(&vmx->pt_desc.guest, vmx->pt_desc.num_address_ranges); 1235 } 1236 } 1237 1238 static void pt_guest_exit(struct vcpu_vmx *vmx) 1239 { 1240 if (vmx_pt_mode_is_system()) 1241 return; 1242 1243 if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) { 1244 pt_save_msr(&vmx->pt_desc.guest, vmx->pt_desc.num_address_ranges); 1245 pt_load_msr(&vmx->pt_desc.host, vmx->pt_desc.num_address_ranges); 1246 } 1247 1248 /* 1249 * KVM requires VM_EXIT_CLEAR_IA32_RTIT_CTL to expose PT to the guest, 1250 * i.e. RTIT_CTL is always cleared on VM-Exit. Restore it if necessary. 1251 */ 1252 if (vmx->pt_desc.host.ctl) 1253 wrmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); 1254 } 1255 1256 void vmx_set_host_fs_gs(struct vmcs_host_state *host, u16 fs_sel, u16 gs_sel, 1257 unsigned long fs_base, unsigned long gs_base) 1258 { 1259 if (unlikely(fs_sel != host->fs_sel)) { 1260 if (!(fs_sel & 7)) 1261 vmcs_write16(HOST_FS_SELECTOR, fs_sel); 1262 else 1263 vmcs_write16(HOST_FS_SELECTOR, 0); 1264 host->fs_sel = fs_sel; 1265 } 1266 if (unlikely(gs_sel != host->gs_sel)) { 1267 if (!(gs_sel & 7)) 1268 vmcs_write16(HOST_GS_SELECTOR, gs_sel); 1269 else 1270 vmcs_write16(HOST_GS_SELECTOR, 0); 1271 host->gs_sel = gs_sel; 1272 } 1273 if (unlikely(fs_base != host->fs_base)) { 1274 vmcs_writel(HOST_FS_BASE, fs_base); 1275 host->fs_base = fs_base; 1276 } 1277 if (unlikely(gs_base != host->gs_base)) { 1278 vmcs_writel(HOST_GS_BASE, gs_base); 1279 host->gs_base = gs_base; 1280 } 1281 } 1282 1283 void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu) 1284 { 1285 struct vcpu_vmx *vmx = to_vmx(vcpu); 1286 struct vmcs_host_state *host_state; 1287 #ifdef CONFIG_X86_64 1288 int cpu = raw_smp_processor_id(); 1289 #endif 1290 unsigned long fs_base, gs_base; 1291 u16 fs_sel, gs_sel; 1292 int i; 1293 1294 /* 1295 * Note that guest MSRs to be saved/restored can also be changed 1296 * when guest state is loaded. This happens when guest transitions 1297 * to/from long-mode by setting MSR_EFER.LMA. 1298 */ 1299 if (!vmx->guest_uret_msrs_loaded) { 1300 vmx->guest_uret_msrs_loaded = true; 1301 for (i = 0; i < kvm_nr_uret_msrs; ++i) { 1302 if (!vmx->guest_uret_msrs[i].load_into_hardware) 1303 continue; 1304 1305 kvm_set_user_return_msr(i, 1306 vmx->guest_uret_msrs[i].data, 1307 vmx->guest_uret_msrs[i].mask); 1308 } 1309 } 1310 1311 if (vmx->nested.need_vmcs12_to_shadow_sync) 1312 nested_sync_vmcs12_to_shadow(vcpu); 1313 1314 if (vmx->guest_state_loaded) 1315 return; 1316 1317 host_state = &vmx->loaded_vmcs->host_state; 1318 1319 /* 1320 * Set host fs and gs selectors. Unfortunately, 22.2.3 does not 1321 * allow segment selectors with cpl > 0 or ti == 1. 1322 */ 1323 host_state->ldt_sel = kvm_read_ldt(); 1324 1325 #ifdef CONFIG_X86_64 1326 savesegment(ds, host_state->ds_sel); 1327 savesegment(es, host_state->es_sel); 1328 1329 gs_base = cpu_kernelmode_gs_base(cpu); 1330 if (likely(is_64bit_mm(current->mm))) { 1331 current_save_fsgs(); 1332 fs_sel = current->thread.fsindex; 1333 gs_sel = current->thread.gsindex; 1334 fs_base = current->thread.fsbase; 1335 vmx->msr_host_kernel_gs_base = current->thread.gsbase; 1336 } else { 1337 savesegment(fs, fs_sel); 1338 savesegment(gs, gs_sel); 1339 fs_base = read_msr(MSR_FS_BASE); 1340 vmx->msr_host_kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE); 1341 } 1342 1343 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); 1344 #else 1345 savesegment(fs, fs_sel); 1346 savesegment(gs, gs_sel); 1347 fs_base = segment_base(fs_sel); 1348 gs_base = segment_base(gs_sel); 1349 #endif 1350 1351 vmx_set_host_fs_gs(host_state, fs_sel, gs_sel, fs_base, gs_base); 1352 vmx->guest_state_loaded = true; 1353 } 1354 1355 static void vmx_prepare_switch_to_host(struct vcpu_vmx *vmx) 1356 { 1357 struct vmcs_host_state *host_state; 1358 1359 if (!vmx->guest_state_loaded) 1360 return; 1361 1362 host_state = &vmx->loaded_vmcs->host_state; 1363 1364 ++vmx->vcpu.stat.host_state_reload; 1365 1366 #ifdef CONFIG_X86_64 1367 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); 1368 #endif 1369 if (host_state->ldt_sel || (host_state->gs_sel & 7)) { 1370 kvm_load_ldt(host_state->ldt_sel); 1371 #ifdef CONFIG_X86_64 1372 load_gs_index(host_state->gs_sel); 1373 #else 1374 loadsegment(gs, host_state->gs_sel); 1375 #endif 1376 } 1377 if (host_state->fs_sel & 7) 1378 loadsegment(fs, host_state->fs_sel); 1379 #ifdef CONFIG_X86_64 1380 if (unlikely(host_state->ds_sel | host_state->es_sel)) { 1381 loadsegment(ds, host_state->ds_sel); 1382 loadsegment(es, host_state->es_sel); 1383 } 1384 #endif 1385 invalidate_tss_limit(); 1386 #ifdef CONFIG_X86_64 1387 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base); 1388 #endif 1389 load_fixmap_gdt(raw_smp_processor_id()); 1390 vmx->guest_state_loaded = false; 1391 vmx->guest_uret_msrs_loaded = false; 1392 } 1393 1394 #ifdef CONFIG_X86_64 1395 static u64 vmx_read_guest_kernel_gs_base(struct vcpu_vmx *vmx) 1396 { 1397 preempt_disable(); 1398 if (vmx->guest_state_loaded) 1399 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); 1400 preempt_enable(); 1401 return vmx->msr_guest_kernel_gs_base; 1402 } 1403 1404 static void vmx_write_guest_kernel_gs_base(struct vcpu_vmx *vmx, u64 data) 1405 { 1406 preempt_disable(); 1407 if (vmx->guest_state_loaded) 1408 wrmsrl(MSR_KERNEL_GS_BASE, data); 1409 preempt_enable(); 1410 vmx->msr_guest_kernel_gs_base = data; 1411 } 1412 #endif 1413 1414 void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu, 1415 struct loaded_vmcs *buddy) 1416 { 1417 struct vcpu_vmx *vmx = to_vmx(vcpu); 1418 bool already_loaded = vmx->loaded_vmcs->cpu == cpu; 1419 struct vmcs *prev; 1420 1421 if (!already_loaded) { 1422 loaded_vmcs_clear(vmx->loaded_vmcs); 1423 local_irq_disable(); 1424 1425 /* 1426 * Ensure loaded_vmcs->cpu is read before adding loaded_vmcs to 1427 * this cpu's percpu list, otherwise it may not yet be deleted 1428 * from its previous cpu's percpu list. Pairs with the 1429 * smb_wmb() in __loaded_vmcs_clear(). 1430 */ 1431 smp_rmb(); 1432 1433 list_add(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link, 1434 &per_cpu(loaded_vmcss_on_cpu, cpu)); 1435 local_irq_enable(); 1436 } 1437 1438 prev = per_cpu(current_vmcs, cpu); 1439 if (prev != vmx->loaded_vmcs->vmcs) { 1440 per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs; 1441 vmcs_load(vmx->loaded_vmcs->vmcs); 1442 1443 /* 1444 * No indirect branch prediction barrier needed when switching 1445 * the active VMCS within a vCPU, unless IBRS is advertised to 1446 * the vCPU. To minimize the number of IBPBs executed, KVM 1447 * performs IBPB on nested VM-Exit (a single nested transition 1448 * may switch the active VMCS multiple times). 1449 */ 1450 if (!buddy || WARN_ON_ONCE(buddy->vmcs != prev)) 1451 indirect_branch_prediction_barrier(); 1452 } 1453 1454 if (!already_loaded) { 1455 void *gdt = get_current_gdt_ro(); 1456 1457 /* 1458 * Flush all EPTP/VPID contexts, the new pCPU may have stale 1459 * TLB entries from its previous association with the vCPU. 1460 */ 1461 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 1462 1463 /* 1464 * Linux uses per-cpu TSS and GDT, so set these when switching 1465 * processors. See 22.2.4. 1466 */ 1467 vmcs_writel(HOST_TR_BASE, 1468 (unsigned long)&get_cpu_entry_area(cpu)->tss.x86_tss); 1469 vmcs_writel(HOST_GDTR_BASE, (unsigned long)gdt); /* 22.2.4 */ 1470 1471 if (IS_ENABLED(CONFIG_IA32_EMULATION) || IS_ENABLED(CONFIG_X86_32)) { 1472 /* 22.2.3 */ 1473 vmcs_writel(HOST_IA32_SYSENTER_ESP, 1474 (unsigned long)(cpu_entry_stack(cpu) + 1)); 1475 } 1476 1477 vmx->loaded_vmcs->cpu = cpu; 1478 } 1479 } 1480 1481 /* 1482 * Switches to specified vcpu, until a matching vcpu_put(), but assumes 1483 * vcpu mutex is already taken. 1484 */ 1485 void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1486 { 1487 struct vcpu_vmx *vmx = to_vmx(vcpu); 1488 1489 vmx_vcpu_load_vmcs(vcpu, cpu, NULL); 1490 1491 vmx_vcpu_pi_load(vcpu, cpu); 1492 1493 vmx->host_debugctlmsr = get_debugctlmsr(); 1494 } 1495 1496 void vmx_vcpu_put(struct kvm_vcpu *vcpu) 1497 { 1498 vmx_vcpu_pi_put(vcpu); 1499 1500 vmx_prepare_switch_to_host(to_vmx(vcpu)); 1501 } 1502 1503 bool vmx_emulation_required(struct kvm_vcpu *vcpu) 1504 { 1505 return emulate_invalid_guest_state && !vmx_guest_state_valid(vcpu); 1506 } 1507 1508 unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu) 1509 { 1510 struct vcpu_vmx *vmx = to_vmx(vcpu); 1511 unsigned long rflags, save_rflags; 1512 1513 if (!kvm_register_is_available(vcpu, VCPU_EXREG_RFLAGS)) { 1514 kvm_register_mark_available(vcpu, VCPU_EXREG_RFLAGS); 1515 rflags = vmcs_readl(GUEST_RFLAGS); 1516 if (vmx->rmode.vm86_active) { 1517 rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS; 1518 save_rflags = vmx->rmode.save_rflags; 1519 rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS; 1520 } 1521 vmx->rflags = rflags; 1522 } 1523 return vmx->rflags; 1524 } 1525 1526 void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) 1527 { 1528 struct vcpu_vmx *vmx = to_vmx(vcpu); 1529 unsigned long old_rflags; 1530 1531 /* 1532 * Unlike CR0 and CR4, RFLAGS handling requires checking if the vCPU 1533 * is an unrestricted guest in order to mark L2 as needing emulation 1534 * if L1 runs L2 as a restricted guest. 1535 */ 1536 if (is_unrestricted_guest(vcpu)) { 1537 kvm_register_mark_available(vcpu, VCPU_EXREG_RFLAGS); 1538 vmx->rflags = rflags; 1539 vmcs_writel(GUEST_RFLAGS, rflags); 1540 return; 1541 } 1542 1543 old_rflags = vmx_get_rflags(vcpu); 1544 vmx->rflags = rflags; 1545 if (vmx->rmode.vm86_active) { 1546 vmx->rmode.save_rflags = rflags; 1547 rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM; 1548 } 1549 vmcs_writel(GUEST_RFLAGS, rflags); 1550 1551 if ((old_rflags ^ vmx->rflags) & X86_EFLAGS_VM) 1552 vmx->emulation_required = vmx_emulation_required(vcpu); 1553 } 1554 1555 bool vmx_get_if_flag(struct kvm_vcpu *vcpu) 1556 { 1557 return vmx_get_rflags(vcpu) & X86_EFLAGS_IF; 1558 } 1559 1560 u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu) 1561 { 1562 u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 1563 int ret = 0; 1564 1565 if (interruptibility & GUEST_INTR_STATE_STI) 1566 ret |= KVM_X86_SHADOW_INT_STI; 1567 if (interruptibility & GUEST_INTR_STATE_MOV_SS) 1568 ret |= KVM_X86_SHADOW_INT_MOV_SS; 1569 1570 return ret; 1571 } 1572 1573 void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask) 1574 { 1575 u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 1576 u32 interruptibility = interruptibility_old; 1577 1578 interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS); 1579 1580 if (mask & KVM_X86_SHADOW_INT_MOV_SS) 1581 interruptibility |= GUEST_INTR_STATE_MOV_SS; 1582 else if (mask & KVM_X86_SHADOW_INT_STI) 1583 interruptibility |= GUEST_INTR_STATE_STI; 1584 1585 if ((interruptibility != interruptibility_old)) 1586 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility); 1587 } 1588 1589 static int vmx_rtit_ctl_check(struct kvm_vcpu *vcpu, u64 data) 1590 { 1591 struct vcpu_vmx *vmx = to_vmx(vcpu); 1592 unsigned long value; 1593 1594 /* 1595 * Any MSR write that attempts to change bits marked reserved will 1596 * case a #GP fault. 1597 */ 1598 if (data & vmx->pt_desc.ctl_bitmask) 1599 return 1; 1600 1601 /* 1602 * Any attempt to modify IA32_RTIT_CTL while TraceEn is set will 1603 * result in a #GP unless the same write also clears TraceEn. 1604 */ 1605 if ((vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) && 1606 ((vmx->pt_desc.guest.ctl ^ data) & ~RTIT_CTL_TRACEEN)) 1607 return 1; 1608 1609 /* 1610 * WRMSR to IA32_RTIT_CTL that sets TraceEn but clears this bit 1611 * and FabricEn would cause #GP, if 1612 * CPUID.(EAX=14H, ECX=0):ECX.SNGLRGNOUT[bit 2] = 0 1613 */ 1614 if ((data & RTIT_CTL_TRACEEN) && !(data & RTIT_CTL_TOPA) && 1615 !(data & RTIT_CTL_FABRIC_EN) && 1616 !intel_pt_validate_cap(vmx->pt_desc.caps, 1617 PT_CAP_single_range_output)) 1618 return 1; 1619 1620 /* 1621 * MTCFreq, CycThresh and PSBFreq encodings check, any MSR write that 1622 * utilize encodings marked reserved will cause a #GP fault. 1623 */ 1624 value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc_periods); 1625 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc) && 1626 !test_bit((data & RTIT_CTL_MTC_RANGE) >> 1627 RTIT_CTL_MTC_RANGE_OFFSET, &value)) 1628 return 1; 1629 value = intel_pt_validate_cap(vmx->pt_desc.caps, 1630 PT_CAP_cycle_thresholds); 1631 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) && 1632 !test_bit((data & RTIT_CTL_CYC_THRESH) >> 1633 RTIT_CTL_CYC_THRESH_OFFSET, &value)) 1634 return 1; 1635 value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_periods); 1636 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) && 1637 !test_bit((data & RTIT_CTL_PSB_FREQ) >> 1638 RTIT_CTL_PSB_FREQ_OFFSET, &value)) 1639 return 1; 1640 1641 /* 1642 * If ADDRx_CFG is reserved or the encodings is >2 will 1643 * cause a #GP fault. 1644 */ 1645 value = (data & RTIT_CTL_ADDR0) >> RTIT_CTL_ADDR0_OFFSET; 1646 if ((value && (vmx->pt_desc.num_address_ranges < 1)) || (value > 2)) 1647 return 1; 1648 value = (data & RTIT_CTL_ADDR1) >> RTIT_CTL_ADDR1_OFFSET; 1649 if ((value && (vmx->pt_desc.num_address_ranges < 2)) || (value > 2)) 1650 return 1; 1651 value = (data & RTIT_CTL_ADDR2) >> RTIT_CTL_ADDR2_OFFSET; 1652 if ((value && (vmx->pt_desc.num_address_ranges < 3)) || (value > 2)) 1653 return 1; 1654 value = (data & RTIT_CTL_ADDR3) >> RTIT_CTL_ADDR3_OFFSET; 1655 if ((value && (vmx->pt_desc.num_address_ranges < 4)) || (value > 2)) 1656 return 1; 1657 1658 return 0; 1659 } 1660 1661 int vmx_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, 1662 void *insn, int insn_len) 1663 { 1664 /* 1665 * Emulation of instructions in SGX enclaves is impossible as RIP does 1666 * not point at the failing instruction, and even if it did, the code 1667 * stream is inaccessible. Inject #UD instead of exiting to userspace 1668 * so that guest userspace can't DoS the guest simply by triggering 1669 * emulation (enclaves are CPL3 only). 1670 */ 1671 if (to_vmx(vcpu)->exit_reason.enclave_mode) { 1672 kvm_queue_exception(vcpu, UD_VECTOR); 1673 return X86EMUL_PROPAGATE_FAULT; 1674 } 1675 return X86EMUL_CONTINUE; 1676 } 1677 1678 static int skip_emulated_instruction(struct kvm_vcpu *vcpu) 1679 { 1680 union vmx_exit_reason exit_reason = to_vmx(vcpu)->exit_reason; 1681 unsigned long rip, orig_rip; 1682 u32 instr_len; 1683 1684 /* 1685 * Using VMCS.VM_EXIT_INSTRUCTION_LEN on EPT misconfig depends on 1686 * undefined behavior: Intel's SDM doesn't mandate the VMCS field be 1687 * set when EPT misconfig occurs. In practice, real hardware updates 1688 * VM_EXIT_INSTRUCTION_LEN on EPT misconfig, but other hypervisors 1689 * (namely Hyper-V) don't set it due to it being undefined behavior, 1690 * i.e. we end up advancing IP with some random value. 1691 */ 1692 if (!static_cpu_has(X86_FEATURE_HYPERVISOR) || 1693 exit_reason.basic != EXIT_REASON_EPT_MISCONFIG) { 1694 instr_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 1695 1696 /* 1697 * Emulating an enclave's instructions isn't supported as KVM 1698 * cannot access the enclave's memory or its true RIP, e.g. the 1699 * vmcs.GUEST_RIP points at the exit point of the enclave, not 1700 * the RIP that actually triggered the VM-Exit. But, because 1701 * most instructions that cause VM-Exit will #UD in an enclave, 1702 * most instruction-based VM-Exits simply do not occur. 1703 * 1704 * There are a few exceptions, notably the debug instructions 1705 * INT1ICEBRK and INT3, as they are allowed in debug enclaves 1706 * and generate #DB/#BP as expected, which KVM might intercept. 1707 * But again, the CPU does the dirty work and saves an instr 1708 * length of zero so VMMs don't shoot themselves in the foot. 1709 * WARN if KVM tries to skip a non-zero length instruction on 1710 * a VM-Exit from an enclave. 1711 */ 1712 if (!instr_len) 1713 goto rip_updated; 1714 1715 WARN_ONCE(exit_reason.enclave_mode, 1716 "skipping instruction after SGX enclave VM-Exit"); 1717 1718 orig_rip = kvm_rip_read(vcpu); 1719 rip = orig_rip + instr_len; 1720 #ifdef CONFIG_X86_64 1721 /* 1722 * We need to mask out the high 32 bits of RIP if not in 64-bit 1723 * mode, but just finding out that we are in 64-bit mode is 1724 * quite expensive. Only do it if there was a carry. 1725 */ 1726 if (unlikely(((rip ^ orig_rip) >> 31) == 3) && !is_64_bit_mode(vcpu)) 1727 rip = (u32)rip; 1728 #endif 1729 kvm_rip_write(vcpu, rip); 1730 } else { 1731 if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP)) 1732 return 0; 1733 } 1734 1735 rip_updated: 1736 /* skipping an emulated instruction also counts */ 1737 vmx_set_interrupt_shadow(vcpu, 0); 1738 1739 return 1; 1740 } 1741 1742 /* 1743 * Recognizes a pending MTF VM-exit and records the nested state for later 1744 * delivery. 1745 */ 1746 void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu) 1747 { 1748 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1749 struct vcpu_vmx *vmx = to_vmx(vcpu); 1750 1751 if (!is_guest_mode(vcpu)) 1752 return; 1753 1754 /* 1755 * Per the SDM, MTF takes priority over debug-trap exceptions besides 1756 * TSS T-bit traps and ICEBP (INT1). KVM doesn't emulate T-bit traps 1757 * or ICEBP (in the emulator proper), and skipping of ICEBP after an 1758 * intercepted #DB deliberately avoids single-step #DB and MTF updates 1759 * as ICEBP is higher priority than both. As instruction emulation is 1760 * completed at this point (i.e. KVM is at the instruction boundary), 1761 * any #DB exception pending delivery must be a debug-trap of lower 1762 * priority than MTF. Record the pending MTF state to be delivered in 1763 * vmx_check_nested_events(). 1764 */ 1765 if (nested_cpu_has_mtf(vmcs12) && 1766 (!vcpu->arch.exception.pending || 1767 vcpu->arch.exception.vector == DB_VECTOR) && 1768 (!vcpu->arch.exception_vmexit.pending || 1769 vcpu->arch.exception_vmexit.vector == DB_VECTOR)) { 1770 vmx->nested.mtf_pending = true; 1771 kvm_make_request(KVM_REQ_EVENT, vcpu); 1772 } else { 1773 vmx->nested.mtf_pending = false; 1774 } 1775 } 1776 1777 int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu) 1778 { 1779 vmx_update_emulated_instruction(vcpu); 1780 return skip_emulated_instruction(vcpu); 1781 } 1782 1783 static void vmx_clear_hlt(struct kvm_vcpu *vcpu) 1784 { 1785 /* 1786 * Ensure that we clear the HLT state in the VMCS. We don't need to 1787 * explicitly skip the instruction because if the HLT state is set, 1788 * then the instruction is already executing and RIP has already been 1789 * advanced. 1790 */ 1791 if (kvm_hlt_in_guest(vcpu->kvm) && 1792 vmcs_read32(GUEST_ACTIVITY_STATE) == GUEST_ACTIVITY_HLT) 1793 vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE); 1794 } 1795 1796 void vmx_inject_exception(struct kvm_vcpu *vcpu) 1797 { 1798 struct kvm_queued_exception *ex = &vcpu->arch.exception; 1799 u32 intr_info = ex->vector | INTR_INFO_VALID_MASK; 1800 struct vcpu_vmx *vmx = to_vmx(vcpu); 1801 1802 kvm_deliver_exception_payload(vcpu, ex); 1803 1804 if (ex->has_error_code) { 1805 /* 1806 * Despite the error code being architecturally defined as 32 1807 * bits, and the VMCS field being 32 bits, Intel CPUs and thus 1808 * VMX don't actually supporting setting bits 31:16. Hardware 1809 * will (should) never provide a bogus error code, but AMD CPUs 1810 * do generate error codes with bits 31:16 set, and so KVM's 1811 * ABI lets userspace shove in arbitrary 32-bit values. Drop 1812 * the upper bits to avoid VM-Fail, losing information that 1813 * doesn't really exist is preferable to killing the VM. 1814 */ 1815 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, (u16)ex->error_code); 1816 intr_info |= INTR_INFO_DELIVER_CODE_MASK; 1817 } 1818 1819 if (vmx->rmode.vm86_active) { 1820 int inc_eip = 0; 1821 if (kvm_exception_is_soft(ex->vector)) 1822 inc_eip = vcpu->arch.event_exit_inst_len; 1823 kvm_inject_realmode_interrupt(vcpu, ex->vector, inc_eip); 1824 return; 1825 } 1826 1827 WARN_ON_ONCE(vmx->emulation_required); 1828 1829 if (kvm_exception_is_soft(ex->vector)) { 1830 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1831 vmx->vcpu.arch.event_exit_inst_len); 1832 intr_info |= INTR_TYPE_SOFT_EXCEPTION; 1833 } else 1834 intr_info |= INTR_TYPE_HARD_EXCEPTION; 1835 1836 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info); 1837 1838 vmx_clear_hlt(vcpu); 1839 } 1840 1841 static void vmx_setup_uret_msr(struct vcpu_vmx *vmx, unsigned int msr, 1842 bool load_into_hardware) 1843 { 1844 struct vmx_uret_msr *uret_msr; 1845 1846 uret_msr = vmx_find_uret_msr(vmx, msr); 1847 if (!uret_msr) 1848 return; 1849 1850 uret_msr->load_into_hardware = load_into_hardware; 1851 } 1852 1853 /* 1854 * Configuring user return MSRs to automatically save, load, and restore MSRs 1855 * that need to be shoved into hardware when running the guest. Note, omitting 1856 * an MSR here does _NOT_ mean it's not emulated, only that it will not be 1857 * loaded into hardware when running the guest. 1858 */ 1859 static void vmx_setup_uret_msrs(struct vcpu_vmx *vmx) 1860 { 1861 #ifdef CONFIG_X86_64 1862 bool load_syscall_msrs; 1863 1864 /* 1865 * The SYSCALL MSRs are only needed on long mode guests, and only 1866 * when EFER.SCE is set. 1867 */ 1868 load_syscall_msrs = is_long_mode(&vmx->vcpu) && 1869 (vmx->vcpu.arch.efer & EFER_SCE); 1870 1871 vmx_setup_uret_msr(vmx, MSR_STAR, load_syscall_msrs); 1872 vmx_setup_uret_msr(vmx, MSR_LSTAR, load_syscall_msrs); 1873 vmx_setup_uret_msr(vmx, MSR_SYSCALL_MASK, load_syscall_msrs); 1874 #endif 1875 vmx_setup_uret_msr(vmx, MSR_EFER, update_transition_efer(vmx)); 1876 1877 vmx_setup_uret_msr(vmx, MSR_TSC_AUX, 1878 guest_cpuid_has(&vmx->vcpu, X86_FEATURE_RDTSCP) || 1879 guest_cpuid_has(&vmx->vcpu, X86_FEATURE_RDPID)); 1880 1881 /* 1882 * hle=0, rtm=0, tsx_ctrl=1 can be found with some combinations of new 1883 * kernel and old userspace. If those guests run on a tsx=off host, do 1884 * allow guests to use TSX_CTRL, but don't change the value in hardware 1885 * so that TSX remains always disabled. 1886 */ 1887 vmx_setup_uret_msr(vmx, MSR_IA32_TSX_CTRL, boot_cpu_has(X86_FEATURE_RTM)); 1888 1889 /* 1890 * The set of MSRs to load may have changed, reload MSRs before the 1891 * next VM-Enter. 1892 */ 1893 vmx->guest_uret_msrs_loaded = false; 1894 } 1895 1896 u64 vmx_get_l2_tsc_offset(struct kvm_vcpu *vcpu) 1897 { 1898 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1899 1900 if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING)) 1901 return vmcs12->tsc_offset; 1902 1903 return 0; 1904 } 1905 1906 u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu) 1907 { 1908 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1909 1910 if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING) && 1911 nested_cpu_has2(vmcs12, SECONDARY_EXEC_TSC_SCALING)) 1912 return vmcs12->tsc_multiplier; 1913 1914 return kvm_caps.default_tsc_scaling_ratio; 1915 } 1916 1917 void vmx_write_tsc_offset(struct kvm_vcpu *vcpu) 1918 { 1919 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 1920 } 1921 1922 void vmx_write_tsc_multiplier(struct kvm_vcpu *vcpu) 1923 { 1924 vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio); 1925 } 1926 1927 /* 1928 * Userspace is allowed to set any supported IA32_FEATURE_CONTROL regardless of 1929 * guest CPUID. Note, KVM allows userspace to set "VMX in SMX" to maintain 1930 * backwards compatibility even though KVM doesn't support emulating SMX. And 1931 * because userspace set "VMX in SMX", the guest must also be allowed to set it, 1932 * e.g. if the MSR is left unlocked and the guest does a RMW operation. 1933 */ 1934 #define KVM_SUPPORTED_FEATURE_CONTROL (FEAT_CTL_LOCKED | \ 1935 FEAT_CTL_VMX_ENABLED_INSIDE_SMX | \ 1936 FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX | \ 1937 FEAT_CTL_SGX_LC_ENABLED | \ 1938 FEAT_CTL_SGX_ENABLED | \ 1939 FEAT_CTL_LMCE_ENABLED) 1940 1941 static inline bool is_vmx_feature_control_msr_valid(struct vcpu_vmx *vmx, 1942 struct msr_data *msr) 1943 { 1944 uint64_t valid_bits; 1945 1946 /* 1947 * Ensure KVM_SUPPORTED_FEATURE_CONTROL is updated when new bits are 1948 * exposed to the guest. 1949 */ 1950 WARN_ON_ONCE(vmx->msr_ia32_feature_control_valid_bits & 1951 ~KVM_SUPPORTED_FEATURE_CONTROL); 1952 1953 if (!msr->host_initiated && 1954 (vmx->msr_ia32_feature_control & FEAT_CTL_LOCKED)) 1955 return false; 1956 1957 if (msr->host_initiated) 1958 valid_bits = KVM_SUPPORTED_FEATURE_CONTROL; 1959 else 1960 valid_bits = vmx->msr_ia32_feature_control_valid_bits; 1961 1962 return !(msr->data & ~valid_bits); 1963 } 1964 1965 int vmx_get_msr_feature(struct kvm_msr_entry *msr) 1966 { 1967 switch (msr->index) { 1968 case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR: 1969 if (!nested) 1970 return 1; 1971 return vmx_get_vmx_msr(&vmcs_config.nested, msr->index, &msr->data); 1972 default: 1973 return KVM_MSR_RET_INVALID; 1974 } 1975 } 1976 1977 /* 1978 * Reads an msr value (of 'msr_info->index') into 'msr_info->data'. 1979 * Returns 0 on success, non-0 otherwise. 1980 * Assumes vcpu_load() was already called. 1981 */ 1982 int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 1983 { 1984 struct vcpu_vmx *vmx = to_vmx(vcpu); 1985 struct vmx_uret_msr *msr; 1986 u32 index; 1987 1988 switch (msr_info->index) { 1989 #ifdef CONFIG_X86_64 1990 case MSR_FS_BASE: 1991 msr_info->data = vmcs_readl(GUEST_FS_BASE); 1992 break; 1993 case MSR_GS_BASE: 1994 msr_info->data = vmcs_readl(GUEST_GS_BASE); 1995 break; 1996 case MSR_KERNEL_GS_BASE: 1997 msr_info->data = vmx_read_guest_kernel_gs_base(vmx); 1998 break; 1999 #endif 2000 case MSR_EFER: 2001 return kvm_get_msr_common(vcpu, msr_info); 2002 case MSR_IA32_TSX_CTRL: 2003 if (!msr_info->host_initiated && 2004 !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR)) 2005 return 1; 2006 goto find_uret_msr; 2007 case MSR_IA32_UMWAIT_CONTROL: 2008 if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx)) 2009 return 1; 2010 2011 msr_info->data = vmx->msr_ia32_umwait_control; 2012 break; 2013 case MSR_IA32_SPEC_CTRL: 2014 if (!msr_info->host_initiated && 2015 !guest_has_spec_ctrl_msr(vcpu)) 2016 return 1; 2017 2018 msr_info->data = to_vmx(vcpu)->spec_ctrl; 2019 break; 2020 case MSR_IA32_SYSENTER_CS: 2021 msr_info->data = vmcs_read32(GUEST_SYSENTER_CS); 2022 break; 2023 case MSR_IA32_SYSENTER_EIP: 2024 msr_info->data = vmcs_readl(GUEST_SYSENTER_EIP); 2025 break; 2026 case MSR_IA32_SYSENTER_ESP: 2027 msr_info->data = vmcs_readl(GUEST_SYSENTER_ESP); 2028 break; 2029 case MSR_IA32_BNDCFGS: 2030 if (!kvm_mpx_supported() || 2031 (!msr_info->host_initiated && 2032 !guest_cpuid_has(vcpu, X86_FEATURE_MPX))) 2033 return 1; 2034 msr_info->data = vmcs_read64(GUEST_BNDCFGS); 2035 break; 2036 case MSR_IA32_MCG_EXT_CTL: 2037 if (!msr_info->host_initiated && 2038 !(vmx->msr_ia32_feature_control & 2039 FEAT_CTL_LMCE_ENABLED)) 2040 return 1; 2041 msr_info->data = vcpu->arch.mcg_ext_ctl; 2042 break; 2043 case MSR_IA32_FEAT_CTL: 2044 msr_info->data = vmx->msr_ia32_feature_control; 2045 break; 2046 case MSR_IA32_SGXLEPUBKEYHASH0 ... MSR_IA32_SGXLEPUBKEYHASH3: 2047 if (!msr_info->host_initiated && 2048 !guest_cpuid_has(vcpu, X86_FEATURE_SGX_LC)) 2049 return 1; 2050 msr_info->data = to_vmx(vcpu)->msr_ia32_sgxlepubkeyhash 2051 [msr_info->index - MSR_IA32_SGXLEPUBKEYHASH0]; 2052 break; 2053 case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR: 2054 if (!guest_can_use(vcpu, X86_FEATURE_VMX)) 2055 return 1; 2056 if (vmx_get_vmx_msr(&vmx->nested.msrs, msr_info->index, 2057 &msr_info->data)) 2058 return 1; 2059 #ifdef CONFIG_KVM_HYPERV 2060 /* 2061 * Enlightened VMCS v1 doesn't have certain VMCS fields but 2062 * instead of just ignoring the features, different Hyper-V 2063 * versions are either trying to use them and fail or do some 2064 * sanity checking and refuse to boot. Filter all unsupported 2065 * features out. 2066 */ 2067 if (!msr_info->host_initiated && guest_cpuid_has_evmcs(vcpu)) 2068 nested_evmcs_filter_control_msr(vcpu, msr_info->index, 2069 &msr_info->data); 2070 #endif 2071 break; 2072 case MSR_IA32_RTIT_CTL: 2073 if (!vmx_pt_mode_is_host_guest()) 2074 return 1; 2075 msr_info->data = vmx->pt_desc.guest.ctl; 2076 break; 2077 case MSR_IA32_RTIT_STATUS: 2078 if (!vmx_pt_mode_is_host_guest()) 2079 return 1; 2080 msr_info->data = vmx->pt_desc.guest.status; 2081 break; 2082 case MSR_IA32_RTIT_CR3_MATCH: 2083 if (!vmx_pt_mode_is_host_guest() || 2084 !intel_pt_validate_cap(vmx->pt_desc.caps, 2085 PT_CAP_cr3_filtering)) 2086 return 1; 2087 msr_info->data = vmx->pt_desc.guest.cr3_match; 2088 break; 2089 case MSR_IA32_RTIT_OUTPUT_BASE: 2090 if (!vmx_pt_mode_is_host_guest() || 2091 (!intel_pt_validate_cap(vmx->pt_desc.caps, 2092 PT_CAP_topa_output) && 2093 !intel_pt_validate_cap(vmx->pt_desc.caps, 2094 PT_CAP_single_range_output))) 2095 return 1; 2096 msr_info->data = vmx->pt_desc.guest.output_base; 2097 break; 2098 case MSR_IA32_RTIT_OUTPUT_MASK: 2099 if (!vmx_pt_mode_is_host_guest() || 2100 (!intel_pt_validate_cap(vmx->pt_desc.caps, 2101 PT_CAP_topa_output) && 2102 !intel_pt_validate_cap(vmx->pt_desc.caps, 2103 PT_CAP_single_range_output))) 2104 return 1; 2105 msr_info->data = vmx->pt_desc.guest.output_mask; 2106 break; 2107 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: 2108 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A; 2109 if (!vmx_pt_mode_is_host_guest() || 2110 (index >= 2 * vmx->pt_desc.num_address_ranges)) 2111 return 1; 2112 if (index % 2) 2113 msr_info->data = vmx->pt_desc.guest.addr_b[index / 2]; 2114 else 2115 msr_info->data = vmx->pt_desc.guest.addr_a[index / 2]; 2116 break; 2117 case MSR_IA32_DEBUGCTLMSR: 2118 msr_info->data = vmcs_read64(GUEST_IA32_DEBUGCTL); 2119 break; 2120 default: 2121 find_uret_msr: 2122 msr = vmx_find_uret_msr(vmx, msr_info->index); 2123 if (msr) { 2124 msr_info->data = msr->data; 2125 break; 2126 } 2127 return kvm_get_msr_common(vcpu, msr_info); 2128 } 2129 2130 return 0; 2131 } 2132 2133 static u64 nested_vmx_truncate_sysenter_addr(struct kvm_vcpu *vcpu, 2134 u64 data) 2135 { 2136 #ifdef CONFIG_X86_64 2137 if (!guest_cpuid_has(vcpu, X86_FEATURE_LM)) 2138 return (u32)data; 2139 #endif 2140 return (unsigned long)data; 2141 } 2142 2143 static u64 vmx_get_supported_debugctl(struct kvm_vcpu *vcpu, bool host_initiated) 2144 { 2145 u64 debugctl = 0; 2146 2147 if (boot_cpu_has(X86_FEATURE_BUS_LOCK_DETECT) && 2148 (host_initiated || guest_cpuid_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT))) 2149 debugctl |= DEBUGCTLMSR_BUS_LOCK_DETECT; 2150 2151 if ((kvm_caps.supported_perf_cap & PMU_CAP_LBR_FMT) && 2152 (host_initiated || intel_pmu_lbr_is_enabled(vcpu))) 2153 debugctl |= DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI; 2154 2155 return debugctl; 2156 } 2157 2158 /* 2159 * Writes msr value into the appropriate "register". 2160 * Returns 0 on success, non-0 otherwise. 2161 * Assumes vcpu_load() was already called. 2162 */ 2163 int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 2164 { 2165 struct vcpu_vmx *vmx = to_vmx(vcpu); 2166 struct vmx_uret_msr *msr; 2167 int ret = 0; 2168 u32 msr_index = msr_info->index; 2169 u64 data = msr_info->data; 2170 u32 index; 2171 2172 switch (msr_index) { 2173 case MSR_EFER: 2174 ret = kvm_set_msr_common(vcpu, msr_info); 2175 break; 2176 #ifdef CONFIG_X86_64 2177 case MSR_FS_BASE: 2178 vmx_segment_cache_clear(vmx); 2179 vmcs_writel(GUEST_FS_BASE, data); 2180 break; 2181 case MSR_GS_BASE: 2182 vmx_segment_cache_clear(vmx); 2183 vmcs_writel(GUEST_GS_BASE, data); 2184 break; 2185 case MSR_KERNEL_GS_BASE: 2186 vmx_write_guest_kernel_gs_base(vmx, data); 2187 break; 2188 case MSR_IA32_XFD: 2189 ret = kvm_set_msr_common(vcpu, msr_info); 2190 /* 2191 * Always intercepting WRMSR could incur non-negligible 2192 * overhead given xfd might be changed frequently in 2193 * guest context switch. Disable write interception 2194 * upon the first write with a non-zero value (indicating 2195 * potential usage on dynamic xfeatures). Also update 2196 * exception bitmap to trap #NM for proper virtualization 2197 * of guest xfd_err. 2198 */ 2199 if (!ret && data) { 2200 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_XFD, 2201 MSR_TYPE_RW); 2202 vcpu->arch.xfd_no_write_intercept = true; 2203 vmx_update_exception_bitmap(vcpu); 2204 } 2205 break; 2206 #endif 2207 case MSR_IA32_SYSENTER_CS: 2208 if (is_guest_mode(vcpu)) 2209 get_vmcs12(vcpu)->guest_sysenter_cs = data; 2210 vmcs_write32(GUEST_SYSENTER_CS, data); 2211 break; 2212 case MSR_IA32_SYSENTER_EIP: 2213 if (is_guest_mode(vcpu)) { 2214 data = nested_vmx_truncate_sysenter_addr(vcpu, data); 2215 get_vmcs12(vcpu)->guest_sysenter_eip = data; 2216 } 2217 vmcs_writel(GUEST_SYSENTER_EIP, data); 2218 break; 2219 case MSR_IA32_SYSENTER_ESP: 2220 if (is_guest_mode(vcpu)) { 2221 data = nested_vmx_truncate_sysenter_addr(vcpu, data); 2222 get_vmcs12(vcpu)->guest_sysenter_esp = data; 2223 } 2224 vmcs_writel(GUEST_SYSENTER_ESP, data); 2225 break; 2226 case MSR_IA32_DEBUGCTLMSR: { 2227 u64 invalid; 2228 2229 invalid = data & ~vmx_get_supported_debugctl(vcpu, msr_info->host_initiated); 2230 if (invalid & (DEBUGCTLMSR_BTF|DEBUGCTLMSR_LBR)) { 2231 kvm_pr_unimpl_wrmsr(vcpu, msr_index, data); 2232 data &= ~(DEBUGCTLMSR_BTF|DEBUGCTLMSR_LBR); 2233 invalid &= ~(DEBUGCTLMSR_BTF|DEBUGCTLMSR_LBR); 2234 } 2235 2236 if (invalid) 2237 return 1; 2238 2239 if (is_guest_mode(vcpu) && get_vmcs12(vcpu)->vm_exit_controls & 2240 VM_EXIT_SAVE_DEBUG_CONTROLS) 2241 get_vmcs12(vcpu)->guest_ia32_debugctl = data; 2242 2243 vmcs_write64(GUEST_IA32_DEBUGCTL, data); 2244 if (intel_pmu_lbr_is_enabled(vcpu) && !to_vmx(vcpu)->lbr_desc.event && 2245 (data & DEBUGCTLMSR_LBR)) 2246 intel_pmu_create_guest_lbr_event(vcpu); 2247 return 0; 2248 } 2249 case MSR_IA32_BNDCFGS: 2250 if (!kvm_mpx_supported() || 2251 (!msr_info->host_initiated && 2252 !guest_cpuid_has(vcpu, X86_FEATURE_MPX))) 2253 return 1; 2254 if (is_noncanonical_address(data & PAGE_MASK, vcpu) || 2255 (data & MSR_IA32_BNDCFGS_RSVD)) 2256 return 1; 2257 2258 if (is_guest_mode(vcpu) && 2259 ((vmx->nested.msrs.entry_ctls_high & VM_ENTRY_LOAD_BNDCFGS) || 2260 (vmx->nested.msrs.exit_ctls_high & VM_EXIT_CLEAR_BNDCFGS))) 2261 get_vmcs12(vcpu)->guest_bndcfgs = data; 2262 2263 vmcs_write64(GUEST_BNDCFGS, data); 2264 break; 2265 case MSR_IA32_UMWAIT_CONTROL: 2266 if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx)) 2267 return 1; 2268 2269 /* The reserved bit 1 and non-32 bit [63:32] should be zero */ 2270 if (data & (BIT_ULL(1) | GENMASK_ULL(63, 32))) 2271 return 1; 2272 2273 vmx->msr_ia32_umwait_control = data; 2274 break; 2275 case MSR_IA32_SPEC_CTRL: 2276 if (!msr_info->host_initiated && 2277 !guest_has_spec_ctrl_msr(vcpu)) 2278 return 1; 2279 2280 if (kvm_spec_ctrl_test_value(data)) 2281 return 1; 2282 2283 vmx->spec_ctrl = data; 2284 if (!data) 2285 break; 2286 2287 /* 2288 * For non-nested: 2289 * When it's written (to non-zero) for the first time, pass 2290 * it through. 2291 * 2292 * For nested: 2293 * The handling of the MSR bitmap for L2 guests is done in 2294 * nested_vmx_prepare_msr_bitmap. We should not touch the 2295 * vmcs02.msr_bitmap here since it gets completely overwritten 2296 * in the merging. We update the vmcs01 here for L1 as well 2297 * since it will end up touching the MSR anyway now. 2298 */ 2299 vmx_disable_intercept_for_msr(vcpu, 2300 MSR_IA32_SPEC_CTRL, 2301 MSR_TYPE_RW); 2302 break; 2303 case MSR_IA32_TSX_CTRL: 2304 if (!msr_info->host_initiated && 2305 !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR)) 2306 return 1; 2307 if (data & ~(TSX_CTRL_RTM_DISABLE | TSX_CTRL_CPUID_CLEAR)) 2308 return 1; 2309 goto find_uret_msr; 2310 case MSR_IA32_CR_PAT: 2311 ret = kvm_set_msr_common(vcpu, msr_info); 2312 if (ret) 2313 break; 2314 2315 if (is_guest_mode(vcpu) && 2316 get_vmcs12(vcpu)->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT) 2317 get_vmcs12(vcpu)->guest_ia32_pat = data; 2318 2319 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) 2320 vmcs_write64(GUEST_IA32_PAT, data); 2321 break; 2322 case MSR_IA32_MCG_EXT_CTL: 2323 if ((!msr_info->host_initiated && 2324 !(to_vmx(vcpu)->msr_ia32_feature_control & 2325 FEAT_CTL_LMCE_ENABLED)) || 2326 (data & ~MCG_EXT_CTL_LMCE_EN)) 2327 return 1; 2328 vcpu->arch.mcg_ext_ctl = data; 2329 break; 2330 case MSR_IA32_FEAT_CTL: 2331 if (!is_vmx_feature_control_msr_valid(vmx, msr_info)) 2332 return 1; 2333 2334 vmx->msr_ia32_feature_control = data; 2335 if (msr_info->host_initiated && data == 0) 2336 vmx_leave_nested(vcpu); 2337 2338 /* SGX may be enabled/disabled by guest's firmware */ 2339 vmx_write_encls_bitmap(vcpu, NULL); 2340 break; 2341 case MSR_IA32_SGXLEPUBKEYHASH0 ... MSR_IA32_SGXLEPUBKEYHASH3: 2342 /* 2343 * On real hardware, the LE hash MSRs are writable before 2344 * the firmware sets bit 0 in MSR 0x7a ("activating" SGX), 2345 * at which point SGX related bits in IA32_FEATURE_CONTROL 2346 * become writable. 2347 * 2348 * KVM does not emulate SGX activation for simplicity, so 2349 * allow writes to the LE hash MSRs if IA32_FEATURE_CONTROL 2350 * is unlocked. This is technically not architectural 2351 * behavior, but it's close enough. 2352 */ 2353 if (!msr_info->host_initiated && 2354 (!guest_cpuid_has(vcpu, X86_FEATURE_SGX_LC) || 2355 ((vmx->msr_ia32_feature_control & FEAT_CTL_LOCKED) && 2356 !(vmx->msr_ia32_feature_control & FEAT_CTL_SGX_LC_ENABLED)))) 2357 return 1; 2358 vmx->msr_ia32_sgxlepubkeyhash 2359 [msr_index - MSR_IA32_SGXLEPUBKEYHASH0] = data; 2360 break; 2361 case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR: 2362 if (!msr_info->host_initiated) 2363 return 1; /* they are read-only */ 2364 if (!guest_can_use(vcpu, X86_FEATURE_VMX)) 2365 return 1; 2366 return vmx_set_vmx_msr(vcpu, msr_index, data); 2367 case MSR_IA32_RTIT_CTL: 2368 if (!vmx_pt_mode_is_host_guest() || 2369 vmx_rtit_ctl_check(vcpu, data) || 2370 vmx->nested.vmxon) 2371 return 1; 2372 vmcs_write64(GUEST_IA32_RTIT_CTL, data); 2373 vmx->pt_desc.guest.ctl = data; 2374 pt_update_intercept_for_msr(vcpu); 2375 break; 2376 case MSR_IA32_RTIT_STATUS: 2377 if (!pt_can_write_msr(vmx)) 2378 return 1; 2379 if (data & MSR_IA32_RTIT_STATUS_MASK) 2380 return 1; 2381 vmx->pt_desc.guest.status = data; 2382 break; 2383 case MSR_IA32_RTIT_CR3_MATCH: 2384 if (!pt_can_write_msr(vmx)) 2385 return 1; 2386 if (!intel_pt_validate_cap(vmx->pt_desc.caps, 2387 PT_CAP_cr3_filtering)) 2388 return 1; 2389 vmx->pt_desc.guest.cr3_match = data; 2390 break; 2391 case MSR_IA32_RTIT_OUTPUT_BASE: 2392 if (!pt_can_write_msr(vmx)) 2393 return 1; 2394 if (!intel_pt_validate_cap(vmx->pt_desc.caps, 2395 PT_CAP_topa_output) && 2396 !intel_pt_validate_cap(vmx->pt_desc.caps, 2397 PT_CAP_single_range_output)) 2398 return 1; 2399 if (!pt_output_base_valid(vcpu, data)) 2400 return 1; 2401 vmx->pt_desc.guest.output_base = data; 2402 break; 2403 case MSR_IA32_RTIT_OUTPUT_MASK: 2404 if (!pt_can_write_msr(vmx)) 2405 return 1; 2406 if (!intel_pt_validate_cap(vmx->pt_desc.caps, 2407 PT_CAP_topa_output) && 2408 !intel_pt_validate_cap(vmx->pt_desc.caps, 2409 PT_CAP_single_range_output)) 2410 return 1; 2411 vmx->pt_desc.guest.output_mask = data; 2412 break; 2413 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: 2414 if (!pt_can_write_msr(vmx)) 2415 return 1; 2416 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A; 2417 if (index >= 2 * vmx->pt_desc.num_address_ranges) 2418 return 1; 2419 if (is_noncanonical_address(data, vcpu)) 2420 return 1; 2421 if (index % 2) 2422 vmx->pt_desc.guest.addr_b[index / 2] = data; 2423 else 2424 vmx->pt_desc.guest.addr_a[index / 2] = data; 2425 break; 2426 case MSR_IA32_PERF_CAPABILITIES: 2427 if (data && !vcpu_to_pmu(vcpu)->version) 2428 return 1; 2429 if (data & PMU_CAP_LBR_FMT) { 2430 if ((data & PMU_CAP_LBR_FMT) != 2431 (kvm_caps.supported_perf_cap & PMU_CAP_LBR_FMT)) 2432 return 1; 2433 if (!cpuid_model_is_consistent(vcpu)) 2434 return 1; 2435 } 2436 if (data & PERF_CAP_PEBS_FORMAT) { 2437 if ((data & PERF_CAP_PEBS_MASK) != 2438 (kvm_caps.supported_perf_cap & PERF_CAP_PEBS_MASK)) 2439 return 1; 2440 if (!guest_cpuid_has(vcpu, X86_FEATURE_DS)) 2441 return 1; 2442 if (!guest_cpuid_has(vcpu, X86_FEATURE_DTES64)) 2443 return 1; 2444 if (!cpuid_model_is_consistent(vcpu)) 2445 return 1; 2446 } 2447 ret = kvm_set_msr_common(vcpu, msr_info); 2448 break; 2449 2450 default: 2451 find_uret_msr: 2452 msr = vmx_find_uret_msr(vmx, msr_index); 2453 if (msr) 2454 ret = vmx_set_guest_uret_msr(vmx, msr, data); 2455 else 2456 ret = kvm_set_msr_common(vcpu, msr_info); 2457 } 2458 2459 /* FB_CLEAR may have changed, also update the FB_CLEAR_DIS behavior */ 2460 if (msr_index == MSR_IA32_ARCH_CAPABILITIES) 2461 vmx_update_fb_clear_dis(vcpu, vmx); 2462 2463 return ret; 2464 } 2465 2466 void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) 2467 { 2468 unsigned long guest_owned_bits; 2469 2470 kvm_register_mark_available(vcpu, reg); 2471 2472 switch (reg) { 2473 case VCPU_REGS_RSP: 2474 vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP); 2475 break; 2476 case VCPU_REGS_RIP: 2477 vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP); 2478 break; 2479 case VCPU_EXREG_PDPTR: 2480 if (enable_ept) 2481 ept_save_pdptrs(vcpu); 2482 break; 2483 case VCPU_EXREG_CR0: 2484 guest_owned_bits = vcpu->arch.cr0_guest_owned_bits; 2485 2486 vcpu->arch.cr0 &= ~guest_owned_bits; 2487 vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & guest_owned_bits; 2488 break; 2489 case VCPU_EXREG_CR3: 2490 /* 2491 * When intercepting CR3 loads, e.g. for shadowing paging, KVM's 2492 * CR3 is loaded into hardware, not the guest's CR3. 2493 */ 2494 if (!(exec_controls_get(to_vmx(vcpu)) & CPU_BASED_CR3_LOAD_EXITING)) 2495 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); 2496 break; 2497 case VCPU_EXREG_CR4: 2498 guest_owned_bits = vcpu->arch.cr4_guest_owned_bits; 2499 2500 vcpu->arch.cr4 &= ~guest_owned_bits; 2501 vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & guest_owned_bits; 2502 break; 2503 default: 2504 KVM_BUG_ON(1, vcpu->kvm); 2505 break; 2506 } 2507 } 2508 2509 /* 2510 * There is no X86_FEATURE for SGX yet, but anyway we need to query CPUID 2511 * directly instead of going through cpu_has(), to ensure KVM is trapping 2512 * ENCLS whenever it's supported in hardware. It does not matter whether 2513 * the host OS supports or has enabled SGX. 2514 */ 2515 static bool cpu_has_sgx(void) 2516 { 2517 return cpuid_eax(0) >= 0x12 && (cpuid_eax(0x12) & BIT(0)); 2518 } 2519 2520 /* 2521 * Some cpus support VM_{ENTRY,EXIT}_IA32_PERF_GLOBAL_CTRL but they 2522 * can't be used due to errata where VM Exit may incorrectly clear 2523 * IA32_PERF_GLOBAL_CTRL[34:32]. Work around the errata by using the 2524 * MSR load mechanism to switch IA32_PERF_GLOBAL_CTRL. 2525 */ 2526 static bool cpu_has_perf_global_ctrl_bug(void) 2527 { 2528 if (boot_cpu_data.x86 == 0x6) { 2529 switch (boot_cpu_data.x86_model) { 2530 case INTEL_FAM6_NEHALEM_EP: /* AAK155 */ 2531 case INTEL_FAM6_NEHALEM: /* AAP115 */ 2532 case INTEL_FAM6_WESTMERE: /* AAT100 */ 2533 case INTEL_FAM6_WESTMERE_EP: /* BC86,AAY89,BD102 */ 2534 case INTEL_FAM6_NEHALEM_EX: /* BA97 */ 2535 return true; 2536 default: 2537 break; 2538 } 2539 } 2540 2541 return false; 2542 } 2543 2544 static int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt, u32 msr, u32 *result) 2545 { 2546 u32 vmx_msr_low, vmx_msr_high; 2547 u32 ctl = ctl_min | ctl_opt; 2548 2549 rdmsr(msr, vmx_msr_low, vmx_msr_high); 2550 2551 ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */ 2552 ctl |= vmx_msr_low; /* bit == 1 in low word ==> must be one */ 2553 2554 /* Ensure minimum (required) set of control bits are supported. */ 2555 if (ctl_min & ~ctl) 2556 return -EIO; 2557 2558 *result = ctl; 2559 return 0; 2560 } 2561 2562 static u64 adjust_vmx_controls64(u64 ctl_opt, u32 msr) 2563 { 2564 u64 allowed; 2565 2566 rdmsrl(msr, allowed); 2567 2568 return ctl_opt & allowed; 2569 } 2570 2571 static int setup_vmcs_config(struct vmcs_config *vmcs_conf, 2572 struct vmx_capability *vmx_cap) 2573 { 2574 u32 vmx_msr_low, vmx_msr_high; 2575 u32 _pin_based_exec_control = 0; 2576 u32 _cpu_based_exec_control = 0; 2577 u32 _cpu_based_2nd_exec_control = 0; 2578 u64 _cpu_based_3rd_exec_control = 0; 2579 u32 _vmexit_control = 0; 2580 u32 _vmentry_control = 0; 2581 u64 misc_msr; 2582 int i; 2583 2584 /* 2585 * LOAD/SAVE_DEBUG_CONTROLS are absent because both are mandatory. 2586 * SAVE_IA32_PAT and SAVE_IA32_EFER are absent because KVM always 2587 * intercepts writes to PAT and EFER, i.e. never enables those controls. 2588 */ 2589 struct { 2590 u32 entry_control; 2591 u32 exit_control; 2592 } const vmcs_entry_exit_pairs[] = { 2593 { VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL, VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL }, 2594 { VM_ENTRY_LOAD_IA32_PAT, VM_EXIT_LOAD_IA32_PAT }, 2595 { VM_ENTRY_LOAD_IA32_EFER, VM_EXIT_LOAD_IA32_EFER }, 2596 { VM_ENTRY_LOAD_BNDCFGS, VM_EXIT_CLEAR_BNDCFGS }, 2597 { VM_ENTRY_LOAD_IA32_RTIT_CTL, VM_EXIT_CLEAR_IA32_RTIT_CTL }, 2598 }; 2599 2600 memset(vmcs_conf, 0, sizeof(*vmcs_conf)); 2601 2602 if (adjust_vmx_controls(KVM_REQUIRED_VMX_CPU_BASED_VM_EXEC_CONTROL, 2603 KVM_OPTIONAL_VMX_CPU_BASED_VM_EXEC_CONTROL, 2604 MSR_IA32_VMX_PROCBASED_CTLS, 2605 &_cpu_based_exec_control)) 2606 return -EIO; 2607 if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) { 2608 if (adjust_vmx_controls(KVM_REQUIRED_VMX_SECONDARY_VM_EXEC_CONTROL, 2609 KVM_OPTIONAL_VMX_SECONDARY_VM_EXEC_CONTROL, 2610 MSR_IA32_VMX_PROCBASED_CTLS2, 2611 &_cpu_based_2nd_exec_control)) 2612 return -EIO; 2613 } 2614 if (!IS_ENABLED(CONFIG_KVM_INTEL_PROVE_VE)) 2615 _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_EPT_VIOLATION_VE; 2616 2617 #ifndef CONFIG_X86_64 2618 if (!(_cpu_based_2nd_exec_control & 2619 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) 2620 _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW; 2621 #endif 2622 2623 if (!(_cpu_based_exec_control & CPU_BASED_TPR_SHADOW)) 2624 _cpu_based_2nd_exec_control &= ~( 2625 SECONDARY_EXEC_APIC_REGISTER_VIRT | 2626 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 2627 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 2628 2629 rdmsr_safe(MSR_IA32_VMX_EPT_VPID_CAP, 2630 &vmx_cap->ept, &vmx_cap->vpid); 2631 2632 if (!(_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) && 2633 vmx_cap->ept) { 2634 pr_warn_once("EPT CAP should not exist if not support " 2635 "1-setting enable EPT VM-execution control\n"); 2636 2637 if (error_on_inconsistent_vmcs_config) 2638 return -EIO; 2639 2640 vmx_cap->ept = 0; 2641 _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_EPT_VIOLATION_VE; 2642 } 2643 if (!(_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_VPID) && 2644 vmx_cap->vpid) { 2645 pr_warn_once("VPID CAP should not exist if not support " 2646 "1-setting enable VPID VM-execution control\n"); 2647 2648 if (error_on_inconsistent_vmcs_config) 2649 return -EIO; 2650 2651 vmx_cap->vpid = 0; 2652 } 2653 2654 if (!cpu_has_sgx()) 2655 _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_ENCLS_EXITING; 2656 2657 if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_TERTIARY_CONTROLS) 2658 _cpu_based_3rd_exec_control = 2659 adjust_vmx_controls64(KVM_OPTIONAL_VMX_TERTIARY_VM_EXEC_CONTROL, 2660 MSR_IA32_VMX_PROCBASED_CTLS3); 2661 2662 if (adjust_vmx_controls(KVM_REQUIRED_VMX_VM_EXIT_CONTROLS, 2663 KVM_OPTIONAL_VMX_VM_EXIT_CONTROLS, 2664 MSR_IA32_VMX_EXIT_CTLS, 2665 &_vmexit_control)) 2666 return -EIO; 2667 2668 if (adjust_vmx_controls(KVM_REQUIRED_VMX_PIN_BASED_VM_EXEC_CONTROL, 2669 KVM_OPTIONAL_VMX_PIN_BASED_VM_EXEC_CONTROL, 2670 MSR_IA32_VMX_PINBASED_CTLS, 2671 &_pin_based_exec_control)) 2672 return -EIO; 2673 2674 if (cpu_has_broken_vmx_preemption_timer()) 2675 _pin_based_exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER; 2676 if (!(_cpu_based_2nd_exec_control & 2677 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)) 2678 _pin_based_exec_control &= ~PIN_BASED_POSTED_INTR; 2679 2680 if (adjust_vmx_controls(KVM_REQUIRED_VMX_VM_ENTRY_CONTROLS, 2681 KVM_OPTIONAL_VMX_VM_ENTRY_CONTROLS, 2682 MSR_IA32_VMX_ENTRY_CTLS, 2683 &_vmentry_control)) 2684 return -EIO; 2685 2686 for (i = 0; i < ARRAY_SIZE(vmcs_entry_exit_pairs); i++) { 2687 u32 n_ctrl = vmcs_entry_exit_pairs[i].entry_control; 2688 u32 x_ctrl = vmcs_entry_exit_pairs[i].exit_control; 2689 2690 if (!(_vmentry_control & n_ctrl) == !(_vmexit_control & x_ctrl)) 2691 continue; 2692 2693 pr_warn_once("Inconsistent VM-Entry/VM-Exit pair, entry = %x, exit = %x\n", 2694 _vmentry_control & n_ctrl, _vmexit_control & x_ctrl); 2695 2696 if (error_on_inconsistent_vmcs_config) 2697 return -EIO; 2698 2699 _vmentry_control &= ~n_ctrl; 2700 _vmexit_control &= ~x_ctrl; 2701 } 2702 2703 rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high); 2704 2705 /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */ 2706 if ((vmx_msr_high & 0x1fff) > PAGE_SIZE) 2707 return -EIO; 2708 2709 #ifdef CONFIG_X86_64 2710 /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */ 2711 if (vmx_msr_high & (1u<<16)) 2712 return -EIO; 2713 #endif 2714 2715 /* Require Write-Back (WB) memory type for VMCS accesses. */ 2716 if (((vmx_msr_high >> 18) & 15) != 6) 2717 return -EIO; 2718 2719 rdmsrl(MSR_IA32_VMX_MISC, misc_msr); 2720 2721 vmcs_conf->size = vmx_msr_high & 0x1fff; 2722 vmcs_conf->basic_cap = vmx_msr_high & ~0x1fff; 2723 2724 vmcs_conf->revision_id = vmx_msr_low; 2725 2726 vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control; 2727 vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control; 2728 vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control; 2729 vmcs_conf->cpu_based_3rd_exec_ctrl = _cpu_based_3rd_exec_control; 2730 vmcs_conf->vmexit_ctrl = _vmexit_control; 2731 vmcs_conf->vmentry_ctrl = _vmentry_control; 2732 vmcs_conf->misc = misc_msr; 2733 2734 #if IS_ENABLED(CONFIG_HYPERV) 2735 if (enlightened_vmcs) 2736 evmcs_sanitize_exec_ctrls(vmcs_conf); 2737 #endif 2738 2739 return 0; 2740 } 2741 2742 static bool __kvm_is_vmx_supported(void) 2743 { 2744 int cpu = smp_processor_id(); 2745 2746 if (!(cpuid_ecx(1) & feature_bit(VMX))) { 2747 pr_err("VMX not supported by CPU %d\n", cpu); 2748 return false; 2749 } 2750 2751 if (!this_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) || 2752 !this_cpu_has(X86_FEATURE_VMX)) { 2753 pr_err("VMX not enabled (by BIOS) in MSR_IA32_FEAT_CTL on CPU %d\n", cpu); 2754 return false; 2755 } 2756 2757 return true; 2758 } 2759 2760 static bool kvm_is_vmx_supported(void) 2761 { 2762 bool supported; 2763 2764 migrate_disable(); 2765 supported = __kvm_is_vmx_supported(); 2766 migrate_enable(); 2767 2768 return supported; 2769 } 2770 2771 int vmx_check_processor_compat(void) 2772 { 2773 int cpu = raw_smp_processor_id(); 2774 struct vmcs_config vmcs_conf; 2775 struct vmx_capability vmx_cap; 2776 2777 if (!__kvm_is_vmx_supported()) 2778 return -EIO; 2779 2780 if (setup_vmcs_config(&vmcs_conf, &vmx_cap) < 0) { 2781 pr_err("Failed to setup VMCS config on CPU %d\n", cpu); 2782 return -EIO; 2783 } 2784 if (nested) 2785 nested_vmx_setup_ctls_msrs(&vmcs_conf, vmx_cap.ept); 2786 if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config))) { 2787 pr_err("Inconsistent VMCS config on CPU %d\n", cpu); 2788 return -EIO; 2789 } 2790 return 0; 2791 } 2792 2793 static int kvm_cpu_vmxon(u64 vmxon_pointer) 2794 { 2795 u64 msr; 2796 2797 cr4_set_bits(X86_CR4_VMXE); 2798 2799 asm goto("1: vmxon %[vmxon_pointer]\n\t" 2800 _ASM_EXTABLE(1b, %l[fault]) 2801 : : [vmxon_pointer] "m"(vmxon_pointer) 2802 : : fault); 2803 return 0; 2804 2805 fault: 2806 WARN_ONCE(1, "VMXON faulted, MSR_IA32_FEAT_CTL (0x3a) = 0x%llx\n", 2807 rdmsrl_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr); 2808 cr4_clear_bits(X86_CR4_VMXE); 2809 2810 return -EFAULT; 2811 } 2812 2813 int vmx_hardware_enable(void) 2814 { 2815 int cpu = raw_smp_processor_id(); 2816 u64 phys_addr = __pa(per_cpu(vmxarea, cpu)); 2817 int r; 2818 2819 if (cr4_read_shadow() & X86_CR4_VMXE) 2820 return -EBUSY; 2821 2822 /* 2823 * This can happen if we hot-added a CPU but failed to allocate 2824 * VP assist page for it. 2825 */ 2826 if (kvm_is_using_evmcs() && !hv_get_vp_assist_page(cpu)) 2827 return -EFAULT; 2828 2829 intel_pt_handle_vmx(1); 2830 2831 r = kvm_cpu_vmxon(phys_addr); 2832 if (r) { 2833 intel_pt_handle_vmx(0); 2834 return r; 2835 } 2836 2837 if (enable_ept) 2838 ept_sync_global(); 2839 2840 return 0; 2841 } 2842 2843 static void vmclear_local_loaded_vmcss(void) 2844 { 2845 int cpu = raw_smp_processor_id(); 2846 struct loaded_vmcs *v, *n; 2847 2848 list_for_each_entry_safe(v, n, &per_cpu(loaded_vmcss_on_cpu, cpu), 2849 loaded_vmcss_on_cpu_link) 2850 __loaded_vmcs_clear(v); 2851 } 2852 2853 void vmx_hardware_disable(void) 2854 { 2855 vmclear_local_loaded_vmcss(); 2856 2857 if (kvm_cpu_vmxoff()) 2858 kvm_spurious_fault(); 2859 2860 hv_reset_evmcs(); 2861 2862 intel_pt_handle_vmx(0); 2863 } 2864 2865 struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags) 2866 { 2867 int node = cpu_to_node(cpu); 2868 struct page *pages; 2869 struct vmcs *vmcs; 2870 2871 pages = __alloc_pages_node(node, flags, 0); 2872 if (!pages) 2873 return NULL; 2874 vmcs = page_address(pages); 2875 memset(vmcs, 0, vmcs_config.size); 2876 2877 /* KVM supports Enlightened VMCS v1 only */ 2878 if (kvm_is_using_evmcs()) 2879 vmcs->hdr.revision_id = KVM_EVMCS_VERSION; 2880 else 2881 vmcs->hdr.revision_id = vmcs_config.revision_id; 2882 2883 if (shadow) 2884 vmcs->hdr.shadow_vmcs = 1; 2885 return vmcs; 2886 } 2887 2888 void free_vmcs(struct vmcs *vmcs) 2889 { 2890 free_page((unsigned long)vmcs); 2891 } 2892 2893 /* 2894 * Free a VMCS, but before that VMCLEAR it on the CPU where it was last loaded 2895 */ 2896 void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs) 2897 { 2898 if (!loaded_vmcs->vmcs) 2899 return; 2900 loaded_vmcs_clear(loaded_vmcs); 2901 free_vmcs(loaded_vmcs->vmcs); 2902 loaded_vmcs->vmcs = NULL; 2903 if (loaded_vmcs->msr_bitmap) 2904 free_page((unsigned long)loaded_vmcs->msr_bitmap); 2905 WARN_ON(loaded_vmcs->shadow_vmcs != NULL); 2906 } 2907 2908 int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs) 2909 { 2910 loaded_vmcs->vmcs = alloc_vmcs(false); 2911 if (!loaded_vmcs->vmcs) 2912 return -ENOMEM; 2913 2914 vmcs_clear(loaded_vmcs->vmcs); 2915 2916 loaded_vmcs->shadow_vmcs = NULL; 2917 loaded_vmcs->hv_timer_soft_disabled = false; 2918 loaded_vmcs->cpu = -1; 2919 loaded_vmcs->launched = 0; 2920 2921 if (cpu_has_vmx_msr_bitmap()) { 2922 loaded_vmcs->msr_bitmap = (unsigned long *) 2923 __get_free_page(GFP_KERNEL_ACCOUNT); 2924 if (!loaded_vmcs->msr_bitmap) 2925 goto out_vmcs; 2926 memset(loaded_vmcs->msr_bitmap, 0xff, PAGE_SIZE); 2927 } 2928 2929 memset(&loaded_vmcs->host_state, 0, sizeof(struct vmcs_host_state)); 2930 memset(&loaded_vmcs->controls_shadow, 0, 2931 sizeof(struct vmcs_controls_shadow)); 2932 2933 return 0; 2934 2935 out_vmcs: 2936 free_loaded_vmcs(loaded_vmcs); 2937 return -ENOMEM; 2938 } 2939 2940 static void free_kvm_area(void) 2941 { 2942 int cpu; 2943 2944 for_each_possible_cpu(cpu) { 2945 free_vmcs(per_cpu(vmxarea, cpu)); 2946 per_cpu(vmxarea, cpu) = NULL; 2947 } 2948 } 2949 2950 static __init int alloc_kvm_area(void) 2951 { 2952 int cpu; 2953 2954 for_each_possible_cpu(cpu) { 2955 struct vmcs *vmcs; 2956 2957 vmcs = alloc_vmcs_cpu(false, cpu, GFP_KERNEL); 2958 if (!vmcs) { 2959 free_kvm_area(); 2960 return -ENOMEM; 2961 } 2962 2963 /* 2964 * When eVMCS is enabled, alloc_vmcs_cpu() sets 2965 * vmcs->revision_id to KVM_EVMCS_VERSION instead of 2966 * revision_id reported by MSR_IA32_VMX_BASIC. 2967 * 2968 * However, even though not explicitly documented by 2969 * TLFS, VMXArea passed as VMXON argument should 2970 * still be marked with revision_id reported by 2971 * physical CPU. 2972 */ 2973 if (kvm_is_using_evmcs()) 2974 vmcs->hdr.revision_id = vmcs_config.revision_id; 2975 2976 per_cpu(vmxarea, cpu) = vmcs; 2977 } 2978 return 0; 2979 } 2980 2981 static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg, 2982 struct kvm_segment *save) 2983 { 2984 if (!emulate_invalid_guest_state) { 2985 /* 2986 * CS and SS RPL should be equal during guest entry according 2987 * to VMX spec, but in reality it is not always so. Since vcpu 2988 * is in the middle of the transition from real mode to 2989 * protected mode it is safe to assume that RPL 0 is a good 2990 * default value. 2991 */ 2992 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS) 2993 save->selector &= ~SEGMENT_RPL_MASK; 2994 save->dpl = save->selector & SEGMENT_RPL_MASK; 2995 save->s = 1; 2996 } 2997 __vmx_set_segment(vcpu, save, seg); 2998 } 2999 3000 static void enter_pmode(struct kvm_vcpu *vcpu) 3001 { 3002 unsigned long flags; 3003 struct vcpu_vmx *vmx = to_vmx(vcpu); 3004 3005 /* 3006 * Update real mode segment cache. It may be not up-to-date if segment 3007 * register was written while vcpu was in a guest mode. 3008 */ 3009 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES); 3010 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS); 3011 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS); 3012 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS); 3013 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS); 3014 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS); 3015 3016 vmx->rmode.vm86_active = 0; 3017 3018 __vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR); 3019 3020 flags = vmcs_readl(GUEST_RFLAGS); 3021 flags &= RMODE_GUEST_OWNED_EFLAGS_BITS; 3022 flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS; 3023 vmcs_writel(GUEST_RFLAGS, flags); 3024 3025 vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) | 3026 (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME)); 3027 3028 vmx_update_exception_bitmap(vcpu); 3029 3030 fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]); 3031 fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]); 3032 fix_pmode_seg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]); 3033 fix_pmode_seg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]); 3034 fix_pmode_seg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]); 3035 fix_pmode_seg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]); 3036 } 3037 3038 static void fix_rmode_seg(int seg, struct kvm_segment *save) 3039 { 3040 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 3041 struct kvm_segment var = *save; 3042 3043 var.dpl = 0x3; 3044 if (seg == VCPU_SREG_CS) 3045 var.type = 0x3; 3046 3047 if (!emulate_invalid_guest_state) { 3048 var.selector = var.base >> 4; 3049 var.base = var.base & 0xffff0; 3050 var.limit = 0xffff; 3051 var.g = 0; 3052 var.db = 0; 3053 var.present = 1; 3054 var.s = 1; 3055 var.l = 0; 3056 var.unusable = 0; 3057 var.type = 0x3; 3058 var.avl = 0; 3059 if (save->base & 0xf) 3060 pr_warn_once("segment base is not paragraph aligned " 3061 "when entering protected mode (seg=%d)", seg); 3062 } 3063 3064 vmcs_write16(sf->selector, var.selector); 3065 vmcs_writel(sf->base, var.base); 3066 vmcs_write32(sf->limit, var.limit); 3067 vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(&var)); 3068 } 3069 3070 static void enter_rmode(struct kvm_vcpu *vcpu) 3071 { 3072 unsigned long flags; 3073 struct vcpu_vmx *vmx = to_vmx(vcpu); 3074 struct kvm_vmx *kvm_vmx = to_kvm_vmx(vcpu->kvm); 3075 3076 /* 3077 * KVM should never use VM86 to virtualize Real Mode when L2 is active, 3078 * as using VM86 is unnecessary if unrestricted guest is enabled, and 3079 * if unrestricted guest is disabled, VM-Enter (from L1) with CR0.PG=0 3080 * should VM-Fail and KVM should reject userspace attempts to stuff 3081 * CR0.PG=0 when L2 is active. 3082 */ 3083 WARN_ON_ONCE(is_guest_mode(vcpu)); 3084 3085 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR); 3086 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES); 3087 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS); 3088 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS); 3089 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS); 3090 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS); 3091 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS); 3092 3093 vmx->rmode.vm86_active = 1; 3094 3095 vmx_segment_cache_clear(vmx); 3096 3097 vmcs_writel(GUEST_TR_BASE, kvm_vmx->tss_addr); 3098 vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1); 3099 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); 3100 3101 flags = vmcs_readl(GUEST_RFLAGS); 3102 vmx->rmode.save_rflags = flags; 3103 3104 flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM; 3105 3106 vmcs_writel(GUEST_RFLAGS, flags); 3107 vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME); 3108 vmx_update_exception_bitmap(vcpu); 3109 3110 fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]); 3111 fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]); 3112 fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]); 3113 fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]); 3114 fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]); 3115 fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]); 3116 } 3117 3118 int vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer) 3119 { 3120 struct vcpu_vmx *vmx = to_vmx(vcpu); 3121 3122 /* Nothing to do if hardware doesn't support EFER. */ 3123 if (!vmx_find_uret_msr(vmx, MSR_EFER)) 3124 return 0; 3125 3126 vcpu->arch.efer = efer; 3127 #ifdef CONFIG_X86_64 3128 if (efer & EFER_LMA) 3129 vm_entry_controls_setbit(vmx, VM_ENTRY_IA32E_MODE); 3130 else 3131 vm_entry_controls_clearbit(vmx, VM_ENTRY_IA32E_MODE); 3132 #else 3133 if (KVM_BUG_ON(efer & EFER_LMA, vcpu->kvm)) 3134 return 1; 3135 #endif 3136 3137 vmx_setup_uret_msrs(vmx); 3138 return 0; 3139 } 3140 3141 #ifdef CONFIG_X86_64 3142 3143 static void enter_lmode(struct kvm_vcpu *vcpu) 3144 { 3145 u32 guest_tr_ar; 3146 3147 vmx_segment_cache_clear(to_vmx(vcpu)); 3148 3149 guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES); 3150 if ((guest_tr_ar & VMX_AR_TYPE_MASK) != VMX_AR_TYPE_BUSY_64_TSS) { 3151 pr_debug_ratelimited("%s: tss fixup for long mode. \n", 3152 __func__); 3153 vmcs_write32(GUEST_TR_AR_BYTES, 3154 (guest_tr_ar & ~VMX_AR_TYPE_MASK) 3155 | VMX_AR_TYPE_BUSY_64_TSS); 3156 } 3157 vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA); 3158 } 3159 3160 static void exit_lmode(struct kvm_vcpu *vcpu) 3161 { 3162 vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA); 3163 } 3164 3165 #endif 3166 3167 void vmx_flush_tlb_all(struct kvm_vcpu *vcpu) 3168 { 3169 struct vcpu_vmx *vmx = to_vmx(vcpu); 3170 3171 /* 3172 * INVEPT must be issued when EPT is enabled, irrespective of VPID, as 3173 * the CPU is not required to invalidate guest-physical mappings on 3174 * VM-Entry, even if VPID is disabled. Guest-physical mappings are 3175 * associated with the root EPT structure and not any particular VPID 3176 * (INVVPID also isn't required to invalidate guest-physical mappings). 3177 */ 3178 if (enable_ept) { 3179 ept_sync_global(); 3180 } else if (enable_vpid) { 3181 if (cpu_has_vmx_invvpid_global()) { 3182 vpid_sync_vcpu_global(); 3183 } else { 3184 vpid_sync_vcpu_single(vmx->vpid); 3185 vpid_sync_vcpu_single(vmx->nested.vpid02); 3186 } 3187 } 3188 } 3189 3190 static inline int vmx_get_current_vpid(struct kvm_vcpu *vcpu) 3191 { 3192 if (is_guest_mode(vcpu)) 3193 return nested_get_vpid02(vcpu); 3194 return to_vmx(vcpu)->vpid; 3195 } 3196 3197 void vmx_flush_tlb_current(struct kvm_vcpu *vcpu) 3198 { 3199 struct kvm_mmu *mmu = vcpu->arch.mmu; 3200 u64 root_hpa = mmu->root.hpa; 3201 3202 /* No flush required if the current context is invalid. */ 3203 if (!VALID_PAGE(root_hpa)) 3204 return; 3205 3206 if (enable_ept) 3207 ept_sync_context(construct_eptp(vcpu, root_hpa, 3208 mmu->root_role.level)); 3209 else 3210 vpid_sync_context(vmx_get_current_vpid(vcpu)); 3211 } 3212 3213 void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) 3214 { 3215 /* 3216 * vpid_sync_vcpu_addr() is a nop if vpid==0, see the comment in 3217 * vmx_flush_tlb_guest() for an explanation of why this is ok. 3218 */ 3219 vpid_sync_vcpu_addr(vmx_get_current_vpid(vcpu), addr); 3220 } 3221 3222 void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu) 3223 { 3224 /* 3225 * vpid_sync_context() is a nop if vpid==0, e.g. if enable_vpid==0 or a 3226 * vpid couldn't be allocated for this vCPU. VM-Enter and VM-Exit are 3227 * required to flush GVA->{G,H}PA mappings from the TLB if vpid is 3228 * disabled (VM-Enter with vpid enabled and vpid==0 is disallowed), 3229 * i.e. no explicit INVVPID is necessary. 3230 */ 3231 vpid_sync_context(vmx_get_current_vpid(vcpu)); 3232 } 3233 3234 void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu) 3235 { 3236 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 3237 3238 if (!kvm_register_is_dirty(vcpu, VCPU_EXREG_PDPTR)) 3239 return; 3240 3241 if (is_pae_paging(vcpu)) { 3242 vmcs_write64(GUEST_PDPTR0, mmu->pdptrs[0]); 3243 vmcs_write64(GUEST_PDPTR1, mmu->pdptrs[1]); 3244 vmcs_write64(GUEST_PDPTR2, mmu->pdptrs[2]); 3245 vmcs_write64(GUEST_PDPTR3, mmu->pdptrs[3]); 3246 } 3247 } 3248 3249 void ept_save_pdptrs(struct kvm_vcpu *vcpu) 3250 { 3251 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 3252 3253 if (WARN_ON_ONCE(!is_pae_paging(vcpu))) 3254 return; 3255 3256 mmu->pdptrs[0] = vmcs_read64(GUEST_PDPTR0); 3257 mmu->pdptrs[1] = vmcs_read64(GUEST_PDPTR1); 3258 mmu->pdptrs[2] = vmcs_read64(GUEST_PDPTR2); 3259 mmu->pdptrs[3] = vmcs_read64(GUEST_PDPTR3); 3260 3261 kvm_register_mark_available(vcpu, VCPU_EXREG_PDPTR); 3262 } 3263 3264 #define CR3_EXITING_BITS (CPU_BASED_CR3_LOAD_EXITING | \ 3265 CPU_BASED_CR3_STORE_EXITING) 3266 3267 bool vmx_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 3268 { 3269 if (is_guest_mode(vcpu)) 3270 return nested_guest_cr0_valid(vcpu, cr0); 3271 3272 if (to_vmx(vcpu)->nested.vmxon) 3273 return nested_host_cr0_valid(vcpu, cr0); 3274 3275 return true; 3276 } 3277 3278 void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 3279 { 3280 struct vcpu_vmx *vmx = to_vmx(vcpu); 3281 unsigned long hw_cr0, old_cr0_pg; 3282 u32 tmp; 3283 3284 old_cr0_pg = kvm_read_cr0_bits(vcpu, X86_CR0_PG); 3285 3286 hw_cr0 = (cr0 & ~KVM_VM_CR0_ALWAYS_OFF); 3287 if (enable_unrestricted_guest) 3288 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST; 3289 else { 3290 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON; 3291 if (!enable_ept) 3292 hw_cr0 |= X86_CR0_WP; 3293 3294 if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE)) 3295 enter_pmode(vcpu); 3296 3297 if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE)) 3298 enter_rmode(vcpu); 3299 } 3300 3301 vmcs_writel(CR0_READ_SHADOW, cr0); 3302 vmcs_writel(GUEST_CR0, hw_cr0); 3303 vcpu->arch.cr0 = cr0; 3304 kvm_register_mark_available(vcpu, VCPU_EXREG_CR0); 3305 3306 #ifdef CONFIG_X86_64 3307 if (vcpu->arch.efer & EFER_LME) { 3308 if (!old_cr0_pg && (cr0 & X86_CR0_PG)) 3309 enter_lmode(vcpu); 3310 else if (old_cr0_pg && !(cr0 & X86_CR0_PG)) 3311 exit_lmode(vcpu); 3312 } 3313 #endif 3314 3315 if (enable_ept && !enable_unrestricted_guest) { 3316 /* 3317 * Ensure KVM has an up-to-date snapshot of the guest's CR3. If 3318 * the below code _enables_ CR3 exiting, vmx_cache_reg() will 3319 * (correctly) stop reading vmcs.GUEST_CR3 because it thinks 3320 * KVM's CR3 is installed. 3321 */ 3322 if (!kvm_register_is_available(vcpu, VCPU_EXREG_CR3)) 3323 vmx_cache_reg(vcpu, VCPU_EXREG_CR3); 3324 3325 /* 3326 * When running with EPT but not unrestricted guest, KVM must 3327 * intercept CR3 accesses when paging is _disabled_. This is 3328 * necessary because restricted guests can't actually run with 3329 * paging disabled, and so KVM stuffs its own CR3 in order to 3330 * run the guest when identity mapped page tables. 3331 * 3332 * Do _NOT_ check the old CR0.PG, e.g. to optimize away the 3333 * update, it may be stale with respect to CR3 interception, 3334 * e.g. after nested VM-Enter. 3335 * 3336 * Lastly, honor L1's desires, i.e. intercept CR3 loads and/or 3337 * stores to forward them to L1, even if KVM does not need to 3338 * intercept them to preserve its identity mapped page tables. 3339 */ 3340 if (!(cr0 & X86_CR0_PG)) { 3341 exec_controls_setbit(vmx, CR3_EXITING_BITS); 3342 } else if (!is_guest_mode(vcpu)) { 3343 exec_controls_clearbit(vmx, CR3_EXITING_BITS); 3344 } else { 3345 tmp = exec_controls_get(vmx); 3346 tmp &= ~CR3_EXITING_BITS; 3347 tmp |= get_vmcs12(vcpu)->cpu_based_vm_exec_control & CR3_EXITING_BITS; 3348 exec_controls_set(vmx, tmp); 3349 } 3350 3351 /* Note, vmx_set_cr4() consumes the new vcpu->arch.cr0. */ 3352 if ((old_cr0_pg ^ cr0) & X86_CR0_PG) 3353 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu)); 3354 3355 /* 3356 * When !CR0_PG -> CR0_PG, vcpu->arch.cr3 becomes active, but 3357 * GUEST_CR3 is still vmx->ept_identity_map_addr if EPT + !URG. 3358 */ 3359 if (!(old_cr0_pg & X86_CR0_PG) && (cr0 & X86_CR0_PG)) 3360 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); 3361 } 3362 3363 /* depends on vcpu->arch.cr0 to be set to a new value */ 3364 vmx->emulation_required = vmx_emulation_required(vcpu); 3365 } 3366 3367 static int vmx_get_max_ept_level(void) 3368 { 3369 if (cpu_has_vmx_ept_5levels()) 3370 return 5; 3371 return 4; 3372 } 3373 3374 u64 construct_eptp(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level) 3375 { 3376 u64 eptp = VMX_EPTP_MT_WB; 3377 3378 eptp |= (root_level == 5) ? VMX_EPTP_PWL_5 : VMX_EPTP_PWL_4; 3379 3380 if (enable_ept_ad_bits && 3381 (!is_guest_mode(vcpu) || nested_ept_ad_enabled(vcpu))) 3382 eptp |= VMX_EPTP_AD_ENABLE_BIT; 3383 eptp |= root_hpa; 3384 3385 return eptp; 3386 } 3387 3388 void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level) 3389 { 3390 struct kvm *kvm = vcpu->kvm; 3391 bool update_guest_cr3 = true; 3392 unsigned long guest_cr3; 3393 u64 eptp; 3394 3395 if (enable_ept) { 3396 eptp = construct_eptp(vcpu, root_hpa, root_level); 3397 vmcs_write64(EPT_POINTER, eptp); 3398 3399 hv_track_root_tdp(vcpu, root_hpa); 3400 3401 if (!enable_unrestricted_guest && !is_paging(vcpu)) 3402 guest_cr3 = to_kvm_vmx(kvm)->ept_identity_map_addr; 3403 else if (kvm_register_is_dirty(vcpu, VCPU_EXREG_CR3)) 3404 guest_cr3 = vcpu->arch.cr3; 3405 else /* vmcs.GUEST_CR3 is already up-to-date. */ 3406 update_guest_cr3 = false; 3407 vmx_ept_load_pdptrs(vcpu); 3408 } else { 3409 guest_cr3 = root_hpa | kvm_get_active_pcid(vcpu) | 3410 kvm_get_active_cr3_lam_bits(vcpu); 3411 } 3412 3413 if (update_guest_cr3) 3414 vmcs_writel(GUEST_CR3, guest_cr3); 3415 } 3416 3417 bool vmx_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 3418 { 3419 /* 3420 * We operate under the default treatment of SMM, so VMX cannot be 3421 * enabled under SMM. Note, whether or not VMXE is allowed at all, 3422 * i.e. is a reserved bit, is handled by common x86 code. 3423 */ 3424 if ((cr4 & X86_CR4_VMXE) && is_smm(vcpu)) 3425 return false; 3426 3427 if (to_vmx(vcpu)->nested.vmxon && !nested_cr4_valid(vcpu, cr4)) 3428 return false; 3429 3430 return true; 3431 } 3432 3433 void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 3434 { 3435 unsigned long old_cr4 = kvm_read_cr4(vcpu); 3436 struct vcpu_vmx *vmx = to_vmx(vcpu); 3437 unsigned long hw_cr4; 3438 3439 /* 3440 * Pass through host's Machine Check Enable value to hw_cr4, which 3441 * is in force while we are in guest mode. Do not let guests control 3442 * this bit, even if host CR4.MCE == 0. 3443 */ 3444 hw_cr4 = (cr4_read_shadow() & X86_CR4_MCE) | (cr4 & ~X86_CR4_MCE); 3445 if (enable_unrestricted_guest) 3446 hw_cr4 |= KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST; 3447 else if (vmx->rmode.vm86_active) 3448 hw_cr4 |= KVM_RMODE_VM_CR4_ALWAYS_ON; 3449 else 3450 hw_cr4 |= KVM_PMODE_VM_CR4_ALWAYS_ON; 3451 3452 if (vmx_umip_emulated()) { 3453 if (cr4 & X86_CR4_UMIP) { 3454 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_DESC); 3455 hw_cr4 &= ~X86_CR4_UMIP; 3456 } else if (!is_guest_mode(vcpu) || 3457 !nested_cpu_has2(get_vmcs12(vcpu), SECONDARY_EXEC_DESC)) { 3458 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_DESC); 3459 } 3460 } 3461 3462 vcpu->arch.cr4 = cr4; 3463 kvm_register_mark_available(vcpu, VCPU_EXREG_CR4); 3464 3465 if (!enable_unrestricted_guest) { 3466 if (enable_ept) { 3467 if (!is_paging(vcpu)) { 3468 hw_cr4 &= ~X86_CR4_PAE; 3469 hw_cr4 |= X86_CR4_PSE; 3470 } else if (!(cr4 & X86_CR4_PAE)) { 3471 hw_cr4 &= ~X86_CR4_PAE; 3472 } 3473 } 3474 3475 /* 3476 * SMEP/SMAP/PKU is disabled if CPU is in non-paging mode in 3477 * hardware. To emulate this behavior, SMEP/SMAP/PKU needs 3478 * to be manually disabled when guest switches to non-paging 3479 * mode. 3480 * 3481 * If !enable_unrestricted_guest, the CPU is always running 3482 * with CR0.PG=1 and CR4 needs to be modified. 3483 * If enable_unrestricted_guest, the CPU automatically 3484 * disables SMEP/SMAP/PKU when the guest sets CR0.PG=0. 3485 */ 3486 if (!is_paging(vcpu)) 3487 hw_cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE); 3488 } 3489 3490 vmcs_writel(CR4_READ_SHADOW, cr4); 3491 vmcs_writel(GUEST_CR4, hw_cr4); 3492 3493 if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE)) 3494 kvm_update_cpuid_runtime(vcpu); 3495 } 3496 3497 void vmx_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) 3498 { 3499 struct vcpu_vmx *vmx = to_vmx(vcpu); 3500 u32 ar; 3501 3502 if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) { 3503 *var = vmx->rmode.segs[seg]; 3504 if (seg == VCPU_SREG_TR 3505 || var->selector == vmx_read_guest_seg_selector(vmx, seg)) 3506 return; 3507 var->base = vmx_read_guest_seg_base(vmx, seg); 3508 var->selector = vmx_read_guest_seg_selector(vmx, seg); 3509 return; 3510 } 3511 var->base = vmx_read_guest_seg_base(vmx, seg); 3512 var->limit = vmx_read_guest_seg_limit(vmx, seg); 3513 var->selector = vmx_read_guest_seg_selector(vmx, seg); 3514 ar = vmx_read_guest_seg_ar(vmx, seg); 3515 var->unusable = (ar >> 16) & 1; 3516 var->type = ar & 15; 3517 var->s = (ar >> 4) & 1; 3518 var->dpl = (ar >> 5) & 3; 3519 /* 3520 * Some userspaces do not preserve unusable property. Since usable 3521 * segment has to be present according to VMX spec we can use present 3522 * property to amend userspace bug by making unusable segment always 3523 * nonpresent. vmx_segment_access_rights() already marks nonpresent 3524 * segment as unusable. 3525 */ 3526 var->present = !var->unusable; 3527 var->avl = (ar >> 12) & 1; 3528 var->l = (ar >> 13) & 1; 3529 var->db = (ar >> 14) & 1; 3530 var->g = (ar >> 15) & 1; 3531 } 3532 3533 u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg) 3534 { 3535 struct kvm_segment s; 3536 3537 if (to_vmx(vcpu)->rmode.vm86_active) { 3538 vmx_get_segment(vcpu, &s, seg); 3539 return s.base; 3540 } 3541 return vmx_read_guest_seg_base(to_vmx(vcpu), seg); 3542 } 3543 3544 int vmx_get_cpl(struct kvm_vcpu *vcpu) 3545 { 3546 struct vcpu_vmx *vmx = to_vmx(vcpu); 3547 3548 if (unlikely(vmx->rmode.vm86_active)) 3549 return 0; 3550 else { 3551 int ar = vmx_read_guest_seg_ar(vmx, VCPU_SREG_SS); 3552 return VMX_AR_DPL(ar); 3553 } 3554 } 3555 3556 static u32 vmx_segment_access_rights(struct kvm_segment *var) 3557 { 3558 u32 ar; 3559 3560 ar = var->type & 15; 3561 ar |= (var->s & 1) << 4; 3562 ar |= (var->dpl & 3) << 5; 3563 ar |= (var->present & 1) << 7; 3564 ar |= (var->avl & 1) << 12; 3565 ar |= (var->l & 1) << 13; 3566 ar |= (var->db & 1) << 14; 3567 ar |= (var->g & 1) << 15; 3568 ar |= (var->unusable || !var->present) << 16; 3569 3570 return ar; 3571 } 3572 3573 void __vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) 3574 { 3575 struct vcpu_vmx *vmx = to_vmx(vcpu); 3576 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 3577 3578 vmx_segment_cache_clear(vmx); 3579 3580 if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) { 3581 vmx->rmode.segs[seg] = *var; 3582 if (seg == VCPU_SREG_TR) 3583 vmcs_write16(sf->selector, var->selector); 3584 else if (var->s) 3585 fix_rmode_seg(seg, &vmx->rmode.segs[seg]); 3586 return; 3587 } 3588 3589 vmcs_writel(sf->base, var->base); 3590 vmcs_write32(sf->limit, var->limit); 3591 vmcs_write16(sf->selector, var->selector); 3592 3593 /* 3594 * Fix the "Accessed" bit in AR field of segment registers for older 3595 * qemu binaries. 3596 * IA32 arch specifies that at the time of processor reset the 3597 * "Accessed" bit in the AR field of segment registers is 1. And qemu 3598 * is setting it to 0 in the userland code. This causes invalid guest 3599 * state vmexit when "unrestricted guest" mode is turned on. 3600 * Fix for this setup issue in cpu_reset is being pushed in the qemu 3601 * tree. Newer qemu binaries with that qemu fix would not need this 3602 * kvm hack. 3603 */ 3604 if (is_unrestricted_guest(vcpu) && (seg != VCPU_SREG_LDTR)) 3605 var->type |= 0x1; /* Accessed */ 3606 3607 vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(var)); 3608 } 3609 3610 void vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) 3611 { 3612 __vmx_set_segment(vcpu, var, seg); 3613 3614 to_vmx(vcpu)->emulation_required = vmx_emulation_required(vcpu); 3615 } 3616 3617 void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) 3618 { 3619 u32 ar = vmx_read_guest_seg_ar(to_vmx(vcpu), VCPU_SREG_CS); 3620 3621 *db = (ar >> 14) & 1; 3622 *l = (ar >> 13) & 1; 3623 } 3624 3625 void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3626 { 3627 dt->size = vmcs_read32(GUEST_IDTR_LIMIT); 3628 dt->address = vmcs_readl(GUEST_IDTR_BASE); 3629 } 3630 3631 void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3632 { 3633 vmcs_write32(GUEST_IDTR_LIMIT, dt->size); 3634 vmcs_writel(GUEST_IDTR_BASE, dt->address); 3635 } 3636 3637 void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3638 { 3639 dt->size = vmcs_read32(GUEST_GDTR_LIMIT); 3640 dt->address = vmcs_readl(GUEST_GDTR_BASE); 3641 } 3642 3643 void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3644 { 3645 vmcs_write32(GUEST_GDTR_LIMIT, dt->size); 3646 vmcs_writel(GUEST_GDTR_BASE, dt->address); 3647 } 3648 3649 static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg) 3650 { 3651 struct kvm_segment var; 3652 u32 ar; 3653 3654 vmx_get_segment(vcpu, &var, seg); 3655 var.dpl = 0x3; 3656 if (seg == VCPU_SREG_CS) 3657 var.type = 0x3; 3658 ar = vmx_segment_access_rights(&var); 3659 3660 if (var.base != (var.selector << 4)) 3661 return false; 3662 if (var.limit != 0xffff) 3663 return false; 3664 if (ar != 0xf3) 3665 return false; 3666 3667 return true; 3668 } 3669 3670 static bool code_segment_valid(struct kvm_vcpu *vcpu) 3671 { 3672 struct kvm_segment cs; 3673 unsigned int cs_rpl; 3674 3675 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS); 3676 cs_rpl = cs.selector & SEGMENT_RPL_MASK; 3677 3678 if (cs.unusable) 3679 return false; 3680 if (~cs.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_ACCESSES_MASK)) 3681 return false; 3682 if (!cs.s) 3683 return false; 3684 if (cs.type & VMX_AR_TYPE_WRITEABLE_MASK) { 3685 if (cs.dpl > cs_rpl) 3686 return false; 3687 } else { 3688 if (cs.dpl != cs_rpl) 3689 return false; 3690 } 3691 if (!cs.present) 3692 return false; 3693 3694 /* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */ 3695 return true; 3696 } 3697 3698 static bool stack_segment_valid(struct kvm_vcpu *vcpu) 3699 { 3700 struct kvm_segment ss; 3701 unsigned int ss_rpl; 3702 3703 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS); 3704 ss_rpl = ss.selector & SEGMENT_RPL_MASK; 3705 3706 if (ss.unusable) 3707 return true; 3708 if (ss.type != 3 && ss.type != 7) 3709 return false; 3710 if (!ss.s) 3711 return false; 3712 if (ss.dpl != ss_rpl) /* DPL != RPL */ 3713 return false; 3714 if (!ss.present) 3715 return false; 3716 3717 return true; 3718 } 3719 3720 static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg) 3721 { 3722 struct kvm_segment var; 3723 unsigned int rpl; 3724 3725 vmx_get_segment(vcpu, &var, seg); 3726 rpl = var.selector & SEGMENT_RPL_MASK; 3727 3728 if (var.unusable) 3729 return true; 3730 if (!var.s) 3731 return false; 3732 if (!var.present) 3733 return false; 3734 if (~var.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_WRITEABLE_MASK)) { 3735 if (var.dpl < rpl) /* DPL < RPL */ 3736 return false; 3737 } 3738 3739 /* TODO: Add other members to kvm_segment_field to allow checking for other access 3740 * rights flags 3741 */ 3742 return true; 3743 } 3744 3745 static bool tr_valid(struct kvm_vcpu *vcpu) 3746 { 3747 struct kvm_segment tr; 3748 3749 vmx_get_segment(vcpu, &tr, VCPU_SREG_TR); 3750 3751 if (tr.unusable) 3752 return false; 3753 if (tr.selector & SEGMENT_TI_MASK) /* TI = 1 */ 3754 return false; 3755 if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */ 3756 return false; 3757 if (!tr.present) 3758 return false; 3759 3760 return true; 3761 } 3762 3763 static bool ldtr_valid(struct kvm_vcpu *vcpu) 3764 { 3765 struct kvm_segment ldtr; 3766 3767 vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR); 3768 3769 if (ldtr.unusable) 3770 return true; 3771 if (ldtr.selector & SEGMENT_TI_MASK) /* TI = 1 */ 3772 return false; 3773 if (ldtr.type != 2) 3774 return false; 3775 if (!ldtr.present) 3776 return false; 3777 3778 return true; 3779 } 3780 3781 static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu) 3782 { 3783 struct kvm_segment cs, ss; 3784 3785 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS); 3786 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS); 3787 3788 return ((cs.selector & SEGMENT_RPL_MASK) == 3789 (ss.selector & SEGMENT_RPL_MASK)); 3790 } 3791 3792 /* 3793 * Check if guest state is valid. Returns true if valid, false if 3794 * not. 3795 * We assume that registers are always usable 3796 */ 3797 bool __vmx_guest_state_valid(struct kvm_vcpu *vcpu) 3798 { 3799 /* real mode guest state checks */ 3800 if (!is_protmode(vcpu) || (vmx_get_rflags(vcpu) & X86_EFLAGS_VM)) { 3801 if (!rmode_segment_valid(vcpu, VCPU_SREG_CS)) 3802 return false; 3803 if (!rmode_segment_valid(vcpu, VCPU_SREG_SS)) 3804 return false; 3805 if (!rmode_segment_valid(vcpu, VCPU_SREG_DS)) 3806 return false; 3807 if (!rmode_segment_valid(vcpu, VCPU_SREG_ES)) 3808 return false; 3809 if (!rmode_segment_valid(vcpu, VCPU_SREG_FS)) 3810 return false; 3811 if (!rmode_segment_valid(vcpu, VCPU_SREG_GS)) 3812 return false; 3813 } else { 3814 /* protected mode guest state checks */ 3815 if (!cs_ss_rpl_check(vcpu)) 3816 return false; 3817 if (!code_segment_valid(vcpu)) 3818 return false; 3819 if (!stack_segment_valid(vcpu)) 3820 return false; 3821 if (!data_segment_valid(vcpu, VCPU_SREG_DS)) 3822 return false; 3823 if (!data_segment_valid(vcpu, VCPU_SREG_ES)) 3824 return false; 3825 if (!data_segment_valid(vcpu, VCPU_SREG_FS)) 3826 return false; 3827 if (!data_segment_valid(vcpu, VCPU_SREG_GS)) 3828 return false; 3829 if (!tr_valid(vcpu)) 3830 return false; 3831 if (!ldtr_valid(vcpu)) 3832 return false; 3833 } 3834 /* TODO: 3835 * - Add checks on RIP 3836 * - Add checks on RFLAGS 3837 */ 3838 3839 return true; 3840 } 3841 3842 static int init_rmode_tss(struct kvm *kvm, void __user *ua) 3843 { 3844 const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0))); 3845 u16 data; 3846 int i; 3847 3848 for (i = 0; i < 3; i++) { 3849 if (__copy_to_user(ua + PAGE_SIZE * i, zero_page, PAGE_SIZE)) 3850 return -EFAULT; 3851 } 3852 3853 data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE; 3854 if (__copy_to_user(ua + TSS_IOPB_BASE_OFFSET, &data, sizeof(u16))) 3855 return -EFAULT; 3856 3857 data = ~0; 3858 if (__copy_to_user(ua + RMODE_TSS_SIZE - 1, &data, sizeof(u8))) 3859 return -EFAULT; 3860 3861 return 0; 3862 } 3863 3864 static int init_rmode_identity_map(struct kvm *kvm) 3865 { 3866 struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); 3867 int i, r = 0; 3868 void __user *uaddr; 3869 u32 tmp; 3870 3871 /* Protect kvm_vmx->ept_identity_pagetable_done. */ 3872 mutex_lock(&kvm->slots_lock); 3873 3874 if (likely(kvm_vmx->ept_identity_pagetable_done)) 3875 goto out; 3876 3877 if (!kvm_vmx->ept_identity_map_addr) 3878 kvm_vmx->ept_identity_map_addr = VMX_EPT_IDENTITY_PAGETABLE_ADDR; 3879 3880 uaddr = __x86_set_memory_region(kvm, 3881 IDENTITY_PAGETABLE_PRIVATE_MEMSLOT, 3882 kvm_vmx->ept_identity_map_addr, 3883 PAGE_SIZE); 3884 if (IS_ERR(uaddr)) { 3885 r = PTR_ERR(uaddr); 3886 goto out; 3887 } 3888 3889 /* Set up identity-mapping pagetable for EPT in real mode */ 3890 for (i = 0; i < (PAGE_SIZE / sizeof(tmp)); i++) { 3891 tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | 3892 _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE); 3893 if (__copy_to_user(uaddr + i * sizeof(tmp), &tmp, sizeof(tmp))) { 3894 r = -EFAULT; 3895 goto out; 3896 } 3897 } 3898 kvm_vmx->ept_identity_pagetable_done = true; 3899 3900 out: 3901 mutex_unlock(&kvm->slots_lock); 3902 return r; 3903 } 3904 3905 static void seg_setup(int seg) 3906 { 3907 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 3908 unsigned int ar; 3909 3910 vmcs_write16(sf->selector, 0); 3911 vmcs_writel(sf->base, 0); 3912 vmcs_write32(sf->limit, 0xffff); 3913 ar = 0x93; 3914 if (seg == VCPU_SREG_CS) 3915 ar |= 0x08; /* code segment */ 3916 3917 vmcs_write32(sf->ar_bytes, ar); 3918 } 3919 3920 int allocate_vpid(void) 3921 { 3922 int vpid; 3923 3924 if (!enable_vpid) 3925 return 0; 3926 spin_lock(&vmx_vpid_lock); 3927 vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS); 3928 if (vpid < VMX_NR_VPIDS) 3929 __set_bit(vpid, vmx_vpid_bitmap); 3930 else 3931 vpid = 0; 3932 spin_unlock(&vmx_vpid_lock); 3933 return vpid; 3934 } 3935 3936 void free_vpid(int vpid) 3937 { 3938 if (!enable_vpid || vpid == 0) 3939 return; 3940 spin_lock(&vmx_vpid_lock); 3941 __clear_bit(vpid, vmx_vpid_bitmap); 3942 spin_unlock(&vmx_vpid_lock); 3943 } 3944 3945 static void vmx_msr_bitmap_l01_changed(struct vcpu_vmx *vmx) 3946 { 3947 /* 3948 * When KVM is a nested hypervisor on top of Hyper-V and uses 3949 * 'Enlightened MSR Bitmap' feature L0 needs to know that MSR 3950 * bitmap has changed. 3951 */ 3952 if (kvm_is_using_evmcs()) { 3953 struct hv_enlightened_vmcs *evmcs = (void *)vmx->vmcs01.vmcs; 3954 3955 if (evmcs->hv_enlightenments_control.msr_bitmap) 3956 evmcs->hv_clean_fields &= 3957 ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP; 3958 } 3959 3960 vmx->nested.force_msr_bitmap_recalc = true; 3961 } 3962 3963 void vmx_disable_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type) 3964 { 3965 struct vcpu_vmx *vmx = to_vmx(vcpu); 3966 unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap; 3967 int idx; 3968 3969 if (!cpu_has_vmx_msr_bitmap()) 3970 return; 3971 3972 vmx_msr_bitmap_l01_changed(vmx); 3973 3974 /* 3975 * Mark the desired intercept state in shadow bitmap, this is needed 3976 * for resync when the MSR filters change. 3977 */ 3978 idx = vmx_get_passthrough_msr_slot(msr); 3979 if (idx >= 0) { 3980 if (type & MSR_TYPE_R) 3981 clear_bit(idx, vmx->shadow_msr_intercept.read); 3982 if (type & MSR_TYPE_W) 3983 clear_bit(idx, vmx->shadow_msr_intercept.write); 3984 } 3985 3986 if ((type & MSR_TYPE_R) && 3987 !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_READ)) { 3988 vmx_set_msr_bitmap_read(msr_bitmap, msr); 3989 type &= ~MSR_TYPE_R; 3990 } 3991 3992 if ((type & MSR_TYPE_W) && 3993 !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_WRITE)) { 3994 vmx_set_msr_bitmap_write(msr_bitmap, msr); 3995 type &= ~MSR_TYPE_W; 3996 } 3997 3998 if (type & MSR_TYPE_R) 3999 vmx_clear_msr_bitmap_read(msr_bitmap, msr); 4000 4001 if (type & MSR_TYPE_W) 4002 vmx_clear_msr_bitmap_write(msr_bitmap, msr); 4003 } 4004 4005 void vmx_enable_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type) 4006 { 4007 struct vcpu_vmx *vmx = to_vmx(vcpu); 4008 unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap; 4009 int idx; 4010 4011 if (!cpu_has_vmx_msr_bitmap()) 4012 return; 4013 4014 vmx_msr_bitmap_l01_changed(vmx); 4015 4016 /* 4017 * Mark the desired intercept state in shadow bitmap, this is needed 4018 * for resync when the MSR filter changes. 4019 */ 4020 idx = vmx_get_passthrough_msr_slot(msr); 4021 if (idx >= 0) { 4022 if (type & MSR_TYPE_R) 4023 set_bit(idx, vmx->shadow_msr_intercept.read); 4024 if (type & MSR_TYPE_W) 4025 set_bit(idx, vmx->shadow_msr_intercept.write); 4026 } 4027 4028 if (type & MSR_TYPE_R) 4029 vmx_set_msr_bitmap_read(msr_bitmap, msr); 4030 4031 if (type & MSR_TYPE_W) 4032 vmx_set_msr_bitmap_write(msr_bitmap, msr); 4033 } 4034 4035 static void vmx_update_msr_bitmap_x2apic(struct kvm_vcpu *vcpu) 4036 { 4037 /* 4038 * x2APIC indices for 64-bit accesses into the RDMSR and WRMSR halves 4039 * of the MSR bitmap. KVM emulates APIC registers up through 0x3f0, 4040 * i.e. MSR 0x83f, and so only needs to dynamically manipulate 64 bits. 4041 */ 4042 const int read_idx = APIC_BASE_MSR / BITS_PER_LONG_LONG; 4043 const int write_idx = read_idx + (0x800 / sizeof(u64)); 4044 struct vcpu_vmx *vmx = to_vmx(vcpu); 4045 u64 *msr_bitmap = (u64 *)vmx->vmcs01.msr_bitmap; 4046 u8 mode; 4047 4048 if (!cpu_has_vmx_msr_bitmap() || WARN_ON_ONCE(!lapic_in_kernel(vcpu))) 4049 return; 4050 4051 if (cpu_has_secondary_exec_ctrls() && 4052 (secondary_exec_controls_get(vmx) & 4053 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE)) { 4054 mode = MSR_BITMAP_MODE_X2APIC; 4055 if (enable_apicv && kvm_vcpu_apicv_active(vcpu)) 4056 mode |= MSR_BITMAP_MODE_X2APIC_APICV; 4057 } else { 4058 mode = 0; 4059 } 4060 4061 if (mode == vmx->x2apic_msr_bitmap_mode) 4062 return; 4063 4064 vmx->x2apic_msr_bitmap_mode = mode; 4065 4066 /* 4067 * Reset the bitmap for MSRs 0x800 - 0x83f. Leave AMD's uber-extended 4068 * registers (0x840 and above) intercepted, KVM doesn't support them. 4069 * Intercept all writes by default and poke holes as needed. Pass 4070 * through reads for all valid registers by default in x2APIC+APICv 4071 * mode, only the current timer count needs on-demand emulation by KVM. 4072 */ 4073 if (mode & MSR_BITMAP_MODE_X2APIC_APICV) 4074 msr_bitmap[read_idx] = ~kvm_lapic_readable_reg_mask(vcpu->arch.apic); 4075 else 4076 msr_bitmap[read_idx] = ~0ull; 4077 msr_bitmap[write_idx] = ~0ull; 4078 4079 /* 4080 * TPR reads and writes can be virtualized even if virtual interrupt 4081 * delivery is not in use. 4082 */ 4083 vmx_set_intercept_for_msr(vcpu, X2APIC_MSR(APIC_TASKPRI), MSR_TYPE_RW, 4084 !(mode & MSR_BITMAP_MODE_X2APIC)); 4085 4086 if (mode & MSR_BITMAP_MODE_X2APIC_APICV) { 4087 vmx_enable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_TMCCT), MSR_TYPE_RW); 4088 vmx_disable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_EOI), MSR_TYPE_W); 4089 vmx_disable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_SELF_IPI), MSR_TYPE_W); 4090 if (enable_ipiv) 4091 vmx_disable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_ICR), MSR_TYPE_RW); 4092 } 4093 } 4094 4095 void pt_update_intercept_for_msr(struct kvm_vcpu *vcpu) 4096 { 4097 struct vcpu_vmx *vmx = to_vmx(vcpu); 4098 bool flag = !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN); 4099 u32 i; 4100 4101 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_STATUS, MSR_TYPE_RW, flag); 4102 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_OUTPUT_BASE, MSR_TYPE_RW, flag); 4103 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_OUTPUT_MASK, MSR_TYPE_RW, flag); 4104 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_CR3_MATCH, MSR_TYPE_RW, flag); 4105 for (i = 0; i < vmx->pt_desc.num_address_ranges; i++) { 4106 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_ADDR0_A + i * 2, MSR_TYPE_RW, flag); 4107 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_ADDR0_B + i * 2, MSR_TYPE_RW, flag); 4108 } 4109 } 4110 4111 bool vmx_guest_apic_has_interrupt(struct kvm_vcpu *vcpu) 4112 { 4113 struct vcpu_vmx *vmx = to_vmx(vcpu); 4114 void *vapic_page; 4115 u32 vppr; 4116 int rvi; 4117 4118 if (WARN_ON_ONCE(!is_guest_mode(vcpu)) || 4119 !nested_cpu_has_vid(get_vmcs12(vcpu)) || 4120 WARN_ON_ONCE(!vmx->nested.virtual_apic_map.gfn)) 4121 return false; 4122 4123 rvi = vmx_get_rvi(); 4124 4125 vapic_page = vmx->nested.virtual_apic_map.hva; 4126 vppr = *((u32 *)(vapic_page + APIC_PROCPRI)); 4127 4128 return ((rvi & 0xf0) > (vppr & 0xf0)); 4129 } 4130 4131 void vmx_msr_filter_changed(struct kvm_vcpu *vcpu) 4132 { 4133 struct vcpu_vmx *vmx = to_vmx(vcpu); 4134 u32 i; 4135 4136 if (!cpu_has_vmx_msr_bitmap()) 4137 return; 4138 4139 /* 4140 * Redo intercept permissions for MSRs that KVM is passing through to 4141 * the guest. Disabling interception will check the new MSR filter and 4142 * ensure that KVM enables interception if usersepace wants to filter 4143 * the MSR. MSRs that KVM is already intercepting don't need to be 4144 * refreshed since KVM is going to intercept them regardless of what 4145 * userspace wants. 4146 */ 4147 for (i = 0; i < ARRAY_SIZE(vmx_possible_passthrough_msrs); i++) { 4148 u32 msr = vmx_possible_passthrough_msrs[i]; 4149 4150 if (!test_bit(i, vmx->shadow_msr_intercept.read)) 4151 vmx_disable_intercept_for_msr(vcpu, msr, MSR_TYPE_R); 4152 4153 if (!test_bit(i, vmx->shadow_msr_intercept.write)) 4154 vmx_disable_intercept_for_msr(vcpu, msr, MSR_TYPE_W); 4155 } 4156 4157 /* PT MSRs can be passed through iff PT is exposed to the guest. */ 4158 if (vmx_pt_mode_is_host_guest()) 4159 pt_update_intercept_for_msr(vcpu); 4160 } 4161 4162 static inline void kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu, 4163 int pi_vec) 4164 { 4165 #ifdef CONFIG_SMP 4166 if (vcpu->mode == IN_GUEST_MODE) { 4167 /* 4168 * The vector of the virtual has already been set in the PIR. 4169 * Send a notification event to deliver the virtual interrupt 4170 * unless the vCPU is the currently running vCPU, i.e. the 4171 * event is being sent from a fastpath VM-Exit handler, in 4172 * which case the PIR will be synced to the vIRR before 4173 * re-entering the guest. 4174 * 4175 * When the target is not the running vCPU, the following 4176 * possibilities emerge: 4177 * 4178 * Case 1: vCPU stays in non-root mode. Sending a notification 4179 * event posts the interrupt to the vCPU. 4180 * 4181 * Case 2: vCPU exits to root mode and is still runnable. The 4182 * PIR will be synced to the vIRR before re-entering the guest. 4183 * Sending a notification event is ok as the host IRQ handler 4184 * will ignore the spurious event. 4185 * 4186 * Case 3: vCPU exits to root mode and is blocked. vcpu_block() 4187 * has already synced PIR to vIRR and never blocks the vCPU if 4188 * the vIRR is not empty. Therefore, a blocked vCPU here does 4189 * not wait for any requested interrupts in PIR, and sending a 4190 * notification event also results in a benign, spurious event. 4191 */ 4192 4193 if (vcpu != kvm_get_running_vcpu()) 4194 __apic_send_IPI_mask(get_cpu_mask(vcpu->cpu), pi_vec); 4195 return; 4196 } 4197 #endif 4198 /* 4199 * The vCPU isn't in the guest; wake the vCPU in case it is blocking, 4200 * otherwise do nothing as KVM will grab the highest priority pending 4201 * IRQ via ->sync_pir_to_irr() in vcpu_enter_guest(). 4202 */ 4203 kvm_vcpu_wake_up(vcpu); 4204 } 4205 4206 static int vmx_deliver_nested_posted_interrupt(struct kvm_vcpu *vcpu, 4207 int vector) 4208 { 4209 struct vcpu_vmx *vmx = to_vmx(vcpu); 4210 4211 if (is_guest_mode(vcpu) && 4212 vector == vmx->nested.posted_intr_nv) { 4213 /* 4214 * If a posted intr is not recognized by hardware, 4215 * we will accomplish it in the next vmentry. 4216 */ 4217 vmx->nested.pi_pending = true; 4218 kvm_make_request(KVM_REQ_EVENT, vcpu); 4219 4220 /* 4221 * This pairs with the smp_mb_*() after setting vcpu->mode in 4222 * vcpu_enter_guest() to guarantee the vCPU sees the event 4223 * request if triggering a posted interrupt "fails" because 4224 * vcpu->mode != IN_GUEST_MODE. The extra barrier is needed as 4225 * the smb_wmb() in kvm_make_request() only ensures everything 4226 * done before making the request is visible when the request 4227 * is visible, it doesn't ensure ordering between the store to 4228 * vcpu->requests and the load from vcpu->mode. 4229 */ 4230 smp_mb__after_atomic(); 4231 4232 /* the PIR and ON have been set by L1. */ 4233 kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_NESTED_VECTOR); 4234 return 0; 4235 } 4236 return -1; 4237 } 4238 /* 4239 * Send interrupt to vcpu via posted interrupt way. 4240 * 1. If target vcpu is running(non-root mode), send posted interrupt 4241 * notification to vcpu and hardware will sync PIR to vIRR atomically. 4242 * 2. If target vcpu isn't running(root mode), kick it to pick up the 4243 * interrupt from PIR in next vmentry. 4244 */ 4245 static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector) 4246 { 4247 struct vcpu_vmx *vmx = to_vmx(vcpu); 4248 int r; 4249 4250 r = vmx_deliver_nested_posted_interrupt(vcpu, vector); 4251 if (!r) 4252 return 0; 4253 4254 /* Note, this is called iff the local APIC is in-kernel. */ 4255 if (!vcpu->arch.apic->apicv_active) 4256 return -1; 4257 4258 if (pi_test_and_set_pir(vector, &vmx->pi_desc)) 4259 return 0; 4260 4261 /* If a previous notification has sent the IPI, nothing to do. */ 4262 if (pi_test_and_set_on(&vmx->pi_desc)) 4263 return 0; 4264 4265 /* 4266 * The implied barrier in pi_test_and_set_on() pairs with the smp_mb_*() 4267 * after setting vcpu->mode in vcpu_enter_guest(), thus the vCPU is 4268 * guaranteed to see PID.ON=1 and sync the PIR to IRR if triggering a 4269 * posted interrupt "fails" because vcpu->mode != IN_GUEST_MODE. 4270 */ 4271 kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_VECTOR); 4272 return 0; 4273 } 4274 4275 void vmx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, 4276 int trig_mode, int vector) 4277 { 4278 struct kvm_vcpu *vcpu = apic->vcpu; 4279 4280 if (vmx_deliver_posted_interrupt(vcpu, vector)) { 4281 kvm_lapic_set_irr(vector, apic); 4282 kvm_make_request(KVM_REQ_EVENT, vcpu); 4283 kvm_vcpu_kick(vcpu); 4284 } else { 4285 trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, 4286 trig_mode, vector); 4287 } 4288 } 4289 4290 /* 4291 * Set up the vmcs's constant host-state fields, i.e., host-state fields that 4292 * will not change in the lifetime of the guest. 4293 * Note that host-state that does change is set elsewhere. E.g., host-state 4294 * that is set differently for each CPU is set in vmx_vcpu_load(), not here. 4295 */ 4296 void vmx_set_constant_host_state(struct vcpu_vmx *vmx) 4297 { 4298 u32 low32, high32; 4299 unsigned long tmpl; 4300 unsigned long cr0, cr3, cr4; 4301 4302 cr0 = read_cr0(); 4303 WARN_ON(cr0 & X86_CR0_TS); 4304 vmcs_writel(HOST_CR0, cr0); /* 22.2.3 */ 4305 4306 /* 4307 * Save the most likely value for this task's CR3 in the VMCS. 4308 * We can't use __get_current_cr3_fast() because we're not atomic. 4309 */ 4310 cr3 = __read_cr3(); 4311 vmcs_writel(HOST_CR3, cr3); /* 22.2.3 FIXME: shadow tables */ 4312 vmx->loaded_vmcs->host_state.cr3 = cr3; 4313 4314 /* Save the most likely value for this task's CR4 in the VMCS. */ 4315 cr4 = cr4_read_shadow(); 4316 vmcs_writel(HOST_CR4, cr4); /* 22.2.3, 22.2.5 */ 4317 vmx->loaded_vmcs->host_state.cr4 = cr4; 4318 4319 vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS); /* 22.2.4 */ 4320 #ifdef CONFIG_X86_64 4321 /* 4322 * Load null selectors, so we can avoid reloading them in 4323 * vmx_prepare_switch_to_host(), in case userspace uses 4324 * the null selectors too (the expected case). 4325 */ 4326 vmcs_write16(HOST_DS_SELECTOR, 0); 4327 vmcs_write16(HOST_ES_SELECTOR, 0); 4328 #else 4329 vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS); /* 22.2.4 */ 4330 vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS); /* 22.2.4 */ 4331 #endif 4332 vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS); /* 22.2.4 */ 4333 vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8); /* 22.2.4 */ 4334 4335 vmcs_writel(HOST_IDTR_BASE, host_idt_base); /* 22.2.4 */ 4336 4337 vmcs_writel(HOST_RIP, (unsigned long)vmx_vmexit); /* 22.2.5 */ 4338 4339 rdmsr(MSR_IA32_SYSENTER_CS, low32, high32); 4340 vmcs_write32(HOST_IA32_SYSENTER_CS, low32); 4341 4342 /* 4343 * SYSENTER is used for 32-bit system calls on either 32-bit or 4344 * 64-bit kernels. It is always zero If neither is allowed, otherwise 4345 * vmx_vcpu_load_vmcs loads it with the per-CPU entry stack (and may 4346 * have already done so!). 4347 */ 4348 if (!IS_ENABLED(CONFIG_IA32_EMULATION) && !IS_ENABLED(CONFIG_X86_32)) 4349 vmcs_writel(HOST_IA32_SYSENTER_ESP, 0); 4350 4351 rdmsrl(MSR_IA32_SYSENTER_EIP, tmpl); 4352 vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl); /* 22.2.3 */ 4353 4354 if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) { 4355 rdmsr(MSR_IA32_CR_PAT, low32, high32); 4356 vmcs_write64(HOST_IA32_PAT, low32 | ((u64) high32 << 32)); 4357 } 4358 4359 if (cpu_has_load_ia32_efer()) 4360 vmcs_write64(HOST_IA32_EFER, host_efer); 4361 } 4362 4363 void set_cr4_guest_host_mask(struct vcpu_vmx *vmx) 4364 { 4365 struct kvm_vcpu *vcpu = &vmx->vcpu; 4366 4367 vcpu->arch.cr4_guest_owned_bits = KVM_POSSIBLE_CR4_GUEST_BITS & 4368 ~vcpu->arch.cr4_guest_rsvd_bits; 4369 if (!enable_ept) { 4370 vcpu->arch.cr4_guest_owned_bits &= ~X86_CR4_TLBFLUSH_BITS; 4371 vcpu->arch.cr4_guest_owned_bits &= ~X86_CR4_PDPTR_BITS; 4372 } 4373 if (is_guest_mode(&vmx->vcpu)) 4374 vcpu->arch.cr4_guest_owned_bits &= 4375 ~get_vmcs12(vcpu)->cr4_guest_host_mask; 4376 vmcs_writel(CR4_GUEST_HOST_MASK, ~vcpu->arch.cr4_guest_owned_bits); 4377 } 4378 4379 static u32 vmx_pin_based_exec_ctrl(struct vcpu_vmx *vmx) 4380 { 4381 u32 pin_based_exec_ctrl = vmcs_config.pin_based_exec_ctrl; 4382 4383 if (!kvm_vcpu_apicv_active(&vmx->vcpu)) 4384 pin_based_exec_ctrl &= ~PIN_BASED_POSTED_INTR; 4385 4386 if (!enable_vnmi) 4387 pin_based_exec_ctrl &= ~PIN_BASED_VIRTUAL_NMIS; 4388 4389 if (!enable_preemption_timer) 4390 pin_based_exec_ctrl &= ~PIN_BASED_VMX_PREEMPTION_TIMER; 4391 4392 return pin_based_exec_ctrl; 4393 } 4394 4395 static u32 vmx_vmentry_ctrl(void) 4396 { 4397 u32 vmentry_ctrl = vmcs_config.vmentry_ctrl; 4398 4399 if (vmx_pt_mode_is_system()) 4400 vmentry_ctrl &= ~(VM_ENTRY_PT_CONCEAL_PIP | 4401 VM_ENTRY_LOAD_IA32_RTIT_CTL); 4402 /* 4403 * IA32e mode, and loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically. 4404 */ 4405 vmentry_ctrl &= ~(VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL | 4406 VM_ENTRY_LOAD_IA32_EFER | 4407 VM_ENTRY_IA32E_MODE); 4408 4409 if (cpu_has_perf_global_ctrl_bug()) 4410 vmentry_ctrl &= ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL; 4411 4412 return vmentry_ctrl; 4413 } 4414 4415 static u32 vmx_vmexit_ctrl(void) 4416 { 4417 u32 vmexit_ctrl = vmcs_config.vmexit_ctrl; 4418 4419 /* 4420 * Not used by KVM and never set in vmcs01 or vmcs02, but emulated for 4421 * nested virtualization and thus allowed to be set in vmcs12. 4422 */ 4423 vmexit_ctrl &= ~(VM_EXIT_SAVE_IA32_PAT | VM_EXIT_SAVE_IA32_EFER | 4424 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER); 4425 4426 if (vmx_pt_mode_is_system()) 4427 vmexit_ctrl &= ~(VM_EXIT_PT_CONCEAL_PIP | 4428 VM_EXIT_CLEAR_IA32_RTIT_CTL); 4429 4430 if (cpu_has_perf_global_ctrl_bug()) 4431 vmexit_ctrl &= ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; 4432 4433 /* Loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically */ 4434 return vmexit_ctrl & 4435 ~(VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | VM_EXIT_LOAD_IA32_EFER); 4436 } 4437 4438 void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) 4439 { 4440 struct vcpu_vmx *vmx = to_vmx(vcpu); 4441 4442 if (is_guest_mode(vcpu)) { 4443 vmx->nested.update_vmcs01_apicv_status = true; 4444 return; 4445 } 4446 4447 pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx)); 4448 4449 if (kvm_vcpu_apicv_active(vcpu)) { 4450 secondary_exec_controls_setbit(vmx, 4451 SECONDARY_EXEC_APIC_REGISTER_VIRT | 4452 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 4453 if (enable_ipiv) 4454 tertiary_exec_controls_setbit(vmx, TERTIARY_EXEC_IPI_VIRT); 4455 } else { 4456 secondary_exec_controls_clearbit(vmx, 4457 SECONDARY_EXEC_APIC_REGISTER_VIRT | 4458 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 4459 if (enable_ipiv) 4460 tertiary_exec_controls_clearbit(vmx, TERTIARY_EXEC_IPI_VIRT); 4461 } 4462 4463 vmx_update_msr_bitmap_x2apic(vcpu); 4464 } 4465 4466 static u32 vmx_exec_control(struct vcpu_vmx *vmx) 4467 { 4468 u32 exec_control = vmcs_config.cpu_based_exec_ctrl; 4469 4470 /* 4471 * Not used by KVM, but fully supported for nesting, i.e. are allowed in 4472 * vmcs12 and propagated to vmcs02 when set in vmcs12. 4473 */ 4474 exec_control &= ~(CPU_BASED_RDTSC_EXITING | 4475 CPU_BASED_USE_IO_BITMAPS | 4476 CPU_BASED_MONITOR_TRAP_FLAG | 4477 CPU_BASED_PAUSE_EXITING); 4478 4479 /* INTR_WINDOW_EXITING and NMI_WINDOW_EXITING are toggled dynamically */ 4480 exec_control &= ~(CPU_BASED_INTR_WINDOW_EXITING | 4481 CPU_BASED_NMI_WINDOW_EXITING); 4482 4483 if (vmx->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT) 4484 exec_control &= ~CPU_BASED_MOV_DR_EXITING; 4485 4486 if (!cpu_need_tpr_shadow(&vmx->vcpu)) 4487 exec_control &= ~CPU_BASED_TPR_SHADOW; 4488 4489 #ifdef CONFIG_X86_64 4490 if (exec_control & CPU_BASED_TPR_SHADOW) 4491 exec_control &= ~(CPU_BASED_CR8_LOAD_EXITING | 4492 CPU_BASED_CR8_STORE_EXITING); 4493 else 4494 exec_control |= CPU_BASED_CR8_STORE_EXITING | 4495 CPU_BASED_CR8_LOAD_EXITING; 4496 #endif 4497 /* No need to intercept CR3 access or INVPLG when using EPT. */ 4498 if (enable_ept) 4499 exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING | 4500 CPU_BASED_CR3_STORE_EXITING | 4501 CPU_BASED_INVLPG_EXITING); 4502 if (kvm_mwait_in_guest(vmx->vcpu.kvm)) 4503 exec_control &= ~(CPU_BASED_MWAIT_EXITING | 4504 CPU_BASED_MONITOR_EXITING); 4505 if (kvm_hlt_in_guest(vmx->vcpu.kvm)) 4506 exec_control &= ~CPU_BASED_HLT_EXITING; 4507 return exec_control; 4508 } 4509 4510 static u64 vmx_tertiary_exec_control(struct vcpu_vmx *vmx) 4511 { 4512 u64 exec_control = vmcs_config.cpu_based_3rd_exec_ctrl; 4513 4514 /* 4515 * IPI virtualization relies on APICv. Disable IPI virtualization if 4516 * APICv is inhibited. 4517 */ 4518 if (!enable_ipiv || !kvm_vcpu_apicv_active(&vmx->vcpu)) 4519 exec_control &= ~TERTIARY_EXEC_IPI_VIRT; 4520 4521 return exec_control; 4522 } 4523 4524 /* 4525 * Adjust a single secondary execution control bit to intercept/allow an 4526 * instruction in the guest. This is usually done based on whether or not a 4527 * feature has been exposed to the guest in order to correctly emulate faults. 4528 */ 4529 static inline void 4530 vmx_adjust_secondary_exec_control(struct vcpu_vmx *vmx, u32 *exec_control, 4531 u32 control, bool enabled, bool exiting) 4532 { 4533 /* 4534 * If the control is for an opt-in feature, clear the control if the 4535 * feature is not exposed to the guest, i.e. not enabled. If the 4536 * control is opt-out, i.e. an exiting control, clear the control if 4537 * the feature _is_ exposed to the guest, i.e. exiting/interception is 4538 * disabled for the associated instruction. Note, the caller is 4539 * responsible presetting exec_control to set all supported bits. 4540 */ 4541 if (enabled == exiting) 4542 *exec_control &= ~control; 4543 4544 /* 4545 * Update the nested MSR settings so that a nested VMM can/can't set 4546 * controls for features that are/aren't exposed to the guest. 4547 */ 4548 if (nested) { 4549 /* 4550 * All features that can be added or removed to VMX MSRs must 4551 * be supported in the first place for nested virtualization. 4552 */ 4553 if (WARN_ON_ONCE(!(vmcs_config.nested.secondary_ctls_high & control))) 4554 enabled = false; 4555 4556 if (enabled) 4557 vmx->nested.msrs.secondary_ctls_high |= control; 4558 else 4559 vmx->nested.msrs.secondary_ctls_high &= ~control; 4560 } 4561 } 4562 4563 /* 4564 * Wrapper macro for the common case of adjusting a secondary execution control 4565 * based on a single guest CPUID bit, with a dedicated feature bit. This also 4566 * verifies that the control is actually supported by KVM and hardware. 4567 */ 4568 #define vmx_adjust_sec_exec_control(vmx, exec_control, name, feat_name, ctrl_name, exiting) \ 4569 ({ \ 4570 struct kvm_vcpu *__vcpu = &(vmx)->vcpu; \ 4571 bool __enabled; \ 4572 \ 4573 if (cpu_has_vmx_##name()) { \ 4574 if (kvm_is_governed_feature(X86_FEATURE_##feat_name)) \ 4575 __enabled = guest_can_use(__vcpu, X86_FEATURE_##feat_name); \ 4576 else \ 4577 __enabled = guest_cpuid_has(__vcpu, X86_FEATURE_##feat_name); \ 4578 vmx_adjust_secondary_exec_control(vmx, exec_control, SECONDARY_EXEC_##ctrl_name,\ 4579 __enabled, exiting); \ 4580 } \ 4581 }) 4582 4583 /* More macro magic for ENABLE_/opt-in versus _EXITING/opt-out controls. */ 4584 #define vmx_adjust_sec_exec_feature(vmx, exec_control, lname, uname) \ 4585 vmx_adjust_sec_exec_control(vmx, exec_control, lname, uname, ENABLE_##uname, false) 4586 4587 #define vmx_adjust_sec_exec_exiting(vmx, exec_control, lname, uname) \ 4588 vmx_adjust_sec_exec_control(vmx, exec_control, lname, uname, uname##_EXITING, true) 4589 4590 static u32 vmx_secondary_exec_control(struct vcpu_vmx *vmx) 4591 { 4592 struct kvm_vcpu *vcpu = &vmx->vcpu; 4593 4594 u32 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl; 4595 4596 if (vmx_pt_mode_is_system()) 4597 exec_control &= ~(SECONDARY_EXEC_PT_USE_GPA | SECONDARY_EXEC_PT_CONCEAL_VMX); 4598 if (!cpu_need_virtualize_apic_accesses(vcpu)) 4599 exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 4600 if (vmx->vpid == 0) 4601 exec_control &= ~SECONDARY_EXEC_ENABLE_VPID; 4602 if (!enable_ept) { 4603 exec_control &= ~SECONDARY_EXEC_ENABLE_EPT; 4604 exec_control &= ~SECONDARY_EXEC_EPT_VIOLATION_VE; 4605 enable_unrestricted_guest = 0; 4606 } 4607 if (!enable_unrestricted_guest) 4608 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST; 4609 if (kvm_pause_in_guest(vmx->vcpu.kvm)) 4610 exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING; 4611 if (!kvm_vcpu_apicv_active(vcpu)) 4612 exec_control &= ~(SECONDARY_EXEC_APIC_REGISTER_VIRT | 4613 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 4614 exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE; 4615 4616 /* 4617 * KVM doesn't support VMFUNC for L1, but the control is set in KVM's 4618 * base configuration as KVM emulates VMFUNC[EPTP_SWITCHING] for L2. 4619 */ 4620 exec_control &= ~SECONDARY_EXEC_ENABLE_VMFUNC; 4621 4622 /* SECONDARY_EXEC_DESC is enabled/disabled on writes to CR4.UMIP, 4623 * in vmx_set_cr4. */ 4624 exec_control &= ~SECONDARY_EXEC_DESC; 4625 4626 /* SECONDARY_EXEC_SHADOW_VMCS is enabled when L1 executes VMPTRLD 4627 (handle_vmptrld). 4628 We can NOT enable shadow_vmcs here because we don't have yet 4629 a current VMCS12 4630 */ 4631 exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS; 4632 4633 /* 4634 * PML is enabled/disabled when dirty logging of memsmlots changes, but 4635 * it needs to be set here when dirty logging is already active, e.g. 4636 * if this vCPU was created after dirty logging was enabled. 4637 */ 4638 if (!enable_pml || !atomic_read(&vcpu->kvm->nr_memslots_dirty_logging)) 4639 exec_control &= ~SECONDARY_EXEC_ENABLE_PML; 4640 4641 vmx_adjust_sec_exec_feature(vmx, &exec_control, xsaves, XSAVES); 4642 4643 /* 4644 * RDPID is also gated by ENABLE_RDTSCP, turn on the control if either 4645 * feature is exposed to the guest. This creates a virtualization hole 4646 * if both are supported in hardware but only one is exposed to the 4647 * guest, but letting the guest execute RDTSCP or RDPID when either one 4648 * is advertised is preferable to emulating the advertised instruction 4649 * in KVM on #UD, and obviously better than incorrectly injecting #UD. 4650 */ 4651 if (cpu_has_vmx_rdtscp()) { 4652 bool rdpid_or_rdtscp_enabled = 4653 guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) || 4654 guest_cpuid_has(vcpu, X86_FEATURE_RDPID); 4655 4656 vmx_adjust_secondary_exec_control(vmx, &exec_control, 4657 SECONDARY_EXEC_ENABLE_RDTSCP, 4658 rdpid_or_rdtscp_enabled, false); 4659 } 4660 4661 vmx_adjust_sec_exec_feature(vmx, &exec_control, invpcid, INVPCID); 4662 4663 vmx_adjust_sec_exec_exiting(vmx, &exec_control, rdrand, RDRAND); 4664 vmx_adjust_sec_exec_exiting(vmx, &exec_control, rdseed, RDSEED); 4665 4666 vmx_adjust_sec_exec_control(vmx, &exec_control, waitpkg, WAITPKG, 4667 ENABLE_USR_WAIT_PAUSE, false); 4668 4669 if (!vcpu->kvm->arch.bus_lock_detection_enabled) 4670 exec_control &= ~SECONDARY_EXEC_BUS_LOCK_DETECTION; 4671 4672 if (!kvm_notify_vmexit_enabled(vcpu->kvm)) 4673 exec_control &= ~SECONDARY_EXEC_NOTIFY_VM_EXITING; 4674 4675 return exec_control; 4676 } 4677 4678 static inline int vmx_get_pid_table_order(struct kvm *kvm) 4679 { 4680 return get_order(kvm->arch.max_vcpu_ids * sizeof(*to_kvm_vmx(kvm)->pid_table)); 4681 } 4682 4683 static int vmx_alloc_ipiv_pid_table(struct kvm *kvm) 4684 { 4685 struct page *pages; 4686 struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); 4687 4688 if (!irqchip_in_kernel(kvm) || !enable_ipiv) 4689 return 0; 4690 4691 if (kvm_vmx->pid_table) 4692 return 0; 4693 4694 pages = alloc_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO, 4695 vmx_get_pid_table_order(kvm)); 4696 if (!pages) 4697 return -ENOMEM; 4698 4699 kvm_vmx->pid_table = (void *)page_address(pages); 4700 return 0; 4701 } 4702 4703 int vmx_vcpu_precreate(struct kvm *kvm) 4704 { 4705 return vmx_alloc_ipiv_pid_table(kvm); 4706 } 4707 4708 #define VMX_XSS_EXIT_BITMAP 0 4709 4710 static void init_vmcs(struct vcpu_vmx *vmx) 4711 { 4712 struct kvm *kvm = vmx->vcpu.kvm; 4713 struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); 4714 4715 if (nested) 4716 nested_vmx_set_vmcs_shadowing_bitmap(); 4717 4718 if (cpu_has_vmx_msr_bitmap()) 4719 vmcs_write64(MSR_BITMAP, __pa(vmx->vmcs01.msr_bitmap)); 4720 4721 vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); /* 22.3.1.5 */ 4722 4723 /* Control */ 4724 pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx)); 4725 4726 exec_controls_set(vmx, vmx_exec_control(vmx)); 4727 4728 if (cpu_has_secondary_exec_ctrls()) { 4729 secondary_exec_controls_set(vmx, vmx_secondary_exec_control(vmx)); 4730 if (vmx->ve_info) 4731 vmcs_write64(VE_INFORMATION_ADDRESS, 4732 __pa(vmx->ve_info)); 4733 } 4734 4735 if (cpu_has_tertiary_exec_ctrls()) 4736 tertiary_exec_controls_set(vmx, vmx_tertiary_exec_control(vmx)); 4737 4738 if (enable_apicv && lapic_in_kernel(&vmx->vcpu)) { 4739 vmcs_write64(EOI_EXIT_BITMAP0, 0); 4740 vmcs_write64(EOI_EXIT_BITMAP1, 0); 4741 vmcs_write64(EOI_EXIT_BITMAP2, 0); 4742 vmcs_write64(EOI_EXIT_BITMAP3, 0); 4743 4744 vmcs_write16(GUEST_INTR_STATUS, 0); 4745 4746 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR); 4747 vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->pi_desc))); 4748 } 4749 4750 if (vmx_can_use_ipiv(&vmx->vcpu)) { 4751 vmcs_write64(PID_POINTER_TABLE, __pa(kvm_vmx->pid_table)); 4752 vmcs_write16(LAST_PID_POINTER_INDEX, kvm->arch.max_vcpu_ids - 1); 4753 } 4754 4755 if (!kvm_pause_in_guest(kvm)) { 4756 vmcs_write32(PLE_GAP, ple_gap); 4757 vmx->ple_window = ple_window; 4758 vmx->ple_window_dirty = true; 4759 } 4760 4761 if (kvm_notify_vmexit_enabled(kvm)) 4762 vmcs_write32(NOTIFY_WINDOW, kvm->arch.notify_window); 4763 4764 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0); 4765 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0); 4766 vmcs_write32(CR3_TARGET_COUNT, 0); /* 22.2.1 */ 4767 4768 vmcs_write16(HOST_FS_SELECTOR, 0); /* 22.2.4 */ 4769 vmcs_write16(HOST_GS_SELECTOR, 0); /* 22.2.4 */ 4770 vmx_set_constant_host_state(vmx); 4771 vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */ 4772 vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */ 4773 4774 if (cpu_has_vmx_vmfunc()) 4775 vmcs_write64(VM_FUNCTION_CONTROL, 0); 4776 4777 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0); 4778 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0); 4779 vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val)); 4780 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0); 4781 vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); 4782 4783 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) 4784 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); 4785 4786 vm_exit_controls_set(vmx, vmx_vmexit_ctrl()); 4787 4788 /* 22.2.1, 20.8.1 */ 4789 vm_entry_controls_set(vmx, vmx_vmentry_ctrl()); 4790 4791 vmx->vcpu.arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits(); 4792 vmcs_writel(CR0_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr0_guest_owned_bits); 4793 4794 set_cr4_guest_host_mask(vmx); 4795 4796 if (vmx->vpid != 0) 4797 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid); 4798 4799 if (cpu_has_vmx_xsaves()) 4800 vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP); 4801 4802 if (enable_pml) { 4803 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg)); 4804 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1); 4805 } 4806 4807 vmx_write_encls_bitmap(&vmx->vcpu, NULL); 4808 4809 if (vmx_pt_mode_is_host_guest()) { 4810 memset(&vmx->pt_desc, 0, sizeof(vmx->pt_desc)); 4811 /* Bit[6~0] are forced to 1, writes are ignored. */ 4812 vmx->pt_desc.guest.output_mask = 0x7F; 4813 vmcs_write64(GUEST_IA32_RTIT_CTL, 0); 4814 } 4815 4816 vmcs_write32(GUEST_SYSENTER_CS, 0); 4817 vmcs_writel(GUEST_SYSENTER_ESP, 0); 4818 vmcs_writel(GUEST_SYSENTER_EIP, 0); 4819 vmcs_write64(GUEST_IA32_DEBUGCTL, 0); 4820 4821 if (cpu_has_vmx_tpr_shadow()) { 4822 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0); 4823 if (cpu_need_tpr_shadow(&vmx->vcpu)) 4824 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 4825 __pa(vmx->vcpu.arch.apic->regs)); 4826 vmcs_write32(TPR_THRESHOLD, 0); 4827 } 4828 4829 vmx_setup_uret_msrs(vmx); 4830 } 4831 4832 static void __vmx_vcpu_reset(struct kvm_vcpu *vcpu) 4833 { 4834 struct vcpu_vmx *vmx = to_vmx(vcpu); 4835 4836 init_vmcs(vmx); 4837 4838 if (nested) 4839 memcpy(&vmx->nested.msrs, &vmcs_config.nested, sizeof(vmx->nested.msrs)); 4840 4841 vcpu_setup_sgx_lepubkeyhash(vcpu); 4842 4843 vmx->nested.posted_intr_nv = -1; 4844 vmx->nested.vmxon_ptr = INVALID_GPA; 4845 vmx->nested.current_vmptr = INVALID_GPA; 4846 4847 #ifdef CONFIG_KVM_HYPERV 4848 vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID; 4849 #endif 4850 4851 vcpu->arch.microcode_version = 0x100000000ULL; 4852 vmx->msr_ia32_feature_control_valid_bits = FEAT_CTL_LOCKED; 4853 4854 /* 4855 * Enforce invariant: pi_desc.nv is always either POSTED_INTR_VECTOR 4856 * or POSTED_INTR_WAKEUP_VECTOR. 4857 */ 4858 vmx->pi_desc.nv = POSTED_INTR_VECTOR; 4859 __pi_set_sn(&vmx->pi_desc); 4860 } 4861 4862 void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) 4863 { 4864 struct vcpu_vmx *vmx = to_vmx(vcpu); 4865 4866 if (!init_event) 4867 __vmx_vcpu_reset(vcpu); 4868 4869 vmx->rmode.vm86_active = 0; 4870 vmx->spec_ctrl = 0; 4871 4872 vmx->msr_ia32_umwait_control = 0; 4873 4874 vmx->hv_deadline_tsc = -1; 4875 kvm_set_cr8(vcpu, 0); 4876 4877 vmx_segment_cache_clear(vmx); 4878 kvm_register_mark_available(vcpu, VCPU_EXREG_SEGMENTS); 4879 4880 seg_setup(VCPU_SREG_CS); 4881 vmcs_write16(GUEST_CS_SELECTOR, 0xf000); 4882 vmcs_writel(GUEST_CS_BASE, 0xffff0000ul); 4883 4884 seg_setup(VCPU_SREG_DS); 4885 seg_setup(VCPU_SREG_ES); 4886 seg_setup(VCPU_SREG_FS); 4887 seg_setup(VCPU_SREG_GS); 4888 seg_setup(VCPU_SREG_SS); 4889 4890 vmcs_write16(GUEST_TR_SELECTOR, 0); 4891 vmcs_writel(GUEST_TR_BASE, 0); 4892 vmcs_write32(GUEST_TR_LIMIT, 0xffff); 4893 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); 4894 4895 vmcs_write16(GUEST_LDTR_SELECTOR, 0); 4896 vmcs_writel(GUEST_LDTR_BASE, 0); 4897 vmcs_write32(GUEST_LDTR_LIMIT, 0xffff); 4898 vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082); 4899 4900 vmcs_writel(GUEST_GDTR_BASE, 0); 4901 vmcs_write32(GUEST_GDTR_LIMIT, 0xffff); 4902 4903 vmcs_writel(GUEST_IDTR_BASE, 0); 4904 vmcs_write32(GUEST_IDTR_LIMIT, 0xffff); 4905 4906 vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE); 4907 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0); 4908 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 0); 4909 if (kvm_mpx_supported()) 4910 vmcs_write64(GUEST_BNDCFGS, 0); 4911 4912 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); /* 22.2.1 */ 4913 4914 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 4915 4916 vpid_sync_context(vmx->vpid); 4917 4918 vmx_update_fb_clear_dis(vcpu, vmx); 4919 } 4920 4921 void vmx_enable_irq_window(struct kvm_vcpu *vcpu) 4922 { 4923 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING); 4924 } 4925 4926 void vmx_enable_nmi_window(struct kvm_vcpu *vcpu) 4927 { 4928 if (!enable_vnmi || 4929 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) { 4930 vmx_enable_irq_window(vcpu); 4931 return; 4932 } 4933 4934 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_NMI_WINDOW_EXITING); 4935 } 4936 4937 void vmx_inject_irq(struct kvm_vcpu *vcpu, bool reinjected) 4938 { 4939 struct vcpu_vmx *vmx = to_vmx(vcpu); 4940 uint32_t intr; 4941 int irq = vcpu->arch.interrupt.nr; 4942 4943 trace_kvm_inj_virq(irq, vcpu->arch.interrupt.soft, reinjected); 4944 4945 ++vcpu->stat.irq_injections; 4946 if (vmx->rmode.vm86_active) { 4947 int inc_eip = 0; 4948 if (vcpu->arch.interrupt.soft) 4949 inc_eip = vcpu->arch.event_exit_inst_len; 4950 kvm_inject_realmode_interrupt(vcpu, irq, inc_eip); 4951 return; 4952 } 4953 intr = irq | INTR_INFO_VALID_MASK; 4954 if (vcpu->arch.interrupt.soft) { 4955 intr |= INTR_TYPE_SOFT_INTR; 4956 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 4957 vmx->vcpu.arch.event_exit_inst_len); 4958 } else 4959 intr |= INTR_TYPE_EXT_INTR; 4960 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr); 4961 4962 vmx_clear_hlt(vcpu); 4963 } 4964 4965 void vmx_inject_nmi(struct kvm_vcpu *vcpu) 4966 { 4967 struct vcpu_vmx *vmx = to_vmx(vcpu); 4968 4969 if (!enable_vnmi) { 4970 /* 4971 * Tracking the NMI-blocked state in software is built upon 4972 * finding the next open IRQ window. This, in turn, depends on 4973 * well-behaving guests: They have to keep IRQs disabled at 4974 * least as long as the NMI handler runs. Otherwise we may 4975 * cause NMI nesting, maybe breaking the guest. But as this is 4976 * highly unlikely, we can live with the residual risk. 4977 */ 4978 vmx->loaded_vmcs->soft_vnmi_blocked = 1; 4979 vmx->loaded_vmcs->vnmi_blocked_time = 0; 4980 } 4981 4982 ++vcpu->stat.nmi_injections; 4983 vmx->loaded_vmcs->nmi_known_unmasked = false; 4984 4985 if (vmx->rmode.vm86_active) { 4986 kvm_inject_realmode_interrupt(vcpu, NMI_VECTOR, 0); 4987 return; 4988 } 4989 4990 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 4991 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR); 4992 4993 vmx_clear_hlt(vcpu); 4994 } 4995 4996 bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu) 4997 { 4998 struct vcpu_vmx *vmx = to_vmx(vcpu); 4999 bool masked; 5000 5001 if (!enable_vnmi) 5002 return vmx->loaded_vmcs->soft_vnmi_blocked; 5003 if (vmx->loaded_vmcs->nmi_known_unmasked) 5004 return false; 5005 masked = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_NMI; 5006 vmx->loaded_vmcs->nmi_known_unmasked = !masked; 5007 return masked; 5008 } 5009 5010 void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) 5011 { 5012 struct vcpu_vmx *vmx = to_vmx(vcpu); 5013 5014 if (!enable_vnmi) { 5015 if (vmx->loaded_vmcs->soft_vnmi_blocked != masked) { 5016 vmx->loaded_vmcs->soft_vnmi_blocked = masked; 5017 vmx->loaded_vmcs->vnmi_blocked_time = 0; 5018 } 5019 } else { 5020 vmx->loaded_vmcs->nmi_known_unmasked = !masked; 5021 if (masked) 5022 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 5023 GUEST_INTR_STATE_NMI); 5024 else 5025 vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO, 5026 GUEST_INTR_STATE_NMI); 5027 } 5028 } 5029 5030 bool vmx_nmi_blocked(struct kvm_vcpu *vcpu) 5031 { 5032 if (is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu)) 5033 return false; 5034 5035 if (!enable_vnmi && to_vmx(vcpu)->loaded_vmcs->soft_vnmi_blocked) 5036 return true; 5037 5038 return (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 5039 (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI | 5040 GUEST_INTR_STATE_NMI)); 5041 } 5042 5043 int vmx_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 5044 { 5045 if (to_vmx(vcpu)->nested.nested_run_pending) 5046 return -EBUSY; 5047 5048 /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */ 5049 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu)) 5050 return -EBUSY; 5051 5052 return !vmx_nmi_blocked(vcpu); 5053 } 5054 5055 bool vmx_interrupt_blocked(struct kvm_vcpu *vcpu) 5056 { 5057 if (is_guest_mode(vcpu) && nested_exit_on_intr(vcpu)) 5058 return false; 5059 5060 return !(vmx_get_rflags(vcpu) & X86_EFLAGS_IF) || 5061 (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 5062 (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS)); 5063 } 5064 5065 int vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) 5066 { 5067 if (to_vmx(vcpu)->nested.nested_run_pending) 5068 return -EBUSY; 5069 5070 /* 5071 * An IRQ must not be injected into L2 if it's supposed to VM-Exit, 5072 * e.g. if the IRQ arrived asynchronously after checking nested events. 5073 */ 5074 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(vcpu)) 5075 return -EBUSY; 5076 5077 return !vmx_interrupt_blocked(vcpu); 5078 } 5079 5080 int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr) 5081 { 5082 void __user *ret; 5083 5084 if (enable_unrestricted_guest) 5085 return 0; 5086 5087 mutex_lock(&kvm->slots_lock); 5088 ret = __x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, addr, 5089 PAGE_SIZE * 3); 5090 mutex_unlock(&kvm->slots_lock); 5091 5092 if (IS_ERR(ret)) 5093 return PTR_ERR(ret); 5094 5095 to_kvm_vmx(kvm)->tss_addr = addr; 5096 5097 return init_rmode_tss(kvm, ret); 5098 } 5099 5100 int vmx_set_identity_map_addr(struct kvm *kvm, u64 ident_addr) 5101 { 5102 to_kvm_vmx(kvm)->ept_identity_map_addr = ident_addr; 5103 return 0; 5104 } 5105 5106 static bool rmode_exception(struct kvm_vcpu *vcpu, int vec) 5107 { 5108 switch (vec) { 5109 case BP_VECTOR: 5110 /* 5111 * Update instruction length as we may reinject the exception 5112 * from user space while in guest debugging mode. 5113 */ 5114 to_vmx(vcpu)->vcpu.arch.event_exit_inst_len = 5115 vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 5116 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) 5117 return false; 5118 fallthrough; 5119 case DB_VECTOR: 5120 return !(vcpu->guest_debug & 5121 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)); 5122 case DE_VECTOR: 5123 case OF_VECTOR: 5124 case BR_VECTOR: 5125 case UD_VECTOR: 5126 case DF_VECTOR: 5127 case SS_VECTOR: 5128 case GP_VECTOR: 5129 case MF_VECTOR: 5130 return true; 5131 } 5132 return false; 5133 } 5134 5135 static int handle_rmode_exception(struct kvm_vcpu *vcpu, 5136 int vec, u32 err_code) 5137 { 5138 /* 5139 * Instruction with address size override prefix opcode 0x67 5140 * Cause the #SS fault with 0 error code in VM86 mode. 5141 */ 5142 if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) { 5143 if (kvm_emulate_instruction(vcpu, 0)) { 5144 if (vcpu->arch.halt_request) { 5145 vcpu->arch.halt_request = 0; 5146 return kvm_emulate_halt_noskip(vcpu); 5147 } 5148 return 1; 5149 } 5150 return 0; 5151 } 5152 5153 /* 5154 * Forward all other exceptions that are valid in real mode. 5155 * FIXME: Breaks guest debugging in real mode, needs to be fixed with 5156 * the required debugging infrastructure rework. 5157 */ 5158 kvm_queue_exception(vcpu, vec); 5159 return 1; 5160 } 5161 5162 static int handle_machine_check(struct kvm_vcpu *vcpu) 5163 { 5164 /* handled by vmx_vcpu_run() */ 5165 return 1; 5166 } 5167 5168 /* 5169 * If the host has split lock detection disabled, then #AC is 5170 * unconditionally injected into the guest, which is the pre split lock 5171 * detection behaviour. 5172 * 5173 * If the host has split lock detection enabled then #AC is 5174 * only injected into the guest when: 5175 * - Guest CPL == 3 (user mode) 5176 * - Guest has #AC detection enabled in CR0 5177 * - Guest EFLAGS has AC bit set 5178 */ 5179 bool vmx_guest_inject_ac(struct kvm_vcpu *vcpu) 5180 { 5181 if (!boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT)) 5182 return true; 5183 5184 return vmx_get_cpl(vcpu) == 3 && kvm_is_cr0_bit_set(vcpu, X86_CR0_AM) && 5185 (kvm_get_rflags(vcpu) & X86_EFLAGS_AC); 5186 } 5187 5188 static int handle_exception_nmi(struct kvm_vcpu *vcpu) 5189 { 5190 struct vcpu_vmx *vmx = to_vmx(vcpu); 5191 struct kvm_run *kvm_run = vcpu->run; 5192 u32 intr_info, ex_no, error_code; 5193 unsigned long cr2, dr6; 5194 u32 vect_info; 5195 5196 vect_info = vmx->idt_vectoring_info; 5197 intr_info = vmx_get_intr_info(vcpu); 5198 5199 /* 5200 * Machine checks are handled by handle_exception_irqoff(), or by 5201 * vmx_vcpu_run() if a #MC occurs on VM-Entry. NMIs are handled by 5202 * vmx_vcpu_enter_exit(). 5203 */ 5204 if (is_machine_check(intr_info) || is_nmi(intr_info)) 5205 return 1; 5206 5207 /* 5208 * Queue the exception here instead of in handle_nm_fault_irqoff(). 5209 * This ensures the nested_vmx check is not skipped so vmexit can 5210 * be reflected to L1 (when it intercepts #NM) before reaching this 5211 * point. 5212 */ 5213 if (is_nm_fault(intr_info)) { 5214 kvm_queue_exception(vcpu, NM_VECTOR); 5215 return 1; 5216 } 5217 5218 if (is_invalid_opcode(intr_info)) 5219 return handle_ud(vcpu); 5220 5221 if (WARN_ON_ONCE(is_ve_fault(intr_info))) { 5222 struct vmx_ve_information *ve_info = vmx->ve_info; 5223 5224 WARN_ONCE(ve_info->exit_reason != EXIT_REASON_EPT_VIOLATION, 5225 "Unexpected #VE on VM-Exit reason 0x%x", ve_info->exit_reason); 5226 dump_vmcs(vcpu); 5227 kvm_mmu_print_sptes(vcpu, ve_info->guest_physical_address, "#VE"); 5228 return 1; 5229 } 5230 5231 error_code = 0; 5232 if (intr_info & INTR_INFO_DELIVER_CODE_MASK) 5233 error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); 5234 5235 if (!vmx->rmode.vm86_active && is_gp_fault(intr_info)) { 5236 WARN_ON_ONCE(!enable_vmware_backdoor); 5237 5238 /* 5239 * VMware backdoor emulation on #GP interception only handles 5240 * IN{S}, OUT{S}, and RDPMC, none of which generate a non-zero 5241 * error code on #GP. 5242 */ 5243 if (error_code) { 5244 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code); 5245 return 1; 5246 } 5247 return kvm_emulate_instruction(vcpu, EMULTYPE_VMWARE_GP); 5248 } 5249 5250 /* 5251 * The #PF with PFEC.RSVD = 1 indicates the guest is accessing 5252 * MMIO, it is better to report an internal error. 5253 * See the comments in vmx_handle_exit. 5254 */ 5255 if ((vect_info & VECTORING_INFO_VALID_MASK) && 5256 !(is_page_fault(intr_info) && !(error_code & PFERR_RSVD_MASK))) { 5257 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 5258 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX; 5259 vcpu->run->internal.ndata = 4; 5260 vcpu->run->internal.data[0] = vect_info; 5261 vcpu->run->internal.data[1] = intr_info; 5262 vcpu->run->internal.data[2] = error_code; 5263 vcpu->run->internal.data[3] = vcpu->arch.last_vmentry_cpu; 5264 return 0; 5265 } 5266 5267 if (is_page_fault(intr_info)) { 5268 cr2 = vmx_get_exit_qual(vcpu); 5269 if (enable_ept && !vcpu->arch.apf.host_apf_flags) { 5270 /* 5271 * EPT will cause page fault only if we need to 5272 * detect illegal GPAs. 5273 */ 5274 WARN_ON_ONCE(!allow_smaller_maxphyaddr); 5275 kvm_fixup_and_inject_pf_error(vcpu, cr2, error_code); 5276 return 1; 5277 } else 5278 return kvm_handle_page_fault(vcpu, error_code, cr2, NULL, 0); 5279 } 5280 5281 ex_no = intr_info & INTR_INFO_VECTOR_MASK; 5282 5283 if (vmx->rmode.vm86_active && rmode_exception(vcpu, ex_no)) 5284 return handle_rmode_exception(vcpu, ex_no, error_code); 5285 5286 switch (ex_no) { 5287 case DB_VECTOR: 5288 dr6 = vmx_get_exit_qual(vcpu); 5289 if (!(vcpu->guest_debug & 5290 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) { 5291 /* 5292 * If the #DB was due to ICEBP, a.k.a. INT1, skip the 5293 * instruction. ICEBP generates a trap-like #DB, but 5294 * despite its interception control being tied to #DB, 5295 * is an instruction intercept, i.e. the VM-Exit occurs 5296 * on the ICEBP itself. Use the inner "skip" helper to 5297 * avoid single-step #DB and MTF updates, as ICEBP is 5298 * higher priority. Note, skipping ICEBP still clears 5299 * STI and MOVSS blocking. 5300 * 5301 * For all other #DBs, set vmcs.PENDING_DBG_EXCEPTIONS.BS 5302 * if single-step is enabled in RFLAGS and STI or MOVSS 5303 * blocking is active, as the CPU doesn't set the bit 5304 * on VM-Exit due to #DB interception. VM-Entry has a 5305 * consistency check that a single-step #DB is pending 5306 * in this scenario as the previous instruction cannot 5307 * have toggled RFLAGS.TF 0=>1 (because STI and POP/MOV 5308 * don't modify RFLAGS), therefore the one instruction 5309 * delay when activating single-step breakpoints must 5310 * have already expired. Note, the CPU sets/clears BS 5311 * as appropriate for all other VM-Exits types. 5312 */ 5313 if (is_icebp(intr_info)) 5314 WARN_ON(!skip_emulated_instruction(vcpu)); 5315 else if ((vmx_get_rflags(vcpu) & X86_EFLAGS_TF) && 5316 (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 5317 (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS))) 5318 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 5319 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS) | DR6_BS); 5320 5321 kvm_queue_exception_p(vcpu, DB_VECTOR, dr6); 5322 return 1; 5323 } 5324 kvm_run->debug.arch.dr6 = dr6 | DR6_ACTIVE_LOW; 5325 kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7); 5326 fallthrough; 5327 case BP_VECTOR: 5328 /* 5329 * Update instruction length as we may reinject #BP from 5330 * user space while in guest debugging mode. Reading it for 5331 * #DB as well causes no harm, it is not used in that case. 5332 */ 5333 vmx->vcpu.arch.event_exit_inst_len = 5334 vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 5335 kvm_run->exit_reason = KVM_EXIT_DEBUG; 5336 kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu); 5337 kvm_run->debug.arch.exception = ex_no; 5338 break; 5339 case AC_VECTOR: 5340 if (vmx_guest_inject_ac(vcpu)) { 5341 kvm_queue_exception_e(vcpu, AC_VECTOR, error_code); 5342 return 1; 5343 } 5344 5345 /* 5346 * Handle split lock. Depending on detection mode this will 5347 * either warn and disable split lock detection for this 5348 * task or force SIGBUS on it. 5349 */ 5350 if (handle_guest_split_lock(kvm_rip_read(vcpu))) 5351 return 1; 5352 fallthrough; 5353 default: 5354 kvm_run->exit_reason = KVM_EXIT_EXCEPTION; 5355 kvm_run->ex.exception = ex_no; 5356 kvm_run->ex.error_code = error_code; 5357 break; 5358 } 5359 return 0; 5360 } 5361 5362 static __always_inline int handle_external_interrupt(struct kvm_vcpu *vcpu) 5363 { 5364 ++vcpu->stat.irq_exits; 5365 return 1; 5366 } 5367 5368 static int handle_triple_fault(struct kvm_vcpu *vcpu) 5369 { 5370 vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN; 5371 vcpu->mmio_needed = 0; 5372 return 0; 5373 } 5374 5375 static int handle_io(struct kvm_vcpu *vcpu) 5376 { 5377 unsigned long exit_qualification; 5378 int size, in, string; 5379 unsigned port; 5380 5381 exit_qualification = vmx_get_exit_qual(vcpu); 5382 string = (exit_qualification & 16) != 0; 5383 5384 ++vcpu->stat.io_exits; 5385 5386 if (string) 5387 return kvm_emulate_instruction(vcpu, 0); 5388 5389 port = exit_qualification >> 16; 5390 size = (exit_qualification & 7) + 1; 5391 in = (exit_qualification & 8) != 0; 5392 5393 return kvm_fast_pio(vcpu, size, port, in); 5394 } 5395 5396 void vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) 5397 { 5398 /* 5399 * Patch in the VMCALL instruction: 5400 */ 5401 hypercall[0] = 0x0f; 5402 hypercall[1] = 0x01; 5403 hypercall[2] = 0xc1; 5404 } 5405 5406 /* called to set cr0 as appropriate for a mov-to-cr0 exit. */ 5407 static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val) 5408 { 5409 if (is_guest_mode(vcpu)) { 5410 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5411 unsigned long orig_val = val; 5412 5413 /* 5414 * We get here when L2 changed cr0 in a way that did not change 5415 * any of L1's shadowed bits (see nested_vmx_exit_handled_cr), 5416 * but did change L0 shadowed bits. So we first calculate the 5417 * effective cr0 value that L1 would like to write into the 5418 * hardware. It consists of the L2-owned bits from the new 5419 * value combined with the L1-owned bits from L1's guest_cr0. 5420 */ 5421 val = (val & ~vmcs12->cr0_guest_host_mask) | 5422 (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask); 5423 5424 if (kvm_set_cr0(vcpu, val)) 5425 return 1; 5426 vmcs_writel(CR0_READ_SHADOW, orig_val); 5427 return 0; 5428 } else { 5429 return kvm_set_cr0(vcpu, val); 5430 } 5431 } 5432 5433 static int handle_set_cr4(struct kvm_vcpu *vcpu, unsigned long val) 5434 { 5435 if (is_guest_mode(vcpu)) { 5436 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5437 unsigned long orig_val = val; 5438 5439 /* analogously to handle_set_cr0 */ 5440 val = (val & ~vmcs12->cr4_guest_host_mask) | 5441 (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask); 5442 if (kvm_set_cr4(vcpu, val)) 5443 return 1; 5444 vmcs_writel(CR4_READ_SHADOW, orig_val); 5445 return 0; 5446 } else 5447 return kvm_set_cr4(vcpu, val); 5448 } 5449 5450 static int handle_desc(struct kvm_vcpu *vcpu) 5451 { 5452 /* 5453 * UMIP emulation relies on intercepting writes to CR4.UMIP, i.e. this 5454 * and other code needs to be updated if UMIP can be guest owned. 5455 */ 5456 BUILD_BUG_ON(KVM_POSSIBLE_CR4_GUEST_BITS & X86_CR4_UMIP); 5457 5458 WARN_ON_ONCE(!kvm_is_cr4_bit_set(vcpu, X86_CR4_UMIP)); 5459 return kvm_emulate_instruction(vcpu, 0); 5460 } 5461 5462 static int handle_cr(struct kvm_vcpu *vcpu) 5463 { 5464 unsigned long exit_qualification, val; 5465 int cr; 5466 int reg; 5467 int err; 5468 int ret; 5469 5470 exit_qualification = vmx_get_exit_qual(vcpu); 5471 cr = exit_qualification & 15; 5472 reg = (exit_qualification >> 8) & 15; 5473 switch ((exit_qualification >> 4) & 3) { 5474 case 0: /* mov to cr */ 5475 val = kvm_register_read(vcpu, reg); 5476 trace_kvm_cr_write(cr, val); 5477 switch (cr) { 5478 case 0: 5479 err = handle_set_cr0(vcpu, val); 5480 return kvm_complete_insn_gp(vcpu, err); 5481 case 3: 5482 WARN_ON_ONCE(enable_unrestricted_guest); 5483 5484 err = kvm_set_cr3(vcpu, val); 5485 return kvm_complete_insn_gp(vcpu, err); 5486 case 4: 5487 err = handle_set_cr4(vcpu, val); 5488 return kvm_complete_insn_gp(vcpu, err); 5489 case 8: { 5490 u8 cr8_prev = kvm_get_cr8(vcpu); 5491 u8 cr8 = (u8)val; 5492 err = kvm_set_cr8(vcpu, cr8); 5493 ret = kvm_complete_insn_gp(vcpu, err); 5494 if (lapic_in_kernel(vcpu)) 5495 return ret; 5496 if (cr8_prev <= cr8) 5497 return ret; 5498 /* 5499 * TODO: we might be squashing a 5500 * KVM_GUESTDBG_SINGLESTEP-triggered 5501 * KVM_EXIT_DEBUG here. 5502 */ 5503 vcpu->run->exit_reason = KVM_EXIT_SET_TPR; 5504 return 0; 5505 } 5506 } 5507 break; 5508 case 2: /* clts */ 5509 KVM_BUG(1, vcpu->kvm, "Guest always owns CR0.TS"); 5510 return -EIO; 5511 case 1: /*mov from cr*/ 5512 switch (cr) { 5513 case 3: 5514 WARN_ON_ONCE(enable_unrestricted_guest); 5515 5516 val = kvm_read_cr3(vcpu); 5517 kvm_register_write(vcpu, reg, val); 5518 trace_kvm_cr_read(cr, val); 5519 return kvm_skip_emulated_instruction(vcpu); 5520 case 8: 5521 val = kvm_get_cr8(vcpu); 5522 kvm_register_write(vcpu, reg, val); 5523 trace_kvm_cr_read(cr, val); 5524 return kvm_skip_emulated_instruction(vcpu); 5525 } 5526 break; 5527 case 3: /* lmsw */ 5528 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f; 5529 trace_kvm_cr_write(0, (kvm_read_cr0_bits(vcpu, ~0xful) | val)); 5530 kvm_lmsw(vcpu, val); 5531 5532 return kvm_skip_emulated_instruction(vcpu); 5533 default: 5534 break; 5535 } 5536 vcpu->run->exit_reason = 0; 5537 vcpu_unimpl(vcpu, "unhandled control register: op %d cr %d\n", 5538 (int)(exit_qualification >> 4) & 3, cr); 5539 return 0; 5540 } 5541 5542 static int handle_dr(struct kvm_vcpu *vcpu) 5543 { 5544 unsigned long exit_qualification; 5545 int dr, dr7, reg; 5546 int err = 1; 5547 5548 exit_qualification = vmx_get_exit_qual(vcpu); 5549 dr = exit_qualification & DEBUG_REG_ACCESS_NUM; 5550 5551 /* First, if DR does not exist, trigger UD */ 5552 if (!kvm_require_dr(vcpu, dr)) 5553 return 1; 5554 5555 if (vmx_get_cpl(vcpu) > 0) 5556 goto out; 5557 5558 dr7 = vmcs_readl(GUEST_DR7); 5559 if (dr7 & DR7_GD) { 5560 /* 5561 * As the vm-exit takes precedence over the debug trap, we 5562 * need to emulate the latter, either for the host or the 5563 * guest debugging itself. 5564 */ 5565 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) { 5566 vcpu->run->debug.arch.dr6 = DR6_BD | DR6_ACTIVE_LOW; 5567 vcpu->run->debug.arch.dr7 = dr7; 5568 vcpu->run->debug.arch.pc = kvm_get_linear_rip(vcpu); 5569 vcpu->run->debug.arch.exception = DB_VECTOR; 5570 vcpu->run->exit_reason = KVM_EXIT_DEBUG; 5571 return 0; 5572 } else { 5573 kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BD); 5574 return 1; 5575 } 5576 } 5577 5578 if (vcpu->guest_debug == 0) { 5579 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_MOV_DR_EXITING); 5580 5581 /* 5582 * No more DR vmexits; force a reload of the debug registers 5583 * and reenter on this instruction. The next vmexit will 5584 * retrieve the full state of the debug registers. 5585 */ 5586 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT; 5587 return 1; 5588 } 5589 5590 reg = DEBUG_REG_ACCESS_REG(exit_qualification); 5591 if (exit_qualification & TYPE_MOV_FROM_DR) { 5592 kvm_register_write(vcpu, reg, kvm_get_dr(vcpu, dr)); 5593 err = 0; 5594 } else { 5595 err = kvm_set_dr(vcpu, dr, kvm_register_read(vcpu, reg)); 5596 } 5597 5598 out: 5599 return kvm_complete_insn_gp(vcpu, err); 5600 } 5601 5602 void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu) 5603 { 5604 get_debugreg(vcpu->arch.db[0], 0); 5605 get_debugreg(vcpu->arch.db[1], 1); 5606 get_debugreg(vcpu->arch.db[2], 2); 5607 get_debugreg(vcpu->arch.db[3], 3); 5608 get_debugreg(vcpu->arch.dr6, 6); 5609 vcpu->arch.dr7 = vmcs_readl(GUEST_DR7); 5610 5611 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT; 5612 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_MOV_DR_EXITING); 5613 5614 /* 5615 * exc_debug expects dr6 to be cleared after it runs, avoid that it sees 5616 * a stale dr6 from the guest. 5617 */ 5618 set_debugreg(DR6_RESERVED, 6); 5619 } 5620 5621 void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val) 5622 { 5623 vmcs_writel(GUEST_DR7, val); 5624 } 5625 5626 static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu) 5627 { 5628 kvm_apic_update_ppr(vcpu); 5629 return 1; 5630 } 5631 5632 static int handle_interrupt_window(struct kvm_vcpu *vcpu) 5633 { 5634 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING); 5635 5636 kvm_make_request(KVM_REQ_EVENT, vcpu); 5637 5638 ++vcpu->stat.irq_window_exits; 5639 return 1; 5640 } 5641 5642 static int handle_invlpg(struct kvm_vcpu *vcpu) 5643 { 5644 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5645 5646 kvm_mmu_invlpg(vcpu, exit_qualification); 5647 return kvm_skip_emulated_instruction(vcpu); 5648 } 5649 5650 static int handle_apic_access(struct kvm_vcpu *vcpu) 5651 { 5652 if (likely(fasteoi)) { 5653 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5654 int access_type, offset; 5655 5656 access_type = exit_qualification & APIC_ACCESS_TYPE; 5657 offset = exit_qualification & APIC_ACCESS_OFFSET; 5658 /* 5659 * Sane guest uses MOV to write EOI, with written value 5660 * not cared. So make a short-circuit here by avoiding 5661 * heavy instruction emulation. 5662 */ 5663 if ((access_type == TYPE_LINEAR_APIC_INST_WRITE) && 5664 (offset == APIC_EOI)) { 5665 kvm_lapic_set_eoi(vcpu); 5666 return kvm_skip_emulated_instruction(vcpu); 5667 } 5668 } 5669 return kvm_emulate_instruction(vcpu, 0); 5670 } 5671 5672 static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu) 5673 { 5674 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5675 int vector = exit_qualification & 0xff; 5676 5677 /* EOI-induced VM exit is trap-like and thus no need to adjust IP */ 5678 kvm_apic_set_eoi_accelerated(vcpu, vector); 5679 return 1; 5680 } 5681 5682 static int handle_apic_write(struct kvm_vcpu *vcpu) 5683 { 5684 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5685 5686 /* 5687 * APIC-write VM-Exit is trap-like, KVM doesn't need to advance RIP and 5688 * hardware has done any necessary aliasing, offset adjustments, etc... 5689 * for the access. I.e. the correct value has already been written to 5690 * the vAPIC page for the correct 16-byte chunk. KVM needs only to 5691 * retrieve the register value and emulate the access. 5692 */ 5693 u32 offset = exit_qualification & 0xff0; 5694 5695 kvm_apic_write_nodecode(vcpu, offset); 5696 return 1; 5697 } 5698 5699 static int handle_task_switch(struct kvm_vcpu *vcpu) 5700 { 5701 struct vcpu_vmx *vmx = to_vmx(vcpu); 5702 unsigned long exit_qualification; 5703 bool has_error_code = false; 5704 u32 error_code = 0; 5705 u16 tss_selector; 5706 int reason, type, idt_v, idt_index; 5707 5708 idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK); 5709 idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK); 5710 type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK); 5711 5712 exit_qualification = vmx_get_exit_qual(vcpu); 5713 5714 reason = (u32)exit_qualification >> 30; 5715 if (reason == TASK_SWITCH_GATE && idt_v) { 5716 switch (type) { 5717 case INTR_TYPE_NMI_INTR: 5718 vcpu->arch.nmi_injected = false; 5719 vmx_set_nmi_mask(vcpu, true); 5720 break; 5721 case INTR_TYPE_EXT_INTR: 5722 case INTR_TYPE_SOFT_INTR: 5723 kvm_clear_interrupt_queue(vcpu); 5724 break; 5725 case INTR_TYPE_HARD_EXCEPTION: 5726 if (vmx->idt_vectoring_info & 5727 VECTORING_INFO_DELIVER_CODE_MASK) { 5728 has_error_code = true; 5729 error_code = 5730 vmcs_read32(IDT_VECTORING_ERROR_CODE); 5731 } 5732 fallthrough; 5733 case INTR_TYPE_SOFT_EXCEPTION: 5734 kvm_clear_exception_queue(vcpu); 5735 break; 5736 default: 5737 break; 5738 } 5739 } 5740 tss_selector = exit_qualification; 5741 5742 if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION && 5743 type != INTR_TYPE_EXT_INTR && 5744 type != INTR_TYPE_NMI_INTR)) 5745 WARN_ON(!skip_emulated_instruction(vcpu)); 5746 5747 /* 5748 * TODO: What about debug traps on tss switch? 5749 * Are we supposed to inject them and update dr6? 5750 */ 5751 return kvm_task_switch(vcpu, tss_selector, 5752 type == INTR_TYPE_SOFT_INTR ? idt_index : -1, 5753 reason, has_error_code, error_code); 5754 } 5755 5756 static int handle_ept_violation(struct kvm_vcpu *vcpu) 5757 { 5758 unsigned long exit_qualification; 5759 gpa_t gpa; 5760 u64 error_code; 5761 5762 exit_qualification = vmx_get_exit_qual(vcpu); 5763 5764 /* 5765 * EPT violation happened while executing iret from NMI, 5766 * "blocked by NMI" bit has to be set before next VM entry. 5767 * There are errata that may cause this bit to not be set: 5768 * AAK134, BY25. 5769 */ 5770 if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) && 5771 enable_vnmi && 5772 (exit_qualification & INTR_INFO_UNBLOCK_NMI)) 5773 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI); 5774 5775 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS); 5776 trace_kvm_page_fault(vcpu, gpa, exit_qualification); 5777 5778 /* Is it a read fault? */ 5779 error_code = (exit_qualification & EPT_VIOLATION_ACC_READ) 5780 ? PFERR_USER_MASK : 0; 5781 /* Is it a write fault? */ 5782 error_code |= (exit_qualification & EPT_VIOLATION_ACC_WRITE) 5783 ? PFERR_WRITE_MASK : 0; 5784 /* Is it a fetch fault? */ 5785 error_code |= (exit_qualification & EPT_VIOLATION_ACC_INSTR) 5786 ? PFERR_FETCH_MASK : 0; 5787 /* ept page table entry is present? */ 5788 error_code |= (exit_qualification & EPT_VIOLATION_RWX_MASK) 5789 ? PFERR_PRESENT_MASK : 0; 5790 5791 error_code |= (exit_qualification & EPT_VIOLATION_GVA_TRANSLATED) != 0 ? 5792 PFERR_GUEST_FINAL_MASK : PFERR_GUEST_PAGE_MASK; 5793 5794 /* 5795 * Check that the GPA doesn't exceed physical memory limits, as that is 5796 * a guest page fault. We have to emulate the instruction here, because 5797 * if the illegal address is that of a paging structure, then 5798 * EPT_VIOLATION_ACC_WRITE bit is set. Alternatively, if supported we 5799 * would also use advanced VM-exit information for EPT violations to 5800 * reconstruct the page fault error code. 5801 */ 5802 if (unlikely(allow_smaller_maxphyaddr && !kvm_vcpu_is_legal_gpa(vcpu, gpa))) 5803 return kvm_emulate_instruction(vcpu, 0); 5804 5805 return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0); 5806 } 5807 5808 static int handle_ept_misconfig(struct kvm_vcpu *vcpu) 5809 { 5810 gpa_t gpa; 5811 5812 if (vmx_check_emulate_instruction(vcpu, EMULTYPE_PF, NULL, 0)) 5813 return 1; 5814 5815 /* 5816 * A nested guest cannot optimize MMIO vmexits, because we have an 5817 * nGPA here instead of the required GPA. 5818 */ 5819 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS); 5820 if (!is_guest_mode(vcpu) && 5821 !kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) { 5822 trace_kvm_fast_mmio(gpa); 5823 return kvm_skip_emulated_instruction(vcpu); 5824 } 5825 5826 return kvm_mmu_page_fault(vcpu, gpa, PFERR_RSVD_MASK, NULL, 0); 5827 } 5828 5829 static int handle_nmi_window(struct kvm_vcpu *vcpu) 5830 { 5831 if (KVM_BUG_ON(!enable_vnmi, vcpu->kvm)) 5832 return -EIO; 5833 5834 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_NMI_WINDOW_EXITING); 5835 ++vcpu->stat.nmi_window_exits; 5836 kvm_make_request(KVM_REQ_EVENT, vcpu); 5837 5838 return 1; 5839 } 5840 5841 static bool vmx_emulation_required_with_pending_exception(struct kvm_vcpu *vcpu) 5842 { 5843 struct vcpu_vmx *vmx = to_vmx(vcpu); 5844 5845 return vmx->emulation_required && !vmx->rmode.vm86_active && 5846 (kvm_is_exception_pending(vcpu) || vcpu->arch.exception.injected); 5847 } 5848 5849 static int handle_invalid_guest_state(struct kvm_vcpu *vcpu) 5850 { 5851 struct vcpu_vmx *vmx = to_vmx(vcpu); 5852 bool intr_window_requested; 5853 unsigned count = 130; 5854 5855 intr_window_requested = exec_controls_get(vmx) & 5856 CPU_BASED_INTR_WINDOW_EXITING; 5857 5858 while (vmx->emulation_required && count-- != 0) { 5859 if (intr_window_requested && !vmx_interrupt_blocked(vcpu)) 5860 return handle_interrupt_window(&vmx->vcpu); 5861 5862 if (kvm_test_request(KVM_REQ_EVENT, vcpu)) 5863 return 1; 5864 5865 if (!kvm_emulate_instruction(vcpu, 0)) 5866 return 0; 5867 5868 if (vmx_emulation_required_with_pending_exception(vcpu)) { 5869 kvm_prepare_emulation_failure_exit(vcpu); 5870 return 0; 5871 } 5872 5873 if (vcpu->arch.halt_request) { 5874 vcpu->arch.halt_request = 0; 5875 return kvm_emulate_halt_noskip(vcpu); 5876 } 5877 5878 /* 5879 * Note, return 1 and not 0, vcpu_run() will invoke 5880 * xfer_to_guest_mode() which will create a proper return 5881 * code. 5882 */ 5883 if (__xfer_to_guest_mode_work_pending()) 5884 return 1; 5885 } 5886 5887 return 1; 5888 } 5889 5890 int vmx_vcpu_pre_run(struct kvm_vcpu *vcpu) 5891 { 5892 if (vmx_emulation_required_with_pending_exception(vcpu)) { 5893 kvm_prepare_emulation_failure_exit(vcpu); 5894 return 0; 5895 } 5896 5897 return 1; 5898 } 5899 5900 static void grow_ple_window(struct kvm_vcpu *vcpu) 5901 { 5902 struct vcpu_vmx *vmx = to_vmx(vcpu); 5903 unsigned int old = vmx->ple_window; 5904 5905 vmx->ple_window = __grow_ple_window(old, ple_window, 5906 ple_window_grow, 5907 ple_window_max); 5908 5909 if (vmx->ple_window != old) { 5910 vmx->ple_window_dirty = true; 5911 trace_kvm_ple_window_update(vcpu->vcpu_id, 5912 vmx->ple_window, old); 5913 } 5914 } 5915 5916 static void shrink_ple_window(struct kvm_vcpu *vcpu) 5917 { 5918 struct vcpu_vmx *vmx = to_vmx(vcpu); 5919 unsigned int old = vmx->ple_window; 5920 5921 vmx->ple_window = __shrink_ple_window(old, ple_window, 5922 ple_window_shrink, 5923 ple_window); 5924 5925 if (vmx->ple_window != old) { 5926 vmx->ple_window_dirty = true; 5927 trace_kvm_ple_window_update(vcpu->vcpu_id, 5928 vmx->ple_window, old); 5929 } 5930 } 5931 5932 /* 5933 * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE 5934 * exiting, so only get here on cpu with PAUSE-Loop-Exiting. 5935 */ 5936 static int handle_pause(struct kvm_vcpu *vcpu) 5937 { 5938 if (!kvm_pause_in_guest(vcpu->kvm)) 5939 grow_ple_window(vcpu); 5940 5941 /* 5942 * Intel sdm vol3 ch-25.1.3 says: The "PAUSE-loop exiting" 5943 * VM-execution control is ignored if CPL > 0. OTOH, KVM 5944 * never set PAUSE_EXITING and just set PLE if supported, 5945 * so the vcpu must be CPL=0 if it gets a PAUSE exit. 5946 */ 5947 kvm_vcpu_on_spin(vcpu, true); 5948 return kvm_skip_emulated_instruction(vcpu); 5949 } 5950 5951 static int handle_monitor_trap(struct kvm_vcpu *vcpu) 5952 { 5953 return 1; 5954 } 5955 5956 static int handle_invpcid(struct kvm_vcpu *vcpu) 5957 { 5958 u32 vmx_instruction_info; 5959 unsigned long type; 5960 gva_t gva; 5961 struct { 5962 u64 pcid; 5963 u64 gla; 5964 } operand; 5965 int gpr_index; 5966 5967 if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) { 5968 kvm_queue_exception(vcpu, UD_VECTOR); 5969 return 1; 5970 } 5971 5972 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5973 gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info); 5974 type = kvm_register_read(vcpu, gpr_index); 5975 5976 /* According to the Intel instruction reference, the memory operand 5977 * is read even if it isn't needed (e.g., for type==all) 5978 */ 5979 if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), 5980 vmx_instruction_info, false, 5981 sizeof(operand), &gva)) 5982 return 1; 5983 5984 return kvm_handle_invpcid(vcpu, type, gva); 5985 } 5986 5987 static int handle_pml_full(struct kvm_vcpu *vcpu) 5988 { 5989 unsigned long exit_qualification; 5990 5991 trace_kvm_pml_full(vcpu->vcpu_id); 5992 5993 exit_qualification = vmx_get_exit_qual(vcpu); 5994 5995 /* 5996 * PML buffer FULL happened while executing iret from NMI, 5997 * "blocked by NMI" bit has to be set before next VM entry. 5998 */ 5999 if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) && 6000 enable_vnmi && 6001 (exit_qualification & INTR_INFO_UNBLOCK_NMI)) 6002 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 6003 GUEST_INTR_STATE_NMI); 6004 6005 /* 6006 * PML buffer already flushed at beginning of VMEXIT. Nothing to do 6007 * here.., and there's no userspace involvement needed for PML. 6008 */ 6009 return 1; 6010 } 6011 6012 static fastpath_t handle_fastpath_preemption_timer(struct kvm_vcpu *vcpu, 6013 bool force_immediate_exit) 6014 { 6015 struct vcpu_vmx *vmx = to_vmx(vcpu); 6016 6017 /* 6018 * In the *extremely* unlikely scenario that this is a spurious VM-Exit 6019 * due to the timer expiring while it was "soft" disabled, just eat the 6020 * exit and re-enter the guest. 6021 */ 6022 if (unlikely(vmx->loaded_vmcs->hv_timer_soft_disabled)) 6023 return EXIT_FASTPATH_REENTER_GUEST; 6024 6025 /* 6026 * If the timer expired because KVM used it to force an immediate exit, 6027 * then mission accomplished. 6028 */ 6029 if (force_immediate_exit) 6030 return EXIT_FASTPATH_EXIT_HANDLED; 6031 6032 /* 6033 * If L2 is active, go down the slow path as emulating the guest timer 6034 * expiration likely requires synthesizing a nested VM-Exit. 6035 */ 6036 if (is_guest_mode(vcpu)) 6037 return EXIT_FASTPATH_NONE; 6038 6039 kvm_lapic_expired_hv_timer(vcpu); 6040 return EXIT_FASTPATH_REENTER_GUEST; 6041 } 6042 6043 static int handle_preemption_timer(struct kvm_vcpu *vcpu) 6044 { 6045 /* 6046 * This non-fastpath handler is reached if and only if the preemption 6047 * timer was being used to emulate a guest timer while L2 is active. 6048 * All other scenarios are supposed to be handled in the fastpath. 6049 */ 6050 WARN_ON_ONCE(!is_guest_mode(vcpu)); 6051 kvm_lapic_expired_hv_timer(vcpu); 6052 return 1; 6053 } 6054 6055 /* 6056 * When nested=0, all VMX instruction VM Exits filter here. The handlers 6057 * are overwritten by nested_vmx_setup() when nested=1. 6058 */ 6059 static int handle_vmx_instruction(struct kvm_vcpu *vcpu) 6060 { 6061 kvm_queue_exception(vcpu, UD_VECTOR); 6062 return 1; 6063 } 6064 6065 #ifndef CONFIG_X86_SGX_KVM 6066 static int handle_encls(struct kvm_vcpu *vcpu) 6067 { 6068 /* 6069 * SGX virtualization is disabled. There is no software enable bit for 6070 * SGX, so KVM intercepts all ENCLS leafs and injects a #UD to prevent 6071 * the guest from executing ENCLS (when SGX is supported by hardware). 6072 */ 6073 kvm_queue_exception(vcpu, UD_VECTOR); 6074 return 1; 6075 } 6076 #endif /* CONFIG_X86_SGX_KVM */ 6077 6078 static int handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) 6079 { 6080 /* 6081 * Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK 6082 * VM-Exits. Unconditionally set the flag here and leave the handling to 6083 * vmx_handle_exit(). 6084 */ 6085 to_vmx(vcpu)->exit_reason.bus_lock_detected = true; 6086 return 1; 6087 } 6088 6089 static int handle_notify(struct kvm_vcpu *vcpu) 6090 { 6091 unsigned long exit_qual = vmx_get_exit_qual(vcpu); 6092 bool context_invalid = exit_qual & NOTIFY_VM_CONTEXT_INVALID; 6093 6094 ++vcpu->stat.notify_window_exits; 6095 6096 /* 6097 * Notify VM exit happened while executing iret from NMI, 6098 * "blocked by NMI" bit has to be set before next VM entry. 6099 */ 6100 if (enable_vnmi && (exit_qual & INTR_INFO_UNBLOCK_NMI)) 6101 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 6102 GUEST_INTR_STATE_NMI); 6103 6104 if (vcpu->kvm->arch.notify_vmexit_flags & KVM_X86_NOTIFY_VMEXIT_USER || 6105 context_invalid) { 6106 vcpu->run->exit_reason = KVM_EXIT_NOTIFY; 6107 vcpu->run->notify.flags = context_invalid ? 6108 KVM_NOTIFY_CONTEXT_INVALID : 0; 6109 return 0; 6110 } 6111 6112 return 1; 6113 } 6114 6115 /* 6116 * The exit handlers return 1 if the exit was handled fully and guest execution 6117 * may resume. Otherwise they set the kvm_run parameter to indicate what needs 6118 * to be done to userspace and return 0. 6119 */ 6120 static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { 6121 [EXIT_REASON_EXCEPTION_NMI] = handle_exception_nmi, 6122 [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt, 6123 [EXIT_REASON_TRIPLE_FAULT] = handle_triple_fault, 6124 [EXIT_REASON_NMI_WINDOW] = handle_nmi_window, 6125 [EXIT_REASON_IO_INSTRUCTION] = handle_io, 6126 [EXIT_REASON_CR_ACCESS] = handle_cr, 6127 [EXIT_REASON_DR_ACCESS] = handle_dr, 6128 [EXIT_REASON_CPUID] = kvm_emulate_cpuid, 6129 [EXIT_REASON_MSR_READ] = kvm_emulate_rdmsr, 6130 [EXIT_REASON_MSR_WRITE] = kvm_emulate_wrmsr, 6131 [EXIT_REASON_INTERRUPT_WINDOW] = handle_interrupt_window, 6132 [EXIT_REASON_HLT] = kvm_emulate_halt, 6133 [EXIT_REASON_INVD] = kvm_emulate_invd, 6134 [EXIT_REASON_INVLPG] = handle_invlpg, 6135 [EXIT_REASON_RDPMC] = kvm_emulate_rdpmc, 6136 [EXIT_REASON_VMCALL] = kvm_emulate_hypercall, 6137 [EXIT_REASON_VMCLEAR] = handle_vmx_instruction, 6138 [EXIT_REASON_VMLAUNCH] = handle_vmx_instruction, 6139 [EXIT_REASON_VMPTRLD] = handle_vmx_instruction, 6140 [EXIT_REASON_VMPTRST] = handle_vmx_instruction, 6141 [EXIT_REASON_VMREAD] = handle_vmx_instruction, 6142 [EXIT_REASON_VMRESUME] = handle_vmx_instruction, 6143 [EXIT_REASON_VMWRITE] = handle_vmx_instruction, 6144 [EXIT_REASON_VMOFF] = handle_vmx_instruction, 6145 [EXIT_REASON_VMON] = handle_vmx_instruction, 6146 [EXIT_REASON_TPR_BELOW_THRESHOLD] = handle_tpr_below_threshold, 6147 [EXIT_REASON_APIC_ACCESS] = handle_apic_access, 6148 [EXIT_REASON_APIC_WRITE] = handle_apic_write, 6149 [EXIT_REASON_EOI_INDUCED] = handle_apic_eoi_induced, 6150 [EXIT_REASON_WBINVD] = kvm_emulate_wbinvd, 6151 [EXIT_REASON_XSETBV] = kvm_emulate_xsetbv, 6152 [EXIT_REASON_TASK_SWITCH] = handle_task_switch, 6153 [EXIT_REASON_MCE_DURING_VMENTRY] = handle_machine_check, 6154 [EXIT_REASON_GDTR_IDTR] = handle_desc, 6155 [EXIT_REASON_LDTR_TR] = handle_desc, 6156 [EXIT_REASON_EPT_VIOLATION] = handle_ept_violation, 6157 [EXIT_REASON_EPT_MISCONFIG] = handle_ept_misconfig, 6158 [EXIT_REASON_PAUSE_INSTRUCTION] = handle_pause, 6159 [EXIT_REASON_MWAIT_INSTRUCTION] = kvm_emulate_mwait, 6160 [EXIT_REASON_MONITOR_TRAP_FLAG] = handle_monitor_trap, 6161 [EXIT_REASON_MONITOR_INSTRUCTION] = kvm_emulate_monitor, 6162 [EXIT_REASON_INVEPT] = handle_vmx_instruction, 6163 [EXIT_REASON_INVVPID] = handle_vmx_instruction, 6164 [EXIT_REASON_RDRAND] = kvm_handle_invalid_op, 6165 [EXIT_REASON_RDSEED] = kvm_handle_invalid_op, 6166 [EXIT_REASON_PML_FULL] = handle_pml_full, 6167 [EXIT_REASON_INVPCID] = handle_invpcid, 6168 [EXIT_REASON_VMFUNC] = handle_vmx_instruction, 6169 [EXIT_REASON_PREEMPTION_TIMER] = handle_preemption_timer, 6170 [EXIT_REASON_ENCLS] = handle_encls, 6171 [EXIT_REASON_BUS_LOCK] = handle_bus_lock_vmexit, 6172 [EXIT_REASON_NOTIFY] = handle_notify, 6173 }; 6174 6175 static const int kvm_vmx_max_exit_handlers = 6176 ARRAY_SIZE(kvm_vmx_exit_handlers); 6177 6178 void vmx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, 6179 u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code) 6180 { 6181 struct vcpu_vmx *vmx = to_vmx(vcpu); 6182 6183 *reason = vmx->exit_reason.full; 6184 *info1 = vmx_get_exit_qual(vcpu); 6185 if (!(vmx->exit_reason.failed_vmentry)) { 6186 *info2 = vmx->idt_vectoring_info; 6187 *intr_info = vmx_get_intr_info(vcpu); 6188 if (is_exception_with_error_code(*intr_info)) 6189 *error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); 6190 else 6191 *error_code = 0; 6192 } else { 6193 *info2 = 0; 6194 *intr_info = 0; 6195 *error_code = 0; 6196 } 6197 } 6198 6199 static void vmx_destroy_pml_buffer(struct vcpu_vmx *vmx) 6200 { 6201 if (vmx->pml_pg) { 6202 __free_page(vmx->pml_pg); 6203 vmx->pml_pg = NULL; 6204 } 6205 } 6206 6207 static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu) 6208 { 6209 struct vcpu_vmx *vmx = to_vmx(vcpu); 6210 u64 *pml_buf; 6211 u16 pml_idx; 6212 6213 pml_idx = vmcs_read16(GUEST_PML_INDEX); 6214 6215 /* Do nothing if PML buffer is empty */ 6216 if (pml_idx == (PML_ENTITY_NUM - 1)) 6217 return; 6218 6219 /* PML index always points to next available PML buffer entity */ 6220 if (pml_idx >= PML_ENTITY_NUM) 6221 pml_idx = 0; 6222 else 6223 pml_idx++; 6224 6225 pml_buf = page_address(vmx->pml_pg); 6226 for (; pml_idx < PML_ENTITY_NUM; pml_idx++) { 6227 u64 gpa; 6228 6229 gpa = pml_buf[pml_idx]; 6230 WARN_ON(gpa & (PAGE_SIZE - 1)); 6231 kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT); 6232 } 6233 6234 /* reset PML index */ 6235 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1); 6236 } 6237 6238 static void vmx_dump_sel(char *name, uint32_t sel) 6239 { 6240 pr_err("%s sel=0x%04x, attr=0x%05x, limit=0x%08x, base=0x%016lx\n", 6241 name, vmcs_read16(sel), 6242 vmcs_read32(sel + GUEST_ES_AR_BYTES - GUEST_ES_SELECTOR), 6243 vmcs_read32(sel + GUEST_ES_LIMIT - GUEST_ES_SELECTOR), 6244 vmcs_readl(sel + GUEST_ES_BASE - GUEST_ES_SELECTOR)); 6245 } 6246 6247 static void vmx_dump_dtsel(char *name, uint32_t limit) 6248 { 6249 pr_err("%s limit=0x%08x, base=0x%016lx\n", 6250 name, vmcs_read32(limit), 6251 vmcs_readl(limit + GUEST_GDTR_BASE - GUEST_GDTR_LIMIT)); 6252 } 6253 6254 static void vmx_dump_msrs(char *name, struct vmx_msrs *m) 6255 { 6256 unsigned int i; 6257 struct vmx_msr_entry *e; 6258 6259 pr_err("MSR %s:\n", name); 6260 for (i = 0, e = m->val; i < m->nr; ++i, ++e) 6261 pr_err(" %2d: msr=0x%08x value=0x%016llx\n", i, e->index, e->value); 6262 } 6263 6264 void dump_vmcs(struct kvm_vcpu *vcpu) 6265 { 6266 struct vcpu_vmx *vmx = to_vmx(vcpu); 6267 u32 vmentry_ctl, vmexit_ctl; 6268 u32 cpu_based_exec_ctrl, pin_based_exec_ctrl, secondary_exec_control; 6269 u64 tertiary_exec_control; 6270 unsigned long cr4; 6271 int efer_slot; 6272 6273 if (!dump_invalid_vmcs) { 6274 pr_warn_ratelimited("set kvm_intel.dump_invalid_vmcs=1 to dump internal KVM state.\n"); 6275 return; 6276 } 6277 6278 vmentry_ctl = vmcs_read32(VM_ENTRY_CONTROLS); 6279 vmexit_ctl = vmcs_read32(VM_EXIT_CONTROLS); 6280 cpu_based_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL); 6281 pin_based_exec_ctrl = vmcs_read32(PIN_BASED_VM_EXEC_CONTROL); 6282 cr4 = vmcs_readl(GUEST_CR4); 6283 6284 if (cpu_has_secondary_exec_ctrls()) 6285 secondary_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL); 6286 else 6287 secondary_exec_control = 0; 6288 6289 if (cpu_has_tertiary_exec_ctrls()) 6290 tertiary_exec_control = vmcs_read64(TERTIARY_VM_EXEC_CONTROL); 6291 else 6292 tertiary_exec_control = 0; 6293 6294 pr_err("VMCS %p, last attempted VM-entry on CPU %d\n", 6295 vmx->loaded_vmcs->vmcs, vcpu->arch.last_vmentry_cpu); 6296 pr_err("*** Guest State ***\n"); 6297 pr_err("CR0: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n", 6298 vmcs_readl(GUEST_CR0), vmcs_readl(CR0_READ_SHADOW), 6299 vmcs_readl(CR0_GUEST_HOST_MASK)); 6300 pr_err("CR4: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n", 6301 cr4, vmcs_readl(CR4_READ_SHADOW), vmcs_readl(CR4_GUEST_HOST_MASK)); 6302 pr_err("CR3 = 0x%016lx\n", vmcs_readl(GUEST_CR3)); 6303 if (cpu_has_vmx_ept()) { 6304 pr_err("PDPTR0 = 0x%016llx PDPTR1 = 0x%016llx\n", 6305 vmcs_read64(GUEST_PDPTR0), vmcs_read64(GUEST_PDPTR1)); 6306 pr_err("PDPTR2 = 0x%016llx PDPTR3 = 0x%016llx\n", 6307 vmcs_read64(GUEST_PDPTR2), vmcs_read64(GUEST_PDPTR3)); 6308 } 6309 pr_err("RSP = 0x%016lx RIP = 0x%016lx\n", 6310 vmcs_readl(GUEST_RSP), vmcs_readl(GUEST_RIP)); 6311 pr_err("RFLAGS=0x%08lx DR7 = 0x%016lx\n", 6312 vmcs_readl(GUEST_RFLAGS), vmcs_readl(GUEST_DR7)); 6313 pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n", 6314 vmcs_readl(GUEST_SYSENTER_ESP), 6315 vmcs_read32(GUEST_SYSENTER_CS), vmcs_readl(GUEST_SYSENTER_EIP)); 6316 vmx_dump_sel("CS: ", GUEST_CS_SELECTOR); 6317 vmx_dump_sel("DS: ", GUEST_DS_SELECTOR); 6318 vmx_dump_sel("SS: ", GUEST_SS_SELECTOR); 6319 vmx_dump_sel("ES: ", GUEST_ES_SELECTOR); 6320 vmx_dump_sel("FS: ", GUEST_FS_SELECTOR); 6321 vmx_dump_sel("GS: ", GUEST_GS_SELECTOR); 6322 vmx_dump_dtsel("GDTR:", GUEST_GDTR_LIMIT); 6323 vmx_dump_sel("LDTR:", GUEST_LDTR_SELECTOR); 6324 vmx_dump_dtsel("IDTR:", GUEST_IDTR_LIMIT); 6325 vmx_dump_sel("TR: ", GUEST_TR_SELECTOR); 6326 efer_slot = vmx_find_loadstore_msr_slot(&vmx->msr_autoload.guest, MSR_EFER); 6327 if (vmentry_ctl & VM_ENTRY_LOAD_IA32_EFER) 6328 pr_err("EFER= 0x%016llx\n", vmcs_read64(GUEST_IA32_EFER)); 6329 else if (efer_slot >= 0) 6330 pr_err("EFER= 0x%016llx (autoload)\n", 6331 vmx->msr_autoload.guest.val[efer_slot].value); 6332 else if (vmentry_ctl & VM_ENTRY_IA32E_MODE) 6333 pr_err("EFER= 0x%016llx (effective)\n", 6334 vcpu->arch.efer | (EFER_LMA | EFER_LME)); 6335 else 6336 pr_err("EFER= 0x%016llx (effective)\n", 6337 vcpu->arch.efer & ~(EFER_LMA | EFER_LME)); 6338 if (vmentry_ctl & VM_ENTRY_LOAD_IA32_PAT) 6339 pr_err("PAT = 0x%016llx\n", vmcs_read64(GUEST_IA32_PAT)); 6340 pr_err("DebugCtl = 0x%016llx DebugExceptions = 0x%016lx\n", 6341 vmcs_read64(GUEST_IA32_DEBUGCTL), 6342 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS)); 6343 if (cpu_has_load_perf_global_ctrl() && 6344 vmentry_ctl & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) 6345 pr_err("PerfGlobCtl = 0x%016llx\n", 6346 vmcs_read64(GUEST_IA32_PERF_GLOBAL_CTRL)); 6347 if (vmentry_ctl & VM_ENTRY_LOAD_BNDCFGS) 6348 pr_err("BndCfgS = 0x%016llx\n", vmcs_read64(GUEST_BNDCFGS)); 6349 pr_err("Interruptibility = %08x ActivityState = %08x\n", 6350 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO), 6351 vmcs_read32(GUEST_ACTIVITY_STATE)); 6352 if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) 6353 pr_err("InterruptStatus = %04x\n", 6354 vmcs_read16(GUEST_INTR_STATUS)); 6355 if (vmcs_read32(VM_ENTRY_MSR_LOAD_COUNT) > 0) 6356 vmx_dump_msrs("guest autoload", &vmx->msr_autoload.guest); 6357 if (vmcs_read32(VM_EXIT_MSR_STORE_COUNT) > 0) 6358 vmx_dump_msrs("guest autostore", &vmx->msr_autostore.guest); 6359 6360 pr_err("*** Host State ***\n"); 6361 pr_err("RIP = 0x%016lx RSP = 0x%016lx\n", 6362 vmcs_readl(HOST_RIP), vmcs_readl(HOST_RSP)); 6363 pr_err("CS=%04x SS=%04x DS=%04x ES=%04x FS=%04x GS=%04x TR=%04x\n", 6364 vmcs_read16(HOST_CS_SELECTOR), vmcs_read16(HOST_SS_SELECTOR), 6365 vmcs_read16(HOST_DS_SELECTOR), vmcs_read16(HOST_ES_SELECTOR), 6366 vmcs_read16(HOST_FS_SELECTOR), vmcs_read16(HOST_GS_SELECTOR), 6367 vmcs_read16(HOST_TR_SELECTOR)); 6368 pr_err("FSBase=%016lx GSBase=%016lx TRBase=%016lx\n", 6369 vmcs_readl(HOST_FS_BASE), vmcs_readl(HOST_GS_BASE), 6370 vmcs_readl(HOST_TR_BASE)); 6371 pr_err("GDTBase=%016lx IDTBase=%016lx\n", 6372 vmcs_readl(HOST_GDTR_BASE), vmcs_readl(HOST_IDTR_BASE)); 6373 pr_err("CR0=%016lx CR3=%016lx CR4=%016lx\n", 6374 vmcs_readl(HOST_CR0), vmcs_readl(HOST_CR3), 6375 vmcs_readl(HOST_CR4)); 6376 pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n", 6377 vmcs_readl(HOST_IA32_SYSENTER_ESP), 6378 vmcs_read32(HOST_IA32_SYSENTER_CS), 6379 vmcs_readl(HOST_IA32_SYSENTER_EIP)); 6380 if (vmexit_ctl & VM_EXIT_LOAD_IA32_EFER) 6381 pr_err("EFER= 0x%016llx\n", vmcs_read64(HOST_IA32_EFER)); 6382 if (vmexit_ctl & VM_EXIT_LOAD_IA32_PAT) 6383 pr_err("PAT = 0x%016llx\n", vmcs_read64(HOST_IA32_PAT)); 6384 if (cpu_has_load_perf_global_ctrl() && 6385 vmexit_ctl & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) 6386 pr_err("PerfGlobCtl = 0x%016llx\n", 6387 vmcs_read64(HOST_IA32_PERF_GLOBAL_CTRL)); 6388 if (vmcs_read32(VM_EXIT_MSR_LOAD_COUNT) > 0) 6389 vmx_dump_msrs("host autoload", &vmx->msr_autoload.host); 6390 6391 pr_err("*** Control State ***\n"); 6392 pr_err("CPUBased=0x%08x SecondaryExec=0x%08x TertiaryExec=0x%016llx\n", 6393 cpu_based_exec_ctrl, secondary_exec_control, tertiary_exec_control); 6394 pr_err("PinBased=0x%08x EntryControls=%08x ExitControls=%08x\n", 6395 pin_based_exec_ctrl, vmentry_ctl, vmexit_ctl); 6396 pr_err("ExceptionBitmap=%08x PFECmask=%08x PFECmatch=%08x\n", 6397 vmcs_read32(EXCEPTION_BITMAP), 6398 vmcs_read32(PAGE_FAULT_ERROR_CODE_MASK), 6399 vmcs_read32(PAGE_FAULT_ERROR_CODE_MATCH)); 6400 pr_err("VMEntry: intr_info=%08x errcode=%08x ilen=%08x\n", 6401 vmcs_read32(VM_ENTRY_INTR_INFO_FIELD), 6402 vmcs_read32(VM_ENTRY_EXCEPTION_ERROR_CODE), 6403 vmcs_read32(VM_ENTRY_INSTRUCTION_LEN)); 6404 pr_err("VMExit: intr_info=%08x errcode=%08x ilen=%08x\n", 6405 vmcs_read32(VM_EXIT_INTR_INFO), 6406 vmcs_read32(VM_EXIT_INTR_ERROR_CODE), 6407 vmcs_read32(VM_EXIT_INSTRUCTION_LEN)); 6408 pr_err(" reason=%08x qualification=%016lx\n", 6409 vmcs_read32(VM_EXIT_REASON), vmcs_readl(EXIT_QUALIFICATION)); 6410 pr_err("IDTVectoring: info=%08x errcode=%08x\n", 6411 vmcs_read32(IDT_VECTORING_INFO_FIELD), 6412 vmcs_read32(IDT_VECTORING_ERROR_CODE)); 6413 pr_err("TSC Offset = 0x%016llx\n", vmcs_read64(TSC_OFFSET)); 6414 if (secondary_exec_control & SECONDARY_EXEC_TSC_SCALING) 6415 pr_err("TSC Multiplier = 0x%016llx\n", 6416 vmcs_read64(TSC_MULTIPLIER)); 6417 if (cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW) { 6418 if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) { 6419 u16 status = vmcs_read16(GUEST_INTR_STATUS); 6420 pr_err("SVI|RVI = %02x|%02x ", status >> 8, status & 0xff); 6421 } 6422 pr_cont("TPR Threshold = 0x%02x\n", vmcs_read32(TPR_THRESHOLD)); 6423 if (secondary_exec_control & SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) 6424 pr_err("APIC-access addr = 0x%016llx ", vmcs_read64(APIC_ACCESS_ADDR)); 6425 pr_cont("virt-APIC addr = 0x%016llx\n", vmcs_read64(VIRTUAL_APIC_PAGE_ADDR)); 6426 } 6427 if (pin_based_exec_ctrl & PIN_BASED_POSTED_INTR) 6428 pr_err("PostedIntrVec = 0x%02x\n", vmcs_read16(POSTED_INTR_NV)); 6429 if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT)) 6430 pr_err("EPT pointer = 0x%016llx\n", vmcs_read64(EPT_POINTER)); 6431 if (secondary_exec_control & SECONDARY_EXEC_PAUSE_LOOP_EXITING) 6432 pr_err("PLE Gap=%08x Window=%08x\n", 6433 vmcs_read32(PLE_GAP), vmcs_read32(PLE_WINDOW)); 6434 if (secondary_exec_control & SECONDARY_EXEC_ENABLE_VPID) 6435 pr_err("Virtual processor ID = 0x%04x\n", 6436 vmcs_read16(VIRTUAL_PROCESSOR_ID)); 6437 if (secondary_exec_control & SECONDARY_EXEC_EPT_VIOLATION_VE) { 6438 struct vmx_ve_information *ve_info = vmx->ve_info; 6439 u64 ve_info_pa = vmcs_read64(VE_INFORMATION_ADDRESS); 6440 6441 /* 6442 * If KVM is dumping the VMCS, then something has gone wrong 6443 * already. Derefencing an address from the VMCS, which could 6444 * very well be corrupted, is a terrible idea. The virtual 6445 * address is known so use it. 6446 */ 6447 pr_err("VE info address = 0x%016llx%s\n", ve_info_pa, 6448 ve_info_pa == __pa(ve_info) ? "" : "(corrupted!)"); 6449 pr_err("ve_info: 0x%08x 0x%08x 0x%016llx 0x%016llx 0x%016llx 0x%04x\n", 6450 ve_info->exit_reason, ve_info->delivery, 6451 ve_info->exit_qualification, 6452 ve_info->guest_linear_address, 6453 ve_info->guest_physical_address, ve_info->eptp_index); 6454 } 6455 } 6456 6457 /* 6458 * The guest has exited. See if we can fix it or if we need userspace 6459 * assistance. 6460 */ 6461 static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) 6462 { 6463 struct vcpu_vmx *vmx = to_vmx(vcpu); 6464 union vmx_exit_reason exit_reason = vmx->exit_reason; 6465 u32 vectoring_info = vmx->idt_vectoring_info; 6466 u16 exit_handler_index; 6467 6468 /* 6469 * Flush logged GPAs PML buffer, this will make dirty_bitmap more 6470 * updated. Another good is, in kvm_vm_ioctl_get_dirty_log, before 6471 * querying dirty_bitmap, we only need to kick all vcpus out of guest 6472 * mode as if vcpus is in root mode, the PML buffer must has been 6473 * flushed already. Note, PML is never enabled in hardware while 6474 * running L2. 6475 */ 6476 if (enable_pml && !is_guest_mode(vcpu)) 6477 vmx_flush_pml_buffer(vcpu); 6478 6479 /* 6480 * KVM should never reach this point with a pending nested VM-Enter. 6481 * More specifically, short-circuiting VM-Entry to emulate L2 due to 6482 * invalid guest state should never happen as that means KVM knowingly 6483 * allowed a nested VM-Enter with an invalid vmcs12. More below. 6484 */ 6485 if (KVM_BUG_ON(vmx->nested.nested_run_pending, vcpu->kvm)) 6486 return -EIO; 6487 6488 if (is_guest_mode(vcpu)) { 6489 /* 6490 * PML is never enabled when running L2, bail immediately if a 6491 * PML full exit occurs as something is horribly wrong. 6492 */ 6493 if (exit_reason.basic == EXIT_REASON_PML_FULL) 6494 goto unexpected_vmexit; 6495 6496 /* 6497 * The host physical addresses of some pages of guest memory 6498 * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC 6499 * Page). The CPU may write to these pages via their host 6500 * physical address while L2 is running, bypassing any 6501 * address-translation-based dirty tracking (e.g. EPT write 6502 * protection). 6503 * 6504 * Mark them dirty on every exit from L2 to prevent them from 6505 * getting out of sync with dirty tracking. 6506 */ 6507 nested_mark_vmcs12_pages_dirty(vcpu); 6508 6509 /* 6510 * Synthesize a triple fault if L2 state is invalid. In normal 6511 * operation, nested VM-Enter rejects any attempt to enter L2 6512 * with invalid state. However, those checks are skipped if 6513 * state is being stuffed via RSM or KVM_SET_NESTED_STATE. If 6514 * L2 state is invalid, it means either L1 modified SMRAM state 6515 * or userspace provided bad state. Synthesize TRIPLE_FAULT as 6516 * doing so is architecturally allowed in the RSM case, and is 6517 * the least awful solution for the userspace case without 6518 * risking false positives. 6519 */ 6520 if (vmx->emulation_required) { 6521 nested_vmx_vmexit(vcpu, EXIT_REASON_TRIPLE_FAULT, 0, 0); 6522 return 1; 6523 } 6524 6525 if (nested_vmx_reflect_vmexit(vcpu)) 6526 return 1; 6527 } 6528 6529 /* If guest state is invalid, start emulating. L2 is handled above. */ 6530 if (vmx->emulation_required) 6531 return handle_invalid_guest_state(vcpu); 6532 6533 if (exit_reason.failed_vmentry) { 6534 dump_vmcs(vcpu); 6535 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; 6536 vcpu->run->fail_entry.hardware_entry_failure_reason 6537 = exit_reason.full; 6538 vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu; 6539 return 0; 6540 } 6541 6542 if (unlikely(vmx->fail)) { 6543 dump_vmcs(vcpu); 6544 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; 6545 vcpu->run->fail_entry.hardware_entry_failure_reason 6546 = vmcs_read32(VM_INSTRUCTION_ERROR); 6547 vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu; 6548 return 0; 6549 } 6550 6551 /* 6552 * Note: 6553 * Do not try to fix EXIT_REASON_EPT_MISCONFIG if it caused by 6554 * delivery event since it indicates guest is accessing MMIO. 6555 * The vm-exit can be triggered again after return to guest that 6556 * will cause infinite loop. 6557 */ 6558 if ((vectoring_info & VECTORING_INFO_VALID_MASK) && 6559 (exit_reason.basic != EXIT_REASON_EXCEPTION_NMI && 6560 exit_reason.basic != EXIT_REASON_EPT_VIOLATION && 6561 exit_reason.basic != EXIT_REASON_PML_FULL && 6562 exit_reason.basic != EXIT_REASON_APIC_ACCESS && 6563 exit_reason.basic != EXIT_REASON_TASK_SWITCH && 6564 exit_reason.basic != EXIT_REASON_NOTIFY)) { 6565 int ndata = 3; 6566 6567 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 6568 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_DELIVERY_EV; 6569 vcpu->run->internal.data[0] = vectoring_info; 6570 vcpu->run->internal.data[1] = exit_reason.full; 6571 vcpu->run->internal.data[2] = vmx_get_exit_qual(vcpu); 6572 if (exit_reason.basic == EXIT_REASON_EPT_MISCONFIG) { 6573 vcpu->run->internal.data[ndata++] = 6574 vmcs_read64(GUEST_PHYSICAL_ADDRESS); 6575 } 6576 vcpu->run->internal.data[ndata++] = vcpu->arch.last_vmentry_cpu; 6577 vcpu->run->internal.ndata = ndata; 6578 return 0; 6579 } 6580 6581 if (unlikely(!enable_vnmi && 6582 vmx->loaded_vmcs->soft_vnmi_blocked)) { 6583 if (!vmx_interrupt_blocked(vcpu)) { 6584 vmx->loaded_vmcs->soft_vnmi_blocked = 0; 6585 } else if (vmx->loaded_vmcs->vnmi_blocked_time > 1000000000LL && 6586 vcpu->arch.nmi_pending) { 6587 /* 6588 * This CPU don't support us in finding the end of an 6589 * NMI-blocked window if the guest runs with IRQs 6590 * disabled. So we pull the trigger after 1 s of 6591 * futile waiting, but inform the user about this. 6592 */ 6593 printk(KERN_WARNING "%s: Breaking out of NMI-blocked " 6594 "state on VCPU %d after 1 s timeout\n", 6595 __func__, vcpu->vcpu_id); 6596 vmx->loaded_vmcs->soft_vnmi_blocked = 0; 6597 } 6598 } 6599 6600 if (exit_fastpath != EXIT_FASTPATH_NONE) 6601 return 1; 6602 6603 if (exit_reason.basic >= kvm_vmx_max_exit_handlers) 6604 goto unexpected_vmexit; 6605 #ifdef CONFIG_MITIGATION_RETPOLINE 6606 if (exit_reason.basic == EXIT_REASON_MSR_WRITE) 6607 return kvm_emulate_wrmsr(vcpu); 6608 else if (exit_reason.basic == EXIT_REASON_PREEMPTION_TIMER) 6609 return handle_preemption_timer(vcpu); 6610 else if (exit_reason.basic == EXIT_REASON_INTERRUPT_WINDOW) 6611 return handle_interrupt_window(vcpu); 6612 else if (exit_reason.basic == EXIT_REASON_EXTERNAL_INTERRUPT) 6613 return handle_external_interrupt(vcpu); 6614 else if (exit_reason.basic == EXIT_REASON_HLT) 6615 return kvm_emulate_halt(vcpu); 6616 else if (exit_reason.basic == EXIT_REASON_EPT_MISCONFIG) 6617 return handle_ept_misconfig(vcpu); 6618 #endif 6619 6620 exit_handler_index = array_index_nospec((u16)exit_reason.basic, 6621 kvm_vmx_max_exit_handlers); 6622 if (!kvm_vmx_exit_handlers[exit_handler_index]) 6623 goto unexpected_vmexit; 6624 6625 return kvm_vmx_exit_handlers[exit_handler_index](vcpu); 6626 6627 unexpected_vmexit: 6628 vcpu_unimpl(vcpu, "vmx: unexpected exit reason 0x%x\n", 6629 exit_reason.full); 6630 dump_vmcs(vcpu); 6631 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 6632 vcpu->run->internal.suberror = 6633 KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON; 6634 vcpu->run->internal.ndata = 2; 6635 vcpu->run->internal.data[0] = exit_reason.full; 6636 vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu; 6637 return 0; 6638 } 6639 6640 int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) 6641 { 6642 int ret = __vmx_handle_exit(vcpu, exit_fastpath); 6643 6644 /* 6645 * Exit to user space when bus lock detected to inform that there is 6646 * a bus lock in guest. 6647 */ 6648 if (to_vmx(vcpu)->exit_reason.bus_lock_detected) { 6649 if (ret > 0) 6650 vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; 6651 6652 vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK; 6653 return 0; 6654 } 6655 return ret; 6656 } 6657 6658 /* 6659 * Software based L1D cache flush which is used when microcode providing 6660 * the cache control MSR is not loaded. 6661 * 6662 * The L1D cache is 32 KiB on Nehalem and later microarchitectures, but to 6663 * flush it is required to read in 64 KiB because the replacement algorithm 6664 * is not exactly LRU. This could be sized at runtime via topology 6665 * information but as all relevant affected CPUs have 32KiB L1D cache size 6666 * there is no point in doing so. 6667 */ 6668 static noinstr void vmx_l1d_flush(struct kvm_vcpu *vcpu) 6669 { 6670 int size = PAGE_SIZE << L1D_CACHE_ORDER; 6671 6672 /* 6673 * This code is only executed when the flush mode is 'cond' or 6674 * 'always' 6675 */ 6676 if (static_branch_likely(&vmx_l1d_flush_cond)) { 6677 bool flush_l1d; 6678 6679 /* 6680 * Clear the per-vcpu flush bit, it gets set again 6681 * either from vcpu_run() or from one of the unsafe 6682 * VMEXIT handlers. 6683 */ 6684 flush_l1d = vcpu->arch.l1tf_flush_l1d; 6685 vcpu->arch.l1tf_flush_l1d = false; 6686 6687 /* 6688 * Clear the per-cpu flush bit, it gets set again from 6689 * the interrupt handlers. 6690 */ 6691 flush_l1d |= kvm_get_cpu_l1tf_flush_l1d(); 6692 kvm_clear_cpu_l1tf_flush_l1d(); 6693 6694 if (!flush_l1d) 6695 return; 6696 } 6697 6698 vcpu->stat.l1d_flush++; 6699 6700 if (static_cpu_has(X86_FEATURE_FLUSH_L1D)) { 6701 native_wrmsrl(MSR_IA32_FLUSH_CMD, L1D_FLUSH); 6702 return; 6703 } 6704 6705 asm volatile( 6706 /* First ensure the pages are in the TLB */ 6707 "xorl %%eax, %%eax\n" 6708 ".Lpopulate_tlb:\n\t" 6709 "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t" 6710 "addl $4096, %%eax\n\t" 6711 "cmpl %%eax, %[size]\n\t" 6712 "jne .Lpopulate_tlb\n\t" 6713 "xorl %%eax, %%eax\n\t" 6714 "cpuid\n\t" 6715 /* Now fill the cache */ 6716 "xorl %%eax, %%eax\n" 6717 ".Lfill_cache:\n" 6718 "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t" 6719 "addl $64, %%eax\n\t" 6720 "cmpl %%eax, %[size]\n\t" 6721 "jne .Lfill_cache\n\t" 6722 "lfence\n" 6723 :: [flush_pages] "r" (vmx_l1d_flush_pages), 6724 [size] "r" (size) 6725 : "eax", "ebx", "ecx", "edx"); 6726 } 6727 6728 void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) 6729 { 6730 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 6731 int tpr_threshold; 6732 6733 if (is_guest_mode(vcpu) && 6734 nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) 6735 return; 6736 6737 tpr_threshold = (irr == -1 || tpr < irr) ? 0 : irr; 6738 if (is_guest_mode(vcpu)) 6739 to_vmx(vcpu)->nested.l1_tpr_threshold = tpr_threshold; 6740 else 6741 vmcs_write32(TPR_THRESHOLD, tpr_threshold); 6742 } 6743 6744 void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu) 6745 { 6746 struct vcpu_vmx *vmx = to_vmx(vcpu); 6747 u32 sec_exec_control; 6748 6749 if (!lapic_in_kernel(vcpu)) 6750 return; 6751 6752 if (!flexpriority_enabled && 6753 !cpu_has_vmx_virtualize_x2apic_mode()) 6754 return; 6755 6756 /* Postpone execution until vmcs01 is the current VMCS. */ 6757 if (is_guest_mode(vcpu)) { 6758 vmx->nested.change_vmcs01_virtual_apic_mode = true; 6759 return; 6760 } 6761 6762 sec_exec_control = secondary_exec_controls_get(vmx); 6763 sec_exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 6764 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE); 6765 6766 switch (kvm_get_apic_mode(vcpu)) { 6767 case LAPIC_MODE_INVALID: 6768 WARN_ONCE(true, "Invalid local APIC state"); 6769 break; 6770 case LAPIC_MODE_DISABLED: 6771 break; 6772 case LAPIC_MODE_XAPIC: 6773 if (flexpriority_enabled) { 6774 sec_exec_control |= 6775 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 6776 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 6777 6778 /* 6779 * Flush the TLB, reloading the APIC access page will 6780 * only do so if its physical address has changed, but 6781 * the guest may have inserted a non-APIC mapping into 6782 * the TLB while the APIC access page was disabled. 6783 */ 6784 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 6785 } 6786 break; 6787 case LAPIC_MODE_X2APIC: 6788 if (cpu_has_vmx_virtualize_x2apic_mode()) 6789 sec_exec_control |= 6790 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE; 6791 break; 6792 } 6793 secondary_exec_controls_set(vmx, sec_exec_control); 6794 6795 vmx_update_msr_bitmap_x2apic(vcpu); 6796 } 6797 6798 void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu) 6799 { 6800 const gfn_t gfn = APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT; 6801 struct kvm *kvm = vcpu->kvm; 6802 struct kvm_memslots *slots = kvm_memslots(kvm); 6803 struct kvm_memory_slot *slot; 6804 unsigned long mmu_seq; 6805 kvm_pfn_t pfn; 6806 6807 /* Defer reload until vmcs01 is the current VMCS. */ 6808 if (is_guest_mode(vcpu)) { 6809 to_vmx(vcpu)->nested.reload_vmcs01_apic_access_page = true; 6810 return; 6811 } 6812 6813 if (!(secondary_exec_controls_get(to_vmx(vcpu)) & 6814 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) 6815 return; 6816 6817 /* 6818 * Explicitly grab the memslot using KVM's internal slot ID to ensure 6819 * KVM doesn't unintentionally grab a userspace memslot. It _should_ 6820 * be impossible for userspace to create a memslot for the APIC when 6821 * APICv is enabled, but paranoia won't hurt in this case. 6822 */ 6823 slot = id_to_memslot(slots, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT); 6824 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) 6825 return; 6826 6827 /* 6828 * Ensure that the mmu_notifier sequence count is read before KVM 6829 * retrieves the pfn from the primary MMU. Note, the memslot is 6830 * protected by SRCU, not the mmu_notifier. Pairs with the smp_wmb() 6831 * in kvm_mmu_invalidate_end(). 6832 */ 6833 mmu_seq = kvm->mmu_invalidate_seq; 6834 smp_rmb(); 6835 6836 /* 6837 * No need to retry if the memslot does not exist or is invalid. KVM 6838 * controls the APIC-access page memslot, and only deletes the memslot 6839 * if APICv is permanently inhibited, i.e. the memslot won't reappear. 6840 */ 6841 pfn = gfn_to_pfn_memslot(slot, gfn); 6842 if (is_error_noslot_pfn(pfn)) 6843 return; 6844 6845 read_lock(&vcpu->kvm->mmu_lock); 6846 if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) { 6847 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 6848 read_unlock(&vcpu->kvm->mmu_lock); 6849 goto out; 6850 } 6851 6852 vmcs_write64(APIC_ACCESS_ADDR, pfn_to_hpa(pfn)); 6853 read_unlock(&vcpu->kvm->mmu_lock); 6854 6855 /* 6856 * No need for a manual TLB flush at this point, KVM has already done a 6857 * flush if there were SPTEs pointing at the previous page. 6858 */ 6859 out: 6860 /* 6861 * Do not pin apic access page in memory, the MMU notifier 6862 * will call us again if it is migrated or swapped out. 6863 */ 6864 kvm_release_pfn_clean(pfn); 6865 } 6866 6867 void vmx_hwapic_isr_update(int max_isr) 6868 { 6869 u16 status; 6870 u8 old; 6871 6872 if (max_isr == -1) 6873 max_isr = 0; 6874 6875 status = vmcs_read16(GUEST_INTR_STATUS); 6876 old = status >> 8; 6877 if (max_isr != old) { 6878 status &= 0xff; 6879 status |= max_isr << 8; 6880 vmcs_write16(GUEST_INTR_STATUS, status); 6881 } 6882 } 6883 6884 static void vmx_set_rvi(int vector) 6885 { 6886 u16 status; 6887 u8 old; 6888 6889 if (vector == -1) 6890 vector = 0; 6891 6892 status = vmcs_read16(GUEST_INTR_STATUS); 6893 old = (u8)status & 0xff; 6894 if ((u8)vector != old) { 6895 status &= ~0xff; 6896 status |= (u8)vector; 6897 vmcs_write16(GUEST_INTR_STATUS, status); 6898 } 6899 } 6900 6901 void vmx_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr) 6902 { 6903 /* 6904 * When running L2, updating RVI is only relevant when 6905 * vmcs12 virtual-interrupt-delivery enabled. 6906 * However, it can be enabled only when L1 also 6907 * intercepts external-interrupts and in that case 6908 * we should not update vmcs02 RVI but instead intercept 6909 * interrupt. Therefore, do nothing when running L2. 6910 */ 6911 if (!is_guest_mode(vcpu)) 6912 vmx_set_rvi(max_irr); 6913 } 6914 6915 int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu) 6916 { 6917 struct vcpu_vmx *vmx = to_vmx(vcpu); 6918 int max_irr; 6919 bool got_posted_interrupt; 6920 6921 if (KVM_BUG_ON(!enable_apicv, vcpu->kvm)) 6922 return -EIO; 6923 6924 if (pi_test_on(&vmx->pi_desc)) { 6925 pi_clear_on(&vmx->pi_desc); 6926 /* 6927 * IOMMU can write to PID.ON, so the barrier matters even on UP. 6928 * But on x86 this is just a compiler barrier anyway. 6929 */ 6930 smp_mb__after_atomic(); 6931 got_posted_interrupt = 6932 kvm_apic_update_irr(vcpu, vmx->pi_desc.pir, &max_irr); 6933 } else { 6934 max_irr = kvm_lapic_find_highest_irr(vcpu); 6935 got_posted_interrupt = false; 6936 } 6937 6938 /* 6939 * Newly recognized interrupts are injected via either virtual interrupt 6940 * delivery (RVI) or KVM_REQ_EVENT. Virtual interrupt delivery is 6941 * disabled in two cases: 6942 * 6943 * 1) If L2 is running and the vCPU has a new pending interrupt. If L1 6944 * wants to exit on interrupts, KVM_REQ_EVENT is needed to synthesize a 6945 * VM-Exit to L1. If L1 doesn't want to exit, the interrupt is injected 6946 * into L2, but KVM doesn't use virtual interrupt delivery to inject 6947 * interrupts into L2, and so KVM_REQ_EVENT is again needed. 6948 * 6949 * 2) If APICv is disabled for this vCPU, assigned devices may still 6950 * attempt to post interrupts. The posted interrupt vector will cause 6951 * a VM-Exit and the subsequent entry will call sync_pir_to_irr. 6952 */ 6953 if (!is_guest_mode(vcpu) && kvm_vcpu_apicv_active(vcpu)) 6954 vmx_set_rvi(max_irr); 6955 else if (got_posted_interrupt) 6956 kvm_make_request(KVM_REQ_EVENT, vcpu); 6957 6958 return max_irr; 6959 } 6960 6961 void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) 6962 { 6963 if (!kvm_vcpu_apicv_active(vcpu)) 6964 return; 6965 6966 vmcs_write64(EOI_EXIT_BITMAP0, eoi_exit_bitmap[0]); 6967 vmcs_write64(EOI_EXIT_BITMAP1, eoi_exit_bitmap[1]); 6968 vmcs_write64(EOI_EXIT_BITMAP2, eoi_exit_bitmap[2]); 6969 vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]); 6970 } 6971 6972 void vmx_apicv_pre_state_restore(struct kvm_vcpu *vcpu) 6973 { 6974 struct vcpu_vmx *vmx = to_vmx(vcpu); 6975 6976 pi_clear_on(&vmx->pi_desc); 6977 memset(vmx->pi_desc.pir, 0, sizeof(vmx->pi_desc.pir)); 6978 } 6979 6980 void vmx_do_interrupt_irqoff(unsigned long entry); 6981 void vmx_do_nmi_irqoff(void); 6982 6983 static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu) 6984 { 6985 /* 6986 * Save xfd_err to guest_fpu before interrupt is enabled, so the 6987 * MSR value is not clobbered by the host activity before the guest 6988 * has chance to consume it. 6989 * 6990 * Do not blindly read xfd_err here, since this exception might 6991 * be caused by L1 interception on a platform which doesn't 6992 * support xfd at all. 6993 * 6994 * Do it conditionally upon guest_fpu::xfd. xfd_err matters 6995 * only when xfd contains a non-zero value. 6996 * 6997 * Queuing exception is done in vmx_handle_exit. See comment there. 6998 */ 6999 if (vcpu->arch.guest_fpu.fpstate->xfd) 7000 rdmsrl(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); 7001 } 7002 7003 static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info) 7004 { 7005 /* if exit due to PF check for async PF */ 7006 if (is_page_fault(intr_info)) 7007 vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags(); 7008 /* if exit due to NM, handle before interrupts are enabled */ 7009 else if (is_nm_fault(intr_info)) 7010 handle_nm_fault_irqoff(vcpu); 7011 /* Handle machine checks before interrupts are enabled */ 7012 else if (is_machine_check(intr_info)) 7013 kvm_machine_check(); 7014 } 7015 7016 static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu, 7017 u32 intr_info) 7018 { 7019 unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK; 7020 7021 if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm, 7022 "unexpected VM-Exit interrupt info: 0x%x", intr_info)) 7023 return; 7024 7025 kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ); 7026 if (cpu_feature_enabled(X86_FEATURE_FRED)) 7027 fred_entry_from_kvm(EVENT_TYPE_EXTINT, vector); 7028 else 7029 vmx_do_interrupt_irqoff(gate_offset((gate_desc *)host_idt_base + vector)); 7030 kvm_after_interrupt(vcpu); 7031 7032 vcpu->arch.at_instruction_boundary = true; 7033 } 7034 7035 void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu) 7036 { 7037 struct vcpu_vmx *vmx = to_vmx(vcpu); 7038 7039 if (vmx->emulation_required) 7040 return; 7041 7042 if (vmx->exit_reason.basic == EXIT_REASON_EXTERNAL_INTERRUPT) 7043 handle_external_interrupt_irqoff(vcpu, vmx_get_intr_info(vcpu)); 7044 else if (vmx->exit_reason.basic == EXIT_REASON_EXCEPTION_NMI) 7045 handle_exception_irqoff(vcpu, vmx_get_intr_info(vcpu)); 7046 } 7047 7048 /* 7049 * The kvm parameter can be NULL (module initialization, or invocation before 7050 * VM creation). Be sure to check the kvm parameter before using it. 7051 */ 7052 bool vmx_has_emulated_msr(struct kvm *kvm, u32 index) 7053 { 7054 switch (index) { 7055 case MSR_IA32_SMBASE: 7056 if (!IS_ENABLED(CONFIG_KVM_SMM)) 7057 return false; 7058 /* 7059 * We cannot do SMM unless we can run the guest in big 7060 * real mode. 7061 */ 7062 return enable_unrestricted_guest || emulate_invalid_guest_state; 7063 case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR: 7064 return nested; 7065 case MSR_AMD64_VIRT_SPEC_CTRL: 7066 case MSR_AMD64_TSC_RATIO: 7067 /* This is AMD only. */ 7068 return false; 7069 default: 7070 return true; 7071 } 7072 } 7073 7074 static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx) 7075 { 7076 u32 exit_intr_info; 7077 bool unblock_nmi; 7078 u8 vector; 7079 bool idtv_info_valid; 7080 7081 idtv_info_valid = vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK; 7082 7083 if (enable_vnmi) { 7084 if (vmx->loaded_vmcs->nmi_known_unmasked) 7085 return; 7086 7087 exit_intr_info = vmx_get_intr_info(&vmx->vcpu); 7088 unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0; 7089 vector = exit_intr_info & INTR_INFO_VECTOR_MASK; 7090 /* 7091 * SDM 3: 27.7.1.2 (September 2008) 7092 * Re-set bit "block by NMI" before VM entry if vmexit caused by 7093 * a guest IRET fault. 7094 * SDM 3: 23.2.2 (September 2008) 7095 * Bit 12 is undefined in any of the following cases: 7096 * If the VM exit sets the valid bit in the IDT-vectoring 7097 * information field. 7098 * If the VM exit is due to a double fault. 7099 */ 7100 if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi && 7101 vector != DF_VECTOR && !idtv_info_valid) 7102 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 7103 GUEST_INTR_STATE_NMI); 7104 else 7105 vmx->loaded_vmcs->nmi_known_unmasked = 7106 !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) 7107 & GUEST_INTR_STATE_NMI); 7108 } else if (unlikely(vmx->loaded_vmcs->soft_vnmi_blocked)) 7109 vmx->loaded_vmcs->vnmi_blocked_time += 7110 ktime_to_ns(ktime_sub(ktime_get(), 7111 vmx->loaded_vmcs->entry_time)); 7112 } 7113 7114 static void __vmx_complete_interrupts(struct kvm_vcpu *vcpu, 7115 u32 idt_vectoring_info, 7116 int instr_len_field, 7117 int error_code_field) 7118 { 7119 u8 vector; 7120 int type; 7121 bool idtv_info_valid; 7122 7123 idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK; 7124 7125 vcpu->arch.nmi_injected = false; 7126 kvm_clear_exception_queue(vcpu); 7127 kvm_clear_interrupt_queue(vcpu); 7128 7129 if (!idtv_info_valid) 7130 return; 7131 7132 kvm_make_request(KVM_REQ_EVENT, vcpu); 7133 7134 vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK; 7135 type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK; 7136 7137 switch (type) { 7138 case INTR_TYPE_NMI_INTR: 7139 vcpu->arch.nmi_injected = true; 7140 /* 7141 * SDM 3: 27.7.1.2 (September 2008) 7142 * Clear bit "block by NMI" before VM entry if a NMI 7143 * delivery faulted. 7144 */ 7145 vmx_set_nmi_mask(vcpu, false); 7146 break; 7147 case INTR_TYPE_SOFT_EXCEPTION: 7148 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field); 7149 fallthrough; 7150 case INTR_TYPE_HARD_EXCEPTION: 7151 if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) { 7152 u32 err = vmcs_read32(error_code_field); 7153 kvm_requeue_exception_e(vcpu, vector, err); 7154 } else 7155 kvm_requeue_exception(vcpu, vector); 7156 break; 7157 case INTR_TYPE_SOFT_INTR: 7158 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field); 7159 fallthrough; 7160 case INTR_TYPE_EXT_INTR: 7161 kvm_queue_interrupt(vcpu, vector, type == INTR_TYPE_SOFT_INTR); 7162 break; 7163 default: 7164 break; 7165 } 7166 } 7167 7168 static void vmx_complete_interrupts(struct vcpu_vmx *vmx) 7169 { 7170 __vmx_complete_interrupts(&vmx->vcpu, vmx->idt_vectoring_info, 7171 VM_EXIT_INSTRUCTION_LEN, 7172 IDT_VECTORING_ERROR_CODE); 7173 } 7174 7175 void vmx_cancel_injection(struct kvm_vcpu *vcpu) 7176 { 7177 __vmx_complete_interrupts(vcpu, 7178 vmcs_read32(VM_ENTRY_INTR_INFO_FIELD), 7179 VM_ENTRY_INSTRUCTION_LEN, 7180 VM_ENTRY_EXCEPTION_ERROR_CODE); 7181 7182 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); 7183 } 7184 7185 static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx) 7186 { 7187 int i, nr_msrs; 7188 struct perf_guest_switch_msr *msrs; 7189 struct kvm_pmu *pmu = vcpu_to_pmu(&vmx->vcpu); 7190 7191 pmu->host_cross_mapped_mask = 0; 7192 if (pmu->pebs_enable & pmu->global_ctrl) 7193 intel_pmu_cross_mapped_check(pmu); 7194 7195 /* Note, nr_msrs may be garbage if perf_guest_get_msrs() returns NULL. */ 7196 msrs = perf_guest_get_msrs(&nr_msrs, (void *)pmu); 7197 if (!msrs) 7198 return; 7199 7200 for (i = 0; i < nr_msrs; i++) 7201 if (msrs[i].host == msrs[i].guest) 7202 clear_atomic_switch_msr(vmx, msrs[i].msr); 7203 else 7204 add_atomic_switch_msr(vmx, msrs[i].msr, msrs[i].guest, 7205 msrs[i].host, false); 7206 } 7207 7208 static void vmx_update_hv_timer(struct kvm_vcpu *vcpu, bool force_immediate_exit) 7209 { 7210 struct vcpu_vmx *vmx = to_vmx(vcpu); 7211 u64 tscl; 7212 u32 delta_tsc; 7213 7214 if (force_immediate_exit) { 7215 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, 0); 7216 vmx->loaded_vmcs->hv_timer_soft_disabled = false; 7217 } else if (vmx->hv_deadline_tsc != -1) { 7218 tscl = rdtsc(); 7219 if (vmx->hv_deadline_tsc > tscl) 7220 /* set_hv_timer ensures the delta fits in 32-bits */ 7221 delta_tsc = (u32)((vmx->hv_deadline_tsc - tscl) >> 7222 cpu_preemption_timer_multi); 7223 else 7224 delta_tsc = 0; 7225 7226 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, delta_tsc); 7227 vmx->loaded_vmcs->hv_timer_soft_disabled = false; 7228 } else if (!vmx->loaded_vmcs->hv_timer_soft_disabled) { 7229 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, -1); 7230 vmx->loaded_vmcs->hv_timer_soft_disabled = true; 7231 } 7232 } 7233 7234 void noinstr vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp) 7235 { 7236 if (unlikely(host_rsp != vmx->loaded_vmcs->host_state.rsp)) { 7237 vmx->loaded_vmcs->host_state.rsp = host_rsp; 7238 vmcs_writel(HOST_RSP, host_rsp); 7239 } 7240 } 7241 7242 void noinstr vmx_spec_ctrl_restore_host(struct vcpu_vmx *vmx, 7243 unsigned int flags) 7244 { 7245 u64 hostval = this_cpu_read(x86_spec_ctrl_current); 7246 7247 if (!cpu_feature_enabled(X86_FEATURE_MSR_SPEC_CTRL)) 7248 return; 7249 7250 if (flags & VMX_RUN_SAVE_SPEC_CTRL) 7251 vmx->spec_ctrl = __rdmsr(MSR_IA32_SPEC_CTRL); 7252 7253 /* 7254 * If the guest/host SPEC_CTRL values differ, restore the host value. 7255 * 7256 * For legacy IBRS, the IBRS bit always needs to be written after 7257 * transitioning from a less privileged predictor mode, regardless of 7258 * whether the guest/host values differ. 7259 */ 7260 if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) || 7261 vmx->spec_ctrl != hostval) 7262 native_wrmsrl(MSR_IA32_SPEC_CTRL, hostval); 7263 7264 barrier_nospec(); 7265 } 7266 7267 static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu, 7268 bool force_immediate_exit) 7269 { 7270 /* 7271 * If L2 is active, some VMX preemption timer exits can be handled in 7272 * the fastpath even, all other exits must use the slow path. 7273 */ 7274 if (is_guest_mode(vcpu) && 7275 to_vmx(vcpu)->exit_reason.basic != EXIT_REASON_PREEMPTION_TIMER) 7276 return EXIT_FASTPATH_NONE; 7277 7278 switch (to_vmx(vcpu)->exit_reason.basic) { 7279 case EXIT_REASON_MSR_WRITE: 7280 return handle_fastpath_set_msr_irqoff(vcpu); 7281 case EXIT_REASON_PREEMPTION_TIMER: 7282 return handle_fastpath_preemption_timer(vcpu, force_immediate_exit); 7283 default: 7284 return EXIT_FASTPATH_NONE; 7285 } 7286 } 7287 7288 static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, 7289 unsigned int flags) 7290 { 7291 struct vcpu_vmx *vmx = to_vmx(vcpu); 7292 7293 guest_state_enter_irqoff(); 7294 7295 /* 7296 * L1D Flush includes CPU buffer clear to mitigate MDS, but VERW 7297 * mitigation for MDS is done late in VMentry and is still 7298 * executed in spite of L1D Flush. This is because an extra VERW 7299 * should not matter much after the big hammer L1D Flush. 7300 */ 7301 if (static_branch_unlikely(&vmx_l1d_should_flush)) 7302 vmx_l1d_flush(vcpu); 7303 else if (static_branch_unlikely(&mmio_stale_data_clear) && 7304 kvm_arch_has_assigned_device(vcpu->kvm)) 7305 mds_clear_cpu_buffers(); 7306 7307 vmx_disable_fb_clear(vmx); 7308 7309 if (vcpu->arch.cr2 != native_read_cr2()) 7310 native_write_cr2(vcpu->arch.cr2); 7311 7312 vmx->fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs, 7313 flags); 7314 7315 vcpu->arch.cr2 = native_read_cr2(); 7316 vcpu->arch.regs_avail &= ~VMX_REGS_LAZY_LOAD_SET; 7317 7318 vmx->idt_vectoring_info = 0; 7319 7320 vmx_enable_fb_clear(vmx); 7321 7322 if (unlikely(vmx->fail)) { 7323 vmx->exit_reason.full = 0xdead; 7324 goto out; 7325 } 7326 7327 vmx->exit_reason.full = vmcs_read32(VM_EXIT_REASON); 7328 if (likely(!vmx->exit_reason.failed_vmentry)) 7329 vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); 7330 7331 if ((u16)vmx->exit_reason.basic == EXIT_REASON_EXCEPTION_NMI && 7332 is_nmi(vmx_get_intr_info(vcpu))) { 7333 kvm_before_interrupt(vcpu, KVM_HANDLING_NMI); 7334 if (cpu_feature_enabled(X86_FEATURE_FRED)) 7335 fred_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR); 7336 else 7337 vmx_do_nmi_irqoff(); 7338 kvm_after_interrupt(vcpu); 7339 } 7340 7341 out: 7342 guest_state_exit_irqoff(); 7343 } 7344 7345 fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, bool force_immediate_exit) 7346 { 7347 struct vcpu_vmx *vmx = to_vmx(vcpu); 7348 unsigned long cr3, cr4; 7349 7350 /* Record the guest's net vcpu time for enforced NMI injections. */ 7351 if (unlikely(!enable_vnmi && 7352 vmx->loaded_vmcs->soft_vnmi_blocked)) 7353 vmx->loaded_vmcs->entry_time = ktime_get(); 7354 7355 /* 7356 * Don't enter VMX if guest state is invalid, let the exit handler 7357 * start emulation until we arrive back to a valid state. Synthesize a 7358 * consistency check VM-Exit due to invalid guest state and bail. 7359 */ 7360 if (unlikely(vmx->emulation_required)) { 7361 vmx->fail = 0; 7362 7363 vmx->exit_reason.full = EXIT_REASON_INVALID_STATE; 7364 vmx->exit_reason.failed_vmentry = 1; 7365 kvm_register_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_1); 7366 vmx->exit_qualification = ENTRY_FAIL_DEFAULT; 7367 kvm_register_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_2); 7368 vmx->exit_intr_info = 0; 7369 return EXIT_FASTPATH_NONE; 7370 } 7371 7372 trace_kvm_entry(vcpu, force_immediate_exit); 7373 7374 if (vmx->ple_window_dirty) { 7375 vmx->ple_window_dirty = false; 7376 vmcs_write32(PLE_WINDOW, vmx->ple_window); 7377 } 7378 7379 /* 7380 * We did this in prepare_switch_to_guest, because it needs to 7381 * be within srcu_read_lock. 7382 */ 7383 WARN_ON_ONCE(vmx->nested.need_vmcs12_to_shadow_sync); 7384 7385 if (kvm_register_is_dirty(vcpu, VCPU_REGS_RSP)) 7386 vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]); 7387 if (kvm_register_is_dirty(vcpu, VCPU_REGS_RIP)) 7388 vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]); 7389 vcpu->arch.regs_dirty = 0; 7390 7391 /* 7392 * Refresh vmcs.HOST_CR3 if necessary. This must be done immediately 7393 * prior to VM-Enter, as the kernel may load a new ASID (PCID) any time 7394 * it switches back to the current->mm, which can occur in KVM context 7395 * when switching to a temporary mm to patch kernel code, e.g. if KVM 7396 * toggles a static key while handling a VM-Exit. 7397 */ 7398 cr3 = __get_current_cr3_fast(); 7399 if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) { 7400 vmcs_writel(HOST_CR3, cr3); 7401 vmx->loaded_vmcs->host_state.cr3 = cr3; 7402 } 7403 7404 cr4 = cr4_read_shadow(); 7405 if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) { 7406 vmcs_writel(HOST_CR4, cr4); 7407 vmx->loaded_vmcs->host_state.cr4 = cr4; 7408 } 7409 7410 /* When KVM_DEBUGREG_WONT_EXIT, dr6 is accessible in guest. */ 7411 if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) 7412 set_debugreg(vcpu->arch.dr6, 6); 7413 7414 /* When single-stepping over STI and MOV SS, we must clear the 7415 * corresponding interruptibility bits in the guest state. Otherwise 7416 * vmentry fails as it then expects bit 14 (BS) in pending debug 7417 * exceptions being set, but that's not correct for the guest debugging 7418 * case. */ 7419 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) 7420 vmx_set_interrupt_shadow(vcpu, 0); 7421 7422 kvm_load_guest_xsave_state(vcpu); 7423 7424 pt_guest_enter(vmx); 7425 7426 atomic_switch_perf_msrs(vmx); 7427 if (intel_pmu_lbr_is_enabled(vcpu)) 7428 vmx_passthrough_lbr_msrs(vcpu); 7429 7430 if (enable_preemption_timer) 7431 vmx_update_hv_timer(vcpu, force_immediate_exit); 7432 else if (force_immediate_exit) 7433 smp_send_reschedule(vcpu->cpu); 7434 7435 kvm_wait_lapic_expire(vcpu); 7436 7437 /* The actual VMENTER/EXIT is in the .noinstr.text section. */ 7438 vmx_vcpu_enter_exit(vcpu, __vmx_vcpu_run_flags(vmx)); 7439 7440 /* All fields are clean at this point */ 7441 if (kvm_is_using_evmcs()) { 7442 current_evmcs->hv_clean_fields |= 7443 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 7444 7445 current_evmcs->hv_vp_id = kvm_hv_get_vpindex(vcpu); 7446 } 7447 7448 /* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */ 7449 if (vmx->host_debugctlmsr) 7450 update_debugctlmsr(vmx->host_debugctlmsr); 7451 7452 #ifndef CONFIG_X86_64 7453 /* 7454 * The sysexit path does not restore ds/es, so we must set them to 7455 * a reasonable value ourselves. 7456 * 7457 * We can't defer this to vmx_prepare_switch_to_host() since that 7458 * function may be executed in interrupt context, which saves and 7459 * restore segments around it, nullifying its effect. 7460 */ 7461 loadsegment(ds, __USER_DS); 7462 loadsegment(es, __USER_DS); 7463 #endif 7464 7465 pt_guest_exit(vmx); 7466 7467 kvm_load_host_xsave_state(vcpu); 7468 7469 if (is_guest_mode(vcpu)) { 7470 /* 7471 * Track VMLAUNCH/VMRESUME that have made past guest state 7472 * checking. 7473 */ 7474 if (vmx->nested.nested_run_pending && 7475 !vmx->exit_reason.failed_vmentry) 7476 ++vcpu->stat.nested_run; 7477 7478 vmx->nested.nested_run_pending = 0; 7479 } 7480 7481 if (unlikely(vmx->fail)) 7482 return EXIT_FASTPATH_NONE; 7483 7484 if (unlikely((u16)vmx->exit_reason.basic == EXIT_REASON_MCE_DURING_VMENTRY)) 7485 kvm_machine_check(); 7486 7487 trace_kvm_exit(vcpu, KVM_ISA_VMX); 7488 7489 if (unlikely(vmx->exit_reason.failed_vmentry)) 7490 return EXIT_FASTPATH_NONE; 7491 7492 vmx->loaded_vmcs->launched = 1; 7493 7494 vmx_recover_nmi_blocking(vmx); 7495 vmx_complete_interrupts(vmx); 7496 7497 return vmx_exit_handlers_fastpath(vcpu, force_immediate_exit); 7498 } 7499 7500 void vmx_vcpu_free(struct kvm_vcpu *vcpu) 7501 { 7502 struct vcpu_vmx *vmx = to_vmx(vcpu); 7503 7504 if (enable_pml) 7505 vmx_destroy_pml_buffer(vmx); 7506 free_vpid(vmx->vpid); 7507 nested_vmx_free_vcpu(vcpu); 7508 free_loaded_vmcs(vmx->loaded_vmcs); 7509 free_page((unsigned long)vmx->ve_info); 7510 } 7511 7512 int vmx_vcpu_create(struct kvm_vcpu *vcpu) 7513 { 7514 struct vmx_uret_msr *tsx_ctrl; 7515 struct vcpu_vmx *vmx; 7516 int i, err; 7517 7518 BUILD_BUG_ON(offsetof(struct vcpu_vmx, vcpu) != 0); 7519 vmx = to_vmx(vcpu); 7520 7521 INIT_LIST_HEAD(&vmx->pi_wakeup_list); 7522 7523 err = -ENOMEM; 7524 7525 vmx->vpid = allocate_vpid(); 7526 7527 /* 7528 * If PML is turned on, failure on enabling PML just results in failure 7529 * of creating the vcpu, therefore we can simplify PML logic (by 7530 * avoiding dealing with cases, such as enabling PML partially on vcpus 7531 * for the guest), etc. 7532 */ 7533 if (enable_pml) { 7534 vmx->pml_pg = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 7535 if (!vmx->pml_pg) 7536 goto free_vpid; 7537 } 7538 7539 for (i = 0; i < kvm_nr_uret_msrs; ++i) 7540 vmx->guest_uret_msrs[i].mask = -1ull; 7541 if (boot_cpu_has(X86_FEATURE_RTM)) { 7542 /* 7543 * TSX_CTRL_CPUID_CLEAR is handled in the CPUID interception. 7544 * Keep the host value unchanged to avoid changing CPUID bits 7545 * under the host kernel's feet. 7546 */ 7547 tsx_ctrl = vmx_find_uret_msr(vmx, MSR_IA32_TSX_CTRL); 7548 if (tsx_ctrl) 7549 tsx_ctrl->mask = ~(u64)TSX_CTRL_CPUID_CLEAR; 7550 } 7551 7552 err = alloc_loaded_vmcs(&vmx->vmcs01); 7553 if (err < 0) 7554 goto free_pml; 7555 7556 /* 7557 * Use Hyper-V 'Enlightened MSR Bitmap' feature when KVM runs as a 7558 * nested (L1) hypervisor and Hyper-V in L0 supports it. Enable the 7559 * feature only for vmcs01, KVM currently isn't equipped to realize any 7560 * performance benefits from enabling it for vmcs02. 7561 */ 7562 if (kvm_is_using_evmcs() && 7563 (ms_hyperv.nested_features & HV_X64_NESTED_MSR_BITMAP)) { 7564 struct hv_enlightened_vmcs *evmcs = (void *)vmx->vmcs01.vmcs; 7565 7566 evmcs->hv_enlightenments_control.msr_bitmap = 1; 7567 } 7568 7569 /* The MSR bitmap starts with all ones */ 7570 bitmap_fill(vmx->shadow_msr_intercept.read, MAX_POSSIBLE_PASSTHROUGH_MSRS); 7571 bitmap_fill(vmx->shadow_msr_intercept.write, MAX_POSSIBLE_PASSTHROUGH_MSRS); 7572 7573 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_TSC, MSR_TYPE_R); 7574 #ifdef CONFIG_X86_64 7575 vmx_disable_intercept_for_msr(vcpu, MSR_FS_BASE, MSR_TYPE_RW); 7576 vmx_disable_intercept_for_msr(vcpu, MSR_GS_BASE, MSR_TYPE_RW); 7577 vmx_disable_intercept_for_msr(vcpu, MSR_KERNEL_GS_BASE, MSR_TYPE_RW); 7578 #endif 7579 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_CS, MSR_TYPE_RW); 7580 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_ESP, MSR_TYPE_RW); 7581 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_EIP, MSR_TYPE_RW); 7582 if (kvm_cstate_in_guest(vcpu->kvm)) { 7583 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C1_RES, MSR_TYPE_R); 7584 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C3_RESIDENCY, MSR_TYPE_R); 7585 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C6_RESIDENCY, MSR_TYPE_R); 7586 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C7_RESIDENCY, MSR_TYPE_R); 7587 } 7588 7589 vmx->loaded_vmcs = &vmx->vmcs01; 7590 7591 if (cpu_need_virtualize_apic_accesses(vcpu)) { 7592 err = kvm_alloc_apic_access_page(vcpu->kvm); 7593 if (err) 7594 goto free_vmcs; 7595 } 7596 7597 if (enable_ept && !enable_unrestricted_guest) { 7598 err = init_rmode_identity_map(vcpu->kvm); 7599 if (err) 7600 goto free_vmcs; 7601 } 7602 7603 err = -ENOMEM; 7604 if (vmcs_config.cpu_based_2nd_exec_ctrl & SECONDARY_EXEC_EPT_VIOLATION_VE) { 7605 struct page *page; 7606 7607 BUILD_BUG_ON(sizeof(*vmx->ve_info) > PAGE_SIZE); 7608 7609 /* ve_info must be page aligned. */ 7610 page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 7611 if (!page) 7612 goto free_vmcs; 7613 7614 vmx->ve_info = page_to_virt(page); 7615 } 7616 7617 if (vmx_can_use_ipiv(vcpu)) 7618 WRITE_ONCE(to_kvm_vmx(vcpu->kvm)->pid_table[vcpu->vcpu_id], 7619 __pa(&vmx->pi_desc) | PID_TABLE_ENTRY_VALID); 7620 7621 return 0; 7622 7623 free_vmcs: 7624 free_loaded_vmcs(vmx->loaded_vmcs); 7625 free_pml: 7626 vmx_destroy_pml_buffer(vmx); 7627 free_vpid: 7628 free_vpid(vmx->vpid); 7629 return err; 7630 } 7631 7632 #define L1TF_MSG_SMT "L1TF CPU bug present and SMT on, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n" 7633 #define L1TF_MSG_L1D "L1TF CPU bug present and virtualization mitigation disabled, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n" 7634 7635 int vmx_vm_init(struct kvm *kvm) 7636 { 7637 if (!ple_gap) 7638 kvm->arch.pause_in_guest = true; 7639 7640 if (boot_cpu_has(X86_BUG_L1TF) && enable_ept) { 7641 switch (l1tf_mitigation) { 7642 case L1TF_MITIGATION_OFF: 7643 case L1TF_MITIGATION_FLUSH_NOWARN: 7644 /* 'I explicitly don't care' is set */ 7645 break; 7646 case L1TF_MITIGATION_FLUSH: 7647 case L1TF_MITIGATION_FLUSH_NOSMT: 7648 case L1TF_MITIGATION_FULL: 7649 /* 7650 * Warn upon starting the first VM in a potentially 7651 * insecure environment. 7652 */ 7653 if (sched_smt_active()) 7654 pr_warn_once(L1TF_MSG_SMT); 7655 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER) 7656 pr_warn_once(L1TF_MSG_L1D); 7657 break; 7658 case L1TF_MITIGATION_FULL_FORCE: 7659 /* Flush is enforced */ 7660 break; 7661 } 7662 } 7663 return 0; 7664 } 7665 7666 u8 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio) 7667 { 7668 /* We wanted to honor guest CD/MTRR/PAT, but doing so could result in 7669 * memory aliases with conflicting memory types and sometimes MCEs. 7670 * We have to be careful as to what are honored and when. 7671 * 7672 * For MMIO, guest CD/MTRR are ignored. The EPT memory type is set to 7673 * UC. The effective memory type is UC or WC depending on guest PAT. 7674 * This was historically the source of MCEs and we want to be 7675 * conservative. 7676 * 7677 * When there is no need to deal with noncoherent DMA (e.g., no VT-d 7678 * or VT-d has snoop control), guest CD/MTRR/PAT are all ignored. The 7679 * EPT memory type is set to WB. The effective memory type is forced 7680 * WB. 7681 * 7682 * Otherwise, we trust guest. Guest CD/MTRR/PAT are all honored. The 7683 * EPT memory type is used to emulate guest CD/MTRR. 7684 */ 7685 7686 if (is_mmio) 7687 return MTRR_TYPE_UNCACHABLE << VMX_EPT_MT_EPTE_SHIFT; 7688 7689 if (!kvm_arch_has_noncoherent_dma(vcpu->kvm)) 7690 return (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT) | VMX_EPT_IPAT_BIT; 7691 7692 if (kvm_read_cr0_bits(vcpu, X86_CR0_CD)) { 7693 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED)) 7694 return MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT; 7695 else 7696 return (MTRR_TYPE_UNCACHABLE << VMX_EPT_MT_EPTE_SHIFT) | 7697 VMX_EPT_IPAT_BIT; 7698 } 7699 7700 return kvm_mtrr_get_guest_memory_type(vcpu, gfn) << VMX_EPT_MT_EPTE_SHIFT; 7701 } 7702 7703 static void vmcs_set_secondary_exec_control(struct vcpu_vmx *vmx, u32 new_ctl) 7704 { 7705 /* 7706 * These bits in the secondary execution controls field 7707 * are dynamic, the others are mostly based on the hypervisor 7708 * architecture and the guest's CPUID. Do not touch the 7709 * dynamic bits. 7710 */ 7711 u32 mask = 7712 SECONDARY_EXEC_SHADOW_VMCS | 7713 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 7714 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 7715 SECONDARY_EXEC_DESC; 7716 7717 u32 cur_ctl = secondary_exec_controls_get(vmx); 7718 7719 secondary_exec_controls_set(vmx, (new_ctl & ~mask) | (cur_ctl & mask)); 7720 } 7721 7722 /* 7723 * Generate MSR_IA32_VMX_CR{0,4}_FIXED1 according to CPUID. Only set bits 7724 * (indicating "allowed-1") if they are supported in the guest's CPUID. 7725 */ 7726 static void nested_vmx_cr_fixed1_bits_update(struct kvm_vcpu *vcpu) 7727 { 7728 struct vcpu_vmx *vmx = to_vmx(vcpu); 7729 struct kvm_cpuid_entry2 *entry; 7730 7731 vmx->nested.msrs.cr0_fixed1 = 0xffffffff; 7732 vmx->nested.msrs.cr4_fixed1 = X86_CR4_PCE; 7733 7734 #define cr4_fixed1_update(_cr4_mask, _reg, _cpuid_mask) do { \ 7735 if (entry && (entry->_reg & (_cpuid_mask))) \ 7736 vmx->nested.msrs.cr4_fixed1 |= (_cr4_mask); \ 7737 } while (0) 7738 7739 entry = kvm_find_cpuid_entry(vcpu, 0x1); 7740 cr4_fixed1_update(X86_CR4_VME, edx, feature_bit(VME)); 7741 cr4_fixed1_update(X86_CR4_PVI, edx, feature_bit(VME)); 7742 cr4_fixed1_update(X86_CR4_TSD, edx, feature_bit(TSC)); 7743 cr4_fixed1_update(X86_CR4_DE, edx, feature_bit(DE)); 7744 cr4_fixed1_update(X86_CR4_PSE, edx, feature_bit(PSE)); 7745 cr4_fixed1_update(X86_CR4_PAE, edx, feature_bit(PAE)); 7746 cr4_fixed1_update(X86_CR4_MCE, edx, feature_bit(MCE)); 7747 cr4_fixed1_update(X86_CR4_PGE, edx, feature_bit(PGE)); 7748 cr4_fixed1_update(X86_CR4_OSFXSR, edx, feature_bit(FXSR)); 7749 cr4_fixed1_update(X86_CR4_OSXMMEXCPT, edx, feature_bit(XMM)); 7750 cr4_fixed1_update(X86_CR4_VMXE, ecx, feature_bit(VMX)); 7751 cr4_fixed1_update(X86_CR4_SMXE, ecx, feature_bit(SMX)); 7752 cr4_fixed1_update(X86_CR4_PCIDE, ecx, feature_bit(PCID)); 7753 cr4_fixed1_update(X86_CR4_OSXSAVE, ecx, feature_bit(XSAVE)); 7754 7755 entry = kvm_find_cpuid_entry_index(vcpu, 0x7, 0); 7756 cr4_fixed1_update(X86_CR4_FSGSBASE, ebx, feature_bit(FSGSBASE)); 7757 cr4_fixed1_update(X86_CR4_SMEP, ebx, feature_bit(SMEP)); 7758 cr4_fixed1_update(X86_CR4_SMAP, ebx, feature_bit(SMAP)); 7759 cr4_fixed1_update(X86_CR4_PKE, ecx, feature_bit(PKU)); 7760 cr4_fixed1_update(X86_CR4_UMIP, ecx, feature_bit(UMIP)); 7761 cr4_fixed1_update(X86_CR4_LA57, ecx, feature_bit(LA57)); 7762 7763 entry = kvm_find_cpuid_entry_index(vcpu, 0x7, 1); 7764 cr4_fixed1_update(X86_CR4_LAM_SUP, eax, feature_bit(LAM)); 7765 7766 #undef cr4_fixed1_update 7767 } 7768 7769 static void update_intel_pt_cfg(struct kvm_vcpu *vcpu) 7770 { 7771 struct vcpu_vmx *vmx = to_vmx(vcpu); 7772 struct kvm_cpuid_entry2 *best = NULL; 7773 int i; 7774 7775 for (i = 0; i < PT_CPUID_LEAVES; i++) { 7776 best = kvm_find_cpuid_entry_index(vcpu, 0x14, i); 7777 if (!best) 7778 return; 7779 vmx->pt_desc.caps[CPUID_EAX + i*PT_CPUID_REGS_NUM] = best->eax; 7780 vmx->pt_desc.caps[CPUID_EBX + i*PT_CPUID_REGS_NUM] = best->ebx; 7781 vmx->pt_desc.caps[CPUID_ECX + i*PT_CPUID_REGS_NUM] = best->ecx; 7782 vmx->pt_desc.caps[CPUID_EDX + i*PT_CPUID_REGS_NUM] = best->edx; 7783 } 7784 7785 /* Get the number of configurable Address Ranges for filtering */ 7786 vmx->pt_desc.num_address_ranges = intel_pt_validate_cap(vmx->pt_desc.caps, 7787 PT_CAP_num_address_ranges); 7788 7789 /* Initialize and clear the no dependency bits */ 7790 vmx->pt_desc.ctl_bitmask = ~(RTIT_CTL_TRACEEN | RTIT_CTL_OS | 7791 RTIT_CTL_USR | RTIT_CTL_TSC_EN | RTIT_CTL_DISRETC | 7792 RTIT_CTL_BRANCH_EN); 7793 7794 /* 7795 * If CPUID.(EAX=14H,ECX=0):EBX[0]=1 CR3Filter can be set otherwise 7796 * will inject an #GP 7797 */ 7798 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_cr3_filtering)) 7799 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_CR3EN; 7800 7801 /* 7802 * If CPUID.(EAX=14H,ECX=0):EBX[1]=1 CYCEn, CycThresh and 7803 * PSBFreq can be set 7804 */ 7805 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc)) 7806 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_CYCLEACC | 7807 RTIT_CTL_CYC_THRESH | RTIT_CTL_PSB_FREQ); 7808 7809 /* 7810 * If CPUID.(EAX=14H,ECX=0):EBX[3]=1 MTCEn and MTCFreq can be set 7811 */ 7812 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc)) 7813 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_MTC_EN | 7814 RTIT_CTL_MTC_RANGE); 7815 7816 /* If CPUID.(EAX=14H,ECX=0):EBX[4]=1 FUPonPTW and PTWEn can be set */ 7817 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_ptwrite)) 7818 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_FUP_ON_PTW | 7819 RTIT_CTL_PTW_EN); 7820 7821 /* If CPUID.(EAX=14H,ECX=0):EBX[5]=1 PwrEvEn can be set */ 7822 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_power_event_trace)) 7823 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_PWR_EVT_EN; 7824 7825 /* If CPUID.(EAX=14H,ECX=0):ECX[0]=1 ToPA can be set */ 7826 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_topa_output)) 7827 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_TOPA; 7828 7829 /* If CPUID.(EAX=14H,ECX=0):ECX[3]=1 FabricEn can be set */ 7830 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_output_subsys)) 7831 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_FABRIC_EN; 7832 7833 /* unmask address range configure area */ 7834 for (i = 0; i < vmx->pt_desc.num_address_ranges; i++) 7835 vmx->pt_desc.ctl_bitmask &= ~(0xfULL << (32 + i * 4)); 7836 } 7837 7838 void vmx_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) 7839 { 7840 struct vcpu_vmx *vmx = to_vmx(vcpu); 7841 7842 /* 7843 * XSAVES is effectively enabled if and only if XSAVE is also exposed 7844 * to the guest. XSAVES depends on CR4.OSXSAVE, and CR4.OSXSAVE can be 7845 * set if and only if XSAVE is supported. 7846 */ 7847 if (boot_cpu_has(X86_FEATURE_XSAVE) && 7848 guest_cpuid_has(vcpu, X86_FEATURE_XSAVE)) 7849 kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_XSAVES); 7850 7851 kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_VMX); 7852 kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_LAM); 7853 7854 vmx_setup_uret_msrs(vmx); 7855 7856 if (cpu_has_secondary_exec_ctrls()) 7857 vmcs_set_secondary_exec_control(vmx, 7858 vmx_secondary_exec_control(vmx)); 7859 7860 if (guest_can_use(vcpu, X86_FEATURE_VMX)) 7861 vmx->msr_ia32_feature_control_valid_bits |= 7862 FEAT_CTL_VMX_ENABLED_INSIDE_SMX | 7863 FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX; 7864 else 7865 vmx->msr_ia32_feature_control_valid_bits &= 7866 ~(FEAT_CTL_VMX_ENABLED_INSIDE_SMX | 7867 FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX); 7868 7869 if (guest_can_use(vcpu, X86_FEATURE_VMX)) 7870 nested_vmx_cr_fixed1_bits_update(vcpu); 7871 7872 if (boot_cpu_has(X86_FEATURE_INTEL_PT) && 7873 guest_cpuid_has(vcpu, X86_FEATURE_INTEL_PT)) 7874 update_intel_pt_cfg(vcpu); 7875 7876 if (boot_cpu_has(X86_FEATURE_RTM)) { 7877 struct vmx_uret_msr *msr; 7878 msr = vmx_find_uret_msr(vmx, MSR_IA32_TSX_CTRL); 7879 if (msr) { 7880 bool enabled = guest_cpuid_has(vcpu, X86_FEATURE_RTM); 7881 vmx_set_guest_uret_msr(vmx, msr, enabled ? 0 : TSX_CTRL_RTM_DISABLE); 7882 } 7883 } 7884 7885 if (kvm_cpu_cap_has(X86_FEATURE_XFD)) 7886 vmx_set_intercept_for_msr(vcpu, MSR_IA32_XFD_ERR, MSR_TYPE_R, 7887 !guest_cpuid_has(vcpu, X86_FEATURE_XFD)); 7888 7889 if (boot_cpu_has(X86_FEATURE_IBPB)) 7890 vmx_set_intercept_for_msr(vcpu, MSR_IA32_PRED_CMD, MSR_TYPE_W, 7891 !guest_has_pred_cmd_msr(vcpu)); 7892 7893 if (boot_cpu_has(X86_FEATURE_FLUSH_L1D)) 7894 vmx_set_intercept_for_msr(vcpu, MSR_IA32_FLUSH_CMD, MSR_TYPE_W, 7895 !guest_cpuid_has(vcpu, X86_FEATURE_FLUSH_L1D)); 7896 7897 set_cr4_guest_host_mask(vmx); 7898 7899 vmx_write_encls_bitmap(vcpu, NULL); 7900 if (guest_cpuid_has(vcpu, X86_FEATURE_SGX)) 7901 vmx->msr_ia32_feature_control_valid_bits |= FEAT_CTL_SGX_ENABLED; 7902 else 7903 vmx->msr_ia32_feature_control_valid_bits &= ~FEAT_CTL_SGX_ENABLED; 7904 7905 if (guest_cpuid_has(vcpu, X86_FEATURE_SGX_LC)) 7906 vmx->msr_ia32_feature_control_valid_bits |= 7907 FEAT_CTL_SGX_LC_ENABLED; 7908 else 7909 vmx->msr_ia32_feature_control_valid_bits &= 7910 ~FEAT_CTL_SGX_LC_ENABLED; 7911 7912 /* Refresh #PF interception to account for MAXPHYADDR changes. */ 7913 vmx_update_exception_bitmap(vcpu); 7914 } 7915 7916 static __init u64 vmx_get_perf_capabilities(void) 7917 { 7918 u64 perf_cap = PMU_CAP_FW_WRITES; 7919 u64 host_perf_cap = 0; 7920 7921 if (!enable_pmu) 7922 return 0; 7923 7924 if (boot_cpu_has(X86_FEATURE_PDCM)) 7925 rdmsrl(MSR_IA32_PERF_CAPABILITIES, host_perf_cap); 7926 7927 if (!cpu_feature_enabled(X86_FEATURE_ARCH_LBR)) { 7928 x86_perf_get_lbr(&vmx_lbr_caps); 7929 7930 /* 7931 * KVM requires LBR callstack support, as the overhead due to 7932 * context switching LBRs without said support is too high. 7933 * See intel_pmu_create_guest_lbr_event() for more info. 7934 */ 7935 if (!vmx_lbr_caps.has_callstack) 7936 memset(&vmx_lbr_caps, 0, sizeof(vmx_lbr_caps)); 7937 else if (vmx_lbr_caps.nr) 7938 perf_cap |= host_perf_cap & PMU_CAP_LBR_FMT; 7939 } 7940 7941 if (vmx_pebs_supported()) { 7942 perf_cap |= host_perf_cap & PERF_CAP_PEBS_MASK; 7943 7944 /* 7945 * Disallow adaptive PEBS as it is functionally broken, can be 7946 * used by the guest to read *host* LBRs, and can be used to 7947 * bypass userspace event filters. To correctly and safely 7948 * support adaptive PEBS, KVM needs to: 7949 * 7950 * 1. Account for the ADAPTIVE flag when (re)programming fixed 7951 * counters. 7952 * 7953 * 2. Gain support from perf (or take direct control of counter 7954 * programming) to support events without adaptive PEBS 7955 * enabled for the hardware counter. 7956 * 7957 * 3. Ensure LBR MSRs cannot hold host data on VM-Entry with 7958 * adaptive PEBS enabled and MSR_PEBS_DATA_CFG.LBRS=1. 7959 * 7960 * 4. Document which PMU events are effectively exposed to the 7961 * guest via adaptive PEBS, and make adaptive PEBS mutually 7962 * exclusive with KVM_SET_PMU_EVENT_FILTER if necessary. 7963 */ 7964 perf_cap &= ~PERF_CAP_PEBS_BASELINE; 7965 } 7966 7967 return perf_cap; 7968 } 7969 7970 static __init void vmx_set_cpu_caps(void) 7971 { 7972 kvm_set_cpu_caps(); 7973 7974 /* CPUID 0x1 */ 7975 if (nested) 7976 kvm_cpu_cap_set(X86_FEATURE_VMX); 7977 7978 /* CPUID 0x7 */ 7979 if (kvm_mpx_supported()) 7980 kvm_cpu_cap_check_and_set(X86_FEATURE_MPX); 7981 if (!cpu_has_vmx_invpcid()) 7982 kvm_cpu_cap_clear(X86_FEATURE_INVPCID); 7983 if (vmx_pt_mode_is_host_guest()) 7984 kvm_cpu_cap_check_and_set(X86_FEATURE_INTEL_PT); 7985 if (vmx_pebs_supported()) { 7986 kvm_cpu_cap_check_and_set(X86_FEATURE_DS); 7987 kvm_cpu_cap_check_and_set(X86_FEATURE_DTES64); 7988 } 7989 7990 if (!enable_pmu) 7991 kvm_cpu_cap_clear(X86_FEATURE_PDCM); 7992 kvm_caps.supported_perf_cap = vmx_get_perf_capabilities(); 7993 7994 if (!enable_sgx) { 7995 kvm_cpu_cap_clear(X86_FEATURE_SGX); 7996 kvm_cpu_cap_clear(X86_FEATURE_SGX_LC); 7997 kvm_cpu_cap_clear(X86_FEATURE_SGX1); 7998 kvm_cpu_cap_clear(X86_FEATURE_SGX2); 7999 } 8000 8001 if (vmx_umip_emulated()) 8002 kvm_cpu_cap_set(X86_FEATURE_UMIP); 8003 8004 /* CPUID 0xD.1 */ 8005 kvm_caps.supported_xss = 0; 8006 if (!cpu_has_vmx_xsaves()) 8007 kvm_cpu_cap_clear(X86_FEATURE_XSAVES); 8008 8009 /* CPUID 0x80000001 and 0x7 (RDPID) */ 8010 if (!cpu_has_vmx_rdtscp()) { 8011 kvm_cpu_cap_clear(X86_FEATURE_RDTSCP); 8012 kvm_cpu_cap_clear(X86_FEATURE_RDPID); 8013 } 8014 8015 if (cpu_has_vmx_waitpkg()) 8016 kvm_cpu_cap_check_and_set(X86_FEATURE_WAITPKG); 8017 } 8018 8019 static int vmx_check_intercept_io(struct kvm_vcpu *vcpu, 8020 struct x86_instruction_info *info) 8021 { 8022 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 8023 unsigned short port; 8024 bool intercept; 8025 int size; 8026 8027 if (info->intercept == x86_intercept_in || 8028 info->intercept == x86_intercept_ins) { 8029 port = info->src_val; 8030 size = info->dst_bytes; 8031 } else { 8032 port = info->dst_val; 8033 size = info->src_bytes; 8034 } 8035 8036 /* 8037 * If the 'use IO bitmaps' VM-execution control is 0, IO instruction 8038 * VM-exits depend on the 'unconditional IO exiting' VM-execution 8039 * control. 8040 * 8041 * Otherwise, IO instruction VM-exits are controlled by the IO bitmaps. 8042 */ 8043 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 8044 intercept = nested_cpu_has(vmcs12, 8045 CPU_BASED_UNCOND_IO_EXITING); 8046 else 8047 intercept = nested_vmx_check_io_bitmaps(vcpu, port, size); 8048 8049 /* FIXME: produce nested vmexit and return X86EMUL_INTERCEPTED. */ 8050 return intercept ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; 8051 } 8052 8053 int vmx_check_intercept(struct kvm_vcpu *vcpu, 8054 struct x86_instruction_info *info, 8055 enum x86_intercept_stage stage, 8056 struct x86_exception *exception) 8057 { 8058 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 8059 8060 switch (info->intercept) { 8061 /* 8062 * RDPID causes #UD if disabled through secondary execution controls. 8063 * Because it is marked as EmulateOnUD, we need to intercept it here. 8064 * Note, RDPID is hidden behind ENABLE_RDTSCP. 8065 */ 8066 case x86_intercept_rdpid: 8067 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_RDTSCP)) { 8068 exception->vector = UD_VECTOR; 8069 exception->error_code_valid = false; 8070 return X86EMUL_PROPAGATE_FAULT; 8071 } 8072 break; 8073 8074 case x86_intercept_in: 8075 case x86_intercept_ins: 8076 case x86_intercept_out: 8077 case x86_intercept_outs: 8078 return vmx_check_intercept_io(vcpu, info); 8079 8080 case x86_intercept_lgdt: 8081 case x86_intercept_lidt: 8082 case x86_intercept_lldt: 8083 case x86_intercept_ltr: 8084 case x86_intercept_sgdt: 8085 case x86_intercept_sidt: 8086 case x86_intercept_sldt: 8087 case x86_intercept_str: 8088 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC)) 8089 return X86EMUL_CONTINUE; 8090 8091 /* FIXME: produce nested vmexit and return X86EMUL_INTERCEPTED. */ 8092 break; 8093 8094 case x86_intercept_pause: 8095 /* 8096 * PAUSE is a single-byte NOP with a REPE prefix, i.e. collides 8097 * with vanilla NOPs in the emulator. Apply the interception 8098 * check only to actual PAUSE instructions. Don't check 8099 * PAUSE-loop-exiting, software can't expect a given PAUSE to 8100 * exit, i.e. KVM is within its rights to allow L2 to execute 8101 * the PAUSE. 8102 */ 8103 if ((info->rep_prefix != REPE_PREFIX) || 8104 !nested_cpu_has2(vmcs12, CPU_BASED_PAUSE_EXITING)) 8105 return X86EMUL_CONTINUE; 8106 8107 break; 8108 8109 /* TODO: check more intercepts... */ 8110 default: 8111 break; 8112 } 8113 8114 return X86EMUL_UNHANDLEABLE; 8115 } 8116 8117 #ifdef CONFIG_X86_64 8118 /* (a << shift) / divisor, return 1 if overflow otherwise 0 */ 8119 static inline int u64_shl_div_u64(u64 a, unsigned int shift, 8120 u64 divisor, u64 *result) 8121 { 8122 u64 low = a << shift, high = a >> (64 - shift); 8123 8124 /* To avoid the overflow on divq */ 8125 if (high >= divisor) 8126 return 1; 8127 8128 /* Low hold the result, high hold rem which is discarded */ 8129 asm("divq %2\n\t" : "=a" (low), "=d" (high) : 8130 "rm" (divisor), "0" (low), "1" (high)); 8131 *result = low; 8132 8133 return 0; 8134 } 8135 8136 int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, 8137 bool *expired) 8138 { 8139 struct vcpu_vmx *vmx; 8140 u64 tscl, guest_tscl, delta_tsc, lapic_timer_advance_cycles; 8141 struct kvm_timer *ktimer = &vcpu->arch.apic->lapic_timer; 8142 8143 vmx = to_vmx(vcpu); 8144 tscl = rdtsc(); 8145 guest_tscl = kvm_read_l1_tsc(vcpu, tscl); 8146 delta_tsc = max(guest_deadline_tsc, guest_tscl) - guest_tscl; 8147 lapic_timer_advance_cycles = nsec_to_cycles(vcpu, 8148 ktimer->timer_advance_ns); 8149 8150 if (delta_tsc > lapic_timer_advance_cycles) 8151 delta_tsc -= lapic_timer_advance_cycles; 8152 else 8153 delta_tsc = 0; 8154 8155 /* Convert to host delta tsc if tsc scaling is enabled */ 8156 if (vcpu->arch.l1_tsc_scaling_ratio != kvm_caps.default_tsc_scaling_ratio && 8157 delta_tsc && u64_shl_div_u64(delta_tsc, 8158 kvm_caps.tsc_scaling_ratio_frac_bits, 8159 vcpu->arch.l1_tsc_scaling_ratio, &delta_tsc)) 8160 return -ERANGE; 8161 8162 /* 8163 * If the delta tsc can't fit in the 32 bit after the multi shift, 8164 * we can't use the preemption timer. 8165 * It's possible that it fits on later vmentries, but checking 8166 * on every vmentry is costly so we just use an hrtimer. 8167 */ 8168 if (delta_tsc >> (cpu_preemption_timer_multi + 32)) 8169 return -ERANGE; 8170 8171 vmx->hv_deadline_tsc = tscl + delta_tsc; 8172 *expired = !delta_tsc; 8173 return 0; 8174 } 8175 8176 void vmx_cancel_hv_timer(struct kvm_vcpu *vcpu) 8177 { 8178 to_vmx(vcpu)->hv_deadline_tsc = -1; 8179 } 8180 #endif 8181 8182 void vmx_sched_in(struct kvm_vcpu *vcpu, int cpu) 8183 { 8184 if (!kvm_pause_in_guest(vcpu->kvm)) 8185 shrink_ple_window(vcpu); 8186 } 8187 8188 void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) 8189 { 8190 struct vcpu_vmx *vmx = to_vmx(vcpu); 8191 8192 if (WARN_ON_ONCE(!enable_pml)) 8193 return; 8194 8195 if (is_guest_mode(vcpu)) { 8196 vmx->nested.update_vmcs01_cpu_dirty_logging = true; 8197 return; 8198 } 8199 8200 /* 8201 * Note, nr_memslots_dirty_logging can be changed concurrent with this 8202 * code, but in that case another update request will be made and so 8203 * the guest will never run with a stale PML value. 8204 */ 8205 if (atomic_read(&vcpu->kvm->nr_memslots_dirty_logging)) 8206 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_ENABLE_PML); 8207 else 8208 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_ENABLE_PML); 8209 } 8210 8211 void vmx_setup_mce(struct kvm_vcpu *vcpu) 8212 { 8213 if (vcpu->arch.mcg_cap & MCG_LMCE_P) 8214 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |= 8215 FEAT_CTL_LMCE_ENABLED; 8216 else 8217 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &= 8218 ~FEAT_CTL_LMCE_ENABLED; 8219 } 8220 8221 #ifdef CONFIG_KVM_SMM 8222 int vmx_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 8223 { 8224 /* we need a nested vmexit to enter SMM, postpone if run is pending */ 8225 if (to_vmx(vcpu)->nested.nested_run_pending) 8226 return -EBUSY; 8227 return !is_smm(vcpu); 8228 } 8229 8230 int vmx_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram) 8231 { 8232 struct vcpu_vmx *vmx = to_vmx(vcpu); 8233 8234 /* 8235 * TODO: Implement custom flows for forcing the vCPU out/in of L2 on 8236 * SMI and RSM. Using the common VM-Exit + VM-Enter routines is wrong 8237 * SMI and RSM only modify state that is saved and restored via SMRAM. 8238 * E.g. most MSRs are left untouched, but many are modified by VM-Exit 8239 * and VM-Enter, and thus L2's values may be corrupted on SMI+RSM. 8240 */ 8241 vmx->nested.smm.guest_mode = is_guest_mode(vcpu); 8242 if (vmx->nested.smm.guest_mode) 8243 nested_vmx_vmexit(vcpu, -1, 0, 0); 8244 8245 vmx->nested.smm.vmxon = vmx->nested.vmxon; 8246 vmx->nested.vmxon = false; 8247 vmx_clear_hlt(vcpu); 8248 return 0; 8249 } 8250 8251 int vmx_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram) 8252 { 8253 struct vcpu_vmx *vmx = to_vmx(vcpu); 8254 int ret; 8255 8256 if (vmx->nested.smm.vmxon) { 8257 vmx->nested.vmxon = true; 8258 vmx->nested.smm.vmxon = false; 8259 } 8260 8261 if (vmx->nested.smm.guest_mode) { 8262 ret = nested_vmx_enter_non_root_mode(vcpu, false); 8263 if (ret) 8264 return ret; 8265 8266 vmx->nested.nested_run_pending = 1; 8267 vmx->nested.smm.guest_mode = false; 8268 } 8269 return 0; 8270 } 8271 8272 void vmx_enable_smi_window(struct kvm_vcpu *vcpu) 8273 { 8274 /* RSM will cause a vmexit anyway. */ 8275 } 8276 #endif 8277 8278 bool vmx_apic_init_signal_blocked(struct kvm_vcpu *vcpu) 8279 { 8280 return to_vmx(vcpu)->nested.vmxon && !is_guest_mode(vcpu); 8281 } 8282 8283 void vmx_migrate_timers(struct kvm_vcpu *vcpu) 8284 { 8285 if (is_guest_mode(vcpu)) { 8286 struct hrtimer *timer = &to_vmx(vcpu)->nested.preemption_timer; 8287 8288 if (hrtimer_try_to_cancel(timer) == 1) 8289 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED); 8290 } 8291 } 8292 8293 void vmx_hardware_unsetup(void) 8294 { 8295 kvm_set_posted_intr_wakeup_handler(NULL); 8296 8297 if (nested) 8298 nested_vmx_hardware_unsetup(); 8299 8300 free_kvm_area(); 8301 } 8302 8303 void vmx_vm_destroy(struct kvm *kvm) 8304 { 8305 struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); 8306 8307 free_pages((unsigned long)kvm_vmx->pid_table, vmx_get_pid_table_order(kvm)); 8308 } 8309 8310 /* 8311 * Note, the SDM states that the linear address is masked *after* the modified 8312 * canonicality check, whereas KVM masks (untags) the address and then performs 8313 * a "normal" canonicality check. Functionally, the two methods are identical, 8314 * and when the masking occurs relative to the canonicality check isn't visible 8315 * to software, i.e. KVM's behavior doesn't violate the SDM. 8316 */ 8317 gva_t vmx_get_untagged_addr(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags) 8318 { 8319 int lam_bit; 8320 unsigned long cr3_bits; 8321 8322 if (flags & (X86EMUL_F_FETCH | X86EMUL_F_IMPLICIT | X86EMUL_F_INVLPG)) 8323 return gva; 8324 8325 if (!is_64_bit_mode(vcpu)) 8326 return gva; 8327 8328 /* 8329 * Bit 63 determines if the address should be treated as user address 8330 * or a supervisor address. 8331 */ 8332 if (!(gva & BIT_ULL(63))) { 8333 cr3_bits = kvm_get_active_cr3_lam_bits(vcpu); 8334 if (!(cr3_bits & (X86_CR3_LAM_U57 | X86_CR3_LAM_U48))) 8335 return gva; 8336 8337 /* LAM_U48 is ignored if LAM_U57 is set. */ 8338 lam_bit = cr3_bits & X86_CR3_LAM_U57 ? 56 : 47; 8339 } else { 8340 if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_LAM_SUP)) 8341 return gva; 8342 8343 lam_bit = kvm_is_cr4_bit_set(vcpu, X86_CR4_LA57) ? 56 : 47; 8344 } 8345 8346 /* 8347 * Untag the address by sign-extending the lam_bit, but NOT to bit 63. 8348 * Bit 63 is retained from the raw virtual address so that untagging 8349 * doesn't change a user access to a supervisor access, and vice versa. 8350 */ 8351 return (sign_extend64(gva, lam_bit) & ~BIT_ULL(63)) | (gva & BIT_ULL(63)); 8352 } 8353 8354 static unsigned int vmx_handle_intel_pt_intr(void) 8355 { 8356 struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); 8357 8358 /* '0' on failure so that the !PT case can use a RET0 static call. */ 8359 if (!vcpu || !kvm_handling_nmi_from_guest(vcpu)) 8360 return 0; 8361 8362 kvm_make_request(KVM_REQ_PMI, vcpu); 8363 __set_bit(MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT, 8364 (unsigned long *)&vcpu->arch.pmu.global_status); 8365 return 1; 8366 } 8367 8368 static __init void vmx_setup_user_return_msrs(void) 8369 { 8370 8371 /* 8372 * Though SYSCALL is only supported in 64-bit mode on Intel CPUs, kvm 8373 * will emulate SYSCALL in legacy mode if the vendor string in guest 8374 * CPUID.0:{EBX,ECX,EDX} is "AuthenticAMD" or "AMDisbetter!" To 8375 * support this emulation, MSR_STAR is included in the list for i386, 8376 * but is never loaded into hardware. MSR_CSTAR is also never loaded 8377 * into hardware and is here purely for emulation purposes. 8378 */ 8379 const u32 vmx_uret_msrs_list[] = { 8380 #ifdef CONFIG_X86_64 8381 MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR, 8382 #endif 8383 MSR_EFER, MSR_TSC_AUX, MSR_STAR, 8384 MSR_IA32_TSX_CTRL, 8385 }; 8386 int i; 8387 8388 BUILD_BUG_ON(ARRAY_SIZE(vmx_uret_msrs_list) != MAX_NR_USER_RETURN_MSRS); 8389 8390 for (i = 0; i < ARRAY_SIZE(vmx_uret_msrs_list); ++i) 8391 kvm_add_user_return_msr(vmx_uret_msrs_list[i]); 8392 } 8393 8394 static void __init vmx_setup_me_spte_mask(void) 8395 { 8396 u64 me_mask = 0; 8397 8398 /* 8399 * kvm_get_shadow_phys_bits() returns shadow_phys_bits. Use 8400 * the former to avoid exposing shadow_phys_bits. 8401 * 8402 * On pre-MKTME system, boot_cpu_data.x86_phys_bits equals to 8403 * shadow_phys_bits. On MKTME and/or TDX capable systems, 8404 * boot_cpu_data.x86_phys_bits holds the actual physical address 8405 * w/o the KeyID bits, and shadow_phys_bits equals to MAXPHYADDR 8406 * reported by CPUID. Those bits between are KeyID bits. 8407 */ 8408 if (boot_cpu_data.x86_phys_bits != kvm_get_shadow_phys_bits()) 8409 me_mask = rsvd_bits(boot_cpu_data.x86_phys_bits, 8410 kvm_get_shadow_phys_bits() - 1); 8411 /* 8412 * Unlike SME, host kernel doesn't support setting up any 8413 * MKTME KeyID on Intel platforms. No memory encryption 8414 * bits should be included into the SPTE. 8415 */ 8416 kvm_mmu_set_me_spte_mask(0, me_mask); 8417 } 8418 8419 __init int vmx_hardware_setup(void) 8420 { 8421 unsigned long host_bndcfgs; 8422 struct desc_ptr dt; 8423 int r; 8424 8425 store_idt(&dt); 8426 host_idt_base = dt.address; 8427 8428 vmx_setup_user_return_msrs(); 8429 8430 if (setup_vmcs_config(&vmcs_config, &vmx_capability) < 0) 8431 return -EIO; 8432 8433 if (cpu_has_perf_global_ctrl_bug()) 8434 pr_warn_once("VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL " 8435 "does not work properly. Using workaround\n"); 8436 8437 if (boot_cpu_has(X86_FEATURE_NX)) 8438 kvm_enable_efer_bits(EFER_NX); 8439 8440 if (boot_cpu_has(X86_FEATURE_MPX)) { 8441 rdmsrl(MSR_IA32_BNDCFGS, host_bndcfgs); 8442 WARN_ONCE(host_bndcfgs, "BNDCFGS in host will be lost"); 8443 } 8444 8445 if (!cpu_has_vmx_mpx()) 8446 kvm_caps.supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS | 8447 XFEATURE_MASK_BNDCSR); 8448 8449 if (!cpu_has_vmx_vpid() || !cpu_has_vmx_invvpid() || 8450 !(cpu_has_vmx_invvpid_single() || cpu_has_vmx_invvpid_global())) 8451 enable_vpid = 0; 8452 8453 if (!cpu_has_vmx_ept() || 8454 !cpu_has_vmx_ept_4levels() || 8455 !cpu_has_vmx_ept_mt_wb() || 8456 !cpu_has_vmx_invept_global()) 8457 enable_ept = 0; 8458 8459 /* NX support is required for shadow paging. */ 8460 if (!enable_ept && !boot_cpu_has(X86_FEATURE_NX)) { 8461 pr_err_ratelimited("NX (Execute Disable) not supported\n"); 8462 return -EOPNOTSUPP; 8463 } 8464 8465 if (!cpu_has_vmx_ept_ad_bits() || !enable_ept) 8466 enable_ept_ad_bits = 0; 8467 8468 if (!cpu_has_vmx_unrestricted_guest() || !enable_ept) 8469 enable_unrestricted_guest = 0; 8470 8471 if (!cpu_has_vmx_flexpriority()) 8472 flexpriority_enabled = 0; 8473 8474 if (!cpu_has_virtual_nmis()) 8475 enable_vnmi = 0; 8476 8477 #ifdef CONFIG_X86_SGX_KVM 8478 if (!cpu_has_vmx_encls_vmexit()) 8479 enable_sgx = false; 8480 #endif 8481 8482 /* 8483 * set_apic_access_page_addr() is used to reload apic access 8484 * page upon invalidation. No need to do anything if not 8485 * using the APIC_ACCESS_ADDR VMCS field. 8486 */ 8487 if (!flexpriority_enabled) 8488 vt_x86_ops.set_apic_access_page_addr = NULL; 8489 8490 if (!cpu_has_vmx_tpr_shadow()) 8491 vt_x86_ops.update_cr8_intercept = NULL; 8492 8493 #if IS_ENABLED(CONFIG_HYPERV) 8494 if (ms_hyperv.nested_features & HV_X64_NESTED_GUEST_MAPPING_FLUSH 8495 && enable_ept) { 8496 vt_x86_ops.flush_remote_tlbs = hv_flush_remote_tlbs; 8497 vt_x86_ops.flush_remote_tlbs_range = hv_flush_remote_tlbs_range; 8498 } 8499 #endif 8500 8501 if (!cpu_has_vmx_ple()) { 8502 ple_gap = 0; 8503 ple_window = 0; 8504 ple_window_grow = 0; 8505 ple_window_max = 0; 8506 ple_window_shrink = 0; 8507 } 8508 8509 if (!cpu_has_vmx_apicv()) 8510 enable_apicv = 0; 8511 if (!enable_apicv) 8512 vt_x86_ops.sync_pir_to_irr = NULL; 8513 8514 if (!enable_apicv || !cpu_has_vmx_ipiv()) 8515 enable_ipiv = false; 8516 8517 if (cpu_has_vmx_tsc_scaling()) 8518 kvm_caps.has_tsc_control = true; 8519 8520 kvm_caps.max_tsc_scaling_ratio = KVM_VMX_TSC_MULTIPLIER_MAX; 8521 kvm_caps.tsc_scaling_ratio_frac_bits = 48; 8522 kvm_caps.has_bus_lock_exit = cpu_has_vmx_bus_lock_detection(); 8523 kvm_caps.has_notify_vmexit = cpu_has_notify_vmexit(); 8524 8525 set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */ 8526 8527 if (enable_ept) 8528 kvm_mmu_set_ept_masks(enable_ept_ad_bits, 8529 cpu_has_vmx_ept_execute_only()); 8530 8531 /* 8532 * Setup shadow_me_value/shadow_me_mask to include MKTME KeyID 8533 * bits to shadow_zero_check. 8534 */ 8535 vmx_setup_me_spte_mask(); 8536 8537 kvm_configure_mmu(enable_ept, 0, vmx_get_max_ept_level(), 8538 ept_caps_to_lpage_level(vmx_capability.ept)); 8539 8540 /* 8541 * Only enable PML when hardware supports PML feature, and both EPT 8542 * and EPT A/D bit features are enabled -- PML depends on them to work. 8543 */ 8544 if (!enable_ept || !enable_ept_ad_bits || !cpu_has_vmx_pml()) 8545 enable_pml = 0; 8546 8547 if (!enable_pml) 8548 vt_x86_ops.cpu_dirty_log_size = 0; 8549 8550 if (!cpu_has_vmx_preemption_timer()) 8551 enable_preemption_timer = false; 8552 8553 if (enable_preemption_timer) { 8554 u64 use_timer_freq = 5000ULL * 1000 * 1000; 8555 8556 cpu_preemption_timer_multi = 8557 vmcs_config.misc & VMX_MISC_PREEMPTION_TIMER_RATE_MASK; 8558 8559 if (tsc_khz) 8560 use_timer_freq = (u64)tsc_khz * 1000; 8561 use_timer_freq >>= cpu_preemption_timer_multi; 8562 8563 /* 8564 * KVM "disables" the preemption timer by setting it to its max 8565 * value. Don't use the timer if it might cause spurious exits 8566 * at a rate faster than 0.1 Hz (of uninterrupted guest time). 8567 */ 8568 if (use_timer_freq > 0xffffffffu / 10) 8569 enable_preemption_timer = false; 8570 } 8571 8572 if (!enable_preemption_timer) { 8573 vt_x86_ops.set_hv_timer = NULL; 8574 vt_x86_ops.cancel_hv_timer = NULL; 8575 } 8576 8577 kvm_caps.supported_mce_cap |= MCG_LMCE_P; 8578 kvm_caps.supported_mce_cap |= MCG_CMCI_P; 8579 8580 if (pt_mode != PT_MODE_SYSTEM && pt_mode != PT_MODE_HOST_GUEST) 8581 return -EINVAL; 8582 if (!enable_ept || !enable_pmu || !cpu_has_vmx_intel_pt()) 8583 pt_mode = PT_MODE_SYSTEM; 8584 if (pt_mode == PT_MODE_HOST_GUEST) 8585 vt_init_ops.handle_intel_pt_intr = vmx_handle_intel_pt_intr; 8586 else 8587 vt_init_ops.handle_intel_pt_intr = NULL; 8588 8589 setup_default_sgx_lepubkeyhash(); 8590 8591 if (nested) { 8592 nested_vmx_setup_ctls_msrs(&vmcs_config, vmx_capability.ept); 8593 8594 r = nested_vmx_hardware_setup(kvm_vmx_exit_handlers); 8595 if (r) 8596 return r; 8597 } 8598 8599 vmx_set_cpu_caps(); 8600 8601 r = alloc_kvm_area(); 8602 if (r && nested) 8603 nested_vmx_hardware_unsetup(); 8604 8605 kvm_set_posted_intr_wakeup_handler(pi_wakeup_handler); 8606 8607 return r; 8608 } 8609 8610 static void vmx_cleanup_l1d_flush(void) 8611 { 8612 if (vmx_l1d_flush_pages) { 8613 free_pages((unsigned long)vmx_l1d_flush_pages, L1D_CACHE_ORDER); 8614 vmx_l1d_flush_pages = NULL; 8615 } 8616 /* Restore state so sysfs ignores VMX */ 8617 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; 8618 } 8619 8620 static void __vmx_exit(void) 8621 { 8622 allow_smaller_maxphyaddr = false; 8623 8624 cpu_emergency_unregister_virt_callback(vmx_emergency_disable); 8625 8626 vmx_cleanup_l1d_flush(); 8627 } 8628 8629 static void vmx_exit(void) 8630 { 8631 kvm_exit(); 8632 kvm_x86_vendor_exit(); 8633 8634 __vmx_exit(); 8635 } 8636 module_exit(vmx_exit); 8637 8638 static int __init vmx_init(void) 8639 { 8640 int r, cpu; 8641 8642 if (!kvm_is_vmx_supported()) 8643 return -EOPNOTSUPP; 8644 8645 /* 8646 * Note, hv_init_evmcs() touches only VMX knobs, i.e. there's nothing 8647 * to unwind if a later step fails. 8648 */ 8649 hv_init_evmcs(); 8650 8651 r = kvm_x86_vendor_init(&vt_init_ops); 8652 if (r) 8653 return r; 8654 8655 /* 8656 * Must be called after common x86 init so enable_ept is properly set 8657 * up. Hand the parameter mitigation value in which was stored in 8658 * the pre module init parser. If no parameter was given, it will 8659 * contain 'auto' which will be turned into the default 'cond' 8660 * mitigation mode. 8661 */ 8662 r = vmx_setup_l1d_flush(vmentry_l1d_flush_param); 8663 if (r) 8664 goto err_l1d_flush; 8665 8666 for_each_possible_cpu(cpu) { 8667 INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu)); 8668 8669 pi_init_cpu(cpu); 8670 } 8671 8672 cpu_emergency_register_virt_callback(vmx_emergency_disable); 8673 8674 vmx_check_vmcs12_offsets(); 8675 8676 /* 8677 * Shadow paging doesn't have a (further) performance penalty 8678 * from GUEST_MAXPHYADDR < HOST_MAXPHYADDR so enable it 8679 * by default 8680 */ 8681 if (!enable_ept) 8682 allow_smaller_maxphyaddr = true; 8683 8684 /* 8685 * Common KVM initialization _must_ come last, after this, /dev/kvm is 8686 * exposed to userspace! 8687 */ 8688 r = kvm_init(sizeof(struct vcpu_vmx), __alignof__(struct vcpu_vmx), 8689 THIS_MODULE); 8690 if (r) 8691 goto err_kvm_init; 8692 8693 return 0; 8694 8695 err_kvm_init: 8696 __vmx_exit(); 8697 err_l1d_flush: 8698 kvm_x86_vendor_exit(); 8699 return r; 8700 } 8701 module_init(vmx_init); 8702