1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * This module enables machines with Intel VT-x extensions to run virtual 6 * machines without emulation or binary translation. 7 * 8 * Copyright (C) 2006 Qumranet, Inc. 9 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 10 * 11 * Authors: 12 * Avi Kivity <avi@qumranet.com> 13 * Yaniv Kamay <yaniv@qumranet.com> 14 */ 15 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 16 17 #include <linux/highmem.h> 18 #include <linux/hrtimer.h> 19 #include <linux/kernel.h> 20 #include <linux/kvm_host.h> 21 #include <linux/module.h> 22 #include <linux/moduleparam.h> 23 #include <linux/mm.h> 24 #include <linux/objtool.h> 25 #include <linux/sched.h> 26 #include <linux/sched/smt.h> 27 #include <linux/slab.h> 28 #include <linux/tboot.h> 29 #include <linux/trace_events.h> 30 31 #include <asm/apic.h> 32 #include <asm/asm.h> 33 #include <asm/cpu.h> 34 #include <asm/cpu_device_id.h> 35 #include <asm/cpuid/api.h> 36 #include <asm/debugreg.h> 37 #include <asm/desc.h> 38 #include <asm/fpu/api.h> 39 #include <asm/fpu/xstate.h> 40 #include <asm/fred.h> 41 #include <asm/idtentry.h> 42 #include <asm/io.h> 43 #include <asm/irq_remapping.h> 44 #include <asm/reboot.h> 45 #include <asm/perf_event.h> 46 #include <asm/mmu_context.h> 47 #include <asm/mshyperv.h> 48 #include <asm/msr.h> 49 #include <asm/mwait.h> 50 #include <asm/spec-ctrl.h> 51 #include <asm/virt.h> 52 #include <asm/vmx.h> 53 54 #include <trace/events/ipi.h> 55 56 #include "capabilities.h" 57 #include "common.h" 58 #include "cpuid.h" 59 #include "hyperv.h" 60 #include "kvm_onhyperv.h" 61 #include "irq.h" 62 #include "regs.h" 63 #include "lapic.h" 64 #include "mmu.h" 65 #include "nested.h" 66 #include "pmu.h" 67 #include "sgx.h" 68 #include "trace.h" 69 #include "vmcs.h" 70 #include "vmcs12.h" 71 #include "vmx.h" 72 #include "x86.h" 73 #include "x86_ops.h" 74 #include "smm.h" 75 #include "tss.h" 76 #include "vmx_onhyperv.h" 77 #include "vmenter.h" 78 #include "posted_intr.h" 79 80 #include "mmu/spte.h" 81 82 MODULE_AUTHOR("Qumranet"); 83 MODULE_DESCRIPTION("KVM support for VMX (Intel VT-x) extensions"); 84 MODULE_LICENSE("GPL"); 85 86 #ifdef MODULE 87 static const struct x86_cpu_id vmx_cpu_id[] = { 88 X86_MATCH_FEATURE(X86_FEATURE_VMX, NULL), 89 {} 90 }; 91 MODULE_DEVICE_TABLE(x86cpu, vmx_cpu_id); 92 #endif 93 94 bool __read_mostly enable_vpid = 1; 95 module_param_named(vpid, enable_vpid, bool, 0444); 96 97 static bool __read_mostly enable_vnmi = 1; 98 module_param_named(vnmi, enable_vnmi, bool, 0444); 99 100 bool __read_mostly flexpriority_enabled = 1; 101 module_param_named(flexpriority, flexpriority_enabled, bool, 0444); 102 103 bool __read_mostly enable_ept = 1; 104 module_param_named(ept, enable_ept, bool, 0444); 105 106 bool __read_mostly enable_unrestricted_guest = 1; 107 module_param_named(unrestricted_guest, 108 enable_unrestricted_guest, bool, 0444); 109 110 bool __read_mostly enable_ept_ad_bits = 1; 111 module_param_named(eptad, enable_ept_ad_bits, bool, 0444); 112 113 bool __read_mostly enable_cet = 1; 114 module_param_named(cet, enable_cet, bool, 0444); 115 116 static bool __read_mostly emulate_invalid_guest_state = true; 117 module_param(emulate_invalid_guest_state, bool, 0444); 118 119 static bool __read_mostly fasteoi = 1; 120 module_param(fasteoi, bool, 0444); 121 122 bool __read_mostly enable_mbec = 1; 123 module_param_named(mbec, enable_mbec, bool, 0444); 124 125 module_param(enable_apicv, bool, 0444); 126 module_param(enable_ipiv, bool, 0444); 127 128 module_param(enable_device_posted_irqs, bool, 0444); 129 130 /* 131 * If nested=1, nested virtualization is supported, i.e., guests may use 132 * VMX and be a hypervisor for its own guests. If nested=0, guests may not 133 * use VMX instructions. 134 */ 135 static bool __read_mostly nested = 1; 136 module_param(nested, bool, 0444); 137 138 bool __read_mostly enable_pml = 1; 139 module_param_named(pml, enable_pml, bool, 0444); 140 141 static bool __read_mostly error_on_inconsistent_vmcs_config = true; 142 module_param(error_on_inconsistent_vmcs_config, bool, 0444); 143 144 static bool __read_mostly dump_invalid_vmcs = 0; 145 module_param(dump_invalid_vmcs, bool, 0644); 146 147 #define MSR_BITMAP_MODE_X2APIC 1 148 #define MSR_BITMAP_MODE_X2APIC_APICV 2 149 150 #define KVM_VMX_TSC_MULTIPLIER_MAX 0xffffffffffffffffULL 151 152 /* Guest_tsc -> host_tsc conversion requires 64-bit division. */ 153 #ifdef CONFIG_X86_64 154 static int __read_mostly cpu_preemption_timer_multi; 155 static bool __read_mostly enable_preemption_timer = 1; 156 static u64 __ro_after_init preemption_timer_max_value; 157 module_param_named(preemption_timer, enable_preemption_timer, bool, S_IRUGO); 158 #else 159 #define enable_preemption_timer false 160 #endif 161 162 extern bool __read_mostly allow_smaller_maxphyaddr; 163 module_param(allow_smaller_maxphyaddr, bool, S_IRUGO); 164 165 module_param(enable_mediated_pmu, bool, 0444); 166 167 #define KVM_VM_CR0_ALWAYS_OFF (X86_CR0_NW | X86_CR0_CD) 168 #define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR0_NE 169 #define KVM_VM_CR0_ALWAYS_ON \ 170 (KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | X86_CR0_PG | X86_CR0_PE) 171 172 #define KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR4_VMXE 173 #define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE) 174 #define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE) 175 176 #define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM)) 177 178 #define MSR_IA32_RTIT_STATUS_MASK (~(RTIT_STATUS_FILTEREN | \ 179 RTIT_STATUS_CONTEXTEN | RTIT_STATUS_TRIGGEREN | \ 180 RTIT_STATUS_ERROR | RTIT_STATUS_STOPPED | \ 181 RTIT_STATUS_BYTECNT)) 182 183 /* 184 * These 2 parameters are used to config the controls for Pause-Loop Exiting: 185 * ple_gap: upper bound on the amount of time between two successive 186 * executions of PAUSE in a loop. Also indicate if ple enabled. 187 * According to test, this time is usually smaller than 128 cycles. 188 * ple_window: upper bound on the amount of time a guest is allowed to execute 189 * in a PAUSE loop. Tests indicate that most spinlocks are held for 190 * less than 2^12 cycles 191 * Time is measured based on a counter that runs at the same rate as the TSC, 192 * refer SDM volume 3b section 21.6.13 & 22.1.3. 193 */ 194 static unsigned int ple_gap = KVM_DEFAULT_PLE_GAP; 195 module_param(ple_gap, uint, 0444); 196 197 static unsigned int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW; 198 module_param(ple_window, uint, 0444); 199 200 /* Default doubles per-vcpu window every exit. */ 201 static unsigned int ple_window_grow = KVM_DEFAULT_PLE_WINDOW_GROW; 202 module_param(ple_window_grow, uint, 0444); 203 204 /* Default resets per-vcpu window every exit to ple_window. */ 205 static unsigned int ple_window_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK; 206 module_param(ple_window_shrink, uint, 0444); 207 208 /* Default is to compute the maximum so we can never overflow. */ 209 static unsigned int ple_window_max = KVM_VMX_DEFAULT_PLE_WINDOW_MAX; 210 module_param(ple_window_max, uint, 0444); 211 212 /* Default is SYSTEM mode, 1 for host-guest mode (which is BROKEN) */ 213 int __read_mostly pt_mode = PT_MODE_SYSTEM; 214 #ifdef CONFIG_BROKEN 215 module_param(pt_mode, int, S_IRUGO); 216 #endif 217 218 struct x86_pmu_lbr __ro_after_init vmx_lbr_caps; 219 220 #ifdef CONFIG_CPU_MITIGATIONS 221 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_should_flush); 222 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_flush_cond); 223 static DEFINE_MUTEX(vmx_l1d_flush_mutex); 224 225 /* Storage for pre module init parameter parsing */ 226 static enum vmx_l1d_flush_state __read_mostly vmentry_l1d_flush_param = VMENTER_L1D_FLUSH_AUTO; 227 228 static const struct { 229 const char *option; 230 bool for_parse; 231 } vmentry_l1d_param[] = { 232 [VMENTER_L1D_FLUSH_AUTO] = {"auto", true}, 233 [VMENTER_L1D_FLUSH_NEVER] = {"never", true}, 234 [VMENTER_L1D_FLUSH_COND] = {"cond", true}, 235 [VMENTER_L1D_FLUSH_ALWAYS] = {"always", true}, 236 [VMENTER_L1D_FLUSH_EPT_DISABLED] = {"EPT disabled", false}, 237 [VMENTER_L1D_FLUSH_NOT_REQUIRED] = {"not required", false}, 238 }; 239 240 #define L1D_CACHE_ORDER 4 241 static void *vmx_l1d_flush_pages; 242 243 static int __vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf) 244 { 245 struct page *page; 246 unsigned int i; 247 248 if (!boot_cpu_has_bug(X86_BUG_L1TF)) { 249 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED; 250 return 0; 251 } 252 253 if (!enable_ept) { 254 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_EPT_DISABLED; 255 return 0; 256 } 257 258 if (kvm_host.arch_capabilities & ARCH_CAP_SKIP_VMENTRY_L1DFLUSH) { 259 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED; 260 return 0; 261 } 262 263 /* If set to auto use the default l1tf mitigation method */ 264 if (l1tf == VMENTER_L1D_FLUSH_AUTO) { 265 switch (l1tf_mitigation) { 266 case L1TF_MITIGATION_OFF: 267 l1tf = VMENTER_L1D_FLUSH_NEVER; 268 break; 269 case L1TF_MITIGATION_AUTO: 270 case L1TF_MITIGATION_FLUSH_NOWARN: 271 case L1TF_MITIGATION_FLUSH: 272 case L1TF_MITIGATION_FLUSH_NOSMT: 273 l1tf = VMENTER_L1D_FLUSH_COND; 274 break; 275 case L1TF_MITIGATION_FULL: 276 case L1TF_MITIGATION_FULL_FORCE: 277 l1tf = VMENTER_L1D_FLUSH_ALWAYS; 278 break; 279 } 280 } else if (l1tf_mitigation == L1TF_MITIGATION_FULL_FORCE) { 281 l1tf = VMENTER_L1D_FLUSH_ALWAYS; 282 } 283 284 if (l1tf != VMENTER_L1D_FLUSH_NEVER && !vmx_l1d_flush_pages && 285 !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) { 286 /* 287 * This allocation for vmx_l1d_flush_pages is not tied to a VM 288 * lifetime and so should not be charged to a memcg. 289 */ 290 page = alloc_pages(GFP_KERNEL, L1D_CACHE_ORDER); 291 if (!page) 292 return -ENOMEM; 293 vmx_l1d_flush_pages = page_address(page); 294 295 /* 296 * Initialize each page with a different pattern in 297 * order to protect against KSM in the nested 298 * virtualization case. 299 */ 300 for (i = 0; i < 1u << L1D_CACHE_ORDER; ++i) { 301 memset(vmx_l1d_flush_pages + i * PAGE_SIZE, i + 1, 302 PAGE_SIZE); 303 } 304 } 305 306 l1tf_vmx_mitigation = l1tf; 307 308 if (l1tf != VMENTER_L1D_FLUSH_NEVER) 309 static_branch_enable(&vmx_l1d_should_flush); 310 else 311 static_branch_disable(&vmx_l1d_should_flush); 312 313 if (l1tf == VMENTER_L1D_FLUSH_COND) 314 static_branch_enable(&vmx_l1d_flush_cond); 315 else 316 static_branch_disable(&vmx_l1d_flush_cond); 317 return 0; 318 } 319 320 static int vmx_setup_l1d_flush(void) 321 { 322 /* 323 * Hand the parameter mitigation value in which was stored in the pre 324 * module init parser. If no parameter was given, it will contain 325 * 'auto' which will be turned into the default 'cond' mitigation mode. 326 */ 327 return __vmx_setup_l1d_flush(vmentry_l1d_flush_param); 328 } 329 330 static void vmx_cleanup_l1d_flush(void) 331 { 332 if (vmx_l1d_flush_pages) { 333 free_pages((unsigned long)vmx_l1d_flush_pages, L1D_CACHE_ORDER); 334 vmx_l1d_flush_pages = NULL; 335 } 336 /* Restore state so sysfs ignores VMX */ 337 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; 338 } 339 340 static int vmentry_l1d_flush_parse(const char *s) 341 { 342 unsigned int i; 343 344 if (s) { 345 for (i = 0; i < ARRAY_SIZE(vmentry_l1d_param); i++) { 346 if (vmentry_l1d_param[i].for_parse && 347 sysfs_streq(s, vmentry_l1d_param[i].option)) 348 return i; 349 } 350 } 351 return -EINVAL; 352 } 353 354 static int vmentry_l1d_flush_set(const char *s, const struct kernel_param *kp) 355 { 356 int l1tf, ret; 357 358 l1tf = vmentry_l1d_flush_parse(s); 359 if (l1tf < 0) 360 return l1tf; 361 362 if (!boot_cpu_has(X86_BUG_L1TF)) 363 return 0; 364 365 /* 366 * Has vmx_init() run already? If not then this is the pre init 367 * parameter parsing. In that case just store the value and let 368 * vmx_init() do the proper setup after enable_ept has been 369 * established. 370 */ 371 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) { 372 vmentry_l1d_flush_param = l1tf; 373 return 0; 374 } 375 376 mutex_lock(&vmx_l1d_flush_mutex); 377 ret = __vmx_setup_l1d_flush(l1tf); 378 mutex_unlock(&vmx_l1d_flush_mutex); 379 return ret; 380 } 381 382 static int vmentry_l1d_flush_get(char *s, const struct kernel_param *kp) 383 { 384 if (WARN_ON_ONCE(l1tf_vmx_mitigation >= ARRAY_SIZE(vmentry_l1d_param))) 385 return sysfs_emit(s, "???\n"); 386 387 return sysfs_emit(s, "%s\n", vmentry_l1d_param[l1tf_vmx_mitigation].option); 388 } 389 390 /* 391 * Software based L1D cache flush which is used when microcode providing 392 * the cache control MSR is not loaded. 393 * 394 * The L1D cache is 32 KiB on Nehalem and later microarchitectures, but to 395 * flush it is required to read in 64 KiB because the replacement algorithm 396 * is not exactly LRU. This could be sized at runtime via topology 397 * information but as all relevant affected CPUs have 32KiB L1D cache size 398 * there is no point in doing so. 399 */ 400 static noinstr void vmx_l1d_flush(struct kvm_vcpu *vcpu) 401 { 402 int size = PAGE_SIZE << L1D_CACHE_ORDER; 403 404 if (!static_branch_unlikely(&vmx_l1d_should_flush)) 405 return; 406 407 /* 408 * This code is only executed when the flush mode is 'cond' or 409 * 'always' 410 */ 411 if (static_branch_likely(&vmx_l1d_flush_cond)) { 412 /* 413 * Clear the per-cpu flush bit, it gets set again if the vCPU 414 * is reloaded, i.e. if the vCPU is scheduled out or if KVM 415 * exits to userspace, or if KVM reaches one of the unsafe 416 * VMEXIT handlers, e.g. if KVM calls into the emulator, 417 * or from the interrupt handlers. 418 */ 419 if (!kvm_get_cpu_l1tf_flush_l1d()) 420 return; 421 kvm_clear_cpu_l1tf_flush_l1d(); 422 } 423 424 vcpu->stat.l1d_flush++; 425 426 if (cpu_feature_enabled(X86_FEATURE_FLUSH_L1D)) { 427 native_wrmsrq(MSR_IA32_FLUSH_CMD, L1D_FLUSH); 428 return; 429 } 430 431 asm volatile( 432 /* First ensure the pages are in the TLB */ 433 "xorl %%eax, %%eax\n" 434 ".Lpopulate_tlb:\n\t" 435 "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t" 436 "addl $4096, %%eax\n\t" 437 "cmpl %%eax, %[size]\n\t" 438 "jne .Lpopulate_tlb\n\t" 439 "xorl %%eax, %%eax\n\t" 440 "cpuid\n\t" 441 /* Now fill the cache */ 442 "xorl %%eax, %%eax\n" 443 ".Lfill_cache:\n" 444 "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t" 445 "addl $64, %%eax\n\t" 446 "cmpl %%eax, %[size]\n\t" 447 "jne .Lfill_cache\n\t" 448 "lfence\n" 449 :: [flush_pages] "r" (vmx_l1d_flush_pages), 450 [size] "r" (size) 451 : "eax", "ebx", "ecx", "edx"); 452 } 453 454 #else /* CONFIG_CPU_MITIGATIONS*/ 455 static int vmx_setup_l1d_flush(void) 456 { 457 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NEVER; 458 return 0; 459 } 460 static void vmx_cleanup_l1d_flush(void) 461 { 462 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; 463 } 464 static __always_inline void vmx_l1d_flush(struct kvm_vcpu *vcpu) 465 { 466 467 } 468 static int vmentry_l1d_flush_set(const char *s, const struct kernel_param *kp) 469 { 470 pr_warn_once("Kernel compiled without mitigations, ignoring vmentry_l1d_flush\n"); 471 return 0; 472 } 473 static int vmentry_l1d_flush_get(char *s, const struct kernel_param *kp) 474 { 475 return sysfs_emit(s, "never\n"); 476 } 477 #endif 478 479 static const struct kernel_param_ops vmentry_l1d_flush_ops = { 480 .set = vmentry_l1d_flush_set, 481 .get = vmentry_l1d_flush_get, 482 }; 483 module_param_cb(vmentry_l1d_flush, &vmentry_l1d_flush_ops, NULL, 0644); 484 485 static __always_inline void vmx_disable_fb_clear(struct vcpu_vmx *vmx) 486 { 487 u64 msr; 488 489 if (!vmx->disable_fb_clear) 490 return; 491 492 msr = native_rdmsrq(MSR_IA32_MCU_OPT_CTRL); 493 msr |= FB_CLEAR_DIS; 494 native_wrmsrq(MSR_IA32_MCU_OPT_CTRL, msr); 495 /* Cache the MSR value to avoid reading it later */ 496 vmx->msr_ia32_mcu_opt_ctrl = msr; 497 } 498 499 static __always_inline void vmx_enable_fb_clear(struct vcpu_vmx *vmx) 500 { 501 if (!vmx->disable_fb_clear) 502 return; 503 504 vmx->msr_ia32_mcu_opt_ctrl &= ~FB_CLEAR_DIS; 505 native_wrmsrq(MSR_IA32_MCU_OPT_CTRL, vmx->msr_ia32_mcu_opt_ctrl); 506 } 507 508 static void vmx_update_fb_clear_dis(struct kvm_vcpu *vcpu, struct vcpu_vmx *vmx) 509 { 510 /* 511 * Disable VERW's behavior of clearing CPU buffers for the guest if the 512 * CPU isn't affected by MDS/TAA, and the host hasn't forcefully enabled 513 * the mitigation. Disabling the clearing behavior provides a 514 * performance boost for guests that aren't aware that manually clearing 515 * CPU buffers is unnecessary, at the cost of MSR accesses on VM-Entry 516 * and VM-Exit. 517 */ 518 vmx->disable_fb_clear = !cpu_feature_enabled(X86_FEATURE_CLEAR_CPU_BUF) && 519 (kvm_host.arch_capabilities & ARCH_CAP_FB_CLEAR_CTRL) && 520 !boot_cpu_has_bug(X86_BUG_MDS) && 521 !boot_cpu_has_bug(X86_BUG_TAA); 522 523 /* 524 * If guest will not execute VERW, there is no need to set FB_CLEAR_DIS 525 * at VMEntry. Skip the MSR read/write when a guest has no use case to 526 * execute VERW. 527 */ 528 if ((vcpu->arch.arch_capabilities & ARCH_CAP_FB_CLEAR) || 529 ((vcpu->arch.arch_capabilities & ARCH_CAP_MDS_NO) && 530 (vcpu->arch.arch_capabilities & ARCH_CAP_TAA_NO) && 531 (vcpu->arch.arch_capabilities & ARCH_CAP_PSDP_NO) && 532 (vcpu->arch.arch_capabilities & ARCH_CAP_FBSDP_NO) && 533 (vcpu->arch.arch_capabilities & ARCH_CAP_SBDR_SSDP_NO))) 534 vmx->disable_fb_clear = false; 535 } 536 537 static u32 vmx_segment_access_rights(struct kvm_segment *var); 538 539 void vmx_vmexit(void); 540 541 #define vmx_insn_failed(fmt...) \ 542 do { \ 543 WARN_ONCE(1, fmt); \ 544 pr_warn_ratelimited(fmt); \ 545 } while (0) 546 547 noinline void vmread_error(unsigned long field) 548 { 549 vmx_insn_failed("vmread failed: field=%lx\n", field); 550 } 551 552 #ifndef CONFIG_CC_HAS_ASM_GOTO_OUTPUT 553 noinstr void vmread_error_trampoline2(unsigned long field, bool fault) 554 { 555 if (fault) { 556 kvm_spurious_fault(); 557 } else { 558 instrumentation_begin(); 559 vmread_error(field); 560 instrumentation_end(); 561 } 562 } 563 #endif 564 565 noinline void vmwrite_error(unsigned long field, unsigned long value) 566 { 567 vmx_insn_failed("vmwrite failed: field=%lx val=%lx err=%u\n", 568 field, value, vmcs_read32(VM_INSTRUCTION_ERROR)); 569 } 570 571 noinline void vmclear_error(struct vmcs *vmcs, u64 phys_addr) 572 { 573 vmx_insn_failed("vmclear failed: %p/%llx err=%u\n", 574 vmcs, phys_addr, vmcs_read32(VM_INSTRUCTION_ERROR)); 575 } 576 577 noinline void vmptrld_error(struct vmcs *vmcs, u64 phys_addr) 578 { 579 vmx_insn_failed("vmptrld failed: %p/%llx err=%u\n", 580 vmcs, phys_addr, vmcs_read32(VM_INSTRUCTION_ERROR)); 581 } 582 583 noinline void invvpid_error(unsigned long ext, u16 vpid, gva_t gva) 584 { 585 vmx_insn_failed("invvpid failed: ext=0x%lx vpid=%u gva=0x%lx\n", 586 ext, vpid, gva); 587 } 588 589 noinline void invept_error(unsigned long ext, u64 eptp) 590 { 591 vmx_insn_failed("invept failed: ext=0x%lx eptp=%llx\n", ext, eptp); 592 } 593 594 DEFINE_PER_CPU(struct vmcs *, current_vmcs); 595 /* 596 * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed 597 * when a CPU is brought down, and we need to VMCLEAR all VMCSs loaded on it. 598 */ 599 static DEFINE_PER_CPU(struct list_head, loaded_vmcss_on_cpu); 600 601 static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS); 602 static DEFINE_SPINLOCK(vmx_vpid_lock); 603 604 struct vmcs_config vmcs_config __ro_after_init; 605 struct vmx_capability vmx_capability __ro_after_init; 606 607 #define VMX_SEGMENT_FIELD(seg) \ 608 [VCPU_SREG_##seg] = { \ 609 .selector = GUEST_##seg##_SELECTOR, \ 610 .base = GUEST_##seg##_BASE, \ 611 .limit = GUEST_##seg##_LIMIT, \ 612 .ar_bytes = GUEST_##seg##_AR_BYTES, \ 613 } 614 615 static const struct kvm_vmx_segment_field { 616 unsigned selector; 617 unsigned base; 618 unsigned limit; 619 unsigned ar_bytes; 620 } kvm_vmx_segment_fields[] = { 621 VMX_SEGMENT_FIELD(CS), 622 VMX_SEGMENT_FIELD(DS), 623 VMX_SEGMENT_FIELD(ES), 624 VMX_SEGMENT_FIELD(FS), 625 VMX_SEGMENT_FIELD(GS), 626 VMX_SEGMENT_FIELD(SS), 627 VMX_SEGMENT_FIELD(TR), 628 VMX_SEGMENT_FIELD(LDTR), 629 }; 630 631 632 static unsigned long host_idt_base; 633 634 #if IS_ENABLED(CONFIG_HYPERV) 635 static bool __read_mostly enlightened_vmcs = true; 636 module_param(enlightened_vmcs, bool, 0444); 637 638 static int hv_enable_l2_tlb_flush(struct kvm_vcpu *vcpu) 639 { 640 struct hv_enlightened_vmcs *evmcs; 641 hpa_t partition_assist_page = hv_get_partition_assist_page(vcpu); 642 643 if (partition_assist_page == INVALID_PAGE) 644 return -ENOMEM; 645 646 evmcs = (struct hv_enlightened_vmcs *)to_vmx(vcpu)->loaded_vmcs->vmcs; 647 648 evmcs->partition_assist_page = partition_assist_page; 649 evmcs->hv_vm_id = (unsigned long)vcpu->kvm; 650 evmcs->hv_enlightenments_control.nested_flush_hypercall = 1; 651 652 return 0; 653 } 654 655 static __init void hv_init_evmcs(void) 656 { 657 int cpu; 658 659 if (!enlightened_vmcs) 660 return; 661 662 /* 663 * Enlightened VMCS usage should be recommended and the host needs 664 * to support eVMCS v1 or above. 665 */ 666 if (ms_hyperv.hints & HV_X64_ENLIGHTENED_VMCS_RECOMMENDED && 667 (ms_hyperv.nested_features & HV_X64_ENLIGHTENED_VMCS_VERSION) >= 668 KVM_EVMCS_VERSION) { 669 670 /* Check that we have assist pages on all online CPUs */ 671 for_each_online_cpu(cpu) { 672 if (!hv_get_vp_assist_page(cpu)) { 673 enlightened_vmcs = false; 674 break; 675 } 676 } 677 678 if (enlightened_vmcs) { 679 pr_info("Using Hyper-V Enlightened VMCS\n"); 680 static_branch_enable(&__kvm_is_using_evmcs); 681 } 682 683 if (ms_hyperv.nested_features & HV_X64_NESTED_DIRECT_FLUSH) 684 vt_x86_ops.enable_l2_tlb_flush 685 = hv_enable_l2_tlb_flush; 686 } else { 687 enlightened_vmcs = false; 688 } 689 } 690 691 static void hv_reset_evmcs(void) 692 { 693 struct hv_vp_assist_page *vp_ap; 694 695 if (!kvm_is_using_evmcs()) 696 return; 697 698 /* 699 * KVM should enable eVMCS if and only if all CPUs have a VP assist 700 * page, and should reject CPU onlining if eVMCS is enabled the CPU 701 * doesn't have a VP assist page allocated. 702 */ 703 vp_ap = hv_get_vp_assist_page(smp_processor_id()); 704 if (WARN_ON_ONCE(!vp_ap)) 705 return; 706 707 /* 708 * Reset everything to support using non-enlightened VMCS access later 709 * (e.g. when we reload the module with enlightened_vmcs=0) 710 */ 711 vp_ap->nested_control.features.directhypercall = 0; 712 vp_ap->current_nested_vmcs = 0; 713 vp_ap->enlighten_vmentry = 0; 714 } 715 716 #else /* IS_ENABLED(CONFIG_HYPERV) */ 717 static void hv_init_evmcs(void) {} 718 static void hv_reset_evmcs(void) {} 719 #endif /* IS_ENABLED(CONFIG_HYPERV) */ 720 721 /* 722 * Comment's format: document - errata name - stepping - processor name. 723 * Refer from 724 * https://www.virtualbox.org/svn/vbox/trunk/src/VBox/VMM/VMMR0/HMR0.cpp 725 */ 726 static u32 vmx_preemption_cpu_tfms[] = { 727 /* 323344.pdf - BA86 - D0 - Xeon 7500 Series */ 728 0x000206E6, 729 /* 323056.pdf - AAX65 - C2 - Xeon L3406 */ 730 /* 322814.pdf - AAT59 - C2 - i7-600, i5-500, i5-400 and i3-300 Mobile */ 731 /* 322911.pdf - AAU65 - C2 - i5-600, i3-500 Desktop and Pentium G6950 */ 732 0x00020652, 733 /* 322911.pdf - AAU65 - K0 - i5-600, i3-500 Desktop and Pentium G6950 */ 734 0x00020655, 735 /* 322373.pdf - AAO95 - B1 - Xeon 3400 Series */ 736 /* 322166.pdf - AAN92 - B1 - i7-800 and i5-700 Desktop */ 737 /* 738 * 320767.pdf - AAP86 - B1 - 739 * i7-900 Mobile Extreme, i7-800 and i7-700 Mobile 740 */ 741 0x000106E5, 742 /* 321333.pdf - AAM126 - C0 - Xeon 3500 */ 743 0x000106A0, 744 /* 321333.pdf - AAM126 - C1 - Xeon 3500 */ 745 0x000106A1, 746 /* 320836.pdf - AAJ124 - C0 - i7-900 Desktop Extreme and i7-900 Desktop */ 747 0x000106A4, 748 /* 321333.pdf - AAM126 - D0 - Xeon 3500 */ 749 /* 321324.pdf - AAK139 - D0 - Xeon 5500 */ 750 /* 320836.pdf - AAJ124 - D0 - i7-900 Extreme and i7-900 Desktop */ 751 0x000106A5, 752 /* Xeon E3-1220 V2 */ 753 0x000306A8, 754 }; 755 756 static inline bool cpu_has_broken_vmx_preemption_timer(void) 757 { 758 u32 eax = cpuid_eax(0x00000001), i; 759 760 /* Clear the reserved bits */ 761 eax &= ~(0x3U << 14 | 0xfU << 28); 762 for (i = 0; i < ARRAY_SIZE(vmx_preemption_cpu_tfms); i++) 763 if (eax == vmx_preemption_cpu_tfms[i]) 764 return true; 765 766 return false; 767 } 768 769 static inline bool cpu_need_virtualize_apic_accesses(struct kvm_vcpu *vcpu) 770 { 771 return flexpriority_enabled && lapic_in_kernel(vcpu); 772 } 773 774 struct vmx_uret_msr *vmx_find_uret_msr(struct vcpu_vmx *vmx, u32 msr) 775 { 776 int i; 777 778 i = kvm_find_user_return_msr(msr); 779 if (i >= 0) 780 return &vmx->guest_uret_msrs[i]; 781 return NULL; 782 } 783 784 static int vmx_set_guest_uret_msr(struct vcpu_vmx *vmx, 785 struct vmx_uret_msr *msr, u64 data) 786 { 787 unsigned int slot = msr - vmx->guest_uret_msrs; 788 int ret = 0; 789 790 if (msr->load_into_hardware) { 791 preempt_disable(); 792 ret = kvm_set_user_return_msr(slot, data, msr->mask); 793 preempt_enable(); 794 } 795 if (!ret) 796 msr->data = data; 797 return ret; 798 } 799 800 void vmx_emergency_disable_virtualization_cpu(void) 801 { 802 int cpu = raw_smp_processor_id(); 803 struct loaded_vmcs *v; 804 805 list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu), 806 loaded_vmcss_on_cpu_link) { 807 vmcs_clear(v->vmcs); 808 if (v->shadow_vmcs) 809 vmcs_clear(v->shadow_vmcs); 810 } 811 } 812 813 static void __loaded_vmcs_clear(void *arg) 814 { 815 struct loaded_vmcs *loaded_vmcs = arg; 816 int cpu = raw_smp_processor_id(); 817 818 if (loaded_vmcs->cpu != cpu) 819 return; /* vcpu migration can race with cpu offline */ 820 if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs) 821 per_cpu(current_vmcs, cpu) = NULL; 822 823 vmcs_clear(loaded_vmcs->vmcs); 824 if (loaded_vmcs->shadow_vmcs && loaded_vmcs->launched) 825 vmcs_clear(loaded_vmcs->shadow_vmcs); 826 827 list_del(&loaded_vmcs->loaded_vmcss_on_cpu_link); 828 829 /* 830 * Ensure all writes to loaded_vmcs, including deleting it from its 831 * current percpu list, complete before setting loaded_vmcs->cpu to 832 * -1, otherwise a different cpu can see loaded_vmcs->cpu == -1 first 833 * and add loaded_vmcs to its percpu list before it's deleted from this 834 * cpu's list. Pairs with the smp_rmb() in vmx_vcpu_load_vmcs(). 835 */ 836 smp_wmb(); 837 838 loaded_vmcs->cpu = -1; 839 loaded_vmcs->launched = 0; 840 } 841 842 static void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs) 843 { 844 int cpu = loaded_vmcs->cpu; 845 846 if (cpu != -1) 847 smp_call_function_single(cpu, 848 __loaded_vmcs_clear, loaded_vmcs, 1); 849 } 850 851 static bool vmx_segment_cache_test_set(struct vcpu_vmx *vmx, unsigned seg, 852 unsigned field) 853 { 854 bool ret; 855 u32 mask = 1 << (seg * SEG_FIELD_NR + field); 856 857 if (!kvm_register_is_available(&vmx->vcpu, VCPU_REG_SEGMENTS)) { 858 kvm_register_mark_available(&vmx->vcpu, VCPU_REG_SEGMENTS); 859 vmx->segment_cache.bitmask = 0; 860 } 861 ret = vmx->segment_cache.bitmask & mask; 862 vmx->segment_cache.bitmask |= mask; 863 return ret; 864 } 865 866 static u16 vmx_read_guest_seg_selector(struct vcpu_vmx *vmx, unsigned seg) 867 { 868 u16 *p = &vmx->segment_cache.seg[seg].selector; 869 870 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_SEL)) 871 *p = vmcs_read16(kvm_vmx_segment_fields[seg].selector); 872 return *p; 873 } 874 875 static ulong vmx_read_guest_seg_base(struct vcpu_vmx *vmx, unsigned seg) 876 { 877 ulong *p = &vmx->segment_cache.seg[seg].base; 878 879 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_BASE)) 880 *p = vmcs_readl(kvm_vmx_segment_fields[seg].base); 881 return *p; 882 } 883 884 static u32 vmx_read_guest_seg_limit(struct vcpu_vmx *vmx, unsigned seg) 885 { 886 u32 *p = &vmx->segment_cache.seg[seg].limit; 887 888 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_LIMIT)) 889 *p = vmcs_read32(kvm_vmx_segment_fields[seg].limit); 890 return *p; 891 } 892 893 static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg) 894 { 895 u32 *p = &vmx->segment_cache.seg[seg].ar; 896 897 if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_AR)) 898 *p = vmcs_read32(kvm_vmx_segment_fields[seg].ar_bytes); 899 return *p; 900 } 901 902 void vmx_update_exception_bitmap(struct kvm_vcpu *vcpu) 903 { 904 u32 eb; 905 906 eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) | 907 (1u << DB_VECTOR) | (1u << AC_VECTOR); 908 /* 909 * #VE isn't used for VMX. To test against unexpected changes 910 * related to #VE for VMX, intercept unexpected #VE and warn on it. 911 */ 912 if (IS_ENABLED(CONFIG_KVM_INTEL_PROVE_VE)) 913 eb |= 1u << VE_VECTOR; 914 /* 915 * Guest access to VMware backdoor ports could legitimately 916 * trigger #GP because of TSS I/O permission bitmap. 917 * We intercept those #GP and allow access to them anyway 918 * as VMware does. 919 */ 920 if (enable_vmware_backdoor) 921 eb |= (1u << GP_VECTOR); 922 if ((vcpu->guest_debug & 923 (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) == 924 (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) 925 eb |= 1u << BP_VECTOR; 926 if (to_vmx(vcpu)->rmode.vm86_active) 927 eb = ~0; 928 if (!vmx_need_pf_intercept(vcpu)) 929 eb &= ~(1u << PF_VECTOR); 930 931 /* When we are running a nested L2 guest and L1 specified for it a 932 * certain exception bitmap, we must trap the same exceptions and pass 933 * them to L1. When running L2, we will only handle the exceptions 934 * specified above if L1 did not want them. 935 */ 936 if (is_guest_mode(vcpu)) 937 eb |= get_vmcs12(vcpu)->exception_bitmap; 938 else { 939 int mask = 0, match = 0; 940 941 if (enable_ept && (eb & (1u << PF_VECTOR))) { 942 /* 943 * If EPT is enabled, #PF is currently only intercepted 944 * if MAXPHYADDR is smaller on the guest than on the 945 * host. In that case we only care about present, 946 * non-reserved faults. For vmcs02, however, PFEC_MASK 947 * and PFEC_MATCH are set in prepare_vmcs02_rare. 948 */ 949 mask = PFERR_PRESENT_MASK | PFERR_RSVD_MASK; 950 match = PFERR_PRESENT_MASK; 951 } 952 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, mask); 953 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, match); 954 } 955 956 /* 957 * Disabling xfd interception indicates that dynamic xfeatures 958 * might be used in the guest. Always trap #NM in this case 959 * to save guest xfd_err timely. 960 */ 961 if (vcpu->arch.xfd_no_write_intercept) 962 eb |= (1u << NM_VECTOR); 963 964 vmcs_write32(EXCEPTION_BITMAP, eb); 965 } 966 967 /* 968 * Check if MSR is intercepted for currently loaded MSR bitmap. 969 */ 970 static bool msr_write_intercepted(struct vcpu_vmx *vmx, u32 msr) 971 { 972 if (!(exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS)) 973 return true; 974 975 return vmx_test_msr_bitmap_write(vmx->loaded_vmcs->msr_bitmap, msr); 976 } 977 978 unsigned int __vmx_vcpu_enter_flags(struct vcpu_vmx *vmx) 979 { 980 unsigned int flags = 0; 981 982 if (vmx->loaded_vmcs->launched) 983 flags |= KVM_ENTER_VMRESUME; 984 985 /* 986 * If writes to the SPEC_CTRL MSR aren't intercepted, the guest is free 987 * to change it directly without causing a vmexit. In that case read 988 * it after vmexit and store it in vmx->spec_ctrl. 989 */ 990 if (!msr_write_intercepted(vmx, MSR_IA32_SPEC_CTRL)) 991 flags |= KVM_ENTER_SAVE_SPEC_CTRL; 992 993 if (cpu_feature_enabled(X86_FEATURE_CLEAR_CPU_BUF_VM_MMIO) && 994 kvm_vcpu_can_access_host_mmio(&vmx->vcpu)) 995 flags |= KVM_ENTER_CLEAR_CPU_BUFFERS_FOR_MMIO; 996 997 return flags; 998 } 999 1000 static __always_inline void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx, 1001 unsigned long entry, unsigned long exit) 1002 { 1003 vm_entry_controls_clearbit(vmx, entry); 1004 vm_exit_controls_clearbit(vmx, exit); 1005 } 1006 1007 static int vmx_find_loadstore_msr_slot(struct vmx_msrs *m, u32 msr) 1008 { 1009 unsigned int i; 1010 1011 for (i = 0; i < m->nr; ++i) { 1012 if (m->val[i].index == msr) 1013 return i; 1014 } 1015 return -ENOENT; 1016 } 1017 1018 static void vmx_remove_auto_msr(struct vmx_msrs *m, u32 msr, 1019 unsigned long vmcs_count_field) 1020 { 1021 int i; 1022 1023 i = vmx_find_loadstore_msr_slot(m, msr); 1024 if (i < 0) 1025 return; 1026 1027 --m->nr; 1028 m->val[i] = m->val[m->nr]; 1029 vmcs_write32(vmcs_count_field, m->nr); 1030 } 1031 1032 static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr) 1033 { 1034 struct msr_autoload *m = &vmx->msr_autoload; 1035 1036 switch (msr) { 1037 case MSR_EFER: 1038 if (cpu_has_load_ia32_efer()) { 1039 clear_atomic_switch_msr_special(vmx, 1040 VM_ENTRY_LOAD_IA32_EFER, 1041 VM_EXIT_LOAD_IA32_EFER); 1042 return; 1043 } 1044 break; 1045 case MSR_CORE_PERF_GLOBAL_CTRL: 1046 if (cpu_has_load_perf_global_ctrl()) { 1047 clear_atomic_switch_msr_special(vmx, 1048 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL, 1049 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL); 1050 return; 1051 } 1052 break; 1053 } 1054 1055 vmx_remove_auto_msr(&m->guest, msr, VM_ENTRY_MSR_LOAD_COUNT); 1056 vmx_remove_auto_msr(&m->host, msr, VM_EXIT_MSR_LOAD_COUNT); 1057 } 1058 1059 static __always_inline void add_atomic_switch_msr_special(struct vcpu_vmx *vmx, 1060 unsigned long entry, unsigned long exit, 1061 unsigned long guest_val_vmcs, unsigned long host_val_vmcs, 1062 u64 guest_val, u64 host_val) 1063 { 1064 vmcs_write64(guest_val_vmcs, guest_val); 1065 if (host_val_vmcs != HOST_IA32_EFER) 1066 vmcs_write64(host_val_vmcs, host_val); 1067 vm_entry_controls_setbit(vmx, entry); 1068 vm_exit_controls_setbit(vmx, exit); 1069 } 1070 1071 static void vmx_add_auto_msr(struct vmx_msrs *m, u32 msr, u64 value, 1072 unsigned long vmcs_count_field, struct kvm *kvm) 1073 { 1074 int i; 1075 1076 i = vmx_find_loadstore_msr_slot(m, msr); 1077 if (i < 0) { 1078 if (KVM_BUG_ON(m->nr == MAX_NR_LOADSTORE_MSRS, kvm)) 1079 return; 1080 1081 i = m->nr++; 1082 m->val[i].index = msr; 1083 vmcs_write32(vmcs_count_field, m->nr); 1084 } 1085 m->val[i].value = value; 1086 } 1087 1088 static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr, 1089 u64 guest_val, u64 host_val) 1090 { 1091 struct msr_autoload *m = &vmx->msr_autoload; 1092 struct kvm *kvm = vmx->vcpu.kvm; 1093 1094 switch (msr) { 1095 case MSR_EFER: 1096 if (cpu_has_load_ia32_efer()) { 1097 add_atomic_switch_msr_special(vmx, 1098 VM_ENTRY_LOAD_IA32_EFER, 1099 VM_EXIT_LOAD_IA32_EFER, 1100 GUEST_IA32_EFER, 1101 HOST_IA32_EFER, 1102 guest_val, host_val); 1103 return; 1104 } 1105 break; 1106 case MSR_CORE_PERF_GLOBAL_CTRL: 1107 if (cpu_has_load_perf_global_ctrl()) { 1108 add_atomic_switch_msr_special(vmx, 1109 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL, 1110 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL, 1111 GUEST_IA32_PERF_GLOBAL_CTRL, 1112 HOST_IA32_PERF_GLOBAL_CTRL, 1113 guest_val, host_val); 1114 return; 1115 } 1116 break; 1117 case MSR_IA32_PEBS_ENABLE: 1118 /* PEBS needs a quiescent period after being disabled (to write 1119 * a record). Disabling PEBS through VMX MSR swapping doesn't 1120 * provide that period, so a CPU could write host's record into 1121 * guest's memory. 1122 */ 1123 wrmsrq(MSR_IA32_PEBS_ENABLE, 0); 1124 } 1125 1126 vmx_add_auto_msr(&m->guest, msr, guest_val, VM_ENTRY_MSR_LOAD_COUNT, kvm); 1127 vmx_add_auto_msr(&m->host, msr, host_val, VM_EXIT_MSR_LOAD_COUNT, kvm); 1128 } 1129 1130 static bool update_transition_efer(struct vcpu_vmx *vmx) 1131 { 1132 u64 guest_efer = vmx->vcpu.arch.efer; 1133 u64 ignore_bits = 0; 1134 int i; 1135 1136 /* Shadow paging assumes NX to be available. */ 1137 if (!enable_ept) 1138 guest_efer |= EFER_NX; 1139 1140 /* 1141 * LMA and LME handled by hardware; SCE meaningless outside long mode. 1142 */ 1143 ignore_bits |= EFER_SCE; 1144 #ifdef CONFIG_X86_64 1145 ignore_bits |= EFER_LMA | EFER_LME; 1146 /* SCE is meaningful only in long mode on Intel */ 1147 if (guest_efer & EFER_LMA) 1148 ignore_bits &= ~(u64)EFER_SCE; 1149 #endif 1150 1151 /* 1152 * On EPT, we can't emulate NX, so we must switch EFER atomically. 1153 * On CPUs that support "load IA32_EFER", always switch EFER 1154 * atomically, since it's faster than switching it manually. 1155 */ 1156 if (cpu_has_load_ia32_efer() || 1157 (enable_ept && ((vmx->vcpu.arch.efer ^ kvm_host.efer) & EFER_NX))) { 1158 if (!(guest_efer & EFER_LMA)) 1159 guest_efer &= ~EFER_LME; 1160 if (guest_efer != kvm_host.efer) 1161 add_atomic_switch_msr(vmx, MSR_EFER, guest_efer, kvm_host.efer); 1162 else 1163 clear_atomic_switch_msr(vmx, MSR_EFER); 1164 return false; 1165 } 1166 1167 i = kvm_find_user_return_msr(MSR_EFER); 1168 if (i < 0) 1169 return false; 1170 1171 clear_atomic_switch_msr(vmx, MSR_EFER); 1172 1173 guest_efer &= ~ignore_bits; 1174 guest_efer |= kvm_host.efer & ignore_bits; 1175 1176 vmx->guest_uret_msrs[i].data = guest_efer; 1177 vmx->guest_uret_msrs[i].mask = ~ignore_bits; 1178 1179 return true; 1180 } 1181 1182 static void vmx_add_autostore_msr(struct vcpu_vmx *vmx, u32 msr) 1183 { 1184 vmx_add_auto_msr(&vmx->msr_autostore, msr, 0, VM_EXIT_MSR_STORE_COUNT, 1185 vmx->vcpu.kvm); 1186 } 1187 1188 static void vmx_remove_autostore_msr(struct vcpu_vmx *vmx, u32 msr) 1189 { 1190 vmx_remove_auto_msr(&vmx->msr_autostore, msr, VM_EXIT_MSR_STORE_COUNT); 1191 } 1192 1193 static u16 vmx_store_ldt(void) 1194 { 1195 u16 ldt; 1196 asm("sldt %0" : "=g"(ldt)); 1197 return ldt; 1198 } 1199 1200 static void vmx_load_ldt(u16 sel) 1201 { 1202 asm("lldt %0" : : "rm"(sel)); 1203 } 1204 1205 #ifdef CONFIG_X86_32 1206 /* 1207 * On 32-bit kernels, VM exits still load the FS and GS bases from the 1208 * VMCS rather than the segment table. KVM uses this helper to figure 1209 * out the current bases to poke them into the VMCS before entry. 1210 */ 1211 static unsigned long segment_base(u16 selector) 1212 { 1213 struct desc_struct *table; 1214 unsigned long v; 1215 1216 if (!(selector & ~SEGMENT_RPL_MASK)) 1217 return 0; 1218 1219 table = get_current_gdt_ro(); 1220 1221 if ((selector & SEGMENT_TI_MASK) == SEGMENT_LDT) { 1222 u16 ldt_selector = vmx_store_ldt(); 1223 1224 if (!(ldt_selector & ~SEGMENT_RPL_MASK)) 1225 return 0; 1226 1227 table = (struct desc_struct *)segment_base(ldt_selector); 1228 } 1229 v = get_desc_base(&table[selector >> 3]); 1230 return v; 1231 } 1232 #endif 1233 1234 static inline bool pt_can_write_msr(struct vcpu_vmx *vmx) 1235 { 1236 return vmx_pt_mode_is_host_guest() && 1237 !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN); 1238 } 1239 1240 static inline bool pt_output_base_valid(struct kvm_vcpu *vcpu, u64 base) 1241 { 1242 /* The base must be 128-byte aligned and a legal physical address. */ 1243 return kvm_vcpu_is_legal_aligned_gpa(vcpu, base, 128); 1244 } 1245 1246 static inline void pt_load_msr(struct pt_ctx *ctx, u32 addr_range) 1247 { 1248 u32 i; 1249 1250 wrmsrq(MSR_IA32_RTIT_STATUS, ctx->status); 1251 wrmsrq(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base); 1252 wrmsrq(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask); 1253 wrmsrq(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match); 1254 for (i = 0; i < addr_range; i++) { 1255 wrmsrq(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]); 1256 wrmsrq(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]); 1257 } 1258 } 1259 1260 static inline void pt_save_msr(struct pt_ctx *ctx, u32 addr_range) 1261 { 1262 u32 i; 1263 1264 rdmsrq(MSR_IA32_RTIT_STATUS, ctx->status); 1265 rdmsrq(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base); 1266 rdmsrq(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask); 1267 rdmsrq(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match); 1268 for (i = 0; i < addr_range; i++) { 1269 rdmsrq(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]); 1270 rdmsrq(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]); 1271 } 1272 } 1273 1274 static void pt_guest_enter(struct vcpu_vmx *vmx) 1275 { 1276 if (vmx_pt_mode_is_system()) 1277 return; 1278 1279 /* 1280 * GUEST_IA32_RTIT_CTL is already set in the VMCS. 1281 * Save host state before VM entry. 1282 */ 1283 rdmsrq(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); 1284 if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) { 1285 wrmsrq(MSR_IA32_RTIT_CTL, 0); 1286 pt_save_msr(&vmx->pt_desc.host, vmx->pt_desc.num_address_ranges); 1287 pt_load_msr(&vmx->pt_desc.guest, vmx->pt_desc.num_address_ranges); 1288 } 1289 } 1290 1291 static void pt_guest_exit(struct vcpu_vmx *vmx) 1292 { 1293 if (vmx_pt_mode_is_system()) 1294 return; 1295 1296 if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) { 1297 pt_save_msr(&vmx->pt_desc.guest, vmx->pt_desc.num_address_ranges); 1298 pt_load_msr(&vmx->pt_desc.host, vmx->pt_desc.num_address_ranges); 1299 } 1300 1301 /* 1302 * KVM requires VM_EXIT_CLEAR_IA32_RTIT_CTL to expose PT to the guest, 1303 * i.e. RTIT_CTL is always cleared on VM-Exit. Restore it if necessary. 1304 */ 1305 if (vmx->pt_desc.host.ctl) 1306 wrmsrq(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl); 1307 } 1308 1309 void vmx_set_host_fs_gs(struct vmcs_host_state *host, u16 fs_sel, u16 gs_sel, 1310 unsigned long fs_base, unsigned long gs_base) 1311 { 1312 if (unlikely(fs_sel != host->fs_sel)) { 1313 if (!(fs_sel & 7)) 1314 vmcs_write16(HOST_FS_SELECTOR, fs_sel); 1315 else 1316 vmcs_write16(HOST_FS_SELECTOR, 0); 1317 host->fs_sel = fs_sel; 1318 } 1319 if (unlikely(gs_sel != host->gs_sel)) { 1320 if (!(gs_sel & 7)) 1321 vmcs_write16(HOST_GS_SELECTOR, gs_sel); 1322 else 1323 vmcs_write16(HOST_GS_SELECTOR, 0); 1324 host->gs_sel = gs_sel; 1325 } 1326 if (unlikely(fs_base != host->fs_base)) { 1327 vmcs_writel(HOST_FS_BASE, fs_base); 1328 host->fs_base = fs_base; 1329 } 1330 if (unlikely(gs_base != host->gs_base)) { 1331 vmcs_writel(HOST_GS_BASE, gs_base); 1332 host->gs_base = gs_base; 1333 } 1334 } 1335 1336 void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu) 1337 { 1338 struct vcpu_vmx *vmx = to_vmx(vcpu); 1339 struct vcpu_vt *vt = to_vt(vcpu); 1340 struct vmcs_host_state *host_state; 1341 #ifdef CONFIG_X86_64 1342 int cpu = raw_smp_processor_id(); 1343 #endif 1344 unsigned long fs_base, gs_base; 1345 u16 fs_sel, gs_sel; 1346 int i; 1347 1348 /* 1349 * Note that guest MSRs to be saved/restored can also be changed 1350 * when guest state is loaded. This happens when guest transitions 1351 * to/from long-mode by setting MSR_EFER.LMA. 1352 */ 1353 if (!vmx->guest_uret_msrs_loaded) { 1354 vmx->guest_uret_msrs_loaded = true; 1355 for (i = 0; i < kvm_nr_uret_msrs; ++i) { 1356 if (!vmx->guest_uret_msrs[i].load_into_hardware) 1357 continue; 1358 1359 kvm_set_user_return_msr(i, 1360 vmx->guest_uret_msrs[i].data, 1361 vmx->guest_uret_msrs[i].mask); 1362 } 1363 } 1364 1365 if (vmx->nested.need_vmcs12_to_shadow_sync) 1366 nested_sync_vmcs12_to_shadow(vcpu); 1367 1368 if (vt->guest_state_loaded) 1369 return; 1370 1371 host_state = &vmx->loaded_vmcs->host_state; 1372 1373 /* 1374 * Set host fs and gs selectors. Unfortunately, 22.2.3 does not 1375 * allow segment selectors with cpl > 0 or ti == 1. 1376 */ 1377 host_state->ldt_sel = vmx_store_ldt(); 1378 1379 #ifdef CONFIG_X86_64 1380 savesegment(ds, host_state->ds_sel); 1381 savesegment(es, host_state->es_sel); 1382 1383 gs_base = cpu_kernelmode_gs_base(cpu); 1384 if (likely(is_64bit_mm(current->mm))) { 1385 current_save_fsgs(); 1386 fs_sel = current->thread.fsindex; 1387 gs_sel = current->thread.gsindex; 1388 fs_base = current->thread.fsbase; 1389 vt->msr_host_kernel_gs_base = current->thread.gsbase; 1390 } else { 1391 savesegment(fs, fs_sel); 1392 savesegment(gs, gs_sel); 1393 fs_base = read_msr(MSR_FS_BASE); 1394 vt->msr_host_kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE); 1395 } 1396 1397 wrmsrq(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); 1398 #else 1399 savesegment(fs, fs_sel); 1400 savesegment(gs, gs_sel); 1401 fs_base = segment_base(fs_sel); 1402 gs_base = segment_base(gs_sel); 1403 #endif 1404 1405 vmx_set_host_fs_gs(host_state, fs_sel, gs_sel, fs_base, gs_base); 1406 vt->guest_state_loaded = true; 1407 } 1408 1409 static void vmx_prepare_switch_to_host(struct vcpu_vmx *vmx) 1410 { 1411 struct vmcs_host_state *host_state; 1412 1413 if (!vmx->vt.guest_state_loaded) 1414 return; 1415 1416 host_state = &vmx->loaded_vmcs->host_state; 1417 1418 ++vmx->vcpu.stat.host_state_reload; 1419 1420 #ifdef CONFIG_X86_64 1421 rdmsrq(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base); 1422 #endif 1423 if (host_state->ldt_sel || (host_state->gs_sel & 7)) { 1424 vmx_load_ldt(host_state->ldt_sel); 1425 #ifdef CONFIG_X86_64 1426 load_gs_index(host_state->gs_sel); 1427 #else 1428 loadsegment(gs, host_state->gs_sel); 1429 #endif 1430 } 1431 if (host_state->fs_sel & 7) 1432 loadsegment(fs, host_state->fs_sel); 1433 #ifdef CONFIG_X86_64 1434 if (unlikely(host_state->ds_sel | host_state->es_sel)) { 1435 loadsegment(ds, host_state->ds_sel); 1436 loadsegment(es, host_state->es_sel); 1437 } 1438 #endif 1439 invalidate_tss_limit(); 1440 #ifdef CONFIG_X86_64 1441 wrmsrq(MSR_KERNEL_GS_BASE, vmx->vt.msr_host_kernel_gs_base); 1442 #endif 1443 load_fixmap_gdt(raw_smp_processor_id()); 1444 vmx->vt.guest_state_loaded = false; 1445 vmx->guest_uret_msrs_loaded = false; 1446 } 1447 1448 #ifdef CONFIG_X86_64 1449 static u64 vmx_read_guest_host_msr(struct vcpu_vmx *vmx, u32 msr, u64 *cache) 1450 { 1451 preempt_disable(); 1452 if (vmx->vt.guest_state_loaded) 1453 *cache = read_msr(msr); 1454 preempt_enable(); 1455 return *cache; 1456 } 1457 1458 static void vmx_write_guest_host_msr(struct vcpu_vmx *vmx, u32 msr, u64 data, 1459 u64 *cache) 1460 { 1461 preempt_disable(); 1462 if (vmx->vt.guest_state_loaded) 1463 wrmsrns(msr, data); 1464 preempt_enable(); 1465 *cache = data; 1466 } 1467 1468 static u64 vmx_read_guest_kernel_gs_base(struct vcpu_vmx *vmx) 1469 { 1470 return vmx_read_guest_host_msr(vmx, MSR_KERNEL_GS_BASE, 1471 &vmx->msr_guest_kernel_gs_base); 1472 } 1473 1474 static void vmx_write_guest_kernel_gs_base(struct vcpu_vmx *vmx, u64 data) 1475 { 1476 vmx_write_guest_host_msr(vmx, MSR_KERNEL_GS_BASE, data, 1477 &vmx->msr_guest_kernel_gs_base); 1478 } 1479 #endif 1480 1481 static void grow_ple_window(struct kvm_vcpu *vcpu) 1482 { 1483 struct vcpu_vmx *vmx = to_vmx(vcpu); 1484 unsigned int old = vmx->ple_window; 1485 1486 vmx->ple_window = __grow_ple_window(old, ple_window, 1487 ple_window_grow, 1488 ple_window_max); 1489 1490 if (vmx->ple_window != old) { 1491 vmx->ple_window_dirty = true; 1492 trace_kvm_ple_window_update(vcpu->vcpu_id, 1493 vmx->ple_window, old); 1494 } 1495 } 1496 1497 static void shrink_ple_window(struct kvm_vcpu *vcpu) 1498 { 1499 struct vcpu_vmx *vmx = to_vmx(vcpu); 1500 unsigned int old = vmx->ple_window; 1501 1502 vmx->ple_window = __shrink_ple_window(old, ple_window, 1503 ple_window_shrink, 1504 ple_window); 1505 1506 if (vmx->ple_window != old) { 1507 vmx->ple_window_dirty = true; 1508 trace_kvm_ple_window_update(vcpu->vcpu_id, 1509 vmx->ple_window, old); 1510 } 1511 } 1512 1513 void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu) 1514 { 1515 struct vcpu_vmx *vmx = to_vmx(vcpu); 1516 bool already_loaded = vmx->loaded_vmcs->cpu == cpu; 1517 struct vmcs *prev; 1518 1519 if (!already_loaded) { 1520 loaded_vmcs_clear(vmx->loaded_vmcs); 1521 local_irq_disable(); 1522 1523 /* 1524 * Ensure loaded_vmcs->cpu is read before adding loaded_vmcs to 1525 * this cpu's percpu list, otherwise it may not yet be deleted 1526 * from its previous cpu's percpu list. Pairs with the 1527 * smb_wmb() in __loaded_vmcs_clear(). 1528 */ 1529 smp_rmb(); 1530 1531 list_add(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link, 1532 &per_cpu(loaded_vmcss_on_cpu, cpu)); 1533 local_irq_enable(); 1534 } 1535 1536 prev = per_cpu(current_vmcs, cpu); 1537 if (prev != vmx->loaded_vmcs->vmcs) { 1538 per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs; 1539 vmcs_load(vmx->loaded_vmcs->vmcs); 1540 } 1541 1542 if (!already_loaded) { 1543 void *gdt = get_current_gdt_ro(); 1544 1545 /* 1546 * Flush all EPTP/VPID contexts, the new pCPU may have stale 1547 * TLB entries from its previous association with the vCPU. 1548 */ 1549 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 1550 1551 /* 1552 * Linux uses per-cpu TSS and GDT, so set these when switching 1553 * processors. See 22.2.4. 1554 */ 1555 vmcs_writel(HOST_TR_BASE, 1556 (unsigned long)&get_cpu_entry_area(cpu)->tss.x86_tss); 1557 vmcs_writel(HOST_GDTR_BASE, (unsigned long)gdt); /* 22.2.4 */ 1558 1559 if (IS_ENABLED(CONFIG_IA32_EMULATION) || IS_ENABLED(CONFIG_X86_32)) { 1560 /* 22.2.3 */ 1561 vmcs_writel(HOST_IA32_SYSENTER_ESP, 1562 (unsigned long)(cpu_entry_stack(cpu) + 1)); 1563 } 1564 1565 vmx->loaded_vmcs->cpu = cpu; 1566 } 1567 } 1568 1569 /* 1570 * Switches to specified vcpu, until a matching vcpu_put(), but assumes 1571 * vcpu mutex is already taken. 1572 */ 1573 void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1574 { 1575 if (vcpu->scheduled_out && !kvm_pause_in_guest(vcpu->kvm)) 1576 shrink_ple_window(vcpu); 1577 1578 vmx_vcpu_load_vmcs(vcpu, cpu); 1579 1580 vt_vcpu_pi_load(vcpu, cpu); 1581 } 1582 1583 void vmx_vcpu_put(struct kvm_vcpu *vcpu) 1584 { 1585 vt_vcpu_pi_put(vcpu); 1586 1587 vmx_prepare_switch_to_host(to_vmx(vcpu)); 1588 } 1589 1590 static void vmx_switch_loaded_vmcs(struct kvm_vcpu *vcpu, 1591 struct loaded_vmcs *vmcs) 1592 { 1593 struct vcpu_vmx *vmx = to_vmx(vcpu); 1594 int cpu; 1595 1596 cpu = get_cpu(); 1597 vmx->loaded_vmcs = vmcs; 1598 vmx_vcpu_load_vmcs(vcpu, cpu); 1599 put_cpu(); 1600 } 1601 1602 static void vmx_load_vmcs01(struct kvm_vcpu *vcpu) 1603 { 1604 struct vcpu_vmx *vmx = to_vmx(vcpu); 1605 1606 if (!is_guest_mode(vcpu)) { 1607 WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01); 1608 return; 1609 } 1610 1611 WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->nested.vmcs02); 1612 vmx_switch_loaded_vmcs(vcpu, &vmx->vmcs01); 1613 } 1614 1615 static void vmx_put_vmcs01(struct kvm_vcpu *vcpu) 1616 { 1617 if (!is_guest_mode(vcpu)) 1618 return; 1619 1620 vmx_switch_loaded_vmcs(vcpu, &to_vmx(vcpu)->nested.vmcs02); 1621 } 1622 DEFINE_GUARD(vmx_vmcs01, struct kvm_vcpu *, 1623 vmx_load_vmcs01(_T), vmx_put_vmcs01(_T)) 1624 1625 bool vmx_emulation_required(struct kvm_vcpu *vcpu) 1626 { 1627 return emulate_invalid_guest_state && !vmx_guest_state_valid(vcpu); 1628 } 1629 1630 unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu) 1631 { 1632 struct vcpu_vmx *vmx = to_vmx(vcpu); 1633 unsigned long rflags, save_rflags; 1634 1635 if (!kvm_register_is_available(vcpu, VCPU_REG_RFLAGS)) { 1636 kvm_register_mark_available(vcpu, VCPU_REG_RFLAGS); 1637 rflags = vmcs_readl(GUEST_RFLAGS); 1638 if (vmx->rmode.vm86_active) { 1639 rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS; 1640 save_rflags = vmx->rmode.save_rflags; 1641 rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS; 1642 } 1643 vmx->rflags = rflags; 1644 } 1645 return vmx->rflags; 1646 } 1647 1648 void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) 1649 { 1650 struct vcpu_vmx *vmx = to_vmx(vcpu); 1651 unsigned long old_rflags; 1652 1653 /* 1654 * Unlike CR0 and CR4, RFLAGS handling requires checking if the vCPU 1655 * is an unrestricted guest in order to mark L2 as needing emulation 1656 * if L1 runs L2 as a restricted guest. 1657 */ 1658 if (is_unrestricted_guest(vcpu)) { 1659 kvm_register_mark_available(vcpu, VCPU_REG_RFLAGS); 1660 vmx->rflags = rflags; 1661 vmcs_writel(GUEST_RFLAGS, rflags); 1662 return; 1663 } 1664 1665 old_rflags = vmx_get_rflags(vcpu); 1666 vmx->rflags = rflags; 1667 if (vmx->rmode.vm86_active) { 1668 vmx->rmode.save_rflags = rflags; 1669 rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM; 1670 } 1671 vmcs_writel(GUEST_RFLAGS, rflags); 1672 1673 if ((old_rflags ^ vmx->rflags) & X86_EFLAGS_VM) 1674 vmx->vt.emulation_required = vmx_emulation_required(vcpu); 1675 } 1676 1677 bool vmx_get_if_flag(struct kvm_vcpu *vcpu) 1678 { 1679 return vmx_get_rflags(vcpu) & X86_EFLAGS_IF; 1680 } 1681 1682 u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu) 1683 { 1684 u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 1685 int ret = 0; 1686 1687 if (interruptibility & GUEST_INTR_STATE_STI) 1688 ret |= KVM_X86_SHADOW_INT_STI; 1689 if (interruptibility & GUEST_INTR_STATE_MOV_SS) 1690 ret |= KVM_X86_SHADOW_INT_MOV_SS; 1691 1692 return ret; 1693 } 1694 1695 void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask) 1696 { 1697 u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 1698 u32 interruptibility = interruptibility_old; 1699 1700 interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS); 1701 1702 if (mask & KVM_X86_SHADOW_INT_MOV_SS) 1703 interruptibility |= GUEST_INTR_STATE_MOV_SS; 1704 else if (mask & KVM_X86_SHADOW_INT_STI) 1705 interruptibility |= GUEST_INTR_STATE_STI; 1706 1707 if ((interruptibility != interruptibility_old)) 1708 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility); 1709 } 1710 1711 static int vmx_rtit_ctl_check(struct kvm_vcpu *vcpu, u64 data) 1712 { 1713 struct vcpu_vmx *vmx = to_vmx(vcpu); 1714 unsigned long value; 1715 1716 /* 1717 * Any MSR write that attempts to change bits marked reserved will 1718 * case a #GP fault. 1719 */ 1720 if (data & vmx->pt_desc.ctl_bitmask) 1721 return 1; 1722 1723 /* 1724 * Any attempt to modify IA32_RTIT_CTL while TraceEn is set will 1725 * result in a #GP unless the same write also clears TraceEn. 1726 */ 1727 if ((vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) && 1728 (data & RTIT_CTL_TRACEEN) && 1729 data != vmx->pt_desc.guest.ctl) 1730 return 1; 1731 1732 /* 1733 * WRMSR to IA32_RTIT_CTL that sets TraceEn but clears this bit 1734 * and FabricEn would cause #GP, if 1735 * CPUID.(EAX=14H, ECX=0):ECX.SNGLRGNOUT[bit 2] = 0 1736 */ 1737 if ((data & RTIT_CTL_TRACEEN) && !(data & RTIT_CTL_TOPA) && 1738 !(data & RTIT_CTL_FABRIC_EN) && 1739 !intel_pt_validate_cap(vmx->pt_desc.caps, 1740 PT_CAP_single_range_output)) 1741 return 1; 1742 1743 /* 1744 * MTCFreq, CycThresh and PSBFreq encodings check, any MSR write that 1745 * utilize encodings marked reserved will cause a #GP fault. 1746 */ 1747 value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc_periods); 1748 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc) && 1749 !test_bit((data & RTIT_CTL_MTC_RANGE) >> 1750 RTIT_CTL_MTC_RANGE_OFFSET, &value)) 1751 return 1; 1752 value = intel_pt_validate_cap(vmx->pt_desc.caps, 1753 PT_CAP_cycle_thresholds); 1754 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) && 1755 !test_bit((data & RTIT_CTL_CYC_THRESH) >> 1756 RTIT_CTL_CYC_THRESH_OFFSET, &value)) 1757 return 1; 1758 value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_periods); 1759 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) && 1760 !test_bit((data & RTIT_CTL_PSB_FREQ) >> 1761 RTIT_CTL_PSB_FREQ_OFFSET, &value)) 1762 return 1; 1763 1764 /* 1765 * If ADDRx_CFG is reserved or the encodings is >2 will 1766 * cause a #GP fault. 1767 */ 1768 value = (data & RTIT_CTL_ADDR0) >> RTIT_CTL_ADDR0_OFFSET; 1769 if ((value && (vmx->pt_desc.num_address_ranges < 1)) || (value > 2)) 1770 return 1; 1771 value = (data & RTIT_CTL_ADDR1) >> RTIT_CTL_ADDR1_OFFSET; 1772 if ((value && (vmx->pt_desc.num_address_ranges < 2)) || (value > 2)) 1773 return 1; 1774 value = (data & RTIT_CTL_ADDR2) >> RTIT_CTL_ADDR2_OFFSET; 1775 if ((value && (vmx->pt_desc.num_address_ranges < 3)) || (value > 2)) 1776 return 1; 1777 value = (data & RTIT_CTL_ADDR3) >> RTIT_CTL_ADDR3_OFFSET; 1778 if ((value && (vmx->pt_desc.num_address_ranges < 4)) || (value > 2)) 1779 return 1; 1780 1781 return 0; 1782 } 1783 1784 int vmx_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, 1785 void *insn, int insn_len) 1786 { 1787 /* 1788 * Emulation of instructions in SGX enclaves is impossible as RIP does 1789 * not point at the failing instruction, and even if it did, the code 1790 * stream is inaccessible. Inject #UD instead of exiting to userspace 1791 * so that guest userspace can't DoS the guest simply by triggering 1792 * emulation (enclaves are CPL3 only). 1793 */ 1794 if (vt_get_exit_reason(vcpu).enclave_mode) { 1795 kvm_queue_exception(vcpu, UD_VECTOR); 1796 return X86EMUL_PROPAGATE_FAULT; 1797 } 1798 1799 /* Check that emulation is possible during event vectoring */ 1800 if ((to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) && 1801 !kvm_can_emulate_event_vectoring(emul_type)) 1802 return X86EMUL_UNHANDLEABLE_VECTORING; 1803 1804 return X86EMUL_CONTINUE; 1805 } 1806 1807 static int skip_emulated_instruction(struct kvm_vcpu *vcpu) 1808 { 1809 union vmx_exit_reason exit_reason = vt_get_exit_reason(vcpu); 1810 unsigned long rip, orig_rip; 1811 u32 instr_len; 1812 1813 /* 1814 * Using VMCS.VM_EXIT_INSTRUCTION_LEN on EPT misconfig depends on 1815 * undefined behavior: Intel's SDM doesn't mandate the VMCS field be 1816 * set when EPT misconfig occurs. In practice, real hardware updates 1817 * VM_EXIT_INSTRUCTION_LEN on EPT misconfig, but other hypervisors 1818 * (namely Hyper-V) don't set it due to it being undefined behavior, 1819 * i.e. we end up advancing IP with some random value. 1820 */ 1821 if (!cpu_feature_enabled(X86_FEATURE_HYPERVISOR) || 1822 exit_reason.basic != EXIT_REASON_EPT_MISCONFIG) { 1823 instr_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 1824 1825 /* 1826 * Emulating an enclave's instructions isn't supported as KVM 1827 * cannot access the enclave's memory or its true RIP, e.g. the 1828 * vmcs.GUEST_RIP points at the exit point of the enclave, not 1829 * the RIP that actually triggered the VM-Exit. But, because 1830 * most instructions that cause VM-Exit will #UD in an enclave, 1831 * most instruction-based VM-Exits simply do not occur. 1832 * 1833 * There are a few exceptions, notably the debug instructions 1834 * INT1ICEBRK and INT3, as they are allowed in debug enclaves 1835 * and generate #DB/#BP as expected, which KVM might intercept. 1836 * But again, the CPU does the dirty work and saves an instr 1837 * length of zero so VMMs don't shoot themselves in the foot. 1838 * WARN if KVM tries to skip a non-zero length instruction on 1839 * a VM-Exit from an enclave. 1840 */ 1841 if (!instr_len) 1842 goto rip_updated; 1843 1844 WARN_ONCE(exit_reason.enclave_mode, 1845 "skipping instruction after SGX enclave VM-Exit"); 1846 1847 orig_rip = kvm_rip_read(vcpu); 1848 rip = orig_rip + instr_len; 1849 #ifdef CONFIG_X86_64 1850 /* 1851 * We need to mask out the high 32 bits of RIP if not in 64-bit 1852 * mode, but just finding out that we are in 64-bit mode is 1853 * quite expensive. Only do it if there was a carry. 1854 */ 1855 if (unlikely(((rip ^ orig_rip) >> 31) == 3) && !is_64_bit_mode(vcpu)) 1856 rip = (u32)rip; 1857 #endif 1858 kvm_rip_write(vcpu, rip); 1859 } else { 1860 if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP)) 1861 return 0; 1862 } 1863 1864 rip_updated: 1865 /* skipping an emulated instruction also counts */ 1866 vmx_set_interrupt_shadow(vcpu, 0); 1867 1868 return 1; 1869 } 1870 1871 /* 1872 * Recognizes a pending MTF VM-exit and records the nested state for later 1873 * delivery. 1874 */ 1875 void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu) 1876 { 1877 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1878 struct vcpu_vmx *vmx = to_vmx(vcpu); 1879 1880 if (!is_guest_mode(vcpu)) 1881 return; 1882 1883 /* 1884 * Per the SDM, MTF takes priority over debug-trap exceptions besides 1885 * TSS T-bit traps and ICEBP (INT1). KVM doesn't emulate T-bit traps 1886 * or ICEBP (in the emulator proper), and skipping of ICEBP after an 1887 * intercepted #DB deliberately avoids single-step #DB and MTF updates 1888 * as ICEBP is higher priority than both. As instruction emulation is 1889 * completed at this point (i.e. KVM is at the instruction boundary), 1890 * any #DB exception pending delivery must be a debug-trap of lower 1891 * priority than MTF. Record the pending MTF state to be delivered in 1892 * vmx_check_nested_events(). 1893 */ 1894 if (nested_cpu_has_mtf(vmcs12) && 1895 (!vcpu->arch.exception.pending || 1896 vcpu->arch.exception.vector == DB_VECTOR) && 1897 (!vcpu->arch.exception_vmexit.pending || 1898 vcpu->arch.exception_vmexit.vector == DB_VECTOR)) { 1899 vmx->nested.mtf_pending = true; 1900 kvm_make_request(KVM_REQ_EVENT, vcpu); 1901 } else { 1902 vmx->nested.mtf_pending = false; 1903 } 1904 } 1905 1906 int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu) 1907 { 1908 vmx_update_emulated_instruction(vcpu); 1909 return skip_emulated_instruction(vcpu); 1910 } 1911 1912 void vmx_clear_hlt(struct kvm_vcpu *vcpu) 1913 { 1914 /* 1915 * Ensure that we clear the HLT state in the VMCS. We don't need to 1916 * explicitly skip the instruction because if the HLT state is set, 1917 * then the instruction is already executing and RIP has already been 1918 * advanced. 1919 */ 1920 if (kvm_hlt_in_guest(vcpu->kvm) && 1921 vmcs_read32(GUEST_ACTIVITY_STATE) == GUEST_ACTIVITY_HLT) 1922 vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE); 1923 } 1924 1925 void vmx_inject_exception(struct kvm_vcpu *vcpu) 1926 { 1927 struct kvm_queued_exception *ex = &vcpu->arch.exception; 1928 u32 intr_info = ex->vector | INTR_INFO_VALID_MASK; 1929 struct vcpu_vmx *vmx = to_vmx(vcpu); 1930 1931 /* 1932 * When injecting a #DB, single-stepping is enabled in RFLAGS, and STI 1933 * or MOV-SS blocking is active, set vmcs.PENDING_DBG_EXCEPTIONS.BS to 1934 * prevent a false positive from VM-Entry consistency check. VM-Entry 1935 * asserts that a single-step #DB _must_ be pending in this scenario, 1936 * as the previous instruction cannot have toggled RFLAGS.TF 0=>1 1937 * (because STI and POP/MOV don't modify RFLAGS), therefore the one 1938 * instruction delay when activating single-step breakpoints must have 1939 * already expired. However, the CPU isn't smart enough to peek at 1940 * vmcs.VM_ENTRY_INTR_INFO_FIELD and so doesn't realize that yes, there 1941 * is indeed a #DB pending/imminent. 1942 */ 1943 if (ex->vector == DB_VECTOR && 1944 (vmx_get_rflags(vcpu) & X86_EFLAGS_TF) && 1945 vmx_get_interrupt_shadow(vcpu)) 1946 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 1947 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS) | DR6_BS); 1948 1949 kvm_deliver_exception_payload(vcpu, ex); 1950 1951 if (ex->has_error_code) { 1952 /* 1953 * Despite the error code being architecturally defined as 32 1954 * bits, and the VMCS field being 32 bits, Intel CPUs and thus 1955 * VMX don't actually supporting setting bits 31:16. Hardware 1956 * will (should) never provide a bogus error code, but AMD CPUs 1957 * do generate error codes with bits 31:16 set, and so KVM's 1958 * ABI lets userspace shove in arbitrary 32-bit values. Drop 1959 * the upper bits to avoid VM-Fail, losing information that 1960 * doesn't really exist is preferable to killing the VM. 1961 */ 1962 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, (u16)ex->error_code); 1963 intr_info |= INTR_INFO_DELIVER_CODE_MASK; 1964 } 1965 1966 if (vmx->rmode.vm86_active) { 1967 int inc_eip = 0; 1968 if (kvm_exception_is_soft(ex->vector)) 1969 inc_eip = vcpu->arch.event_exit_inst_len; 1970 kvm_inject_realmode_interrupt(vcpu, ex->vector, inc_eip); 1971 return; 1972 } 1973 1974 WARN_ON_ONCE(vmx->vt.emulation_required); 1975 1976 if (kvm_exception_is_soft(ex->vector)) { 1977 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1978 vmx->vcpu.arch.event_exit_inst_len); 1979 intr_info |= INTR_TYPE_SOFT_EXCEPTION; 1980 } else 1981 intr_info |= INTR_TYPE_HARD_EXCEPTION; 1982 1983 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info); 1984 1985 vmx_clear_hlt(vcpu); 1986 } 1987 1988 static void vmx_setup_uret_msr(struct vcpu_vmx *vmx, unsigned int msr, 1989 bool load_into_hardware) 1990 { 1991 struct vmx_uret_msr *uret_msr; 1992 1993 uret_msr = vmx_find_uret_msr(vmx, msr); 1994 if (!uret_msr) 1995 return; 1996 1997 uret_msr->load_into_hardware = load_into_hardware; 1998 } 1999 2000 /* 2001 * Configuring user return MSRs to automatically save, load, and restore MSRs 2002 * that need to be shoved into hardware when running the guest. Note, omitting 2003 * an MSR here does _NOT_ mean it's not emulated, only that it will not be 2004 * loaded into hardware when running the guest. 2005 */ 2006 static void vmx_setup_uret_msrs(struct vcpu_vmx *vmx) 2007 { 2008 #ifdef CONFIG_X86_64 2009 bool load_syscall_msrs; 2010 2011 /* 2012 * The SYSCALL MSRs are only needed on long mode guests, and only 2013 * when EFER.SCE is set. 2014 */ 2015 load_syscall_msrs = is_long_mode(&vmx->vcpu) && 2016 (vmx->vcpu.arch.efer & EFER_SCE); 2017 2018 vmx_setup_uret_msr(vmx, MSR_STAR, load_syscall_msrs); 2019 vmx_setup_uret_msr(vmx, MSR_LSTAR, load_syscall_msrs); 2020 vmx_setup_uret_msr(vmx, MSR_SYSCALL_MASK, load_syscall_msrs); 2021 #endif 2022 vmx_setup_uret_msr(vmx, MSR_EFER, update_transition_efer(vmx)); 2023 2024 vmx_setup_uret_msr(vmx, MSR_TSC_AUX, 2025 guest_cpu_cap_has(&vmx->vcpu, X86_FEATURE_RDTSCP) || 2026 guest_cpu_cap_has(&vmx->vcpu, X86_FEATURE_RDPID)); 2027 2028 /* 2029 * hle=0, rtm=0, tsx_ctrl=1 can be found with some combinations of new 2030 * kernel and old userspace. If those guests run on a tsx=off host, do 2031 * allow guests to use TSX_CTRL, but don't change the value in hardware 2032 * so that TSX remains always disabled. 2033 */ 2034 vmx_setup_uret_msr(vmx, MSR_IA32_TSX_CTRL, boot_cpu_has(X86_FEATURE_RTM)); 2035 2036 /* 2037 * The set of MSRs to load may have changed, reload MSRs before the 2038 * next VM-Enter. 2039 */ 2040 vmx->guest_uret_msrs_loaded = false; 2041 } 2042 2043 u64 vmx_get_l2_tsc_offset(struct kvm_vcpu *vcpu) 2044 { 2045 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 2046 2047 if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING)) 2048 return vmcs12->tsc_offset; 2049 2050 return 0; 2051 } 2052 2053 u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu) 2054 { 2055 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 2056 2057 if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING) && 2058 nested_cpu_has2(vmcs12, SECONDARY_EXEC_TSC_SCALING)) 2059 return vmcs12->tsc_multiplier; 2060 2061 return kvm_caps.default_tsc_scaling_ratio; 2062 } 2063 2064 void vmx_write_tsc_offset(struct kvm_vcpu *vcpu) 2065 { 2066 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 2067 } 2068 2069 void vmx_write_tsc_multiplier(struct kvm_vcpu *vcpu) 2070 { 2071 vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio); 2072 } 2073 2074 /* 2075 * Userspace is allowed to set any supported IA32_FEATURE_CONTROL regardless of 2076 * guest CPUID. Note, KVM allows userspace to set "VMX in SMX" to maintain 2077 * backwards compatibility even though KVM doesn't support emulating SMX. And 2078 * because userspace set "VMX in SMX", the guest must also be allowed to set it, 2079 * e.g. if the MSR is left unlocked and the guest does a RMW operation. 2080 */ 2081 #define KVM_SUPPORTED_FEATURE_CONTROL (FEAT_CTL_LOCKED | \ 2082 FEAT_CTL_VMX_ENABLED_INSIDE_SMX | \ 2083 FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX | \ 2084 FEAT_CTL_SGX_LC_ENABLED | \ 2085 FEAT_CTL_SGX_ENABLED | \ 2086 FEAT_CTL_LMCE_ENABLED) 2087 2088 static inline bool is_vmx_feature_control_msr_valid(struct vcpu_vmx *vmx, 2089 struct msr_data *msr) 2090 { 2091 uint64_t valid_bits; 2092 2093 /* 2094 * Ensure KVM_SUPPORTED_FEATURE_CONTROL is updated when new bits are 2095 * exposed to the guest. 2096 */ 2097 WARN_ON_ONCE(vmx->msr_ia32_feature_control_valid_bits & 2098 ~KVM_SUPPORTED_FEATURE_CONTROL); 2099 2100 if (!msr->host_initiated && 2101 (vmx->msr_ia32_feature_control & FEAT_CTL_LOCKED)) 2102 return false; 2103 2104 if (msr->host_initiated) 2105 valid_bits = KVM_SUPPORTED_FEATURE_CONTROL; 2106 else 2107 valid_bits = vmx->msr_ia32_feature_control_valid_bits; 2108 2109 return !(msr->data & ~valid_bits); 2110 } 2111 2112 int vmx_get_feature_msr(u32 msr, u64 *data) 2113 { 2114 switch (msr) { 2115 case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR: 2116 if (!nested) 2117 return 1; 2118 return vmx_get_vmx_msr(&vmcs_config.nested, msr, data); 2119 default: 2120 return KVM_MSR_RET_UNSUPPORTED; 2121 } 2122 } 2123 2124 /* 2125 * Reads an msr value (of 'msr_info->index') into 'msr_info->data'. 2126 * Returns 0 on success, non-0 otherwise. 2127 * Assumes vcpu_load() was already called. 2128 */ 2129 int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 2130 { 2131 struct vcpu_vmx *vmx = to_vmx(vcpu); 2132 struct vmx_uret_msr *msr; 2133 u32 index; 2134 2135 switch (msr_info->index) { 2136 #ifdef CONFIG_X86_64 2137 case MSR_FS_BASE: 2138 msr_info->data = vmcs_readl(GUEST_FS_BASE); 2139 break; 2140 case MSR_GS_BASE: 2141 msr_info->data = vmcs_readl(GUEST_GS_BASE); 2142 break; 2143 case MSR_KERNEL_GS_BASE: 2144 msr_info->data = vmx_read_guest_kernel_gs_base(vmx); 2145 break; 2146 #endif 2147 case MSR_EFER: 2148 return kvm_get_msr_common(vcpu, msr_info); 2149 case MSR_IA32_TSX_CTRL: 2150 if (!msr_info->host_initiated && 2151 !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR)) 2152 return 1; 2153 goto find_uret_msr; 2154 case MSR_IA32_UMWAIT_CONTROL: 2155 if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx)) 2156 return 1; 2157 2158 msr_info->data = vmx->msr_ia32_umwait_control; 2159 break; 2160 case MSR_IA32_SPEC_CTRL: 2161 if (!msr_info->host_initiated && 2162 !guest_has_spec_ctrl_msr(vcpu)) 2163 return 1; 2164 2165 msr_info->data = vmx->spec_ctrl; 2166 break; 2167 case MSR_IA32_SYSENTER_CS: 2168 msr_info->data = vmcs_read32(GUEST_SYSENTER_CS); 2169 break; 2170 case MSR_IA32_SYSENTER_EIP: 2171 msr_info->data = vmcs_readl(GUEST_SYSENTER_EIP); 2172 break; 2173 case MSR_IA32_SYSENTER_ESP: 2174 msr_info->data = vmcs_readl(GUEST_SYSENTER_ESP); 2175 break; 2176 case MSR_IA32_BNDCFGS: 2177 if (!kvm_mpx_supported() || 2178 (!msr_info->host_initiated && 2179 !guest_cpu_cap_has(vcpu, X86_FEATURE_MPX))) 2180 return 1; 2181 msr_info->data = vmcs_read64(GUEST_BNDCFGS); 2182 break; 2183 case MSR_IA32_MCG_EXT_CTL: 2184 if (!msr_info->host_initiated && 2185 !(vmx->msr_ia32_feature_control & 2186 FEAT_CTL_LMCE_ENABLED)) 2187 return 1; 2188 msr_info->data = vcpu->arch.mcg_ext_ctl; 2189 break; 2190 case MSR_IA32_FEAT_CTL: 2191 msr_info->data = vmx->msr_ia32_feature_control; 2192 break; 2193 case MSR_IA32_SGXLEPUBKEYHASH0 ... MSR_IA32_SGXLEPUBKEYHASH3: 2194 if (!msr_info->host_initiated && 2195 !guest_cpu_cap_has(vcpu, X86_FEATURE_SGX_LC)) 2196 return 1; 2197 msr_info->data = vmx->msr_ia32_sgxlepubkeyhash 2198 [msr_info->index - MSR_IA32_SGXLEPUBKEYHASH0]; 2199 break; 2200 case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR: 2201 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_VMX)) 2202 return 1; 2203 if (vmx_get_vmx_msr(&vmx->nested.msrs, msr_info->index, 2204 &msr_info->data)) 2205 return 1; 2206 #ifdef CONFIG_KVM_HYPERV 2207 /* 2208 * Enlightened VMCS v1 doesn't have certain VMCS fields but 2209 * instead of just ignoring the features, different Hyper-V 2210 * versions are either trying to use them and fail or do some 2211 * sanity checking and refuse to boot. Filter all unsupported 2212 * features out. 2213 */ 2214 if (!msr_info->host_initiated && guest_cpu_cap_has_evmcs(vcpu)) 2215 nested_evmcs_filter_control_msr(vcpu, msr_info->index, 2216 &msr_info->data); 2217 #endif 2218 break; 2219 case MSR_IA32_RTIT_CTL: 2220 if (!vmx_pt_mode_is_host_guest()) 2221 return 1; 2222 msr_info->data = vmx->pt_desc.guest.ctl; 2223 break; 2224 case MSR_IA32_RTIT_STATUS: 2225 if (!vmx_pt_mode_is_host_guest()) 2226 return 1; 2227 msr_info->data = vmx->pt_desc.guest.status; 2228 break; 2229 case MSR_IA32_RTIT_CR3_MATCH: 2230 if (!vmx_pt_mode_is_host_guest() || 2231 !intel_pt_validate_cap(vmx->pt_desc.caps, 2232 PT_CAP_cr3_filtering)) 2233 return 1; 2234 msr_info->data = vmx->pt_desc.guest.cr3_match; 2235 break; 2236 case MSR_IA32_RTIT_OUTPUT_BASE: 2237 if (!vmx_pt_mode_is_host_guest() || 2238 (!intel_pt_validate_cap(vmx->pt_desc.caps, 2239 PT_CAP_topa_output) && 2240 !intel_pt_validate_cap(vmx->pt_desc.caps, 2241 PT_CAP_single_range_output))) 2242 return 1; 2243 msr_info->data = vmx->pt_desc.guest.output_base; 2244 break; 2245 case MSR_IA32_RTIT_OUTPUT_MASK: 2246 if (!vmx_pt_mode_is_host_guest() || 2247 (!intel_pt_validate_cap(vmx->pt_desc.caps, 2248 PT_CAP_topa_output) && 2249 !intel_pt_validate_cap(vmx->pt_desc.caps, 2250 PT_CAP_single_range_output))) 2251 return 1; 2252 msr_info->data = vmx->pt_desc.guest.output_mask; 2253 break; 2254 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: 2255 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A; 2256 if (!vmx_pt_mode_is_host_guest() || 2257 (index >= 2 * vmx->pt_desc.num_address_ranges)) 2258 return 1; 2259 if (index % 2) 2260 msr_info->data = vmx->pt_desc.guest.addr_b[index / 2]; 2261 else 2262 msr_info->data = vmx->pt_desc.guest.addr_a[index / 2]; 2263 break; 2264 case MSR_IA32_S_CET: 2265 msr_info->data = vmcs_readl(GUEST_S_CET); 2266 break; 2267 case MSR_KVM_INTERNAL_GUEST_SSP: 2268 msr_info->data = vmcs_readl(GUEST_SSP); 2269 break; 2270 case MSR_IA32_INT_SSP_TAB: 2271 msr_info->data = vmcs_readl(GUEST_INTR_SSP_TABLE); 2272 break; 2273 case MSR_IA32_DEBUGCTLMSR: 2274 msr_info->data = vmx_guest_debugctl_read(); 2275 break; 2276 default: 2277 find_uret_msr: 2278 msr = vmx_find_uret_msr(vmx, msr_info->index); 2279 if (msr) { 2280 msr_info->data = msr->data; 2281 break; 2282 } 2283 return kvm_get_msr_common(vcpu, msr_info); 2284 } 2285 2286 return 0; 2287 } 2288 2289 static u64 nested_vmx_truncate_sysenter_addr(struct kvm_vcpu *vcpu, 2290 u64 data) 2291 { 2292 #ifdef CONFIG_X86_64 2293 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_LM)) 2294 return (u32)data; 2295 #endif 2296 return (unsigned long)data; 2297 } 2298 2299 u64 vmx_get_supported_debugctl(struct kvm_vcpu *vcpu, bool host_initiated) 2300 { 2301 u64 debugctl = 0; 2302 2303 if (boot_cpu_has(X86_FEATURE_BUS_LOCK_DETECT) && 2304 (host_initiated || guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT))) 2305 debugctl |= DEBUGCTLMSR_BUS_LOCK_DETECT; 2306 2307 if ((kvm_caps.supported_perf_cap & PERF_CAP_LBR_FMT) && 2308 (host_initiated || intel_pmu_lbr_is_enabled(vcpu))) 2309 debugctl |= DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI; 2310 2311 if (boot_cpu_has(X86_FEATURE_RTM) && 2312 (host_initiated || guest_cpu_cap_has(vcpu, X86_FEATURE_RTM))) 2313 debugctl |= DEBUGCTLMSR_RTM_DEBUG; 2314 2315 return debugctl; 2316 } 2317 2318 bool vmx_is_valid_debugctl(struct kvm_vcpu *vcpu, u64 data, bool host_initiated) 2319 { 2320 u64 invalid; 2321 2322 invalid = data & ~vmx_get_supported_debugctl(vcpu, host_initiated); 2323 if (invalid & (DEBUGCTLMSR_BTF | DEBUGCTLMSR_LBR)) { 2324 kvm_pr_unimpl_wrmsr(vcpu, MSR_IA32_DEBUGCTLMSR, data); 2325 invalid &= ~(DEBUGCTLMSR_BTF | DEBUGCTLMSR_LBR); 2326 } 2327 return !invalid; 2328 } 2329 2330 /* 2331 * Writes msr value into the appropriate "register". 2332 * Returns 0 on success, non-0 otherwise. 2333 * Assumes vcpu_load() was already called. 2334 */ 2335 int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 2336 { 2337 struct vcpu_vmx *vmx = to_vmx(vcpu); 2338 struct vmx_uret_msr *msr; 2339 int ret = 0; 2340 u32 msr_index = msr_info->index; 2341 u64 data = msr_info->data; 2342 u32 index; 2343 2344 switch (msr_index) { 2345 case MSR_EFER: 2346 ret = kvm_set_msr_common(vcpu, msr_info); 2347 break; 2348 #ifdef CONFIG_X86_64 2349 case MSR_FS_BASE: 2350 vmx_segment_cache_clear(vmx); 2351 vmcs_writel(GUEST_FS_BASE, data); 2352 break; 2353 case MSR_GS_BASE: 2354 vmx_segment_cache_clear(vmx); 2355 vmcs_writel(GUEST_GS_BASE, data); 2356 break; 2357 case MSR_KERNEL_GS_BASE: 2358 vmx_write_guest_kernel_gs_base(vmx, data); 2359 break; 2360 case MSR_IA32_XFD: 2361 ret = kvm_set_msr_common(vcpu, msr_info); 2362 /* 2363 * Always intercepting WRMSR could incur non-negligible 2364 * overhead given xfd might be changed frequently in 2365 * guest context switch. Disable write interception 2366 * upon the first write with a non-zero value (indicating 2367 * potential usage on dynamic xfeatures). Also update 2368 * exception bitmap to trap #NM for proper virtualization 2369 * of guest xfd_err. 2370 */ 2371 if (!ret && data) { 2372 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_XFD, 2373 MSR_TYPE_RW); 2374 vcpu->arch.xfd_no_write_intercept = true; 2375 vmx_update_exception_bitmap(vcpu); 2376 } 2377 break; 2378 #endif 2379 case MSR_IA32_SYSENTER_CS: 2380 if (is_guest_mode(vcpu)) 2381 get_vmcs12(vcpu)->guest_sysenter_cs = data; 2382 vmcs_write32(GUEST_SYSENTER_CS, data); 2383 break; 2384 case MSR_IA32_SYSENTER_EIP: 2385 if (is_guest_mode(vcpu)) { 2386 data = nested_vmx_truncate_sysenter_addr(vcpu, data); 2387 get_vmcs12(vcpu)->guest_sysenter_eip = data; 2388 } 2389 vmcs_writel(GUEST_SYSENTER_EIP, data); 2390 break; 2391 case MSR_IA32_SYSENTER_ESP: 2392 if (is_guest_mode(vcpu)) { 2393 data = nested_vmx_truncate_sysenter_addr(vcpu, data); 2394 get_vmcs12(vcpu)->guest_sysenter_esp = data; 2395 } 2396 vmcs_writel(GUEST_SYSENTER_ESP, data); 2397 break; 2398 case MSR_IA32_DEBUGCTLMSR: 2399 if (!vmx_is_valid_debugctl(vcpu, data, msr_info->host_initiated)) 2400 return 1; 2401 2402 data &= vmx_get_supported_debugctl(vcpu, msr_info->host_initiated); 2403 2404 if (is_guest_mode(vcpu) && get_vmcs12(vcpu)->vm_exit_controls & 2405 VM_EXIT_SAVE_DEBUG_CONTROLS) 2406 get_vmcs12(vcpu)->guest_ia32_debugctl = data; 2407 2408 vmx_guest_debugctl_write(vcpu, data); 2409 2410 if (intel_pmu_lbr_is_enabled(vcpu) && !vmx->lbr_desc.event && 2411 (data & DEBUGCTLMSR_LBR)) 2412 intel_pmu_create_guest_lbr_event(vcpu); 2413 return 0; 2414 case MSR_IA32_BNDCFGS: 2415 if (!kvm_mpx_supported() || 2416 (!msr_info->host_initiated && 2417 !guest_cpu_cap_has(vcpu, X86_FEATURE_MPX))) 2418 return 1; 2419 if (is_noncanonical_msr_address(data & PAGE_MASK, vcpu) || 2420 (data & MSR_IA32_BNDCFGS_RSVD)) 2421 return 1; 2422 2423 if (is_guest_mode(vcpu) && 2424 ((vmx->nested.msrs.entry_ctls_high & VM_ENTRY_LOAD_BNDCFGS) || 2425 (vmx->nested.msrs.exit_ctls_high & VM_EXIT_CLEAR_BNDCFGS))) 2426 get_vmcs12(vcpu)->guest_bndcfgs = data; 2427 2428 vmcs_write64(GUEST_BNDCFGS, data); 2429 break; 2430 case MSR_IA32_UMWAIT_CONTROL: 2431 if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx)) 2432 return 1; 2433 2434 /* The reserved bit 1 and non-32 bit [63:32] should be zero */ 2435 if (data & (BIT_ULL(1) | GENMASK_ULL(63, 32))) 2436 return 1; 2437 2438 vmx->msr_ia32_umwait_control = data; 2439 break; 2440 case MSR_IA32_SPEC_CTRL: 2441 if (!msr_info->host_initiated && 2442 !guest_has_spec_ctrl_msr(vcpu)) 2443 return 1; 2444 2445 if (kvm_spec_ctrl_test_value(data)) 2446 return 1; 2447 2448 vmx->spec_ctrl = data; 2449 if (!data) 2450 break; 2451 2452 /* 2453 * For non-nested: 2454 * When it's written (to non-zero) for the first time, pass 2455 * it through. 2456 * 2457 * For nested: 2458 * The handling of the MSR bitmap for L2 guests is done in 2459 * nested_vmx_prepare_msr_bitmap. We should not touch the 2460 * vmcs02.msr_bitmap here since it gets completely overwritten 2461 * in the merging. We update the vmcs01 here for L1 as well 2462 * since it will end up touching the MSR anyway now. 2463 */ 2464 vmx_disable_intercept_for_msr(vcpu, 2465 MSR_IA32_SPEC_CTRL, 2466 MSR_TYPE_RW); 2467 break; 2468 case MSR_IA32_TSX_CTRL: 2469 if (!msr_info->host_initiated && 2470 !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR)) 2471 return 1; 2472 if (data & ~(TSX_CTRL_RTM_DISABLE | TSX_CTRL_CPUID_CLEAR)) 2473 return 1; 2474 goto find_uret_msr; 2475 case MSR_IA32_CR_PAT: 2476 ret = kvm_set_msr_common(vcpu, msr_info); 2477 if (ret) 2478 break; 2479 2480 if (is_guest_mode(vcpu) && 2481 get_vmcs12(vcpu)->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT) 2482 get_vmcs12(vcpu)->guest_ia32_pat = data; 2483 2484 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) 2485 vmcs_write64(GUEST_IA32_PAT, data); 2486 break; 2487 case MSR_IA32_MCG_EXT_CTL: 2488 if ((!msr_info->host_initiated && 2489 !(vmx->msr_ia32_feature_control & 2490 FEAT_CTL_LMCE_ENABLED)) || 2491 (data & ~MCG_EXT_CTL_LMCE_EN)) 2492 return 1; 2493 vcpu->arch.mcg_ext_ctl = data; 2494 break; 2495 case MSR_IA32_FEAT_CTL: 2496 if (!is_vmx_feature_control_msr_valid(vmx, msr_info)) 2497 return 1; 2498 2499 vmx->msr_ia32_feature_control = data; 2500 if (msr_info->host_initiated && data == 0) 2501 vmx_leave_nested(vcpu); 2502 2503 /* SGX may be enabled/disabled by guest's firmware */ 2504 vmx_write_encls_bitmap(vcpu, NULL); 2505 break; 2506 case MSR_IA32_SGXLEPUBKEYHASH0 ... MSR_IA32_SGXLEPUBKEYHASH3: 2507 /* 2508 * On real hardware, the LE hash MSRs are writable before 2509 * the firmware sets bit 0 in MSR 0x7a ("activating" SGX), 2510 * at which point SGX related bits in IA32_FEATURE_CONTROL 2511 * become writable. 2512 * 2513 * KVM does not emulate SGX activation for simplicity, so 2514 * allow writes to the LE hash MSRs if IA32_FEATURE_CONTROL 2515 * is unlocked. This is technically not architectural 2516 * behavior, but it's close enough. 2517 */ 2518 if (!msr_info->host_initiated && 2519 (!guest_cpu_cap_has(vcpu, X86_FEATURE_SGX_LC) || 2520 ((vmx->msr_ia32_feature_control & FEAT_CTL_LOCKED) && 2521 !(vmx->msr_ia32_feature_control & FEAT_CTL_SGX_LC_ENABLED)))) 2522 return 1; 2523 vmx->msr_ia32_sgxlepubkeyhash 2524 [msr_index - MSR_IA32_SGXLEPUBKEYHASH0] = data; 2525 break; 2526 case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR: 2527 if (!msr_info->host_initiated) 2528 return 1; /* they are read-only */ 2529 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_VMX)) 2530 return 1; 2531 return vmx_set_vmx_msr(vcpu, msr_index, data); 2532 case MSR_IA32_RTIT_CTL: 2533 if (!vmx_pt_mode_is_host_guest() || 2534 vmx_rtit_ctl_check(vcpu, data) || 2535 vmx->nested.vmxon) 2536 return 1; 2537 vmcs_write64(GUEST_IA32_RTIT_CTL, data); 2538 vmx->pt_desc.guest.ctl = data; 2539 pt_update_intercept_for_msr(vcpu); 2540 break; 2541 case MSR_IA32_RTIT_STATUS: 2542 if (!pt_can_write_msr(vmx)) 2543 return 1; 2544 if (data & MSR_IA32_RTIT_STATUS_MASK) 2545 return 1; 2546 vmx->pt_desc.guest.status = data; 2547 break; 2548 case MSR_IA32_RTIT_CR3_MATCH: 2549 if (!pt_can_write_msr(vmx)) 2550 return 1; 2551 if (!intel_pt_validate_cap(vmx->pt_desc.caps, 2552 PT_CAP_cr3_filtering)) 2553 return 1; 2554 vmx->pt_desc.guest.cr3_match = data; 2555 break; 2556 case MSR_IA32_RTIT_OUTPUT_BASE: 2557 if (!pt_can_write_msr(vmx)) 2558 return 1; 2559 if (!intel_pt_validate_cap(vmx->pt_desc.caps, 2560 PT_CAP_topa_output) && 2561 !intel_pt_validate_cap(vmx->pt_desc.caps, 2562 PT_CAP_single_range_output)) 2563 return 1; 2564 if (!pt_output_base_valid(vcpu, data)) 2565 return 1; 2566 vmx->pt_desc.guest.output_base = data; 2567 break; 2568 case MSR_IA32_RTIT_OUTPUT_MASK: 2569 if (!pt_can_write_msr(vmx)) 2570 return 1; 2571 if (!intel_pt_validate_cap(vmx->pt_desc.caps, 2572 PT_CAP_topa_output) && 2573 !intel_pt_validate_cap(vmx->pt_desc.caps, 2574 PT_CAP_single_range_output)) 2575 return 1; 2576 vmx->pt_desc.guest.output_mask = data; 2577 break; 2578 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: 2579 if (!pt_can_write_msr(vmx)) 2580 return 1; 2581 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A; 2582 if (index >= 2 * vmx->pt_desc.num_address_ranges) 2583 return 1; 2584 if (is_noncanonical_msr_address(data, vcpu)) 2585 return 1; 2586 if (index % 2) 2587 vmx->pt_desc.guest.addr_b[index / 2] = data; 2588 else 2589 vmx->pt_desc.guest.addr_a[index / 2] = data; 2590 break; 2591 case MSR_IA32_S_CET: 2592 vmcs_writel(GUEST_S_CET, data); 2593 break; 2594 case MSR_KVM_INTERNAL_GUEST_SSP: 2595 vmcs_writel(GUEST_SSP, data); 2596 break; 2597 case MSR_IA32_INT_SSP_TAB: 2598 vmcs_writel(GUEST_INTR_SSP_TABLE, data); 2599 break; 2600 case MSR_IA32_PERF_CAPABILITIES: 2601 if (data & PERF_CAP_LBR_FMT) { 2602 if ((data & PERF_CAP_LBR_FMT) != 2603 (kvm_caps.supported_perf_cap & PERF_CAP_LBR_FMT)) 2604 return 1; 2605 if (!cpuid_model_is_consistent(vcpu)) 2606 return 1; 2607 } 2608 if (data & PERF_CAP_PEBS_FORMAT) { 2609 if ((data & PERF_CAP_PEBS_MASK) != 2610 (kvm_caps.supported_perf_cap & PERF_CAP_PEBS_MASK)) 2611 return 1; 2612 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_DS)) 2613 return 1; 2614 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_DTES64)) 2615 return 1; 2616 if (!cpuid_model_is_consistent(vcpu)) 2617 return 1; 2618 } 2619 ret = kvm_set_msr_common(vcpu, msr_info); 2620 break; 2621 2622 default: 2623 find_uret_msr: 2624 msr = vmx_find_uret_msr(vmx, msr_index); 2625 if (msr) 2626 ret = vmx_set_guest_uret_msr(vmx, msr, data); 2627 else 2628 ret = kvm_set_msr_common(vcpu, msr_info); 2629 } 2630 2631 /* FB_CLEAR may have changed, also update the FB_CLEAR_DIS behavior */ 2632 if (msr_index == MSR_IA32_ARCH_CAPABILITIES) 2633 vmx_update_fb_clear_dis(vcpu, vmx); 2634 2635 return ret; 2636 } 2637 2638 void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) 2639 { 2640 unsigned long guest_owned_bits; 2641 2642 kvm_register_mark_available(vcpu, reg); 2643 2644 switch (reg) { 2645 case VCPU_REGS_RSP: 2646 vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP); 2647 break; 2648 case VCPU_REG_RIP: 2649 vcpu->arch.rip = vmcs_readl(GUEST_RIP); 2650 break; 2651 case VCPU_REG_PDPTR: 2652 if (enable_ept) 2653 ept_save_pdptrs(vcpu); 2654 break; 2655 case VCPU_REG_CR0: 2656 guest_owned_bits = vcpu->arch.cr0_guest_owned_bits; 2657 2658 vcpu->arch.cr0 &= ~guest_owned_bits; 2659 vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & guest_owned_bits; 2660 break; 2661 case VCPU_REG_CR3: 2662 /* 2663 * When intercepting CR3 loads, e.g. for shadowing paging, KVM's 2664 * CR3 is loaded into hardware, not the guest's CR3. 2665 */ 2666 if (!(exec_controls_get(to_vmx(vcpu)) & CPU_BASED_CR3_LOAD_EXITING)) 2667 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); 2668 break; 2669 case VCPU_REG_CR4: 2670 guest_owned_bits = vcpu->arch.cr4_guest_owned_bits; 2671 2672 vcpu->arch.cr4 &= ~guest_owned_bits; 2673 vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & guest_owned_bits; 2674 break; 2675 default: 2676 KVM_BUG_ON(1, vcpu->kvm); 2677 break; 2678 } 2679 } 2680 2681 /* 2682 * There is no X86_FEATURE for SGX yet, but anyway we need to query CPUID 2683 * directly instead of going through cpu_has(), to ensure KVM is trapping 2684 * ENCLS whenever it's supported in hardware. It does not matter whether 2685 * the host OS supports or has enabled SGX. 2686 */ 2687 static bool cpu_has_sgx(void) 2688 { 2689 return cpuid_eax(0) >= 0x12 && (cpuid_eax(0x12) & BIT(0)); 2690 } 2691 2692 static int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt, u32 msr, u32 *result) 2693 { 2694 struct msr vmx_msr; 2695 u32 ctl = ctl_min | ctl_opt; 2696 2697 rdmsrq(msr, vmx_msr.q); 2698 2699 ctl &= vmx_msr.h; /* bit == 0 in high word ==> must be zero */ 2700 ctl |= vmx_msr.l; /* bit == 1 in low word ==> must be one */ 2701 2702 /* Ensure minimum (required) set of control bits are supported. */ 2703 if (ctl_min & ~ctl) 2704 return -EIO; 2705 2706 *result = ctl; 2707 return 0; 2708 } 2709 2710 static u64 adjust_vmx_controls64(u64 ctl_opt, u32 msr) 2711 { 2712 u64 allowed; 2713 2714 rdmsrq(msr, allowed); 2715 2716 return ctl_opt & allowed; 2717 } 2718 2719 #define vmx_check_entry_exit_pairs(pairs, entry_controls, exit_controls) \ 2720 ({ \ 2721 int i, r = 0; \ 2722 \ 2723 BUILD_BUG_ON(sizeof(pairs[0].entry_control) != sizeof(entry_controls)); \ 2724 BUILD_BUG_ON(sizeof(pairs[0].exit_control) != sizeof(exit_controls)); \ 2725 \ 2726 for (i = 0; i < ARRAY_SIZE(pairs); i++) { \ 2727 typeof(entry_controls) n_ctrl = pairs[i].entry_control; \ 2728 typeof(exit_controls) x_ctrl = pairs[i].exit_control; \ 2729 \ 2730 if (!(entry_controls & n_ctrl) == !(exit_controls & x_ctrl)) \ 2731 continue; \ 2732 \ 2733 pr_warn_once("Inconsistent VM-Entry/VM-Exit pair, " \ 2734 "entry = %llx (%llx), exit = %llx (%llx)\n", \ 2735 (u64)(entry_controls & n_ctrl), (u64)n_ctrl, \ 2736 (u64)(exit_controls & x_ctrl), (u64)x_ctrl); \ 2737 \ 2738 if (error_on_inconsistent_vmcs_config) \ 2739 r = -EIO; \ 2740 \ 2741 entry_controls &= ~n_ctrl; \ 2742 exit_controls &= ~x_ctrl; \ 2743 } \ 2744 r; \ 2745 }) 2746 2747 static int setup_vmcs_config(struct vmcs_config *vmcs_conf, 2748 struct vmx_capability *vmx_cap) 2749 { 2750 u32 _pin_based_exec_control = 0; 2751 u32 _cpu_based_exec_control = 0; 2752 u32 _cpu_based_2nd_exec_control = 0; 2753 u64 _cpu_based_3rd_exec_control = 0; 2754 u32 _vmexit_control = 0; 2755 u32 _vmentry_control = 0; 2756 struct msr val; 2757 u64 basic_msr; 2758 u64 misc_msr; 2759 2760 /* 2761 * LOAD/SAVE_DEBUG_CONTROLS are absent because both are mandatory. 2762 * SAVE_IA32_PAT and SAVE_IA32_EFER are absent because KVM always 2763 * intercepts writes to PAT and EFER, i.e. never enables those controls. 2764 */ 2765 struct { 2766 u32 entry_control; 2767 u32 exit_control; 2768 } const vmcs_entry_exit_pairs[] = { 2769 { VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL, VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL }, 2770 { VM_ENTRY_LOAD_IA32_PAT, VM_EXIT_LOAD_IA32_PAT }, 2771 { VM_ENTRY_LOAD_IA32_EFER, VM_EXIT_LOAD_IA32_EFER }, 2772 { VM_ENTRY_LOAD_BNDCFGS, VM_EXIT_CLEAR_BNDCFGS }, 2773 { VM_ENTRY_LOAD_IA32_RTIT_CTL, VM_EXIT_CLEAR_IA32_RTIT_CTL }, 2774 { VM_ENTRY_LOAD_CET_STATE, VM_EXIT_LOAD_CET_STATE }, 2775 }; 2776 2777 memset(vmcs_conf, 0, sizeof(*vmcs_conf)); 2778 2779 if (adjust_vmx_controls(KVM_REQUIRED_VMX_CPU_BASED_VM_EXEC_CONTROL, 2780 KVM_OPTIONAL_VMX_CPU_BASED_VM_EXEC_CONTROL, 2781 MSR_IA32_VMX_PROCBASED_CTLS, 2782 &_cpu_based_exec_control)) 2783 return -EIO; 2784 if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) { 2785 if (adjust_vmx_controls(KVM_REQUIRED_VMX_SECONDARY_VM_EXEC_CONTROL, 2786 KVM_OPTIONAL_VMX_SECONDARY_VM_EXEC_CONTROL, 2787 MSR_IA32_VMX_PROCBASED_CTLS2, 2788 &_cpu_based_2nd_exec_control)) 2789 return -EIO; 2790 } 2791 if (!IS_ENABLED(CONFIG_KVM_INTEL_PROVE_VE)) 2792 _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_EPT_VIOLATION_VE; 2793 2794 #ifndef CONFIG_X86_64 2795 if (!(_cpu_based_2nd_exec_control & 2796 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) 2797 _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW; 2798 #endif 2799 2800 if (!(_cpu_based_exec_control & CPU_BASED_TPR_SHADOW)) 2801 _cpu_based_2nd_exec_control &= ~( 2802 SECONDARY_EXEC_APIC_REGISTER_VIRT | 2803 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 2804 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 2805 2806 rdmsrq_safe(MSR_IA32_VMX_EPT_VPID_CAP, &val.q); 2807 vmx_cap->ept = val.l; 2808 vmx_cap->vpid = val.h; 2809 2810 if (!(_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) && 2811 vmx_cap->ept) { 2812 pr_warn_once("EPT CAP should not exist if not support " 2813 "1-setting enable EPT VM-execution control\n"); 2814 2815 if (error_on_inconsistent_vmcs_config) 2816 return -EIO; 2817 2818 vmx_cap->ept = 0; 2819 _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_MODE_BASED_EPT_EXEC; 2820 _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_EPT_VIOLATION_VE; 2821 } 2822 if (!(_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_VPID) && 2823 vmx_cap->vpid) { 2824 pr_warn_once("VPID CAP should not exist if not support " 2825 "1-setting enable VPID VM-execution control\n"); 2826 2827 if (error_on_inconsistent_vmcs_config) 2828 return -EIO; 2829 2830 vmx_cap->vpid = 0; 2831 } 2832 2833 /* 2834 * Virtualizing MBEC requires advanced vmexit information in order to 2835 * distinguish supervisor and user accesses. For simplicity and clarity 2836 * disable MBEC entirely if advanced vmexit information is not available, 2837 * this way mbec=1 in the kvm_intel module parameters implies availability 2838 * to nested guests as well. 2839 */ 2840 if (!(vmx_cap->ept & VMX_EPT_ADVANCED_VMEXIT_INFO_BIT)) 2841 _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_MODE_BASED_EPT_EXEC; 2842 2843 if (!cpu_has_sgx()) 2844 _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_ENCLS_EXITING; 2845 2846 /* 2847 * KVM doesn't re-derive the TSC scaling ratio when the host TSC 2848 * frequency changes, so TSC scaling is only usable with a constant 2849 * TSC. Clear the control here rather than in vmx_hardware_setup() so 2850 * that the per-CPU configs recomputed by vmx_check_processor_compat() 2851 * stay consistent with the golden vmcs_config. 2852 */ 2853 if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) 2854 _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_TSC_SCALING; 2855 2856 if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_TERTIARY_CONTROLS) 2857 _cpu_based_3rd_exec_control = 2858 adjust_vmx_controls64(KVM_OPTIONAL_VMX_TERTIARY_VM_EXEC_CONTROL, 2859 MSR_IA32_VMX_PROCBASED_CTLS3); 2860 2861 if (adjust_vmx_controls(KVM_REQUIRED_VMX_VM_EXIT_CONTROLS, 2862 KVM_OPTIONAL_VMX_VM_EXIT_CONTROLS, 2863 MSR_IA32_VMX_EXIT_CTLS, 2864 &_vmexit_control)) 2865 return -EIO; 2866 2867 if (adjust_vmx_controls(KVM_REQUIRED_VMX_PIN_BASED_VM_EXEC_CONTROL, 2868 KVM_OPTIONAL_VMX_PIN_BASED_VM_EXEC_CONTROL, 2869 MSR_IA32_VMX_PINBASED_CTLS, 2870 &_pin_based_exec_control)) 2871 return -EIO; 2872 2873 if (cpu_has_broken_vmx_preemption_timer()) 2874 _pin_based_exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER; 2875 if (!(_cpu_based_2nd_exec_control & 2876 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)) 2877 _pin_based_exec_control &= ~PIN_BASED_POSTED_INTR; 2878 2879 if (adjust_vmx_controls(KVM_REQUIRED_VMX_VM_ENTRY_CONTROLS, 2880 KVM_OPTIONAL_VMX_VM_ENTRY_CONTROLS, 2881 MSR_IA32_VMX_ENTRY_CTLS, 2882 &_vmentry_control)) 2883 return -EIO; 2884 2885 if (vmx_check_entry_exit_pairs(vmcs_entry_exit_pairs, 2886 _vmentry_control, _vmexit_control)) 2887 return -EIO; 2888 2889 /* 2890 * Some cpus support VM_{ENTRY,EXIT}_IA32_PERF_GLOBAL_CTRL but they 2891 * can't be used due to an errata where VM Exit may incorrectly clear 2892 * IA32_PERF_GLOBAL_CTRL[34:32]. Workaround the errata by using the 2893 * MSR load mechanism to switch IA32_PERF_GLOBAL_CTRL. 2894 */ 2895 switch (boot_cpu_data.x86_vfm) { 2896 case INTEL_NEHALEM_EP: /* AAK155 */ 2897 case INTEL_NEHALEM: /* AAP115 */ 2898 case INTEL_WESTMERE: /* AAT100 */ 2899 case INTEL_WESTMERE_EP: /* BC86,AAY89,BD102 */ 2900 case INTEL_NEHALEM_EX: /* BA97 */ 2901 _vmentry_control &= ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL; 2902 _vmexit_control &= ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; 2903 pr_warn_once("VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL " 2904 "does not work properly. Using workaround\n"); 2905 break; 2906 default: 2907 break; 2908 } 2909 2910 rdmsrq(MSR_IA32_VMX_BASIC, basic_msr); 2911 2912 /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */ 2913 if (vmx_basic_vmcs_size(basic_msr) > PAGE_SIZE) 2914 return -EIO; 2915 2916 #ifdef CONFIG_X86_64 2917 /* 2918 * KVM expects to be able to shove all legal physical addresses into 2919 * VMCS fields for 64-bit kernels, and per the SDM, "This bit is always 2920 * 0 for processors that support Intel 64 architecture". 2921 */ 2922 if (basic_msr & VMX_BASIC_32BIT_PHYS_ADDR_ONLY) 2923 return -EIO; 2924 #endif 2925 2926 /* Require Write-Back (WB) memory type for VMCS accesses. */ 2927 if (vmx_basic_vmcs_mem_type(basic_msr) != X86_MEMTYPE_WB) 2928 return -EIO; 2929 2930 rdmsrq(MSR_IA32_VMX_MISC, misc_msr); 2931 2932 vmcs_conf->basic = basic_msr; 2933 vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control; 2934 vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control; 2935 vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control; 2936 vmcs_conf->cpu_based_3rd_exec_ctrl = _cpu_based_3rd_exec_control; 2937 vmcs_conf->vmexit_ctrl = _vmexit_control; 2938 vmcs_conf->vmentry_ctrl = _vmentry_control; 2939 vmcs_conf->misc = misc_msr; 2940 2941 #if IS_ENABLED(CONFIG_HYPERV) 2942 if (enlightened_vmcs) 2943 evmcs_sanitize_exec_ctrls(vmcs_conf); 2944 #endif 2945 2946 return 0; 2947 } 2948 2949 static bool __kvm_is_vmx_supported(void) 2950 { 2951 int cpu = smp_processor_id(); 2952 2953 if (!(cpuid_ecx(1) & feature_bit(VMX))) { 2954 pr_err("VMX not supported by CPU %d\n", cpu); 2955 return false; 2956 } 2957 2958 if (!this_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL)) { 2959 pr_err("VMX not enabled (by BIOS) in MSR_IA32_FEAT_CTL on CPU %d\n", cpu); 2960 return false; 2961 } 2962 2963 if (!this_cpu_has(X86_FEATURE_VMX)) { 2964 pr_err("VMX not fully enabled on CPU %d. Check kernel logs and/or BIOS\n", cpu); 2965 return false; 2966 } 2967 2968 return true; 2969 } 2970 2971 static bool kvm_is_vmx_supported(void) 2972 { 2973 bool supported; 2974 2975 migrate_disable(); 2976 supported = __kvm_is_vmx_supported(); 2977 migrate_enable(); 2978 2979 return supported; 2980 } 2981 2982 int vmx_check_processor_compat(void) 2983 { 2984 int cpu = raw_smp_processor_id(); 2985 struct vmcs_config vmcs_conf; 2986 struct vmx_capability vmx_cap; 2987 2988 if (!__kvm_is_vmx_supported()) 2989 return -EIO; 2990 2991 if (setup_vmcs_config(&vmcs_conf, &vmx_cap) < 0) { 2992 pr_err("Failed to setup VMCS config on CPU %d\n", cpu); 2993 return -EIO; 2994 } 2995 if (nested) 2996 nested_vmx_setup_ctls_msrs(&vmcs_conf, vmx_cap.ept); 2997 2998 if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config))) { 2999 u32 *gold = (void *)&vmcs_config; 3000 u32 *mine = (void *)&vmcs_conf; 3001 int i; 3002 3003 BUILD_BUG_ON(sizeof(struct vmcs_config) % sizeof(u32)); 3004 3005 pr_err("VMCS config on CPU %d doesn't match reference config:", cpu); 3006 for (i = 0; i < sizeof(struct vmcs_config) / sizeof(u32); i++) { 3007 if (gold[i] == mine[i]) 3008 continue; 3009 3010 pr_cont("\n Offset %u REF = 0x%08x, CPU%u = 0x%08x, mismatch = 0x%08x", 3011 i * (int)sizeof(u32), gold[i], cpu, mine[i], gold[i] ^ mine[i]); 3012 } 3013 pr_cont("\n"); 3014 return -EIO; 3015 } 3016 return 0; 3017 } 3018 3019 int vmx_enable_virtualization_cpu(void) 3020 { 3021 int cpu = raw_smp_processor_id(); 3022 3023 /* 3024 * This can happen if we hot-added a CPU but failed to allocate 3025 * VP assist page for it. 3026 */ 3027 if (kvm_is_using_evmcs() && !hv_get_vp_assist_page(cpu)) 3028 return -EFAULT; 3029 3030 return x86_virt_get_ref(X86_FEATURE_VMX); 3031 } 3032 3033 static void vmclear_local_loaded_vmcss(void) 3034 { 3035 int cpu = raw_smp_processor_id(); 3036 struct loaded_vmcs *v, *n; 3037 3038 list_for_each_entry_safe(v, n, &per_cpu(loaded_vmcss_on_cpu, cpu), 3039 loaded_vmcss_on_cpu_link) 3040 __loaded_vmcs_clear(v); 3041 } 3042 3043 void vmx_disable_virtualization_cpu(void) 3044 { 3045 vmclear_local_loaded_vmcss(); 3046 3047 x86_virt_put_ref(X86_FEATURE_VMX); 3048 3049 hv_reset_evmcs(); 3050 } 3051 3052 struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags) 3053 { 3054 int node = cpu_to_node(cpu); 3055 struct page *pages; 3056 struct vmcs *vmcs; 3057 3058 pages = alloc_pages_node(node, flags, 0); 3059 if (!pages) 3060 return NULL; 3061 vmcs = page_address(pages); 3062 memset(vmcs, 0, vmx_basic_vmcs_size(vmcs_config.basic)); 3063 3064 /* KVM supports Enlightened VMCS v1 only */ 3065 if (kvm_is_using_evmcs()) 3066 vmcs->hdr.revision_id = KVM_EVMCS_VERSION; 3067 else 3068 vmcs->hdr.revision_id = vmx_basic_vmcs_revision_id(vmcs_config.basic); 3069 3070 if (shadow) 3071 vmcs->hdr.shadow_vmcs = 1; 3072 return vmcs; 3073 } 3074 3075 void free_vmcs(struct vmcs *vmcs) 3076 { 3077 free_page((unsigned long)vmcs); 3078 } 3079 3080 /* 3081 * Free a VMCS, but before that VMCLEAR it on the CPU where it was last loaded 3082 */ 3083 void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs) 3084 { 3085 if (!loaded_vmcs->vmcs) 3086 return; 3087 loaded_vmcs_clear(loaded_vmcs); 3088 free_vmcs(loaded_vmcs->vmcs); 3089 loaded_vmcs->vmcs = NULL; 3090 if (loaded_vmcs->msr_bitmap) 3091 free_page((unsigned long)loaded_vmcs->msr_bitmap); 3092 WARN_ON(loaded_vmcs->shadow_vmcs != NULL); 3093 } 3094 3095 int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs) 3096 { 3097 loaded_vmcs->vmcs = alloc_vmcs(false); 3098 if (!loaded_vmcs->vmcs) 3099 return -ENOMEM; 3100 3101 vmcs_clear(loaded_vmcs->vmcs); 3102 3103 loaded_vmcs->shadow_vmcs = NULL; 3104 loaded_vmcs->hv_timer_soft_disabled = false; 3105 loaded_vmcs->cpu = -1; 3106 loaded_vmcs->launched = 0; 3107 3108 if (cpu_has_vmx_msr_bitmap()) { 3109 loaded_vmcs->msr_bitmap = (unsigned long *) 3110 __get_free_page(GFP_KERNEL_ACCOUNT); 3111 if (!loaded_vmcs->msr_bitmap) 3112 goto out_vmcs; 3113 memset(loaded_vmcs->msr_bitmap, 0xff, PAGE_SIZE); 3114 } 3115 3116 memset(&loaded_vmcs->host_state, 0, sizeof(struct vmcs_host_state)); 3117 memset(&loaded_vmcs->controls_shadow, 0, 3118 sizeof(struct vmcs_controls_shadow)); 3119 3120 return 0; 3121 3122 out_vmcs: 3123 free_loaded_vmcs(loaded_vmcs); 3124 return -ENOMEM; 3125 } 3126 3127 static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg, 3128 struct kvm_segment *save) 3129 { 3130 if (!emulate_invalid_guest_state) { 3131 /* 3132 * CS and SS RPL should be equal during guest entry according 3133 * to VMX spec, but in reality it is not always so. Since vcpu 3134 * is in the middle of the transition from real mode to 3135 * protected mode it is safe to assume that RPL 0 is a good 3136 * default value. 3137 */ 3138 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS) 3139 save->selector &= ~SEGMENT_RPL_MASK; 3140 save->dpl = save->selector & SEGMENT_RPL_MASK; 3141 save->s = 1; 3142 } 3143 __vmx_set_segment(vcpu, save, seg); 3144 } 3145 3146 static void enter_pmode(struct kvm_vcpu *vcpu) 3147 { 3148 unsigned long flags; 3149 struct vcpu_vmx *vmx = to_vmx(vcpu); 3150 3151 /* 3152 * Update real mode segment cache. It may be not up-to-date if segment 3153 * register was written while vcpu was in a guest mode. 3154 */ 3155 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES); 3156 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS); 3157 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS); 3158 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS); 3159 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS); 3160 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS); 3161 3162 vmx->rmode.vm86_active = 0; 3163 3164 __vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR); 3165 3166 flags = vmcs_readl(GUEST_RFLAGS); 3167 flags &= RMODE_GUEST_OWNED_EFLAGS_BITS; 3168 flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS; 3169 vmcs_writel(GUEST_RFLAGS, flags); 3170 3171 vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) | 3172 (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME)); 3173 3174 vmx_update_exception_bitmap(vcpu); 3175 3176 fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]); 3177 fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]); 3178 fix_pmode_seg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]); 3179 fix_pmode_seg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]); 3180 fix_pmode_seg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]); 3181 fix_pmode_seg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]); 3182 } 3183 3184 static void fix_rmode_seg(int seg, struct kvm_segment *save) 3185 { 3186 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 3187 struct kvm_segment var = *save; 3188 3189 var.dpl = 0x3; 3190 if (seg == VCPU_SREG_CS) 3191 var.type = 0x3; 3192 3193 if (!emulate_invalid_guest_state) { 3194 var.selector = var.base >> 4; 3195 var.base = var.base & 0xffff0; 3196 var.limit = 0xffff; 3197 var.g = 0; 3198 var.db = 0; 3199 var.present = 1; 3200 var.s = 1; 3201 var.l = 0; 3202 var.unusable = 0; 3203 var.type = 0x3; 3204 var.avl = 0; 3205 if (save->base & 0xf) 3206 pr_warn_once("segment base is not paragraph aligned " 3207 "when entering protected mode (seg=%d)", seg); 3208 } 3209 3210 vmcs_write16(sf->selector, var.selector); 3211 vmcs_writel(sf->base, var.base); 3212 vmcs_write32(sf->limit, var.limit); 3213 vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(&var)); 3214 } 3215 3216 static void enter_rmode(struct kvm_vcpu *vcpu) 3217 { 3218 unsigned long flags; 3219 struct vcpu_vmx *vmx = to_vmx(vcpu); 3220 struct kvm_vmx *kvm_vmx = to_kvm_vmx(vcpu->kvm); 3221 3222 /* 3223 * KVM should never use VM86 to virtualize Real Mode when L2 is active, 3224 * as using VM86 is unnecessary if unrestricted guest is enabled, and 3225 * if unrestricted guest is disabled, VM-Enter (from L1) with CR0.PG=0 3226 * should VM-Fail and KVM should reject userspace attempts to stuff 3227 * CR0.PG=0 when L2 is active. 3228 */ 3229 WARN_ON_ONCE(is_guest_mode(vcpu)); 3230 3231 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR); 3232 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES); 3233 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS); 3234 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS); 3235 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS); 3236 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS); 3237 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS); 3238 3239 vmx->rmode.vm86_active = 1; 3240 3241 vmx_segment_cache_clear(vmx); 3242 3243 vmcs_writel(GUEST_TR_BASE, kvm_vmx->tss_addr); 3244 vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1); 3245 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); 3246 3247 flags = vmcs_readl(GUEST_RFLAGS); 3248 vmx->rmode.save_rflags = flags; 3249 3250 flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM; 3251 3252 vmcs_writel(GUEST_RFLAGS, flags); 3253 vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME); 3254 vmx_update_exception_bitmap(vcpu); 3255 3256 fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]); 3257 fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]); 3258 fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]); 3259 fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]); 3260 fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]); 3261 fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]); 3262 } 3263 3264 int vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer) 3265 { 3266 struct vcpu_vmx *vmx = to_vmx(vcpu); 3267 3268 /* Nothing to do if hardware doesn't support EFER. */ 3269 if (!vmx_find_uret_msr(vmx, MSR_EFER)) 3270 return 0; 3271 3272 vcpu->arch.efer = efer; 3273 #ifdef CONFIG_X86_64 3274 if (efer & EFER_LMA) 3275 vm_entry_controls_setbit(vmx, VM_ENTRY_IA32E_MODE); 3276 else 3277 vm_entry_controls_clearbit(vmx, VM_ENTRY_IA32E_MODE); 3278 #else 3279 if (KVM_BUG_ON(efer & EFER_LMA, vcpu->kvm)) 3280 return 1; 3281 #endif 3282 3283 vmx_setup_uret_msrs(vmx); 3284 return 0; 3285 } 3286 3287 #ifdef CONFIG_X86_64 3288 3289 static void enter_lmode(struct kvm_vcpu *vcpu) 3290 { 3291 u32 guest_tr_ar; 3292 3293 vmx_segment_cache_clear(to_vmx(vcpu)); 3294 3295 guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES); 3296 if ((guest_tr_ar & VMX_AR_TYPE_MASK) != VMX_AR_TYPE_BUSY_64_TSS) { 3297 pr_debug_ratelimited("%s: tss fixup for long mode. \n", 3298 __func__); 3299 vmcs_write32(GUEST_TR_AR_BYTES, 3300 (guest_tr_ar & ~VMX_AR_TYPE_MASK) 3301 | VMX_AR_TYPE_BUSY_64_TSS); 3302 } 3303 vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA); 3304 } 3305 3306 static void exit_lmode(struct kvm_vcpu *vcpu) 3307 { 3308 vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA); 3309 } 3310 3311 #endif 3312 3313 void vmx_flush_tlb_all(struct kvm_vcpu *vcpu) 3314 { 3315 struct vcpu_vmx *vmx = to_vmx(vcpu); 3316 3317 /* 3318 * INVEPT must be issued when EPT is enabled, irrespective of VPID, as 3319 * the CPU is not required to invalidate guest-physical mappings on 3320 * VM-Entry, even if VPID is disabled. Guest-physical mappings are 3321 * associated with the root EPT structure and not any particular VPID 3322 * (INVVPID also isn't required to invalidate guest-physical mappings). 3323 */ 3324 if (enable_ept) { 3325 ept_sync_global(); 3326 } else if (enable_vpid) { 3327 if (cpu_has_vmx_invvpid_global()) { 3328 vpid_sync_vcpu_global(); 3329 } else { 3330 vpid_sync_vcpu_single(vmx->vpid); 3331 vpid_sync_vcpu_single(vmx->nested.vpid02); 3332 } 3333 } 3334 } 3335 3336 static inline int vmx_get_current_vpid(struct kvm_vcpu *vcpu) 3337 { 3338 if (is_guest_mode(vcpu) && nested_cpu_has_vpid(get_vmcs12(vcpu))) 3339 return nested_get_vpid02(vcpu); 3340 return to_vmx(vcpu)->vpid; 3341 } 3342 3343 static u64 construct_eptp(hpa_t root_hpa) 3344 { 3345 u64 eptp = root_hpa | VMX_EPTP_MT_WB; 3346 struct kvm_mmu_page *root; 3347 3348 if (kvm_mmu_is_dummy_root(root_hpa)) 3349 return eptp | VMX_EPTP_PWL_4; 3350 3351 /* 3352 * EPT roots should always have an associated MMU page. Return a "bad" 3353 * EPTP to induce VM-Fail instead of continuing on in a unknown state. 3354 */ 3355 root = root_to_sp(root_hpa); 3356 if (WARN_ON_ONCE(!root)) 3357 return INVALID_PAGE; 3358 3359 eptp |= (root->role.level == 5) ? VMX_EPTP_PWL_5 : VMX_EPTP_PWL_4; 3360 3361 if (enable_ept_ad_bits && !root->role.ad_disabled) 3362 eptp |= VMX_EPTP_AD_ENABLE_BIT; 3363 3364 return eptp; 3365 } 3366 3367 static void vmx_flush_tlb_ept_root(hpa_t root_hpa) 3368 { 3369 u64 eptp = construct_eptp(root_hpa); 3370 3371 if (VALID_PAGE(eptp)) 3372 ept_sync_context(eptp); 3373 else 3374 ept_sync_global(); 3375 } 3376 3377 void vmx_flush_tlb_current(struct kvm_vcpu *vcpu) 3378 { 3379 struct kvm_mmu *mmu = vcpu->arch.mmu; 3380 u64 root_hpa = mmu->root.hpa; 3381 3382 /* No flush required if the current context is invalid. */ 3383 if (!VALID_PAGE(root_hpa)) 3384 return; 3385 3386 if (enable_ept) 3387 vmx_flush_tlb_ept_root(root_hpa); 3388 else 3389 vpid_sync_context(vmx_get_current_vpid(vcpu)); 3390 } 3391 3392 void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) 3393 { 3394 /* 3395 * vpid_sync_vcpu_addr() is a nop if vpid==0, see the comment in 3396 * vmx_flush_tlb_guest() for an explanation of why this is ok. 3397 */ 3398 vpid_sync_vcpu_addr(vmx_get_current_vpid(vcpu), addr); 3399 } 3400 3401 void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu) 3402 { 3403 /* 3404 * vpid_sync_context() is a nop if vpid==0, e.g. if enable_vpid==0 or a 3405 * vpid couldn't be allocated for this vCPU. VM-Enter and VM-Exit are 3406 * required to flush GVA->{G,H}PA mappings from the TLB if vpid is 3407 * disabled (VM-Enter with vpid enabled and vpid==0 is disallowed), 3408 * i.e. no explicit INVVPID is necessary. 3409 */ 3410 vpid_sync_context(vmx_get_current_vpid(vcpu)); 3411 } 3412 3413 void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu) 3414 { 3415 if (!kvm_register_is_dirty(vcpu, VCPU_REG_PDPTR)) 3416 return; 3417 3418 if (is_pae_paging(vcpu)) { 3419 vmcs_write64(GUEST_PDPTR0, vcpu->arch.pdptrs[0]); 3420 vmcs_write64(GUEST_PDPTR1, vcpu->arch.pdptrs[1]); 3421 vmcs_write64(GUEST_PDPTR2, vcpu->arch.pdptrs[2]); 3422 vmcs_write64(GUEST_PDPTR3, vcpu->arch.pdptrs[3]); 3423 } 3424 } 3425 3426 void ept_save_pdptrs(struct kvm_vcpu *vcpu) 3427 { 3428 if (WARN_ON_ONCE(!is_pae_paging(vcpu))) 3429 return; 3430 3431 vcpu->arch.pdptrs[0] = vmcs_read64(GUEST_PDPTR0); 3432 vcpu->arch.pdptrs[1] = vmcs_read64(GUEST_PDPTR1); 3433 vcpu->arch.pdptrs[2] = vmcs_read64(GUEST_PDPTR2); 3434 vcpu->arch.pdptrs[3] = vmcs_read64(GUEST_PDPTR3); 3435 3436 kvm_register_mark_available(vcpu, VCPU_REG_PDPTR); 3437 } 3438 3439 #define CR3_EXITING_BITS (CPU_BASED_CR3_LOAD_EXITING | \ 3440 CPU_BASED_CR3_STORE_EXITING) 3441 3442 bool vmx_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 3443 { 3444 if (is_guest_mode(vcpu)) 3445 return nested_guest_cr0_valid(vcpu, cr0); 3446 3447 if (to_vmx(vcpu)->nested.vmxon) 3448 return nested_host_cr0_valid(vcpu, cr0); 3449 3450 return true; 3451 } 3452 3453 void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 3454 { 3455 struct vcpu_vmx *vmx = to_vmx(vcpu); 3456 unsigned long hw_cr0, old_cr0_pg; 3457 u32 tmp; 3458 3459 old_cr0_pg = kvm_read_cr0_bits(vcpu, X86_CR0_PG); 3460 3461 hw_cr0 = (cr0 & ~KVM_VM_CR0_ALWAYS_OFF); 3462 if (enable_unrestricted_guest) 3463 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST; 3464 else { 3465 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON; 3466 if (!enable_ept) 3467 hw_cr0 |= X86_CR0_WP; 3468 3469 if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE)) 3470 enter_pmode(vcpu); 3471 3472 if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE)) 3473 enter_rmode(vcpu); 3474 } 3475 3476 vmcs_writel(CR0_READ_SHADOW, cr0); 3477 vmcs_writel(GUEST_CR0, hw_cr0); 3478 vcpu->arch.cr0 = cr0; 3479 kvm_register_mark_available(vcpu, VCPU_REG_CR0); 3480 3481 #ifdef CONFIG_X86_64 3482 if (vcpu->arch.efer & EFER_LME) { 3483 if (!old_cr0_pg && (cr0 & X86_CR0_PG)) 3484 enter_lmode(vcpu); 3485 else if (old_cr0_pg && !(cr0 & X86_CR0_PG)) 3486 exit_lmode(vcpu); 3487 } 3488 #endif 3489 3490 if (enable_ept && !enable_unrestricted_guest) { 3491 /* 3492 * Ensure KVM has an up-to-date snapshot of the guest's CR3. If 3493 * the below code _enables_ CR3 exiting, vmx_cache_reg() will 3494 * (correctly) stop reading vmcs.GUEST_CR3 because it thinks 3495 * KVM's CR3 is installed. 3496 */ 3497 if (!kvm_register_is_available(vcpu, VCPU_REG_CR3)) 3498 vmx_cache_reg(vcpu, VCPU_REG_CR3); 3499 3500 /* 3501 * When running with EPT but not unrestricted guest, KVM must 3502 * intercept CR3 accesses when paging is _disabled_. This is 3503 * necessary because restricted guests can't actually run with 3504 * paging disabled, and so KVM stuffs its own CR3 in order to 3505 * run the guest when identity mapped page tables. 3506 * 3507 * Do _NOT_ check the old CR0.PG, e.g. to optimize away the 3508 * update, it may be stale with respect to CR3 interception, 3509 * e.g. after nested VM-Enter. 3510 * 3511 * Lastly, honor L1's desires, i.e. intercept CR3 loads and/or 3512 * stores to forward them to L1, even if KVM does not need to 3513 * intercept them to preserve its identity mapped page tables. 3514 */ 3515 if (!(cr0 & X86_CR0_PG)) { 3516 exec_controls_setbit(vmx, CR3_EXITING_BITS); 3517 } else if (!is_guest_mode(vcpu)) { 3518 exec_controls_clearbit(vmx, CR3_EXITING_BITS); 3519 } else { 3520 tmp = exec_controls_get(vmx); 3521 tmp &= ~CR3_EXITING_BITS; 3522 tmp |= get_vmcs12(vcpu)->cpu_based_vm_exec_control & CR3_EXITING_BITS; 3523 exec_controls_set(vmx, tmp); 3524 } 3525 3526 /* Note, vmx_set_cr4() consumes the new vcpu->arch.cr0. */ 3527 if ((old_cr0_pg ^ cr0) & X86_CR0_PG) 3528 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu)); 3529 3530 /* 3531 * When !CR0_PG -> CR0_PG, vcpu->arch.cr3 becomes active, but 3532 * GUEST_CR3 is still vmx->ept_identity_map_addr if EPT + !URG. 3533 */ 3534 if (!(old_cr0_pg & X86_CR0_PG) && (cr0 & X86_CR0_PG)) 3535 kvm_register_mark_dirty(vcpu, VCPU_REG_CR3); 3536 } 3537 3538 /* depends on vcpu->arch.cr0 to be set to a new value */ 3539 vmx->vt.emulation_required = vmx_emulation_required(vcpu); 3540 } 3541 3542 static int vmx_get_max_ept_level(void) 3543 { 3544 if (cpu_has_vmx_ept_5levels()) 3545 return 5; 3546 return 4; 3547 } 3548 3549 void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level) 3550 { 3551 struct kvm *kvm = vcpu->kvm; 3552 bool update_guest_cr3 = true; 3553 unsigned long guest_cr3; 3554 3555 if (enable_ept) { 3556 KVM_MMU_WARN_ON(root_to_sp(root_hpa) && 3557 root_level != root_to_sp(root_hpa)->role.level); 3558 vmcs_write64(EPT_POINTER, construct_eptp(root_hpa)); 3559 3560 hv_track_root_tdp(vcpu, root_hpa); 3561 3562 if (!enable_unrestricted_guest && !is_paging(vcpu)) 3563 guest_cr3 = to_kvm_vmx(kvm)->ept_identity_map_addr; 3564 else if (kvm_register_is_dirty(vcpu, VCPU_REG_CR3)) 3565 guest_cr3 = vcpu->arch.cr3; 3566 else /* vmcs.GUEST_CR3 is already up-to-date. */ 3567 update_guest_cr3 = false; 3568 vmx_ept_load_pdptrs(vcpu); 3569 } else { 3570 guest_cr3 = root_hpa | kvm_get_active_pcid(vcpu) | 3571 kvm_get_active_cr3_lam_bits(vcpu); 3572 } 3573 3574 if (update_guest_cr3) 3575 vmcs_writel(GUEST_CR3, guest_cr3); 3576 } 3577 3578 bool vmx_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 3579 { 3580 /* 3581 * We operate under the default treatment of SMM, so VMX cannot be 3582 * enabled under SMM. Note, whether or not VMXE is allowed at all, 3583 * i.e. is a reserved bit, is handled by common x86 code. 3584 */ 3585 if ((cr4 & X86_CR4_VMXE) && is_smm(vcpu)) 3586 return false; 3587 3588 if (to_vmx(vcpu)->nested.vmxon && !nested_cr4_valid(vcpu, cr4)) 3589 return false; 3590 3591 return true; 3592 } 3593 3594 void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 3595 { 3596 unsigned long old_cr4 = kvm_read_cr4(vcpu); 3597 struct vcpu_vmx *vmx = to_vmx(vcpu); 3598 unsigned long hw_cr4; 3599 3600 /* 3601 * Pass through host's Machine Check Enable value to hw_cr4, which 3602 * is in force while we are in guest mode. Do not let guests control 3603 * this bit, even if host CR4.MCE == 0. 3604 */ 3605 hw_cr4 = (cr4_read_shadow() & X86_CR4_MCE) | (cr4 & ~X86_CR4_MCE); 3606 if (enable_unrestricted_guest) 3607 hw_cr4 |= KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST; 3608 else if (vmx->rmode.vm86_active) 3609 hw_cr4 |= KVM_RMODE_VM_CR4_ALWAYS_ON; 3610 else 3611 hw_cr4 |= KVM_PMODE_VM_CR4_ALWAYS_ON; 3612 3613 if (vmx_umip_emulated()) { 3614 if (cr4 & X86_CR4_UMIP) { 3615 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_DESC); 3616 hw_cr4 &= ~X86_CR4_UMIP; 3617 } else if (!is_guest_mode(vcpu) || 3618 !nested_cpu_has2(get_vmcs12(vcpu), SECONDARY_EXEC_DESC)) { 3619 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_DESC); 3620 } 3621 } 3622 3623 vcpu->arch.cr4 = cr4; 3624 kvm_register_mark_available(vcpu, VCPU_REG_CR4); 3625 3626 if (!enable_unrestricted_guest) { 3627 if (enable_ept) { 3628 if (!is_paging(vcpu)) { 3629 hw_cr4 &= ~X86_CR4_PAE; 3630 hw_cr4 |= X86_CR4_PSE; 3631 } else if (!(cr4 & X86_CR4_PAE)) { 3632 hw_cr4 &= ~X86_CR4_PAE; 3633 } 3634 } 3635 3636 /* 3637 * SMEP/SMAP/PKU is disabled if CPU is in non-paging mode in 3638 * hardware. To emulate this behavior, SMEP/SMAP/PKU needs 3639 * to be manually disabled when guest switches to non-paging 3640 * mode. 3641 * 3642 * If !enable_unrestricted_guest, the CPU is always running 3643 * with CR0.PG=1 and CR4 needs to be modified. 3644 * If enable_unrestricted_guest, the CPU automatically 3645 * disables SMEP/SMAP/PKU when the guest sets CR0.PG=0. 3646 */ 3647 if (!is_paging(vcpu)) 3648 hw_cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE); 3649 } 3650 3651 vmcs_writel(CR4_READ_SHADOW, cr4); 3652 vmcs_writel(GUEST_CR4, hw_cr4); 3653 3654 if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE)) 3655 vcpu->arch.cpuid_dynamic_bits_dirty = true; 3656 } 3657 3658 void vmx_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) 3659 { 3660 struct vcpu_vmx *vmx = to_vmx(vcpu); 3661 u32 ar; 3662 3663 if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) { 3664 *var = vmx->rmode.segs[seg]; 3665 if (seg == VCPU_SREG_TR 3666 || var->selector == vmx_read_guest_seg_selector(vmx, seg)) 3667 return; 3668 var->base = vmx_read_guest_seg_base(vmx, seg); 3669 var->selector = vmx_read_guest_seg_selector(vmx, seg); 3670 return; 3671 } 3672 var->base = vmx_read_guest_seg_base(vmx, seg); 3673 var->limit = vmx_read_guest_seg_limit(vmx, seg); 3674 var->selector = vmx_read_guest_seg_selector(vmx, seg); 3675 ar = vmx_read_guest_seg_ar(vmx, seg); 3676 var->unusable = (ar >> 16) & 1; 3677 var->type = ar & 15; 3678 var->s = (ar >> 4) & 1; 3679 var->dpl = (ar >> 5) & 3; 3680 /* 3681 * Some userspaces do not preserve unusable property. Since usable 3682 * segment has to be present according to VMX spec we can use present 3683 * property to amend userspace bug by making unusable segment always 3684 * nonpresent. vmx_segment_access_rights() already marks nonpresent 3685 * segment as unusable. 3686 */ 3687 var->present = !var->unusable; 3688 var->avl = (ar >> 12) & 1; 3689 var->l = (ar >> 13) & 1; 3690 var->db = (ar >> 14) & 1; 3691 var->g = (ar >> 15) & 1; 3692 } 3693 3694 u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg) 3695 { 3696 struct vcpu_vmx *vmx = to_vmx(vcpu); 3697 struct kvm_segment s; 3698 3699 if (vmx->rmode.vm86_active) { 3700 vmx_get_segment(vcpu, &s, seg); 3701 return s.base; 3702 } 3703 return vmx_read_guest_seg_base(vmx, seg); 3704 } 3705 3706 static int __vmx_get_cpl(struct kvm_vcpu *vcpu, bool no_cache) 3707 { 3708 struct vcpu_vmx *vmx = to_vmx(vcpu); 3709 int ar; 3710 3711 if (unlikely(vmx->rmode.vm86_active)) 3712 return 0; 3713 3714 if (no_cache) 3715 ar = vmcs_read32(GUEST_SS_AR_BYTES); 3716 else 3717 ar = vmx_read_guest_seg_ar(vmx, VCPU_SREG_SS); 3718 return VMX_AR_DPL(ar); 3719 } 3720 3721 int vmx_get_cpl(struct kvm_vcpu *vcpu) 3722 { 3723 return __vmx_get_cpl(vcpu, false); 3724 } 3725 3726 int vmx_get_cpl_no_cache(struct kvm_vcpu *vcpu) 3727 { 3728 return __vmx_get_cpl(vcpu, true); 3729 } 3730 3731 static u32 vmx_segment_access_rights(struct kvm_segment *var) 3732 { 3733 u32 ar; 3734 3735 ar = var->type & 15; 3736 ar |= (var->s & 1) << 4; 3737 ar |= (var->dpl & 3) << 5; 3738 ar |= (var->present & 1) << 7; 3739 ar |= (var->avl & 1) << 12; 3740 ar |= (var->l & 1) << 13; 3741 ar |= (var->db & 1) << 14; 3742 ar |= (var->g & 1) << 15; 3743 ar |= (var->unusable || !var->present) << 16; 3744 3745 return ar; 3746 } 3747 3748 void __vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) 3749 { 3750 struct vcpu_vmx *vmx = to_vmx(vcpu); 3751 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 3752 3753 vmx_segment_cache_clear(vmx); 3754 3755 if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) { 3756 vmx->rmode.segs[seg] = *var; 3757 if (seg == VCPU_SREG_TR) 3758 vmcs_write16(sf->selector, var->selector); 3759 else if (var->s) 3760 fix_rmode_seg(seg, &vmx->rmode.segs[seg]); 3761 return; 3762 } 3763 3764 vmcs_writel(sf->base, var->base); 3765 vmcs_write32(sf->limit, var->limit); 3766 vmcs_write16(sf->selector, var->selector); 3767 3768 /* 3769 * Fix the "Accessed" bit in AR field of segment registers for older 3770 * qemu binaries. 3771 * IA32 arch specifies that at the time of processor reset the 3772 * "Accessed" bit in the AR field of segment registers is 1. And qemu 3773 * is setting it to 0 in the userland code. This causes invalid guest 3774 * state vmexit when "unrestricted guest" mode is turned on. 3775 * Fix for this setup issue in cpu_reset is being pushed in the qemu 3776 * tree. Newer qemu binaries with that qemu fix would not need this 3777 * kvm hack. 3778 */ 3779 if (is_unrestricted_guest(vcpu) && (seg != VCPU_SREG_LDTR)) 3780 var->type |= 0x1; /* Accessed */ 3781 3782 vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(var)); 3783 } 3784 3785 void vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) 3786 { 3787 __vmx_set_segment(vcpu, var, seg); 3788 3789 to_vmx(vcpu)->vt.emulation_required = vmx_emulation_required(vcpu); 3790 } 3791 3792 void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) 3793 { 3794 u32 ar = vmx_read_guest_seg_ar(to_vmx(vcpu), VCPU_SREG_CS); 3795 3796 *db = (ar >> 14) & 1; 3797 *l = (ar >> 13) & 1; 3798 } 3799 3800 void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3801 { 3802 dt->size = vmcs_read32(GUEST_IDTR_LIMIT); 3803 dt->address = vmcs_readl(GUEST_IDTR_BASE); 3804 } 3805 3806 void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3807 { 3808 vmcs_write32(GUEST_IDTR_LIMIT, dt->size); 3809 vmcs_writel(GUEST_IDTR_BASE, dt->address); 3810 } 3811 3812 void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3813 { 3814 dt->size = vmcs_read32(GUEST_GDTR_LIMIT); 3815 dt->address = vmcs_readl(GUEST_GDTR_BASE); 3816 } 3817 3818 void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) 3819 { 3820 vmcs_write32(GUEST_GDTR_LIMIT, dt->size); 3821 vmcs_writel(GUEST_GDTR_BASE, dt->address); 3822 } 3823 3824 static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg) 3825 { 3826 struct kvm_segment var; 3827 u32 ar; 3828 3829 vmx_get_segment(vcpu, &var, seg); 3830 var.dpl = 0x3; 3831 if (seg == VCPU_SREG_CS) 3832 var.type = 0x3; 3833 ar = vmx_segment_access_rights(&var); 3834 3835 if (var.base != (var.selector << 4)) 3836 return false; 3837 if (var.limit != 0xffff) 3838 return false; 3839 if (ar != 0xf3) 3840 return false; 3841 3842 return true; 3843 } 3844 3845 static bool code_segment_valid(struct kvm_vcpu *vcpu) 3846 { 3847 struct kvm_segment cs; 3848 unsigned int cs_rpl; 3849 3850 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS); 3851 cs_rpl = cs.selector & SEGMENT_RPL_MASK; 3852 3853 if (cs.unusable) 3854 return false; 3855 if (~cs.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_ACCESSES_MASK)) 3856 return false; 3857 if (!cs.s) 3858 return false; 3859 if (cs.type & VMX_AR_TYPE_WRITEABLE_MASK) { 3860 if (cs.dpl > cs_rpl) 3861 return false; 3862 } else { 3863 if (cs.dpl != cs_rpl) 3864 return false; 3865 } 3866 if (!cs.present) 3867 return false; 3868 3869 /* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */ 3870 return true; 3871 } 3872 3873 static bool stack_segment_valid(struct kvm_vcpu *vcpu) 3874 { 3875 struct kvm_segment ss; 3876 unsigned int ss_rpl; 3877 3878 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS); 3879 ss_rpl = ss.selector & SEGMENT_RPL_MASK; 3880 3881 if (ss.unusable) 3882 return true; 3883 if (ss.type != 3 && ss.type != 7) 3884 return false; 3885 if (!ss.s) 3886 return false; 3887 if (ss.dpl != ss_rpl) /* DPL != RPL */ 3888 return false; 3889 if (!ss.present) 3890 return false; 3891 3892 return true; 3893 } 3894 3895 static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg) 3896 { 3897 struct kvm_segment var; 3898 unsigned int rpl; 3899 3900 vmx_get_segment(vcpu, &var, seg); 3901 rpl = var.selector & SEGMENT_RPL_MASK; 3902 3903 if (var.unusable) 3904 return true; 3905 if (!var.s) 3906 return false; 3907 if (!var.present) 3908 return false; 3909 if (~var.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_WRITEABLE_MASK)) { 3910 if (var.dpl < rpl) /* DPL < RPL */ 3911 return false; 3912 } 3913 3914 /* TODO: Add other members to kvm_segment_field to allow checking for other access 3915 * rights flags 3916 */ 3917 return true; 3918 } 3919 3920 static bool tr_valid(struct kvm_vcpu *vcpu) 3921 { 3922 struct kvm_segment tr; 3923 3924 vmx_get_segment(vcpu, &tr, VCPU_SREG_TR); 3925 3926 if (tr.unusable) 3927 return false; 3928 if (tr.selector & SEGMENT_TI_MASK) /* TI = 1 */ 3929 return false; 3930 if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */ 3931 return false; 3932 if (!tr.present) 3933 return false; 3934 3935 return true; 3936 } 3937 3938 static bool ldtr_valid(struct kvm_vcpu *vcpu) 3939 { 3940 struct kvm_segment ldtr; 3941 3942 vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR); 3943 3944 if (ldtr.unusable) 3945 return true; 3946 if (ldtr.selector & SEGMENT_TI_MASK) /* TI = 1 */ 3947 return false; 3948 if (ldtr.type != 2) 3949 return false; 3950 if (!ldtr.present) 3951 return false; 3952 3953 return true; 3954 } 3955 3956 static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu) 3957 { 3958 struct kvm_segment cs, ss; 3959 3960 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS); 3961 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS); 3962 3963 return ((cs.selector & SEGMENT_RPL_MASK) == 3964 (ss.selector & SEGMENT_RPL_MASK)); 3965 } 3966 3967 /* 3968 * Check if guest state is valid. Returns true if valid, false if 3969 * not. 3970 * We assume that registers are always usable 3971 */ 3972 bool __vmx_guest_state_valid(struct kvm_vcpu *vcpu) 3973 { 3974 /* real mode guest state checks */ 3975 if (!is_protmode(vcpu) || (vmx_get_rflags(vcpu) & X86_EFLAGS_VM)) { 3976 if (!rmode_segment_valid(vcpu, VCPU_SREG_CS)) 3977 return false; 3978 if (!rmode_segment_valid(vcpu, VCPU_SREG_SS)) 3979 return false; 3980 if (!rmode_segment_valid(vcpu, VCPU_SREG_DS)) 3981 return false; 3982 if (!rmode_segment_valid(vcpu, VCPU_SREG_ES)) 3983 return false; 3984 if (!rmode_segment_valid(vcpu, VCPU_SREG_FS)) 3985 return false; 3986 if (!rmode_segment_valid(vcpu, VCPU_SREG_GS)) 3987 return false; 3988 } else { 3989 /* protected mode guest state checks */ 3990 if (!cs_ss_rpl_check(vcpu)) 3991 return false; 3992 if (!code_segment_valid(vcpu)) 3993 return false; 3994 if (!stack_segment_valid(vcpu)) 3995 return false; 3996 if (!data_segment_valid(vcpu, VCPU_SREG_DS)) 3997 return false; 3998 if (!data_segment_valid(vcpu, VCPU_SREG_ES)) 3999 return false; 4000 if (!data_segment_valid(vcpu, VCPU_SREG_FS)) 4001 return false; 4002 if (!data_segment_valid(vcpu, VCPU_SREG_GS)) 4003 return false; 4004 if (!tr_valid(vcpu)) 4005 return false; 4006 if (!ldtr_valid(vcpu)) 4007 return false; 4008 } 4009 /* TODO: 4010 * - Add checks on RIP 4011 * - Add checks on RFLAGS 4012 */ 4013 4014 return true; 4015 } 4016 4017 static int init_rmode_tss(struct kvm *kvm, void __user *ua) 4018 { 4019 const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0))); 4020 u16 data; 4021 int i; 4022 4023 for (i = 0; i < 3; i++) { 4024 if (__copy_to_user(ua + PAGE_SIZE * i, zero_page, PAGE_SIZE)) 4025 return -EFAULT; 4026 } 4027 4028 data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE; 4029 if (__copy_to_user(ua + TSS_IOPB_BASE_OFFSET, &data, sizeof(u16))) 4030 return -EFAULT; 4031 4032 data = ~0; 4033 if (__copy_to_user(ua + RMODE_TSS_SIZE - 1, &data, sizeof(u8))) 4034 return -EFAULT; 4035 4036 return 0; 4037 } 4038 4039 static int init_rmode_identity_map(struct kvm *kvm) 4040 { 4041 struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); 4042 int i, r = 0; 4043 void __user *uaddr; 4044 u32 tmp; 4045 4046 /* Protect kvm_vmx->ept_identity_pagetable_done. */ 4047 mutex_lock(&kvm->slots_lock); 4048 4049 if (likely(kvm_vmx->ept_identity_pagetable_done)) 4050 goto out; 4051 4052 if (!kvm_vmx->ept_identity_map_addr) 4053 kvm_vmx->ept_identity_map_addr = VMX_EPT_IDENTITY_PAGETABLE_ADDR; 4054 4055 uaddr = __x86_set_memory_region(kvm, 4056 IDENTITY_PAGETABLE_PRIVATE_MEMSLOT, 4057 kvm_vmx->ept_identity_map_addr, 4058 PAGE_SIZE); 4059 if (IS_ERR(uaddr)) { 4060 r = PTR_ERR(uaddr); 4061 goto out; 4062 } 4063 4064 /* Set up identity-mapping pagetable for EPT in real mode */ 4065 for (i = 0; i < (PAGE_SIZE / sizeof(tmp)); i++) { 4066 tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | 4067 _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE); 4068 if (__copy_to_user(uaddr + i * sizeof(tmp), &tmp, sizeof(tmp))) { 4069 r = -EFAULT; 4070 goto out; 4071 } 4072 } 4073 kvm_vmx->ept_identity_pagetable_done = true; 4074 4075 out: 4076 mutex_unlock(&kvm->slots_lock); 4077 return r; 4078 } 4079 4080 static void seg_setup(int seg) 4081 { 4082 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 4083 unsigned int ar; 4084 4085 vmcs_write16(sf->selector, 0); 4086 vmcs_writel(sf->base, 0); 4087 vmcs_write32(sf->limit, 0xffff); 4088 ar = 0x93; 4089 if (seg == VCPU_SREG_CS) 4090 ar |= 0x08; /* code segment */ 4091 4092 vmcs_write32(sf->ar_bytes, ar); 4093 } 4094 4095 int allocate_vpid(void) 4096 { 4097 int vpid; 4098 4099 if (!enable_vpid) 4100 return 0; 4101 spin_lock(&vmx_vpid_lock); 4102 vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS); 4103 if (vpid < VMX_NR_VPIDS) 4104 __set_bit(vpid, vmx_vpid_bitmap); 4105 else 4106 vpid = 0; 4107 spin_unlock(&vmx_vpid_lock); 4108 return vpid; 4109 } 4110 4111 void free_vpid(int vpid) 4112 { 4113 if (!enable_vpid || vpid == 0) 4114 return; 4115 spin_lock(&vmx_vpid_lock); 4116 __clear_bit(vpid, vmx_vpid_bitmap); 4117 spin_unlock(&vmx_vpid_lock); 4118 } 4119 4120 static void vmx_msr_bitmap_l01_changed(struct vcpu_vmx *vmx) 4121 { 4122 /* 4123 * When KVM is a nested hypervisor on top of Hyper-V and uses 4124 * 'Enlightened MSR Bitmap' feature L0 needs to know that MSR 4125 * bitmap has changed. 4126 */ 4127 if (kvm_is_using_evmcs()) { 4128 struct hv_enlightened_vmcs *evmcs = (void *)vmx->vmcs01.vmcs; 4129 4130 if (evmcs->hv_enlightenments_control.msr_bitmap) 4131 evmcs->hv_clean_fields &= 4132 ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP; 4133 } 4134 4135 vmx->nested.force_msr_bitmap_recalc = true; 4136 } 4137 4138 void vmx_set_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type, bool set) 4139 { 4140 struct vcpu_vmx *vmx = to_vmx(vcpu); 4141 unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap; 4142 4143 if (!cpu_has_vmx_msr_bitmap()) 4144 return; 4145 4146 vmx_msr_bitmap_l01_changed(vmx); 4147 4148 if (type & MSR_TYPE_R) { 4149 if (!set && kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_READ)) 4150 vmx_clear_msr_bitmap_read(msr_bitmap, msr); 4151 else 4152 vmx_set_msr_bitmap_read(msr_bitmap, msr); 4153 } 4154 4155 if (type & MSR_TYPE_W) { 4156 if (!set && kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_WRITE)) 4157 vmx_clear_msr_bitmap_write(msr_bitmap, msr); 4158 else 4159 vmx_set_msr_bitmap_write(msr_bitmap, msr); 4160 } 4161 } 4162 4163 static void vmx_update_msr_bitmap_x2apic(struct kvm_vcpu *vcpu) 4164 { 4165 /* 4166 * x2APIC indices for 64-bit accesses into the RDMSR and WRMSR halves 4167 * of the MSR bitmap. KVM emulates APIC registers up through 0x3f0, 4168 * i.e. MSR 0x83f, and so only needs to dynamically manipulate 64 bits. 4169 */ 4170 const int read_idx = APIC_BASE_MSR / BITS_PER_LONG_LONG; 4171 const int write_idx = read_idx + (0x800 / sizeof(u64)); 4172 struct vcpu_vmx *vmx = to_vmx(vcpu); 4173 u64 *msr_bitmap = (u64 *)vmx->vmcs01.msr_bitmap; 4174 u8 mode; 4175 4176 if (!cpu_has_vmx_msr_bitmap() || WARN_ON_ONCE(!lapic_in_kernel(vcpu))) 4177 return; 4178 4179 if (cpu_has_secondary_exec_ctrls() && 4180 (secondary_exec_controls_get(vmx) & 4181 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE)) { 4182 mode = MSR_BITMAP_MODE_X2APIC; 4183 if (enable_apicv && kvm_vcpu_apicv_active(vcpu)) 4184 mode |= MSR_BITMAP_MODE_X2APIC_APICV; 4185 } else { 4186 mode = 0; 4187 } 4188 4189 if (mode == vmx->x2apic_msr_bitmap_mode) 4190 return; 4191 4192 vmx->x2apic_msr_bitmap_mode = mode; 4193 4194 /* 4195 * Reset the bitmap for MSRs 0x800 - 0x83f. Leave AMD's uber-extended 4196 * registers (0x840 and above) intercepted, KVM doesn't support them. 4197 * Intercept all writes by default and poke holes as needed. Pass 4198 * through reads for all valid registers by default in x2APIC+APICv 4199 * mode, only the current timer count needs on-demand emulation by KVM. 4200 */ 4201 if (mode & MSR_BITMAP_MODE_X2APIC_APICV) 4202 msr_bitmap[read_idx] = ~kvm_x2apic_disable_read_intercept_reg_mask(vcpu); 4203 else 4204 msr_bitmap[read_idx] = ~0ull; 4205 msr_bitmap[write_idx] = ~0ull; 4206 4207 /* 4208 * TPR reads and writes can be virtualized even if virtual interrupt 4209 * delivery is not in use. 4210 */ 4211 vmx_set_intercept_for_msr(vcpu, X2APIC_MSR(APIC_TASKPRI), MSR_TYPE_RW, 4212 !(mode & MSR_BITMAP_MODE_X2APIC)); 4213 4214 if (mode & MSR_BITMAP_MODE_X2APIC_APICV) { 4215 vmx_disable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_EOI), MSR_TYPE_W); 4216 vmx_disable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_SELF_IPI), MSR_TYPE_W); 4217 if (enable_ipiv) 4218 vmx_disable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_ICR), MSR_TYPE_RW); 4219 } 4220 } 4221 4222 void pt_update_intercept_for_msr(struct kvm_vcpu *vcpu) 4223 { 4224 struct vcpu_vmx *vmx = to_vmx(vcpu); 4225 bool flag = !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN); 4226 u32 i; 4227 4228 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_STATUS, MSR_TYPE_RW, flag); 4229 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_OUTPUT_BASE, MSR_TYPE_RW, flag); 4230 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_OUTPUT_MASK, MSR_TYPE_RW, flag); 4231 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_CR3_MATCH, MSR_TYPE_RW, flag); 4232 for (i = 0; i < vmx->pt_desc.num_address_ranges; i++) { 4233 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_ADDR0_A + i * 2, MSR_TYPE_RW, flag); 4234 vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_ADDR0_B + i * 2, MSR_TYPE_RW, flag); 4235 } 4236 } 4237 4238 static void vmx_recalc_pmu_msr_intercepts(struct kvm_vcpu *vcpu) 4239 { 4240 u64 vm_exit_controls_bits = VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | 4241 VM_EXIT_SAVE_IA32_PERF_GLOBAL_CTRL; 4242 bool has_mediated_pmu = kvm_vcpu_has_mediated_pmu(vcpu); 4243 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 4244 struct vcpu_vmx *vmx = to_vmx(vcpu); 4245 bool intercept = !has_mediated_pmu; 4246 int i; 4247 4248 if (!enable_mediated_pmu) 4249 return; 4250 4251 if (!cpu_has_save_perf_global_ctrl()) { 4252 vm_exit_controls_bits &= ~VM_EXIT_SAVE_IA32_PERF_GLOBAL_CTRL; 4253 4254 if (has_mediated_pmu) 4255 vmx_add_autostore_msr(vmx, MSR_CORE_PERF_GLOBAL_CTRL); 4256 else 4257 vmx_remove_autostore_msr(vmx, MSR_CORE_PERF_GLOBAL_CTRL); 4258 } 4259 4260 vm_entry_controls_changebit(vmx, VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL, 4261 has_mediated_pmu); 4262 4263 vm_exit_controls_changebit(vmx, vm_exit_controls_bits, has_mediated_pmu); 4264 4265 for (i = 0; i < pmu->nr_arch_gp_counters; i++) { 4266 vmx_set_intercept_for_msr(vcpu, MSR_IA32_PERFCTR0 + i, 4267 MSR_TYPE_RW, intercept); 4268 vmx_set_intercept_for_msr(vcpu, MSR_IA32_PMC0 + i, MSR_TYPE_RW, 4269 intercept || !fw_writes_is_enabled(vcpu)); 4270 } 4271 for ( ; i < kvm_pmu_cap.num_counters_gp; i++) { 4272 vmx_set_intercept_for_msr(vcpu, MSR_IA32_PERFCTR0 + i, 4273 MSR_TYPE_RW, true); 4274 vmx_set_intercept_for_msr(vcpu, MSR_IA32_PMC0 + i, 4275 MSR_TYPE_RW, true); 4276 } 4277 4278 for (i = 0; i < pmu->nr_arch_fixed_counters; i++) 4279 vmx_set_intercept_for_msr(vcpu, MSR_CORE_PERF_FIXED_CTR0 + i, 4280 MSR_TYPE_RW, intercept); 4281 for ( ; i < kvm_pmu_cap.num_counters_fixed; i++) 4282 vmx_set_intercept_for_msr(vcpu, MSR_CORE_PERF_FIXED_CTR0 + i, 4283 MSR_TYPE_RW, true); 4284 4285 intercept = kvm_need_perf_global_ctrl_intercept(vcpu); 4286 vmx_set_intercept_for_msr(vcpu, MSR_CORE_PERF_GLOBAL_STATUS, 4287 MSR_TYPE_RW, intercept); 4288 vmx_set_intercept_for_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL, 4289 MSR_TYPE_RW, intercept); 4290 vmx_set_intercept_for_msr(vcpu, MSR_CORE_PERF_GLOBAL_OVF_CTRL, 4291 MSR_TYPE_RW, intercept); 4292 } 4293 4294 static void vmx_recalc_msr_intercepts(struct kvm_vcpu *vcpu) 4295 { 4296 bool intercept; 4297 4298 if (!cpu_has_vmx_msr_bitmap()) 4299 return; 4300 4301 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_TSC, MSR_TYPE_R); 4302 #ifdef CONFIG_X86_64 4303 vmx_disable_intercept_for_msr(vcpu, MSR_FS_BASE, MSR_TYPE_RW); 4304 vmx_disable_intercept_for_msr(vcpu, MSR_GS_BASE, MSR_TYPE_RW); 4305 vmx_disable_intercept_for_msr(vcpu, MSR_KERNEL_GS_BASE, MSR_TYPE_RW); 4306 #endif 4307 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_CS, MSR_TYPE_RW); 4308 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_ESP, MSR_TYPE_RW); 4309 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_EIP, MSR_TYPE_RW); 4310 if (kvm_cstate_in_guest(vcpu->kvm)) { 4311 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C1_RES, MSR_TYPE_R); 4312 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C3_RESIDENCY, MSR_TYPE_R); 4313 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C6_RESIDENCY, MSR_TYPE_R); 4314 vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C7_RESIDENCY, MSR_TYPE_R); 4315 } 4316 if (kvm_aperfmperf_in_guest(vcpu->kvm)) { 4317 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_APERF, MSR_TYPE_R); 4318 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_MPERF, MSR_TYPE_R); 4319 } 4320 4321 /* PT MSRs can be passed through iff PT is exposed to the guest. */ 4322 if (vmx_pt_mode_is_host_guest()) 4323 pt_update_intercept_for_msr(vcpu); 4324 4325 if (vcpu->arch.xfd_no_write_intercept) 4326 vmx_disable_intercept_for_msr(vcpu, MSR_IA32_XFD, MSR_TYPE_RW); 4327 4328 vmx_set_intercept_for_msr(vcpu, MSR_IA32_SPEC_CTRL, MSR_TYPE_RW, 4329 !to_vmx(vcpu)->spec_ctrl); 4330 4331 if (kvm_cpu_cap_has(X86_FEATURE_XFD)) 4332 vmx_set_intercept_for_msr(vcpu, MSR_IA32_XFD_ERR, MSR_TYPE_R, 4333 !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD)); 4334 4335 if (cpu_feature_enabled(X86_FEATURE_IBPB)) 4336 vmx_set_intercept_for_msr(vcpu, MSR_IA32_PRED_CMD, MSR_TYPE_W, 4337 !guest_has_pred_cmd_msr(vcpu)); 4338 4339 if (cpu_feature_enabled(X86_FEATURE_FLUSH_L1D)) 4340 vmx_set_intercept_for_msr(vcpu, MSR_IA32_FLUSH_CMD, MSR_TYPE_W, 4341 !guest_cpu_cap_has(vcpu, X86_FEATURE_FLUSH_L1D)); 4342 4343 if (kvm_cpu_cap_has(X86_FEATURE_SHSTK)) { 4344 intercept = !guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK); 4345 4346 vmx_set_intercept_for_msr(vcpu, MSR_IA32_PL0_SSP, MSR_TYPE_RW, intercept); 4347 vmx_set_intercept_for_msr(vcpu, MSR_IA32_PL1_SSP, MSR_TYPE_RW, intercept); 4348 vmx_set_intercept_for_msr(vcpu, MSR_IA32_PL2_SSP, MSR_TYPE_RW, intercept); 4349 vmx_set_intercept_for_msr(vcpu, MSR_IA32_PL3_SSP, MSR_TYPE_RW, intercept); 4350 } 4351 4352 if (kvm_cpu_cap_has(X86_FEATURE_SHSTK) || kvm_cpu_cap_has(X86_FEATURE_IBT)) { 4353 intercept = !guest_cpu_cap_has(vcpu, X86_FEATURE_IBT) && 4354 !guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK); 4355 4356 vmx_set_intercept_for_msr(vcpu, MSR_IA32_U_CET, MSR_TYPE_RW, intercept); 4357 vmx_set_intercept_for_msr(vcpu, MSR_IA32_S_CET, MSR_TYPE_RW, intercept); 4358 } 4359 4360 vmx_recalc_pmu_msr_intercepts(vcpu); 4361 4362 /* 4363 * x2APIC and LBR MSR intercepts are modified on-demand and cannot be 4364 * filtered by userspace. 4365 */ 4366 } 4367 4368 static void vmx_recalc_instruction_intercepts(struct kvm_vcpu *vcpu) 4369 { 4370 exec_controls_changebit(to_vmx(vcpu), CPU_BASED_RDPMC_EXITING, 4371 kvm_need_rdpmc_intercept(vcpu)); 4372 } 4373 4374 void vmx_recalc_intercepts(struct kvm_vcpu *vcpu) 4375 { 4376 vmx_recalc_instruction_intercepts(vcpu); 4377 vmx_recalc_msr_intercepts(vcpu); 4378 } 4379 4380 static int vmx_deliver_nested_posted_interrupt(struct kvm_vcpu *vcpu, 4381 int vector) 4382 { 4383 struct vcpu_vmx *vmx = to_vmx(vcpu); 4384 4385 /* 4386 * DO NOT query the vCPU's vmcs12, as vmcs12 is dynamically allocated 4387 * and freed, and must not be accessed outside of vcpu->mutex. The 4388 * vCPU's cached PI NV is valid if and only if posted interrupts 4389 * enabled in its vmcs12, i.e. checking the vector also checks that 4390 * L1 has enabled posted interrupts for L2. 4391 */ 4392 if (is_guest_mode(vcpu) && 4393 vector == vmx->nested.posted_intr_nv) { 4394 /* 4395 * If a posted intr is not recognized by hardware, 4396 * we will accomplish it in the next vmentry. 4397 */ 4398 vmx->nested.pi_pending = true; 4399 kvm_make_request(KVM_REQ_EVENT, vcpu); 4400 4401 /* 4402 * This pairs with the smp_mb_*() after setting vcpu->mode in 4403 * vcpu_enter_guest() to guarantee the vCPU sees the event 4404 * request if triggering a posted interrupt "fails" because 4405 * vcpu->mode != IN_GUEST_MODE. The extra barrier is needed as 4406 * the smb_wmb() in kvm_make_request() only ensures everything 4407 * done before making the request is visible when the request 4408 * is visible, it doesn't ensure ordering between the store to 4409 * vcpu->requests and the load from vcpu->mode. 4410 */ 4411 smp_mb__after_atomic(); 4412 4413 /* the PIR and ON have been set by L1. */ 4414 kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_NESTED_VECTOR); 4415 return 0; 4416 } 4417 return -1; 4418 } 4419 /* 4420 * Send interrupt to vcpu via posted interrupt way. 4421 * 1. If target vcpu is running(non-root mode), send posted interrupt 4422 * notification to vcpu and hardware will sync PIR to vIRR atomically. 4423 * 2. If target vcpu isn't running(root mode), kick it to pick up the 4424 * interrupt from PIR in next vmentry. 4425 */ 4426 static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector) 4427 { 4428 struct vcpu_vt *vt = to_vt(vcpu); 4429 int r; 4430 4431 r = vmx_deliver_nested_posted_interrupt(vcpu, vector); 4432 if (!r) 4433 return 0; 4434 4435 /* Note, this is called iff the local APIC is in-kernel. */ 4436 if (!vcpu->arch.apic->apicv_active) 4437 return -1; 4438 4439 __vt_deliver_posted_interrupt(vcpu, &vt->pi_desc, vector); 4440 return 0; 4441 } 4442 4443 void vmx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, 4444 int trig_mode, int vector) 4445 { 4446 struct kvm_vcpu *vcpu = apic->vcpu; 4447 4448 if (vmx_deliver_posted_interrupt(vcpu, vector)) { 4449 kvm_lapic_set_irr(vector, apic); 4450 kvm_make_request(KVM_REQ_EVENT, vcpu); 4451 kvm_vcpu_kick(vcpu); 4452 } else { 4453 trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, 4454 trig_mode, vector); 4455 } 4456 } 4457 4458 /* 4459 * Set up the vmcs's constant host-state fields, i.e., host-state fields that 4460 * will not change in the lifetime of the guest. 4461 * Note that host-state that does change is set elsewhere. E.g., host-state 4462 * that is set differently for each CPU is set in vmx_vcpu_load(), not here. 4463 */ 4464 void vmx_set_constant_host_state(struct vcpu_vmx *vmx) 4465 { 4466 struct msr val; 4467 unsigned long tmpl; 4468 unsigned long cr0, cr3, cr4; 4469 4470 cr0 = read_cr0(); 4471 WARN_ON(cr0 & X86_CR0_TS); 4472 vmcs_writel(HOST_CR0, cr0); /* 22.2.3 */ 4473 4474 /* 4475 * Save the most likely value for this task's CR3 in the VMCS. 4476 * We can't use __get_current_cr3_fast() because we're not atomic. 4477 */ 4478 cr3 = __read_cr3(); 4479 vmcs_writel(HOST_CR3, cr3); /* 22.2.3 FIXME: shadow tables */ 4480 vmx->loaded_vmcs->host_state.cr3 = cr3; 4481 4482 /* Save the most likely value for this task's CR4 in the VMCS. */ 4483 cr4 = cr4_read_shadow(); 4484 vmcs_writel(HOST_CR4, cr4); /* 22.2.3, 22.2.5 */ 4485 vmx->loaded_vmcs->host_state.cr4 = cr4; 4486 4487 vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS); /* 22.2.4 */ 4488 #ifdef CONFIG_X86_64 4489 /* 4490 * Load null selectors, so we can avoid reloading them in 4491 * vmx_prepare_switch_to_host(), in case userspace uses 4492 * the null selectors too (the expected case). 4493 */ 4494 vmcs_write16(HOST_DS_SELECTOR, 0); 4495 vmcs_write16(HOST_ES_SELECTOR, 0); 4496 #else 4497 vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS); /* 22.2.4 */ 4498 vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS); /* 22.2.4 */ 4499 #endif 4500 vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS); /* 22.2.4 */ 4501 vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8); /* 22.2.4 */ 4502 4503 vmcs_writel(HOST_IDTR_BASE, host_idt_base); /* 22.2.4 */ 4504 4505 vmcs_writel(HOST_RIP, (unsigned long)vmx_vmexit); /* 22.2.5 */ 4506 4507 rdmsrq(MSR_IA32_SYSENTER_CS, val.q); 4508 vmcs_write32(HOST_IA32_SYSENTER_CS, val.l); 4509 4510 /* 4511 * SYSENTER is used for 32-bit system calls on either 32-bit or 4512 * 64-bit kernels. It is always zero If neither is allowed, otherwise 4513 * vmx_vcpu_load_vmcs loads it with the per-CPU entry stack (and may 4514 * have already done so!). 4515 */ 4516 if (!IS_ENABLED(CONFIG_IA32_EMULATION) && !IS_ENABLED(CONFIG_X86_32)) 4517 vmcs_writel(HOST_IA32_SYSENTER_ESP, 0); 4518 4519 rdmsrq(MSR_IA32_SYSENTER_EIP, tmpl); 4520 vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl); /* 22.2.3 */ 4521 4522 if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) { 4523 rdmsrq(MSR_IA32_CR_PAT, val.q); 4524 vmcs_write64(HOST_IA32_PAT, val.q); 4525 } 4526 4527 if (cpu_has_load_ia32_efer()) 4528 vmcs_write64(HOST_IA32_EFER, kvm_host.efer); 4529 4530 /* 4531 * Supervisor shadow stack is not enabled on host side, i.e., 4532 * host IA32_S_CET.SHSTK_EN bit is guaranteed to 0 now, per SDM 4533 * description(RDSSP instruction), SSP is not readable in CPL0, 4534 * so resetting the two registers to 0s at VM-Exit does no harm 4535 * to kernel execution. When execution flow exits to userspace, 4536 * SSP is reloaded from IA32_PL3_SSP. Check SDM Vol.2A/B Chapter 4537 * 3 and 4 for details. 4538 */ 4539 if (enable_cet) { 4540 vmcs_writel(HOST_S_CET, kvm_host.s_cet); 4541 vmcs_writel(HOST_SSP, 0); 4542 vmcs_writel(HOST_INTR_SSP_TABLE, 0); 4543 } 4544 4545 /* 4546 * When running a guest with a mediated PMU, guest state is resident in 4547 * hardware after VM-Exit. Zero PERF_GLOBAL_CTRL on exit so that host 4548 * activity doesn't bleed into the guest counters. When running with 4549 * an emulated PMU, PERF_GLOBAL_CTRL is dynamically computed on every 4550 * entry/exit to merge guest and host PMU usage. 4551 */ 4552 if (enable_mediated_pmu) 4553 vmcs_write64(HOST_IA32_PERF_GLOBAL_CTRL, 0); 4554 } 4555 4556 void set_cr4_guest_host_mask(struct vcpu_vmx *vmx) 4557 { 4558 struct kvm_vcpu *vcpu = &vmx->vcpu; 4559 4560 vcpu->arch.cr4_guest_owned_bits = KVM_POSSIBLE_CR4_GUEST_BITS & 4561 ~vcpu->arch.cr4_guest_rsvd_bits; 4562 if (!enable_ept) { 4563 vcpu->arch.cr4_guest_owned_bits &= ~X86_CR4_TLBFLUSH_BITS; 4564 vcpu->arch.cr4_guest_owned_bits &= ~X86_CR4_PDPTR_BITS; 4565 } 4566 if (is_guest_mode(&vmx->vcpu)) 4567 vcpu->arch.cr4_guest_owned_bits &= 4568 ~get_vmcs12(vcpu)->cr4_guest_host_mask; 4569 vmcs_writel(CR4_GUEST_HOST_MASK, ~vcpu->arch.cr4_guest_owned_bits); 4570 } 4571 4572 static u32 vmx_pin_based_exec_ctrl(struct vcpu_vmx *vmx) 4573 { 4574 u32 pin_based_exec_ctrl = vmcs_config.pin_based_exec_ctrl; 4575 4576 if (!kvm_vcpu_apicv_active(&vmx->vcpu)) 4577 pin_based_exec_ctrl &= ~PIN_BASED_POSTED_INTR; 4578 4579 if (!enable_vnmi) 4580 pin_based_exec_ctrl &= ~PIN_BASED_VIRTUAL_NMIS; 4581 4582 if (!enable_preemption_timer) 4583 pin_based_exec_ctrl &= ~PIN_BASED_VMX_PREEMPTION_TIMER; 4584 4585 return pin_based_exec_ctrl; 4586 } 4587 4588 static u32 vmx_get_initial_vmentry_ctrl(void) 4589 { 4590 u32 vmentry_ctrl = vmcs_config.vmentry_ctrl; 4591 4592 if (vmx_pt_mode_is_system()) 4593 vmentry_ctrl &= ~(VM_ENTRY_PT_CONCEAL_PIP | 4594 VM_ENTRY_LOAD_IA32_RTIT_CTL); 4595 4596 if (!enable_cet) 4597 vmentry_ctrl &= ~VM_ENTRY_LOAD_CET_STATE; 4598 4599 /* 4600 * IA32e mode, and loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically. 4601 */ 4602 vmentry_ctrl &= ~(VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL | 4603 VM_ENTRY_LOAD_IA32_EFER | 4604 VM_ENTRY_IA32E_MODE); 4605 4606 return vmentry_ctrl; 4607 } 4608 4609 static u32 vmx_get_initial_vmexit_ctrl(void) 4610 { 4611 u32 vmexit_ctrl = vmcs_config.vmexit_ctrl; 4612 4613 if (!enable_cet) 4614 vmexit_ctrl &= ~VM_EXIT_LOAD_CET_STATE; 4615 4616 /* 4617 * Not used by KVM and never set in vmcs01 or vmcs02, but emulated for 4618 * nested virtualization and thus allowed to be set in vmcs12. 4619 */ 4620 vmexit_ctrl &= ~(VM_EXIT_SAVE_IA32_PAT | VM_EXIT_SAVE_IA32_EFER | 4621 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER); 4622 4623 if (vmx_pt_mode_is_system()) 4624 vmexit_ctrl &= ~(VM_EXIT_PT_CONCEAL_PIP | 4625 VM_EXIT_CLEAR_IA32_RTIT_CTL); 4626 /* Loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically */ 4627 return vmexit_ctrl & 4628 ~(VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | VM_EXIT_LOAD_IA32_EFER | 4629 VM_EXIT_SAVE_IA32_PERF_GLOBAL_CTRL); 4630 } 4631 4632 void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) 4633 { 4634 struct vcpu_vmx *vmx = to_vmx(vcpu); 4635 4636 guard(vmx_vmcs01)(vcpu); 4637 4638 pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx)); 4639 4640 secondary_exec_controls_changebit(vmx, 4641 SECONDARY_EXEC_APIC_REGISTER_VIRT | 4642 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY, 4643 kvm_vcpu_apicv_active(vcpu)); 4644 if (enable_ipiv) 4645 tertiary_exec_controls_changebit(vmx, TERTIARY_EXEC_IPI_VIRT, 4646 kvm_vcpu_apicv_active(vcpu)); 4647 4648 vmx_update_msr_bitmap_x2apic(vcpu); 4649 } 4650 4651 static u32 vmx_exec_control(struct vcpu_vmx *vmx) 4652 { 4653 u32 exec_control = vmcs_config.cpu_based_exec_ctrl; 4654 4655 /* 4656 * Not used by KVM, but fully supported for nesting, i.e. are allowed in 4657 * vmcs12 and propagated to vmcs02 when set in vmcs12. 4658 */ 4659 exec_control &= ~(CPU_BASED_RDTSC_EXITING | 4660 CPU_BASED_USE_IO_BITMAPS | 4661 CPU_BASED_MONITOR_TRAP_FLAG | 4662 CPU_BASED_PAUSE_EXITING); 4663 4664 /* INTR_WINDOW_EXITING and NMI_WINDOW_EXITING are toggled dynamically */ 4665 exec_control &= ~(CPU_BASED_INTR_WINDOW_EXITING | 4666 CPU_BASED_NMI_WINDOW_EXITING); 4667 4668 if (vmx->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT) 4669 exec_control &= ~CPU_BASED_MOV_DR_EXITING; 4670 4671 if (!cpu_need_tpr_shadow(&vmx->vcpu)) 4672 exec_control &= ~CPU_BASED_TPR_SHADOW; 4673 4674 #ifdef CONFIG_X86_64 4675 if (exec_control & CPU_BASED_TPR_SHADOW) 4676 exec_control &= ~(CPU_BASED_CR8_LOAD_EXITING | 4677 CPU_BASED_CR8_STORE_EXITING); 4678 else 4679 exec_control |= CPU_BASED_CR8_STORE_EXITING | 4680 CPU_BASED_CR8_LOAD_EXITING; 4681 #endif 4682 /* No need to intercept CR3 access or INVPLG when using EPT. */ 4683 if (enable_ept) 4684 exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING | 4685 CPU_BASED_CR3_STORE_EXITING | 4686 CPU_BASED_INVLPG_EXITING); 4687 if (kvm_mwait_in_guest(vmx->vcpu.kvm)) 4688 exec_control &= ~(CPU_BASED_MWAIT_EXITING | 4689 CPU_BASED_MONITOR_EXITING); 4690 if (kvm_hlt_in_guest(vmx->vcpu.kvm)) 4691 exec_control &= ~CPU_BASED_HLT_EXITING; 4692 return exec_control; 4693 } 4694 4695 static u64 vmx_tertiary_exec_control(struct vcpu_vmx *vmx) 4696 { 4697 u64 exec_control = vmcs_config.cpu_based_3rd_exec_ctrl; 4698 4699 /* 4700 * IPI virtualization relies on APICv. Disable IPI virtualization if 4701 * APICv is inhibited. 4702 */ 4703 if (!enable_ipiv || !kvm_vcpu_apicv_active(&vmx->vcpu)) 4704 exec_control &= ~TERTIARY_EXEC_IPI_VIRT; 4705 4706 return exec_control; 4707 } 4708 4709 /* 4710 * Adjust a single secondary execution control bit to intercept/allow an 4711 * instruction in the guest. This is usually done based on whether or not a 4712 * feature has been exposed to the guest in order to correctly emulate faults. 4713 */ 4714 static inline void 4715 vmx_adjust_secondary_exec_control(struct vcpu_vmx *vmx, u32 *exec_control, 4716 u32 control, bool enabled, bool exiting) 4717 { 4718 /* 4719 * If the control is for an opt-in feature, clear the control if the 4720 * feature is not exposed to the guest, i.e. not enabled. If the 4721 * control is opt-out, i.e. an exiting control, clear the control if 4722 * the feature _is_ exposed to the guest, i.e. exiting/interception is 4723 * disabled for the associated instruction. Note, the caller is 4724 * responsible presetting exec_control to set all supported bits. 4725 */ 4726 if (enabled == exiting) 4727 *exec_control &= ~control; 4728 4729 /* 4730 * Update the nested MSR settings so that a nested VMM can/can't set 4731 * controls for features that are/aren't exposed to the guest. 4732 */ 4733 if (nested && 4734 kvm_check_has_quirk(vmx->vcpu.kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS)) { 4735 /* 4736 * All features that can be added or removed to VMX MSRs must 4737 * be supported in the first place for nested virtualization. 4738 */ 4739 if (WARN_ON_ONCE(!(vmcs_config.nested.secondary_ctls_high & control))) 4740 enabled = false; 4741 4742 if (enabled) 4743 vmx->nested.msrs.secondary_ctls_high |= control; 4744 else 4745 vmx->nested.msrs.secondary_ctls_high &= ~control; 4746 } 4747 } 4748 4749 /* 4750 * Wrapper macro for the common case of adjusting a secondary execution control 4751 * based on a single guest CPUID bit, with a dedicated feature bit. This also 4752 * verifies that the control is actually supported by KVM and hardware. 4753 */ 4754 #define vmx_adjust_sec_exec_control(vmx, exec_control, name, feat_name, ctrl_name, exiting) \ 4755 ({ \ 4756 struct kvm_vcpu *__vcpu = &(vmx)->vcpu; \ 4757 bool __enabled; \ 4758 \ 4759 if (cpu_has_vmx_##name()) { \ 4760 __enabled = guest_cpu_cap_has(__vcpu, X86_FEATURE_##feat_name); \ 4761 vmx_adjust_secondary_exec_control(vmx, exec_control, SECONDARY_EXEC_##ctrl_name,\ 4762 __enabled, exiting); \ 4763 } \ 4764 }) 4765 4766 /* More macro magic for ENABLE_/opt-in versus _EXITING/opt-out controls. */ 4767 #define vmx_adjust_sec_exec_feature(vmx, exec_control, lname, uname) \ 4768 vmx_adjust_sec_exec_control(vmx, exec_control, lname, uname, ENABLE_##uname, false) 4769 4770 #define vmx_adjust_sec_exec_exiting(vmx, exec_control, lname, uname) \ 4771 vmx_adjust_sec_exec_control(vmx, exec_control, lname, uname, uname##_EXITING, true) 4772 4773 static u32 vmx_secondary_exec_control(struct vcpu_vmx *vmx) 4774 { 4775 struct kvm_vcpu *vcpu = &vmx->vcpu; 4776 4777 u32 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl; 4778 4779 if (vmx_pt_mode_is_system()) 4780 exec_control &= ~(SECONDARY_EXEC_PT_USE_GPA | SECONDARY_EXEC_PT_CONCEAL_VMX); 4781 if (!cpu_need_virtualize_apic_accesses(vcpu)) 4782 exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 4783 if (vmx->vpid == 0) 4784 exec_control &= ~SECONDARY_EXEC_ENABLE_VPID; 4785 if (!enable_ept) { 4786 exec_control &= ~SECONDARY_EXEC_ENABLE_EPT; 4787 exec_control &= ~SECONDARY_EXEC_EPT_VIOLATION_VE; 4788 enable_unrestricted_guest = 0; 4789 } 4790 if (!enable_unrestricted_guest) 4791 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST; 4792 if (kvm_pause_in_guest(vmx->vcpu.kvm)) 4793 exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING; 4794 if (!kvm_vcpu_apicv_active(vcpu)) 4795 exec_control &= ~(SECONDARY_EXEC_APIC_REGISTER_VIRT | 4796 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); 4797 exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE; 4798 4799 /* 4800 * KVM doesn't support VMFUNC for L1, but the control is set in KVM's 4801 * base configuration as KVM emulates VMFUNC[EPTP_SWITCHING] for L2. 4802 */ 4803 exec_control &= ~SECONDARY_EXEC_ENABLE_VMFUNC; 4804 4805 if (!enable_mbec) 4806 exec_control &= ~SECONDARY_EXEC_MODE_BASED_EPT_EXEC; 4807 4808 /* SECONDARY_EXEC_DESC is enabled/disabled on writes to CR4.UMIP, 4809 * in vmx_set_cr4. */ 4810 exec_control &= ~SECONDARY_EXEC_DESC; 4811 4812 /* SECONDARY_EXEC_SHADOW_VMCS is enabled when L1 executes VMPTRLD 4813 (handle_vmptrld). 4814 We can NOT enable shadow_vmcs here because we don't have yet 4815 a current VMCS12 4816 */ 4817 exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS; 4818 4819 /* 4820 * PML is enabled/disabled when dirty logging of memsmlots changes, but 4821 * it needs to be set here when dirty logging is already active, e.g. 4822 * if this vCPU was created after dirty logging was enabled. 4823 */ 4824 if (!enable_pml || !atomic_read(&vcpu->kvm->nr_memslots_dirty_logging)) 4825 exec_control &= ~SECONDARY_EXEC_ENABLE_PML; 4826 4827 vmx_adjust_sec_exec_feature(vmx, &exec_control, xsaves, XSAVES); 4828 4829 /* 4830 * RDPID is also gated by ENABLE_RDTSCP, turn on the control if either 4831 * feature is exposed to the guest. This creates a virtualization hole 4832 * if both are supported in hardware but only one is exposed to the 4833 * guest, but letting the guest execute RDTSCP or RDPID when either one 4834 * is advertised is preferable to emulating the advertised instruction 4835 * in KVM on #UD, and obviously better than incorrectly injecting #UD. 4836 */ 4837 if (cpu_has_vmx_rdtscp()) { 4838 bool rdpid_or_rdtscp_enabled = 4839 guest_cpu_cap_has(vcpu, X86_FEATURE_RDTSCP) || 4840 guest_cpu_cap_has(vcpu, X86_FEATURE_RDPID); 4841 4842 vmx_adjust_secondary_exec_control(vmx, &exec_control, 4843 SECONDARY_EXEC_ENABLE_RDTSCP, 4844 rdpid_or_rdtscp_enabled, false); 4845 } 4846 4847 vmx_adjust_sec_exec_feature(vmx, &exec_control, invpcid, INVPCID); 4848 4849 vmx_adjust_sec_exec_exiting(vmx, &exec_control, rdrand, RDRAND); 4850 vmx_adjust_sec_exec_exiting(vmx, &exec_control, rdseed, RDSEED); 4851 4852 vmx_adjust_sec_exec_control(vmx, &exec_control, waitpkg, WAITPKG, 4853 ENABLE_USR_WAIT_PAUSE, false); 4854 4855 if (!vcpu->kvm->arch.bus_lock_detection_enabled) 4856 exec_control &= ~SECONDARY_EXEC_BUS_LOCK_DETECTION; 4857 4858 if (!kvm_notify_vmexit_enabled(vcpu->kvm)) 4859 exec_control &= ~SECONDARY_EXEC_NOTIFY_VM_EXITING; 4860 4861 return exec_control; 4862 } 4863 4864 static inline int vmx_get_pid_table_order(struct kvm *kvm) 4865 { 4866 return get_order(kvm->arch.max_vcpu_ids * sizeof(*to_kvm_vmx(kvm)->pid_table)); 4867 } 4868 4869 static int vmx_alloc_ipiv_pid_table(struct kvm *kvm) 4870 { 4871 struct page *pages; 4872 struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); 4873 4874 if (!irqchip_in_kernel(kvm) || !enable_ipiv) 4875 return 0; 4876 4877 if (kvm_vmx->pid_table) 4878 return 0; 4879 4880 pages = alloc_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO, 4881 vmx_get_pid_table_order(kvm)); 4882 if (!pages) 4883 return -ENOMEM; 4884 4885 kvm_vmx->pid_table = (void *)page_address(pages); 4886 return 0; 4887 } 4888 4889 int vmx_vcpu_precreate(struct kvm *kvm) 4890 { 4891 return vmx_alloc_ipiv_pid_table(kvm); 4892 } 4893 4894 #define VMX_XSS_EXIT_BITMAP 0 4895 4896 static void init_vmcs(struct kvm_vcpu *vcpu) 4897 { 4898 struct kvm *kvm = vcpu->kvm; 4899 struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); 4900 struct vcpu_vmx *vmx = to_vmx(vcpu); 4901 4902 if (nested) 4903 nested_vmx_set_vmcs_shadowing_bitmap(); 4904 4905 if (cpu_has_vmx_msr_bitmap()) 4906 vmcs_write64(MSR_BITMAP, __pa(vmx->vmcs01.msr_bitmap)); 4907 4908 vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); /* 22.3.1.5 */ 4909 4910 /* Control */ 4911 pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx)); 4912 4913 exec_controls_set(vmx, vmx_exec_control(vmx)); 4914 4915 if (cpu_has_secondary_exec_ctrls()) { 4916 secondary_exec_controls_set(vmx, vmx_secondary_exec_control(vmx)); 4917 if (vmx->ve_info) 4918 vmcs_write64(VE_INFORMATION_ADDRESS, 4919 __pa(vmx->ve_info)); 4920 } 4921 4922 if (cpu_has_tertiary_exec_ctrls()) 4923 tertiary_exec_controls_set(vmx, vmx_tertiary_exec_control(vmx)); 4924 4925 if (enable_apicv && lapic_in_kernel(vcpu)) { 4926 vmcs_write64(EOI_EXIT_BITMAP0, 0); 4927 vmcs_write64(EOI_EXIT_BITMAP1, 0); 4928 vmcs_write64(EOI_EXIT_BITMAP2, 0); 4929 vmcs_write64(EOI_EXIT_BITMAP3, 0); 4930 4931 vmcs_write16(GUEST_INTR_STATUS, 0); 4932 4933 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR); 4934 vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->vt.pi_desc))); 4935 } 4936 4937 if (vmx_can_use_ipiv(vcpu)) { 4938 vmcs_write64(PID_POINTER_TABLE, __pa(kvm_vmx->pid_table)); 4939 vmcs_write16(LAST_PID_POINTER_INDEX, kvm->arch.max_vcpu_ids - 1); 4940 } 4941 4942 if (!kvm_pause_in_guest(kvm)) { 4943 vmcs_write32(PLE_GAP, ple_gap); 4944 vmx->ple_window = ple_window; 4945 vmx->ple_window_dirty = true; 4946 } 4947 4948 if (kvm_notify_vmexit_enabled(kvm)) 4949 vmcs_write32(NOTIFY_WINDOW, kvm->arch.notify_window); 4950 4951 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0); 4952 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0); 4953 vmcs_write32(CR3_TARGET_COUNT, 0); /* 22.2.1 */ 4954 4955 vmcs_write16(HOST_FS_SELECTOR, 0); /* 22.2.4 */ 4956 vmcs_write16(HOST_GS_SELECTOR, 0); /* 22.2.4 */ 4957 vmx_set_constant_host_state(vmx); 4958 vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */ 4959 vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */ 4960 4961 if (cpu_has_vmx_vmfunc()) 4962 vmcs_write64(VM_FUNCTION_CONTROL, 0); 4963 4964 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0); 4965 vmcs_write64(VM_EXIT_MSR_STORE_ADDR, __pa(vmx->msr_autostore.val)); 4966 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0); 4967 vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val)); 4968 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0); 4969 vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); 4970 4971 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) 4972 vmcs_write64(GUEST_IA32_PAT, vcpu->arch.pat); 4973 4974 vm_exit_controls_set(vmx, vmx_get_initial_vmexit_ctrl()); 4975 4976 /* 22.2.1, 20.8.1 */ 4977 vm_entry_controls_set(vmx, vmx_get_initial_vmentry_ctrl()); 4978 4979 vcpu->arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits(); 4980 vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits); 4981 4982 set_cr4_guest_host_mask(vmx); 4983 4984 if (vmx->vpid != 0) 4985 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid); 4986 4987 if (cpu_has_vmx_xsaves()) 4988 vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP); 4989 4990 if (enable_pml) { 4991 vmcs_write64(PML_ADDRESS, page_to_phys(vcpu->arch.pml_page)); 4992 vmcs_write16(GUEST_PML_INDEX, PML_HEAD_INDEX); 4993 } 4994 4995 vmx_write_encls_bitmap(vcpu, NULL); 4996 4997 if (vmx_pt_mode_is_host_guest()) { 4998 memset(&vmx->pt_desc, 0, sizeof(vmx->pt_desc)); 4999 /* Bit[6~0] are forced to 1, writes are ignored. */ 5000 vmx->pt_desc.guest.output_mask = 0x7F; 5001 vmcs_write64(GUEST_IA32_RTIT_CTL, 0); 5002 } 5003 5004 vmcs_write32(GUEST_SYSENTER_CS, 0); 5005 vmcs_writel(GUEST_SYSENTER_ESP, 0); 5006 vmcs_writel(GUEST_SYSENTER_EIP, 0); 5007 5008 vmx_guest_debugctl_write(vcpu, 0); 5009 5010 if (cpu_has_vmx_tpr_shadow()) { 5011 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0); 5012 if (cpu_need_tpr_shadow(vcpu)) 5013 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 5014 __pa(vcpu->arch.apic->regs)); 5015 vmcs_write32(TPR_THRESHOLD, 0); 5016 } 5017 5018 vmx_setup_uret_msrs(vmx); 5019 } 5020 5021 static void __vmx_vcpu_reset(struct kvm_vcpu *vcpu) 5022 { 5023 struct vcpu_vmx *vmx = to_vmx(vcpu); 5024 5025 init_vmcs(vcpu); 5026 5027 if (nested && 5028 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS)) 5029 memcpy(&vmx->nested.msrs, &vmcs_config.nested, sizeof(vmx->nested.msrs)); 5030 5031 vcpu_setup_sgx_lepubkeyhash(vcpu); 5032 5033 vmx->nested.posted_intr_nv = -1; 5034 vmx->nested.vmxon_ptr = INVALID_GPA; 5035 vmx->nested.current_vmptr = INVALID_GPA; 5036 5037 #ifdef CONFIG_KVM_HYPERV 5038 vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID; 5039 #endif 5040 5041 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS)) 5042 vcpu->arch.microcode_version = 0x100000000ULL; 5043 vmx->msr_ia32_feature_control_valid_bits = FEAT_CTL_LOCKED; 5044 5045 /* 5046 * Enforce invariant: pi_desc.nv is always either POSTED_INTR_VECTOR 5047 * or POSTED_INTR_WAKEUP_VECTOR. 5048 */ 5049 vmx->vt.pi_desc.nv = POSTED_INTR_VECTOR; 5050 __pi_set_sn(&vmx->vt.pi_desc); 5051 } 5052 5053 void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) 5054 { 5055 struct vcpu_vmx *vmx = to_vmx(vcpu); 5056 5057 if (!init_event) 5058 __vmx_vcpu_reset(vcpu); 5059 5060 vmx->rmode.vm86_active = 0; 5061 vmx->spec_ctrl = 0; 5062 5063 vmx->msr_ia32_umwait_control = 0; 5064 5065 vmx->hv_deadline_tsc = -1; 5066 kvm_set_cr8(vcpu, 0); 5067 5068 seg_setup(VCPU_SREG_CS); 5069 vmcs_write16(GUEST_CS_SELECTOR, 0xf000); 5070 vmcs_writel(GUEST_CS_BASE, 0xffff0000ul); 5071 5072 seg_setup(VCPU_SREG_DS); 5073 seg_setup(VCPU_SREG_ES); 5074 seg_setup(VCPU_SREG_FS); 5075 seg_setup(VCPU_SREG_GS); 5076 seg_setup(VCPU_SREG_SS); 5077 5078 vmcs_write16(GUEST_TR_SELECTOR, 0); 5079 vmcs_writel(GUEST_TR_BASE, 0); 5080 vmcs_write32(GUEST_TR_LIMIT, 0xffff); 5081 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); 5082 5083 vmcs_write16(GUEST_LDTR_SELECTOR, 0); 5084 vmcs_writel(GUEST_LDTR_BASE, 0); 5085 vmcs_write32(GUEST_LDTR_LIMIT, 0xffff); 5086 vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082); 5087 5088 vmcs_writel(GUEST_GDTR_BASE, 0); 5089 vmcs_write32(GUEST_GDTR_LIMIT, 0xffff); 5090 5091 vmcs_writel(GUEST_IDTR_BASE, 0); 5092 vmcs_write32(GUEST_IDTR_LIMIT, 0xffff); 5093 5094 vmx_segment_cache_clear(vmx); 5095 kvm_register_mark_available(vcpu, VCPU_REG_SEGMENTS); 5096 5097 vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE); 5098 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0); 5099 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 0); 5100 if (kvm_mpx_supported()) 5101 vmcs_write64(GUEST_BNDCFGS, 0); 5102 5103 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); /* 22.2.1 */ 5104 5105 if (kvm_cpu_cap_has(X86_FEATURE_SHSTK)) { 5106 vmcs_writel(GUEST_SSP, 0); 5107 vmcs_writel(GUEST_INTR_SSP_TABLE, 0); 5108 } 5109 if (kvm_cpu_cap_has(X86_FEATURE_IBT) || 5110 kvm_cpu_cap_has(X86_FEATURE_SHSTK)) 5111 vmcs_writel(GUEST_S_CET, 0); 5112 5113 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 5114 5115 vpid_sync_context(vmx->vpid); 5116 5117 vmx_update_fb_clear_dis(vcpu, vmx); 5118 } 5119 5120 void vmx_enable_irq_window(struct kvm_vcpu *vcpu) 5121 { 5122 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING); 5123 } 5124 5125 void vmx_enable_nmi_window(struct kvm_vcpu *vcpu) 5126 { 5127 if (!enable_vnmi || 5128 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) { 5129 vmx_enable_irq_window(vcpu); 5130 return; 5131 } 5132 5133 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_NMI_WINDOW_EXITING); 5134 } 5135 5136 void vmx_inject_irq(struct kvm_vcpu *vcpu, bool reinjected) 5137 { 5138 struct vcpu_vmx *vmx = to_vmx(vcpu); 5139 uint32_t intr; 5140 int irq = vcpu->arch.interrupt.nr; 5141 5142 trace_kvm_inj_virq(irq, vcpu->arch.interrupt.soft, reinjected); 5143 5144 ++vcpu->stat.irq_injections; 5145 if (vmx->rmode.vm86_active) { 5146 int inc_eip = 0; 5147 if (vcpu->arch.interrupt.soft) 5148 inc_eip = vcpu->arch.event_exit_inst_len; 5149 kvm_inject_realmode_interrupt(vcpu, irq, inc_eip); 5150 return; 5151 } 5152 intr = irq | INTR_INFO_VALID_MASK; 5153 if (vcpu->arch.interrupt.soft) { 5154 intr |= INTR_TYPE_SOFT_INTR; 5155 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 5156 vmx->vcpu.arch.event_exit_inst_len); 5157 } else 5158 intr |= INTR_TYPE_EXT_INTR; 5159 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr); 5160 5161 vmx_clear_hlt(vcpu); 5162 } 5163 5164 void vmx_inject_nmi(struct kvm_vcpu *vcpu) 5165 { 5166 struct vcpu_vmx *vmx = to_vmx(vcpu); 5167 5168 if (!enable_vnmi) { 5169 /* 5170 * Tracking the NMI-blocked state in software is built upon 5171 * finding the next open IRQ window. This, in turn, depends on 5172 * well-behaving guests: They have to keep IRQs disabled at 5173 * least as long as the NMI handler runs. Otherwise we may 5174 * cause NMI nesting, maybe breaking the guest. But as this is 5175 * highly unlikely, we can live with the residual risk. 5176 */ 5177 vmx->loaded_vmcs->soft_vnmi_blocked = 1; 5178 vmx->loaded_vmcs->vnmi_blocked_time = 0; 5179 } 5180 5181 ++vcpu->stat.nmi_injections; 5182 vmx->loaded_vmcs->nmi_known_unmasked = false; 5183 5184 if (vmx->rmode.vm86_active) { 5185 kvm_inject_realmode_interrupt(vcpu, NMI_VECTOR, 0); 5186 return; 5187 } 5188 5189 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 5190 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR); 5191 5192 vmx_clear_hlt(vcpu); 5193 } 5194 5195 bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu) 5196 { 5197 struct vcpu_vmx *vmx = to_vmx(vcpu); 5198 bool masked; 5199 5200 if (!enable_vnmi) 5201 return vmx->loaded_vmcs->soft_vnmi_blocked; 5202 if (vmx->loaded_vmcs->nmi_known_unmasked) 5203 return false; 5204 masked = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_NMI; 5205 vmx->loaded_vmcs->nmi_known_unmasked = !masked; 5206 return masked; 5207 } 5208 5209 void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) 5210 { 5211 struct vcpu_vmx *vmx = to_vmx(vcpu); 5212 5213 if (!enable_vnmi) { 5214 if (vmx->loaded_vmcs->soft_vnmi_blocked != masked) { 5215 vmx->loaded_vmcs->soft_vnmi_blocked = masked; 5216 vmx->loaded_vmcs->vnmi_blocked_time = 0; 5217 } 5218 } else { 5219 vmx->loaded_vmcs->nmi_known_unmasked = !masked; 5220 if (masked) 5221 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 5222 GUEST_INTR_STATE_NMI); 5223 else 5224 vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO, 5225 GUEST_INTR_STATE_NMI); 5226 } 5227 } 5228 5229 bool vmx_nmi_blocked(struct kvm_vcpu *vcpu) 5230 { 5231 if (is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu)) 5232 return false; 5233 5234 if (!enable_vnmi && to_vmx(vcpu)->loaded_vmcs->soft_vnmi_blocked) 5235 return true; 5236 5237 return (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 5238 (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI | 5239 GUEST_INTR_STATE_NMI)); 5240 } 5241 5242 int vmx_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 5243 { 5244 if (vcpu->arch.nested_run_pending) 5245 return -EBUSY; 5246 5247 /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */ 5248 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu)) 5249 return -EBUSY; 5250 5251 return !vmx_nmi_blocked(vcpu); 5252 } 5253 5254 bool __vmx_interrupt_blocked(struct kvm_vcpu *vcpu) 5255 { 5256 return !(vmx_get_rflags(vcpu) & X86_EFLAGS_IF) || 5257 (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 5258 (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS)); 5259 } 5260 5261 bool vmx_interrupt_blocked(struct kvm_vcpu *vcpu) 5262 { 5263 if (is_guest_mode(vcpu) && nested_exit_on_intr(vcpu)) 5264 return false; 5265 5266 return __vmx_interrupt_blocked(vcpu); 5267 } 5268 5269 int vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) 5270 { 5271 if (vmx_interrupt_blocked(vcpu)) 5272 return 0; 5273 5274 if (vcpu->arch.nested_run_pending) 5275 return -EBUSY; 5276 5277 /* 5278 * An IRQ must not be injected into L2 if it's supposed to VM-Exit, 5279 * e.g. if the IRQ arrived asynchronously after checking nested events. 5280 */ 5281 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(vcpu)) 5282 return -EBUSY; 5283 5284 return 1; 5285 } 5286 5287 int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr) 5288 { 5289 void __user *ret; 5290 5291 if (enable_unrestricted_guest) 5292 return 0; 5293 5294 mutex_lock(&kvm->slots_lock); 5295 ret = __x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, addr, 5296 PAGE_SIZE * 3); 5297 mutex_unlock(&kvm->slots_lock); 5298 5299 if (IS_ERR(ret)) 5300 return PTR_ERR(ret); 5301 5302 to_kvm_vmx(kvm)->tss_addr = addr; 5303 5304 return init_rmode_tss(kvm, ret); 5305 } 5306 5307 int vmx_set_identity_map_addr(struct kvm *kvm, u64 ident_addr) 5308 { 5309 to_kvm_vmx(kvm)->ept_identity_map_addr = ident_addr; 5310 return 0; 5311 } 5312 5313 static bool rmode_exception(struct kvm_vcpu *vcpu, int vec) 5314 { 5315 switch (vec) { 5316 case BP_VECTOR: 5317 /* 5318 * Update instruction length as we may reinject the exception 5319 * from user space while in guest debugging mode. 5320 */ 5321 to_vmx(vcpu)->vcpu.arch.event_exit_inst_len = 5322 vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 5323 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) 5324 return false; 5325 fallthrough; 5326 case DB_VECTOR: 5327 return !(vcpu->guest_debug & 5328 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)); 5329 case DE_VECTOR: 5330 case OF_VECTOR: 5331 case BR_VECTOR: 5332 case UD_VECTOR: 5333 case DF_VECTOR: 5334 case SS_VECTOR: 5335 case GP_VECTOR: 5336 case MF_VECTOR: 5337 return true; 5338 } 5339 return false; 5340 } 5341 5342 static int handle_rmode_exception(struct kvm_vcpu *vcpu, 5343 int vec, u32 err_code) 5344 { 5345 /* 5346 * Instruction with address size override prefix opcode 0x67 5347 * Cause the #SS fault with 0 error code in VM86 mode. 5348 */ 5349 if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) { 5350 if (kvm_emulate_instruction(vcpu, 0)) { 5351 if (vcpu->arch.halt_request) { 5352 vcpu->arch.halt_request = 0; 5353 return kvm_emulate_halt_noskip(vcpu); 5354 } 5355 return 1; 5356 } 5357 return 0; 5358 } 5359 5360 /* 5361 * Forward all other exceptions that are valid in real mode. 5362 * FIXME: Breaks guest debugging in real mode, needs to be fixed with 5363 * the required debugging infrastructure rework. 5364 */ 5365 kvm_queue_exception(vcpu, vec); 5366 return 1; 5367 } 5368 5369 static int handle_machine_check(struct kvm_vcpu *vcpu) 5370 { 5371 /* handled by vmx_vcpu_run() */ 5372 return 1; 5373 } 5374 5375 /* 5376 * If the host has split lock detection disabled, then #AC is 5377 * unconditionally injected into the guest, which is the pre split lock 5378 * detection behaviour. 5379 * 5380 * If the host has split lock detection enabled then #AC is 5381 * only injected into the guest when: 5382 * - Guest CPL == 3 (user mode) 5383 * - Guest has #AC detection enabled in CR0 5384 * - Guest EFLAGS has AC bit set 5385 */ 5386 bool vmx_guest_inject_ac(struct kvm_vcpu *vcpu) 5387 { 5388 if (!boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT)) 5389 return true; 5390 5391 return vmx_get_cpl(vcpu) == 3 && kvm_is_cr0_bit_set(vcpu, X86_CR0_AM) && 5392 (kvm_get_rflags(vcpu) & X86_EFLAGS_AC); 5393 } 5394 5395 static int vmx_handle_page_fault(struct kvm_vcpu *vcpu, u32 error_code) 5396 { 5397 unsigned long cr2 = vt_get_exit_qual(vcpu); 5398 5399 if (vcpu->arch.apf.host_apf_flags) 5400 goto handle_pf; 5401 5402 /* When using EPT, KVM intercepts #PF only to detect illegal GPAs. */ 5403 WARN_ON_ONCE(enable_ept && !allow_smaller_maxphyaddr); 5404 5405 /* 5406 * On SGX2 hardware, EPCM violations are delivered as #PF with the SGX 5407 * flag set in the error code (SGX1 hardware generates #GP(0)). EPCM 5408 * violations have nothing to do with shadow paging and can never be 5409 * resolved by KVM; always reflect them into the guest. 5410 */ 5411 if (error_code & PFERR_SGX_MASK) { 5412 WARN_ON_ONCE(!IS_ENABLED(CONFIG_X86_SGX_KVM) || 5413 !cpu_feature_enabled(X86_FEATURE_SGX2)); 5414 5415 if (guest_cpu_cap_has(vcpu, X86_FEATURE_SGX2)) 5416 kvm_fixup_and_inject_pf_error(vcpu, cr2, error_code); 5417 else 5418 kvm_inject_gp(vcpu, 0); 5419 return 1; 5420 } 5421 5422 /* 5423 * If EPT is enabled, fixup and inject the #PF. KVM intercepts #PFs 5424 * only to set PFERR_RSVD as appropriate (hardware won't set RSVD due 5425 * to the GPA being legal with respect to host.MAXPHYADDR). 5426 */ 5427 if (enable_ept) { 5428 kvm_fixup_and_inject_pf_error(vcpu, cr2, error_code); 5429 return 1; 5430 } 5431 5432 handle_pf: 5433 return kvm_handle_page_fault(vcpu, error_code, cr2, NULL, 0); 5434 } 5435 5436 static int handle_exception_nmi(struct kvm_vcpu *vcpu) 5437 { 5438 struct vcpu_vmx *vmx = to_vmx(vcpu); 5439 struct kvm_run *kvm_run = vcpu->run; 5440 u32 intr_info, ex_no, error_code; 5441 unsigned long dr6; 5442 u32 vect_info; 5443 5444 vect_info = vmx->idt_vectoring_info; 5445 intr_info = vt_get_intr_info(vcpu); 5446 5447 /* 5448 * Machine checks are handled by handle_exception_irqoff(), or by 5449 * vmx_vcpu_run() if a #MC occurs on VM-Entry. NMIs are handled by 5450 * vmx_vcpu_enter_exit(). 5451 */ 5452 if (is_machine_check(intr_info) || is_nmi(intr_info)) 5453 return 1; 5454 5455 /* 5456 * Queue the exception here instead of in handle_nm_fault_irqoff(). 5457 * This ensures the nested_vmx check is not skipped so vmexit can 5458 * be reflected to L1 (when it intercepts #NM) before reaching this 5459 * point. 5460 */ 5461 if (is_nm_fault(intr_info)) { 5462 kvm_queue_exception_p(vcpu, NM_VECTOR, 5463 is_xfd_nm_fault(vcpu) ? vcpu->arch.guest_fpu.xfd_err : 0); 5464 return 1; 5465 } 5466 5467 if (is_invalid_opcode(intr_info)) 5468 return handle_ud(vcpu); 5469 5470 if (WARN_ON_ONCE(is_ve_fault(intr_info))) { 5471 struct vmx_ve_information *ve_info = vmx->ve_info; 5472 5473 WARN_ONCE(ve_info->exit_reason != EXIT_REASON_EPT_VIOLATION, 5474 "Unexpected #VE on VM-Exit reason 0x%x", ve_info->exit_reason); 5475 dump_vmcs(vcpu); 5476 kvm_mmu_print_sptes(vcpu, ve_info->guest_physical_address, "#VE"); 5477 return 1; 5478 } 5479 5480 error_code = 0; 5481 if (intr_info & INTR_INFO_DELIVER_CODE_MASK) 5482 error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); 5483 5484 if (!vmx->rmode.vm86_active && is_gp_fault(intr_info)) { 5485 WARN_ON_ONCE(!enable_vmware_backdoor); 5486 5487 /* 5488 * VMware backdoor emulation on #GP interception only handles 5489 * IN{S}, OUT{S}, and RDPMC, none of which generate a non-zero 5490 * error code on #GP. 5491 */ 5492 if (error_code) { 5493 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code); 5494 return 1; 5495 } 5496 return kvm_emulate_instruction(vcpu, EMULTYPE_VMWARE_GP); 5497 } 5498 5499 /* 5500 * The #PF with PFEC.RSVD = 1 indicates the guest is accessing 5501 * MMIO, it is better to report an internal error. 5502 * See the comments in vmx_handle_exit. 5503 */ 5504 if ((vect_info & VECTORING_INFO_VALID_MASK) && 5505 !(is_page_fault(intr_info) && !(error_code & PFERR_RSVD_MASK))) { 5506 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 5507 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX; 5508 vcpu->run->internal.ndata = 4; 5509 vcpu->run->internal.data[0] = vect_info; 5510 vcpu->run->internal.data[1] = intr_info; 5511 vcpu->run->internal.data[2] = error_code; 5512 vcpu->run->internal.data[3] = vcpu->arch.last_vmentry_cpu; 5513 return 0; 5514 } 5515 5516 if (is_page_fault(intr_info)) 5517 return vmx_handle_page_fault(vcpu, error_code); 5518 5519 ex_no = intr_info & INTR_INFO_VECTOR_MASK; 5520 5521 if (vmx->rmode.vm86_active && rmode_exception(vcpu, ex_no)) 5522 return handle_rmode_exception(vcpu, ex_no, error_code); 5523 5524 switch (ex_no) { 5525 case DB_VECTOR: 5526 dr6 = vt_get_exit_qual(vcpu); 5527 if (!(vcpu->guest_debug & 5528 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) { 5529 /* 5530 * If the #DB was due to ICEBP, a.k.a. INT1, skip the 5531 * instruction. ICEBP generates a trap-like #DB, but 5532 * despite its interception control being tied to #DB, 5533 * is an instruction intercept, i.e. the VM-Exit occurs 5534 * on the ICEBP itself. Use the inner "skip" helper to 5535 * avoid single-step #DB and MTF updates, as ICEBP is 5536 * higher priority. Note, skipping ICEBP still clears 5537 * STI and MOVSS blocking. 5538 */ 5539 if (is_icebp(intr_info)) 5540 WARN_ON(!skip_emulated_instruction(vcpu)); 5541 5542 kvm_queue_exception_p(vcpu, DB_VECTOR, dr6); 5543 return 1; 5544 } 5545 kvm_run->debug.arch.dr6 = dr6 | DR6_ACTIVE_LOW; 5546 kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7); 5547 fallthrough; 5548 case BP_VECTOR: 5549 /* 5550 * Update instruction length as we may reinject #BP from 5551 * user space while in guest debugging mode. Reading it for 5552 * #DB as well causes no harm, it is not used in that case. 5553 */ 5554 vmx->vcpu.arch.event_exit_inst_len = 5555 vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 5556 kvm_run->exit_reason = KVM_EXIT_DEBUG; 5557 kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu); 5558 kvm_run->debug.arch.exception = ex_no; 5559 break; 5560 case AC_VECTOR: 5561 if (vmx_guest_inject_ac(vcpu)) { 5562 kvm_queue_exception_e(vcpu, AC_VECTOR, error_code); 5563 return 1; 5564 } 5565 5566 /* 5567 * Handle split lock. Depending on detection mode this will 5568 * either warn and disable split lock detection for this 5569 * task or force SIGBUS on it. 5570 */ 5571 if (handle_guest_split_lock(kvm_rip_read(vcpu))) 5572 return 1; 5573 fallthrough; 5574 default: 5575 kvm_run->exit_reason = KVM_EXIT_EXCEPTION; 5576 kvm_run->ex.exception = ex_no; 5577 kvm_run->ex.error_code = error_code; 5578 break; 5579 } 5580 return 0; 5581 } 5582 5583 static __always_inline int handle_external_interrupt(struct kvm_vcpu *vcpu) 5584 { 5585 ++vcpu->stat.irq_exits; 5586 return 1; 5587 } 5588 5589 static int handle_triple_fault(struct kvm_vcpu *vcpu) 5590 { 5591 vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN; 5592 vcpu->mmio_needed = 0; 5593 return 0; 5594 } 5595 5596 static int handle_io(struct kvm_vcpu *vcpu) 5597 { 5598 unsigned long exit_qualification; 5599 int size, in, string; 5600 unsigned port; 5601 5602 exit_qualification = vt_get_exit_qual(vcpu); 5603 string = (exit_qualification & 16) != 0; 5604 5605 ++vcpu->stat.io_exits; 5606 5607 if (string) 5608 return kvm_emulate_instruction(vcpu, 0); 5609 5610 port = exit_qualification >> 16; 5611 size = (exit_qualification & 7) + 1; 5612 in = (exit_qualification & 8) != 0; 5613 5614 return kvm_fast_pio(vcpu, size, port, in); 5615 } 5616 5617 void vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) 5618 { 5619 /* 5620 * Patch in the VMCALL instruction: 5621 */ 5622 hypercall[0] = 0x0f; 5623 hypercall[1] = 0x01; 5624 hypercall[2] = 0xc1; 5625 } 5626 5627 /* called to set cr0 as appropriate for a mov-to-cr0 exit. */ 5628 static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val) 5629 { 5630 if (is_guest_mode(vcpu)) { 5631 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5632 unsigned long orig_val = val; 5633 5634 /* 5635 * We get here when L2 changed cr0 in a way that did not change 5636 * any of L1's shadowed bits (see nested_vmx_exit_handled_cr), 5637 * but did change L0 shadowed bits. So we first calculate the 5638 * effective cr0 value that L1 would like to write into the 5639 * hardware. It consists of the L2-owned bits from the new 5640 * value combined with the L1-owned bits from L1's guest_cr0. 5641 */ 5642 val = (val & ~vmcs12->cr0_guest_host_mask) | 5643 (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask); 5644 5645 if (kvm_set_cr0(vcpu, val)) 5646 return 1; 5647 vmcs_writel(CR0_READ_SHADOW, orig_val); 5648 return 0; 5649 } else { 5650 return kvm_set_cr0(vcpu, val); 5651 } 5652 } 5653 5654 static int handle_set_cr4(struct kvm_vcpu *vcpu, unsigned long val) 5655 { 5656 if (is_guest_mode(vcpu)) { 5657 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5658 unsigned long orig_val = val; 5659 5660 /* analogously to handle_set_cr0 */ 5661 val = (val & ~vmcs12->cr4_guest_host_mask) | 5662 (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask); 5663 if (kvm_set_cr4(vcpu, val)) 5664 return 1; 5665 vmcs_writel(CR4_READ_SHADOW, orig_val); 5666 return 0; 5667 } else 5668 return kvm_set_cr4(vcpu, val); 5669 } 5670 5671 static int handle_desc(struct kvm_vcpu *vcpu) 5672 { 5673 /* 5674 * UMIP emulation relies on intercepting writes to CR4.UMIP, i.e. this 5675 * and other code needs to be updated if UMIP can be guest owned. 5676 */ 5677 BUILD_BUG_ON(KVM_POSSIBLE_CR4_GUEST_BITS & X86_CR4_UMIP); 5678 5679 WARN_ON_ONCE(!kvm_is_cr4_bit_set(vcpu, X86_CR4_UMIP)); 5680 return kvm_emulate_instruction(vcpu, 0); 5681 } 5682 5683 static int handle_cr(struct kvm_vcpu *vcpu) 5684 { 5685 unsigned long exit_qualification, val; 5686 int cr; 5687 int reg; 5688 int err; 5689 int ret; 5690 5691 exit_qualification = vt_get_exit_qual(vcpu); 5692 cr = exit_qualification & 15; 5693 reg = (exit_qualification >> 8) & 15; 5694 switch ((exit_qualification >> 4) & 3) { 5695 case 0: /* mov to cr */ 5696 val = kvm_register_read(vcpu, reg); 5697 trace_kvm_cr_write(cr, val); 5698 switch (cr) { 5699 case 0: 5700 err = handle_set_cr0(vcpu, val); 5701 return kvm_complete_insn_gp(vcpu, err); 5702 case 3: 5703 WARN_ON_ONCE(enable_unrestricted_guest); 5704 5705 err = kvm_set_cr3(vcpu, val); 5706 return kvm_complete_insn_gp(vcpu, err); 5707 case 4: 5708 err = handle_set_cr4(vcpu, val); 5709 return kvm_complete_insn_gp(vcpu, err); 5710 case 8: { 5711 u8 cr8_prev = kvm_get_cr8(vcpu); 5712 u8 cr8 = (u8)val; 5713 err = kvm_set_cr8(vcpu, cr8); 5714 ret = kvm_complete_insn_gp(vcpu, err); 5715 if (lapic_in_kernel(vcpu)) 5716 return ret; 5717 if (cr8_prev <= cr8) 5718 return ret; 5719 /* 5720 * TODO: we might be squashing a 5721 * KVM_GUESTDBG_SINGLESTEP-triggered 5722 * KVM_EXIT_DEBUG here. 5723 */ 5724 vcpu->run->exit_reason = KVM_EXIT_SET_TPR; 5725 return 0; 5726 } 5727 } 5728 break; 5729 case 2: /* clts */ 5730 KVM_BUG(1, vcpu->kvm, "Guest always owns CR0.TS"); 5731 return -EIO; 5732 case 1: /*mov from cr*/ 5733 switch (cr) { 5734 case 3: 5735 WARN_ON_ONCE(enable_unrestricted_guest); 5736 5737 val = kvm_read_cr3(vcpu); 5738 kvm_register_write(vcpu, reg, val); 5739 trace_kvm_cr_read(cr, val); 5740 return kvm_skip_emulated_instruction(vcpu); 5741 case 8: 5742 val = kvm_get_cr8(vcpu); 5743 kvm_register_write(vcpu, reg, val); 5744 trace_kvm_cr_read(cr, val); 5745 return kvm_skip_emulated_instruction(vcpu); 5746 } 5747 break; 5748 case 3: /* lmsw */ 5749 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f; 5750 trace_kvm_cr_write(0, (kvm_read_cr0_bits(vcpu, ~0xful) | val)); 5751 kvm_lmsw(vcpu, val); 5752 5753 return kvm_skip_emulated_instruction(vcpu); 5754 default: 5755 break; 5756 } 5757 vcpu->run->exit_reason = 0; 5758 vcpu_unimpl(vcpu, "unhandled control register: op %d cr %d\n", 5759 (int)(exit_qualification >> 4) & 3, cr); 5760 return 0; 5761 } 5762 5763 static int handle_dr(struct kvm_vcpu *vcpu) 5764 { 5765 unsigned long exit_qualification; 5766 int dr, dr7, reg; 5767 int err = 1; 5768 5769 exit_qualification = vt_get_exit_qual(vcpu); 5770 dr = exit_qualification & DEBUG_REG_ACCESS_NUM; 5771 5772 /* First, if DR does not exist, trigger UD */ 5773 if (!kvm_require_dr(vcpu, dr)) 5774 return 1; 5775 5776 dr7 = vmcs_readl(GUEST_DR7); 5777 if (dr7 & DR7_GD) { 5778 /* 5779 * As the vm-exit takes precedence over the debug trap, we 5780 * need to emulate the latter, either for the host or the 5781 * guest debugging itself. 5782 */ 5783 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) { 5784 vcpu->run->debug.arch.dr6 = DR6_BD | DR6_ACTIVE_LOW; 5785 vcpu->run->debug.arch.dr7 = dr7; 5786 vcpu->run->debug.arch.pc = kvm_get_linear_rip(vcpu); 5787 vcpu->run->debug.arch.exception = DB_VECTOR; 5788 vcpu->run->exit_reason = KVM_EXIT_DEBUG; 5789 return 0; 5790 } else { 5791 kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BD); 5792 return 1; 5793 } 5794 } 5795 5796 if (vmx_get_cpl(vcpu) > 0) 5797 goto out; 5798 5799 if (vcpu->guest_debug == 0) { 5800 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_MOV_DR_EXITING); 5801 5802 /* 5803 * No more DR vmexits; force a reload of the debug registers 5804 * and reenter on this instruction. The next vmexit will 5805 * retrieve the full state of the debug registers. 5806 */ 5807 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT; 5808 return 1; 5809 } 5810 5811 reg = DEBUG_REG_ACCESS_REG(exit_qualification); 5812 if (exit_qualification & TYPE_MOV_FROM_DR) { 5813 kvm_register_write(vcpu, reg, kvm_get_dr(vcpu, dr)); 5814 err = 0; 5815 } else { 5816 err = kvm_set_dr(vcpu, dr, kvm_register_read(vcpu, reg)); 5817 } 5818 5819 out: 5820 return kvm_complete_insn_gp(vcpu, err); 5821 } 5822 5823 void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu) 5824 { 5825 get_debugreg(vcpu->arch.db[0], 0); 5826 get_debugreg(vcpu->arch.db[1], 1); 5827 get_debugreg(vcpu->arch.db[2], 2); 5828 get_debugreg(vcpu->arch.db[3], 3); 5829 get_debugreg(vcpu->arch.dr6, 6); 5830 vcpu->arch.dr7 = vmcs_readl(GUEST_DR7); 5831 5832 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT; 5833 exec_controls_setbit(to_vmx(vcpu), CPU_BASED_MOV_DR_EXITING); 5834 5835 /* 5836 * exc_debug expects dr6 to be cleared after it runs, avoid that it sees 5837 * a stale dr6 from the guest. 5838 */ 5839 set_debugreg(DR6_RESERVED, 6); 5840 } 5841 5842 void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val) 5843 { 5844 vmcs_writel(GUEST_DR7, val); 5845 } 5846 5847 static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu) 5848 { 5849 kvm_apic_update_ppr(vcpu); 5850 return 1; 5851 } 5852 5853 static int handle_interrupt_window(struct kvm_vcpu *vcpu) 5854 { 5855 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING); 5856 5857 kvm_make_request(KVM_REQ_EVENT, vcpu); 5858 5859 ++vcpu->stat.irq_window_exits; 5860 return 1; 5861 } 5862 5863 static int handle_invlpg(struct kvm_vcpu *vcpu) 5864 { 5865 unsigned long exit_qualification = vt_get_exit_qual(vcpu); 5866 5867 kvm_mmu_invlpg(vcpu, exit_qualification); 5868 return kvm_skip_emulated_instruction(vcpu); 5869 } 5870 5871 static int handle_apic_access(struct kvm_vcpu *vcpu) 5872 { 5873 if (likely(fasteoi)) { 5874 unsigned long exit_qualification = vt_get_exit_qual(vcpu); 5875 int access_type, offset; 5876 5877 access_type = exit_qualification & APIC_ACCESS_TYPE; 5878 offset = exit_qualification & APIC_ACCESS_OFFSET; 5879 /* 5880 * Sane guest uses MOV to write EOI, with written value 5881 * not cared. So make a short-circuit here by avoiding 5882 * heavy instruction emulation. 5883 */ 5884 if ((access_type == TYPE_LINEAR_APIC_INST_WRITE) && 5885 (offset == APIC_EOI)) { 5886 kvm_lapic_set_eoi(vcpu); 5887 return kvm_skip_emulated_instruction(vcpu); 5888 } 5889 } 5890 return kvm_emulate_instruction(vcpu, 0); 5891 } 5892 5893 static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu) 5894 { 5895 unsigned long exit_qualification = vt_get_exit_qual(vcpu); 5896 5897 /* EOI-induced VM exit is trap-like and thus no need to adjust IP */ 5898 kvm_apic_set_eoi_accelerated(vcpu, exit_qualification & 0xff); 5899 return 1; 5900 } 5901 5902 static int handle_apic_write(struct kvm_vcpu *vcpu) 5903 { 5904 unsigned long exit_qualification = vt_get_exit_qual(vcpu); 5905 5906 /* 5907 * APIC-write VM-Exit is trap-like, KVM doesn't need to advance RIP and 5908 * hardware has done any necessary aliasing, offset adjustments, etc... 5909 * for the access. I.e. the correct value has already been written to 5910 * the vAPIC page for the correct 16-byte chunk. KVM needs only to 5911 * retrieve the register value and emulate the access. 5912 */ 5913 u32 offset = exit_qualification & 0xff0; 5914 5915 kvm_apic_write_nodecode(vcpu, offset); 5916 return 1; 5917 } 5918 5919 static int handle_task_switch(struct kvm_vcpu *vcpu) 5920 { 5921 struct vcpu_vmx *vmx = to_vmx(vcpu); 5922 unsigned long exit_qualification; 5923 bool has_error_code = false; 5924 u32 error_code = 0; 5925 u16 tss_selector; 5926 int reason, type, idt_v, idt_index; 5927 5928 idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK); 5929 idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK); 5930 type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK); 5931 5932 exit_qualification = vt_get_exit_qual(vcpu); 5933 5934 reason = (u32)exit_qualification >> 30; 5935 if (reason == TASK_SWITCH_GATE && idt_v) { 5936 switch (type) { 5937 case INTR_TYPE_NMI_INTR: 5938 vcpu->arch.nmi_injected = false; 5939 vmx_set_nmi_mask(vcpu, true); 5940 break; 5941 case INTR_TYPE_EXT_INTR: 5942 case INTR_TYPE_SOFT_INTR: 5943 kvm_clear_interrupt_queue(vcpu); 5944 break; 5945 case INTR_TYPE_HARD_EXCEPTION: 5946 if (vmx->idt_vectoring_info & 5947 VECTORING_INFO_DELIVER_CODE_MASK) { 5948 has_error_code = true; 5949 error_code = 5950 vmcs_read32(IDT_VECTORING_ERROR_CODE); 5951 } 5952 fallthrough; 5953 case INTR_TYPE_SOFT_EXCEPTION: 5954 kvm_clear_exception_queue(vcpu); 5955 break; 5956 default: 5957 break; 5958 } 5959 } 5960 tss_selector = exit_qualification; 5961 5962 if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION && 5963 type != INTR_TYPE_EXT_INTR && 5964 type != INTR_TYPE_NMI_INTR)) 5965 WARN_ON(!skip_emulated_instruction(vcpu)); 5966 5967 /* 5968 * TODO: What about debug traps on tss switch? 5969 * Are we supposed to inject them and update dr6? 5970 */ 5971 return kvm_task_switch(vcpu, tss_selector, 5972 type == INTR_TYPE_SOFT_INTR ? idt_index : -1, 5973 reason, has_error_code, error_code); 5974 } 5975 5976 static int handle_ept_violation(struct kvm_vcpu *vcpu) 5977 { 5978 unsigned long exit_qualification = vt_get_exit_qual(vcpu); 5979 gpa_t gpa; 5980 5981 /* 5982 * EPT violation happened while executing iret from NMI, 5983 * "blocked by NMI" bit has to be set before next VM entry. 5984 * There are errata that may cause this bit to not be set: 5985 * AAK134, BY25. 5986 */ 5987 if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) && 5988 enable_vnmi && 5989 (exit_qualification & INTR_INFO_UNBLOCK_NMI)) 5990 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI); 5991 5992 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS); 5993 trace_kvm_page_fault(vcpu, gpa, exit_qualification); 5994 5995 /* 5996 * Check that the GPA doesn't exceed physical memory limits, as that is 5997 * a guest page fault. We have to emulate the instruction here, because 5998 * if the illegal address is that of a paging structure, then 5999 * EPT_VIOLATION_ACC_WRITE bit is set. Alternatively, if supported we 6000 * would also use advanced VM-exit information for EPT violations to 6001 * reconstruct the page fault error code. 6002 */ 6003 if (unlikely(allow_smaller_maxphyaddr && !kvm_vcpu_is_legal_gpa(vcpu, gpa))) 6004 return kvm_emulate_instruction(vcpu, 0); 6005 6006 return __vt_handle_ept_violation(vcpu, gpa, exit_qualification); 6007 } 6008 6009 static int handle_ept_misconfig(struct kvm_vcpu *vcpu) 6010 { 6011 gpa_t gpa; 6012 6013 if (vmx_check_emulate_instruction(vcpu, EMULTYPE_PF, NULL, 0)) 6014 return 1; 6015 6016 /* 6017 * A nested guest cannot optimize MMIO vmexits, because we have an 6018 * nGPA here instead of the required GPA. 6019 */ 6020 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS); 6021 if (!is_guest_mode(vcpu) && 6022 !kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) { 6023 trace_kvm_fast_mmio(gpa); 6024 return kvm_skip_emulated_instruction(vcpu); 6025 } 6026 6027 return kvm_mmu_page_fault(vcpu, gpa, PFERR_RSVD_MASK, NULL, 0); 6028 } 6029 6030 static int handle_nmi_window(struct kvm_vcpu *vcpu) 6031 { 6032 if (KVM_BUG_ON(!enable_vnmi, vcpu->kvm)) 6033 return -EIO; 6034 6035 exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_NMI_WINDOW_EXITING); 6036 ++vcpu->stat.nmi_window_exits; 6037 kvm_make_request(KVM_REQ_EVENT, vcpu); 6038 6039 return 1; 6040 } 6041 6042 /* 6043 * Returns true if emulation is required (due to the vCPU having invalid state 6044 * with unsrestricted guest mode disabled) and KVM can't faithfully emulate the 6045 * current vCPU state. 6046 */ 6047 bool vmx_unhandleable_emulation_required(struct kvm_vcpu *vcpu) 6048 { 6049 struct vcpu_vmx *vmx = to_vmx(vcpu); 6050 6051 if (!vmx->vt.emulation_required) 6052 return false; 6053 6054 /* 6055 * It is architecturally impossible for emulation to be required when a 6056 * nested VM-Enter is pending completion, as VM-Enter will VM-Fail if 6057 * guest state is invalid and unrestricted guest is disabled, i.e. KVM 6058 * should synthesize VM-Fail instead emulation L2 code. This path is 6059 * only reachable if userspace modifies L2 guest state after KVM has 6060 * performed the nested VM-Enter consistency checks. 6061 */ 6062 if (vcpu->arch.nested_run_pending) 6063 return true; 6064 6065 /* 6066 * KVM only supports emulating exceptions if the vCPU is in Real Mode. 6067 * If emulation is required, KVM can't perform a successful VM-Enter to 6068 * inject the exception. 6069 */ 6070 return !vmx->rmode.vm86_active && 6071 (kvm_is_exception_pending(vcpu) || vcpu->arch.exception.injected); 6072 } 6073 6074 static int handle_invalid_guest_state(struct kvm_vcpu *vcpu) 6075 { 6076 struct vcpu_vmx *vmx = to_vmx(vcpu); 6077 bool intr_window_requested; 6078 unsigned count = 130; 6079 6080 intr_window_requested = exec_controls_get(vmx) & 6081 CPU_BASED_INTR_WINDOW_EXITING; 6082 6083 while (vmx->vt.emulation_required && count-- != 0) { 6084 if (intr_window_requested && !vmx_interrupt_blocked(vcpu)) 6085 return handle_interrupt_window(&vmx->vcpu); 6086 6087 if (kvm_test_request(KVM_REQ_EVENT, vcpu)) 6088 return 1; 6089 6090 /* 6091 * Ensure that any updates to kvm->buses[] observed by the 6092 * previous instruction (emulated or otherwise) are also 6093 * visible to the instruction KVM is about to emulate. 6094 */ 6095 smp_rmb(); 6096 6097 if (!kvm_emulate_instruction(vcpu, 0)) 6098 return 0; 6099 6100 if (vmx_unhandleable_emulation_required(vcpu)) { 6101 kvm_prepare_emulation_failure_exit(vcpu); 6102 return 0; 6103 } 6104 6105 if (vcpu->arch.halt_request) { 6106 vcpu->arch.halt_request = 0; 6107 return kvm_emulate_halt_noskip(vcpu); 6108 } 6109 6110 /* 6111 * Note, return 1 and not 0, vcpu_run() will invoke 6112 * xfer_to_guest_mode() which will create a proper return 6113 * code. 6114 */ 6115 if (__xfer_to_guest_mode_work_pending()) 6116 return 1; 6117 } 6118 6119 return 1; 6120 } 6121 6122 /* 6123 * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE 6124 * exiting, so only get here on cpu with PAUSE-Loop-Exiting. 6125 */ 6126 static int handle_pause(struct kvm_vcpu *vcpu) 6127 { 6128 if (!kvm_pause_in_guest(vcpu->kvm)) 6129 grow_ple_window(vcpu); 6130 6131 /* 6132 * Intel sdm vol3 ch-25.1.3 says: The "PAUSE-loop exiting" 6133 * VM-execution control is ignored if CPL > 0. OTOH, KVM 6134 * never set PAUSE_EXITING and just set PLE if supported, 6135 * so the vcpu must be CPL=0 if it gets a PAUSE exit. 6136 */ 6137 kvm_vcpu_on_spin(vcpu, true); 6138 return kvm_skip_emulated_instruction(vcpu); 6139 } 6140 6141 static int handle_monitor_trap(struct kvm_vcpu *vcpu) 6142 { 6143 return 1; 6144 } 6145 6146 static int handle_invpcid(struct kvm_vcpu *vcpu) 6147 { 6148 u32 vmx_instruction_info; 6149 unsigned long type; 6150 gva_t gva; 6151 struct { 6152 u64 pcid; 6153 u64 gla; 6154 } operand; 6155 int gpr_index; 6156 6157 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_INVPCID)) { 6158 kvm_queue_exception(vcpu, UD_VECTOR); 6159 return 1; 6160 } 6161 6162 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 6163 gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info); 6164 type = kvm_register_read(vcpu, gpr_index); 6165 6166 /* According to the Intel instruction reference, the memory operand 6167 * is read even if it isn't needed (e.g., for type==all) 6168 */ 6169 if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), 6170 vmx_instruction_info, false, 6171 sizeof(operand), &gva)) 6172 return 1; 6173 6174 return kvm_handle_invpcid(vcpu, type, gva); 6175 } 6176 6177 static int handle_pml_full(struct kvm_vcpu *vcpu) 6178 { 6179 unsigned long exit_qualification; 6180 6181 trace_kvm_pml_full(vcpu->vcpu_id); 6182 6183 exit_qualification = vt_get_exit_qual(vcpu); 6184 6185 /* 6186 * PML buffer FULL happened while executing iret from NMI, 6187 * "blocked by NMI" bit has to be set before next VM entry. 6188 */ 6189 if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) && 6190 enable_vnmi && 6191 (exit_qualification & INTR_INFO_UNBLOCK_NMI)) 6192 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 6193 GUEST_INTR_STATE_NMI); 6194 6195 /* 6196 * PML buffer already flushed at beginning of VMEXIT. Nothing to do 6197 * here.., and there's no userspace involvement needed for PML. 6198 */ 6199 return 1; 6200 } 6201 6202 static fastpath_t handle_fastpath_preemption_timer(struct kvm_vcpu *vcpu, 6203 bool force_immediate_exit) 6204 { 6205 struct vcpu_vmx *vmx = to_vmx(vcpu); 6206 6207 /* 6208 * In the *extremely* unlikely scenario that this is a spurious VM-Exit 6209 * due to the timer expiring while it was "soft" disabled, just eat the 6210 * exit and re-enter the guest. 6211 */ 6212 if (unlikely(vmx->loaded_vmcs->hv_timer_soft_disabled)) 6213 return EXIT_FASTPATH_REENTER_GUEST; 6214 6215 /* 6216 * If the timer expired because KVM used it to force an immediate exit, 6217 * then mission accomplished. 6218 */ 6219 if (force_immediate_exit) 6220 return EXIT_FASTPATH_EXIT_HANDLED; 6221 6222 /* 6223 * If L2 is active, go down the slow path as emulating the guest timer 6224 * expiration likely requires synthesizing a nested VM-Exit. 6225 */ 6226 if (is_guest_mode(vcpu)) 6227 return EXIT_FASTPATH_NONE; 6228 6229 kvm_lapic_expired_hv_timer(vcpu); 6230 return EXIT_FASTPATH_REENTER_GUEST; 6231 } 6232 6233 static int handle_preemption_timer(struct kvm_vcpu *vcpu) 6234 { 6235 /* 6236 * This non-fastpath handler is reached if and only if the preemption 6237 * timer was being used to emulate a guest timer while L2 is active. 6238 * All other scenarios are supposed to be handled in the fastpath. 6239 */ 6240 WARN_ON_ONCE(!is_guest_mode(vcpu)); 6241 kvm_lapic_expired_hv_timer(vcpu); 6242 return 1; 6243 } 6244 6245 /* 6246 * When nested=0, all VMX instruction VM Exits filter here. The handlers 6247 * are overwritten by nested_vmx_hardware_setup() when nested=1. 6248 */ 6249 static int handle_vmx_instruction(struct kvm_vcpu *vcpu) 6250 { 6251 kvm_queue_exception(vcpu, UD_VECTOR); 6252 return 1; 6253 } 6254 6255 static int handle_tdx_instruction(struct kvm_vcpu *vcpu) 6256 { 6257 kvm_queue_exception(vcpu, UD_VECTOR); 6258 return 1; 6259 } 6260 6261 #ifndef CONFIG_X86_SGX_KVM 6262 static int handle_encls(struct kvm_vcpu *vcpu) 6263 { 6264 /* 6265 * SGX virtualization is disabled. There is no software enable bit for 6266 * SGX, so KVM intercepts all ENCLS leafs and injects a #UD to prevent 6267 * the guest from executing ENCLS (when SGX is supported by hardware). 6268 */ 6269 kvm_queue_exception(vcpu, UD_VECTOR); 6270 return 1; 6271 } 6272 #endif /* CONFIG_X86_SGX_KVM */ 6273 6274 static int handle_notify(struct kvm_vcpu *vcpu) 6275 { 6276 unsigned long exit_qual = vt_get_exit_qual(vcpu); 6277 6278 /* 6279 * Notify VM exit happened while executing iret from NMI, 6280 * "blocked by NMI" bit has to be set before next VM entry. 6281 */ 6282 if (enable_vnmi && (exit_qual & INTR_INFO_UNBLOCK_NMI)) 6283 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 6284 GUEST_INTR_STATE_NMI); 6285 6286 return __vt_handle_notify(vcpu, exit_qual); 6287 } 6288 6289 static int vmx_get_msr_imm_reg(struct kvm_vcpu *vcpu) 6290 { 6291 return vmx_get_instr_info_reg(vmcs_read32(VMX_INSTRUCTION_INFO)); 6292 } 6293 6294 static int handle_rdmsr_imm(struct kvm_vcpu *vcpu) 6295 { 6296 return kvm_emulate_rdmsr_imm(vcpu, vt_get_exit_qual(vcpu), 6297 vmx_get_msr_imm_reg(vcpu)); 6298 } 6299 6300 static int handle_wrmsr_imm(struct kvm_vcpu *vcpu) 6301 { 6302 return kvm_emulate_wrmsr_imm(vcpu, vt_get_exit_qual(vcpu), 6303 vmx_get_msr_imm_reg(vcpu)); 6304 } 6305 6306 /* 6307 * The exit handlers return 1 if the exit was handled fully and guest execution 6308 * may resume. Otherwise they set the kvm_run parameter to indicate what needs 6309 * to be done to userspace and return 0. 6310 */ 6311 static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { 6312 [EXIT_REASON_EXCEPTION_NMI] = handle_exception_nmi, 6313 [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt, 6314 [EXIT_REASON_TRIPLE_FAULT] = handle_triple_fault, 6315 [EXIT_REASON_NMI_WINDOW] = handle_nmi_window, 6316 [EXIT_REASON_IO_INSTRUCTION] = handle_io, 6317 [EXIT_REASON_CR_ACCESS] = handle_cr, 6318 [EXIT_REASON_DR_ACCESS] = handle_dr, 6319 [EXIT_REASON_CPUID] = kvm_emulate_cpuid, 6320 [EXIT_REASON_MSR_READ] = kvm_emulate_rdmsr, 6321 [EXIT_REASON_MSR_WRITE] = kvm_emulate_wrmsr, 6322 [EXIT_REASON_INTERRUPT_WINDOW] = handle_interrupt_window, 6323 [EXIT_REASON_HLT] = kvm_emulate_halt, 6324 [EXIT_REASON_INVD] = kvm_emulate_invd, 6325 [EXIT_REASON_INVLPG] = handle_invlpg, 6326 [EXIT_REASON_RDPMC] = kvm_emulate_rdpmc, 6327 [EXIT_REASON_VMCALL] = kvm_emulate_hypercall, 6328 [EXIT_REASON_VMCLEAR] = handle_vmx_instruction, 6329 [EXIT_REASON_VMLAUNCH] = handle_vmx_instruction, 6330 [EXIT_REASON_VMPTRLD] = handle_vmx_instruction, 6331 [EXIT_REASON_VMPTRST] = handle_vmx_instruction, 6332 [EXIT_REASON_VMREAD] = handle_vmx_instruction, 6333 [EXIT_REASON_VMRESUME] = handle_vmx_instruction, 6334 [EXIT_REASON_VMWRITE] = handle_vmx_instruction, 6335 [EXIT_REASON_VMOFF] = handle_vmx_instruction, 6336 [EXIT_REASON_VMON] = handle_vmx_instruction, 6337 [EXIT_REASON_TPR_BELOW_THRESHOLD] = handle_tpr_below_threshold, 6338 [EXIT_REASON_APIC_ACCESS] = handle_apic_access, 6339 [EXIT_REASON_APIC_WRITE] = handle_apic_write, 6340 [EXIT_REASON_EOI_INDUCED] = handle_apic_eoi_induced, 6341 [EXIT_REASON_WBINVD] = kvm_emulate_wbinvd, 6342 [EXIT_REASON_XSETBV] = kvm_emulate_xsetbv, 6343 [EXIT_REASON_TASK_SWITCH] = handle_task_switch, 6344 [EXIT_REASON_MCE_DURING_VMENTRY] = handle_machine_check, 6345 [EXIT_REASON_GDTR_IDTR] = handle_desc, 6346 [EXIT_REASON_LDTR_TR] = handle_desc, 6347 [EXIT_REASON_EPT_VIOLATION] = handle_ept_violation, 6348 [EXIT_REASON_EPT_MISCONFIG] = handle_ept_misconfig, 6349 [EXIT_REASON_PAUSE_INSTRUCTION] = handle_pause, 6350 [EXIT_REASON_MWAIT_INSTRUCTION] = kvm_emulate_mwait, 6351 [EXIT_REASON_MONITOR_TRAP_FLAG] = handle_monitor_trap, 6352 [EXIT_REASON_MONITOR_INSTRUCTION] = kvm_emulate_monitor, 6353 [EXIT_REASON_INVEPT] = handle_vmx_instruction, 6354 [EXIT_REASON_INVVPID] = handle_vmx_instruction, 6355 [EXIT_REASON_RDRAND] = kvm_handle_invalid_op, 6356 [EXIT_REASON_RDSEED] = kvm_handle_invalid_op, 6357 [EXIT_REASON_PML_FULL] = handle_pml_full, 6358 [EXIT_REASON_INVPCID] = handle_invpcid, 6359 [EXIT_REASON_VMFUNC] = handle_vmx_instruction, 6360 [EXIT_REASON_PREEMPTION_TIMER] = handle_preemption_timer, 6361 [EXIT_REASON_ENCLS] = handle_encls, 6362 [EXIT_REASON_BUS_LOCK] = vt_handle_bus_lock_vmexit, 6363 [EXIT_REASON_NOTIFY] = handle_notify, 6364 [EXIT_REASON_SEAMCALL] = handle_tdx_instruction, 6365 [EXIT_REASON_TDCALL] = handle_tdx_instruction, 6366 [EXIT_REASON_MSR_READ_IMM] = handle_rdmsr_imm, 6367 [EXIT_REASON_MSR_WRITE_IMM] = handle_wrmsr_imm, 6368 }; 6369 6370 static const int kvm_vmx_max_exit_handlers = 6371 ARRAY_SIZE(kvm_vmx_exit_handlers); 6372 6373 static_assert(EXIT_REASON_UNDEFINED >= ARRAY_SIZE(kvm_vmx_exit_handlers)); 6374 6375 void vmx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, 6376 u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code) 6377 { 6378 struct vcpu_vmx *vmx = to_vmx(vcpu); 6379 6380 *reason = vmx->vt.exit_reason.full; 6381 *info1 = vt_get_exit_qual(vcpu); 6382 if (!(vmx->vt.exit_reason.failed_vmentry)) { 6383 *info2 = vmx->idt_vectoring_info; 6384 *intr_info = vt_get_intr_info(vcpu); 6385 if (is_exception_with_error_code(*intr_info)) 6386 *error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); 6387 else 6388 *error_code = 0; 6389 } else { 6390 *info2 = 0; 6391 *intr_info = 0; 6392 *error_code = 0; 6393 } 6394 } 6395 6396 void vmx_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info, u32 *error_code) 6397 { 6398 *intr_info = vmcs_read32(VM_ENTRY_INTR_INFO_FIELD); 6399 if (is_exception_with_error_code(*intr_info)) 6400 *error_code = vmcs_read32(VM_ENTRY_EXCEPTION_ERROR_CODE); 6401 else 6402 *error_code = 0; 6403 } 6404 6405 static void vmx_destroy_pml_buffer(struct kvm_vcpu *vcpu) 6406 { 6407 if (vcpu->arch.pml_page) { 6408 __free_page(vcpu->arch.pml_page); 6409 vcpu->arch.pml_page = NULL; 6410 } 6411 } 6412 6413 static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu) 6414 { 6415 u16 pml_idx; 6416 6417 pml_idx = vmcs_read16(GUEST_PML_INDEX); 6418 6419 /* Do nothing if PML buffer is empty */ 6420 if (pml_idx == PML_HEAD_INDEX) 6421 return; 6422 6423 kvm_flush_pml_buffer(vcpu, pml_idx); 6424 6425 /* reset PML index */ 6426 vmcs_write16(GUEST_PML_INDEX, PML_HEAD_INDEX); 6427 } 6428 6429 static void nested_vmx_mark_all_vmcs12_pages_dirty(struct kvm_vcpu *vcpu) 6430 { 6431 struct vcpu_vmx *vmx = to_vmx(vcpu); 6432 6433 kvm_vcpu_map_mark_dirty(vcpu, &vmx->nested.apic_access_page_map); 6434 kvm_vcpu_map_mark_dirty(vcpu, &vmx->nested.virtual_apic_map); 6435 kvm_vcpu_map_mark_dirty(vcpu, &vmx->nested.pi_desc_map); 6436 } 6437 6438 static void vmx_dump_sel(char *name, uint32_t sel) 6439 { 6440 pr_err("%s sel=0x%04x, attr=0x%05x, limit=0x%08x, base=0x%016lx\n", 6441 name, vmcs_read16(sel), 6442 vmcs_read32(sel + GUEST_ES_AR_BYTES - GUEST_ES_SELECTOR), 6443 vmcs_read32(sel + GUEST_ES_LIMIT - GUEST_ES_SELECTOR), 6444 vmcs_readl(sel + GUEST_ES_BASE - GUEST_ES_SELECTOR)); 6445 } 6446 6447 static void vmx_dump_dtsel(char *name, uint32_t limit) 6448 { 6449 pr_err("%s limit=0x%08x, base=0x%016lx\n", 6450 name, vmcs_read32(limit), 6451 vmcs_readl(limit + GUEST_GDTR_BASE - GUEST_GDTR_LIMIT)); 6452 } 6453 6454 static void vmx_dump_msrs(char *name, struct vmx_msrs *m) 6455 { 6456 unsigned int i; 6457 struct vmx_msr_entry *e; 6458 6459 pr_err("MSR %s:\n", name); 6460 for (i = 0, e = m->val; i < m->nr; ++i, ++e) 6461 pr_err(" %2d: msr=0x%08x value=0x%016llx\n", i, e->index, e->value); 6462 } 6463 6464 void dump_vmcs(struct kvm_vcpu *vcpu) 6465 { 6466 struct vcpu_vmx *vmx = to_vmx(vcpu); 6467 u32 vmentry_ctl, vmexit_ctl; 6468 u32 cpu_based_exec_ctrl, pin_based_exec_ctrl, secondary_exec_control; 6469 u64 tertiary_exec_control; 6470 unsigned long cr4; 6471 int efer_slot; 6472 6473 if (!dump_invalid_vmcs) { 6474 pr_warn_ratelimited("set kvm_intel.dump_invalid_vmcs=1 to dump internal KVM state.\n"); 6475 return; 6476 } 6477 6478 vmentry_ctl = vmcs_read32(VM_ENTRY_CONTROLS); 6479 vmexit_ctl = vmcs_read32(VM_EXIT_CONTROLS); 6480 cpu_based_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL); 6481 pin_based_exec_ctrl = vmcs_read32(PIN_BASED_VM_EXEC_CONTROL); 6482 cr4 = vmcs_readl(GUEST_CR4); 6483 6484 if (cpu_has_secondary_exec_ctrls()) 6485 secondary_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL); 6486 else 6487 secondary_exec_control = 0; 6488 6489 if (cpu_has_tertiary_exec_ctrls()) 6490 tertiary_exec_control = vmcs_read64(TERTIARY_VM_EXEC_CONTROL); 6491 else 6492 tertiary_exec_control = 0; 6493 6494 pr_err("VMCS %p, last attempted VM-entry on CPU %d\n", 6495 vmx->loaded_vmcs->vmcs, vcpu->arch.last_vmentry_cpu); 6496 pr_err("*** Guest State ***\n"); 6497 pr_err("CR0: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n", 6498 vmcs_readl(GUEST_CR0), vmcs_readl(CR0_READ_SHADOW), 6499 vmcs_readl(CR0_GUEST_HOST_MASK)); 6500 pr_err("CR4: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n", 6501 cr4, vmcs_readl(CR4_READ_SHADOW), vmcs_readl(CR4_GUEST_HOST_MASK)); 6502 pr_err("CR3 = 0x%016lx\n", vmcs_readl(GUEST_CR3)); 6503 if (cpu_has_vmx_ept()) { 6504 pr_err("PDPTR0 = 0x%016llx PDPTR1 = 0x%016llx\n", 6505 vmcs_read64(GUEST_PDPTR0), vmcs_read64(GUEST_PDPTR1)); 6506 pr_err("PDPTR2 = 0x%016llx PDPTR3 = 0x%016llx\n", 6507 vmcs_read64(GUEST_PDPTR2), vmcs_read64(GUEST_PDPTR3)); 6508 } 6509 pr_err("RSP = 0x%016lx RIP = 0x%016lx\n", 6510 vmcs_readl(GUEST_RSP), vmcs_readl(GUEST_RIP)); 6511 pr_err("RFLAGS=0x%08lx DR7 = 0x%016lx\n", 6512 vmcs_readl(GUEST_RFLAGS), vmcs_readl(GUEST_DR7)); 6513 pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n", 6514 vmcs_readl(GUEST_SYSENTER_ESP), 6515 vmcs_read32(GUEST_SYSENTER_CS), vmcs_readl(GUEST_SYSENTER_EIP)); 6516 vmx_dump_sel("CS: ", GUEST_CS_SELECTOR); 6517 vmx_dump_sel("DS: ", GUEST_DS_SELECTOR); 6518 vmx_dump_sel("SS: ", GUEST_SS_SELECTOR); 6519 vmx_dump_sel("ES: ", GUEST_ES_SELECTOR); 6520 vmx_dump_sel("FS: ", GUEST_FS_SELECTOR); 6521 vmx_dump_sel("GS: ", GUEST_GS_SELECTOR); 6522 vmx_dump_dtsel("GDTR:", GUEST_GDTR_LIMIT); 6523 vmx_dump_sel("LDTR:", GUEST_LDTR_SELECTOR); 6524 vmx_dump_dtsel("IDTR:", GUEST_IDTR_LIMIT); 6525 vmx_dump_sel("TR: ", GUEST_TR_SELECTOR); 6526 efer_slot = vmx_find_loadstore_msr_slot(&vmx->msr_autoload.guest, MSR_EFER); 6527 if (vmentry_ctl & VM_ENTRY_LOAD_IA32_EFER) 6528 pr_err("EFER= 0x%016llx\n", vmcs_read64(GUEST_IA32_EFER)); 6529 else if (efer_slot >= 0) 6530 pr_err("EFER= 0x%016llx (autoload)\n", 6531 vmx->msr_autoload.guest.val[efer_slot].value); 6532 else if (vmentry_ctl & VM_ENTRY_IA32E_MODE) 6533 pr_err("EFER= 0x%016llx (effective)\n", 6534 vcpu->arch.efer | (EFER_LMA | EFER_LME)); 6535 else 6536 pr_err("EFER= 0x%016llx (effective)\n", 6537 vcpu->arch.efer & ~(EFER_LMA | EFER_LME)); 6538 if (vmentry_ctl & VM_ENTRY_LOAD_IA32_PAT) 6539 pr_err("PAT = 0x%016llx\n", vmcs_read64(GUEST_IA32_PAT)); 6540 pr_err("DebugCtl = 0x%016llx DebugExceptions = 0x%016lx\n", 6541 vmcs_read64(GUEST_IA32_DEBUGCTL), 6542 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS)); 6543 if (cpu_has_load_perf_global_ctrl() && 6544 vmentry_ctl & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) 6545 pr_err("PerfGlobCtl = 0x%016llx\n", 6546 vmcs_read64(GUEST_IA32_PERF_GLOBAL_CTRL)); 6547 if (vmentry_ctl & VM_ENTRY_LOAD_BNDCFGS) 6548 pr_err("BndCfgS = 0x%016llx\n", vmcs_read64(GUEST_BNDCFGS)); 6549 pr_err("Interruptibility = %08x ActivityState = %08x\n", 6550 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO), 6551 vmcs_read32(GUEST_ACTIVITY_STATE)); 6552 if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) 6553 pr_err("InterruptStatus = %04x\n", 6554 vmcs_read16(GUEST_INTR_STATUS)); 6555 if (vmcs_read32(VM_ENTRY_MSR_LOAD_COUNT) > 0) 6556 vmx_dump_msrs("guest autoload", &vmx->msr_autoload.guest); 6557 if (vmcs_read32(VM_EXIT_MSR_STORE_COUNT) > 0) 6558 vmx_dump_msrs("autostore", &vmx->msr_autostore); 6559 6560 if (vmentry_ctl & VM_ENTRY_LOAD_CET_STATE) 6561 pr_err("S_CET = 0x%016lx, SSP = 0x%016lx, SSP TABLE = 0x%016lx\n", 6562 vmcs_readl(GUEST_S_CET), vmcs_readl(GUEST_SSP), 6563 vmcs_readl(GUEST_INTR_SSP_TABLE)); 6564 pr_err("*** Host State ***\n"); 6565 pr_err("RIP = 0x%016lx RSP = 0x%016lx\n", 6566 vmcs_readl(HOST_RIP), vmcs_readl(HOST_RSP)); 6567 pr_err("CS=%04x SS=%04x DS=%04x ES=%04x FS=%04x GS=%04x TR=%04x\n", 6568 vmcs_read16(HOST_CS_SELECTOR), vmcs_read16(HOST_SS_SELECTOR), 6569 vmcs_read16(HOST_DS_SELECTOR), vmcs_read16(HOST_ES_SELECTOR), 6570 vmcs_read16(HOST_FS_SELECTOR), vmcs_read16(HOST_GS_SELECTOR), 6571 vmcs_read16(HOST_TR_SELECTOR)); 6572 pr_err("FSBase=%016lx GSBase=%016lx TRBase=%016lx\n", 6573 vmcs_readl(HOST_FS_BASE), vmcs_readl(HOST_GS_BASE), 6574 vmcs_readl(HOST_TR_BASE)); 6575 pr_err("GDTBase=%016lx IDTBase=%016lx\n", 6576 vmcs_readl(HOST_GDTR_BASE), vmcs_readl(HOST_IDTR_BASE)); 6577 pr_err("CR0=%016lx CR3=%016lx CR4=%016lx\n", 6578 vmcs_readl(HOST_CR0), vmcs_readl(HOST_CR3), 6579 vmcs_readl(HOST_CR4)); 6580 pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n", 6581 vmcs_readl(HOST_IA32_SYSENTER_ESP), 6582 vmcs_read32(HOST_IA32_SYSENTER_CS), 6583 vmcs_readl(HOST_IA32_SYSENTER_EIP)); 6584 if (vmexit_ctl & VM_EXIT_LOAD_IA32_EFER) 6585 pr_err("EFER= 0x%016llx\n", vmcs_read64(HOST_IA32_EFER)); 6586 if (vmexit_ctl & VM_EXIT_LOAD_IA32_PAT) 6587 pr_err("PAT = 0x%016llx\n", vmcs_read64(HOST_IA32_PAT)); 6588 if (cpu_has_load_perf_global_ctrl() && 6589 vmexit_ctl & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) 6590 pr_err("PerfGlobCtl = 0x%016llx\n", 6591 vmcs_read64(HOST_IA32_PERF_GLOBAL_CTRL)); 6592 if (vmcs_read32(VM_EXIT_MSR_LOAD_COUNT) > 0) 6593 vmx_dump_msrs("host autoload", &vmx->msr_autoload.host); 6594 if (vmexit_ctl & VM_EXIT_LOAD_CET_STATE) 6595 pr_err("S_CET = 0x%016lx, SSP = 0x%016lx, SSP TABLE = 0x%016lx\n", 6596 vmcs_readl(HOST_S_CET), vmcs_readl(HOST_SSP), 6597 vmcs_readl(HOST_INTR_SSP_TABLE)); 6598 6599 pr_err("*** Control State ***\n"); 6600 pr_err("CPUBased=0x%08x SecondaryExec=0x%08x TertiaryExec=0x%016llx\n", 6601 cpu_based_exec_ctrl, secondary_exec_control, tertiary_exec_control); 6602 pr_err("PinBased=0x%08x EntryControls=%08x ExitControls=%08x\n", 6603 pin_based_exec_ctrl, vmentry_ctl, vmexit_ctl); 6604 pr_err("ExceptionBitmap=%08x PFECmask=%08x PFECmatch=%08x\n", 6605 vmcs_read32(EXCEPTION_BITMAP), 6606 vmcs_read32(PAGE_FAULT_ERROR_CODE_MASK), 6607 vmcs_read32(PAGE_FAULT_ERROR_CODE_MATCH)); 6608 pr_err("VMEntry: intr_info=%08x errcode=%08x ilen=%08x\n", 6609 vmcs_read32(VM_ENTRY_INTR_INFO_FIELD), 6610 vmcs_read32(VM_ENTRY_EXCEPTION_ERROR_CODE), 6611 vmcs_read32(VM_ENTRY_INSTRUCTION_LEN)); 6612 pr_err("VMExit: intr_info=%08x errcode=%08x ilen=%08x\n", 6613 vmcs_read32(VM_EXIT_INTR_INFO), 6614 vmcs_read32(VM_EXIT_INTR_ERROR_CODE), 6615 vmcs_read32(VM_EXIT_INSTRUCTION_LEN)); 6616 pr_err(" reason=%08x qualification=%016lx\n", 6617 vmcs_read32(VM_EXIT_REASON), vmcs_readl(EXIT_QUALIFICATION)); 6618 pr_err("IDTVectoring: info=%08x errcode=%08x\n", 6619 vmcs_read32(IDT_VECTORING_INFO_FIELD), 6620 vmcs_read32(IDT_VECTORING_ERROR_CODE)); 6621 pr_err("TSC Offset = 0x%016llx\n", vmcs_read64(TSC_OFFSET)); 6622 if (secondary_exec_control & SECONDARY_EXEC_TSC_SCALING) 6623 pr_err("TSC Multiplier = 0x%016llx\n", 6624 vmcs_read64(TSC_MULTIPLIER)); 6625 if (cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW) { 6626 if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) { 6627 u16 status = vmcs_read16(GUEST_INTR_STATUS); 6628 pr_err("SVI|RVI = %02x|%02x ", status >> 8, status & 0xff); 6629 } 6630 pr_cont("TPR Threshold = 0x%02x\n", vmcs_read32(TPR_THRESHOLD)); 6631 if (secondary_exec_control & SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) 6632 pr_err("APIC-access addr = 0x%016llx ", vmcs_read64(APIC_ACCESS_ADDR)); 6633 pr_cont("virt-APIC addr = 0x%016llx\n", vmcs_read64(VIRTUAL_APIC_PAGE_ADDR)); 6634 } 6635 if (pin_based_exec_ctrl & PIN_BASED_POSTED_INTR) 6636 pr_err("PostedIntrVec = 0x%02x\n", vmcs_read16(POSTED_INTR_NV)); 6637 if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT)) 6638 pr_err("EPT pointer = 0x%016llx\n", vmcs_read64(EPT_POINTER)); 6639 if (secondary_exec_control & SECONDARY_EXEC_PAUSE_LOOP_EXITING) 6640 pr_err("PLE Gap=%08x Window=%08x\n", 6641 vmcs_read32(PLE_GAP), vmcs_read32(PLE_WINDOW)); 6642 if (secondary_exec_control & SECONDARY_EXEC_ENABLE_VPID) 6643 pr_err("Virtual processor ID = 0x%04x\n", 6644 vmcs_read16(VIRTUAL_PROCESSOR_ID)); 6645 if (secondary_exec_control & SECONDARY_EXEC_EPT_VIOLATION_VE) { 6646 struct vmx_ve_information *ve_info = vmx->ve_info; 6647 u64 ve_info_pa = vmcs_read64(VE_INFORMATION_ADDRESS); 6648 6649 /* 6650 * If KVM is dumping the VMCS, then something has gone wrong 6651 * already. Derefencing an address from the VMCS, which could 6652 * very well be corrupted, is a terrible idea. The virtual 6653 * address is known so use it. 6654 */ 6655 pr_err("VE info address = 0x%016llx%s\n", ve_info_pa, 6656 ve_info_pa == __pa(ve_info) ? "" : "(corrupted!)"); 6657 pr_err("ve_info: 0x%08x 0x%08x 0x%016llx 0x%016llx 0x%016llx 0x%04x\n", 6658 ve_info->exit_reason, ve_info->delivery, 6659 ve_info->exit_qualification, 6660 ve_info->guest_linear_address, 6661 ve_info->guest_physical_address, ve_info->eptp_index); 6662 } 6663 } 6664 6665 /* 6666 * The guest has exited. See if we can fix it or if we need userspace 6667 * assistance. 6668 */ 6669 int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) 6670 { 6671 struct vcpu_vmx *vmx = to_vmx(vcpu); 6672 union vmx_exit_reason exit_reason = vt_get_exit_reason(vcpu); 6673 u32 vectoring_info = vmx->idt_vectoring_info; 6674 u16 exit_handler_index; 6675 6676 /* 6677 * Flush logged GPAs PML buffer, this will make dirty_bitmap more 6678 * updated. Another good is, in kvm_vm_ioctl_get_dirty_log, before 6679 * querying dirty_bitmap, we only need to kick all vcpus out of guest 6680 * mode as if vcpus is in root mode, the PML buffer must has been 6681 * flushed already. Note, PML is never enabled in hardware while 6682 * running L2. 6683 */ 6684 if (enable_pml && !is_guest_mode(vcpu)) 6685 vmx_flush_pml_buffer(vcpu); 6686 6687 if (unlikely(exit_fastpath == EXIT_FASTPATH_EXIT_USERSPACE)) 6688 return 0; 6689 6690 /* 6691 * KVM should never reach this point with a pending nested VM-Enter. 6692 * More specifically, short-circuiting VM-Entry to emulate L2 due to 6693 * invalid guest state should never happen as that means KVM knowingly 6694 * allowed a nested VM-Enter with an invalid vmcs12. More below. 6695 */ 6696 if (KVM_BUG_ON(vcpu->arch.nested_run_pending, vcpu->kvm)) 6697 return -EIO; 6698 6699 if (is_guest_mode(vcpu)) { 6700 /* 6701 * PML is never enabled when running L2, bail immediately if a 6702 * PML full exit occurs as something is horribly wrong. 6703 */ 6704 if (exit_reason.basic == EXIT_REASON_PML_FULL) 6705 goto unexpected_vmexit; 6706 6707 /* 6708 * The host physical addresses of some pages of guest memory 6709 * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC 6710 * Page). The CPU may write to these pages via their host 6711 * physical address while L2 is running, bypassing any 6712 * address-translation-based dirty tracking (e.g. EPT write 6713 * protection). 6714 * 6715 * Mark them dirty on every exit from L2 to prevent them from 6716 * getting out of sync with dirty tracking. 6717 */ 6718 nested_vmx_mark_all_vmcs12_pages_dirty(vcpu); 6719 6720 /* 6721 * Synthesize a triple fault if L2 state is invalid. In normal 6722 * operation, nested VM-Enter rejects any attempt to enter L2 6723 * with invalid state. However, those checks are skipped if 6724 * state is being stuffed via RSM or KVM_SET_NESTED_STATE. If 6725 * L2 state is invalid, it means either L1 modified SMRAM state 6726 * or userspace provided bad state. Synthesize TRIPLE_FAULT as 6727 * doing so is architecturally allowed in the RSM case, and is 6728 * the least awful solution for the userspace case without 6729 * risking false positives. 6730 */ 6731 if (vmx->vt.emulation_required) { 6732 nested_vmx_vmexit(vcpu, EXIT_REASON_TRIPLE_FAULT, 0, 0); 6733 return 1; 6734 } 6735 6736 if (nested_vmx_reflect_vmexit(vcpu)) 6737 return 1; 6738 } 6739 6740 /* If guest state is invalid, start emulating. L2 is handled above. */ 6741 if (vmx->vt.emulation_required) 6742 return handle_invalid_guest_state(vcpu); 6743 6744 if (exit_reason.failed_vmentry) { 6745 dump_vmcs(vcpu); 6746 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; 6747 vcpu->run->fail_entry.hardware_entry_failure_reason 6748 = exit_reason.full; 6749 vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu; 6750 return 0; 6751 } 6752 6753 if (unlikely(vmx->fail)) { 6754 dump_vmcs(vcpu); 6755 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; 6756 vcpu->run->fail_entry.hardware_entry_failure_reason 6757 = vmcs_read32(VM_INSTRUCTION_ERROR); 6758 vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu; 6759 return 0; 6760 } 6761 6762 if ((vectoring_info & VECTORING_INFO_VALID_MASK) && 6763 (exit_reason.basic != EXIT_REASON_EXCEPTION_NMI && 6764 exit_reason.basic != EXIT_REASON_EPT_VIOLATION && 6765 exit_reason.basic != EXIT_REASON_PML_FULL && 6766 exit_reason.basic != EXIT_REASON_APIC_ACCESS && 6767 exit_reason.basic != EXIT_REASON_TASK_SWITCH && 6768 exit_reason.basic != EXIT_REASON_NOTIFY && 6769 exit_reason.basic != EXIT_REASON_EPT_MISCONFIG)) { 6770 kvm_prepare_event_vectoring_exit(vcpu, INVALID_GPA); 6771 return 0; 6772 } 6773 6774 if (unlikely(!enable_vnmi && 6775 vmx->loaded_vmcs->soft_vnmi_blocked)) { 6776 if (!vmx_interrupt_blocked(vcpu)) { 6777 vmx->loaded_vmcs->soft_vnmi_blocked = 0; 6778 } else if (vmx->loaded_vmcs->vnmi_blocked_time > 1000000000LL && 6779 vcpu->arch.nmi_pending) { 6780 /* 6781 * This CPU don't support us in finding the end of an 6782 * NMI-blocked window if the guest runs with IRQs 6783 * disabled. So we pull the trigger after 1 s of 6784 * futile waiting, but inform the user about this. 6785 */ 6786 printk(KERN_WARNING "%s: Breaking out of NMI-blocked " 6787 "state on VCPU %d after 1 s timeout\n", 6788 __func__, vcpu->vcpu_id); 6789 vmx->loaded_vmcs->soft_vnmi_blocked = 0; 6790 } 6791 } 6792 6793 if (exit_fastpath != EXIT_FASTPATH_NONE) 6794 return 1; 6795 6796 if (exit_reason.basic >= kvm_vmx_max_exit_handlers) 6797 goto unexpected_vmexit; 6798 #ifdef CONFIG_MITIGATION_RETPOLINE 6799 if (exit_reason.basic == EXIT_REASON_MSR_WRITE) 6800 return kvm_emulate_wrmsr(vcpu); 6801 else if (exit_reason.basic == EXIT_REASON_MSR_WRITE_IMM) 6802 return handle_wrmsr_imm(vcpu); 6803 else if (exit_reason.basic == EXIT_REASON_PREEMPTION_TIMER) 6804 return handle_preemption_timer(vcpu); 6805 else if (exit_reason.basic == EXIT_REASON_INTERRUPT_WINDOW) 6806 return handle_interrupt_window(vcpu); 6807 else if (exit_reason.basic == EXIT_REASON_EXTERNAL_INTERRUPT) 6808 return handle_external_interrupt(vcpu); 6809 else if (exit_reason.basic == EXIT_REASON_HLT) 6810 return kvm_emulate_halt(vcpu); 6811 else if (exit_reason.basic == EXIT_REASON_EPT_MISCONFIG) 6812 return handle_ept_misconfig(vcpu); 6813 #endif 6814 6815 exit_handler_index = array_index_nospec((u16)exit_reason.basic, 6816 kvm_vmx_max_exit_handlers); 6817 if (!kvm_vmx_exit_handlers[exit_handler_index]) 6818 goto unexpected_vmexit; 6819 6820 return kvm_vmx_exit_handlers[exit_handler_index](vcpu); 6821 6822 unexpected_vmexit: 6823 dump_vmcs(vcpu); 6824 kvm_prepare_unexpected_reason_exit(vcpu, exit_reason.full); 6825 return 0; 6826 } 6827 6828 void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) 6829 { 6830 int tpr_threshold; 6831 6832 if (is_guest_mode(vcpu) && 6833 nested_cpu_has(get_vmcs12(vcpu), CPU_BASED_TPR_SHADOW)) 6834 return; 6835 6836 guard(vmx_vmcs01)(vcpu); 6837 6838 tpr_threshold = (irr == -1 || tpr < irr) ? 0 : irr; 6839 vmcs_write32(TPR_THRESHOLD, tpr_threshold); 6840 } 6841 6842 void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu) 6843 { 6844 struct vcpu_vmx *vmx = to_vmx(vcpu); 6845 u32 sec_exec_control; 6846 6847 if (!lapic_in_kernel(vcpu)) 6848 return; 6849 6850 if (!flexpriority_enabled && 6851 !cpu_has_vmx_virtualize_x2apic_mode()) 6852 return; 6853 6854 guard(vmx_vmcs01)(vcpu); 6855 6856 sec_exec_control = secondary_exec_controls_get(vmx); 6857 sec_exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 6858 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE); 6859 6860 switch (kvm_get_apic_mode(vcpu)) { 6861 case LAPIC_MODE_INVALID: 6862 WARN_ONCE(true, "Invalid local APIC state"); 6863 break; 6864 case LAPIC_MODE_DISABLED: 6865 break; 6866 case LAPIC_MODE_XAPIC: 6867 if (flexpriority_enabled) { 6868 sec_exec_control |= 6869 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 6870 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 6871 6872 /* 6873 * Flush the TLB, reloading the APIC access page will 6874 * only do so if its physical address has changed, but 6875 * the guest may have inserted a non-APIC mapping into 6876 * the TLB while the APIC access page was disabled. 6877 * 6878 * If L2 is active, immediately flush L1's TLB instead 6879 * of requesting a flush of the current TLB, because 6880 * the current TLB context is L2's. 6881 */ 6882 if (!is_guest_mode(vcpu)) 6883 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 6884 else if (!enable_ept) 6885 vpid_sync_context(vmx->vpid); 6886 else if (VALID_PAGE(vcpu->arch.root_mmu.root.hpa)) 6887 vmx_flush_tlb_ept_root(vcpu->arch.root_mmu.root.hpa); 6888 } 6889 break; 6890 case LAPIC_MODE_X2APIC: 6891 if (cpu_has_vmx_virtualize_x2apic_mode()) 6892 sec_exec_control |= 6893 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE; 6894 break; 6895 } 6896 secondary_exec_controls_set(vmx, sec_exec_control); 6897 6898 vmx_update_msr_bitmap_x2apic(vcpu); 6899 } 6900 6901 void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu) 6902 { 6903 const gfn_t gfn = APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT; 6904 struct kvm *kvm = vcpu->kvm; 6905 struct kvm_memslots *slots = kvm_memslots(kvm); 6906 struct kvm_memory_slot *slot; 6907 struct page *refcounted_page; 6908 unsigned long mmu_seq; 6909 kvm_pfn_t pfn; 6910 bool writable; 6911 6912 /* Note, the VIRTUALIZE_APIC_ACCESSES check needs to query vmcs01. */ 6913 guard(vmx_vmcs01)(vcpu); 6914 6915 if (!(secondary_exec_controls_get(to_vmx(vcpu)) & 6916 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) 6917 return; 6918 6919 /* 6920 * Explicitly grab the memslot using KVM's internal slot ID to ensure 6921 * KVM doesn't unintentionally grab a userspace memslot. It _should_ 6922 * be impossible for userspace to create a memslot for the APIC when 6923 * APICv is enabled, but paranoia won't hurt in this case. 6924 */ 6925 slot = id_to_memslot(slots, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT); 6926 if (!slot || slot->flags & KVM_MEMSLOT_INVALID) 6927 return; 6928 6929 /* 6930 * Ensure that the mmu_notifier sequence count is read before KVM 6931 * retrieves the pfn from the primary MMU. Note, the memslot is 6932 * protected by SRCU, not the mmu_notifier. Pairs with the smp_wmb() 6933 * in kvm_mmu_invalidate_end(). 6934 */ 6935 mmu_seq = kvm->mmu_invalidate_seq; 6936 smp_rmb(); 6937 6938 /* 6939 * No need to retry if the memslot does not exist or is invalid. KVM 6940 * controls the APIC-access page memslot, and only deletes the memslot 6941 * if APICv is permanently inhibited, i.e. the memslot won't reappear. 6942 */ 6943 pfn = __kvm_faultin_pfn(slot, gfn, FOLL_WRITE, &writable, &refcounted_page); 6944 if (is_error_noslot_pfn(pfn)) 6945 return; 6946 6947 read_lock(&vcpu->kvm->mmu_lock); 6948 if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) 6949 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 6950 else 6951 vmcs_write64(APIC_ACCESS_ADDR, pfn_to_hpa(pfn)); 6952 6953 /* 6954 * Do not pin the APIC access page in memory so that it can be freely 6955 * migrated, the MMU notifier will call us again if it is migrated or 6956 * swapped out. KVM backs the memslot with anonymous memory, the pfn 6957 * should always point at a refcounted page (if the pfn is valid). 6958 */ 6959 if (!WARN_ON_ONCE(!refcounted_page)) 6960 kvm_release_page_clean(refcounted_page); 6961 6962 /* 6963 * No need for a manual TLB flush at this point, KVM has already done a 6964 * flush if there were SPTEs pointing at the previous page. 6965 */ 6966 read_unlock(&vcpu->kvm->mmu_lock); 6967 } 6968 6969 void vmx_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr) 6970 { 6971 u16 status; 6972 u8 old; 6973 6974 if (max_isr == -1) 6975 max_isr = 0; 6976 6977 /* 6978 * Always update SVI in vmcs01, as SVI is only relevant for L2 if and 6979 * only if Virtual Interrupt Delivery is enabled in vmcs12, and if VID 6980 * is enabled then L2 EOIs affect L2's vAPIC, not L1's vAPIC. 6981 */ 6982 guard(vmx_vmcs01)(vcpu); 6983 6984 status = vmcs_read16(GUEST_INTR_STATUS); 6985 old = status >> 8; 6986 if (max_isr != old) { 6987 status &= 0xff; 6988 status |= max_isr << 8; 6989 vmcs_write16(GUEST_INTR_STATUS, status); 6990 } 6991 } 6992 6993 static void vmx_set_rvi(int vector) 6994 { 6995 u16 status; 6996 u8 old; 6997 6998 if (vector == -1) 6999 vector = 0; 7000 7001 status = vmcs_read16(GUEST_INTR_STATUS); 7002 old = (u8)status & 0xff; 7003 if ((u8)vector != old) { 7004 status &= ~0xff; 7005 status |= (u8)vector; 7006 vmcs_write16(GUEST_INTR_STATUS, status); 7007 } 7008 } 7009 7010 int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu) 7011 { 7012 struct vcpu_vt *vt = to_vt(vcpu); 7013 bool max_irr_is_from_pir; 7014 int max_irr; 7015 7016 if (KVM_BUG_ON(!enable_apicv, vcpu->kvm)) 7017 return -EIO; 7018 7019 if (pi_test_on(&vt->pi_desc)) { 7020 pi_clear_on(&vt->pi_desc); 7021 /* 7022 * IOMMU can write to PID.ON, so the barrier matters even on UP. 7023 * But on x86 this is just a compiler barrier anyway. 7024 */ 7025 smp_mb__after_atomic(); 7026 max_irr_is_from_pir = kvm_apic_update_irr(vcpu, vt->pi_desc.pir, 7027 &max_irr); 7028 } else { 7029 max_irr = kvm_lapic_find_highest_irr(vcpu); 7030 max_irr_is_from_pir = false; 7031 } 7032 7033 /* 7034 * If APICv is enabled and L2 is not active, then update the Requesting 7035 * Virtual Interrupt (RVI) portion of vmcs01.GUEST_INTR_STATUS with the 7036 * highest priority IRR to deliver the IRQ via Virtual Interrupt 7037 * Delivery. Note, this is required even if the highest priority IRQ 7038 * was already pending in the IRR, as RVI isn't updated in lockstep with 7039 * the IRR (unlike apic->irr_pending). 7040 * 7041 * For the cases where Virtual Interrupt Delivery can't be used: 7042 * 7043 * 1) If L2 is running and the vCPU has a new pending interrupt. If L1 7044 * wants to exit on interrupts, KVM_REQ_EVENT is needed to synthesize a 7045 * VM-Exit to L1. If L1 doesn't want to exit, the interrupt is injected 7046 * into L2, but KVM doesn't use virtual interrupt delivery to inject 7047 * interrupts into L2, and so KVM_REQ_EVENT is again needed. 7048 * 7049 * 2) If APICv is disabled for this vCPU, assigned devices may still 7050 * attempt to post interrupts. The posted interrupt vector will cause 7051 * a VM-Exit and the subsequent entry will call sync_pir_to_irr. 7052 * 7053 * In both cases, set KVM_REQ_EVENT if and only if the highest priority 7054 * pending IRQ came from the PIR, as setting KVM_REQ_EVENT if any IRQ 7055 * is pending may put the vCPU into an infinite loop, e.g. if the IRQ 7056 * is blocked, then it will stay pending until an IRQ window is opened. 7057 * 7058 * Note! It's possible that one or more IRQs were moved from the PIR 7059 * to the IRR _without_ max_irr_is_from_pir being true! I.e. if there 7060 * was a higher priority IRQ already pending in the IRR. Not setting 7061 * KVM_REQ_EVENT in this case is intentional and safe. If APICv is 7062 * inactive, or L2 is running with exit-on-interrupt off (in vmcs12), 7063 * i.e. without nested virtual interrupt delivery, then there's no need 7064 * to request an IRQ window as the lower priority IRQ only needs to be 7065 * delivered when the higher priority IRQ is dismissed from the ISR, 7066 * i.e. on the next EOI, and EOIs are always intercepted if APICv is 7067 * disabled or if L2 is running without nested VID. If L2 is running 7068 * exit-on-interrupt on (in vmcs12), then the higher priority IRQ will 7069 * trigger a nested VM-Exit, at which point KVM will re-evaluate L1's 7070 * pending IRQs. 7071 */ 7072 if (!is_guest_mode(vcpu) && kvm_vcpu_apicv_active(vcpu)) 7073 vmx_set_rvi(max_irr); 7074 else if (max_irr_is_from_pir) 7075 kvm_make_request(KVM_REQ_EVENT, vcpu); 7076 7077 return max_irr; 7078 } 7079 7080 void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) 7081 { 7082 if (!kvm_vcpu_apicv_active(vcpu)) 7083 return; 7084 7085 vmcs_write64(EOI_EXIT_BITMAP0, eoi_exit_bitmap[0]); 7086 vmcs_write64(EOI_EXIT_BITMAP1, eoi_exit_bitmap[1]); 7087 vmcs_write64(EOI_EXIT_BITMAP2, eoi_exit_bitmap[2]); 7088 vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]); 7089 } 7090 7091 /* 7092 * The kvm parameter can be NULL (module initialization, or invocation before 7093 * VM creation). Be sure to check the kvm parameter before using it. 7094 */ 7095 bool vmx_has_emulated_msr(struct kvm *kvm, u32 index) 7096 { 7097 switch (index) { 7098 case MSR_IA32_SMBASE: 7099 if (!IS_ENABLED(CONFIG_KVM_SMM)) 7100 return false; 7101 /* 7102 * We cannot do SMM unless we can run the guest in big 7103 * real mode. 7104 */ 7105 return enable_unrestricted_guest || emulate_invalid_guest_state; 7106 case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR: 7107 return nested; 7108 case MSR_AMD64_VIRT_SPEC_CTRL: 7109 case MSR_AMD64_TSC_RATIO: 7110 /* This is AMD only. */ 7111 return false; 7112 default: 7113 return true; 7114 } 7115 } 7116 7117 static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx) 7118 { 7119 u32 exit_intr_info; 7120 bool unblock_nmi; 7121 u8 vector; 7122 bool idtv_info_valid; 7123 7124 idtv_info_valid = vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK; 7125 7126 if (enable_vnmi) { 7127 if (vmx->loaded_vmcs->nmi_known_unmasked) 7128 return; 7129 7130 exit_intr_info = vt_get_intr_info(&vmx->vcpu); 7131 unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0; 7132 vector = exit_intr_info & INTR_INFO_VECTOR_MASK; 7133 /* 7134 * SDM 3: 27.7.1.2 (September 2008) 7135 * Re-set bit "block by NMI" before VM entry if vmexit caused by 7136 * a guest IRET fault. 7137 * SDM 3: 23.2.2 (September 2008) 7138 * Bit 12 is undefined in any of the following cases: 7139 * If the VM exit sets the valid bit in the IDT-vectoring 7140 * information field. 7141 * If the VM exit is due to a double fault. 7142 */ 7143 if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi && 7144 vector != DF_VECTOR && !idtv_info_valid) 7145 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, 7146 GUEST_INTR_STATE_NMI); 7147 else 7148 vmx->loaded_vmcs->nmi_known_unmasked = 7149 !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) 7150 & GUEST_INTR_STATE_NMI); 7151 } else if (unlikely(vmx->loaded_vmcs->soft_vnmi_blocked)) 7152 vmx->loaded_vmcs->vnmi_blocked_time += 7153 ktime_to_ns(ktime_sub(ktime_get(), 7154 vmx->loaded_vmcs->entry_time)); 7155 } 7156 7157 static void __vmx_complete_interrupts(struct kvm_vcpu *vcpu, 7158 u32 idt_vectoring_info, 7159 int instr_len_field, 7160 int error_code_field) 7161 { 7162 u8 vector; 7163 int type; 7164 bool idtv_info_valid; 7165 7166 idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK; 7167 7168 vcpu->arch.nmi_injected = false; 7169 kvm_clear_exception_queue(vcpu); 7170 kvm_clear_interrupt_queue(vcpu); 7171 7172 if (!idtv_info_valid) 7173 return; 7174 7175 kvm_make_request(KVM_REQ_EVENT, vcpu); 7176 7177 vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK; 7178 type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK; 7179 7180 switch (type) { 7181 case INTR_TYPE_NMI_INTR: 7182 vcpu->arch.nmi_injected = true; 7183 /* 7184 * SDM 3: 27.7.1.2 (September 2008) 7185 * Clear bit "block by NMI" before VM entry if a NMI 7186 * delivery faulted. 7187 */ 7188 vmx_set_nmi_mask(vcpu, false); 7189 break; 7190 case INTR_TYPE_SOFT_EXCEPTION: 7191 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field); 7192 fallthrough; 7193 case INTR_TYPE_HARD_EXCEPTION: { 7194 u32 error_code = 0; 7195 7196 if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) 7197 error_code = vmcs_read32(error_code_field); 7198 7199 kvm_requeue_exception(vcpu, vector, 7200 idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK, 7201 error_code); 7202 break; 7203 } 7204 case INTR_TYPE_SOFT_INTR: 7205 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field); 7206 fallthrough; 7207 case INTR_TYPE_EXT_INTR: 7208 kvm_queue_interrupt(vcpu, vector, type == INTR_TYPE_SOFT_INTR); 7209 break; 7210 default: 7211 break; 7212 } 7213 } 7214 7215 static void vmx_complete_interrupts(struct vcpu_vmx *vmx) 7216 { 7217 __vmx_complete_interrupts(&vmx->vcpu, vmx->idt_vectoring_info, 7218 VM_EXIT_INSTRUCTION_LEN, 7219 IDT_VECTORING_ERROR_CODE); 7220 } 7221 7222 void vmx_cancel_injection(struct kvm_vcpu *vcpu) 7223 { 7224 __vmx_complete_interrupts(vcpu, 7225 vmcs_read32(VM_ENTRY_INTR_INFO_FIELD), 7226 VM_ENTRY_INSTRUCTION_LEN, 7227 VM_ENTRY_EXCEPTION_ERROR_CODE); 7228 7229 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); 7230 } 7231 7232 static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx) 7233 { 7234 int i, nr_msrs; 7235 struct perf_guest_switch_msr *msrs; 7236 struct kvm_pmu *pmu = vcpu_to_pmu(&vmx->vcpu); 7237 7238 if (kvm_vcpu_has_mediated_pmu(&vmx->vcpu)) 7239 return; 7240 7241 struct x86_guest_pebs guest_pebs = { 7242 .enable = intel_pmu_compute_pebs_enable(pmu), 7243 .ds_area = pmu->ds_area, 7244 .data_cfg = pmu->pebs_data_cfg, 7245 }; 7246 7247 /* Note, nr_msrs may be garbage if perf_guest_get_msrs() returns NULL. */ 7248 msrs = perf_guest_get_msrs(&nr_msrs, &guest_pebs); 7249 if (!msrs) 7250 return; 7251 7252 for (i = 0; i < nr_msrs; i++) 7253 if (msrs[i].host == msrs[i].guest) 7254 clear_atomic_switch_msr(vmx, msrs[i].msr); 7255 else 7256 add_atomic_switch_msr(vmx, msrs[i].msr, msrs[i].guest, 7257 msrs[i].host); 7258 } 7259 7260 static void vmx_refresh_guest_perf_global_control(struct kvm_vcpu *vcpu) 7261 { 7262 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 7263 struct vcpu_vmx *vmx = to_vmx(vcpu); 7264 7265 if (msr_write_intercepted(vmx, MSR_CORE_PERF_GLOBAL_CTRL)) 7266 return; 7267 7268 if (!cpu_has_save_perf_global_ctrl()) { 7269 int slot = vmx_find_loadstore_msr_slot(&vmx->msr_autostore, 7270 MSR_CORE_PERF_GLOBAL_CTRL); 7271 7272 if (WARN_ON_ONCE(slot < 0)) 7273 return; 7274 7275 pmu->global_ctrl = vmx->msr_autostore.val[slot].value; 7276 vmcs_write64(GUEST_IA32_PERF_GLOBAL_CTRL, pmu->global_ctrl); 7277 return; 7278 } 7279 7280 pmu->global_ctrl = vmcs_read64(GUEST_IA32_PERF_GLOBAL_CTRL); 7281 } 7282 7283 void noinstr vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp) 7284 { 7285 if (unlikely(host_rsp != vmx->loaded_vmcs->host_state.rsp)) { 7286 vmx->loaded_vmcs->host_state.rsp = host_rsp; 7287 vmcs_writel(HOST_RSP, host_rsp); 7288 } 7289 } 7290 7291 static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu, 7292 bool force_immediate_exit) 7293 { 7294 /* 7295 * If L2 is active, some VMX preemption timer exits can be handled in 7296 * the fastpath even, all other exits must use the slow path. 7297 */ 7298 if (is_guest_mode(vcpu) && 7299 vt_get_exit_reason(vcpu).basic != EXIT_REASON_PREEMPTION_TIMER) 7300 return EXIT_FASTPATH_NONE; 7301 7302 switch (vt_get_exit_reason(vcpu).basic) { 7303 case EXIT_REASON_MSR_WRITE: 7304 return handle_fastpath_wrmsr(vcpu); 7305 case EXIT_REASON_MSR_WRITE_IMM: 7306 return handle_fastpath_wrmsr_imm(vcpu, vt_get_exit_qual(vcpu), 7307 vmx_get_msr_imm_reg(vcpu)); 7308 case EXIT_REASON_PREEMPTION_TIMER: 7309 return handle_fastpath_preemption_timer(vcpu, force_immediate_exit); 7310 case EXIT_REASON_HLT: 7311 return handle_fastpath_hlt(vcpu); 7312 case EXIT_REASON_INVD: 7313 return handle_fastpath_invd(vcpu); 7314 default: 7315 return EXIT_FASTPATH_NONE; 7316 } 7317 } 7318 7319 static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, 7320 unsigned int flags) 7321 { 7322 struct vcpu_vmx *vmx = to_vmx(vcpu); 7323 7324 guest_state_enter_irqoff(); 7325 7326 vmx_l1d_flush(vcpu); 7327 7328 vmx_disable_fb_clear(vmx); 7329 7330 if (vcpu->arch.cr2 != native_read_cr2()) 7331 native_write_cr2(vcpu->arch.cr2); 7332 7333 vmx->fail = __vmx_vcpu_run(vmx, flags); 7334 7335 vcpu->arch.cr2 = native_read_cr2(); 7336 kvm_clear_available_registers(vcpu, VMX_REGS_LAZY_LOAD_SET); 7337 7338 vmx->idt_vectoring_info = 0; 7339 7340 vmx_enable_fb_clear(vmx); 7341 7342 if (unlikely(vmx->fail)) { 7343 vmx->vt.exit_reason.full = EXIT_REASON_UNDEFINED; 7344 goto out; 7345 } 7346 7347 vmx->vt.exit_reason.full = vmcs_read32(VM_EXIT_REASON); 7348 if (likely(!vt_get_exit_reason(vcpu).failed_vmentry)) 7349 vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); 7350 7351 vt_handle_nmi(vcpu); 7352 7353 out: 7354 guest_state_exit_irqoff(); 7355 } 7356 7357 static void vmx_update_hv_timer(struct kvm_vcpu *vcpu, bool force_immediate_exit); 7358 7359 fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) 7360 { 7361 bool force_immediate_exit = run_flags & KVM_RUN_FORCE_IMMEDIATE_EXIT; 7362 struct vcpu_vmx *vmx = to_vmx(vcpu); 7363 unsigned long cr3, cr4; 7364 7365 /* Record the guest's net vcpu time for enforced NMI injections. */ 7366 if (unlikely(!enable_vnmi && 7367 vmx->loaded_vmcs->soft_vnmi_blocked)) 7368 vmx->loaded_vmcs->entry_time = ktime_get(); 7369 7370 /* 7371 * Don't enter VMX if guest state is invalid, let the exit handler 7372 * start emulation until we arrive back to a valid state. Synthesize a 7373 * consistency check VM-Exit due to invalid guest state and bail. 7374 */ 7375 if (unlikely(vmx->vt.emulation_required)) { 7376 vmx->fail = 0; 7377 7378 vmx->vt.exit_reason.full = EXIT_REASON_INVALID_STATE; 7379 vmx->vt.exit_reason.failed_vmentry = 1; 7380 kvm_register_mark_available(vcpu, VCPU_REG_EXIT_INFO_1); 7381 vmx->vt.exit_qualification = ENTRY_FAIL_DEFAULT; 7382 kvm_register_mark_available(vcpu, VCPU_REG_EXIT_INFO_2); 7383 vmx->vt.exit_intr_info = 0; 7384 return EXIT_FASTPATH_NONE; 7385 } 7386 7387 trace_kvm_entry(vcpu, force_immediate_exit); 7388 7389 if (vmx->ple_window_dirty) { 7390 vmx->ple_window_dirty = false; 7391 vmcs_write32(PLE_WINDOW, vmx->ple_window); 7392 } 7393 7394 /* 7395 * We did this in prepare_switch_to_guest, because it needs to 7396 * be within srcu_read_lock. 7397 */ 7398 WARN_ON_ONCE(vmx->nested.need_vmcs12_to_shadow_sync); 7399 7400 if (kvm_register_is_dirty(vcpu, VCPU_REGS_RSP)) 7401 vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]); 7402 if (kvm_register_is_dirty(vcpu, VCPU_REG_RIP)) 7403 vmcs_writel(GUEST_RIP, vcpu->arch.rip); 7404 kvm_reset_dirty_registers(vcpu); 7405 7406 if (run_flags & KVM_RUN_LOAD_GUEST_DR6) 7407 set_debugreg(vcpu->arch.dr6, 6); 7408 7409 if (run_flags & KVM_RUN_LOAD_DEBUGCTL) 7410 vmx_reload_guest_debugctl(vcpu); 7411 7412 /* 7413 * Refresh vmcs.HOST_CR3 if necessary. This must be done immediately 7414 * prior to VM-Enter, as the kernel may load a new ASID (PCID) any time 7415 * it switches back to the current->mm, which can occur in KVM context 7416 * when switching to a temporary mm to patch kernel code, e.g. if KVM 7417 * toggles a static key while handling a VM-Exit. 7418 */ 7419 cr3 = __get_current_cr3_fast(); 7420 if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) { 7421 vmcs_writel(HOST_CR3, cr3); 7422 vmx->loaded_vmcs->host_state.cr3 = cr3; 7423 } 7424 7425 cr4 = cr4_read_shadow(); 7426 if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) { 7427 vmcs_writel(HOST_CR4, cr4); 7428 vmx->loaded_vmcs->host_state.cr4 = cr4; 7429 } 7430 7431 /* When single-stepping over STI and MOV SS, we must clear the 7432 * corresponding interruptibility bits in the guest state. Otherwise 7433 * vmentry fails as it then expects bit 14 (BS) in pending debug 7434 * exceptions being set, but that's not correct for the guest debugging 7435 * case. */ 7436 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) 7437 vmx_set_interrupt_shadow(vcpu, 0); 7438 7439 pt_guest_enter(vmx); 7440 7441 atomic_switch_perf_msrs(vmx); 7442 if (intel_pmu_lbr_is_enabled(vcpu)) 7443 vmx_passthrough_lbr_msrs(vcpu); 7444 7445 if (enable_preemption_timer) 7446 vmx_update_hv_timer(vcpu, force_immediate_exit); 7447 else if (force_immediate_exit) 7448 smp_send_reschedule(vcpu->cpu); 7449 7450 kvm_wait_lapic_expire(vcpu); 7451 7452 /* The actual VMENTER/EXIT is in the .noinstr.text section. */ 7453 vmx_vcpu_enter_exit(vcpu, __vmx_vcpu_enter_flags(vmx)); 7454 7455 /* All fields are clean at this point */ 7456 if (kvm_is_using_evmcs()) { 7457 current_evmcs->hv_clean_fields |= 7458 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 7459 7460 current_evmcs->hv_vp_id = kvm_hv_get_vpindex(vcpu); 7461 } 7462 7463 /* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */ 7464 if (vcpu->arch.host_debugctl) 7465 update_debugctlmsr(vcpu->arch.host_debugctl); 7466 7467 #ifndef CONFIG_X86_64 7468 /* 7469 * The sysexit path does not restore ds/es, so we must set them to 7470 * a reasonable value ourselves. 7471 * 7472 * We can't defer this to vmx_prepare_switch_to_host() since that 7473 * function may be executed in interrupt context, which saves and 7474 * restore segments around it, nullifying its effect. 7475 */ 7476 loadsegment(ds, __USER_DS); 7477 loadsegment(es, __USER_DS); 7478 #endif 7479 7480 pt_guest_exit(vmx); 7481 7482 if (is_guest_mode(vcpu)) { 7483 /* 7484 * Track VMLAUNCH/VMRESUME that have made past guest state 7485 * checking. 7486 */ 7487 if (vcpu->arch.nested_run_pending && 7488 !vt_get_exit_reason(vcpu).failed_vmentry) 7489 ++vcpu->stat.nested_run; 7490 7491 vcpu->arch.nested_run_pending = 0; 7492 } 7493 7494 if (unlikely(vmx->fail)) 7495 return EXIT_FASTPATH_NONE; 7496 7497 trace_kvm_exit(vcpu, KVM_ISA_VMX); 7498 7499 if (unlikely(vt_get_exit_reason(vcpu).failed_vmentry)) 7500 return EXIT_FASTPATH_NONE; 7501 7502 vmx->loaded_vmcs->launched = 1; 7503 7504 vmx_refresh_guest_perf_global_control(vcpu); 7505 7506 vmx_recover_nmi_blocking(vmx); 7507 vmx_complete_interrupts(vmx); 7508 7509 return vmx_exit_handlers_fastpath(vcpu, force_immediate_exit); 7510 } 7511 7512 void vmx_vcpu_free(struct kvm_vcpu *vcpu) 7513 { 7514 struct vcpu_vmx *vmx = to_vmx(vcpu); 7515 7516 if (enable_pml) 7517 vmx_destroy_pml_buffer(vcpu); 7518 free_vpid(vmx->vpid); 7519 nested_vmx_free_vcpu(vcpu); 7520 free_loaded_vmcs(vmx->loaded_vmcs); 7521 free_page((unsigned long)vmx->ve_info); 7522 7523 if (vmx_can_use_ipiv(vcpu)) 7524 WRITE_ONCE(to_kvm_vmx(vcpu->kvm)->pid_table[vcpu->vcpu_id], 0); 7525 } 7526 7527 int vmx_vcpu_create(struct kvm_vcpu *vcpu) 7528 { 7529 struct vmx_uret_msr *tsx_ctrl; 7530 struct vcpu_vmx *vmx; 7531 int i, err; 7532 7533 BUILD_BUG_ON(offsetof(struct vcpu_vmx, vcpu) != 0); 7534 vmx = to_vmx(vcpu); 7535 7536 INIT_LIST_HEAD(&vmx->vt.pi_wakeup_list); 7537 7538 err = -ENOMEM; 7539 7540 vmx->vpid = allocate_vpid(); 7541 7542 /* 7543 * If PML is turned on, failure on enabling PML just results in failure 7544 * of creating the vcpu, therefore we can simplify PML logic (by 7545 * avoiding dealing with cases, such as enabling PML partially on vcpus 7546 * for the guest), etc. 7547 */ 7548 if (enable_pml) { 7549 vcpu->arch.pml_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 7550 if (!vcpu->arch.pml_page) 7551 goto free_vpid; 7552 } 7553 7554 for (i = 0; i < kvm_nr_uret_msrs; ++i) 7555 vmx->guest_uret_msrs[i].mask = -1ull; 7556 if (boot_cpu_has(X86_FEATURE_RTM)) { 7557 /* 7558 * TSX_CTRL_CPUID_CLEAR is handled in the CPUID interception. 7559 * Keep the host value unchanged to avoid changing CPUID bits 7560 * under the host kernel's feet. 7561 */ 7562 tsx_ctrl = vmx_find_uret_msr(vmx, MSR_IA32_TSX_CTRL); 7563 if (tsx_ctrl) 7564 tsx_ctrl->mask = ~(u64)TSX_CTRL_CPUID_CLEAR; 7565 } 7566 7567 err = alloc_loaded_vmcs(&vmx->vmcs01); 7568 if (err < 0) 7569 goto free_pml; 7570 7571 /* 7572 * Use Hyper-V 'Enlightened MSR Bitmap' feature when KVM runs as a 7573 * nested (L1) hypervisor and Hyper-V in L0 supports it. Enable the 7574 * feature only for vmcs01, KVM currently isn't equipped to realize any 7575 * performance benefits from enabling it for vmcs02. 7576 */ 7577 if (kvm_is_using_evmcs() && 7578 (ms_hyperv.nested_features & HV_X64_NESTED_MSR_BITMAP)) { 7579 struct hv_enlightened_vmcs *evmcs = (void *)vmx->vmcs01.vmcs; 7580 7581 evmcs->hv_enlightenments_control.msr_bitmap = 1; 7582 } 7583 7584 vmx->loaded_vmcs = &vmx->vmcs01; 7585 7586 if (cpu_need_virtualize_apic_accesses(vcpu)) { 7587 err = kvm_alloc_apic_access_page(vcpu->kvm); 7588 if (err) 7589 goto free_vmcs; 7590 } 7591 7592 if (enable_ept && !enable_unrestricted_guest) { 7593 err = init_rmode_identity_map(vcpu->kvm); 7594 if (err) 7595 goto free_vmcs; 7596 } 7597 7598 err = -ENOMEM; 7599 if (vmcs_config.cpu_based_2nd_exec_ctrl & SECONDARY_EXEC_EPT_VIOLATION_VE) { 7600 struct page *page; 7601 7602 BUILD_BUG_ON(sizeof(*vmx->ve_info) > PAGE_SIZE); 7603 7604 /* ve_info must be page aligned. */ 7605 page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 7606 if (!page) 7607 goto free_vmcs; 7608 7609 vmx->ve_info = page_to_virt(page); 7610 } 7611 7612 if (vmx_can_use_ipiv(vcpu)) 7613 WRITE_ONCE(to_kvm_vmx(vcpu->kvm)->pid_table[vcpu->vcpu_id], 7614 __pa(&vmx->vt.pi_desc) | PID_TABLE_ENTRY_VALID); 7615 7616 return 0; 7617 7618 free_vmcs: 7619 free_loaded_vmcs(vmx->loaded_vmcs); 7620 free_pml: 7621 vmx_destroy_pml_buffer(vcpu); 7622 free_vpid: 7623 free_vpid(vmx->vpid); 7624 return err; 7625 } 7626 7627 #define L1TF_MSG_SMT "L1TF CPU bug present and SMT on, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n" 7628 #define L1TF_MSG_L1D "L1TF CPU bug present and virtualization mitigation disabled, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n" 7629 7630 int vmx_vm_init(struct kvm *kvm) 7631 { 7632 if (!ple_gap) 7633 kvm_disable_exits(kvm, KVM_X86_DISABLE_EXITS_PAUSE); 7634 7635 if (boot_cpu_has(X86_BUG_L1TF) && enable_ept) { 7636 switch (l1tf_mitigation) { 7637 case L1TF_MITIGATION_OFF: 7638 case L1TF_MITIGATION_FLUSH_NOWARN: 7639 /* 'I explicitly don't care' is set */ 7640 break; 7641 case L1TF_MITIGATION_AUTO: 7642 case L1TF_MITIGATION_FLUSH: 7643 case L1TF_MITIGATION_FLUSH_NOSMT: 7644 case L1TF_MITIGATION_FULL: 7645 /* 7646 * Warn upon starting the first VM in a potentially 7647 * insecure environment. 7648 */ 7649 if (sched_smt_active()) 7650 pr_warn_once(L1TF_MSG_SMT); 7651 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER) 7652 pr_warn_once(L1TF_MSG_L1D); 7653 break; 7654 case L1TF_MITIGATION_FULL_FORCE: 7655 /* Flush is enforced */ 7656 break; 7657 } 7658 } 7659 7660 if (enable_pml) 7661 kvm->arch.cpu_dirty_log_size = PML_LOG_NR_ENTRIES; 7662 return 0; 7663 } 7664 7665 static inline bool vmx_ignore_guest_pat(struct kvm *kvm) 7666 { 7667 /* 7668 * Non-coherent DMA devices need the guest to flush CPU properly. 7669 * In that case it is not possible to map all guest RAM as WB, so 7670 * always trust guest PAT. 7671 */ 7672 return !kvm_arch_has_noncoherent_dma(kvm) && 7673 kvm_check_has_quirk(kvm, KVM_X86_QUIRK_IGNORE_GUEST_PAT); 7674 } 7675 7676 u8 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio) 7677 { 7678 /* 7679 * Force UC for host MMIO regions, as allowing the guest to access MMIO 7680 * with cacheable accesses will result in Machine Checks. 7681 */ 7682 if (is_mmio) 7683 return MTRR_TYPE_UNCACHABLE << VMX_EPT_MT_EPTE_SHIFT; 7684 7685 /* Force WB if ignoring guest PAT */ 7686 if (vmx_ignore_guest_pat(vcpu->kvm)) 7687 return (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT) | VMX_EPT_IPAT_BIT; 7688 7689 return (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT); 7690 } 7691 7692 static void vmcs_set_secondary_exec_control(struct vcpu_vmx *vmx, u32 new_ctl) 7693 { 7694 /* 7695 * These bits in the secondary execution controls field 7696 * are dynamic, the others are mostly based on the hypervisor 7697 * architecture and the guest's CPUID. Do not touch the 7698 * dynamic bits. 7699 */ 7700 u32 mask = 7701 SECONDARY_EXEC_SHADOW_VMCS | 7702 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 7703 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 7704 SECONDARY_EXEC_DESC; 7705 7706 u32 cur_ctl = secondary_exec_controls_get(vmx); 7707 7708 secondary_exec_controls_set(vmx, (new_ctl & ~mask) | (cur_ctl & mask)); 7709 } 7710 7711 /* 7712 * Generate MSR_IA32_VMX_CR{0,4}_FIXED1 according to CPUID. Only set bits 7713 * (indicating "allowed-1") if they are supported in the guest's CPUID. 7714 */ 7715 static void nested_vmx_cr_fixed1_bits_update(struct kvm_vcpu *vcpu) 7716 { 7717 struct vcpu_vmx *vmx = to_vmx(vcpu); 7718 struct kvm_cpuid_entry2 *entry; 7719 7720 vmx->nested.msrs.cr0_fixed1 = 0xffffffff; 7721 vmx->nested.msrs.cr4_fixed1 = X86_CR4_PCE; 7722 7723 #define cr4_fixed1_update(_cr4_mask, _reg, _cpuid_mask) do { \ 7724 if (entry && (entry->_reg & (_cpuid_mask))) \ 7725 vmx->nested.msrs.cr4_fixed1 |= (_cr4_mask); \ 7726 } while (0) 7727 7728 entry = kvm_find_cpuid_entry(vcpu, 0x1); 7729 cr4_fixed1_update(X86_CR4_VME, edx, feature_bit(VME)); 7730 cr4_fixed1_update(X86_CR4_PVI, edx, feature_bit(VME)); 7731 cr4_fixed1_update(X86_CR4_TSD, edx, feature_bit(TSC)); 7732 cr4_fixed1_update(X86_CR4_DE, edx, feature_bit(DE)); 7733 cr4_fixed1_update(X86_CR4_PSE, edx, feature_bit(PSE)); 7734 cr4_fixed1_update(X86_CR4_PAE, edx, feature_bit(PAE)); 7735 cr4_fixed1_update(X86_CR4_MCE, edx, feature_bit(MCE)); 7736 cr4_fixed1_update(X86_CR4_PGE, edx, feature_bit(PGE)); 7737 cr4_fixed1_update(X86_CR4_OSFXSR, edx, feature_bit(FXSR)); 7738 cr4_fixed1_update(X86_CR4_OSXMMEXCPT, edx, feature_bit(XMM)); 7739 cr4_fixed1_update(X86_CR4_VMXE, ecx, feature_bit(VMX)); 7740 cr4_fixed1_update(X86_CR4_SMXE, ecx, feature_bit(SMX)); 7741 cr4_fixed1_update(X86_CR4_PCIDE, ecx, feature_bit(PCID)); 7742 cr4_fixed1_update(X86_CR4_OSXSAVE, ecx, feature_bit(XSAVE)); 7743 7744 entry = kvm_find_cpuid_entry_index(vcpu, 0x7, 0); 7745 cr4_fixed1_update(X86_CR4_FSGSBASE, ebx, feature_bit(FSGSBASE)); 7746 cr4_fixed1_update(X86_CR4_SMEP, ebx, feature_bit(SMEP)); 7747 cr4_fixed1_update(X86_CR4_SMAP, ebx, feature_bit(SMAP)); 7748 cr4_fixed1_update(X86_CR4_PKE, ecx, feature_bit(PKU)); 7749 cr4_fixed1_update(X86_CR4_UMIP, ecx, feature_bit(UMIP)); 7750 cr4_fixed1_update(X86_CR4_LA57, ecx, feature_bit(LA57)); 7751 cr4_fixed1_update(X86_CR4_CET, ecx, feature_bit(SHSTK)); 7752 cr4_fixed1_update(X86_CR4_CET, edx, feature_bit(IBT)); 7753 7754 entry = kvm_find_cpuid_entry_index(vcpu, 0x7, 1); 7755 cr4_fixed1_update(X86_CR4_LAM_SUP, eax, feature_bit(LAM)); 7756 7757 #undef cr4_fixed1_update 7758 } 7759 7760 static void update_intel_pt_cfg(struct kvm_vcpu *vcpu) 7761 { 7762 struct vcpu_vmx *vmx = to_vmx(vcpu); 7763 struct kvm_cpuid_entry2 *best = NULL; 7764 int i; 7765 7766 for (i = 0; i < PT_CPUID_LEAVES; i++) { 7767 best = kvm_find_cpuid_entry_index(vcpu, 0x14, i); 7768 if (!best) 7769 return; 7770 vmx->pt_desc.caps[CPUID_EAX + i*PT_CPUID_REGS_NUM] = best->eax; 7771 vmx->pt_desc.caps[CPUID_EBX + i*PT_CPUID_REGS_NUM] = best->ebx; 7772 vmx->pt_desc.caps[CPUID_ECX + i*PT_CPUID_REGS_NUM] = best->ecx; 7773 vmx->pt_desc.caps[CPUID_EDX + i*PT_CPUID_REGS_NUM] = best->edx; 7774 } 7775 7776 /* Get the number of configurable Address Ranges for filtering */ 7777 vmx->pt_desc.num_address_ranges = intel_pt_validate_cap(vmx->pt_desc.caps, 7778 PT_CAP_num_address_ranges); 7779 7780 /* Initialize and clear the no dependency bits */ 7781 vmx->pt_desc.ctl_bitmask = ~(RTIT_CTL_TRACEEN | RTIT_CTL_OS | 7782 RTIT_CTL_USR | RTIT_CTL_TSC_EN | RTIT_CTL_DISRETC | 7783 RTIT_CTL_BRANCH_EN); 7784 7785 /* 7786 * If CPUID.(EAX=14H,ECX=0):EBX[0]=1 CR3Filter can be set otherwise 7787 * will inject an #GP 7788 */ 7789 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_cr3_filtering)) 7790 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_CR3EN; 7791 7792 /* 7793 * If CPUID.(EAX=14H,ECX=0):EBX[1]=1 CYCEn, CycThresh and 7794 * PSBFreq can be set 7795 */ 7796 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc)) 7797 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_CYCLEACC | 7798 RTIT_CTL_CYC_THRESH | RTIT_CTL_PSB_FREQ); 7799 7800 /* 7801 * If CPUID.(EAX=14H,ECX=0):EBX[3]=1 MTCEn and MTCFreq can be set 7802 */ 7803 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc)) 7804 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_MTC_EN | 7805 RTIT_CTL_MTC_RANGE); 7806 7807 /* If CPUID.(EAX=14H,ECX=0):EBX[4]=1 FUPonPTW and PTWEn can be set */ 7808 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_ptwrite)) 7809 vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_FUP_ON_PTW | 7810 RTIT_CTL_PTW_EN); 7811 7812 /* If CPUID.(EAX=14H,ECX=0):EBX[5]=1 PwrEvEn can be set */ 7813 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_power_event_trace)) 7814 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_PWR_EVT_EN; 7815 7816 /* If CPUID.(EAX=14H,ECX=0):ECX[0]=1 ToPA can be set */ 7817 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_topa_output)) 7818 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_TOPA; 7819 7820 /* If CPUID.(EAX=14H,ECX=0):ECX[3]=1 FabricEn can be set */ 7821 if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_output_subsys)) 7822 vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_FABRIC_EN; 7823 7824 /* unmask address range configure area */ 7825 for (i = 0; i < vmx->pt_desc.num_address_ranges; i++) 7826 vmx->pt_desc.ctl_bitmask &= ~(0xfULL << (32 + i * 4)); 7827 } 7828 7829 void vmx_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) 7830 { 7831 struct vcpu_vmx *vmx = to_vmx(vcpu); 7832 7833 /* 7834 * XSAVES is effectively enabled if and only if XSAVE is also exposed 7835 * to the guest. XSAVES depends on CR4.OSXSAVE, and CR4.OSXSAVE can be 7836 * set if and only if XSAVE is supported. 7837 */ 7838 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_XSAVE)) 7839 guest_cpu_cap_clear(vcpu, X86_FEATURE_XSAVES); 7840 7841 vmx_setup_uret_msrs(vmx); 7842 7843 if (cpu_has_secondary_exec_ctrls()) 7844 vmcs_set_secondary_exec_control(vmx, 7845 vmx_secondary_exec_control(vmx)); 7846 7847 if (guest_cpu_cap_has(vcpu, X86_FEATURE_VMX)) 7848 vmx->msr_ia32_feature_control_valid_bits |= 7849 FEAT_CTL_VMX_ENABLED_INSIDE_SMX | 7850 FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX; 7851 else 7852 vmx->msr_ia32_feature_control_valid_bits &= 7853 ~(FEAT_CTL_VMX_ENABLED_INSIDE_SMX | 7854 FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX); 7855 7856 if (guest_cpu_cap_has(vcpu, X86_FEATURE_VMX)) 7857 nested_vmx_cr_fixed1_bits_update(vcpu); 7858 7859 if (boot_cpu_has(X86_FEATURE_INTEL_PT) && 7860 guest_cpu_cap_has(vcpu, X86_FEATURE_INTEL_PT)) 7861 update_intel_pt_cfg(vcpu); 7862 7863 if (boot_cpu_has(X86_FEATURE_RTM)) { 7864 struct vmx_uret_msr *msr; 7865 msr = vmx_find_uret_msr(vmx, MSR_IA32_TSX_CTRL); 7866 if (msr) { 7867 bool enabled = guest_cpu_cap_has(vcpu, X86_FEATURE_RTM); 7868 vmx_set_guest_uret_msr(vmx, msr, enabled ? 0 : TSX_CTRL_RTM_DISABLE); 7869 } 7870 } 7871 7872 set_cr4_guest_host_mask(vmx); 7873 7874 vmx_write_encls_bitmap(vcpu, NULL); 7875 if (guest_cpu_cap_has(vcpu, X86_FEATURE_SGX)) 7876 vmx->msr_ia32_feature_control_valid_bits |= FEAT_CTL_SGX_ENABLED; 7877 else 7878 vmx->msr_ia32_feature_control_valid_bits &= ~FEAT_CTL_SGX_ENABLED; 7879 7880 if (guest_cpu_cap_has(vcpu, X86_FEATURE_SGX_LC)) 7881 vmx->msr_ia32_feature_control_valid_bits |= 7882 FEAT_CTL_SGX_LC_ENABLED; 7883 else 7884 vmx->msr_ia32_feature_control_valid_bits &= 7885 ~FEAT_CTL_SGX_LC_ENABLED; 7886 7887 /* Refresh #PF interception to account for MAXPHYADDR changes. */ 7888 vmx_update_exception_bitmap(vcpu); 7889 } 7890 7891 static __init u64 vmx_get_perf_capabilities(void) 7892 { 7893 u64 perf_cap = PERF_CAP_FW_WRITES; 7894 u64 host_perf_cap = 0; 7895 7896 if (!enable_pmu) 7897 return 0; 7898 7899 if (boot_cpu_has(X86_FEATURE_PDCM)) 7900 rdmsrq(MSR_IA32_PERF_CAPABILITIES, host_perf_cap); 7901 7902 if (!cpu_feature_enabled(X86_FEATURE_ARCH_LBR) && 7903 !enable_mediated_pmu) { 7904 x86_perf_get_lbr(&vmx_lbr_caps); 7905 7906 /* 7907 * KVM requires LBR callstack support, as the overhead due to 7908 * context switching LBRs without said support is too high. 7909 * See intel_pmu_create_guest_lbr_event() for more info. 7910 */ 7911 if (!vmx_lbr_caps.has_callstack) 7912 memset(&vmx_lbr_caps, 0, sizeof(vmx_lbr_caps)); 7913 else if (vmx_lbr_caps.nr) 7914 perf_cap |= host_perf_cap & PERF_CAP_LBR_FMT; 7915 } 7916 7917 if (vmx_pebs_supported()) { 7918 perf_cap |= host_perf_cap & PERF_CAP_PEBS_MASK; 7919 7920 /* 7921 * Disallow adaptive PEBS as it is functionally broken, can be 7922 * used by the guest to read *host* LBRs, and can be used to 7923 * bypass userspace event filters. To correctly and safely 7924 * support adaptive PEBS, KVM needs to: 7925 * 7926 * 1. Account for the ADAPTIVE flag when (re)programming fixed 7927 * counters. 7928 * 7929 * 2. Gain support from perf (or take direct control of counter 7930 * programming) to support events without adaptive PEBS 7931 * enabled for the hardware counter. 7932 * 7933 * 3. Ensure LBR MSRs cannot hold host data on VM-Entry with 7934 * adaptive PEBS enabled and MSR_PEBS_DATA_CFG.LBRS=1. 7935 * 7936 * 4. Document which PMU events are effectively exposed to the 7937 * guest via adaptive PEBS, and make adaptive PEBS mutually 7938 * exclusive with KVM_SET_PMU_EVENT_FILTER if necessary. 7939 */ 7940 perf_cap &= ~PERF_CAP_PEBS_BASELINE; 7941 } 7942 7943 return perf_cap; 7944 } 7945 7946 static __init void vmx_set_cpu_caps(void) 7947 { 7948 kvm_initialize_cpu_caps(); 7949 7950 /* CPUID 0x1 */ 7951 if (nested) 7952 kvm_cpu_cap_set(X86_FEATURE_VMX); 7953 7954 /* CPUID 0x7 */ 7955 if (kvm_mpx_supported()) 7956 kvm_cpu_cap_check_and_set(X86_FEATURE_MPX); 7957 if (!cpu_has_vmx_invpcid()) 7958 kvm_cpu_cap_clear(X86_FEATURE_INVPCID); 7959 if (vmx_pt_mode_is_host_guest()) 7960 kvm_cpu_cap_check_and_set(X86_FEATURE_INTEL_PT); 7961 if (vmx_pebs_supported()) { 7962 kvm_cpu_cap_check_and_set(X86_FEATURE_DS); 7963 kvm_cpu_cap_check_and_set(X86_FEATURE_DTES64); 7964 } 7965 7966 if (!enable_pmu) 7967 kvm_cpu_cap_clear(X86_FEATURE_PDCM); 7968 kvm_caps.supported_perf_cap = vmx_get_perf_capabilities(); 7969 7970 if (!enable_sgx) { 7971 kvm_cpu_cap_clear(X86_FEATURE_SGX); 7972 kvm_cpu_cap_clear(X86_FEATURE_SGX_LC); 7973 kvm_cpu_cap_clear(X86_FEATURE_SGX1); 7974 kvm_cpu_cap_clear(X86_FEATURE_SGX2); 7975 kvm_cpu_cap_clear(X86_FEATURE_SGX_EDECCSSA); 7976 } 7977 7978 if (vmx_umip_emulated()) 7979 kvm_cpu_cap_set(X86_FEATURE_UMIP); 7980 7981 /* CPUID 0xD.1 */ 7982 if (!cpu_has_vmx_xsaves()) 7983 kvm_cpu_cap_clear(X86_FEATURE_XSAVES); 7984 7985 /* CPUID 0x80000001 and 0x7 (RDPID) */ 7986 if (!cpu_has_vmx_rdtscp()) { 7987 kvm_cpu_cap_clear(X86_FEATURE_RDTSCP); 7988 kvm_cpu_cap_clear(X86_FEATURE_RDPID); 7989 } 7990 7991 if (cpu_has_vmx_waitpkg()) 7992 kvm_cpu_cap_check_and_set(X86_FEATURE_WAITPKG); 7993 7994 /* 7995 * Disable CET if unrestricted_guest is unsupported as KVM doesn't 7996 * enforce CET HW behaviors in emulator. On platforms with 7997 * VMX_BASIC[bit56] == 0, inject #CP at VMX entry with error code 7998 * fails, so disable CET in this case too. 7999 */ 8000 if (!enable_cet || !enable_unrestricted_guest || 8001 !cpu_has_vmx_basic_no_hw_errcode_cc()) { 8002 kvm_cpu_cap_clear(X86_FEATURE_SHSTK); 8003 kvm_cpu_cap_clear(X86_FEATURE_IBT); 8004 } 8005 8006 /* 8007 * CPUID 0x80000008. Tell userspace that EFER.LMSLE must be zero; KVM 8008 * never allows EFER.LMSLE to be set on Intel CPUs, as KVM supports long 8009 * mode segment limits only in conjunction with nested SVM. 8010 */ 8011 kvm_cpu_cap_set(X86_FEATURE_EFER_LMSLE_MBZ); 8012 8013 kvm_setup_xss_caps(); 8014 kvm_finalize_cpu_caps(); 8015 } 8016 8017 static bool vmx_is_io_intercepted(struct kvm_vcpu *vcpu, 8018 struct x86_instruction_info *info, 8019 unsigned long *exit_qualification) 8020 { 8021 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 8022 unsigned short port; 8023 int size; 8024 bool imm; 8025 8026 /* 8027 * If the 'use IO bitmaps' VM-execution control is 0, IO instruction 8028 * VM-exits depend on the 'unconditional IO exiting' VM-execution 8029 * control. 8030 * 8031 * Otherwise, IO instruction VM-exits are controlled by the IO bitmaps. 8032 */ 8033 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 8034 return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING); 8035 8036 if (info->intercept == x86_intercept_in || 8037 info->intercept == x86_intercept_ins) { 8038 port = info->src_val; 8039 size = info->dst_bytes; 8040 imm = info->src_type == OP_IMM; 8041 } else { 8042 port = info->dst_val; 8043 size = info->src_bytes; 8044 imm = info->dst_type == OP_IMM; 8045 } 8046 8047 8048 *exit_qualification = ((unsigned long)port << 16) | (size - 1); 8049 8050 if (info->intercept == x86_intercept_ins || 8051 info->intercept == x86_intercept_outs) 8052 *exit_qualification |= BIT(4); 8053 8054 if (info->rep_prefix) 8055 *exit_qualification |= BIT(5); 8056 8057 if (imm) 8058 *exit_qualification |= BIT(6); 8059 8060 return nested_vmx_check_io_bitmaps(vcpu, port, size); 8061 } 8062 8063 int vmx_check_intercept(struct kvm_vcpu *vcpu, 8064 struct x86_instruction_info *info, 8065 enum x86_intercept_stage stage, 8066 struct x86_exception *exception) 8067 { 8068 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 8069 unsigned long exit_qualification = 0; 8070 u32 vm_exit_reason; 8071 u64 exit_insn_len; 8072 8073 switch (info->intercept) { 8074 case x86_intercept_rdpid: 8075 /* 8076 * RDPID causes #UD if not enabled through secondary execution 8077 * controls (ENABLE_RDTSCP). Note, the implicit MSR access to 8078 * TSC_AUX is NOT subject to interception, i.e. checking only 8079 * the dedicated execution control is architecturally correct. 8080 */ 8081 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_RDTSCP)) { 8082 exception->vector = UD_VECTOR; 8083 exception->error_code_valid = false; 8084 return X86EMUL_PROPAGATE_FAULT; 8085 } 8086 return X86EMUL_CONTINUE; 8087 8088 case x86_intercept_in: 8089 case x86_intercept_ins: 8090 case x86_intercept_out: 8091 case x86_intercept_outs: 8092 if (!vmx_is_io_intercepted(vcpu, info, &exit_qualification)) 8093 return X86EMUL_CONTINUE; 8094 8095 vm_exit_reason = EXIT_REASON_IO_INSTRUCTION; 8096 break; 8097 8098 case x86_intercept_lgdt: 8099 case x86_intercept_lidt: 8100 case x86_intercept_lldt: 8101 case x86_intercept_ltr: 8102 case x86_intercept_sgdt: 8103 case x86_intercept_sidt: 8104 case x86_intercept_sldt: 8105 case x86_intercept_str: 8106 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC)) 8107 return X86EMUL_CONTINUE; 8108 8109 if (info->intercept == x86_intercept_lldt || 8110 info->intercept == x86_intercept_ltr || 8111 info->intercept == x86_intercept_sldt || 8112 info->intercept == x86_intercept_str) 8113 vm_exit_reason = EXIT_REASON_LDTR_TR; 8114 else 8115 vm_exit_reason = EXIT_REASON_GDTR_IDTR; 8116 /* 8117 * FIXME: Decode the ModR/M to generate the correct exit 8118 * qualification for memory operands. 8119 */ 8120 break; 8121 8122 case x86_intercept_hlt: 8123 if (!nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING)) 8124 return X86EMUL_CONTINUE; 8125 8126 vm_exit_reason = EXIT_REASON_HLT; 8127 break; 8128 8129 case x86_intercept_pause: 8130 /* 8131 * PAUSE is a single-byte NOP with a REPE prefix, i.e. collides 8132 * with vanilla NOPs in the emulator. Apply the interception 8133 * check only to actual PAUSE instructions. Don't check 8134 * PAUSE-loop-exiting, software can't expect a given PAUSE to 8135 * exit, i.e. KVM is within its rights to allow L2 to execute 8136 * the PAUSE. 8137 */ 8138 if ((info->rep_prefix != REPE_PREFIX) || 8139 !nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING)) 8140 return X86EMUL_CONTINUE; 8141 8142 vm_exit_reason = EXIT_REASON_PAUSE_INSTRUCTION; 8143 break; 8144 8145 /* TODO: check more intercepts... */ 8146 default: 8147 return X86EMUL_UNHANDLEABLE; 8148 } 8149 8150 exit_insn_len = abs_diff((s64)info->next_rip, (s64)info->rip); 8151 if (!exit_insn_len || exit_insn_len > X86_MAX_INSTRUCTION_LENGTH) 8152 return X86EMUL_UNHANDLEABLE; 8153 8154 __nested_vmx_vmexit(vcpu, vm_exit_reason, 0, exit_qualification, 8155 exit_insn_len); 8156 return X86EMUL_INTERCEPTED; 8157 } 8158 8159 #ifdef CONFIG_X86_64 8160 /* (a << shift) / divisor, return 1 if overflow otherwise 0 */ 8161 static inline int u64_shl_div_u64(u64 a, unsigned int shift, 8162 u64 divisor, u64 *result) 8163 { 8164 u64 low = a << shift, high = a >> (64 - shift); 8165 8166 /* To avoid the overflow on divq */ 8167 if (high >= divisor) 8168 return 1; 8169 8170 /* Low hold the result, high hold rem which is discarded */ 8171 asm("divq %2\n\t" : "=a" (low), "=d" (high) : 8172 "rm" (divisor), "0" (low), "1" (high)); 8173 *result = low; 8174 8175 return 0; 8176 } 8177 8178 /* 8179 * Workaround for a widespread Intel erratum (e.g. EMR158) where the 8180 * VMX-preemption timer may expire earlier than expected when programmed 8181 * with large values. The workaround is to cap the timer value to strictly 8182 * less than 2^25 * CPUID.15H:EBX / CPUID.15H:EAX. 8183 */ 8184 static __init u64 calc_preemption_timer_max_value(void) 8185 { 8186 const u64 ARCHITECTURAL_MAX_VALUE = UINT_MAX; 8187 u32 eax, ebx, ecx, edx; 8188 8189 if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR)) 8190 return ARCHITECTURAL_MAX_VALUE; 8191 8192 if (cpuid_eax(0) < 0x15) 8193 return ARCHITECTURAL_MAX_VALUE; 8194 8195 cpuid(0x15, &eax, &ebx, &ecx, &edx); 8196 if (!eax || !ebx) 8197 return ARCHITECTURAL_MAX_VALUE; 8198 8199 if (WARN_ON_ONCE(!(((u64)ebx << 25) / eax))) 8200 return ARCHITECTURAL_MAX_VALUE; 8201 8202 return min((((u64)ebx << 25) / eax) - 1, ARCHITECTURAL_MAX_VALUE); 8203 } 8204 8205 static __init void vmx_setup_preemption_timer(void) 8206 { 8207 if (!cpu_has_vmx_preemption_timer()) 8208 enable_preemption_timer = false; 8209 8210 if (enable_preemption_timer) { 8211 u64 use_timer_freq = 5000ULL * 1000 * 1000; 8212 8213 cpu_preemption_timer_multi = 8214 vmx_misc_preemption_timer_rate(vmcs_config.misc); 8215 8216 preemption_timer_max_value = calc_preemption_timer_max_value(); 8217 8218 if (tsc_khz) 8219 use_timer_freq = (u64)tsc_khz * 1000; 8220 use_timer_freq >>= cpu_preemption_timer_multi; 8221 8222 /* 8223 * KVM "disables" the preemption timer by setting it to its max 8224 * value. Don't use the timer if it might cause spurious exits 8225 * at a rate faster than 0.1 Hz (of uninterrupted guest time). 8226 */ 8227 if (use_timer_freq > preemption_timer_max_value / 10) 8228 enable_preemption_timer = false; 8229 } 8230 8231 if (!enable_preemption_timer) { 8232 vt_x86_ops.set_hv_timer = NULL; 8233 vt_x86_ops.cancel_hv_timer = NULL; 8234 } 8235 } 8236 8237 int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, 8238 bool *expired) 8239 { 8240 struct vcpu_vmx *vmx; 8241 u64 tscl, guest_tscl, delta_tsc, lapic_timer_advance_cycles; 8242 struct kvm_timer *ktimer = &vcpu->arch.apic->lapic_timer; 8243 8244 vmx = to_vmx(vcpu); 8245 tscl = rdtsc(); 8246 guest_tscl = kvm_read_l1_tsc(vcpu, tscl); 8247 delta_tsc = max(guest_deadline_tsc, guest_tscl) - guest_tscl; 8248 lapic_timer_advance_cycles = nsec_to_cycles(vcpu, 8249 ktimer->timer_advance_ns); 8250 8251 if (delta_tsc > lapic_timer_advance_cycles) 8252 delta_tsc -= lapic_timer_advance_cycles; 8253 else 8254 delta_tsc = 0; 8255 8256 /* Convert to host delta tsc if tsc scaling is enabled */ 8257 if (vcpu->arch.l1_tsc_scaling_ratio != kvm_caps.default_tsc_scaling_ratio && 8258 delta_tsc && u64_shl_div_u64(delta_tsc, 8259 kvm_caps.tsc_scaling_ratio_frac_bits, 8260 vcpu->arch.l1_tsc_scaling_ratio, &delta_tsc)) 8261 return -ERANGE; 8262 8263 /* 8264 * If the delta tsc exceeds the preemption timer limit after the 8265 * multi shift, we can't use the preemption timer. 8266 * It's possible that it fits on later vmentries, but checking 8267 * on every vmentry is costly so we just use an hrtimer. 8268 */ 8269 if ((delta_tsc >> cpu_preemption_timer_multi) > preemption_timer_max_value) 8270 return -ERANGE; 8271 8272 vmx->hv_deadline_tsc = tscl + delta_tsc; 8273 *expired = !delta_tsc; 8274 return 0; 8275 } 8276 8277 void vmx_cancel_hv_timer(struct kvm_vcpu *vcpu) 8278 { 8279 to_vmx(vcpu)->hv_deadline_tsc = -1; 8280 } 8281 8282 static void vmx_update_hv_timer(struct kvm_vcpu *vcpu, bool force_immediate_exit) 8283 { 8284 struct vcpu_vmx *vmx = to_vmx(vcpu); 8285 u64 tscl; 8286 u32 delta_tsc; 8287 8288 if (force_immediate_exit) { 8289 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, 0); 8290 vmx->loaded_vmcs->hv_timer_soft_disabled = false; 8291 } else if (vmx->hv_deadline_tsc != -1) { 8292 tscl = rdtsc(); 8293 if (vmx->hv_deadline_tsc > tscl) 8294 /* set_hv_timer ensures the delta fits in 32-bits */ 8295 delta_tsc = (u32)((vmx->hv_deadline_tsc - tscl) >> 8296 cpu_preemption_timer_multi); 8297 else 8298 delta_tsc = 0; 8299 8300 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, delta_tsc); 8301 vmx->loaded_vmcs->hv_timer_soft_disabled = false; 8302 } else if (!vmx->loaded_vmcs->hv_timer_soft_disabled) { 8303 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, preemption_timer_max_value); 8304 vmx->loaded_vmcs->hv_timer_soft_disabled = true; 8305 } 8306 } 8307 #else 8308 static __init void vmx_setup_preemption_timer(void) { } 8309 8310 static void vmx_update_hv_timer(struct kvm_vcpu *vcpu, bool force_immediate_exit) 8311 { 8312 BUILD_BUG_ON(1); 8313 } 8314 #endif 8315 8316 void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable) 8317 { 8318 struct vcpu_vmx *vmx = to_vmx(vcpu); 8319 8320 guard(vmx_vmcs01)(vcpu); 8321 8322 if (enable) 8323 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_ENABLE_PML); 8324 else 8325 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_ENABLE_PML); 8326 } 8327 8328 void vmx_setup_mce(struct kvm_vcpu *vcpu) 8329 { 8330 if (vcpu->arch.mcg_cap & MCG_LMCE_P) 8331 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |= 8332 FEAT_CTL_LMCE_ENABLED; 8333 else 8334 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &= 8335 ~FEAT_CTL_LMCE_ENABLED; 8336 } 8337 8338 #ifdef CONFIG_KVM_SMM 8339 int vmx_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) 8340 { 8341 /* we need a nested vmexit to enter SMM, postpone if run is pending */ 8342 if (vcpu->arch.nested_run_pending) 8343 return -EBUSY; 8344 return !is_smm(vcpu); 8345 } 8346 8347 int vmx_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram) 8348 { 8349 struct vcpu_vmx *vmx = to_vmx(vcpu); 8350 8351 /* 8352 * TODO: Implement custom flows for forcing the vCPU out/in of L2 on 8353 * SMI and RSM. Using the common VM-Exit + VM-Enter routines is wrong 8354 * SMI and RSM only modify state that is saved and restored via SMRAM. 8355 * E.g. most MSRs are left untouched, but many are modified by VM-Exit 8356 * and VM-Enter, and thus L2's values may be corrupted on SMI+RSM. 8357 */ 8358 vmx->nested.smm.guest_mode = is_guest_mode(vcpu); 8359 if (vmx->nested.smm.guest_mode) 8360 nested_vmx_vmexit(vcpu, -1, 0, 0); 8361 8362 vmx->nested.smm.vmxon = vmx->nested.vmxon; 8363 vmx->nested.vmxon = false; 8364 vmx_clear_hlt(vcpu); 8365 return 0; 8366 } 8367 8368 int vmx_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram) 8369 { 8370 struct vcpu_vmx *vmx = to_vmx(vcpu); 8371 int ret; 8372 8373 if (vmx->nested.smm.vmxon) { 8374 vmx->nested.vmxon = true; 8375 vmx->nested.smm.vmxon = false; 8376 } 8377 8378 if (vmx->nested.smm.guest_mode) { 8379 /* Triple fault if the state is invalid. */ 8380 if (nested_vmx_check_restored_vmcs12(vcpu) < 0) 8381 return 1; 8382 8383 ret = nested_vmx_enter_non_root_mode(vcpu, false); 8384 if (ret != NVMX_VMENTRY_SUCCESS) 8385 return 1; 8386 8387 vcpu->arch.nested_run_pending = KVM_NESTED_RUN_PENDING; 8388 vmx->nested.smm.guest_mode = false; 8389 } 8390 return 0; 8391 } 8392 8393 void vmx_enable_smi_window(struct kvm_vcpu *vcpu) 8394 { 8395 /* RSM will cause a vmexit anyway. */ 8396 } 8397 #endif 8398 8399 bool vmx_apic_init_signal_blocked(struct kvm_vcpu *vcpu) 8400 { 8401 return to_vmx(vcpu)->nested.vmxon && !is_guest_mode(vcpu); 8402 } 8403 8404 void vmx_migrate_timers(struct kvm_vcpu *vcpu) 8405 { 8406 if (is_guest_mode(vcpu)) { 8407 struct hrtimer *timer = &to_vmx(vcpu)->nested.preemption_timer; 8408 8409 if (hrtimer_try_to_cancel(timer) == 1) 8410 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED); 8411 } 8412 } 8413 8414 void vmx_hardware_unsetup(void) 8415 { 8416 kvm_set_posted_intr_wakeup_handler(NULL); 8417 8418 if (nested) 8419 nested_vmx_hardware_unsetup(); 8420 } 8421 8422 void vmx_vm_destroy(struct kvm *kvm) 8423 { 8424 struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); 8425 8426 free_pages((unsigned long)kvm_vmx->pid_table, vmx_get_pid_table_order(kvm)); 8427 } 8428 8429 /* 8430 * Note, the SDM states that the linear address is masked *after* the modified 8431 * canonicality check, whereas KVM masks (untags) the address and then performs 8432 * a "normal" canonicality check. Functionally, the two methods are identical, 8433 * and when the masking occurs relative to the canonicality check isn't visible 8434 * to software, i.e. KVM's behavior doesn't violate the SDM. 8435 */ 8436 gva_t vmx_get_untagged_addr(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags) 8437 { 8438 int lam_bit; 8439 unsigned long cr3_bits; 8440 8441 if (flags & (X86EMUL_F_FETCH | X86EMUL_F_IMPLICIT | X86EMUL_F_INVLPG)) 8442 return gva; 8443 8444 if (!is_64_bit_mode(vcpu)) 8445 return gva; 8446 8447 /* 8448 * Bit 63 determines if the address should be treated as user address 8449 * or a supervisor address. 8450 */ 8451 if (!(gva & BIT_ULL(63))) { 8452 cr3_bits = kvm_get_active_cr3_lam_bits(vcpu); 8453 if (!(cr3_bits & (X86_CR3_LAM_U57 | X86_CR3_LAM_U48))) 8454 return gva; 8455 8456 /* LAM_U48 is ignored if LAM_U57 is set. */ 8457 lam_bit = cr3_bits & X86_CR3_LAM_U57 ? 56 : 47; 8458 } else { 8459 if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_LAM_SUP)) 8460 return gva; 8461 8462 lam_bit = kvm_is_cr4_bit_set(vcpu, X86_CR4_LA57) ? 56 : 47; 8463 } 8464 8465 /* 8466 * Untag the address by sign-extending the lam_bit, but NOT to bit 63. 8467 * Bit 63 is retained from the raw virtual address so that untagging 8468 * doesn't change a user access to a supervisor access, and vice versa. 8469 */ 8470 return (sign_extend64(gva, lam_bit) & ~BIT_ULL(63)) | (gva & BIT_ULL(63)); 8471 } 8472 8473 static unsigned int vmx_handle_intel_pt_intr(void) 8474 { 8475 struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); 8476 8477 /* '0' on failure so that the !PT case can use a RET0 static call. */ 8478 if (!vcpu || !kvm_handling_nmi_from_guest(vcpu)) 8479 return 0; 8480 8481 kvm_make_request(KVM_REQ_PMI, vcpu); 8482 __set_bit(MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT, 8483 (unsigned long *)&vcpu->arch.pmu.global_status); 8484 return 1; 8485 } 8486 8487 static __init void vmx_setup_user_return_msrs(void) 8488 { 8489 8490 /* 8491 * Though SYSCALL is only supported in 64-bit mode on Intel CPUs, kvm 8492 * will emulate SYSCALL in legacy mode if the vendor string in guest 8493 * CPUID.0:{EBX,ECX,EDX} is "AuthenticAMD" or "AMDisbetter!" To 8494 * support this emulation, MSR_STAR is included in the list for i386, 8495 * but is never loaded into hardware. MSR_CSTAR is also never loaded 8496 * into hardware and is here purely for emulation purposes. 8497 */ 8498 const u32 vmx_uret_msrs_list[] = { 8499 #ifdef CONFIG_X86_64 8500 MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR, 8501 #endif 8502 MSR_EFER, MSR_TSC_AUX, MSR_STAR, 8503 MSR_IA32_TSX_CTRL, 8504 }; 8505 int i; 8506 8507 BUILD_BUG_ON(ARRAY_SIZE(vmx_uret_msrs_list) != MAX_NR_USER_RETURN_MSRS); 8508 8509 for (i = 0; i < ARRAY_SIZE(vmx_uret_msrs_list); ++i) 8510 kvm_add_user_return_msr(vmx_uret_msrs_list[i]); 8511 } 8512 8513 static void __init vmx_setup_me_spte_mask(void) 8514 { 8515 u64 me_mask = 0; 8516 8517 /* 8518 * On pre-MKTME system, boot_cpu_data.x86_phys_bits equals to 8519 * kvm_host.maxphyaddr. On MKTME and/or TDX capable systems, 8520 * boot_cpu_data.x86_phys_bits holds the actual physical address 8521 * w/o the KeyID bits, and kvm_host.maxphyaddr equals to 8522 * MAXPHYADDR reported by CPUID. Those bits between are KeyID bits. 8523 */ 8524 if (boot_cpu_data.x86_phys_bits != kvm_host.maxphyaddr) 8525 me_mask = rsvd_bits(boot_cpu_data.x86_phys_bits, 8526 kvm_host.maxphyaddr - 1); 8527 8528 /* 8529 * Unlike SME, host kernel doesn't support setting up any 8530 * MKTME KeyID on Intel platforms. No memory encryption 8531 * bits should be included into the SPTE. 8532 */ 8533 kvm_mmu_set_me_spte_mask(0, me_mask); 8534 } 8535 8536 __init int vmx_hardware_setup(void) 8537 { 8538 unsigned long host_bndcfgs; 8539 struct desc_ptr dt; 8540 int r; 8541 8542 store_idt(&dt); 8543 host_idt_base = dt.address; 8544 8545 vmx_setup_user_return_msrs(); 8546 8547 if (boot_cpu_has(X86_FEATURE_MPX)) { 8548 rdmsrq(MSR_IA32_BNDCFGS, host_bndcfgs); 8549 WARN_ONCE(host_bndcfgs, "BNDCFGS in host will be lost"); 8550 } 8551 8552 if (!cpu_has_vmx_mpx()) 8553 kvm_caps.supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS | 8554 XFEATURE_MASK_BNDCSR); 8555 8556 if (!cpu_has_vmx_vpid() || !cpu_has_vmx_invvpid() || 8557 !(cpu_has_vmx_invvpid_single() || cpu_has_vmx_invvpid_global())) 8558 enable_vpid = 0; 8559 8560 if (!cpu_has_vmx_ept() || 8561 !cpu_has_vmx_ept_4levels() || 8562 !cpu_has_vmx_ept_mt_wb() || 8563 !cpu_has_vmx_invept_global()) 8564 enable_ept = 0; 8565 8566 if (!cpu_has_load_cet_ctrl()) 8567 enable_cet = 0; 8568 8569 /* NX support is required for shadow paging. */ 8570 if (!enable_ept && !boot_cpu_has(X86_FEATURE_NX)) { 8571 pr_err_ratelimited("NX (Execute Disable) not supported\n"); 8572 return -EOPNOTSUPP; 8573 } 8574 8575 /* 8576 * Shadow paging doesn't have a (further) performance penalty 8577 * from GUEST_MAXPHYADDR < HOST_MAXPHYADDR so enable it 8578 * by default 8579 */ 8580 if (!enable_ept) 8581 allow_smaller_maxphyaddr = true; 8582 8583 if (!cpu_has_vmx_ept_ad_bits() || !enable_ept) 8584 enable_ept_ad_bits = 0; 8585 if (!cpu_has_ept_mbec() || !enable_ept) 8586 enable_mbec = 0; 8587 8588 if (!cpu_has_vmx_unrestricted_guest() || !enable_ept) 8589 enable_unrestricted_guest = 0; 8590 8591 if (!cpu_has_vmx_flexpriority()) 8592 flexpriority_enabled = 0; 8593 8594 if (!cpu_has_virtual_nmis()) 8595 enable_vnmi = 0; 8596 8597 #ifdef CONFIG_X86_SGX_KVM 8598 if (!cpu_has_vmx_encls_vmexit()) 8599 enable_sgx = false; 8600 #endif 8601 8602 /* 8603 * set_apic_access_page_addr() is used to reload apic access 8604 * page upon invalidation. No need to do anything if not 8605 * using the APIC_ACCESS_ADDR VMCS field. 8606 */ 8607 if (!flexpriority_enabled) 8608 vt_x86_ops.set_apic_access_page_addr = NULL; 8609 8610 if (!cpu_has_vmx_tpr_shadow()) 8611 vt_x86_ops.update_cr8_intercept = NULL; 8612 8613 #if IS_ENABLED(CONFIG_HYPERV) 8614 if (ms_hyperv.nested_features & HV_X64_NESTED_GUEST_MAPPING_FLUSH 8615 && enable_ept) { 8616 vt_x86_ops.flush_remote_tlbs = hv_flush_remote_tlbs; 8617 vt_x86_ops.flush_remote_tlbs_range = hv_flush_remote_tlbs_range; 8618 } 8619 #endif 8620 8621 if (!cpu_has_vmx_ple()) { 8622 ple_gap = 0; 8623 ple_window = 0; 8624 ple_window_grow = 0; 8625 ple_window_max = 0; 8626 ple_window_shrink = 0; 8627 } 8628 8629 if (!cpu_has_vmx_apicv()) 8630 enable_apicv = 0; 8631 if (!enable_apicv) 8632 vt_x86_ops.sync_pir_to_irr = NULL; 8633 8634 if (!enable_apicv || !cpu_has_vmx_ipiv()) 8635 enable_ipiv = false; 8636 8637 if (cpu_has_vmx_tsc_scaling()) 8638 kvm_caps.has_tsc_control = true; 8639 8640 kvm_caps.max_tsc_scaling_ratio = KVM_VMX_TSC_MULTIPLIER_MAX; 8641 kvm_caps.tsc_scaling_ratio_frac_bits = 48; 8642 kvm_caps.has_bus_lock_exit = cpu_has_vmx_bus_lock_detection(); 8643 kvm_caps.has_notify_vmexit = cpu_has_notify_vmexit(); 8644 8645 set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */ 8646 8647 if (enable_ept) 8648 kvm_mmu_set_ept_masks(enable_ept_ad_bits); 8649 else 8650 vt_x86_ops.get_mt_mask = NULL; 8651 8652 /* 8653 * Setup shadow_me_value/shadow_me_mask to include MKTME KeyID 8654 * bits into the MMU's struct kvm_page_format. 8655 */ 8656 vmx_setup_me_spte_mask(); 8657 8658 kvm_configure_mmu(enable_ept, 0, vmx_get_max_ept_level(), 8659 ept_caps_to_lpage_level(vmx_capability.ept)); 8660 8661 /* 8662 * Only enable PML when hardware supports PML feature, and both EPT 8663 * and EPT A/D bit features are enabled -- PML depends on them to work. 8664 */ 8665 if (!enable_ept || !enable_ept_ad_bits || !cpu_has_vmx_pml()) 8666 enable_pml = 0; 8667 8668 vmx_setup_preemption_timer(); 8669 8670 kvm_caps.supported_mce_cap |= MCG_LMCE_P; 8671 kvm_caps.supported_mce_cap |= MCG_CMCI_P; 8672 8673 if (pt_mode != PT_MODE_SYSTEM && pt_mode != PT_MODE_HOST_GUEST) 8674 return -EINVAL; 8675 if (!enable_ept || !enable_pmu || !cpu_has_vmx_intel_pt()) 8676 pt_mode = PT_MODE_SYSTEM; 8677 if (pt_mode == PT_MODE_HOST_GUEST) 8678 vt_init_ops.handle_intel_pt_intr = vmx_handle_intel_pt_intr; 8679 else 8680 vt_init_ops.handle_intel_pt_intr = NULL; 8681 8682 setup_default_sgx_lepubkeyhash(); 8683 8684 vmx_set_cpu_caps(); 8685 8686 /* 8687 * Configure nested capabilities after core CPU capabilities so that 8688 * nested support can be conditional on base support, e.g. so that KVM 8689 * can hide/show features based on kvm_cpu_cap_has(). 8690 */ 8691 if (nested) { 8692 r = nested_vmx_hardware_setup(kvm_vmx_exit_handlers); 8693 if (r) 8694 return r; 8695 } 8696 vmx_nested_ops.enabled = nested; 8697 8698 kvm_set_posted_intr_wakeup_handler(pi_wakeup_handler); 8699 8700 /* 8701 * On Intel CPUs that lack self-snoop feature, letting the guest control 8702 * memory types may result in unexpected behavior. So always ignore guest 8703 * PAT on those CPUs and map VM as writeback, not allowing userspace to 8704 * disable the quirk. 8705 * 8706 * On certain Intel CPUs (e.g. SPR, ICX), though self-snoop feature is 8707 * supported, UC is slow enough to cause issues with some older guests (e.g. 8708 * an old version of bochs driver uses ioremap() instead of ioremap_wc() to 8709 * map the video RAM, causing wayland desktop to fail to get started 8710 * correctly). To avoid breaking those older guests that rely on KVM to force 8711 * memory type to WB, provide KVM_X86_QUIRK_IGNORE_GUEST_PAT to preserve the 8712 * safer (for performance) default behavior. 8713 * 8714 * On top of this, non-coherent DMA devices need the guest to flush CPU 8715 * caches properly. This also requires honoring guest PAT, and is forced 8716 * independent of the quirk in vmx_ignore_guest_pat(). 8717 */ 8718 if (!cpu_feature_enabled(X86_FEATURE_SELFSNOOP)) 8719 kvm_caps.supported_quirks &= ~KVM_X86_QUIRK_IGNORE_GUEST_PAT; 8720 8721 kvm_caps.inapplicable_quirks &= ~KVM_X86_QUIRK_IGNORE_GUEST_PAT; 8722 8723 return 0; 8724 } 8725 8726 void vmx_exit(void) 8727 { 8728 allow_smaller_maxphyaddr = false; 8729 8730 vmx_cleanup_l1d_flush(); 8731 8732 kvm_x86_vendor_exit(); 8733 } 8734 8735 int __init vmx_init(void) 8736 { 8737 int r, cpu; 8738 8739 KVM_SANITY_CHECK_VM_STRUCT_SIZE(kvm_vmx); 8740 8741 if (!kvm_is_vmx_supported()) 8742 return -EOPNOTSUPP; 8743 8744 /* 8745 * Note, VMCS and eVMCS configuration only touch VMX knobs/variables, 8746 * i.e. there's nothing to unwind if a later step fails. 8747 */ 8748 hv_init_evmcs(); 8749 8750 /* 8751 * Parse the VMCS config and VMX capabilities before anything else, so 8752 * that the information is available to all setup flows. 8753 */ 8754 if (setup_vmcs_config(&vmcs_config, &vmx_capability) < 0) 8755 return -EIO; 8756 8757 r = kvm_x86_vendor_init(&vt_init_ops); 8758 if (r) 8759 return r; 8760 8761 /* Must be called after common x86 init so enable_ept is setup. */ 8762 r = vmx_setup_l1d_flush(); 8763 if (r) 8764 goto err_l1d_flush; 8765 8766 for_each_possible_cpu(cpu) { 8767 INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu)); 8768 8769 pi_init_cpu(cpu); 8770 } 8771 8772 vmx_check_vmcs12_offsets(); 8773 8774 return 0; 8775 8776 err_l1d_flush: 8777 kvm_x86_vendor_exit(); 8778 return r; 8779 } 8780