1 // SPDX-License-Identifier: GPL-2.0 2 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 3 4 #include <linux/objtool.h> 5 #include <linux/percpu.h> 6 7 #include <asm/debugreg.h> 8 #include <asm/mmu_context.h> 9 #include <asm/msr.h> 10 11 #include "x86.h" 12 #include "cpuid.h" 13 #include "hyperv.h" 14 #include "irq.h" 15 #include "mmu.h" 16 #include "nested.h" 17 #include "pmu.h" 18 #include "posted_intr.h" 19 #include "sgx.h" 20 #include "trace.h" 21 #include "vmx.h" 22 #include "smm.h" 23 #include "x86_ops.h" 24 25 static bool __read_mostly enable_shadow_vmcs = 1; 26 module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO); 27 28 static bool __ro_after_init warn_on_missed_cc; 29 module_param(warn_on_missed_cc, bool, 0444); 30 31 #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK 32 33 /* 34 * Hyper-V requires all of these, so mark them as supported even though 35 * they are just treated the same as all-context. 36 */ 37 #define VMX_VPID_EXTENT_SUPPORTED_MASK \ 38 (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT | \ 39 VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT | \ 40 VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT | \ 41 VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT) 42 43 #define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5 44 45 enum { 46 VMX_VMREAD_BITMAP, 47 VMX_VMWRITE_BITMAP, 48 VMX_BITMAP_NR 49 }; 50 static unsigned long *vmx_bitmap[VMX_BITMAP_NR]; 51 52 #define vmx_vmread_bitmap (vmx_bitmap[VMX_VMREAD_BITMAP]) 53 #define vmx_vmwrite_bitmap (vmx_bitmap[VMX_VMWRITE_BITMAP]) 54 55 struct shadow_vmcs_field { 56 u16 encoding; 57 u16 offset; 58 }; 59 static struct shadow_vmcs_field shadow_read_only_fields[] = { 60 #define SHADOW_FIELD_RO(x, y) { x, offsetof(struct vmcs12, y) }, 61 #include "vmcs_shadow_fields.h" 62 }; 63 static int max_shadow_read_only_fields = 64 ARRAY_SIZE(shadow_read_only_fields); 65 66 static struct shadow_vmcs_field shadow_read_write_fields[] = { 67 #define SHADOW_FIELD_RW(x, y) { x, offsetof(struct vmcs12, y) }, 68 #include "vmcs_shadow_fields.h" 69 }; 70 static int max_shadow_read_write_fields = 71 ARRAY_SIZE(shadow_read_write_fields); 72 73 static void init_vmcs_shadow_fields(void) 74 { 75 int i, j; 76 77 memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE); 78 memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE); 79 80 for (i = j = 0; i < max_shadow_read_only_fields; i++) { 81 struct shadow_vmcs_field entry = shadow_read_only_fields[i]; 82 u16 field = entry.encoding; 83 84 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && 85 (i + 1 == max_shadow_read_only_fields || 86 shadow_read_only_fields[i + 1].encoding != field + 1)) 87 pr_err("Missing field from shadow_read_only_field %x\n", 88 field + 1); 89 90 if (get_vmcs12_field_offset(field) < 0) 91 continue; 92 93 clear_bit(field, vmx_vmread_bitmap); 94 if (field & 1) 95 #ifdef CONFIG_X86_64 96 continue; 97 #else 98 entry.offset += sizeof(u32); 99 #endif 100 shadow_read_only_fields[j++] = entry; 101 } 102 max_shadow_read_only_fields = j; 103 104 for (i = j = 0; i < max_shadow_read_write_fields; i++) { 105 struct shadow_vmcs_field entry = shadow_read_write_fields[i]; 106 u16 field = entry.encoding; 107 108 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && 109 (i + 1 == max_shadow_read_write_fields || 110 shadow_read_write_fields[i + 1].encoding != field + 1)) 111 pr_err("Missing field from shadow_read_write_field %x\n", 112 field + 1); 113 114 WARN_ONCE(field >= GUEST_ES_AR_BYTES && 115 field <= GUEST_TR_AR_BYTES, 116 "Update vmcs12_write_any() to drop reserved bits from AR_BYTES"); 117 118 if (get_vmcs12_field_offset(field) < 0) 119 continue; 120 121 /* 122 * KVM emulates PML and the VMX preemption timer irrespective 123 * of hardware support, but shadowing their related VMCS fields 124 * requires hardware support as the CPU will reject VMWRITEs to 125 * fields that don't exist. 126 */ 127 switch (field) { 128 case GUEST_PML_INDEX: 129 if (!cpu_has_vmx_pml()) 130 continue; 131 break; 132 case VMX_PREEMPTION_TIMER_VALUE: 133 if (!cpu_has_vmx_preemption_timer()) 134 continue; 135 break; 136 default: 137 break; 138 } 139 140 clear_bit(field, vmx_vmwrite_bitmap); 141 clear_bit(field, vmx_vmread_bitmap); 142 if (field & 1) 143 #ifdef CONFIG_X86_64 144 continue; 145 #else 146 entry.offset += sizeof(u32); 147 #endif 148 shadow_read_write_fields[j++] = entry; 149 } 150 max_shadow_read_write_fields = j; 151 } 152 153 /* 154 * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(), 155 * set the success or error code of an emulated VMX instruction (as specified 156 * by Vol 2B, VMX Instruction Reference, "Conventions"), and skip the emulated 157 * instruction. 158 */ 159 static int nested_vmx_succeed(struct kvm_vcpu *vcpu) 160 { 161 vmx_set_rflags(vcpu, vmx_get_rflags(vcpu) 162 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | 163 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF)); 164 return kvm_skip_emulated_instruction(vcpu); 165 } 166 167 static int nested_vmx_failInvalid(struct kvm_vcpu *vcpu) 168 { 169 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) 170 & ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF | 171 X86_EFLAGS_SF | X86_EFLAGS_OF)) 172 | X86_EFLAGS_CF); 173 return kvm_skip_emulated_instruction(vcpu); 174 } 175 176 static int nested_vmx_failValid(struct kvm_vcpu *vcpu, 177 u32 vm_instruction_error) 178 { 179 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) 180 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | 181 X86_EFLAGS_SF | X86_EFLAGS_OF)) 182 | X86_EFLAGS_ZF); 183 get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error; 184 /* 185 * We don't need to force sync to shadow VMCS because 186 * VM_INSTRUCTION_ERROR is not shadowed. Enlightened VMCS 'shadows' all 187 * fields and thus must be synced. 188 */ 189 if (nested_vmx_is_evmptr12_set(to_vmx(vcpu))) 190 to_vmx(vcpu)->nested.need_vmcs12_to_shadow_sync = true; 191 192 return kvm_skip_emulated_instruction(vcpu); 193 } 194 195 static int nested_vmx_fail(struct kvm_vcpu *vcpu, u32 vm_instruction_error) 196 { 197 struct vcpu_vmx *vmx = to_vmx(vcpu); 198 199 /* 200 * failValid writes the error number to the current VMCS, which 201 * can't be done if there isn't a current VMCS. 202 */ 203 if (vmx->nested.current_vmptr == INVALID_GPA && 204 !nested_vmx_is_evmptr12_valid(vmx)) 205 return nested_vmx_failInvalid(vcpu); 206 207 return nested_vmx_failValid(vcpu, vm_instruction_error); 208 } 209 210 static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator) 211 { 212 /* TODO: not to reset guest simply here. */ 213 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 214 pr_debug_ratelimited("nested vmx abort, indicator %d\n", indicator); 215 } 216 217 static inline bool vmx_control_verify(u32 control, u32 low, u32 high) 218 { 219 return fixed_bits_valid(control, low, high); 220 } 221 222 static inline u64 vmx_control_msr(u32 low, u32 high) 223 { 224 return low | ((u64)high << 32); 225 } 226 227 static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx) 228 { 229 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_SHADOW_VMCS); 230 vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); 231 vmx->nested.need_vmcs12_to_shadow_sync = false; 232 } 233 234 static inline void nested_release_evmcs(struct kvm_vcpu *vcpu) 235 { 236 #ifdef CONFIG_KVM_HYPERV 237 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 238 struct vcpu_vmx *vmx = to_vmx(vcpu); 239 240 kvm_vcpu_unmap(vcpu, &vmx->nested.hv_evmcs_map); 241 vmx->nested.hv_evmcs = NULL; 242 vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID; 243 244 if (hv_vcpu) { 245 hv_vcpu->nested.pa_page_gpa = INVALID_GPA; 246 hv_vcpu->nested.vm_id = 0; 247 hv_vcpu->nested.vp_id = 0; 248 } 249 #endif 250 } 251 252 static bool nested_evmcs_handle_vmclear(struct kvm_vcpu *vcpu, gpa_t vmptr) 253 { 254 #ifdef CONFIG_KVM_HYPERV 255 struct vcpu_vmx *vmx = to_vmx(vcpu); 256 /* 257 * When Enlightened VMEntry is enabled on the calling CPU we treat 258 * memory area pointer by vmptr as Enlightened VMCS (as there's no good 259 * way to distinguish it from VMCS12) and we must not corrupt it by 260 * writing to the non-existent 'launch_state' field. The area doesn't 261 * have to be the currently active EVMCS on the calling CPU and there's 262 * nothing KVM has to do to transition it from 'active' to 'non-active' 263 * state. It is possible that the area will stay mapped as 264 * vmx->nested.hv_evmcs but this shouldn't be a problem. 265 */ 266 if (!guest_cpu_cap_has_evmcs(vcpu) || 267 !evmptr_is_valid(nested_get_evmptr(vcpu))) 268 return false; 269 270 if (nested_vmx_evmcs(vmx) && vmptr == vmx->nested.hv_evmcs_vmptr) 271 nested_release_evmcs(vcpu); 272 273 return true; 274 #else 275 return false; 276 #endif 277 } 278 279 static void vmx_sync_vmcs_host_state(struct vcpu_vmx *vmx, 280 struct loaded_vmcs *prev) 281 { 282 struct vmcs_host_state *dest, *src; 283 284 if (unlikely(!vmx->vt.guest_state_loaded)) 285 return; 286 287 src = &prev->host_state; 288 dest = &vmx->loaded_vmcs->host_state; 289 290 vmx_set_host_fs_gs(dest, src->fs_sel, src->gs_sel, src->fs_base, src->gs_base); 291 dest->ldt_sel = src->ldt_sel; 292 #ifdef CONFIG_X86_64 293 dest->ds_sel = src->ds_sel; 294 dest->es_sel = src->es_sel; 295 #endif 296 } 297 298 static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs) 299 { 300 struct vcpu_vmx *vmx = to_vmx(vcpu); 301 struct loaded_vmcs *prev; 302 int cpu; 303 304 if (WARN_ON_ONCE(vmx->loaded_vmcs == vmcs)) 305 return; 306 307 cpu = get_cpu(); 308 prev = vmx->loaded_vmcs; 309 vmx->loaded_vmcs = vmcs; 310 vmx_vcpu_load_vmcs(vcpu, cpu); 311 vmx_sync_vmcs_host_state(vmx, prev); 312 put_cpu(); 313 314 kvm_clear_available_registers(vcpu, VMX_REGS_LAZY_LOAD_SET); 315 316 /* 317 * All lazily updated registers will be reloaded from VMCS12 on both 318 * vmentry and vmexit. 319 */ 320 kvm_reset_dirty_registers(vcpu); 321 } 322 323 static void nested_put_vmcs12_pages(struct kvm_vcpu *vcpu) 324 { 325 struct vcpu_vmx *vmx = to_vmx(vcpu); 326 327 kvm_vcpu_unmap(vcpu, &vmx->nested.apic_access_page_map); 328 kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map); 329 kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map); 330 vmx->nested.pi_desc = NULL; 331 } 332 333 /* 334 * Free whatever needs to be freed from vmx->nested when L1 goes down, or 335 * just stops using VMX. 336 */ 337 static void free_nested(struct kvm_vcpu *vcpu) 338 { 339 struct vcpu_vmx *vmx = to_vmx(vcpu); 340 struct vmcs *shadow_vmcs; 341 342 if (WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01)) 343 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 344 345 if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon) 346 return; 347 348 kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 349 350 vmx->nested.vmxon = false; 351 vmx->nested.smm.vmxon = false; 352 vmx->nested.vmxon_ptr = INVALID_GPA; 353 free_vpid(vmx->nested.vpid02); 354 vmx->nested.posted_intr_nv = -1; 355 vmx->nested.current_vmptr = INVALID_GPA; 356 if (enable_shadow_vmcs) { 357 vmx_disable_shadow_vmcs(vmx); 358 359 /* 360 * Keep the pointer visible until after VMCLEAR, so migration 361 * can clear an active shadow VMCS on the old CPU. 362 */ 363 shadow_vmcs = vmx->vmcs01.shadow_vmcs; 364 vmcs_clear(shadow_vmcs); 365 vmx->vmcs01.shadow_vmcs = NULL; 366 free_vmcs(shadow_vmcs); 367 } 368 kfree(vmx->nested.cached_vmcs12); 369 vmx->nested.cached_vmcs12 = NULL; 370 kfree(vmx->nested.cached_shadow_vmcs12); 371 vmx->nested.cached_shadow_vmcs12 = NULL; 372 373 nested_put_vmcs12_pages(vcpu); 374 375 kvm_mmu_free_roots(vcpu->kvm, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 376 377 nested_release_evmcs(vcpu); 378 379 free_loaded_vmcs(&vmx->nested.vmcs02); 380 } 381 382 /* 383 * Ensure that the current vmcs of the logical processor is the 384 * vmcs01 of the vcpu before calling free_nested(). 385 */ 386 void nested_vmx_free_vcpu(struct kvm_vcpu *vcpu) 387 { 388 vcpu_load(vcpu); 389 vmx_leave_nested(vcpu); 390 vcpu_put(vcpu); 391 } 392 393 #define EPTP_PA_MASK GENMASK_ULL(51, 12) 394 395 static bool nested_ept_root_matches(hpa_t root_hpa, u64 root_eptp, u64 eptp) 396 { 397 return VALID_PAGE(root_hpa) && 398 ((root_eptp & EPTP_PA_MASK) == (eptp & EPTP_PA_MASK)); 399 } 400 401 static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp, 402 gpa_t addr) 403 { 404 unsigned long roots = 0; 405 uint i; 406 struct kvm_mmu_root_info *cached_root; 407 408 WARN_ON_ONCE(!mmu_is_nested(vcpu)); 409 410 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { 411 cached_root = &vcpu->arch.mmu->prev_roots[i]; 412 413 if (nested_ept_root_matches(cached_root->hpa, cached_root->pgd, 414 eptp)) 415 roots |= KVM_MMU_ROOT_PREVIOUS(i); 416 } 417 if (roots) 418 kvm_mmu_invalidate_addr(vcpu, &vcpu->arch.ngpa_walk, addr, roots); 419 } 420 421 static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, 422 struct x86_exception *fault, 423 bool from_hardware) 424 { 425 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 426 struct vcpu_vmx *vmx = to_vmx(vcpu); 427 unsigned long exit_qualification; 428 u32 vm_exit_reason; 429 430 if (vmx->nested.pml_full) { 431 vm_exit_reason = EXIT_REASON_PML_FULL; 432 vmx->nested.pml_full = false; 433 434 /* 435 * It should be impossible to trigger a nested PML Full VM-Exit 436 * for anything other than an EPT Violation from L2. KVM *can* 437 * trigger nEPT page fault injection in response to an EPT 438 * Misconfig, e.g. if the MMIO SPTE was stale and L1's EPT 439 * tables also changed, but KVM should not treat EPT Misconfig 440 * VM-Exits as writes. 441 */ 442 WARN_ON_ONCE(vmx->vt.exit_reason.basic != EXIT_REASON_EPT_VIOLATION); 443 444 /* 445 * PML Full and EPT Violation VM-Exits both use bit 12 to report 446 * "NMI unblocking due to IRET", i.e. the bit can be propagated 447 * as-is from the original EXIT_QUALIFICATION. 448 */ 449 exit_qualification = vt_get_exit_qual(vcpu) & INTR_INFO_UNBLOCK_NMI; 450 } else { 451 if (fault->error_code & PFERR_RSVD_MASK) { 452 vm_exit_reason = EXIT_REASON_EPT_MISCONFIG; 453 exit_qualification = 0; 454 } else { 455 u64 mask = EPT_VIOLATION_GVA_IS_VALID | 456 EPT_VIOLATION_GVA_TRANSLATED; 457 458 if (vmx->nested.msrs.ept_caps & VMX_EPT_ADVANCED_VMEXIT_INFO_BIT) 459 mask |= EPT_VIOLATION_GVA_USER | 460 EPT_VIOLATION_GVA_WRITABLE | 461 EPT_VIOLATION_GVA_NX; 462 463 exit_qualification = fault->exit_qualification & ~mask; 464 465 /* 466 * Use the EXIT_QUALIFICATION from the VMCS if and only 467 * if the hardware VM-Exit from L2 was an EPT Violation. 468 * If the fault is synthesized, then EXIT_QUALIFICATION 469 * is stale and/or holds entirely different data. And 470 * conversely, KVM _must_ rely on EXIT_QUALIFICATION if 471 * the fault came from hardware, because KVM only sees 472 * and walks the faulting GPA. 473 */ 474 if (from_hardware) 475 exit_qualification |= vt_get_exit_qual(vcpu) & mask; 476 else 477 exit_qualification |= fault->exit_qualification & mask; 478 479 vm_exit_reason = EXIT_REASON_EPT_VIOLATION; 480 } 481 482 /* 483 * Although the caller (kvm_inject_emulated_page_fault) would 484 * have already synced the faulting address in the shadow EPT 485 * tables for the current EPTP12, we also need to sync it for 486 * any other cached EPTP02s based on the same EP4TA, since the 487 * TLB associates mappings to the EP4TA rather than the full EPTP. 488 */ 489 nested_ept_invalidate_addr(vcpu, vmcs12->ept_pointer, 490 fault->address); 491 } 492 493 nested_vmx_vmexit(vcpu, vm_exit_reason, 0, exit_qualification); 494 vmcs12->guest_physical_address = fault->address; 495 } 496 497 static inline bool nested_ept_mbec_enabled(struct kvm_vcpu *vcpu) 498 { 499 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 500 501 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_MODE_BASED_EPT_EXEC); 502 } 503 504 static void nested_ept_new_eptp(struct kvm_vcpu *vcpu) 505 { 506 struct vcpu_vmx *vmx = to_vmx(vcpu); 507 bool execonly = vmx->nested.msrs.ept_caps & VMX_EPT_EXECUTE_ONLY_BIT; 508 int ept_lpage_level = ept_caps_to_lpage_level(vmx->nested.msrs.ept_caps); 509 510 kvm_init_shadow_ept_mmu(vcpu, execonly, ept_lpage_level, 511 nested_ept_ad_enabled(vcpu), 512 nested_ept_mbec_enabled(vcpu), 513 nested_ept_get_eptp(vcpu)); 514 } 515 516 static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu) 517 { 518 WARN_ON(mmu_is_nested(vcpu)); 519 520 vcpu->arch.mmu = &vcpu->arch.guest_mmu; 521 nested_ept_new_eptp(vcpu); 522 vcpu->arch.ngpa_walk.get_guest_pgd = nested_ept_get_eptp; 523 vcpu->arch.ngpa_walk.get_pdptr = kvm_pdptr_read; 524 525 vcpu->arch.ngpa_walk.inject_page_fault = nested_ept_inject_page_fault; 526 } 527 528 static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu) 529 { 530 vcpu->arch.mmu = &vcpu->arch.root_mmu; 531 } 532 533 static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12, 534 u16 error_code) 535 { 536 bool inequality, bit; 537 538 bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0; 539 inequality = 540 (error_code & vmcs12->page_fault_error_code_mask) != 541 vmcs12->page_fault_error_code_match; 542 return inequality ^ bit; 543 } 544 545 static bool nested_vmx_is_exception_vmexit(struct kvm_vcpu *vcpu, u8 vector, 546 u32 error_code) 547 { 548 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 549 550 /* 551 * Drop bits 31:16 of the error code when performing the #PF mask+match 552 * check. All VMCS fields involved are 32 bits, but Intel CPUs never 553 * set bits 31:16 and VMX disallows setting bits 31:16 in the injected 554 * error code. Including the to-be-dropped bits in the check might 555 * result in an "impossible" or missed exit from L1's perspective. 556 */ 557 if (vector == PF_VECTOR) 558 return nested_vmx_is_page_fault_vmexit(vmcs12, (u16)error_code); 559 560 return (vmcs12->exception_bitmap & (1u << vector)); 561 } 562 563 static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu, 564 struct vmcs12 *vmcs12) 565 { 566 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 567 return 0; 568 569 if (CC(!page_address_valid(vcpu, vmcs12->io_bitmap_a)) || 570 CC(!page_address_valid(vcpu, vmcs12->io_bitmap_b))) 571 return -EINVAL; 572 573 return 0; 574 } 575 576 static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu, 577 struct vmcs12 *vmcs12) 578 { 579 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 580 return 0; 581 582 if (CC(!page_address_valid(vcpu, vmcs12->msr_bitmap))) 583 return -EINVAL; 584 585 return 0; 586 } 587 588 static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, 589 struct vmcs12 *vmcs12) 590 { 591 gpa_t vtpr_gpa = vmcs12->virtual_apic_page_addr + APIC_TASKPRI; 592 u32 vtpr; 593 594 if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) 595 return 0; 596 597 if (CC(!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr))) 598 return -EINVAL; 599 600 if (CC(!nested_cpu_has_vid(vmcs12) && vmcs12->tpr_threshold >> 4)) 601 return -EINVAL; 602 603 /* 604 * Do the illegal vTPR vs. TPR Threshold consistency check if and only 605 * if KVM is configured to WARN on missed consistency checks, otherwise 606 * it's a waste of time. KVM needs to rely on hardware to fully detect 607 * an illegal combination due to the vTPR being writable by L1 at all 608 * times (it's an in-memory value, not a VMCS field). I.e. even if the 609 * check passes now, it might fail at the actual VM-Enter. 610 * 611 * If reading guest memory fails, skip the check as KVM's de facto ABI 612 * for VMX instruction accesses to non-existent memory is to provide 613 * PCI Bus Error semantics (reads return 0xFFs), in which case the vTPR 614 * is guaranteed to greater than or equal to the threshold. 615 * 616 * Note! Deliberately use the VM-scoped API when reading guest memory, 617 * to ensure the read doesn't hit SMRAM when restoring L2 state on RSM, 618 * and only perform the check when in KVM_RUN, to avoid a false failure 619 * if userspace hasn't yet configured memslots during state restore. 620 */ 621 if (warn_on_missed_cc && vcpu->wants_to_run && 622 nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW) && 623 !nested_cpu_has_vid(vmcs12) && 624 !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && 625 !kvm_read_guest(vcpu->kvm, vtpr_gpa, &vtpr, sizeof(vtpr)) && 626 CC((vmcs12->tpr_threshold & GENMASK(3, 0)) > ((vtpr >> 4) & GENMASK(3, 0)))) 627 return -EINVAL; 628 629 return 0; 630 } 631 632 /* 633 * For x2APIC MSRs, ignore the vmcs01 bitmap. L1 can enable x2APIC without L1 634 * itself utilizing x2APIC. All MSRs were previously set to be intercepted, 635 * only the "disable intercept" case needs to be handled. 636 */ 637 static void nested_vmx_disable_intercept_for_x2apic_msr(unsigned long *msr_bitmap_l1, 638 unsigned long *msr_bitmap_l0, 639 u32 msr, int type) 640 { 641 if (type & MSR_TYPE_R && !vmx_test_msr_bitmap_read(msr_bitmap_l1, msr)) 642 vmx_clear_msr_bitmap_read(msr_bitmap_l0, msr); 643 644 if (type & MSR_TYPE_W && !vmx_test_msr_bitmap_write(msr_bitmap_l1, msr)) 645 vmx_clear_msr_bitmap_write(msr_bitmap_l0, msr); 646 } 647 648 static inline void enable_x2apic_msr_intercepts(unsigned long *msr_bitmap) 649 { 650 int msr; 651 652 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 653 unsigned word = msr / BITS_PER_LONG; 654 655 msr_bitmap[word] = ~0; 656 msr_bitmap[word + (0x800 / sizeof(long))] = ~0; 657 } 658 } 659 660 #define BUILD_NVMX_MSR_INTERCEPT_HELPER(rw) \ 661 static inline \ 662 void nested_vmx_set_msr_##rw##_intercept(struct vcpu_vmx *vmx, \ 663 unsigned long *msr_bitmap_l1, \ 664 unsigned long *msr_bitmap_l0, u32 msr) \ 665 { \ 666 if (vmx_test_msr_bitmap_##rw(vmx->vmcs01.msr_bitmap, msr) || \ 667 vmx_test_msr_bitmap_##rw(msr_bitmap_l1, msr)) \ 668 vmx_set_msr_bitmap_##rw(msr_bitmap_l0, msr); \ 669 else \ 670 vmx_clear_msr_bitmap_##rw(msr_bitmap_l0, msr); \ 671 } 672 BUILD_NVMX_MSR_INTERCEPT_HELPER(read) 673 BUILD_NVMX_MSR_INTERCEPT_HELPER(write) 674 675 static inline void nested_vmx_set_intercept_for_msr(struct vcpu_vmx *vmx, 676 unsigned long *msr_bitmap_l1, 677 unsigned long *msr_bitmap_l0, 678 u32 msr, int types) 679 { 680 if (types & MSR_TYPE_R) 681 nested_vmx_set_msr_read_intercept(vmx, msr_bitmap_l1, 682 msr_bitmap_l0, msr); 683 if (types & MSR_TYPE_W) 684 nested_vmx_set_msr_write_intercept(vmx, msr_bitmap_l1, 685 msr_bitmap_l0, msr); 686 } 687 688 #define nested_vmx_merge_msr_bitmaps(msr, type) \ 689 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, \ 690 msr_bitmap_l0, msr, type) 691 692 #define nested_vmx_merge_msr_bitmaps_read(msr) \ 693 nested_vmx_merge_msr_bitmaps(msr, MSR_TYPE_R) 694 695 #define nested_vmx_merge_msr_bitmaps_write(msr) \ 696 nested_vmx_merge_msr_bitmaps(msr, MSR_TYPE_W) 697 698 #define nested_vmx_merge_msr_bitmaps_rw(msr) \ 699 nested_vmx_merge_msr_bitmaps(msr, MSR_TYPE_RW) 700 701 static void nested_vmx_merge_pmu_msr_bitmaps(struct kvm_vcpu *vcpu, 702 unsigned long *msr_bitmap_l1, 703 unsigned long *msr_bitmap_l0) 704 { 705 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 706 struct vcpu_vmx *vmx = to_vmx(vcpu); 707 int i; 708 709 /* 710 * Skip the merges if the vCPU doesn't have a mediated PMU MSR, i.e. if 711 * none of the MSRs can possibly be passed through to L1. 712 */ 713 if (!kvm_vcpu_has_mediated_pmu(vcpu)) 714 return; 715 716 for (i = 0; i < pmu->nr_arch_gp_counters; i++) { 717 nested_vmx_merge_msr_bitmaps_rw(MSR_IA32_PERFCTR0 + i); 718 nested_vmx_merge_msr_bitmaps_rw(MSR_IA32_PMC0 + i); 719 } 720 721 for (i = 0; i < pmu->nr_arch_fixed_counters; i++) 722 nested_vmx_merge_msr_bitmaps_rw(MSR_CORE_PERF_FIXED_CTR0 + i); 723 724 nested_vmx_merge_msr_bitmaps_rw(MSR_CORE_PERF_GLOBAL_CTRL); 725 nested_vmx_merge_msr_bitmaps_read(MSR_CORE_PERF_GLOBAL_STATUS); 726 nested_vmx_merge_msr_bitmaps_write(MSR_CORE_PERF_GLOBAL_OVF_CTRL); 727 } 728 729 /* 730 * Merge L0's and L1's MSR bitmap, return false to indicate that 731 * we do not use the hardware. 732 */ 733 static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, 734 struct vmcs12 *vmcs12) 735 { 736 struct vcpu_vmx *vmx = to_vmx(vcpu); 737 int msr; 738 unsigned long *msr_bitmap_l1; 739 unsigned long *msr_bitmap_l0 = vmx->nested.vmcs02.msr_bitmap; 740 741 /* Nothing to do if the MSR bitmap is not in use. */ 742 if (!cpu_has_vmx_msr_bitmap() || 743 !nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 744 return false; 745 746 /* 747 * MSR bitmap update can be skipped when: 748 * - MSR bitmap for L1 hasn't changed. 749 * - Nested hypervisor (L1) is attempting to launch the same L2 as 750 * before. 751 * - Nested hypervisor (L1) has enabled 'Enlightened MSR Bitmap' feature 752 * and tells KVM (L0) there were no changes in MSR bitmap for L2. 753 */ 754 if (!vmx->nested.force_msr_bitmap_recalc) { 755 struct hv_enlightened_vmcs *evmcs = nested_vmx_evmcs(vmx); 756 757 if (evmcs && evmcs->hv_enlightenments_control.msr_bitmap && 758 evmcs->hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP) 759 return true; 760 } 761 762 CLASS(kvm_vcpu_map_local_readonly, m)(vcpu, gpa_to_gfn(vmcs12->msr_bitmap)); 763 if (m.ret) 764 return false; 765 766 msr_bitmap_l1 = (unsigned long *)m.map.hva; 767 768 /* 769 * To keep the control flow simple, pay eight 8-byte writes (sixteen 770 * 4-byte writes on 32-bit systems) up front to enable intercepts for 771 * the x2APIC MSR range and selectively toggle those relevant to L2. 772 */ 773 enable_x2apic_msr_intercepts(msr_bitmap_l0); 774 775 if (nested_cpu_has_virt_x2apic_mode(vmcs12)) { 776 if (nested_cpu_has_apic_reg_virt(vmcs12)) { 777 /* 778 * L0 need not intercept reads for MSRs between 0x800 779 * and 0x8ff, it just lets the processor take the value 780 * from the virtual-APIC page; take those 256 bits 781 * directly from the L1 bitmap. 782 */ 783 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 784 unsigned word = msr / BITS_PER_LONG; 785 786 msr_bitmap_l0[word] = msr_bitmap_l1[word]; 787 } 788 } 789 790 nested_vmx_disable_intercept_for_x2apic_msr( 791 msr_bitmap_l1, msr_bitmap_l0, 792 X2APIC_MSR(APIC_TASKPRI), 793 MSR_TYPE_R | MSR_TYPE_W); 794 795 if (nested_cpu_has_vid(vmcs12)) { 796 nested_vmx_disable_intercept_for_x2apic_msr( 797 msr_bitmap_l1, msr_bitmap_l0, 798 X2APIC_MSR(APIC_EOI), 799 MSR_TYPE_W); 800 nested_vmx_disable_intercept_for_x2apic_msr( 801 msr_bitmap_l1, msr_bitmap_l0, 802 X2APIC_MSR(APIC_SELF_IPI), 803 MSR_TYPE_W); 804 } 805 } 806 807 /* 808 * Always check vmcs01's bitmap to honor userspace MSR filters and any 809 * other runtime changes to vmcs01's bitmap, e.g. dynamic pass-through. 810 */ 811 #ifdef CONFIG_X86_64 812 nested_vmx_merge_msr_bitmaps_rw(MSR_FS_BASE); 813 nested_vmx_merge_msr_bitmaps_rw(MSR_GS_BASE); 814 nested_vmx_merge_msr_bitmaps_rw(MSR_KERNEL_GS_BASE); 815 #endif 816 nested_vmx_merge_msr_bitmaps_rw(MSR_IA32_SPEC_CTRL); 817 nested_vmx_merge_msr_bitmaps_write(MSR_IA32_PRED_CMD); 818 nested_vmx_merge_msr_bitmaps_write(MSR_IA32_FLUSH_CMD); 819 820 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 821 MSR_IA32_APERF, MSR_TYPE_R); 822 823 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 824 MSR_IA32_MPERF, MSR_TYPE_R); 825 826 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 827 MSR_IA32_U_CET, MSR_TYPE_RW); 828 829 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 830 MSR_IA32_S_CET, MSR_TYPE_RW); 831 832 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 833 MSR_IA32_PL0_SSP, MSR_TYPE_RW); 834 835 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 836 MSR_IA32_PL1_SSP, MSR_TYPE_RW); 837 838 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 839 MSR_IA32_PL2_SSP, MSR_TYPE_RW); 840 841 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 842 MSR_IA32_PL3_SSP, MSR_TYPE_RW); 843 844 nested_vmx_merge_pmu_msr_bitmaps(vcpu, msr_bitmap_l1, msr_bitmap_l0); 845 846 vmx->nested.force_msr_bitmap_recalc = false; 847 848 return true; 849 } 850 851 static void nested_cache_shadow_vmcs12(struct kvm_vcpu *vcpu, 852 struct vmcs12 *vmcs12) 853 { 854 struct vcpu_vmx *vmx = to_vmx(vcpu); 855 struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; 856 857 if (!nested_cpu_has_shadow_vmcs(vmcs12) || 858 vmcs12->vmcs_link_pointer == INVALID_GPA) 859 return; 860 861 if (ghc->gpa != vmcs12->vmcs_link_pointer && 862 kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, 863 vmcs12->vmcs_link_pointer, VMCS12_SIZE)) 864 return; 865 866 kvm_read_guest_cached(vcpu->kvm, ghc, get_shadow_vmcs12(vcpu), 867 VMCS12_SIZE); 868 } 869 870 static void nested_flush_cached_shadow_vmcs12(struct kvm_vcpu *vcpu, 871 struct vmcs12 *vmcs12) 872 { 873 struct vcpu_vmx *vmx = to_vmx(vcpu); 874 struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; 875 876 if (!nested_cpu_has_shadow_vmcs(vmcs12) || 877 vmcs12->vmcs_link_pointer == INVALID_GPA) 878 return; 879 880 if (ghc->gpa != vmcs12->vmcs_link_pointer && 881 kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, 882 vmcs12->vmcs_link_pointer, VMCS12_SIZE)) 883 return; 884 885 kvm_write_guest_cached(vcpu->kvm, ghc, get_shadow_vmcs12(vcpu), 886 VMCS12_SIZE); 887 } 888 889 /* 890 * In nested virtualization, check if L1 has set 891 * VM_EXIT_ACK_INTR_ON_EXIT 892 */ 893 static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu) 894 { 895 return get_vmcs12(vcpu)->vm_exit_controls & 896 VM_EXIT_ACK_INTR_ON_EXIT; 897 } 898 899 static int nested_vmx_check_apic_access_controls(struct kvm_vcpu *vcpu, 900 struct vmcs12 *vmcs12) 901 { 902 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && 903 CC(!page_address_valid(vcpu, vmcs12->apic_access_addr))) 904 return -EINVAL; 905 else 906 return 0; 907 } 908 909 static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu, 910 struct vmcs12 *vmcs12) 911 { 912 if (!nested_cpu_has_virt_x2apic_mode(vmcs12) && 913 !nested_cpu_has_apic_reg_virt(vmcs12) && 914 !nested_cpu_has_vid(vmcs12) && 915 !nested_cpu_has_posted_intr(vmcs12)) 916 return 0; 917 918 /* 919 * If virtualize x2apic mode is enabled, 920 * virtualize apic access must be disabled. 921 */ 922 if (CC(nested_cpu_has_virt_x2apic_mode(vmcs12) && 923 nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))) 924 return -EINVAL; 925 926 /* 927 * If virtual interrupt delivery is enabled, 928 * we must exit on external interrupts. 929 */ 930 if (CC(nested_cpu_has_vid(vmcs12) && !nested_exit_on_intr(vcpu))) 931 return -EINVAL; 932 933 /* 934 * bits 15:8 should be zero in posted_intr_nv, 935 * the descriptor address has been already checked 936 * in nested_get_vmcs12_pages. 937 * 938 * bits 5:0 of posted_intr_desc_addr should be zero. 939 */ 940 if (nested_cpu_has_posted_intr(vmcs12) && 941 (CC(!nested_cpu_has_vid(vmcs12)) || 942 CC(!nested_exit_intr_ack_set(vcpu)) || 943 CC((vmcs12->posted_intr_nv & 0xff00)) || 944 CC(!kvm_vcpu_is_legal_aligned_gpa(vcpu, vmcs12->posted_intr_desc_addr, 64)))) 945 return -EINVAL; 946 947 /* tpr shadow is needed by all apicv features. */ 948 if (CC(!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))) 949 return -EINVAL; 950 951 return 0; 952 } 953 954 static u32 nested_vmx_max_atomic_switch_msrs(struct kvm_vcpu *vcpu) 955 { 956 struct vcpu_vmx *vmx = to_vmx(vcpu); 957 u64 vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low, 958 vmx->nested.msrs.misc_high); 959 960 return (vmx_misc_max_msr(vmx_misc) + 1) * VMX_MISC_MSR_LIST_MULTIPLIER; 961 } 962 963 static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu, 964 u32 count, u64 addr) 965 { 966 if (count == 0) 967 return 0; 968 969 /* 970 * Exceeding the limit results in architecturally _undefined_ behavior, 971 * i.e. KVM is allowed to do literally anything in response to a bad 972 * limit. Immediately generate a consistency check so that code that 973 * consumes the count doesn't need to worry about extreme edge cases. 974 */ 975 if (count > nested_vmx_max_atomic_switch_msrs(vcpu)) 976 return -EINVAL; 977 978 if (!kvm_vcpu_is_legal_aligned_gpa(vcpu, addr, 16) || 979 !kvm_vcpu_is_legal_gpa(vcpu, (addr + count * sizeof(struct vmx_msr_entry) - 1))) 980 return -EINVAL; 981 982 return 0; 983 } 984 985 static int nested_vmx_check_exit_msr_switch_controls(struct kvm_vcpu *vcpu, 986 struct vmcs12 *vmcs12) 987 { 988 if (CC(nested_vmx_check_msr_switch(vcpu, 989 vmcs12->vm_exit_msr_load_count, 990 vmcs12->vm_exit_msr_load_addr)) || 991 CC(nested_vmx_check_msr_switch(vcpu, 992 vmcs12->vm_exit_msr_store_count, 993 vmcs12->vm_exit_msr_store_addr))) 994 return -EINVAL; 995 996 return 0; 997 } 998 999 static int nested_vmx_check_entry_msr_switch_controls(struct kvm_vcpu *vcpu, 1000 struct vmcs12 *vmcs12) 1001 { 1002 if (CC(nested_vmx_check_msr_switch(vcpu, 1003 vmcs12->vm_entry_msr_load_count, 1004 vmcs12->vm_entry_msr_load_addr))) 1005 return -EINVAL; 1006 1007 return 0; 1008 } 1009 1010 static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu, 1011 struct vmcs12 *vmcs12) 1012 { 1013 if (!nested_cpu_has_pml(vmcs12)) 1014 return 0; 1015 1016 if (CC(!nested_cpu_has_ept(vmcs12)) || 1017 CC(!page_address_valid(vcpu, vmcs12->pml_address))) 1018 return -EINVAL; 1019 1020 return 0; 1021 } 1022 1023 static int nested_vmx_check_unrestricted_guest_controls(struct kvm_vcpu *vcpu, 1024 struct vmcs12 *vmcs12) 1025 { 1026 if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST) && 1027 !nested_cpu_has_ept(vmcs12))) 1028 return -EINVAL; 1029 return 0; 1030 } 1031 1032 static int nested_vmx_check_mode_based_ept_exec_controls(struct kvm_vcpu *vcpu, 1033 struct vmcs12 *vmcs12) 1034 { 1035 if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_MODE_BASED_EPT_EXEC) && 1036 !nested_cpu_has_ept(vmcs12))) 1037 return -EINVAL; 1038 return 0; 1039 } 1040 1041 static int nested_vmx_check_shadow_vmcs_controls(struct kvm_vcpu *vcpu, 1042 struct vmcs12 *vmcs12) 1043 { 1044 if (!nested_cpu_has_shadow_vmcs(vmcs12)) 1045 return 0; 1046 1047 if (CC(!page_address_valid(vcpu, vmcs12->vmread_bitmap)) || 1048 CC(!page_address_valid(vcpu, vmcs12->vmwrite_bitmap))) 1049 return -EINVAL; 1050 1051 return 0; 1052 } 1053 1054 static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu, 1055 struct vmx_msr_entry *e) 1056 { 1057 /* x2APIC MSR accesses are not allowed */ 1058 if (CC(vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8)) 1059 return -EINVAL; 1060 if (CC(e->index == MSR_IA32_UCODE_WRITE) || /* SDM Table 35-2 */ 1061 CC(e->index == MSR_IA32_UCODE_REV)) 1062 return -EINVAL; 1063 if (CC(e->reserved != 0)) 1064 return -EINVAL; 1065 return 0; 1066 } 1067 1068 static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu, 1069 struct vmx_msr_entry *e) 1070 { 1071 if (CC(e->index == MSR_FS_BASE) || 1072 CC(e->index == MSR_GS_BASE) || 1073 CC(e->index == MSR_IA32_SMM_MONITOR_CTL) || /* SMM is not supported */ 1074 nested_vmx_msr_check_common(vcpu, e)) 1075 return -EINVAL; 1076 return 0; 1077 } 1078 1079 static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu, 1080 struct vmx_msr_entry *e) 1081 { 1082 if (CC(e->index == MSR_IA32_SMBASE) || /* SMM is not supported */ 1083 nested_vmx_msr_check_common(vcpu, e)) 1084 return -EINVAL; 1085 return 0; 1086 } 1087 1088 /* 1089 * Load guest's/host's msr at nested entry/exit. 1090 * return 0 for success, entry index for failure. 1091 * 1092 * One of the failure modes for MSR load/store is when a list exceeds the 1093 * virtual hardware's capacity. To maintain compatibility with hardware inasmuch 1094 * as possible, process all valid entries before failing rather than precheck 1095 * for a capacity violation. 1096 */ 1097 static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) 1098 { 1099 u32 i; 1100 struct vmx_msr_entry e; 1101 u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu); 1102 1103 for (i = 0; i < count; i++) { 1104 if (WARN_ON_ONCE(i >= max_msr_list_size)) 1105 goto fail; 1106 1107 if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e), 1108 &e, sizeof(e))) { 1109 pr_debug_ratelimited( 1110 "%s cannot read MSR entry (%u, 0x%08llx)\n", 1111 __func__, i, gpa + i * sizeof(e)); 1112 goto fail; 1113 } 1114 if (nested_vmx_load_msr_check(vcpu, &e)) { 1115 pr_debug_ratelimited( 1116 "%s check failed (%u, 0x%x, 0x%x)\n", 1117 __func__, i, e.index, e.reserved); 1118 goto fail; 1119 } 1120 if (kvm_emulate_msr_write(vcpu, e.index, e.value)) { 1121 pr_debug_ratelimited( 1122 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", 1123 __func__, i, e.index, e.value); 1124 goto fail; 1125 } 1126 } 1127 return 0; 1128 fail: 1129 /* Note, max_msr_list_size is at most 4096, i.e. this can't wrap. */ 1130 return i + 1; 1131 } 1132 1133 static bool nested_vmx_get_vmexit_msr_value(struct kvm_vcpu *vcpu, 1134 u32 msr_index, 1135 u64 *data) 1136 { 1137 struct vcpu_vmx *vmx = to_vmx(vcpu); 1138 1139 /* 1140 * If the L0 hypervisor stored a more accurate value for the TSC that 1141 * does not include the time taken for emulation of the L2->L1 1142 * VM-exit in L0, use the more accurate value. 1143 */ 1144 if (msr_index == MSR_IA32_TSC && vmx->nested.tsc_autostore_slot >= 0) { 1145 int slot = vmx->nested.tsc_autostore_slot; 1146 u64 host_tsc = vmx->msr_autostore.val[slot].value; 1147 1148 *data = kvm_read_l1_tsc(vcpu, host_tsc); 1149 return true; 1150 } 1151 1152 if (kvm_emulate_msr_read(vcpu, msr_index, data)) { 1153 pr_debug_ratelimited("%s cannot read MSR (0x%x)\n", __func__, 1154 msr_index); 1155 return false; 1156 } 1157 return true; 1158 } 1159 1160 static bool read_and_check_msr_entry(struct kvm_vcpu *vcpu, u64 gpa, int i, 1161 struct vmx_msr_entry *e) 1162 { 1163 if (kvm_vcpu_read_guest(vcpu, 1164 gpa + i * sizeof(*e), 1165 e, 2 * sizeof(u32))) { 1166 pr_debug_ratelimited( 1167 "%s cannot read MSR entry (%u, 0x%08llx)\n", 1168 __func__, i, gpa + i * sizeof(*e)); 1169 return false; 1170 } 1171 if (nested_vmx_store_msr_check(vcpu, e)) { 1172 pr_debug_ratelimited( 1173 "%s check failed (%u, 0x%x, 0x%x)\n", 1174 __func__, i, e->index, e->reserved); 1175 return false; 1176 } 1177 return true; 1178 } 1179 1180 static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) 1181 { 1182 u64 data; 1183 u32 i; 1184 struct vmx_msr_entry e; 1185 u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu); 1186 1187 for (i = 0; i < count; i++) { 1188 if (WARN_ON_ONCE(i >= max_msr_list_size)) 1189 return -EINVAL; 1190 1191 if (!read_and_check_msr_entry(vcpu, gpa, i, &e)) 1192 return -EINVAL; 1193 1194 if (!nested_vmx_get_vmexit_msr_value(vcpu, e.index, &data)) 1195 return -EINVAL; 1196 1197 if (kvm_vcpu_write_guest(vcpu, 1198 gpa + i * sizeof(e) + 1199 offsetof(struct vmx_msr_entry, value), 1200 &data, sizeof(data))) { 1201 pr_debug_ratelimited( 1202 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", 1203 __func__, i, e.index, data); 1204 return -EINVAL; 1205 } 1206 } 1207 return 0; 1208 } 1209 1210 static bool nested_msr_store_list_has_msr(struct kvm_vcpu *vcpu, u32 msr_index) 1211 { 1212 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1213 u32 count = vmcs12->vm_exit_msr_store_count; 1214 u64 gpa = vmcs12->vm_exit_msr_store_addr; 1215 struct vmx_msr_entry e; 1216 u32 i; 1217 1218 for (i = 0; i < count; i++) { 1219 if (!read_and_check_msr_entry(vcpu, gpa, i, &e)) 1220 return false; 1221 1222 if (e.index == msr_index) 1223 return true; 1224 } 1225 return false; 1226 } 1227 1228 /* 1229 * Load guest's/host's cr3 at nested entry/exit. @nested_ept is true if we are 1230 * emulating VM-Entry into a guest with EPT enabled. On failure, the expected 1231 * Exit Qualification (for a VM-Entry consistency check VM-Exit) is assigned to 1232 * @entry_failure_code. 1233 */ 1234 static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, 1235 bool nested_ept, bool reload_pdptrs, 1236 enum vm_entry_failure_code *entry_failure_code) 1237 { 1238 if (CC(!kvm_vcpu_is_legal_cr3(vcpu, cr3))) { 1239 *entry_failure_code = ENTRY_FAIL_DEFAULT; 1240 return -EINVAL; 1241 } 1242 1243 /* 1244 * If PAE paging and EPT are both on, CR3 is not used by the CPU and 1245 * must not be dereferenced. 1246 */ 1247 if (reload_pdptrs && !nested_ept && is_pae_paging(vcpu) && 1248 CC(!load_pdptrs(vcpu, cr3))) { 1249 *entry_failure_code = ENTRY_FAIL_PDPTE; 1250 return -EINVAL; 1251 } 1252 1253 vcpu->arch.cr3 = cr3; 1254 kvm_register_mark_dirty(vcpu, VCPU_REG_CR3); 1255 1256 /* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */ 1257 kvm_init_mmu(vcpu); 1258 1259 if (!nested_ept) 1260 kvm_mmu_new_pgd(vcpu, cr3); 1261 1262 return 0; 1263 } 1264 1265 /* 1266 * Returns if KVM is able to config CPU to tag TLB entries 1267 * populated by L2 differently than TLB entries populated 1268 * by L1. 1269 * 1270 * If L0 uses EPT, L1 and L2 run with different EPTP because 1271 * guest_mode is part of kvm_mmu_page_role. Thus, TLB entries 1272 * are tagged with different EPTP. 1273 * 1274 * If L1 uses VPID and we allocated a vpid02, TLB entries are tagged 1275 * with different VPID (L1 entries are tagged with vmx->vpid 1276 * while L2 entries are tagged with vmx->nested.vpid02). 1277 */ 1278 static bool nested_has_guest_tlb_tag(struct kvm_vcpu *vcpu) 1279 { 1280 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1281 1282 return enable_ept || 1283 (nested_cpu_has_vpid(vmcs12) && to_vmx(vcpu)->nested.vpid02); 1284 } 1285 1286 static void nested_vmx_transition_tlb_flush(struct kvm_vcpu *vcpu, 1287 struct vmcs12 *vmcs12, 1288 bool is_vmenter) 1289 { 1290 struct vcpu_vmx *vmx = to_vmx(vcpu); 1291 1292 /* Handle pending Hyper-V TLB flush requests */ 1293 kvm_hv_nested_transtion_tlb_flush(vcpu, enable_ept); 1294 1295 /* 1296 * If VPID is disabled, then guest TLB accesses use VPID=0, i.e. the 1297 * same VPID as the host, and so architecturally, linear and combined 1298 * mappings for VPID=0 must be flushed at VM-Enter and VM-Exit. KVM 1299 * emulates L2 sharing L1's VPID=0 by using vpid01 while running L2, 1300 * and so KVM must also emulate TLB flush of VPID=0, i.e. vpid01. This 1301 * is required if VPID is disabled in KVM, as a TLB flush (there are no 1302 * VPIDs) still occurs from L1's perspective, and KVM may need to 1303 * synchronize the MMU in response to the guest TLB flush. 1304 * 1305 * Note, using TLB_FLUSH_GUEST is correct even if nested EPT is in use. 1306 * EPT is a special snowflake, as guest-physical mappings aren't 1307 * flushed on VPID invalidations, including VM-Enter or VM-Exit with 1308 * VPID disabled. As a result, KVM _never_ needs to sync nEPT 1309 * entries on VM-Enter because L1 can't rely on VM-Enter to flush 1310 * those mappings. 1311 */ 1312 if (!nested_cpu_has_vpid(vmcs12)) { 1313 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 1314 return; 1315 } 1316 1317 /* L2 should never have a VPID if VPID is disabled. */ 1318 WARN_ON(!enable_vpid); 1319 1320 /* 1321 * VPID is enabled and in use by vmcs12. If vpid12 is changing, then 1322 * emulate a guest TLB flush as KVM does not track vpid12 history nor 1323 * is the VPID incorporated into the MMU context. I.e. KVM must assume 1324 * that the new vpid12 has never been used and thus represents a new 1325 * guest ASID that cannot have entries in the TLB. 1326 * 1327 * Note, last_vpid is initialized as 0, so the first nested VM-Enter 1328 * after VMXON will always flush the TLB to avoid using stale entries. 1329 */ 1330 if (is_vmenter && vmcs12->virtual_processor_id != vmx->nested.last_vpid) { 1331 vmx->nested.last_vpid = vmcs12->virtual_processor_id; 1332 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 1333 return; 1334 } 1335 1336 /* 1337 * If VPID is enabled, used by vmc12, and vpid12 is not changing but 1338 * does not have a unique TLB tag (ASID), i.e. EPT is disabled and 1339 * KVM was unable to allocate a VPID for L2, flush the current context 1340 * as the effective ASID is common to both L1 and L2. 1341 */ 1342 if (!nested_has_guest_tlb_tag(vcpu)) 1343 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 1344 } 1345 1346 static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask) 1347 { 1348 superset &= mask; 1349 subset &= mask; 1350 1351 return (superset | subset) == superset; 1352 } 1353 1354 static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data) 1355 { 1356 const u64 feature_bits = VMX_BASIC_DUAL_MONITOR_TREATMENT | 1357 VMX_BASIC_INOUT | 1358 VMX_BASIC_TRUE_CTLS | 1359 VMX_BASIC_NO_HW_ERROR_CODE_CC; 1360 1361 const u64 reserved_bits = GENMASK_ULL(63, 57) | 1362 GENMASK_ULL(47, 45) | 1363 BIT_ULL(31); 1364 1365 u64 vmx_basic = vmcs_config.nested.basic; 1366 1367 BUILD_BUG_ON(feature_bits & reserved_bits); 1368 1369 /* 1370 * Except for 32BIT_PHYS_ADDR_ONLY, which is an anti-feature bit (has 1371 * inverted polarity), the incoming value must not set feature bits or 1372 * reserved bits that aren't allowed/supported by KVM. Fields, i.e. 1373 * multi-bit values, are explicitly checked below. 1374 */ 1375 if (!is_bitwise_subset(vmx_basic, data, feature_bits | reserved_bits)) 1376 return -EINVAL; 1377 1378 /* 1379 * KVM does not emulate a version of VMX that constrains physical 1380 * addresses of VMX structures (e.g. VMCS) to 32-bits. 1381 */ 1382 if (data & VMX_BASIC_32BIT_PHYS_ADDR_ONLY) 1383 return -EINVAL; 1384 1385 if (vmx_basic_vmcs_revision_id(vmx_basic) != 1386 vmx_basic_vmcs_revision_id(data)) 1387 return -EINVAL; 1388 1389 if (vmx_basic_vmcs_size(vmx_basic) > vmx_basic_vmcs_size(data)) 1390 return -EINVAL; 1391 1392 vmx->nested.msrs.basic = data; 1393 return 0; 1394 } 1395 1396 static void vmx_get_control_msr(struct nested_vmx_msrs *msrs, u32 msr_index, 1397 u32 **low, u32 **high) 1398 { 1399 switch (msr_index) { 1400 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1401 *low = &msrs->pinbased_ctls_low; 1402 *high = &msrs->pinbased_ctls_high; 1403 break; 1404 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1405 *low = &msrs->procbased_ctls_low; 1406 *high = &msrs->procbased_ctls_high; 1407 break; 1408 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1409 *low = &msrs->exit_ctls_low; 1410 *high = &msrs->exit_ctls_high; 1411 break; 1412 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1413 *low = &msrs->entry_ctls_low; 1414 *high = &msrs->entry_ctls_high; 1415 break; 1416 case MSR_IA32_VMX_PROCBASED_CTLS2: 1417 *low = &msrs->secondary_ctls_low; 1418 *high = &msrs->secondary_ctls_high; 1419 break; 1420 default: 1421 BUG(); 1422 } 1423 } 1424 1425 static int 1426 vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) 1427 { 1428 u32 *lowp, *highp; 1429 u64 supported; 1430 1431 vmx_get_control_msr(&vmcs_config.nested, msr_index, &lowp, &highp); 1432 1433 supported = vmx_control_msr(*lowp, *highp); 1434 1435 /* Check must-be-1 bits are still 1. */ 1436 if (!is_bitwise_subset(data, supported, GENMASK_ULL(31, 0))) 1437 return -EINVAL; 1438 1439 /* Check must-be-0 bits are still 0. */ 1440 if (!is_bitwise_subset(supported, data, GENMASK_ULL(63, 32))) 1441 return -EINVAL; 1442 1443 vmx_get_control_msr(&vmx->nested.msrs, msr_index, &lowp, &highp); 1444 *lowp = data; 1445 *highp = data >> 32; 1446 return 0; 1447 } 1448 1449 static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data) 1450 { 1451 const u64 feature_bits = VMX_MISC_SAVE_EFER_LMA | 1452 VMX_MISC_ACTIVITY_HLT | 1453 VMX_MISC_ACTIVITY_SHUTDOWN | 1454 VMX_MISC_ACTIVITY_WAIT_SIPI | 1455 VMX_MISC_INTEL_PT | 1456 VMX_MISC_RDMSR_IN_SMM | 1457 VMX_MISC_VMWRITE_SHADOW_RO_FIELDS | 1458 VMX_MISC_VMXOFF_BLOCK_SMI | 1459 VMX_MISC_ZERO_LEN_INS; 1460 1461 const u64 reserved_bits = BIT_ULL(31) | GENMASK_ULL(13, 9); 1462 1463 u64 vmx_misc = vmx_control_msr(vmcs_config.nested.misc_low, 1464 vmcs_config.nested.misc_high); 1465 1466 BUILD_BUG_ON(feature_bits & reserved_bits); 1467 1468 /* 1469 * The incoming value must not set feature bits or reserved bits that 1470 * aren't allowed/supported by KVM. Fields, i.e. multi-bit values, are 1471 * explicitly checked below. 1472 */ 1473 if (!is_bitwise_subset(vmx_misc, data, feature_bits | reserved_bits)) 1474 return -EINVAL; 1475 1476 if ((vmx->nested.msrs.pinbased_ctls_high & 1477 PIN_BASED_VMX_PREEMPTION_TIMER) && 1478 vmx_misc_preemption_timer_rate(data) != 1479 vmx_misc_preemption_timer_rate(vmx_misc)) 1480 return -EINVAL; 1481 1482 if (vmx_misc_cr3_count(data) > vmx_misc_cr3_count(vmx_misc)) 1483 return -EINVAL; 1484 1485 if (vmx_misc_max_msr(data) > vmx_misc_max_msr(vmx_misc)) 1486 return -EINVAL; 1487 1488 if (vmx_misc_mseg_revid(data) != vmx_misc_mseg_revid(vmx_misc)) 1489 return -EINVAL; 1490 1491 vmx->nested.msrs.misc_low = data; 1492 vmx->nested.msrs.misc_high = data >> 32; 1493 1494 return 0; 1495 } 1496 1497 static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data) 1498 { 1499 u64 vmx_ept_vpid_cap = vmx_control_msr(vmcs_config.nested.ept_caps, 1500 vmcs_config.nested.vpid_caps); 1501 1502 /* Every bit is either reserved or a feature bit. */ 1503 if (!is_bitwise_subset(vmx_ept_vpid_cap, data, -1ULL)) 1504 return -EINVAL; 1505 1506 vmx->nested.msrs.ept_caps = data; 1507 vmx->nested.msrs.vpid_caps = data >> 32; 1508 return 0; 1509 } 1510 1511 static u64 *vmx_get_fixed0_msr(struct nested_vmx_msrs *msrs, u32 msr_index) 1512 { 1513 switch (msr_index) { 1514 case MSR_IA32_VMX_CR0_FIXED0: 1515 return &msrs->cr0_fixed0; 1516 case MSR_IA32_VMX_CR4_FIXED0: 1517 return &msrs->cr4_fixed0; 1518 default: 1519 BUG(); 1520 } 1521 } 1522 1523 static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) 1524 { 1525 const u64 *msr = vmx_get_fixed0_msr(&vmcs_config.nested, msr_index); 1526 1527 /* 1528 * 1 bits (which indicates bits which "must-be-1" during VMX operation) 1529 * must be 1 in the restored value. 1530 */ 1531 if (!is_bitwise_subset(data, *msr, -1ULL)) 1532 return -EINVAL; 1533 1534 *vmx_get_fixed0_msr(&vmx->nested.msrs, msr_index) = data; 1535 return 0; 1536 } 1537 1538 /* 1539 * Called when userspace is restoring VMX MSRs. 1540 * 1541 * Returns 0 on success, non-0 otherwise. 1542 */ 1543 int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) 1544 { 1545 struct vcpu_vmx *vmx = to_vmx(vcpu); 1546 1547 /* 1548 * Don't allow changes to the VMX capability MSRs while the vCPU 1549 * is in VMX operation. 1550 */ 1551 if (vmx->nested.vmxon) 1552 return -EBUSY; 1553 1554 switch (msr_index) { 1555 case MSR_IA32_VMX_BASIC: 1556 return vmx_restore_vmx_basic(vmx, data); 1557 case MSR_IA32_VMX_PINBASED_CTLS: 1558 case MSR_IA32_VMX_PROCBASED_CTLS: 1559 case MSR_IA32_VMX_EXIT_CTLS: 1560 case MSR_IA32_VMX_ENTRY_CTLS: 1561 /* 1562 * The "non-true" VMX capability MSRs are generated from the 1563 * "true" MSRs, so we do not support restoring them directly. 1564 * 1565 * If userspace wants to emulate VMX_BASIC[55]=0, userspace 1566 * should restore the "true" MSRs with the must-be-1 bits 1567 * set according to the SDM Vol 3. A.2 "RESERVED CONTROLS AND 1568 * DEFAULT SETTINGS". 1569 */ 1570 return -EINVAL; 1571 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1572 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1573 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1574 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1575 case MSR_IA32_VMX_PROCBASED_CTLS2: 1576 return vmx_restore_control_msr(vmx, msr_index, data); 1577 case MSR_IA32_VMX_MISC: 1578 return vmx_restore_vmx_misc(vmx, data); 1579 case MSR_IA32_VMX_CR0_FIXED0: 1580 case MSR_IA32_VMX_CR4_FIXED0: 1581 return vmx_restore_fixed0_msr(vmx, msr_index, data); 1582 case MSR_IA32_VMX_CR0_FIXED1: 1583 case MSR_IA32_VMX_CR4_FIXED1: 1584 /* 1585 * These MSRs are generated based on the vCPU's CPUID, so we 1586 * do not support restoring them directly. 1587 */ 1588 return -EINVAL; 1589 case MSR_IA32_VMX_EPT_VPID_CAP: 1590 return vmx_restore_vmx_ept_vpid_cap(vmx, data); 1591 case MSR_IA32_VMX_VMCS_ENUM: 1592 vmx->nested.msrs.vmcs_enum = data; 1593 return 0; 1594 case MSR_IA32_VMX_VMFUNC: 1595 if (data & ~vmcs_config.nested.vmfunc_controls) 1596 return -EINVAL; 1597 vmx->nested.msrs.vmfunc_controls = data; 1598 return 0; 1599 default: 1600 /* 1601 * The rest of the VMX capability MSRs do not support restore. 1602 */ 1603 return -EINVAL; 1604 } 1605 } 1606 1607 /* Returns 0 on success, non-0 otherwise. */ 1608 int vmx_get_vmx_msr(struct nested_vmx_msrs *msrs, u32 msr_index, u64 *pdata) 1609 { 1610 switch (msr_index) { 1611 case MSR_IA32_VMX_BASIC: 1612 *pdata = msrs->basic; 1613 break; 1614 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1615 case MSR_IA32_VMX_PINBASED_CTLS: 1616 *pdata = vmx_control_msr( 1617 msrs->pinbased_ctls_low, 1618 msrs->pinbased_ctls_high); 1619 if (msr_index == MSR_IA32_VMX_PINBASED_CTLS) 1620 *pdata |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 1621 break; 1622 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1623 case MSR_IA32_VMX_PROCBASED_CTLS: 1624 *pdata = vmx_control_msr( 1625 msrs->procbased_ctls_low, 1626 msrs->procbased_ctls_high); 1627 if (msr_index == MSR_IA32_VMX_PROCBASED_CTLS) 1628 *pdata |= CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 1629 break; 1630 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1631 case MSR_IA32_VMX_EXIT_CTLS: 1632 *pdata = vmx_control_msr( 1633 msrs->exit_ctls_low, 1634 msrs->exit_ctls_high); 1635 if (msr_index == MSR_IA32_VMX_EXIT_CTLS) 1636 *pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; 1637 break; 1638 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1639 case MSR_IA32_VMX_ENTRY_CTLS: 1640 *pdata = vmx_control_msr( 1641 msrs->entry_ctls_low, 1642 msrs->entry_ctls_high); 1643 if (msr_index == MSR_IA32_VMX_ENTRY_CTLS) 1644 *pdata |= VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; 1645 break; 1646 case MSR_IA32_VMX_MISC: 1647 *pdata = vmx_control_msr( 1648 msrs->misc_low, 1649 msrs->misc_high); 1650 break; 1651 case MSR_IA32_VMX_CR0_FIXED0: 1652 *pdata = msrs->cr0_fixed0; 1653 break; 1654 case MSR_IA32_VMX_CR0_FIXED1: 1655 *pdata = msrs->cr0_fixed1; 1656 break; 1657 case MSR_IA32_VMX_CR4_FIXED0: 1658 *pdata = msrs->cr4_fixed0; 1659 break; 1660 case MSR_IA32_VMX_CR4_FIXED1: 1661 *pdata = msrs->cr4_fixed1; 1662 break; 1663 case MSR_IA32_VMX_VMCS_ENUM: 1664 *pdata = msrs->vmcs_enum; 1665 break; 1666 case MSR_IA32_VMX_PROCBASED_CTLS2: 1667 *pdata = vmx_control_msr( 1668 msrs->secondary_ctls_low, 1669 msrs->secondary_ctls_high); 1670 break; 1671 case MSR_IA32_VMX_EPT_VPID_CAP: 1672 *pdata = msrs->ept_caps | 1673 ((u64)msrs->vpid_caps << 32); 1674 break; 1675 case MSR_IA32_VMX_VMFUNC: 1676 *pdata = msrs->vmfunc_controls; 1677 break; 1678 default: 1679 return 1; 1680 } 1681 1682 return 0; 1683 } 1684 1685 /* 1686 * Copy the writable VMCS shadow fields back to the VMCS12, in case they have 1687 * been modified by the L1 guest. Note, "writable" in this context means 1688 * "writable by the guest", i.e. tagged SHADOW_FIELD_RW; the set of 1689 * fields tagged SHADOW_FIELD_RO may or may not align with the "read-only" 1690 * VM-exit information fields (which are actually writable if the vCPU is 1691 * configured to support "VMWRITE to any supported field in the VMCS"). 1692 */ 1693 static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx) 1694 { 1695 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; 1696 struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu); 1697 struct shadow_vmcs_field field; 1698 unsigned long val; 1699 int i; 1700 1701 if (WARN_ON(!shadow_vmcs)) 1702 return; 1703 1704 preempt_disable(); 1705 1706 vmcs_load(shadow_vmcs); 1707 1708 for (i = 0; i < max_shadow_read_write_fields; i++) { 1709 field = shadow_read_write_fields[i]; 1710 val = __vmcs_readl(field.encoding); 1711 vmcs12_write_any(vmcs12, field.encoding, field.offset, val); 1712 } 1713 1714 vmcs_clear(shadow_vmcs); 1715 vmcs_load(vmx->loaded_vmcs->vmcs); 1716 1717 preempt_enable(); 1718 } 1719 1720 static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx) 1721 { 1722 const struct shadow_vmcs_field *fields[] = { 1723 shadow_read_write_fields, 1724 shadow_read_only_fields 1725 }; 1726 const int max_fields[] = { 1727 max_shadow_read_write_fields, 1728 max_shadow_read_only_fields 1729 }; 1730 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; 1731 struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu); 1732 struct shadow_vmcs_field field; 1733 unsigned long val; 1734 int i, q; 1735 1736 if (WARN_ON(!shadow_vmcs)) 1737 return; 1738 1739 vmcs_load(shadow_vmcs); 1740 1741 for (q = 0; q < ARRAY_SIZE(fields); q++) { 1742 for (i = 0; i < max_fields[q]; i++) { 1743 field = fields[q][i]; 1744 val = vmcs12_read_any(vmcs12, field.encoding, 1745 field.offset); 1746 __vmcs_writel(field.encoding, val); 1747 } 1748 } 1749 1750 vmcs_clear(shadow_vmcs); 1751 vmcs_load(vmx->loaded_vmcs->vmcs); 1752 } 1753 1754 static void copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx, u32 hv_clean_fields) 1755 { 1756 #ifdef CONFIG_KVM_HYPERV 1757 const u64 runtime_controls = HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1 | 1758 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2 | 1759 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC; 1760 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; 1761 struct hv_enlightened_vmcs *evmcs = nested_vmx_evmcs(vmx); 1762 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(&vmx->vcpu); 1763 1764 /* HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE */ 1765 vmcs12->tpr_threshold = evmcs->tpr_threshold; 1766 vmcs12->guest_rip = evmcs->guest_rip; 1767 1768 if ((hv_clean_fields & runtime_controls) != runtime_controls) 1769 vmx->nested.force_msr_bitmap_recalc = true; 1770 1771 if (unlikely(!(hv_clean_fields & 1772 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ENLIGHTENMENTSCONTROL))) { 1773 hv_vcpu->nested.pa_page_gpa = evmcs->partition_assist_page; 1774 hv_vcpu->nested.vm_id = evmcs->hv_vm_id; 1775 hv_vcpu->nested.vp_id = evmcs->hv_vp_id; 1776 } 1777 1778 if (unlikely(!(hv_clean_fields & 1779 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC))) { 1780 vmcs12->guest_rsp = evmcs->guest_rsp; 1781 vmcs12->guest_rflags = evmcs->guest_rflags; 1782 vmcs12->guest_interruptibility_info = 1783 evmcs->guest_interruptibility_info; 1784 /* 1785 * Not present in struct vmcs12: 1786 * vmcs12->guest_ssp = evmcs->guest_ssp; 1787 */ 1788 } 1789 1790 if (unlikely(!(hv_clean_fields & 1791 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC))) { 1792 vmcs12->cpu_based_vm_exec_control = 1793 evmcs->cpu_based_vm_exec_control; 1794 } 1795 1796 if (unlikely(!(hv_clean_fields & 1797 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EXCPN))) { 1798 vmcs12->exception_bitmap = evmcs->exception_bitmap; 1799 } 1800 1801 if (unlikely(!(hv_clean_fields & 1802 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY))) { 1803 vmcs12->vm_entry_controls = evmcs->vm_entry_controls; 1804 } 1805 1806 if (unlikely(!(hv_clean_fields & 1807 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT))) { 1808 vmcs12->vm_entry_intr_info_field = 1809 evmcs->vm_entry_intr_info_field; 1810 vmcs12->vm_entry_exception_error_code = 1811 evmcs->vm_entry_exception_error_code; 1812 vmcs12->vm_entry_instruction_len = 1813 evmcs->vm_entry_instruction_len; 1814 } 1815 1816 if (unlikely(!(hv_clean_fields & 1817 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1))) { 1818 vmcs12->host_ia32_pat = evmcs->host_ia32_pat; 1819 vmcs12->host_ia32_efer = evmcs->host_ia32_efer; 1820 vmcs12->host_cr0 = evmcs->host_cr0; 1821 vmcs12->host_cr3 = evmcs->host_cr3; 1822 vmcs12->host_cr4 = evmcs->host_cr4; 1823 vmcs12->host_ia32_sysenter_esp = evmcs->host_ia32_sysenter_esp; 1824 vmcs12->host_ia32_sysenter_eip = evmcs->host_ia32_sysenter_eip; 1825 vmcs12->host_rip = evmcs->host_rip; 1826 vmcs12->host_ia32_sysenter_cs = evmcs->host_ia32_sysenter_cs; 1827 vmcs12->host_es_selector = evmcs->host_es_selector; 1828 vmcs12->host_cs_selector = evmcs->host_cs_selector; 1829 vmcs12->host_ss_selector = evmcs->host_ss_selector; 1830 vmcs12->host_ds_selector = evmcs->host_ds_selector; 1831 vmcs12->host_fs_selector = evmcs->host_fs_selector; 1832 vmcs12->host_gs_selector = evmcs->host_gs_selector; 1833 vmcs12->host_tr_selector = evmcs->host_tr_selector; 1834 vmcs12->host_ia32_perf_global_ctrl = evmcs->host_ia32_perf_global_ctrl; 1835 /* 1836 * Not present in struct vmcs12: 1837 * vmcs12->host_ia32_s_cet = evmcs->host_ia32_s_cet; 1838 * vmcs12->host_ssp = evmcs->host_ssp; 1839 * vmcs12->host_ia32_int_ssp_table_addr = evmcs->host_ia32_int_ssp_table_addr; 1840 */ 1841 } 1842 1843 if (unlikely(!(hv_clean_fields & 1844 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1))) { 1845 vmcs12->pin_based_vm_exec_control = 1846 evmcs->pin_based_vm_exec_control; 1847 vmcs12->vm_exit_controls = evmcs->vm_exit_controls; 1848 vmcs12->secondary_vm_exec_control = 1849 evmcs->secondary_vm_exec_control; 1850 } 1851 1852 if (unlikely(!(hv_clean_fields & 1853 HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP))) { 1854 vmcs12->io_bitmap_a = evmcs->io_bitmap_a; 1855 vmcs12->io_bitmap_b = evmcs->io_bitmap_b; 1856 } 1857 1858 if (unlikely(!(hv_clean_fields & 1859 HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP))) { 1860 vmcs12->msr_bitmap = evmcs->msr_bitmap; 1861 } 1862 1863 if (unlikely(!(hv_clean_fields & 1864 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2))) { 1865 vmcs12->guest_es_base = evmcs->guest_es_base; 1866 vmcs12->guest_cs_base = evmcs->guest_cs_base; 1867 vmcs12->guest_ss_base = evmcs->guest_ss_base; 1868 vmcs12->guest_ds_base = evmcs->guest_ds_base; 1869 vmcs12->guest_fs_base = evmcs->guest_fs_base; 1870 vmcs12->guest_gs_base = evmcs->guest_gs_base; 1871 vmcs12->guest_ldtr_base = evmcs->guest_ldtr_base; 1872 vmcs12->guest_tr_base = evmcs->guest_tr_base; 1873 vmcs12->guest_gdtr_base = evmcs->guest_gdtr_base; 1874 vmcs12->guest_idtr_base = evmcs->guest_idtr_base; 1875 vmcs12->guest_es_limit = evmcs->guest_es_limit; 1876 vmcs12->guest_cs_limit = evmcs->guest_cs_limit; 1877 vmcs12->guest_ss_limit = evmcs->guest_ss_limit; 1878 vmcs12->guest_ds_limit = evmcs->guest_ds_limit; 1879 vmcs12->guest_fs_limit = evmcs->guest_fs_limit; 1880 vmcs12->guest_gs_limit = evmcs->guest_gs_limit; 1881 vmcs12->guest_ldtr_limit = evmcs->guest_ldtr_limit; 1882 vmcs12->guest_tr_limit = evmcs->guest_tr_limit; 1883 vmcs12->guest_gdtr_limit = evmcs->guest_gdtr_limit; 1884 vmcs12->guest_idtr_limit = evmcs->guest_idtr_limit; 1885 vmcs12->guest_es_ar_bytes = evmcs->guest_es_ar_bytes; 1886 vmcs12->guest_cs_ar_bytes = evmcs->guest_cs_ar_bytes; 1887 vmcs12->guest_ss_ar_bytes = evmcs->guest_ss_ar_bytes; 1888 vmcs12->guest_ds_ar_bytes = evmcs->guest_ds_ar_bytes; 1889 vmcs12->guest_fs_ar_bytes = evmcs->guest_fs_ar_bytes; 1890 vmcs12->guest_gs_ar_bytes = evmcs->guest_gs_ar_bytes; 1891 vmcs12->guest_ldtr_ar_bytes = evmcs->guest_ldtr_ar_bytes; 1892 vmcs12->guest_tr_ar_bytes = evmcs->guest_tr_ar_bytes; 1893 vmcs12->guest_es_selector = evmcs->guest_es_selector; 1894 vmcs12->guest_cs_selector = evmcs->guest_cs_selector; 1895 vmcs12->guest_ss_selector = evmcs->guest_ss_selector; 1896 vmcs12->guest_ds_selector = evmcs->guest_ds_selector; 1897 vmcs12->guest_fs_selector = evmcs->guest_fs_selector; 1898 vmcs12->guest_gs_selector = evmcs->guest_gs_selector; 1899 vmcs12->guest_ldtr_selector = evmcs->guest_ldtr_selector; 1900 vmcs12->guest_tr_selector = evmcs->guest_tr_selector; 1901 } 1902 1903 if (unlikely(!(hv_clean_fields & 1904 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2))) { 1905 vmcs12->tsc_offset = evmcs->tsc_offset; 1906 vmcs12->virtual_apic_page_addr = evmcs->virtual_apic_page_addr; 1907 vmcs12->xss_exit_bitmap = evmcs->xss_exit_bitmap; 1908 vmcs12->encls_exiting_bitmap = evmcs->encls_exiting_bitmap; 1909 vmcs12->tsc_multiplier = evmcs->tsc_multiplier; 1910 } 1911 1912 if (unlikely(!(hv_clean_fields & 1913 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR))) { 1914 vmcs12->cr0_guest_host_mask = evmcs->cr0_guest_host_mask; 1915 vmcs12->cr4_guest_host_mask = evmcs->cr4_guest_host_mask; 1916 vmcs12->cr0_read_shadow = evmcs->cr0_read_shadow; 1917 vmcs12->cr4_read_shadow = evmcs->cr4_read_shadow; 1918 vmcs12->guest_cr0 = evmcs->guest_cr0; 1919 vmcs12->guest_cr3 = evmcs->guest_cr3; 1920 vmcs12->guest_cr4 = evmcs->guest_cr4; 1921 vmcs12->guest_dr7 = evmcs->guest_dr7; 1922 } 1923 1924 if (unlikely(!(hv_clean_fields & 1925 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER))) { 1926 vmcs12->host_fs_base = evmcs->host_fs_base; 1927 vmcs12->host_gs_base = evmcs->host_gs_base; 1928 vmcs12->host_tr_base = evmcs->host_tr_base; 1929 vmcs12->host_gdtr_base = evmcs->host_gdtr_base; 1930 vmcs12->host_idtr_base = evmcs->host_idtr_base; 1931 vmcs12->host_rsp = evmcs->host_rsp; 1932 } 1933 1934 if (unlikely(!(hv_clean_fields & 1935 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT))) { 1936 vmcs12->ept_pointer = evmcs->ept_pointer; 1937 vmcs12->virtual_processor_id = evmcs->virtual_processor_id; 1938 } 1939 1940 if (unlikely(!(hv_clean_fields & 1941 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1))) { 1942 vmcs12->vmcs_link_pointer = evmcs->vmcs_link_pointer; 1943 vmcs12->guest_ia32_debugctl = evmcs->guest_ia32_debugctl; 1944 vmcs12->guest_ia32_pat = evmcs->guest_ia32_pat; 1945 vmcs12->guest_ia32_efer = evmcs->guest_ia32_efer; 1946 vmcs12->guest_pdptr0 = evmcs->guest_pdptr0; 1947 vmcs12->guest_pdptr1 = evmcs->guest_pdptr1; 1948 vmcs12->guest_pdptr2 = evmcs->guest_pdptr2; 1949 vmcs12->guest_pdptr3 = evmcs->guest_pdptr3; 1950 vmcs12->guest_pending_dbg_exceptions = 1951 evmcs->guest_pending_dbg_exceptions; 1952 vmcs12->guest_sysenter_esp = evmcs->guest_sysenter_esp; 1953 vmcs12->guest_sysenter_eip = evmcs->guest_sysenter_eip; 1954 vmcs12->guest_bndcfgs = evmcs->guest_bndcfgs; 1955 vmcs12->guest_activity_state = evmcs->guest_activity_state; 1956 vmcs12->guest_sysenter_cs = evmcs->guest_sysenter_cs; 1957 vmcs12->guest_ia32_perf_global_ctrl = evmcs->guest_ia32_perf_global_ctrl; 1958 /* 1959 * Not present in struct vmcs12: 1960 * vmcs12->guest_ia32_s_cet = evmcs->guest_ia32_s_cet; 1961 * vmcs12->guest_ia32_lbr_ctl = evmcs->guest_ia32_lbr_ctl; 1962 * vmcs12->guest_ia32_int_ssp_table_addr = evmcs->guest_ia32_int_ssp_table_addr; 1963 */ 1964 } 1965 1966 /* 1967 * Not used? 1968 * vmcs12->vm_exit_msr_store_addr = evmcs->vm_exit_msr_store_addr; 1969 * vmcs12->vm_exit_msr_load_addr = evmcs->vm_exit_msr_load_addr; 1970 * vmcs12->vm_entry_msr_load_addr = evmcs->vm_entry_msr_load_addr; 1971 * vmcs12->page_fault_error_code_mask = 1972 * evmcs->page_fault_error_code_mask; 1973 * vmcs12->page_fault_error_code_match = 1974 * evmcs->page_fault_error_code_match; 1975 * vmcs12->cr3_target_count = evmcs->cr3_target_count; 1976 * vmcs12->vm_exit_msr_store_count = evmcs->vm_exit_msr_store_count; 1977 * vmcs12->vm_exit_msr_load_count = evmcs->vm_exit_msr_load_count; 1978 * vmcs12->vm_entry_msr_load_count = evmcs->vm_entry_msr_load_count; 1979 */ 1980 1981 /* 1982 * Read only fields: 1983 * vmcs12->guest_physical_address = evmcs->guest_physical_address; 1984 * vmcs12->vm_instruction_error = evmcs->vm_instruction_error; 1985 * vmcs12->vm_exit_reason = evmcs->vm_exit_reason; 1986 * vmcs12->vm_exit_intr_info = evmcs->vm_exit_intr_info; 1987 * vmcs12->vm_exit_intr_error_code = evmcs->vm_exit_intr_error_code; 1988 * vmcs12->idt_vectoring_info_field = evmcs->idt_vectoring_info_field; 1989 * vmcs12->idt_vectoring_error_code = evmcs->idt_vectoring_error_code; 1990 * vmcs12->vm_exit_instruction_len = evmcs->vm_exit_instruction_len; 1991 * vmcs12->vmx_instruction_info = evmcs->vmx_instruction_info; 1992 * vmcs12->exit_qualification = evmcs->exit_qualification; 1993 * vmcs12->guest_linear_address = evmcs->guest_linear_address; 1994 * 1995 * Not present in struct vmcs12: 1996 * vmcs12->exit_io_instruction_ecx = evmcs->exit_io_instruction_ecx; 1997 * vmcs12->exit_io_instruction_esi = evmcs->exit_io_instruction_esi; 1998 * vmcs12->exit_io_instruction_edi = evmcs->exit_io_instruction_edi; 1999 * vmcs12->exit_io_instruction_eip = evmcs->exit_io_instruction_eip; 2000 */ 2001 2002 return; 2003 #else /* CONFIG_KVM_HYPERV */ 2004 KVM_BUG_ON(1, vmx->vcpu.kvm); 2005 #endif /* CONFIG_KVM_HYPERV */ 2006 } 2007 2008 static void copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx) 2009 { 2010 #ifdef CONFIG_KVM_HYPERV 2011 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; 2012 struct hv_enlightened_vmcs *evmcs = nested_vmx_evmcs(vmx); 2013 2014 /* 2015 * Should not be changed by KVM: 2016 * 2017 * evmcs->host_es_selector = vmcs12->host_es_selector; 2018 * evmcs->host_cs_selector = vmcs12->host_cs_selector; 2019 * evmcs->host_ss_selector = vmcs12->host_ss_selector; 2020 * evmcs->host_ds_selector = vmcs12->host_ds_selector; 2021 * evmcs->host_fs_selector = vmcs12->host_fs_selector; 2022 * evmcs->host_gs_selector = vmcs12->host_gs_selector; 2023 * evmcs->host_tr_selector = vmcs12->host_tr_selector; 2024 * evmcs->host_ia32_pat = vmcs12->host_ia32_pat; 2025 * evmcs->host_ia32_efer = vmcs12->host_ia32_efer; 2026 * evmcs->host_cr0 = vmcs12->host_cr0; 2027 * evmcs->host_cr3 = vmcs12->host_cr3; 2028 * evmcs->host_cr4 = vmcs12->host_cr4; 2029 * evmcs->host_ia32_sysenter_esp = vmcs12->host_ia32_sysenter_esp; 2030 * evmcs->host_ia32_sysenter_eip = vmcs12->host_ia32_sysenter_eip; 2031 * evmcs->host_rip = vmcs12->host_rip; 2032 * evmcs->host_ia32_sysenter_cs = vmcs12->host_ia32_sysenter_cs; 2033 * evmcs->host_fs_base = vmcs12->host_fs_base; 2034 * evmcs->host_gs_base = vmcs12->host_gs_base; 2035 * evmcs->host_tr_base = vmcs12->host_tr_base; 2036 * evmcs->host_gdtr_base = vmcs12->host_gdtr_base; 2037 * evmcs->host_idtr_base = vmcs12->host_idtr_base; 2038 * evmcs->host_rsp = vmcs12->host_rsp; 2039 * sync_vmcs02_to_vmcs12() doesn't read these: 2040 * evmcs->io_bitmap_a = vmcs12->io_bitmap_a; 2041 * evmcs->io_bitmap_b = vmcs12->io_bitmap_b; 2042 * evmcs->msr_bitmap = vmcs12->msr_bitmap; 2043 * evmcs->ept_pointer = vmcs12->ept_pointer; 2044 * evmcs->xss_exit_bitmap = vmcs12->xss_exit_bitmap; 2045 * evmcs->vm_exit_msr_store_addr = vmcs12->vm_exit_msr_store_addr; 2046 * evmcs->vm_exit_msr_load_addr = vmcs12->vm_exit_msr_load_addr; 2047 * evmcs->vm_entry_msr_load_addr = vmcs12->vm_entry_msr_load_addr; 2048 * evmcs->tpr_threshold = vmcs12->tpr_threshold; 2049 * evmcs->virtual_processor_id = vmcs12->virtual_processor_id; 2050 * evmcs->exception_bitmap = vmcs12->exception_bitmap; 2051 * evmcs->vmcs_link_pointer = vmcs12->vmcs_link_pointer; 2052 * evmcs->pin_based_vm_exec_control = vmcs12->pin_based_vm_exec_control; 2053 * evmcs->vm_exit_controls = vmcs12->vm_exit_controls; 2054 * evmcs->secondary_vm_exec_control = vmcs12->secondary_vm_exec_control; 2055 * evmcs->page_fault_error_code_mask = 2056 * vmcs12->page_fault_error_code_mask; 2057 * evmcs->page_fault_error_code_match = 2058 * vmcs12->page_fault_error_code_match; 2059 * evmcs->cr3_target_count = vmcs12->cr3_target_count; 2060 * evmcs->virtual_apic_page_addr = vmcs12->virtual_apic_page_addr; 2061 * evmcs->tsc_offset = vmcs12->tsc_offset; 2062 * evmcs->guest_ia32_debugctl = vmcs12->guest_ia32_debugctl; 2063 * evmcs->cr0_guest_host_mask = vmcs12->cr0_guest_host_mask; 2064 * evmcs->cr4_guest_host_mask = vmcs12->cr4_guest_host_mask; 2065 * evmcs->cr0_read_shadow = vmcs12->cr0_read_shadow; 2066 * evmcs->cr4_read_shadow = vmcs12->cr4_read_shadow; 2067 * evmcs->vm_exit_msr_store_count = vmcs12->vm_exit_msr_store_count; 2068 * evmcs->vm_exit_msr_load_count = vmcs12->vm_exit_msr_load_count; 2069 * evmcs->vm_entry_msr_load_count = vmcs12->vm_entry_msr_load_count; 2070 * evmcs->guest_ia32_perf_global_ctrl = vmcs12->guest_ia32_perf_global_ctrl; 2071 * evmcs->host_ia32_perf_global_ctrl = vmcs12->host_ia32_perf_global_ctrl; 2072 * evmcs->encls_exiting_bitmap = vmcs12->encls_exiting_bitmap; 2073 * evmcs->tsc_multiplier = vmcs12->tsc_multiplier; 2074 * 2075 * Not present in struct vmcs12: 2076 * evmcs->exit_io_instruction_ecx = vmcs12->exit_io_instruction_ecx; 2077 * evmcs->exit_io_instruction_esi = vmcs12->exit_io_instruction_esi; 2078 * evmcs->exit_io_instruction_edi = vmcs12->exit_io_instruction_edi; 2079 * evmcs->exit_io_instruction_eip = vmcs12->exit_io_instruction_eip; 2080 * evmcs->host_ia32_s_cet = vmcs12->host_ia32_s_cet; 2081 * evmcs->host_ssp = vmcs12->host_ssp; 2082 * evmcs->host_ia32_int_ssp_table_addr = vmcs12->host_ia32_int_ssp_table_addr; 2083 * evmcs->guest_ia32_s_cet = vmcs12->guest_ia32_s_cet; 2084 * evmcs->guest_ia32_lbr_ctl = vmcs12->guest_ia32_lbr_ctl; 2085 * evmcs->guest_ia32_int_ssp_table_addr = vmcs12->guest_ia32_int_ssp_table_addr; 2086 * evmcs->guest_ssp = vmcs12->guest_ssp; 2087 */ 2088 2089 evmcs->guest_es_selector = vmcs12->guest_es_selector; 2090 evmcs->guest_cs_selector = vmcs12->guest_cs_selector; 2091 evmcs->guest_ss_selector = vmcs12->guest_ss_selector; 2092 evmcs->guest_ds_selector = vmcs12->guest_ds_selector; 2093 evmcs->guest_fs_selector = vmcs12->guest_fs_selector; 2094 evmcs->guest_gs_selector = vmcs12->guest_gs_selector; 2095 evmcs->guest_ldtr_selector = vmcs12->guest_ldtr_selector; 2096 evmcs->guest_tr_selector = vmcs12->guest_tr_selector; 2097 2098 evmcs->guest_es_limit = vmcs12->guest_es_limit; 2099 evmcs->guest_cs_limit = vmcs12->guest_cs_limit; 2100 evmcs->guest_ss_limit = vmcs12->guest_ss_limit; 2101 evmcs->guest_ds_limit = vmcs12->guest_ds_limit; 2102 evmcs->guest_fs_limit = vmcs12->guest_fs_limit; 2103 evmcs->guest_gs_limit = vmcs12->guest_gs_limit; 2104 evmcs->guest_ldtr_limit = vmcs12->guest_ldtr_limit; 2105 evmcs->guest_tr_limit = vmcs12->guest_tr_limit; 2106 evmcs->guest_gdtr_limit = vmcs12->guest_gdtr_limit; 2107 evmcs->guest_idtr_limit = vmcs12->guest_idtr_limit; 2108 2109 evmcs->guest_es_ar_bytes = vmcs12->guest_es_ar_bytes; 2110 evmcs->guest_cs_ar_bytes = vmcs12->guest_cs_ar_bytes; 2111 evmcs->guest_ss_ar_bytes = vmcs12->guest_ss_ar_bytes; 2112 evmcs->guest_ds_ar_bytes = vmcs12->guest_ds_ar_bytes; 2113 evmcs->guest_fs_ar_bytes = vmcs12->guest_fs_ar_bytes; 2114 evmcs->guest_gs_ar_bytes = vmcs12->guest_gs_ar_bytes; 2115 evmcs->guest_ldtr_ar_bytes = vmcs12->guest_ldtr_ar_bytes; 2116 evmcs->guest_tr_ar_bytes = vmcs12->guest_tr_ar_bytes; 2117 2118 evmcs->guest_es_base = vmcs12->guest_es_base; 2119 evmcs->guest_cs_base = vmcs12->guest_cs_base; 2120 evmcs->guest_ss_base = vmcs12->guest_ss_base; 2121 evmcs->guest_ds_base = vmcs12->guest_ds_base; 2122 evmcs->guest_fs_base = vmcs12->guest_fs_base; 2123 evmcs->guest_gs_base = vmcs12->guest_gs_base; 2124 evmcs->guest_ldtr_base = vmcs12->guest_ldtr_base; 2125 evmcs->guest_tr_base = vmcs12->guest_tr_base; 2126 evmcs->guest_gdtr_base = vmcs12->guest_gdtr_base; 2127 evmcs->guest_idtr_base = vmcs12->guest_idtr_base; 2128 2129 evmcs->guest_ia32_pat = vmcs12->guest_ia32_pat; 2130 evmcs->guest_ia32_efer = vmcs12->guest_ia32_efer; 2131 2132 evmcs->guest_pdptr0 = vmcs12->guest_pdptr0; 2133 evmcs->guest_pdptr1 = vmcs12->guest_pdptr1; 2134 evmcs->guest_pdptr2 = vmcs12->guest_pdptr2; 2135 evmcs->guest_pdptr3 = vmcs12->guest_pdptr3; 2136 2137 evmcs->guest_pending_dbg_exceptions = 2138 vmcs12->guest_pending_dbg_exceptions; 2139 evmcs->guest_sysenter_esp = vmcs12->guest_sysenter_esp; 2140 evmcs->guest_sysenter_eip = vmcs12->guest_sysenter_eip; 2141 2142 evmcs->guest_activity_state = vmcs12->guest_activity_state; 2143 evmcs->guest_sysenter_cs = vmcs12->guest_sysenter_cs; 2144 2145 evmcs->guest_cr0 = vmcs12->guest_cr0; 2146 evmcs->guest_cr3 = vmcs12->guest_cr3; 2147 evmcs->guest_cr4 = vmcs12->guest_cr4; 2148 evmcs->guest_dr7 = vmcs12->guest_dr7; 2149 2150 evmcs->guest_physical_address = vmcs12->guest_physical_address; 2151 2152 evmcs->vm_instruction_error = vmcs12->vm_instruction_error; 2153 evmcs->vm_exit_reason = vmcs12->vm_exit_reason; 2154 evmcs->vm_exit_intr_info = vmcs12->vm_exit_intr_info; 2155 evmcs->vm_exit_intr_error_code = vmcs12->vm_exit_intr_error_code; 2156 evmcs->idt_vectoring_info_field = vmcs12->idt_vectoring_info_field; 2157 evmcs->idt_vectoring_error_code = vmcs12->idt_vectoring_error_code; 2158 evmcs->vm_exit_instruction_len = vmcs12->vm_exit_instruction_len; 2159 evmcs->vmx_instruction_info = vmcs12->vmx_instruction_info; 2160 2161 evmcs->exit_qualification = vmcs12->exit_qualification; 2162 2163 evmcs->guest_linear_address = vmcs12->guest_linear_address; 2164 evmcs->guest_rsp = vmcs12->guest_rsp; 2165 evmcs->guest_rflags = vmcs12->guest_rflags; 2166 2167 evmcs->guest_interruptibility_info = 2168 vmcs12->guest_interruptibility_info; 2169 evmcs->cpu_based_vm_exec_control = vmcs12->cpu_based_vm_exec_control; 2170 evmcs->vm_entry_controls = vmcs12->vm_entry_controls; 2171 evmcs->vm_entry_intr_info_field = vmcs12->vm_entry_intr_info_field; 2172 evmcs->vm_entry_exception_error_code = 2173 vmcs12->vm_entry_exception_error_code; 2174 evmcs->vm_entry_instruction_len = vmcs12->vm_entry_instruction_len; 2175 2176 evmcs->guest_rip = vmcs12->guest_rip; 2177 2178 evmcs->guest_bndcfgs = vmcs12->guest_bndcfgs; 2179 2180 return; 2181 #else /* CONFIG_KVM_HYPERV */ 2182 KVM_BUG_ON(1, vmx->vcpu.kvm); 2183 #endif /* CONFIG_KVM_HYPERV */ 2184 } 2185 2186 /* 2187 * This is an equivalent of the nested hypervisor executing the vmptrld 2188 * instruction. 2189 */ 2190 static enum nested_evmptrld_status nested_vmx_handle_enlightened_vmptrld( 2191 struct kvm_vcpu *vcpu, bool from_launch) 2192 { 2193 #ifdef CONFIG_KVM_HYPERV 2194 struct vcpu_vmx *vmx = to_vmx(vcpu); 2195 bool evmcs_gpa_changed = false; 2196 u64 evmcs_gpa; 2197 2198 if (likely(!guest_cpu_cap_has_evmcs(vcpu))) 2199 return EVMPTRLD_DISABLED; 2200 2201 evmcs_gpa = nested_get_evmptr(vcpu); 2202 if (!evmptr_is_valid(evmcs_gpa)) { 2203 nested_release_evmcs(vcpu); 2204 return EVMPTRLD_DISABLED; 2205 } 2206 2207 if (unlikely(evmcs_gpa != vmx->nested.hv_evmcs_vmptr)) { 2208 vmx->nested.current_vmptr = INVALID_GPA; 2209 2210 nested_release_evmcs(vcpu); 2211 2212 if (kvm_vcpu_map(vcpu, gpa_to_gfn(evmcs_gpa), 2213 &vmx->nested.hv_evmcs_map)) 2214 return EVMPTRLD_ERROR; 2215 2216 vmx->nested.hv_evmcs = vmx->nested.hv_evmcs_map.hva; 2217 2218 /* 2219 * Currently, KVM only supports eVMCS version 1 2220 * (== KVM_EVMCS_VERSION) and thus we expect guest to set this 2221 * value to first u32 field of eVMCS which should specify eVMCS 2222 * VersionNumber. 2223 * 2224 * Guest should be aware of supported eVMCS versions by host by 2225 * examining CPUID.0x4000000A.EAX[0:15]. Host userspace VMM is 2226 * expected to set this CPUID leaf according to the value 2227 * returned in vmcs_version from nested_enable_evmcs(). 2228 * 2229 * However, it turns out that Microsoft Hyper-V fails to comply 2230 * to their own invented interface: When Hyper-V use eVMCS, it 2231 * just sets first u32 field of eVMCS to revision_id specified 2232 * in MSR_IA32_VMX_BASIC. Instead of used eVMCS version number 2233 * which is one of the supported versions specified in 2234 * CPUID.0x4000000A.EAX[0:15]. 2235 * 2236 * To overcome Hyper-V bug, we accept here either a supported 2237 * eVMCS version or VMCS12 revision_id as valid values for first 2238 * u32 field of eVMCS. 2239 */ 2240 if ((vmx->nested.hv_evmcs->revision_id != KVM_EVMCS_VERSION) && 2241 (vmx->nested.hv_evmcs->revision_id != VMCS12_REVISION)) { 2242 nested_release_evmcs(vcpu); 2243 return EVMPTRLD_VMFAIL; 2244 } 2245 2246 vmx->nested.hv_evmcs_vmptr = evmcs_gpa; 2247 2248 evmcs_gpa_changed = true; 2249 /* 2250 * Unlike normal vmcs12, enlightened vmcs12 is not fully 2251 * reloaded from guest's memory (read only fields, fields not 2252 * present in struct hv_enlightened_vmcs, ...). Make sure there 2253 * are no leftovers. 2254 */ 2255 if (from_launch) { 2256 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 2257 memset(vmcs12, 0, sizeof(*vmcs12)); 2258 vmcs12->hdr.revision_id = VMCS12_REVISION; 2259 } 2260 2261 } 2262 2263 /* 2264 * Clean fields data can't be used on VMLAUNCH and when we switch 2265 * between different L2 guests as KVM keeps a single VMCS12 per L1. 2266 */ 2267 if (from_launch || evmcs_gpa_changed) { 2268 vmx->nested.hv_evmcs->hv_clean_fields &= 2269 ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 2270 2271 vmx->nested.force_msr_bitmap_recalc = true; 2272 } 2273 2274 return EVMPTRLD_SUCCEEDED; 2275 #else 2276 return EVMPTRLD_DISABLED; 2277 #endif 2278 } 2279 2280 void nested_sync_vmcs12_to_shadow(struct kvm_vcpu *vcpu) 2281 { 2282 struct vcpu_vmx *vmx = to_vmx(vcpu); 2283 2284 if (nested_vmx_is_evmptr12_valid(vmx)) 2285 copy_vmcs12_to_enlightened(vmx); 2286 else 2287 copy_vmcs12_to_shadow(vmx); 2288 2289 vmx->nested.need_vmcs12_to_shadow_sync = false; 2290 } 2291 2292 static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer) 2293 { 2294 struct vcpu_vmx *vmx = 2295 container_of(timer, struct vcpu_vmx, nested.preemption_timer); 2296 2297 vmx->nested.preemption_timer_expired = true; 2298 kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu); 2299 kvm_vcpu_kick(&vmx->vcpu); 2300 2301 return HRTIMER_NORESTART; 2302 } 2303 2304 static u64 vmx_calc_preemption_timer_value(struct kvm_vcpu *vcpu) 2305 { 2306 struct vcpu_vmx *vmx = to_vmx(vcpu); 2307 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 2308 2309 u64 l1_scaled_tsc = kvm_read_l1_tsc(vcpu, rdtsc()) >> 2310 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 2311 2312 if (!vmx->nested.has_preemption_timer_deadline) { 2313 vmx->nested.preemption_timer_deadline = 2314 vmcs12->vmx_preemption_timer_value + l1_scaled_tsc; 2315 vmx->nested.has_preemption_timer_deadline = true; 2316 } 2317 return vmx->nested.preemption_timer_deadline - l1_scaled_tsc; 2318 } 2319 2320 static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu, 2321 u64 preemption_timeout) 2322 { 2323 struct vcpu_vmx *vmx = to_vmx(vcpu); 2324 2325 /* 2326 * A timer value of zero is architecturally guaranteed to cause 2327 * a VMExit prior to executing any instructions in the guest. 2328 */ 2329 if (preemption_timeout == 0) { 2330 vmx_preemption_timer_fn(&vmx->nested.preemption_timer); 2331 return; 2332 } 2333 2334 if (vcpu->arch.virtual_tsc_khz == 0) 2335 return; 2336 2337 preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 2338 preemption_timeout *= 1000000; 2339 do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz); 2340 hrtimer_start(&vmx->nested.preemption_timer, 2341 ktime_add_ns(ktime_get(), preemption_timeout), 2342 HRTIMER_MODE_ABS_PINNED); 2343 } 2344 2345 static u64 nested_vmx_calc_efer(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 2346 { 2347 if (vmx->vcpu.arch.nested_run_pending && 2348 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) 2349 return vmcs12->guest_ia32_efer; 2350 else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) 2351 return vmx->vcpu.arch.efer | (EFER_LMA | EFER_LME); 2352 else 2353 return vmx->vcpu.arch.efer & ~(EFER_LMA | EFER_LME); 2354 } 2355 2356 static void prepare_vmcs02_constant_state(struct vcpu_vmx *vmx) 2357 { 2358 struct kvm *kvm = vmx->vcpu.kvm; 2359 2360 /* 2361 * If vmcs02 hasn't been initialized, set the constant vmcs02 state 2362 * according to L0's settings (vmcs12 is irrelevant here). Host 2363 * fields that come from L0 and are not constant, e.g. HOST_CR3, 2364 * will be set as needed prior to VMLAUNCH/VMRESUME. 2365 */ 2366 if (vmx->nested.vmcs02_initialized) 2367 return; 2368 vmx->nested.vmcs02_initialized = true; 2369 2370 if (vmx->ve_info) 2371 vmcs_write64(VE_INFORMATION_ADDRESS, __pa(vmx->ve_info)); 2372 2373 /* All VMFUNCs are currently emulated through L0 vmexits. */ 2374 if (cpu_has_vmx_vmfunc()) 2375 vmcs_write64(VM_FUNCTION_CONTROL, 0); 2376 2377 if (cpu_has_vmx_posted_intr()) 2378 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_NESTED_VECTOR); 2379 2380 if (cpu_has_vmx_msr_bitmap()) 2381 vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap)); 2382 2383 /* 2384 * PML is emulated for L2, but never enabled in hardware as the MMU 2385 * handles A/D emulation. Disabling PML for L2 also avoids having to 2386 * deal with filtering out L2 GPAs from the buffer. 2387 */ 2388 if (enable_pml) { 2389 vmcs_write64(PML_ADDRESS, 0); 2390 vmcs_write16(GUEST_PML_INDEX, -1); 2391 } 2392 2393 if (cpu_has_vmx_encls_vmexit()) 2394 vmcs_write64(ENCLS_EXITING_BITMAP, INVALID_GPA); 2395 2396 if (kvm_notify_vmexit_enabled(kvm)) 2397 vmcs_write32(NOTIFY_WINDOW, kvm->arch.notify_window); 2398 2399 /* 2400 * Set the MSR load/store lists to match L0's settings. Only the 2401 * addresses are constant (for vmcs02), the counts can change based 2402 * on L2's behavior, e.g. switching to/from long mode. 2403 */ 2404 vmcs_write64(VM_EXIT_MSR_STORE_ADDR, __pa(vmx->msr_autostore.val)); 2405 vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val)); 2406 vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); 2407 2408 vmx_set_constant_host_state(vmx); 2409 } 2410 2411 static void prepare_vmcs02_early_rare(struct vcpu_vmx *vmx, 2412 struct vmcs12 *vmcs12) 2413 { 2414 prepare_vmcs02_constant_state(vmx); 2415 2416 vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); 2417 2418 /* 2419 * If VPID is disabled, then guest TLB accesses use VPID=0, i.e. the 2420 * same VPID as the host. Emulate this behavior by using vpid01 for L2 2421 * if VPID is disabled in vmcs12. Note, if VPID is disabled, VM-Enter 2422 * and VM-Exit are architecturally required to flush VPID=0, but *only* 2423 * VPID=0. I.e. using vpid02 would be ok (so long as KVM emulates the 2424 * required flushes), but doing so would cause KVM to over-flush. E.g. 2425 * if L1 runs L2 X with VPID12=1, then runs L2 Y with VPID12 disabled, 2426 * and then runs L2 X again, then KVM can and should retain TLB entries 2427 * for VPID12=1. 2428 */ 2429 if (enable_vpid) { 2430 if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02) 2431 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02); 2432 else 2433 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid); 2434 } 2435 } 2436 2437 static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct loaded_vmcs *vmcs01, 2438 struct vmcs12 *vmcs12) 2439 { 2440 u32 exec_control; 2441 u64 guest_efer = nested_vmx_calc_efer(vmx, vmcs12); 2442 2443 if (vmx->nested.dirty_vmcs12 || nested_vmx_is_evmptr12_valid(vmx)) 2444 prepare_vmcs02_early_rare(vmx, vmcs12); 2445 2446 /* 2447 * PIN CONTROLS 2448 */ 2449 exec_control = __pin_controls_get(vmcs01); 2450 exec_control |= (vmcs12->pin_based_vm_exec_control & 2451 ~PIN_BASED_VMX_PREEMPTION_TIMER); 2452 2453 /* Posted interrupts setting is only taken from vmcs12. */ 2454 vmx->nested.pi_pending = false; 2455 if (nested_cpu_has_posted_intr(vmcs12)) { 2456 vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv; 2457 } else { 2458 vmx->nested.posted_intr_nv = -1; 2459 exec_control &= ~PIN_BASED_POSTED_INTR; 2460 } 2461 pin_controls_set(vmx, exec_control); 2462 2463 /* 2464 * EXEC CONTROLS 2465 */ 2466 exec_control = __exec_controls_get(vmcs01); /* L0's desires */ 2467 exec_control &= ~CPU_BASED_INTR_WINDOW_EXITING; 2468 exec_control &= ~CPU_BASED_NMI_WINDOW_EXITING; 2469 exec_control &= ~CPU_BASED_TPR_SHADOW; 2470 exec_control |= vmcs12->cpu_based_vm_exec_control; 2471 2472 if (exec_control & CPU_BASED_TPR_SHADOW) 2473 vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold); 2474 #ifdef CONFIG_X86_64 2475 else 2476 exec_control |= CPU_BASED_CR8_LOAD_EXITING | 2477 CPU_BASED_CR8_STORE_EXITING; 2478 #endif 2479 2480 /* 2481 * A vmexit (to either L1 hypervisor or L0 userspace) is always needed 2482 * for I/O port accesses. 2483 */ 2484 exec_control |= CPU_BASED_UNCOND_IO_EXITING; 2485 exec_control &= ~CPU_BASED_USE_IO_BITMAPS; 2486 2487 /* 2488 * This bit will be computed in nested_get_vmcs12_pages, because 2489 * we do not have access to L1's MSR bitmap yet. For now, keep 2490 * the same bit as before, hoping to avoid multiple VMWRITEs that 2491 * only set/clear this bit. 2492 */ 2493 exec_control &= ~CPU_BASED_USE_MSR_BITMAPS; 2494 exec_control |= exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS; 2495 2496 exec_controls_set(vmx, exec_control); 2497 2498 /* 2499 * SECONDARY EXEC CONTROLS 2500 */ 2501 if (cpu_has_secondary_exec_ctrls()) { 2502 exec_control = __secondary_exec_controls_get(vmcs01); 2503 2504 /* Take the following fields only from vmcs12 */ 2505 exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 2506 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 2507 SECONDARY_EXEC_ENABLE_INVPCID | 2508 SECONDARY_EXEC_ENABLE_RDTSCP | 2509 SECONDARY_EXEC_ENABLE_XSAVES | 2510 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE | 2511 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 2512 SECONDARY_EXEC_APIC_REGISTER_VIRT | 2513 SECONDARY_EXEC_ENABLE_VMFUNC | 2514 SECONDARY_EXEC_MODE_BASED_EPT_EXEC | 2515 SECONDARY_EXEC_DESC); 2516 2517 if (nested_cpu_has(vmcs12, 2518 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) 2519 exec_control |= vmcs12->secondary_vm_exec_control; 2520 2521 /* PML is emulated and never enabled in hardware for L2. */ 2522 exec_control &= ~SECONDARY_EXEC_ENABLE_PML; 2523 2524 /* VMCS shadowing for L2 is emulated for now */ 2525 exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS; 2526 2527 /* 2528 * Preset *DT exiting when emulating UMIP, so that vmx_set_cr4() 2529 * will not have to rewrite the controls just for this bit. 2530 */ 2531 if (vmx_umip_emulated() && (vmcs12->guest_cr4 & X86_CR4_UMIP)) 2532 exec_control |= SECONDARY_EXEC_DESC; 2533 2534 if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) 2535 vmcs_write16(GUEST_INTR_STATUS, 2536 vmcs12->guest_intr_status); 2537 2538 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST)) 2539 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST; 2540 2541 if (exec_control & SECONDARY_EXEC_ENCLS_EXITING) 2542 vmx_write_encls_bitmap(&vmx->vcpu, vmcs12); 2543 2544 secondary_exec_controls_set(vmx, exec_control); 2545 } 2546 2547 /* 2548 * ENTRY CONTROLS 2549 * 2550 * vmcs12's VM_{ENTRY,EXIT}_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE 2551 * are emulated by vmx_set_efer() in prepare_vmcs02(), but speculate 2552 * on the related bits (if supported by the CPU) in the hope that 2553 * we can avoid VMWrites during vmx_set_efer(). 2554 * 2555 * Similarly, take vmcs01's PERF_GLOBAL_CTRL in the hope that if KVM is 2556 * loading PERF_GLOBAL_CTRL via the VMCS for L1, then KVM will want to 2557 * do the same for L2. 2558 */ 2559 exec_control = __vm_entry_controls_get(vmcs01); 2560 exec_control |= (vmcs12->vm_entry_controls & 2561 ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL); 2562 exec_control &= ~(VM_ENTRY_IA32E_MODE | VM_ENTRY_LOAD_IA32_EFER); 2563 if (cpu_has_load_ia32_efer()) { 2564 if (guest_efer & EFER_LMA) 2565 exec_control |= VM_ENTRY_IA32E_MODE; 2566 if (guest_efer != kvm_host.efer) 2567 exec_control |= VM_ENTRY_LOAD_IA32_EFER; 2568 } 2569 vm_entry_controls_set(vmx, exec_control); 2570 2571 /* 2572 * EXIT CONTROLS 2573 * 2574 * L2->L1 exit controls are emulated - the hardware exit is to L0 so 2575 * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER 2576 * bits may be modified by vmx_set_efer() in prepare_vmcs02(). 2577 */ 2578 exec_control = __vm_exit_controls_get(vmcs01); 2579 if (cpu_has_load_ia32_efer() && guest_efer != kvm_host.efer) 2580 exec_control |= VM_EXIT_LOAD_IA32_EFER; 2581 else 2582 exec_control &= ~VM_EXIT_LOAD_IA32_EFER; 2583 vm_exit_controls_set(vmx, exec_control); 2584 2585 /* 2586 * Interrupt/Exception Fields 2587 */ 2588 if (vmx->vcpu.arch.nested_run_pending) { 2589 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 2590 vmcs12->vm_entry_intr_info_field); 2591 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, 2592 vmcs12->vm_entry_exception_error_code); 2593 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 2594 vmcs12->vm_entry_instruction_len); 2595 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 2596 vmcs12->guest_interruptibility_info); 2597 vmx->loaded_vmcs->nmi_known_unmasked = 2598 !(vmcs12->guest_interruptibility_info & GUEST_INTR_STATE_NMI); 2599 } else { 2600 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); 2601 } 2602 } 2603 2604 static void vmcs_read_cet_state(struct kvm_vcpu *vcpu, u64 *s_cet, 2605 u64 *ssp, u64 *ssp_tbl) 2606 { 2607 if (guest_cpu_cap_has(vcpu, X86_FEATURE_IBT) || 2608 guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK)) 2609 *s_cet = vmcs_readl(GUEST_S_CET); 2610 2611 if (guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK)) { 2612 *ssp = vmcs_readl(GUEST_SSP); 2613 *ssp_tbl = vmcs_readl(GUEST_INTR_SSP_TABLE); 2614 } 2615 } 2616 2617 static void vmcs_write_cet_state(struct kvm_vcpu *vcpu, u64 s_cet, 2618 u64 ssp, u64 ssp_tbl) 2619 { 2620 if (guest_cpu_cap_has(vcpu, X86_FEATURE_IBT) || 2621 guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK)) 2622 vmcs_writel(GUEST_S_CET, s_cet); 2623 2624 if (guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK)) { 2625 vmcs_writel(GUEST_SSP, ssp); 2626 vmcs_writel(GUEST_INTR_SSP_TABLE, ssp_tbl); 2627 } 2628 } 2629 2630 static void prepare_vmcs02_rare(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 2631 { 2632 struct hv_enlightened_vmcs *hv_evmcs = nested_vmx_evmcs(vmx); 2633 2634 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2635 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2)) { 2636 2637 vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector); 2638 vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector); 2639 vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector); 2640 vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector); 2641 vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector); 2642 vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector); 2643 vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector); 2644 vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector); 2645 vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit); 2646 vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit); 2647 vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit); 2648 vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit); 2649 vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit); 2650 vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit); 2651 vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit); 2652 vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit); 2653 vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit); 2654 vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit); 2655 vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes); 2656 vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes); 2657 vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes); 2658 vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes); 2659 vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes); 2660 vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes); 2661 vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes); 2662 vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes); 2663 vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base); 2664 vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base); 2665 vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base); 2666 vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base); 2667 vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base); 2668 vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base); 2669 vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base); 2670 vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base); 2671 vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base); 2672 vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base); 2673 2674 vmx_segment_cache_clear(vmx); 2675 } 2676 2677 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2678 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1)) { 2679 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs); 2680 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 2681 vmcs12->guest_pending_dbg_exceptions); 2682 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp); 2683 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip); 2684 2685 if (kvm_mpx_supported() && vmx->vcpu.arch.nested_run_pending && 2686 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)) 2687 vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs); 2688 } 2689 2690 if (nested_cpu_has_xsaves(vmcs12)) 2691 vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap); 2692 2693 /* 2694 * Whether page-faults are trapped is determined by a combination of 2695 * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF. If L0 2696 * doesn't care about page faults then we should set all of these to 2697 * L1's desires. However, if L0 does care about (some) page faults, it 2698 * is not easy (if at all possible?) to merge L0 and L1's desires, we 2699 * simply ask to exit on each and every L2 page fault. This is done by 2700 * setting MASK=MATCH=0 and (see below) EB.PF=1. 2701 * Note that below we don't need special code to set EB.PF beyond the 2702 * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept, 2703 * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when 2704 * !enable_ept, EB.PF is 1, so the "or" will always be 1. 2705 */ 2706 if (vmx_need_pf_intercept(&vmx->vcpu)) { 2707 /* 2708 * TODO: if both L0 and L1 need the same MASK and MATCH, 2709 * go ahead and use it? 2710 */ 2711 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0); 2712 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0); 2713 } else { 2714 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, vmcs12->page_fault_error_code_mask); 2715 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, vmcs12->page_fault_error_code_match); 2716 } 2717 2718 if (cpu_has_vmx_apicv()) { 2719 vmcs_write64(EOI_EXIT_BITMAP0, vmcs12->eoi_exit_bitmap0); 2720 vmcs_write64(EOI_EXIT_BITMAP1, vmcs12->eoi_exit_bitmap1); 2721 vmcs_write64(EOI_EXIT_BITMAP2, vmcs12->eoi_exit_bitmap2); 2722 vmcs_write64(EOI_EXIT_BITMAP3, vmcs12->eoi_exit_bitmap3); 2723 } 2724 2725 /* 2726 * If vmcs12 is configured to save TSC on exit via the auto-store list, 2727 * append the MSR to vmcs02's auto-store list so that KVM effectively 2728 * reads TSC at the time of VM-Exit from L2. The saved value will be 2729 * propagated to vmcs12's list on nested VM-Exit. 2730 * 2731 * Don't increment the number of MSRs in the vCPU structure, as saving 2732 * TSC is specific to this particular incarnation of vmcb02, i.e. must 2733 * not bleed into vmcs01. 2734 */ 2735 if (nested_msr_store_list_has_msr(&vmx->vcpu, MSR_IA32_TSC) && 2736 !WARN_ON_ONCE(vmx->msr_autostore.nr >= ARRAY_SIZE(vmx->msr_autostore.val))) { 2737 vmx->nested.tsc_autostore_slot = vmx->msr_autostore.nr; 2738 vmx->msr_autostore.val[vmx->msr_autostore.nr].index = MSR_IA32_TSC; 2739 2740 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, vmx->msr_autostore.nr + 1); 2741 } else { 2742 vmx->nested.tsc_autostore_slot = -1; 2743 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, vmx->msr_autostore.nr); 2744 } 2745 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 2746 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 2747 2748 if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_CET_STATE) 2749 vmcs_write_cet_state(&vmx->vcpu, vmcs12->guest_s_cet, 2750 vmcs12->guest_ssp, vmcs12->guest_ssp_tbl); 2751 2752 set_cr4_guest_host_mask(vmx); 2753 } 2754 2755 /* 2756 * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested 2757 * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it 2758 * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2 2759 * guest in a way that will both be appropriate to L1's requests, and our 2760 * needs. In addition to modifying the active vmcs (which is vmcs02), this 2761 * function also has additional necessary side-effects, like setting various 2762 * vcpu->arch fields. 2763 * Returns 0 on success, 1 on failure. Invalid state exit qualification code 2764 * is assigned to entry_failure_code on failure. 2765 */ 2766 static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, 2767 bool from_vmentry, 2768 enum vm_entry_failure_code *entry_failure_code) 2769 { 2770 struct vcpu_vmx *vmx = to_vmx(vcpu); 2771 struct hv_enlightened_vmcs *evmcs = nested_vmx_evmcs(vmx); 2772 bool load_guest_pdptrs_vmcs12 = false; 2773 2774 if (vmx->nested.dirty_vmcs12 || nested_vmx_is_evmptr12_valid(vmx)) { 2775 prepare_vmcs02_rare(vmx, vmcs12); 2776 vmx->nested.dirty_vmcs12 = false; 2777 2778 load_guest_pdptrs_vmcs12 = !nested_vmx_is_evmptr12_valid(vmx) || 2779 !(evmcs->hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1); 2780 } 2781 2782 if (vcpu->arch.nested_run_pending && 2783 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) { 2784 kvm_set_dr(vcpu, 7, vmcs12->guest_dr7); 2785 vmx_guest_debugctl_write(vcpu, vmcs12->guest_ia32_debugctl & 2786 vmx_get_supported_debugctl(vcpu, false)); 2787 } else { 2788 kvm_set_dr(vcpu, 7, vcpu->arch.dr7); 2789 vmx_guest_debugctl_write(vcpu, vmx->nested.pre_vmenter_debugctl); 2790 } 2791 2792 if (!vcpu->arch.nested_run_pending || 2793 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_CET_STATE)) 2794 vmcs_write_cet_state(vcpu, vmx->nested.pre_vmenter_s_cet, 2795 vmx->nested.pre_vmenter_ssp, 2796 vmx->nested.pre_vmenter_ssp_tbl); 2797 2798 if (kvm_mpx_supported() && (!vcpu->arch.nested_run_pending || 2799 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))) 2800 vmcs_write64(GUEST_BNDCFGS, vmx->nested.pre_vmenter_bndcfgs); 2801 vmx_set_rflags(vcpu, vmcs12->guest_rflags); 2802 2803 /* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the 2804 * bitwise-or of what L1 wants to trap for L2, and what we want to 2805 * trap. Note that CR0.TS also needs updating - we do this later. 2806 */ 2807 vmx_update_exception_bitmap(vcpu); 2808 vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask; 2809 vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits); 2810 2811 if (vcpu->arch.nested_run_pending && 2812 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT)) { 2813 vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat); 2814 vcpu->arch.pat = vmcs12->guest_ia32_pat; 2815 } else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) { 2816 vmcs_write64(GUEST_IA32_PAT, vcpu->arch.pat); 2817 } 2818 2819 vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset( 2820 vcpu->arch.l1_tsc_offset, 2821 vmx_get_l2_tsc_offset(vcpu), 2822 vmx_get_l2_tsc_multiplier(vcpu)); 2823 2824 vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier( 2825 vcpu->arch.l1_tsc_scaling_ratio, 2826 vmx_get_l2_tsc_multiplier(vcpu)); 2827 2828 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 2829 if (kvm_caps.has_tsc_control) 2830 vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio); 2831 2832 nested_vmx_transition_tlb_flush(vcpu, vmcs12, true); 2833 2834 if (nested_cpu_has_ept(vmcs12)) 2835 nested_ept_init_mmu_context(vcpu); 2836 2837 /* 2838 * Override the CR0/CR4 read shadows after setting the effective guest 2839 * CR0/CR4. The common helpers also set the shadows, but they don't 2840 * account for vmcs12's cr0/4_guest_host_mask. 2841 */ 2842 vmx_set_cr0(vcpu, vmcs12->guest_cr0); 2843 vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12)); 2844 2845 vmx_set_cr4(vcpu, vmcs12->guest_cr4); 2846 vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12)); 2847 2848 vcpu->arch.efer = nested_vmx_calc_efer(vmx, vmcs12); 2849 /* Note: may modify VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */ 2850 vmx_set_efer(vcpu, vcpu->arch.efer); 2851 2852 /* 2853 * Guest state is invalid and unrestricted guest is disabled, 2854 * which means L1 attempted VMEntry to L2 with invalid state. 2855 * Fail the VMEntry. 2856 * 2857 * However when force loading the guest state (SMM exit or 2858 * loading nested state after migration, it is possible to 2859 * have invalid guest state now, which will be later fixed by 2860 * restoring L2 register state 2861 */ 2862 if (CC(from_vmentry && !vmx_guest_state_valid(vcpu))) { 2863 *entry_failure_code = ENTRY_FAIL_DEFAULT; 2864 return -EINVAL; 2865 } 2866 2867 /* Shadow page tables on either EPT or shadow page tables. */ 2868 if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_cpu_has_ept(vmcs12), 2869 from_vmentry, entry_failure_code)) 2870 return -EINVAL; 2871 2872 /* 2873 * Immediately write vmcs02.GUEST_CR3. It will be propagated to vmcs12 2874 * on nested VM-Exit, which can occur without actually running L2 and 2875 * thus without hitting vmx_load_mmu_pgd(), e.g. if L1 is entering L2 with 2876 * vmcs12.GUEST_ACTIVITYSTATE=HLT, in which case KVM will intercept the 2877 * transition to HLT instead of running L2. 2878 */ 2879 if (enable_ept) 2880 vmcs_writel(GUEST_CR3, vmcs12->guest_cr3); 2881 2882 /* Late preparation of GUEST_PDPTRs now that EFER and CRs are set. */ 2883 if (load_guest_pdptrs_vmcs12 && nested_cpu_has_ept(vmcs12) && 2884 is_pae_paging(vcpu)) { 2885 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0); 2886 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1); 2887 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2); 2888 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3); 2889 } 2890 2891 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) && 2892 kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu)) && 2893 WARN_ON_ONCE(__kvm_emulate_msr_write(vcpu, MSR_CORE_PERF_GLOBAL_CTRL, 2894 vmcs12->guest_ia32_perf_global_ctrl))) { 2895 *entry_failure_code = ENTRY_FAIL_DEFAULT; 2896 return -EINVAL; 2897 } 2898 2899 kvm_rsp_write(vcpu, vmcs12->guest_rsp); 2900 kvm_rip_write(vcpu, vmcs12->guest_rip); 2901 2902 /* 2903 * It was observed that genuine Hyper-V running in L1 doesn't reset 2904 * 'hv_clean_fields' by itself, it only sets the corresponding dirty 2905 * bits when it changes a field in eVMCS. Mark all fields as clean 2906 * here. 2907 */ 2908 if (nested_vmx_is_evmptr12_valid(vmx)) 2909 evmcs->hv_clean_fields |= HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 2910 2911 return 0; 2912 } 2913 2914 static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12) 2915 { 2916 if (CC(!nested_cpu_has_nmi_exiting(vmcs12) && 2917 nested_cpu_has_virtual_nmis(vmcs12))) 2918 return -EINVAL; 2919 2920 if (CC(!nested_cpu_has_virtual_nmis(vmcs12) && 2921 nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING))) 2922 return -EINVAL; 2923 2924 return 0; 2925 } 2926 2927 static bool nested_vmx_check_eptp(struct kvm_vcpu *vcpu, u64 new_eptp) 2928 { 2929 struct vcpu_vmx *vmx = to_vmx(vcpu); 2930 2931 /* Check for memory type validity */ 2932 switch (new_eptp & VMX_EPTP_MT_MASK) { 2933 case VMX_EPTP_MT_UC: 2934 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT))) 2935 return false; 2936 break; 2937 case VMX_EPTP_MT_WB: 2938 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT))) 2939 return false; 2940 break; 2941 default: 2942 return false; 2943 } 2944 2945 /* Page-walk levels validity. */ 2946 switch (new_eptp & VMX_EPTP_PWL_MASK) { 2947 case VMX_EPTP_PWL_5: 2948 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_5_BIT))) 2949 return false; 2950 break; 2951 case VMX_EPTP_PWL_4: 2952 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_4_BIT))) 2953 return false; 2954 break; 2955 default: 2956 return false; 2957 } 2958 2959 /* Reserved bits should not be set */ 2960 if (CC(!kvm_vcpu_is_legal_gpa(vcpu, new_eptp) || ((new_eptp >> 7) & 0x1f))) 2961 return false; 2962 2963 /* AD, if set, should be supported */ 2964 if (new_eptp & VMX_EPTP_AD_ENABLE_BIT) { 2965 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT))) 2966 return false; 2967 } 2968 2969 return true; 2970 } 2971 2972 /* 2973 * Checks related to VM-Execution Control Fields 2974 */ 2975 static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu, 2976 struct vmcs12 *vmcs12) 2977 { 2978 struct vcpu_vmx *vmx = to_vmx(vcpu); 2979 2980 if (CC(!vmx_control_verify(vmcs12->pin_based_vm_exec_control, 2981 vmx->nested.msrs.pinbased_ctls_low, 2982 vmx->nested.msrs.pinbased_ctls_high)) || 2983 CC(!vmx_control_verify(vmcs12->cpu_based_vm_exec_control, 2984 vmx->nested.msrs.procbased_ctls_low, 2985 vmx->nested.msrs.procbased_ctls_high))) 2986 return -EINVAL; 2987 2988 if (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) && 2989 CC(!vmx_control_verify(vmcs12->secondary_vm_exec_control, 2990 vmx->nested.msrs.secondary_ctls_low, 2991 vmx->nested.msrs.secondary_ctls_high))) 2992 return -EINVAL; 2993 2994 if (CC(vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu)) || 2995 nested_vmx_check_io_bitmap_controls(vcpu, vmcs12) || 2996 nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12) || 2997 nested_vmx_check_tpr_shadow_controls(vcpu, vmcs12) || 2998 nested_vmx_check_apic_access_controls(vcpu, vmcs12) || 2999 nested_vmx_check_apicv_controls(vcpu, vmcs12) || 3000 nested_vmx_check_nmi_controls(vmcs12) || 3001 nested_vmx_check_pml_controls(vcpu, vmcs12) || 3002 nested_vmx_check_unrestricted_guest_controls(vcpu, vmcs12) || 3003 nested_vmx_check_mode_based_ept_exec_controls(vcpu, vmcs12) || 3004 nested_vmx_check_shadow_vmcs_controls(vcpu, vmcs12) || 3005 CC(nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id)) 3006 return -EINVAL; 3007 3008 if (!nested_cpu_has_preemption_timer(vmcs12) && 3009 nested_cpu_has_save_preemption_timer(vmcs12)) 3010 return -EINVAL; 3011 3012 if (nested_cpu_has_ept(vmcs12) && 3013 CC(!nested_vmx_check_eptp(vcpu, vmcs12->ept_pointer))) 3014 return -EINVAL; 3015 3016 if (nested_cpu_has_vmfunc(vmcs12)) { 3017 if (CC(vmcs12->vm_function_control & 3018 ~vmx->nested.msrs.vmfunc_controls)) 3019 return -EINVAL; 3020 3021 if (nested_cpu_has_eptp_switching(vmcs12)) { 3022 if (CC(!nested_cpu_has_ept(vmcs12)) || 3023 CC(!page_address_valid(vcpu, vmcs12->eptp_list_address))) 3024 return -EINVAL; 3025 } 3026 } 3027 3028 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_TSC_SCALING) && 3029 CC(!vmcs12->tsc_multiplier)) 3030 return -EINVAL; 3031 3032 return 0; 3033 } 3034 3035 /* 3036 * Checks related to VM-Exit Control Fields 3037 */ 3038 static int nested_check_vm_exit_controls(struct kvm_vcpu *vcpu, 3039 struct vmcs12 *vmcs12) 3040 { 3041 struct vcpu_vmx *vmx = to_vmx(vcpu); 3042 3043 if (CC(!vmx_control_verify(vmcs12->vm_exit_controls, 3044 vmx->nested.msrs.exit_ctls_low, 3045 vmx->nested.msrs.exit_ctls_high)) || 3046 CC(nested_vmx_check_exit_msr_switch_controls(vcpu, vmcs12))) 3047 return -EINVAL; 3048 3049 return 0; 3050 } 3051 3052 /* 3053 * Checks related to VM-Entry Control Fields 3054 */ 3055 static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu, 3056 struct vmcs12 *vmcs12) 3057 { 3058 struct vcpu_vmx *vmx = to_vmx(vcpu); 3059 3060 if (CC(!vmx_control_verify(vmcs12->vm_entry_controls, 3061 vmx->nested.msrs.entry_ctls_low, 3062 vmx->nested.msrs.entry_ctls_high))) 3063 return -EINVAL; 3064 3065 /* 3066 * From the Intel SDM, volume 3: 3067 * Fields relevant to VM-entry event injection must be set properly. 3068 * These fields are the VM-entry interruption-information field, the 3069 * VM-entry exception error code, and the VM-entry instruction length. 3070 */ 3071 if (vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) { 3072 u32 intr_info = vmcs12->vm_entry_intr_info_field; 3073 u8 vector = intr_info & INTR_INFO_VECTOR_MASK; 3074 u32 intr_type = intr_info & INTR_INFO_INTR_TYPE_MASK; 3075 bool has_error_code = intr_info & INTR_INFO_DELIVER_CODE_MASK; 3076 bool urg = nested_cpu_has2(vmcs12, 3077 SECONDARY_EXEC_UNRESTRICTED_GUEST); 3078 bool prot_mode = !urg || vmcs12->guest_cr0 & X86_CR0_PE; 3079 3080 /* VM-entry interruption-info field: interruption type */ 3081 if (CC(intr_type == INTR_TYPE_RESERVED) || 3082 CC(intr_type == INTR_TYPE_OTHER_EVENT && 3083 !nested_cpu_supports_monitor_trap_flag(vcpu))) 3084 return -EINVAL; 3085 3086 /* VM-entry interruption-info field: vector */ 3087 if (CC(intr_type == INTR_TYPE_NMI_INTR && vector != NMI_VECTOR) || 3088 CC(intr_type == INTR_TYPE_HARD_EXCEPTION && vector > 31) || 3089 CC(intr_type == INTR_TYPE_OTHER_EVENT && vector != 0)) 3090 return -EINVAL; 3091 3092 /* 3093 * Cannot deliver error code in real mode or if the interrupt 3094 * type is not hardware exception. For other cases, do the 3095 * consistency check only if the vCPU doesn't enumerate 3096 * VMX_BASIC_NO_HW_ERROR_CODE_CC. 3097 */ 3098 if (!prot_mode || intr_type != INTR_TYPE_HARD_EXCEPTION) { 3099 if (CC(has_error_code)) 3100 return -EINVAL; 3101 } else if (!nested_cpu_has_no_hw_errcode_cc(vcpu)) { 3102 if (CC(has_error_code != x86_exception_has_error_code(vector))) 3103 return -EINVAL; 3104 } 3105 3106 /* VM-entry exception error code */ 3107 if (CC(has_error_code && 3108 vmcs12->vm_entry_exception_error_code & GENMASK(31, 16))) 3109 return -EINVAL; 3110 3111 /* VM-entry interruption-info field: reserved bits */ 3112 if (CC(intr_info & INTR_INFO_RESVD_BITS_MASK)) 3113 return -EINVAL; 3114 3115 /* VM-entry instruction length */ 3116 switch (intr_type) { 3117 case INTR_TYPE_SOFT_EXCEPTION: 3118 case INTR_TYPE_SOFT_INTR: 3119 case INTR_TYPE_PRIV_SW_EXCEPTION: 3120 if (CC(vmcs12->vm_entry_instruction_len > X86_MAX_INSTRUCTION_LENGTH) || 3121 CC(vmcs12->vm_entry_instruction_len == 0 && 3122 CC(!nested_cpu_has_zero_length_injection(vcpu)))) 3123 return -EINVAL; 3124 } 3125 } 3126 3127 if (nested_vmx_check_entry_msr_switch_controls(vcpu, vmcs12)) 3128 return -EINVAL; 3129 3130 return 0; 3131 } 3132 3133 static int nested_vmx_check_controls(struct kvm_vcpu *vcpu, 3134 struct vmcs12 *vmcs12) 3135 { 3136 if (nested_check_vm_execution_controls(vcpu, vmcs12) || 3137 nested_check_vm_exit_controls(vcpu, vmcs12) || 3138 nested_check_vm_entry_controls(vcpu, vmcs12)) 3139 return -EINVAL; 3140 3141 #ifdef CONFIG_KVM_HYPERV 3142 if (guest_cpu_cap_has_evmcs(vcpu)) 3143 return nested_evmcs_check_controls(vmcs12); 3144 #endif 3145 3146 return 0; 3147 } 3148 3149 static int nested_vmx_check_address_space_size(struct kvm_vcpu *vcpu, 3150 struct vmcs12 *vmcs12) 3151 { 3152 #ifdef CONFIG_X86_64 3153 if (CC(!!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) != 3154 !!(vcpu->arch.efer & EFER_LMA))) 3155 return -EINVAL; 3156 #endif 3157 return 0; 3158 } 3159 3160 static bool is_l1_noncanonical_address_on_vmexit(u64 la, struct vmcs12 *vmcs12) 3161 { 3162 /* 3163 * Check that the given linear address is canonical after a VM exit 3164 * from L2, based on HOST_CR4.LA57 value that will be loaded for L1. 3165 */ 3166 u8 l1_address_bits_on_exit = (vmcs12->host_cr4 & X86_CR4_LA57) ? 57 : 48; 3167 3168 return !__is_canonical_address(la, l1_address_bits_on_exit); 3169 } 3170 3171 static int nested_vmx_check_cet_state_common(struct kvm_vcpu *vcpu, u64 s_cet, 3172 u64 ssp, u64 ssp_tbl) 3173 { 3174 if (CC(!kvm_is_valid_u_s_cet(vcpu, s_cet)) || CC(!IS_ALIGNED(ssp, 4)) || 3175 CC(is_noncanonical_msr_address(ssp_tbl, vcpu))) 3176 return -EINVAL; 3177 3178 return 0; 3179 } 3180 3181 static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu, 3182 struct vmcs12 *vmcs12) 3183 { 3184 bool ia32e = !!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE); 3185 3186 if (CC(!nested_host_cr0_valid(vcpu, vmcs12->host_cr0)) || 3187 CC(!nested_host_cr4_valid(vcpu, vmcs12->host_cr4)) || 3188 CC(!kvm_vcpu_is_legal_cr3(vcpu, vmcs12->host_cr3))) 3189 return -EINVAL; 3190 3191 if (CC(vmcs12->host_cr4 & X86_CR4_CET && !(vmcs12->host_cr0 & X86_CR0_WP))) 3192 return -EINVAL; 3193 3194 if (CC(is_noncanonical_msr_address(vmcs12->host_ia32_sysenter_esp, vcpu)) || 3195 CC(is_noncanonical_msr_address(vmcs12->host_ia32_sysenter_eip, vcpu))) 3196 return -EINVAL; 3197 3198 if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) && 3199 CC(!kvm_pat_valid(vmcs12->host_ia32_pat))) 3200 return -EINVAL; 3201 3202 if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) && 3203 CC(!kvm_valid_perf_global_ctrl(vcpu_to_pmu(vcpu), 3204 vmcs12->host_ia32_perf_global_ctrl))) 3205 return -EINVAL; 3206 3207 if (ia32e) { 3208 if (CC(!(vmcs12->host_cr4 & X86_CR4_PAE))) 3209 return -EINVAL; 3210 } else { 3211 if (CC(vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) || 3212 CC(vmcs12->host_cr4 & X86_CR4_PCIDE) || 3213 CC((vmcs12->host_rip) >> 32)) 3214 return -EINVAL; 3215 } 3216 3217 if (CC(vmcs12->host_cs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 3218 CC(vmcs12->host_ss_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 3219 CC(vmcs12->host_ds_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 3220 CC(vmcs12->host_es_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 3221 CC(vmcs12->host_fs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 3222 CC(vmcs12->host_gs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 3223 CC(vmcs12->host_tr_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 3224 CC(vmcs12->host_cs_selector == 0) || 3225 CC(vmcs12->host_tr_selector == 0) || 3226 CC(vmcs12->host_ss_selector == 0 && !ia32e)) 3227 return -EINVAL; 3228 3229 if (CC(is_noncanonical_base_address(vmcs12->host_fs_base, vcpu)) || 3230 CC(is_noncanonical_base_address(vmcs12->host_gs_base, vcpu)) || 3231 CC(is_noncanonical_base_address(vmcs12->host_gdtr_base, vcpu)) || 3232 CC(is_noncanonical_base_address(vmcs12->host_idtr_base, vcpu)) || 3233 CC(is_noncanonical_base_address(vmcs12->host_tr_base, vcpu)) || 3234 CC(is_l1_noncanonical_address_on_vmexit(vmcs12->host_rip, vmcs12))) 3235 return -EINVAL; 3236 3237 /* 3238 * If the load IA32_EFER VM-exit control is 1, bits reserved in the 3239 * IA32_EFER MSR must be 0 in the field for that register. In addition, 3240 * the values of the LMA and LME bits in the field must each be that of 3241 * the host address-space size VM-exit control. 3242 */ 3243 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) { 3244 if (CC(!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer)) || 3245 CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA)) || 3246 CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LME))) 3247 return -EINVAL; 3248 } 3249 3250 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_CET_STATE) { 3251 if (nested_vmx_check_cet_state_common(vcpu, vmcs12->host_s_cet, 3252 vmcs12->host_ssp, 3253 vmcs12->host_ssp_tbl)) 3254 return -EINVAL; 3255 3256 /* 3257 * IA32_S_CET and SSP must be canonical if the host will 3258 * enter 64-bit mode after VM-exit; otherwise, higher 3259 * 32-bits must be all 0s. 3260 */ 3261 if (ia32e) { 3262 if (CC(is_noncanonical_msr_address(vmcs12->host_s_cet, vcpu)) || 3263 CC(is_noncanonical_msr_address(vmcs12->host_ssp, vcpu))) 3264 return -EINVAL; 3265 } else { 3266 if (CC(vmcs12->host_s_cet >> 32) || CC(vmcs12->host_ssp >> 32)) 3267 return -EINVAL; 3268 } 3269 } 3270 3271 return 0; 3272 } 3273 3274 static int nested_vmx_check_vmcs_link_ptr(struct kvm_vcpu *vcpu, 3275 struct vmcs12 *vmcs12) 3276 { 3277 struct vcpu_vmx *vmx = to_vmx(vcpu); 3278 struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; 3279 struct vmcs_hdr hdr; 3280 3281 if (vmcs12->vmcs_link_pointer == INVALID_GPA) 3282 return 0; 3283 3284 if (CC(!page_address_valid(vcpu, vmcs12->vmcs_link_pointer))) 3285 return -EINVAL; 3286 3287 if (ghc->gpa != vmcs12->vmcs_link_pointer && 3288 CC(kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, 3289 vmcs12->vmcs_link_pointer, VMCS12_SIZE))) 3290 return -EINVAL; 3291 3292 if (CC(kvm_read_guest_offset_cached(vcpu->kvm, ghc, &hdr, 3293 offsetof(struct vmcs12, hdr), 3294 sizeof(hdr)))) 3295 return -EINVAL; 3296 3297 if (CC(hdr.revision_id != VMCS12_REVISION) || 3298 CC(hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12))) 3299 return -EINVAL; 3300 3301 return 0; 3302 } 3303 3304 /* 3305 * Checks related to Guest Non-register State 3306 */ 3307 static int nested_check_guest_non_reg_state(struct vmcs12 *vmcs12) 3308 { 3309 if (CC(vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE && 3310 vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT && 3311 vmcs12->guest_activity_state != GUEST_ACTIVITY_WAIT_SIPI)) 3312 return -EINVAL; 3313 3314 return 0; 3315 } 3316 3317 static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu, 3318 struct vmcs12 *vmcs12, 3319 enum vm_entry_failure_code *entry_failure_code) 3320 { 3321 bool ia32e = !!(vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE); 3322 3323 *entry_failure_code = ENTRY_FAIL_DEFAULT; 3324 3325 if (CC(!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0)) || 3326 CC(!nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4))) 3327 return -EINVAL; 3328 3329 if (CC(vmcs12->guest_cr4 & X86_CR4_CET && !(vmcs12->guest_cr0 & X86_CR0_WP))) 3330 return -EINVAL; 3331 3332 if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) { 3333 u64 debugctl = vmcs12->guest_ia32_debugctl; 3334 3335 /* 3336 * FREEZE_IN_SMM is not virtualized, but allow L1 to set it in 3337 * vmcs12's DEBUGCTL under a quirk for backwards compatibility. 3338 * Note that the quirk only relaxes the consistency check. The 3339 * vmcc02 bit is still under the control of the host. In 3340 * particular, if a host administrator decides to clear the bit, 3341 * then L1 has no say in the matter. 3342 */ 3343 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_VMCS12_ALLOW_FREEZE_IN_SMM)) 3344 debugctl &= ~DEBUGCTLMSR_FREEZE_IN_SMM; 3345 3346 if (CC(!kvm_dr7_valid(vmcs12->guest_dr7)) || 3347 CC(!vmx_is_valid_debugctl(vcpu, debugctl, false))) 3348 return -EINVAL; 3349 } 3350 3351 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT) && 3352 CC(!kvm_pat_valid(vmcs12->guest_ia32_pat))) 3353 return -EINVAL; 3354 3355 if (nested_vmx_check_vmcs_link_ptr(vcpu, vmcs12)) { 3356 *entry_failure_code = ENTRY_FAIL_VMCS_LINK_PTR; 3357 return -EINVAL; 3358 } 3359 3360 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) && 3361 CC(!kvm_valid_perf_global_ctrl(vcpu_to_pmu(vcpu), 3362 vmcs12->guest_ia32_perf_global_ctrl))) 3363 return -EINVAL; 3364 3365 if (CC((vmcs12->guest_cr0 & (X86_CR0_PG | X86_CR0_PE)) == X86_CR0_PG)) 3366 return -EINVAL; 3367 3368 if (CC(ia32e && !(vmcs12->guest_cr4 & X86_CR4_PAE)) || 3369 CC(ia32e && !(vmcs12->guest_cr0 & X86_CR0_PG))) 3370 return -EINVAL; 3371 3372 /* 3373 * If the load IA32_EFER VM-entry control is 1, the following checks 3374 * are performed on the field for the IA32_EFER MSR: 3375 * - Bits reserved in the IA32_EFER MSR must be 0. 3376 * - Bit 10 (corresponding to IA32_EFER.LMA) must equal the value of 3377 * the IA-32e mode guest VM-exit control. It must also be identical 3378 * to bit 8 (LME) if bit 31 in the CR0 field (corresponding to 3379 * CR0.PG) is 1. 3380 */ 3381 if (vcpu->arch.nested_run_pending && 3382 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) { 3383 if (CC(!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer)) || 3384 CC(ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA)) || 3385 CC(((vmcs12->guest_cr0 & X86_CR0_PG) && 3386 ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME)))) 3387 return -EINVAL; 3388 } 3389 3390 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS) && 3391 (CC(is_noncanonical_msr_address(vmcs12->guest_bndcfgs & PAGE_MASK, vcpu)) || 3392 CC((vmcs12->guest_bndcfgs & MSR_IA32_BNDCFGS_RSVD)))) 3393 return -EINVAL; 3394 3395 if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_CET_STATE) { 3396 if (nested_vmx_check_cet_state_common(vcpu, vmcs12->guest_s_cet, 3397 vmcs12->guest_ssp, 3398 vmcs12->guest_ssp_tbl)) 3399 return -EINVAL; 3400 3401 /* 3402 * Guest SSP must have 63:N bits identical, rather than 3403 * be canonical (i.e., 63:N-1 bits identical), where N is 3404 * the CPU's maximum linear-address width. Similar to 3405 * is_noncanonical_msr_address(), use the host's 3406 * linear-address width. 3407 */ 3408 if (CC(!__is_canonical_address(vmcs12->guest_ssp, max_host_virt_addr_bits() + 1))) 3409 return -EINVAL; 3410 } 3411 3412 if (nested_check_guest_non_reg_state(vmcs12)) 3413 return -EINVAL; 3414 3415 return 0; 3416 } 3417 3418 #ifdef CONFIG_KVM_HYPERV 3419 static bool nested_get_evmcs_page(struct kvm_vcpu *vcpu) 3420 { 3421 struct vcpu_vmx *vmx = to_vmx(vcpu); 3422 3423 /* 3424 * hv_evmcs may end up being not mapped after migration (when 3425 * L2 was running), map it here to make sure vmcs12 changes are 3426 * properly reflected. 3427 */ 3428 if (guest_cpu_cap_has_evmcs(vcpu) && 3429 vmx->nested.hv_evmcs_vmptr == EVMPTR_MAP_PENDING) { 3430 enum nested_evmptrld_status evmptrld_status = 3431 nested_vmx_handle_enlightened_vmptrld(vcpu, false); 3432 3433 if (evmptrld_status == EVMPTRLD_VMFAIL || 3434 evmptrld_status == EVMPTRLD_ERROR) 3435 return false; 3436 3437 /* 3438 * Post migration VMCS12 always provides the most actual 3439 * information, copy it to eVMCS upon entry. 3440 */ 3441 vmx->nested.need_vmcs12_to_shadow_sync = true; 3442 } 3443 3444 return true; 3445 } 3446 #endif 3447 3448 static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu) 3449 { 3450 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3451 struct vcpu_vmx *vmx = to_vmx(vcpu); 3452 struct kvm_host_map *map; 3453 3454 if (!vcpu->arch.pdptrs_from_userspace && 3455 !nested_cpu_has_ept(vmcs12) && is_pae_paging(vcpu)) { 3456 /* 3457 * Reload the guest's PDPTRs since after a migration 3458 * the guest CR3 might be restored prior to setting the nested 3459 * state which can lead to a load of wrong PDPTRs. 3460 */ 3461 if (CC(!load_pdptrs(vcpu, vcpu->arch.cr3))) 3462 return false; 3463 } 3464 3465 3466 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 3467 map = &vmx->nested.apic_access_page_map; 3468 3469 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->apic_access_addr), map)) { 3470 vmcs_write64(APIC_ACCESS_ADDR, pfn_to_hpa(map->pfn)); 3471 } else { 3472 pr_debug_ratelimited("%s: no backing for APIC-access address in vmcs12\n", 3473 __func__); 3474 return false; 3475 } 3476 } 3477 3478 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { 3479 map = &vmx->nested.virtual_apic_map; 3480 3481 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->virtual_apic_page_addr), map)) { 3482 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, pfn_to_hpa(map->pfn)); 3483 } else if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING) && 3484 nested_cpu_has(vmcs12, CPU_BASED_CR8_STORE_EXITING) && 3485 !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 3486 /* 3487 * The processor will never use the TPR shadow, simply 3488 * clear the bit from the execution control. Such a 3489 * configuration is useless, but it happens in tests. 3490 * For any other configuration, failing the vm entry is 3491 * _not_ what the processor does but it's basically the 3492 * only possibility we have. 3493 */ 3494 exec_controls_clearbit(vmx, CPU_BASED_TPR_SHADOW); 3495 } else { 3496 /* 3497 * Write an illegal value to VIRTUAL_APIC_PAGE_ADDR to 3498 * force VM-Entry to fail. 3499 */ 3500 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, INVALID_GPA); 3501 } 3502 } 3503 3504 if (nested_cpu_has_posted_intr(vmcs12)) { 3505 map = &vmx->nested.pi_desc_map; 3506 3507 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->posted_intr_desc_addr), map)) { 3508 vmx->nested.pi_desc = 3509 (struct pi_desc *)(((void *)map->hva) + 3510 offset_in_page(vmcs12->posted_intr_desc_addr)); 3511 vmcs_write64(POSTED_INTR_DESC_ADDR, 3512 pfn_to_hpa(map->pfn) + offset_in_page(vmcs12->posted_intr_desc_addr)); 3513 } else { 3514 /* 3515 * Defer the KVM_INTERNAL_EXIT until KVM tries to 3516 * access the contents of the VMCS12 posted interrupt 3517 * descriptor. (Note that KVM may do this when it 3518 * should not, per the architectural specification.) 3519 */ 3520 vmx->nested.pi_desc = NULL; 3521 pin_controls_clearbit(vmx, PIN_BASED_POSTED_INTR); 3522 } 3523 } 3524 if (nested_vmx_prepare_msr_bitmap(vcpu, vmcs12)) 3525 exec_controls_setbit(vmx, CPU_BASED_USE_MSR_BITMAPS); 3526 else 3527 exec_controls_clearbit(vmx, CPU_BASED_USE_MSR_BITMAPS); 3528 3529 return true; 3530 } 3531 3532 static bool vmx_get_nested_state_pages(struct kvm_vcpu *vcpu) 3533 { 3534 #ifdef CONFIG_KVM_HYPERV 3535 /* 3536 * Note: nested_get_evmcs_page() also updates 'vp_assist_page' copy 3537 * in 'struct kvm_vcpu_hv' in case eVMCS is in use, this is mandatory 3538 * to make nested_evmcs_l2_tlb_flush_enabled() work correctly post 3539 * migration. 3540 */ 3541 if (!nested_get_evmcs_page(vcpu)) { 3542 pr_debug_ratelimited("%s: enlightened vmptrld failed\n", 3543 __func__); 3544 return false; 3545 } 3546 #endif 3547 3548 if (is_guest_mode(vcpu) && !nested_get_vmcs12_pages(vcpu)) 3549 return false; 3550 3551 return true; 3552 } 3553 3554 static int nested_vmx_write_pml_buffer(struct kvm_vcpu *vcpu, gpa_t gpa) 3555 { 3556 struct vmcs12 *vmcs12; 3557 struct vcpu_vmx *vmx = to_vmx(vcpu); 3558 gpa_t dst; 3559 3560 if (WARN_ON_ONCE(!is_guest_mode(vcpu))) 3561 return 0; 3562 3563 if (WARN_ON_ONCE(vmx->nested.pml_full)) 3564 return 1; 3565 3566 /* 3567 * Check if PML is enabled for the nested guest. Whether eptp bit 6 is 3568 * set is already checked as part of A/D emulation. 3569 */ 3570 vmcs12 = get_vmcs12(vcpu); 3571 if (!nested_cpu_has_pml(vmcs12)) 3572 return 0; 3573 3574 if (vmcs12->guest_pml_index >= PML_LOG_NR_ENTRIES) { 3575 vmx->nested.pml_full = true; 3576 return 1; 3577 } 3578 3579 gpa &= ~0xFFFull; 3580 dst = vmcs12->pml_address + sizeof(u64) * vmcs12->guest_pml_index; 3581 3582 if (kvm_write_guest_page(vcpu->kvm, gpa_to_gfn(dst), &gpa, 3583 offset_in_page(dst), sizeof(gpa))) 3584 return 0; 3585 3586 vmcs12->guest_pml_index--; 3587 3588 return 0; 3589 } 3590 3591 /* 3592 * Intel's VMX Instruction Reference specifies a common set of prerequisites 3593 * for running VMX instructions (except VMXON, whose prerequisites are 3594 * slightly different). It also specifies what exception to inject otherwise. 3595 * Note that many of these exceptions have priority over VM exits, so they 3596 * don't have to be checked again here. 3597 */ 3598 static int nested_vmx_check_permission(struct kvm_vcpu *vcpu) 3599 { 3600 if (!to_vmx(vcpu)->nested.vmxon) { 3601 kvm_queue_exception(vcpu, UD_VECTOR); 3602 return 0; 3603 } 3604 3605 if (vmx_get_cpl(vcpu)) { 3606 kvm_inject_gp(vcpu, 0); 3607 return 0; 3608 } 3609 3610 return 1; 3611 } 3612 3613 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, 3614 struct vmcs12 *vmcs12); 3615 3616 /* 3617 * If from_vmentry is false, this is being called from state restore (either RSM 3618 * or KVM_SET_NESTED_STATE). Otherwise it's called from vmlaunch/vmresume. 3619 * 3620 * Returns: 3621 * NVMX_VMENTRY_SUCCESS: Entered VMX non-root mode 3622 * NVMX_VMENTRY_VMFAIL: Consistency check VMFail 3623 * NVMX_VMENTRY_VMEXIT: Consistency check VMExit 3624 * NVMX_VMENTRY_KVM_INTERNAL_ERROR: KVM internal error 3625 */ 3626 enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, 3627 bool from_vmentry) 3628 { 3629 struct vcpu_vmx *vmx = to_vmx(vcpu); 3630 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3631 enum vm_entry_failure_code entry_failure_code; 3632 union vmx_exit_reason exit_reason = { 3633 .basic = EXIT_REASON_INVALID_STATE, 3634 .failed_vmentry = 1, 3635 }; 3636 u32 failed_index; 3637 3638 trace_kvm_nested_vmenter(kvm_rip_read(vcpu), 3639 vmx->nested.current_vmptr, 3640 vmcs12->guest_rip, 3641 vmcs12->guest_intr_status, 3642 vmcs12->vm_entry_intr_info_field, 3643 vmcs12->secondary_vm_exec_control & SECONDARY_EXEC_ENABLE_EPT, 3644 vmcs12->ept_pointer, 3645 vmcs12->guest_cr3, 3646 KVM_ISA_VMX); 3647 3648 kvm_service_local_tlb_flush_requests(vcpu); 3649 3650 if (!vcpu->arch.nested_run_pending || 3651 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) 3652 vmx->nested.pre_vmenter_debugctl = vmx_guest_debugctl_read(); 3653 if (kvm_mpx_supported() && 3654 (!vcpu->arch.nested_run_pending || 3655 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))) 3656 vmx->nested.pre_vmenter_bndcfgs = vmcs_read64(GUEST_BNDCFGS); 3657 3658 if (!vcpu->arch.nested_run_pending || 3659 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_CET_STATE)) 3660 vmcs_read_cet_state(vcpu, &vmx->nested.pre_vmenter_s_cet, 3661 &vmx->nested.pre_vmenter_ssp, 3662 &vmx->nested.pre_vmenter_ssp_tbl); 3663 3664 /* 3665 * Stash L1's CR3, so that in the event of a "late" VM-Fail, i.e. a 3666 * VM-Fail detected by hardware but not KVM, KVM can unwind its 3667 * software model to the pre-VM-Entry host state. When EPT is 3668 * disabled, GUEST_CR3 holds KVM's shadow CR3, not L1's "real" CR3, 3669 * and so simply restoring from vmcs01.GUEST_CR3 would corrupt 3670 * vcpu->arch.cr3. 3671 */ 3672 vmx->nested.pre_vmenter_cr3 = kvm_read_cr3(vcpu); 3673 3674 vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02); 3675 3676 prepare_vmcs02_early(vmx, &vmx->vmcs01, vmcs12); 3677 3678 if (from_vmentry) { 3679 if (unlikely(!nested_get_vmcs12_pages(vcpu))) { 3680 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3681 return NVMX_VMENTRY_KVM_INTERNAL_ERROR; 3682 } 3683 3684 if (nested_vmx_check_guest_state(vcpu, vmcs12, 3685 &entry_failure_code)) { 3686 exit_reason.basic = EXIT_REASON_INVALID_STATE; 3687 vmcs12->exit_qualification = entry_failure_code; 3688 goto vmentry_fail_vmexit; 3689 } 3690 } 3691 3692 enter_guest_mode(vcpu); 3693 3694 if (prepare_vmcs02(vcpu, vmcs12, from_vmentry, &entry_failure_code)) { 3695 exit_reason.basic = EXIT_REASON_INVALID_STATE; 3696 vmcs12->exit_qualification = entry_failure_code; 3697 goto vmentry_fail_vmexit_guest_mode; 3698 } 3699 3700 if (from_vmentry) { 3701 failed_index = nested_vmx_load_msr(vcpu, 3702 vmcs12->vm_entry_msr_load_addr, 3703 vmcs12->vm_entry_msr_load_count); 3704 if (failed_index) { 3705 exit_reason.basic = EXIT_REASON_MSR_LOAD_FAIL; 3706 vmcs12->exit_qualification = failed_index; 3707 goto vmentry_fail_vmexit_guest_mode; 3708 } 3709 } else { 3710 /* 3711 * The MMU is not initialized to point at the right entities yet and 3712 * "get pages" would need to read data from the guest (i.e. we will 3713 * need to perform gpa to hpa translation). Request a call 3714 * to nested_get_vmcs12_pages before the next VM-entry. The MSRs 3715 * have already been set at vmentry time and should not be reset. 3716 */ 3717 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 3718 } 3719 3720 /* 3721 * Re-evaluate pending events if L1 had a pending IRQ/NMI/INIT/SIPI 3722 * when it executed VMLAUNCH/VMRESUME, as entering non-root mode can 3723 * effectively unblock various events, e.g. INIT/SIPI cause VM-Exit 3724 * unconditionally. Take care to pull data from vmcs01 as appropriate, 3725 * e.g. when checking for interrupt windows, as vmcs02 is now loaded. 3726 */ 3727 if ((__exec_controls_get(&vmx->vmcs01) & (CPU_BASED_INTR_WINDOW_EXITING | 3728 CPU_BASED_NMI_WINDOW_EXITING)) || 3729 kvm_apic_has_pending_init_or_sipi(vcpu) || 3730 kvm_apic_has_interrupt(vcpu)) 3731 kvm_make_request(KVM_REQ_EVENT, vcpu); 3732 3733 /* 3734 * Do not start the preemption timer hrtimer until after we know 3735 * we are successful, so that only nested_vmx_vmexit needs to cancel 3736 * the timer. 3737 */ 3738 vmx->nested.preemption_timer_expired = false; 3739 if (nested_cpu_has_preemption_timer(vmcs12)) { 3740 u64 timer_value = vmx_calc_preemption_timer_value(vcpu); 3741 vmx_start_preemption_timer(vcpu, timer_value); 3742 } 3743 3744 /* 3745 * Note no nested_vmx_succeed or nested_vmx_fail here. At this point 3746 * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet 3747 * returned as far as L1 is concerned. It will only return (and set 3748 * the success flag) when L2 exits (see nested_vmx_vmexit()). 3749 */ 3750 return NVMX_VMENTRY_SUCCESS; 3751 3752 /* 3753 * A failed consistency check that leads to a VMExit during L1's 3754 * VMEnter to L2 is a variation of a normal VMexit, as explained in 3755 * 26.7 "VM-entry failures during or after loading guest state". 3756 */ 3757 vmentry_fail_vmexit_guest_mode: 3758 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING) 3759 vcpu->arch.tsc_offset -= vmcs12->tsc_offset; 3760 3761 /* 3762 * Handle any TLB flush requests that were queued for L2 if KVM made it 3763 * far enough along to switch to L2 context. Note, loading host state 3764 * will generate any flushes for L1 required by VM-Exit. 3765 */ 3766 kvm_service_local_tlb_flush_requests(vcpu); 3767 3768 leave_guest_mode(vcpu); 3769 3770 vmentry_fail_vmexit: 3771 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3772 3773 if (!from_vmentry) { 3774 nested_ept_uninit_mmu_context(vcpu); 3775 return NVMX_VMENTRY_VMEXIT; 3776 } 3777 3778 nested_put_vmcs12_pages(vcpu); 3779 3780 load_vmcs12_host_state(vcpu, vmcs12); 3781 vmcs12->vm_exit_reason = exit_reason.full; 3782 if (enable_shadow_vmcs || nested_vmx_is_evmptr12_valid(vmx)) 3783 vmx->nested.need_vmcs12_to_shadow_sync = true; 3784 return NVMX_VMENTRY_VMEXIT; 3785 } 3786 3787 /* 3788 * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1 3789 * for running an L2 nested guest. 3790 */ 3791 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) 3792 { 3793 struct vmcs12 *vmcs12; 3794 enum nvmx_vmentry_status status; 3795 struct vcpu_vmx *vmx = to_vmx(vcpu); 3796 u32 interrupt_shadow = vmx_get_interrupt_shadow(vcpu); 3797 enum nested_evmptrld_status evmptrld_status; 3798 3799 if (!nested_vmx_check_permission(vcpu)) 3800 return 1; 3801 3802 evmptrld_status = nested_vmx_handle_enlightened_vmptrld(vcpu, launch); 3803 if (evmptrld_status == EVMPTRLD_ERROR) { 3804 kvm_queue_exception(vcpu, UD_VECTOR); 3805 return 1; 3806 } 3807 3808 kvm_pmu_branch_retired(vcpu); 3809 3810 if (CC(evmptrld_status == EVMPTRLD_VMFAIL)) 3811 return nested_vmx_failInvalid(vcpu); 3812 3813 if (CC(!nested_vmx_is_evmptr12_valid(vmx) && 3814 vmx->nested.current_vmptr == INVALID_GPA)) 3815 return nested_vmx_failInvalid(vcpu); 3816 3817 vmcs12 = get_vmcs12(vcpu); 3818 3819 /* 3820 * Can't VMLAUNCH or VMRESUME a shadow VMCS. Despite the fact 3821 * that there *is* a valid VMCS pointer, RFLAGS.CF is set 3822 * rather than RFLAGS.ZF, and no error number is stored to the 3823 * VM-instruction error field. 3824 */ 3825 if (CC(vmcs12->hdr.shadow_vmcs)) 3826 return nested_vmx_failInvalid(vcpu); 3827 3828 if (nested_vmx_is_evmptr12_valid(vmx)) { 3829 struct hv_enlightened_vmcs *evmcs = nested_vmx_evmcs(vmx); 3830 3831 copy_enlightened_to_vmcs12(vmx, evmcs->hv_clean_fields); 3832 /* Enlightened VMCS doesn't have launch state */ 3833 vmcs12->launch_state = !launch; 3834 } else if (enable_shadow_vmcs) { 3835 copy_shadow_to_vmcs12(vmx); 3836 } 3837 3838 /* 3839 * The nested entry process starts with enforcing various prerequisites 3840 * on vmcs12 as required by the Intel SDM, and act appropriately when 3841 * they fail: As the SDM explains, some conditions should cause the 3842 * instruction to fail, while others will cause the instruction to seem 3843 * to succeed, but return an EXIT_REASON_INVALID_STATE. 3844 * To speed up the normal (success) code path, we should avoid checking 3845 * for misconfigurations which will anyway be caught by the processor 3846 * when using the merged vmcs02. 3847 */ 3848 if (CC(interrupt_shadow & KVM_X86_SHADOW_INT_MOV_SS)) 3849 return nested_vmx_fail(vcpu, VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS); 3850 3851 if (CC(vmcs12->launch_state == launch)) 3852 return nested_vmx_fail(vcpu, 3853 launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS 3854 : VMXERR_VMRESUME_NONLAUNCHED_VMCS); 3855 3856 if (nested_vmx_check_controls(vcpu, vmcs12)) 3857 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3858 3859 if (nested_vmx_check_address_space_size(vcpu, vmcs12)) 3860 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_HOST_STATE_FIELD); 3861 3862 if (nested_vmx_check_host_state(vcpu, vmcs12)) 3863 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_HOST_STATE_FIELD); 3864 3865 /* 3866 * We're finally done with prerequisite checking, and can start with 3867 * the nested entry. 3868 */ 3869 vcpu->arch.nested_run_pending = KVM_NESTED_RUN_PENDING; 3870 vmx->nested.has_preemption_timer_deadline = false; 3871 status = nested_vmx_enter_non_root_mode(vcpu, true); 3872 if (unlikely(status != NVMX_VMENTRY_SUCCESS)) 3873 goto vmentry_failed; 3874 3875 /* Hide L1D cache contents from the nested guest. */ 3876 kvm_request_l1tf_flush_l1d(); 3877 3878 /* 3879 * Must happen outside of nested_vmx_enter_non_root_mode() as it will 3880 * also be used as part of restoring nVMX state for 3881 * snapshot restore (migration). 3882 * 3883 * In this flow, it is assumed that vmcs12 cache was 3884 * transferred as part of captured nVMX state and should 3885 * therefore not be read from guest memory (which may not 3886 * exist on destination host yet). 3887 */ 3888 nested_cache_shadow_vmcs12(vcpu, vmcs12); 3889 3890 switch (vmcs12->guest_activity_state) { 3891 case GUEST_ACTIVITY_HLT: 3892 /* 3893 * If we're entering a halted L2 vcpu and the L2 vcpu won't be 3894 * awakened by event injection or by an NMI-window VM-exit or 3895 * by an interrupt-window VM-exit, halt the vcpu. 3896 */ 3897 if (!(vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) && 3898 !nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING) && 3899 !(nested_cpu_has(vmcs12, CPU_BASED_INTR_WINDOW_EXITING) && 3900 (vmcs12->guest_rflags & X86_EFLAGS_IF))) { 3901 vcpu->arch.nested_run_pending = 0; 3902 return kvm_emulate_halt_noskip(vcpu); 3903 } 3904 break; 3905 case GUEST_ACTIVITY_WAIT_SIPI: 3906 vcpu->arch.nested_run_pending = 0; 3907 kvm_set_mp_state(vcpu, KVM_MP_STATE_INIT_RECEIVED); 3908 break; 3909 default: 3910 break; 3911 } 3912 3913 return 1; 3914 3915 vmentry_failed: 3916 vcpu->arch.nested_run_pending = 0; 3917 if (status == NVMX_VMENTRY_KVM_INTERNAL_ERROR) { 3918 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 3919 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; 3920 vcpu->run->internal.ndata = 0; 3921 return 0; 3922 } 3923 if (status == NVMX_VMENTRY_VMEXIT) 3924 return 1; 3925 WARN_ON_ONCE(status != NVMX_VMENTRY_VMFAIL); 3926 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3927 } 3928 3929 /* 3930 * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date 3931 * because L2 may have changed some cr0 bits directly (CR0_GUEST_HOST_MASK). 3932 * This function returns the new value we should put in vmcs12.guest_cr0. 3933 * It's not enough to just return the vmcs02 GUEST_CR0. Rather, 3934 * 1. Bits that neither L0 nor L1 trapped, were set directly by L2 and are now 3935 * available in vmcs02 GUEST_CR0. (Note: It's enough to check that L0 3936 * didn't trap the bit, because if L1 did, so would L0). 3937 * 2. Bits that L1 asked to trap (and therefore L0 also did) could not have 3938 * been modified by L2, and L1 knows it. So just leave the old value of 3939 * the bit from vmcs12.guest_cr0. Note that the bit from vmcs02 GUEST_CR0 3940 * isn't relevant, because if L0 traps this bit it can set it to anything. 3941 * 3. Bits that L1 didn't trap, but L0 did. L1 believes the guest could have 3942 * changed these bits, and therefore they need to be updated, but L0 3943 * didn't necessarily allow them to be changed in GUEST_CR0 - and rather 3944 * put them in vmcs02 CR0_READ_SHADOW. So take these bits from there. 3945 */ 3946 static inline unsigned long 3947 vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3948 { 3949 return 3950 /*1*/ (vmcs_readl(GUEST_CR0) & vcpu->arch.cr0_guest_owned_bits) | 3951 /*2*/ (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) | 3952 /*3*/ (vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask | 3953 vcpu->arch.cr0_guest_owned_bits)); 3954 } 3955 3956 static inline unsigned long 3957 vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3958 { 3959 return 3960 /*1*/ (vmcs_readl(GUEST_CR4) & vcpu->arch.cr4_guest_owned_bits) | 3961 /*2*/ (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) | 3962 /*3*/ (vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask | 3963 vcpu->arch.cr4_guest_owned_bits)); 3964 } 3965 3966 static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu, 3967 struct vmcs12 *vmcs12, 3968 u32 vm_exit_reason, u32 exit_intr_info) 3969 { 3970 u32 idt_vectoring; 3971 unsigned int nr; 3972 3973 /* 3974 * Per the SDM, VM-Exits due to double and triple faults are never 3975 * considered to occur during event delivery, even if the double/triple 3976 * fault is the result of an escalating vectoring issue. 3977 * 3978 * Note, the SDM qualifies the double fault behavior with "The original 3979 * event results in a double-fault exception". It's unclear why the 3980 * qualification exists since exits due to double fault can occur only 3981 * while vectoring a different exception (injected events are never 3982 * subject to interception), i.e. there's _always_ an original event. 3983 * 3984 * The SDM also uses NMI as a confusing example for the "original event 3985 * causes the VM exit directly" clause. NMI isn't special in any way, 3986 * the same rule applies to all events that cause an exit directly. 3987 * NMI is an odd choice for the example because NMIs can only occur on 3988 * instruction boundaries, i.e. they _can't_ occur during vectoring. 3989 */ 3990 if ((u16)vm_exit_reason == EXIT_REASON_TRIPLE_FAULT || 3991 ((u16)vm_exit_reason == EXIT_REASON_EXCEPTION_NMI && 3992 is_double_fault(exit_intr_info))) { 3993 vmcs12->idt_vectoring_info_field = 0; 3994 } else if (vcpu->arch.exception.injected) { 3995 nr = vcpu->arch.exception.vector; 3996 idt_vectoring = nr | VECTORING_INFO_VALID_MASK; 3997 3998 if (kvm_exception_is_soft(nr)) { 3999 vmcs12->vm_exit_instruction_len = 4000 vcpu->arch.event_exit_inst_len; 4001 idt_vectoring |= INTR_TYPE_SOFT_EXCEPTION; 4002 } else 4003 idt_vectoring |= INTR_TYPE_HARD_EXCEPTION; 4004 4005 if (vcpu->arch.exception.has_error_code) { 4006 idt_vectoring |= VECTORING_INFO_DELIVER_CODE_MASK; 4007 vmcs12->idt_vectoring_error_code = 4008 vcpu->arch.exception.error_code; 4009 } 4010 4011 vmcs12->idt_vectoring_info_field = idt_vectoring; 4012 } else if (vcpu->arch.nmi_injected) { 4013 vmcs12->idt_vectoring_info_field = 4014 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR; 4015 } else if (vcpu->arch.interrupt.injected) { 4016 nr = vcpu->arch.interrupt.nr; 4017 idt_vectoring = nr | VECTORING_INFO_VALID_MASK; 4018 4019 if (vcpu->arch.interrupt.soft) { 4020 idt_vectoring |= INTR_TYPE_SOFT_INTR; 4021 vmcs12->vm_entry_instruction_len = 4022 vcpu->arch.event_exit_inst_len; 4023 } else 4024 idt_vectoring |= INTR_TYPE_EXT_INTR; 4025 4026 vmcs12->idt_vectoring_info_field = idt_vectoring; 4027 } else { 4028 vmcs12->idt_vectoring_info_field = 0; 4029 } 4030 } 4031 4032 static int vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu) 4033 { 4034 struct vcpu_vmx *vmx = to_vmx(vcpu); 4035 int max_irr; 4036 void *vapic_page; 4037 u16 status; 4038 4039 if (!vmx->nested.pi_pending) 4040 return 0; 4041 4042 if (!vmx->nested.pi_desc) 4043 goto mmio_needed; 4044 4045 vmx->nested.pi_pending = false; 4046 4047 if (!pi_test_and_clear_on(vmx->nested.pi_desc)) 4048 return 0; 4049 4050 max_irr = pi_find_highest_vector(vmx->nested.pi_desc); 4051 if (max_irr > 0) { 4052 vapic_page = vmx->nested.virtual_apic_map.hva; 4053 if (!vapic_page) 4054 goto mmio_needed; 4055 4056 __kvm_apic_update_irr(vmx->nested.pi_desc->pir, 4057 vapic_page, &max_irr); 4058 status = vmcs_read16(GUEST_INTR_STATUS); 4059 if ((u8)max_irr > ((u8)status & 0xff)) { 4060 status &= ~0xff; 4061 status |= (u8)max_irr; 4062 vmcs_write16(GUEST_INTR_STATUS, status); 4063 } 4064 } 4065 4066 kvm_vcpu_map_mark_dirty(vcpu, &vmx->nested.virtual_apic_map); 4067 kvm_vcpu_map_mark_dirty(vcpu, &vmx->nested.pi_desc_map); 4068 return 0; 4069 4070 mmio_needed: 4071 kvm_handle_memory_failure(vcpu, X86EMUL_IO_NEEDED, NULL); 4072 return -ENXIO; 4073 } 4074 4075 static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu) 4076 { 4077 struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit; 4078 u32 intr_info = ex->vector | INTR_INFO_VALID_MASK; 4079 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 4080 unsigned long exit_qual; 4081 4082 if (ex->has_payload) { 4083 exit_qual = ex->payload; 4084 } else if (ex->vector == PF_VECTOR) { 4085 exit_qual = vcpu->arch.cr2; 4086 } else if (ex->vector == DB_VECTOR) { 4087 exit_qual = vcpu->arch.dr6; 4088 exit_qual &= ~DR6_BT; 4089 exit_qual ^= DR6_ACTIVE_LOW; 4090 } else { 4091 exit_qual = 0; 4092 } 4093 4094 /* 4095 * Unlike AMD's Paged Real Mode, which reports an error code on #PF 4096 * VM-Exits even if the CPU is in Real Mode, Intel VMX never sets the 4097 * "has error code" flags on VM-Exit if the CPU is in Real Mode. 4098 */ 4099 if (ex->has_error_code && is_protmode(vcpu)) { 4100 /* 4101 * Intel CPUs do not generate error codes with bits 31:16 set, 4102 * and more importantly VMX disallows setting bits 31:16 in the 4103 * injected error code for VM-Entry. Drop the bits to mimic 4104 * hardware and avoid inducing failure on nested VM-Entry if L1 4105 * chooses to inject the exception back to L2. AMD CPUs _do_ 4106 * generate "full" 32-bit error codes, so KVM allows userspace 4107 * to inject exception error codes with bits 31:16 set. 4108 */ 4109 vmcs12->vm_exit_intr_error_code = (u16)ex->error_code; 4110 intr_info |= INTR_INFO_DELIVER_CODE_MASK; 4111 } 4112 4113 if (kvm_exception_is_soft(ex->vector)) 4114 intr_info |= INTR_TYPE_SOFT_EXCEPTION; 4115 else 4116 intr_info |= INTR_TYPE_HARD_EXCEPTION; 4117 4118 if (!(vmcs12->idt_vectoring_info_field & VECTORING_INFO_VALID_MASK) && 4119 vmx_get_nmi_mask(vcpu)) 4120 intr_info |= INTR_INFO_UNBLOCK_NMI; 4121 4122 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, intr_info, exit_qual); 4123 } 4124 4125 /* 4126 * Returns true if a debug trap is (likely) pending delivery. Infer the class 4127 * of a #DB (trap-like vs. fault-like) from the exception payload (to-be-DR6). 4128 * Using the payload is flawed because code breakpoints (fault-like) and data 4129 * breakpoints (trap-like) set the same bits in DR6 (breakpoint detected), i.e. 4130 * this will return false positives if a to-be-injected code breakpoint #DB is 4131 * pending (from KVM's perspective, but not "pending" across an instruction 4132 * boundary). ICEBP, a.k.a. INT1, is also not reflected here even though it 4133 * too is trap-like. 4134 * 4135 * KVM "works" despite these flaws as ICEBP isn't currently supported by the 4136 * emulator, Monitor Trap Flag is not marked pending on intercepted #DBs (the 4137 * #DB has already happened), and MTF isn't marked pending on code breakpoints 4138 * from the emulator (because such #DBs are fault-like and thus don't trigger 4139 * actions that fire on instruction retire). 4140 */ 4141 static unsigned long vmx_get_pending_dbg_trap(struct kvm_queued_exception *ex) 4142 { 4143 if (!ex->pending || ex->vector != DB_VECTOR) 4144 return 0; 4145 4146 /* General Detect #DBs are always fault-like. */ 4147 return ex->payload & ~DR6_BD; 4148 } 4149 4150 /* 4151 * Returns true if there's a pending #DB exception that is lower priority than 4152 * a pending Monitor Trap Flag VM-Exit. TSS T-flag #DBs are not emulated by 4153 * KVM, but could theoretically be injected by userspace. Note, this code is 4154 * imperfect, see above. 4155 */ 4156 static bool vmx_is_low_priority_db_trap(struct kvm_queued_exception *ex) 4157 { 4158 return vmx_get_pending_dbg_trap(ex) & ~DR6_BT; 4159 } 4160 4161 /* 4162 * Certain VM-exits set the 'pending debug exceptions' field to indicate a 4163 * recognized #DB (data or single-step) that has yet to be delivered. Since KVM 4164 * represents these debug traps with a payload that is said to be compatible 4165 * with the 'pending debug exceptions' field, write the payload to the VMCS 4166 * field if a VM-exit is delivered before the debug trap. 4167 */ 4168 static void nested_vmx_update_pending_dbg(struct kvm_vcpu *vcpu) 4169 { 4170 unsigned long pending_dbg; 4171 4172 pending_dbg = vmx_get_pending_dbg_trap(&vcpu->arch.exception); 4173 if (pending_dbg) 4174 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, pending_dbg); 4175 } 4176 4177 static bool nested_vmx_preemption_timer_pending(struct kvm_vcpu *vcpu) 4178 { 4179 return nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) && 4180 to_vmx(vcpu)->nested.preemption_timer_expired; 4181 } 4182 4183 static bool vmx_has_nested_events(struct kvm_vcpu *vcpu, bool for_injection) 4184 { 4185 struct vcpu_vmx *vmx = to_vmx(vcpu); 4186 void *vapic = vmx->nested.virtual_apic_map.hva; 4187 int max_irr, vppr; 4188 4189 if (nested_vmx_preemption_timer_pending(vcpu) || 4190 vmx->nested.mtf_pending) 4191 return true; 4192 4193 /* 4194 * Virtual Interrupt Delivery doesn't require manual injection. Either 4195 * the interrupt is already in GUEST_RVI and will be recognized by CPU 4196 * at VM-Entry, or there is a KVM_REQ_EVENT pending and KVM will move 4197 * the interrupt from the PIR to RVI prior to entering the guest. 4198 */ 4199 if (for_injection) 4200 return false; 4201 4202 if (!nested_cpu_has_vid(get_vmcs12(vcpu)) || 4203 __vmx_interrupt_blocked(vcpu)) 4204 return false; 4205 4206 if (!vapic) 4207 return false; 4208 4209 vppr = *((u32 *)(vapic + APIC_PROCPRI)); 4210 4211 max_irr = vmx_get_rvi(); 4212 if ((max_irr & 0xf0) > (vppr & 0xf0)) 4213 return true; 4214 4215 if (vmx->nested.pi_pending && vmx->nested.pi_desc && 4216 pi_test_on(vmx->nested.pi_desc)) { 4217 max_irr = pi_find_highest_vector(vmx->nested.pi_desc); 4218 if (max_irr > 0 && (max_irr & 0xf0) > (vppr & 0xf0)) 4219 return true; 4220 } 4221 4222 return false; 4223 } 4224 4225 /* 4226 * Per the Intel SDM's table "Priority Among Concurrent Events", with minor 4227 * edits to fill in missing examples, e.g. #DB due to split-lock accesses, 4228 * and less minor edits to splice in the priority of VMX Non-Root specific 4229 * events, e.g. MTF and NMI/INTR-window exiting. 4230 * 4231 * 1 Hardware Reset and Machine Checks 4232 * - RESET 4233 * - Machine Check 4234 * 4235 * 2 Trap on Task Switch 4236 * - T flag in TSS is set (on task switch) 4237 * 4238 * 3 External Hardware Interventions 4239 * - FLUSH 4240 * - STOPCLK 4241 * - SMI 4242 * - INIT 4243 * 4244 * 3.5 Monitor Trap Flag (MTF) VM-exit[1] 4245 * 4246 * 4 Traps on Previous Instruction 4247 * - Breakpoints 4248 * - Trap-class Debug Exceptions (#DB due to TF flag set, data/I-O 4249 * breakpoint, or #DB due to a split-lock access) 4250 * 4251 * 4.3 VMX-preemption timer expired VM-exit 4252 * 4253 * 4.6 NMI-window exiting VM-exit[2] 4254 * 4255 * 5 Nonmaskable Interrupts (NMI) 4256 * 4257 * 5.5 Interrupt-window exiting VM-exit and Virtual-interrupt delivery 4258 * 4259 * 6 Maskable Hardware Interrupts 4260 * 4261 * 7 Code Breakpoint Fault 4262 * 4263 * 8 Faults from Fetching Next Instruction 4264 * - Code-Segment Limit Violation 4265 * - Code Page Fault 4266 * - Control protection exception (missing ENDBRANCH at target of indirect 4267 * call or jump) 4268 * 4269 * 9 Faults from Decoding Next Instruction 4270 * - Instruction length > 15 bytes 4271 * - Invalid Opcode 4272 * - Coprocessor Not Available 4273 * 4274 *10 Faults on Executing Instruction 4275 * - Overflow 4276 * - Bound error 4277 * - Invalid TSS 4278 * - Segment Not Present 4279 * - Stack fault 4280 * - General Protection 4281 * - Data Page Fault 4282 * - Alignment Check 4283 * - x86 FPU Floating-point exception 4284 * - SIMD floating-point exception 4285 * - Virtualization exception 4286 * - Control protection exception 4287 * 4288 * [1] Per the "Monitor Trap Flag" section: System-management interrupts (SMIs), 4289 * INIT signals, and higher priority events take priority over MTF VM exits. 4290 * MTF VM exits take priority over debug-trap exceptions and lower priority 4291 * events. 4292 * 4293 * [2] Debug-trap exceptions and higher priority events take priority over VM exits 4294 * caused by the VMX-preemption timer. VM exits caused by the VMX-preemption 4295 * timer take priority over VM exits caused by the "NMI-window exiting" 4296 * VM-execution control and lower priority events. 4297 * 4298 * [3] Debug-trap exceptions and higher priority events take priority over VM exits 4299 * caused by "NMI-window exiting". VM exits caused by this control take 4300 * priority over non-maskable interrupts (NMIs) and lower priority events. 4301 * 4302 * [4] Virtual-interrupt delivery has the same priority as that of VM exits due to 4303 * the 1-setting of the "interrupt-window exiting" VM-execution control. Thus, 4304 * non-maskable interrupts (NMIs) and higher priority events take priority over 4305 * delivery of a virtual interrupt; delivery of a virtual interrupt takes 4306 * priority over external interrupts and lower priority events. 4307 */ 4308 static int vmx_check_nested_events(struct kvm_vcpu *vcpu) 4309 { 4310 struct kvm_lapic *apic = vcpu->arch.apic; 4311 struct vcpu_vmx *vmx = to_vmx(vcpu); 4312 /* 4313 * Only a pending nested run blocks a pending exception. If there is a 4314 * previously injected event, the pending exception occurred while said 4315 * event was being delivered and thus needs to be handled. 4316 */ 4317 bool block_nested_exceptions = vcpu->arch.nested_run_pending; 4318 /* 4319 * Events that don't require injection, i.e. that are virtualized by 4320 * hardware, aren't blocked by a pending VM-Enter as KVM doesn't need 4321 * to regain control in order to deliver the event, and hardware will 4322 * handle event ordering, e.g. with respect to injected exceptions. 4323 * 4324 * But, new events (not exceptions) are only recognized at instruction 4325 * boundaries. If an event needs reinjection, then KVM is handling a 4326 * VM-Exit that occurred _during_ instruction execution; new events, 4327 * irrespective of whether or not they're injected, are blocked until 4328 * the instruction completes. 4329 */ 4330 bool block_non_injected_events = kvm_event_needs_reinjection(vcpu); 4331 /* 4332 * Inject events are blocked by nested VM-Enter, as KVM is responsible 4333 * for managing priority between concurrent events, i.e. KVM needs to 4334 * wait until after VM-Enter completes to deliver injected events. 4335 */ 4336 bool block_nested_events = block_nested_exceptions || 4337 block_non_injected_events; 4338 4339 if (lapic_in_kernel(vcpu) && 4340 test_bit(KVM_APIC_INIT, &apic->pending_events)) { 4341 if (block_nested_events) 4342 return -EBUSY; 4343 nested_vmx_update_pending_dbg(vcpu); 4344 clear_bit(KVM_APIC_INIT, &apic->pending_events); 4345 if (vcpu->arch.mp_state != KVM_MP_STATE_INIT_RECEIVED) 4346 nested_vmx_vmexit(vcpu, EXIT_REASON_INIT_SIGNAL, 0, 0); 4347 4348 /* MTF is discarded if the vCPU is in WFS. */ 4349 vmx->nested.mtf_pending = false; 4350 return 0; 4351 } 4352 4353 if (lapic_in_kernel(vcpu) && 4354 test_bit(KVM_APIC_SIPI, &apic->pending_events)) { 4355 if (block_nested_events) 4356 return -EBUSY; 4357 4358 clear_bit(KVM_APIC_SIPI, &apic->pending_events); 4359 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { 4360 nested_vmx_vmexit(vcpu, EXIT_REASON_SIPI_SIGNAL, 0, 4361 apic->sipi_vector & 0xFFUL); 4362 return 0; 4363 } 4364 /* Fallthrough, the SIPI is completely ignored. */ 4365 } 4366 4367 /* 4368 * Process exceptions that are higher priority than Monitor Trap Flag: 4369 * fault-like exceptions, TSS T flag #DB (not emulated by KVM, but 4370 * could theoretically come in from userspace), and ICEBP (INT1). 4371 * 4372 * TODO: SMIs have higher priority than MTF and trap-like #DBs (except 4373 * for TSS T flag #DBs). KVM also doesn't save/restore pending MTF 4374 * across SMI/RSM as it should; that needs to be addressed in order to 4375 * prioritize SMI over MTF and trap-like #DBs. 4376 */ 4377 if (vcpu->arch.exception_vmexit.pending && 4378 !vmx_is_low_priority_db_trap(&vcpu->arch.exception_vmexit)) { 4379 if (block_nested_exceptions) 4380 return -EBUSY; 4381 4382 nested_vmx_inject_exception_vmexit(vcpu); 4383 return 0; 4384 } 4385 4386 if (vcpu->arch.exception.pending && 4387 !vmx_is_low_priority_db_trap(&vcpu->arch.exception)) { 4388 if (block_nested_exceptions) 4389 return -EBUSY; 4390 goto no_vmexit; 4391 } 4392 4393 if (vmx->nested.mtf_pending) { 4394 if (block_nested_events) 4395 return -EBUSY; 4396 nested_vmx_update_pending_dbg(vcpu); 4397 nested_vmx_vmexit(vcpu, EXIT_REASON_MONITOR_TRAP_FLAG, 0, 0); 4398 return 0; 4399 } 4400 4401 if (vcpu->arch.exception_vmexit.pending) { 4402 if (block_nested_exceptions) 4403 return -EBUSY; 4404 4405 nested_vmx_inject_exception_vmexit(vcpu); 4406 return 0; 4407 } 4408 4409 if (vcpu->arch.exception.pending) { 4410 if (block_nested_exceptions) 4411 return -EBUSY; 4412 goto no_vmexit; 4413 } 4414 4415 if (nested_vmx_preemption_timer_pending(vcpu)) { 4416 if (block_nested_events) 4417 return -EBUSY; 4418 nested_vmx_vmexit(vcpu, EXIT_REASON_PREEMPTION_TIMER, 0, 0); 4419 return 0; 4420 } 4421 4422 if (vcpu->arch.smi_pending && !is_smm(vcpu)) { 4423 if (block_nested_events) 4424 return -EBUSY; 4425 goto no_vmexit; 4426 } 4427 4428 if (vcpu->arch.nmi_pending && !vmx_nmi_blocked(vcpu)) { 4429 if (block_nested_events) 4430 return -EBUSY; 4431 if (!nested_exit_on_nmi(vcpu)) 4432 goto no_vmexit; 4433 4434 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, 4435 NMI_VECTOR | INTR_TYPE_NMI_INTR | 4436 INTR_INFO_VALID_MASK, 0); 4437 /* 4438 * The NMI-triggered VM exit counts as injection: 4439 * clear this one and block further NMIs. 4440 */ 4441 vcpu->arch.nmi_pending = 0; 4442 vmx_set_nmi_mask(vcpu, true); 4443 return 0; 4444 } 4445 4446 if (kvm_cpu_has_interrupt(vcpu) && !vmx_interrupt_blocked(vcpu)) { 4447 int irq; 4448 4449 if (!nested_exit_on_intr(vcpu)) { 4450 if (block_nested_events) 4451 return -EBUSY; 4452 4453 goto no_vmexit; 4454 } 4455 4456 if (!nested_exit_intr_ack_set(vcpu)) { 4457 if (block_nested_events) 4458 return -EBUSY; 4459 4460 nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0); 4461 return 0; 4462 } 4463 4464 irq = kvm_cpu_get_extint(vcpu); 4465 if (irq != -1) { 4466 if (block_nested_events) 4467 return -EBUSY; 4468 4469 nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 4470 INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR | irq, 0); 4471 return 0; 4472 } 4473 4474 irq = kvm_apic_has_interrupt(vcpu); 4475 if (unlikely(irq < 0)) { 4476 kvm_warn_on_lost_irq(vcpu); 4477 goto no_vmexit; 4478 } 4479 4480 /* 4481 * If the IRQ is L2's PI notification vector, process posted 4482 * interrupts for L2 instead of injecting VM-Exit, as the 4483 * detection/morphing architecturally occurs when the IRQ is 4484 * delivered to the CPU. Note, only interrupts that are routed 4485 * through the local APIC trigger posted interrupt processing, 4486 * and enabling posted interrupts requires ACK-on-exit. 4487 */ 4488 if (irq == vmx->nested.posted_intr_nv) { 4489 /* 4490 * Nested posted interrupts are delivered via RVI, i.e. 4491 * aren't injected by KVM, and so can be queued even if 4492 * manual event injection is disallowed. 4493 */ 4494 if (block_non_injected_events) 4495 return -EBUSY; 4496 4497 vmx->nested.pi_pending = true; 4498 kvm_apic_clear_irr(vcpu, irq); 4499 goto no_vmexit; 4500 } 4501 4502 if (block_nested_events) 4503 return -EBUSY; 4504 4505 nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 4506 INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR | irq, 0); 4507 4508 /* 4509 * ACK the interrupt _after_ emulating VM-Exit, as the IRQ must 4510 * be marked as in-service in vmcs01.GUEST_INTERRUPT_STATUS.SVI 4511 * if APICv is active. 4512 */ 4513 kvm_apic_ack_interrupt(vcpu, irq); 4514 return 0; 4515 } 4516 4517 no_vmexit: 4518 return vmx_complete_nested_posted_interrupt(vcpu); 4519 } 4520 4521 static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu) 4522 { 4523 ktime_t remaining = 4524 hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer); 4525 u64 value; 4526 4527 if (ktime_to_ns(remaining) <= 0) 4528 return 0; 4529 4530 value = ktime_to_ns(remaining) * vcpu->arch.virtual_tsc_khz; 4531 do_div(value, 1000000); 4532 return value >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 4533 } 4534 4535 static bool is_vmcs12_ext_field(unsigned long field) 4536 { 4537 switch (field) { 4538 case GUEST_ES_SELECTOR: 4539 case GUEST_CS_SELECTOR: 4540 case GUEST_SS_SELECTOR: 4541 case GUEST_DS_SELECTOR: 4542 case GUEST_FS_SELECTOR: 4543 case GUEST_GS_SELECTOR: 4544 case GUEST_LDTR_SELECTOR: 4545 case GUEST_TR_SELECTOR: 4546 case GUEST_ES_LIMIT: 4547 case GUEST_CS_LIMIT: 4548 case GUEST_SS_LIMIT: 4549 case GUEST_DS_LIMIT: 4550 case GUEST_FS_LIMIT: 4551 case GUEST_GS_LIMIT: 4552 case GUEST_LDTR_LIMIT: 4553 case GUEST_TR_LIMIT: 4554 case GUEST_GDTR_LIMIT: 4555 case GUEST_IDTR_LIMIT: 4556 case GUEST_ES_AR_BYTES: 4557 case GUEST_DS_AR_BYTES: 4558 case GUEST_FS_AR_BYTES: 4559 case GUEST_GS_AR_BYTES: 4560 case GUEST_LDTR_AR_BYTES: 4561 case GUEST_TR_AR_BYTES: 4562 case GUEST_ES_BASE: 4563 case GUEST_CS_BASE: 4564 case GUEST_SS_BASE: 4565 case GUEST_DS_BASE: 4566 case GUEST_FS_BASE: 4567 case GUEST_GS_BASE: 4568 case GUEST_LDTR_BASE: 4569 case GUEST_TR_BASE: 4570 case GUEST_GDTR_BASE: 4571 case GUEST_IDTR_BASE: 4572 case GUEST_PENDING_DBG_EXCEPTIONS: 4573 case GUEST_BNDCFGS: 4574 return true; 4575 default: 4576 break; 4577 } 4578 4579 return false; 4580 } 4581 4582 static void sync_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu, 4583 struct vmcs12 *vmcs12) 4584 { 4585 struct vcpu_vmx *vmx = to_vmx(vcpu); 4586 4587 vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR); 4588 vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR); 4589 vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR); 4590 vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR); 4591 vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR); 4592 vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR); 4593 vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR); 4594 vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR); 4595 vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT); 4596 vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT); 4597 vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT); 4598 vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT); 4599 vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT); 4600 vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT); 4601 vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT); 4602 vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT); 4603 vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT); 4604 vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT); 4605 vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES); 4606 vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES); 4607 vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES); 4608 vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES); 4609 vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES); 4610 vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES); 4611 vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE); 4612 vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE); 4613 vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE); 4614 vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE); 4615 vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE); 4616 vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE); 4617 vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE); 4618 vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE); 4619 vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE); 4620 vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE); 4621 vmcs12->guest_pending_dbg_exceptions = 4622 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS); 4623 4624 vmx->nested.need_sync_vmcs02_to_vmcs12_rare = false; 4625 } 4626 4627 static void copy_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu, 4628 struct vmcs12 *vmcs12) 4629 { 4630 struct vcpu_vmx *vmx = to_vmx(vcpu); 4631 int cpu; 4632 4633 if (!vmx->nested.need_sync_vmcs02_to_vmcs12_rare) 4634 return; 4635 4636 4637 WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01); 4638 4639 cpu = get_cpu(); 4640 vmx->loaded_vmcs = &vmx->nested.vmcs02; 4641 vmx_vcpu_load_vmcs(vcpu, cpu); 4642 4643 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 4644 4645 vmx->loaded_vmcs = &vmx->vmcs01; 4646 vmx_vcpu_load_vmcs(vcpu, cpu); 4647 put_cpu(); 4648 } 4649 4650 /* 4651 * Update the guest state fields of vmcs12 to reflect changes that 4652 * occurred while L2 was running. (The "IA-32e mode guest" bit of the 4653 * VM-entry controls is also updated, since this is really a guest 4654 * state bit.) 4655 */ 4656 static void sync_vmcs02_to_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 4657 { 4658 struct vcpu_vmx *vmx = to_vmx(vcpu); 4659 4660 if (nested_vmx_is_evmptr12_valid(vmx)) 4661 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 4662 4663 vmx->nested.need_sync_vmcs02_to_vmcs12_rare = 4664 !nested_vmx_is_evmptr12_valid(vmx); 4665 4666 vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12); 4667 vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12); 4668 4669 vmcs12->guest_rsp = kvm_rsp_read(vcpu); 4670 vmcs12->guest_rip = kvm_rip_read(vcpu); 4671 vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS); 4672 4673 vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES); 4674 vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES); 4675 4676 vmcs12->guest_interruptibility_info = 4677 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 4678 4679 if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED) 4680 vmcs12->guest_activity_state = GUEST_ACTIVITY_HLT; 4681 else if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) 4682 vmcs12->guest_activity_state = GUEST_ACTIVITY_WAIT_SIPI; 4683 else 4684 vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE; 4685 4686 if (nested_cpu_has_preemption_timer(vmcs12) && 4687 vmcs12->vm_exit_controls & VM_EXIT_SAVE_VMX_PREEMPTION_TIMER && 4688 !vcpu->arch.nested_run_pending) 4689 vmcs12->vmx_preemption_timer_value = 4690 vmx_get_preemption_timer_value(vcpu); 4691 4692 /* 4693 * In some cases (usually, nested EPT), L2 is allowed to change its 4694 * own CR3 without exiting. If it has changed it, we must keep it. 4695 * Of course, if L0 is using shadow page tables, GUEST_CR3 was defined 4696 * by L0, not L1 or L2, so we mustn't unconditionally copy it to vmcs12. 4697 * 4698 * Additionally, restore L2's PDPTR to vmcs12. 4699 */ 4700 if (enable_ept) { 4701 vmcs12->guest_cr3 = vmcs_readl(GUEST_CR3); 4702 if (nested_cpu_has_ept(vmcs12) && is_pae_paging(vcpu)) { 4703 vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0); 4704 vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1); 4705 vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2); 4706 vmcs12->guest_pdptr3 = vmcs_read64(GUEST_PDPTR3); 4707 } 4708 } 4709 4710 vmcs12->guest_linear_address = vmcs_readl(GUEST_LINEAR_ADDRESS); 4711 4712 if (nested_cpu_has_vid(vmcs12)) 4713 vmcs12->guest_intr_status = vmcs_read16(GUEST_INTR_STATUS); 4714 4715 vmcs12->vm_entry_controls = 4716 (vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) | 4717 (vm_entry_controls_get(to_vmx(vcpu)) & VM_ENTRY_IA32E_MODE); 4718 4719 /* 4720 * Note! Save DR7, but intentionally don't grab DEBUGCTL from vmcs02. 4721 * Writes to DEBUGCTL that aren't intercepted by L1 are immediately 4722 * propagated to vmcs12 (see vmx_set_msr()), as the value loaded into 4723 * vmcs02 doesn't strictly track vmcs12. 4724 */ 4725 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS) 4726 vmcs12->guest_dr7 = vcpu->arch.dr7; 4727 4728 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_EFER) 4729 vmcs12->guest_ia32_efer = vcpu->arch.efer; 4730 4731 vmcs_read_cet_state(&vmx->vcpu, &vmcs12->guest_s_cet, 4732 &vmcs12->guest_ssp, 4733 &vmcs12->guest_ssp_tbl); 4734 } 4735 4736 /* 4737 * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits 4738 * and we want to prepare to run its L1 parent. L1 keeps a vmcs for L2 (vmcs12), 4739 * and this function updates it to reflect the changes to the guest state while 4740 * L2 was running (and perhaps made some exits which were handled directly by L0 4741 * without going back to L1), and to reflect the exit reason. 4742 * Note that we do not have to copy here all VMCS fields, just those that 4743 * could have changed by the L2 guest or the exit - i.e., the guest-state and 4744 * exit-information fields only. Other fields are modified by L1 with VMWRITE, 4745 * which already writes to vmcs12 directly. 4746 */ 4747 static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, 4748 u32 vm_exit_reason, u32 exit_intr_info, 4749 unsigned long exit_qualification, u32 exit_insn_len) 4750 { 4751 /* update exit information fields: */ 4752 vmcs12->vm_exit_reason = vm_exit_reason; 4753 if (vt_get_exit_reason(vcpu).enclave_mode) 4754 vmcs12->vm_exit_reason |= VMX_EXIT_REASONS_SGX_ENCLAVE_MODE; 4755 vmcs12->exit_qualification = exit_qualification; 4756 4757 /* 4758 * On VM-Exit due to a failed VM-Entry, the VMCS isn't marked launched 4759 * and only EXIT_REASON and EXIT_QUALIFICATION are updated, all other 4760 * exit info fields are unmodified. 4761 */ 4762 if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) { 4763 vmcs12->launch_state = 1; 4764 4765 /* vm_entry_intr_info_field is cleared on exit. Emulate this 4766 * instead of reading the real value. */ 4767 vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK; 4768 4769 /* 4770 * Transfer the event that L0 or L1 may wanted to inject into 4771 * L2 to IDT_VECTORING_INFO_FIELD. 4772 */ 4773 vmcs12_save_pending_event(vcpu, vmcs12, 4774 vm_exit_reason, exit_intr_info); 4775 4776 vmcs12->vm_exit_intr_info = exit_intr_info; 4777 vmcs12->vm_exit_instruction_len = exit_insn_len; 4778 vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4779 4780 /* 4781 * According to spec, there's no need to store the guest's 4782 * MSRs if the exit is due to a VM-entry failure that occurs 4783 * during or after loading the guest state. Since this exit 4784 * does not fall in that category, we need to save the MSRs. 4785 */ 4786 if (nested_vmx_store_msr(vcpu, 4787 vmcs12->vm_exit_msr_store_addr, 4788 vmcs12->vm_exit_msr_store_count)) 4789 nested_vmx_abort(vcpu, 4790 VMX_ABORT_SAVE_GUEST_MSR_FAIL); 4791 } 4792 } 4793 4794 /* 4795 * A part of what we need to when the nested L2 guest exits and we want to 4796 * run its L1 parent, is to reset L1's guest state to the host state specified 4797 * in vmcs12. 4798 * This function is to be called not only on normal nested exit, but also on 4799 * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry 4800 * Failures During or After Loading Guest State"). 4801 * This function should be called when the active VMCS is L1's (vmcs01). 4802 */ 4803 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, 4804 struct vmcs12 *vmcs12) 4805 { 4806 enum vm_entry_failure_code ignored; 4807 struct kvm_segment seg; 4808 4809 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) 4810 vcpu->arch.efer = vmcs12->host_ia32_efer; 4811 else if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) 4812 vcpu->arch.efer |= (EFER_LMA | EFER_LME); 4813 else 4814 vcpu->arch.efer &= ~(EFER_LMA | EFER_LME); 4815 vmx_set_efer(vcpu, vcpu->arch.efer); 4816 4817 kvm_rsp_write(vcpu, vmcs12->host_rsp); 4818 kvm_rip_write(vcpu, vmcs12->host_rip); 4819 vmx_set_rflags(vcpu, X86_EFLAGS_FIXED); 4820 vmx_set_interrupt_shadow(vcpu, 0); 4821 4822 /* 4823 * Note that calling vmx_set_cr0 is important, even if cr0 hasn't 4824 * actually changed, because vmx_set_cr0 refers to efer set above. 4825 * 4826 * CR0_GUEST_HOST_MASK is already set in the original vmcs01 4827 * (KVM doesn't change it); 4828 */ 4829 vcpu->arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits(); 4830 vmx_set_cr0(vcpu, vmcs12->host_cr0); 4831 4832 /* Same as above - no reason to call set_cr4_guest_host_mask(). */ 4833 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK); 4834 vmx_set_cr4(vcpu, vmcs12->host_cr4); 4835 4836 nested_ept_uninit_mmu_context(vcpu); 4837 4838 /* 4839 * Only PDPTE load can fail as the value of cr3 was checked on entry and 4840 * couldn't have changed. 4841 */ 4842 if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, true, &ignored)) 4843 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL); 4844 4845 nested_vmx_transition_tlb_flush(vcpu, vmcs12, false); 4846 4847 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs); 4848 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp); 4849 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip); 4850 vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base); 4851 vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base); 4852 vmcs_write32(GUEST_IDTR_LIMIT, 0xFFFF); 4853 vmcs_write32(GUEST_GDTR_LIMIT, 0xFFFF); 4854 4855 /* If not VM_EXIT_CLEAR_BNDCFGS, the L2 value propagates to L1. */ 4856 if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS) 4857 vmcs_write64(GUEST_BNDCFGS, 0); 4858 4859 /* 4860 * Load CET state from host state if VM_EXIT_LOAD_CET_STATE is set. 4861 * otherwise CET state should be retained across VM-exit, i.e., 4862 * guest values should be propagated from vmcs12 to vmcs01. 4863 */ 4864 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_CET_STATE) 4865 vmcs_write_cet_state(vcpu, vmcs12->host_s_cet, vmcs12->host_ssp, 4866 vmcs12->host_ssp_tbl); 4867 else 4868 vmcs_write_cet_state(vcpu, vmcs12->guest_s_cet, vmcs12->guest_ssp, 4869 vmcs12->guest_ssp_tbl); 4870 4871 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) { 4872 vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat); 4873 vcpu->arch.pat = vmcs12->host_ia32_pat; 4874 } 4875 if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) && 4876 kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu))) 4877 WARN_ON_ONCE(__kvm_emulate_msr_write(vcpu, MSR_CORE_PERF_GLOBAL_CTRL, 4878 vmcs12->host_ia32_perf_global_ctrl)); 4879 4880 /* Set L1 segment info according to Intel SDM 4881 27.5.2 Loading Host Segment and Descriptor-Table Registers */ 4882 seg = (struct kvm_segment) { 4883 .base = 0, 4884 .limit = 0xFFFFFFFF, 4885 .selector = vmcs12->host_cs_selector, 4886 .type = 11, 4887 .present = 1, 4888 .s = 1, 4889 .g = 1 4890 }; 4891 if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) 4892 seg.l = 1; 4893 else 4894 seg.db = 1; 4895 __vmx_set_segment(vcpu, &seg, VCPU_SREG_CS); 4896 seg = (struct kvm_segment) { 4897 .base = 0, 4898 .limit = 0xFFFFFFFF, 4899 .type = 3, 4900 .present = 1, 4901 .s = 1, 4902 .db = 1, 4903 .g = 1 4904 }; 4905 seg.selector = vmcs12->host_ds_selector; 4906 __vmx_set_segment(vcpu, &seg, VCPU_SREG_DS); 4907 seg.selector = vmcs12->host_es_selector; 4908 __vmx_set_segment(vcpu, &seg, VCPU_SREG_ES); 4909 seg.selector = vmcs12->host_ss_selector; 4910 __vmx_set_segment(vcpu, &seg, VCPU_SREG_SS); 4911 seg.selector = vmcs12->host_fs_selector; 4912 seg.base = vmcs12->host_fs_base; 4913 __vmx_set_segment(vcpu, &seg, VCPU_SREG_FS); 4914 seg.selector = vmcs12->host_gs_selector; 4915 seg.base = vmcs12->host_gs_base; 4916 __vmx_set_segment(vcpu, &seg, VCPU_SREG_GS); 4917 seg = (struct kvm_segment) { 4918 .base = vmcs12->host_tr_base, 4919 .limit = 0x67, 4920 .selector = vmcs12->host_tr_selector, 4921 .type = 11, 4922 .present = 1 4923 }; 4924 __vmx_set_segment(vcpu, &seg, VCPU_SREG_TR); 4925 4926 memset(&seg, 0, sizeof(seg)); 4927 seg.unusable = 1; 4928 __vmx_set_segment(vcpu, &seg, VCPU_SREG_LDTR); 4929 4930 kvm_set_dr(vcpu, 7, 0x400); 4931 vmx_guest_debugctl_write(vcpu, 0); 4932 4933 if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr, 4934 vmcs12->vm_exit_msr_load_count)) 4935 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL); 4936 4937 to_vt(vcpu)->emulation_required = vmx_emulation_required(vcpu); 4938 } 4939 4940 static inline u64 nested_vmx_get_vmcs01_guest_efer(struct vcpu_vmx *vmx) 4941 { 4942 struct vmx_uret_msr *efer_msr; 4943 unsigned int i; 4944 4945 if (vm_entry_controls_get(vmx) & VM_ENTRY_LOAD_IA32_EFER) 4946 return vmcs_read64(GUEST_IA32_EFER); 4947 4948 if (cpu_has_load_ia32_efer()) 4949 return kvm_host.efer; 4950 4951 for (i = 0; i < vmx->msr_autoload.guest.nr; ++i) { 4952 if (vmx->msr_autoload.guest.val[i].index == MSR_EFER) 4953 return vmx->msr_autoload.guest.val[i].value; 4954 } 4955 4956 efer_msr = vmx_find_uret_msr(vmx, MSR_EFER); 4957 if (efer_msr) 4958 return efer_msr->data; 4959 4960 return kvm_host.efer; 4961 } 4962 4963 static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) 4964 { 4965 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 4966 struct vcpu_vmx *vmx = to_vmx(vcpu); 4967 struct vmx_msr_entry g, h; 4968 gpa_t gpa; 4969 u32 i, j; 4970 4971 vcpu->arch.pat = vmcs_read64(GUEST_IA32_PAT); 4972 4973 if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) { 4974 /* 4975 * L1's host DR7 is lost if KVM_GUESTDBG_USE_HW_BP is set 4976 * as vmcs01.GUEST_DR7 contains a userspace defined value 4977 * and vcpu->arch.dr7 is not squirreled away before the 4978 * nested VMENTER (not worth adding a variable in nested_vmx). 4979 */ 4980 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) 4981 kvm_set_dr(vcpu, 7, DR7_FIXED_1); 4982 else 4983 WARN_ON(kvm_set_dr(vcpu, 7, vmcs_readl(GUEST_DR7))); 4984 } 4985 4986 /* Reload DEBUGCTL to ensure vmcs01 has a fresh FREEZE_IN_SMM value. */ 4987 vmx_reload_guest_debugctl(vcpu); 4988 4989 /* 4990 * Note that calling vmx_set_{efer,cr0,cr4} is important as they 4991 * handle a variety of side effects to KVM's software model. 4992 */ 4993 vmx_set_efer(vcpu, nested_vmx_get_vmcs01_guest_efer(vmx)); 4994 4995 vcpu->arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits(); 4996 vmx_set_cr0(vcpu, vmcs_readl(CR0_READ_SHADOW)); 4997 4998 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK); 4999 vmx_set_cr4(vcpu, vmcs_readl(CR4_READ_SHADOW)); 5000 5001 nested_ept_uninit_mmu_context(vcpu); 5002 vcpu->arch.cr3 = vmx->nested.pre_vmenter_cr3; 5003 kvm_register_mark_available(vcpu, VCPU_REG_CR3); 5004 5005 /* 5006 * Use ept_save_pdptrs(vcpu) to load the MMU's cached PDPTRs 5007 * from vmcs01 (if necessary). The PDPTRs are not loaded on 5008 * VMFail, like everything else we just need to ensure our 5009 * software model is up-to-date. 5010 */ 5011 if (enable_ept && is_pae_paging(vcpu)) 5012 ept_save_pdptrs(vcpu); 5013 5014 kvm_mmu_reset_context(vcpu); 5015 5016 /* 5017 * This nasty bit of open coding is a compromise between blindly 5018 * loading L1's MSRs using the exit load lists (incorrect emulation 5019 * of VMFail), leaving the nested VM's MSRs in the software model 5020 * (incorrect behavior) and snapshotting the modified MSRs (too 5021 * expensive since the lists are unbound by hardware). For each 5022 * MSR that was (prematurely) loaded from the nested VMEntry load 5023 * list, reload it from the exit load list if it exists and differs 5024 * from the guest value. The intent is to stuff host state as 5025 * silently as possible, not to fully process the exit load list. 5026 */ 5027 for (i = 0; i < vmcs12->vm_entry_msr_load_count; i++) { 5028 gpa = vmcs12->vm_entry_msr_load_addr + (i * sizeof(g)); 5029 if (kvm_vcpu_read_guest(vcpu, gpa, &g, sizeof(g))) { 5030 pr_debug_ratelimited( 5031 "%s read MSR index failed (%u, 0x%08llx)\n", 5032 __func__, i, gpa); 5033 goto vmabort; 5034 } 5035 5036 for (j = 0; j < vmcs12->vm_exit_msr_load_count; j++) { 5037 gpa = vmcs12->vm_exit_msr_load_addr + (j * sizeof(h)); 5038 if (kvm_vcpu_read_guest(vcpu, gpa, &h, sizeof(h))) { 5039 pr_debug_ratelimited( 5040 "%s read MSR failed (%u, 0x%08llx)\n", 5041 __func__, j, gpa); 5042 goto vmabort; 5043 } 5044 if (h.index != g.index) 5045 continue; 5046 if (h.value == g.value) 5047 break; 5048 5049 if (nested_vmx_load_msr_check(vcpu, &h)) { 5050 pr_debug_ratelimited( 5051 "%s check failed (%u, 0x%x, 0x%x)\n", 5052 __func__, j, h.index, h.reserved); 5053 goto vmabort; 5054 } 5055 5056 if (kvm_emulate_msr_write(vcpu, h.index, h.value)) { 5057 pr_debug_ratelimited( 5058 "%s WRMSR failed (%u, 0x%x, 0x%llx)\n", 5059 __func__, j, h.index, h.value); 5060 goto vmabort; 5061 } 5062 } 5063 } 5064 5065 return; 5066 5067 vmabort: 5068 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL); 5069 } 5070 5071 /* 5072 * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1 5073 * and modify vmcs12 to make it see what it would expect to see there if 5074 * L2 was its real guest. Must only be called when in L2 (is_guest_mode()) 5075 */ 5076 void __nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, 5077 u32 exit_intr_info, unsigned long exit_qualification, 5078 u32 exit_insn_len) 5079 { 5080 struct vcpu_vmx *vmx = to_vmx(vcpu); 5081 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5082 5083 /* Pending MTF traps are discarded on VM-Exit. */ 5084 vmx->nested.mtf_pending = false; 5085 5086 /* trying to cancel vmlaunch/vmresume is a bug */ 5087 kvm_warn_on_nested_run_pending(vcpu); 5088 5089 /* Note, "checking" the request also clears the request. */ 5090 if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) { 5091 #ifdef CONFIG_KVM_HYPERV 5092 /* 5093 * KVM_REQ_GET_NESTED_STATE_PAGES is also used to map 5094 * Enlightened VMCS after migration and we still need to 5095 * do that when something is forcing L2->L1 exit prior to 5096 * the first L2 run. 5097 */ 5098 (void)nested_get_evmcs_page(vcpu); 5099 #endif 5100 } 5101 5102 /* Service pending TLB flush requests for L2 before switching to L1. */ 5103 kvm_service_local_tlb_flush_requests(vcpu); 5104 5105 /* 5106 * VCPU_REG_PDPTR will be clobbered in arch/x86/kvm/vmx/vmx.h between 5107 * now and the new vmentry. Ensure that the VMCS02 PDPTR fields are 5108 * up-to-date before switching to L1. 5109 */ 5110 if (enable_ept && is_pae_paging(vcpu)) 5111 vmx_ept_load_pdptrs(vcpu); 5112 5113 leave_guest_mode(vcpu); 5114 5115 if (nested_cpu_has_preemption_timer(vmcs12)) 5116 hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer); 5117 5118 if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING)) { 5119 vcpu->arch.tsc_offset = vcpu->arch.l1_tsc_offset; 5120 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_TSC_SCALING)) 5121 vcpu->arch.tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; 5122 } 5123 5124 if (likely(!vmx->fail)) { 5125 sync_vmcs02_to_vmcs12(vcpu, vmcs12); 5126 5127 if (vm_exit_reason != -1) 5128 prepare_vmcs12(vcpu, vmcs12, vm_exit_reason, 5129 exit_intr_info, exit_qualification, 5130 exit_insn_len); 5131 5132 /* 5133 * Must happen outside of sync_vmcs02_to_vmcs12() as it will 5134 * also be used to capture vmcs12 cache as part of 5135 * capturing nVMX state for snapshot (migration). 5136 * 5137 * Otherwise, this flush will dirty guest memory at a 5138 * point it is already assumed by user-space to be 5139 * immutable. 5140 */ 5141 nested_flush_cached_shadow_vmcs12(vcpu, vmcs12); 5142 } else { 5143 /* 5144 * The only expected VM-instruction error is "VM entry with 5145 * invalid control field(s)." Anything else indicates a 5146 * problem with L0. 5147 */ 5148 WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) != 5149 VMXERR_ENTRY_INVALID_CONTROL_FIELD); 5150 5151 /* VM-Fail at VM-Entry means KVM missed a consistency check. */ 5152 WARN_ON_ONCE(warn_on_missed_cc); 5153 } 5154 5155 /* 5156 * Drop events/exceptions that were queued for re-injection to L2 5157 * (picked up via vmx_complete_interrupts()), as well as exceptions 5158 * that were pending for L2. Note, this must NOT be hoisted above 5159 * prepare_vmcs12(), events/exceptions queued for re-injection need to 5160 * be captured in vmcs12 (see vmcs12_save_pending_event()). 5161 */ 5162 vcpu->arch.nmi_injected = false; 5163 kvm_clear_exception_queue(vcpu); 5164 kvm_clear_interrupt_queue(vcpu); 5165 5166 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 5167 5168 kvm_nested_vmexit_handle_ibrs(vcpu); 5169 5170 /* 5171 * Update any VMCS fields that might have changed while vmcs02 was the 5172 * active VMCS. The tracking is per-vCPU, not per-VMCS. 5173 */ 5174 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, vmx->msr_autostore.nr); 5175 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 5176 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 5177 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 5178 if (kvm_caps.has_tsc_control) 5179 vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio); 5180 5181 nested_put_vmcs12_pages(vcpu); 5182 5183 if ((vm_exit_reason != -1) && 5184 (enable_shadow_vmcs || nested_vmx_is_evmptr12_valid(vmx))) 5185 vmx->nested.need_vmcs12_to_shadow_sync = true; 5186 5187 /* in case we halted in L2 */ 5188 kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); 5189 5190 if (likely(!vmx->fail)) { 5191 if (vm_exit_reason != -1) 5192 trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason, 5193 vmcs12->exit_qualification, 5194 vmcs12->idt_vectoring_info_field, 5195 vmcs12->vm_exit_intr_info, 5196 vmcs12->vm_exit_intr_error_code, 5197 KVM_ISA_VMX); 5198 5199 load_vmcs12_host_state(vcpu, vmcs12); 5200 5201 /* 5202 * Process events if an injectable IRQ or NMI is pending, even 5203 * if the event is blocked (RFLAGS.IF is cleared on VM-Exit). 5204 * If an event became pending while L2 was active, KVM needs to 5205 * either inject the event or request an IRQ/NMI window. SMIs 5206 * don't need to be processed as SMM is mutually exclusive with 5207 * non-root mode. INIT/SIPI don't need to be checked as INIT 5208 * is blocked post-VMXON, and SIPIs are ignored. 5209 */ 5210 if (kvm_cpu_has_injectable_intr(vcpu) || vcpu->arch.nmi_pending) 5211 kvm_make_request(KVM_REQ_EVENT, vcpu); 5212 return; 5213 } 5214 5215 /* 5216 * After an early L2 VM-entry failure, we're now back 5217 * in L1 which thinks it just finished a VMLAUNCH or 5218 * VMRESUME instruction, so we need to set the failure 5219 * flag and the VM-instruction error field of the VMCS 5220 * accordingly, and skip the emulated instruction. 5221 */ 5222 (void)nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 5223 5224 /* 5225 * Restore L1's host state to KVM's software model. We're here 5226 * because a consistency check was caught by hardware, which 5227 * means some amount of guest state has been propagated to KVM's 5228 * model and needs to be unwound to the host's state. 5229 */ 5230 nested_vmx_restore_host_state(vcpu); 5231 5232 vmx->fail = 0; 5233 } 5234 5235 static void nested_vmx_triple_fault(struct kvm_vcpu *vcpu) 5236 { 5237 kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu); 5238 nested_vmx_vmexit(vcpu, EXIT_REASON_TRIPLE_FAULT, 0, 0); 5239 } 5240 5241 /* 5242 * Decode the memory-address operand of a vmx instruction, as recorded on an 5243 * exit caused by such an instruction (run by a guest hypervisor). 5244 * On success, returns 0. When the operand is invalid, returns 1 and throws 5245 * #UD, #GP, or #SS. 5246 */ 5247 int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification, 5248 u32 vmx_instruction_info, bool wr, int len, gva_t *ret) 5249 { 5250 gva_t off; 5251 bool exn; 5252 struct kvm_segment s; 5253 5254 /* 5255 * According to Vol. 3B, "Information for VM Exits Due to Instruction 5256 * Execution", on an exit, vmx_instruction_info holds most of the 5257 * addressing components of the operand. Only the displacement part 5258 * is put in exit_qualification (see 3B, "Basic VM-Exit Information"). 5259 * For how an actual address is calculated from all these components, 5260 * refer to Vol. 1, "Operand Addressing". 5261 */ 5262 int scaling = vmx_instruction_info & 3; 5263 int addr_size = (vmx_instruction_info >> 7) & 7; 5264 bool is_reg = vmx_instruction_info & (1u << 10); 5265 int seg_reg = (vmx_instruction_info >> 15) & 7; 5266 int index_reg = (vmx_instruction_info >> 18) & 0xf; 5267 bool index_is_valid = !(vmx_instruction_info & (1u << 22)); 5268 int base_reg = (vmx_instruction_info >> 23) & 0xf; 5269 bool base_is_valid = !(vmx_instruction_info & (1u << 27)); 5270 5271 if (is_reg) { 5272 kvm_queue_exception(vcpu, UD_VECTOR); 5273 return 1; 5274 } 5275 5276 /* Addr = segment_base + offset */ 5277 /* offset = base + [index * scale] + displacement */ 5278 off = exit_qualification; /* holds the displacement */ 5279 if (addr_size == 1) 5280 off = (gva_t)sign_extend64(off, 31); 5281 else if (addr_size == 0) 5282 off = (gva_t)sign_extend64(off, 15); 5283 if (base_is_valid) 5284 off += kvm_register_read(vcpu, base_reg); 5285 if (index_is_valid) 5286 off += kvm_register_read(vcpu, index_reg) << scaling; 5287 vmx_get_segment(vcpu, &s, seg_reg); 5288 5289 /* 5290 * The effective address, i.e. @off, of a memory operand is truncated 5291 * based on the address size of the instruction. Note that this is 5292 * the *effective address*, i.e. the address prior to accounting for 5293 * the segment's base. 5294 */ 5295 if (addr_size == 1) /* 32 bit */ 5296 off &= 0xffffffff; 5297 else if (addr_size == 0) /* 16 bit */ 5298 off &= 0xffff; 5299 5300 /* Checks for #GP/#SS exceptions. */ 5301 exn = false; 5302 if (is_long_mode(vcpu)) { 5303 /* 5304 * The virtual/linear address is never truncated in 64-bit 5305 * mode, e.g. a 32-bit address size can yield a 64-bit virtual 5306 * address when using FS/GS with a non-zero base. 5307 */ 5308 if (seg_reg == VCPU_SREG_FS || seg_reg == VCPU_SREG_GS) 5309 *ret = s.base + off; 5310 else 5311 *ret = off; 5312 5313 *ret = vmx_get_untagged_addr(vcpu, *ret, 0); 5314 /* Long mode: #GP(0)/#SS(0) if the memory address is in a 5315 * non-canonical form. This is the only check on the memory 5316 * destination for long mode! 5317 */ 5318 exn = is_noncanonical_address(*ret, vcpu, 0); 5319 } else { 5320 /* 5321 * When not in long mode, the virtual/linear address is 5322 * unconditionally truncated to 32 bits regardless of the 5323 * address size. 5324 */ 5325 *ret = (s.base + off) & 0xffffffff; 5326 5327 /* Protected mode: apply checks for segment validity in the 5328 * following order: 5329 * - segment type check (#GP(0) may be thrown) 5330 * - usability check (#GP(0)/#SS(0)) 5331 * - limit check (#GP(0)/#SS(0)) 5332 */ 5333 if (wr) 5334 /* #GP(0) if the destination operand is located in a 5335 * read-only data segment or any code segment. 5336 */ 5337 exn = ((s.type & 0xa) == 0 || (s.type & 8)); 5338 else 5339 /* #GP(0) if the source operand is located in an 5340 * execute-only code segment 5341 */ 5342 exn = ((s.type & 0xa) == 8); 5343 if (exn) { 5344 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); 5345 return 1; 5346 } 5347 /* Protected mode: #GP(0)/#SS(0) if the segment is unusable. 5348 */ 5349 exn = (s.unusable != 0); 5350 5351 /* 5352 * Protected mode: #GP(0)/#SS(0) if the memory operand is 5353 * outside the segment limit. All CPUs that support VMX ignore 5354 * limit checks for flat segments, i.e. segments with base==0, 5355 * limit==0xffffffff and of type expand-up data or code. 5356 */ 5357 if (!(s.base == 0 && s.limit == 0xffffffff && 5358 ((s.type & 8) || !(s.type & 4)))) 5359 exn = exn || ((u64)off + len - 1 > s.limit); 5360 } 5361 if (exn) { 5362 kvm_queue_exception_e(vcpu, 5363 seg_reg == VCPU_SREG_SS ? 5364 SS_VECTOR : GP_VECTOR, 5365 0); 5366 return 1; 5367 } 5368 5369 return 0; 5370 } 5371 5372 static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer, 5373 int *ret) 5374 { 5375 gva_t gva; 5376 struct x86_exception e; 5377 int r; 5378 5379 if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), 5380 vmcs_read32(VMX_INSTRUCTION_INFO), false, 5381 sizeof(*vmpointer), &gva)) { 5382 *ret = 1; 5383 return -EINVAL; 5384 } 5385 5386 r = kvm_read_guest_virt(vcpu, gva, vmpointer, sizeof(*vmpointer), &e); 5387 if (r != X86EMUL_CONTINUE) { 5388 *ret = kvm_handle_memory_failure(vcpu, r, &e); 5389 return -EINVAL; 5390 } 5391 5392 return 0; 5393 } 5394 5395 /* 5396 * Allocate a shadow VMCS and associate it with the currently loaded 5397 * VMCS, unless such a shadow VMCS already exists. The newly allocated 5398 * VMCS is also VMCLEARed, so that it is ready for use. 5399 */ 5400 static struct vmcs *alloc_shadow_vmcs(struct kvm_vcpu *vcpu) 5401 { 5402 struct vcpu_vmx *vmx = to_vmx(vcpu); 5403 struct loaded_vmcs *loaded_vmcs = vmx->loaded_vmcs; 5404 5405 /* 5406 * KVM allocates a shadow VMCS only when L1 executes VMXON and frees it 5407 * when L1 executes VMXOFF or the vCPU is forced out of nested 5408 * operation. VMXON faults if the CPU is already post-VMXON, so it 5409 * should be impossible to already have an allocated shadow VMCS. KVM 5410 * doesn't support virtualization of VMCS shadowing, so vmcs01 should 5411 * always be the loaded VMCS. 5412 */ 5413 if (WARN_ON(loaded_vmcs != &vmx->vmcs01 || loaded_vmcs->shadow_vmcs)) 5414 return loaded_vmcs->shadow_vmcs; 5415 5416 loaded_vmcs->shadow_vmcs = alloc_vmcs(true); 5417 if (loaded_vmcs->shadow_vmcs) 5418 vmcs_clear(loaded_vmcs->shadow_vmcs); 5419 5420 return loaded_vmcs->shadow_vmcs; 5421 } 5422 5423 static int enter_vmx_operation(struct kvm_vcpu *vcpu) 5424 { 5425 struct vcpu_vmx *vmx = to_vmx(vcpu); 5426 int r; 5427 5428 r = alloc_loaded_vmcs(&vmx->nested.vmcs02); 5429 if (r < 0) 5430 goto out_vmcs02; 5431 5432 vmx->nested.cached_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT); 5433 if (!vmx->nested.cached_vmcs12) 5434 goto out_cached_vmcs12; 5435 5436 vmx->nested.shadow_vmcs12_cache.gpa = INVALID_GPA; 5437 vmx->nested.cached_shadow_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT); 5438 if (!vmx->nested.cached_shadow_vmcs12) 5439 goto out_cached_shadow_vmcs12; 5440 5441 if (enable_shadow_vmcs && !alloc_shadow_vmcs(vcpu)) 5442 goto out_shadow_vmcs; 5443 5444 hrtimer_setup(&vmx->nested.preemption_timer, vmx_preemption_timer_fn, CLOCK_MONOTONIC, 5445 HRTIMER_MODE_ABS_PINNED); 5446 5447 vmx->nested.vpid02 = allocate_vpid(); 5448 5449 /* 5450 * Clear last_vpid to ensure that the VPID is flushed on the first 5451 * nested VM-Enter. Otherwise, stale TLB entries from a previous life of 5452 * the VPID (e.g. different vCPU or even different VM) could be used. 5453 */ 5454 vmx->nested.last_vpid = 0; 5455 5456 vmx->nested.vmcs02_initialized = false; 5457 vmx->nested.vmxon = true; 5458 5459 if (vmx_pt_mode_is_host_guest()) { 5460 vmx->pt_desc.guest.ctl = 0; 5461 pt_update_intercept_for_msr(vcpu); 5462 } 5463 5464 return 0; 5465 5466 out_shadow_vmcs: 5467 kfree(vmx->nested.cached_shadow_vmcs12); 5468 5469 out_cached_shadow_vmcs12: 5470 kfree(vmx->nested.cached_vmcs12); 5471 5472 out_cached_vmcs12: 5473 free_loaded_vmcs(&vmx->nested.vmcs02); 5474 5475 out_vmcs02: 5476 return -ENOMEM; 5477 } 5478 5479 /* Emulate the VMXON instruction. */ 5480 static int handle_vmxon(struct kvm_vcpu *vcpu) 5481 { 5482 int ret; 5483 gpa_t vmptr; 5484 uint32_t revision; 5485 struct vcpu_vmx *vmx = to_vmx(vcpu); 5486 const u64 VMXON_NEEDED_FEATURES = FEAT_CTL_LOCKED 5487 | FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX; 5488 5489 /* 5490 * Manually check CR4.VMXE checks, KVM must force CR4.VMXE=1 to enter 5491 * the guest and so cannot rely on hardware to perform the check, 5492 * which has higher priority than VM-Exit (see Intel SDM's pseudocode 5493 * for VMXON). 5494 * 5495 * Rely on hardware for the other pre-VM-Exit checks, CR0.PE=1, !VM86 5496 * and !COMPATIBILITY modes. For an unrestricted guest, KVM doesn't 5497 * force any of the relevant guest state. For a restricted guest, KVM 5498 * does force CR0.PE=1, but only to also force VM86 in order to emulate 5499 * Real Mode, and so there's no need to check CR0.PE manually. 5500 */ 5501 if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_VMXE)) { 5502 kvm_queue_exception(vcpu, UD_VECTOR); 5503 return 1; 5504 } 5505 5506 /* 5507 * The CPL is checked for "not in VMX operation" and for "in VMX root", 5508 * and has higher priority than the VM-Fail due to being post-VMXON, 5509 * i.e. VMXON #GPs outside of VMX non-root if CPL!=0. In VMX non-root, 5510 * VMXON causes VM-Exit and KVM unconditionally forwards VMXON VM-Exits 5511 * from L2 to L1, i.e. there's no need to check for the vCPU being in 5512 * VMX non-root. 5513 * 5514 * Forwarding the VM-Exit unconditionally, i.e. without performing the 5515 * #UD checks (see above), is functionally ok because KVM doesn't allow 5516 * L1 to run L2 without CR4.VMXE=0, and because KVM never modifies L2's 5517 * CR0 or CR4, i.e. it's L2's responsibility to emulate #UDs that are 5518 * missed by hardware due to shadowing CR0 and/or CR4. 5519 */ 5520 if (vmx_get_cpl(vcpu)) { 5521 kvm_inject_gp(vcpu, 0); 5522 return 1; 5523 } 5524 5525 if (vmx->nested.vmxon) 5526 return nested_vmx_fail(vcpu, VMXERR_VMXON_IN_VMX_ROOT_OPERATION); 5527 5528 /* 5529 * Invalid CR0/CR4 generates #GP. These checks are performed if and 5530 * only if the vCPU isn't already in VMX operation, i.e. effectively 5531 * have lower priority than the VM-Fail above. 5532 */ 5533 if (!nested_host_cr0_valid(vcpu, kvm_read_cr0(vcpu)) || 5534 !nested_host_cr4_valid(vcpu, kvm_read_cr4(vcpu))) { 5535 kvm_inject_gp(vcpu, 0); 5536 return 1; 5537 } 5538 5539 if ((vmx->msr_ia32_feature_control & VMXON_NEEDED_FEATURES) 5540 != VMXON_NEEDED_FEATURES) { 5541 kvm_inject_gp(vcpu, 0); 5542 return 1; 5543 } 5544 5545 if (nested_vmx_get_vmptr(vcpu, &vmptr, &ret)) 5546 return ret; 5547 5548 /* 5549 * SDM 3: 24.11.5 5550 * The first 4 bytes of VMXON region contain the supported 5551 * VMCS revision identifier 5552 * 5553 * Note - IA32_VMX_BASIC[48] will never be 1 for the nested case; 5554 * which replaces physical address width with 32 5555 */ 5556 if (!page_address_valid(vcpu, vmptr)) 5557 return nested_vmx_failInvalid(vcpu); 5558 5559 if (kvm_read_guest(vcpu->kvm, vmptr, &revision, sizeof(revision)) || 5560 revision != VMCS12_REVISION) 5561 return nested_vmx_failInvalid(vcpu); 5562 5563 vmx->nested.vmxon_ptr = vmptr; 5564 ret = enter_vmx_operation(vcpu); 5565 if (ret) 5566 return ret; 5567 5568 return nested_vmx_succeed(vcpu); 5569 } 5570 5571 static inline void nested_release_vmcs12(struct kvm_vcpu *vcpu) 5572 { 5573 struct vcpu_vmx *vmx = to_vmx(vcpu); 5574 5575 if (vmx->nested.current_vmptr == INVALID_GPA) 5576 return; 5577 5578 copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu)); 5579 5580 if (enable_shadow_vmcs) { 5581 /* copy to memory all shadowed fields in case 5582 they were modified */ 5583 copy_shadow_to_vmcs12(vmx); 5584 vmx_disable_shadow_vmcs(vmx); 5585 } 5586 vmx->nested.posted_intr_nv = -1; 5587 5588 /* Flush VMCS12 to guest memory */ 5589 kvm_vcpu_write_guest_page(vcpu, 5590 vmx->nested.current_vmptr >> PAGE_SHIFT, 5591 vmx->nested.cached_vmcs12, 0, VMCS12_SIZE); 5592 5593 kvm_mmu_free_roots(vcpu->kvm, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 5594 5595 vmx->nested.current_vmptr = INVALID_GPA; 5596 } 5597 5598 /* Emulate the VMXOFF instruction */ 5599 static int handle_vmxoff(struct kvm_vcpu *vcpu) 5600 { 5601 if (!nested_vmx_check_permission(vcpu)) 5602 return 1; 5603 5604 free_nested(vcpu); 5605 5606 if (kvm_apic_has_pending_init_or_sipi(vcpu)) 5607 kvm_make_request(KVM_REQ_EVENT, vcpu); 5608 5609 return nested_vmx_succeed(vcpu); 5610 } 5611 5612 /* Emulate the VMCLEAR instruction */ 5613 static int handle_vmclear(struct kvm_vcpu *vcpu) 5614 { 5615 struct vcpu_vmx *vmx = to_vmx(vcpu); 5616 u32 zero = 0; 5617 gpa_t vmptr; 5618 int r; 5619 5620 if (!nested_vmx_check_permission(vcpu)) 5621 return 1; 5622 5623 if (nested_vmx_get_vmptr(vcpu, &vmptr, &r)) 5624 return r; 5625 5626 if (!page_address_valid(vcpu, vmptr)) 5627 return nested_vmx_fail(vcpu, VMXERR_VMCLEAR_INVALID_ADDRESS); 5628 5629 if (vmptr == vmx->nested.vmxon_ptr) 5630 return nested_vmx_fail(vcpu, VMXERR_VMCLEAR_VMXON_POINTER); 5631 5632 if (likely(!nested_evmcs_handle_vmclear(vcpu, vmptr))) { 5633 if (vmptr == vmx->nested.current_vmptr) 5634 nested_release_vmcs12(vcpu); 5635 5636 /* 5637 * Silently ignore memory errors on VMCLEAR, Intel's pseudocode 5638 * for VMCLEAR includes a "ensure that data for VMCS referenced 5639 * by the operand is in memory" clause that guards writes to 5640 * memory, i.e. doing nothing for I/O is architecturally valid. 5641 * 5642 * FIXME: Suppress failures if and only if no memslot is found, 5643 * i.e. exit to userspace if __copy_to_user() fails. 5644 */ 5645 (void)kvm_vcpu_write_guest(vcpu, 5646 vmptr + offsetof(struct vmcs12, 5647 launch_state), 5648 &zero, sizeof(zero)); 5649 } 5650 5651 return nested_vmx_succeed(vcpu); 5652 } 5653 5654 /* Emulate the VMLAUNCH instruction */ 5655 static int handle_vmlaunch(struct kvm_vcpu *vcpu) 5656 { 5657 return nested_vmx_run(vcpu, true); 5658 } 5659 5660 /* Emulate the VMRESUME instruction */ 5661 static int handle_vmresume(struct kvm_vcpu *vcpu) 5662 { 5663 5664 return nested_vmx_run(vcpu, false); 5665 } 5666 5667 static int handle_vmread(struct kvm_vcpu *vcpu) 5668 { 5669 struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) 5670 : get_vmcs12(vcpu); 5671 unsigned long exit_qualification = vt_get_exit_qual(vcpu); 5672 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5673 struct vcpu_vmx *vmx = to_vmx(vcpu); 5674 struct x86_exception e; 5675 unsigned long field; 5676 u64 value; 5677 gva_t gva = 0; 5678 short offset; 5679 int len, r; 5680 5681 if (!nested_vmx_check_permission(vcpu)) 5682 return 1; 5683 5684 /* Decode instruction info and find the field to read */ 5685 field = kvm_register_read(vcpu, (((instr_info) >> 28) & 0xf)); 5686 5687 if (!nested_vmx_is_evmptr12_valid(vmx)) { 5688 /* 5689 * In VMX non-root operation, when the VMCS-link pointer is INVALID_GPA, 5690 * any VMREAD sets the ALU flags for VMfailInvalid. 5691 */ 5692 if (vmx->nested.current_vmptr == INVALID_GPA || 5693 (is_guest_mode(vcpu) && 5694 get_vmcs12(vcpu)->vmcs_link_pointer == INVALID_GPA)) 5695 return nested_vmx_failInvalid(vcpu); 5696 5697 offset = get_vmcs12_field_offset(field); 5698 if (offset < 0) 5699 return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); 5700 5701 if (!is_guest_mode(vcpu) && is_vmcs12_ext_field(field)) 5702 copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 5703 5704 /* Read the field, zero-extended to a u64 value */ 5705 value = vmcs12_read_any(vmcs12, field, offset); 5706 } else { 5707 /* 5708 * Hyper-V TLFS (as of 6.0b) explicitly states, that while an 5709 * enlightened VMCS is active VMREAD/VMWRITE instructions are 5710 * unsupported. Unfortunately, certain versions of Windows 11 5711 * don't comply with this requirement which is not enforced in 5712 * genuine Hyper-V. Allow VMREAD from an enlightened VMCS as a 5713 * workaround, as misbehaving guests will panic on VM-Fail. 5714 * Note, enlightened VMCS is incompatible with shadow VMCS so 5715 * all VMREADs from L2 should go to L1. 5716 */ 5717 if (WARN_ON_ONCE(is_guest_mode(vcpu))) 5718 return nested_vmx_failInvalid(vcpu); 5719 5720 offset = evmcs_field_offset(field, NULL); 5721 if (offset < 0) 5722 return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); 5723 5724 /* Read the field, zero-extended to a u64 value */ 5725 value = evmcs_read_any(nested_vmx_evmcs(vmx), field, offset); 5726 } 5727 5728 /* 5729 * Now copy part of this value to register or memory, as requested. 5730 * Note that the number of bits actually copied is 32 or 64 depending 5731 * on the guest's mode (32 or 64 bit), not on the given field's length. 5732 */ 5733 if (instr_info & BIT(10)) { 5734 kvm_register_write(vcpu, (((instr_info) >> 3) & 0xf), value); 5735 } else { 5736 len = is_64_bit_mode(vcpu) ? 8 : 4; 5737 if (get_vmx_mem_address(vcpu, exit_qualification, 5738 instr_info, true, len, &gva)) 5739 return 1; 5740 /* _system ok, nested_vmx_check_permission has verified cpl=0 */ 5741 r = kvm_write_guest_virt_system(vcpu, gva, &value, len, &e); 5742 if (r != X86EMUL_CONTINUE) 5743 return kvm_handle_memory_failure(vcpu, r, &e); 5744 } 5745 5746 return nested_vmx_succeed(vcpu); 5747 } 5748 5749 static bool is_shadow_field_rw(unsigned long field) 5750 { 5751 switch (field) { 5752 #define SHADOW_FIELD_RW(x, y) case x: 5753 #include "vmcs_shadow_fields.h" 5754 return true; 5755 default: 5756 break; 5757 } 5758 return false; 5759 } 5760 5761 static bool is_shadow_field_ro(unsigned long field) 5762 { 5763 switch (field) { 5764 #define SHADOW_FIELD_RO(x, y) case x: 5765 #include "vmcs_shadow_fields.h" 5766 return true; 5767 default: 5768 break; 5769 } 5770 return false; 5771 } 5772 5773 static int handle_vmwrite(struct kvm_vcpu *vcpu) 5774 { 5775 struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) 5776 : get_vmcs12(vcpu); 5777 unsigned long exit_qualification = vt_get_exit_qual(vcpu); 5778 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5779 struct vcpu_vmx *vmx = to_vmx(vcpu); 5780 struct x86_exception e; 5781 unsigned long field; 5782 short offset; 5783 gva_t gva; 5784 int len, r; 5785 5786 /* 5787 * The value to write might be 32 or 64 bits, depending on L1's long 5788 * mode, and eventually we need to write that into a field of several 5789 * possible lengths. The code below first zero-extends the value to 64 5790 * bit (value), and then copies only the appropriate number of 5791 * bits into the vmcs12 field. 5792 */ 5793 u64 value = 0; 5794 5795 if (!nested_vmx_check_permission(vcpu)) 5796 return 1; 5797 5798 /* 5799 * In VMX non-root operation, when the VMCS-link pointer is INVALID_GPA, 5800 * any VMWRITE sets the ALU flags for VMfailInvalid. 5801 */ 5802 if (vmx->nested.current_vmptr == INVALID_GPA || 5803 (is_guest_mode(vcpu) && 5804 get_vmcs12(vcpu)->vmcs_link_pointer == INVALID_GPA)) 5805 return nested_vmx_failInvalid(vcpu); 5806 5807 if (instr_info & BIT(10)) 5808 value = kvm_register_read(vcpu, (((instr_info) >> 3) & 0xf)); 5809 else { 5810 len = is_64_bit_mode(vcpu) ? 8 : 4; 5811 if (get_vmx_mem_address(vcpu, exit_qualification, 5812 instr_info, false, len, &gva)) 5813 return 1; 5814 r = kvm_read_guest_virt(vcpu, gva, &value, len, &e); 5815 if (r != X86EMUL_CONTINUE) 5816 return kvm_handle_memory_failure(vcpu, r, &e); 5817 } 5818 5819 field = kvm_register_read(vcpu, (((instr_info) >> 28) & 0xf)); 5820 5821 offset = get_vmcs12_field_offset(field); 5822 if (offset < 0) 5823 return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); 5824 5825 /* 5826 * If the vCPU supports "VMWRITE to any supported field in the 5827 * VMCS," then the "read-only" fields are actually read/write. 5828 */ 5829 if (vmcs_field_readonly(field) && 5830 !nested_cpu_has_vmwrite_any_field(vcpu)) 5831 return nested_vmx_fail(vcpu, VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT); 5832 5833 /* 5834 * Ensure vmcs12 is up-to-date before any VMWRITE that dirties 5835 * vmcs12, else we may crush a field or consume a stale value. 5836 */ 5837 if (!is_guest_mode(vcpu) && !is_shadow_field_rw(field)) 5838 copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 5839 5840 /* 5841 * Some Intel CPUs intentionally drop the reserved bits of the AR byte 5842 * fields on VMWRITE. Emulate this behavior to ensure consistent KVM 5843 * behavior regardless of the underlying hardware, e.g. if an AR_BYTE 5844 * field is intercepted for VMWRITE but not VMREAD (in L1), then VMREAD 5845 * from L1 will return a different value than VMREAD from L2 (L1 sees 5846 * the stripped down value, L2 sees the full value as stored by KVM). 5847 */ 5848 if (field >= GUEST_ES_AR_BYTES && field <= GUEST_TR_AR_BYTES) 5849 value &= 0x1f0ff; 5850 5851 vmcs12_write_any(vmcs12, field, offset, value); 5852 5853 /* 5854 * Do not track vmcs12 dirty-state if in guest-mode as we actually 5855 * dirty shadow vmcs12 instead of vmcs12. Fields that can be updated 5856 * by L1 without a vmexit are always updated in the vmcs02, i.e. don't 5857 * "dirty" vmcs12, all others go down the prepare_vmcs02() slow path. 5858 */ 5859 if (!is_guest_mode(vcpu) && !is_shadow_field_rw(field)) { 5860 /* 5861 * L1 can read these fields without exiting, ensure the 5862 * shadow VMCS is up-to-date. 5863 */ 5864 if (enable_shadow_vmcs && is_shadow_field_ro(field)) { 5865 preempt_disable(); 5866 vmcs_load(vmx->vmcs01.shadow_vmcs); 5867 5868 __vmcs_writel(field, value); 5869 5870 vmcs_clear(vmx->vmcs01.shadow_vmcs); 5871 vmcs_load(vmx->loaded_vmcs->vmcs); 5872 preempt_enable(); 5873 } 5874 vmx->nested.dirty_vmcs12 = true; 5875 } 5876 5877 return nested_vmx_succeed(vcpu); 5878 } 5879 5880 static void set_current_vmptr(struct vcpu_vmx *vmx, gpa_t vmptr) 5881 { 5882 vmx->nested.current_vmptr = vmptr; 5883 if (enable_shadow_vmcs) { 5884 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_SHADOW_VMCS); 5885 vmcs_write64(VMCS_LINK_POINTER, 5886 __pa(vmx->vmcs01.shadow_vmcs)); 5887 vmx->nested.need_vmcs12_to_shadow_sync = true; 5888 } 5889 vmx->nested.dirty_vmcs12 = true; 5890 vmx->nested.force_msr_bitmap_recalc = true; 5891 } 5892 5893 /* Emulate the VMPTRLD instruction */ 5894 static int handle_vmptrld(struct kvm_vcpu *vcpu) 5895 { 5896 struct vcpu_vmx *vmx = to_vmx(vcpu); 5897 gpa_t vmptr; 5898 int r; 5899 5900 if (!nested_vmx_check_permission(vcpu)) 5901 return 1; 5902 5903 /* Forbid normal VMPTRLD if Enlightened version was used */ 5904 if (nested_vmx_is_evmptr12_valid(vmx)) { 5905 kvm_queue_exception(vcpu, UD_VECTOR); 5906 return 1; 5907 } 5908 5909 if (nested_vmx_get_vmptr(vcpu, &vmptr, &r)) 5910 return r; 5911 5912 if (!page_address_valid(vcpu, vmptr)) 5913 return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_INVALID_ADDRESS); 5914 5915 if (vmptr == vmx->nested.vmxon_ptr) 5916 return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_VMXON_POINTER); 5917 5918 if (vmx->nested.current_vmptr != vmptr) { 5919 struct gfn_to_hva_cache *ghc = &vmx->nested.vmcs12_cache; 5920 struct vmcs_hdr hdr; 5921 5922 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, vmptr, VMCS12_SIZE)) { 5923 /* 5924 * Reads from an unbacked page return all 1s, 5925 * which means that the 32 bits located at the 5926 * given physical address won't match the required 5927 * VMCS12_REVISION identifier. 5928 */ 5929 return nested_vmx_fail(vcpu, 5930 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5931 } 5932 5933 if (kvm_read_guest_offset_cached(vcpu->kvm, ghc, &hdr, 5934 offsetof(struct vmcs12, hdr), 5935 sizeof(hdr))) { 5936 return nested_vmx_fail(vcpu, 5937 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5938 } 5939 5940 if (hdr.revision_id != VMCS12_REVISION || 5941 (hdr.shadow_vmcs && 5942 !nested_cpu_has_vmx_shadow_vmcs(vcpu))) { 5943 return nested_vmx_fail(vcpu, 5944 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5945 } 5946 5947 nested_release_vmcs12(vcpu); 5948 5949 /* 5950 * Load VMCS12 from guest memory since it is not already 5951 * cached. 5952 */ 5953 if (kvm_read_guest_cached(vcpu->kvm, ghc, vmx->nested.cached_vmcs12, 5954 VMCS12_SIZE)) { 5955 return nested_vmx_fail(vcpu, 5956 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5957 } 5958 5959 set_current_vmptr(vmx, vmptr); 5960 } 5961 5962 return nested_vmx_succeed(vcpu); 5963 } 5964 5965 /* Emulate the VMPTRST instruction */ 5966 static int handle_vmptrst(struct kvm_vcpu *vcpu) 5967 { 5968 unsigned long exit_qual = vt_get_exit_qual(vcpu); 5969 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5970 gpa_t current_vmptr; 5971 struct x86_exception e; 5972 gva_t gva; 5973 int r; 5974 5975 if (!nested_vmx_check_permission(vcpu)) 5976 return 1; 5977 5978 /* 5979 * Hyper-V TLFS does not specify the behavior of VMPTRST when eVMCS is used 5980 * but genuine Hyper-V seems to be returning eVMCS GPA. 5981 */ 5982 #ifdef CONFIG_KVM_HYPERV 5983 if (nested_vmx_is_evmptr12_valid(to_vmx(vcpu))) 5984 current_vmptr = to_vmx(vcpu)->nested.hv_evmcs_vmptr; 5985 else 5986 #endif 5987 current_vmptr = to_vmx(vcpu)->nested.current_vmptr; 5988 5989 if (get_vmx_mem_address(vcpu, exit_qual, instr_info, 5990 true, sizeof(gpa_t), &gva)) 5991 return 1; 5992 /* *_system ok, nested_vmx_check_permission has verified cpl=0 */ 5993 r = kvm_write_guest_virt_system(vcpu, gva, (void *)¤t_vmptr, 5994 sizeof(gpa_t), &e); 5995 if (r != X86EMUL_CONTINUE) 5996 return kvm_handle_memory_failure(vcpu, r, &e); 5997 5998 return nested_vmx_succeed(vcpu); 5999 } 6000 6001 /* Emulate the INVEPT instruction */ 6002 static int handle_invept(struct kvm_vcpu *vcpu) 6003 { 6004 struct vcpu_vmx *vmx = to_vmx(vcpu); 6005 u32 vmx_instruction_info, types; 6006 unsigned long type, roots_to_free; 6007 struct kvm_mmu *mmu; 6008 gva_t gva; 6009 struct x86_exception e; 6010 struct { 6011 u64 eptp, gpa; 6012 } operand; 6013 int i, r, gpr_index; 6014 6015 if (!(vmx->nested.msrs.secondary_ctls_high & 6016 SECONDARY_EXEC_ENABLE_EPT) || 6017 !(vmx->nested.msrs.ept_caps & VMX_EPT_INVEPT_BIT)) { 6018 kvm_queue_exception(vcpu, UD_VECTOR); 6019 return 1; 6020 } 6021 6022 if (!nested_vmx_check_permission(vcpu)) 6023 return 1; 6024 6025 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 6026 gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info); 6027 type = kvm_register_read(vcpu, gpr_index); 6028 6029 types = (vmx->nested.msrs.ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6; 6030 6031 if (type >= 32 || !(types & (1 << type))) 6032 return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 6033 6034 /* According to the Intel VMX instruction reference, the memory 6035 * operand is read even if it isn't needed (e.g., for type==global) 6036 */ 6037 if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), 6038 vmx_instruction_info, false, sizeof(operand), &gva)) 6039 return 1; 6040 r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); 6041 if (r != X86EMUL_CONTINUE) 6042 return kvm_handle_memory_failure(vcpu, r, &e); 6043 6044 /* 6045 * Nested EPT roots are always held through guest_mmu, 6046 * not root_mmu. 6047 */ 6048 mmu = &vcpu->arch.guest_mmu; 6049 6050 switch (type) { 6051 case VMX_EPT_EXTENT_CONTEXT: 6052 if (!nested_vmx_check_eptp(vcpu, operand.eptp)) 6053 return nested_vmx_fail(vcpu, 6054 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 6055 6056 roots_to_free = 0; 6057 if (nested_ept_root_matches(mmu->root.hpa, mmu->root.pgd, 6058 operand.eptp)) 6059 roots_to_free |= KVM_MMU_ROOT_CURRENT; 6060 6061 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { 6062 if (nested_ept_root_matches(mmu->prev_roots[i].hpa, 6063 mmu->prev_roots[i].pgd, 6064 operand.eptp)) 6065 roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i); 6066 } 6067 break; 6068 case VMX_EPT_EXTENT_GLOBAL: 6069 roots_to_free = KVM_MMU_ROOTS_ALL; 6070 break; 6071 default: 6072 BUG(); 6073 break; 6074 } 6075 6076 if (roots_to_free) 6077 kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free); 6078 6079 return nested_vmx_succeed(vcpu); 6080 } 6081 6082 static int handle_invvpid(struct kvm_vcpu *vcpu) 6083 { 6084 struct vcpu_vmx *vmx = to_vmx(vcpu); 6085 u32 vmx_instruction_info; 6086 unsigned long type, types; 6087 gva_t gva; 6088 struct x86_exception e; 6089 struct { 6090 u64 vpid; 6091 u64 gla; 6092 } operand; 6093 int r, gpr_index; 6094 int cpu; 6095 6096 if (!(vmx->nested.msrs.secondary_ctls_high & 6097 SECONDARY_EXEC_ENABLE_VPID) || 6098 !(vmx->nested.msrs.vpid_caps & VMX_VPID_INVVPID_BIT)) { 6099 kvm_queue_exception(vcpu, UD_VECTOR); 6100 return 1; 6101 } 6102 6103 if (!nested_vmx_check_permission(vcpu)) 6104 return 1; 6105 6106 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 6107 gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info); 6108 type = kvm_register_read(vcpu, gpr_index); 6109 6110 types = (vmx->nested.msrs.vpid_caps & 6111 VMX_VPID_EXTENT_SUPPORTED_MASK) >> 8; 6112 6113 if (type >= 32 || !(types & (1 << type))) 6114 return nested_vmx_fail(vcpu, 6115 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 6116 6117 /* according to the intel vmx instruction reference, the memory 6118 * operand is read even if it isn't needed (e.g., for type==global) 6119 */ 6120 if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), 6121 vmx_instruction_info, false, sizeof(operand), &gva)) 6122 return 1; 6123 r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); 6124 if (r != X86EMUL_CONTINUE) 6125 return kvm_handle_memory_failure(vcpu, r, &e); 6126 6127 if (operand.vpid >> 16) 6128 return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 6129 6130 if (type != VMX_VPID_EXTENT_ALL_CONTEXT && !operand.vpid) 6131 return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 6132 6133 /* LAM doesn't apply to addresses that are inputs to TLB invalidation. */ 6134 if (type == VMX_VPID_EXTENT_INDIVIDUAL_ADDR && 6135 is_noncanonical_invlpg_address(operand.gla, vcpu)) 6136 return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 6137 6138 /* 6139 * Always flush the effective vpid02, i.e. never flush the current VPID 6140 * and never explicitly flush vpid01. INVVPID targets a VPID, not a 6141 * VMCS, and so whether or not the current vmcs12 has VPID enabled is 6142 * irrelevant (and there may not be a loaded vmcs12). 6143 * 6144 * If vmcs02 was last loaded on a different pCPU, then defer the flush 6145 * by invalidating the nested VPID tracking to ensure that KVM performs 6146 * the invalidation on the correct pCPU. 6147 */ 6148 cpu = get_cpu(); 6149 if (cpu != vmx->nested.vmcs02.cpu) 6150 vmx->nested.last_vpid = 0; 6151 else if (type == VMX_VPID_EXTENT_INDIVIDUAL_ADDR) 6152 vpid_sync_vcpu_addr(nested_get_vpid02(vcpu), operand.gla); 6153 else 6154 vpid_sync_context(nested_get_vpid02(vcpu)); 6155 put_cpu(); 6156 6157 /* 6158 * Sync the shadow page tables if EPT is disabled, L1 is invalidating 6159 * linear mappings for L2 (tagged with L2's VPID). Free all guest 6160 * roots as VPIDs are not tracked in the MMU role. 6161 * 6162 * Note, this operates on root_mmu, not guest_mmu, as L1 and L2 share 6163 * an MMU when EPT is disabled. 6164 * 6165 * TODO: sync only the affected SPTEs for INVDIVIDUAL_ADDR. 6166 */ 6167 if (!enable_ept) 6168 kvm_mmu_free_guest_mode_roots(vcpu->kvm, &vcpu->arch.root_mmu); 6169 6170 return nested_vmx_succeed(vcpu); 6171 } 6172 6173 static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu, 6174 struct vmcs12 *vmcs12) 6175 { 6176 u32 index = kvm_ecx_read(vcpu); 6177 u64 new_eptp; 6178 6179 if (WARN_ON_ONCE(!nested_cpu_has_ept(vmcs12))) 6180 return 1; 6181 if (index >= VMFUNC_EPTP_ENTRIES) 6182 return 1; 6183 6184 if (kvm_vcpu_read_guest_page(vcpu, vmcs12->eptp_list_address >> PAGE_SHIFT, 6185 &new_eptp, index * 8, 8)) 6186 return 1; 6187 6188 /* 6189 * If the (L2) guest does a vmfunc to the currently 6190 * active ept pointer, we don't have to do anything else 6191 */ 6192 if (vmcs12->ept_pointer != new_eptp) { 6193 if (!nested_vmx_check_eptp(vcpu, new_eptp)) 6194 return 1; 6195 6196 vmcs12->ept_pointer = new_eptp; 6197 nested_ept_new_eptp(vcpu); 6198 6199 if (!nested_cpu_has_vpid(vmcs12)) 6200 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 6201 } 6202 6203 return 0; 6204 } 6205 6206 static int handle_vmfunc(struct kvm_vcpu *vcpu) 6207 { 6208 struct vcpu_vmx *vmx = to_vmx(vcpu); 6209 struct vmcs12 *vmcs12; 6210 u32 function = kvm_eax_read(vcpu); 6211 6212 /* 6213 * VMFUNC should never execute cleanly while L1 is active; KVM supports 6214 * VMFUNC for nested VMs, but not for L1. 6215 */ 6216 if (WARN_ON_ONCE(!is_guest_mode(vcpu))) { 6217 kvm_queue_exception(vcpu, UD_VECTOR); 6218 return 1; 6219 } 6220 6221 vmcs12 = get_vmcs12(vcpu); 6222 6223 /* 6224 * #UD on out-of-bounds function has priority over VM-Exit, and VMFUNC 6225 * is enabled in vmcs02 if and only if it's enabled in vmcs12. 6226 */ 6227 if (WARN_ON_ONCE((function > 63) || !nested_cpu_has_vmfunc(vmcs12))) { 6228 kvm_queue_exception(vcpu, UD_VECTOR); 6229 return 1; 6230 } 6231 6232 if (!(vmcs12->vm_function_control & BIT_ULL(function))) 6233 goto fail; 6234 6235 switch (function) { 6236 case 0: 6237 if (nested_vmx_eptp_switching(vcpu, vmcs12)) 6238 goto fail; 6239 break; 6240 default: 6241 goto fail; 6242 } 6243 return kvm_skip_emulated_instruction(vcpu); 6244 6245 fail: 6246 /* 6247 * This is effectively a reflected VM-Exit, as opposed to a synthesized 6248 * nested VM-Exit. Pass the original exit reason, i.e. don't hardcode 6249 * EXIT_REASON_VMFUNC as the exit reason. 6250 */ 6251 nested_vmx_vmexit(vcpu, vmx->vt.exit_reason.full, 6252 vt_get_intr_info(vcpu), 6253 vt_get_exit_qual(vcpu)); 6254 return 1; 6255 } 6256 6257 /* 6258 * Return true if an IO instruction with the specified port and size should cause 6259 * a VM-exit into L1. 6260 */ 6261 bool nested_vmx_check_io_bitmaps(struct kvm_vcpu *vcpu, unsigned int port, 6262 int size) 6263 { 6264 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 6265 gpa_t bitmap, last_bitmap; 6266 u8 b; 6267 6268 last_bitmap = INVALID_GPA; 6269 b = -1; 6270 6271 while (size > 0) { 6272 if (port < 0x8000) 6273 bitmap = vmcs12->io_bitmap_a; 6274 else if (port < 0x10000) 6275 bitmap = vmcs12->io_bitmap_b; 6276 else 6277 return true; 6278 bitmap += (port & 0x7fff) / 8; 6279 6280 if (last_bitmap != bitmap) 6281 if (kvm_vcpu_read_guest(vcpu, bitmap, &b, 1)) 6282 return true; 6283 if (b & (1 << (port & 7))) 6284 return true; 6285 6286 port++; 6287 size--; 6288 last_bitmap = bitmap; 6289 } 6290 6291 return false; 6292 } 6293 6294 static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu, 6295 struct vmcs12 *vmcs12) 6296 { 6297 unsigned long exit_qualification; 6298 unsigned short port; 6299 int size; 6300 6301 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 6302 return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING); 6303 6304 exit_qualification = vt_get_exit_qual(vcpu); 6305 6306 port = exit_qualification >> 16; 6307 size = (exit_qualification & 7) + 1; 6308 6309 return nested_vmx_check_io_bitmaps(vcpu, port, size); 6310 } 6311 6312 /* 6313 * Return 1 if we should exit from L2 to L1 to handle an MSR access, 6314 * rather than handle it ourselves in L0. I.e., check whether L1 expressed 6315 * disinterest in the current event (read or write a specific MSR) by using an 6316 * MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps. 6317 */ 6318 static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, 6319 struct vmcs12 *vmcs12, 6320 union vmx_exit_reason exit_reason) 6321 { 6322 u32 msr_index; 6323 gpa_t bitmap; 6324 6325 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 6326 return true; 6327 6328 if (exit_reason.basic == EXIT_REASON_MSR_READ_IMM || 6329 exit_reason.basic == EXIT_REASON_MSR_WRITE_IMM) 6330 msr_index = vt_get_exit_qual(vcpu); 6331 else 6332 msr_index = kvm_ecx_read(vcpu); 6333 6334 /* 6335 * The MSR_BITMAP page is divided into four 1024-byte bitmaps, 6336 * for the four combinations of read/write and low/high MSR numbers. 6337 * First we need to figure out which of the four to use: 6338 */ 6339 bitmap = vmcs12->msr_bitmap; 6340 if (exit_reason.basic == EXIT_REASON_MSR_WRITE || 6341 exit_reason.basic == EXIT_REASON_MSR_WRITE_IMM) 6342 bitmap += 2048; 6343 if (msr_index >= 0xc0000000) { 6344 msr_index -= 0xc0000000; 6345 bitmap += 1024; 6346 } 6347 6348 /* Then read the msr_index'th bit from this bitmap: */ 6349 if (msr_index < 1024*8) { 6350 unsigned char b; 6351 if (kvm_vcpu_read_guest(vcpu, bitmap + msr_index/8, &b, 1)) 6352 return true; 6353 return 1 & (b >> (msr_index & 7)); 6354 } else 6355 return true; /* let L1 handle the wrong parameter */ 6356 } 6357 6358 /* 6359 * Return 1 if we should exit from L2 to L1 to handle a CR access exit, 6360 * rather than handle it ourselves in L0. I.e., check if L1 wanted to 6361 * intercept (via guest_host_mask etc.) the current event. 6362 */ 6363 static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu, 6364 struct vmcs12 *vmcs12) 6365 { 6366 unsigned long exit_qualification = vt_get_exit_qual(vcpu); 6367 int cr = exit_qualification & 15; 6368 int reg; 6369 unsigned long val; 6370 6371 switch ((exit_qualification >> 4) & 3) { 6372 case 0: /* mov to cr */ 6373 reg = (exit_qualification >> 8) & 15; 6374 val = kvm_register_read(vcpu, reg); 6375 switch (cr) { 6376 case 0: 6377 if (vmcs12->cr0_guest_host_mask & 6378 (val ^ vmcs12->cr0_read_shadow)) 6379 return true; 6380 break; 6381 case 3: 6382 if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING)) 6383 return true; 6384 break; 6385 case 4: 6386 if (vmcs12->cr4_guest_host_mask & 6387 (vmcs12->cr4_read_shadow ^ val)) 6388 return true; 6389 break; 6390 case 8: 6391 if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING)) 6392 return true; 6393 break; 6394 } 6395 break; 6396 case 2: /* clts */ 6397 if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) && 6398 (vmcs12->cr0_read_shadow & X86_CR0_TS)) 6399 return true; 6400 break; 6401 case 1: /* mov from cr */ 6402 switch (cr) { 6403 case 3: 6404 if (vmcs12->cpu_based_vm_exec_control & 6405 CPU_BASED_CR3_STORE_EXITING) 6406 return true; 6407 break; 6408 case 8: 6409 if (vmcs12->cpu_based_vm_exec_control & 6410 CPU_BASED_CR8_STORE_EXITING) 6411 return true; 6412 break; 6413 } 6414 break; 6415 case 3: /* lmsw */ 6416 /* 6417 * lmsw can change bits 1..3 of cr0, and only set bit 0 of 6418 * cr0. Other attempted changes are ignored, with no exit. 6419 */ 6420 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f; 6421 if (vmcs12->cr0_guest_host_mask & 0xe & 6422 (val ^ vmcs12->cr0_read_shadow)) 6423 return true; 6424 if ((vmcs12->cr0_guest_host_mask & 0x1) && 6425 !(vmcs12->cr0_read_shadow & 0x1) && 6426 (val & 0x1)) 6427 return true; 6428 break; 6429 } 6430 return false; 6431 } 6432 6433 static bool nested_vmx_exit_handled_encls(struct kvm_vcpu *vcpu, 6434 struct vmcs12 *vmcs12) 6435 { 6436 u32 encls_leaf; 6437 6438 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SGX) || 6439 !nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENCLS_EXITING)) 6440 return false; 6441 6442 encls_leaf = kvm_eax_read(vcpu); 6443 if (encls_leaf > 62) 6444 encls_leaf = 63; 6445 return vmcs12->encls_exiting_bitmap & BIT_ULL(encls_leaf); 6446 } 6447 6448 static bool nested_vmx_exit_handled_vmcs_access(struct kvm_vcpu *vcpu, 6449 struct vmcs12 *vmcs12, gpa_t bitmap) 6450 { 6451 u32 vmx_instruction_info; 6452 unsigned long field; 6453 u8 b; 6454 6455 if (!nested_cpu_has_shadow_vmcs(vmcs12)) 6456 return true; 6457 6458 /* Decode instruction info and find the field to access */ 6459 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 6460 field = kvm_register_read(vcpu, (((vmx_instruction_info) >> 28) & 0xf)); 6461 6462 /* Out-of-range fields always cause a VM exit from L2 to L1 */ 6463 if (field >> 15) 6464 return true; 6465 6466 if (kvm_vcpu_read_guest(vcpu, bitmap + field/8, &b, 1)) 6467 return true; 6468 6469 return 1 & (b >> (field & 7)); 6470 } 6471 6472 static bool nested_vmx_exit_handled_mtf(struct vmcs12 *vmcs12) 6473 { 6474 u32 entry_intr_info = vmcs12->vm_entry_intr_info_field; 6475 6476 if (nested_cpu_has_mtf(vmcs12)) 6477 return true; 6478 6479 /* 6480 * An MTF VM-exit may be injected into the guest by setting the 6481 * interruption-type to 7 (other event) and the vector field to 0. Such 6482 * is the case regardless of the 'monitor trap flag' VM-execution 6483 * control. 6484 */ 6485 return entry_intr_info == (INTR_INFO_VALID_MASK 6486 | INTR_TYPE_OTHER_EVENT); 6487 } 6488 6489 /* 6490 * Return true if L0 wants to handle an exit from L2 regardless of whether or not 6491 * L1 wants the exit. Only call this when in is_guest_mode (L2). 6492 */ 6493 static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, 6494 union vmx_exit_reason exit_reason) 6495 { 6496 u32 intr_info; 6497 6498 switch ((u16)exit_reason.basic) { 6499 case EXIT_REASON_EXCEPTION_NMI: 6500 intr_info = vt_get_intr_info(vcpu); 6501 if (is_nmi(intr_info)) 6502 return true; 6503 else if (is_page_fault(intr_info)) 6504 return vcpu->arch.apf.host_apf_flags || 6505 vmx_need_pf_intercept(vcpu); 6506 else if (is_debug(intr_info) && 6507 vcpu->guest_debug & 6508 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) 6509 return true; 6510 else if (is_breakpoint(intr_info) && 6511 vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) 6512 return true; 6513 else if (is_alignment_check(intr_info) && 6514 !vmx_guest_inject_ac(vcpu)) 6515 return true; 6516 else if (is_ve_fault(intr_info)) 6517 return true; 6518 return false; 6519 case EXIT_REASON_EXTERNAL_INTERRUPT: 6520 return true; 6521 case EXIT_REASON_MCE_DURING_VMENTRY: 6522 return true; 6523 case EXIT_REASON_EPT_VIOLATION: 6524 /* 6525 * L0 always deals with the EPT violation. If nested EPT is 6526 * used, and the nested mmu code discovers that the address is 6527 * missing in the guest EPT table (EPT12), the EPT violation 6528 * will be injected with nested_ept_inject_page_fault() 6529 */ 6530 return true; 6531 case EXIT_REASON_EPT_MISCONFIG: 6532 /* 6533 * L2 never uses directly L1's EPT, but rather L0's own EPT 6534 * table (shadow on EPT) or a merged EPT table that L0 built 6535 * (EPT on EPT). So any problems with the structure of the 6536 * table is L0's fault. 6537 */ 6538 return true; 6539 case EXIT_REASON_PREEMPTION_TIMER: 6540 return true; 6541 case EXIT_REASON_PML_FULL: 6542 /* 6543 * PML is emulated for an L1 VMM and should never be enabled in 6544 * vmcs02, always "handle" PML_FULL by exiting to userspace. 6545 */ 6546 return true; 6547 case EXIT_REASON_VMFUNC: 6548 /* VM functions are emulated through L2->L0 vmexits. */ 6549 return true; 6550 case EXIT_REASON_BUS_LOCK: 6551 /* 6552 * At present, bus lock VM exit is never exposed to L1. 6553 * Handle L2's bus locks in L0 directly. 6554 */ 6555 return true; 6556 #ifdef CONFIG_KVM_HYPERV 6557 case EXIT_REASON_VMCALL: 6558 /* Hyper-V L2 TLB flush hypercall is handled by L0 */ 6559 return guest_hv_cpuid_has_l2_tlb_flush(vcpu) && 6560 nested_evmcs_l2_tlb_flush_enabled(vcpu) && 6561 kvm_hv_is_tlb_flush_hcall(vcpu); 6562 #endif 6563 case EXIT_REASON_CPUID: 6564 return !kvm_is_cpuid_allowed(vcpu); 6565 default: 6566 break; 6567 } 6568 return false; 6569 } 6570 6571 /* 6572 * Return 1 if L1 wants to intercept an exit from L2. Only call this when in 6573 * is_guest_mode (L2). 6574 */ 6575 static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu, 6576 union vmx_exit_reason exit_reason) 6577 { 6578 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 6579 u32 intr_info; 6580 6581 switch ((u16)exit_reason.basic) { 6582 case EXIT_REASON_EXCEPTION_NMI: 6583 intr_info = vt_get_intr_info(vcpu); 6584 if (is_nmi(intr_info)) 6585 return true; 6586 else if (is_page_fault(intr_info)) 6587 return true; 6588 return vmcs12->exception_bitmap & 6589 (1u << (intr_info & INTR_INFO_VECTOR_MASK)); 6590 case EXIT_REASON_EXTERNAL_INTERRUPT: 6591 return nested_exit_on_intr(vcpu); 6592 case EXIT_REASON_TRIPLE_FAULT: 6593 return true; 6594 case EXIT_REASON_INTERRUPT_WINDOW: 6595 return nested_cpu_has(vmcs12, CPU_BASED_INTR_WINDOW_EXITING); 6596 case EXIT_REASON_NMI_WINDOW: 6597 return nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING); 6598 case EXIT_REASON_TASK_SWITCH: 6599 return true; 6600 case EXIT_REASON_CPUID: 6601 return true; 6602 case EXIT_REASON_HLT: 6603 return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING); 6604 case EXIT_REASON_INVD: 6605 return true; 6606 case EXIT_REASON_INVLPG: 6607 return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 6608 case EXIT_REASON_RDPMC: 6609 return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING); 6610 case EXIT_REASON_RDRAND: 6611 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDRAND_EXITING); 6612 case EXIT_REASON_RDSEED: 6613 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDSEED_EXITING); 6614 case EXIT_REASON_RDTSC: case EXIT_REASON_RDTSCP: 6615 return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING); 6616 case EXIT_REASON_VMREAD: 6617 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12, 6618 vmcs12->vmread_bitmap); 6619 case EXIT_REASON_VMWRITE: 6620 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12, 6621 vmcs12->vmwrite_bitmap); 6622 case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR: 6623 case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD: 6624 case EXIT_REASON_VMPTRST: case EXIT_REASON_VMRESUME: 6625 case EXIT_REASON_VMOFF: case EXIT_REASON_VMON: 6626 case EXIT_REASON_INVEPT: case EXIT_REASON_INVVPID: 6627 /* 6628 * VMX instructions trap unconditionally. This allows L1 to 6629 * emulate them for its L2 guest, i.e., allows 3-level nesting! 6630 */ 6631 return true; 6632 case EXIT_REASON_CR_ACCESS: 6633 return nested_vmx_exit_handled_cr(vcpu, vmcs12); 6634 case EXIT_REASON_DR_ACCESS: 6635 return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING); 6636 case EXIT_REASON_IO_INSTRUCTION: 6637 return nested_vmx_exit_handled_io(vcpu, vmcs12); 6638 case EXIT_REASON_GDTR_IDTR: case EXIT_REASON_LDTR_TR: 6639 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC); 6640 case EXIT_REASON_MSR_READ: 6641 case EXIT_REASON_MSR_WRITE: 6642 case EXIT_REASON_MSR_READ_IMM: 6643 case EXIT_REASON_MSR_WRITE_IMM: 6644 return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason); 6645 case EXIT_REASON_INVALID_STATE: 6646 return true; 6647 case EXIT_REASON_MWAIT_INSTRUCTION: 6648 return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING); 6649 case EXIT_REASON_MONITOR_TRAP_FLAG: 6650 return nested_vmx_exit_handled_mtf(vmcs12); 6651 case EXIT_REASON_MONITOR_INSTRUCTION: 6652 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING); 6653 case EXIT_REASON_PAUSE_INSTRUCTION: 6654 return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) || 6655 nested_cpu_has2(vmcs12, 6656 SECONDARY_EXEC_PAUSE_LOOP_EXITING); 6657 case EXIT_REASON_MCE_DURING_VMENTRY: 6658 return true; 6659 case EXIT_REASON_TPR_BELOW_THRESHOLD: 6660 return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW); 6661 case EXIT_REASON_APIC_ACCESS: 6662 case EXIT_REASON_APIC_WRITE: 6663 case EXIT_REASON_EOI_INDUCED: 6664 /* 6665 * The controls for "virtualize APIC accesses," "APIC- 6666 * register virtualization," and "virtual-interrupt 6667 * delivery" only come from vmcs12. 6668 */ 6669 return true; 6670 case EXIT_REASON_INVPCID: 6671 return 6672 nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_INVPCID) && 6673 nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 6674 case EXIT_REASON_WBINVD: 6675 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING); 6676 case EXIT_REASON_XSETBV: 6677 return true; 6678 case EXIT_REASON_XSAVES: 6679 case EXIT_REASON_XRSTORS: 6680 /* 6681 * Always forward XSAVES/XRSTORS to L1 as KVM doesn't utilize 6682 * XSS-bitmap, and always loads vmcs02 with vmcs12's XSS-bitmap 6683 * verbatim, i.e. any exit is due to L1's bitmap. WARN if 6684 * XSAVES isn't enabled, as the CPU is supposed to inject #UD 6685 * in that case, before consulting the XSS-bitmap. 6686 */ 6687 WARN_ON_ONCE(!nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_XSAVES)); 6688 return true; 6689 case EXIT_REASON_UMWAIT: 6690 case EXIT_REASON_TPAUSE: 6691 return nested_cpu_has2(vmcs12, 6692 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE); 6693 case EXIT_REASON_ENCLS: 6694 return nested_vmx_exit_handled_encls(vcpu, vmcs12); 6695 case EXIT_REASON_NOTIFY: 6696 /* Notify VM exit is not exposed to L1 */ 6697 return false; 6698 case EXIT_REASON_SEAMCALL: 6699 case EXIT_REASON_TDCALL: 6700 /* 6701 * SEAMCALL and TDCALL unconditionally VM-Exit, but aren't 6702 * virtualized by KVM for L1 hypervisors, i.e. L1 should 6703 * never want or expect such an exit. 6704 */ 6705 return false; 6706 default: 6707 return true; 6708 } 6709 } 6710 6711 /* 6712 * Conditionally reflect a VM-Exit into L1. Returns %true if the VM-Exit was 6713 * reflected into L1. 6714 */ 6715 bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu) 6716 { 6717 struct vcpu_vmx *vmx = to_vmx(vcpu); 6718 union vmx_exit_reason exit_reason = vmx->vt.exit_reason; 6719 unsigned long exit_qual; 6720 u32 exit_intr_info; 6721 6722 kvm_warn_on_nested_run_pending(vcpu); 6723 6724 /* 6725 * Late nested VM-Fail shares the same flow as nested VM-Exit since KVM 6726 * has already loaded L2's state. 6727 */ 6728 if (unlikely(vmx->fail)) { 6729 trace_kvm_nested_vmenter_failed( 6730 "hardware VM-instruction error: ", 6731 vmcs_read32(VM_INSTRUCTION_ERROR)); 6732 exit_intr_info = 0; 6733 exit_qual = 0; 6734 goto reflect_vmexit; 6735 } 6736 6737 trace_kvm_nested_vmexit(vcpu, KVM_ISA_VMX); 6738 6739 /* If L0 (KVM) wants the exit, it trumps L1's desires. */ 6740 if (nested_vmx_l0_wants_exit(vcpu, exit_reason)) 6741 return false; 6742 6743 /* If L1 doesn't want the exit, handle it in L0. */ 6744 if (!nested_vmx_l1_wants_exit(vcpu, exit_reason)) 6745 return false; 6746 6747 /* 6748 * vmcs.VM_EXIT_INTR_INFO is only valid for EXCEPTION_NMI exits. For 6749 * EXTERNAL_INTERRUPT, the value for vmcs12->vm_exit_intr_info would 6750 * need to be synthesized by querying the in-kernel LAPIC, but external 6751 * interrupts are never reflected to L1 so it's a non-issue. 6752 */ 6753 exit_intr_info = vt_get_intr_info(vcpu); 6754 if (is_exception_with_error_code(exit_intr_info)) { 6755 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 6756 6757 vmcs12->vm_exit_intr_error_code = 6758 vmcs_read32(VM_EXIT_INTR_ERROR_CODE); 6759 } 6760 exit_qual = vt_get_exit_qual(vcpu); 6761 6762 reflect_vmexit: 6763 nested_vmx_vmexit(vcpu, exit_reason.full, exit_intr_info, exit_qual); 6764 return true; 6765 } 6766 6767 static int vmx_get_nested_state(struct kvm_vcpu *vcpu, 6768 struct kvm_nested_state __user *user_kvm_nested_state, 6769 u32 user_data_size) 6770 { 6771 struct vcpu_vmx *vmx; 6772 struct vmcs12 *vmcs12; 6773 struct kvm_nested_state kvm_state = { 6774 .flags = 0, 6775 .format = KVM_STATE_NESTED_FORMAT_VMX, 6776 .size = sizeof(kvm_state), 6777 .hdr.vmx.flags = 0, 6778 .hdr.vmx.vmxon_pa = INVALID_GPA, 6779 .hdr.vmx.vmcs12_pa = INVALID_GPA, 6780 .hdr.vmx.preemption_timer_deadline = 0, 6781 }; 6782 struct kvm_vmx_nested_state_data __user *user_vmx_nested_state = 6783 &user_kvm_nested_state->data.vmx[0]; 6784 6785 if (!vcpu) 6786 return kvm_state.size + sizeof(*user_vmx_nested_state); 6787 6788 vmx = to_vmx(vcpu); 6789 vmcs12 = get_vmcs12(vcpu); 6790 6791 if (guest_cpu_cap_has(vcpu, X86_FEATURE_VMX) && 6792 (vmx->nested.vmxon || vmx->nested.smm.vmxon)) { 6793 kvm_state.hdr.vmx.vmxon_pa = vmx->nested.vmxon_ptr; 6794 kvm_state.hdr.vmx.vmcs12_pa = vmx->nested.current_vmptr; 6795 6796 if (vmx_has_valid_vmcs12(vcpu)) { 6797 kvm_state.size += sizeof(user_vmx_nested_state->vmcs12); 6798 6799 /* 'hv_evmcs_vmptr' can also be EVMPTR_MAP_PENDING here */ 6800 if (nested_vmx_is_evmptr12_set(vmx)) 6801 kvm_state.flags |= KVM_STATE_NESTED_EVMCS; 6802 6803 if (is_guest_mode(vcpu) && 6804 nested_cpu_has_shadow_vmcs(vmcs12) && 6805 vmcs12->vmcs_link_pointer != INVALID_GPA) 6806 kvm_state.size += sizeof(user_vmx_nested_state->shadow_vmcs12); 6807 } 6808 6809 if (vmx->nested.smm.vmxon) 6810 kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_VMXON; 6811 6812 if (vmx->nested.smm.guest_mode) 6813 kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_GUEST_MODE; 6814 6815 if (is_guest_mode(vcpu)) { 6816 kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE; 6817 6818 if (vcpu->arch.nested_run_pending) 6819 kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING; 6820 6821 if (vmx->nested.mtf_pending) 6822 kvm_state.flags |= KVM_STATE_NESTED_MTF_PENDING; 6823 6824 if (nested_cpu_has_preemption_timer(vmcs12) && 6825 vmx->nested.has_preemption_timer_deadline) { 6826 kvm_state.hdr.vmx.flags |= 6827 KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE; 6828 kvm_state.hdr.vmx.preemption_timer_deadline = 6829 vmx->nested.preemption_timer_deadline; 6830 } 6831 } 6832 } 6833 6834 if (user_data_size < kvm_state.size) 6835 goto out; 6836 6837 if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state))) 6838 return -EFAULT; 6839 6840 if (!vmx_has_valid_vmcs12(vcpu)) 6841 goto out; 6842 6843 /* 6844 * When running L2, the authoritative vmcs12 state is in the 6845 * vmcs02. When running L1, the authoritative vmcs12 state is 6846 * in the shadow or enlightened vmcs linked to vmcs01, unless 6847 * need_vmcs12_to_shadow_sync is set, in which case, the authoritative 6848 * vmcs12 state is in the vmcs12 already. 6849 */ 6850 if (is_guest_mode(vcpu)) { 6851 sync_vmcs02_to_vmcs12(vcpu, vmcs12); 6852 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 6853 } else { 6854 copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu)); 6855 if (!vmx->nested.need_vmcs12_to_shadow_sync) { 6856 if (nested_vmx_is_evmptr12_valid(vmx)) 6857 /* 6858 * L1 hypervisor is not obliged to keep eVMCS 6859 * clean fields data always up-to-date while 6860 * not in guest mode, 'hv_clean_fields' is only 6861 * supposed to be actual upon vmentry so we need 6862 * to ignore it here and do full copy. 6863 */ 6864 copy_enlightened_to_vmcs12(vmx, 0); 6865 else if (enable_shadow_vmcs) 6866 copy_shadow_to_vmcs12(vmx); 6867 } 6868 } 6869 6870 BUILD_BUG_ON(sizeof(user_vmx_nested_state->vmcs12) < VMCS12_SIZE); 6871 BUILD_BUG_ON(sizeof(user_vmx_nested_state->shadow_vmcs12) < VMCS12_SIZE); 6872 6873 /* 6874 * Copy over the full allocated size of vmcs12 rather than just the size 6875 * of the struct. 6876 */ 6877 if (copy_to_user(user_vmx_nested_state->vmcs12, vmcs12, VMCS12_SIZE)) 6878 return -EFAULT; 6879 6880 if (nested_cpu_has_shadow_vmcs(vmcs12) && 6881 vmcs12->vmcs_link_pointer != INVALID_GPA) { 6882 if (copy_to_user(user_vmx_nested_state->shadow_vmcs12, 6883 get_shadow_vmcs12(vcpu), VMCS12_SIZE)) 6884 return -EFAULT; 6885 } 6886 out: 6887 return kvm_state.size; 6888 } 6889 6890 void vmx_leave_nested(struct kvm_vcpu *vcpu) 6891 { 6892 if (is_guest_mode(vcpu)) { 6893 vcpu->arch.nested_run_pending = 0; 6894 nested_vmx_vmexit(vcpu, -1, 0, 0); 6895 } 6896 free_nested(vcpu); 6897 } 6898 6899 int nested_vmx_check_restored_vmcs12(struct kvm_vcpu *vcpu) 6900 { 6901 enum vm_entry_failure_code ignored; 6902 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 6903 6904 if (nested_cpu_has_shadow_vmcs(vmcs12) && 6905 vmcs12->vmcs_link_pointer != INVALID_GPA) { 6906 struct vmcs12 *shadow_vmcs12 = get_shadow_vmcs12(vcpu); 6907 6908 if (shadow_vmcs12->hdr.revision_id != VMCS12_REVISION || 6909 !shadow_vmcs12->hdr.shadow_vmcs) 6910 return -EINVAL; 6911 } 6912 6913 if (nested_vmx_check_controls(vcpu, vmcs12) || 6914 nested_vmx_check_host_state(vcpu, vmcs12) || 6915 nested_vmx_check_guest_state(vcpu, vmcs12, &ignored)) 6916 return -EINVAL; 6917 6918 return 0; 6919 } 6920 6921 static int vmx_set_nested_state(struct kvm_vcpu *vcpu, 6922 struct kvm_nested_state __user *user_kvm_nested_state, 6923 struct kvm_nested_state *kvm_state) 6924 { 6925 struct vcpu_vmx *vmx = to_vmx(vcpu); 6926 struct vmcs12 *vmcs12; 6927 struct kvm_vmx_nested_state_data __user *user_vmx_nested_state = 6928 &user_kvm_nested_state->data.vmx[0]; 6929 int ret; 6930 6931 if (kvm_state->format != KVM_STATE_NESTED_FORMAT_VMX) 6932 return -EINVAL; 6933 6934 if (kvm_state->hdr.vmx.vmxon_pa == INVALID_GPA) { 6935 if (kvm_state->hdr.vmx.smm.flags) 6936 return -EINVAL; 6937 6938 if (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA) 6939 return -EINVAL; 6940 6941 /* 6942 * KVM_STATE_NESTED_EVMCS used to signal that KVM should 6943 * enable eVMCS capability on vCPU. However, since then 6944 * code was changed such that flag signals vmcs12 should 6945 * be copied into eVMCS in guest memory. 6946 * 6947 * To preserve backwards compatibility, allow user 6948 * to set this flag even when there is no VMXON region. 6949 */ 6950 if (kvm_state->flags & ~KVM_STATE_NESTED_EVMCS) 6951 return -EINVAL; 6952 } else { 6953 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_VMX)) 6954 return -EINVAL; 6955 6956 if (!page_address_valid(vcpu, kvm_state->hdr.vmx.vmxon_pa)) 6957 return -EINVAL; 6958 } 6959 6960 if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) && 6961 (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 6962 return -EINVAL; 6963 6964 if (kvm_state->hdr.vmx.smm.flags & 6965 ~(KVM_STATE_NESTED_SMM_GUEST_MODE | KVM_STATE_NESTED_SMM_VMXON)) 6966 return -EINVAL; 6967 6968 if (kvm_state->hdr.vmx.flags & ~KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE) 6969 return -EINVAL; 6970 6971 /* 6972 * SMM temporarily disables VMX, so we cannot be in guest mode, 6973 * nor can VMLAUNCH/VMRESUME be pending. Outside SMM, SMM flags 6974 * must be zero. 6975 */ 6976 if (is_smm(vcpu) ? 6977 (kvm_state->flags & 6978 (KVM_STATE_NESTED_GUEST_MODE | KVM_STATE_NESTED_RUN_PENDING)) 6979 : kvm_state->hdr.vmx.smm.flags) 6980 return -EINVAL; 6981 6982 if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) && 6983 !(kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON)) 6984 return -EINVAL; 6985 6986 if ((kvm_state->flags & KVM_STATE_NESTED_EVMCS) && 6987 (!guest_cpu_cap_has(vcpu, X86_FEATURE_VMX) || 6988 !vmx->nested.enlightened_vmcs_enabled)) 6989 return -EINVAL; 6990 6991 vmx_leave_nested(vcpu); 6992 6993 if (kvm_state->hdr.vmx.vmxon_pa == INVALID_GPA) 6994 return 0; 6995 6996 vmx->nested.vmxon_ptr = kvm_state->hdr.vmx.vmxon_pa; 6997 ret = enter_vmx_operation(vcpu); 6998 if (ret) 6999 return ret; 7000 7001 /* Empty 'VMXON' state is permitted if no VMCS loaded */ 7002 if (kvm_state->size < sizeof(*kvm_state) + sizeof(*vmcs12)) { 7003 /* See vmx_has_valid_vmcs12. */ 7004 if ((kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE) || 7005 (kvm_state->flags & KVM_STATE_NESTED_EVMCS) || 7006 (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA)) 7007 return -EINVAL; 7008 else 7009 return 0; 7010 } 7011 7012 if (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA) { 7013 if (kvm_state->hdr.vmx.vmcs12_pa == kvm_state->hdr.vmx.vmxon_pa || 7014 !page_address_valid(vcpu, kvm_state->hdr.vmx.vmcs12_pa)) 7015 return -EINVAL; 7016 7017 set_current_vmptr(vmx, kvm_state->hdr.vmx.vmcs12_pa); 7018 #ifdef CONFIG_KVM_HYPERV 7019 } else if (kvm_state->flags & KVM_STATE_NESTED_EVMCS) { 7020 /* 7021 * nested_vmx_handle_enlightened_vmptrld() cannot be called 7022 * directly from here as HV_X64_MSR_VP_ASSIST_PAGE may not be 7023 * restored yet. EVMCS will be mapped from 7024 * nested_get_vmcs12_pages(). 7025 */ 7026 vmx->nested.hv_evmcs_vmptr = EVMPTR_MAP_PENDING; 7027 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 7028 #endif 7029 } else { 7030 return -EINVAL; 7031 } 7032 7033 if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON) { 7034 vmx->nested.smm.vmxon = true; 7035 vmx->nested.vmxon = false; 7036 7037 if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) 7038 vmx->nested.smm.guest_mode = true; 7039 } 7040 7041 vmcs12 = get_vmcs12(vcpu); 7042 if (copy_from_user(vmcs12, user_vmx_nested_state->vmcs12, sizeof(*vmcs12))) 7043 return -EFAULT; 7044 7045 if (vmcs12->hdr.revision_id != VMCS12_REVISION) 7046 return -EINVAL; 7047 7048 if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 7049 return 0; 7050 7051 if (kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING) 7052 vcpu->arch.nested_run_pending = KVM_NESTED_RUN_PENDING_UNTRUSTED; 7053 else 7054 vcpu->arch.nested_run_pending = 0; 7055 7056 vmx->nested.mtf_pending = 7057 !!(kvm_state->flags & KVM_STATE_NESTED_MTF_PENDING); 7058 7059 if (nested_cpu_has_shadow_vmcs(vmcs12) && 7060 vmcs12->vmcs_link_pointer != INVALID_GPA) { 7061 struct vmcs12 *shadow_vmcs12 = get_shadow_vmcs12(vcpu); 7062 7063 ret = -EINVAL; 7064 if (kvm_state->size < 7065 sizeof(*kvm_state) + 7066 sizeof(user_vmx_nested_state->vmcs12) + sizeof(*shadow_vmcs12)) 7067 goto error_guest_mode; 7068 7069 ret = -EFAULT; 7070 if (copy_from_user(shadow_vmcs12, 7071 user_vmx_nested_state->shadow_vmcs12, 7072 sizeof(*shadow_vmcs12))) 7073 goto error_guest_mode; 7074 } 7075 7076 vmx->nested.has_preemption_timer_deadline = false; 7077 if (kvm_state->hdr.vmx.flags & KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE) { 7078 vmx->nested.has_preemption_timer_deadline = true; 7079 vmx->nested.preemption_timer_deadline = 7080 kvm_state->hdr.vmx.preemption_timer_deadline; 7081 } 7082 7083 ret = nested_vmx_check_restored_vmcs12(vcpu); 7084 if (ret < 0) 7085 goto error_guest_mode; 7086 7087 vmx->nested.dirty_vmcs12 = true; 7088 vmx->nested.force_msr_bitmap_recalc = true; 7089 ret = nested_vmx_enter_non_root_mode(vcpu, false); 7090 if (ret) 7091 goto error_guest_mode; 7092 7093 if (vmx->nested.mtf_pending) 7094 kvm_make_request(KVM_REQ_EVENT, vcpu); 7095 7096 return 0; 7097 7098 error_guest_mode: 7099 vcpu->arch.nested_run_pending = 0; 7100 return ret; 7101 } 7102 7103 void nested_vmx_set_vmcs_shadowing_bitmap(void) 7104 { 7105 if (enable_shadow_vmcs) { 7106 vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap)); 7107 vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap)); 7108 } 7109 } 7110 7111 static u64 nested_vmx_calc_vmcs_enum_msr(void) 7112 { 7113 /* 7114 * Note these are the so called "index" of the VMCS field encoding, not 7115 * the index into vmcs12. 7116 */ 7117 unsigned int max_idx, idx; 7118 int i; 7119 7120 /* 7121 * For better or worse, KVM allows VMREAD/VMWRITE to all fields in 7122 * vmcs12, regardless of whether or not the associated feature is 7123 * exposed to L1. Simply find the field with the highest index. 7124 */ 7125 max_idx = 0; 7126 for (i = 0; i < nr_vmcs12_fields; i++) { 7127 /* The vmcs12 table is very, very sparsely populated. */ 7128 if (!vmcs12_field_offsets[i]) 7129 continue; 7130 7131 idx = vmcs_field_index(VMCS12_IDX_TO_ENC(i)); 7132 if (idx > max_idx) 7133 max_idx = idx; 7134 } 7135 7136 return (u64)max_idx << VMCS_FIELD_INDEX_SHIFT; 7137 } 7138 7139 static void nested_vmx_setup_pinbased_ctls(struct vmcs_config *vmcs_conf, 7140 struct nested_vmx_msrs *msrs) 7141 { 7142 msrs->pinbased_ctls_low = 7143 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 7144 7145 msrs->pinbased_ctls_high = vmcs_conf->pin_based_exec_ctrl; 7146 msrs->pinbased_ctls_high &= 7147 PIN_BASED_EXT_INTR_MASK | 7148 PIN_BASED_NMI_EXITING | 7149 PIN_BASED_VIRTUAL_NMIS | 7150 (enable_apicv ? PIN_BASED_POSTED_INTR : 0); 7151 msrs->pinbased_ctls_high |= 7152 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR | 7153 PIN_BASED_VMX_PREEMPTION_TIMER; 7154 } 7155 7156 static void nested_vmx_setup_exit_ctls(struct vmcs_config *vmcs_conf, 7157 struct nested_vmx_msrs *msrs) 7158 { 7159 msrs->exit_ctls_low = 7160 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; 7161 7162 msrs->exit_ctls_high = vmcs_conf->vmexit_ctrl; 7163 msrs->exit_ctls_high &= 7164 #ifdef CONFIG_X86_64 7165 VM_EXIT_HOST_ADDR_SPACE_SIZE | 7166 #endif 7167 VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT | 7168 VM_EXIT_CLEAR_BNDCFGS | VM_EXIT_LOAD_CET_STATE; 7169 msrs->exit_ctls_high |= 7170 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR | 7171 VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER | 7172 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT | 7173 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; 7174 7175 if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK) && 7176 !kvm_cpu_cap_has(X86_FEATURE_IBT)) 7177 msrs->exit_ctls_high &= ~VM_EXIT_LOAD_CET_STATE; 7178 7179 /* We support free control of debug control saving. */ 7180 msrs->exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS; 7181 } 7182 7183 static void nested_vmx_setup_entry_ctls(struct vmcs_config *vmcs_conf, 7184 struct nested_vmx_msrs *msrs) 7185 { 7186 msrs->entry_ctls_low = 7187 VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; 7188 7189 msrs->entry_ctls_high = vmcs_conf->vmentry_ctrl; 7190 msrs->entry_ctls_high &= 7191 #ifdef CONFIG_X86_64 7192 VM_ENTRY_IA32E_MODE | 7193 #endif 7194 VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_BNDCFGS | 7195 VM_ENTRY_LOAD_CET_STATE; 7196 msrs->entry_ctls_high |= 7197 (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER | 7198 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL); 7199 7200 if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK) && 7201 !kvm_cpu_cap_has(X86_FEATURE_IBT)) 7202 msrs->entry_ctls_high &= ~VM_ENTRY_LOAD_CET_STATE; 7203 7204 /* We support free control of debug control loading. */ 7205 msrs->entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS; 7206 } 7207 7208 static void nested_vmx_setup_cpubased_ctls(struct vmcs_config *vmcs_conf, 7209 struct nested_vmx_msrs *msrs) 7210 { 7211 msrs->procbased_ctls_low = 7212 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 7213 7214 msrs->procbased_ctls_high = vmcs_conf->cpu_based_exec_ctrl; 7215 msrs->procbased_ctls_high &= 7216 CPU_BASED_INTR_WINDOW_EXITING | 7217 CPU_BASED_NMI_WINDOW_EXITING | CPU_BASED_USE_TSC_OFFSETTING | 7218 CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING | 7219 CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING | 7220 CPU_BASED_CR3_STORE_EXITING | 7221 #ifdef CONFIG_X86_64 7222 CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING | 7223 #endif 7224 CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING | 7225 CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_TRAP_FLAG | 7226 CPU_BASED_MONITOR_EXITING | CPU_BASED_RDPMC_EXITING | 7227 CPU_BASED_RDTSC_EXITING | CPU_BASED_PAUSE_EXITING | 7228 CPU_BASED_TPR_SHADOW | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; 7229 /* 7230 * We can allow some features even when not supported by the 7231 * hardware. For example, L1 can specify an MSR bitmap - and we 7232 * can use it to avoid exits to L1 - even when L0 runs L2 7233 * without MSR bitmaps. 7234 */ 7235 msrs->procbased_ctls_high |= 7236 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR | 7237 CPU_BASED_USE_MSR_BITMAPS; 7238 7239 /* We support free control of CR3 access interception. */ 7240 msrs->procbased_ctls_low &= 7241 ~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING); 7242 } 7243 7244 static void nested_vmx_setup_secondary_ctls(u32 ept_caps, 7245 struct vmcs_config *vmcs_conf, 7246 struct nested_vmx_msrs *msrs) 7247 { 7248 msrs->secondary_ctls_low = 0; 7249 7250 msrs->secondary_ctls_high = vmcs_conf->cpu_based_2nd_exec_ctrl; 7251 msrs->secondary_ctls_high &= 7252 SECONDARY_EXEC_DESC | 7253 SECONDARY_EXEC_ENABLE_RDTSCP | 7254 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 7255 SECONDARY_EXEC_WBINVD_EXITING | 7256 SECONDARY_EXEC_APIC_REGISTER_VIRT | 7257 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 7258 SECONDARY_EXEC_RDRAND_EXITING | 7259 SECONDARY_EXEC_ENABLE_INVPCID | 7260 SECONDARY_EXEC_ENABLE_VMFUNC | 7261 SECONDARY_EXEC_RDSEED_EXITING | 7262 SECONDARY_EXEC_ENABLE_XSAVES | 7263 SECONDARY_EXEC_TSC_SCALING | 7264 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE; 7265 7266 /* 7267 * We can emulate "VMCS shadowing," even if the hardware 7268 * doesn't support it. 7269 */ 7270 msrs->secondary_ctls_high |= 7271 SECONDARY_EXEC_SHADOW_VMCS; 7272 7273 if (enable_ept) { 7274 /* nested EPT: emulate EPT also to L1 */ 7275 msrs->secondary_ctls_high |= 7276 SECONDARY_EXEC_ENABLE_EPT; 7277 msrs->ept_caps = 7278 VMX_EPT_PAGE_WALK_4_BIT | 7279 VMX_EPT_PAGE_WALK_5_BIT | 7280 VMX_EPTP_WB_BIT | 7281 VMX_EPT_INVEPT_BIT | 7282 VMX_EPT_EXECUTE_ONLY_BIT | 7283 VMX_EPT_ADVANCED_VMEXIT_INFO_BIT; 7284 7285 msrs->ept_caps &= ept_caps; 7286 msrs->ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT | 7287 VMX_EPT_EXTENT_CONTEXT_BIT | VMX_EPT_2MB_PAGE_BIT | 7288 VMX_EPT_1GB_PAGE_BIT; 7289 if (enable_ept_ad_bits) { 7290 msrs->secondary_ctls_high |= 7291 SECONDARY_EXEC_ENABLE_PML; 7292 msrs->ept_caps |= VMX_EPT_AD_BIT; 7293 } 7294 7295 if (enable_mbec) 7296 msrs->secondary_ctls_high |= 7297 SECONDARY_EXEC_MODE_BASED_EPT_EXEC; 7298 /* 7299 * Advertise EPTP switching irrespective of hardware support, 7300 * KVM emulates it in software so long as VMFUNC is supported. 7301 */ 7302 if (cpu_has_vmx_vmfunc()) 7303 msrs->vmfunc_controls = VMX_VMFUNC_EPTP_SWITCHING; 7304 } 7305 7306 /* 7307 * Old versions of KVM use the single-context version without 7308 * checking for support, so declare that it is supported even 7309 * though it is treated as global context. The alternative is 7310 * not failing the single-context invvpid, and it is worse. 7311 */ 7312 if (enable_vpid) { 7313 msrs->secondary_ctls_high |= 7314 SECONDARY_EXEC_ENABLE_VPID; 7315 msrs->vpid_caps = VMX_VPID_INVVPID_BIT | 7316 VMX_VPID_EXTENT_SUPPORTED_MASK; 7317 } 7318 7319 if (enable_unrestricted_guest) 7320 msrs->secondary_ctls_high |= 7321 SECONDARY_EXEC_UNRESTRICTED_GUEST; 7322 7323 if (flexpriority_enabled) 7324 msrs->secondary_ctls_high |= 7325 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 7326 7327 if (enable_sgx) 7328 msrs->secondary_ctls_high |= SECONDARY_EXEC_ENCLS_EXITING; 7329 } 7330 7331 static void nested_vmx_setup_misc_data(struct vmcs_config *vmcs_conf, 7332 struct nested_vmx_msrs *msrs) 7333 { 7334 msrs->misc_low = (u32)vmcs_conf->misc & VMX_MISC_SAVE_EFER_LMA; 7335 msrs->misc_low |= 7336 VMX_MISC_VMWRITE_SHADOW_RO_FIELDS | 7337 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE | 7338 VMX_MISC_ACTIVITY_HLT | 7339 VMX_MISC_ACTIVITY_WAIT_SIPI; 7340 msrs->misc_high = 0; 7341 } 7342 7343 static void nested_vmx_setup_basic(struct nested_vmx_msrs *msrs) 7344 { 7345 /* 7346 * This MSR reports some information about VMX support. We 7347 * should return information about the VMX we emulate for the 7348 * guest, and the VMCS structure we give it - not about the 7349 * VMX support of the underlying hardware. 7350 */ 7351 msrs->basic = vmx_basic_encode_vmcs_info(VMCS12_REVISION, VMCS12_SIZE, 7352 X86_MEMTYPE_WB); 7353 7354 msrs->basic |= VMX_BASIC_TRUE_CTLS; 7355 if (cpu_has_vmx_basic_inout()) 7356 msrs->basic |= VMX_BASIC_INOUT; 7357 if (cpu_has_vmx_basic_no_hw_errcode_cc()) 7358 msrs->basic |= VMX_BASIC_NO_HW_ERROR_CODE_CC; 7359 } 7360 7361 static void nested_vmx_setup_cr_fixed(struct nested_vmx_msrs *msrs) 7362 { 7363 /* 7364 * These MSRs specify bits which the guest must keep fixed on 7365 * while L1 is in VMXON mode (in L1's root mode, or running an L2). 7366 * We picked the standard core2 setting. 7367 */ 7368 #define VMXON_CR0_ALWAYSON (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE) 7369 #define VMXON_CR4_ALWAYSON X86_CR4_VMXE 7370 msrs->cr0_fixed0 = VMXON_CR0_ALWAYSON; 7371 msrs->cr4_fixed0 = VMXON_CR4_ALWAYSON; 7372 7373 /* These MSRs specify bits which the guest must keep fixed off. */ 7374 rdmsrq(MSR_IA32_VMX_CR0_FIXED1, msrs->cr0_fixed1); 7375 rdmsrq(MSR_IA32_VMX_CR4_FIXED1, msrs->cr4_fixed1); 7376 7377 if (vmx_umip_emulated()) 7378 msrs->cr4_fixed1 |= X86_CR4_UMIP; 7379 } 7380 7381 /* 7382 * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be 7383 * returned for the various VMX controls MSRs when nested VMX is enabled. 7384 * The same values should also be used to verify that vmcs12 control fields are 7385 * valid during nested entry from L1 to L2. 7386 * Each of these control msrs has a low and high 32-bit half: A low bit is on 7387 * if the corresponding bit in the (32-bit) control field *must* be on, and a 7388 * bit in the high half is on if the corresponding bit in the control field 7389 * may be on. See also vmx_control_verify(). 7390 */ 7391 void nested_vmx_setup_ctls_msrs(struct vmcs_config *vmcs_conf, u32 ept_caps) 7392 { 7393 struct nested_vmx_msrs *msrs = &vmcs_conf->nested; 7394 7395 /* 7396 * Note that as a general rule, the high half of the MSRs (bits in 7397 * the control fields which may be 1) should be initialized by the 7398 * intersection of the underlying hardware's MSR (i.e., features which 7399 * can be supported) and the list of features we want to expose - 7400 * because they are known to be properly supported in our code. 7401 * Also, usually, the low half of the MSRs (bits which must be 1) can 7402 * be set to 0, meaning that L1 may turn off any of these bits. The 7403 * reason is that if one of these bits is necessary, it will appear 7404 * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control 7405 * fields of vmcs01 and vmcs02, will turn these bits off - and 7406 * nested_vmx_l1_wants_exit() will not pass related exits to L1. 7407 * These rules have exceptions below. 7408 */ 7409 nested_vmx_setup_pinbased_ctls(vmcs_conf, msrs); 7410 7411 nested_vmx_setup_exit_ctls(vmcs_conf, msrs); 7412 7413 nested_vmx_setup_entry_ctls(vmcs_conf, msrs); 7414 7415 nested_vmx_setup_cpubased_ctls(vmcs_conf, msrs); 7416 7417 nested_vmx_setup_secondary_ctls(ept_caps, vmcs_conf, msrs); 7418 7419 nested_vmx_setup_misc_data(vmcs_conf, msrs); 7420 7421 nested_vmx_setup_basic(msrs); 7422 7423 nested_vmx_setup_cr_fixed(msrs); 7424 7425 msrs->vmcs_enum = nested_vmx_calc_vmcs_enum_msr(); 7426 } 7427 7428 void nested_vmx_hardware_unsetup(void) 7429 { 7430 int i; 7431 7432 if (enable_shadow_vmcs) { 7433 for (i = 0; i < VMX_BITMAP_NR; i++) 7434 free_page((unsigned long)vmx_bitmap[i]); 7435 } 7436 } 7437 7438 __init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *)) 7439 { 7440 int i; 7441 7442 /* 7443 * Note! The set of supported vmcs12 fields is consumed by both VMX 7444 * MSR and shadow VMCS setup. 7445 */ 7446 nested_vmx_setup_vmcs12_fields(); 7447 7448 nested_vmx_setup_ctls_msrs(&vmcs_config, vmx_capability.ept); 7449 7450 if (!cpu_has_vmx_shadow_vmcs()) 7451 enable_shadow_vmcs = 0; 7452 if (enable_shadow_vmcs) { 7453 for (i = 0; i < VMX_BITMAP_NR; i++) { 7454 /* 7455 * The vmx_bitmap is not tied to a VM and so should 7456 * not be charged to a memcg. 7457 */ 7458 vmx_bitmap[i] = (unsigned long *) 7459 __get_free_page(GFP_KERNEL); 7460 if (!vmx_bitmap[i]) { 7461 nested_vmx_hardware_unsetup(); 7462 return -ENOMEM; 7463 } 7464 } 7465 7466 init_vmcs_shadow_fields(); 7467 } 7468 7469 exit_handlers[EXIT_REASON_VMCLEAR] = handle_vmclear; 7470 exit_handlers[EXIT_REASON_VMLAUNCH] = handle_vmlaunch; 7471 exit_handlers[EXIT_REASON_VMPTRLD] = handle_vmptrld; 7472 exit_handlers[EXIT_REASON_VMPTRST] = handle_vmptrst; 7473 exit_handlers[EXIT_REASON_VMREAD] = handle_vmread; 7474 exit_handlers[EXIT_REASON_VMRESUME] = handle_vmresume; 7475 exit_handlers[EXIT_REASON_VMWRITE] = handle_vmwrite; 7476 exit_handlers[EXIT_REASON_VMOFF] = handle_vmxoff; 7477 exit_handlers[EXIT_REASON_VMON] = handle_vmxon; 7478 exit_handlers[EXIT_REASON_INVEPT] = handle_invept; 7479 exit_handlers[EXIT_REASON_INVVPID] = handle_invvpid; 7480 exit_handlers[EXIT_REASON_VMFUNC] = handle_vmfunc; 7481 7482 return 0; 7483 } 7484 7485 7486 static gpa_t vmx_translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, 7487 u64 access, 7488 struct x86_exception *exception, 7489 u64 pte_access) 7490 { 7491 struct kvm_pagewalk *w = &vcpu->arch.ngpa_walk; 7492 7493 if (WARN_ON_ONCE(!mmu_is_nested(vcpu))) 7494 return gpa; 7495 7496 /* 7497 * MBEC differentiates based on the effective U/S bit of 7498 * the guest page tables; not the processor CPL. 7499 */ 7500 access &= ~PFERR_USER_MASK; 7501 if ((pte_access & ACC_USER_MASK) && (access & PFERR_GUEST_FINAL_MASK)) 7502 access |= PFERR_USER_MASK; 7503 7504 return w->gva_to_gpa(vcpu, w, gpa, access, exception); 7505 } 7506 7507 struct kvm_x86_nested_ops vmx_nested_ops __initdata = { 7508 .leave_nested = vmx_leave_nested, 7509 .translate_nested_gpa = vmx_translate_nested_gpa, 7510 .is_exception_vmexit = nested_vmx_is_exception_vmexit, 7511 .check_events = vmx_check_nested_events, 7512 .has_events = vmx_has_nested_events, 7513 .triple_fault = nested_vmx_triple_fault, 7514 .get_state = vmx_get_nested_state, 7515 .set_state = vmx_set_nested_state, 7516 .get_nested_state_pages = vmx_get_nested_state_pages, 7517 .write_log_dirty = nested_vmx_write_pml_buffer, 7518 #ifdef CONFIG_KVM_HYPERV 7519 .enable_evmcs = nested_enable_evmcs, 7520 .get_evmcs_version = nested_get_evmcs_version, 7521 .hv_inject_synthetic_vmexit_post_tlb_flush = vmx_hv_inject_synthetic_vmexit_post_tlb_flush, 7522 #endif 7523 }; 7524