1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * AMD SVM support 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Yaniv Kamay <yaniv@qumranet.com> 12 * Avi Kivity <avi@qumranet.com> 13 */ 14 15 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 16 17 #include <linux/kvm_types.h> 18 #include <linux/kvm_host.h> 19 #include <linux/kernel.h> 20 21 #include <asm/msr-index.h> 22 #include <asm/debugreg.h> 23 24 #include "kvm_emulate.h" 25 #include "trace.h" 26 #include "irq.h" 27 #include "mmu.h" 28 #include "x86.h" 29 #include "smm.h" 30 #include "cpuid.h" 31 #include "lapic.h" 32 #include "svm.h" 33 #include "hyperv.h" 34 #include "pmu.h" 35 36 #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK 37 38 static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu, 39 struct x86_exception *fault, 40 bool from_hardware) 41 { 42 struct vcpu_svm *svm = to_svm(vcpu); 43 struct vmcb *vmcb = svm->vmcb; 44 u64 fault_stage; 45 46 /* 47 * For hardware NPF exits, the GUEST_FAULT_STAGE bits are only 48 * available in the hardware exit_info_1, since the guest_mmu 49 * walker doesn't know whether the faulting GPA was a page table 50 * page or final page from L2's perspective. 51 */ 52 if (from_hardware) 53 fault_stage = vmcb->control.exit_info_1 & 54 PFERR_GUEST_FAULT_STAGE_MASK; 55 else 56 fault_stage = fault->error_code & PFERR_GUEST_FAULT_STAGE_MASK; 57 58 /* 59 * All nested page faults should be annotated as occurring on the 60 * final translation *or* the page walk. Arbitrarily choose "final" 61 * if KVM is buggy and enumerated both or neither. 62 */ 63 if (WARN_ON_ONCE(hweight64(fault_stage) != 1)) 64 fault_stage = PFERR_GUEST_FINAL_MASK; 65 66 vmcb->control.exit_code = SVM_EXIT_NPF; 67 vmcb->control.exit_info_1 = fault_stage | 68 (fault->error_code & ~PFERR_GUEST_FAULT_STAGE_MASK); 69 vmcb->control.exit_info_2 = fault->address; 70 71 nested_svm_vmexit(svm); 72 } 73 74 static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index) 75 { 76 struct vcpu_svm *svm = to_svm(vcpu); 77 u64 cr3 = svm->nested.ctl.nested_cr3; 78 u64 pdpte; 79 int ret; 80 81 /* 82 * Note, nCR3 is "assumed" to be 32-byte aligned, i.e. the CPU ignores 83 * nCR3[4:0] when loading PDPTEs from memory. 84 */ 85 ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(cr3), &pdpte, 86 (cr3 & GENMASK(11, 5)) + index * 8, 8); 87 if (ret) 88 return 0; 89 return pdpte; 90 } 91 92 static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu) 93 { 94 struct vcpu_svm *svm = to_svm(vcpu); 95 96 return svm->nested.ctl.nested_cr3; 97 } 98 99 static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu) 100 { 101 struct vcpu_svm *svm = to_svm(vcpu); 102 103 WARN_ON(mmu_is_nested(vcpu)); 104 105 vcpu->arch.mmu = &vcpu->arch.guest_mmu; 106 107 /* 108 * The NPT format depends on L1's CR4 and EFER, which is in vmcb01. Note, 109 * when called via KVM_SET_NESTED_STATE, that state may _not_ match current 110 * vCPU state. CR0.WP is explicitly ignored, while CR0.PG is required. 111 */ 112 kvm_init_shadow_npt_mmu(vcpu, svm->vmcb01.ptr->save.cr4, 113 svm->vmcb01.ptr->save.efer, 114 svm->nested.ctl.nested_cr3, 115 svm->nested.ctl.misc_ctl); 116 117 vcpu->arch.ngpa_walk.get_guest_pgd = nested_svm_get_tdp_cr3; 118 vcpu->arch.ngpa_walk.get_pdptr = nested_svm_get_tdp_pdptr; 119 vcpu->arch.ngpa_walk.inject_page_fault = nested_svm_inject_npf_exit; 120 } 121 122 static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu) 123 { 124 vcpu->arch.mmu = &vcpu->arch.root_mmu; 125 } 126 127 static bool nested_vmcb_needs_vls_intercept(struct vcpu_svm *svm) 128 { 129 if (!guest_cpu_cap_has(&svm->vcpu, X86_FEATURE_V_VMSAVE_VMLOAD)) 130 return true; 131 132 if (!nested_npt_enabled(svm)) 133 return true; 134 135 if (!(svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE)) 136 return true; 137 138 return false; 139 } 140 141 void nested_vmcb02_recalc_intercepts(struct vcpu_svm *svm) 142 { 143 struct vmcb_ctrl_area_cached *vmcb12_ctrl = &svm->nested.ctl; 144 struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; 145 struct vmcb *vmcb01 = svm->vmcb01.ptr; 146 unsigned int i; 147 148 if (WARN_ON_ONCE(svm->vmcb != vmcb02)) 149 return; 150 151 vmcb_mark_dirty(vmcb02, VMCB_INTERCEPTS); 152 153 for (i = 0; i < MAX_INTERCEPT; i++) 154 vmcb02->control.intercepts[i] = vmcb01->control.intercepts[i]; 155 156 if (vmcb12_ctrl->int_ctl & V_INTR_MASKING_MASK) { 157 /* 158 * If L2 is active and V_INTR_MASKING is enabled in vmcb12, 159 * disable intercept of CR8 writes as L2's CR8 does not affect 160 * any interrupt KVM may want to inject. 161 * 162 * Similarly, disable intercept of virtual interrupts (used to 163 * detect interrupt windows) if the saved RFLAGS.IF is '0', as 164 * the effective RFLAGS.IF for L1 interrupts will never be set 165 * while L2 is running (L2's RFLAGS.IF doesn't affect L1 IRQs). 166 */ 167 vmcb_clr_intercept(&vmcb02->control, INTERCEPT_CR8_WRITE); 168 if (!(vmcb01->save.rflags & X86_EFLAGS_IF)) 169 vmcb_clr_intercept(&vmcb02->control, INTERCEPT_VINTR); 170 } 171 172 for (i = 0; i < MAX_INTERCEPT; i++) 173 vmcb02->control.intercepts[i] |= vmcb12_ctrl->intercepts[i]; 174 175 /* If SMI is not intercepted, ignore guest SMI intercept as well */ 176 if (!intercept_smi) 177 vmcb_clr_intercept(&vmcb02->control, INTERCEPT_SMI); 178 179 /* 180 * Intercept PAUSE if and only if L1 wants to. KVM intercepts PAUSE so 181 * that a vCPU that may be spinning waiting for a lock can be scheduled 182 * out in favor of the vCPU that holds said lock. KVM doesn't support 183 * yielding across L2 vCPUs, as KVM has limited visilibity into which 184 * L2 vCPUs are in the same L2 VM, i.e. may be contending for locks. 185 */ 186 if (!vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_PAUSE)) 187 vmcb_clr_intercept(&vmcb02->control, INTERCEPT_PAUSE); 188 189 if (nested_vmcb_needs_vls_intercept(svm)) { 190 /* 191 * If the virtual VMLOAD/VMSAVE is not enabled for the L2, 192 * we must intercept these instructions to correctly 193 * emulate them in case L1 doesn't intercept them. 194 */ 195 vmcb_set_intercept(&vmcb02->control, INTERCEPT_VMLOAD); 196 vmcb_set_intercept(&vmcb02->control, INTERCEPT_VMSAVE); 197 } else { 198 WARN_ON_ONCE(!(vmcb02->control.misc_ctl2 & SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE)); 199 } 200 } 201 202 /* 203 * This array (and its actual size) holds the set of offsets (indexing by chunk 204 * size) to process when merging vmcb12's MSRPM with vmcb01's MSRPM. Note, the 205 * set of MSRs for which interception is disabled in vmcb01 is per-vCPU, e.g. 206 * based on CPUID features. This array only tracks MSRs that *might* be passed 207 * through to the guest. 208 * 209 * Hardcode the capacity of the array based on the maximum number of _offsets_. 210 * MSRs are batched together, so there are fewer offsets than MSRs. 211 */ 212 static int nested_svm_msrpm_merge_offsets[10] __ro_after_init; 213 static int nested_svm_nr_msrpm_merge_offsets __ro_after_init; 214 typedef unsigned long nsvm_msrpm_merge_t; 215 216 int __init nested_svm_init_msrpm_merge_offsets(void) 217 { 218 static const u32 merge_msrs[] __initconst = { 219 MSR_STAR, 220 MSR_IA32_SYSENTER_CS, 221 MSR_IA32_SYSENTER_EIP, 222 MSR_IA32_SYSENTER_ESP, 223 #ifdef CONFIG_X86_64 224 MSR_GS_BASE, 225 MSR_FS_BASE, 226 MSR_KERNEL_GS_BASE, 227 MSR_LSTAR, 228 MSR_CSTAR, 229 MSR_SYSCALL_MASK, 230 #endif 231 MSR_IA32_SPEC_CTRL, 232 MSR_IA32_PRED_CMD, 233 MSR_IA32_FLUSH_CMD, 234 MSR_IA32_APERF, 235 MSR_IA32_MPERF, 236 MSR_IA32_LASTBRANCHFROMIP, 237 MSR_IA32_LASTBRANCHTOIP, 238 MSR_IA32_LASTINTFROMIP, 239 MSR_IA32_LASTINTTOIP, 240 241 MSR_K7_PERFCTR0, 242 MSR_K7_PERFCTR1, 243 MSR_K7_PERFCTR2, 244 MSR_K7_PERFCTR3, 245 MSR_F15H_PERF_CTR0, 246 MSR_F15H_PERF_CTR1, 247 MSR_F15H_PERF_CTR2, 248 MSR_F15H_PERF_CTR3, 249 MSR_F15H_PERF_CTR4, 250 MSR_F15H_PERF_CTR5, 251 252 MSR_AMD64_PERF_CNTR_GLOBAL_CTL, 253 MSR_AMD64_PERF_CNTR_GLOBAL_STATUS, 254 MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR, 255 MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET, 256 }; 257 int i, j; 258 259 for (i = 0; i < ARRAY_SIZE(merge_msrs); i++) { 260 int bit_nr = svm_msrpm_bit_nr(merge_msrs[i]); 261 u32 offset; 262 263 if (WARN_ON(bit_nr < 0)) 264 return -EIO; 265 266 /* 267 * Merging is done in chunks to reduce the number of accesses 268 * to L1's bitmap. 269 */ 270 offset = bit_nr / BITS_PER_BYTE / sizeof(nsvm_msrpm_merge_t); 271 272 for (j = 0; j < nested_svm_nr_msrpm_merge_offsets; j++) { 273 if (nested_svm_msrpm_merge_offsets[j] == offset) 274 break; 275 } 276 277 if (j < nested_svm_nr_msrpm_merge_offsets) 278 continue; 279 280 if (WARN_ON(j >= ARRAY_SIZE(nested_svm_msrpm_merge_offsets))) 281 return -EIO; 282 283 nested_svm_msrpm_merge_offsets[j] = offset; 284 nested_svm_nr_msrpm_merge_offsets++; 285 } 286 287 return 0; 288 } 289 290 /* 291 * Merge L0's (KVM) and L1's (Nested VMCB) MSR permission bitmaps. The function 292 * is optimized in that it only merges the parts where KVM MSR permission bitmap 293 * may contain zero bits. 294 */ 295 static bool nested_svm_merge_msrpm(struct kvm_vcpu *vcpu) 296 { 297 struct vcpu_svm *svm = to_svm(vcpu); 298 nsvm_msrpm_merge_t *msrpm02 = svm->nested.msrpm; 299 nsvm_msrpm_merge_t *msrpm01 = svm->msrpm; 300 int i; 301 302 /* 303 * MSR bitmap update can be skipped when: 304 * - MSR bitmap for L1 hasn't changed. 305 * - Nested hypervisor (L1) is attempting to launch the same L2 as 306 * before. 307 * - Nested hypervisor (L1) is using Hyper-V emulation interface and 308 * tells KVM (L0) there were no changes in MSR bitmap for L2. 309 */ 310 #ifdef CONFIG_KVM_HYPERV 311 if (!svm->nested.force_msr_bitmap_recalc) { 312 struct hv_vmcb_enlightenments *hve = &svm->nested.ctl.hv_enlightenments; 313 314 if (kvm_hv_hypercall_enabled(vcpu) && 315 hve->hv_enlightenments_control.msr_bitmap && 316 (svm->nested.ctl.clean & BIT(HV_VMCB_NESTED_ENLIGHTENMENTS))) 317 goto set_msrpm_base_pa; 318 } 319 #endif 320 321 if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT))) 322 return true; 323 324 for (i = 0; i < nested_svm_nr_msrpm_merge_offsets; i++) { 325 const int p = nested_svm_msrpm_merge_offsets[i]; 326 nsvm_msrpm_merge_t l1_val; 327 gpa_t gpa; 328 329 gpa = svm->nested.ctl.msrpm_base_pa + (p * sizeof(l1_val)); 330 331 if (kvm_vcpu_read_guest(vcpu, gpa, &l1_val, sizeof(l1_val))) 332 return false; 333 334 msrpm02[p] = msrpm01[p] | l1_val; 335 } 336 337 svm->nested.force_msr_bitmap_recalc = false; 338 339 #ifdef CONFIG_KVM_HYPERV 340 set_msrpm_base_pa: 341 #endif 342 svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm)); 343 344 return true; 345 } 346 347 /* 348 * Bits 11:0 of bitmap address are ignored by hardware 349 */ 350 static bool nested_svm_check_bitmap_pa(struct kvm_vcpu *vcpu, u64 pa, u32 size) 351 { 352 u64 addr = PAGE_ALIGN(pa); 353 354 return kvm_vcpu_is_legal_gpa(vcpu, addr) && 355 kvm_vcpu_is_legal_gpa(vcpu, addr + size - 1); 356 } 357 358 static bool nested_svm_event_inj_valid_exept(struct kvm_vcpu *vcpu, u8 vector) 359 { 360 /* 361 * Vectors that do not correspond to a defined exception are invalid 362 * (including #NMI and reserved vectors). In a best effort to define 363 * valid exceptions based on the virtual CPU, make all exceptions always 364 * valid except those obviously tied to a CPU feature. 365 */ 366 switch (vector) { 367 case DE_VECTOR: case DB_VECTOR: case BP_VECTOR: case OF_VECTOR: 368 case BR_VECTOR: case UD_VECTOR: case NM_VECTOR: case DF_VECTOR: 369 case TS_VECTOR: case NP_VECTOR: case SS_VECTOR: case GP_VECTOR: 370 case PF_VECTOR: case MF_VECTOR: case AC_VECTOR: case MC_VECTOR: 371 case XM_VECTOR: case HV_VECTOR: case SX_VECTOR: 372 return true; 373 case CP_VECTOR: 374 return guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK); 375 case VC_VECTOR: 376 return guest_cpu_cap_has(vcpu, X86_FEATURE_SEV_ES); 377 } 378 return false; 379 } 380 381 /* 382 * According to the APM, VMRUN exits with SVM_EXIT_ERR if SVM_EVTINJ_VALID is 383 * set and: 384 * - The type of event_inj is not one of the defined values. 385 * - The type is SVM_EVTINJ_TYPE_EXEPT, but the vector is not a valid exception. 386 */ 387 static bool nested_svm_check_event_inj(struct kvm_vcpu *vcpu, u32 event_inj) 388 { 389 u32 type = event_inj & SVM_EVTINJ_TYPE_MASK; 390 u8 vector = event_inj & SVM_EVTINJ_VEC_MASK; 391 392 if (!(event_inj & SVM_EVTINJ_VALID)) 393 return true; 394 395 if (type != SVM_EVTINJ_TYPE_INTR && type != SVM_EVTINJ_TYPE_NMI && 396 type != SVM_EVTINJ_TYPE_EXEPT && type != SVM_EVTINJ_TYPE_SOFT) 397 return false; 398 399 if (type == SVM_EVTINJ_TYPE_EXEPT && 400 !nested_svm_event_inj_valid_exept(vcpu, vector)) 401 return false; 402 403 return true; 404 } 405 406 static bool nested_vmcb_check_controls(struct kvm_vcpu *vcpu, 407 struct vmcb_ctrl_area_cached *control) 408 { 409 if (CC(!vmcb12_is_intercept(control, INTERCEPT_VMRUN))) 410 return false; 411 412 if (CC(control->asid == 0)) 413 return false; 414 415 if (CC((control->misc_ctl & SVM_MISC_ENABLE_NP) && 416 !kvm_vcpu_is_legal_gpa(vcpu, control->nested_cr3))) 417 return false; 418 419 if (CC(!nested_svm_check_bitmap_pa(vcpu, control->msrpm_base_pa, 420 MSRPM_SIZE))) 421 return false; 422 if (CC(!nested_svm_check_bitmap_pa(vcpu, control->iopm_base_pa, 423 IOPM_SIZE))) 424 return false; 425 426 if (CC((control->int_ctl & V_NMI_ENABLE_MASK) && 427 !vmcb12_is_intercept(control, INTERCEPT_NMI))) { 428 return false; 429 } 430 431 if (CC(!nested_svm_check_event_inj(vcpu, control->event_inj))) 432 return false; 433 434 return true; 435 } 436 437 /* Common checks that apply to both L1 and L2 state. */ 438 static bool nested_vmcb_check_save(struct kvm_vcpu *vcpu, 439 struct vmcb_save_area_cached *save, 440 bool check_gpat) 441 { 442 if (CC(!(save->efer & EFER_SVME))) 443 return false; 444 445 if (CC((save->cr0 & X86_CR0_CD) == 0 && (save->cr0 & X86_CR0_NW)) || 446 CC(save->cr0 & ~0xffffffffULL)) 447 return false; 448 449 if (CC(!kvm_dr6_valid(save->dr6)) || CC(!kvm_dr7_valid(save->dr7))) 450 return false; 451 452 /* 453 * These checks are also performed by KVM_SET_SREGS, 454 * except that EFER.LMA is not checked by SVM against 455 * CR0.PG && EFER.LME. 456 */ 457 if ((save->efer & EFER_LME) && (save->cr0 & X86_CR0_PG)) { 458 if (CC(!(save->cr4 & X86_CR4_PAE)) || 459 CC(!(save->cr0 & X86_CR0_PE)) || 460 CC(!kvm_vcpu_is_legal_cr3(vcpu, save->cr3))) 461 return false; 462 463 if (CC((save->cs.attrib & SVM_SELECTOR_L_MASK) && 464 (save->cs.attrib & SVM_SELECTOR_DB_MASK))) 465 return false; 466 } 467 468 /* Note, SVM doesn't have any additional restrictions on CR4. */ 469 if (CC(!__kvm_is_valid_cr4(vcpu, save->cr4))) 470 return false; 471 472 if (CC(!kvm_valid_efer(vcpu, save->efer))) 473 return false; 474 475 /* 476 * If userspace contrives to get an invalid g_pat into vmcb02 by 477 * disabling KVM_X86_QUIRK_NESTED_SVM_SHARED_PAT in a race with 478 * this check, it should be prepared for the KVM_EXIT_FAIL_ENTRY 479 * that will follow. 480 */ 481 if (check_gpat && CC(!kvm_pat_valid(save->g_pat))) 482 return false; 483 484 return true; 485 } 486 487 int nested_svm_check_cached_vmcb12(struct kvm_vcpu *vcpu) 488 { 489 struct vcpu_svm *svm = to_svm(vcpu); 490 491 if (!nested_vmcb_check_save(vcpu, &svm->nested.save, 492 l2_has_separate_pat(vcpu)) || 493 !nested_vmcb_check_controls(vcpu, &svm->nested.ctl)) 494 return -EINVAL; 495 496 return 0; 497 } 498 499 /* 500 * If a feature is not advertised to L1, clear the corresponding vmcb12 501 * intercept. 502 */ 503 #define __nested_svm_sanitize_intercept(__vcpu, __control, fname, iname) \ 504 do { \ 505 if (!guest_cpu_cap_has(__vcpu, X86_FEATURE_##fname)) \ 506 vmcb12_clr_intercept(__control, INTERCEPT_##iname); \ 507 } while (0) 508 509 #define nested_svm_sanitize_intercept(__vcpu, __control, name) \ 510 __nested_svm_sanitize_intercept(__vcpu, __control, name, name) 511 512 static 513 void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu, 514 struct vmcb_ctrl_area_cached *to, 515 struct vmcb_control_area *from) 516 { 517 unsigned int i; 518 519 for (i = 0; i < MAX_INTERCEPT; i++) 520 to->intercepts[i] = from->intercepts[i]; 521 522 __nested_svm_sanitize_intercept(vcpu, to, XSAVE, XSETBV); 523 nested_svm_sanitize_intercept(vcpu, to, INVPCID); 524 nested_svm_sanitize_intercept(vcpu, to, RDTSCP); 525 nested_svm_sanitize_intercept(vcpu, to, SKINIT); 526 nested_svm_sanitize_intercept(vcpu, to, RDPRU); 527 528 /* Always clear misc_ctl bits that the guest cannot use */ 529 to->misc_ctl = from->misc_ctl; 530 to->misc_ctl2 = from->misc_ctl2; 531 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_NPT)) 532 to->misc_ctl &= ~SVM_MISC_ENABLE_NP; 533 534 if (!gmet_enabled || !guest_cpu_cap_has(vcpu, X86_FEATURE_GMET)) 535 to->misc_ctl &= ~SVM_MISC_ENABLE_GMET; 536 537 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV)) 538 to->misc_ctl2 &= ~SVM_MISC2_ENABLE_V_LBR; 539 540 to->iopm_base_pa = from->iopm_base_pa & PAGE_MASK; 541 to->msrpm_base_pa = from->msrpm_base_pa & PAGE_MASK; 542 to->tsc_offset = from->tsc_offset; 543 to->tlb_ctl = from->tlb_ctl & TLB_CONTROL_MASK; 544 to->erap_ctl = from->erap_ctl; 545 to->int_ctl = from->int_ctl; 546 to->int_vector = from->int_vector & SVM_INT_VECTOR_MASK; 547 to->int_state = from->int_state & SVM_INTERRUPT_SHADOW_MASK; 548 to->exit_code = from->exit_code; 549 to->exit_info_1 = from->exit_info_1; 550 to->exit_info_2 = from->exit_info_2; 551 to->exit_int_info = from->exit_int_info; 552 to->exit_int_info_err = from->exit_int_info_err; 553 to->event_inj = from->event_inj & ~SVM_EVTINJ_RESERVED_BITS; 554 to->event_inj_err = from->event_inj_err; 555 to->next_rip = from->next_rip; 556 to->nested_cr3 = from->nested_cr3; 557 to->pause_filter_count = from->pause_filter_count; 558 to->pause_filter_thresh = from->pause_filter_thresh; 559 560 /* Copy asid here because nested_vmcb_check_controls() will check it */ 561 to->asid = from->asid; 562 to->clean = from->clean; 563 564 #ifdef CONFIG_KVM_HYPERV 565 /* Hyper-V extensions (Enlightened VMCB) */ 566 if (kvm_hv_hypercall_enabled(vcpu)) { 567 memcpy(&to->hv_enlightenments, &from->hv_enlightenments, 568 sizeof(to->hv_enlightenments)); 569 } 570 #endif 571 } 572 573 void nested_copy_vmcb_control_to_cache(struct vcpu_svm *svm, 574 struct vmcb_control_area *control) 575 { 576 __nested_copy_vmcb_control_to_cache(&svm->vcpu, &svm->nested.ctl, control); 577 } 578 579 static void __nested_copy_vmcb_save_to_cache(struct vmcb_save_area_cached *to, 580 struct vmcb_save_area *from) 581 { 582 to->es = from->es; 583 to->cs = from->cs; 584 to->ss = from->ss; 585 to->ds = from->ds; 586 to->gdtr = from->gdtr; 587 to->idtr = from->idtr; 588 589 to->cpl = from->cpl; 590 591 to->efer = from->efer; 592 to->cr4 = from->cr4; 593 to->cr3 = from->cr3; 594 to->cr0 = from->cr0; 595 to->dr7 = from->dr7; 596 to->dr6 = from->dr6; 597 598 to->rflags = from->rflags; 599 to->rip = from->rip; 600 to->rsp = from->rsp; 601 602 to->s_cet = from->s_cet; 603 to->ssp = from->ssp; 604 to->isst_addr = from->isst_addr; 605 606 to->rax = from->rax; 607 to->cr2 = from->cr2; 608 to->g_pat = from->g_pat; 609 610 svm_copy_lbrs(to, from); 611 } 612 613 void nested_copy_vmcb_save_to_cache(struct vcpu_svm *svm, 614 struct vmcb_save_area *save) 615 { 616 __nested_copy_vmcb_save_to_cache(&svm->nested.save, save); 617 } 618 619 /* 620 * Synchronize fields that are written by the processor, so that 621 * they can be copied back into the vmcb12. 622 */ 623 void nested_sync_control_from_vmcb02(struct vcpu_svm *svm) 624 { 625 u32 mask; 626 svm->nested.ctl.event_inj = svm->vmcb->control.event_inj; 627 svm->nested.ctl.event_inj_err = svm->vmcb->control.event_inj_err; 628 svm->nested.ctl.int_state = svm->vmcb->control.int_state; 629 630 /* Only a few fields of int_ctl are written by the processor. */ 631 mask = V_IRQ_MASK | V_TPR_MASK; 632 /* 633 * Don't sync vmcb02 V_IRQ back to vmcb12 if KVM (L0) is intercepting 634 * virtual interrupts in order to request an interrupt window, as KVM 635 * has usurped vmcb02's int_ctl. If an interrupt window opens before 636 * the next VM-Exit, svm_clear_vintr() will restore vmcb12's int_ctl. 637 * If no window opens, V_IRQ will be correctly preserved in vmcb12's 638 * int_ctl (because it was never recognized while L2 was running). 639 */ 640 if (svm_is_intercept(svm, INTERCEPT_VINTR) && 641 !vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_VINTR)) 642 mask &= ~V_IRQ_MASK; 643 644 if (nested_vgif_enabled(svm)) 645 mask |= V_GIF_MASK; 646 647 if (nested_vnmi_enabled(svm)) 648 mask |= V_NMI_BLOCKING_MASK | V_NMI_PENDING_MASK; 649 650 svm->nested.ctl.int_ctl &= ~mask; 651 svm->nested.ctl.int_ctl |= svm->vmcb->control.int_ctl & mask; 652 } 653 654 /* 655 * Transfer any event that L0 or L1 wanted to inject into L2 to 656 * EXIT_INT_INFO. 657 */ 658 static void nested_save_pending_event_to_vmcb12(struct vcpu_svm *svm, 659 struct vmcb *vmcb12) 660 { 661 struct kvm_vcpu *vcpu = &svm->vcpu; 662 u32 exit_int_info = 0; 663 unsigned int nr; 664 665 if (vcpu->arch.exception.injected) { 666 nr = vcpu->arch.exception.vector; 667 exit_int_info = nr | SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_EXEPT; 668 669 if (vcpu->arch.exception.has_error_code) { 670 exit_int_info |= SVM_EVTINJ_VALID_ERR; 671 vmcb12->control.exit_int_info_err = 672 vcpu->arch.exception.error_code; 673 } 674 675 } else if (vcpu->arch.nmi_injected) { 676 exit_int_info = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI; 677 678 } else if (vcpu->arch.interrupt.injected) { 679 nr = vcpu->arch.interrupt.nr; 680 exit_int_info = nr | SVM_EVTINJ_VALID; 681 682 if (vcpu->arch.interrupt.soft) 683 exit_int_info |= SVM_EVTINJ_TYPE_SOFT; 684 else 685 exit_int_info |= SVM_EVTINJ_TYPE_INTR; 686 } 687 688 vmcb12->control.exit_int_info = exit_int_info; 689 } 690 691 static void nested_svm_transition_tlb_flush(struct kvm_vcpu *vcpu) 692 { 693 /* Handle pending Hyper-V TLB flush requests */ 694 kvm_hv_nested_transtion_tlb_flush(vcpu, npt_enabled); 695 696 /* 697 * TODO: optimize unconditional TLB flush/MMU sync. A partial list of 698 * things to fix before this can be conditional: 699 * 700 * - Flush TLBs for both L1 and L2 remote TLB flush 701 * - Honor L1's request to flush an ASID on nested VMRUN 702 * - Sync nested NPT MMU on VMRUN that flushes L2's ASID[*] 703 * - Don't crush a pending TLB flush in vmcb02 on nested VMRUN 704 * - Flush L1's ASID on KVM_REQ_TLB_FLUSH_GUEST 705 * 706 * [*] Unlike nested EPT, SVM's ASID management can invalidate nested 707 * NPT guest-physical mappings on VMRUN. 708 */ 709 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); 710 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 711 } 712 713 /* 714 * Load guest's/host's cr3 on nested vmentry or vmexit. @nested_npt is true 715 * if we are emulating VM-Entry into a guest with NPT enabled. 716 */ 717 static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, 718 bool nested_npt, bool reload_pdptrs) 719 { 720 if (CC(!kvm_vcpu_is_legal_cr3(vcpu, cr3))) 721 return -EINVAL; 722 723 if (reload_pdptrs && is_pae_paging(vcpu)) { 724 if (nested_npt) 725 kvm_register_mark_for_reload(vcpu, VCPU_REG_PDPTR); 726 else if (CC(!load_pdptrs(vcpu, cr3))) 727 return -EINVAL; 728 } 729 730 vcpu->arch.cr3 = cr3; 731 732 /* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */ 733 kvm_init_mmu(vcpu); 734 735 if (!nested_npt) 736 kvm_mmu_new_pgd(vcpu, cr3); 737 738 return 0; 739 } 740 741 static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) 742 { 743 struct vmcb_ctrl_area_cached *control = &svm->nested.ctl; 744 struct vmcb_save_area_cached *save = &svm->nested.save; 745 bool new_vmcb12 = false; 746 struct vmcb *vmcb01 = svm->vmcb01.ptr; 747 struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; 748 struct kvm_vcpu *vcpu = &svm->vcpu; 749 750 /* Load the nested guest state */ 751 if (svm->nested.vmcb12_gpa != svm->nested.last_vmcb12_gpa) { 752 new_vmcb12 = true; 753 svm->nested.last_vmcb12_gpa = svm->nested.vmcb12_gpa; 754 svm->nested.force_msr_bitmap_recalc = true; 755 } 756 757 if (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_SEG))) { 758 vmcb02->save.es = save->es; 759 vmcb02->save.cs = save->cs; 760 vmcb02->save.ss = save->ss; 761 vmcb02->save.ds = save->ds; 762 vmcb02->save.cpl = save->cpl; 763 vmcb_mark_dirty(vmcb02, VMCB_SEG); 764 } 765 766 if (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_DT))) { 767 vmcb02->save.gdtr = save->gdtr; 768 vmcb02->save.idtr = save->idtr; 769 vmcb_mark_dirty(vmcb02, VMCB_DT); 770 } 771 772 if (guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) && 773 (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_CET)))) { 774 vmcb02->save.s_cet = save->s_cet; 775 vmcb02->save.isst_addr = save->isst_addr; 776 vmcb02->save.ssp = save->ssp; 777 vmcb_mark_dirty(vmcb02, VMCB_CET); 778 } 779 780 if (l2_has_separate_pat(vcpu)) { 781 if (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_NPT))) 782 vmcb_set_gpat(vmcb02, svm->nested.save.g_pat); 783 } else if (npt_enabled) { 784 vmcb_set_gpat(vmcb02, vcpu->arch.pat); 785 } 786 787 kvm_set_rflags(vcpu, save->rflags | X86_EFLAGS_FIXED); 788 789 /* SVM ignores EFER.LMA if EFER.LME=0 (instead of failing VMRUN). */ 790 if (!(svm->nested.save.efer & EFER_LME)) 791 svm->nested.save.efer &= ~EFER_LMA; 792 793 svm_set_efer(vcpu, svm->nested.save.efer); 794 795 svm_set_cr0(vcpu, svm->nested.save.cr0); 796 svm_set_cr4(vcpu, svm->nested.save.cr4); 797 798 svm->vcpu.arch.cr2 = save->cr2; 799 800 kvm_rax_write_raw(vcpu, save->rax); 801 kvm_rsp_write(vcpu, save->rsp); 802 kvm_rip_write(vcpu, save->rip); 803 804 /* In case we don't even reach vcpu_run, the fields are not updated */ 805 vmcb02->save.rax = save->rax; 806 vmcb02->save.rsp = save->rsp; 807 vmcb02->save.rip = save->rip; 808 809 if (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_DR))) { 810 vmcb02->save.dr7 = svm->nested.save.dr7 | DR7_FIXED_1; 811 svm->vcpu.arch.dr6 = svm->nested.save.dr6 | kvm_get_dr6_fixed_1(vcpu); 812 vmcb_mark_dirty(vmcb02, VMCB_DR); 813 } 814 815 if (control->misc_ctl2 & SVM_MISC2_ENABLE_V_LBR) { 816 /* 817 * Reserved bits of DEBUGCTL are ignored. Be consistent with 818 * svm_set_msr's definition of reserved bits. 819 */ 820 svm_copy_lbrs(&vmcb02->save, save); 821 vmcb02->save.dbgctl &= svm_get_supported_debugctl(vcpu); 822 } else { 823 svm_copy_lbrs(&vmcb02->save, &vmcb01->save); 824 } 825 vmcb_mark_dirty(vmcb02, VMCB_LBR); 826 svm_update_lbrv(&svm->vcpu); 827 } 828 829 static inline bool is_evtinj_soft(u32 evtinj) 830 { 831 u32 type = evtinj & SVM_EVTINJ_TYPE_MASK; 832 u8 vector = evtinj & SVM_EVTINJ_VEC_MASK; 833 834 if (!(evtinj & SVM_EVTINJ_VALID)) 835 return false; 836 837 if (type == SVM_EVTINJ_TYPE_SOFT) 838 return true; 839 840 return type == SVM_EVTINJ_TYPE_EXEPT && kvm_exception_is_soft(vector); 841 } 842 843 static bool is_evtinj_nmi(u32 evtinj) 844 { 845 u32 type = evtinj & SVM_EVTINJ_TYPE_MASK; 846 847 if (!(evtinj & SVM_EVTINJ_VALID)) 848 return false; 849 850 return type == SVM_EVTINJ_TYPE_NMI; 851 } 852 853 static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) 854 { 855 u32 int_ctl_vmcb01_bits = V_INTR_MASKING_MASK; 856 u32 int_ctl_vmcb12_bits = V_TPR_MASK | V_IRQ_INJECTION_BITS_MASK; 857 858 struct vmcb_ctrl_area_cached *vmcb12_ctrl = &svm->nested.ctl; 859 struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; 860 struct vmcb *vmcb01 = svm->vmcb01.ptr; 861 struct kvm_vcpu *vcpu = &svm->vcpu; 862 863 nested_svm_transition_tlb_flush(vcpu); 864 865 /* Enter Guest-Mode */ 866 enter_guest_mode(vcpu); 867 svm_pmu_handle_nested_transition(svm); 868 869 /* 870 * Filled at exit: exit_code, exit_info_1, exit_info_2, exit_int_info, 871 * exit_int_info_err, next_rip, insn_len, insn_bytes. 872 */ 873 874 if (guest_cpu_cap_has(vcpu, X86_FEATURE_VGIF) && 875 (vmcb12_ctrl->int_ctl & V_GIF_ENABLE_MASK)) 876 int_ctl_vmcb12_bits |= (V_GIF_MASK | V_GIF_ENABLE_MASK); 877 else 878 int_ctl_vmcb01_bits |= (V_GIF_MASK | V_GIF_ENABLE_MASK); 879 880 if (vnmi) { 881 if (vmcb01->control.int_ctl & V_NMI_PENDING_MASK) { 882 svm->vcpu.arch.nmi_pending++; 883 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); 884 } 885 if (nested_vnmi_enabled(svm)) 886 int_ctl_vmcb12_bits |= (V_NMI_PENDING_MASK | 887 V_NMI_ENABLE_MASK | 888 V_NMI_BLOCKING_MASK); 889 } 890 891 /* 892 * Copied from vmcb01. msrpm_base can be overwritten later. 893 * 894 * SVM_MISC_ENABLE_NP in vmcb12 is only used for consistency checks. If 895 * L1 enables NPTs, KVM shadows L1's NPTs and uses those to run L2. If 896 * L1 disables NPT, KVM runs L2 with the same NPTs used to run L1. For 897 * the latter, L1 runs L2 with shadow page tables that translate L2 GVAs 898 * to L1 GPAs, so the same NPTs can be used for L1 and L2. 899 */ 900 vmcb02->control.misc_ctl = vmcb01->control.misc_ctl & (SVM_MISC_ENABLE_NP | SVM_MISC_ENABLE_GMET); 901 vmcb02->control.iopm_base_pa = vmcb01->control.iopm_base_pa; 902 vmcb02->control.msrpm_base_pa = vmcb01->control.msrpm_base_pa; 903 vmcb_mark_dirty(vmcb02, VMCB_PERM_MAP); 904 905 /* 906 * PML is never enabled in hardware for L2. Make sure that an 907 * unexpected PML write would trigger a PML_FULL VM-Exit. 908 */ 909 if (pml) 910 vmcb02->control.pml_index = -1; 911 912 /* 913 * Stash vmcb02's counter if the guest hasn't moved past the guilty 914 * instruction; otherwise, reset the counter to '0'. 915 * 916 * In order to detect if L2 has made forward progress or not, track the 917 * RIP at which a bus lock has occurred on a per-vmcb12 basis. If RIP 918 * is changed, guest has clearly made forward progress, bus_lock_counter 919 * still remained '1', so reset bus_lock_counter to '0'. Eg. In the 920 * scenario, where a buslock happened in L1 before VMRUN, the bus lock 921 * firmly happened on an instruction in the past. Even if vmcb01's 922 * counter is still '1', (because the guilty instruction got patched), 923 * the vCPU has clearly made forward progress and so KVM should reset 924 * vmcb02's counter to '0'. 925 * 926 * If the RIP hasn't changed, stash the bus lock counter at nested VMRUN 927 * to prevent the same guilty instruction from triggering a VM-Exit. Eg. 928 * if userspace rate-limits the vCPU, then it's entirely possible that 929 * L1's tick interrupt is pending by the time userspace re-runs the 930 * vCPU. If KVM unconditionally clears the counter on VMRUN, then when 931 * L1 re-enters L2, the same instruction will trigger a VM-Exit and the 932 * entire cycle start over. 933 */ 934 if (vmcb02->save.rip && (svm->nested.last_bus_lock_rip == vmcb02->save.rip)) 935 vmcb02->control.bus_lock_counter = 1; 936 else 937 vmcb02->control.bus_lock_counter = 0; 938 939 /* Done at vmrun: asid. */ 940 941 /* Also overwritten later if necessary. */ 942 vmcb02->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; 943 944 /* Use vmcb01 MMU and format if guest does not use nNPT */ 945 if (nested_npt_enabled(svm)) { 946 vmcb02->control.misc_ctl &= ~SVM_MISC_ENABLE_GMET; 947 vmcb02->control.misc_ctl |= (svm->nested.ctl.misc_ctl & SVM_MISC_ENABLE_GMET); 948 949 nested_svm_init_mmu_context(vcpu); 950 } 951 952 vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(vcpu->arch.l1_tsc_offset, 953 vmcb12_ctrl->tsc_offset, 954 svm->tsc_ratio_msr); 955 956 vmcb02->control.tsc_offset = vcpu->arch.tsc_offset; 957 958 if (guest_cpu_cap_has(vcpu, X86_FEATURE_TSCRATEMSR) && 959 svm->tsc_ratio_msr != kvm_caps.default_tsc_scaling_ratio) 960 nested_svm_update_tsc_ratio_msr(vcpu); 961 962 vmcb02->control.int_ctl = 963 (vmcb12_ctrl->int_ctl & int_ctl_vmcb12_bits) | 964 (vmcb01->control.int_ctl & int_ctl_vmcb01_bits); 965 966 vmcb02->control.int_vector = vmcb12_ctrl->int_vector; 967 vmcb02->control.int_state = vmcb12_ctrl->int_state; 968 vmcb02->control.event_inj = vmcb12_ctrl->event_inj; 969 vmcb02->control.event_inj_err = vmcb12_ctrl->event_inj_err; 970 971 /* 972 * If nrips is exposed to L1, take NextRIP as-is. Otherwise, L1 973 * advances L2's RIP before VMRUN instead of using NextRIP. KVM will 974 * stuff the current RIP as vmcb02's NextRIP before L2 is run. After 975 * the first run of L2 (e.g. after save+restore), NextRIP is updated by 976 * the CPU and/or KVM and should be used regardless of L1's support. 977 */ 978 if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS) || 979 !vcpu->arch.nested_run_pending) 980 vmcb02->control.next_rip = vmcb12_ctrl->next_rip; 981 982 svm->nmi_l1_to_l2 = is_evtinj_nmi(vmcb02->control.event_inj); 983 984 /* 985 * soft_int_csbase, soft_int_old_rip, and soft_int_next_rip (if L1 986 * doesn't have NRIPS) are initialized later, before the vCPU is run. 987 */ 988 if (is_evtinj_soft(vmcb02->control.event_inj)) { 989 svm->soft_int_injected = true; 990 if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS) || 991 !vcpu->arch.nested_run_pending) 992 svm->soft_int_next_rip = vmcb12_ctrl->next_rip; 993 } 994 995 /* SVM_MISC2_ENABLE_V_LBR is controlled by svm_update_lbrv() */ 996 997 if (!nested_vmcb_needs_vls_intercept(svm)) 998 vmcb02->control.misc_ctl2 |= SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE; 999 1000 if (guest_cpu_cap_has(vcpu, X86_FEATURE_PAUSEFILTER)) 1001 vmcb02->control.pause_filter_count = vmcb12_ctrl->pause_filter_count; 1002 else 1003 vmcb02->control.pause_filter_count = 0; 1004 if (guest_cpu_cap_has(vcpu, X86_FEATURE_PFTHRESHOLD)) 1005 vmcb02->control.pause_filter_thresh = vmcb12_ctrl->pause_filter_thresh; 1006 else 1007 vmcb02->control.pause_filter_thresh = 0; 1008 1009 /* 1010 * Take ALLOW_LARGER_RAP from vmcb12 even though it should be safe to 1011 * let L2 use a larger RAP since KVM will emulate the necessary clears, 1012 * as it's possible L1 deliberately wants to restrict L2 to the legacy 1013 * RAP size. Unconditionally clear the RAP on nested VMRUN, as KVM is 1014 * responsible for emulating the host vs. guest tags (L1 is the "host", 1015 * L2 is the "guest"). 1016 */ 1017 if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS)) 1018 vmcb02->control.erap_ctl = (vmcb12_ctrl->erap_ctl & 1019 ERAP_CONTROL_ALLOW_LARGER_RAP) | 1020 ERAP_CONTROL_CLEAR_RAP; 1021 1022 /* 1023 * Merge guest and host intercepts - must be called with vcpu in 1024 * guest-mode to take effect. 1025 */ 1026 nested_vmcb02_recalc_intercepts(svm); 1027 } 1028 1029 static void nested_svm_copy_common_state(struct vmcb *from_vmcb, struct vmcb *to_vmcb) 1030 { 1031 /* 1032 * Some VMCB state is shared between L1 and L2 and thus has to be 1033 * moved at the time of nested vmrun and vmexit. 1034 * 1035 * VMLOAD/VMSAVE state would also belong in this category, but KVM 1036 * always performs VMLOAD and VMSAVE from the VMCB01. 1037 */ 1038 to_vmcb->save.spec_ctrl = from_vmcb->save.spec_ctrl; 1039 } 1040 1041 int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb12_gpa, bool from_vmrun) 1042 { 1043 struct vcpu_svm *svm = to_svm(vcpu); 1044 struct vmcb_ctrl_area_cached *control = &svm->nested.ctl; 1045 struct vmcb_save_area_cached *save = &svm->nested.save; 1046 int ret; 1047 1048 trace_kvm_nested_vmenter(svm->vmcb->save.rip, 1049 vmcb12_gpa, 1050 save->rip, 1051 control->int_ctl, 1052 control->event_inj, 1053 control->misc_ctl, 1054 control->nested_cr3, 1055 save->cr3, 1056 KVM_ISA_SVM); 1057 1058 trace_kvm_nested_intercepts(control->intercepts[INTERCEPT_CR] & 0xffff, 1059 control->intercepts[INTERCEPT_CR] >> 16, 1060 control->intercepts[INTERCEPT_EXCEPTION], 1061 control->intercepts[INTERCEPT_WORD3], 1062 control->intercepts[INTERCEPT_WORD4], 1063 control->intercepts[INTERCEPT_WORD5]); 1064 1065 1066 svm->nested.vmcb12_gpa = vmcb12_gpa; 1067 1068 WARN_ON(svm->vmcb == svm->nested.vmcb02.ptr); 1069 1070 nested_svm_copy_common_state(svm->vmcb01.ptr, svm->nested.vmcb02.ptr); 1071 1072 svm_switch_vmcb(svm, &svm->nested.vmcb02); 1073 nested_vmcb02_prepare_control(svm); 1074 nested_vmcb02_prepare_save(svm); 1075 1076 ret = nested_svm_load_cr3(&svm->vcpu, svm->nested.save.cr3, 1077 nested_npt_enabled(svm), from_vmrun); 1078 if (ret) 1079 return ret; 1080 1081 if (!from_vmrun) 1082 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 1083 1084 svm_set_gif(svm, true); 1085 1086 if (kvm_vcpu_apicv_active(vcpu)) 1087 kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu); 1088 1089 nested_svm_hv_update_vm_vp_ids(vcpu); 1090 1091 return 0; 1092 } 1093 1094 static int nested_svm_copy_vmcb12_to_cache(struct kvm_vcpu *vcpu, u64 vmcb12_gpa) 1095 { 1096 struct vcpu_svm *svm = to_svm(vcpu); 1097 struct vmcb *vmcb12; 1098 int r = 0; 1099 1100 CLASS(kvm_vcpu_map_local, m)(vcpu, gpa_to_gfn(vmcb12_gpa)); 1101 if (m.ret) 1102 return -EFAULT; 1103 1104 vmcb12 = m.map.hva; 1105 nested_copy_vmcb_control_to_cache(svm, &vmcb12->control); 1106 nested_copy_vmcb_save_to_cache(svm, &vmcb12->save); 1107 1108 if (nested_svm_check_cached_vmcb12(vcpu) < 0) { 1109 vmcb12->control.exit_code = SVM_EXIT_ERR; 1110 vmcb12->control.exit_info_1 = 0; 1111 vmcb12->control.exit_info_2 = 0; 1112 vmcb12->control.event_inj = 0; 1113 vmcb12->control.event_inj_err = 0; 1114 svm_set_gif(svm, false); 1115 r = -EINVAL; 1116 } 1117 1118 return r; 1119 } 1120 1121 int nested_svm_vmrun(struct kvm_vcpu *vcpu) 1122 { 1123 struct vcpu_svm *svm = to_svm(vcpu); 1124 int ret; 1125 u64 vmcb12_gpa; 1126 struct vmcb *vmcb01 = svm->vmcb01.ptr; 1127 1128 if (!svm->nested.hsave_msr) { 1129 kvm_inject_gp(vcpu, 0); 1130 return 1; 1131 } 1132 1133 if (is_smm(vcpu)) { 1134 kvm_queue_exception(vcpu, UD_VECTOR); 1135 return 1; 1136 } 1137 1138 /* This fails when VP assist page is enabled but the supplied GPA is bogus */ 1139 ret = kvm_hv_verify_vp_assist(vcpu); 1140 if (ret) { 1141 kvm_inject_gp(vcpu, 0); 1142 return ret; 1143 } 1144 1145 if (WARN_ON_ONCE(!svm->nested.initialized)) 1146 return -EINVAL; 1147 1148 vmcb12_gpa = kvm_rax_read(vcpu); 1149 if (!page_address_valid(vcpu, vmcb12_gpa)) { 1150 kvm_inject_gp(vcpu, 0); 1151 return 1; 1152 } 1153 1154 ret = nested_svm_copy_vmcb12_to_cache(vcpu, vmcb12_gpa); 1155 if (ret == -EFAULT) 1156 return kvm_handle_memory_failure(vcpu, X86EMUL_IO_NEEDED, NULL); 1157 1158 /* 1159 * At this point, VMRUN is guaranteed to not fault; advance RIP. If 1160 * caching vmcb12 failed for other reasons, return immediately afterward 1161 * as a nested #VMEXIT was already set up. 1162 * 1163 * FIXME: If TF is set on VMRUN should inject a #DB (or handle guest 1164 * debugging) right after #VMEXIT, right now it's just ignored. 1165 */ 1166 if (!svm_skip_emulated_instruction(vcpu)) 1167 return 0; 1168 1169 if (ret) 1170 goto insn_retired; 1171 1172 /* 1173 * Since vmcb01 is not in use, we can use it to store some of the L1 1174 * state. 1175 */ 1176 vmcb01->save.efer = vcpu->arch.efer; 1177 vmcb01->save.cr0 = kvm_read_cr0(vcpu); 1178 vmcb01->save.cr4 = vcpu->arch.cr4; 1179 vmcb01->save.rflags = kvm_get_rflags(vcpu); 1180 vmcb01->save.rip = kvm_rip_read(vcpu); 1181 1182 if (!npt_enabled) 1183 vmcb01->save.cr3 = kvm_read_cr3(vcpu); 1184 1185 vcpu->arch.nested_run_pending = KVM_NESTED_RUN_PENDING; 1186 1187 if (enter_svm_guest_mode(vcpu, vmcb12_gpa, true) || 1188 !nested_svm_merge_msrpm(vcpu)) { 1189 vcpu->arch.nested_run_pending = 0; 1190 svm->nmi_l1_to_l2 = false; 1191 svm->soft_int_injected = false; 1192 1193 svm->vmcb->control.exit_code = SVM_EXIT_ERR; 1194 svm->vmcb->control.exit_info_1 = 0; 1195 svm->vmcb->control.exit_info_2 = 0; 1196 1197 nested_svm_vmexit(svm); 1198 } 1199 1200 insn_retired: 1201 /* 1202 * A successful VMRUN is counted by the PMU in guest mode, so only 1203 * retire the instruction after potentially entering guest mode. 1204 */ 1205 kvm_pmu_instruction_retired(vcpu); 1206 return 1; 1207 } 1208 1209 /* Copy state save area fields which are handled by VMRUN */ 1210 void svm_copy_vmrun_state(struct kvm_vcpu *vcpu, struct vmcb_save_area *to_save, 1211 struct vmcb_save_area *from_save) 1212 { 1213 to_save->es = from_save->es; 1214 to_save->cs = from_save->cs; 1215 to_save->ss = from_save->ss; 1216 to_save->ds = from_save->ds; 1217 to_save->gdtr = from_save->gdtr; 1218 to_save->idtr = from_save->idtr; 1219 to_save->rflags = from_save->rflags | X86_EFLAGS_FIXED; 1220 to_save->efer = from_save->efer; 1221 to_save->cr0 = from_save->cr0; 1222 to_save->cr3 = from_save->cr3; 1223 to_save->cr4 = from_save->cr4; 1224 to_save->rax = from_save->rax; 1225 to_save->rsp = from_save->rsp; 1226 to_save->rip = from_save->rip; 1227 to_save->cpl = 0; 1228 1229 if (kvm_cpu_cap_has(X86_FEATURE_SHSTK)) { 1230 to_save->s_cet = from_save->s_cet; 1231 to_save->isst_addr = from_save->isst_addr; 1232 to_save->ssp = from_save->ssp; 1233 } 1234 1235 if (kvm_cpu_cap_has(X86_FEATURE_LBRV)) { 1236 svm_copy_lbrs(to_save, from_save); 1237 to_save->dbgctl &= svm_get_supported_debugctl(vcpu); 1238 } 1239 } 1240 1241 void svm_copy_vmloadsave_state(struct vmcb *to_vmcb, struct vmcb *from_vmcb) 1242 { 1243 to_vmcb->save.fs = from_vmcb->save.fs; 1244 to_vmcb->save.gs = from_vmcb->save.gs; 1245 to_vmcb->save.tr = from_vmcb->save.tr; 1246 to_vmcb->save.ldtr = from_vmcb->save.ldtr; 1247 to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base; 1248 to_vmcb->save.star = from_vmcb->save.star; 1249 to_vmcb->save.lstar = from_vmcb->save.lstar; 1250 to_vmcb->save.cstar = from_vmcb->save.cstar; 1251 to_vmcb->save.sfmask = from_vmcb->save.sfmask; 1252 to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs; 1253 to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp; 1254 to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip; 1255 } 1256 1257 static int nested_svm_vmexit_update_vmcb12(struct kvm_vcpu *vcpu) 1258 { 1259 struct vcpu_svm *svm = to_svm(vcpu); 1260 struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; 1261 struct vmcb *vmcb12; 1262 1263 CLASS(kvm_vcpu_map_local, m)(vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa)); 1264 if (m.ret) 1265 return m.ret; 1266 1267 vmcb12 = m.map.hva; 1268 1269 vmcb12->save.es = vmcb02->save.es; 1270 vmcb12->save.cs = vmcb02->save.cs; 1271 vmcb12->save.ss = vmcb02->save.ss; 1272 vmcb12->save.ds = vmcb02->save.ds; 1273 vmcb12->save.gdtr = vmcb02->save.gdtr; 1274 vmcb12->save.idtr = vmcb02->save.idtr; 1275 vmcb12->save.efer = svm->vcpu.arch.efer; 1276 vmcb12->save.cr0 = kvm_read_cr0(vcpu); 1277 vmcb12->save.cr3 = kvm_read_cr3(vcpu); 1278 vmcb12->save.cr2 = vcpu->arch.cr2; 1279 vmcb12->save.cr4 = svm->vcpu.arch.cr4; 1280 vmcb12->save.rflags = kvm_get_rflags(vcpu); 1281 vmcb12->save.rip = kvm_rip_read(vcpu); 1282 vmcb12->save.rsp = kvm_rsp_read(vcpu); 1283 vmcb12->save.rax = kvm_rax_read_raw(vcpu); 1284 vmcb12->save.dr7 = vmcb02->save.dr7; 1285 vmcb12->save.dr6 = svm->vcpu.arch.dr6; 1286 vmcb12->save.cpl = vmcb02->save.cpl; 1287 1288 if (l2_has_separate_pat(vcpu)) 1289 vmcb12->save.g_pat = vmcb02->save.g_pat; 1290 1291 if (guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK)) { 1292 vmcb12->save.s_cet = vmcb02->save.s_cet; 1293 vmcb12->save.isst_addr = vmcb02->save.isst_addr; 1294 vmcb12->save.ssp = vmcb02->save.ssp; 1295 } 1296 1297 vmcb12->control.int_state = vmcb02->control.int_state; 1298 vmcb12->control.exit_code = vmcb02->control.exit_code; 1299 vmcb12->control.exit_info_1 = vmcb02->control.exit_info_1; 1300 vmcb12->control.exit_info_2 = vmcb02->control.exit_info_2; 1301 1302 if (!svm_is_vmrun_failure(vmcb12->control.exit_code)) 1303 nested_save_pending_event_to_vmcb12(svm, vmcb12); 1304 1305 if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS)) 1306 vmcb12->control.next_rip = vmcb02->control.next_rip; 1307 1308 if (svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR) 1309 svm_copy_lbrs(&vmcb12->save, &vmcb02->save); 1310 1311 vmcb12->control.event_inj = 0; 1312 vmcb12->control.event_inj_err = 0; 1313 vmcb12->control.int_ctl = svm->nested.ctl.int_ctl; 1314 1315 trace_kvm_nested_vmexit_inject(vmcb12->control.exit_code, 1316 vmcb12->control.exit_info_1, 1317 vmcb12->control.exit_info_2, 1318 vmcb12->control.exit_int_info, 1319 vmcb12->control.exit_int_info_err, 1320 KVM_ISA_SVM); 1321 1322 return 0; 1323 } 1324 1325 void nested_svm_vmexit(struct vcpu_svm *svm) 1326 { 1327 struct kvm_vcpu *vcpu = &svm->vcpu; 1328 struct vmcb *vmcb01 = svm->vmcb01.ptr; 1329 struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; 1330 1331 if (nested_svm_vmexit_update_vmcb12(vcpu)) 1332 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 1333 1334 /* Exit Guest-Mode */ 1335 leave_guest_mode(vcpu); 1336 svm_pmu_handle_nested_transition(svm); 1337 1338 svm->nested.vmcb12_gpa = 0; 1339 1340 kvm_warn_on_nested_run_pending(vcpu); 1341 1342 kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 1343 1344 /* in case we halted in L2 */ 1345 kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); 1346 1347 /* 1348 * Invalidate last_bus_lock_rip unless KVM is still waiting for the 1349 * guest to make forward progress before re-enabling bus lock detection. 1350 */ 1351 if (!vmcb02->control.bus_lock_counter) 1352 svm->nested.last_bus_lock_rip = INVALID_GPA; 1353 1354 nested_svm_copy_common_state(svm->nested.vmcb02.ptr, svm->vmcb01.ptr); 1355 1356 kvm_nested_vmexit_handle_ibrs(vcpu); 1357 1358 if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS)) 1359 vmcb01->control.erap_ctl |= ERAP_CONTROL_CLEAR_RAP; 1360 1361 svm_switch_vmcb(svm, &svm->vmcb01); 1362 1363 /* 1364 * Rules for synchronizing int_ctl bits from vmcb02 to vmcb01: 1365 * 1366 * V_IRQ, V_IRQ_VECTOR, V_INTR_PRIO_MASK, V_IGN_TPR: If L1 doesn't 1367 * intercept interrupts, then KVM will use vmcb02's V_IRQ (and related 1368 * flags) to detect interrupt windows for L1 IRQs (even if L1 uses 1369 * virtual interrupt masking). Raise KVM_REQ_EVENT to ensure that 1370 * KVM re-requests an interrupt window if necessary, which implicitly 1371 * copies this bits from vmcb02 to vmcb01. 1372 * 1373 * V_TPR: If L1 doesn't use virtual interrupt masking, then L1's vTPR 1374 * is stored in vmcb02, but its value doesn't need to be copied from/to 1375 * vmcb01 because it is copied from/to the virtual APIC's TPR register 1376 * on each VM entry/exit. 1377 * 1378 * V_GIF: If nested vGIF is not used, KVM uses vmcb02's V_GIF for L1's 1379 * V_GIF. However, GIF is architecturally clear on each VM exit, thus 1380 * there is no need to copy V_GIF from vmcb02 to vmcb01. 1381 */ 1382 if (!nested_exit_on_intr(svm)) 1383 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); 1384 1385 if (!(svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR)) { 1386 svm_copy_lbrs(&vmcb01->save, &vmcb02->save); 1387 vmcb_mark_dirty(vmcb01, VMCB_LBR); 1388 } 1389 1390 svm_update_lbrv(vcpu); 1391 1392 if (vnmi) { 1393 if (vmcb02->control.int_ctl & V_NMI_BLOCKING_MASK) 1394 vmcb01->control.int_ctl |= V_NMI_BLOCKING_MASK; 1395 else 1396 vmcb01->control.int_ctl &= ~V_NMI_BLOCKING_MASK; 1397 1398 if (vcpu->arch.nmi_pending) { 1399 vcpu->arch.nmi_pending--; 1400 vmcb01->control.int_ctl |= V_NMI_PENDING_MASK; 1401 } else { 1402 vmcb01->control.int_ctl &= ~V_NMI_PENDING_MASK; 1403 } 1404 } 1405 1406 /* 1407 * On vmexit the GIF is set to false and 1408 * no event can be injected in L1. 1409 */ 1410 svm_set_gif(svm, false); 1411 vmcb01->control.exit_int_info = 0; 1412 1413 svm->vcpu.arch.tsc_offset = svm->vcpu.arch.l1_tsc_offset; 1414 if (vmcb01->control.tsc_offset != svm->vcpu.arch.tsc_offset) { 1415 vmcb01->control.tsc_offset = svm->vcpu.arch.tsc_offset; 1416 vmcb_mark_dirty(vmcb01, VMCB_INTERCEPTS); 1417 } 1418 1419 if (kvm_caps.has_tsc_control && 1420 vcpu->arch.tsc_scaling_ratio != vcpu->arch.l1_tsc_scaling_ratio) { 1421 vcpu->arch.tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; 1422 svm_write_tsc_multiplier(vcpu); 1423 } 1424 1425 svm->nested.ctl.nested_cr3 = 0; 1426 1427 /* 1428 * Restore processor state that had been saved in vmcb01 1429 */ 1430 kvm_set_rflags(vcpu, vmcb01->save.rflags); 1431 svm_set_efer(vcpu, vmcb01->save.efer); 1432 svm_set_cr0(vcpu, vmcb01->save.cr0 | X86_CR0_PE); 1433 svm_set_cr4(vcpu, vmcb01->save.cr4); 1434 kvm_rax_write_raw(vcpu, vmcb01->save.rax); 1435 kvm_rsp_write(vcpu, vmcb01->save.rsp); 1436 kvm_rip_write(vcpu, vmcb01->save.rip); 1437 1438 svm->vcpu.arch.dr7 = DR7_FIXED_1; 1439 kvm_update_dr7(&svm->vcpu); 1440 1441 nested_svm_transition_tlb_flush(vcpu); 1442 1443 nested_svm_uninit_mmu_context(vcpu); 1444 1445 if (nested_svm_load_cr3(vcpu, vmcb01->save.cr3, false, true)) 1446 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 1447 1448 /* Drop tracking for L1->L2 injected NMIs and soft IRQs */ 1449 svm->nmi_l1_to_l2 = false; 1450 svm->soft_int_injected = false; 1451 1452 /* 1453 * Drop what we picked up for L2 via svm_complete_interrupts() so it 1454 * doesn't end up in L1. 1455 */ 1456 svm->vcpu.arch.nmi_injected = false; 1457 kvm_clear_exception_queue(vcpu); 1458 kvm_clear_interrupt_queue(vcpu); 1459 1460 /* 1461 * If we are here following the completion of a VMRUN that 1462 * is being single-stepped, queue the pending #DB intercept 1463 * right now so that it an be accounted for before we execute 1464 * L1's next instruction. 1465 */ 1466 if (unlikely(vmcb01->save.rflags & X86_EFLAGS_TF)) 1467 kvm_queue_exception(&(svm->vcpu), DB_VECTOR); 1468 1469 /* 1470 * Un-inhibit the AVIC right away, so that other vCPUs can start 1471 * to benefit from it right away. 1472 */ 1473 if (kvm_apicv_activated(vcpu->kvm)) 1474 __kvm_vcpu_update_apicv(vcpu); 1475 } 1476 1477 static void nested_svm_triple_fault(struct kvm_vcpu *vcpu) 1478 { 1479 struct vcpu_svm *svm = to_svm(vcpu); 1480 1481 if (!vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SHUTDOWN)) 1482 return; 1483 1484 kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu); 1485 nested_svm_simple_vmexit(to_svm(vcpu), SVM_EXIT_SHUTDOWN); 1486 } 1487 1488 int svm_allocate_nested(struct vcpu_svm *svm) 1489 { 1490 struct page *vmcb02_page; 1491 1492 if (svm->nested.initialized) 1493 return 0; 1494 1495 vmcb02_page = snp_safe_alloc_page(); 1496 if (!vmcb02_page) 1497 return -ENOMEM; 1498 svm->nested.vmcb02.ptr = page_address(vmcb02_page); 1499 svm->nested.vmcb02.pa = __sme_set(page_to_pfn(vmcb02_page) << PAGE_SHIFT); 1500 1501 svm->nested.msrpm = svm_vcpu_alloc_msrpm(); 1502 if (!svm->nested.msrpm) 1503 goto err_free_vmcb02; 1504 1505 svm->nested.vmcb02.cpu = -1; 1506 svm->nested.initialized = true; 1507 return 0; 1508 1509 err_free_vmcb02: 1510 __free_page(vmcb02_page); 1511 return -ENOMEM; 1512 } 1513 1514 void svm_free_nested(struct vcpu_svm *svm) 1515 { 1516 if (!svm->nested.initialized) 1517 return; 1518 1519 if (WARN_ON_ONCE(svm->vmcb != svm->vmcb01.ptr)) 1520 svm_switch_vmcb(svm, &svm->vmcb01); 1521 1522 svm_vcpu_free_msrpm(svm->nested.msrpm); 1523 svm->nested.msrpm = NULL; 1524 1525 __free_page(virt_to_page(svm->nested.vmcb02.ptr)); 1526 svm->nested.vmcb02.ptr = NULL; 1527 1528 /* 1529 * When last_vmcb12_gpa matches the current vmcb12 gpa, 1530 * some vmcb12 fields are not loaded if they are marked clean 1531 * in the vmcb12, since in this case they are up to date already. 1532 * 1533 * When the vmcb02 is freed, this optimization becomes invalid. 1534 */ 1535 svm->nested.last_vmcb12_gpa = INVALID_GPA; 1536 1537 svm->nested.initialized = false; 1538 } 1539 1540 void svm_leave_nested(struct kvm_vcpu *vcpu) 1541 { 1542 struct vcpu_svm *svm = to_svm(vcpu); 1543 1544 if (is_guest_mode(vcpu)) { 1545 vcpu->arch.nested_run_pending = 0; 1546 svm->nested.vmcb12_gpa = INVALID_GPA; 1547 1548 leave_guest_mode(vcpu); 1549 1550 /* 1551 * Force leaving nested is a non-architectural flow so precision 1552 * isn't a priority. Defer updating the PMU until the next vCPU 1553 * run, potentially tolerating some imprecision to avoid poking 1554 * into PMU state from arbitrary contexts (e.g. to avoid using 1555 * stale state). 1556 */ 1557 __svm_pmu_handle_nested_transition(svm, true); 1558 1559 svm_switch_vmcb(svm, &svm->vmcb01); 1560 1561 nested_svm_uninit_mmu_context(vcpu); 1562 vmcb_mark_all_dirty(svm->vmcb); 1563 1564 svm_set_gif(svm, true); 1565 1566 if (kvm_apicv_activated(vcpu->kvm)) 1567 kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu); 1568 } 1569 1570 kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 1571 } 1572 1573 static int nested_svm_exit_handled_msr(struct vcpu_svm *svm) 1574 { 1575 gpa_t base = svm->nested.ctl.msrpm_base_pa; 1576 int write, bit_nr; 1577 u8 value, mask; 1578 u32 msr; 1579 1580 if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT))) 1581 return NESTED_EXIT_HOST; 1582 1583 msr = svm->vcpu.arch.regs[VCPU_REGS_RCX]; 1584 bit_nr = svm_msrpm_bit_nr(msr); 1585 write = svm->vmcb->control.exit_info_1 & 1; 1586 1587 if (bit_nr < 0) 1588 return NESTED_EXIT_DONE; 1589 1590 if (kvm_vcpu_read_guest(&svm->vcpu, base + bit_nr / BITS_PER_BYTE, 1591 &value, sizeof(value))) 1592 return NESTED_EXIT_DONE; 1593 1594 mask = BIT(write) << (bit_nr & (BITS_PER_BYTE - 1)); 1595 return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; 1596 } 1597 1598 static int nested_svm_intercept_ioio(struct vcpu_svm *svm) 1599 { 1600 unsigned port, size, iopm_len; 1601 u16 val, mask; 1602 u8 start_bit; 1603 u64 gpa; 1604 1605 if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_IOIO_PROT))) 1606 return NESTED_EXIT_HOST; 1607 1608 port = svm->vmcb->control.exit_info_1 >> 16; 1609 size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >> 1610 SVM_IOIO_SIZE_SHIFT; 1611 gpa = svm->nested.ctl.iopm_base_pa + (port / 8); 1612 start_bit = port % 8; 1613 iopm_len = (start_bit + size > 8) ? 2 : 1; 1614 mask = (0xf >> (4 - size)) << start_bit; 1615 val = 0; 1616 1617 if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len)) 1618 return NESTED_EXIT_DONE; 1619 1620 return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; 1621 } 1622 1623 static int nested_svm_intercept(struct vcpu_svm *svm) 1624 { 1625 u64 exit_code = svm->vmcb->control.exit_code; 1626 int vmexit = NESTED_EXIT_HOST; 1627 1628 if (svm_is_vmrun_failure(exit_code)) 1629 return NESTED_EXIT_DONE; 1630 1631 switch (exit_code) { 1632 case SVM_EXIT_MSR: 1633 vmexit = nested_svm_exit_handled_msr(svm); 1634 break; 1635 case SVM_EXIT_IOIO: 1636 vmexit = nested_svm_intercept_ioio(svm); 1637 break; 1638 case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: 1639 /* 1640 * Host-intercepted exceptions have been checked already in 1641 * nested_svm_exit_special. There is nothing to do here, 1642 * the vmexit is injected by svm_check_nested_events. 1643 */ 1644 vmexit = NESTED_EXIT_DONE; 1645 break; 1646 default: 1647 if (vmcb12_is_intercept(&svm->nested.ctl, exit_code)) 1648 vmexit = NESTED_EXIT_DONE; 1649 break; 1650 } 1651 1652 return vmexit; 1653 } 1654 1655 int nested_svm_exit_handled(struct vcpu_svm *svm) 1656 { 1657 int vmexit; 1658 1659 vmexit = nested_svm_intercept(svm); 1660 1661 if (vmexit == NESTED_EXIT_DONE) 1662 nested_svm_vmexit(svm); 1663 1664 return vmexit; 1665 } 1666 1667 int nested_svm_check_permissions(struct kvm_vcpu *vcpu) 1668 { 1669 if (!(vcpu->arch.efer & EFER_SVME) || !is_paging(vcpu)) { 1670 kvm_queue_exception(vcpu, UD_VECTOR); 1671 return 1; 1672 } 1673 1674 if (to_svm(vcpu)->vmcb->save.cpl) { 1675 kvm_inject_gp(vcpu, 0); 1676 return 1; 1677 } 1678 1679 return 0; 1680 } 1681 1682 static bool nested_svm_is_exception_vmexit(struct kvm_vcpu *vcpu, u8 vector, 1683 u32 error_code) 1684 { 1685 struct vcpu_svm *svm = to_svm(vcpu); 1686 1687 return (svm->nested.ctl.intercepts[INTERCEPT_EXCEPTION] & BIT(vector)); 1688 } 1689 1690 static void nested_svm_inject_exception_vmexit(struct kvm_vcpu *vcpu) 1691 { 1692 struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit; 1693 struct vcpu_svm *svm = to_svm(vcpu); 1694 struct vmcb *vmcb = svm->vmcb; 1695 1696 vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + ex->vector; 1697 1698 if (ex->has_error_code) 1699 vmcb->control.exit_info_1 = ex->error_code; 1700 1701 /* 1702 * EXITINFO2 is undefined for all exception intercepts other 1703 * than #PF. 1704 */ 1705 if (ex->vector == PF_VECTOR) { 1706 if (ex->has_payload) 1707 vmcb->control.exit_info_2 = ex->payload; 1708 else 1709 vmcb->control.exit_info_2 = vcpu->arch.cr2; 1710 } else if (ex->vector == DB_VECTOR) { 1711 /* See kvm_check_and_inject_events(). */ 1712 kvm_deliver_exception_payload(vcpu, ex); 1713 1714 if (vcpu->arch.dr7 & DR7_GD) { 1715 vcpu->arch.dr7 &= ~DR7_GD; 1716 kvm_update_dr7(vcpu); 1717 } 1718 } else { 1719 WARN_ON(ex->has_payload); 1720 } 1721 1722 nested_svm_vmexit(svm); 1723 } 1724 1725 static inline bool nested_exit_on_init(struct vcpu_svm *svm) 1726 { 1727 return vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_INIT); 1728 } 1729 1730 static int svm_check_nested_events(struct kvm_vcpu *vcpu) 1731 { 1732 struct kvm_lapic *apic = vcpu->arch.apic; 1733 struct vcpu_svm *svm = to_svm(vcpu); 1734 /* 1735 * Only a pending nested run blocks a pending exception. If there is a 1736 * previously injected event, the pending exception occurred while said 1737 * event was being delivered and thus needs to be handled. 1738 */ 1739 bool block_nested_exceptions = vcpu->arch.nested_run_pending; 1740 /* 1741 * New events (not exceptions) are only recognized at instruction 1742 * boundaries. If an event needs reinjection, then KVM is handling a 1743 * VM-Exit that occurred _during_ instruction execution; new events are 1744 * blocked until the instruction completes. 1745 */ 1746 bool block_nested_events = block_nested_exceptions || 1747 kvm_event_needs_reinjection(vcpu); 1748 1749 if (lapic_in_kernel(vcpu) && 1750 test_bit(KVM_APIC_INIT, &apic->pending_events)) { 1751 if (block_nested_events) 1752 return -EBUSY; 1753 if (!nested_exit_on_init(svm)) 1754 return 0; 1755 nested_svm_simple_vmexit(svm, SVM_EXIT_INIT); 1756 return 0; 1757 } 1758 1759 if (vcpu->arch.exception_vmexit.pending) { 1760 if (block_nested_exceptions) 1761 return -EBUSY; 1762 nested_svm_inject_exception_vmexit(vcpu); 1763 return 0; 1764 } 1765 1766 if (vcpu->arch.exception.pending) { 1767 if (block_nested_exceptions) 1768 return -EBUSY; 1769 return 0; 1770 } 1771 1772 #ifdef CONFIG_KVM_SMM 1773 if (vcpu->arch.smi_pending && !svm_smi_blocked(vcpu)) { 1774 if (block_nested_events) 1775 return -EBUSY; 1776 if (!nested_exit_on_smi(svm)) 1777 return 0; 1778 nested_svm_simple_vmexit(svm, SVM_EXIT_SMI); 1779 return 0; 1780 } 1781 #endif 1782 1783 if (vcpu->arch.nmi_pending && !svm_nmi_blocked(vcpu)) { 1784 if (block_nested_events) 1785 return -EBUSY; 1786 if (!nested_exit_on_nmi(svm)) 1787 return 0; 1788 nested_svm_simple_vmexit(svm, SVM_EXIT_NMI); 1789 return 0; 1790 } 1791 1792 if (kvm_cpu_has_interrupt(vcpu) && !svm_interrupt_blocked(vcpu)) { 1793 if (block_nested_events) 1794 return -EBUSY; 1795 if (!nested_exit_on_intr(svm)) 1796 return 0; 1797 trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip); 1798 nested_svm_simple_vmexit(svm, SVM_EXIT_INTR); 1799 return 0; 1800 } 1801 1802 return 0; 1803 } 1804 1805 int nested_svm_exit_special(struct vcpu_svm *svm) 1806 { 1807 u32 exit_code = svm->vmcb->control.exit_code; 1808 struct kvm_vcpu *vcpu = &svm->vcpu; 1809 1810 switch (exit_code) { 1811 case SVM_EXIT_INTR: 1812 case SVM_EXIT_NMI: 1813 case SVM_EXIT_NPF: 1814 return NESTED_EXIT_HOST; 1815 case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: { 1816 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE); 1817 1818 if (svm->vmcb01.ptr->control.intercepts[INTERCEPT_EXCEPTION] & 1819 excp_bits) 1820 return NESTED_EXIT_HOST; 1821 else if (exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR && 1822 svm->vcpu.arch.apf.host_apf_flags) 1823 /* Trap async PF even if not shadowing */ 1824 return NESTED_EXIT_HOST; 1825 break; 1826 } 1827 case SVM_EXIT_VMMCALL: 1828 /* Hyper-V L2 TLB flush hypercall is handled by L0 */ 1829 if (nested_svm_is_l2_tlb_flush_hcall(vcpu)) 1830 return NESTED_EXIT_HOST; 1831 break; 1832 case SVM_EXIT_PML_FULL: 1833 /* 1834 * All PML full exits are handled by KVM. KVM emulates PML in 1835 * software for L1, but never enables PML in hardware on behalf 1836 * of L1. 1837 */ 1838 return NESTED_EXIT_HOST; 1839 default: 1840 break; 1841 } 1842 1843 return NESTED_EXIT_CONTINUE; 1844 } 1845 1846 void nested_svm_update_tsc_ratio_msr(struct kvm_vcpu *vcpu) 1847 { 1848 struct vcpu_svm *svm = to_svm(vcpu); 1849 1850 vcpu->arch.tsc_scaling_ratio = 1851 kvm_calc_nested_tsc_multiplier(vcpu->arch.l1_tsc_scaling_ratio, 1852 svm->tsc_ratio_msr); 1853 svm_write_tsc_multiplier(vcpu); 1854 } 1855 1856 /* Inverse operation of nested_copy_vmcb_control_to_cache(). asid is copied too. */ 1857 static void nested_copy_vmcb_cache_to_control(struct vmcb_control_area *dst, 1858 struct vmcb_ctrl_area_cached *from) 1859 { 1860 unsigned int i; 1861 1862 memset(dst, 0, sizeof(struct vmcb_control_area)); 1863 1864 for (i = 0; i < MAX_INTERCEPT; i++) 1865 dst->intercepts[i] = from->intercepts[i]; 1866 1867 dst->iopm_base_pa = from->iopm_base_pa; 1868 dst->msrpm_base_pa = from->msrpm_base_pa; 1869 dst->tsc_offset = from->tsc_offset; 1870 dst->asid = from->asid; 1871 dst->tlb_ctl = from->tlb_ctl; 1872 dst->erap_ctl = from->erap_ctl; 1873 dst->int_ctl = from->int_ctl; 1874 dst->int_vector = from->int_vector; 1875 dst->int_state = from->int_state; 1876 dst->exit_code = from->exit_code; 1877 dst->exit_info_1 = from->exit_info_1; 1878 dst->exit_info_2 = from->exit_info_2; 1879 dst->exit_int_info = from->exit_int_info; 1880 dst->exit_int_info_err = from->exit_int_info_err; 1881 dst->misc_ctl = from->misc_ctl; 1882 dst->event_inj = from->event_inj; 1883 dst->event_inj_err = from->event_inj_err; 1884 dst->next_rip = from->next_rip; 1885 dst->nested_cr3 = from->nested_cr3; 1886 dst->misc_ctl2 = from->misc_ctl2; 1887 dst->pause_filter_count = from->pause_filter_count; 1888 dst->pause_filter_thresh = from->pause_filter_thresh; 1889 /* 'clean' and 'hv_enlightenments' are not changed by KVM */ 1890 } 1891 1892 static int svm_get_nested_state(struct kvm_vcpu *vcpu, 1893 struct kvm_nested_state __user *user_kvm_nested_state, 1894 u32 user_data_size) 1895 { 1896 struct vcpu_svm *svm; 1897 struct vmcb_control_area *ctl; 1898 unsigned long r; 1899 struct kvm_nested_state kvm_state = { 1900 .flags = 0, 1901 .format = KVM_STATE_NESTED_FORMAT_SVM, 1902 .size = sizeof(kvm_state), 1903 }; 1904 struct vmcb __user *user_vmcb = (struct vmcb __user *) 1905 &user_kvm_nested_state->data.svm[0]; 1906 1907 if (!vcpu) 1908 return kvm_state.size + KVM_STATE_NESTED_SVM_VMCB_SIZE; 1909 1910 svm = to_svm(vcpu); 1911 1912 if (user_data_size < kvm_state.size) 1913 goto out; 1914 1915 /* First fill in the header and copy it out. */ 1916 if (is_guest_mode(vcpu)) { 1917 kvm_state.hdr.svm.vmcb_pa = svm->nested.vmcb12_gpa; 1918 kvm_state.hdr.svm.gpat = 0; 1919 if (l2_has_separate_pat(vcpu)) 1920 kvm_state.hdr.svm.gpat = svm->vmcb->save.g_pat; 1921 kvm_state.size += KVM_STATE_NESTED_SVM_VMCB_SIZE; 1922 kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE; 1923 1924 if (vcpu->arch.nested_run_pending) 1925 kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING; 1926 } 1927 1928 if (gif_set(svm)) 1929 kvm_state.flags |= KVM_STATE_NESTED_GIF_SET; 1930 1931 if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state))) 1932 return -EFAULT; 1933 1934 if (!is_guest_mode(vcpu)) 1935 goto out; 1936 1937 /* 1938 * Copy over the full size of the VMCB rather than just the size 1939 * of the structs. 1940 */ 1941 if (clear_user(user_vmcb, KVM_STATE_NESTED_SVM_VMCB_SIZE)) 1942 return -EFAULT; 1943 1944 ctl = kzalloc_obj(*ctl); 1945 if (!ctl) 1946 return -ENOMEM; 1947 1948 nested_copy_vmcb_cache_to_control(ctl, &svm->nested.ctl); 1949 r = copy_to_user(&user_vmcb->control, ctl, 1950 sizeof(user_vmcb->control)); 1951 kfree(ctl); 1952 if (r) 1953 return -EFAULT; 1954 1955 if (copy_to_user(&user_vmcb->save, &svm->vmcb01.ptr->save, 1956 sizeof(user_vmcb->save))) 1957 return -EFAULT; 1958 out: 1959 return kvm_state.size; 1960 } 1961 1962 static int svm_set_nested_state(struct kvm_vcpu *vcpu, 1963 struct kvm_nested_state __user *user_kvm_nested_state, 1964 struct kvm_nested_state *kvm_state) 1965 { 1966 struct vcpu_svm *svm = to_svm(vcpu); 1967 struct vmcb __user *user_vmcb = (struct vmcb __user *) 1968 &user_kvm_nested_state->data.svm[0]; 1969 struct vmcb_control_area *ctl; 1970 struct vmcb_save_area *save; 1971 struct vmcb_save_area_cached save_cached; 1972 struct vmcb_ctrl_area_cached ctl_cached; 1973 bool use_separate_l2_pat; 1974 unsigned long cr0; 1975 int ret; 1976 1977 BUILD_BUG_ON(sizeof(struct vmcb_control_area) + sizeof(struct vmcb_save_area) > 1978 KVM_STATE_NESTED_SVM_VMCB_SIZE); 1979 1980 if (kvm_state->format != KVM_STATE_NESTED_FORMAT_SVM) 1981 return -EINVAL; 1982 1983 if (kvm_state->flags & ~(KVM_STATE_NESTED_GUEST_MODE | 1984 KVM_STATE_NESTED_RUN_PENDING | 1985 KVM_STATE_NESTED_GIF_SET)) 1986 return -EINVAL; 1987 1988 /* 1989 * If in guest mode, vcpu->arch.efer actually refers to the L2 guest's 1990 * EFER.SVME, but EFER.SVME still has to be 1 for VMRUN to succeed. 1991 * If SVME is disabled, the only valid states are "none" and GIF=1 1992 * (clearing SVME does NOT set GIF, i.e. GIF=0 is allowed). 1993 */ 1994 if (!(vcpu->arch.efer & EFER_SVME) && kvm_state->flags && 1995 kvm_state->flags != KVM_STATE_NESTED_GIF_SET) 1996 return -EINVAL; 1997 1998 /* SMM temporarily disables SVM, so we cannot be in guest mode. */ 1999 if (is_smm(vcpu) && (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 2000 return -EINVAL; 2001 2002 if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) { 2003 svm_leave_nested(vcpu); 2004 svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET)); 2005 return 0; 2006 } 2007 2008 if (!page_address_valid(vcpu, kvm_state->hdr.svm.vmcb_pa)) 2009 return -EINVAL; 2010 if (kvm_state->size < sizeof(*kvm_state) + KVM_STATE_NESTED_SVM_VMCB_SIZE) 2011 return -EINVAL; 2012 2013 ctl = memdup_user(&user_vmcb->control, sizeof(*ctl)); 2014 if (IS_ERR(ctl)) 2015 return PTR_ERR(ctl); 2016 2017 save = memdup_user(&user_vmcb->save, sizeof(*save)); 2018 if (IS_ERR(save)) { 2019 kfree(ctl); 2020 return PTR_ERR(save); 2021 } 2022 2023 ret = -EINVAL; 2024 __nested_copy_vmcb_control_to_cache(vcpu, &ctl_cached, ctl); 2025 if (!nested_vmcb_check_controls(vcpu, &ctl_cached)) 2026 goto out_free; 2027 2028 /* 2029 * Processor state contains L2 state. Check that it is 2030 * valid for guest mode (see nested_vmcb_check_save()). 2031 */ 2032 cr0 = kvm_read_cr0(vcpu); 2033 if (((cr0 & X86_CR0_CD) == 0) && (cr0 & X86_CR0_NW)) 2034 goto out_free; 2035 2036 /* 2037 * Validate host state saved from before VMRUN (see 2038 * nested_svm_check_permissions). Note that the g_pat field is not 2039 * validated, because (a) it may have been clobbered by SMM before 2040 * KVM_GET_NESTED_STATE, and (b) it is not loaded at emulated 2041 * #VMEXIT. 2042 */ 2043 __nested_copy_vmcb_save_to_cache(&save_cached, save); 2044 if (!(save->cr0 & X86_CR0_PG) || 2045 !(save->cr0 & X86_CR0_PE) || 2046 (save->rflags & X86_EFLAGS_VM) || 2047 ((save->efer & EFER_LMA) && !(save->efer & EFER_LME)) || 2048 !nested_vmcb_check_save(vcpu, &save_cached, false)) 2049 goto out_free; 2050 2051 /* 2052 * Validate gPAT when the shared PAT quirk is disabled (i.e. L2 2053 * has its own gPAT). This is done separately from the 2054 * vmcb_save_area_cached validation above, because gPAT is L2 2055 * state, but the vmcb_save_area_cached is populated with L1 state. 2056 */ 2057 use_separate_l2_pat = (ctl_cached.misc_ctl & SVM_MISC_ENABLE_NP) && 2058 !kvm_check_has_quirk(vcpu->kvm, 2059 KVM_X86_QUIRK_NESTED_SVM_SHARED_PAT); 2060 if (use_separate_l2_pat && !kvm_pat_valid(kvm_state->hdr.svm.gpat)) 2061 goto out_free; 2062 2063 /* 2064 * All checks done, we can enter guest mode. Userspace provides 2065 * vmcb12.control, which will be combined with L1 and stored into 2066 * vmcb02, and the L1 save state which we store in vmcb01. 2067 * L2 registers if needed are moved from the current VMCB to VMCB02. 2068 */ 2069 2070 if (is_guest_mode(vcpu)) 2071 svm_leave_nested(vcpu); 2072 else 2073 svm->nested.vmcb02.ptr->save = svm->vmcb01.ptr->save; 2074 2075 svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET)); 2076 2077 if (kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING) 2078 vcpu->arch.nested_run_pending = KVM_NESTED_RUN_PENDING_UNTRUSTED; 2079 else 2080 vcpu->arch.nested_run_pending = 0; 2081 2082 svm->nested.vmcb12_gpa = kvm_state->hdr.svm.vmcb_pa; 2083 2084 svm_copy_vmrun_state(vcpu, &svm->vmcb01.ptr->save, save); 2085 nested_copy_vmcb_control_to_cache(svm, ctl); 2086 2087 svm_switch_vmcb(svm, &svm->nested.vmcb02); 2088 2089 if (use_separate_l2_pat) 2090 vmcb_set_gpat(svm->vmcb, kvm_state->hdr.svm.gpat); 2091 2092 nested_vmcb02_prepare_control(svm); 2093 2094 /* 2095 * Any previously restored state (e.g. KVM_SET_SREGS) would mark fields 2096 * dirty in vmcb01 instead of vmcb02, so mark all of vmcb02 dirty here. 2097 */ 2098 vmcb_mark_all_dirty(svm->vmcb); 2099 2100 /* 2101 * While the nested guest CR3 is already checked and set by 2102 * KVM_SET_SREGS, it was set when nested state was yet loaded, 2103 * thus MMU might not be initialized correctly. 2104 * Set it again to fix this. 2105 */ 2106 ret = nested_svm_load_cr3(&svm->vcpu, vcpu->arch.cr3, 2107 nested_npt_enabled(svm), false); 2108 if (ret) 2109 goto out_free; 2110 2111 svm->nested.force_msr_bitmap_recalc = true; 2112 2113 if (kvm_vcpu_apicv_active(vcpu)) 2114 kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu); 2115 2116 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 2117 out_free: 2118 kfree(save); 2119 kfree(ctl); 2120 2121 return ret; 2122 } 2123 2124 static bool svm_get_nested_state_pages(struct kvm_vcpu *vcpu) 2125 { 2126 if (WARN_ON(!is_guest_mode(vcpu))) 2127 return true; 2128 2129 if (is_pae_paging(vcpu)) { 2130 /* 2131 * After migration, CR3 may have been restored before 2132 * KVM_SET_NESTED_STATE, so the PDPTR load into mmu->pdptrs[] 2133 * may have treated CR3 as an L1 GPA. For nNPT, drop the 2134 * cache so the next access reloads them with the proper 2135 * nGPA translation. For !nNPT, reload eagerly unless userspace 2136 * already supplied authoritative PDPTRs via KVM_SET_SREGS2. 2137 */ 2138 if (nested_npt_enabled(to_svm(vcpu))) 2139 kvm_register_mark_for_reload(vcpu, VCPU_REG_PDPTR); 2140 else if (!vcpu->arch.pdptrs_from_userspace && 2141 CC(!load_pdptrs(vcpu, vcpu->arch.cr3))) 2142 return false; 2143 } 2144 2145 if (!nested_svm_merge_msrpm(vcpu)) 2146 return false; 2147 2148 if (kvm_hv_verify_vp_assist(vcpu)) 2149 return false; 2150 2151 return true; 2152 } 2153 2154 static gpa_t svm_translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, 2155 u64 access, 2156 struct x86_exception *exception, 2157 u64 pte_access) 2158 { 2159 struct vcpu_svm *svm = to_svm(vcpu); 2160 struct kvm_pagewalk *w = &vcpu->arch.ngpa_walk; 2161 2162 if (WARN_ON_ONCE(!mmu_is_nested(vcpu))) 2163 return gpa; 2164 2165 /* Non-GMET walks are always user-walks */ 2166 if (!(svm->nested.ctl.misc_ctl & SVM_MISC_ENABLE_GMET)) 2167 access |= PFERR_USER_MASK; 2168 2169 return w->gva_to_gpa(vcpu, w, gpa, access, exception); 2170 } 2171 2172 struct kvm_x86_nested_ops svm_nested_ops __initdata = { 2173 .leave_nested = svm_leave_nested, 2174 .translate_nested_gpa = svm_translate_nested_gpa, 2175 .is_exception_vmexit = nested_svm_is_exception_vmexit, 2176 .check_events = svm_check_nested_events, 2177 .triple_fault = nested_svm_triple_fault, 2178 .get_nested_state_pages = svm_get_nested_state_pages, 2179 .get_state = svm_get_nested_state, 2180 .set_state = svm_set_nested_state, 2181 .hv_inject_synthetic_vmexit_post_tlb_flush = svm_hv_inject_synthetic_vmexit_post_tlb_flush, 2182 }; 2183