1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * AMD SVM support 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Yaniv Kamay <yaniv@qumranet.com> 12 * Avi Kivity <avi@qumranet.com> 13 */ 14 15 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 16 17 #include <linux/kvm_types.h> 18 #include <linux/kvm_host.h> 19 #include <linux/kernel.h> 20 21 #include <asm/msr-index.h> 22 #include <asm/debugreg.h> 23 24 #include "kvm_emulate.h" 25 #include "trace.h" 26 #include "mmu.h" 27 #include "x86.h" 28 #include "smm.h" 29 #include "cpuid.h" 30 #include "lapic.h" 31 #include "svm.h" 32 #include "hyperv.h" 33 34 #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK 35 36 static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu, 37 struct x86_exception *fault) 38 { 39 struct vcpu_svm *svm = to_svm(vcpu); 40 struct vmcb *vmcb = svm->vmcb; 41 42 if (vmcb->control.exit_code != SVM_EXIT_NPF) { 43 /* 44 * TODO: track the cause of the nested page fault, and 45 * correctly fill in the high bits of exit_info_1. 46 */ 47 vmcb->control.exit_code = SVM_EXIT_NPF; 48 vmcb->control.exit_info_1 = (1ULL << 32); 49 vmcb->control.exit_info_2 = fault->address; 50 } 51 52 vmcb->control.exit_info_1 &= ~0xffffffffULL; 53 vmcb->control.exit_info_1 |= fault->error_code; 54 55 nested_svm_vmexit(svm); 56 } 57 58 static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index) 59 { 60 struct vcpu_svm *svm = to_svm(vcpu); 61 u64 cr3 = svm->nested.ctl.nested_cr3; 62 u64 pdpte; 63 int ret; 64 65 /* 66 * Note, nCR3 is "assumed" to be 32-byte aligned, i.e. the CPU ignores 67 * nCR3[4:0] when loading PDPTEs from memory. 68 */ 69 ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(cr3), &pdpte, 70 (cr3 & GENMASK(11, 5)) + index * 8, 8); 71 if (ret) 72 return 0; 73 return pdpte; 74 } 75 76 static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu) 77 { 78 struct vcpu_svm *svm = to_svm(vcpu); 79 80 return svm->nested.ctl.nested_cr3; 81 } 82 83 static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu) 84 { 85 struct vcpu_svm *svm = to_svm(vcpu); 86 87 WARN_ON(mmu_is_nested(vcpu)); 88 89 vcpu->arch.mmu = &vcpu->arch.guest_mmu; 90 91 /* 92 * The NPT format depends on L1's CR4 and EFER, which is in vmcb01. Note, 93 * when called via KVM_SET_NESTED_STATE, that state may _not_ match current 94 * vCPU state. CR0.WP is explicitly ignored, while CR0.PG is required. 95 */ 96 kvm_init_shadow_npt_mmu(vcpu, X86_CR0_PG, svm->vmcb01.ptr->save.cr4, 97 svm->vmcb01.ptr->save.efer, 98 svm->nested.ctl.nested_cr3); 99 vcpu->arch.mmu->get_guest_pgd = nested_svm_get_tdp_cr3; 100 vcpu->arch.mmu->get_pdptr = nested_svm_get_tdp_pdptr; 101 vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit; 102 vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; 103 } 104 105 static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu) 106 { 107 vcpu->arch.mmu = &vcpu->arch.root_mmu; 108 vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; 109 } 110 111 static bool nested_vmcb_needs_vls_intercept(struct vcpu_svm *svm) 112 { 113 if (!guest_cpu_cap_has(&svm->vcpu, X86_FEATURE_V_VMSAVE_VMLOAD)) 114 return true; 115 116 if (!nested_npt_enabled(svm)) 117 return true; 118 119 if (!(svm->nested.ctl.virt_ext & VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK)) 120 return true; 121 122 return false; 123 } 124 125 void recalc_intercepts(struct vcpu_svm *svm) 126 { 127 struct vmcb_control_area *c, *h; 128 struct vmcb_ctrl_area_cached *g; 129 unsigned int i; 130 131 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 132 133 if (!is_guest_mode(&svm->vcpu)) 134 return; 135 136 c = &svm->vmcb->control; 137 h = &svm->vmcb01.ptr->control; 138 g = &svm->nested.ctl; 139 140 for (i = 0; i < MAX_INTERCEPT; i++) 141 c->intercepts[i] = h->intercepts[i]; 142 143 if (g->int_ctl & V_INTR_MASKING_MASK) { 144 /* 145 * If L2 is active and V_INTR_MASKING is enabled in vmcb12, 146 * disable intercept of CR8 writes as L2's CR8 does not affect 147 * any interrupt KVM may want to inject. 148 * 149 * Similarly, disable intercept of virtual interrupts (used to 150 * detect interrupt windows) if the saved RFLAGS.IF is '0', as 151 * the effective RFLAGS.IF for L1 interrupts will never be set 152 * while L2 is running (L2's RFLAGS.IF doesn't affect L1 IRQs). 153 */ 154 vmcb_clr_intercept(c, INTERCEPT_CR8_WRITE); 155 if (!(svm->vmcb01.ptr->save.rflags & X86_EFLAGS_IF)) 156 vmcb_clr_intercept(c, INTERCEPT_VINTR); 157 } 158 159 /* 160 * We want to see VMMCALLs from a nested guest only when Hyper-V L2 TLB 161 * flush feature is enabled. 162 */ 163 if (!nested_svm_l2_tlb_flush_enabled(&svm->vcpu)) 164 vmcb_clr_intercept(c, INTERCEPT_VMMCALL); 165 166 for (i = 0; i < MAX_INTERCEPT; i++) 167 c->intercepts[i] |= g->intercepts[i]; 168 169 /* If SMI is not intercepted, ignore guest SMI intercept as well */ 170 if (!intercept_smi) 171 vmcb_clr_intercept(c, INTERCEPT_SMI); 172 173 if (nested_vmcb_needs_vls_intercept(svm)) { 174 /* 175 * If the virtual VMLOAD/VMSAVE is not enabled for the L2, 176 * we must intercept these instructions to correctly 177 * emulate them in case L1 doesn't intercept them. 178 */ 179 vmcb_set_intercept(c, INTERCEPT_VMLOAD); 180 vmcb_set_intercept(c, INTERCEPT_VMSAVE); 181 } else { 182 WARN_ON(!(c->virt_ext & VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK)); 183 } 184 } 185 186 /* 187 * This array (and its actual size) holds the set of offsets (indexing by chunk 188 * size) to process when merging vmcb12's MSRPM with vmcb01's MSRPM. Note, the 189 * set of MSRs for which interception is disabled in vmcb01 is per-vCPU, e.g. 190 * based on CPUID features. This array only tracks MSRs that *might* be passed 191 * through to the guest. 192 * 193 * Hardcode the capacity of the array based on the maximum number of _offsets_. 194 * MSRs are batched together, so there are fewer offsets than MSRs. 195 */ 196 static int nested_svm_msrpm_merge_offsets[10] __ro_after_init; 197 static int nested_svm_nr_msrpm_merge_offsets __ro_after_init; 198 typedef unsigned long nsvm_msrpm_merge_t; 199 200 int __init nested_svm_init_msrpm_merge_offsets(void) 201 { 202 static const u32 merge_msrs[] __initconst = { 203 MSR_STAR, 204 MSR_IA32_SYSENTER_CS, 205 MSR_IA32_SYSENTER_EIP, 206 MSR_IA32_SYSENTER_ESP, 207 #ifdef CONFIG_X86_64 208 MSR_GS_BASE, 209 MSR_FS_BASE, 210 MSR_KERNEL_GS_BASE, 211 MSR_LSTAR, 212 MSR_CSTAR, 213 MSR_SYSCALL_MASK, 214 #endif 215 MSR_IA32_SPEC_CTRL, 216 MSR_IA32_PRED_CMD, 217 MSR_IA32_FLUSH_CMD, 218 MSR_IA32_APERF, 219 MSR_IA32_MPERF, 220 MSR_IA32_LASTBRANCHFROMIP, 221 MSR_IA32_LASTBRANCHTOIP, 222 MSR_IA32_LASTINTFROMIP, 223 MSR_IA32_LASTINTTOIP, 224 225 MSR_K7_PERFCTR0, 226 MSR_K7_PERFCTR1, 227 MSR_K7_PERFCTR2, 228 MSR_K7_PERFCTR3, 229 MSR_F15H_PERF_CTR0, 230 MSR_F15H_PERF_CTR1, 231 MSR_F15H_PERF_CTR2, 232 MSR_F15H_PERF_CTR3, 233 MSR_F15H_PERF_CTR4, 234 MSR_F15H_PERF_CTR5, 235 236 MSR_AMD64_PERF_CNTR_GLOBAL_CTL, 237 MSR_AMD64_PERF_CNTR_GLOBAL_STATUS, 238 MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR, 239 MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET, 240 }; 241 int i, j; 242 243 for (i = 0; i < ARRAY_SIZE(merge_msrs); i++) { 244 int bit_nr = svm_msrpm_bit_nr(merge_msrs[i]); 245 u32 offset; 246 247 if (WARN_ON(bit_nr < 0)) 248 return -EIO; 249 250 /* 251 * Merging is done in chunks to reduce the number of accesses 252 * to L1's bitmap. 253 */ 254 offset = bit_nr / BITS_PER_BYTE / sizeof(nsvm_msrpm_merge_t); 255 256 for (j = 0; j < nested_svm_nr_msrpm_merge_offsets; j++) { 257 if (nested_svm_msrpm_merge_offsets[j] == offset) 258 break; 259 } 260 261 if (j < nested_svm_nr_msrpm_merge_offsets) 262 continue; 263 264 if (WARN_ON(j >= ARRAY_SIZE(nested_svm_msrpm_merge_offsets))) 265 return -EIO; 266 267 nested_svm_msrpm_merge_offsets[j] = offset; 268 nested_svm_nr_msrpm_merge_offsets++; 269 } 270 271 return 0; 272 } 273 274 /* 275 * Merge L0's (KVM) and L1's (Nested VMCB) MSR permission bitmaps. The function 276 * is optimized in that it only merges the parts where KVM MSR permission bitmap 277 * may contain zero bits. 278 */ 279 static bool nested_svm_merge_msrpm(struct kvm_vcpu *vcpu) 280 { 281 struct vcpu_svm *svm = to_svm(vcpu); 282 nsvm_msrpm_merge_t *msrpm02 = svm->nested.msrpm; 283 nsvm_msrpm_merge_t *msrpm01 = svm->msrpm; 284 int i; 285 286 /* 287 * MSR bitmap update can be skipped when: 288 * - MSR bitmap for L1 hasn't changed. 289 * - Nested hypervisor (L1) is attempting to launch the same L2 as 290 * before. 291 * - Nested hypervisor (L1) is using Hyper-V emulation interface and 292 * tells KVM (L0) there were no changes in MSR bitmap for L2. 293 */ 294 #ifdef CONFIG_KVM_HYPERV 295 if (!svm->nested.force_msr_bitmap_recalc) { 296 struct hv_vmcb_enlightenments *hve = &svm->nested.ctl.hv_enlightenments; 297 298 if (kvm_hv_hypercall_enabled(vcpu) && 299 hve->hv_enlightenments_control.msr_bitmap && 300 (svm->nested.ctl.clean & BIT(HV_VMCB_NESTED_ENLIGHTENMENTS))) 301 goto set_msrpm_base_pa; 302 } 303 #endif 304 305 if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT))) 306 return true; 307 308 for (i = 0; i < nested_svm_nr_msrpm_merge_offsets; i++) { 309 const int p = nested_svm_msrpm_merge_offsets[i]; 310 nsvm_msrpm_merge_t l1_val; 311 gpa_t gpa; 312 313 gpa = svm->nested.ctl.msrpm_base_pa + (p * sizeof(l1_val)); 314 315 if (kvm_vcpu_read_guest(vcpu, gpa, &l1_val, sizeof(l1_val))) 316 return false; 317 318 msrpm02[p] = msrpm01[p] | l1_val; 319 } 320 321 svm->nested.force_msr_bitmap_recalc = false; 322 323 #ifdef CONFIG_KVM_HYPERV 324 set_msrpm_base_pa: 325 #endif 326 svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm)); 327 328 return true; 329 } 330 331 /* 332 * Bits 11:0 of bitmap address are ignored by hardware 333 */ 334 static bool nested_svm_check_bitmap_pa(struct kvm_vcpu *vcpu, u64 pa, u32 size) 335 { 336 u64 addr = PAGE_ALIGN(pa); 337 338 return kvm_vcpu_is_legal_gpa(vcpu, addr) && 339 kvm_vcpu_is_legal_gpa(vcpu, addr + size - 1); 340 } 341 342 static bool __nested_vmcb_check_controls(struct kvm_vcpu *vcpu, 343 struct vmcb_ctrl_area_cached *control) 344 { 345 if (CC(!vmcb12_is_intercept(control, INTERCEPT_VMRUN))) 346 return false; 347 348 if (CC(control->asid == 0)) 349 return false; 350 351 if (CC((control->nested_ctl & SVM_NESTED_CTL_NP_ENABLE) && !npt_enabled)) 352 return false; 353 354 if (CC(!nested_svm_check_bitmap_pa(vcpu, control->msrpm_base_pa, 355 MSRPM_SIZE))) 356 return false; 357 if (CC(!nested_svm_check_bitmap_pa(vcpu, control->iopm_base_pa, 358 IOPM_SIZE))) 359 return false; 360 361 if (CC((control->int_ctl & V_NMI_ENABLE_MASK) && 362 !vmcb12_is_intercept(control, INTERCEPT_NMI))) { 363 return false; 364 } 365 366 return true; 367 } 368 369 /* Common checks that apply to both L1 and L2 state. */ 370 static bool __nested_vmcb_check_save(struct kvm_vcpu *vcpu, 371 struct vmcb_save_area_cached *save) 372 { 373 if (CC(!(save->efer & EFER_SVME))) 374 return false; 375 376 if (CC((save->cr0 & X86_CR0_CD) == 0 && (save->cr0 & X86_CR0_NW)) || 377 CC(save->cr0 & ~0xffffffffULL)) 378 return false; 379 380 if (CC(!kvm_dr6_valid(save->dr6)) || CC(!kvm_dr7_valid(save->dr7))) 381 return false; 382 383 /* 384 * These checks are also performed by KVM_SET_SREGS, 385 * except that EFER.LMA is not checked by SVM against 386 * CR0.PG && EFER.LME. 387 */ 388 if ((save->efer & EFER_LME) && (save->cr0 & X86_CR0_PG)) { 389 if (CC(!(save->cr4 & X86_CR4_PAE)) || 390 CC(!(save->cr0 & X86_CR0_PE)) || 391 CC(!kvm_vcpu_is_legal_cr3(vcpu, save->cr3))) 392 return false; 393 } 394 395 /* Note, SVM doesn't have any additional restrictions on CR4. */ 396 if (CC(!__kvm_is_valid_cr4(vcpu, save->cr4))) 397 return false; 398 399 if (CC(!kvm_valid_efer(vcpu, save->efer))) 400 return false; 401 402 return true; 403 } 404 405 static bool nested_vmcb_check_save(struct kvm_vcpu *vcpu) 406 { 407 struct vcpu_svm *svm = to_svm(vcpu); 408 struct vmcb_save_area_cached *save = &svm->nested.save; 409 410 return __nested_vmcb_check_save(vcpu, save); 411 } 412 413 static bool nested_vmcb_check_controls(struct kvm_vcpu *vcpu) 414 { 415 struct vcpu_svm *svm = to_svm(vcpu); 416 struct vmcb_ctrl_area_cached *ctl = &svm->nested.ctl; 417 418 return __nested_vmcb_check_controls(vcpu, ctl); 419 } 420 421 int nested_svm_check_cached_vmcb12(struct kvm_vcpu *vcpu) 422 { 423 if (!nested_vmcb_check_save(vcpu) || 424 !nested_vmcb_check_controls(vcpu)) 425 return -EINVAL; 426 427 return 0; 428 } 429 430 /* 431 * If a feature is not advertised to L1, clear the corresponding vmcb12 432 * intercept. 433 */ 434 #define __nested_svm_sanitize_intercept(__vcpu, __control, fname, iname) \ 435 do { \ 436 if (!guest_cpu_cap_has(__vcpu, X86_FEATURE_##fname)) \ 437 vmcb12_clr_intercept(__control, INTERCEPT_##iname); \ 438 } while (0) 439 440 #define nested_svm_sanitize_intercept(__vcpu, __control, name) \ 441 __nested_svm_sanitize_intercept(__vcpu, __control, name, name) 442 443 static 444 void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu, 445 struct vmcb_ctrl_area_cached *to, 446 struct vmcb_control_area *from) 447 { 448 unsigned int i; 449 450 for (i = 0; i < MAX_INTERCEPT; i++) 451 to->intercepts[i] = from->intercepts[i]; 452 453 __nested_svm_sanitize_intercept(vcpu, to, XSAVE, XSETBV); 454 nested_svm_sanitize_intercept(vcpu, to, INVPCID); 455 nested_svm_sanitize_intercept(vcpu, to, RDTSCP); 456 nested_svm_sanitize_intercept(vcpu, to, SKINIT); 457 nested_svm_sanitize_intercept(vcpu, to, RDPRU); 458 459 to->iopm_base_pa = from->iopm_base_pa; 460 to->msrpm_base_pa = from->msrpm_base_pa; 461 to->tsc_offset = from->tsc_offset; 462 to->tlb_ctl = from->tlb_ctl; 463 to->erap_ctl = from->erap_ctl; 464 to->int_ctl = from->int_ctl; 465 to->int_vector = from->int_vector; 466 to->int_state = from->int_state; 467 to->exit_code = from->exit_code; 468 to->exit_info_1 = from->exit_info_1; 469 to->exit_info_2 = from->exit_info_2; 470 to->exit_int_info = from->exit_int_info; 471 to->exit_int_info_err = from->exit_int_info_err; 472 to->nested_ctl = from->nested_ctl; 473 to->event_inj = from->event_inj; 474 to->event_inj_err = from->event_inj_err; 475 to->next_rip = from->next_rip; 476 to->nested_cr3 = from->nested_cr3; 477 to->virt_ext = from->virt_ext; 478 to->pause_filter_count = from->pause_filter_count; 479 to->pause_filter_thresh = from->pause_filter_thresh; 480 481 /* Copy asid here because nested_vmcb_check_controls will check it. */ 482 to->asid = from->asid; 483 to->msrpm_base_pa &= ~0x0fffULL; 484 to->iopm_base_pa &= ~0x0fffULL; 485 486 #ifdef CONFIG_KVM_HYPERV 487 /* Hyper-V extensions (Enlightened VMCB) */ 488 if (kvm_hv_hypercall_enabled(vcpu)) { 489 to->clean = from->clean; 490 memcpy(&to->hv_enlightenments, &from->hv_enlightenments, 491 sizeof(to->hv_enlightenments)); 492 } 493 #endif 494 } 495 496 void nested_copy_vmcb_control_to_cache(struct vcpu_svm *svm, 497 struct vmcb_control_area *control) 498 { 499 __nested_copy_vmcb_control_to_cache(&svm->vcpu, &svm->nested.ctl, control); 500 } 501 502 static void __nested_copy_vmcb_save_to_cache(struct vmcb_save_area_cached *to, 503 struct vmcb_save_area *from) 504 { 505 /* 506 * Copy only fields that are validated, as we need them 507 * to avoid TOC/TOU races. 508 */ 509 to->efer = from->efer; 510 to->cr0 = from->cr0; 511 to->cr3 = from->cr3; 512 to->cr4 = from->cr4; 513 514 to->dr6 = from->dr6; 515 to->dr7 = from->dr7; 516 } 517 518 void nested_copy_vmcb_save_to_cache(struct vcpu_svm *svm, 519 struct vmcb_save_area *save) 520 { 521 __nested_copy_vmcb_save_to_cache(&svm->nested.save, save); 522 } 523 524 /* 525 * Synchronize fields that are written by the processor, so that 526 * they can be copied back into the vmcb12. 527 */ 528 void nested_sync_control_from_vmcb02(struct vcpu_svm *svm) 529 { 530 u32 mask; 531 svm->nested.ctl.event_inj = svm->vmcb->control.event_inj; 532 svm->nested.ctl.event_inj_err = svm->vmcb->control.event_inj_err; 533 534 /* Only a few fields of int_ctl are written by the processor. */ 535 mask = V_IRQ_MASK | V_TPR_MASK; 536 /* 537 * Don't sync vmcb02 V_IRQ back to vmcb12 if KVM (L0) is intercepting 538 * virtual interrupts in order to request an interrupt window, as KVM 539 * has usurped vmcb02's int_ctl. If an interrupt window opens before 540 * the next VM-Exit, svm_clear_vintr() will restore vmcb12's int_ctl. 541 * If no window opens, V_IRQ will be correctly preserved in vmcb12's 542 * int_ctl (because it was never recognized while L2 was running). 543 */ 544 if (svm_is_intercept(svm, INTERCEPT_VINTR) && 545 !test_bit(INTERCEPT_VINTR, (unsigned long *)svm->nested.ctl.intercepts)) 546 mask &= ~V_IRQ_MASK; 547 548 if (nested_vgif_enabled(svm)) 549 mask |= V_GIF_MASK; 550 551 if (nested_vnmi_enabled(svm)) 552 mask |= V_NMI_BLOCKING_MASK | V_NMI_PENDING_MASK; 553 554 svm->nested.ctl.int_ctl &= ~mask; 555 svm->nested.ctl.int_ctl |= svm->vmcb->control.int_ctl & mask; 556 } 557 558 /* 559 * Transfer any event that L0 or L1 wanted to inject into L2 to 560 * EXIT_INT_INFO. 561 */ 562 static void nested_save_pending_event_to_vmcb12(struct vcpu_svm *svm, 563 struct vmcb *vmcb12) 564 { 565 struct kvm_vcpu *vcpu = &svm->vcpu; 566 u32 exit_int_info = 0; 567 unsigned int nr; 568 569 if (vcpu->arch.exception.injected) { 570 nr = vcpu->arch.exception.vector; 571 exit_int_info = nr | SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_EXEPT; 572 573 if (vcpu->arch.exception.has_error_code) { 574 exit_int_info |= SVM_EVTINJ_VALID_ERR; 575 vmcb12->control.exit_int_info_err = 576 vcpu->arch.exception.error_code; 577 } 578 579 } else if (vcpu->arch.nmi_injected) { 580 exit_int_info = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI; 581 582 } else if (vcpu->arch.interrupt.injected) { 583 nr = vcpu->arch.interrupt.nr; 584 exit_int_info = nr | SVM_EVTINJ_VALID; 585 586 if (vcpu->arch.interrupt.soft) 587 exit_int_info |= SVM_EVTINJ_TYPE_SOFT; 588 else 589 exit_int_info |= SVM_EVTINJ_TYPE_INTR; 590 } 591 592 vmcb12->control.exit_int_info = exit_int_info; 593 } 594 595 static void nested_svm_transition_tlb_flush(struct kvm_vcpu *vcpu) 596 { 597 /* Handle pending Hyper-V TLB flush requests */ 598 kvm_hv_nested_transtion_tlb_flush(vcpu, npt_enabled); 599 600 /* 601 * TODO: optimize unconditional TLB flush/MMU sync. A partial list of 602 * things to fix before this can be conditional: 603 * 604 * - Flush TLBs for both L1 and L2 remote TLB flush 605 * - Honor L1's request to flush an ASID on nested VMRUN 606 * - Sync nested NPT MMU on VMRUN that flushes L2's ASID[*] 607 * - Don't crush a pending TLB flush in vmcb02 on nested VMRUN 608 * - Flush L1's ASID on KVM_REQ_TLB_FLUSH_GUEST 609 * 610 * [*] Unlike nested EPT, SVM's ASID management can invalidate nested 611 * NPT guest-physical mappings on VMRUN. 612 */ 613 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); 614 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 615 } 616 617 /* 618 * Load guest's/host's cr3 on nested vmentry or vmexit. @nested_npt is true 619 * if we are emulating VM-Entry into a guest with NPT enabled. 620 */ 621 static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, 622 bool nested_npt, bool reload_pdptrs) 623 { 624 if (CC(!kvm_vcpu_is_legal_cr3(vcpu, cr3))) 625 return -EINVAL; 626 627 if (reload_pdptrs && !nested_npt && is_pae_paging(vcpu) && 628 CC(!load_pdptrs(vcpu, cr3))) 629 return -EINVAL; 630 631 vcpu->arch.cr3 = cr3; 632 633 /* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */ 634 kvm_init_mmu(vcpu); 635 636 if (!nested_npt) 637 kvm_mmu_new_pgd(vcpu, cr3); 638 639 return 0; 640 } 641 642 void nested_vmcb02_compute_g_pat(struct vcpu_svm *svm) 643 { 644 if (!svm->nested.vmcb02.ptr) 645 return; 646 647 /* FIXME: merge g_pat from vmcb01 and vmcb12. */ 648 svm->nested.vmcb02.ptr->save.g_pat = svm->vmcb01.ptr->save.g_pat; 649 } 650 651 static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12) 652 { 653 bool new_vmcb12 = false; 654 struct vmcb *vmcb01 = svm->vmcb01.ptr; 655 struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; 656 struct kvm_vcpu *vcpu = &svm->vcpu; 657 658 nested_vmcb02_compute_g_pat(svm); 659 vmcb_mark_dirty(vmcb02, VMCB_NPT); 660 661 /* Load the nested guest state */ 662 if (svm->nested.vmcb12_gpa != svm->nested.last_vmcb12_gpa) { 663 new_vmcb12 = true; 664 svm->nested.last_vmcb12_gpa = svm->nested.vmcb12_gpa; 665 svm->nested.force_msr_bitmap_recalc = true; 666 } 667 668 if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_SEG))) { 669 vmcb02->save.es = vmcb12->save.es; 670 vmcb02->save.cs = vmcb12->save.cs; 671 vmcb02->save.ss = vmcb12->save.ss; 672 vmcb02->save.ds = vmcb12->save.ds; 673 vmcb02->save.cpl = vmcb12->save.cpl; 674 vmcb_mark_dirty(vmcb02, VMCB_SEG); 675 } 676 677 if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_DT))) { 678 vmcb02->save.gdtr = vmcb12->save.gdtr; 679 vmcb02->save.idtr = vmcb12->save.idtr; 680 vmcb_mark_dirty(vmcb02, VMCB_DT); 681 } 682 683 if (guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) && 684 (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_CET)))) { 685 vmcb02->save.s_cet = vmcb12->save.s_cet; 686 vmcb02->save.isst_addr = vmcb12->save.isst_addr; 687 vmcb02->save.ssp = vmcb12->save.ssp; 688 vmcb_mark_dirty(vmcb02, VMCB_CET); 689 } 690 691 kvm_set_rflags(vcpu, vmcb12->save.rflags | X86_EFLAGS_FIXED); 692 693 svm_set_efer(vcpu, svm->nested.save.efer); 694 695 svm_set_cr0(vcpu, svm->nested.save.cr0); 696 svm_set_cr4(vcpu, svm->nested.save.cr4); 697 698 svm->vcpu.arch.cr2 = vmcb12->save.cr2; 699 700 kvm_rax_write(vcpu, vmcb12->save.rax); 701 kvm_rsp_write(vcpu, vmcb12->save.rsp); 702 kvm_rip_write(vcpu, vmcb12->save.rip); 703 704 /* In case we don't even reach vcpu_run, the fields are not updated */ 705 vmcb02->save.rax = vmcb12->save.rax; 706 vmcb02->save.rsp = vmcb12->save.rsp; 707 vmcb02->save.rip = vmcb12->save.rip; 708 709 if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_DR))) { 710 vmcb02->save.dr7 = svm->nested.save.dr7 | DR7_FIXED_1; 711 svm->vcpu.arch.dr6 = svm->nested.save.dr6 | DR6_ACTIVE_LOW; 712 vmcb_mark_dirty(vmcb02, VMCB_DR); 713 } 714 715 if (unlikely(guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) && 716 (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK))) { 717 /* 718 * Reserved bits of DEBUGCTL are ignored. Be consistent with 719 * svm_set_msr's definition of reserved bits. 720 */ 721 svm_copy_lbrs(vmcb02, vmcb12); 722 vmcb02->save.dbgctl &= ~DEBUGCTL_RESERVED_BITS; 723 } else { 724 svm_copy_lbrs(vmcb02, vmcb01); 725 } 726 svm_update_lbrv(&svm->vcpu); 727 } 728 729 static inline bool is_evtinj_soft(u32 evtinj) 730 { 731 u32 type = evtinj & SVM_EVTINJ_TYPE_MASK; 732 u8 vector = evtinj & SVM_EVTINJ_VEC_MASK; 733 734 if (!(evtinj & SVM_EVTINJ_VALID)) 735 return false; 736 737 if (type == SVM_EVTINJ_TYPE_SOFT) 738 return true; 739 740 return type == SVM_EVTINJ_TYPE_EXEPT && kvm_exception_is_soft(vector); 741 } 742 743 static bool is_evtinj_nmi(u32 evtinj) 744 { 745 u32 type = evtinj & SVM_EVTINJ_TYPE_MASK; 746 747 if (!(evtinj & SVM_EVTINJ_VALID)) 748 return false; 749 750 return type == SVM_EVTINJ_TYPE_NMI; 751 } 752 753 static void nested_vmcb02_prepare_control(struct vcpu_svm *svm, 754 unsigned long vmcb12_rip, 755 unsigned long vmcb12_csbase) 756 { 757 u32 int_ctl_vmcb01_bits = V_INTR_MASKING_MASK; 758 u32 int_ctl_vmcb12_bits = V_TPR_MASK | V_IRQ_INJECTION_BITS_MASK; 759 760 struct kvm_vcpu *vcpu = &svm->vcpu; 761 struct vmcb *vmcb01 = svm->vmcb01.ptr; 762 struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; 763 u32 pause_count12; 764 u32 pause_thresh12; 765 766 nested_svm_transition_tlb_flush(vcpu); 767 768 /* Enter Guest-Mode */ 769 enter_guest_mode(vcpu); 770 771 /* 772 * Filled at exit: exit_code, exit_info_1, exit_info_2, exit_int_info, 773 * exit_int_info_err, next_rip, insn_len, insn_bytes. 774 */ 775 776 if (guest_cpu_cap_has(vcpu, X86_FEATURE_VGIF) && 777 (svm->nested.ctl.int_ctl & V_GIF_ENABLE_MASK)) 778 int_ctl_vmcb12_bits |= (V_GIF_MASK | V_GIF_ENABLE_MASK); 779 else 780 int_ctl_vmcb01_bits |= (V_GIF_MASK | V_GIF_ENABLE_MASK); 781 782 if (vnmi) { 783 if (vmcb01->control.int_ctl & V_NMI_PENDING_MASK) { 784 svm->vcpu.arch.nmi_pending++; 785 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); 786 } 787 if (nested_vnmi_enabled(svm)) 788 int_ctl_vmcb12_bits |= (V_NMI_PENDING_MASK | 789 V_NMI_ENABLE_MASK | 790 V_NMI_BLOCKING_MASK); 791 } 792 793 /* Copied from vmcb01. msrpm_base can be overwritten later. */ 794 vmcb02->control.nested_ctl = vmcb01->control.nested_ctl; 795 vmcb02->control.iopm_base_pa = vmcb01->control.iopm_base_pa; 796 vmcb02->control.msrpm_base_pa = vmcb01->control.msrpm_base_pa; 797 vmcb_mark_dirty(vmcb02, VMCB_PERM_MAP); 798 799 /* 800 * Stash vmcb02's counter if the guest hasn't moved past the guilty 801 * instruction; otherwise, reset the counter to '0'. 802 * 803 * In order to detect if L2 has made forward progress or not, track the 804 * RIP at which a bus lock has occurred on a per-vmcb12 basis. If RIP 805 * is changed, guest has clearly made forward progress, bus_lock_counter 806 * still remained '1', so reset bus_lock_counter to '0'. Eg. In the 807 * scenario, where a buslock happened in L1 before VMRUN, the bus lock 808 * firmly happened on an instruction in the past. Even if vmcb01's 809 * counter is still '1', (because the guilty instruction got patched), 810 * the vCPU has clearly made forward progress and so KVM should reset 811 * vmcb02's counter to '0'. 812 * 813 * If the RIP hasn't changed, stash the bus lock counter at nested VMRUN 814 * to prevent the same guilty instruction from triggering a VM-Exit. Eg. 815 * if userspace rate-limits the vCPU, then it's entirely possible that 816 * L1's tick interrupt is pending by the time userspace re-runs the 817 * vCPU. If KVM unconditionally clears the counter on VMRUN, then when 818 * L1 re-enters L2, the same instruction will trigger a VM-Exit and the 819 * entire cycle start over. 820 */ 821 if (vmcb02->save.rip && (svm->nested.ctl.bus_lock_rip == vmcb02->save.rip)) 822 vmcb02->control.bus_lock_counter = 1; 823 else 824 vmcb02->control.bus_lock_counter = 0; 825 826 /* Done at vmrun: asid. */ 827 828 /* Also overwritten later if necessary. */ 829 vmcb02->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; 830 831 /* nested_cr3. */ 832 if (nested_npt_enabled(svm)) 833 nested_svm_init_mmu_context(vcpu); 834 835 vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset( 836 vcpu->arch.l1_tsc_offset, 837 svm->nested.ctl.tsc_offset, 838 svm->tsc_ratio_msr); 839 840 vmcb02->control.tsc_offset = vcpu->arch.tsc_offset; 841 842 if (guest_cpu_cap_has(vcpu, X86_FEATURE_TSCRATEMSR) && 843 svm->tsc_ratio_msr != kvm_caps.default_tsc_scaling_ratio) 844 nested_svm_update_tsc_ratio_msr(vcpu); 845 846 vmcb02->control.int_ctl = 847 (svm->nested.ctl.int_ctl & int_ctl_vmcb12_bits) | 848 (vmcb01->control.int_ctl & int_ctl_vmcb01_bits); 849 850 vmcb02->control.int_vector = svm->nested.ctl.int_vector; 851 vmcb02->control.int_state = svm->nested.ctl.int_state; 852 vmcb02->control.event_inj = svm->nested.ctl.event_inj; 853 vmcb02->control.event_inj_err = svm->nested.ctl.event_inj_err; 854 855 /* 856 * next_rip is consumed on VMRUN as the return address pushed on the 857 * stack for injected soft exceptions/interrupts. If nrips is exposed 858 * to L1, take it verbatim from vmcb12. If nrips is supported in 859 * hardware but not exposed to L1, stuff the actual L2 RIP to emulate 860 * what a nrips=0 CPU would do (L1 is responsible for advancing RIP 861 * prior to injecting the event). 862 */ 863 if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS)) 864 vmcb02->control.next_rip = svm->nested.ctl.next_rip; 865 else if (boot_cpu_has(X86_FEATURE_NRIPS)) 866 vmcb02->control.next_rip = vmcb12_rip; 867 868 svm->nmi_l1_to_l2 = is_evtinj_nmi(vmcb02->control.event_inj); 869 if (is_evtinj_soft(vmcb02->control.event_inj)) { 870 svm->soft_int_injected = true; 871 svm->soft_int_csbase = vmcb12_csbase; 872 svm->soft_int_old_rip = vmcb12_rip; 873 if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS)) 874 svm->soft_int_next_rip = svm->nested.ctl.next_rip; 875 else 876 svm->soft_int_next_rip = vmcb12_rip; 877 } 878 879 /* LBR_CTL_ENABLE_MASK is controlled by svm_update_lbrv() */ 880 881 if (!nested_vmcb_needs_vls_intercept(svm)) 882 vmcb02->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; 883 884 if (guest_cpu_cap_has(vcpu, X86_FEATURE_PAUSEFILTER)) 885 pause_count12 = svm->nested.ctl.pause_filter_count; 886 else 887 pause_count12 = 0; 888 if (guest_cpu_cap_has(vcpu, X86_FEATURE_PFTHRESHOLD)) 889 pause_thresh12 = svm->nested.ctl.pause_filter_thresh; 890 else 891 pause_thresh12 = 0; 892 if (kvm_pause_in_guest(svm->vcpu.kvm)) { 893 /* use guest values since host doesn't intercept PAUSE */ 894 vmcb02->control.pause_filter_count = pause_count12; 895 vmcb02->control.pause_filter_thresh = pause_thresh12; 896 897 } else { 898 /* start from host values otherwise */ 899 vmcb02->control.pause_filter_count = vmcb01->control.pause_filter_count; 900 vmcb02->control.pause_filter_thresh = vmcb01->control.pause_filter_thresh; 901 902 /* ... but ensure filtering is disabled if so requested. */ 903 if (vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_PAUSE)) { 904 if (!pause_count12) 905 vmcb02->control.pause_filter_count = 0; 906 if (!pause_thresh12) 907 vmcb02->control.pause_filter_thresh = 0; 908 } 909 } 910 911 /* 912 * Take ALLOW_LARGER_RAP from vmcb12 even though it should be safe to 913 * let L2 use a larger RAP since KVM will emulate the necessary clears, 914 * as it's possible L1 deliberately wants to restrict L2 to the legacy 915 * RAP size. Unconditionally clear the RAP on nested VMRUN, as KVM is 916 * responsible for emulating the host vs. guest tags (L1 is the "host", 917 * L2 is the "guest"). 918 */ 919 if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS)) 920 vmcb02->control.erap_ctl = (svm->nested.ctl.erap_ctl & 921 ERAP_CONTROL_ALLOW_LARGER_RAP) | 922 ERAP_CONTROL_CLEAR_RAP; 923 924 /* 925 * Merge guest and host intercepts - must be called with vcpu in 926 * guest-mode to take effect. 927 */ 928 recalc_intercepts(svm); 929 } 930 931 static void nested_svm_copy_common_state(struct vmcb *from_vmcb, struct vmcb *to_vmcb) 932 { 933 /* 934 * Some VMCB state is shared between L1 and L2 and thus has to be 935 * moved at the time of nested vmrun and vmexit. 936 * 937 * VMLOAD/VMSAVE state would also belong in this category, but KVM 938 * always performs VMLOAD and VMSAVE from the VMCB01. 939 */ 940 to_vmcb->save.spec_ctrl = from_vmcb->save.spec_ctrl; 941 } 942 943 int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb12_gpa, 944 struct vmcb *vmcb12, bool from_vmrun) 945 { 946 struct vcpu_svm *svm = to_svm(vcpu); 947 int ret; 948 949 trace_kvm_nested_vmenter(svm->vmcb->save.rip, 950 vmcb12_gpa, 951 vmcb12->save.rip, 952 vmcb12->control.int_ctl, 953 vmcb12->control.event_inj, 954 vmcb12->control.nested_ctl, 955 vmcb12->control.nested_cr3, 956 vmcb12->save.cr3, 957 KVM_ISA_SVM); 958 959 trace_kvm_nested_intercepts(vmcb12->control.intercepts[INTERCEPT_CR] & 0xffff, 960 vmcb12->control.intercepts[INTERCEPT_CR] >> 16, 961 vmcb12->control.intercepts[INTERCEPT_EXCEPTION], 962 vmcb12->control.intercepts[INTERCEPT_WORD3], 963 vmcb12->control.intercepts[INTERCEPT_WORD4], 964 vmcb12->control.intercepts[INTERCEPT_WORD5]); 965 966 967 svm->nested.vmcb12_gpa = vmcb12_gpa; 968 969 WARN_ON(svm->vmcb == svm->nested.vmcb02.ptr); 970 971 nested_svm_copy_common_state(svm->vmcb01.ptr, svm->nested.vmcb02.ptr); 972 973 svm_switch_vmcb(svm, &svm->nested.vmcb02); 974 nested_vmcb02_prepare_control(svm, vmcb12->save.rip, vmcb12->save.cs.base); 975 nested_vmcb02_prepare_save(svm, vmcb12); 976 977 ret = nested_svm_load_cr3(&svm->vcpu, svm->nested.save.cr3, 978 nested_npt_enabled(svm), from_vmrun); 979 if (ret) 980 return ret; 981 982 if (!from_vmrun) 983 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 984 985 svm_set_gif(svm, true); 986 987 if (kvm_vcpu_apicv_active(vcpu)) 988 kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu); 989 990 nested_svm_hv_update_vm_vp_ids(vcpu); 991 992 return 0; 993 } 994 995 int nested_svm_vmrun(struct kvm_vcpu *vcpu) 996 { 997 struct vcpu_svm *svm = to_svm(vcpu); 998 int ret; 999 struct vmcb *vmcb12; 1000 struct kvm_host_map map; 1001 u64 vmcb12_gpa; 1002 struct vmcb *vmcb01 = svm->vmcb01.ptr; 1003 1004 if (!svm->nested.hsave_msr) { 1005 kvm_inject_gp(vcpu, 0); 1006 return 1; 1007 } 1008 1009 if (is_smm(vcpu)) { 1010 kvm_queue_exception(vcpu, UD_VECTOR); 1011 return 1; 1012 } 1013 1014 /* This fails when VP assist page is enabled but the supplied GPA is bogus */ 1015 ret = kvm_hv_verify_vp_assist(vcpu); 1016 if (ret) { 1017 kvm_inject_gp(vcpu, 0); 1018 return ret; 1019 } 1020 1021 vmcb12_gpa = svm->vmcb->save.rax; 1022 ret = kvm_vcpu_map(vcpu, gpa_to_gfn(vmcb12_gpa), &map); 1023 if (ret == -EINVAL) { 1024 kvm_inject_gp(vcpu, 0); 1025 return 1; 1026 } else if (ret) { 1027 return kvm_skip_emulated_instruction(vcpu); 1028 } 1029 1030 ret = kvm_skip_emulated_instruction(vcpu); 1031 1032 vmcb12 = map.hva; 1033 1034 if (WARN_ON_ONCE(!svm->nested.initialized)) 1035 return -EINVAL; 1036 1037 nested_copy_vmcb_control_to_cache(svm, &vmcb12->control); 1038 nested_copy_vmcb_save_to_cache(svm, &vmcb12->save); 1039 1040 if (nested_svm_check_cached_vmcb12(vcpu) < 0) { 1041 vmcb12->control.exit_code = SVM_EXIT_ERR; 1042 vmcb12->control.exit_info_1 = 0; 1043 vmcb12->control.exit_info_2 = 0; 1044 goto out; 1045 } 1046 1047 /* 1048 * Since vmcb01 is not in use, we can use it to store some of the L1 1049 * state. 1050 */ 1051 vmcb01->save.efer = vcpu->arch.efer; 1052 vmcb01->save.cr0 = kvm_read_cr0(vcpu); 1053 vmcb01->save.cr4 = vcpu->arch.cr4; 1054 vmcb01->save.rflags = kvm_get_rflags(vcpu); 1055 vmcb01->save.rip = kvm_rip_read(vcpu); 1056 1057 if (!npt_enabled) 1058 vmcb01->save.cr3 = kvm_read_cr3(vcpu); 1059 1060 svm->nested.nested_run_pending = 1; 1061 1062 if (enter_svm_guest_mode(vcpu, vmcb12_gpa, vmcb12, true)) 1063 goto out_exit_err; 1064 1065 if (nested_svm_merge_msrpm(vcpu)) 1066 goto out; 1067 1068 out_exit_err: 1069 svm->nested.nested_run_pending = 0; 1070 svm->nmi_l1_to_l2 = false; 1071 svm->soft_int_injected = false; 1072 1073 svm->vmcb->control.exit_code = SVM_EXIT_ERR; 1074 svm->vmcb->control.exit_info_1 = 0; 1075 svm->vmcb->control.exit_info_2 = 0; 1076 1077 nested_svm_vmexit(svm); 1078 1079 out: 1080 kvm_vcpu_unmap(vcpu, &map); 1081 1082 return ret; 1083 } 1084 1085 /* Copy state save area fields which are handled by VMRUN */ 1086 void svm_copy_vmrun_state(struct vmcb_save_area *to_save, 1087 struct vmcb_save_area *from_save) 1088 { 1089 to_save->es = from_save->es; 1090 to_save->cs = from_save->cs; 1091 to_save->ss = from_save->ss; 1092 to_save->ds = from_save->ds; 1093 to_save->gdtr = from_save->gdtr; 1094 to_save->idtr = from_save->idtr; 1095 to_save->rflags = from_save->rflags | X86_EFLAGS_FIXED; 1096 to_save->efer = from_save->efer; 1097 to_save->cr0 = from_save->cr0; 1098 to_save->cr3 = from_save->cr3; 1099 to_save->cr4 = from_save->cr4; 1100 to_save->rax = from_save->rax; 1101 to_save->rsp = from_save->rsp; 1102 to_save->rip = from_save->rip; 1103 to_save->cpl = 0; 1104 1105 if (kvm_cpu_cap_has(X86_FEATURE_SHSTK)) { 1106 to_save->s_cet = from_save->s_cet; 1107 to_save->isst_addr = from_save->isst_addr; 1108 to_save->ssp = from_save->ssp; 1109 } 1110 } 1111 1112 void svm_copy_vmloadsave_state(struct vmcb *to_vmcb, struct vmcb *from_vmcb) 1113 { 1114 to_vmcb->save.fs = from_vmcb->save.fs; 1115 to_vmcb->save.gs = from_vmcb->save.gs; 1116 to_vmcb->save.tr = from_vmcb->save.tr; 1117 to_vmcb->save.ldtr = from_vmcb->save.ldtr; 1118 to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base; 1119 to_vmcb->save.star = from_vmcb->save.star; 1120 to_vmcb->save.lstar = from_vmcb->save.lstar; 1121 to_vmcb->save.cstar = from_vmcb->save.cstar; 1122 to_vmcb->save.sfmask = from_vmcb->save.sfmask; 1123 to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs; 1124 to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp; 1125 to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip; 1126 } 1127 1128 int nested_svm_vmexit(struct vcpu_svm *svm) 1129 { 1130 struct kvm_vcpu *vcpu = &svm->vcpu; 1131 struct vmcb *vmcb01 = svm->vmcb01.ptr; 1132 struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; 1133 struct vmcb *vmcb12; 1134 struct kvm_host_map map; 1135 int rc; 1136 1137 rc = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa), &map); 1138 if (rc) { 1139 if (rc == -EINVAL) 1140 kvm_inject_gp(vcpu, 0); 1141 return 1; 1142 } 1143 1144 vmcb12 = map.hva; 1145 1146 /* Exit Guest-Mode */ 1147 leave_guest_mode(vcpu); 1148 svm->nested.vmcb12_gpa = 0; 1149 WARN_ON_ONCE(svm->nested.nested_run_pending); 1150 1151 kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 1152 1153 /* in case we halted in L2 */ 1154 kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); 1155 1156 /* Give the current vmcb to the guest */ 1157 1158 vmcb12->save.es = vmcb02->save.es; 1159 vmcb12->save.cs = vmcb02->save.cs; 1160 vmcb12->save.ss = vmcb02->save.ss; 1161 vmcb12->save.ds = vmcb02->save.ds; 1162 vmcb12->save.gdtr = vmcb02->save.gdtr; 1163 vmcb12->save.idtr = vmcb02->save.idtr; 1164 vmcb12->save.efer = svm->vcpu.arch.efer; 1165 vmcb12->save.cr0 = kvm_read_cr0(vcpu); 1166 vmcb12->save.cr3 = kvm_read_cr3(vcpu); 1167 vmcb12->save.cr2 = vmcb02->save.cr2; 1168 vmcb12->save.cr4 = svm->vcpu.arch.cr4; 1169 vmcb12->save.rflags = kvm_get_rflags(vcpu); 1170 vmcb12->save.rip = kvm_rip_read(vcpu); 1171 vmcb12->save.rsp = kvm_rsp_read(vcpu); 1172 vmcb12->save.rax = kvm_rax_read(vcpu); 1173 vmcb12->save.dr7 = vmcb02->save.dr7; 1174 vmcb12->save.dr6 = svm->vcpu.arch.dr6; 1175 vmcb12->save.cpl = vmcb02->save.cpl; 1176 1177 if (guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK)) { 1178 vmcb12->save.s_cet = vmcb02->save.s_cet; 1179 vmcb12->save.isst_addr = vmcb02->save.isst_addr; 1180 vmcb12->save.ssp = vmcb02->save.ssp; 1181 } 1182 1183 vmcb12->control.int_state = vmcb02->control.int_state; 1184 vmcb12->control.exit_code = vmcb02->control.exit_code; 1185 vmcb12->control.exit_info_1 = vmcb02->control.exit_info_1; 1186 vmcb12->control.exit_info_2 = vmcb02->control.exit_info_2; 1187 1188 if (!svm_is_vmrun_failure(vmcb12->control.exit_code)) 1189 nested_save_pending_event_to_vmcb12(svm, vmcb12); 1190 1191 if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS)) 1192 vmcb12->control.next_rip = vmcb02->control.next_rip; 1193 1194 vmcb12->control.int_ctl = svm->nested.ctl.int_ctl; 1195 vmcb12->control.event_inj = svm->nested.ctl.event_inj; 1196 vmcb12->control.event_inj_err = svm->nested.ctl.event_inj_err; 1197 1198 if (!kvm_pause_in_guest(vcpu->kvm)) { 1199 vmcb01->control.pause_filter_count = vmcb02->control.pause_filter_count; 1200 vmcb_mark_dirty(vmcb01, VMCB_INTERCEPTS); 1201 1202 } 1203 1204 /* 1205 * Invalidate bus_lock_rip unless KVM is still waiting for the guest 1206 * to make forward progress before re-enabling bus lock detection. 1207 */ 1208 if (!vmcb02->control.bus_lock_counter) 1209 svm->nested.ctl.bus_lock_rip = INVALID_GPA; 1210 1211 nested_svm_copy_common_state(svm->nested.vmcb02.ptr, svm->vmcb01.ptr); 1212 1213 kvm_nested_vmexit_handle_ibrs(vcpu); 1214 1215 if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS)) 1216 vmcb01->control.erap_ctl |= ERAP_CONTROL_CLEAR_RAP; 1217 1218 svm_switch_vmcb(svm, &svm->vmcb01); 1219 1220 /* 1221 * Rules for synchronizing int_ctl bits from vmcb02 to vmcb01: 1222 * 1223 * V_IRQ, V_IRQ_VECTOR, V_INTR_PRIO_MASK, V_IGN_TPR: If L1 doesn't 1224 * intercept interrupts, then KVM will use vmcb02's V_IRQ (and related 1225 * flags) to detect interrupt windows for L1 IRQs (even if L1 uses 1226 * virtual interrupt masking). Raise KVM_REQ_EVENT to ensure that 1227 * KVM re-requests an interrupt window if necessary, which implicitly 1228 * copies this bits from vmcb02 to vmcb01. 1229 * 1230 * V_TPR: If L1 doesn't use virtual interrupt masking, then L1's vTPR 1231 * is stored in vmcb02, but its value doesn't need to be copied from/to 1232 * vmcb01 because it is copied from/to the virtual APIC's TPR register 1233 * on each VM entry/exit. 1234 * 1235 * V_GIF: If nested vGIF is not used, KVM uses vmcb02's V_GIF for L1's 1236 * V_GIF. However, GIF is architecturally clear on each VM exit, thus 1237 * there is no need to copy V_GIF from vmcb02 to vmcb01. 1238 */ 1239 if (!nested_exit_on_intr(svm)) 1240 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); 1241 1242 if (unlikely(guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) && 1243 (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK))) 1244 svm_copy_lbrs(vmcb12, vmcb02); 1245 else 1246 svm_copy_lbrs(vmcb01, vmcb02); 1247 1248 svm_update_lbrv(vcpu); 1249 1250 if (vnmi) { 1251 if (vmcb02->control.int_ctl & V_NMI_BLOCKING_MASK) 1252 vmcb01->control.int_ctl |= V_NMI_BLOCKING_MASK; 1253 else 1254 vmcb01->control.int_ctl &= ~V_NMI_BLOCKING_MASK; 1255 1256 if (vcpu->arch.nmi_pending) { 1257 vcpu->arch.nmi_pending--; 1258 vmcb01->control.int_ctl |= V_NMI_PENDING_MASK; 1259 } else { 1260 vmcb01->control.int_ctl &= ~V_NMI_PENDING_MASK; 1261 } 1262 } 1263 1264 /* 1265 * On vmexit the GIF is set to false and 1266 * no event can be injected in L1. 1267 */ 1268 svm_set_gif(svm, false); 1269 vmcb01->control.exit_int_info = 0; 1270 1271 svm->vcpu.arch.tsc_offset = svm->vcpu.arch.l1_tsc_offset; 1272 if (vmcb01->control.tsc_offset != svm->vcpu.arch.tsc_offset) { 1273 vmcb01->control.tsc_offset = svm->vcpu.arch.tsc_offset; 1274 vmcb_mark_dirty(vmcb01, VMCB_INTERCEPTS); 1275 } 1276 1277 if (kvm_caps.has_tsc_control && 1278 vcpu->arch.tsc_scaling_ratio != vcpu->arch.l1_tsc_scaling_ratio) { 1279 vcpu->arch.tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; 1280 svm_write_tsc_multiplier(vcpu); 1281 } 1282 1283 svm->nested.ctl.nested_cr3 = 0; 1284 1285 /* 1286 * Restore processor state that had been saved in vmcb01 1287 */ 1288 kvm_set_rflags(vcpu, vmcb01->save.rflags); 1289 svm_set_efer(vcpu, vmcb01->save.efer); 1290 svm_set_cr0(vcpu, vmcb01->save.cr0 | X86_CR0_PE); 1291 svm_set_cr4(vcpu, vmcb01->save.cr4); 1292 kvm_rax_write(vcpu, vmcb01->save.rax); 1293 kvm_rsp_write(vcpu, vmcb01->save.rsp); 1294 kvm_rip_write(vcpu, vmcb01->save.rip); 1295 1296 svm->vcpu.arch.dr7 = DR7_FIXED_1; 1297 kvm_update_dr7(&svm->vcpu); 1298 1299 trace_kvm_nested_vmexit_inject(vmcb12->control.exit_code, 1300 vmcb12->control.exit_info_1, 1301 vmcb12->control.exit_info_2, 1302 vmcb12->control.exit_int_info, 1303 vmcb12->control.exit_int_info_err, 1304 KVM_ISA_SVM); 1305 1306 kvm_vcpu_unmap(vcpu, &map); 1307 1308 nested_svm_transition_tlb_flush(vcpu); 1309 1310 nested_svm_uninit_mmu_context(vcpu); 1311 1312 rc = nested_svm_load_cr3(vcpu, vmcb01->save.cr3, false, true); 1313 if (rc) 1314 return 1; 1315 1316 /* 1317 * Drop what we picked up for L2 via svm_complete_interrupts() so it 1318 * doesn't end up in L1. 1319 */ 1320 svm->vcpu.arch.nmi_injected = false; 1321 kvm_clear_exception_queue(vcpu); 1322 kvm_clear_interrupt_queue(vcpu); 1323 1324 /* 1325 * If we are here following the completion of a VMRUN that 1326 * is being single-stepped, queue the pending #DB intercept 1327 * right now so that it an be accounted for before we execute 1328 * L1's next instruction. 1329 */ 1330 if (unlikely(vmcb01->save.rflags & X86_EFLAGS_TF)) 1331 kvm_queue_exception(&(svm->vcpu), DB_VECTOR); 1332 1333 /* 1334 * Un-inhibit the AVIC right away, so that other vCPUs can start 1335 * to benefit from it right away. 1336 */ 1337 if (kvm_apicv_activated(vcpu->kvm)) 1338 __kvm_vcpu_update_apicv(vcpu); 1339 1340 return 0; 1341 } 1342 1343 static void nested_svm_triple_fault(struct kvm_vcpu *vcpu) 1344 { 1345 struct vcpu_svm *svm = to_svm(vcpu); 1346 1347 if (!vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SHUTDOWN)) 1348 return; 1349 1350 kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu); 1351 nested_svm_simple_vmexit(to_svm(vcpu), SVM_EXIT_SHUTDOWN); 1352 } 1353 1354 int svm_allocate_nested(struct vcpu_svm *svm) 1355 { 1356 struct page *vmcb02_page; 1357 1358 if (svm->nested.initialized) 1359 return 0; 1360 1361 vmcb02_page = snp_safe_alloc_page(); 1362 if (!vmcb02_page) 1363 return -ENOMEM; 1364 svm->nested.vmcb02.ptr = page_address(vmcb02_page); 1365 svm->nested.vmcb02.pa = __sme_set(page_to_pfn(vmcb02_page) << PAGE_SHIFT); 1366 1367 svm->nested.msrpm = svm_vcpu_alloc_msrpm(); 1368 if (!svm->nested.msrpm) 1369 goto err_free_vmcb02; 1370 1371 svm->nested.initialized = true; 1372 return 0; 1373 1374 err_free_vmcb02: 1375 __free_page(vmcb02_page); 1376 return -ENOMEM; 1377 } 1378 1379 void svm_free_nested(struct vcpu_svm *svm) 1380 { 1381 if (!svm->nested.initialized) 1382 return; 1383 1384 if (WARN_ON_ONCE(svm->vmcb != svm->vmcb01.ptr)) 1385 svm_switch_vmcb(svm, &svm->vmcb01); 1386 1387 svm_vcpu_free_msrpm(svm->nested.msrpm); 1388 svm->nested.msrpm = NULL; 1389 1390 __free_page(virt_to_page(svm->nested.vmcb02.ptr)); 1391 svm->nested.vmcb02.ptr = NULL; 1392 1393 /* 1394 * When last_vmcb12_gpa matches the current vmcb12 gpa, 1395 * some vmcb12 fields are not loaded if they are marked clean 1396 * in the vmcb12, since in this case they are up to date already. 1397 * 1398 * When the vmcb02 is freed, this optimization becomes invalid. 1399 */ 1400 svm->nested.last_vmcb12_gpa = INVALID_GPA; 1401 1402 svm->nested.initialized = false; 1403 } 1404 1405 void svm_leave_nested(struct kvm_vcpu *vcpu) 1406 { 1407 struct vcpu_svm *svm = to_svm(vcpu); 1408 1409 if (is_guest_mode(vcpu)) { 1410 svm->nested.nested_run_pending = 0; 1411 svm->nested.vmcb12_gpa = INVALID_GPA; 1412 1413 leave_guest_mode(vcpu); 1414 1415 svm_switch_vmcb(svm, &svm->vmcb01); 1416 1417 nested_svm_uninit_mmu_context(vcpu); 1418 vmcb_mark_all_dirty(svm->vmcb); 1419 1420 svm_set_gif(svm, true); 1421 1422 if (kvm_apicv_activated(vcpu->kvm)) 1423 kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu); 1424 } 1425 1426 kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 1427 } 1428 1429 static int nested_svm_exit_handled_msr(struct vcpu_svm *svm) 1430 { 1431 gpa_t base = svm->nested.ctl.msrpm_base_pa; 1432 int write, bit_nr; 1433 u8 value, mask; 1434 u32 msr; 1435 1436 if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT))) 1437 return NESTED_EXIT_HOST; 1438 1439 msr = svm->vcpu.arch.regs[VCPU_REGS_RCX]; 1440 bit_nr = svm_msrpm_bit_nr(msr); 1441 write = svm->vmcb->control.exit_info_1 & 1; 1442 1443 if (bit_nr < 0) 1444 return NESTED_EXIT_DONE; 1445 1446 if (kvm_vcpu_read_guest(&svm->vcpu, base + bit_nr / BITS_PER_BYTE, 1447 &value, sizeof(value))) 1448 return NESTED_EXIT_DONE; 1449 1450 mask = BIT(write) << (bit_nr & (BITS_PER_BYTE - 1)); 1451 return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; 1452 } 1453 1454 static int nested_svm_intercept_ioio(struct vcpu_svm *svm) 1455 { 1456 unsigned port, size, iopm_len; 1457 u16 val, mask; 1458 u8 start_bit; 1459 u64 gpa; 1460 1461 if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_IOIO_PROT))) 1462 return NESTED_EXIT_HOST; 1463 1464 port = svm->vmcb->control.exit_info_1 >> 16; 1465 size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >> 1466 SVM_IOIO_SIZE_SHIFT; 1467 gpa = svm->nested.ctl.iopm_base_pa + (port / 8); 1468 start_bit = port % 8; 1469 iopm_len = (start_bit + size > 8) ? 2 : 1; 1470 mask = (0xf >> (4 - size)) << start_bit; 1471 val = 0; 1472 1473 if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len)) 1474 return NESTED_EXIT_DONE; 1475 1476 return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; 1477 } 1478 1479 static int nested_svm_intercept(struct vcpu_svm *svm) 1480 { 1481 u64 exit_code = svm->vmcb->control.exit_code; 1482 int vmexit = NESTED_EXIT_HOST; 1483 1484 if (svm_is_vmrun_failure(exit_code)) 1485 return NESTED_EXIT_DONE; 1486 1487 switch (exit_code) { 1488 case SVM_EXIT_MSR: 1489 vmexit = nested_svm_exit_handled_msr(svm); 1490 break; 1491 case SVM_EXIT_IOIO: 1492 vmexit = nested_svm_intercept_ioio(svm); 1493 break; 1494 case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: 1495 /* 1496 * Host-intercepted exceptions have been checked already in 1497 * nested_svm_exit_special. There is nothing to do here, 1498 * the vmexit is injected by svm_check_nested_events. 1499 */ 1500 vmexit = NESTED_EXIT_DONE; 1501 break; 1502 default: 1503 if (vmcb12_is_intercept(&svm->nested.ctl, exit_code)) 1504 vmexit = NESTED_EXIT_DONE; 1505 break; 1506 } 1507 1508 return vmexit; 1509 } 1510 1511 int nested_svm_exit_handled(struct vcpu_svm *svm) 1512 { 1513 int vmexit; 1514 1515 vmexit = nested_svm_intercept(svm); 1516 1517 if (vmexit == NESTED_EXIT_DONE) 1518 nested_svm_vmexit(svm); 1519 1520 return vmexit; 1521 } 1522 1523 int nested_svm_check_permissions(struct kvm_vcpu *vcpu) 1524 { 1525 if (!(vcpu->arch.efer & EFER_SVME) || !is_paging(vcpu)) { 1526 kvm_queue_exception(vcpu, UD_VECTOR); 1527 return 1; 1528 } 1529 1530 if (to_svm(vcpu)->vmcb->save.cpl) { 1531 kvm_inject_gp(vcpu, 0); 1532 return 1; 1533 } 1534 1535 return 0; 1536 } 1537 1538 static bool nested_svm_is_exception_vmexit(struct kvm_vcpu *vcpu, u8 vector, 1539 u32 error_code) 1540 { 1541 struct vcpu_svm *svm = to_svm(vcpu); 1542 1543 return (svm->nested.ctl.intercepts[INTERCEPT_EXCEPTION] & BIT(vector)); 1544 } 1545 1546 static void nested_svm_inject_exception_vmexit(struct kvm_vcpu *vcpu) 1547 { 1548 struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit; 1549 struct vcpu_svm *svm = to_svm(vcpu); 1550 struct vmcb *vmcb = svm->vmcb; 1551 1552 vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + ex->vector; 1553 1554 if (ex->has_error_code) 1555 vmcb->control.exit_info_1 = ex->error_code; 1556 1557 /* 1558 * EXITINFO2 is undefined for all exception intercepts other 1559 * than #PF. 1560 */ 1561 if (ex->vector == PF_VECTOR) { 1562 if (ex->has_payload) 1563 vmcb->control.exit_info_2 = ex->payload; 1564 else 1565 vmcb->control.exit_info_2 = vcpu->arch.cr2; 1566 } else if (ex->vector == DB_VECTOR) { 1567 /* See kvm_check_and_inject_events(). */ 1568 kvm_deliver_exception_payload(vcpu, ex); 1569 1570 if (vcpu->arch.dr7 & DR7_GD) { 1571 vcpu->arch.dr7 &= ~DR7_GD; 1572 kvm_update_dr7(vcpu); 1573 } 1574 } else { 1575 WARN_ON(ex->has_payload); 1576 } 1577 1578 nested_svm_vmexit(svm); 1579 } 1580 1581 static inline bool nested_exit_on_init(struct vcpu_svm *svm) 1582 { 1583 return vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_INIT); 1584 } 1585 1586 static int svm_check_nested_events(struct kvm_vcpu *vcpu) 1587 { 1588 struct kvm_lapic *apic = vcpu->arch.apic; 1589 struct vcpu_svm *svm = to_svm(vcpu); 1590 /* 1591 * Only a pending nested run blocks a pending exception. If there is a 1592 * previously injected event, the pending exception occurred while said 1593 * event was being delivered and thus needs to be handled. 1594 */ 1595 bool block_nested_exceptions = svm->nested.nested_run_pending; 1596 /* 1597 * New events (not exceptions) are only recognized at instruction 1598 * boundaries. If an event needs reinjection, then KVM is handling a 1599 * VM-Exit that occurred _during_ instruction execution; new events are 1600 * blocked until the instruction completes. 1601 */ 1602 bool block_nested_events = block_nested_exceptions || 1603 kvm_event_needs_reinjection(vcpu); 1604 1605 if (lapic_in_kernel(vcpu) && 1606 test_bit(KVM_APIC_INIT, &apic->pending_events)) { 1607 if (block_nested_events) 1608 return -EBUSY; 1609 if (!nested_exit_on_init(svm)) 1610 return 0; 1611 nested_svm_simple_vmexit(svm, SVM_EXIT_INIT); 1612 return 0; 1613 } 1614 1615 if (vcpu->arch.exception_vmexit.pending) { 1616 if (block_nested_exceptions) 1617 return -EBUSY; 1618 nested_svm_inject_exception_vmexit(vcpu); 1619 return 0; 1620 } 1621 1622 if (vcpu->arch.exception.pending) { 1623 if (block_nested_exceptions) 1624 return -EBUSY; 1625 return 0; 1626 } 1627 1628 #ifdef CONFIG_KVM_SMM 1629 if (vcpu->arch.smi_pending && !svm_smi_blocked(vcpu)) { 1630 if (block_nested_events) 1631 return -EBUSY; 1632 if (!nested_exit_on_smi(svm)) 1633 return 0; 1634 nested_svm_simple_vmexit(svm, SVM_EXIT_SMI); 1635 return 0; 1636 } 1637 #endif 1638 1639 if (vcpu->arch.nmi_pending && !svm_nmi_blocked(vcpu)) { 1640 if (block_nested_events) 1641 return -EBUSY; 1642 if (!nested_exit_on_nmi(svm)) 1643 return 0; 1644 nested_svm_simple_vmexit(svm, SVM_EXIT_NMI); 1645 return 0; 1646 } 1647 1648 if (kvm_cpu_has_interrupt(vcpu) && !svm_interrupt_blocked(vcpu)) { 1649 if (block_nested_events) 1650 return -EBUSY; 1651 if (!nested_exit_on_intr(svm)) 1652 return 0; 1653 trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip); 1654 nested_svm_simple_vmexit(svm, SVM_EXIT_INTR); 1655 return 0; 1656 } 1657 1658 return 0; 1659 } 1660 1661 int nested_svm_exit_special(struct vcpu_svm *svm) 1662 { 1663 u32 exit_code = svm->vmcb->control.exit_code; 1664 struct kvm_vcpu *vcpu = &svm->vcpu; 1665 1666 switch (exit_code) { 1667 case SVM_EXIT_INTR: 1668 case SVM_EXIT_NMI: 1669 case SVM_EXIT_NPF: 1670 return NESTED_EXIT_HOST; 1671 case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: { 1672 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE); 1673 1674 if (svm->vmcb01.ptr->control.intercepts[INTERCEPT_EXCEPTION] & 1675 excp_bits) 1676 return NESTED_EXIT_HOST; 1677 else if (exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR && 1678 svm->vcpu.arch.apf.host_apf_flags) 1679 /* Trap async PF even if not shadowing */ 1680 return NESTED_EXIT_HOST; 1681 break; 1682 } 1683 case SVM_EXIT_VMMCALL: 1684 /* Hyper-V L2 TLB flush hypercall is handled by L0 */ 1685 if (guest_hv_cpuid_has_l2_tlb_flush(vcpu) && 1686 nested_svm_l2_tlb_flush_enabled(vcpu) && 1687 kvm_hv_is_tlb_flush_hcall(vcpu)) 1688 return NESTED_EXIT_HOST; 1689 break; 1690 default: 1691 break; 1692 } 1693 1694 return NESTED_EXIT_CONTINUE; 1695 } 1696 1697 void nested_svm_update_tsc_ratio_msr(struct kvm_vcpu *vcpu) 1698 { 1699 struct vcpu_svm *svm = to_svm(vcpu); 1700 1701 vcpu->arch.tsc_scaling_ratio = 1702 kvm_calc_nested_tsc_multiplier(vcpu->arch.l1_tsc_scaling_ratio, 1703 svm->tsc_ratio_msr); 1704 svm_write_tsc_multiplier(vcpu); 1705 } 1706 1707 /* Inverse operation of nested_copy_vmcb_control_to_cache(). asid is copied too. */ 1708 static void nested_copy_vmcb_cache_to_control(struct vmcb_control_area *dst, 1709 struct vmcb_ctrl_area_cached *from) 1710 { 1711 unsigned int i; 1712 1713 memset(dst, 0, sizeof(struct vmcb_control_area)); 1714 1715 for (i = 0; i < MAX_INTERCEPT; i++) 1716 dst->intercepts[i] = from->intercepts[i]; 1717 1718 dst->iopm_base_pa = from->iopm_base_pa; 1719 dst->msrpm_base_pa = from->msrpm_base_pa; 1720 dst->tsc_offset = from->tsc_offset; 1721 dst->asid = from->asid; 1722 dst->tlb_ctl = from->tlb_ctl; 1723 dst->erap_ctl = from->erap_ctl; 1724 dst->int_ctl = from->int_ctl; 1725 dst->int_vector = from->int_vector; 1726 dst->int_state = from->int_state; 1727 dst->exit_code = from->exit_code; 1728 dst->exit_info_1 = from->exit_info_1; 1729 dst->exit_info_2 = from->exit_info_2; 1730 dst->exit_int_info = from->exit_int_info; 1731 dst->exit_int_info_err = from->exit_int_info_err; 1732 dst->nested_ctl = from->nested_ctl; 1733 dst->event_inj = from->event_inj; 1734 dst->event_inj_err = from->event_inj_err; 1735 dst->next_rip = from->next_rip; 1736 dst->nested_cr3 = from->nested_cr3; 1737 dst->virt_ext = from->virt_ext; 1738 dst->pause_filter_count = from->pause_filter_count; 1739 dst->pause_filter_thresh = from->pause_filter_thresh; 1740 /* 'clean' and 'hv_enlightenments' are not changed by KVM */ 1741 } 1742 1743 static int svm_get_nested_state(struct kvm_vcpu *vcpu, 1744 struct kvm_nested_state __user *user_kvm_nested_state, 1745 u32 user_data_size) 1746 { 1747 struct vcpu_svm *svm; 1748 struct vmcb_control_area *ctl; 1749 unsigned long r; 1750 struct kvm_nested_state kvm_state = { 1751 .flags = 0, 1752 .format = KVM_STATE_NESTED_FORMAT_SVM, 1753 .size = sizeof(kvm_state), 1754 }; 1755 struct vmcb __user *user_vmcb = (struct vmcb __user *) 1756 &user_kvm_nested_state->data.svm[0]; 1757 1758 if (!vcpu) 1759 return kvm_state.size + KVM_STATE_NESTED_SVM_VMCB_SIZE; 1760 1761 svm = to_svm(vcpu); 1762 1763 if (user_data_size < kvm_state.size) 1764 goto out; 1765 1766 /* First fill in the header and copy it out. */ 1767 if (is_guest_mode(vcpu)) { 1768 kvm_state.hdr.svm.vmcb_pa = svm->nested.vmcb12_gpa; 1769 kvm_state.size += KVM_STATE_NESTED_SVM_VMCB_SIZE; 1770 kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE; 1771 1772 if (svm->nested.nested_run_pending) 1773 kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING; 1774 } 1775 1776 if (gif_set(svm)) 1777 kvm_state.flags |= KVM_STATE_NESTED_GIF_SET; 1778 1779 if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state))) 1780 return -EFAULT; 1781 1782 if (!is_guest_mode(vcpu)) 1783 goto out; 1784 1785 /* 1786 * Copy over the full size of the VMCB rather than just the size 1787 * of the structs. 1788 */ 1789 if (clear_user(user_vmcb, KVM_STATE_NESTED_SVM_VMCB_SIZE)) 1790 return -EFAULT; 1791 1792 ctl = kzalloc_obj(*ctl); 1793 if (!ctl) 1794 return -ENOMEM; 1795 1796 nested_copy_vmcb_cache_to_control(ctl, &svm->nested.ctl); 1797 r = copy_to_user(&user_vmcb->control, ctl, 1798 sizeof(user_vmcb->control)); 1799 kfree(ctl); 1800 if (r) 1801 return -EFAULT; 1802 1803 if (copy_to_user(&user_vmcb->save, &svm->vmcb01.ptr->save, 1804 sizeof(user_vmcb->save))) 1805 return -EFAULT; 1806 out: 1807 return kvm_state.size; 1808 } 1809 1810 static int svm_set_nested_state(struct kvm_vcpu *vcpu, 1811 struct kvm_nested_state __user *user_kvm_nested_state, 1812 struct kvm_nested_state *kvm_state) 1813 { 1814 struct vcpu_svm *svm = to_svm(vcpu); 1815 struct vmcb __user *user_vmcb = (struct vmcb __user *) 1816 &user_kvm_nested_state->data.svm[0]; 1817 struct vmcb_control_area *ctl; 1818 struct vmcb_save_area *save; 1819 struct vmcb_save_area_cached save_cached; 1820 struct vmcb_ctrl_area_cached ctl_cached; 1821 unsigned long cr0; 1822 int ret; 1823 1824 BUILD_BUG_ON(sizeof(struct vmcb_control_area) + sizeof(struct vmcb_save_area) > 1825 KVM_STATE_NESTED_SVM_VMCB_SIZE); 1826 1827 if (kvm_state->format != KVM_STATE_NESTED_FORMAT_SVM) 1828 return -EINVAL; 1829 1830 if (kvm_state->flags & ~(KVM_STATE_NESTED_GUEST_MODE | 1831 KVM_STATE_NESTED_RUN_PENDING | 1832 KVM_STATE_NESTED_GIF_SET)) 1833 return -EINVAL; 1834 1835 /* 1836 * If in guest mode, vcpu->arch.efer actually refers to the L2 guest's 1837 * EFER.SVME, but EFER.SVME still has to be 1 for VMRUN to succeed. 1838 * If SVME is disabled, the only valid states are "none" and GIF=1 1839 * (clearing SVME does NOT set GIF, i.e. GIF=0 is allowed). 1840 */ 1841 if (!(vcpu->arch.efer & EFER_SVME) && kvm_state->flags && 1842 kvm_state->flags != KVM_STATE_NESTED_GIF_SET) 1843 return -EINVAL; 1844 1845 /* SMM temporarily disables SVM, so we cannot be in guest mode. */ 1846 if (is_smm(vcpu) && (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 1847 return -EINVAL; 1848 1849 if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) { 1850 svm_leave_nested(vcpu); 1851 svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET)); 1852 return 0; 1853 } 1854 1855 if (!page_address_valid(vcpu, kvm_state->hdr.svm.vmcb_pa)) 1856 return -EINVAL; 1857 if (kvm_state->size < sizeof(*kvm_state) + KVM_STATE_NESTED_SVM_VMCB_SIZE) 1858 return -EINVAL; 1859 1860 ctl = memdup_user(&user_vmcb->control, sizeof(*ctl)); 1861 if (IS_ERR(ctl)) 1862 return PTR_ERR(ctl); 1863 1864 save = memdup_user(&user_vmcb->save, sizeof(*save)); 1865 if (IS_ERR(save)) { 1866 kfree(ctl); 1867 return PTR_ERR(save); 1868 } 1869 1870 ret = -EINVAL; 1871 __nested_copy_vmcb_control_to_cache(vcpu, &ctl_cached, ctl); 1872 if (!__nested_vmcb_check_controls(vcpu, &ctl_cached)) 1873 goto out_free; 1874 1875 /* 1876 * Processor state contains L2 state. Check that it is 1877 * valid for guest mode (see nested_vmcb_check_save). 1878 */ 1879 cr0 = kvm_read_cr0(vcpu); 1880 if (((cr0 & X86_CR0_CD) == 0) && (cr0 & X86_CR0_NW)) 1881 goto out_free; 1882 1883 /* 1884 * Validate host state saved from before VMRUN (see 1885 * nested_svm_check_permissions). 1886 */ 1887 __nested_copy_vmcb_save_to_cache(&save_cached, save); 1888 if (!(save->cr0 & X86_CR0_PG) || 1889 !(save->cr0 & X86_CR0_PE) || 1890 (save->rflags & X86_EFLAGS_VM) || 1891 !__nested_vmcb_check_save(vcpu, &save_cached)) 1892 goto out_free; 1893 1894 1895 /* 1896 * All checks done, we can enter guest mode. Userspace provides 1897 * vmcb12.control, which will be combined with L1 and stored into 1898 * vmcb02, and the L1 save state which we store in vmcb01. 1899 * L2 registers if needed are moved from the current VMCB to VMCB02. 1900 */ 1901 1902 if (is_guest_mode(vcpu)) 1903 svm_leave_nested(vcpu); 1904 else 1905 svm->nested.vmcb02.ptr->save = svm->vmcb01.ptr->save; 1906 1907 svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET)); 1908 1909 svm->nested.nested_run_pending = 1910 !!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING); 1911 1912 svm->nested.vmcb12_gpa = kvm_state->hdr.svm.vmcb_pa; 1913 1914 svm_copy_vmrun_state(&svm->vmcb01.ptr->save, save); 1915 nested_copy_vmcb_control_to_cache(svm, ctl); 1916 1917 svm_switch_vmcb(svm, &svm->nested.vmcb02); 1918 nested_vmcb02_prepare_control(svm, svm->vmcb->save.rip, svm->vmcb->save.cs.base); 1919 1920 /* 1921 * While the nested guest CR3 is already checked and set by 1922 * KVM_SET_SREGS, it was set when nested state was yet loaded, 1923 * thus MMU might not be initialized correctly. 1924 * Set it again to fix this. 1925 */ 1926 ret = nested_svm_load_cr3(&svm->vcpu, vcpu->arch.cr3, 1927 nested_npt_enabled(svm), false); 1928 if (ret) 1929 goto out_free; 1930 1931 svm->nested.force_msr_bitmap_recalc = true; 1932 1933 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 1934 ret = 0; 1935 out_free: 1936 kfree(save); 1937 kfree(ctl); 1938 1939 return ret; 1940 } 1941 1942 static bool svm_get_nested_state_pages(struct kvm_vcpu *vcpu) 1943 { 1944 if (WARN_ON(!is_guest_mode(vcpu))) 1945 return true; 1946 1947 if (!vcpu->arch.pdptrs_from_userspace && 1948 !nested_npt_enabled(to_svm(vcpu)) && is_pae_paging(vcpu)) 1949 /* 1950 * Reload the guest's PDPTRs since after a migration 1951 * the guest CR3 might be restored prior to setting the nested 1952 * state which can lead to a load of wrong PDPTRs. 1953 */ 1954 if (CC(!load_pdptrs(vcpu, vcpu->arch.cr3))) 1955 return false; 1956 1957 if (!nested_svm_merge_msrpm(vcpu)) { 1958 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1959 vcpu->run->internal.suberror = 1960 KVM_INTERNAL_ERROR_EMULATION; 1961 vcpu->run->internal.ndata = 0; 1962 return false; 1963 } 1964 1965 if (kvm_hv_verify_vp_assist(vcpu)) 1966 return false; 1967 1968 return true; 1969 } 1970 1971 struct kvm_x86_nested_ops svm_nested_ops = { 1972 .leave_nested = svm_leave_nested, 1973 .is_exception_vmexit = nested_svm_is_exception_vmexit, 1974 .check_events = svm_check_nested_events, 1975 .triple_fault = nested_svm_triple_fault, 1976 .get_nested_state_pages = svm_get_nested_state_pages, 1977 .get_state = svm_get_nested_state, 1978 .set_state = svm_set_nested_state, 1979 .hv_inject_synthetic_vmexit_post_tlb_flush = svm_hv_inject_synthetic_vmexit_post_tlb_flush, 1980 }; 1981