1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * AMD SVM support 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Yaniv Kamay <yaniv@qumranet.com> 12 * Avi Kivity <avi@qumranet.com> 13 */ 14 15 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 16 17 #include <linux/kvm_types.h> 18 #include <linux/kvm_host.h> 19 #include <linux/kernel.h> 20 21 #include <asm/msr-index.h> 22 #include <asm/debugreg.h> 23 24 #include "kvm_emulate.h" 25 #include "trace.h" 26 #include "mmu.h" 27 #include "x86.h" 28 #include "smm.h" 29 #include "cpuid.h" 30 #include "lapic.h" 31 #include "svm.h" 32 #include "hyperv.h" 33 34 #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK 35 36 static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu, 37 struct x86_exception *fault) 38 { 39 struct vcpu_svm *svm = to_svm(vcpu); 40 struct vmcb *vmcb = svm->vmcb; 41 42 if (vmcb->control.exit_code != SVM_EXIT_NPF) { 43 /* 44 * TODO: track the cause of the nested page fault, and 45 * correctly fill in the high bits of exit_info_1. 46 */ 47 vmcb->control.exit_code = SVM_EXIT_NPF; 48 vmcb->control.exit_info_1 = (1ULL << 32); 49 vmcb->control.exit_info_2 = fault->address; 50 } 51 52 vmcb->control.exit_info_1 &= ~0xffffffffULL; 53 vmcb->control.exit_info_1 |= fault->error_code; 54 55 nested_svm_vmexit(svm); 56 } 57 58 static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index) 59 { 60 struct vcpu_svm *svm = to_svm(vcpu); 61 u64 cr3 = svm->nested.ctl.nested_cr3; 62 u64 pdpte; 63 int ret; 64 65 /* 66 * Note, nCR3 is "assumed" to be 32-byte aligned, i.e. the CPU ignores 67 * nCR3[4:0] when loading PDPTEs from memory. 68 */ 69 ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(cr3), &pdpte, 70 (cr3 & GENMASK(11, 5)) + index * 8, 8); 71 if (ret) 72 return 0; 73 return pdpte; 74 } 75 76 static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu) 77 { 78 struct vcpu_svm *svm = to_svm(vcpu); 79 80 return svm->nested.ctl.nested_cr3; 81 } 82 83 static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu) 84 { 85 struct vcpu_svm *svm = to_svm(vcpu); 86 87 WARN_ON(mmu_is_nested(vcpu)); 88 89 vcpu->arch.mmu = &vcpu->arch.guest_mmu; 90 91 /* 92 * The NPT format depends on L1's CR4 and EFER, which is in vmcb01. Note, 93 * when called via KVM_SET_NESTED_STATE, that state may _not_ match current 94 * vCPU state. CR0.WP is explicitly ignored, while CR0.PG is required. 95 */ 96 kvm_init_shadow_npt_mmu(vcpu, svm->vmcb01.ptr->save.cr4, 97 svm->vmcb01.ptr->save.efer, 98 svm->nested.ctl.nested_cr3, 99 svm->nested.ctl.misc_ctl); 100 vcpu->arch.mmu->get_guest_pgd = nested_svm_get_tdp_cr3; 101 vcpu->arch.mmu->get_pdptr = nested_svm_get_tdp_pdptr; 102 vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit; 103 vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; 104 } 105 106 static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu) 107 { 108 vcpu->arch.mmu = &vcpu->arch.root_mmu; 109 vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; 110 } 111 112 static bool nested_vmcb_needs_vls_intercept(struct vcpu_svm *svm) 113 { 114 if (!guest_cpu_cap_has(&svm->vcpu, X86_FEATURE_V_VMSAVE_VMLOAD)) 115 return true; 116 117 if (!nested_npt_enabled(svm)) 118 return true; 119 120 if (!(svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE)) 121 return true; 122 123 return false; 124 } 125 126 void nested_vmcb02_recalc_intercepts(struct vcpu_svm *svm) 127 { 128 struct vmcb_ctrl_area_cached *vmcb12_ctrl = &svm->nested.ctl; 129 struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; 130 struct vmcb *vmcb01 = svm->vmcb01.ptr; 131 unsigned int i; 132 133 if (WARN_ON_ONCE(svm->vmcb != vmcb02)) 134 return; 135 136 vmcb_mark_dirty(vmcb02, VMCB_INTERCEPTS); 137 138 for (i = 0; i < MAX_INTERCEPT; i++) 139 vmcb02->control.intercepts[i] = vmcb01->control.intercepts[i]; 140 141 if (vmcb12_ctrl->int_ctl & V_INTR_MASKING_MASK) { 142 /* 143 * If L2 is active and V_INTR_MASKING is enabled in vmcb12, 144 * disable intercept of CR8 writes as L2's CR8 does not affect 145 * any interrupt KVM may want to inject. 146 * 147 * Similarly, disable intercept of virtual interrupts (used to 148 * detect interrupt windows) if the saved RFLAGS.IF is '0', as 149 * the effective RFLAGS.IF for L1 interrupts will never be set 150 * while L2 is running (L2's RFLAGS.IF doesn't affect L1 IRQs). 151 */ 152 vmcb_clr_intercept(&vmcb02->control, INTERCEPT_CR8_WRITE); 153 if (!(vmcb01->save.rflags & X86_EFLAGS_IF)) 154 vmcb_clr_intercept(&vmcb02->control, INTERCEPT_VINTR); 155 } 156 157 for (i = 0; i < MAX_INTERCEPT; i++) 158 vmcb02->control.intercepts[i] |= vmcb12_ctrl->intercepts[i]; 159 160 /* If SMI is not intercepted, ignore guest SMI intercept as well */ 161 if (!intercept_smi) 162 vmcb_clr_intercept(&vmcb02->control, INTERCEPT_SMI); 163 164 if (nested_vmcb_needs_vls_intercept(svm)) { 165 /* 166 * If the virtual VMLOAD/VMSAVE is not enabled for the L2, 167 * we must intercept these instructions to correctly 168 * emulate them in case L1 doesn't intercept them. 169 */ 170 vmcb_set_intercept(&vmcb02->control, INTERCEPT_VMLOAD); 171 vmcb_set_intercept(&vmcb02->control, INTERCEPT_VMSAVE); 172 } else { 173 WARN_ON_ONCE(!(vmcb02->control.misc_ctl2 & SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE)); 174 } 175 } 176 177 /* 178 * This array (and its actual size) holds the set of offsets (indexing by chunk 179 * size) to process when merging vmcb12's MSRPM with vmcb01's MSRPM. Note, the 180 * set of MSRs for which interception is disabled in vmcb01 is per-vCPU, e.g. 181 * based on CPUID features. This array only tracks MSRs that *might* be passed 182 * through to the guest. 183 * 184 * Hardcode the capacity of the array based on the maximum number of _offsets_. 185 * MSRs are batched together, so there are fewer offsets than MSRs. 186 */ 187 static int nested_svm_msrpm_merge_offsets[10] __ro_after_init; 188 static int nested_svm_nr_msrpm_merge_offsets __ro_after_init; 189 typedef unsigned long nsvm_msrpm_merge_t; 190 191 int __init nested_svm_init_msrpm_merge_offsets(void) 192 { 193 static const u32 merge_msrs[] __initconst = { 194 MSR_STAR, 195 MSR_IA32_SYSENTER_CS, 196 MSR_IA32_SYSENTER_EIP, 197 MSR_IA32_SYSENTER_ESP, 198 #ifdef CONFIG_X86_64 199 MSR_GS_BASE, 200 MSR_FS_BASE, 201 MSR_KERNEL_GS_BASE, 202 MSR_LSTAR, 203 MSR_CSTAR, 204 MSR_SYSCALL_MASK, 205 #endif 206 MSR_IA32_SPEC_CTRL, 207 MSR_IA32_PRED_CMD, 208 MSR_IA32_FLUSH_CMD, 209 MSR_IA32_APERF, 210 MSR_IA32_MPERF, 211 MSR_IA32_LASTBRANCHFROMIP, 212 MSR_IA32_LASTBRANCHTOIP, 213 MSR_IA32_LASTINTFROMIP, 214 MSR_IA32_LASTINTTOIP, 215 216 MSR_K7_PERFCTR0, 217 MSR_K7_PERFCTR1, 218 MSR_K7_PERFCTR2, 219 MSR_K7_PERFCTR3, 220 MSR_F15H_PERF_CTR0, 221 MSR_F15H_PERF_CTR1, 222 MSR_F15H_PERF_CTR2, 223 MSR_F15H_PERF_CTR3, 224 MSR_F15H_PERF_CTR4, 225 MSR_F15H_PERF_CTR5, 226 227 MSR_AMD64_PERF_CNTR_GLOBAL_CTL, 228 MSR_AMD64_PERF_CNTR_GLOBAL_STATUS, 229 MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR, 230 MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET, 231 }; 232 int i, j; 233 234 for (i = 0; i < ARRAY_SIZE(merge_msrs); i++) { 235 int bit_nr = svm_msrpm_bit_nr(merge_msrs[i]); 236 u32 offset; 237 238 if (WARN_ON(bit_nr < 0)) 239 return -EIO; 240 241 /* 242 * Merging is done in chunks to reduce the number of accesses 243 * to L1's bitmap. 244 */ 245 offset = bit_nr / BITS_PER_BYTE / sizeof(nsvm_msrpm_merge_t); 246 247 for (j = 0; j < nested_svm_nr_msrpm_merge_offsets; j++) { 248 if (nested_svm_msrpm_merge_offsets[j] == offset) 249 break; 250 } 251 252 if (j < nested_svm_nr_msrpm_merge_offsets) 253 continue; 254 255 if (WARN_ON(j >= ARRAY_SIZE(nested_svm_msrpm_merge_offsets))) 256 return -EIO; 257 258 nested_svm_msrpm_merge_offsets[j] = offset; 259 nested_svm_nr_msrpm_merge_offsets++; 260 } 261 262 return 0; 263 } 264 265 /* 266 * Merge L0's (KVM) and L1's (Nested VMCB) MSR permission bitmaps. The function 267 * is optimized in that it only merges the parts where KVM MSR permission bitmap 268 * may contain zero bits. 269 */ 270 static bool nested_svm_merge_msrpm(struct kvm_vcpu *vcpu) 271 { 272 struct vcpu_svm *svm = to_svm(vcpu); 273 nsvm_msrpm_merge_t *msrpm02 = svm->nested.msrpm; 274 nsvm_msrpm_merge_t *msrpm01 = svm->msrpm; 275 int i; 276 277 /* 278 * MSR bitmap update can be skipped when: 279 * - MSR bitmap for L1 hasn't changed. 280 * - Nested hypervisor (L1) is attempting to launch the same L2 as 281 * before. 282 * - Nested hypervisor (L1) is using Hyper-V emulation interface and 283 * tells KVM (L0) there were no changes in MSR bitmap for L2. 284 */ 285 #ifdef CONFIG_KVM_HYPERV 286 if (!svm->nested.force_msr_bitmap_recalc) { 287 struct hv_vmcb_enlightenments *hve = &svm->nested.ctl.hv_enlightenments; 288 289 if (kvm_hv_hypercall_enabled(vcpu) && 290 hve->hv_enlightenments_control.msr_bitmap && 291 (svm->nested.ctl.clean & BIT(HV_VMCB_NESTED_ENLIGHTENMENTS))) 292 goto set_msrpm_base_pa; 293 } 294 #endif 295 296 if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT))) 297 return true; 298 299 for (i = 0; i < nested_svm_nr_msrpm_merge_offsets; i++) { 300 const int p = nested_svm_msrpm_merge_offsets[i]; 301 nsvm_msrpm_merge_t l1_val; 302 gpa_t gpa; 303 304 gpa = svm->nested.ctl.msrpm_base_pa + (p * sizeof(l1_val)); 305 306 if (kvm_vcpu_read_guest(vcpu, gpa, &l1_val, sizeof(l1_val))) 307 return false; 308 309 msrpm02[p] = msrpm01[p] | l1_val; 310 } 311 312 svm->nested.force_msr_bitmap_recalc = false; 313 314 #ifdef CONFIG_KVM_HYPERV 315 set_msrpm_base_pa: 316 #endif 317 svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm)); 318 319 return true; 320 } 321 322 /* 323 * Bits 11:0 of bitmap address are ignored by hardware 324 */ 325 static bool nested_svm_check_bitmap_pa(struct kvm_vcpu *vcpu, u64 pa, u32 size) 326 { 327 u64 addr = PAGE_ALIGN(pa); 328 329 return kvm_vcpu_is_legal_gpa(vcpu, addr) && 330 kvm_vcpu_is_legal_gpa(vcpu, addr + size - 1); 331 } 332 333 static bool nested_svm_event_inj_valid_exept(struct kvm_vcpu *vcpu, u8 vector) 334 { 335 /* 336 * Vectors that do not correspond to a defined exception are invalid 337 * (including #NMI and reserved vectors). In a best effort to define 338 * valid exceptions based on the virtual CPU, make all exceptions always 339 * valid except those obviously tied to a CPU feature. 340 */ 341 switch (vector) { 342 case DE_VECTOR: case DB_VECTOR: case BP_VECTOR: case OF_VECTOR: 343 case BR_VECTOR: case UD_VECTOR: case NM_VECTOR: case DF_VECTOR: 344 case TS_VECTOR: case NP_VECTOR: case SS_VECTOR: case GP_VECTOR: 345 case PF_VECTOR: case MF_VECTOR: case AC_VECTOR: case MC_VECTOR: 346 case XM_VECTOR: case HV_VECTOR: case SX_VECTOR: 347 return true; 348 case CP_VECTOR: 349 return guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK); 350 case VC_VECTOR: 351 return guest_cpu_cap_has(vcpu, X86_FEATURE_SEV_ES); 352 } 353 return false; 354 } 355 356 /* 357 * According to the APM, VMRUN exits with SVM_EXIT_ERR if SVM_EVTINJ_VALID is 358 * set and: 359 * - The type of event_inj is not one of the defined values. 360 * - The type is SVM_EVTINJ_TYPE_EXEPT, but the vector is not a valid exception. 361 */ 362 static bool nested_svm_check_event_inj(struct kvm_vcpu *vcpu, u32 event_inj) 363 { 364 u32 type = event_inj & SVM_EVTINJ_TYPE_MASK; 365 u8 vector = event_inj & SVM_EVTINJ_VEC_MASK; 366 367 if (!(event_inj & SVM_EVTINJ_VALID)) 368 return true; 369 370 if (type != SVM_EVTINJ_TYPE_INTR && type != SVM_EVTINJ_TYPE_NMI && 371 type != SVM_EVTINJ_TYPE_EXEPT && type != SVM_EVTINJ_TYPE_SOFT) 372 return false; 373 374 if (type == SVM_EVTINJ_TYPE_EXEPT && 375 !nested_svm_event_inj_valid_exept(vcpu, vector)) 376 return false; 377 378 return true; 379 } 380 381 static bool nested_vmcb_check_controls(struct kvm_vcpu *vcpu, 382 struct vmcb_ctrl_area_cached *control) 383 { 384 if (CC(!vmcb12_is_intercept(control, INTERCEPT_VMRUN))) 385 return false; 386 387 if (CC(control->asid == 0)) 388 return false; 389 390 if (CC((control->misc_ctl & SVM_MISC_ENABLE_NP) && 391 !kvm_vcpu_is_legal_gpa(vcpu, control->nested_cr3))) 392 return false; 393 394 if (CC(!nested_svm_check_bitmap_pa(vcpu, control->msrpm_base_pa, 395 MSRPM_SIZE))) 396 return false; 397 if (CC(!nested_svm_check_bitmap_pa(vcpu, control->iopm_base_pa, 398 IOPM_SIZE))) 399 return false; 400 401 if (CC((control->int_ctl & V_NMI_ENABLE_MASK) && 402 !vmcb12_is_intercept(control, INTERCEPT_NMI))) { 403 return false; 404 } 405 406 if (CC(!nested_svm_check_event_inj(vcpu, control->event_inj))) 407 return false; 408 409 return true; 410 } 411 412 /* Common checks that apply to both L1 and L2 state. */ 413 static bool nested_vmcb_check_save(struct kvm_vcpu *vcpu, 414 struct vmcb_save_area_cached *save) 415 { 416 if (CC(!(save->efer & EFER_SVME))) 417 return false; 418 419 if (CC((save->cr0 & X86_CR0_CD) == 0 && (save->cr0 & X86_CR0_NW)) || 420 CC(save->cr0 & ~0xffffffffULL)) 421 return false; 422 423 if (CC(!kvm_dr6_valid(save->dr6)) || CC(!kvm_dr7_valid(save->dr7))) 424 return false; 425 426 /* 427 * These checks are also performed by KVM_SET_SREGS, 428 * except that EFER.LMA is not checked by SVM against 429 * CR0.PG && EFER.LME. 430 */ 431 if ((save->efer & EFER_LME) && (save->cr0 & X86_CR0_PG)) { 432 if (CC(!(save->cr4 & X86_CR4_PAE)) || 433 CC(!(save->cr0 & X86_CR0_PE)) || 434 CC(!kvm_vcpu_is_legal_cr3(vcpu, save->cr3))) 435 return false; 436 437 if (CC((save->cs.attrib & SVM_SELECTOR_L_MASK) && 438 (save->cs.attrib & SVM_SELECTOR_DB_MASK))) 439 return false; 440 } 441 442 /* Note, SVM doesn't have any additional restrictions on CR4. */ 443 if (CC(!__kvm_is_valid_cr4(vcpu, save->cr4))) 444 return false; 445 446 if (CC(!kvm_valid_efer(vcpu, save->efer))) 447 return false; 448 449 return true; 450 } 451 452 int nested_svm_check_cached_vmcb12(struct kvm_vcpu *vcpu) 453 { 454 struct vcpu_svm *svm = to_svm(vcpu); 455 456 if (!nested_vmcb_check_save(vcpu, &svm->nested.save) || 457 !nested_vmcb_check_controls(vcpu, &svm->nested.ctl)) 458 return -EINVAL; 459 460 return 0; 461 } 462 463 /* 464 * If a feature is not advertised to L1, clear the corresponding vmcb12 465 * intercept. 466 */ 467 #define __nested_svm_sanitize_intercept(__vcpu, __control, fname, iname) \ 468 do { \ 469 if (!guest_cpu_cap_has(__vcpu, X86_FEATURE_##fname)) \ 470 vmcb12_clr_intercept(__control, INTERCEPT_##iname); \ 471 } while (0) 472 473 #define nested_svm_sanitize_intercept(__vcpu, __control, name) \ 474 __nested_svm_sanitize_intercept(__vcpu, __control, name, name) 475 476 static 477 void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu, 478 struct vmcb_ctrl_area_cached *to, 479 struct vmcb_control_area *from) 480 { 481 unsigned int i; 482 483 for (i = 0; i < MAX_INTERCEPT; i++) 484 to->intercepts[i] = from->intercepts[i]; 485 486 __nested_svm_sanitize_intercept(vcpu, to, XSAVE, XSETBV); 487 nested_svm_sanitize_intercept(vcpu, to, INVPCID); 488 nested_svm_sanitize_intercept(vcpu, to, RDTSCP); 489 nested_svm_sanitize_intercept(vcpu, to, SKINIT); 490 nested_svm_sanitize_intercept(vcpu, to, RDPRU); 491 492 /* Always clear misc_ctl bits that the guest cannot use */ 493 to->misc_ctl = from->misc_ctl; 494 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_NPT)) 495 to->misc_ctl &= ~SVM_MISC_ENABLE_NP; 496 497 if (!gmet_enabled || !guest_cpu_cap_has(vcpu, X86_FEATURE_GMET)) 498 to->misc_ctl &= ~SVM_MISC_ENABLE_GMET; 499 500 to->iopm_base_pa = from->iopm_base_pa & PAGE_MASK; 501 to->msrpm_base_pa = from->msrpm_base_pa & PAGE_MASK; 502 to->tsc_offset = from->tsc_offset; 503 to->tlb_ctl = from->tlb_ctl & TLB_CONTROL_MASK; 504 to->erap_ctl = from->erap_ctl; 505 to->int_ctl = from->int_ctl; 506 to->int_vector = from->int_vector & SVM_INT_VECTOR_MASK; 507 to->int_state = from->int_state & SVM_INTERRUPT_SHADOW_MASK; 508 to->exit_code = from->exit_code; 509 to->exit_info_1 = from->exit_info_1; 510 to->exit_info_2 = from->exit_info_2; 511 to->exit_int_info = from->exit_int_info; 512 to->exit_int_info_err = from->exit_int_info_err; 513 to->event_inj = from->event_inj & ~SVM_EVTINJ_RESERVED_BITS; 514 to->event_inj_err = from->event_inj_err; 515 to->next_rip = from->next_rip; 516 to->nested_cr3 = from->nested_cr3; 517 to->misc_ctl2 = from->misc_ctl2; 518 to->pause_filter_count = from->pause_filter_count; 519 to->pause_filter_thresh = from->pause_filter_thresh; 520 521 /* Copy asid here because nested_vmcb_check_controls() will check it */ 522 to->asid = from->asid; 523 to->clean = from->clean; 524 525 #ifdef CONFIG_KVM_HYPERV 526 /* Hyper-V extensions (Enlightened VMCB) */ 527 if (kvm_hv_hypercall_enabled(vcpu)) { 528 memcpy(&to->hv_enlightenments, &from->hv_enlightenments, 529 sizeof(to->hv_enlightenments)); 530 } 531 #endif 532 } 533 534 void nested_copy_vmcb_control_to_cache(struct vcpu_svm *svm, 535 struct vmcb_control_area *control) 536 { 537 __nested_copy_vmcb_control_to_cache(&svm->vcpu, &svm->nested.ctl, control); 538 } 539 540 static void __nested_copy_vmcb_save_to_cache(struct vmcb_save_area_cached *to, 541 struct vmcb_save_area *from) 542 { 543 to->es = from->es; 544 to->cs = from->cs; 545 to->ss = from->ss; 546 to->ds = from->ds; 547 to->gdtr = from->gdtr; 548 to->idtr = from->idtr; 549 550 to->cpl = from->cpl; 551 552 to->efer = from->efer; 553 to->cr4 = from->cr4; 554 to->cr3 = from->cr3; 555 to->cr0 = from->cr0; 556 to->dr7 = from->dr7; 557 to->dr6 = from->dr6; 558 559 to->rflags = from->rflags; 560 to->rip = from->rip; 561 to->rsp = from->rsp; 562 563 to->s_cet = from->s_cet; 564 to->ssp = from->ssp; 565 to->isst_addr = from->isst_addr; 566 567 to->rax = from->rax; 568 to->cr2 = from->cr2; 569 570 svm_copy_lbrs(to, from); 571 } 572 573 void nested_copy_vmcb_save_to_cache(struct vcpu_svm *svm, 574 struct vmcb_save_area *save) 575 { 576 __nested_copy_vmcb_save_to_cache(&svm->nested.save, save); 577 } 578 579 /* 580 * Synchronize fields that are written by the processor, so that 581 * they can be copied back into the vmcb12. 582 */ 583 void nested_sync_control_from_vmcb02(struct vcpu_svm *svm) 584 { 585 u32 mask; 586 svm->nested.ctl.event_inj = svm->vmcb->control.event_inj; 587 svm->nested.ctl.event_inj_err = svm->vmcb->control.event_inj_err; 588 svm->nested.ctl.int_state = svm->vmcb->control.int_state; 589 590 /* Only a few fields of int_ctl are written by the processor. */ 591 mask = V_IRQ_MASK | V_TPR_MASK; 592 /* 593 * Don't sync vmcb02 V_IRQ back to vmcb12 if KVM (L0) is intercepting 594 * virtual interrupts in order to request an interrupt window, as KVM 595 * has usurped vmcb02's int_ctl. If an interrupt window opens before 596 * the next VM-Exit, svm_clear_vintr() will restore vmcb12's int_ctl. 597 * If no window opens, V_IRQ will be correctly preserved in vmcb12's 598 * int_ctl (because it was never recognized while L2 was running). 599 */ 600 if (svm_is_intercept(svm, INTERCEPT_VINTR) && 601 !vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_VINTR)) 602 mask &= ~V_IRQ_MASK; 603 604 if (nested_vgif_enabled(svm)) 605 mask |= V_GIF_MASK; 606 607 if (nested_vnmi_enabled(svm)) 608 mask |= V_NMI_BLOCKING_MASK | V_NMI_PENDING_MASK; 609 610 svm->nested.ctl.int_ctl &= ~mask; 611 svm->nested.ctl.int_ctl |= svm->vmcb->control.int_ctl & mask; 612 } 613 614 /* 615 * Transfer any event that L0 or L1 wanted to inject into L2 to 616 * EXIT_INT_INFO. 617 */ 618 static void nested_save_pending_event_to_vmcb12(struct vcpu_svm *svm, 619 struct vmcb *vmcb12) 620 { 621 struct kvm_vcpu *vcpu = &svm->vcpu; 622 u32 exit_int_info = 0; 623 unsigned int nr; 624 625 if (vcpu->arch.exception.injected) { 626 nr = vcpu->arch.exception.vector; 627 exit_int_info = nr | SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_EXEPT; 628 629 if (vcpu->arch.exception.has_error_code) { 630 exit_int_info |= SVM_EVTINJ_VALID_ERR; 631 vmcb12->control.exit_int_info_err = 632 vcpu->arch.exception.error_code; 633 } 634 635 } else if (vcpu->arch.nmi_injected) { 636 exit_int_info = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI; 637 638 } else if (vcpu->arch.interrupt.injected) { 639 nr = vcpu->arch.interrupt.nr; 640 exit_int_info = nr | SVM_EVTINJ_VALID; 641 642 if (vcpu->arch.interrupt.soft) 643 exit_int_info |= SVM_EVTINJ_TYPE_SOFT; 644 else 645 exit_int_info |= SVM_EVTINJ_TYPE_INTR; 646 } 647 648 vmcb12->control.exit_int_info = exit_int_info; 649 } 650 651 static void nested_svm_transition_tlb_flush(struct kvm_vcpu *vcpu) 652 { 653 /* Handle pending Hyper-V TLB flush requests */ 654 kvm_hv_nested_transtion_tlb_flush(vcpu, npt_enabled); 655 656 /* 657 * TODO: optimize unconditional TLB flush/MMU sync. A partial list of 658 * things to fix before this can be conditional: 659 * 660 * - Flush TLBs for both L1 and L2 remote TLB flush 661 * - Honor L1's request to flush an ASID on nested VMRUN 662 * - Sync nested NPT MMU on VMRUN that flushes L2's ASID[*] 663 * - Don't crush a pending TLB flush in vmcb02 on nested VMRUN 664 * - Flush L1's ASID on KVM_REQ_TLB_FLUSH_GUEST 665 * 666 * [*] Unlike nested EPT, SVM's ASID management can invalidate nested 667 * NPT guest-physical mappings on VMRUN. 668 */ 669 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); 670 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 671 } 672 673 /* 674 * Load guest's/host's cr3 on nested vmentry or vmexit. @nested_npt is true 675 * if we are emulating VM-Entry into a guest with NPT enabled. 676 */ 677 static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, 678 bool nested_npt, bool reload_pdptrs) 679 { 680 if (CC(!kvm_vcpu_is_legal_cr3(vcpu, cr3))) 681 return -EINVAL; 682 683 if (reload_pdptrs && !nested_npt && is_pae_paging(vcpu) && 684 CC(!load_pdptrs(vcpu, cr3))) 685 return -EINVAL; 686 687 vcpu->arch.cr3 = cr3; 688 689 /* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */ 690 kvm_init_mmu(vcpu); 691 692 if (!nested_npt) 693 kvm_mmu_new_pgd(vcpu, cr3); 694 695 return 0; 696 } 697 698 void nested_vmcb02_compute_g_pat(struct vcpu_svm *svm) 699 { 700 if (!svm->nested.vmcb02.ptr) 701 return; 702 703 /* FIXME: merge g_pat from vmcb01 and vmcb12. */ 704 svm->nested.vmcb02.ptr->save.g_pat = svm->vmcb01.ptr->save.g_pat; 705 } 706 707 static bool nested_vmcb12_has_lbrv(struct kvm_vcpu *vcpu) 708 { 709 return guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) && 710 (to_svm(vcpu)->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR); 711 } 712 713 static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) 714 { 715 struct vmcb_ctrl_area_cached *control = &svm->nested.ctl; 716 struct vmcb_save_area_cached *save = &svm->nested.save; 717 bool new_vmcb12 = false; 718 struct vmcb *vmcb01 = svm->vmcb01.ptr; 719 struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; 720 struct kvm_vcpu *vcpu = &svm->vcpu; 721 722 nested_vmcb02_compute_g_pat(svm); 723 vmcb_mark_dirty(vmcb02, VMCB_NPT); 724 725 /* Load the nested guest state */ 726 if (svm->nested.vmcb12_gpa != svm->nested.last_vmcb12_gpa) { 727 new_vmcb12 = true; 728 svm->nested.last_vmcb12_gpa = svm->nested.vmcb12_gpa; 729 svm->nested.force_msr_bitmap_recalc = true; 730 } 731 732 if (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_SEG))) { 733 vmcb02->save.es = save->es; 734 vmcb02->save.cs = save->cs; 735 vmcb02->save.ss = save->ss; 736 vmcb02->save.ds = save->ds; 737 vmcb02->save.cpl = save->cpl; 738 vmcb_mark_dirty(vmcb02, VMCB_SEG); 739 } 740 741 if (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_DT))) { 742 vmcb02->save.gdtr = save->gdtr; 743 vmcb02->save.idtr = save->idtr; 744 vmcb_mark_dirty(vmcb02, VMCB_DT); 745 } 746 747 if (guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) && 748 (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_CET)))) { 749 vmcb02->save.s_cet = save->s_cet; 750 vmcb02->save.isst_addr = save->isst_addr; 751 vmcb02->save.ssp = save->ssp; 752 vmcb_mark_dirty(vmcb02, VMCB_CET); 753 } 754 755 kvm_set_rflags(vcpu, save->rflags | X86_EFLAGS_FIXED); 756 757 svm_set_efer(vcpu, svm->nested.save.efer); 758 759 svm_set_cr0(vcpu, svm->nested.save.cr0); 760 svm_set_cr4(vcpu, svm->nested.save.cr4); 761 762 svm->vcpu.arch.cr2 = save->cr2; 763 764 kvm_rax_write(vcpu, save->rax); 765 kvm_rsp_write(vcpu, save->rsp); 766 kvm_rip_write(vcpu, save->rip); 767 768 /* In case we don't even reach vcpu_run, the fields are not updated */ 769 vmcb02->save.rax = save->rax; 770 vmcb02->save.rsp = save->rsp; 771 vmcb02->save.rip = save->rip; 772 773 if (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_DR))) { 774 vmcb02->save.dr7 = svm->nested.save.dr7 | DR7_FIXED_1; 775 svm->vcpu.arch.dr6 = svm->nested.save.dr6 | DR6_ACTIVE_LOW; 776 vmcb_mark_dirty(vmcb02, VMCB_DR); 777 } 778 779 if (nested_vmcb12_has_lbrv(vcpu)) { 780 /* 781 * Reserved bits of DEBUGCTL are ignored. Be consistent with 782 * svm_set_msr's definition of reserved bits. 783 */ 784 svm_copy_lbrs(&vmcb02->save, save); 785 vmcb02->save.dbgctl &= ~DEBUGCTL_RESERVED_BITS; 786 } else { 787 svm_copy_lbrs(&vmcb02->save, &vmcb01->save); 788 } 789 vmcb_mark_dirty(vmcb02, VMCB_LBR); 790 svm_update_lbrv(&svm->vcpu); 791 } 792 793 static inline bool is_evtinj_soft(u32 evtinj) 794 { 795 u32 type = evtinj & SVM_EVTINJ_TYPE_MASK; 796 u8 vector = evtinj & SVM_EVTINJ_VEC_MASK; 797 798 if (!(evtinj & SVM_EVTINJ_VALID)) 799 return false; 800 801 if (type == SVM_EVTINJ_TYPE_SOFT) 802 return true; 803 804 return type == SVM_EVTINJ_TYPE_EXEPT && kvm_exception_is_soft(vector); 805 } 806 807 static bool is_evtinj_nmi(u32 evtinj) 808 { 809 u32 type = evtinj & SVM_EVTINJ_TYPE_MASK; 810 811 if (!(evtinj & SVM_EVTINJ_VALID)) 812 return false; 813 814 return type == SVM_EVTINJ_TYPE_NMI; 815 } 816 817 static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) 818 { 819 u32 int_ctl_vmcb01_bits = V_INTR_MASKING_MASK; 820 u32 int_ctl_vmcb12_bits = V_TPR_MASK | V_IRQ_INJECTION_BITS_MASK; 821 822 struct vmcb_ctrl_area_cached *vmcb12_ctrl = &svm->nested.ctl; 823 struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; 824 struct vmcb *vmcb01 = svm->vmcb01.ptr; 825 struct kvm_vcpu *vcpu = &svm->vcpu; 826 u32 pause_count12, pause_thresh12; 827 828 nested_svm_transition_tlb_flush(vcpu); 829 830 /* Enter Guest-Mode */ 831 enter_guest_mode(vcpu); 832 833 /* 834 * Filled at exit: exit_code, exit_info_1, exit_info_2, exit_int_info, 835 * exit_int_info_err, next_rip, insn_len, insn_bytes. 836 */ 837 838 if (guest_cpu_cap_has(vcpu, X86_FEATURE_VGIF) && 839 (vmcb12_ctrl->int_ctl & V_GIF_ENABLE_MASK)) 840 int_ctl_vmcb12_bits |= (V_GIF_MASK | V_GIF_ENABLE_MASK); 841 else 842 int_ctl_vmcb01_bits |= (V_GIF_MASK | V_GIF_ENABLE_MASK); 843 844 if (vnmi) { 845 if (vmcb01->control.int_ctl & V_NMI_PENDING_MASK) { 846 svm->vcpu.arch.nmi_pending++; 847 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); 848 } 849 if (nested_vnmi_enabled(svm)) 850 int_ctl_vmcb12_bits |= (V_NMI_PENDING_MASK | 851 V_NMI_ENABLE_MASK | 852 V_NMI_BLOCKING_MASK); 853 } 854 855 /* 856 * Copied from vmcb01. msrpm_base can be overwritten later. 857 * 858 * SVM_MISC_ENABLE_NP in vmcb12 is only used for consistency checks. If 859 * L1 enables NPTs, KVM shadows L1's NPTs and uses those to run L2. If 860 * L1 disables NPT, KVM runs L2 with the same NPTs used to run L1. For 861 * the latter, L1 runs L2 with shadow page tables that translate L2 GVAs 862 * to L1 GPAs, so the same NPTs can be used for L1 and L2. 863 */ 864 vmcb02->control.misc_ctl = vmcb01->control.misc_ctl & (SVM_MISC_ENABLE_NP | SVM_MISC_ENABLE_GMET); 865 vmcb02->control.iopm_base_pa = vmcb01->control.iopm_base_pa; 866 vmcb02->control.msrpm_base_pa = vmcb01->control.msrpm_base_pa; 867 vmcb_mark_dirty(vmcb02, VMCB_PERM_MAP); 868 869 /* 870 * Stash vmcb02's counter if the guest hasn't moved past the guilty 871 * instruction; otherwise, reset the counter to '0'. 872 * 873 * In order to detect if L2 has made forward progress or not, track the 874 * RIP at which a bus lock has occurred on a per-vmcb12 basis. If RIP 875 * is changed, guest has clearly made forward progress, bus_lock_counter 876 * still remained '1', so reset bus_lock_counter to '0'. Eg. In the 877 * scenario, where a buslock happened in L1 before VMRUN, the bus lock 878 * firmly happened on an instruction in the past. Even if vmcb01's 879 * counter is still '1', (because the guilty instruction got patched), 880 * the vCPU has clearly made forward progress and so KVM should reset 881 * vmcb02's counter to '0'. 882 * 883 * If the RIP hasn't changed, stash the bus lock counter at nested VMRUN 884 * to prevent the same guilty instruction from triggering a VM-Exit. Eg. 885 * if userspace rate-limits the vCPU, then it's entirely possible that 886 * L1's tick interrupt is pending by the time userspace re-runs the 887 * vCPU. If KVM unconditionally clears the counter on VMRUN, then when 888 * L1 re-enters L2, the same instruction will trigger a VM-Exit and the 889 * entire cycle start over. 890 */ 891 if (vmcb02->save.rip && (svm->nested.last_bus_lock_rip == vmcb02->save.rip)) 892 vmcb02->control.bus_lock_counter = 1; 893 else 894 vmcb02->control.bus_lock_counter = 0; 895 896 /* Done at vmrun: asid. */ 897 898 /* Also overwritten later if necessary. */ 899 vmcb02->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; 900 901 /* Use vmcb01 MMU and format if guest does not use nNPT */ 902 if (nested_npt_enabled(svm)) { 903 vmcb02->control.misc_ctl &= ~SVM_MISC_ENABLE_GMET; 904 vmcb02->control.misc_ctl |= (svm->nested.ctl.misc_ctl & SVM_MISC_ENABLE_GMET); 905 906 nested_svm_init_mmu_context(vcpu); 907 } 908 909 vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(vcpu->arch.l1_tsc_offset, 910 vmcb12_ctrl->tsc_offset, 911 svm->tsc_ratio_msr); 912 913 vmcb02->control.tsc_offset = vcpu->arch.tsc_offset; 914 915 if (guest_cpu_cap_has(vcpu, X86_FEATURE_TSCRATEMSR) && 916 svm->tsc_ratio_msr != kvm_caps.default_tsc_scaling_ratio) 917 nested_svm_update_tsc_ratio_msr(vcpu); 918 919 vmcb02->control.int_ctl = 920 (vmcb12_ctrl->int_ctl & int_ctl_vmcb12_bits) | 921 (vmcb01->control.int_ctl & int_ctl_vmcb01_bits); 922 923 vmcb02->control.int_vector = vmcb12_ctrl->int_vector; 924 vmcb02->control.int_state = vmcb12_ctrl->int_state; 925 vmcb02->control.event_inj = vmcb12_ctrl->event_inj; 926 vmcb02->control.event_inj_err = vmcb12_ctrl->event_inj_err; 927 928 /* 929 * If nrips is exposed to L1, take NextRIP as-is. Otherwise, L1 930 * advances L2's RIP before VMRUN instead of using NextRIP. KVM will 931 * stuff the current RIP as vmcb02's NextRIP before L2 is run. After 932 * the first run of L2 (e.g. after save+restore), NextRIP is updated by 933 * the CPU and/or KVM and should be used regardless of L1's support. 934 */ 935 if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS) || 936 !vcpu->arch.nested_run_pending) 937 vmcb02->control.next_rip = vmcb12_ctrl->next_rip; 938 939 svm->nmi_l1_to_l2 = is_evtinj_nmi(vmcb02->control.event_inj); 940 941 /* 942 * soft_int_csbase, soft_int_old_rip, and soft_int_next_rip (if L1 943 * doesn't have NRIPS) are initialized later, before the vCPU is run. 944 */ 945 if (is_evtinj_soft(vmcb02->control.event_inj)) { 946 svm->soft_int_injected = true; 947 if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS) || 948 !vcpu->arch.nested_run_pending) 949 svm->soft_int_next_rip = vmcb12_ctrl->next_rip; 950 } 951 952 /* SVM_MISC2_ENABLE_V_LBR is controlled by svm_update_lbrv() */ 953 954 if (!nested_vmcb_needs_vls_intercept(svm)) 955 vmcb02->control.misc_ctl2 |= SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE; 956 957 if (guest_cpu_cap_has(vcpu, X86_FEATURE_PAUSEFILTER)) 958 pause_count12 = vmcb12_ctrl->pause_filter_count; 959 else 960 pause_count12 = 0; 961 if (guest_cpu_cap_has(vcpu, X86_FEATURE_PFTHRESHOLD)) 962 pause_thresh12 = vmcb12_ctrl->pause_filter_thresh; 963 else 964 pause_thresh12 = 0; 965 if (kvm_pause_in_guest(svm->vcpu.kvm)) { 966 /* use guest values since host doesn't intercept PAUSE */ 967 vmcb02->control.pause_filter_count = pause_count12; 968 vmcb02->control.pause_filter_thresh = pause_thresh12; 969 970 } else { 971 /* start from host values otherwise */ 972 vmcb02->control.pause_filter_count = vmcb01->control.pause_filter_count; 973 vmcb02->control.pause_filter_thresh = vmcb01->control.pause_filter_thresh; 974 975 /* ... but ensure filtering is disabled if so requested. */ 976 if (vmcb12_is_intercept(vmcb12_ctrl, INTERCEPT_PAUSE)) { 977 if (!pause_count12) 978 vmcb02->control.pause_filter_count = 0; 979 if (!pause_thresh12) 980 vmcb02->control.pause_filter_thresh = 0; 981 } 982 } 983 984 /* 985 * Take ALLOW_LARGER_RAP from vmcb12 even though it should be safe to 986 * let L2 use a larger RAP since KVM will emulate the necessary clears, 987 * as it's possible L1 deliberately wants to restrict L2 to the legacy 988 * RAP size. Unconditionally clear the RAP on nested VMRUN, as KVM is 989 * responsible for emulating the host vs. guest tags (L1 is the "host", 990 * L2 is the "guest"). 991 */ 992 if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS)) 993 vmcb02->control.erap_ctl = (vmcb12_ctrl->erap_ctl & 994 ERAP_CONTROL_ALLOW_LARGER_RAP) | 995 ERAP_CONTROL_CLEAR_RAP; 996 997 /* 998 * Merge guest and host intercepts - must be called with vcpu in 999 * guest-mode to take effect. 1000 */ 1001 nested_vmcb02_recalc_intercepts(svm); 1002 } 1003 1004 static void nested_svm_copy_common_state(struct vmcb *from_vmcb, struct vmcb *to_vmcb) 1005 { 1006 /* 1007 * Some VMCB state is shared between L1 and L2 and thus has to be 1008 * moved at the time of nested vmrun and vmexit. 1009 * 1010 * VMLOAD/VMSAVE state would also belong in this category, but KVM 1011 * always performs VMLOAD and VMSAVE from the VMCB01. 1012 */ 1013 to_vmcb->save.spec_ctrl = from_vmcb->save.spec_ctrl; 1014 } 1015 1016 int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb12_gpa, bool from_vmrun) 1017 { 1018 struct vcpu_svm *svm = to_svm(vcpu); 1019 struct vmcb_ctrl_area_cached *control = &svm->nested.ctl; 1020 struct vmcb_save_area_cached *save = &svm->nested.save; 1021 int ret; 1022 1023 trace_kvm_nested_vmenter(svm->vmcb->save.rip, 1024 vmcb12_gpa, 1025 save->rip, 1026 control->int_ctl, 1027 control->event_inj, 1028 control->misc_ctl, 1029 control->nested_cr3, 1030 save->cr3, 1031 KVM_ISA_SVM); 1032 1033 trace_kvm_nested_intercepts(control->intercepts[INTERCEPT_CR] & 0xffff, 1034 control->intercepts[INTERCEPT_CR] >> 16, 1035 control->intercepts[INTERCEPT_EXCEPTION], 1036 control->intercepts[INTERCEPT_WORD3], 1037 control->intercepts[INTERCEPT_WORD4], 1038 control->intercepts[INTERCEPT_WORD5]); 1039 1040 1041 svm->nested.vmcb12_gpa = vmcb12_gpa; 1042 1043 WARN_ON(svm->vmcb == svm->nested.vmcb02.ptr); 1044 1045 nested_svm_copy_common_state(svm->vmcb01.ptr, svm->nested.vmcb02.ptr); 1046 1047 svm_switch_vmcb(svm, &svm->nested.vmcb02); 1048 nested_vmcb02_prepare_control(svm); 1049 nested_vmcb02_prepare_save(svm); 1050 1051 ret = nested_svm_load_cr3(&svm->vcpu, svm->nested.save.cr3, 1052 nested_npt_enabled(svm), from_vmrun); 1053 if (ret) 1054 return ret; 1055 1056 if (!from_vmrun) 1057 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 1058 1059 svm_set_gif(svm, true); 1060 1061 if (kvm_vcpu_apicv_active(vcpu)) 1062 kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu); 1063 1064 nested_svm_hv_update_vm_vp_ids(vcpu); 1065 1066 return 0; 1067 } 1068 1069 static int nested_svm_copy_vmcb12_to_cache(struct kvm_vcpu *vcpu, u64 vmcb12_gpa) 1070 { 1071 struct vcpu_svm *svm = to_svm(vcpu); 1072 struct kvm_host_map map; 1073 struct vmcb *vmcb12; 1074 int r = 0; 1075 1076 if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcb12_gpa), &map)) 1077 return -EFAULT; 1078 1079 vmcb12 = map.hva; 1080 nested_copy_vmcb_control_to_cache(svm, &vmcb12->control); 1081 nested_copy_vmcb_save_to_cache(svm, &vmcb12->save); 1082 1083 if (nested_svm_check_cached_vmcb12(vcpu) < 0) { 1084 vmcb12->control.exit_code = SVM_EXIT_ERR; 1085 vmcb12->control.exit_info_1 = 0; 1086 vmcb12->control.exit_info_2 = 0; 1087 vmcb12->control.event_inj = 0; 1088 vmcb12->control.event_inj_err = 0; 1089 svm_set_gif(svm, false); 1090 r = -EINVAL; 1091 } 1092 1093 kvm_vcpu_unmap(vcpu, &map); 1094 return r; 1095 } 1096 1097 int nested_svm_vmrun(struct kvm_vcpu *vcpu) 1098 { 1099 struct vcpu_svm *svm = to_svm(vcpu); 1100 int ret; 1101 u64 vmcb12_gpa; 1102 struct vmcb *vmcb01 = svm->vmcb01.ptr; 1103 1104 if (!svm->nested.hsave_msr) { 1105 kvm_inject_gp(vcpu, 0); 1106 return 1; 1107 } 1108 1109 if (is_smm(vcpu)) { 1110 kvm_queue_exception(vcpu, UD_VECTOR); 1111 return 1; 1112 } 1113 1114 /* This fails when VP assist page is enabled but the supplied GPA is bogus */ 1115 ret = kvm_hv_verify_vp_assist(vcpu); 1116 if (ret) { 1117 kvm_inject_gp(vcpu, 0); 1118 return ret; 1119 } 1120 1121 if (WARN_ON_ONCE(!svm->nested.initialized)) 1122 return -EINVAL; 1123 1124 vmcb12_gpa = kvm_register_read(vcpu, VCPU_REGS_RAX); 1125 if (!page_address_valid(vcpu, vmcb12_gpa)) { 1126 kvm_inject_gp(vcpu, 0); 1127 return 1; 1128 } 1129 1130 ret = nested_svm_copy_vmcb12_to_cache(vcpu, vmcb12_gpa); 1131 if (ret) { 1132 if (ret == -EFAULT) 1133 return kvm_handle_memory_failure(vcpu, X86EMUL_IO_NEEDED, NULL); 1134 1135 /* Advance RIP past VMRUN as part of the nested #VMEXIT. */ 1136 return kvm_skip_emulated_instruction(vcpu); 1137 } 1138 1139 /* At this point, VMRUN is guaranteed to not fault; advance RIP. */ 1140 ret = kvm_skip_emulated_instruction(vcpu); 1141 1142 /* 1143 * Since vmcb01 is not in use, we can use it to store some of the L1 1144 * state. 1145 */ 1146 vmcb01->save.efer = vcpu->arch.efer; 1147 vmcb01->save.cr0 = kvm_read_cr0(vcpu); 1148 vmcb01->save.cr4 = vcpu->arch.cr4; 1149 vmcb01->save.rflags = kvm_get_rflags(vcpu); 1150 vmcb01->save.rip = kvm_rip_read(vcpu); 1151 1152 if (!npt_enabled) 1153 vmcb01->save.cr3 = kvm_read_cr3(vcpu); 1154 1155 vcpu->arch.nested_run_pending = KVM_NESTED_RUN_PENDING; 1156 1157 if (enter_svm_guest_mode(vcpu, vmcb12_gpa, true) || 1158 !nested_svm_merge_msrpm(vcpu)) { 1159 vcpu->arch.nested_run_pending = 0; 1160 svm->nmi_l1_to_l2 = false; 1161 svm->soft_int_injected = false; 1162 1163 svm->vmcb->control.exit_code = SVM_EXIT_ERR; 1164 svm->vmcb->control.exit_info_1 = 0; 1165 svm->vmcb->control.exit_info_2 = 0; 1166 1167 nested_svm_vmexit(svm); 1168 } 1169 1170 return ret; 1171 } 1172 1173 /* Copy state save area fields which are handled by VMRUN */ 1174 void svm_copy_vmrun_state(struct vmcb_save_area *to_save, 1175 struct vmcb_save_area *from_save) 1176 { 1177 to_save->es = from_save->es; 1178 to_save->cs = from_save->cs; 1179 to_save->ss = from_save->ss; 1180 to_save->ds = from_save->ds; 1181 to_save->gdtr = from_save->gdtr; 1182 to_save->idtr = from_save->idtr; 1183 to_save->rflags = from_save->rflags | X86_EFLAGS_FIXED; 1184 to_save->efer = from_save->efer; 1185 to_save->cr0 = from_save->cr0; 1186 to_save->cr3 = from_save->cr3; 1187 to_save->cr4 = from_save->cr4; 1188 to_save->rax = from_save->rax; 1189 to_save->rsp = from_save->rsp; 1190 to_save->rip = from_save->rip; 1191 to_save->cpl = 0; 1192 1193 if (kvm_cpu_cap_has(X86_FEATURE_SHSTK)) { 1194 to_save->s_cet = from_save->s_cet; 1195 to_save->isst_addr = from_save->isst_addr; 1196 to_save->ssp = from_save->ssp; 1197 } 1198 1199 if (kvm_cpu_cap_has(X86_FEATURE_LBRV)) { 1200 svm_copy_lbrs(to_save, from_save); 1201 to_save->dbgctl &= ~DEBUGCTL_RESERVED_BITS; 1202 } 1203 } 1204 1205 void svm_copy_vmloadsave_state(struct vmcb *to_vmcb, struct vmcb *from_vmcb) 1206 { 1207 to_vmcb->save.fs = from_vmcb->save.fs; 1208 to_vmcb->save.gs = from_vmcb->save.gs; 1209 to_vmcb->save.tr = from_vmcb->save.tr; 1210 to_vmcb->save.ldtr = from_vmcb->save.ldtr; 1211 to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base; 1212 to_vmcb->save.star = from_vmcb->save.star; 1213 to_vmcb->save.lstar = from_vmcb->save.lstar; 1214 to_vmcb->save.cstar = from_vmcb->save.cstar; 1215 to_vmcb->save.sfmask = from_vmcb->save.sfmask; 1216 to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs; 1217 to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp; 1218 to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip; 1219 } 1220 1221 static int nested_svm_vmexit_update_vmcb12(struct kvm_vcpu *vcpu) 1222 { 1223 struct vcpu_svm *svm = to_svm(vcpu); 1224 struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; 1225 struct kvm_host_map map; 1226 struct vmcb *vmcb12; 1227 int rc; 1228 1229 rc = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa), &map); 1230 if (rc) 1231 return rc; 1232 1233 vmcb12 = map.hva; 1234 1235 vmcb12->save.es = vmcb02->save.es; 1236 vmcb12->save.cs = vmcb02->save.cs; 1237 vmcb12->save.ss = vmcb02->save.ss; 1238 vmcb12->save.ds = vmcb02->save.ds; 1239 vmcb12->save.gdtr = vmcb02->save.gdtr; 1240 vmcb12->save.idtr = vmcb02->save.idtr; 1241 vmcb12->save.efer = svm->vcpu.arch.efer; 1242 vmcb12->save.cr0 = kvm_read_cr0(vcpu); 1243 vmcb12->save.cr3 = kvm_read_cr3(vcpu); 1244 vmcb12->save.cr2 = vcpu->arch.cr2; 1245 vmcb12->save.cr4 = svm->vcpu.arch.cr4; 1246 vmcb12->save.rflags = kvm_get_rflags(vcpu); 1247 vmcb12->save.rip = kvm_rip_read(vcpu); 1248 vmcb12->save.rsp = kvm_rsp_read(vcpu); 1249 vmcb12->save.rax = kvm_rax_read(vcpu); 1250 vmcb12->save.dr7 = vmcb02->save.dr7; 1251 vmcb12->save.dr6 = svm->vcpu.arch.dr6; 1252 vmcb12->save.cpl = vmcb02->save.cpl; 1253 1254 if (guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK)) { 1255 vmcb12->save.s_cet = vmcb02->save.s_cet; 1256 vmcb12->save.isst_addr = vmcb02->save.isst_addr; 1257 vmcb12->save.ssp = vmcb02->save.ssp; 1258 } 1259 1260 vmcb12->control.int_state = vmcb02->control.int_state; 1261 vmcb12->control.exit_code = vmcb02->control.exit_code; 1262 vmcb12->control.exit_info_1 = vmcb02->control.exit_info_1; 1263 vmcb12->control.exit_info_2 = vmcb02->control.exit_info_2; 1264 1265 if (!svm_is_vmrun_failure(vmcb12->control.exit_code)) 1266 nested_save_pending_event_to_vmcb12(svm, vmcb12); 1267 1268 if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS)) 1269 vmcb12->control.next_rip = vmcb02->control.next_rip; 1270 1271 if (nested_vmcb12_has_lbrv(vcpu)) 1272 svm_copy_lbrs(&vmcb12->save, &vmcb02->save); 1273 1274 vmcb12->control.event_inj = 0; 1275 vmcb12->control.event_inj_err = 0; 1276 vmcb12->control.int_ctl = svm->nested.ctl.int_ctl; 1277 1278 trace_kvm_nested_vmexit_inject(vmcb12->control.exit_code, 1279 vmcb12->control.exit_info_1, 1280 vmcb12->control.exit_info_2, 1281 vmcb12->control.exit_int_info, 1282 vmcb12->control.exit_int_info_err, 1283 KVM_ISA_SVM); 1284 1285 kvm_vcpu_unmap(vcpu, &map); 1286 return 0; 1287 } 1288 1289 void nested_svm_vmexit(struct vcpu_svm *svm) 1290 { 1291 struct kvm_vcpu *vcpu = &svm->vcpu; 1292 struct vmcb *vmcb01 = svm->vmcb01.ptr; 1293 struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; 1294 1295 if (nested_svm_vmexit_update_vmcb12(vcpu)) 1296 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 1297 1298 /* Exit Guest-Mode */ 1299 leave_guest_mode(vcpu); 1300 svm->nested.vmcb12_gpa = 0; 1301 1302 kvm_warn_on_nested_run_pending(vcpu); 1303 1304 kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 1305 1306 /* in case we halted in L2 */ 1307 kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); 1308 1309 if (!kvm_pause_in_guest(vcpu->kvm)) { 1310 vmcb01->control.pause_filter_count = vmcb02->control.pause_filter_count; 1311 vmcb_mark_dirty(vmcb01, VMCB_INTERCEPTS); 1312 1313 } 1314 1315 /* 1316 * Invalidate last_bus_lock_rip unless KVM is still waiting for the 1317 * guest to make forward progress before re-enabling bus lock detection. 1318 */ 1319 if (!vmcb02->control.bus_lock_counter) 1320 svm->nested.last_bus_lock_rip = INVALID_GPA; 1321 1322 nested_svm_copy_common_state(svm->nested.vmcb02.ptr, svm->vmcb01.ptr); 1323 1324 kvm_nested_vmexit_handle_ibrs(vcpu); 1325 1326 if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS)) 1327 vmcb01->control.erap_ctl |= ERAP_CONTROL_CLEAR_RAP; 1328 1329 svm_switch_vmcb(svm, &svm->vmcb01); 1330 1331 /* 1332 * Rules for synchronizing int_ctl bits from vmcb02 to vmcb01: 1333 * 1334 * V_IRQ, V_IRQ_VECTOR, V_INTR_PRIO_MASK, V_IGN_TPR: If L1 doesn't 1335 * intercept interrupts, then KVM will use vmcb02's V_IRQ (and related 1336 * flags) to detect interrupt windows for L1 IRQs (even if L1 uses 1337 * virtual interrupt masking). Raise KVM_REQ_EVENT to ensure that 1338 * KVM re-requests an interrupt window if necessary, which implicitly 1339 * copies this bits from vmcb02 to vmcb01. 1340 * 1341 * V_TPR: If L1 doesn't use virtual interrupt masking, then L1's vTPR 1342 * is stored in vmcb02, but its value doesn't need to be copied from/to 1343 * vmcb01 because it is copied from/to the virtual APIC's TPR register 1344 * on each VM entry/exit. 1345 * 1346 * V_GIF: If nested vGIF is not used, KVM uses vmcb02's V_GIF for L1's 1347 * V_GIF. However, GIF is architecturally clear on each VM exit, thus 1348 * there is no need to copy V_GIF from vmcb02 to vmcb01. 1349 */ 1350 if (!nested_exit_on_intr(svm)) 1351 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); 1352 1353 if (!nested_vmcb12_has_lbrv(vcpu)) { 1354 svm_copy_lbrs(&vmcb01->save, &vmcb02->save); 1355 vmcb_mark_dirty(vmcb01, VMCB_LBR); 1356 } 1357 1358 svm_update_lbrv(vcpu); 1359 1360 if (vnmi) { 1361 if (vmcb02->control.int_ctl & V_NMI_BLOCKING_MASK) 1362 vmcb01->control.int_ctl |= V_NMI_BLOCKING_MASK; 1363 else 1364 vmcb01->control.int_ctl &= ~V_NMI_BLOCKING_MASK; 1365 1366 if (vcpu->arch.nmi_pending) { 1367 vcpu->arch.nmi_pending--; 1368 vmcb01->control.int_ctl |= V_NMI_PENDING_MASK; 1369 } else { 1370 vmcb01->control.int_ctl &= ~V_NMI_PENDING_MASK; 1371 } 1372 } 1373 1374 /* 1375 * On vmexit the GIF is set to false and 1376 * no event can be injected in L1. 1377 */ 1378 svm_set_gif(svm, false); 1379 vmcb01->control.exit_int_info = 0; 1380 1381 svm->vcpu.arch.tsc_offset = svm->vcpu.arch.l1_tsc_offset; 1382 if (vmcb01->control.tsc_offset != svm->vcpu.arch.tsc_offset) { 1383 vmcb01->control.tsc_offset = svm->vcpu.arch.tsc_offset; 1384 vmcb_mark_dirty(vmcb01, VMCB_INTERCEPTS); 1385 } 1386 1387 if (kvm_caps.has_tsc_control && 1388 vcpu->arch.tsc_scaling_ratio != vcpu->arch.l1_tsc_scaling_ratio) { 1389 vcpu->arch.tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; 1390 svm_write_tsc_multiplier(vcpu); 1391 } 1392 1393 svm->nested.ctl.nested_cr3 = 0; 1394 1395 /* 1396 * Restore processor state that had been saved in vmcb01 1397 */ 1398 kvm_set_rflags(vcpu, vmcb01->save.rflags); 1399 svm_set_efer(vcpu, vmcb01->save.efer); 1400 svm_set_cr0(vcpu, vmcb01->save.cr0 | X86_CR0_PE); 1401 svm_set_cr4(vcpu, vmcb01->save.cr4); 1402 kvm_rax_write(vcpu, vmcb01->save.rax); 1403 kvm_rsp_write(vcpu, vmcb01->save.rsp); 1404 kvm_rip_write(vcpu, vmcb01->save.rip); 1405 1406 svm->vcpu.arch.dr7 = DR7_FIXED_1; 1407 kvm_update_dr7(&svm->vcpu); 1408 1409 nested_svm_transition_tlb_flush(vcpu); 1410 1411 nested_svm_uninit_mmu_context(vcpu); 1412 1413 if (nested_svm_load_cr3(vcpu, vmcb01->save.cr3, false, true)) 1414 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 1415 1416 /* Drop tracking for L1->L2 injected NMIs and soft IRQs */ 1417 svm->nmi_l1_to_l2 = false; 1418 svm->soft_int_injected = false; 1419 1420 /* 1421 * Drop what we picked up for L2 via svm_complete_interrupts() so it 1422 * doesn't end up in L1. 1423 */ 1424 svm->vcpu.arch.nmi_injected = false; 1425 kvm_clear_exception_queue(vcpu); 1426 kvm_clear_interrupt_queue(vcpu); 1427 1428 /* 1429 * If we are here following the completion of a VMRUN that 1430 * is being single-stepped, queue the pending #DB intercept 1431 * right now so that it an be accounted for before we execute 1432 * L1's next instruction. 1433 */ 1434 if (unlikely(vmcb01->save.rflags & X86_EFLAGS_TF)) 1435 kvm_queue_exception(&(svm->vcpu), DB_VECTOR); 1436 1437 /* 1438 * Un-inhibit the AVIC right away, so that other vCPUs can start 1439 * to benefit from it right away. 1440 */ 1441 if (kvm_apicv_activated(vcpu->kvm)) 1442 __kvm_vcpu_update_apicv(vcpu); 1443 } 1444 1445 static void nested_svm_triple_fault(struct kvm_vcpu *vcpu) 1446 { 1447 struct vcpu_svm *svm = to_svm(vcpu); 1448 1449 if (!vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SHUTDOWN)) 1450 return; 1451 1452 kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu); 1453 nested_svm_simple_vmexit(to_svm(vcpu), SVM_EXIT_SHUTDOWN); 1454 } 1455 1456 int svm_allocate_nested(struct vcpu_svm *svm) 1457 { 1458 struct page *vmcb02_page; 1459 1460 if (svm->nested.initialized) 1461 return 0; 1462 1463 vmcb02_page = snp_safe_alloc_page(); 1464 if (!vmcb02_page) 1465 return -ENOMEM; 1466 svm->nested.vmcb02.ptr = page_address(vmcb02_page); 1467 svm->nested.vmcb02.pa = __sme_set(page_to_pfn(vmcb02_page) << PAGE_SHIFT); 1468 1469 svm->nested.msrpm = svm_vcpu_alloc_msrpm(); 1470 if (!svm->nested.msrpm) 1471 goto err_free_vmcb02; 1472 1473 svm->nested.initialized = true; 1474 return 0; 1475 1476 err_free_vmcb02: 1477 __free_page(vmcb02_page); 1478 return -ENOMEM; 1479 } 1480 1481 void svm_free_nested(struct vcpu_svm *svm) 1482 { 1483 if (!svm->nested.initialized) 1484 return; 1485 1486 if (WARN_ON_ONCE(svm->vmcb != svm->vmcb01.ptr)) 1487 svm_switch_vmcb(svm, &svm->vmcb01); 1488 1489 svm_vcpu_free_msrpm(svm->nested.msrpm); 1490 svm->nested.msrpm = NULL; 1491 1492 __free_page(virt_to_page(svm->nested.vmcb02.ptr)); 1493 svm->nested.vmcb02.ptr = NULL; 1494 1495 /* 1496 * When last_vmcb12_gpa matches the current vmcb12 gpa, 1497 * some vmcb12 fields are not loaded if they are marked clean 1498 * in the vmcb12, since in this case they are up to date already. 1499 * 1500 * When the vmcb02 is freed, this optimization becomes invalid. 1501 */ 1502 svm->nested.last_vmcb12_gpa = INVALID_GPA; 1503 1504 svm->nested.initialized = false; 1505 } 1506 1507 void svm_leave_nested(struct kvm_vcpu *vcpu) 1508 { 1509 struct vcpu_svm *svm = to_svm(vcpu); 1510 1511 if (is_guest_mode(vcpu)) { 1512 vcpu->arch.nested_run_pending = 0; 1513 svm->nested.vmcb12_gpa = INVALID_GPA; 1514 1515 leave_guest_mode(vcpu); 1516 1517 svm_switch_vmcb(svm, &svm->vmcb01); 1518 1519 nested_svm_uninit_mmu_context(vcpu); 1520 vmcb_mark_all_dirty(svm->vmcb); 1521 1522 svm_set_gif(svm, true); 1523 1524 if (kvm_apicv_activated(vcpu->kvm)) 1525 kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu); 1526 } 1527 1528 kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 1529 } 1530 1531 static int nested_svm_exit_handled_msr(struct vcpu_svm *svm) 1532 { 1533 gpa_t base = svm->nested.ctl.msrpm_base_pa; 1534 int write, bit_nr; 1535 u8 value, mask; 1536 u32 msr; 1537 1538 if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT))) 1539 return NESTED_EXIT_HOST; 1540 1541 msr = svm->vcpu.arch.regs[VCPU_REGS_RCX]; 1542 bit_nr = svm_msrpm_bit_nr(msr); 1543 write = svm->vmcb->control.exit_info_1 & 1; 1544 1545 if (bit_nr < 0) 1546 return NESTED_EXIT_DONE; 1547 1548 if (kvm_vcpu_read_guest(&svm->vcpu, base + bit_nr / BITS_PER_BYTE, 1549 &value, sizeof(value))) 1550 return NESTED_EXIT_DONE; 1551 1552 mask = BIT(write) << (bit_nr & (BITS_PER_BYTE - 1)); 1553 return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; 1554 } 1555 1556 static int nested_svm_intercept_ioio(struct vcpu_svm *svm) 1557 { 1558 unsigned port, size, iopm_len; 1559 u16 val, mask; 1560 u8 start_bit; 1561 u64 gpa; 1562 1563 if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_IOIO_PROT))) 1564 return NESTED_EXIT_HOST; 1565 1566 port = svm->vmcb->control.exit_info_1 >> 16; 1567 size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >> 1568 SVM_IOIO_SIZE_SHIFT; 1569 gpa = svm->nested.ctl.iopm_base_pa + (port / 8); 1570 start_bit = port % 8; 1571 iopm_len = (start_bit + size > 8) ? 2 : 1; 1572 mask = (0xf >> (4 - size)) << start_bit; 1573 val = 0; 1574 1575 if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len)) 1576 return NESTED_EXIT_DONE; 1577 1578 return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; 1579 } 1580 1581 static int nested_svm_intercept(struct vcpu_svm *svm) 1582 { 1583 u64 exit_code = svm->vmcb->control.exit_code; 1584 int vmexit = NESTED_EXIT_HOST; 1585 1586 if (svm_is_vmrun_failure(exit_code)) 1587 return NESTED_EXIT_DONE; 1588 1589 switch (exit_code) { 1590 case SVM_EXIT_MSR: 1591 vmexit = nested_svm_exit_handled_msr(svm); 1592 break; 1593 case SVM_EXIT_IOIO: 1594 vmexit = nested_svm_intercept_ioio(svm); 1595 break; 1596 case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: 1597 /* 1598 * Host-intercepted exceptions have been checked already in 1599 * nested_svm_exit_special. There is nothing to do here, 1600 * the vmexit is injected by svm_check_nested_events. 1601 */ 1602 vmexit = NESTED_EXIT_DONE; 1603 break; 1604 default: 1605 if (vmcb12_is_intercept(&svm->nested.ctl, exit_code)) 1606 vmexit = NESTED_EXIT_DONE; 1607 break; 1608 } 1609 1610 return vmexit; 1611 } 1612 1613 int nested_svm_exit_handled(struct vcpu_svm *svm) 1614 { 1615 int vmexit; 1616 1617 vmexit = nested_svm_intercept(svm); 1618 1619 if (vmexit == NESTED_EXIT_DONE) 1620 nested_svm_vmexit(svm); 1621 1622 return vmexit; 1623 } 1624 1625 int nested_svm_check_permissions(struct kvm_vcpu *vcpu) 1626 { 1627 if (!(vcpu->arch.efer & EFER_SVME) || !is_paging(vcpu)) { 1628 kvm_queue_exception(vcpu, UD_VECTOR); 1629 return 1; 1630 } 1631 1632 if (to_svm(vcpu)->vmcb->save.cpl) { 1633 kvm_inject_gp(vcpu, 0); 1634 return 1; 1635 } 1636 1637 return 0; 1638 } 1639 1640 static bool nested_svm_is_exception_vmexit(struct kvm_vcpu *vcpu, u8 vector, 1641 u32 error_code) 1642 { 1643 struct vcpu_svm *svm = to_svm(vcpu); 1644 1645 return (svm->nested.ctl.intercepts[INTERCEPT_EXCEPTION] & BIT(vector)); 1646 } 1647 1648 static void nested_svm_inject_exception_vmexit(struct kvm_vcpu *vcpu) 1649 { 1650 struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit; 1651 struct vcpu_svm *svm = to_svm(vcpu); 1652 struct vmcb *vmcb = svm->vmcb; 1653 1654 vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + ex->vector; 1655 1656 if (ex->has_error_code) 1657 vmcb->control.exit_info_1 = ex->error_code; 1658 1659 /* 1660 * EXITINFO2 is undefined for all exception intercepts other 1661 * than #PF. 1662 */ 1663 if (ex->vector == PF_VECTOR) { 1664 if (ex->has_payload) 1665 vmcb->control.exit_info_2 = ex->payload; 1666 else 1667 vmcb->control.exit_info_2 = vcpu->arch.cr2; 1668 } else if (ex->vector == DB_VECTOR) { 1669 /* See kvm_check_and_inject_events(). */ 1670 kvm_deliver_exception_payload(vcpu, ex); 1671 1672 if (vcpu->arch.dr7 & DR7_GD) { 1673 vcpu->arch.dr7 &= ~DR7_GD; 1674 kvm_update_dr7(vcpu); 1675 } 1676 } else { 1677 WARN_ON(ex->has_payload); 1678 } 1679 1680 nested_svm_vmexit(svm); 1681 } 1682 1683 static inline bool nested_exit_on_init(struct vcpu_svm *svm) 1684 { 1685 return vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_INIT); 1686 } 1687 1688 static int svm_check_nested_events(struct kvm_vcpu *vcpu) 1689 { 1690 struct kvm_lapic *apic = vcpu->arch.apic; 1691 struct vcpu_svm *svm = to_svm(vcpu); 1692 /* 1693 * Only a pending nested run blocks a pending exception. If there is a 1694 * previously injected event, the pending exception occurred while said 1695 * event was being delivered and thus needs to be handled. 1696 */ 1697 bool block_nested_exceptions = vcpu->arch.nested_run_pending; 1698 /* 1699 * New events (not exceptions) are only recognized at instruction 1700 * boundaries. If an event needs reinjection, then KVM is handling a 1701 * VM-Exit that occurred _during_ instruction execution; new events are 1702 * blocked until the instruction completes. 1703 */ 1704 bool block_nested_events = block_nested_exceptions || 1705 kvm_event_needs_reinjection(vcpu); 1706 1707 if (lapic_in_kernel(vcpu) && 1708 test_bit(KVM_APIC_INIT, &apic->pending_events)) { 1709 if (block_nested_events) 1710 return -EBUSY; 1711 if (!nested_exit_on_init(svm)) 1712 return 0; 1713 nested_svm_simple_vmexit(svm, SVM_EXIT_INIT); 1714 return 0; 1715 } 1716 1717 if (vcpu->arch.exception_vmexit.pending) { 1718 if (block_nested_exceptions) 1719 return -EBUSY; 1720 nested_svm_inject_exception_vmexit(vcpu); 1721 return 0; 1722 } 1723 1724 if (vcpu->arch.exception.pending) { 1725 if (block_nested_exceptions) 1726 return -EBUSY; 1727 return 0; 1728 } 1729 1730 #ifdef CONFIG_KVM_SMM 1731 if (vcpu->arch.smi_pending && !svm_smi_blocked(vcpu)) { 1732 if (block_nested_events) 1733 return -EBUSY; 1734 if (!nested_exit_on_smi(svm)) 1735 return 0; 1736 nested_svm_simple_vmexit(svm, SVM_EXIT_SMI); 1737 return 0; 1738 } 1739 #endif 1740 1741 if (vcpu->arch.nmi_pending && !svm_nmi_blocked(vcpu)) { 1742 if (block_nested_events) 1743 return -EBUSY; 1744 if (!nested_exit_on_nmi(svm)) 1745 return 0; 1746 nested_svm_simple_vmexit(svm, SVM_EXIT_NMI); 1747 return 0; 1748 } 1749 1750 if (kvm_cpu_has_interrupt(vcpu) && !svm_interrupt_blocked(vcpu)) { 1751 if (block_nested_events) 1752 return -EBUSY; 1753 if (!nested_exit_on_intr(svm)) 1754 return 0; 1755 trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip); 1756 nested_svm_simple_vmexit(svm, SVM_EXIT_INTR); 1757 return 0; 1758 } 1759 1760 return 0; 1761 } 1762 1763 int nested_svm_exit_special(struct vcpu_svm *svm) 1764 { 1765 u32 exit_code = svm->vmcb->control.exit_code; 1766 struct kvm_vcpu *vcpu = &svm->vcpu; 1767 1768 switch (exit_code) { 1769 case SVM_EXIT_INTR: 1770 case SVM_EXIT_NMI: 1771 case SVM_EXIT_NPF: 1772 return NESTED_EXIT_HOST; 1773 case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: { 1774 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE); 1775 1776 if (svm->vmcb01.ptr->control.intercepts[INTERCEPT_EXCEPTION] & 1777 excp_bits) 1778 return NESTED_EXIT_HOST; 1779 else if (exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR && 1780 svm->vcpu.arch.apf.host_apf_flags) 1781 /* Trap async PF even if not shadowing */ 1782 return NESTED_EXIT_HOST; 1783 break; 1784 } 1785 case SVM_EXIT_VMMCALL: 1786 /* Hyper-V L2 TLB flush hypercall is handled by L0 */ 1787 if (nested_svm_is_l2_tlb_flush_hcall(vcpu)) 1788 return NESTED_EXIT_HOST; 1789 break; 1790 default: 1791 break; 1792 } 1793 1794 return NESTED_EXIT_CONTINUE; 1795 } 1796 1797 void nested_svm_update_tsc_ratio_msr(struct kvm_vcpu *vcpu) 1798 { 1799 struct vcpu_svm *svm = to_svm(vcpu); 1800 1801 vcpu->arch.tsc_scaling_ratio = 1802 kvm_calc_nested_tsc_multiplier(vcpu->arch.l1_tsc_scaling_ratio, 1803 svm->tsc_ratio_msr); 1804 svm_write_tsc_multiplier(vcpu); 1805 } 1806 1807 /* Inverse operation of nested_copy_vmcb_control_to_cache(). asid is copied too. */ 1808 static void nested_copy_vmcb_cache_to_control(struct vmcb_control_area *dst, 1809 struct vmcb_ctrl_area_cached *from) 1810 { 1811 unsigned int i; 1812 1813 memset(dst, 0, sizeof(struct vmcb_control_area)); 1814 1815 for (i = 0; i < MAX_INTERCEPT; i++) 1816 dst->intercepts[i] = from->intercepts[i]; 1817 1818 dst->iopm_base_pa = from->iopm_base_pa; 1819 dst->msrpm_base_pa = from->msrpm_base_pa; 1820 dst->tsc_offset = from->tsc_offset; 1821 dst->asid = from->asid; 1822 dst->tlb_ctl = from->tlb_ctl; 1823 dst->erap_ctl = from->erap_ctl; 1824 dst->int_ctl = from->int_ctl; 1825 dst->int_vector = from->int_vector; 1826 dst->int_state = from->int_state; 1827 dst->exit_code = from->exit_code; 1828 dst->exit_info_1 = from->exit_info_1; 1829 dst->exit_info_2 = from->exit_info_2; 1830 dst->exit_int_info = from->exit_int_info; 1831 dst->exit_int_info_err = from->exit_int_info_err; 1832 dst->misc_ctl = from->misc_ctl; 1833 dst->event_inj = from->event_inj; 1834 dst->event_inj_err = from->event_inj_err; 1835 dst->next_rip = from->next_rip; 1836 dst->nested_cr3 = from->nested_cr3; 1837 dst->misc_ctl2 = from->misc_ctl2; 1838 dst->pause_filter_count = from->pause_filter_count; 1839 dst->pause_filter_thresh = from->pause_filter_thresh; 1840 /* 'clean' and 'hv_enlightenments' are not changed by KVM */ 1841 } 1842 1843 static int svm_get_nested_state(struct kvm_vcpu *vcpu, 1844 struct kvm_nested_state __user *user_kvm_nested_state, 1845 u32 user_data_size) 1846 { 1847 struct vcpu_svm *svm; 1848 struct vmcb_control_area *ctl; 1849 unsigned long r; 1850 struct kvm_nested_state kvm_state = { 1851 .flags = 0, 1852 .format = KVM_STATE_NESTED_FORMAT_SVM, 1853 .size = sizeof(kvm_state), 1854 }; 1855 struct vmcb __user *user_vmcb = (struct vmcb __user *) 1856 &user_kvm_nested_state->data.svm[0]; 1857 1858 if (!vcpu) 1859 return kvm_state.size + KVM_STATE_NESTED_SVM_VMCB_SIZE; 1860 1861 svm = to_svm(vcpu); 1862 1863 if (user_data_size < kvm_state.size) 1864 goto out; 1865 1866 /* First fill in the header and copy it out. */ 1867 if (is_guest_mode(vcpu)) { 1868 kvm_state.hdr.svm.vmcb_pa = svm->nested.vmcb12_gpa; 1869 kvm_state.size += KVM_STATE_NESTED_SVM_VMCB_SIZE; 1870 kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE; 1871 1872 if (vcpu->arch.nested_run_pending) 1873 kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING; 1874 } 1875 1876 if (gif_set(svm)) 1877 kvm_state.flags |= KVM_STATE_NESTED_GIF_SET; 1878 1879 if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state))) 1880 return -EFAULT; 1881 1882 if (!is_guest_mode(vcpu)) 1883 goto out; 1884 1885 /* 1886 * Copy over the full size of the VMCB rather than just the size 1887 * of the structs. 1888 */ 1889 if (clear_user(user_vmcb, KVM_STATE_NESTED_SVM_VMCB_SIZE)) 1890 return -EFAULT; 1891 1892 ctl = kzalloc_obj(*ctl); 1893 if (!ctl) 1894 return -ENOMEM; 1895 1896 nested_copy_vmcb_cache_to_control(ctl, &svm->nested.ctl); 1897 r = copy_to_user(&user_vmcb->control, ctl, 1898 sizeof(user_vmcb->control)); 1899 kfree(ctl); 1900 if (r) 1901 return -EFAULT; 1902 1903 if (copy_to_user(&user_vmcb->save, &svm->vmcb01.ptr->save, 1904 sizeof(user_vmcb->save))) 1905 return -EFAULT; 1906 out: 1907 return kvm_state.size; 1908 } 1909 1910 static int svm_set_nested_state(struct kvm_vcpu *vcpu, 1911 struct kvm_nested_state __user *user_kvm_nested_state, 1912 struct kvm_nested_state *kvm_state) 1913 { 1914 struct vcpu_svm *svm = to_svm(vcpu); 1915 struct vmcb __user *user_vmcb = (struct vmcb __user *) 1916 &user_kvm_nested_state->data.svm[0]; 1917 struct vmcb_control_area *ctl; 1918 struct vmcb_save_area *save; 1919 struct vmcb_save_area_cached save_cached; 1920 struct vmcb_ctrl_area_cached ctl_cached; 1921 unsigned long cr0; 1922 int ret; 1923 1924 BUILD_BUG_ON(sizeof(struct vmcb_control_area) + sizeof(struct vmcb_save_area) > 1925 KVM_STATE_NESTED_SVM_VMCB_SIZE); 1926 1927 if (kvm_state->format != KVM_STATE_NESTED_FORMAT_SVM) 1928 return -EINVAL; 1929 1930 if (kvm_state->flags & ~(KVM_STATE_NESTED_GUEST_MODE | 1931 KVM_STATE_NESTED_RUN_PENDING | 1932 KVM_STATE_NESTED_GIF_SET)) 1933 return -EINVAL; 1934 1935 /* 1936 * If in guest mode, vcpu->arch.efer actually refers to the L2 guest's 1937 * EFER.SVME, but EFER.SVME still has to be 1 for VMRUN to succeed. 1938 * If SVME is disabled, the only valid states are "none" and GIF=1 1939 * (clearing SVME does NOT set GIF, i.e. GIF=0 is allowed). 1940 */ 1941 if (!(vcpu->arch.efer & EFER_SVME) && kvm_state->flags && 1942 kvm_state->flags != KVM_STATE_NESTED_GIF_SET) 1943 return -EINVAL; 1944 1945 /* SMM temporarily disables SVM, so we cannot be in guest mode. */ 1946 if (is_smm(vcpu) && (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 1947 return -EINVAL; 1948 1949 if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) { 1950 svm_leave_nested(vcpu); 1951 svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET)); 1952 return 0; 1953 } 1954 1955 if (!page_address_valid(vcpu, kvm_state->hdr.svm.vmcb_pa)) 1956 return -EINVAL; 1957 if (kvm_state->size < sizeof(*kvm_state) + KVM_STATE_NESTED_SVM_VMCB_SIZE) 1958 return -EINVAL; 1959 1960 ctl = memdup_user(&user_vmcb->control, sizeof(*ctl)); 1961 if (IS_ERR(ctl)) 1962 return PTR_ERR(ctl); 1963 1964 save = memdup_user(&user_vmcb->save, sizeof(*save)); 1965 if (IS_ERR(save)) { 1966 kfree(ctl); 1967 return PTR_ERR(save); 1968 } 1969 1970 ret = -EINVAL; 1971 __nested_copy_vmcb_control_to_cache(vcpu, &ctl_cached, ctl); 1972 if (!nested_vmcb_check_controls(vcpu, &ctl_cached)) 1973 goto out_free; 1974 1975 /* 1976 * Processor state contains L2 state. Check that it is 1977 * valid for guest mode (see nested_vmcb_check_save()). 1978 */ 1979 cr0 = kvm_read_cr0(vcpu); 1980 if (((cr0 & X86_CR0_CD) == 0) && (cr0 & X86_CR0_NW)) 1981 goto out_free; 1982 1983 /* 1984 * Validate host state saved from before VMRUN (see 1985 * nested_svm_check_permissions). 1986 */ 1987 __nested_copy_vmcb_save_to_cache(&save_cached, save); 1988 if (!(save->cr0 & X86_CR0_PG) || 1989 !(save->cr0 & X86_CR0_PE) || 1990 (save->rflags & X86_EFLAGS_VM) || 1991 !nested_vmcb_check_save(vcpu, &save_cached)) 1992 goto out_free; 1993 1994 1995 /* 1996 * All checks done, we can enter guest mode. Userspace provides 1997 * vmcb12.control, which will be combined with L1 and stored into 1998 * vmcb02, and the L1 save state which we store in vmcb01. 1999 * L2 registers if needed are moved from the current VMCB to VMCB02. 2000 */ 2001 2002 if (is_guest_mode(vcpu)) 2003 svm_leave_nested(vcpu); 2004 else 2005 svm->nested.vmcb02.ptr->save = svm->vmcb01.ptr->save; 2006 2007 svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET)); 2008 2009 if (kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING) 2010 vcpu->arch.nested_run_pending = KVM_NESTED_RUN_PENDING_UNTRUSTED; 2011 else 2012 vcpu->arch.nested_run_pending = 0; 2013 2014 svm->nested.vmcb12_gpa = kvm_state->hdr.svm.vmcb_pa; 2015 2016 svm_copy_vmrun_state(&svm->vmcb01.ptr->save, save); 2017 nested_copy_vmcb_control_to_cache(svm, ctl); 2018 2019 svm_switch_vmcb(svm, &svm->nested.vmcb02); 2020 nested_vmcb02_prepare_control(svm); 2021 2022 /* 2023 * Any previously restored state (e.g. KVM_SET_SREGS) would mark fields 2024 * dirty in vmcb01 instead of vmcb02, so mark all of vmcb02 dirty here. 2025 */ 2026 vmcb_mark_all_dirty(svm->vmcb); 2027 2028 /* 2029 * While the nested guest CR3 is already checked and set by 2030 * KVM_SET_SREGS, it was set when nested state was yet loaded, 2031 * thus MMU might not be initialized correctly. 2032 * Set it again to fix this. 2033 */ 2034 ret = nested_svm_load_cr3(&svm->vcpu, vcpu->arch.cr3, 2035 nested_npt_enabled(svm), false); 2036 if (ret) 2037 goto out_free; 2038 2039 svm->nested.force_msr_bitmap_recalc = true; 2040 2041 if (kvm_vcpu_apicv_active(vcpu)) 2042 kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu); 2043 2044 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 2045 ret = 0; 2046 out_free: 2047 kfree(save); 2048 kfree(ctl); 2049 2050 return ret; 2051 } 2052 2053 static bool svm_get_nested_state_pages(struct kvm_vcpu *vcpu) 2054 { 2055 if (WARN_ON(!is_guest_mode(vcpu))) 2056 return true; 2057 2058 if (!vcpu->arch.pdptrs_from_userspace && 2059 !nested_npt_enabled(to_svm(vcpu)) && is_pae_paging(vcpu)) 2060 /* 2061 * Reload the guest's PDPTRs since after a migration 2062 * the guest CR3 might be restored prior to setting the nested 2063 * state which can lead to a load of wrong PDPTRs. 2064 */ 2065 if (CC(!load_pdptrs(vcpu, vcpu->arch.cr3))) 2066 return false; 2067 2068 if (!nested_svm_merge_msrpm(vcpu)) { 2069 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 2070 vcpu->run->internal.suberror = 2071 KVM_INTERNAL_ERROR_EMULATION; 2072 vcpu->run->internal.ndata = 0; 2073 return false; 2074 } 2075 2076 if (kvm_hv_verify_vp_assist(vcpu)) 2077 return false; 2078 2079 return true; 2080 } 2081 2082 static gpa_t svm_translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, 2083 u64 access, 2084 struct x86_exception *exception, 2085 u64 pte_access) 2086 { 2087 struct vcpu_svm *svm = to_svm(vcpu); 2088 struct kvm_mmu *mmu = vcpu->arch.mmu; 2089 2090 BUG_ON(!mmu_is_nested(vcpu)); 2091 2092 /* Non-GMET walks are always user-walks */ 2093 if (!(svm->nested.ctl.misc_ctl & SVM_MISC_ENABLE_GMET)) 2094 access |= PFERR_USER_MASK; 2095 2096 return mmu->gva_to_gpa(vcpu, mmu, gpa, access, exception); 2097 } 2098 2099 struct kvm_x86_nested_ops svm_nested_ops = { 2100 .leave_nested = svm_leave_nested, 2101 .translate_nested_gpa = svm_translate_nested_gpa, 2102 .is_exception_vmexit = nested_svm_is_exception_vmexit, 2103 .check_events = svm_check_nested_events, 2104 .triple_fault = nested_svm_triple_fault, 2105 .get_nested_state_pages = svm_get_nested_state_pages, 2106 .get_state = svm_get_nested_state, 2107 .set_state = svm_set_nested_state, 2108 .hv_inject_synthetic_vmexit_post_tlb_flush = svm_hv_inject_synthetic_vmexit_post_tlb_flush, 2109 }; 2110