1883b0a91SJoerg Roedel // SPDX-License-Identifier: GPL-2.0-only 2883b0a91SJoerg Roedel /* 3883b0a91SJoerg Roedel * Kernel-based Virtual Machine driver for Linux 4883b0a91SJoerg Roedel * 5883b0a91SJoerg Roedel * AMD SVM support 6883b0a91SJoerg Roedel * 7883b0a91SJoerg Roedel * Copyright (C) 2006 Qumranet, Inc. 8883b0a91SJoerg Roedel * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9883b0a91SJoerg Roedel * 10883b0a91SJoerg Roedel * Authors: 11883b0a91SJoerg Roedel * Yaniv Kamay <yaniv@qumranet.com> 12883b0a91SJoerg Roedel * Avi Kivity <avi@qumranet.com> 13883b0a91SJoerg Roedel */ 14883b0a91SJoerg Roedel 15883b0a91SJoerg Roedel #define pr_fmt(fmt) "SVM: " fmt 16883b0a91SJoerg Roedel 17883b0a91SJoerg Roedel #include <linux/kvm_types.h> 18883b0a91SJoerg Roedel #include <linux/kvm_host.h> 19883b0a91SJoerg Roedel #include <linux/kernel.h> 20883b0a91SJoerg Roedel 21883b0a91SJoerg Roedel #include <asm/msr-index.h> 225679b803SPaolo Bonzini #include <asm/debugreg.h> 23883b0a91SJoerg Roedel 24883b0a91SJoerg Roedel #include "kvm_emulate.h" 25883b0a91SJoerg Roedel #include "trace.h" 26883b0a91SJoerg Roedel #include "mmu.h" 27883b0a91SJoerg Roedel #include "x86.h" 28cc440cdaSPaolo Bonzini #include "cpuid.h" 295b672408SPaolo Bonzini #include "lapic.h" 30883b0a91SJoerg Roedel #include "svm.h" 3166c03a92SVitaly Kuznetsov #include "hyperv.h" 32883b0a91SJoerg Roedel 3311f0cbf0SSean Christopherson #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK 3411f0cbf0SSean Christopherson 35883b0a91SJoerg Roedel static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu, 36883b0a91SJoerg Roedel struct x86_exception *fault) 37883b0a91SJoerg Roedel { 38883b0a91SJoerg Roedel struct vcpu_svm *svm = to_svm(vcpu); 39883b0a91SJoerg Roedel 40883b0a91SJoerg Roedel if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) { 41883b0a91SJoerg Roedel /* 42883b0a91SJoerg Roedel * TODO: track the cause of the nested page fault, and 43883b0a91SJoerg Roedel * correctly fill in the high bits of exit_info_1. 44883b0a91SJoerg Roedel */ 45883b0a91SJoerg Roedel svm->vmcb->control.exit_code = SVM_EXIT_NPF; 46883b0a91SJoerg Roedel svm->vmcb->control.exit_code_hi = 0; 47883b0a91SJoerg Roedel svm->vmcb->control.exit_info_1 = (1ULL << 32); 48883b0a91SJoerg Roedel svm->vmcb->control.exit_info_2 = fault->address; 49883b0a91SJoerg Roedel } 50883b0a91SJoerg Roedel 51883b0a91SJoerg Roedel svm->vmcb->control.exit_info_1 &= ~0xffffffffULL; 52883b0a91SJoerg Roedel svm->vmcb->control.exit_info_1 |= fault->error_code; 53883b0a91SJoerg Roedel 54883b0a91SJoerg Roedel nested_svm_vmexit(svm); 55883b0a91SJoerg Roedel } 56883b0a91SJoerg Roedel 57a04aead1SPaolo Bonzini static void svm_inject_page_fault_nested(struct kvm_vcpu *vcpu, struct x86_exception *fault) 58a04aead1SPaolo Bonzini { 59a04aead1SPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 60a04aead1SPaolo Bonzini WARN_ON(!is_guest_mode(vcpu)); 61a04aead1SPaolo Bonzini 628fc78909SEmanuele Giuseppe Esposito if (vmcb12_is_intercept(&svm->nested.ctl, 638fc78909SEmanuele Giuseppe Esposito INTERCEPT_EXCEPTION_OFFSET + PF_VECTOR) && 64a04aead1SPaolo Bonzini !svm->nested.nested_run_pending) { 65a04aead1SPaolo Bonzini svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + PF_VECTOR; 66a04aead1SPaolo Bonzini svm->vmcb->control.exit_code_hi = 0; 67a04aead1SPaolo Bonzini svm->vmcb->control.exit_info_1 = fault->error_code; 68a04aead1SPaolo Bonzini svm->vmcb->control.exit_info_2 = fault->address; 69a04aead1SPaolo Bonzini nested_svm_vmexit(svm); 70a04aead1SPaolo Bonzini } else { 71a04aead1SPaolo Bonzini kvm_inject_page_fault(vcpu, fault); 72a04aead1SPaolo Bonzini } 73a04aead1SPaolo Bonzini } 74a04aead1SPaolo Bonzini 75883b0a91SJoerg Roedel static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index) 76883b0a91SJoerg Roedel { 77883b0a91SJoerg Roedel struct vcpu_svm *svm = to_svm(vcpu); 78e670bf68SPaolo Bonzini u64 cr3 = svm->nested.ctl.nested_cr3; 79883b0a91SJoerg Roedel u64 pdpte; 80883b0a91SJoerg Roedel int ret; 81883b0a91SJoerg Roedel 822732be90SSean Christopherson ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(cr3), &pdpte, 83883b0a91SJoerg Roedel offset_in_page(cr3) + index * 8, 8); 84883b0a91SJoerg Roedel if (ret) 85883b0a91SJoerg Roedel return 0; 86883b0a91SJoerg Roedel return pdpte; 87883b0a91SJoerg Roedel } 88883b0a91SJoerg Roedel 89883b0a91SJoerg Roedel static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu) 90883b0a91SJoerg Roedel { 91883b0a91SJoerg Roedel struct vcpu_svm *svm = to_svm(vcpu); 92883b0a91SJoerg Roedel 93e670bf68SPaolo Bonzini return svm->nested.ctl.nested_cr3; 94883b0a91SJoerg Roedel } 95883b0a91SJoerg Roedel 96883b0a91SJoerg Roedel static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu) 97883b0a91SJoerg Roedel { 98929d1cfaSPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 99929d1cfaSPaolo Bonzini 100883b0a91SJoerg Roedel WARN_ON(mmu_is_nested(vcpu)); 101883b0a91SJoerg Roedel 102883b0a91SJoerg Roedel vcpu->arch.mmu = &vcpu->arch.guest_mmu; 10331e96bc6SSean Christopherson 10431e96bc6SSean Christopherson /* 10531e96bc6SSean Christopherson * The NPT format depends on L1's CR4 and EFER, which is in vmcb01. Note, 10631e96bc6SSean Christopherson * when called via KVM_SET_NESTED_STATE, that state may _not_ match current 10731e96bc6SSean Christopherson * vCPU state. CR0.WP is explicitly ignored, while CR0.PG is required. 10831e96bc6SSean Christopherson */ 1094995a368SCathy Avery kvm_init_shadow_npt_mmu(vcpu, X86_CR0_PG, svm->vmcb01.ptr->save.cr4, 1104995a368SCathy Avery svm->vmcb01.ptr->save.efer, 1110f04a2acSVitaly Kuznetsov svm->nested.ctl.nested_cr3); 112883b0a91SJoerg Roedel vcpu->arch.mmu->get_guest_pgd = nested_svm_get_tdp_cr3; 113883b0a91SJoerg Roedel vcpu->arch.mmu->get_pdptr = nested_svm_get_tdp_pdptr; 114883b0a91SJoerg Roedel vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit; 115883b0a91SJoerg Roedel vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; 116883b0a91SJoerg Roedel } 117883b0a91SJoerg Roedel 118883b0a91SJoerg Roedel static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu) 119883b0a91SJoerg Roedel { 120883b0a91SJoerg Roedel vcpu->arch.mmu = &vcpu->arch.root_mmu; 121883b0a91SJoerg Roedel vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; 122883b0a91SJoerg Roedel } 123883b0a91SJoerg Roedel 124883b0a91SJoerg Roedel void recalc_intercepts(struct vcpu_svm *svm) 125883b0a91SJoerg Roedel { 1268fc78909SEmanuele Giuseppe Esposito struct vmcb_control_area *c, *h; 1278fc78909SEmanuele Giuseppe Esposito struct vmcb_ctrl_area_cached *g; 128c45ad722SBabu Moger unsigned int i; 129883b0a91SJoerg Roedel 13006e7852cSJoerg Roedel vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 131883b0a91SJoerg Roedel 132883b0a91SJoerg Roedel if (!is_guest_mode(&svm->vcpu)) 133883b0a91SJoerg Roedel return; 134883b0a91SJoerg Roedel 135883b0a91SJoerg Roedel c = &svm->vmcb->control; 1364995a368SCathy Avery h = &svm->vmcb01.ptr->control; 137e670bf68SPaolo Bonzini g = &svm->nested.ctl; 138883b0a91SJoerg Roedel 139c45ad722SBabu Moger for (i = 0; i < MAX_INTERCEPT; i++) 140c45ad722SBabu Moger c->intercepts[i] = h->intercepts[i]; 141c45ad722SBabu Moger 142e9fd761aSPaolo Bonzini if (g->int_ctl & V_INTR_MASKING_MASK) { 143883b0a91SJoerg Roedel /* We only want the cr8 intercept bits of L1 */ 14403bfeeb9SBabu Moger vmcb_clr_intercept(c, INTERCEPT_CR8_READ); 14503bfeeb9SBabu Moger vmcb_clr_intercept(c, INTERCEPT_CR8_WRITE); 146883b0a91SJoerg Roedel 147883b0a91SJoerg Roedel /* 148883b0a91SJoerg Roedel * Once running L2 with HF_VINTR_MASK, EFLAGS.IF does not 149883b0a91SJoerg Roedel * affect any interrupt we may want to inject; therefore, 150883b0a91SJoerg Roedel * interrupt window vmexits are irrelevant to L0. 151883b0a91SJoerg Roedel */ 152c62e2e94SBabu Moger vmcb_clr_intercept(c, INTERCEPT_VINTR); 153883b0a91SJoerg Roedel } 154883b0a91SJoerg Roedel 155883b0a91SJoerg Roedel /* We don't want to see VMMCALLs from a nested guest */ 156c62e2e94SBabu Moger vmcb_clr_intercept(c, INTERCEPT_VMMCALL); 157883b0a91SJoerg Roedel 158c45ad722SBabu Moger for (i = 0; i < MAX_INTERCEPT; i++) 159c45ad722SBabu Moger c->intercepts[i] |= g->intercepts[i]; 1604b639a9fSMaxim Levitsky 1614b639a9fSMaxim Levitsky /* If SMI is not intercepted, ignore guest SMI intercept as well */ 1624b639a9fSMaxim Levitsky if (!intercept_smi) 1634b639a9fSMaxim Levitsky vmcb_clr_intercept(c, INTERCEPT_SMI); 164c7dfa400SMaxim Levitsky 165c7dfa400SMaxim Levitsky vmcb_set_intercept(c, INTERCEPT_VMLOAD); 166c7dfa400SMaxim Levitsky vmcb_set_intercept(c, INTERCEPT_VMSAVE); 167883b0a91SJoerg Roedel } 168883b0a91SJoerg Roedel 16966c03a92SVitaly Kuznetsov /* 17066c03a92SVitaly Kuznetsov * Merge L0's (KVM) and L1's (Nested VMCB) MSR permission bitmaps. The function 17166c03a92SVitaly Kuznetsov * is optimized in that it only merges the parts where KVM MSR permission bitmap 17266c03a92SVitaly Kuznetsov * may contain zero bits. 17366c03a92SVitaly Kuznetsov */ 174883b0a91SJoerg Roedel static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm) 175883b0a91SJoerg Roedel { 17666c03a92SVitaly Kuznetsov struct hv_enlightenments *hve = 17766c03a92SVitaly Kuznetsov (struct hv_enlightenments *)svm->nested.ctl.reserved_sw; 178883b0a91SJoerg Roedel int i; 179883b0a91SJoerg Roedel 18066c03a92SVitaly Kuznetsov /* 18166c03a92SVitaly Kuznetsov * MSR bitmap update can be skipped when: 18266c03a92SVitaly Kuznetsov * - MSR bitmap for L1 hasn't changed. 18366c03a92SVitaly Kuznetsov * - Nested hypervisor (L1) is attempting to launch the same L2 as 18466c03a92SVitaly Kuznetsov * before. 18566c03a92SVitaly Kuznetsov * - Nested hypervisor (L1) is using Hyper-V emulation interface and 18666c03a92SVitaly Kuznetsov * tells KVM (L0) there were no changes in MSR bitmap for L2. 18766c03a92SVitaly Kuznetsov */ 18866c03a92SVitaly Kuznetsov if (!svm->nested.force_msr_bitmap_recalc && 18966c03a92SVitaly Kuznetsov kvm_hv_hypercall_enabled(&svm->vcpu) && 19066c03a92SVitaly Kuznetsov hve->hv_enlightenments_control.msr_bitmap && 19166c03a92SVitaly Kuznetsov (svm->nested.ctl.clean & BIT(VMCB_HV_NESTED_ENLIGHTENMENTS))) 19266c03a92SVitaly Kuznetsov goto set_msrpm_base_pa; 19366c03a92SVitaly Kuznetsov 1948fc78909SEmanuele Giuseppe Esposito if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT))) 195883b0a91SJoerg Roedel return true; 196883b0a91SJoerg Roedel 197883b0a91SJoerg Roedel for (i = 0; i < MSRPM_OFFSETS; i++) { 198883b0a91SJoerg Roedel u32 value, p; 199883b0a91SJoerg Roedel u64 offset; 200883b0a91SJoerg Roedel 201883b0a91SJoerg Roedel if (msrpm_offsets[i] == 0xffffffff) 202883b0a91SJoerg Roedel break; 203883b0a91SJoerg Roedel 204883b0a91SJoerg Roedel p = msrpm_offsets[i]; 205e670bf68SPaolo Bonzini offset = svm->nested.ctl.msrpm_base_pa + (p * 4); 206883b0a91SJoerg Roedel 207883b0a91SJoerg Roedel if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4)) 208883b0a91SJoerg Roedel return false; 209883b0a91SJoerg Roedel 210883b0a91SJoerg Roedel svm->nested.msrpm[p] = svm->msrpm[p] | value; 211883b0a91SJoerg Roedel } 212883b0a91SJoerg Roedel 21373c25546SVitaly Kuznetsov svm->nested.force_msr_bitmap_recalc = false; 21473c25546SVitaly Kuznetsov 21566c03a92SVitaly Kuznetsov set_msrpm_base_pa: 216883b0a91SJoerg Roedel svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm)); 217883b0a91SJoerg Roedel 218883b0a91SJoerg Roedel return true; 219883b0a91SJoerg Roedel } 220883b0a91SJoerg Roedel 221ee695f22SKrish Sadhukhan /* 222ee695f22SKrish Sadhukhan * Bits 11:0 of bitmap address are ignored by hardware 223ee695f22SKrish Sadhukhan */ 224ee695f22SKrish Sadhukhan static bool nested_svm_check_bitmap_pa(struct kvm_vcpu *vcpu, u64 pa, u32 size) 225ee695f22SKrish Sadhukhan { 226ee695f22SKrish Sadhukhan u64 addr = PAGE_ALIGN(pa); 227ee695f22SKrish Sadhukhan 228ee695f22SKrish Sadhukhan return kvm_vcpu_is_legal_gpa(vcpu, addr) && 229ee695f22SKrish Sadhukhan kvm_vcpu_is_legal_gpa(vcpu, addr + size - 1); 230ee695f22SKrish Sadhukhan } 231ee695f22SKrish Sadhukhan 232174a921bSKrish Sadhukhan static bool nested_svm_check_tlb_ctl(struct kvm_vcpu *vcpu, u8 tlb_ctl) 233174a921bSKrish Sadhukhan { 234174a921bSKrish Sadhukhan /* Nested FLUSHBYASID is not supported yet. */ 235174a921bSKrish Sadhukhan switch(tlb_ctl) { 236174a921bSKrish Sadhukhan case TLB_CONTROL_DO_NOTHING: 237174a921bSKrish Sadhukhan case TLB_CONTROL_FLUSH_ALL_ASID: 238174a921bSKrish Sadhukhan return true; 239174a921bSKrish Sadhukhan default: 240174a921bSKrish Sadhukhan return false; 241174a921bSKrish Sadhukhan } 242174a921bSKrish Sadhukhan } 243174a921bSKrish Sadhukhan 244bd95926cSPaolo Bonzini static bool __nested_vmcb_check_controls(struct kvm_vcpu *vcpu, 2458fc78909SEmanuele Giuseppe Esposito struct vmcb_ctrl_area_cached *control) 246ca46d739SPaolo Bonzini { 2478fc78909SEmanuele Giuseppe Esposito if (CC(!vmcb12_is_intercept(control, INTERCEPT_VMRUN))) 248ca46d739SPaolo Bonzini return false; 249ca46d739SPaolo Bonzini 25011f0cbf0SSean Christopherson if (CC(control->asid == 0)) 251ca46d739SPaolo Bonzini return false; 252ca46d739SPaolo Bonzini 25311f0cbf0SSean Christopherson if (CC((control->nested_ctl & SVM_NESTED_CTL_NP_ENABLE) && !npt_enabled)) 254ca46d739SPaolo Bonzini return false; 255ca46d739SPaolo Bonzini 256ee695f22SKrish Sadhukhan if (CC(!nested_svm_check_bitmap_pa(vcpu, control->msrpm_base_pa, 257ee695f22SKrish Sadhukhan MSRPM_SIZE))) 258ee695f22SKrish Sadhukhan return false; 259ee695f22SKrish Sadhukhan if (CC(!nested_svm_check_bitmap_pa(vcpu, control->iopm_base_pa, 260ee695f22SKrish Sadhukhan IOPM_SIZE))) 261ee695f22SKrish Sadhukhan return false; 262ee695f22SKrish Sadhukhan 263174a921bSKrish Sadhukhan if (CC(!nested_svm_check_tlb_ctl(vcpu, control->tlb_ctl))) 264174a921bSKrish Sadhukhan return false; 265174a921bSKrish Sadhukhan 266ca46d739SPaolo Bonzini return true; 267ca46d739SPaolo Bonzini } 268ca46d739SPaolo Bonzini 2696906e06dSKrish Sadhukhan /* Common checks that apply to both L1 and L2 state. */ 270b7a3d8b6SEmanuele Giuseppe Esposito static bool __nested_vmcb_check_save(struct kvm_vcpu *vcpu, 271b7a3d8b6SEmanuele Giuseppe Esposito struct vmcb_save_area_cached *save) 2726906e06dSKrish Sadhukhan { 27311f0cbf0SSean Christopherson if (CC(!(save->efer & EFER_SVME))) 2746906e06dSKrish Sadhukhan return false; 2756906e06dSKrish Sadhukhan 27611f0cbf0SSean Christopherson if (CC((save->cr0 & X86_CR0_CD) == 0 && (save->cr0 & X86_CR0_NW)) || 27711f0cbf0SSean Christopherson CC(save->cr0 & ~0xffffffffULL)) 2786906e06dSKrish Sadhukhan return false; 2796906e06dSKrish Sadhukhan 28011f0cbf0SSean Christopherson if (CC(!kvm_dr6_valid(save->dr6)) || CC(!kvm_dr7_valid(save->dr7))) 2816906e06dSKrish Sadhukhan return false; 2826906e06dSKrish Sadhukhan 283907afa48SEmanuele Giuseppe Esposito /* 284907afa48SEmanuele Giuseppe Esposito * These checks are also performed by KVM_SET_SREGS, 285907afa48SEmanuele Giuseppe Esposito * except that EFER.LMA is not checked by SVM against 286907afa48SEmanuele Giuseppe Esposito * CR0.PG && EFER.LME. 287907afa48SEmanuele Giuseppe Esposito */ 288907afa48SEmanuele Giuseppe Esposito if ((save->efer & EFER_LME) && (save->cr0 & X86_CR0_PG)) { 289907afa48SEmanuele Giuseppe Esposito if (CC(!(save->cr4 & X86_CR4_PAE)) || 290907afa48SEmanuele Giuseppe Esposito CC(!(save->cr0 & X86_CR0_PE)) || 291907afa48SEmanuele Giuseppe Esposito CC(kvm_vcpu_is_illegal_gpa(vcpu, save->cr3))) 292907afa48SEmanuele Giuseppe Esposito return false; 293907afa48SEmanuele Giuseppe Esposito } 294907afa48SEmanuele Giuseppe Esposito 295907afa48SEmanuele Giuseppe Esposito if (CC(!kvm_is_valid_cr4(vcpu, save->cr4))) 2966906e06dSKrish Sadhukhan return false; 2976906e06dSKrish Sadhukhan 29863129754SPaolo Bonzini if (CC(!kvm_valid_efer(vcpu, save->efer))) 2996906e06dSKrish Sadhukhan return false; 3006906e06dSKrish Sadhukhan 3016906e06dSKrish Sadhukhan return true; 3026906e06dSKrish Sadhukhan } 3036906e06dSKrish Sadhukhan 304b7a3d8b6SEmanuele Giuseppe Esposito static bool nested_vmcb_check_save(struct kvm_vcpu *vcpu) 305b7a3d8b6SEmanuele Giuseppe Esposito { 306b7a3d8b6SEmanuele Giuseppe Esposito struct vcpu_svm *svm = to_svm(vcpu); 307b7a3d8b6SEmanuele Giuseppe Esposito struct vmcb_save_area_cached *save = &svm->nested.save; 308b7a3d8b6SEmanuele Giuseppe Esposito 309b7a3d8b6SEmanuele Giuseppe Esposito return __nested_vmcb_check_save(vcpu, save); 310b7a3d8b6SEmanuele Giuseppe Esposito } 311b7a3d8b6SEmanuele Giuseppe Esposito 312bd95926cSPaolo Bonzini static bool nested_vmcb_check_controls(struct kvm_vcpu *vcpu) 313bd95926cSPaolo Bonzini { 314bd95926cSPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 3158fc78909SEmanuele Giuseppe Esposito struct vmcb_ctrl_area_cached *ctl = &svm->nested.ctl; 316bd95926cSPaolo Bonzini 317bd95926cSPaolo Bonzini return __nested_vmcb_check_controls(vcpu, ctl); 318bd95926cSPaolo Bonzini } 319bd95926cSPaolo Bonzini 3207907160dSEmanuele Giuseppe Esposito static 32166c03a92SVitaly Kuznetsov void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu, 32266c03a92SVitaly Kuznetsov struct vmcb_ctrl_area_cached *to, 3237907160dSEmanuele Giuseppe Esposito struct vmcb_control_area *from) 3247907160dSEmanuele Giuseppe Esposito { 3257907160dSEmanuele Giuseppe Esposito unsigned int i; 3267907160dSEmanuele Giuseppe Esposito 3277907160dSEmanuele Giuseppe Esposito for (i = 0; i < MAX_INTERCEPT; i++) 3287907160dSEmanuele Giuseppe Esposito to->intercepts[i] = from->intercepts[i]; 3297907160dSEmanuele Giuseppe Esposito 3307907160dSEmanuele Giuseppe Esposito to->iopm_base_pa = from->iopm_base_pa; 3317907160dSEmanuele Giuseppe Esposito to->msrpm_base_pa = from->msrpm_base_pa; 3327907160dSEmanuele Giuseppe Esposito to->tsc_offset = from->tsc_offset; 3337907160dSEmanuele Giuseppe Esposito to->tlb_ctl = from->tlb_ctl; 3347907160dSEmanuele Giuseppe Esposito to->int_ctl = from->int_ctl; 3357907160dSEmanuele Giuseppe Esposito to->int_vector = from->int_vector; 3367907160dSEmanuele Giuseppe Esposito to->int_state = from->int_state; 3377907160dSEmanuele Giuseppe Esposito to->exit_code = from->exit_code; 3387907160dSEmanuele Giuseppe Esposito to->exit_code_hi = from->exit_code_hi; 3397907160dSEmanuele Giuseppe Esposito to->exit_info_1 = from->exit_info_1; 3407907160dSEmanuele Giuseppe Esposito to->exit_info_2 = from->exit_info_2; 3417907160dSEmanuele Giuseppe Esposito to->exit_int_info = from->exit_int_info; 3427907160dSEmanuele Giuseppe Esposito to->exit_int_info_err = from->exit_int_info_err; 3437907160dSEmanuele Giuseppe Esposito to->nested_ctl = from->nested_ctl; 3447907160dSEmanuele Giuseppe Esposito to->event_inj = from->event_inj; 3457907160dSEmanuele Giuseppe Esposito to->event_inj_err = from->event_inj_err; 3467907160dSEmanuele Giuseppe Esposito to->nested_cr3 = from->nested_cr3; 3477907160dSEmanuele Giuseppe Esposito to->virt_ext = from->virt_ext; 3487907160dSEmanuele Giuseppe Esposito to->pause_filter_count = from->pause_filter_count; 3497907160dSEmanuele Giuseppe Esposito to->pause_filter_thresh = from->pause_filter_thresh; 3507907160dSEmanuele Giuseppe Esposito 3517907160dSEmanuele Giuseppe Esposito /* Copy asid here because nested_vmcb_check_controls will check it. */ 3527907160dSEmanuele Giuseppe Esposito to->asid = from->asid; 3537907160dSEmanuele Giuseppe Esposito to->msrpm_base_pa &= ~0x0fffULL; 3547907160dSEmanuele Giuseppe Esposito to->iopm_base_pa &= ~0x0fffULL; 35566c03a92SVitaly Kuznetsov 35666c03a92SVitaly Kuznetsov /* Hyper-V extensions (Enlightened VMCB) */ 35766c03a92SVitaly Kuznetsov if (kvm_hv_hypercall_enabled(vcpu)) { 35866c03a92SVitaly Kuznetsov to->clean = from->clean; 35966c03a92SVitaly Kuznetsov memcpy(to->reserved_sw, from->reserved_sw, 36066c03a92SVitaly Kuznetsov sizeof(struct hv_enlightenments)); 36166c03a92SVitaly Kuznetsov } 3627907160dSEmanuele Giuseppe Esposito } 3637907160dSEmanuele Giuseppe Esposito 3647907160dSEmanuele Giuseppe Esposito void nested_copy_vmcb_control_to_cache(struct vcpu_svm *svm, 3653e06f016SPaolo Bonzini struct vmcb_control_area *control) 3663e06f016SPaolo Bonzini { 36766c03a92SVitaly Kuznetsov __nested_copy_vmcb_control_to_cache(&svm->vcpu, &svm->nested.ctl, control); 3683e06f016SPaolo Bonzini } 3693e06f016SPaolo Bonzini 370f2740a8dSEmanuele Giuseppe Esposito static void __nested_copy_vmcb_save_to_cache(struct vmcb_save_area_cached *to, 371f2740a8dSEmanuele Giuseppe Esposito struct vmcb_save_area *from) 372f2740a8dSEmanuele Giuseppe Esposito { 373f2740a8dSEmanuele Giuseppe Esposito /* 374f2740a8dSEmanuele Giuseppe Esposito * Copy only fields that are validated, as we need them 375f2740a8dSEmanuele Giuseppe Esposito * to avoid TOC/TOU races. 376f2740a8dSEmanuele Giuseppe Esposito */ 377f2740a8dSEmanuele Giuseppe Esposito to->efer = from->efer; 378f2740a8dSEmanuele Giuseppe Esposito to->cr0 = from->cr0; 379f2740a8dSEmanuele Giuseppe Esposito to->cr3 = from->cr3; 380f2740a8dSEmanuele Giuseppe Esposito to->cr4 = from->cr4; 381f2740a8dSEmanuele Giuseppe Esposito 382f2740a8dSEmanuele Giuseppe Esposito to->dr6 = from->dr6; 383f2740a8dSEmanuele Giuseppe Esposito to->dr7 = from->dr7; 384f2740a8dSEmanuele Giuseppe Esposito } 385f2740a8dSEmanuele Giuseppe Esposito 386f2740a8dSEmanuele Giuseppe Esposito void nested_copy_vmcb_save_to_cache(struct vcpu_svm *svm, 387f2740a8dSEmanuele Giuseppe Esposito struct vmcb_save_area *save) 388f2740a8dSEmanuele Giuseppe Esposito { 389f2740a8dSEmanuele Giuseppe Esposito __nested_copy_vmcb_save_to_cache(&svm->nested.save, save); 390f2740a8dSEmanuele Giuseppe Esposito } 391f2740a8dSEmanuele Giuseppe Esposito 3922d8a42beSPaolo Bonzini /* 3932d8a42beSPaolo Bonzini * Synchronize fields that are written by the processor, so that 3949e8f0fbfSPaolo Bonzini * they can be copied back into the vmcb12. 3952d8a42beSPaolo Bonzini */ 3969e8f0fbfSPaolo Bonzini void nested_sync_control_from_vmcb02(struct vcpu_svm *svm) 3972d8a42beSPaolo Bonzini { 3982d8a42beSPaolo Bonzini u32 mask; 3992d8a42beSPaolo Bonzini svm->nested.ctl.event_inj = svm->vmcb->control.event_inj; 4002d8a42beSPaolo Bonzini svm->nested.ctl.event_inj_err = svm->vmcb->control.event_inj_err; 4012d8a42beSPaolo Bonzini 4022d8a42beSPaolo Bonzini /* Only a few fields of int_ctl are written by the processor. */ 4032d8a42beSPaolo Bonzini mask = V_IRQ_MASK | V_TPR_MASK; 4042d8a42beSPaolo Bonzini if (!(svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK) && 405a284ba56SJoerg Roedel svm_is_intercept(svm, INTERCEPT_VINTR)) { 4062d8a42beSPaolo Bonzini /* 4072d8a42beSPaolo Bonzini * In order to request an interrupt window, L0 is usurping 4082d8a42beSPaolo Bonzini * svm->vmcb->control.int_ctl and possibly setting V_IRQ 4092d8a42beSPaolo Bonzini * even if it was clear in L1's VMCB. Restoring it would be 4102d8a42beSPaolo Bonzini * wrong. However, in this case V_IRQ will remain true until 4112d8a42beSPaolo Bonzini * interrupt_window_interception calls svm_clear_vintr and 4122d8a42beSPaolo Bonzini * restores int_ctl. We can just leave it aside. 4132d8a42beSPaolo Bonzini */ 4142d8a42beSPaolo Bonzini mask &= ~V_IRQ_MASK; 4152d8a42beSPaolo Bonzini } 4162d8a42beSPaolo Bonzini svm->nested.ctl.int_ctl &= ~mask; 4172d8a42beSPaolo Bonzini svm->nested.ctl.int_ctl |= svm->vmcb->control.int_ctl & mask; 4182d8a42beSPaolo Bonzini } 4192d8a42beSPaolo Bonzini 42036e2e983SPaolo Bonzini /* 42136e2e983SPaolo Bonzini * Transfer any event that L0 or L1 wanted to inject into L2 to 42236e2e983SPaolo Bonzini * EXIT_INT_INFO. 42336e2e983SPaolo Bonzini */ 4249e8f0fbfSPaolo Bonzini static void nested_save_pending_event_to_vmcb12(struct vcpu_svm *svm, 4250dd16b5bSMaxim Levitsky struct vmcb *vmcb12) 42636e2e983SPaolo Bonzini { 42736e2e983SPaolo Bonzini struct kvm_vcpu *vcpu = &svm->vcpu; 42836e2e983SPaolo Bonzini u32 exit_int_info = 0; 42936e2e983SPaolo Bonzini unsigned int nr; 43036e2e983SPaolo Bonzini 43136e2e983SPaolo Bonzini if (vcpu->arch.exception.injected) { 43236e2e983SPaolo Bonzini nr = vcpu->arch.exception.nr; 43336e2e983SPaolo Bonzini exit_int_info = nr | SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_EXEPT; 43436e2e983SPaolo Bonzini 43536e2e983SPaolo Bonzini if (vcpu->arch.exception.has_error_code) { 43636e2e983SPaolo Bonzini exit_int_info |= SVM_EVTINJ_VALID_ERR; 4370dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info_err = 43836e2e983SPaolo Bonzini vcpu->arch.exception.error_code; 43936e2e983SPaolo Bonzini } 44036e2e983SPaolo Bonzini 44136e2e983SPaolo Bonzini } else if (vcpu->arch.nmi_injected) { 44236e2e983SPaolo Bonzini exit_int_info = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI; 44336e2e983SPaolo Bonzini 44436e2e983SPaolo Bonzini } else if (vcpu->arch.interrupt.injected) { 44536e2e983SPaolo Bonzini nr = vcpu->arch.interrupt.nr; 44636e2e983SPaolo Bonzini exit_int_info = nr | SVM_EVTINJ_VALID; 44736e2e983SPaolo Bonzini 44836e2e983SPaolo Bonzini if (vcpu->arch.interrupt.soft) 44936e2e983SPaolo Bonzini exit_int_info |= SVM_EVTINJ_TYPE_SOFT; 45036e2e983SPaolo Bonzini else 45136e2e983SPaolo Bonzini exit_int_info |= SVM_EVTINJ_TYPE_INTR; 45236e2e983SPaolo Bonzini } 45336e2e983SPaolo Bonzini 4540dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info = exit_int_info; 45536e2e983SPaolo Bonzini } 45636e2e983SPaolo Bonzini 45762156f6cSVitaly Kuznetsov static inline bool nested_npt_enabled(struct vcpu_svm *svm) 45862156f6cSVitaly Kuznetsov { 45962156f6cSVitaly Kuznetsov return svm->nested.ctl.nested_ctl & SVM_NESTED_CTL_NP_ENABLE; 46062156f6cSVitaly Kuznetsov } 46162156f6cSVitaly Kuznetsov 462d2e56019SSean Christopherson static void nested_svm_transition_tlb_flush(struct kvm_vcpu *vcpu) 463d2e56019SSean Christopherson { 464d2e56019SSean Christopherson /* 465d2e56019SSean Christopherson * TODO: optimize unconditional TLB flush/MMU sync. A partial list of 466d2e56019SSean Christopherson * things to fix before this can be conditional: 467d2e56019SSean Christopherson * 468d2e56019SSean Christopherson * - Flush TLBs for both L1 and L2 remote TLB flush 469d2e56019SSean Christopherson * - Honor L1's request to flush an ASID on nested VMRUN 470d2e56019SSean Christopherson * - Sync nested NPT MMU on VMRUN that flushes L2's ASID[*] 471d2e56019SSean Christopherson * - Don't crush a pending TLB flush in vmcb02 on nested VMRUN 472d2e56019SSean Christopherson * - Flush L1's ASID on KVM_REQ_TLB_FLUSH_GUEST 473d2e56019SSean Christopherson * 474d2e56019SSean Christopherson * [*] Unlike nested EPT, SVM's ASID management can invalidate nested 475d2e56019SSean Christopherson * NPT guest-physical mappings on VMRUN. 476d2e56019SSean Christopherson */ 477d2e56019SSean Christopherson kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); 478d2e56019SSean Christopherson kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 479d2e56019SSean Christopherson } 480d2e56019SSean Christopherson 48162156f6cSVitaly Kuznetsov /* 482d82aaef9SVitaly Kuznetsov * Load guest's/host's cr3 on nested vmentry or vmexit. @nested_npt is true 483d82aaef9SVitaly Kuznetsov * if we are emulating VM-Entry into a guest with NPT enabled. 48462156f6cSVitaly Kuznetsov */ 48562156f6cSVitaly Kuznetsov static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, 486b222b0b8SMaxim Levitsky bool nested_npt, bool reload_pdptrs) 48762156f6cSVitaly Kuznetsov { 48811f0cbf0SSean Christopherson if (CC(kvm_vcpu_is_illegal_gpa(vcpu, cr3))) 489a506fdd2SVitaly Kuznetsov return -EINVAL; 490a506fdd2SVitaly Kuznetsov 491b222b0b8SMaxim Levitsky if (reload_pdptrs && !nested_npt && is_pae_paging(vcpu) && 4922df4a5ebSLai Jiangshan CC(!load_pdptrs(vcpu, cr3))) 493a506fdd2SVitaly Kuznetsov return -EINVAL; 494a506fdd2SVitaly Kuznetsov 495a506fdd2SVitaly Kuznetsov vcpu->arch.cr3 = cr3; 496a506fdd2SVitaly Kuznetsov 497616007c8SSean Christopherson /* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */ 498c9060662SSean Christopherson kvm_init_mmu(vcpu); 499a506fdd2SVitaly Kuznetsov 500*3cffc89dSPaolo Bonzini if (!nested_npt) 501*3cffc89dSPaolo Bonzini kvm_mmu_new_pgd(vcpu, cr3); 502*3cffc89dSPaolo Bonzini 503a506fdd2SVitaly Kuznetsov return 0; 50462156f6cSVitaly Kuznetsov } 50562156f6cSVitaly Kuznetsov 5064995a368SCathy Avery void nested_vmcb02_compute_g_pat(struct vcpu_svm *svm) 5074995a368SCathy Avery { 5084995a368SCathy Avery if (!svm->nested.vmcb02.ptr) 5094995a368SCathy Avery return; 5104995a368SCathy Avery 5114995a368SCathy Avery /* FIXME: merge g_pat from vmcb01 and vmcb12. */ 5124995a368SCathy Avery svm->nested.vmcb02.ptr->save.g_pat = svm->vmcb01.ptr->save.g_pat; 5134995a368SCathy Avery } 5144995a368SCathy Avery 5159e8f0fbfSPaolo Bonzini static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12) 516883b0a91SJoerg Roedel { 5178173396eSCathy Avery bool new_vmcb12 = false; 5188173396eSCathy Avery 5194995a368SCathy Avery nested_vmcb02_compute_g_pat(svm); 5204995a368SCathy Avery 521883b0a91SJoerg Roedel /* Load the nested guest state */ 5228173396eSCathy Avery if (svm->nested.vmcb12_gpa != svm->nested.last_vmcb12_gpa) { 5238173396eSCathy Avery new_vmcb12 = true; 5248173396eSCathy Avery svm->nested.last_vmcb12_gpa = svm->nested.vmcb12_gpa; 52573c25546SVitaly Kuznetsov svm->nested.force_msr_bitmap_recalc = true; 5268173396eSCathy Avery } 5278173396eSCathy Avery 5288173396eSCathy Avery if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_SEG))) { 5290dd16b5bSMaxim Levitsky svm->vmcb->save.es = vmcb12->save.es; 5300dd16b5bSMaxim Levitsky svm->vmcb->save.cs = vmcb12->save.cs; 5310dd16b5bSMaxim Levitsky svm->vmcb->save.ss = vmcb12->save.ss; 5320dd16b5bSMaxim Levitsky svm->vmcb->save.ds = vmcb12->save.ds; 5334bb170a5SPaolo Bonzini svm->vmcb->save.cpl = vmcb12->save.cpl; 5344bb170a5SPaolo Bonzini vmcb_mark_dirty(svm->vmcb, VMCB_SEG); 5358173396eSCathy Avery } 5364bb170a5SPaolo Bonzini 5378173396eSCathy Avery if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_DT))) { 5380dd16b5bSMaxim Levitsky svm->vmcb->save.gdtr = vmcb12->save.gdtr; 5390dd16b5bSMaxim Levitsky svm->vmcb->save.idtr = vmcb12->save.idtr; 5404bb170a5SPaolo Bonzini vmcb_mark_dirty(svm->vmcb, VMCB_DT); 5418173396eSCathy Avery } 5424bb170a5SPaolo Bonzini 5438cce12b3SPaolo Bonzini kvm_set_rflags(&svm->vcpu, vmcb12->save.rflags | X86_EFLAGS_FIXED); 5443c346c0cSPaolo Bonzini 545355d0473SEmanuele Giuseppe Esposito svm_set_efer(&svm->vcpu, svm->nested.save.efer); 5463c346c0cSPaolo Bonzini 547355d0473SEmanuele Giuseppe Esposito svm_set_cr0(&svm->vcpu, svm->nested.save.cr0); 548355d0473SEmanuele Giuseppe Esposito svm_set_cr4(&svm->vcpu, svm->nested.save.cr4); 5494bb170a5SPaolo Bonzini 5504bb170a5SPaolo Bonzini svm->vcpu.arch.cr2 = vmcb12->save.cr2; 5518173396eSCathy Avery 5520dd16b5bSMaxim Levitsky kvm_rax_write(&svm->vcpu, vmcb12->save.rax); 5530dd16b5bSMaxim Levitsky kvm_rsp_write(&svm->vcpu, vmcb12->save.rsp); 5540dd16b5bSMaxim Levitsky kvm_rip_write(&svm->vcpu, vmcb12->save.rip); 555883b0a91SJoerg Roedel 556883b0a91SJoerg Roedel /* In case we don't even reach vcpu_run, the fields are not updated */ 5570dd16b5bSMaxim Levitsky svm->vmcb->save.rax = vmcb12->save.rax; 5580dd16b5bSMaxim Levitsky svm->vmcb->save.rsp = vmcb12->save.rsp; 5590dd16b5bSMaxim Levitsky svm->vmcb->save.rip = vmcb12->save.rip; 5604bb170a5SPaolo Bonzini 5618173396eSCathy Avery /* These bits will be set properly on the first execution when new_vmc12 is true */ 5628173396eSCathy Avery if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_DR))) { 563355d0473SEmanuele Giuseppe Esposito svm->vmcb->save.dr7 = svm->nested.save.dr7 | DR7_FIXED_1; 564355d0473SEmanuele Giuseppe Esposito svm->vcpu.arch.dr6 = svm->nested.save.dr6 | DR6_ACTIVE_LOW; 5654bb170a5SPaolo Bonzini vmcb_mark_dirty(svm->vmcb, VMCB_DR); 566f241d711SPaolo Bonzini } 5678173396eSCathy Avery } 568883b0a91SJoerg Roedel 5699e8f0fbfSPaolo Bonzini static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) 570f241d711SPaolo Bonzini { 5710f923e07SMaxim Levitsky const u32 int_ctl_vmcb01_bits = 5720f923e07SMaxim Levitsky V_INTR_MASKING_MASK | V_GIF_MASK | V_GIF_ENABLE_MASK; 5730f923e07SMaxim Levitsky 5740f923e07SMaxim Levitsky const u32 int_ctl_vmcb12_bits = V_TPR_MASK | V_IRQ_INJECTION_BITS_MASK; 5750f923e07SMaxim Levitsky 576d2e56019SSean Christopherson struct kvm_vcpu *vcpu = &svm->vcpu; 57762156f6cSVitaly Kuznetsov 5787c3ecfcdSPaolo Bonzini /* 5797c3ecfcdSPaolo Bonzini * Filled at exit: exit_code, exit_code_hi, exit_info_1, exit_info_2, 5807c3ecfcdSPaolo Bonzini * exit_int_info, exit_int_info_err, next_rip, insn_len, insn_bytes. 5817c3ecfcdSPaolo Bonzini */ 5824995a368SCathy Avery 5837c3ecfcdSPaolo Bonzini /* 5847c3ecfcdSPaolo Bonzini * Also covers avic_vapic_bar, avic_backing_page, avic_logical_id, 5857c3ecfcdSPaolo Bonzini * avic_physical_id. 5867c3ecfcdSPaolo Bonzini */ 587feea0136SMaxim Levitsky WARN_ON(kvm_apicv_activated(svm->vcpu.kvm)); 5887c3ecfcdSPaolo Bonzini 5897c3ecfcdSPaolo Bonzini /* Copied from vmcb01. msrpm_base can be overwritten later. */ 5907c3ecfcdSPaolo Bonzini svm->vmcb->control.nested_ctl = svm->vmcb01.ptr->control.nested_ctl; 5917c3ecfcdSPaolo Bonzini svm->vmcb->control.iopm_base_pa = svm->vmcb01.ptr->control.iopm_base_pa; 5927c3ecfcdSPaolo Bonzini svm->vmcb->control.msrpm_base_pa = svm->vmcb01.ptr->control.msrpm_base_pa; 5937c3ecfcdSPaolo Bonzini 5947c3ecfcdSPaolo Bonzini /* Done at vmrun: asid. */ 5957c3ecfcdSPaolo Bonzini 5967c3ecfcdSPaolo Bonzini /* Also overwritten later if necessary. */ 5977c3ecfcdSPaolo Bonzini svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; 5987c3ecfcdSPaolo Bonzini 5997c3ecfcdSPaolo Bonzini /* nested_cr3. */ 60062156f6cSVitaly Kuznetsov if (nested_npt_enabled(svm)) 601d2e56019SSean Christopherson nested_svm_init_mmu_context(vcpu); 60269cb8774SPaolo Bonzini 6035228eb96SMaxim Levitsky vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset( 6045228eb96SMaxim Levitsky vcpu->arch.l1_tsc_offset, 6055228eb96SMaxim Levitsky svm->nested.ctl.tsc_offset, 6065228eb96SMaxim Levitsky svm->tsc_ratio_msr); 6075228eb96SMaxim Levitsky 6085228eb96SMaxim Levitsky svm->vmcb->control.tsc_offset = vcpu->arch.tsc_offset; 6095228eb96SMaxim Levitsky 6105228eb96SMaxim Levitsky if (svm->tsc_ratio_msr != kvm_default_tsc_scaling_ratio) { 6115228eb96SMaxim Levitsky WARN_ON(!svm->tsc_scaling_enabled); 6125228eb96SMaxim Levitsky nested_svm_update_tsc_ratio_msr(vcpu); 6135228eb96SMaxim Levitsky } 614883b0a91SJoerg Roedel 61591b7130cSPaolo Bonzini svm->vmcb->control.int_ctl = 6160f923e07SMaxim Levitsky (svm->nested.ctl.int_ctl & int_ctl_vmcb12_bits) | 6170f923e07SMaxim Levitsky (svm->vmcb01.ptr->control.int_ctl & int_ctl_vmcb01_bits); 61891b7130cSPaolo Bonzini 619e670bf68SPaolo Bonzini svm->vmcb->control.int_vector = svm->nested.ctl.int_vector; 620e670bf68SPaolo Bonzini svm->vmcb->control.int_state = svm->nested.ctl.int_state; 621e670bf68SPaolo Bonzini svm->vmcb->control.event_inj = svm->nested.ctl.event_inj; 622e670bf68SPaolo Bonzini svm->vmcb->control.event_inj_err = svm->nested.ctl.event_inj_err; 623883b0a91SJoerg Roedel 624d2e56019SSean Christopherson nested_svm_transition_tlb_flush(vcpu); 625d2e56019SSean Christopherson 626883b0a91SJoerg Roedel /* Enter Guest-Mode */ 627d2e56019SSean Christopherson enter_guest_mode(vcpu); 628883b0a91SJoerg Roedel 629883b0a91SJoerg Roedel /* 630883b0a91SJoerg Roedel * Merge guest and host intercepts - must be called with vcpu in 6314bb170a5SPaolo Bonzini * guest-mode to take effect. 632883b0a91SJoerg Roedel */ 633883b0a91SJoerg Roedel recalc_intercepts(svm); 634f241d711SPaolo Bonzini } 635f241d711SPaolo Bonzini 636d00b99c5SBabu Moger static void nested_svm_copy_common_state(struct vmcb *from_vmcb, struct vmcb *to_vmcb) 637d00b99c5SBabu Moger { 638d00b99c5SBabu Moger /* 639d00b99c5SBabu Moger * Some VMCB state is shared between L1 and L2 and thus has to be 640d00b99c5SBabu Moger * moved at the time of nested vmrun and vmexit. 641d00b99c5SBabu Moger * 642d00b99c5SBabu Moger * VMLOAD/VMSAVE state would also belong in this category, but KVM 643d00b99c5SBabu Moger * always performs VMLOAD and VMSAVE from the VMCB01. 644d00b99c5SBabu Moger */ 645d00b99c5SBabu Moger to_vmcb->save.spec_ctrl = from_vmcb->save.spec_ctrl; 646d00b99c5SBabu Moger } 647d00b99c5SBabu Moger 64863129754SPaolo Bonzini int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb12_gpa, 649e85d3e7bSMaxim Levitsky struct vmcb *vmcb12, bool from_vmrun) 650f241d711SPaolo Bonzini { 65163129754SPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 652a506fdd2SVitaly Kuznetsov int ret; 653a506fdd2SVitaly Kuznetsov 654954f419bSMaxim Levitsky trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb12_gpa, 655954f419bSMaxim Levitsky vmcb12->save.rip, 656954f419bSMaxim Levitsky vmcb12->control.int_ctl, 657954f419bSMaxim Levitsky vmcb12->control.event_inj, 658954f419bSMaxim Levitsky vmcb12->control.nested_ctl); 659954f419bSMaxim Levitsky 660954f419bSMaxim Levitsky trace_kvm_nested_intercepts(vmcb12->control.intercepts[INTERCEPT_CR] & 0xffff, 661954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_CR] >> 16, 662954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_EXCEPTION], 663954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_WORD3], 664954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_WORD4], 665954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_WORD5]); 666954f419bSMaxim Levitsky 667954f419bSMaxim Levitsky 6680dd16b5bSMaxim Levitsky svm->nested.vmcb12_gpa = vmcb12_gpa; 6694995a368SCathy Avery 6704995a368SCathy Avery WARN_ON(svm->vmcb == svm->nested.vmcb02.ptr); 6714995a368SCathy Avery 672d00b99c5SBabu Moger nested_svm_copy_common_state(svm->vmcb01.ptr, svm->nested.vmcb02.ptr); 6734995a368SCathy Avery 6744995a368SCathy Avery svm_switch_vmcb(svm, &svm->nested.vmcb02); 6759e8f0fbfSPaolo Bonzini nested_vmcb02_prepare_control(svm); 6769e8f0fbfSPaolo Bonzini nested_vmcb02_prepare_save(svm, vmcb12); 677f241d711SPaolo Bonzini 678355d0473SEmanuele Giuseppe Esposito ret = nested_svm_load_cr3(&svm->vcpu, svm->nested.save.cr3, 679e85d3e7bSMaxim Levitsky nested_npt_enabled(svm), from_vmrun); 680a506fdd2SVitaly Kuznetsov if (ret) 681a506fdd2SVitaly Kuznetsov return ret; 682a506fdd2SVitaly Kuznetsov 683a04aead1SPaolo Bonzini if (!npt_enabled) 68463129754SPaolo Bonzini vcpu->arch.mmu->inject_page_fault = svm_inject_page_fault_nested; 685a04aead1SPaolo Bonzini 686e85d3e7bSMaxim Levitsky if (!from_vmrun) 687e85d3e7bSMaxim Levitsky kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 688e85d3e7bSMaxim Levitsky 689ffdf7f9eSPaolo Bonzini svm_set_gif(svm, true); 69059cd9bc5SVitaly Kuznetsov 69159cd9bc5SVitaly Kuznetsov return 0; 692883b0a91SJoerg Roedel } 693883b0a91SJoerg Roedel 69463129754SPaolo Bonzini int nested_svm_vmrun(struct kvm_vcpu *vcpu) 695883b0a91SJoerg Roedel { 69663129754SPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 697883b0a91SJoerg Roedel int ret; 6980dd16b5bSMaxim Levitsky struct vmcb *vmcb12; 699883b0a91SJoerg Roedel struct kvm_host_map map; 7000dd16b5bSMaxim Levitsky u64 vmcb12_gpa; 701883b0a91SJoerg Roedel 702fb79f566SVitaly Kuznetsov if (!svm->nested.hsave_msr) { 703fb79f566SVitaly Kuznetsov kvm_inject_gp(vcpu, 0); 704fb79f566SVitaly Kuznetsov return 1; 705fb79f566SVitaly Kuznetsov } 706fb79f566SVitaly Kuznetsov 70763129754SPaolo Bonzini if (is_smm(vcpu)) { 70863129754SPaolo Bonzini kvm_queue_exception(vcpu, UD_VECTOR); 7097c67f546SPaolo Bonzini return 1; 7107c67f546SPaolo Bonzini } 711883b0a91SJoerg Roedel 7120dd16b5bSMaxim Levitsky vmcb12_gpa = svm->vmcb->save.rax; 71363129754SPaolo Bonzini ret = kvm_vcpu_map(vcpu, gpa_to_gfn(vmcb12_gpa), &map); 714883b0a91SJoerg Roedel if (ret == -EINVAL) { 71563129754SPaolo Bonzini kvm_inject_gp(vcpu, 0); 716883b0a91SJoerg Roedel return 1; 717883b0a91SJoerg Roedel } else if (ret) { 71863129754SPaolo Bonzini return kvm_skip_emulated_instruction(vcpu); 719883b0a91SJoerg Roedel } 720883b0a91SJoerg Roedel 72163129754SPaolo Bonzini ret = kvm_skip_emulated_instruction(vcpu); 722883b0a91SJoerg Roedel 7230dd16b5bSMaxim Levitsky vmcb12 = map.hva; 724883b0a91SJoerg Roedel 7252fcf4876SMaxim Levitsky if (WARN_ON_ONCE(!svm->nested.initialized)) 7262fcf4876SMaxim Levitsky return -EINVAL; 7272fcf4876SMaxim Levitsky 7287907160dSEmanuele Giuseppe Esposito nested_copy_vmcb_control_to_cache(svm, &vmcb12->control); 729f2740a8dSEmanuele Giuseppe Esposito nested_copy_vmcb_save_to_cache(svm, &vmcb12->save); 730cb9b6a1bSPaolo Bonzini 731b7a3d8b6SEmanuele Giuseppe Esposito if (!nested_vmcb_check_save(vcpu) || 732bd95926cSPaolo Bonzini !nested_vmcb_check_controls(vcpu)) { 7330dd16b5bSMaxim Levitsky vmcb12->control.exit_code = SVM_EXIT_ERR; 7340dd16b5bSMaxim Levitsky vmcb12->control.exit_code_hi = 0; 7350dd16b5bSMaxim Levitsky vmcb12->control.exit_info_1 = 0; 7360dd16b5bSMaxim Levitsky vmcb12->control.exit_info_2 = 0; 73769c9dfa2SPaolo Bonzini goto out; 738883b0a91SJoerg Roedel } 739883b0a91SJoerg Roedel 740883b0a91SJoerg Roedel /* 7414995a368SCathy Avery * Since vmcb01 is not in use, we can use it to store some of the L1 7424995a368SCathy Avery * state. 743883b0a91SJoerg Roedel */ 74463129754SPaolo Bonzini svm->vmcb01.ptr->save.efer = vcpu->arch.efer; 74563129754SPaolo Bonzini svm->vmcb01.ptr->save.cr0 = kvm_read_cr0(vcpu); 74663129754SPaolo Bonzini svm->vmcb01.ptr->save.cr4 = vcpu->arch.cr4; 74763129754SPaolo Bonzini svm->vmcb01.ptr->save.rflags = kvm_get_rflags(vcpu); 74863129754SPaolo Bonzini svm->vmcb01.ptr->save.rip = kvm_rip_read(vcpu); 749883b0a91SJoerg Roedel 7504995a368SCathy Avery if (!npt_enabled) 75163129754SPaolo Bonzini svm->vmcb01.ptr->save.cr3 = kvm_read_cr3(vcpu); 752883b0a91SJoerg Roedel 753f74f9414SPaolo Bonzini svm->nested.nested_run_pending = 1; 754883b0a91SJoerg Roedel 755e85d3e7bSMaxim Levitsky if (enter_svm_guest_mode(vcpu, vmcb12_gpa, vmcb12, true)) 75659cd9bc5SVitaly Kuznetsov goto out_exit_err; 75759cd9bc5SVitaly Kuznetsov 75859cd9bc5SVitaly Kuznetsov if (nested_svm_vmrun_msrpm(svm)) 75959cd9bc5SVitaly Kuznetsov goto out; 76059cd9bc5SVitaly Kuznetsov 76159cd9bc5SVitaly Kuznetsov out_exit_err: 762ebdb3dbaSVitaly Kuznetsov svm->nested.nested_run_pending = 0; 763ebdb3dbaSVitaly Kuznetsov 764883b0a91SJoerg Roedel svm->vmcb->control.exit_code = SVM_EXIT_ERR; 765883b0a91SJoerg Roedel svm->vmcb->control.exit_code_hi = 0; 766883b0a91SJoerg Roedel svm->vmcb->control.exit_info_1 = 0; 767883b0a91SJoerg Roedel svm->vmcb->control.exit_info_2 = 0; 768883b0a91SJoerg Roedel 769883b0a91SJoerg Roedel nested_svm_vmexit(svm); 770883b0a91SJoerg Roedel 77169c9dfa2SPaolo Bonzini out: 77263129754SPaolo Bonzini kvm_vcpu_unmap(vcpu, &map, true); 77369c9dfa2SPaolo Bonzini 774883b0a91SJoerg Roedel return ret; 775883b0a91SJoerg Roedel } 776883b0a91SJoerg Roedel 7770a758290SVitaly Kuznetsov /* Copy state save area fields which are handled by VMRUN */ 7782bb16beaSVitaly Kuznetsov void svm_copy_vmrun_state(struct vmcb_save_area *to_save, 7792bb16beaSVitaly Kuznetsov struct vmcb_save_area *from_save) 7800a758290SVitaly Kuznetsov { 7810a758290SVitaly Kuznetsov to_save->es = from_save->es; 7820a758290SVitaly Kuznetsov to_save->cs = from_save->cs; 7830a758290SVitaly Kuznetsov to_save->ss = from_save->ss; 7840a758290SVitaly Kuznetsov to_save->ds = from_save->ds; 7850a758290SVitaly Kuznetsov to_save->gdtr = from_save->gdtr; 7860a758290SVitaly Kuznetsov to_save->idtr = from_save->idtr; 7870a758290SVitaly Kuznetsov to_save->rflags = from_save->rflags | X86_EFLAGS_FIXED; 7880a758290SVitaly Kuznetsov to_save->efer = from_save->efer; 7890a758290SVitaly Kuznetsov to_save->cr0 = from_save->cr0; 7900a758290SVitaly Kuznetsov to_save->cr3 = from_save->cr3; 7910a758290SVitaly Kuznetsov to_save->cr4 = from_save->cr4; 7920a758290SVitaly Kuznetsov to_save->rax = from_save->rax; 7930a758290SVitaly Kuznetsov to_save->rsp = from_save->rsp; 7940a758290SVitaly Kuznetsov to_save->rip = from_save->rip; 7950a758290SVitaly Kuznetsov to_save->cpl = 0; 7960a758290SVitaly Kuznetsov } 7970a758290SVitaly Kuznetsov 7982bb16beaSVitaly Kuznetsov void svm_copy_vmloadsave_state(struct vmcb *to_vmcb, struct vmcb *from_vmcb) 799883b0a91SJoerg Roedel { 800883b0a91SJoerg Roedel to_vmcb->save.fs = from_vmcb->save.fs; 801883b0a91SJoerg Roedel to_vmcb->save.gs = from_vmcb->save.gs; 802883b0a91SJoerg Roedel to_vmcb->save.tr = from_vmcb->save.tr; 803883b0a91SJoerg Roedel to_vmcb->save.ldtr = from_vmcb->save.ldtr; 804883b0a91SJoerg Roedel to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base; 805883b0a91SJoerg Roedel to_vmcb->save.star = from_vmcb->save.star; 806883b0a91SJoerg Roedel to_vmcb->save.lstar = from_vmcb->save.lstar; 807883b0a91SJoerg Roedel to_vmcb->save.cstar = from_vmcb->save.cstar; 808883b0a91SJoerg Roedel to_vmcb->save.sfmask = from_vmcb->save.sfmask; 809883b0a91SJoerg Roedel to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs; 810883b0a91SJoerg Roedel to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp; 811883b0a91SJoerg Roedel to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip; 812883b0a91SJoerg Roedel } 813883b0a91SJoerg Roedel 814883b0a91SJoerg Roedel int nested_svm_vmexit(struct vcpu_svm *svm) 815883b0a91SJoerg Roedel { 81663129754SPaolo Bonzini struct kvm_vcpu *vcpu = &svm->vcpu; 8170dd16b5bSMaxim Levitsky struct vmcb *vmcb12; 818883b0a91SJoerg Roedel struct vmcb *vmcb = svm->vmcb; 819883b0a91SJoerg Roedel struct kvm_host_map map; 82063129754SPaolo Bonzini int rc; 821883b0a91SJoerg Roedel 822cb6a32c2SSean Christopherson /* Triple faults in L2 should never escape. */ 823cb6a32c2SSean Christopherson WARN_ON_ONCE(kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)); 824cb6a32c2SSean Christopherson 82563129754SPaolo Bonzini rc = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa), &map); 826883b0a91SJoerg Roedel if (rc) { 827883b0a91SJoerg Roedel if (rc == -EINVAL) 82863129754SPaolo Bonzini kvm_inject_gp(vcpu, 0); 829883b0a91SJoerg Roedel return 1; 830883b0a91SJoerg Roedel } 831883b0a91SJoerg Roedel 8320dd16b5bSMaxim Levitsky vmcb12 = map.hva; 833883b0a91SJoerg Roedel 834883b0a91SJoerg Roedel /* Exit Guest-Mode */ 83563129754SPaolo Bonzini leave_guest_mode(vcpu); 8360dd16b5bSMaxim Levitsky svm->nested.vmcb12_gpa = 0; 8372d8a42beSPaolo Bonzini WARN_ON_ONCE(svm->nested.nested_run_pending); 838883b0a91SJoerg Roedel 83963129754SPaolo Bonzini kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 840f2c7ef3bSMaxim Levitsky 84138c0b192SPaolo Bonzini /* in case we halted in L2 */ 84238c0b192SPaolo Bonzini svm->vcpu.arch.mp_state = KVM_MP_STATE_RUNNABLE; 84338c0b192SPaolo Bonzini 844883b0a91SJoerg Roedel /* Give the current vmcb to the guest */ 845883b0a91SJoerg Roedel 8460dd16b5bSMaxim Levitsky vmcb12->save.es = vmcb->save.es; 8470dd16b5bSMaxim Levitsky vmcb12->save.cs = vmcb->save.cs; 8480dd16b5bSMaxim Levitsky vmcb12->save.ss = vmcb->save.ss; 8490dd16b5bSMaxim Levitsky vmcb12->save.ds = vmcb->save.ds; 8500dd16b5bSMaxim Levitsky vmcb12->save.gdtr = vmcb->save.gdtr; 8510dd16b5bSMaxim Levitsky vmcb12->save.idtr = vmcb->save.idtr; 8520dd16b5bSMaxim Levitsky vmcb12->save.efer = svm->vcpu.arch.efer; 85363129754SPaolo Bonzini vmcb12->save.cr0 = kvm_read_cr0(vcpu); 85463129754SPaolo Bonzini vmcb12->save.cr3 = kvm_read_cr3(vcpu); 8550dd16b5bSMaxim Levitsky vmcb12->save.cr2 = vmcb->save.cr2; 8560dd16b5bSMaxim Levitsky vmcb12->save.cr4 = svm->vcpu.arch.cr4; 85763129754SPaolo Bonzini vmcb12->save.rflags = kvm_get_rflags(vcpu); 85863129754SPaolo Bonzini vmcb12->save.rip = kvm_rip_read(vcpu); 85963129754SPaolo Bonzini vmcb12->save.rsp = kvm_rsp_read(vcpu); 86063129754SPaolo Bonzini vmcb12->save.rax = kvm_rax_read(vcpu); 8610dd16b5bSMaxim Levitsky vmcb12->save.dr7 = vmcb->save.dr7; 8620dd16b5bSMaxim Levitsky vmcb12->save.dr6 = svm->vcpu.arch.dr6; 8630dd16b5bSMaxim Levitsky vmcb12->save.cpl = vmcb->save.cpl; 864883b0a91SJoerg Roedel 8650dd16b5bSMaxim Levitsky vmcb12->control.int_state = vmcb->control.int_state; 8660dd16b5bSMaxim Levitsky vmcb12->control.exit_code = vmcb->control.exit_code; 8670dd16b5bSMaxim Levitsky vmcb12->control.exit_code_hi = vmcb->control.exit_code_hi; 8680dd16b5bSMaxim Levitsky vmcb12->control.exit_info_1 = vmcb->control.exit_info_1; 8690dd16b5bSMaxim Levitsky vmcb12->control.exit_info_2 = vmcb->control.exit_info_2; 87036e2e983SPaolo Bonzini 8710dd16b5bSMaxim Levitsky if (vmcb12->control.exit_code != SVM_EXIT_ERR) 8729e8f0fbfSPaolo Bonzini nested_save_pending_event_to_vmcb12(svm, vmcb12); 873883b0a91SJoerg Roedel 874883b0a91SJoerg Roedel if (svm->nrips_enabled) 8750dd16b5bSMaxim Levitsky vmcb12->control.next_rip = vmcb->control.next_rip; 876883b0a91SJoerg Roedel 8770dd16b5bSMaxim Levitsky vmcb12->control.int_ctl = svm->nested.ctl.int_ctl; 8780dd16b5bSMaxim Levitsky vmcb12->control.tlb_ctl = svm->nested.ctl.tlb_ctl; 8790dd16b5bSMaxim Levitsky vmcb12->control.event_inj = svm->nested.ctl.event_inj; 8800dd16b5bSMaxim Levitsky vmcb12->control.event_inj_err = svm->nested.ctl.event_inj_err; 881883b0a91SJoerg Roedel 882d00b99c5SBabu Moger nested_svm_copy_common_state(svm->nested.vmcb02.ptr, svm->vmcb01.ptr); 883d00b99c5SBabu Moger 8844995a368SCathy Avery svm_switch_vmcb(svm, &svm->vmcb01); 8854995a368SCathy Avery 8864995a368SCathy Avery /* 8874995a368SCathy Avery * On vmexit the GIF is set to false and 8884995a368SCathy Avery * no event can be injected in L1. 8894995a368SCathy Avery */ 8909883764aSMaxim Levitsky svm_set_gif(svm, false); 8914995a368SCathy Avery svm->vmcb->control.exit_int_info = 0; 8929883764aSMaxim Levitsky 8937ca62d13SPaolo Bonzini svm->vcpu.arch.tsc_offset = svm->vcpu.arch.l1_tsc_offset; 8947ca62d13SPaolo Bonzini if (svm->vmcb->control.tsc_offset != svm->vcpu.arch.tsc_offset) { 8957ca62d13SPaolo Bonzini svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset; 8967ca62d13SPaolo Bonzini vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 8977ca62d13SPaolo Bonzini } 89818fc6c55SPaolo Bonzini 8995228eb96SMaxim Levitsky if (svm->tsc_ratio_msr != kvm_default_tsc_scaling_ratio) { 9005228eb96SMaxim Levitsky WARN_ON(!svm->tsc_scaling_enabled); 9015228eb96SMaxim Levitsky vcpu->arch.tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; 9025228eb96SMaxim Levitsky svm_write_tsc_multiplier(vcpu, vcpu->arch.tsc_scaling_ratio); 9035228eb96SMaxim Levitsky } 9045228eb96SMaxim Levitsky 905e670bf68SPaolo Bonzini svm->nested.ctl.nested_cr3 = 0; 906883b0a91SJoerg Roedel 9074995a368SCathy Avery /* 9084995a368SCathy Avery * Restore processor state that had been saved in vmcb01 9094995a368SCathy Avery */ 91063129754SPaolo Bonzini kvm_set_rflags(vcpu, svm->vmcb->save.rflags); 91163129754SPaolo Bonzini svm_set_efer(vcpu, svm->vmcb->save.efer); 91263129754SPaolo Bonzini svm_set_cr0(vcpu, svm->vmcb->save.cr0 | X86_CR0_PE); 91363129754SPaolo Bonzini svm_set_cr4(vcpu, svm->vmcb->save.cr4); 91463129754SPaolo Bonzini kvm_rax_write(vcpu, svm->vmcb->save.rax); 91563129754SPaolo Bonzini kvm_rsp_write(vcpu, svm->vmcb->save.rsp); 91663129754SPaolo Bonzini kvm_rip_write(vcpu, svm->vmcb->save.rip); 9174995a368SCathy Avery 9184995a368SCathy Avery svm->vcpu.arch.dr7 = DR7_FIXED_1; 9194995a368SCathy Avery kvm_update_dr7(&svm->vcpu); 920883b0a91SJoerg Roedel 9210dd16b5bSMaxim Levitsky trace_kvm_nested_vmexit_inject(vmcb12->control.exit_code, 9220dd16b5bSMaxim Levitsky vmcb12->control.exit_info_1, 9230dd16b5bSMaxim Levitsky vmcb12->control.exit_info_2, 9240dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info, 9250dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info_err, 92636e2e983SPaolo Bonzini KVM_ISA_SVM); 92736e2e983SPaolo Bonzini 92863129754SPaolo Bonzini kvm_vcpu_unmap(vcpu, &map, true); 929883b0a91SJoerg Roedel 930d2e56019SSean Christopherson nested_svm_transition_tlb_flush(vcpu); 931d2e56019SSean Christopherson 93263129754SPaolo Bonzini nested_svm_uninit_mmu_context(vcpu); 933bf7dea42SVitaly Kuznetsov 934b222b0b8SMaxim Levitsky rc = nested_svm_load_cr3(vcpu, svm->vmcb->save.cr3, false, true); 935d82aaef9SVitaly Kuznetsov if (rc) 936d82aaef9SVitaly Kuznetsov return 1; 937bf7dea42SVitaly Kuznetsov 938883b0a91SJoerg Roedel /* 939883b0a91SJoerg Roedel * Drop what we picked up for L2 via svm_complete_interrupts() so it 940883b0a91SJoerg Roedel * doesn't end up in L1. 941883b0a91SJoerg Roedel */ 942883b0a91SJoerg Roedel svm->vcpu.arch.nmi_injected = false; 94363129754SPaolo Bonzini kvm_clear_exception_queue(vcpu); 94463129754SPaolo Bonzini kvm_clear_interrupt_queue(vcpu); 945883b0a91SJoerg Roedel 9469a7de6ecSKrish Sadhukhan /* 9479a7de6ecSKrish Sadhukhan * If we are here following the completion of a VMRUN that 9489a7de6ecSKrish Sadhukhan * is being single-stepped, queue the pending #DB intercept 9499a7de6ecSKrish Sadhukhan * right now so that it an be accounted for before we execute 9509a7de6ecSKrish Sadhukhan * L1's next instruction. 9519a7de6ecSKrish Sadhukhan */ 9529a7de6ecSKrish Sadhukhan if (unlikely(svm->vmcb->save.rflags & X86_EFLAGS_TF)) 9539a7de6ecSKrish Sadhukhan kvm_queue_exception(&(svm->vcpu), DB_VECTOR); 9549a7de6ecSKrish Sadhukhan 955883b0a91SJoerg Roedel return 0; 956883b0a91SJoerg Roedel } 957883b0a91SJoerg Roedel 958cb6a32c2SSean Christopherson static void nested_svm_triple_fault(struct kvm_vcpu *vcpu) 959cb6a32c2SSean Christopherson { 9603a87c7e0SSean Christopherson nested_svm_simple_vmexit(to_svm(vcpu), SVM_EXIT_SHUTDOWN); 961cb6a32c2SSean Christopherson } 962cb6a32c2SSean Christopherson 9632fcf4876SMaxim Levitsky int svm_allocate_nested(struct vcpu_svm *svm) 9642fcf4876SMaxim Levitsky { 9654995a368SCathy Avery struct page *vmcb02_page; 9662fcf4876SMaxim Levitsky 9672fcf4876SMaxim Levitsky if (svm->nested.initialized) 9682fcf4876SMaxim Levitsky return 0; 9692fcf4876SMaxim Levitsky 9704995a368SCathy Avery vmcb02_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 9714995a368SCathy Avery if (!vmcb02_page) 9722fcf4876SMaxim Levitsky return -ENOMEM; 9734995a368SCathy Avery svm->nested.vmcb02.ptr = page_address(vmcb02_page); 9744995a368SCathy Avery svm->nested.vmcb02.pa = __sme_set(page_to_pfn(vmcb02_page) << PAGE_SHIFT); 9752fcf4876SMaxim Levitsky 9762fcf4876SMaxim Levitsky svm->nested.msrpm = svm_vcpu_alloc_msrpm(); 9772fcf4876SMaxim Levitsky if (!svm->nested.msrpm) 9784995a368SCathy Avery goto err_free_vmcb02; 9792fcf4876SMaxim Levitsky svm_vcpu_init_msrpm(&svm->vcpu, svm->nested.msrpm); 9802fcf4876SMaxim Levitsky 9812fcf4876SMaxim Levitsky svm->nested.initialized = true; 9822fcf4876SMaxim Levitsky return 0; 9832fcf4876SMaxim Levitsky 9844995a368SCathy Avery err_free_vmcb02: 9854995a368SCathy Avery __free_page(vmcb02_page); 9862fcf4876SMaxim Levitsky return -ENOMEM; 9872fcf4876SMaxim Levitsky } 9882fcf4876SMaxim Levitsky 9892fcf4876SMaxim Levitsky void svm_free_nested(struct vcpu_svm *svm) 9902fcf4876SMaxim Levitsky { 9912fcf4876SMaxim Levitsky if (!svm->nested.initialized) 9922fcf4876SMaxim Levitsky return; 9932fcf4876SMaxim Levitsky 9942fcf4876SMaxim Levitsky svm_vcpu_free_msrpm(svm->nested.msrpm); 9952fcf4876SMaxim Levitsky svm->nested.msrpm = NULL; 9962fcf4876SMaxim Levitsky 9974995a368SCathy Avery __free_page(virt_to_page(svm->nested.vmcb02.ptr)); 9984995a368SCathy Avery svm->nested.vmcb02.ptr = NULL; 9992fcf4876SMaxim Levitsky 1000c74ad08fSMaxim Levitsky /* 1001c74ad08fSMaxim Levitsky * When last_vmcb12_gpa matches the current vmcb12 gpa, 1002c74ad08fSMaxim Levitsky * some vmcb12 fields are not loaded if they are marked clean 1003c74ad08fSMaxim Levitsky * in the vmcb12, since in this case they are up to date already. 1004c74ad08fSMaxim Levitsky * 1005c74ad08fSMaxim Levitsky * When the vmcb02 is freed, this optimization becomes invalid. 1006c74ad08fSMaxim Levitsky */ 1007c74ad08fSMaxim Levitsky svm->nested.last_vmcb12_gpa = INVALID_GPA; 1008c74ad08fSMaxim Levitsky 10092fcf4876SMaxim Levitsky svm->nested.initialized = false; 10102fcf4876SMaxim Levitsky } 10112fcf4876SMaxim Levitsky 1012c513f484SPaolo Bonzini /* 1013c513f484SPaolo Bonzini * Forcibly leave nested mode in order to be able to reset the VCPU later on. 1014c513f484SPaolo Bonzini */ 1015f7e57078SSean Christopherson void svm_leave_nested(struct kvm_vcpu *vcpu) 1016c513f484SPaolo Bonzini { 1017f7e57078SSean Christopherson struct vcpu_svm *svm = to_svm(vcpu); 101863129754SPaolo Bonzini 101963129754SPaolo Bonzini if (is_guest_mode(vcpu)) { 1020c513f484SPaolo Bonzini svm->nested.nested_run_pending = 0; 1021c74ad08fSMaxim Levitsky svm->nested.vmcb12_gpa = INVALID_GPA; 1022c74ad08fSMaxim Levitsky 102363129754SPaolo Bonzini leave_guest_mode(vcpu); 10244995a368SCathy Avery 1025deee59baSMaxim Levitsky svm_switch_vmcb(svm, &svm->vmcb01); 10264995a368SCathy Avery 102763129754SPaolo Bonzini nested_svm_uninit_mmu_context(vcpu); 102856fe28deSMaxim Levitsky vmcb_mark_all_dirty(svm->vmcb); 1029c513f484SPaolo Bonzini } 1030a7d5c7ceSPaolo Bonzini 103163129754SPaolo Bonzini kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 1032c513f484SPaolo Bonzini } 1033c513f484SPaolo Bonzini 1034883b0a91SJoerg Roedel static int nested_svm_exit_handled_msr(struct vcpu_svm *svm) 1035883b0a91SJoerg Roedel { 1036883b0a91SJoerg Roedel u32 offset, msr, value; 1037883b0a91SJoerg Roedel int write, mask; 1038883b0a91SJoerg Roedel 10398fc78909SEmanuele Giuseppe Esposito if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT))) 1040883b0a91SJoerg Roedel return NESTED_EXIT_HOST; 1041883b0a91SJoerg Roedel 1042883b0a91SJoerg Roedel msr = svm->vcpu.arch.regs[VCPU_REGS_RCX]; 1043883b0a91SJoerg Roedel offset = svm_msrpm_offset(msr); 1044883b0a91SJoerg Roedel write = svm->vmcb->control.exit_info_1 & 1; 1045883b0a91SJoerg Roedel mask = 1 << ((2 * (msr & 0xf)) + write); 1046883b0a91SJoerg Roedel 1047883b0a91SJoerg Roedel if (offset == MSR_INVALID) 1048883b0a91SJoerg Roedel return NESTED_EXIT_DONE; 1049883b0a91SJoerg Roedel 1050883b0a91SJoerg Roedel /* Offset is in 32 bit units but need in 8 bit units */ 1051883b0a91SJoerg Roedel offset *= 4; 1052883b0a91SJoerg Roedel 1053e670bf68SPaolo Bonzini if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.ctl.msrpm_base_pa + offset, &value, 4)) 1054883b0a91SJoerg Roedel return NESTED_EXIT_DONE; 1055883b0a91SJoerg Roedel 1056883b0a91SJoerg Roedel return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; 1057883b0a91SJoerg Roedel } 1058883b0a91SJoerg Roedel 1059883b0a91SJoerg Roedel static int nested_svm_intercept_ioio(struct vcpu_svm *svm) 1060883b0a91SJoerg Roedel { 1061883b0a91SJoerg Roedel unsigned port, size, iopm_len; 1062883b0a91SJoerg Roedel u16 val, mask; 1063883b0a91SJoerg Roedel u8 start_bit; 1064883b0a91SJoerg Roedel u64 gpa; 1065883b0a91SJoerg Roedel 10668fc78909SEmanuele Giuseppe Esposito if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_IOIO_PROT))) 1067883b0a91SJoerg Roedel return NESTED_EXIT_HOST; 1068883b0a91SJoerg Roedel 1069883b0a91SJoerg Roedel port = svm->vmcb->control.exit_info_1 >> 16; 1070883b0a91SJoerg Roedel size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >> 1071883b0a91SJoerg Roedel SVM_IOIO_SIZE_SHIFT; 1072e670bf68SPaolo Bonzini gpa = svm->nested.ctl.iopm_base_pa + (port / 8); 1073883b0a91SJoerg Roedel start_bit = port % 8; 1074883b0a91SJoerg Roedel iopm_len = (start_bit + size > 8) ? 2 : 1; 1075883b0a91SJoerg Roedel mask = (0xf >> (4 - size)) << start_bit; 1076883b0a91SJoerg Roedel val = 0; 1077883b0a91SJoerg Roedel 1078883b0a91SJoerg Roedel if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len)) 1079883b0a91SJoerg Roedel return NESTED_EXIT_DONE; 1080883b0a91SJoerg Roedel 1081883b0a91SJoerg Roedel return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; 1082883b0a91SJoerg Roedel } 1083883b0a91SJoerg Roedel 1084883b0a91SJoerg Roedel static int nested_svm_intercept(struct vcpu_svm *svm) 1085883b0a91SJoerg Roedel { 1086883b0a91SJoerg Roedel u32 exit_code = svm->vmcb->control.exit_code; 1087883b0a91SJoerg Roedel int vmexit = NESTED_EXIT_HOST; 1088883b0a91SJoerg Roedel 1089883b0a91SJoerg Roedel switch (exit_code) { 1090883b0a91SJoerg Roedel case SVM_EXIT_MSR: 1091883b0a91SJoerg Roedel vmexit = nested_svm_exit_handled_msr(svm); 1092883b0a91SJoerg Roedel break; 1093883b0a91SJoerg Roedel case SVM_EXIT_IOIO: 1094883b0a91SJoerg Roedel vmexit = nested_svm_intercept_ioio(svm); 1095883b0a91SJoerg Roedel break; 1096883b0a91SJoerg Roedel case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: { 10978fc78909SEmanuele Giuseppe Esposito if (vmcb12_is_intercept(&svm->nested.ctl, exit_code)) 1098883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 1099883b0a91SJoerg Roedel break; 1100883b0a91SJoerg Roedel } 1101883b0a91SJoerg Roedel case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: { 11028fc78909SEmanuele Giuseppe Esposito if (vmcb12_is_intercept(&svm->nested.ctl, exit_code)) 1103883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 1104883b0a91SJoerg Roedel break; 1105883b0a91SJoerg Roedel } 1106883b0a91SJoerg Roedel case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: { 11077c86663bSPaolo Bonzini /* 11087c86663bSPaolo Bonzini * Host-intercepted exceptions have been checked already in 11097c86663bSPaolo Bonzini * nested_svm_exit_special. There is nothing to do here, 11107c86663bSPaolo Bonzini * the vmexit is injected by svm_check_nested_events. 11117c86663bSPaolo Bonzini */ 1112883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 1113883b0a91SJoerg Roedel break; 1114883b0a91SJoerg Roedel } 1115883b0a91SJoerg Roedel case SVM_EXIT_ERR: { 1116883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 1117883b0a91SJoerg Roedel break; 1118883b0a91SJoerg Roedel } 1119883b0a91SJoerg Roedel default: { 11208fc78909SEmanuele Giuseppe Esposito if (vmcb12_is_intercept(&svm->nested.ctl, exit_code)) 1121883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 1122883b0a91SJoerg Roedel } 1123883b0a91SJoerg Roedel } 1124883b0a91SJoerg Roedel 1125883b0a91SJoerg Roedel return vmexit; 1126883b0a91SJoerg Roedel } 1127883b0a91SJoerg Roedel 1128883b0a91SJoerg Roedel int nested_svm_exit_handled(struct vcpu_svm *svm) 1129883b0a91SJoerg Roedel { 1130883b0a91SJoerg Roedel int vmexit; 1131883b0a91SJoerg Roedel 1132883b0a91SJoerg Roedel vmexit = nested_svm_intercept(svm); 1133883b0a91SJoerg Roedel 1134883b0a91SJoerg Roedel if (vmexit == NESTED_EXIT_DONE) 1135883b0a91SJoerg Roedel nested_svm_vmexit(svm); 1136883b0a91SJoerg Roedel 1137883b0a91SJoerg Roedel return vmexit; 1138883b0a91SJoerg Roedel } 1139883b0a91SJoerg Roedel 114063129754SPaolo Bonzini int nested_svm_check_permissions(struct kvm_vcpu *vcpu) 1141883b0a91SJoerg Roedel { 114263129754SPaolo Bonzini if (!(vcpu->arch.efer & EFER_SVME) || !is_paging(vcpu)) { 114363129754SPaolo Bonzini kvm_queue_exception(vcpu, UD_VECTOR); 1144883b0a91SJoerg Roedel return 1; 1145883b0a91SJoerg Roedel } 1146883b0a91SJoerg Roedel 114763129754SPaolo Bonzini if (to_svm(vcpu)->vmcb->save.cpl) { 114863129754SPaolo Bonzini kvm_inject_gp(vcpu, 0); 1149883b0a91SJoerg Roedel return 1; 1150883b0a91SJoerg Roedel } 1151883b0a91SJoerg Roedel 1152883b0a91SJoerg Roedel return 0; 1153883b0a91SJoerg Roedel } 1154883b0a91SJoerg Roedel 11557c86663bSPaolo Bonzini static bool nested_exit_on_exception(struct vcpu_svm *svm) 1156883b0a91SJoerg Roedel { 11577c86663bSPaolo Bonzini unsigned int nr = svm->vcpu.arch.exception.nr; 1158883b0a91SJoerg Roedel 11599780d51dSBabu Moger return (svm->nested.ctl.intercepts[INTERCEPT_EXCEPTION] & BIT(nr)); 11607c86663bSPaolo Bonzini } 1161883b0a91SJoerg Roedel 11627c86663bSPaolo Bonzini static void nested_svm_inject_exception_vmexit(struct vcpu_svm *svm) 11637c86663bSPaolo Bonzini { 11647c86663bSPaolo Bonzini unsigned int nr = svm->vcpu.arch.exception.nr; 1165883b0a91SJoerg Roedel 1166883b0a91SJoerg Roedel svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr; 1167883b0a91SJoerg Roedel svm->vmcb->control.exit_code_hi = 0; 11687c86663bSPaolo Bonzini 11697c86663bSPaolo Bonzini if (svm->vcpu.arch.exception.has_error_code) 11707c86663bSPaolo Bonzini svm->vmcb->control.exit_info_1 = svm->vcpu.arch.exception.error_code; 1171883b0a91SJoerg Roedel 1172883b0a91SJoerg Roedel /* 1173883b0a91SJoerg Roedel * EXITINFO2 is undefined for all exception intercepts other 1174883b0a91SJoerg Roedel * than #PF. 1175883b0a91SJoerg Roedel */ 11767c86663bSPaolo Bonzini if (nr == PF_VECTOR) { 1177883b0a91SJoerg Roedel if (svm->vcpu.arch.exception.nested_apf) 1178883b0a91SJoerg Roedel svm->vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token; 1179883b0a91SJoerg Roedel else if (svm->vcpu.arch.exception.has_payload) 1180883b0a91SJoerg Roedel svm->vmcb->control.exit_info_2 = svm->vcpu.arch.exception.payload; 1181883b0a91SJoerg Roedel else 1182883b0a91SJoerg Roedel svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2; 11837c86663bSPaolo Bonzini } else if (nr == DB_VECTOR) { 11847c86663bSPaolo Bonzini /* See inject_pending_event. */ 11857c86663bSPaolo Bonzini kvm_deliver_exception_payload(&svm->vcpu); 11867c86663bSPaolo Bonzini if (svm->vcpu.arch.dr7 & DR7_GD) { 11877c86663bSPaolo Bonzini svm->vcpu.arch.dr7 &= ~DR7_GD; 11887c86663bSPaolo Bonzini kvm_update_dr7(&svm->vcpu); 11897c86663bSPaolo Bonzini } 11907c86663bSPaolo Bonzini } else 11917c86663bSPaolo Bonzini WARN_ON(svm->vcpu.arch.exception.has_payload); 1192883b0a91SJoerg Roedel 11937c86663bSPaolo Bonzini nested_svm_vmexit(svm); 1194883b0a91SJoerg Roedel } 1195883b0a91SJoerg Roedel 11965b672408SPaolo Bonzini static inline bool nested_exit_on_init(struct vcpu_svm *svm) 11975b672408SPaolo Bonzini { 11988fc78909SEmanuele Giuseppe Esposito return vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_INIT); 11995b672408SPaolo Bonzini } 12005b672408SPaolo Bonzini 120133b22172SPaolo Bonzini static int svm_check_nested_events(struct kvm_vcpu *vcpu) 1202883b0a91SJoerg Roedel { 1203883b0a91SJoerg Roedel struct vcpu_svm *svm = to_svm(vcpu); 1204883b0a91SJoerg Roedel bool block_nested_events = 1205bd279629SPaolo Bonzini kvm_event_needs_reinjection(vcpu) || svm->nested.nested_run_pending; 12065b672408SPaolo Bonzini struct kvm_lapic *apic = vcpu->arch.apic; 12075b672408SPaolo Bonzini 12085b672408SPaolo Bonzini if (lapic_in_kernel(vcpu) && 12095b672408SPaolo Bonzini test_bit(KVM_APIC_INIT, &apic->pending_events)) { 12105b672408SPaolo Bonzini if (block_nested_events) 12115b672408SPaolo Bonzini return -EBUSY; 12125b672408SPaolo Bonzini if (!nested_exit_on_init(svm)) 12135b672408SPaolo Bonzini return 0; 12143a87c7e0SSean Christopherson nested_svm_simple_vmexit(svm, SVM_EXIT_INIT); 12155b672408SPaolo Bonzini return 0; 12165b672408SPaolo Bonzini } 1217883b0a91SJoerg Roedel 12187c86663bSPaolo Bonzini if (vcpu->arch.exception.pending) { 12194020da3bSMaxim Levitsky /* 12204020da3bSMaxim Levitsky * Only a pending nested run can block a pending exception. 12214020da3bSMaxim Levitsky * Otherwise an injected NMI/interrupt should either be 12224020da3bSMaxim Levitsky * lost or delivered to the nested hypervisor in the EXITINTINFO 12234020da3bSMaxim Levitsky * vmcb field, while delivering the pending exception. 12244020da3bSMaxim Levitsky */ 12254020da3bSMaxim Levitsky if (svm->nested.nested_run_pending) 12267c86663bSPaolo Bonzini return -EBUSY; 12277c86663bSPaolo Bonzini if (!nested_exit_on_exception(svm)) 12287c86663bSPaolo Bonzini return 0; 12297c86663bSPaolo Bonzini nested_svm_inject_exception_vmexit(svm); 12307c86663bSPaolo Bonzini return 0; 12317c86663bSPaolo Bonzini } 12327c86663bSPaolo Bonzini 1233221e7610SPaolo Bonzini if (vcpu->arch.smi_pending && !svm_smi_blocked(vcpu)) { 123455714cddSPaolo Bonzini if (block_nested_events) 123555714cddSPaolo Bonzini return -EBUSY; 1236221e7610SPaolo Bonzini if (!nested_exit_on_smi(svm)) 1237221e7610SPaolo Bonzini return 0; 12383a87c7e0SSean Christopherson nested_svm_simple_vmexit(svm, SVM_EXIT_SMI); 123955714cddSPaolo Bonzini return 0; 124055714cddSPaolo Bonzini } 124155714cddSPaolo Bonzini 1242221e7610SPaolo Bonzini if (vcpu->arch.nmi_pending && !svm_nmi_blocked(vcpu)) { 12439c3d370aSCathy Avery if (block_nested_events) 12449c3d370aSCathy Avery return -EBUSY; 1245221e7610SPaolo Bonzini if (!nested_exit_on_nmi(svm)) 1246221e7610SPaolo Bonzini return 0; 12473a87c7e0SSean Christopherson nested_svm_simple_vmexit(svm, SVM_EXIT_NMI); 12489c3d370aSCathy Avery return 0; 12499c3d370aSCathy Avery } 12509c3d370aSCathy Avery 1251221e7610SPaolo Bonzini if (kvm_cpu_has_interrupt(vcpu) && !svm_interrupt_blocked(vcpu)) { 1252883b0a91SJoerg Roedel if (block_nested_events) 1253883b0a91SJoerg Roedel return -EBUSY; 1254221e7610SPaolo Bonzini if (!nested_exit_on_intr(svm)) 1255221e7610SPaolo Bonzini return 0; 12563a87c7e0SSean Christopherson trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip); 12573a87c7e0SSean Christopherson nested_svm_simple_vmexit(svm, SVM_EXIT_INTR); 1258883b0a91SJoerg Roedel return 0; 1259883b0a91SJoerg Roedel } 1260883b0a91SJoerg Roedel 1261883b0a91SJoerg Roedel return 0; 1262883b0a91SJoerg Roedel } 1263883b0a91SJoerg Roedel 1264883b0a91SJoerg Roedel int nested_svm_exit_special(struct vcpu_svm *svm) 1265883b0a91SJoerg Roedel { 1266883b0a91SJoerg Roedel u32 exit_code = svm->vmcb->control.exit_code; 1267883b0a91SJoerg Roedel 1268883b0a91SJoerg Roedel switch (exit_code) { 1269883b0a91SJoerg Roedel case SVM_EXIT_INTR: 1270883b0a91SJoerg Roedel case SVM_EXIT_NMI: 1271883b0a91SJoerg Roedel case SVM_EXIT_NPF: 1272883b0a91SJoerg Roedel return NESTED_EXIT_HOST; 12737c86663bSPaolo Bonzini case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: { 12747c86663bSPaolo Bonzini u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE); 12757c86663bSPaolo Bonzini 12764995a368SCathy Avery if (svm->vmcb01.ptr->control.intercepts[INTERCEPT_EXCEPTION] & 12779780d51dSBabu Moger excp_bits) 12787c86663bSPaolo Bonzini return NESTED_EXIT_HOST; 12797c86663bSPaolo Bonzini else if (exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR && 128068fd66f1SVitaly Kuznetsov svm->vcpu.arch.apf.host_apf_flags) 1281a3535be7SPaolo Bonzini /* Trap async PF even if not shadowing */ 1282883b0a91SJoerg Roedel return NESTED_EXIT_HOST; 1283883b0a91SJoerg Roedel break; 12847c86663bSPaolo Bonzini } 1285883b0a91SJoerg Roedel default: 1286883b0a91SJoerg Roedel break; 1287883b0a91SJoerg Roedel } 1288883b0a91SJoerg Roedel 1289883b0a91SJoerg Roedel return NESTED_EXIT_CONTINUE; 1290883b0a91SJoerg Roedel } 129133b22172SPaolo Bonzini 12925228eb96SMaxim Levitsky void nested_svm_update_tsc_ratio_msr(struct kvm_vcpu *vcpu) 12935228eb96SMaxim Levitsky { 12945228eb96SMaxim Levitsky struct vcpu_svm *svm = to_svm(vcpu); 12955228eb96SMaxim Levitsky 12965228eb96SMaxim Levitsky vcpu->arch.tsc_scaling_ratio = 12975228eb96SMaxim Levitsky kvm_calc_nested_tsc_multiplier(vcpu->arch.l1_tsc_scaling_ratio, 12985228eb96SMaxim Levitsky svm->tsc_ratio_msr); 12995228eb96SMaxim Levitsky svm_write_tsc_multiplier(vcpu, vcpu->arch.tsc_scaling_ratio); 13005228eb96SMaxim Levitsky } 13015228eb96SMaxim Levitsky 13028fc78909SEmanuele Giuseppe Esposito /* Inverse operation of nested_copy_vmcb_control_to_cache(). asid is copied too. */ 13038fc78909SEmanuele Giuseppe Esposito static void nested_copy_vmcb_cache_to_control(struct vmcb_control_area *dst, 13048fc78909SEmanuele Giuseppe Esposito struct vmcb_ctrl_area_cached *from) 13058fc78909SEmanuele Giuseppe Esposito { 13068fc78909SEmanuele Giuseppe Esposito unsigned int i; 13078fc78909SEmanuele Giuseppe Esposito 13088fc78909SEmanuele Giuseppe Esposito memset(dst, 0, sizeof(struct vmcb_control_area)); 13098fc78909SEmanuele Giuseppe Esposito 13108fc78909SEmanuele Giuseppe Esposito for (i = 0; i < MAX_INTERCEPT; i++) 13118fc78909SEmanuele Giuseppe Esposito dst->intercepts[i] = from->intercepts[i]; 13128fc78909SEmanuele Giuseppe Esposito 13138fc78909SEmanuele Giuseppe Esposito dst->iopm_base_pa = from->iopm_base_pa; 13148fc78909SEmanuele Giuseppe Esposito dst->msrpm_base_pa = from->msrpm_base_pa; 13158fc78909SEmanuele Giuseppe Esposito dst->tsc_offset = from->tsc_offset; 13168fc78909SEmanuele Giuseppe Esposito dst->asid = from->asid; 13178fc78909SEmanuele Giuseppe Esposito dst->tlb_ctl = from->tlb_ctl; 13188fc78909SEmanuele Giuseppe Esposito dst->int_ctl = from->int_ctl; 13198fc78909SEmanuele Giuseppe Esposito dst->int_vector = from->int_vector; 13208fc78909SEmanuele Giuseppe Esposito dst->int_state = from->int_state; 13218fc78909SEmanuele Giuseppe Esposito dst->exit_code = from->exit_code; 13228fc78909SEmanuele Giuseppe Esposito dst->exit_code_hi = from->exit_code_hi; 13238fc78909SEmanuele Giuseppe Esposito dst->exit_info_1 = from->exit_info_1; 13248fc78909SEmanuele Giuseppe Esposito dst->exit_info_2 = from->exit_info_2; 13258fc78909SEmanuele Giuseppe Esposito dst->exit_int_info = from->exit_int_info; 13268fc78909SEmanuele Giuseppe Esposito dst->exit_int_info_err = from->exit_int_info_err; 13278fc78909SEmanuele Giuseppe Esposito dst->nested_ctl = from->nested_ctl; 13288fc78909SEmanuele Giuseppe Esposito dst->event_inj = from->event_inj; 13298fc78909SEmanuele Giuseppe Esposito dst->event_inj_err = from->event_inj_err; 13308fc78909SEmanuele Giuseppe Esposito dst->nested_cr3 = from->nested_cr3; 13318fc78909SEmanuele Giuseppe Esposito dst->virt_ext = from->virt_ext; 13328fc78909SEmanuele Giuseppe Esposito dst->pause_filter_count = from->pause_filter_count; 13338fc78909SEmanuele Giuseppe Esposito dst->pause_filter_thresh = from->pause_filter_thresh; 133466c03a92SVitaly Kuznetsov /* 'clean' and 'reserved_sw' are not changed by KVM */ 13358fc78909SEmanuele Giuseppe Esposito } 13368fc78909SEmanuele Giuseppe Esposito 1337cc440cdaSPaolo Bonzini static int svm_get_nested_state(struct kvm_vcpu *vcpu, 1338cc440cdaSPaolo Bonzini struct kvm_nested_state __user *user_kvm_nested_state, 1339cc440cdaSPaolo Bonzini u32 user_data_size) 1340cc440cdaSPaolo Bonzini { 1341cc440cdaSPaolo Bonzini struct vcpu_svm *svm; 13428fc78909SEmanuele Giuseppe Esposito struct vmcb_control_area *ctl; 13438fc78909SEmanuele Giuseppe Esposito unsigned long r; 1344cc440cdaSPaolo Bonzini struct kvm_nested_state kvm_state = { 1345cc440cdaSPaolo Bonzini .flags = 0, 1346cc440cdaSPaolo Bonzini .format = KVM_STATE_NESTED_FORMAT_SVM, 1347cc440cdaSPaolo Bonzini .size = sizeof(kvm_state), 1348cc440cdaSPaolo Bonzini }; 1349cc440cdaSPaolo Bonzini struct vmcb __user *user_vmcb = (struct vmcb __user *) 1350cc440cdaSPaolo Bonzini &user_kvm_nested_state->data.svm[0]; 1351cc440cdaSPaolo Bonzini 1352cc440cdaSPaolo Bonzini if (!vcpu) 1353cc440cdaSPaolo Bonzini return kvm_state.size + KVM_STATE_NESTED_SVM_VMCB_SIZE; 1354cc440cdaSPaolo Bonzini 1355cc440cdaSPaolo Bonzini svm = to_svm(vcpu); 1356cc440cdaSPaolo Bonzini 1357cc440cdaSPaolo Bonzini if (user_data_size < kvm_state.size) 1358cc440cdaSPaolo Bonzini goto out; 1359cc440cdaSPaolo Bonzini 1360cc440cdaSPaolo Bonzini /* First fill in the header and copy it out. */ 1361cc440cdaSPaolo Bonzini if (is_guest_mode(vcpu)) { 13620dd16b5bSMaxim Levitsky kvm_state.hdr.svm.vmcb_pa = svm->nested.vmcb12_gpa; 1363cc440cdaSPaolo Bonzini kvm_state.size += KVM_STATE_NESTED_SVM_VMCB_SIZE; 1364cc440cdaSPaolo Bonzini kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE; 1365cc440cdaSPaolo Bonzini 1366cc440cdaSPaolo Bonzini if (svm->nested.nested_run_pending) 1367cc440cdaSPaolo Bonzini kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING; 1368cc440cdaSPaolo Bonzini } 1369cc440cdaSPaolo Bonzini 1370cc440cdaSPaolo Bonzini if (gif_set(svm)) 1371cc440cdaSPaolo Bonzini kvm_state.flags |= KVM_STATE_NESTED_GIF_SET; 1372cc440cdaSPaolo Bonzini 1373cc440cdaSPaolo Bonzini if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state))) 1374cc440cdaSPaolo Bonzini return -EFAULT; 1375cc440cdaSPaolo Bonzini 1376cc440cdaSPaolo Bonzini if (!is_guest_mode(vcpu)) 1377cc440cdaSPaolo Bonzini goto out; 1378cc440cdaSPaolo Bonzini 1379cc440cdaSPaolo Bonzini /* 1380cc440cdaSPaolo Bonzini * Copy over the full size of the VMCB rather than just the size 1381cc440cdaSPaolo Bonzini * of the structs. 1382cc440cdaSPaolo Bonzini */ 1383cc440cdaSPaolo Bonzini if (clear_user(user_vmcb, KVM_STATE_NESTED_SVM_VMCB_SIZE)) 1384cc440cdaSPaolo Bonzini return -EFAULT; 13858fc78909SEmanuele Giuseppe Esposito 13868fc78909SEmanuele Giuseppe Esposito ctl = kzalloc(sizeof(*ctl), GFP_KERNEL); 13878fc78909SEmanuele Giuseppe Esposito if (!ctl) 13888fc78909SEmanuele Giuseppe Esposito return -ENOMEM; 13898fc78909SEmanuele Giuseppe Esposito 13908fc78909SEmanuele Giuseppe Esposito nested_copy_vmcb_cache_to_control(ctl, &svm->nested.ctl); 13918fc78909SEmanuele Giuseppe Esposito r = copy_to_user(&user_vmcb->control, ctl, 13928fc78909SEmanuele Giuseppe Esposito sizeof(user_vmcb->control)); 13938fc78909SEmanuele Giuseppe Esposito kfree(ctl); 13948fc78909SEmanuele Giuseppe Esposito if (r) 1395cc440cdaSPaolo Bonzini return -EFAULT; 13968fc78909SEmanuele Giuseppe Esposito 13974995a368SCathy Avery if (copy_to_user(&user_vmcb->save, &svm->vmcb01.ptr->save, 1398cc440cdaSPaolo Bonzini sizeof(user_vmcb->save))) 1399cc440cdaSPaolo Bonzini return -EFAULT; 1400cc440cdaSPaolo Bonzini out: 1401cc440cdaSPaolo Bonzini return kvm_state.size; 1402cc440cdaSPaolo Bonzini } 1403cc440cdaSPaolo Bonzini 1404cc440cdaSPaolo Bonzini static int svm_set_nested_state(struct kvm_vcpu *vcpu, 1405cc440cdaSPaolo Bonzini struct kvm_nested_state __user *user_kvm_nested_state, 1406cc440cdaSPaolo Bonzini struct kvm_nested_state *kvm_state) 1407cc440cdaSPaolo Bonzini { 1408cc440cdaSPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 1409cc440cdaSPaolo Bonzini struct vmcb __user *user_vmcb = (struct vmcb __user *) 1410cc440cdaSPaolo Bonzini &user_kvm_nested_state->data.svm[0]; 14116ccbd29aSJoerg Roedel struct vmcb_control_area *ctl; 14126ccbd29aSJoerg Roedel struct vmcb_save_area *save; 1413b7a3d8b6SEmanuele Giuseppe Esposito struct vmcb_save_area_cached save_cached; 14148fc78909SEmanuele Giuseppe Esposito struct vmcb_ctrl_area_cached ctl_cached; 1415dbc4739bSSean Christopherson unsigned long cr0; 14166ccbd29aSJoerg Roedel int ret; 1417cc440cdaSPaolo Bonzini 14186ccbd29aSJoerg Roedel BUILD_BUG_ON(sizeof(struct vmcb_control_area) + sizeof(struct vmcb_save_area) > 14196ccbd29aSJoerg Roedel KVM_STATE_NESTED_SVM_VMCB_SIZE); 14206ccbd29aSJoerg Roedel 1421cc440cdaSPaolo Bonzini if (kvm_state->format != KVM_STATE_NESTED_FORMAT_SVM) 1422cc440cdaSPaolo Bonzini return -EINVAL; 1423cc440cdaSPaolo Bonzini 1424cc440cdaSPaolo Bonzini if (kvm_state->flags & ~(KVM_STATE_NESTED_GUEST_MODE | 1425cc440cdaSPaolo Bonzini KVM_STATE_NESTED_RUN_PENDING | 1426cc440cdaSPaolo Bonzini KVM_STATE_NESTED_GIF_SET)) 1427cc440cdaSPaolo Bonzini return -EINVAL; 1428cc440cdaSPaolo Bonzini 1429cc440cdaSPaolo Bonzini /* 1430cc440cdaSPaolo Bonzini * If in guest mode, vcpu->arch.efer actually refers to the L2 guest's 1431cc440cdaSPaolo Bonzini * EFER.SVME, but EFER.SVME still has to be 1 for VMRUN to succeed. 1432cc440cdaSPaolo Bonzini */ 1433cc440cdaSPaolo Bonzini if (!(vcpu->arch.efer & EFER_SVME)) { 1434cc440cdaSPaolo Bonzini /* GIF=1 and no guest mode are required if SVME=0. */ 1435cc440cdaSPaolo Bonzini if (kvm_state->flags != KVM_STATE_NESTED_GIF_SET) 1436cc440cdaSPaolo Bonzini return -EINVAL; 1437cc440cdaSPaolo Bonzini } 1438cc440cdaSPaolo Bonzini 1439cc440cdaSPaolo Bonzini /* SMM temporarily disables SVM, so we cannot be in guest mode. */ 1440cc440cdaSPaolo Bonzini if (is_smm(vcpu) && (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 1441cc440cdaSPaolo Bonzini return -EINVAL; 1442cc440cdaSPaolo Bonzini 1443cc440cdaSPaolo Bonzini if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) { 1444f7e57078SSean Christopherson svm_leave_nested(vcpu); 1445d5cd6f34SVitaly Kuznetsov svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET)); 1446d5cd6f34SVitaly Kuznetsov return 0; 1447cc440cdaSPaolo Bonzini } 1448cc440cdaSPaolo Bonzini 1449cc440cdaSPaolo Bonzini if (!page_address_valid(vcpu, kvm_state->hdr.svm.vmcb_pa)) 1450cc440cdaSPaolo Bonzini return -EINVAL; 1451cc440cdaSPaolo Bonzini if (kvm_state->size < sizeof(*kvm_state) + KVM_STATE_NESTED_SVM_VMCB_SIZE) 1452cc440cdaSPaolo Bonzini return -EINVAL; 1453cc440cdaSPaolo Bonzini 14546ccbd29aSJoerg Roedel ret = -ENOMEM; 1455eba04b20SSean Christopherson ctl = kzalloc(sizeof(*ctl), GFP_KERNEL_ACCOUNT); 1456eba04b20SSean Christopherson save = kzalloc(sizeof(*save), GFP_KERNEL_ACCOUNT); 14576ccbd29aSJoerg Roedel if (!ctl || !save) 14586ccbd29aSJoerg Roedel goto out_free; 14596ccbd29aSJoerg Roedel 14606ccbd29aSJoerg Roedel ret = -EFAULT; 14616ccbd29aSJoerg Roedel if (copy_from_user(ctl, &user_vmcb->control, sizeof(*ctl))) 14626ccbd29aSJoerg Roedel goto out_free; 14636ccbd29aSJoerg Roedel if (copy_from_user(save, &user_vmcb->save, sizeof(*save))) 14646ccbd29aSJoerg Roedel goto out_free; 14656ccbd29aSJoerg Roedel 14666ccbd29aSJoerg Roedel ret = -EINVAL; 146766c03a92SVitaly Kuznetsov __nested_copy_vmcb_control_to_cache(vcpu, &ctl_cached, ctl); 14688fc78909SEmanuele Giuseppe Esposito if (!__nested_vmcb_check_controls(vcpu, &ctl_cached)) 14696ccbd29aSJoerg Roedel goto out_free; 1470cc440cdaSPaolo Bonzini 1471cc440cdaSPaolo Bonzini /* 1472cc440cdaSPaolo Bonzini * Processor state contains L2 state. Check that it is 1473cb9b6a1bSPaolo Bonzini * valid for guest mode (see nested_vmcb_check_save). 1474cc440cdaSPaolo Bonzini */ 1475cc440cdaSPaolo Bonzini cr0 = kvm_read_cr0(vcpu); 1476cc440cdaSPaolo Bonzini if (((cr0 & X86_CR0_CD) == 0) && (cr0 & X86_CR0_NW)) 14776ccbd29aSJoerg Roedel goto out_free; 1478cc440cdaSPaolo Bonzini 1479cc440cdaSPaolo Bonzini /* 1480cc440cdaSPaolo Bonzini * Validate host state saved from before VMRUN (see 1481cc440cdaSPaolo Bonzini * nested_svm_check_permissions). 1482cc440cdaSPaolo Bonzini */ 1483b7a3d8b6SEmanuele Giuseppe Esposito __nested_copy_vmcb_save_to_cache(&save_cached, save); 14846906e06dSKrish Sadhukhan if (!(save->cr0 & X86_CR0_PG) || 14856906e06dSKrish Sadhukhan !(save->cr0 & X86_CR0_PE) || 14866906e06dSKrish Sadhukhan (save->rflags & X86_EFLAGS_VM) || 1487b7a3d8b6SEmanuele Giuseppe Esposito !__nested_vmcb_check_save(vcpu, &save_cached)) 14886ccbd29aSJoerg Roedel goto out_free; 1489cc440cdaSPaolo Bonzini 1490b222b0b8SMaxim Levitsky 1491b222b0b8SMaxim Levitsky /* 14924995a368SCathy Avery * All checks done, we can enter guest mode. Userspace provides 14934995a368SCathy Avery * vmcb12.control, which will be combined with L1 and stored into 14944995a368SCathy Avery * vmcb02, and the L1 save state which we store in vmcb01. 14954995a368SCathy Avery * L2 registers if needed are moved from the current VMCB to VMCB02. 1496cc440cdaSPaolo Bonzini */ 149781f76adaSMaxim Levitsky 14989d290e16SMaxim Levitsky if (is_guest_mode(vcpu)) 1499f7e57078SSean Christopherson svm_leave_nested(vcpu); 15009d290e16SMaxim Levitsky else 15019d290e16SMaxim Levitsky svm->nested.vmcb02.ptr->save = svm->vmcb01.ptr->save; 15029d290e16SMaxim Levitsky 1503063ab16cSMaxim Levitsky svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET)); 1504063ab16cSMaxim Levitsky 150581f76adaSMaxim Levitsky svm->nested.nested_run_pending = 150681f76adaSMaxim Levitsky !!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING); 150781f76adaSMaxim Levitsky 15080dd16b5bSMaxim Levitsky svm->nested.vmcb12_gpa = kvm_state->hdr.svm.vmcb_pa; 1509c08f390aSPaolo Bonzini 15102bb16beaSVitaly Kuznetsov svm_copy_vmrun_state(&svm->vmcb01.ptr->save, save); 15117907160dSEmanuele Giuseppe Esposito nested_copy_vmcb_control_to_cache(svm, ctl); 15124995a368SCathy Avery 15134995a368SCathy Avery svm_switch_vmcb(svm, &svm->nested.vmcb02); 15149e8f0fbfSPaolo Bonzini nested_vmcb02_prepare_control(svm); 1515e1779c27SMaxim Levitsky 1516e1779c27SMaxim Levitsky /* 1517e1779c27SMaxim Levitsky * While the nested guest CR3 is already checked and set by 1518e1779c27SMaxim Levitsky * KVM_SET_SREGS, it was set when nested state was yet loaded, 1519e1779c27SMaxim Levitsky * thus MMU might not be initialized correctly. 1520e1779c27SMaxim Levitsky * Set it again to fix this. 1521e1779c27SMaxim Levitsky */ 1522e1779c27SMaxim Levitsky 1523e1779c27SMaxim Levitsky ret = nested_svm_load_cr3(&svm->vcpu, vcpu->arch.cr3, 1524e1779c27SMaxim Levitsky nested_npt_enabled(svm), false); 1525e1779c27SMaxim Levitsky if (WARN_ON_ONCE(ret)) 1526e1779c27SMaxim Levitsky goto out_free; 1527e1779c27SMaxim Levitsky 152873c25546SVitaly Kuznetsov svm->nested.force_msr_bitmap_recalc = true; 1529e1779c27SMaxim Levitsky 1530a7d5c7ceSPaolo Bonzini kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 15316ccbd29aSJoerg Roedel ret = 0; 15326ccbd29aSJoerg Roedel out_free: 15336ccbd29aSJoerg Roedel kfree(save); 15346ccbd29aSJoerg Roedel kfree(ctl); 15356ccbd29aSJoerg Roedel 15366ccbd29aSJoerg Roedel return ret; 1537cc440cdaSPaolo Bonzini } 1538cc440cdaSPaolo Bonzini 1539232f75d3SMaxim Levitsky static bool svm_get_nested_state_pages(struct kvm_vcpu *vcpu) 1540232f75d3SMaxim Levitsky { 1541232f75d3SMaxim Levitsky struct vcpu_svm *svm = to_svm(vcpu); 1542232f75d3SMaxim Levitsky 1543232f75d3SMaxim Levitsky if (WARN_ON(!is_guest_mode(vcpu))) 1544232f75d3SMaxim Levitsky return true; 1545232f75d3SMaxim Levitsky 1546158a48ecSMaxim Levitsky if (!vcpu->arch.pdptrs_from_userspace && 1547158a48ecSMaxim Levitsky !nested_npt_enabled(svm) && is_pae_paging(vcpu)) 1548b222b0b8SMaxim Levitsky /* 1549b222b0b8SMaxim Levitsky * Reload the guest's PDPTRs since after a migration 1550b222b0b8SMaxim Levitsky * the guest CR3 might be restored prior to setting the nested 1551b222b0b8SMaxim Levitsky * state which can lead to a load of wrong PDPTRs. 1552b222b0b8SMaxim Levitsky */ 15532df4a5ebSLai Jiangshan if (CC(!load_pdptrs(vcpu, vcpu->arch.cr3))) 1554232f75d3SMaxim Levitsky return false; 1555232f75d3SMaxim Levitsky 1556232f75d3SMaxim Levitsky if (!nested_svm_vmrun_msrpm(svm)) { 1557232f75d3SMaxim Levitsky vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1558232f75d3SMaxim Levitsky vcpu->run->internal.suberror = 1559232f75d3SMaxim Levitsky KVM_INTERNAL_ERROR_EMULATION; 1560232f75d3SMaxim Levitsky vcpu->run->internal.ndata = 0; 1561232f75d3SMaxim Levitsky return false; 1562232f75d3SMaxim Levitsky } 1563232f75d3SMaxim Levitsky 1564232f75d3SMaxim Levitsky return true; 1565232f75d3SMaxim Levitsky } 1566232f75d3SMaxim Levitsky 156733b22172SPaolo Bonzini struct kvm_x86_nested_ops svm_nested_ops = { 1568f7e57078SSean Christopherson .leave_nested = svm_leave_nested, 156933b22172SPaolo Bonzini .check_events = svm_check_nested_events, 1570cb6a32c2SSean Christopherson .triple_fault = nested_svm_triple_fault, 1571a7d5c7ceSPaolo Bonzini .get_nested_state_pages = svm_get_nested_state_pages, 1572cc440cdaSPaolo Bonzini .get_state = svm_get_nested_state, 1573cc440cdaSPaolo Bonzini .set_state = svm_set_nested_state, 157433b22172SPaolo Bonzini }; 1575