1883b0a91SJoerg Roedel // SPDX-License-Identifier: GPL-2.0-only 2883b0a91SJoerg Roedel /* 3883b0a91SJoerg Roedel * Kernel-based Virtual Machine driver for Linux 4883b0a91SJoerg Roedel * 5883b0a91SJoerg Roedel * AMD SVM support 6883b0a91SJoerg Roedel * 7883b0a91SJoerg Roedel * Copyright (C) 2006 Qumranet, Inc. 8883b0a91SJoerg Roedel * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9883b0a91SJoerg Roedel * 10883b0a91SJoerg Roedel * Authors: 11883b0a91SJoerg Roedel * Yaniv Kamay <yaniv@qumranet.com> 12883b0a91SJoerg Roedel * Avi Kivity <avi@qumranet.com> 13883b0a91SJoerg Roedel */ 14883b0a91SJoerg Roedel 15883b0a91SJoerg Roedel #define pr_fmt(fmt) "SVM: " fmt 16883b0a91SJoerg Roedel 17883b0a91SJoerg Roedel #include <linux/kvm_types.h> 18883b0a91SJoerg Roedel #include <linux/kvm_host.h> 19883b0a91SJoerg Roedel #include <linux/kernel.h> 20883b0a91SJoerg Roedel 21883b0a91SJoerg Roedel #include <asm/msr-index.h> 225679b803SPaolo Bonzini #include <asm/debugreg.h> 23883b0a91SJoerg Roedel 24883b0a91SJoerg Roedel #include "kvm_emulate.h" 25883b0a91SJoerg Roedel #include "trace.h" 26883b0a91SJoerg Roedel #include "mmu.h" 27883b0a91SJoerg Roedel #include "x86.h" 28cc440cdaSPaolo Bonzini #include "cpuid.h" 295b672408SPaolo Bonzini #include "lapic.h" 30883b0a91SJoerg Roedel #include "svm.h" 3166c03a92SVitaly Kuznetsov #include "hyperv.h" 32883b0a91SJoerg Roedel 3311f0cbf0SSean Christopherson #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK 3411f0cbf0SSean Christopherson 35883b0a91SJoerg Roedel static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu, 36883b0a91SJoerg Roedel struct x86_exception *fault) 37883b0a91SJoerg Roedel { 38883b0a91SJoerg Roedel struct vcpu_svm *svm = to_svm(vcpu); 39db663af4SMaxim Levitsky struct vmcb *vmcb = svm->vmcb; 40883b0a91SJoerg Roedel 41db663af4SMaxim Levitsky if (vmcb->control.exit_code != SVM_EXIT_NPF) { 42883b0a91SJoerg Roedel /* 43883b0a91SJoerg Roedel * TODO: track the cause of the nested page fault, and 44883b0a91SJoerg Roedel * correctly fill in the high bits of exit_info_1. 45883b0a91SJoerg Roedel */ 46db663af4SMaxim Levitsky vmcb->control.exit_code = SVM_EXIT_NPF; 47db663af4SMaxim Levitsky vmcb->control.exit_code_hi = 0; 48db663af4SMaxim Levitsky vmcb->control.exit_info_1 = (1ULL << 32); 49db663af4SMaxim Levitsky vmcb->control.exit_info_2 = fault->address; 50883b0a91SJoerg Roedel } 51883b0a91SJoerg Roedel 52db663af4SMaxim Levitsky vmcb->control.exit_info_1 &= ~0xffffffffULL; 53db663af4SMaxim Levitsky vmcb->control.exit_info_1 |= fault->error_code; 54883b0a91SJoerg Roedel 55883b0a91SJoerg Roedel nested_svm_vmexit(svm); 56883b0a91SJoerg Roedel } 57883b0a91SJoerg Roedel 58a04aead1SPaolo Bonzini static void svm_inject_page_fault_nested(struct kvm_vcpu *vcpu, struct x86_exception *fault) 59a04aead1SPaolo Bonzini { 60a04aead1SPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 61db663af4SMaxim Levitsky struct vmcb *vmcb = svm->vmcb; 62db663af4SMaxim Levitsky 63a04aead1SPaolo Bonzini WARN_ON(!is_guest_mode(vcpu)); 64a04aead1SPaolo Bonzini 658fc78909SEmanuele Giuseppe Esposito if (vmcb12_is_intercept(&svm->nested.ctl, 668fc78909SEmanuele Giuseppe Esposito INTERCEPT_EXCEPTION_OFFSET + PF_VECTOR) && 67a04aead1SPaolo Bonzini !svm->nested.nested_run_pending) { 68db663af4SMaxim Levitsky vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + PF_VECTOR; 69db663af4SMaxim Levitsky vmcb->control.exit_code_hi = 0; 70db663af4SMaxim Levitsky vmcb->control.exit_info_1 = fault->error_code; 71db663af4SMaxim Levitsky vmcb->control.exit_info_2 = fault->address; 72a04aead1SPaolo Bonzini nested_svm_vmexit(svm); 73a04aead1SPaolo Bonzini } else { 74a04aead1SPaolo Bonzini kvm_inject_page_fault(vcpu, fault); 75a04aead1SPaolo Bonzini } 76a04aead1SPaolo Bonzini } 77a04aead1SPaolo Bonzini 78883b0a91SJoerg Roedel static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index) 79883b0a91SJoerg Roedel { 80883b0a91SJoerg Roedel struct vcpu_svm *svm = to_svm(vcpu); 81e670bf68SPaolo Bonzini u64 cr3 = svm->nested.ctl.nested_cr3; 82883b0a91SJoerg Roedel u64 pdpte; 83883b0a91SJoerg Roedel int ret; 84883b0a91SJoerg Roedel 852732be90SSean Christopherson ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(cr3), &pdpte, 86883b0a91SJoerg Roedel offset_in_page(cr3) + index * 8, 8); 87883b0a91SJoerg Roedel if (ret) 88883b0a91SJoerg Roedel return 0; 89883b0a91SJoerg Roedel return pdpte; 90883b0a91SJoerg Roedel } 91883b0a91SJoerg Roedel 92883b0a91SJoerg Roedel static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu) 93883b0a91SJoerg Roedel { 94883b0a91SJoerg Roedel struct vcpu_svm *svm = to_svm(vcpu); 95883b0a91SJoerg Roedel 96e670bf68SPaolo Bonzini return svm->nested.ctl.nested_cr3; 97883b0a91SJoerg Roedel } 98883b0a91SJoerg Roedel 99883b0a91SJoerg Roedel static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu) 100883b0a91SJoerg Roedel { 101929d1cfaSPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 102929d1cfaSPaolo Bonzini 103883b0a91SJoerg Roedel WARN_ON(mmu_is_nested(vcpu)); 104883b0a91SJoerg Roedel 105883b0a91SJoerg Roedel vcpu->arch.mmu = &vcpu->arch.guest_mmu; 10631e96bc6SSean Christopherson 10731e96bc6SSean Christopherson /* 10831e96bc6SSean Christopherson * The NPT format depends on L1's CR4 and EFER, which is in vmcb01. Note, 10931e96bc6SSean Christopherson * when called via KVM_SET_NESTED_STATE, that state may _not_ match current 11031e96bc6SSean Christopherson * vCPU state. CR0.WP is explicitly ignored, while CR0.PG is required. 11131e96bc6SSean Christopherson */ 1124995a368SCathy Avery kvm_init_shadow_npt_mmu(vcpu, X86_CR0_PG, svm->vmcb01.ptr->save.cr4, 1134995a368SCathy Avery svm->vmcb01.ptr->save.efer, 1140f04a2acSVitaly Kuznetsov svm->nested.ctl.nested_cr3); 115883b0a91SJoerg Roedel vcpu->arch.mmu->get_guest_pgd = nested_svm_get_tdp_cr3; 116883b0a91SJoerg Roedel vcpu->arch.mmu->get_pdptr = nested_svm_get_tdp_pdptr; 117883b0a91SJoerg Roedel vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit; 118883b0a91SJoerg Roedel vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; 119883b0a91SJoerg Roedel } 120883b0a91SJoerg Roedel 121883b0a91SJoerg Roedel static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu) 122883b0a91SJoerg Roedel { 123883b0a91SJoerg Roedel vcpu->arch.mmu = &vcpu->arch.root_mmu; 124883b0a91SJoerg Roedel vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; 125883b0a91SJoerg Roedel } 126883b0a91SJoerg Roedel 127b9f3973aSMaxim Levitsky static bool nested_vmcb_needs_vls_intercept(struct vcpu_svm *svm) 128b9f3973aSMaxim Levitsky { 129b9f3973aSMaxim Levitsky if (!svm->v_vmload_vmsave_enabled) 130b9f3973aSMaxim Levitsky return true; 131b9f3973aSMaxim Levitsky 132b9f3973aSMaxim Levitsky if (!nested_npt_enabled(svm)) 133b9f3973aSMaxim Levitsky return true; 134b9f3973aSMaxim Levitsky 135b9f3973aSMaxim Levitsky if (!(svm->nested.ctl.virt_ext & VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK)) 136b9f3973aSMaxim Levitsky return true; 137b9f3973aSMaxim Levitsky 138b9f3973aSMaxim Levitsky return false; 139b9f3973aSMaxim Levitsky } 140b9f3973aSMaxim Levitsky 141883b0a91SJoerg Roedel void recalc_intercepts(struct vcpu_svm *svm) 142883b0a91SJoerg Roedel { 1438fc78909SEmanuele Giuseppe Esposito struct vmcb_control_area *c, *h; 1448fc78909SEmanuele Giuseppe Esposito struct vmcb_ctrl_area_cached *g; 145c45ad722SBabu Moger unsigned int i; 146883b0a91SJoerg Roedel 14706e7852cSJoerg Roedel vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 148883b0a91SJoerg Roedel 149883b0a91SJoerg Roedel if (!is_guest_mode(&svm->vcpu)) 150883b0a91SJoerg Roedel return; 151883b0a91SJoerg Roedel 152883b0a91SJoerg Roedel c = &svm->vmcb->control; 1534995a368SCathy Avery h = &svm->vmcb01.ptr->control; 154e670bf68SPaolo Bonzini g = &svm->nested.ctl; 155883b0a91SJoerg Roedel 156c45ad722SBabu Moger for (i = 0; i < MAX_INTERCEPT; i++) 157c45ad722SBabu Moger c->intercepts[i] = h->intercepts[i]; 158c45ad722SBabu Moger 159e9fd761aSPaolo Bonzini if (g->int_ctl & V_INTR_MASKING_MASK) { 160883b0a91SJoerg Roedel /* We only want the cr8 intercept bits of L1 */ 16103bfeeb9SBabu Moger vmcb_clr_intercept(c, INTERCEPT_CR8_READ); 16203bfeeb9SBabu Moger vmcb_clr_intercept(c, INTERCEPT_CR8_WRITE); 163883b0a91SJoerg Roedel 164883b0a91SJoerg Roedel /* 165883b0a91SJoerg Roedel * Once running L2 with HF_VINTR_MASK, EFLAGS.IF does not 166883b0a91SJoerg Roedel * affect any interrupt we may want to inject; therefore, 167883b0a91SJoerg Roedel * interrupt window vmexits are irrelevant to L0. 168883b0a91SJoerg Roedel */ 169c62e2e94SBabu Moger vmcb_clr_intercept(c, INTERCEPT_VINTR); 170883b0a91SJoerg Roedel } 171883b0a91SJoerg Roedel 172883b0a91SJoerg Roedel /* We don't want to see VMMCALLs from a nested guest */ 173c62e2e94SBabu Moger vmcb_clr_intercept(c, INTERCEPT_VMMCALL); 174883b0a91SJoerg Roedel 175c45ad722SBabu Moger for (i = 0; i < MAX_INTERCEPT; i++) 176c45ad722SBabu Moger c->intercepts[i] |= g->intercepts[i]; 1774b639a9fSMaxim Levitsky 1784b639a9fSMaxim Levitsky /* If SMI is not intercepted, ignore guest SMI intercept as well */ 1794b639a9fSMaxim Levitsky if (!intercept_smi) 1804b639a9fSMaxim Levitsky vmcb_clr_intercept(c, INTERCEPT_SMI); 181c7dfa400SMaxim Levitsky 182b9f3973aSMaxim Levitsky if (nested_vmcb_needs_vls_intercept(svm)) { 183b9f3973aSMaxim Levitsky /* 184b9f3973aSMaxim Levitsky * If the virtual VMLOAD/VMSAVE is not enabled for the L2, 185b9f3973aSMaxim Levitsky * we must intercept these instructions to correctly 186b9f3973aSMaxim Levitsky * emulate them in case L1 doesn't intercept them. 187b9f3973aSMaxim Levitsky */ 188c7dfa400SMaxim Levitsky vmcb_set_intercept(c, INTERCEPT_VMLOAD); 189c7dfa400SMaxim Levitsky vmcb_set_intercept(c, INTERCEPT_VMSAVE); 190b9f3973aSMaxim Levitsky } else { 191b9f3973aSMaxim Levitsky WARN_ON(!(c->virt_ext & VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK)); 192b9f3973aSMaxim Levitsky } 193883b0a91SJoerg Roedel } 194883b0a91SJoerg Roedel 19566c03a92SVitaly Kuznetsov /* 19666c03a92SVitaly Kuznetsov * Merge L0's (KVM) and L1's (Nested VMCB) MSR permission bitmaps. The function 19766c03a92SVitaly Kuznetsov * is optimized in that it only merges the parts where KVM MSR permission bitmap 19866c03a92SVitaly Kuznetsov * may contain zero bits. 19966c03a92SVitaly Kuznetsov */ 200883b0a91SJoerg Roedel static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm) 201883b0a91SJoerg Roedel { 20266c03a92SVitaly Kuznetsov struct hv_enlightenments *hve = 20366c03a92SVitaly Kuznetsov (struct hv_enlightenments *)svm->nested.ctl.reserved_sw; 204883b0a91SJoerg Roedel int i; 205883b0a91SJoerg Roedel 20666c03a92SVitaly Kuznetsov /* 20766c03a92SVitaly Kuznetsov * MSR bitmap update can be skipped when: 20866c03a92SVitaly Kuznetsov * - MSR bitmap for L1 hasn't changed. 20966c03a92SVitaly Kuznetsov * - Nested hypervisor (L1) is attempting to launch the same L2 as 21066c03a92SVitaly Kuznetsov * before. 21166c03a92SVitaly Kuznetsov * - Nested hypervisor (L1) is using Hyper-V emulation interface and 21266c03a92SVitaly Kuznetsov * tells KVM (L0) there were no changes in MSR bitmap for L2. 21366c03a92SVitaly Kuznetsov */ 21466c03a92SVitaly Kuznetsov if (!svm->nested.force_msr_bitmap_recalc && 21566c03a92SVitaly Kuznetsov kvm_hv_hypercall_enabled(&svm->vcpu) && 21666c03a92SVitaly Kuznetsov hve->hv_enlightenments_control.msr_bitmap && 21766c03a92SVitaly Kuznetsov (svm->nested.ctl.clean & BIT(VMCB_HV_NESTED_ENLIGHTENMENTS))) 21866c03a92SVitaly Kuznetsov goto set_msrpm_base_pa; 21966c03a92SVitaly Kuznetsov 2208fc78909SEmanuele Giuseppe Esposito if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT))) 221883b0a91SJoerg Roedel return true; 222883b0a91SJoerg Roedel 223883b0a91SJoerg Roedel for (i = 0; i < MSRPM_OFFSETS; i++) { 224883b0a91SJoerg Roedel u32 value, p; 225883b0a91SJoerg Roedel u64 offset; 226883b0a91SJoerg Roedel 227883b0a91SJoerg Roedel if (msrpm_offsets[i] == 0xffffffff) 228883b0a91SJoerg Roedel break; 229883b0a91SJoerg Roedel 230883b0a91SJoerg Roedel p = msrpm_offsets[i]; 231e670bf68SPaolo Bonzini offset = svm->nested.ctl.msrpm_base_pa + (p * 4); 232883b0a91SJoerg Roedel 233883b0a91SJoerg Roedel if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4)) 234883b0a91SJoerg Roedel return false; 235883b0a91SJoerg Roedel 236883b0a91SJoerg Roedel svm->nested.msrpm[p] = svm->msrpm[p] | value; 237883b0a91SJoerg Roedel } 238883b0a91SJoerg Roedel 23973c25546SVitaly Kuznetsov svm->nested.force_msr_bitmap_recalc = false; 24073c25546SVitaly Kuznetsov 24166c03a92SVitaly Kuznetsov set_msrpm_base_pa: 242883b0a91SJoerg Roedel svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm)); 243883b0a91SJoerg Roedel 244883b0a91SJoerg Roedel return true; 245883b0a91SJoerg Roedel } 246883b0a91SJoerg Roedel 247ee695f22SKrish Sadhukhan /* 248ee695f22SKrish Sadhukhan * Bits 11:0 of bitmap address are ignored by hardware 249ee695f22SKrish Sadhukhan */ 250ee695f22SKrish Sadhukhan static bool nested_svm_check_bitmap_pa(struct kvm_vcpu *vcpu, u64 pa, u32 size) 251ee695f22SKrish Sadhukhan { 252ee695f22SKrish Sadhukhan u64 addr = PAGE_ALIGN(pa); 253ee695f22SKrish Sadhukhan 254ee695f22SKrish Sadhukhan return kvm_vcpu_is_legal_gpa(vcpu, addr) && 255ee695f22SKrish Sadhukhan kvm_vcpu_is_legal_gpa(vcpu, addr + size - 1); 256ee695f22SKrish Sadhukhan } 257ee695f22SKrish Sadhukhan 258174a921bSKrish Sadhukhan static bool nested_svm_check_tlb_ctl(struct kvm_vcpu *vcpu, u8 tlb_ctl) 259174a921bSKrish Sadhukhan { 260174a921bSKrish Sadhukhan /* Nested FLUSHBYASID is not supported yet. */ 261174a921bSKrish Sadhukhan switch(tlb_ctl) { 262174a921bSKrish Sadhukhan case TLB_CONTROL_DO_NOTHING: 263174a921bSKrish Sadhukhan case TLB_CONTROL_FLUSH_ALL_ASID: 264174a921bSKrish Sadhukhan return true; 265174a921bSKrish Sadhukhan default: 266174a921bSKrish Sadhukhan return false; 267174a921bSKrish Sadhukhan } 268174a921bSKrish Sadhukhan } 269174a921bSKrish Sadhukhan 270bd95926cSPaolo Bonzini static bool __nested_vmcb_check_controls(struct kvm_vcpu *vcpu, 2718fc78909SEmanuele Giuseppe Esposito struct vmcb_ctrl_area_cached *control) 272ca46d739SPaolo Bonzini { 2738fc78909SEmanuele Giuseppe Esposito if (CC(!vmcb12_is_intercept(control, INTERCEPT_VMRUN))) 274ca46d739SPaolo Bonzini return false; 275ca46d739SPaolo Bonzini 27611f0cbf0SSean Christopherson if (CC(control->asid == 0)) 277ca46d739SPaolo Bonzini return false; 278ca46d739SPaolo Bonzini 27911f0cbf0SSean Christopherson if (CC((control->nested_ctl & SVM_NESTED_CTL_NP_ENABLE) && !npt_enabled)) 280ca46d739SPaolo Bonzini return false; 281ca46d739SPaolo Bonzini 282ee695f22SKrish Sadhukhan if (CC(!nested_svm_check_bitmap_pa(vcpu, control->msrpm_base_pa, 283ee695f22SKrish Sadhukhan MSRPM_SIZE))) 284ee695f22SKrish Sadhukhan return false; 285ee695f22SKrish Sadhukhan if (CC(!nested_svm_check_bitmap_pa(vcpu, control->iopm_base_pa, 286ee695f22SKrish Sadhukhan IOPM_SIZE))) 287ee695f22SKrish Sadhukhan return false; 288ee695f22SKrish Sadhukhan 289174a921bSKrish Sadhukhan if (CC(!nested_svm_check_tlb_ctl(vcpu, control->tlb_ctl))) 290174a921bSKrish Sadhukhan return false; 291174a921bSKrish Sadhukhan 292ca46d739SPaolo Bonzini return true; 293ca46d739SPaolo Bonzini } 294ca46d739SPaolo Bonzini 2956906e06dSKrish Sadhukhan /* Common checks that apply to both L1 and L2 state. */ 296b7a3d8b6SEmanuele Giuseppe Esposito static bool __nested_vmcb_check_save(struct kvm_vcpu *vcpu, 297b7a3d8b6SEmanuele Giuseppe Esposito struct vmcb_save_area_cached *save) 2986906e06dSKrish Sadhukhan { 29911f0cbf0SSean Christopherson if (CC(!(save->efer & EFER_SVME))) 3006906e06dSKrish Sadhukhan return false; 3016906e06dSKrish Sadhukhan 30211f0cbf0SSean Christopherson if (CC((save->cr0 & X86_CR0_CD) == 0 && (save->cr0 & X86_CR0_NW)) || 30311f0cbf0SSean Christopherson CC(save->cr0 & ~0xffffffffULL)) 3046906e06dSKrish Sadhukhan return false; 3056906e06dSKrish Sadhukhan 30611f0cbf0SSean Christopherson if (CC(!kvm_dr6_valid(save->dr6)) || CC(!kvm_dr7_valid(save->dr7))) 3076906e06dSKrish Sadhukhan return false; 3086906e06dSKrish Sadhukhan 309907afa48SEmanuele Giuseppe Esposito /* 310907afa48SEmanuele Giuseppe Esposito * These checks are also performed by KVM_SET_SREGS, 311907afa48SEmanuele Giuseppe Esposito * except that EFER.LMA is not checked by SVM against 312907afa48SEmanuele Giuseppe Esposito * CR0.PG && EFER.LME. 313907afa48SEmanuele Giuseppe Esposito */ 314907afa48SEmanuele Giuseppe Esposito if ((save->efer & EFER_LME) && (save->cr0 & X86_CR0_PG)) { 315907afa48SEmanuele Giuseppe Esposito if (CC(!(save->cr4 & X86_CR4_PAE)) || 316907afa48SEmanuele Giuseppe Esposito CC(!(save->cr0 & X86_CR0_PE)) || 317907afa48SEmanuele Giuseppe Esposito CC(kvm_vcpu_is_illegal_gpa(vcpu, save->cr3))) 318907afa48SEmanuele Giuseppe Esposito return false; 319907afa48SEmanuele Giuseppe Esposito } 320907afa48SEmanuele Giuseppe Esposito 321907afa48SEmanuele Giuseppe Esposito if (CC(!kvm_is_valid_cr4(vcpu, save->cr4))) 3226906e06dSKrish Sadhukhan return false; 3236906e06dSKrish Sadhukhan 32463129754SPaolo Bonzini if (CC(!kvm_valid_efer(vcpu, save->efer))) 3256906e06dSKrish Sadhukhan return false; 3266906e06dSKrish Sadhukhan 3276906e06dSKrish Sadhukhan return true; 3286906e06dSKrish Sadhukhan } 3296906e06dSKrish Sadhukhan 330b7a3d8b6SEmanuele Giuseppe Esposito static bool nested_vmcb_check_save(struct kvm_vcpu *vcpu) 331b7a3d8b6SEmanuele Giuseppe Esposito { 332b7a3d8b6SEmanuele Giuseppe Esposito struct vcpu_svm *svm = to_svm(vcpu); 333b7a3d8b6SEmanuele Giuseppe Esposito struct vmcb_save_area_cached *save = &svm->nested.save; 334b7a3d8b6SEmanuele Giuseppe Esposito 335b7a3d8b6SEmanuele Giuseppe Esposito return __nested_vmcb_check_save(vcpu, save); 336b7a3d8b6SEmanuele Giuseppe Esposito } 337b7a3d8b6SEmanuele Giuseppe Esposito 338bd95926cSPaolo Bonzini static bool nested_vmcb_check_controls(struct kvm_vcpu *vcpu) 339bd95926cSPaolo Bonzini { 340bd95926cSPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 3418fc78909SEmanuele Giuseppe Esposito struct vmcb_ctrl_area_cached *ctl = &svm->nested.ctl; 342bd95926cSPaolo Bonzini 343bd95926cSPaolo Bonzini return __nested_vmcb_check_controls(vcpu, ctl); 344bd95926cSPaolo Bonzini } 345bd95926cSPaolo Bonzini 3467907160dSEmanuele Giuseppe Esposito static 34766c03a92SVitaly Kuznetsov void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu, 34866c03a92SVitaly Kuznetsov struct vmcb_ctrl_area_cached *to, 3497907160dSEmanuele Giuseppe Esposito struct vmcb_control_area *from) 3507907160dSEmanuele Giuseppe Esposito { 3517907160dSEmanuele Giuseppe Esposito unsigned int i; 3527907160dSEmanuele Giuseppe Esposito 3537907160dSEmanuele Giuseppe Esposito for (i = 0; i < MAX_INTERCEPT; i++) 3547907160dSEmanuele Giuseppe Esposito to->intercepts[i] = from->intercepts[i]; 3557907160dSEmanuele Giuseppe Esposito 3567907160dSEmanuele Giuseppe Esposito to->iopm_base_pa = from->iopm_base_pa; 3577907160dSEmanuele Giuseppe Esposito to->msrpm_base_pa = from->msrpm_base_pa; 3587907160dSEmanuele Giuseppe Esposito to->tsc_offset = from->tsc_offset; 3597907160dSEmanuele Giuseppe Esposito to->tlb_ctl = from->tlb_ctl; 3607907160dSEmanuele Giuseppe Esposito to->int_ctl = from->int_ctl; 3617907160dSEmanuele Giuseppe Esposito to->int_vector = from->int_vector; 3627907160dSEmanuele Giuseppe Esposito to->int_state = from->int_state; 3637907160dSEmanuele Giuseppe Esposito to->exit_code = from->exit_code; 3647907160dSEmanuele Giuseppe Esposito to->exit_code_hi = from->exit_code_hi; 3657907160dSEmanuele Giuseppe Esposito to->exit_info_1 = from->exit_info_1; 3667907160dSEmanuele Giuseppe Esposito to->exit_info_2 = from->exit_info_2; 3677907160dSEmanuele Giuseppe Esposito to->exit_int_info = from->exit_int_info; 3687907160dSEmanuele Giuseppe Esposito to->exit_int_info_err = from->exit_int_info_err; 3697907160dSEmanuele Giuseppe Esposito to->nested_ctl = from->nested_ctl; 3707907160dSEmanuele Giuseppe Esposito to->event_inj = from->event_inj; 3717907160dSEmanuele Giuseppe Esposito to->event_inj_err = from->event_inj_err; 3727907160dSEmanuele Giuseppe Esposito to->nested_cr3 = from->nested_cr3; 3737907160dSEmanuele Giuseppe Esposito to->virt_ext = from->virt_ext; 3747907160dSEmanuele Giuseppe Esposito to->pause_filter_count = from->pause_filter_count; 3757907160dSEmanuele Giuseppe Esposito to->pause_filter_thresh = from->pause_filter_thresh; 3767907160dSEmanuele Giuseppe Esposito 3777907160dSEmanuele Giuseppe Esposito /* Copy asid here because nested_vmcb_check_controls will check it. */ 3787907160dSEmanuele Giuseppe Esposito to->asid = from->asid; 3797907160dSEmanuele Giuseppe Esposito to->msrpm_base_pa &= ~0x0fffULL; 3807907160dSEmanuele Giuseppe Esposito to->iopm_base_pa &= ~0x0fffULL; 38166c03a92SVitaly Kuznetsov 38266c03a92SVitaly Kuznetsov /* Hyper-V extensions (Enlightened VMCB) */ 38366c03a92SVitaly Kuznetsov if (kvm_hv_hypercall_enabled(vcpu)) { 38466c03a92SVitaly Kuznetsov to->clean = from->clean; 38566c03a92SVitaly Kuznetsov memcpy(to->reserved_sw, from->reserved_sw, 38666c03a92SVitaly Kuznetsov sizeof(struct hv_enlightenments)); 38766c03a92SVitaly Kuznetsov } 3887907160dSEmanuele Giuseppe Esposito } 3897907160dSEmanuele Giuseppe Esposito 3907907160dSEmanuele Giuseppe Esposito void nested_copy_vmcb_control_to_cache(struct vcpu_svm *svm, 3913e06f016SPaolo Bonzini struct vmcb_control_area *control) 3923e06f016SPaolo Bonzini { 39366c03a92SVitaly Kuznetsov __nested_copy_vmcb_control_to_cache(&svm->vcpu, &svm->nested.ctl, control); 3943e06f016SPaolo Bonzini } 3953e06f016SPaolo Bonzini 396f2740a8dSEmanuele Giuseppe Esposito static void __nested_copy_vmcb_save_to_cache(struct vmcb_save_area_cached *to, 397f2740a8dSEmanuele Giuseppe Esposito struct vmcb_save_area *from) 398f2740a8dSEmanuele Giuseppe Esposito { 399f2740a8dSEmanuele Giuseppe Esposito /* 400f2740a8dSEmanuele Giuseppe Esposito * Copy only fields that are validated, as we need them 401f2740a8dSEmanuele Giuseppe Esposito * to avoid TOC/TOU races. 402f2740a8dSEmanuele Giuseppe Esposito */ 403f2740a8dSEmanuele Giuseppe Esposito to->efer = from->efer; 404f2740a8dSEmanuele Giuseppe Esposito to->cr0 = from->cr0; 405f2740a8dSEmanuele Giuseppe Esposito to->cr3 = from->cr3; 406f2740a8dSEmanuele Giuseppe Esposito to->cr4 = from->cr4; 407f2740a8dSEmanuele Giuseppe Esposito 408f2740a8dSEmanuele Giuseppe Esposito to->dr6 = from->dr6; 409f2740a8dSEmanuele Giuseppe Esposito to->dr7 = from->dr7; 410f2740a8dSEmanuele Giuseppe Esposito } 411f2740a8dSEmanuele Giuseppe Esposito 412f2740a8dSEmanuele Giuseppe Esposito void nested_copy_vmcb_save_to_cache(struct vcpu_svm *svm, 413f2740a8dSEmanuele Giuseppe Esposito struct vmcb_save_area *save) 414f2740a8dSEmanuele Giuseppe Esposito { 415f2740a8dSEmanuele Giuseppe Esposito __nested_copy_vmcb_save_to_cache(&svm->nested.save, save); 416f2740a8dSEmanuele Giuseppe Esposito } 417f2740a8dSEmanuele Giuseppe Esposito 4182d8a42beSPaolo Bonzini /* 4192d8a42beSPaolo Bonzini * Synchronize fields that are written by the processor, so that 4209e8f0fbfSPaolo Bonzini * they can be copied back into the vmcb12. 4212d8a42beSPaolo Bonzini */ 4229e8f0fbfSPaolo Bonzini void nested_sync_control_from_vmcb02(struct vcpu_svm *svm) 4232d8a42beSPaolo Bonzini { 4242d8a42beSPaolo Bonzini u32 mask; 4252d8a42beSPaolo Bonzini svm->nested.ctl.event_inj = svm->vmcb->control.event_inj; 4262d8a42beSPaolo Bonzini svm->nested.ctl.event_inj_err = svm->vmcb->control.event_inj_err; 4272d8a42beSPaolo Bonzini 4282d8a42beSPaolo Bonzini /* Only a few fields of int_ctl are written by the processor. */ 4292d8a42beSPaolo Bonzini mask = V_IRQ_MASK | V_TPR_MASK; 4302d8a42beSPaolo Bonzini if (!(svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK) && 431a284ba56SJoerg Roedel svm_is_intercept(svm, INTERCEPT_VINTR)) { 4322d8a42beSPaolo Bonzini /* 4332d8a42beSPaolo Bonzini * In order to request an interrupt window, L0 is usurping 4342d8a42beSPaolo Bonzini * svm->vmcb->control.int_ctl and possibly setting V_IRQ 4352d8a42beSPaolo Bonzini * even if it was clear in L1's VMCB. Restoring it would be 4362d8a42beSPaolo Bonzini * wrong. However, in this case V_IRQ will remain true until 4372d8a42beSPaolo Bonzini * interrupt_window_interception calls svm_clear_vintr and 4382d8a42beSPaolo Bonzini * restores int_ctl. We can just leave it aside. 4392d8a42beSPaolo Bonzini */ 4402d8a42beSPaolo Bonzini mask &= ~V_IRQ_MASK; 4412d8a42beSPaolo Bonzini } 4422d8a42beSPaolo Bonzini svm->nested.ctl.int_ctl &= ~mask; 4432d8a42beSPaolo Bonzini svm->nested.ctl.int_ctl |= svm->vmcb->control.int_ctl & mask; 4442d8a42beSPaolo Bonzini } 4452d8a42beSPaolo Bonzini 44636e2e983SPaolo Bonzini /* 44736e2e983SPaolo Bonzini * Transfer any event that L0 or L1 wanted to inject into L2 to 44836e2e983SPaolo Bonzini * EXIT_INT_INFO. 44936e2e983SPaolo Bonzini */ 4509e8f0fbfSPaolo Bonzini static void nested_save_pending_event_to_vmcb12(struct vcpu_svm *svm, 4510dd16b5bSMaxim Levitsky struct vmcb *vmcb12) 45236e2e983SPaolo Bonzini { 45336e2e983SPaolo Bonzini struct kvm_vcpu *vcpu = &svm->vcpu; 45436e2e983SPaolo Bonzini u32 exit_int_info = 0; 45536e2e983SPaolo Bonzini unsigned int nr; 45636e2e983SPaolo Bonzini 45736e2e983SPaolo Bonzini if (vcpu->arch.exception.injected) { 45836e2e983SPaolo Bonzini nr = vcpu->arch.exception.nr; 45936e2e983SPaolo Bonzini exit_int_info = nr | SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_EXEPT; 46036e2e983SPaolo Bonzini 46136e2e983SPaolo Bonzini if (vcpu->arch.exception.has_error_code) { 46236e2e983SPaolo Bonzini exit_int_info |= SVM_EVTINJ_VALID_ERR; 4630dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info_err = 46436e2e983SPaolo Bonzini vcpu->arch.exception.error_code; 46536e2e983SPaolo Bonzini } 46636e2e983SPaolo Bonzini 46736e2e983SPaolo Bonzini } else if (vcpu->arch.nmi_injected) { 46836e2e983SPaolo Bonzini exit_int_info = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI; 46936e2e983SPaolo Bonzini 47036e2e983SPaolo Bonzini } else if (vcpu->arch.interrupt.injected) { 47136e2e983SPaolo Bonzini nr = vcpu->arch.interrupt.nr; 47236e2e983SPaolo Bonzini exit_int_info = nr | SVM_EVTINJ_VALID; 47336e2e983SPaolo Bonzini 47436e2e983SPaolo Bonzini if (vcpu->arch.interrupt.soft) 47536e2e983SPaolo Bonzini exit_int_info |= SVM_EVTINJ_TYPE_SOFT; 47636e2e983SPaolo Bonzini else 47736e2e983SPaolo Bonzini exit_int_info |= SVM_EVTINJ_TYPE_INTR; 47836e2e983SPaolo Bonzini } 47936e2e983SPaolo Bonzini 4800dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info = exit_int_info; 48136e2e983SPaolo Bonzini } 48236e2e983SPaolo Bonzini 483d2e56019SSean Christopherson static void nested_svm_transition_tlb_flush(struct kvm_vcpu *vcpu) 484d2e56019SSean Christopherson { 485d2e56019SSean Christopherson /* 486d2e56019SSean Christopherson * TODO: optimize unconditional TLB flush/MMU sync. A partial list of 487d2e56019SSean Christopherson * things to fix before this can be conditional: 488d2e56019SSean Christopherson * 489d2e56019SSean Christopherson * - Flush TLBs for both L1 and L2 remote TLB flush 490d2e56019SSean Christopherson * - Honor L1's request to flush an ASID on nested VMRUN 491d2e56019SSean Christopherson * - Sync nested NPT MMU on VMRUN that flushes L2's ASID[*] 492d2e56019SSean Christopherson * - Don't crush a pending TLB flush in vmcb02 on nested VMRUN 493d2e56019SSean Christopherson * - Flush L1's ASID on KVM_REQ_TLB_FLUSH_GUEST 494d2e56019SSean Christopherson * 495d2e56019SSean Christopherson * [*] Unlike nested EPT, SVM's ASID management can invalidate nested 496d2e56019SSean Christopherson * NPT guest-physical mappings on VMRUN. 497d2e56019SSean Christopherson */ 498d2e56019SSean Christopherson kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); 499d2e56019SSean Christopherson kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 500d2e56019SSean Christopherson } 501d2e56019SSean Christopherson 50262156f6cSVitaly Kuznetsov /* 503d82aaef9SVitaly Kuznetsov * Load guest's/host's cr3 on nested vmentry or vmexit. @nested_npt is true 504d82aaef9SVitaly Kuznetsov * if we are emulating VM-Entry into a guest with NPT enabled. 50562156f6cSVitaly Kuznetsov */ 50662156f6cSVitaly Kuznetsov static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, 507b222b0b8SMaxim Levitsky bool nested_npt, bool reload_pdptrs) 50862156f6cSVitaly Kuznetsov { 50911f0cbf0SSean Christopherson if (CC(kvm_vcpu_is_illegal_gpa(vcpu, cr3))) 510a506fdd2SVitaly Kuznetsov return -EINVAL; 511a506fdd2SVitaly Kuznetsov 512b222b0b8SMaxim Levitsky if (reload_pdptrs && !nested_npt && is_pae_paging(vcpu) && 5132df4a5ebSLai Jiangshan CC(!load_pdptrs(vcpu, cr3))) 514a506fdd2SVitaly Kuznetsov return -EINVAL; 515a506fdd2SVitaly Kuznetsov 516a506fdd2SVitaly Kuznetsov vcpu->arch.cr3 = cr3; 517a506fdd2SVitaly Kuznetsov 518616007c8SSean Christopherson /* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */ 519c9060662SSean Christopherson kvm_init_mmu(vcpu); 520a506fdd2SVitaly Kuznetsov 5213cffc89dSPaolo Bonzini if (!nested_npt) 5223cffc89dSPaolo Bonzini kvm_mmu_new_pgd(vcpu, cr3); 5233cffc89dSPaolo Bonzini 524a506fdd2SVitaly Kuznetsov return 0; 52562156f6cSVitaly Kuznetsov } 52662156f6cSVitaly Kuznetsov 5274995a368SCathy Avery void nested_vmcb02_compute_g_pat(struct vcpu_svm *svm) 5284995a368SCathy Avery { 5294995a368SCathy Avery if (!svm->nested.vmcb02.ptr) 5304995a368SCathy Avery return; 5314995a368SCathy Avery 5324995a368SCathy Avery /* FIXME: merge g_pat from vmcb01 and vmcb12. */ 5334995a368SCathy Avery svm->nested.vmcb02.ptr->save.g_pat = svm->vmcb01.ptr->save.g_pat; 5344995a368SCathy Avery } 5354995a368SCathy Avery 5369e8f0fbfSPaolo Bonzini static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12) 537883b0a91SJoerg Roedel { 5388173396eSCathy Avery bool new_vmcb12 = false; 5391d5a1b58SMaxim Levitsky struct vmcb *vmcb01 = svm->vmcb01.ptr; 540db663af4SMaxim Levitsky struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; 5418173396eSCathy Avery 5424995a368SCathy Avery nested_vmcb02_compute_g_pat(svm); 5434995a368SCathy Avery 544883b0a91SJoerg Roedel /* Load the nested guest state */ 5458173396eSCathy Avery if (svm->nested.vmcb12_gpa != svm->nested.last_vmcb12_gpa) { 5468173396eSCathy Avery new_vmcb12 = true; 5478173396eSCathy Avery svm->nested.last_vmcb12_gpa = svm->nested.vmcb12_gpa; 54873c25546SVitaly Kuznetsov svm->nested.force_msr_bitmap_recalc = true; 5498173396eSCathy Avery } 5508173396eSCathy Avery 5518173396eSCathy Avery if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_SEG))) { 552db663af4SMaxim Levitsky vmcb02->save.es = vmcb12->save.es; 553db663af4SMaxim Levitsky vmcb02->save.cs = vmcb12->save.cs; 554db663af4SMaxim Levitsky vmcb02->save.ss = vmcb12->save.ss; 555db663af4SMaxim Levitsky vmcb02->save.ds = vmcb12->save.ds; 556db663af4SMaxim Levitsky vmcb02->save.cpl = vmcb12->save.cpl; 557db663af4SMaxim Levitsky vmcb_mark_dirty(vmcb02, VMCB_SEG); 5588173396eSCathy Avery } 5594bb170a5SPaolo Bonzini 5608173396eSCathy Avery if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_DT))) { 561db663af4SMaxim Levitsky vmcb02->save.gdtr = vmcb12->save.gdtr; 562db663af4SMaxim Levitsky vmcb02->save.idtr = vmcb12->save.idtr; 563db663af4SMaxim Levitsky vmcb_mark_dirty(vmcb02, VMCB_DT); 5648173396eSCathy Avery } 5654bb170a5SPaolo Bonzini 5668cce12b3SPaolo Bonzini kvm_set_rflags(&svm->vcpu, vmcb12->save.rflags | X86_EFLAGS_FIXED); 5673c346c0cSPaolo Bonzini 568355d0473SEmanuele Giuseppe Esposito svm_set_efer(&svm->vcpu, svm->nested.save.efer); 5693c346c0cSPaolo Bonzini 570355d0473SEmanuele Giuseppe Esposito svm_set_cr0(&svm->vcpu, svm->nested.save.cr0); 571355d0473SEmanuele Giuseppe Esposito svm_set_cr4(&svm->vcpu, svm->nested.save.cr4); 5724bb170a5SPaolo Bonzini 5734bb170a5SPaolo Bonzini svm->vcpu.arch.cr2 = vmcb12->save.cr2; 5748173396eSCathy Avery 5750dd16b5bSMaxim Levitsky kvm_rax_write(&svm->vcpu, vmcb12->save.rax); 5760dd16b5bSMaxim Levitsky kvm_rsp_write(&svm->vcpu, vmcb12->save.rsp); 5770dd16b5bSMaxim Levitsky kvm_rip_write(&svm->vcpu, vmcb12->save.rip); 578883b0a91SJoerg Roedel 579883b0a91SJoerg Roedel /* In case we don't even reach vcpu_run, the fields are not updated */ 580db663af4SMaxim Levitsky vmcb02->save.rax = vmcb12->save.rax; 581db663af4SMaxim Levitsky vmcb02->save.rsp = vmcb12->save.rsp; 582db663af4SMaxim Levitsky vmcb02->save.rip = vmcb12->save.rip; 5834bb170a5SPaolo Bonzini 5848173396eSCathy Avery /* These bits will be set properly on the first execution when new_vmc12 is true */ 5858173396eSCathy Avery if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_DR))) { 586db663af4SMaxim Levitsky vmcb02->save.dr7 = svm->nested.save.dr7 | DR7_FIXED_1; 587355d0473SEmanuele Giuseppe Esposito svm->vcpu.arch.dr6 = svm->nested.save.dr6 | DR6_ACTIVE_LOW; 588db663af4SMaxim Levitsky vmcb_mark_dirty(vmcb02, VMCB_DR); 589f241d711SPaolo Bonzini } 5901d5a1b58SMaxim Levitsky 591d20c796cSMaxim Levitsky if (unlikely(svm->lbrv_enabled && (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK))) { 592d20c796cSMaxim Levitsky /* 593d20c796cSMaxim Levitsky * Reserved bits of DEBUGCTL are ignored. Be consistent with 594d20c796cSMaxim Levitsky * svm_set_msr's definition of reserved bits. 595d20c796cSMaxim Levitsky */ 596d20c796cSMaxim Levitsky svm_copy_lbrs(vmcb02, vmcb12); 597d20c796cSMaxim Levitsky vmcb02->save.dbgctl &= ~DEBUGCTL_RESERVED_BITS; 598d20c796cSMaxim Levitsky svm_update_lbrv(&svm->vcpu); 599d20c796cSMaxim Levitsky 600d20c796cSMaxim Levitsky } else if (unlikely(vmcb01->control.virt_ext & LBR_CTL_ENABLE_MASK)) { 6011d5a1b58SMaxim Levitsky svm_copy_lbrs(vmcb02, vmcb01); 6028173396eSCathy Avery } 603d20c796cSMaxim Levitsky } 604883b0a91SJoerg Roedel 6059e8f0fbfSPaolo Bonzini static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) 606f241d711SPaolo Bonzini { 6070f923e07SMaxim Levitsky const u32 int_ctl_vmcb01_bits = 6080f923e07SMaxim Levitsky V_INTR_MASKING_MASK | V_GIF_MASK | V_GIF_ENABLE_MASK; 6090f923e07SMaxim Levitsky 6100f923e07SMaxim Levitsky const u32 int_ctl_vmcb12_bits = V_TPR_MASK | V_IRQ_INJECTION_BITS_MASK; 6110f923e07SMaxim Levitsky 612d2e56019SSean Christopherson struct kvm_vcpu *vcpu = &svm->vcpu; 613db663af4SMaxim Levitsky struct vmcb *vmcb01 = svm->vmcb01.ptr; 614db663af4SMaxim Levitsky struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; 61562156f6cSVitaly Kuznetsov 6167c3ecfcdSPaolo Bonzini /* 6177c3ecfcdSPaolo Bonzini * Filled at exit: exit_code, exit_code_hi, exit_info_1, exit_info_2, 6187c3ecfcdSPaolo Bonzini * exit_int_info, exit_int_info_err, next_rip, insn_len, insn_bytes. 6197c3ecfcdSPaolo Bonzini */ 6204995a368SCathy Avery 6217c3ecfcdSPaolo Bonzini /* 6227c3ecfcdSPaolo Bonzini * Also covers avic_vapic_bar, avic_backing_page, avic_logical_id, 6237c3ecfcdSPaolo Bonzini * avic_physical_id. 6247c3ecfcdSPaolo Bonzini */ 625feea0136SMaxim Levitsky WARN_ON(kvm_apicv_activated(svm->vcpu.kvm)); 6267c3ecfcdSPaolo Bonzini 6277c3ecfcdSPaolo Bonzini /* Copied from vmcb01. msrpm_base can be overwritten later. */ 628db663af4SMaxim Levitsky vmcb02->control.nested_ctl = vmcb01->control.nested_ctl; 629db663af4SMaxim Levitsky vmcb02->control.iopm_base_pa = vmcb01->control.iopm_base_pa; 630db663af4SMaxim Levitsky vmcb02->control.msrpm_base_pa = vmcb01->control.msrpm_base_pa; 6317c3ecfcdSPaolo Bonzini 6327c3ecfcdSPaolo Bonzini /* Done at vmrun: asid. */ 6337c3ecfcdSPaolo Bonzini 6347c3ecfcdSPaolo Bonzini /* Also overwritten later if necessary. */ 635db663af4SMaxim Levitsky vmcb02->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; 6367c3ecfcdSPaolo Bonzini 6377c3ecfcdSPaolo Bonzini /* nested_cr3. */ 63862156f6cSVitaly Kuznetsov if (nested_npt_enabled(svm)) 639d2e56019SSean Christopherson nested_svm_init_mmu_context(vcpu); 64069cb8774SPaolo Bonzini 6415228eb96SMaxim Levitsky vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset( 6425228eb96SMaxim Levitsky vcpu->arch.l1_tsc_offset, 6435228eb96SMaxim Levitsky svm->nested.ctl.tsc_offset, 6445228eb96SMaxim Levitsky svm->tsc_ratio_msr); 6455228eb96SMaxim Levitsky 646db663af4SMaxim Levitsky vmcb02->control.tsc_offset = vcpu->arch.tsc_offset; 6475228eb96SMaxim Levitsky 6485228eb96SMaxim Levitsky if (svm->tsc_ratio_msr != kvm_default_tsc_scaling_ratio) { 6495228eb96SMaxim Levitsky WARN_ON(!svm->tsc_scaling_enabled); 6505228eb96SMaxim Levitsky nested_svm_update_tsc_ratio_msr(vcpu); 6515228eb96SMaxim Levitsky } 652883b0a91SJoerg Roedel 653db663af4SMaxim Levitsky vmcb02->control.int_ctl = 6540f923e07SMaxim Levitsky (svm->nested.ctl.int_ctl & int_ctl_vmcb12_bits) | 655db663af4SMaxim Levitsky (vmcb01->control.int_ctl & int_ctl_vmcb01_bits); 65691b7130cSPaolo Bonzini 657db663af4SMaxim Levitsky vmcb02->control.int_vector = svm->nested.ctl.int_vector; 658db663af4SMaxim Levitsky vmcb02->control.int_state = svm->nested.ctl.int_state; 659db663af4SMaxim Levitsky vmcb02->control.event_inj = svm->nested.ctl.event_inj; 660db663af4SMaxim Levitsky vmcb02->control.event_inj_err = svm->nested.ctl.event_inj_err; 661883b0a91SJoerg Roedel 6621d5a1b58SMaxim Levitsky vmcb02->control.virt_ext = vmcb01->control.virt_ext & 6631d5a1b58SMaxim Levitsky LBR_CTL_ENABLE_MASK; 664d20c796cSMaxim Levitsky if (svm->lbrv_enabled) 665d20c796cSMaxim Levitsky vmcb02->control.virt_ext |= 666d20c796cSMaxim Levitsky (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK); 6671d5a1b58SMaxim Levitsky 668b9f3973aSMaxim Levitsky if (!nested_vmcb_needs_vls_intercept(svm)) 669db663af4SMaxim Levitsky vmcb02->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; 670b9f3973aSMaxim Levitsky 671*74fd41edSMaxim Levitsky if (kvm_pause_in_guest(svm->vcpu.kvm)) { 672*74fd41edSMaxim Levitsky /* use guest values since host doesn't use them */ 673*74fd41edSMaxim Levitsky vmcb02->control.pause_filter_count = 674*74fd41edSMaxim Levitsky svm->pause_filter_enabled ? 675*74fd41edSMaxim Levitsky svm->nested.ctl.pause_filter_count : 0; 676*74fd41edSMaxim Levitsky 677*74fd41edSMaxim Levitsky vmcb02->control.pause_filter_thresh = 678*74fd41edSMaxim Levitsky svm->pause_threshold_enabled ? 679*74fd41edSMaxim Levitsky svm->nested.ctl.pause_filter_thresh : 0; 680*74fd41edSMaxim Levitsky 681*74fd41edSMaxim Levitsky } else if (!vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_PAUSE)) { 682*74fd41edSMaxim Levitsky /* use host values when guest doesn't use them */ 683*74fd41edSMaxim Levitsky vmcb02->control.pause_filter_count = vmcb01->control.pause_filter_count; 684*74fd41edSMaxim Levitsky vmcb02->control.pause_filter_thresh = vmcb01->control.pause_filter_thresh; 685*74fd41edSMaxim Levitsky } else { 686*74fd41edSMaxim Levitsky /* 687*74fd41edSMaxim Levitsky * Intercept every PAUSE otherwise and 688*74fd41edSMaxim Levitsky * ignore both host and guest values 689*74fd41edSMaxim Levitsky */ 690*74fd41edSMaxim Levitsky vmcb02->control.pause_filter_count = 0; 691*74fd41edSMaxim Levitsky vmcb02->control.pause_filter_thresh = 0; 692*74fd41edSMaxim Levitsky } 693*74fd41edSMaxim Levitsky 694d2e56019SSean Christopherson nested_svm_transition_tlb_flush(vcpu); 695d2e56019SSean Christopherson 696883b0a91SJoerg Roedel /* Enter Guest-Mode */ 697d2e56019SSean Christopherson enter_guest_mode(vcpu); 698883b0a91SJoerg Roedel 699883b0a91SJoerg Roedel /* 700883b0a91SJoerg Roedel * Merge guest and host intercepts - must be called with vcpu in 7014bb170a5SPaolo Bonzini * guest-mode to take effect. 702883b0a91SJoerg Roedel */ 703883b0a91SJoerg Roedel recalc_intercepts(svm); 704f241d711SPaolo Bonzini } 705f241d711SPaolo Bonzini 706d00b99c5SBabu Moger static void nested_svm_copy_common_state(struct vmcb *from_vmcb, struct vmcb *to_vmcb) 707d00b99c5SBabu Moger { 708d00b99c5SBabu Moger /* 709d00b99c5SBabu Moger * Some VMCB state is shared between L1 and L2 and thus has to be 710d00b99c5SBabu Moger * moved at the time of nested vmrun and vmexit. 711d00b99c5SBabu Moger * 712d00b99c5SBabu Moger * VMLOAD/VMSAVE state would also belong in this category, but KVM 713d00b99c5SBabu Moger * always performs VMLOAD and VMSAVE from the VMCB01. 714d00b99c5SBabu Moger */ 715d00b99c5SBabu Moger to_vmcb->save.spec_ctrl = from_vmcb->save.spec_ctrl; 716d00b99c5SBabu Moger } 717d00b99c5SBabu Moger 71863129754SPaolo Bonzini int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb12_gpa, 719e85d3e7bSMaxim Levitsky struct vmcb *vmcb12, bool from_vmrun) 720f241d711SPaolo Bonzini { 72163129754SPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 722a506fdd2SVitaly Kuznetsov int ret; 723a506fdd2SVitaly Kuznetsov 724954f419bSMaxim Levitsky trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb12_gpa, 725954f419bSMaxim Levitsky vmcb12->save.rip, 726954f419bSMaxim Levitsky vmcb12->control.int_ctl, 727954f419bSMaxim Levitsky vmcb12->control.event_inj, 728954f419bSMaxim Levitsky vmcb12->control.nested_ctl); 729954f419bSMaxim Levitsky 730954f419bSMaxim Levitsky trace_kvm_nested_intercepts(vmcb12->control.intercepts[INTERCEPT_CR] & 0xffff, 731954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_CR] >> 16, 732954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_EXCEPTION], 733954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_WORD3], 734954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_WORD4], 735954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_WORD5]); 736954f419bSMaxim Levitsky 737954f419bSMaxim Levitsky 7380dd16b5bSMaxim Levitsky svm->nested.vmcb12_gpa = vmcb12_gpa; 7394995a368SCathy Avery 7404995a368SCathy Avery WARN_ON(svm->vmcb == svm->nested.vmcb02.ptr); 7414995a368SCathy Avery 742d00b99c5SBabu Moger nested_svm_copy_common_state(svm->vmcb01.ptr, svm->nested.vmcb02.ptr); 7434995a368SCathy Avery 7444995a368SCathy Avery svm_switch_vmcb(svm, &svm->nested.vmcb02); 7459e8f0fbfSPaolo Bonzini nested_vmcb02_prepare_control(svm); 7469e8f0fbfSPaolo Bonzini nested_vmcb02_prepare_save(svm, vmcb12); 747f241d711SPaolo Bonzini 748355d0473SEmanuele Giuseppe Esposito ret = nested_svm_load_cr3(&svm->vcpu, svm->nested.save.cr3, 749e85d3e7bSMaxim Levitsky nested_npt_enabled(svm), from_vmrun); 750a506fdd2SVitaly Kuznetsov if (ret) 751a506fdd2SVitaly Kuznetsov return ret; 752a506fdd2SVitaly Kuznetsov 753a04aead1SPaolo Bonzini if (!npt_enabled) 75463129754SPaolo Bonzini vcpu->arch.mmu->inject_page_fault = svm_inject_page_fault_nested; 755a04aead1SPaolo Bonzini 756e85d3e7bSMaxim Levitsky if (!from_vmrun) 757e85d3e7bSMaxim Levitsky kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 758e85d3e7bSMaxim Levitsky 759ffdf7f9eSPaolo Bonzini svm_set_gif(svm, true); 76059cd9bc5SVitaly Kuznetsov 76159cd9bc5SVitaly Kuznetsov return 0; 762883b0a91SJoerg Roedel } 763883b0a91SJoerg Roedel 76463129754SPaolo Bonzini int nested_svm_vmrun(struct kvm_vcpu *vcpu) 765883b0a91SJoerg Roedel { 76663129754SPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 767883b0a91SJoerg Roedel int ret; 7680dd16b5bSMaxim Levitsky struct vmcb *vmcb12; 769883b0a91SJoerg Roedel struct kvm_host_map map; 7700dd16b5bSMaxim Levitsky u64 vmcb12_gpa; 771db663af4SMaxim Levitsky struct vmcb *vmcb01 = svm->vmcb01.ptr; 772883b0a91SJoerg Roedel 773fb79f566SVitaly Kuznetsov if (!svm->nested.hsave_msr) { 774fb79f566SVitaly Kuznetsov kvm_inject_gp(vcpu, 0); 775fb79f566SVitaly Kuznetsov return 1; 776fb79f566SVitaly Kuznetsov } 777fb79f566SVitaly Kuznetsov 77863129754SPaolo Bonzini if (is_smm(vcpu)) { 77963129754SPaolo Bonzini kvm_queue_exception(vcpu, UD_VECTOR); 7807c67f546SPaolo Bonzini return 1; 7817c67f546SPaolo Bonzini } 782883b0a91SJoerg Roedel 7830dd16b5bSMaxim Levitsky vmcb12_gpa = svm->vmcb->save.rax; 78463129754SPaolo Bonzini ret = kvm_vcpu_map(vcpu, gpa_to_gfn(vmcb12_gpa), &map); 785883b0a91SJoerg Roedel if (ret == -EINVAL) { 78663129754SPaolo Bonzini kvm_inject_gp(vcpu, 0); 787883b0a91SJoerg Roedel return 1; 788883b0a91SJoerg Roedel } else if (ret) { 78963129754SPaolo Bonzini return kvm_skip_emulated_instruction(vcpu); 790883b0a91SJoerg Roedel } 791883b0a91SJoerg Roedel 79263129754SPaolo Bonzini ret = kvm_skip_emulated_instruction(vcpu); 793883b0a91SJoerg Roedel 7940dd16b5bSMaxim Levitsky vmcb12 = map.hva; 795883b0a91SJoerg Roedel 7962fcf4876SMaxim Levitsky if (WARN_ON_ONCE(!svm->nested.initialized)) 7972fcf4876SMaxim Levitsky return -EINVAL; 7982fcf4876SMaxim Levitsky 7997907160dSEmanuele Giuseppe Esposito nested_copy_vmcb_control_to_cache(svm, &vmcb12->control); 800f2740a8dSEmanuele Giuseppe Esposito nested_copy_vmcb_save_to_cache(svm, &vmcb12->save); 801cb9b6a1bSPaolo Bonzini 802b7a3d8b6SEmanuele Giuseppe Esposito if (!nested_vmcb_check_save(vcpu) || 803bd95926cSPaolo Bonzini !nested_vmcb_check_controls(vcpu)) { 8040dd16b5bSMaxim Levitsky vmcb12->control.exit_code = SVM_EXIT_ERR; 8050dd16b5bSMaxim Levitsky vmcb12->control.exit_code_hi = 0; 8060dd16b5bSMaxim Levitsky vmcb12->control.exit_info_1 = 0; 8070dd16b5bSMaxim Levitsky vmcb12->control.exit_info_2 = 0; 80869c9dfa2SPaolo Bonzini goto out; 809883b0a91SJoerg Roedel } 810883b0a91SJoerg Roedel 811883b0a91SJoerg Roedel /* 8124995a368SCathy Avery * Since vmcb01 is not in use, we can use it to store some of the L1 8134995a368SCathy Avery * state. 814883b0a91SJoerg Roedel */ 815db663af4SMaxim Levitsky vmcb01->save.efer = vcpu->arch.efer; 816db663af4SMaxim Levitsky vmcb01->save.cr0 = kvm_read_cr0(vcpu); 817db663af4SMaxim Levitsky vmcb01->save.cr4 = vcpu->arch.cr4; 818db663af4SMaxim Levitsky vmcb01->save.rflags = kvm_get_rflags(vcpu); 819db663af4SMaxim Levitsky vmcb01->save.rip = kvm_rip_read(vcpu); 820883b0a91SJoerg Roedel 8214995a368SCathy Avery if (!npt_enabled) 822db663af4SMaxim Levitsky vmcb01->save.cr3 = kvm_read_cr3(vcpu); 823883b0a91SJoerg Roedel 824f74f9414SPaolo Bonzini svm->nested.nested_run_pending = 1; 825883b0a91SJoerg Roedel 826e85d3e7bSMaxim Levitsky if (enter_svm_guest_mode(vcpu, vmcb12_gpa, vmcb12, true)) 82759cd9bc5SVitaly Kuznetsov goto out_exit_err; 82859cd9bc5SVitaly Kuznetsov 82959cd9bc5SVitaly Kuznetsov if (nested_svm_vmrun_msrpm(svm)) 83059cd9bc5SVitaly Kuznetsov goto out; 83159cd9bc5SVitaly Kuznetsov 83259cd9bc5SVitaly Kuznetsov out_exit_err: 833ebdb3dbaSVitaly Kuznetsov svm->nested.nested_run_pending = 0; 834ebdb3dbaSVitaly Kuznetsov 835883b0a91SJoerg Roedel svm->vmcb->control.exit_code = SVM_EXIT_ERR; 836883b0a91SJoerg Roedel svm->vmcb->control.exit_code_hi = 0; 837883b0a91SJoerg Roedel svm->vmcb->control.exit_info_1 = 0; 838883b0a91SJoerg Roedel svm->vmcb->control.exit_info_2 = 0; 839883b0a91SJoerg Roedel 840883b0a91SJoerg Roedel nested_svm_vmexit(svm); 841883b0a91SJoerg Roedel 84269c9dfa2SPaolo Bonzini out: 84363129754SPaolo Bonzini kvm_vcpu_unmap(vcpu, &map, true); 84469c9dfa2SPaolo Bonzini 845883b0a91SJoerg Roedel return ret; 846883b0a91SJoerg Roedel } 847883b0a91SJoerg Roedel 8480a758290SVitaly Kuznetsov /* Copy state save area fields which are handled by VMRUN */ 8492bb16beaSVitaly Kuznetsov void svm_copy_vmrun_state(struct vmcb_save_area *to_save, 8502bb16beaSVitaly Kuznetsov struct vmcb_save_area *from_save) 8510a758290SVitaly Kuznetsov { 8520a758290SVitaly Kuznetsov to_save->es = from_save->es; 8530a758290SVitaly Kuznetsov to_save->cs = from_save->cs; 8540a758290SVitaly Kuznetsov to_save->ss = from_save->ss; 8550a758290SVitaly Kuznetsov to_save->ds = from_save->ds; 8560a758290SVitaly Kuznetsov to_save->gdtr = from_save->gdtr; 8570a758290SVitaly Kuznetsov to_save->idtr = from_save->idtr; 8580a758290SVitaly Kuznetsov to_save->rflags = from_save->rflags | X86_EFLAGS_FIXED; 8590a758290SVitaly Kuznetsov to_save->efer = from_save->efer; 8600a758290SVitaly Kuznetsov to_save->cr0 = from_save->cr0; 8610a758290SVitaly Kuznetsov to_save->cr3 = from_save->cr3; 8620a758290SVitaly Kuznetsov to_save->cr4 = from_save->cr4; 8630a758290SVitaly Kuznetsov to_save->rax = from_save->rax; 8640a758290SVitaly Kuznetsov to_save->rsp = from_save->rsp; 8650a758290SVitaly Kuznetsov to_save->rip = from_save->rip; 8660a758290SVitaly Kuznetsov to_save->cpl = 0; 8670a758290SVitaly Kuznetsov } 8680a758290SVitaly Kuznetsov 8692bb16beaSVitaly Kuznetsov void svm_copy_vmloadsave_state(struct vmcb *to_vmcb, struct vmcb *from_vmcb) 870883b0a91SJoerg Roedel { 871883b0a91SJoerg Roedel to_vmcb->save.fs = from_vmcb->save.fs; 872883b0a91SJoerg Roedel to_vmcb->save.gs = from_vmcb->save.gs; 873883b0a91SJoerg Roedel to_vmcb->save.tr = from_vmcb->save.tr; 874883b0a91SJoerg Roedel to_vmcb->save.ldtr = from_vmcb->save.ldtr; 875883b0a91SJoerg Roedel to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base; 876883b0a91SJoerg Roedel to_vmcb->save.star = from_vmcb->save.star; 877883b0a91SJoerg Roedel to_vmcb->save.lstar = from_vmcb->save.lstar; 878883b0a91SJoerg Roedel to_vmcb->save.cstar = from_vmcb->save.cstar; 879883b0a91SJoerg Roedel to_vmcb->save.sfmask = from_vmcb->save.sfmask; 880883b0a91SJoerg Roedel to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs; 881883b0a91SJoerg Roedel to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp; 882883b0a91SJoerg Roedel to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip; 883883b0a91SJoerg Roedel } 884883b0a91SJoerg Roedel 885883b0a91SJoerg Roedel int nested_svm_vmexit(struct vcpu_svm *svm) 886883b0a91SJoerg Roedel { 88763129754SPaolo Bonzini struct kvm_vcpu *vcpu = &svm->vcpu; 888db663af4SMaxim Levitsky struct vmcb *vmcb01 = svm->vmcb01.ptr; 889db663af4SMaxim Levitsky struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; 8900dd16b5bSMaxim Levitsky struct vmcb *vmcb12; 891883b0a91SJoerg Roedel struct kvm_host_map map; 89263129754SPaolo Bonzini int rc; 893883b0a91SJoerg Roedel 894cb6a32c2SSean Christopherson /* Triple faults in L2 should never escape. */ 895cb6a32c2SSean Christopherson WARN_ON_ONCE(kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)); 896cb6a32c2SSean Christopherson 89763129754SPaolo Bonzini rc = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa), &map); 898883b0a91SJoerg Roedel if (rc) { 899883b0a91SJoerg Roedel if (rc == -EINVAL) 90063129754SPaolo Bonzini kvm_inject_gp(vcpu, 0); 901883b0a91SJoerg Roedel return 1; 902883b0a91SJoerg Roedel } 903883b0a91SJoerg Roedel 9040dd16b5bSMaxim Levitsky vmcb12 = map.hva; 905883b0a91SJoerg Roedel 906883b0a91SJoerg Roedel /* Exit Guest-Mode */ 90763129754SPaolo Bonzini leave_guest_mode(vcpu); 9080dd16b5bSMaxim Levitsky svm->nested.vmcb12_gpa = 0; 9092d8a42beSPaolo Bonzini WARN_ON_ONCE(svm->nested.nested_run_pending); 910883b0a91SJoerg Roedel 91163129754SPaolo Bonzini kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 912f2c7ef3bSMaxim Levitsky 91338c0b192SPaolo Bonzini /* in case we halted in L2 */ 91438c0b192SPaolo Bonzini svm->vcpu.arch.mp_state = KVM_MP_STATE_RUNNABLE; 91538c0b192SPaolo Bonzini 916883b0a91SJoerg Roedel /* Give the current vmcb to the guest */ 917883b0a91SJoerg Roedel 918db663af4SMaxim Levitsky vmcb12->save.es = vmcb02->save.es; 919db663af4SMaxim Levitsky vmcb12->save.cs = vmcb02->save.cs; 920db663af4SMaxim Levitsky vmcb12->save.ss = vmcb02->save.ss; 921db663af4SMaxim Levitsky vmcb12->save.ds = vmcb02->save.ds; 922db663af4SMaxim Levitsky vmcb12->save.gdtr = vmcb02->save.gdtr; 923db663af4SMaxim Levitsky vmcb12->save.idtr = vmcb02->save.idtr; 9240dd16b5bSMaxim Levitsky vmcb12->save.efer = svm->vcpu.arch.efer; 92563129754SPaolo Bonzini vmcb12->save.cr0 = kvm_read_cr0(vcpu); 92663129754SPaolo Bonzini vmcb12->save.cr3 = kvm_read_cr3(vcpu); 927db663af4SMaxim Levitsky vmcb12->save.cr2 = vmcb02->save.cr2; 9280dd16b5bSMaxim Levitsky vmcb12->save.cr4 = svm->vcpu.arch.cr4; 92963129754SPaolo Bonzini vmcb12->save.rflags = kvm_get_rflags(vcpu); 93063129754SPaolo Bonzini vmcb12->save.rip = kvm_rip_read(vcpu); 93163129754SPaolo Bonzini vmcb12->save.rsp = kvm_rsp_read(vcpu); 93263129754SPaolo Bonzini vmcb12->save.rax = kvm_rax_read(vcpu); 933db663af4SMaxim Levitsky vmcb12->save.dr7 = vmcb02->save.dr7; 9340dd16b5bSMaxim Levitsky vmcb12->save.dr6 = svm->vcpu.arch.dr6; 935db663af4SMaxim Levitsky vmcb12->save.cpl = vmcb02->save.cpl; 936883b0a91SJoerg Roedel 937db663af4SMaxim Levitsky vmcb12->control.int_state = vmcb02->control.int_state; 938db663af4SMaxim Levitsky vmcb12->control.exit_code = vmcb02->control.exit_code; 939db663af4SMaxim Levitsky vmcb12->control.exit_code_hi = vmcb02->control.exit_code_hi; 940db663af4SMaxim Levitsky vmcb12->control.exit_info_1 = vmcb02->control.exit_info_1; 941db663af4SMaxim Levitsky vmcb12->control.exit_info_2 = vmcb02->control.exit_info_2; 94236e2e983SPaolo Bonzini 9430dd16b5bSMaxim Levitsky if (vmcb12->control.exit_code != SVM_EXIT_ERR) 9449e8f0fbfSPaolo Bonzini nested_save_pending_event_to_vmcb12(svm, vmcb12); 945883b0a91SJoerg Roedel 946883b0a91SJoerg Roedel if (svm->nrips_enabled) 947db663af4SMaxim Levitsky vmcb12->control.next_rip = vmcb02->control.next_rip; 948883b0a91SJoerg Roedel 9490dd16b5bSMaxim Levitsky vmcb12->control.int_ctl = svm->nested.ctl.int_ctl; 9500dd16b5bSMaxim Levitsky vmcb12->control.tlb_ctl = svm->nested.ctl.tlb_ctl; 9510dd16b5bSMaxim Levitsky vmcb12->control.event_inj = svm->nested.ctl.event_inj; 9520dd16b5bSMaxim Levitsky vmcb12->control.event_inj_err = svm->nested.ctl.event_inj_err; 953883b0a91SJoerg Roedel 954*74fd41edSMaxim Levitsky if (!kvm_pause_in_guest(vcpu->kvm) && vmcb02->control.pause_filter_count) 955*74fd41edSMaxim Levitsky vmcb01->control.pause_filter_count = vmcb02->control.pause_filter_count; 956*74fd41edSMaxim Levitsky 957d00b99c5SBabu Moger nested_svm_copy_common_state(svm->nested.vmcb02.ptr, svm->vmcb01.ptr); 958d00b99c5SBabu Moger 9594995a368SCathy Avery svm_switch_vmcb(svm, &svm->vmcb01); 9604995a368SCathy Avery 961d20c796cSMaxim Levitsky if (unlikely(svm->lbrv_enabled && (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK))) { 962d20c796cSMaxim Levitsky svm_copy_lbrs(vmcb12, vmcb02); 963d20c796cSMaxim Levitsky svm_update_lbrv(vcpu); 964d20c796cSMaxim Levitsky } else if (unlikely(vmcb01->control.virt_ext & LBR_CTL_ENABLE_MASK)) { 9651d5a1b58SMaxim Levitsky svm_copy_lbrs(vmcb01, vmcb02); 9661d5a1b58SMaxim Levitsky svm_update_lbrv(vcpu); 9671d5a1b58SMaxim Levitsky } 9681d5a1b58SMaxim Levitsky 9694995a368SCathy Avery /* 9704995a368SCathy Avery * On vmexit the GIF is set to false and 9714995a368SCathy Avery * no event can be injected in L1. 9724995a368SCathy Avery */ 9739883764aSMaxim Levitsky svm_set_gif(svm, false); 974db663af4SMaxim Levitsky vmcb01->control.exit_int_info = 0; 9759883764aSMaxim Levitsky 9767ca62d13SPaolo Bonzini svm->vcpu.arch.tsc_offset = svm->vcpu.arch.l1_tsc_offset; 977db663af4SMaxim Levitsky if (vmcb01->control.tsc_offset != svm->vcpu.arch.tsc_offset) { 978db663af4SMaxim Levitsky vmcb01->control.tsc_offset = svm->vcpu.arch.tsc_offset; 979db663af4SMaxim Levitsky vmcb_mark_dirty(vmcb01, VMCB_INTERCEPTS); 9807ca62d13SPaolo Bonzini } 98118fc6c55SPaolo Bonzini 9825228eb96SMaxim Levitsky if (svm->tsc_ratio_msr != kvm_default_tsc_scaling_ratio) { 9835228eb96SMaxim Levitsky WARN_ON(!svm->tsc_scaling_enabled); 9845228eb96SMaxim Levitsky vcpu->arch.tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; 9855228eb96SMaxim Levitsky svm_write_tsc_multiplier(vcpu, vcpu->arch.tsc_scaling_ratio); 9865228eb96SMaxim Levitsky } 9875228eb96SMaxim Levitsky 988e670bf68SPaolo Bonzini svm->nested.ctl.nested_cr3 = 0; 989883b0a91SJoerg Roedel 9904995a368SCathy Avery /* 9914995a368SCathy Avery * Restore processor state that had been saved in vmcb01 9924995a368SCathy Avery */ 993db663af4SMaxim Levitsky kvm_set_rflags(vcpu, vmcb01->save.rflags); 994db663af4SMaxim Levitsky svm_set_efer(vcpu, vmcb01->save.efer); 995db663af4SMaxim Levitsky svm_set_cr0(vcpu, vmcb01->save.cr0 | X86_CR0_PE); 996db663af4SMaxim Levitsky svm_set_cr4(vcpu, vmcb01->save.cr4); 997db663af4SMaxim Levitsky kvm_rax_write(vcpu, vmcb01->save.rax); 998db663af4SMaxim Levitsky kvm_rsp_write(vcpu, vmcb01->save.rsp); 999db663af4SMaxim Levitsky kvm_rip_write(vcpu, vmcb01->save.rip); 10004995a368SCathy Avery 10014995a368SCathy Avery svm->vcpu.arch.dr7 = DR7_FIXED_1; 10024995a368SCathy Avery kvm_update_dr7(&svm->vcpu); 1003883b0a91SJoerg Roedel 10040dd16b5bSMaxim Levitsky trace_kvm_nested_vmexit_inject(vmcb12->control.exit_code, 10050dd16b5bSMaxim Levitsky vmcb12->control.exit_info_1, 10060dd16b5bSMaxim Levitsky vmcb12->control.exit_info_2, 10070dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info, 10080dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info_err, 100936e2e983SPaolo Bonzini KVM_ISA_SVM); 101036e2e983SPaolo Bonzini 101163129754SPaolo Bonzini kvm_vcpu_unmap(vcpu, &map, true); 1012883b0a91SJoerg Roedel 1013d2e56019SSean Christopherson nested_svm_transition_tlb_flush(vcpu); 1014d2e56019SSean Christopherson 101563129754SPaolo Bonzini nested_svm_uninit_mmu_context(vcpu); 1016bf7dea42SVitaly Kuznetsov 1017db663af4SMaxim Levitsky rc = nested_svm_load_cr3(vcpu, vmcb01->save.cr3, false, true); 1018d82aaef9SVitaly Kuznetsov if (rc) 1019d82aaef9SVitaly Kuznetsov return 1; 1020bf7dea42SVitaly Kuznetsov 1021883b0a91SJoerg Roedel /* 1022883b0a91SJoerg Roedel * Drop what we picked up for L2 via svm_complete_interrupts() so it 1023883b0a91SJoerg Roedel * doesn't end up in L1. 1024883b0a91SJoerg Roedel */ 1025883b0a91SJoerg Roedel svm->vcpu.arch.nmi_injected = false; 102663129754SPaolo Bonzini kvm_clear_exception_queue(vcpu); 102763129754SPaolo Bonzini kvm_clear_interrupt_queue(vcpu); 1028883b0a91SJoerg Roedel 10299a7de6ecSKrish Sadhukhan /* 10309a7de6ecSKrish Sadhukhan * If we are here following the completion of a VMRUN that 10319a7de6ecSKrish Sadhukhan * is being single-stepped, queue the pending #DB intercept 10329a7de6ecSKrish Sadhukhan * right now so that it an be accounted for before we execute 10339a7de6ecSKrish Sadhukhan * L1's next instruction. 10349a7de6ecSKrish Sadhukhan */ 1035db663af4SMaxim Levitsky if (unlikely(vmcb01->save.rflags & X86_EFLAGS_TF)) 10369a7de6ecSKrish Sadhukhan kvm_queue_exception(&(svm->vcpu), DB_VECTOR); 10379a7de6ecSKrish Sadhukhan 1038883b0a91SJoerg Roedel return 0; 1039883b0a91SJoerg Roedel } 1040883b0a91SJoerg Roedel 1041cb6a32c2SSean Christopherson static void nested_svm_triple_fault(struct kvm_vcpu *vcpu) 1042cb6a32c2SSean Christopherson { 10433a87c7e0SSean Christopherson nested_svm_simple_vmexit(to_svm(vcpu), SVM_EXIT_SHUTDOWN); 1044cb6a32c2SSean Christopherson } 1045cb6a32c2SSean Christopherson 10462fcf4876SMaxim Levitsky int svm_allocate_nested(struct vcpu_svm *svm) 10472fcf4876SMaxim Levitsky { 10484995a368SCathy Avery struct page *vmcb02_page; 10492fcf4876SMaxim Levitsky 10502fcf4876SMaxim Levitsky if (svm->nested.initialized) 10512fcf4876SMaxim Levitsky return 0; 10522fcf4876SMaxim Levitsky 10534995a368SCathy Avery vmcb02_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 10544995a368SCathy Avery if (!vmcb02_page) 10552fcf4876SMaxim Levitsky return -ENOMEM; 10564995a368SCathy Avery svm->nested.vmcb02.ptr = page_address(vmcb02_page); 10574995a368SCathy Avery svm->nested.vmcb02.pa = __sme_set(page_to_pfn(vmcb02_page) << PAGE_SHIFT); 10582fcf4876SMaxim Levitsky 10592fcf4876SMaxim Levitsky svm->nested.msrpm = svm_vcpu_alloc_msrpm(); 10602fcf4876SMaxim Levitsky if (!svm->nested.msrpm) 10614995a368SCathy Avery goto err_free_vmcb02; 10622fcf4876SMaxim Levitsky svm_vcpu_init_msrpm(&svm->vcpu, svm->nested.msrpm); 10632fcf4876SMaxim Levitsky 10642fcf4876SMaxim Levitsky svm->nested.initialized = true; 10652fcf4876SMaxim Levitsky return 0; 10662fcf4876SMaxim Levitsky 10674995a368SCathy Avery err_free_vmcb02: 10684995a368SCathy Avery __free_page(vmcb02_page); 10692fcf4876SMaxim Levitsky return -ENOMEM; 10702fcf4876SMaxim Levitsky } 10712fcf4876SMaxim Levitsky 10722fcf4876SMaxim Levitsky void svm_free_nested(struct vcpu_svm *svm) 10732fcf4876SMaxim Levitsky { 10742fcf4876SMaxim Levitsky if (!svm->nested.initialized) 10752fcf4876SMaxim Levitsky return; 10762fcf4876SMaxim Levitsky 10772fcf4876SMaxim Levitsky svm_vcpu_free_msrpm(svm->nested.msrpm); 10782fcf4876SMaxim Levitsky svm->nested.msrpm = NULL; 10792fcf4876SMaxim Levitsky 10804995a368SCathy Avery __free_page(virt_to_page(svm->nested.vmcb02.ptr)); 10814995a368SCathy Avery svm->nested.vmcb02.ptr = NULL; 10822fcf4876SMaxim Levitsky 1083c74ad08fSMaxim Levitsky /* 1084c74ad08fSMaxim Levitsky * When last_vmcb12_gpa matches the current vmcb12 gpa, 1085c74ad08fSMaxim Levitsky * some vmcb12 fields are not loaded if they are marked clean 1086c74ad08fSMaxim Levitsky * in the vmcb12, since in this case they are up to date already. 1087c74ad08fSMaxim Levitsky * 1088c74ad08fSMaxim Levitsky * When the vmcb02 is freed, this optimization becomes invalid. 1089c74ad08fSMaxim Levitsky */ 1090c74ad08fSMaxim Levitsky svm->nested.last_vmcb12_gpa = INVALID_GPA; 1091c74ad08fSMaxim Levitsky 10922fcf4876SMaxim Levitsky svm->nested.initialized = false; 10932fcf4876SMaxim Levitsky } 10942fcf4876SMaxim Levitsky 1095c513f484SPaolo Bonzini /* 1096c513f484SPaolo Bonzini * Forcibly leave nested mode in order to be able to reset the VCPU later on. 1097c513f484SPaolo Bonzini */ 1098f7e57078SSean Christopherson void svm_leave_nested(struct kvm_vcpu *vcpu) 1099c513f484SPaolo Bonzini { 1100f7e57078SSean Christopherson struct vcpu_svm *svm = to_svm(vcpu); 110163129754SPaolo Bonzini 110263129754SPaolo Bonzini if (is_guest_mode(vcpu)) { 1103c513f484SPaolo Bonzini svm->nested.nested_run_pending = 0; 1104c74ad08fSMaxim Levitsky svm->nested.vmcb12_gpa = INVALID_GPA; 1105c74ad08fSMaxim Levitsky 110663129754SPaolo Bonzini leave_guest_mode(vcpu); 11074995a368SCathy Avery 1108deee59baSMaxim Levitsky svm_switch_vmcb(svm, &svm->vmcb01); 11094995a368SCathy Avery 111063129754SPaolo Bonzini nested_svm_uninit_mmu_context(vcpu); 111156fe28deSMaxim Levitsky vmcb_mark_all_dirty(svm->vmcb); 1112c513f484SPaolo Bonzini } 1113a7d5c7ceSPaolo Bonzini 111463129754SPaolo Bonzini kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 1115c513f484SPaolo Bonzini } 1116c513f484SPaolo Bonzini 1117883b0a91SJoerg Roedel static int nested_svm_exit_handled_msr(struct vcpu_svm *svm) 1118883b0a91SJoerg Roedel { 1119883b0a91SJoerg Roedel u32 offset, msr, value; 1120883b0a91SJoerg Roedel int write, mask; 1121883b0a91SJoerg Roedel 11228fc78909SEmanuele Giuseppe Esposito if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT))) 1123883b0a91SJoerg Roedel return NESTED_EXIT_HOST; 1124883b0a91SJoerg Roedel 1125883b0a91SJoerg Roedel msr = svm->vcpu.arch.regs[VCPU_REGS_RCX]; 1126883b0a91SJoerg Roedel offset = svm_msrpm_offset(msr); 1127883b0a91SJoerg Roedel write = svm->vmcb->control.exit_info_1 & 1; 1128883b0a91SJoerg Roedel mask = 1 << ((2 * (msr & 0xf)) + write); 1129883b0a91SJoerg Roedel 1130883b0a91SJoerg Roedel if (offset == MSR_INVALID) 1131883b0a91SJoerg Roedel return NESTED_EXIT_DONE; 1132883b0a91SJoerg Roedel 1133883b0a91SJoerg Roedel /* Offset is in 32 bit units but need in 8 bit units */ 1134883b0a91SJoerg Roedel offset *= 4; 1135883b0a91SJoerg Roedel 1136e670bf68SPaolo Bonzini if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.ctl.msrpm_base_pa + offset, &value, 4)) 1137883b0a91SJoerg Roedel return NESTED_EXIT_DONE; 1138883b0a91SJoerg Roedel 1139883b0a91SJoerg Roedel return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; 1140883b0a91SJoerg Roedel } 1141883b0a91SJoerg Roedel 1142883b0a91SJoerg Roedel static int nested_svm_intercept_ioio(struct vcpu_svm *svm) 1143883b0a91SJoerg Roedel { 1144883b0a91SJoerg Roedel unsigned port, size, iopm_len; 1145883b0a91SJoerg Roedel u16 val, mask; 1146883b0a91SJoerg Roedel u8 start_bit; 1147883b0a91SJoerg Roedel u64 gpa; 1148883b0a91SJoerg Roedel 11498fc78909SEmanuele Giuseppe Esposito if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_IOIO_PROT))) 1150883b0a91SJoerg Roedel return NESTED_EXIT_HOST; 1151883b0a91SJoerg Roedel 1152883b0a91SJoerg Roedel port = svm->vmcb->control.exit_info_1 >> 16; 1153883b0a91SJoerg Roedel size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >> 1154883b0a91SJoerg Roedel SVM_IOIO_SIZE_SHIFT; 1155e670bf68SPaolo Bonzini gpa = svm->nested.ctl.iopm_base_pa + (port / 8); 1156883b0a91SJoerg Roedel start_bit = port % 8; 1157883b0a91SJoerg Roedel iopm_len = (start_bit + size > 8) ? 2 : 1; 1158883b0a91SJoerg Roedel mask = (0xf >> (4 - size)) << start_bit; 1159883b0a91SJoerg Roedel val = 0; 1160883b0a91SJoerg Roedel 1161883b0a91SJoerg Roedel if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len)) 1162883b0a91SJoerg Roedel return NESTED_EXIT_DONE; 1163883b0a91SJoerg Roedel 1164883b0a91SJoerg Roedel return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; 1165883b0a91SJoerg Roedel } 1166883b0a91SJoerg Roedel 1167883b0a91SJoerg Roedel static int nested_svm_intercept(struct vcpu_svm *svm) 1168883b0a91SJoerg Roedel { 1169883b0a91SJoerg Roedel u32 exit_code = svm->vmcb->control.exit_code; 1170883b0a91SJoerg Roedel int vmexit = NESTED_EXIT_HOST; 1171883b0a91SJoerg Roedel 1172883b0a91SJoerg Roedel switch (exit_code) { 1173883b0a91SJoerg Roedel case SVM_EXIT_MSR: 1174883b0a91SJoerg Roedel vmexit = nested_svm_exit_handled_msr(svm); 1175883b0a91SJoerg Roedel break; 1176883b0a91SJoerg Roedel case SVM_EXIT_IOIO: 1177883b0a91SJoerg Roedel vmexit = nested_svm_intercept_ioio(svm); 1178883b0a91SJoerg Roedel break; 1179883b0a91SJoerg Roedel case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: { 11808fc78909SEmanuele Giuseppe Esposito if (vmcb12_is_intercept(&svm->nested.ctl, exit_code)) 1181883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 1182883b0a91SJoerg Roedel break; 1183883b0a91SJoerg Roedel } 1184883b0a91SJoerg Roedel case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: { 11858fc78909SEmanuele Giuseppe Esposito if (vmcb12_is_intercept(&svm->nested.ctl, exit_code)) 1186883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 1187883b0a91SJoerg Roedel break; 1188883b0a91SJoerg Roedel } 1189883b0a91SJoerg Roedel case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: { 11907c86663bSPaolo Bonzini /* 11917c86663bSPaolo Bonzini * Host-intercepted exceptions have been checked already in 11927c86663bSPaolo Bonzini * nested_svm_exit_special. There is nothing to do here, 11937c86663bSPaolo Bonzini * the vmexit is injected by svm_check_nested_events. 11947c86663bSPaolo Bonzini */ 1195883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 1196883b0a91SJoerg Roedel break; 1197883b0a91SJoerg Roedel } 1198883b0a91SJoerg Roedel case SVM_EXIT_ERR: { 1199883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 1200883b0a91SJoerg Roedel break; 1201883b0a91SJoerg Roedel } 1202883b0a91SJoerg Roedel default: { 12038fc78909SEmanuele Giuseppe Esposito if (vmcb12_is_intercept(&svm->nested.ctl, exit_code)) 1204883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 1205883b0a91SJoerg Roedel } 1206883b0a91SJoerg Roedel } 1207883b0a91SJoerg Roedel 1208883b0a91SJoerg Roedel return vmexit; 1209883b0a91SJoerg Roedel } 1210883b0a91SJoerg Roedel 1211883b0a91SJoerg Roedel int nested_svm_exit_handled(struct vcpu_svm *svm) 1212883b0a91SJoerg Roedel { 1213883b0a91SJoerg Roedel int vmexit; 1214883b0a91SJoerg Roedel 1215883b0a91SJoerg Roedel vmexit = nested_svm_intercept(svm); 1216883b0a91SJoerg Roedel 1217883b0a91SJoerg Roedel if (vmexit == NESTED_EXIT_DONE) 1218883b0a91SJoerg Roedel nested_svm_vmexit(svm); 1219883b0a91SJoerg Roedel 1220883b0a91SJoerg Roedel return vmexit; 1221883b0a91SJoerg Roedel } 1222883b0a91SJoerg Roedel 122363129754SPaolo Bonzini int nested_svm_check_permissions(struct kvm_vcpu *vcpu) 1224883b0a91SJoerg Roedel { 122563129754SPaolo Bonzini if (!(vcpu->arch.efer & EFER_SVME) || !is_paging(vcpu)) { 122663129754SPaolo Bonzini kvm_queue_exception(vcpu, UD_VECTOR); 1227883b0a91SJoerg Roedel return 1; 1228883b0a91SJoerg Roedel } 1229883b0a91SJoerg Roedel 123063129754SPaolo Bonzini if (to_svm(vcpu)->vmcb->save.cpl) { 123163129754SPaolo Bonzini kvm_inject_gp(vcpu, 0); 1232883b0a91SJoerg Roedel return 1; 1233883b0a91SJoerg Roedel } 1234883b0a91SJoerg Roedel 1235883b0a91SJoerg Roedel return 0; 1236883b0a91SJoerg Roedel } 1237883b0a91SJoerg Roedel 12387c86663bSPaolo Bonzini static bool nested_exit_on_exception(struct vcpu_svm *svm) 1239883b0a91SJoerg Roedel { 12407c86663bSPaolo Bonzini unsigned int nr = svm->vcpu.arch.exception.nr; 1241883b0a91SJoerg Roedel 12429780d51dSBabu Moger return (svm->nested.ctl.intercepts[INTERCEPT_EXCEPTION] & BIT(nr)); 12437c86663bSPaolo Bonzini } 1244883b0a91SJoerg Roedel 12457c86663bSPaolo Bonzini static void nested_svm_inject_exception_vmexit(struct vcpu_svm *svm) 12467c86663bSPaolo Bonzini { 12477c86663bSPaolo Bonzini unsigned int nr = svm->vcpu.arch.exception.nr; 1248db663af4SMaxim Levitsky struct vmcb *vmcb = svm->vmcb; 1249883b0a91SJoerg Roedel 1250db663af4SMaxim Levitsky vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr; 1251db663af4SMaxim Levitsky vmcb->control.exit_code_hi = 0; 12527c86663bSPaolo Bonzini 12537c86663bSPaolo Bonzini if (svm->vcpu.arch.exception.has_error_code) 1254db663af4SMaxim Levitsky vmcb->control.exit_info_1 = svm->vcpu.arch.exception.error_code; 1255883b0a91SJoerg Roedel 1256883b0a91SJoerg Roedel /* 1257883b0a91SJoerg Roedel * EXITINFO2 is undefined for all exception intercepts other 1258883b0a91SJoerg Roedel * than #PF. 1259883b0a91SJoerg Roedel */ 12607c86663bSPaolo Bonzini if (nr == PF_VECTOR) { 1261883b0a91SJoerg Roedel if (svm->vcpu.arch.exception.nested_apf) 1262db663af4SMaxim Levitsky vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token; 1263883b0a91SJoerg Roedel else if (svm->vcpu.arch.exception.has_payload) 1264db663af4SMaxim Levitsky vmcb->control.exit_info_2 = svm->vcpu.arch.exception.payload; 1265883b0a91SJoerg Roedel else 1266db663af4SMaxim Levitsky vmcb->control.exit_info_2 = svm->vcpu.arch.cr2; 12677c86663bSPaolo Bonzini } else if (nr == DB_VECTOR) { 12687c86663bSPaolo Bonzini /* See inject_pending_event. */ 12697c86663bSPaolo Bonzini kvm_deliver_exception_payload(&svm->vcpu); 12707c86663bSPaolo Bonzini if (svm->vcpu.arch.dr7 & DR7_GD) { 12717c86663bSPaolo Bonzini svm->vcpu.arch.dr7 &= ~DR7_GD; 12727c86663bSPaolo Bonzini kvm_update_dr7(&svm->vcpu); 12737c86663bSPaolo Bonzini } 12747c86663bSPaolo Bonzini } else 12757c86663bSPaolo Bonzini WARN_ON(svm->vcpu.arch.exception.has_payload); 1276883b0a91SJoerg Roedel 12777c86663bSPaolo Bonzini nested_svm_vmexit(svm); 1278883b0a91SJoerg Roedel } 1279883b0a91SJoerg Roedel 12805b672408SPaolo Bonzini static inline bool nested_exit_on_init(struct vcpu_svm *svm) 12815b672408SPaolo Bonzini { 12828fc78909SEmanuele Giuseppe Esposito return vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_INIT); 12835b672408SPaolo Bonzini } 12845b672408SPaolo Bonzini 128533b22172SPaolo Bonzini static int svm_check_nested_events(struct kvm_vcpu *vcpu) 1286883b0a91SJoerg Roedel { 1287883b0a91SJoerg Roedel struct vcpu_svm *svm = to_svm(vcpu); 1288883b0a91SJoerg Roedel bool block_nested_events = 1289bd279629SPaolo Bonzini kvm_event_needs_reinjection(vcpu) || svm->nested.nested_run_pending; 12905b672408SPaolo Bonzini struct kvm_lapic *apic = vcpu->arch.apic; 12915b672408SPaolo Bonzini 12925b672408SPaolo Bonzini if (lapic_in_kernel(vcpu) && 12935b672408SPaolo Bonzini test_bit(KVM_APIC_INIT, &apic->pending_events)) { 12945b672408SPaolo Bonzini if (block_nested_events) 12955b672408SPaolo Bonzini return -EBUSY; 12965b672408SPaolo Bonzini if (!nested_exit_on_init(svm)) 12975b672408SPaolo Bonzini return 0; 12983a87c7e0SSean Christopherson nested_svm_simple_vmexit(svm, SVM_EXIT_INIT); 12995b672408SPaolo Bonzini return 0; 13005b672408SPaolo Bonzini } 1301883b0a91SJoerg Roedel 13027c86663bSPaolo Bonzini if (vcpu->arch.exception.pending) { 13034020da3bSMaxim Levitsky /* 13044020da3bSMaxim Levitsky * Only a pending nested run can block a pending exception. 13054020da3bSMaxim Levitsky * Otherwise an injected NMI/interrupt should either be 13064020da3bSMaxim Levitsky * lost or delivered to the nested hypervisor in the EXITINTINFO 13074020da3bSMaxim Levitsky * vmcb field, while delivering the pending exception. 13084020da3bSMaxim Levitsky */ 13094020da3bSMaxim Levitsky if (svm->nested.nested_run_pending) 13107c86663bSPaolo Bonzini return -EBUSY; 13117c86663bSPaolo Bonzini if (!nested_exit_on_exception(svm)) 13127c86663bSPaolo Bonzini return 0; 13137c86663bSPaolo Bonzini nested_svm_inject_exception_vmexit(svm); 13147c86663bSPaolo Bonzini return 0; 13157c86663bSPaolo Bonzini } 13167c86663bSPaolo Bonzini 1317221e7610SPaolo Bonzini if (vcpu->arch.smi_pending && !svm_smi_blocked(vcpu)) { 131855714cddSPaolo Bonzini if (block_nested_events) 131955714cddSPaolo Bonzini return -EBUSY; 1320221e7610SPaolo Bonzini if (!nested_exit_on_smi(svm)) 1321221e7610SPaolo Bonzini return 0; 13223a87c7e0SSean Christopherson nested_svm_simple_vmexit(svm, SVM_EXIT_SMI); 132355714cddSPaolo Bonzini return 0; 132455714cddSPaolo Bonzini } 132555714cddSPaolo Bonzini 1326221e7610SPaolo Bonzini if (vcpu->arch.nmi_pending && !svm_nmi_blocked(vcpu)) { 13279c3d370aSCathy Avery if (block_nested_events) 13289c3d370aSCathy Avery return -EBUSY; 1329221e7610SPaolo Bonzini if (!nested_exit_on_nmi(svm)) 1330221e7610SPaolo Bonzini return 0; 13313a87c7e0SSean Christopherson nested_svm_simple_vmexit(svm, SVM_EXIT_NMI); 13329c3d370aSCathy Avery return 0; 13339c3d370aSCathy Avery } 13349c3d370aSCathy Avery 1335221e7610SPaolo Bonzini if (kvm_cpu_has_interrupt(vcpu) && !svm_interrupt_blocked(vcpu)) { 1336883b0a91SJoerg Roedel if (block_nested_events) 1337883b0a91SJoerg Roedel return -EBUSY; 1338221e7610SPaolo Bonzini if (!nested_exit_on_intr(svm)) 1339221e7610SPaolo Bonzini return 0; 13403a87c7e0SSean Christopherson trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip); 13413a87c7e0SSean Christopherson nested_svm_simple_vmexit(svm, SVM_EXIT_INTR); 1342883b0a91SJoerg Roedel return 0; 1343883b0a91SJoerg Roedel } 1344883b0a91SJoerg Roedel 1345883b0a91SJoerg Roedel return 0; 1346883b0a91SJoerg Roedel } 1347883b0a91SJoerg Roedel 1348883b0a91SJoerg Roedel int nested_svm_exit_special(struct vcpu_svm *svm) 1349883b0a91SJoerg Roedel { 1350883b0a91SJoerg Roedel u32 exit_code = svm->vmcb->control.exit_code; 1351883b0a91SJoerg Roedel 1352883b0a91SJoerg Roedel switch (exit_code) { 1353883b0a91SJoerg Roedel case SVM_EXIT_INTR: 1354883b0a91SJoerg Roedel case SVM_EXIT_NMI: 1355883b0a91SJoerg Roedel case SVM_EXIT_NPF: 1356883b0a91SJoerg Roedel return NESTED_EXIT_HOST; 13577c86663bSPaolo Bonzini case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: { 13587c86663bSPaolo Bonzini u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE); 13597c86663bSPaolo Bonzini 13604995a368SCathy Avery if (svm->vmcb01.ptr->control.intercepts[INTERCEPT_EXCEPTION] & 13619780d51dSBabu Moger excp_bits) 13627c86663bSPaolo Bonzini return NESTED_EXIT_HOST; 13637c86663bSPaolo Bonzini else if (exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR && 136468fd66f1SVitaly Kuznetsov svm->vcpu.arch.apf.host_apf_flags) 1365a3535be7SPaolo Bonzini /* Trap async PF even if not shadowing */ 1366883b0a91SJoerg Roedel return NESTED_EXIT_HOST; 1367883b0a91SJoerg Roedel break; 13687c86663bSPaolo Bonzini } 1369883b0a91SJoerg Roedel default: 1370883b0a91SJoerg Roedel break; 1371883b0a91SJoerg Roedel } 1372883b0a91SJoerg Roedel 1373883b0a91SJoerg Roedel return NESTED_EXIT_CONTINUE; 1374883b0a91SJoerg Roedel } 137533b22172SPaolo Bonzini 13765228eb96SMaxim Levitsky void nested_svm_update_tsc_ratio_msr(struct kvm_vcpu *vcpu) 13775228eb96SMaxim Levitsky { 13785228eb96SMaxim Levitsky struct vcpu_svm *svm = to_svm(vcpu); 13795228eb96SMaxim Levitsky 13805228eb96SMaxim Levitsky vcpu->arch.tsc_scaling_ratio = 13815228eb96SMaxim Levitsky kvm_calc_nested_tsc_multiplier(vcpu->arch.l1_tsc_scaling_ratio, 13825228eb96SMaxim Levitsky svm->tsc_ratio_msr); 13835228eb96SMaxim Levitsky svm_write_tsc_multiplier(vcpu, vcpu->arch.tsc_scaling_ratio); 13845228eb96SMaxim Levitsky } 13855228eb96SMaxim Levitsky 13868fc78909SEmanuele Giuseppe Esposito /* Inverse operation of nested_copy_vmcb_control_to_cache(). asid is copied too. */ 13878fc78909SEmanuele Giuseppe Esposito static void nested_copy_vmcb_cache_to_control(struct vmcb_control_area *dst, 13888fc78909SEmanuele Giuseppe Esposito struct vmcb_ctrl_area_cached *from) 13898fc78909SEmanuele Giuseppe Esposito { 13908fc78909SEmanuele Giuseppe Esposito unsigned int i; 13918fc78909SEmanuele Giuseppe Esposito 13928fc78909SEmanuele Giuseppe Esposito memset(dst, 0, sizeof(struct vmcb_control_area)); 13938fc78909SEmanuele Giuseppe Esposito 13948fc78909SEmanuele Giuseppe Esposito for (i = 0; i < MAX_INTERCEPT; i++) 13958fc78909SEmanuele Giuseppe Esposito dst->intercepts[i] = from->intercepts[i]; 13968fc78909SEmanuele Giuseppe Esposito 13978fc78909SEmanuele Giuseppe Esposito dst->iopm_base_pa = from->iopm_base_pa; 13988fc78909SEmanuele Giuseppe Esposito dst->msrpm_base_pa = from->msrpm_base_pa; 13998fc78909SEmanuele Giuseppe Esposito dst->tsc_offset = from->tsc_offset; 14008fc78909SEmanuele Giuseppe Esposito dst->asid = from->asid; 14018fc78909SEmanuele Giuseppe Esposito dst->tlb_ctl = from->tlb_ctl; 14028fc78909SEmanuele Giuseppe Esposito dst->int_ctl = from->int_ctl; 14038fc78909SEmanuele Giuseppe Esposito dst->int_vector = from->int_vector; 14048fc78909SEmanuele Giuseppe Esposito dst->int_state = from->int_state; 14058fc78909SEmanuele Giuseppe Esposito dst->exit_code = from->exit_code; 14068fc78909SEmanuele Giuseppe Esposito dst->exit_code_hi = from->exit_code_hi; 14078fc78909SEmanuele Giuseppe Esposito dst->exit_info_1 = from->exit_info_1; 14088fc78909SEmanuele Giuseppe Esposito dst->exit_info_2 = from->exit_info_2; 14098fc78909SEmanuele Giuseppe Esposito dst->exit_int_info = from->exit_int_info; 14108fc78909SEmanuele Giuseppe Esposito dst->exit_int_info_err = from->exit_int_info_err; 14118fc78909SEmanuele Giuseppe Esposito dst->nested_ctl = from->nested_ctl; 14128fc78909SEmanuele Giuseppe Esposito dst->event_inj = from->event_inj; 14138fc78909SEmanuele Giuseppe Esposito dst->event_inj_err = from->event_inj_err; 14148fc78909SEmanuele Giuseppe Esposito dst->nested_cr3 = from->nested_cr3; 14158fc78909SEmanuele Giuseppe Esposito dst->virt_ext = from->virt_ext; 14168fc78909SEmanuele Giuseppe Esposito dst->pause_filter_count = from->pause_filter_count; 14178fc78909SEmanuele Giuseppe Esposito dst->pause_filter_thresh = from->pause_filter_thresh; 141866c03a92SVitaly Kuznetsov /* 'clean' and 'reserved_sw' are not changed by KVM */ 14198fc78909SEmanuele Giuseppe Esposito } 14208fc78909SEmanuele Giuseppe Esposito 1421cc440cdaSPaolo Bonzini static int svm_get_nested_state(struct kvm_vcpu *vcpu, 1422cc440cdaSPaolo Bonzini struct kvm_nested_state __user *user_kvm_nested_state, 1423cc440cdaSPaolo Bonzini u32 user_data_size) 1424cc440cdaSPaolo Bonzini { 1425cc440cdaSPaolo Bonzini struct vcpu_svm *svm; 14268fc78909SEmanuele Giuseppe Esposito struct vmcb_control_area *ctl; 14278fc78909SEmanuele Giuseppe Esposito unsigned long r; 1428cc440cdaSPaolo Bonzini struct kvm_nested_state kvm_state = { 1429cc440cdaSPaolo Bonzini .flags = 0, 1430cc440cdaSPaolo Bonzini .format = KVM_STATE_NESTED_FORMAT_SVM, 1431cc440cdaSPaolo Bonzini .size = sizeof(kvm_state), 1432cc440cdaSPaolo Bonzini }; 1433cc440cdaSPaolo Bonzini struct vmcb __user *user_vmcb = (struct vmcb __user *) 1434cc440cdaSPaolo Bonzini &user_kvm_nested_state->data.svm[0]; 1435cc440cdaSPaolo Bonzini 1436cc440cdaSPaolo Bonzini if (!vcpu) 1437cc440cdaSPaolo Bonzini return kvm_state.size + KVM_STATE_NESTED_SVM_VMCB_SIZE; 1438cc440cdaSPaolo Bonzini 1439cc440cdaSPaolo Bonzini svm = to_svm(vcpu); 1440cc440cdaSPaolo Bonzini 1441cc440cdaSPaolo Bonzini if (user_data_size < kvm_state.size) 1442cc440cdaSPaolo Bonzini goto out; 1443cc440cdaSPaolo Bonzini 1444cc440cdaSPaolo Bonzini /* First fill in the header and copy it out. */ 1445cc440cdaSPaolo Bonzini if (is_guest_mode(vcpu)) { 14460dd16b5bSMaxim Levitsky kvm_state.hdr.svm.vmcb_pa = svm->nested.vmcb12_gpa; 1447cc440cdaSPaolo Bonzini kvm_state.size += KVM_STATE_NESTED_SVM_VMCB_SIZE; 1448cc440cdaSPaolo Bonzini kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE; 1449cc440cdaSPaolo Bonzini 1450cc440cdaSPaolo Bonzini if (svm->nested.nested_run_pending) 1451cc440cdaSPaolo Bonzini kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING; 1452cc440cdaSPaolo Bonzini } 1453cc440cdaSPaolo Bonzini 1454cc440cdaSPaolo Bonzini if (gif_set(svm)) 1455cc440cdaSPaolo Bonzini kvm_state.flags |= KVM_STATE_NESTED_GIF_SET; 1456cc440cdaSPaolo Bonzini 1457cc440cdaSPaolo Bonzini if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state))) 1458cc440cdaSPaolo Bonzini return -EFAULT; 1459cc440cdaSPaolo Bonzini 1460cc440cdaSPaolo Bonzini if (!is_guest_mode(vcpu)) 1461cc440cdaSPaolo Bonzini goto out; 1462cc440cdaSPaolo Bonzini 1463cc440cdaSPaolo Bonzini /* 1464cc440cdaSPaolo Bonzini * Copy over the full size of the VMCB rather than just the size 1465cc440cdaSPaolo Bonzini * of the structs. 1466cc440cdaSPaolo Bonzini */ 1467cc440cdaSPaolo Bonzini if (clear_user(user_vmcb, KVM_STATE_NESTED_SVM_VMCB_SIZE)) 1468cc440cdaSPaolo Bonzini return -EFAULT; 14698fc78909SEmanuele Giuseppe Esposito 14708fc78909SEmanuele Giuseppe Esposito ctl = kzalloc(sizeof(*ctl), GFP_KERNEL); 14718fc78909SEmanuele Giuseppe Esposito if (!ctl) 14728fc78909SEmanuele Giuseppe Esposito return -ENOMEM; 14738fc78909SEmanuele Giuseppe Esposito 14748fc78909SEmanuele Giuseppe Esposito nested_copy_vmcb_cache_to_control(ctl, &svm->nested.ctl); 14758fc78909SEmanuele Giuseppe Esposito r = copy_to_user(&user_vmcb->control, ctl, 14768fc78909SEmanuele Giuseppe Esposito sizeof(user_vmcb->control)); 14778fc78909SEmanuele Giuseppe Esposito kfree(ctl); 14788fc78909SEmanuele Giuseppe Esposito if (r) 1479cc440cdaSPaolo Bonzini return -EFAULT; 14808fc78909SEmanuele Giuseppe Esposito 14814995a368SCathy Avery if (copy_to_user(&user_vmcb->save, &svm->vmcb01.ptr->save, 1482cc440cdaSPaolo Bonzini sizeof(user_vmcb->save))) 1483cc440cdaSPaolo Bonzini return -EFAULT; 1484cc440cdaSPaolo Bonzini out: 1485cc440cdaSPaolo Bonzini return kvm_state.size; 1486cc440cdaSPaolo Bonzini } 1487cc440cdaSPaolo Bonzini 1488cc440cdaSPaolo Bonzini static int svm_set_nested_state(struct kvm_vcpu *vcpu, 1489cc440cdaSPaolo Bonzini struct kvm_nested_state __user *user_kvm_nested_state, 1490cc440cdaSPaolo Bonzini struct kvm_nested_state *kvm_state) 1491cc440cdaSPaolo Bonzini { 1492cc440cdaSPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 1493cc440cdaSPaolo Bonzini struct vmcb __user *user_vmcb = (struct vmcb __user *) 1494cc440cdaSPaolo Bonzini &user_kvm_nested_state->data.svm[0]; 14956ccbd29aSJoerg Roedel struct vmcb_control_area *ctl; 14966ccbd29aSJoerg Roedel struct vmcb_save_area *save; 1497b7a3d8b6SEmanuele Giuseppe Esposito struct vmcb_save_area_cached save_cached; 14988fc78909SEmanuele Giuseppe Esposito struct vmcb_ctrl_area_cached ctl_cached; 1499dbc4739bSSean Christopherson unsigned long cr0; 15006ccbd29aSJoerg Roedel int ret; 1501cc440cdaSPaolo Bonzini 15026ccbd29aSJoerg Roedel BUILD_BUG_ON(sizeof(struct vmcb_control_area) + sizeof(struct vmcb_save_area) > 15036ccbd29aSJoerg Roedel KVM_STATE_NESTED_SVM_VMCB_SIZE); 15046ccbd29aSJoerg Roedel 1505cc440cdaSPaolo Bonzini if (kvm_state->format != KVM_STATE_NESTED_FORMAT_SVM) 1506cc440cdaSPaolo Bonzini return -EINVAL; 1507cc440cdaSPaolo Bonzini 1508cc440cdaSPaolo Bonzini if (kvm_state->flags & ~(KVM_STATE_NESTED_GUEST_MODE | 1509cc440cdaSPaolo Bonzini KVM_STATE_NESTED_RUN_PENDING | 1510cc440cdaSPaolo Bonzini KVM_STATE_NESTED_GIF_SET)) 1511cc440cdaSPaolo Bonzini return -EINVAL; 1512cc440cdaSPaolo Bonzini 1513cc440cdaSPaolo Bonzini /* 1514cc440cdaSPaolo Bonzini * If in guest mode, vcpu->arch.efer actually refers to the L2 guest's 1515cc440cdaSPaolo Bonzini * EFER.SVME, but EFER.SVME still has to be 1 for VMRUN to succeed. 1516cc440cdaSPaolo Bonzini */ 1517cc440cdaSPaolo Bonzini if (!(vcpu->arch.efer & EFER_SVME)) { 1518cc440cdaSPaolo Bonzini /* GIF=1 and no guest mode are required if SVME=0. */ 1519cc440cdaSPaolo Bonzini if (kvm_state->flags != KVM_STATE_NESTED_GIF_SET) 1520cc440cdaSPaolo Bonzini return -EINVAL; 1521cc440cdaSPaolo Bonzini } 1522cc440cdaSPaolo Bonzini 1523cc440cdaSPaolo Bonzini /* SMM temporarily disables SVM, so we cannot be in guest mode. */ 1524cc440cdaSPaolo Bonzini if (is_smm(vcpu) && (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 1525cc440cdaSPaolo Bonzini return -EINVAL; 1526cc440cdaSPaolo Bonzini 1527cc440cdaSPaolo Bonzini if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) { 1528f7e57078SSean Christopherson svm_leave_nested(vcpu); 1529d5cd6f34SVitaly Kuznetsov svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET)); 1530d5cd6f34SVitaly Kuznetsov return 0; 1531cc440cdaSPaolo Bonzini } 1532cc440cdaSPaolo Bonzini 1533cc440cdaSPaolo Bonzini if (!page_address_valid(vcpu, kvm_state->hdr.svm.vmcb_pa)) 1534cc440cdaSPaolo Bonzini return -EINVAL; 1535cc440cdaSPaolo Bonzini if (kvm_state->size < sizeof(*kvm_state) + KVM_STATE_NESTED_SVM_VMCB_SIZE) 1536cc440cdaSPaolo Bonzini return -EINVAL; 1537cc440cdaSPaolo Bonzini 15386ccbd29aSJoerg Roedel ret = -ENOMEM; 1539eba04b20SSean Christopherson ctl = kzalloc(sizeof(*ctl), GFP_KERNEL_ACCOUNT); 1540eba04b20SSean Christopherson save = kzalloc(sizeof(*save), GFP_KERNEL_ACCOUNT); 15416ccbd29aSJoerg Roedel if (!ctl || !save) 15426ccbd29aSJoerg Roedel goto out_free; 15436ccbd29aSJoerg Roedel 15446ccbd29aSJoerg Roedel ret = -EFAULT; 15456ccbd29aSJoerg Roedel if (copy_from_user(ctl, &user_vmcb->control, sizeof(*ctl))) 15466ccbd29aSJoerg Roedel goto out_free; 15476ccbd29aSJoerg Roedel if (copy_from_user(save, &user_vmcb->save, sizeof(*save))) 15486ccbd29aSJoerg Roedel goto out_free; 15496ccbd29aSJoerg Roedel 15506ccbd29aSJoerg Roedel ret = -EINVAL; 155166c03a92SVitaly Kuznetsov __nested_copy_vmcb_control_to_cache(vcpu, &ctl_cached, ctl); 15528fc78909SEmanuele Giuseppe Esposito if (!__nested_vmcb_check_controls(vcpu, &ctl_cached)) 15536ccbd29aSJoerg Roedel goto out_free; 1554cc440cdaSPaolo Bonzini 1555cc440cdaSPaolo Bonzini /* 1556cc440cdaSPaolo Bonzini * Processor state contains L2 state. Check that it is 1557cb9b6a1bSPaolo Bonzini * valid for guest mode (see nested_vmcb_check_save). 1558cc440cdaSPaolo Bonzini */ 1559cc440cdaSPaolo Bonzini cr0 = kvm_read_cr0(vcpu); 1560cc440cdaSPaolo Bonzini if (((cr0 & X86_CR0_CD) == 0) && (cr0 & X86_CR0_NW)) 15616ccbd29aSJoerg Roedel goto out_free; 1562cc440cdaSPaolo Bonzini 1563cc440cdaSPaolo Bonzini /* 1564cc440cdaSPaolo Bonzini * Validate host state saved from before VMRUN (see 1565cc440cdaSPaolo Bonzini * nested_svm_check_permissions). 1566cc440cdaSPaolo Bonzini */ 1567b7a3d8b6SEmanuele Giuseppe Esposito __nested_copy_vmcb_save_to_cache(&save_cached, save); 15686906e06dSKrish Sadhukhan if (!(save->cr0 & X86_CR0_PG) || 15696906e06dSKrish Sadhukhan !(save->cr0 & X86_CR0_PE) || 15706906e06dSKrish Sadhukhan (save->rflags & X86_EFLAGS_VM) || 1571b7a3d8b6SEmanuele Giuseppe Esposito !__nested_vmcb_check_save(vcpu, &save_cached)) 15726ccbd29aSJoerg Roedel goto out_free; 1573cc440cdaSPaolo Bonzini 1574b222b0b8SMaxim Levitsky 1575b222b0b8SMaxim Levitsky /* 15764995a368SCathy Avery * All checks done, we can enter guest mode. Userspace provides 15774995a368SCathy Avery * vmcb12.control, which will be combined with L1 and stored into 15784995a368SCathy Avery * vmcb02, and the L1 save state which we store in vmcb01. 15794995a368SCathy Avery * L2 registers if needed are moved from the current VMCB to VMCB02. 1580cc440cdaSPaolo Bonzini */ 158181f76adaSMaxim Levitsky 15829d290e16SMaxim Levitsky if (is_guest_mode(vcpu)) 1583f7e57078SSean Christopherson svm_leave_nested(vcpu); 15849d290e16SMaxim Levitsky else 15859d290e16SMaxim Levitsky svm->nested.vmcb02.ptr->save = svm->vmcb01.ptr->save; 15869d290e16SMaxim Levitsky 1587063ab16cSMaxim Levitsky svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET)); 1588063ab16cSMaxim Levitsky 158981f76adaSMaxim Levitsky svm->nested.nested_run_pending = 159081f76adaSMaxim Levitsky !!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING); 159181f76adaSMaxim Levitsky 15920dd16b5bSMaxim Levitsky svm->nested.vmcb12_gpa = kvm_state->hdr.svm.vmcb_pa; 1593c08f390aSPaolo Bonzini 15942bb16beaSVitaly Kuznetsov svm_copy_vmrun_state(&svm->vmcb01.ptr->save, save); 15957907160dSEmanuele Giuseppe Esposito nested_copy_vmcb_control_to_cache(svm, ctl); 15964995a368SCathy Avery 15974995a368SCathy Avery svm_switch_vmcb(svm, &svm->nested.vmcb02); 15989e8f0fbfSPaolo Bonzini nested_vmcb02_prepare_control(svm); 1599e1779c27SMaxim Levitsky 1600e1779c27SMaxim Levitsky /* 1601e1779c27SMaxim Levitsky * While the nested guest CR3 is already checked and set by 1602e1779c27SMaxim Levitsky * KVM_SET_SREGS, it was set when nested state was yet loaded, 1603e1779c27SMaxim Levitsky * thus MMU might not be initialized correctly. 1604e1779c27SMaxim Levitsky * Set it again to fix this. 1605e1779c27SMaxim Levitsky */ 1606e1779c27SMaxim Levitsky 1607e1779c27SMaxim Levitsky ret = nested_svm_load_cr3(&svm->vcpu, vcpu->arch.cr3, 1608e1779c27SMaxim Levitsky nested_npt_enabled(svm), false); 1609e1779c27SMaxim Levitsky if (WARN_ON_ONCE(ret)) 1610e1779c27SMaxim Levitsky goto out_free; 1611e1779c27SMaxim Levitsky 161273c25546SVitaly Kuznetsov svm->nested.force_msr_bitmap_recalc = true; 1613e1779c27SMaxim Levitsky 1614a7d5c7ceSPaolo Bonzini kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 16156ccbd29aSJoerg Roedel ret = 0; 16166ccbd29aSJoerg Roedel out_free: 16176ccbd29aSJoerg Roedel kfree(save); 16186ccbd29aSJoerg Roedel kfree(ctl); 16196ccbd29aSJoerg Roedel 16206ccbd29aSJoerg Roedel return ret; 1621cc440cdaSPaolo Bonzini } 1622cc440cdaSPaolo Bonzini 1623232f75d3SMaxim Levitsky static bool svm_get_nested_state_pages(struct kvm_vcpu *vcpu) 1624232f75d3SMaxim Levitsky { 1625232f75d3SMaxim Levitsky struct vcpu_svm *svm = to_svm(vcpu); 1626232f75d3SMaxim Levitsky 1627232f75d3SMaxim Levitsky if (WARN_ON(!is_guest_mode(vcpu))) 1628232f75d3SMaxim Levitsky return true; 1629232f75d3SMaxim Levitsky 1630158a48ecSMaxim Levitsky if (!vcpu->arch.pdptrs_from_userspace && 1631158a48ecSMaxim Levitsky !nested_npt_enabled(svm) && is_pae_paging(vcpu)) 1632b222b0b8SMaxim Levitsky /* 1633b222b0b8SMaxim Levitsky * Reload the guest's PDPTRs since after a migration 1634b222b0b8SMaxim Levitsky * the guest CR3 might be restored prior to setting the nested 1635b222b0b8SMaxim Levitsky * state which can lead to a load of wrong PDPTRs. 1636b222b0b8SMaxim Levitsky */ 16372df4a5ebSLai Jiangshan if (CC(!load_pdptrs(vcpu, vcpu->arch.cr3))) 1638232f75d3SMaxim Levitsky return false; 1639232f75d3SMaxim Levitsky 1640232f75d3SMaxim Levitsky if (!nested_svm_vmrun_msrpm(svm)) { 1641232f75d3SMaxim Levitsky vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1642232f75d3SMaxim Levitsky vcpu->run->internal.suberror = 1643232f75d3SMaxim Levitsky KVM_INTERNAL_ERROR_EMULATION; 1644232f75d3SMaxim Levitsky vcpu->run->internal.ndata = 0; 1645232f75d3SMaxim Levitsky return false; 1646232f75d3SMaxim Levitsky } 1647232f75d3SMaxim Levitsky 1648232f75d3SMaxim Levitsky return true; 1649232f75d3SMaxim Levitsky } 1650232f75d3SMaxim Levitsky 165133b22172SPaolo Bonzini struct kvm_x86_nested_ops svm_nested_ops = { 1652f7e57078SSean Christopherson .leave_nested = svm_leave_nested, 165333b22172SPaolo Bonzini .check_events = svm_check_nested_events, 1654cb6a32c2SSean Christopherson .triple_fault = nested_svm_triple_fault, 1655a7d5c7ceSPaolo Bonzini .get_nested_state_pages = svm_get_nested_state_pages, 1656cc440cdaSPaolo Bonzini .get_state = svm_get_nested_state, 1657cc440cdaSPaolo Bonzini .set_state = svm_set_nested_state, 165833b22172SPaolo Bonzini }; 1659