1883b0a91SJoerg Roedel // SPDX-License-Identifier: GPL-2.0-only 2883b0a91SJoerg Roedel /* 3883b0a91SJoerg Roedel * Kernel-based Virtual Machine driver for Linux 4883b0a91SJoerg Roedel * 5883b0a91SJoerg Roedel * AMD SVM support 6883b0a91SJoerg Roedel * 7883b0a91SJoerg Roedel * Copyright (C) 2006 Qumranet, Inc. 8883b0a91SJoerg Roedel * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9883b0a91SJoerg Roedel * 10883b0a91SJoerg Roedel * Authors: 11883b0a91SJoerg Roedel * Yaniv Kamay <yaniv@qumranet.com> 12883b0a91SJoerg Roedel * Avi Kivity <avi@qumranet.com> 13883b0a91SJoerg Roedel */ 14883b0a91SJoerg Roedel 15883b0a91SJoerg Roedel #define pr_fmt(fmt) "SVM: " fmt 16883b0a91SJoerg Roedel 17883b0a91SJoerg Roedel #include <linux/kvm_types.h> 18883b0a91SJoerg Roedel #include <linux/kvm_host.h> 19883b0a91SJoerg Roedel #include <linux/kernel.h> 20883b0a91SJoerg Roedel 21883b0a91SJoerg Roedel #include <asm/msr-index.h> 225679b803SPaolo Bonzini #include <asm/debugreg.h> 23883b0a91SJoerg Roedel 24883b0a91SJoerg Roedel #include "kvm_emulate.h" 25883b0a91SJoerg Roedel #include "trace.h" 26883b0a91SJoerg Roedel #include "mmu.h" 27883b0a91SJoerg Roedel #include "x86.h" 28cc440cdaSPaolo Bonzini #include "cpuid.h" 295b672408SPaolo Bonzini #include "lapic.h" 30883b0a91SJoerg Roedel #include "svm.h" 31883b0a91SJoerg Roedel 3211f0cbf0SSean Christopherson #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK 3311f0cbf0SSean Christopherson 34883b0a91SJoerg Roedel static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu, 35883b0a91SJoerg Roedel struct x86_exception *fault) 36883b0a91SJoerg Roedel { 37883b0a91SJoerg Roedel struct vcpu_svm *svm = to_svm(vcpu); 38883b0a91SJoerg Roedel 39883b0a91SJoerg Roedel if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) { 40883b0a91SJoerg Roedel /* 41883b0a91SJoerg Roedel * TODO: track the cause of the nested page fault, and 42883b0a91SJoerg Roedel * correctly fill in the high bits of exit_info_1. 43883b0a91SJoerg Roedel */ 44883b0a91SJoerg Roedel svm->vmcb->control.exit_code = SVM_EXIT_NPF; 45883b0a91SJoerg Roedel svm->vmcb->control.exit_code_hi = 0; 46883b0a91SJoerg Roedel svm->vmcb->control.exit_info_1 = (1ULL << 32); 47883b0a91SJoerg Roedel svm->vmcb->control.exit_info_2 = fault->address; 48883b0a91SJoerg Roedel } 49883b0a91SJoerg Roedel 50883b0a91SJoerg Roedel svm->vmcb->control.exit_info_1 &= ~0xffffffffULL; 51883b0a91SJoerg Roedel svm->vmcb->control.exit_info_1 |= fault->error_code; 52883b0a91SJoerg Roedel 53883b0a91SJoerg Roedel nested_svm_vmexit(svm); 54883b0a91SJoerg Roedel } 55883b0a91SJoerg Roedel 56a04aead1SPaolo Bonzini static void svm_inject_page_fault_nested(struct kvm_vcpu *vcpu, struct x86_exception *fault) 57a04aead1SPaolo Bonzini { 58a04aead1SPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 59a04aead1SPaolo Bonzini WARN_ON(!is_guest_mode(vcpu)); 60a04aead1SPaolo Bonzini 61a04aead1SPaolo Bonzini if (vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_EXCEPTION_OFFSET + PF_VECTOR) && 62a04aead1SPaolo Bonzini !svm->nested.nested_run_pending) { 63a04aead1SPaolo Bonzini svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + PF_VECTOR; 64a04aead1SPaolo Bonzini svm->vmcb->control.exit_code_hi = 0; 65a04aead1SPaolo Bonzini svm->vmcb->control.exit_info_1 = fault->error_code; 66a04aead1SPaolo Bonzini svm->vmcb->control.exit_info_2 = fault->address; 67a04aead1SPaolo Bonzini nested_svm_vmexit(svm); 68a04aead1SPaolo Bonzini } else { 69a04aead1SPaolo Bonzini kvm_inject_page_fault(vcpu, fault); 70a04aead1SPaolo Bonzini } 71a04aead1SPaolo Bonzini } 72a04aead1SPaolo Bonzini 73883b0a91SJoerg Roedel static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index) 74883b0a91SJoerg Roedel { 75883b0a91SJoerg Roedel struct vcpu_svm *svm = to_svm(vcpu); 76e670bf68SPaolo Bonzini u64 cr3 = svm->nested.ctl.nested_cr3; 77883b0a91SJoerg Roedel u64 pdpte; 78883b0a91SJoerg Roedel int ret; 79883b0a91SJoerg Roedel 802732be90SSean Christopherson ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(cr3), &pdpte, 81883b0a91SJoerg Roedel offset_in_page(cr3) + index * 8, 8); 82883b0a91SJoerg Roedel if (ret) 83883b0a91SJoerg Roedel return 0; 84883b0a91SJoerg Roedel return pdpte; 85883b0a91SJoerg Roedel } 86883b0a91SJoerg Roedel 87883b0a91SJoerg Roedel static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu) 88883b0a91SJoerg Roedel { 89883b0a91SJoerg Roedel struct vcpu_svm *svm = to_svm(vcpu); 90883b0a91SJoerg Roedel 91e670bf68SPaolo Bonzini return svm->nested.ctl.nested_cr3; 92883b0a91SJoerg Roedel } 93883b0a91SJoerg Roedel 94883b0a91SJoerg Roedel static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu) 95883b0a91SJoerg Roedel { 96929d1cfaSPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 97929d1cfaSPaolo Bonzini 98883b0a91SJoerg Roedel WARN_ON(mmu_is_nested(vcpu)); 99883b0a91SJoerg Roedel 100883b0a91SJoerg Roedel vcpu->arch.mmu = &vcpu->arch.guest_mmu; 1014995a368SCathy Avery kvm_init_shadow_npt_mmu(vcpu, X86_CR0_PG, svm->vmcb01.ptr->save.cr4, 1024995a368SCathy Avery svm->vmcb01.ptr->save.efer, 1030f04a2acSVitaly Kuznetsov svm->nested.ctl.nested_cr3); 104883b0a91SJoerg Roedel vcpu->arch.mmu->get_guest_pgd = nested_svm_get_tdp_cr3; 105883b0a91SJoerg Roedel vcpu->arch.mmu->get_pdptr = nested_svm_get_tdp_pdptr; 106883b0a91SJoerg Roedel vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit; 107883b0a91SJoerg Roedel reset_shadow_zero_bits_mask(vcpu, vcpu->arch.mmu); 108883b0a91SJoerg Roedel vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; 109883b0a91SJoerg Roedel } 110883b0a91SJoerg Roedel 111883b0a91SJoerg Roedel static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu) 112883b0a91SJoerg Roedel { 113883b0a91SJoerg Roedel vcpu->arch.mmu = &vcpu->arch.root_mmu; 114883b0a91SJoerg Roedel vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; 115883b0a91SJoerg Roedel } 116883b0a91SJoerg Roedel 117883b0a91SJoerg Roedel void recalc_intercepts(struct vcpu_svm *svm) 118883b0a91SJoerg Roedel { 119e670bf68SPaolo Bonzini struct vmcb_control_area *c, *h, *g; 120c45ad722SBabu Moger unsigned int i; 121883b0a91SJoerg Roedel 12206e7852cSJoerg Roedel vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 123883b0a91SJoerg Roedel 124883b0a91SJoerg Roedel if (!is_guest_mode(&svm->vcpu)) 125883b0a91SJoerg Roedel return; 126883b0a91SJoerg Roedel 127883b0a91SJoerg Roedel c = &svm->vmcb->control; 1284995a368SCathy Avery h = &svm->vmcb01.ptr->control; 129e670bf68SPaolo Bonzini g = &svm->nested.ctl; 130883b0a91SJoerg Roedel 131c45ad722SBabu Moger for (i = 0; i < MAX_INTERCEPT; i++) 132c45ad722SBabu Moger c->intercepts[i] = h->intercepts[i]; 133c45ad722SBabu Moger 134e9fd761aSPaolo Bonzini if (g->int_ctl & V_INTR_MASKING_MASK) { 135883b0a91SJoerg Roedel /* We only want the cr8 intercept bits of L1 */ 13603bfeeb9SBabu Moger vmcb_clr_intercept(c, INTERCEPT_CR8_READ); 13703bfeeb9SBabu Moger vmcb_clr_intercept(c, INTERCEPT_CR8_WRITE); 138883b0a91SJoerg Roedel 139883b0a91SJoerg Roedel /* 140883b0a91SJoerg Roedel * Once running L2 with HF_VINTR_MASK, EFLAGS.IF does not 141883b0a91SJoerg Roedel * affect any interrupt we may want to inject; therefore, 142883b0a91SJoerg Roedel * interrupt window vmexits are irrelevant to L0. 143883b0a91SJoerg Roedel */ 144c62e2e94SBabu Moger vmcb_clr_intercept(c, INTERCEPT_VINTR); 145883b0a91SJoerg Roedel } 146883b0a91SJoerg Roedel 147883b0a91SJoerg Roedel /* We don't want to see VMMCALLs from a nested guest */ 148c62e2e94SBabu Moger vmcb_clr_intercept(c, INTERCEPT_VMMCALL); 149883b0a91SJoerg Roedel 150c45ad722SBabu Moger for (i = 0; i < MAX_INTERCEPT; i++) 151c45ad722SBabu Moger c->intercepts[i] |= g->intercepts[i]; 152883b0a91SJoerg Roedel } 153883b0a91SJoerg Roedel 1542f675917SPaolo Bonzini static void copy_vmcb_control_area(struct vmcb_control_area *dst, 1552f675917SPaolo Bonzini struct vmcb_control_area *from) 156883b0a91SJoerg Roedel { 157c45ad722SBabu Moger unsigned int i; 158c45ad722SBabu Moger 159c45ad722SBabu Moger for (i = 0; i < MAX_INTERCEPT; i++) 160c45ad722SBabu Moger dst->intercepts[i] = from->intercepts[i]; 161c45ad722SBabu Moger 162883b0a91SJoerg Roedel dst->iopm_base_pa = from->iopm_base_pa; 163883b0a91SJoerg Roedel dst->msrpm_base_pa = from->msrpm_base_pa; 164883b0a91SJoerg Roedel dst->tsc_offset = from->tsc_offset; 1656c0238c4SPaolo Bonzini /* asid not copied, it is handled manually for svm->vmcb. */ 166883b0a91SJoerg Roedel dst->tlb_ctl = from->tlb_ctl; 167883b0a91SJoerg Roedel dst->int_ctl = from->int_ctl; 168883b0a91SJoerg Roedel dst->int_vector = from->int_vector; 169883b0a91SJoerg Roedel dst->int_state = from->int_state; 170883b0a91SJoerg Roedel dst->exit_code = from->exit_code; 171883b0a91SJoerg Roedel dst->exit_code_hi = from->exit_code_hi; 172883b0a91SJoerg Roedel dst->exit_info_1 = from->exit_info_1; 173883b0a91SJoerg Roedel dst->exit_info_2 = from->exit_info_2; 174883b0a91SJoerg Roedel dst->exit_int_info = from->exit_int_info; 175883b0a91SJoerg Roedel dst->exit_int_info_err = from->exit_int_info_err; 176883b0a91SJoerg Roedel dst->nested_ctl = from->nested_ctl; 177883b0a91SJoerg Roedel dst->event_inj = from->event_inj; 178883b0a91SJoerg Roedel dst->event_inj_err = from->event_inj_err; 179883b0a91SJoerg Roedel dst->nested_cr3 = from->nested_cr3; 180883b0a91SJoerg Roedel dst->virt_ext = from->virt_ext; 181883b0a91SJoerg Roedel dst->pause_filter_count = from->pause_filter_count; 182883b0a91SJoerg Roedel dst->pause_filter_thresh = from->pause_filter_thresh; 183883b0a91SJoerg Roedel } 184883b0a91SJoerg Roedel 185883b0a91SJoerg Roedel static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm) 186883b0a91SJoerg Roedel { 187883b0a91SJoerg Roedel /* 188883b0a91SJoerg Roedel * This function merges the msr permission bitmaps of kvm and the 189883b0a91SJoerg Roedel * nested vmcb. It is optimized in that it only merges the parts where 190883b0a91SJoerg Roedel * the kvm msr permission bitmap may contain zero bits 191883b0a91SJoerg Roedel */ 192883b0a91SJoerg Roedel int i; 193883b0a91SJoerg Roedel 194c62e2e94SBabu Moger if (!(vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT))) 195883b0a91SJoerg Roedel return true; 196883b0a91SJoerg Roedel 197883b0a91SJoerg Roedel for (i = 0; i < MSRPM_OFFSETS; i++) { 198883b0a91SJoerg Roedel u32 value, p; 199883b0a91SJoerg Roedel u64 offset; 200883b0a91SJoerg Roedel 201883b0a91SJoerg Roedel if (msrpm_offsets[i] == 0xffffffff) 202883b0a91SJoerg Roedel break; 203883b0a91SJoerg Roedel 204883b0a91SJoerg Roedel p = msrpm_offsets[i]; 205e670bf68SPaolo Bonzini offset = svm->nested.ctl.msrpm_base_pa + (p * 4); 206883b0a91SJoerg Roedel 207883b0a91SJoerg Roedel if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4)) 208883b0a91SJoerg Roedel return false; 209883b0a91SJoerg Roedel 210883b0a91SJoerg Roedel svm->nested.msrpm[p] = svm->msrpm[p] | value; 211883b0a91SJoerg Roedel } 212883b0a91SJoerg Roedel 213883b0a91SJoerg Roedel svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm)); 214883b0a91SJoerg Roedel 215883b0a91SJoerg Roedel return true; 216883b0a91SJoerg Roedel } 217883b0a91SJoerg Roedel 218ee695f22SKrish Sadhukhan /* 219ee695f22SKrish Sadhukhan * Bits 11:0 of bitmap address are ignored by hardware 220ee695f22SKrish Sadhukhan */ 221ee695f22SKrish Sadhukhan static bool nested_svm_check_bitmap_pa(struct kvm_vcpu *vcpu, u64 pa, u32 size) 222ee695f22SKrish Sadhukhan { 223ee695f22SKrish Sadhukhan u64 addr = PAGE_ALIGN(pa); 224ee695f22SKrish Sadhukhan 225ee695f22SKrish Sadhukhan return kvm_vcpu_is_legal_gpa(vcpu, addr) && 226ee695f22SKrish Sadhukhan kvm_vcpu_is_legal_gpa(vcpu, addr + size - 1); 227ee695f22SKrish Sadhukhan } 228ee695f22SKrish Sadhukhan 229ee695f22SKrish Sadhukhan static bool nested_vmcb_check_controls(struct kvm_vcpu *vcpu, 230ee695f22SKrish Sadhukhan struct vmcb_control_area *control) 231ca46d739SPaolo Bonzini { 23211f0cbf0SSean Christopherson if (CC(!vmcb_is_intercept(control, INTERCEPT_VMRUN))) 233ca46d739SPaolo Bonzini return false; 234ca46d739SPaolo Bonzini 23511f0cbf0SSean Christopherson if (CC(control->asid == 0)) 236ca46d739SPaolo Bonzini return false; 237ca46d739SPaolo Bonzini 23811f0cbf0SSean Christopherson if (CC((control->nested_ctl & SVM_NESTED_CTL_NP_ENABLE) && !npt_enabled)) 239ca46d739SPaolo Bonzini return false; 240ca46d739SPaolo Bonzini 241ee695f22SKrish Sadhukhan if (CC(!nested_svm_check_bitmap_pa(vcpu, control->msrpm_base_pa, 242ee695f22SKrish Sadhukhan MSRPM_SIZE))) 243ee695f22SKrish Sadhukhan return false; 244ee695f22SKrish Sadhukhan if (CC(!nested_svm_check_bitmap_pa(vcpu, control->iopm_base_pa, 245ee695f22SKrish Sadhukhan IOPM_SIZE))) 246ee695f22SKrish Sadhukhan return false; 247ee695f22SKrish Sadhukhan 248ca46d739SPaolo Bonzini return true; 249ca46d739SPaolo Bonzini } 250ca46d739SPaolo Bonzini 25163129754SPaolo Bonzini static bool nested_vmcb_check_cr3_cr4(struct kvm_vcpu *vcpu, 2526906e06dSKrish Sadhukhan struct vmcb_save_area *save) 253883b0a91SJoerg Roedel { 2546906e06dSKrish Sadhukhan /* 2556906e06dSKrish Sadhukhan * These checks are also performed by KVM_SET_SREGS, 2566906e06dSKrish Sadhukhan * except that EFER.LMA is not checked by SVM against 2576906e06dSKrish Sadhukhan * CR0.PG && EFER.LME. 2586906e06dSKrish Sadhukhan */ 2596906e06dSKrish Sadhukhan if ((save->efer & EFER_LME) && (save->cr0 & X86_CR0_PG)) { 26011f0cbf0SSean Christopherson if (CC(!(save->cr4 & X86_CR4_PAE)) || 26111f0cbf0SSean Christopherson CC(!(save->cr0 & X86_CR0_PE)) || 26211f0cbf0SSean Christopherson CC(kvm_vcpu_is_illegal_gpa(vcpu, save->cr3))) 263761e4169SKrish Sadhukhan return false; 264761e4169SKrish Sadhukhan } 2656906e06dSKrish Sadhukhan 26611f0cbf0SSean Christopherson if (CC(!kvm_is_valid_cr4(vcpu, save->cr4))) 26711f0cbf0SSean Christopherson return false; 26811f0cbf0SSean Christopherson 26911f0cbf0SSean Christopherson return true; 2706906e06dSKrish Sadhukhan } 2716906e06dSKrish Sadhukhan 2726906e06dSKrish Sadhukhan /* Common checks that apply to both L1 and L2 state. */ 27363129754SPaolo Bonzini static bool nested_vmcb_valid_sregs(struct kvm_vcpu *vcpu, 2746906e06dSKrish Sadhukhan struct vmcb_save_area *save) 2756906e06dSKrish Sadhukhan { 2763c346c0cSPaolo Bonzini /* 2773c346c0cSPaolo Bonzini * FIXME: these should be done after copying the fields, 2783c346c0cSPaolo Bonzini * to avoid TOC/TOU races. For these save area checks 2793c346c0cSPaolo Bonzini * the possible damage is limited since kvm_set_cr0 and 2803c346c0cSPaolo Bonzini * kvm_set_cr4 handle failure; EFER_SVME is an exception 2813c346c0cSPaolo Bonzini * so it is force-set later in nested_prepare_vmcb_save. 2823c346c0cSPaolo Bonzini */ 28311f0cbf0SSean Christopherson if (CC(!(save->efer & EFER_SVME))) 2846906e06dSKrish Sadhukhan return false; 2856906e06dSKrish Sadhukhan 28611f0cbf0SSean Christopherson if (CC((save->cr0 & X86_CR0_CD) == 0 && (save->cr0 & X86_CR0_NW)) || 28711f0cbf0SSean Christopherson CC(save->cr0 & ~0xffffffffULL)) 2886906e06dSKrish Sadhukhan return false; 2896906e06dSKrish Sadhukhan 29011f0cbf0SSean Christopherson if (CC(!kvm_dr6_valid(save->dr6)) || CC(!kvm_dr7_valid(save->dr7))) 2916906e06dSKrish Sadhukhan return false; 2926906e06dSKrish Sadhukhan 29363129754SPaolo Bonzini if (!nested_vmcb_check_cr3_cr4(vcpu, save)) 2946906e06dSKrish Sadhukhan return false; 2956906e06dSKrish Sadhukhan 29663129754SPaolo Bonzini if (CC(!kvm_valid_efer(vcpu, save->efer))) 2976906e06dSKrish Sadhukhan return false; 2986906e06dSKrish Sadhukhan 2996906e06dSKrish Sadhukhan return true; 3006906e06dSKrish Sadhukhan } 3016906e06dSKrish Sadhukhan 3029e8f0fbfSPaolo Bonzini static void nested_load_control_from_vmcb12(struct vcpu_svm *svm, 3033e06f016SPaolo Bonzini struct vmcb_control_area *control) 3043e06f016SPaolo Bonzini { 305e670bf68SPaolo Bonzini copy_vmcb_control_area(&svm->nested.ctl, control); 3063e06f016SPaolo Bonzini 307cc440cdaSPaolo Bonzini /* Copy it here because nested_svm_check_controls will check it. */ 308cc440cdaSPaolo Bonzini svm->nested.ctl.asid = control->asid; 309e670bf68SPaolo Bonzini svm->nested.ctl.msrpm_base_pa &= ~0x0fffULL; 310e670bf68SPaolo Bonzini svm->nested.ctl.iopm_base_pa &= ~0x0fffULL; 3113e06f016SPaolo Bonzini } 3123e06f016SPaolo Bonzini 3132d8a42beSPaolo Bonzini /* 3142d8a42beSPaolo Bonzini * Synchronize fields that are written by the processor, so that 3159e8f0fbfSPaolo Bonzini * they can be copied back into the vmcb12. 3162d8a42beSPaolo Bonzini */ 3179e8f0fbfSPaolo Bonzini void nested_sync_control_from_vmcb02(struct vcpu_svm *svm) 3182d8a42beSPaolo Bonzini { 3192d8a42beSPaolo Bonzini u32 mask; 3202d8a42beSPaolo Bonzini svm->nested.ctl.event_inj = svm->vmcb->control.event_inj; 3212d8a42beSPaolo Bonzini svm->nested.ctl.event_inj_err = svm->vmcb->control.event_inj_err; 3222d8a42beSPaolo Bonzini 3232d8a42beSPaolo Bonzini /* Only a few fields of int_ctl are written by the processor. */ 3242d8a42beSPaolo Bonzini mask = V_IRQ_MASK | V_TPR_MASK; 3252d8a42beSPaolo Bonzini if (!(svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK) && 326a284ba56SJoerg Roedel svm_is_intercept(svm, INTERCEPT_VINTR)) { 3272d8a42beSPaolo Bonzini /* 3282d8a42beSPaolo Bonzini * In order to request an interrupt window, L0 is usurping 3292d8a42beSPaolo Bonzini * svm->vmcb->control.int_ctl and possibly setting V_IRQ 3302d8a42beSPaolo Bonzini * even if it was clear in L1's VMCB. Restoring it would be 3312d8a42beSPaolo Bonzini * wrong. However, in this case V_IRQ will remain true until 3322d8a42beSPaolo Bonzini * interrupt_window_interception calls svm_clear_vintr and 3332d8a42beSPaolo Bonzini * restores int_ctl. We can just leave it aside. 3342d8a42beSPaolo Bonzini */ 3352d8a42beSPaolo Bonzini mask &= ~V_IRQ_MASK; 3362d8a42beSPaolo Bonzini } 3372d8a42beSPaolo Bonzini svm->nested.ctl.int_ctl &= ~mask; 3382d8a42beSPaolo Bonzini svm->nested.ctl.int_ctl |= svm->vmcb->control.int_ctl & mask; 3392d8a42beSPaolo Bonzini } 3402d8a42beSPaolo Bonzini 34136e2e983SPaolo Bonzini /* 34236e2e983SPaolo Bonzini * Transfer any event that L0 or L1 wanted to inject into L2 to 34336e2e983SPaolo Bonzini * EXIT_INT_INFO. 34436e2e983SPaolo Bonzini */ 3459e8f0fbfSPaolo Bonzini static void nested_save_pending_event_to_vmcb12(struct vcpu_svm *svm, 3460dd16b5bSMaxim Levitsky struct vmcb *vmcb12) 34736e2e983SPaolo Bonzini { 34836e2e983SPaolo Bonzini struct kvm_vcpu *vcpu = &svm->vcpu; 34936e2e983SPaolo Bonzini u32 exit_int_info = 0; 35036e2e983SPaolo Bonzini unsigned int nr; 35136e2e983SPaolo Bonzini 35236e2e983SPaolo Bonzini if (vcpu->arch.exception.injected) { 35336e2e983SPaolo Bonzini nr = vcpu->arch.exception.nr; 35436e2e983SPaolo Bonzini exit_int_info = nr | SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_EXEPT; 35536e2e983SPaolo Bonzini 35636e2e983SPaolo Bonzini if (vcpu->arch.exception.has_error_code) { 35736e2e983SPaolo Bonzini exit_int_info |= SVM_EVTINJ_VALID_ERR; 3580dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info_err = 35936e2e983SPaolo Bonzini vcpu->arch.exception.error_code; 36036e2e983SPaolo Bonzini } 36136e2e983SPaolo Bonzini 36236e2e983SPaolo Bonzini } else if (vcpu->arch.nmi_injected) { 36336e2e983SPaolo Bonzini exit_int_info = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI; 36436e2e983SPaolo Bonzini 36536e2e983SPaolo Bonzini } else if (vcpu->arch.interrupt.injected) { 36636e2e983SPaolo Bonzini nr = vcpu->arch.interrupt.nr; 36736e2e983SPaolo Bonzini exit_int_info = nr | SVM_EVTINJ_VALID; 36836e2e983SPaolo Bonzini 36936e2e983SPaolo Bonzini if (vcpu->arch.interrupt.soft) 37036e2e983SPaolo Bonzini exit_int_info |= SVM_EVTINJ_TYPE_SOFT; 37136e2e983SPaolo Bonzini else 37236e2e983SPaolo Bonzini exit_int_info |= SVM_EVTINJ_TYPE_INTR; 37336e2e983SPaolo Bonzini } 37436e2e983SPaolo Bonzini 3750dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info = exit_int_info; 37636e2e983SPaolo Bonzini } 37736e2e983SPaolo Bonzini 37862156f6cSVitaly Kuznetsov static inline bool nested_npt_enabled(struct vcpu_svm *svm) 37962156f6cSVitaly Kuznetsov { 38062156f6cSVitaly Kuznetsov return svm->nested.ctl.nested_ctl & SVM_NESTED_CTL_NP_ENABLE; 38162156f6cSVitaly Kuznetsov } 38262156f6cSVitaly Kuznetsov 383d2e56019SSean Christopherson static void nested_svm_transition_tlb_flush(struct kvm_vcpu *vcpu) 384d2e56019SSean Christopherson { 385d2e56019SSean Christopherson /* 386d2e56019SSean Christopherson * TODO: optimize unconditional TLB flush/MMU sync. A partial list of 387d2e56019SSean Christopherson * things to fix before this can be conditional: 388d2e56019SSean Christopherson * 389d2e56019SSean Christopherson * - Flush TLBs for both L1 and L2 remote TLB flush 390d2e56019SSean Christopherson * - Honor L1's request to flush an ASID on nested VMRUN 391d2e56019SSean Christopherson * - Sync nested NPT MMU on VMRUN that flushes L2's ASID[*] 392d2e56019SSean Christopherson * - Don't crush a pending TLB flush in vmcb02 on nested VMRUN 393d2e56019SSean Christopherson * - Flush L1's ASID on KVM_REQ_TLB_FLUSH_GUEST 394d2e56019SSean Christopherson * 395d2e56019SSean Christopherson * [*] Unlike nested EPT, SVM's ASID management can invalidate nested 396d2e56019SSean Christopherson * NPT guest-physical mappings on VMRUN. 397d2e56019SSean Christopherson */ 398d2e56019SSean Christopherson kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); 399d2e56019SSean Christopherson kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 400d2e56019SSean Christopherson } 401d2e56019SSean Christopherson 40262156f6cSVitaly Kuznetsov /* 403d82aaef9SVitaly Kuznetsov * Load guest's/host's cr3 on nested vmentry or vmexit. @nested_npt is true 404d82aaef9SVitaly Kuznetsov * if we are emulating VM-Entry into a guest with NPT enabled. 40562156f6cSVitaly Kuznetsov */ 40662156f6cSVitaly Kuznetsov static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, 407b222b0b8SMaxim Levitsky bool nested_npt, bool reload_pdptrs) 40862156f6cSVitaly Kuznetsov { 40911f0cbf0SSean Christopherson if (CC(kvm_vcpu_is_illegal_gpa(vcpu, cr3))) 410a506fdd2SVitaly Kuznetsov return -EINVAL; 411a506fdd2SVitaly Kuznetsov 412b222b0b8SMaxim Levitsky if (reload_pdptrs && !nested_npt && is_pae_paging(vcpu) && 413a36dbec6SSean Christopherson CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))) 414a506fdd2SVitaly Kuznetsov return -EINVAL; 415a506fdd2SVitaly Kuznetsov 416a506fdd2SVitaly Kuznetsov if (!nested_npt) 417b5129100SSean Christopherson kvm_mmu_new_pgd(vcpu, cr3); 418a506fdd2SVitaly Kuznetsov 419a506fdd2SVitaly Kuznetsov vcpu->arch.cr3 = cr3; 420a506fdd2SVitaly Kuznetsov kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); 421a506fdd2SVitaly Kuznetsov 422c9060662SSean Christopherson kvm_init_mmu(vcpu); 423a506fdd2SVitaly Kuznetsov 424a506fdd2SVitaly Kuznetsov return 0; 42562156f6cSVitaly Kuznetsov } 42662156f6cSVitaly Kuznetsov 4274995a368SCathy Avery void nested_vmcb02_compute_g_pat(struct vcpu_svm *svm) 4284995a368SCathy Avery { 4294995a368SCathy Avery if (!svm->nested.vmcb02.ptr) 4304995a368SCathy Avery return; 4314995a368SCathy Avery 4324995a368SCathy Avery /* FIXME: merge g_pat from vmcb01 and vmcb12. */ 4334995a368SCathy Avery svm->nested.vmcb02.ptr->save.g_pat = svm->vmcb01.ptr->save.g_pat; 4344995a368SCathy Avery } 4354995a368SCathy Avery 4369e8f0fbfSPaolo Bonzini static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12) 437883b0a91SJoerg Roedel { 4388173396eSCathy Avery bool new_vmcb12 = false; 4398173396eSCathy Avery 4404995a368SCathy Avery nested_vmcb02_compute_g_pat(svm); 4414995a368SCathy Avery 442883b0a91SJoerg Roedel /* Load the nested guest state */ 4438173396eSCathy Avery if (svm->nested.vmcb12_gpa != svm->nested.last_vmcb12_gpa) { 4448173396eSCathy Avery new_vmcb12 = true; 4458173396eSCathy Avery svm->nested.last_vmcb12_gpa = svm->nested.vmcb12_gpa; 4468173396eSCathy Avery } 4478173396eSCathy Avery 4488173396eSCathy Avery if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_SEG))) { 4490dd16b5bSMaxim Levitsky svm->vmcb->save.es = vmcb12->save.es; 4500dd16b5bSMaxim Levitsky svm->vmcb->save.cs = vmcb12->save.cs; 4510dd16b5bSMaxim Levitsky svm->vmcb->save.ss = vmcb12->save.ss; 4520dd16b5bSMaxim Levitsky svm->vmcb->save.ds = vmcb12->save.ds; 4534bb170a5SPaolo Bonzini svm->vmcb->save.cpl = vmcb12->save.cpl; 4544bb170a5SPaolo Bonzini vmcb_mark_dirty(svm->vmcb, VMCB_SEG); 4558173396eSCathy Avery } 4564bb170a5SPaolo Bonzini 4578173396eSCathy Avery if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_DT))) { 4580dd16b5bSMaxim Levitsky svm->vmcb->save.gdtr = vmcb12->save.gdtr; 4590dd16b5bSMaxim Levitsky svm->vmcb->save.idtr = vmcb12->save.idtr; 4604bb170a5SPaolo Bonzini vmcb_mark_dirty(svm->vmcb, VMCB_DT); 4618173396eSCathy Avery } 4624bb170a5SPaolo Bonzini 4638cce12b3SPaolo Bonzini kvm_set_rflags(&svm->vcpu, vmcb12->save.rflags | X86_EFLAGS_FIXED); 4643c346c0cSPaolo Bonzini 4653c346c0cSPaolo Bonzini /* 4663c346c0cSPaolo Bonzini * Force-set EFER_SVME even though it is checked earlier on the 4673c346c0cSPaolo Bonzini * VMCB12, because the guest can flip the bit between the check 4683c346c0cSPaolo Bonzini * and now. Clearing EFER_SVME would call svm_free_nested. 4693c346c0cSPaolo Bonzini */ 4703c346c0cSPaolo Bonzini svm_set_efer(&svm->vcpu, vmcb12->save.efer | EFER_SVME); 4713c346c0cSPaolo Bonzini 4720dd16b5bSMaxim Levitsky svm_set_cr0(&svm->vcpu, vmcb12->save.cr0); 4730dd16b5bSMaxim Levitsky svm_set_cr4(&svm->vcpu, vmcb12->save.cr4); 4744bb170a5SPaolo Bonzini 4754bb170a5SPaolo Bonzini svm->vcpu.arch.cr2 = vmcb12->save.cr2; 4768173396eSCathy Avery 4770dd16b5bSMaxim Levitsky kvm_rax_write(&svm->vcpu, vmcb12->save.rax); 4780dd16b5bSMaxim Levitsky kvm_rsp_write(&svm->vcpu, vmcb12->save.rsp); 4790dd16b5bSMaxim Levitsky kvm_rip_write(&svm->vcpu, vmcb12->save.rip); 480883b0a91SJoerg Roedel 481883b0a91SJoerg Roedel /* In case we don't even reach vcpu_run, the fields are not updated */ 4820dd16b5bSMaxim Levitsky svm->vmcb->save.rax = vmcb12->save.rax; 4830dd16b5bSMaxim Levitsky svm->vmcb->save.rsp = vmcb12->save.rsp; 4840dd16b5bSMaxim Levitsky svm->vmcb->save.rip = vmcb12->save.rip; 4854bb170a5SPaolo Bonzini 4868173396eSCathy Avery /* These bits will be set properly on the first execution when new_vmc12 is true */ 4878173396eSCathy Avery if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_DR))) { 4888cce12b3SPaolo Bonzini svm->vmcb->save.dr7 = vmcb12->save.dr7 | DR7_FIXED_1; 4899a3ecd5eSChenyi Qiang svm->vcpu.arch.dr6 = vmcb12->save.dr6 | DR6_ACTIVE_LOW; 4904bb170a5SPaolo Bonzini vmcb_mark_dirty(svm->vmcb, VMCB_DR); 491f241d711SPaolo Bonzini } 4928173396eSCathy Avery } 493883b0a91SJoerg Roedel 4949e8f0fbfSPaolo Bonzini static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) 495f241d711SPaolo Bonzini { 49691b7130cSPaolo Bonzini const u32 mask = V_INTR_MASKING_MASK | V_GIF_ENABLE_MASK | V_GIF_MASK; 497d2e56019SSean Christopherson struct kvm_vcpu *vcpu = &svm->vcpu; 49862156f6cSVitaly Kuznetsov 4997c3ecfcdSPaolo Bonzini /* 5007c3ecfcdSPaolo Bonzini * Filled at exit: exit_code, exit_code_hi, exit_info_1, exit_info_2, 5017c3ecfcdSPaolo Bonzini * exit_int_info, exit_int_info_err, next_rip, insn_len, insn_bytes. 5027c3ecfcdSPaolo Bonzini */ 5034995a368SCathy Avery 5047c3ecfcdSPaolo Bonzini /* 5057c3ecfcdSPaolo Bonzini * Also covers avic_vapic_bar, avic_backing_page, avic_logical_id, 5067c3ecfcdSPaolo Bonzini * avic_physical_id. 5077c3ecfcdSPaolo Bonzini */ 5087c3ecfcdSPaolo Bonzini WARN_ON(svm->vmcb01.ptr->control.int_ctl & AVIC_ENABLE_MASK); 5097c3ecfcdSPaolo Bonzini 5107c3ecfcdSPaolo Bonzini /* Copied from vmcb01. msrpm_base can be overwritten later. */ 5117c3ecfcdSPaolo Bonzini svm->vmcb->control.nested_ctl = svm->vmcb01.ptr->control.nested_ctl; 5127c3ecfcdSPaolo Bonzini svm->vmcb->control.iopm_base_pa = svm->vmcb01.ptr->control.iopm_base_pa; 5137c3ecfcdSPaolo Bonzini svm->vmcb->control.msrpm_base_pa = svm->vmcb01.ptr->control.msrpm_base_pa; 5147c3ecfcdSPaolo Bonzini 5157c3ecfcdSPaolo Bonzini /* Done at vmrun: asid. */ 5167c3ecfcdSPaolo Bonzini 5177c3ecfcdSPaolo Bonzini /* Also overwritten later if necessary. */ 5187c3ecfcdSPaolo Bonzini svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; 5197c3ecfcdSPaolo Bonzini 5207c3ecfcdSPaolo Bonzini /* nested_cr3. */ 52162156f6cSVitaly Kuznetsov if (nested_npt_enabled(svm)) 522d2e56019SSean Christopherson nested_svm_init_mmu_context(vcpu); 52369cb8774SPaolo Bonzini 524d2e56019SSean Christopherson svm->vmcb->control.tsc_offset = vcpu->arch.tsc_offset = 525d2e56019SSean Christopherson vcpu->arch.l1_tsc_offset + svm->nested.ctl.tsc_offset; 526883b0a91SJoerg Roedel 52791b7130cSPaolo Bonzini svm->vmcb->control.int_ctl = 52891b7130cSPaolo Bonzini (svm->nested.ctl.int_ctl & ~mask) | 5294995a368SCathy Avery (svm->vmcb01.ptr->control.int_ctl & mask); 53091b7130cSPaolo Bonzini 531e670bf68SPaolo Bonzini svm->vmcb->control.virt_ext = svm->nested.ctl.virt_ext; 532e670bf68SPaolo Bonzini svm->vmcb->control.int_vector = svm->nested.ctl.int_vector; 533e670bf68SPaolo Bonzini svm->vmcb->control.int_state = svm->nested.ctl.int_state; 534e670bf68SPaolo Bonzini svm->vmcb->control.event_inj = svm->nested.ctl.event_inj; 535e670bf68SPaolo Bonzini svm->vmcb->control.event_inj_err = svm->nested.ctl.event_inj_err; 536883b0a91SJoerg Roedel 537e670bf68SPaolo Bonzini svm->vmcb->control.pause_filter_count = svm->nested.ctl.pause_filter_count; 538e670bf68SPaolo Bonzini svm->vmcb->control.pause_filter_thresh = svm->nested.ctl.pause_filter_thresh; 539883b0a91SJoerg Roedel 540d2e56019SSean Christopherson nested_svm_transition_tlb_flush(vcpu); 541d2e56019SSean Christopherson 542883b0a91SJoerg Roedel /* Enter Guest-Mode */ 543d2e56019SSean Christopherson enter_guest_mode(vcpu); 544883b0a91SJoerg Roedel 545883b0a91SJoerg Roedel /* 546883b0a91SJoerg Roedel * Merge guest and host intercepts - must be called with vcpu in 5474bb170a5SPaolo Bonzini * guest-mode to take effect. 548883b0a91SJoerg Roedel */ 549883b0a91SJoerg Roedel recalc_intercepts(svm); 550f241d711SPaolo Bonzini } 551f241d711SPaolo Bonzini 552d00b99c5SBabu Moger static void nested_svm_copy_common_state(struct vmcb *from_vmcb, struct vmcb *to_vmcb) 553d00b99c5SBabu Moger { 554d00b99c5SBabu Moger /* 555d00b99c5SBabu Moger * Some VMCB state is shared between L1 and L2 and thus has to be 556d00b99c5SBabu Moger * moved at the time of nested vmrun and vmexit. 557d00b99c5SBabu Moger * 558d00b99c5SBabu Moger * VMLOAD/VMSAVE state would also belong in this category, but KVM 559d00b99c5SBabu Moger * always performs VMLOAD and VMSAVE from the VMCB01. 560d00b99c5SBabu Moger */ 561d00b99c5SBabu Moger to_vmcb->save.spec_ctrl = from_vmcb->save.spec_ctrl; 562d00b99c5SBabu Moger } 563d00b99c5SBabu Moger 56463129754SPaolo Bonzini int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb12_gpa, 5650dd16b5bSMaxim Levitsky struct vmcb *vmcb12) 566f241d711SPaolo Bonzini { 56763129754SPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 568a506fdd2SVitaly Kuznetsov int ret; 569a506fdd2SVitaly Kuznetsov 570954f419bSMaxim Levitsky trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb12_gpa, 571954f419bSMaxim Levitsky vmcb12->save.rip, 572954f419bSMaxim Levitsky vmcb12->control.int_ctl, 573954f419bSMaxim Levitsky vmcb12->control.event_inj, 574954f419bSMaxim Levitsky vmcb12->control.nested_ctl); 575954f419bSMaxim Levitsky 576954f419bSMaxim Levitsky trace_kvm_nested_intercepts(vmcb12->control.intercepts[INTERCEPT_CR] & 0xffff, 577954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_CR] >> 16, 578954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_EXCEPTION], 579954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_WORD3], 580954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_WORD4], 581954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_WORD5]); 582954f419bSMaxim Levitsky 583954f419bSMaxim Levitsky 5840dd16b5bSMaxim Levitsky svm->nested.vmcb12_gpa = vmcb12_gpa; 5854995a368SCathy Avery 5864995a368SCathy Avery WARN_ON(svm->vmcb == svm->nested.vmcb02.ptr); 5874995a368SCathy Avery 588d00b99c5SBabu Moger nested_svm_copy_common_state(svm->vmcb01.ptr, svm->nested.vmcb02.ptr); 5894995a368SCathy Avery 5904995a368SCathy Avery svm_switch_vmcb(svm, &svm->nested.vmcb02); 5919e8f0fbfSPaolo Bonzini nested_vmcb02_prepare_control(svm); 5929e8f0fbfSPaolo Bonzini nested_vmcb02_prepare_save(svm, vmcb12); 593f241d711SPaolo Bonzini 5940dd16b5bSMaxim Levitsky ret = nested_svm_load_cr3(&svm->vcpu, vmcb12->save.cr3, 595b222b0b8SMaxim Levitsky nested_npt_enabled(svm), true); 596a506fdd2SVitaly Kuznetsov if (ret) 597a506fdd2SVitaly Kuznetsov return ret; 598a506fdd2SVitaly Kuznetsov 599a04aead1SPaolo Bonzini if (!npt_enabled) 60063129754SPaolo Bonzini vcpu->arch.mmu->inject_page_fault = svm_inject_page_fault_nested; 601a04aead1SPaolo Bonzini 602ffdf7f9eSPaolo Bonzini svm_set_gif(svm, true); 60359cd9bc5SVitaly Kuznetsov 60459cd9bc5SVitaly Kuznetsov return 0; 605883b0a91SJoerg Roedel } 606883b0a91SJoerg Roedel 60763129754SPaolo Bonzini int nested_svm_vmrun(struct kvm_vcpu *vcpu) 608883b0a91SJoerg Roedel { 60963129754SPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 610883b0a91SJoerg Roedel int ret; 6110dd16b5bSMaxim Levitsky struct vmcb *vmcb12; 612883b0a91SJoerg Roedel struct kvm_host_map map; 6130dd16b5bSMaxim Levitsky u64 vmcb12_gpa; 614883b0a91SJoerg Roedel 61563129754SPaolo Bonzini if (is_smm(vcpu)) { 61663129754SPaolo Bonzini kvm_queue_exception(vcpu, UD_VECTOR); 6177c67f546SPaolo Bonzini return 1; 6187c67f546SPaolo Bonzini } 619883b0a91SJoerg Roedel 6200dd16b5bSMaxim Levitsky vmcb12_gpa = svm->vmcb->save.rax; 62163129754SPaolo Bonzini ret = kvm_vcpu_map(vcpu, gpa_to_gfn(vmcb12_gpa), &map); 622883b0a91SJoerg Roedel if (ret == -EINVAL) { 62363129754SPaolo Bonzini kvm_inject_gp(vcpu, 0); 624883b0a91SJoerg Roedel return 1; 625883b0a91SJoerg Roedel } else if (ret) { 62663129754SPaolo Bonzini return kvm_skip_emulated_instruction(vcpu); 627883b0a91SJoerg Roedel } 628883b0a91SJoerg Roedel 62963129754SPaolo Bonzini ret = kvm_skip_emulated_instruction(vcpu); 630883b0a91SJoerg Roedel 6310dd16b5bSMaxim Levitsky vmcb12 = map.hva; 632883b0a91SJoerg Roedel 6332fcf4876SMaxim Levitsky if (WARN_ON_ONCE(!svm->nested.initialized)) 6342fcf4876SMaxim Levitsky return -EINVAL; 6352fcf4876SMaxim Levitsky 636cb9b6a1bSPaolo Bonzini nested_load_control_from_vmcb12(svm, &vmcb12->control); 637cb9b6a1bSPaolo Bonzini 638cb9b6a1bSPaolo Bonzini if (!nested_vmcb_valid_sregs(vcpu, &vmcb12->save) || 639ee695f22SKrish Sadhukhan !nested_vmcb_check_controls(vcpu, &svm->nested.ctl)) { 6400dd16b5bSMaxim Levitsky vmcb12->control.exit_code = SVM_EXIT_ERR; 6410dd16b5bSMaxim Levitsky vmcb12->control.exit_code_hi = 0; 6420dd16b5bSMaxim Levitsky vmcb12->control.exit_info_1 = 0; 6430dd16b5bSMaxim Levitsky vmcb12->control.exit_info_2 = 0; 64469c9dfa2SPaolo Bonzini goto out; 645883b0a91SJoerg Roedel } 646883b0a91SJoerg Roedel 647883b0a91SJoerg Roedel 648883b0a91SJoerg Roedel /* Clear internal status */ 64963129754SPaolo Bonzini kvm_clear_exception_queue(vcpu); 65063129754SPaolo Bonzini kvm_clear_interrupt_queue(vcpu); 651883b0a91SJoerg Roedel 652883b0a91SJoerg Roedel /* 6534995a368SCathy Avery * Since vmcb01 is not in use, we can use it to store some of the L1 6544995a368SCathy Avery * state. 655883b0a91SJoerg Roedel */ 65663129754SPaolo Bonzini svm->vmcb01.ptr->save.efer = vcpu->arch.efer; 65763129754SPaolo Bonzini svm->vmcb01.ptr->save.cr0 = kvm_read_cr0(vcpu); 65863129754SPaolo Bonzini svm->vmcb01.ptr->save.cr4 = vcpu->arch.cr4; 65963129754SPaolo Bonzini svm->vmcb01.ptr->save.rflags = kvm_get_rflags(vcpu); 66063129754SPaolo Bonzini svm->vmcb01.ptr->save.rip = kvm_rip_read(vcpu); 661883b0a91SJoerg Roedel 6624995a368SCathy Avery if (!npt_enabled) 66363129754SPaolo Bonzini svm->vmcb01.ptr->save.cr3 = kvm_read_cr3(vcpu); 664883b0a91SJoerg Roedel 665f74f9414SPaolo Bonzini svm->nested.nested_run_pending = 1; 666883b0a91SJoerg Roedel 66763129754SPaolo Bonzini if (enter_svm_guest_mode(vcpu, vmcb12_gpa, vmcb12)) 66859cd9bc5SVitaly Kuznetsov goto out_exit_err; 66959cd9bc5SVitaly Kuznetsov 67059cd9bc5SVitaly Kuznetsov if (nested_svm_vmrun_msrpm(svm)) 67159cd9bc5SVitaly Kuznetsov goto out; 67259cd9bc5SVitaly Kuznetsov 67359cd9bc5SVitaly Kuznetsov out_exit_err: 674ebdb3dbaSVitaly Kuznetsov svm->nested.nested_run_pending = 0; 675ebdb3dbaSVitaly Kuznetsov 676883b0a91SJoerg Roedel svm->vmcb->control.exit_code = SVM_EXIT_ERR; 677883b0a91SJoerg Roedel svm->vmcb->control.exit_code_hi = 0; 678883b0a91SJoerg Roedel svm->vmcb->control.exit_info_1 = 0; 679883b0a91SJoerg Roedel svm->vmcb->control.exit_info_2 = 0; 680883b0a91SJoerg Roedel 681883b0a91SJoerg Roedel nested_svm_vmexit(svm); 682883b0a91SJoerg Roedel 68369c9dfa2SPaolo Bonzini out: 68463129754SPaolo Bonzini kvm_vcpu_unmap(vcpu, &map, true); 68569c9dfa2SPaolo Bonzini 686883b0a91SJoerg Roedel return ret; 687883b0a91SJoerg Roedel } 688883b0a91SJoerg Roedel 689883b0a91SJoerg Roedel void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb) 690883b0a91SJoerg Roedel { 691883b0a91SJoerg Roedel to_vmcb->save.fs = from_vmcb->save.fs; 692883b0a91SJoerg Roedel to_vmcb->save.gs = from_vmcb->save.gs; 693883b0a91SJoerg Roedel to_vmcb->save.tr = from_vmcb->save.tr; 694883b0a91SJoerg Roedel to_vmcb->save.ldtr = from_vmcb->save.ldtr; 695883b0a91SJoerg Roedel to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base; 696883b0a91SJoerg Roedel to_vmcb->save.star = from_vmcb->save.star; 697883b0a91SJoerg Roedel to_vmcb->save.lstar = from_vmcb->save.lstar; 698883b0a91SJoerg Roedel to_vmcb->save.cstar = from_vmcb->save.cstar; 699883b0a91SJoerg Roedel to_vmcb->save.sfmask = from_vmcb->save.sfmask; 700883b0a91SJoerg Roedel to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs; 701883b0a91SJoerg Roedel to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp; 702883b0a91SJoerg Roedel to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip; 703883b0a91SJoerg Roedel } 704883b0a91SJoerg Roedel 705883b0a91SJoerg Roedel int nested_svm_vmexit(struct vcpu_svm *svm) 706883b0a91SJoerg Roedel { 70763129754SPaolo Bonzini struct kvm_vcpu *vcpu = &svm->vcpu; 7080dd16b5bSMaxim Levitsky struct vmcb *vmcb12; 709883b0a91SJoerg Roedel struct vmcb *vmcb = svm->vmcb; 710883b0a91SJoerg Roedel struct kvm_host_map map; 71163129754SPaolo Bonzini int rc; 712883b0a91SJoerg Roedel 713cb6a32c2SSean Christopherson /* Triple faults in L2 should never escape. */ 714cb6a32c2SSean Christopherson WARN_ON_ONCE(kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)); 715cb6a32c2SSean Christopherson 71663129754SPaolo Bonzini rc = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa), &map); 717883b0a91SJoerg Roedel if (rc) { 718883b0a91SJoerg Roedel if (rc == -EINVAL) 71963129754SPaolo Bonzini kvm_inject_gp(vcpu, 0); 720883b0a91SJoerg Roedel return 1; 721883b0a91SJoerg Roedel } 722883b0a91SJoerg Roedel 7230dd16b5bSMaxim Levitsky vmcb12 = map.hva; 724883b0a91SJoerg Roedel 725883b0a91SJoerg Roedel /* Exit Guest-Mode */ 72663129754SPaolo Bonzini leave_guest_mode(vcpu); 7270dd16b5bSMaxim Levitsky svm->nested.vmcb12_gpa = 0; 7282d8a42beSPaolo Bonzini WARN_ON_ONCE(svm->nested.nested_run_pending); 729883b0a91SJoerg Roedel 73063129754SPaolo Bonzini kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 731f2c7ef3bSMaxim Levitsky 73238c0b192SPaolo Bonzini /* in case we halted in L2 */ 73338c0b192SPaolo Bonzini svm->vcpu.arch.mp_state = KVM_MP_STATE_RUNNABLE; 73438c0b192SPaolo Bonzini 735883b0a91SJoerg Roedel /* Give the current vmcb to the guest */ 736883b0a91SJoerg Roedel 7370dd16b5bSMaxim Levitsky vmcb12->save.es = vmcb->save.es; 7380dd16b5bSMaxim Levitsky vmcb12->save.cs = vmcb->save.cs; 7390dd16b5bSMaxim Levitsky vmcb12->save.ss = vmcb->save.ss; 7400dd16b5bSMaxim Levitsky vmcb12->save.ds = vmcb->save.ds; 7410dd16b5bSMaxim Levitsky vmcb12->save.gdtr = vmcb->save.gdtr; 7420dd16b5bSMaxim Levitsky vmcb12->save.idtr = vmcb->save.idtr; 7430dd16b5bSMaxim Levitsky vmcb12->save.efer = svm->vcpu.arch.efer; 74463129754SPaolo Bonzini vmcb12->save.cr0 = kvm_read_cr0(vcpu); 74563129754SPaolo Bonzini vmcb12->save.cr3 = kvm_read_cr3(vcpu); 7460dd16b5bSMaxim Levitsky vmcb12->save.cr2 = vmcb->save.cr2; 7470dd16b5bSMaxim Levitsky vmcb12->save.cr4 = svm->vcpu.arch.cr4; 74863129754SPaolo Bonzini vmcb12->save.rflags = kvm_get_rflags(vcpu); 74963129754SPaolo Bonzini vmcb12->save.rip = kvm_rip_read(vcpu); 75063129754SPaolo Bonzini vmcb12->save.rsp = kvm_rsp_read(vcpu); 75163129754SPaolo Bonzini vmcb12->save.rax = kvm_rax_read(vcpu); 7520dd16b5bSMaxim Levitsky vmcb12->save.dr7 = vmcb->save.dr7; 7530dd16b5bSMaxim Levitsky vmcb12->save.dr6 = svm->vcpu.arch.dr6; 7540dd16b5bSMaxim Levitsky vmcb12->save.cpl = vmcb->save.cpl; 755883b0a91SJoerg Roedel 7560dd16b5bSMaxim Levitsky vmcb12->control.int_state = vmcb->control.int_state; 7570dd16b5bSMaxim Levitsky vmcb12->control.exit_code = vmcb->control.exit_code; 7580dd16b5bSMaxim Levitsky vmcb12->control.exit_code_hi = vmcb->control.exit_code_hi; 7590dd16b5bSMaxim Levitsky vmcb12->control.exit_info_1 = vmcb->control.exit_info_1; 7600dd16b5bSMaxim Levitsky vmcb12->control.exit_info_2 = vmcb->control.exit_info_2; 76136e2e983SPaolo Bonzini 7620dd16b5bSMaxim Levitsky if (vmcb12->control.exit_code != SVM_EXIT_ERR) 7639e8f0fbfSPaolo Bonzini nested_save_pending_event_to_vmcb12(svm, vmcb12); 764883b0a91SJoerg Roedel 765883b0a91SJoerg Roedel if (svm->nrips_enabled) 7660dd16b5bSMaxim Levitsky vmcb12->control.next_rip = vmcb->control.next_rip; 767883b0a91SJoerg Roedel 7680dd16b5bSMaxim Levitsky vmcb12->control.int_ctl = svm->nested.ctl.int_ctl; 7690dd16b5bSMaxim Levitsky vmcb12->control.tlb_ctl = svm->nested.ctl.tlb_ctl; 7700dd16b5bSMaxim Levitsky vmcb12->control.event_inj = svm->nested.ctl.event_inj; 7710dd16b5bSMaxim Levitsky vmcb12->control.event_inj_err = svm->nested.ctl.event_inj_err; 772883b0a91SJoerg Roedel 7730dd16b5bSMaxim Levitsky vmcb12->control.pause_filter_count = 774883b0a91SJoerg Roedel svm->vmcb->control.pause_filter_count; 7750dd16b5bSMaxim Levitsky vmcb12->control.pause_filter_thresh = 776883b0a91SJoerg Roedel svm->vmcb->control.pause_filter_thresh; 777883b0a91SJoerg Roedel 778d00b99c5SBabu Moger nested_svm_copy_common_state(svm->nested.vmcb02.ptr, svm->vmcb01.ptr); 779d00b99c5SBabu Moger 7804995a368SCathy Avery svm_switch_vmcb(svm, &svm->vmcb01); 7814995a368SCathy Avery 7824995a368SCathy Avery /* 7834995a368SCathy Avery * On vmexit the GIF is set to false and 7844995a368SCathy Avery * no event can be injected in L1. 7854995a368SCathy Avery */ 7869883764aSMaxim Levitsky svm_set_gif(svm, false); 7874995a368SCathy Avery svm->vmcb->control.exit_int_info = 0; 7889883764aSMaxim Levitsky 7897ca62d13SPaolo Bonzini svm->vcpu.arch.tsc_offset = svm->vcpu.arch.l1_tsc_offset; 7907ca62d13SPaolo Bonzini if (svm->vmcb->control.tsc_offset != svm->vcpu.arch.tsc_offset) { 7917ca62d13SPaolo Bonzini svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset; 7927ca62d13SPaolo Bonzini vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 7937ca62d13SPaolo Bonzini } 79418fc6c55SPaolo Bonzini 795e670bf68SPaolo Bonzini svm->nested.ctl.nested_cr3 = 0; 796883b0a91SJoerg Roedel 7974995a368SCathy Avery /* 7984995a368SCathy Avery * Restore processor state that had been saved in vmcb01 7994995a368SCathy Avery */ 80063129754SPaolo Bonzini kvm_set_rflags(vcpu, svm->vmcb->save.rflags); 80163129754SPaolo Bonzini svm_set_efer(vcpu, svm->vmcb->save.efer); 80263129754SPaolo Bonzini svm_set_cr0(vcpu, svm->vmcb->save.cr0 | X86_CR0_PE); 80363129754SPaolo Bonzini svm_set_cr4(vcpu, svm->vmcb->save.cr4); 80463129754SPaolo Bonzini kvm_rax_write(vcpu, svm->vmcb->save.rax); 80563129754SPaolo Bonzini kvm_rsp_write(vcpu, svm->vmcb->save.rsp); 80663129754SPaolo Bonzini kvm_rip_write(vcpu, svm->vmcb->save.rip); 8074995a368SCathy Avery 8084995a368SCathy Avery svm->vcpu.arch.dr7 = DR7_FIXED_1; 8094995a368SCathy Avery kvm_update_dr7(&svm->vcpu); 810883b0a91SJoerg Roedel 8110dd16b5bSMaxim Levitsky trace_kvm_nested_vmexit_inject(vmcb12->control.exit_code, 8120dd16b5bSMaxim Levitsky vmcb12->control.exit_info_1, 8130dd16b5bSMaxim Levitsky vmcb12->control.exit_info_2, 8140dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info, 8150dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info_err, 81636e2e983SPaolo Bonzini KVM_ISA_SVM); 81736e2e983SPaolo Bonzini 81863129754SPaolo Bonzini kvm_vcpu_unmap(vcpu, &map, true); 819883b0a91SJoerg Roedel 820d2e56019SSean Christopherson nested_svm_transition_tlb_flush(vcpu); 821d2e56019SSean Christopherson 82263129754SPaolo Bonzini nested_svm_uninit_mmu_context(vcpu); 823bf7dea42SVitaly Kuznetsov 824b222b0b8SMaxim Levitsky rc = nested_svm_load_cr3(vcpu, svm->vmcb->save.cr3, false, true); 825d82aaef9SVitaly Kuznetsov if (rc) 826d82aaef9SVitaly Kuznetsov return 1; 827bf7dea42SVitaly Kuznetsov 828883b0a91SJoerg Roedel /* 829883b0a91SJoerg Roedel * Drop what we picked up for L2 via svm_complete_interrupts() so it 830883b0a91SJoerg Roedel * doesn't end up in L1. 831883b0a91SJoerg Roedel */ 832883b0a91SJoerg Roedel svm->vcpu.arch.nmi_injected = false; 83363129754SPaolo Bonzini kvm_clear_exception_queue(vcpu); 83463129754SPaolo Bonzini kvm_clear_interrupt_queue(vcpu); 835883b0a91SJoerg Roedel 8369a7de6ecSKrish Sadhukhan /* 8379a7de6ecSKrish Sadhukhan * If we are here following the completion of a VMRUN that 8389a7de6ecSKrish Sadhukhan * is being single-stepped, queue the pending #DB intercept 8399a7de6ecSKrish Sadhukhan * right now so that it an be accounted for before we execute 8409a7de6ecSKrish Sadhukhan * L1's next instruction. 8419a7de6ecSKrish Sadhukhan */ 8429a7de6ecSKrish Sadhukhan if (unlikely(svm->vmcb->save.rflags & X86_EFLAGS_TF)) 8439a7de6ecSKrish Sadhukhan kvm_queue_exception(&(svm->vcpu), DB_VECTOR); 8449a7de6ecSKrish Sadhukhan 845883b0a91SJoerg Roedel return 0; 846883b0a91SJoerg Roedel } 847883b0a91SJoerg Roedel 848cb6a32c2SSean Christopherson static void nested_svm_triple_fault(struct kvm_vcpu *vcpu) 849cb6a32c2SSean Christopherson { 8503a87c7e0SSean Christopherson nested_svm_simple_vmexit(to_svm(vcpu), SVM_EXIT_SHUTDOWN); 851cb6a32c2SSean Christopherson } 852cb6a32c2SSean Christopherson 8532fcf4876SMaxim Levitsky int svm_allocate_nested(struct vcpu_svm *svm) 8542fcf4876SMaxim Levitsky { 8554995a368SCathy Avery struct page *vmcb02_page; 8562fcf4876SMaxim Levitsky 8572fcf4876SMaxim Levitsky if (svm->nested.initialized) 8582fcf4876SMaxim Levitsky return 0; 8592fcf4876SMaxim Levitsky 8604995a368SCathy Avery vmcb02_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 8614995a368SCathy Avery if (!vmcb02_page) 8622fcf4876SMaxim Levitsky return -ENOMEM; 8634995a368SCathy Avery svm->nested.vmcb02.ptr = page_address(vmcb02_page); 8644995a368SCathy Avery svm->nested.vmcb02.pa = __sme_set(page_to_pfn(vmcb02_page) << PAGE_SHIFT); 8652fcf4876SMaxim Levitsky 8662fcf4876SMaxim Levitsky svm->nested.msrpm = svm_vcpu_alloc_msrpm(); 8672fcf4876SMaxim Levitsky if (!svm->nested.msrpm) 8684995a368SCathy Avery goto err_free_vmcb02; 8692fcf4876SMaxim Levitsky svm_vcpu_init_msrpm(&svm->vcpu, svm->nested.msrpm); 8702fcf4876SMaxim Levitsky 8712fcf4876SMaxim Levitsky svm->nested.initialized = true; 8722fcf4876SMaxim Levitsky return 0; 8732fcf4876SMaxim Levitsky 8744995a368SCathy Avery err_free_vmcb02: 8754995a368SCathy Avery __free_page(vmcb02_page); 8762fcf4876SMaxim Levitsky return -ENOMEM; 8772fcf4876SMaxim Levitsky } 8782fcf4876SMaxim Levitsky 8792fcf4876SMaxim Levitsky void svm_free_nested(struct vcpu_svm *svm) 8802fcf4876SMaxim Levitsky { 8812fcf4876SMaxim Levitsky if (!svm->nested.initialized) 8822fcf4876SMaxim Levitsky return; 8832fcf4876SMaxim Levitsky 8842fcf4876SMaxim Levitsky svm_vcpu_free_msrpm(svm->nested.msrpm); 8852fcf4876SMaxim Levitsky svm->nested.msrpm = NULL; 8862fcf4876SMaxim Levitsky 8874995a368SCathy Avery __free_page(virt_to_page(svm->nested.vmcb02.ptr)); 8884995a368SCathy Avery svm->nested.vmcb02.ptr = NULL; 8892fcf4876SMaxim Levitsky 890c74ad08fSMaxim Levitsky /* 891c74ad08fSMaxim Levitsky * When last_vmcb12_gpa matches the current vmcb12 gpa, 892c74ad08fSMaxim Levitsky * some vmcb12 fields are not loaded if they are marked clean 893c74ad08fSMaxim Levitsky * in the vmcb12, since in this case they are up to date already. 894c74ad08fSMaxim Levitsky * 895c74ad08fSMaxim Levitsky * When the vmcb02 is freed, this optimization becomes invalid. 896c74ad08fSMaxim Levitsky */ 897c74ad08fSMaxim Levitsky svm->nested.last_vmcb12_gpa = INVALID_GPA; 898c74ad08fSMaxim Levitsky 8992fcf4876SMaxim Levitsky svm->nested.initialized = false; 9002fcf4876SMaxim Levitsky } 9012fcf4876SMaxim Levitsky 902c513f484SPaolo Bonzini /* 903c513f484SPaolo Bonzini * Forcibly leave nested mode in order to be able to reset the VCPU later on. 904c513f484SPaolo Bonzini */ 905c513f484SPaolo Bonzini void svm_leave_nested(struct vcpu_svm *svm) 906c513f484SPaolo Bonzini { 90763129754SPaolo Bonzini struct kvm_vcpu *vcpu = &svm->vcpu; 90863129754SPaolo Bonzini 90963129754SPaolo Bonzini if (is_guest_mode(vcpu)) { 910c513f484SPaolo Bonzini svm->nested.nested_run_pending = 0; 911c74ad08fSMaxim Levitsky svm->nested.vmcb12_gpa = INVALID_GPA; 912c74ad08fSMaxim Levitsky 91363129754SPaolo Bonzini leave_guest_mode(vcpu); 9144995a368SCathy Avery 915deee59baSMaxim Levitsky svm_switch_vmcb(svm, &svm->vmcb01); 9164995a368SCathy Avery 91763129754SPaolo Bonzini nested_svm_uninit_mmu_context(vcpu); 91856fe28deSMaxim Levitsky vmcb_mark_all_dirty(svm->vmcb); 919c513f484SPaolo Bonzini } 920a7d5c7ceSPaolo Bonzini 92163129754SPaolo Bonzini kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 922c513f484SPaolo Bonzini } 923c513f484SPaolo Bonzini 924883b0a91SJoerg Roedel static int nested_svm_exit_handled_msr(struct vcpu_svm *svm) 925883b0a91SJoerg Roedel { 926883b0a91SJoerg Roedel u32 offset, msr, value; 927883b0a91SJoerg Roedel int write, mask; 928883b0a91SJoerg Roedel 929c62e2e94SBabu Moger if (!(vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT))) 930883b0a91SJoerg Roedel return NESTED_EXIT_HOST; 931883b0a91SJoerg Roedel 932883b0a91SJoerg Roedel msr = svm->vcpu.arch.regs[VCPU_REGS_RCX]; 933883b0a91SJoerg Roedel offset = svm_msrpm_offset(msr); 934883b0a91SJoerg Roedel write = svm->vmcb->control.exit_info_1 & 1; 935883b0a91SJoerg Roedel mask = 1 << ((2 * (msr & 0xf)) + write); 936883b0a91SJoerg Roedel 937883b0a91SJoerg Roedel if (offset == MSR_INVALID) 938883b0a91SJoerg Roedel return NESTED_EXIT_DONE; 939883b0a91SJoerg Roedel 940883b0a91SJoerg Roedel /* Offset is in 32 bit units but need in 8 bit units */ 941883b0a91SJoerg Roedel offset *= 4; 942883b0a91SJoerg Roedel 943e670bf68SPaolo Bonzini if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.ctl.msrpm_base_pa + offset, &value, 4)) 944883b0a91SJoerg Roedel return NESTED_EXIT_DONE; 945883b0a91SJoerg Roedel 946883b0a91SJoerg Roedel return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; 947883b0a91SJoerg Roedel } 948883b0a91SJoerg Roedel 949883b0a91SJoerg Roedel static int nested_svm_intercept_ioio(struct vcpu_svm *svm) 950883b0a91SJoerg Roedel { 951883b0a91SJoerg Roedel unsigned port, size, iopm_len; 952883b0a91SJoerg Roedel u16 val, mask; 953883b0a91SJoerg Roedel u8 start_bit; 954883b0a91SJoerg Roedel u64 gpa; 955883b0a91SJoerg Roedel 956c62e2e94SBabu Moger if (!(vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_IOIO_PROT))) 957883b0a91SJoerg Roedel return NESTED_EXIT_HOST; 958883b0a91SJoerg Roedel 959883b0a91SJoerg Roedel port = svm->vmcb->control.exit_info_1 >> 16; 960883b0a91SJoerg Roedel size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >> 961883b0a91SJoerg Roedel SVM_IOIO_SIZE_SHIFT; 962e670bf68SPaolo Bonzini gpa = svm->nested.ctl.iopm_base_pa + (port / 8); 963883b0a91SJoerg Roedel start_bit = port % 8; 964883b0a91SJoerg Roedel iopm_len = (start_bit + size > 8) ? 2 : 1; 965883b0a91SJoerg Roedel mask = (0xf >> (4 - size)) << start_bit; 966883b0a91SJoerg Roedel val = 0; 967883b0a91SJoerg Roedel 968883b0a91SJoerg Roedel if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len)) 969883b0a91SJoerg Roedel return NESTED_EXIT_DONE; 970883b0a91SJoerg Roedel 971883b0a91SJoerg Roedel return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; 972883b0a91SJoerg Roedel } 973883b0a91SJoerg Roedel 974883b0a91SJoerg Roedel static int nested_svm_intercept(struct vcpu_svm *svm) 975883b0a91SJoerg Roedel { 976883b0a91SJoerg Roedel u32 exit_code = svm->vmcb->control.exit_code; 977883b0a91SJoerg Roedel int vmexit = NESTED_EXIT_HOST; 978883b0a91SJoerg Roedel 979883b0a91SJoerg Roedel switch (exit_code) { 980883b0a91SJoerg Roedel case SVM_EXIT_MSR: 981883b0a91SJoerg Roedel vmexit = nested_svm_exit_handled_msr(svm); 982883b0a91SJoerg Roedel break; 983883b0a91SJoerg Roedel case SVM_EXIT_IOIO: 984883b0a91SJoerg Roedel vmexit = nested_svm_intercept_ioio(svm); 985883b0a91SJoerg Roedel break; 986883b0a91SJoerg Roedel case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: { 98703bfeeb9SBabu Moger if (vmcb_is_intercept(&svm->nested.ctl, exit_code)) 988883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 989883b0a91SJoerg Roedel break; 990883b0a91SJoerg Roedel } 991883b0a91SJoerg Roedel case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: { 99230abaa88SBabu Moger if (vmcb_is_intercept(&svm->nested.ctl, exit_code)) 993883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 994883b0a91SJoerg Roedel break; 995883b0a91SJoerg Roedel } 996883b0a91SJoerg Roedel case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: { 9977c86663bSPaolo Bonzini /* 9987c86663bSPaolo Bonzini * Host-intercepted exceptions have been checked already in 9997c86663bSPaolo Bonzini * nested_svm_exit_special. There is nothing to do here, 10007c86663bSPaolo Bonzini * the vmexit is injected by svm_check_nested_events. 10017c86663bSPaolo Bonzini */ 1002883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 1003883b0a91SJoerg Roedel break; 1004883b0a91SJoerg Roedel } 1005883b0a91SJoerg Roedel case SVM_EXIT_ERR: { 1006883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 1007883b0a91SJoerg Roedel break; 1008883b0a91SJoerg Roedel } 1009883b0a91SJoerg Roedel default: { 1010c62e2e94SBabu Moger if (vmcb_is_intercept(&svm->nested.ctl, exit_code)) 1011883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 1012883b0a91SJoerg Roedel } 1013883b0a91SJoerg Roedel } 1014883b0a91SJoerg Roedel 1015883b0a91SJoerg Roedel return vmexit; 1016883b0a91SJoerg Roedel } 1017883b0a91SJoerg Roedel 1018883b0a91SJoerg Roedel int nested_svm_exit_handled(struct vcpu_svm *svm) 1019883b0a91SJoerg Roedel { 1020883b0a91SJoerg Roedel int vmexit; 1021883b0a91SJoerg Roedel 1022883b0a91SJoerg Roedel vmexit = nested_svm_intercept(svm); 1023883b0a91SJoerg Roedel 1024883b0a91SJoerg Roedel if (vmexit == NESTED_EXIT_DONE) 1025883b0a91SJoerg Roedel nested_svm_vmexit(svm); 1026883b0a91SJoerg Roedel 1027883b0a91SJoerg Roedel return vmexit; 1028883b0a91SJoerg Roedel } 1029883b0a91SJoerg Roedel 103063129754SPaolo Bonzini int nested_svm_check_permissions(struct kvm_vcpu *vcpu) 1031883b0a91SJoerg Roedel { 103263129754SPaolo Bonzini if (!(vcpu->arch.efer & EFER_SVME) || !is_paging(vcpu)) { 103363129754SPaolo Bonzini kvm_queue_exception(vcpu, UD_VECTOR); 1034883b0a91SJoerg Roedel return 1; 1035883b0a91SJoerg Roedel } 1036883b0a91SJoerg Roedel 103763129754SPaolo Bonzini if (to_svm(vcpu)->vmcb->save.cpl) { 103863129754SPaolo Bonzini kvm_inject_gp(vcpu, 0); 1039883b0a91SJoerg Roedel return 1; 1040883b0a91SJoerg Roedel } 1041883b0a91SJoerg Roedel 1042883b0a91SJoerg Roedel return 0; 1043883b0a91SJoerg Roedel } 1044883b0a91SJoerg Roedel 10457c86663bSPaolo Bonzini static bool nested_exit_on_exception(struct vcpu_svm *svm) 1046883b0a91SJoerg Roedel { 10477c86663bSPaolo Bonzini unsigned int nr = svm->vcpu.arch.exception.nr; 1048883b0a91SJoerg Roedel 10499780d51dSBabu Moger return (svm->nested.ctl.intercepts[INTERCEPT_EXCEPTION] & BIT(nr)); 10507c86663bSPaolo Bonzini } 1051883b0a91SJoerg Roedel 10527c86663bSPaolo Bonzini static void nested_svm_inject_exception_vmexit(struct vcpu_svm *svm) 10537c86663bSPaolo Bonzini { 10547c86663bSPaolo Bonzini unsigned int nr = svm->vcpu.arch.exception.nr; 1055883b0a91SJoerg Roedel 1056883b0a91SJoerg Roedel svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr; 1057883b0a91SJoerg Roedel svm->vmcb->control.exit_code_hi = 0; 10587c86663bSPaolo Bonzini 10597c86663bSPaolo Bonzini if (svm->vcpu.arch.exception.has_error_code) 10607c86663bSPaolo Bonzini svm->vmcb->control.exit_info_1 = svm->vcpu.arch.exception.error_code; 1061883b0a91SJoerg Roedel 1062883b0a91SJoerg Roedel /* 1063883b0a91SJoerg Roedel * EXITINFO2 is undefined for all exception intercepts other 1064883b0a91SJoerg Roedel * than #PF. 1065883b0a91SJoerg Roedel */ 10667c86663bSPaolo Bonzini if (nr == PF_VECTOR) { 1067883b0a91SJoerg Roedel if (svm->vcpu.arch.exception.nested_apf) 1068883b0a91SJoerg Roedel svm->vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token; 1069883b0a91SJoerg Roedel else if (svm->vcpu.arch.exception.has_payload) 1070883b0a91SJoerg Roedel svm->vmcb->control.exit_info_2 = svm->vcpu.arch.exception.payload; 1071883b0a91SJoerg Roedel else 1072883b0a91SJoerg Roedel svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2; 10737c86663bSPaolo Bonzini } else if (nr == DB_VECTOR) { 10747c86663bSPaolo Bonzini /* See inject_pending_event. */ 10757c86663bSPaolo Bonzini kvm_deliver_exception_payload(&svm->vcpu); 10767c86663bSPaolo Bonzini if (svm->vcpu.arch.dr7 & DR7_GD) { 10777c86663bSPaolo Bonzini svm->vcpu.arch.dr7 &= ~DR7_GD; 10787c86663bSPaolo Bonzini kvm_update_dr7(&svm->vcpu); 10797c86663bSPaolo Bonzini } 10807c86663bSPaolo Bonzini } else 10817c86663bSPaolo Bonzini WARN_ON(svm->vcpu.arch.exception.has_payload); 1082883b0a91SJoerg Roedel 10837c86663bSPaolo Bonzini nested_svm_vmexit(svm); 1084883b0a91SJoerg Roedel } 1085883b0a91SJoerg Roedel 10865b672408SPaolo Bonzini static inline bool nested_exit_on_init(struct vcpu_svm *svm) 10875b672408SPaolo Bonzini { 1088c62e2e94SBabu Moger return vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_INIT); 10895b672408SPaolo Bonzini } 10905b672408SPaolo Bonzini 109133b22172SPaolo Bonzini static int svm_check_nested_events(struct kvm_vcpu *vcpu) 1092883b0a91SJoerg Roedel { 1093883b0a91SJoerg Roedel struct vcpu_svm *svm = to_svm(vcpu); 1094883b0a91SJoerg Roedel bool block_nested_events = 1095bd279629SPaolo Bonzini kvm_event_needs_reinjection(vcpu) || svm->nested.nested_run_pending; 10965b672408SPaolo Bonzini struct kvm_lapic *apic = vcpu->arch.apic; 10975b672408SPaolo Bonzini 10985b672408SPaolo Bonzini if (lapic_in_kernel(vcpu) && 10995b672408SPaolo Bonzini test_bit(KVM_APIC_INIT, &apic->pending_events)) { 11005b672408SPaolo Bonzini if (block_nested_events) 11015b672408SPaolo Bonzini return -EBUSY; 11025b672408SPaolo Bonzini if (!nested_exit_on_init(svm)) 11035b672408SPaolo Bonzini return 0; 11043a87c7e0SSean Christopherson nested_svm_simple_vmexit(svm, SVM_EXIT_INIT); 11055b672408SPaolo Bonzini return 0; 11065b672408SPaolo Bonzini } 1107883b0a91SJoerg Roedel 11087c86663bSPaolo Bonzini if (vcpu->arch.exception.pending) { 11094020da3bSMaxim Levitsky /* 11104020da3bSMaxim Levitsky * Only a pending nested run can block a pending exception. 11114020da3bSMaxim Levitsky * Otherwise an injected NMI/interrupt should either be 11124020da3bSMaxim Levitsky * lost or delivered to the nested hypervisor in the EXITINTINFO 11134020da3bSMaxim Levitsky * vmcb field, while delivering the pending exception. 11144020da3bSMaxim Levitsky */ 11154020da3bSMaxim Levitsky if (svm->nested.nested_run_pending) 11167c86663bSPaolo Bonzini return -EBUSY; 11177c86663bSPaolo Bonzini if (!nested_exit_on_exception(svm)) 11187c86663bSPaolo Bonzini return 0; 11197c86663bSPaolo Bonzini nested_svm_inject_exception_vmexit(svm); 11207c86663bSPaolo Bonzini return 0; 11217c86663bSPaolo Bonzini } 11227c86663bSPaolo Bonzini 1123221e7610SPaolo Bonzini if (vcpu->arch.smi_pending && !svm_smi_blocked(vcpu)) { 112455714cddSPaolo Bonzini if (block_nested_events) 112555714cddSPaolo Bonzini return -EBUSY; 1126221e7610SPaolo Bonzini if (!nested_exit_on_smi(svm)) 1127221e7610SPaolo Bonzini return 0; 11283a87c7e0SSean Christopherson nested_svm_simple_vmexit(svm, SVM_EXIT_SMI); 112955714cddSPaolo Bonzini return 0; 113055714cddSPaolo Bonzini } 113155714cddSPaolo Bonzini 1132221e7610SPaolo Bonzini if (vcpu->arch.nmi_pending && !svm_nmi_blocked(vcpu)) { 11339c3d370aSCathy Avery if (block_nested_events) 11349c3d370aSCathy Avery return -EBUSY; 1135221e7610SPaolo Bonzini if (!nested_exit_on_nmi(svm)) 1136221e7610SPaolo Bonzini return 0; 11373a87c7e0SSean Christopherson nested_svm_simple_vmexit(svm, SVM_EXIT_NMI); 11389c3d370aSCathy Avery return 0; 11399c3d370aSCathy Avery } 11409c3d370aSCathy Avery 1141221e7610SPaolo Bonzini if (kvm_cpu_has_interrupt(vcpu) && !svm_interrupt_blocked(vcpu)) { 1142883b0a91SJoerg Roedel if (block_nested_events) 1143883b0a91SJoerg Roedel return -EBUSY; 1144221e7610SPaolo Bonzini if (!nested_exit_on_intr(svm)) 1145221e7610SPaolo Bonzini return 0; 11463a87c7e0SSean Christopherson trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip); 11473a87c7e0SSean Christopherson nested_svm_simple_vmexit(svm, SVM_EXIT_INTR); 1148883b0a91SJoerg Roedel return 0; 1149883b0a91SJoerg Roedel } 1150883b0a91SJoerg Roedel 1151883b0a91SJoerg Roedel return 0; 1152883b0a91SJoerg Roedel } 1153883b0a91SJoerg Roedel 1154883b0a91SJoerg Roedel int nested_svm_exit_special(struct vcpu_svm *svm) 1155883b0a91SJoerg Roedel { 1156883b0a91SJoerg Roedel u32 exit_code = svm->vmcb->control.exit_code; 1157883b0a91SJoerg Roedel 1158883b0a91SJoerg Roedel switch (exit_code) { 1159883b0a91SJoerg Roedel case SVM_EXIT_INTR: 1160883b0a91SJoerg Roedel case SVM_EXIT_NMI: 1161883b0a91SJoerg Roedel case SVM_EXIT_NPF: 1162883b0a91SJoerg Roedel return NESTED_EXIT_HOST; 11637c86663bSPaolo Bonzini case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: { 11647c86663bSPaolo Bonzini u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE); 11657c86663bSPaolo Bonzini 11664995a368SCathy Avery if (svm->vmcb01.ptr->control.intercepts[INTERCEPT_EXCEPTION] & 11679780d51dSBabu Moger excp_bits) 11687c86663bSPaolo Bonzini return NESTED_EXIT_HOST; 11697c86663bSPaolo Bonzini else if (exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR && 117068fd66f1SVitaly Kuznetsov svm->vcpu.arch.apf.host_apf_flags) 1171a3535be7SPaolo Bonzini /* Trap async PF even if not shadowing */ 1172883b0a91SJoerg Roedel return NESTED_EXIT_HOST; 1173883b0a91SJoerg Roedel break; 11747c86663bSPaolo Bonzini } 1175883b0a91SJoerg Roedel default: 1176883b0a91SJoerg Roedel break; 1177883b0a91SJoerg Roedel } 1178883b0a91SJoerg Roedel 1179883b0a91SJoerg Roedel return NESTED_EXIT_CONTINUE; 1180883b0a91SJoerg Roedel } 118133b22172SPaolo Bonzini 1182cc440cdaSPaolo Bonzini static int svm_get_nested_state(struct kvm_vcpu *vcpu, 1183cc440cdaSPaolo Bonzini struct kvm_nested_state __user *user_kvm_nested_state, 1184cc440cdaSPaolo Bonzini u32 user_data_size) 1185cc440cdaSPaolo Bonzini { 1186cc440cdaSPaolo Bonzini struct vcpu_svm *svm; 1187cc440cdaSPaolo Bonzini struct kvm_nested_state kvm_state = { 1188cc440cdaSPaolo Bonzini .flags = 0, 1189cc440cdaSPaolo Bonzini .format = KVM_STATE_NESTED_FORMAT_SVM, 1190cc440cdaSPaolo Bonzini .size = sizeof(kvm_state), 1191cc440cdaSPaolo Bonzini }; 1192cc440cdaSPaolo Bonzini struct vmcb __user *user_vmcb = (struct vmcb __user *) 1193cc440cdaSPaolo Bonzini &user_kvm_nested_state->data.svm[0]; 1194cc440cdaSPaolo Bonzini 1195cc440cdaSPaolo Bonzini if (!vcpu) 1196cc440cdaSPaolo Bonzini return kvm_state.size + KVM_STATE_NESTED_SVM_VMCB_SIZE; 1197cc440cdaSPaolo Bonzini 1198cc440cdaSPaolo Bonzini svm = to_svm(vcpu); 1199cc440cdaSPaolo Bonzini 1200cc440cdaSPaolo Bonzini if (user_data_size < kvm_state.size) 1201cc440cdaSPaolo Bonzini goto out; 1202cc440cdaSPaolo Bonzini 1203cc440cdaSPaolo Bonzini /* First fill in the header and copy it out. */ 1204cc440cdaSPaolo Bonzini if (is_guest_mode(vcpu)) { 12050dd16b5bSMaxim Levitsky kvm_state.hdr.svm.vmcb_pa = svm->nested.vmcb12_gpa; 1206cc440cdaSPaolo Bonzini kvm_state.size += KVM_STATE_NESTED_SVM_VMCB_SIZE; 1207cc440cdaSPaolo Bonzini kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE; 1208cc440cdaSPaolo Bonzini 1209cc440cdaSPaolo Bonzini if (svm->nested.nested_run_pending) 1210cc440cdaSPaolo Bonzini kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING; 1211cc440cdaSPaolo Bonzini } 1212cc440cdaSPaolo Bonzini 1213cc440cdaSPaolo Bonzini if (gif_set(svm)) 1214cc440cdaSPaolo Bonzini kvm_state.flags |= KVM_STATE_NESTED_GIF_SET; 1215cc440cdaSPaolo Bonzini 1216cc440cdaSPaolo Bonzini if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state))) 1217cc440cdaSPaolo Bonzini return -EFAULT; 1218cc440cdaSPaolo Bonzini 1219cc440cdaSPaolo Bonzini if (!is_guest_mode(vcpu)) 1220cc440cdaSPaolo Bonzini goto out; 1221cc440cdaSPaolo Bonzini 1222cc440cdaSPaolo Bonzini /* 1223cc440cdaSPaolo Bonzini * Copy over the full size of the VMCB rather than just the size 1224cc440cdaSPaolo Bonzini * of the structs. 1225cc440cdaSPaolo Bonzini */ 1226cc440cdaSPaolo Bonzini if (clear_user(user_vmcb, KVM_STATE_NESTED_SVM_VMCB_SIZE)) 1227cc440cdaSPaolo Bonzini return -EFAULT; 1228cc440cdaSPaolo Bonzini if (copy_to_user(&user_vmcb->control, &svm->nested.ctl, 1229cc440cdaSPaolo Bonzini sizeof(user_vmcb->control))) 1230cc440cdaSPaolo Bonzini return -EFAULT; 12314995a368SCathy Avery if (copy_to_user(&user_vmcb->save, &svm->vmcb01.ptr->save, 1232cc440cdaSPaolo Bonzini sizeof(user_vmcb->save))) 1233cc440cdaSPaolo Bonzini return -EFAULT; 1234cc440cdaSPaolo Bonzini out: 1235cc440cdaSPaolo Bonzini return kvm_state.size; 1236cc440cdaSPaolo Bonzini } 1237cc440cdaSPaolo Bonzini 1238cc440cdaSPaolo Bonzini static int svm_set_nested_state(struct kvm_vcpu *vcpu, 1239cc440cdaSPaolo Bonzini struct kvm_nested_state __user *user_kvm_nested_state, 1240cc440cdaSPaolo Bonzini struct kvm_nested_state *kvm_state) 1241cc440cdaSPaolo Bonzini { 1242cc440cdaSPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 1243cc440cdaSPaolo Bonzini struct vmcb __user *user_vmcb = (struct vmcb __user *) 1244cc440cdaSPaolo Bonzini &user_kvm_nested_state->data.svm[0]; 12456ccbd29aSJoerg Roedel struct vmcb_control_area *ctl; 12466ccbd29aSJoerg Roedel struct vmcb_save_area *save; 1247*dbc4739bSSean Christopherson unsigned long cr0; 12486ccbd29aSJoerg Roedel int ret; 1249cc440cdaSPaolo Bonzini 12506ccbd29aSJoerg Roedel BUILD_BUG_ON(sizeof(struct vmcb_control_area) + sizeof(struct vmcb_save_area) > 12516ccbd29aSJoerg Roedel KVM_STATE_NESTED_SVM_VMCB_SIZE); 12526ccbd29aSJoerg Roedel 1253cc440cdaSPaolo Bonzini if (kvm_state->format != KVM_STATE_NESTED_FORMAT_SVM) 1254cc440cdaSPaolo Bonzini return -EINVAL; 1255cc440cdaSPaolo Bonzini 1256cc440cdaSPaolo Bonzini if (kvm_state->flags & ~(KVM_STATE_NESTED_GUEST_MODE | 1257cc440cdaSPaolo Bonzini KVM_STATE_NESTED_RUN_PENDING | 1258cc440cdaSPaolo Bonzini KVM_STATE_NESTED_GIF_SET)) 1259cc440cdaSPaolo Bonzini return -EINVAL; 1260cc440cdaSPaolo Bonzini 1261cc440cdaSPaolo Bonzini /* 1262cc440cdaSPaolo Bonzini * If in guest mode, vcpu->arch.efer actually refers to the L2 guest's 1263cc440cdaSPaolo Bonzini * EFER.SVME, but EFER.SVME still has to be 1 for VMRUN to succeed. 1264cc440cdaSPaolo Bonzini */ 1265cc440cdaSPaolo Bonzini if (!(vcpu->arch.efer & EFER_SVME)) { 1266cc440cdaSPaolo Bonzini /* GIF=1 and no guest mode are required if SVME=0. */ 1267cc440cdaSPaolo Bonzini if (kvm_state->flags != KVM_STATE_NESTED_GIF_SET) 1268cc440cdaSPaolo Bonzini return -EINVAL; 1269cc440cdaSPaolo Bonzini } 1270cc440cdaSPaolo Bonzini 1271cc440cdaSPaolo Bonzini /* SMM temporarily disables SVM, so we cannot be in guest mode. */ 1272cc440cdaSPaolo Bonzini if (is_smm(vcpu) && (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 1273cc440cdaSPaolo Bonzini return -EINVAL; 1274cc440cdaSPaolo Bonzini 1275cc440cdaSPaolo Bonzini if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) { 1276cc440cdaSPaolo Bonzini svm_leave_nested(svm); 1277d5cd6f34SVitaly Kuznetsov svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET)); 1278d5cd6f34SVitaly Kuznetsov return 0; 1279cc440cdaSPaolo Bonzini } 1280cc440cdaSPaolo Bonzini 1281cc440cdaSPaolo Bonzini if (!page_address_valid(vcpu, kvm_state->hdr.svm.vmcb_pa)) 1282cc440cdaSPaolo Bonzini return -EINVAL; 1283cc440cdaSPaolo Bonzini if (kvm_state->size < sizeof(*kvm_state) + KVM_STATE_NESTED_SVM_VMCB_SIZE) 1284cc440cdaSPaolo Bonzini return -EINVAL; 1285cc440cdaSPaolo Bonzini 12866ccbd29aSJoerg Roedel ret = -ENOMEM; 1287eba04b20SSean Christopherson ctl = kzalloc(sizeof(*ctl), GFP_KERNEL_ACCOUNT); 1288eba04b20SSean Christopherson save = kzalloc(sizeof(*save), GFP_KERNEL_ACCOUNT); 12896ccbd29aSJoerg Roedel if (!ctl || !save) 12906ccbd29aSJoerg Roedel goto out_free; 12916ccbd29aSJoerg Roedel 12926ccbd29aSJoerg Roedel ret = -EFAULT; 12936ccbd29aSJoerg Roedel if (copy_from_user(ctl, &user_vmcb->control, sizeof(*ctl))) 12946ccbd29aSJoerg Roedel goto out_free; 12956ccbd29aSJoerg Roedel if (copy_from_user(save, &user_vmcb->save, sizeof(*save))) 12966ccbd29aSJoerg Roedel goto out_free; 12976ccbd29aSJoerg Roedel 12986ccbd29aSJoerg Roedel ret = -EINVAL; 1299ee695f22SKrish Sadhukhan if (!nested_vmcb_check_controls(vcpu, ctl)) 13006ccbd29aSJoerg Roedel goto out_free; 1301cc440cdaSPaolo Bonzini 1302cc440cdaSPaolo Bonzini /* 1303cc440cdaSPaolo Bonzini * Processor state contains L2 state. Check that it is 1304cb9b6a1bSPaolo Bonzini * valid for guest mode (see nested_vmcb_check_save). 1305cc440cdaSPaolo Bonzini */ 1306cc440cdaSPaolo Bonzini cr0 = kvm_read_cr0(vcpu); 1307cc440cdaSPaolo Bonzini if (((cr0 & X86_CR0_CD) == 0) && (cr0 & X86_CR0_NW)) 13086ccbd29aSJoerg Roedel goto out_free; 1309cc440cdaSPaolo Bonzini 1310cc440cdaSPaolo Bonzini /* 1311cc440cdaSPaolo Bonzini * Validate host state saved from before VMRUN (see 1312cc440cdaSPaolo Bonzini * nested_svm_check_permissions). 1313cc440cdaSPaolo Bonzini */ 13146906e06dSKrish Sadhukhan if (!(save->cr0 & X86_CR0_PG) || 13156906e06dSKrish Sadhukhan !(save->cr0 & X86_CR0_PE) || 13166906e06dSKrish Sadhukhan (save->rflags & X86_EFLAGS_VM) || 131763129754SPaolo Bonzini !nested_vmcb_valid_sregs(vcpu, save)) 13186ccbd29aSJoerg Roedel goto out_free; 1319cc440cdaSPaolo Bonzini 1320cc440cdaSPaolo Bonzini /* 1321b222b0b8SMaxim Levitsky * While the nested guest CR3 is already checked and set by 1322b222b0b8SMaxim Levitsky * KVM_SET_SREGS, it was set when nested state was yet loaded, 1323b222b0b8SMaxim Levitsky * thus MMU might not be initialized correctly. 1324b222b0b8SMaxim Levitsky * Set it again to fix this. 1325b222b0b8SMaxim Levitsky */ 1326b222b0b8SMaxim Levitsky 1327b222b0b8SMaxim Levitsky ret = nested_svm_load_cr3(&svm->vcpu, vcpu->arch.cr3, 1328b222b0b8SMaxim Levitsky nested_npt_enabled(svm), false); 1329b222b0b8SMaxim Levitsky if (WARN_ON_ONCE(ret)) 1330b222b0b8SMaxim Levitsky goto out_free; 1331b222b0b8SMaxim Levitsky 1332b222b0b8SMaxim Levitsky 1333b222b0b8SMaxim Levitsky /* 13344995a368SCathy Avery * All checks done, we can enter guest mode. Userspace provides 13354995a368SCathy Avery * vmcb12.control, which will be combined with L1 and stored into 13364995a368SCathy Avery * vmcb02, and the L1 save state which we store in vmcb01. 13374995a368SCathy Avery * L2 registers if needed are moved from the current VMCB to VMCB02. 1338cc440cdaSPaolo Bonzini */ 133981f76adaSMaxim Levitsky 13409d290e16SMaxim Levitsky if (is_guest_mode(vcpu)) 13419d290e16SMaxim Levitsky svm_leave_nested(svm); 13429d290e16SMaxim Levitsky else 13439d290e16SMaxim Levitsky svm->nested.vmcb02.ptr->save = svm->vmcb01.ptr->save; 13449d290e16SMaxim Levitsky 1345063ab16cSMaxim Levitsky svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET)); 1346063ab16cSMaxim Levitsky 134781f76adaSMaxim Levitsky svm->nested.nested_run_pending = 134881f76adaSMaxim Levitsky !!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING); 134981f76adaSMaxim Levitsky 13500dd16b5bSMaxim Levitsky svm->nested.vmcb12_gpa = kvm_state->hdr.svm.vmcb_pa; 1351c08f390aSPaolo Bonzini 1352c08f390aSPaolo Bonzini svm->vmcb01.ptr->save.es = save->es; 1353c08f390aSPaolo Bonzini svm->vmcb01.ptr->save.cs = save->cs; 1354c08f390aSPaolo Bonzini svm->vmcb01.ptr->save.ss = save->ss; 1355c08f390aSPaolo Bonzini svm->vmcb01.ptr->save.ds = save->ds; 1356c08f390aSPaolo Bonzini svm->vmcb01.ptr->save.gdtr = save->gdtr; 1357c08f390aSPaolo Bonzini svm->vmcb01.ptr->save.idtr = save->idtr; 1358c08f390aSPaolo Bonzini svm->vmcb01.ptr->save.rflags = save->rflags | X86_EFLAGS_FIXED; 1359c08f390aSPaolo Bonzini svm->vmcb01.ptr->save.efer = save->efer; 1360c08f390aSPaolo Bonzini svm->vmcb01.ptr->save.cr0 = save->cr0; 1361c08f390aSPaolo Bonzini svm->vmcb01.ptr->save.cr3 = save->cr3; 1362c08f390aSPaolo Bonzini svm->vmcb01.ptr->save.cr4 = save->cr4; 1363c08f390aSPaolo Bonzini svm->vmcb01.ptr->save.rax = save->rax; 1364c08f390aSPaolo Bonzini svm->vmcb01.ptr->save.rsp = save->rsp; 1365c08f390aSPaolo Bonzini svm->vmcb01.ptr->save.rip = save->rip; 1366c08f390aSPaolo Bonzini svm->vmcb01.ptr->save.cpl = 0; 1367c08f390aSPaolo Bonzini 13689e8f0fbfSPaolo Bonzini nested_load_control_from_vmcb12(svm, ctl); 13694995a368SCathy Avery 13704995a368SCathy Avery svm_switch_vmcb(svm, &svm->nested.vmcb02); 13714995a368SCathy Avery 13729e8f0fbfSPaolo Bonzini nested_vmcb02_prepare_control(svm); 1373cc440cdaSPaolo Bonzini 1374a7d5c7ceSPaolo Bonzini kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 13756ccbd29aSJoerg Roedel ret = 0; 13766ccbd29aSJoerg Roedel out_free: 13776ccbd29aSJoerg Roedel kfree(save); 13786ccbd29aSJoerg Roedel kfree(ctl); 13796ccbd29aSJoerg Roedel 13806ccbd29aSJoerg Roedel return ret; 1381cc440cdaSPaolo Bonzini } 1382cc440cdaSPaolo Bonzini 1383232f75d3SMaxim Levitsky static bool svm_get_nested_state_pages(struct kvm_vcpu *vcpu) 1384232f75d3SMaxim Levitsky { 1385232f75d3SMaxim Levitsky struct vcpu_svm *svm = to_svm(vcpu); 1386232f75d3SMaxim Levitsky 1387232f75d3SMaxim Levitsky if (WARN_ON(!is_guest_mode(vcpu))) 1388232f75d3SMaxim Levitsky return true; 1389232f75d3SMaxim Levitsky 1390158a48ecSMaxim Levitsky if (!vcpu->arch.pdptrs_from_userspace && 1391158a48ecSMaxim Levitsky !nested_npt_enabled(svm) && is_pae_paging(vcpu)) 1392b222b0b8SMaxim Levitsky /* 1393b222b0b8SMaxim Levitsky * Reload the guest's PDPTRs since after a migration 1394b222b0b8SMaxim Levitsky * the guest CR3 might be restored prior to setting the nested 1395b222b0b8SMaxim Levitsky * state which can lead to a load of wrong PDPTRs. 1396b222b0b8SMaxim Levitsky */ 1397b222b0b8SMaxim Levitsky if (CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3))) 1398232f75d3SMaxim Levitsky return false; 1399232f75d3SMaxim Levitsky 1400232f75d3SMaxim Levitsky if (!nested_svm_vmrun_msrpm(svm)) { 1401232f75d3SMaxim Levitsky vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1402232f75d3SMaxim Levitsky vcpu->run->internal.suberror = 1403232f75d3SMaxim Levitsky KVM_INTERNAL_ERROR_EMULATION; 1404232f75d3SMaxim Levitsky vcpu->run->internal.ndata = 0; 1405232f75d3SMaxim Levitsky return false; 1406232f75d3SMaxim Levitsky } 1407232f75d3SMaxim Levitsky 1408232f75d3SMaxim Levitsky return true; 1409232f75d3SMaxim Levitsky } 1410232f75d3SMaxim Levitsky 141133b22172SPaolo Bonzini struct kvm_x86_nested_ops svm_nested_ops = { 141233b22172SPaolo Bonzini .check_events = svm_check_nested_events, 1413cb6a32c2SSean Christopherson .triple_fault = nested_svm_triple_fault, 1414a7d5c7ceSPaolo Bonzini .get_nested_state_pages = svm_get_nested_state_pages, 1415cc440cdaSPaolo Bonzini .get_state = svm_get_nested_state, 1416cc440cdaSPaolo Bonzini .set_state = svm_set_nested_state, 141733b22172SPaolo Bonzini }; 1418