1883b0a91SJoerg Roedel // SPDX-License-Identifier: GPL-2.0-only 2883b0a91SJoerg Roedel /* 3883b0a91SJoerg Roedel * Kernel-based Virtual Machine driver for Linux 4883b0a91SJoerg Roedel * 5883b0a91SJoerg Roedel * AMD SVM support 6883b0a91SJoerg Roedel * 7883b0a91SJoerg Roedel * Copyright (C) 2006 Qumranet, Inc. 8883b0a91SJoerg Roedel * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9883b0a91SJoerg Roedel * 10883b0a91SJoerg Roedel * Authors: 11883b0a91SJoerg Roedel * Yaniv Kamay <yaniv@qumranet.com> 12883b0a91SJoerg Roedel * Avi Kivity <avi@qumranet.com> 13883b0a91SJoerg Roedel */ 14883b0a91SJoerg Roedel 15883b0a91SJoerg Roedel #define pr_fmt(fmt) "SVM: " fmt 16883b0a91SJoerg Roedel 17883b0a91SJoerg Roedel #include <linux/kvm_types.h> 18883b0a91SJoerg Roedel #include <linux/kvm_host.h> 19883b0a91SJoerg Roedel #include <linux/kernel.h> 20883b0a91SJoerg Roedel 21883b0a91SJoerg Roedel #include <asm/msr-index.h> 225679b803SPaolo Bonzini #include <asm/debugreg.h> 23883b0a91SJoerg Roedel 24883b0a91SJoerg Roedel #include "kvm_emulate.h" 25883b0a91SJoerg Roedel #include "trace.h" 26883b0a91SJoerg Roedel #include "mmu.h" 27883b0a91SJoerg Roedel #include "x86.h" 28cc440cdaSPaolo Bonzini #include "cpuid.h" 295b672408SPaolo Bonzini #include "lapic.h" 30883b0a91SJoerg Roedel #include "svm.h" 31883b0a91SJoerg Roedel 32883b0a91SJoerg Roedel static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu, 33883b0a91SJoerg Roedel struct x86_exception *fault) 34883b0a91SJoerg Roedel { 35883b0a91SJoerg Roedel struct vcpu_svm *svm = to_svm(vcpu); 36883b0a91SJoerg Roedel 37883b0a91SJoerg Roedel if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) { 38883b0a91SJoerg Roedel /* 39883b0a91SJoerg Roedel * TODO: track the cause of the nested page fault, and 40883b0a91SJoerg Roedel * correctly fill in the high bits of exit_info_1. 41883b0a91SJoerg Roedel */ 42883b0a91SJoerg Roedel svm->vmcb->control.exit_code = SVM_EXIT_NPF; 43883b0a91SJoerg Roedel svm->vmcb->control.exit_code_hi = 0; 44883b0a91SJoerg Roedel svm->vmcb->control.exit_info_1 = (1ULL << 32); 45883b0a91SJoerg Roedel svm->vmcb->control.exit_info_2 = fault->address; 46883b0a91SJoerg Roedel } 47883b0a91SJoerg Roedel 48883b0a91SJoerg Roedel svm->vmcb->control.exit_info_1 &= ~0xffffffffULL; 49883b0a91SJoerg Roedel svm->vmcb->control.exit_info_1 |= fault->error_code; 50883b0a91SJoerg Roedel 51883b0a91SJoerg Roedel nested_svm_vmexit(svm); 52883b0a91SJoerg Roedel } 53883b0a91SJoerg Roedel 54883b0a91SJoerg Roedel static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index) 55883b0a91SJoerg Roedel { 56883b0a91SJoerg Roedel struct vcpu_svm *svm = to_svm(vcpu); 57e670bf68SPaolo Bonzini u64 cr3 = svm->nested.ctl.nested_cr3; 58883b0a91SJoerg Roedel u64 pdpte; 59883b0a91SJoerg Roedel int ret; 60883b0a91SJoerg Roedel 61883b0a91SJoerg Roedel ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(__sme_clr(cr3)), &pdpte, 62883b0a91SJoerg Roedel offset_in_page(cr3) + index * 8, 8); 63883b0a91SJoerg Roedel if (ret) 64883b0a91SJoerg Roedel return 0; 65883b0a91SJoerg Roedel return pdpte; 66883b0a91SJoerg Roedel } 67883b0a91SJoerg Roedel 68883b0a91SJoerg Roedel static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu) 69883b0a91SJoerg Roedel { 70883b0a91SJoerg Roedel struct vcpu_svm *svm = to_svm(vcpu); 71883b0a91SJoerg Roedel 72e670bf68SPaolo Bonzini return svm->nested.ctl.nested_cr3; 73883b0a91SJoerg Roedel } 74883b0a91SJoerg Roedel 75883b0a91SJoerg Roedel static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu) 76883b0a91SJoerg Roedel { 77929d1cfaSPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 78929d1cfaSPaolo Bonzini struct vmcb *hsave = svm->nested.hsave; 79929d1cfaSPaolo Bonzini 80883b0a91SJoerg Roedel WARN_ON(mmu_is_nested(vcpu)); 81883b0a91SJoerg Roedel 82883b0a91SJoerg Roedel vcpu->arch.mmu = &vcpu->arch.guest_mmu; 830f04a2acSVitaly Kuznetsov kvm_init_shadow_npt_mmu(vcpu, X86_CR0_PG, hsave->save.cr4, hsave->save.efer, 840f04a2acSVitaly Kuznetsov svm->nested.ctl.nested_cr3); 85883b0a91SJoerg Roedel vcpu->arch.mmu->get_guest_pgd = nested_svm_get_tdp_cr3; 86883b0a91SJoerg Roedel vcpu->arch.mmu->get_pdptr = nested_svm_get_tdp_pdptr; 87883b0a91SJoerg Roedel vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit; 88883b0a91SJoerg Roedel reset_shadow_zero_bits_mask(vcpu, vcpu->arch.mmu); 89883b0a91SJoerg Roedel vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; 90883b0a91SJoerg Roedel } 91883b0a91SJoerg Roedel 92883b0a91SJoerg Roedel static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu) 93883b0a91SJoerg Roedel { 94883b0a91SJoerg Roedel vcpu->arch.mmu = &vcpu->arch.root_mmu; 95883b0a91SJoerg Roedel vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; 96883b0a91SJoerg Roedel } 97883b0a91SJoerg Roedel 98883b0a91SJoerg Roedel void recalc_intercepts(struct vcpu_svm *svm) 99883b0a91SJoerg Roedel { 100e670bf68SPaolo Bonzini struct vmcb_control_area *c, *h, *g; 101c45ad722SBabu Moger unsigned int i; 102883b0a91SJoerg Roedel 10306e7852cSJoerg Roedel vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 104883b0a91SJoerg Roedel 105883b0a91SJoerg Roedel if (!is_guest_mode(&svm->vcpu)) 106883b0a91SJoerg Roedel return; 107883b0a91SJoerg Roedel 108883b0a91SJoerg Roedel c = &svm->vmcb->control; 109883b0a91SJoerg Roedel h = &svm->nested.hsave->control; 110e670bf68SPaolo Bonzini g = &svm->nested.ctl; 111883b0a91SJoerg Roedel 112c45ad722SBabu Moger for (i = 0; i < MAX_INTERCEPT; i++) 113c45ad722SBabu Moger c->intercepts[i] = h->intercepts[i]; 114c45ad722SBabu Moger 115883b0a91SJoerg Roedel c->intercept_dr = h->intercept_dr; 116883b0a91SJoerg Roedel c->intercept_exceptions = h->intercept_exceptions; 117883b0a91SJoerg Roedel c->intercept = h->intercept; 118883b0a91SJoerg Roedel 119e9fd761aSPaolo Bonzini if (g->int_ctl & V_INTR_MASKING_MASK) { 120883b0a91SJoerg Roedel /* We only want the cr8 intercept bits of L1 */ 121*03bfeeb9SBabu Moger vmcb_clr_intercept(c, INTERCEPT_CR8_READ); 122*03bfeeb9SBabu Moger vmcb_clr_intercept(c, INTERCEPT_CR8_WRITE); 123883b0a91SJoerg Roedel 124883b0a91SJoerg Roedel /* 125883b0a91SJoerg Roedel * Once running L2 with HF_VINTR_MASK, EFLAGS.IF does not 126883b0a91SJoerg Roedel * affect any interrupt we may want to inject; therefore, 127883b0a91SJoerg Roedel * interrupt window vmexits are irrelevant to L0. 128883b0a91SJoerg Roedel */ 129883b0a91SJoerg Roedel c->intercept &= ~(1ULL << INTERCEPT_VINTR); 130883b0a91SJoerg Roedel } 131883b0a91SJoerg Roedel 132883b0a91SJoerg Roedel /* We don't want to see VMMCALLs from a nested guest */ 133883b0a91SJoerg Roedel c->intercept &= ~(1ULL << INTERCEPT_VMMCALL); 134883b0a91SJoerg Roedel 135c45ad722SBabu Moger for (i = 0; i < MAX_INTERCEPT; i++) 136c45ad722SBabu Moger c->intercepts[i] |= g->intercepts[i]; 137c45ad722SBabu Moger 138883b0a91SJoerg Roedel c->intercept_dr |= g->intercept_dr; 139883b0a91SJoerg Roedel c->intercept_exceptions |= g->intercept_exceptions; 140883b0a91SJoerg Roedel c->intercept |= g->intercept; 141883b0a91SJoerg Roedel } 142883b0a91SJoerg Roedel 1432f675917SPaolo Bonzini static void copy_vmcb_control_area(struct vmcb_control_area *dst, 1442f675917SPaolo Bonzini struct vmcb_control_area *from) 145883b0a91SJoerg Roedel { 146c45ad722SBabu Moger unsigned int i; 147c45ad722SBabu Moger 148c45ad722SBabu Moger for (i = 0; i < MAX_INTERCEPT; i++) 149c45ad722SBabu Moger dst->intercepts[i] = from->intercepts[i]; 150c45ad722SBabu Moger 151883b0a91SJoerg Roedel dst->intercept_dr = from->intercept_dr; 152883b0a91SJoerg Roedel dst->intercept_exceptions = from->intercept_exceptions; 153883b0a91SJoerg Roedel dst->intercept = from->intercept; 154883b0a91SJoerg Roedel dst->iopm_base_pa = from->iopm_base_pa; 155883b0a91SJoerg Roedel dst->msrpm_base_pa = from->msrpm_base_pa; 156883b0a91SJoerg Roedel dst->tsc_offset = from->tsc_offset; 1576c0238c4SPaolo Bonzini /* asid not copied, it is handled manually for svm->vmcb. */ 158883b0a91SJoerg Roedel dst->tlb_ctl = from->tlb_ctl; 159883b0a91SJoerg Roedel dst->int_ctl = from->int_ctl; 160883b0a91SJoerg Roedel dst->int_vector = from->int_vector; 161883b0a91SJoerg Roedel dst->int_state = from->int_state; 162883b0a91SJoerg Roedel dst->exit_code = from->exit_code; 163883b0a91SJoerg Roedel dst->exit_code_hi = from->exit_code_hi; 164883b0a91SJoerg Roedel dst->exit_info_1 = from->exit_info_1; 165883b0a91SJoerg Roedel dst->exit_info_2 = from->exit_info_2; 166883b0a91SJoerg Roedel dst->exit_int_info = from->exit_int_info; 167883b0a91SJoerg Roedel dst->exit_int_info_err = from->exit_int_info_err; 168883b0a91SJoerg Roedel dst->nested_ctl = from->nested_ctl; 169883b0a91SJoerg Roedel dst->event_inj = from->event_inj; 170883b0a91SJoerg Roedel dst->event_inj_err = from->event_inj_err; 171883b0a91SJoerg Roedel dst->nested_cr3 = from->nested_cr3; 172883b0a91SJoerg Roedel dst->virt_ext = from->virt_ext; 173883b0a91SJoerg Roedel dst->pause_filter_count = from->pause_filter_count; 174883b0a91SJoerg Roedel dst->pause_filter_thresh = from->pause_filter_thresh; 175883b0a91SJoerg Roedel } 176883b0a91SJoerg Roedel 177883b0a91SJoerg Roedel static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm) 178883b0a91SJoerg Roedel { 179883b0a91SJoerg Roedel /* 180883b0a91SJoerg Roedel * This function merges the msr permission bitmaps of kvm and the 181883b0a91SJoerg Roedel * nested vmcb. It is optimized in that it only merges the parts where 182883b0a91SJoerg Roedel * the kvm msr permission bitmap may contain zero bits 183883b0a91SJoerg Roedel */ 184883b0a91SJoerg Roedel int i; 185883b0a91SJoerg Roedel 186e670bf68SPaolo Bonzini if (!(svm->nested.ctl.intercept & (1ULL << INTERCEPT_MSR_PROT))) 187883b0a91SJoerg Roedel return true; 188883b0a91SJoerg Roedel 189883b0a91SJoerg Roedel for (i = 0; i < MSRPM_OFFSETS; i++) { 190883b0a91SJoerg Roedel u32 value, p; 191883b0a91SJoerg Roedel u64 offset; 192883b0a91SJoerg Roedel 193883b0a91SJoerg Roedel if (msrpm_offsets[i] == 0xffffffff) 194883b0a91SJoerg Roedel break; 195883b0a91SJoerg Roedel 196883b0a91SJoerg Roedel p = msrpm_offsets[i]; 197e670bf68SPaolo Bonzini offset = svm->nested.ctl.msrpm_base_pa + (p * 4); 198883b0a91SJoerg Roedel 199883b0a91SJoerg Roedel if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4)) 200883b0a91SJoerg Roedel return false; 201883b0a91SJoerg Roedel 202883b0a91SJoerg Roedel svm->nested.msrpm[p] = svm->msrpm[p] | value; 203883b0a91SJoerg Roedel } 204883b0a91SJoerg Roedel 205883b0a91SJoerg Roedel svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm)); 206883b0a91SJoerg Roedel 207883b0a91SJoerg Roedel return true; 208883b0a91SJoerg Roedel } 209883b0a91SJoerg Roedel 210ca46d739SPaolo Bonzini static bool nested_vmcb_check_controls(struct vmcb_control_area *control) 211ca46d739SPaolo Bonzini { 212ca46d739SPaolo Bonzini if ((control->intercept & (1ULL << INTERCEPT_VMRUN)) == 0) 213ca46d739SPaolo Bonzini return false; 214ca46d739SPaolo Bonzini 215ca46d739SPaolo Bonzini if (control->asid == 0) 216ca46d739SPaolo Bonzini return false; 217ca46d739SPaolo Bonzini 218ca46d739SPaolo Bonzini if ((control->nested_ctl & SVM_NESTED_CTL_NP_ENABLE) && 219ca46d739SPaolo Bonzini !npt_enabled) 220ca46d739SPaolo Bonzini return false; 221ca46d739SPaolo Bonzini 222ca46d739SPaolo Bonzini return true; 223ca46d739SPaolo Bonzini } 224ca46d739SPaolo Bonzini 2250dd16b5bSMaxim Levitsky static bool nested_vmcb_checks(struct vcpu_svm *svm, struct vmcb *vmcb12) 226883b0a91SJoerg Roedel { 2270dd16b5bSMaxim Levitsky bool vmcb12_lma; 2280dd16b5bSMaxim Levitsky 2290dd16b5bSMaxim Levitsky if ((vmcb12->save.efer & EFER_SVME) == 0) 230883b0a91SJoerg Roedel return false; 231883b0a91SJoerg Roedel 2320dd16b5bSMaxim Levitsky if (((vmcb12->save.cr0 & X86_CR0_CD) == 0) && (vmcb12->save.cr0 & X86_CR0_NW)) 2334f233371SKrish Sadhukhan return false; 2344f233371SKrish Sadhukhan 2350dd16b5bSMaxim Levitsky if (!kvm_dr6_valid(vmcb12->save.dr6) || !kvm_dr7_valid(vmcb12->save.dr7)) 2361aef8161SKrish Sadhukhan return false; 2371aef8161SKrish Sadhukhan 2380dd16b5bSMaxim Levitsky vmcb12_lma = (vmcb12->save.efer & EFER_LME) && (vmcb12->save.cr0 & X86_CR0_PG); 239761e4169SKrish Sadhukhan 2400dd16b5bSMaxim Levitsky if (!vmcb12_lma) { 2410dd16b5bSMaxim Levitsky if (vmcb12->save.cr4 & X86_CR4_PAE) { 2420dd16b5bSMaxim Levitsky if (vmcb12->save.cr3 & MSR_CR3_LEGACY_PAE_RESERVED_MASK) 243761e4169SKrish Sadhukhan return false; 244761e4169SKrish Sadhukhan } else { 2450dd16b5bSMaxim Levitsky if (vmcb12->save.cr3 & MSR_CR3_LEGACY_RESERVED_MASK) 246761e4169SKrish Sadhukhan return false; 247761e4169SKrish Sadhukhan } 248761e4169SKrish Sadhukhan } else { 2490dd16b5bSMaxim Levitsky if (!(vmcb12->save.cr4 & X86_CR4_PAE) || 2500dd16b5bSMaxim Levitsky !(vmcb12->save.cr0 & X86_CR0_PE) || 2510dd16b5bSMaxim Levitsky (vmcb12->save.cr3 & MSR_CR3_LONG_RESERVED_MASK)) 252761e4169SKrish Sadhukhan return false; 253761e4169SKrish Sadhukhan } 2540dd16b5bSMaxim Levitsky if (kvm_valid_cr4(&svm->vcpu, vmcb12->save.cr4)) 255761e4169SKrish Sadhukhan return false; 256761e4169SKrish Sadhukhan 2570dd16b5bSMaxim Levitsky return nested_vmcb_check_controls(&vmcb12->control); 258883b0a91SJoerg Roedel } 259883b0a91SJoerg Roedel 2603e06f016SPaolo Bonzini static void load_nested_vmcb_control(struct vcpu_svm *svm, 2613e06f016SPaolo Bonzini struct vmcb_control_area *control) 2623e06f016SPaolo Bonzini { 263e670bf68SPaolo Bonzini copy_vmcb_control_area(&svm->nested.ctl, control); 2643e06f016SPaolo Bonzini 265cc440cdaSPaolo Bonzini /* Copy it here because nested_svm_check_controls will check it. */ 266cc440cdaSPaolo Bonzini svm->nested.ctl.asid = control->asid; 267e670bf68SPaolo Bonzini svm->nested.ctl.msrpm_base_pa &= ~0x0fffULL; 268e670bf68SPaolo Bonzini svm->nested.ctl.iopm_base_pa &= ~0x0fffULL; 2693e06f016SPaolo Bonzini } 2703e06f016SPaolo Bonzini 2712d8a42beSPaolo Bonzini /* 2722d8a42beSPaolo Bonzini * Synchronize fields that are written by the processor, so that 2732d8a42beSPaolo Bonzini * they can be copied back into the nested_vmcb. 2742d8a42beSPaolo Bonzini */ 2752d8a42beSPaolo Bonzini void sync_nested_vmcb_control(struct vcpu_svm *svm) 2762d8a42beSPaolo Bonzini { 2772d8a42beSPaolo Bonzini u32 mask; 2782d8a42beSPaolo Bonzini svm->nested.ctl.event_inj = svm->vmcb->control.event_inj; 2792d8a42beSPaolo Bonzini svm->nested.ctl.event_inj_err = svm->vmcb->control.event_inj_err; 2802d8a42beSPaolo Bonzini 2812d8a42beSPaolo Bonzini /* Only a few fields of int_ctl are written by the processor. */ 2822d8a42beSPaolo Bonzini mask = V_IRQ_MASK | V_TPR_MASK; 2832d8a42beSPaolo Bonzini if (!(svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK) && 284a284ba56SJoerg Roedel svm_is_intercept(svm, INTERCEPT_VINTR)) { 2852d8a42beSPaolo Bonzini /* 2862d8a42beSPaolo Bonzini * In order to request an interrupt window, L0 is usurping 2872d8a42beSPaolo Bonzini * svm->vmcb->control.int_ctl and possibly setting V_IRQ 2882d8a42beSPaolo Bonzini * even if it was clear in L1's VMCB. Restoring it would be 2892d8a42beSPaolo Bonzini * wrong. However, in this case V_IRQ will remain true until 2902d8a42beSPaolo Bonzini * interrupt_window_interception calls svm_clear_vintr and 2912d8a42beSPaolo Bonzini * restores int_ctl. We can just leave it aside. 2922d8a42beSPaolo Bonzini */ 2932d8a42beSPaolo Bonzini mask &= ~V_IRQ_MASK; 2942d8a42beSPaolo Bonzini } 2952d8a42beSPaolo Bonzini svm->nested.ctl.int_ctl &= ~mask; 2962d8a42beSPaolo Bonzini svm->nested.ctl.int_ctl |= svm->vmcb->control.int_ctl & mask; 2972d8a42beSPaolo Bonzini } 2982d8a42beSPaolo Bonzini 29936e2e983SPaolo Bonzini /* 30036e2e983SPaolo Bonzini * Transfer any event that L0 or L1 wanted to inject into L2 to 30136e2e983SPaolo Bonzini * EXIT_INT_INFO. 30236e2e983SPaolo Bonzini */ 30336e2e983SPaolo Bonzini static void nested_vmcb_save_pending_event(struct vcpu_svm *svm, 3040dd16b5bSMaxim Levitsky struct vmcb *vmcb12) 30536e2e983SPaolo Bonzini { 30636e2e983SPaolo Bonzini struct kvm_vcpu *vcpu = &svm->vcpu; 30736e2e983SPaolo Bonzini u32 exit_int_info = 0; 30836e2e983SPaolo Bonzini unsigned int nr; 30936e2e983SPaolo Bonzini 31036e2e983SPaolo Bonzini if (vcpu->arch.exception.injected) { 31136e2e983SPaolo Bonzini nr = vcpu->arch.exception.nr; 31236e2e983SPaolo Bonzini exit_int_info = nr | SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_EXEPT; 31336e2e983SPaolo Bonzini 31436e2e983SPaolo Bonzini if (vcpu->arch.exception.has_error_code) { 31536e2e983SPaolo Bonzini exit_int_info |= SVM_EVTINJ_VALID_ERR; 3160dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info_err = 31736e2e983SPaolo Bonzini vcpu->arch.exception.error_code; 31836e2e983SPaolo Bonzini } 31936e2e983SPaolo Bonzini 32036e2e983SPaolo Bonzini } else if (vcpu->arch.nmi_injected) { 32136e2e983SPaolo Bonzini exit_int_info = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI; 32236e2e983SPaolo Bonzini 32336e2e983SPaolo Bonzini } else if (vcpu->arch.interrupt.injected) { 32436e2e983SPaolo Bonzini nr = vcpu->arch.interrupt.nr; 32536e2e983SPaolo Bonzini exit_int_info = nr | SVM_EVTINJ_VALID; 32636e2e983SPaolo Bonzini 32736e2e983SPaolo Bonzini if (vcpu->arch.interrupt.soft) 32836e2e983SPaolo Bonzini exit_int_info |= SVM_EVTINJ_TYPE_SOFT; 32936e2e983SPaolo Bonzini else 33036e2e983SPaolo Bonzini exit_int_info |= SVM_EVTINJ_TYPE_INTR; 33136e2e983SPaolo Bonzini } 33236e2e983SPaolo Bonzini 3330dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info = exit_int_info; 33436e2e983SPaolo Bonzini } 33536e2e983SPaolo Bonzini 33662156f6cSVitaly Kuznetsov static inline bool nested_npt_enabled(struct vcpu_svm *svm) 33762156f6cSVitaly Kuznetsov { 33862156f6cSVitaly Kuznetsov return svm->nested.ctl.nested_ctl & SVM_NESTED_CTL_NP_ENABLE; 33962156f6cSVitaly Kuznetsov } 34062156f6cSVitaly Kuznetsov 34162156f6cSVitaly Kuznetsov /* 342d82aaef9SVitaly Kuznetsov * Load guest's/host's cr3 on nested vmentry or vmexit. @nested_npt is true 343d82aaef9SVitaly Kuznetsov * if we are emulating VM-Entry into a guest with NPT enabled. 34462156f6cSVitaly Kuznetsov */ 34562156f6cSVitaly Kuznetsov static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, 34662156f6cSVitaly Kuznetsov bool nested_npt) 34762156f6cSVitaly Kuznetsov { 348a506fdd2SVitaly Kuznetsov if (cr3 & rsvd_bits(cpuid_maxphyaddr(vcpu), 63)) 349a506fdd2SVitaly Kuznetsov return -EINVAL; 350a506fdd2SVitaly Kuznetsov 351a506fdd2SVitaly Kuznetsov if (!nested_npt && is_pae_paging(vcpu) && 352a506fdd2SVitaly Kuznetsov (cr3 != kvm_read_cr3(vcpu) || pdptrs_changed(vcpu))) { 353a506fdd2SVitaly Kuznetsov if (!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3)) 354a506fdd2SVitaly Kuznetsov return -EINVAL; 355a506fdd2SVitaly Kuznetsov } 356a506fdd2SVitaly Kuznetsov 357a506fdd2SVitaly Kuznetsov /* 358a506fdd2SVitaly Kuznetsov * TODO: optimize unconditional TLB flush/MMU sync here and in 359a506fdd2SVitaly Kuznetsov * kvm_init_shadow_npt_mmu(). 360a506fdd2SVitaly Kuznetsov */ 361a506fdd2SVitaly Kuznetsov if (!nested_npt) 362a506fdd2SVitaly Kuznetsov kvm_mmu_new_pgd(vcpu, cr3, false, false); 363a506fdd2SVitaly Kuznetsov 364a506fdd2SVitaly Kuznetsov vcpu->arch.cr3 = cr3; 365a506fdd2SVitaly Kuznetsov kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); 366a506fdd2SVitaly Kuznetsov 367a506fdd2SVitaly Kuznetsov kvm_init_mmu(vcpu, false); 368a506fdd2SVitaly Kuznetsov 369a506fdd2SVitaly Kuznetsov return 0; 37062156f6cSVitaly Kuznetsov } 37162156f6cSVitaly Kuznetsov 3720dd16b5bSMaxim Levitsky static void nested_prepare_vmcb_save(struct vcpu_svm *svm, struct vmcb *vmcb12) 373883b0a91SJoerg Roedel { 374883b0a91SJoerg Roedel /* Load the nested guest state */ 3750dd16b5bSMaxim Levitsky svm->vmcb->save.es = vmcb12->save.es; 3760dd16b5bSMaxim Levitsky svm->vmcb->save.cs = vmcb12->save.cs; 3770dd16b5bSMaxim Levitsky svm->vmcb->save.ss = vmcb12->save.ss; 3780dd16b5bSMaxim Levitsky svm->vmcb->save.ds = vmcb12->save.ds; 3790dd16b5bSMaxim Levitsky svm->vmcb->save.gdtr = vmcb12->save.gdtr; 3800dd16b5bSMaxim Levitsky svm->vmcb->save.idtr = vmcb12->save.idtr; 3810dd16b5bSMaxim Levitsky kvm_set_rflags(&svm->vcpu, vmcb12->save.rflags); 3820dd16b5bSMaxim Levitsky svm_set_efer(&svm->vcpu, vmcb12->save.efer); 3830dd16b5bSMaxim Levitsky svm_set_cr0(&svm->vcpu, vmcb12->save.cr0); 3840dd16b5bSMaxim Levitsky svm_set_cr4(&svm->vcpu, vmcb12->save.cr4); 3850dd16b5bSMaxim Levitsky svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = vmcb12->save.cr2; 3860dd16b5bSMaxim Levitsky kvm_rax_write(&svm->vcpu, vmcb12->save.rax); 3870dd16b5bSMaxim Levitsky kvm_rsp_write(&svm->vcpu, vmcb12->save.rsp); 3880dd16b5bSMaxim Levitsky kvm_rip_write(&svm->vcpu, vmcb12->save.rip); 389883b0a91SJoerg Roedel 390883b0a91SJoerg Roedel /* In case we don't even reach vcpu_run, the fields are not updated */ 3910dd16b5bSMaxim Levitsky svm->vmcb->save.rax = vmcb12->save.rax; 3920dd16b5bSMaxim Levitsky svm->vmcb->save.rsp = vmcb12->save.rsp; 3930dd16b5bSMaxim Levitsky svm->vmcb->save.rip = vmcb12->save.rip; 3940dd16b5bSMaxim Levitsky svm->vmcb->save.dr7 = vmcb12->save.dr7; 3950dd16b5bSMaxim Levitsky svm->vcpu.arch.dr6 = vmcb12->save.dr6; 3960dd16b5bSMaxim Levitsky svm->vmcb->save.cpl = vmcb12->save.cpl; 397f241d711SPaolo Bonzini } 398883b0a91SJoerg Roedel 399e670bf68SPaolo Bonzini static void nested_prepare_vmcb_control(struct vcpu_svm *svm) 400f241d711SPaolo Bonzini { 40191b7130cSPaolo Bonzini const u32 mask = V_INTR_MASKING_MASK | V_GIF_ENABLE_MASK | V_GIF_MASK; 40262156f6cSVitaly Kuznetsov 40362156f6cSVitaly Kuznetsov if (nested_npt_enabled(svm)) 40469cb8774SPaolo Bonzini nested_svm_init_mmu_context(&svm->vcpu); 40569cb8774SPaolo Bonzini 40618fc6c55SPaolo Bonzini svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset = 407e670bf68SPaolo Bonzini svm->vcpu.arch.l1_tsc_offset + svm->nested.ctl.tsc_offset; 408883b0a91SJoerg Roedel 40991b7130cSPaolo Bonzini svm->vmcb->control.int_ctl = 41091b7130cSPaolo Bonzini (svm->nested.ctl.int_ctl & ~mask) | 41191b7130cSPaolo Bonzini (svm->nested.hsave->control.int_ctl & mask); 41291b7130cSPaolo Bonzini 413e670bf68SPaolo Bonzini svm->vmcb->control.virt_ext = svm->nested.ctl.virt_ext; 414e670bf68SPaolo Bonzini svm->vmcb->control.int_vector = svm->nested.ctl.int_vector; 415e670bf68SPaolo Bonzini svm->vmcb->control.int_state = svm->nested.ctl.int_state; 416e670bf68SPaolo Bonzini svm->vmcb->control.event_inj = svm->nested.ctl.event_inj; 417e670bf68SPaolo Bonzini svm->vmcb->control.event_inj_err = svm->nested.ctl.event_inj_err; 418883b0a91SJoerg Roedel 419e670bf68SPaolo Bonzini svm->vmcb->control.pause_filter_count = svm->nested.ctl.pause_filter_count; 420e670bf68SPaolo Bonzini svm->vmcb->control.pause_filter_thresh = svm->nested.ctl.pause_filter_thresh; 421883b0a91SJoerg Roedel 422883b0a91SJoerg Roedel /* Enter Guest-Mode */ 423883b0a91SJoerg Roedel enter_guest_mode(&svm->vcpu); 424883b0a91SJoerg Roedel 425883b0a91SJoerg Roedel /* 426883b0a91SJoerg Roedel * Merge guest and host intercepts - must be called with vcpu in 427883b0a91SJoerg Roedel * guest-mode to take affect here 428883b0a91SJoerg Roedel */ 429883b0a91SJoerg Roedel recalc_intercepts(svm); 430883b0a91SJoerg Roedel 43106e7852cSJoerg Roedel vmcb_mark_all_dirty(svm->vmcb); 432f241d711SPaolo Bonzini } 433f241d711SPaolo Bonzini 4340dd16b5bSMaxim Levitsky int enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb12_gpa, 4350dd16b5bSMaxim Levitsky struct vmcb *vmcb12) 436f241d711SPaolo Bonzini { 437a506fdd2SVitaly Kuznetsov int ret; 438a506fdd2SVitaly Kuznetsov 4390dd16b5bSMaxim Levitsky svm->nested.vmcb12_gpa = vmcb12_gpa; 4400dd16b5bSMaxim Levitsky load_nested_vmcb_control(svm, &vmcb12->control); 4410dd16b5bSMaxim Levitsky nested_prepare_vmcb_save(svm, vmcb12); 442e670bf68SPaolo Bonzini nested_prepare_vmcb_control(svm); 443f241d711SPaolo Bonzini 4440dd16b5bSMaxim Levitsky ret = nested_svm_load_cr3(&svm->vcpu, vmcb12->save.cr3, 445a506fdd2SVitaly Kuznetsov nested_npt_enabled(svm)); 446a506fdd2SVitaly Kuznetsov if (ret) 447a506fdd2SVitaly Kuznetsov return ret; 448a506fdd2SVitaly Kuznetsov 449ffdf7f9eSPaolo Bonzini svm_set_gif(svm, true); 45059cd9bc5SVitaly Kuznetsov 45159cd9bc5SVitaly Kuznetsov return 0; 452883b0a91SJoerg Roedel } 453883b0a91SJoerg Roedel 454883b0a91SJoerg Roedel int nested_svm_vmrun(struct vcpu_svm *svm) 455883b0a91SJoerg Roedel { 456883b0a91SJoerg Roedel int ret; 4570dd16b5bSMaxim Levitsky struct vmcb *vmcb12; 458883b0a91SJoerg Roedel struct vmcb *hsave = svm->nested.hsave; 459883b0a91SJoerg Roedel struct vmcb *vmcb = svm->vmcb; 460883b0a91SJoerg Roedel struct kvm_host_map map; 4610dd16b5bSMaxim Levitsky u64 vmcb12_gpa; 462883b0a91SJoerg Roedel 4637c67f546SPaolo Bonzini if (is_smm(&svm->vcpu)) { 4647c67f546SPaolo Bonzini kvm_queue_exception(&svm->vcpu, UD_VECTOR); 4657c67f546SPaolo Bonzini return 1; 4667c67f546SPaolo Bonzini } 467883b0a91SJoerg Roedel 4680dd16b5bSMaxim Levitsky vmcb12_gpa = svm->vmcb->save.rax; 4690dd16b5bSMaxim Levitsky ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(vmcb12_gpa), &map); 470883b0a91SJoerg Roedel if (ret == -EINVAL) { 471883b0a91SJoerg Roedel kvm_inject_gp(&svm->vcpu, 0); 472883b0a91SJoerg Roedel return 1; 473883b0a91SJoerg Roedel } else if (ret) { 474883b0a91SJoerg Roedel return kvm_skip_emulated_instruction(&svm->vcpu); 475883b0a91SJoerg Roedel } 476883b0a91SJoerg Roedel 477883b0a91SJoerg Roedel ret = kvm_skip_emulated_instruction(&svm->vcpu); 478883b0a91SJoerg Roedel 4790dd16b5bSMaxim Levitsky vmcb12 = map.hva; 480883b0a91SJoerg Roedel 4810dd16b5bSMaxim Levitsky if (!nested_vmcb_checks(svm, vmcb12)) { 4820dd16b5bSMaxim Levitsky vmcb12->control.exit_code = SVM_EXIT_ERR; 4830dd16b5bSMaxim Levitsky vmcb12->control.exit_code_hi = 0; 4840dd16b5bSMaxim Levitsky vmcb12->control.exit_info_1 = 0; 4850dd16b5bSMaxim Levitsky vmcb12->control.exit_info_2 = 0; 48669c9dfa2SPaolo Bonzini goto out; 487883b0a91SJoerg Roedel } 488883b0a91SJoerg Roedel 4890dd16b5bSMaxim Levitsky trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb12_gpa, 4900dd16b5bSMaxim Levitsky vmcb12->save.rip, 4910dd16b5bSMaxim Levitsky vmcb12->control.int_ctl, 4920dd16b5bSMaxim Levitsky vmcb12->control.event_inj, 4930dd16b5bSMaxim Levitsky vmcb12->control.nested_ctl); 494883b0a91SJoerg Roedel 495*03bfeeb9SBabu Moger trace_kvm_nested_intercepts(vmcb12->control.intercepts[INTERCEPT_CR] & 0xffff, 496*03bfeeb9SBabu Moger vmcb12->control.intercepts[INTERCEPT_CR] >> 16, 4970dd16b5bSMaxim Levitsky vmcb12->control.intercept_exceptions, 4980dd16b5bSMaxim Levitsky vmcb12->control.intercept); 499883b0a91SJoerg Roedel 500883b0a91SJoerg Roedel /* Clear internal status */ 501883b0a91SJoerg Roedel kvm_clear_exception_queue(&svm->vcpu); 502883b0a91SJoerg Roedel kvm_clear_interrupt_queue(&svm->vcpu); 503883b0a91SJoerg Roedel 504883b0a91SJoerg Roedel /* 505883b0a91SJoerg Roedel * Save the old vmcb, so we don't need to pick what we save, but can 506883b0a91SJoerg Roedel * restore everything when a VMEXIT occurs 507883b0a91SJoerg Roedel */ 508883b0a91SJoerg Roedel hsave->save.es = vmcb->save.es; 509883b0a91SJoerg Roedel hsave->save.cs = vmcb->save.cs; 510883b0a91SJoerg Roedel hsave->save.ss = vmcb->save.ss; 511883b0a91SJoerg Roedel hsave->save.ds = vmcb->save.ds; 512883b0a91SJoerg Roedel hsave->save.gdtr = vmcb->save.gdtr; 513883b0a91SJoerg Roedel hsave->save.idtr = vmcb->save.idtr; 514883b0a91SJoerg Roedel hsave->save.efer = svm->vcpu.arch.efer; 515883b0a91SJoerg Roedel hsave->save.cr0 = kvm_read_cr0(&svm->vcpu); 516883b0a91SJoerg Roedel hsave->save.cr4 = svm->vcpu.arch.cr4; 517883b0a91SJoerg Roedel hsave->save.rflags = kvm_get_rflags(&svm->vcpu); 518883b0a91SJoerg Roedel hsave->save.rip = kvm_rip_read(&svm->vcpu); 519883b0a91SJoerg Roedel hsave->save.rsp = vmcb->save.rsp; 520883b0a91SJoerg Roedel hsave->save.rax = vmcb->save.rax; 521883b0a91SJoerg Roedel if (npt_enabled) 522883b0a91SJoerg Roedel hsave->save.cr3 = vmcb->save.cr3; 523883b0a91SJoerg Roedel else 524883b0a91SJoerg Roedel hsave->save.cr3 = kvm_read_cr3(&svm->vcpu); 525883b0a91SJoerg Roedel 5262f675917SPaolo Bonzini copy_vmcb_control_area(&hsave->control, &vmcb->control); 527883b0a91SJoerg Roedel 528f74f9414SPaolo Bonzini svm->nested.nested_run_pending = 1; 529883b0a91SJoerg Roedel 5300dd16b5bSMaxim Levitsky if (enter_svm_guest_mode(svm, vmcb12_gpa, vmcb12)) 53159cd9bc5SVitaly Kuznetsov goto out_exit_err; 53259cd9bc5SVitaly Kuznetsov 53359cd9bc5SVitaly Kuznetsov if (nested_svm_vmrun_msrpm(svm)) 53459cd9bc5SVitaly Kuznetsov goto out; 53559cd9bc5SVitaly Kuznetsov 53659cd9bc5SVitaly Kuznetsov out_exit_err: 537ebdb3dbaSVitaly Kuznetsov svm->nested.nested_run_pending = 0; 538ebdb3dbaSVitaly Kuznetsov 539883b0a91SJoerg Roedel svm->vmcb->control.exit_code = SVM_EXIT_ERR; 540883b0a91SJoerg Roedel svm->vmcb->control.exit_code_hi = 0; 541883b0a91SJoerg Roedel svm->vmcb->control.exit_info_1 = 0; 542883b0a91SJoerg Roedel svm->vmcb->control.exit_info_2 = 0; 543883b0a91SJoerg Roedel 544883b0a91SJoerg Roedel nested_svm_vmexit(svm); 545883b0a91SJoerg Roedel 54669c9dfa2SPaolo Bonzini out: 54769c9dfa2SPaolo Bonzini kvm_vcpu_unmap(&svm->vcpu, &map, true); 54869c9dfa2SPaolo Bonzini 549883b0a91SJoerg Roedel return ret; 550883b0a91SJoerg Roedel } 551883b0a91SJoerg Roedel 552883b0a91SJoerg Roedel void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb) 553883b0a91SJoerg Roedel { 554883b0a91SJoerg Roedel to_vmcb->save.fs = from_vmcb->save.fs; 555883b0a91SJoerg Roedel to_vmcb->save.gs = from_vmcb->save.gs; 556883b0a91SJoerg Roedel to_vmcb->save.tr = from_vmcb->save.tr; 557883b0a91SJoerg Roedel to_vmcb->save.ldtr = from_vmcb->save.ldtr; 558883b0a91SJoerg Roedel to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base; 559883b0a91SJoerg Roedel to_vmcb->save.star = from_vmcb->save.star; 560883b0a91SJoerg Roedel to_vmcb->save.lstar = from_vmcb->save.lstar; 561883b0a91SJoerg Roedel to_vmcb->save.cstar = from_vmcb->save.cstar; 562883b0a91SJoerg Roedel to_vmcb->save.sfmask = from_vmcb->save.sfmask; 563883b0a91SJoerg Roedel to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs; 564883b0a91SJoerg Roedel to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp; 565883b0a91SJoerg Roedel to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip; 566883b0a91SJoerg Roedel } 567883b0a91SJoerg Roedel 568883b0a91SJoerg Roedel int nested_svm_vmexit(struct vcpu_svm *svm) 569883b0a91SJoerg Roedel { 570883b0a91SJoerg Roedel int rc; 5710dd16b5bSMaxim Levitsky struct vmcb *vmcb12; 572883b0a91SJoerg Roedel struct vmcb *hsave = svm->nested.hsave; 573883b0a91SJoerg Roedel struct vmcb *vmcb = svm->vmcb; 574883b0a91SJoerg Roedel struct kvm_host_map map; 575883b0a91SJoerg Roedel 5760dd16b5bSMaxim Levitsky rc = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa), &map); 577883b0a91SJoerg Roedel if (rc) { 578883b0a91SJoerg Roedel if (rc == -EINVAL) 579883b0a91SJoerg Roedel kvm_inject_gp(&svm->vcpu, 0); 580883b0a91SJoerg Roedel return 1; 581883b0a91SJoerg Roedel } 582883b0a91SJoerg Roedel 5830dd16b5bSMaxim Levitsky vmcb12 = map.hva; 584883b0a91SJoerg Roedel 585883b0a91SJoerg Roedel /* Exit Guest-Mode */ 586883b0a91SJoerg Roedel leave_guest_mode(&svm->vcpu); 5870dd16b5bSMaxim Levitsky svm->nested.vmcb12_gpa = 0; 5882d8a42beSPaolo Bonzini WARN_ON_ONCE(svm->nested.nested_run_pending); 589883b0a91SJoerg Roedel 59038c0b192SPaolo Bonzini /* in case we halted in L2 */ 59138c0b192SPaolo Bonzini svm->vcpu.arch.mp_state = KVM_MP_STATE_RUNNABLE; 59238c0b192SPaolo Bonzini 593883b0a91SJoerg Roedel /* Give the current vmcb to the guest */ 594883b0a91SJoerg Roedel 5950dd16b5bSMaxim Levitsky vmcb12->save.es = vmcb->save.es; 5960dd16b5bSMaxim Levitsky vmcb12->save.cs = vmcb->save.cs; 5970dd16b5bSMaxim Levitsky vmcb12->save.ss = vmcb->save.ss; 5980dd16b5bSMaxim Levitsky vmcb12->save.ds = vmcb->save.ds; 5990dd16b5bSMaxim Levitsky vmcb12->save.gdtr = vmcb->save.gdtr; 6000dd16b5bSMaxim Levitsky vmcb12->save.idtr = vmcb->save.idtr; 6010dd16b5bSMaxim Levitsky vmcb12->save.efer = svm->vcpu.arch.efer; 6020dd16b5bSMaxim Levitsky vmcb12->save.cr0 = kvm_read_cr0(&svm->vcpu); 6030dd16b5bSMaxim Levitsky vmcb12->save.cr3 = kvm_read_cr3(&svm->vcpu); 6040dd16b5bSMaxim Levitsky vmcb12->save.cr2 = vmcb->save.cr2; 6050dd16b5bSMaxim Levitsky vmcb12->save.cr4 = svm->vcpu.arch.cr4; 6060dd16b5bSMaxim Levitsky vmcb12->save.rflags = kvm_get_rflags(&svm->vcpu); 6070dd16b5bSMaxim Levitsky vmcb12->save.rip = kvm_rip_read(&svm->vcpu); 6080dd16b5bSMaxim Levitsky vmcb12->save.rsp = kvm_rsp_read(&svm->vcpu); 6090dd16b5bSMaxim Levitsky vmcb12->save.rax = kvm_rax_read(&svm->vcpu); 6100dd16b5bSMaxim Levitsky vmcb12->save.dr7 = vmcb->save.dr7; 6110dd16b5bSMaxim Levitsky vmcb12->save.dr6 = svm->vcpu.arch.dr6; 6120dd16b5bSMaxim Levitsky vmcb12->save.cpl = vmcb->save.cpl; 613883b0a91SJoerg Roedel 6140dd16b5bSMaxim Levitsky vmcb12->control.int_state = vmcb->control.int_state; 6150dd16b5bSMaxim Levitsky vmcb12->control.exit_code = vmcb->control.exit_code; 6160dd16b5bSMaxim Levitsky vmcb12->control.exit_code_hi = vmcb->control.exit_code_hi; 6170dd16b5bSMaxim Levitsky vmcb12->control.exit_info_1 = vmcb->control.exit_info_1; 6180dd16b5bSMaxim Levitsky vmcb12->control.exit_info_2 = vmcb->control.exit_info_2; 61936e2e983SPaolo Bonzini 6200dd16b5bSMaxim Levitsky if (vmcb12->control.exit_code != SVM_EXIT_ERR) 6210dd16b5bSMaxim Levitsky nested_vmcb_save_pending_event(svm, vmcb12); 622883b0a91SJoerg Roedel 623883b0a91SJoerg Roedel if (svm->nrips_enabled) 6240dd16b5bSMaxim Levitsky vmcb12->control.next_rip = vmcb->control.next_rip; 625883b0a91SJoerg Roedel 6260dd16b5bSMaxim Levitsky vmcb12->control.int_ctl = svm->nested.ctl.int_ctl; 6270dd16b5bSMaxim Levitsky vmcb12->control.tlb_ctl = svm->nested.ctl.tlb_ctl; 6280dd16b5bSMaxim Levitsky vmcb12->control.event_inj = svm->nested.ctl.event_inj; 6290dd16b5bSMaxim Levitsky vmcb12->control.event_inj_err = svm->nested.ctl.event_inj_err; 630883b0a91SJoerg Roedel 6310dd16b5bSMaxim Levitsky vmcb12->control.pause_filter_count = 632883b0a91SJoerg Roedel svm->vmcb->control.pause_filter_count; 6330dd16b5bSMaxim Levitsky vmcb12->control.pause_filter_thresh = 634883b0a91SJoerg Roedel svm->vmcb->control.pause_filter_thresh; 635883b0a91SJoerg Roedel 636883b0a91SJoerg Roedel /* Restore the original control entries */ 6372f675917SPaolo Bonzini copy_vmcb_control_area(&vmcb->control, &hsave->control); 638883b0a91SJoerg Roedel 6399883764aSMaxim Levitsky /* On vmexit the GIF is set to false */ 6409883764aSMaxim Levitsky svm_set_gif(svm, false); 6419883764aSMaxim Levitsky 64218fc6c55SPaolo Bonzini svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset = 64318fc6c55SPaolo Bonzini svm->vcpu.arch.l1_tsc_offset; 64418fc6c55SPaolo Bonzini 645e670bf68SPaolo Bonzini svm->nested.ctl.nested_cr3 = 0; 646883b0a91SJoerg Roedel 647883b0a91SJoerg Roedel /* Restore selected save entries */ 648883b0a91SJoerg Roedel svm->vmcb->save.es = hsave->save.es; 649883b0a91SJoerg Roedel svm->vmcb->save.cs = hsave->save.cs; 650883b0a91SJoerg Roedel svm->vmcb->save.ss = hsave->save.ss; 651883b0a91SJoerg Roedel svm->vmcb->save.ds = hsave->save.ds; 652883b0a91SJoerg Roedel svm->vmcb->save.gdtr = hsave->save.gdtr; 653883b0a91SJoerg Roedel svm->vmcb->save.idtr = hsave->save.idtr; 654883b0a91SJoerg Roedel kvm_set_rflags(&svm->vcpu, hsave->save.rflags); 655883b0a91SJoerg Roedel svm_set_efer(&svm->vcpu, hsave->save.efer); 656883b0a91SJoerg Roedel svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE); 657883b0a91SJoerg Roedel svm_set_cr4(&svm->vcpu, hsave->save.cr4); 658883b0a91SJoerg Roedel kvm_rax_write(&svm->vcpu, hsave->save.rax); 659883b0a91SJoerg Roedel kvm_rsp_write(&svm->vcpu, hsave->save.rsp); 660883b0a91SJoerg Roedel kvm_rip_write(&svm->vcpu, hsave->save.rip); 661883b0a91SJoerg Roedel svm->vmcb->save.dr7 = 0; 662883b0a91SJoerg Roedel svm->vmcb->save.cpl = 0; 663883b0a91SJoerg Roedel svm->vmcb->control.exit_int_info = 0; 664883b0a91SJoerg Roedel 66506e7852cSJoerg Roedel vmcb_mark_all_dirty(svm->vmcb); 666883b0a91SJoerg Roedel 6670dd16b5bSMaxim Levitsky trace_kvm_nested_vmexit_inject(vmcb12->control.exit_code, 6680dd16b5bSMaxim Levitsky vmcb12->control.exit_info_1, 6690dd16b5bSMaxim Levitsky vmcb12->control.exit_info_2, 6700dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info, 6710dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info_err, 67236e2e983SPaolo Bonzini KVM_ISA_SVM); 67336e2e983SPaolo Bonzini 674883b0a91SJoerg Roedel kvm_vcpu_unmap(&svm->vcpu, &map, true); 675883b0a91SJoerg Roedel 676883b0a91SJoerg Roedel nested_svm_uninit_mmu_context(&svm->vcpu); 677bf7dea42SVitaly Kuznetsov 678d82aaef9SVitaly Kuznetsov rc = nested_svm_load_cr3(&svm->vcpu, hsave->save.cr3, false); 679d82aaef9SVitaly Kuznetsov if (rc) 680d82aaef9SVitaly Kuznetsov return 1; 681bf7dea42SVitaly Kuznetsov 682d82aaef9SVitaly Kuznetsov if (npt_enabled) 683d82aaef9SVitaly Kuznetsov svm->vmcb->save.cr3 = hsave->save.cr3; 684883b0a91SJoerg Roedel 685883b0a91SJoerg Roedel /* 686883b0a91SJoerg Roedel * Drop what we picked up for L2 via svm_complete_interrupts() so it 687883b0a91SJoerg Roedel * doesn't end up in L1. 688883b0a91SJoerg Roedel */ 689883b0a91SJoerg Roedel svm->vcpu.arch.nmi_injected = false; 690883b0a91SJoerg Roedel kvm_clear_exception_queue(&svm->vcpu); 691883b0a91SJoerg Roedel kvm_clear_interrupt_queue(&svm->vcpu); 692883b0a91SJoerg Roedel 693883b0a91SJoerg Roedel return 0; 694883b0a91SJoerg Roedel } 695883b0a91SJoerg Roedel 696c513f484SPaolo Bonzini /* 697c513f484SPaolo Bonzini * Forcibly leave nested mode in order to be able to reset the VCPU later on. 698c513f484SPaolo Bonzini */ 699c513f484SPaolo Bonzini void svm_leave_nested(struct vcpu_svm *svm) 700c513f484SPaolo Bonzini { 701c513f484SPaolo Bonzini if (is_guest_mode(&svm->vcpu)) { 702c513f484SPaolo Bonzini struct vmcb *hsave = svm->nested.hsave; 703c513f484SPaolo Bonzini struct vmcb *vmcb = svm->vmcb; 704c513f484SPaolo Bonzini 705c513f484SPaolo Bonzini svm->nested.nested_run_pending = 0; 706c513f484SPaolo Bonzini leave_guest_mode(&svm->vcpu); 707c513f484SPaolo Bonzini copy_vmcb_control_area(&vmcb->control, &hsave->control); 708c513f484SPaolo Bonzini nested_svm_uninit_mmu_context(&svm->vcpu); 709c513f484SPaolo Bonzini } 710c513f484SPaolo Bonzini } 711c513f484SPaolo Bonzini 712883b0a91SJoerg Roedel static int nested_svm_exit_handled_msr(struct vcpu_svm *svm) 713883b0a91SJoerg Roedel { 714883b0a91SJoerg Roedel u32 offset, msr, value; 715883b0a91SJoerg Roedel int write, mask; 716883b0a91SJoerg Roedel 717e670bf68SPaolo Bonzini if (!(svm->nested.ctl.intercept & (1ULL << INTERCEPT_MSR_PROT))) 718883b0a91SJoerg Roedel return NESTED_EXIT_HOST; 719883b0a91SJoerg Roedel 720883b0a91SJoerg Roedel msr = svm->vcpu.arch.regs[VCPU_REGS_RCX]; 721883b0a91SJoerg Roedel offset = svm_msrpm_offset(msr); 722883b0a91SJoerg Roedel write = svm->vmcb->control.exit_info_1 & 1; 723883b0a91SJoerg Roedel mask = 1 << ((2 * (msr & 0xf)) + write); 724883b0a91SJoerg Roedel 725883b0a91SJoerg Roedel if (offset == MSR_INVALID) 726883b0a91SJoerg Roedel return NESTED_EXIT_DONE; 727883b0a91SJoerg Roedel 728883b0a91SJoerg Roedel /* Offset is in 32 bit units but need in 8 bit units */ 729883b0a91SJoerg Roedel offset *= 4; 730883b0a91SJoerg Roedel 731e670bf68SPaolo Bonzini if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.ctl.msrpm_base_pa + offset, &value, 4)) 732883b0a91SJoerg Roedel return NESTED_EXIT_DONE; 733883b0a91SJoerg Roedel 734883b0a91SJoerg Roedel return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; 735883b0a91SJoerg Roedel } 736883b0a91SJoerg Roedel 737883b0a91SJoerg Roedel static int nested_svm_intercept_ioio(struct vcpu_svm *svm) 738883b0a91SJoerg Roedel { 739883b0a91SJoerg Roedel unsigned port, size, iopm_len; 740883b0a91SJoerg Roedel u16 val, mask; 741883b0a91SJoerg Roedel u8 start_bit; 742883b0a91SJoerg Roedel u64 gpa; 743883b0a91SJoerg Roedel 744e670bf68SPaolo Bonzini if (!(svm->nested.ctl.intercept & (1ULL << INTERCEPT_IOIO_PROT))) 745883b0a91SJoerg Roedel return NESTED_EXIT_HOST; 746883b0a91SJoerg Roedel 747883b0a91SJoerg Roedel port = svm->vmcb->control.exit_info_1 >> 16; 748883b0a91SJoerg Roedel size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >> 749883b0a91SJoerg Roedel SVM_IOIO_SIZE_SHIFT; 750e670bf68SPaolo Bonzini gpa = svm->nested.ctl.iopm_base_pa + (port / 8); 751883b0a91SJoerg Roedel start_bit = port % 8; 752883b0a91SJoerg Roedel iopm_len = (start_bit + size > 8) ? 2 : 1; 753883b0a91SJoerg Roedel mask = (0xf >> (4 - size)) << start_bit; 754883b0a91SJoerg Roedel val = 0; 755883b0a91SJoerg Roedel 756883b0a91SJoerg Roedel if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len)) 757883b0a91SJoerg Roedel return NESTED_EXIT_DONE; 758883b0a91SJoerg Roedel 759883b0a91SJoerg Roedel return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; 760883b0a91SJoerg Roedel } 761883b0a91SJoerg Roedel 762883b0a91SJoerg Roedel static int nested_svm_intercept(struct vcpu_svm *svm) 763883b0a91SJoerg Roedel { 764883b0a91SJoerg Roedel u32 exit_code = svm->vmcb->control.exit_code; 765883b0a91SJoerg Roedel int vmexit = NESTED_EXIT_HOST; 766883b0a91SJoerg Roedel 767883b0a91SJoerg Roedel switch (exit_code) { 768883b0a91SJoerg Roedel case SVM_EXIT_MSR: 769883b0a91SJoerg Roedel vmexit = nested_svm_exit_handled_msr(svm); 770883b0a91SJoerg Roedel break; 771883b0a91SJoerg Roedel case SVM_EXIT_IOIO: 772883b0a91SJoerg Roedel vmexit = nested_svm_intercept_ioio(svm); 773883b0a91SJoerg Roedel break; 774883b0a91SJoerg Roedel case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: { 775*03bfeeb9SBabu Moger if (vmcb_is_intercept(&svm->nested.ctl, exit_code)) 776883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 777883b0a91SJoerg Roedel break; 778883b0a91SJoerg Roedel } 779883b0a91SJoerg Roedel case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: { 780883b0a91SJoerg Roedel u32 bit = 1U << (exit_code - SVM_EXIT_READ_DR0); 781e670bf68SPaolo Bonzini if (svm->nested.ctl.intercept_dr & bit) 782883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 783883b0a91SJoerg Roedel break; 784883b0a91SJoerg Roedel } 785883b0a91SJoerg Roedel case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: { 7867c86663bSPaolo Bonzini /* 7877c86663bSPaolo Bonzini * Host-intercepted exceptions have been checked already in 7887c86663bSPaolo Bonzini * nested_svm_exit_special. There is nothing to do here, 7897c86663bSPaolo Bonzini * the vmexit is injected by svm_check_nested_events. 7907c86663bSPaolo Bonzini */ 791883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 792883b0a91SJoerg Roedel break; 793883b0a91SJoerg Roedel } 794883b0a91SJoerg Roedel case SVM_EXIT_ERR: { 795883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 796883b0a91SJoerg Roedel break; 797883b0a91SJoerg Roedel } 798883b0a91SJoerg Roedel default: { 799883b0a91SJoerg Roedel u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR); 800e670bf68SPaolo Bonzini if (svm->nested.ctl.intercept & exit_bits) 801883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 802883b0a91SJoerg Roedel } 803883b0a91SJoerg Roedel } 804883b0a91SJoerg Roedel 805883b0a91SJoerg Roedel return vmexit; 806883b0a91SJoerg Roedel } 807883b0a91SJoerg Roedel 808883b0a91SJoerg Roedel int nested_svm_exit_handled(struct vcpu_svm *svm) 809883b0a91SJoerg Roedel { 810883b0a91SJoerg Roedel int vmexit; 811883b0a91SJoerg Roedel 812883b0a91SJoerg Roedel vmexit = nested_svm_intercept(svm); 813883b0a91SJoerg Roedel 814883b0a91SJoerg Roedel if (vmexit == NESTED_EXIT_DONE) 815883b0a91SJoerg Roedel nested_svm_vmexit(svm); 816883b0a91SJoerg Roedel 817883b0a91SJoerg Roedel return vmexit; 818883b0a91SJoerg Roedel } 819883b0a91SJoerg Roedel 820883b0a91SJoerg Roedel int nested_svm_check_permissions(struct vcpu_svm *svm) 821883b0a91SJoerg Roedel { 822883b0a91SJoerg Roedel if (!(svm->vcpu.arch.efer & EFER_SVME) || 823883b0a91SJoerg Roedel !is_paging(&svm->vcpu)) { 824883b0a91SJoerg Roedel kvm_queue_exception(&svm->vcpu, UD_VECTOR); 825883b0a91SJoerg Roedel return 1; 826883b0a91SJoerg Roedel } 827883b0a91SJoerg Roedel 828883b0a91SJoerg Roedel if (svm->vmcb->save.cpl) { 829883b0a91SJoerg Roedel kvm_inject_gp(&svm->vcpu, 0); 830883b0a91SJoerg Roedel return 1; 831883b0a91SJoerg Roedel } 832883b0a91SJoerg Roedel 833883b0a91SJoerg Roedel return 0; 834883b0a91SJoerg Roedel } 835883b0a91SJoerg Roedel 8367c86663bSPaolo Bonzini static bool nested_exit_on_exception(struct vcpu_svm *svm) 837883b0a91SJoerg Roedel { 8387c86663bSPaolo Bonzini unsigned int nr = svm->vcpu.arch.exception.nr; 839883b0a91SJoerg Roedel 840e670bf68SPaolo Bonzini return (svm->nested.ctl.intercept_exceptions & (1 << nr)); 8417c86663bSPaolo Bonzini } 842883b0a91SJoerg Roedel 8437c86663bSPaolo Bonzini static void nested_svm_inject_exception_vmexit(struct vcpu_svm *svm) 8447c86663bSPaolo Bonzini { 8457c86663bSPaolo Bonzini unsigned int nr = svm->vcpu.arch.exception.nr; 846883b0a91SJoerg Roedel 847883b0a91SJoerg Roedel svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr; 848883b0a91SJoerg Roedel svm->vmcb->control.exit_code_hi = 0; 8497c86663bSPaolo Bonzini 8507c86663bSPaolo Bonzini if (svm->vcpu.arch.exception.has_error_code) 8517c86663bSPaolo Bonzini svm->vmcb->control.exit_info_1 = svm->vcpu.arch.exception.error_code; 852883b0a91SJoerg Roedel 853883b0a91SJoerg Roedel /* 854883b0a91SJoerg Roedel * EXITINFO2 is undefined for all exception intercepts other 855883b0a91SJoerg Roedel * than #PF. 856883b0a91SJoerg Roedel */ 8577c86663bSPaolo Bonzini if (nr == PF_VECTOR) { 858883b0a91SJoerg Roedel if (svm->vcpu.arch.exception.nested_apf) 859883b0a91SJoerg Roedel svm->vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token; 860883b0a91SJoerg Roedel else if (svm->vcpu.arch.exception.has_payload) 861883b0a91SJoerg Roedel svm->vmcb->control.exit_info_2 = svm->vcpu.arch.exception.payload; 862883b0a91SJoerg Roedel else 863883b0a91SJoerg Roedel svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2; 8647c86663bSPaolo Bonzini } else if (nr == DB_VECTOR) { 8657c86663bSPaolo Bonzini /* See inject_pending_event. */ 8667c86663bSPaolo Bonzini kvm_deliver_exception_payload(&svm->vcpu); 8677c86663bSPaolo Bonzini if (svm->vcpu.arch.dr7 & DR7_GD) { 8687c86663bSPaolo Bonzini svm->vcpu.arch.dr7 &= ~DR7_GD; 8697c86663bSPaolo Bonzini kvm_update_dr7(&svm->vcpu); 8707c86663bSPaolo Bonzini } 8717c86663bSPaolo Bonzini } else 8727c86663bSPaolo Bonzini WARN_ON(svm->vcpu.arch.exception.has_payload); 873883b0a91SJoerg Roedel 8747c86663bSPaolo Bonzini nested_svm_vmexit(svm); 875883b0a91SJoerg Roedel } 876883b0a91SJoerg Roedel 87755714cddSPaolo Bonzini static void nested_svm_smi(struct vcpu_svm *svm) 87855714cddSPaolo Bonzini { 87955714cddSPaolo Bonzini svm->vmcb->control.exit_code = SVM_EXIT_SMI; 88055714cddSPaolo Bonzini svm->vmcb->control.exit_info_1 = 0; 88155714cddSPaolo Bonzini svm->vmcb->control.exit_info_2 = 0; 88255714cddSPaolo Bonzini 88355714cddSPaolo Bonzini nested_svm_vmexit(svm); 88455714cddSPaolo Bonzini } 88555714cddSPaolo Bonzini 8869c3d370aSCathy Avery static void nested_svm_nmi(struct vcpu_svm *svm) 8879c3d370aSCathy Avery { 8889c3d370aSCathy Avery svm->vmcb->control.exit_code = SVM_EXIT_NMI; 8899c3d370aSCathy Avery svm->vmcb->control.exit_info_1 = 0; 8909c3d370aSCathy Avery svm->vmcb->control.exit_info_2 = 0; 8919c3d370aSCathy Avery 8929c3d370aSCathy Avery nested_svm_vmexit(svm); 8939c3d370aSCathy Avery } 8949c3d370aSCathy Avery 895883b0a91SJoerg Roedel static void nested_svm_intr(struct vcpu_svm *svm) 896883b0a91SJoerg Roedel { 8976e085cbfSPaolo Bonzini trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip); 8986e085cbfSPaolo Bonzini 899883b0a91SJoerg Roedel svm->vmcb->control.exit_code = SVM_EXIT_INTR; 900883b0a91SJoerg Roedel svm->vmcb->control.exit_info_1 = 0; 901883b0a91SJoerg Roedel svm->vmcb->control.exit_info_2 = 0; 902883b0a91SJoerg Roedel 9036e085cbfSPaolo Bonzini nested_svm_vmexit(svm); 904883b0a91SJoerg Roedel } 905883b0a91SJoerg Roedel 9065b672408SPaolo Bonzini static inline bool nested_exit_on_init(struct vcpu_svm *svm) 9075b672408SPaolo Bonzini { 908e670bf68SPaolo Bonzini return (svm->nested.ctl.intercept & (1ULL << INTERCEPT_INIT)); 9095b672408SPaolo Bonzini } 9105b672408SPaolo Bonzini 9115b672408SPaolo Bonzini static void nested_svm_init(struct vcpu_svm *svm) 9125b672408SPaolo Bonzini { 9135b672408SPaolo Bonzini svm->vmcb->control.exit_code = SVM_EXIT_INIT; 9145b672408SPaolo Bonzini svm->vmcb->control.exit_info_1 = 0; 9155b672408SPaolo Bonzini svm->vmcb->control.exit_info_2 = 0; 9165b672408SPaolo Bonzini 9175b672408SPaolo Bonzini nested_svm_vmexit(svm); 9185b672408SPaolo Bonzini } 9195b672408SPaolo Bonzini 9205b672408SPaolo Bonzini 92133b22172SPaolo Bonzini static int svm_check_nested_events(struct kvm_vcpu *vcpu) 922883b0a91SJoerg Roedel { 923883b0a91SJoerg Roedel struct vcpu_svm *svm = to_svm(vcpu); 924883b0a91SJoerg Roedel bool block_nested_events = 925bd279629SPaolo Bonzini kvm_event_needs_reinjection(vcpu) || svm->nested.nested_run_pending; 9265b672408SPaolo Bonzini struct kvm_lapic *apic = vcpu->arch.apic; 9275b672408SPaolo Bonzini 9285b672408SPaolo Bonzini if (lapic_in_kernel(vcpu) && 9295b672408SPaolo Bonzini test_bit(KVM_APIC_INIT, &apic->pending_events)) { 9305b672408SPaolo Bonzini if (block_nested_events) 9315b672408SPaolo Bonzini return -EBUSY; 9325b672408SPaolo Bonzini if (!nested_exit_on_init(svm)) 9335b672408SPaolo Bonzini return 0; 9345b672408SPaolo Bonzini nested_svm_init(svm); 9355b672408SPaolo Bonzini return 0; 9365b672408SPaolo Bonzini } 937883b0a91SJoerg Roedel 9387c86663bSPaolo Bonzini if (vcpu->arch.exception.pending) { 9397c86663bSPaolo Bonzini if (block_nested_events) 9407c86663bSPaolo Bonzini return -EBUSY; 9417c86663bSPaolo Bonzini if (!nested_exit_on_exception(svm)) 9427c86663bSPaolo Bonzini return 0; 9437c86663bSPaolo Bonzini nested_svm_inject_exception_vmexit(svm); 9447c86663bSPaolo Bonzini return 0; 9457c86663bSPaolo Bonzini } 9467c86663bSPaolo Bonzini 947221e7610SPaolo Bonzini if (vcpu->arch.smi_pending && !svm_smi_blocked(vcpu)) { 94855714cddSPaolo Bonzini if (block_nested_events) 94955714cddSPaolo Bonzini return -EBUSY; 950221e7610SPaolo Bonzini if (!nested_exit_on_smi(svm)) 951221e7610SPaolo Bonzini return 0; 95255714cddSPaolo Bonzini nested_svm_smi(svm); 95355714cddSPaolo Bonzini return 0; 95455714cddSPaolo Bonzini } 95555714cddSPaolo Bonzini 956221e7610SPaolo Bonzini if (vcpu->arch.nmi_pending && !svm_nmi_blocked(vcpu)) { 9579c3d370aSCathy Avery if (block_nested_events) 9589c3d370aSCathy Avery return -EBUSY; 959221e7610SPaolo Bonzini if (!nested_exit_on_nmi(svm)) 960221e7610SPaolo Bonzini return 0; 9619c3d370aSCathy Avery nested_svm_nmi(svm); 9629c3d370aSCathy Avery return 0; 9639c3d370aSCathy Avery } 9649c3d370aSCathy Avery 965221e7610SPaolo Bonzini if (kvm_cpu_has_interrupt(vcpu) && !svm_interrupt_blocked(vcpu)) { 966883b0a91SJoerg Roedel if (block_nested_events) 967883b0a91SJoerg Roedel return -EBUSY; 968221e7610SPaolo Bonzini if (!nested_exit_on_intr(svm)) 969221e7610SPaolo Bonzini return 0; 970883b0a91SJoerg Roedel nested_svm_intr(svm); 971883b0a91SJoerg Roedel return 0; 972883b0a91SJoerg Roedel } 973883b0a91SJoerg Roedel 974883b0a91SJoerg Roedel return 0; 975883b0a91SJoerg Roedel } 976883b0a91SJoerg Roedel 977883b0a91SJoerg Roedel int nested_svm_exit_special(struct vcpu_svm *svm) 978883b0a91SJoerg Roedel { 979883b0a91SJoerg Roedel u32 exit_code = svm->vmcb->control.exit_code; 980883b0a91SJoerg Roedel 981883b0a91SJoerg Roedel switch (exit_code) { 982883b0a91SJoerg Roedel case SVM_EXIT_INTR: 983883b0a91SJoerg Roedel case SVM_EXIT_NMI: 984883b0a91SJoerg Roedel case SVM_EXIT_NPF: 985883b0a91SJoerg Roedel return NESTED_EXIT_HOST; 9867c86663bSPaolo Bonzini case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: { 9877c86663bSPaolo Bonzini u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE); 9887c86663bSPaolo Bonzini 9897c86663bSPaolo Bonzini if (get_host_vmcb(svm)->control.intercept_exceptions & excp_bits) 9907c86663bSPaolo Bonzini return NESTED_EXIT_HOST; 9917c86663bSPaolo Bonzini else if (exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR && 99268fd66f1SVitaly Kuznetsov svm->vcpu.arch.apf.host_apf_flags) 993a3535be7SPaolo Bonzini /* Trap async PF even if not shadowing */ 994883b0a91SJoerg Roedel return NESTED_EXIT_HOST; 995883b0a91SJoerg Roedel break; 9967c86663bSPaolo Bonzini } 997883b0a91SJoerg Roedel default: 998883b0a91SJoerg Roedel break; 999883b0a91SJoerg Roedel } 1000883b0a91SJoerg Roedel 1001883b0a91SJoerg Roedel return NESTED_EXIT_CONTINUE; 1002883b0a91SJoerg Roedel } 100333b22172SPaolo Bonzini 1004cc440cdaSPaolo Bonzini static int svm_get_nested_state(struct kvm_vcpu *vcpu, 1005cc440cdaSPaolo Bonzini struct kvm_nested_state __user *user_kvm_nested_state, 1006cc440cdaSPaolo Bonzini u32 user_data_size) 1007cc440cdaSPaolo Bonzini { 1008cc440cdaSPaolo Bonzini struct vcpu_svm *svm; 1009cc440cdaSPaolo Bonzini struct kvm_nested_state kvm_state = { 1010cc440cdaSPaolo Bonzini .flags = 0, 1011cc440cdaSPaolo Bonzini .format = KVM_STATE_NESTED_FORMAT_SVM, 1012cc440cdaSPaolo Bonzini .size = sizeof(kvm_state), 1013cc440cdaSPaolo Bonzini }; 1014cc440cdaSPaolo Bonzini struct vmcb __user *user_vmcb = (struct vmcb __user *) 1015cc440cdaSPaolo Bonzini &user_kvm_nested_state->data.svm[0]; 1016cc440cdaSPaolo Bonzini 1017cc440cdaSPaolo Bonzini if (!vcpu) 1018cc440cdaSPaolo Bonzini return kvm_state.size + KVM_STATE_NESTED_SVM_VMCB_SIZE; 1019cc440cdaSPaolo Bonzini 1020cc440cdaSPaolo Bonzini svm = to_svm(vcpu); 1021cc440cdaSPaolo Bonzini 1022cc440cdaSPaolo Bonzini if (user_data_size < kvm_state.size) 1023cc440cdaSPaolo Bonzini goto out; 1024cc440cdaSPaolo Bonzini 1025cc440cdaSPaolo Bonzini /* First fill in the header and copy it out. */ 1026cc440cdaSPaolo Bonzini if (is_guest_mode(vcpu)) { 10270dd16b5bSMaxim Levitsky kvm_state.hdr.svm.vmcb_pa = svm->nested.vmcb12_gpa; 1028cc440cdaSPaolo Bonzini kvm_state.size += KVM_STATE_NESTED_SVM_VMCB_SIZE; 1029cc440cdaSPaolo Bonzini kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE; 1030cc440cdaSPaolo Bonzini 1031cc440cdaSPaolo Bonzini if (svm->nested.nested_run_pending) 1032cc440cdaSPaolo Bonzini kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING; 1033cc440cdaSPaolo Bonzini } 1034cc440cdaSPaolo Bonzini 1035cc440cdaSPaolo Bonzini if (gif_set(svm)) 1036cc440cdaSPaolo Bonzini kvm_state.flags |= KVM_STATE_NESTED_GIF_SET; 1037cc440cdaSPaolo Bonzini 1038cc440cdaSPaolo Bonzini if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state))) 1039cc440cdaSPaolo Bonzini return -EFAULT; 1040cc440cdaSPaolo Bonzini 1041cc440cdaSPaolo Bonzini if (!is_guest_mode(vcpu)) 1042cc440cdaSPaolo Bonzini goto out; 1043cc440cdaSPaolo Bonzini 1044cc440cdaSPaolo Bonzini /* 1045cc440cdaSPaolo Bonzini * Copy over the full size of the VMCB rather than just the size 1046cc440cdaSPaolo Bonzini * of the structs. 1047cc440cdaSPaolo Bonzini */ 1048cc440cdaSPaolo Bonzini if (clear_user(user_vmcb, KVM_STATE_NESTED_SVM_VMCB_SIZE)) 1049cc440cdaSPaolo Bonzini return -EFAULT; 1050cc440cdaSPaolo Bonzini if (copy_to_user(&user_vmcb->control, &svm->nested.ctl, 1051cc440cdaSPaolo Bonzini sizeof(user_vmcb->control))) 1052cc440cdaSPaolo Bonzini return -EFAULT; 1053cc440cdaSPaolo Bonzini if (copy_to_user(&user_vmcb->save, &svm->nested.hsave->save, 1054cc440cdaSPaolo Bonzini sizeof(user_vmcb->save))) 1055cc440cdaSPaolo Bonzini return -EFAULT; 1056cc440cdaSPaolo Bonzini 1057cc440cdaSPaolo Bonzini out: 1058cc440cdaSPaolo Bonzini return kvm_state.size; 1059cc440cdaSPaolo Bonzini } 1060cc440cdaSPaolo Bonzini 1061cc440cdaSPaolo Bonzini static int svm_set_nested_state(struct kvm_vcpu *vcpu, 1062cc440cdaSPaolo Bonzini struct kvm_nested_state __user *user_kvm_nested_state, 1063cc440cdaSPaolo Bonzini struct kvm_nested_state *kvm_state) 1064cc440cdaSPaolo Bonzini { 1065cc440cdaSPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 1066cc440cdaSPaolo Bonzini struct vmcb *hsave = svm->nested.hsave; 1067cc440cdaSPaolo Bonzini struct vmcb __user *user_vmcb = (struct vmcb __user *) 1068cc440cdaSPaolo Bonzini &user_kvm_nested_state->data.svm[0]; 10696ccbd29aSJoerg Roedel struct vmcb_control_area *ctl; 10706ccbd29aSJoerg Roedel struct vmcb_save_area *save; 10716ccbd29aSJoerg Roedel int ret; 1072cc440cdaSPaolo Bonzini u32 cr0; 1073cc440cdaSPaolo Bonzini 10746ccbd29aSJoerg Roedel BUILD_BUG_ON(sizeof(struct vmcb_control_area) + sizeof(struct vmcb_save_area) > 10756ccbd29aSJoerg Roedel KVM_STATE_NESTED_SVM_VMCB_SIZE); 10766ccbd29aSJoerg Roedel 1077cc440cdaSPaolo Bonzini if (kvm_state->format != KVM_STATE_NESTED_FORMAT_SVM) 1078cc440cdaSPaolo Bonzini return -EINVAL; 1079cc440cdaSPaolo Bonzini 1080cc440cdaSPaolo Bonzini if (kvm_state->flags & ~(KVM_STATE_NESTED_GUEST_MODE | 1081cc440cdaSPaolo Bonzini KVM_STATE_NESTED_RUN_PENDING | 1082cc440cdaSPaolo Bonzini KVM_STATE_NESTED_GIF_SET)) 1083cc440cdaSPaolo Bonzini return -EINVAL; 1084cc440cdaSPaolo Bonzini 1085cc440cdaSPaolo Bonzini /* 1086cc440cdaSPaolo Bonzini * If in guest mode, vcpu->arch.efer actually refers to the L2 guest's 1087cc440cdaSPaolo Bonzini * EFER.SVME, but EFER.SVME still has to be 1 for VMRUN to succeed. 1088cc440cdaSPaolo Bonzini */ 1089cc440cdaSPaolo Bonzini if (!(vcpu->arch.efer & EFER_SVME)) { 1090cc440cdaSPaolo Bonzini /* GIF=1 and no guest mode are required if SVME=0. */ 1091cc440cdaSPaolo Bonzini if (kvm_state->flags != KVM_STATE_NESTED_GIF_SET) 1092cc440cdaSPaolo Bonzini return -EINVAL; 1093cc440cdaSPaolo Bonzini } 1094cc440cdaSPaolo Bonzini 1095cc440cdaSPaolo Bonzini /* SMM temporarily disables SVM, so we cannot be in guest mode. */ 1096cc440cdaSPaolo Bonzini if (is_smm(vcpu) && (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 1097cc440cdaSPaolo Bonzini return -EINVAL; 1098cc440cdaSPaolo Bonzini 1099cc440cdaSPaolo Bonzini if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) { 1100cc440cdaSPaolo Bonzini svm_leave_nested(svm); 1101d5cd6f34SVitaly Kuznetsov svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET)); 1102d5cd6f34SVitaly Kuznetsov return 0; 1103cc440cdaSPaolo Bonzini } 1104cc440cdaSPaolo Bonzini 1105cc440cdaSPaolo Bonzini if (!page_address_valid(vcpu, kvm_state->hdr.svm.vmcb_pa)) 1106cc440cdaSPaolo Bonzini return -EINVAL; 1107cc440cdaSPaolo Bonzini if (kvm_state->size < sizeof(*kvm_state) + KVM_STATE_NESTED_SVM_VMCB_SIZE) 1108cc440cdaSPaolo Bonzini return -EINVAL; 1109cc440cdaSPaolo Bonzini 11106ccbd29aSJoerg Roedel ret = -ENOMEM; 11116ccbd29aSJoerg Roedel ctl = kzalloc(sizeof(*ctl), GFP_KERNEL); 11126ccbd29aSJoerg Roedel save = kzalloc(sizeof(*save), GFP_KERNEL); 11136ccbd29aSJoerg Roedel if (!ctl || !save) 11146ccbd29aSJoerg Roedel goto out_free; 11156ccbd29aSJoerg Roedel 11166ccbd29aSJoerg Roedel ret = -EFAULT; 11176ccbd29aSJoerg Roedel if (copy_from_user(ctl, &user_vmcb->control, sizeof(*ctl))) 11186ccbd29aSJoerg Roedel goto out_free; 11196ccbd29aSJoerg Roedel if (copy_from_user(save, &user_vmcb->save, sizeof(*save))) 11206ccbd29aSJoerg Roedel goto out_free; 11216ccbd29aSJoerg Roedel 11226ccbd29aSJoerg Roedel ret = -EINVAL; 11236ccbd29aSJoerg Roedel if (!nested_vmcb_check_controls(ctl)) 11246ccbd29aSJoerg Roedel goto out_free; 1125cc440cdaSPaolo Bonzini 1126cc440cdaSPaolo Bonzini /* 1127cc440cdaSPaolo Bonzini * Processor state contains L2 state. Check that it is 1128cc440cdaSPaolo Bonzini * valid for guest mode (see nested_vmcb_checks). 1129cc440cdaSPaolo Bonzini */ 1130cc440cdaSPaolo Bonzini cr0 = kvm_read_cr0(vcpu); 1131cc440cdaSPaolo Bonzini if (((cr0 & X86_CR0_CD) == 0) && (cr0 & X86_CR0_NW)) 11326ccbd29aSJoerg Roedel goto out_free; 1133cc440cdaSPaolo Bonzini 1134cc440cdaSPaolo Bonzini /* 1135cc440cdaSPaolo Bonzini * Validate host state saved from before VMRUN (see 1136cc440cdaSPaolo Bonzini * nested_svm_check_permissions). 1137cc440cdaSPaolo Bonzini * TODO: validate reserved bits for all saved state. 1138cc440cdaSPaolo Bonzini */ 11396ccbd29aSJoerg Roedel if (!(save->cr0 & X86_CR0_PG)) 11406ccbd29aSJoerg Roedel goto out_free; 1141cc440cdaSPaolo Bonzini 1142cc440cdaSPaolo Bonzini /* 1143cc440cdaSPaolo Bonzini * All checks done, we can enter guest mode. L1 control fields 1144cc440cdaSPaolo Bonzini * come from the nested save state. Guest state is already 1145cc440cdaSPaolo Bonzini * in the registers, the save area of the nested state instead 1146cc440cdaSPaolo Bonzini * contains saved L1 state. 1147cc440cdaSPaolo Bonzini */ 1148cc440cdaSPaolo Bonzini copy_vmcb_control_area(&hsave->control, &svm->vmcb->control); 11496ccbd29aSJoerg Roedel hsave->save = *save; 1150cc440cdaSPaolo Bonzini 11510dd16b5bSMaxim Levitsky svm->nested.vmcb12_gpa = kvm_state->hdr.svm.vmcb_pa; 11526ccbd29aSJoerg Roedel load_nested_vmcb_control(svm, ctl); 1153cc440cdaSPaolo Bonzini nested_prepare_vmcb_control(svm); 1154cc440cdaSPaolo Bonzini 1155772b81bbSMaxim Levitsky if (!nested_svm_vmrun_msrpm(svm)) 1156d5cd6f34SVitaly Kuznetsov goto out_free; 11576ccbd29aSJoerg Roedel 11586ccbd29aSJoerg Roedel ret = 0; 11596ccbd29aSJoerg Roedel out_free: 11606ccbd29aSJoerg Roedel kfree(save); 11616ccbd29aSJoerg Roedel kfree(ctl); 11626ccbd29aSJoerg Roedel 11636ccbd29aSJoerg Roedel return ret; 1164cc440cdaSPaolo Bonzini } 1165cc440cdaSPaolo Bonzini 116633b22172SPaolo Bonzini struct kvm_x86_nested_ops svm_nested_ops = { 116733b22172SPaolo Bonzini .check_events = svm_check_nested_events, 1168cc440cdaSPaolo Bonzini .get_state = svm_get_nested_state, 1169cc440cdaSPaolo Bonzini .set_state = svm_set_nested_state, 117033b22172SPaolo Bonzini }; 1171