1883b0a91SJoerg Roedel // SPDX-License-Identifier: GPL-2.0-only 2883b0a91SJoerg Roedel /* 3883b0a91SJoerg Roedel * Kernel-based Virtual Machine driver for Linux 4883b0a91SJoerg Roedel * 5883b0a91SJoerg Roedel * AMD SVM support 6883b0a91SJoerg Roedel * 7883b0a91SJoerg Roedel * Copyright (C) 2006 Qumranet, Inc. 8883b0a91SJoerg Roedel * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9883b0a91SJoerg Roedel * 10883b0a91SJoerg Roedel * Authors: 11883b0a91SJoerg Roedel * Yaniv Kamay <yaniv@qumranet.com> 12883b0a91SJoerg Roedel * Avi Kivity <avi@qumranet.com> 13883b0a91SJoerg Roedel */ 14883b0a91SJoerg Roedel 15883b0a91SJoerg Roedel #define pr_fmt(fmt) "SVM: " fmt 16883b0a91SJoerg Roedel 17883b0a91SJoerg Roedel #include <linux/kvm_types.h> 18883b0a91SJoerg Roedel #include <linux/kvm_host.h> 19883b0a91SJoerg Roedel #include <linux/kernel.h> 20883b0a91SJoerg Roedel 21883b0a91SJoerg Roedel #include <asm/msr-index.h> 225679b803SPaolo Bonzini #include <asm/debugreg.h> 23883b0a91SJoerg Roedel 24883b0a91SJoerg Roedel #include "kvm_emulate.h" 25883b0a91SJoerg Roedel #include "trace.h" 26883b0a91SJoerg Roedel #include "mmu.h" 27883b0a91SJoerg Roedel #include "x86.h" 28cc440cdaSPaolo Bonzini #include "cpuid.h" 295b672408SPaolo Bonzini #include "lapic.h" 30883b0a91SJoerg Roedel #include "svm.h" 31883b0a91SJoerg Roedel 3211f0cbf0SSean Christopherson #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK 3311f0cbf0SSean Christopherson 34883b0a91SJoerg Roedel static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu, 35883b0a91SJoerg Roedel struct x86_exception *fault) 36883b0a91SJoerg Roedel { 37883b0a91SJoerg Roedel struct vcpu_svm *svm = to_svm(vcpu); 38883b0a91SJoerg Roedel 39883b0a91SJoerg Roedel if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) { 40883b0a91SJoerg Roedel /* 41883b0a91SJoerg Roedel * TODO: track the cause of the nested page fault, and 42883b0a91SJoerg Roedel * correctly fill in the high bits of exit_info_1. 43883b0a91SJoerg Roedel */ 44883b0a91SJoerg Roedel svm->vmcb->control.exit_code = SVM_EXIT_NPF; 45883b0a91SJoerg Roedel svm->vmcb->control.exit_code_hi = 0; 46883b0a91SJoerg Roedel svm->vmcb->control.exit_info_1 = (1ULL << 32); 47883b0a91SJoerg Roedel svm->vmcb->control.exit_info_2 = fault->address; 48883b0a91SJoerg Roedel } 49883b0a91SJoerg Roedel 50883b0a91SJoerg Roedel svm->vmcb->control.exit_info_1 &= ~0xffffffffULL; 51883b0a91SJoerg Roedel svm->vmcb->control.exit_info_1 |= fault->error_code; 52883b0a91SJoerg Roedel 53883b0a91SJoerg Roedel nested_svm_vmexit(svm); 54883b0a91SJoerg Roedel } 55883b0a91SJoerg Roedel 56a04aead1SPaolo Bonzini static void svm_inject_page_fault_nested(struct kvm_vcpu *vcpu, struct x86_exception *fault) 57a04aead1SPaolo Bonzini { 58a04aead1SPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 59a04aead1SPaolo Bonzini WARN_ON(!is_guest_mode(vcpu)); 60a04aead1SPaolo Bonzini 61a04aead1SPaolo Bonzini if (vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_EXCEPTION_OFFSET + PF_VECTOR) && 62a04aead1SPaolo Bonzini !svm->nested.nested_run_pending) { 63a04aead1SPaolo Bonzini svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + PF_VECTOR; 64a04aead1SPaolo Bonzini svm->vmcb->control.exit_code_hi = 0; 65a04aead1SPaolo Bonzini svm->vmcb->control.exit_info_1 = fault->error_code; 66a04aead1SPaolo Bonzini svm->vmcb->control.exit_info_2 = fault->address; 67a04aead1SPaolo Bonzini nested_svm_vmexit(svm); 68a04aead1SPaolo Bonzini } else { 69a04aead1SPaolo Bonzini kvm_inject_page_fault(vcpu, fault); 70a04aead1SPaolo Bonzini } 71a04aead1SPaolo Bonzini } 72a04aead1SPaolo Bonzini 73883b0a91SJoerg Roedel static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index) 74883b0a91SJoerg Roedel { 75883b0a91SJoerg Roedel struct vcpu_svm *svm = to_svm(vcpu); 76e670bf68SPaolo Bonzini u64 cr3 = svm->nested.ctl.nested_cr3; 77883b0a91SJoerg Roedel u64 pdpte; 78883b0a91SJoerg Roedel int ret; 79883b0a91SJoerg Roedel 802732be90SSean Christopherson ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(cr3), &pdpte, 81883b0a91SJoerg Roedel offset_in_page(cr3) + index * 8, 8); 82883b0a91SJoerg Roedel if (ret) 83883b0a91SJoerg Roedel return 0; 84883b0a91SJoerg Roedel return pdpte; 85883b0a91SJoerg Roedel } 86883b0a91SJoerg Roedel 87883b0a91SJoerg Roedel static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu) 88883b0a91SJoerg Roedel { 89883b0a91SJoerg Roedel struct vcpu_svm *svm = to_svm(vcpu); 90883b0a91SJoerg Roedel 91e670bf68SPaolo Bonzini return svm->nested.ctl.nested_cr3; 92883b0a91SJoerg Roedel } 93883b0a91SJoerg Roedel 94883b0a91SJoerg Roedel static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu) 95883b0a91SJoerg Roedel { 96929d1cfaSPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 97929d1cfaSPaolo Bonzini 98883b0a91SJoerg Roedel WARN_ON(mmu_is_nested(vcpu)); 99883b0a91SJoerg Roedel 100883b0a91SJoerg Roedel vcpu->arch.mmu = &vcpu->arch.guest_mmu; 10131e96bc6SSean Christopherson 10231e96bc6SSean Christopherson /* 10331e96bc6SSean Christopherson * The NPT format depends on L1's CR4 and EFER, which is in vmcb01. Note, 10431e96bc6SSean Christopherson * when called via KVM_SET_NESTED_STATE, that state may _not_ match current 10531e96bc6SSean Christopherson * vCPU state. CR0.WP is explicitly ignored, while CR0.PG is required. 10631e96bc6SSean Christopherson */ 1074995a368SCathy Avery kvm_init_shadow_npt_mmu(vcpu, X86_CR0_PG, svm->vmcb01.ptr->save.cr4, 1084995a368SCathy Avery svm->vmcb01.ptr->save.efer, 1090f04a2acSVitaly Kuznetsov svm->nested.ctl.nested_cr3); 110883b0a91SJoerg Roedel vcpu->arch.mmu->get_guest_pgd = nested_svm_get_tdp_cr3; 111883b0a91SJoerg Roedel vcpu->arch.mmu->get_pdptr = nested_svm_get_tdp_pdptr; 112883b0a91SJoerg Roedel vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit; 113883b0a91SJoerg Roedel vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; 114883b0a91SJoerg Roedel } 115883b0a91SJoerg Roedel 116883b0a91SJoerg Roedel static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu) 117883b0a91SJoerg Roedel { 118883b0a91SJoerg Roedel vcpu->arch.mmu = &vcpu->arch.root_mmu; 119883b0a91SJoerg Roedel vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; 120883b0a91SJoerg Roedel } 121883b0a91SJoerg Roedel 122883b0a91SJoerg Roedel void recalc_intercepts(struct vcpu_svm *svm) 123883b0a91SJoerg Roedel { 124e670bf68SPaolo Bonzini struct vmcb_control_area *c, *h, *g; 125c45ad722SBabu Moger unsigned int i; 126883b0a91SJoerg Roedel 12706e7852cSJoerg Roedel vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 128883b0a91SJoerg Roedel 129883b0a91SJoerg Roedel if (!is_guest_mode(&svm->vcpu)) 130883b0a91SJoerg Roedel return; 131883b0a91SJoerg Roedel 132883b0a91SJoerg Roedel c = &svm->vmcb->control; 1334995a368SCathy Avery h = &svm->vmcb01.ptr->control; 134e670bf68SPaolo Bonzini g = &svm->nested.ctl; 135883b0a91SJoerg Roedel 136c45ad722SBabu Moger for (i = 0; i < MAX_INTERCEPT; i++) 137c45ad722SBabu Moger c->intercepts[i] = h->intercepts[i]; 138c45ad722SBabu Moger 139e9fd761aSPaolo Bonzini if (g->int_ctl & V_INTR_MASKING_MASK) { 140883b0a91SJoerg Roedel /* We only want the cr8 intercept bits of L1 */ 14103bfeeb9SBabu Moger vmcb_clr_intercept(c, INTERCEPT_CR8_READ); 14203bfeeb9SBabu Moger vmcb_clr_intercept(c, INTERCEPT_CR8_WRITE); 143883b0a91SJoerg Roedel 144883b0a91SJoerg Roedel /* 145883b0a91SJoerg Roedel * Once running L2 with HF_VINTR_MASK, EFLAGS.IF does not 146883b0a91SJoerg Roedel * affect any interrupt we may want to inject; therefore, 147883b0a91SJoerg Roedel * interrupt window vmexits are irrelevant to L0. 148883b0a91SJoerg Roedel */ 149c62e2e94SBabu Moger vmcb_clr_intercept(c, INTERCEPT_VINTR); 150883b0a91SJoerg Roedel } 151883b0a91SJoerg Roedel 152883b0a91SJoerg Roedel /* We don't want to see VMMCALLs from a nested guest */ 153c62e2e94SBabu Moger vmcb_clr_intercept(c, INTERCEPT_VMMCALL); 154883b0a91SJoerg Roedel 155c45ad722SBabu Moger for (i = 0; i < MAX_INTERCEPT; i++) 156c45ad722SBabu Moger c->intercepts[i] |= g->intercepts[i]; 1574b639a9fSMaxim Levitsky 1584b639a9fSMaxim Levitsky /* If SMI is not intercepted, ignore guest SMI intercept as well */ 1594b639a9fSMaxim Levitsky if (!intercept_smi) 1604b639a9fSMaxim Levitsky vmcb_clr_intercept(c, INTERCEPT_SMI); 161c7dfa400SMaxim Levitsky 162c7dfa400SMaxim Levitsky vmcb_set_intercept(c, INTERCEPT_VMLOAD); 163c7dfa400SMaxim Levitsky vmcb_set_intercept(c, INTERCEPT_VMSAVE); 164883b0a91SJoerg Roedel } 165883b0a91SJoerg Roedel 1662f675917SPaolo Bonzini static void copy_vmcb_control_area(struct vmcb_control_area *dst, 1672f675917SPaolo Bonzini struct vmcb_control_area *from) 168883b0a91SJoerg Roedel { 169c45ad722SBabu Moger unsigned int i; 170c45ad722SBabu Moger 171c45ad722SBabu Moger for (i = 0; i < MAX_INTERCEPT; i++) 172c45ad722SBabu Moger dst->intercepts[i] = from->intercepts[i]; 173c45ad722SBabu Moger 174883b0a91SJoerg Roedel dst->iopm_base_pa = from->iopm_base_pa; 175883b0a91SJoerg Roedel dst->msrpm_base_pa = from->msrpm_base_pa; 176883b0a91SJoerg Roedel dst->tsc_offset = from->tsc_offset; 1776c0238c4SPaolo Bonzini /* asid not copied, it is handled manually for svm->vmcb. */ 178883b0a91SJoerg Roedel dst->tlb_ctl = from->tlb_ctl; 179883b0a91SJoerg Roedel dst->int_ctl = from->int_ctl; 180883b0a91SJoerg Roedel dst->int_vector = from->int_vector; 181883b0a91SJoerg Roedel dst->int_state = from->int_state; 182883b0a91SJoerg Roedel dst->exit_code = from->exit_code; 183883b0a91SJoerg Roedel dst->exit_code_hi = from->exit_code_hi; 184883b0a91SJoerg Roedel dst->exit_info_1 = from->exit_info_1; 185883b0a91SJoerg Roedel dst->exit_info_2 = from->exit_info_2; 186883b0a91SJoerg Roedel dst->exit_int_info = from->exit_int_info; 187883b0a91SJoerg Roedel dst->exit_int_info_err = from->exit_int_info_err; 188883b0a91SJoerg Roedel dst->nested_ctl = from->nested_ctl; 189883b0a91SJoerg Roedel dst->event_inj = from->event_inj; 190883b0a91SJoerg Roedel dst->event_inj_err = from->event_inj_err; 191883b0a91SJoerg Roedel dst->nested_cr3 = from->nested_cr3; 192883b0a91SJoerg Roedel dst->virt_ext = from->virt_ext; 193883b0a91SJoerg Roedel dst->pause_filter_count = from->pause_filter_count; 194883b0a91SJoerg Roedel dst->pause_filter_thresh = from->pause_filter_thresh; 195883b0a91SJoerg Roedel } 196883b0a91SJoerg Roedel 197883b0a91SJoerg Roedel static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm) 198883b0a91SJoerg Roedel { 199883b0a91SJoerg Roedel /* 200883b0a91SJoerg Roedel * This function merges the msr permission bitmaps of kvm and the 201883b0a91SJoerg Roedel * nested vmcb. It is optimized in that it only merges the parts where 202883b0a91SJoerg Roedel * the kvm msr permission bitmap may contain zero bits 203883b0a91SJoerg Roedel */ 204883b0a91SJoerg Roedel int i; 205883b0a91SJoerg Roedel 206c62e2e94SBabu Moger if (!(vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT))) 207883b0a91SJoerg Roedel return true; 208883b0a91SJoerg Roedel 209883b0a91SJoerg Roedel for (i = 0; i < MSRPM_OFFSETS; i++) { 210883b0a91SJoerg Roedel u32 value, p; 211883b0a91SJoerg Roedel u64 offset; 212883b0a91SJoerg Roedel 213883b0a91SJoerg Roedel if (msrpm_offsets[i] == 0xffffffff) 214883b0a91SJoerg Roedel break; 215883b0a91SJoerg Roedel 216883b0a91SJoerg Roedel p = msrpm_offsets[i]; 217e670bf68SPaolo Bonzini offset = svm->nested.ctl.msrpm_base_pa + (p * 4); 218883b0a91SJoerg Roedel 219883b0a91SJoerg Roedel if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4)) 220883b0a91SJoerg Roedel return false; 221883b0a91SJoerg Roedel 222883b0a91SJoerg Roedel svm->nested.msrpm[p] = svm->msrpm[p] | value; 223883b0a91SJoerg Roedel } 224883b0a91SJoerg Roedel 225883b0a91SJoerg Roedel svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm)); 226883b0a91SJoerg Roedel 227883b0a91SJoerg Roedel return true; 228883b0a91SJoerg Roedel } 229883b0a91SJoerg Roedel 230ee695f22SKrish Sadhukhan /* 231ee695f22SKrish Sadhukhan * Bits 11:0 of bitmap address are ignored by hardware 232ee695f22SKrish Sadhukhan */ 233ee695f22SKrish Sadhukhan static bool nested_svm_check_bitmap_pa(struct kvm_vcpu *vcpu, u64 pa, u32 size) 234ee695f22SKrish Sadhukhan { 235ee695f22SKrish Sadhukhan u64 addr = PAGE_ALIGN(pa); 236ee695f22SKrish Sadhukhan 237ee695f22SKrish Sadhukhan return kvm_vcpu_is_legal_gpa(vcpu, addr) && 238ee695f22SKrish Sadhukhan kvm_vcpu_is_legal_gpa(vcpu, addr + size - 1); 239ee695f22SKrish Sadhukhan } 240ee695f22SKrish Sadhukhan 241174a921bSKrish Sadhukhan static bool nested_svm_check_tlb_ctl(struct kvm_vcpu *vcpu, u8 tlb_ctl) 242174a921bSKrish Sadhukhan { 243174a921bSKrish Sadhukhan /* Nested FLUSHBYASID is not supported yet. */ 244174a921bSKrish Sadhukhan switch(tlb_ctl) { 245174a921bSKrish Sadhukhan case TLB_CONTROL_DO_NOTHING: 246174a921bSKrish Sadhukhan case TLB_CONTROL_FLUSH_ALL_ASID: 247174a921bSKrish Sadhukhan return true; 248174a921bSKrish Sadhukhan default: 249174a921bSKrish Sadhukhan return false; 250174a921bSKrish Sadhukhan } 251174a921bSKrish Sadhukhan } 252174a921bSKrish Sadhukhan 253ee695f22SKrish Sadhukhan static bool nested_vmcb_check_controls(struct kvm_vcpu *vcpu, 254ee695f22SKrish Sadhukhan struct vmcb_control_area *control) 255ca46d739SPaolo Bonzini { 25611f0cbf0SSean Christopherson if (CC(!vmcb_is_intercept(control, INTERCEPT_VMRUN))) 257ca46d739SPaolo Bonzini return false; 258ca46d739SPaolo Bonzini 25911f0cbf0SSean Christopherson if (CC(control->asid == 0)) 260ca46d739SPaolo Bonzini return false; 261ca46d739SPaolo Bonzini 26211f0cbf0SSean Christopherson if (CC((control->nested_ctl & SVM_NESTED_CTL_NP_ENABLE) && !npt_enabled)) 263ca46d739SPaolo Bonzini return false; 264ca46d739SPaolo Bonzini 265ee695f22SKrish Sadhukhan if (CC(!nested_svm_check_bitmap_pa(vcpu, control->msrpm_base_pa, 266ee695f22SKrish Sadhukhan MSRPM_SIZE))) 267ee695f22SKrish Sadhukhan return false; 268ee695f22SKrish Sadhukhan if (CC(!nested_svm_check_bitmap_pa(vcpu, control->iopm_base_pa, 269ee695f22SKrish Sadhukhan IOPM_SIZE))) 270ee695f22SKrish Sadhukhan return false; 271ee695f22SKrish Sadhukhan 272174a921bSKrish Sadhukhan if (CC(!nested_svm_check_tlb_ctl(vcpu, control->tlb_ctl))) 273174a921bSKrish Sadhukhan return false; 274174a921bSKrish Sadhukhan 275ca46d739SPaolo Bonzini return true; 276ca46d739SPaolo Bonzini } 277ca46d739SPaolo Bonzini 2786906e06dSKrish Sadhukhan /* Common checks that apply to both L1 and L2 state. */ 27963129754SPaolo Bonzini static bool nested_vmcb_valid_sregs(struct kvm_vcpu *vcpu, 2806906e06dSKrish Sadhukhan struct vmcb_save_area *save) 2816906e06dSKrish Sadhukhan { 2823c346c0cSPaolo Bonzini /* 2833c346c0cSPaolo Bonzini * FIXME: these should be done after copying the fields, 2843c346c0cSPaolo Bonzini * to avoid TOC/TOU races. For these save area checks 2853c346c0cSPaolo Bonzini * the possible damage is limited since kvm_set_cr0 and 2863c346c0cSPaolo Bonzini * kvm_set_cr4 handle failure; EFER_SVME is an exception 2873c346c0cSPaolo Bonzini * so it is force-set later in nested_prepare_vmcb_save. 2883c346c0cSPaolo Bonzini */ 28911f0cbf0SSean Christopherson if (CC(!(save->efer & EFER_SVME))) 2906906e06dSKrish Sadhukhan return false; 2916906e06dSKrish Sadhukhan 29211f0cbf0SSean Christopherson if (CC((save->cr0 & X86_CR0_CD) == 0 && (save->cr0 & X86_CR0_NW)) || 29311f0cbf0SSean Christopherson CC(save->cr0 & ~0xffffffffULL)) 2946906e06dSKrish Sadhukhan return false; 2956906e06dSKrish Sadhukhan 29611f0cbf0SSean Christopherson if (CC(!kvm_dr6_valid(save->dr6)) || CC(!kvm_dr7_valid(save->dr7))) 2976906e06dSKrish Sadhukhan return false; 2986906e06dSKrish Sadhukhan 299907afa48SEmanuele Giuseppe Esposito /* 300907afa48SEmanuele Giuseppe Esposito * These checks are also performed by KVM_SET_SREGS, 301907afa48SEmanuele Giuseppe Esposito * except that EFER.LMA is not checked by SVM against 302907afa48SEmanuele Giuseppe Esposito * CR0.PG && EFER.LME. 303907afa48SEmanuele Giuseppe Esposito */ 304907afa48SEmanuele Giuseppe Esposito if ((save->efer & EFER_LME) && (save->cr0 & X86_CR0_PG)) { 305907afa48SEmanuele Giuseppe Esposito if (CC(!(save->cr4 & X86_CR4_PAE)) || 306907afa48SEmanuele Giuseppe Esposito CC(!(save->cr0 & X86_CR0_PE)) || 307907afa48SEmanuele Giuseppe Esposito CC(kvm_vcpu_is_illegal_gpa(vcpu, save->cr3))) 308907afa48SEmanuele Giuseppe Esposito return false; 309907afa48SEmanuele Giuseppe Esposito } 310907afa48SEmanuele Giuseppe Esposito 311907afa48SEmanuele Giuseppe Esposito if (CC(!kvm_is_valid_cr4(vcpu, save->cr4))) 3126906e06dSKrish Sadhukhan return false; 3136906e06dSKrish Sadhukhan 31463129754SPaolo Bonzini if (CC(!kvm_valid_efer(vcpu, save->efer))) 3156906e06dSKrish Sadhukhan return false; 3166906e06dSKrish Sadhukhan 3176906e06dSKrish Sadhukhan return true; 3186906e06dSKrish Sadhukhan } 3196906e06dSKrish Sadhukhan 320bb00bd9cSVitaly Kuznetsov void nested_load_control_from_vmcb12(struct vcpu_svm *svm, 3213e06f016SPaolo Bonzini struct vmcb_control_area *control) 3223e06f016SPaolo Bonzini { 323e670bf68SPaolo Bonzini copy_vmcb_control_area(&svm->nested.ctl, control); 3243e06f016SPaolo Bonzini 325cc440cdaSPaolo Bonzini /* Copy it here because nested_svm_check_controls will check it. */ 326cc440cdaSPaolo Bonzini svm->nested.ctl.asid = control->asid; 327e670bf68SPaolo Bonzini svm->nested.ctl.msrpm_base_pa &= ~0x0fffULL; 328e670bf68SPaolo Bonzini svm->nested.ctl.iopm_base_pa &= ~0x0fffULL; 3293e06f016SPaolo Bonzini } 3303e06f016SPaolo Bonzini 331*f2740a8dSEmanuele Giuseppe Esposito static void __nested_copy_vmcb_save_to_cache(struct vmcb_save_area_cached *to, 332*f2740a8dSEmanuele Giuseppe Esposito struct vmcb_save_area *from) 333*f2740a8dSEmanuele Giuseppe Esposito { 334*f2740a8dSEmanuele Giuseppe Esposito /* 335*f2740a8dSEmanuele Giuseppe Esposito * Copy only fields that are validated, as we need them 336*f2740a8dSEmanuele Giuseppe Esposito * to avoid TOC/TOU races. 337*f2740a8dSEmanuele Giuseppe Esposito */ 338*f2740a8dSEmanuele Giuseppe Esposito to->efer = from->efer; 339*f2740a8dSEmanuele Giuseppe Esposito to->cr0 = from->cr0; 340*f2740a8dSEmanuele Giuseppe Esposito to->cr3 = from->cr3; 341*f2740a8dSEmanuele Giuseppe Esposito to->cr4 = from->cr4; 342*f2740a8dSEmanuele Giuseppe Esposito 343*f2740a8dSEmanuele Giuseppe Esposito to->dr6 = from->dr6; 344*f2740a8dSEmanuele Giuseppe Esposito to->dr7 = from->dr7; 345*f2740a8dSEmanuele Giuseppe Esposito } 346*f2740a8dSEmanuele Giuseppe Esposito 347*f2740a8dSEmanuele Giuseppe Esposito void nested_copy_vmcb_save_to_cache(struct vcpu_svm *svm, 348*f2740a8dSEmanuele Giuseppe Esposito struct vmcb_save_area *save) 349*f2740a8dSEmanuele Giuseppe Esposito { 350*f2740a8dSEmanuele Giuseppe Esposito __nested_copy_vmcb_save_to_cache(&svm->nested.save, save); 351*f2740a8dSEmanuele Giuseppe Esposito } 352*f2740a8dSEmanuele Giuseppe Esposito 3532d8a42beSPaolo Bonzini /* 3542d8a42beSPaolo Bonzini * Synchronize fields that are written by the processor, so that 3559e8f0fbfSPaolo Bonzini * they can be copied back into the vmcb12. 3562d8a42beSPaolo Bonzini */ 3579e8f0fbfSPaolo Bonzini void nested_sync_control_from_vmcb02(struct vcpu_svm *svm) 3582d8a42beSPaolo Bonzini { 3592d8a42beSPaolo Bonzini u32 mask; 3602d8a42beSPaolo Bonzini svm->nested.ctl.event_inj = svm->vmcb->control.event_inj; 3612d8a42beSPaolo Bonzini svm->nested.ctl.event_inj_err = svm->vmcb->control.event_inj_err; 3622d8a42beSPaolo Bonzini 3632d8a42beSPaolo Bonzini /* Only a few fields of int_ctl are written by the processor. */ 3642d8a42beSPaolo Bonzini mask = V_IRQ_MASK | V_TPR_MASK; 3652d8a42beSPaolo Bonzini if (!(svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK) && 366a284ba56SJoerg Roedel svm_is_intercept(svm, INTERCEPT_VINTR)) { 3672d8a42beSPaolo Bonzini /* 3682d8a42beSPaolo Bonzini * In order to request an interrupt window, L0 is usurping 3692d8a42beSPaolo Bonzini * svm->vmcb->control.int_ctl and possibly setting V_IRQ 3702d8a42beSPaolo Bonzini * even if it was clear in L1's VMCB. Restoring it would be 3712d8a42beSPaolo Bonzini * wrong. However, in this case V_IRQ will remain true until 3722d8a42beSPaolo Bonzini * interrupt_window_interception calls svm_clear_vintr and 3732d8a42beSPaolo Bonzini * restores int_ctl. We can just leave it aside. 3742d8a42beSPaolo Bonzini */ 3752d8a42beSPaolo Bonzini mask &= ~V_IRQ_MASK; 3762d8a42beSPaolo Bonzini } 3772d8a42beSPaolo Bonzini svm->nested.ctl.int_ctl &= ~mask; 3782d8a42beSPaolo Bonzini svm->nested.ctl.int_ctl |= svm->vmcb->control.int_ctl & mask; 3792d8a42beSPaolo Bonzini } 3802d8a42beSPaolo Bonzini 38136e2e983SPaolo Bonzini /* 38236e2e983SPaolo Bonzini * Transfer any event that L0 or L1 wanted to inject into L2 to 38336e2e983SPaolo Bonzini * EXIT_INT_INFO. 38436e2e983SPaolo Bonzini */ 3859e8f0fbfSPaolo Bonzini static void nested_save_pending_event_to_vmcb12(struct vcpu_svm *svm, 3860dd16b5bSMaxim Levitsky struct vmcb *vmcb12) 38736e2e983SPaolo Bonzini { 38836e2e983SPaolo Bonzini struct kvm_vcpu *vcpu = &svm->vcpu; 38936e2e983SPaolo Bonzini u32 exit_int_info = 0; 39036e2e983SPaolo Bonzini unsigned int nr; 39136e2e983SPaolo Bonzini 39236e2e983SPaolo Bonzini if (vcpu->arch.exception.injected) { 39336e2e983SPaolo Bonzini nr = vcpu->arch.exception.nr; 39436e2e983SPaolo Bonzini exit_int_info = nr | SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_EXEPT; 39536e2e983SPaolo Bonzini 39636e2e983SPaolo Bonzini if (vcpu->arch.exception.has_error_code) { 39736e2e983SPaolo Bonzini exit_int_info |= SVM_EVTINJ_VALID_ERR; 3980dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info_err = 39936e2e983SPaolo Bonzini vcpu->arch.exception.error_code; 40036e2e983SPaolo Bonzini } 40136e2e983SPaolo Bonzini 40236e2e983SPaolo Bonzini } else if (vcpu->arch.nmi_injected) { 40336e2e983SPaolo Bonzini exit_int_info = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI; 40436e2e983SPaolo Bonzini 40536e2e983SPaolo Bonzini } else if (vcpu->arch.interrupt.injected) { 40636e2e983SPaolo Bonzini nr = vcpu->arch.interrupt.nr; 40736e2e983SPaolo Bonzini exit_int_info = nr | SVM_EVTINJ_VALID; 40836e2e983SPaolo Bonzini 40936e2e983SPaolo Bonzini if (vcpu->arch.interrupt.soft) 41036e2e983SPaolo Bonzini exit_int_info |= SVM_EVTINJ_TYPE_SOFT; 41136e2e983SPaolo Bonzini else 41236e2e983SPaolo Bonzini exit_int_info |= SVM_EVTINJ_TYPE_INTR; 41336e2e983SPaolo Bonzini } 41436e2e983SPaolo Bonzini 4150dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info = exit_int_info; 41636e2e983SPaolo Bonzini } 41736e2e983SPaolo Bonzini 41862156f6cSVitaly Kuznetsov static inline bool nested_npt_enabled(struct vcpu_svm *svm) 41962156f6cSVitaly Kuznetsov { 42062156f6cSVitaly Kuznetsov return svm->nested.ctl.nested_ctl & SVM_NESTED_CTL_NP_ENABLE; 42162156f6cSVitaly Kuznetsov } 42262156f6cSVitaly Kuznetsov 423d2e56019SSean Christopherson static void nested_svm_transition_tlb_flush(struct kvm_vcpu *vcpu) 424d2e56019SSean Christopherson { 425d2e56019SSean Christopherson /* 426d2e56019SSean Christopherson * TODO: optimize unconditional TLB flush/MMU sync. A partial list of 427d2e56019SSean Christopherson * things to fix before this can be conditional: 428d2e56019SSean Christopherson * 429d2e56019SSean Christopherson * - Flush TLBs for both L1 and L2 remote TLB flush 430d2e56019SSean Christopherson * - Honor L1's request to flush an ASID on nested VMRUN 431d2e56019SSean Christopherson * - Sync nested NPT MMU on VMRUN that flushes L2's ASID[*] 432d2e56019SSean Christopherson * - Don't crush a pending TLB flush in vmcb02 on nested VMRUN 433d2e56019SSean Christopherson * - Flush L1's ASID on KVM_REQ_TLB_FLUSH_GUEST 434d2e56019SSean Christopherson * 435d2e56019SSean Christopherson * [*] Unlike nested EPT, SVM's ASID management can invalidate nested 436d2e56019SSean Christopherson * NPT guest-physical mappings on VMRUN. 437d2e56019SSean Christopherson */ 438d2e56019SSean Christopherson kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); 439d2e56019SSean Christopherson kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 440d2e56019SSean Christopherson } 441d2e56019SSean Christopherson 44262156f6cSVitaly Kuznetsov /* 443d82aaef9SVitaly Kuznetsov * Load guest's/host's cr3 on nested vmentry or vmexit. @nested_npt is true 444d82aaef9SVitaly Kuznetsov * if we are emulating VM-Entry into a guest with NPT enabled. 44562156f6cSVitaly Kuznetsov */ 44662156f6cSVitaly Kuznetsov static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, 447b222b0b8SMaxim Levitsky bool nested_npt, bool reload_pdptrs) 44862156f6cSVitaly Kuznetsov { 44911f0cbf0SSean Christopherson if (CC(kvm_vcpu_is_illegal_gpa(vcpu, cr3))) 450a506fdd2SVitaly Kuznetsov return -EINVAL; 451a506fdd2SVitaly Kuznetsov 452b222b0b8SMaxim Levitsky if (reload_pdptrs && !nested_npt && is_pae_paging(vcpu) && 453a36dbec6SSean Christopherson CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))) 454a506fdd2SVitaly Kuznetsov return -EINVAL; 455a506fdd2SVitaly Kuznetsov 456a506fdd2SVitaly Kuznetsov if (!nested_npt) 457b5129100SSean Christopherson kvm_mmu_new_pgd(vcpu, cr3); 458a506fdd2SVitaly Kuznetsov 459a506fdd2SVitaly Kuznetsov vcpu->arch.cr3 = cr3; 460a506fdd2SVitaly Kuznetsov kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); 461a506fdd2SVitaly Kuznetsov 462616007c8SSean Christopherson /* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */ 463c9060662SSean Christopherson kvm_init_mmu(vcpu); 464a506fdd2SVitaly Kuznetsov 465a506fdd2SVitaly Kuznetsov return 0; 46662156f6cSVitaly Kuznetsov } 46762156f6cSVitaly Kuznetsov 4684995a368SCathy Avery void nested_vmcb02_compute_g_pat(struct vcpu_svm *svm) 4694995a368SCathy Avery { 4704995a368SCathy Avery if (!svm->nested.vmcb02.ptr) 4714995a368SCathy Avery return; 4724995a368SCathy Avery 4734995a368SCathy Avery /* FIXME: merge g_pat from vmcb01 and vmcb12. */ 4744995a368SCathy Avery svm->nested.vmcb02.ptr->save.g_pat = svm->vmcb01.ptr->save.g_pat; 4754995a368SCathy Avery } 4764995a368SCathy Avery 4779e8f0fbfSPaolo Bonzini static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12) 478883b0a91SJoerg Roedel { 4798173396eSCathy Avery bool new_vmcb12 = false; 4808173396eSCathy Avery 4814995a368SCathy Avery nested_vmcb02_compute_g_pat(svm); 4824995a368SCathy Avery 483883b0a91SJoerg Roedel /* Load the nested guest state */ 4848173396eSCathy Avery if (svm->nested.vmcb12_gpa != svm->nested.last_vmcb12_gpa) { 4858173396eSCathy Avery new_vmcb12 = true; 4868173396eSCathy Avery svm->nested.last_vmcb12_gpa = svm->nested.vmcb12_gpa; 4878173396eSCathy Avery } 4888173396eSCathy Avery 4898173396eSCathy Avery if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_SEG))) { 4900dd16b5bSMaxim Levitsky svm->vmcb->save.es = vmcb12->save.es; 4910dd16b5bSMaxim Levitsky svm->vmcb->save.cs = vmcb12->save.cs; 4920dd16b5bSMaxim Levitsky svm->vmcb->save.ss = vmcb12->save.ss; 4930dd16b5bSMaxim Levitsky svm->vmcb->save.ds = vmcb12->save.ds; 4944bb170a5SPaolo Bonzini svm->vmcb->save.cpl = vmcb12->save.cpl; 4954bb170a5SPaolo Bonzini vmcb_mark_dirty(svm->vmcb, VMCB_SEG); 4968173396eSCathy Avery } 4974bb170a5SPaolo Bonzini 4988173396eSCathy Avery if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_DT))) { 4990dd16b5bSMaxim Levitsky svm->vmcb->save.gdtr = vmcb12->save.gdtr; 5000dd16b5bSMaxim Levitsky svm->vmcb->save.idtr = vmcb12->save.idtr; 5014bb170a5SPaolo Bonzini vmcb_mark_dirty(svm->vmcb, VMCB_DT); 5028173396eSCathy Avery } 5034bb170a5SPaolo Bonzini 5048cce12b3SPaolo Bonzini kvm_set_rflags(&svm->vcpu, vmcb12->save.rflags | X86_EFLAGS_FIXED); 5053c346c0cSPaolo Bonzini 5063c346c0cSPaolo Bonzini /* 5073c346c0cSPaolo Bonzini * Force-set EFER_SVME even though it is checked earlier on the 5083c346c0cSPaolo Bonzini * VMCB12, because the guest can flip the bit between the check 5093c346c0cSPaolo Bonzini * and now. Clearing EFER_SVME would call svm_free_nested. 5103c346c0cSPaolo Bonzini */ 5113c346c0cSPaolo Bonzini svm_set_efer(&svm->vcpu, vmcb12->save.efer | EFER_SVME); 5123c346c0cSPaolo Bonzini 5130dd16b5bSMaxim Levitsky svm_set_cr0(&svm->vcpu, vmcb12->save.cr0); 5140dd16b5bSMaxim Levitsky svm_set_cr4(&svm->vcpu, vmcb12->save.cr4); 5154bb170a5SPaolo Bonzini 5164bb170a5SPaolo Bonzini svm->vcpu.arch.cr2 = vmcb12->save.cr2; 5178173396eSCathy Avery 5180dd16b5bSMaxim Levitsky kvm_rax_write(&svm->vcpu, vmcb12->save.rax); 5190dd16b5bSMaxim Levitsky kvm_rsp_write(&svm->vcpu, vmcb12->save.rsp); 5200dd16b5bSMaxim Levitsky kvm_rip_write(&svm->vcpu, vmcb12->save.rip); 521883b0a91SJoerg Roedel 522883b0a91SJoerg Roedel /* In case we don't even reach vcpu_run, the fields are not updated */ 5230dd16b5bSMaxim Levitsky svm->vmcb->save.rax = vmcb12->save.rax; 5240dd16b5bSMaxim Levitsky svm->vmcb->save.rsp = vmcb12->save.rsp; 5250dd16b5bSMaxim Levitsky svm->vmcb->save.rip = vmcb12->save.rip; 5264bb170a5SPaolo Bonzini 5278173396eSCathy Avery /* These bits will be set properly on the first execution when new_vmc12 is true */ 5288173396eSCathy Avery if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_DR))) { 5298cce12b3SPaolo Bonzini svm->vmcb->save.dr7 = vmcb12->save.dr7 | DR7_FIXED_1; 5309a3ecd5eSChenyi Qiang svm->vcpu.arch.dr6 = vmcb12->save.dr6 | DR6_ACTIVE_LOW; 5314bb170a5SPaolo Bonzini vmcb_mark_dirty(svm->vmcb, VMCB_DR); 532f241d711SPaolo Bonzini } 5338173396eSCathy Avery } 534883b0a91SJoerg Roedel 5359e8f0fbfSPaolo Bonzini static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) 536f241d711SPaolo Bonzini { 5370f923e07SMaxim Levitsky const u32 int_ctl_vmcb01_bits = 5380f923e07SMaxim Levitsky V_INTR_MASKING_MASK | V_GIF_MASK | V_GIF_ENABLE_MASK; 5390f923e07SMaxim Levitsky 5400f923e07SMaxim Levitsky const u32 int_ctl_vmcb12_bits = V_TPR_MASK | V_IRQ_INJECTION_BITS_MASK; 5410f923e07SMaxim Levitsky 542d2e56019SSean Christopherson struct kvm_vcpu *vcpu = &svm->vcpu; 54362156f6cSVitaly Kuznetsov 5447c3ecfcdSPaolo Bonzini /* 5457c3ecfcdSPaolo Bonzini * Filled at exit: exit_code, exit_code_hi, exit_info_1, exit_info_2, 5467c3ecfcdSPaolo Bonzini * exit_int_info, exit_int_info_err, next_rip, insn_len, insn_bytes. 5477c3ecfcdSPaolo Bonzini */ 5484995a368SCathy Avery 5497c3ecfcdSPaolo Bonzini /* 5507c3ecfcdSPaolo Bonzini * Also covers avic_vapic_bar, avic_backing_page, avic_logical_id, 5517c3ecfcdSPaolo Bonzini * avic_physical_id. 5527c3ecfcdSPaolo Bonzini */ 553feea0136SMaxim Levitsky WARN_ON(kvm_apicv_activated(svm->vcpu.kvm)); 5547c3ecfcdSPaolo Bonzini 5557c3ecfcdSPaolo Bonzini /* Copied from vmcb01. msrpm_base can be overwritten later. */ 5567c3ecfcdSPaolo Bonzini svm->vmcb->control.nested_ctl = svm->vmcb01.ptr->control.nested_ctl; 5577c3ecfcdSPaolo Bonzini svm->vmcb->control.iopm_base_pa = svm->vmcb01.ptr->control.iopm_base_pa; 5587c3ecfcdSPaolo Bonzini svm->vmcb->control.msrpm_base_pa = svm->vmcb01.ptr->control.msrpm_base_pa; 5597c3ecfcdSPaolo Bonzini 5607c3ecfcdSPaolo Bonzini /* Done at vmrun: asid. */ 5617c3ecfcdSPaolo Bonzini 5627c3ecfcdSPaolo Bonzini /* Also overwritten later if necessary. */ 5637c3ecfcdSPaolo Bonzini svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; 5647c3ecfcdSPaolo Bonzini 5657c3ecfcdSPaolo Bonzini /* nested_cr3. */ 56662156f6cSVitaly Kuznetsov if (nested_npt_enabled(svm)) 567d2e56019SSean Christopherson nested_svm_init_mmu_context(vcpu); 56869cb8774SPaolo Bonzini 5695228eb96SMaxim Levitsky vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset( 5705228eb96SMaxim Levitsky vcpu->arch.l1_tsc_offset, 5715228eb96SMaxim Levitsky svm->nested.ctl.tsc_offset, 5725228eb96SMaxim Levitsky svm->tsc_ratio_msr); 5735228eb96SMaxim Levitsky 5745228eb96SMaxim Levitsky svm->vmcb->control.tsc_offset = vcpu->arch.tsc_offset; 5755228eb96SMaxim Levitsky 5765228eb96SMaxim Levitsky if (svm->tsc_ratio_msr != kvm_default_tsc_scaling_ratio) { 5775228eb96SMaxim Levitsky WARN_ON(!svm->tsc_scaling_enabled); 5785228eb96SMaxim Levitsky nested_svm_update_tsc_ratio_msr(vcpu); 5795228eb96SMaxim Levitsky } 580883b0a91SJoerg Roedel 58191b7130cSPaolo Bonzini svm->vmcb->control.int_ctl = 5820f923e07SMaxim Levitsky (svm->nested.ctl.int_ctl & int_ctl_vmcb12_bits) | 5830f923e07SMaxim Levitsky (svm->vmcb01.ptr->control.int_ctl & int_ctl_vmcb01_bits); 58491b7130cSPaolo Bonzini 585e670bf68SPaolo Bonzini svm->vmcb->control.int_vector = svm->nested.ctl.int_vector; 586e670bf68SPaolo Bonzini svm->vmcb->control.int_state = svm->nested.ctl.int_state; 587e670bf68SPaolo Bonzini svm->vmcb->control.event_inj = svm->nested.ctl.event_inj; 588e670bf68SPaolo Bonzini svm->vmcb->control.event_inj_err = svm->nested.ctl.event_inj_err; 589883b0a91SJoerg Roedel 590d2e56019SSean Christopherson nested_svm_transition_tlb_flush(vcpu); 591d2e56019SSean Christopherson 592883b0a91SJoerg Roedel /* Enter Guest-Mode */ 593d2e56019SSean Christopherson enter_guest_mode(vcpu); 594883b0a91SJoerg Roedel 595883b0a91SJoerg Roedel /* 596883b0a91SJoerg Roedel * Merge guest and host intercepts - must be called with vcpu in 5974bb170a5SPaolo Bonzini * guest-mode to take effect. 598883b0a91SJoerg Roedel */ 599883b0a91SJoerg Roedel recalc_intercepts(svm); 600f241d711SPaolo Bonzini } 601f241d711SPaolo Bonzini 602d00b99c5SBabu Moger static void nested_svm_copy_common_state(struct vmcb *from_vmcb, struct vmcb *to_vmcb) 603d00b99c5SBabu Moger { 604d00b99c5SBabu Moger /* 605d00b99c5SBabu Moger * Some VMCB state is shared between L1 and L2 and thus has to be 606d00b99c5SBabu Moger * moved at the time of nested vmrun and vmexit. 607d00b99c5SBabu Moger * 608d00b99c5SBabu Moger * VMLOAD/VMSAVE state would also belong in this category, but KVM 609d00b99c5SBabu Moger * always performs VMLOAD and VMSAVE from the VMCB01. 610d00b99c5SBabu Moger */ 611d00b99c5SBabu Moger to_vmcb->save.spec_ctrl = from_vmcb->save.spec_ctrl; 612d00b99c5SBabu Moger } 613d00b99c5SBabu Moger 61463129754SPaolo Bonzini int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb12_gpa, 615e85d3e7bSMaxim Levitsky struct vmcb *vmcb12, bool from_vmrun) 616f241d711SPaolo Bonzini { 61763129754SPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 618a506fdd2SVitaly Kuznetsov int ret; 619a506fdd2SVitaly Kuznetsov 620954f419bSMaxim Levitsky trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb12_gpa, 621954f419bSMaxim Levitsky vmcb12->save.rip, 622954f419bSMaxim Levitsky vmcb12->control.int_ctl, 623954f419bSMaxim Levitsky vmcb12->control.event_inj, 624954f419bSMaxim Levitsky vmcb12->control.nested_ctl); 625954f419bSMaxim Levitsky 626954f419bSMaxim Levitsky trace_kvm_nested_intercepts(vmcb12->control.intercepts[INTERCEPT_CR] & 0xffff, 627954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_CR] >> 16, 628954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_EXCEPTION], 629954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_WORD3], 630954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_WORD4], 631954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_WORD5]); 632954f419bSMaxim Levitsky 633954f419bSMaxim Levitsky 6340dd16b5bSMaxim Levitsky svm->nested.vmcb12_gpa = vmcb12_gpa; 6354995a368SCathy Avery 6364995a368SCathy Avery WARN_ON(svm->vmcb == svm->nested.vmcb02.ptr); 6374995a368SCathy Avery 638d00b99c5SBabu Moger nested_svm_copy_common_state(svm->vmcb01.ptr, svm->nested.vmcb02.ptr); 6394995a368SCathy Avery 6404995a368SCathy Avery svm_switch_vmcb(svm, &svm->nested.vmcb02); 6419e8f0fbfSPaolo Bonzini nested_vmcb02_prepare_control(svm); 6429e8f0fbfSPaolo Bonzini nested_vmcb02_prepare_save(svm, vmcb12); 643f241d711SPaolo Bonzini 6440dd16b5bSMaxim Levitsky ret = nested_svm_load_cr3(&svm->vcpu, vmcb12->save.cr3, 645e85d3e7bSMaxim Levitsky nested_npt_enabled(svm), from_vmrun); 646a506fdd2SVitaly Kuznetsov if (ret) 647a506fdd2SVitaly Kuznetsov return ret; 648a506fdd2SVitaly Kuznetsov 649a04aead1SPaolo Bonzini if (!npt_enabled) 65063129754SPaolo Bonzini vcpu->arch.mmu->inject_page_fault = svm_inject_page_fault_nested; 651a04aead1SPaolo Bonzini 652e85d3e7bSMaxim Levitsky if (!from_vmrun) 653e85d3e7bSMaxim Levitsky kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 654e85d3e7bSMaxim Levitsky 655ffdf7f9eSPaolo Bonzini svm_set_gif(svm, true); 65659cd9bc5SVitaly Kuznetsov 65759cd9bc5SVitaly Kuznetsov return 0; 658883b0a91SJoerg Roedel } 659883b0a91SJoerg Roedel 66063129754SPaolo Bonzini int nested_svm_vmrun(struct kvm_vcpu *vcpu) 661883b0a91SJoerg Roedel { 66263129754SPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 663883b0a91SJoerg Roedel int ret; 6640dd16b5bSMaxim Levitsky struct vmcb *vmcb12; 665883b0a91SJoerg Roedel struct kvm_host_map map; 6660dd16b5bSMaxim Levitsky u64 vmcb12_gpa; 667883b0a91SJoerg Roedel 668fb79f566SVitaly Kuznetsov if (!svm->nested.hsave_msr) { 669fb79f566SVitaly Kuznetsov kvm_inject_gp(vcpu, 0); 670fb79f566SVitaly Kuznetsov return 1; 671fb79f566SVitaly Kuznetsov } 672fb79f566SVitaly Kuznetsov 67363129754SPaolo Bonzini if (is_smm(vcpu)) { 67463129754SPaolo Bonzini kvm_queue_exception(vcpu, UD_VECTOR); 6757c67f546SPaolo Bonzini return 1; 6767c67f546SPaolo Bonzini } 677883b0a91SJoerg Roedel 6780dd16b5bSMaxim Levitsky vmcb12_gpa = svm->vmcb->save.rax; 67963129754SPaolo Bonzini ret = kvm_vcpu_map(vcpu, gpa_to_gfn(vmcb12_gpa), &map); 680883b0a91SJoerg Roedel if (ret == -EINVAL) { 68163129754SPaolo Bonzini kvm_inject_gp(vcpu, 0); 682883b0a91SJoerg Roedel return 1; 683883b0a91SJoerg Roedel } else if (ret) { 68463129754SPaolo Bonzini return kvm_skip_emulated_instruction(vcpu); 685883b0a91SJoerg Roedel } 686883b0a91SJoerg Roedel 68763129754SPaolo Bonzini ret = kvm_skip_emulated_instruction(vcpu); 688883b0a91SJoerg Roedel 6890dd16b5bSMaxim Levitsky vmcb12 = map.hva; 690883b0a91SJoerg Roedel 6912fcf4876SMaxim Levitsky if (WARN_ON_ONCE(!svm->nested.initialized)) 6922fcf4876SMaxim Levitsky return -EINVAL; 6932fcf4876SMaxim Levitsky 694cb9b6a1bSPaolo Bonzini nested_load_control_from_vmcb12(svm, &vmcb12->control); 695*f2740a8dSEmanuele Giuseppe Esposito nested_copy_vmcb_save_to_cache(svm, &vmcb12->save); 696cb9b6a1bSPaolo Bonzini 697cb9b6a1bSPaolo Bonzini if (!nested_vmcb_valid_sregs(vcpu, &vmcb12->save) || 698ee695f22SKrish Sadhukhan !nested_vmcb_check_controls(vcpu, &svm->nested.ctl)) { 6990dd16b5bSMaxim Levitsky vmcb12->control.exit_code = SVM_EXIT_ERR; 7000dd16b5bSMaxim Levitsky vmcb12->control.exit_code_hi = 0; 7010dd16b5bSMaxim Levitsky vmcb12->control.exit_info_1 = 0; 7020dd16b5bSMaxim Levitsky vmcb12->control.exit_info_2 = 0; 70369c9dfa2SPaolo Bonzini goto out; 704883b0a91SJoerg Roedel } 705883b0a91SJoerg Roedel 706883b0a91SJoerg Roedel /* 7074995a368SCathy Avery * Since vmcb01 is not in use, we can use it to store some of the L1 7084995a368SCathy Avery * state. 709883b0a91SJoerg Roedel */ 71063129754SPaolo Bonzini svm->vmcb01.ptr->save.efer = vcpu->arch.efer; 71163129754SPaolo Bonzini svm->vmcb01.ptr->save.cr0 = kvm_read_cr0(vcpu); 71263129754SPaolo Bonzini svm->vmcb01.ptr->save.cr4 = vcpu->arch.cr4; 71363129754SPaolo Bonzini svm->vmcb01.ptr->save.rflags = kvm_get_rflags(vcpu); 71463129754SPaolo Bonzini svm->vmcb01.ptr->save.rip = kvm_rip_read(vcpu); 715883b0a91SJoerg Roedel 7164995a368SCathy Avery if (!npt_enabled) 71763129754SPaolo Bonzini svm->vmcb01.ptr->save.cr3 = kvm_read_cr3(vcpu); 718883b0a91SJoerg Roedel 719f74f9414SPaolo Bonzini svm->nested.nested_run_pending = 1; 720883b0a91SJoerg Roedel 721e85d3e7bSMaxim Levitsky if (enter_svm_guest_mode(vcpu, vmcb12_gpa, vmcb12, true)) 72259cd9bc5SVitaly Kuznetsov goto out_exit_err; 72359cd9bc5SVitaly Kuznetsov 72459cd9bc5SVitaly Kuznetsov if (nested_svm_vmrun_msrpm(svm)) 72559cd9bc5SVitaly Kuznetsov goto out; 72659cd9bc5SVitaly Kuznetsov 72759cd9bc5SVitaly Kuznetsov out_exit_err: 728ebdb3dbaSVitaly Kuznetsov svm->nested.nested_run_pending = 0; 729ebdb3dbaSVitaly Kuznetsov 730883b0a91SJoerg Roedel svm->vmcb->control.exit_code = SVM_EXIT_ERR; 731883b0a91SJoerg Roedel svm->vmcb->control.exit_code_hi = 0; 732883b0a91SJoerg Roedel svm->vmcb->control.exit_info_1 = 0; 733883b0a91SJoerg Roedel svm->vmcb->control.exit_info_2 = 0; 734883b0a91SJoerg Roedel 735883b0a91SJoerg Roedel nested_svm_vmexit(svm); 736883b0a91SJoerg Roedel 73769c9dfa2SPaolo Bonzini out: 73863129754SPaolo Bonzini kvm_vcpu_unmap(vcpu, &map, true); 73969c9dfa2SPaolo Bonzini 740883b0a91SJoerg Roedel return ret; 741883b0a91SJoerg Roedel } 742883b0a91SJoerg Roedel 7430a758290SVitaly Kuznetsov /* Copy state save area fields which are handled by VMRUN */ 7442bb16beaSVitaly Kuznetsov void svm_copy_vmrun_state(struct vmcb_save_area *to_save, 7452bb16beaSVitaly Kuznetsov struct vmcb_save_area *from_save) 7460a758290SVitaly Kuznetsov { 7470a758290SVitaly Kuznetsov to_save->es = from_save->es; 7480a758290SVitaly Kuznetsov to_save->cs = from_save->cs; 7490a758290SVitaly Kuznetsov to_save->ss = from_save->ss; 7500a758290SVitaly Kuznetsov to_save->ds = from_save->ds; 7510a758290SVitaly Kuznetsov to_save->gdtr = from_save->gdtr; 7520a758290SVitaly Kuznetsov to_save->idtr = from_save->idtr; 7530a758290SVitaly Kuznetsov to_save->rflags = from_save->rflags | X86_EFLAGS_FIXED; 7540a758290SVitaly Kuznetsov to_save->efer = from_save->efer; 7550a758290SVitaly Kuznetsov to_save->cr0 = from_save->cr0; 7560a758290SVitaly Kuznetsov to_save->cr3 = from_save->cr3; 7570a758290SVitaly Kuznetsov to_save->cr4 = from_save->cr4; 7580a758290SVitaly Kuznetsov to_save->rax = from_save->rax; 7590a758290SVitaly Kuznetsov to_save->rsp = from_save->rsp; 7600a758290SVitaly Kuznetsov to_save->rip = from_save->rip; 7610a758290SVitaly Kuznetsov to_save->cpl = 0; 7620a758290SVitaly Kuznetsov } 7630a758290SVitaly Kuznetsov 7642bb16beaSVitaly Kuznetsov void svm_copy_vmloadsave_state(struct vmcb *to_vmcb, struct vmcb *from_vmcb) 765883b0a91SJoerg Roedel { 766883b0a91SJoerg Roedel to_vmcb->save.fs = from_vmcb->save.fs; 767883b0a91SJoerg Roedel to_vmcb->save.gs = from_vmcb->save.gs; 768883b0a91SJoerg Roedel to_vmcb->save.tr = from_vmcb->save.tr; 769883b0a91SJoerg Roedel to_vmcb->save.ldtr = from_vmcb->save.ldtr; 770883b0a91SJoerg Roedel to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base; 771883b0a91SJoerg Roedel to_vmcb->save.star = from_vmcb->save.star; 772883b0a91SJoerg Roedel to_vmcb->save.lstar = from_vmcb->save.lstar; 773883b0a91SJoerg Roedel to_vmcb->save.cstar = from_vmcb->save.cstar; 774883b0a91SJoerg Roedel to_vmcb->save.sfmask = from_vmcb->save.sfmask; 775883b0a91SJoerg Roedel to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs; 776883b0a91SJoerg Roedel to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp; 777883b0a91SJoerg Roedel to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip; 778883b0a91SJoerg Roedel } 779883b0a91SJoerg Roedel 780883b0a91SJoerg Roedel int nested_svm_vmexit(struct vcpu_svm *svm) 781883b0a91SJoerg Roedel { 78263129754SPaolo Bonzini struct kvm_vcpu *vcpu = &svm->vcpu; 7830dd16b5bSMaxim Levitsky struct vmcb *vmcb12; 784883b0a91SJoerg Roedel struct vmcb *vmcb = svm->vmcb; 785883b0a91SJoerg Roedel struct kvm_host_map map; 78663129754SPaolo Bonzini int rc; 787883b0a91SJoerg Roedel 788cb6a32c2SSean Christopherson /* Triple faults in L2 should never escape. */ 789cb6a32c2SSean Christopherson WARN_ON_ONCE(kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)); 790cb6a32c2SSean Christopherson 79163129754SPaolo Bonzini rc = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa), &map); 792883b0a91SJoerg Roedel if (rc) { 793883b0a91SJoerg Roedel if (rc == -EINVAL) 79463129754SPaolo Bonzini kvm_inject_gp(vcpu, 0); 795883b0a91SJoerg Roedel return 1; 796883b0a91SJoerg Roedel } 797883b0a91SJoerg Roedel 7980dd16b5bSMaxim Levitsky vmcb12 = map.hva; 799883b0a91SJoerg Roedel 800883b0a91SJoerg Roedel /* Exit Guest-Mode */ 80163129754SPaolo Bonzini leave_guest_mode(vcpu); 8020dd16b5bSMaxim Levitsky svm->nested.vmcb12_gpa = 0; 8032d8a42beSPaolo Bonzini WARN_ON_ONCE(svm->nested.nested_run_pending); 804883b0a91SJoerg Roedel 80563129754SPaolo Bonzini kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 806f2c7ef3bSMaxim Levitsky 80738c0b192SPaolo Bonzini /* in case we halted in L2 */ 80838c0b192SPaolo Bonzini svm->vcpu.arch.mp_state = KVM_MP_STATE_RUNNABLE; 80938c0b192SPaolo Bonzini 810883b0a91SJoerg Roedel /* Give the current vmcb to the guest */ 811883b0a91SJoerg Roedel 8120dd16b5bSMaxim Levitsky vmcb12->save.es = vmcb->save.es; 8130dd16b5bSMaxim Levitsky vmcb12->save.cs = vmcb->save.cs; 8140dd16b5bSMaxim Levitsky vmcb12->save.ss = vmcb->save.ss; 8150dd16b5bSMaxim Levitsky vmcb12->save.ds = vmcb->save.ds; 8160dd16b5bSMaxim Levitsky vmcb12->save.gdtr = vmcb->save.gdtr; 8170dd16b5bSMaxim Levitsky vmcb12->save.idtr = vmcb->save.idtr; 8180dd16b5bSMaxim Levitsky vmcb12->save.efer = svm->vcpu.arch.efer; 81963129754SPaolo Bonzini vmcb12->save.cr0 = kvm_read_cr0(vcpu); 82063129754SPaolo Bonzini vmcb12->save.cr3 = kvm_read_cr3(vcpu); 8210dd16b5bSMaxim Levitsky vmcb12->save.cr2 = vmcb->save.cr2; 8220dd16b5bSMaxim Levitsky vmcb12->save.cr4 = svm->vcpu.arch.cr4; 82363129754SPaolo Bonzini vmcb12->save.rflags = kvm_get_rflags(vcpu); 82463129754SPaolo Bonzini vmcb12->save.rip = kvm_rip_read(vcpu); 82563129754SPaolo Bonzini vmcb12->save.rsp = kvm_rsp_read(vcpu); 82663129754SPaolo Bonzini vmcb12->save.rax = kvm_rax_read(vcpu); 8270dd16b5bSMaxim Levitsky vmcb12->save.dr7 = vmcb->save.dr7; 8280dd16b5bSMaxim Levitsky vmcb12->save.dr6 = svm->vcpu.arch.dr6; 8290dd16b5bSMaxim Levitsky vmcb12->save.cpl = vmcb->save.cpl; 830883b0a91SJoerg Roedel 8310dd16b5bSMaxim Levitsky vmcb12->control.int_state = vmcb->control.int_state; 8320dd16b5bSMaxim Levitsky vmcb12->control.exit_code = vmcb->control.exit_code; 8330dd16b5bSMaxim Levitsky vmcb12->control.exit_code_hi = vmcb->control.exit_code_hi; 8340dd16b5bSMaxim Levitsky vmcb12->control.exit_info_1 = vmcb->control.exit_info_1; 8350dd16b5bSMaxim Levitsky vmcb12->control.exit_info_2 = vmcb->control.exit_info_2; 83636e2e983SPaolo Bonzini 8370dd16b5bSMaxim Levitsky if (vmcb12->control.exit_code != SVM_EXIT_ERR) 8389e8f0fbfSPaolo Bonzini nested_save_pending_event_to_vmcb12(svm, vmcb12); 839883b0a91SJoerg Roedel 840883b0a91SJoerg Roedel if (svm->nrips_enabled) 8410dd16b5bSMaxim Levitsky vmcb12->control.next_rip = vmcb->control.next_rip; 842883b0a91SJoerg Roedel 8430dd16b5bSMaxim Levitsky vmcb12->control.int_ctl = svm->nested.ctl.int_ctl; 8440dd16b5bSMaxim Levitsky vmcb12->control.tlb_ctl = svm->nested.ctl.tlb_ctl; 8450dd16b5bSMaxim Levitsky vmcb12->control.event_inj = svm->nested.ctl.event_inj; 8460dd16b5bSMaxim Levitsky vmcb12->control.event_inj_err = svm->nested.ctl.event_inj_err; 847883b0a91SJoerg Roedel 848d00b99c5SBabu Moger nested_svm_copy_common_state(svm->nested.vmcb02.ptr, svm->vmcb01.ptr); 849d00b99c5SBabu Moger 8504995a368SCathy Avery svm_switch_vmcb(svm, &svm->vmcb01); 8514995a368SCathy Avery 8524995a368SCathy Avery /* 8534995a368SCathy Avery * On vmexit the GIF is set to false and 8544995a368SCathy Avery * no event can be injected in L1. 8554995a368SCathy Avery */ 8569883764aSMaxim Levitsky svm_set_gif(svm, false); 8574995a368SCathy Avery svm->vmcb->control.exit_int_info = 0; 8589883764aSMaxim Levitsky 8597ca62d13SPaolo Bonzini svm->vcpu.arch.tsc_offset = svm->vcpu.arch.l1_tsc_offset; 8607ca62d13SPaolo Bonzini if (svm->vmcb->control.tsc_offset != svm->vcpu.arch.tsc_offset) { 8617ca62d13SPaolo Bonzini svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset; 8627ca62d13SPaolo Bonzini vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 8637ca62d13SPaolo Bonzini } 86418fc6c55SPaolo Bonzini 8655228eb96SMaxim Levitsky if (svm->tsc_ratio_msr != kvm_default_tsc_scaling_ratio) { 8665228eb96SMaxim Levitsky WARN_ON(!svm->tsc_scaling_enabled); 8675228eb96SMaxim Levitsky vcpu->arch.tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; 8685228eb96SMaxim Levitsky svm_write_tsc_multiplier(vcpu, vcpu->arch.tsc_scaling_ratio); 8695228eb96SMaxim Levitsky } 8705228eb96SMaxim Levitsky 871e670bf68SPaolo Bonzini svm->nested.ctl.nested_cr3 = 0; 872883b0a91SJoerg Roedel 8734995a368SCathy Avery /* 8744995a368SCathy Avery * Restore processor state that had been saved in vmcb01 8754995a368SCathy Avery */ 87663129754SPaolo Bonzini kvm_set_rflags(vcpu, svm->vmcb->save.rflags); 87763129754SPaolo Bonzini svm_set_efer(vcpu, svm->vmcb->save.efer); 87863129754SPaolo Bonzini svm_set_cr0(vcpu, svm->vmcb->save.cr0 | X86_CR0_PE); 87963129754SPaolo Bonzini svm_set_cr4(vcpu, svm->vmcb->save.cr4); 88063129754SPaolo Bonzini kvm_rax_write(vcpu, svm->vmcb->save.rax); 88163129754SPaolo Bonzini kvm_rsp_write(vcpu, svm->vmcb->save.rsp); 88263129754SPaolo Bonzini kvm_rip_write(vcpu, svm->vmcb->save.rip); 8834995a368SCathy Avery 8844995a368SCathy Avery svm->vcpu.arch.dr7 = DR7_FIXED_1; 8854995a368SCathy Avery kvm_update_dr7(&svm->vcpu); 886883b0a91SJoerg Roedel 8870dd16b5bSMaxim Levitsky trace_kvm_nested_vmexit_inject(vmcb12->control.exit_code, 8880dd16b5bSMaxim Levitsky vmcb12->control.exit_info_1, 8890dd16b5bSMaxim Levitsky vmcb12->control.exit_info_2, 8900dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info, 8910dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info_err, 89236e2e983SPaolo Bonzini KVM_ISA_SVM); 89336e2e983SPaolo Bonzini 89463129754SPaolo Bonzini kvm_vcpu_unmap(vcpu, &map, true); 895883b0a91SJoerg Roedel 896d2e56019SSean Christopherson nested_svm_transition_tlb_flush(vcpu); 897d2e56019SSean Christopherson 89863129754SPaolo Bonzini nested_svm_uninit_mmu_context(vcpu); 899bf7dea42SVitaly Kuznetsov 900b222b0b8SMaxim Levitsky rc = nested_svm_load_cr3(vcpu, svm->vmcb->save.cr3, false, true); 901d82aaef9SVitaly Kuznetsov if (rc) 902d82aaef9SVitaly Kuznetsov return 1; 903bf7dea42SVitaly Kuznetsov 904883b0a91SJoerg Roedel /* 905883b0a91SJoerg Roedel * Drop what we picked up for L2 via svm_complete_interrupts() so it 906883b0a91SJoerg Roedel * doesn't end up in L1. 907883b0a91SJoerg Roedel */ 908883b0a91SJoerg Roedel svm->vcpu.arch.nmi_injected = false; 90963129754SPaolo Bonzini kvm_clear_exception_queue(vcpu); 91063129754SPaolo Bonzini kvm_clear_interrupt_queue(vcpu); 911883b0a91SJoerg Roedel 9129a7de6ecSKrish Sadhukhan /* 9139a7de6ecSKrish Sadhukhan * If we are here following the completion of a VMRUN that 9149a7de6ecSKrish Sadhukhan * is being single-stepped, queue the pending #DB intercept 9159a7de6ecSKrish Sadhukhan * right now so that it an be accounted for before we execute 9169a7de6ecSKrish Sadhukhan * L1's next instruction. 9179a7de6ecSKrish Sadhukhan */ 9189a7de6ecSKrish Sadhukhan if (unlikely(svm->vmcb->save.rflags & X86_EFLAGS_TF)) 9199a7de6ecSKrish Sadhukhan kvm_queue_exception(&(svm->vcpu), DB_VECTOR); 9209a7de6ecSKrish Sadhukhan 921883b0a91SJoerg Roedel return 0; 922883b0a91SJoerg Roedel } 923883b0a91SJoerg Roedel 924cb6a32c2SSean Christopherson static void nested_svm_triple_fault(struct kvm_vcpu *vcpu) 925cb6a32c2SSean Christopherson { 9263a87c7e0SSean Christopherson nested_svm_simple_vmexit(to_svm(vcpu), SVM_EXIT_SHUTDOWN); 927cb6a32c2SSean Christopherson } 928cb6a32c2SSean Christopherson 9292fcf4876SMaxim Levitsky int svm_allocate_nested(struct vcpu_svm *svm) 9302fcf4876SMaxim Levitsky { 9314995a368SCathy Avery struct page *vmcb02_page; 9322fcf4876SMaxim Levitsky 9332fcf4876SMaxim Levitsky if (svm->nested.initialized) 9342fcf4876SMaxim Levitsky return 0; 9352fcf4876SMaxim Levitsky 9364995a368SCathy Avery vmcb02_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 9374995a368SCathy Avery if (!vmcb02_page) 9382fcf4876SMaxim Levitsky return -ENOMEM; 9394995a368SCathy Avery svm->nested.vmcb02.ptr = page_address(vmcb02_page); 9404995a368SCathy Avery svm->nested.vmcb02.pa = __sme_set(page_to_pfn(vmcb02_page) << PAGE_SHIFT); 9412fcf4876SMaxim Levitsky 9422fcf4876SMaxim Levitsky svm->nested.msrpm = svm_vcpu_alloc_msrpm(); 9432fcf4876SMaxim Levitsky if (!svm->nested.msrpm) 9444995a368SCathy Avery goto err_free_vmcb02; 9452fcf4876SMaxim Levitsky svm_vcpu_init_msrpm(&svm->vcpu, svm->nested.msrpm); 9462fcf4876SMaxim Levitsky 9472fcf4876SMaxim Levitsky svm->nested.initialized = true; 9482fcf4876SMaxim Levitsky return 0; 9492fcf4876SMaxim Levitsky 9504995a368SCathy Avery err_free_vmcb02: 9514995a368SCathy Avery __free_page(vmcb02_page); 9522fcf4876SMaxim Levitsky return -ENOMEM; 9532fcf4876SMaxim Levitsky } 9542fcf4876SMaxim Levitsky 9552fcf4876SMaxim Levitsky void svm_free_nested(struct vcpu_svm *svm) 9562fcf4876SMaxim Levitsky { 9572fcf4876SMaxim Levitsky if (!svm->nested.initialized) 9582fcf4876SMaxim Levitsky return; 9592fcf4876SMaxim Levitsky 9602fcf4876SMaxim Levitsky svm_vcpu_free_msrpm(svm->nested.msrpm); 9612fcf4876SMaxim Levitsky svm->nested.msrpm = NULL; 9622fcf4876SMaxim Levitsky 9634995a368SCathy Avery __free_page(virt_to_page(svm->nested.vmcb02.ptr)); 9644995a368SCathy Avery svm->nested.vmcb02.ptr = NULL; 9652fcf4876SMaxim Levitsky 966c74ad08fSMaxim Levitsky /* 967c74ad08fSMaxim Levitsky * When last_vmcb12_gpa matches the current vmcb12 gpa, 968c74ad08fSMaxim Levitsky * some vmcb12 fields are not loaded if they are marked clean 969c74ad08fSMaxim Levitsky * in the vmcb12, since in this case they are up to date already. 970c74ad08fSMaxim Levitsky * 971c74ad08fSMaxim Levitsky * When the vmcb02 is freed, this optimization becomes invalid. 972c74ad08fSMaxim Levitsky */ 973c74ad08fSMaxim Levitsky svm->nested.last_vmcb12_gpa = INVALID_GPA; 974c74ad08fSMaxim Levitsky 9752fcf4876SMaxim Levitsky svm->nested.initialized = false; 9762fcf4876SMaxim Levitsky } 9772fcf4876SMaxim Levitsky 978c513f484SPaolo Bonzini /* 979c513f484SPaolo Bonzini * Forcibly leave nested mode in order to be able to reset the VCPU later on. 980c513f484SPaolo Bonzini */ 981c513f484SPaolo Bonzini void svm_leave_nested(struct vcpu_svm *svm) 982c513f484SPaolo Bonzini { 98363129754SPaolo Bonzini struct kvm_vcpu *vcpu = &svm->vcpu; 98463129754SPaolo Bonzini 98563129754SPaolo Bonzini if (is_guest_mode(vcpu)) { 986c513f484SPaolo Bonzini svm->nested.nested_run_pending = 0; 987c74ad08fSMaxim Levitsky svm->nested.vmcb12_gpa = INVALID_GPA; 988c74ad08fSMaxim Levitsky 98963129754SPaolo Bonzini leave_guest_mode(vcpu); 9904995a368SCathy Avery 991deee59baSMaxim Levitsky svm_switch_vmcb(svm, &svm->vmcb01); 9924995a368SCathy Avery 99363129754SPaolo Bonzini nested_svm_uninit_mmu_context(vcpu); 99456fe28deSMaxim Levitsky vmcb_mark_all_dirty(svm->vmcb); 995c513f484SPaolo Bonzini } 996a7d5c7ceSPaolo Bonzini 99763129754SPaolo Bonzini kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 998c513f484SPaolo Bonzini } 999c513f484SPaolo Bonzini 1000883b0a91SJoerg Roedel static int nested_svm_exit_handled_msr(struct vcpu_svm *svm) 1001883b0a91SJoerg Roedel { 1002883b0a91SJoerg Roedel u32 offset, msr, value; 1003883b0a91SJoerg Roedel int write, mask; 1004883b0a91SJoerg Roedel 1005c62e2e94SBabu Moger if (!(vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT))) 1006883b0a91SJoerg Roedel return NESTED_EXIT_HOST; 1007883b0a91SJoerg Roedel 1008883b0a91SJoerg Roedel msr = svm->vcpu.arch.regs[VCPU_REGS_RCX]; 1009883b0a91SJoerg Roedel offset = svm_msrpm_offset(msr); 1010883b0a91SJoerg Roedel write = svm->vmcb->control.exit_info_1 & 1; 1011883b0a91SJoerg Roedel mask = 1 << ((2 * (msr & 0xf)) + write); 1012883b0a91SJoerg Roedel 1013883b0a91SJoerg Roedel if (offset == MSR_INVALID) 1014883b0a91SJoerg Roedel return NESTED_EXIT_DONE; 1015883b0a91SJoerg Roedel 1016883b0a91SJoerg Roedel /* Offset is in 32 bit units but need in 8 bit units */ 1017883b0a91SJoerg Roedel offset *= 4; 1018883b0a91SJoerg Roedel 1019e670bf68SPaolo Bonzini if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.ctl.msrpm_base_pa + offset, &value, 4)) 1020883b0a91SJoerg Roedel return NESTED_EXIT_DONE; 1021883b0a91SJoerg Roedel 1022883b0a91SJoerg Roedel return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; 1023883b0a91SJoerg Roedel } 1024883b0a91SJoerg Roedel 1025883b0a91SJoerg Roedel static int nested_svm_intercept_ioio(struct vcpu_svm *svm) 1026883b0a91SJoerg Roedel { 1027883b0a91SJoerg Roedel unsigned port, size, iopm_len; 1028883b0a91SJoerg Roedel u16 val, mask; 1029883b0a91SJoerg Roedel u8 start_bit; 1030883b0a91SJoerg Roedel u64 gpa; 1031883b0a91SJoerg Roedel 1032c62e2e94SBabu Moger if (!(vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_IOIO_PROT))) 1033883b0a91SJoerg Roedel return NESTED_EXIT_HOST; 1034883b0a91SJoerg Roedel 1035883b0a91SJoerg Roedel port = svm->vmcb->control.exit_info_1 >> 16; 1036883b0a91SJoerg Roedel size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >> 1037883b0a91SJoerg Roedel SVM_IOIO_SIZE_SHIFT; 1038e670bf68SPaolo Bonzini gpa = svm->nested.ctl.iopm_base_pa + (port / 8); 1039883b0a91SJoerg Roedel start_bit = port % 8; 1040883b0a91SJoerg Roedel iopm_len = (start_bit + size > 8) ? 2 : 1; 1041883b0a91SJoerg Roedel mask = (0xf >> (4 - size)) << start_bit; 1042883b0a91SJoerg Roedel val = 0; 1043883b0a91SJoerg Roedel 1044883b0a91SJoerg Roedel if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len)) 1045883b0a91SJoerg Roedel return NESTED_EXIT_DONE; 1046883b0a91SJoerg Roedel 1047883b0a91SJoerg Roedel return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; 1048883b0a91SJoerg Roedel } 1049883b0a91SJoerg Roedel 1050883b0a91SJoerg Roedel static int nested_svm_intercept(struct vcpu_svm *svm) 1051883b0a91SJoerg Roedel { 1052883b0a91SJoerg Roedel u32 exit_code = svm->vmcb->control.exit_code; 1053883b0a91SJoerg Roedel int vmexit = NESTED_EXIT_HOST; 1054883b0a91SJoerg Roedel 1055883b0a91SJoerg Roedel switch (exit_code) { 1056883b0a91SJoerg Roedel case SVM_EXIT_MSR: 1057883b0a91SJoerg Roedel vmexit = nested_svm_exit_handled_msr(svm); 1058883b0a91SJoerg Roedel break; 1059883b0a91SJoerg Roedel case SVM_EXIT_IOIO: 1060883b0a91SJoerg Roedel vmexit = nested_svm_intercept_ioio(svm); 1061883b0a91SJoerg Roedel break; 1062883b0a91SJoerg Roedel case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: { 106303bfeeb9SBabu Moger if (vmcb_is_intercept(&svm->nested.ctl, exit_code)) 1064883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 1065883b0a91SJoerg Roedel break; 1066883b0a91SJoerg Roedel } 1067883b0a91SJoerg Roedel case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: { 106830abaa88SBabu Moger if (vmcb_is_intercept(&svm->nested.ctl, exit_code)) 1069883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 1070883b0a91SJoerg Roedel break; 1071883b0a91SJoerg Roedel } 1072883b0a91SJoerg Roedel case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: { 10737c86663bSPaolo Bonzini /* 10747c86663bSPaolo Bonzini * Host-intercepted exceptions have been checked already in 10757c86663bSPaolo Bonzini * nested_svm_exit_special. There is nothing to do here, 10767c86663bSPaolo Bonzini * the vmexit is injected by svm_check_nested_events. 10777c86663bSPaolo Bonzini */ 1078883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 1079883b0a91SJoerg Roedel break; 1080883b0a91SJoerg Roedel } 1081883b0a91SJoerg Roedel case SVM_EXIT_ERR: { 1082883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 1083883b0a91SJoerg Roedel break; 1084883b0a91SJoerg Roedel } 1085883b0a91SJoerg Roedel default: { 1086c62e2e94SBabu Moger if (vmcb_is_intercept(&svm->nested.ctl, exit_code)) 1087883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE; 1088883b0a91SJoerg Roedel } 1089883b0a91SJoerg Roedel } 1090883b0a91SJoerg Roedel 1091883b0a91SJoerg Roedel return vmexit; 1092883b0a91SJoerg Roedel } 1093883b0a91SJoerg Roedel 1094883b0a91SJoerg Roedel int nested_svm_exit_handled(struct vcpu_svm *svm) 1095883b0a91SJoerg Roedel { 1096883b0a91SJoerg Roedel int vmexit; 1097883b0a91SJoerg Roedel 1098883b0a91SJoerg Roedel vmexit = nested_svm_intercept(svm); 1099883b0a91SJoerg Roedel 1100883b0a91SJoerg Roedel if (vmexit == NESTED_EXIT_DONE) 1101883b0a91SJoerg Roedel nested_svm_vmexit(svm); 1102883b0a91SJoerg Roedel 1103883b0a91SJoerg Roedel return vmexit; 1104883b0a91SJoerg Roedel } 1105883b0a91SJoerg Roedel 110663129754SPaolo Bonzini int nested_svm_check_permissions(struct kvm_vcpu *vcpu) 1107883b0a91SJoerg Roedel { 110863129754SPaolo Bonzini if (!(vcpu->arch.efer & EFER_SVME) || !is_paging(vcpu)) { 110963129754SPaolo Bonzini kvm_queue_exception(vcpu, UD_VECTOR); 1110883b0a91SJoerg Roedel return 1; 1111883b0a91SJoerg Roedel } 1112883b0a91SJoerg Roedel 111363129754SPaolo Bonzini if (to_svm(vcpu)->vmcb->save.cpl) { 111463129754SPaolo Bonzini kvm_inject_gp(vcpu, 0); 1115883b0a91SJoerg Roedel return 1; 1116883b0a91SJoerg Roedel } 1117883b0a91SJoerg Roedel 1118883b0a91SJoerg Roedel return 0; 1119883b0a91SJoerg Roedel } 1120883b0a91SJoerg Roedel 11217c86663bSPaolo Bonzini static bool nested_exit_on_exception(struct vcpu_svm *svm) 1122883b0a91SJoerg Roedel { 11237c86663bSPaolo Bonzini unsigned int nr = svm->vcpu.arch.exception.nr; 1124883b0a91SJoerg Roedel 11259780d51dSBabu Moger return (svm->nested.ctl.intercepts[INTERCEPT_EXCEPTION] & BIT(nr)); 11267c86663bSPaolo Bonzini } 1127883b0a91SJoerg Roedel 11287c86663bSPaolo Bonzini static void nested_svm_inject_exception_vmexit(struct vcpu_svm *svm) 11297c86663bSPaolo Bonzini { 11307c86663bSPaolo Bonzini unsigned int nr = svm->vcpu.arch.exception.nr; 1131883b0a91SJoerg Roedel 1132883b0a91SJoerg Roedel svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr; 1133883b0a91SJoerg Roedel svm->vmcb->control.exit_code_hi = 0; 11347c86663bSPaolo Bonzini 11357c86663bSPaolo Bonzini if (svm->vcpu.arch.exception.has_error_code) 11367c86663bSPaolo Bonzini svm->vmcb->control.exit_info_1 = svm->vcpu.arch.exception.error_code; 1137883b0a91SJoerg Roedel 1138883b0a91SJoerg Roedel /* 1139883b0a91SJoerg Roedel * EXITINFO2 is undefined for all exception intercepts other 1140883b0a91SJoerg Roedel * than #PF. 1141883b0a91SJoerg Roedel */ 11427c86663bSPaolo Bonzini if (nr == PF_VECTOR) { 1143883b0a91SJoerg Roedel if (svm->vcpu.arch.exception.nested_apf) 1144883b0a91SJoerg Roedel svm->vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token; 1145883b0a91SJoerg Roedel else if (svm->vcpu.arch.exception.has_payload) 1146883b0a91SJoerg Roedel svm->vmcb->control.exit_info_2 = svm->vcpu.arch.exception.payload; 1147883b0a91SJoerg Roedel else 1148883b0a91SJoerg Roedel svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2; 11497c86663bSPaolo Bonzini } else if (nr == DB_VECTOR) { 11507c86663bSPaolo Bonzini /* See inject_pending_event. */ 11517c86663bSPaolo Bonzini kvm_deliver_exception_payload(&svm->vcpu); 11527c86663bSPaolo Bonzini if (svm->vcpu.arch.dr7 & DR7_GD) { 11537c86663bSPaolo Bonzini svm->vcpu.arch.dr7 &= ~DR7_GD; 11547c86663bSPaolo Bonzini kvm_update_dr7(&svm->vcpu); 11557c86663bSPaolo Bonzini } 11567c86663bSPaolo Bonzini } else 11577c86663bSPaolo Bonzini WARN_ON(svm->vcpu.arch.exception.has_payload); 1158883b0a91SJoerg Roedel 11597c86663bSPaolo Bonzini nested_svm_vmexit(svm); 1160883b0a91SJoerg Roedel } 1161883b0a91SJoerg Roedel 11625b672408SPaolo Bonzini static inline bool nested_exit_on_init(struct vcpu_svm *svm) 11635b672408SPaolo Bonzini { 1164c62e2e94SBabu Moger return vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_INIT); 11655b672408SPaolo Bonzini } 11665b672408SPaolo Bonzini 116733b22172SPaolo Bonzini static int svm_check_nested_events(struct kvm_vcpu *vcpu) 1168883b0a91SJoerg Roedel { 1169883b0a91SJoerg Roedel struct vcpu_svm *svm = to_svm(vcpu); 1170883b0a91SJoerg Roedel bool block_nested_events = 1171bd279629SPaolo Bonzini kvm_event_needs_reinjection(vcpu) || svm->nested.nested_run_pending; 11725b672408SPaolo Bonzini struct kvm_lapic *apic = vcpu->arch.apic; 11735b672408SPaolo Bonzini 11745b672408SPaolo Bonzini if (lapic_in_kernel(vcpu) && 11755b672408SPaolo Bonzini test_bit(KVM_APIC_INIT, &apic->pending_events)) { 11765b672408SPaolo Bonzini if (block_nested_events) 11775b672408SPaolo Bonzini return -EBUSY; 11785b672408SPaolo Bonzini if (!nested_exit_on_init(svm)) 11795b672408SPaolo Bonzini return 0; 11803a87c7e0SSean Christopherson nested_svm_simple_vmexit(svm, SVM_EXIT_INIT); 11815b672408SPaolo Bonzini return 0; 11825b672408SPaolo Bonzini } 1183883b0a91SJoerg Roedel 11847c86663bSPaolo Bonzini if (vcpu->arch.exception.pending) { 11854020da3bSMaxim Levitsky /* 11864020da3bSMaxim Levitsky * Only a pending nested run can block a pending exception. 11874020da3bSMaxim Levitsky * Otherwise an injected NMI/interrupt should either be 11884020da3bSMaxim Levitsky * lost or delivered to the nested hypervisor in the EXITINTINFO 11894020da3bSMaxim Levitsky * vmcb field, while delivering the pending exception. 11904020da3bSMaxim Levitsky */ 11914020da3bSMaxim Levitsky if (svm->nested.nested_run_pending) 11927c86663bSPaolo Bonzini return -EBUSY; 11937c86663bSPaolo Bonzini if (!nested_exit_on_exception(svm)) 11947c86663bSPaolo Bonzini return 0; 11957c86663bSPaolo Bonzini nested_svm_inject_exception_vmexit(svm); 11967c86663bSPaolo Bonzini return 0; 11977c86663bSPaolo Bonzini } 11987c86663bSPaolo Bonzini 1199221e7610SPaolo Bonzini if (vcpu->arch.smi_pending && !svm_smi_blocked(vcpu)) { 120055714cddSPaolo Bonzini if (block_nested_events) 120155714cddSPaolo Bonzini return -EBUSY; 1202221e7610SPaolo Bonzini if (!nested_exit_on_smi(svm)) 1203221e7610SPaolo Bonzini return 0; 12043a87c7e0SSean Christopherson nested_svm_simple_vmexit(svm, SVM_EXIT_SMI); 120555714cddSPaolo Bonzini return 0; 120655714cddSPaolo Bonzini } 120755714cddSPaolo Bonzini 1208221e7610SPaolo Bonzini if (vcpu->arch.nmi_pending && !svm_nmi_blocked(vcpu)) { 12099c3d370aSCathy Avery if (block_nested_events) 12109c3d370aSCathy Avery return -EBUSY; 1211221e7610SPaolo Bonzini if (!nested_exit_on_nmi(svm)) 1212221e7610SPaolo Bonzini return 0; 12133a87c7e0SSean Christopherson nested_svm_simple_vmexit(svm, SVM_EXIT_NMI); 12149c3d370aSCathy Avery return 0; 12159c3d370aSCathy Avery } 12169c3d370aSCathy Avery 1217221e7610SPaolo Bonzini if (kvm_cpu_has_interrupt(vcpu) && !svm_interrupt_blocked(vcpu)) { 1218883b0a91SJoerg Roedel if (block_nested_events) 1219883b0a91SJoerg Roedel return -EBUSY; 1220221e7610SPaolo Bonzini if (!nested_exit_on_intr(svm)) 1221221e7610SPaolo Bonzini return 0; 12223a87c7e0SSean Christopherson trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip); 12233a87c7e0SSean Christopherson nested_svm_simple_vmexit(svm, SVM_EXIT_INTR); 1224883b0a91SJoerg Roedel return 0; 1225883b0a91SJoerg Roedel } 1226883b0a91SJoerg Roedel 1227883b0a91SJoerg Roedel return 0; 1228883b0a91SJoerg Roedel } 1229883b0a91SJoerg Roedel 1230883b0a91SJoerg Roedel int nested_svm_exit_special(struct vcpu_svm *svm) 1231883b0a91SJoerg Roedel { 1232883b0a91SJoerg Roedel u32 exit_code = svm->vmcb->control.exit_code; 1233883b0a91SJoerg Roedel 1234883b0a91SJoerg Roedel switch (exit_code) { 1235883b0a91SJoerg Roedel case SVM_EXIT_INTR: 1236883b0a91SJoerg Roedel case SVM_EXIT_NMI: 1237883b0a91SJoerg Roedel case SVM_EXIT_NPF: 1238883b0a91SJoerg Roedel return NESTED_EXIT_HOST; 12397c86663bSPaolo Bonzini case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: { 12407c86663bSPaolo Bonzini u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE); 12417c86663bSPaolo Bonzini 12424995a368SCathy Avery if (svm->vmcb01.ptr->control.intercepts[INTERCEPT_EXCEPTION] & 12439780d51dSBabu Moger excp_bits) 12447c86663bSPaolo Bonzini return NESTED_EXIT_HOST; 12457c86663bSPaolo Bonzini else if (exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR && 124668fd66f1SVitaly Kuznetsov svm->vcpu.arch.apf.host_apf_flags) 1247a3535be7SPaolo Bonzini /* Trap async PF even if not shadowing */ 1248883b0a91SJoerg Roedel return NESTED_EXIT_HOST; 1249883b0a91SJoerg Roedel break; 12507c86663bSPaolo Bonzini } 1251883b0a91SJoerg Roedel default: 1252883b0a91SJoerg Roedel break; 1253883b0a91SJoerg Roedel } 1254883b0a91SJoerg Roedel 1255883b0a91SJoerg Roedel return NESTED_EXIT_CONTINUE; 1256883b0a91SJoerg Roedel } 125733b22172SPaolo Bonzini 12585228eb96SMaxim Levitsky void nested_svm_update_tsc_ratio_msr(struct kvm_vcpu *vcpu) 12595228eb96SMaxim Levitsky { 12605228eb96SMaxim Levitsky struct vcpu_svm *svm = to_svm(vcpu); 12615228eb96SMaxim Levitsky 12625228eb96SMaxim Levitsky vcpu->arch.tsc_scaling_ratio = 12635228eb96SMaxim Levitsky kvm_calc_nested_tsc_multiplier(vcpu->arch.l1_tsc_scaling_ratio, 12645228eb96SMaxim Levitsky svm->tsc_ratio_msr); 12655228eb96SMaxim Levitsky svm_write_tsc_multiplier(vcpu, vcpu->arch.tsc_scaling_ratio); 12665228eb96SMaxim Levitsky } 12675228eb96SMaxim Levitsky 1268cc440cdaSPaolo Bonzini static int svm_get_nested_state(struct kvm_vcpu *vcpu, 1269cc440cdaSPaolo Bonzini struct kvm_nested_state __user *user_kvm_nested_state, 1270cc440cdaSPaolo Bonzini u32 user_data_size) 1271cc440cdaSPaolo Bonzini { 1272cc440cdaSPaolo Bonzini struct vcpu_svm *svm; 1273cc440cdaSPaolo Bonzini struct kvm_nested_state kvm_state = { 1274cc440cdaSPaolo Bonzini .flags = 0, 1275cc440cdaSPaolo Bonzini .format = KVM_STATE_NESTED_FORMAT_SVM, 1276cc440cdaSPaolo Bonzini .size = sizeof(kvm_state), 1277cc440cdaSPaolo Bonzini }; 1278cc440cdaSPaolo Bonzini struct vmcb __user *user_vmcb = (struct vmcb __user *) 1279cc440cdaSPaolo Bonzini &user_kvm_nested_state->data.svm[0]; 1280cc440cdaSPaolo Bonzini 1281cc440cdaSPaolo Bonzini if (!vcpu) 1282cc440cdaSPaolo Bonzini return kvm_state.size + KVM_STATE_NESTED_SVM_VMCB_SIZE; 1283cc440cdaSPaolo Bonzini 1284cc440cdaSPaolo Bonzini svm = to_svm(vcpu); 1285cc440cdaSPaolo Bonzini 1286cc440cdaSPaolo Bonzini if (user_data_size < kvm_state.size) 1287cc440cdaSPaolo Bonzini goto out; 1288cc440cdaSPaolo Bonzini 1289cc440cdaSPaolo Bonzini /* First fill in the header and copy it out. */ 1290cc440cdaSPaolo Bonzini if (is_guest_mode(vcpu)) { 12910dd16b5bSMaxim Levitsky kvm_state.hdr.svm.vmcb_pa = svm->nested.vmcb12_gpa; 1292cc440cdaSPaolo Bonzini kvm_state.size += KVM_STATE_NESTED_SVM_VMCB_SIZE; 1293cc440cdaSPaolo Bonzini kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE; 1294cc440cdaSPaolo Bonzini 1295cc440cdaSPaolo Bonzini if (svm->nested.nested_run_pending) 1296cc440cdaSPaolo Bonzini kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING; 1297cc440cdaSPaolo Bonzini } 1298cc440cdaSPaolo Bonzini 1299cc440cdaSPaolo Bonzini if (gif_set(svm)) 1300cc440cdaSPaolo Bonzini kvm_state.flags |= KVM_STATE_NESTED_GIF_SET; 1301cc440cdaSPaolo Bonzini 1302cc440cdaSPaolo Bonzini if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state))) 1303cc440cdaSPaolo Bonzini return -EFAULT; 1304cc440cdaSPaolo Bonzini 1305cc440cdaSPaolo Bonzini if (!is_guest_mode(vcpu)) 1306cc440cdaSPaolo Bonzini goto out; 1307cc440cdaSPaolo Bonzini 1308cc440cdaSPaolo Bonzini /* 1309cc440cdaSPaolo Bonzini * Copy over the full size of the VMCB rather than just the size 1310cc440cdaSPaolo Bonzini * of the structs. 1311cc440cdaSPaolo Bonzini */ 1312cc440cdaSPaolo Bonzini if (clear_user(user_vmcb, KVM_STATE_NESTED_SVM_VMCB_SIZE)) 1313cc440cdaSPaolo Bonzini return -EFAULT; 1314cc440cdaSPaolo Bonzini if (copy_to_user(&user_vmcb->control, &svm->nested.ctl, 1315cc440cdaSPaolo Bonzini sizeof(user_vmcb->control))) 1316cc440cdaSPaolo Bonzini return -EFAULT; 13174995a368SCathy Avery if (copy_to_user(&user_vmcb->save, &svm->vmcb01.ptr->save, 1318cc440cdaSPaolo Bonzini sizeof(user_vmcb->save))) 1319cc440cdaSPaolo Bonzini return -EFAULT; 1320cc440cdaSPaolo Bonzini out: 1321cc440cdaSPaolo Bonzini return kvm_state.size; 1322cc440cdaSPaolo Bonzini } 1323cc440cdaSPaolo Bonzini 1324cc440cdaSPaolo Bonzini static int svm_set_nested_state(struct kvm_vcpu *vcpu, 1325cc440cdaSPaolo Bonzini struct kvm_nested_state __user *user_kvm_nested_state, 1326cc440cdaSPaolo Bonzini struct kvm_nested_state *kvm_state) 1327cc440cdaSPaolo Bonzini { 1328cc440cdaSPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu); 1329cc440cdaSPaolo Bonzini struct vmcb __user *user_vmcb = (struct vmcb __user *) 1330cc440cdaSPaolo Bonzini &user_kvm_nested_state->data.svm[0]; 13316ccbd29aSJoerg Roedel struct vmcb_control_area *ctl; 13326ccbd29aSJoerg Roedel struct vmcb_save_area *save; 1333dbc4739bSSean Christopherson unsigned long cr0; 13346ccbd29aSJoerg Roedel int ret; 1335cc440cdaSPaolo Bonzini 13366ccbd29aSJoerg Roedel BUILD_BUG_ON(sizeof(struct vmcb_control_area) + sizeof(struct vmcb_save_area) > 13376ccbd29aSJoerg Roedel KVM_STATE_NESTED_SVM_VMCB_SIZE); 13386ccbd29aSJoerg Roedel 1339cc440cdaSPaolo Bonzini if (kvm_state->format != KVM_STATE_NESTED_FORMAT_SVM) 1340cc440cdaSPaolo Bonzini return -EINVAL; 1341cc440cdaSPaolo Bonzini 1342cc440cdaSPaolo Bonzini if (kvm_state->flags & ~(KVM_STATE_NESTED_GUEST_MODE | 1343cc440cdaSPaolo Bonzini KVM_STATE_NESTED_RUN_PENDING | 1344cc440cdaSPaolo Bonzini KVM_STATE_NESTED_GIF_SET)) 1345cc440cdaSPaolo Bonzini return -EINVAL; 1346cc440cdaSPaolo Bonzini 1347cc440cdaSPaolo Bonzini /* 1348cc440cdaSPaolo Bonzini * If in guest mode, vcpu->arch.efer actually refers to the L2 guest's 1349cc440cdaSPaolo Bonzini * EFER.SVME, but EFER.SVME still has to be 1 for VMRUN to succeed. 1350cc440cdaSPaolo Bonzini */ 1351cc440cdaSPaolo Bonzini if (!(vcpu->arch.efer & EFER_SVME)) { 1352cc440cdaSPaolo Bonzini /* GIF=1 and no guest mode are required if SVME=0. */ 1353cc440cdaSPaolo Bonzini if (kvm_state->flags != KVM_STATE_NESTED_GIF_SET) 1354cc440cdaSPaolo Bonzini return -EINVAL; 1355cc440cdaSPaolo Bonzini } 1356cc440cdaSPaolo Bonzini 1357cc440cdaSPaolo Bonzini /* SMM temporarily disables SVM, so we cannot be in guest mode. */ 1358cc440cdaSPaolo Bonzini if (is_smm(vcpu) && (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 1359cc440cdaSPaolo Bonzini return -EINVAL; 1360cc440cdaSPaolo Bonzini 1361cc440cdaSPaolo Bonzini if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) { 1362cc440cdaSPaolo Bonzini svm_leave_nested(svm); 1363d5cd6f34SVitaly Kuznetsov svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET)); 1364d5cd6f34SVitaly Kuznetsov return 0; 1365cc440cdaSPaolo Bonzini } 1366cc440cdaSPaolo Bonzini 1367cc440cdaSPaolo Bonzini if (!page_address_valid(vcpu, kvm_state->hdr.svm.vmcb_pa)) 1368cc440cdaSPaolo Bonzini return -EINVAL; 1369cc440cdaSPaolo Bonzini if (kvm_state->size < sizeof(*kvm_state) + KVM_STATE_NESTED_SVM_VMCB_SIZE) 1370cc440cdaSPaolo Bonzini return -EINVAL; 1371cc440cdaSPaolo Bonzini 13726ccbd29aSJoerg Roedel ret = -ENOMEM; 1373eba04b20SSean Christopherson ctl = kzalloc(sizeof(*ctl), GFP_KERNEL_ACCOUNT); 1374eba04b20SSean Christopherson save = kzalloc(sizeof(*save), GFP_KERNEL_ACCOUNT); 13756ccbd29aSJoerg Roedel if (!ctl || !save) 13766ccbd29aSJoerg Roedel goto out_free; 13776ccbd29aSJoerg Roedel 13786ccbd29aSJoerg Roedel ret = -EFAULT; 13796ccbd29aSJoerg Roedel if (copy_from_user(ctl, &user_vmcb->control, sizeof(*ctl))) 13806ccbd29aSJoerg Roedel goto out_free; 13816ccbd29aSJoerg Roedel if (copy_from_user(save, &user_vmcb->save, sizeof(*save))) 13826ccbd29aSJoerg Roedel goto out_free; 13836ccbd29aSJoerg Roedel 13846ccbd29aSJoerg Roedel ret = -EINVAL; 1385ee695f22SKrish Sadhukhan if (!nested_vmcb_check_controls(vcpu, ctl)) 13866ccbd29aSJoerg Roedel goto out_free; 1387cc440cdaSPaolo Bonzini 1388cc440cdaSPaolo Bonzini /* 1389cc440cdaSPaolo Bonzini * Processor state contains L2 state. Check that it is 1390cb9b6a1bSPaolo Bonzini * valid for guest mode (see nested_vmcb_check_save). 1391cc440cdaSPaolo Bonzini */ 1392cc440cdaSPaolo Bonzini cr0 = kvm_read_cr0(vcpu); 1393cc440cdaSPaolo Bonzini if (((cr0 & X86_CR0_CD) == 0) && (cr0 & X86_CR0_NW)) 13946ccbd29aSJoerg Roedel goto out_free; 1395cc440cdaSPaolo Bonzini 1396cc440cdaSPaolo Bonzini /* 1397cc440cdaSPaolo Bonzini * Validate host state saved from before VMRUN (see 1398cc440cdaSPaolo Bonzini * nested_svm_check_permissions). 1399cc440cdaSPaolo Bonzini */ 14006906e06dSKrish Sadhukhan if (!(save->cr0 & X86_CR0_PG) || 14016906e06dSKrish Sadhukhan !(save->cr0 & X86_CR0_PE) || 14026906e06dSKrish Sadhukhan (save->rflags & X86_EFLAGS_VM) || 140363129754SPaolo Bonzini !nested_vmcb_valid_sregs(vcpu, save)) 14046ccbd29aSJoerg Roedel goto out_free; 1405cc440cdaSPaolo Bonzini 1406cc440cdaSPaolo Bonzini /* 1407b222b0b8SMaxim Levitsky * While the nested guest CR3 is already checked and set by 1408b222b0b8SMaxim Levitsky * KVM_SET_SREGS, it was set when nested state was yet loaded, 1409b222b0b8SMaxim Levitsky * thus MMU might not be initialized correctly. 1410b222b0b8SMaxim Levitsky * Set it again to fix this. 1411b222b0b8SMaxim Levitsky */ 1412b222b0b8SMaxim Levitsky 1413b222b0b8SMaxim Levitsky ret = nested_svm_load_cr3(&svm->vcpu, vcpu->arch.cr3, 1414b222b0b8SMaxim Levitsky nested_npt_enabled(svm), false); 1415b222b0b8SMaxim Levitsky if (WARN_ON_ONCE(ret)) 1416b222b0b8SMaxim Levitsky goto out_free; 1417b222b0b8SMaxim Levitsky 1418b222b0b8SMaxim Levitsky 1419b222b0b8SMaxim Levitsky /* 14204995a368SCathy Avery * All checks done, we can enter guest mode. Userspace provides 14214995a368SCathy Avery * vmcb12.control, which will be combined with L1 and stored into 14224995a368SCathy Avery * vmcb02, and the L1 save state which we store in vmcb01. 14234995a368SCathy Avery * L2 registers if needed are moved from the current VMCB to VMCB02. 1424cc440cdaSPaolo Bonzini */ 142581f76adaSMaxim Levitsky 14269d290e16SMaxim Levitsky if (is_guest_mode(vcpu)) 14279d290e16SMaxim Levitsky svm_leave_nested(svm); 14289d290e16SMaxim Levitsky else 14299d290e16SMaxim Levitsky svm->nested.vmcb02.ptr->save = svm->vmcb01.ptr->save; 14309d290e16SMaxim Levitsky 1431063ab16cSMaxim Levitsky svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET)); 1432063ab16cSMaxim Levitsky 143381f76adaSMaxim Levitsky svm->nested.nested_run_pending = 143481f76adaSMaxim Levitsky !!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING); 143581f76adaSMaxim Levitsky 14360dd16b5bSMaxim Levitsky svm->nested.vmcb12_gpa = kvm_state->hdr.svm.vmcb_pa; 1437c08f390aSPaolo Bonzini 14382bb16beaSVitaly Kuznetsov svm_copy_vmrun_state(&svm->vmcb01.ptr->save, save); 14399e8f0fbfSPaolo Bonzini nested_load_control_from_vmcb12(svm, ctl); 14404995a368SCathy Avery 14414995a368SCathy Avery svm_switch_vmcb(svm, &svm->nested.vmcb02); 14429e8f0fbfSPaolo Bonzini nested_vmcb02_prepare_control(svm); 1443a7d5c7ceSPaolo Bonzini kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 14446ccbd29aSJoerg Roedel ret = 0; 14456ccbd29aSJoerg Roedel out_free: 14466ccbd29aSJoerg Roedel kfree(save); 14476ccbd29aSJoerg Roedel kfree(ctl); 14486ccbd29aSJoerg Roedel 14496ccbd29aSJoerg Roedel return ret; 1450cc440cdaSPaolo Bonzini } 1451cc440cdaSPaolo Bonzini 1452232f75d3SMaxim Levitsky static bool svm_get_nested_state_pages(struct kvm_vcpu *vcpu) 1453232f75d3SMaxim Levitsky { 1454232f75d3SMaxim Levitsky struct vcpu_svm *svm = to_svm(vcpu); 1455232f75d3SMaxim Levitsky 1456232f75d3SMaxim Levitsky if (WARN_ON(!is_guest_mode(vcpu))) 1457232f75d3SMaxim Levitsky return true; 1458232f75d3SMaxim Levitsky 1459158a48ecSMaxim Levitsky if (!vcpu->arch.pdptrs_from_userspace && 1460158a48ecSMaxim Levitsky !nested_npt_enabled(svm) && is_pae_paging(vcpu)) 1461b222b0b8SMaxim Levitsky /* 1462b222b0b8SMaxim Levitsky * Reload the guest's PDPTRs since after a migration 1463b222b0b8SMaxim Levitsky * the guest CR3 might be restored prior to setting the nested 1464b222b0b8SMaxim Levitsky * state which can lead to a load of wrong PDPTRs. 1465b222b0b8SMaxim Levitsky */ 1466b222b0b8SMaxim Levitsky if (CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3))) 1467232f75d3SMaxim Levitsky return false; 1468232f75d3SMaxim Levitsky 1469232f75d3SMaxim Levitsky if (!nested_svm_vmrun_msrpm(svm)) { 1470232f75d3SMaxim Levitsky vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1471232f75d3SMaxim Levitsky vcpu->run->internal.suberror = 1472232f75d3SMaxim Levitsky KVM_INTERNAL_ERROR_EMULATION; 1473232f75d3SMaxim Levitsky vcpu->run->internal.ndata = 0; 1474232f75d3SMaxim Levitsky return false; 1475232f75d3SMaxim Levitsky } 1476232f75d3SMaxim Levitsky 1477232f75d3SMaxim Levitsky return true; 1478232f75d3SMaxim Levitsky } 1479232f75d3SMaxim Levitsky 148033b22172SPaolo Bonzini struct kvm_x86_nested_ops svm_nested_ops = { 148133b22172SPaolo Bonzini .check_events = svm_check_nested_events, 1482cb6a32c2SSean Christopherson .triple_fault = nested_svm_triple_fault, 1483a7d5c7ceSPaolo Bonzini .get_nested_state_pages = svm_get_nested_state_pages, 1484cc440cdaSPaolo Bonzini .get_state = svm_get_nested_state, 1485cc440cdaSPaolo Bonzini .set_state = svm_set_nested_state, 148633b22172SPaolo Bonzini }; 1487