1883b0a91SJoerg Roedel // SPDX-License-Identifier: GPL-2.0-only
2883b0a91SJoerg Roedel /*
3883b0a91SJoerg Roedel * Kernel-based Virtual Machine driver for Linux
4883b0a91SJoerg Roedel *
5883b0a91SJoerg Roedel * AMD SVM support
6883b0a91SJoerg Roedel *
7883b0a91SJoerg Roedel * Copyright (C) 2006 Qumranet, Inc.
8883b0a91SJoerg Roedel * Copyright 2010 Red Hat, Inc. and/or its affiliates.
9883b0a91SJoerg Roedel *
10883b0a91SJoerg Roedel * Authors:
11883b0a91SJoerg Roedel * Yaniv Kamay <yaniv@qumranet.com>
12883b0a91SJoerg Roedel * Avi Kivity <avi@qumranet.com>
13883b0a91SJoerg Roedel */
14883b0a91SJoerg Roedel
158d20bd63SSean Christopherson #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
16883b0a91SJoerg Roedel
17883b0a91SJoerg Roedel #include <linux/kvm_types.h>
18883b0a91SJoerg Roedel #include <linux/kvm_host.h>
19883b0a91SJoerg Roedel #include <linux/kernel.h>
20883b0a91SJoerg Roedel
21883b0a91SJoerg Roedel #include <asm/msr-index.h>
225679b803SPaolo Bonzini #include <asm/debugreg.h>
23883b0a91SJoerg Roedel
24883b0a91SJoerg Roedel #include "kvm_emulate.h"
25883b0a91SJoerg Roedel #include "trace.h"
26883b0a91SJoerg Roedel #include "mmu.h"
27883b0a91SJoerg Roedel #include "x86.h"
28b0b42197SPaolo Bonzini #include "smm.h"
29cc440cdaSPaolo Bonzini #include "cpuid.h"
305b672408SPaolo Bonzini #include "lapic.h"
31883b0a91SJoerg Roedel #include "svm.h"
3266c03a92SVitaly Kuznetsov #include "hyperv.h"
33883b0a91SJoerg Roedel
3411f0cbf0SSean Christopherson #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK
3511f0cbf0SSean Christopherson
nested_svm_inject_npf_exit(struct kvm_vcpu * vcpu,struct x86_exception * fault)36883b0a91SJoerg Roedel static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
37883b0a91SJoerg Roedel struct x86_exception *fault)
38883b0a91SJoerg Roedel {
39883b0a91SJoerg Roedel struct vcpu_svm *svm = to_svm(vcpu);
40db663af4SMaxim Levitsky struct vmcb *vmcb = svm->vmcb;
41883b0a91SJoerg Roedel
42db663af4SMaxim Levitsky if (vmcb->control.exit_code != SVM_EXIT_NPF) {
43883b0a91SJoerg Roedel /*
44883b0a91SJoerg Roedel * TODO: track the cause of the nested page fault, and
45883b0a91SJoerg Roedel * correctly fill in the high bits of exit_info_1.
46883b0a91SJoerg Roedel */
47db663af4SMaxim Levitsky vmcb->control.exit_code = SVM_EXIT_NPF;
48db663af4SMaxim Levitsky vmcb->control.exit_code_hi = 0;
49db663af4SMaxim Levitsky vmcb->control.exit_info_1 = (1ULL << 32);
50db663af4SMaxim Levitsky vmcb->control.exit_info_2 = fault->address;
51883b0a91SJoerg Roedel }
52883b0a91SJoerg Roedel
53db663af4SMaxim Levitsky vmcb->control.exit_info_1 &= ~0xffffffffULL;
54db663af4SMaxim Levitsky vmcb->control.exit_info_1 |= fault->error_code;
55883b0a91SJoerg Roedel
56883b0a91SJoerg Roedel nested_svm_vmexit(svm);
57883b0a91SJoerg Roedel }
58883b0a91SJoerg Roedel
nested_svm_get_tdp_pdptr(struct kvm_vcpu * vcpu,int index)59883b0a91SJoerg Roedel static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index)
60883b0a91SJoerg Roedel {
61883b0a91SJoerg Roedel struct vcpu_svm *svm = to_svm(vcpu);
62e670bf68SPaolo Bonzini u64 cr3 = svm->nested.ctl.nested_cr3;
63883b0a91SJoerg Roedel u64 pdpte;
64883b0a91SJoerg Roedel int ret;
65883b0a91SJoerg Roedel
662732be90SSean Christopherson ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(cr3), &pdpte,
67883b0a91SJoerg Roedel offset_in_page(cr3) + index * 8, 8);
68883b0a91SJoerg Roedel if (ret)
69883b0a91SJoerg Roedel return 0;
70883b0a91SJoerg Roedel return pdpte;
71883b0a91SJoerg Roedel }
72883b0a91SJoerg Roedel
nested_svm_get_tdp_cr3(struct kvm_vcpu * vcpu)73883b0a91SJoerg Roedel static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu)
74883b0a91SJoerg Roedel {
75883b0a91SJoerg Roedel struct vcpu_svm *svm = to_svm(vcpu);
76883b0a91SJoerg Roedel
77e670bf68SPaolo Bonzini return svm->nested.ctl.nested_cr3;
78883b0a91SJoerg Roedel }
79883b0a91SJoerg Roedel
nested_svm_init_mmu_context(struct kvm_vcpu * vcpu)80883b0a91SJoerg Roedel static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
81883b0a91SJoerg Roedel {
82929d1cfaSPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu);
83929d1cfaSPaolo Bonzini
84883b0a91SJoerg Roedel WARN_ON(mmu_is_nested(vcpu));
85883b0a91SJoerg Roedel
86883b0a91SJoerg Roedel vcpu->arch.mmu = &vcpu->arch.guest_mmu;
8731e96bc6SSean Christopherson
8831e96bc6SSean Christopherson /*
8931e96bc6SSean Christopherson * The NPT format depends on L1's CR4 and EFER, which is in vmcb01. Note,
9031e96bc6SSean Christopherson * when called via KVM_SET_NESTED_STATE, that state may _not_ match current
9131e96bc6SSean Christopherson * vCPU state. CR0.WP is explicitly ignored, while CR0.PG is required.
9231e96bc6SSean Christopherson */
934995a368SCathy Avery kvm_init_shadow_npt_mmu(vcpu, X86_CR0_PG, svm->vmcb01.ptr->save.cr4,
944995a368SCathy Avery svm->vmcb01.ptr->save.efer,
950f04a2acSVitaly Kuznetsov svm->nested.ctl.nested_cr3);
96883b0a91SJoerg Roedel vcpu->arch.mmu->get_guest_pgd = nested_svm_get_tdp_cr3;
97883b0a91SJoerg Roedel vcpu->arch.mmu->get_pdptr = nested_svm_get_tdp_pdptr;
98883b0a91SJoerg Roedel vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit;
99883b0a91SJoerg Roedel vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu;
100883b0a91SJoerg Roedel }
101883b0a91SJoerg Roedel
nested_svm_uninit_mmu_context(struct kvm_vcpu * vcpu)102883b0a91SJoerg Roedel static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
103883b0a91SJoerg Roedel {
104883b0a91SJoerg Roedel vcpu->arch.mmu = &vcpu->arch.root_mmu;
105883b0a91SJoerg Roedel vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
106883b0a91SJoerg Roedel }
107883b0a91SJoerg Roedel
nested_vmcb_needs_vls_intercept(struct vcpu_svm * svm)108b9f3973aSMaxim Levitsky static bool nested_vmcb_needs_vls_intercept(struct vcpu_svm *svm)
109b9f3973aSMaxim Levitsky {
1104d2a1560SSean Christopherson if (!guest_can_use(&svm->vcpu, X86_FEATURE_V_VMSAVE_VMLOAD))
111b9f3973aSMaxim Levitsky return true;
112b9f3973aSMaxim Levitsky
113b9f3973aSMaxim Levitsky if (!nested_npt_enabled(svm))
114b9f3973aSMaxim Levitsky return true;
115b9f3973aSMaxim Levitsky
116b9f3973aSMaxim Levitsky if (!(svm->nested.ctl.virt_ext & VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK))
117b9f3973aSMaxim Levitsky return true;
118b9f3973aSMaxim Levitsky
119b9f3973aSMaxim Levitsky return false;
120b9f3973aSMaxim Levitsky }
121b9f3973aSMaxim Levitsky
recalc_intercepts(struct vcpu_svm * svm)122883b0a91SJoerg Roedel void recalc_intercepts(struct vcpu_svm *svm)
123883b0a91SJoerg Roedel {
1248fc78909SEmanuele Giuseppe Esposito struct vmcb_control_area *c, *h;
1258fc78909SEmanuele Giuseppe Esposito struct vmcb_ctrl_area_cached *g;
126c45ad722SBabu Moger unsigned int i;
127883b0a91SJoerg Roedel
12806e7852cSJoerg Roedel vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
129883b0a91SJoerg Roedel
130883b0a91SJoerg Roedel if (!is_guest_mode(&svm->vcpu))
131883b0a91SJoerg Roedel return;
132883b0a91SJoerg Roedel
133883b0a91SJoerg Roedel c = &svm->vmcb->control;
1344995a368SCathy Avery h = &svm->vmcb01.ptr->control;
135e670bf68SPaolo Bonzini g = &svm->nested.ctl;
136883b0a91SJoerg Roedel
137c45ad722SBabu Moger for (i = 0; i < MAX_INTERCEPT; i++)
138c45ad722SBabu Moger c->intercepts[i] = h->intercepts[i];
139c45ad722SBabu Moger
140e9fd761aSPaolo Bonzini if (g->int_ctl & V_INTR_MASKING_MASK) {
141883b0a91SJoerg Roedel /*
1427334ede4SSantosh Shukla * If L2 is active and V_INTR_MASKING is enabled in vmcb12,
1437334ede4SSantosh Shukla * disable intercept of CR8 writes as L2's CR8 does not affect
1447334ede4SSantosh Shukla * any interrupt KVM may want to inject.
1457334ede4SSantosh Shukla *
1467334ede4SSantosh Shukla * Similarly, disable intercept of virtual interrupts (used to
1477334ede4SSantosh Shukla * detect interrupt windows) if the saved RFLAGS.IF is '0', as
1487334ede4SSantosh Shukla * the effective RFLAGS.IF for L1 interrupts will never be set
1497334ede4SSantosh Shukla * while L2 is running (L2's RFLAGS.IF doesn't affect L1 IRQs).
150883b0a91SJoerg Roedel */
15174905e3dSPaolo Bonzini vmcb_clr_intercept(c, INTERCEPT_CR8_WRITE);
1527334ede4SSantosh Shukla if (!(svm->vmcb01.ptr->save.rflags & X86_EFLAGS_IF))
153c62e2e94SBabu Moger vmcb_clr_intercept(c, INTERCEPT_VINTR);
154883b0a91SJoerg Roedel }
155883b0a91SJoerg Roedel
1563f4a812eSVitaly Kuznetsov /*
1573f4a812eSVitaly Kuznetsov * We want to see VMMCALLs from a nested guest only when Hyper-V L2 TLB
1583f4a812eSVitaly Kuznetsov * flush feature is enabled.
1593f4a812eSVitaly Kuznetsov */
1603f4a812eSVitaly Kuznetsov if (!nested_svm_l2_tlb_flush_enabled(&svm->vcpu))
161c62e2e94SBabu Moger vmcb_clr_intercept(c, INTERCEPT_VMMCALL);
162883b0a91SJoerg Roedel
163c45ad722SBabu Moger for (i = 0; i < MAX_INTERCEPT; i++)
164c45ad722SBabu Moger c->intercepts[i] |= g->intercepts[i];
1654b639a9fSMaxim Levitsky
1664b639a9fSMaxim Levitsky /* If SMI is not intercepted, ignore guest SMI intercept as well */
1674b639a9fSMaxim Levitsky if (!intercept_smi)
1684b639a9fSMaxim Levitsky vmcb_clr_intercept(c, INTERCEPT_SMI);
169c7dfa400SMaxim Levitsky
170b9f3973aSMaxim Levitsky if (nested_vmcb_needs_vls_intercept(svm)) {
171b9f3973aSMaxim Levitsky /*
172b9f3973aSMaxim Levitsky * If the virtual VMLOAD/VMSAVE is not enabled for the L2,
173b9f3973aSMaxim Levitsky * we must intercept these instructions to correctly
174b9f3973aSMaxim Levitsky * emulate them in case L1 doesn't intercept them.
175b9f3973aSMaxim Levitsky */
176c7dfa400SMaxim Levitsky vmcb_set_intercept(c, INTERCEPT_VMLOAD);
177c7dfa400SMaxim Levitsky vmcb_set_intercept(c, INTERCEPT_VMSAVE);
178b9f3973aSMaxim Levitsky } else {
179b9f3973aSMaxim Levitsky WARN_ON(!(c->virt_ext & VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK));
180b9f3973aSMaxim Levitsky }
181883b0a91SJoerg Roedel }
182883b0a91SJoerg Roedel
18366c03a92SVitaly Kuznetsov /*
18466c03a92SVitaly Kuznetsov * Merge L0's (KVM) and L1's (Nested VMCB) MSR permission bitmaps. The function
18566c03a92SVitaly Kuznetsov * is optimized in that it only merges the parts where KVM MSR permission bitmap
18666c03a92SVitaly Kuznetsov * may contain zero bits.
18766c03a92SVitaly Kuznetsov */
nested_svm_vmrun_msrpm(struct vcpu_svm * svm)188883b0a91SJoerg Roedel static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
189883b0a91SJoerg Roedel {
190883b0a91SJoerg Roedel int i;
191883b0a91SJoerg Roedel
19266c03a92SVitaly Kuznetsov /*
19366c03a92SVitaly Kuznetsov * MSR bitmap update can be skipped when:
19466c03a92SVitaly Kuznetsov * - MSR bitmap for L1 hasn't changed.
19566c03a92SVitaly Kuznetsov * - Nested hypervisor (L1) is attempting to launch the same L2 as
19666c03a92SVitaly Kuznetsov * before.
19766c03a92SVitaly Kuznetsov * - Nested hypervisor (L1) is using Hyper-V emulation interface and
19866c03a92SVitaly Kuznetsov * tells KVM (L0) there were no changes in MSR bitmap for L2.
19966c03a92SVitaly Kuznetsov */
200017a99a9SVitaly Kuznetsov #ifdef CONFIG_KVM_HYPERV
201017a99a9SVitaly Kuznetsov if (!svm->nested.force_msr_bitmap_recalc) {
202017a99a9SVitaly Kuznetsov struct hv_vmcb_enlightenments *hve = &svm->nested.ctl.hv_enlightenments;
203017a99a9SVitaly Kuznetsov
204017a99a9SVitaly Kuznetsov if (kvm_hv_hypercall_enabled(&svm->vcpu) &&
20566c03a92SVitaly Kuznetsov hve->hv_enlightenments_control.msr_bitmap &&
206089fe572SSean Christopherson (svm->nested.ctl.clean & BIT(HV_VMCB_NESTED_ENLIGHTENMENTS)))
20766c03a92SVitaly Kuznetsov goto set_msrpm_base_pa;
208017a99a9SVitaly Kuznetsov }
209017a99a9SVitaly Kuznetsov #endif
21066c03a92SVitaly Kuznetsov
2118fc78909SEmanuele Giuseppe Esposito if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT)))
212883b0a91SJoerg Roedel return true;
213883b0a91SJoerg Roedel
214883b0a91SJoerg Roedel for (i = 0; i < MSRPM_OFFSETS; i++) {
215883b0a91SJoerg Roedel u32 value, p;
216883b0a91SJoerg Roedel u64 offset;
217883b0a91SJoerg Roedel
218883b0a91SJoerg Roedel if (msrpm_offsets[i] == 0xffffffff)
219883b0a91SJoerg Roedel break;
220883b0a91SJoerg Roedel
221883b0a91SJoerg Roedel p = msrpm_offsets[i];
2227a8f7c1fSMaxim Levitsky
2237a8f7c1fSMaxim Levitsky /* x2apic msrs are intercepted always for the nested guest */
2247a8f7c1fSMaxim Levitsky if (is_x2apic_msrpm_offset(p))
2257a8f7c1fSMaxim Levitsky continue;
2267a8f7c1fSMaxim Levitsky
227e670bf68SPaolo Bonzini offset = svm->nested.ctl.msrpm_base_pa + (p * 4);
228883b0a91SJoerg Roedel
229883b0a91SJoerg Roedel if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4))
230883b0a91SJoerg Roedel return false;
231883b0a91SJoerg Roedel
232883b0a91SJoerg Roedel svm->nested.msrpm[p] = svm->msrpm[p] | value;
233883b0a91SJoerg Roedel }
234883b0a91SJoerg Roedel
23573c25546SVitaly Kuznetsov svm->nested.force_msr_bitmap_recalc = false;
23673c25546SVitaly Kuznetsov
237017a99a9SVitaly Kuznetsov #ifdef CONFIG_KVM_HYPERV
23866c03a92SVitaly Kuznetsov set_msrpm_base_pa:
239017a99a9SVitaly Kuznetsov #endif
240883b0a91SJoerg Roedel svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm));
241883b0a91SJoerg Roedel
242883b0a91SJoerg Roedel return true;
243883b0a91SJoerg Roedel }
244883b0a91SJoerg Roedel
245ee695f22SKrish Sadhukhan /*
246ee695f22SKrish Sadhukhan * Bits 11:0 of bitmap address are ignored by hardware
247ee695f22SKrish Sadhukhan */
nested_svm_check_bitmap_pa(struct kvm_vcpu * vcpu,u64 pa,u32 size)248ee695f22SKrish Sadhukhan static bool nested_svm_check_bitmap_pa(struct kvm_vcpu *vcpu, u64 pa, u32 size)
249ee695f22SKrish Sadhukhan {
250ee695f22SKrish Sadhukhan u64 addr = PAGE_ALIGN(pa);
251ee695f22SKrish Sadhukhan
252ee695f22SKrish Sadhukhan return kvm_vcpu_is_legal_gpa(vcpu, addr) &&
253ee695f22SKrish Sadhukhan kvm_vcpu_is_legal_gpa(vcpu, addr + size - 1);
254ee695f22SKrish Sadhukhan }
255ee695f22SKrish Sadhukhan
__nested_vmcb_check_controls(struct kvm_vcpu * vcpu,struct vmcb_ctrl_area_cached * control)256bd95926cSPaolo Bonzini static bool __nested_vmcb_check_controls(struct kvm_vcpu *vcpu,
2578fc78909SEmanuele Giuseppe Esposito struct vmcb_ctrl_area_cached *control)
258ca46d739SPaolo Bonzini {
2598fc78909SEmanuele Giuseppe Esposito if (CC(!vmcb12_is_intercept(control, INTERCEPT_VMRUN)))
260ca46d739SPaolo Bonzini return false;
261ca46d739SPaolo Bonzini
26211f0cbf0SSean Christopherson if (CC(control->asid == 0))
263ca46d739SPaolo Bonzini return false;
264ca46d739SPaolo Bonzini
26511f0cbf0SSean Christopherson if (CC((control->nested_ctl & SVM_NESTED_CTL_NP_ENABLE) && !npt_enabled))
266ca46d739SPaolo Bonzini return false;
267ca46d739SPaolo Bonzini
268ee695f22SKrish Sadhukhan if (CC(!nested_svm_check_bitmap_pa(vcpu, control->msrpm_base_pa,
269ee695f22SKrish Sadhukhan MSRPM_SIZE)))
270ee695f22SKrish Sadhukhan return false;
271ee695f22SKrish Sadhukhan if (CC(!nested_svm_check_bitmap_pa(vcpu, control->iopm_base_pa,
272ee695f22SKrish Sadhukhan IOPM_SIZE)))
273ee695f22SKrish Sadhukhan return false;
274ee695f22SKrish Sadhukhan
2750977cfacSSantosh Shukla if (CC((control->int_ctl & V_NMI_ENABLE_MASK) &&
2760977cfacSSantosh Shukla !vmcb12_is_intercept(control, INTERCEPT_NMI))) {
2770977cfacSSantosh Shukla return false;
2780977cfacSSantosh Shukla }
2790977cfacSSantosh Shukla
280ca46d739SPaolo Bonzini return true;
281ca46d739SPaolo Bonzini }
282ca46d739SPaolo Bonzini
2836906e06dSKrish Sadhukhan /* Common checks that apply to both L1 and L2 state. */
__nested_vmcb_check_save(struct kvm_vcpu * vcpu,struct vmcb_save_area_cached * save)284b7a3d8b6SEmanuele Giuseppe Esposito static bool __nested_vmcb_check_save(struct kvm_vcpu *vcpu,
285b7a3d8b6SEmanuele Giuseppe Esposito struct vmcb_save_area_cached *save)
2866906e06dSKrish Sadhukhan {
28711f0cbf0SSean Christopherson if (CC(!(save->efer & EFER_SVME)))
2886906e06dSKrish Sadhukhan return false;
2896906e06dSKrish Sadhukhan
29011f0cbf0SSean Christopherson if (CC((save->cr0 & X86_CR0_CD) == 0 && (save->cr0 & X86_CR0_NW)) ||
29111f0cbf0SSean Christopherson CC(save->cr0 & ~0xffffffffULL))
2926906e06dSKrish Sadhukhan return false;
2936906e06dSKrish Sadhukhan
29411f0cbf0SSean Christopherson if (CC(!kvm_dr6_valid(save->dr6)) || CC(!kvm_dr7_valid(save->dr7)))
2956906e06dSKrish Sadhukhan return false;
2966906e06dSKrish Sadhukhan
297907afa48SEmanuele Giuseppe Esposito /*
298907afa48SEmanuele Giuseppe Esposito * These checks are also performed by KVM_SET_SREGS,
299907afa48SEmanuele Giuseppe Esposito * except that EFER.LMA is not checked by SVM against
300907afa48SEmanuele Giuseppe Esposito * CR0.PG && EFER.LME.
301907afa48SEmanuele Giuseppe Esposito */
302907afa48SEmanuele Giuseppe Esposito if ((save->efer & EFER_LME) && (save->cr0 & X86_CR0_PG)) {
303907afa48SEmanuele Giuseppe Esposito if (CC(!(save->cr4 & X86_CR4_PAE)) ||
304907afa48SEmanuele Giuseppe Esposito CC(!(save->cr0 & X86_CR0_PE)) ||
3052c49db45SBinbin Wu CC(!kvm_vcpu_is_legal_cr3(vcpu, save->cr3)))
306907afa48SEmanuele Giuseppe Esposito return false;
307907afa48SEmanuele Giuseppe Esposito }
308907afa48SEmanuele Giuseppe Esposito
309c33f6f22SSean Christopherson /* Note, SVM doesn't have any additional restrictions on CR4. */
310c33f6f22SSean Christopherson if (CC(!__kvm_is_valid_cr4(vcpu, save->cr4)))
3116906e06dSKrish Sadhukhan return false;
3126906e06dSKrish Sadhukhan
31363129754SPaolo Bonzini if (CC(!kvm_valid_efer(vcpu, save->efer)))
3146906e06dSKrish Sadhukhan return false;
3156906e06dSKrish Sadhukhan
3166906e06dSKrish Sadhukhan return true;
3176906e06dSKrish Sadhukhan }
3186906e06dSKrish Sadhukhan
nested_vmcb_check_save(struct kvm_vcpu * vcpu)319b7a3d8b6SEmanuele Giuseppe Esposito static bool nested_vmcb_check_save(struct kvm_vcpu *vcpu)
320b7a3d8b6SEmanuele Giuseppe Esposito {
321b7a3d8b6SEmanuele Giuseppe Esposito struct vcpu_svm *svm = to_svm(vcpu);
322b7a3d8b6SEmanuele Giuseppe Esposito struct vmcb_save_area_cached *save = &svm->nested.save;
323b7a3d8b6SEmanuele Giuseppe Esposito
324b7a3d8b6SEmanuele Giuseppe Esposito return __nested_vmcb_check_save(vcpu, save);
325b7a3d8b6SEmanuele Giuseppe Esposito }
326b7a3d8b6SEmanuele Giuseppe Esposito
nested_vmcb_check_controls(struct kvm_vcpu * vcpu)327bd95926cSPaolo Bonzini static bool nested_vmcb_check_controls(struct kvm_vcpu *vcpu)
328bd95926cSPaolo Bonzini {
329bd95926cSPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu);
3308fc78909SEmanuele Giuseppe Esposito struct vmcb_ctrl_area_cached *ctl = &svm->nested.ctl;
331bd95926cSPaolo Bonzini
332bd95926cSPaolo Bonzini return __nested_vmcb_check_controls(vcpu, ctl);
333bd95926cSPaolo Bonzini }
334bd95926cSPaolo Bonzini
3357907160dSEmanuele Giuseppe Esposito static
__nested_copy_vmcb_control_to_cache(struct kvm_vcpu * vcpu,struct vmcb_ctrl_area_cached * to,struct vmcb_control_area * from)33666c03a92SVitaly Kuznetsov void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu,
33766c03a92SVitaly Kuznetsov struct vmcb_ctrl_area_cached *to,
3387907160dSEmanuele Giuseppe Esposito struct vmcb_control_area *from)
3397907160dSEmanuele Giuseppe Esposito {
3407907160dSEmanuele Giuseppe Esposito unsigned int i;
3417907160dSEmanuele Giuseppe Esposito
3427907160dSEmanuele Giuseppe Esposito for (i = 0; i < MAX_INTERCEPT; i++)
3437907160dSEmanuele Giuseppe Esposito to->intercepts[i] = from->intercepts[i];
3447907160dSEmanuele Giuseppe Esposito
3457907160dSEmanuele Giuseppe Esposito to->iopm_base_pa = from->iopm_base_pa;
3467907160dSEmanuele Giuseppe Esposito to->msrpm_base_pa = from->msrpm_base_pa;
3477907160dSEmanuele Giuseppe Esposito to->tsc_offset = from->tsc_offset;
3487907160dSEmanuele Giuseppe Esposito to->tlb_ctl = from->tlb_ctl;
3497907160dSEmanuele Giuseppe Esposito to->int_ctl = from->int_ctl;
3507907160dSEmanuele Giuseppe Esposito to->int_vector = from->int_vector;
3517907160dSEmanuele Giuseppe Esposito to->int_state = from->int_state;
3527907160dSEmanuele Giuseppe Esposito to->exit_code = from->exit_code;
3537907160dSEmanuele Giuseppe Esposito to->exit_code_hi = from->exit_code_hi;
3547907160dSEmanuele Giuseppe Esposito to->exit_info_1 = from->exit_info_1;
3557907160dSEmanuele Giuseppe Esposito to->exit_info_2 = from->exit_info_2;
3567907160dSEmanuele Giuseppe Esposito to->exit_int_info = from->exit_int_info;
3577907160dSEmanuele Giuseppe Esposito to->exit_int_info_err = from->exit_int_info_err;
3587907160dSEmanuele Giuseppe Esposito to->nested_ctl = from->nested_ctl;
3597907160dSEmanuele Giuseppe Esposito to->event_inj = from->event_inj;
3607907160dSEmanuele Giuseppe Esposito to->event_inj_err = from->event_inj_err;
36100f08d99SMaciej S. Szmigiero to->next_rip = from->next_rip;
3627907160dSEmanuele Giuseppe Esposito to->nested_cr3 = from->nested_cr3;
3637907160dSEmanuele Giuseppe Esposito to->virt_ext = from->virt_ext;
3647907160dSEmanuele Giuseppe Esposito to->pause_filter_count = from->pause_filter_count;
3657907160dSEmanuele Giuseppe Esposito to->pause_filter_thresh = from->pause_filter_thresh;
3667907160dSEmanuele Giuseppe Esposito
3677907160dSEmanuele Giuseppe Esposito /* Copy asid here because nested_vmcb_check_controls will check it. */
3687907160dSEmanuele Giuseppe Esposito to->asid = from->asid;
3697907160dSEmanuele Giuseppe Esposito to->msrpm_base_pa &= ~0x0fffULL;
3707907160dSEmanuele Giuseppe Esposito to->iopm_base_pa &= ~0x0fffULL;
37166c03a92SVitaly Kuznetsov
372017a99a9SVitaly Kuznetsov #ifdef CONFIG_KVM_HYPERV
37366c03a92SVitaly Kuznetsov /* Hyper-V extensions (Enlightened VMCB) */
37466c03a92SVitaly Kuznetsov if (kvm_hv_hypercall_enabled(vcpu)) {
37566c03a92SVitaly Kuznetsov to->clean = from->clean;
37668ae7c7bSSean Christopherson memcpy(&to->hv_enlightenments, &from->hv_enlightenments,
37768ae7c7bSSean Christopherson sizeof(to->hv_enlightenments));
37866c03a92SVitaly Kuznetsov }
379017a99a9SVitaly Kuznetsov #endif
3807907160dSEmanuele Giuseppe Esposito }
3817907160dSEmanuele Giuseppe Esposito
nested_copy_vmcb_control_to_cache(struct vcpu_svm * svm,struct vmcb_control_area * control)3827907160dSEmanuele Giuseppe Esposito void nested_copy_vmcb_control_to_cache(struct vcpu_svm *svm,
3833e06f016SPaolo Bonzini struct vmcb_control_area *control)
3843e06f016SPaolo Bonzini {
38566c03a92SVitaly Kuznetsov __nested_copy_vmcb_control_to_cache(&svm->vcpu, &svm->nested.ctl, control);
3863e06f016SPaolo Bonzini }
3873e06f016SPaolo Bonzini
__nested_copy_vmcb_save_to_cache(struct vmcb_save_area_cached * to,struct vmcb_save_area * from)388f2740a8dSEmanuele Giuseppe Esposito static void __nested_copy_vmcb_save_to_cache(struct vmcb_save_area_cached *to,
389f2740a8dSEmanuele Giuseppe Esposito struct vmcb_save_area *from)
390f2740a8dSEmanuele Giuseppe Esposito {
391f2740a8dSEmanuele Giuseppe Esposito /*
392f2740a8dSEmanuele Giuseppe Esposito * Copy only fields that are validated, as we need them
393f2740a8dSEmanuele Giuseppe Esposito * to avoid TOC/TOU races.
394f2740a8dSEmanuele Giuseppe Esposito */
395f2740a8dSEmanuele Giuseppe Esposito to->efer = from->efer;
396f2740a8dSEmanuele Giuseppe Esposito to->cr0 = from->cr0;
397f2740a8dSEmanuele Giuseppe Esposito to->cr3 = from->cr3;
398f2740a8dSEmanuele Giuseppe Esposito to->cr4 = from->cr4;
399f2740a8dSEmanuele Giuseppe Esposito
400f2740a8dSEmanuele Giuseppe Esposito to->dr6 = from->dr6;
401f2740a8dSEmanuele Giuseppe Esposito to->dr7 = from->dr7;
402f2740a8dSEmanuele Giuseppe Esposito }
403f2740a8dSEmanuele Giuseppe Esposito
nested_copy_vmcb_save_to_cache(struct vcpu_svm * svm,struct vmcb_save_area * save)404f2740a8dSEmanuele Giuseppe Esposito void nested_copy_vmcb_save_to_cache(struct vcpu_svm *svm,
405f2740a8dSEmanuele Giuseppe Esposito struct vmcb_save_area *save)
406f2740a8dSEmanuele Giuseppe Esposito {
407f2740a8dSEmanuele Giuseppe Esposito __nested_copy_vmcb_save_to_cache(&svm->nested.save, save);
408f2740a8dSEmanuele Giuseppe Esposito }
409f2740a8dSEmanuele Giuseppe Esposito
4102d8a42beSPaolo Bonzini /*
4112d8a42beSPaolo Bonzini * Synchronize fields that are written by the processor, so that
4129e8f0fbfSPaolo Bonzini * they can be copied back into the vmcb12.
4132d8a42beSPaolo Bonzini */
nested_sync_control_from_vmcb02(struct vcpu_svm * svm)4149e8f0fbfSPaolo Bonzini void nested_sync_control_from_vmcb02(struct vcpu_svm *svm)
4152d8a42beSPaolo Bonzini {
4162d8a42beSPaolo Bonzini u32 mask;
4172d8a42beSPaolo Bonzini svm->nested.ctl.event_inj = svm->vmcb->control.event_inj;
4182d8a42beSPaolo Bonzini svm->nested.ctl.event_inj_err = svm->vmcb->control.event_inj_err;
4192d8a42beSPaolo Bonzini
4202d8a42beSPaolo Bonzini /* Only a few fields of int_ctl are written by the processor. */
4212d8a42beSPaolo Bonzini mask = V_IRQ_MASK | V_TPR_MASK;
4222d8a42beSPaolo Bonzini /*
4235faaffabSSantosh Shukla * Don't sync vmcb02 V_IRQ back to vmcb12 if KVM (L0) is intercepting
4245faaffabSSantosh Shukla * virtual interrupts in order to request an interrupt window, as KVM
4255faaffabSSantosh Shukla * has usurped vmcb02's int_ctl. If an interrupt window opens before
4265faaffabSSantosh Shukla * the next VM-Exit, svm_clear_vintr() will restore vmcb12's int_ctl.
4275faaffabSSantosh Shukla * If no window opens, V_IRQ will be correctly preserved in vmcb12's
4285faaffabSSantosh Shukla * int_ctl (because it was never recognized while L2 was running).
4292d8a42beSPaolo Bonzini */
4305faaffabSSantosh Shukla if (svm_is_intercept(svm, INTERCEPT_VINTR) &&
4315faaffabSSantosh Shukla !test_bit(INTERCEPT_VINTR, (unsigned long *)svm->nested.ctl.intercepts))
4322d8a42beSPaolo Bonzini mask &= ~V_IRQ_MASK;
4330b349662SMaxim Levitsky
4340b349662SMaxim Levitsky if (nested_vgif_enabled(svm))
4350b349662SMaxim Levitsky mask |= V_GIF_MASK;
4360b349662SMaxim Levitsky
4370977cfacSSantosh Shukla if (nested_vnmi_enabled(svm))
4380977cfacSSantosh Shukla mask |= V_NMI_BLOCKING_MASK | V_NMI_PENDING_MASK;
4390977cfacSSantosh Shukla
4402d8a42beSPaolo Bonzini svm->nested.ctl.int_ctl &= ~mask;
4412d8a42beSPaolo Bonzini svm->nested.ctl.int_ctl |= svm->vmcb->control.int_ctl & mask;
4422d8a42beSPaolo Bonzini }
4432d8a42beSPaolo Bonzini
44436e2e983SPaolo Bonzini /*
44536e2e983SPaolo Bonzini * Transfer any event that L0 or L1 wanted to inject into L2 to
44636e2e983SPaolo Bonzini * EXIT_INT_INFO.
44736e2e983SPaolo Bonzini */
nested_save_pending_event_to_vmcb12(struct vcpu_svm * svm,struct vmcb * vmcb12)4489e8f0fbfSPaolo Bonzini static void nested_save_pending_event_to_vmcb12(struct vcpu_svm *svm,
4490dd16b5bSMaxim Levitsky struct vmcb *vmcb12)
45036e2e983SPaolo Bonzini {
45136e2e983SPaolo Bonzini struct kvm_vcpu *vcpu = &svm->vcpu;
45236e2e983SPaolo Bonzini u32 exit_int_info = 0;
45336e2e983SPaolo Bonzini unsigned int nr;
45436e2e983SPaolo Bonzini
45536e2e983SPaolo Bonzini if (vcpu->arch.exception.injected) {
456d4963e31SSean Christopherson nr = vcpu->arch.exception.vector;
45736e2e983SPaolo Bonzini exit_int_info = nr | SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_EXEPT;
45836e2e983SPaolo Bonzini
45936e2e983SPaolo Bonzini if (vcpu->arch.exception.has_error_code) {
46036e2e983SPaolo Bonzini exit_int_info |= SVM_EVTINJ_VALID_ERR;
4610dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info_err =
46236e2e983SPaolo Bonzini vcpu->arch.exception.error_code;
46336e2e983SPaolo Bonzini }
46436e2e983SPaolo Bonzini
46536e2e983SPaolo Bonzini } else if (vcpu->arch.nmi_injected) {
46636e2e983SPaolo Bonzini exit_int_info = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
46736e2e983SPaolo Bonzini
46836e2e983SPaolo Bonzini } else if (vcpu->arch.interrupt.injected) {
46936e2e983SPaolo Bonzini nr = vcpu->arch.interrupt.nr;
47036e2e983SPaolo Bonzini exit_int_info = nr | SVM_EVTINJ_VALID;
47136e2e983SPaolo Bonzini
47236e2e983SPaolo Bonzini if (vcpu->arch.interrupt.soft)
47336e2e983SPaolo Bonzini exit_int_info |= SVM_EVTINJ_TYPE_SOFT;
47436e2e983SPaolo Bonzini else
47536e2e983SPaolo Bonzini exit_int_info |= SVM_EVTINJ_TYPE_INTR;
47636e2e983SPaolo Bonzini }
47736e2e983SPaolo Bonzini
4780dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info = exit_int_info;
47936e2e983SPaolo Bonzini }
48036e2e983SPaolo Bonzini
nested_svm_transition_tlb_flush(struct kvm_vcpu * vcpu)481d2e56019SSean Christopherson static void nested_svm_transition_tlb_flush(struct kvm_vcpu *vcpu)
482d2e56019SSean Christopherson {
483af9d544aSVitaly Kuznetsov /* Handle pending Hyper-V TLB flush requests */
484af9d544aSVitaly Kuznetsov kvm_hv_nested_transtion_tlb_flush(vcpu, npt_enabled);
4853f4a812eSVitaly Kuznetsov
4863f4a812eSVitaly Kuznetsov /*
487d2e56019SSean Christopherson * TODO: optimize unconditional TLB flush/MMU sync. A partial list of
488d2e56019SSean Christopherson * things to fix before this can be conditional:
489d2e56019SSean Christopherson *
490d2e56019SSean Christopherson * - Flush TLBs for both L1 and L2 remote TLB flush
491d2e56019SSean Christopherson * - Honor L1's request to flush an ASID on nested VMRUN
492d2e56019SSean Christopherson * - Sync nested NPT MMU on VMRUN that flushes L2's ASID[*]
493d2e56019SSean Christopherson * - Don't crush a pending TLB flush in vmcb02 on nested VMRUN
494d2e56019SSean Christopherson * - Flush L1's ASID on KVM_REQ_TLB_FLUSH_GUEST
495d2e56019SSean Christopherson *
496d2e56019SSean Christopherson * [*] Unlike nested EPT, SVM's ASID management can invalidate nested
497d2e56019SSean Christopherson * NPT guest-physical mappings on VMRUN.
498d2e56019SSean Christopherson */
499d2e56019SSean Christopherson kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
500d2e56019SSean Christopherson kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
501d2e56019SSean Christopherson }
502d2e56019SSean Christopherson
50362156f6cSVitaly Kuznetsov /*
504d82aaef9SVitaly Kuznetsov * Load guest's/host's cr3 on nested vmentry or vmexit. @nested_npt is true
505d82aaef9SVitaly Kuznetsov * if we are emulating VM-Entry into a guest with NPT enabled.
50662156f6cSVitaly Kuznetsov */
nested_svm_load_cr3(struct kvm_vcpu * vcpu,unsigned long cr3,bool nested_npt,bool reload_pdptrs)50762156f6cSVitaly Kuznetsov static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3,
508b222b0b8SMaxim Levitsky bool nested_npt, bool reload_pdptrs)
50962156f6cSVitaly Kuznetsov {
5102c49db45SBinbin Wu if (CC(!kvm_vcpu_is_legal_cr3(vcpu, cr3)))
511a506fdd2SVitaly Kuznetsov return -EINVAL;
512a506fdd2SVitaly Kuznetsov
513b222b0b8SMaxim Levitsky if (reload_pdptrs && !nested_npt && is_pae_paging(vcpu) &&
5142df4a5ebSLai Jiangshan CC(!load_pdptrs(vcpu, cr3)))
515a506fdd2SVitaly Kuznetsov return -EINVAL;
516a506fdd2SVitaly Kuznetsov
517a506fdd2SVitaly Kuznetsov vcpu->arch.cr3 = cr3;
518a506fdd2SVitaly Kuznetsov
519616007c8SSean Christopherson /* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */
520c9060662SSean Christopherson kvm_init_mmu(vcpu);
521a506fdd2SVitaly Kuznetsov
5223cffc89dSPaolo Bonzini if (!nested_npt)
5233cffc89dSPaolo Bonzini kvm_mmu_new_pgd(vcpu, cr3);
5243cffc89dSPaolo Bonzini
525a506fdd2SVitaly Kuznetsov return 0;
52662156f6cSVitaly Kuznetsov }
52762156f6cSVitaly Kuznetsov
nested_vmcb02_compute_g_pat(struct vcpu_svm * svm)5284995a368SCathy Avery void nested_vmcb02_compute_g_pat(struct vcpu_svm *svm)
5294995a368SCathy Avery {
5304995a368SCathy Avery if (!svm->nested.vmcb02.ptr)
5314995a368SCathy Avery return;
5324995a368SCathy Avery
5334995a368SCathy Avery /* FIXME: merge g_pat from vmcb01 and vmcb12. */
5344995a368SCathy Avery svm->nested.vmcb02.ptr->save.g_pat = svm->vmcb01.ptr->save.g_pat;
5354995a368SCathy Avery }
5364995a368SCathy Avery
nested_vmcb02_prepare_save(struct vcpu_svm * svm,struct vmcb * vmcb12)5379e8f0fbfSPaolo Bonzini static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12)
538883b0a91SJoerg Roedel {
5398173396eSCathy Avery bool new_vmcb12 = false;
5401d5a1b58SMaxim Levitsky struct vmcb *vmcb01 = svm->vmcb01.ptr;
541db663af4SMaxim Levitsky struct vmcb *vmcb02 = svm->nested.vmcb02.ptr;
542e183d17aSSean Christopherson struct kvm_vcpu *vcpu = &svm->vcpu;
5438173396eSCathy Avery
5444995a368SCathy Avery nested_vmcb02_compute_g_pat(svm);
5454995a368SCathy Avery
546883b0a91SJoerg Roedel /* Load the nested guest state */
5478173396eSCathy Avery if (svm->nested.vmcb12_gpa != svm->nested.last_vmcb12_gpa) {
5488173396eSCathy Avery new_vmcb12 = true;
5498173396eSCathy Avery svm->nested.last_vmcb12_gpa = svm->nested.vmcb12_gpa;
55073c25546SVitaly Kuznetsov svm->nested.force_msr_bitmap_recalc = true;
5518173396eSCathy Avery }
5528173396eSCathy Avery
5538173396eSCathy Avery if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_SEG))) {
554db663af4SMaxim Levitsky vmcb02->save.es = vmcb12->save.es;
555db663af4SMaxim Levitsky vmcb02->save.cs = vmcb12->save.cs;
556db663af4SMaxim Levitsky vmcb02->save.ss = vmcb12->save.ss;
557db663af4SMaxim Levitsky vmcb02->save.ds = vmcb12->save.ds;
558db663af4SMaxim Levitsky vmcb02->save.cpl = vmcb12->save.cpl;
559db663af4SMaxim Levitsky vmcb_mark_dirty(vmcb02, VMCB_SEG);
5608173396eSCathy Avery }
5614bb170a5SPaolo Bonzini
5628173396eSCathy Avery if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_DT))) {
563db663af4SMaxim Levitsky vmcb02->save.gdtr = vmcb12->save.gdtr;
564db663af4SMaxim Levitsky vmcb02->save.idtr = vmcb12->save.idtr;
565db663af4SMaxim Levitsky vmcb_mark_dirty(vmcb02, VMCB_DT);
5668173396eSCathy Avery }
5674bb170a5SPaolo Bonzini
568e183d17aSSean Christopherson kvm_set_rflags(vcpu, vmcb12->save.rflags | X86_EFLAGS_FIXED);
5693c346c0cSPaolo Bonzini
570e183d17aSSean Christopherson svm_set_efer(vcpu, svm->nested.save.efer);
5713c346c0cSPaolo Bonzini
572e183d17aSSean Christopherson svm_set_cr0(vcpu, svm->nested.save.cr0);
573e183d17aSSean Christopherson svm_set_cr4(vcpu, svm->nested.save.cr4);
5744bb170a5SPaolo Bonzini
5754bb170a5SPaolo Bonzini svm->vcpu.arch.cr2 = vmcb12->save.cr2;
5768173396eSCathy Avery
577e183d17aSSean Christopherson kvm_rax_write(vcpu, vmcb12->save.rax);
578e183d17aSSean Christopherson kvm_rsp_write(vcpu, vmcb12->save.rsp);
579e183d17aSSean Christopherson kvm_rip_write(vcpu, vmcb12->save.rip);
580883b0a91SJoerg Roedel
581883b0a91SJoerg Roedel /* In case we don't even reach vcpu_run, the fields are not updated */
582db663af4SMaxim Levitsky vmcb02->save.rax = vmcb12->save.rax;
583db663af4SMaxim Levitsky vmcb02->save.rsp = vmcb12->save.rsp;
584db663af4SMaxim Levitsky vmcb02->save.rip = vmcb12->save.rip;
5854bb170a5SPaolo Bonzini
5868173396eSCathy Avery /* These bits will be set properly on the first execution when new_vmc12 is true */
5878173396eSCathy Avery if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_DR))) {
588db663af4SMaxim Levitsky vmcb02->save.dr7 = svm->nested.save.dr7 | DR7_FIXED_1;
589355d0473SEmanuele Giuseppe Esposito svm->vcpu.arch.dr6 = svm->nested.save.dr6 | DR6_ACTIVE_LOW;
590db663af4SMaxim Levitsky vmcb_mark_dirty(vmcb02, VMCB_DR);
591f241d711SPaolo Bonzini }
5921d5a1b58SMaxim Levitsky
593e183d17aSSean Christopherson if (unlikely(guest_can_use(vcpu, X86_FEATURE_LBRV) &&
594e183d17aSSean Christopherson (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK))) {
595d20c796cSMaxim Levitsky /*
596d20c796cSMaxim Levitsky * Reserved bits of DEBUGCTL are ignored. Be consistent with
597d20c796cSMaxim Levitsky * svm_set_msr's definition of reserved bits.
598d20c796cSMaxim Levitsky */
599d20c796cSMaxim Levitsky svm_copy_lbrs(vmcb02, vmcb12);
600d20c796cSMaxim Levitsky vmcb02->save.dbgctl &= ~DEBUGCTL_RESERVED_BITS;
601d20c796cSMaxim Levitsky svm_update_lbrv(&svm->vcpu);
602d20c796cSMaxim Levitsky
603d20c796cSMaxim Levitsky } else if (unlikely(vmcb01->control.virt_ext & LBR_CTL_ENABLE_MASK)) {
6041d5a1b58SMaxim Levitsky svm_copy_lbrs(vmcb02, vmcb01);
6058173396eSCathy Avery }
606d20c796cSMaxim Levitsky }
607883b0a91SJoerg Roedel
is_evtinj_soft(u32 evtinj)6086ef88d6eSSean Christopherson static inline bool is_evtinj_soft(u32 evtinj)
6096ef88d6eSSean Christopherson {
6106ef88d6eSSean Christopherson u32 type = evtinj & SVM_EVTINJ_TYPE_MASK;
6116ef88d6eSSean Christopherson u8 vector = evtinj & SVM_EVTINJ_VEC_MASK;
6126ef88d6eSSean Christopherson
6136ef88d6eSSean Christopherson if (!(evtinj & SVM_EVTINJ_VALID))
6146ef88d6eSSean Christopherson return false;
6156ef88d6eSSean Christopherson
6167e5b5ef8SSean Christopherson if (type == SVM_EVTINJ_TYPE_SOFT)
6177e5b5ef8SSean Christopherson return true;
6187e5b5ef8SSean Christopherson
6196ef88d6eSSean Christopherson return type == SVM_EVTINJ_TYPE_EXEPT && kvm_exception_is_soft(vector);
6206ef88d6eSSean Christopherson }
6216ef88d6eSSean Christopherson
is_evtinj_nmi(u32 evtinj)622159fc6faSMaciej S. Szmigiero static bool is_evtinj_nmi(u32 evtinj)
623159fc6faSMaciej S. Szmigiero {
624159fc6faSMaciej S. Szmigiero u32 type = evtinj & SVM_EVTINJ_TYPE_MASK;
625159fc6faSMaciej S. Szmigiero
626159fc6faSMaciej S. Szmigiero if (!(evtinj & SVM_EVTINJ_VALID))
627159fc6faSMaciej S. Szmigiero return false;
628159fc6faSMaciej S. Szmigiero
629159fc6faSMaciej S. Szmigiero return type == SVM_EVTINJ_TYPE_NMI;
630159fc6faSMaciej S. Szmigiero }
631159fc6faSMaciej S. Szmigiero
nested_vmcb02_prepare_control(struct vcpu_svm * svm,unsigned long vmcb12_rip,unsigned long vmcb12_csbase)63200f08d99SMaciej S. Szmigiero static void nested_vmcb02_prepare_control(struct vcpu_svm *svm,
633da0b93d6SMaciej S. Szmigiero unsigned long vmcb12_rip,
634da0b93d6SMaciej S. Szmigiero unsigned long vmcb12_csbase)
635f241d711SPaolo Bonzini {
6360b349662SMaxim Levitsky u32 int_ctl_vmcb01_bits = V_INTR_MASKING_MASK;
6370b349662SMaxim Levitsky u32 int_ctl_vmcb12_bits = V_TPR_MASK | V_IRQ_INJECTION_BITS_MASK;
6380f923e07SMaxim Levitsky
639d2e56019SSean Christopherson struct kvm_vcpu *vcpu = &svm->vcpu;
640db663af4SMaxim Levitsky struct vmcb *vmcb01 = svm->vmcb01.ptr;
641db663af4SMaxim Levitsky struct vmcb *vmcb02 = svm->nested.vmcb02.ptr;
642e3cdaab5SPaolo Bonzini u32 pause_count12;
643e3cdaab5SPaolo Bonzini u32 pause_thresh12;
64462156f6cSVitaly Kuznetsov
6457c3ecfcdSPaolo Bonzini /*
6467c3ecfcdSPaolo Bonzini * Filled at exit: exit_code, exit_code_hi, exit_info_1, exit_info_2,
6477c3ecfcdSPaolo Bonzini * exit_int_info, exit_int_info_err, next_rip, insn_len, insn_bytes.
6487c3ecfcdSPaolo Bonzini */
6494995a368SCathy Avery
650b89456aeSSean Christopherson if (guest_can_use(vcpu, X86_FEATURE_VGIF) &&
651b89456aeSSean Christopherson (svm->nested.ctl.int_ctl & V_GIF_ENABLE_MASK))
6520b349662SMaxim Levitsky int_ctl_vmcb12_bits |= (V_GIF_MASK | V_GIF_ENABLE_MASK);
6530b349662SMaxim Levitsky else
6540b349662SMaxim Levitsky int_ctl_vmcb01_bits |= (V_GIF_MASK | V_GIF_ENABLE_MASK);
6550b349662SMaxim Levitsky
6560977cfacSSantosh Shukla if (vnmi) {
6570977cfacSSantosh Shukla if (vmcb01->control.int_ctl & V_NMI_PENDING_MASK) {
6580977cfacSSantosh Shukla svm->vcpu.arch.nmi_pending++;
6590977cfacSSantosh Shukla kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
6600977cfacSSantosh Shukla }
6610977cfacSSantosh Shukla if (nested_vnmi_enabled(svm))
6620977cfacSSantosh Shukla int_ctl_vmcb12_bits |= (V_NMI_PENDING_MASK |
6630977cfacSSantosh Shukla V_NMI_ENABLE_MASK |
6640977cfacSSantosh Shukla V_NMI_BLOCKING_MASK);
6650977cfacSSantosh Shukla }
6660977cfacSSantosh Shukla
6677c3ecfcdSPaolo Bonzini /* Copied from vmcb01. msrpm_base can be overwritten later. */
668db663af4SMaxim Levitsky vmcb02->control.nested_ctl = vmcb01->control.nested_ctl;
669db663af4SMaxim Levitsky vmcb02->control.iopm_base_pa = vmcb01->control.iopm_base_pa;
670db663af4SMaxim Levitsky vmcb02->control.msrpm_base_pa = vmcb01->control.msrpm_base_pa;
6717c3ecfcdSPaolo Bonzini
6727c3ecfcdSPaolo Bonzini /* Done at vmrun: asid. */
6737c3ecfcdSPaolo Bonzini
6747c3ecfcdSPaolo Bonzini /* Also overwritten later if necessary. */
675db663af4SMaxim Levitsky vmcb02->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
6767c3ecfcdSPaolo Bonzini
6777c3ecfcdSPaolo Bonzini /* nested_cr3. */
67862156f6cSVitaly Kuznetsov if (nested_npt_enabled(svm))
679d2e56019SSean Christopherson nested_svm_init_mmu_context(vcpu);
68069cb8774SPaolo Bonzini
6815228eb96SMaxim Levitsky vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(
6825228eb96SMaxim Levitsky vcpu->arch.l1_tsc_offset,
6835228eb96SMaxim Levitsky svm->nested.ctl.tsc_offset,
6845228eb96SMaxim Levitsky svm->tsc_ratio_msr);
6855228eb96SMaxim Levitsky
686db663af4SMaxim Levitsky vmcb02->control.tsc_offset = vcpu->arch.tsc_offset;
6875228eb96SMaxim Levitsky
6884365a455SSean Christopherson if (guest_can_use(vcpu, X86_FEATURE_TSCRATEMSR) &&
6897cafe9b8SSean Christopherson svm->tsc_ratio_msr != kvm_caps.default_tsc_scaling_ratio)
6905228eb96SMaxim Levitsky nested_svm_update_tsc_ratio_msr(vcpu);
691883b0a91SJoerg Roedel
692db663af4SMaxim Levitsky vmcb02->control.int_ctl =
6930f923e07SMaxim Levitsky (svm->nested.ctl.int_ctl & int_ctl_vmcb12_bits) |
694db663af4SMaxim Levitsky (vmcb01->control.int_ctl & int_ctl_vmcb01_bits);
69591b7130cSPaolo Bonzini
696db663af4SMaxim Levitsky vmcb02->control.int_vector = svm->nested.ctl.int_vector;
697db663af4SMaxim Levitsky vmcb02->control.int_state = svm->nested.ctl.int_state;
698db663af4SMaxim Levitsky vmcb02->control.event_inj = svm->nested.ctl.event_inj;
699db663af4SMaxim Levitsky vmcb02->control.event_inj_err = svm->nested.ctl.event_inj_err;
700883b0a91SJoerg Roedel
70100f08d99SMaciej S. Szmigiero /*
70200f08d99SMaciej S. Szmigiero * next_rip is consumed on VMRUN as the return address pushed on the
70300f08d99SMaciej S. Szmigiero * stack for injected soft exceptions/interrupts. If nrips is exposed
70400f08d99SMaciej S. Szmigiero * to L1, take it verbatim from vmcb12. If nrips is supported in
70500f08d99SMaciej S. Szmigiero * hardware but not exposed to L1, stuff the actual L2 RIP to emulate
70600f08d99SMaciej S. Szmigiero * what a nrips=0 CPU would do (L1 is responsible for advancing RIP
70700f08d99SMaciej S. Szmigiero * prior to injecting the event).
70800f08d99SMaciej S. Szmigiero */
7097a6a6a3bSSean Christopherson if (guest_can_use(vcpu, X86_FEATURE_NRIPS))
71000f08d99SMaciej S. Szmigiero vmcb02->control.next_rip = svm->nested.ctl.next_rip;
71100f08d99SMaciej S. Szmigiero else if (boot_cpu_has(X86_FEATURE_NRIPS))
71200f08d99SMaciej S. Szmigiero vmcb02->control.next_rip = vmcb12_rip;
71300f08d99SMaciej S. Szmigiero
714159fc6faSMaciej S. Szmigiero svm->nmi_l1_to_l2 = is_evtinj_nmi(vmcb02->control.event_inj);
7156ef88d6eSSean Christopherson if (is_evtinj_soft(vmcb02->control.event_inj)) {
7166ef88d6eSSean Christopherson svm->soft_int_injected = true;
717da0b93d6SMaciej S. Szmigiero svm->soft_int_csbase = vmcb12_csbase;
7186ef88d6eSSean Christopherson svm->soft_int_old_rip = vmcb12_rip;
7197a6a6a3bSSean Christopherson if (guest_can_use(vcpu, X86_FEATURE_NRIPS))
7206ef88d6eSSean Christopherson svm->soft_int_next_rip = svm->nested.ctl.next_rip;
7216ef88d6eSSean Christopherson else
7226ef88d6eSSean Christopherson svm->soft_int_next_rip = vmcb12_rip;
7236ef88d6eSSean Christopherson }
7246ef88d6eSSean Christopherson
7251d5a1b58SMaxim Levitsky vmcb02->control.virt_ext = vmcb01->control.virt_ext &
7261d5a1b58SMaxim Levitsky LBR_CTL_ENABLE_MASK;
727e183d17aSSean Christopherson if (guest_can_use(vcpu, X86_FEATURE_LBRV))
728d20c796cSMaxim Levitsky vmcb02->control.virt_ext |=
729d20c796cSMaxim Levitsky (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK);
7301d5a1b58SMaxim Levitsky
731b9f3973aSMaxim Levitsky if (!nested_vmcb_needs_vls_intercept(svm))
732db663af4SMaxim Levitsky vmcb02->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
733b9f3973aSMaxim Levitsky
73459d67fc1SSean Christopherson if (guest_can_use(vcpu, X86_FEATURE_PAUSEFILTER))
73559d67fc1SSean Christopherson pause_count12 = svm->nested.ctl.pause_filter_count;
73659d67fc1SSean Christopherson else
73759d67fc1SSean Christopherson pause_count12 = 0;
73859d67fc1SSean Christopherson if (guest_can_use(vcpu, X86_FEATURE_PFTHRESHOLD))
73959d67fc1SSean Christopherson pause_thresh12 = svm->nested.ctl.pause_filter_thresh;
74059d67fc1SSean Christopherson else
74159d67fc1SSean Christopherson pause_thresh12 = 0;
74274fd41edSMaxim Levitsky if (kvm_pause_in_guest(svm->vcpu.kvm)) {
743e3cdaab5SPaolo Bonzini /* use guest values since host doesn't intercept PAUSE */
744e3cdaab5SPaolo Bonzini vmcb02->control.pause_filter_count = pause_count12;
745e3cdaab5SPaolo Bonzini vmcb02->control.pause_filter_thresh = pause_thresh12;
74674fd41edSMaxim Levitsky
747e3cdaab5SPaolo Bonzini } else {
748e3cdaab5SPaolo Bonzini /* start from host values otherwise */
74974fd41edSMaxim Levitsky vmcb02->control.pause_filter_count = vmcb01->control.pause_filter_count;
75074fd41edSMaxim Levitsky vmcb02->control.pause_filter_thresh = vmcb01->control.pause_filter_thresh;
751e3cdaab5SPaolo Bonzini
752e3cdaab5SPaolo Bonzini /* ... but ensure filtering is disabled if so requested. */
753e3cdaab5SPaolo Bonzini if (vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_PAUSE)) {
754e3cdaab5SPaolo Bonzini if (!pause_count12)
75574fd41edSMaxim Levitsky vmcb02->control.pause_filter_count = 0;
756e3cdaab5SPaolo Bonzini if (!pause_thresh12)
75774fd41edSMaxim Levitsky vmcb02->control.pause_filter_thresh = 0;
75874fd41edSMaxim Levitsky }
759e3cdaab5SPaolo Bonzini }
76074fd41edSMaxim Levitsky
761d2e56019SSean Christopherson nested_svm_transition_tlb_flush(vcpu);
762d2e56019SSean Christopherson
763883b0a91SJoerg Roedel /* Enter Guest-Mode */
764d2e56019SSean Christopherson enter_guest_mode(vcpu);
765883b0a91SJoerg Roedel
766883b0a91SJoerg Roedel /*
767883b0a91SJoerg Roedel * Merge guest and host intercepts - must be called with vcpu in
7684bb170a5SPaolo Bonzini * guest-mode to take effect.
769883b0a91SJoerg Roedel */
770883b0a91SJoerg Roedel recalc_intercepts(svm);
771f241d711SPaolo Bonzini }
772f241d711SPaolo Bonzini
nested_svm_copy_common_state(struct vmcb * from_vmcb,struct vmcb * to_vmcb)773d00b99c5SBabu Moger static void nested_svm_copy_common_state(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
774d00b99c5SBabu Moger {
775d00b99c5SBabu Moger /*
776d00b99c5SBabu Moger * Some VMCB state is shared between L1 and L2 and thus has to be
777d00b99c5SBabu Moger * moved at the time of nested vmrun and vmexit.
778d00b99c5SBabu Moger *
779d00b99c5SBabu Moger * VMLOAD/VMSAVE state would also belong in this category, but KVM
780d00b99c5SBabu Moger * always performs VMLOAD and VMSAVE from the VMCB01.
781d00b99c5SBabu Moger */
782d00b99c5SBabu Moger to_vmcb->save.spec_ctrl = from_vmcb->save.spec_ctrl;
783d00b99c5SBabu Moger }
784d00b99c5SBabu Moger
enter_svm_guest_mode(struct kvm_vcpu * vcpu,u64 vmcb12_gpa,struct vmcb * vmcb12,bool from_vmrun)78563129754SPaolo Bonzini int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb12_gpa,
786e85d3e7bSMaxim Levitsky struct vmcb *vmcb12, bool from_vmrun)
787f241d711SPaolo Bonzini {
78863129754SPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu);
789a506fdd2SVitaly Kuznetsov int ret;
790a506fdd2SVitaly Kuznetsov
79189e54ec5SMingwei Zhang trace_kvm_nested_vmenter(svm->vmcb->save.rip,
79289e54ec5SMingwei Zhang vmcb12_gpa,
793954f419bSMaxim Levitsky vmcb12->save.rip,
794954f419bSMaxim Levitsky vmcb12->control.int_ctl,
795954f419bSMaxim Levitsky vmcb12->control.event_inj,
79689e54ec5SMingwei Zhang vmcb12->control.nested_ctl,
79702dfc44fSMingwei Zhang vmcb12->control.nested_cr3,
79802dfc44fSMingwei Zhang vmcb12->save.cr3,
79989e54ec5SMingwei Zhang KVM_ISA_SVM);
800954f419bSMaxim Levitsky
801954f419bSMaxim Levitsky trace_kvm_nested_intercepts(vmcb12->control.intercepts[INTERCEPT_CR] & 0xffff,
802954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_CR] >> 16,
803954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_EXCEPTION],
804954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_WORD3],
805954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_WORD4],
806954f419bSMaxim Levitsky vmcb12->control.intercepts[INTERCEPT_WORD5]);
807954f419bSMaxim Levitsky
808954f419bSMaxim Levitsky
8090dd16b5bSMaxim Levitsky svm->nested.vmcb12_gpa = vmcb12_gpa;
8104995a368SCathy Avery
8114995a368SCathy Avery WARN_ON(svm->vmcb == svm->nested.vmcb02.ptr);
8124995a368SCathy Avery
813d00b99c5SBabu Moger nested_svm_copy_common_state(svm->vmcb01.ptr, svm->nested.vmcb02.ptr);
8144995a368SCathy Avery
8154995a368SCathy Avery svm_switch_vmcb(svm, &svm->nested.vmcb02);
816da0b93d6SMaciej S. Szmigiero nested_vmcb02_prepare_control(svm, vmcb12->save.rip, vmcb12->save.cs.base);
8179e8f0fbfSPaolo Bonzini nested_vmcb02_prepare_save(svm, vmcb12);
818f241d711SPaolo Bonzini
819355d0473SEmanuele Giuseppe Esposito ret = nested_svm_load_cr3(&svm->vcpu, svm->nested.save.cr3,
820e85d3e7bSMaxim Levitsky nested_npt_enabled(svm), from_vmrun);
821a506fdd2SVitaly Kuznetsov if (ret)
822a506fdd2SVitaly Kuznetsov return ret;
823a506fdd2SVitaly Kuznetsov
824e85d3e7bSMaxim Levitsky if (!from_vmrun)
825e85d3e7bSMaxim Levitsky kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
826e85d3e7bSMaxim Levitsky
827ffdf7f9eSPaolo Bonzini svm_set_gif(svm, true);
82859cd9bc5SVitaly Kuznetsov
829f44509f8SMaxim Levitsky if (kvm_vcpu_apicv_active(vcpu))
830f44509f8SMaxim Levitsky kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);
831f44509f8SMaxim Levitsky
832e45aa244SVitaly Kuznetsov nested_svm_hv_update_vm_vp_ids(vcpu);
833e45aa244SVitaly Kuznetsov
83459cd9bc5SVitaly Kuznetsov return 0;
835883b0a91SJoerg Roedel }
836883b0a91SJoerg Roedel
nested_svm_vmrun(struct kvm_vcpu * vcpu)83763129754SPaolo Bonzini int nested_svm_vmrun(struct kvm_vcpu *vcpu)
838883b0a91SJoerg Roedel {
83963129754SPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu);
840883b0a91SJoerg Roedel int ret;
8410dd16b5bSMaxim Levitsky struct vmcb *vmcb12;
842883b0a91SJoerg Roedel struct kvm_host_map map;
8430dd16b5bSMaxim Levitsky u64 vmcb12_gpa;
844db663af4SMaxim Levitsky struct vmcb *vmcb01 = svm->vmcb01.ptr;
845883b0a91SJoerg Roedel
846fb79f566SVitaly Kuznetsov if (!svm->nested.hsave_msr) {
847fb79f566SVitaly Kuznetsov kvm_inject_gp(vcpu, 0);
848fb79f566SVitaly Kuznetsov return 1;
849fb79f566SVitaly Kuznetsov }
850fb79f566SVitaly Kuznetsov
85163129754SPaolo Bonzini if (is_smm(vcpu)) {
85263129754SPaolo Bonzini kvm_queue_exception(vcpu, UD_VECTOR);
8537c67f546SPaolo Bonzini return 1;
8547c67f546SPaolo Bonzini }
855883b0a91SJoerg Roedel
8563f4a812eSVitaly Kuznetsov /* This fails when VP assist page is enabled but the supplied GPA is bogus */
8573f4a812eSVitaly Kuznetsov ret = kvm_hv_verify_vp_assist(vcpu);
8583f4a812eSVitaly Kuznetsov if (ret) {
8593f4a812eSVitaly Kuznetsov kvm_inject_gp(vcpu, 0);
8603f4a812eSVitaly Kuznetsov return ret;
8613f4a812eSVitaly Kuznetsov }
8623f4a812eSVitaly Kuznetsov
8630dd16b5bSMaxim Levitsky vmcb12_gpa = svm->vmcb->save.rax;
86463129754SPaolo Bonzini ret = kvm_vcpu_map(vcpu, gpa_to_gfn(vmcb12_gpa), &map);
865883b0a91SJoerg Roedel if (ret == -EINVAL) {
86663129754SPaolo Bonzini kvm_inject_gp(vcpu, 0);
867883b0a91SJoerg Roedel return 1;
868883b0a91SJoerg Roedel } else if (ret) {
86963129754SPaolo Bonzini return kvm_skip_emulated_instruction(vcpu);
870883b0a91SJoerg Roedel }
871883b0a91SJoerg Roedel
87263129754SPaolo Bonzini ret = kvm_skip_emulated_instruction(vcpu);
873883b0a91SJoerg Roedel
8740dd16b5bSMaxim Levitsky vmcb12 = map.hva;
875883b0a91SJoerg Roedel
8762fcf4876SMaxim Levitsky if (WARN_ON_ONCE(!svm->nested.initialized))
8772fcf4876SMaxim Levitsky return -EINVAL;
8782fcf4876SMaxim Levitsky
8797907160dSEmanuele Giuseppe Esposito nested_copy_vmcb_control_to_cache(svm, &vmcb12->control);
880f2740a8dSEmanuele Giuseppe Esposito nested_copy_vmcb_save_to_cache(svm, &vmcb12->save);
881cb9b6a1bSPaolo Bonzini
882b7a3d8b6SEmanuele Giuseppe Esposito if (!nested_vmcb_check_save(vcpu) ||
883bd95926cSPaolo Bonzini !nested_vmcb_check_controls(vcpu)) {
8840dd16b5bSMaxim Levitsky vmcb12->control.exit_code = SVM_EXIT_ERR;
8850dd16b5bSMaxim Levitsky vmcb12->control.exit_code_hi = 0;
8860dd16b5bSMaxim Levitsky vmcb12->control.exit_info_1 = 0;
8870dd16b5bSMaxim Levitsky vmcb12->control.exit_info_2 = 0;
88869c9dfa2SPaolo Bonzini goto out;
889883b0a91SJoerg Roedel }
890883b0a91SJoerg Roedel
891883b0a91SJoerg Roedel /*
8924995a368SCathy Avery * Since vmcb01 is not in use, we can use it to store some of the L1
8934995a368SCathy Avery * state.
894883b0a91SJoerg Roedel */
895db663af4SMaxim Levitsky vmcb01->save.efer = vcpu->arch.efer;
896db663af4SMaxim Levitsky vmcb01->save.cr0 = kvm_read_cr0(vcpu);
897db663af4SMaxim Levitsky vmcb01->save.cr4 = vcpu->arch.cr4;
898db663af4SMaxim Levitsky vmcb01->save.rflags = kvm_get_rflags(vcpu);
899db663af4SMaxim Levitsky vmcb01->save.rip = kvm_rip_read(vcpu);
900883b0a91SJoerg Roedel
9014995a368SCathy Avery if (!npt_enabled)
902db663af4SMaxim Levitsky vmcb01->save.cr3 = kvm_read_cr3(vcpu);
903883b0a91SJoerg Roedel
904f74f9414SPaolo Bonzini svm->nested.nested_run_pending = 1;
905883b0a91SJoerg Roedel
906e85d3e7bSMaxim Levitsky if (enter_svm_guest_mode(vcpu, vmcb12_gpa, vmcb12, true))
90759cd9bc5SVitaly Kuznetsov goto out_exit_err;
90859cd9bc5SVitaly Kuznetsov
90959cd9bc5SVitaly Kuznetsov if (nested_svm_vmrun_msrpm(svm))
91059cd9bc5SVitaly Kuznetsov goto out;
91159cd9bc5SVitaly Kuznetsov
91259cd9bc5SVitaly Kuznetsov out_exit_err:
913ebdb3dbaSVitaly Kuznetsov svm->nested.nested_run_pending = 0;
914159fc6faSMaciej S. Szmigiero svm->nmi_l1_to_l2 = false;
9156ef88d6eSSean Christopherson svm->soft_int_injected = false;
916ebdb3dbaSVitaly Kuznetsov
917883b0a91SJoerg Roedel svm->vmcb->control.exit_code = SVM_EXIT_ERR;
918883b0a91SJoerg Roedel svm->vmcb->control.exit_code_hi = 0;
919883b0a91SJoerg Roedel svm->vmcb->control.exit_info_1 = 0;
920883b0a91SJoerg Roedel svm->vmcb->control.exit_info_2 = 0;
921883b0a91SJoerg Roedel
922883b0a91SJoerg Roedel nested_svm_vmexit(svm);
923883b0a91SJoerg Roedel
92469c9dfa2SPaolo Bonzini out:
92563129754SPaolo Bonzini kvm_vcpu_unmap(vcpu, &map, true);
92669c9dfa2SPaolo Bonzini
927883b0a91SJoerg Roedel return ret;
928883b0a91SJoerg Roedel }
929883b0a91SJoerg Roedel
9300a758290SVitaly Kuznetsov /* Copy state save area fields which are handled by VMRUN */
svm_copy_vmrun_state(struct vmcb_save_area * to_save,struct vmcb_save_area * from_save)9312bb16beaSVitaly Kuznetsov void svm_copy_vmrun_state(struct vmcb_save_area *to_save,
9322bb16beaSVitaly Kuznetsov struct vmcb_save_area *from_save)
9330a758290SVitaly Kuznetsov {
9340a758290SVitaly Kuznetsov to_save->es = from_save->es;
9350a758290SVitaly Kuznetsov to_save->cs = from_save->cs;
9360a758290SVitaly Kuznetsov to_save->ss = from_save->ss;
9370a758290SVitaly Kuznetsov to_save->ds = from_save->ds;
9380a758290SVitaly Kuznetsov to_save->gdtr = from_save->gdtr;
9390a758290SVitaly Kuznetsov to_save->idtr = from_save->idtr;
9400a758290SVitaly Kuznetsov to_save->rflags = from_save->rflags | X86_EFLAGS_FIXED;
9410a758290SVitaly Kuznetsov to_save->efer = from_save->efer;
9420a758290SVitaly Kuznetsov to_save->cr0 = from_save->cr0;
9430a758290SVitaly Kuznetsov to_save->cr3 = from_save->cr3;
9440a758290SVitaly Kuznetsov to_save->cr4 = from_save->cr4;
9450a758290SVitaly Kuznetsov to_save->rax = from_save->rax;
9460a758290SVitaly Kuznetsov to_save->rsp = from_save->rsp;
9470a758290SVitaly Kuznetsov to_save->rip = from_save->rip;
9480a758290SVitaly Kuznetsov to_save->cpl = 0;
9490a758290SVitaly Kuznetsov }
9500a758290SVitaly Kuznetsov
svm_copy_vmloadsave_state(struct vmcb * to_vmcb,struct vmcb * from_vmcb)9512bb16beaSVitaly Kuznetsov void svm_copy_vmloadsave_state(struct vmcb *to_vmcb, struct vmcb *from_vmcb)
952883b0a91SJoerg Roedel {
953883b0a91SJoerg Roedel to_vmcb->save.fs = from_vmcb->save.fs;
954883b0a91SJoerg Roedel to_vmcb->save.gs = from_vmcb->save.gs;
955883b0a91SJoerg Roedel to_vmcb->save.tr = from_vmcb->save.tr;
956883b0a91SJoerg Roedel to_vmcb->save.ldtr = from_vmcb->save.ldtr;
957883b0a91SJoerg Roedel to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
958883b0a91SJoerg Roedel to_vmcb->save.star = from_vmcb->save.star;
959883b0a91SJoerg Roedel to_vmcb->save.lstar = from_vmcb->save.lstar;
960883b0a91SJoerg Roedel to_vmcb->save.cstar = from_vmcb->save.cstar;
961883b0a91SJoerg Roedel to_vmcb->save.sfmask = from_vmcb->save.sfmask;
962883b0a91SJoerg Roedel to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
963883b0a91SJoerg Roedel to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
964883b0a91SJoerg Roedel to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
965883b0a91SJoerg Roedel }
966883b0a91SJoerg Roedel
nested_svm_vmexit(struct vcpu_svm * svm)967883b0a91SJoerg Roedel int nested_svm_vmexit(struct vcpu_svm *svm)
968883b0a91SJoerg Roedel {
96963129754SPaolo Bonzini struct kvm_vcpu *vcpu = &svm->vcpu;
970db663af4SMaxim Levitsky struct vmcb *vmcb01 = svm->vmcb01.ptr;
971db663af4SMaxim Levitsky struct vmcb *vmcb02 = svm->nested.vmcb02.ptr;
9720dd16b5bSMaxim Levitsky struct vmcb *vmcb12;
973883b0a91SJoerg Roedel struct kvm_host_map map;
97463129754SPaolo Bonzini int rc;
975883b0a91SJoerg Roedel
97663129754SPaolo Bonzini rc = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa), &map);
977883b0a91SJoerg Roedel if (rc) {
978883b0a91SJoerg Roedel if (rc == -EINVAL)
97963129754SPaolo Bonzini kvm_inject_gp(vcpu, 0);
980883b0a91SJoerg Roedel return 1;
981883b0a91SJoerg Roedel }
982883b0a91SJoerg Roedel
9830dd16b5bSMaxim Levitsky vmcb12 = map.hva;
984883b0a91SJoerg Roedel
985883b0a91SJoerg Roedel /* Exit Guest-Mode */
98663129754SPaolo Bonzini leave_guest_mode(vcpu);
9870dd16b5bSMaxim Levitsky svm->nested.vmcb12_gpa = 0;
9882d8a42beSPaolo Bonzini WARN_ON_ONCE(svm->nested.nested_run_pending);
989883b0a91SJoerg Roedel
99063129754SPaolo Bonzini kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
991f2c7ef3bSMaxim Levitsky
99238c0b192SPaolo Bonzini /* in case we halted in L2 */
99338c0b192SPaolo Bonzini svm->vcpu.arch.mp_state = KVM_MP_STATE_RUNNABLE;
99438c0b192SPaolo Bonzini
995883b0a91SJoerg Roedel /* Give the current vmcb to the guest */
996883b0a91SJoerg Roedel
997db663af4SMaxim Levitsky vmcb12->save.es = vmcb02->save.es;
998db663af4SMaxim Levitsky vmcb12->save.cs = vmcb02->save.cs;
999db663af4SMaxim Levitsky vmcb12->save.ss = vmcb02->save.ss;
1000db663af4SMaxim Levitsky vmcb12->save.ds = vmcb02->save.ds;
1001db663af4SMaxim Levitsky vmcb12->save.gdtr = vmcb02->save.gdtr;
1002db663af4SMaxim Levitsky vmcb12->save.idtr = vmcb02->save.idtr;
10030dd16b5bSMaxim Levitsky vmcb12->save.efer = svm->vcpu.arch.efer;
100463129754SPaolo Bonzini vmcb12->save.cr0 = kvm_read_cr0(vcpu);
100563129754SPaolo Bonzini vmcb12->save.cr3 = kvm_read_cr3(vcpu);
1006db663af4SMaxim Levitsky vmcb12->save.cr2 = vmcb02->save.cr2;
10070dd16b5bSMaxim Levitsky vmcb12->save.cr4 = svm->vcpu.arch.cr4;
100863129754SPaolo Bonzini vmcb12->save.rflags = kvm_get_rflags(vcpu);
100963129754SPaolo Bonzini vmcb12->save.rip = kvm_rip_read(vcpu);
101063129754SPaolo Bonzini vmcb12->save.rsp = kvm_rsp_read(vcpu);
101163129754SPaolo Bonzini vmcb12->save.rax = kvm_rax_read(vcpu);
1012db663af4SMaxim Levitsky vmcb12->save.dr7 = vmcb02->save.dr7;
10130dd16b5bSMaxim Levitsky vmcb12->save.dr6 = svm->vcpu.arch.dr6;
1014db663af4SMaxim Levitsky vmcb12->save.cpl = vmcb02->save.cpl;
1015883b0a91SJoerg Roedel
1016db663af4SMaxim Levitsky vmcb12->control.int_state = vmcb02->control.int_state;
1017db663af4SMaxim Levitsky vmcb12->control.exit_code = vmcb02->control.exit_code;
1018db663af4SMaxim Levitsky vmcb12->control.exit_code_hi = vmcb02->control.exit_code_hi;
1019db663af4SMaxim Levitsky vmcb12->control.exit_info_1 = vmcb02->control.exit_info_1;
1020db663af4SMaxim Levitsky vmcb12->control.exit_info_2 = vmcb02->control.exit_info_2;
102136e2e983SPaolo Bonzini
10220dd16b5bSMaxim Levitsky if (vmcb12->control.exit_code != SVM_EXIT_ERR)
10239e8f0fbfSPaolo Bonzini nested_save_pending_event_to_vmcb12(svm, vmcb12);
1024883b0a91SJoerg Roedel
10257a6a6a3bSSean Christopherson if (guest_can_use(vcpu, X86_FEATURE_NRIPS))
1026db663af4SMaxim Levitsky vmcb12->control.next_rip = vmcb02->control.next_rip;
1027883b0a91SJoerg Roedel
10280dd16b5bSMaxim Levitsky vmcb12->control.int_ctl = svm->nested.ctl.int_ctl;
10290dd16b5bSMaxim Levitsky vmcb12->control.event_inj = svm->nested.ctl.event_inj;
10300dd16b5bSMaxim Levitsky vmcb12->control.event_inj_err = svm->nested.ctl.event_inj_err;
1031883b0a91SJoerg Roedel
1032e3cdaab5SPaolo Bonzini if (!kvm_pause_in_guest(vcpu->kvm)) {
103374fd41edSMaxim Levitsky vmcb01->control.pause_filter_count = vmcb02->control.pause_filter_count;
1034e3cdaab5SPaolo Bonzini vmcb_mark_dirty(vmcb01, VMCB_INTERCEPTS);
1035e3cdaab5SPaolo Bonzini
1036e3cdaab5SPaolo Bonzini }
103774fd41edSMaxim Levitsky
1038d00b99c5SBabu Moger nested_svm_copy_common_state(svm->nested.vmcb02.ptr, svm->vmcb01.ptr);
1039d00b99c5SBabu Moger
10404995a368SCathy Avery svm_switch_vmcb(svm, &svm->vmcb01);
10414995a368SCathy Avery
10425d1ec456SMaxim Levitsky /*
10435d1ec456SMaxim Levitsky * Rules for synchronizing int_ctl bits from vmcb02 to vmcb01:
10445d1ec456SMaxim Levitsky *
10455d1ec456SMaxim Levitsky * V_IRQ, V_IRQ_VECTOR, V_INTR_PRIO_MASK, V_IGN_TPR: If L1 doesn't
10465d1ec456SMaxim Levitsky * intercept interrupts, then KVM will use vmcb02's V_IRQ (and related
10475d1ec456SMaxim Levitsky * flags) to detect interrupt windows for L1 IRQs (even if L1 uses
10485d1ec456SMaxim Levitsky * virtual interrupt masking). Raise KVM_REQ_EVENT to ensure that
10495d1ec456SMaxim Levitsky * KVM re-requests an interrupt window if necessary, which implicitly
10505d1ec456SMaxim Levitsky * copies this bits from vmcb02 to vmcb01.
10515d1ec456SMaxim Levitsky *
10525d1ec456SMaxim Levitsky * V_TPR: If L1 doesn't use virtual interrupt masking, then L1's vTPR
10535d1ec456SMaxim Levitsky * is stored in vmcb02, but its value doesn't need to be copied from/to
10545d1ec456SMaxim Levitsky * vmcb01 because it is copied from/to the virtual APIC's TPR register
10555d1ec456SMaxim Levitsky * on each VM entry/exit.
10565d1ec456SMaxim Levitsky *
10575d1ec456SMaxim Levitsky * V_GIF: If nested vGIF is not used, KVM uses vmcb02's V_GIF for L1's
10585d1ec456SMaxim Levitsky * V_GIF. However, GIF is architecturally clear on each VM exit, thus
10595d1ec456SMaxim Levitsky * there is no need to copy V_GIF from vmcb02 to vmcb01.
10605d1ec456SMaxim Levitsky */
10615d1ec456SMaxim Levitsky if (!nested_exit_on_intr(svm))
10625d1ec456SMaxim Levitsky kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
10635d1ec456SMaxim Levitsky
1064e183d17aSSean Christopherson if (unlikely(guest_can_use(vcpu, X86_FEATURE_LBRV) &&
1065e183d17aSSean Christopherson (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK))) {
1066d20c796cSMaxim Levitsky svm_copy_lbrs(vmcb12, vmcb02);
1067d20c796cSMaxim Levitsky svm_update_lbrv(vcpu);
1068d20c796cSMaxim Levitsky } else if (unlikely(vmcb01->control.virt_ext & LBR_CTL_ENABLE_MASK)) {
10691d5a1b58SMaxim Levitsky svm_copy_lbrs(vmcb01, vmcb02);
10701d5a1b58SMaxim Levitsky svm_update_lbrv(vcpu);
10711d5a1b58SMaxim Levitsky }
10721d5a1b58SMaxim Levitsky
10730977cfacSSantosh Shukla if (vnmi) {
10740977cfacSSantosh Shukla if (vmcb02->control.int_ctl & V_NMI_BLOCKING_MASK)
10750977cfacSSantosh Shukla vmcb01->control.int_ctl |= V_NMI_BLOCKING_MASK;
10760977cfacSSantosh Shukla else
10770977cfacSSantosh Shukla vmcb01->control.int_ctl &= ~V_NMI_BLOCKING_MASK;
10780977cfacSSantosh Shukla
10790977cfacSSantosh Shukla if (vcpu->arch.nmi_pending) {
10800977cfacSSantosh Shukla vcpu->arch.nmi_pending--;
10810977cfacSSantosh Shukla vmcb01->control.int_ctl |= V_NMI_PENDING_MASK;
10820977cfacSSantosh Shukla } else {
10830977cfacSSantosh Shukla vmcb01->control.int_ctl &= ~V_NMI_PENDING_MASK;
10840977cfacSSantosh Shukla }
10850977cfacSSantosh Shukla }
10860977cfacSSantosh Shukla
10874995a368SCathy Avery /*
10884995a368SCathy Avery * On vmexit the GIF is set to false and
10894995a368SCathy Avery * no event can be injected in L1.
10904995a368SCathy Avery */
10919883764aSMaxim Levitsky svm_set_gif(svm, false);
1092db663af4SMaxim Levitsky vmcb01->control.exit_int_info = 0;
10939883764aSMaxim Levitsky
10947ca62d13SPaolo Bonzini svm->vcpu.arch.tsc_offset = svm->vcpu.arch.l1_tsc_offset;
1095db663af4SMaxim Levitsky if (vmcb01->control.tsc_offset != svm->vcpu.arch.tsc_offset) {
1096db663af4SMaxim Levitsky vmcb01->control.tsc_offset = svm->vcpu.arch.tsc_offset;
1097db663af4SMaxim Levitsky vmcb_mark_dirty(vmcb01, VMCB_INTERCEPTS);
10987ca62d13SPaolo Bonzini }
109918fc6c55SPaolo Bonzini
11000c94e246SSean Christopherson if (kvm_caps.has_tsc_control &&
11010c94e246SSean Christopherson vcpu->arch.tsc_scaling_ratio != vcpu->arch.l1_tsc_scaling_ratio) {
11025228eb96SMaxim Levitsky vcpu->arch.tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio;
11032d636990SSean Christopherson svm_write_tsc_multiplier(vcpu);
11045228eb96SMaxim Levitsky }
11055228eb96SMaxim Levitsky
1106e670bf68SPaolo Bonzini svm->nested.ctl.nested_cr3 = 0;
1107883b0a91SJoerg Roedel
11084995a368SCathy Avery /*
11094995a368SCathy Avery * Restore processor state that had been saved in vmcb01
11104995a368SCathy Avery */
1111db663af4SMaxim Levitsky kvm_set_rflags(vcpu, vmcb01->save.rflags);
1112db663af4SMaxim Levitsky svm_set_efer(vcpu, vmcb01->save.efer);
1113db663af4SMaxim Levitsky svm_set_cr0(vcpu, vmcb01->save.cr0 | X86_CR0_PE);
1114db663af4SMaxim Levitsky svm_set_cr4(vcpu, vmcb01->save.cr4);
1115db663af4SMaxim Levitsky kvm_rax_write(vcpu, vmcb01->save.rax);
1116db663af4SMaxim Levitsky kvm_rsp_write(vcpu, vmcb01->save.rsp);
1117db663af4SMaxim Levitsky kvm_rip_write(vcpu, vmcb01->save.rip);
11184995a368SCathy Avery
11194995a368SCathy Avery svm->vcpu.arch.dr7 = DR7_FIXED_1;
11204995a368SCathy Avery kvm_update_dr7(&svm->vcpu);
1121883b0a91SJoerg Roedel
11220dd16b5bSMaxim Levitsky trace_kvm_nested_vmexit_inject(vmcb12->control.exit_code,
11230dd16b5bSMaxim Levitsky vmcb12->control.exit_info_1,
11240dd16b5bSMaxim Levitsky vmcb12->control.exit_info_2,
11250dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info,
11260dd16b5bSMaxim Levitsky vmcb12->control.exit_int_info_err,
112736e2e983SPaolo Bonzini KVM_ISA_SVM);
112836e2e983SPaolo Bonzini
112963129754SPaolo Bonzini kvm_vcpu_unmap(vcpu, &map, true);
1130883b0a91SJoerg Roedel
1131d2e56019SSean Christopherson nested_svm_transition_tlb_flush(vcpu);
1132d2e56019SSean Christopherson
113363129754SPaolo Bonzini nested_svm_uninit_mmu_context(vcpu);
1134bf7dea42SVitaly Kuznetsov
1135db663af4SMaxim Levitsky rc = nested_svm_load_cr3(vcpu, vmcb01->save.cr3, false, true);
1136d82aaef9SVitaly Kuznetsov if (rc)
1137d82aaef9SVitaly Kuznetsov return 1;
1138bf7dea42SVitaly Kuznetsov
1139883b0a91SJoerg Roedel /*
1140883b0a91SJoerg Roedel * Drop what we picked up for L2 via svm_complete_interrupts() so it
1141883b0a91SJoerg Roedel * doesn't end up in L1.
1142883b0a91SJoerg Roedel */
1143883b0a91SJoerg Roedel svm->vcpu.arch.nmi_injected = false;
114463129754SPaolo Bonzini kvm_clear_exception_queue(vcpu);
114563129754SPaolo Bonzini kvm_clear_interrupt_queue(vcpu);
1146883b0a91SJoerg Roedel
11479a7de6ecSKrish Sadhukhan /*
11489a7de6ecSKrish Sadhukhan * If we are here following the completion of a VMRUN that
11499a7de6ecSKrish Sadhukhan * is being single-stepped, queue the pending #DB intercept
11509a7de6ecSKrish Sadhukhan * right now so that it an be accounted for before we execute
11519a7de6ecSKrish Sadhukhan * L1's next instruction.
11529a7de6ecSKrish Sadhukhan */
1153db663af4SMaxim Levitsky if (unlikely(vmcb01->save.rflags & X86_EFLAGS_TF))
11549a7de6ecSKrish Sadhukhan kvm_queue_exception(&(svm->vcpu), DB_VECTOR);
11559a7de6ecSKrish Sadhukhan
1156f44509f8SMaxim Levitsky /*
1157f44509f8SMaxim Levitsky * Un-inhibit the AVIC right away, so that other vCPUs can start
1158f44509f8SMaxim Levitsky * to benefit from it right away.
1159f44509f8SMaxim Levitsky */
1160f44509f8SMaxim Levitsky if (kvm_apicv_activated(vcpu->kvm))
11612008fab3SSean Christopherson __kvm_vcpu_update_apicv(vcpu);
1162f44509f8SMaxim Levitsky
1163883b0a91SJoerg Roedel return 0;
1164883b0a91SJoerg Roedel }
1165883b0a91SJoerg Roedel
nested_svm_triple_fault(struct kvm_vcpu * vcpu)1166cb6a32c2SSean Christopherson static void nested_svm_triple_fault(struct kvm_vcpu *vcpu)
1167cb6a32c2SSean Christopherson {
116892e7d5c8SMaxim Levitsky struct vcpu_svm *svm = to_svm(vcpu);
116992e7d5c8SMaxim Levitsky
117092e7d5c8SMaxim Levitsky if (!vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SHUTDOWN))
117192e7d5c8SMaxim Levitsky return;
117292e7d5c8SMaxim Levitsky
117392e7d5c8SMaxim Levitsky kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu);
11743a87c7e0SSean Christopherson nested_svm_simple_vmexit(to_svm(vcpu), SVM_EXIT_SHUTDOWN);
1175cb6a32c2SSean Christopherson }
1176cb6a32c2SSean Christopherson
svm_allocate_nested(struct vcpu_svm * svm)11772fcf4876SMaxim Levitsky int svm_allocate_nested(struct vcpu_svm *svm)
11782fcf4876SMaxim Levitsky {
11794995a368SCathy Avery struct page *vmcb02_page;
11802fcf4876SMaxim Levitsky
11812fcf4876SMaxim Levitsky if (svm->nested.initialized)
11822fcf4876SMaxim Levitsky return 0;
11832fcf4876SMaxim Levitsky
1184*f51af346SLi RongQing vmcb02_page = snp_safe_alloc_page();
11854995a368SCathy Avery if (!vmcb02_page)
11862fcf4876SMaxim Levitsky return -ENOMEM;
11874995a368SCathy Avery svm->nested.vmcb02.ptr = page_address(vmcb02_page);
11884995a368SCathy Avery svm->nested.vmcb02.pa = __sme_set(page_to_pfn(vmcb02_page) << PAGE_SHIFT);
11892fcf4876SMaxim Levitsky
11902fcf4876SMaxim Levitsky svm->nested.msrpm = svm_vcpu_alloc_msrpm();
11912fcf4876SMaxim Levitsky if (!svm->nested.msrpm)
11924995a368SCathy Avery goto err_free_vmcb02;
11932fcf4876SMaxim Levitsky svm_vcpu_init_msrpm(&svm->vcpu, svm->nested.msrpm);
11942fcf4876SMaxim Levitsky
11952fcf4876SMaxim Levitsky svm->nested.initialized = true;
11962fcf4876SMaxim Levitsky return 0;
11972fcf4876SMaxim Levitsky
11984995a368SCathy Avery err_free_vmcb02:
11994995a368SCathy Avery __free_page(vmcb02_page);
12002fcf4876SMaxim Levitsky return -ENOMEM;
12012fcf4876SMaxim Levitsky }
12022fcf4876SMaxim Levitsky
svm_free_nested(struct vcpu_svm * svm)12032fcf4876SMaxim Levitsky void svm_free_nested(struct vcpu_svm *svm)
12042fcf4876SMaxim Levitsky {
12052fcf4876SMaxim Levitsky if (!svm->nested.initialized)
12062fcf4876SMaxim Levitsky return;
12072fcf4876SMaxim Levitsky
120816ae56d7SMaxim Levitsky if (WARN_ON_ONCE(svm->vmcb != svm->vmcb01.ptr))
120916ae56d7SMaxim Levitsky svm_switch_vmcb(svm, &svm->vmcb01);
121016ae56d7SMaxim Levitsky
12112fcf4876SMaxim Levitsky svm_vcpu_free_msrpm(svm->nested.msrpm);
12122fcf4876SMaxim Levitsky svm->nested.msrpm = NULL;
12132fcf4876SMaxim Levitsky
12144995a368SCathy Avery __free_page(virt_to_page(svm->nested.vmcb02.ptr));
12154995a368SCathy Avery svm->nested.vmcb02.ptr = NULL;
12162fcf4876SMaxim Levitsky
1217c74ad08fSMaxim Levitsky /*
1218c74ad08fSMaxim Levitsky * When last_vmcb12_gpa matches the current vmcb12 gpa,
1219c74ad08fSMaxim Levitsky * some vmcb12 fields are not loaded if they are marked clean
1220c74ad08fSMaxim Levitsky * in the vmcb12, since in this case they are up to date already.
1221c74ad08fSMaxim Levitsky *
1222c74ad08fSMaxim Levitsky * When the vmcb02 is freed, this optimization becomes invalid.
1223c74ad08fSMaxim Levitsky */
1224c74ad08fSMaxim Levitsky svm->nested.last_vmcb12_gpa = INVALID_GPA;
1225c74ad08fSMaxim Levitsky
12262fcf4876SMaxim Levitsky svm->nested.initialized = false;
12272fcf4876SMaxim Levitsky }
12282fcf4876SMaxim Levitsky
svm_leave_nested(struct kvm_vcpu * vcpu)1229f7e57078SSean Christopherson void svm_leave_nested(struct kvm_vcpu *vcpu)
1230c513f484SPaolo Bonzini {
1231f7e57078SSean Christopherson struct vcpu_svm *svm = to_svm(vcpu);
123263129754SPaolo Bonzini
123363129754SPaolo Bonzini if (is_guest_mode(vcpu)) {
1234c513f484SPaolo Bonzini svm->nested.nested_run_pending = 0;
1235c74ad08fSMaxim Levitsky svm->nested.vmcb12_gpa = INVALID_GPA;
1236c74ad08fSMaxim Levitsky
123763129754SPaolo Bonzini leave_guest_mode(vcpu);
12384995a368SCathy Avery
1239deee59baSMaxim Levitsky svm_switch_vmcb(svm, &svm->vmcb01);
12404995a368SCathy Avery
124163129754SPaolo Bonzini nested_svm_uninit_mmu_context(vcpu);
124256fe28deSMaxim Levitsky vmcb_mark_all_dirty(svm->vmcb);
12433fdc6087SMaxim Levitsky
12443fdc6087SMaxim Levitsky if (kvm_apicv_activated(vcpu->kvm))
12453fdc6087SMaxim Levitsky kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);
1246c513f484SPaolo Bonzini }
1247a7d5c7ceSPaolo Bonzini
124863129754SPaolo Bonzini kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
1249c513f484SPaolo Bonzini }
1250c513f484SPaolo Bonzini
nested_svm_exit_handled_msr(struct vcpu_svm * svm)1251883b0a91SJoerg Roedel static int nested_svm_exit_handled_msr(struct vcpu_svm *svm)
1252883b0a91SJoerg Roedel {
1253883b0a91SJoerg Roedel u32 offset, msr, value;
1254883b0a91SJoerg Roedel int write, mask;
1255883b0a91SJoerg Roedel
12568fc78909SEmanuele Giuseppe Esposito if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT)))
1257883b0a91SJoerg Roedel return NESTED_EXIT_HOST;
1258883b0a91SJoerg Roedel
1259883b0a91SJoerg Roedel msr = svm->vcpu.arch.regs[VCPU_REGS_RCX];
1260883b0a91SJoerg Roedel offset = svm_msrpm_offset(msr);
1261883b0a91SJoerg Roedel write = svm->vmcb->control.exit_info_1 & 1;
1262883b0a91SJoerg Roedel mask = 1 << ((2 * (msr & 0xf)) + write);
1263883b0a91SJoerg Roedel
1264883b0a91SJoerg Roedel if (offset == MSR_INVALID)
1265883b0a91SJoerg Roedel return NESTED_EXIT_DONE;
1266883b0a91SJoerg Roedel
1267883b0a91SJoerg Roedel /* Offset is in 32 bit units but need in 8 bit units */
1268883b0a91SJoerg Roedel offset *= 4;
1269883b0a91SJoerg Roedel
1270e670bf68SPaolo Bonzini if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.ctl.msrpm_base_pa + offset, &value, 4))
1271883b0a91SJoerg Roedel return NESTED_EXIT_DONE;
1272883b0a91SJoerg Roedel
1273883b0a91SJoerg Roedel return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
1274883b0a91SJoerg Roedel }
1275883b0a91SJoerg Roedel
nested_svm_intercept_ioio(struct vcpu_svm * svm)1276883b0a91SJoerg Roedel static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
1277883b0a91SJoerg Roedel {
1278883b0a91SJoerg Roedel unsigned port, size, iopm_len;
1279883b0a91SJoerg Roedel u16 val, mask;
1280883b0a91SJoerg Roedel u8 start_bit;
1281883b0a91SJoerg Roedel u64 gpa;
1282883b0a91SJoerg Roedel
12838fc78909SEmanuele Giuseppe Esposito if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_IOIO_PROT)))
1284883b0a91SJoerg Roedel return NESTED_EXIT_HOST;
1285883b0a91SJoerg Roedel
1286883b0a91SJoerg Roedel port = svm->vmcb->control.exit_info_1 >> 16;
1287883b0a91SJoerg Roedel size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >>
1288883b0a91SJoerg Roedel SVM_IOIO_SIZE_SHIFT;
1289e670bf68SPaolo Bonzini gpa = svm->nested.ctl.iopm_base_pa + (port / 8);
1290883b0a91SJoerg Roedel start_bit = port % 8;
1291883b0a91SJoerg Roedel iopm_len = (start_bit + size > 8) ? 2 : 1;
1292883b0a91SJoerg Roedel mask = (0xf >> (4 - size)) << start_bit;
1293883b0a91SJoerg Roedel val = 0;
1294883b0a91SJoerg Roedel
1295883b0a91SJoerg Roedel if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len))
1296883b0a91SJoerg Roedel return NESTED_EXIT_DONE;
1297883b0a91SJoerg Roedel
1298883b0a91SJoerg Roedel return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
1299883b0a91SJoerg Roedel }
1300883b0a91SJoerg Roedel
nested_svm_intercept(struct vcpu_svm * svm)1301883b0a91SJoerg Roedel static int nested_svm_intercept(struct vcpu_svm *svm)
1302883b0a91SJoerg Roedel {
1303883b0a91SJoerg Roedel u32 exit_code = svm->vmcb->control.exit_code;
1304883b0a91SJoerg Roedel int vmexit = NESTED_EXIT_HOST;
1305883b0a91SJoerg Roedel
1306883b0a91SJoerg Roedel switch (exit_code) {
1307883b0a91SJoerg Roedel case SVM_EXIT_MSR:
1308883b0a91SJoerg Roedel vmexit = nested_svm_exit_handled_msr(svm);
1309883b0a91SJoerg Roedel break;
1310883b0a91SJoerg Roedel case SVM_EXIT_IOIO:
1311883b0a91SJoerg Roedel vmexit = nested_svm_intercept_ioio(svm);
1312883b0a91SJoerg Roedel break;
1313883b0a91SJoerg Roedel case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: {
13148fc78909SEmanuele Giuseppe Esposito if (vmcb12_is_intercept(&svm->nested.ctl, exit_code))
1315883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE;
1316883b0a91SJoerg Roedel break;
1317883b0a91SJoerg Roedel }
1318883b0a91SJoerg Roedel case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: {
13198fc78909SEmanuele Giuseppe Esposito if (vmcb12_is_intercept(&svm->nested.ctl, exit_code))
1320883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE;
1321883b0a91SJoerg Roedel break;
1322883b0a91SJoerg Roedel }
1323883b0a91SJoerg Roedel case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
13247c86663bSPaolo Bonzini /*
13257c86663bSPaolo Bonzini * Host-intercepted exceptions have been checked already in
13267c86663bSPaolo Bonzini * nested_svm_exit_special. There is nothing to do here,
13277c86663bSPaolo Bonzini * the vmexit is injected by svm_check_nested_events.
13287c86663bSPaolo Bonzini */
1329883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE;
1330883b0a91SJoerg Roedel break;
1331883b0a91SJoerg Roedel }
1332883b0a91SJoerg Roedel case SVM_EXIT_ERR: {
1333883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE;
1334883b0a91SJoerg Roedel break;
1335883b0a91SJoerg Roedel }
1336883b0a91SJoerg Roedel default: {
13378fc78909SEmanuele Giuseppe Esposito if (vmcb12_is_intercept(&svm->nested.ctl, exit_code))
1338883b0a91SJoerg Roedel vmexit = NESTED_EXIT_DONE;
1339883b0a91SJoerg Roedel }
1340883b0a91SJoerg Roedel }
1341883b0a91SJoerg Roedel
1342883b0a91SJoerg Roedel return vmexit;
1343883b0a91SJoerg Roedel }
1344883b0a91SJoerg Roedel
nested_svm_exit_handled(struct vcpu_svm * svm)1345883b0a91SJoerg Roedel int nested_svm_exit_handled(struct vcpu_svm *svm)
1346883b0a91SJoerg Roedel {
1347883b0a91SJoerg Roedel int vmexit;
1348883b0a91SJoerg Roedel
1349883b0a91SJoerg Roedel vmexit = nested_svm_intercept(svm);
1350883b0a91SJoerg Roedel
1351883b0a91SJoerg Roedel if (vmexit == NESTED_EXIT_DONE)
1352883b0a91SJoerg Roedel nested_svm_vmexit(svm);
1353883b0a91SJoerg Roedel
1354883b0a91SJoerg Roedel return vmexit;
1355883b0a91SJoerg Roedel }
1356883b0a91SJoerg Roedel
nested_svm_check_permissions(struct kvm_vcpu * vcpu)135763129754SPaolo Bonzini int nested_svm_check_permissions(struct kvm_vcpu *vcpu)
1358883b0a91SJoerg Roedel {
135963129754SPaolo Bonzini if (!(vcpu->arch.efer & EFER_SVME) || !is_paging(vcpu)) {
136063129754SPaolo Bonzini kvm_queue_exception(vcpu, UD_VECTOR);
1361883b0a91SJoerg Roedel return 1;
1362883b0a91SJoerg Roedel }
1363883b0a91SJoerg Roedel
136463129754SPaolo Bonzini if (to_svm(vcpu)->vmcb->save.cpl) {
136563129754SPaolo Bonzini kvm_inject_gp(vcpu, 0);
1366883b0a91SJoerg Roedel return 1;
1367883b0a91SJoerg Roedel }
1368883b0a91SJoerg Roedel
1369883b0a91SJoerg Roedel return 0;
1370883b0a91SJoerg Roedel }
1371883b0a91SJoerg Roedel
nested_svm_is_exception_vmexit(struct kvm_vcpu * vcpu,u8 vector,u32 error_code)13727709aba8SSean Christopherson static bool nested_svm_is_exception_vmexit(struct kvm_vcpu *vcpu, u8 vector,
13737709aba8SSean Christopherson u32 error_code)
1374883b0a91SJoerg Roedel {
13757709aba8SSean Christopherson struct vcpu_svm *svm = to_svm(vcpu);
1376883b0a91SJoerg Roedel
1377d4963e31SSean Christopherson return (svm->nested.ctl.intercepts[INTERCEPT_EXCEPTION] & BIT(vector));
13787c86663bSPaolo Bonzini }
1379883b0a91SJoerg Roedel
nested_svm_inject_exception_vmexit(struct kvm_vcpu * vcpu)1380d4963e31SSean Christopherson static void nested_svm_inject_exception_vmexit(struct kvm_vcpu *vcpu)
13817c86663bSPaolo Bonzini {
13827709aba8SSean Christopherson struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit;
1383d4963e31SSean Christopherson struct vcpu_svm *svm = to_svm(vcpu);
1384db663af4SMaxim Levitsky struct vmcb *vmcb = svm->vmcb;
1385883b0a91SJoerg Roedel
1386d4963e31SSean Christopherson vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + ex->vector;
1387db663af4SMaxim Levitsky vmcb->control.exit_code_hi = 0;
13887c86663bSPaolo Bonzini
1389d4963e31SSean Christopherson if (ex->has_error_code)
1390d4963e31SSean Christopherson vmcb->control.exit_info_1 = ex->error_code;
1391883b0a91SJoerg Roedel
1392883b0a91SJoerg Roedel /*
1393883b0a91SJoerg Roedel * EXITINFO2 is undefined for all exception intercepts other
1394883b0a91SJoerg Roedel * than #PF.
1395883b0a91SJoerg Roedel */
1396d4963e31SSean Christopherson if (ex->vector == PF_VECTOR) {
13977709aba8SSean Christopherson if (ex->has_payload)
1398d4963e31SSean Christopherson vmcb->control.exit_info_2 = ex->payload;
1399883b0a91SJoerg Roedel else
1400d4963e31SSean Christopherson vmcb->control.exit_info_2 = vcpu->arch.cr2;
1401d4963e31SSean Christopherson } else if (ex->vector == DB_VECTOR) {
1402e746c1f1SSean Christopherson /* See kvm_check_and_inject_events(). */
1403d4963e31SSean Christopherson kvm_deliver_exception_payload(vcpu, ex);
1404d4963e31SSean Christopherson
1405d4963e31SSean Christopherson if (vcpu->arch.dr7 & DR7_GD) {
1406d4963e31SSean Christopherson vcpu->arch.dr7 &= ~DR7_GD;
1407d4963e31SSean Christopherson kvm_update_dr7(vcpu);
14087c86663bSPaolo Bonzini }
1409d4963e31SSean Christopherson } else {
1410d4963e31SSean Christopherson WARN_ON(ex->has_payload);
1411d4963e31SSean Christopherson }
1412883b0a91SJoerg Roedel
14137c86663bSPaolo Bonzini nested_svm_vmexit(svm);
1414883b0a91SJoerg Roedel }
1415883b0a91SJoerg Roedel
nested_exit_on_init(struct vcpu_svm * svm)14165b672408SPaolo Bonzini static inline bool nested_exit_on_init(struct vcpu_svm *svm)
14175b672408SPaolo Bonzini {
14188fc78909SEmanuele Giuseppe Esposito return vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_INIT);
14195b672408SPaolo Bonzini }
14205b672408SPaolo Bonzini
svm_check_nested_events(struct kvm_vcpu * vcpu)142133b22172SPaolo Bonzini static int svm_check_nested_events(struct kvm_vcpu *vcpu)
1422883b0a91SJoerg Roedel {
14235b672408SPaolo Bonzini struct kvm_lapic *apic = vcpu->arch.apic;
142472c14e00SSean Christopherson struct vcpu_svm *svm = to_svm(vcpu);
142572c14e00SSean Christopherson /*
142672c14e00SSean Christopherson * Only a pending nested run blocks a pending exception. If there is a
142772c14e00SSean Christopherson * previously injected event, the pending exception occurred while said
142872c14e00SSean Christopherson * event was being delivered and thus needs to be handled.
142972c14e00SSean Christopherson */
143072c14e00SSean Christopherson bool block_nested_exceptions = svm->nested.nested_run_pending;
143172c14e00SSean Christopherson /*
143272c14e00SSean Christopherson * New events (not exceptions) are only recognized at instruction
143372c14e00SSean Christopherson * boundaries. If an event needs reinjection, then KVM is handling a
143472c14e00SSean Christopherson * VM-Exit that occurred _during_ instruction execution; new events are
143572c14e00SSean Christopherson * blocked until the instruction completes.
143672c14e00SSean Christopherson */
143772c14e00SSean Christopherson bool block_nested_events = block_nested_exceptions ||
143872c14e00SSean Christopherson kvm_event_needs_reinjection(vcpu);
14395b672408SPaolo Bonzini
14405b672408SPaolo Bonzini if (lapic_in_kernel(vcpu) &&
14415b672408SPaolo Bonzini test_bit(KVM_APIC_INIT, &apic->pending_events)) {
14425b672408SPaolo Bonzini if (block_nested_events)
14435b672408SPaolo Bonzini return -EBUSY;
14445b672408SPaolo Bonzini if (!nested_exit_on_init(svm))
14455b672408SPaolo Bonzini return 0;
14463a87c7e0SSean Christopherson nested_svm_simple_vmexit(svm, SVM_EXIT_INIT);
14475b672408SPaolo Bonzini return 0;
14485b672408SPaolo Bonzini }
1449883b0a91SJoerg Roedel
14507709aba8SSean Christopherson if (vcpu->arch.exception_vmexit.pending) {
14517709aba8SSean Christopherson if (block_nested_exceptions)
14527709aba8SSean Christopherson return -EBUSY;
14537709aba8SSean Christopherson nested_svm_inject_exception_vmexit(vcpu);
14547709aba8SSean Christopherson return 0;
14557709aba8SSean Christopherson }
14567709aba8SSean Christopherson
14577c86663bSPaolo Bonzini if (vcpu->arch.exception.pending) {
145872c14e00SSean Christopherson if (block_nested_exceptions)
14597c86663bSPaolo Bonzini return -EBUSY;
14607c86663bSPaolo Bonzini return 0;
14617c86663bSPaolo Bonzini }
14627c86663bSPaolo Bonzini
146331e83e21SPaolo Bonzini #ifdef CONFIG_KVM_SMM
1464221e7610SPaolo Bonzini if (vcpu->arch.smi_pending && !svm_smi_blocked(vcpu)) {
146555714cddSPaolo Bonzini if (block_nested_events)
146655714cddSPaolo Bonzini return -EBUSY;
1467221e7610SPaolo Bonzini if (!nested_exit_on_smi(svm))
1468221e7610SPaolo Bonzini return 0;
14693a87c7e0SSean Christopherson nested_svm_simple_vmexit(svm, SVM_EXIT_SMI);
147055714cddSPaolo Bonzini return 0;
147155714cddSPaolo Bonzini }
147231e83e21SPaolo Bonzini #endif
147355714cddSPaolo Bonzini
1474221e7610SPaolo Bonzini if (vcpu->arch.nmi_pending && !svm_nmi_blocked(vcpu)) {
14759c3d370aSCathy Avery if (block_nested_events)
14769c3d370aSCathy Avery return -EBUSY;
1477221e7610SPaolo Bonzini if (!nested_exit_on_nmi(svm))
1478221e7610SPaolo Bonzini return 0;
14793a87c7e0SSean Christopherson nested_svm_simple_vmexit(svm, SVM_EXIT_NMI);
14809c3d370aSCathy Avery return 0;
14819c3d370aSCathy Avery }
14829c3d370aSCathy Avery
1483221e7610SPaolo Bonzini if (kvm_cpu_has_interrupt(vcpu) && !svm_interrupt_blocked(vcpu)) {
1484883b0a91SJoerg Roedel if (block_nested_events)
1485883b0a91SJoerg Roedel return -EBUSY;
1486221e7610SPaolo Bonzini if (!nested_exit_on_intr(svm))
1487221e7610SPaolo Bonzini return 0;
14883a87c7e0SSean Christopherson trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
14893a87c7e0SSean Christopherson nested_svm_simple_vmexit(svm, SVM_EXIT_INTR);
1490883b0a91SJoerg Roedel return 0;
1491883b0a91SJoerg Roedel }
1492883b0a91SJoerg Roedel
1493883b0a91SJoerg Roedel return 0;
1494883b0a91SJoerg Roedel }
1495883b0a91SJoerg Roedel
nested_svm_exit_special(struct vcpu_svm * svm)1496883b0a91SJoerg Roedel int nested_svm_exit_special(struct vcpu_svm *svm)
1497883b0a91SJoerg Roedel {
1498883b0a91SJoerg Roedel u32 exit_code = svm->vmcb->control.exit_code;
14993f4a812eSVitaly Kuznetsov struct kvm_vcpu *vcpu = &svm->vcpu;
1500883b0a91SJoerg Roedel
1501883b0a91SJoerg Roedel switch (exit_code) {
1502883b0a91SJoerg Roedel case SVM_EXIT_INTR:
1503883b0a91SJoerg Roedel case SVM_EXIT_NMI:
1504883b0a91SJoerg Roedel case SVM_EXIT_NPF:
1505883b0a91SJoerg Roedel return NESTED_EXIT_HOST;
15067c86663bSPaolo Bonzini case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
15077c86663bSPaolo Bonzini u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
15087c86663bSPaolo Bonzini
15094995a368SCathy Avery if (svm->vmcb01.ptr->control.intercepts[INTERCEPT_EXCEPTION] &
15109780d51dSBabu Moger excp_bits)
15117c86663bSPaolo Bonzini return NESTED_EXIT_HOST;
15127c86663bSPaolo Bonzini else if (exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR &&
151368fd66f1SVitaly Kuznetsov svm->vcpu.arch.apf.host_apf_flags)
1514a3535be7SPaolo Bonzini /* Trap async PF even if not shadowing */
1515883b0a91SJoerg Roedel return NESTED_EXIT_HOST;
1516883b0a91SJoerg Roedel break;
15177c86663bSPaolo Bonzini }
15183f4a812eSVitaly Kuznetsov case SVM_EXIT_VMMCALL:
15193f4a812eSVitaly Kuznetsov /* Hyper-V L2 TLB flush hypercall is handled by L0 */
15203f4a812eSVitaly Kuznetsov if (guest_hv_cpuid_has_l2_tlb_flush(vcpu) &&
15213f4a812eSVitaly Kuznetsov nested_svm_l2_tlb_flush_enabled(vcpu) &&
15223f4a812eSVitaly Kuznetsov kvm_hv_is_tlb_flush_hcall(vcpu))
15233f4a812eSVitaly Kuznetsov return NESTED_EXIT_HOST;
15243f4a812eSVitaly Kuznetsov break;
1525883b0a91SJoerg Roedel default:
1526883b0a91SJoerg Roedel break;
1527883b0a91SJoerg Roedel }
1528883b0a91SJoerg Roedel
1529883b0a91SJoerg Roedel return NESTED_EXIT_CONTINUE;
1530883b0a91SJoerg Roedel }
153133b22172SPaolo Bonzini
nested_svm_update_tsc_ratio_msr(struct kvm_vcpu * vcpu)15325228eb96SMaxim Levitsky void nested_svm_update_tsc_ratio_msr(struct kvm_vcpu *vcpu)
15335228eb96SMaxim Levitsky {
15345228eb96SMaxim Levitsky struct vcpu_svm *svm = to_svm(vcpu);
15355228eb96SMaxim Levitsky
15365228eb96SMaxim Levitsky vcpu->arch.tsc_scaling_ratio =
15375228eb96SMaxim Levitsky kvm_calc_nested_tsc_multiplier(vcpu->arch.l1_tsc_scaling_ratio,
15385228eb96SMaxim Levitsky svm->tsc_ratio_msr);
15392d636990SSean Christopherson svm_write_tsc_multiplier(vcpu);
15405228eb96SMaxim Levitsky }
15415228eb96SMaxim Levitsky
15428fc78909SEmanuele Giuseppe Esposito /* Inverse operation of nested_copy_vmcb_control_to_cache(). asid is copied too. */
nested_copy_vmcb_cache_to_control(struct vmcb_control_area * dst,struct vmcb_ctrl_area_cached * from)15438fc78909SEmanuele Giuseppe Esposito static void nested_copy_vmcb_cache_to_control(struct vmcb_control_area *dst,
15448fc78909SEmanuele Giuseppe Esposito struct vmcb_ctrl_area_cached *from)
15458fc78909SEmanuele Giuseppe Esposito {
15468fc78909SEmanuele Giuseppe Esposito unsigned int i;
15478fc78909SEmanuele Giuseppe Esposito
15488fc78909SEmanuele Giuseppe Esposito memset(dst, 0, sizeof(struct vmcb_control_area));
15498fc78909SEmanuele Giuseppe Esposito
15508fc78909SEmanuele Giuseppe Esposito for (i = 0; i < MAX_INTERCEPT; i++)
15518fc78909SEmanuele Giuseppe Esposito dst->intercepts[i] = from->intercepts[i];
15528fc78909SEmanuele Giuseppe Esposito
15538fc78909SEmanuele Giuseppe Esposito dst->iopm_base_pa = from->iopm_base_pa;
15548fc78909SEmanuele Giuseppe Esposito dst->msrpm_base_pa = from->msrpm_base_pa;
15558fc78909SEmanuele Giuseppe Esposito dst->tsc_offset = from->tsc_offset;
15568fc78909SEmanuele Giuseppe Esposito dst->asid = from->asid;
15578fc78909SEmanuele Giuseppe Esposito dst->tlb_ctl = from->tlb_ctl;
15588fc78909SEmanuele Giuseppe Esposito dst->int_ctl = from->int_ctl;
15598fc78909SEmanuele Giuseppe Esposito dst->int_vector = from->int_vector;
15608fc78909SEmanuele Giuseppe Esposito dst->int_state = from->int_state;
15618fc78909SEmanuele Giuseppe Esposito dst->exit_code = from->exit_code;
15628fc78909SEmanuele Giuseppe Esposito dst->exit_code_hi = from->exit_code_hi;
15638fc78909SEmanuele Giuseppe Esposito dst->exit_info_1 = from->exit_info_1;
15648fc78909SEmanuele Giuseppe Esposito dst->exit_info_2 = from->exit_info_2;
15658fc78909SEmanuele Giuseppe Esposito dst->exit_int_info = from->exit_int_info;
15668fc78909SEmanuele Giuseppe Esposito dst->exit_int_info_err = from->exit_int_info_err;
15678fc78909SEmanuele Giuseppe Esposito dst->nested_ctl = from->nested_ctl;
15688fc78909SEmanuele Giuseppe Esposito dst->event_inj = from->event_inj;
15698fc78909SEmanuele Giuseppe Esposito dst->event_inj_err = from->event_inj_err;
157000f08d99SMaciej S. Szmigiero dst->next_rip = from->next_rip;
15718fc78909SEmanuele Giuseppe Esposito dst->nested_cr3 = from->nested_cr3;
15728fc78909SEmanuele Giuseppe Esposito dst->virt_ext = from->virt_ext;
15738fc78909SEmanuele Giuseppe Esposito dst->pause_filter_count = from->pause_filter_count;
15748fc78909SEmanuele Giuseppe Esposito dst->pause_filter_thresh = from->pause_filter_thresh;
157568ae7c7bSSean Christopherson /* 'clean' and 'hv_enlightenments' are not changed by KVM */
15768fc78909SEmanuele Giuseppe Esposito }
15778fc78909SEmanuele Giuseppe Esposito
svm_get_nested_state(struct kvm_vcpu * vcpu,struct kvm_nested_state __user * user_kvm_nested_state,u32 user_data_size)1578cc440cdaSPaolo Bonzini static int svm_get_nested_state(struct kvm_vcpu *vcpu,
1579cc440cdaSPaolo Bonzini struct kvm_nested_state __user *user_kvm_nested_state,
1580cc440cdaSPaolo Bonzini u32 user_data_size)
1581cc440cdaSPaolo Bonzini {
1582cc440cdaSPaolo Bonzini struct vcpu_svm *svm;
15838fc78909SEmanuele Giuseppe Esposito struct vmcb_control_area *ctl;
15848fc78909SEmanuele Giuseppe Esposito unsigned long r;
1585cc440cdaSPaolo Bonzini struct kvm_nested_state kvm_state = {
1586cc440cdaSPaolo Bonzini .flags = 0,
1587cc440cdaSPaolo Bonzini .format = KVM_STATE_NESTED_FORMAT_SVM,
1588cc440cdaSPaolo Bonzini .size = sizeof(kvm_state),
1589cc440cdaSPaolo Bonzini };
1590cc440cdaSPaolo Bonzini struct vmcb __user *user_vmcb = (struct vmcb __user *)
1591cc440cdaSPaolo Bonzini &user_kvm_nested_state->data.svm[0];
1592cc440cdaSPaolo Bonzini
1593cc440cdaSPaolo Bonzini if (!vcpu)
1594cc440cdaSPaolo Bonzini return kvm_state.size + KVM_STATE_NESTED_SVM_VMCB_SIZE;
1595cc440cdaSPaolo Bonzini
1596cc440cdaSPaolo Bonzini svm = to_svm(vcpu);
1597cc440cdaSPaolo Bonzini
1598cc440cdaSPaolo Bonzini if (user_data_size < kvm_state.size)
1599cc440cdaSPaolo Bonzini goto out;
1600cc440cdaSPaolo Bonzini
1601cc440cdaSPaolo Bonzini /* First fill in the header and copy it out. */
1602cc440cdaSPaolo Bonzini if (is_guest_mode(vcpu)) {
16030dd16b5bSMaxim Levitsky kvm_state.hdr.svm.vmcb_pa = svm->nested.vmcb12_gpa;
1604cc440cdaSPaolo Bonzini kvm_state.size += KVM_STATE_NESTED_SVM_VMCB_SIZE;
1605cc440cdaSPaolo Bonzini kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE;
1606cc440cdaSPaolo Bonzini
1607cc440cdaSPaolo Bonzini if (svm->nested.nested_run_pending)
1608cc440cdaSPaolo Bonzini kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING;
1609cc440cdaSPaolo Bonzini }
1610cc440cdaSPaolo Bonzini
1611cc440cdaSPaolo Bonzini if (gif_set(svm))
1612cc440cdaSPaolo Bonzini kvm_state.flags |= KVM_STATE_NESTED_GIF_SET;
1613cc440cdaSPaolo Bonzini
1614cc440cdaSPaolo Bonzini if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state)))
1615cc440cdaSPaolo Bonzini return -EFAULT;
1616cc440cdaSPaolo Bonzini
1617cc440cdaSPaolo Bonzini if (!is_guest_mode(vcpu))
1618cc440cdaSPaolo Bonzini goto out;
1619cc440cdaSPaolo Bonzini
1620cc440cdaSPaolo Bonzini /*
1621cc440cdaSPaolo Bonzini * Copy over the full size of the VMCB rather than just the size
1622cc440cdaSPaolo Bonzini * of the structs.
1623cc440cdaSPaolo Bonzini */
1624cc440cdaSPaolo Bonzini if (clear_user(user_vmcb, KVM_STATE_NESTED_SVM_VMCB_SIZE))
1625cc440cdaSPaolo Bonzini return -EFAULT;
16268fc78909SEmanuele Giuseppe Esposito
16278fc78909SEmanuele Giuseppe Esposito ctl = kzalloc(sizeof(*ctl), GFP_KERNEL);
16288fc78909SEmanuele Giuseppe Esposito if (!ctl)
16298fc78909SEmanuele Giuseppe Esposito return -ENOMEM;
16308fc78909SEmanuele Giuseppe Esposito
16318fc78909SEmanuele Giuseppe Esposito nested_copy_vmcb_cache_to_control(ctl, &svm->nested.ctl);
16328fc78909SEmanuele Giuseppe Esposito r = copy_to_user(&user_vmcb->control, ctl,
16338fc78909SEmanuele Giuseppe Esposito sizeof(user_vmcb->control));
16348fc78909SEmanuele Giuseppe Esposito kfree(ctl);
16358fc78909SEmanuele Giuseppe Esposito if (r)
1636cc440cdaSPaolo Bonzini return -EFAULT;
16378fc78909SEmanuele Giuseppe Esposito
16384995a368SCathy Avery if (copy_to_user(&user_vmcb->save, &svm->vmcb01.ptr->save,
1639cc440cdaSPaolo Bonzini sizeof(user_vmcb->save)))
1640cc440cdaSPaolo Bonzini return -EFAULT;
1641cc440cdaSPaolo Bonzini out:
1642cc440cdaSPaolo Bonzini return kvm_state.size;
1643cc440cdaSPaolo Bonzini }
1644cc440cdaSPaolo Bonzini
svm_set_nested_state(struct kvm_vcpu * vcpu,struct kvm_nested_state __user * user_kvm_nested_state,struct kvm_nested_state * kvm_state)1645cc440cdaSPaolo Bonzini static int svm_set_nested_state(struct kvm_vcpu *vcpu,
1646cc440cdaSPaolo Bonzini struct kvm_nested_state __user *user_kvm_nested_state,
1647cc440cdaSPaolo Bonzini struct kvm_nested_state *kvm_state)
1648cc440cdaSPaolo Bonzini {
1649cc440cdaSPaolo Bonzini struct vcpu_svm *svm = to_svm(vcpu);
1650cc440cdaSPaolo Bonzini struct vmcb __user *user_vmcb = (struct vmcb __user *)
1651cc440cdaSPaolo Bonzini &user_kvm_nested_state->data.svm[0];
16526ccbd29aSJoerg Roedel struct vmcb_control_area *ctl;
16536ccbd29aSJoerg Roedel struct vmcb_save_area *save;
1654b7a3d8b6SEmanuele Giuseppe Esposito struct vmcb_save_area_cached save_cached;
16558fc78909SEmanuele Giuseppe Esposito struct vmcb_ctrl_area_cached ctl_cached;
1656dbc4739bSSean Christopherson unsigned long cr0;
16576ccbd29aSJoerg Roedel int ret;
1658cc440cdaSPaolo Bonzini
16596ccbd29aSJoerg Roedel BUILD_BUG_ON(sizeof(struct vmcb_control_area) + sizeof(struct vmcb_save_area) >
16606ccbd29aSJoerg Roedel KVM_STATE_NESTED_SVM_VMCB_SIZE);
16616ccbd29aSJoerg Roedel
1662cc440cdaSPaolo Bonzini if (kvm_state->format != KVM_STATE_NESTED_FORMAT_SVM)
1663cc440cdaSPaolo Bonzini return -EINVAL;
1664cc440cdaSPaolo Bonzini
1665cc440cdaSPaolo Bonzini if (kvm_state->flags & ~(KVM_STATE_NESTED_GUEST_MODE |
1666cc440cdaSPaolo Bonzini KVM_STATE_NESTED_RUN_PENDING |
1667cc440cdaSPaolo Bonzini KVM_STATE_NESTED_GIF_SET))
1668cc440cdaSPaolo Bonzini return -EINVAL;
1669cc440cdaSPaolo Bonzini
1670cc440cdaSPaolo Bonzini /*
1671cc440cdaSPaolo Bonzini * If in guest mode, vcpu->arch.efer actually refers to the L2 guest's
1672cc440cdaSPaolo Bonzini * EFER.SVME, but EFER.SVME still has to be 1 for VMRUN to succeed.
1673cc440cdaSPaolo Bonzini */
1674cc440cdaSPaolo Bonzini if (!(vcpu->arch.efer & EFER_SVME)) {
1675cc440cdaSPaolo Bonzini /* GIF=1 and no guest mode are required if SVME=0. */
1676cc440cdaSPaolo Bonzini if (kvm_state->flags != KVM_STATE_NESTED_GIF_SET)
1677cc440cdaSPaolo Bonzini return -EINVAL;
1678cc440cdaSPaolo Bonzini }
1679cc440cdaSPaolo Bonzini
1680cc440cdaSPaolo Bonzini /* SMM temporarily disables SVM, so we cannot be in guest mode. */
1681cc440cdaSPaolo Bonzini if (is_smm(vcpu) && (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE))
1682cc440cdaSPaolo Bonzini return -EINVAL;
1683cc440cdaSPaolo Bonzini
1684cc440cdaSPaolo Bonzini if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) {
1685f7e57078SSean Christopherson svm_leave_nested(vcpu);
1686d5cd6f34SVitaly Kuznetsov svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET));
1687d5cd6f34SVitaly Kuznetsov return 0;
1688cc440cdaSPaolo Bonzini }
1689cc440cdaSPaolo Bonzini
1690cc440cdaSPaolo Bonzini if (!page_address_valid(vcpu, kvm_state->hdr.svm.vmcb_pa))
1691cc440cdaSPaolo Bonzini return -EINVAL;
1692cc440cdaSPaolo Bonzini if (kvm_state->size < sizeof(*kvm_state) + KVM_STATE_NESTED_SVM_VMCB_SIZE)
1693cc440cdaSPaolo Bonzini return -EINVAL;
1694cc440cdaSPaolo Bonzini
16956ccbd29aSJoerg Roedel ret = -ENOMEM;
1696eba04b20SSean Christopherson ctl = kzalloc(sizeof(*ctl), GFP_KERNEL_ACCOUNT);
1697eba04b20SSean Christopherson save = kzalloc(sizeof(*save), GFP_KERNEL_ACCOUNT);
16986ccbd29aSJoerg Roedel if (!ctl || !save)
16996ccbd29aSJoerg Roedel goto out_free;
17006ccbd29aSJoerg Roedel
17016ccbd29aSJoerg Roedel ret = -EFAULT;
17026ccbd29aSJoerg Roedel if (copy_from_user(ctl, &user_vmcb->control, sizeof(*ctl)))
17036ccbd29aSJoerg Roedel goto out_free;
17046ccbd29aSJoerg Roedel if (copy_from_user(save, &user_vmcb->save, sizeof(*save)))
17056ccbd29aSJoerg Roedel goto out_free;
17066ccbd29aSJoerg Roedel
17076ccbd29aSJoerg Roedel ret = -EINVAL;
170866c03a92SVitaly Kuznetsov __nested_copy_vmcb_control_to_cache(vcpu, &ctl_cached, ctl);
17098fc78909SEmanuele Giuseppe Esposito if (!__nested_vmcb_check_controls(vcpu, &ctl_cached))
17106ccbd29aSJoerg Roedel goto out_free;
1711cc440cdaSPaolo Bonzini
1712cc440cdaSPaolo Bonzini /*
1713cc440cdaSPaolo Bonzini * Processor state contains L2 state. Check that it is
1714cb9b6a1bSPaolo Bonzini * valid for guest mode (see nested_vmcb_check_save).
1715cc440cdaSPaolo Bonzini */
1716cc440cdaSPaolo Bonzini cr0 = kvm_read_cr0(vcpu);
1717cc440cdaSPaolo Bonzini if (((cr0 & X86_CR0_CD) == 0) && (cr0 & X86_CR0_NW))
17186ccbd29aSJoerg Roedel goto out_free;
1719cc440cdaSPaolo Bonzini
1720cc440cdaSPaolo Bonzini /*
1721cc440cdaSPaolo Bonzini * Validate host state saved from before VMRUN (see
1722cc440cdaSPaolo Bonzini * nested_svm_check_permissions).
1723cc440cdaSPaolo Bonzini */
1724b7a3d8b6SEmanuele Giuseppe Esposito __nested_copy_vmcb_save_to_cache(&save_cached, save);
17256906e06dSKrish Sadhukhan if (!(save->cr0 & X86_CR0_PG) ||
17266906e06dSKrish Sadhukhan !(save->cr0 & X86_CR0_PE) ||
17276906e06dSKrish Sadhukhan (save->rflags & X86_EFLAGS_VM) ||
1728b7a3d8b6SEmanuele Giuseppe Esposito !__nested_vmcb_check_save(vcpu, &save_cached))
17296ccbd29aSJoerg Roedel goto out_free;
1730cc440cdaSPaolo Bonzini
1731b222b0b8SMaxim Levitsky
1732b222b0b8SMaxim Levitsky /*
17334995a368SCathy Avery * All checks done, we can enter guest mode. Userspace provides
17344995a368SCathy Avery * vmcb12.control, which will be combined with L1 and stored into
17354995a368SCathy Avery * vmcb02, and the L1 save state which we store in vmcb01.
17364995a368SCathy Avery * L2 registers if needed are moved from the current VMCB to VMCB02.
1737cc440cdaSPaolo Bonzini */
173881f76adaSMaxim Levitsky
17399d290e16SMaxim Levitsky if (is_guest_mode(vcpu))
1740f7e57078SSean Christopherson svm_leave_nested(vcpu);
17419d290e16SMaxim Levitsky else
17429d290e16SMaxim Levitsky svm->nested.vmcb02.ptr->save = svm->vmcb01.ptr->save;
17439d290e16SMaxim Levitsky
1744063ab16cSMaxim Levitsky svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET));
1745063ab16cSMaxim Levitsky
174681f76adaSMaxim Levitsky svm->nested.nested_run_pending =
174781f76adaSMaxim Levitsky !!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING);
174881f76adaSMaxim Levitsky
17490dd16b5bSMaxim Levitsky svm->nested.vmcb12_gpa = kvm_state->hdr.svm.vmcb_pa;
1750c08f390aSPaolo Bonzini
17512bb16beaSVitaly Kuznetsov svm_copy_vmrun_state(&svm->vmcb01.ptr->save, save);
17527907160dSEmanuele Giuseppe Esposito nested_copy_vmcb_control_to_cache(svm, ctl);
17534995a368SCathy Avery
17544995a368SCathy Avery svm_switch_vmcb(svm, &svm->nested.vmcb02);
1755da0b93d6SMaciej S. Szmigiero nested_vmcb02_prepare_control(svm, svm->vmcb->save.rip, svm->vmcb->save.cs.base);
1756e1779c27SMaxim Levitsky
1757e1779c27SMaxim Levitsky /*
1758e1779c27SMaxim Levitsky * While the nested guest CR3 is already checked and set by
1759e1779c27SMaxim Levitsky * KVM_SET_SREGS, it was set when nested state was yet loaded,
1760e1779c27SMaxim Levitsky * thus MMU might not be initialized correctly.
1761e1779c27SMaxim Levitsky * Set it again to fix this.
1762e1779c27SMaxim Levitsky */
1763e1779c27SMaxim Levitsky
1764e1779c27SMaxim Levitsky ret = nested_svm_load_cr3(&svm->vcpu, vcpu->arch.cr3,
1765e1779c27SMaxim Levitsky nested_npt_enabled(svm), false);
1766e1779c27SMaxim Levitsky if (WARN_ON_ONCE(ret))
1767e1779c27SMaxim Levitsky goto out_free;
1768e1779c27SMaxim Levitsky
176973c25546SVitaly Kuznetsov svm->nested.force_msr_bitmap_recalc = true;
1770e1779c27SMaxim Levitsky
1771a7d5c7ceSPaolo Bonzini kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
17726ccbd29aSJoerg Roedel ret = 0;
17736ccbd29aSJoerg Roedel out_free:
17746ccbd29aSJoerg Roedel kfree(save);
17756ccbd29aSJoerg Roedel kfree(ctl);
17766ccbd29aSJoerg Roedel
17776ccbd29aSJoerg Roedel return ret;
1778cc440cdaSPaolo Bonzini }
1779cc440cdaSPaolo Bonzini
svm_get_nested_state_pages(struct kvm_vcpu * vcpu)1780232f75d3SMaxim Levitsky static bool svm_get_nested_state_pages(struct kvm_vcpu *vcpu)
1781232f75d3SMaxim Levitsky {
1782232f75d3SMaxim Levitsky struct vcpu_svm *svm = to_svm(vcpu);
1783232f75d3SMaxim Levitsky
1784232f75d3SMaxim Levitsky if (WARN_ON(!is_guest_mode(vcpu)))
1785232f75d3SMaxim Levitsky return true;
1786232f75d3SMaxim Levitsky
1787158a48ecSMaxim Levitsky if (!vcpu->arch.pdptrs_from_userspace &&
1788158a48ecSMaxim Levitsky !nested_npt_enabled(svm) && is_pae_paging(vcpu))
1789b222b0b8SMaxim Levitsky /*
1790b222b0b8SMaxim Levitsky * Reload the guest's PDPTRs since after a migration
1791b222b0b8SMaxim Levitsky * the guest CR3 might be restored prior to setting the nested
1792b222b0b8SMaxim Levitsky * state which can lead to a load of wrong PDPTRs.
1793b222b0b8SMaxim Levitsky */
17942df4a5ebSLai Jiangshan if (CC(!load_pdptrs(vcpu, vcpu->arch.cr3)))
1795232f75d3SMaxim Levitsky return false;
1796232f75d3SMaxim Levitsky
1797232f75d3SMaxim Levitsky if (!nested_svm_vmrun_msrpm(svm)) {
1798232f75d3SMaxim Levitsky vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1799232f75d3SMaxim Levitsky vcpu->run->internal.suberror =
1800232f75d3SMaxim Levitsky KVM_INTERNAL_ERROR_EMULATION;
1801232f75d3SMaxim Levitsky vcpu->run->internal.ndata = 0;
1802232f75d3SMaxim Levitsky return false;
1803232f75d3SMaxim Levitsky }
1804232f75d3SMaxim Levitsky
18053f4a812eSVitaly Kuznetsov if (kvm_hv_verify_vp_assist(vcpu))
18063f4a812eSVitaly Kuznetsov return false;
18073f4a812eSVitaly Kuznetsov
1808232f75d3SMaxim Levitsky return true;
1809232f75d3SMaxim Levitsky }
1810232f75d3SMaxim Levitsky
181133b22172SPaolo Bonzini struct kvm_x86_nested_ops svm_nested_ops = {
1812f7e57078SSean Christopherson .leave_nested = svm_leave_nested,
18137709aba8SSean Christopherson .is_exception_vmexit = nested_svm_is_exception_vmexit,
181433b22172SPaolo Bonzini .check_events = svm_check_nested_events,
1815cb6a32c2SSean Christopherson .triple_fault = nested_svm_triple_fault,
1816a7d5c7ceSPaolo Bonzini .get_nested_state_pages = svm_get_nested_state_pages,
1817cc440cdaSPaolo Bonzini .get_state = svm_get_nested_state,
1818cc440cdaSPaolo Bonzini .set_state = svm_set_nested_state,
1819b0c9c25eSVitaly Kuznetsov .hv_inject_synthetic_vmexit_post_tlb_flush = svm_hv_inject_synthetic_vmexit_post_tlb_flush,
182033b22172SPaolo Bonzini };
1821