1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Kernel-based Virtual Machine driver for Linux
4 *
5 * AMD SVM support
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
8 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
9 *
10 * Authors:
11 * Yaniv Kamay <yaniv@qumranet.com>
12 * Avi Kivity <avi@qumranet.com>
13 */
14
15 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
16
17 #include <linux/kvm_types.h>
18 #include <linux/kvm_host.h>
19 #include <linux/kernel.h>
20
21 #include <asm/msr-index.h>
22 #include <asm/debugreg.h>
23
24 #include "kvm_emulate.h"
25 #include "trace.h"
26 #include "irq.h"
27 #include "mmu.h"
28 #include "x86.h"
29 #include "smm.h"
30 #include "cpuid.h"
31 #include "lapic.h"
32 #include "svm.h"
33 #include "hyperv.h"
34 #include "pmu.h"
35
36 #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK
37
nested_svm_inject_npf_exit(struct kvm_vcpu * vcpu,struct x86_exception * fault,bool from_hardware)38 static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
39 struct x86_exception *fault,
40 bool from_hardware)
41 {
42 struct vcpu_svm *svm = to_svm(vcpu);
43 struct vmcb *vmcb = svm->vmcb;
44 u64 fault_stage;
45
46 /*
47 * For hardware NPF exits, the GUEST_FAULT_STAGE bits are only
48 * available in the hardware exit_info_1, since the guest_mmu
49 * walker doesn't know whether the faulting GPA was a page table
50 * page or final page from L2's perspective.
51 */
52 if (from_hardware)
53 fault_stage = vmcb->control.exit_info_1 &
54 PFERR_GUEST_FAULT_STAGE_MASK;
55 else
56 fault_stage = fault->error_code & PFERR_GUEST_FAULT_STAGE_MASK;
57
58 /*
59 * All nested page faults should be annotated as occurring on the
60 * final translation *or* the page walk. Arbitrarily choose "final"
61 * if KVM is buggy and enumerated both or neither.
62 */
63 if (WARN_ON_ONCE(hweight64(fault_stage) != 1))
64 fault_stage = PFERR_GUEST_FINAL_MASK;
65
66 vmcb->control.exit_code = SVM_EXIT_NPF;
67 vmcb->control.exit_info_1 = fault_stage |
68 (fault->error_code & ~PFERR_GUEST_FAULT_STAGE_MASK);
69 vmcb->control.exit_info_2 = fault->address;
70
71 nested_svm_vmexit(svm);
72 }
73
nested_svm_get_tdp_pdptr(struct kvm_vcpu * vcpu,int index)74 static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index)
75 {
76 struct vcpu_svm *svm = to_svm(vcpu);
77 u64 cr3 = svm->nested.ctl.nested_cr3;
78 u64 pdpte;
79 int ret;
80
81 /*
82 * Note, nCR3 is "assumed" to be 32-byte aligned, i.e. the CPU ignores
83 * nCR3[4:0] when loading PDPTEs from memory.
84 */
85 ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(cr3), &pdpte,
86 (cr3 & GENMASK(11, 5)) + index * 8, 8);
87 if (ret)
88 return 0;
89 return pdpte;
90 }
91
nested_svm_get_tdp_cr3(struct kvm_vcpu * vcpu)92 static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu)
93 {
94 struct vcpu_svm *svm = to_svm(vcpu);
95
96 return svm->nested.ctl.nested_cr3;
97 }
98
nested_svm_init_mmu_context(struct kvm_vcpu * vcpu)99 static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
100 {
101 struct vcpu_svm *svm = to_svm(vcpu);
102
103 WARN_ON(mmu_is_nested(vcpu));
104
105 vcpu->arch.mmu = &vcpu->arch.guest_mmu;
106
107 /*
108 * The NPT format depends on L1's CR4 and EFER, which is in vmcb01. Note,
109 * when called via KVM_SET_NESTED_STATE, that state may _not_ match current
110 * vCPU state. CR0.WP is explicitly ignored, while CR0.PG is required.
111 */
112 kvm_init_shadow_npt_mmu(vcpu, svm->vmcb01.ptr->save.cr4,
113 svm->vmcb01.ptr->save.efer,
114 svm->nested.ctl.nested_cr3,
115 svm->nested.ctl.misc_ctl);
116
117 vcpu->arch.ngpa_walk.get_guest_pgd = nested_svm_get_tdp_cr3;
118 vcpu->arch.ngpa_walk.get_pdptr = nested_svm_get_tdp_pdptr;
119 vcpu->arch.ngpa_walk.inject_page_fault = nested_svm_inject_npf_exit;
120 }
121
nested_svm_uninit_mmu_context(struct kvm_vcpu * vcpu)122 static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
123 {
124 vcpu->arch.mmu = &vcpu->arch.root_mmu;
125 }
126
nested_vmcb_needs_vls_intercept(struct vcpu_svm * svm)127 static bool nested_vmcb_needs_vls_intercept(struct vcpu_svm *svm)
128 {
129 if (!guest_cpu_cap_has(&svm->vcpu, X86_FEATURE_V_VMSAVE_VMLOAD))
130 return true;
131
132 if (!nested_npt_enabled(svm))
133 return true;
134
135 if (!(svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE))
136 return true;
137
138 return false;
139 }
140
nested_vmcb02_recalc_intercepts(struct vcpu_svm * svm)141 void nested_vmcb02_recalc_intercepts(struct vcpu_svm *svm)
142 {
143 struct vmcb_ctrl_area_cached *vmcb12_ctrl = &svm->nested.ctl;
144 struct vmcb *vmcb02 = svm->nested.vmcb02.ptr;
145 struct vmcb *vmcb01 = svm->vmcb01.ptr;
146 unsigned int i;
147
148 if (WARN_ON_ONCE(svm->vmcb != vmcb02))
149 return;
150
151 vmcb_mark_dirty(vmcb02, VMCB_INTERCEPTS);
152
153 for (i = 0; i < MAX_INTERCEPT; i++)
154 vmcb02->control.intercepts[i] = vmcb01->control.intercepts[i];
155
156 if (vmcb12_ctrl->int_ctl & V_INTR_MASKING_MASK) {
157 /*
158 * If L2 is active and V_INTR_MASKING is enabled in vmcb12,
159 * disable intercept of CR8 writes as L2's CR8 does not affect
160 * any interrupt KVM may want to inject.
161 *
162 * Similarly, disable intercept of virtual interrupts (used to
163 * detect interrupt windows) if the saved RFLAGS.IF is '0', as
164 * the effective RFLAGS.IF for L1 interrupts will never be set
165 * while L2 is running (L2's RFLAGS.IF doesn't affect L1 IRQs).
166 */
167 vmcb_clr_intercept(&vmcb02->control, INTERCEPT_CR8_WRITE);
168 if (!(vmcb01->save.rflags & X86_EFLAGS_IF))
169 vmcb_clr_intercept(&vmcb02->control, INTERCEPT_VINTR);
170 }
171
172 for (i = 0; i < MAX_INTERCEPT; i++)
173 vmcb02->control.intercepts[i] |= vmcb12_ctrl->intercepts[i];
174
175 /* If SMI is not intercepted, ignore guest SMI intercept as well */
176 if (!intercept_smi)
177 vmcb_clr_intercept(&vmcb02->control, INTERCEPT_SMI);
178
179 /*
180 * Intercept PAUSE if and only if L1 wants to. KVM intercepts PAUSE so
181 * that a vCPU that may be spinning waiting for a lock can be scheduled
182 * out in favor of the vCPU that holds said lock. KVM doesn't support
183 * yielding across L2 vCPUs, as KVM has limited visilibity into which
184 * L2 vCPUs are in the same L2 VM, i.e. may be contending for locks.
185 */
186 if (!vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_PAUSE))
187 vmcb_clr_intercept(&vmcb02->control, INTERCEPT_PAUSE);
188
189 if (nested_vmcb_needs_vls_intercept(svm)) {
190 /*
191 * If the virtual VMLOAD/VMSAVE is not enabled for the L2,
192 * we must intercept these instructions to correctly
193 * emulate them in case L1 doesn't intercept them.
194 */
195 vmcb_set_intercept(&vmcb02->control, INTERCEPT_VMLOAD);
196 vmcb_set_intercept(&vmcb02->control, INTERCEPT_VMSAVE);
197 } else {
198 WARN_ON_ONCE(!(vmcb02->control.misc_ctl2 & SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE));
199 }
200 }
201
202 /*
203 * This array (and its actual size) holds the set of offsets (indexing by chunk
204 * size) to process when merging vmcb12's MSRPM with vmcb01's MSRPM. Note, the
205 * set of MSRs for which interception is disabled in vmcb01 is per-vCPU, e.g.
206 * based on CPUID features. This array only tracks MSRs that *might* be passed
207 * through to the guest.
208 *
209 * Hardcode the capacity of the array based on the maximum number of _offsets_.
210 * MSRs are batched together, so there are fewer offsets than MSRs.
211 */
212 static int nested_svm_msrpm_merge_offsets[10] __ro_after_init;
213 static int nested_svm_nr_msrpm_merge_offsets __ro_after_init;
214 typedef unsigned long nsvm_msrpm_merge_t;
215
nested_svm_init_msrpm_merge_offsets(void)216 int __init nested_svm_init_msrpm_merge_offsets(void)
217 {
218 static const u32 merge_msrs[] __initconst = {
219 MSR_STAR,
220 MSR_IA32_SYSENTER_CS,
221 MSR_IA32_SYSENTER_EIP,
222 MSR_IA32_SYSENTER_ESP,
223 #ifdef CONFIG_X86_64
224 MSR_GS_BASE,
225 MSR_FS_BASE,
226 MSR_KERNEL_GS_BASE,
227 MSR_LSTAR,
228 MSR_CSTAR,
229 MSR_SYSCALL_MASK,
230 #endif
231 MSR_IA32_SPEC_CTRL,
232 MSR_IA32_PRED_CMD,
233 MSR_IA32_FLUSH_CMD,
234 MSR_IA32_APERF,
235 MSR_IA32_MPERF,
236 MSR_IA32_LASTBRANCHFROMIP,
237 MSR_IA32_LASTBRANCHTOIP,
238 MSR_IA32_LASTINTFROMIP,
239 MSR_IA32_LASTINTTOIP,
240
241 MSR_K7_PERFCTR0,
242 MSR_K7_PERFCTR1,
243 MSR_K7_PERFCTR2,
244 MSR_K7_PERFCTR3,
245 MSR_F15H_PERF_CTR0,
246 MSR_F15H_PERF_CTR1,
247 MSR_F15H_PERF_CTR2,
248 MSR_F15H_PERF_CTR3,
249 MSR_F15H_PERF_CTR4,
250 MSR_F15H_PERF_CTR5,
251
252 MSR_AMD64_PERF_CNTR_GLOBAL_CTL,
253 MSR_AMD64_PERF_CNTR_GLOBAL_STATUS,
254 MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR,
255 MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET,
256 };
257 int i, j;
258
259 for (i = 0; i < ARRAY_SIZE(merge_msrs); i++) {
260 int bit_nr = svm_msrpm_bit_nr(merge_msrs[i]);
261 u32 offset;
262
263 if (WARN_ON(bit_nr < 0))
264 return -EIO;
265
266 /*
267 * Merging is done in chunks to reduce the number of accesses
268 * to L1's bitmap.
269 */
270 offset = bit_nr / BITS_PER_BYTE / sizeof(nsvm_msrpm_merge_t);
271
272 for (j = 0; j < nested_svm_nr_msrpm_merge_offsets; j++) {
273 if (nested_svm_msrpm_merge_offsets[j] == offset)
274 break;
275 }
276
277 if (j < nested_svm_nr_msrpm_merge_offsets)
278 continue;
279
280 if (WARN_ON(j >= ARRAY_SIZE(nested_svm_msrpm_merge_offsets)))
281 return -EIO;
282
283 nested_svm_msrpm_merge_offsets[j] = offset;
284 nested_svm_nr_msrpm_merge_offsets++;
285 }
286
287 return 0;
288 }
289
290 /*
291 * Merge L0's (KVM) and L1's (Nested VMCB) MSR permission bitmaps. The function
292 * is optimized in that it only merges the parts where KVM MSR permission bitmap
293 * may contain zero bits.
294 */
nested_svm_merge_msrpm(struct kvm_vcpu * vcpu)295 static bool nested_svm_merge_msrpm(struct kvm_vcpu *vcpu)
296 {
297 struct vcpu_svm *svm = to_svm(vcpu);
298 nsvm_msrpm_merge_t *msrpm02 = svm->nested.msrpm;
299 nsvm_msrpm_merge_t *msrpm01 = svm->msrpm;
300 int i;
301
302 /*
303 * MSR bitmap update can be skipped when:
304 * - MSR bitmap for L1 hasn't changed.
305 * - Nested hypervisor (L1) is attempting to launch the same L2 as
306 * before.
307 * - Nested hypervisor (L1) is using Hyper-V emulation interface and
308 * tells KVM (L0) there were no changes in MSR bitmap for L2.
309 */
310 #ifdef CONFIG_KVM_HYPERV
311 if (!svm->nested.force_msr_bitmap_recalc) {
312 struct hv_vmcb_enlightenments *hve = &svm->nested.ctl.hv_enlightenments;
313
314 if (kvm_hv_hypercall_enabled(vcpu) &&
315 hve->hv_enlightenments_control.msr_bitmap &&
316 (svm->nested.ctl.clean & BIT(HV_VMCB_NESTED_ENLIGHTENMENTS)))
317 goto set_msrpm_base_pa;
318 }
319 #endif
320
321 if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT)))
322 return true;
323
324 for (i = 0; i < nested_svm_nr_msrpm_merge_offsets; i++) {
325 const int p = nested_svm_msrpm_merge_offsets[i];
326 nsvm_msrpm_merge_t l1_val;
327 gpa_t gpa;
328
329 gpa = svm->nested.ctl.msrpm_base_pa + (p * sizeof(l1_val));
330
331 if (kvm_vcpu_read_guest(vcpu, gpa, &l1_val, sizeof(l1_val)))
332 return false;
333
334 msrpm02[p] = msrpm01[p] | l1_val;
335 }
336
337 svm->nested.force_msr_bitmap_recalc = false;
338
339 #ifdef CONFIG_KVM_HYPERV
340 set_msrpm_base_pa:
341 #endif
342 svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm));
343
344 return true;
345 }
346
347 /*
348 * Bits 11:0 of bitmap address are ignored by hardware
349 */
nested_svm_check_bitmap_pa(struct kvm_vcpu * vcpu,u64 pa,u32 size)350 static bool nested_svm_check_bitmap_pa(struct kvm_vcpu *vcpu, u64 pa, u32 size)
351 {
352 u64 addr = PAGE_ALIGN(pa);
353
354 return kvm_vcpu_is_legal_gpa(vcpu, addr) &&
355 kvm_vcpu_is_legal_gpa(vcpu, addr + size - 1);
356 }
357
nested_svm_event_inj_valid_exept(struct kvm_vcpu * vcpu,u8 vector)358 static bool nested_svm_event_inj_valid_exept(struct kvm_vcpu *vcpu, u8 vector)
359 {
360 /*
361 * Vectors that do not correspond to a defined exception are invalid
362 * (including #NMI and reserved vectors). In a best effort to define
363 * valid exceptions based on the virtual CPU, make all exceptions always
364 * valid except those obviously tied to a CPU feature.
365 */
366 switch (vector) {
367 case DE_VECTOR: case DB_VECTOR: case BP_VECTOR: case OF_VECTOR:
368 case BR_VECTOR: case UD_VECTOR: case NM_VECTOR: case DF_VECTOR:
369 case TS_VECTOR: case NP_VECTOR: case SS_VECTOR: case GP_VECTOR:
370 case PF_VECTOR: case MF_VECTOR: case AC_VECTOR: case MC_VECTOR:
371 case XM_VECTOR: case HV_VECTOR: case SX_VECTOR:
372 return true;
373 case CP_VECTOR:
374 return guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK);
375 case VC_VECTOR:
376 return guest_cpu_cap_has(vcpu, X86_FEATURE_SEV_ES);
377 }
378 return false;
379 }
380
381 /*
382 * According to the APM, VMRUN exits with SVM_EXIT_ERR if SVM_EVTINJ_VALID is
383 * set and:
384 * - The type of event_inj is not one of the defined values.
385 * - The type is SVM_EVTINJ_TYPE_EXEPT, but the vector is not a valid exception.
386 */
nested_svm_check_event_inj(struct kvm_vcpu * vcpu,u32 event_inj)387 static bool nested_svm_check_event_inj(struct kvm_vcpu *vcpu, u32 event_inj)
388 {
389 u32 type = event_inj & SVM_EVTINJ_TYPE_MASK;
390 u8 vector = event_inj & SVM_EVTINJ_VEC_MASK;
391
392 if (!(event_inj & SVM_EVTINJ_VALID))
393 return true;
394
395 if (type != SVM_EVTINJ_TYPE_INTR && type != SVM_EVTINJ_TYPE_NMI &&
396 type != SVM_EVTINJ_TYPE_EXEPT && type != SVM_EVTINJ_TYPE_SOFT)
397 return false;
398
399 if (type == SVM_EVTINJ_TYPE_EXEPT &&
400 !nested_svm_event_inj_valid_exept(vcpu, vector))
401 return false;
402
403 return true;
404 }
405
nested_vmcb_check_controls(struct kvm_vcpu * vcpu,struct vmcb_ctrl_area_cached * control)406 static bool nested_vmcb_check_controls(struct kvm_vcpu *vcpu,
407 struct vmcb_ctrl_area_cached *control)
408 {
409 if (CC(!vmcb12_is_intercept(control, INTERCEPT_VMRUN)))
410 return false;
411
412 if (CC(control->asid == 0))
413 return false;
414
415 if (CC((control->misc_ctl & SVM_MISC_ENABLE_NP) &&
416 !kvm_vcpu_is_legal_gpa(vcpu, control->nested_cr3)))
417 return false;
418
419 if (CC(!nested_svm_check_bitmap_pa(vcpu, control->msrpm_base_pa,
420 MSRPM_SIZE)))
421 return false;
422 if (CC(!nested_svm_check_bitmap_pa(vcpu, control->iopm_base_pa,
423 IOPM_SIZE)))
424 return false;
425
426 if (CC((control->int_ctl & V_NMI_ENABLE_MASK) &&
427 !vmcb12_is_intercept(control, INTERCEPT_NMI))) {
428 return false;
429 }
430
431 if (CC(!nested_svm_check_event_inj(vcpu, control->event_inj)))
432 return false;
433
434 return true;
435 }
436
437 /* Common checks that apply to both L1 and L2 state. */
nested_vmcb_check_save(struct kvm_vcpu * vcpu,struct vmcb_save_area_cached * save,bool check_gpat)438 static bool nested_vmcb_check_save(struct kvm_vcpu *vcpu,
439 struct vmcb_save_area_cached *save,
440 bool check_gpat)
441 {
442 if (CC(!(save->efer & EFER_SVME)))
443 return false;
444
445 if (CC((save->cr0 & X86_CR0_CD) == 0 && (save->cr0 & X86_CR0_NW)) ||
446 CC(save->cr0 & ~0xffffffffULL))
447 return false;
448
449 if (CC(!kvm_dr6_valid(save->dr6)) || CC(!kvm_dr7_valid(save->dr7)))
450 return false;
451
452 /*
453 * These checks are also performed by KVM_SET_SREGS,
454 * except that EFER.LMA is not checked by SVM against
455 * CR0.PG && EFER.LME.
456 */
457 if ((save->efer & EFER_LME) && (save->cr0 & X86_CR0_PG)) {
458 if (CC(!(save->cr4 & X86_CR4_PAE)) ||
459 CC(!(save->cr0 & X86_CR0_PE)) ||
460 CC(!kvm_vcpu_is_legal_cr3(vcpu, save->cr3)))
461 return false;
462
463 if (CC((save->cs.attrib & SVM_SELECTOR_L_MASK) &&
464 (save->cs.attrib & SVM_SELECTOR_DB_MASK)))
465 return false;
466 }
467
468 /* Note, SVM doesn't have any additional restrictions on CR4. */
469 if (CC(!__kvm_is_valid_cr4(vcpu, save->cr4)))
470 return false;
471
472 if (CC(!kvm_valid_efer(vcpu, save->efer)))
473 return false;
474
475 /*
476 * If userspace contrives to get an invalid g_pat into vmcb02 by
477 * disabling KVM_X86_QUIRK_NESTED_SVM_SHARED_PAT in a race with
478 * this check, it should be prepared for the KVM_EXIT_FAIL_ENTRY
479 * that will follow.
480 */
481 if (check_gpat && CC(!kvm_pat_valid(save->g_pat)))
482 return false;
483
484 return true;
485 }
486
nested_svm_check_cached_vmcb12(struct kvm_vcpu * vcpu)487 int nested_svm_check_cached_vmcb12(struct kvm_vcpu *vcpu)
488 {
489 struct vcpu_svm *svm = to_svm(vcpu);
490
491 if (!nested_vmcb_check_save(vcpu, &svm->nested.save,
492 l2_has_separate_pat(vcpu)) ||
493 !nested_vmcb_check_controls(vcpu, &svm->nested.ctl))
494 return -EINVAL;
495
496 return 0;
497 }
498
499 /*
500 * If a feature is not advertised to L1, clear the corresponding vmcb12
501 * intercept.
502 */
503 #define __nested_svm_sanitize_intercept(__vcpu, __control, fname, iname) \
504 do { \
505 if (!guest_cpu_cap_has(__vcpu, X86_FEATURE_##fname)) \
506 vmcb12_clr_intercept(__control, INTERCEPT_##iname); \
507 } while (0)
508
509 #define nested_svm_sanitize_intercept(__vcpu, __control, name) \
510 __nested_svm_sanitize_intercept(__vcpu, __control, name, name)
511
512 static
__nested_copy_vmcb_control_to_cache(struct kvm_vcpu * vcpu,struct vmcb_ctrl_area_cached * to,struct vmcb_control_area * from)513 void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu,
514 struct vmcb_ctrl_area_cached *to,
515 struct vmcb_control_area *from)
516 {
517 unsigned int i;
518
519 for (i = 0; i < MAX_INTERCEPT; i++)
520 to->intercepts[i] = from->intercepts[i];
521
522 __nested_svm_sanitize_intercept(vcpu, to, XSAVE, XSETBV);
523 nested_svm_sanitize_intercept(vcpu, to, INVPCID);
524 nested_svm_sanitize_intercept(vcpu, to, RDTSCP);
525 nested_svm_sanitize_intercept(vcpu, to, SKINIT);
526 nested_svm_sanitize_intercept(vcpu, to, RDPRU);
527
528 /* Always clear misc_ctl bits that the guest cannot use */
529 to->misc_ctl = from->misc_ctl;
530 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_NPT))
531 to->misc_ctl &= ~SVM_MISC_ENABLE_NP;
532
533 if (!gmet_enabled || !guest_cpu_cap_has(vcpu, X86_FEATURE_GMET))
534 to->misc_ctl &= ~SVM_MISC_ENABLE_GMET;
535
536 to->iopm_base_pa = from->iopm_base_pa & PAGE_MASK;
537 to->msrpm_base_pa = from->msrpm_base_pa & PAGE_MASK;
538 to->tsc_offset = from->tsc_offset;
539 to->tlb_ctl = from->tlb_ctl & TLB_CONTROL_MASK;
540 to->erap_ctl = from->erap_ctl;
541 to->int_ctl = from->int_ctl;
542 to->int_vector = from->int_vector & SVM_INT_VECTOR_MASK;
543 to->int_state = from->int_state & SVM_INTERRUPT_SHADOW_MASK;
544 to->exit_code = from->exit_code;
545 to->exit_info_1 = from->exit_info_1;
546 to->exit_info_2 = from->exit_info_2;
547 to->exit_int_info = from->exit_int_info;
548 to->exit_int_info_err = from->exit_int_info_err;
549 to->event_inj = from->event_inj & ~SVM_EVTINJ_RESERVED_BITS;
550 to->event_inj_err = from->event_inj_err;
551 to->next_rip = from->next_rip;
552 to->nested_cr3 = from->nested_cr3;
553 to->misc_ctl2 = from->misc_ctl2;
554 to->pause_filter_count = from->pause_filter_count;
555 to->pause_filter_thresh = from->pause_filter_thresh;
556
557 /* Copy asid here because nested_vmcb_check_controls() will check it */
558 to->asid = from->asid;
559 to->clean = from->clean;
560
561 #ifdef CONFIG_KVM_HYPERV
562 /* Hyper-V extensions (Enlightened VMCB) */
563 if (kvm_hv_hypercall_enabled(vcpu)) {
564 memcpy(&to->hv_enlightenments, &from->hv_enlightenments,
565 sizeof(to->hv_enlightenments));
566 }
567 #endif
568 }
569
nested_copy_vmcb_control_to_cache(struct vcpu_svm * svm,struct vmcb_control_area * control)570 void nested_copy_vmcb_control_to_cache(struct vcpu_svm *svm,
571 struct vmcb_control_area *control)
572 {
573 __nested_copy_vmcb_control_to_cache(&svm->vcpu, &svm->nested.ctl, control);
574 }
575
__nested_copy_vmcb_save_to_cache(struct vmcb_save_area_cached * to,struct vmcb_save_area * from)576 static void __nested_copy_vmcb_save_to_cache(struct vmcb_save_area_cached *to,
577 struct vmcb_save_area *from)
578 {
579 to->es = from->es;
580 to->cs = from->cs;
581 to->ss = from->ss;
582 to->ds = from->ds;
583 to->gdtr = from->gdtr;
584 to->idtr = from->idtr;
585
586 to->cpl = from->cpl;
587
588 to->efer = from->efer;
589 to->cr4 = from->cr4;
590 to->cr3 = from->cr3;
591 to->cr0 = from->cr0;
592 to->dr7 = from->dr7;
593 to->dr6 = from->dr6;
594
595 to->rflags = from->rflags;
596 to->rip = from->rip;
597 to->rsp = from->rsp;
598
599 to->s_cet = from->s_cet;
600 to->ssp = from->ssp;
601 to->isst_addr = from->isst_addr;
602
603 to->rax = from->rax;
604 to->cr2 = from->cr2;
605 to->g_pat = from->g_pat;
606
607 svm_copy_lbrs(to, from);
608 }
609
nested_copy_vmcb_save_to_cache(struct vcpu_svm * svm,struct vmcb_save_area * save)610 void nested_copy_vmcb_save_to_cache(struct vcpu_svm *svm,
611 struct vmcb_save_area *save)
612 {
613 __nested_copy_vmcb_save_to_cache(&svm->nested.save, save);
614 }
615
616 /*
617 * Synchronize fields that are written by the processor, so that
618 * they can be copied back into the vmcb12.
619 */
nested_sync_control_from_vmcb02(struct vcpu_svm * svm)620 void nested_sync_control_from_vmcb02(struct vcpu_svm *svm)
621 {
622 u32 mask;
623 svm->nested.ctl.event_inj = svm->vmcb->control.event_inj;
624 svm->nested.ctl.event_inj_err = svm->vmcb->control.event_inj_err;
625 svm->nested.ctl.int_state = svm->vmcb->control.int_state;
626
627 /* Only a few fields of int_ctl are written by the processor. */
628 mask = V_IRQ_MASK | V_TPR_MASK;
629 /*
630 * Don't sync vmcb02 V_IRQ back to vmcb12 if KVM (L0) is intercepting
631 * virtual interrupts in order to request an interrupt window, as KVM
632 * has usurped vmcb02's int_ctl. If an interrupt window opens before
633 * the next VM-Exit, svm_clear_vintr() will restore vmcb12's int_ctl.
634 * If no window opens, V_IRQ will be correctly preserved in vmcb12's
635 * int_ctl (because it was never recognized while L2 was running).
636 */
637 if (svm_is_intercept(svm, INTERCEPT_VINTR) &&
638 !vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_VINTR))
639 mask &= ~V_IRQ_MASK;
640
641 if (nested_vgif_enabled(svm))
642 mask |= V_GIF_MASK;
643
644 if (nested_vnmi_enabled(svm))
645 mask |= V_NMI_BLOCKING_MASK | V_NMI_PENDING_MASK;
646
647 svm->nested.ctl.int_ctl &= ~mask;
648 svm->nested.ctl.int_ctl |= svm->vmcb->control.int_ctl & mask;
649 }
650
651 /*
652 * Transfer any event that L0 or L1 wanted to inject into L2 to
653 * EXIT_INT_INFO.
654 */
nested_save_pending_event_to_vmcb12(struct vcpu_svm * svm,struct vmcb * vmcb12)655 static void nested_save_pending_event_to_vmcb12(struct vcpu_svm *svm,
656 struct vmcb *vmcb12)
657 {
658 struct kvm_vcpu *vcpu = &svm->vcpu;
659 u32 exit_int_info = 0;
660 unsigned int nr;
661
662 if (vcpu->arch.exception.injected) {
663 nr = vcpu->arch.exception.vector;
664 exit_int_info = nr | SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_EXEPT;
665
666 if (vcpu->arch.exception.has_error_code) {
667 exit_int_info |= SVM_EVTINJ_VALID_ERR;
668 vmcb12->control.exit_int_info_err =
669 vcpu->arch.exception.error_code;
670 }
671
672 } else if (vcpu->arch.nmi_injected) {
673 exit_int_info = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
674
675 } else if (vcpu->arch.interrupt.injected) {
676 nr = vcpu->arch.interrupt.nr;
677 exit_int_info = nr | SVM_EVTINJ_VALID;
678
679 if (vcpu->arch.interrupt.soft)
680 exit_int_info |= SVM_EVTINJ_TYPE_SOFT;
681 else
682 exit_int_info |= SVM_EVTINJ_TYPE_INTR;
683 }
684
685 vmcb12->control.exit_int_info = exit_int_info;
686 }
687
nested_svm_transition_tlb_flush(struct kvm_vcpu * vcpu)688 static void nested_svm_transition_tlb_flush(struct kvm_vcpu *vcpu)
689 {
690 /* Handle pending Hyper-V TLB flush requests */
691 kvm_hv_nested_transtion_tlb_flush(vcpu, npt_enabled);
692
693 /*
694 * TODO: optimize unconditional TLB flush/MMU sync. A partial list of
695 * things to fix before this can be conditional:
696 *
697 * - Flush TLBs for both L1 and L2 remote TLB flush
698 * - Honor L1's request to flush an ASID on nested VMRUN
699 * - Sync nested NPT MMU on VMRUN that flushes L2's ASID[*]
700 * - Don't crush a pending TLB flush in vmcb02 on nested VMRUN
701 * - Flush L1's ASID on KVM_REQ_TLB_FLUSH_GUEST
702 *
703 * [*] Unlike nested EPT, SVM's ASID management can invalidate nested
704 * NPT guest-physical mappings on VMRUN.
705 */
706 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
707 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
708 }
709
710 /*
711 * Load guest's/host's cr3 on nested vmentry or vmexit. @nested_npt is true
712 * if we are emulating VM-Entry into a guest with NPT enabled.
713 */
nested_svm_load_cr3(struct kvm_vcpu * vcpu,unsigned long cr3,bool nested_npt,bool reload_pdptrs)714 static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3,
715 bool nested_npt, bool reload_pdptrs)
716 {
717 if (CC(!kvm_vcpu_is_legal_cr3(vcpu, cr3)))
718 return -EINVAL;
719
720 if (reload_pdptrs && is_pae_paging(vcpu)) {
721 if (nested_npt)
722 kvm_register_mark_for_reload(vcpu, VCPU_REG_PDPTR);
723 else if (CC(!load_pdptrs(vcpu, cr3)))
724 return -EINVAL;
725 }
726
727 vcpu->arch.cr3 = cr3;
728
729 /* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */
730 kvm_init_mmu(vcpu);
731
732 if (!nested_npt)
733 kvm_mmu_new_pgd(vcpu, cr3);
734
735 return 0;
736 }
737
nested_vmcb12_has_lbrv(struct kvm_vcpu * vcpu)738 static bool nested_vmcb12_has_lbrv(struct kvm_vcpu *vcpu)
739 {
740 return guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) &&
741 (to_svm(vcpu)->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR);
742 }
743
nested_vmcb02_prepare_save(struct vcpu_svm * svm)744 static void nested_vmcb02_prepare_save(struct vcpu_svm *svm)
745 {
746 struct vmcb_ctrl_area_cached *control = &svm->nested.ctl;
747 struct vmcb_save_area_cached *save = &svm->nested.save;
748 bool new_vmcb12 = false;
749 struct vmcb *vmcb01 = svm->vmcb01.ptr;
750 struct vmcb *vmcb02 = svm->nested.vmcb02.ptr;
751 struct kvm_vcpu *vcpu = &svm->vcpu;
752
753 /* Load the nested guest state */
754 if (svm->nested.vmcb12_gpa != svm->nested.last_vmcb12_gpa) {
755 new_vmcb12 = true;
756 svm->nested.last_vmcb12_gpa = svm->nested.vmcb12_gpa;
757 svm->nested.force_msr_bitmap_recalc = true;
758 }
759
760 if (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_SEG))) {
761 vmcb02->save.es = save->es;
762 vmcb02->save.cs = save->cs;
763 vmcb02->save.ss = save->ss;
764 vmcb02->save.ds = save->ds;
765 vmcb02->save.cpl = save->cpl;
766 vmcb_mark_dirty(vmcb02, VMCB_SEG);
767 }
768
769 if (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_DT))) {
770 vmcb02->save.gdtr = save->gdtr;
771 vmcb02->save.idtr = save->idtr;
772 vmcb_mark_dirty(vmcb02, VMCB_DT);
773 }
774
775 if (guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) &&
776 (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_CET)))) {
777 vmcb02->save.s_cet = save->s_cet;
778 vmcb02->save.isst_addr = save->isst_addr;
779 vmcb02->save.ssp = save->ssp;
780 vmcb_mark_dirty(vmcb02, VMCB_CET);
781 }
782
783 if (l2_has_separate_pat(vcpu)) {
784 if (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_NPT)))
785 vmcb_set_gpat(vmcb02, svm->nested.save.g_pat);
786 } else if (npt_enabled) {
787 vmcb_set_gpat(vmcb02, vcpu->arch.pat);
788 }
789
790 kvm_set_rflags(vcpu, save->rflags | X86_EFLAGS_FIXED);
791
792 svm_set_efer(vcpu, svm->nested.save.efer);
793
794 svm_set_cr0(vcpu, svm->nested.save.cr0);
795 svm_set_cr4(vcpu, svm->nested.save.cr4);
796
797 svm->vcpu.arch.cr2 = save->cr2;
798
799 kvm_rax_write_raw(vcpu, save->rax);
800 kvm_rsp_write(vcpu, save->rsp);
801 kvm_rip_write(vcpu, save->rip);
802
803 /* In case we don't even reach vcpu_run, the fields are not updated */
804 vmcb02->save.rax = save->rax;
805 vmcb02->save.rsp = save->rsp;
806 vmcb02->save.rip = save->rip;
807
808 if (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_DR))) {
809 vmcb02->save.dr7 = svm->nested.save.dr7 | DR7_FIXED_1;
810 svm->vcpu.arch.dr6 = svm->nested.save.dr6 | DR6_ACTIVE_LOW;
811 vmcb_mark_dirty(vmcb02, VMCB_DR);
812 }
813
814 if (nested_vmcb12_has_lbrv(vcpu)) {
815 /*
816 * Reserved bits of DEBUGCTL are ignored. Be consistent with
817 * svm_set_msr's definition of reserved bits.
818 */
819 svm_copy_lbrs(&vmcb02->save, save);
820 vmcb02->save.dbgctl &= ~DEBUGCTL_RESERVED_BITS;
821 } else {
822 svm_copy_lbrs(&vmcb02->save, &vmcb01->save);
823 }
824 vmcb_mark_dirty(vmcb02, VMCB_LBR);
825 svm_update_lbrv(&svm->vcpu);
826 }
827
is_evtinj_soft(u32 evtinj)828 static inline bool is_evtinj_soft(u32 evtinj)
829 {
830 u32 type = evtinj & SVM_EVTINJ_TYPE_MASK;
831 u8 vector = evtinj & SVM_EVTINJ_VEC_MASK;
832
833 if (!(evtinj & SVM_EVTINJ_VALID))
834 return false;
835
836 if (type == SVM_EVTINJ_TYPE_SOFT)
837 return true;
838
839 return type == SVM_EVTINJ_TYPE_EXEPT && kvm_exception_is_soft(vector);
840 }
841
is_evtinj_nmi(u32 evtinj)842 static bool is_evtinj_nmi(u32 evtinj)
843 {
844 u32 type = evtinj & SVM_EVTINJ_TYPE_MASK;
845
846 if (!(evtinj & SVM_EVTINJ_VALID))
847 return false;
848
849 return type == SVM_EVTINJ_TYPE_NMI;
850 }
851
nested_vmcb02_prepare_control(struct vcpu_svm * svm)852 static void nested_vmcb02_prepare_control(struct vcpu_svm *svm)
853 {
854 u32 int_ctl_vmcb01_bits = V_INTR_MASKING_MASK;
855 u32 int_ctl_vmcb12_bits = V_TPR_MASK | V_IRQ_INJECTION_BITS_MASK;
856
857 struct vmcb_ctrl_area_cached *vmcb12_ctrl = &svm->nested.ctl;
858 struct vmcb *vmcb02 = svm->nested.vmcb02.ptr;
859 struct vmcb *vmcb01 = svm->vmcb01.ptr;
860 struct kvm_vcpu *vcpu = &svm->vcpu;
861
862 nested_svm_transition_tlb_flush(vcpu);
863
864 /* Enter Guest-Mode */
865 enter_guest_mode(vcpu);
866 svm_pmu_handle_nested_transition(svm);
867
868 /*
869 * Filled at exit: exit_code, exit_info_1, exit_info_2, exit_int_info,
870 * exit_int_info_err, next_rip, insn_len, insn_bytes.
871 */
872
873 if (guest_cpu_cap_has(vcpu, X86_FEATURE_VGIF) &&
874 (vmcb12_ctrl->int_ctl & V_GIF_ENABLE_MASK))
875 int_ctl_vmcb12_bits |= (V_GIF_MASK | V_GIF_ENABLE_MASK);
876 else
877 int_ctl_vmcb01_bits |= (V_GIF_MASK | V_GIF_ENABLE_MASK);
878
879 if (vnmi) {
880 if (vmcb01->control.int_ctl & V_NMI_PENDING_MASK) {
881 svm->vcpu.arch.nmi_pending++;
882 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
883 }
884 if (nested_vnmi_enabled(svm))
885 int_ctl_vmcb12_bits |= (V_NMI_PENDING_MASK |
886 V_NMI_ENABLE_MASK |
887 V_NMI_BLOCKING_MASK);
888 }
889
890 /*
891 * Copied from vmcb01. msrpm_base can be overwritten later.
892 *
893 * SVM_MISC_ENABLE_NP in vmcb12 is only used for consistency checks. If
894 * L1 enables NPTs, KVM shadows L1's NPTs and uses those to run L2. If
895 * L1 disables NPT, KVM runs L2 with the same NPTs used to run L1. For
896 * the latter, L1 runs L2 with shadow page tables that translate L2 GVAs
897 * to L1 GPAs, so the same NPTs can be used for L1 and L2.
898 */
899 vmcb02->control.misc_ctl = vmcb01->control.misc_ctl & (SVM_MISC_ENABLE_NP | SVM_MISC_ENABLE_GMET);
900 vmcb02->control.iopm_base_pa = vmcb01->control.iopm_base_pa;
901 vmcb02->control.msrpm_base_pa = vmcb01->control.msrpm_base_pa;
902 vmcb_mark_dirty(vmcb02, VMCB_PERM_MAP);
903
904 /*
905 * Stash vmcb02's counter if the guest hasn't moved past the guilty
906 * instruction; otherwise, reset the counter to '0'.
907 *
908 * In order to detect if L2 has made forward progress or not, track the
909 * RIP at which a bus lock has occurred on a per-vmcb12 basis. If RIP
910 * is changed, guest has clearly made forward progress, bus_lock_counter
911 * still remained '1', so reset bus_lock_counter to '0'. Eg. In the
912 * scenario, where a buslock happened in L1 before VMRUN, the bus lock
913 * firmly happened on an instruction in the past. Even if vmcb01's
914 * counter is still '1', (because the guilty instruction got patched),
915 * the vCPU has clearly made forward progress and so KVM should reset
916 * vmcb02's counter to '0'.
917 *
918 * If the RIP hasn't changed, stash the bus lock counter at nested VMRUN
919 * to prevent the same guilty instruction from triggering a VM-Exit. Eg.
920 * if userspace rate-limits the vCPU, then it's entirely possible that
921 * L1's tick interrupt is pending by the time userspace re-runs the
922 * vCPU. If KVM unconditionally clears the counter on VMRUN, then when
923 * L1 re-enters L2, the same instruction will trigger a VM-Exit and the
924 * entire cycle start over.
925 */
926 if (vmcb02->save.rip && (svm->nested.last_bus_lock_rip == vmcb02->save.rip))
927 vmcb02->control.bus_lock_counter = 1;
928 else
929 vmcb02->control.bus_lock_counter = 0;
930
931 /* Done at vmrun: asid. */
932
933 /* Also overwritten later if necessary. */
934 vmcb02->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
935
936 /* Use vmcb01 MMU and format if guest does not use nNPT */
937 if (nested_npt_enabled(svm)) {
938 vmcb02->control.misc_ctl &= ~SVM_MISC_ENABLE_GMET;
939 vmcb02->control.misc_ctl |= (svm->nested.ctl.misc_ctl & SVM_MISC_ENABLE_GMET);
940
941 nested_svm_init_mmu_context(vcpu);
942 }
943
944 vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(vcpu->arch.l1_tsc_offset,
945 vmcb12_ctrl->tsc_offset,
946 svm->tsc_ratio_msr);
947
948 vmcb02->control.tsc_offset = vcpu->arch.tsc_offset;
949
950 if (guest_cpu_cap_has(vcpu, X86_FEATURE_TSCRATEMSR) &&
951 svm->tsc_ratio_msr != kvm_caps.default_tsc_scaling_ratio)
952 nested_svm_update_tsc_ratio_msr(vcpu);
953
954 vmcb02->control.int_ctl =
955 (vmcb12_ctrl->int_ctl & int_ctl_vmcb12_bits) |
956 (vmcb01->control.int_ctl & int_ctl_vmcb01_bits);
957
958 vmcb02->control.int_vector = vmcb12_ctrl->int_vector;
959 vmcb02->control.int_state = vmcb12_ctrl->int_state;
960 vmcb02->control.event_inj = vmcb12_ctrl->event_inj;
961 vmcb02->control.event_inj_err = vmcb12_ctrl->event_inj_err;
962
963 /*
964 * If nrips is exposed to L1, take NextRIP as-is. Otherwise, L1
965 * advances L2's RIP before VMRUN instead of using NextRIP. KVM will
966 * stuff the current RIP as vmcb02's NextRIP before L2 is run. After
967 * the first run of L2 (e.g. after save+restore), NextRIP is updated by
968 * the CPU and/or KVM and should be used regardless of L1's support.
969 */
970 if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS) ||
971 !vcpu->arch.nested_run_pending)
972 vmcb02->control.next_rip = vmcb12_ctrl->next_rip;
973
974 svm->nmi_l1_to_l2 = is_evtinj_nmi(vmcb02->control.event_inj);
975
976 /*
977 * soft_int_csbase, soft_int_old_rip, and soft_int_next_rip (if L1
978 * doesn't have NRIPS) are initialized later, before the vCPU is run.
979 */
980 if (is_evtinj_soft(vmcb02->control.event_inj)) {
981 svm->soft_int_injected = true;
982 if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS) ||
983 !vcpu->arch.nested_run_pending)
984 svm->soft_int_next_rip = vmcb12_ctrl->next_rip;
985 }
986
987 /* SVM_MISC2_ENABLE_V_LBR is controlled by svm_update_lbrv() */
988
989 if (!nested_vmcb_needs_vls_intercept(svm))
990 vmcb02->control.misc_ctl2 |= SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE;
991
992 if (guest_cpu_cap_has(vcpu, X86_FEATURE_PAUSEFILTER))
993 vmcb02->control.pause_filter_count = vmcb12_ctrl->pause_filter_count;
994 else
995 vmcb02->control.pause_filter_count = 0;
996 if (guest_cpu_cap_has(vcpu, X86_FEATURE_PFTHRESHOLD))
997 vmcb02->control.pause_filter_thresh = vmcb12_ctrl->pause_filter_thresh;
998 else
999 vmcb02->control.pause_filter_thresh = 0;
1000
1001 /*
1002 * Take ALLOW_LARGER_RAP from vmcb12 even though it should be safe to
1003 * let L2 use a larger RAP since KVM will emulate the necessary clears,
1004 * as it's possible L1 deliberately wants to restrict L2 to the legacy
1005 * RAP size. Unconditionally clear the RAP on nested VMRUN, as KVM is
1006 * responsible for emulating the host vs. guest tags (L1 is the "host",
1007 * L2 is the "guest").
1008 */
1009 if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS))
1010 vmcb02->control.erap_ctl = (vmcb12_ctrl->erap_ctl &
1011 ERAP_CONTROL_ALLOW_LARGER_RAP) |
1012 ERAP_CONTROL_CLEAR_RAP;
1013
1014 /*
1015 * Merge guest and host intercepts - must be called with vcpu in
1016 * guest-mode to take effect.
1017 */
1018 nested_vmcb02_recalc_intercepts(svm);
1019 }
1020
nested_svm_copy_common_state(struct vmcb * from_vmcb,struct vmcb * to_vmcb)1021 static void nested_svm_copy_common_state(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
1022 {
1023 /*
1024 * Some VMCB state is shared between L1 and L2 and thus has to be
1025 * moved at the time of nested vmrun and vmexit.
1026 *
1027 * VMLOAD/VMSAVE state would also belong in this category, but KVM
1028 * always performs VMLOAD and VMSAVE from the VMCB01.
1029 */
1030 to_vmcb->save.spec_ctrl = from_vmcb->save.spec_ctrl;
1031 }
1032
enter_svm_guest_mode(struct kvm_vcpu * vcpu,u64 vmcb12_gpa,bool from_vmrun)1033 int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb12_gpa, bool from_vmrun)
1034 {
1035 struct vcpu_svm *svm = to_svm(vcpu);
1036 struct vmcb_ctrl_area_cached *control = &svm->nested.ctl;
1037 struct vmcb_save_area_cached *save = &svm->nested.save;
1038 int ret;
1039
1040 trace_kvm_nested_vmenter(svm->vmcb->save.rip,
1041 vmcb12_gpa,
1042 save->rip,
1043 control->int_ctl,
1044 control->event_inj,
1045 control->misc_ctl,
1046 control->nested_cr3,
1047 save->cr3,
1048 KVM_ISA_SVM);
1049
1050 trace_kvm_nested_intercepts(control->intercepts[INTERCEPT_CR] & 0xffff,
1051 control->intercepts[INTERCEPT_CR] >> 16,
1052 control->intercepts[INTERCEPT_EXCEPTION],
1053 control->intercepts[INTERCEPT_WORD3],
1054 control->intercepts[INTERCEPT_WORD4],
1055 control->intercepts[INTERCEPT_WORD5]);
1056
1057
1058 svm->nested.vmcb12_gpa = vmcb12_gpa;
1059
1060 WARN_ON(svm->vmcb == svm->nested.vmcb02.ptr);
1061
1062 nested_svm_copy_common_state(svm->vmcb01.ptr, svm->nested.vmcb02.ptr);
1063
1064 svm_switch_vmcb(svm, &svm->nested.vmcb02);
1065 nested_vmcb02_prepare_control(svm);
1066 nested_vmcb02_prepare_save(svm);
1067
1068 ret = nested_svm_load_cr3(&svm->vcpu, svm->nested.save.cr3,
1069 nested_npt_enabled(svm), from_vmrun);
1070 if (ret)
1071 return ret;
1072
1073 if (!from_vmrun)
1074 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
1075
1076 svm_set_gif(svm, true);
1077
1078 if (kvm_vcpu_apicv_active(vcpu))
1079 kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);
1080
1081 nested_svm_hv_update_vm_vp_ids(vcpu);
1082
1083 return 0;
1084 }
1085
nested_svm_copy_vmcb12_to_cache(struct kvm_vcpu * vcpu,u64 vmcb12_gpa)1086 static int nested_svm_copy_vmcb12_to_cache(struct kvm_vcpu *vcpu, u64 vmcb12_gpa)
1087 {
1088 struct vcpu_svm *svm = to_svm(vcpu);
1089 struct vmcb *vmcb12;
1090 int r = 0;
1091
1092 CLASS(kvm_vcpu_map_local, m)(vcpu, gpa_to_gfn(vmcb12_gpa));
1093 if (m.ret)
1094 return -EFAULT;
1095
1096 vmcb12 = m.map.hva;
1097 nested_copy_vmcb_control_to_cache(svm, &vmcb12->control);
1098 nested_copy_vmcb_save_to_cache(svm, &vmcb12->save);
1099
1100 if (nested_svm_check_cached_vmcb12(vcpu) < 0) {
1101 vmcb12->control.exit_code = SVM_EXIT_ERR;
1102 vmcb12->control.exit_info_1 = 0;
1103 vmcb12->control.exit_info_2 = 0;
1104 vmcb12->control.event_inj = 0;
1105 vmcb12->control.event_inj_err = 0;
1106 svm_set_gif(svm, false);
1107 r = -EINVAL;
1108 }
1109
1110 return r;
1111 }
1112
nested_svm_vmrun(struct kvm_vcpu * vcpu)1113 int nested_svm_vmrun(struct kvm_vcpu *vcpu)
1114 {
1115 struct vcpu_svm *svm = to_svm(vcpu);
1116 int ret;
1117 u64 vmcb12_gpa;
1118 struct vmcb *vmcb01 = svm->vmcb01.ptr;
1119
1120 if (!svm->nested.hsave_msr) {
1121 kvm_inject_gp(vcpu, 0);
1122 return 1;
1123 }
1124
1125 if (is_smm(vcpu)) {
1126 kvm_queue_exception(vcpu, UD_VECTOR);
1127 return 1;
1128 }
1129
1130 /* This fails when VP assist page is enabled but the supplied GPA is bogus */
1131 ret = kvm_hv_verify_vp_assist(vcpu);
1132 if (ret) {
1133 kvm_inject_gp(vcpu, 0);
1134 return ret;
1135 }
1136
1137 if (WARN_ON_ONCE(!svm->nested.initialized))
1138 return -EINVAL;
1139
1140 vmcb12_gpa = kvm_rax_read(vcpu);
1141 if (!page_address_valid(vcpu, vmcb12_gpa)) {
1142 kvm_inject_gp(vcpu, 0);
1143 return 1;
1144 }
1145
1146 ret = nested_svm_copy_vmcb12_to_cache(vcpu, vmcb12_gpa);
1147 if (ret == -EFAULT)
1148 return kvm_handle_memory_failure(vcpu, X86EMUL_IO_NEEDED, NULL);
1149
1150 /*
1151 * At this point, VMRUN is guaranteed to not fault; advance RIP. If
1152 * caching vmcb12 failed for other reasons, return immediately afterward
1153 * as a nested #VMEXIT was already set up.
1154 *
1155 * FIXME: If TF is set on VMRUN should inject a #DB (or handle guest
1156 * debugging) right after #VMEXIT, right now it's just ignored.
1157 */
1158 if (!svm_skip_emulated_instruction(vcpu))
1159 return 0;
1160
1161 if (ret)
1162 goto insn_retired;
1163
1164 /*
1165 * Since vmcb01 is not in use, we can use it to store some of the L1
1166 * state.
1167 */
1168 vmcb01->save.efer = vcpu->arch.efer;
1169 vmcb01->save.cr0 = kvm_read_cr0(vcpu);
1170 vmcb01->save.cr4 = vcpu->arch.cr4;
1171 vmcb01->save.rflags = kvm_get_rflags(vcpu);
1172 vmcb01->save.rip = kvm_rip_read(vcpu);
1173
1174 if (!npt_enabled)
1175 vmcb01->save.cr3 = kvm_read_cr3(vcpu);
1176
1177 vcpu->arch.nested_run_pending = KVM_NESTED_RUN_PENDING;
1178
1179 if (enter_svm_guest_mode(vcpu, vmcb12_gpa, true) ||
1180 !nested_svm_merge_msrpm(vcpu)) {
1181 vcpu->arch.nested_run_pending = 0;
1182 svm->nmi_l1_to_l2 = false;
1183 svm->soft_int_injected = false;
1184
1185 svm->vmcb->control.exit_code = SVM_EXIT_ERR;
1186 svm->vmcb->control.exit_info_1 = 0;
1187 svm->vmcb->control.exit_info_2 = 0;
1188
1189 nested_svm_vmexit(svm);
1190 }
1191
1192 insn_retired:
1193 /*
1194 * A successful VMRUN is counted by the PMU in guest mode, so only
1195 * retire the instruction after potentially entering guest mode.
1196 */
1197 kvm_pmu_instruction_retired(vcpu);
1198 return 1;
1199 }
1200
1201 /* Copy state save area fields which are handled by VMRUN */
svm_copy_vmrun_state(struct vmcb_save_area * to_save,struct vmcb_save_area * from_save)1202 void svm_copy_vmrun_state(struct vmcb_save_area *to_save,
1203 struct vmcb_save_area *from_save)
1204 {
1205 to_save->es = from_save->es;
1206 to_save->cs = from_save->cs;
1207 to_save->ss = from_save->ss;
1208 to_save->ds = from_save->ds;
1209 to_save->gdtr = from_save->gdtr;
1210 to_save->idtr = from_save->idtr;
1211 to_save->rflags = from_save->rflags | X86_EFLAGS_FIXED;
1212 to_save->efer = from_save->efer;
1213 to_save->cr0 = from_save->cr0;
1214 to_save->cr3 = from_save->cr3;
1215 to_save->cr4 = from_save->cr4;
1216 to_save->rax = from_save->rax;
1217 to_save->rsp = from_save->rsp;
1218 to_save->rip = from_save->rip;
1219 to_save->cpl = 0;
1220
1221 if (kvm_cpu_cap_has(X86_FEATURE_SHSTK)) {
1222 to_save->s_cet = from_save->s_cet;
1223 to_save->isst_addr = from_save->isst_addr;
1224 to_save->ssp = from_save->ssp;
1225 }
1226
1227 if (kvm_cpu_cap_has(X86_FEATURE_LBRV)) {
1228 svm_copy_lbrs(to_save, from_save);
1229 to_save->dbgctl &= ~DEBUGCTL_RESERVED_BITS;
1230 }
1231 }
1232
svm_copy_vmloadsave_state(struct vmcb * to_vmcb,struct vmcb * from_vmcb)1233 void svm_copy_vmloadsave_state(struct vmcb *to_vmcb, struct vmcb *from_vmcb)
1234 {
1235 to_vmcb->save.fs = from_vmcb->save.fs;
1236 to_vmcb->save.gs = from_vmcb->save.gs;
1237 to_vmcb->save.tr = from_vmcb->save.tr;
1238 to_vmcb->save.ldtr = from_vmcb->save.ldtr;
1239 to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
1240 to_vmcb->save.star = from_vmcb->save.star;
1241 to_vmcb->save.lstar = from_vmcb->save.lstar;
1242 to_vmcb->save.cstar = from_vmcb->save.cstar;
1243 to_vmcb->save.sfmask = from_vmcb->save.sfmask;
1244 to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
1245 to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
1246 to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
1247 }
1248
nested_svm_vmexit_update_vmcb12(struct kvm_vcpu * vcpu)1249 static int nested_svm_vmexit_update_vmcb12(struct kvm_vcpu *vcpu)
1250 {
1251 struct vcpu_svm *svm = to_svm(vcpu);
1252 struct vmcb *vmcb02 = svm->nested.vmcb02.ptr;
1253 struct vmcb *vmcb12;
1254
1255 CLASS(kvm_vcpu_map_local, m)(vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa));
1256 if (m.ret)
1257 return m.ret;
1258
1259 vmcb12 = m.map.hva;
1260
1261 vmcb12->save.es = vmcb02->save.es;
1262 vmcb12->save.cs = vmcb02->save.cs;
1263 vmcb12->save.ss = vmcb02->save.ss;
1264 vmcb12->save.ds = vmcb02->save.ds;
1265 vmcb12->save.gdtr = vmcb02->save.gdtr;
1266 vmcb12->save.idtr = vmcb02->save.idtr;
1267 vmcb12->save.efer = svm->vcpu.arch.efer;
1268 vmcb12->save.cr0 = kvm_read_cr0(vcpu);
1269 vmcb12->save.cr3 = kvm_read_cr3(vcpu);
1270 vmcb12->save.cr2 = vcpu->arch.cr2;
1271 vmcb12->save.cr4 = svm->vcpu.arch.cr4;
1272 vmcb12->save.rflags = kvm_get_rflags(vcpu);
1273 vmcb12->save.rip = kvm_rip_read(vcpu);
1274 vmcb12->save.rsp = kvm_rsp_read(vcpu);
1275 vmcb12->save.rax = kvm_rax_read_raw(vcpu);
1276 vmcb12->save.dr7 = vmcb02->save.dr7;
1277 vmcb12->save.dr6 = svm->vcpu.arch.dr6;
1278 vmcb12->save.cpl = vmcb02->save.cpl;
1279
1280 if (l2_has_separate_pat(vcpu))
1281 vmcb12->save.g_pat = vmcb02->save.g_pat;
1282
1283 if (guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK)) {
1284 vmcb12->save.s_cet = vmcb02->save.s_cet;
1285 vmcb12->save.isst_addr = vmcb02->save.isst_addr;
1286 vmcb12->save.ssp = vmcb02->save.ssp;
1287 }
1288
1289 vmcb12->control.int_state = vmcb02->control.int_state;
1290 vmcb12->control.exit_code = vmcb02->control.exit_code;
1291 vmcb12->control.exit_info_1 = vmcb02->control.exit_info_1;
1292 vmcb12->control.exit_info_2 = vmcb02->control.exit_info_2;
1293
1294 if (!svm_is_vmrun_failure(vmcb12->control.exit_code))
1295 nested_save_pending_event_to_vmcb12(svm, vmcb12);
1296
1297 if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS))
1298 vmcb12->control.next_rip = vmcb02->control.next_rip;
1299
1300 if (nested_vmcb12_has_lbrv(vcpu))
1301 svm_copy_lbrs(&vmcb12->save, &vmcb02->save);
1302
1303 vmcb12->control.event_inj = 0;
1304 vmcb12->control.event_inj_err = 0;
1305 vmcb12->control.int_ctl = svm->nested.ctl.int_ctl;
1306
1307 trace_kvm_nested_vmexit_inject(vmcb12->control.exit_code,
1308 vmcb12->control.exit_info_1,
1309 vmcb12->control.exit_info_2,
1310 vmcb12->control.exit_int_info,
1311 vmcb12->control.exit_int_info_err,
1312 KVM_ISA_SVM);
1313
1314 return 0;
1315 }
1316
nested_svm_vmexit(struct vcpu_svm * svm)1317 void nested_svm_vmexit(struct vcpu_svm *svm)
1318 {
1319 struct kvm_vcpu *vcpu = &svm->vcpu;
1320 struct vmcb *vmcb01 = svm->vmcb01.ptr;
1321 struct vmcb *vmcb02 = svm->nested.vmcb02.ptr;
1322
1323 if (nested_svm_vmexit_update_vmcb12(vcpu))
1324 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
1325
1326 /* Exit Guest-Mode */
1327 leave_guest_mode(vcpu);
1328 svm_pmu_handle_nested_transition(svm);
1329
1330 svm->nested.vmcb12_gpa = 0;
1331
1332 kvm_warn_on_nested_run_pending(vcpu);
1333
1334 kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
1335
1336 /* in case we halted in L2 */
1337 kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE);
1338
1339 /*
1340 * Invalidate last_bus_lock_rip unless KVM is still waiting for the
1341 * guest to make forward progress before re-enabling bus lock detection.
1342 */
1343 if (!vmcb02->control.bus_lock_counter)
1344 svm->nested.last_bus_lock_rip = INVALID_GPA;
1345
1346 nested_svm_copy_common_state(svm->nested.vmcb02.ptr, svm->vmcb01.ptr);
1347
1348 kvm_nested_vmexit_handle_ibrs(vcpu);
1349
1350 if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS))
1351 vmcb01->control.erap_ctl |= ERAP_CONTROL_CLEAR_RAP;
1352
1353 svm_switch_vmcb(svm, &svm->vmcb01);
1354
1355 /*
1356 * Rules for synchronizing int_ctl bits from vmcb02 to vmcb01:
1357 *
1358 * V_IRQ, V_IRQ_VECTOR, V_INTR_PRIO_MASK, V_IGN_TPR: If L1 doesn't
1359 * intercept interrupts, then KVM will use vmcb02's V_IRQ (and related
1360 * flags) to detect interrupt windows for L1 IRQs (even if L1 uses
1361 * virtual interrupt masking). Raise KVM_REQ_EVENT to ensure that
1362 * KVM re-requests an interrupt window if necessary, which implicitly
1363 * copies this bits from vmcb02 to vmcb01.
1364 *
1365 * V_TPR: If L1 doesn't use virtual interrupt masking, then L1's vTPR
1366 * is stored in vmcb02, but its value doesn't need to be copied from/to
1367 * vmcb01 because it is copied from/to the virtual APIC's TPR register
1368 * on each VM entry/exit.
1369 *
1370 * V_GIF: If nested vGIF is not used, KVM uses vmcb02's V_GIF for L1's
1371 * V_GIF. However, GIF is architecturally clear on each VM exit, thus
1372 * there is no need to copy V_GIF from vmcb02 to vmcb01.
1373 */
1374 if (!nested_exit_on_intr(svm))
1375 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
1376
1377 if (!nested_vmcb12_has_lbrv(vcpu)) {
1378 svm_copy_lbrs(&vmcb01->save, &vmcb02->save);
1379 vmcb_mark_dirty(vmcb01, VMCB_LBR);
1380 }
1381
1382 svm_update_lbrv(vcpu);
1383
1384 if (vnmi) {
1385 if (vmcb02->control.int_ctl & V_NMI_BLOCKING_MASK)
1386 vmcb01->control.int_ctl |= V_NMI_BLOCKING_MASK;
1387 else
1388 vmcb01->control.int_ctl &= ~V_NMI_BLOCKING_MASK;
1389
1390 if (vcpu->arch.nmi_pending) {
1391 vcpu->arch.nmi_pending--;
1392 vmcb01->control.int_ctl |= V_NMI_PENDING_MASK;
1393 } else {
1394 vmcb01->control.int_ctl &= ~V_NMI_PENDING_MASK;
1395 }
1396 }
1397
1398 /*
1399 * On vmexit the GIF is set to false and
1400 * no event can be injected in L1.
1401 */
1402 svm_set_gif(svm, false);
1403 vmcb01->control.exit_int_info = 0;
1404
1405 svm->vcpu.arch.tsc_offset = svm->vcpu.arch.l1_tsc_offset;
1406 if (vmcb01->control.tsc_offset != svm->vcpu.arch.tsc_offset) {
1407 vmcb01->control.tsc_offset = svm->vcpu.arch.tsc_offset;
1408 vmcb_mark_dirty(vmcb01, VMCB_INTERCEPTS);
1409 }
1410
1411 if (kvm_caps.has_tsc_control &&
1412 vcpu->arch.tsc_scaling_ratio != vcpu->arch.l1_tsc_scaling_ratio) {
1413 vcpu->arch.tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio;
1414 svm_write_tsc_multiplier(vcpu);
1415 }
1416
1417 svm->nested.ctl.nested_cr3 = 0;
1418
1419 /*
1420 * Restore processor state that had been saved in vmcb01
1421 */
1422 kvm_set_rflags(vcpu, vmcb01->save.rflags);
1423 svm_set_efer(vcpu, vmcb01->save.efer);
1424 svm_set_cr0(vcpu, vmcb01->save.cr0 | X86_CR0_PE);
1425 svm_set_cr4(vcpu, vmcb01->save.cr4);
1426 kvm_rax_write_raw(vcpu, vmcb01->save.rax);
1427 kvm_rsp_write(vcpu, vmcb01->save.rsp);
1428 kvm_rip_write(vcpu, vmcb01->save.rip);
1429
1430 svm->vcpu.arch.dr7 = DR7_FIXED_1;
1431 kvm_update_dr7(&svm->vcpu);
1432
1433 nested_svm_transition_tlb_flush(vcpu);
1434
1435 nested_svm_uninit_mmu_context(vcpu);
1436
1437 if (nested_svm_load_cr3(vcpu, vmcb01->save.cr3, false, true))
1438 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
1439
1440 /* Drop tracking for L1->L2 injected NMIs and soft IRQs */
1441 svm->nmi_l1_to_l2 = false;
1442 svm->soft_int_injected = false;
1443
1444 /*
1445 * Drop what we picked up for L2 via svm_complete_interrupts() so it
1446 * doesn't end up in L1.
1447 */
1448 svm->vcpu.arch.nmi_injected = false;
1449 kvm_clear_exception_queue(vcpu);
1450 kvm_clear_interrupt_queue(vcpu);
1451
1452 /*
1453 * If we are here following the completion of a VMRUN that
1454 * is being single-stepped, queue the pending #DB intercept
1455 * right now so that it an be accounted for before we execute
1456 * L1's next instruction.
1457 */
1458 if (unlikely(vmcb01->save.rflags & X86_EFLAGS_TF))
1459 kvm_queue_exception(&(svm->vcpu), DB_VECTOR);
1460
1461 /*
1462 * Un-inhibit the AVIC right away, so that other vCPUs can start
1463 * to benefit from it right away.
1464 */
1465 if (kvm_apicv_activated(vcpu->kvm))
1466 __kvm_vcpu_update_apicv(vcpu);
1467 }
1468
nested_svm_triple_fault(struct kvm_vcpu * vcpu)1469 static void nested_svm_triple_fault(struct kvm_vcpu *vcpu)
1470 {
1471 struct vcpu_svm *svm = to_svm(vcpu);
1472
1473 if (!vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SHUTDOWN))
1474 return;
1475
1476 kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu);
1477 nested_svm_simple_vmexit(to_svm(vcpu), SVM_EXIT_SHUTDOWN);
1478 }
1479
svm_allocate_nested(struct vcpu_svm * svm)1480 int svm_allocate_nested(struct vcpu_svm *svm)
1481 {
1482 struct page *vmcb02_page;
1483
1484 if (svm->nested.initialized)
1485 return 0;
1486
1487 vmcb02_page = snp_safe_alloc_page();
1488 if (!vmcb02_page)
1489 return -ENOMEM;
1490 svm->nested.vmcb02.ptr = page_address(vmcb02_page);
1491 svm->nested.vmcb02.pa = __sme_set(page_to_pfn(vmcb02_page) << PAGE_SHIFT);
1492
1493 svm->nested.msrpm = svm_vcpu_alloc_msrpm();
1494 if (!svm->nested.msrpm)
1495 goto err_free_vmcb02;
1496
1497 svm->nested.initialized = true;
1498 return 0;
1499
1500 err_free_vmcb02:
1501 __free_page(vmcb02_page);
1502 return -ENOMEM;
1503 }
1504
svm_free_nested(struct vcpu_svm * svm)1505 void svm_free_nested(struct vcpu_svm *svm)
1506 {
1507 if (!svm->nested.initialized)
1508 return;
1509
1510 if (WARN_ON_ONCE(svm->vmcb != svm->vmcb01.ptr))
1511 svm_switch_vmcb(svm, &svm->vmcb01);
1512
1513 svm_vcpu_free_msrpm(svm->nested.msrpm);
1514 svm->nested.msrpm = NULL;
1515
1516 __free_page(virt_to_page(svm->nested.vmcb02.ptr));
1517 svm->nested.vmcb02.ptr = NULL;
1518
1519 /*
1520 * When last_vmcb12_gpa matches the current vmcb12 gpa,
1521 * some vmcb12 fields are not loaded if they are marked clean
1522 * in the vmcb12, since in this case they are up to date already.
1523 *
1524 * When the vmcb02 is freed, this optimization becomes invalid.
1525 */
1526 svm->nested.last_vmcb12_gpa = INVALID_GPA;
1527
1528 svm->nested.initialized = false;
1529 }
1530
svm_leave_nested(struct kvm_vcpu * vcpu)1531 void svm_leave_nested(struct kvm_vcpu *vcpu)
1532 {
1533 struct vcpu_svm *svm = to_svm(vcpu);
1534
1535 if (is_guest_mode(vcpu)) {
1536 vcpu->arch.nested_run_pending = 0;
1537 svm->nested.vmcb12_gpa = INVALID_GPA;
1538
1539 leave_guest_mode(vcpu);
1540
1541 /*
1542 * Force leaving nested is a non-architectural flow so precision
1543 * isn't a priority. Defer updating the PMU until the next vCPU
1544 * run, potentially tolerating some imprecision to avoid poking
1545 * into PMU state from arbitrary contexts (e.g. to avoid using
1546 * stale state).
1547 */
1548 __svm_pmu_handle_nested_transition(svm, true);
1549
1550 svm_switch_vmcb(svm, &svm->vmcb01);
1551
1552 nested_svm_uninit_mmu_context(vcpu);
1553 vmcb_mark_all_dirty(svm->vmcb);
1554
1555 svm_set_gif(svm, true);
1556
1557 if (kvm_apicv_activated(vcpu->kvm))
1558 kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);
1559 }
1560
1561 kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
1562 }
1563
nested_svm_exit_handled_msr(struct vcpu_svm * svm)1564 static int nested_svm_exit_handled_msr(struct vcpu_svm *svm)
1565 {
1566 gpa_t base = svm->nested.ctl.msrpm_base_pa;
1567 int write, bit_nr;
1568 u8 value, mask;
1569 u32 msr;
1570
1571 if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT)))
1572 return NESTED_EXIT_HOST;
1573
1574 msr = svm->vcpu.arch.regs[VCPU_REGS_RCX];
1575 bit_nr = svm_msrpm_bit_nr(msr);
1576 write = svm->vmcb->control.exit_info_1 & 1;
1577
1578 if (bit_nr < 0)
1579 return NESTED_EXIT_DONE;
1580
1581 if (kvm_vcpu_read_guest(&svm->vcpu, base + bit_nr / BITS_PER_BYTE,
1582 &value, sizeof(value)))
1583 return NESTED_EXIT_DONE;
1584
1585 mask = BIT(write) << (bit_nr & (BITS_PER_BYTE - 1));
1586 return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
1587 }
1588
nested_svm_intercept_ioio(struct vcpu_svm * svm)1589 static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
1590 {
1591 unsigned port, size, iopm_len;
1592 u16 val, mask;
1593 u8 start_bit;
1594 u64 gpa;
1595
1596 if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_IOIO_PROT)))
1597 return NESTED_EXIT_HOST;
1598
1599 port = svm->vmcb->control.exit_info_1 >> 16;
1600 size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >>
1601 SVM_IOIO_SIZE_SHIFT;
1602 gpa = svm->nested.ctl.iopm_base_pa + (port / 8);
1603 start_bit = port % 8;
1604 iopm_len = (start_bit + size > 8) ? 2 : 1;
1605 mask = (0xf >> (4 - size)) << start_bit;
1606 val = 0;
1607
1608 if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len))
1609 return NESTED_EXIT_DONE;
1610
1611 return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
1612 }
1613
nested_svm_intercept(struct vcpu_svm * svm)1614 static int nested_svm_intercept(struct vcpu_svm *svm)
1615 {
1616 u64 exit_code = svm->vmcb->control.exit_code;
1617 int vmexit = NESTED_EXIT_HOST;
1618
1619 if (svm_is_vmrun_failure(exit_code))
1620 return NESTED_EXIT_DONE;
1621
1622 switch (exit_code) {
1623 case SVM_EXIT_MSR:
1624 vmexit = nested_svm_exit_handled_msr(svm);
1625 break;
1626 case SVM_EXIT_IOIO:
1627 vmexit = nested_svm_intercept_ioio(svm);
1628 break;
1629 case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f:
1630 /*
1631 * Host-intercepted exceptions have been checked already in
1632 * nested_svm_exit_special. There is nothing to do here,
1633 * the vmexit is injected by svm_check_nested_events.
1634 */
1635 vmexit = NESTED_EXIT_DONE;
1636 break;
1637 default:
1638 if (vmcb12_is_intercept(&svm->nested.ctl, exit_code))
1639 vmexit = NESTED_EXIT_DONE;
1640 break;
1641 }
1642
1643 return vmexit;
1644 }
1645
nested_svm_exit_handled(struct vcpu_svm * svm)1646 int nested_svm_exit_handled(struct vcpu_svm *svm)
1647 {
1648 int vmexit;
1649
1650 vmexit = nested_svm_intercept(svm);
1651
1652 if (vmexit == NESTED_EXIT_DONE)
1653 nested_svm_vmexit(svm);
1654
1655 return vmexit;
1656 }
1657
nested_svm_check_permissions(struct kvm_vcpu * vcpu)1658 int nested_svm_check_permissions(struct kvm_vcpu *vcpu)
1659 {
1660 if (!(vcpu->arch.efer & EFER_SVME) || !is_paging(vcpu)) {
1661 kvm_queue_exception(vcpu, UD_VECTOR);
1662 return 1;
1663 }
1664
1665 if (to_svm(vcpu)->vmcb->save.cpl) {
1666 kvm_inject_gp(vcpu, 0);
1667 return 1;
1668 }
1669
1670 return 0;
1671 }
1672
nested_svm_is_exception_vmexit(struct kvm_vcpu * vcpu,u8 vector,u32 error_code)1673 static bool nested_svm_is_exception_vmexit(struct kvm_vcpu *vcpu, u8 vector,
1674 u32 error_code)
1675 {
1676 struct vcpu_svm *svm = to_svm(vcpu);
1677
1678 return (svm->nested.ctl.intercepts[INTERCEPT_EXCEPTION] & BIT(vector));
1679 }
1680
nested_svm_inject_exception_vmexit(struct kvm_vcpu * vcpu)1681 static void nested_svm_inject_exception_vmexit(struct kvm_vcpu *vcpu)
1682 {
1683 struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit;
1684 struct vcpu_svm *svm = to_svm(vcpu);
1685 struct vmcb *vmcb = svm->vmcb;
1686
1687 vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + ex->vector;
1688
1689 if (ex->has_error_code)
1690 vmcb->control.exit_info_1 = ex->error_code;
1691
1692 /*
1693 * EXITINFO2 is undefined for all exception intercepts other
1694 * than #PF.
1695 */
1696 if (ex->vector == PF_VECTOR) {
1697 if (ex->has_payload)
1698 vmcb->control.exit_info_2 = ex->payload;
1699 else
1700 vmcb->control.exit_info_2 = vcpu->arch.cr2;
1701 } else if (ex->vector == DB_VECTOR) {
1702 /* See kvm_check_and_inject_events(). */
1703 kvm_deliver_exception_payload(vcpu, ex);
1704
1705 if (vcpu->arch.dr7 & DR7_GD) {
1706 vcpu->arch.dr7 &= ~DR7_GD;
1707 kvm_update_dr7(vcpu);
1708 }
1709 } else {
1710 WARN_ON(ex->has_payload);
1711 }
1712
1713 nested_svm_vmexit(svm);
1714 }
1715
nested_exit_on_init(struct vcpu_svm * svm)1716 static inline bool nested_exit_on_init(struct vcpu_svm *svm)
1717 {
1718 return vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_INIT);
1719 }
1720
svm_check_nested_events(struct kvm_vcpu * vcpu)1721 static int svm_check_nested_events(struct kvm_vcpu *vcpu)
1722 {
1723 struct kvm_lapic *apic = vcpu->arch.apic;
1724 struct vcpu_svm *svm = to_svm(vcpu);
1725 /*
1726 * Only a pending nested run blocks a pending exception. If there is a
1727 * previously injected event, the pending exception occurred while said
1728 * event was being delivered and thus needs to be handled.
1729 */
1730 bool block_nested_exceptions = vcpu->arch.nested_run_pending;
1731 /*
1732 * New events (not exceptions) are only recognized at instruction
1733 * boundaries. If an event needs reinjection, then KVM is handling a
1734 * VM-Exit that occurred _during_ instruction execution; new events are
1735 * blocked until the instruction completes.
1736 */
1737 bool block_nested_events = block_nested_exceptions ||
1738 kvm_event_needs_reinjection(vcpu);
1739
1740 if (lapic_in_kernel(vcpu) &&
1741 test_bit(KVM_APIC_INIT, &apic->pending_events)) {
1742 if (block_nested_events)
1743 return -EBUSY;
1744 if (!nested_exit_on_init(svm))
1745 return 0;
1746 nested_svm_simple_vmexit(svm, SVM_EXIT_INIT);
1747 return 0;
1748 }
1749
1750 if (vcpu->arch.exception_vmexit.pending) {
1751 if (block_nested_exceptions)
1752 return -EBUSY;
1753 nested_svm_inject_exception_vmexit(vcpu);
1754 return 0;
1755 }
1756
1757 if (vcpu->arch.exception.pending) {
1758 if (block_nested_exceptions)
1759 return -EBUSY;
1760 return 0;
1761 }
1762
1763 #ifdef CONFIG_KVM_SMM
1764 if (vcpu->arch.smi_pending && !svm_smi_blocked(vcpu)) {
1765 if (block_nested_events)
1766 return -EBUSY;
1767 if (!nested_exit_on_smi(svm))
1768 return 0;
1769 nested_svm_simple_vmexit(svm, SVM_EXIT_SMI);
1770 return 0;
1771 }
1772 #endif
1773
1774 if (vcpu->arch.nmi_pending && !svm_nmi_blocked(vcpu)) {
1775 if (block_nested_events)
1776 return -EBUSY;
1777 if (!nested_exit_on_nmi(svm))
1778 return 0;
1779 nested_svm_simple_vmexit(svm, SVM_EXIT_NMI);
1780 return 0;
1781 }
1782
1783 if (kvm_cpu_has_interrupt(vcpu) && !svm_interrupt_blocked(vcpu)) {
1784 if (block_nested_events)
1785 return -EBUSY;
1786 if (!nested_exit_on_intr(svm))
1787 return 0;
1788 trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
1789 nested_svm_simple_vmexit(svm, SVM_EXIT_INTR);
1790 return 0;
1791 }
1792
1793 return 0;
1794 }
1795
nested_svm_exit_special(struct vcpu_svm * svm)1796 int nested_svm_exit_special(struct vcpu_svm *svm)
1797 {
1798 u32 exit_code = svm->vmcb->control.exit_code;
1799 struct kvm_vcpu *vcpu = &svm->vcpu;
1800
1801 switch (exit_code) {
1802 case SVM_EXIT_INTR:
1803 case SVM_EXIT_NMI:
1804 case SVM_EXIT_NPF:
1805 return NESTED_EXIT_HOST;
1806 case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
1807 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
1808
1809 if (svm->vmcb01.ptr->control.intercepts[INTERCEPT_EXCEPTION] &
1810 excp_bits)
1811 return NESTED_EXIT_HOST;
1812 else if (exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR &&
1813 svm->vcpu.arch.apf.host_apf_flags)
1814 /* Trap async PF even if not shadowing */
1815 return NESTED_EXIT_HOST;
1816 break;
1817 }
1818 case SVM_EXIT_VMMCALL:
1819 /* Hyper-V L2 TLB flush hypercall is handled by L0 */
1820 if (nested_svm_is_l2_tlb_flush_hcall(vcpu))
1821 return NESTED_EXIT_HOST;
1822 break;
1823 default:
1824 break;
1825 }
1826
1827 return NESTED_EXIT_CONTINUE;
1828 }
1829
nested_svm_update_tsc_ratio_msr(struct kvm_vcpu * vcpu)1830 void nested_svm_update_tsc_ratio_msr(struct kvm_vcpu *vcpu)
1831 {
1832 struct vcpu_svm *svm = to_svm(vcpu);
1833
1834 vcpu->arch.tsc_scaling_ratio =
1835 kvm_calc_nested_tsc_multiplier(vcpu->arch.l1_tsc_scaling_ratio,
1836 svm->tsc_ratio_msr);
1837 svm_write_tsc_multiplier(vcpu);
1838 }
1839
1840 /* Inverse operation of nested_copy_vmcb_control_to_cache(). asid is copied too. */
nested_copy_vmcb_cache_to_control(struct vmcb_control_area * dst,struct vmcb_ctrl_area_cached * from)1841 static void nested_copy_vmcb_cache_to_control(struct vmcb_control_area *dst,
1842 struct vmcb_ctrl_area_cached *from)
1843 {
1844 unsigned int i;
1845
1846 memset(dst, 0, sizeof(struct vmcb_control_area));
1847
1848 for (i = 0; i < MAX_INTERCEPT; i++)
1849 dst->intercepts[i] = from->intercepts[i];
1850
1851 dst->iopm_base_pa = from->iopm_base_pa;
1852 dst->msrpm_base_pa = from->msrpm_base_pa;
1853 dst->tsc_offset = from->tsc_offset;
1854 dst->asid = from->asid;
1855 dst->tlb_ctl = from->tlb_ctl;
1856 dst->erap_ctl = from->erap_ctl;
1857 dst->int_ctl = from->int_ctl;
1858 dst->int_vector = from->int_vector;
1859 dst->int_state = from->int_state;
1860 dst->exit_code = from->exit_code;
1861 dst->exit_info_1 = from->exit_info_1;
1862 dst->exit_info_2 = from->exit_info_2;
1863 dst->exit_int_info = from->exit_int_info;
1864 dst->exit_int_info_err = from->exit_int_info_err;
1865 dst->misc_ctl = from->misc_ctl;
1866 dst->event_inj = from->event_inj;
1867 dst->event_inj_err = from->event_inj_err;
1868 dst->next_rip = from->next_rip;
1869 dst->nested_cr3 = from->nested_cr3;
1870 dst->misc_ctl2 = from->misc_ctl2;
1871 dst->pause_filter_count = from->pause_filter_count;
1872 dst->pause_filter_thresh = from->pause_filter_thresh;
1873 /* 'clean' and 'hv_enlightenments' are not changed by KVM */
1874 }
1875
svm_get_nested_state(struct kvm_vcpu * vcpu,struct kvm_nested_state __user * user_kvm_nested_state,u32 user_data_size)1876 static int svm_get_nested_state(struct kvm_vcpu *vcpu,
1877 struct kvm_nested_state __user *user_kvm_nested_state,
1878 u32 user_data_size)
1879 {
1880 struct vcpu_svm *svm;
1881 struct vmcb_control_area *ctl;
1882 unsigned long r;
1883 struct kvm_nested_state kvm_state = {
1884 .flags = 0,
1885 .format = KVM_STATE_NESTED_FORMAT_SVM,
1886 .size = sizeof(kvm_state),
1887 };
1888 struct vmcb __user *user_vmcb = (struct vmcb __user *)
1889 &user_kvm_nested_state->data.svm[0];
1890
1891 if (!vcpu)
1892 return kvm_state.size + KVM_STATE_NESTED_SVM_VMCB_SIZE;
1893
1894 svm = to_svm(vcpu);
1895
1896 if (user_data_size < kvm_state.size)
1897 goto out;
1898
1899 /* First fill in the header and copy it out. */
1900 if (is_guest_mode(vcpu)) {
1901 kvm_state.hdr.svm.vmcb_pa = svm->nested.vmcb12_gpa;
1902 kvm_state.hdr.svm.gpat = 0;
1903 if (l2_has_separate_pat(vcpu))
1904 kvm_state.hdr.svm.gpat = svm->vmcb->save.g_pat;
1905 kvm_state.size += KVM_STATE_NESTED_SVM_VMCB_SIZE;
1906 kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE;
1907
1908 if (vcpu->arch.nested_run_pending)
1909 kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING;
1910 }
1911
1912 if (gif_set(svm))
1913 kvm_state.flags |= KVM_STATE_NESTED_GIF_SET;
1914
1915 if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state)))
1916 return -EFAULT;
1917
1918 if (!is_guest_mode(vcpu))
1919 goto out;
1920
1921 /*
1922 * Copy over the full size of the VMCB rather than just the size
1923 * of the structs.
1924 */
1925 if (clear_user(user_vmcb, KVM_STATE_NESTED_SVM_VMCB_SIZE))
1926 return -EFAULT;
1927
1928 ctl = kzalloc_obj(*ctl);
1929 if (!ctl)
1930 return -ENOMEM;
1931
1932 nested_copy_vmcb_cache_to_control(ctl, &svm->nested.ctl);
1933 r = copy_to_user(&user_vmcb->control, ctl,
1934 sizeof(user_vmcb->control));
1935 kfree(ctl);
1936 if (r)
1937 return -EFAULT;
1938
1939 if (copy_to_user(&user_vmcb->save, &svm->vmcb01.ptr->save,
1940 sizeof(user_vmcb->save)))
1941 return -EFAULT;
1942 out:
1943 return kvm_state.size;
1944 }
1945
svm_set_nested_state(struct kvm_vcpu * vcpu,struct kvm_nested_state __user * user_kvm_nested_state,struct kvm_nested_state * kvm_state)1946 static int svm_set_nested_state(struct kvm_vcpu *vcpu,
1947 struct kvm_nested_state __user *user_kvm_nested_state,
1948 struct kvm_nested_state *kvm_state)
1949 {
1950 struct vcpu_svm *svm = to_svm(vcpu);
1951 struct vmcb __user *user_vmcb = (struct vmcb __user *)
1952 &user_kvm_nested_state->data.svm[0];
1953 struct vmcb_control_area *ctl;
1954 struct vmcb_save_area *save;
1955 struct vmcb_save_area_cached save_cached;
1956 struct vmcb_ctrl_area_cached ctl_cached;
1957 bool use_separate_l2_pat;
1958 unsigned long cr0;
1959 int ret;
1960
1961 BUILD_BUG_ON(sizeof(struct vmcb_control_area) + sizeof(struct vmcb_save_area) >
1962 KVM_STATE_NESTED_SVM_VMCB_SIZE);
1963
1964 if (kvm_state->format != KVM_STATE_NESTED_FORMAT_SVM)
1965 return -EINVAL;
1966
1967 if (kvm_state->flags & ~(KVM_STATE_NESTED_GUEST_MODE |
1968 KVM_STATE_NESTED_RUN_PENDING |
1969 KVM_STATE_NESTED_GIF_SET))
1970 return -EINVAL;
1971
1972 /*
1973 * If in guest mode, vcpu->arch.efer actually refers to the L2 guest's
1974 * EFER.SVME, but EFER.SVME still has to be 1 for VMRUN to succeed.
1975 * If SVME is disabled, the only valid states are "none" and GIF=1
1976 * (clearing SVME does NOT set GIF, i.e. GIF=0 is allowed).
1977 */
1978 if (!(vcpu->arch.efer & EFER_SVME) && kvm_state->flags &&
1979 kvm_state->flags != KVM_STATE_NESTED_GIF_SET)
1980 return -EINVAL;
1981
1982 /* SMM temporarily disables SVM, so we cannot be in guest mode. */
1983 if (is_smm(vcpu) && (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE))
1984 return -EINVAL;
1985
1986 if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) {
1987 svm_leave_nested(vcpu);
1988 svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET));
1989 return 0;
1990 }
1991
1992 if (!page_address_valid(vcpu, kvm_state->hdr.svm.vmcb_pa))
1993 return -EINVAL;
1994 if (kvm_state->size < sizeof(*kvm_state) + KVM_STATE_NESTED_SVM_VMCB_SIZE)
1995 return -EINVAL;
1996
1997 ctl = memdup_user(&user_vmcb->control, sizeof(*ctl));
1998 if (IS_ERR(ctl))
1999 return PTR_ERR(ctl);
2000
2001 save = memdup_user(&user_vmcb->save, sizeof(*save));
2002 if (IS_ERR(save)) {
2003 kfree(ctl);
2004 return PTR_ERR(save);
2005 }
2006
2007 ret = -EINVAL;
2008 __nested_copy_vmcb_control_to_cache(vcpu, &ctl_cached, ctl);
2009 if (!nested_vmcb_check_controls(vcpu, &ctl_cached))
2010 goto out_free;
2011
2012 /*
2013 * Processor state contains L2 state. Check that it is
2014 * valid for guest mode (see nested_vmcb_check_save()).
2015 */
2016 cr0 = kvm_read_cr0(vcpu);
2017 if (((cr0 & X86_CR0_CD) == 0) && (cr0 & X86_CR0_NW))
2018 goto out_free;
2019
2020 /*
2021 * Validate host state saved from before VMRUN (see
2022 * nested_svm_check_permissions). Note that the g_pat field is not
2023 * validated, because (a) it may have been clobbered by SMM before
2024 * KVM_GET_NESTED_STATE, and (b) it is not loaded at emulated
2025 * #VMEXIT.
2026 */
2027 __nested_copy_vmcb_save_to_cache(&save_cached, save);
2028 if (!(save->cr0 & X86_CR0_PG) ||
2029 !(save->cr0 & X86_CR0_PE) ||
2030 (save->rflags & X86_EFLAGS_VM) ||
2031 !nested_vmcb_check_save(vcpu, &save_cached, false))
2032 goto out_free;
2033
2034 /*
2035 * Validate gPAT when the shared PAT quirk is disabled (i.e. L2
2036 * has its own gPAT). This is done separately from the
2037 * vmcb_save_area_cached validation above, because gPAT is L2
2038 * state, but the vmcb_save_area_cached is populated with L1 state.
2039 */
2040 use_separate_l2_pat = (ctl_cached.misc_ctl & SVM_MISC_ENABLE_NP) &&
2041 !kvm_check_has_quirk(vcpu->kvm,
2042 KVM_X86_QUIRK_NESTED_SVM_SHARED_PAT);
2043 if (use_separate_l2_pat && !kvm_pat_valid(kvm_state->hdr.svm.gpat))
2044 goto out_free;
2045
2046 /*
2047 * All checks done, we can enter guest mode. Userspace provides
2048 * vmcb12.control, which will be combined with L1 and stored into
2049 * vmcb02, and the L1 save state which we store in vmcb01.
2050 * L2 registers if needed are moved from the current VMCB to VMCB02.
2051 */
2052
2053 if (is_guest_mode(vcpu))
2054 svm_leave_nested(vcpu);
2055 else
2056 svm->nested.vmcb02.ptr->save = svm->vmcb01.ptr->save;
2057
2058 svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET));
2059
2060 if (kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING)
2061 vcpu->arch.nested_run_pending = KVM_NESTED_RUN_PENDING_UNTRUSTED;
2062 else
2063 vcpu->arch.nested_run_pending = 0;
2064
2065 svm->nested.vmcb12_gpa = kvm_state->hdr.svm.vmcb_pa;
2066
2067 svm_copy_vmrun_state(&svm->vmcb01.ptr->save, save);
2068 nested_copy_vmcb_control_to_cache(svm, ctl);
2069
2070 svm_switch_vmcb(svm, &svm->nested.vmcb02);
2071
2072 if (use_separate_l2_pat)
2073 vmcb_set_gpat(svm->vmcb, kvm_state->hdr.svm.gpat);
2074
2075 nested_vmcb02_prepare_control(svm);
2076
2077 /*
2078 * Any previously restored state (e.g. KVM_SET_SREGS) would mark fields
2079 * dirty in vmcb01 instead of vmcb02, so mark all of vmcb02 dirty here.
2080 */
2081 vmcb_mark_all_dirty(svm->vmcb);
2082
2083 /*
2084 * While the nested guest CR3 is already checked and set by
2085 * KVM_SET_SREGS, it was set when nested state was yet loaded,
2086 * thus MMU might not be initialized correctly.
2087 * Set it again to fix this.
2088 */
2089 ret = nested_svm_load_cr3(&svm->vcpu, vcpu->arch.cr3,
2090 nested_npt_enabled(svm), false);
2091 if (ret)
2092 goto out_free;
2093
2094 svm->nested.force_msr_bitmap_recalc = true;
2095
2096 if (kvm_vcpu_apicv_active(vcpu))
2097 kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);
2098
2099 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
2100 out_free:
2101 kfree(save);
2102 kfree(ctl);
2103
2104 return ret;
2105 }
2106
svm_get_nested_state_pages(struct kvm_vcpu * vcpu)2107 static bool svm_get_nested_state_pages(struct kvm_vcpu *vcpu)
2108 {
2109 if (WARN_ON(!is_guest_mode(vcpu)))
2110 return true;
2111
2112 if (is_pae_paging(vcpu)) {
2113 /*
2114 * After migration, CR3 may have been restored before
2115 * KVM_SET_NESTED_STATE, so the PDPTR load into mmu->pdptrs[]
2116 * may have treated CR3 as an L1 GPA. For nNPT, drop the
2117 * cache so the next access reloads them with the proper
2118 * nGPA translation. For !nNPT, reload eagerly unless userspace
2119 * already supplied authoritative PDPTRs via KVM_SET_SREGS2.
2120 */
2121 if (nested_npt_enabled(to_svm(vcpu)))
2122 kvm_register_mark_for_reload(vcpu, VCPU_REG_PDPTR);
2123 else if (!vcpu->arch.pdptrs_from_userspace &&
2124 CC(!load_pdptrs(vcpu, vcpu->arch.cr3)))
2125 return false;
2126 }
2127
2128 if (!nested_svm_merge_msrpm(vcpu))
2129 return false;
2130
2131 if (kvm_hv_verify_vp_assist(vcpu))
2132 return false;
2133
2134 return true;
2135 }
2136
svm_translate_nested_gpa(struct kvm_vcpu * vcpu,gpa_t gpa,u64 access,struct x86_exception * exception,u64 pte_access)2137 static gpa_t svm_translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa,
2138 u64 access,
2139 struct x86_exception *exception,
2140 u64 pte_access)
2141 {
2142 struct vcpu_svm *svm = to_svm(vcpu);
2143 struct kvm_pagewalk *w = &vcpu->arch.ngpa_walk;
2144
2145 if (WARN_ON_ONCE(!mmu_is_nested(vcpu)))
2146 return gpa;
2147
2148 /* Non-GMET walks are always user-walks */
2149 if (!(svm->nested.ctl.misc_ctl & SVM_MISC_ENABLE_GMET))
2150 access |= PFERR_USER_MASK;
2151
2152 return w->gva_to_gpa(vcpu, w, gpa, access, exception);
2153 }
2154
2155 struct kvm_x86_nested_ops svm_nested_ops __initdata = {
2156 .leave_nested = svm_leave_nested,
2157 .translate_nested_gpa = svm_translate_nested_gpa,
2158 .is_exception_vmexit = nested_svm_is_exception_vmexit,
2159 .check_events = svm_check_nested_events,
2160 .triple_fault = nested_svm_triple_fault,
2161 .get_nested_state_pages = svm_get_nested_state_pages,
2162 .get_state = svm_get_nested_state,
2163 .set_state = svm_set_nested_state,
2164 .hv_inject_synthetic_vmexit_post_tlb_flush = svm_hv_inject_synthetic_vmexit_post_tlb_flush,
2165 };
2166