xref: /linux/arch/x86/kvm/svm/svm.c (revision d7f39aee79f04eeaa42085728423501b33ac5be5)
1 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
2 
3 #include <linux/kvm_host.h>
4 
5 #include "irq.h"
6 #include "mmu.h"
7 #include "kvm_cache_regs.h"
8 #include "x86.h"
9 #include "smm.h"
10 #include "cpuid.h"
11 #include "pmu.h"
12 
13 #include <linux/module.h>
14 #include <linux/mod_devicetable.h>
15 #include <linux/kernel.h>
16 #include <linux/vmalloc.h>
17 #include <linux/highmem.h>
18 #include <linux/amd-iommu.h>
19 #include <linux/sched.h>
20 #include <linux/trace_events.h>
21 #include <linux/slab.h>
22 #include <linux/hashtable.h>
23 #include <linux/objtool.h>
24 #include <linux/psp-sev.h>
25 #include <linux/file.h>
26 #include <linux/pagemap.h>
27 #include <linux/swap.h>
28 #include <linux/rwsem.h>
29 #include <linux/cc_platform.h>
30 #include <linux/smp.h>
31 
32 #include <asm/apic.h>
33 #include <asm/perf_event.h>
34 #include <asm/tlbflush.h>
35 #include <asm/desc.h>
36 #include <asm/debugreg.h>
37 #include <asm/kvm_para.h>
38 #include <asm/irq_remapping.h>
39 #include <asm/spec-ctrl.h>
40 #include <asm/cpu_device_id.h>
41 #include <asm/traps.h>
42 #include <asm/reboot.h>
43 #include <asm/fpu/api.h>
44 
45 #include <trace/events/ipi.h>
46 
47 #include "trace.h"
48 
49 #include "svm.h"
50 #include "svm_ops.h"
51 
52 #include "kvm_onhyperv.h"
53 #include "svm_onhyperv.h"
54 
55 MODULE_AUTHOR("Qumranet");
56 MODULE_LICENSE("GPL");
57 
58 #ifdef MODULE
59 static const struct x86_cpu_id svm_cpu_id[] = {
60 	X86_MATCH_FEATURE(X86_FEATURE_SVM, NULL),
61 	{}
62 };
63 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
64 #endif
65 
66 #define SEG_TYPE_LDT 2
67 #define SEG_TYPE_BUSY_TSS16 3
68 
69 static bool erratum_383_found __read_mostly;
70 
71 u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
72 
73 /*
74  * Set osvw_len to higher value when updated Revision Guides
75  * are published and we know what the new status bits are
76  */
77 static uint64_t osvw_len = 4, osvw_status;
78 
79 static DEFINE_PER_CPU(u64, current_tsc_ratio);
80 
81 #define X2APIC_MSR(x)	(APIC_BASE_MSR + (x >> 4))
82 
83 static const struct svm_direct_access_msrs {
84 	u32 index;   /* Index of the MSR */
85 	bool always; /* True if intercept is initially cleared */
86 } direct_access_msrs[MAX_DIRECT_ACCESS_MSRS] = {
87 	{ .index = MSR_STAR,				.always = true  },
88 	{ .index = MSR_IA32_SYSENTER_CS,		.always = true  },
89 	{ .index = MSR_IA32_SYSENTER_EIP,		.always = false },
90 	{ .index = MSR_IA32_SYSENTER_ESP,		.always = false },
91 #ifdef CONFIG_X86_64
92 	{ .index = MSR_GS_BASE,				.always = true  },
93 	{ .index = MSR_FS_BASE,				.always = true  },
94 	{ .index = MSR_KERNEL_GS_BASE,			.always = true  },
95 	{ .index = MSR_LSTAR,				.always = true  },
96 	{ .index = MSR_CSTAR,				.always = true  },
97 	{ .index = MSR_SYSCALL_MASK,			.always = true  },
98 #endif
99 	{ .index = MSR_IA32_SPEC_CTRL,			.always = false },
100 	{ .index = MSR_IA32_PRED_CMD,			.always = false },
101 	{ .index = MSR_IA32_FLUSH_CMD,			.always = false },
102 	{ .index = MSR_IA32_DEBUGCTLMSR,		.always = false },
103 	{ .index = MSR_IA32_LASTBRANCHFROMIP,		.always = false },
104 	{ .index = MSR_IA32_LASTBRANCHTOIP,		.always = false },
105 	{ .index = MSR_IA32_LASTINTFROMIP,		.always = false },
106 	{ .index = MSR_IA32_LASTINTTOIP,		.always = false },
107 	{ .index = MSR_IA32_XSS,			.always = false },
108 	{ .index = MSR_EFER,				.always = false },
109 	{ .index = MSR_IA32_CR_PAT,			.always = false },
110 	{ .index = MSR_AMD64_SEV_ES_GHCB,		.always = true  },
111 	{ .index = MSR_TSC_AUX,				.always = false },
112 	{ .index = X2APIC_MSR(APIC_ID),			.always = false },
113 	{ .index = X2APIC_MSR(APIC_LVR),		.always = false },
114 	{ .index = X2APIC_MSR(APIC_TASKPRI),		.always = false },
115 	{ .index = X2APIC_MSR(APIC_ARBPRI),		.always = false },
116 	{ .index = X2APIC_MSR(APIC_PROCPRI),		.always = false },
117 	{ .index = X2APIC_MSR(APIC_EOI),		.always = false },
118 	{ .index = X2APIC_MSR(APIC_RRR),		.always = false },
119 	{ .index = X2APIC_MSR(APIC_LDR),		.always = false },
120 	{ .index = X2APIC_MSR(APIC_DFR),		.always = false },
121 	{ .index = X2APIC_MSR(APIC_SPIV),		.always = false },
122 	{ .index = X2APIC_MSR(APIC_ISR),		.always = false },
123 	{ .index = X2APIC_MSR(APIC_TMR),		.always = false },
124 	{ .index = X2APIC_MSR(APIC_IRR),		.always = false },
125 	{ .index = X2APIC_MSR(APIC_ESR),		.always = false },
126 	{ .index = X2APIC_MSR(APIC_ICR),		.always = false },
127 	{ .index = X2APIC_MSR(APIC_ICR2),		.always = false },
128 
129 	/*
130 	 * Note:
131 	 * AMD does not virtualize APIC TSC-deadline timer mode, but it is
132 	 * emulated by KVM. When setting APIC LVTT (0x832) register bit 18,
133 	 * the AVIC hardware would generate GP fault. Therefore, always
134 	 * intercept the MSR 0x832, and do not setup direct_access_msr.
135 	 */
136 	{ .index = X2APIC_MSR(APIC_LVTTHMR),		.always = false },
137 	{ .index = X2APIC_MSR(APIC_LVTPC),		.always = false },
138 	{ .index = X2APIC_MSR(APIC_LVT0),		.always = false },
139 	{ .index = X2APIC_MSR(APIC_LVT1),		.always = false },
140 	{ .index = X2APIC_MSR(APIC_LVTERR),		.always = false },
141 	{ .index = X2APIC_MSR(APIC_TMICT),		.always = false },
142 	{ .index = X2APIC_MSR(APIC_TMCCT),		.always = false },
143 	{ .index = X2APIC_MSR(APIC_TDCR),		.always = false },
144 	{ .index = MSR_INVALID,				.always = false },
145 };
146 
147 /*
148  * These 2 parameters are used to config the controls for Pause-Loop Exiting:
149  * pause_filter_count: On processors that support Pause filtering(indicated
150  *	by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter
151  *	count value. On VMRUN this value is loaded into an internal counter.
152  *	Each time a pause instruction is executed, this counter is decremented
153  *	until it reaches zero at which time a #VMEXIT is generated if pause
154  *	intercept is enabled. Refer to  AMD APM Vol 2 Section 15.14.4 Pause
155  *	Intercept Filtering for more details.
156  *	This also indicate if ple logic enabled.
157  *
158  * pause_filter_thresh: In addition, some processor families support advanced
159  *	pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on
160  *	the amount of time a guest is allowed to execute in a pause loop.
161  *	In this mode, a 16-bit pause filter threshold field is added in the
162  *	VMCB. The threshold value is a cycle count that is used to reset the
163  *	pause counter. As with simple pause filtering, VMRUN loads the pause
164  *	count value from VMCB into an internal counter. Then, on each pause
165  *	instruction the hardware checks the elapsed number of cycles since
166  *	the most recent pause instruction against the pause filter threshold.
167  *	If the elapsed cycle count is greater than the pause filter threshold,
168  *	then the internal pause count is reloaded from the VMCB and execution
169  *	continues. If the elapsed cycle count is less than the pause filter
170  *	threshold, then the internal pause count is decremented. If the count
171  *	value is less than zero and PAUSE intercept is enabled, a #VMEXIT is
172  *	triggered. If advanced pause filtering is supported and pause filter
173  *	threshold field is set to zero, the filter will operate in the simpler,
174  *	count only mode.
175  */
176 
177 static unsigned short pause_filter_thresh = KVM_DEFAULT_PLE_GAP;
178 module_param(pause_filter_thresh, ushort, 0444);
179 
180 static unsigned short pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW;
181 module_param(pause_filter_count, ushort, 0444);
182 
183 /* Default doubles per-vcpu window every exit. */
184 static unsigned short pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
185 module_param(pause_filter_count_grow, ushort, 0444);
186 
187 /* Default resets per-vcpu window every exit to pause_filter_count. */
188 static unsigned short pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
189 module_param(pause_filter_count_shrink, ushort, 0444);
190 
191 /* Default is to compute the maximum so we can never overflow. */
192 static unsigned short pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX;
193 module_param(pause_filter_count_max, ushort, 0444);
194 
195 /*
196  * Use nested page tables by default.  Note, NPT may get forced off by
197  * svm_hardware_setup() if it's unsupported by hardware or the host kernel.
198  */
199 bool npt_enabled = true;
200 module_param_named(npt, npt_enabled, bool, 0444);
201 
202 /* allow nested virtualization in KVM/SVM */
203 static int nested = true;
204 module_param(nested, int, 0444);
205 
206 /* enable/disable Next RIP Save */
207 int nrips = true;
208 module_param(nrips, int, 0444);
209 
210 /* enable/disable Virtual VMLOAD VMSAVE */
211 static int vls = true;
212 module_param(vls, int, 0444);
213 
214 /* enable/disable Virtual GIF */
215 int vgif = true;
216 module_param(vgif, int, 0444);
217 
218 /* enable/disable LBR virtualization */
219 int lbrv = true;
220 module_param(lbrv, int, 0444);
221 
222 static int tsc_scaling = true;
223 module_param(tsc_scaling, int, 0444);
224 
225 /*
226  * enable / disable AVIC.  Because the defaults differ for APICv
227  * support between VMX and SVM we cannot use module_param_named.
228  */
229 static bool avic;
230 module_param(avic, bool, 0444);
231 
232 bool __read_mostly dump_invalid_vmcb;
233 module_param(dump_invalid_vmcb, bool, 0644);
234 
235 
236 bool intercept_smi = true;
237 module_param(intercept_smi, bool, 0444);
238 
239 bool vnmi = true;
240 module_param(vnmi, bool, 0444);
241 
242 static bool svm_gp_erratum_intercept = true;
243 
244 static u8 rsm_ins_bytes[] = "\x0f\xaa";
245 
246 static unsigned long iopm_base;
247 
248 DEFINE_PER_CPU(struct svm_cpu_data, svm_data);
249 
250 /*
251  * Only MSR_TSC_AUX is switched via the user return hook.  EFER is switched via
252  * the VMCB, and the SYSCALL/SYSENTER MSRs are handled by VMLOAD/VMSAVE.
253  *
254  * RDTSCP and RDPID are not used in the kernel, specifically to allow KVM to
255  * defer the restoration of TSC_AUX until the CPU returns to userspace.
256  */
257 static int tsc_aux_uret_slot __read_mostly = -1;
258 
259 static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
260 
261 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
262 #define MSRS_RANGE_SIZE 2048
263 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
264 
265 u32 svm_msrpm_offset(u32 msr)
266 {
267 	u32 offset;
268 	int i;
269 
270 	for (i = 0; i < NUM_MSR_MAPS; i++) {
271 		if (msr < msrpm_ranges[i] ||
272 		    msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
273 			continue;
274 
275 		offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
276 		offset += (i * MSRS_RANGE_SIZE);       /* add range offset */
277 
278 		/* Now we have the u8 offset - but need the u32 offset */
279 		return offset / 4;
280 	}
281 
282 	/* MSR not in any range */
283 	return MSR_INVALID;
284 }
285 
286 static void svm_flush_tlb_current(struct kvm_vcpu *vcpu);
287 
288 static int get_npt_level(void)
289 {
290 #ifdef CONFIG_X86_64
291 	return pgtable_l5_enabled() ? PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL;
292 #else
293 	return PT32E_ROOT_LEVEL;
294 #endif
295 }
296 
297 int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
298 {
299 	struct vcpu_svm *svm = to_svm(vcpu);
300 	u64 old_efer = vcpu->arch.efer;
301 	vcpu->arch.efer = efer;
302 
303 	if (!npt_enabled) {
304 		/* Shadow paging assumes NX to be available.  */
305 		efer |= EFER_NX;
306 
307 		if (!(efer & EFER_LMA))
308 			efer &= ~EFER_LME;
309 	}
310 
311 	if ((old_efer & EFER_SVME) != (efer & EFER_SVME)) {
312 		if (!(efer & EFER_SVME)) {
313 			svm_leave_nested(vcpu);
314 			svm_set_gif(svm, true);
315 			/* #GP intercept is still needed for vmware backdoor */
316 			if (!enable_vmware_backdoor)
317 				clr_exception_intercept(svm, GP_VECTOR);
318 
319 			/*
320 			 * Free the nested guest state, unless we are in SMM.
321 			 * In this case we will return to the nested guest
322 			 * as soon as we leave SMM.
323 			 */
324 			if (!is_smm(vcpu))
325 				svm_free_nested(svm);
326 
327 		} else {
328 			int ret = svm_allocate_nested(svm);
329 
330 			if (ret) {
331 				vcpu->arch.efer = old_efer;
332 				return ret;
333 			}
334 
335 			/*
336 			 * Never intercept #GP for SEV guests, KVM can't
337 			 * decrypt guest memory to workaround the erratum.
338 			 */
339 			if (svm_gp_erratum_intercept && !sev_guest(vcpu->kvm))
340 				set_exception_intercept(svm, GP_VECTOR);
341 		}
342 	}
343 
344 	svm->vmcb->save.efer = efer | EFER_SVME;
345 	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
346 	return 0;
347 }
348 
349 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
350 {
351 	struct vcpu_svm *svm = to_svm(vcpu);
352 	u32 ret = 0;
353 
354 	if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
355 		ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
356 	return ret;
357 }
358 
359 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
360 {
361 	struct vcpu_svm *svm = to_svm(vcpu);
362 
363 	if (mask == 0)
364 		svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
365 	else
366 		svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
367 
368 }
369 
370 static int __svm_skip_emulated_instruction(struct kvm_vcpu *vcpu,
371 					   bool commit_side_effects)
372 {
373 	struct vcpu_svm *svm = to_svm(vcpu);
374 	unsigned long old_rflags;
375 
376 	/*
377 	 * SEV-ES does not expose the next RIP. The RIP update is controlled by
378 	 * the type of exit and the #VC handler in the guest.
379 	 */
380 	if (sev_es_guest(vcpu->kvm))
381 		goto done;
382 
383 	if (nrips && svm->vmcb->control.next_rip != 0) {
384 		WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
385 		svm->next_rip = svm->vmcb->control.next_rip;
386 	}
387 
388 	if (!svm->next_rip) {
389 		if (unlikely(!commit_side_effects))
390 			old_rflags = svm->vmcb->save.rflags;
391 
392 		if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP))
393 			return 0;
394 
395 		if (unlikely(!commit_side_effects))
396 			svm->vmcb->save.rflags = old_rflags;
397 	} else {
398 		kvm_rip_write(vcpu, svm->next_rip);
399 	}
400 
401 done:
402 	if (likely(commit_side_effects))
403 		svm_set_interrupt_shadow(vcpu, 0);
404 
405 	return 1;
406 }
407 
408 static int svm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
409 {
410 	return __svm_skip_emulated_instruction(vcpu, true);
411 }
412 
413 static int svm_update_soft_interrupt_rip(struct kvm_vcpu *vcpu)
414 {
415 	unsigned long rip, old_rip = kvm_rip_read(vcpu);
416 	struct vcpu_svm *svm = to_svm(vcpu);
417 
418 	/*
419 	 * Due to architectural shortcomings, the CPU doesn't always provide
420 	 * NextRIP, e.g. if KVM intercepted an exception that occurred while
421 	 * the CPU was vectoring an INTO/INT3 in the guest.  Temporarily skip
422 	 * the instruction even if NextRIP is supported to acquire the next
423 	 * RIP so that it can be shoved into the NextRIP field, otherwise
424 	 * hardware will fail to advance guest RIP during event injection.
425 	 * Drop the exception/interrupt if emulation fails and effectively
426 	 * retry the instruction, it's the least awful option.  If NRIPS is
427 	 * in use, the skip must not commit any side effects such as clearing
428 	 * the interrupt shadow or RFLAGS.RF.
429 	 */
430 	if (!__svm_skip_emulated_instruction(vcpu, !nrips))
431 		return -EIO;
432 
433 	rip = kvm_rip_read(vcpu);
434 
435 	/*
436 	 * Save the injection information, even when using next_rip, as the
437 	 * VMCB's next_rip will be lost (cleared on VM-Exit) if the injection
438 	 * doesn't complete due to a VM-Exit occurring while the CPU is
439 	 * vectoring the event.   Decoding the instruction isn't guaranteed to
440 	 * work as there may be no backing instruction, e.g. if the event is
441 	 * being injected by L1 for L2, or if the guest is patching INT3 into
442 	 * a different instruction.
443 	 */
444 	svm->soft_int_injected = true;
445 	svm->soft_int_csbase = svm->vmcb->save.cs.base;
446 	svm->soft_int_old_rip = old_rip;
447 	svm->soft_int_next_rip = rip;
448 
449 	if (nrips)
450 		kvm_rip_write(vcpu, old_rip);
451 
452 	if (static_cpu_has(X86_FEATURE_NRIPS))
453 		svm->vmcb->control.next_rip = rip;
454 
455 	return 0;
456 }
457 
458 static void svm_inject_exception(struct kvm_vcpu *vcpu)
459 {
460 	struct kvm_queued_exception *ex = &vcpu->arch.exception;
461 	struct vcpu_svm *svm = to_svm(vcpu);
462 
463 	kvm_deliver_exception_payload(vcpu, ex);
464 
465 	if (kvm_exception_is_soft(ex->vector) &&
466 	    svm_update_soft_interrupt_rip(vcpu))
467 		return;
468 
469 	svm->vmcb->control.event_inj = ex->vector
470 		| SVM_EVTINJ_VALID
471 		| (ex->has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
472 		| SVM_EVTINJ_TYPE_EXEPT;
473 	svm->vmcb->control.event_inj_err = ex->error_code;
474 }
475 
476 static void svm_init_erratum_383(void)
477 {
478 	u32 low, high;
479 	int err;
480 	u64 val;
481 
482 	if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
483 		return;
484 
485 	/* Use _safe variants to not break nested virtualization */
486 	val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
487 	if (err)
488 		return;
489 
490 	val |= (1ULL << 47);
491 
492 	low  = lower_32_bits(val);
493 	high = upper_32_bits(val);
494 
495 	native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
496 
497 	erratum_383_found = true;
498 }
499 
500 static void svm_init_osvw(struct kvm_vcpu *vcpu)
501 {
502 	/*
503 	 * Guests should see errata 400 and 415 as fixed (assuming that
504 	 * HLT and IO instructions are intercepted).
505 	 */
506 	vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
507 	vcpu->arch.osvw.status = osvw_status & ~(6ULL);
508 
509 	/*
510 	 * By increasing VCPU's osvw.length to 3 we are telling the guest that
511 	 * all osvw.status bits inside that length, including bit 0 (which is
512 	 * reserved for erratum 298), are valid. However, if host processor's
513 	 * osvw_len is 0 then osvw_status[0] carries no information. We need to
514 	 * be conservative here and therefore we tell the guest that erratum 298
515 	 * is present (because we really don't know).
516 	 */
517 	if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
518 		vcpu->arch.osvw.status |= 1;
519 }
520 
521 static bool __kvm_is_svm_supported(void)
522 {
523 	int cpu = smp_processor_id();
524 	struct cpuinfo_x86 *c = &cpu_data(cpu);
525 
526 	if (c->x86_vendor != X86_VENDOR_AMD &&
527 	    c->x86_vendor != X86_VENDOR_HYGON) {
528 		pr_err("CPU %d isn't AMD or Hygon\n", cpu);
529 		return false;
530 	}
531 
532 	if (!cpu_has(c, X86_FEATURE_SVM)) {
533 		pr_err("SVM not supported by CPU %d\n", cpu);
534 		return false;
535 	}
536 
537 	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) {
538 		pr_info("KVM is unsupported when running as an SEV guest\n");
539 		return false;
540 	}
541 
542 	return true;
543 }
544 
545 static bool kvm_is_svm_supported(void)
546 {
547 	bool supported;
548 
549 	migrate_disable();
550 	supported = __kvm_is_svm_supported();
551 	migrate_enable();
552 
553 	return supported;
554 }
555 
556 static int svm_check_processor_compat(void)
557 {
558 	if (!__kvm_is_svm_supported())
559 		return -EIO;
560 
561 	return 0;
562 }
563 
564 static void __svm_write_tsc_multiplier(u64 multiplier)
565 {
566 	if (multiplier == __this_cpu_read(current_tsc_ratio))
567 		return;
568 
569 	wrmsrl(MSR_AMD64_TSC_RATIO, multiplier);
570 	__this_cpu_write(current_tsc_ratio, multiplier);
571 }
572 
573 static inline void kvm_cpu_svm_disable(void)
574 {
575 	uint64_t efer;
576 
577 	wrmsrl(MSR_VM_HSAVE_PA, 0);
578 	rdmsrl(MSR_EFER, efer);
579 	if (efer & EFER_SVME) {
580 		/*
581 		 * Force GIF=1 prior to disabling SVM, e.g. to ensure INIT and
582 		 * NMI aren't blocked.
583 		 */
584 		stgi();
585 		wrmsrl(MSR_EFER, efer & ~EFER_SVME);
586 	}
587 }
588 
589 static void svm_emergency_disable(void)
590 {
591 	kvm_rebooting = true;
592 
593 	kvm_cpu_svm_disable();
594 }
595 
596 static void svm_hardware_disable(void)
597 {
598 	/* Make sure we clean up behind us */
599 	if (tsc_scaling)
600 		__svm_write_tsc_multiplier(SVM_TSC_RATIO_DEFAULT);
601 
602 	kvm_cpu_svm_disable();
603 
604 	amd_pmu_disable_virt();
605 }
606 
607 static int svm_hardware_enable(void)
608 {
609 
610 	struct svm_cpu_data *sd;
611 	uint64_t efer;
612 	int me = raw_smp_processor_id();
613 
614 	rdmsrl(MSR_EFER, efer);
615 	if (efer & EFER_SVME)
616 		return -EBUSY;
617 
618 	sd = per_cpu_ptr(&svm_data, me);
619 	sd->asid_generation = 1;
620 	sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
621 	sd->next_asid = sd->max_asid + 1;
622 	sd->min_asid = max_sev_asid + 1;
623 
624 	wrmsrl(MSR_EFER, efer | EFER_SVME);
625 
626 	wrmsrl(MSR_VM_HSAVE_PA, sd->save_area_pa);
627 
628 	if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
629 		/*
630 		 * Set the default value, even if we don't use TSC scaling
631 		 * to avoid having stale value in the msr
632 		 */
633 		__svm_write_tsc_multiplier(SVM_TSC_RATIO_DEFAULT);
634 	}
635 
636 
637 	/*
638 	 * Get OSVW bits.
639 	 *
640 	 * Note that it is possible to have a system with mixed processor
641 	 * revisions and therefore different OSVW bits. If bits are not the same
642 	 * on different processors then choose the worst case (i.e. if erratum
643 	 * is present on one processor and not on another then assume that the
644 	 * erratum is present everywhere).
645 	 */
646 	if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
647 		uint64_t len, status = 0;
648 		int err;
649 
650 		len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
651 		if (!err)
652 			status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
653 						      &err);
654 
655 		if (err)
656 			osvw_status = osvw_len = 0;
657 		else {
658 			if (len < osvw_len)
659 				osvw_len = len;
660 			osvw_status |= status;
661 			osvw_status &= (1ULL << osvw_len) - 1;
662 		}
663 	} else
664 		osvw_status = osvw_len = 0;
665 
666 	svm_init_erratum_383();
667 
668 	amd_pmu_enable_virt();
669 
670 	/*
671 	 * If TSC_AUX virtualization is supported, TSC_AUX becomes a swap type
672 	 * "B" field (see sev_es_prepare_switch_to_guest()) for SEV-ES guests.
673 	 * Since Linux does not change the value of TSC_AUX once set, prime the
674 	 * TSC_AUX field now to avoid a RDMSR on every vCPU run.
675 	 */
676 	if (boot_cpu_has(X86_FEATURE_V_TSC_AUX)) {
677 		struct sev_es_save_area *hostsa;
678 		u32 __maybe_unused msr_hi;
679 
680 		hostsa = (struct sev_es_save_area *)(page_address(sd->save_area) + 0x400);
681 
682 		rdmsr(MSR_TSC_AUX, hostsa->tsc_aux, msr_hi);
683 	}
684 
685 	return 0;
686 }
687 
688 static void svm_cpu_uninit(int cpu)
689 {
690 	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu);
691 
692 	if (!sd->save_area)
693 		return;
694 
695 	kfree(sd->sev_vmcbs);
696 	__free_page(sd->save_area);
697 	sd->save_area_pa = 0;
698 	sd->save_area = NULL;
699 }
700 
701 static int svm_cpu_init(int cpu)
702 {
703 	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu);
704 	int ret = -ENOMEM;
705 
706 	memset(sd, 0, sizeof(struct svm_cpu_data));
707 	sd->save_area = snp_safe_alloc_page(NULL);
708 	if (!sd->save_area)
709 		return ret;
710 
711 	ret = sev_cpu_init(sd);
712 	if (ret)
713 		goto free_save_area;
714 
715 	sd->save_area_pa = __sme_page_pa(sd->save_area);
716 	return 0;
717 
718 free_save_area:
719 	__free_page(sd->save_area);
720 	sd->save_area = NULL;
721 	return ret;
722 
723 }
724 
725 static void set_dr_intercepts(struct vcpu_svm *svm)
726 {
727 	struct vmcb *vmcb = svm->vmcb01.ptr;
728 
729 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR0_READ);
730 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR1_READ);
731 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR2_READ);
732 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR3_READ);
733 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR4_READ);
734 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR5_READ);
735 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR6_READ);
736 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR0_WRITE);
737 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR1_WRITE);
738 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR2_WRITE);
739 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR3_WRITE);
740 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR4_WRITE);
741 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR5_WRITE);
742 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR6_WRITE);
743 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_READ);
744 	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_WRITE);
745 
746 	recalc_intercepts(svm);
747 }
748 
749 static void clr_dr_intercepts(struct vcpu_svm *svm)
750 {
751 	struct vmcb *vmcb = svm->vmcb01.ptr;
752 
753 	vmcb->control.intercepts[INTERCEPT_DR] = 0;
754 
755 	recalc_intercepts(svm);
756 }
757 
758 static int direct_access_msr_slot(u32 msr)
759 {
760 	u32 i;
761 
762 	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
763 		if (direct_access_msrs[i].index == msr)
764 			return i;
765 
766 	return -ENOENT;
767 }
768 
769 static void set_shadow_msr_intercept(struct kvm_vcpu *vcpu, u32 msr, int read,
770 				     int write)
771 {
772 	struct vcpu_svm *svm = to_svm(vcpu);
773 	int slot = direct_access_msr_slot(msr);
774 
775 	if (slot == -ENOENT)
776 		return;
777 
778 	/* Set the shadow bitmaps to the desired intercept states */
779 	if (read)
780 		set_bit(slot, svm->shadow_msr_intercept.read);
781 	else
782 		clear_bit(slot, svm->shadow_msr_intercept.read);
783 
784 	if (write)
785 		set_bit(slot, svm->shadow_msr_intercept.write);
786 	else
787 		clear_bit(slot, svm->shadow_msr_intercept.write);
788 }
789 
790 static bool valid_msr_intercept(u32 index)
791 {
792 	return direct_access_msr_slot(index) != -ENOENT;
793 }
794 
795 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr)
796 {
797 	u8 bit_write;
798 	unsigned long tmp;
799 	u32 offset;
800 	u32 *msrpm;
801 
802 	/*
803 	 * For non-nested case:
804 	 * If the L01 MSR bitmap does not intercept the MSR, then we need to
805 	 * save it.
806 	 *
807 	 * For nested case:
808 	 * If the L02 MSR bitmap does not intercept the MSR, then we need to
809 	 * save it.
810 	 */
811 	msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm:
812 				      to_svm(vcpu)->msrpm;
813 
814 	offset    = svm_msrpm_offset(msr);
815 	bit_write = 2 * (msr & 0x0f) + 1;
816 	tmp       = msrpm[offset];
817 
818 	BUG_ON(offset == MSR_INVALID);
819 
820 	return test_bit(bit_write, &tmp);
821 }
822 
823 static void set_msr_interception_bitmap(struct kvm_vcpu *vcpu, u32 *msrpm,
824 					u32 msr, int read, int write)
825 {
826 	struct vcpu_svm *svm = to_svm(vcpu);
827 	u8 bit_read, bit_write;
828 	unsigned long tmp;
829 	u32 offset;
830 
831 	/*
832 	 * If this warning triggers extend the direct_access_msrs list at the
833 	 * beginning of the file
834 	 */
835 	WARN_ON(!valid_msr_intercept(msr));
836 
837 	/* Enforce non allowed MSRs to trap */
838 	if (read && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_READ))
839 		read = 0;
840 
841 	if (write && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_WRITE))
842 		write = 0;
843 
844 	offset    = svm_msrpm_offset(msr);
845 	bit_read  = 2 * (msr & 0x0f);
846 	bit_write = 2 * (msr & 0x0f) + 1;
847 	tmp       = msrpm[offset];
848 
849 	BUG_ON(offset == MSR_INVALID);
850 
851 	read  ? clear_bit(bit_read,  &tmp) : set_bit(bit_read,  &tmp);
852 	write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
853 
854 	msrpm[offset] = tmp;
855 
856 	svm_hv_vmcb_dirty_nested_enlightenments(vcpu);
857 	svm->nested.force_msr_bitmap_recalc = true;
858 }
859 
860 void set_msr_interception(struct kvm_vcpu *vcpu, u32 *msrpm, u32 msr,
861 			  int read, int write)
862 {
863 	set_shadow_msr_intercept(vcpu, msr, read, write);
864 	set_msr_interception_bitmap(vcpu, msrpm, msr, read, write);
865 }
866 
867 u32 *svm_vcpu_alloc_msrpm(void)
868 {
869 	unsigned int order = get_order(MSRPM_SIZE);
870 	struct page *pages = alloc_pages(GFP_KERNEL_ACCOUNT, order);
871 	u32 *msrpm;
872 
873 	if (!pages)
874 		return NULL;
875 
876 	msrpm = page_address(pages);
877 	memset(msrpm, 0xff, PAGE_SIZE * (1 << order));
878 
879 	return msrpm;
880 }
881 
882 void svm_vcpu_init_msrpm(struct kvm_vcpu *vcpu, u32 *msrpm)
883 {
884 	int i;
885 
886 	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
887 		if (!direct_access_msrs[i].always)
888 			continue;
889 		set_msr_interception(vcpu, msrpm, direct_access_msrs[i].index, 1, 1);
890 	}
891 }
892 
893 void svm_set_x2apic_msr_interception(struct vcpu_svm *svm, bool intercept)
894 {
895 	int i;
896 
897 	if (intercept == svm->x2avic_msrs_intercepted)
898 		return;
899 
900 	if (!x2avic_enabled)
901 		return;
902 
903 	for (i = 0; i < MAX_DIRECT_ACCESS_MSRS; i++) {
904 		int index = direct_access_msrs[i].index;
905 
906 		if ((index < APIC_BASE_MSR) ||
907 		    (index > APIC_BASE_MSR + 0xff))
908 			continue;
909 		set_msr_interception(&svm->vcpu, svm->msrpm, index,
910 				     !intercept, !intercept);
911 	}
912 
913 	svm->x2avic_msrs_intercepted = intercept;
914 }
915 
916 void svm_vcpu_free_msrpm(u32 *msrpm)
917 {
918 	__free_pages(virt_to_page(msrpm), get_order(MSRPM_SIZE));
919 }
920 
921 static void svm_msr_filter_changed(struct kvm_vcpu *vcpu)
922 {
923 	struct vcpu_svm *svm = to_svm(vcpu);
924 	u32 i;
925 
926 	/*
927 	 * Set intercept permissions for all direct access MSRs again. They
928 	 * will automatically get filtered through the MSR filter, so we are
929 	 * back in sync after this.
930 	 */
931 	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
932 		u32 msr = direct_access_msrs[i].index;
933 		u32 read = test_bit(i, svm->shadow_msr_intercept.read);
934 		u32 write = test_bit(i, svm->shadow_msr_intercept.write);
935 
936 		set_msr_interception_bitmap(vcpu, svm->msrpm, msr, read, write);
937 	}
938 }
939 
940 static void add_msr_offset(u32 offset)
941 {
942 	int i;
943 
944 	for (i = 0; i < MSRPM_OFFSETS; ++i) {
945 
946 		/* Offset already in list? */
947 		if (msrpm_offsets[i] == offset)
948 			return;
949 
950 		/* Slot used by another offset? */
951 		if (msrpm_offsets[i] != MSR_INVALID)
952 			continue;
953 
954 		/* Add offset to list */
955 		msrpm_offsets[i] = offset;
956 
957 		return;
958 	}
959 
960 	/*
961 	 * If this BUG triggers the msrpm_offsets table has an overflow. Just
962 	 * increase MSRPM_OFFSETS in this case.
963 	 */
964 	BUG();
965 }
966 
967 static void init_msrpm_offsets(void)
968 {
969 	int i;
970 
971 	memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
972 
973 	for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
974 		u32 offset;
975 
976 		offset = svm_msrpm_offset(direct_access_msrs[i].index);
977 		BUG_ON(offset == MSR_INVALID);
978 
979 		add_msr_offset(offset);
980 	}
981 }
982 
983 void svm_copy_lbrs(struct vmcb *to_vmcb, struct vmcb *from_vmcb)
984 {
985 	to_vmcb->save.dbgctl		= from_vmcb->save.dbgctl;
986 	to_vmcb->save.br_from		= from_vmcb->save.br_from;
987 	to_vmcb->save.br_to		= from_vmcb->save.br_to;
988 	to_vmcb->save.last_excp_from	= from_vmcb->save.last_excp_from;
989 	to_vmcb->save.last_excp_to	= from_vmcb->save.last_excp_to;
990 
991 	vmcb_mark_dirty(to_vmcb, VMCB_LBR);
992 }
993 
994 void svm_enable_lbrv(struct kvm_vcpu *vcpu)
995 {
996 	struct vcpu_svm *svm = to_svm(vcpu);
997 
998 	svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
999 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
1000 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
1001 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
1002 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
1003 
1004 	if (sev_es_guest(vcpu->kvm))
1005 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_DEBUGCTLMSR, 1, 1);
1006 
1007 	/* Move the LBR msrs to the vmcb02 so that the guest can see them. */
1008 	if (is_guest_mode(vcpu))
1009 		svm_copy_lbrs(svm->vmcb, svm->vmcb01.ptr);
1010 }
1011 
1012 static void svm_disable_lbrv(struct kvm_vcpu *vcpu)
1013 {
1014 	struct vcpu_svm *svm = to_svm(vcpu);
1015 
1016 	KVM_BUG_ON(sev_es_guest(vcpu->kvm), vcpu->kvm);
1017 
1018 	svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;
1019 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
1020 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
1021 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
1022 	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
1023 
1024 	/*
1025 	 * Move the LBR msrs back to the vmcb01 to avoid copying them
1026 	 * on nested guest entries.
1027 	 */
1028 	if (is_guest_mode(vcpu))
1029 		svm_copy_lbrs(svm->vmcb01.ptr, svm->vmcb);
1030 }
1031 
1032 static struct vmcb *svm_get_lbr_vmcb(struct vcpu_svm *svm)
1033 {
1034 	/*
1035 	 * If LBR virtualization is disabled, the LBR MSRs are always kept in
1036 	 * vmcb01.  If LBR virtualization is enabled and L1 is running VMs of
1037 	 * its own, the MSRs are moved between vmcb01 and vmcb02 as needed.
1038 	 */
1039 	return svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK ? svm->vmcb :
1040 								   svm->vmcb01.ptr;
1041 }
1042 
1043 void svm_update_lbrv(struct kvm_vcpu *vcpu)
1044 {
1045 	struct vcpu_svm *svm = to_svm(vcpu);
1046 	bool current_enable_lbrv = svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK;
1047 	bool enable_lbrv = (svm_get_lbr_vmcb(svm)->save.dbgctl & DEBUGCTLMSR_LBR) ||
1048 			    (is_guest_mode(vcpu) && guest_can_use(vcpu, X86_FEATURE_LBRV) &&
1049 			    (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK));
1050 
1051 	if (enable_lbrv == current_enable_lbrv)
1052 		return;
1053 
1054 	if (enable_lbrv)
1055 		svm_enable_lbrv(vcpu);
1056 	else
1057 		svm_disable_lbrv(vcpu);
1058 }
1059 
1060 void disable_nmi_singlestep(struct vcpu_svm *svm)
1061 {
1062 	svm->nmi_singlestep = false;
1063 
1064 	if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) {
1065 		/* Clear our flags if they were not set by the guest */
1066 		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1067 			svm->vmcb->save.rflags &= ~X86_EFLAGS_TF;
1068 		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1069 			svm->vmcb->save.rflags &= ~X86_EFLAGS_RF;
1070 	}
1071 }
1072 
1073 static void grow_ple_window(struct kvm_vcpu *vcpu)
1074 {
1075 	struct vcpu_svm *svm = to_svm(vcpu);
1076 	struct vmcb_control_area *control = &svm->vmcb->control;
1077 	int old = control->pause_filter_count;
1078 
1079 	if (kvm_pause_in_guest(vcpu->kvm))
1080 		return;
1081 
1082 	control->pause_filter_count = __grow_ple_window(old,
1083 							pause_filter_count,
1084 							pause_filter_count_grow,
1085 							pause_filter_count_max);
1086 
1087 	if (control->pause_filter_count != old) {
1088 		vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1089 		trace_kvm_ple_window_update(vcpu->vcpu_id,
1090 					    control->pause_filter_count, old);
1091 	}
1092 }
1093 
1094 static void shrink_ple_window(struct kvm_vcpu *vcpu)
1095 {
1096 	struct vcpu_svm *svm = to_svm(vcpu);
1097 	struct vmcb_control_area *control = &svm->vmcb->control;
1098 	int old = control->pause_filter_count;
1099 
1100 	if (kvm_pause_in_guest(vcpu->kvm))
1101 		return;
1102 
1103 	control->pause_filter_count =
1104 				__shrink_ple_window(old,
1105 						    pause_filter_count,
1106 						    pause_filter_count_shrink,
1107 						    pause_filter_count);
1108 	if (control->pause_filter_count != old) {
1109 		vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1110 		trace_kvm_ple_window_update(vcpu->vcpu_id,
1111 					    control->pause_filter_count, old);
1112 	}
1113 }
1114 
1115 static void svm_hardware_unsetup(void)
1116 {
1117 	int cpu;
1118 
1119 	sev_hardware_unsetup();
1120 
1121 	for_each_possible_cpu(cpu)
1122 		svm_cpu_uninit(cpu);
1123 
1124 	__free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT),
1125 	get_order(IOPM_SIZE));
1126 	iopm_base = 0;
1127 }
1128 
1129 static void init_seg(struct vmcb_seg *seg)
1130 {
1131 	seg->selector = 0;
1132 	seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
1133 		      SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
1134 	seg->limit = 0xffff;
1135 	seg->base = 0;
1136 }
1137 
1138 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
1139 {
1140 	seg->selector = 0;
1141 	seg->attrib = SVM_SELECTOR_P_MASK | type;
1142 	seg->limit = 0xffff;
1143 	seg->base = 0;
1144 }
1145 
1146 static u64 svm_get_l2_tsc_offset(struct kvm_vcpu *vcpu)
1147 {
1148 	struct vcpu_svm *svm = to_svm(vcpu);
1149 
1150 	return svm->nested.ctl.tsc_offset;
1151 }
1152 
1153 static u64 svm_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu)
1154 {
1155 	struct vcpu_svm *svm = to_svm(vcpu);
1156 
1157 	return svm->tsc_ratio_msr;
1158 }
1159 
1160 static void svm_write_tsc_offset(struct kvm_vcpu *vcpu)
1161 {
1162 	struct vcpu_svm *svm = to_svm(vcpu);
1163 
1164 	svm->vmcb01.ptr->control.tsc_offset = vcpu->arch.l1_tsc_offset;
1165 	svm->vmcb->control.tsc_offset = vcpu->arch.tsc_offset;
1166 	vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1167 }
1168 
1169 void svm_write_tsc_multiplier(struct kvm_vcpu *vcpu)
1170 {
1171 	preempt_disable();
1172 	if (to_svm(vcpu)->guest_state_loaded)
1173 		__svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio);
1174 	preempt_enable();
1175 }
1176 
1177 /* Evaluate instruction intercepts that depend on guest CPUID features. */
1178 static void svm_recalc_instruction_intercepts(struct kvm_vcpu *vcpu,
1179 					      struct vcpu_svm *svm)
1180 {
1181 	/*
1182 	 * Intercept INVPCID if shadow paging is enabled to sync/free shadow
1183 	 * roots, or if INVPCID is disabled in the guest to inject #UD.
1184 	 */
1185 	if (kvm_cpu_cap_has(X86_FEATURE_INVPCID)) {
1186 		if (!npt_enabled ||
1187 		    !guest_cpuid_has(&svm->vcpu, X86_FEATURE_INVPCID))
1188 			svm_set_intercept(svm, INTERCEPT_INVPCID);
1189 		else
1190 			svm_clr_intercept(svm, INTERCEPT_INVPCID);
1191 	}
1192 
1193 	if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP)) {
1194 		if (guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
1195 			svm_clr_intercept(svm, INTERCEPT_RDTSCP);
1196 		else
1197 			svm_set_intercept(svm, INTERCEPT_RDTSCP);
1198 	}
1199 }
1200 
1201 static inline void init_vmcb_after_set_cpuid(struct kvm_vcpu *vcpu)
1202 {
1203 	struct vcpu_svm *svm = to_svm(vcpu);
1204 
1205 	if (guest_cpuid_is_intel(vcpu)) {
1206 		/*
1207 		 * We must intercept SYSENTER_EIP and SYSENTER_ESP
1208 		 * accesses because the processor only stores 32 bits.
1209 		 * For the same reason we cannot use virtual VMLOAD/VMSAVE.
1210 		 */
1211 		svm_set_intercept(svm, INTERCEPT_VMLOAD);
1212 		svm_set_intercept(svm, INTERCEPT_VMSAVE);
1213 		svm->vmcb->control.virt_ext &= ~VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1214 
1215 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 0, 0);
1216 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 0, 0);
1217 	} else {
1218 		/*
1219 		 * If hardware supports Virtual VMLOAD VMSAVE then enable it
1220 		 * in VMCB and clear intercepts to avoid #VMEXIT.
1221 		 */
1222 		if (vls) {
1223 			svm_clr_intercept(svm, INTERCEPT_VMLOAD);
1224 			svm_clr_intercept(svm, INTERCEPT_VMSAVE);
1225 			svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1226 		}
1227 		/* No need to intercept these MSRs */
1228 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 1, 1);
1229 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 1, 1);
1230 	}
1231 }
1232 
1233 static void init_vmcb(struct kvm_vcpu *vcpu)
1234 {
1235 	struct vcpu_svm *svm = to_svm(vcpu);
1236 	struct vmcb *vmcb = svm->vmcb01.ptr;
1237 	struct vmcb_control_area *control = &vmcb->control;
1238 	struct vmcb_save_area *save = &vmcb->save;
1239 
1240 	svm_set_intercept(svm, INTERCEPT_CR0_READ);
1241 	svm_set_intercept(svm, INTERCEPT_CR3_READ);
1242 	svm_set_intercept(svm, INTERCEPT_CR4_READ);
1243 	svm_set_intercept(svm, INTERCEPT_CR0_WRITE);
1244 	svm_set_intercept(svm, INTERCEPT_CR3_WRITE);
1245 	svm_set_intercept(svm, INTERCEPT_CR4_WRITE);
1246 	if (!kvm_vcpu_apicv_active(vcpu))
1247 		svm_set_intercept(svm, INTERCEPT_CR8_WRITE);
1248 
1249 	set_dr_intercepts(svm);
1250 
1251 	set_exception_intercept(svm, PF_VECTOR);
1252 	set_exception_intercept(svm, UD_VECTOR);
1253 	set_exception_intercept(svm, MC_VECTOR);
1254 	set_exception_intercept(svm, AC_VECTOR);
1255 	set_exception_intercept(svm, DB_VECTOR);
1256 	/*
1257 	 * Guest access to VMware backdoor ports could legitimately
1258 	 * trigger #GP because of TSS I/O permission bitmap.
1259 	 * We intercept those #GP and allow access to them anyway
1260 	 * as VMware does.
1261 	 */
1262 	if (enable_vmware_backdoor)
1263 		set_exception_intercept(svm, GP_VECTOR);
1264 
1265 	svm_set_intercept(svm, INTERCEPT_INTR);
1266 	svm_set_intercept(svm, INTERCEPT_NMI);
1267 
1268 	if (intercept_smi)
1269 		svm_set_intercept(svm, INTERCEPT_SMI);
1270 
1271 	svm_set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
1272 	svm_set_intercept(svm, INTERCEPT_RDPMC);
1273 	svm_set_intercept(svm, INTERCEPT_CPUID);
1274 	svm_set_intercept(svm, INTERCEPT_INVD);
1275 	svm_set_intercept(svm, INTERCEPT_INVLPG);
1276 	svm_set_intercept(svm, INTERCEPT_INVLPGA);
1277 	svm_set_intercept(svm, INTERCEPT_IOIO_PROT);
1278 	svm_set_intercept(svm, INTERCEPT_MSR_PROT);
1279 	svm_set_intercept(svm, INTERCEPT_TASK_SWITCH);
1280 	svm_set_intercept(svm, INTERCEPT_SHUTDOWN);
1281 	svm_set_intercept(svm, INTERCEPT_VMRUN);
1282 	svm_set_intercept(svm, INTERCEPT_VMMCALL);
1283 	svm_set_intercept(svm, INTERCEPT_VMLOAD);
1284 	svm_set_intercept(svm, INTERCEPT_VMSAVE);
1285 	svm_set_intercept(svm, INTERCEPT_STGI);
1286 	svm_set_intercept(svm, INTERCEPT_CLGI);
1287 	svm_set_intercept(svm, INTERCEPT_SKINIT);
1288 	svm_set_intercept(svm, INTERCEPT_WBINVD);
1289 	svm_set_intercept(svm, INTERCEPT_XSETBV);
1290 	svm_set_intercept(svm, INTERCEPT_RDPRU);
1291 	svm_set_intercept(svm, INTERCEPT_RSM);
1292 
1293 	if (!kvm_mwait_in_guest(vcpu->kvm)) {
1294 		svm_set_intercept(svm, INTERCEPT_MONITOR);
1295 		svm_set_intercept(svm, INTERCEPT_MWAIT);
1296 	}
1297 
1298 	if (!kvm_hlt_in_guest(vcpu->kvm))
1299 		svm_set_intercept(svm, INTERCEPT_HLT);
1300 
1301 	control->iopm_base_pa = __sme_set(iopm_base);
1302 	control->msrpm_base_pa = __sme_set(__pa(svm->msrpm));
1303 	control->int_ctl = V_INTR_MASKING_MASK;
1304 
1305 	init_seg(&save->es);
1306 	init_seg(&save->ss);
1307 	init_seg(&save->ds);
1308 	init_seg(&save->fs);
1309 	init_seg(&save->gs);
1310 
1311 	save->cs.selector = 0xf000;
1312 	save->cs.base = 0xffff0000;
1313 	/* Executable/Readable Code Segment */
1314 	save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1315 		SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1316 	save->cs.limit = 0xffff;
1317 
1318 	save->gdtr.base = 0;
1319 	save->gdtr.limit = 0xffff;
1320 	save->idtr.base = 0;
1321 	save->idtr.limit = 0xffff;
1322 
1323 	init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1324 	init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1325 
1326 	if (npt_enabled) {
1327 		/* Setup VMCB for Nested Paging */
1328 		control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE;
1329 		svm_clr_intercept(svm, INTERCEPT_INVLPG);
1330 		clr_exception_intercept(svm, PF_VECTOR);
1331 		svm_clr_intercept(svm, INTERCEPT_CR3_READ);
1332 		svm_clr_intercept(svm, INTERCEPT_CR3_WRITE);
1333 		save->g_pat = vcpu->arch.pat;
1334 		save->cr3 = 0;
1335 	}
1336 	svm->current_vmcb->asid_generation = 0;
1337 	svm->asid = 0;
1338 
1339 	svm->nested.vmcb12_gpa = INVALID_GPA;
1340 	svm->nested.last_vmcb12_gpa = INVALID_GPA;
1341 
1342 	if (!kvm_pause_in_guest(vcpu->kvm)) {
1343 		control->pause_filter_count = pause_filter_count;
1344 		if (pause_filter_thresh)
1345 			control->pause_filter_thresh = pause_filter_thresh;
1346 		svm_set_intercept(svm, INTERCEPT_PAUSE);
1347 	} else {
1348 		svm_clr_intercept(svm, INTERCEPT_PAUSE);
1349 	}
1350 
1351 	svm_recalc_instruction_intercepts(vcpu, svm);
1352 
1353 	/*
1354 	 * If the host supports V_SPEC_CTRL then disable the interception
1355 	 * of MSR_IA32_SPEC_CTRL.
1356 	 */
1357 	if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
1358 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
1359 
1360 	if (kvm_vcpu_apicv_active(vcpu))
1361 		avic_init_vmcb(svm, vmcb);
1362 
1363 	if (vnmi)
1364 		svm->vmcb->control.int_ctl |= V_NMI_ENABLE_MASK;
1365 
1366 	if (vgif) {
1367 		svm_clr_intercept(svm, INTERCEPT_STGI);
1368 		svm_clr_intercept(svm, INTERCEPT_CLGI);
1369 		svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK;
1370 	}
1371 
1372 	if (sev_guest(vcpu->kvm))
1373 		sev_init_vmcb(svm);
1374 
1375 	svm_hv_init_vmcb(vmcb);
1376 	init_vmcb_after_set_cpuid(vcpu);
1377 
1378 	vmcb_mark_all_dirty(vmcb);
1379 
1380 	enable_gif(svm);
1381 }
1382 
1383 static void __svm_vcpu_reset(struct kvm_vcpu *vcpu)
1384 {
1385 	struct vcpu_svm *svm = to_svm(vcpu);
1386 
1387 	svm_vcpu_init_msrpm(vcpu, svm->msrpm);
1388 
1389 	svm_init_osvw(vcpu);
1390 	vcpu->arch.microcode_version = 0x01000065;
1391 	svm->tsc_ratio_msr = kvm_caps.default_tsc_scaling_ratio;
1392 
1393 	svm->nmi_masked = false;
1394 	svm->awaiting_iret_completion = false;
1395 
1396 	if (sev_es_guest(vcpu->kvm))
1397 		sev_es_vcpu_reset(svm);
1398 }
1399 
1400 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
1401 {
1402 	struct vcpu_svm *svm = to_svm(vcpu);
1403 
1404 	svm->spec_ctrl = 0;
1405 	svm->virt_spec_ctrl = 0;
1406 
1407 	init_vmcb(vcpu);
1408 
1409 	if (!init_event)
1410 		__svm_vcpu_reset(vcpu);
1411 }
1412 
1413 void svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb)
1414 {
1415 	svm->current_vmcb = target_vmcb;
1416 	svm->vmcb = target_vmcb->ptr;
1417 }
1418 
1419 static int svm_vcpu_create(struct kvm_vcpu *vcpu)
1420 {
1421 	struct vcpu_svm *svm;
1422 	struct page *vmcb01_page;
1423 	struct page *vmsa_page = NULL;
1424 	int err;
1425 
1426 	BUILD_BUG_ON(offsetof(struct vcpu_svm, vcpu) != 0);
1427 	svm = to_svm(vcpu);
1428 
1429 	err = -ENOMEM;
1430 	vmcb01_page = snp_safe_alloc_page(vcpu);
1431 	if (!vmcb01_page)
1432 		goto out;
1433 
1434 	if (sev_es_guest(vcpu->kvm)) {
1435 		/*
1436 		 * SEV-ES guests require a separate VMSA page used to contain
1437 		 * the encrypted register state of the guest.
1438 		 */
1439 		vmsa_page = snp_safe_alloc_page(vcpu);
1440 		if (!vmsa_page)
1441 			goto error_free_vmcb_page;
1442 	}
1443 
1444 	err = avic_init_vcpu(svm);
1445 	if (err)
1446 		goto error_free_vmsa_page;
1447 
1448 	svm->msrpm = svm_vcpu_alloc_msrpm();
1449 	if (!svm->msrpm) {
1450 		err = -ENOMEM;
1451 		goto error_free_vmsa_page;
1452 	}
1453 
1454 	svm->x2avic_msrs_intercepted = true;
1455 
1456 	svm->vmcb01.ptr = page_address(vmcb01_page);
1457 	svm->vmcb01.pa = __sme_set(page_to_pfn(vmcb01_page) << PAGE_SHIFT);
1458 	svm_switch_vmcb(svm, &svm->vmcb01);
1459 
1460 	if (vmsa_page)
1461 		svm->sev_es.vmsa = page_address(vmsa_page);
1462 
1463 	svm->guest_state_loaded = false;
1464 
1465 	return 0;
1466 
1467 error_free_vmsa_page:
1468 	if (vmsa_page)
1469 		__free_page(vmsa_page);
1470 error_free_vmcb_page:
1471 	__free_page(vmcb01_page);
1472 out:
1473 	return err;
1474 }
1475 
1476 static void svm_clear_current_vmcb(struct vmcb *vmcb)
1477 {
1478 	int i;
1479 
1480 	for_each_online_cpu(i)
1481 		cmpxchg(per_cpu_ptr(&svm_data.current_vmcb, i), vmcb, NULL);
1482 }
1483 
1484 static void svm_vcpu_free(struct kvm_vcpu *vcpu)
1485 {
1486 	struct vcpu_svm *svm = to_svm(vcpu);
1487 
1488 	/*
1489 	 * The vmcb page can be recycled, causing a false negative in
1490 	 * svm_vcpu_load(). So, ensure that no logical CPU has this
1491 	 * vmcb page recorded as its current vmcb.
1492 	 */
1493 	svm_clear_current_vmcb(svm->vmcb);
1494 
1495 	svm_leave_nested(vcpu);
1496 	svm_free_nested(svm);
1497 
1498 	sev_free_vcpu(vcpu);
1499 
1500 	__free_page(pfn_to_page(__sme_clr(svm->vmcb01.pa) >> PAGE_SHIFT));
1501 	__free_pages(virt_to_page(svm->msrpm), get_order(MSRPM_SIZE));
1502 }
1503 
1504 static struct sev_es_save_area *sev_es_host_save_area(struct svm_cpu_data *sd)
1505 {
1506 	return page_address(sd->save_area) + 0x400;
1507 }
1508 
1509 static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
1510 {
1511 	struct vcpu_svm *svm = to_svm(vcpu);
1512 	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu);
1513 
1514 	if (sev_es_guest(vcpu->kvm))
1515 		sev_es_unmap_ghcb(svm);
1516 
1517 	if (svm->guest_state_loaded)
1518 		return;
1519 
1520 	/*
1521 	 * Save additional host state that will be restored on VMEXIT (sev-es)
1522 	 * or subsequent vmload of host save area.
1523 	 */
1524 	vmsave(sd->save_area_pa);
1525 	if (sev_es_guest(vcpu->kvm))
1526 		sev_es_prepare_switch_to_guest(svm, sev_es_host_save_area(sd));
1527 
1528 	if (tsc_scaling)
1529 		__svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio);
1530 
1531 	/*
1532 	 * TSC_AUX is always virtualized for SEV-ES guests when the feature is
1533 	 * available. The user return MSR support is not required in this case
1534 	 * because TSC_AUX is restored on #VMEXIT from the host save area
1535 	 * (which has been initialized in svm_hardware_enable()).
1536 	 */
1537 	if (likely(tsc_aux_uret_slot >= 0) &&
1538 	    (!boot_cpu_has(X86_FEATURE_V_TSC_AUX) || !sev_es_guest(vcpu->kvm)))
1539 		kvm_set_user_return_msr(tsc_aux_uret_slot, svm->tsc_aux, -1ull);
1540 
1541 	svm->guest_state_loaded = true;
1542 }
1543 
1544 static void svm_prepare_host_switch(struct kvm_vcpu *vcpu)
1545 {
1546 	to_svm(vcpu)->guest_state_loaded = false;
1547 }
1548 
1549 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1550 {
1551 	struct vcpu_svm *svm = to_svm(vcpu);
1552 	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu);
1553 
1554 	if (sd->current_vmcb != svm->vmcb) {
1555 		sd->current_vmcb = svm->vmcb;
1556 
1557 		if (!cpu_feature_enabled(X86_FEATURE_IBPB_ON_VMEXIT))
1558 			indirect_branch_prediction_barrier();
1559 	}
1560 	if (kvm_vcpu_apicv_active(vcpu))
1561 		avic_vcpu_load(vcpu, cpu);
1562 }
1563 
1564 static void svm_vcpu_put(struct kvm_vcpu *vcpu)
1565 {
1566 	if (kvm_vcpu_apicv_active(vcpu))
1567 		avic_vcpu_put(vcpu);
1568 
1569 	svm_prepare_host_switch(vcpu);
1570 
1571 	++vcpu->stat.host_state_reload;
1572 }
1573 
1574 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
1575 {
1576 	struct vcpu_svm *svm = to_svm(vcpu);
1577 	unsigned long rflags = svm->vmcb->save.rflags;
1578 
1579 	if (svm->nmi_singlestep) {
1580 		/* Hide our flags if they were not set by the guest */
1581 		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1582 			rflags &= ~X86_EFLAGS_TF;
1583 		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1584 			rflags &= ~X86_EFLAGS_RF;
1585 	}
1586 	return rflags;
1587 }
1588 
1589 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
1590 {
1591 	if (to_svm(vcpu)->nmi_singlestep)
1592 		rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
1593 
1594        /*
1595         * Any change of EFLAGS.VM is accompanied by a reload of SS
1596         * (caused by either a task switch or an inter-privilege IRET),
1597         * so we do not need to update the CPL here.
1598         */
1599 	to_svm(vcpu)->vmcb->save.rflags = rflags;
1600 }
1601 
1602 static bool svm_get_if_flag(struct kvm_vcpu *vcpu)
1603 {
1604 	struct vmcb *vmcb = to_svm(vcpu)->vmcb;
1605 
1606 	return sev_es_guest(vcpu->kvm)
1607 		? vmcb->control.int_state & SVM_GUEST_INTERRUPT_MASK
1608 		: kvm_get_rflags(vcpu) & X86_EFLAGS_IF;
1609 }
1610 
1611 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
1612 {
1613 	kvm_register_mark_available(vcpu, reg);
1614 
1615 	switch (reg) {
1616 	case VCPU_EXREG_PDPTR:
1617 		/*
1618 		 * When !npt_enabled, mmu->pdptrs[] is already available since
1619 		 * it is always updated per SDM when moving to CRs.
1620 		 */
1621 		if (npt_enabled)
1622 			load_pdptrs(vcpu, kvm_read_cr3(vcpu));
1623 		break;
1624 	default:
1625 		KVM_BUG_ON(1, vcpu->kvm);
1626 	}
1627 }
1628 
1629 static void svm_set_vintr(struct vcpu_svm *svm)
1630 {
1631 	struct vmcb_control_area *control;
1632 
1633 	/*
1634 	 * The following fields are ignored when AVIC is enabled
1635 	 */
1636 	WARN_ON(kvm_vcpu_apicv_activated(&svm->vcpu));
1637 
1638 	svm_set_intercept(svm, INTERCEPT_VINTR);
1639 
1640 	/*
1641 	 * Recalculating intercepts may have cleared the VINTR intercept.  If
1642 	 * V_INTR_MASKING is enabled in vmcb12, then the effective RFLAGS.IF
1643 	 * for L1 physical interrupts is L1's RFLAGS.IF at the time of VMRUN.
1644 	 * Requesting an interrupt window if save.RFLAGS.IF=0 is pointless as
1645 	 * interrupts will never be unblocked while L2 is running.
1646 	 */
1647 	if (!svm_is_intercept(svm, INTERCEPT_VINTR))
1648 		return;
1649 
1650 	/*
1651 	 * This is just a dummy VINTR to actually cause a vmexit to happen.
1652 	 * Actual injection of virtual interrupts happens through EVENTINJ.
1653 	 */
1654 	control = &svm->vmcb->control;
1655 	control->int_vector = 0x0;
1656 	control->int_ctl &= ~V_INTR_PRIO_MASK;
1657 	control->int_ctl |= V_IRQ_MASK |
1658 		((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
1659 	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
1660 }
1661 
1662 static void svm_clear_vintr(struct vcpu_svm *svm)
1663 {
1664 	svm_clr_intercept(svm, INTERCEPT_VINTR);
1665 
1666 	/* Drop int_ctl fields related to VINTR injection.  */
1667 	svm->vmcb->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK;
1668 	if (is_guest_mode(&svm->vcpu)) {
1669 		svm->vmcb01.ptr->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK;
1670 
1671 		WARN_ON((svm->vmcb->control.int_ctl & V_TPR_MASK) !=
1672 			(svm->nested.ctl.int_ctl & V_TPR_MASK));
1673 
1674 		svm->vmcb->control.int_ctl |= svm->nested.ctl.int_ctl &
1675 			V_IRQ_INJECTION_BITS_MASK;
1676 
1677 		svm->vmcb->control.int_vector = svm->nested.ctl.int_vector;
1678 	}
1679 
1680 	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
1681 }
1682 
1683 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
1684 {
1685 	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1686 	struct vmcb_save_area *save01 = &to_svm(vcpu)->vmcb01.ptr->save;
1687 
1688 	switch (seg) {
1689 	case VCPU_SREG_CS: return &save->cs;
1690 	case VCPU_SREG_DS: return &save->ds;
1691 	case VCPU_SREG_ES: return &save->es;
1692 	case VCPU_SREG_FS: return &save01->fs;
1693 	case VCPU_SREG_GS: return &save01->gs;
1694 	case VCPU_SREG_SS: return &save->ss;
1695 	case VCPU_SREG_TR: return &save01->tr;
1696 	case VCPU_SREG_LDTR: return &save01->ldtr;
1697 	}
1698 	BUG();
1699 	return NULL;
1700 }
1701 
1702 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1703 {
1704 	struct vmcb_seg *s = svm_seg(vcpu, seg);
1705 
1706 	return s->base;
1707 }
1708 
1709 static void svm_get_segment(struct kvm_vcpu *vcpu,
1710 			    struct kvm_segment *var, int seg)
1711 {
1712 	struct vmcb_seg *s = svm_seg(vcpu, seg);
1713 
1714 	var->base = s->base;
1715 	var->limit = s->limit;
1716 	var->selector = s->selector;
1717 	var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
1718 	var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
1719 	var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
1720 	var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
1721 	var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
1722 	var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
1723 	var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
1724 
1725 	/*
1726 	 * AMD CPUs circa 2014 track the G bit for all segments except CS.
1727 	 * However, the SVM spec states that the G bit is not observed by the
1728 	 * CPU, and some VMware virtual CPUs drop the G bit for all segments.
1729 	 * So let's synthesize a legal G bit for all segments, this helps
1730 	 * running KVM nested. It also helps cross-vendor migration, because
1731 	 * Intel's vmentry has a check on the 'G' bit.
1732 	 */
1733 	var->g = s->limit > 0xfffff;
1734 
1735 	/*
1736 	 * AMD's VMCB does not have an explicit unusable field, so emulate it
1737 	 * for cross vendor migration purposes by "not present"
1738 	 */
1739 	var->unusable = !var->present;
1740 
1741 	switch (seg) {
1742 	case VCPU_SREG_TR:
1743 		/*
1744 		 * Work around a bug where the busy flag in the tr selector
1745 		 * isn't exposed
1746 		 */
1747 		var->type |= 0x2;
1748 		break;
1749 	case VCPU_SREG_DS:
1750 	case VCPU_SREG_ES:
1751 	case VCPU_SREG_FS:
1752 	case VCPU_SREG_GS:
1753 		/*
1754 		 * The accessed bit must always be set in the segment
1755 		 * descriptor cache, although it can be cleared in the
1756 		 * descriptor, the cached bit always remains at 1. Since
1757 		 * Intel has a check on this, set it here to support
1758 		 * cross-vendor migration.
1759 		 */
1760 		if (!var->unusable)
1761 			var->type |= 0x1;
1762 		break;
1763 	case VCPU_SREG_SS:
1764 		/*
1765 		 * On AMD CPUs sometimes the DB bit in the segment
1766 		 * descriptor is left as 1, although the whole segment has
1767 		 * been made unusable. Clear it here to pass an Intel VMX
1768 		 * entry check when cross vendor migrating.
1769 		 */
1770 		if (var->unusable)
1771 			var->db = 0;
1772 		/* This is symmetric with svm_set_segment() */
1773 		var->dpl = to_svm(vcpu)->vmcb->save.cpl;
1774 		break;
1775 	}
1776 }
1777 
1778 static int svm_get_cpl(struct kvm_vcpu *vcpu)
1779 {
1780 	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1781 
1782 	return save->cpl;
1783 }
1784 
1785 static void svm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
1786 {
1787 	struct kvm_segment cs;
1788 
1789 	svm_get_segment(vcpu, &cs, VCPU_SREG_CS);
1790 	*db = cs.db;
1791 	*l = cs.l;
1792 }
1793 
1794 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1795 {
1796 	struct vcpu_svm *svm = to_svm(vcpu);
1797 
1798 	dt->size = svm->vmcb->save.idtr.limit;
1799 	dt->address = svm->vmcb->save.idtr.base;
1800 }
1801 
1802 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1803 {
1804 	struct vcpu_svm *svm = to_svm(vcpu);
1805 
1806 	svm->vmcb->save.idtr.limit = dt->size;
1807 	svm->vmcb->save.idtr.base = dt->address ;
1808 	vmcb_mark_dirty(svm->vmcb, VMCB_DT);
1809 }
1810 
1811 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1812 {
1813 	struct vcpu_svm *svm = to_svm(vcpu);
1814 
1815 	dt->size = svm->vmcb->save.gdtr.limit;
1816 	dt->address = svm->vmcb->save.gdtr.base;
1817 }
1818 
1819 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1820 {
1821 	struct vcpu_svm *svm = to_svm(vcpu);
1822 
1823 	svm->vmcb->save.gdtr.limit = dt->size;
1824 	svm->vmcb->save.gdtr.base = dt->address ;
1825 	vmcb_mark_dirty(svm->vmcb, VMCB_DT);
1826 }
1827 
1828 static void sev_post_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
1829 {
1830 	struct vcpu_svm *svm = to_svm(vcpu);
1831 
1832 	/*
1833 	 * For guests that don't set guest_state_protected, the cr3 update is
1834 	 * handled via kvm_mmu_load() while entering the guest. For guests
1835 	 * that do (SEV-ES/SEV-SNP), the cr3 update needs to be written to
1836 	 * VMCB save area now, since the save area will become the initial
1837 	 * contents of the VMSA, and future VMCB save area updates won't be
1838 	 * seen.
1839 	 */
1840 	if (sev_es_guest(vcpu->kvm)) {
1841 		svm->vmcb->save.cr3 = cr3;
1842 		vmcb_mark_dirty(svm->vmcb, VMCB_CR);
1843 	}
1844 }
1845 
1846 static bool svm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1847 {
1848 	return true;
1849 }
1850 
1851 void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1852 {
1853 	struct vcpu_svm *svm = to_svm(vcpu);
1854 	u64 hcr0 = cr0;
1855 	bool old_paging = is_paging(vcpu);
1856 
1857 #ifdef CONFIG_X86_64
1858 	if (vcpu->arch.efer & EFER_LME) {
1859 		if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
1860 			vcpu->arch.efer |= EFER_LMA;
1861 			if (!vcpu->arch.guest_state_protected)
1862 				svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
1863 		}
1864 
1865 		if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
1866 			vcpu->arch.efer &= ~EFER_LMA;
1867 			if (!vcpu->arch.guest_state_protected)
1868 				svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
1869 		}
1870 	}
1871 #endif
1872 	vcpu->arch.cr0 = cr0;
1873 
1874 	if (!npt_enabled) {
1875 		hcr0 |= X86_CR0_PG | X86_CR0_WP;
1876 		if (old_paging != is_paging(vcpu))
1877 			svm_set_cr4(vcpu, kvm_read_cr4(vcpu));
1878 	}
1879 
1880 	/*
1881 	 * re-enable caching here because the QEMU bios
1882 	 * does not do it - this results in some delay at
1883 	 * reboot
1884 	 */
1885 	if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
1886 		hcr0 &= ~(X86_CR0_CD | X86_CR0_NW);
1887 
1888 	svm->vmcb->save.cr0 = hcr0;
1889 	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
1890 
1891 	/*
1892 	 * SEV-ES guests must always keep the CR intercepts cleared. CR
1893 	 * tracking is done using the CR write traps.
1894 	 */
1895 	if (sev_es_guest(vcpu->kvm))
1896 		return;
1897 
1898 	if (hcr0 == cr0) {
1899 		/* Selective CR0 write remains on.  */
1900 		svm_clr_intercept(svm, INTERCEPT_CR0_READ);
1901 		svm_clr_intercept(svm, INTERCEPT_CR0_WRITE);
1902 	} else {
1903 		svm_set_intercept(svm, INTERCEPT_CR0_READ);
1904 		svm_set_intercept(svm, INTERCEPT_CR0_WRITE);
1905 	}
1906 }
1907 
1908 static bool svm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1909 {
1910 	return true;
1911 }
1912 
1913 void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1914 {
1915 	unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
1916 	unsigned long old_cr4 = vcpu->arch.cr4;
1917 
1918 	if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
1919 		svm_flush_tlb_current(vcpu);
1920 
1921 	vcpu->arch.cr4 = cr4;
1922 	if (!npt_enabled) {
1923 		cr4 |= X86_CR4_PAE;
1924 
1925 		if (!is_paging(vcpu))
1926 			cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE);
1927 	}
1928 	cr4 |= host_cr4_mce;
1929 	to_svm(vcpu)->vmcb->save.cr4 = cr4;
1930 	vmcb_mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
1931 
1932 	if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE))
1933 		kvm_update_cpuid_runtime(vcpu);
1934 }
1935 
1936 static void svm_set_segment(struct kvm_vcpu *vcpu,
1937 			    struct kvm_segment *var, int seg)
1938 {
1939 	struct vcpu_svm *svm = to_svm(vcpu);
1940 	struct vmcb_seg *s = svm_seg(vcpu, seg);
1941 
1942 	s->base = var->base;
1943 	s->limit = var->limit;
1944 	s->selector = var->selector;
1945 	s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
1946 	s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
1947 	s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
1948 	s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT;
1949 	s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
1950 	s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
1951 	s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
1952 	s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
1953 
1954 	/*
1955 	 * This is always accurate, except if SYSRET returned to a segment
1956 	 * with SS.DPL != 3.  Intel does not have this quirk, and always
1957 	 * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
1958 	 * would entail passing the CPL to userspace and back.
1959 	 */
1960 	if (seg == VCPU_SREG_SS)
1961 		/* This is symmetric with svm_get_segment() */
1962 		svm->vmcb->save.cpl = (var->dpl & 3);
1963 
1964 	vmcb_mark_dirty(svm->vmcb, VMCB_SEG);
1965 }
1966 
1967 static void svm_update_exception_bitmap(struct kvm_vcpu *vcpu)
1968 {
1969 	struct vcpu_svm *svm = to_svm(vcpu);
1970 
1971 	clr_exception_intercept(svm, BP_VECTOR);
1972 
1973 	if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
1974 		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
1975 			set_exception_intercept(svm, BP_VECTOR);
1976 	}
1977 }
1978 
1979 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
1980 {
1981 	if (sd->next_asid > sd->max_asid) {
1982 		++sd->asid_generation;
1983 		sd->next_asid = sd->min_asid;
1984 		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
1985 		vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
1986 	}
1987 
1988 	svm->current_vmcb->asid_generation = sd->asid_generation;
1989 	svm->asid = sd->next_asid++;
1990 }
1991 
1992 static void svm_set_dr6(struct vcpu_svm *svm, unsigned long value)
1993 {
1994 	struct vmcb *vmcb = svm->vmcb;
1995 
1996 	if (svm->vcpu.arch.guest_state_protected)
1997 		return;
1998 
1999 	if (unlikely(value != vmcb->save.dr6)) {
2000 		vmcb->save.dr6 = value;
2001 		vmcb_mark_dirty(vmcb, VMCB_DR);
2002 	}
2003 }
2004 
2005 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
2006 {
2007 	struct vcpu_svm *svm = to_svm(vcpu);
2008 
2009 	if (WARN_ON_ONCE(sev_es_guest(vcpu->kvm)))
2010 		return;
2011 
2012 	get_debugreg(vcpu->arch.db[0], 0);
2013 	get_debugreg(vcpu->arch.db[1], 1);
2014 	get_debugreg(vcpu->arch.db[2], 2);
2015 	get_debugreg(vcpu->arch.db[3], 3);
2016 	/*
2017 	 * We cannot reset svm->vmcb->save.dr6 to DR6_ACTIVE_LOW here,
2018 	 * because db_interception might need it.  We can do it before vmentry.
2019 	 */
2020 	vcpu->arch.dr6 = svm->vmcb->save.dr6;
2021 	vcpu->arch.dr7 = svm->vmcb->save.dr7;
2022 	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
2023 	set_dr_intercepts(svm);
2024 }
2025 
2026 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
2027 {
2028 	struct vcpu_svm *svm = to_svm(vcpu);
2029 
2030 	if (vcpu->arch.guest_state_protected)
2031 		return;
2032 
2033 	svm->vmcb->save.dr7 = value;
2034 	vmcb_mark_dirty(svm->vmcb, VMCB_DR);
2035 }
2036 
2037 static int pf_interception(struct kvm_vcpu *vcpu)
2038 {
2039 	struct vcpu_svm *svm = to_svm(vcpu);
2040 
2041 	u64 fault_address = svm->vmcb->control.exit_info_2;
2042 	u64 error_code = svm->vmcb->control.exit_info_1;
2043 
2044 	return kvm_handle_page_fault(vcpu, error_code, fault_address,
2045 			static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2046 			svm->vmcb->control.insn_bytes : NULL,
2047 			svm->vmcb->control.insn_len);
2048 }
2049 
2050 static int npf_interception(struct kvm_vcpu *vcpu)
2051 {
2052 	struct vcpu_svm *svm = to_svm(vcpu);
2053 
2054 	u64 fault_address = svm->vmcb->control.exit_info_2;
2055 	u64 error_code = svm->vmcb->control.exit_info_1;
2056 
2057 	/*
2058 	 * WARN if hardware generates a fault with an error code that collides
2059 	 * with KVM-defined sythentic flags.  Clear the flags and continue on,
2060 	 * i.e. don't terminate the VM, as KVM can't possibly be relying on a
2061 	 * flag that KVM doesn't know about.
2062 	 */
2063 	if (WARN_ON_ONCE(error_code & PFERR_SYNTHETIC_MASK))
2064 		error_code &= ~PFERR_SYNTHETIC_MASK;
2065 
2066 	trace_kvm_page_fault(vcpu, fault_address, error_code);
2067 	return kvm_mmu_page_fault(vcpu, fault_address, error_code,
2068 			static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2069 			svm->vmcb->control.insn_bytes : NULL,
2070 			svm->vmcb->control.insn_len);
2071 }
2072 
2073 static int db_interception(struct kvm_vcpu *vcpu)
2074 {
2075 	struct kvm_run *kvm_run = vcpu->run;
2076 	struct vcpu_svm *svm = to_svm(vcpu);
2077 
2078 	if (!(vcpu->guest_debug &
2079 	      (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
2080 		!svm->nmi_singlestep) {
2081 		u32 payload = svm->vmcb->save.dr6 ^ DR6_ACTIVE_LOW;
2082 		kvm_queue_exception_p(vcpu, DB_VECTOR, payload);
2083 		return 1;
2084 	}
2085 
2086 	if (svm->nmi_singlestep) {
2087 		disable_nmi_singlestep(svm);
2088 		/* Make sure we check for pending NMIs upon entry */
2089 		kvm_make_request(KVM_REQ_EVENT, vcpu);
2090 	}
2091 
2092 	if (vcpu->guest_debug &
2093 	    (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
2094 		kvm_run->exit_reason = KVM_EXIT_DEBUG;
2095 		kvm_run->debug.arch.dr6 = svm->vmcb->save.dr6;
2096 		kvm_run->debug.arch.dr7 = svm->vmcb->save.dr7;
2097 		kvm_run->debug.arch.pc =
2098 			svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2099 		kvm_run->debug.arch.exception = DB_VECTOR;
2100 		return 0;
2101 	}
2102 
2103 	return 1;
2104 }
2105 
2106 static int bp_interception(struct kvm_vcpu *vcpu)
2107 {
2108 	struct vcpu_svm *svm = to_svm(vcpu);
2109 	struct kvm_run *kvm_run = vcpu->run;
2110 
2111 	kvm_run->exit_reason = KVM_EXIT_DEBUG;
2112 	kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2113 	kvm_run->debug.arch.exception = BP_VECTOR;
2114 	return 0;
2115 }
2116 
2117 static int ud_interception(struct kvm_vcpu *vcpu)
2118 {
2119 	return handle_ud(vcpu);
2120 }
2121 
2122 static int ac_interception(struct kvm_vcpu *vcpu)
2123 {
2124 	kvm_queue_exception_e(vcpu, AC_VECTOR, 0);
2125 	return 1;
2126 }
2127 
2128 static bool is_erratum_383(void)
2129 {
2130 	int err, i;
2131 	u64 value;
2132 
2133 	if (!erratum_383_found)
2134 		return false;
2135 
2136 	value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
2137 	if (err)
2138 		return false;
2139 
2140 	/* Bit 62 may or may not be set for this mce */
2141 	value &= ~(1ULL << 62);
2142 
2143 	if (value != 0xb600000000010015ULL)
2144 		return false;
2145 
2146 	/* Clear MCi_STATUS registers */
2147 	for (i = 0; i < 6; ++i)
2148 		native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
2149 
2150 	value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
2151 	if (!err) {
2152 		u32 low, high;
2153 
2154 		value &= ~(1ULL << 2);
2155 		low    = lower_32_bits(value);
2156 		high   = upper_32_bits(value);
2157 
2158 		native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
2159 	}
2160 
2161 	/* Flush tlb to evict multi-match entries */
2162 	__flush_tlb_all();
2163 
2164 	return true;
2165 }
2166 
2167 static void svm_handle_mce(struct kvm_vcpu *vcpu)
2168 {
2169 	if (is_erratum_383()) {
2170 		/*
2171 		 * Erratum 383 triggered. Guest state is corrupt so kill the
2172 		 * guest.
2173 		 */
2174 		pr_err("Guest triggered AMD Erratum 383\n");
2175 
2176 		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
2177 
2178 		return;
2179 	}
2180 
2181 	/*
2182 	 * On an #MC intercept the MCE handler is not called automatically in
2183 	 * the host. So do it by hand here.
2184 	 */
2185 	kvm_machine_check();
2186 }
2187 
2188 static int mc_interception(struct kvm_vcpu *vcpu)
2189 {
2190 	return 1;
2191 }
2192 
2193 static int shutdown_interception(struct kvm_vcpu *vcpu)
2194 {
2195 	struct kvm_run *kvm_run = vcpu->run;
2196 	struct vcpu_svm *svm = to_svm(vcpu);
2197 
2198 
2199 	/*
2200 	 * VMCB is undefined after a SHUTDOWN intercept.  INIT the vCPU to put
2201 	 * the VMCB in a known good state.  Unfortuately, KVM doesn't have
2202 	 * KVM_MP_STATE_SHUTDOWN and can't add it without potentially breaking
2203 	 * userspace.  At a platform view, INIT is acceptable behavior as
2204 	 * there exist bare metal platforms that automatically INIT the CPU
2205 	 * in response to shutdown.
2206 	 *
2207 	 * The VM save area for SEV-ES guests has already been encrypted so it
2208 	 * cannot be reinitialized, i.e. synthesizing INIT is futile.
2209 	 */
2210 	if (!sev_es_guest(vcpu->kvm)) {
2211 		clear_page(svm->vmcb);
2212 		kvm_vcpu_reset(vcpu, true);
2213 	}
2214 
2215 	kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
2216 	return 0;
2217 }
2218 
2219 static int io_interception(struct kvm_vcpu *vcpu)
2220 {
2221 	struct vcpu_svm *svm = to_svm(vcpu);
2222 	u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
2223 	int size, in, string;
2224 	unsigned port;
2225 
2226 	++vcpu->stat.io_exits;
2227 	string = (io_info & SVM_IOIO_STR_MASK) != 0;
2228 	in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
2229 	port = io_info >> 16;
2230 	size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
2231 
2232 	if (string) {
2233 		if (sev_es_guest(vcpu->kvm))
2234 			return sev_es_string_io(svm, size, port, in);
2235 		else
2236 			return kvm_emulate_instruction(vcpu, 0);
2237 	}
2238 
2239 	svm->next_rip = svm->vmcb->control.exit_info_2;
2240 
2241 	return kvm_fast_pio(vcpu, size, port, in);
2242 }
2243 
2244 static int nmi_interception(struct kvm_vcpu *vcpu)
2245 {
2246 	return 1;
2247 }
2248 
2249 static int smi_interception(struct kvm_vcpu *vcpu)
2250 {
2251 	return 1;
2252 }
2253 
2254 static int intr_interception(struct kvm_vcpu *vcpu)
2255 {
2256 	++vcpu->stat.irq_exits;
2257 	return 1;
2258 }
2259 
2260 static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload)
2261 {
2262 	struct vcpu_svm *svm = to_svm(vcpu);
2263 	struct vmcb *vmcb12;
2264 	struct kvm_host_map map;
2265 	int ret;
2266 
2267 	if (nested_svm_check_permissions(vcpu))
2268 		return 1;
2269 
2270 	ret = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map);
2271 	if (ret) {
2272 		if (ret == -EINVAL)
2273 			kvm_inject_gp(vcpu, 0);
2274 		return 1;
2275 	}
2276 
2277 	vmcb12 = map.hva;
2278 
2279 	ret = kvm_skip_emulated_instruction(vcpu);
2280 
2281 	if (vmload) {
2282 		svm_copy_vmloadsave_state(svm->vmcb, vmcb12);
2283 		svm->sysenter_eip_hi = 0;
2284 		svm->sysenter_esp_hi = 0;
2285 	} else {
2286 		svm_copy_vmloadsave_state(vmcb12, svm->vmcb);
2287 	}
2288 
2289 	kvm_vcpu_unmap(vcpu, &map, true);
2290 
2291 	return ret;
2292 }
2293 
2294 static int vmload_interception(struct kvm_vcpu *vcpu)
2295 {
2296 	return vmload_vmsave_interception(vcpu, true);
2297 }
2298 
2299 static int vmsave_interception(struct kvm_vcpu *vcpu)
2300 {
2301 	return vmload_vmsave_interception(vcpu, false);
2302 }
2303 
2304 static int vmrun_interception(struct kvm_vcpu *vcpu)
2305 {
2306 	if (nested_svm_check_permissions(vcpu))
2307 		return 1;
2308 
2309 	return nested_svm_vmrun(vcpu);
2310 }
2311 
2312 enum {
2313 	NONE_SVM_INSTR,
2314 	SVM_INSTR_VMRUN,
2315 	SVM_INSTR_VMLOAD,
2316 	SVM_INSTR_VMSAVE,
2317 };
2318 
2319 /* Return NONE_SVM_INSTR if not SVM instrs, otherwise return decode result */
2320 static int svm_instr_opcode(struct kvm_vcpu *vcpu)
2321 {
2322 	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
2323 
2324 	if (ctxt->b != 0x1 || ctxt->opcode_len != 2)
2325 		return NONE_SVM_INSTR;
2326 
2327 	switch (ctxt->modrm) {
2328 	case 0xd8: /* VMRUN */
2329 		return SVM_INSTR_VMRUN;
2330 	case 0xda: /* VMLOAD */
2331 		return SVM_INSTR_VMLOAD;
2332 	case 0xdb: /* VMSAVE */
2333 		return SVM_INSTR_VMSAVE;
2334 	default:
2335 		break;
2336 	}
2337 
2338 	return NONE_SVM_INSTR;
2339 }
2340 
2341 static int emulate_svm_instr(struct kvm_vcpu *vcpu, int opcode)
2342 {
2343 	const int guest_mode_exit_codes[] = {
2344 		[SVM_INSTR_VMRUN] = SVM_EXIT_VMRUN,
2345 		[SVM_INSTR_VMLOAD] = SVM_EXIT_VMLOAD,
2346 		[SVM_INSTR_VMSAVE] = SVM_EXIT_VMSAVE,
2347 	};
2348 	int (*const svm_instr_handlers[])(struct kvm_vcpu *vcpu) = {
2349 		[SVM_INSTR_VMRUN] = vmrun_interception,
2350 		[SVM_INSTR_VMLOAD] = vmload_interception,
2351 		[SVM_INSTR_VMSAVE] = vmsave_interception,
2352 	};
2353 	struct vcpu_svm *svm = to_svm(vcpu);
2354 	int ret;
2355 
2356 	if (is_guest_mode(vcpu)) {
2357 		/* Returns '1' or -errno on failure, '0' on success. */
2358 		ret = nested_svm_simple_vmexit(svm, guest_mode_exit_codes[opcode]);
2359 		if (ret)
2360 			return ret;
2361 		return 1;
2362 	}
2363 	return svm_instr_handlers[opcode](vcpu);
2364 }
2365 
2366 /*
2367  * #GP handling code. Note that #GP can be triggered under the following two
2368  * cases:
2369  *   1) SVM VM-related instructions (VMRUN/VMSAVE/VMLOAD) that trigger #GP on
2370  *      some AMD CPUs when EAX of these instructions are in the reserved memory
2371  *      regions (e.g. SMM memory on host).
2372  *   2) VMware backdoor
2373  */
2374 static int gp_interception(struct kvm_vcpu *vcpu)
2375 {
2376 	struct vcpu_svm *svm = to_svm(vcpu);
2377 	u32 error_code = svm->vmcb->control.exit_info_1;
2378 	int opcode;
2379 
2380 	/* Both #GP cases have zero error_code */
2381 	if (error_code)
2382 		goto reinject;
2383 
2384 	/* Decode the instruction for usage later */
2385 	if (x86_decode_emulated_instruction(vcpu, 0, NULL, 0) != EMULATION_OK)
2386 		goto reinject;
2387 
2388 	opcode = svm_instr_opcode(vcpu);
2389 
2390 	if (opcode == NONE_SVM_INSTR) {
2391 		if (!enable_vmware_backdoor)
2392 			goto reinject;
2393 
2394 		/*
2395 		 * VMware backdoor emulation on #GP interception only handles
2396 		 * IN{S}, OUT{S}, and RDPMC.
2397 		 */
2398 		if (!is_guest_mode(vcpu))
2399 			return kvm_emulate_instruction(vcpu,
2400 				EMULTYPE_VMWARE_GP | EMULTYPE_NO_DECODE);
2401 	} else {
2402 		/* All SVM instructions expect page aligned RAX */
2403 		if (svm->vmcb->save.rax & ~PAGE_MASK)
2404 			goto reinject;
2405 
2406 		return emulate_svm_instr(vcpu, opcode);
2407 	}
2408 
2409 reinject:
2410 	kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
2411 	return 1;
2412 }
2413 
2414 void svm_set_gif(struct vcpu_svm *svm, bool value)
2415 {
2416 	if (value) {
2417 		/*
2418 		 * If VGIF is enabled, the STGI intercept is only added to
2419 		 * detect the opening of the SMI/NMI window; remove it now.
2420 		 * Likewise, clear the VINTR intercept, we will set it
2421 		 * again while processing KVM_REQ_EVENT if needed.
2422 		 */
2423 		if (vgif)
2424 			svm_clr_intercept(svm, INTERCEPT_STGI);
2425 		if (svm_is_intercept(svm, INTERCEPT_VINTR))
2426 			svm_clear_vintr(svm);
2427 
2428 		enable_gif(svm);
2429 		if (svm->vcpu.arch.smi_pending ||
2430 		    svm->vcpu.arch.nmi_pending ||
2431 		    kvm_cpu_has_injectable_intr(&svm->vcpu) ||
2432 		    kvm_apic_has_pending_init_or_sipi(&svm->vcpu))
2433 			kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2434 	} else {
2435 		disable_gif(svm);
2436 
2437 		/*
2438 		 * After a CLGI no interrupts should come.  But if vGIF is
2439 		 * in use, we still rely on the VINTR intercept (rather than
2440 		 * STGI) to detect an open interrupt window.
2441 		*/
2442 		if (!vgif)
2443 			svm_clear_vintr(svm);
2444 	}
2445 }
2446 
2447 static int stgi_interception(struct kvm_vcpu *vcpu)
2448 {
2449 	int ret;
2450 
2451 	if (nested_svm_check_permissions(vcpu))
2452 		return 1;
2453 
2454 	ret = kvm_skip_emulated_instruction(vcpu);
2455 	svm_set_gif(to_svm(vcpu), true);
2456 	return ret;
2457 }
2458 
2459 static int clgi_interception(struct kvm_vcpu *vcpu)
2460 {
2461 	int ret;
2462 
2463 	if (nested_svm_check_permissions(vcpu))
2464 		return 1;
2465 
2466 	ret = kvm_skip_emulated_instruction(vcpu);
2467 	svm_set_gif(to_svm(vcpu), false);
2468 	return ret;
2469 }
2470 
2471 static int invlpga_interception(struct kvm_vcpu *vcpu)
2472 {
2473 	gva_t gva = kvm_rax_read(vcpu);
2474 	u32 asid = kvm_rcx_read(vcpu);
2475 
2476 	/* FIXME: Handle an address size prefix. */
2477 	if (!is_long_mode(vcpu))
2478 		gva = (u32)gva;
2479 
2480 	trace_kvm_invlpga(to_svm(vcpu)->vmcb->save.rip, asid, gva);
2481 
2482 	/* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
2483 	kvm_mmu_invlpg(vcpu, gva);
2484 
2485 	return kvm_skip_emulated_instruction(vcpu);
2486 }
2487 
2488 static int skinit_interception(struct kvm_vcpu *vcpu)
2489 {
2490 	trace_kvm_skinit(to_svm(vcpu)->vmcb->save.rip, kvm_rax_read(vcpu));
2491 
2492 	kvm_queue_exception(vcpu, UD_VECTOR);
2493 	return 1;
2494 }
2495 
2496 static int task_switch_interception(struct kvm_vcpu *vcpu)
2497 {
2498 	struct vcpu_svm *svm = to_svm(vcpu);
2499 	u16 tss_selector;
2500 	int reason;
2501 	int int_type = svm->vmcb->control.exit_int_info &
2502 		SVM_EXITINTINFO_TYPE_MASK;
2503 	int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
2504 	uint32_t type =
2505 		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
2506 	uint32_t idt_v =
2507 		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
2508 	bool has_error_code = false;
2509 	u32 error_code = 0;
2510 
2511 	tss_selector = (u16)svm->vmcb->control.exit_info_1;
2512 
2513 	if (svm->vmcb->control.exit_info_2 &
2514 	    (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
2515 		reason = TASK_SWITCH_IRET;
2516 	else if (svm->vmcb->control.exit_info_2 &
2517 		 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
2518 		reason = TASK_SWITCH_JMP;
2519 	else if (idt_v)
2520 		reason = TASK_SWITCH_GATE;
2521 	else
2522 		reason = TASK_SWITCH_CALL;
2523 
2524 	if (reason == TASK_SWITCH_GATE) {
2525 		switch (type) {
2526 		case SVM_EXITINTINFO_TYPE_NMI:
2527 			vcpu->arch.nmi_injected = false;
2528 			break;
2529 		case SVM_EXITINTINFO_TYPE_EXEPT:
2530 			if (svm->vmcb->control.exit_info_2 &
2531 			    (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
2532 				has_error_code = true;
2533 				error_code =
2534 					(u32)svm->vmcb->control.exit_info_2;
2535 			}
2536 			kvm_clear_exception_queue(vcpu);
2537 			break;
2538 		case SVM_EXITINTINFO_TYPE_INTR:
2539 		case SVM_EXITINTINFO_TYPE_SOFT:
2540 			kvm_clear_interrupt_queue(vcpu);
2541 			break;
2542 		default:
2543 			break;
2544 		}
2545 	}
2546 
2547 	if (reason != TASK_SWITCH_GATE ||
2548 	    int_type == SVM_EXITINTINFO_TYPE_SOFT ||
2549 	    (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
2550 	     (int_vec == OF_VECTOR || int_vec == BP_VECTOR))) {
2551 		if (!svm_skip_emulated_instruction(vcpu))
2552 			return 0;
2553 	}
2554 
2555 	if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
2556 		int_vec = -1;
2557 
2558 	return kvm_task_switch(vcpu, tss_selector, int_vec, reason,
2559 			       has_error_code, error_code);
2560 }
2561 
2562 static void svm_clr_iret_intercept(struct vcpu_svm *svm)
2563 {
2564 	if (!sev_es_guest(svm->vcpu.kvm))
2565 		svm_clr_intercept(svm, INTERCEPT_IRET);
2566 }
2567 
2568 static void svm_set_iret_intercept(struct vcpu_svm *svm)
2569 {
2570 	if (!sev_es_guest(svm->vcpu.kvm))
2571 		svm_set_intercept(svm, INTERCEPT_IRET);
2572 }
2573 
2574 static int iret_interception(struct kvm_vcpu *vcpu)
2575 {
2576 	struct vcpu_svm *svm = to_svm(vcpu);
2577 
2578 	WARN_ON_ONCE(sev_es_guest(vcpu->kvm));
2579 
2580 	++vcpu->stat.nmi_window_exits;
2581 	svm->awaiting_iret_completion = true;
2582 
2583 	svm_clr_iret_intercept(svm);
2584 	svm->nmi_iret_rip = kvm_rip_read(vcpu);
2585 
2586 	kvm_make_request(KVM_REQ_EVENT, vcpu);
2587 	return 1;
2588 }
2589 
2590 static int invlpg_interception(struct kvm_vcpu *vcpu)
2591 {
2592 	if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2593 		return kvm_emulate_instruction(vcpu, 0);
2594 
2595 	kvm_mmu_invlpg(vcpu, to_svm(vcpu)->vmcb->control.exit_info_1);
2596 	return kvm_skip_emulated_instruction(vcpu);
2597 }
2598 
2599 static int emulate_on_interception(struct kvm_vcpu *vcpu)
2600 {
2601 	return kvm_emulate_instruction(vcpu, 0);
2602 }
2603 
2604 static int rsm_interception(struct kvm_vcpu *vcpu)
2605 {
2606 	return kvm_emulate_instruction_from_buffer(vcpu, rsm_ins_bytes, 2);
2607 }
2608 
2609 static bool check_selective_cr0_intercepted(struct kvm_vcpu *vcpu,
2610 					    unsigned long val)
2611 {
2612 	struct vcpu_svm *svm = to_svm(vcpu);
2613 	unsigned long cr0 = vcpu->arch.cr0;
2614 	bool ret = false;
2615 
2616 	if (!is_guest_mode(vcpu) ||
2617 	    (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0))))
2618 		return false;
2619 
2620 	cr0 &= ~SVM_CR0_SELECTIVE_MASK;
2621 	val &= ~SVM_CR0_SELECTIVE_MASK;
2622 
2623 	if (cr0 ^ val) {
2624 		svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
2625 		ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
2626 	}
2627 
2628 	return ret;
2629 }
2630 
2631 #define CR_VALID (1ULL << 63)
2632 
2633 static int cr_interception(struct kvm_vcpu *vcpu)
2634 {
2635 	struct vcpu_svm *svm = to_svm(vcpu);
2636 	int reg, cr;
2637 	unsigned long val;
2638 	int err;
2639 
2640 	if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2641 		return emulate_on_interception(vcpu);
2642 
2643 	if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
2644 		return emulate_on_interception(vcpu);
2645 
2646 	reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2647 	if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
2648 		cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
2649 	else
2650 		cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
2651 
2652 	err = 0;
2653 	if (cr >= 16) { /* mov to cr */
2654 		cr -= 16;
2655 		val = kvm_register_read(vcpu, reg);
2656 		trace_kvm_cr_write(cr, val);
2657 		switch (cr) {
2658 		case 0:
2659 			if (!check_selective_cr0_intercepted(vcpu, val))
2660 				err = kvm_set_cr0(vcpu, val);
2661 			else
2662 				return 1;
2663 
2664 			break;
2665 		case 3:
2666 			err = kvm_set_cr3(vcpu, val);
2667 			break;
2668 		case 4:
2669 			err = kvm_set_cr4(vcpu, val);
2670 			break;
2671 		case 8:
2672 			err = kvm_set_cr8(vcpu, val);
2673 			break;
2674 		default:
2675 			WARN(1, "unhandled write to CR%d", cr);
2676 			kvm_queue_exception(vcpu, UD_VECTOR);
2677 			return 1;
2678 		}
2679 	} else { /* mov from cr */
2680 		switch (cr) {
2681 		case 0:
2682 			val = kvm_read_cr0(vcpu);
2683 			break;
2684 		case 2:
2685 			val = vcpu->arch.cr2;
2686 			break;
2687 		case 3:
2688 			val = kvm_read_cr3(vcpu);
2689 			break;
2690 		case 4:
2691 			val = kvm_read_cr4(vcpu);
2692 			break;
2693 		case 8:
2694 			val = kvm_get_cr8(vcpu);
2695 			break;
2696 		default:
2697 			WARN(1, "unhandled read from CR%d", cr);
2698 			kvm_queue_exception(vcpu, UD_VECTOR);
2699 			return 1;
2700 		}
2701 		kvm_register_write(vcpu, reg, val);
2702 		trace_kvm_cr_read(cr, val);
2703 	}
2704 	return kvm_complete_insn_gp(vcpu, err);
2705 }
2706 
2707 static int cr_trap(struct kvm_vcpu *vcpu)
2708 {
2709 	struct vcpu_svm *svm = to_svm(vcpu);
2710 	unsigned long old_value, new_value;
2711 	unsigned int cr;
2712 	int ret = 0;
2713 
2714 	new_value = (unsigned long)svm->vmcb->control.exit_info_1;
2715 
2716 	cr = svm->vmcb->control.exit_code - SVM_EXIT_CR0_WRITE_TRAP;
2717 	switch (cr) {
2718 	case 0:
2719 		old_value = kvm_read_cr0(vcpu);
2720 		svm_set_cr0(vcpu, new_value);
2721 
2722 		kvm_post_set_cr0(vcpu, old_value, new_value);
2723 		break;
2724 	case 4:
2725 		old_value = kvm_read_cr4(vcpu);
2726 		svm_set_cr4(vcpu, new_value);
2727 
2728 		kvm_post_set_cr4(vcpu, old_value, new_value);
2729 		break;
2730 	case 8:
2731 		ret = kvm_set_cr8(vcpu, new_value);
2732 		break;
2733 	default:
2734 		WARN(1, "unhandled CR%d write trap", cr);
2735 		kvm_queue_exception(vcpu, UD_VECTOR);
2736 		return 1;
2737 	}
2738 
2739 	return kvm_complete_insn_gp(vcpu, ret);
2740 }
2741 
2742 static int dr_interception(struct kvm_vcpu *vcpu)
2743 {
2744 	struct vcpu_svm *svm = to_svm(vcpu);
2745 	int reg, dr;
2746 	int err = 0;
2747 
2748 	/*
2749 	 * SEV-ES intercepts DR7 only to disable guest debugging and the guest issues a VMGEXIT
2750 	 * for DR7 write only. KVM cannot change DR7 (always swapped as type 'A') so return early.
2751 	 */
2752 	if (sev_es_guest(vcpu->kvm))
2753 		return 1;
2754 
2755 	if (vcpu->guest_debug == 0) {
2756 		/*
2757 		 * No more DR vmexits; force a reload of the debug registers
2758 		 * and reenter on this instruction.  The next vmexit will
2759 		 * retrieve the full state of the debug registers.
2760 		 */
2761 		clr_dr_intercepts(svm);
2762 		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
2763 		return 1;
2764 	}
2765 
2766 	if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
2767 		return emulate_on_interception(vcpu);
2768 
2769 	reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2770 	dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
2771 	if (dr >= 16) { /* mov to DRn  */
2772 		dr -= 16;
2773 		err = kvm_set_dr(vcpu, dr, kvm_register_read(vcpu, reg));
2774 	} else {
2775 		kvm_register_write(vcpu, reg, kvm_get_dr(vcpu, dr));
2776 	}
2777 
2778 	return kvm_complete_insn_gp(vcpu, err);
2779 }
2780 
2781 static int cr8_write_interception(struct kvm_vcpu *vcpu)
2782 {
2783 	int r;
2784 
2785 	u8 cr8_prev = kvm_get_cr8(vcpu);
2786 	/* instruction emulation calls kvm_set_cr8() */
2787 	r = cr_interception(vcpu);
2788 	if (lapic_in_kernel(vcpu))
2789 		return r;
2790 	if (cr8_prev <= kvm_get_cr8(vcpu))
2791 		return r;
2792 	vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
2793 	return 0;
2794 }
2795 
2796 static int efer_trap(struct kvm_vcpu *vcpu)
2797 {
2798 	struct msr_data msr_info;
2799 	int ret;
2800 
2801 	/*
2802 	 * Clear the EFER_SVME bit from EFER. The SVM code always sets this
2803 	 * bit in svm_set_efer(), but __kvm_valid_efer() checks it against
2804 	 * whether the guest has X86_FEATURE_SVM - this avoids a failure if
2805 	 * the guest doesn't have X86_FEATURE_SVM.
2806 	 */
2807 	msr_info.host_initiated = false;
2808 	msr_info.index = MSR_EFER;
2809 	msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & ~EFER_SVME;
2810 	ret = kvm_set_msr_common(vcpu, &msr_info);
2811 
2812 	return kvm_complete_insn_gp(vcpu, ret);
2813 }
2814 
2815 static int svm_get_msr_feature(struct kvm_msr_entry *msr)
2816 {
2817 	msr->data = 0;
2818 
2819 	switch (msr->index) {
2820 	case MSR_AMD64_DE_CFG:
2821 		if (cpu_feature_enabled(X86_FEATURE_LFENCE_RDTSC))
2822 			msr->data |= MSR_AMD64_DE_CFG_LFENCE_SERIALIZE;
2823 		break;
2824 	default:
2825 		return KVM_MSR_RET_INVALID;
2826 	}
2827 
2828 	return 0;
2829 }
2830 
2831 static bool
2832 sev_es_prevent_msr_access(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2833 {
2834 	return sev_es_guest(vcpu->kvm) &&
2835 	       vcpu->arch.guest_state_protected &&
2836 	       svm_msrpm_offset(msr_info->index) != MSR_INVALID &&
2837 	       !msr_write_intercepted(vcpu, msr_info->index);
2838 }
2839 
2840 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2841 {
2842 	struct vcpu_svm *svm = to_svm(vcpu);
2843 
2844 	if (sev_es_prevent_msr_access(vcpu, msr_info)) {
2845 		msr_info->data = 0;
2846 		return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;
2847 	}
2848 
2849 	switch (msr_info->index) {
2850 	case MSR_AMD64_TSC_RATIO:
2851 		if (!msr_info->host_initiated &&
2852 		    !guest_can_use(vcpu, X86_FEATURE_TSCRATEMSR))
2853 			return 1;
2854 		msr_info->data = svm->tsc_ratio_msr;
2855 		break;
2856 	case MSR_STAR:
2857 		msr_info->data = svm->vmcb01.ptr->save.star;
2858 		break;
2859 #ifdef CONFIG_X86_64
2860 	case MSR_LSTAR:
2861 		msr_info->data = svm->vmcb01.ptr->save.lstar;
2862 		break;
2863 	case MSR_CSTAR:
2864 		msr_info->data = svm->vmcb01.ptr->save.cstar;
2865 		break;
2866 	case MSR_KERNEL_GS_BASE:
2867 		msr_info->data = svm->vmcb01.ptr->save.kernel_gs_base;
2868 		break;
2869 	case MSR_SYSCALL_MASK:
2870 		msr_info->data = svm->vmcb01.ptr->save.sfmask;
2871 		break;
2872 #endif
2873 	case MSR_IA32_SYSENTER_CS:
2874 		msr_info->data = svm->vmcb01.ptr->save.sysenter_cs;
2875 		break;
2876 	case MSR_IA32_SYSENTER_EIP:
2877 		msr_info->data = (u32)svm->vmcb01.ptr->save.sysenter_eip;
2878 		if (guest_cpuid_is_intel(vcpu))
2879 			msr_info->data |= (u64)svm->sysenter_eip_hi << 32;
2880 		break;
2881 	case MSR_IA32_SYSENTER_ESP:
2882 		msr_info->data = svm->vmcb01.ptr->save.sysenter_esp;
2883 		if (guest_cpuid_is_intel(vcpu))
2884 			msr_info->data |= (u64)svm->sysenter_esp_hi << 32;
2885 		break;
2886 	case MSR_TSC_AUX:
2887 		msr_info->data = svm->tsc_aux;
2888 		break;
2889 	case MSR_IA32_DEBUGCTLMSR:
2890 		msr_info->data = svm_get_lbr_vmcb(svm)->save.dbgctl;
2891 		break;
2892 	case MSR_IA32_LASTBRANCHFROMIP:
2893 		msr_info->data = svm_get_lbr_vmcb(svm)->save.br_from;
2894 		break;
2895 	case MSR_IA32_LASTBRANCHTOIP:
2896 		msr_info->data = svm_get_lbr_vmcb(svm)->save.br_to;
2897 		break;
2898 	case MSR_IA32_LASTINTFROMIP:
2899 		msr_info->data = svm_get_lbr_vmcb(svm)->save.last_excp_from;
2900 		break;
2901 	case MSR_IA32_LASTINTTOIP:
2902 		msr_info->data = svm_get_lbr_vmcb(svm)->save.last_excp_to;
2903 		break;
2904 	case MSR_VM_HSAVE_PA:
2905 		msr_info->data = svm->nested.hsave_msr;
2906 		break;
2907 	case MSR_VM_CR:
2908 		msr_info->data = svm->nested.vm_cr_msr;
2909 		break;
2910 	case MSR_IA32_SPEC_CTRL:
2911 		if (!msr_info->host_initiated &&
2912 		    !guest_has_spec_ctrl_msr(vcpu))
2913 			return 1;
2914 
2915 		if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
2916 			msr_info->data = svm->vmcb->save.spec_ctrl;
2917 		else
2918 			msr_info->data = svm->spec_ctrl;
2919 		break;
2920 	case MSR_AMD64_VIRT_SPEC_CTRL:
2921 		if (!msr_info->host_initiated &&
2922 		    !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
2923 			return 1;
2924 
2925 		msr_info->data = svm->virt_spec_ctrl;
2926 		break;
2927 	case MSR_F15H_IC_CFG: {
2928 
2929 		int family, model;
2930 
2931 		family = guest_cpuid_family(vcpu);
2932 		model  = guest_cpuid_model(vcpu);
2933 
2934 		if (family < 0 || model < 0)
2935 			return kvm_get_msr_common(vcpu, msr_info);
2936 
2937 		msr_info->data = 0;
2938 
2939 		if (family == 0x15 &&
2940 		    (model >= 0x2 && model < 0x20))
2941 			msr_info->data = 0x1E;
2942 		}
2943 		break;
2944 	case MSR_AMD64_DE_CFG:
2945 		msr_info->data = svm->msr_decfg;
2946 		break;
2947 	default:
2948 		return kvm_get_msr_common(vcpu, msr_info);
2949 	}
2950 	return 0;
2951 }
2952 
2953 static int svm_complete_emulated_msr(struct kvm_vcpu *vcpu, int err)
2954 {
2955 	struct vcpu_svm *svm = to_svm(vcpu);
2956 	if (!err || !sev_es_guest(vcpu->kvm) || WARN_ON_ONCE(!svm->sev_es.ghcb))
2957 		return kvm_complete_insn_gp(vcpu, err);
2958 
2959 	ghcb_set_sw_exit_info_1(svm->sev_es.ghcb, 1);
2960 	ghcb_set_sw_exit_info_2(svm->sev_es.ghcb,
2961 				X86_TRAP_GP |
2962 				SVM_EVTINJ_TYPE_EXEPT |
2963 				SVM_EVTINJ_VALID);
2964 	return 1;
2965 }
2966 
2967 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
2968 {
2969 	struct vcpu_svm *svm = to_svm(vcpu);
2970 	int svm_dis, chg_mask;
2971 
2972 	if (data & ~SVM_VM_CR_VALID_MASK)
2973 		return 1;
2974 
2975 	chg_mask = SVM_VM_CR_VALID_MASK;
2976 
2977 	if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
2978 		chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
2979 
2980 	svm->nested.vm_cr_msr &= ~chg_mask;
2981 	svm->nested.vm_cr_msr |= (data & chg_mask);
2982 
2983 	svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
2984 
2985 	/* check for svm_disable while efer.svme is set */
2986 	if (svm_dis && (vcpu->arch.efer & EFER_SVME))
2987 		return 1;
2988 
2989 	return 0;
2990 }
2991 
2992 static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
2993 {
2994 	struct vcpu_svm *svm = to_svm(vcpu);
2995 	int ret = 0;
2996 
2997 	u32 ecx = msr->index;
2998 	u64 data = msr->data;
2999 
3000 	if (sev_es_prevent_msr_access(vcpu, msr))
3001 		return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;
3002 
3003 	switch (ecx) {
3004 	case MSR_AMD64_TSC_RATIO:
3005 
3006 		if (!guest_can_use(vcpu, X86_FEATURE_TSCRATEMSR)) {
3007 
3008 			if (!msr->host_initiated)
3009 				return 1;
3010 			/*
3011 			 * In case TSC scaling is not enabled, always
3012 			 * leave this MSR at the default value.
3013 			 *
3014 			 * Due to bug in qemu 6.2.0, it would try to set
3015 			 * this msr to 0 if tsc scaling is not enabled.
3016 			 * Ignore this value as well.
3017 			 */
3018 			if (data != 0 && data != svm->tsc_ratio_msr)
3019 				return 1;
3020 			break;
3021 		}
3022 
3023 		if (data & SVM_TSC_RATIO_RSVD)
3024 			return 1;
3025 
3026 		svm->tsc_ratio_msr = data;
3027 
3028 		if (guest_can_use(vcpu, X86_FEATURE_TSCRATEMSR) &&
3029 		    is_guest_mode(vcpu))
3030 			nested_svm_update_tsc_ratio_msr(vcpu);
3031 
3032 		break;
3033 	case MSR_IA32_CR_PAT:
3034 		ret = kvm_set_msr_common(vcpu, msr);
3035 		if (ret)
3036 			break;
3037 
3038 		svm->vmcb01.ptr->save.g_pat = data;
3039 		if (is_guest_mode(vcpu))
3040 			nested_vmcb02_compute_g_pat(svm);
3041 		vmcb_mark_dirty(svm->vmcb, VMCB_NPT);
3042 		break;
3043 	case MSR_IA32_SPEC_CTRL:
3044 		if (!msr->host_initiated &&
3045 		    !guest_has_spec_ctrl_msr(vcpu))
3046 			return 1;
3047 
3048 		if (kvm_spec_ctrl_test_value(data))
3049 			return 1;
3050 
3051 		if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
3052 			svm->vmcb->save.spec_ctrl = data;
3053 		else
3054 			svm->spec_ctrl = data;
3055 		if (!data)
3056 			break;
3057 
3058 		/*
3059 		 * For non-nested:
3060 		 * When it's written (to non-zero) for the first time, pass
3061 		 * it through.
3062 		 *
3063 		 * For nested:
3064 		 * The handling of the MSR bitmap for L2 guests is done in
3065 		 * nested_svm_vmrun_msrpm.
3066 		 * We update the L1 MSR bit as well since it will end up
3067 		 * touching the MSR anyway now.
3068 		 */
3069 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
3070 		break;
3071 	case MSR_AMD64_VIRT_SPEC_CTRL:
3072 		if (!msr->host_initiated &&
3073 		    !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
3074 			return 1;
3075 
3076 		if (data & ~SPEC_CTRL_SSBD)
3077 			return 1;
3078 
3079 		svm->virt_spec_ctrl = data;
3080 		break;
3081 	case MSR_STAR:
3082 		svm->vmcb01.ptr->save.star = data;
3083 		break;
3084 #ifdef CONFIG_X86_64
3085 	case MSR_LSTAR:
3086 		svm->vmcb01.ptr->save.lstar = data;
3087 		break;
3088 	case MSR_CSTAR:
3089 		svm->vmcb01.ptr->save.cstar = data;
3090 		break;
3091 	case MSR_KERNEL_GS_BASE:
3092 		svm->vmcb01.ptr->save.kernel_gs_base = data;
3093 		break;
3094 	case MSR_SYSCALL_MASK:
3095 		svm->vmcb01.ptr->save.sfmask = data;
3096 		break;
3097 #endif
3098 	case MSR_IA32_SYSENTER_CS:
3099 		svm->vmcb01.ptr->save.sysenter_cs = data;
3100 		break;
3101 	case MSR_IA32_SYSENTER_EIP:
3102 		svm->vmcb01.ptr->save.sysenter_eip = (u32)data;
3103 		/*
3104 		 * We only intercept the MSR_IA32_SYSENTER_{EIP|ESP} msrs
3105 		 * when we spoof an Intel vendor ID (for cross vendor migration).
3106 		 * In this case we use this intercept to track the high
3107 		 * 32 bit part of these msrs to support Intel's
3108 		 * implementation of SYSENTER/SYSEXIT.
3109 		 */
3110 		svm->sysenter_eip_hi = guest_cpuid_is_intel(vcpu) ? (data >> 32) : 0;
3111 		break;
3112 	case MSR_IA32_SYSENTER_ESP:
3113 		svm->vmcb01.ptr->save.sysenter_esp = (u32)data;
3114 		svm->sysenter_esp_hi = guest_cpuid_is_intel(vcpu) ? (data >> 32) : 0;
3115 		break;
3116 	case MSR_TSC_AUX:
3117 		/*
3118 		 * TSC_AUX is always virtualized for SEV-ES guests when the
3119 		 * feature is available. The user return MSR support is not
3120 		 * required in this case because TSC_AUX is restored on #VMEXIT
3121 		 * from the host save area (which has been initialized in
3122 		 * svm_hardware_enable()).
3123 		 */
3124 		if (boot_cpu_has(X86_FEATURE_V_TSC_AUX) && sev_es_guest(vcpu->kvm))
3125 			break;
3126 
3127 		/*
3128 		 * TSC_AUX is usually changed only during boot and never read
3129 		 * directly.  Intercept TSC_AUX instead of exposing it to the
3130 		 * guest via direct_access_msrs, and switch it via user return.
3131 		 */
3132 		preempt_disable();
3133 		ret = kvm_set_user_return_msr(tsc_aux_uret_slot, data, -1ull);
3134 		preempt_enable();
3135 		if (ret)
3136 			break;
3137 
3138 		svm->tsc_aux = data;
3139 		break;
3140 	case MSR_IA32_DEBUGCTLMSR:
3141 		if (!lbrv) {
3142 			kvm_pr_unimpl_wrmsr(vcpu, ecx, data);
3143 			break;
3144 		}
3145 		if (data & DEBUGCTL_RESERVED_BITS)
3146 			return 1;
3147 
3148 		svm_get_lbr_vmcb(svm)->save.dbgctl = data;
3149 		svm_update_lbrv(vcpu);
3150 		break;
3151 	case MSR_VM_HSAVE_PA:
3152 		/*
3153 		 * Old kernels did not validate the value written to
3154 		 * MSR_VM_HSAVE_PA.  Allow KVM_SET_MSR to set an invalid
3155 		 * value to allow live migrating buggy or malicious guests
3156 		 * originating from those kernels.
3157 		 */
3158 		if (!msr->host_initiated && !page_address_valid(vcpu, data))
3159 			return 1;
3160 
3161 		svm->nested.hsave_msr = data & PAGE_MASK;
3162 		break;
3163 	case MSR_VM_CR:
3164 		return svm_set_vm_cr(vcpu, data);
3165 	case MSR_VM_IGNNE:
3166 		kvm_pr_unimpl_wrmsr(vcpu, ecx, data);
3167 		break;
3168 	case MSR_AMD64_DE_CFG: {
3169 		struct kvm_msr_entry msr_entry;
3170 
3171 		msr_entry.index = msr->index;
3172 		if (svm_get_msr_feature(&msr_entry))
3173 			return 1;
3174 
3175 		/* Check the supported bits */
3176 		if (data & ~msr_entry.data)
3177 			return 1;
3178 
3179 		/* Don't allow the guest to change a bit, #GP */
3180 		if (!msr->host_initiated && (data ^ msr_entry.data))
3181 			return 1;
3182 
3183 		svm->msr_decfg = data;
3184 		break;
3185 	}
3186 	default:
3187 		return kvm_set_msr_common(vcpu, msr);
3188 	}
3189 	return ret;
3190 }
3191 
3192 static int msr_interception(struct kvm_vcpu *vcpu)
3193 {
3194 	if (to_svm(vcpu)->vmcb->control.exit_info_1)
3195 		return kvm_emulate_wrmsr(vcpu);
3196 	else
3197 		return kvm_emulate_rdmsr(vcpu);
3198 }
3199 
3200 static int interrupt_window_interception(struct kvm_vcpu *vcpu)
3201 {
3202 	kvm_make_request(KVM_REQ_EVENT, vcpu);
3203 	svm_clear_vintr(to_svm(vcpu));
3204 
3205 	/*
3206 	 * If not running nested, for AVIC, the only reason to end up here is ExtINTs.
3207 	 * In this case AVIC was temporarily disabled for
3208 	 * requesting the IRQ window and we have to re-enable it.
3209 	 *
3210 	 * If running nested, still remove the VM wide AVIC inhibit to
3211 	 * support case in which the interrupt window was requested when the
3212 	 * vCPU was not running nested.
3213 
3214 	 * All vCPUs which run still run nested, will remain to have their
3215 	 * AVIC still inhibited due to per-cpu AVIC inhibition.
3216 	 */
3217 	kvm_clear_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN);
3218 
3219 	++vcpu->stat.irq_window_exits;
3220 	return 1;
3221 }
3222 
3223 static int pause_interception(struct kvm_vcpu *vcpu)
3224 {
3225 	bool in_kernel;
3226 	/*
3227 	 * CPL is not made available for an SEV-ES guest, therefore
3228 	 * vcpu->arch.preempted_in_kernel can never be true.  Just
3229 	 * set in_kernel to false as well.
3230 	 */
3231 	in_kernel = !sev_es_guest(vcpu->kvm) && svm_get_cpl(vcpu) == 0;
3232 
3233 	grow_ple_window(vcpu);
3234 
3235 	kvm_vcpu_on_spin(vcpu, in_kernel);
3236 	return kvm_skip_emulated_instruction(vcpu);
3237 }
3238 
3239 static int invpcid_interception(struct kvm_vcpu *vcpu)
3240 {
3241 	struct vcpu_svm *svm = to_svm(vcpu);
3242 	unsigned long type;
3243 	gva_t gva;
3244 
3245 	if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) {
3246 		kvm_queue_exception(vcpu, UD_VECTOR);
3247 		return 1;
3248 	}
3249 
3250 	/*
3251 	 * For an INVPCID intercept:
3252 	 * EXITINFO1 provides the linear address of the memory operand.
3253 	 * EXITINFO2 provides the contents of the register operand.
3254 	 */
3255 	type = svm->vmcb->control.exit_info_2;
3256 	gva = svm->vmcb->control.exit_info_1;
3257 
3258 	return kvm_handle_invpcid(vcpu, type, gva);
3259 }
3260 
3261 static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = {
3262 	[SVM_EXIT_READ_CR0]			= cr_interception,
3263 	[SVM_EXIT_READ_CR3]			= cr_interception,
3264 	[SVM_EXIT_READ_CR4]			= cr_interception,
3265 	[SVM_EXIT_READ_CR8]			= cr_interception,
3266 	[SVM_EXIT_CR0_SEL_WRITE]		= cr_interception,
3267 	[SVM_EXIT_WRITE_CR0]			= cr_interception,
3268 	[SVM_EXIT_WRITE_CR3]			= cr_interception,
3269 	[SVM_EXIT_WRITE_CR4]			= cr_interception,
3270 	[SVM_EXIT_WRITE_CR8]			= cr8_write_interception,
3271 	[SVM_EXIT_READ_DR0]			= dr_interception,
3272 	[SVM_EXIT_READ_DR1]			= dr_interception,
3273 	[SVM_EXIT_READ_DR2]			= dr_interception,
3274 	[SVM_EXIT_READ_DR3]			= dr_interception,
3275 	[SVM_EXIT_READ_DR4]			= dr_interception,
3276 	[SVM_EXIT_READ_DR5]			= dr_interception,
3277 	[SVM_EXIT_READ_DR6]			= dr_interception,
3278 	[SVM_EXIT_READ_DR7]			= dr_interception,
3279 	[SVM_EXIT_WRITE_DR0]			= dr_interception,
3280 	[SVM_EXIT_WRITE_DR1]			= dr_interception,
3281 	[SVM_EXIT_WRITE_DR2]			= dr_interception,
3282 	[SVM_EXIT_WRITE_DR3]			= dr_interception,
3283 	[SVM_EXIT_WRITE_DR4]			= dr_interception,
3284 	[SVM_EXIT_WRITE_DR5]			= dr_interception,
3285 	[SVM_EXIT_WRITE_DR6]			= dr_interception,
3286 	[SVM_EXIT_WRITE_DR7]			= dr_interception,
3287 	[SVM_EXIT_EXCP_BASE + DB_VECTOR]	= db_interception,
3288 	[SVM_EXIT_EXCP_BASE + BP_VECTOR]	= bp_interception,
3289 	[SVM_EXIT_EXCP_BASE + UD_VECTOR]	= ud_interception,
3290 	[SVM_EXIT_EXCP_BASE + PF_VECTOR]	= pf_interception,
3291 	[SVM_EXIT_EXCP_BASE + MC_VECTOR]	= mc_interception,
3292 	[SVM_EXIT_EXCP_BASE + AC_VECTOR]	= ac_interception,
3293 	[SVM_EXIT_EXCP_BASE + GP_VECTOR]	= gp_interception,
3294 	[SVM_EXIT_INTR]				= intr_interception,
3295 	[SVM_EXIT_NMI]				= nmi_interception,
3296 	[SVM_EXIT_SMI]				= smi_interception,
3297 	[SVM_EXIT_VINTR]			= interrupt_window_interception,
3298 	[SVM_EXIT_RDPMC]			= kvm_emulate_rdpmc,
3299 	[SVM_EXIT_CPUID]			= kvm_emulate_cpuid,
3300 	[SVM_EXIT_IRET]                         = iret_interception,
3301 	[SVM_EXIT_INVD]                         = kvm_emulate_invd,
3302 	[SVM_EXIT_PAUSE]			= pause_interception,
3303 	[SVM_EXIT_HLT]				= kvm_emulate_halt,
3304 	[SVM_EXIT_INVLPG]			= invlpg_interception,
3305 	[SVM_EXIT_INVLPGA]			= invlpga_interception,
3306 	[SVM_EXIT_IOIO]				= io_interception,
3307 	[SVM_EXIT_MSR]				= msr_interception,
3308 	[SVM_EXIT_TASK_SWITCH]			= task_switch_interception,
3309 	[SVM_EXIT_SHUTDOWN]			= shutdown_interception,
3310 	[SVM_EXIT_VMRUN]			= vmrun_interception,
3311 	[SVM_EXIT_VMMCALL]			= kvm_emulate_hypercall,
3312 	[SVM_EXIT_VMLOAD]			= vmload_interception,
3313 	[SVM_EXIT_VMSAVE]			= vmsave_interception,
3314 	[SVM_EXIT_STGI]				= stgi_interception,
3315 	[SVM_EXIT_CLGI]				= clgi_interception,
3316 	[SVM_EXIT_SKINIT]			= skinit_interception,
3317 	[SVM_EXIT_RDTSCP]			= kvm_handle_invalid_op,
3318 	[SVM_EXIT_WBINVD]                       = kvm_emulate_wbinvd,
3319 	[SVM_EXIT_MONITOR]			= kvm_emulate_monitor,
3320 	[SVM_EXIT_MWAIT]			= kvm_emulate_mwait,
3321 	[SVM_EXIT_XSETBV]			= kvm_emulate_xsetbv,
3322 	[SVM_EXIT_RDPRU]			= kvm_handle_invalid_op,
3323 	[SVM_EXIT_EFER_WRITE_TRAP]		= efer_trap,
3324 	[SVM_EXIT_CR0_WRITE_TRAP]		= cr_trap,
3325 	[SVM_EXIT_CR4_WRITE_TRAP]		= cr_trap,
3326 	[SVM_EXIT_CR8_WRITE_TRAP]		= cr_trap,
3327 	[SVM_EXIT_INVPCID]                      = invpcid_interception,
3328 	[SVM_EXIT_NPF]				= npf_interception,
3329 	[SVM_EXIT_RSM]                          = rsm_interception,
3330 	[SVM_EXIT_AVIC_INCOMPLETE_IPI]		= avic_incomplete_ipi_interception,
3331 	[SVM_EXIT_AVIC_UNACCELERATED_ACCESS]	= avic_unaccelerated_access_interception,
3332 #ifdef CONFIG_KVM_AMD_SEV
3333 	[SVM_EXIT_VMGEXIT]			= sev_handle_vmgexit,
3334 #endif
3335 };
3336 
3337 static void dump_vmcb(struct kvm_vcpu *vcpu)
3338 {
3339 	struct vcpu_svm *svm = to_svm(vcpu);
3340 	struct vmcb_control_area *control = &svm->vmcb->control;
3341 	struct vmcb_save_area *save = &svm->vmcb->save;
3342 	struct vmcb_save_area *save01 = &svm->vmcb01.ptr->save;
3343 
3344 	if (!dump_invalid_vmcb) {
3345 		pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n");
3346 		return;
3347 	}
3348 
3349 	pr_err("VMCB %p, last attempted VMRUN on CPU %d\n",
3350 	       svm->current_vmcb->ptr, vcpu->arch.last_vmentry_cpu);
3351 	pr_err("VMCB Control Area:\n");
3352 	pr_err("%-20s%04x\n", "cr_read:", control->intercepts[INTERCEPT_CR] & 0xffff);
3353 	pr_err("%-20s%04x\n", "cr_write:", control->intercepts[INTERCEPT_CR] >> 16);
3354 	pr_err("%-20s%04x\n", "dr_read:", control->intercepts[INTERCEPT_DR] & 0xffff);
3355 	pr_err("%-20s%04x\n", "dr_write:", control->intercepts[INTERCEPT_DR] >> 16);
3356 	pr_err("%-20s%08x\n", "exceptions:", control->intercepts[INTERCEPT_EXCEPTION]);
3357 	pr_err("%-20s%08x %08x\n", "intercepts:",
3358               control->intercepts[INTERCEPT_WORD3],
3359 	       control->intercepts[INTERCEPT_WORD4]);
3360 	pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
3361 	pr_err("%-20s%d\n", "pause filter threshold:",
3362 	       control->pause_filter_thresh);
3363 	pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
3364 	pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
3365 	pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
3366 	pr_err("%-20s%d\n", "asid:", control->asid);
3367 	pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
3368 	pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
3369 	pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
3370 	pr_err("%-20s%08x\n", "int_state:", control->int_state);
3371 	pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
3372 	pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
3373 	pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
3374 	pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
3375 	pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
3376 	pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
3377 	pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
3378 	pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar);
3379 	pr_err("%-20s%016llx\n", "ghcb:", control->ghcb_gpa);
3380 	pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
3381 	pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
3382 	pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext);
3383 	pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
3384 	pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page);
3385 	pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id);
3386 	pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id);
3387 	pr_err("%-20s%016llx\n", "vmsa_pa:", control->vmsa_pa);
3388 	pr_err("VMCB State Save Area:\n");
3389 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3390 	       "es:",
3391 	       save->es.selector, save->es.attrib,
3392 	       save->es.limit, save->es.base);
3393 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3394 	       "cs:",
3395 	       save->cs.selector, save->cs.attrib,
3396 	       save->cs.limit, save->cs.base);
3397 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3398 	       "ss:",
3399 	       save->ss.selector, save->ss.attrib,
3400 	       save->ss.limit, save->ss.base);
3401 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3402 	       "ds:",
3403 	       save->ds.selector, save->ds.attrib,
3404 	       save->ds.limit, save->ds.base);
3405 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3406 	       "fs:",
3407 	       save01->fs.selector, save01->fs.attrib,
3408 	       save01->fs.limit, save01->fs.base);
3409 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3410 	       "gs:",
3411 	       save01->gs.selector, save01->gs.attrib,
3412 	       save01->gs.limit, save01->gs.base);
3413 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3414 	       "gdtr:",
3415 	       save->gdtr.selector, save->gdtr.attrib,
3416 	       save->gdtr.limit, save->gdtr.base);
3417 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3418 	       "ldtr:",
3419 	       save01->ldtr.selector, save01->ldtr.attrib,
3420 	       save01->ldtr.limit, save01->ldtr.base);
3421 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3422 	       "idtr:",
3423 	       save->idtr.selector, save->idtr.attrib,
3424 	       save->idtr.limit, save->idtr.base);
3425 	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3426 	       "tr:",
3427 	       save01->tr.selector, save01->tr.attrib,
3428 	       save01->tr.limit, save01->tr.base);
3429 	pr_err("vmpl: %d   cpl:  %d               efer:          %016llx\n",
3430 	       save->vmpl, save->cpl, save->efer);
3431 	pr_err("%-15s %016llx %-13s %016llx\n",
3432 	       "cr0:", save->cr0, "cr2:", save->cr2);
3433 	pr_err("%-15s %016llx %-13s %016llx\n",
3434 	       "cr3:", save->cr3, "cr4:", save->cr4);
3435 	pr_err("%-15s %016llx %-13s %016llx\n",
3436 	       "dr6:", save->dr6, "dr7:", save->dr7);
3437 	pr_err("%-15s %016llx %-13s %016llx\n",
3438 	       "rip:", save->rip, "rflags:", save->rflags);
3439 	pr_err("%-15s %016llx %-13s %016llx\n",
3440 	       "rsp:", save->rsp, "rax:", save->rax);
3441 	pr_err("%-15s %016llx %-13s %016llx\n",
3442 	       "star:", save01->star, "lstar:", save01->lstar);
3443 	pr_err("%-15s %016llx %-13s %016llx\n",
3444 	       "cstar:", save01->cstar, "sfmask:", save01->sfmask);
3445 	pr_err("%-15s %016llx %-13s %016llx\n",
3446 	       "kernel_gs_base:", save01->kernel_gs_base,
3447 	       "sysenter_cs:", save01->sysenter_cs);
3448 	pr_err("%-15s %016llx %-13s %016llx\n",
3449 	       "sysenter_esp:", save01->sysenter_esp,
3450 	       "sysenter_eip:", save01->sysenter_eip);
3451 	pr_err("%-15s %016llx %-13s %016llx\n",
3452 	       "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
3453 	pr_err("%-15s %016llx %-13s %016llx\n",
3454 	       "br_from:", save->br_from, "br_to:", save->br_to);
3455 	pr_err("%-15s %016llx %-13s %016llx\n",
3456 	       "excp_from:", save->last_excp_from,
3457 	       "excp_to:", save->last_excp_to);
3458 }
3459 
3460 static bool svm_check_exit_valid(u64 exit_code)
3461 {
3462 	return (exit_code < ARRAY_SIZE(svm_exit_handlers) &&
3463 		svm_exit_handlers[exit_code]);
3464 }
3465 
3466 static int svm_handle_invalid_exit(struct kvm_vcpu *vcpu, u64 exit_code)
3467 {
3468 	vcpu_unimpl(vcpu, "svm: unexpected exit reason 0x%llx\n", exit_code);
3469 	dump_vmcb(vcpu);
3470 	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
3471 	vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
3472 	vcpu->run->internal.ndata = 2;
3473 	vcpu->run->internal.data[0] = exit_code;
3474 	vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu;
3475 	return 0;
3476 }
3477 
3478 int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 exit_code)
3479 {
3480 	if (!svm_check_exit_valid(exit_code))
3481 		return svm_handle_invalid_exit(vcpu, exit_code);
3482 
3483 #ifdef CONFIG_MITIGATION_RETPOLINE
3484 	if (exit_code == SVM_EXIT_MSR)
3485 		return msr_interception(vcpu);
3486 	else if (exit_code == SVM_EXIT_VINTR)
3487 		return interrupt_window_interception(vcpu);
3488 	else if (exit_code == SVM_EXIT_INTR)
3489 		return intr_interception(vcpu);
3490 	else if (exit_code == SVM_EXIT_HLT)
3491 		return kvm_emulate_halt(vcpu);
3492 	else if (exit_code == SVM_EXIT_NPF)
3493 		return npf_interception(vcpu);
3494 #endif
3495 	return svm_exit_handlers[exit_code](vcpu);
3496 }
3497 
3498 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason,
3499 			      u64 *info1, u64 *info2,
3500 			      u32 *intr_info, u32 *error_code)
3501 {
3502 	struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
3503 
3504 	*reason = control->exit_code;
3505 	*info1 = control->exit_info_1;
3506 	*info2 = control->exit_info_2;
3507 	*intr_info = control->exit_int_info;
3508 	if ((*intr_info & SVM_EXITINTINFO_VALID) &&
3509 	    (*intr_info & SVM_EXITINTINFO_VALID_ERR))
3510 		*error_code = control->exit_int_info_err;
3511 	else
3512 		*error_code = 0;
3513 }
3514 
3515 static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
3516 {
3517 	struct vcpu_svm *svm = to_svm(vcpu);
3518 	struct kvm_run *kvm_run = vcpu->run;
3519 	u32 exit_code = svm->vmcb->control.exit_code;
3520 
3521 	/* SEV-ES guests must use the CR write traps to track CR registers. */
3522 	if (!sev_es_guest(vcpu->kvm)) {
3523 		if (!svm_is_intercept(svm, INTERCEPT_CR0_WRITE))
3524 			vcpu->arch.cr0 = svm->vmcb->save.cr0;
3525 		if (npt_enabled)
3526 			vcpu->arch.cr3 = svm->vmcb->save.cr3;
3527 	}
3528 
3529 	if (is_guest_mode(vcpu)) {
3530 		int vmexit;
3531 
3532 		trace_kvm_nested_vmexit(vcpu, KVM_ISA_SVM);
3533 
3534 		vmexit = nested_svm_exit_special(svm);
3535 
3536 		if (vmexit == NESTED_EXIT_CONTINUE)
3537 			vmexit = nested_svm_exit_handled(svm);
3538 
3539 		if (vmexit == NESTED_EXIT_DONE)
3540 			return 1;
3541 	}
3542 
3543 	if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
3544 		kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3545 		kvm_run->fail_entry.hardware_entry_failure_reason
3546 			= svm->vmcb->control.exit_code;
3547 		kvm_run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu;
3548 		dump_vmcb(vcpu);
3549 		return 0;
3550 	}
3551 
3552 	if (exit_fastpath != EXIT_FASTPATH_NONE)
3553 		return 1;
3554 
3555 	return svm_invoke_exit_handler(vcpu, exit_code);
3556 }
3557 
3558 static void pre_svm_run(struct kvm_vcpu *vcpu)
3559 {
3560 	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu);
3561 	struct vcpu_svm *svm = to_svm(vcpu);
3562 
3563 	/*
3564 	 * If the previous vmrun of the vmcb occurred on a different physical
3565 	 * cpu, then mark the vmcb dirty and assign a new asid.  Hardware's
3566 	 * vmcb clean bits are per logical CPU, as are KVM's asid assignments.
3567 	 */
3568 	if (unlikely(svm->current_vmcb->cpu != vcpu->cpu)) {
3569 		svm->current_vmcb->asid_generation = 0;
3570 		vmcb_mark_all_dirty(svm->vmcb);
3571 		svm->current_vmcb->cpu = vcpu->cpu;
3572         }
3573 
3574 	if (sev_guest(vcpu->kvm))
3575 		return pre_sev_run(svm, vcpu->cpu);
3576 
3577 	/* FIXME: handle wraparound of asid_generation */
3578 	if (svm->current_vmcb->asid_generation != sd->asid_generation)
3579 		new_asid(svm, sd);
3580 }
3581 
3582 static void svm_inject_nmi(struct kvm_vcpu *vcpu)
3583 {
3584 	struct vcpu_svm *svm = to_svm(vcpu);
3585 
3586 	svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
3587 
3588 	if (svm->nmi_l1_to_l2)
3589 		return;
3590 
3591 	/*
3592 	 * No need to manually track NMI masking when vNMI is enabled, hardware
3593 	 * automatically sets V_NMI_BLOCKING_MASK as appropriate, including the
3594 	 * case where software directly injects an NMI.
3595 	 */
3596 	if (!is_vnmi_enabled(svm)) {
3597 		svm->nmi_masked = true;
3598 		svm_set_iret_intercept(svm);
3599 	}
3600 	++vcpu->stat.nmi_injections;
3601 }
3602 
3603 static bool svm_is_vnmi_pending(struct kvm_vcpu *vcpu)
3604 {
3605 	struct vcpu_svm *svm = to_svm(vcpu);
3606 
3607 	if (!is_vnmi_enabled(svm))
3608 		return false;
3609 
3610 	return !!(svm->vmcb->control.int_ctl & V_NMI_PENDING_MASK);
3611 }
3612 
3613 static bool svm_set_vnmi_pending(struct kvm_vcpu *vcpu)
3614 {
3615 	struct vcpu_svm *svm = to_svm(vcpu);
3616 
3617 	if (!is_vnmi_enabled(svm))
3618 		return false;
3619 
3620 	if (svm->vmcb->control.int_ctl & V_NMI_PENDING_MASK)
3621 		return false;
3622 
3623 	svm->vmcb->control.int_ctl |= V_NMI_PENDING_MASK;
3624 	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
3625 
3626 	/*
3627 	 * Because the pending NMI is serviced by hardware, KVM can't know when
3628 	 * the NMI is "injected", but for all intents and purposes, passing the
3629 	 * NMI off to hardware counts as injection.
3630 	 */
3631 	++vcpu->stat.nmi_injections;
3632 
3633 	return true;
3634 }
3635 
3636 static void svm_inject_irq(struct kvm_vcpu *vcpu, bool reinjected)
3637 {
3638 	struct vcpu_svm *svm = to_svm(vcpu);
3639 	u32 type;
3640 
3641 	if (vcpu->arch.interrupt.soft) {
3642 		if (svm_update_soft_interrupt_rip(vcpu))
3643 			return;
3644 
3645 		type = SVM_EVTINJ_TYPE_SOFT;
3646 	} else {
3647 		type = SVM_EVTINJ_TYPE_INTR;
3648 	}
3649 
3650 	trace_kvm_inj_virq(vcpu->arch.interrupt.nr,
3651 			   vcpu->arch.interrupt.soft, reinjected);
3652 	++vcpu->stat.irq_injections;
3653 
3654 	svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
3655 				       SVM_EVTINJ_VALID | type;
3656 }
3657 
3658 void svm_complete_interrupt_delivery(struct kvm_vcpu *vcpu, int delivery_mode,
3659 				     int trig_mode, int vector)
3660 {
3661 	/*
3662 	 * apic->apicv_active must be read after vcpu->mode.
3663 	 * Pairs with smp_store_release in vcpu_enter_guest.
3664 	 */
3665 	bool in_guest_mode = (smp_load_acquire(&vcpu->mode) == IN_GUEST_MODE);
3666 
3667 	/* Note, this is called iff the local APIC is in-kernel. */
3668 	if (!READ_ONCE(vcpu->arch.apic->apicv_active)) {
3669 		/* Process the interrupt via kvm_check_and_inject_events(). */
3670 		kvm_make_request(KVM_REQ_EVENT, vcpu);
3671 		kvm_vcpu_kick(vcpu);
3672 		return;
3673 	}
3674 
3675 	trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, trig_mode, vector);
3676 	if (in_guest_mode) {
3677 		/*
3678 		 * Signal the doorbell to tell hardware to inject the IRQ.  If
3679 		 * the vCPU exits the guest before the doorbell chimes, hardware
3680 		 * will automatically process AVIC interrupts at the next VMRUN.
3681 		 */
3682 		avic_ring_doorbell(vcpu);
3683 	} else {
3684 		/*
3685 		 * Wake the vCPU if it was blocking.  KVM will then detect the
3686 		 * pending IRQ when checking if the vCPU has a wake event.
3687 		 */
3688 		kvm_vcpu_wake_up(vcpu);
3689 	}
3690 }
3691 
3692 static void svm_deliver_interrupt(struct kvm_lapic *apic,  int delivery_mode,
3693 				  int trig_mode, int vector)
3694 {
3695 	kvm_lapic_set_irr(vector, apic);
3696 
3697 	/*
3698 	 * Pairs with the smp_mb_*() after setting vcpu->guest_mode in
3699 	 * vcpu_enter_guest() to ensure the write to the vIRR is ordered before
3700 	 * the read of guest_mode.  This guarantees that either VMRUN will see
3701 	 * and process the new vIRR entry, or that svm_complete_interrupt_delivery
3702 	 * will signal the doorbell if the CPU has already entered the guest.
3703 	 */
3704 	smp_mb__after_atomic();
3705 	svm_complete_interrupt_delivery(apic->vcpu, delivery_mode, trig_mode, vector);
3706 }
3707 
3708 static void svm_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
3709 {
3710 	struct vcpu_svm *svm = to_svm(vcpu);
3711 
3712 	/*
3713 	 * SEV-ES guests must always keep the CR intercepts cleared. CR
3714 	 * tracking is done using the CR write traps.
3715 	 */
3716 	if (sev_es_guest(vcpu->kvm))
3717 		return;
3718 
3719 	if (nested_svm_virtualize_tpr(vcpu))
3720 		return;
3721 
3722 	svm_clr_intercept(svm, INTERCEPT_CR8_WRITE);
3723 
3724 	if (irr == -1)
3725 		return;
3726 
3727 	if (tpr >= irr)
3728 		svm_set_intercept(svm, INTERCEPT_CR8_WRITE);
3729 }
3730 
3731 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
3732 {
3733 	struct vcpu_svm *svm = to_svm(vcpu);
3734 
3735 	if (is_vnmi_enabled(svm))
3736 		return svm->vmcb->control.int_ctl & V_NMI_BLOCKING_MASK;
3737 	else
3738 		return svm->nmi_masked;
3739 }
3740 
3741 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
3742 {
3743 	struct vcpu_svm *svm = to_svm(vcpu);
3744 
3745 	if (is_vnmi_enabled(svm)) {
3746 		if (masked)
3747 			svm->vmcb->control.int_ctl |= V_NMI_BLOCKING_MASK;
3748 		else
3749 			svm->vmcb->control.int_ctl &= ~V_NMI_BLOCKING_MASK;
3750 
3751 	} else {
3752 		svm->nmi_masked = masked;
3753 		if (masked)
3754 			svm_set_iret_intercept(svm);
3755 		else
3756 			svm_clr_iret_intercept(svm);
3757 	}
3758 }
3759 
3760 bool svm_nmi_blocked(struct kvm_vcpu *vcpu)
3761 {
3762 	struct vcpu_svm *svm = to_svm(vcpu);
3763 	struct vmcb *vmcb = svm->vmcb;
3764 
3765 	if (!gif_set(svm))
3766 		return true;
3767 
3768 	if (is_guest_mode(vcpu) && nested_exit_on_nmi(svm))
3769 		return false;
3770 
3771 	if (svm_get_nmi_mask(vcpu))
3772 		return true;
3773 
3774 	return vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK;
3775 }
3776 
3777 static int svm_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
3778 {
3779 	struct vcpu_svm *svm = to_svm(vcpu);
3780 	if (svm->nested.nested_run_pending)
3781 		return -EBUSY;
3782 
3783 	if (svm_nmi_blocked(vcpu))
3784 		return 0;
3785 
3786 	/* An NMI must not be injected into L2 if it's supposed to VM-Exit.  */
3787 	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(svm))
3788 		return -EBUSY;
3789 	return 1;
3790 }
3791 
3792 bool svm_interrupt_blocked(struct kvm_vcpu *vcpu)
3793 {
3794 	struct vcpu_svm *svm = to_svm(vcpu);
3795 	struct vmcb *vmcb = svm->vmcb;
3796 
3797 	if (!gif_set(svm))
3798 		return true;
3799 
3800 	if (is_guest_mode(vcpu)) {
3801 		/* As long as interrupts are being delivered...  */
3802 		if ((svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK)
3803 		    ? !(svm->vmcb01.ptr->save.rflags & X86_EFLAGS_IF)
3804 		    : !(kvm_get_rflags(vcpu) & X86_EFLAGS_IF))
3805 			return true;
3806 
3807 		/* ... vmexits aren't blocked by the interrupt shadow  */
3808 		if (nested_exit_on_intr(svm))
3809 			return false;
3810 	} else {
3811 		if (!svm_get_if_flag(vcpu))
3812 			return true;
3813 	}
3814 
3815 	return (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK);
3816 }
3817 
3818 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection)
3819 {
3820 	struct vcpu_svm *svm = to_svm(vcpu);
3821 
3822 	if (svm->nested.nested_run_pending)
3823 		return -EBUSY;
3824 
3825 	if (svm_interrupt_blocked(vcpu))
3826 		return 0;
3827 
3828 	/*
3829 	 * An IRQ must not be injected into L2 if it's supposed to VM-Exit,
3830 	 * e.g. if the IRQ arrived asynchronously after checking nested events.
3831 	 */
3832 	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(svm))
3833 		return -EBUSY;
3834 
3835 	return 1;
3836 }
3837 
3838 static void svm_enable_irq_window(struct kvm_vcpu *vcpu)
3839 {
3840 	struct vcpu_svm *svm = to_svm(vcpu);
3841 
3842 	/*
3843 	 * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
3844 	 * 1, because that's a separate STGI/VMRUN intercept.  The next time we
3845 	 * get that intercept, this function will be called again though and
3846 	 * we'll get the vintr intercept. However, if the vGIF feature is
3847 	 * enabled, the STGI interception will not occur. Enable the irq
3848 	 * window under the assumption that the hardware will set the GIF.
3849 	 */
3850 	if (vgif || gif_set(svm)) {
3851 		/*
3852 		 * IRQ window is not needed when AVIC is enabled,
3853 		 * unless we have pending ExtINT since it cannot be injected
3854 		 * via AVIC. In such case, KVM needs to temporarily disable AVIC,
3855 		 * and fallback to injecting IRQ via V_IRQ.
3856 		 *
3857 		 * If running nested, AVIC is already locally inhibited
3858 		 * on this vCPU, therefore there is no need to request
3859 		 * the VM wide AVIC inhibition.
3860 		 */
3861 		if (!is_guest_mode(vcpu))
3862 			kvm_set_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN);
3863 
3864 		svm_set_vintr(svm);
3865 	}
3866 }
3867 
3868 static void svm_enable_nmi_window(struct kvm_vcpu *vcpu)
3869 {
3870 	struct vcpu_svm *svm = to_svm(vcpu);
3871 
3872 	/*
3873 	 * If NMIs are outright masked, i.e. the vCPU is already handling an
3874 	 * NMI, and KVM has not yet intercepted an IRET, then there is nothing
3875 	 * more to do at this time as KVM has already enabled IRET intercepts.
3876 	 * If KVM has already intercepted IRET, then single-step over the IRET,
3877 	 * as NMIs aren't architecturally unmasked until the IRET completes.
3878 	 *
3879 	 * If vNMI is enabled, KVM should never request an NMI window if NMIs
3880 	 * are masked, as KVM allows at most one to-be-injected NMI and one
3881 	 * pending NMI.  If two NMIs arrive simultaneously, KVM will inject one
3882 	 * NMI and set V_NMI_PENDING for the other, but if and only if NMIs are
3883 	 * unmasked.  KVM _will_ request an NMI window in some situations, e.g.
3884 	 * if the vCPU is in an STI shadow or if GIF=0, KVM can't immediately
3885 	 * inject the NMI.  In those situations, KVM needs to single-step over
3886 	 * the STI shadow or intercept STGI.
3887 	 */
3888 	if (svm_get_nmi_mask(vcpu)) {
3889 		WARN_ON_ONCE(is_vnmi_enabled(svm));
3890 
3891 		if (!svm->awaiting_iret_completion)
3892 			return; /* IRET will cause a vm exit */
3893 	}
3894 
3895 	/*
3896 	 * SEV-ES guests are responsible for signaling when a vCPU is ready to
3897 	 * receive a new NMI, as SEV-ES guests can't be single-stepped, i.e.
3898 	 * KVM can't intercept and single-step IRET to detect when NMIs are
3899 	 * unblocked (architecturally speaking).  See SVM_VMGEXIT_NMI_COMPLETE.
3900 	 *
3901 	 * Note, GIF is guaranteed to be '1' for SEV-ES guests as hardware
3902 	 * ignores SEV-ES guest writes to EFER.SVME *and* CLGI/STGI are not
3903 	 * supported NAEs in the GHCB protocol.
3904 	 */
3905 	if (sev_es_guest(vcpu->kvm))
3906 		return;
3907 
3908 	if (!gif_set(svm)) {
3909 		if (vgif)
3910 			svm_set_intercept(svm, INTERCEPT_STGI);
3911 		return; /* STGI will cause a vm exit */
3912 	}
3913 
3914 	/*
3915 	 * Something prevents NMI from been injected. Single step over possible
3916 	 * problem (IRET or exception injection or interrupt shadow)
3917 	 */
3918 	svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu);
3919 	svm->nmi_singlestep = true;
3920 	svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
3921 }
3922 
3923 static void svm_flush_tlb_asid(struct kvm_vcpu *vcpu)
3924 {
3925 	struct vcpu_svm *svm = to_svm(vcpu);
3926 
3927 	/*
3928 	 * Unlike VMX, SVM doesn't provide a way to flush only NPT TLB entries.
3929 	 * A TLB flush for the current ASID flushes both "host" and "guest" TLB
3930 	 * entries, and thus is a superset of Hyper-V's fine grained flushing.
3931 	 */
3932 	kvm_hv_vcpu_purge_flush_tlb(vcpu);
3933 
3934 	/*
3935 	 * Flush only the current ASID even if the TLB flush was invoked via
3936 	 * kvm_flush_remote_tlbs().  Although flushing remote TLBs requires all
3937 	 * ASIDs to be flushed, KVM uses a single ASID for L1 and L2, and
3938 	 * unconditionally does a TLB flush on both nested VM-Enter and nested
3939 	 * VM-Exit (via kvm_mmu_reset_context()).
3940 	 */
3941 	if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
3942 		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
3943 	else
3944 		svm->current_vmcb->asid_generation--;
3945 }
3946 
3947 static void svm_flush_tlb_current(struct kvm_vcpu *vcpu)
3948 {
3949 	hpa_t root_tdp = vcpu->arch.mmu->root.hpa;
3950 
3951 	/*
3952 	 * When running on Hyper-V with EnlightenedNptTlb enabled, explicitly
3953 	 * flush the NPT mappings via hypercall as flushing the ASID only
3954 	 * affects virtual to physical mappings, it does not invalidate guest
3955 	 * physical to host physical mappings.
3956 	 */
3957 	if (svm_hv_is_enlightened_tlb_enabled(vcpu) && VALID_PAGE(root_tdp))
3958 		hyperv_flush_guest_mapping(root_tdp);
3959 
3960 	svm_flush_tlb_asid(vcpu);
3961 }
3962 
3963 static void svm_flush_tlb_all(struct kvm_vcpu *vcpu)
3964 {
3965 	/*
3966 	 * When running on Hyper-V with EnlightenedNptTlb enabled, remote TLB
3967 	 * flushes should be routed to hv_flush_remote_tlbs() without requesting
3968 	 * a "regular" remote flush.  Reaching this point means either there's
3969 	 * a KVM bug or a prior hv_flush_remote_tlbs() call failed, both of
3970 	 * which might be fatal to the guest.  Yell, but try to recover.
3971 	 */
3972 	if (WARN_ON_ONCE(svm_hv_is_enlightened_tlb_enabled(vcpu)))
3973 		hv_flush_remote_tlbs(vcpu->kvm);
3974 
3975 	svm_flush_tlb_asid(vcpu);
3976 }
3977 
3978 static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva)
3979 {
3980 	struct vcpu_svm *svm = to_svm(vcpu);
3981 
3982 	invlpga(gva, svm->vmcb->control.asid);
3983 }
3984 
3985 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
3986 {
3987 	struct vcpu_svm *svm = to_svm(vcpu);
3988 
3989 	if (nested_svm_virtualize_tpr(vcpu))
3990 		return;
3991 
3992 	if (!svm_is_intercept(svm, INTERCEPT_CR8_WRITE)) {
3993 		int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
3994 		kvm_set_cr8(vcpu, cr8);
3995 	}
3996 }
3997 
3998 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
3999 {
4000 	struct vcpu_svm *svm = to_svm(vcpu);
4001 	u64 cr8;
4002 
4003 	if (nested_svm_virtualize_tpr(vcpu) ||
4004 	    kvm_vcpu_apicv_active(vcpu))
4005 		return;
4006 
4007 	cr8 = kvm_get_cr8(vcpu);
4008 	svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
4009 	svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
4010 }
4011 
4012 static void svm_complete_soft_interrupt(struct kvm_vcpu *vcpu, u8 vector,
4013 					int type)
4014 {
4015 	bool is_exception = (type == SVM_EXITINTINFO_TYPE_EXEPT);
4016 	bool is_soft = (type == SVM_EXITINTINFO_TYPE_SOFT);
4017 	struct vcpu_svm *svm = to_svm(vcpu);
4018 
4019 	/*
4020 	 * If NRIPS is enabled, KVM must snapshot the pre-VMRUN next_rip that's
4021 	 * associated with the original soft exception/interrupt.  next_rip is
4022 	 * cleared on all exits that can occur while vectoring an event, so KVM
4023 	 * needs to manually set next_rip for re-injection.  Unlike the !nrips
4024 	 * case below, this needs to be done if and only if KVM is re-injecting
4025 	 * the same event, i.e. if the event is a soft exception/interrupt,
4026 	 * otherwise next_rip is unused on VMRUN.
4027 	 */
4028 	if (nrips && (is_soft || (is_exception && kvm_exception_is_soft(vector))) &&
4029 	    kvm_is_linear_rip(vcpu, svm->soft_int_old_rip + svm->soft_int_csbase))
4030 		svm->vmcb->control.next_rip = svm->soft_int_next_rip;
4031 	/*
4032 	 * If NRIPS isn't enabled, KVM must manually advance RIP prior to
4033 	 * injecting the soft exception/interrupt.  That advancement needs to
4034 	 * be unwound if vectoring didn't complete.  Note, the new event may
4035 	 * not be the injected event, e.g. if KVM injected an INTn, the INTn
4036 	 * hit a #NP in the guest, and the #NP encountered a #PF, the #NP will
4037 	 * be the reported vectored event, but RIP still needs to be unwound.
4038 	 */
4039 	else if (!nrips && (is_soft || is_exception) &&
4040 		 kvm_is_linear_rip(vcpu, svm->soft_int_next_rip + svm->soft_int_csbase))
4041 		kvm_rip_write(vcpu, svm->soft_int_old_rip);
4042 }
4043 
4044 static void svm_complete_interrupts(struct kvm_vcpu *vcpu)
4045 {
4046 	struct vcpu_svm *svm = to_svm(vcpu);
4047 	u8 vector;
4048 	int type;
4049 	u32 exitintinfo = svm->vmcb->control.exit_int_info;
4050 	bool nmi_l1_to_l2 = svm->nmi_l1_to_l2;
4051 	bool soft_int_injected = svm->soft_int_injected;
4052 
4053 	svm->nmi_l1_to_l2 = false;
4054 	svm->soft_int_injected = false;
4055 
4056 	/*
4057 	 * If we've made progress since setting awaiting_iret_completion, we've
4058 	 * executed an IRET and can allow NMI injection.
4059 	 */
4060 	if (svm->awaiting_iret_completion &&
4061 	    kvm_rip_read(vcpu) != svm->nmi_iret_rip) {
4062 		svm->awaiting_iret_completion = false;
4063 		svm->nmi_masked = false;
4064 		kvm_make_request(KVM_REQ_EVENT, vcpu);
4065 	}
4066 
4067 	vcpu->arch.nmi_injected = false;
4068 	kvm_clear_exception_queue(vcpu);
4069 	kvm_clear_interrupt_queue(vcpu);
4070 
4071 	if (!(exitintinfo & SVM_EXITINTINFO_VALID))
4072 		return;
4073 
4074 	kvm_make_request(KVM_REQ_EVENT, vcpu);
4075 
4076 	vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
4077 	type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
4078 
4079 	if (soft_int_injected)
4080 		svm_complete_soft_interrupt(vcpu, vector, type);
4081 
4082 	switch (type) {
4083 	case SVM_EXITINTINFO_TYPE_NMI:
4084 		vcpu->arch.nmi_injected = true;
4085 		svm->nmi_l1_to_l2 = nmi_l1_to_l2;
4086 		break;
4087 	case SVM_EXITINTINFO_TYPE_EXEPT:
4088 		/*
4089 		 * Never re-inject a #VC exception.
4090 		 */
4091 		if (vector == X86_TRAP_VC)
4092 			break;
4093 
4094 		if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
4095 			u32 err = svm->vmcb->control.exit_int_info_err;
4096 			kvm_requeue_exception_e(vcpu, vector, err);
4097 
4098 		} else
4099 			kvm_requeue_exception(vcpu, vector);
4100 		break;
4101 	case SVM_EXITINTINFO_TYPE_INTR:
4102 		kvm_queue_interrupt(vcpu, vector, false);
4103 		break;
4104 	case SVM_EXITINTINFO_TYPE_SOFT:
4105 		kvm_queue_interrupt(vcpu, vector, true);
4106 		break;
4107 	default:
4108 		break;
4109 	}
4110 
4111 }
4112 
4113 static void svm_cancel_injection(struct kvm_vcpu *vcpu)
4114 {
4115 	struct vcpu_svm *svm = to_svm(vcpu);
4116 	struct vmcb_control_area *control = &svm->vmcb->control;
4117 
4118 	control->exit_int_info = control->event_inj;
4119 	control->exit_int_info_err = control->event_inj_err;
4120 	control->event_inj = 0;
4121 	svm_complete_interrupts(vcpu);
4122 }
4123 
4124 static int svm_vcpu_pre_run(struct kvm_vcpu *vcpu)
4125 {
4126 	if (to_kvm_sev_info(vcpu->kvm)->need_init)
4127 		return -EINVAL;
4128 
4129 	return 1;
4130 }
4131 
4132 static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
4133 {
4134 	if (is_guest_mode(vcpu))
4135 		return EXIT_FASTPATH_NONE;
4136 
4137 	if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_MSR &&
4138 	    to_svm(vcpu)->vmcb->control.exit_info_1)
4139 		return handle_fastpath_set_msr_irqoff(vcpu);
4140 
4141 	return EXIT_FASTPATH_NONE;
4142 }
4143 
4144 static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_intercepted)
4145 {
4146 	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu);
4147 	struct vcpu_svm *svm = to_svm(vcpu);
4148 
4149 	guest_state_enter_irqoff();
4150 
4151 	amd_clear_divider();
4152 
4153 	if (sev_es_guest(vcpu->kvm))
4154 		__svm_sev_es_vcpu_run(svm, spec_ctrl_intercepted,
4155 				      sev_es_host_save_area(sd));
4156 	else
4157 		__svm_vcpu_run(svm, spec_ctrl_intercepted);
4158 
4159 	guest_state_exit_irqoff();
4160 }
4161 
4162 static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu,
4163 					  bool force_immediate_exit)
4164 {
4165 	struct vcpu_svm *svm = to_svm(vcpu);
4166 	bool spec_ctrl_intercepted = msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL);
4167 
4168 	trace_kvm_entry(vcpu, force_immediate_exit);
4169 
4170 	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
4171 	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
4172 	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
4173 
4174 	/*
4175 	 * Disable singlestep if we're injecting an interrupt/exception.
4176 	 * We don't want our modified rflags to be pushed on the stack where
4177 	 * we might not be able to easily reset them if we disabled NMI
4178 	 * singlestep later.
4179 	 */
4180 	if (svm->nmi_singlestep && svm->vmcb->control.event_inj) {
4181 		/*
4182 		 * Event injection happens before external interrupts cause a
4183 		 * vmexit and interrupts are disabled here, so smp_send_reschedule
4184 		 * is enough to force an immediate vmexit.
4185 		 */
4186 		disable_nmi_singlestep(svm);
4187 		force_immediate_exit = true;
4188 	}
4189 
4190 	if (force_immediate_exit)
4191 		smp_send_reschedule(vcpu->cpu);
4192 
4193 	pre_svm_run(vcpu);
4194 
4195 	sync_lapic_to_cr8(vcpu);
4196 
4197 	if (unlikely(svm->asid != svm->vmcb->control.asid)) {
4198 		svm->vmcb->control.asid = svm->asid;
4199 		vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
4200 	}
4201 	svm->vmcb->save.cr2 = vcpu->arch.cr2;
4202 
4203 	svm_hv_update_vp_id(svm->vmcb, vcpu);
4204 
4205 	/*
4206 	 * Run with all-zero DR6 unless needed, so that we can get the exact cause
4207 	 * of a #DB.
4208 	 */
4209 	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT))
4210 		svm_set_dr6(svm, vcpu->arch.dr6);
4211 	else
4212 		svm_set_dr6(svm, DR6_ACTIVE_LOW);
4213 
4214 	clgi();
4215 	kvm_load_guest_xsave_state(vcpu);
4216 
4217 	kvm_wait_lapic_expire(vcpu);
4218 
4219 	/*
4220 	 * If this vCPU has touched SPEC_CTRL, restore the guest's value if
4221 	 * it's non-zero. Since vmentry is serialising on affected CPUs, there
4222 	 * is no need to worry about the conditional branch over the wrmsr
4223 	 * being speculatively taken.
4224 	 */
4225 	if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
4226 		x86_spec_ctrl_set_guest(svm->virt_spec_ctrl);
4227 
4228 	svm_vcpu_enter_exit(vcpu, spec_ctrl_intercepted);
4229 
4230 	if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
4231 		x86_spec_ctrl_restore_host(svm->virt_spec_ctrl);
4232 
4233 	if (!sev_es_guest(vcpu->kvm)) {
4234 		vcpu->arch.cr2 = svm->vmcb->save.cr2;
4235 		vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
4236 		vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
4237 		vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
4238 	}
4239 	vcpu->arch.regs_dirty = 0;
4240 
4241 	if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
4242 		kvm_before_interrupt(vcpu, KVM_HANDLING_NMI);
4243 
4244 	kvm_load_host_xsave_state(vcpu);
4245 	stgi();
4246 
4247 	/* Any pending NMI will happen here */
4248 
4249 	if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
4250 		kvm_after_interrupt(vcpu);
4251 
4252 	sync_cr8_to_lapic(vcpu);
4253 
4254 	svm->next_rip = 0;
4255 	if (is_guest_mode(vcpu)) {
4256 		nested_sync_control_from_vmcb02(svm);
4257 
4258 		/* Track VMRUNs that have made past consistency checking */
4259 		if (svm->nested.nested_run_pending &&
4260 		    svm->vmcb->control.exit_code != SVM_EXIT_ERR)
4261                         ++vcpu->stat.nested_run;
4262 
4263 		svm->nested.nested_run_pending = 0;
4264 	}
4265 
4266 	svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
4267 	vmcb_mark_all_clean(svm->vmcb);
4268 
4269 	/* if exit due to PF check for async PF */
4270 	if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
4271 		vcpu->arch.apf.host_apf_flags =
4272 			kvm_read_and_reset_apf_flags();
4273 
4274 	vcpu->arch.regs_avail &= ~SVM_REGS_LAZY_LOAD_SET;
4275 
4276 	/*
4277 	 * We need to handle MC intercepts here before the vcpu has a chance to
4278 	 * change the physical cpu
4279 	 */
4280 	if (unlikely(svm->vmcb->control.exit_code ==
4281 		     SVM_EXIT_EXCP_BASE + MC_VECTOR))
4282 		svm_handle_mce(vcpu);
4283 
4284 	trace_kvm_exit(vcpu, KVM_ISA_SVM);
4285 
4286 	svm_complete_interrupts(vcpu);
4287 
4288 	return svm_exit_handlers_fastpath(vcpu);
4289 }
4290 
4291 static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa,
4292 			     int root_level)
4293 {
4294 	struct vcpu_svm *svm = to_svm(vcpu);
4295 	unsigned long cr3;
4296 
4297 	if (npt_enabled) {
4298 		svm->vmcb->control.nested_cr3 = __sme_set(root_hpa);
4299 		vmcb_mark_dirty(svm->vmcb, VMCB_NPT);
4300 
4301 		hv_track_root_tdp(vcpu, root_hpa);
4302 
4303 		cr3 = vcpu->arch.cr3;
4304 	} else if (root_level >= PT64_ROOT_4LEVEL) {
4305 		cr3 = __sme_set(root_hpa) | kvm_get_active_pcid(vcpu);
4306 	} else {
4307 		/* PCID in the guest should be impossible with a 32-bit MMU. */
4308 		WARN_ON_ONCE(kvm_get_active_pcid(vcpu));
4309 		cr3 = root_hpa;
4310 	}
4311 
4312 	svm->vmcb->save.cr3 = cr3;
4313 	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
4314 }
4315 
4316 static void
4317 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
4318 {
4319 	/*
4320 	 * Patch in the VMMCALL instruction:
4321 	 */
4322 	hypercall[0] = 0x0f;
4323 	hypercall[1] = 0x01;
4324 	hypercall[2] = 0xd9;
4325 }
4326 
4327 /*
4328  * The kvm parameter can be NULL (module initialization, or invocation before
4329  * VM creation). Be sure to check the kvm parameter before using it.
4330  */
4331 static bool svm_has_emulated_msr(struct kvm *kvm, u32 index)
4332 {
4333 	switch (index) {
4334 	case MSR_IA32_MCG_EXT_CTL:
4335 	case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR:
4336 		return false;
4337 	case MSR_IA32_SMBASE:
4338 		if (!IS_ENABLED(CONFIG_KVM_SMM))
4339 			return false;
4340 		/* SEV-ES guests do not support SMM, so report false */
4341 		if (kvm && sev_es_guest(kvm))
4342 			return false;
4343 		break;
4344 	default:
4345 		break;
4346 	}
4347 
4348 	return true;
4349 }
4350 
4351 static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
4352 {
4353 	struct vcpu_svm *svm = to_svm(vcpu);
4354 
4355 	/*
4356 	 * SVM doesn't provide a way to disable just XSAVES in the guest, KVM
4357 	 * can only disable all variants of by disallowing CR4.OSXSAVE from
4358 	 * being set.  As a result, if the host has XSAVE and XSAVES, and the
4359 	 * guest has XSAVE enabled, the guest can execute XSAVES without
4360 	 * faulting.  Treat XSAVES as enabled in this case regardless of
4361 	 * whether it's advertised to the guest so that KVM context switches
4362 	 * XSS on VM-Enter/VM-Exit.  Failure to do so would effectively give
4363 	 * the guest read/write access to the host's XSS.
4364 	 */
4365 	if (boot_cpu_has(X86_FEATURE_XSAVE) &&
4366 	    boot_cpu_has(X86_FEATURE_XSAVES) &&
4367 	    guest_cpuid_has(vcpu, X86_FEATURE_XSAVE))
4368 		kvm_governed_feature_set(vcpu, X86_FEATURE_XSAVES);
4369 
4370 	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_NRIPS);
4371 	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_TSCRATEMSR);
4372 	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_LBRV);
4373 
4374 	/*
4375 	 * Intercept VMLOAD if the vCPU mode is Intel in order to emulate that
4376 	 * VMLOAD drops bits 63:32 of SYSENTER (ignoring the fact that exposing
4377 	 * SVM on Intel is bonkers and extremely unlikely to work).
4378 	 */
4379 	if (!guest_cpuid_is_intel(vcpu))
4380 		kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_V_VMSAVE_VMLOAD);
4381 
4382 	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_PAUSEFILTER);
4383 	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_PFTHRESHOLD);
4384 	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_VGIF);
4385 	kvm_governed_feature_check_and_set(vcpu, X86_FEATURE_VNMI);
4386 
4387 	svm_recalc_instruction_intercepts(vcpu, svm);
4388 
4389 	if (boot_cpu_has(X86_FEATURE_IBPB))
4390 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_PRED_CMD, 0,
4391 				     !!guest_has_pred_cmd_msr(vcpu));
4392 
4393 	if (boot_cpu_has(X86_FEATURE_FLUSH_L1D))
4394 		set_msr_interception(vcpu, svm->msrpm, MSR_IA32_FLUSH_CMD, 0,
4395 				     !!guest_cpuid_has(vcpu, X86_FEATURE_FLUSH_L1D));
4396 
4397 	if (sev_guest(vcpu->kvm))
4398 		sev_vcpu_after_set_cpuid(svm);
4399 
4400 	init_vmcb_after_set_cpuid(vcpu);
4401 }
4402 
4403 static bool svm_has_wbinvd_exit(void)
4404 {
4405 	return true;
4406 }
4407 
4408 #define PRE_EX(exit)  { .exit_code = (exit), \
4409 			.stage = X86_ICPT_PRE_EXCEPT, }
4410 #define POST_EX(exit) { .exit_code = (exit), \
4411 			.stage = X86_ICPT_POST_EXCEPT, }
4412 #define POST_MEM(exit) { .exit_code = (exit), \
4413 			.stage = X86_ICPT_POST_MEMACCESS, }
4414 
4415 static const struct __x86_intercept {
4416 	u32 exit_code;
4417 	enum x86_intercept_stage stage;
4418 } x86_intercept_map[] = {
4419 	[x86_intercept_cr_read]		= POST_EX(SVM_EXIT_READ_CR0),
4420 	[x86_intercept_cr_write]	= POST_EX(SVM_EXIT_WRITE_CR0),
4421 	[x86_intercept_clts]		= POST_EX(SVM_EXIT_WRITE_CR0),
4422 	[x86_intercept_lmsw]		= POST_EX(SVM_EXIT_WRITE_CR0),
4423 	[x86_intercept_smsw]		= POST_EX(SVM_EXIT_READ_CR0),
4424 	[x86_intercept_dr_read]		= POST_EX(SVM_EXIT_READ_DR0),
4425 	[x86_intercept_dr_write]	= POST_EX(SVM_EXIT_WRITE_DR0),
4426 	[x86_intercept_sldt]		= POST_EX(SVM_EXIT_LDTR_READ),
4427 	[x86_intercept_str]		= POST_EX(SVM_EXIT_TR_READ),
4428 	[x86_intercept_lldt]		= POST_EX(SVM_EXIT_LDTR_WRITE),
4429 	[x86_intercept_ltr]		= POST_EX(SVM_EXIT_TR_WRITE),
4430 	[x86_intercept_sgdt]		= POST_EX(SVM_EXIT_GDTR_READ),
4431 	[x86_intercept_sidt]		= POST_EX(SVM_EXIT_IDTR_READ),
4432 	[x86_intercept_lgdt]		= POST_EX(SVM_EXIT_GDTR_WRITE),
4433 	[x86_intercept_lidt]		= POST_EX(SVM_EXIT_IDTR_WRITE),
4434 	[x86_intercept_vmrun]		= POST_EX(SVM_EXIT_VMRUN),
4435 	[x86_intercept_vmmcall]		= POST_EX(SVM_EXIT_VMMCALL),
4436 	[x86_intercept_vmload]		= POST_EX(SVM_EXIT_VMLOAD),
4437 	[x86_intercept_vmsave]		= POST_EX(SVM_EXIT_VMSAVE),
4438 	[x86_intercept_stgi]		= POST_EX(SVM_EXIT_STGI),
4439 	[x86_intercept_clgi]		= POST_EX(SVM_EXIT_CLGI),
4440 	[x86_intercept_skinit]		= POST_EX(SVM_EXIT_SKINIT),
4441 	[x86_intercept_invlpga]		= POST_EX(SVM_EXIT_INVLPGA),
4442 	[x86_intercept_rdtscp]		= POST_EX(SVM_EXIT_RDTSCP),
4443 	[x86_intercept_monitor]		= POST_MEM(SVM_EXIT_MONITOR),
4444 	[x86_intercept_mwait]		= POST_EX(SVM_EXIT_MWAIT),
4445 	[x86_intercept_invlpg]		= POST_EX(SVM_EXIT_INVLPG),
4446 	[x86_intercept_invd]		= POST_EX(SVM_EXIT_INVD),
4447 	[x86_intercept_wbinvd]		= POST_EX(SVM_EXIT_WBINVD),
4448 	[x86_intercept_wrmsr]		= POST_EX(SVM_EXIT_MSR),
4449 	[x86_intercept_rdtsc]		= POST_EX(SVM_EXIT_RDTSC),
4450 	[x86_intercept_rdmsr]		= POST_EX(SVM_EXIT_MSR),
4451 	[x86_intercept_rdpmc]		= POST_EX(SVM_EXIT_RDPMC),
4452 	[x86_intercept_cpuid]		= PRE_EX(SVM_EXIT_CPUID),
4453 	[x86_intercept_rsm]		= PRE_EX(SVM_EXIT_RSM),
4454 	[x86_intercept_pause]		= PRE_EX(SVM_EXIT_PAUSE),
4455 	[x86_intercept_pushf]		= PRE_EX(SVM_EXIT_PUSHF),
4456 	[x86_intercept_popf]		= PRE_EX(SVM_EXIT_POPF),
4457 	[x86_intercept_intn]		= PRE_EX(SVM_EXIT_SWINT),
4458 	[x86_intercept_iret]		= PRE_EX(SVM_EXIT_IRET),
4459 	[x86_intercept_icebp]		= PRE_EX(SVM_EXIT_ICEBP),
4460 	[x86_intercept_hlt]		= POST_EX(SVM_EXIT_HLT),
4461 	[x86_intercept_in]		= POST_EX(SVM_EXIT_IOIO),
4462 	[x86_intercept_ins]		= POST_EX(SVM_EXIT_IOIO),
4463 	[x86_intercept_out]		= POST_EX(SVM_EXIT_IOIO),
4464 	[x86_intercept_outs]		= POST_EX(SVM_EXIT_IOIO),
4465 	[x86_intercept_xsetbv]		= PRE_EX(SVM_EXIT_XSETBV),
4466 };
4467 
4468 #undef PRE_EX
4469 #undef POST_EX
4470 #undef POST_MEM
4471 
4472 static int svm_check_intercept(struct kvm_vcpu *vcpu,
4473 			       struct x86_instruction_info *info,
4474 			       enum x86_intercept_stage stage,
4475 			       struct x86_exception *exception)
4476 {
4477 	struct vcpu_svm *svm = to_svm(vcpu);
4478 	int vmexit, ret = X86EMUL_CONTINUE;
4479 	struct __x86_intercept icpt_info;
4480 	struct vmcb *vmcb = svm->vmcb;
4481 
4482 	if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
4483 		goto out;
4484 
4485 	icpt_info = x86_intercept_map[info->intercept];
4486 
4487 	if (stage != icpt_info.stage)
4488 		goto out;
4489 
4490 	switch (icpt_info.exit_code) {
4491 	case SVM_EXIT_READ_CR0:
4492 		if (info->intercept == x86_intercept_cr_read)
4493 			icpt_info.exit_code += info->modrm_reg;
4494 		break;
4495 	case SVM_EXIT_WRITE_CR0: {
4496 		unsigned long cr0, val;
4497 
4498 		if (info->intercept == x86_intercept_cr_write)
4499 			icpt_info.exit_code += info->modrm_reg;
4500 
4501 		if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
4502 		    info->intercept == x86_intercept_clts)
4503 			break;
4504 
4505 		if (!(vmcb12_is_intercept(&svm->nested.ctl,
4506 					INTERCEPT_SELECTIVE_CR0)))
4507 			break;
4508 
4509 		cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
4510 		val = info->src_val  & ~SVM_CR0_SELECTIVE_MASK;
4511 
4512 		if (info->intercept == x86_intercept_lmsw) {
4513 			cr0 &= 0xfUL;
4514 			val &= 0xfUL;
4515 			/* lmsw can't clear PE - catch this here */
4516 			if (cr0 & X86_CR0_PE)
4517 				val |= X86_CR0_PE;
4518 		}
4519 
4520 		if (cr0 ^ val)
4521 			icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
4522 
4523 		break;
4524 	}
4525 	case SVM_EXIT_READ_DR0:
4526 	case SVM_EXIT_WRITE_DR0:
4527 		icpt_info.exit_code += info->modrm_reg;
4528 		break;
4529 	case SVM_EXIT_MSR:
4530 		if (info->intercept == x86_intercept_wrmsr)
4531 			vmcb->control.exit_info_1 = 1;
4532 		else
4533 			vmcb->control.exit_info_1 = 0;
4534 		break;
4535 	case SVM_EXIT_PAUSE:
4536 		/*
4537 		 * We get this for NOP only, but pause
4538 		 * is rep not, check this here
4539 		 */
4540 		if (info->rep_prefix != REPE_PREFIX)
4541 			goto out;
4542 		break;
4543 	case SVM_EXIT_IOIO: {
4544 		u64 exit_info;
4545 		u32 bytes;
4546 
4547 		if (info->intercept == x86_intercept_in ||
4548 		    info->intercept == x86_intercept_ins) {
4549 			exit_info = ((info->src_val & 0xffff) << 16) |
4550 				SVM_IOIO_TYPE_MASK;
4551 			bytes = info->dst_bytes;
4552 		} else {
4553 			exit_info = (info->dst_val & 0xffff) << 16;
4554 			bytes = info->src_bytes;
4555 		}
4556 
4557 		if (info->intercept == x86_intercept_outs ||
4558 		    info->intercept == x86_intercept_ins)
4559 			exit_info |= SVM_IOIO_STR_MASK;
4560 
4561 		if (info->rep_prefix)
4562 			exit_info |= SVM_IOIO_REP_MASK;
4563 
4564 		bytes = min(bytes, 4u);
4565 
4566 		exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;
4567 
4568 		exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);
4569 
4570 		vmcb->control.exit_info_1 = exit_info;
4571 		vmcb->control.exit_info_2 = info->next_rip;
4572 
4573 		break;
4574 	}
4575 	default:
4576 		break;
4577 	}
4578 
4579 	/* TODO: Advertise NRIPS to guest hypervisor unconditionally */
4580 	if (static_cpu_has(X86_FEATURE_NRIPS))
4581 		vmcb->control.next_rip  = info->next_rip;
4582 	vmcb->control.exit_code = icpt_info.exit_code;
4583 	vmexit = nested_svm_exit_handled(svm);
4584 
4585 	ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
4586 					   : X86EMUL_CONTINUE;
4587 
4588 out:
4589 	return ret;
4590 }
4591 
4592 static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu)
4593 {
4594 	if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_INTR)
4595 		vcpu->arch.at_instruction_boundary = true;
4596 }
4597 
4598 static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu)
4599 {
4600 	if (!kvm_pause_in_guest(vcpu->kvm))
4601 		shrink_ple_window(vcpu);
4602 }
4603 
4604 static void svm_setup_mce(struct kvm_vcpu *vcpu)
4605 {
4606 	/* [63:9] are reserved. */
4607 	vcpu->arch.mcg_cap &= 0x1ff;
4608 }
4609 
4610 #ifdef CONFIG_KVM_SMM
4611 bool svm_smi_blocked(struct kvm_vcpu *vcpu)
4612 {
4613 	struct vcpu_svm *svm = to_svm(vcpu);
4614 
4615 	/* Per APM Vol.2 15.22.2 "Response to SMI" */
4616 	if (!gif_set(svm))
4617 		return true;
4618 
4619 	return is_smm(vcpu);
4620 }
4621 
4622 static int svm_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
4623 {
4624 	struct vcpu_svm *svm = to_svm(vcpu);
4625 	if (svm->nested.nested_run_pending)
4626 		return -EBUSY;
4627 
4628 	if (svm_smi_blocked(vcpu))
4629 		return 0;
4630 
4631 	/* An SMI must not be injected into L2 if it's supposed to VM-Exit.  */
4632 	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_smi(svm))
4633 		return -EBUSY;
4634 
4635 	return 1;
4636 }
4637 
4638 static int svm_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram)
4639 {
4640 	struct vcpu_svm *svm = to_svm(vcpu);
4641 	struct kvm_host_map map_save;
4642 	int ret;
4643 
4644 	if (!is_guest_mode(vcpu))
4645 		return 0;
4646 
4647 	/*
4648 	 * 32-bit SMRAM format doesn't preserve EFER and SVM state.  Userspace is
4649 	 * responsible for ensuring nested SVM and SMIs are mutually exclusive.
4650 	 */
4651 
4652 	if (!guest_cpuid_has(vcpu, X86_FEATURE_LM))
4653 		return 1;
4654 
4655 	smram->smram64.svm_guest_flag = 1;
4656 	smram->smram64.svm_guest_vmcb_gpa = svm->nested.vmcb12_gpa;
4657 
4658 	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
4659 	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
4660 	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
4661 
4662 	ret = nested_svm_simple_vmexit(svm, SVM_EXIT_SW);
4663 	if (ret)
4664 		return ret;
4665 
4666 	/*
4667 	 * KVM uses VMCB01 to store L1 host state while L2 runs but
4668 	 * VMCB01 is going to be used during SMM and thus the state will
4669 	 * be lost. Temporary save non-VMLOAD/VMSAVE state to the host save
4670 	 * area pointed to by MSR_VM_HSAVE_PA. APM guarantees that the
4671 	 * format of the area is identical to guest save area offsetted
4672 	 * by 0x400 (matches the offset of 'struct vmcb_save_area'
4673 	 * within 'struct vmcb'). Note: HSAVE area may also be used by
4674 	 * L1 hypervisor to save additional host context (e.g. KVM does
4675 	 * that, see svm_prepare_switch_to_guest()) which must be
4676 	 * preserved.
4677 	 */
4678 	if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save))
4679 		return 1;
4680 
4681 	BUILD_BUG_ON(offsetof(struct vmcb, save) != 0x400);
4682 
4683 	svm_copy_vmrun_state(map_save.hva + 0x400,
4684 			     &svm->vmcb01.ptr->save);
4685 
4686 	kvm_vcpu_unmap(vcpu, &map_save, true);
4687 	return 0;
4688 }
4689 
4690 static int svm_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram)
4691 {
4692 	struct vcpu_svm *svm = to_svm(vcpu);
4693 	struct kvm_host_map map, map_save;
4694 	struct vmcb *vmcb12;
4695 	int ret;
4696 
4697 	const struct kvm_smram_state_64 *smram64 = &smram->smram64;
4698 
4699 	if (!guest_cpuid_has(vcpu, X86_FEATURE_LM))
4700 		return 0;
4701 
4702 	/* Non-zero if SMI arrived while vCPU was in guest mode. */
4703 	if (!smram64->svm_guest_flag)
4704 		return 0;
4705 
4706 	if (!guest_cpuid_has(vcpu, X86_FEATURE_SVM))
4707 		return 1;
4708 
4709 	if (!(smram64->efer & EFER_SVME))
4710 		return 1;
4711 
4712 	if (kvm_vcpu_map(vcpu, gpa_to_gfn(smram64->svm_guest_vmcb_gpa), &map))
4713 		return 1;
4714 
4715 	ret = 1;
4716 	if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save))
4717 		goto unmap_map;
4718 
4719 	if (svm_allocate_nested(svm))
4720 		goto unmap_save;
4721 
4722 	/*
4723 	 * Restore L1 host state from L1 HSAVE area as VMCB01 was
4724 	 * used during SMM (see svm_enter_smm())
4725 	 */
4726 
4727 	svm_copy_vmrun_state(&svm->vmcb01.ptr->save, map_save.hva + 0x400);
4728 
4729 	/*
4730 	 * Enter the nested guest now
4731 	 */
4732 
4733 	vmcb_mark_all_dirty(svm->vmcb01.ptr);
4734 
4735 	vmcb12 = map.hva;
4736 	nested_copy_vmcb_control_to_cache(svm, &vmcb12->control);
4737 	nested_copy_vmcb_save_to_cache(svm, &vmcb12->save);
4738 	ret = enter_svm_guest_mode(vcpu, smram64->svm_guest_vmcb_gpa, vmcb12, false);
4739 
4740 	if (ret)
4741 		goto unmap_save;
4742 
4743 	svm->nested.nested_run_pending = 1;
4744 
4745 unmap_save:
4746 	kvm_vcpu_unmap(vcpu, &map_save, true);
4747 unmap_map:
4748 	kvm_vcpu_unmap(vcpu, &map, true);
4749 	return ret;
4750 }
4751 
4752 static void svm_enable_smi_window(struct kvm_vcpu *vcpu)
4753 {
4754 	struct vcpu_svm *svm = to_svm(vcpu);
4755 
4756 	if (!gif_set(svm)) {
4757 		if (vgif)
4758 			svm_set_intercept(svm, INTERCEPT_STGI);
4759 		/* STGI will cause a vm exit */
4760 	} else {
4761 		/* We must be in SMM; RSM will cause a vmexit anyway.  */
4762 	}
4763 }
4764 #endif
4765 
4766 static int svm_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type,
4767 					 void *insn, int insn_len)
4768 {
4769 	bool smep, smap, is_user;
4770 	u64 error_code;
4771 
4772 	/* Emulation is always possible when KVM has access to all guest state. */
4773 	if (!sev_guest(vcpu->kvm))
4774 		return X86EMUL_CONTINUE;
4775 
4776 	/* #UD and #GP should never be intercepted for SEV guests. */
4777 	WARN_ON_ONCE(emul_type & (EMULTYPE_TRAP_UD |
4778 				  EMULTYPE_TRAP_UD_FORCED |
4779 				  EMULTYPE_VMWARE_GP));
4780 
4781 	/*
4782 	 * Emulation is impossible for SEV-ES guests as KVM doesn't have access
4783 	 * to guest register state.
4784 	 */
4785 	if (sev_es_guest(vcpu->kvm))
4786 		return X86EMUL_RETRY_INSTR;
4787 
4788 	/*
4789 	 * Emulation is possible if the instruction is already decoded, e.g.
4790 	 * when completing I/O after returning from userspace.
4791 	 */
4792 	if (emul_type & EMULTYPE_NO_DECODE)
4793 		return X86EMUL_CONTINUE;
4794 
4795 	/*
4796 	 * Emulation is possible for SEV guests if and only if a prefilled
4797 	 * buffer containing the bytes of the intercepted instruction is
4798 	 * available. SEV guest memory is encrypted with a guest specific key
4799 	 * and cannot be decrypted by KVM, i.e. KVM would read ciphertext and
4800 	 * decode garbage.
4801 	 *
4802 	 * If KVM is NOT trying to simply skip an instruction, inject #UD if
4803 	 * KVM reached this point without an instruction buffer.  In practice,
4804 	 * this path should never be hit by a well-behaved guest, e.g. KVM
4805 	 * doesn't intercept #UD or #GP for SEV guests, but this path is still
4806 	 * theoretically reachable, e.g. via unaccelerated fault-like AVIC
4807 	 * access, and needs to be handled by KVM to avoid putting the guest
4808 	 * into an infinite loop.   Injecting #UD is somewhat arbitrary, but
4809 	 * its the least awful option given lack of insight into the guest.
4810 	 *
4811 	 * If KVM is trying to skip an instruction, simply resume the guest.
4812 	 * If a #NPF occurs while the guest is vectoring an INT3/INTO, then KVM
4813 	 * will attempt to re-inject the INT3/INTO and skip the instruction.
4814 	 * In that scenario, retrying the INT3/INTO and hoping the guest will
4815 	 * make forward progress is the only option that has a chance of
4816 	 * success (and in practice it will work the vast majority of the time).
4817 	 */
4818 	if (unlikely(!insn)) {
4819 		if (emul_type & EMULTYPE_SKIP)
4820 			return X86EMUL_UNHANDLEABLE;
4821 
4822 		kvm_queue_exception(vcpu, UD_VECTOR);
4823 		return X86EMUL_PROPAGATE_FAULT;
4824 	}
4825 
4826 	/*
4827 	 * Emulate for SEV guests if the insn buffer is not empty.  The buffer
4828 	 * will be empty if the DecodeAssist microcode cannot fetch bytes for
4829 	 * the faulting instruction because the code fetch itself faulted, e.g.
4830 	 * the guest attempted to fetch from emulated MMIO or a guest page
4831 	 * table used to translate CS:RIP resides in emulated MMIO.
4832 	 */
4833 	if (likely(insn_len))
4834 		return X86EMUL_CONTINUE;
4835 
4836 	/*
4837 	 * Detect and workaround Errata 1096 Fam_17h_00_0Fh.
4838 	 *
4839 	 * Errata:
4840 	 * When CPU raises #NPF on guest data access and vCPU CR4.SMAP=1, it is
4841 	 * possible that CPU microcode implementing DecodeAssist will fail to
4842 	 * read guest memory at CS:RIP and vmcb.GuestIntrBytes will incorrectly
4843 	 * be '0'.  This happens because microcode reads CS:RIP using a _data_
4844 	 * loap uop with CPL=0 privileges.  If the load hits a SMAP #PF, ucode
4845 	 * gives up and does not fill the instruction bytes buffer.
4846 	 *
4847 	 * As above, KVM reaches this point iff the VM is an SEV guest, the CPU
4848 	 * supports DecodeAssist, a #NPF was raised, KVM's page fault handler
4849 	 * triggered emulation (e.g. for MMIO), and the CPU returned 0 in the
4850 	 * GuestIntrBytes field of the VMCB.
4851 	 *
4852 	 * This does _not_ mean that the erratum has been encountered, as the
4853 	 * DecodeAssist will also fail if the load for CS:RIP hits a legitimate
4854 	 * #PF, e.g. if the guest attempt to execute from emulated MMIO and
4855 	 * encountered a reserved/not-present #PF.
4856 	 *
4857 	 * To hit the erratum, the following conditions must be true:
4858 	 *    1. CR4.SMAP=1 (obviously).
4859 	 *    2. CR4.SMEP=0 || CPL=3.  If SMEP=1 and CPL<3, the erratum cannot
4860 	 *       have been hit as the guest would have encountered a SMEP
4861 	 *       violation #PF, not a #NPF.
4862 	 *    3. The #NPF is not due to a code fetch, in which case failure to
4863 	 *       retrieve the instruction bytes is legitimate (see abvoe).
4864 	 *
4865 	 * In addition, don't apply the erratum workaround if the #NPF occurred
4866 	 * while translating guest page tables (see below).
4867 	 */
4868 	error_code = to_svm(vcpu)->vmcb->control.exit_info_1;
4869 	if (error_code & (PFERR_GUEST_PAGE_MASK | PFERR_FETCH_MASK))
4870 		goto resume_guest;
4871 
4872 	smep = kvm_is_cr4_bit_set(vcpu, X86_CR4_SMEP);
4873 	smap = kvm_is_cr4_bit_set(vcpu, X86_CR4_SMAP);
4874 	is_user = svm_get_cpl(vcpu) == 3;
4875 	if (smap && (!smep || is_user)) {
4876 		pr_err_ratelimited("SEV Guest triggered AMD Erratum 1096\n");
4877 
4878 		/*
4879 		 * If the fault occurred in userspace, arbitrarily inject #GP
4880 		 * to avoid killing the guest and to hopefully avoid confusing
4881 		 * the guest kernel too much, e.g. injecting #PF would not be
4882 		 * coherent with respect to the guest's page tables.  Request
4883 		 * triple fault if the fault occurred in the kernel as there's
4884 		 * no fault that KVM can inject without confusing the guest.
4885 		 * In practice, the triple fault is moot as no sane SEV kernel
4886 		 * will execute from user memory while also running with SMAP=1.
4887 		 */
4888 		if (is_user)
4889 			kvm_inject_gp(vcpu, 0);
4890 		else
4891 			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
4892 		return X86EMUL_PROPAGATE_FAULT;
4893 	}
4894 
4895 resume_guest:
4896 	/*
4897 	 * If the erratum was not hit, simply resume the guest and let it fault
4898 	 * again.  While awful, e.g. the vCPU may get stuck in an infinite loop
4899 	 * if the fault is at CPL=0, it's the lesser of all evils.  Exiting to
4900 	 * userspace will kill the guest, and letting the emulator read garbage
4901 	 * will yield random behavior and potentially corrupt the guest.
4902 	 *
4903 	 * Simply resuming the guest is technically not a violation of the SEV
4904 	 * architecture.  AMD's APM states that all code fetches and page table
4905 	 * accesses for SEV guest are encrypted, regardless of the C-Bit.  The
4906 	 * APM also states that encrypted accesses to MMIO are "ignored", but
4907 	 * doesn't explicitly define "ignored", i.e. doing nothing and letting
4908 	 * the guest spin is technically "ignoring" the access.
4909 	 */
4910 	return X86EMUL_RETRY_INSTR;
4911 }
4912 
4913 static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
4914 {
4915 	struct vcpu_svm *svm = to_svm(vcpu);
4916 
4917 	return !gif_set(svm);
4918 }
4919 
4920 static void svm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
4921 {
4922 	if (!sev_es_guest(vcpu->kvm))
4923 		return kvm_vcpu_deliver_sipi_vector(vcpu, vector);
4924 
4925 	sev_vcpu_deliver_sipi_vector(vcpu, vector);
4926 }
4927 
4928 static void svm_vm_destroy(struct kvm *kvm)
4929 {
4930 	avic_vm_destroy(kvm);
4931 	sev_vm_destroy(kvm);
4932 }
4933 
4934 static int svm_vm_init(struct kvm *kvm)
4935 {
4936 	int type = kvm->arch.vm_type;
4937 
4938 	if (type != KVM_X86_DEFAULT_VM &&
4939 	    type != KVM_X86_SW_PROTECTED_VM) {
4940 		kvm->arch.has_protected_state = (type == KVM_X86_SEV_ES_VM);
4941 		to_kvm_sev_info(kvm)->need_init = true;
4942 	}
4943 
4944 	if (!pause_filter_count || !pause_filter_thresh)
4945 		kvm->arch.pause_in_guest = true;
4946 
4947 	if (enable_apicv) {
4948 		int ret = avic_vm_init(kvm);
4949 		if (ret)
4950 			return ret;
4951 	}
4952 
4953 	return 0;
4954 }
4955 
4956 static void *svm_alloc_apic_backing_page(struct kvm_vcpu *vcpu)
4957 {
4958 	struct page *page = snp_safe_alloc_page(vcpu);
4959 
4960 	if (!page)
4961 		return NULL;
4962 
4963 	return page_address(page);
4964 }
4965 
4966 static struct kvm_x86_ops svm_x86_ops __initdata = {
4967 	.name = KBUILD_MODNAME,
4968 
4969 	.check_processor_compatibility = svm_check_processor_compat,
4970 
4971 	.hardware_unsetup = svm_hardware_unsetup,
4972 	.hardware_enable = svm_hardware_enable,
4973 	.hardware_disable = svm_hardware_disable,
4974 	.has_emulated_msr = svm_has_emulated_msr,
4975 
4976 	.vcpu_create = svm_vcpu_create,
4977 	.vcpu_free = svm_vcpu_free,
4978 	.vcpu_reset = svm_vcpu_reset,
4979 
4980 	.vm_size = sizeof(struct kvm_svm),
4981 	.vm_init = svm_vm_init,
4982 	.vm_destroy = svm_vm_destroy,
4983 
4984 	.prepare_switch_to_guest = svm_prepare_switch_to_guest,
4985 	.vcpu_load = svm_vcpu_load,
4986 	.vcpu_put = svm_vcpu_put,
4987 	.vcpu_blocking = avic_vcpu_blocking,
4988 	.vcpu_unblocking = avic_vcpu_unblocking,
4989 
4990 	.update_exception_bitmap = svm_update_exception_bitmap,
4991 	.get_msr_feature = svm_get_msr_feature,
4992 	.get_msr = svm_get_msr,
4993 	.set_msr = svm_set_msr,
4994 	.get_segment_base = svm_get_segment_base,
4995 	.get_segment = svm_get_segment,
4996 	.set_segment = svm_set_segment,
4997 	.get_cpl = svm_get_cpl,
4998 	.get_cs_db_l_bits = svm_get_cs_db_l_bits,
4999 	.is_valid_cr0 = svm_is_valid_cr0,
5000 	.set_cr0 = svm_set_cr0,
5001 	.post_set_cr3 = sev_post_set_cr3,
5002 	.is_valid_cr4 = svm_is_valid_cr4,
5003 	.set_cr4 = svm_set_cr4,
5004 	.set_efer = svm_set_efer,
5005 	.get_idt = svm_get_idt,
5006 	.set_idt = svm_set_idt,
5007 	.get_gdt = svm_get_gdt,
5008 	.set_gdt = svm_set_gdt,
5009 	.set_dr7 = svm_set_dr7,
5010 	.sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
5011 	.cache_reg = svm_cache_reg,
5012 	.get_rflags = svm_get_rflags,
5013 	.set_rflags = svm_set_rflags,
5014 	.get_if_flag = svm_get_if_flag,
5015 
5016 	.flush_tlb_all = svm_flush_tlb_all,
5017 	.flush_tlb_current = svm_flush_tlb_current,
5018 	.flush_tlb_gva = svm_flush_tlb_gva,
5019 	.flush_tlb_guest = svm_flush_tlb_asid,
5020 
5021 	.vcpu_pre_run = svm_vcpu_pre_run,
5022 	.vcpu_run = svm_vcpu_run,
5023 	.handle_exit = svm_handle_exit,
5024 	.skip_emulated_instruction = svm_skip_emulated_instruction,
5025 	.update_emulated_instruction = NULL,
5026 	.set_interrupt_shadow = svm_set_interrupt_shadow,
5027 	.get_interrupt_shadow = svm_get_interrupt_shadow,
5028 	.patch_hypercall = svm_patch_hypercall,
5029 	.inject_irq = svm_inject_irq,
5030 	.inject_nmi = svm_inject_nmi,
5031 	.is_vnmi_pending = svm_is_vnmi_pending,
5032 	.set_vnmi_pending = svm_set_vnmi_pending,
5033 	.inject_exception = svm_inject_exception,
5034 	.cancel_injection = svm_cancel_injection,
5035 	.interrupt_allowed = svm_interrupt_allowed,
5036 	.nmi_allowed = svm_nmi_allowed,
5037 	.get_nmi_mask = svm_get_nmi_mask,
5038 	.set_nmi_mask = svm_set_nmi_mask,
5039 	.enable_nmi_window = svm_enable_nmi_window,
5040 	.enable_irq_window = svm_enable_irq_window,
5041 	.update_cr8_intercept = svm_update_cr8_intercept,
5042 	.set_virtual_apic_mode = avic_refresh_virtual_apic_mode,
5043 	.refresh_apicv_exec_ctrl = avic_refresh_apicv_exec_ctrl,
5044 	.apicv_post_state_restore = avic_apicv_post_state_restore,
5045 	.required_apicv_inhibits = AVIC_REQUIRED_APICV_INHIBITS,
5046 
5047 	.get_exit_info = svm_get_exit_info,
5048 
5049 	.vcpu_after_set_cpuid = svm_vcpu_after_set_cpuid,
5050 
5051 	.has_wbinvd_exit = svm_has_wbinvd_exit,
5052 
5053 	.get_l2_tsc_offset = svm_get_l2_tsc_offset,
5054 	.get_l2_tsc_multiplier = svm_get_l2_tsc_multiplier,
5055 	.write_tsc_offset = svm_write_tsc_offset,
5056 	.write_tsc_multiplier = svm_write_tsc_multiplier,
5057 
5058 	.load_mmu_pgd = svm_load_mmu_pgd,
5059 
5060 	.check_intercept = svm_check_intercept,
5061 	.handle_exit_irqoff = svm_handle_exit_irqoff,
5062 
5063 	.sched_in = svm_sched_in,
5064 
5065 	.nested_ops = &svm_nested_ops,
5066 
5067 	.deliver_interrupt = svm_deliver_interrupt,
5068 	.pi_update_irte = avic_pi_update_irte,
5069 	.setup_mce = svm_setup_mce,
5070 
5071 #ifdef CONFIG_KVM_SMM
5072 	.smi_allowed = svm_smi_allowed,
5073 	.enter_smm = svm_enter_smm,
5074 	.leave_smm = svm_leave_smm,
5075 	.enable_smi_window = svm_enable_smi_window,
5076 #endif
5077 
5078 #ifdef CONFIG_KVM_AMD_SEV
5079 	.dev_get_attr = sev_dev_get_attr,
5080 	.mem_enc_ioctl = sev_mem_enc_ioctl,
5081 	.mem_enc_register_region = sev_mem_enc_register_region,
5082 	.mem_enc_unregister_region = sev_mem_enc_unregister_region,
5083 	.guest_memory_reclaimed = sev_guest_memory_reclaimed,
5084 
5085 	.vm_copy_enc_context_from = sev_vm_copy_enc_context_from,
5086 	.vm_move_enc_context_from = sev_vm_move_enc_context_from,
5087 #endif
5088 	.check_emulate_instruction = svm_check_emulate_instruction,
5089 
5090 	.apic_init_signal_blocked = svm_apic_init_signal_blocked,
5091 
5092 	.msr_filter_changed = svm_msr_filter_changed,
5093 	.complete_emulated_msr = svm_complete_emulated_msr,
5094 
5095 	.vcpu_deliver_sipi_vector = svm_vcpu_deliver_sipi_vector,
5096 	.vcpu_get_apicv_inhibit_reasons = avic_vcpu_get_apicv_inhibit_reasons,
5097 	.alloc_apic_backing_page = svm_alloc_apic_backing_page,
5098 };
5099 
5100 /*
5101  * The default MMIO mask is a single bit (excluding the present bit),
5102  * which could conflict with the memory encryption bit. Check for
5103  * memory encryption support and override the default MMIO mask if
5104  * memory encryption is enabled.
5105  */
5106 static __init void svm_adjust_mmio_mask(void)
5107 {
5108 	unsigned int enc_bit, mask_bit;
5109 	u64 msr, mask;
5110 
5111 	/* If there is no memory encryption support, use existing mask */
5112 	if (cpuid_eax(0x80000000) < 0x8000001f)
5113 		return;
5114 
5115 	/* If memory encryption is not enabled, use existing mask */
5116 	rdmsrl(MSR_AMD64_SYSCFG, msr);
5117 	if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT))
5118 		return;
5119 
5120 	enc_bit = cpuid_ebx(0x8000001f) & 0x3f;
5121 	mask_bit = boot_cpu_data.x86_phys_bits;
5122 
5123 	/* Increment the mask bit if it is the same as the encryption bit */
5124 	if (enc_bit == mask_bit)
5125 		mask_bit++;
5126 
5127 	/*
5128 	 * If the mask bit location is below 52, then some bits above the
5129 	 * physical addressing limit will always be reserved, so use the
5130 	 * rsvd_bits() function to generate the mask. This mask, along with
5131 	 * the present bit, will be used to generate a page fault with
5132 	 * PFER.RSV = 1.
5133 	 *
5134 	 * If the mask bit location is 52 (or above), then clear the mask.
5135 	 */
5136 	mask = (mask_bit < 52) ? rsvd_bits(mask_bit, 51) | PT_PRESENT_MASK : 0;
5137 
5138 	kvm_mmu_set_mmio_spte_mask(mask, mask, PT_WRITABLE_MASK | PT_USER_MASK);
5139 }
5140 
5141 static __init void svm_set_cpu_caps(void)
5142 {
5143 	kvm_set_cpu_caps();
5144 
5145 	kvm_caps.supported_perf_cap = 0;
5146 	kvm_caps.supported_xss = 0;
5147 
5148 	/* CPUID 0x80000001 and 0x8000000A (SVM features) */
5149 	if (nested) {
5150 		kvm_cpu_cap_set(X86_FEATURE_SVM);
5151 		kvm_cpu_cap_set(X86_FEATURE_VMCBCLEAN);
5152 
5153 		/*
5154 		 * KVM currently flushes TLBs on *every* nested SVM transition,
5155 		 * and so for all intents and purposes KVM supports flushing by
5156 		 * ASID, i.e. KVM is guaranteed to honor every L1 ASID flush.
5157 		 */
5158 		kvm_cpu_cap_set(X86_FEATURE_FLUSHBYASID);
5159 
5160 		if (nrips)
5161 			kvm_cpu_cap_set(X86_FEATURE_NRIPS);
5162 
5163 		if (npt_enabled)
5164 			kvm_cpu_cap_set(X86_FEATURE_NPT);
5165 
5166 		if (tsc_scaling)
5167 			kvm_cpu_cap_set(X86_FEATURE_TSCRATEMSR);
5168 
5169 		if (vls)
5170 			kvm_cpu_cap_set(X86_FEATURE_V_VMSAVE_VMLOAD);
5171 		if (lbrv)
5172 			kvm_cpu_cap_set(X86_FEATURE_LBRV);
5173 
5174 		if (boot_cpu_has(X86_FEATURE_PAUSEFILTER))
5175 			kvm_cpu_cap_set(X86_FEATURE_PAUSEFILTER);
5176 
5177 		if (boot_cpu_has(X86_FEATURE_PFTHRESHOLD))
5178 			kvm_cpu_cap_set(X86_FEATURE_PFTHRESHOLD);
5179 
5180 		if (vgif)
5181 			kvm_cpu_cap_set(X86_FEATURE_VGIF);
5182 
5183 		if (vnmi)
5184 			kvm_cpu_cap_set(X86_FEATURE_VNMI);
5185 
5186 		/* Nested VM can receive #VMEXIT instead of triggering #GP */
5187 		kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK);
5188 	}
5189 
5190 	/* CPUID 0x80000008 */
5191 	if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) ||
5192 	    boot_cpu_has(X86_FEATURE_AMD_SSBD))
5193 		kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD);
5194 
5195 	if (enable_pmu) {
5196 		/*
5197 		 * Enumerate support for PERFCTR_CORE if and only if KVM has
5198 		 * access to enough counters to virtualize "core" support,
5199 		 * otherwise limit vPMU support to the legacy number of counters.
5200 		 */
5201 		if (kvm_pmu_cap.num_counters_gp < AMD64_NUM_COUNTERS_CORE)
5202 			kvm_pmu_cap.num_counters_gp = min(AMD64_NUM_COUNTERS,
5203 							  kvm_pmu_cap.num_counters_gp);
5204 		else
5205 			kvm_cpu_cap_check_and_set(X86_FEATURE_PERFCTR_CORE);
5206 
5207 		if (kvm_pmu_cap.version != 2 ||
5208 		    !kvm_cpu_cap_has(X86_FEATURE_PERFCTR_CORE))
5209 			kvm_cpu_cap_clear(X86_FEATURE_PERFMON_V2);
5210 	}
5211 
5212 	/* CPUID 0x8000001F (SME/SEV features) */
5213 	sev_set_cpu_caps();
5214 }
5215 
5216 static __init int svm_hardware_setup(void)
5217 {
5218 	int cpu;
5219 	struct page *iopm_pages;
5220 	void *iopm_va;
5221 	int r;
5222 	unsigned int order = get_order(IOPM_SIZE);
5223 
5224 	/*
5225 	 * NX is required for shadow paging and for NPT if the NX huge pages
5226 	 * mitigation is enabled.
5227 	 */
5228 	if (!boot_cpu_has(X86_FEATURE_NX)) {
5229 		pr_err_ratelimited("NX (Execute Disable) not supported\n");
5230 		return -EOPNOTSUPP;
5231 	}
5232 	kvm_enable_efer_bits(EFER_NX);
5233 
5234 	iopm_pages = alloc_pages(GFP_KERNEL, order);
5235 
5236 	if (!iopm_pages)
5237 		return -ENOMEM;
5238 
5239 	iopm_va = page_address(iopm_pages);
5240 	memset(iopm_va, 0xff, PAGE_SIZE * (1 << order));
5241 	iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
5242 
5243 	init_msrpm_offsets();
5244 
5245 	kvm_caps.supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS |
5246 				     XFEATURE_MASK_BNDCSR);
5247 
5248 	if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
5249 		kvm_enable_efer_bits(EFER_FFXSR);
5250 
5251 	if (tsc_scaling) {
5252 		if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
5253 			tsc_scaling = false;
5254 		} else {
5255 			pr_info("TSC scaling supported\n");
5256 			kvm_caps.has_tsc_control = true;
5257 		}
5258 	}
5259 	kvm_caps.max_tsc_scaling_ratio = SVM_TSC_RATIO_MAX;
5260 	kvm_caps.tsc_scaling_ratio_frac_bits = 32;
5261 
5262 	tsc_aux_uret_slot = kvm_add_user_return_msr(MSR_TSC_AUX);
5263 
5264 	if (boot_cpu_has(X86_FEATURE_AUTOIBRS))
5265 		kvm_enable_efer_bits(EFER_AUTOIBRS);
5266 
5267 	/* Check for pause filtering support */
5268 	if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
5269 		pause_filter_count = 0;
5270 		pause_filter_thresh = 0;
5271 	} else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) {
5272 		pause_filter_thresh = 0;
5273 	}
5274 
5275 	if (nested) {
5276 		pr_info("Nested Virtualization enabled\n");
5277 		kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
5278 	}
5279 
5280 	/*
5281 	 * KVM's MMU doesn't support using 2-level paging for itself, and thus
5282 	 * NPT isn't supported if the host is using 2-level paging since host
5283 	 * CR4 is unchanged on VMRUN.
5284 	 */
5285 	if (!IS_ENABLED(CONFIG_X86_64) && !IS_ENABLED(CONFIG_X86_PAE))
5286 		npt_enabled = false;
5287 
5288 	if (!boot_cpu_has(X86_FEATURE_NPT))
5289 		npt_enabled = false;
5290 
5291 	/* Force VM NPT level equal to the host's paging level */
5292 	kvm_configure_mmu(npt_enabled, get_npt_level(),
5293 			  get_npt_level(), PG_LEVEL_1G);
5294 	pr_info("Nested Paging %sabled\n", npt_enabled ? "en" : "dis");
5295 
5296 	/* Setup shadow_me_value and shadow_me_mask */
5297 	kvm_mmu_set_me_spte_mask(sme_me_mask, sme_me_mask);
5298 
5299 	svm_adjust_mmio_mask();
5300 
5301 	nrips = nrips && boot_cpu_has(X86_FEATURE_NRIPS);
5302 
5303 	if (lbrv) {
5304 		if (!boot_cpu_has(X86_FEATURE_LBRV))
5305 			lbrv = false;
5306 		else
5307 			pr_info("LBR virtualization supported\n");
5308 	}
5309 	/*
5310 	 * Note, SEV setup consumes npt_enabled and enable_mmio_caching (which
5311 	 * may be modified by svm_adjust_mmio_mask()), as well as nrips.
5312 	 */
5313 	sev_hardware_setup();
5314 
5315 	svm_hv_hardware_setup();
5316 
5317 	for_each_possible_cpu(cpu) {
5318 		r = svm_cpu_init(cpu);
5319 		if (r)
5320 			goto err;
5321 	}
5322 
5323 	enable_apicv = avic = avic && avic_hardware_setup();
5324 
5325 	if (!enable_apicv) {
5326 		svm_x86_ops.vcpu_blocking = NULL;
5327 		svm_x86_ops.vcpu_unblocking = NULL;
5328 		svm_x86_ops.vcpu_get_apicv_inhibit_reasons = NULL;
5329 	} else if (!x2avic_enabled) {
5330 		svm_x86_ops.allow_apicv_in_x2apic_without_x2apic_virtualization = true;
5331 	}
5332 
5333 	if (vls) {
5334 		if (!npt_enabled ||
5335 		    !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) ||
5336 		    !IS_ENABLED(CONFIG_X86_64)) {
5337 			vls = false;
5338 		} else {
5339 			pr_info("Virtual VMLOAD VMSAVE supported\n");
5340 		}
5341 	}
5342 
5343 	if (boot_cpu_has(X86_FEATURE_SVME_ADDR_CHK))
5344 		svm_gp_erratum_intercept = false;
5345 
5346 	if (vgif) {
5347 		if (!boot_cpu_has(X86_FEATURE_VGIF))
5348 			vgif = false;
5349 		else
5350 			pr_info("Virtual GIF supported\n");
5351 	}
5352 
5353 	vnmi = vgif && vnmi && boot_cpu_has(X86_FEATURE_VNMI);
5354 	if (vnmi)
5355 		pr_info("Virtual NMI enabled\n");
5356 
5357 	if (!vnmi) {
5358 		svm_x86_ops.is_vnmi_pending = NULL;
5359 		svm_x86_ops.set_vnmi_pending = NULL;
5360 	}
5361 
5362 	if (!enable_pmu)
5363 		pr_info("PMU virtualization is disabled\n");
5364 
5365 	svm_set_cpu_caps();
5366 
5367 	/*
5368 	 * It seems that on AMD processors PTE's accessed bit is
5369 	 * being set by the CPU hardware before the NPF vmexit.
5370 	 * This is not expected behaviour and our tests fail because
5371 	 * of it.
5372 	 * A workaround here is to disable support for
5373 	 * GUEST_MAXPHYADDR < HOST_MAXPHYADDR if NPT is enabled.
5374 	 * In this case userspace can know if there is support using
5375 	 * KVM_CAP_SMALLER_MAXPHYADDR extension and decide how to handle
5376 	 * it
5377 	 * If future AMD CPU models change the behaviour described above,
5378 	 * this variable can be changed accordingly
5379 	 */
5380 	allow_smaller_maxphyaddr = !npt_enabled;
5381 
5382 	return 0;
5383 
5384 err:
5385 	svm_hardware_unsetup();
5386 	return r;
5387 }
5388 
5389 
5390 static struct kvm_x86_init_ops svm_init_ops __initdata = {
5391 	.hardware_setup = svm_hardware_setup,
5392 
5393 	.runtime_ops = &svm_x86_ops,
5394 	.pmu_ops = &amd_pmu_ops,
5395 };
5396 
5397 static void __svm_exit(void)
5398 {
5399 	kvm_x86_vendor_exit();
5400 
5401 	cpu_emergency_unregister_virt_callback(svm_emergency_disable);
5402 }
5403 
5404 static int __init svm_init(void)
5405 {
5406 	int r;
5407 
5408 	__unused_size_checks();
5409 
5410 	if (!kvm_is_svm_supported())
5411 		return -EOPNOTSUPP;
5412 
5413 	r = kvm_x86_vendor_init(&svm_init_ops);
5414 	if (r)
5415 		return r;
5416 
5417 	cpu_emergency_register_virt_callback(svm_emergency_disable);
5418 
5419 	/*
5420 	 * Common KVM initialization _must_ come last, after this, /dev/kvm is
5421 	 * exposed to userspace!
5422 	 */
5423 	r = kvm_init(sizeof(struct vcpu_svm), __alignof__(struct vcpu_svm),
5424 		     THIS_MODULE);
5425 	if (r)
5426 		goto err_kvm_init;
5427 
5428 	return 0;
5429 
5430 err_kvm_init:
5431 	__svm_exit();
5432 	return r;
5433 }
5434 
5435 static void __exit svm_exit(void)
5436 {
5437 	kvm_exit();
5438 	__svm_exit();
5439 }
5440 
5441 module_init(svm_init)
5442 module_exit(svm_exit)
5443