xref: /freebsd/sys/amd64/vmm/intel/vmx.c (revision 39c21c2db2d5f600f8f7ed87d1f6e7dc2e3073c8)
1366f6083SPeter Grehan /*-
2366f6083SPeter Grehan  * Copyright (c) 2011 NetApp, Inc.
3366f6083SPeter Grehan  * All rights reserved.
4366f6083SPeter Grehan  *
5366f6083SPeter Grehan  * Redistribution and use in source and binary forms, with or without
6366f6083SPeter Grehan  * modification, are permitted provided that the following conditions
7366f6083SPeter Grehan  * are met:
8366f6083SPeter Grehan  * 1. Redistributions of source code must retain the above copyright
9366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer.
10366f6083SPeter Grehan  * 2. Redistributions in binary form must reproduce the above copyright
11366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer in the
12366f6083SPeter Grehan  *    documentation and/or other materials provided with the distribution.
13366f6083SPeter Grehan  *
14366f6083SPeter Grehan  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
15366f6083SPeter Grehan  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16366f6083SPeter Grehan  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17366f6083SPeter Grehan  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
18366f6083SPeter Grehan  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19366f6083SPeter Grehan  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20366f6083SPeter Grehan  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21366f6083SPeter Grehan  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22366f6083SPeter Grehan  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23366f6083SPeter Grehan  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24366f6083SPeter Grehan  * SUCH DAMAGE.
25366f6083SPeter Grehan  *
26366f6083SPeter Grehan  * $FreeBSD$
27366f6083SPeter Grehan  */
28366f6083SPeter Grehan 
29366f6083SPeter Grehan #include <sys/cdefs.h>
30366f6083SPeter Grehan __FBSDID("$FreeBSD$");
31366f6083SPeter Grehan 
32366f6083SPeter Grehan #include <sys/param.h>
33366f6083SPeter Grehan #include <sys/systm.h>
34366f6083SPeter Grehan #include <sys/smp.h>
35366f6083SPeter Grehan #include <sys/kernel.h>
36366f6083SPeter Grehan #include <sys/malloc.h>
37366f6083SPeter Grehan #include <sys/pcpu.h>
38366f6083SPeter Grehan #include <sys/proc.h>
39366f6083SPeter Grehan 
40366f6083SPeter Grehan #include <vm/vm.h>
41366f6083SPeter Grehan #include <vm/pmap.h>
42366f6083SPeter Grehan 
43366f6083SPeter Grehan #include <machine/psl.h>
44366f6083SPeter Grehan #include <machine/cpufunc.h>
458b287612SJohn Baldwin #include <machine/md_var.h>
46366f6083SPeter Grehan #include <machine/pmap.h>
47366f6083SPeter Grehan #include <machine/segments.h>
48608f97c3SPeter Grehan #include <machine/specialreg.h>
49366f6083SPeter Grehan #include <machine/vmparam.h>
50366f6083SPeter Grehan 
51366f6083SPeter Grehan #include <machine/vmm.h>
52366f6083SPeter Grehan #include "vmm_lapic.h"
53366f6083SPeter Grehan #include "vmm_msr.h"
54366f6083SPeter Grehan #include "vmm_ktr.h"
55366f6083SPeter Grehan #include "vmm_stat.h"
56366f6083SPeter Grehan 
57366f6083SPeter Grehan #include "vmx_msr.h"
58366f6083SPeter Grehan #include "ept.h"
59366f6083SPeter Grehan #include "vmx_cpufunc.h"
60366f6083SPeter Grehan #include "vmx.h"
61366f6083SPeter Grehan #include "x86.h"
62366f6083SPeter Grehan #include "vmx_controls.h"
63366f6083SPeter Grehan 
64366f6083SPeter Grehan #define	CR4_VMXE	(1UL << 13)
65366f6083SPeter Grehan 
66366f6083SPeter Grehan #define	PINBASED_CTLS_ONE_SETTING					\
67366f6083SPeter Grehan 	(PINBASED_EXTINT_EXITING	|				\
68366f6083SPeter Grehan 	 PINBASED_NMI_EXITING		|				\
69366f6083SPeter Grehan 	 PINBASED_VIRTUAL_NMI)
70366f6083SPeter Grehan #define	PINBASED_CTLS_ZERO_SETTING	0
71366f6083SPeter Grehan 
72366f6083SPeter Grehan #define PROCBASED_CTLS_WINDOW_SETTING					\
73366f6083SPeter Grehan 	(PROCBASED_INT_WINDOW_EXITING	|				\
74366f6083SPeter Grehan 	 PROCBASED_NMI_WINDOW_EXITING)
75366f6083SPeter Grehan 
76366f6083SPeter Grehan #define	PROCBASED_CTLS_ONE_SETTING 					\
77366f6083SPeter Grehan 	(PROCBASED_SECONDARY_CONTROLS	|				\
78366f6083SPeter Grehan 	 PROCBASED_IO_EXITING		|				\
79366f6083SPeter Grehan 	 PROCBASED_MSR_BITMAPS		|				\
80366f6083SPeter Grehan 	 PROCBASED_CTLS_WINDOW_SETTING)
81366f6083SPeter Grehan #define	PROCBASED_CTLS_ZERO_SETTING	\
82366f6083SPeter Grehan 	(PROCBASED_CR3_LOAD_EXITING |	\
83366f6083SPeter Grehan 	PROCBASED_CR3_STORE_EXITING |	\
84366f6083SPeter Grehan 	PROCBASED_IO_BITMAPS)
85366f6083SPeter Grehan 
86366f6083SPeter Grehan #define	PROCBASED_CTLS2_ONE_SETTING	PROCBASED2_ENABLE_EPT
87366f6083SPeter Grehan #define	PROCBASED_CTLS2_ZERO_SETTING	0
88366f6083SPeter Grehan 
89608f97c3SPeter Grehan #define VM_EXIT_CTLS_ONE_SETTING_NO_PAT					\
90366f6083SPeter Grehan 	(VM_EXIT_HOST_LMA			|			\
91366f6083SPeter Grehan 	VM_EXIT_SAVE_EFER			|			\
92366f6083SPeter Grehan 	VM_EXIT_LOAD_EFER)
93608f97c3SPeter Grehan 
94608f97c3SPeter Grehan #define	VM_EXIT_CTLS_ONE_SETTING					\
95608f97c3SPeter Grehan 	(VM_EXIT_CTLS_ONE_SETTING_NO_PAT       	|			\
96608f97c3SPeter Grehan 	VM_EXIT_SAVE_PAT			|			\
97608f97c3SPeter Grehan 	VM_EXIT_LOAD_PAT)
98366f6083SPeter Grehan #define	VM_EXIT_CTLS_ZERO_SETTING	VM_EXIT_SAVE_DEBUG_CONTROLS
99366f6083SPeter Grehan 
100608f97c3SPeter Grehan #define	VM_ENTRY_CTLS_ONE_SETTING_NO_PAT	VM_ENTRY_LOAD_EFER
101608f97c3SPeter Grehan 
102366f6083SPeter Grehan #define	VM_ENTRY_CTLS_ONE_SETTING					\
103608f97c3SPeter Grehan 	(VM_ENTRY_CTLS_ONE_SETTING_NO_PAT     	|			\
104608f97c3SPeter Grehan 	VM_ENTRY_LOAD_PAT)
105366f6083SPeter Grehan #define	VM_ENTRY_CTLS_ZERO_SETTING					\
106366f6083SPeter Grehan 	(VM_ENTRY_LOAD_DEBUG_CONTROLS		|			\
107366f6083SPeter Grehan 	VM_ENTRY_INTO_SMM			|			\
108366f6083SPeter Grehan 	VM_ENTRY_DEACTIVATE_DUAL_MONITOR)
109366f6083SPeter Grehan 
110366f6083SPeter Grehan #define	guest_msr_rw(vmx, msr) \
111366f6083SPeter Grehan 	msr_bitmap_change_access((vmx)->msr_bitmap, (msr), MSR_BITMAP_ACCESS_RW)
112366f6083SPeter Grehan 
113366f6083SPeter Grehan #define	HANDLED		1
114366f6083SPeter Grehan #define	UNHANDLED	0
115366f6083SPeter Grehan 
116366f6083SPeter Grehan MALLOC_DEFINE(M_VMX, "vmx", "vmx");
117366f6083SPeter Grehan 
118366f6083SPeter Grehan extern  struct pcpu __pcpu[];
119366f6083SPeter Grehan 
120b3996dd4SJohn Baldwin int vmxon_enabled[MAXCPU];
121366f6083SPeter Grehan static char vmxon_region[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE);
122366f6083SPeter Grehan 
123366f6083SPeter Grehan static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2;
124366f6083SPeter Grehan static uint32_t exit_ctls, entry_ctls;
125366f6083SPeter Grehan 
126366f6083SPeter Grehan static uint64_t cr0_ones_mask, cr0_zeros_mask;
127366f6083SPeter Grehan static uint64_t cr4_ones_mask, cr4_zeros_mask;
128366f6083SPeter Grehan 
129366f6083SPeter Grehan static volatile u_int nextvpid;
130366f6083SPeter Grehan 
131608f97c3SPeter Grehan static int vmx_no_patmsr;
132608f97c3SPeter Grehan 
133366f6083SPeter Grehan /*
134366f6083SPeter Grehan  * Virtual NMI blocking conditions.
135366f6083SPeter Grehan  *
136366f6083SPeter Grehan  * Some processor implementations also require NMI to be blocked if
137366f6083SPeter Grehan  * the STI_BLOCKING bit is set. It is possible to detect this at runtime
138366f6083SPeter Grehan  * based on the (exit_reason,exit_qual) tuple being set to
139366f6083SPeter Grehan  * (EXIT_REASON_INVAL_VMCS, EXIT_QUAL_NMI_WHILE_STI_BLOCKING).
140366f6083SPeter Grehan  *
141366f6083SPeter Grehan  * We take the easy way out and also include STI_BLOCKING as one of the
142366f6083SPeter Grehan  * gating items for vNMI injection.
143366f6083SPeter Grehan  */
144366f6083SPeter Grehan static uint64_t nmi_blocking_bits = VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING |
145366f6083SPeter Grehan 				    VMCS_INTERRUPTIBILITY_NMI_BLOCKING |
146366f6083SPeter Grehan 				    VMCS_INTERRUPTIBILITY_STI_BLOCKING;
147366f6083SPeter Grehan 
148366f6083SPeter Grehan /*
149366f6083SPeter Grehan  * Optional capabilities
150366f6083SPeter Grehan  */
151366f6083SPeter Grehan static int cap_halt_exit;
152366f6083SPeter Grehan static int cap_pause_exit;
153366f6083SPeter Grehan static int cap_unrestricted_guest;
154366f6083SPeter Grehan static int cap_monitor_trap;
155366f6083SPeter Grehan 
156366f6083SPeter Grehan /* statistics */
157366f6083SPeter Grehan static VMM_STAT_DEFINE(VCPU_MIGRATIONS, "vcpu migration across host cpus");
158366f6083SPeter Grehan static VMM_STAT_DEFINE(VMEXIT_EXTINT, "vm exits due to external interrupt");
159366f6083SPeter Grehan 
160366f6083SPeter Grehan #ifdef KTR
161366f6083SPeter Grehan static const char *
162366f6083SPeter Grehan exit_reason_to_str(int reason)
163366f6083SPeter Grehan {
164366f6083SPeter Grehan 	static char reasonbuf[32];
165366f6083SPeter Grehan 
166366f6083SPeter Grehan 	switch (reason) {
167366f6083SPeter Grehan 	case EXIT_REASON_EXCEPTION:
168366f6083SPeter Grehan 		return "exception";
169366f6083SPeter Grehan 	case EXIT_REASON_EXT_INTR:
170366f6083SPeter Grehan 		return "extint";
171366f6083SPeter Grehan 	case EXIT_REASON_TRIPLE_FAULT:
172366f6083SPeter Grehan 		return "triplefault";
173366f6083SPeter Grehan 	case EXIT_REASON_INIT:
174366f6083SPeter Grehan 		return "init";
175366f6083SPeter Grehan 	case EXIT_REASON_SIPI:
176366f6083SPeter Grehan 		return "sipi";
177366f6083SPeter Grehan 	case EXIT_REASON_IO_SMI:
178366f6083SPeter Grehan 		return "iosmi";
179366f6083SPeter Grehan 	case EXIT_REASON_SMI:
180366f6083SPeter Grehan 		return "smi";
181366f6083SPeter Grehan 	case EXIT_REASON_INTR_WINDOW:
182366f6083SPeter Grehan 		return "intrwindow";
183366f6083SPeter Grehan 	case EXIT_REASON_NMI_WINDOW:
184366f6083SPeter Grehan 		return "nmiwindow";
185366f6083SPeter Grehan 	case EXIT_REASON_TASK_SWITCH:
186366f6083SPeter Grehan 		return "taskswitch";
187366f6083SPeter Grehan 	case EXIT_REASON_CPUID:
188366f6083SPeter Grehan 		return "cpuid";
189366f6083SPeter Grehan 	case EXIT_REASON_GETSEC:
190366f6083SPeter Grehan 		return "getsec";
191366f6083SPeter Grehan 	case EXIT_REASON_HLT:
192366f6083SPeter Grehan 		return "hlt";
193366f6083SPeter Grehan 	case EXIT_REASON_INVD:
194366f6083SPeter Grehan 		return "invd";
195366f6083SPeter Grehan 	case EXIT_REASON_INVLPG:
196366f6083SPeter Grehan 		return "invlpg";
197366f6083SPeter Grehan 	case EXIT_REASON_RDPMC:
198366f6083SPeter Grehan 		return "rdpmc";
199366f6083SPeter Grehan 	case EXIT_REASON_RDTSC:
200366f6083SPeter Grehan 		return "rdtsc";
201366f6083SPeter Grehan 	case EXIT_REASON_RSM:
202366f6083SPeter Grehan 		return "rsm";
203366f6083SPeter Grehan 	case EXIT_REASON_VMCALL:
204366f6083SPeter Grehan 		return "vmcall";
205366f6083SPeter Grehan 	case EXIT_REASON_VMCLEAR:
206366f6083SPeter Grehan 		return "vmclear";
207366f6083SPeter Grehan 	case EXIT_REASON_VMLAUNCH:
208366f6083SPeter Grehan 		return "vmlaunch";
209366f6083SPeter Grehan 	case EXIT_REASON_VMPTRLD:
210366f6083SPeter Grehan 		return "vmptrld";
211366f6083SPeter Grehan 	case EXIT_REASON_VMPTRST:
212366f6083SPeter Grehan 		return "vmptrst";
213366f6083SPeter Grehan 	case EXIT_REASON_VMREAD:
214366f6083SPeter Grehan 		return "vmread";
215366f6083SPeter Grehan 	case EXIT_REASON_VMRESUME:
216366f6083SPeter Grehan 		return "vmresume";
217366f6083SPeter Grehan 	case EXIT_REASON_VMWRITE:
218366f6083SPeter Grehan 		return "vmwrite";
219366f6083SPeter Grehan 	case EXIT_REASON_VMXOFF:
220366f6083SPeter Grehan 		return "vmxoff";
221366f6083SPeter Grehan 	case EXIT_REASON_VMXON:
222366f6083SPeter Grehan 		return "vmxon";
223366f6083SPeter Grehan 	case EXIT_REASON_CR_ACCESS:
224366f6083SPeter Grehan 		return "craccess";
225366f6083SPeter Grehan 	case EXIT_REASON_DR_ACCESS:
226366f6083SPeter Grehan 		return "draccess";
227366f6083SPeter Grehan 	case EXIT_REASON_INOUT:
228366f6083SPeter Grehan 		return "inout";
229366f6083SPeter Grehan 	case EXIT_REASON_RDMSR:
230366f6083SPeter Grehan 		return "rdmsr";
231366f6083SPeter Grehan 	case EXIT_REASON_WRMSR:
232366f6083SPeter Grehan 		return "wrmsr";
233366f6083SPeter Grehan 	case EXIT_REASON_INVAL_VMCS:
234366f6083SPeter Grehan 		return "invalvmcs";
235366f6083SPeter Grehan 	case EXIT_REASON_INVAL_MSR:
236366f6083SPeter Grehan 		return "invalmsr";
237366f6083SPeter Grehan 	case EXIT_REASON_MWAIT:
238366f6083SPeter Grehan 		return "mwait";
239366f6083SPeter Grehan 	case EXIT_REASON_MTF:
240366f6083SPeter Grehan 		return "mtf";
241366f6083SPeter Grehan 	case EXIT_REASON_MONITOR:
242366f6083SPeter Grehan 		return "monitor";
243366f6083SPeter Grehan 	case EXIT_REASON_PAUSE:
244366f6083SPeter Grehan 		return "pause";
245366f6083SPeter Grehan 	case EXIT_REASON_MCE:
246366f6083SPeter Grehan 		return "mce";
247366f6083SPeter Grehan 	case EXIT_REASON_TPR:
248366f6083SPeter Grehan 		return "tpr";
249366f6083SPeter Grehan 	case EXIT_REASON_APIC:
250366f6083SPeter Grehan 		return "apic";
251366f6083SPeter Grehan 	case EXIT_REASON_GDTR_IDTR:
252366f6083SPeter Grehan 		return "gdtridtr";
253366f6083SPeter Grehan 	case EXIT_REASON_LDTR_TR:
254366f6083SPeter Grehan 		return "ldtrtr";
255366f6083SPeter Grehan 	case EXIT_REASON_EPT_FAULT:
256366f6083SPeter Grehan 		return "eptfault";
257366f6083SPeter Grehan 	case EXIT_REASON_EPT_MISCONFIG:
258366f6083SPeter Grehan 		return "eptmisconfig";
259366f6083SPeter Grehan 	case EXIT_REASON_INVEPT:
260366f6083SPeter Grehan 		return "invept";
261366f6083SPeter Grehan 	case EXIT_REASON_RDTSCP:
262366f6083SPeter Grehan 		return "rdtscp";
263366f6083SPeter Grehan 	case EXIT_REASON_VMX_PREEMPT:
264366f6083SPeter Grehan 		return "vmxpreempt";
265366f6083SPeter Grehan 	case EXIT_REASON_INVVPID:
266366f6083SPeter Grehan 		return "invvpid";
267366f6083SPeter Grehan 	case EXIT_REASON_WBINVD:
268366f6083SPeter Grehan 		return "wbinvd";
269366f6083SPeter Grehan 	case EXIT_REASON_XSETBV:
270366f6083SPeter Grehan 		return "xsetbv";
271366f6083SPeter Grehan 	default:
272366f6083SPeter Grehan 		snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason);
273366f6083SPeter Grehan 		return (reasonbuf);
274366f6083SPeter Grehan 	}
275366f6083SPeter Grehan }
276366f6083SPeter Grehan 
277366f6083SPeter Grehan #ifdef SETJMP_TRACE
278366f6083SPeter Grehan static const char *
279366f6083SPeter Grehan vmx_setjmp_rc2str(int rc)
280366f6083SPeter Grehan {
281366f6083SPeter Grehan 	switch (rc) {
282366f6083SPeter Grehan 	case VMX_RETURN_DIRECT:
283366f6083SPeter Grehan 		return "direct";
284366f6083SPeter Grehan 	case VMX_RETURN_LONGJMP:
285366f6083SPeter Grehan 		return "longjmp";
286366f6083SPeter Grehan 	case VMX_RETURN_VMRESUME:
287366f6083SPeter Grehan 		return "vmresume";
288366f6083SPeter Grehan 	case VMX_RETURN_VMLAUNCH:
289366f6083SPeter Grehan 		return "vmlaunch";
290366f6083SPeter Grehan 	default:
291366f6083SPeter Grehan 		return "unknown";
292366f6083SPeter Grehan 	}
293366f6083SPeter Grehan }
294366f6083SPeter Grehan 
295366f6083SPeter Grehan #define	SETJMP_TRACE(vmx, vcpu, vmxctx, regname)			  \
296366f6083SPeter Grehan 	VMM_CTR1((vmx)->vm, (vcpu), "setjmp trace " #regname " 0x%016lx", \
297366f6083SPeter Grehan 		 (vmxctx)->regname)
298366f6083SPeter Grehan 
299366f6083SPeter Grehan static void
300366f6083SPeter Grehan vmx_setjmp_trace(struct vmx *vmx, int vcpu, struct vmxctx *vmxctx, int rc)
301366f6083SPeter Grehan {
302366f6083SPeter Grehan 	uint64_t host_rip, host_rsp;
303366f6083SPeter Grehan 
304366f6083SPeter Grehan 	if (vmxctx != &vmx->ctx[vcpu])
305366f6083SPeter Grehan 		panic("vmx_setjmp_trace: invalid vmxctx %p; should be %p",
306366f6083SPeter Grehan 			vmxctx, &vmx->ctx[vcpu]);
307366f6083SPeter Grehan 
308366f6083SPeter Grehan 	VMM_CTR1((vmx)->vm, (vcpu), "vmxctx = %p", vmxctx);
309366f6083SPeter Grehan 	VMM_CTR2((vmx)->vm, (vcpu), "setjmp return code %s(%d)",
310366f6083SPeter Grehan 		 vmx_setjmp_rc2str(rc), rc);
311366f6083SPeter Grehan 
312366f6083SPeter Grehan 	host_rsp = host_rip = ~0;
313366f6083SPeter Grehan 	vmread(VMCS_HOST_RIP, &host_rip);
314366f6083SPeter Grehan 	vmread(VMCS_HOST_RSP, &host_rsp);
315366f6083SPeter Grehan 	VMM_CTR2((vmx)->vm, (vcpu), "vmcs host_rip 0x%016lx, host_rsp 0x%016lx",
316366f6083SPeter Grehan 		 host_rip, host_rsp);
317366f6083SPeter Grehan 
318366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, host_r15);
319366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, host_r14);
320366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, host_r13);
321366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, host_r12);
322366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, host_rbp);
323366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, host_rsp);
324366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, host_rbx);
325366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, host_rip);
326366f6083SPeter Grehan 
327366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rdi);
328366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rsi);
329366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rdx);
330366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rcx);
331366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r8);
332366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r9);
333366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rax);
334366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rbx);
335366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rbp);
336366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r10);
337366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r11);
338366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r12);
339366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r13);
340366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r14);
341366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r15);
342366f6083SPeter Grehan 	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_cr2);
343366f6083SPeter Grehan }
344366f6083SPeter Grehan #endif
345366f6083SPeter Grehan #else
346366f6083SPeter Grehan static void __inline
347366f6083SPeter Grehan vmx_setjmp_trace(struct vmx *vmx, int vcpu, struct vmxctx *vmxctx, int rc)
348366f6083SPeter Grehan {
349366f6083SPeter Grehan 	return;
350366f6083SPeter Grehan }
351366f6083SPeter Grehan #endif	/* KTR */
352366f6083SPeter Grehan 
353366f6083SPeter Grehan u_long
354366f6083SPeter Grehan vmx_fix_cr0(u_long cr0)
355366f6083SPeter Grehan {
356366f6083SPeter Grehan 
357366f6083SPeter Grehan 	return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask);
358366f6083SPeter Grehan }
359366f6083SPeter Grehan 
360366f6083SPeter Grehan u_long
361366f6083SPeter Grehan vmx_fix_cr4(u_long cr4)
362366f6083SPeter Grehan {
363366f6083SPeter Grehan 
364366f6083SPeter Grehan 	return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask);
365366f6083SPeter Grehan }
366366f6083SPeter Grehan 
367366f6083SPeter Grehan static void
368366f6083SPeter Grehan msr_save_area_init(struct msr_entry *g_area, int *g_count)
369366f6083SPeter Grehan {
370366f6083SPeter Grehan 	int cnt;
371366f6083SPeter Grehan 
372366f6083SPeter Grehan 	static struct msr_entry guest_msrs[] = {
373366f6083SPeter Grehan 		{ MSR_KGSBASE, 0, 0 },
374366f6083SPeter Grehan 	};
375366f6083SPeter Grehan 
376366f6083SPeter Grehan 	cnt = sizeof(guest_msrs) / sizeof(guest_msrs[0]);
377366f6083SPeter Grehan 	if (cnt > GUEST_MSR_MAX_ENTRIES)
378366f6083SPeter Grehan 		panic("guest msr save area overrun");
379366f6083SPeter Grehan 	bcopy(guest_msrs, g_area, sizeof(guest_msrs));
380366f6083SPeter Grehan 	*g_count = cnt;
381366f6083SPeter Grehan }
382366f6083SPeter Grehan 
383366f6083SPeter Grehan static void
384366f6083SPeter Grehan vmx_disable(void *arg __unused)
385366f6083SPeter Grehan {
386366f6083SPeter Grehan 	struct invvpid_desc invvpid_desc = { 0 };
387366f6083SPeter Grehan 	struct invept_desc invept_desc = { 0 };
388366f6083SPeter Grehan 
389366f6083SPeter Grehan 	if (vmxon_enabled[curcpu]) {
390366f6083SPeter Grehan 		/*
391366f6083SPeter Grehan 		 * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b.
392366f6083SPeter Grehan 		 *
393366f6083SPeter Grehan 		 * VMXON or VMXOFF are not required to invalidate any TLB
394366f6083SPeter Grehan 		 * caching structures. This prevents potential retention of
395366f6083SPeter Grehan 		 * cached information in the TLB between distinct VMX episodes.
396366f6083SPeter Grehan 		 */
397366f6083SPeter Grehan 		invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc);
398366f6083SPeter Grehan 		invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc);
399366f6083SPeter Grehan 		vmxoff();
400366f6083SPeter Grehan 	}
401366f6083SPeter Grehan 	load_cr4(rcr4() & ~CR4_VMXE);
402366f6083SPeter Grehan }
403366f6083SPeter Grehan 
404366f6083SPeter Grehan static int
405366f6083SPeter Grehan vmx_cleanup(void)
406366f6083SPeter Grehan {
407366f6083SPeter Grehan 
408366f6083SPeter Grehan 	smp_rendezvous(NULL, vmx_disable, NULL, NULL);
409366f6083SPeter Grehan 
410366f6083SPeter Grehan 	return (0);
411366f6083SPeter Grehan }
412366f6083SPeter Grehan 
413366f6083SPeter Grehan static void
414366f6083SPeter Grehan vmx_enable(void *arg __unused)
415366f6083SPeter Grehan {
416366f6083SPeter Grehan 	int error;
417366f6083SPeter Grehan 
418366f6083SPeter Grehan 	load_cr4(rcr4() | CR4_VMXE);
419366f6083SPeter Grehan 
420366f6083SPeter Grehan 	*(uint32_t *)vmxon_region[curcpu] = vmx_revision();
421366f6083SPeter Grehan 	error = vmxon(vmxon_region[curcpu]);
422366f6083SPeter Grehan 	if (error == 0)
423366f6083SPeter Grehan 		vmxon_enabled[curcpu] = 1;
424366f6083SPeter Grehan }
425366f6083SPeter Grehan 
426366f6083SPeter Grehan static int
427366f6083SPeter Grehan vmx_init(void)
428366f6083SPeter Grehan {
429366f6083SPeter Grehan 	int error;
4304bff7fadSNeel Natu 	uint64_t fixed0, fixed1, feature_control;
431366f6083SPeter Grehan 	uint32_t tmp;
432366f6083SPeter Grehan 
433366f6083SPeter Grehan 	/* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */
4348b287612SJohn Baldwin 	if (!(cpu_feature2 & CPUID2_VMX)) {
435366f6083SPeter Grehan 		printf("vmx_init: processor does not support VMX operation\n");
436366f6083SPeter Grehan 		return (ENXIO);
437366f6083SPeter Grehan 	}
438366f6083SPeter Grehan 
4394bff7fadSNeel Natu 	/*
4404bff7fadSNeel Natu 	 * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits
4414bff7fadSNeel Natu 	 * are set (bits 0 and 2 respectively).
4424bff7fadSNeel Natu 	 */
4434bff7fadSNeel Natu 	feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
4444bff7fadSNeel Natu 	if ((feature_control & 0x5) != 0x5) {
4454bff7fadSNeel Natu 		printf("vmx_init: VMX operation disabled by BIOS\n");
4464bff7fadSNeel Natu 		return (ENXIO);
4474bff7fadSNeel Natu 	}
4484bff7fadSNeel Natu 
449366f6083SPeter Grehan 	/* Check support for primary processor-based VM-execution controls */
450366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
451366f6083SPeter Grehan 			       MSR_VMX_TRUE_PROCBASED_CTLS,
452366f6083SPeter Grehan 			       PROCBASED_CTLS_ONE_SETTING,
453366f6083SPeter Grehan 			       PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls);
454366f6083SPeter Grehan 	if (error) {
455366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired primary "
456366f6083SPeter Grehan 		       "processor-based controls\n");
457366f6083SPeter Grehan 		return (error);
458366f6083SPeter Grehan 	}
459366f6083SPeter Grehan 
460366f6083SPeter Grehan 	/* Clear the processor-based ctl bits that are set on demand */
461366f6083SPeter Grehan 	procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING;
462366f6083SPeter Grehan 
463366f6083SPeter Grehan 	/* Check support for secondary processor-based VM-execution controls */
464366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
465366f6083SPeter Grehan 			       MSR_VMX_PROCBASED_CTLS2,
466366f6083SPeter Grehan 			       PROCBASED_CTLS2_ONE_SETTING,
467366f6083SPeter Grehan 			       PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2);
468366f6083SPeter Grehan 	if (error) {
469366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired secondary "
470366f6083SPeter Grehan 		       "processor-based controls\n");
471366f6083SPeter Grehan 		return (error);
472366f6083SPeter Grehan 	}
473366f6083SPeter Grehan 
474366f6083SPeter Grehan 	/* Check support for VPID */
475366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
476366f6083SPeter Grehan 			       PROCBASED2_ENABLE_VPID, 0, &tmp);
477366f6083SPeter Grehan 	if (error == 0)
478366f6083SPeter Grehan 		procbased_ctls2 |= PROCBASED2_ENABLE_VPID;
479366f6083SPeter Grehan 
480366f6083SPeter Grehan 	/* Check support for pin-based VM-execution controls */
481366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
482366f6083SPeter Grehan 			       MSR_VMX_TRUE_PINBASED_CTLS,
483366f6083SPeter Grehan 			       PINBASED_CTLS_ONE_SETTING,
484366f6083SPeter Grehan 			       PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls);
485366f6083SPeter Grehan 	if (error) {
486366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired "
487366f6083SPeter Grehan 		       "pin-based controls\n");
488366f6083SPeter Grehan 		return (error);
489366f6083SPeter Grehan 	}
490366f6083SPeter Grehan 
491366f6083SPeter Grehan 	/* Check support for VM-exit controls */
492366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS,
493366f6083SPeter Grehan 			       VM_EXIT_CTLS_ONE_SETTING,
494366f6083SPeter Grehan 			       VM_EXIT_CTLS_ZERO_SETTING,
495366f6083SPeter Grehan 			       &exit_ctls);
496366f6083SPeter Grehan 	if (error) {
497608f97c3SPeter Grehan 		/* Try again without the PAT MSR bits */
498608f97c3SPeter Grehan 		error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS,
499608f97c3SPeter Grehan 				       MSR_VMX_TRUE_EXIT_CTLS,
500608f97c3SPeter Grehan 				       VM_EXIT_CTLS_ONE_SETTING_NO_PAT,
501608f97c3SPeter Grehan 				       VM_EXIT_CTLS_ZERO_SETTING,
502608f97c3SPeter Grehan 				       &exit_ctls);
503608f97c3SPeter Grehan 		if (error) {
504366f6083SPeter Grehan 			printf("vmx_init: processor does not support desired "
505366f6083SPeter Grehan 			       "exit controls\n");
506366f6083SPeter Grehan 			return (error);
507608f97c3SPeter Grehan 		} else {
508608f97c3SPeter Grehan 			if (bootverbose)
509608f97c3SPeter Grehan 				printf("vmm: PAT MSR access not supported\n");
510608f97c3SPeter Grehan 			guest_msr_valid(MSR_PAT);
511608f97c3SPeter Grehan 			vmx_no_patmsr = 1;
512608f97c3SPeter Grehan 		}
513366f6083SPeter Grehan 	}
514366f6083SPeter Grehan 
515366f6083SPeter Grehan 	/* Check support for VM-entry controls */
516608f97c3SPeter Grehan 	if (!vmx_no_patmsr) {
517608f97c3SPeter Grehan 		error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS,
518608f97c3SPeter Grehan 				       MSR_VMX_TRUE_ENTRY_CTLS,
519366f6083SPeter Grehan 				       VM_ENTRY_CTLS_ONE_SETTING,
520366f6083SPeter Grehan 				       VM_ENTRY_CTLS_ZERO_SETTING,
521366f6083SPeter Grehan 				       &entry_ctls);
522608f97c3SPeter Grehan 	} else {
523608f97c3SPeter Grehan 		error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS,
524608f97c3SPeter Grehan 				       MSR_VMX_TRUE_ENTRY_CTLS,
525608f97c3SPeter Grehan 				       VM_ENTRY_CTLS_ONE_SETTING_NO_PAT,
526608f97c3SPeter Grehan 				       VM_ENTRY_CTLS_ZERO_SETTING,
527608f97c3SPeter Grehan 				       &entry_ctls);
528608f97c3SPeter Grehan 	}
529608f97c3SPeter Grehan 
530366f6083SPeter Grehan 	if (error) {
531366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired "
532366f6083SPeter Grehan 		       "entry controls\n");
533366f6083SPeter Grehan 		       return (error);
534366f6083SPeter Grehan 	}
535366f6083SPeter Grehan 
536366f6083SPeter Grehan 	/*
537366f6083SPeter Grehan 	 * Check support for optional features by testing them
538366f6083SPeter Grehan 	 * as individual bits
539366f6083SPeter Grehan 	 */
540366f6083SPeter Grehan 	cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
541366f6083SPeter Grehan 					MSR_VMX_TRUE_PROCBASED_CTLS,
542366f6083SPeter Grehan 					PROCBASED_HLT_EXITING, 0,
543366f6083SPeter Grehan 					&tmp) == 0);
544366f6083SPeter Grehan 
545366f6083SPeter Grehan 	cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
546366f6083SPeter Grehan 					MSR_VMX_PROCBASED_CTLS,
547366f6083SPeter Grehan 					PROCBASED_MTF, 0,
548366f6083SPeter Grehan 					&tmp) == 0);
549366f6083SPeter Grehan 
550366f6083SPeter Grehan 	cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
551366f6083SPeter Grehan 					 MSR_VMX_TRUE_PROCBASED_CTLS,
552366f6083SPeter Grehan 					 PROCBASED_PAUSE_EXITING, 0,
553366f6083SPeter Grehan 					 &tmp) == 0);
554366f6083SPeter Grehan 
555366f6083SPeter Grehan 	cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
556366f6083SPeter Grehan 					MSR_VMX_PROCBASED_CTLS2,
557366f6083SPeter Grehan 					PROCBASED2_UNRESTRICTED_GUEST, 0,
558366f6083SPeter Grehan 				        &tmp) == 0);
559366f6083SPeter Grehan 
560366f6083SPeter Grehan 	/* Initialize EPT */
561366f6083SPeter Grehan 	error = ept_init();
562366f6083SPeter Grehan 	if (error) {
563366f6083SPeter Grehan 		printf("vmx_init: ept initialization failed (%d)\n", error);
564366f6083SPeter Grehan 		return (error);
565366f6083SPeter Grehan 	}
566366f6083SPeter Grehan 
567366f6083SPeter Grehan 	/*
568366f6083SPeter Grehan 	 * Stash the cr0 and cr4 bits that must be fixed to 0 or 1
569366f6083SPeter Grehan 	 */
570366f6083SPeter Grehan 	fixed0 = rdmsr(MSR_VMX_CR0_FIXED0);
571366f6083SPeter Grehan 	fixed1 = rdmsr(MSR_VMX_CR0_FIXED1);
572366f6083SPeter Grehan 	cr0_ones_mask = fixed0 & fixed1;
573366f6083SPeter Grehan 	cr0_zeros_mask = ~fixed0 & ~fixed1;
574366f6083SPeter Grehan 
575366f6083SPeter Grehan 	/*
576366f6083SPeter Grehan 	 * CR0_PE and CR0_PG can be set to zero in VMX non-root operation
577366f6083SPeter Grehan 	 * if unrestricted guest execution is allowed.
578366f6083SPeter Grehan 	 */
579366f6083SPeter Grehan 	if (cap_unrestricted_guest)
580366f6083SPeter Grehan 		cr0_ones_mask &= ~(CR0_PG | CR0_PE);
581366f6083SPeter Grehan 
582366f6083SPeter Grehan 	/*
583366f6083SPeter Grehan 	 * Do not allow the guest to set CR0_NW or CR0_CD.
584366f6083SPeter Grehan 	 */
585366f6083SPeter Grehan 	cr0_zeros_mask |= (CR0_NW | CR0_CD);
586366f6083SPeter Grehan 
587366f6083SPeter Grehan 	fixed0 = rdmsr(MSR_VMX_CR4_FIXED0);
588366f6083SPeter Grehan 	fixed1 = rdmsr(MSR_VMX_CR4_FIXED1);
589366f6083SPeter Grehan 	cr4_ones_mask = fixed0 & fixed1;
590366f6083SPeter Grehan 	cr4_zeros_mask = ~fixed0 & ~fixed1;
591366f6083SPeter Grehan 
592366f6083SPeter Grehan 	/* enable VMX operation */
593366f6083SPeter Grehan 	smp_rendezvous(NULL, vmx_enable, NULL, NULL);
594366f6083SPeter Grehan 
595366f6083SPeter Grehan 	return (0);
596366f6083SPeter Grehan }
597366f6083SPeter Grehan 
598366f6083SPeter Grehan /*
599366f6083SPeter Grehan  * If this processor does not support VPIDs then simply return 0.
600366f6083SPeter Grehan  *
601366f6083SPeter Grehan  * Otherwise generate the next value of VPID to use. Any value is alright
602366f6083SPeter Grehan  * as long as it is non-zero.
603366f6083SPeter Grehan  *
604366f6083SPeter Grehan  * We always execute in VMX non-root context with EPT enabled. Thus all
605366f6083SPeter Grehan  * combined mappings are tagged with the (EP4TA, VPID, PCID) tuple. This
606366f6083SPeter Grehan  * in turn means that multiple VMs can share the same VPID as long as
607366f6083SPeter Grehan  * they have distinct EPT page tables.
608366f6083SPeter Grehan  *
609366f6083SPeter Grehan  * XXX
610366f6083SPeter Grehan  * We should optimize this so that it returns VPIDs that are not in
611366f6083SPeter Grehan  * use. Then we will not unnecessarily invalidate mappings in
612366f6083SPeter Grehan  * vmx_set_pcpu_defaults() just because two or more vcpus happen to
613366f6083SPeter Grehan  * use the same 'vpid'.
614366f6083SPeter Grehan  */
615366f6083SPeter Grehan static uint16_t
616366f6083SPeter Grehan vmx_vpid(void)
617366f6083SPeter Grehan {
618366f6083SPeter Grehan 	uint16_t vpid = 0;
619366f6083SPeter Grehan 
620366f6083SPeter Grehan 	if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) != 0) {
621366f6083SPeter Grehan 		do {
622366f6083SPeter Grehan 			vpid = atomic_fetchadd_int(&nextvpid, 1);
623366f6083SPeter Grehan 		} while (vpid == 0);
624366f6083SPeter Grehan 	}
625366f6083SPeter Grehan 
626366f6083SPeter Grehan 	return (vpid);
627366f6083SPeter Grehan }
628366f6083SPeter Grehan 
629366f6083SPeter Grehan static int
630*39c21c2dSNeel Natu vmx_setup_cr_shadow(int which, struct vmcs *vmcs)
631366f6083SPeter Grehan {
632*39c21c2dSNeel Natu 	int error, mask_ident, shadow_ident;
633*39c21c2dSNeel Natu 	uint64_t mask_value, shadow_value;
634366f6083SPeter Grehan 
635*39c21c2dSNeel Natu 	if (which != 0 && which != 4)
636*39c21c2dSNeel Natu 		panic("vmx_setup_cr_shadow: unknown cr%d", which);
637*39c21c2dSNeel Natu 
638*39c21c2dSNeel Natu 	if (which == 0) {
639*39c21c2dSNeel Natu 		mask_ident = VMCS_CR0_MASK;
640*39c21c2dSNeel Natu 		mask_value = cr0_ones_mask | cr0_zeros_mask;
641*39c21c2dSNeel Natu 		shadow_ident = VMCS_CR0_SHADOW;
642*39c21c2dSNeel Natu 		shadow_value = cr0_ones_mask;
643*39c21c2dSNeel Natu 	} else {
644*39c21c2dSNeel Natu 		mask_ident = VMCS_CR4_MASK;
645*39c21c2dSNeel Natu 		mask_value = cr4_ones_mask | cr4_zeros_mask;
646*39c21c2dSNeel Natu 		shadow_ident = VMCS_CR4_SHADOW;
647*39c21c2dSNeel Natu 		shadow_value = cr4_ones_mask;
648*39c21c2dSNeel Natu 	}
649*39c21c2dSNeel Natu 
650*39c21c2dSNeel Natu 	error = vmcs_setreg(vmcs, VMCS_IDENT(mask_ident), mask_value);
651366f6083SPeter Grehan 	if (error)
652366f6083SPeter Grehan 		return (error);
653366f6083SPeter Grehan 
654*39c21c2dSNeel Natu 	error = vmcs_setreg(vmcs, VMCS_IDENT(shadow_ident), shadow_value);
655366f6083SPeter Grehan 	if (error)
656366f6083SPeter Grehan 		return (error);
657366f6083SPeter Grehan 
658366f6083SPeter Grehan 	return (0);
659366f6083SPeter Grehan }
660*39c21c2dSNeel Natu #define	vmx_setup_cr0_shadow(vmcs)	vmx_setup_cr_shadow(0, (vmcs))
661*39c21c2dSNeel Natu #define	vmx_setup_cr4_shadow(vmcs)	vmx_setup_cr_shadow(4, (vmcs))
662366f6083SPeter Grehan 
663366f6083SPeter Grehan static void *
664366f6083SPeter Grehan vmx_vminit(struct vm *vm)
665366f6083SPeter Grehan {
666366f6083SPeter Grehan 	uint16_t vpid;
667366f6083SPeter Grehan 	int i, error, guest_msr_count;
668366f6083SPeter Grehan 	struct vmx *vmx;
669366f6083SPeter Grehan 
670366f6083SPeter Grehan 	vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO);
671366f6083SPeter Grehan 	if ((uintptr_t)vmx & PAGE_MASK) {
672366f6083SPeter Grehan 		panic("malloc of struct vmx not aligned on %d byte boundary",
673366f6083SPeter Grehan 		      PAGE_SIZE);
674366f6083SPeter Grehan 	}
675366f6083SPeter Grehan 	vmx->vm = vm;
676366f6083SPeter Grehan 
677366f6083SPeter Grehan 	/*
678366f6083SPeter Grehan 	 * Clean up EPTP-tagged guest physical and combined mappings
679366f6083SPeter Grehan 	 *
680366f6083SPeter Grehan 	 * VMX transitions are not required to invalidate any guest physical
681366f6083SPeter Grehan 	 * mappings. So, it may be possible for stale guest physical mappings
682366f6083SPeter Grehan 	 * to be present in the processor TLBs.
683366f6083SPeter Grehan 	 *
684366f6083SPeter Grehan 	 * Combined mappings for this EP4TA are also invalidated for all VPIDs.
685366f6083SPeter Grehan 	 */
686366f6083SPeter Grehan 	ept_invalidate_mappings(vtophys(vmx->pml4ept));
687366f6083SPeter Grehan 
688366f6083SPeter Grehan 	msr_bitmap_initialize(vmx->msr_bitmap);
689366f6083SPeter Grehan 
690366f6083SPeter Grehan 	/*
691366f6083SPeter Grehan 	 * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE.
692366f6083SPeter Grehan 	 * The guest FSBASE and GSBASE are saved and restored during
693366f6083SPeter Grehan 	 * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are
694366f6083SPeter Grehan 	 * always restored from the vmcs host state area on vm-exit.
695366f6083SPeter Grehan 	 *
696366f6083SPeter Grehan 	 * Guest KGSBASE is saved and restored in the guest MSR save area.
697366f6083SPeter Grehan 	 * Host KGSBASE is restored before returning to userland from the pcb.
698366f6083SPeter Grehan 	 * There will be a window of time when we are executing in the host
699366f6083SPeter Grehan 	 * kernel context with a value of KGSBASE from the guest. This is ok
700366f6083SPeter Grehan 	 * because the value of KGSBASE is inconsequential in kernel context.
701366f6083SPeter Grehan 	 *
702366f6083SPeter Grehan 	 * MSR_EFER is saved and restored in the guest VMCS area on a
703366f6083SPeter Grehan 	 * VM exit and entry respectively. It is also restored from the
704366f6083SPeter Grehan 	 * host VMCS area on a VM exit.
705366f6083SPeter Grehan 	 */
706366f6083SPeter Grehan 	if (guest_msr_rw(vmx, MSR_GSBASE) ||
707366f6083SPeter Grehan 	    guest_msr_rw(vmx, MSR_FSBASE) ||
708366f6083SPeter Grehan 	    guest_msr_rw(vmx, MSR_KGSBASE) ||
709608f97c3SPeter Grehan 	    guest_msr_rw(vmx, MSR_EFER))
710366f6083SPeter Grehan 		panic("vmx_vminit: error setting guest msr access");
711366f6083SPeter Grehan 
712608f97c3SPeter Grehan 	/*
713608f97c3SPeter Grehan 	 * MSR_PAT is saved and restored in the guest VMCS are on a VM exit
714608f97c3SPeter Grehan 	 * and entry respectively. It is also restored from the host VMCS
715608f97c3SPeter Grehan 	 * area on a VM exit. However, if running on a system with no
716608f97c3SPeter Grehan 	 * MSR_PAT save/restore support, leave access disabled so accesses
717608f97c3SPeter Grehan 	 * will be trapped.
718608f97c3SPeter Grehan 	 */
719608f97c3SPeter Grehan 	if (!vmx_no_patmsr && guest_msr_rw(vmx, MSR_PAT))
720608f97c3SPeter Grehan 		panic("vmx_vminit: error setting guest pat msr access");
721608f97c3SPeter Grehan 
722366f6083SPeter Grehan 	for (i = 0; i < VM_MAXCPU; i++) {
723366f6083SPeter Grehan 		vmx->vmcs[i].identifier = vmx_revision();
724366f6083SPeter Grehan 		error = vmclear(&vmx->vmcs[i]);
725366f6083SPeter Grehan 		if (error != 0) {
726366f6083SPeter Grehan 			panic("vmx_vminit: vmclear error %d on vcpu %d\n",
727366f6083SPeter Grehan 			      error, i);
728366f6083SPeter Grehan 		}
729366f6083SPeter Grehan 
730366f6083SPeter Grehan 		vpid = vmx_vpid();
731366f6083SPeter Grehan 
732366f6083SPeter Grehan 		error = vmcs_set_defaults(&vmx->vmcs[i],
733366f6083SPeter Grehan 					  (u_long)vmx_longjmp,
734366f6083SPeter Grehan 					  (u_long)&vmx->ctx[i],
735366f6083SPeter Grehan 					  vtophys(vmx->pml4ept),
736366f6083SPeter Grehan 					  pinbased_ctls,
737366f6083SPeter Grehan 					  procbased_ctls,
738366f6083SPeter Grehan 					  procbased_ctls2,
739366f6083SPeter Grehan 					  exit_ctls, entry_ctls,
740366f6083SPeter Grehan 					  vtophys(vmx->msr_bitmap),
741366f6083SPeter Grehan 					  vpid);
742366f6083SPeter Grehan 
743366f6083SPeter Grehan 		if (error != 0)
744366f6083SPeter Grehan 			panic("vmx_vminit: vmcs_set_defaults error %d", error);
745366f6083SPeter Grehan 
746366f6083SPeter Grehan 		vmx->cap[i].set = 0;
747366f6083SPeter Grehan 		vmx->cap[i].proc_ctls = procbased_ctls;
748366f6083SPeter Grehan 
749366f6083SPeter Grehan 		vmx->state[i].request_nmi = 0;
750366f6083SPeter Grehan 		vmx->state[i].lastcpu = -1;
751366f6083SPeter Grehan 		vmx->state[i].vpid = vpid;
752366f6083SPeter Grehan 
753366f6083SPeter Grehan 		msr_save_area_init(vmx->guest_msrs[i], &guest_msr_count);
754366f6083SPeter Grehan 
755366f6083SPeter Grehan 		error = vmcs_set_msr_save(&vmx->vmcs[i],
756366f6083SPeter Grehan 					  vtophys(vmx->guest_msrs[i]),
757366f6083SPeter Grehan 					  guest_msr_count);
758366f6083SPeter Grehan 		if (error != 0)
759366f6083SPeter Grehan 			panic("vmcs_set_msr_save error %d", error);
760366f6083SPeter Grehan 
761366f6083SPeter Grehan 		error = vmx_setup_cr0_shadow(&vmx->vmcs[i]);
762*39c21c2dSNeel Natu 		if (error != 0)
763*39c21c2dSNeel Natu 			panic("vmx_setup_cr0_shadow %d", error);
764*39c21c2dSNeel Natu 
765*39c21c2dSNeel Natu 		error = vmx_setup_cr4_shadow(&vmx->vmcs[i]);
766*39c21c2dSNeel Natu 		if (error != 0)
767*39c21c2dSNeel Natu 			panic("vmx_setup_cr4_shadow %d", error);
768366f6083SPeter Grehan 	}
769366f6083SPeter Grehan 
770366f6083SPeter Grehan 	return (vmx);
771366f6083SPeter Grehan }
772366f6083SPeter Grehan 
773366f6083SPeter Grehan static int
7748b287612SJohn Baldwin vmx_handle_cpuid(int vcpu, struct vmxctx *vmxctx)
775366f6083SPeter Grehan {
776366f6083SPeter Grehan 	int handled, func;
777366f6083SPeter Grehan 
778366f6083SPeter Grehan 	func = vmxctx->guest_rax;
779366f6083SPeter Grehan 
780366f6083SPeter Grehan 	handled = x86_emulate_cpuid((uint32_t*)(&vmxctx->guest_rax),
781366f6083SPeter Grehan 	    (uint32_t*)(&vmxctx->guest_rbx), (uint32_t*)(&vmxctx->guest_rcx),
7828b287612SJohn Baldwin 	    (uint32_t*)(&vmxctx->guest_rdx), vcpu);
783366f6083SPeter Grehan #if 0
784366f6083SPeter Grehan 	printf("%s: func %x rax %lx rbx %lx rcx %lx rdx %lx handled %d\n",
785366f6083SPeter Grehan 		__func__, func, vmxctx->guest_rax, vmxctx->guest_rbx,
786366f6083SPeter Grehan 		vmxctx->guest_rcx, vmxctx->guest_rdx, handled);
787366f6083SPeter Grehan #endif
788366f6083SPeter Grehan 
789366f6083SPeter Grehan 	return (handled);
790366f6083SPeter Grehan }
791366f6083SPeter Grehan 
792366f6083SPeter Grehan static __inline void
793366f6083SPeter Grehan vmx_run_trace(struct vmx *vmx, int vcpu)
794366f6083SPeter Grehan {
795366f6083SPeter Grehan #ifdef KTR
796366f6083SPeter Grehan 	VMM_CTR1(vmx->vm, vcpu, "Resume execution at 0x%0lx", vmcs_guest_rip());
797366f6083SPeter Grehan #endif
798366f6083SPeter Grehan }
799366f6083SPeter Grehan 
800366f6083SPeter Grehan static __inline void
801366f6083SPeter Grehan vmx_exit_trace(struct vmx *vmx, int vcpu, uint64_t rip, uint32_t exit_reason,
802366f6083SPeter Grehan 	       int handled, int astpending)
803366f6083SPeter Grehan {
804366f6083SPeter Grehan #ifdef KTR
805366f6083SPeter Grehan 	VMM_CTR3(vmx->vm, vcpu, "%s %s vmexit at 0x%0lx",
806366f6083SPeter Grehan 		 handled ? "handled" : "unhandled",
807366f6083SPeter Grehan 		 exit_reason_to_str(exit_reason), rip);
808366f6083SPeter Grehan 
809366f6083SPeter Grehan 	if (astpending)
810366f6083SPeter Grehan 		VMM_CTR0(vmx->vm, vcpu, "astpending");
811366f6083SPeter Grehan #endif
812366f6083SPeter Grehan }
813366f6083SPeter Grehan 
814366f6083SPeter Grehan static int
815366f6083SPeter Grehan vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu)
816366f6083SPeter Grehan {
817366f6083SPeter Grehan 	int error, lastcpu;
818366f6083SPeter Grehan 	struct vmxstate *vmxstate;
819366f6083SPeter Grehan 	struct invvpid_desc invvpid_desc = { 0 };
820366f6083SPeter Grehan 
821366f6083SPeter Grehan 	vmxstate = &vmx->state[vcpu];
822366f6083SPeter Grehan 	lastcpu = vmxstate->lastcpu;
823366f6083SPeter Grehan 	vmxstate->lastcpu = curcpu;
824366f6083SPeter Grehan 
825366f6083SPeter Grehan 	if (lastcpu == curcpu) {
826366f6083SPeter Grehan 		error = 0;
827366f6083SPeter Grehan 		goto done;
828366f6083SPeter Grehan 	}
829366f6083SPeter Grehan 
830366f6083SPeter Grehan 	vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1);
831366f6083SPeter Grehan 
832366f6083SPeter Grehan 	error = vmwrite(VMCS_HOST_TR_BASE, (u_long)PCPU_GET(tssp));
833366f6083SPeter Grehan 	if (error != 0)
834366f6083SPeter Grehan 		goto done;
835366f6083SPeter Grehan 
836366f6083SPeter Grehan 	error = vmwrite(VMCS_HOST_GDTR_BASE, (u_long)&gdt[NGDT * curcpu]);
837366f6083SPeter Grehan 	if (error != 0)
838366f6083SPeter Grehan 		goto done;
839366f6083SPeter Grehan 
840366f6083SPeter Grehan 	error = vmwrite(VMCS_HOST_GS_BASE, (u_long)&__pcpu[curcpu]);
841366f6083SPeter Grehan 	if (error != 0)
842366f6083SPeter Grehan 		goto done;
843366f6083SPeter Grehan 
844366f6083SPeter Grehan 	/*
845366f6083SPeter Grehan 	 * If we are using VPIDs then invalidate all mappings tagged with 'vpid'
846366f6083SPeter Grehan 	 *
847366f6083SPeter Grehan 	 * We do this because this vcpu was executing on a different host
848366f6083SPeter Grehan 	 * cpu when it last ran. We do not track whether it invalidated
849366f6083SPeter Grehan 	 * mappings associated with its 'vpid' during that run. So we must
850366f6083SPeter Grehan 	 * assume that the mappings associated with 'vpid' on 'curcpu' are
851366f6083SPeter Grehan 	 * stale and invalidate them.
852366f6083SPeter Grehan 	 *
853366f6083SPeter Grehan 	 * Note that we incur this penalty only when the scheduler chooses to
854366f6083SPeter Grehan 	 * move the thread associated with this vcpu between host cpus.
855366f6083SPeter Grehan 	 *
856366f6083SPeter Grehan 	 * Note also that this will invalidate mappings tagged with 'vpid'
857366f6083SPeter Grehan 	 * for "all" EP4TAs.
858366f6083SPeter Grehan 	 */
859366f6083SPeter Grehan 	if (vmxstate->vpid != 0) {
860366f6083SPeter Grehan 		invvpid_desc.vpid = vmxstate->vpid;
861366f6083SPeter Grehan 		invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc);
862366f6083SPeter Grehan 	}
863366f6083SPeter Grehan done:
864366f6083SPeter Grehan 	return (error);
865366f6083SPeter Grehan }
866366f6083SPeter Grehan 
867366f6083SPeter Grehan static void
868366f6083SPeter Grehan vm_exit_update_rip(struct vm_exit *vmexit)
869366f6083SPeter Grehan {
870366f6083SPeter Grehan 	int error;
871366f6083SPeter Grehan 
872366f6083SPeter Grehan 	error = vmwrite(VMCS_GUEST_RIP, vmexit->rip + vmexit->inst_length);
873366f6083SPeter Grehan 	if (error)
874366f6083SPeter Grehan 		panic("vmx_run: error %d writing to VMCS_GUEST_RIP", error);
875366f6083SPeter Grehan }
876366f6083SPeter Grehan 
877366f6083SPeter Grehan /*
878366f6083SPeter Grehan  * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set.
879366f6083SPeter Grehan  */
880366f6083SPeter Grehan CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0);
881366f6083SPeter Grehan 
882366f6083SPeter Grehan static void __inline
883366f6083SPeter Grehan vmx_set_int_window_exiting(struct vmx *vmx, int vcpu)
884366f6083SPeter Grehan {
885366f6083SPeter Grehan 	int error;
886366f6083SPeter Grehan 
887366f6083SPeter Grehan 	vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING;
888366f6083SPeter Grehan 
889366f6083SPeter Grehan 	error = vmwrite(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
890366f6083SPeter Grehan 	if (error)
891366f6083SPeter Grehan 		panic("vmx_set_int_window_exiting: vmwrite error %d", error);
892366f6083SPeter Grehan }
893366f6083SPeter Grehan 
894366f6083SPeter Grehan static void __inline
895366f6083SPeter Grehan vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu)
896366f6083SPeter Grehan {
897366f6083SPeter Grehan 	int error;
898366f6083SPeter Grehan 
899366f6083SPeter Grehan 	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING;
900366f6083SPeter Grehan 
901366f6083SPeter Grehan 	error = vmwrite(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
902366f6083SPeter Grehan 	if (error)
903366f6083SPeter Grehan 		panic("vmx_clear_int_window_exiting: vmwrite error %d", error);
904366f6083SPeter Grehan }
905366f6083SPeter Grehan 
906366f6083SPeter Grehan static void __inline
907366f6083SPeter Grehan vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu)
908366f6083SPeter Grehan {
909366f6083SPeter Grehan 	int error;
910366f6083SPeter Grehan 
911366f6083SPeter Grehan 	vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING;
912366f6083SPeter Grehan 
913366f6083SPeter Grehan 	error = vmwrite(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
914366f6083SPeter Grehan 	if (error)
915366f6083SPeter Grehan 		panic("vmx_set_nmi_window_exiting: vmwrite error %d", error);
916366f6083SPeter Grehan }
917366f6083SPeter Grehan 
918366f6083SPeter Grehan static void __inline
919366f6083SPeter Grehan vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu)
920366f6083SPeter Grehan {
921366f6083SPeter Grehan 	int error;
922366f6083SPeter Grehan 
923366f6083SPeter Grehan 	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING;
924366f6083SPeter Grehan 
925366f6083SPeter Grehan 	error = vmwrite(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
926366f6083SPeter Grehan 	if (error)
927366f6083SPeter Grehan 		panic("vmx_clear_nmi_window_exiting: vmwrite error %d", error);
928366f6083SPeter Grehan }
929366f6083SPeter Grehan 
930366f6083SPeter Grehan static int
931366f6083SPeter Grehan vmx_inject_nmi(struct vmx *vmx, int vcpu)
932366f6083SPeter Grehan {
933366f6083SPeter Grehan 	int error;
934366f6083SPeter Grehan 	uint64_t info, interruptibility;
935366f6083SPeter Grehan 
936366f6083SPeter Grehan 	/* Bail out if no NMI requested */
937366f6083SPeter Grehan 	if (vmx->state[vcpu].request_nmi == 0)
938366f6083SPeter Grehan 		return (0);
939366f6083SPeter Grehan 
940366f6083SPeter Grehan 	error = vmread(VMCS_GUEST_INTERRUPTIBILITY, &interruptibility);
941366f6083SPeter Grehan 	if (error) {
942366f6083SPeter Grehan 		panic("vmx_inject_nmi: vmread(interruptibility) %d",
943366f6083SPeter Grehan 			error);
944366f6083SPeter Grehan 	}
945366f6083SPeter Grehan 	if (interruptibility & nmi_blocking_bits)
946366f6083SPeter Grehan 		goto nmiblocked;
947366f6083SPeter Grehan 
948366f6083SPeter Grehan 	/*
949366f6083SPeter Grehan 	 * Inject the virtual NMI. The vector must be the NMI IDT entry
950366f6083SPeter Grehan 	 * or the VMCS entry check will fail.
951366f6083SPeter Grehan 	 */
952366f6083SPeter Grehan 	info = VMCS_INTERRUPTION_INFO_NMI | VMCS_INTERRUPTION_INFO_VALID;
953366f6083SPeter Grehan 	info |= IDT_NMI;
954366f6083SPeter Grehan 
955366f6083SPeter Grehan 	error = vmwrite(VMCS_ENTRY_INTR_INFO, info);
956366f6083SPeter Grehan 	if (error)
957366f6083SPeter Grehan 		panic("vmx_inject_nmi: vmwrite(intrinfo) %d", error);
958366f6083SPeter Grehan 
959366f6083SPeter Grehan 	VMM_CTR0(vmx->vm, vcpu, "Injecting vNMI");
960366f6083SPeter Grehan 
961366f6083SPeter Grehan 	/* Clear the request */
962366f6083SPeter Grehan 	vmx->state[vcpu].request_nmi = 0;
963366f6083SPeter Grehan 	return (1);
964366f6083SPeter Grehan 
965366f6083SPeter Grehan nmiblocked:
966366f6083SPeter Grehan 	/*
967366f6083SPeter Grehan 	 * Set the NMI Window Exiting execution control so we can inject
968366f6083SPeter Grehan 	 * the virtual NMI as soon as blocking condition goes away.
969366f6083SPeter Grehan 	 */
970366f6083SPeter Grehan 	vmx_set_nmi_window_exiting(vmx, vcpu);
971366f6083SPeter Grehan 
972366f6083SPeter Grehan 	VMM_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting");
973366f6083SPeter Grehan 	return (1);
974366f6083SPeter Grehan }
975366f6083SPeter Grehan 
976366f6083SPeter Grehan static void
977366f6083SPeter Grehan vmx_inject_interrupts(struct vmx *vmx, int vcpu)
978366f6083SPeter Grehan {
979366f6083SPeter Grehan 	int error, vector;
980366f6083SPeter Grehan 	uint64_t info, rflags, interruptibility;
981366f6083SPeter Grehan 
982366f6083SPeter Grehan 	const int HWINTR_BLOCKED = VMCS_INTERRUPTIBILITY_STI_BLOCKING |
983366f6083SPeter Grehan 				   VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING;
984366f6083SPeter Grehan 
985366f6083SPeter Grehan #if 1
986366f6083SPeter Grehan 	/*
987366f6083SPeter Grehan 	 * XXX
988366f6083SPeter Grehan 	 * If an event is being injected from userland then just return.
989366f6083SPeter Grehan 	 * For e.g. we may inject a breakpoint exception to cause the
990366f6083SPeter Grehan 	 * guest to enter the debugger so we can inspect its state.
991366f6083SPeter Grehan 	 */
992366f6083SPeter Grehan 	error = vmread(VMCS_ENTRY_INTR_INFO, &info);
993366f6083SPeter Grehan 	if (error)
994366f6083SPeter Grehan 		panic("vmx_inject_interrupts: vmread(intrinfo) %d", error);
995366f6083SPeter Grehan 	if (info & VMCS_INTERRUPTION_INFO_VALID)
996366f6083SPeter Grehan 		return;
997366f6083SPeter Grehan #endif
998366f6083SPeter Grehan 	/*
999366f6083SPeter Grehan 	 * NMI injection has priority so deal with those first
1000366f6083SPeter Grehan 	 */
1001366f6083SPeter Grehan 	if (vmx_inject_nmi(vmx, vcpu))
1002366f6083SPeter Grehan 		return;
1003366f6083SPeter Grehan 
1004366f6083SPeter Grehan 	/* Ask the local apic for a vector to inject */
1005366f6083SPeter Grehan 	vector = lapic_pending_intr(vmx->vm, vcpu);
1006366f6083SPeter Grehan 	if (vector < 0)
1007366f6083SPeter Grehan 		return;
1008366f6083SPeter Grehan 
1009366f6083SPeter Grehan 	if (vector < 32 || vector > 255)
1010366f6083SPeter Grehan 		panic("vmx_inject_interrupts: invalid vector %d\n", vector);
1011366f6083SPeter Grehan 
1012366f6083SPeter Grehan 	/* Check RFLAGS.IF and the interruptibility state of the guest */
1013366f6083SPeter Grehan 	error = vmread(VMCS_GUEST_RFLAGS, &rflags);
1014366f6083SPeter Grehan 	if (error)
1015366f6083SPeter Grehan 		panic("vmx_inject_interrupts: vmread(rflags) %d", error);
1016366f6083SPeter Grehan 
1017366f6083SPeter Grehan 	if ((rflags & PSL_I) == 0)
1018366f6083SPeter Grehan 		goto cantinject;
1019366f6083SPeter Grehan 
1020366f6083SPeter Grehan 	error = vmread(VMCS_GUEST_INTERRUPTIBILITY, &interruptibility);
1021366f6083SPeter Grehan 	if (error) {
1022366f6083SPeter Grehan 		panic("vmx_inject_interrupts: vmread(interruptibility) %d",
1023366f6083SPeter Grehan 			error);
1024366f6083SPeter Grehan 	}
1025366f6083SPeter Grehan 	if (interruptibility & HWINTR_BLOCKED)
1026366f6083SPeter Grehan 		goto cantinject;
1027366f6083SPeter Grehan 
1028366f6083SPeter Grehan 	/* Inject the interrupt */
1029366f6083SPeter Grehan 	info = VMCS_INTERRUPTION_INFO_HW_INTR | VMCS_INTERRUPTION_INFO_VALID;
1030366f6083SPeter Grehan 	info |= vector;
1031366f6083SPeter Grehan 	error = vmwrite(VMCS_ENTRY_INTR_INFO, info);
1032366f6083SPeter Grehan 	if (error)
1033366f6083SPeter Grehan 		panic("vmx_inject_interrupts: vmwrite(intrinfo) %d", error);
1034366f6083SPeter Grehan 
1035366f6083SPeter Grehan 	/* Update the Local APIC ISR */
1036366f6083SPeter Grehan 	lapic_intr_accepted(vmx->vm, vcpu, vector);
1037366f6083SPeter Grehan 
1038366f6083SPeter Grehan 	VMM_CTR1(vmx->vm, vcpu, "Injecting hwintr at vector %d", vector);
1039366f6083SPeter Grehan 
1040366f6083SPeter Grehan 	return;
1041366f6083SPeter Grehan 
1042366f6083SPeter Grehan cantinject:
1043366f6083SPeter Grehan 	/*
1044366f6083SPeter Grehan 	 * Set the Interrupt Window Exiting execution control so we can inject
1045366f6083SPeter Grehan 	 * the interrupt as soon as blocking condition goes away.
1046366f6083SPeter Grehan 	 */
1047366f6083SPeter Grehan 	vmx_set_int_window_exiting(vmx, vcpu);
1048366f6083SPeter Grehan 
1049366f6083SPeter Grehan 	VMM_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting");
1050366f6083SPeter Grehan }
1051366f6083SPeter Grehan 
1052366f6083SPeter Grehan static int
1053366f6083SPeter Grehan vmx_emulate_cr_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1054366f6083SPeter Grehan {
1055*39c21c2dSNeel Natu 	int error, cr, vmcs_guest_cr;
1056*39c21c2dSNeel Natu 	uint64_t regval, ones_mask, zeros_mask;
1057366f6083SPeter Grehan 	const struct vmxctx *vmxctx;
1058366f6083SPeter Grehan 
1059*39c21c2dSNeel Natu 	/* We only handle mov to %cr0 or %cr4 at this time */
1060*39c21c2dSNeel Natu 	if ((exitqual & 0xf0) != 0x00)
1061*39c21c2dSNeel Natu 		return (UNHANDLED);
1062*39c21c2dSNeel Natu 
1063*39c21c2dSNeel Natu 	cr = exitqual & 0xf;
1064*39c21c2dSNeel Natu 	if (cr != 0 && cr != 4)
1065366f6083SPeter Grehan 		return (UNHANDLED);
1066366f6083SPeter Grehan 
1067366f6083SPeter Grehan 	vmxctx = &vmx->ctx[vcpu];
1068366f6083SPeter Grehan 
1069366f6083SPeter Grehan 	/*
1070366f6083SPeter Grehan 	 * We must use vmwrite() directly here because vmcs_setreg() will
1071366f6083SPeter Grehan 	 * call vmclear(vmcs) as a side-effect which we certainly don't want.
1072366f6083SPeter Grehan 	 */
1073366f6083SPeter Grehan 	switch ((exitqual >> 8) & 0xf) {
1074366f6083SPeter Grehan 	case 0:
1075366f6083SPeter Grehan 		regval = vmxctx->guest_rax;
1076366f6083SPeter Grehan 		break;
1077366f6083SPeter Grehan 	case 1:
1078366f6083SPeter Grehan 		regval = vmxctx->guest_rcx;
1079366f6083SPeter Grehan 		break;
1080366f6083SPeter Grehan 	case 2:
1081366f6083SPeter Grehan 		regval = vmxctx->guest_rdx;
1082366f6083SPeter Grehan 		break;
1083366f6083SPeter Grehan 	case 3:
1084366f6083SPeter Grehan 		regval = vmxctx->guest_rbx;
1085366f6083SPeter Grehan 		break;
1086366f6083SPeter Grehan 	case 4:
1087366f6083SPeter Grehan 		error = vmread(VMCS_GUEST_RSP, &regval);
1088366f6083SPeter Grehan 		if (error) {
1089366f6083SPeter Grehan 			panic("vmx_emulate_cr_access: "
1090366f6083SPeter Grehan 			      "error %d reading guest rsp", error);
1091366f6083SPeter Grehan 		}
1092366f6083SPeter Grehan 		break;
1093366f6083SPeter Grehan 	case 5:
1094366f6083SPeter Grehan 		regval = vmxctx->guest_rbp;
1095366f6083SPeter Grehan 		break;
1096366f6083SPeter Grehan 	case 6:
1097366f6083SPeter Grehan 		regval = vmxctx->guest_rsi;
1098366f6083SPeter Grehan 		break;
1099366f6083SPeter Grehan 	case 7:
1100366f6083SPeter Grehan 		regval = vmxctx->guest_rdi;
1101366f6083SPeter Grehan 		break;
1102366f6083SPeter Grehan 	case 8:
1103366f6083SPeter Grehan 		regval = vmxctx->guest_r8;
1104366f6083SPeter Grehan 		break;
1105366f6083SPeter Grehan 	case 9:
1106366f6083SPeter Grehan 		regval = vmxctx->guest_r9;
1107366f6083SPeter Grehan 		break;
1108366f6083SPeter Grehan 	case 10:
1109366f6083SPeter Grehan 		regval = vmxctx->guest_r10;
1110366f6083SPeter Grehan 		break;
1111366f6083SPeter Grehan 	case 11:
1112366f6083SPeter Grehan 		regval = vmxctx->guest_r11;
1113366f6083SPeter Grehan 		break;
1114366f6083SPeter Grehan 	case 12:
1115366f6083SPeter Grehan 		regval = vmxctx->guest_r12;
1116366f6083SPeter Grehan 		break;
1117366f6083SPeter Grehan 	case 13:
1118366f6083SPeter Grehan 		regval = vmxctx->guest_r13;
1119366f6083SPeter Grehan 		break;
1120366f6083SPeter Grehan 	case 14:
1121366f6083SPeter Grehan 		regval = vmxctx->guest_r14;
1122366f6083SPeter Grehan 		break;
1123366f6083SPeter Grehan 	case 15:
1124366f6083SPeter Grehan 		regval = vmxctx->guest_r15;
1125366f6083SPeter Grehan 		break;
1126366f6083SPeter Grehan 	}
1127366f6083SPeter Grehan 
1128*39c21c2dSNeel Natu 	if (cr == 0) {
1129*39c21c2dSNeel Natu 		ones_mask = cr0_ones_mask;
1130*39c21c2dSNeel Natu 		zeros_mask = cr0_zeros_mask;
1131*39c21c2dSNeel Natu 		vmcs_guest_cr = VMCS_GUEST_CR0;
1132*39c21c2dSNeel Natu 	} else {
1133*39c21c2dSNeel Natu 		ones_mask = cr4_ones_mask;
1134*39c21c2dSNeel Natu 		zeros_mask = cr4_zeros_mask;
1135*39c21c2dSNeel Natu 		vmcs_guest_cr = VMCS_GUEST_CR4;
1136*39c21c2dSNeel Natu 	}
1137*39c21c2dSNeel Natu 	regval |= ones_mask;
1138*39c21c2dSNeel Natu 	regval &= ~zeros_mask;
1139*39c21c2dSNeel Natu 	error = vmwrite(vmcs_guest_cr, regval);
1140*39c21c2dSNeel Natu 	if (error) {
1141*39c21c2dSNeel Natu 		panic("vmx_emulate_cr_access: error %d writing cr%d",
1142*39c21c2dSNeel Natu 		      error, cr);
1143*39c21c2dSNeel Natu 	}
1144366f6083SPeter Grehan 
1145366f6083SPeter Grehan 	return (HANDLED);
1146366f6083SPeter Grehan }
1147366f6083SPeter Grehan 
1148366f6083SPeter Grehan static int
1149366f6083SPeter Grehan vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1150366f6083SPeter Grehan {
1151366f6083SPeter Grehan 	int handled;
1152366f6083SPeter Grehan 	struct vmcs *vmcs;
1153366f6083SPeter Grehan 	struct vmxctx *vmxctx;
1154366f6083SPeter Grehan 	uint32_t eax, ecx, edx;
1155366f6083SPeter Grehan 	uint64_t qual;
1156366f6083SPeter Grehan 
1157366f6083SPeter Grehan 	handled = 0;
1158366f6083SPeter Grehan 	vmcs = &vmx->vmcs[vcpu];
1159366f6083SPeter Grehan 	vmxctx = &vmx->ctx[vcpu];
1160366f6083SPeter Grehan 	qual = vmexit->u.vmx.exit_qualification;
1161366f6083SPeter Grehan 	vmexit->exitcode = VM_EXITCODE_BOGUS;
1162366f6083SPeter Grehan 
1163366f6083SPeter Grehan 	switch (vmexit->u.vmx.exit_reason) {
1164366f6083SPeter Grehan 	case EXIT_REASON_CR_ACCESS:
1165366f6083SPeter Grehan 		handled = vmx_emulate_cr_access(vmx, vcpu, qual);
1166366f6083SPeter Grehan 		break;
1167366f6083SPeter Grehan 	case EXIT_REASON_RDMSR:
1168366f6083SPeter Grehan 		ecx = vmxctx->guest_rcx;
1169366f6083SPeter Grehan 		handled = emulate_rdmsr(vmx->vm, vcpu, ecx);
1170366f6083SPeter Grehan 		if (!handled) {
1171366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_RDMSR;
1172366f6083SPeter Grehan 			vmexit->u.msr.code = ecx;
1173366f6083SPeter Grehan 		}
1174366f6083SPeter Grehan 		break;
1175366f6083SPeter Grehan 	case EXIT_REASON_WRMSR:
1176366f6083SPeter Grehan 		eax = vmxctx->guest_rax;
1177366f6083SPeter Grehan 		ecx = vmxctx->guest_rcx;
1178366f6083SPeter Grehan 		edx = vmxctx->guest_rdx;
1179366f6083SPeter Grehan 		handled = emulate_wrmsr(vmx->vm, vcpu, ecx,
1180366f6083SPeter Grehan 					(uint64_t)edx << 32 | eax);
1181366f6083SPeter Grehan 		if (!handled) {
1182366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_WRMSR;
1183366f6083SPeter Grehan 			vmexit->u.msr.code = ecx;
1184366f6083SPeter Grehan 			vmexit->u.msr.wval = (uint64_t)edx << 32 | eax;
1185366f6083SPeter Grehan 		}
1186366f6083SPeter Grehan 		break;
1187366f6083SPeter Grehan 	case EXIT_REASON_HLT:
1188366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_HLT;
1189366f6083SPeter Grehan 		break;
1190366f6083SPeter Grehan 	case EXIT_REASON_MTF:
1191366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_MTRAP;
1192366f6083SPeter Grehan 		break;
1193366f6083SPeter Grehan 	case EXIT_REASON_PAUSE:
1194366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_PAUSE;
1195366f6083SPeter Grehan 		break;
1196366f6083SPeter Grehan 	case EXIT_REASON_INTR_WINDOW:
1197366f6083SPeter Grehan 		vmx_clear_int_window_exiting(vmx, vcpu);
1198366f6083SPeter Grehan 		VMM_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting");
1199366f6083SPeter Grehan 		/* FALLTHRU */
1200366f6083SPeter Grehan 	case EXIT_REASON_EXT_INTR:
1201366f6083SPeter Grehan 		/*
1202366f6083SPeter Grehan 		 * External interrupts serve only to cause VM exits and allow
1203366f6083SPeter Grehan 		 * the host interrupt handler to run.
1204366f6083SPeter Grehan 		 *
1205366f6083SPeter Grehan 		 * If this external interrupt triggers a virtual interrupt
1206366f6083SPeter Grehan 		 * to a VM, then that state will be recorded by the
1207366f6083SPeter Grehan 		 * host interrupt handler in the VM's softc. We will inject
1208366f6083SPeter Grehan 		 * this virtual interrupt during the subsequent VM enter.
1209366f6083SPeter Grehan 		 */
1210366f6083SPeter Grehan 
1211366f6083SPeter Grehan 		/*
1212366f6083SPeter Grehan 		 * This is special. We want to treat this as an 'handled'
1213366f6083SPeter Grehan 		 * VM-exit but not increment the instruction pointer.
1214366f6083SPeter Grehan 		 */
1215366f6083SPeter Grehan 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1);
1216366f6083SPeter Grehan 		return (1);
1217366f6083SPeter Grehan 	case EXIT_REASON_NMI_WINDOW:
1218366f6083SPeter Grehan 		/* Exit to allow the pending virtual NMI to be injected */
1219366f6083SPeter Grehan 		vmx_clear_nmi_window_exiting(vmx, vcpu);
1220366f6083SPeter Grehan 		VMM_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting");
1221366f6083SPeter Grehan 		return (1);
1222366f6083SPeter Grehan 	case EXIT_REASON_INOUT:
1223366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_INOUT;
1224366f6083SPeter Grehan 		vmexit->u.inout.bytes = (qual & 0x7) + 1;
1225366f6083SPeter Grehan 		vmexit->u.inout.in = (qual & 0x8) ? 1 : 0;
1226366f6083SPeter Grehan 		vmexit->u.inout.string = (qual & 0x10) ? 1 : 0;
1227366f6083SPeter Grehan 		vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0;
1228366f6083SPeter Grehan 		vmexit->u.inout.port = (uint16_t)(qual >> 16);
1229366f6083SPeter Grehan 		vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax);
1230366f6083SPeter Grehan 		break;
1231366f6083SPeter Grehan 	case EXIT_REASON_CPUID:
12328b287612SJohn Baldwin 		handled = vmx_handle_cpuid(vcpu, vmxctx);
1233366f6083SPeter Grehan 		break;
1234cd942e0fSPeter Grehan 	case EXIT_REASON_EPT_FAULT:
1235cd942e0fSPeter Grehan 		vmexit->exitcode = VM_EXITCODE_PAGING;
1236cd942e0fSPeter Grehan 		vmexit->u.paging.cr3 = vmcs_guest_cr3();
1237cd942e0fSPeter Grehan 		break;
1238366f6083SPeter Grehan 	default:
1239366f6083SPeter Grehan 		break;
1240366f6083SPeter Grehan 	}
1241366f6083SPeter Grehan 
1242366f6083SPeter Grehan 	if (handled) {
1243366f6083SPeter Grehan 		/*
1244366f6083SPeter Grehan 		 * It is possible that control is returned to userland
1245366f6083SPeter Grehan 		 * even though we were able to handle the VM exit in the
1246366f6083SPeter Grehan 		 * kernel (for e.g. 'astpending' is set in the run loop).
1247366f6083SPeter Grehan 		 *
1248366f6083SPeter Grehan 		 * In such a case we want to make sure that the userland
1249366f6083SPeter Grehan 		 * restarts guest execution at the instruction *after*
1250366f6083SPeter Grehan 		 * the one we just processed. Therefore we update the
1251366f6083SPeter Grehan 		 * guest rip in the VMCS and in 'vmexit'.
1252366f6083SPeter Grehan 		 */
1253366f6083SPeter Grehan 		vm_exit_update_rip(vmexit);
1254366f6083SPeter Grehan 		vmexit->rip += vmexit->inst_length;
1255366f6083SPeter Grehan 		vmexit->inst_length = 0;
1256366f6083SPeter Grehan 	} else {
1257366f6083SPeter Grehan 		if (vmexit->exitcode == VM_EXITCODE_BOGUS) {
1258366f6083SPeter Grehan 			/*
1259366f6083SPeter Grehan 			 * If this VM exit was not claimed by anybody then
1260366f6083SPeter Grehan 			 * treat it as a generic VMX exit.
1261366f6083SPeter Grehan 			 */
1262366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_VMX;
1263366f6083SPeter Grehan 			vmexit->u.vmx.error = 0;
1264366f6083SPeter Grehan 		} else {
1265366f6083SPeter Grehan 			/*
1266366f6083SPeter Grehan 			 * The exitcode and collateral have been populated.
1267366f6083SPeter Grehan 			 * The VM exit will be processed further in userland.
1268366f6083SPeter Grehan 			 */
1269366f6083SPeter Grehan 		}
1270366f6083SPeter Grehan 	}
1271366f6083SPeter Grehan 	return (handled);
1272366f6083SPeter Grehan }
1273366f6083SPeter Grehan 
1274366f6083SPeter Grehan static int
1275366f6083SPeter Grehan vmx_run(void *arg, int vcpu, register_t rip, struct vm_exit *vmexit)
1276366f6083SPeter Grehan {
1277ad54f374SNeel Natu 	int error, vie, rc, handled, astpending;
1278366f6083SPeter Grehan 	uint32_t exit_reason;
1279366f6083SPeter Grehan 	struct vmx *vmx;
1280366f6083SPeter Grehan 	struct vmxctx *vmxctx;
1281366f6083SPeter Grehan 	struct vmcs *vmcs;
1282366f6083SPeter Grehan 
1283366f6083SPeter Grehan 	vmx = arg;
1284366f6083SPeter Grehan 	vmcs = &vmx->vmcs[vcpu];
1285366f6083SPeter Grehan 	vmxctx = &vmx->ctx[vcpu];
1286ad54f374SNeel Natu 	vmxctx->launched = 0;
1287366f6083SPeter Grehan 
1288366f6083SPeter Grehan 	/*
1289366f6083SPeter Grehan 	 * XXX Can we avoid doing this every time we do a vm run?
1290366f6083SPeter Grehan 	 */
1291366f6083SPeter Grehan 	VMPTRLD(vmcs);
1292366f6083SPeter Grehan 
1293366f6083SPeter Grehan 	/*
1294366f6083SPeter Grehan 	 * XXX
1295366f6083SPeter Grehan 	 * We do this every time because we may setup the virtual machine
1296366f6083SPeter Grehan 	 * from a different process than the one that actually runs it.
1297366f6083SPeter Grehan 	 *
1298366f6083SPeter Grehan 	 * If the life of a virtual machine was spent entirely in the context
1299366f6083SPeter Grehan 	 * of a single process we could do this once in vmcs_set_defaults().
1300366f6083SPeter Grehan 	 */
1301366f6083SPeter Grehan 	if ((error = vmwrite(VMCS_HOST_CR3, rcr3())) != 0)
1302366f6083SPeter Grehan 		panic("vmx_run: error %d writing to VMCS_HOST_CR3", error);
1303366f6083SPeter Grehan 
1304366f6083SPeter Grehan 	if ((error = vmwrite(VMCS_GUEST_RIP, rip)) != 0)
1305366f6083SPeter Grehan 		panic("vmx_run: error %d writing to VMCS_GUEST_RIP", error);
1306366f6083SPeter Grehan 
1307366f6083SPeter Grehan 	if ((error = vmx_set_pcpu_defaults(vmx, vcpu)) != 0)
1308366f6083SPeter Grehan 		panic("vmx_run: error %d setting up pcpu defaults", error);
1309366f6083SPeter Grehan 
1310366f6083SPeter Grehan 	do {
1311366f6083SPeter Grehan 		lapic_timer_tick(vmx->vm, vcpu);
1312366f6083SPeter Grehan 		vmx_inject_interrupts(vmx, vcpu);
1313366f6083SPeter Grehan 		vmx_run_trace(vmx, vcpu);
1314366f6083SPeter Grehan 		rc = vmx_setjmp(vmxctx);
1315366f6083SPeter Grehan #ifdef SETJMP_TRACE
1316366f6083SPeter Grehan 		vmx_setjmp_trace(vmx, vcpu, vmxctx, rc);
1317366f6083SPeter Grehan #endif
1318366f6083SPeter Grehan 		switch (rc) {
1319366f6083SPeter Grehan 		case VMX_RETURN_DIRECT:
1320ad54f374SNeel Natu 			if (vmxctx->launched == 0) {
1321ad54f374SNeel Natu 				vmxctx->launched = 1;
1322366f6083SPeter Grehan 				vmx_launch(vmxctx);
1323366f6083SPeter Grehan 			} else
1324366f6083SPeter Grehan 				vmx_resume(vmxctx);
1325366f6083SPeter Grehan 			panic("vmx_launch/resume should not return");
1326366f6083SPeter Grehan 			break;
1327366f6083SPeter Grehan 		case VMX_RETURN_LONGJMP:
1328366f6083SPeter Grehan 			break;			/* vm exit */
1329366f6083SPeter Grehan 		case VMX_RETURN_VMRESUME:
1330366f6083SPeter Grehan 			vie = vmcs_instruction_error();
1331366f6083SPeter Grehan 			if (vmxctx->launch_error == VM_FAIL_INVALID ||
1332366f6083SPeter Grehan 			    vie != VMRESUME_WITH_NON_LAUNCHED_VMCS) {
1333366f6083SPeter Grehan 				printf("vmresume error %d vmcs inst error %d\n",
1334366f6083SPeter Grehan 					vmxctx->launch_error, vie);
1335366f6083SPeter Grehan 				goto err_exit;
1336366f6083SPeter Grehan 			}
1337366f6083SPeter Grehan 			vmx_launch(vmxctx);	/* try to launch the guest */
1338366f6083SPeter Grehan 			panic("vmx_launch should not return");
1339366f6083SPeter Grehan 			break;
1340366f6083SPeter Grehan 		case VMX_RETURN_VMLAUNCH:
1341366f6083SPeter Grehan 			vie = vmcs_instruction_error();
1342366f6083SPeter Grehan #if 1
1343366f6083SPeter Grehan 			printf("vmlaunch error %d vmcs inst error %d\n",
1344366f6083SPeter Grehan 				vmxctx->launch_error, vie);
1345366f6083SPeter Grehan #endif
1346366f6083SPeter Grehan 			goto err_exit;
1347366f6083SPeter Grehan 		default:
1348366f6083SPeter Grehan 			panic("vmx_setjmp returned %d", rc);
1349366f6083SPeter Grehan 		}
1350366f6083SPeter Grehan 
1351366f6083SPeter Grehan 		/*
1352366f6083SPeter Grehan 		 * XXX locking?
1353366f6083SPeter Grehan 		 * See comments in exception.S about checking for ASTs
1354366f6083SPeter Grehan 		 * atomically while interrupts are disabled. But it is
1355366f6083SPeter Grehan 		 * not clear that they apply in our case.
1356366f6083SPeter Grehan 		 */
1357366f6083SPeter Grehan 		astpending = curthread->td_flags & TDF_ASTPENDING;
1358366f6083SPeter Grehan 
1359366f6083SPeter Grehan 		/* enable interrupts */
1360366f6083SPeter Grehan 		enable_intr();
1361366f6083SPeter Grehan 
1362366f6083SPeter Grehan 		/* collect some basic information for VM exit processing */
1363366f6083SPeter Grehan 		vmexit->rip = rip = vmcs_guest_rip();
1364366f6083SPeter Grehan 		vmexit->inst_length = vmexit_instruction_length();
1365366f6083SPeter Grehan 		vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason();
1366366f6083SPeter Grehan 		vmexit->u.vmx.exit_qualification = vmcs_exit_qualification();
1367366f6083SPeter Grehan 
1368366f6083SPeter Grehan 		handled = vmx_exit_process(vmx, vcpu, vmexit);
1369366f6083SPeter Grehan 
1370366f6083SPeter Grehan 		vmx_exit_trace(vmx, vcpu, rip, exit_reason, handled,
1371366f6083SPeter Grehan 			       astpending);
1372366f6083SPeter Grehan 	} while (handled && !astpending);
1373366f6083SPeter Grehan 
1374366f6083SPeter Grehan 	/*
1375366f6083SPeter Grehan 	 * If a VM exit has been handled then the exitcode must be BOGUS
1376366f6083SPeter Grehan 	 * If a VM exit is not handled then the exitcode must not be BOGUS
1377366f6083SPeter Grehan 	 */
1378366f6083SPeter Grehan 	if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) ||
1379366f6083SPeter Grehan 	    (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) {
1380366f6083SPeter Grehan 		panic("Mismatch between handled (%d) and exitcode (%d)",
1381366f6083SPeter Grehan 		      handled, vmexit->exitcode);
1382366f6083SPeter Grehan 	}
1383366f6083SPeter Grehan 
1384366f6083SPeter Grehan 	VMM_CTR1(vmx->vm, vcpu, "goto userland: exitcode %d",vmexit->exitcode);
1385366f6083SPeter Grehan 
1386366f6083SPeter Grehan 	/*
1387366f6083SPeter Grehan 	 * XXX
1388366f6083SPeter Grehan 	 * We need to do this to ensure that any VMCS state cached by the
1389366f6083SPeter Grehan 	 * processor is flushed to memory. We need to do this in case the
1390366f6083SPeter Grehan 	 * VM moves to a different cpu the next time it runs.
1391366f6083SPeter Grehan 	 *
1392366f6083SPeter Grehan 	 * Can we avoid doing this?
1393366f6083SPeter Grehan 	 */
1394366f6083SPeter Grehan 	VMCLEAR(vmcs);
1395366f6083SPeter Grehan 	return (0);
1396366f6083SPeter Grehan 
1397366f6083SPeter Grehan err_exit:
1398366f6083SPeter Grehan 	vmexit->exitcode = VM_EXITCODE_VMX;
1399366f6083SPeter Grehan 	vmexit->u.vmx.exit_reason = (uint32_t)-1;
1400366f6083SPeter Grehan 	vmexit->u.vmx.exit_qualification = (uint32_t)-1;
1401366f6083SPeter Grehan 	vmexit->u.vmx.error = vie;
1402366f6083SPeter Grehan 	VMCLEAR(vmcs);
1403366f6083SPeter Grehan 	return (ENOEXEC);
1404366f6083SPeter Grehan }
1405366f6083SPeter Grehan 
1406366f6083SPeter Grehan static void
1407366f6083SPeter Grehan vmx_vmcleanup(void *arg)
1408366f6083SPeter Grehan {
1409366f6083SPeter Grehan 	int error;
1410366f6083SPeter Grehan 	struct vmx *vmx = arg;
1411366f6083SPeter Grehan 
1412366f6083SPeter Grehan 	/*
1413366f6083SPeter Grehan 	 * XXXSMP we also need to clear the VMCS active on the other vcpus.
1414366f6083SPeter Grehan 	 */
1415366f6083SPeter Grehan 	error = vmclear(&vmx->vmcs[0]);
1416366f6083SPeter Grehan 	if (error != 0)
1417366f6083SPeter Grehan 		panic("vmx_vmcleanup: vmclear error %d on vcpu 0", error);
1418366f6083SPeter Grehan 
1419366f6083SPeter Grehan 	ept_vmcleanup(vmx);
1420366f6083SPeter Grehan 	free(vmx, M_VMX);
1421366f6083SPeter Grehan 
1422366f6083SPeter Grehan 	return;
1423366f6083SPeter Grehan }
1424366f6083SPeter Grehan 
1425366f6083SPeter Grehan static register_t *
1426366f6083SPeter Grehan vmxctx_regptr(struct vmxctx *vmxctx, int reg)
1427366f6083SPeter Grehan {
1428366f6083SPeter Grehan 
1429366f6083SPeter Grehan 	switch (reg) {
1430366f6083SPeter Grehan 	case VM_REG_GUEST_RAX:
1431366f6083SPeter Grehan 		return (&vmxctx->guest_rax);
1432366f6083SPeter Grehan 	case VM_REG_GUEST_RBX:
1433366f6083SPeter Grehan 		return (&vmxctx->guest_rbx);
1434366f6083SPeter Grehan 	case VM_REG_GUEST_RCX:
1435366f6083SPeter Grehan 		return (&vmxctx->guest_rcx);
1436366f6083SPeter Grehan 	case VM_REG_GUEST_RDX:
1437366f6083SPeter Grehan 		return (&vmxctx->guest_rdx);
1438366f6083SPeter Grehan 	case VM_REG_GUEST_RSI:
1439366f6083SPeter Grehan 		return (&vmxctx->guest_rsi);
1440366f6083SPeter Grehan 	case VM_REG_GUEST_RDI:
1441366f6083SPeter Grehan 		return (&vmxctx->guest_rdi);
1442366f6083SPeter Grehan 	case VM_REG_GUEST_RBP:
1443366f6083SPeter Grehan 		return (&vmxctx->guest_rbp);
1444366f6083SPeter Grehan 	case VM_REG_GUEST_R8:
1445366f6083SPeter Grehan 		return (&vmxctx->guest_r8);
1446366f6083SPeter Grehan 	case VM_REG_GUEST_R9:
1447366f6083SPeter Grehan 		return (&vmxctx->guest_r9);
1448366f6083SPeter Grehan 	case VM_REG_GUEST_R10:
1449366f6083SPeter Grehan 		return (&vmxctx->guest_r10);
1450366f6083SPeter Grehan 	case VM_REG_GUEST_R11:
1451366f6083SPeter Grehan 		return (&vmxctx->guest_r11);
1452366f6083SPeter Grehan 	case VM_REG_GUEST_R12:
1453366f6083SPeter Grehan 		return (&vmxctx->guest_r12);
1454366f6083SPeter Grehan 	case VM_REG_GUEST_R13:
1455366f6083SPeter Grehan 		return (&vmxctx->guest_r13);
1456366f6083SPeter Grehan 	case VM_REG_GUEST_R14:
1457366f6083SPeter Grehan 		return (&vmxctx->guest_r14);
1458366f6083SPeter Grehan 	case VM_REG_GUEST_R15:
1459366f6083SPeter Grehan 		return (&vmxctx->guest_r15);
1460366f6083SPeter Grehan 	default:
1461366f6083SPeter Grehan 		break;
1462366f6083SPeter Grehan 	}
1463366f6083SPeter Grehan 	return (NULL);
1464366f6083SPeter Grehan }
1465366f6083SPeter Grehan 
1466366f6083SPeter Grehan static int
1467366f6083SPeter Grehan vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval)
1468366f6083SPeter Grehan {
1469366f6083SPeter Grehan 	register_t *regp;
1470366f6083SPeter Grehan 
1471366f6083SPeter Grehan 	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
1472366f6083SPeter Grehan 		*retval = *regp;
1473366f6083SPeter Grehan 		return (0);
1474366f6083SPeter Grehan 	} else
1475366f6083SPeter Grehan 		return (EINVAL);
1476366f6083SPeter Grehan }
1477366f6083SPeter Grehan 
1478366f6083SPeter Grehan static int
1479366f6083SPeter Grehan vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val)
1480366f6083SPeter Grehan {
1481366f6083SPeter Grehan 	register_t *regp;
1482366f6083SPeter Grehan 
1483366f6083SPeter Grehan 	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
1484366f6083SPeter Grehan 		*regp = val;
1485366f6083SPeter Grehan 		return (0);
1486366f6083SPeter Grehan 	} else
1487366f6083SPeter Grehan 		return (EINVAL);
1488366f6083SPeter Grehan }
1489366f6083SPeter Grehan 
1490366f6083SPeter Grehan static int
1491366f6083SPeter Grehan vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval)
1492366f6083SPeter Grehan {
1493366f6083SPeter Grehan 	struct vmx *vmx = arg;
1494366f6083SPeter Grehan 
1495366f6083SPeter Grehan 	if (vmxctx_getreg(&vmx->ctx[vcpu], reg, retval) == 0)
1496366f6083SPeter Grehan 		return (0);
1497366f6083SPeter Grehan 
1498366f6083SPeter Grehan 	/*
1499366f6083SPeter Grehan 	 * If the vcpu is running then don't mess with the VMCS.
1500366f6083SPeter Grehan 	 *
1501366f6083SPeter Grehan 	 * vmcs_getreg will VMCLEAR the vmcs when it is done which will cause
1502366f6083SPeter Grehan 	 * the subsequent vmlaunch/vmresume to fail.
1503366f6083SPeter Grehan 	 */
1504366f6083SPeter Grehan 	if (vcpu_is_running(vmx->vm, vcpu, NULL))
1505366f6083SPeter Grehan 		panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu);
1506366f6083SPeter Grehan 
1507366f6083SPeter Grehan 	return (vmcs_getreg(&vmx->vmcs[vcpu], reg, retval));
1508366f6083SPeter Grehan }
1509366f6083SPeter Grehan 
1510366f6083SPeter Grehan static int
1511366f6083SPeter Grehan vmx_setreg(void *arg, int vcpu, int reg, uint64_t val)
1512366f6083SPeter Grehan {
1513366f6083SPeter Grehan 	int error;
1514366f6083SPeter Grehan 	uint64_t ctls;
1515366f6083SPeter Grehan 	struct vmx *vmx = arg;
1516366f6083SPeter Grehan 
1517366f6083SPeter Grehan 	/*
1518366f6083SPeter Grehan 	 * XXX Allow caller to set contents of the guest registers saved in
1519366f6083SPeter Grehan 	 * the 'vmxctx' even though the vcpu might be running. We need this
1520366f6083SPeter Grehan 	 * specifically to support the rdmsr emulation that will set the
1521366f6083SPeter Grehan 	 * %eax and %edx registers during vm exit processing.
1522366f6083SPeter Grehan 	 */
1523366f6083SPeter Grehan 	if (vmxctx_setreg(&vmx->ctx[vcpu], reg, val) == 0)
1524366f6083SPeter Grehan 		return (0);
1525366f6083SPeter Grehan 
1526366f6083SPeter Grehan 	/*
1527366f6083SPeter Grehan 	 * If the vcpu is running then don't mess with the VMCS.
1528366f6083SPeter Grehan 	 *
1529366f6083SPeter Grehan 	 * vmcs_setreg will VMCLEAR the vmcs when it is done which will cause
1530366f6083SPeter Grehan 	 * the subsequent vmlaunch/vmresume to fail.
1531366f6083SPeter Grehan 	 */
1532366f6083SPeter Grehan 	if (vcpu_is_running(vmx->vm, vcpu, NULL))
1533366f6083SPeter Grehan 		panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu);
1534366f6083SPeter Grehan 
1535366f6083SPeter Grehan 	error = vmcs_setreg(&vmx->vmcs[vcpu], reg, val);
1536366f6083SPeter Grehan 
1537366f6083SPeter Grehan 	if (error == 0) {
1538366f6083SPeter Grehan 		/*
1539366f6083SPeter Grehan 		 * If the "load EFER" VM-entry control is 1 then the
1540366f6083SPeter Grehan 		 * value of EFER.LMA must be identical to "IA-32e mode guest"
1541366f6083SPeter Grehan 		 * bit in the VM-entry control.
1542366f6083SPeter Grehan 		 */
1543366f6083SPeter Grehan 		if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 &&
1544366f6083SPeter Grehan 		    (reg == VM_REG_GUEST_EFER)) {
1545366f6083SPeter Grehan 			vmcs_getreg(&vmx->vmcs[vcpu],
1546366f6083SPeter Grehan 				    VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls);
1547366f6083SPeter Grehan 			if (val & EFER_LMA)
1548366f6083SPeter Grehan 				ctls |= VM_ENTRY_GUEST_LMA;
1549366f6083SPeter Grehan 			else
1550366f6083SPeter Grehan 				ctls &= ~VM_ENTRY_GUEST_LMA;
1551366f6083SPeter Grehan 			vmcs_setreg(&vmx->vmcs[vcpu],
1552366f6083SPeter Grehan 				    VMCS_IDENT(VMCS_ENTRY_CTLS), ctls);
1553366f6083SPeter Grehan 		}
1554366f6083SPeter Grehan 	}
1555366f6083SPeter Grehan 
1556366f6083SPeter Grehan 	return (error);
1557366f6083SPeter Grehan }
1558366f6083SPeter Grehan 
1559366f6083SPeter Grehan static int
1560366f6083SPeter Grehan vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
1561366f6083SPeter Grehan {
1562366f6083SPeter Grehan 	struct vmx *vmx = arg;
1563366f6083SPeter Grehan 
1564366f6083SPeter Grehan 	return (vmcs_getdesc(&vmx->vmcs[vcpu], reg, desc));
1565366f6083SPeter Grehan }
1566366f6083SPeter Grehan 
1567366f6083SPeter Grehan static int
1568366f6083SPeter Grehan vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
1569366f6083SPeter Grehan {
1570366f6083SPeter Grehan 	struct vmx *vmx = arg;
1571366f6083SPeter Grehan 
1572366f6083SPeter Grehan 	return (vmcs_setdesc(&vmx->vmcs[vcpu], reg, desc));
1573366f6083SPeter Grehan }
1574366f6083SPeter Grehan 
1575366f6083SPeter Grehan static int
1576366f6083SPeter Grehan vmx_inject(void *arg, int vcpu, int type, int vector, uint32_t code,
1577366f6083SPeter Grehan 	   int code_valid)
1578366f6083SPeter Grehan {
1579366f6083SPeter Grehan 	int error;
1580366f6083SPeter Grehan 	uint32_t info;
1581366f6083SPeter Grehan 	struct vmx *vmx = arg;
1582366f6083SPeter Grehan 	struct vmcs *vmcs = &vmx->vmcs[vcpu];
1583366f6083SPeter Grehan 
1584366f6083SPeter Grehan 	static uint32_t type_map[VM_EVENT_MAX] = {
1585366f6083SPeter Grehan 		0x1,		/* VM_EVENT_NONE */
1586366f6083SPeter Grehan 		0x0,		/* VM_HW_INTR */
1587366f6083SPeter Grehan 		0x2,		/* VM_NMI */
1588366f6083SPeter Grehan 		0x3,		/* VM_HW_EXCEPTION */
1589366f6083SPeter Grehan 		0x4,		/* VM_SW_INTR */
1590366f6083SPeter Grehan 		0x5,		/* VM_PRIV_SW_EXCEPTION */
1591366f6083SPeter Grehan 		0x6,		/* VM_SW_EXCEPTION */
1592366f6083SPeter Grehan 	};
1593366f6083SPeter Grehan 
1594366f6083SPeter Grehan 	info = vector | (type_map[type] << 8) | (code_valid ? 1 << 11 : 0);
1595366f6083SPeter Grehan 	info |= VMCS_INTERRUPTION_INFO_VALID;
1596366f6083SPeter Grehan 	error = vmcs_setreg(vmcs, VMCS_IDENT(VMCS_ENTRY_INTR_INFO), info);
1597366f6083SPeter Grehan 	if (error != 0)
1598366f6083SPeter Grehan 		return (error);
1599366f6083SPeter Grehan 
1600366f6083SPeter Grehan 	if (code_valid) {
1601366f6083SPeter Grehan 		error = vmcs_setreg(vmcs,
1602366f6083SPeter Grehan 				    VMCS_IDENT(VMCS_ENTRY_EXCEPTION_ERROR),
1603366f6083SPeter Grehan 				    code);
1604366f6083SPeter Grehan 	}
1605366f6083SPeter Grehan 	return (error);
1606366f6083SPeter Grehan }
1607366f6083SPeter Grehan 
1608366f6083SPeter Grehan static int
1609366f6083SPeter Grehan vmx_nmi(void *arg, int vcpu)
1610366f6083SPeter Grehan {
1611366f6083SPeter Grehan 	struct vmx *vmx = arg;
1612366f6083SPeter Grehan 
1613366f6083SPeter Grehan 	atomic_set_int(&vmx->state[vcpu].request_nmi, 1);
1614366f6083SPeter Grehan 
1615366f6083SPeter Grehan 	return (0);
1616366f6083SPeter Grehan }
1617366f6083SPeter Grehan 
1618366f6083SPeter Grehan static int
1619366f6083SPeter Grehan vmx_getcap(void *arg, int vcpu, int type, int *retval)
1620366f6083SPeter Grehan {
1621366f6083SPeter Grehan 	struct vmx *vmx = arg;
1622366f6083SPeter Grehan 	int vcap;
1623366f6083SPeter Grehan 	int ret;
1624366f6083SPeter Grehan 
1625366f6083SPeter Grehan 	ret = ENOENT;
1626366f6083SPeter Grehan 
1627366f6083SPeter Grehan 	vcap = vmx->cap[vcpu].set;
1628366f6083SPeter Grehan 
1629366f6083SPeter Grehan 	switch (type) {
1630366f6083SPeter Grehan 	case VM_CAP_HALT_EXIT:
1631366f6083SPeter Grehan 		if (cap_halt_exit)
1632366f6083SPeter Grehan 			ret = 0;
1633366f6083SPeter Grehan 		break;
1634366f6083SPeter Grehan 	case VM_CAP_PAUSE_EXIT:
1635366f6083SPeter Grehan 		if (cap_pause_exit)
1636366f6083SPeter Grehan 			ret = 0;
1637366f6083SPeter Grehan 		break;
1638366f6083SPeter Grehan 	case VM_CAP_MTRAP_EXIT:
1639366f6083SPeter Grehan 		if (cap_monitor_trap)
1640366f6083SPeter Grehan 			ret = 0;
1641366f6083SPeter Grehan 		break;
1642366f6083SPeter Grehan 	case VM_CAP_UNRESTRICTED_GUEST:
1643366f6083SPeter Grehan 		if (cap_unrestricted_guest)
1644366f6083SPeter Grehan 			ret = 0;
1645366f6083SPeter Grehan 		break;
1646366f6083SPeter Grehan 	default:
1647366f6083SPeter Grehan 		break;
1648366f6083SPeter Grehan 	}
1649366f6083SPeter Grehan 
1650366f6083SPeter Grehan 	if (ret == 0)
1651366f6083SPeter Grehan 		*retval = (vcap & (1 << type)) ? 1 : 0;
1652366f6083SPeter Grehan 
1653366f6083SPeter Grehan 	return (ret);
1654366f6083SPeter Grehan }
1655366f6083SPeter Grehan 
1656366f6083SPeter Grehan static int
1657366f6083SPeter Grehan vmx_setcap(void *arg, int vcpu, int type, int val)
1658366f6083SPeter Grehan {
1659366f6083SPeter Grehan 	struct vmx *vmx = arg;
1660366f6083SPeter Grehan 	struct vmcs *vmcs = &vmx->vmcs[vcpu];
1661366f6083SPeter Grehan 	uint32_t baseval;
1662366f6083SPeter Grehan 	uint32_t *pptr;
1663366f6083SPeter Grehan 	int error;
1664366f6083SPeter Grehan 	int flag;
1665366f6083SPeter Grehan 	int reg;
1666366f6083SPeter Grehan 	int retval;
1667366f6083SPeter Grehan 
1668366f6083SPeter Grehan 	retval = ENOENT;
1669366f6083SPeter Grehan 	pptr = NULL;
1670366f6083SPeter Grehan 
1671366f6083SPeter Grehan 	switch (type) {
1672366f6083SPeter Grehan 	case VM_CAP_HALT_EXIT:
1673366f6083SPeter Grehan 		if (cap_halt_exit) {
1674366f6083SPeter Grehan 			retval = 0;
1675366f6083SPeter Grehan 			pptr = &vmx->cap[vcpu].proc_ctls;
1676366f6083SPeter Grehan 			baseval = *pptr;
1677366f6083SPeter Grehan 			flag = PROCBASED_HLT_EXITING;
1678366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
1679366f6083SPeter Grehan 		}
1680366f6083SPeter Grehan 		break;
1681366f6083SPeter Grehan 	case VM_CAP_MTRAP_EXIT:
1682366f6083SPeter Grehan 		if (cap_monitor_trap) {
1683366f6083SPeter Grehan 			retval = 0;
1684366f6083SPeter Grehan 			pptr = &vmx->cap[vcpu].proc_ctls;
1685366f6083SPeter Grehan 			baseval = *pptr;
1686366f6083SPeter Grehan 			flag = PROCBASED_MTF;
1687366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
1688366f6083SPeter Grehan 		}
1689366f6083SPeter Grehan 		break;
1690366f6083SPeter Grehan 	case VM_CAP_PAUSE_EXIT:
1691366f6083SPeter Grehan 		if (cap_pause_exit) {
1692366f6083SPeter Grehan 			retval = 0;
1693366f6083SPeter Grehan 			pptr = &vmx->cap[vcpu].proc_ctls;
1694366f6083SPeter Grehan 			baseval = *pptr;
1695366f6083SPeter Grehan 			flag = PROCBASED_PAUSE_EXITING;
1696366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
1697366f6083SPeter Grehan 		}
1698366f6083SPeter Grehan 		break;
1699366f6083SPeter Grehan 	case VM_CAP_UNRESTRICTED_GUEST:
1700366f6083SPeter Grehan 		if (cap_unrestricted_guest) {
1701366f6083SPeter Grehan 			retval = 0;
1702366f6083SPeter Grehan 			baseval = procbased_ctls2;
1703366f6083SPeter Grehan 			flag = PROCBASED2_UNRESTRICTED_GUEST;
1704366f6083SPeter Grehan 			reg = VMCS_SEC_PROC_BASED_CTLS;
1705366f6083SPeter Grehan 		}
1706366f6083SPeter Grehan 		break;
1707366f6083SPeter Grehan 	default:
1708366f6083SPeter Grehan 		break;
1709366f6083SPeter Grehan 	}
1710366f6083SPeter Grehan 
1711366f6083SPeter Grehan 	if (retval == 0) {
1712366f6083SPeter Grehan 		if (val) {
1713366f6083SPeter Grehan 			baseval |= flag;
1714366f6083SPeter Grehan 		} else {
1715366f6083SPeter Grehan 			baseval &= ~flag;
1716366f6083SPeter Grehan 		}
1717366f6083SPeter Grehan 		VMPTRLD(vmcs);
1718366f6083SPeter Grehan 		error = vmwrite(reg, baseval);
1719366f6083SPeter Grehan 		VMCLEAR(vmcs);
1720366f6083SPeter Grehan 
1721366f6083SPeter Grehan 		if (error) {
1722366f6083SPeter Grehan 			retval = error;
1723366f6083SPeter Grehan 		} else {
1724366f6083SPeter Grehan 			/*
1725366f6083SPeter Grehan 			 * Update optional stored flags, and record
1726366f6083SPeter Grehan 			 * setting
1727366f6083SPeter Grehan 			 */
1728366f6083SPeter Grehan 			if (pptr != NULL) {
1729366f6083SPeter Grehan 				*pptr = baseval;
1730366f6083SPeter Grehan 			}
1731366f6083SPeter Grehan 
1732366f6083SPeter Grehan 			if (val) {
1733366f6083SPeter Grehan 				vmx->cap[vcpu].set |= (1 << type);
1734366f6083SPeter Grehan 			} else {
1735366f6083SPeter Grehan 				vmx->cap[vcpu].set &= ~(1 << type);
1736366f6083SPeter Grehan 			}
1737366f6083SPeter Grehan 		}
1738366f6083SPeter Grehan 	}
1739366f6083SPeter Grehan 
1740366f6083SPeter Grehan         return (retval);
1741366f6083SPeter Grehan }
1742366f6083SPeter Grehan 
1743366f6083SPeter Grehan struct vmm_ops vmm_ops_intel = {
1744366f6083SPeter Grehan 	vmx_init,
1745366f6083SPeter Grehan 	vmx_cleanup,
1746366f6083SPeter Grehan 	vmx_vminit,
1747366f6083SPeter Grehan 	vmx_run,
1748366f6083SPeter Grehan 	vmx_vmcleanup,
1749366f6083SPeter Grehan 	ept_vmmmap,
1750366f6083SPeter Grehan 	vmx_getreg,
1751366f6083SPeter Grehan 	vmx_setreg,
1752366f6083SPeter Grehan 	vmx_getdesc,
1753366f6083SPeter Grehan 	vmx_setdesc,
1754366f6083SPeter Grehan 	vmx_inject,
1755366f6083SPeter Grehan 	vmx_nmi,
1756366f6083SPeter Grehan 	vmx_getcap,
1757366f6083SPeter Grehan 	vmx_setcap
1758366f6083SPeter Grehan };
1759