xref: /freebsd/sys/amd64/vmm/intel/vmx.c (revision 159dd56f947fb0d05692ca13ca4fcc389fd52dca)
1366f6083SPeter Grehan /*-
2366f6083SPeter Grehan  * Copyright (c) 2011 NetApp, Inc.
3366f6083SPeter Grehan  * All rights reserved.
4366f6083SPeter Grehan  *
5366f6083SPeter Grehan  * Redistribution and use in source and binary forms, with or without
6366f6083SPeter Grehan  * modification, are permitted provided that the following conditions
7366f6083SPeter Grehan  * are met:
8366f6083SPeter Grehan  * 1. Redistributions of source code must retain the above copyright
9366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer.
10366f6083SPeter Grehan  * 2. Redistributions in binary form must reproduce the above copyright
11366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer in the
12366f6083SPeter Grehan  *    documentation and/or other materials provided with the distribution.
13366f6083SPeter Grehan  *
14366f6083SPeter Grehan  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
15366f6083SPeter Grehan  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16366f6083SPeter Grehan  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17366f6083SPeter Grehan  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
18366f6083SPeter Grehan  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19366f6083SPeter Grehan  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20366f6083SPeter Grehan  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21366f6083SPeter Grehan  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22366f6083SPeter Grehan  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23366f6083SPeter Grehan  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24366f6083SPeter Grehan  * SUCH DAMAGE.
25366f6083SPeter Grehan  *
26366f6083SPeter Grehan  * $FreeBSD$
27366f6083SPeter Grehan  */
28366f6083SPeter Grehan 
29366f6083SPeter Grehan #include <sys/cdefs.h>
30366f6083SPeter Grehan __FBSDID("$FreeBSD$");
31366f6083SPeter Grehan 
32366f6083SPeter Grehan #include <sys/param.h>
33366f6083SPeter Grehan #include <sys/systm.h>
34366f6083SPeter Grehan #include <sys/smp.h>
35366f6083SPeter Grehan #include <sys/kernel.h>
36366f6083SPeter Grehan #include <sys/malloc.h>
37366f6083SPeter Grehan #include <sys/pcpu.h>
38366f6083SPeter Grehan #include <sys/proc.h>
393565b59eSNeel Natu #include <sys/sysctl.h>
40366f6083SPeter Grehan 
41366f6083SPeter Grehan #include <vm/vm.h>
42366f6083SPeter Grehan #include <vm/pmap.h>
43366f6083SPeter Grehan 
44366f6083SPeter Grehan #include <machine/psl.h>
45366f6083SPeter Grehan #include <machine/cpufunc.h>
468b287612SJohn Baldwin #include <machine/md_var.h>
47366f6083SPeter Grehan #include <machine/segments.h>
48176666c2SNeel Natu #include <machine/smp.h>
49608f97c3SPeter Grehan #include <machine/specialreg.h>
50366f6083SPeter Grehan #include <machine/vmparam.h>
51366f6083SPeter Grehan 
52366f6083SPeter Grehan #include <machine/vmm.h>
53b01c2033SNeel Natu #include "vmm_host.h"
54176666c2SNeel Natu #include "vmm_ipi.h"
55366f6083SPeter Grehan #include "vmm_msr.h"
56366f6083SPeter Grehan #include "vmm_ktr.h"
57366f6083SPeter Grehan #include "vmm_stat.h"
58de5ea6b6SNeel Natu #include "vlapic.h"
59de5ea6b6SNeel Natu #include "vlapic_priv.h"
60366f6083SPeter Grehan 
61366f6083SPeter Grehan #include "vmx_msr.h"
62366f6083SPeter Grehan #include "ept.h"
63366f6083SPeter Grehan #include "vmx_cpufunc.h"
64366f6083SPeter Grehan #include "vmx.h"
65366f6083SPeter Grehan #include "x86.h"
66366f6083SPeter Grehan #include "vmx_controls.h"
67366f6083SPeter Grehan 
68366f6083SPeter Grehan #define	PINBASED_CTLS_ONE_SETTING					\
69366f6083SPeter Grehan 	(PINBASED_EXTINT_EXITING	|				\
70366f6083SPeter Grehan 	 PINBASED_NMI_EXITING		|				\
71366f6083SPeter Grehan 	 PINBASED_VIRTUAL_NMI)
72366f6083SPeter Grehan #define	PINBASED_CTLS_ZERO_SETTING	0
73366f6083SPeter Grehan 
74366f6083SPeter Grehan #define PROCBASED_CTLS_WINDOW_SETTING					\
75366f6083SPeter Grehan 	(PROCBASED_INT_WINDOW_EXITING	|				\
76366f6083SPeter Grehan 	 PROCBASED_NMI_WINDOW_EXITING)
77366f6083SPeter Grehan 
78366f6083SPeter Grehan #define	PROCBASED_CTLS_ONE_SETTING 					\
79366f6083SPeter Grehan 	(PROCBASED_SECONDARY_CONTROLS	|				\
80366f6083SPeter Grehan 	 PROCBASED_IO_EXITING		|				\
81366f6083SPeter Grehan 	 PROCBASED_MSR_BITMAPS		|				\
82366f6083SPeter Grehan 	 PROCBASED_CTLS_WINDOW_SETTING)
83366f6083SPeter Grehan #define	PROCBASED_CTLS_ZERO_SETTING	\
84366f6083SPeter Grehan 	(PROCBASED_CR3_LOAD_EXITING |	\
85366f6083SPeter Grehan 	PROCBASED_CR3_STORE_EXITING |	\
86366f6083SPeter Grehan 	PROCBASED_IO_BITMAPS)
87366f6083SPeter Grehan 
88366f6083SPeter Grehan #define	PROCBASED_CTLS2_ONE_SETTING	PROCBASED2_ENABLE_EPT
89366f6083SPeter Grehan #define	PROCBASED_CTLS2_ZERO_SETTING	0
90366f6083SPeter Grehan 
91608f97c3SPeter Grehan #define VM_EXIT_CTLS_ONE_SETTING_NO_PAT					\
92366f6083SPeter Grehan 	(VM_EXIT_HOST_LMA			|			\
93366f6083SPeter Grehan 	VM_EXIT_SAVE_EFER			|			\
94366f6083SPeter Grehan 	VM_EXIT_LOAD_EFER)
95608f97c3SPeter Grehan 
96608f97c3SPeter Grehan #define	VM_EXIT_CTLS_ONE_SETTING					\
97608f97c3SPeter Grehan 	(VM_EXIT_CTLS_ONE_SETTING_NO_PAT       	|			\
98f7d47425SNeel Natu 	VM_EXIT_ACKNOWLEDGE_INTERRUPT		|			\
99608f97c3SPeter Grehan 	VM_EXIT_SAVE_PAT			|			\
100608f97c3SPeter Grehan 	VM_EXIT_LOAD_PAT)
101366f6083SPeter Grehan #define	VM_EXIT_CTLS_ZERO_SETTING	VM_EXIT_SAVE_DEBUG_CONTROLS
102366f6083SPeter Grehan 
103608f97c3SPeter Grehan #define	VM_ENTRY_CTLS_ONE_SETTING_NO_PAT	VM_ENTRY_LOAD_EFER
104608f97c3SPeter Grehan 
105366f6083SPeter Grehan #define	VM_ENTRY_CTLS_ONE_SETTING					\
106608f97c3SPeter Grehan 	(VM_ENTRY_CTLS_ONE_SETTING_NO_PAT     	|			\
107608f97c3SPeter Grehan 	VM_ENTRY_LOAD_PAT)
108366f6083SPeter Grehan #define	VM_ENTRY_CTLS_ZERO_SETTING					\
109366f6083SPeter Grehan 	(VM_ENTRY_LOAD_DEBUG_CONTROLS		|			\
110366f6083SPeter Grehan 	VM_ENTRY_INTO_SMM			|			\
111366f6083SPeter Grehan 	VM_ENTRY_DEACTIVATE_DUAL_MONITOR)
112366f6083SPeter Grehan 
113366f6083SPeter Grehan #define	guest_msr_rw(vmx, msr) \
114366f6083SPeter Grehan 	msr_bitmap_change_access((vmx)->msr_bitmap, (msr), MSR_BITMAP_ACCESS_RW)
115366f6083SPeter Grehan 
116*159dd56fSNeel Natu #define	guest_msr_ro(vmx, msr) \
117*159dd56fSNeel Natu     msr_bitmap_change_access((vmx)->msr_bitmap, (msr), MSR_BITMAP_ACCESS_READ)
118*159dd56fSNeel Natu 
119366f6083SPeter Grehan #define	HANDLED		1
120366f6083SPeter Grehan #define	UNHANDLED	0
121366f6083SPeter Grehan 
122de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VMX, "vmx", "vmx");
123de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
124366f6083SPeter Grehan 
1253565b59eSNeel Natu SYSCTL_DECL(_hw_vmm);
1263565b59eSNeel Natu SYSCTL_NODE(_hw_vmm, OID_AUTO, vmx, CTLFLAG_RW, NULL, NULL);
1273565b59eSNeel Natu 
128b3996dd4SJohn Baldwin int vmxon_enabled[MAXCPU];
129366f6083SPeter Grehan static char vmxon_region[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE);
130366f6083SPeter Grehan 
131366f6083SPeter Grehan static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2;
132366f6083SPeter Grehan static uint32_t exit_ctls, entry_ctls;
133366f6083SPeter Grehan 
134366f6083SPeter Grehan static uint64_t cr0_ones_mask, cr0_zeros_mask;
1353565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_ones_mask, CTLFLAG_RD,
1363565b59eSNeel Natu 	     &cr0_ones_mask, 0, NULL);
1373565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_zeros_mask, CTLFLAG_RD,
1383565b59eSNeel Natu 	     &cr0_zeros_mask, 0, NULL);
1393565b59eSNeel Natu 
140366f6083SPeter Grehan static uint64_t cr4_ones_mask, cr4_zeros_mask;
1413565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_ones_mask, CTLFLAG_RD,
1423565b59eSNeel Natu 	     &cr4_ones_mask, 0, NULL);
1433565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_zeros_mask, CTLFLAG_RD,
1443565b59eSNeel Natu 	     &cr4_zeros_mask, 0, NULL);
145366f6083SPeter Grehan 
146608f97c3SPeter Grehan static int vmx_no_patmsr;
147608f97c3SPeter Grehan 
1483565b59eSNeel Natu static int vmx_initialized;
1493565b59eSNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, initialized, CTLFLAG_RD,
1503565b59eSNeel Natu 	   &vmx_initialized, 0, "Intel VMX initialized");
1513565b59eSNeel Natu 
152366f6083SPeter Grehan /*
153366f6083SPeter Grehan  * Optional capabilities
154366f6083SPeter Grehan  */
155366f6083SPeter Grehan static int cap_halt_exit;
156366f6083SPeter Grehan static int cap_pause_exit;
157366f6083SPeter Grehan static int cap_unrestricted_guest;
158366f6083SPeter Grehan static int cap_monitor_trap;
15949cc03daSNeel Natu static int cap_invpcid;
160366f6083SPeter Grehan 
16188c4b8d1SNeel Natu static int virtual_interrupt_delivery;
16288c4b8d1SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, virtual_interrupt_delivery, CTLFLAG_RD,
16388c4b8d1SNeel Natu     &virtual_interrupt_delivery, 0, "APICv virtual interrupt delivery support");
16488c4b8d1SNeel Natu 
165176666c2SNeel Natu static int posted_interrupts;
166176666c2SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupts, CTLFLAG_RD,
167176666c2SNeel Natu     &posted_interrupts, 0, "APICv posted interrupt support");
168176666c2SNeel Natu 
169176666c2SNeel Natu static int pirvec;
170176666c2SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupt_vector, CTLFLAG_RD,
171176666c2SNeel Natu     &pirvec, 0, "APICv posted interrupt vector");
172176666c2SNeel Natu 
17345e51299SNeel Natu static struct unrhdr *vpid_unr;
17445e51299SNeel Natu static u_int vpid_alloc_failed;
17545e51299SNeel Natu SYSCTL_UINT(_hw_vmm_vmx, OID_AUTO, vpid_alloc_failed, CTLFLAG_RD,
17645e51299SNeel Natu 	    &vpid_alloc_failed, 0, NULL);
17745e51299SNeel Natu 
17888c4b8d1SNeel Natu /*
17988c4b8d1SNeel Natu  * Use the last page below 4GB as the APIC access address. This address is
18088c4b8d1SNeel Natu  * occupied by the boot firmware so it is guaranteed that it will not conflict
18188c4b8d1SNeel Natu  * with a page in system memory.
18288c4b8d1SNeel Natu  */
18388c4b8d1SNeel Natu #define	APIC_ACCESS_ADDRESS	0xFFFFF000
18488c4b8d1SNeel Natu 
18588c4b8d1SNeel Natu static void vmx_inject_pir(struct vlapic *vlapic);
18688c4b8d1SNeel Natu 
187366f6083SPeter Grehan #ifdef KTR
188366f6083SPeter Grehan static const char *
189366f6083SPeter Grehan exit_reason_to_str(int reason)
190366f6083SPeter Grehan {
191366f6083SPeter Grehan 	static char reasonbuf[32];
192366f6083SPeter Grehan 
193366f6083SPeter Grehan 	switch (reason) {
194366f6083SPeter Grehan 	case EXIT_REASON_EXCEPTION:
195366f6083SPeter Grehan 		return "exception";
196366f6083SPeter Grehan 	case EXIT_REASON_EXT_INTR:
197366f6083SPeter Grehan 		return "extint";
198366f6083SPeter Grehan 	case EXIT_REASON_TRIPLE_FAULT:
199366f6083SPeter Grehan 		return "triplefault";
200366f6083SPeter Grehan 	case EXIT_REASON_INIT:
201366f6083SPeter Grehan 		return "init";
202366f6083SPeter Grehan 	case EXIT_REASON_SIPI:
203366f6083SPeter Grehan 		return "sipi";
204366f6083SPeter Grehan 	case EXIT_REASON_IO_SMI:
205366f6083SPeter Grehan 		return "iosmi";
206366f6083SPeter Grehan 	case EXIT_REASON_SMI:
207366f6083SPeter Grehan 		return "smi";
208366f6083SPeter Grehan 	case EXIT_REASON_INTR_WINDOW:
209366f6083SPeter Grehan 		return "intrwindow";
210366f6083SPeter Grehan 	case EXIT_REASON_NMI_WINDOW:
211366f6083SPeter Grehan 		return "nmiwindow";
212366f6083SPeter Grehan 	case EXIT_REASON_TASK_SWITCH:
213366f6083SPeter Grehan 		return "taskswitch";
214366f6083SPeter Grehan 	case EXIT_REASON_CPUID:
215366f6083SPeter Grehan 		return "cpuid";
216366f6083SPeter Grehan 	case EXIT_REASON_GETSEC:
217366f6083SPeter Grehan 		return "getsec";
218366f6083SPeter Grehan 	case EXIT_REASON_HLT:
219366f6083SPeter Grehan 		return "hlt";
220366f6083SPeter Grehan 	case EXIT_REASON_INVD:
221366f6083SPeter Grehan 		return "invd";
222366f6083SPeter Grehan 	case EXIT_REASON_INVLPG:
223366f6083SPeter Grehan 		return "invlpg";
224366f6083SPeter Grehan 	case EXIT_REASON_RDPMC:
225366f6083SPeter Grehan 		return "rdpmc";
226366f6083SPeter Grehan 	case EXIT_REASON_RDTSC:
227366f6083SPeter Grehan 		return "rdtsc";
228366f6083SPeter Grehan 	case EXIT_REASON_RSM:
229366f6083SPeter Grehan 		return "rsm";
230366f6083SPeter Grehan 	case EXIT_REASON_VMCALL:
231366f6083SPeter Grehan 		return "vmcall";
232366f6083SPeter Grehan 	case EXIT_REASON_VMCLEAR:
233366f6083SPeter Grehan 		return "vmclear";
234366f6083SPeter Grehan 	case EXIT_REASON_VMLAUNCH:
235366f6083SPeter Grehan 		return "vmlaunch";
236366f6083SPeter Grehan 	case EXIT_REASON_VMPTRLD:
237366f6083SPeter Grehan 		return "vmptrld";
238366f6083SPeter Grehan 	case EXIT_REASON_VMPTRST:
239366f6083SPeter Grehan 		return "vmptrst";
240366f6083SPeter Grehan 	case EXIT_REASON_VMREAD:
241366f6083SPeter Grehan 		return "vmread";
242366f6083SPeter Grehan 	case EXIT_REASON_VMRESUME:
243366f6083SPeter Grehan 		return "vmresume";
244366f6083SPeter Grehan 	case EXIT_REASON_VMWRITE:
245366f6083SPeter Grehan 		return "vmwrite";
246366f6083SPeter Grehan 	case EXIT_REASON_VMXOFF:
247366f6083SPeter Grehan 		return "vmxoff";
248366f6083SPeter Grehan 	case EXIT_REASON_VMXON:
249366f6083SPeter Grehan 		return "vmxon";
250366f6083SPeter Grehan 	case EXIT_REASON_CR_ACCESS:
251366f6083SPeter Grehan 		return "craccess";
252366f6083SPeter Grehan 	case EXIT_REASON_DR_ACCESS:
253366f6083SPeter Grehan 		return "draccess";
254366f6083SPeter Grehan 	case EXIT_REASON_INOUT:
255366f6083SPeter Grehan 		return "inout";
256366f6083SPeter Grehan 	case EXIT_REASON_RDMSR:
257366f6083SPeter Grehan 		return "rdmsr";
258366f6083SPeter Grehan 	case EXIT_REASON_WRMSR:
259366f6083SPeter Grehan 		return "wrmsr";
260366f6083SPeter Grehan 	case EXIT_REASON_INVAL_VMCS:
261366f6083SPeter Grehan 		return "invalvmcs";
262366f6083SPeter Grehan 	case EXIT_REASON_INVAL_MSR:
263366f6083SPeter Grehan 		return "invalmsr";
264366f6083SPeter Grehan 	case EXIT_REASON_MWAIT:
265366f6083SPeter Grehan 		return "mwait";
266366f6083SPeter Grehan 	case EXIT_REASON_MTF:
267366f6083SPeter Grehan 		return "mtf";
268366f6083SPeter Grehan 	case EXIT_REASON_MONITOR:
269366f6083SPeter Grehan 		return "monitor";
270366f6083SPeter Grehan 	case EXIT_REASON_PAUSE:
271366f6083SPeter Grehan 		return "pause";
272366f6083SPeter Grehan 	case EXIT_REASON_MCE:
273366f6083SPeter Grehan 		return "mce";
274366f6083SPeter Grehan 	case EXIT_REASON_TPR:
275366f6083SPeter Grehan 		return "tpr";
27688c4b8d1SNeel Natu 	case EXIT_REASON_APIC_ACCESS:
27788c4b8d1SNeel Natu 		return "apic-access";
278366f6083SPeter Grehan 	case EXIT_REASON_GDTR_IDTR:
279366f6083SPeter Grehan 		return "gdtridtr";
280366f6083SPeter Grehan 	case EXIT_REASON_LDTR_TR:
281366f6083SPeter Grehan 		return "ldtrtr";
282366f6083SPeter Grehan 	case EXIT_REASON_EPT_FAULT:
283366f6083SPeter Grehan 		return "eptfault";
284366f6083SPeter Grehan 	case EXIT_REASON_EPT_MISCONFIG:
285366f6083SPeter Grehan 		return "eptmisconfig";
286366f6083SPeter Grehan 	case EXIT_REASON_INVEPT:
287366f6083SPeter Grehan 		return "invept";
288366f6083SPeter Grehan 	case EXIT_REASON_RDTSCP:
289366f6083SPeter Grehan 		return "rdtscp";
290366f6083SPeter Grehan 	case EXIT_REASON_VMX_PREEMPT:
291366f6083SPeter Grehan 		return "vmxpreempt";
292366f6083SPeter Grehan 	case EXIT_REASON_INVVPID:
293366f6083SPeter Grehan 		return "invvpid";
294366f6083SPeter Grehan 	case EXIT_REASON_WBINVD:
295366f6083SPeter Grehan 		return "wbinvd";
296366f6083SPeter Grehan 	case EXIT_REASON_XSETBV:
297366f6083SPeter Grehan 		return "xsetbv";
29888c4b8d1SNeel Natu 	case EXIT_REASON_APIC_WRITE:
29988c4b8d1SNeel Natu 		return "apic-write";
300366f6083SPeter Grehan 	default:
301366f6083SPeter Grehan 		snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason);
302366f6083SPeter Grehan 		return (reasonbuf);
303366f6083SPeter Grehan 	}
304366f6083SPeter Grehan }
305366f6083SPeter Grehan #endif	/* KTR */
306366f6083SPeter Grehan 
307*159dd56fSNeel Natu static int
308*159dd56fSNeel Natu vmx_allow_x2apic_msrs(struct vmx *vmx)
309*159dd56fSNeel Natu {
310*159dd56fSNeel Natu 	int i, error;
311*159dd56fSNeel Natu 
312*159dd56fSNeel Natu 	error = 0;
313*159dd56fSNeel Natu 
314*159dd56fSNeel Natu 	/*
315*159dd56fSNeel Natu 	 * Allow readonly access to the following x2APIC MSRs from the guest.
316*159dd56fSNeel Natu 	 */
317*159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ID);
318*159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_VERSION);
319*159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LDR);
320*159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_SVR);
321*159dd56fSNeel Natu 
322*159dd56fSNeel Natu 	for (i = 0; i < 8; i++)
323*159dd56fSNeel Natu 		error += guest_msr_ro(vmx, MSR_APIC_ISR0 + i);
324*159dd56fSNeel Natu 
325*159dd56fSNeel Natu 	for (i = 0; i < 8; i++)
326*159dd56fSNeel Natu 		error += guest_msr_ro(vmx, MSR_APIC_TMR0 + i);
327*159dd56fSNeel Natu 
328*159dd56fSNeel Natu 	for (i = 0; i < 8; i++)
329*159dd56fSNeel Natu 		error += guest_msr_ro(vmx, MSR_APIC_IRR0 + i);
330*159dd56fSNeel Natu 
331*159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ESR);
332*159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_TIMER);
333*159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_THERMAL);
334*159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_PCINT);
335*159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT0);
336*159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT1);
337*159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_ERROR);
338*159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ICR_TIMER);
339*159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_DCR_TIMER);
340*159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ICR);
341*159dd56fSNeel Natu 
342*159dd56fSNeel Natu 	/*
343*159dd56fSNeel Natu 	 * Allow TPR, EOI and SELF_IPI MSRs to be read and written by the guest.
344*159dd56fSNeel Natu 	 *
345*159dd56fSNeel Natu 	 * These registers get special treatment described in the section
346*159dd56fSNeel Natu 	 * "Virtualizing MSR-Based APIC Accesses".
347*159dd56fSNeel Natu 	 */
348*159dd56fSNeel Natu 	error += guest_msr_rw(vmx, MSR_APIC_TPR);
349*159dd56fSNeel Natu 	error += guest_msr_rw(vmx, MSR_APIC_EOI);
350*159dd56fSNeel Natu 	error += guest_msr_rw(vmx, MSR_APIC_SELF_IPI);
351*159dd56fSNeel Natu 
352*159dd56fSNeel Natu 	return (error);
353*159dd56fSNeel Natu }
354*159dd56fSNeel Natu 
355366f6083SPeter Grehan u_long
356366f6083SPeter Grehan vmx_fix_cr0(u_long cr0)
357366f6083SPeter Grehan {
358366f6083SPeter Grehan 
359366f6083SPeter Grehan 	return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask);
360366f6083SPeter Grehan }
361366f6083SPeter Grehan 
362366f6083SPeter Grehan u_long
363366f6083SPeter Grehan vmx_fix_cr4(u_long cr4)
364366f6083SPeter Grehan {
365366f6083SPeter Grehan 
366366f6083SPeter Grehan 	return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask);
367366f6083SPeter Grehan }
368366f6083SPeter Grehan 
369366f6083SPeter Grehan static void
37045e51299SNeel Natu vpid_free(int vpid)
37145e51299SNeel Natu {
37245e51299SNeel Natu 	if (vpid < 0 || vpid > 0xffff)
37345e51299SNeel Natu 		panic("vpid_free: invalid vpid %d", vpid);
37445e51299SNeel Natu 
37545e51299SNeel Natu 	/*
37645e51299SNeel Natu 	 * VPIDs [0,VM_MAXCPU] are special and are not allocated from
37745e51299SNeel Natu 	 * the unit number allocator.
37845e51299SNeel Natu 	 */
37945e51299SNeel Natu 
38045e51299SNeel Natu 	if (vpid > VM_MAXCPU)
38145e51299SNeel Natu 		free_unr(vpid_unr, vpid);
38245e51299SNeel Natu }
38345e51299SNeel Natu 
38445e51299SNeel Natu static void
38545e51299SNeel Natu vpid_alloc(uint16_t *vpid, int num)
38645e51299SNeel Natu {
38745e51299SNeel Natu 	int i, x;
38845e51299SNeel Natu 
38945e51299SNeel Natu 	if (num <= 0 || num > VM_MAXCPU)
39045e51299SNeel Natu 		panic("invalid number of vpids requested: %d", num);
39145e51299SNeel Natu 
39245e51299SNeel Natu 	/*
39345e51299SNeel Natu 	 * If the "enable vpid" execution control is not enabled then the
39445e51299SNeel Natu 	 * VPID is required to be 0 for all vcpus.
39545e51299SNeel Natu 	 */
39645e51299SNeel Natu 	if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) == 0) {
39745e51299SNeel Natu 		for (i = 0; i < num; i++)
39845e51299SNeel Natu 			vpid[i] = 0;
39945e51299SNeel Natu 		return;
40045e51299SNeel Natu 	}
40145e51299SNeel Natu 
40245e51299SNeel Natu 	/*
40345e51299SNeel Natu 	 * Allocate a unique VPID for each vcpu from the unit number allocator.
40445e51299SNeel Natu 	 */
40545e51299SNeel Natu 	for (i = 0; i < num; i++) {
40645e51299SNeel Natu 		x = alloc_unr(vpid_unr);
40745e51299SNeel Natu 		if (x == -1)
40845e51299SNeel Natu 			break;
40945e51299SNeel Natu 		else
41045e51299SNeel Natu 			vpid[i] = x;
41145e51299SNeel Natu 	}
41245e51299SNeel Natu 
41345e51299SNeel Natu 	if (i < num) {
41445e51299SNeel Natu 		atomic_add_int(&vpid_alloc_failed, 1);
41545e51299SNeel Natu 
41645e51299SNeel Natu 		/*
41745e51299SNeel Natu 		 * If the unit number allocator does not have enough unique
41845e51299SNeel Natu 		 * VPIDs then we need to allocate from the [1,VM_MAXCPU] range.
41945e51299SNeel Natu 		 *
42045e51299SNeel Natu 		 * These VPIDs are not be unique across VMs but this does not
42145e51299SNeel Natu 		 * affect correctness because the combined mappings are also
42245e51299SNeel Natu 		 * tagged with the EP4TA which is unique for each VM.
42345e51299SNeel Natu 		 *
42445e51299SNeel Natu 		 * It is still sub-optimal because the invvpid will invalidate
42545e51299SNeel Natu 		 * combined mappings for a particular VPID across all EP4TAs.
42645e51299SNeel Natu 		 */
42745e51299SNeel Natu 		while (i-- > 0)
42845e51299SNeel Natu 			vpid_free(vpid[i]);
42945e51299SNeel Natu 
43045e51299SNeel Natu 		for (i = 0; i < num; i++)
43145e51299SNeel Natu 			vpid[i] = i + 1;
43245e51299SNeel Natu 	}
43345e51299SNeel Natu }
43445e51299SNeel Natu 
43545e51299SNeel Natu static void
43645e51299SNeel Natu vpid_init(void)
43745e51299SNeel Natu {
43845e51299SNeel Natu 	/*
43945e51299SNeel Natu 	 * VPID 0 is required when the "enable VPID" execution control is
44045e51299SNeel Natu 	 * disabled.
44145e51299SNeel Natu 	 *
44245e51299SNeel Natu 	 * VPIDs [1,VM_MAXCPU] are used as the "overflow namespace" when the
44345e51299SNeel Natu 	 * unit number allocator does not have sufficient unique VPIDs to
44445e51299SNeel Natu 	 * satisfy the allocation.
44545e51299SNeel Natu 	 *
44645e51299SNeel Natu 	 * The remaining VPIDs are managed by the unit number allocator.
44745e51299SNeel Natu 	 */
44845e51299SNeel Natu 	vpid_unr = new_unrhdr(VM_MAXCPU + 1, 0xffff, NULL);
44945e51299SNeel Natu }
45045e51299SNeel Natu 
45145e51299SNeel Natu static void
452366f6083SPeter Grehan msr_save_area_init(struct msr_entry *g_area, int *g_count)
453366f6083SPeter Grehan {
454366f6083SPeter Grehan 	int cnt;
455366f6083SPeter Grehan 
456366f6083SPeter Grehan 	static struct msr_entry guest_msrs[] = {
457366f6083SPeter Grehan 		{ MSR_KGSBASE, 0, 0 },
458366f6083SPeter Grehan 	};
459366f6083SPeter Grehan 
460366f6083SPeter Grehan 	cnt = sizeof(guest_msrs) / sizeof(guest_msrs[0]);
461366f6083SPeter Grehan 	if (cnt > GUEST_MSR_MAX_ENTRIES)
462366f6083SPeter Grehan 		panic("guest msr save area overrun");
463366f6083SPeter Grehan 	bcopy(guest_msrs, g_area, sizeof(guest_msrs));
464366f6083SPeter Grehan 	*g_count = cnt;
465366f6083SPeter Grehan }
466366f6083SPeter Grehan 
467366f6083SPeter Grehan static void
468366f6083SPeter Grehan vmx_disable(void *arg __unused)
469366f6083SPeter Grehan {
470366f6083SPeter Grehan 	struct invvpid_desc invvpid_desc = { 0 };
471366f6083SPeter Grehan 	struct invept_desc invept_desc = { 0 };
472366f6083SPeter Grehan 
473366f6083SPeter Grehan 	if (vmxon_enabled[curcpu]) {
474366f6083SPeter Grehan 		/*
475366f6083SPeter Grehan 		 * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b.
476366f6083SPeter Grehan 		 *
477366f6083SPeter Grehan 		 * VMXON or VMXOFF are not required to invalidate any TLB
478366f6083SPeter Grehan 		 * caching structures. This prevents potential retention of
479366f6083SPeter Grehan 		 * cached information in the TLB between distinct VMX episodes.
480366f6083SPeter Grehan 		 */
481366f6083SPeter Grehan 		invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc);
482366f6083SPeter Grehan 		invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc);
483366f6083SPeter Grehan 		vmxoff();
484366f6083SPeter Grehan 	}
485366f6083SPeter Grehan 	load_cr4(rcr4() & ~CR4_VMXE);
486366f6083SPeter Grehan }
487366f6083SPeter Grehan 
488366f6083SPeter Grehan static int
489366f6083SPeter Grehan vmx_cleanup(void)
490366f6083SPeter Grehan {
491366f6083SPeter Grehan 
492176666c2SNeel Natu 	if (pirvec != 0)
493176666c2SNeel Natu 		vmm_ipi_free(pirvec);
494176666c2SNeel Natu 
49545e51299SNeel Natu 	if (vpid_unr != NULL) {
49645e51299SNeel Natu 		delete_unrhdr(vpid_unr);
49745e51299SNeel Natu 		vpid_unr = NULL;
49845e51299SNeel Natu 	}
49945e51299SNeel Natu 
500366f6083SPeter Grehan 	smp_rendezvous(NULL, vmx_disable, NULL, NULL);
501366f6083SPeter Grehan 
502366f6083SPeter Grehan 	return (0);
503366f6083SPeter Grehan }
504366f6083SPeter Grehan 
505366f6083SPeter Grehan static void
506366f6083SPeter Grehan vmx_enable(void *arg __unused)
507366f6083SPeter Grehan {
508366f6083SPeter Grehan 	int error;
509366f6083SPeter Grehan 
510366f6083SPeter Grehan 	load_cr4(rcr4() | CR4_VMXE);
511366f6083SPeter Grehan 
512366f6083SPeter Grehan 	*(uint32_t *)vmxon_region[curcpu] = vmx_revision();
513366f6083SPeter Grehan 	error = vmxon(vmxon_region[curcpu]);
514366f6083SPeter Grehan 	if (error == 0)
515366f6083SPeter Grehan 		vmxon_enabled[curcpu] = 1;
516366f6083SPeter Grehan }
517366f6083SPeter Grehan 
51863e62d39SJohn Baldwin static void
51963e62d39SJohn Baldwin vmx_restore(void)
52063e62d39SJohn Baldwin {
52163e62d39SJohn Baldwin 
52263e62d39SJohn Baldwin 	if (vmxon_enabled[curcpu])
52363e62d39SJohn Baldwin 		vmxon(vmxon_region[curcpu]);
52463e62d39SJohn Baldwin }
52563e62d39SJohn Baldwin 
526366f6083SPeter Grehan static int
527add611fdSNeel Natu vmx_init(int ipinum)
528366f6083SPeter Grehan {
52988c4b8d1SNeel Natu 	int error, use_tpr_shadow;
5304bff7fadSNeel Natu 	uint64_t fixed0, fixed1, feature_control;
53188c4b8d1SNeel Natu 	uint32_t tmp, procbased2_vid_bits;
532366f6083SPeter Grehan 
533366f6083SPeter Grehan 	/* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */
5348b287612SJohn Baldwin 	if (!(cpu_feature2 & CPUID2_VMX)) {
535366f6083SPeter Grehan 		printf("vmx_init: processor does not support VMX operation\n");
536366f6083SPeter Grehan 		return (ENXIO);
537366f6083SPeter Grehan 	}
538366f6083SPeter Grehan 
5394bff7fadSNeel Natu 	/*
5404bff7fadSNeel Natu 	 * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits
5414bff7fadSNeel Natu 	 * are set (bits 0 and 2 respectively).
5424bff7fadSNeel Natu 	 */
5434bff7fadSNeel Natu 	feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
544150369abSNeel Natu 	if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 0 ||
545150369abSNeel Natu 	    (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) {
5464bff7fadSNeel Natu 		printf("vmx_init: VMX operation disabled by BIOS\n");
5474bff7fadSNeel Natu 		return (ENXIO);
5484bff7fadSNeel Natu 	}
5494bff7fadSNeel Natu 
550366f6083SPeter Grehan 	/* Check support for primary processor-based VM-execution controls */
551366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
552366f6083SPeter Grehan 			       MSR_VMX_TRUE_PROCBASED_CTLS,
553366f6083SPeter Grehan 			       PROCBASED_CTLS_ONE_SETTING,
554366f6083SPeter Grehan 			       PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls);
555366f6083SPeter Grehan 	if (error) {
556366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired primary "
557366f6083SPeter Grehan 		       "processor-based controls\n");
558366f6083SPeter Grehan 		return (error);
559366f6083SPeter Grehan 	}
560366f6083SPeter Grehan 
561366f6083SPeter Grehan 	/* Clear the processor-based ctl bits that are set on demand */
562366f6083SPeter Grehan 	procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING;
563366f6083SPeter Grehan 
564366f6083SPeter Grehan 	/* Check support for secondary processor-based VM-execution controls */
565366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
566366f6083SPeter Grehan 			       MSR_VMX_PROCBASED_CTLS2,
567366f6083SPeter Grehan 			       PROCBASED_CTLS2_ONE_SETTING,
568366f6083SPeter Grehan 			       PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2);
569366f6083SPeter Grehan 	if (error) {
570366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired secondary "
571366f6083SPeter Grehan 		       "processor-based controls\n");
572366f6083SPeter Grehan 		return (error);
573366f6083SPeter Grehan 	}
574366f6083SPeter Grehan 
575366f6083SPeter Grehan 	/* Check support for VPID */
576366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
577366f6083SPeter Grehan 			       PROCBASED2_ENABLE_VPID, 0, &tmp);
578366f6083SPeter Grehan 	if (error == 0)
579366f6083SPeter Grehan 		procbased_ctls2 |= PROCBASED2_ENABLE_VPID;
580366f6083SPeter Grehan 
581366f6083SPeter Grehan 	/* Check support for pin-based VM-execution controls */
582366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
583366f6083SPeter Grehan 			       MSR_VMX_TRUE_PINBASED_CTLS,
584366f6083SPeter Grehan 			       PINBASED_CTLS_ONE_SETTING,
585366f6083SPeter Grehan 			       PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls);
586366f6083SPeter Grehan 	if (error) {
587366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired "
588366f6083SPeter Grehan 		       "pin-based controls\n");
589366f6083SPeter Grehan 		return (error);
590366f6083SPeter Grehan 	}
591366f6083SPeter Grehan 
592366f6083SPeter Grehan 	/* Check support for VM-exit controls */
593366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS,
594366f6083SPeter Grehan 			       VM_EXIT_CTLS_ONE_SETTING,
595366f6083SPeter Grehan 			       VM_EXIT_CTLS_ZERO_SETTING,
596366f6083SPeter Grehan 			       &exit_ctls);
597366f6083SPeter Grehan 	if (error) {
598608f97c3SPeter Grehan 		/* Try again without the PAT MSR bits */
599608f97c3SPeter Grehan 		error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS,
600608f97c3SPeter Grehan 				       MSR_VMX_TRUE_EXIT_CTLS,
601608f97c3SPeter Grehan 				       VM_EXIT_CTLS_ONE_SETTING_NO_PAT,
602608f97c3SPeter Grehan 				       VM_EXIT_CTLS_ZERO_SETTING,
603608f97c3SPeter Grehan 				       &exit_ctls);
604608f97c3SPeter Grehan 		if (error) {
605366f6083SPeter Grehan 			printf("vmx_init: processor does not support desired "
606366f6083SPeter Grehan 			       "exit controls\n");
607366f6083SPeter Grehan 			return (error);
608608f97c3SPeter Grehan 		} else {
609608f97c3SPeter Grehan 			if (bootverbose)
610608f97c3SPeter Grehan 				printf("vmm: PAT MSR access not supported\n");
611608f97c3SPeter Grehan 			guest_msr_valid(MSR_PAT);
612608f97c3SPeter Grehan 			vmx_no_patmsr = 1;
613608f97c3SPeter Grehan 		}
614366f6083SPeter Grehan 	}
615366f6083SPeter Grehan 
616366f6083SPeter Grehan 	/* Check support for VM-entry controls */
617608f97c3SPeter Grehan 	if (!vmx_no_patmsr) {
618608f97c3SPeter Grehan 		error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS,
619608f97c3SPeter Grehan 				       MSR_VMX_TRUE_ENTRY_CTLS,
620366f6083SPeter Grehan 				       VM_ENTRY_CTLS_ONE_SETTING,
621366f6083SPeter Grehan 				       VM_ENTRY_CTLS_ZERO_SETTING,
622366f6083SPeter Grehan 				       &entry_ctls);
623608f97c3SPeter Grehan 	} else {
624608f97c3SPeter Grehan 		error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS,
625608f97c3SPeter Grehan 				       MSR_VMX_TRUE_ENTRY_CTLS,
626608f97c3SPeter Grehan 				       VM_ENTRY_CTLS_ONE_SETTING_NO_PAT,
627608f97c3SPeter Grehan 				       VM_ENTRY_CTLS_ZERO_SETTING,
628608f97c3SPeter Grehan 				       &entry_ctls);
629608f97c3SPeter Grehan 	}
630608f97c3SPeter Grehan 
631366f6083SPeter Grehan 	if (error) {
632366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired "
633366f6083SPeter Grehan 		       "entry controls\n");
634366f6083SPeter Grehan 		       return (error);
635366f6083SPeter Grehan 	}
636366f6083SPeter Grehan 
637366f6083SPeter Grehan 	/*
638366f6083SPeter Grehan 	 * Check support for optional features by testing them
639366f6083SPeter Grehan 	 * as individual bits
640366f6083SPeter Grehan 	 */
641366f6083SPeter Grehan 	cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
642366f6083SPeter Grehan 					MSR_VMX_TRUE_PROCBASED_CTLS,
643366f6083SPeter Grehan 					PROCBASED_HLT_EXITING, 0,
644366f6083SPeter Grehan 					&tmp) == 0);
645366f6083SPeter Grehan 
646366f6083SPeter Grehan 	cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
647366f6083SPeter Grehan 					MSR_VMX_PROCBASED_CTLS,
648366f6083SPeter Grehan 					PROCBASED_MTF, 0,
649366f6083SPeter Grehan 					&tmp) == 0);
650366f6083SPeter Grehan 
651366f6083SPeter Grehan 	cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
652366f6083SPeter Grehan 					 MSR_VMX_TRUE_PROCBASED_CTLS,
653366f6083SPeter Grehan 					 PROCBASED_PAUSE_EXITING, 0,
654366f6083SPeter Grehan 					 &tmp) == 0);
655366f6083SPeter Grehan 
656366f6083SPeter Grehan 	cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
657366f6083SPeter Grehan 					MSR_VMX_PROCBASED_CTLS2,
658366f6083SPeter Grehan 					PROCBASED2_UNRESTRICTED_GUEST, 0,
659366f6083SPeter Grehan 				        &tmp) == 0);
660366f6083SPeter Grehan 
66149cc03daSNeel Natu 	cap_invpcid = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
66249cc03daSNeel Natu 	    MSR_VMX_PROCBASED_CTLS2, PROCBASED2_ENABLE_INVPCID, 0,
66349cc03daSNeel Natu 	    &tmp) == 0);
66449cc03daSNeel Natu 
66588c4b8d1SNeel Natu 	/*
66688c4b8d1SNeel Natu 	 * Check support for virtual interrupt delivery.
66788c4b8d1SNeel Natu 	 */
66888c4b8d1SNeel Natu 	procbased2_vid_bits = (PROCBASED2_VIRTUALIZE_APIC_ACCESSES |
66988c4b8d1SNeel Natu 	    PROCBASED2_VIRTUALIZE_X2APIC_MODE |
67088c4b8d1SNeel Natu 	    PROCBASED2_APIC_REGISTER_VIRTUALIZATION |
67188c4b8d1SNeel Natu 	    PROCBASED2_VIRTUAL_INTERRUPT_DELIVERY);
67288c4b8d1SNeel Natu 
67388c4b8d1SNeel Natu 	use_tpr_shadow = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
67488c4b8d1SNeel Natu 	    MSR_VMX_TRUE_PROCBASED_CTLS, PROCBASED_USE_TPR_SHADOW, 0,
67588c4b8d1SNeel Natu 	    &tmp) == 0);
67688c4b8d1SNeel Natu 
67788c4b8d1SNeel Natu 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
67888c4b8d1SNeel Natu 	    procbased2_vid_bits, 0, &tmp);
67988c4b8d1SNeel Natu 	if (error == 0 && use_tpr_shadow) {
68088c4b8d1SNeel Natu 		virtual_interrupt_delivery = 1;
68188c4b8d1SNeel Natu 		TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_vid",
68288c4b8d1SNeel Natu 		    &virtual_interrupt_delivery);
68388c4b8d1SNeel Natu 	}
68488c4b8d1SNeel Natu 
68588c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
68688c4b8d1SNeel Natu 		procbased_ctls |= PROCBASED_USE_TPR_SHADOW;
68788c4b8d1SNeel Natu 		procbased_ctls2 |= procbased2_vid_bits;
68888c4b8d1SNeel Natu 		procbased_ctls2 &= ~PROCBASED2_VIRTUALIZE_X2APIC_MODE;
689176666c2SNeel Natu 
690176666c2SNeel Natu 		/*
691176666c2SNeel Natu 		 * Check for Posted Interrupts only if Virtual Interrupt
692176666c2SNeel Natu 		 * Delivery is enabled.
693176666c2SNeel Natu 		 */
694176666c2SNeel Natu 		error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
695176666c2SNeel Natu 		    MSR_VMX_TRUE_PINBASED_CTLS, PINBASED_POSTED_INTERRUPT, 0,
696176666c2SNeel Natu 		    &tmp);
697176666c2SNeel Natu 		if (error == 0) {
698176666c2SNeel Natu 			pirvec = vmm_ipi_alloc();
699176666c2SNeel Natu 			if (pirvec == 0) {
700176666c2SNeel Natu 				if (bootverbose) {
701176666c2SNeel Natu 					printf("vmx_init: unable to allocate "
702176666c2SNeel Natu 					    "posted interrupt vector\n");
70388c4b8d1SNeel Natu 				}
704176666c2SNeel Natu 			} else {
705176666c2SNeel Natu 				posted_interrupts = 1;
706176666c2SNeel Natu 				TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_pir",
707176666c2SNeel Natu 				    &posted_interrupts);
708176666c2SNeel Natu 			}
709176666c2SNeel Natu 		}
710176666c2SNeel Natu 	}
711176666c2SNeel Natu 
712176666c2SNeel Natu 	if (posted_interrupts)
713176666c2SNeel Natu 		    pinbased_ctls |= PINBASED_POSTED_INTERRUPT;
71449cc03daSNeel Natu 
715366f6083SPeter Grehan 	/* Initialize EPT */
716add611fdSNeel Natu 	error = ept_init(ipinum);
717366f6083SPeter Grehan 	if (error) {
718366f6083SPeter Grehan 		printf("vmx_init: ept initialization failed (%d)\n", error);
719366f6083SPeter Grehan 		return (error);
720366f6083SPeter Grehan 	}
721366f6083SPeter Grehan 
722366f6083SPeter Grehan 	/*
723366f6083SPeter Grehan 	 * Stash the cr0 and cr4 bits that must be fixed to 0 or 1
724366f6083SPeter Grehan 	 */
725366f6083SPeter Grehan 	fixed0 = rdmsr(MSR_VMX_CR0_FIXED0);
726366f6083SPeter Grehan 	fixed1 = rdmsr(MSR_VMX_CR0_FIXED1);
727366f6083SPeter Grehan 	cr0_ones_mask = fixed0 & fixed1;
728366f6083SPeter Grehan 	cr0_zeros_mask = ~fixed0 & ~fixed1;
729366f6083SPeter Grehan 
730366f6083SPeter Grehan 	/*
731366f6083SPeter Grehan 	 * CR0_PE and CR0_PG can be set to zero in VMX non-root operation
732366f6083SPeter Grehan 	 * if unrestricted guest execution is allowed.
733366f6083SPeter Grehan 	 */
734366f6083SPeter Grehan 	if (cap_unrestricted_guest)
735366f6083SPeter Grehan 		cr0_ones_mask &= ~(CR0_PG | CR0_PE);
736366f6083SPeter Grehan 
737366f6083SPeter Grehan 	/*
738366f6083SPeter Grehan 	 * Do not allow the guest to set CR0_NW or CR0_CD.
739366f6083SPeter Grehan 	 */
740366f6083SPeter Grehan 	cr0_zeros_mask |= (CR0_NW | CR0_CD);
741366f6083SPeter Grehan 
742366f6083SPeter Grehan 	fixed0 = rdmsr(MSR_VMX_CR4_FIXED0);
743366f6083SPeter Grehan 	fixed1 = rdmsr(MSR_VMX_CR4_FIXED1);
744366f6083SPeter Grehan 	cr4_ones_mask = fixed0 & fixed1;
745366f6083SPeter Grehan 	cr4_zeros_mask = ~fixed0 & ~fixed1;
746366f6083SPeter Grehan 
74745e51299SNeel Natu 	vpid_init();
74845e51299SNeel Natu 
749366f6083SPeter Grehan 	/* enable VMX operation */
750366f6083SPeter Grehan 	smp_rendezvous(NULL, vmx_enable, NULL, NULL);
751366f6083SPeter Grehan 
7523565b59eSNeel Natu 	vmx_initialized = 1;
7533565b59eSNeel Natu 
754366f6083SPeter Grehan 	return (0);
755366f6083SPeter Grehan }
756366f6083SPeter Grehan 
757f7d47425SNeel Natu static void
758f7d47425SNeel Natu vmx_trigger_hostintr(int vector)
759f7d47425SNeel Natu {
760f7d47425SNeel Natu 	uintptr_t func;
761f7d47425SNeel Natu 	struct gate_descriptor *gd;
762f7d47425SNeel Natu 
763f7d47425SNeel Natu 	gd = &idt[vector];
764f7d47425SNeel Natu 
765f7d47425SNeel Natu 	KASSERT(vector >= 32 && vector <= 255, ("vmx_trigger_hostintr: "
766f7d47425SNeel Natu 	    "invalid vector %d", vector));
767f7d47425SNeel Natu 	KASSERT(gd->gd_p == 1, ("gate descriptor for vector %d not present",
768f7d47425SNeel Natu 	    vector));
769f7d47425SNeel Natu 	KASSERT(gd->gd_type == SDT_SYSIGT, ("gate descriptor for vector %d "
770f7d47425SNeel Natu 	    "has invalid type %d", vector, gd->gd_type));
771f7d47425SNeel Natu 	KASSERT(gd->gd_dpl == SEL_KPL, ("gate descriptor for vector %d "
772f7d47425SNeel Natu 	    "has invalid dpl %d", vector, gd->gd_dpl));
773f7d47425SNeel Natu 	KASSERT(gd->gd_selector == GSEL(GCODE_SEL, SEL_KPL), ("gate descriptor "
774f7d47425SNeel Natu 	    "for vector %d has invalid selector %d", vector, gd->gd_selector));
775f7d47425SNeel Natu 	KASSERT(gd->gd_ist == 0, ("gate descriptor for vector %d has invalid "
776f7d47425SNeel Natu 	    "IST %d", vector, gd->gd_ist));
777f7d47425SNeel Natu 
778f7d47425SNeel Natu 	func = ((long)gd->gd_hioffset << 16 | gd->gd_looffset);
779f7d47425SNeel Natu 	vmx_call_isr(func);
780f7d47425SNeel Natu }
781f7d47425SNeel Natu 
782366f6083SPeter Grehan static int
783aaaa0656SPeter Grehan vmx_setup_cr_shadow(int which, struct vmcs *vmcs, uint32_t initial)
784366f6083SPeter Grehan {
78539c21c2dSNeel Natu 	int error, mask_ident, shadow_ident;
786aaaa0656SPeter Grehan 	uint64_t mask_value;
787366f6083SPeter Grehan 
78839c21c2dSNeel Natu 	if (which != 0 && which != 4)
78939c21c2dSNeel Natu 		panic("vmx_setup_cr_shadow: unknown cr%d", which);
79039c21c2dSNeel Natu 
79139c21c2dSNeel Natu 	if (which == 0) {
79239c21c2dSNeel Natu 		mask_ident = VMCS_CR0_MASK;
79339c21c2dSNeel Natu 		mask_value = cr0_ones_mask | cr0_zeros_mask;
79439c21c2dSNeel Natu 		shadow_ident = VMCS_CR0_SHADOW;
79539c21c2dSNeel Natu 	} else {
79639c21c2dSNeel Natu 		mask_ident = VMCS_CR4_MASK;
79739c21c2dSNeel Natu 		mask_value = cr4_ones_mask | cr4_zeros_mask;
79839c21c2dSNeel Natu 		shadow_ident = VMCS_CR4_SHADOW;
79939c21c2dSNeel Natu 	}
80039c21c2dSNeel Natu 
801d3c11f40SPeter Grehan 	error = vmcs_setreg(vmcs, 0, VMCS_IDENT(mask_ident), mask_value);
802366f6083SPeter Grehan 	if (error)
803366f6083SPeter Grehan 		return (error);
804366f6083SPeter Grehan 
805aaaa0656SPeter Grehan 	error = vmcs_setreg(vmcs, 0, VMCS_IDENT(shadow_ident), initial);
806366f6083SPeter Grehan 	if (error)
807366f6083SPeter Grehan 		return (error);
808366f6083SPeter Grehan 
809366f6083SPeter Grehan 	return (0);
810366f6083SPeter Grehan }
811aaaa0656SPeter Grehan #define	vmx_setup_cr0_shadow(vmcs,init)	vmx_setup_cr_shadow(0, (vmcs), (init))
812aaaa0656SPeter Grehan #define	vmx_setup_cr4_shadow(vmcs,init)	vmx_setup_cr_shadow(4, (vmcs), (init))
813366f6083SPeter Grehan 
814366f6083SPeter Grehan static void *
815318224bbSNeel Natu vmx_vminit(struct vm *vm, pmap_t pmap)
816366f6083SPeter Grehan {
81745e51299SNeel Natu 	uint16_t vpid[VM_MAXCPU];
818366f6083SPeter Grehan 	int i, error, guest_msr_count;
819366f6083SPeter Grehan 	struct vmx *vmx;
820c847a506SNeel Natu 	struct vmcs *vmcs;
821366f6083SPeter Grehan 
822366f6083SPeter Grehan 	vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO);
823366f6083SPeter Grehan 	if ((uintptr_t)vmx & PAGE_MASK) {
824366f6083SPeter Grehan 		panic("malloc of struct vmx not aligned on %d byte boundary",
825366f6083SPeter Grehan 		      PAGE_SIZE);
826366f6083SPeter Grehan 	}
827366f6083SPeter Grehan 	vmx->vm = vm;
828366f6083SPeter Grehan 
829318224bbSNeel Natu 	vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pml4));
830318224bbSNeel Natu 
831366f6083SPeter Grehan 	/*
832366f6083SPeter Grehan 	 * Clean up EPTP-tagged guest physical and combined mappings
833366f6083SPeter Grehan 	 *
834366f6083SPeter Grehan 	 * VMX transitions are not required to invalidate any guest physical
835366f6083SPeter Grehan 	 * mappings. So, it may be possible for stale guest physical mappings
836366f6083SPeter Grehan 	 * to be present in the processor TLBs.
837366f6083SPeter Grehan 	 *
838366f6083SPeter Grehan 	 * Combined mappings for this EP4TA are also invalidated for all VPIDs.
839366f6083SPeter Grehan 	 */
840318224bbSNeel Natu 	ept_invalidate_mappings(vmx->eptp);
841366f6083SPeter Grehan 
842366f6083SPeter Grehan 	msr_bitmap_initialize(vmx->msr_bitmap);
843366f6083SPeter Grehan 
844366f6083SPeter Grehan 	/*
845366f6083SPeter Grehan 	 * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE.
846366f6083SPeter Grehan 	 * The guest FSBASE and GSBASE are saved and restored during
847366f6083SPeter Grehan 	 * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are
848366f6083SPeter Grehan 	 * always restored from the vmcs host state area on vm-exit.
849366f6083SPeter Grehan 	 *
8501fb0ea3fSPeter Grehan 	 * The SYSENTER_CS/ESP/EIP MSRs are identical to FS/GSBASE in
8511fb0ea3fSPeter Grehan 	 * how they are saved/restored so can be directly accessed by the
8521fb0ea3fSPeter Grehan 	 * guest.
8531fb0ea3fSPeter Grehan 	 *
854366f6083SPeter Grehan 	 * Guest KGSBASE is saved and restored in the guest MSR save area.
855366f6083SPeter Grehan 	 * Host KGSBASE is restored before returning to userland from the pcb.
856366f6083SPeter Grehan 	 * There will be a window of time when we are executing in the host
857366f6083SPeter Grehan 	 * kernel context with a value of KGSBASE from the guest. This is ok
858366f6083SPeter Grehan 	 * because the value of KGSBASE is inconsequential in kernel context.
859366f6083SPeter Grehan 	 *
860366f6083SPeter Grehan 	 * MSR_EFER is saved and restored in the guest VMCS area on a
861366f6083SPeter Grehan 	 * VM exit and entry respectively. It is also restored from the
862366f6083SPeter Grehan 	 * host VMCS area on a VM exit.
863366f6083SPeter Grehan 	 */
864366f6083SPeter Grehan 	if (guest_msr_rw(vmx, MSR_GSBASE) ||
865366f6083SPeter Grehan 	    guest_msr_rw(vmx, MSR_FSBASE) ||
8661fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_CS_MSR) ||
8671fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_ESP_MSR) ||
8681fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_EIP_MSR) ||
869366f6083SPeter Grehan 	    guest_msr_rw(vmx, MSR_KGSBASE) ||
870608f97c3SPeter Grehan 	    guest_msr_rw(vmx, MSR_EFER))
871366f6083SPeter Grehan 		panic("vmx_vminit: error setting guest msr access");
872366f6083SPeter Grehan 
873608f97c3SPeter Grehan 	/*
874608f97c3SPeter Grehan 	 * MSR_PAT is saved and restored in the guest VMCS are on a VM exit
875608f97c3SPeter Grehan 	 * and entry respectively. It is also restored from the host VMCS
876608f97c3SPeter Grehan 	 * area on a VM exit. However, if running on a system with no
877608f97c3SPeter Grehan 	 * MSR_PAT save/restore support, leave access disabled so accesses
878608f97c3SPeter Grehan 	 * will be trapped.
879608f97c3SPeter Grehan 	 */
880608f97c3SPeter Grehan 	if (!vmx_no_patmsr && guest_msr_rw(vmx, MSR_PAT))
881608f97c3SPeter Grehan 		panic("vmx_vminit: error setting guest pat msr access");
882608f97c3SPeter Grehan 
88345e51299SNeel Natu 	vpid_alloc(vpid, VM_MAXCPU);
88445e51299SNeel Natu 
88588c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
88688c4b8d1SNeel Natu 		error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE,
88788c4b8d1SNeel Natu 		    APIC_ACCESS_ADDRESS);
88888c4b8d1SNeel Natu 		/* XXX this should really return an error to the caller */
88988c4b8d1SNeel Natu 		KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error));
89088c4b8d1SNeel Natu 	}
89188c4b8d1SNeel Natu 
892366f6083SPeter Grehan 	for (i = 0; i < VM_MAXCPU; i++) {
893c847a506SNeel Natu 		vmcs = &vmx->vmcs[i];
894c847a506SNeel Natu 		vmcs->identifier = vmx_revision();
895c847a506SNeel Natu 		error = vmclear(vmcs);
896366f6083SPeter Grehan 		if (error != 0) {
897366f6083SPeter Grehan 			panic("vmx_vminit: vmclear error %d on vcpu %d\n",
898366f6083SPeter Grehan 			      error, i);
899366f6083SPeter Grehan 		}
900366f6083SPeter Grehan 
901c847a506SNeel Natu 		error = vmcs_init(vmcs);
902c847a506SNeel Natu 		KASSERT(error == 0, ("vmcs_init error %d", error));
903366f6083SPeter Grehan 
904c847a506SNeel Natu 		VMPTRLD(vmcs);
905c847a506SNeel Natu 		error = 0;
906c847a506SNeel Natu 		error += vmwrite(VMCS_HOST_RSP, (u_long)&vmx->ctx[i]);
907c847a506SNeel Natu 		error += vmwrite(VMCS_EPTP, vmx->eptp);
908c847a506SNeel Natu 		error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls);
909c847a506SNeel Natu 		error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls);
910c847a506SNeel Natu 		error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2);
911c847a506SNeel Natu 		error += vmwrite(VMCS_EXIT_CTLS, exit_ctls);
912c847a506SNeel Natu 		error += vmwrite(VMCS_ENTRY_CTLS, entry_ctls);
913c847a506SNeel Natu 		error += vmwrite(VMCS_MSR_BITMAP, vtophys(vmx->msr_bitmap));
914c847a506SNeel Natu 		error += vmwrite(VMCS_VPID, vpid[i]);
91588c4b8d1SNeel Natu 		if (virtual_interrupt_delivery) {
91688c4b8d1SNeel Natu 			error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS);
91788c4b8d1SNeel Natu 			error += vmwrite(VMCS_VIRTUAL_APIC,
91888c4b8d1SNeel Natu 			    vtophys(&vmx->apic_page[i]));
91988c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT0, 0);
92088c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT1, 0);
92188c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT2, 0);
92288c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT3, 0);
92388c4b8d1SNeel Natu 		}
924176666c2SNeel Natu 		if (posted_interrupts) {
925176666c2SNeel Natu 			error += vmwrite(VMCS_PIR_VECTOR, pirvec);
926176666c2SNeel Natu 			error += vmwrite(VMCS_PIR_DESC,
927176666c2SNeel Natu 			    vtophys(&vmx->pir_desc[i]));
928176666c2SNeel Natu 		}
929c847a506SNeel Natu 		VMCLEAR(vmcs);
930c847a506SNeel Natu 		KASSERT(error == 0, ("vmx_vminit: error customizing the vmcs"));
931366f6083SPeter Grehan 
932366f6083SPeter Grehan 		vmx->cap[i].set = 0;
933366f6083SPeter Grehan 		vmx->cap[i].proc_ctls = procbased_ctls;
93449cc03daSNeel Natu 		vmx->cap[i].proc_ctls2 = procbased_ctls2;
935366f6083SPeter Grehan 
936366f6083SPeter Grehan 		vmx->state[i].lastcpu = -1;
93745e51299SNeel Natu 		vmx->state[i].vpid = vpid[i];
938a0efd3fbSJohn Baldwin 		vmx->state[i].user_event.intr_info = 0;
939366f6083SPeter Grehan 
940366f6083SPeter Grehan 		msr_save_area_init(vmx->guest_msrs[i], &guest_msr_count);
941366f6083SPeter Grehan 
942c847a506SNeel Natu 		error = vmcs_set_msr_save(vmcs, vtophys(vmx->guest_msrs[i]),
943366f6083SPeter Grehan 		    guest_msr_count);
944366f6083SPeter Grehan 		if (error != 0)
945366f6083SPeter Grehan 			panic("vmcs_set_msr_save error %d", error);
946366f6083SPeter Grehan 
947aaaa0656SPeter Grehan 		/*
948aaaa0656SPeter Grehan 		 * Set up the CR0/4 shadows, and init the read shadow
949aaaa0656SPeter Grehan 		 * to the power-on register value from the Intel Sys Arch.
950aaaa0656SPeter Grehan 		 *  CR0 - 0x60000010
951aaaa0656SPeter Grehan 		 *  CR4 - 0
952aaaa0656SPeter Grehan 		 */
953c847a506SNeel Natu 		error = vmx_setup_cr0_shadow(vmcs, 0x60000010);
95439c21c2dSNeel Natu 		if (error != 0)
95539c21c2dSNeel Natu 			panic("vmx_setup_cr0_shadow %d", error);
95639c21c2dSNeel Natu 
957c847a506SNeel Natu 		error = vmx_setup_cr4_shadow(vmcs, 0);
95839c21c2dSNeel Natu 		if (error != 0)
95939c21c2dSNeel Natu 			panic("vmx_setup_cr4_shadow %d", error);
960318224bbSNeel Natu 
961318224bbSNeel Natu 		vmx->ctx[i].pmap = pmap;
962366f6083SPeter Grehan 	}
963366f6083SPeter Grehan 
964366f6083SPeter Grehan 	return (vmx);
965366f6083SPeter Grehan }
966366f6083SPeter Grehan 
967366f6083SPeter Grehan static int
968a2da7af6SNeel Natu vmx_handle_cpuid(struct vm *vm, int vcpu, struct vmxctx *vmxctx)
969366f6083SPeter Grehan {
970366f6083SPeter Grehan 	int handled, func;
971366f6083SPeter Grehan 
972366f6083SPeter Grehan 	func = vmxctx->guest_rax;
973366f6083SPeter Grehan 
974a2da7af6SNeel Natu 	handled = x86_emulate_cpuid(vm, vcpu,
975a2da7af6SNeel Natu 				    (uint32_t*)(&vmxctx->guest_rax),
976a2da7af6SNeel Natu 				    (uint32_t*)(&vmxctx->guest_rbx),
977a2da7af6SNeel Natu 				    (uint32_t*)(&vmxctx->guest_rcx),
978a2da7af6SNeel Natu 				    (uint32_t*)(&vmxctx->guest_rdx));
979366f6083SPeter Grehan 	return (handled);
980366f6083SPeter Grehan }
981366f6083SPeter Grehan 
982366f6083SPeter Grehan static __inline void
983366f6083SPeter Grehan vmx_run_trace(struct vmx *vmx, int vcpu)
984366f6083SPeter Grehan {
985366f6083SPeter Grehan #ifdef KTR
986513c8d33SNeel Natu 	VCPU_CTR1(vmx->vm, vcpu, "Resume execution at %#lx", vmcs_guest_rip());
987366f6083SPeter Grehan #endif
988366f6083SPeter Grehan }
989366f6083SPeter Grehan 
990366f6083SPeter Grehan static __inline void
991366f6083SPeter Grehan vmx_exit_trace(struct vmx *vmx, int vcpu, uint64_t rip, uint32_t exit_reason,
992eeefa4e4SNeel Natu 	       int handled)
993366f6083SPeter Grehan {
994366f6083SPeter Grehan #ifdef KTR
995513c8d33SNeel Natu 	VCPU_CTR3(vmx->vm, vcpu, "%s %s vmexit at 0x%0lx",
996366f6083SPeter Grehan 		 handled ? "handled" : "unhandled",
997366f6083SPeter Grehan 		 exit_reason_to_str(exit_reason), rip);
998eeefa4e4SNeel Natu #endif
999eeefa4e4SNeel Natu }
1000366f6083SPeter Grehan 
1001eeefa4e4SNeel Natu static __inline void
1002eeefa4e4SNeel Natu vmx_astpending_trace(struct vmx *vmx, int vcpu, uint64_t rip)
1003eeefa4e4SNeel Natu {
1004eeefa4e4SNeel Natu #ifdef KTR
1005513c8d33SNeel Natu 	VCPU_CTR1(vmx->vm, vcpu, "astpending vmexit at 0x%0lx", rip);
1006366f6083SPeter Grehan #endif
1007366f6083SPeter Grehan }
1008366f6083SPeter Grehan 
1009953c2c47SNeel Natu static VMM_STAT_INTEL(VCPU_INVVPID_SAVED, "Number of vpid invalidations saved");
1010953c2c47SNeel Natu 
10113de83862SNeel Natu static void
1012953c2c47SNeel Natu vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu, pmap_t pmap)
1013366f6083SPeter Grehan {
1014366f6083SPeter Grehan 	struct vmxstate *vmxstate;
1015953c2c47SNeel Natu 	struct invvpid_desc invvpid_desc;
1016366f6083SPeter Grehan 
1017366f6083SPeter Grehan 	vmxstate = &vmx->state[vcpu];
1018953c2c47SNeel Natu 	if (vmxstate->lastcpu == curcpu)
10193de83862SNeel Natu 		return;
1020366f6083SPeter Grehan 
1021953c2c47SNeel Natu 	vmxstate->lastcpu = curcpu;
1022953c2c47SNeel Natu 
1023366f6083SPeter Grehan 	vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1);
1024366f6083SPeter Grehan 
10253de83862SNeel Natu 	vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase());
10263de83862SNeel Natu 	vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase());
10273de83862SNeel Natu 	vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase());
1028366f6083SPeter Grehan 
1029366f6083SPeter Grehan 	/*
1030366f6083SPeter Grehan 	 * If we are using VPIDs then invalidate all mappings tagged with 'vpid'
1031366f6083SPeter Grehan 	 *
1032366f6083SPeter Grehan 	 * We do this because this vcpu was executing on a different host
1033366f6083SPeter Grehan 	 * cpu when it last ran. We do not track whether it invalidated
1034366f6083SPeter Grehan 	 * mappings associated with its 'vpid' during that run. So we must
1035366f6083SPeter Grehan 	 * assume that the mappings associated with 'vpid' on 'curcpu' are
1036366f6083SPeter Grehan 	 * stale and invalidate them.
1037366f6083SPeter Grehan 	 *
1038366f6083SPeter Grehan 	 * Note that we incur this penalty only when the scheduler chooses to
1039366f6083SPeter Grehan 	 * move the thread associated with this vcpu between host cpus.
1040366f6083SPeter Grehan 	 *
1041366f6083SPeter Grehan 	 * Note also that this will invalidate mappings tagged with 'vpid'
1042366f6083SPeter Grehan 	 * for "all" EP4TAs.
1043366f6083SPeter Grehan 	 */
1044366f6083SPeter Grehan 	if (vmxstate->vpid != 0) {
1045953c2c47SNeel Natu 		if (pmap->pm_eptgen == vmx->eptgen[curcpu]) {
1046953c2c47SNeel Natu 			invvpid_desc._res1 = 0;
1047953c2c47SNeel Natu 			invvpid_desc._res2 = 0;
1048366f6083SPeter Grehan 			invvpid_desc.vpid = vmxstate->vpid;
1049366f6083SPeter Grehan 			invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc);
1050953c2c47SNeel Natu 		} else {
1051953c2c47SNeel Natu 			/*
1052953c2c47SNeel Natu 			 * The invvpid can be skipped if an invept is going to
1053953c2c47SNeel Natu 			 * be performed before entering the guest. The invept
1054953c2c47SNeel Natu 			 * will invalidate combined mappings tagged with
1055953c2c47SNeel Natu 			 * 'vmx->eptp' for all vpids.
1056953c2c47SNeel Natu 			 */
1057953c2c47SNeel Natu 			vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_SAVED, 1);
1058953c2c47SNeel Natu 		}
1059366f6083SPeter Grehan 	}
1060366f6083SPeter Grehan }
1061366f6083SPeter Grehan 
1062366f6083SPeter Grehan /*
1063366f6083SPeter Grehan  * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set.
1064366f6083SPeter Grehan  */
1065366f6083SPeter Grehan CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0);
1066366f6083SPeter Grehan 
1067366f6083SPeter Grehan static void __inline
1068366f6083SPeter Grehan vmx_set_int_window_exiting(struct vmx *vmx, int vcpu)
1069366f6083SPeter Grehan {
1070366f6083SPeter Grehan 
107148b2d828SNeel Natu 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) {
1072366f6083SPeter Grehan 		vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING;
10733de83862SNeel Natu 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
107448b2d828SNeel Natu 		VCPU_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting");
107548b2d828SNeel Natu 	}
1076366f6083SPeter Grehan }
1077366f6083SPeter Grehan 
1078366f6083SPeter Grehan static void __inline
1079366f6083SPeter Grehan vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu)
1080366f6083SPeter Grehan {
1081366f6083SPeter Grehan 
108248b2d828SNeel Natu 	KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0,
108348b2d828SNeel Natu 	    ("intr_window_exiting not set: %#x", vmx->cap[vcpu].proc_ctls));
1084366f6083SPeter Grehan 	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING;
10853de83862SNeel Natu 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
108648b2d828SNeel Natu 	VCPU_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting");
1087366f6083SPeter Grehan }
1088366f6083SPeter Grehan 
1089366f6083SPeter Grehan static void __inline
1090366f6083SPeter Grehan vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu)
1091366f6083SPeter Grehan {
1092366f6083SPeter Grehan 
109348b2d828SNeel Natu 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) {
1094366f6083SPeter Grehan 		vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING;
10953de83862SNeel Natu 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
109648b2d828SNeel Natu 		VCPU_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting");
109748b2d828SNeel Natu 	}
1098366f6083SPeter Grehan }
1099366f6083SPeter Grehan 
1100366f6083SPeter Grehan static void __inline
1101366f6083SPeter Grehan vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu)
1102366f6083SPeter Grehan {
1103366f6083SPeter Grehan 
110448b2d828SNeel Natu 	KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0,
110548b2d828SNeel Natu 	    ("nmi_window_exiting not set %#x", vmx->cap[vcpu].proc_ctls));
1106366f6083SPeter Grehan 	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING;
11073de83862SNeel Natu 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
110848b2d828SNeel Natu 	VCPU_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting");
1109366f6083SPeter Grehan }
1110366f6083SPeter Grehan 
111148b2d828SNeel Natu #define	NMI_BLOCKING	(VMCS_INTERRUPTIBILITY_NMI_BLOCKING |		\
111248b2d828SNeel Natu 			 VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
111348b2d828SNeel Natu #define	HWINTR_BLOCKING	(VMCS_INTERRUPTIBILITY_STI_BLOCKING |		\
111448b2d828SNeel Natu 			 VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
111548b2d828SNeel Natu 
111648b2d828SNeel Natu static void
1117a0efd3fbSJohn Baldwin vmx_inject_user_event(struct vmx *vmx, int vcpu)
1118a0efd3fbSJohn Baldwin {
1119a0efd3fbSJohn Baldwin 	struct vmxevent *user_event;
1120a0efd3fbSJohn Baldwin 	uint32_t info;
1121a0efd3fbSJohn Baldwin 
1122a0efd3fbSJohn Baldwin 	user_event = &vmx->state[vcpu].user_event;
1123a0efd3fbSJohn Baldwin 
1124a0efd3fbSJohn Baldwin 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1125a0efd3fbSJohn Baldwin 	KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_user_event: invalid "
1126a0efd3fbSJohn Baldwin 	    "VM-entry interruption information %#x", info));
1127a0efd3fbSJohn Baldwin 
1128a0efd3fbSJohn Baldwin 	vmcs_write(VMCS_ENTRY_INTR_INFO, user_event->intr_info);
1129a0efd3fbSJohn Baldwin 	if (user_event->intr_info & VMCS_INTR_DEL_ERRCODE)
1130a0efd3fbSJohn Baldwin 		vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR, user_event->error_code);
1131a0efd3fbSJohn Baldwin 	user_event->intr_info = 0;
1132a0efd3fbSJohn Baldwin }
1133a0efd3fbSJohn Baldwin 
1134a0efd3fbSJohn Baldwin static void
1135a0efd3fbSJohn Baldwin vmx_inject_exception(struct vmx *vmx, int vcpu, struct vm_exit *vmexit,
1136a0efd3fbSJohn Baldwin     int fault, int errvalid, int errcode)
1137a0efd3fbSJohn Baldwin {
1138a0efd3fbSJohn Baldwin 	uint32_t info;
1139a0efd3fbSJohn Baldwin 
1140a0efd3fbSJohn Baldwin 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1141a0efd3fbSJohn Baldwin 	KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_exception: invalid "
1142a0efd3fbSJohn Baldwin 	    "VM-entry interruption information %#x", info));
1143a0efd3fbSJohn Baldwin 
1144a0efd3fbSJohn Baldwin 	/*
1145a0efd3fbSJohn Baldwin 	 * Although INTR_T_HWEXCEPTION does not advance %rip, vmx_run()
1146a0efd3fbSJohn Baldwin 	 * always advances it, so we clear the instruction length to zero
1147a0efd3fbSJohn Baldwin 	 * explicitly.
1148a0efd3fbSJohn Baldwin 	 */
1149a0efd3fbSJohn Baldwin 	vmexit->inst_length = 0;
1150a0efd3fbSJohn Baldwin 	info = fault | VMCS_INTR_T_HWEXCEPTION | VMCS_INTR_VALID;
1151a0efd3fbSJohn Baldwin 	if (errvalid) {
1152a0efd3fbSJohn Baldwin 		info |= VMCS_INTR_DEL_ERRCODE;
1153a0efd3fbSJohn Baldwin 		vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR, errcode);
1154a0efd3fbSJohn Baldwin 	}
1155a0efd3fbSJohn Baldwin 	vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1156a0efd3fbSJohn Baldwin 
1157a0efd3fbSJohn Baldwin 	VCPU_CTR2(vmx->vm, vcpu, "Injecting fault %d (errcode %d)", fault,
1158a0efd3fbSJohn Baldwin 	    errcode);
1159a0efd3fbSJohn Baldwin }
1160a0efd3fbSJohn Baldwin 
1161a0efd3fbSJohn Baldwin /* All GP# faults VMM injects use an error code of 0. */
1162a0efd3fbSJohn Baldwin static void
1163a0efd3fbSJohn Baldwin vmx_inject_gp(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1164a0efd3fbSJohn Baldwin {
1165a0efd3fbSJohn Baldwin 
1166a0efd3fbSJohn Baldwin 	vmx_inject_exception(vmx, vcpu, vmexit, IDT_GP, 1, 0);
1167a0efd3fbSJohn Baldwin }
1168a0efd3fbSJohn Baldwin 
1169a0efd3fbSJohn Baldwin static void
1170a0efd3fbSJohn Baldwin vmx_inject_ud(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1171a0efd3fbSJohn Baldwin {
1172a0efd3fbSJohn Baldwin 
1173a0efd3fbSJohn Baldwin 	vmx_inject_exception(vmx, vcpu, vmexit, IDT_UD, 0, 0);
1174a0efd3fbSJohn Baldwin }
1175a0efd3fbSJohn Baldwin 
1176a0efd3fbSJohn Baldwin static void
1177366f6083SPeter Grehan vmx_inject_nmi(struct vmx *vmx, int vcpu)
1178366f6083SPeter Grehan {
117948b2d828SNeel Natu 	uint32_t gi, info;
1180366f6083SPeter Grehan 
118148b2d828SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
118248b2d828SNeel Natu 	KASSERT((gi & NMI_BLOCKING) == 0, ("vmx_inject_nmi: invalid guest "
118348b2d828SNeel Natu 	    "interruptibility-state %#x", gi));
1184366f6083SPeter Grehan 
118548b2d828SNeel Natu 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
118648b2d828SNeel Natu 	KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_nmi: invalid "
118748b2d828SNeel Natu 	    "VM-entry interruption information %#x", info));
1188366f6083SPeter Grehan 
1189366f6083SPeter Grehan 	/*
1190366f6083SPeter Grehan 	 * Inject the virtual NMI. The vector must be the NMI IDT entry
1191366f6083SPeter Grehan 	 * or the VMCS entry check will fail.
1192366f6083SPeter Grehan 	 */
119348b2d828SNeel Natu 	info = IDT_NMI | VMCS_INTR_T_NMI | VMCS_INTR_VALID;
11943de83862SNeel Natu 	vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1195366f6083SPeter Grehan 
1196513c8d33SNeel Natu 	VCPU_CTR0(vmx->vm, vcpu, "Injecting vNMI");
1197366f6083SPeter Grehan 
1198366f6083SPeter Grehan 	/* Clear the request */
1199f352ff0cSNeel Natu 	vm_nmi_clear(vmx->vm, vcpu);
1200366f6083SPeter Grehan }
1201366f6083SPeter Grehan 
1202366f6083SPeter Grehan static void
1203de5ea6b6SNeel Natu vmx_inject_interrupts(struct vmx *vmx, int vcpu, struct vlapic *vlapic)
1204366f6083SPeter Grehan {
120548b2d828SNeel Natu 	int vector, need_nmi_exiting;
120648b2d828SNeel Natu 	uint64_t rflags;
120748b2d828SNeel Natu 	uint32_t gi, info;
1208366f6083SPeter Grehan 
120948b2d828SNeel Natu 	if (vm_nmi_pending(vmx->vm, vcpu)) {
1210366f6083SPeter Grehan 		/*
121148b2d828SNeel Natu 		 * If there are no conditions blocking NMI injection then
121248b2d828SNeel Natu 		 * inject it directly here otherwise enable "NMI window
121348b2d828SNeel Natu 		 * exiting" to inject it as soon as we can.
1214eeefa4e4SNeel Natu 		 *
121548b2d828SNeel Natu 		 * We also check for STI_BLOCKING because some implementations
121648b2d828SNeel Natu 		 * don't allow NMI injection in this case. If we are running
121748b2d828SNeel Natu 		 * on a processor that doesn't have this restriction it will
121848b2d828SNeel Natu 		 * immediately exit and the NMI will be injected in the
121948b2d828SNeel Natu 		 * "NMI window exiting" handler.
1220366f6083SPeter Grehan 		 */
122148b2d828SNeel Natu 		need_nmi_exiting = 1;
122248b2d828SNeel Natu 		gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
122348b2d828SNeel Natu 		if ((gi & (HWINTR_BLOCKING | NMI_BLOCKING)) == 0) {
12243de83862SNeel Natu 			info = vmcs_read(VMCS_ENTRY_INTR_INFO);
122548b2d828SNeel Natu 			if ((info & VMCS_INTR_VALID) == 0) {
122648b2d828SNeel Natu 				vmx_inject_nmi(vmx, vcpu);
122748b2d828SNeel Natu 				need_nmi_exiting = 0;
122848b2d828SNeel Natu 			} else {
122948b2d828SNeel Natu 				VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI "
123048b2d828SNeel Natu 				    "due to VM-entry intr info %#x", info);
123148b2d828SNeel Natu 			}
123248b2d828SNeel Natu 		} else {
123348b2d828SNeel Natu 			VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI due to "
123448b2d828SNeel Natu 			    "Guest Interruptibility-state %#x", gi);
123548b2d828SNeel Natu 		}
1236eeefa4e4SNeel Natu 
123748b2d828SNeel Natu 		if (need_nmi_exiting)
123848b2d828SNeel Natu 			vmx_set_nmi_window_exiting(vmx, vcpu);
123948b2d828SNeel Natu 	}
1240366f6083SPeter Grehan 
1241a0efd3fbSJohn Baldwin 	/*
1242a0efd3fbSJohn Baldwin 	 * If there is a user injection event pending and there isn't
1243a0efd3fbSJohn Baldwin 	 * an interrupt queued already, inject the user event.
1244a0efd3fbSJohn Baldwin 	 */
1245a0efd3fbSJohn Baldwin 	if (vmx->state[vcpu].user_event.intr_info & VMCS_INTR_VALID) {
1246a0efd3fbSJohn Baldwin 		info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1247a0efd3fbSJohn Baldwin 		if ((info & VMCS_INTR_VALID) == 0) {
1248a0efd3fbSJohn Baldwin 			vmx_inject_user_event(vmx, vcpu);
1249a0efd3fbSJohn Baldwin 		} else {
1250a0efd3fbSJohn Baldwin 			/*
1251a0efd3fbSJohn Baldwin 			 * XXX: Do we need to force an exit so this can
1252a0efd3fbSJohn Baldwin 			 * be injected?
1253a0efd3fbSJohn Baldwin 			 */
1254a0efd3fbSJohn Baldwin 			VCPU_CTR1(vmx->vm, vcpu, "Cannot inject user event "
1255a0efd3fbSJohn Baldwin 			    "due to VM-entry intr info %#x", info);
1256a0efd3fbSJohn Baldwin 		}
1257a0efd3fbSJohn Baldwin 	}
1258a0efd3fbSJohn Baldwin 
125988c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
126088c4b8d1SNeel Natu 		vmx_inject_pir(vlapic);
126188c4b8d1SNeel Natu 		return;
126288c4b8d1SNeel Natu 	}
126388c4b8d1SNeel Natu 
126448b2d828SNeel Natu 	/*
126536736912SNeel Natu 	 * If interrupt-window exiting is already in effect then don't bother
126636736912SNeel Natu 	 * checking for pending interrupts. This is just an optimization and
126736736912SNeel Natu 	 * not needed for correctness.
126848b2d828SNeel Natu 	 */
126936736912SNeel Natu 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) {
127036736912SNeel Natu 		VCPU_CTR0(vmx->vm, vcpu, "Skip interrupt injection due to "
127136736912SNeel Natu 		    "pending int_window_exiting");
127248b2d828SNeel Natu 		return;
127336736912SNeel Natu 	}
127448b2d828SNeel Natu 
1275366f6083SPeter Grehan 	/* Ask the local apic for a vector to inject */
12764d1e82a8SNeel Natu 	if (!vlapic_pending_intr(vlapic, &vector))
1277366f6083SPeter Grehan 		return;
1278366f6083SPeter Grehan 
127948b2d828SNeel Natu 	KASSERT(vector >= 32 && vector <= 255, ("invalid vector %d", vector));
1280366f6083SPeter Grehan 
1281366f6083SPeter Grehan 	/* Check RFLAGS.IF and the interruptibility state of the guest */
12823de83862SNeel Natu 	rflags = vmcs_read(VMCS_GUEST_RFLAGS);
128336736912SNeel Natu 	if ((rflags & PSL_I) == 0) {
128436736912SNeel Natu 		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
128536736912SNeel Natu 		    "rflags %#lx", vector, rflags);
1286366f6083SPeter Grehan 		goto cantinject;
128736736912SNeel Natu 	}
1288366f6083SPeter Grehan 
128948b2d828SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
129036736912SNeel Natu 	if (gi & HWINTR_BLOCKING) {
129136736912SNeel Natu 		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
129236736912SNeel Natu 		    "Guest Interruptibility-state %#x", vector, gi);
1293366f6083SPeter Grehan 		goto cantinject;
129436736912SNeel Natu 	}
129536736912SNeel Natu 
129636736912SNeel Natu 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
129736736912SNeel Natu 	if (info & VMCS_INTR_VALID) {
129836736912SNeel Natu 		/*
129936736912SNeel Natu 		 * This is expected and could happen for multiple reasons:
130036736912SNeel Natu 		 * - A vectoring VM-entry was aborted due to astpending
130136736912SNeel Natu 		 * - A VM-exit happened during event injection.
130236736912SNeel Natu 		 * - An NMI was injected above or after "NMI window exiting"
130336736912SNeel Natu 		 */
130436736912SNeel Natu 		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
130536736912SNeel Natu 		    "VM-entry intr info %#x", vector, info);
130636736912SNeel Natu 		goto cantinject;
130736736912SNeel Natu 	}
1308366f6083SPeter Grehan 
1309366f6083SPeter Grehan 	/* Inject the interrupt */
1310160471d2SNeel Natu 	info = VMCS_INTR_T_HWINTR | VMCS_INTR_VALID;
1311366f6083SPeter Grehan 	info |= vector;
13123de83862SNeel Natu 	vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1313366f6083SPeter Grehan 
1314366f6083SPeter Grehan 	/* Update the Local APIC ISR */
1315de5ea6b6SNeel Natu 	vlapic_intr_accepted(vlapic, vector);
1316366f6083SPeter Grehan 
1317513c8d33SNeel Natu 	VCPU_CTR1(vmx->vm, vcpu, "Injecting hwintr at vector %d", vector);
1318366f6083SPeter Grehan 
1319366f6083SPeter Grehan 	return;
1320366f6083SPeter Grehan 
1321366f6083SPeter Grehan cantinject:
1322366f6083SPeter Grehan 	/*
1323366f6083SPeter Grehan 	 * Set the Interrupt Window Exiting execution control so we can inject
1324366f6083SPeter Grehan 	 * the interrupt as soon as blocking condition goes away.
1325366f6083SPeter Grehan 	 */
1326366f6083SPeter Grehan 	vmx_set_int_window_exiting(vmx, vcpu);
1327366f6083SPeter Grehan }
1328366f6083SPeter Grehan 
1329e5a1d950SNeel Natu /*
1330e5a1d950SNeel Natu  * If the Virtual NMIs execution control is '1' then the logical processor
1331e5a1d950SNeel Natu  * tracks virtual-NMI blocking in the Guest Interruptibility-state field of
1332e5a1d950SNeel Natu  * the VMCS. An IRET instruction in VMX non-root operation will remove any
1333e5a1d950SNeel Natu  * virtual-NMI blocking.
1334e5a1d950SNeel Natu  *
1335e5a1d950SNeel Natu  * This unblocking occurs even if the IRET causes a fault. In this case the
1336e5a1d950SNeel Natu  * hypervisor needs to restore virtual-NMI blocking before resuming the guest.
1337e5a1d950SNeel Natu  */
1338e5a1d950SNeel Natu static void
1339e5a1d950SNeel Natu vmx_restore_nmi_blocking(struct vmx *vmx, int vcpuid)
1340e5a1d950SNeel Natu {
1341e5a1d950SNeel Natu 	uint32_t gi;
1342e5a1d950SNeel Natu 
1343e5a1d950SNeel Natu 	VCPU_CTR0(vmx->vm, vcpuid, "Restore Virtual-NMI blocking");
1344e5a1d950SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1345e5a1d950SNeel Natu 	gi |= VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1346e5a1d950SNeel Natu 	vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1347e5a1d950SNeel Natu }
1348e5a1d950SNeel Natu 
1349e5a1d950SNeel Natu static void
1350e5a1d950SNeel Natu vmx_clear_nmi_blocking(struct vmx *vmx, int vcpuid)
1351e5a1d950SNeel Natu {
1352e5a1d950SNeel Natu 	uint32_t gi;
1353e5a1d950SNeel Natu 
1354e5a1d950SNeel Natu 	VCPU_CTR0(vmx->vm, vcpuid, "Clear Virtual-NMI blocking");
1355e5a1d950SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1356e5a1d950SNeel Natu 	gi &= ~VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1357e5a1d950SNeel Natu 	vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1358e5a1d950SNeel Natu }
1359e5a1d950SNeel Natu 
1360366f6083SPeter Grehan static int
1361a0efd3fbSJohn Baldwin vmx_emulate_xsetbv(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1362abb023fbSJohn Baldwin {
1363abb023fbSJohn Baldwin 	struct vmxctx *vmxctx;
1364abb023fbSJohn Baldwin 	uint64_t xcrval;
1365abb023fbSJohn Baldwin 	const struct xsave_limits *limits;
1366abb023fbSJohn Baldwin 
1367abb023fbSJohn Baldwin 	vmxctx = &vmx->ctx[vcpu];
1368abb023fbSJohn Baldwin 	limits = vmm_get_xsave_limits();
1369abb023fbSJohn Baldwin 
1370a0efd3fbSJohn Baldwin 	/*
1371a0efd3fbSJohn Baldwin 	 * Note that the processor raises a GP# fault on its own if
1372a0efd3fbSJohn Baldwin 	 * xsetbv is executed for CPL != 0, so we do not have to
1373a0efd3fbSJohn Baldwin 	 * emulate that fault here.
1374a0efd3fbSJohn Baldwin 	 */
1375a0efd3fbSJohn Baldwin 
1376a0efd3fbSJohn Baldwin 	/* Only xcr0 is supported. */
1377a0efd3fbSJohn Baldwin 	if (vmxctx->guest_rcx != 0) {
1378a0efd3fbSJohn Baldwin 		vmx_inject_gp(vmx, vcpu, vmexit);
1379a0efd3fbSJohn Baldwin 		return (HANDLED);
1380a0efd3fbSJohn Baldwin 	}
1381a0efd3fbSJohn Baldwin 
1382a0efd3fbSJohn Baldwin 	/* We only handle xcr0 if both the host and guest have XSAVE enabled. */
1383a0efd3fbSJohn Baldwin 	if (!limits->xsave_enabled || !(vmcs_read(VMCS_GUEST_CR4) & CR4_XSAVE)) {
1384a0efd3fbSJohn Baldwin 		vmx_inject_ud(vmx, vcpu, vmexit);
1385a0efd3fbSJohn Baldwin 		return (HANDLED);
1386a0efd3fbSJohn Baldwin 	}
1387abb023fbSJohn Baldwin 
1388abb023fbSJohn Baldwin 	xcrval = vmxctx->guest_rdx << 32 | (vmxctx->guest_rax & 0xffffffff);
1389a0efd3fbSJohn Baldwin 	if ((xcrval & ~limits->xcr0_allowed) != 0) {
1390a0efd3fbSJohn Baldwin 		vmx_inject_gp(vmx, vcpu, vmexit);
1391a0efd3fbSJohn Baldwin 		return (HANDLED);
1392a0efd3fbSJohn Baldwin 	}
1393abb023fbSJohn Baldwin 
1394a0efd3fbSJohn Baldwin 	if (!(xcrval & XFEATURE_ENABLED_X87)) {
1395a0efd3fbSJohn Baldwin 		vmx_inject_gp(vmx, vcpu, vmexit);
1396a0efd3fbSJohn Baldwin 		return (HANDLED);
1397a0efd3fbSJohn Baldwin 	}
1398abb023fbSJohn Baldwin 
1399abb023fbSJohn Baldwin 	if ((xcrval & (XFEATURE_ENABLED_AVX | XFEATURE_ENABLED_SSE)) ==
1400a0efd3fbSJohn Baldwin 	    XFEATURE_ENABLED_AVX) {
1401a0efd3fbSJohn Baldwin 		vmx_inject_gp(vmx, vcpu, vmexit);
1402a0efd3fbSJohn Baldwin 		return (HANDLED);
1403a0efd3fbSJohn Baldwin 	}
1404abb023fbSJohn Baldwin 
1405abb023fbSJohn Baldwin 	/*
1406abb023fbSJohn Baldwin 	 * This runs "inside" vmrun() with the guest's FPU state, so
1407abb023fbSJohn Baldwin 	 * modifying xcr0 directly modifies the guest's xcr0, not the
1408abb023fbSJohn Baldwin 	 * host's.
1409abb023fbSJohn Baldwin 	 */
1410abb023fbSJohn Baldwin 	load_xcr(0, xcrval);
1411abb023fbSJohn Baldwin 	return (HANDLED);
1412abb023fbSJohn Baldwin }
1413abb023fbSJohn Baldwin 
1414abb023fbSJohn Baldwin static int
1415366f6083SPeter Grehan vmx_emulate_cr_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1416366f6083SPeter Grehan {
14173de83862SNeel Natu 	int cr, vmcs_guest_cr, vmcs_shadow_cr;
141880a902efSPeter Grehan 	uint64_t crval, regval, ones_mask, zeros_mask;
1419366f6083SPeter Grehan 	const struct vmxctx *vmxctx;
1420366f6083SPeter Grehan 
142139c21c2dSNeel Natu 	/* We only handle mov to %cr0 or %cr4 at this time */
142239c21c2dSNeel Natu 	if ((exitqual & 0xf0) != 0x00)
142339c21c2dSNeel Natu 		return (UNHANDLED);
142439c21c2dSNeel Natu 
142539c21c2dSNeel Natu 	cr = exitqual & 0xf;
142639c21c2dSNeel Natu 	if (cr != 0 && cr != 4)
1427366f6083SPeter Grehan 		return (UNHANDLED);
1428366f6083SPeter Grehan 
14296f0c167fSDimitry Andric 	regval = 0; /* silence gcc */
1430366f6083SPeter Grehan 	vmxctx = &vmx->ctx[vcpu];
1431366f6083SPeter Grehan 
1432366f6083SPeter Grehan 	/*
14333de83862SNeel Natu 	 * We must use vmcs_write() directly here because vmcs_setreg() will
1434366f6083SPeter Grehan 	 * call vmclear(vmcs) as a side-effect which we certainly don't want.
1435366f6083SPeter Grehan 	 */
1436366f6083SPeter Grehan 	switch ((exitqual >> 8) & 0xf) {
1437366f6083SPeter Grehan 	case 0:
1438366f6083SPeter Grehan 		regval = vmxctx->guest_rax;
1439366f6083SPeter Grehan 		break;
1440366f6083SPeter Grehan 	case 1:
1441366f6083SPeter Grehan 		regval = vmxctx->guest_rcx;
1442366f6083SPeter Grehan 		break;
1443366f6083SPeter Grehan 	case 2:
1444366f6083SPeter Grehan 		regval = vmxctx->guest_rdx;
1445366f6083SPeter Grehan 		break;
1446366f6083SPeter Grehan 	case 3:
1447366f6083SPeter Grehan 		regval = vmxctx->guest_rbx;
1448366f6083SPeter Grehan 		break;
1449366f6083SPeter Grehan 	case 4:
14503de83862SNeel Natu 		regval = vmcs_read(VMCS_GUEST_RSP);
1451366f6083SPeter Grehan 		break;
1452366f6083SPeter Grehan 	case 5:
1453366f6083SPeter Grehan 		regval = vmxctx->guest_rbp;
1454366f6083SPeter Grehan 		break;
1455366f6083SPeter Grehan 	case 6:
1456366f6083SPeter Grehan 		regval = vmxctx->guest_rsi;
1457366f6083SPeter Grehan 		break;
1458366f6083SPeter Grehan 	case 7:
1459366f6083SPeter Grehan 		regval = vmxctx->guest_rdi;
1460366f6083SPeter Grehan 		break;
1461366f6083SPeter Grehan 	case 8:
1462366f6083SPeter Grehan 		regval = vmxctx->guest_r8;
1463366f6083SPeter Grehan 		break;
1464366f6083SPeter Grehan 	case 9:
1465366f6083SPeter Grehan 		regval = vmxctx->guest_r9;
1466366f6083SPeter Grehan 		break;
1467366f6083SPeter Grehan 	case 10:
1468366f6083SPeter Grehan 		regval = vmxctx->guest_r10;
1469366f6083SPeter Grehan 		break;
1470366f6083SPeter Grehan 	case 11:
1471366f6083SPeter Grehan 		regval = vmxctx->guest_r11;
1472366f6083SPeter Grehan 		break;
1473366f6083SPeter Grehan 	case 12:
1474366f6083SPeter Grehan 		regval = vmxctx->guest_r12;
1475366f6083SPeter Grehan 		break;
1476366f6083SPeter Grehan 	case 13:
1477366f6083SPeter Grehan 		regval = vmxctx->guest_r13;
1478366f6083SPeter Grehan 		break;
1479366f6083SPeter Grehan 	case 14:
1480366f6083SPeter Grehan 		regval = vmxctx->guest_r14;
1481366f6083SPeter Grehan 		break;
1482366f6083SPeter Grehan 	case 15:
1483366f6083SPeter Grehan 		regval = vmxctx->guest_r15;
1484366f6083SPeter Grehan 		break;
1485366f6083SPeter Grehan 	}
1486366f6083SPeter Grehan 
148739c21c2dSNeel Natu 	if (cr == 0) {
148839c21c2dSNeel Natu 		ones_mask = cr0_ones_mask;
148939c21c2dSNeel Natu 		zeros_mask = cr0_zeros_mask;
149039c21c2dSNeel Natu 		vmcs_guest_cr = VMCS_GUEST_CR0;
1491aaaa0656SPeter Grehan 		vmcs_shadow_cr = VMCS_CR0_SHADOW;
149239c21c2dSNeel Natu 	} else {
149339c21c2dSNeel Natu 		ones_mask = cr4_ones_mask;
149439c21c2dSNeel Natu 		zeros_mask = cr4_zeros_mask;
149539c21c2dSNeel Natu 		vmcs_guest_cr = VMCS_GUEST_CR4;
1496aaaa0656SPeter Grehan 		vmcs_shadow_cr = VMCS_CR4_SHADOW;
149739c21c2dSNeel Natu 	}
14983de83862SNeel Natu 	vmcs_write(vmcs_shadow_cr, regval);
1499aaaa0656SPeter Grehan 
150080a902efSPeter Grehan 	crval = regval | ones_mask;
150180a902efSPeter Grehan 	crval &= ~zeros_mask;
15023de83862SNeel Natu 	vmcs_write(vmcs_guest_cr, crval);
1503366f6083SPeter Grehan 
150480a902efSPeter Grehan 	if (cr == 0 && regval & CR0_PG) {
150580a902efSPeter Grehan 		uint64_t efer, entry_ctls;
150680a902efSPeter Grehan 
150780a902efSPeter Grehan 		/*
150880a902efSPeter Grehan 		 * If CR0.PG is 1 and EFER.LME is 1 then EFER.LMA and
150980a902efSPeter Grehan 		 * the "IA-32e mode guest" bit in VM-entry control must be
151080a902efSPeter Grehan 		 * equal.
151180a902efSPeter Grehan 		 */
15123de83862SNeel Natu 		efer = vmcs_read(VMCS_GUEST_IA32_EFER);
151380a902efSPeter Grehan 		if (efer & EFER_LME) {
151480a902efSPeter Grehan 			efer |= EFER_LMA;
15153de83862SNeel Natu 			vmcs_write(VMCS_GUEST_IA32_EFER, efer);
15163de83862SNeel Natu 			entry_ctls = vmcs_read(VMCS_ENTRY_CTLS);
151780a902efSPeter Grehan 			entry_ctls |= VM_ENTRY_GUEST_LMA;
15183de83862SNeel Natu 			vmcs_write(VMCS_ENTRY_CTLS, entry_ctls);
151980a902efSPeter Grehan 		}
152080a902efSPeter Grehan 	}
152180a902efSPeter Grehan 
1522366f6083SPeter Grehan 	return (HANDLED);
1523366f6083SPeter Grehan }
1524366f6083SPeter Grehan 
152500f3efe1SJohn Baldwin static enum vie_cpu_mode
152600f3efe1SJohn Baldwin vmx_cpu_mode(void)
152700f3efe1SJohn Baldwin {
152800f3efe1SJohn Baldwin 
152900f3efe1SJohn Baldwin 	if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LMA)
153000f3efe1SJohn Baldwin 		return (CPU_MODE_64BIT);
153100f3efe1SJohn Baldwin 	else
153200f3efe1SJohn Baldwin 		return (CPU_MODE_COMPATIBILITY);
153300f3efe1SJohn Baldwin }
153400f3efe1SJohn Baldwin 
153500f3efe1SJohn Baldwin static enum vie_paging_mode
153600f3efe1SJohn Baldwin vmx_paging_mode(void)
153700f3efe1SJohn Baldwin {
153800f3efe1SJohn Baldwin 
153900f3efe1SJohn Baldwin 	if (!(vmcs_read(VMCS_GUEST_CR0) & CR0_PG))
154000f3efe1SJohn Baldwin 		return (PAGING_MODE_FLAT);
154100f3efe1SJohn Baldwin 	if (!(vmcs_read(VMCS_GUEST_CR4) & CR4_PAE))
154200f3efe1SJohn Baldwin 		return (PAGING_MODE_32);
154300f3efe1SJohn Baldwin 	if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LME)
154400f3efe1SJohn Baldwin 		return (PAGING_MODE_64);
154500f3efe1SJohn Baldwin 	else
154600f3efe1SJohn Baldwin 		return (PAGING_MODE_PAE);
154700f3efe1SJohn Baldwin }
154800f3efe1SJohn Baldwin 
1549366f6083SPeter Grehan static int
1550318224bbSNeel Natu ept_fault_type(uint64_t ept_qual)
1551a2da7af6SNeel Natu {
1552318224bbSNeel Natu 	int fault_type;
1553a2da7af6SNeel Natu 
1554318224bbSNeel Natu 	if (ept_qual & EPT_VIOLATION_DATA_WRITE)
1555318224bbSNeel Natu 		fault_type = VM_PROT_WRITE;
1556318224bbSNeel Natu 	else if (ept_qual & EPT_VIOLATION_INST_FETCH)
1557318224bbSNeel Natu 		fault_type = VM_PROT_EXECUTE;
1558318224bbSNeel Natu 	else
1559318224bbSNeel Natu 		fault_type= VM_PROT_READ;
1560318224bbSNeel Natu 
1561318224bbSNeel Natu 	return (fault_type);
1562318224bbSNeel Natu }
1563318224bbSNeel Natu 
1564318224bbSNeel Natu static boolean_t
1565318224bbSNeel Natu ept_emulation_fault(uint64_t ept_qual)
1566318224bbSNeel Natu {
1567318224bbSNeel Natu 	int read, write;
1568318224bbSNeel Natu 
1569318224bbSNeel Natu 	/* EPT fault on an instruction fetch doesn't make sense here */
1570a2da7af6SNeel Natu 	if (ept_qual & EPT_VIOLATION_INST_FETCH)
1571318224bbSNeel Natu 		return (FALSE);
1572a2da7af6SNeel Natu 
1573318224bbSNeel Natu 	/* EPT fault must be a read fault or a write fault */
1574a2da7af6SNeel Natu 	read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0;
1575a2da7af6SNeel Natu 	write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0;
15763b2b0011SPeter Grehan 	if ((read | write) == 0)
1577318224bbSNeel Natu 		return (FALSE);
1578a2da7af6SNeel Natu 
1579a2da7af6SNeel Natu 	/*
15803b2b0011SPeter Grehan 	 * The EPT violation must have been caused by accessing a
15813b2b0011SPeter Grehan 	 * guest-physical address that is a translation of a guest-linear
15823b2b0011SPeter Grehan 	 * address.
1583a2da7af6SNeel Natu 	 */
1584a2da7af6SNeel Natu 	if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 ||
1585a2da7af6SNeel Natu 	    (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) {
1586318224bbSNeel Natu 		return (FALSE);
1587a2da7af6SNeel Natu 	}
1588a2da7af6SNeel Natu 
1589318224bbSNeel Natu 	return (TRUE);
1590a2da7af6SNeel Natu }
1591a2da7af6SNeel Natu 
1592*159dd56fSNeel Natu static __inline int
1593*159dd56fSNeel Natu apic_access_virtualization(struct vmx *vmx, int vcpuid)
1594*159dd56fSNeel Natu {
1595*159dd56fSNeel Natu 	uint32_t proc_ctls2;
1596*159dd56fSNeel Natu 
1597*159dd56fSNeel Natu 	proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
1598*159dd56fSNeel Natu 	return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) ? 1 : 0);
1599*159dd56fSNeel Natu }
1600*159dd56fSNeel Natu 
1601*159dd56fSNeel Natu static __inline int
1602*159dd56fSNeel Natu x2apic_virtualization(struct vmx *vmx, int vcpuid)
1603*159dd56fSNeel Natu {
1604*159dd56fSNeel Natu 	uint32_t proc_ctls2;
1605*159dd56fSNeel Natu 
1606*159dd56fSNeel Natu 	proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
1607*159dd56fSNeel Natu 	return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_X2APIC_MODE) ? 1 : 0);
1608*159dd56fSNeel Natu }
1609*159dd56fSNeel Natu 
1610a2da7af6SNeel Natu static int
1611*159dd56fSNeel Natu vmx_handle_apic_write(struct vmx *vmx, int vcpuid, struct vlapic *vlapic,
1612*159dd56fSNeel Natu     uint64_t qual)
161388c4b8d1SNeel Natu {
161488c4b8d1SNeel Natu 	int error, handled, offset;
1615*159dd56fSNeel Natu 	uint32_t *apic_regs, vector;
161688c4b8d1SNeel Natu 	bool retu;
161788c4b8d1SNeel Natu 
1618a0efd3fbSJohn Baldwin 	handled = HANDLED;
161988c4b8d1SNeel Natu 	offset = APIC_WRITE_OFFSET(qual);
1620*159dd56fSNeel Natu 
1621*159dd56fSNeel Natu 	if (!apic_access_virtualization(vmx, vcpuid)) {
1622*159dd56fSNeel Natu 		/*
1623*159dd56fSNeel Natu 		 * In general there should not be any APIC write VM-exits
1624*159dd56fSNeel Natu 		 * unless APIC-access virtualization is enabled.
1625*159dd56fSNeel Natu 		 *
1626*159dd56fSNeel Natu 		 * However self-IPI virtualization can legitimately trigger
1627*159dd56fSNeel Natu 		 * an APIC-write VM-exit so treat it specially.
1628*159dd56fSNeel Natu 		 */
1629*159dd56fSNeel Natu 		if (x2apic_virtualization(vmx, vcpuid) &&
1630*159dd56fSNeel Natu 		    offset == APIC_OFFSET_SELF_IPI) {
1631*159dd56fSNeel Natu 			apic_regs = (uint32_t *)(vlapic->apic_page);
1632*159dd56fSNeel Natu 			vector = apic_regs[APIC_OFFSET_SELF_IPI / 4];
1633*159dd56fSNeel Natu 			vlapic_self_ipi_handler(vlapic, vector);
1634*159dd56fSNeel Natu 			return (HANDLED);
1635*159dd56fSNeel Natu 		} else
1636*159dd56fSNeel Natu 			return (UNHANDLED);
1637*159dd56fSNeel Natu 	}
1638*159dd56fSNeel Natu 
163988c4b8d1SNeel Natu 	switch (offset) {
164088c4b8d1SNeel Natu 	case APIC_OFFSET_ID:
164188c4b8d1SNeel Natu 		vlapic_id_write_handler(vlapic);
164288c4b8d1SNeel Natu 		break;
164388c4b8d1SNeel Natu 	case APIC_OFFSET_LDR:
164488c4b8d1SNeel Natu 		vlapic_ldr_write_handler(vlapic);
164588c4b8d1SNeel Natu 		break;
164688c4b8d1SNeel Natu 	case APIC_OFFSET_DFR:
164788c4b8d1SNeel Natu 		vlapic_dfr_write_handler(vlapic);
164888c4b8d1SNeel Natu 		break;
164988c4b8d1SNeel Natu 	case APIC_OFFSET_SVR:
165088c4b8d1SNeel Natu 		vlapic_svr_write_handler(vlapic);
165188c4b8d1SNeel Natu 		break;
165288c4b8d1SNeel Natu 	case APIC_OFFSET_ESR:
165388c4b8d1SNeel Natu 		vlapic_esr_write_handler(vlapic);
165488c4b8d1SNeel Natu 		break;
165588c4b8d1SNeel Natu 	case APIC_OFFSET_ICR_LOW:
165688c4b8d1SNeel Natu 		retu = false;
165788c4b8d1SNeel Natu 		error = vlapic_icrlo_write_handler(vlapic, &retu);
165888c4b8d1SNeel Natu 		if (error != 0 || retu)
1659a0efd3fbSJohn Baldwin 			handled = UNHANDLED;
166088c4b8d1SNeel Natu 		break;
166188c4b8d1SNeel Natu 	case APIC_OFFSET_CMCI_LVT:
166288c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
166388c4b8d1SNeel Natu 		vlapic_lvt_write_handler(vlapic, offset);
166488c4b8d1SNeel Natu 		break;
166588c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_ICR:
166688c4b8d1SNeel Natu 		vlapic_icrtmr_write_handler(vlapic);
166788c4b8d1SNeel Natu 		break;
166888c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_DCR:
166988c4b8d1SNeel Natu 		vlapic_dcr_write_handler(vlapic);
167088c4b8d1SNeel Natu 		break;
167188c4b8d1SNeel Natu 	default:
1672a0efd3fbSJohn Baldwin 		handled = UNHANDLED;
167388c4b8d1SNeel Natu 		break;
167488c4b8d1SNeel Natu 	}
167588c4b8d1SNeel Natu 	return (handled);
167688c4b8d1SNeel Natu }
167788c4b8d1SNeel Natu 
167888c4b8d1SNeel Natu static bool
1679*159dd56fSNeel Natu apic_access_fault(struct vmx *vmx, int vcpuid, uint64_t gpa)
168088c4b8d1SNeel Natu {
168188c4b8d1SNeel Natu 
1682*159dd56fSNeel Natu 	if (apic_access_virtualization(vmx, vcpuid) &&
168388c4b8d1SNeel Natu 	    (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE))
168488c4b8d1SNeel Natu 		return (true);
168588c4b8d1SNeel Natu 	else
168688c4b8d1SNeel Natu 		return (false);
168788c4b8d1SNeel Natu }
168888c4b8d1SNeel Natu 
168988c4b8d1SNeel Natu static int
169088c4b8d1SNeel Natu vmx_handle_apic_access(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
169188c4b8d1SNeel Natu {
169288c4b8d1SNeel Natu 	uint64_t qual;
169388c4b8d1SNeel Natu 	int access_type, offset, allowed;
169488c4b8d1SNeel Natu 
1695*159dd56fSNeel Natu 	if (!apic_access_virtualization(vmx, vcpuid))
169688c4b8d1SNeel Natu 		return (UNHANDLED);
169788c4b8d1SNeel Natu 
169888c4b8d1SNeel Natu 	qual = vmexit->u.vmx.exit_qualification;
169988c4b8d1SNeel Natu 	access_type = APIC_ACCESS_TYPE(qual);
170088c4b8d1SNeel Natu 	offset = APIC_ACCESS_OFFSET(qual);
170188c4b8d1SNeel Natu 
170288c4b8d1SNeel Natu 	allowed = 0;
170388c4b8d1SNeel Natu 	if (access_type == 0) {
170488c4b8d1SNeel Natu 		/*
170588c4b8d1SNeel Natu 		 * Read data access to the following registers is expected.
170688c4b8d1SNeel Natu 		 */
170788c4b8d1SNeel Natu 		switch (offset) {
170888c4b8d1SNeel Natu 		case APIC_OFFSET_APR:
170988c4b8d1SNeel Natu 		case APIC_OFFSET_PPR:
171088c4b8d1SNeel Natu 		case APIC_OFFSET_RRR:
171188c4b8d1SNeel Natu 		case APIC_OFFSET_CMCI_LVT:
171288c4b8d1SNeel Natu 		case APIC_OFFSET_TIMER_CCR:
171388c4b8d1SNeel Natu 			allowed = 1;
171488c4b8d1SNeel Natu 			break;
171588c4b8d1SNeel Natu 		default:
171688c4b8d1SNeel Natu 			break;
171788c4b8d1SNeel Natu 		}
171888c4b8d1SNeel Natu 	} else if (access_type == 1) {
171988c4b8d1SNeel Natu 		/*
172088c4b8d1SNeel Natu 		 * Write data access to the following registers is expected.
172188c4b8d1SNeel Natu 		 */
172288c4b8d1SNeel Natu 		switch (offset) {
172388c4b8d1SNeel Natu 		case APIC_OFFSET_VER:
172488c4b8d1SNeel Natu 		case APIC_OFFSET_APR:
172588c4b8d1SNeel Natu 		case APIC_OFFSET_PPR:
172688c4b8d1SNeel Natu 		case APIC_OFFSET_RRR:
172788c4b8d1SNeel Natu 		case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
172888c4b8d1SNeel Natu 		case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
172988c4b8d1SNeel Natu 		case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
173088c4b8d1SNeel Natu 		case APIC_OFFSET_CMCI_LVT:
173188c4b8d1SNeel Natu 		case APIC_OFFSET_TIMER_CCR:
173288c4b8d1SNeel Natu 			allowed = 1;
173388c4b8d1SNeel Natu 			break;
173488c4b8d1SNeel Natu 		default:
173588c4b8d1SNeel Natu 			break;
173688c4b8d1SNeel Natu 		}
173788c4b8d1SNeel Natu 	}
173888c4b8d1SNeel Natu 
173988c4b8d1SNeel Natu 	if (allowed) {
174088c4b8d1SNeel Natu 		vmexit->exitcode = VM_EXITCODE_INST_EMUL;
174188c4b8d1SNeel Natu 		vmexit->u.inst_emul.gpa = DEFAULT_APIC_BASE + offset;
174288c4b8d1SNeel Natu 		vmexit->u.inst_emul.gla = VIE_INVALID_GLA;
174388c4b8d1SNeel Natu 		vmexit->u.inst_emul.cr3 = vmcs_guest_cr3();
174400f3efe1SJohn Baldwin 		vmexit->u.inst_emul.cpu_mode = vmx_cpu_mode();
174500f3efe1SJohn Baldwin 		vmexit->u.inst_emul.paging_mode = vmx_paging_mode();
174688c4b8d1SNeel Natu 	}
174788c4b8d1SNeel Natu 
174888c4b8d1SNeel Natu 	/*
174988c4b8d1SNeel Natu 	 * Regardless of whether the APIC-access is allowed this handler
175088c4b8d1SNeel Natu 	 * always returns UNHANDLED:
175188c4b8d1SNeel Natu 	 * - if the access is allowed then it is handled by emulating the
175288c4b8d1SNeel Natu 	 *   instruction that caused the VM-exit (outside the critical section)
175388c4b8d1SNeel Natu 	 * - if the access is not allowed then it will be converted to an
175488c4b8d1SNeel Natu 	 *   exitcode of VM_EXITCODE_VMX and will be dealt with in userland.
175588c4b8d1SNeel Natu 	 */
175688c4b8d1SNeel Natu 	return (UNHANDLED);
175788c4b8d1SNeel Natu }
175888c4b8d1SNeel Natu 
175988c4b8d1SNeel Natu static int
1760366f6083SPeter Grehan vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1761366f6083SPeter Grehan {
1762f76fc5d4SNeel Natu 	int error, handled;
1763366f6083SPeter Grehan 	struct vmxctx *vmxctx;
176488c4b8d1SNeel Natu 	struct vlapic *vlapic;
1765e5a1d950SNeel Natu 	uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, reason;
17663de83862SNeel Natu 	uint64_t qual, gpa;
1767becd9849SNeel Natu 	bool retu;
1768366f6083SPeter Grehan 
1769160471d2SNeel Natu 	CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0);
1770c308b23bSNeel Natu 	CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0);
1771160471d2SNeel Natu 
1772a0efd3fbSJohn Baldwin 	handled = UNHANDLED;
1773366f6083SPeter Grehan 	vmxctx = &vmx->ctx[vcpu];
17740492757cSNeel Natu 
1775366f6083SPeter Grehan 	qual = vmexit->u.vmx.exit_qualification;
1776318224bbSNeel Natu 	reason = vmexit->u.vmx.exit_reason;
1777366f6083SPeter Grehan 	vmexit->exitcode = VM_EXITCODE_BOGUS;
1778366f6083SPeter Grehan 
177961592433SNeel Natu 	vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1);
178061592433SNeel Natu 
1781318224bbSNeel Natu 	/*
1782318224bbSNeel Natu 	 * VM exits that could be triggered during event injection on the
1783318224bbSNeel Natu 	 * previous VM entry need to be handled specially by re-injecting
1784318224bbSNeel Natu 	 * the event.
1785318224bbSNeel Natu 	 *
1786318224bbSNeel Natu 	 * See "Information for VM Exits During Event Delivery" in Intel SDM
1787318224bbSNeel Natu 	 * for details.
1788318224bbSNeel Natu 	 */
1789318224bbSNeel Natu 	switch (reason) {
1790318224bbSNeel Natu 	case EXIT_REASON_EPT_FAULT:
1791318224bbSNeel Natu 	case EXIT_REASON_EPT_MISCONFIG:
179288c4b8d1SNeel Natu 	case EXIT_REASON_APIC_ACCESS:
1793318224bbSNeel Natu 	case EXIT_REASON_TASK_SWITCH:
1794318224bbSNeel Natu 	case EXIT_REASON_EXCEPTION:
1795318224bbSNeel Natu 		idtvec_info = vmcs_idt_vectoring_info();
1796318224bbSNeel Natu 		if (idtvec_info & VMCS_IDT_VEC_VALID) {
1797318224bbSNeel Natu 			idtvec_info &= ~(1 << 12); /* clear undefined bit */
17983de83862SNeel Natu 			vmcs_write(VMCS_ENTRY_INTR_INFO, idtvec_info);
1799318224bbSNeel Natu 			if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
1800318224bbSNeel Natu 				idtvec_err = vmcs_idt_vectoring_err();
18013de83862SNeel Natu 				vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR,
18023de83862SNeel Natu 				    idtvec_err);
1803318224bbSNeel Natu 			}
1804160471d2SNeel Natu 			/*
1805160471d2SNeel Natu 			 * If 'virtual NMIs' are being used and the VM-exit
1806160471d2SNeel Natu 			 * happened while injecting an NMI during the previous
1807160471d2SNeel Natu 			 * VM-entry, then clear "blocking by NMI" in the Guest
1808160471d2SNeel Natu 			 * Interruptibility-state.
1809160471d2SNeel Natu 			 */
1810160471d2SNeel Natu 			if ((idtvec_info & VMCS_INTR_T_MASK) ==
1811160471d2SNeel Natu 			    VMCS_INTR_T_NMI) {
1812e5a1d950SNeel Natu 				 vmx_clear_nmi_blocking(vmx, vcpu);
1813160471d2SNeel Natu 			}
18143de83862SNeel Natu 			vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
1815318224bbSNeel Natu 		}
1816318224bbSNeel Natu 	default:
1817e5a1d950SNeel Natu 		idtvec_info = 0;
1818318224bbSNeel Natu 		break;
1819318224bbSNeel Natu 	}
1820318224bbSNeel Natu 
1821318224bbSNeel Natu 	switch (reason) {
1822366f6083SPeter Grehan 	case EXIT_REASON_CR_ACCESS:
1823b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CR_ACCESS, 1);
1824366f6083SPeter Grehan 		handled = vmx_emulate_cr_access(vmx, vcpu, qual);
1825366f6083SPeter Grehan 		break;
1826366f6083SPeter Grehan 	case EXIT_REASON_RDMSR:
1827b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RDMSR, 1);
1828becd9849SNeel Natu 		retu = false;
1829366f6083SPeter Grehan 		ecx = vmxctx->guest_rcx;
1830becd9849SNeel Natu 		error = emulate_rdmsr(vmx->vm, vcpu, ecx, &retu);
1831b42206f3SNeel Natu 		if (error) {
1832366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_RDMSR;
1833366f6083SPeter Grehan 			vmexit->u.msr.code = ecx;
1834becd9849SNeel Natu 		} else if (!retu) {
1835a0efd3fbSJohn Baldwin 			handled = HANDLED;
1836becd9849SNeel Natu 		} else {
1837becd9849SNeel Natu 			/* Return to userspace with a valid exitcode */
1838becd9849SNeel Natu 			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
1839becd9849SNeel Natu 			    ("emulate_wrmsr retu with bogus exitcode"));
1840becd9849SNeel Natu 		}
1841366f6083SPeter Grehan 		break;
1842366f6083SPeter Grehan 	case EXIT_REASON_WRMSR:
1843b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_WRMSR, 1);
1844becd9849SNeel Natu 		retu = false;
1845366f6083SPeter Grehan 		eax = vmxctx->guest_rax;
1846366f6083SPeter Grehan 		ecx = vmxctx->guest_rcx;
1847366f6083SPeter Grehan 		edx = vmxctx->guest_rdx;
1848b42206f3SNeel Natu 		error = emulate_wrmsr(vmx->vm, vcpu, ecx,
1849becd9849SNeel Natu 		    (uint64_t)edx << 32 | eax, &retu);
1850b42206f3SNeel Natu 		if (error) {
1851366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_WRMSR;
1852366f6083SPeter Grehan 			vmexit->u.msr.code = ecx;
1853366f6083SPeter Grehan 			vmexit->u.msr.wval = (uint64_t)edx << 32 | eax;
1854becd9849SNeel Natu 		} else if (!retu) {
1855a0efd3fbSJohn Baldwin 			handled = HANDLED;
1856becd9849SNeel Natu 		} else {
1857becd9849SNeel Natu 			/* Return to userspace with a valid exitcode */
1858becd9849SNeel Natu 			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
1859becd9849SNeel Natu 			    ("emulate_wrmsr retu with bogus exitcode"));
1860becd9849SNeel Natu 		}
1861366f6083SPeter Grehan 		break;
1862366f6083SPeter Grehan 	case EXIT_REASON_HLT:
1863f76fc5d4SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_HLT, 1);
1864366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_HLT;
18653de83862SNeel Natu 		vmexit->u.hlt.rflags = vmcs_read(VMCS_GUEST_RFLAGS);
1866366f6083SPeter Grehan 		break;
1867366f6083SPeter Grehan 	case EXIT_REASON_MTF:
1868b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_MTRAP, 1);
1869366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_MTRAP;
1870366f6083SPeter Grehan 		break;
1871366f6083SPeter Grehan 	case EXIT_REASON_PAUSE:
1872b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_PAUSE, 1);
1873366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_PAUSE;
1874366f6083SPeter Grehan 		break;
1875366f6083SPeter Grehan 	case EXIT_REASON_INTR_WINDOW:
1876b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INTR_WINDOW, 1);
1877366f6083SPeter Grehan 		vmx_clear_int_window_exiting(vmx, vcpu);
1878b5aaf7b2SNeel Natu 		return (1);
1879366f6083SPeter Grehan 	case EXIT_REASON_EXT_INTR:
1880366f6083SPeter Grehan 		/*
1881366f6083SPeter Grehan 		 * External interrupts serve only to cause VM exits and allow
1882366f6083SPeter Grehan 		 * the host interrupt handler to run.
1883366f6083SPeter Grehan 		 *
1884366f6083SPeter Grehan 		 * If this external interrupt triggers a virtual interrupt
1885366f6083SPeter Grehan 		 * to a VM, then that state will be recorded by the
1886366f6083SPeter Grehan 		 * host interrupt handler in the VM's softc. We will inject
1887366f6083SPeter Grehan 		 * this virtual interrupt during the subsequent VM enter.
1888366f6083SPeter Grehan 		 */
1889f7d47425SNeel Natu 		intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
1890160471d2SNeel Natu 		KASSERT((intr_info & VMCS_INTR_VALID) != 0 &&
1891160471d2SNeel Natu 		    (intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_HWINTR,
1892f7d47425SNeel Natu 		    ("VM exit interruption info invalid: %#x", intr_info));
1893f7d47425SNeel Natu 		vmx_trigger_hostintr(intr_info & 0xff);
1894366f6083SPeter Grehan 
1895366f6083SPeter Grehan 		/*
1896366f6083SPeter Grehan 		 * This is special. We want to treat this as an 'handled'
1897366f6083SPeter Grehan 		 * VM-exit but not increment the instruction pointer.
1898366f6083SPeter Grehan 		 */
1899366f6083SPeter Grehan 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1);
1900366f6083SPeter Grehan 		return (1);
1901366f6083SPeter Grehan 	case EXIT_REASON_NMI_WINDOW:
1902366f6083SPeter Grehan 		/* Exit to allow the pending virtual NMI to be injected */
190348b2d828SNeel Natu 		if (vm_nmi_pending(vmx->vm, vcpu))
190448b2d828SNeel Natu 			vmx_inject_nmi(vmx, vcpu);
1905366f6083SPeter Grehan 		vmx_clear_nmi_window_exiting(vmx, vcpu);
190648b2d828SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NMI_WINDOW, 1);
1907366f6083SPeter Grehan 		return (1);
1908366f6083SPeter Grehan 	case EXIT_REASON_INOUT:
1909b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INOUT, 1);
1910366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_INOUT;
1911366f6083SPeter Grehan 		vmexit->u.inout.bytes = (qual & 0x7) + 1;
1912366f6083SPeter Grehan 		vmexit->u.inout.in = (qual & 0x8) ? 1 : 0;
1913366f6083SPeter Grehan 		vmexit->u.inout.string = (qual & 0x10) ? 1 : 0;
1914366f6083SPeter Grehan 		vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0;
1915366f6083SPeter Grehan 		vmexit->u.inout.port = (uint16_t)(qual >> 16);
1916366f6083SPeter Grehan 		vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax);
1917366f6083SPeter Grehan 		break;
1918366f6083SPeter Grehan 	case EXIT_REASON_CPUID:
1919b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CPUID, 1);
1920a2da7af6SNeel Natu 		handled = vmx_handle_cpuid(vmx->vm, vcpu, vmxctx);
1921366f6083SPeter Grehan 		break;
1922e5a1d950SNeel Natu 	case EXIT_REASON_EXCEPTION:
1923c308b23bSNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXCEPTION, 1);
1924e5a1d950SNeel Natu 		intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
1925e5a1d950SNeel Natu 		KASSERT((intr_info & VMCS_INTR_VALID) != 0,
1926e5a1d950SNeel Natu 		    ("VM exit interruption info invalid: %#x", intr_info));
1927c308b23bSNeel Natu 
1928e5a1d950SNeel Natu 		/*
1929e5a1d950SNeel Natu 		 * If Virtual NMIs control is 1 and the VM-exit is due to a
1930e5a1d950SNeel Natu 		 * fault encountered during the execution of IRET then we must
1931e5a1d950SNeel Natu 		 * restore the state of "virtual-NMI blocking" before resuming
1932e5a1d950SNeel Natu 		 * the guest.
1933e5a1d950SNeel Natu 		 *
1934e5a1d950SNeel Natu 		 * See "Resuming Guest Software after Handling an Exception".
1935e5a1d950SNeel Natu 		 */
1936e5a1d950SNeel Natu 		if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
1937e5a1d950SNeel Natu 		    (intr_info & 0xff) != IDT_DF &&
1938e5a1d950SNeel Natu 		    (intr_info & EXIT_QUAL_NMIUDTI) != 0)
1939e5a1d950SNeel Natu 			vmx_restore_nmi_blocking(vmx, vcpu);
1940c308b23bSNeel Natu 
1941c308b23bSNeel Natu 		/*
194262fbd7c2SNeel Natu 		 * The NMI has already been handled in vmx_exit_handle_nmi().
1943c308b23bSNeel Natu 		 */
194462fbd7c2SNeel Natu 		if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI)
1945c308b23bSNeel Natu 			return (1);
1946e5a1d950SNeel Natu 		break;
1947cd942e0fSPeter Grehan 	case EXIT_REASON_EPT_FAULT:
1948318224bbSNeel Natu 		/*
1949318224bbSNeel Natu 		 * If 'gpa' lies within the address space allocated to
1950318224bbSNeel Natu 		 * memory then this must be a nested page fault otherwise
1951318224bbSNeel Natu 		 * this must be an instruction that accesses MMIO space.
1952318224bbSNeel Natu 		 */
1953a2da7af6SNeel Natu 		gpa = vmcs_gpa();
1954*159dd56fSNeel Natu 		if (vm_mem_allocated(vmx->vm, gpa) ||
1955*159dd56fSNeel Natu 		    apic_access_fault(vmx, vcpu, gpa)) {
1956cd942e0fSPeter Grehan 			vmexit->exitcode = VM_EXITCODE_PAGING;
195713ec9371SPeter Grehan 			vmexit->u.paging.gpa = gpa;
1958318224bbSNeel Natu 			vmexit->u.paging.fault_type = ept_fault_type(qual);
1959bf73979dSNeel Natu 			vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NESTED_FAULT, 1);
1960318224bbSNeel Natu 		} else if (ept_emulation_fault(qual)) {
1961318224bbSNeel Natu 			vmexit->exitcode = VM_EXITCODE_INST_EMUL;
1962318224bbSNeel Natu 			vmexit->u.inst_emul.gpa = gpa;
1963318224bbSNeel Natu 			vmexit->u.inst_emul.gla = vmcs_gla();
1964318224bbSNeel Natu 			vmexit->u.inst_emul.cr3 = vmcs_guest_cr3();
196500f3efe1SJohn Baldwin 			vmexit->u.inst_emul.cpu_mode = vmx_cpu_mode();
196600f3efe1SJohn Baldwin 			vmexit->u.inst_emul.paging_mode = vmx_paging_mode();
1967bf73979dSNeel Natu 			vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INST_EMUL, 1);
1968a2da7af6SNeel Natu 		}
1969e5a1d950SNeel Natu 		/*
1970e5a1d950SNeel Natu 		 * If Virtual NMIs control is 1 and the VM-exit is due to an
1971e5a1d950SNeel Natu 		 * EPT fault during the execution of IRET then we must restore
1972e5a1d950SNeel Natu 		 * the state of "virtual-NMI blocking" before resuming.
1973e5a1d950SNeel Natu 		 *
1974e5a1d950SNeel Natu 		 * See description of "NMI unblocking due to IRET" in
1975e5a1d950SNeel Natu 		 * "Exit Qualification for EPT Violations".
1976e5a1d950SNeel Natu 		 */
1977e5a1d950SNeel Natu 		if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
1978e5a1d950SNeel Natu 		    (qual & EXIT_QUAL_NMIUDTI) != 0)
1979e5a1d950SNeel Natu 			vmx_restore_nmi_blocking(vmx, vcpu);
1980cd942e0fSPeter Grehan 		break;
198130b94db8SNeel Natu 	case EXIT_REASON_VIRTUALIZED_EOI:
198230b94db8SNeel Natu 		vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI;
198330b94db8SNeel Natu 		vmexit->u.ioapic_eoi.vector = qual & 0xFF;
198430b94db8SNeel Natu 		vmexit->inst_length = 0;	/* trap-like */
198530b94db8SNeel Natu 		break;
198688c4b8d1SNeel Natu 	case EXIT_REASON_APIC_ACCESS:
198788c4b8d1SNeel Natu 		handled = vmx_handle_apic_access(vmx, vcpu, vmexit);
198888c4b8d1SNeel Natu 		break;
198988c4b8d1SNeel Natu 	case EXIT_REASON_APIC_WRITE:
199088c4b8d1SNeel Natu 		/*
199188c4b8d1SNeel Natu 		 * APIC-write VM exit is trap-like so the %rip is already
199288c4b8d1SNeel Natu 		 * pointing to the next instruction.
199388c4b8d1SNeel Natu 		 */
199488c4b8d1SNeel Natu 		vmexit->inst_length = 0;
199588c4b8d1SNeel Natu 		vlapic = vm_lapic(vmx->vm, vcpu);
1996*159dd56fSNeel Natu 		handled = vmx_handle_apic_write(vmx, vcpu, vlapic, qual);
199788c4b8d1SNeel Natu 		break;
1998abb023fbSJohn Baldwin 	case EXIT_REASON_XSETBV:
1999a0efd3fbSJohn Baldwin 		handled = vmx_emulate_xsetbv(vmx, vcpu, vmexit);
2000abb023fbSJohn Baldwin 		break;
2001366f6083SPeter Grehan 	default:
2002b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_UNKNOWN, 1);
2003366f6083SPeter Grehan 		break;
2004366f6083SPeter Grehan 	}
2005366f6083SPeter Grehan 
2006366f6083SPeter Grehan 	if (handled) {
2007366f6083SPeter Grehan 		/*
2008366f6083SPeter Grehan 		 * It is possible that control is returned to userland
2009366f6083SPeter Grehan 		 * even though we were able to handle the VM exit in the
2010eeefa4e4SNeel Natu 		 * kernel.
2011366f6083SPeter Grehan 		 *
2012366f6083SPeter Grehan 		 * In such a case we want to make sure that the userland
2013366f6083SPeter Grehan 		 * restarts guest execution at the instruction *after*
2014366f6083SPeter Grehan 		 * the one we just processed. Therefore we update the
2015366f6083SPeter Grehan 		 * guest rip in the VMCS and in 'vmexit'.
2016366f6083SPeter Grehan 		 */
2017366f6083SPeter Grehan 		vmexit->rip += vmexit->inst_length;
2018366f6083SPeter Grehan 		vmexit->inst_length = 0;
20193de83862SNeel Natu 		vmcs_write(VMCS_GUEST_RIP, vmexit->rip);
2020366f6083SPeter Grehan 	} else {
2021366f6083SPeter Grehan 		if (vmexit->exitcode == VM_EXITCODE_BOGUS) {
2022366f6083SPeter Grehan 			/*
2023366f6083SPeter Grehan 			 * If this VM exit was not claimed by anybody then
2024366f6083SPeter Grehan 			 * treat it as a generic VMX exit.
2025366f6083SPeter Grehan 			 */
2026366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_VMX;
20270492757cSNeel Natu 			vmexit->u.vmx.status = VM_SUCCESS;
2028c308b23bSNeel Natu 			vmexit->u.vmx.inst_type = 0;
2029c308b23bSNeel Natu 			vmexit->u.vmx.inst_error = 0;
2030366f6083SPeter Grehan 		} else {
2031366f6083SPeter Grehan 			/*
2032366f6083SPeter Grehan 			 * The exitcode and collateral have been populated.
2033366f6083SPeter Grehan 			 * The VM exit will be processed further in userland.
2034366f6083SPeter Grehan 			 */
2035366f6083SPeter Grehan 		}
2036366f6083SPeter Grehan 	}
2037366f6083SPeter Grehan 	return (handled);
2038366f6083SPeter Grehan }
2039366f6083SPeter Grehan 
20400492757cSNeel Natu static __inline int
20410492757cSNeel Natu vmx_exit_astpending(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
2042366f6083SPeter Grehan {
20430492757cSNeel Natu 
20440492757cSNeel Natu 	vmexit->rip = vmcs_guest_rip();
20450492757cSNeel Natu 	vmexit->inst_length = 0;
20460492757cSNeel Natu 	vmexit->exitcode = VM_EXITCODE_BOGUS;
20470492757cSNeel Natu 	vmx_astpending_trace(vmx, vcpu, vmexit->rip);
20480492757cSNeel Natu 	vmm_stat_incr(vmx->vm, vcpu, VMEXIT_ASTPENDING, 1);
20490492757cSNeel Natu 
20500492757cSNeel Natu 	return (HANDLED);
20510492757cSNeel Natu }
20520492757cSNeel Natu 
20530492757cSNeel Natu static __inline int
20545b8a8cd1SNeel Natu vmx_exit_rendezvous(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
20555b8a8cd1SNeel Natu {
20565b8a8cd1SNeel Natu 
20575b8a8cd1SNeel Natu 	vmexit->rip = vmcs_guest_rip();
20585b8a8cd1SNeel Natu 	vmexit->inst_length = 0;
20595b8a8cd1SNeel Natu 	vmexit->exitcode = VM_EXITCODE_RENDEZVOUS;
20605b8a8cd1SNeel Natu 	vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RENDEZVOUS, 1);
20615b8a8cd1SNeel Natu 
20625b8a8cd1SNeel Natu 	return (UNHANDLED);
20635b8a8cd1SNeel Natu }
20645b8a8cd1SNeel Natu 
20655b8a8cd1SNeel Natu static __inline int
20660492757cSNeel Natu vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit)
20670492757cSNeel Natu {
20680492757cSNeel Natu 
20690492757cSNeel Natu 	KASSERT(vmxctx->inst_fail_status != VM_SUCCESS,
20700492757cSNeel Natu 	    ("vmx_exit_inst_error: invalid inst_fail_status %d",
20710492757cSNeel Natu 	    vmxctx->inst_fail_status));
20720492757cSNeel Natu 
20730492757cSNeel Natu 	vmexit->inst_length = 0;
20740492757cSNeel Natu 	vmexit->exitcode = VM_EXITCODE_VMX;
20750492757cSNeel Natu 	vmexit->u.vmx.status = vmxctx->inst_fail_status;
20760492757cSNeel Natu 	vmexit->u.vmx.inst_error = vmcs_instruction_error();
20770492757cSNeel Natu 	vmexit->u.vmx.exit_reason = ~0;
20780492757cSNeel Natu 	vmexit->u.vmx.exit_qualification = ~0;
20790492757cSNeel Natu 
20800492757cSNeel Natu 	switch (rc) {
20810492757cSNeel Natu 	case VMX_VMRESUME_ERROR:
20820492757cSNeel Natu 	case VMX_VMLAUNCH_ERROR:
20830492757cSNeel Natu 	case VMX_INVEPT_ERROR:
20840492757cSNeel Natu 		vmexit->u.vmx.inst_type = rc;
20850492757cSNeel Natu 		break;
20860492757cSNeel Natu 	default:
20870492757cSNeel Natu 		panic("vm_exit_inst_error: vmx_enter_guest returned %d", rc);
20880492757cSNeel Natu 	}
20890492757cSNeel Natu 
20900492757cSNeel Natu 	return (UNHANDLED);
20910492757cSNeel Natu }
20920492757cSNeel Natu 
209362fbd7c2SNeel Natu /*
209462fbd7c2SNeel Natu  * If the NMI-exiting VM execution control is set to '1' then an NMI in
209562fbd7c2SNeel Natu  * non-root operation causes a VM-exit. NMI blocking is in effect so it is
209662fbd7c2SNeel Natu  * sufficient to simply vector to the NMI handler via a software interrupt.
209762fbd7c2SNeel Natu  * However, this must be done before maskable interrupts are enabled
209862fbd7c2SNeel Natu  * otherwise the "iret" issued by an interrupt handler will incorrectly
209962fbd7c2SNeel Natu  * clear NMI blocking.
210062fbd7c2SNeel Natu  */
210162fbd7c2SNeel Natu static __inline void
210262fbd7c2SNeel Natu vmx_exit_handle_nmi(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
210362fbd7c2SNeel Natu {
210462fbd7c2SNeel Natu 	uint32_t intr_info;
210562fbd7c2SNeel Natu 
210662fbd7c2SNeel Natu 	KASSERT((read_rflags() & PSL_I) == 0, ("interrupts enabled"));
210762fbd7c2SNeel Natu 
210862fbd7c2SNeel Natu 	if (vmexit->u.vmx.exit_reason != EXIT_REASON_EXCEPTION)
210962fbd7c2SNeel Natu 		return;
211062fbd7c2SNeel Natu 
211162fbd7c2SNeel Natu 	intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
211262fbd7c2SNeel Natu 	KASSERT((intr_info & VMCS_INTR_VALID) != 0,
211362fbd7c2SNeel Natu 	    ("VM exit interruption info invalid: %#x", intr_info));
211462fbd7c2SNeel Natu 
211562fbd7c2SNeel Natu 	if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI) {
211662fbd7c2SNeel Natu 		KASSERT((intr_info & 0xff) == IDT_NMI, ("VM exit due "
211762fbd7c2SNeel Natu 		    "to NMI has invalid vector: %#x", intr_info));
211862fbd7c2SNeel Natu 		VCPU_CTR0(vmx->vm, vcpuid, "Vectoring to NMI handler");
211962fbd7c2SNeel Natu 		__asm __volatile("int $2");
212062fbd7c2SNeel Natu 	}
212162fbd7c2SNeel Natu }
212262fbd7c2SNeel Natu 
21230492757cSNeel Natu static int
21245b8a8cd1SNeel Natu vmx_run(void *arg, int vcpu, register_t startrip, pmap_t pmap,
21255b8a8cd1SNeel Natu     void *rendezvous_cookie)
21260492757cSNeel Natu {
21270492757cSNeel Natu 	int rc, handled, launched;
2128366f6083SPeter Grehan 	struct vmx *vmx;
21295b8a8cd1SNeel Natu 	struct vm *vm;
2130366f6083SPeter Grehan 	struct vmxctx *vmxctx;
2131366f6083SPeter Grehan 	struct vmcs *vmcs;
213298ed632cSNeel Natu 	struct vm_exit *vmexit;
2133de5ea6b6SNeel Natu 	struct vlapic *vlapic;
213479c59630SNeel Natu 	uint64_t rip;
213579c59630SNeel Natu 	uint32_t exit_reason;
2136366f6083SPeter Grehan 
2137366f6083SPeter Grehan 	vmx = arg;
21385b8a8cd1SNeel Natu 	vm = vmx->vm;
2139366f6083SPeter Grehan 	vmcs = &vmx->vmcs[vcpu];
2140366f6083SPeter Grehan 	vmxctx = &vmx->ctx[vcpu];
21415b8a8cd1SNeel Natu 	vlapic = vm_lapic(vm, vcpu);
21425b8a8cd1SNeel Natu 	vmexit = vm_exitinfo(vm, vcpu);
21430492757cSNeel Natu 	launched = 0;
214498ed632cSNeel Natu 
2145318224bbSNeel Natu 	KASSERT(vmxctx->pmap == pmap,
2146318224bbSNeel Natu 	    ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap));
2147318224bbSNeel Natu 
2148366f6083SPeter Grehan 	VMPTRLD(vmcs);
2149366f6083SPeter Grehan 
2150366f6083SPeter Grehan 	/*
2151366f6083SPeter Grehan 	 * XXX
2152366f6083SPeter Grehan 	 * We do this every time because we may setup the virtual machine
2153366f6083SPeter Grehan 	 * from a different process than the one that actually runs it.
2154366f6083SPeter Grehan 	 *
2155366f6083SPeter Grehan 	 * If the life of a virtual machine was spent entirely in the context
2156c847a506SNeel Natu 	 * of a single process we could do this once in vmx_vminit().
2157366f6083SPeter Grehan 	 */
21583de83862SNeel Natu 	vmcs_write(VMCS_HOST_CR3, rcr3());
2159366f6083SPeter Grehan 
21600492757cSNeel Natu 	vmcs_write(VMCS_GUEST_RIP, startrip);
2161953c2c47SNeel Natu 	vmx_set_pcpu_defaults(vmx, vcpu, pmap);
2162366f6083SPeter Grehan 	do {
21630492757cSNeel Natu 		/*
21640492757cSNeel Natu 		 * Interrupts are disabled from this point on until the
21650492757cSNeel Natu 		 * guest starts executing. This is done for the following
21660492757cSNeel Natu 		 * reasons:
21670492757cSNeel Natu 		 *
21680492757cSNeel Natu 		 * If an AST is asserted on this thread after the check below,
21690492757cSNeel Natu 		 * then the IPI_AST notification will not be lost, because it
21700492757cSNeel Natu 		 * will cause a VM exit due to external interrupt as soon as
21710492757cSNeel Natu 		 * the guest state is loaded.
21720492757cSNeel Natu 		 *
21730492757cSNeel Natu 		 * A posted interrupt after 'vmx_inject_interrupts()' will
21740492757cSNeel Natu 		 * not be "lost" because it will be held pending in the host
21750492757cSNeel Natu 		 * APIC because interrupts are disabled. The pending interrupt
21760492757cSNeel Natu 		 * will be recognized as soon as the guest state is loaded.
21770492757cSNeel Natu 		 *
21780492757cSNeel Natu 		 * The same reasoning applies to the IPI generated by
21790492757cSNeel Natu 		 * pmap_invalidate_ept().
21800492757cSNeel Natu 		 */
21810492757cSNeel Natu 		disable_intr();
21820492757cSNeel Natu 		if (curthread->td_flags & (TDF_ASTPENDING | TDF_NEEDRESCHED)) {
21830492757cSNeel Natu 			enable_intr();
21840492757cSNeel Natu 			handled = vmx_exit_astpending(vmx, vcpu, vmexit);
21850492757cSNeel Natu 			break;
21860492757cSNeel Natu 		}
21870492757cSNeel Natu 
21885b8a8cd1SNeel Natu 		if (vcpu_rendezvous_pending(rendezvous_cookie)) {
21895b8a8cd1SNeel Natu 			enable_intr();
21905b8a8cd1SNeel Natu 			handled = vmx_exit_rendezvous(vmx, vcpu, vmexit);
21915b8a8cd1SNeel Natu 			break;
21925b8a8cd1SNeel Natu 		}
21935b8a8cd1SNeel Natu 
2194de5ea6b6SNeel Natu 		vmx_inject_interrupts(vmx, vcpu, vlapic);
2195366f6083SPeter Grehan 		vmx_run_trace(vmx, vcpu);
2196953c2c47SNeel Natu 		rc = vmx_enter_guest(vmxctx, vmx, launched);
219779c59630SNeel Natu 
219879c59630SNeel Natu 		/* Collect some information for VM exit processing */
219979c59630SNeel Natu 		vmexit->rip = rip = vmcs_guest_rip();
220079c59630SNeel Natu 		vmexit->inst_length = vmexit_instruction_length();
220179c59630SNeel Natu 		vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason();
220279c59630SNeel Natu 		vmexit->u.vmx.exit_qualification = vmcs_exit_qualification();
220379c59630SNeel Natu 
22040492757cSNeel Natu 		if (rc == VMX_GUEST_VMEXIT) {
220562fbd7c2SNeel Natu 			vmx_exit_handle_nmi(vmx, vcpu, vmexit);
220662fbd7c2SNeel Natu 			enable_intr();
22070492757cSNeel Natu 			handled = vmx_exit_process(vmx, vcpu, vmexit);
22080492757cSNeel Natu 		} else {
220962fbd7c2SNeel Natu 			enable_intr();
22100492757cSNeel Natu 			handled = vmx_exit_inst_error(vmxctx, rc, vmexit);
2211eeefa4e4SNeel Natu 		}
221262fbd7c2SNeel Natu 		launched = 1;
221379c59630SNeel Natu 		vmx_exit_trace(vmx, vcpu, rip, exit_reason, handled);
2214eeefa4e4SNeel Natu 	} while (handled);
2215366f6083SPeter Grehan 
2216366f6083SPeter Grehan 	/*
2217366f6083SPeter Grehan 	 * If a VM exit has been handled then the exitcode must be BOGUS
2218366f6083SPeter Grehan 	 * If a VM exit is not handled then the exitcode must not be BOGUS
2219366f6083SPeter Grehan 	 */
2220366f6083SPeter Grehan 	if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) ||
2221366f6083SPeter Grehan 	    (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) {
2222366f6083SPeter Grehan 		panic("Mismatch between handled (%d) and exitcode (%d)",
2223366f6083SPeter Grehan 		      handled, vmexit->exitcode);
2224366f6083SPeter Grehan 	}
2225366f6083SPeter Grehan 
2226b5aaf7b2SNeel Natu 	if (!handled)
22275b8a8cd1SNeel Natu 		vmm_stat_incr(vm, vcpu, VMEXIT_USERSPACE, 1);
2228b5aaf7b2SNeel Natu 
22295b8a8cd1SNeel Natu 	VCPU_CTR1(vm, vcpu, "returning from vmx_run: exitcode %d",
22300492757cSNeel Natu 	    vmexit->exitcode);
2231366f6083SPeter Grehan 
2232366f6083SPeter Grehan 	VMCLEAR(vmcs);
2233366f6083SPeter Grehan 	return (0);
2234366f6083SPeter Grehan }
2235366f6083SPeter Grehan 
2236366f6083SPeter Grehan static void
2237366f6083SPeter Grehan vmx_vmcleanup(void *arg)
2238366f6083SPeter Grehan {
223945e51299SNeel Natu 	int i, error;
2240366f6083SPeter Grehan 	struct vmx *vmx = arg;
2241366f6083SPeter Grehan 
2242*159dd56fSNeel Natu 	if (apic_access_virtualization(vmx, 0))
224388c4b8d1SNeel Natu 		vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
224488c4b8d1SNeel Natu 
224545e51299SNeel Natu 	for (i = 0; i < VM_MAXCPU; i++)
224645e51299SNeel Natu 		vpid_free(vmx->state[i].vpid);
224745e51299SNeel Natu 
2248366f6083SPeter Grehan 	/*
2249366f6083SPeter Grehan 	 * XXXSMP we also need to clear the VMCS active on the other vcpus.
2250366f6083SPeter Grehan 	 */
2251366f6083SPeter Grehan 	error = vmclear(&vmx->vmcs[0]);
2252366f6083SPeter Grehan 	if (error != 0)
2253366f6083SPeter Grehan 		panic("vmx_vmcleanup: vmclear error %d on vcpu 0", error);
2254366f6083SPeter Grehan 
2255366f6083SPeter Grehan 	free(vmx, M_VMX);
2256366f6083SPeter Grehan 
2257366f6083SPeter Grehan 	return;
2258366f6083SPeter Grehan }
2259366f6083SPeter Grehan 
2260366f6083SPeter Grehan static register_t *
2261366f6083SPeter Grehan vmxctx_regptr(struct vmxctx *vmxctx, int reg)
2262366f6083SPeter Grehan {
2263366f6083SPeter Grehan 
2264366f6083SPeter Grehan 	switch (reg) {
2265366f6083SPeter Grehan 	case VM_REG_GUEST_RAX:
2266366f6083SPeter Grehan 		return (&vmxctx->guest_rax);
2267366f6083SPeter Grehan 	case VM_REG_GUEST_RBX:
2268366f6083SPeter Grehan 		return (&vmxctx->guest_rbx);
2269366f6083SPeter Grehan 	case VM_REG_GUEST_RCX:
2270366f6083SPeter Grehan 		return (&vmxctx->guest_rcx);
2271366f6083SPeter Grehan 	case VM_REG_GUEST_RDX:
2272366f6083SPeter Grehan 		return (&vmxctx->guest_rdx);
2273366f6083SPeter Grehan 	case VM_REG_GUEST_RSI:
2274366f6083SPeter Grehan 		return (&vmxctx->guest_rsi);
2275366f6083SPeter Grehan 	case VM_REG_GUEST_RDI:
2276366f6083SPeter Grehan 		return (&vmxctx->guest_rdi);
2277366f6083SPeter Grehan 	case VM_REG_GUEST_RBP:
2278366f6083SPeter Grehan 		return (&vmxctx->guest_rbp);
2279366f6083SPeter Grehan 	case VM_REG_GUEST_R8:
2280366f6083SPeter Grehan 		return (&vmxctx->guest_r8);
2281366f6083SPeter Grehan 	case VM_REG_GUEST_R9:
2282366f6083SPeter Grehan 		return (&vmxctx->guest_r9);
2283366f6083SPeter Grehan 	case VM_REG_GUEST_R10:
2284366f6083SPeter Grehan 		return (&vmxctx->guest_r10);
2285366f6083SPeter Grehan 	case VM_REG_GUEST_R11:
2286366f6083SPeter Grehan 		return (&vmxctx->guest_r11);
2287366f6083SPeter Grehan 	case VM_REG_GUEST_R12:
2288366f6083SPeter Grehan 		return (&vmxctx->guest_r12);
2289366f6083SPeter Grehan 	case VM_REG_GUEST_R13:
2290366f6083SPeter Grehan 		return (&vmxctx->guest_r13);
2291366f6083SPeter Grehan 	case VM_REG_GUEST_R14:
2292366f6083SPeter Grehan 		return (&vmxctx->guest_r14);
2293366f6083SPeter Grehan 	case VM_REG_GUEST_R15:
2294366f6083SPeter Grehan 		return (&vmxctx->guest_r15);
2295366f6083SPeter Grehan 	default:
2296366f6083SPeter Grehan 		break;
2297366f6083SPeter Grehan 	}
2298366f6083SPeter Grehan 	return (NULL);
2299366f6083SPeter Grehan }
2300366f6083SPeter Grehan 
2301366f6083SPeter Grehan static int
2302366f6083SPeter Grehan vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval)
2303366f6083SPeter Grehan {
2304366f6083SPeter Grehan 	register_t *regp;
2305366f6083SPeter Grehan 
2306366f6083SPeter Grehan 	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
2307366f6083SPeter Grehan 		*retval = *regp;
2308366f6083SPeter Grehan 		return (0);
2309366f6083SPeter Grehan 	} else
2310366f6083SPeter Grehan 		return (EINVAL);
2311366f6083SPeter Grehan }
2312366f6083SPeter Grehan 
2313366f6083SPeter Grehan static int
2314366f6083SPeter Grehan vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val)
2315366f6083SPeter Grehan {
2316366f6083SPeter Grehan 	register_t *regp;
2317366f6083SPeter Grehan 
2318366f6083SPeter Grehan 	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
2319366f6083SPeter Grehan 		*regp = val;
2320366f6083SPeter Grehan 		return (0);
2321366f6083SPeter Grehan 	} else
2322366f6083SPeter Grehan 		return (EINVAL);
2323366f6083SPeter Grehan }
2324366f6083SPeter Grehan 
2325366f6083SPeter Grehan static int
2326aaaa0656SPeter Grehan vmx_shadow_reg(int reg)
2327aaaa0656SPeter Grehan {
2328aaaa0656SPeter Grehan 	int shreg;
2329aaaa0656SPeter Grehan 
2330aaaa0656SPeter Grehan 	shreg = -1;
2331aaaa0656SPeter Grehan 
2332aaaa0656SPeter Grehan 	switch (reg) {
2333aaaa0656SPeter Grehan 	case VM_REG_GUEST_CR0:
2334aaaa0656SPeter Grehan 		shreg = VMCS_CR0_SHADOW;
2335aaaa0656SPeter Grehan                 break;
2336aaaa0656SPeter Grehan         case VM_REG_GUEST_CR4:
2337aaaa0656SPeter Grehan 		shreg = VMCS_CR4_SHADOW;
2338aaaa0656SPeter Grehan 		break;
2339aaaa0656SPeter Grehan 	default:
2340aaaa0656SPeter Grehan 		break;
2341aaaa0656SPeter Grehan 	}
2342aaaa0656SPeter Grehan 
2343aaaa0656SPeter Grehan 	return (shreg);
2344aaaa0656SPeter Grehan }
2345aaaa0656SPeter Grehan 
2346aaaa0656SPeter Grehan static int
2347366f6083SPeter Grehan vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval)
2348366f6083SPeter Grehan {
2349d3c11f40SPeter Grehan 	int running, hostcpu;
2350366f6083SPeter Grehan 	struct vmx *vmx = arg;
2351366f6083SPeter Grehan 
2352d3c11f40SPeter Grehan 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
2353d3c11f40SPeter Grehan 	if (running && hostcpu != curcpu)
2354d3c11f40SPeter Grehan 		panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu);
2355d3c11f40SPeter Grehan 
2356366f6083SPeter Grehan 	if (vmxctx_getreg(&vmx->ctx[vcpu], reg, retval) == 0)
2357366f6083SPeter Grehan 		return (0);
2358366f6083SPeter Grehan 
2359d3c11f40SPeter Grehan 	return (vmcs_getreg(&vmx->vmcs[vcpu], running, reg, retval));
2360366f6083SPeter Grehan }
2361366f6083SPeter Grehan 
2362366f6083SPeter Grehan static int
2363366f6083SPeter Grehan vmx_setreg(void *arg, int vcpu, int reg, uint64_t val)
2364366f6083SPeter Grehan {
2365aaaa0656SPeter Grehan 	int error, hostcpu, running, shadow;
2366366f6083SPeter Grehan 	uint64_t ctls;
2367366f6083SPeter Grehan 	struct vmx *vmx = arg;
2368366f6083SPeter Grehan 
2369d3c11f40SPeter Grehan 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
2370d3c11f40SPeter Grehan 	if (running && hostcpu != curcpu)
2371d3c11f40SPeter Grehan 		panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu);
2372d3c11f40SPeter Grehan 
2373366f6083SPeter Grehan 	if (vmxctx_setreg(&vmx->ctx[vcpu], reg, val) == 0)
2374366f6083SPeter Grehan 		return (0);
2375366f6083SPeter Grehan 
2376d3c11f40SPeter Grehan 	error = vmcs_setreg(&vmx->vmcs[vcpu], running, reg, val);
2377366f6083SPeter Grehan 
2378366f6083SPeter Grehan 	if (error == 0) {
2379366f6083SPeter Grehan 		/*
2380366f6083SPeter Grehan 		 * If the "load EFER" VM-entry control is 1 then the
2381366f6083SPeter Grehan 		 * value of EFER.LMA must be identical to "IA-32e mode guest"
2382366f6083SPeter Grehan 		 * bit in the VM-entry control.
2383366f6083SPeter Grehan 		 */
2384366f6083SPeter Grehan 		if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 &&
2385366f6083SPeter Grehan 		    (reg == VM_REG_GUEST_EFER)) {
2386d3c11f40SPeter Grehan 			vmcs_getreg(&vmx->vmcs[vcpu], running,
2387366f6083SPeter Grehan 				    VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls);
2388366f6083SPeter Grehan 			if (val & EFER_LMA)
2389366f6083SPeter Grehan 				ctls |= VM_ENTRY_GUEST_LMA;
2390366f6083SPeter Grehan 			else
2391366f6083SPeter Grehan 				ctls &= ~VM_ENTRY_GUEST_LMA;
2392d3c11f40SPeter Grehan 			vmcs_setreg(&vmx->vmcs[vcpu], running,
2393366f6083SPeter Grehan 				    VMCS_IDENT(VMCS_ENTRY_CTLS), ctls);
2394366f6083SPeter Grehan 		}
2395aaaa0656SPeter Grehan 
2396aaaa0656SPeter Grehan 		shadow = vmx_shadow_reg(reg);
2397aaaa0656SPeter Grehan 		if (shadow > 0) {
2398aaaa0656SPeter Grehan 			/*
2399aaaa0656SPeter Grehan 			 * Store the unmodified value in the shadow
2400aaaa0656SPeter Grehan 			 */
2401aaaa0656SPeter Grehan 			error = vmcs_setreg(&vmx->vmcs[vcpu], running,
2402aaaa0656SPeter Grehan 				    VMCS_IDENT(shadow), val);
2403aaaa0656SPeter Grehan 		}
2404366f6083SPeter Grehan 	}
2405366f6083SPeter Grehan 
2406366f6083SPeter Grehan 	return (error);
2407366f6083SPeter Grehan }
2408366f6083SPeter Grehan 
2409366f6083SPeter Grehan static int
2410366f6083SPeter Grehan vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
2411366f6083SPeter Grehan {
2412366f6083SPeter Grehan 	struct vmx *vmx = arg;
2413366f6083SPeter Grehan 
2414366f6083SPeter Grehan 	return (vmcs_getdesc(&vmx->vmcs[vcpu], reg, desc));
2415366f6083SPeter Grehan }
2416366f6083SPeter Grehan 
2417366f6083SPeter Grehan static int
2418366f6083SPeter Grehan vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
2419366f6083SPeter Grehan {
2420366f6083SPeter Grehan 	struct vmx *vmx = arg;
2421366f6083SPeter Grehan 
2422366f6083SPeter Grehan 	return (vmcs_setdesc(&vmx->vmcs[vcpu], reg, desc));
2423366f6083SPeter Grehan }
2424366f6083SPeter Grehan 
2425366f6083SPeter Grehan static int
2426366f6083SPeter Grehan vmx_inject(void *arg, int vcpu, int type, int vector, uint32_t code,
2427366f6083SPeter Grehan 	   int code_valid)
2428366f6083SPeter Grehan {
2429366f6083SPeter Grehan 	struct vmx *vmx = arg;
2430a0efd3fbSJohn Baldwin 	struct vmxevent *user_event = &vmx->state[vcpu].user_event;
2431366f6083SPeter Grehan 
2432366f6083SPeter Grehan 	static uint32_t type_map[VM_EVENT_MAX] = {
2433366f6083SPeter Grehan 		0x1,		/* VM_EVENT_NONE */
2434366f6083SPeter Grehan 		0x0,		/* VM_HW_INTR */
2435366f6083SPeter Grehan 		0x2,		/* VM_NMI */
2436366f6083SPeter Grehan 		0x3,		/* VM_HW_EXCEPTION */
2437366f6083SPeter Grehan 		0x4,		/* VM_SW_INTR */
2438366f6083SPeter Grehan 		0x5,		/* VM_PRIV_SW_EXCEPTION */
2439366f6083SPeter Grehan 		0x6,		/* VM_SW_EXCEPTION */
2440366f6083SPeter Grehan 	};
2441366f6083SPeter Grehan 
2442eeefa4e4SNeel Natu 	/*
2443eeefa4e4SNeel Natu 	 * If there is already an exception pending to be delivered to the
2444eeefa4e4SNeel Natu 	 * vcpu then just return.
2445eeefa4e4SNeel Natu 	 */
2446a0efd3fbSJohn Baldwin 	if (user_event->intr_info & VMCS_INTR_VALID)
2447eeefa4e4SNeel Natu 		return (EAGAIN);
2448eeefa4e4SNeel Natu 
2449a0efd3fbSJohn Baldwin 	user_event->intr_info = vector | (type_map[type] << 8) | VMCS_INTR_VALID;
2450366f6083SPeter Grehan 	if (code_valid) {
2451a0efd3fbSJohn Baldwin 		user_event->intr_info |= VMCS_INTR_DEL_ERRCODE;
2452a0efd3fbSJohn Baldwin 		user_event->error_code = code;
2453366f6083SPeter Grehan 	}
2454a0efd3fbSJohn Baldwin 	return (0);
2455366f6083SPeter Grehan }
2456366f6083SPeter Grehan 
2457366f6083SPeter Grehan static int
2458366f6083SPeter Grehan vmx_getcap(void *arg, int vcpu, int type, int *retval)
2459366f6083SPeter Grehan {
2460366f6083SPeter Grehan 	struct vmx *vmx = arg;
2461366f6083SPeter Grehan 	int vcap;
2462366f6083SPeter Grehan 	int ret;
2463366f6083SPeter Grehan 
2464366f6083SPeter Grehan 	ret = ENOENT;
2465366f6083SPeter Grehan 
2466366f6083SPeter Grehan 	vcap = vmx->cap[vcpu].set;
2467366f6083SPeter Grehan 
2468366f6083SPeter Grehan 	switch (type) {
2469366f6083SPeter Grehan 	case VM_CAP_HALT_EXIT:
2470366f6083SPeter Grehan 		if (cap_halt_exit)
2471366f6083SPeter Grehan 			ret = 0;
2472366f6083SPeter Grehan 		break;
2473366f6083SPeter Grehan 	case VM_CAP_PAUSE_EXIT:
2474366f6083SPeter Grehan 		if (cap_pause_exit)
2475366f6083SPeter Grehan 			ret = 0;
2476366f6083SPeter Grehan 		break;
2477366f6083SPeter Grehan 	case VM_CAP_MTRAP_EXIT:
2478366f6083SPeter Grehan 		if (cap_monitor_trap)
2479366f6083SPeter Grehan 			ret = 0;
2480366f6083SPeter Grehan 		break;
2481366f6083SPeter Grehan 	case VM_CAP_UNRESTRICTED_GUEST:
2482366f6083SPeter Grehan 		if (cap_unrestricted_guest)
2483366f6083SPeter Grehan 			ret = 0;
2484366f6083SPeter Grehan 		break;
248549cc03daSNeel Natu 	case VM_CAP_ENABLE_INVPCID:
248649cc03daSNeel Natu 		if (cap_invpcid)
248749cc03daSNeel Natu 			ret = 0;
248849cc03daSNeel Natu 		break;
2489366f6083SPeter Grehan 	default:
2490366f6083SPeter Grehan 		break;
2491366f6083SPeter Grehan 	}
2492366f6083SPeter Grehan 
2493366f6083SPeter Grehan 	if (ret == 0)
2494366f6083SPeter Grehan 		*retval = (vcap & (1 << type)) ? 1 : 0;
2495366f6083SPeter Grehan 
2496366f6083SPeter Grehan 	return (ret);
2497366f6083SPeter Grehan }
2498366f6083SPeter Grehan 
2499366f6083SPeter Grehan static int
2500366f6083SPeter Grehan vmx_setcap(void *arg, int vcpu, int type, int val)
2501366f6083SPeter Grehan {
2502366f6083SPeter Grehan 	struct vmx *vmx = arg;
2503366f6083SPeter Grehan 	struct vmcs *vmcs = &vmx->vmcs[vcpu];
2504366f6083SPeter Grehan 	uint32_t baseval;
2505366f6083SPeter Grehan 	uint32_t *pptr;
2506366f6083SPeter Grehan 	int error;
2507366f6083SPeter Grehan 	int flag;
2508366f6083SPeter Grehan 	int reg;
2509366f6083SPeter Grehan 	int retval;
2510366f6083SPeter Grehan 
2511366f6083SPeter Grehan 	retval = ENOENT;
2512366f6083SPeter Grehan 	pptr = NULL;
2513366f6083SPeter Grehan 
2514366f6083SPeter Grehan 	switch (type) {
2515366f6083SPeter Grehan 	case VM_CAP_HALT_EXIT:
2516366f6083SPeter Grehan 		if (cap_halt_exit) {
2517366f6083SPeter Grehan 			retval = 0;
2518366f6083SPeter Grehan 			pptr = &vmx->cap[vcpu].proc_ctls;
2519366f6083SPeter Grehan 			baseval = *pptr;
2520366f6083SPeter Grehan 			flag = PROCBASED_HLT_EXITING;
2521366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
2522366f6083SPeter Grehan 		}
2523366f6083SPeter Grehan 		break;
2524366f6083SPeter Grehan 	case VM_CAP_MTRAP_EXIT:
2525366f6083SPeter Grehan 		if (cap_monitor_trap) {
2526366f6083SPeter Grehan 			retval = 0;
2527366f6083SPeter Grehan 			pptr = &vmx->cap[vcpu].proc_ctls;
2528366f6083SPeter Grehan 			baseval = *pptr;
2529366f6083SPeter Grehan 			flag = PROCBASED_MTF;
2530366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
2531366f6083SPeter Grehan 		}
2532366f6083SPeter Grehan 		break;
2533366f6083SPeter Grehan 	case VM_CAP_PAUSE_EXIT:
2534366f6083SPeter Grehan 		if (cap_pause_exit) {
2535366f6083SPeter Grehan 			retval = 0;
2536366f6083SPeter Grehan 			pptr = &vmx->cap[vcpu].proc_ctls;
2537366f6083SPeter Grehan 			baseval = *pptr;
2538366f6083SPeter Grehan 			flag = PROCBASED_PAUSE_EXITING;
2539366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
2540366f6083SPeter Grehan 		}
2541366f6083SPeter Grehan 		break;
2542366f6083SPeter Grehan 	case VM_CAP_UNRESTRICTED_GUEST:
2543366f6083SPeter Grehan 		if (cap_unrestricted_guest) {
2544366f6083SPeter Grehan 			retval = 0;
254549cc03daSNeel Natu 			pptr = &vmx->cap[vcpu].proc_ctls2;
254649cc03daSNeel Natu 			baseval = *pptr;
2547366f6083SPeter Grehan 			flag = PROCBASED2_UNRESTRICTED_GUEST;
2548366f6083SPeter Grehan 			reg = VMCS_SEC_PROC_BASED_CTLS;
2549366f6083SPeter Grehan 		}
2550366f6083SPeter Grehan 		break;
255149cc03daSNeel Natu 	case VM_CAP_ENABLE_INVPCID:
255249cc03daSNeel Natu 		if (cap_invpcid) {
255349cc03daSNeel Natu 			retval = 0;
255449cc03daSNeel Natu 			pptr = &vmx->cap[vcpu].proc_ctls2;
255549cc03daSNeel Natu 			baseval = *pptr;
255649cc03daSNeel Natu 			flag = PROCBASED2_ENABLE_INVPCID;
255749cc03daSNeel Natu 			reg = VMCS_SEC_PROC_BASED_CTLS;
255849cc03daSNeel Natu 		}
255949cc03daSNeel Natu 		break;
2560366f6083SPeter Grehan 	default:
2561366f6083SPeter Grehan 		break;
2562366f6083SPeter Grehan 	}
2563366f6083SPeter Grehan 
2564366f6083SPeter Grehan 	if (retval == 0) {
2565366f6083SPeter Grehan 		if (val) {
2566366f6083SPeter Grehan 			baseval |= flag;
2567366f6083SPeter Grehan 		} else {
2568366f6083SPeter Grehan 			baseval &= ~flag;
2569366f6083SPeter Grehan 		}
2570366f6083SPeter Grehan 		VMPTRLD(vmcs);
2571366f6083SPeter Grehan 		error = vmwrite(reg, baseval);
2572366f6083SPeter Grehan 		VMCLEAR(vmcs);
2573366f6083SPeter Grehan 
2574366f6083SPeter Grehan 		if (error) {
2575366f6083SPeter Grehan 			retval = error;
2576366f6083SPeter Grehan 		} else {
2577366f6083SPeter Grehan 			/*
2578366f6083SPeter Grehan 			 * Update optional stored flags, and record
2579366f6083SPeter Grehan 			 * setting
2580366f6083SPeter Grehan 			 */
2581366f6083SPeter Grehan 			if (pptr != NULL) {
2582366f6083SPeter Grehan 				*pptr = baseval;
2583366f6083SPeter Grehan 			}
2584366f6083SPeter Grehan 
2585366f6083SPeter Grehan 			if (val) {
2586366f6083SPeter Grehan 				vmx->cap[vcpu].set |= (1 << type);
2587366f6083SPeter Grehan 			} else {
2588366f6083SPeter Grehan 				vmx->cap[vcpu].set &= ~(1 << type);
2589366f6083SPeter Grehan 			}
2590366f6083SPeter Grehan 		}
2591366f6083SPeter Grehan 	}
2592366f6083SPeter Grehan 
2593366f6083SPeter Grehan         return (retval);
2594366f6083SPeter Grehan }
2595366f6083SPeter Grehan 
259688c4b8d1SNeel Natu struct vlapic_vtx {
259788c4b8d1SNeel Natu 	struct vlapic	vlapic;
2598176666c2SNeel Natu 	struct pir_desc	*pir_desc;
259930b94db8SNeel Natu 	struct vmx	*vmx;
260088c4b8d1SNeel Natu };
260188c4b8d1SNeel Natu 
260288c4b8d1SNeel Natu #define	VMX_CTR_PIR(vm, vcpuid, pir_desc, notify, vector, level, msg)	\
260388c4b8d1SNeel Natu do {									\
260488c4b8d1SNeel Natu 	VCPU_CTR2(vm, vcpuid, msg " assert %s-triggered vector %d",	\
260588c4b8d1SNeel Natu 	    level ? "level" : "edge", vector);				\
260688c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir0 0x%016lx", pir_desc->pir[0]);	\
260788c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir1 0x%016lx", pir_desc->pir[1]);	\
260888c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir2 0x%016lx", pir_desc->pir[2]);	\
260988c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir3 0x%016lx", pir_desc->pir[3]);	\
261088c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " notify: %s", notify ? "yes" : "no");\
261188c4b8d1SNeel Natu } while (0)
261288c4b8d1SNeel Natu 
261388c4b8d1SNeel Natu /*
261488c4b8d1SNeel Natu  * vlapic->ops handlers that utilize the APICv hardware assist described in
261588c4b8d1SNeel Natu  * Chapter 29 of the Intel SDM.
261688c4b8d1SNeel Natu  */
261788c4b8d1SNeel Natu static int
261888c4b8d1SNeel Natu vmx_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
261988c4b8d1SNeel Natu {
262088c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
262188c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
262288c4b8d1SNeel Natu 	uint64_t mask;
262388c4b8d1SNeel Natu 	int idx, notify;
262488c4b8d1SNeel Natu 
262588c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
2626176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
262788c4b8d1SNeel Natu 
262888c4b8d1SNeel Natu 	/*
262988c4b8d1SNeel Natu 	 * Keep track of interrupt requests in the PIR descriptor. This is
263088c4b8d1SNeel Natu 	 * because the virtual APIC page pointed to by the VMCS cannot be
263188c4b8d1SNeel Natu 	 * modified if the vcpu is running.
263288c4b8d1SNeel Natu 	 */
263388c4b8d1SNeel Natu 	idx = vector / 64;
263488c4b8d1SNeel Natu 	mask = 1UL << (vector % 64);
263588c4b8d1SNeel Natu 	atomic_set_long(&pir_desc->pir[idx], mask);
263688c4b8d1SNeel Natu 	notify = atomic_cmpset_long(&pir_desc->pending, 0, 1);
263788c4b8d1SNeel Natu 
263888c4b8d1SNeel Natu 	VMX_CTR_PIR(vlapic->vm, vlapic->vcpuid, pir_desc, notify, vector,
263988c4b8d1SNeel Natu 	    level, "vmx_set_intr_ready");
264088c4b8d1SNeel Natu 	return (notify);
264188c4b8d1SNeel Natu }
264288c4b8d1SNeel Natu 
264388c4b8d1SNeel Natu static int
264488c4b8d1SNeel Natu vmx_pending_intr(struct vlapic *vlapic, int *vecptr)
264588c4b8d1SNeel Natu {
264688c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
264788c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
264888c4b8d1SNeel Natu 	struct LAPIC *lapic;
264988c4b8d1SNeel Natu 	uint64_t pending, pirval;
265088c4b8d1SNeel Natu 	uint32_t ppr, vpr;
265188c4b8d1SNeel Natu 	int i;
265288c4b8d1SNeel Natu 
265388c4b8d1SNeel Natu 	/*
265488c4b8d1SNeel Natu 	 * This function is only expected to be called from the 'HLT' exit
265588c4b8d1SNeel Natu 	 * handler which does not care about the vector that is pending.
265688c4b8d1SNeel Natu 	 */
265788c4b8d1SNeel Natu 	KASSERT(vecptr == NULL, ("vmx_pending_intr: vecptr must be NULL"));
265888c4b8d1SNeel Natu 
265988c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
2660176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
266188c4b8d1SNeel Natu 
266288c4b8d1SNeel Natu 	pending = atomic_load_acq_long(&pir_desc->pending);
266388c4b8d1SNeel Natu 	if (!pending)
266488c4b8d1SNeel Natu 		return (0);	/* common case */
266588c4b8d1SNeel Natu 
266688c4b8d1SNeel Natu 	/*
266788c4b8d1SNeel Natu 	 * If there is an interrupt pending then it will be recognized only
266888c4b8d1SNeel Natu 	 * if its priority is greater than the processor priority.
266988c4b8d1SNeel Natu 	 *
267088c4b8d1SNeel Natu 	 * Special case: if the processor priority is zero then any pending
267188c4b8d1SNeel Natu 	 * interrupt will be recognized.
267288c4b8d1SNeel Natu 	 */
267388c4b8d1SNeel Natu 	lapic = vlapic->apic_page;
267488c4b8d1SNeel Natu 	ppr = lapic->ppr & 0xf0;
267588c4b8d1SNeel Natu 	if (ppr == 0)
267688c4b8d1SNeel Natu 		return (1);
267788c4b8d1SNeel Natu 
267888c4b8d1SNeel Natu 	VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "HLT with non-zero PPR %d",
267988c4b8d1SNeel Natu 	    lapic->ppr);
268088c4b8d1SNeel Natu 
268188c4b8d1SNeel Natu 	for (i = 3; i >= 0; i--) {
268288c4b8d1SNeel Natu 		pirval = pir_desc->pir[i];
268388c4b8d1SNeel Natu 		if (pirval != 0) {
268488c4b8d1SNeel Natu 			vpr = (i * 64 + flsl(pirval) - 1) & 0xf0;
268588c4b8d1SNeel Natu 			return (vpr > ppr);
268688c4b8d1SNeel Natu 		}
268788c4b8d1SNeel Natu 	}
268888c4b8d1SNeel Natu 	return (0);
268988c4b8d1SNeel Natu }
269088c4b8d1SNeel Natu 
269188c4b8d1SNeel Natu static void
269288c4b8d1SNeel Natu vmx_intr_accepted(struct vlapic *vlapic, int vector)
269388c4b8d1SNeel Natu {
269488c4b8d1SNeel Natu 
269588c4b8d1SNeel Natu 	panic("vmx_intr_accepted: not expected to be called");
269688c4b8d1SNeel Natu }
269788c4b8d1SNeel Natu 
2698176666c2SNeel Natu static void
269930b94db8SNeel Natu vmx_set_tmr(struct vlapic *vlapic, int vector, bool level)
270030b94db8SNeel Natu {
270130b94db8SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
270230b94db8SNeel Natu 	struct vmx *vmx;
270330b94db8SNeel Natu 	struct vmcs *vmcs;
270430b94db8SNeel Natu 	uint64_t mask, val;
270530b94db8SNeel Natu 
270630b94db8SNeel Natu 	KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
270730b94db8SNeel Natu 	KASSERT(!vcpu_is_running(vlapic->vm, vlapic->vcpuid, NULL),
270830b94db8SNeel Natu 	    ("vmx_set_tmr: vcpu cannot be running"));
270930b94db8SNeel Natu 
271030b94db8SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
271130b94db8SNeel Natu 	vmx = vlapic_vtx->vmx;
271230b94db8SNeel Natu 	vmcs = &vmx->vmcs[vlapic->vcpuid];
271330b94db8SNeel Natu 	mask = 1UL << (vector % 64);
271430b94db8SNeel Natu 
271530b94db8SNeel Natu 	VMPTRLD(vmcs);
271630b94db8SNeel Natu 	val = vmcs_read(VMCS_EOI_EXIT(vector));
271730b94db8SNeel Natu 	if (level)
271830b94db8SNeel Natu 		val |= mask;
271930b94db8SNeel Natu 	else
272030b94db8SNeel Natu 		val &= ~mask;
272130b94db8SNeel Natu 	vmcs_write(VMCS_EOI_EXIT(vector), val);
272230b94db8SNeel Natu 	VMCLEAR(vmcs);
272330b94db8SNeel Natu }
272430b94db8SNeel Natu 
272530b94db8SNeel Natu static void
2726*159dd56fSNeel Natu vmx_enable_x2apic_mode(struct vlapic *vlapic)
2727*159dd56fSNeel Natu {
2728*159dd56fSNeel Natu 	struct vmx *vmx;
2729*159dd56fSNeel Natu 	struct vmcs *vmcs;
2730*159dd56fSNeel Natu 	uint32_t proc_ctls2;
2731*159dd56fSNeel Natu 	int vcpuid, error;
2732*159dd56fSNeel Natu 
2733*159dd56fSNeel Natu 	vcpuid = vlapic->vcpuid;
2734*159dd56fSNeel Natu 	vmx = ((struct vlapic_vtx *)vlapic)->vmx;
2735*159dd56fSNeel Natu 	vmcs = &vmx->vmcs[vcpuid];
2736*159dd56fSNeel Natu 
2737*159dd56fSNeel Natu 	proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
2738*159dd56fSNeel Natu 	KASSERT((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) != 0,
2739*159dd56fSNeel Natu 	    ("%s: invalid proc_ctls2 %#x", __func__, proc_ctls2));
2740*159dd56fSNeel Natu 
2741*159dd56fSNeel Natu 	proc_ctls2 &= ~PROCBASED2_VIRTUALIZE_APIC_ACCESSES;
2742*159dd56fSNeel Natu 	proc_ctls2 |= PROCBASED2_VIRTUALIZE_X2APIC_MODE;
2743*159dd56fSNeel Natu 	vmx->cap[vcpuid].proc_ctls2 = proc_ctls2;
2744*159dd56fSNeel Natu 
2745*159dd56fSNeel Natu 	VMPTRLD(vmcs);
2746*159dd56fSNeel Natu 	vmcs_write(VMCS_SEC_PROC_BASED_CTLS, proc_ctls2);
2747*159dd56fSNeel Natu 	VMCLEAR(vmcs);
2748*159dd56fSNeel Natu 
2749*159dd56fSNeel Natu 	if (vlapic->vcpuid == 0) {
2750*159dd56fSNeel Natu 		/*
2751*159dd56fSNeel Natu 		 * The nested page table mappings are shared by all vcpus
2752*159dd56fSNeel Natu 		 * so unmap the APIC access page just once.
2753*159dd56fSNeel Natu 		 */
2754*159dd56fSNeel Natu 		error = vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
2755*159dd56fSNeel Natu 		KASSERT(error == 0, ("%s: vm_unmap_mmio error %d",
2756*159dd56fSNeel Natu 		    __func__, error));
2757*159dd56fSNeel Natu 
2758*159dd56fSNeel Natu 		/*
2759*159dd56fSNeel Natu 		 * The MSR bitmap is shared by all vcpus so modify it only
2760*159dd56fSNeel Natu 		 * once in the context of vcpu 0.
2761*159dd56fSNeel Natu 		 */
2762*159dd56fSNeel Natu 		error = vmx_allow_x2apic_msrs(vmx);
2763*159dd56fSNeel Natu 		KASSERT(error == 0, ("%s: vmx_allow_x2apic_msrs error %d",
2764*159dd56fSNeel Natu 		    __func__, error));
2765*159dd56fSNeel Natu 	}
2766*159dd56fSNeel Natu }
2767*159dd56fSNeel Natu 
2768*159dd56fSNeel Natu static void
2769176666c2SNeel Natu vmx_post_intr(struct vlapic *vlapic, int hostcpu)
2770176666c2SNeel Natu {
2771176666c2SNeel Natu 
2772176666c2SNeel Natu 	ipi_cpu(hostcpu, pirvec);
2773176666c2SNeel Natu }
2774176666c2SNeel Natu 
277588c4b8d1SNeel Natu /*
277688c4b8d1SNeel Natu  * Transfer the pending interrupts in the PIR descriptor to the IRR
277788c4b8d1SNeel Natu  * in the virtual APIC page.
277888c4b8d1SNeel Natu  */
277988c4b8d1SNeel Natu static void
278088c4b8d1SNeel Natu vmx_inject_pir(struct vlapic *vlapic)
278188c4b8d1SNeel Natu {
278288c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
278388c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
278488c4b8d1SNeel Natu 	struct LAPIC *lapic;
278588c4b8d1SNeel Natu 	uint64_t val, pirval;
278688c4b8d1SNeel Natu 	int rvi, pirbase;
278788c4b8d1SNeel Natu 	uint16_t intr_status_old, intr_status_new;
278888c4b8d1SNeel Natu 
278988c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
2790176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
279188c4b8d1SNeel Natu 	if (atomic_cmpset_long(&pir_desc->pending, 1, 0) == 0) {
279288c4b8d1SNeel Natu 		VCPU_CTR0(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
279388c4b8d1SNeel Natu 		    "no posted interrupt pending");
279488c4b8d1SNeel Natu 		return;
279588c4b8d1SNeel Natu 	}
279688c4b8d1SNeel Natu 
279788c4b8d1SNeel Natu 	pirval = 0;
279888c4b8d1SNeel Natu 	lapic = vlapic->apic_page;
279988c4b8d1SNeel Natu 
280088c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[0]);
280188c4b8d1SNeel Natu 	if (val != 0) {
280288c4b8d1SNeel Natu 		lapic->irr0 |= val;
280388c4b8d1SNeel Natu 		lapic->irr1 |= val >> 32;
280488c4b8d1SNeel Natu 		pirbase = 0;
280588c4b8d1SNeel Natu 		pirval = val;
280688c4b8d1SNeel Natu 	}
280788c4b8d1SNeel Natu 
280888c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[1]);
280988c4b8d1SNeel Natu 	if (val != 0) {
281088c4b8d1SNeel Natu 		lapic->irr2 |= val;
281188c4b8d1SNeel Natu 		lapic->irr3 |= val >> 32;
281288c4b8d1SNeel Natu 		pirbase = 64;
281388c4b8d1SNeel Natu 		pirval = val;
281488c4b8d1SNeel Natu 	}
281588c4b8d1SNeel Natu 
281688c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[2]);
281788c4b8d1SNeel Natu 	if (val != 0) {
281888c4b8d1SNeel Natu 		lapic->irr4 |= val;
281988c4b8d1SNeel Natu 		lapic->irr5 |= val >> 32;
282088c4b8d1SNeel Natu 		pirbase = 128;
282188c4b8d1SNeel Natu 		pirval = val;
282288c4b8d1SNeel Natu 	}
282388c4b8d1SNeel Natu 
282488c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[3]);
282588c4b8d1SNeel Natu 	if (val != 0) {
282688c4b8d1SNeel Natu 		lapic->irr6 |= val;
282788c4b8d1SNeel Natu 		lapic->irr7 |= val >> 32;
282888c4b8d1SNeel Natu 		pirbase = 192;
282988c4b8d1SNeel Natu 		pirval = val;
283088c4b8d1SNeel Natu 	}
283188c4b8d1SNeel Natu 	VLAPIC_CTR_IRR(vlapic, "vmx_inject_pir");
283288c4b8d1SNeel Natu 
283388c4b8d1SNeel Natu 	/*
283488c4b8d1SNeel Natu 	 * Update RVI so the processor can evaluate pending virtual
283588c4b8d1SNeel Natu 	 * interrupts on VM-entry.
283688c4b8d1SNeel Natu 	 */
283788c4b8d1SNeel Natu 	if (pirval != 0) {
283888c4b8d1SNeel Natu 		rvi = pirbase + flsl(pirval) - 1;
283988c4b8d1SNeel Natu 		intr_status_old = vmcs_read(VMCS_GUEST_INTR_STATUS);
284088c4b8d1SNeel Natu 		intr_status_new = (intr_status_old & 0xFF00) | rvi;
284188c4b8d1SNeel Natu 		if (intr_status_new > intr_status_old) {
284288c4b8d1SNeel Natu 			vmcs_write(VMCS_GUEST_INTR_STATUS, intr_status_new);
284388c4b8d1SNeel Natu 			VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
284488c4b8d1SNeel Natu 			    "guest_intr_status changed from 0x%04x to 0x%04x",
284588c4b8d1SNeel Natu 			    intr_status_old, intr_status_new);
284688c4b8d1SNeel Natu 		}
284788c4b8d1SNeel Natu 	}
284888c4b8d1SNeel Natu }
284988c4b8d1SNeel Natu 
2850de5ea6b6SNeel Natu static struct vlapic *
2851de5ea6b6SNeel Natu vmx_vlapic_init(void *arg, int vcpuid)
2852de5ea6b6SNeel Natu {
2853de5ea6b6SNeel Natu 	struct vmx *vmx;
2854de5ea6b6SNeel Natu 	struct vlapic *vlapic;
2855176666c2SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
2856de5ea6b6SNeel Natu 
2857de5ea6b6SNeel Natu 	vmx = arg;
2858de5ea6b6SNeel Natu 
285988c4b8d1SNeel Natu 	vlapic = malloc(sizeof(struct vlapic_vtx), M_VLAPIC, M_WAITOK | M_ZERO);
2860de5ea6b6SNeel Natu 	vlapic->vm = vmx->vm;
2861de5ea6b6SNeel Natu 	vlapic->vcpuid = vcpuid;
2862de5ea6b6SNeel Natu 	vlapic->apic_page = (struct LAPIC *)&vmx->apic_page[vcpuid];
2863de5ea6b6SNeel Natu 
2864176666c2SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
2865176666c2SNeel Natu 	vlapic_vtx->pir_desc = &vmx->pir_desc[vcpuid];
286630b94db8SNeel Natu 	vlapic_vtx->vmx = vmx;
2867176666c2SNeel Natu 
286888c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
286988c4b8d1SNeel Natu 		vlapic->ops.set_intr_ready = vmx_set_intr_ready;
287088c4b8d1SNeel Natu 		vlapic->ops.pending_intr = vmx_pending_intr;
287188c4b8d1SNeel Natu 		vlapic->ops.intr_accepted = vmx_intr_accepted;
287230b94db8SNeel Natu 		vlapic->ops.set_tmr = vmx_set_tmr;
2873*159dd56fSNeel Natu 		vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode;
287488c4b8d1SNeel Natu 	}
287588c4b8d1SNeel Natu 
2876176666c2SNeel Natu 	if (posted_interrupts)
2877176666c2SNeel Natu 		vlapic->ops.post_intr = vmx_post_intr;
2878176666c2SNeel Natu 
2879de5ea6b6SNeel Natu 	vlapic_init(vlapic);
2880de5ea6b6SNeel Natu 
2881de5ea6b6SNeel Natu 	return (vlapic);
2882de5ea6b6SNeel Natu }
2883de5ea6b6SNeel Natu 
2884de5ea6b6SNeel Natu static void
2885de5ea6b6SNeel Natu vmx_vlapic_cleanup(void *arg, struct vlapic *vlapic)
2886de5ea6b6SNeel Natu {
2887de5ea6b6SNeel Natu 
2888de5ea6b6SNeel Natu 	vlapic_cleanup(vlapic);
2889de5ea6b6SNeel Natu 	free(vlapic, M_VLAPIC);
2890de5ea6b6SNeel Natu }
2891de5ea6b6SNeel Natu 
2892366f6083SPeter Grehan struct vmm_ops vmm_ops_intel = {
2893366f6083SPeter Grehan 	vmx_init,
2894366f6083SPeter Grehan 	vmx_cleanup,
289563e62d39SJohn Baldwin 	vmx_restore,
2896366f6083SPeter Grehan 	vmx_vminit,
2897366f6083SPeter Grehan 	vmx_run,
2898366f6083SPeter Grehan 	vmx_vmcleanup,
2899366f6083SPeter Grehan 	vmx_getreg,
2900366f6083SPeter Grehan 	vmx_setreg,
2901366f6083SPeter Grehan 	vmx_getdesc,
2902366f6083SPeter Grehan 	vmx_setdesc,
2903366f6083SPeter Grehan 	vmx_inject,
2904366f6083SPeter Grehan 	vmx_getcap,
2905318224bbSNeel Natu 	vmx_setcap,
2906318224bbSNeel Natu 	ept_vmspace_alloc,
2907318224bbSNeel Natu 	ept_vmspace_free,
2908de5ea6b6SNeel Natu 	vmx_vlapic_init,
2909de5ea6b6SNeel Natu 	vmx_vlapic_cleanup,
2910366f6083SPeter Grehan };
2911