xref: /freebsd/sys/amd64/vmm/intel/vmx.c (revision a0efd3fb34f4ef70207fb4498c053f17d2be4d78)
1366f6083SPeter Grehan /*-
2366f6083SPeter Grehan  * Copyright (c) 2011 NetApp, Inc.
3366f6083SPeter Grehan  * All rights reserved.
4366f6083SPeter Grehan  *
5366f6083SPeter Grehan  * Redistribution and use in source and binary forms, with or without
6366f6083SPeter Grehan  * modification, are permitted provided that the following conditions
7366f6083SPeter Grehan  * are met:
8366f6083SPeter Grehan  * 1. Redistributions of source code must retain the above copyright
9366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer.
10366f6083SPeter Grehan  * 2. Redistributions in binary form must reproduce the above copyright
11366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer in the
12366f6083SPeter Grehan  *    documentation and/or other materials provided with the distribution.
13366f6083SPeter Grehan  *
14366f6083SPeter Grehan  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
15366f6083SPeter Grehan  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16366f6083SPeter Grehan  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17366f6083SPeter Grehan  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
18366f6083SPeter Grehan  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19366f6083SPeter Grehan  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20366f6083SPeter Grehan  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21366f6083SPeter Grehan  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22366f6083SPeter Grehan  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23366f6083SPeter Grehan  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24366f6083SPeter Grehan  * SUCH DAMAGE.
25366f6083SPeter Grehan  *
26366f6083SPeter Grehan  * $FreeBSD$
27366f6083SPeter Grehan  */
28366f6083SPeter Grehan 
29366f6083SPeter Grehan #include <sys/cdefs.h>
30366f6083SPeter Grehan __FBSDID("$FreeBSD$");
31366f6083SPeter Grehan 
32366f6083SPeter Grehan #include <sys/param.h>
33366f6083SPeter Grehan #include <sys/systm.h>
34366f6083SPeter Grehan #include <sys/smp.h>
35366f6083SPeter Grehan #include <sys/kernel.h>
36366f6083SPeter Grehan #include <sys/malloc.h>
37366f6083SPeter Grehan #include <sys/pcpu.h>
38366f6083SPeter Grehan #include <sys/proc.h>
393565b59eSNeel Natu #include <sys/sysctl.h>
40366f6083SPeter Grehan 
41366f6083SPeter Grehan #include <vm/vm.h>
42366f6083SPeter Grehan #include <vm/pmap.h>
43366f6083SPeter Grehan 
44366f6083SPeter Grehan #include <machine/psl.h>
45366f6083SPeter Grehan #include <machine/cpufunc.h>
468b287612SJohn Baldwin #include <machine/md_var.h>
47366f6083SPeter Grehan #include <machine/segments.h>
48176666c2SNeel Natu #include <machine/smp.h>
49608f97c3SPeter Grehan #include <machine/specialreg.h>
50366f6083SPeter Grehan #include <machine/vmparam.h>
51366f6083SPeter Grehan 
52366f6083SPeter Grehan #include <machine/vmm.h>
53b01c2033SNeel Natu #include "vmm_host.h"
54176666c2SNeel Natu #include "vmm_ipi.h"
55366f6083SPeter Grehan #include "vmm_msr.h"
56366f6083SPeter Grehan #include "vmm_ktr.h"
57366f6083SPeter Grehan #include "vmm_stat.h"
58de5ea6b6SNeel Natu #include "vlapic.h"
59de5ea6b6SNeel Natu #include "vlapic_priv.h"
60366f6083SPeter Grehan 
61366f6083SPeter Grehan #include "vmx_msr.h"
62366f6083SPeter Grehan #include "ept.h"
63366f6083SPeter Grehan #include "vmx_cpufunc.h"
64366f6083SPeter Grehan #include "vmx.h"
65366f6083SPeter Grehan #include "x86.h"
66366f6083SPeter Grehan #include "vmx_controls.h"
67366f6083SPeter Grehan 
68366f6083SPeter Grehan #define	PINBASED_CTLS_ONE_SETTING					\
69366f6083SPeter Grehan 	(PINBASED_EXTINT_EXITING	|				\
70366f6083SPeter Grehan 	 PINBASED_NMI_EXITING		|				\
71366f6083SPeter Grehan 	 PINBASED_VIRTUAL_NMI)
72366f6083SPeter Grehan #define	PINBASED_CTLS_ZERO_SETTING	0
73366f6083SPeter Grehan 
74366f6083SPeter Grehan #define PROCBASED_CTLS_WINDOW_SETTING					\
75366f6083SPeter Grehan 	(PROCBASED_INT_WINDOW_EXITING	|				\
76366f6083SPeter Grehan 	 PROCBASED_NMI_WINDOW_EXITING)
77366f6083SPeter Grehan 
78366f6083SPeter Grehan #define	PROCBASED_CTLS_ONE_SETTING 					\
79366f6083SPeter Grehan 	(PROCBASED_SECONDARY_CONTROLS	|				\
80366f6083SPeter Grehan 	 PROCBASED_IO_EXITING		|				\
81366f6083SPeter Grehan 	 PROCBASED_MSR_BITMAPS		|				\
82366f6083SPeter Grehan 	 PROCBASED_CTLS_WINDOW_SETTING)
83366f6083SPeter Grehan #define	PROCBASED_CTLS_ZERO_SETTING	\
84366f6083SPeter Grehan 	(PROCBASED_CR3_LOAD_EXITING |	\
85366f6083SPeter Grehan 	PROCBASED_CR3_STORE_EXITING |	\
86366f6083SPeter Grehan 	PROCBASED_IO_BITMAPS)
87366f6083SPeter Grehan 
88366f6083SPeter Grehan #define	PROCBASED_CTLS2_ONE_SETTING	PROCBASED2_ENABLE_EPT
89366f6083SPeter Grehan #define	PROCBASED_CTLS2_ZERO_SETTING	0
90366f6083SPeter Grehan 
91608f97c3SPeter Grehan #define VM_EXIT_CTLS_ONE_SETTING_NO_PAT					\
92366f6083SPeter Grehan 	(VM_EXIT_HOST_LMA			|			\
93366f6083SPeter Grehan 	VM_EXIT_SAVE_EFER			|			\
94366f6083SPeter Grehan 	VM_EXIT_LOAD_EFER)
95608f97c3SPeter Grehan 
96608f97c3SPeter Grehan #define	VM_EXIT_CTLS_ONE_SETTING					\
97608f97c3SPeter Grehan 	(VM_EXIT_CTLS_ONE_SETTING_NO_PAT       	|			\
98f7d47425SNeel Natu 	VM_EXIT_ACKNOWLEDGE_INTERRUPT		|			\
99608f97c3SPeter Grehan 	VM_EXIT_SAVE_PAT			|			\
100608f97c3SPeter Grehan 	VM_EXIT_LOAD_PAT)
101366f6083SPeter Grehan #define	VM_EXIT_CTLS_ZERO_SETTING	VM_EXIT_SAVE_DEBUG_CONTROLS
102366f6083SPeter Grehan 
103608f97c3SPeter Grehan #define	VM_ENTRY_CTLS_ONE_SETTING_NO_PAT	VM_ENTRY_LOAD_EFER
104608f97c3SPeter Grehan 
105366f6083SPeter Grehan #define	VM_ENTRY_CTLS_ONE_SETTING					\
106608f97c3SPeter Grehan 	(VM_ENTRY_CTLS_ONE_SETTING_NO_PAT     	|			\
107608f97c3SPeter Grehan 	VM_ENTRY_LOAD_PAT)
108366f6083SPeter Grehan #define	VM_ENTRY_CTLS_ZERO_SETTING					\
109366f6083SPeter Grehan 	(VM_ENTRY_LOAD_DEBUG_CONTROLS		|			\
110366f6083SPeter Grehan 	VM_ENTRY_INTO_SMM			|			\
111366f6083SPeter Grehan 	VM_ENTRY_DEACTIVATE_DUAL_MONITOR)
112366f6083SPeter Grehan 
113366f6083SPeter Grehan #define	guest_msr_rw(vmx, msr) \
114366f6083SPeter Grehan 	msr_bitmap_change_access((vmx)->msr_bitmap, (msr), MSR_BITMAP_ACCESS_RW)
115366f6083SPeter Grehan 
116366f6083SPeter Grehan #define	HANDLED		1
117366f6083SPeter Grehan #define	UNHANDLED	0
118366f6083SPeter Grehan 
119de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VMX, "vmx", "vmx");
120de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
121366f6083SPeter Grehan 
1223565b59eSNeel Natu SYSCTL_DECL(_hw_vmm);
1233565b59eSNeel Natu SYSCTL_NODE(_hw_vmm, OID_AUTO, vmx, CTLFLAG_RW, NULL, NULL);
1243565b59eSNeel Natu 
125b3996dd4SJohn Baldwin int vmxon_enabled[MAXCPU];
126366f6083SPeter Grehan static char vmxon_region[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE);
127366f6083SPeter Grehan 
128366f6083SPeter Grehan static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2;
129366f6083SPeter Grehan static uint32_t exit_ctls, entry_ctls;
130366f6083SPeter Grehan 
131366f6083SPeter Grehan static uint64_t cr0_ones_mask, cr0_zeros_mask;
1323565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_ones_mask, CTLFLAG_RD,
1333565b59eSNeel Natu 	     &cr0_ones_mask, 0, NULL);
1343565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_zeros_mask, CTLFLAG_RD,
1353565b59eSNeel Natu 	     &cr0_zeros_mask, 0, NULL);
1363565b59eSNeel Natu 
137366f6083SPeter Grehan static uint64_t cr4_ones_mask, cr4_zeros_mask;
1383565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_ones_mask, CTLFLAG_RD,
1393565b59eSNeel Natu 	     &cr4_ones_mask, 0, NULL);
1403565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_zeros_mask, CTLFLAG_RD,
1413565b59eSNeel Natu 	     &cr4_zeros_mask, 0, NULL);
142366f6083SPeter Grehan 
143608f97c3SPeter Grehan static int vmx_no_patmsr;
144608f97c3SPeter Grehan 
1453565b59eSNeel Natu static int vmx_initialized;
1463565b59eSNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, initialized, CTLFLAG_RD,
1473565b59eSNeel Natu 	   &vmx_initialized, 0, "Intel VMX initialized");
1483565b59eSNeel Natu 
149366f6083SPeter Grehan /*
150366f6083SPeter Grehan  * Optional capabilities
151366f6083SPeter Grehan  */
152366f6083SPeter Grehan static int cap_halt_exit;
153366f6083SPeter Grehan static int cap_pause_exit;
154366f6083SPeter Grehan static int cap_unrestricted_guest;
155366f6083SPeter Grehan static int cap_monitor_trap;
15649cc03daSNeel Natu static int cap_invpcid;
157366f6083SPeter Grehan 
15888c4b8d1SNeel Natu static int virtual_interrupt_delivery;
15988c4b8d1SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, virtual_interrupt_delivery, CTLFLAG_RD,
16088c4b8d1SNeel Natu     &virtual_interrupt_delivery, 0, "APICv virtual interrupt delivery support");
16188c4b8d1SNeel Natu 
162176666c2SNeel Natu static int posted_interrupts;
163176666c2SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupts, CTLFLAG_RD,
164176666c2SNeel Natu     &posted_interrupts, 0, "APICv posted interrupt support");
165176666c2SNeel Natu 
166176666c2SNeel Natu static int pirvec;
167176666c2SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupt_vector, CTLFLAG_RD,
168176666c2SNeel Natu     &pirvec, 0, "APICv posted interrupt vector");
169176666c2SNeel Natu 
17045e51299SNeel Natu static struct unrhdr *vpid_unr;
17145e51299SNeel Natu static u_int vpid_alloc_failed;
17245e51299SNeel Natu SYSCTL_UINT(_hw_vmm_vmx, OID_AUTO, vpid_alloc_failed, CTLFLAG_RD,
17345e51299SNeel Natu 	    &vpid_alloc_failed, 0, NULL);
17445e51299SNeel Natu 
17588c4b8d1SNeel Natu /*
17688c4b8d1SNeel Natu  * Use the last page below 4GB as the APIC access address. This address is
17788c4b8d1SNeel Natu  * occupied by the boot firmware so it is guaranteed that it will not conflict
17888c4b8d1SNeel Natu  * with a page in system memory.
17988c4b8d1SNeel Natu  */
18088c4b8d1SNeel Natu #define	APIC_ACCESS_ADDRESS	0xFFFFF000
18188c4b8d1SNeel Natu 
18288c4b8d1SNeel Natu static void vmx_inject_pir(struct vlapic *vlapic);
18388c4b8d1SNeel Natu 
184366f6083SPeter Grehan #ifdef KTR
185366f6083SPeter Grehan static const char *
186366f6083SPeter Grehan exit_reason_to_str(int reason)
187366f6083SPeter Grehan {
188366f6083SPeter Grehan 	static char reasonbuf[32];
189366f6083SPeter Grehan 
190366f6083SPeter Grehan 	switch (reason) {
191366f6083SPeter Grehan 	case EXIT_REASON_EXCEPTION:
192366f6083SPeter Grehan 		return "exception";
193366f6083SPeter Grehan 	case EXIT_REASON_EXT_INTR:
194366f6083SPeter Grehan 		return "extint";
195366f6083SPeter Grehan 	case EXIT_REASON_TRIPLE_FAULT:
196366f6083SPeter Grehan 		return "triplefault";
197366f6083SPeter Grehan 	case EXIT_REASON_INIT:
198366f6083SPeter Grehan 		return "init";
199366f6083SPeter Grehan 	case EXIT_REASON_SIPI:
200366f6083SPeter Grehan 		return "sipi";
201366f6083SPeter Grehan 	case EXIT_REASON_IO_SMI:
202366f6083SPeter Grehan 		return "iosmi";
203366f6083SPeter Grehan 	case EXIT_REASON_SMI:
204366f6083SPeter Grehan 		return "smi";
205366f6083SPeter Grehan 	case EXIT_REASON_INTR_WINDOW:
206366f6083SPeter Grehan 		return "intrwindow";
207366f6083SPeter Grehan 	case EXIT_REASON_NMI_WINDOW:
208366f6083SPeter Grehan 		return "nmiwindow";
209366f6083SPeter Grehan 	case EXIT_REASON_TASK_SWITCH:
210366f6083SPeter Grehan 		return "taskswitch";
211366f6083SPeter Grehan 	case EXIT_REASON_CPUID:
212366f6083SPeter Grehan 		return "cpuid";
213366f6083SPeter Grehan 	case EXIT_REASON_GETSEC:
214366f6083SPeter Grehan 		return "getsec";
215366f6083SPeter Grehan 	case EXIT_REASON_HLT:
216366f6083SPeter Grehan 		return "hlt";
217366f6083SPeter Grehan 	case EXIT_REASON_INVD:
218366f6083SPeter Grehan 		return "invd";
219366f6083SPeter Grehan 	case EXIT_REASON_INVLPG:
220366f6083SPeter Grehan 		return "invlpg";
221366f6083SPeter Grehan 	case EXIT_REASON_RDPMC:
222366f6083SPeter Grehan 		return "rdpmc";
223366f6083SPeter Grehan 	case EXIT_REASON_RDTSC:
224366f6083SPeter Grehan 		return "rdtsc";
225366f6083SPeter Grehan 	case EXIT_REASON_RSM:
226366f6083SPeter Grehan 		return "rsm";
227366f6083SPeter Grehan 	case EXIT_REASON_VMCALL:
228366f6083SPeter Grehan 		return "vmcall";
229366f6083SPeter Grehan 	case EXIT_REASON_VMCLEAR:
230366f6083SPeter Grehan 		return "vmclear";
231366f6083SPeter Grehan 	case EXIT_REASON_VMLAUNCH:
232366f6083SPeter Grehan 		return "vmlaunch";
233366f6083SPeter Grehan 	case EXIT_REASON_VMPTRLD:
234366f6083SPeter Grehan 		return "vmptrld";
235366f6083SPeter Grehan 	case EXIT_REASON_VMPTRST:
236366f6083SPeter Grehan 		return "vmptrst";
237366f6083SPeter Grehan 	case EXIT_REASON_VMREAD:
238366f6083SPeter Grehan 		return "vmread";
239366f6083SPeter Grehan 	case EXIT_REASON_VMRESUME:
240366f6083SPeter Grehan 		return "vmresume";
241366f6083SPeter Grehan 	case EXIT_REASON_VMWRITE:
242366f6083SPeter Grehan 		return "vmwrite";
243366f6083SPeter Grehan 	case EXIT_REASON_VMXOFF:
244366f6083SPeter Grehan 		return "vmxoff";
245366f6083SPeter Grehan 	case EXIT_REASON_VMXON:
246366f6083SPeter Grehan 		return "vmxon";
247366f6083SPeter Grehan 	case EXIT_REASON_CR_ACCESS:
248366f6083SPeter Grehan 		return "craccess";
249366f6083SPeter Grehan 	case EXIT_REASON_DR_ACCESS:
250366f6083SPeter Grehan 		return "draccess";
251366f6083SPeter Grehan 	case EXIT_REASON_INOUT:
252366f6083SPeter Grehan 		return "inout";
253366f6083SPeter Grehan 	case EXIT_REASON_RDMSR:
254366f6083SPeter Grehan 		return "rdmsr";
255366f6083SPeter Grehan 	case EXIT_REASON_WRMSR:
256366f6083SPeter Grehan 		return "wrmsr";
257366f6083SPeter Grehan 	case EXIT_REASON_INVAL_VMCS:
258366f6083SPeter Grehan 		return "invalvmcs";
259366f6083SPeter Grehan 	case EXIT_REASON_INVAL_MSR:
260366f6083SPeter Grehan 		return "invalmsr";
261366f6083SPeter Grehan 	case EXIT_REASON_MWAIT:
262366f6083SPeter Grehan 		return "mwait";
263366f6083SPeter Grehan 	case EXIT_REASON_MTF:
264366f6083SPeter Grehan 		return "mtf";
265366f6083SPeter Grehan 	case EXIT_REASON_MONITOR:
266366f6083SPeter Grehan 		return "monitor";
267366f6083SPeter Grehan 	case EXIT_REASON_PAUSE:
268366f6083SPeter Grehan 		return "pause";
269366f6083SPeter Grehan 	case EXIT_REASON_MCE:
270366f6083SPeter Grehan 		return "mce";
271366f6083SPeter Grehan 	case EXIT_REASON_TPR:
272366f6083SPeter Grehan 		return "tpr";
27388c4b8d1SNeel Natu 	case EXIT_REASON_APIC_ACCESS:
27488c4b8d1SNeel Natu 		return "apic-access";
275366f6083SPeter Grehan 	case EXIT_REASON_GDTR_IDTR:
276366f6083SPeter Grehan 		return "gdtridtr";
277366f6083SPeter Grehan 	case EXIT_REASON_LDTR_TR:
278366f6083SPeter Grehan 		return "ldtrtr";
279366f6083SPeter Grehan 	case EXIT_REASON_EPT_FAULT:
280366f6083SPeter Grehan 		return "eptfault";
281366f6083SPeter Grehan 	case EXIT_REASON_EPT_MISCONFIG:
282366f6083SPeter Grehan 		return "eptmisconfig";
283366f6083SPeter Grehan 	case EXIT_REASON_INVEPT:
284366f6083SPeter Grehan 		return "invept";
285366f6083SPeter Grehan 	case EXIT_REASON_RDTSCP:
286366f6083SPeter Grehan 		return "rdtscp";
287366f6083SPeter Grehan 	case EXIT_REASON_VMX_PREEMPT:
288366f6083SPeter Grehan 		return "vmxpreempt";
289366f6083SPeter Grehan 	case EXIT_REASON_INVVPID:
290366f6083SPeter Grehan 		return "invvpid";
291366f6083SPeter Grehan 	case EXIT_REASON_WBINVD:
292366f6083SPeter Grehan 		return "wbinvd";
293366f6083SPeter Grehan 	case EXIT_REASON_XSETBV:
294366f6083SPeter Grehan 		return "xsetbv";
29588c4b8d1SNeel Natu 	case EXIT_REASON_APIC_WRITE:
29688c4b8d1SNeel Natu 		return "apic-write";
297366f6083SPeter Grehan 	default:
298366f6083SPeter Grehan 		snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason);
299366f6083SPeter Grehan 		return (reasonbuf);
300366f6083SPeter Grehan 	}
301366f6083SPeter Grehan }
302366f6083SPeter Grehan #endif	/* KTR */
303366f6083SPeter Grehan 
304366f6083SPeter Grehan u_long
305366f6083SPeter Grehan vmx_fix_cr0(u_long cr0)
306366f6083SPeter Grehan {
307366f6083SPeter Grehan 
308366f6083SPeter Grehan 	return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask);
309366f6083SPeter Grehan }
310366f6083SPeter Grehan 
311366f6083SPeter Grehan u_long
312366f6083SPeter Grehan vmx_fix_cr4(u_long cr4)
313366f6083SPeter Grehan {
314366f6083SPeter Grehan 
315366f6083SPeter Grehan 	return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask);
316366f6083SPeter Grehan }
317366f6083SPeter Grehan 
318366f6083SPeter Grehan static void
31945e51299SNeel Natu vpid_free(int vpid)
32045e51299SNeel Natu {
32145e51299SNeel Natu 	if (vpid < 0 || vpid > 0xffff)
32245e51299SNeel Natu 		panic("vpid_free: invalid vpid %d", vpid);
32345e51299SNeel Natu 
32445e51299SNeel Natu 	/*
32545e51299SNeel Natu 	 * VPIDs [0,VM_MAXCPU] are special and are not allocated from
32645e51299SNeel Natu 	 * the unit number allocator.
32745e51299SNeel Natu 	 */
32845e51299SNeel Natu 
32945e51299SNeel Natu 	if (vpid > VM_MAXCPU)
33045e51299SNeel Natu 		free_unr(vpid_unr, vpid);
33145e51299SNeel Natu }
33245e51299SNeel Natu 
33345e51299SNeel Natu static void
33445e51299SNeel Natu vpid_alloc(uint16_t *vpid, int num)
33545e51299SNeel Natu {
33645e51299SNeel Natu 	int i, x;
33745e51299SNeel Natu 
33845e51299SNeel Natu 	if (num <= 0 || num > VM_MAXCPU)
33945e51299SNeel Natu 		panic("invalid number of vpids requested: %d", num);
34045e51299SNeel Natu 
34145e51299SNeel Natu 	/*
34245e51299SNeel Natu 	 * If the "enable vpid" execution control is not enabled then the
34345e51299SNeel Natu 	 * VPID is required to be 0 for all vcpus.
34445e51299SNeel Natu 	 */
34545e51299SNeel Natu 	if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) == 0) {
34645e51299SNeel Natu 		for (i = 0; i < num; i++)
34745e51299SNeel Natu 			vpid[i] = 0;
34845e51299SNeel Natu 		return;
34945e51299SNeel Natu 	}
35045e51299SNeel Natu 
35145e51299SNeel Natu 	/*
35245e51299SNeel Natu 	 * Allocate a unique VPID for each vcpu from the unit number allocator.
35345e51299SNeel Natu 	 */
35445e51299SNeel Natu 	for (i = 0; i < num; i++) {
35545e51299SNeel Natu 		x = alloc_unr(vpid_unr);
35645e51299SNeel Natu 		if (x == -1)
35745e51299SNeel Natu 			break;
35845e51299SNeel Natu 		else
35945e51299SNeel Natu 			vpid[i] = x;
36045e51299SNeel Natu 	}
36145e51299SNeel Natu 
36245e51299SNeel Natu 	if (i < num) {
36345e51299SNeel Natu 		atomic_add_int(&vpid_alloc_failed, 1);
36445e51299SNeel Natu 
36545e51299SNeel Natu 		/*
36645e51299SNeel Natu 		 * If the unit number allocator does not have enough unique
36745e51299SNeel Natu 		 * VPIDs then we need to allocate from the [1,VM_MAXCPU] range.
36845e51299SNeel Natu 		 *
36945e51299SNeel Natu 		 * These VPIDs are not be unique across VMs but this does not
37045e51299SNeel Natu 		 * affect correctness because the combined mappings are also
37145e51299SNeel Natu 		 * tagged with the EP4TA which is unique for each VM.
37245e51299SNeel Natu 		 *
37345e51299SNeel Natu 		 * It is still sub-optimal because the invvpid will invalidate
37445e51299SNeel Natu 		 * combined mappings for a particular VPID across all EP4TAs.
37545e51299SNeel Natu 		 */
37645e51299SNeel Natu 		while (i-- > 0)
37745e51299SNeel Natu 			vpid_free(vpid[i]);
37845e51299SNeel Natu 
37945e51299SNeel Natu 		for (i = 0; i < num; i++)
38045e51299SNeel Natu 			vpid[i] = i + 1;
38145e51299SNeel Natu 	}
38245e51299SNeel Natu }
38345e51299SNeel Natu 
38445e51299SNeel Natu static void
38545e51299SNeel Natu vpid_init(void)
38645e51299SNeel Natu {
38745e51299SNeel Natu 	/*
38845e51299SNeel Natu 	 * VPID 0 is required when the "enable VPID" execution control is
38945e51299SNeel Natu 	 * disabled.
39045e51299SNeel Natu 	 *
39145e51299SNeel Natu 	 * VPIDs [1,VM_MAXCPU] are used as the "overflow namespace" when the
39245e51299SNeel Natu 	 * unit number allocator does not have sufficient unique VPIDs to
39345e51299SNeel Natu 	 * satisfy the allocation.
39445e51299SNeel Natu 	 *
39545e51299SNeel Natu 	 * The remaining VPIDs are managed by the unit number allocator.
39645e51299SNeel Natu 	 */
39745e51299SNeel Natu 	vpid_unr = new_unrhdr(VM_MAXCPU + 1, 0xffff, NULL);
39845e51299SNeel Natu }
39945e51299SNeel Natu 
40045e51299SNeel Natu static void
401366f6083SPeter Grehan msr_save_area_init(struct msr_entry *g_area, int *g_count)
402366f6083SPeter Grehan {
403366f6083SPeter Grehan 	int cnt;
404366f6083SPeter Grehan 
405366f6083SPeter Grehan 	static struct msr_entry guest_msrs[] = {
406366f6083SPeter Grehan 		{ MSR_KGSBASE, 0, 0 },
407366f6083SPeter Grehan 	};
408366f6083SPeter Grehan 
409366f6083SPeter Grehan 	cnt = sizeof(guest_msrs) / sizeof(guest_msrs[0]);
410366f6083SPeter Grehan 	if (cnt > GUEST_MSR_MAX_ENTRIES)
411366f6083SPeter Grehan 		panic("guest msr save area overrun");
412366f6083SPeter Grehan 	bcopy(guest_msrs, g_area, sizeof(guest_msrs));
413366f6083SPeter Grehan 	*g_count = cnt;
414366f6083SPeter Grehan }
415366f6083SPeter Grehan 
416366f6083SPeter Grehan static void
417366f6083SPeter Grehan vmx_disable(void *arg __unused)
418366f6083SPeter Grehan {
419366f6083SPeter Grehan 	struct invvpid_desc invvpid_desc = { 0 };
420366f6083SPeter Grehan 	struct invept_desc invept_desc = { 0 };
421366f6083SPeter Grehan 
422366f6083SPeter Grehan 	if (vmxon_enabled[curcpu]) {
423366f6083SPeter Grehan 		/*
424366f6083SPeter Grehan 		 * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b.
425366f6083SPeter Grehan 		 *
426366f6083SPeter Grehan 		 * VMXON or VMXOFF are not required to invalidate any TLB
427366f6083SPeter Grehan 		 * caching structures. This prevents potential retention of
428366f6083SPeter Grehan 		 * cached information in the TLB between distinct VMX episodes.
429366f6083SPeter Grehan 		 */
430366f6083SPeter Grehan 		invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc);
431366f6083SPeter Grehan 		invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc);
432366f6083SPeter Grehan 		vmxoff();
433366f6083SPeter Grehan 	}
434366f6083SPeter Grehan 	load_cr4(rcr4() & ~CR4_VMXE);
435366f6083SPeter Grehan }
436366f6083SPeter Grehan 
437366f6083SPeter Grehan static int
438366f6083SPeter Grehan vmx_cleanup(void)
439366f6083SPeter Grehan {
440366f6083SPeter Grehan 
441176666c2SNeel Natu 	if (pirvec != 0)
442176666c2SNeel Natu 		vmm_ipi_free(pirvec);
443176666c2SNeel Natu 
44445e51299SNeel Natu 	if (vpid_unr != NULL) {
44545e51299SNeel Natu 		delete_unrhdr(vpid_unr);
44645e51299SNeel Natu 		vpid_unr = NULL;
44745e51299SNeel Natu 	}
44845e51299SNeel Natu 
449366f6083SPeter Grehan 	smp_rendezvous(NULL, vmx_disable, NULL, NULL);
450366f6083SPeter Grehan 
451366f6083SPeter Grehan 	return (0);
452366f6083SPeter Grehan }
453366f6083SPeter Grehan 
454366f6083SPeter Grehan static void
455366f6083SPeter Grehan vmx_enable(void *arg __unused)
456366f6083SPeter Grehan {
457366f6083SPeter Grehan 	int error;
458366f6083SPeter Grehan 
459366f6083SPeter Grehan 	load_cr4(rcr4() | CR4_VMXE);
460366f6083SPeter Grehan 
461366f6083SPeter Grehan 	*(uint32_t *)vmxon_region[curcpu] = vmx_revision();
462366f6083SPeter Grehan 	error = vmxon(vmxon_region[curcpu]);
463366f6083SPeter Grehan 	if (error == 0)
464366f6083SPeter Grehan 		vmxon_enabled[curcpu] = 1;
465366f6083SPeter Grehan }
466366f6083SPeter Grehan 
46763e62d39SJohn Baldwin static void
46863e62d39SJohn Baldwin vmx_restore(void)
46963e62d39SJohn Baldwin {
47063e62d39SJohn Baldwin 
47163e62d39SJohn Baldwin 	if (vmxon_enabled[curcpu])
47263e62d39SJohn Baldwin 		vmxon(vmxon_region[curcpu]);
47363e62d39SJohn Baldwin }
47463e62d39SJohn Baldwin 
475366f6083SPeter Grehan static int
476add611fdSNeel Natu vmx_init(int ipinum)
477366f6083SPeter Grehan {
47888c4b8d1SNeel Natu 	int error, use_tpr_shadow;
4794bff7fadSNeel Natu 	uint64_t fixed0, fixed1, feature_control;
48088c4b8d1SNeel Natu 	uint32_t tmp, procbased2_vid_bits;
481366f6083SPeter Grehan 
482366f6083SPeter Grehan 	/* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */
4838b287612SJohn Baldwin 	if (!(cpu_feature2 & CPUID2_VMX)) {
484366f6083SPeter Grehan 		printf("vmx_init: processor does not support VMX operation\n");
485366f6083SPeter Grehan 		return (ENXIO);
486366f6083SPeter Grehan 	}
487366f6083SPeter Grehan 
4884bff7fadSNeel Natu 	/*
4894bff7fadSNeel Natu 	 * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits
4904bff7fadSNeel Natu 	 * are set (bits 0 and 2 respectively).
4914bff7fadSNeel Natu 	 */
4924bff7fadSNeel Natu 	feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
493150369abSNeel Natu 	if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 0 ||
494150369abSNeel Natu 	    (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) {
4954bff7fadSNeel Natu 		printf("vmx_init: VMX operation disabled by BIOS\n");
4964bff7fadSNeel Natu 		return (ENXIO);
4974bff7fadSNeel Natu 	}
4984bff7fadSNeel Natu 
499366f6083SPeter Grehan 	/* Check support for primary processor-based VM-execution controls */
500366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
501366f6083SPeter Grehan 			       MSR_VMX_TRUE_PROCBASED_CTLS,
502366f6083SPeter Grehan 			       PROCBASED_CTLS_ONE_SETTING,
503366f6083SPeter Grehan 			       PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls);
504366f6083SPeter Grehan 	if (error) {
505366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired primary "
506366f6083SPeter Grehan 		       "processor-based controls\n");
507366f6083SPeter Grehan 		return (error);
508366f6083SPeter Grehan 	}
509366f6083SPeter Grehan 
510366f6083SPeter Grehan 	/* Clear the processor-based ctl bits that are set on demand */
511366f6083SPeter Grehan 	procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING;
512366f6083SPeter Grehan 
513366f6083SPeter Grehan 	/* Check support for secondary processor-based VM-execution controls */
514366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
515366f6083SPeter Grehan 			       MSR_VMX_PROCBASED_CTLS2,
516366f6083SPeter Grehan 			       PROCBASED_CTLS2_ONE_SETTING,
517366f6083SPeter Grehan 			       PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2);
518366f6083SPeter Grehan 	if (error) {
519366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired secondary "
520366f6083SPeter Grehan 		       "processor-based controls\n");
521366f6083SPeter Grehan 		return (error);
522366f6083SPeter Grehan 	}
523366f6083SPeter Grehan 
524366f6083SPeter Grehan 	/* Check support for VPID */
525366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
526366f6083SPeter Grehan 			       PROCBASED2_ENABLE_VPID, 0, &tmp);
527366f6083SPeter Grehan 	if (error == 0)
528366f6083SPeter Grehan 		procbased_ctls2 |= PROCBASED2_ENABLE_VPID;
529366f6083SPeter Grehan 
530366f6083SPeter Grehan 	/* Check support for pin-based VM-execution controls */
531366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
532366f6083SPeter Grehan 			       MSR_VMX_TRUE_PINBASED_CTLS,
533366f6083SPeter Grehan 			       PINBASED_CTLS_ONE_SETTING,
534366f6083SPeter Grehan 			       PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls);
535366f6083SPeter Grehan 	if (error) {
536366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired "
537366f6083SPeter Grehan 		       "pin-based controls\n");
538366f6083SPeter Grehan 		return (error);
539366f6083SPeter Grehan 	}
540366f6083SPeter Grehan 
541366f6083SPeter Grehan 	/* Check support for VM-exit controls */
542366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS,
543366f6083SPeter Grehan 			       VM_EXIT_CTLS_ONE_SETTING,
544366f6083SPeter Grehan 			       VM_EXIT_CTLS_ZERO_SETTING,
545366f6083SPeter Grehan 			       &exit_ctls);
546366f6083SPeter Grehan 	if (error) {
547608f97c3SPeter Grehan 		/* Try again without the PAT MSR bits */
548608f97c3SPeter Grehan 		error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS,
549608f97c3SPeter Grehan 				       MSR_VMX_TRUE_EXIT_CTLS,
550608f97c3SPeter Grehan 				       VM_EXIT_CTLS_ONE_SETTING_NO_PAT,
551608f97c3SPeter Grehan 				       VM_EXIT_CTLS_ZERO_SETTING,
552608f97c3SPeter Grehan 				       &exit_ctls);
553608f97c3SPeter Grehan 		if (error) {
554366f6083SPeter Grehan 			printf("vmx_init: processor does not support desired "
555366f6083SPeter Grehan 			       "exit controls\n");
556366f6083SPeter Grehan 			return (error);
557608f97c3SPeter Grehan 		} else {
558608f97c3SPeter Grehan 			if (bootverbose)
559608f97c3SPeter Grehan 				printf("vmm: PAT MSR access not supported\n");
560608f97c3SPeter Grehan 			guest_msr_valid(MSR_PAT);
561608f97c3SPeter Grehan 			vmx_no_patmsr = 1;
562608f97c3SPeter Grehan 		}
563366f6083SPeter Grehan 	}
564366f6083SPeter Grehan 
565366f6083SPeter Grehan 	/* Check support for VM-entry controls */
566608f97c3SPeter Grehan 	if (!vmx_no_patmsr) {
567608f97c3SPeter Grehan 		error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS,
568608f97c3SPeter Grehan 				       MSR_VMX_TRUE_ENTRY_CTLS,
569366f6083SPeter Grehan 				       VM_ENTRY_CTLS_ONE_SETTING,
570366f6083SPeter Grehan 				       VM_ENTRY_CTLS_ZERO_SETTING,
571366f6083SPeter Grehan 				       &entry_ctls);
572608f97c3SPeter Grehan 	} else {
573608f97c3SPeter Grehan 		error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS,
574608f97c3SPeter Grehan 				       MSR_VMX_TRUE_ENTRY_CTLS,
575608f97c3SPeter Grehan 				       VM_ENTRY_CTLS_ONE_SETTING_NO_PAT,
576608f97c3SPeter Grehan 				       VM_ENTRY_CTLS_ZERO_SETTING,
577608f97c3SPeter Grehan 				       &entry_ctls);
578608f97c3SPeter Grehan 	}
579608f97c3SPeter Grehan 
580366f6083SPeter Grehan 	if (error) {
581366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired "
582366f6083SPeter Grehan 		       "entry controls\n");
583366f6083SPeter Grehan 		       return (error);
584366f6083SPeter Grehan 	}
585366f6083SPeter Grehan 
586366f6083SPeter Grehan 	/*
587366f6083SPeter Grehan 	 * Check support for optional features by testing them
588366f6083SPeter Grehan 	 * as individual bits
589366f6083SPeter Grehan 	 */
590366f6083SPeter Grehan 	cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
591366f6083SPeter Grehan 					MSR_VMX_TRUE_PROCBASED_CTLS,
592366f6083SPeter Grehan 					PROCBASED_HLT_EXITING, 0,
593366f6083SPeter Grehan 					&tmp) == 0);
594366f6083SPeter Grehan 
595366f6083SPeter Grehan 	cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
596366f6083SPeter Grehan 					MSR_VMX_PROCBASED_CTLS,
597366f6083SPeter Grehan 					PROCBASED_MTF, 0,
598366f6083SPeter Grehan 					&tmp) == 0);
599366f6083SPeter Grehan 
600366f6083SPeter Grehan 	cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
601366f6083SPeter Grehan 					 MSR_VMX_TRUE_PROCBASED_CTLS,
602366f6083SPeter Grehan 					 PROCBASED_PAUSE_EXITING, 0,
603366f6083SPeter Grehan 					 &tmp) == 0);
604366f6083SPeter Grehan 
605366f6083SPeter Grehan 	cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
606366f6083SPeter Grehan 					MSR_VMX_PROCBASED_CTLS2,
607366f6083SPeter Grehan 					PROCBASED2_UNRESTRICTED_GUEST, 0,
608366f6083SPeter Grehan 				        &tmp) == 0);
609366f6083SPeter Grehan 
61049cc03daSNeel Natu 	cap_invpcid = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
61149cc03daSNeel Natu 	    MSR_VMX_PROCBASED_CTLS2, PROCBASED2_ENABLE_INVPCID, 0,
61249cc03daSNeel Natu 	    &tmp) == 0);
61349cc03daSNeel Natu 
61488c4b8d1SNeel Natu 	/*
61588c4b8d1SNeel Natu 	 * Check support for virtual interrupt delivery.
61688c4b8d1SNeel Natu 	 */
61788c4b8d1SNeel Natu 	procbased2_vid_bits = (PROCBASED2_VIRTUALIZE_APIC_ACCESSES |
61888c4b8d1SNeel Natu 	    PROCBASED2_VIRTUALIZE_X2APIC_MODE |
61988c4b8d1SNeel Natu 	    PROCBASED2_APIC_REGISTER_VIRTUALIZATION |
62088c4b8d1SNeel Natu 	    PROCBASED2_VIRTUAL_INTERRUPT_DELIVERY);
62188c4b8d1SNeel Natu 
62288c4b8d1SNeel Natu 	use_tpr_shadow = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
62388c4b8d1SNeel Natu 	    MSR_VMX_TRUE_PROCBASED_CTLS, PROCBASED_USE_TPR_SHADOW, 0,
62488c4b8d1SNeel Natu 	    &tmp) == 0);
62588c4b8d1SNeel Natu 
62688c4b8d1SNeel Natu 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
62788c4b8d1SNeel Natu 	    procbased2_vid_bits, 0, &tmp);
62888c4b8d1SNeel Natu 	if (error == 0 && use_tpr_shadow) {
62988c4b8d1SNeel Natu 		virtual_interrupt_delivery = 1;
63088c4b8d1SNeel Natu 		TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_vid",
63188c4b8d1SNeel Natu 		    &virtual_interrupt_delivery);
63288c4b8d1SNeel Natu 	}
63388c4b8d1SNeel Natu 
63488c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
63588c4b8d1SNeel Natu 		procbased_ctls |= PROCBASED_USE_TPR_SHADOW;
63688c4b8d1SNeel Natu 		procbased_ctls2 |= procbased2_vid_bits;
63788c4b8d1SNeel Natu 		procbased_ctls2 &= ~PROCBASED2_VIRTUALIZE_X2APIC_MODE;
638176666c2SNeel Natu 
639176666c2SNeel Natu 		/*
640176666c2SNeel Natu 		 * Check for Posted Interrupts only if Virtual Interrupt
641176666c2SNeel Natu 		 * Delivery is enabled.
642176666c2SNeel Natu 		 */
643176666c2SNeel Natu 		error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
644176666c2SNeel Natu 		    MSR_VMX_TRUE_PINBASED_CTLS, PINBASED_POSTED_INTERRUPT, 0,
645176666c2SNeel Natu 		    &tmp);
646176666c2SNeel Natu 		if (error == 0) {
647176666c2SNeel Natu 			pirvec = vmm_ipi_alloc();
648176666c2SNeel Natu 			if (pirvec == 0) {
649176666c2SNeel Natu 				if (bootverbose) {
650176666c2SNeel Natu 					printf("vmx_init: unable to allocate "
651176666c2SNeel Natu 					    "posted interrupt vector\n");
65288c4b8d1SNeel Natu 				}
653176666c2SNeel Natu 			} else {
654176666c2SNeel Natu 				posted_interrupts = 1;
655176666c2SNeel Natu 				TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_pir",
656176666c2SNeel Natu 				    &posted_interrupts);
657176666c2SNeel Natu 			}
658176666c2SNeel Natu 		}
659176666c2SNeel Natu 	}
660176666c2SNeel Natu 
661176666c2SNeel Natu 	if (posted_interrupts)
662176666c2SNeel Natu 		    pinbased_ctls |= PINBASED_POSTED_INTERRUPT;
66349cc03daSNeel Natu 
664366f6083SPeter Grehan 	/* Initialize EPT */
665add611fdSNeel Natu 	error = ept_init(ipinum);
666366f6083SPeter Grehan 	if (error) {
667366f6083SPeter Grehan 		printf("vmx_init: ept initialization failed (%d)\n", error);
668366f6083SPeter Grehan 		return (error);
669366f6083SPeter Grehan 	}
670366f6083SPeter Grehan 
671366f6083SPeter Grehan 	/*
672366f6083SPeter Grehan 	 * Stash the cr0 and cr4 bits that must be fixed to 0 or 1
673366f6083SPeter Grehan 	 */
674366f6083SPeter Grehan 	fixed0 = rdmsr(MSR_VMX_CR0_FIXED0);
675366f6083SPeter Grehan 	fixed1 = rdmsr(MSR_VMX_CR0_FIXED1);
676366f6083SPeter Grehan 	cr0_ones_mask = fixed0 & fixed1;
677366f6083SPeter Grehan 	cr0_zeros_mask = ~fixed0 & ~fixed1;
678366f6083SPeter Grehan 
679366f6083SPeter Grehan 	/*
680366f6083SPeter Grehan 	 * CR0_PE and CR0_PG can be set to zero in VMX non-root operation
681366f6083SPeter Grehan 	 * if unrestricted guest execution is allowed.
682366f6083SPeter Grehan 	 */
683366f6083SPeter Grehan 	if (cap_unrestricted_guest)
684366f6083SPeter Grehan 		cr0_ones_mask &= ~(CR0_PG | CR0_PE);
685366f6083SPeter Grehan 
686366f6083SPeter Grehan 	/*
687366f6083SPeter Grehan 	 * Do not allow the guest to set CR0_NW or CR0_CD.
688366f6083SPeter Grehan 	 */
689366f6083SPeter Grehan 	cr0_zeros_mask |= (CR0_NW | CR0_CD);
690366f6083SPeter Grehan 
691366f6083SPeter Grehan 	fixed0 = rdmsr(MSR_VMX_CR4_FIXED0);
692366f6083SPeter Grehan 	fixed1 = rdmsr(MSR_VMX_CR4_FIXED1);
693366f6083SPeter Grehan 	cr4_ones_mask = fixed0 & fixed1;
694366f6083SPeter Grehan 	cr4_zeros_mask = ~fixed0 & ~fixed1;
695366f6083SPeter Grehan 
69645e51299SNeel Natu 	vpid_init();
69745e51299SNeel Natu 
698366f6083SPeter Grehan 	/* enable VMX operation */
699366f6083SPeter Grehan 	smp_rendezvous(NULL, vmx_enable, NULL, NULL);
700366f6083SPeter Grehan 
7013565b59eSNeel Natu 	vmx_initialized = 1;
7023565b59eSNeel Natu 
703366f6083SPeter Grehan 	return (0);
704366f6083SPeter Grehan }
705366f6083SPeter Grehan 
706f7d47425SNeel Natu static void
707f7d47425SNeel Natu vmx_trigger_hostintr(int vector)
708f7d47425SNeel Natu {
709f7d47425SNeel Natu 	uintptr_t func;
710f7d47425SNeel Natu 	struct gate_descriptor *gd;
711f7d47425SNeel Natu 
712f7d47425SNeel Natu 	gd = &idt[vector];
713f7d47425SNeel Natu 
714f7d47425SNeel Natu 	KASSERT(vector >= 32 && vector <= 255, ("vmx_trigger_hostintr: "
715f7d47425SNeel Natu 	    "invalid vector %d", vector));
716f7d47425SNeel Natu 	KASSERT(gd->gd_p == 1, ("gate descriptor for vector %d not present",
717f7d47425SNeel Natu 	    vector));
718f7d47425SNeel Natu 	KASSERT(gd->gd_type == SDT_SYSIGT, ("gate descriptor for vector %d "
719f7d47425SNeel Natu 	    "has invalid type %d", vector, gd->gd_type));
720f7d47425SNeel Natu 	KASSERT(gd->gd_dpl == SEL_KPL, ("gate descriptor for vector %d "
721f7d47425SNeel Natu 	    "has invalid dpl %d", vector, gd->gd_dpl));
722f7d47425SNeel Natu 	KASSERT(gd->gd_selector == GSEL(GCODE_SEL, SEL_KPL), ("gate descriptor "
723f7d47425SNeel Natu 	    "for vector %d has invalid selector %d", vector, gd->gd_selector));
724f7d47425SNeel Natu 	KASSERT(gd->gd_ist == 0, ("gate descriptor for vector %d has invalid "
725f7d47425SNeel Natu 	    "IST %d", vector, gd->gd_ist));
726f7d47425SNeel Natu 
727f7d47425SNeel Natu 	func = ((long)gd->gd_hioffset << 16 | gd->gd_looffset);
728f7d47425SNeel Natu 	vmx_call_isr(func);
729f7d47425SNeel Natu }
730f7d47425SNeel Natu 
731366f6083SPeter Grehan static int
732aaaa0656SPeter Grehan vmx_setup_cr_shadow(int which, struct vmcs *vmcs, uint32_t initial)
733366f6083SPeter Grehan {
73439c21c2dSNeel Natu 	int error, mask_ident, shadow_ident;
735aaaa0656SPeter Grehan 	uint64_t mask_value;
736366f6083SPeter Grehan 
73739c21c2dSNeel Natu 	if (which != 0 && which != 4)
73839c21c2dSNeel Natu 		panic("vmx_setup_cr_shadow: unknown cr%d", which);
73939c21c2dSNeel Natu 
74039c21c2dSNeel Natu 	if (which == 0) {
74139c21c2dSNeel Natu 		mask_ident = VMCS_CR0_MASK;
74239c21c2dSNeel Natu 		mask_value = cr0_ones_mask | cr0_zeros_mask;
74339c21c2dSNeel Natu 		shadow_ident = VMCS_CR0_SHADOW;
74439c21c2dSNeel Natu 	} else {
74539c21c2dSNeel Natu 		mask_ident = VMCS_CR4_MASK;
74639c21c2dSNeel Natu 		mask_value = cr4_ones_mask | cr4_zeros_mask;
74739c21c2dSNeel Natu 		shadow_ident = VMCS_CR4_SHADOW;
74839c21c2dSNeel Natu 	}
74939c21c2dSNeel Natu 
750d3c11f40SPeter Grehan 	error = vmcs_setreg(vmcs, 0, VMCS_IDENT(mask_ident), mask_value);
751366f6083SPeter Grehan 	if (error)
752366f6083SPeter Grehan 		return (error);
753366f6083SPeter Grehan 
754aaaa0656SPeter Grehan 	error = vmcs_setreg(vmcs, 0, VMCS_IDENT(shadow_ident), initial);
755366f6083SPeter Grehan 	if (error)
756366f6083SPeter Grehan 		return (error);
757366f6083SPeter Grehan 
758366f6083SPeter Grehan 	return (0);
759366f6083SPeter Grehan }
760aaaa0656SPeter Grehan #define	vmx_setup_cr0_shadow(vmcs,init)	vmx_setup_cr_shadow(0, (vmcs), (init))
761aaaa0656SPeter Grehan #define	vmx_setup_cr4_shadow(vmcs,init)	vmx_setup_cr_shadow(4, (vmcs), (init))
762366f6083SPeter Grehan 
763366f6083SPeter Grehan static void *
764318224bbSNeel Natu vmx_vminit(struct vm *vm, pmap_t pmap)
765366f6083SPeter Grehan {
76645e51299SNeel Natu 	uint16_t vpid[VM_MAXCPU];
767366f6083SPeter Grehan 	int i, error, guest_msr_count;
768366f6083SPeter Grehan 	struct vmx *vmx;
769c847a506SNeel Natu 	struct vmcs *vmcs;
770366f6083SPeter Grehan 
771366f6083SPeter Grehan 	vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO);
772366f6083SPeter Grehan 	if ((uintptr_t)vmx & PAGE_MASK) {
773366f6083SPeter Grehan 		panic("malloc of struct vmx not aligned on %d byte boundary",
774366f6083SPeter Grehan 		      PAGE_SIZE);
775366f6083SPeter Grehan 	}
776366f6083SPeter Grehan 	vmx->vm = vm;
777366f6083SPeter Grehan 
778318224bbSNeel Natu 	vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pml4));
779318224bbSNeel Natu 
780366f6083SPeter Grehan 	/*
781366f6083SPeter Grehan 	 * Clean up EPTP-tagged guest physical and combined mappings
782366f6083SPeter Grehan 	 *
783366f6083SPeter Grehan 	 * VMX transitions are not required to invalidate any guest physical
784366f6083SPeter Grehan 	 * mappings. So, it may be possible for stale guest physical mappings
785366f6083SPeter Grehan 	 * to be present in the processor TLBs.
786366f6083SPeter Grehan 	 *
787366f6083SPeter Grehan 	 * Combined mappings for this EP4TA are also invalidated for all VPIDs.
788366f6083SPeter Grehan 	 */
789318224bbSNeel Natu 	ept_invalidate_mappings(vmx->eptp);
790366f6083SPeter Grehan 
791366f6083SPeter Grehan 	msr_bitmap_initialize(vmx->msr_bitmap);
792366f6083SPeter Grehan 
793366f6083SPeter Grehan 	/*
794366f6083SPeter Grehan 	 * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE.
795366f6083SPeter Grehan 	 * The guest FSBASE and GSBASE are saved and restored during
796366f6083SPeter Grehan 	 * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are
797366f6083SPeter Grehan 	 * always restored from the vmcs host state area on vm-exit.
798366f6083SPeter Grehan 	 *
7991fb0ea3fSPeter Grehan 	 * The SYSENTER_CS/ESP/EIP MSRs are identical to FS/GSBASE in
8001fb0ea3fSPeter Grehan 	 * how they are saved/restored so can be directly accessed by the
8011fb0ea3fSPeter Grehan 	 * guest.
8021fb0ea3fSPeter Grehan 	 *
803366f6083SPeter Grehan 	 * Guest KGSBASE is saved and restored in the guest MSR save area.
804366f6083SPeter Grehan 	 * Host KGSBASE is restored before returning to userland from the pcb.
805366f6083SPeter Grehan 	 * There will be a window of time when we are executing in the host
806366f6083SPeter Grehan 	 * kernel context with a value of KGSBASE from the guest. This is ok
807366f6083SPeter Grehan 	 * because the value of KGSBASE is inconsequential in kernel context.
808366f6083SPeter Grehan 	 *
809366f6083SPeter Grehan 	 * MSR_EFER is saved and restored in the guest VMCS area on a
810366f6083SPeter Grehan 	 * VM exit and entry respectively. It is also restored from the
811366f6083SPeter Grehan 	 * host VMCS area on a VM exit.
812366f6083SPeter Grehan 	 */
813366f6083SPeter Grehan 	if (guest_msr_rw(vmx, MSR_GSBASE) ||
814366f6083SPeter Grehan 	    guest_msr_rw(vmx, MSR_FSBASE) ||
8151fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_CS_MSR) ||
8161fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_ESP_MSR) ||
8171fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_EIP_MSR) ||
818366f6083SPeter Grehan 	    guest_msr_rw(vmx, MSR_KGSBASE) ||
819608f97c3SPeter Grehan 	    guest_msr_rw(vmx, MSR_EFER))
820366f6083SPeter Grehan 		panic("vmx_vminit: error setting guest msr access");
821366f6083SPeter Grehan 
822608f97c3SPeter Grehan 	/*
823608f97c3SPeter Grehan 	 * MSR_PAT is saved and restored in the guest VMCS are on a VM exit
824608f97c3SPeter Grehan 	 * and entry respectively. It is also restored from the host VMCS
825608f97c3SPeter Grehan 	 * area on a VM exit. However, if running on a system with no
826608f97c3SPeter Grehan 	 * MSR_PAT save/restore support, leave access disabled so accesses
827608f97c3SPeter Grehan 	 * will be trapped.
828608f97c3SPeter Grehan 	 */
829608f97c3SPeter Grehan 	if (!vmx_no_patmsr && guest_msr_rw(vmx, MSR_PAT))
830608f97c3SPeter Grehan 		panic("vmx_vminit: error setting guest pat msr access");
831608f97c3SPeter Grehan 
83245e51299SNeel Natu 	vpid_alloc(vpid, VM_MAXCPU);
83345e51299SNeel Natu 
83488c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
83588c4b8d1SNeel Natu 		error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE,
83688c4b8d1SNeel Natu 		    APIC_ACCESS_ADDRESS);
83788c4b8d1SNeel Natu 		/* XXX this should really return an error to the caller */
83888c4b8d1SNeel Natu 		KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error));
83988c4b8d1SNeel Natu 	}
84088c4b8d1SNeel Natu 
841366f6083SPeter Grehan 	for (i = 0; i < VM_MAXCPU; i++) {
842c847a506SNeel Natu 		vmcs = &vmx->vmcs[i];
843c847a506SNeel Natu 		vmcs->identifier = vmx_revision();
844c847a506SNeel Natu 		error = vmclear(vmcs);
845366f6083SPeter Grehan 		if (error != 0) {
846366f6083SPeter Grehan 			panic("vmx_vminit: vmclear error %d on vcpu %d\n",
847366f6083SPeter Grehan 			      error, i);
848366f6083SPeter Grehan 		}
849366f6083SPeter Grehan 
850c847a506SNeel Natu 		error = vmcs_init(vmcs);
851c847a506SNeel Natu 		KASSERT(error == 0, ("vmcs_init error %d", error));
852366f6083SPeter Grehan 
853c847a506SNeel Natu 		VMPTRLD(vmcs);
854c847a506SNeel Natu 		error = 0;
855c847a506SNeel Natu 		error += vmwrite(VMCS_HOST_RSP, (u_long)&vmx->ctx[i]);
856c847a506SNeel Natu 		error += vmwrite(VMCS_EPTP, vmx->eptp);
857c847a506SNeel Natu 		error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls);
858c847a506SNeel Natu 		error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls);
859c847a506SNeel Natu 		error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2);
860c847a506SNeel Natu 		error += vmwrite(VMCS_EXIT_CTLS, exit_ctls);
861c847a506SNeel Natu 		error += vmwrite(VMCS_ENTRY_CTLS, entry_ctls);
862c847a506SNeel Natu 		error += vmwrite(VMCS_MSR_BITMAP, vtophys(vmx->msr_bitmap));
863c847a506SNeel Natu 		error += vmwrite(VMCS_VPID, vpid[i]);
86488c4b8d1SNeel Natu 		if (virtual_interrupt_delivery) {
86588c4b8d1SNeel Natu 			error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS);
86688c4b8d1SNeel Natu 			error += vmwrite(VMCS_VIRTUAL_APIC,
86788c4b8d1SNeel Natu 			    vtophys(&vmx->apic_page[i]));
86888c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT0, 0);
86988c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT1, 0);
87088c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT2, 0);
87188c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT3, 0);
87288c4b8d1SNeel Natu 		}
873176666c2SNeel Natu 		if (posted_interrupts) {
874176666c2SNeel Natu 			error += vmwrite(VMCS_PIR_VECTOR, pirvec);
875176666c2SNeel Natu 			error += vmwrite(VMCS_PIR_DESC,
876176666c2SNeel Natu 			    vtophys(&vmx->pir_desc[i]));
877176666c2SNeel Natu 		}
878c847a506SNeel Natu 		VMCLEAR(vmcs);
879c847a506SNeel Natu 		KASSERT(error == 0, ("vmx_vminit: error customizing the vmcs"));
880366f6083SPeter Grehan 
881366f6083SPeter Grehan 		vmx->cap[i].set = 0;
882366f6083SPeter Grehan 		vmx->cap[i].proc_ctls = procbased_ctls;
88349cc03daSNeel Natu 		vmx->cap[i].proc_ctls2 = procbased_ctls2;
884366f6083SPeter Grehan 
885366f6083SPeter Grehan 		vmx->state[i].lastcpu = -1;
88645e51299SNeel Natu 		vmx->state[i].vpid = vpid[i];
887*a0efd3fbSJohn Baldwin 		vmx->state[i].user_event.intr_info = 0;
888366f6083SPeter Grehan 
889366f6083SPeter Grehan 		msr_save_area_init(vmx->guest_msrs[i], &guest_msr_count);
890366f6083SPeter Grehan 
891c847a506SNeel Natu 		error = vmcs_set_msr_save(vmcs, vtophys(vmx->guest_msrs[i]),
892366f6083SPeter Grehan 		    guest_msr_count);
893366f6083SPeter Grehan 		if (error != 0)
894366f6083SPeter Grehan 			panic("vmcs_set_msr_save error %d", error);
895366f6083SPeter Grehan 
896aaaa0656SPeter Grehan 		/*
897aaaa0656SPeter Grehan 		 * Set up the CR0/4 shadows, and init the read shadow
898aaaa0656SPeter Grehan 		 * to the power-on register value from the Intel Sys Arch.
899aaaa0656SPeter Grehan 		 *  CR0 - 0x60000010
900aaaa0656SPeter Grehan 		 *  CR4 - 0
901aaaa0656SPeter Grehan 		 */
902c847a506SNeel Natu 		error = vmx_setup_cr0_shadow(vmcs, 0x60000010);
90339c21c2dSNeel Natu 		if (error != 0)
90439c21c2dSNeel Natu 			panic("vmx_setup_cr0_shadow %d", error);
90539c21c2dSNeel Natu 
906c847a506SNeel Natu 		error = vmx_setup_cr4_shadow(vmcs, 0);
90739c21c2dSNeel Natu 		if (error != 0)
90839c21c2dSNeel Natu 			panic("vmx_setup_cr4_shadow %d", error);
909318224bbSNeel Natu 
910318224bbSNeel Natu 		vmx->ctx[i].pmap = pmap;
911366f6083SPeter Grehan 	}
912366f6083SPeter Grehan 
913366f6083SPeter Grehan 	return (vmx);
914366f6083SPeter Grehan }
915366f6083SPeter Grehan 
916366f6083SPeter Grehan static int
917a2da7af6SNeel Natu vmx_handle_cpuid(struct vm *vm, int vcpu, struct vmxctx *vmxctx)
918366f6083SPeter Grehan {
919366f6083SPeter Grehan 	int handled, func;
920366f6083SPeter Grehan 
921366f6083SPeter Grehan 	func = vmxctx->guest_rax;
922366f6083SPeter Grehan 
923a2da7af6SNeel Natu 	handled = x86_emulate_cpuid(vm, vcpu,
924a2da7af6SNeel Natu 				    (uint32_t*)(&vmxctx->guest_rax),
925a2da7af6SNeel Natu 				    (uint32_t*)(&vmxctx->guest_rbx),
926a2da7af6SNeel Natu 				    (uint32_t*)(&vmxctx->guest_rcx),
927a2da7af6SNeel Natu 				    (uint32_t*)(&vmxctx->guest_rdx));
928366f6083SPeter Grehan 	return (handled);
929366f6083SPeter Grehan }
930366f6083SPeter Grehan 
931366f6083SPeter Grehan static __inline void
932366f6083SPeter Grehan vmx_run_trace(struct vmx *vmx, int vcpu)
933366f6083SPeter Grehan {
934366f6083SPeter Grehan #ifdef KTR
935513c8d33SNeel Natu 	VCPU_CTR1(vmx->vm, vcpu, "Resume execution at %#lx", vmcs_guest_rip());
936366f6083SPeter Grehan #endif
937366f6083SPeter Grehan }
938366f6083SPeter Grehan 
939366f6083SPeter Grehan static __inline void
940366f6083SPeter Grehan vmx_exit_trace(struct vmx *vmx, int vcpu, uint64_t rip, uint32_t exit_reason,
941eeefa4e4SNeel Natu 	       int handled)
942366f6083SPeter Grehan {
943366f6083SPeter Grehan #ifdef KTR
944513c8d33SNeel Natu 	VCPU_CTR3(vmx->vm, vcpu, "%s %s vmexit at 0x%0lx",
945366f6083SPeter Grehan 		 handled ? "handled" : "unhandled",
946366f6083SPeter Grehan 		 exit_reason_to_str(exit_reason), rip);
947eeefa4e4SNeel Natu #endif
948eeefa4e4SNeel Natu }
949366f6083SPeter Grehan 
950eeefa4e4SNeel Natu static __inline void
951eeefa4e4SNeel Natu vmx_astpending_trace(struct vmx *vmx, int vcpu, uint64_t rip)
952eeefa4e4SNeel Natu {
953eeefa4e4SNeel Natu #ifdef KTR
954513c8d33SNeel Natu 	VCPU_CTR1(vmx->vm, vcpu, "astpending vmexit at 0x%0lx", rip);
955366f6083SPeter Grehan #endif
956366f6083SPeter Grehan }
957366f6083SPeter Grehan 
958953c2c47SNeel Natu static VMM_STAT_INTEL(VCPU_INVVPID_SAVED, "Number of vpid invalidations saved");
959953c2c47SNeel Natu 
9603de83862SNeel Natu static void
961953c2c47SNeel Natu vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu, pmap_t pmap)
962366f6083SPeter Grehan {
963366f6083SPeter Grehan 	struct vmxstate *vmxstate;
964953c2c47SNeel Natu 	struct invvpid_desc invvpid_desc;
965366f6083SPeter Grehan 
966366f6083SPeter Grehan 	vmxstate = &vmx->state[vcpu];
967953c2c47SNeel Natu 	if (vmxstate->lastcpu == curcpu)
9683de83862SNeel Natu 		return;
969366f6083SPeter Grehan 
970953c2c47SNeel Natu 	vmxstate->lastcpu = curcpu;
971953c2c47SNeel Natu 
972366f6083SPeter Grehan 	vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1);
973366f6083SPeter Grehan 
9743de83862SNeel Natu 	vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase());
9753de83862SNeel Natu 	vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase());
9763de83862SNeel Natu 	vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase());
977366f6083SPeter Grehan 
978366f6083SPeter Grehan 	/*
979366f6083SPeter Grehan 	 * If we are using VPIDs then invalidate all mappings tagged with 'vpid'
980366f6083SPeter Grehan 	 *
981366f6083SPeter Grehan 	 * We do this because this vcpu was executing on a different host
982366f6083SPeter Grehan 	 * cpu when it last ran. We do not track whether it invalidated
983366f6083SPeter Grehan 	 * mappings associated with its 'vpid' during that run. So we must
984366f6083SPeter Grehan 	 * assume that the mappings associated with 'vpid' on 'curcpu' are
985366f6083SPeter Grehan 	 * stale and invalidate them.
986366f6083SPeter Grehan 	 *
987366f6083SPeter Grehan 	 * Note that we incur this penalty only when the scheduler chooses to
988366f6083SPeter Grehan 	 * move the thread associated with this vcpu between host cpus.
989366f6083SPeter Grehan 	 *
990366f6083SPeter Grehan 	 * Note also that this will invalidate mappings tagged with 'vpid'
991366f6083SPeter Grehan 	 * for "all" EP4TAs.
992366f6083SPeter Grehan 	 */
993366f6083SPeter Grehan 	if (vmxstate->vpid != 0) {
994953c2c47SNeel Natu 		if (pmap->pm_eptgen == vmx->eptgen[curcpu]) {
995953c2c47SNeel Natu 			invvpid_desc._res1 = 0;
996953c2c47SNeel Natu 			invvpid_desc._res2 = 0;
997366f6083SPeter Grehan 			invvpid_desc.vpid = vmxstate->vpid;
998366f6083SPeter Grehan 			invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc);
999953c2c47SNeel Natu 		} else {
1000953c2c47SNeel Natu 			/*
1001953c2c47SNeel Natu 			 * The invvpid can be skipped if an invept is going to
1002953c2c47SNeel Natu 			 * be performed before entering the guest. The invept
1003953c2c47SNeel Natu 			 * will invalidate combined mappings tagged with
1004953c2c47SNeel Natu 			 * 'vmx->eptp' for all vpids.
1005953c2c47SNeel Natu 			 */
1006953c2c47SNeel Natu 			vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_SAVED, 1);
1007953c2c47SNeel Natu 		}
1008366f6083SPeter Grehan 	}
1009366f6083SPeter Grehan }
1010366f6083SPeter Grehan 
1011366f6083SPeter Grehan /*
1012366f6083SPeter Grehan  * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set.
1013366f6083SPeter Grehan  */
1014366f6083SPeter Grehan CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0);
1015366f6083SPeter Grehan 
1016366f6083SPeter Grehan static void __inline
1017366f6083SPeter Grehan vmx_set_int_window_exiting(struct vmx *vmx, int vcpu)
1018366f6083SPeter Grehan {
1019366f6083SPeter Grehan 
102048b2d828SNeel Natu 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) {
1021366f6083SPeter Grehan 		vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING;
10223de83862SNeel Natu 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
102348b2d828SNeel Natu 		VCPU_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting");
102448b2d828SNeel Natu 	}
1025366f6083SPeter Grehan }
1026366f6083SPeter Grehan 
1027366f6083SPeter Grehan static void __inline
1028366f6083SPeter Grehan vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu)
1029366f6083SPeter Grehan {
1030366f6083SPeter Grehan 
103148b2d828SNeel Natu 	KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0,
103248b2d828SNeel Natu 	    ("intr_window_exiting not set: %#x", vmx->cap[vcpu].proc_ctls));
1033366f6083SPeter Grehan 	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING;
10343de83862SNeel Natu 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
103548b2d828SNeel Natu 	VCPU_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting");
1036366f6083SPeter Grehan }
1037366f6083SPeter Grehan 
1038366f6083SPeter Grehan static void __inline
1039366f6083SPeter Grehan vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu)
1040366f6083SPeter Grehan {
1041366f6083SPeter Grehan 
104248b2d828SNeel Natu 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) {
1043366f6083SPeter Grehan 		vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING;
10443de83862SNeel Natu 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
104548b2d828SNeel Natu 		VCPU_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting");
104648b2d828SNeel Natu 	}
1047366f6083SPeter Grehan }
1048366f6083SPeter Grehan 
1049366f6083SPeter Grehan static void __inline
1050366f6083SPeter Grehan vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu)
1051366f6083SPeter Grehan {
1052366f6083SPeter Grehan 
105348b2d828SNeel Natu 	KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0,
105448b2d828SNeel Natu 	    ("nmi_window_exiting not set %#x", vmx->cap[vcpu].proc_ctls));
1055366f6083SPeter Grehan 	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING;
10563de83862SNeel Natu 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
105748b2d828SNeel Natu 	VCPU_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting");
1058366f6083SPeter Grehan }
1059366f6083SPeter Grehan 
106048b2d828SNeel Natu #define	NMI_BLOCKING	(VMCS_INTERRUPTIBILITY_NMI_BLOCKING |		\
106148b2d828SNeel Natu 			 VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
106248b2d828SNeel Natu #define	HWINTR_BLOCKING	(VMCS_INTERRUPTIBILITY_STI_BLOCKING |		\
106348b2d828SNeel Natu 			 VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
106448b2d828SNeel Natu 
106548b2d828SNeel Natu static void
1066*a0efd3fbSJohn Baldwin vmx_inject_user_event(struct vmx *vmx, int vcpu)
1067*a0efd3fbSJohn Baldwin {
1068*a0efd3fbSJohn Baldwin 	struct vmxevent *user_event;
1069*a0efd3fbSJohn Baldwin 	uint32_t info;
1070*a0efd3fbSJohn Baldwin 
1071*a0efd3fbSJohn Baldwin 	user_event = &vmx->state[vcpu].user_event;
1072*a0efd3fbSJohn Baldwin 
1073*a0efd3fbSJohn Baldwin 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1074*a0efd3fbSJohn Baldwin 	KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_user_event: invalid "
1075*a0efd3fbSJohn Baldwin 	    "VM-entry interruption information %#x", info));
1076*a0efd3fbSJohn Baldwin 
1077*a0efd3fbSJohn Baldwin 	vmcs_write(VMCS_ENTRY_INTR_INFO, user_event->intr_info);
1078*a0efd3fbSJohn Baldwin 	if (user_event->intr_info & VMCS_INTR_DEL_ERRCODE)
1079*a0efd3fbSJohn Baldwin 		vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR, user_event->error_code);
1080*a0efd3fbSJohn Baldwin 	user_event->intr_info = 0;
1081*a0efd3fbSJohn Baldwin }
1082*a0efd3fbSJohn Baldwin 
1083*a0efd3fbSJohn Baldwin static void
1084*a0efd3fbSJohn Baldwin vmx_inject_exception(struct vmx *vmx, int vcpu, struct vm_exit *vmexit,
1085*a0efd3fbSJohn Baldwin     int fault, int errvalid, int errcode)
1086*a0efd3fbSJohn Baldwin {
1087*a0efd3fbSJohn Baldwin 	uint32_t info;
1088*a0efd3fbSJohn Baldwin 
1089*a0efd3fbSJohn Baldwin 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1090*a0efd3fbSJohn Baldwin 	KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_exception: invalid "
1091*a0efd3fbSJohn Baldwin 	    "VM-entry interruption information %#x", info));
1092*a0efd3fbSJohn Baldwin 
1093*a0efd3fbSJohn Baldwin 	/*
1094*a0efd3fbSJohn Baldwin 	 * Although INTR_T_HWEXCEPTION does not advance %rip, vmx_run()
1095*a0efd3fbSJohn Baldwin 	 * always advances it, so we clear the instruction length to zero
1096*a0efd3fbSJohn Baldwin 	 * explicitly.
1097*a0efd3fbSJohn Baldwin 	 */
1098*a0efd3fbSJohn Baldwin 	vmexit->inst_length = 0;
1099*a0efd3fbSJohn Baldwin 	info = fault | VMCS_INTR_T_HWEXCEPTION | VMCS_INTR_VALID;
1100*a0efd3fbSJohn Baldwin 	if (errvalid) {
1101*a0efd3fbSJohn Baldwin 		info |= VMCS_INTR_DEL_ERRCODE;
1102*a0efd3fbSJohn Baldwin 		vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR, errcode);
1103*a0efd3fbSJohn Baldwin 	}
1104*a0efd3fbSJohn Baldwin 	vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1105*a0efd3fbSJohn Baldwin 
1106*a0efd3fbSJohn Baldwin 	VCPU_CTR2(vmx->vm, vcpu, "Injecting fault %d (errcode %d)", fault,
1107*a0efd3fbSJohn Baldwin 	    errcode);
1108*a0efd3fbSJohn Baldwin }
1109*a0efd3fbSJohn Baldwin 
1110*a0efd3fbSJohn Baldwin /* All GP# faults VMM injects use an error code of 0. */
1111*a0efd3fbSJohn Baldwin static void
1112*a0efd3fbSJohn Baldwin vmx_inject_gp(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1113*a0efd3fbSJohn Baldwin {
1114*a0efd3fbSJohn Baldwin 
1115*a0efd3fbSJohn Baldwin 	vmx_inject_exception(vmx, vcpu, vmexit, IDT_GP, 1, 0);
1116*a0efd3fbSJohn Baldwin }
1117*a0efd3fbSJohn Baldwin 
1118*a0efd3fbSJohn Baldwin static void
1119*a0efd3fbSJohn Baldwin vmx_inject_ud(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1120*a0efd3fbSJohn Baldwin {
1121*a0efd3fbSJohn Baldwin 
1122*a0efd3fbSJohn Baldwin 	vmx_inject_exception(vmx, vcpu, vmexit, IDT_UD, 0, 0);
1123*a0efd3fbSJohn Baldwin }
1124*a0efd3fbSJohn Baldwin 
1125*a0efd3fbSJohn Baldwin static void
1126366f6083SPeter Grehan vmx_inject_nmi(struct vmx *vmx, int vcpu)
1127366f6083SPeter Grehan {
112848b2d828SNeel Natu 	uint32_t gi, info;
1129366f6083SPeter Grehan 
113048b2d828SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
113148b2d828SNeel Natu 	KASSERT((gi & NMI_BLOCKING) == 0, ("vmx_inject_nmi: invalid guest "
113248b2d828SNeel Natu 	    "interruptibility-state %#x", gi));
1133366f6083SPeter Grehan 
113448b2d828SNeel Natu 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
113548b2d828SNeel Natu 	KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_nmi: invalid "
113648b2d828SNeel Natu 	    "VM-entry interruption information %#x", info));
1137366f6083SPeter Grehan 
1138366f6083SPeter Grehan 	/*
1139366f6083SPeter Grehan 	 * Inject the virtual NMI. The vector must be the NMI IDT entry
1140366f6083SPeter Grehan 	 * or the VMCS entry check will fail.
1141366f6083SPeter Grehan 	 */
114248b2d828SNeel Natu 	info = IDT_NMI | VMCS_INTR_T_NMI | VMCS_INTR_VALID;
11433de83862SNeel Natu 	vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1144366f6083SPeter Grehan 
1145513c8d33SNeel Natu 	VCPU_CTR0(vmx->vm, vcpu, "Injecting vNMI");
1146366f6083SPeter Grehan 
1147366f6083SPeter Grehan 	/* Clear the request */
1148f352ff0cSNeel Natu 	vm_nmi_clear(vmx->vm, vcpu);
1149366f6083SPeter Grehan }
1150366f6083SPeter Grehan 
1151366f6083SPeter Grehan static void
1152de5ea6b6SNeel Natu vmx_inject_interrupts(struct vmx *vmx, int vcpu, struct vlapic *vlapic)
1153366f6083SPeter Grehan {
115448b2d828SNeel Natu 	int vector, need_nmi_exiting;
115548b2d828SNeel Natu 	uint64_t rflags;
115648b2d828SNeel Natu 	uint32_t gi, info;
1157366f6083SPeter Grehan 
115848b2d828SNeel Natu 	if (vm_nmi_pending(vmx->vm, vcpu)) {
1159366f6083SPeter Grehan 		/*
116048b2d828SNeel Natu 		 * If there are no conditions blocking NMI injection then
116148b2d828SNeel Natu 		 * inject it directly here otherwise enable "NMI window
116248b2d828SNeel Natu 		 * exiting" to inject it as soon as we can.
1163eeefa4e4SNeel Natu 		 *
116448b2d828SNeel Natu 		 * We also check for STI_BLOCKING because some implementations
116548b2d828SNeel Natu 		 * don't allow NMI injection in this case. If we are running
116648b2d828SNeel Natu 		 * on a processor that doesn't have this restriction it will
116748b2d828SNeel Natu 		 * immediately exit and the NMI will be injected in the
116848b2d828SNeel Natu 		 * "NMI window exiting" handler.
1169366f6083SPeter Grehan 		 */
117048b2d828SNeel Natu 		need_nmi_exiting = 1;
117148b2d828SNeel Natu 		gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
117248b2d828SNeel Natu 		if ((gi & (HWINTR_BLOCKING | NMI_BLOCKING)) == 0) {
11733de83862SNeel Natu 			info = vmcs_read(VMCS_ENTRY_INTR_INFO);
117448b2d828SNeel Natu 			if ((info & VMCS_INTR_VALID) == 0) {
117548b2d828SNeel Natu 				vmx_inject_nmi(vmx, vcpu);
117648b2d828SNeel Natu 				need_nmi_exiting = 0;
117748b2d828SNeel Natu 			} else {
117848b2d828SNeel Natu 				VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI "
117948b2d828SNeel Natu 				    "due to VM-entry intr info %#x", info);
118048b2d828SNeel Natu 			}
118148b2d828SNeel Natu 		} else {
118248b2d828SNeel Natu 			VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI due to "
118348b2d828SNeel Natu 			    "Guest Interruptibility-state %#x", gi);
118448b2d828SNeel Natu 		}
1185eeefa4e4SNeel Natu 
118648b2d828SNeel Natu 		if (need_nmi_exiting)
118748b2d828SNeel Natu 			vmx_set_nmi_window_exiting(vmx, vcpu);
118848b2d828SNeel Natu 	}
1189366f6083SPeter Grehan 
1190*a0efd3fbSJohn Baldwin 	/*
1191*a0efd3fbSJohn Baldwin 	 * If there is a user injection event pending and there isn't
1192*a0efd3fbSJohn Baldwin 	 * an interrupt queued already, inject the user event.
1193*a0efd3fbSJohn Baldwin 	 */
1194*a0efd3fbSJohn Baldwin 	if (vmx->state[vcpu].user_event.intr_info & VMCS_INTR_VALID) {
1195*a0efd3fbSJohn Baldwin 		info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1196*a0efd3fbSJohn Baldwin 		if ((info & VMCS_INTR_VALID) == 0) {
1197*a0efd3fbSJohn Baldwin 			vmx_inject_user_event(vmx, vcpu);
1198*a0efd3fbSJohn Baldwin 		} else {
1199*a0efd3fbSJohn Baldwin 			/*
1200*a0efd3fbSJohn Baldwin 			 * XXX: Do we need to force an exit so this can
1201*a0efd3fbSJohn Baldwin 			 * be injected?
1202*a0efd3fbSJohn Baldwin 			 */
1203*a0efd3fbSJohn Baldwin 			VCPU_CTR1(vmx->vm, vcpu, "Cannot inject user event "
1204*a0efd3fbSJohn Baldwin 			    "due to VM-entry intr info %#x", info);
1205*a0efd3fbSJohn Baldwin 		}
1206*a0efd3fbSJohn Baldwin 	}
1207*a0efd3fbSJohn Baldwin 
120888c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
120988c4b8d1SNeel Natu 		vmx_inject_pir(vlapic);
121088c4b8d1SNeel Natu 		return;
121188c4b8d1SNeel Natu 	}
121288c4b8d1SNeel Natu 
121348b2d828SNeel Natu 	/*
121436736912SNeel Natu 	 * If interrupt-window exiting is already in effect then don't bother
121536736912SNeel Natu 	 * checking for pending interrupts. This is just an optimization and
121636736912SNeel Natu 	 * not needed for correctness.
121748b2d828SNeel Natu 	 */
121836736912SNeel Natu 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) {
121936736912SNeel Natu 		VCPU_CTR0(vmx->vm, vcpu, "Skip interrupt injection due to "
122036736912SNeel Natu 		    "pending int_window_exiting");
122148b2d828SNeel Natu 		return;
122236736912SNeel Natu 	}
122348b2d828SNeel Natu 
1224366f6083SPeter Grehan 	/* Ask the local apic for a vector to inject */
12254d1e82a8SNeel Natu 	if (!vlapic_pending_intr(vlapic, &vector))
1226366f6083SPeter Grehan 		return;
1227366f6083SPeter Grehan 
122848b2d828SNeel Natu 	KASSERT(vector >= 32 && vector <= 255, ("invalid vector %d", vector));
1229366f6083SPeter Grehan 
1230366f6083SPeter Grehan 	/* Check RFLAGS.IF and the interruptibility state of the guest */
12313de83862SNeel Natu 	rflags = vmcs_read(VMCS_GUEST_RFLAGS);
123236736912SNeel Natu 	if ((rflags & PSL_I) == 0) {
123336736912SNeel Natu 		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
123436736912SNeel Natu 		    "rflags %#lx", vector, rflags);
1235366f6083SPeter Grehan 		goto cantinject;
123636736912SNeel Natu 	}
1237366f6083SPeter Grehan 
123848b2d828SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
123936736912SNeel Natu 	if (gi & HWINTR_BLOCKING) {
124036736912SNeel Natu 		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
124136736912SNeel Natu 		    "Guest Interruptibility-state %#x", vector, gi);
1242366f6083SPeter Grehan 		goto cantinject;
124336736912SNeel Natu 	}
124436736912SNeel Natu 
124536736912SNeel Natu 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
124636736912SNeel Natu 	if (info & VMCS_INTR_VALID) {
124736736912SNeel Natu 		/*
124836736912SNeel Natu 		 * This is expected and could happen for multiple reasons:
124936736912SNeel Natu 		 * - A vectoring VM-entry was aborted due to astpending
125036736912SNeel Natu 		 * - A VM-exit happened during event injection.
125136736912SNeel Natu 		 * - An NMI was injected above or after "NMI window exiting"
125236736912SNeel Natu 		 */
125336736912SNeel Natu 		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
125436736912SNeel Natu 		    "VM-entry intr info %#x", vector, info);
125536736912SNeel Natu 		goto cantinject;
125636736912SNeel Natu 	}
1257366f6083SPeter Grehan 
1258366f6083SPeter Grehan 	/* Inject the interrupt */
1259160471d2SNeel Natu 	info = VMCS_INTR_T_HWINTR | VMCS_INTR_VALID;
1260366f6083SPeter Grehan 	info |= vector;
12613de83862SNeel Natu 	vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1262366f6083SPeter Grehan 
1263366f6083SPeter Grehan 	/* Update the Local APIC ISR */
1264de5ea6b6SNeel Natu 	vlapic_intr_accepted(vlapic, vector);
1265366f6083SPeter Grehan 
1266513c8d33SNeel Natu 	VCPU_CTR1(vmx->vm, vcpu, "Injecting hwintr at vector %d", vector);
1267366f6083SPeter Grehan 
1268366f6083SPeter Grehan 	return;
1269366f6083SPeter Grehan 
1270366f6083SPeter Grehan cantinject:
1271366f6083SPeter Grehan 	/*
1272366f6083SPeter Grehan 	 * Set the Interrupt Window Exiting execution control so we can inject
1273366f6083SPeter Grehan 	 * the interrupt as soon as blocking condition goes away.
1274366f6083SPeter Grehan 	 */
1275366f6083SPeter Grehan 	vmx_set_int_window_exiting(vmx, vcpu);
1276366f6083SPeter Grehan }
1277366f6083SPeter Grehan 
1278e5a1d950SNeel Natu /*
1279e5a1d950SNeel Natu  * If the Virtual NMIs execution control is '1' then the logical processor
1280e5a1d950SNeel Natu  * tracks virtual-NMI blocking in the Guest Interruptibility-state field of
1281e5a1d950SNeel Natu  * the VMCS. An IRET instruction in VMX non-root operation will remove any
1282e5a1d950SNeel Natu  * virtual-NMI blocking.
1283e5a1d950SNeel Natu  *
1284e5a1d950SNeel Natu  * This unblocking occurs even if the IRET causes a fault. In this case the
1285e5a1d950SNeel Natu  * hypervisor needs to restore virtual-NMI blocking before resuming the guest.
1286e5a1d950SNeel Natu  */
1287e5a1d950SNeel Natu static void
1288e5a1d950SNeel Natu vmx_restore_nmi_blocking(struct vmx *vmx, int vcpuid)
1289e5a1d950SNeel Natu {
1290e5a1d950SNeel Natu 	uint32_t gi;
1291e5a1d950SNeel Natu 
1292e5a1d950SNeel Natu 	VCPU_CTR0(vmx->vm, vcpuid, "Restore Virtual-NMI blocking");
1293e5a1d950SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1294e5a1d950SNeel Natu 	gi |= VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1295e5a1d950SNeel Natu 	vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1296e5a1d950SNeel Natu }
1297e5a1d950SNeel Natu 
1298e5a1d950SNeel Natu static void
1299e5a1d950SNeel Natu vmx_clear_nmi_blocking(struct vmx *vmx, int vcpuid)
1300e5a1d950SNeel Natu {
1301e5a1d950SNeel Natu 	uint32_t gi;
1302e5a1d950SNeel Natu 
1303e5a1d950SNeel Natu 	VCPU_CTR0(vmx->vm, vcpuid, "Clear Virtual-NMI blocking");
1304e5a1d950SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1305e5a1d950SNeel Natu 	gi &= ~VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1306e5a1d950SNeel Natu 	vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1307e5a1d950SNeel Natu }
1308e5a1d950SNeel Natu 
1309366f6083SPeter Grehan static int
1310*a0efd3fbSJohn Baldwin vmx_emulate_xsetbv(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1311abb023fbSJohn Baldwin {
1312abb023fbSJohn Baldwin 	struct vmxctx *vmxctx;
1313abb023fbSJohn Baldwin 	uint64_t xcrval;
1314abb023fbSJohn Baldwin 	const struct xsave_limits *limits;
1315abb023fbSJohn Baldwin 
1316abb023fbSJohn Baldwin 	vmxctx = &vmx->ctx[vcpu];
1317abb023fbSJohn Baldwin 	limits = vmm_get_xsave_limits();
1318abb023fbSJohn Baldwin 
1319*a0efd3fbSJohn Baldwin 	/*
1320*a0efd3fbSJohn Baldwin 	 * Note that the processor raises a GP# fault on its own if
1321*a0efd3fbSJohn Baldwin 	 * xsetbv is executed for CPL != 0, so we do not have to
1322*a0efd3fbSJohn Baldwin 	 * emulate that fault here.
1323*a0efd3fbSJohn Baldwin 	 */
1324*a0efd3fbSJohn Baldwin 
1325*a0efd3fbSJohn Baldwin 	/* Only xcr0 is supported. */
1326*a0efd3fbSJohn Baldwin 	if (vmxctx->guest_rcx != 0) {
1327*a0efd3fbSJohn Baldwin 		vmx_inject_gp(vmx, vcpu, vmexit);
1328*a0efd3fbSJohn Baldwin 		return (HANDLED);
1329*a0efd3fbSJohn Baldwin 	}
1330*a0efd3fbSJohn Baldwin 
1331*a0efd3fbSJohn Baldwin 	/* We only handle xcr0 if both the host and guest have XSAVE enabled. */
1332*a0efd3fbSJohn Baldwin 	if (!limits->xsave_enabled || !(vmcs_read(VMCS_GUEST_CR4) & CR4_XSAVE)) {
1333*a0efd3fbSJohn Baldwin 		vmx_inject_ud(vmx, vcpu, vmexit);
1334*a0efd3fbSJohn Baldwin 		return (HANDLED);
1335*a0efd3fbSJohn Baldwin 	}
1336abb023fbSJohn Baldwin 
1337abb023fbSJohn Baldwin 	xcrval = vmxctx->guest_rdx << 32 | (vmxctx->guest_rax & 0xffffffff);
1338*a0efd3fbSJohn Baldwin 	if ((xcrval & ~limits->xcr0_allowed) != 0) {
1339*a0efd3fbSJohn Baldwin 		vmx_inject_gp(vmx, vcpu, vmexit);
1340*a0efd3fbSJohn Baldwin 		return (HANDLED);
1341*a0efd3fbSJohn Baldwin 	}
1342abb023fbSJohn Baldwin 
1343*a0efd3fbSJohn Baldwin 	if (!(xcrval & XFEATURE_ENABLED_X87)) {
1344*a0efd3fbSJohn Baldwin 		vmx_inject_gp(vmx, vcpu, vmexit);
1345*a0efd3fbSJohn Baldwin 		return (HANDLED);
1346*a0efd3fbSJohn Baldwin 	}
1347abb023fbSJohn Baldwin 
1348abb023fbSJohn Baldwin 	if ((xcrval & (XFEATURE_ENABLED_AVX | XFEATURE_ENABLED_SSE)) ==
1349*a0efd3fbSJohn Baldwin 	    XFEATURE_ENABLED_AVX) {
1350*a0efd3fbSJohn Baldwin 		vmx_inject_gp(vmx, vcpu, vmexit);
1351*a0efd3fbSJohn Baldwin 		return (HANDLED);
1352*a0efd3fbSJohn Baldwin 	}
1353abb023fbSJohn Baldwin 
1354abb023fbSJohn Baldwin 	/*
1355abb023fbSJohn Baldwin 	 * This runs "inside" vmrun() with the guest's FPU state, so
1356abb023fbSJohn Baldwin 	 * modifying xcr0 directly modifies the guest's xcr0, not the
1357abb023fbSJohn Baldwin 	 * host's.
1358abb023fbSJohn Baldwin 	 */
1359abb023fbSJohn Baldwin 	load_xcr(0, xcrval);
1360abb023fbSJohn Baldwin 	return (HANDLED);
1361abb023fbSJohn Baldwin }
1362abb023fbSJohn Baldwin 
1363abb023fbSJohn Baldwin static int
1364366f6083SPeter Grehan vmx_emulate_cr_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1365366f6083SPeter Grehan {
13663de83862SNeel Natu 	int cr, vmcs_guest_cr, vmcs_shadow_cr;
136780a902efSPeter Grehan 	uint64_t crval, regval, ones_mask, zeros_mask;
1368366f6083SPeter Grehan 	const struct vmxctx *vmxctx;
1369366f6083SPeter Grehan 
137039c21c2dSNeel Natu 	/* We only handle mov to %cr0 or %cr4 at this time */
137139c21c2dSNeel Natu 	if ((exitqual & 0xf0) != 0x00)
137239c21c2dSNeel Natu 		return (UNHANDLED);
137339c21c2dSNeel Natu 
137439c21c2dSNeel Natu 	cr = exitqual & 0xf;
137539c21c2dSNeel Natu 	if (cr != 0 && cr != 4)
1376366f6083SPeter Grehan 		return (UNHANDLED);
1377366f6083SPeter Grehan 
13786f0c167fSDimitry Andric 	regval = 0; /* silence gcc */
1379366f6083SPeter Grehan 	vmxctx = &vmx->ctx[vcpu];
1380366f6083SPeter Grehan 
1381366f6083SPeter Grehan 	/*
13823de83862SNeel Natu 	 * We must use vmcs_write() directly here because vmcs_setreg() will
1383366f6083SPeter Grehan 	 * call vmclear(vmcs) as a side-effect which we certainly don't want.
1384366f6083SPeter Grehan 	 */
1385366f6083SPeter Grehan 	switch ((exitqual >> 8) & 0xf) {
1386366f6083SPeter Grehan 	case 0:
1387366f6083SPeter Grehan 		regval = vmxctx->guest_rax;
1388366f6083SPeter Grehan 		break;
1389366f6083SPeter Grehan 	case 1:
1390366f6083SPeter Grehan 		regval = vmxctx->guest_rcx;
1391366f6083SPeter Grehan 		break;
1392366f6083SPeter Grehan 	case 2:
1393366f6083SPeter Grehan 		regval = vmxctx->guest_rdx;
1394366f6083SPeter Grehan 		break;
1395366f6083SPeter Grehan 	case 3:
1396366f6083SPeter Grehan 		regval = vmxctx->guest_rbx;
1397366f6083SPeter Grehan 		break;
1398366f6083SPeter Grehan 	case 4:
13993de83862SNeel Natu 		regval = vmcs_read(VMCS_GUEST_RSP);
1400366f6083SPeter Grehan 		break;
1401366f6083SPeter Grehan 	case 5:
1402366f6083SPeter Grehan 		regval = vmxctx->guest_rbp;
1403366f6083SPeter Grehan 		break;
1404366f6083SPeter Grehan 	case 6:
1405366f6083SPeter Grehan 		regval = vmxctx->guest_rsi;
1406366f6083SPeter Grehan 		break;
1407366f6083SPeter Grehan 	case 7:
1408366f6083SPeter Grehan 		regval = vmxctx->guest_rdi;
1409366f6083SPeter Grehan 		break;
1410366f6083SPeter Grehan 	case 8:
1411366f6083SPeter Grehan 		regval = vmxctx->guest_r8;
1412366f6083SPeter Grehan 		break;
1413366f6083SPeter Grehan 	case 9:
1414366f6083SPeter Grehan 		regval = vmxctx->guest_r9;
1415366f6083SPeter Grehan 		break;
1416366f6083SPeter Grehan 	case 10:
1417366f6083SPeter Grehan 		regval = vmxctx->guest_r10;
1418366f6083SPeter Grehan 		break;
1419366f6083SPeter Grehan 	case 11:
1420366f6083SPeter Grehan 		regval = vmxctx->guest_r11;
1421366f6083SPeter Grehan 		break;
1422366f6083SPeter Grehan 	case 12:
1423366f6083SPeter Grehan 		regval = vmxctx->guest_r12;
1424366f6083SPeter Grehan 		break;
1425366f6083SPeter Grehan 	case 13:
1426366f6083SPeter Grehan 		regval = vmxctx->guest_r13;
1427366f6083SPeter Grehan 		break;
1428366f6083SPeter Grehan 	case 14:
1429366f6083SPeter Grehan 		regval = vmxctx->guest_r14;
1430366f6083SPeter Grehan 		break;
1431366f6083SPeter Grehan 	case 15:
1432366f6083SPeter Grehan 		regval = vmxctx->guest_r15;
1433366f6083SPeter Grehan 		break;
1434366f6083SPeter Grehan 	}
1435366f6083SPeter Grehan 
143639c21c2dSNeel Natu 	if (cr == 0) {
143739c21c2dSNeel Natu 		ones_mask = cr0_ones_mask;
143839c21c2dSNeel Natu 		zeros_mask = cr0_zeros_mask;
143939c21c2dSNeel Natu 		vmcs_guest_cr = VMCS_GUEST_CR0;
1440aaaa0656SPeter Grehan 		vmcs_shadow_cr = VMCS_CR0_SHADOW;
144139c21c2dSNeel Natu 	} else {
144239c21c2dSNeel Natu 		ones_mask = cr4_ones_mask;
144339c21c2dSNeel Natu 		zeros_mask = cr4_zeros_mask;
144439c21c2dSNeel Natu 		vmcs_guest_cr = VMCS_GUEST_CR4;
1445aaaa0656SPeter Grehan 		vmcs_shadow_cr = VMCS_CR4_SHADOW;
144639c21c2dSNeel Natu 	}
14473de83862SNeel Natu 	vmcs_write(vmcs_shadow_cr, regval);
1448aaaa0656SPeter Grehan 
144980a902efSPeter Grehan 	crval = regval | ones_mask;
145080a902efSPeter Grehan 	crval &= ~zeros_mask;
14513de83862SNeel Natu 	vmcs_write(vmcs_guest_cr, crval);
1452366f6083SPeter Grehan 
145380a902efSPeter Grehan 	if (cr == 0 && regval & CR0_PG) {
145480a902efSPeter Grehan 		uint64_t efer, entry_ctls;
145580a902efSPeter Grehan 
145680a902efSPeter Grehan 		/*
145780a902efSPeter Grehan 		 * If CR0.PG is 1 and EFER.LME is 1 then EFER.LMA and
145880a902efSPeter Grehan 		 * the "IA-32e mode guest" bit in VM-entry control must be
145980a902efSPeter Grehan 		 * equal.
146080a902efSPeter Grehan 		 */
14613de83862SNeel Natu 		efer = vmcs_read(VMCS_GUEST_IA32_EFER);
146280a902efSPeter Grehan 		if (efer & EFER_LME) {
146380a902efSPeter Grehan 			efer |= EFER_LMA;
14643de83862SNeel Natu 			vmcs_write(VMCS_GUEST_IA32_EFER, efer);
14653de83862SNeel Natu 			entry_ctls = vmcs_read(VMCS_ENTRY_CTLS);
146680a902efSPeter Grehan 			entry_ctls |= VM_ENTRY_GUEST_LMA;
14673de83862SNeel Natu 			vmcs_write(VMCS_ENTRY_CTLS, entry_ctls);
146880a902efSPeter Grehan 		}
146980a902efSPeter Grehan 	}
147080a902efSPeter Grehan 
1471366f6083SPeter Grehan 	return (HANDLED);
1472366f6083SPeter Grehan }
1473366f6083SPeter Grehan 
147400f3efe1SJohn Baldwin static enum vie_cpu_mode
147500f3efe1SJohn Baldwin vmx_cpu_mode(void)
147600f3efe1SJohn Baldwin {
147700f3efe1SJohn Baldwin 
147800f3efe1SJohn Baldwin 	if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LMA)
147900f3efe1SJohn Baldwin 		return (CPU_MODE_64BIT);
148000f3efe1SJohn Baldwin 	else
148100f3efe1SJohn Baldwin 		return (CPU_MODE_COMPATIBILITY);
148200f3efe1SJohn Baldwin }
148300f3efe1SJohn Baldwin 
148400f3efe1SJohn Baldwin static enum vie_paging_mode
148500f3efe1SJohn Baldwin vmx_paging_mode(void)
148600f3efe1SJohn Baldwin {
148700f3efe1SJohn Baldwin 
148800f3efe1SJohn Baldwin 	if (!(vmcs_read(VMCS_GUEST_CR0) & CR0_PG))
148900f3efe1SJohn Baldwin 		return (PAGING_MODE_FLAT);
149000f3efe1SJohn Baldwin 	if (!(vmcs_read(VMCS_GUEST_CR4) & CR4_PAE))
149100f3efe1SJohn Baldwin 		return (PAGING_MODE_32);
149200f3efe1SJohn Baldwin 	if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LME)
149300f3efe1SJohn Baldwin 		return (PAGING_MODE_64);
149400f3efe1SJohn Baldwin 	else
149500f3efe1SJohn Baldwin 		return (PAGING_MODE_PAE);
149600f3efe1SJohn Baldwin }
149700f3efe1SJohn Baldwin 
1498366f6083SPeter Grehan static int
1499318224bbSNeel Natu ept_fault_type(uint64_t ept_qual)
1500a2da7af6SNeel Natu {
1501318224bbSNeel Natu 	int fault_type;
1502a2da7af6SNeel Natu 
1503318224bbSNeel Natu 	if (ept_qual & EPT_VIOLATION_DATA_WRITE)
1504318224bbSNeel Natu 		fault_type = VM_PROT_WRITE;
1505318224bbSNeel Natu 	else if (ept_qual & EPT_VIOLATION_INST_FETCH)
1506318224bbSNeel Natu 		fault_type = VM_PROT_EXECUTE;
1507318224bbSNeel Natu 	else
1508318224bbSNeel Natu 		fault_type= VM_PROT_READ;
1509318224bbSNeel Natu 
1510318224bbSNeel Natu 	return (fault_type);
1511318224bbSNeel Natu }
1512318224bbSNeel Natu 
1513318224bbSNeel Natu static boolean_t
1514318224bbSNeel Natu ept_emulation_fault(uint64_t ept_qual)
1515318224bbSNeel Natu {
1516318224bbSNeel Natu 	int read, write;
1517318224bbSNeel Natu 
1518318224bbSNeel Natu 	/* EPT fault on an instruction fetch doesn't make sense here */
1519a2da7af6SNeel Natu 	if (ept_qual & EPT_VIOLATION_INST_FETCH)
1520318224bbSNeel Natu 		return (FALSE);
1521a2da7af6SNeel Natu 
1522318224bbSNeel Natu 	/* EPT fault must be a read fault or a write fault */
1523a2da7af6SNeel Natu 	read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0;
1524a2da7af6SNeel Natu 	write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0;
15253b2b0011SPeter Grehan 	if ((read | write) == 0)
1526318224bbSNeel Natu 		return (FALSE);
1527a2da7af6SNeel Natu 
1528a2da7af6SNeel Natu 	/*
15293b2b0011SPeter Grehan 	 * The EPT violation must have been caused by accessing a
15303b2b0011SPeter Grehan 	 * guest-physical address that is a translation of a guest-linear
15313b2b0011SPeter Grehan 	 * address.
1532a2da7af6SNeel Natu 	 */
1533a2da7af6SNeel Natu 	if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 ||
1534a2da7af6SNeel Natu 	    (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) {
1535318224bbSNeel Natu 		return (FALSE);
1536a2da7af6SNeel Natu 	}
1537a2da7af6SNeel Natu 
1538318224bbSNeel Natu 	return (TRUE);
1539a2da7af6SNeel Natu }
1540a2da7af6SNeel Natu 
1541a2da7af6SNeel Natu static int
154288c4b8d1SNeel Natu vmx_handle_apic_write(struct vlapic *vlapic, uint64_t qual)
154388c4b8d1SNeel Natu {
154488c4b8d1SNeel Natu 	int error, handled, offset;
154588c4b8d1SNeel Natu 	bool retu;
154688c4b8d1SNeel Natu 
154788c4b8d1SNeel Natu 	if (!virtual_interrupt_delivery)
154888c4b8d1SNeel Natu 		return (UNHANDLED);
154988c4b8d1SNeel Natu 
1550*a0efd3fbSJohn Baldwin 	handled = HANDLED;
155188c4b8d1SNeel Natu 	offset = APIC_WRITE_OFFSET(qual);
155288c4b8d1SNeel Natu 	switch (offset) {
155388c4b8d1SNeel Natu 	case APIC_OFFSET_ID:
155488c4b8d1SNeel Natu 		vlapic_id_write_handler(vlapic);
155588c4b8d1SNeel Natu 		break;
155688c4b8d1SNeel Natu 	case APIC_OFFSET_LDR:
155788c4b8d1SNeel Natu 		vlapic_ldr_write_handler(vlapic);
155888c4b8d1SNeel Natu 		break;
155988c4b8d1SNeel Natu 	case APIC_OFFSET_DFR:
156088c4b8d1SNeel Natu 		vlapic_dfr_write_handler(vlapic);
156188c4b8d1SNeel Natu 		break;
156288c4b8d1SNeel Natu 	case APIC_OFFSET_SVR:
156388c4b8d1SNeel Natu 		vlapic_svr_write_handler(vlapic);
156488c4b8d1SNeel Natu 		break;
156588c4b8d1SNeel Natu 	case APIC_OFFSET_ESR:
156688c4b8d1SNeel Natu 		vlapic_esr_write_handler(vlapic);
156788c4b8d1SNeel Natu 		break;
156888c4b8d1SNeel Natu 	case APIC_OFFSET_ICR_LOW:
156988c4b8d1SNeel Natu 		retu = false;
157088c4b8d1SNeel Natu 		error = vlapic_icrlo_write_handler(vlapic, &retu);
157188c4b8d1SNeel Natu 		if (error != 0 || retu)
1572*a0efd3fbSJohn Baldwin 			handled = UNHANDLED;
157388c4b8d1SNeel Natu 		break;
157488c4b8d1SNeel Natu 	case APIC_OFFSET_CMCI_LVT:
157588c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
157688c4b8d1SNeel Natu 		vlapic_lvt_write_handler(vlapic, offset);
157788c4b8d1SNeel Natu 		break;
157888c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_ICR:
157988c4b8d1SNeel Natu 		vlapic_icrtmr_write_handler(vlapic);
158088c4b8d1SNeel Natu 		break;
158188c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_DCR:
158288c4b8d1SNeel Natu 		vlapic_dcr_write_handler(vlapic);
158388c4b8d1SNeel Natu 		break;
158488c4b8d1SNeel Natu 	default:
1585*a0efd3fbSJohn Baldwin 		handled = UNHANDLED;
158688c4b8d1SNeel Natu 		break;
158788c4b8d1SNeel Natu 	}
158888c4b8d1SNeel Natu 	return (handled);
158988c4b8d1SNeel Natu }
159088c4b8d1SNeel Natu 
159188c4b8d1SNeel Natu static bool
159288c4b8d1SNeel Natu apic_access_fault(uint64_t gpa)
159388c4b8d1SNeel Natu {
159488c4b8d1SNeel Natu 
159588c4b8d1SNeel Natu 	if (virtual_interrupt_delivery &&
159688c4b8d1SNeel Natu 	    (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE))
159788c4b8d1SNeel Natu 		return (true);
159888c4b8d1SNeel Natu 	else
159988c4b8d1SNeel Natu 		return (false);
160088c4b8d1SNeel Natu }
160188c4b8d1SNeel Natu 
160288c4b8d1SNeel Natu static int
160388c4b8d1SNeel Natu vmx_handle_apic_access(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
160488c4b8d1SNeel Natu {
160588c4b8d1SNeel Natu 	uint64_t qual;
160688c4b8d1SNeel Natu 	int access_type, offset, allowed;
160788c4b8d1SNeel Natu 
160888c4b8d1SNeel Natu 	if (!virtual_interrupt_delivery)
160988c4b8d1SNeel Natu 		return (UNHANDLED);
161088c4b8d1SNeel Natu 
161188c4b8d1SNeel Natu 	qual = vmexit->u.vmx.exit_qualification;
161288c4b8d1SNeel Natu 	access_type = APIC_ACCESS_TYPE(qual);
161388c4b8d1SNeel Natu 	offset = APIC_ACCESS_OFFSET(qual);
161488c4b8d1SNeel Natu 
161588c4b8d1SNeel Natu 	allowed = 0;
161688c4b8d1SNeel Natu 	if (access_type == 0) {
161788c4b8d1SNeel Natu 		/*
161888c4b8d1SNeel Natu 		 * Read data access to the following registers is expected.
161988c4b8d1SNeel Natu 		 */
162088c4b8d1SNeel Natu 		switch (offset) {
162188c4b8d1SNeel Natu 		case APIC_OFFSET_APR:
162288c4b8d1SNeel Natu 		case APIC_OFFSET_PPR:
162388c4b8d1SNeel Natu 		case APIC_OFFSET_RRR:
162488c4b8d1SNeel Natu 		case APIC_OFFSET_CMCI_LVT:
162588c4b8d1SNeel Natu 		case APIC_OFFSET_TIMER_CCR:
162688c4b8d1SNeel Natu 			allowed = 1;
162788c4b8d1SNeel Natu 			break;
162888c4b8d1SNeel Natu 		default:
162988c4b8d1SNeel Natu 			break;
163088c4b8d1SNeel Natu 		}
163188c4b8d1SNeel Natu 	} else if (access_type == 1) {
163288c4b8d1SNeel Natu 		/*
163388c4b8d1SNeel Natu 		 * Write data access to the following registers is expected.
163488c4b8d1SNeel Natu 		 */
163588c4b8d1SNeel Natu 		switch (offset) {
163688c4b8d1SNeel Natu 		case APIC_OFFSET_VER:
163788c4b8d1SNeel Natu 		case APIC_OFFSET_APR:
163888c4b8d1SNeel Natu 		case APIC_OFFSET_PPR:
163988c4b8d1SNeel Natu 		case APIC_OFFSET_RRR:
164088c4b8d1SNeel Natu 		case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
164188c4b8d1SNeel Natu 		case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
164288c4b8d1SNeel Natu 		case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
164388c4b8d1SNeel Natu 		case APIC_OFFSET_CMCI_LVT:
164488c4b8d1SNeel Natu 		case APIC_OFFSET_TIMER_CCR:
164588c4b8d1SNeel Natu 			allowed = 1;
164688c4b8d1SNeel Natu 			break;
164788c4b8d1SNeel Natu 		default:
164888c4b8d1SNeel Natu 			break;
164988c4b8d1SNeel Natu 		}
165088c4b8d1SNeel Natu 	}
165188c4b8d1SNeel Natu 
165288c4b8d1SNeel Natu 	if (allowed) {
165388c4b8d1SNeel Natu 		vmexit->exitcode = VM_EXITCODE_INST_EMUL;
165488c4b8d1SNeel Natu 		vmexit->u.inst_emul.gpa = DEFAULT_APIC_BASE + offset;
165588c4b8d1SNeel Natu 		vmexit->u.inst_emul.gla = VIE_INVALID_GLA;
165688c4b8d1SNeel Natu 		vmexit->u.inst_emul.cr3 = vmcs_guest_cr3();
165700f3efe1SJohn Baldwin 		vmexit->u.inst_emul.cpu_mode = vmx_cpu_mode();
165800f3efe1SJohn Baldwin 		vmexit->u.inst_emul.paging_mode = vmx_paging_mode();
165988c4b8d1SNeel Natu 	}
166088c4b8d1SNeel Natu 
166188c4b8d1SNeel Natu 	/*
166288c4b8d1SNeel Natu 	 * Regardless of whether the APIC-access is allowed this handler
166388c4b8d1SNeel Natu 	 * always returns UNHANDLED:
166488c4b8d1SNeel Natu 	 * - if the access is allowed then it is handled by emulating the
166588c4b8d1SNeel Natu 	 *   instruction that caused the VM-exit (outside the critical section)
166688c4b8d1SNeel Natu 	 * - if the access is not allowed then it will be converted to an
166788c4b8d1SNeel Natu 	 *   exitcode of VM_EXITCODE_VMX and will be dealt with in userland.
166888c4b8d1SNeel Natu 	 */
166988c4b8d1SNeel Natu 	return (UNHANDLED);
167088c4b8d1SNeel Natu }
167188c4b8d1SNeel Natu 
167288c4b8d1SNeel Natu static int
1673366f6083SPeter Grehan vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1674366f6083SPeter Grehan {
1675f76fc5d4SNeel Natu 	int error, handled;
1676366f6083SPeter Grehan 	struct vmxctx *vmxctx;
167788c4b8d1SNeel Natu 	struct vlapic *vlapic;
1678e5a1d950SNeel Natu 	uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, reason;
16793de83862SNeel Natu 	uint64_t qual, gpa;
1680becd9849SNeel Natu 	bool retu;
1681366f6083SPeter Grehan 
1682160471d2SNeel Natu 	CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0);
1683c308b23bSNeel Natu 	CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0);
1684160471d2SNeel Natu 
1685*a0efd3fbSJohn Baldwin 	handled = UNHANDLED;
1686366f6083SPeter Grehan 	vmxctx = &vmx->ctx[vcpu];
16870492757cSNeel Natu 
1688366f6083SPeter Grehan 	qual = vmexit->u.vmx.exit_qualification;
1689318224bbSNeel Natu 	reason = vmexit->u.vmx.exit_reason;
1690366f6083SPeter Grehan 	vmexit->exitcode = VM_EXITCODE_BOGUS;
1691366f6083SPeter Grehan 
169261592433SNeel Natu 	vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1);
169361592433SNeel Natu 
1694318224bbSNeel Natu 	/*
1695318224bbSNeel Natu 	 * VM exits that could be triggered during event injection on the
1696318224bbSNeel Natu 	 * previous VM entry need to be handled specially by re-injecting
1697318224bbSNeel Natu 	 * the event.
1698318224bbSNeel Natu 	 *
1699318224bbSNeel Natu 	 * See "Information for VM Exits During Event Delivery" in Intel SDM
1700318224bbSNeel Natu 	 * for details.
1701318224bbSNeel Natu 	 */
1702318224bbSNeel Natu 	switch (reason) {
1703318224bbSNeel Natu 	case EXIT_REASON_EPT_FAULT:
1704318224bbSNeel Natu 	case EXIT_REASON_EPT_MISCONFIG:
170588c4b8d1SNeel Natu 	case EXIT_REASON_APIC_ACCESS:
1706318224bbSNeel Natu 	case EXIT_REASON_TASK_SWITCH:
1707318224bbSNeel Natu 	case EXIT_REASON_EXCEPTION:
1708318224bbSNeel Natu 		idtvec_info = vmcs_idt_vectoring_info();
1709318224bbSNeel Natu 		if (idtvec_info & VMCS_IDT_VEC_VALID) {
1710318224bbSNeel Natu 			idtvec_info &= ~(1 << 12); /* clear undefined bit */
17113de83862SNeel Natu 			vmcs_write(VMCS_ENTRY_INTR_INFO, idtvec_info);
1712318224bbSNeel Natu 			if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
1713318224bbSNeel Natu 				idtvec_err = vmcs_idt_vectoring_err();
17143de83862SNeel Natu 				vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR,
17153de83862SNeel Natu 				    idtvec_err);
1716318224bbSNeel Natu 			}
1717160471d2SNeel Natu 			/*
1718160471d2SNeel Natu 			 * If 'virtual NMIs' are being used and the VM-exit
1719160471d2SNeel Natu 			 * happened while injecting an NMI during the previous
1720160471d2SNeel Natu 			 * VM-entry, then clear "blocking by NMI" in the Guest
1721160471d2SNeel Natu 			 * Interruptibility-state.
1722160471d2SNeel Natu 			 */
1723160471d2SNeel Natu 			if ((idtvec_info & VMCS_INTR_T_MASK) ==
1724160471d2SNeel Natu 			    VMCS_INTR_T_NMI) {
1725e5a1d950SNeel Natu 				 vmx_clear_nmi_blocking(vmx, vcpu);
1726160471d2SNeel Natu 			}
17273de83862SNeel Natu 			vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
1728318224bbSNeel Natu 		}
1729318224bbSNeel Natu 	default:
1730e5a1d950SNeel Natu 		idtvec_info = 0;
1731318224bbSNeel Natu 		break;
1732318224bbSNeel Natu 	}
1733318224bbSNeel Natu 
1734318224bbSNeel Natu 	switch (reason) {
1735366f6083SPeter Grehan 	case EXIT_REASON_CR_ACCESS:
1736b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CR_ACCESS, 1);
1737366f6083SPeter Grehan 		handled = vmx_emulate_cr_access(vmx, vcpu, qual);
1738366f6083SPeter Grehan 		break;
1739366f6083SPeter Grehan 	case EXIT_REASON_RDMSR:
1740b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RDMSR, 1);
1741becd9849SNeel Natu 		retu = false;
1742366f6083SPeter Grehan 		ecx = vmxctx->guest_rcx;
1743becd9849SNeel Natu 		error = emulate_rdmsr(vmx->vm, vcpu, ecx, &retu);
1744b42206f3SNeel Natu 		if (error) {
1745366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_RDMSR;
1746366f6083SPeter Grehan 			vmexit->u.msr.code = ecx;
1747becd9849SNeel Natu 		} else if (!retu) {
1748*a0efd3fbSJohn Baldwin 			handled = HANDLED;
1749becd9849SNeel Natu 		} else {
1750becd9849SNeel Natu 			/* Return to userspace with a valid exitcode */
1751becd9849SNeel Natu 			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
1752becd9849SNeel Natu 			    ("emulate_wrmsr retu with bogus exitcode"));
1753becd9849SNeel Natu 		}
1754366f6083SPeter Grehan 		break;
1755366f6083SPeter Grehan 	case EXIT_REASON_WRMSR:
1756b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_WRMSR, 1);
1757becd9849SNeel Natu 		retu = false;
1758366f6083SPeter Grehan 		eax = vmxctx->guest_rax;
1759366f6083SPeter Grehan 		ecx = vmxctx->guest_rcx;
1760366f6083SPeter Grehan 		edx = vmxctx->guest_rdx;
1761b42206f3SNeel Natu 		error = emulate_wrmsr(vmx->vm, vcpu, ecx,
1762becd9849SNeel Natu 		    (uint64_t)edx << 32 | eax, &retu);
1763b42206f3SNeel Natu 		if (error) {
1764366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_WRMSR;
1765366f6083SPeter Grehan 			vmexit->u.msr.code = ecx;
1766366f6083SPeter Grehan 			vmexit->u.msr.wval = (uint64_t)edx << 32 | eax;
1767becd9849SNeel Natu 		} else if (!retu) {
1768*a0efd3fbSJohn Baldwin 			handled = HANDLED;
1769becd9849SNeel Natu 		} else {
1770becd9849SNeel Natu 			/* Return to userspace with a valid exitcode */
1771becd9849SNeel Natu 			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
1772becd9849SNeel Natu 			    ("emulate_wrmsr retu with bogus exitcode"));
1773becd9849SNeel Natu 		}
1774366f6083SPeter Grehan 		break;
1775366f6083SPeter Grehan 	case EXIT_REASON_HLT:
1776f76fc5d4SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_HLT, 1);
1777366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_HLT;
17783de83862SNeel Natu 		vmexit->u.hlt.rflags = vmcs_read(VMCS_GUEST_RFLAGS);
1779366f6083SPeter Grehan 		break;
1780366f6083SPeter Grehan 	case EXIT_REASON_MTF:
1781b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_MTRAP, 1);
1782366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_MTRAP;
1783366f6083SPeter Grehan 		break;
1784366f6083SPeter Grehan 	case EXIT_REASON_PAUSE:
1785b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_PAUSE, 1);
1786366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_PAUSE;
1787366f6083SPeter Grehan 		break;
1788366f6083SPeter Grehan 	case EXIT_REASON_INTR_WINDOW:
1789b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INTR_WINDOW, 1);
1790366f6083SPeter Grehan 		vmx_clear_int_window_exiting(vmx, vcpu);
1791b5aaf7b2SNeel Natu 		return (1);
1792366f6083SPeter Grehan 	case EXIT_REASON_EXT_INTR:
1793366f6083SPeter Grehan 		/*
1794366f6083SPeter Grehan 		 * External interrupts serve only to cause VM exits and allow
1795366f6083SPeter Grehan 		 * the host interrupt handler to run.
1796366f6083SPeter Grehan 		 *
1797366f6083SPeter Grehan 		 * If this external interrupt triggers a virtual interrupt
1798366f6083SPeter Grehan 		 * to a VM, then that state will be recorded by the
1799366f6083SPeter Grehan 		 * host interrupt handler in the VM's softc. We will inject
1800366f6083SPeter Grehan 		 * this virtual interrupt during the subsequent VM enter.
1801366f6083SPeter Grehan 		 */
1802f7d47425SNeel Natu 		intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
1803160471d2SNeel Natu 		KASSERT((intr_info & VMCS_INTR_VALID) != 0 &&
1804160471d2SNeel Natu 		    (intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_HWINTR,
1805f7d47425SNeel Natu 		    ("VM exit interruption info invalid: %#x", intr_info));
1806f7d47425SNeel Natu 		vmx_trigger_hostintr(intr_info & 0xff);
1807366f6083SPeter Grehan 
1808366f6083SPeter Grehan 		/*
1809366f6083SPeter Grehan 		 * This is special. We want to treat this as an 'handled'
1810366f6083SPeter Grehan 		 * VM-exit but not increment the instruction pointer.
1811366f6083SPeter Grehan 		 */
1812366f6083SPeter Grehan 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1);
1813366f6083SPeter Grehan 		return (1);
1814366f6083SPeter Grehan 	case EXIT_REASON_NMI_WINDOW:
1815366f6083SPeter Grehan 		/* Exit to allow the pending virtual NMI to be injected */
181648b2d828SNeel Natu 		if (vm_nmi_pending(vmx->vm, vcpu))
181748b2d828SNeel Natu 			vmx_inject_nmi(vmx, vcpu);
1818366f6083SPeter Grehan 		vmx_clear_nmi_window_exiting(vmx, vcpu);
181948b2d828SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NMI_WINDOW, 1);
1820366f6083SPeter Grehan 		return (1);
1821366f6083SPeter Grehan 	case EXIT_REASON_INOUT:
1822b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INOUT, 1);
1823366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_INOUT;
1824366f6083SPeter Grehan 		vmexit->u.inout.bytes = (qual & 0x7) + 1;
1825366f6083SPeter Grehan 		vmexit->u.inout.in = (qual & 0x8) ? 1 : 0;
1826366f6083SPeter Grehan 		vmexit->u.inout.string = (qual & 0x10) ? 1 : 0;
1827366f6083SPeter Grehan 		vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0;
1828366f6083SPeter Grehan 		vmexit->u.inout.port = (uint16_t)(qual >> 16);
1829366f6083SPeter Grehan 		vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax);
1830366f6083SPeter Grehan 		break;
1831366f6083SPeter Grehan 	case EXIT_REASON_CPUID:
1832b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CPUID, 1);
1833a2da7af6SNeel Natu 		handled = vmx_handle_cpuid(vmx->vm, vcpu, vmxctx);
1834366f6083SPeter Grehan 		break;
1835e5a1d950SNeel Natu 	case EXIT_REASON_EXCEPTION:
1836c308b23bSNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXCEPTION, 1);
1837e5a1d950SNeel Natu 		intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
1838e5a1d950SNeel Natu 		KASSERT((intr_info & VMCS_INTR_VALID) != 0,
1839e5a1d950SNeel Natu 		    ("VM exit interruption info invalid: %#x", intr_info));
1840c308b23bSNeel Natu 
1841e5a1d950SNeel Natu 		/*
1842e5a1d950SNeel Natu 		 * If Virtual NMIs control is 1 and the VM-exit is due to a
1843e5a1d950SNeel Natu 		 * fault encountered during the execution of IRET then we must
1844e5a1d950SNeel Natu 		 * restore the state of "virtual-NMI blocking" before resuming
1845e5a1d950SNeel Natu 		 * the guest.
1846e5a1d950SNeel Natu 		 *
1847e5a1d950SNeel Natu 		 * See "Resuming Guest Software after Handling an Exception".
1848e5a1d950SNeel Natu 		 */
1849e5a1d950SNeel Natu 		if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
1850e5a1d950SNeel Natu 		    (intr_info & 0xff) != IDT_DF &&
1851e5a1d950SNeel Natu 		    (intr_info & EXIT_QUAL_NMIUDTI) != 0)
1852e5a1d950SNeel Natu 			vmx_restore_nmi_blocking(vmx, vcpu);
1853c308b23bSNeel Natu 
1854c308b23bSNeel Natu 		/*
185562fbd7c2SNeel Natu 		 * The NMI has already been handled in vmx_exit_handle_nmi().
1856c308b23bSNeel Natu 		 */
185762fbd7c2SNeel Natu 		if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI)
1858c308b23bSNeel Natu 			return (1);
1859e5a1d950SNeel Natu 		break;
1860cd942e0fSPeter Grehan 	case EXIT_REASON_EPT_FAULT:
1861318224bbSNeel Natu 		/*
1862318224bbSNeel Natu 		 * If 'gpa' lies within the address space allocated to
1863318224bbSNeel Natu 		 * memory then this must be a nested page fault otherwise
1864318224bbSNeel Natu 		 * this must be an instruction that accesses MMIO space.
1865318224bbSNeel Natu 		 */
1866a2da7af6SNeel Natu 		gpa = vmcs_gpa();
186788c4b8d1SNeel Natu 		if (vm_mem_allocated(vmx->vm, gpa) || apic_access_fault(gpa)) {
1868cd942e0fSPeter Grehan 			vmexit->exitcode = VM_EXITCODE_PAGING;
186913ec9371SPeter Grehan 			vmexit->u.paging.gpa = gpa;
1870318224bbSNeel Natu 			vmexit->u.paging.fault_type = ept_fault_type(qual);
1871bf73979dSNeel Natu 			vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NESTED_FAULT, 1);
1872318224bbSNeel Natu 		} else if (ept_emulation_fault(qual)) {
1873318224bbSNeel Natu 			vmexit->exitcode = VM_EXITCODE_INST_EMUL;
1874318224bbSNeel Natu 			vmexit->u.inst_emul.gpa = gpa;
1875318224bbSNeel Natu 			vmexit->u.inst_emul.gla = vmcs_gla();
1876318224bbSNeel Natu 			vmexit->u.inst_emul.cr3 = vmcs_guest_cr3();
187700f3efe1SJohn Baldwin 			vmexit->u.inst_emul.cpu_mode = vmx_cpu_mode();
187800f3efe1SJohn Baldwin 			vmexit->u.inst_emul.paging_mode = vmx_paging_mode();
1879bf73979dSNeel Natu 			vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INST_EMUL, 1);
1880a2da7af6SNeel Natu 		}
1881e5a1d950SNeel Natu 		/*
1882e5a1d950SNeel Natu 		 * If Virtual NMIs control is 1 and the VM-exit is due to an
1883e5a1d950SNeel Natu 		 * EPT fault during the execution of IRET then we must restore
1884e5a1d950SNeel Natu 		 * the state of "virtual-NMI blocking" before resuming.
1885e5a1d950SNeel Natu 		 *
1886e5a1d950SNeel Natu 		 * See description of "NMI unblocking due to IRET" in
1887e5a1d950SNeel Natu 		 * "Exit Qualification for EPT Violations".
1888e5a1d950SNeel Natu 		 */
1889e5a1d950SNeel Natu 		if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
1890e5a1d950SNeel Natu 		    (qual & EXIT_QUAL_NMIUDTI) != 0)
1891e5a1d950SNeel Natu 			vmx_restore_nmi_blocking(vmx, vcpu);
1892cd942e0fSPeter Grehan 		break;
189330b94db8SNeel Natu 	case EXIT_REASON_VIRTUALIZED_EOI:
189430b94db8SNeel Natu 		vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI;
189530b94db8SNeel Natu 		vmexit->u.ioapic_eoi.vector = qual & 0xFF;
189630b94db8SNeel Natu 		vmexit->inst_length = 0;	/* trap-like */
189730b94db8SNeel Natu 		break;
189888c4b8d1SNeel Natu 	case EXIT_REASON_APIC_ACCESS:
189988c4b8d1SNeel Natu 		handled = vmx_handle_apic_access(vmx, vcpu, vmexit);
190088c4b8d1SNeel Natu 		break;
190188c4b8d1SNeel Natu 	case EXIT_REASON_APIC_WRITE:
190288c4b8d1SNeel Natu 		/*
190388c4b8d1SNeel Natu 		 * APIC-write VM exit is trap-like so the %rip is already
190488c4b8d1SNeel Natu 		 * pointing to the next instruction.
190588c4b8d1SNeel Natu 		 */
190688c4b8d1SNeel Natu 		vmexit->inst_length = 0;
190788c4b8d1SNeel Natu 		vlapic = vm_lapic(vmx->vm, vcpu);
190888c4b8d1SNeel Natu 		handled = vmx_handle_apic_write(vlapic, qual);
190988c4b8d1SNeel Natu 		break;
1910abb023fbSJohn Baldwin 	case EXIT_REASON_XSETBV:
1911*a0efd3fbSJohn Baldwin 		handled = vmx_emulate_xsetbv(vmx, vcpu, vmexit);
1912abb023fbSJohn Baldwin 		break;
1913366f6083SPeter Grehan 	default:
1914b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_UNKNOWN, 1);
1915366f6083SPeter Grehan 		break;
1916366f6083SPeter Grehan 	}
1917366f6083SPeter Grehan 
1918366f6083SPeter Grehan 	if (handled) {
1919366f6083SPeter Grehan 		/*
1920366f6083SPeter Grehan 		 * It is possible that control is returned to userland
1921366f6083SPeter Grehan 		 * even though we were able to handle the VM exit in the
1922eeefa4e4SNeel Natu 		 * kernel.
1923366f6083SPeter Grehan 		 *
1924366f6083SPeter Grehan 		 * In such a case we want to make sure that the userland
1925366f6083SPeter Grehan 		 * restarts guest execution at the instruction *after*
1926366f6083SPeter Grehan 		 * the one we just processed. Therefore we update the
1927366f6083SPeter Grehan 		 * guest rip in the VMCS and in 'vmexit'.
1928366f6083SPeter Grehan 		 */
1929366f6083SPeter Grehan 		vmexit->rip += vmexit->inst_length;
1930366f6083SPeter Grehan 		vmexit->inst_length = 0;
19313de83862SNeel Natu 		vmcs_write(VMCS_GUEST_RIP, vmexit->rip);
1932366f6083SPeter Grehan 	} else {
1933366f6083SPeter Grehan 		if (vmexit->exitcode == VM_EXITCODE_BOGUS) {
1934366f6083SPeter Grehan 			/*
1935366f6083SPeter Grehan 			 * If this VM exit was not claimed by anybody then
1936366f6083SPeter Grehan 			 * treat it as a generic VMX exit.
1937366f6083SPeter Grehan 			 */
1938366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_VMX;
19390492757cSNeel Natu 			vmexit->u.vmx.status = VM_SUCCESS;
1940c308b23bSNeel Natu 			vmexit->u.vmx.inst_type = 0;
1941c308b23bSNeel Natu 			vmexit->u.vmx.inst_error = 0;
1942366f6083SPeter Grehan 		} else {
1943366f6083SPeter Grehan 			/*
1944366f6083SPeter Grehan 			 * The exitcode and collateral have been populated.
1945366f6083SPeter Grehan 			 * The VM exit will be processed further in userland.
1946366f6083SPeter Grehan 			 */
1947366f6083SPeter Grehan 		}
1948366f6083SPeter Grehan 	}
1949366f6083SPeter Grehan 	return (handled);
1950366f6083SPeter Grehan }
1951366f6083SPeter Grehan 
19520492757cSNeel Natu static __inline int
19530492757cSNeel Natu vmx_exit_astpending(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1954366f6083SPeter Grehan {
19550492757cSNeel Natu 
19560492757cSNeel Natu 	vmexit->rip = vmcs_guest_rip();
19570492757cSNeel Natu 	vmexit->inst_length = 0;
19580492757cSNeel Natu 	vmexit->exitcode = VM_EXITCODE_BOGUS;
19590492757cSNeel Natu 	vmx_astpending_trace(vmx, vcpu, vmexit->rip);
19600492757cSNeel Natu 	vmm_stat_incr(vmx->vm, vcpu, VMEXIT_ASTPENDING, 1);
19610492757cSNeel Natu 
19620492757cSNeel Natu 	return (HANDLED);
19630492757cSNeel Natu }
19640492757cSNeel Natu 
19650492757cSNeel Natu static __inline int
19665b8a8cd1SNeel Natu vmx_exit_rendezvous(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
19675b8a8cd1SNeel Natu {
19685b8a8cd1SNeel Natu 
19695b8a8cd1SNeel Natu 	vmexit->rip = vmcs_guest_rip();
19705b8a8cd1SNeel Natu 	vmexit->inst_length = 0;
19715b8a8cd1SNeel Natu 	vmexit->exitcode = VM_EXITCODE_RENDEZVOUS;
19725b8a8cd1SNeel Natu 	vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RENDEZVOUS, 1);
19735b8a8cd1SNeel Natu 
19745b8a8cd1SNeel Natu 	return (UNHANDLED);
19755b8a8cd1SNeel Natu }
19765b8a8cd1SNeel Natu 
19775b8a8cd1SNeel Natu static __inline int
19780492757cSNeel Natu vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit)
19790492757cSNeel Natu {
19800492757cSNeel Natu 
19810492757cSNeel Natu 	KASSERT(vmxctx->inst_fail_status != VM_SUCCESS,
19820492757cSNeel Natu 	    ("vmx_exit_inst_error: invalid inst_fail_status %d",
19830492757cSNeel Natu 	    vmxctx->inst_fail_status));
19840492757cSNeel Natu 
19850492757cSNeel Natu 	vmexit->inst_length = 0;
19860492757cSNeel Natu 	vmexit->exitcode = VM_EXITCODE_VMX;
19870492757cSNeel Natu 	vmexit->u.vmx.status = vmxctx->inst_fail_status;
19880492757cSNeel Natu 	vmexit->u.vmx.inst_error = vmcs_instruction_error();
19890492757cSNeel Natu 	vmexit->u.vmx.exit_reason = ~0;
19900492757cSNeel Natu 	vmexit->u.vmx.exit_qualification = ~0;
19910492757cSNeel Natu 
19920492757cSNeel Natu 	switch (rc) {
19930492757cSNeel Natu 	case VMX_VMRESUME_ERROR:
19940492757cSNeel Natu 	case VMX_VMLAUNCH_ERROR:
19950492757cSNeel Natu 	case VMX_INVEPT_ERROR:
19960492757cSNeel Natu 		vmexit->u.vmx.inst_type = rc;
19970492757cSNeel Natu 		break;
19980492757cSNeel Natu 	default:
19990492757cSNeel Natu 		panic("vm_exit_inst_error: vmx_enter_guest returned %d", rc);
20000492757cSNeel Natu 	}
20010492757cSNeel Natu 
20020492757cSNeel Natu 	return (UNHANDLED);
20030492757cSNeel Natu }
20040492757cSNeel Natu 
200562fbd7c2SNeel Natu /*
200662fbd7c2SNeel Natu  * If the NMI-exiting VM execution control is set to '1' then an NMI in
200762fbd7c2SNeel Natu  * non-root operation causes a VM-exit. NMI blocking is in effect so it is
200862fbd7c2SNeel Natu  * sufficient to simply vector to the NMI handler via a software interrupt.
200962fbd7c2SNeel Natu  * However, this must be done before maskable interrupts are enabled
201062fbd7c2SNeel Natu  * otherwise the "iret" issued by an interrupt handler will incorrectly
201162fbd7c2SNeel Natu  * clear NMI blocking.
201262fbd7c2SNeel Natu  */
201362fbd7c2SNeel Natu static __inline void
201462fbd7c2SNeel Natu vmx_exit_handle_nmi(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
201562fbd7c2SNeel Natu {
201662fbd7c2SNeel Natu 	uint32_t intr_info;
201762fbd7c2SNeel Natu 
201862fbd7c2SNeel Natu 	KASSERT((read_rflags() & PSL_I) == 0, ("interrupts enabled"));
201962fbd7c2SNeel Natu 
202062fbd7c2SNeel Natu 	if (vmexit->u.vmx.exit_reason != EXIT_REASON_EXCEPTION)
202162fbd7c2SNeel Natu 		return;
202262fbd7c2SNeel Natu 
202362fbd7c2SNeel Natu 	intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
202462fbd7c2SNeel Natu 	KASSERT((intr_info & VMCS_INTR_VALID) != 0,
202562fbd7c2SNeel Natu 	    ("VM exit interruption info invalid: %#x", intr_info));
202662fbd7c2SNeel Natu 
202762fbd7c2SNeel Natu 	if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI) {
202862fbd7c2SNeel Natu 		KASSERT((intr_info & 0xff) == IDT_NMI, ("VM exit due "
202962fbd7c2SNeel Natu 		    "to NMI has invalid vector: %#x", intr_info));
203062fbd7c2SNeel Natu 		VCPU_CTR0(vmx->vm, vcpuid, "Vectoring to NMI handler");
203162fbd7c2SNeel Natu 		__asm __volatile("int $2");
203262fbd7c2SNeel Natu 	}
203362fbd7c2SNeel Natu }
203462fbd7c2SNeel Natu 
20350492757cSNeel Natu static int
20365b8a8cd1SNeel Natu vmx_run(void *arg, int vcpu, register_t startrip, pmap_t pmap,
20375b8a8cd1SNeel Natu     void *rendezvous_cookie)
20380492757cSNeel Natu {
20390492757cSNeel Natu 	int rc, handled, launched;
2040366f6083SPeter Grehan 	struct vmx *vmx;
20415b8a8cd1SNeel Natu 	struct vm *vm;
2042366f6083SPeter Grehan 	struct vmxctx *vmxctx;
2043366f6083SPeter Grehan 	struct vmcs *vmcs;
204498ed632cSNeel Natu 	struct vm_exit *vmexit;
2045de5ea6b6SNeel Natu 	struct vlapic *vlapic;
204679c59630SNeel Natu 	uint64_t rip;
204779c59630SNeel Natu 	uint32_t exit_reason;
2048366f6083SPeter Grehan 
2049366f6083SPeter Grehan 	vmx = arg;
20505b8a8cd1SNeel Natu 	vm = vmx->vm;
2051366f6083SPeter Grehan 	vmcs = &vmx->vmcs[vcpu];
2052366f6083SPeter Grehan 	vmxctx = &vmx->ctx[vcpu];
20535b8a8cd1SNeel Natu 	vlapic = vm_lapic(vm, vcpu);
20545b8a8cd1SNeel Natu 	vmexit = vm_exitinfo(vm, vcpu);
20550492757cSNeel Natu 	launched = 0;
205698ed632cSNeel Natu 
2057318224bbSNeel Natu 	KASSERT(vmxctx->pmap == pmap,
2058318224bbSNeel Natu 	    ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap));
2059318224bbSNeel Natu 
2060366f6083SPeter Grehan 	VMPTRLD(vmcs);
2061366f6083SPeter Grehan 
2062366f6083SPeter Grehan 	/*
2063366f6083SPeter Grehan 	 * XXX
2064366f6083SPeter Grehan 	 * We do this every time because we may setup the virtual machine
2065366f6083SPeter Grehan 	 * from a different process than the one that actually runs it.
2066366f6083SPeter Grehan 	 *
2067366f6083SPeter Grehan 	 * If the life of a virtual machine was spent entirely in the context
2068c847a506SNeel Natu 	 * of a single process we could do this once in vmx_vminit().
2069366f6083SPeter Grehan 	 */
20703de83862SNeel Natu 	vmcs_write(VMCS_HOST_CR3, rcr3());
2071366f6083SPeter Grehan 
20720492757cSNeel Natu 	vmcs_write(VMCS_GUEST_RIP, startrip);
2073953c2c47SNeel Natu 	vmx_set_pcpu_defaults(vmx, vcpu, pmap);
2074366f6083SPeter Grehan 	do {
20750492757cSNeel Natu 		/*
20760492757cSNeel Natu 		 * Interrupts are disabled from this point on until the
20770492757cSNeel Natu 		 * guest starts executing. This is done for the following
20780492757cSNeel Natu 		 * reasons:
20790492757cSNeel Natu 		 *
20800492757cSNeel Natu 		 * If an AST is asserted on this thread after the check below,
20810492757cSNeel Natu 		 * then the IPI_AST notification will not be lost, because it
20820492757cSNeel Natu 		 * will cause a VM exit due to external interrupt as soon as
20830492757cSNeel Natu 		 * the guest state is loaded.
20840492757cSNeel Natu 		 *
20850492757cSNeel Natu 		 * A posted interrupt after 'vmx_inject_interrupts()' will
20860492757cSNeel Natu 		 * not be "lost" because it will be held pending in the host
20870492757cSNeel Natu 		 * APIC because interrupts are disabled. The pending interrupt
20880492757cSNeel Natu 		 * will be recognized as soon as the guest state is loaded.
20890492757cSNeel Natu 		 *
20900492757cSNeel Natu 		 * The same reasoning applies to the IPI generated by
20910492757cSNeel Natu 		 * pmap_invalidate_ept().
20920492757cSNeel Natu 		 */
20930492757cSNeel Natu 		disable_intr();
20940492757cSNeel Natu 		if (curthread->td_flags & (TDF_ASTPENDING | TDF_NEEDRESCHED)) {
20950492757cSNeel Natu 			enable_intr();
20960492757cSNeel Natu 			handled = vmx_exit_astpending(vmx, vcpu, vmexit);
20970492757cSNeel Natu 			break;
20980492757cSNeel Natu 		}
20990492757cSNeel Natu 
21005b8a8cd1SNeel Natu 		if (vcpu_rendezvous_pending(rendezvous_cookie)) {
21015b8a8cd1SNeel Natu 			enable_intr();
21025b8a8cd1SNeel Natu 			handled = vmx_exit_rendezvous(vmx, vcpu, vmexit);
21035b8a8cd1SNeel Natu 			break;
21045b8a8cd1SNeel Natu 		}
21055b8a8cd1SNeel Natu 
2106de5ea6b6SNeel Natu 		vmx_inject_interrupts(vmx, vcpu, vlapic);
2107366f6083SPeter Grehan 		vmx_run_trace(vmx, vcpu);
2108953c2c47SNeel Natu 		rc = vmx_enter_guest(vmxctx, vmx, launched);
210979c59630SNeel Natu 
211079c59630SNeel Natu 		/* Collect some information for VM exit processing */
211179c59630SNeel Natu 		vmexit->rip = rip = vmcs_guest_rip();
211279c59630SNeel Natu 		vmexit->inst_length = vmexit_instruction_length();
211379c59630SNeel Natu 		vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason();
211479c59630SNeel Natu 		vmexit->u.vmx.exit_qualification = vmcs_exit_qualification();
211579c59630SNeel Natu 
21160492757cSNeel Natu 		if (rc == VMX_GUEST_VMEXIT) {
211762fbd7c2SNeel Natu 			vmx_exit_handle_nmi(vmx, vcpu, vmexit);
211862fbd7c2SNeel Natu 			enable_intr();
21190492757cSNeel Natu 			handled = vmx_exit_process(vmx, vcpu, vmexit);
21200492757cSNeel Natu 		} else {
212162fbd7c2SNeel Natu 			enable_intr();
21220492757cSNeel Natu 			handled = vmx_exit_inst_error(vmxctx, rc, vmexit);
2123eeefa4e4SNeel Natu 		}
212462fbd7c2SNeel Natu 		launched = 1;
212579c59630SNeel Natu 		vmx_exit_trace(vmx, vcpu, rip, exit_reason, handled);
2126eeefa4e4SNeel Natu 	} while (handled);
2127366f6083SPeter Grehan 
2128366f6083SPeter Grehan 	/*
2129366f6083SPeter Grehan 	 * If a VM exit has been handled then the exitcode must be BOGUS
2130366f6083SPeter Grehan 	 * If a VM exit is not handled then the exitcode must not be BOGUS
2131366f6083SPeter Grehan 	 */
2132366f6083SPeter Grehan 	if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) ||
2133366f6083SPeter Grehan 	    (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) {
2134366f6083SPeter Grehan 		panic("Mismatch between handled (%d) and exitcode (%d)",
2135366f6083SPeter Grehan 		      handled, vmexit->exitcode);
2136366f6083SPeter Grehan 	}
2137366f6083SPeter Grehan 
2138b5aaf7b2SNeel Natu 	if (!handled)
21395b8a8cd1SNeel Natu 		vmm_stat_incr(vm, vcpu, VMEXIT_USERSPACE, 1);
2140b5aaf7b2SNeel Natu 
21415b8a8cd1SNeel Natu 	VCPU_CTR1(vm, vcpu, "returning from vmx_run: exitcode %d",
21420492757cSNeel Natu 	    vmexit->exitcode);
2143366f6083SPeter Grehan 
2144366f6083SPeter Grehan 	VMCLEAR(vmcs);
2145366f6083SPeter Grehan 	return (0);
2146366f6083SPeter Grehan }
2147366f6083SPeter Grehan 
2148366f6083SPeter Grehan static void
2149366f6083SPeter Grehan vmx_vmcleanup(void *arg)
2150366f6083SPeter Grehan {
215145e51299SNeel Natu 	int i, error;
2152366f6083SPeter Grehan 	struct vmx *vmx = arg;
2153366f6083SPeter Grehan 
215488c4b8d1SNeel Natu 	if (virtual_interrupt_delivery)
215588c4b8d1SNeel Natu 		vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
215688c4b8d1SNeel Natu 
215745e51299SNeel Natu 	for (i = 0; i < VM_MAXCPU; i++)
215845e51299SNeel Natu 		vpid_free(vmx->state[i].vpid);
215945e51299SNeel Natu 
2160366f6083SPeter Grehan 	/*
2161366f6083SPeter Grehan 	 * XXXSMP we also need to clear the VMCS active on the other vcpus.
2162366f6083SPeter Grehan 	 */
2163366f6083SPeter Grehan 	error = vmclear(&vmx->vmcs[0]);
2164366f6083SPeter Grehan 	if (error != 0)
2165366f6083SPeter Grehan 		panic("vmx_vmcleanup: vmclear error %d on vcpu 0", error);
2166366f6083SPeter Grehan 
2167366f6083SPeter Grehan 	free(vmx, M_VMX);
2168366f6083SPeter Grehan 
2169366f6083SPeter Grehan 	return;
2170366f6083SPeter Grehan }
2171366f6083SPeter Grehan 
2172366f6083SPeter Grehan static register_t *
2173366f6083SPeter Grehan vmxctx_regptr(struct vmxctx *vmxctx, int reg)
2174366f6083SPeter Grehan {
2175366f6083SPeter Grehan 
2176366f6083SPeter Grehan 	switch (reg) {
2177366f6083SPeter Grehan 	case VM_REG_GUEST_RAX:
2178366f6083SPeter Grehan 		return (&vmxctx->guest_rax);
2179366f6083SPeter Grehan 	case VM_REG_GUEST_RBX:
2180366f6083SPeter Grehan 		return (&vmxctx->guest_rbx);
2181366f6083SPeter Grehan 	case VM_REG_GUEST_RCX:
2182366f6083SPeter Grehan 		return (&vmxctx->guest_rcx);
2183366f6083SPeter Grehan 	case VM_REG_GUEST_RDX:
2184366f6083SPeter Grehan 		return (&vmxctx->guest_rdx);
2185366f6083SPeter Grehan 	case VM_REG_GUEST_RSI:
2186366f6083SPeter Grehan 		return (&vmxctx->guest_rsi);
2187366f6083SPeter Grehan 	case VM_REG_GUEST_RDI:
2188366f6083SPeter Grehan 		return (&vmxctx->guest_rdi);
2189366f6083SPeter Grehan 	case VM_REG_GUEST_RBP:
2190366f6083SPeter Grehan 		return (&vmxctx->guest_rbp);
2191366f6083SPeter Grehan 	case VM_REG_GUEST_R8:
2192366f6083SPeter Grehan 		return (&vmxctx->guest_r8);
2193366f6083SPeter Grehan 	case VM_REG_GUEST_R9:
2194366f6083SPeter Grehan 		return (&vmxctx->guest_r9);
2195366f6083SPeter Grehan 	case VM_REG_GUEST_R10:
2196366f6083SPeter Grehan 		return (&vmxctx->guest_r10);
2197366f6083SPeter Grehan 	case VM_REG_GUEST_R11:
2198366f6083SPeter Grehan 		return (&vmxctx->guest_r11);
2199366f6083SPeter Grehan 	case VM_REG_GUEST_R12:
2200366f6083SPeter Grehan 		return (&vmxctx->guest_r12);
2201366f6083SPeter Grehan 	case VM_REG_GUEST_R13:
2202366f6083SPeter Grehan 		return (&vmxctx->guest_r13);
2203366f6083SPeter Grehan 	case VM_REG_GUEST_R14:
2204366f6083SPeter Grehan 		return (&vmxctx->guest_r14);
2205366f6083SPeter Grehan 	case VM_REG_GUEST_R15:
2206366f6083SPeter Grehan 		return (&vmxctx->guest_r15);
2207366f6083SPeter Grehan 	default:
2208366f6083SPeter Grehan 		break;
2209366f6083SPeter Grehan 	}
2210366f6083SPeter Grehan 	return (NULL);
2211366f6083SPeter Grehan }
2212366f6083SPeter Grehan 
2213366f6083SPeter Grehan static int
2214366f6083SPeter Grehan vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval)
2215366f6083SPeter Grehan {
2216366f6083SPeter Grehan 	register_t *regp;
2217366f6083SPeter Grehan 
2218366f6083SPeter Grehan 	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
2219366f6083SPeter Grehan 		*retval = *regp;
2220366f6083SPeter Grehan 		return (0);
2221366f6083SPeter Grehan 	} else
2222366f6083SPeter Grehan 		return (EINVAL);
2223366f6083SPeter Grehan }
2224366f6083SPeter Grehan 
2225366f6083SPeter Grehan static int
2226366f6083SPeter Grehan vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val)
2227366f6083SPeter Grehan {
2228366f6083SPeter Grehan 	register_t *regp;
2229366f6083SPeter Grehan 
2230366f6083SPeter Grehan 	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
2231366f6083SPeter Grehan 		*regp = val;
2232366f6083SPeter Grehan 		return (0);
2233366f6083SPeter Grehan 	} else
2234366f6083SPeter Grehan 		return (EINVAL);
2235366f6083SPeter Grehan }
2236366f6083SPeter Grehan 
2237366f6083SPeter Grehan static int
2238aaaa0656SPeter Grehan vmx_shadow_reg(int reg)
2239aaaa0656SPeter Grehan {
2240aaaa0656SPeter Grehan 	int shreg;
2241aaaa0656SPeter Grehan 
2242aaaa0656SPeter Grehan 	shreg = -1;
2243aaaa0656SPeter Grehan 
2244aaaa0656SPeter Grehan 	switch (reg) {
2245aaaa0656SPeter Grehan 	case VM_REG_GUEST_CR0:
2246aaaa0656SPeter Grehan 		shreg = VMCS_CR0_SHADOW;
2247aaaa0656SPeter Grehan                 break;
2248aaaa0656SPeter Grehan         case VM_REG_GUEST_CR4:
2249aaaa0656SPeter Grehan 		shreg = VMCS_CR4_SHADOW;
2250aaaa0656SPeter Grehan 		break;
2251aaaa0656SPeter Grehan 	default:
2252aaaa0656SPeter Grehan 		break;
2253aaaa0656SPeter Grehan 	}
2254aaaa0656SPeter Grehan 
2255aaaa0656SPeter Grehan 	return (shreg);
2256aaaa0656SPeter Grehan }
2257aaaa0656SPeter Grehan 
2258aaaa0656SPeter Grehan static int
2259366f6083SPeter Grehan vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval)
2260366f6083SPeter Grehan {
2261d3c11f40SPeter Grehan 	int running, hostcpu;
2262366f6083SPeter Grehan 	struct vmx *vmx = arg;
2263366f6083SPeter Grehan 
2264d3c11f40SPeter Grehan 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
2265d3c11f40SPeter Grehan 	if (running && hostcpu != curcpu)
2266d3c11f40SPeter Grehan 		panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu);
2267d3c11f40SPeter Grehan 
2268366f6083SPeter Grehan 	if (vmxctx_getreg(&vmx->ctx[vcpu], reg, retval) == 0)
2269366f6083SPeter Grehan 		return (0);
2270366f6083SPeter Grehan 
2271d3c11f40SPeter Grehan 	return (vmcs_getreg(&vmx->vmcs[vcpu], running, reg, retval));
2272366f6083SPeter Grehan }
2273366f6083SPeter Grehan 
2274366f6083SPeter Grehan static int
2275366f6083SPeter Grehan vmx_setreg(void *arg, int vcpu, int reg, uint64_t val)
2276366f6083SPeter Grehan {
2277aaaa0656SPeter Grehan 	int error, hostcpu, running, shadow;
2278366f6083SPeter Grehan 	uint64_t ctls;
2279366f6083SPeter Grehan 	struct vmx *vmx = arg;
2280366f6083SPeter Grehan 
2281d3c11f40SPeter Grehan 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
2282d3c11f40SPeter Grehan 	if (running && hostcpu != curcpu)
2283d3c11f40SPeter Grehan 		panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu);
2284d3c11f40SPeter Grehan 
2285366f6083SPeter Grehan 	if (vmxctx_setreg(&vmx->ctx[vcpu], reg, val) == 0)
2286366f6083SPeter Grehan 		return (0);
2287366f6083SPeter Grehan 
2288d3c11f40SPeter Grehan 	error = vmcs_setreg(&vmx->vmcs[vcpu], running, reg, val);
2289366f6083SPeter Grehan 
2290366f6083SPeter Grehan 	if (error == 0) {
2291366f6083SPeter Grehan 		/*
2292366f6083SPeter Grehan 		 * If the "load EFER" VM-entry control is 1 then the
2293366f6083SPeter Grehan 		 * value of EFER.LMA must be identical to "IA-32e mode guest"
2294366f6083SPeter Grehan 		 * bit in the VM-entry control.
2295366f6083SPeter Grehan 		 */
2296366f6083SPeter Grehan 		if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 &&
2297366f6083SPeter Grehan 		    (reg == VM_REG_GUEST_EFER)) {
2298d3c11f40SPeter Grehan 			vmcs_getreg(&vmx->vmcs[vcpu], running,
2299366f6083SPeter Grehan 				    VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls);
2300366f6083SPeter Grehan 			if (val & EFER_LMA)
2301366f6083SPeter Grehan 				ctls |= VM_ENTRY_GUEST_LMA;
2302366f6083SPeter Grehan 			else
2303366f6083SPeter Grehan 				ctls &= ~VM_ENTRY_GUEST_LMA;
2304d3c11f40SPeter Grehan 			vmcs_setreg(&vmx->vmcs[vcpu], running,
2305366f6083SPeter Grehan 				    VMCS_IDENT(VMCS_ENTRY_CTLS), ctls);
2306366f6083SPeter Grehan 		}
2307aaaa0656SPeter Grehan 
2308aaaa0656SPeter Grehan 		shadow = vmx_shadow_reg(reg);
2309aaaa0656SPeter Grehan 		if (shadow > 0) {
2310aaaa0656SPeter Grehan 			/*
2311aaaa0656SPeter Grehan 			 * Store the unmodified value in the shadow
2312aaaa0656SPeter Grehan 			 */
2313aaaa0656SPeter Grehan 			error = vmcs_setreg(&vmx->vmcs[vcpu], running,
2314aaaa0656SPeter Grehan 				    VMCS_IDENT(shadow), val);
2315aaaa0656SPeter Grehan 		}
2316366f6083SPeter Grehan 	}
2317366f6083SPeter Grehan 
2318366f6083SPeter Grehan 	return (error);
2319366f6083SPeter Grehan }
2320366f6083SPeter Grehan 
2321366f6083SPeter Grehan static int
2322366f6083SPeter Grehan vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
2323366f6083SPeter Grehan {
2324366f6083SPeter Grehan 	struct vmx *vmx = arg;
2325366f6083SPeter Grehan 
2326366f6083SPeter Grehan 	return (vmcs_getdesc(&vmx->vmcs[vcpu], reg, desc));
2327366f6083SPeter Grehan }
2328366f6083SPeter Grehan 
2329366f6083SPeter Grehan static int
2330366f6083SPeter Grehan vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
2331366f6083SPeter Grehan {
2332366f6083SPeter Grehan 	struct vmx *vmx = arg;
2333366f6083SPeter Grehan 
2334366f6083SPeter Grehan 	return (vmcs_setdesc(&vmx->vmcs[vcpu], reg, desc));
2335366f6083SPeter Grehan }
2336366f6083SPeter Grehan 
2337366f6083SPeter Grehan static int
2338366f6083SPeter Grehan vmx_inject(void *arg, int vcpu, int type, int vector, uint32_t code,
2339366f6083SPeter Grehan 	   int code_valid)
2340366f6083SPeter Grehan {
2341366f6083SPeter Grehan 	struct vmx *vmx = arg;
2342*a0efd3fbSJohn Baldwin 	struct vmxevent *user_event = &vmx->state[vcpu].user_event;
2343366f6083SPeter Grehan 
2344366f6083SPeter Grehan 	static uint32_t type_map[VM_EVENT_MAX] = {
2345366f6083SPeter Grehan 		0x1,		/* VM_EVENT_NONE */
2346366f6083SPeter Grehan 		0x0,		/* VM_HW_INTR */
2347366f6083SPeter Grehan 		0x2,		/* VM_NMI */
2348366f6083SPeter Grehan 		0x3,		/* VM_HW_EXCEPTION */
2349366f6083SPeter Grehan 		0x4,		/* VM_SW_INTR */
2350366f6083SPeter Grehan 		0x5,		/* VM_PRIV_SW_EXCEPTION */
2351366f6083SPeter Grehan 		0x6,		/* VM_SW_EXCEPTION */
2352366f6083SPeter Grehan 	};
2353366f6083SPeter Grehan 
2354eeefa4e4SNeel Natu 	/*
2355eeefa4e4SNeel Natu 	 * If there is already an exception pending to be delivered to the
2356eeefa4e4SNeel Natu 	 * vcpu then just return.
2357eeefa4e4SNeel Natu 	 */
2358*a0efd3fbSJohn Baldwin 	if (user_event->intr_info & VMCS_INTR_VALID)
2359eeefa4e4SNeel Natu 		return (EAGAIN);
2360eeefa4e4SNeel Natu 
2361*a0efd3fbSJohn Baldwin 	user_event->intr_info = vector | (type_map[type] << 8) | VMCS_INTR_VALID;
2362366f6083SPeter Grehan 	if (code_valid) {
2363*a0efd3fbSJohn Baldwin 		user_event->intr_info |= VMCS_INTR_DEL_ERRCODE;
2364*a0efd3fbSJohn Baldwin 		user_event->error_code = code;
2365366f6083SPeter Grehan 	}
2366*a0efd3fbSJohn Baldwin 	return (0);
2367366f6083SPeter Grehan }
2368366f6083SPeter Grehan 
2369366f6083SPeter Grehan static int
2370366f6083SPeter Grehan vmx_getcap(void *arg, int vcpu, int type, int *retval)
2371366f6083SPeter Grehan {
2372366f6083SPeter Grehan 	struct vmx *vmx = arg;
2373366f6083SPeter Grehan 	int vcap;
2374366f6083SPeter Grehan 	int ret;
2375366f6083SPeter Grehan 
2376366f6083SPeter Grehan 	ret = ENOENT;
2377366f6083SPeter Grehan 
2378366f6083SPeter Grehan 	vcap = vmx->cap[vcpu].set;
2379366f6083SPeter Grehan 
2380366f6083SPeter Grehan 	switch (type) {
2381366f6083SPeter Grehan 	case VM_CAP_HALT_EXIT:
2382366f6083SPeter Grehan 		if (cap_halt_exit)
2383366f6083SPeter Grehan 			ret = 0;
2384366f6083SPeter Grehan 		break;
2385366f6083SPeter Grehan 	case VM_CAP_PAUSE_EXIT:
2386366f6083SPeter Grehan 		if (cap_pause_exit)
2387366f6083SPeter Grehan 			ret = 0;
2388366f6083SPeter Grehan 		break;
2389366f6083SPeter Grehan 	case VM_CAP_MTRAP_EXIT:
2390366f6083SPeter Grehan 		if (cap_monitor_trap)
2391366f6083SPeter Grehan 			ret = 0;
2392366f6083SPeter Grehan 		break;
2393366f6083SPeter Grehan 	case VM_CAP_UNRESTRICTED_GUEST:
2394366f6083SPeter Grehan 		if (cap_unrestricted_guest)
2395366f6083SPeter Grehan 			ret = 0;
2396366f6083SPeter Grehan 		break;
239749cc03daSNeel Natu 	case VM_CAP_ENABLE_INVPCID:
239849cc03daSNeel Natu 		if (cap_invpcid)
239949cc03daSNeel Natu 			ret = 0;
240049cc03daSNeel Natu 		break;
2401366f6083SPeter Grehan 	default:
2402366f6083SPeter Grehan 		break;
2403366f6083SPeter Grehan 	}
2404366f6083SPeter Grehan 
2405366f6083SPeter Grehan 	if (ret == 0)
2406366f6083SPeter Grehan 		*retval = (vcap & (1 << type)) ? 1 : 0;
2407366f6083SPeter Grehan 
2408366f6083SPeter Grehan 	return (ret);
2409366f6083SPeter Grehan }
2410366f6083SPeter Grehan 
2411366f6083SPeter Grehan static int
2412366f6083SPeter Grehan vmx_setcap(void *arg, int vcpu, int type, int val)
2413366f6083SPeter Grehan {
2414366f6083SPeter Grehan 	struct vmx *vmx = arg;
2415366f6083SPeter Grehan 	struct vmcs *vmcs = &vmx->vmcs[vcpu];
2416366f6083SPeter Grehan 	uint32_t baseval;
2417366f6083SPeter Grehan 	uint32_t *pptr;
2418366f6083SPeter Grehan 	int error;
2419366f6083SPeter Grehan 	int flag;
2420366f6083SPeter Grehan 	int reg;
2421366f6083SPeter Grehan 	int retval;
2422366f6083SPeter Grehan 
2423366f6083SPeter Grehan 	retval = ENOENT;
2424366f6083SPeter Grehan 	pptr = NULL;
2425366f6083SPeter Grehan 
2426366f6083SPeter Grehan 	switch (type) {
2427366f6083SPeter Grehan 	case VM_CAP_HALT_EXIT:
2428366f6083SPeter Grehan 		if (cap_halt_exit) {
2429366f6083SPeter Grehan 			retval = 0;
2430366f6083SPeter Grehan 			pptr = &vmx->cap[vcpu].proc_ctls;
2431366f6083SPeter Grehan 			baseval = *pptr;
2432366f6083SPeter Grehan 			flag = PROCBASED_HLT_EXITING;
2433366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
2434366f6083SPeter Grehan 		}
2435366f6083SPeter Grehan 		break;
2436366f6083SPeter Grehan 	case VM_CAP_MTRAP_EXIT:
2437366f6083SPeter Grehan 		if (cap_monitor_trap) {
2438366f6083SPeter Grehan 			retval = 0;
2439366f6083SPeter Grehan 			pptr = &vmx->cap[vcpu].proc_ctls;
2440366f6083SPeter Grehan 			baseval = *pptr;
2441366f6083SPeter Grehan 			flag = PROCBASED_MTF;
2442366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
2443366f6083SPeter Grehan 		}
2444366f6083SPeter Grehan 		break;
2445366f6083SPeter Grehan 	case VM_CAP_PAUSE_EXIT:
2446366f6083SPeter Grehan 		if (cap_pause_exit) {
2447366f6083SPeter Grehan 			retval = 0;
2448366f6083SPeter Grehan 			pptr = &vmx->cap[vcpu].proc_ctls;
2449366f6083SPeter Grehan 			baseval = *pptr;
2450366f6083SPeter Grehan 			flag = PROCBASED_PAUSE_EXITING;
2451366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
2452366f6083SPeter Grehan 		}
2453366f6083SPeter Grehan 		break;
2454366f6083SPeter Grehan 	case VM_CAP_UNRESTRICTED_GUEST:
2455366f6083SPeter Grehan 		if (cap_unrestricted_guest) {
2456366f6083SPeter Grehan 			retval = 0;
245749cc03daSNeel Natu 			pptr = &vmx->cap[vcpu].proc_ctls2;
245849cc03daSNeel Natu 			baseval = *pptr;
2459366f6083SPeter Grehan 			flag = PROCBASED2_UNRESTRICTED_GUEST;
2460366f6083SPeter Grehan 			reg = VMCS_SEC_PROC_BASED_CTLS;
2461366f6083SPeter Grehan 		}
2462366f6083SPeter Grehan 		break;
246349cc03daSNeel Natu 	case VM_CAP_ENABLE_INVPCID:
246449cc03daSNeel Natu 		if (cap_invpcid) {
246549cc03daSNeel Natu 			retval = 0;
246649cc03daSNeel Natu 			pptr = &vmx->cap[vcpu].proc_ctls2;
246749cc03daSNeel Natu 			baseval = *pptr;
246849cc03daSNeel Natu 			flag = PROCBASED2_ENABLE_INVPCID;
246949cc03daSNeel Natu 			reg = VMCS_SEC_PROC_BASED_CTLS;
247049cc03daSNeel Natu 		}
247149cc03daSNeel Natu 		break;
2472366f6083SPeter Grehan 	default:
2473366f6083SPeter Grehan 		break;
2474366f6083SPeter Grehan 	}
2475366f6083SPeter Grehan 
2476366f6083SPeter Grehan 	if (retval == 0) {
2477366f6083SPeter Grehan 		if (val) {
2478366f6083SPeter Grehan 			baseval |= flag;
2479366f6083SPeter Grehan 		} else {
2480366f6083SPeter Grehan 			baseval &= ~flag;
2481366f6083SPeter Grehan 		}
2482366f6083SPeter Grehan 		VMPTRLD(vmcs);
2483366f6083SPeter Grehan 		error = vmwrite(reg, baseval);
2484366f6083SPeter Grehan 		VMCLEAR(vmcs);
2485366f6083SPeter Grehan 
2486366f6083SPeter Grehan 		if (error) {
2487366f6083SPeter Grehan 			retval = error;
2488366f6083SPeter Grehan 		} else {
2489366f6083SPeter Grehan 			/*
2490366f6083SPeter Grehan 			 * Update optional stored flags, and record
2491366f6083SPeter Grehan 			 * setting
2492366f6083SPeter Grehan 			 */
2493366f6083SPeter Grehan 			if (pptr != NULL) {
2494366f6083SPeter Grehan 				*pptr = baseval;
2495366f6083SPeter Grehan 			}
2496366f6083SPeter Grehan 
2497366f6083SPeter Grehan 			if (val) {
2498366f6083SPeter Grehan 				vmx->cap[vcpu].set |= (1 << type);
2499366f6083SPeter Grehan 			} else {
2500366f6083SPeter Grehan 				vmx->cap[vcpu].set &= ~(1 << type);
2501366f6083SPeter Grehan 			}
2502366f6083SPeter Grehan 		}
2503366f6083SPeter Grehan 	}
2504366f6083SPeter Grehan 
2505366f6083SPeter Grehan         return (retval);
2506366f6083SPeter Grehan }
2507366f6083SPeter Grehan 
250888c4b8d1SNeel Natu struct vlapic_vtx {
250988c4b8d1SNeel Natu 	struct vlapic	vlapic;
2510176666c2SNeel Natu 	struct pir_desc	*pir_desc;
251130b94db8SNeel Natu 	struct vmx	*vmx;
251288c4b8d1SNeel Natu };
251388c4b8d1SNeel Natu 
251488c4b8d1SNeel Natu #define	VMX_CTR_PIR(vm, vcpuid, pir_desc, notify, vector, level, msg)	\
251588c4b8d1SNeel Natu do {									\
251688c4b8d1SNeel Natu 	VCPU_CTR2(vm, vcpuid, msg " assert %s-triggered vector %d",	\
251788c4b8d1SNeel Natu 	    level ? "level" : "edge", vector);				\
251888c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir0 0x%016lx", pir_desc->pir[0]);	\
251988c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir1 0x%016lx", pir_desc->pir[1]);	\
252088c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir2 0x%016lx", pir_desc->pir[2]);	\
252188c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir3 0x%016lx", pir_desc->pir[3]);	\
252288c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " notify: %s", notify ? "yes" : "no");\
252388c4b8d1SNeel Natu } while (0)
252488c4b8d1SNeel Natu 
252588c4b8d1SNeel Natu /*
252688c4b8d1SNeel Natu  * vlapic->ops handlers that utilize the APICv hardware assist described in
252788c4b8d1SNeel Natu  * Chapter 29 of the Intel SDM.
252888c4b8d1SNeel Natu  */
252988c4b8d1SNeel Natu static int
253088c4b8d1SNeel Natu vmx_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
253188c4b8d1SNeel Natu {
253288c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
253388c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
253488c4b8d1SNeel Natu 	uint64_t mask;
253588c4b8d1SNeel Natu 	int idx, notify;
253688c4b8d1SNeel Natu 
253788c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
2538176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
253988c4b8d1SNeel Natu 
254088c4b8d1SNeel Natu 	/*
254188c4b8d1SNeel Natu 	 * Keep track of interrupt requests in the PIR descriptor. This is
254288c4b8d1SNeel Natu 	 * because the virtual APIC page pointed to by the VMCS cannot be
254388c4b8d1SNeel Natu 	 * modified if the vcpu is running.
254488c4b8d1SNeel Natu 	 */
254588c4b8d1SNeel Natu 	idx = vector / 64;
254688c4b8d1SNeel Natu 	mask = 1UL << (vector % 64);
254788c4b8d1SNeel Natu 	atomic_set_long(&pir_desc->pir[idx], mask);
254888c4b8d1SNeel Natu 	notify = atomic_cmpset_long(&pir_desc->pending, 0, 1);
254988c4b8d1SNeel Natu 
255088c4b8d1SNeel Natu 	VMX_CTR_PIR(vlapic->vm, vlapic->vcpuid, pir_desc, notify, vector,
255188c4b8d1SNeel Natu 	    level, "vmx_set_intr_ready");
255288c4b8d1SNeel Natu 	return (notify);
255388c4b8d1SNeel Natu }
255488c4b8d1SNeel Natu 
255588c4b8d1SNeel Natu static int
255688c4b8d1SNeel Natu vmx_pending_intr(struct vlapic *vlapic, int *vecptr)
255788c4b8d1SNeel Natu {
255888c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
255988c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
256088c4b8d1SNeel Natu 	struct LAPIC *lapic;
256188c4b8d1SNeel Natu 	uint64_t pending, pirval;
256288c4b8d1SNeel Natu 	uint32_t ppr, vpr;
256388c4b8d1SNeel Natu 	int i;
256488c4b8d1SNeel Natu 
256588c4b8d1SNeel Natu 	/*
256688c4b8d1SNeel Natu 	 * This function is only expected to be called from the 'HLT' exit
256788c4b8d1SNeel Natu 	 * handler which does not care about the vector that is pending.
256888c4b8d1SNeel Natu 	 */
256988c4b8d1SNeel Natu 	KASSERT(vecptr == NULL, ("vmx_pending_intr: vecptr must be NULL"));
257088c4b8d1SNeel Natu 
257188c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
2572176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
257388c4b8d1SNeel Natu 
257488c4b8d1SNeel Natu 	pending = atomic_load_acq_long(&pir_desc->pending);
257588c4b8d1SNeel Natu 	if (!pending)
257688c4b8d1SNeel Natu 		return (0);	/* common case */
257788c4b8d1SNeel Natu 
257888c4b8d1SNeel Natu 	/*
257988c4b8d1SNeel Natu 	 * If there is an interrupt pending then it will be recognized only
258088c4b8d1SNeel Natu 	 * if its priority is greater than the processor priority.
258188c4b8d1SNeel Natu 	 *
258288c4b8d1SNeel Natu 	 * Special case: if the processor priority is zero then any pending
258388c4b8d1SNeel Natu 	 * interrupt will be recognized.
258488c4b8d1SNeel Natu 	 */
258588c4b8d1SNeel Natu 	lapic = vlapic->apic_page;
258688c4b8d1SNeel Natu 	ppr = lapic->ppr & 0xf0;
258788c4b8d1SNeel Natu 	if (ppr == 0)
258888c4b8d1SNeel Natu 		return (1);
258988c4b8d1SNeel Natu 
259088c4b8d1SNeel Natu 	VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "HLT with non-zero PPR %d",
259188c4b8d1SNeel Natu 	    lapic->ppr);
259288c4b8d1SNeel Natu 
259388c4b8d1SNeel Natu 	for (i = 3; i >= 0; i--) {
259488c4b8d1SNeel Natu 		pirval = pir_desc->pir[i];
259588c4b8d1SNeel Natu 		if (pirval != 0) {
259688c4b8d1SNeel Natu 			vpr = (i * 64 + flsl(pirval) - 1) & 0xf0;
259788c4b8d1SNeel Natu 			return (vpr > ppr);
259888c4b8d1SNeel Natu 		}
259988c4b8d1SNeel Natu 	}
260088c4b8d1SNeel Natu 	return (0);
260188c4b8d1SNeel Natu }
260288c4b8d1SNeel Natu 
260388c4b8d1SNeel Natu static void
260488c4b8d1SNeel Natu vmx_intr_accepted(struct vlapic *vlapic, int vector)
260588c4b8d1SNeel Natu {
260688c4b8d1SNeel Natu 
260788c4b8d1SNeel Natu 	panic("vmx_intr_accepted: not expected to be called");
260888c4b8d1SNeel Natu }
260988c4b8d1SNeel Natu 
2610176666c2SNeel Natu static void
261130b94db8SNeel Natu vmx_set_tmr(struct vlapic *vlapic, int vector, bool level)
261230b94db8SNeel Natu {
261330b94db8SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
261430b94db8SNeel Natu 	struct vmx *vmx;
261530b94db8SNeel Natu 	struct vmcs *vmcs;
261630b94db8SNeel Natu 	uint64_t mask, val;
261730b94db8SNeel Natu 
261830b94db8SNeel Natu 	KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
261930b94db8SNeel Natu 	KASSERT(!vcpu_is_running(vlapic->vm, vlapic->vcpuid, NULL),
262030b94db8SNeel Natu 	    ("vmx_set_tmr: vcpu cannot be running"));
262130b94db8SNeel Natu 
262230b94db8SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
262330b94db8SNeel Natu 	vmx = vlapic_vtx->vmx;
262430b94db8SNeel Natu 	vmcs = &vmx->vmcs[vlapic->vcpuid];
262530b94db8SNeel Natu 	mask = 1UL << (vector % 64);
262630b94db8SNeel Natu 
262730b94db8SNeel Natu 	VMPTRLD(vmcs);
262830b94db8SNeel Natu 	val = vmcs_read(VMCS_EOI_EXIT(vector));
262930b94db8SNeel Natu 	if (level)
263030b94db8SNeel Natu 		val |= mask;
263130b94db8SNeel Natu 	else
263230b94db8SNeel Natu 		val &= ~mask;
263330b94db8SNeel Natu 	vmcs_write(VMCS_EOI_EXIT(vector), val);
263430b94db8SNeel Natu 	VMCLEAR(vmcs);
263530b94db8SNeel Natu }
263630b94db8SNeel Natu 
263730b94db8SNeel Natu static void
2638176666c2SNeel Natu vmx_post_intr(struct vlapic *vlapic, int hostcpu)
2639176666c2SNeel Natu {
2640176666c2SNeel Natu 
2641176666c2SNeel Natu 	ipi_cpu(hostcpu, pirvec);
2642176666c2SNeel Natu }
2643176666c2SNeel Natu 
264488c4b8d1SNeel Natu /*
264588c4b8d1SNeel Natu  * Transfer the pending interrupts in the PIR descriptor to the IRR
264688c4b8d1SNeel Natu  * in the virtual APIC page.
264788c4b8d1SNeel Natu  */
264888c4b8d1SNeel Natu static void
264988c4b8d1SNeel Natu vmx_inject_pir(struct vlapic *vlapic)
265088c4b8d1SNeel Natu {
265188c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
265288c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
265388c4b8d1SNeel Natu 	struct LAPIC *lapic;
265488c4b8d1SNeel Natu 	uint64_t val, pirval;
265588c4b8d1SNeel Natu 	int rvi, pirbase;
265688c4b8d1SNeel Natu 	uint16_t intr_status_old, intr_status_new;
265788c4b8d1SNeel Natu 
265888c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
2659176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
266088c4b8d1SNeel Natu 	if (atomic_cmpset_long(&pir_desc->pending, 1, 0) == 0) {
266188c4b8d1SNeel Natu 		VCPU_CTR0(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
266288c4b8d1SNeel Natu 		    "no posted interrupt pending");
266388c4b8d1SNeel Natu 		return;
266488c4b8d1SNeel Natu 	}
266588c4b8d1SNeel Natu 
266688c4b8d1SNeel Natu 	pirval = 0;
266788c4b8d1SNeel Natu 	lapic = vlapic->apic_page;
266888c4b8d1SNeel Natu 
266988c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[0]);
267088c4b8d1SNeel Natu 	if (val != 0) {
267188c4b8d1SNeel Natu 		lapic->irr0 |= val;
267288c4b8d1SNeel Natu 		lapic->irr1 |= val >> 32;
267388c4b8d1SNeel Natu 		pirbase = 0;
267488c4b8d1SNeel Natu 		pirval = val;
267588c4b8d1SNeel Natu 	}
267688c4b8d1SNeel Natu 
267788c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[1]);
267888c4b8d1SNeel Natu 	if (val != 0) {
267988c4b8d1SNeel Natu 		lapic->irr2 |= val;
268088c4b8d1SNeel Natu 		lapic->irr3 |= val >> 32;
268188c4b8d1SNeel Natu 		pirbase = 64;
268288c4b8d1SNeel Natu 		pirval = val;
268388c4b8d1SNeel Natu 	}
268488c4b8d1SNeel Natu 
268588c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[2]);
268688c4b8d1SNeel Natu 	if (val != 0) {
268788c4b8d1SNeel Natu 		lapic->irr4 |= val;
268888c4b8d1SNeel Natu 		lapic->irr5 |= val >> 32;
268988c4b8d1SNeel Natu 		pirbase = 128;
269088c4b8d1SNeel Natu 		pirval = val;
269188c4b8d1SNeel Natu 	}
269288c4b8d1SNeel Natu 
269388c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[3]);
269488c4b8d1SNeel Natu 	if (val != 0) {
269588c4b8d1SNeel Natu 		lapic->irr6 |= val;
269688c4b8d1SNeel Natu 		lapic->irr7 |= val >> 32;
269788c4b8d1SNeel Natu 		pirbase = 192;
269888c4b8d1SNeel Natu 		pirval = val;
269988c4b8d1SNeel Natu 	}
270088c4b8d1SNeel Natu 	VLAPIC_CTR_IRR(vlapic, "vmx_inject_pir");
270188c4b8d1SNeel Natu 
270288c4b8d1SNeel Natu 	/*
270388c4b8d1SNeel Natu 	 * Update RVI so the processor can evaluate pending virtual
270488c4b8d1SNeel Natu 	 * interrupts on VM-entry.
270588c4b8d1SNeel Natu 	 */
270688c4b8d1SNeel Natu 	if (pirval != 0) {
270788c4b8d1SNeel Natu 		rvi = pirbase + flsl(pirval) - 1;
270888c4b8d1SNeel Natu 		intr_status_old = vmcs_read(VMCS_GUEST_INTR_STATUS);
270988c4b8d1SNeel Natu 		intr_status_new = (intr_status_old & 0xFF00) | rvi;
271088c4b8d1SNeel Natu 		if (intr_status_new > intr_status_old) {
271188c4b8d1SNeel Natu 			vmcs_write(VMCS_GUEST_INTR_STATUS, intr_status_new);
271288c4b8d1SNeel Natu 			VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
271388c4b8d1SNeel Natu 			    "guest_intr_status changed from 0x%04x to 0x%04x",
271488c4b8d1SNeel Natu 			    intr_status_old, intr_status_new);
271588c4b8d1SNeel Natu 		}
271688c4b8d1SNeel Natu 	}
271788c4b8d1SNeel Natu }
271888c4b8d1SNeel Natu 
2719de5ea6b6SNeel Natu static struct vlapic *
2720de5ea6b6SNeel Natu vmx_vlapic_init(void *arg, int vcpuid)
2721de5ea6b6SNeel Natu {
2722de5ea6b6SNeel Natu 	struct vmx *vmx;
2723de5ea6b6SNeel Natu 	struct vlapic *vlapic;
2724176666c2SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
2725de5ea6b6SNeel Natu 
2726de5ea6b6SNeel Natu 	vmx = arg;
2727de5ea6b6SNeel Natu 
272888c4b8d1SNeel Natu 	vlapic = malloc(sizeof(struct vlapic_vtx), M_VLAPIC, M_WAITOK | M_ZERO);
2729de5ea6b6SNeel Natu 	vlapic->vm = vmx->vm;
2730de5ea6b6SNeel Natu 	vlapic->vcpuid = vcpuid;
2731de5ea6b6SNeel Natu 	vlapic->apic_page = (struct LAPIC *)&vmx->apic_page[vcpuid];
2732de5ea6b6SNeel Natu 
2733176666c2SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
2734176666c2SNeel Natu 	vlapic_vtx->pir_desc = &vmx->pir_desc[vcpuid];
273530b94db8SNeel Natu 	vlapic_vtx->vmx = vmx;
2736176666c2SNeel Natu 
273788c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
273888c4b8d1SNeel Natu 		vlapic->ops.set_intr_ready = vmx_set_intr_ready;
273988c4b8d1SNeel Natu 		vlapic->ops.pending_intr = vmx_pending_intr;
274088c4b8d1SNeel Natu 		vlapic->ops.intr_accepted = vmx_intr_accepted;
274130b94db8SNeel Natu 		vlapic->ops.set_tmr = vmx_set_tmr;
274288c4b8d1SNeel Natu 	}
274388c4b8d1SNeel Natu 
2744176666c2SNeel Natu 	if (posted_interrupts)
2745176666c2SNeel Natu 		vlapic->ops.post_intr = vmx_post_intr;
2746176666c2SNeel Natu 
2747de5ea6b6SNeel Natu 	vlapic_init(vlapic);
2748de5ea6b6SNeel Natu 
2749de5ea6b6SNeel Natu 	return (vlapic);
2750de5ea6b6SNeel Natu }
2751de5ea6b6SNeel Natu 
2752de5ea6b6SNeel Natu static void
2753de5ea6b6SNeel Natu vmx_vlapic_cleanup(void *arg, struct vlapic *vlapic)
2754de5ea6b6SNeel Natu {
2755de5ea6b6SNeel Natu 
2756de5ea6b6SNeel Natu 	vlapic_cleanup(vlapic);
2757de5ea6b6SNeel Natu 	free(vlapic, M_VLAPIC);
2758de5ea6b6SNeel Natu }
2759de5ea6b6SNeel Natu 
2760366f6083SPeter Grehan struct vmm_ops vmm_ops_intel = {
2761366f6083SPeter Grehan 	vmx_init,
2762366f6083SPeter Grehan 	vmx_cleanup,
276363e62d39SJohn Baldwin 	vmx_restore,
2764366f6083SPeter Grehan 	vmx_vminit,
2765366f6083SPeter Grehan 	vmx_run,
2766366f6083SPeter Grehan 	vmx_vmcleanup,
2767366f6083SPeter Grehan 	vmx_getreg,
2768366f6083SPeter Grehan 	vmx_setreg,
2769366f6083SPeter Grehan 	vmx_getdesc,
2770366f6083SPeter Grehan 	vmx_setdesc,
2771366f6083SPeter Grehan 	vmx_inject,
2772366f6083SPeter Grehan 	vmx_getcap,
2773318224bbSNeel Natu 	vmx_setcap,
2774318224bbSNeel Natu 	ept_vmspace_alloc,
2775318224bbSNeel Natu 	ept_vmspace_free,
2776de5ea6b6SNeel Natu 	vmx_vlapic_init,
2777de5ea6b6SNeel Natu 	vmx_vlapic_cleanup,
2778366f6083SPeter Grehan };
2779