xref: /freebsd/sys/amd64/vmm/intel/vmx.c (revision 30b94db8c0a6d41b4f8c0a9b55d2007b6373288a)
1366f6083SPeter Grehan /*-
2366f6083SPeter Grehan  * Copyright (c) 2011 NetApp, Inc.
3366f6083SPeter Grehan  * All rights reserved.
4366f6083SPeter Grehan  *
5366f6083SPeter Grehan  * Redistribution and use in source and binary forms, with or without
6366f6083SPeter Grehan  * modification, are permitted provided that the following conditions
7366f6083SPeter Grehan  * are met:
8366f6083SPeter Grehan  * 1. Redistributions of source code must retain the above copyright
9366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer.
10366f6083SPeter Grehan  * 2. Redistributions in binary form must reproduce the above copyright
11366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer in the
12366f6083SPeter Grehan  *    documentation and/or other materials provided with the distribution.
13366f6083SPeter Grehan  *
14366f6083SPeter Grehan  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
15366f6083SPeter Grehan  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16366f6083SPeter Grehan  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17366f6083SPeter Grehan  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
18366f6083SPeter Grehan  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19366f6083SPeter Grehan  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20366f6083SPeter Grehan  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21366f6083SPeter Grehan  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22366f6083SPeter Grehan  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23366f6083SPeter Grehan  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24366f6083SPeter Grehan  * SUCH DAMAGE.
25366f6083SPeter Grehan  *
26366f6083SPeter Grehan  * $FreeBSD$
27366f6083SPeter Grehan  */
28366f6083SPeter Grehan 
29366f6083SPeter Grehan #include <sys/cdefs.h>
30366f6083SPeter Grehan __FBSDID("$FreeBSD$");
31366f6083SPeter Grehan 
32366f6083SPeter Grehan #include <sys/param.h>
33366f6083SPeter Grehan #include <sys/systm.h>
34366f6083SPeter Grehan #include <sys/smp.h>
35366f6083SPeter Grehan #include <sys/kernel.h>
36366f6083SPeter Grehan #include <sys/malloc.h>
37366f6083SPeter Grehan #include <sys/pcpu.h>
38366f6083SPeter Grehan #include <sys/proc.h>
393565b59eSNeel Natu #include <sys/sysctl.h>
40366f6083SPeter Grehan 
41366f6083SPeter Grehan #include <vm/vm.h>
42366f6083SPeter Grehan #include <vm/pmap.h>
43366f6083SPeter Grehan 
44366f6083SPeter Grehan #include <machine/psl.h>
45366f6083SPeter Grehan #include <machine/cpufunc.h>
468b287612SJohn Baldwin #include <machine/md_var.h>
47366f6083SPeter Grehan #include <machine/segments.h>
48176666c2SNeel Natu #include <machine/smp.h>
49608f97c3SPeter Grehan #include <machine/specialreg.h>
50366f6083SPeter Grehan #include <machine/vmparam.h>
51366f6083SPeter Grehan 
52366f6083SPeter Grehan #include <machine/vmm.h>
53b01c2033SNeel Natu #include "vmm_host.h"
54176666c2SNeel Natu #include "vmm_ipi.h"
55366f6083SPeter Grehan #include "vmm_msr.h"
56366f6083SPeter Grehan #include "vmm_ktr.h"
57366f6083SPeter Grehan #include "vmm_stat.h"
58de5ea6b6SNeel Natu #include "vlapic.h"
59de5ea6b6SNeel Natu #include "vlapic_priv.h"
60366f6083SPeter Grehan 
61366f6083SPeter Grehan #include "vmx_msr.h"
62366f6083SPeter Grehan #include "ept.h"
63366f6083SPeter Grehan #include "vmx_cpufunc.h"
64366f6083SPeter Grehan #include "vmx.h"
65366f6083SPeter Grehan #include "x86.h"
66366f6083SPeter Grehan #include "vmx_controls.h"
67366f6083SPeter Grehan 
68366f6083SPeter Grehan #define	PINBASED_CTLS_ONE_SETTING					\
69366f6083SPeter Grehan 	(PINBASED_EXTINT_EXITING	|				\
70366f6083SPeter Grehan 	 PINBASED_NMI_EXITING		|				\
71366f6083SPeter Grehan 	 PINBASED_VIRTUAL_NMI)
72366f6083SPeter Grehan #define	PINBASED_CTLS_ZERO_SETTING	0
73366f6083SPeter Grehan 
74366f6083SPeter Grehan #define PROCBASED_CTLS_WINDOW_SETTING					\
75366f6083SPeter Grehan 	(PROCBASED_INT_WINDOW_EXITING	|				\
76366f6083SPeter Grehan 	 PROCBASED_NMI_WINDOW_EXITING)
77366f6083SPeter Grehan 
78366f6083SPeter Grehan #define	PROCBASED_CTLS_ONE_SETTING 					\
79366f6083SPeter Grehan 	(PROCBASED_SECONDARY_CONTROLS	|				\
80366f6083SPeter Grehan 	 PROCBASED_IO_EXITING		|				\
81366f6083SPeter Grehan 	 PROCBASED_MSR_BITMAPS		|				\
82366f6083SPeter Grehan 	 PROCBASED_CTLS_WINDOW_SETTING)
83366f6083SPeter Grehan #define	PROCBASED_CTLS_ZERO_SETTING	\
84366f6083SPeter Grehan 	(PROCBASED_CR3_LOAD_EXITING |	\
85366f6083SPeter Grehan 	PROCBASED_CR3_STORE_EXITING |	\
86366f6083SPeter Grehan 	PROCBASED_IO_BITMAPS)
87366f6083SPeter Grehan 
88366f6083SPeter Grehan #define	PROCBASED_CTLS2_ONE_SETTING	PROCBASED2_ENABLE_EPT
89366f6083SPeter Grehan #define	PROCBASED_CTLS2_ZERO_SETTING	0
90366f6083SPeter Grehan 
91608f97c3SPeter Grehan #define VM_EXIT_CTLS_ONE_SETTING_NO_PAT					\
92366f6083SPeter Grehan 	(VM_EXIT_HOST_LMA			|			\
93366f6083SPeter Grehan 	VM_EXIT_SAVE_EFER			|			\
94366f6083SPeter Grehan 	VM_EXIT_LOAD_EFER)
95608f97c3SPeter Grehan 
96608f97c3SPeter Grehan #define	VM_EXIT_CTLS_ONE_SETTING					\
97608f97c3SPeter Grehan 	(VM_EXIT_CTLS_ONE_SETTING_NO_PAT       	|			\
98f7d47425SNeel Natu 	VM_EXIT_ACKNOWLEDGE_INTERRUPT		|			\
99608f97c3SPeter Grehan 	VM_EXIT_SAVE_PAT			|			\
100608f97c3SPeter Grehan 	VM_EXIT_LOAD_PAT)
101366f6083SPeter Grehan #define	VM_EXIT_CTLS_ZERO_SETTING	VM_EXIT_SAVE_DEBUG_CONTROLS
102366f6083SPeter Grehan 
103608f97c3SPeter Grehan #define	VM_ENTRY_CTLS_ONE_SETTING_NO_PAT	VM_ENTRY_LOAD_EFER
104608f97c3SPeter Grehan 
105366f6083SPeter Grehan #define	VM_ENTRY_CTLS_ONE_SETTING					\
106608f97c3SPeter Grehan 	(VM_ENTRY_CTLS_ONE_SETTING_NO_PAT     	|			\
107608f97c3SPeter Grehan 	VM_ENTRY_LOAD_PAT)
108366f6083SPeter Grehan #define	VM_ENTRY_CTLS_ZERO_SETTING					\
109366f6083SPeter Grehan 	(VM_ENTRY_LOAD_DEBUG_CONTROLS		|			\
110366f6083SPeter Grehan 	VM_ENTRY_INTO_SMM			|			\
111366f6083SPeter Grehan 	VM_ENTRY_DEACTIVATE_DUAL_MONITOR)
112366f6083SPeter Grehan 
113366f6083SPeter Grehan #define	guest_msr_rw(vmx, msr) \
114366f6083SPeter Grehan 	msr_bitmap_change_access((vmx)->msr_bitmap, (msr), MSR_BITMAP_ACCESS_RW)
115366f6083SPeter Grehan 
116366f6083SPeter Grehan #define	HANDLED		1
117366f6083SPeter Grehan #define	UNHANDLED	0
118366f6083SPeter Grehan 
119de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VMX, "vmx", "vmx");
120de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
121366f6083SPeter Grehan 
1223565b59eSNeel Natu SYSCTL_DECL(_hw_vmm);
1233565b59eSNeel Natu SYSCTL_NODE(_hw_vmm, OID_AUTO, vmx, CTLFLAG_RW, NULL, NULL);
1243565b59eSNeel Natu 
125b3996dd4SJohn Baldwin int vmxon_enabled[MAXCPU];
126366f6083SPeter Grehan static char vmxon_region[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE);
127366f6083SPeter Grehan 
128366f6083SPeter Grehan static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2;
129366f6083SPeter Grehan static uint32_t exit_ctls, entry_ctls;
130366f6083SPeter Grehan 
131366f6083SPeter Grehan static uint64_t cr0_ones_mask, cr0_zeros_mask;
1323565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_ones_mask, CTLFLAG_RD,
1333565b59eSNeel Natu 	     &cr0_ones_mask, 0, NULL);
1343565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_zeros_mask, CTLFLAG_RD,
1353565b59eSNeel Natu 	     &cr0_zeros_mask, 0, NULL);
1363565b59eSNeel Natu 
137366f6083SPeter Grehan static uint64_t cr4_ones_mask, cr4_zeros_mask;
1383565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_ones_mask, CTLFLAG_RD,
1393565b59eSNeel Natu 	     &cr4_ones_mask, 0, NULL);
1403565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_zeros_mask, CTLFLAG_RD,
1413565b59eSNeel Natu 	     &cr4_zeros_mask, 0, NULL);
142366f6083SPeter Grehan 
143608f97c3SPeter Grehan static int vmx_no_patmsr;
144608f97c3SPeter Grehan 
1453565b59eSNeel Natu static int vmx_initialized;
1463565b59eSNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, initialized, CTLFLAG_RD,
1473565b59eSNeel Natu 	   &vmx_initialized, 0, "Intel VMX initialized");
1483565b59eSNeel Natu 
149366f6083SPeter Grehan /*
150366f6083SPeter Grehan  * Optional capabilities
151366f6083SPeter Grehan  */
152366f6083SPeter Grehan static int cap_halt_exit;
153366f6083SPeter Grehan static int cap_pause_exit;
154366f6083SPeter Grehan static int cap_unrestricted_guest;
155366f6083SPeter Grehan static int cap_monitor_trap;
15649cc03daSNeel Natu static int cap_invpcid;
157366f6083SPeter Grehan 
15888c4b8d1SNeel Natu static int virtual_interrupt_delivery;
15988c4b8d1SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, virtual_interrupt_delivery, CTLFLAG_RD,
16088c4b8d1SNeel Natu     &virtual_interrupt_delivery, 0, "APICv virtual interrupt delivery support");
16188c4b8d1SNeel Natu 
162176666c2SNeel Natu static int posted_interrupts;
163176666c2SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupts, CTLFLAG_RD,
164176666c2SNeel Natu     &posted_interrupts, 0, "APICv posted interrupt support");
165176666c2SNeel Natu 
166176666c2SNeel Natu static int pirvec;
167176666c2SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupt_vector, CTLFLAG_RD,
168176666c2SNeel Natu     &pirvec, 0, "APICv posted interrupt vector");
169176666c2SNeel Natu 
17045e51299SNeel Natu static struct unrhdr *vpid_unr;
17145e51299SNeel Natu static u_int vpid_alloc_failed;
17245e51299SNeel Natu SYSCTL_UINT(_hw_vmm_vmx, OID_AUTO, vpid_alloc_failed, CTLFLAG_RD,
17345e51299SNeel Natu 	    &vpid_alloc_failed, 0, NULL);
17445e51299SNeel Natu 
17588c4b8d1SNeel Natu /*
17688c4b8d1SNeel Natu  * Use the last page below 4GB as the APIC access address. This address is
17788c4b8d1SNeel Natu  * occupied by the boot firmware so it is guaranteed that it will not conflict
17888c4b8d1SNeel Natu  * with a page in system memory.
17988c4b8d1SNeel Natu  */
18088c4b8d1SNeel Natu #define	APIC_ACCESS_ADDRESS	0xFFFFF000
18188c4b8d1SNeel Natu 
18288c4b8d1SNeel Natu static void vmx_inject_pir(struct vlapic *vlapic);
18388c4b8d1SNeel Natu 
184366f6083SPeter Grehan #ifdef KTR
185366f6083SPeter Grehan static const char *
186366f6083SPeter Grehan exit_reason_to_str(int reason)
187366f6083SPeter Grehan {
188366f6083SPeter Grehan 	static char reasonbuf[32];
189366f6083SPeter Grehan 
190366f6083SPeter Grehan 	switch (reason) {
191366f6083SPeter Grehan 	case EXIT_REASON_EXCEPTION:
192366f6083SPeter Grehan 		return "exception";
193366f6083SPeter Grehan 	case EXIT_REASON_EXT_INTR:
194366f6083SPeter Grehan 		return "extint";
195366f6083SPeter Grehan 	case EXIT_REASON_TRIPLE_FAULT:
196366f6083SPeter Grehan 		return "triplefault";
197366f6083SPeter Grehan 	case EXIT_REASON_INIT:
198366f6083SPeter Grehan 		return "init";
199366f6083SPeter Grehan 	case EXIT_REASON_SIPI:
200366f6083SPeter Grehan 		return "sipi";
201366f6083SPeter Grehan 	case EXIT_REASON_IO_SMI:
202366f6083SPeter Grehan 		return "iosmi";
203366f6083SPeter Grehan 	case EXIT_REASON_SMI:
204366f6083SPeter Grehan 		return "smi";
205366f6083SPeter Grehan 	case EXIT_REASON_INTR_WINDOW:
206366f6083SPeter Grehan 		return "intrwindow";
207366f6083SPeter Grehan 	case EXIT_REASON_NMI_WINDOW:
208366f6083SPeter Grehan 		return "nmiwindow";
209366f6083SPeter Grehan 	case EXIT_REASON_TASK_SWITCH:
210366f6083SPeter Grehan 		return "taskswitch";
211366f6083SPeter Grehan 	case EXIT_REASON_CPUID:
212366f6083SPeter Grehan 		return "cpuid";
213366f6083SPeter Grehan 	case EXIT_REASON_GETSEC:
214366f6083SPeter Grehan 		return "getsec";
215366f6083SPeter Grehan 	case EXIT_REASON_HLT:
216366f6083SPeter Grehan 		return "hlt";
217366f6083SPeter Grehan 	case EXIT_REASON_INVD:
218366f6083SPeter Grehan 		return "invd";
219366f6083SPeter Grehan 	case EXIT_REASON_INVLPG:
220366f6083SPeter Grehan 		return "invlpg";
221366f6083SPeter Grehan 	case EXIT_REASON_RDPMC:
222366f6083SPeter Grehan 		return "rdpmc";
223366f6083SPeter Grehan 	case EXIT_REASON_RDTSC:
224366f6083SPeter Grehan 		return "rdtsc";
225366f6083SPeter Grehan 	case EXIT_REASON_RSM:
226366f6083SPeter Grehan 		return "rsm";
227366f6083SPeter Grehan 	case EXIT_REASON_VMCALL:
228366f6083SPeter Grehan 		return "vmcall";
229366f6083SPeter Grehan 	case EXIT_REASON_VMCLEAR:
230366f6083SPeter Grehan 		return "vmclear";
231366f6083SPeter Grehan 	case EXIT_REASON_VMLAUNCH:
232366f6083SPeter Grehan 		return "vmlaunch";
233366f6083SPeter Grehan 	case EXIT_REASON_VMPTRLD:
234366f6083SPeter Grehan 		return "vmptrld";
235366f6083SPeter Grehan 	case EXIT_REASON_VMPTRST:
236366f6083SPeter Grehan 		return "vmptrst";
237366f6083SPeter Grehan 	case EXIT_REASON_VMREAD:
238366f6083SPeter Grehan 		return "vmread";
239366f6083SPeter Grehan 	case EXIT_REASON_VMRESUME:
240366f6083SPeter Grehan 		return "vmresume";
241366f6083SPeter Grehan 	case EXIT_REASON_VMWRITE:
242366f6083SPeter Grehan 		return "vmwrite";
243366f6083SPeter Grehan 	case EXIT_REASON_VMXOFF:
244366f6083SPeter Grehan 		return "vmxoff";
245366f6083SPeter Grehan 	case EXIT_REASON_VMXON:
246366f6083SPeter Grehan 		return "vmxon";
247366f6083SPeter Grehan 	case EXIT_REASON_CR_ACCESS:
248366f6083SPeter Grehan 		return "craccess";
249366f6083SPeter Grehan 	case EXIT_REASON_DR_ACCESS:
250366f6083SPeter Grehan 		return "draccess";
251366f6083SPeter Grehan 	case EXIT_REASON_INOUT:
252366f6083SPeter Grehan 		return "inout";
253366f6083SPeter Grehan 	case EXIT_REASON_RDMSR:
254366f6083SPeter Grehan 		return "rdmsr";
255366f6083SPeter Grehan 	case EXIT_REASON_WRMSR:
256366f6083SPeter Grehan 		return "wrmsr";
257366f6083SPeter Grehan 	case EXIT_REASON_INVAL_VMCS:
258366f6083SPeter Grehan 		return "invalvmcs";
259366f6083SPeter Grehan 	case EXIT_REASON_INVAL_MSR:
260366f6083SPeter Grehan 		return "invalmsr";
261366f6083SPeter Grehan 	case EXIT_REASON_MWAIT:
262366f6083SPeter Grehan 		return "mwait";
263366f6083SPeter Grehan 	case EXIT_REASON_MTF:
264366f6083SPeter Grehan 		return "mtf";
265366f6083SPeter Grehan 	case EXIT_REASON_MONITOR:
266366f6083SPeter Grehan 		return "monitor";
267366f6083SPeter Grehan 	case EXIT_REASON_PAUSE:
268366f6083SPeter Grehan 		return "pause";
269366f6083SPeter Grehan 	case EXIT_REASON_MCE:
270366f6083SPeter Grehan 		return "mce";
271366f6083SPeter Grehan 	case EXIT_REASON_TPR:
272366f6083SPeter Grehan 		return "tpr";
27388c4b8d1SNeel Natu 	case EXIT_REASON_APIC_ACCESS:
27488c4b8d1SNeel Natu 		return "apic-access";
275366f6083SPeter Grehan 	case EXIT_REASON_GDTR_IDTR:
276366f6083SPeter Grehan 		return "gdtridtr";
277366f6083SPeter Grehan 	case EXIT_REASON_LDTR_TR:
278366f6083SPeter Grehan 		return "ldtrtr";
279366f6083SPeter Grehan 	case EXIT_REASON_EPT_FAULT:
280366f6083SPeter Grehan 		return "eptfault";
281366f6083SPeter Grehan 	case EXIT_REASON_EPT_MISCONFIG:
282366f6083SPeter Grehan 		return "eptmisconfig";
283366f6083SPeter Grehan 	case EXIT_REASON_INVEPT:
284366f6083SPeter Grehan 		return "invept";
285366f6083SPeter Grehan 	case EXIT_REASON_RDTSCP:
286366f6083SPeter Grehan 		return "rdtscp";
287366f6083SPeter Grehan 	case EXIT_REASON_VMX_PREEMPT:
288366f6083SPeter Grehan 		return "vmxpreempt";
289366f6083SPeter Grehan 	case EXIT_REASON_INVVPID:
290366f6083SPeter Grehan 		return "invvpid";
291366f6083SPeter Grehan 	case EXIT_REASON_WBINVD:
292366f6083SPeter Grehan 		return "wbinvd";
293366f6083SPeter Grehan 	case EXIT_REASON_XSETBV:
294366f6083SPeter Grehan 		return "xsetbv";
29588c4b8d1SNeel Natu 	case EXIT_REASON_APIC_WRITE:
29688c4b8d1SNeel Natu 		return "apic-write";
297366f6083SPeter Grehan 	default:
298366f6083SPeter Grehan 		snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason);
299366f6083SPeter Grehan 		return (reasonbuf);
300366f6083SPeter Grehan 	}
301366f6083SPeter Grehan }
302366f6083SPeter Grehan #endif	/* KTR */
303366f6083SPeter Grehan 
304366f6083SPeter Grehan u_long
305366f6083SPeter Grehan vmx_fix_cr0(u_long cr0)
306366f6083SPeter Grehan {
307366f6083SPeter Grehan 
308366f6083SPeter Grehan 	return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask);
309366f6083SPeter Grehan }
310366f6083SPeter Grehan 
311366f6083SPeter Grehan u_long
312366f6083SPeter Grehan vmx_fix_cr4(u_long cr4)
313366f6083SPeter Grehan {
314366f6083SPeter Grehan 
315366f6083SPeter Grehan 	return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask);
316366f6083SPeter Grehan }
317366f6083SPeter Grehan 
318366f6083SPeter Grehan static void
31945e51299SNeel Natu vpid_free(int vpid)
32045e51299SNeel Natu {
32145e51299SNeel Natu 	if (vpid < 0 || vpid > 0xffff)
32245e51299SNeel Natu 		panic("vpid_free: invalid vpid %d", vpid);
32345e51299SNeel Natu 
32445e51299SNeel Natu 	/*
32545e51299SNeel Natu 	 * VPIDs [0,VM_MAXCPU] are special and are not allocated from
32645e51299SNeel Natu 	 * the unit number allocator.
32745e51299SNeel Natu 	 */
32845e51299SNeel Natu 
32945e51299SNeel Natu 	if (vpid > VM_MAXCPU)
33045e51299SNeel Natu 		free_unr(vpid_unr, vpid);
33145e51299SNeel Natu }
33245e51299SNeel Natu 
33345e51299SNeel Natu static void
33445e51299SNeel Natu vpid_alloc(uint16_t *vpid, int num)
33545e51299SNeel Natu {
33645e51299SNeel Natu 	int i, x;
33745e51299SNeel Natu 
33845e51299SNeel Natu 	if (num <= 0 || num > VM_MAXCPU)
33945e51299SNeel Natu 		panic("invalid number of vpids requested: %d", num);
34045e51299SNeel Natu 
34145e51299SNeel Natu 	/*
34245e51299SNeel Natu 	 * If the "enable vpid" execution control is not enabled then the
34345e51299SNeel Natu 	 * VPID is required to be 0 for all vcpus.
34445e51299SNeel Natu 	 */
34545e51299SNeel Natu 	if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) == 0) {
34645e51299SNeel Natu 		for (i = 0; i < num; i++)
34745e51299SNeel Natu 			vpid[i] = 0;
34845e51299SNeel Natu 		return;
34945e51299SNeel Natu 	}
35045e51299SNeel Natu 
35145e51299SNeel Natu 	/*
35245e51299SNeel Natu 	 * Allocate a unique VPID for each vcpu from the unit number allocator.
35345e51299SNeel Natu 	 */
35445e51299SNeel Natu 	for (i = 0; i < num; i++) {
35545e51299SNeel Natu 		x = alloc_unr(vpid_unr);
35645e51299SNeel Natu 		if (x == -1)
35745e51299SNeel Natu 			break;
35845e51299SNeel Natu 		else
35945e51299SNeel Natu 			vpid[i] = x;
36045e51299SNeel Natu 	}
36145e51299SNeel Natu 
36245e51299SNeel Natu 	if (i < num) {
36345e51299SNeel Natu 		atomic_add_int(&vpid_alloc_failed, 1);
36445e51299SNeel Natu 
36545e51299SNeel Natu 		/*
36645e51299SNeel Natu 		 * If the unit number allocator does not have enough unique
36745e51299SNeel Natu 		 * VPIDs then we need to allocate from the [1,VM_MAXCPU] range.
36845e51299SNeel Natu 		 *
36945e51299SNeel Natu 		 * These VPIDs are not be unique across VMs but this does not
37045e51299SNeel Natu 		 * affect correctness because the combined mappings are also
37145e51299SNeel Natu 		 * tagged with the EP4TA which is unique for each VM.
37245e51299SNeel Natu 		 *
37345e51299SNeel Natu 		 * It is still sub-optimal because the invvpid will invalidate
37445e51299SNeel Natu 		 * combined mappings for a particular VPID across all EP4TAs.
37545e51299SNeel Natu 		 */
37645e51299SNeel Natu 		while (i-- > 0)
37745e51299SNeel Natu 			vpid_free(vpid[i]);
37845e51299SNeel Natu 
37945e51299SNeel Natu 		for (i = 0; i < num; i++)
38045e51299SNeel Natu 			vpid[i] = i + 1;
38145e51299SNeel Natu 	}
38245e51299SNeel Natu }
38345e51299SNeel Natu 
38445e51299SNeel Natu static void
38545e51299SNeel Natu vpid_init(void)
38645e51299SNeel Natu {
38745e51299SNeel Natu 	/*
38845e51299SNeel Natu 	 * VPID 0 is required when the "enable VPID" execution control is
38945e51299SNeel Natu 	 * disabled.
39045e51299SNeel Natu 	 *
39145e51299SNeel Natu 	 * VPIDs [1,VM_MAXCPU] are used as the "overflow namespace" when the
39245e51299SNeel Natu 	 * unit number allocator does not have sufficient unique VPIDs to
39345e51299SNeel Natu 	 * satisfy the allocation.
39445e51299SNeel Natu 	 *
39545e51299SNeel Natu 	 * The remaining VPIDs are managed by the unit number allocator.
39645e51299SNeel Natu 	 */
39745e51299SNeel Natu 	vpid_unr = new_unrhdr(VM_MAXCPU + 1, 0xffff, NULL);
39845e51299SNeel Natu }
39945e51299SNeel Natu 
40045e51299SNeel Natu static void
401366f6083SPeter Grehan msr_save_area_init(struct msr_entry *g_area, int *g_count)
402366f6083SPeter Grehan {
403366f6083SPeter Grehan 	int cnt;
404366f6083SPeter Grehan 
405366f6083SPeter Grehan 	static struct msr_entry guest_msrs[] = {
406366f6083SPeter Grehan 		{ MSR_KGSBASE, 0, 0 },
407366f6083SPeter Grehan 	};
408366f6083SPeter Grehan 
409366f6083SPeter Grehan 	cnt = sizeof(guest_msrs) / sizeof(guest_msrs[0]);
410366f6083SPeter Grehan 	if (cnt > GUEST_MSR_MAX_ENTRIES)
411366f6083SPeter Grehan 		panic("guest msr save area overrun");
412366f6083SPeter Grehan 	bcopy(guest_msrs, g_area, sizeof(guest_msrs));
413366f6083SPeter Grehan 	*g_count = cnt;
414366f6083SPeter Grehan }
415366f6083SPeter Grehan 
416366f6083SPeter Grehan static void
417366f6083SPeter Grehan vmx_disable(void *arg __unused)
418366f6083SPeter Grehan {
419366f6083SPeter Grehan 	struct invvpid_desc invvpid_desc = { 0 };
420366f6083SPeter Grehan 	struct invept_desc invept_desc = { 0 };
421366f6083SPeter Grehan 
422366f6083SPeter Grehan 	if (vmxon_enabled[curcpu]) {
423366f6083SPeter Grehan 		/*
424366f6083SPeter Grehan 		 * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b.
425366f6083SPeter Grehan 		 *
426366f6083SPeter Grehan 		 * VMXON or VMXOFF are not required to invalidate any TLB
427366f6083SPeter Grehan 		 * caching structures. This prevents potential retention of
428366f6083SPeter Grehan 		 * cached information in the TLB between distinct VMX episodes.
429366f6083SPeter Grehan 		 */
430366f6083SPeter Grehan 		invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc);
431366f6083SPeter Grehan 		invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc);
432366f6083SPeter Grehan 		vmxoff();
433366f6083SPeter Grehan 	}
434366f6083SPeter Grehan 	load_cr4(rcr4() & ~CR4_VMXE);
435366f6083SPeter Grehan }
436366f6083SPeter Grehan 
437366f6083SPeter Grehan static int
438366f6083SPeter Grehan vmx_cleanup(void)
439366f6083SPeter Grehan {
440366f6083SPeter Grehan 
441176666c2SNeel Natu 	if (pirvec != 0)
442176666c2SNeel Natu 		vmm_ipi_free(pirvec);
443176666c2SNeel Natu 
44445e51299SNeel Natu 	if (vpid_unr != NULL) {
44545e51299SNeel Natu 		delete_unrhdr(vpid_unr);
44645e51299SNeel Natu 		vpid_unr = NULL;
44745e51299SNeel Natu 	}
44845e51299SNeel Natu 
449366f6083SPeter Grehan 	smp_rendezvous(NULL, vmx_disable, NULL, NULL);
450366f6083SPeter Grehan 
451366f6083SPeter Grehan 	return (0);
452366f6083SPeter Grehan }
453366f6083SPeter Grehan 
454366f6083SPeter Grehan static void
455366f6083SPeter Grehan vmx_enable(void *arg __unused)
456366f6083SPeter Grehan {
457366f6083SPeter Grehan 	int error;
458366f6083SPeter Grehan 
459366f6083SPeter Grehan 	load_cr4(rcr4() | CR4_VMXE);
460366f6083SPeter Grehan 
461366f6083SPeter Grehan 	*(uint32_t *)vmxon_region[curcpu] = vmx_revision();
462366f6083SPeter Grehan 	error = vmxon(vmxon_region[curcpu]);
463366f6083SPeter Grehan 	if (error == 0)
464366f6083SPeter Grehan 		vmxon_enabled[curcpu] = 1;
465366f6083SPeter Grehan }
466366f6083SPeter Grehan 
46763e62d39SJohn Baldwin static void
46863e62d39SJohn Baldwin vmx_restore(void)
46963e62d39SJohn Baldwin {
47063e62d39SJohn Baldwin 
47163e62d39SJohn Baldwin 	if (vmxon_enabled[curcpu])
47263e62d39SJohn Baldwin 		vmxon(vmxon_region[curcpu]);
47363e62d39SJohn Baldwin }
47463e62d39SJohn Baldwin 
475366f6083SPeter Grehan static int
476add611fdSNeel Natu vmx_init(int ipinum)
477366f6083SPeter Grehan {
47888c4b8d1SNeel Natu 	int error, use_tpr_shadow;
4794bff7fadSNeel Natu 	uint64_t fixed0, fixed1, feature_control;
48088c4b8d1SNeel Natu 	uint32_t tmp, procbased2_vid_bits;
481366f6083SPeter Grehan 
482366f6083SPeter Grehan 	/* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */
4838b287612SJohn Baldwin 	if (!(cpu_feature2 & CPUID2_VMX)) {
484366f6083SPeter Grehan 		printf("vmx_init: processor does not support VMX operation\n");
485366f6083SPeter Grehan 		return (ENXIO);
486366f6083SPeter Grehan 	}
487366f6083SPeter Grehan 
4884bff7fadSNeel Natu 	/*
4894bff7fadSNeel Natu 	 * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits
4904bff7fadSNeel Natu 	 * are set (bits 0 and 2 respectively).
4914bff7fadSNeel Natu 	 */
4924bff7fadSNeel Natu 	feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
493150369abSNeel Natu 	if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 0 ||
494150369abSNeel Natu 	    (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) {
4954bff7fadSNeel Natu 		printf("vmx_init: VMX operation disabled by BIOS\n");
4964bff7fadSNeel Natu 		return (ENXIO);
4974bff7fadSNeel Natu 	}
4984bff7fadSNeel Natu 
499366f6083SPeter Grehan 	/* Check support for primary processor-based VM-execution controls */
500366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
501366f6083SPeter Grehan 			       MSR_VMX_TRUE_PROCBASED_CTLS,
502366f6083SPeter Grehan 			       PROCBASED_CTLS_ONE_SETTING,
503366f6083SPeter Grehan 			       PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls);
504366f6083SPeter Grehan 	if (error) {
505366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired primary "
506366f6083SPeter Grehan 		       "processor-based controls\n");
507366f6083SPeter Grehan 		return (error);
508366f6083SPeter Grehan 	}
509366f6083SPeter Grehan 
510366f6083SPeter Grehan 	/* Clear the processor-based ctl bits that are set on demand */
511366f6083SPeter Grehan 	procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING;
512366f6083SPeter Grehan 
513366f6083SPeter Grehan 	/* Check support for secondary processor-based VM-execution controls */
514366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
515366f6083SPeter Grehan 			       MSR_VMX_PROCBASED_CTLS2,
516366f6083SPeter Grehan 			       PROCBASED_CTLS2_ONE_SETTING,
517366f6083SPeter Grehan 			       PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2);
518366f6083SPeter Grehan 	if (error) {
519366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired secondary "
520366f6083SPeter Grehan 		       "processor-based controls\n");
521366f6083SPeter Grehan 		return (error);
522366f6083SPeter Grehan 	}
523366f6083SPeter Grehan 
524366f6083SPeter Grehan 	/* Check support for VPID */
525366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
526366f6083SPeter Grehan 			       PROCBASED2_ENABLE_VPID, 0, &tmp);
527366f6083SPeter Grehan 	if (error == 0)
528366f6083SPeter Grehan 		procbased_ctls2 |= PROCBASED2_ENABLE_VPID;
529366f6083SPeter Grehan 
530366f6083SPeter Grehan 	/* Check support for pin-based VM-execution controls */
531366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
532366f6083SPeter Grehan 			       MSR_VMX_TRUE_PINBASED_CTLS,
533366f6083SPeter Grehan 			       PINBASED_CTLS_ONE_SETTING,
534366f6083SPeter Grehan 			       PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls);
535366f6083SPeter Grehan 	if (error) {
536366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired "
537366f6083SPeter Grehan 		       "pin-based controls\n");
538366f6083SPeter Grehan 		return (error);
539366f6083SPeter Grehan 	}
540366f6083SPeter Grehan 
541366f6083SPeter Grehan 	/* Check support for VM-exit controls */
542366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS,
543366f6083SPeter Grehan 			       VM_EXIT_CTLS_ONE_SETTING,
544366f6083SPeter Grehan 			       VM_EXIT_CTLS_ZERO_SETTING,
545366f6083SPeter Grehan 			       &exit_ctls);
546366f6083SPeter Grehan 	if (error) {
547608f97c3SPeter Grehan 		/* Try again without the PAT MSR bits */
548608f97c3SPeter Grehan 		error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS,
549608f97c3SPeter Grehan 				       MSR_VMX_TRUE_EXIT_CTLS,
550608f97c3SPeter Grehan 				       VM_EXIT_CTLS_ONE_SETTING_NO_PAT,
551608f97c3SPeter Grehan 				       VM_EXIT_CTLS_ZERO_SETTING,
552608f97c3SPeter Grehan 				       &exit_ctls);
553608f97c3SPeter Grehan 		if (error) {
554366f6083SPeter Grehan 			printf("vmx_init: processor does not support desired "
555366f6083SPeter Grehan 			       "exit controls\n");
556366f6083SPeter Grehan 			return (error);
557608f97c3SPeter Grehan 		} else {
558608f97c3SPeter Grehan 			if (bootverbose)
559608f97c3SPeter Grehan 				printf("vmm: PAT MSR access not supported\n");
560608f97c3SPeter Grehan 			guest_msr_valid(MSR_PAT);
561608f97c3SPeter Grehan 			vmx_no_patmsr = 1;
562608f97c3SPeter Grehan 		}
563366f6083SPeter Grehan 	}
564366f6083SPeter Grehan 
565366f6083SPeter Grehan 	/* Check support for VM-entry controls */
566608f97c3SPeter Grehan 	if (!vmx_no_patmsr) {
567608f97c3SPeter Grehan 		error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS,
568608f97c3SPeter Grehan 				       MSR_VMX_TRUE_ENTRY_CTLS,
569366f6083SPeter Grehan 				       VM_ENTRY_CTLS_ONE_SETTING,
570366f6083SPeter Grehan 				       VM_ENTRY_CTLS_ZERO_SETTING,
571366f6083SPeter Grehan 				       &entry_ctls);
572608f97c3SPeter Grehan 	} else {
573608f97c3SPeter Grehan 		error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS,
574608f97c3SPeter Grehan 				       MSR_VMX_TRUE_ENTRY_CTLS,
575608f97c3SPeter Grehan 				       VM_ENTRY_CTLS_ONE_SETTING_NO_PAT,
576608f97c3SPeter Grehan 				       VM_ENTRY_CTLS_ZERO_SETTING,
577608f97c3SPeter Grehan 				       &entry_ctls);
578608f97c3SPeter Grehan 	}
579608f97c3SPeter Grehan 
580366f6083SPeter Grehan 	if (error) {
581366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired "
582366f6083SPeter Grehan 		       "entry controls\n");
583366f6083SPeter Grehan 		       return (error);
584366f6083SPeter Grehan 	}
585366f6083SPeter Grehan 
586366f6083SPeter Grehan 	/*
587366f6083SPeter Grehan 	 * Check support for optional features by testing them
588366f6083SPeter Grehan 	 * as individual bits
589366f6083SPeter Grehan 	 */
590366f6083SPeter Grehan 	cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
591366f6083SPeter Grehan 					MSR_VMX_TRUE_PROCBASED_CTLS,
592366f6083SPeter Grehan 					PROCBASED_HLT_EXITING, 0,
593366f6083SPeter Grehan 					&tmp) == 0);
594366f6083SPeter Grehan 
595366f6083SPeter Grehan 	cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
596366f6083SPeter Grehan 					MSR_VMX_PROCBASED_CTLS,
597366f6083SPeter Grehan 					PROCBASED_MTF, 0,
598366f6083SPeter Grehan 					&tmp) == 0);
599366f6083SPeter Grehan 
600366f6083SPeter Grehan 	cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
601366f6083SPeter Grehan 					 MSR_VMX_TRUE_PROCBASED_CTLS,
602366f6083SPeter Grehan 					 PROCBASED_PAUSE_EXITING, 0,
603366f6083SPeter Grehan 					 &tmp) == 0);
604366f6083SPeter Grehan 
605366f6083SPeter Grehan 	cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
606366f6083SPeter Grehan 					MSR_VMX_PROCBASED_CTLS2,
607366f6083SPeter Grehan 					PROCBASED2_UNRESTRICTED_GUEST, 0,
608366f6083SPeter Grehan 				        &tmp) == 0);
609366f6083SPeter Grehan 
61049cc03daSNeel Natu 	cap_invpcid = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
61149cc03daSNeel Natu 	    MSR_VMX_PROCBASED_CTLS2, PROCBASED2_ENABLE_INVPCID, 0,
61249cc03daSNeel Natu 	    &tmp) == 0);
61349cc03daSNeel Natu 
61488c4b8d1SNeel Natu 	/*
61588c4b8d1SNeel Natu 	 * Check support for virtual interrupt delivery.
61688c4b8d1SNeel Natu 	 */
61788c4b8d1SNeel Natu 	procbased2_vid_bits = (PROCBASED2_VIRTUALIZE_APIC_ACCESSES |
61888c4b8d1SNeel Natu 	    PROCBASED2_VIRTUALIZE_X2APIC_MODE |
61988c4b8d1SNeel Natu 	    PROCBASED2_APIC_REGISTER_VIRTUALIZATION |
62088c4b8d1SNeel Natu 	    PROCBASED2_VIRTUAL_INTERRUPT_DELIVERY);
62188c4b8d1SNeel Natu 
62288c4b8d1SNeel Natu 	use_tpr_shadow = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
62388c4b8d1SNeel Natu 	    MSR_VMX_TRUE_PROCBASED_CTLS, PROCBASED_USE_TPR_SHADOW, 0,
62488c4b8d1SNeel Natu 	    &tmp) == 0);
62588c4b8d1SNeel Natu 
62688c4b8d1SNeel Natu 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
62788c4b8d1SNeel Natu 	    procbased2_vid_bits, 0, &tmp);
62888c4b8d1SNeel Natu 	if (error == 0 && use_tpr_shadow) {
62988c4b8d1SNeel Natu 		virtual_interrupt_delivery = 1;
63088c4b8d1SNeel Natu 		TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_vid",
63188c4b8d1SNeel Natu 		    &virtual_interrupt_delivery);
63288c4b8d1SNeel Natu 	}
63388c4b8d1SNeel Natu 
63488c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
63588c4b8d1SNeel Natu 		procbased_ctls |= PROCBASED_USE_TPR_SHADOW;
63688c4b8d1SNeel Natu 		procbased_ctls2 |= procbased2_vid_bits;
63788c4b8d1SNeel Natu 		procbased_ctls2 &= ~PROCBASED2_VIRTUALIZE_X2APIC_MODE;
638176666c2SNeel Natu 
639176666c2SNeel Natu 		/*
640176666c2SNeel Natu 		 * Check for Posted Interrupts only if Virtual Interrupt
641176666c2SNeel Natu 		 * Delivery is enabled.
642176666c2SNeel Natu 		 */
643176666c2SNeel Natu 		error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
644176666c2SNeel Natu 		    MSR_VMX_TRUE_PINBASED_CTLS, PINBASED_POSTED_INTERRUPT, 0,
645176666c2SNeel Natu 		    &tmp);
646176666c2SNeel Natu 		if (error == 0) {
647176666c2SNeel Natu 			pirvec = vmm_ipi_alloc();
648176666c2SNeel Natu 			if (pirvec == 0) {
649176666c2SNeel Natu 				if (bootverbose) {
650176666c2SNeel Natu 					printf("vmx_init: unable to allocate "
651176666c2SNeel Natu 					    "posted interrupt vector\n");
65288c4b8d1SNeel Natu 				}
653176666c2SNeel Natu 			} else {
654176666c2SNeel Natu 				posted_interrupts = 1;
655176666c2SNeel Natu 				TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_pir",
656176666c2SNeel Natu 				    &posted_interrupts);
657176666c2SNeel Natu 			}
658176666c2SNeel Natu 		}
659176666c2SNeel Natu 	}
660176666c2SNeel Natu 
661176666c2SNeel Natu 	if (posted_interrupts)
662176666c2SNeel Natu 		    pinbased_ctls |= PINBASED_POSTED_INTERRUPT;
66349cc03daSNeel Natu 
664366f6083SPeter Grehan 	/* Initialize EPT */
665add611fdSNeel Natu 	error = ept_init(ipinum);
666366f6083SPeter Grehan 	if (error) {
667366f6083SPeter Grehan 		printf("vmx_init: ept initialization failed (%d)\n", error);
668366f6083SPeter Grehan 		return (error);
669366f6083SPeter Grehan 	}
670366f6083SPeter Grehan 
671366f6083SPeter Grehan 	/*
672366f6083SPeter Grehan 	 * Stash the cr0 and cr4 bits that must be fixed to 0 or 1
673366f6083SPeter Grehan 	 */
674366f6083SPeter Grehan 	fixed0 = rdmsr(MSR_VMX_CR0_FIXED0);
675366f6083SPeter Grehan 	fixed1 = rdmsr(MSR_VMX_CR0_FIXED1);
676366f6083SPeter Grehan 	cr0_ones_mask = fixed0 & fixed1;
677366f6083SPeter Grehan 	cr0_zeros_mask = ~fixed0 & ~fixed1;
678366f6083SPeter Grehan 
679366f6083SPeter Grehan 	/*
680366f6083SPeter Grehan 	 * CR0_PE and CR0_PG can be set to zero in VMX non-root operation
681366f6083SPeter Grehan 	 * if unrestricted guest execution is allowed.
682366f6083SPeter Grehan 	 */
683366f6083SPeter Grehan 	if (cap_unrestricted_guest)
684366f6083SPeter Grehan 		cr0_ones_mask &= ~(CR0_PG | CR0_PE);
685366f6083SPeter Grehan 
686366f6083SPeter Grehan 	/*
687366f6083SPeter Grehan 	 * Do not allow the guest to set CR0_NW or CR0_CD.
688366f6083SPeter Grehan 	 */
689366f6083SPeter Grehan 	cr0_zeros_mask |= (CR0_NW | CR0_CD);
690366f6083SPeter Grehan 
691366f6083SPeter Grehan 	fixed0 = rdmsr(MSR_VMX_CR4_FIXED0);
692366f6083SPeter Grehan 	fixed1 = rdmsr(MSR_VMX_CR4_FIXED1);
693366f6083SPeter Grehan 	cr4_ones_mask = fixed0 & fixed1;
694366f6083SPeter Grehan 	cr4_zeros_mask = ~fixed0 & ~fixed1;
695366f6083SPeter Grehan 
69645e51299SNeel Natu 	vpid_init();
69745e51299SNeel Natu 
698366f6083SPeter Grehan 	/* enable VMX operation */
699366f6083SPeter Grehan 	smp_rendezvous(NULL, vmx_enable, NULL, NULL);
700366f6083SPeter Grehan 
7013565b59eSNeel Natu 	vmx_initialized = 1;
7023565b59eSNeel Natu 
703366f6083SPeter Grehan 	return (0);
704366f6083SPeter Grehan }
705366f6083SPeter Grehan 
706f7d47425SNeel Natu static void
707f7d47425SNeel Natu vmx_trigger_hostintr(int vector)
708f7d47425SNeel Natu {
709f7d47425SNeel Natu 	uintptr_t func;
710f7d47425SNeel Natu 	struct gate_descriptor *gd;
711f7d47425SNeel Natu 
712f7d47425SNeel Natu 	gd = &idt[vector];
713f7d47425SNeel Natu 
714f7d47425SNeel Natu 	KASSERT(vector >= 32 && vector <= 255, ("vmx_trigger_hostintr: "
715f7d47425SNeel Natu 	    "invalid vector %d", vector));
716f7d47425SNeel Natu 	KASSERT(gd->gd_p == 1, ("gate descriptor for vector %d not present",
717f7d47425SNeel Natu 	    vector));
718f7d47425SNeel Natu 	KASSERT(gd->gd_type == SDT_SYSIGT, ("gate descriptor for vector %d "
719f7d47425SNeel Natu 	    "has invalid type %d", vector, gd->gd_type));
720f7d47425SNeel Natu 	KASSERT(gd->gd_dpl == SEL_KPL, ("gate descriptor for vector %d "
721f7d47425SNeel Natu 	    "has invalid dpl %d", vector, gd->gd_dpl));
722f7d47425SNeel Natu 	KASSERT(gd->gd_selector == GSEL(GCODE_SEL, SEL_KPL), ("gate descriptor "
723f7d47425SNeel Natu 	    "for vector %d has invalid selector %d", vector, gd->gd_selector));
724f7d47425SNeel Natu 	KASSERT(gd->gd_ist == 0, ("gate descriptor for vector %d has invalid "
725f7d47425SNeel Natu 	    "IST %d", vector, gd->gd_ist));
726f7d47425SNeel Natu 
727f7d47425SNeel Natu 	func = ((long)gd->gd_hioffset << 16 | gd->gd_looffset);
728f7d47425SNeel Natu 	vmx_call_isr(func);
729f7d47425SNeel Natu }
730f7d47425SNeel Natu 
731366f6083SPeter Grehan static int
732aaaa0656SPeter Grehan vmx_setup_cr_shadow(int which, struct vmcs *vmcs, uint32_t initial)
733366f6083SPeter Grehan {
73439c21c2dSNeel Natu 	int error, mask_ident, shadow_ident;
735aaaa0656SPeter Grehan 	uint64_t mask_value;
736366f6083SPeter Grehan 
73739c21c2dSNeel Natu 	if (which != 0 && which != 4)
73839c21c2dSNeel Natu 		panic("vmx_setup_cr_shadow: unknown cr%d", which);
73939c21c2dSNeel Natu 
74039c21c2dSNeel Natu 	if (which == 0) {
74139c21c2dSNeel Natu 		mask_ident = VMCS_CR0_MASK;
74239c21c2dSNeel Natu 		mask_value = cr0_ones_mask | cr0_zeros_mask;
74339c21c2dSNeel Natu 		shadow_ident = VMCS_CR0_SHADOW;
74439c21c2dSNeel Natu 	} else {
74539c21c2dSNeel Natu 		mask_ident = VMCS_CR4_MASK;
74639c21c2dSNeel Natu 		mask_value = cr4_ones_mask | cr4_zeros_mask;
74739c21c2dSNeel Natu 		shadow_ident = VMCS_CR4_SHADOW;
74839c21c2dSNeel Natu 	}
74939c21c2dSNeel Natu 
750d3c11f40SPeter Grehan 	error = vmcs_setreg(vmcs, 0, VMCS_IDENT(mask_ident), mask_value);
751366f6083SPeter Grehan 	if (error)
752366f6083SPeter Grehan 		return (error);
753366f6083SPeter Grehan 
754aaaa0656SPeter Grehan 	error = vmcs_setreg(vmcs, 0, VMCS_IDENT(shadow_ident), initial);
755366f6083SPeter Grehan 	if (error)
756366f6083SPeter Grehan 		return (error);
757366f6083SPeter Grehan 
758366f6083SPeter Grehan 	return (0);
759366f6083SPeter Grehan }
760aaaa0656SPeter Grehan #define	vmx_setup_cr0_shadow(vmcs,init)	vmx_setup_cr_shadow(0, (vmcs), (init))
761aaaa0656SPeter Grehan #define	vmx_setup_cr4_shadow(vmcs,init)	vmx_setup_cr_shadow(4, (vmcs), (init))
762366f6083SPeter Grehan 
763366f6083SPeter Grehan static void *
764318224bbSNeel Natu vmx_vminit(struct vm *vm, pmap_t pmap)
765366f6083SPeter Grehan {
76645e51299SNeel Natu 	uint16_t vpid[VM_MAXCPU];
767366f6083SPeter Grehan 	int i, error, guest_msr_count;
768366f6083SPeter Grehan 	struct vmx *vmx;
769c847a506SNeel Natu 	struct vmcs *vmcs;
770366f6083SPeter Grehan 
771366f6083SPeter Grehan 	vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO);
772366f6083SPeter Grehan 	if ((uintptr_t)vmx & PAGE_MASK) {
773366f6083SPeter Grehan 		panic("malloc of struct vmx not aligned on %d byte boundary",
774366f6083SPeter Grehan 		      PAGE_SIZE);
775366f6083SPeter Grehan 	}
776366f6083SPeter Grehan 	vmx->vm = vm;
777366f6083SPeter Grehan 
778318224bbSNeel Natu 	vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pml4));
779318224bbSNeel Natu 
780366f6083SPeter Grehan 	/*
781366f6083SPeter Grehan 	 * Clean up EPTP-tagged guest physical and combined mappings
782366f6083SPeter Grehan 	 *
783366f6083SPeter Grehan 	 * VMX transitions are not required to invalidate any guest physical
784366f6083SPeter Grehan 	 * mappings. So, it may be possible for stale guest physical mappings
785366f6083SPeter Grehan 	 * to be present in the processor TLBs.
786366f6083SPeter Grehan 	 *
787366f6083SPeter Grehan 	 * Combined mappings for this EP4TA are also invalidated for all VPIDs.
788366f6083SPeter Grehan 	 */
789318224bbSNeel Natu 	ept_invalidate_mappings(vmx->eptp);
790366f6083SPeter Grehan 
791366f6083SPeter Grehan 	msr_bitmap_initialize(vmx->msr_bitmap);
792366f6083SPeter Grehan 
793366f6083SPeter Grehan 	/*
794366f6083SPeter Grehan 	 * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE.
795366f6083SPeter Grehan 	 * The guest FSBASE and GSBASE are saved and restored during
796366f6083SPeter Grehan 	 * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are
797366f6083SPeter Grehan 	 * always restored from the vmcs host state area on vm-exit.
798366f6083SPeter Grehan 	 *
7991fb0ea3fSPeter Grehan 	 * The SYSENTER_CS/ESP/EIP MSRs are identical to FS/GSBASE in
8001fb0ea3fSPeter Grehan 	 * how they are saved/restored so can be directly accessed by the
8011fb0ea3fSPeter Grehan 	 * guest.
8021fb0ea3fSPeter Grehan 	 *
803366f6083SPeter Grehan 	 * Guest KGSBASE is saved and restored in the guest MSR save area.
804366f6083SPeter Grehan 	 * Host KGSBASE is restored before returning to userland from the pcb.
805366f6083SPeter Grehan 	 * There will be a window of time when we are executing in the host
806366f6083SPeter Grehan 	 * kernel context with a value of KGSBASE from the guest. This is ok
807366f6083SPeter Grehan 	 * because the value of KGSBASE is inconsequential in kernel context.
808366f6083SPeter Grehan 	 *
809366f6083SPeter Grehan 	 * MSR_EFER is saved and restored in the guest VMCS area on a
810366f6083SPeter Grehan 	 * VM exit and entry respectively. It is also restored from the
811366f6083SPeter Grehan 	 * host VMCS area on a VM exit.
812366f6083SPeter Grehan 	 */
813366f6083SPeter Grehan 	if (guest_msr_rw(vmx, MSR_GSBASE) ||
814366f6083SPeter Grehan 	    guest_msr_rw(vmx, MSR_FSBASE) ||
8151fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_CS_MSR) ||
8161fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_ESP_MSR) ||
8171fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_EIP_MSR) ||
818366f6083SPeter Grehan 	    guest_msr_rw(vmx, MSR_KGSBASE) ||
819608f97c3SPeter Grehan 	    guest_msr_rw(vmx, MSR_EFER))
820366f6083SPeter Grehan 		panic("vmx_vminit: error setting guest msr access");
821366f6083SPeter Grehan 
822608f97c3SPeter Grehan 	/*
823608f97c3SPeter Grehan 	 * MSR_PAT is saved and restored in the guest VMCS are on a VM exit
824608f97c3SPeter Grehan 	 * and entry respectively. It is also restored from the host VMCS
825608f97c3SPeter Grehan 	 * area on a VM exit. However, if running on a system with no
826608f97c3SPeter Grehan 	 * MSR_PAT save/restore support, leave access disabled so accesses
827608f97c3SPeter Grehan 	 * will be trapped.
828608f97c3SPeter Grehan 	 */
829608f97c3SPeter Grehan 	if (!vmx_no_patmsr && guest_msr_rw(vmx, MSR_PAT))
830608f97c3SPeter Grehan 		panic("vmx_vminit: error setting guest pat msr access");
831608f97c3SPeter Grehan 
83245e51299SNeel Natu 	vpid_alloc(vpid, VM_MAXCPU);
83345e51299SNeel Natu 
83488c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
83588c4b8d1SNeel Natu 		error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE,
83688c4b8d1SNeel Natu 		    APIC_ACCESS_ADDRESS);
83788c4b8d1SNeel Natu 		/* XXX this should really return an error to the caller */
83888c4b8d1SNeel Natu 		KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error));
83988c4b8d1SNeel Natu 	}
84088c4b8d1SNeel Natu 
841366f6083SPeter Grehan 	for (i = 0; i < VM_MAXCPU; i++) {
842c847a506SNeel Natu 		vmcs = &vmx->vmcs[i];
843c847a506SNeel Natu 		vmcs->identifier = vmx_revision();
844c847a506SNeel Natu 		error = vmclear(vmcs);
845366f6083SPeter Grehan 		if (error != 0) {
846366f6083SPeter Grehan 			panic("vmx_vminit: vmclear error %d on vcpu %d\n",
847366f6083SPeter Grehan 			      error, i);
848366f6083SPeter Grehan 		}
849366f6083SPeter Grehan 
850c847a506SNeel Natu 		error = vmcs_init(vmcs);
851c847a506SNeel Natu 		KASSERT(error == 0, ("vmcs_init error %d", error));
852366f6083SPeter Grehan 
853c847a506SNeel Natu 		VMPTRLD(vmcs);
854c847a506SNeel Natu 		error = 0;
855c847a506SNeel Natu 		error += vmwrite(VMCS_HOST_RSP, (u_long)&vmx->ctx[i]);
856c847a506SNeel Natu 		error += vmwrite(VMCS_EPTP, vmx->eptp);
857c847a506SNeel Natu 		error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls);
858c847a506SNeel Natu 		error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls);
859c847a506SNeel Natu 		error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2);
860c847a506SNeel Natu 		error += vmwrite(VMCS_EXIT_CTLS, exit_ctls);
861c847a506SNeel Natu 		error += vmwrite(VMCS_ENTRY_CTLS, entry_ctls);
862c847a506SNeel Natu 		error += vmwrite(VMCS_MSR_BITMAP, vtophys(vmx->msr_bitmap));
863c847a506SNeel Natu 		error += vmwrite(VMCS_VPID, vpid[i]);
86488c4b8d1SNeel Natu 		if (virtual_interrupt_delivery) {
86588c4b8d1SNeel Natu 			error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS);
86688c4b8d1SNeel Natu 			error += vmwrite(VMCS_VIRTUAL_APIC,
86788c4b8d1SNeel Natu 			    vtophys(&vmx->apic_page[i]));
86888c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT0, 0);
86988c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT1, 0);
87088c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT2, 0);
87188c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT3, 0);
87288c4b8d1SNeel Natu 		}
873176666c2SNeel Natu 		if (posted_interrupts) {
874176666c2SNeel Natu 			error += vmwrite(VMCS_PIR_VECTOR, pirvec);
875176666c2SNeel Natu 			error += vmwrite(VMCS_PIR_DESC,
876176666c2SNeel Natu 			    vtophys(&vmx->pir_desc[i]));
877176666c2SNeel Natu 		}
878c847a506SNeel Natu 		VMCLEAR(vmcs);
879c847a506SNeel Natu 		KASSERT(error == 0, ("vmx_vminit: error customizing the vmcs"));
880366f6083SPeter Grehan 
881366f6083SPeter Grehan 		vmx->cap[i].set = 0;
882366f6083SPeter Grehan 		vmx->cap[i].proc_ctls = procbased_ctls;
88349cc03daSNeel Natu 		vmx->cap[i].proc_ctls2 = procbased_ctls2;
884366f6083SPeter Grehan 
885366f6083SPeter Grehan 		vmx->state[i].lastcpu = -1;
88645e51299SNeel Natu 		vmx->state[i].vpid = vpid[i];
887366f6083SPeter Grehan 
888366f6083SPeter Grehan 		msr_save_area_init(vmx->guest_msrs[i], &guest_msr_count);
889366f6083SPeter Grehan 
890c847a506SNeel Natu 		error = vmcs_set_msr_save(vmcs, vtophys(vmx->guest_msrs[i]),
891366f6083SPeter Grehan 		    guest_msr_count);
892366f6083SPeter Grehan 		if (error != 0)
893366f6083SPeter Grehan 			panic("vmcs_set_msr_save error %d", error);
894366f6083SPeter Grehan 
895aaaa0656SPeter Grehan 		/*
896aaaa0656SPeter Grehan 		 * Set up the CR0/4 shadows, and init the read shadow
897aaaa0656SPeter Grehan 		 * to the power-on register value from the Intel Sys Arch.
898aaaa0656SPeter Grehan 		 *  CR0 - 0x60000010
899aaaa0656SPeter Grehan 		 *  CR4 - 0
900aaaa0656SPeter Grehan 		 */
901c847a506SNeel Natu 		error = vmx_setup_cr0_shadow(vmcs, 0x60000010);
90239c21c2dSNeel Natu 		if (error != 0)
90339c21c2dSNeel Natu 			panic("vmx_setup_cr0_shadow %d", error);
90439c21c2dSNeel Natu 
905c847a506SNeel Natu 		error = vmx_setup_cr4_shadow(vmcs, 0);
90639c21c2dSNeel Natu 		if (error != 0)
90739c21c2dSNeel Natu 			panic("vmx_setup_cr4_shadow %d", error);
908318224bbSNeel Natu 
909318224bbSNeel Natu 		vmx->ctx[i].pmap = pmap;
910318224bbSNeel Natu 		vmx->ctx[i].eptp = vmx->eptp;
911366f6083SPeter Grehan 	}
912366f6083SPeter Grehan 
913366f6083SPeter Grehan 	return (vmx);
914366f6083SPeter Grehan }
915366f6083SPeter Grehan 
916366f6083SPeter Grehan static int
917a2da7af6SNeel Natu vmx_handle_cpuid(struct vm *vm, int vcpu, struct vmxctx *vmxctx)
918366f6083SPeter Grehan {
919366f6083SPeter Grehan 	int handled, func;
920366f6083SPeter Grehan 
921366f6083SPeter Grehan 	func = vmxctx->guest_rax;
922366f6083SPeter Grehan 
923a2da7af6SNeel Natu 	handled = x86_emulate_cpuid(vm, vcpu,
924a2da7af6SNeel Natu 				    (uint32_t*)(&vmxctx->guest_rax),
925a2da7af6SNeel Natu 				    (uint32_t*)(&vmxctx->guest_rbx),
926a2da7af6SNeel Natu 				    (uint32_t*)(&vmxctx->guest_rcx),
927a2da7af6SNeel Natu 				    (uint32_t*)(&vmxctx->guest_rdx));
928366f6083SPeter Grehan 	return (handled);
929366f6083SPeter Grehan }
930366f6083SPeter Grehan 
931366f6083SPeter Grehan static __inline void
932366f6083SPeter Grehan vmx_run_trace(struct vmx *vmx, int vcpu)
933366f6083SPeter Grehan {
934366f6083SPeter Grehan #ifdef KTR
935513c8d33SNeel Natu 	VCPU_CTR1(vmx->vm, vcpu, "Resume execution at %#lx", vmcs_guest_rip());
936366f6083SPeter Grehan #endif
937366f6083SPeter Grehan }
938366f6083SPeter Grehan 
939366f6083SPeter Grehan static __inline void
940366f6083SPeter Grehan vmx_exit_trace(struct vmx *vmx, int vcpu, uint64_t rip, uint32_t exit_reason,
941eeefa4e4SNeel Natu 	       int handled)
942366f6083SPeter Grehan {
943366f6083SPeter Grehan #ifdef KTR
944513c8d33SNeel Natu 	VCPU_CTR3(vmx->vm, vcpu, "%s %s vmexit at 0x%0lx",
945366f6083SPeter Grehan 		 handled ? "handled" : "unhandled",
946366f6083SPeter Grehan 		 exit_reason_to_str(exit_reason), rip);
947eeefa4e4SNeel Natu #endif
948eeefa4e4SNeel Natu }
949366f6083SPeter Grehan 
950eeefa4e4SNeel Natu static __inline void
951eeefa4e4SNeel Natu vmx_astpending_trace(struct vmx *vmx, int vcpu, uint64_t rip)
952eeefa4e4SNeel Natu {
953eeefa4e4SNeel Natu #ifdef KTR
954513c8d33SNeel Natu 	VCPU_CTR1(vmx->vm, vcpu, "astpending vmexit at 0x%0lx", rip);
955366f6083SPeter Grehan #endif
956366f6083SPeter Grehan }
957366f6083SPeter Grehan 
9583de83862SNeel Natu static void
959366f6083SPeter Grehan vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu)
960366f6083SPeter Grehan {
9613de83862SNeel Natu 	int lastcpu;
962366f6083SPeter Grehan 	struct vmxstate *vmxstate;
963366f6083SPeter Grehan 	struct invvpid_desc invvpid_desc = { 0 };
964366f6083SPeter Grehan 
965366f6083SPeter Grehan 	vmxstate = &vmx->state[vcpu];
966366f6083SPeter Grehan 	lastcpu = vmxstate->lastcpu;
967366f6083SPeter Grehan 	vmxstate->lastcpu = curcpu;
968366f6083SPeter Grehan 
9693de83862SNeel Natu 	if (lastcpu == curcpu)
9703de83862SNeel Natu 		return;
971366f6083SPeter Grehan 
972366f6083SPeter Grehan 	vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1);
973366f6083SPeter Grehan 
9743de83862SNeel Natu 	vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase());
9753de83862SNeel Natu 	vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase());
9763de83862SNeel Natu 	vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase());
977366f6083SPeter Grehan 
978366f6083SPeter Grehan 	/*
979366f6083SPeter Grehan 	 * If we are using VPIDs then invalidate all mappings tagged with 'vpid'
980366f6083SPeter Grehan 	 *
981366f6083SPeter Grehan 	 * We do this because this vcpu was executing on a different host
982366f6083SPeter Grehan 	 * cpu when it last ran. We do not track whether it invalidated
983366f6083SPeter Grehan 	 * mappings associated with its 'vpid' during that run. So we must
984366f6083SPeter Grehan 	 * assume that the mappings associated with 'vpid' on 'curcpu' are
985366f6083SPeter Grehan 	 * stale and invalidate them.
986366f6083SPeter Grehan 	 *
987366f6083SPeter Grehan 	 * Note that we incur this penalty only when the scheduler chooses to
988366f6083SPeter Grehan 	 * move the thread associated with this vcpu between host cpus.
989366f6083SPeter Grehan 	 *
990366f6083SPeter Grehan 	 * Note also that this will invalidate mappings tagged with 'vpid'
991366f6083SPeter Grehan 	 * for "all" EP4TAs.
992366f6083SPeter Grehan 	 */
993366f6083SPeter Grehan 	if (vmxstate->vpid != 0) {
994366f6083SPeter Grehan 		invvpid_desc.vpid = vmxstate->vpid;
995366f6083SPeter Grehan 		invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc);
996366f6083SPeter Grehan 	}
997366f6083SPeter Grehan }
998366f6083SPeter Grehan 
999366f6083SPeter Grehan /*
1000366f6083SPeter Grehan  * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set.
1001366f6083SPeter Grehan  */
1002366f6083SPeter Grehan CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0);
1003366f6083SPeter Grehan 
1004366f6083SPeter Grehan static void __inline
1005366f6083SPeter Grehan vmx_set_int_window_exiting(struct vmx *vmx, int vcpu)
1006366f6083SPeter Grehan {
1007366f6083SPeter Grehan 
100848b2d828SNeel Natu 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) {
1009366f6083SPeter Grehan 		vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING;
10103de83862SNeel Natu 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
101148b2d828SNeel Natu 		VCPU_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting");
101248b2d828SNeel Natu 	}
1013366f6083SPeter Grehan }
1014366f6083SPeter Grehan 
1015366f6083SPeter Grehan static void __inline
1016366f6083SPeter Grehan vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu)
1017366f6083SPeter Grehan {
1018366f6083SPeter Grehan 
101948b2d828SNeel Natu 	KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0,
102048b2d828SNeel Natu 	    ("intr_window_exiting not set: %#x", vmx->cap[vcpu].proc_ctls));
1021366f6083SPeter Grehan 	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING;
10223de83862SNeel Natu 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
102348b2d828SNeel Natu 	VCPU_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting");
1024366f6083SPeter Grehan }
1025366f6083SPeter Grehan 
1026366f6083SPeter Grehan static void __inline
1027366f6083SPeter Grehan vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu)
1028366f6083SPeter Grehan {
1029366f6083SPeter Grehan 
103048b2d828SNeel Natu 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) {
1031366f6083SPeter Grehan 		vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING;
10323de83862SNeel Natu 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
103348b2d828SNeel Natu 		VCPU_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting");
103448b2d828SNeel Natu 	}
1035366f6083SPeter Grehan }
1036366f6083SPeter Grehan 
1037366f6083SPeter Grehan static void __inline
1038366f6083SPeter Grehan vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu)
1039366f6083SPeter Grehan {
1040366f6083SPeter Grehan 
104148b2d828SNeel Natu 	KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0,
104248b2d828SNeel Natu 	    ("nmi_window_exiting not set %#x", vmx->cap[vcpu].proc_ctls));
1043366f6083SPeter Grehan 	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING;
10443de83862SNeel Natu 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
104548b2d828SNeel Natu 	VCPU_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting");
1046366f6083SPeter Grehan }
1047366f6083SPeter Grehan 
104848b2d828SNeel Natu #define	NMI_BLOCKING	(VMCS_INTERRUPTIBILITY_NMI_BLOCKING |		\
104948b2d828SNeel Natu 			 VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
105048b2d828SNeel Natu #define	HWINTR_BLOCKING	(VMCS_INTERRUPTIBILITY_STI_BLOCKING |		\
105148b2d828SNeel Natu 			 VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
105248b2d828SNeel Natu 
105348b2d828SNeel Natu static void
1054366f6083SPeter Grehan vmx_inject_nmi(struct vmx *vmx, int vcpu)
1055366f6083SPeter Grehan {
105648b2d828SNeel Natu 	uint32_t gi, info;
1057366f6083SPeter Grehan 
105848b2d828SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
105948b2d828SNeel Natu 	KASSERT((gi & NMI_BLOCKING) == 0, ("vmx_inject_nmi: invalid guest "
106048b2d828SNeel Natu 	    "interruptibility-state %#x", gi));
1061366f6083SPeter Grehan 
106248b2d828SNeel Natu 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
106348b2d828SNeel Natu 	KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_nmi: invalid "
106448b2d828SNeel Natu 	    "VM-entry interruption information %#x", info));
1065366f6083SPeter Grehan 
1066366f6083SPeter Grehan 	/*
1067366f6083SPeter Grehan 	 * Inject the virtual NMI. The vector must be the NMI IDT entry
1068366f6083SPeter Grehan 	 * or the VMCS entry check will fail.
1069366f6083SPeter Grehan 	 */
107048b2d828SNeel Natu 	info = IDT_NMI | VMCS_INTR_T_NMI | VMCS_INTR_VALID;
10713de83862SNeel Natu 	vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1072366f6083SPeter Grehan 
1073513c8d33SNeel Natu 	VCPU_CTR0(vmx->vm, vcpu, "Injecting vNMI");
1074366f6083SPeter Grehan 
1075366f6083SPeter Grehan 	/* Clear the request */
1076f352ff0cSNeel Natu 	vm_nmi_clear(vmx->vm, vcpu);
1077366f6083SPeter Grehan }
1078366f6083SPeter Grehan 
1079366f6083SPeter Grehan static void
1080de5ea6b6SNeel Natu vmx_inject_interrupts(struct vmx *vmx, int vcpu, struct vlapic *vlapic)
1081366f6083SPeter Grehan {
108248b2d828SNeel Natu 	int vector, need_nmi_exiting;
108348b2d828SNeel Natu 	uint64_t rflags;
108448b2d828SNeel Natu 	uint32_t gi, info;
1085366f6083SPeter Grehan 
108648b2d828SNeel Natu 	if (vm_nmi_pending(vmx->vm, vcpu)) {
1087366f6083SPeter Grehan 		/*
108848b2d828SNeel Natu 		 * If there are no conditions blocking NMI injection then
108948b2d828SNeel Natu 		 * inject it directly here otherwise enable "NMI window
109048b2d828SNeel Natu 		 * exiting" to inject it as soon as we can.
1091eeefa4e4SNeel Natu 		 *
109248b2d828SNeel Natu 		 * We also check for STI_BLOCKING because some implementations
109348b2d828SNeel Natu 		 * don't allow NMI injection in this case. If we are running
109448b2d828SNeel Natu 		 * on a processor that doesn't have this restriction it will
109548b2d828SNeel Natu 		 * immediately exit and the NMI will be injected in the
109648b2d828SNeel Natu 		 * "NMI window exiting" handler.
1097366f6083SPeter Grehan 		 */
109848b2d828SNeel Natu 		need_nmi_exiting = 1;
109948b2d828SNeel Natu 		gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
110048b2d828SNeel Natu 		if ((gi & (HWINTR_BLOCKING | NMI_BLOCKING)) == 0) {
11013de83862SNeel Natu 			info = vmcs_read(VMCS_ENTRY_INTR_INFO);
110248b2d828SNeel Natu 			if ((info & VMCS_INTR_VALID) == 0) {
110348b2d828SNeel Natu 				vmx_inject_nmi(vmx, vcpu);
110448b2d828SNeel Natu 				need_nmi_exiting = 0;
110548b2d828SNeel Natu 			} else {
110648b2d828SNeel Natu 				VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI "
110748b2d828SNeel Natu 				    "due to VM-entry intr info %#x", info);
110848b2d828SNeel Natu 			}
110948b2d828SNeel Natu 		} else {
111048b2d828SNeel Natu 			VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI due to "
111148b2d828SNeel Natu 			    "Guest Interruptibility-state %#x", gi);
111248b2d828SNeel Natu 		}
1113eeefa4e4SNeel Natu 
111448b2d828SNeel Natu 		if (need_nmi_exiting)
111548b2d828SNeel Natu 			vmx_set_nmi_window_exiting(vmx, vcpu);
111648b2d828SNeel Natu 	}
1117366f6083SPeter Grehan 
111888c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
111988c4b8d1SNeel Natu 		vmx_inject_pir(vlapic);
112088c4b8d1SNeel Natu 		return;
112188c4b8d1SNeel Natu 	}
112288c4b8d1SNeel Natu 
112348b2d828SNeel Natu 	/*
112436736912SNeel Natu 	 * If interrupt-window exiting is already in effect then don't bother
112536736912SNeel Natu 	 * checking for pending interrupts. This is just an optimization and
112636736912SNeel Natu 	 * not needed for correctness.
112748b2d828SNeel Natu 	 */
112836736912SNeel Natu 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) {
112936736912SNeel Natu 		VCPU_CTR0(vmx->vm, vcpu, "Skip interrupt injection due to "
113036736912SNeel Natu 		    "pending int_window_exiting");
113148b2d828SNeel Natu 		return;
113236736912SNeel Natu 	}
113348b2d828SNeel Natu 
1134366f6083SPeter Grehan 	/* Ask the local apic for a vector to inject */
11354d1e82a8SNeel Natu 	if (!vlapic_pending_intr(vlapic, &vector))
1136366f6083SPeter Grehan 		return;
1137366f6083SPeter Grehan 
113848b2d828SNeel Natu 	KASSERT(vector >= 32 && vector <= 255, ("invalid vector %d", vector));
1139366f6083SPeter Grehan 
1140366f6083SPeter Grehan 	/* Check RFLAGS.IF and the interruptibility state of the guest */
11413de83862SNeel Natu 	rflags = vmcs_read(VMCS_GUEST_RFLAGS);
114236736912SNeel Natu 	if ((rflags & PSL_I) == 0) {
114336736912SNeel Natu 		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
114436736912SNeel Natu 		    "rflags %#lx", vector, rflags);
1145366f6083SPeter Grehan 		goto cantinject;
114636736912SNeel Natu 	}
1147366f6083SPeter Grehan 
114848b2d828SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
114936736912SNeel Natu 	if (gi & HWINTR_BLOCKING) {
115036736912SNeel Natu 		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
115136736912SNeel Natu 		    "Guest Interruptibility-state %#x", vector, gi);
1152366f6083SPeter Grehan 		goto cantinject;
115336736912SNeel Natu 	}
115436736912SNeel Natu 
115536736912SNeel Natu 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
115636736912SNeel Natu 	if (info & VMCS_INTR_VALID) {
115736736912SNeel Natu 		/*
115836736912SNeel Natu 		 * This is expected and could happen for multiple reasons:
115936736912SNeel Natu 		 * - A vectoring VM-entry was aborted due to astpending
116036736912SNeel Natu 		 * - A VM-exit happened during event injection.
116136736912SNeel Natu 		 * - An NMI was injected above or after "NMI window exiting"
116236736912SNeel Natu 		 */
116336736912SNeel Natu 		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
116436736912SNeel Natu 		    "VM-entry intr info %#x", vector, info);
116536736912SNeel Natu 		goto cantinject;
116636736912SNeel Natu 	}
1167366f6083SPeter Grehan 
1168366f6083SPeter Grehan 	/* Inject the interrupt */
1169160471d2SNeel Natu 	info = VMCS_INTR_T_HWINTR | VMCS_INTR_VALID;
1170366f6083SPeter Grehan 	info |= vector;
11713de83862SNeel Natu 	vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1172366f6083SPeter Grehan 
1173366f6083SPeter Grehan 	/* Update the Local APIC ISR */
1174de5ea6b6SNeel Natu 	vlapic_intr_accepted(vlapic, vector);
1175366f6083SPeter Grehan 
1176513c8d33SNeel Natu 	VCPU_CTR1(vmx->vm, vcpu, "Injecting hwintr at vector %d", vector);
1177366f6083SPeter Grehan 
1178366f6083SPeter Grehan 	return;
1179366f6083SPeter Grehan 
1180366f6083SPeter Grehan cantinject:
1181366f6083SPeter Grehan 	/*
1182366f6083SPeter Grehan 	 * Set the Interrupt Window Exiting execution control so we can inject
1183366f6083SPeter Grehan 	 * the interrupt as soon as blocking condition goes away.
1184366f6083SPeter Grehan 	 */
1185366f6083SPeter Grehan 	vmx_set_int_window_exiting(vmx, vcpu);
1186366f6083SPeter Grehan }
1187366f6083SPeter Grehan 
1188e5a1d950SNeel Natu /*
1189e5a1d950SNeel Natu  * If the Virtual NMIs execution control is '1' then the logical processor
1190e5a1d950SNeel Natu  * tracks virtual-NMI blocking in the Guest Interruptibility-state field of
1191e5a1d950SNeel Natu  * the VMCS. An IRET instruction in VMX non-root operation will remove any
1192e5a1d950SNeel Natu  * virtual-NMI blocking.
1193e5a1d950SNeel Natu  *
1194e5a1d950SNeel Natu  * This unblocking occurs even if the IRET causes a fault. In this case the
1195e5a1d950SNeel Natu  * hypervisor needs to restore virtual-NMI blocking before resuming the guest.
1196e5a1d950SNeel Natu  */
1197e5a1d950SNeel Natu static void
1198e5a1d950SNeel Natu vmx_restore_nmi_blocking(struct vmx *vmx, int vcpuid)
1199e5a1d950SNeel Natu {
1200e5a1d950SNeel Natu 	uint32_t gi;
1201e5a1d950SNeel Natu 
1202e5a1d950SNeel Natu 	VCPU_CTR0(vmx->vm, vcpuid, "Restore Virtual-NMI blocking");
1203e5a1d950SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1204e5a1d950SNeel Natu 	gi |= VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1205e5a1d950SNeel Natu 	vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1206e5a1d950SNeel Natu }
1207e5a1d950SNeel Natu 
1208e5a1d950SNeel Natu static void
1209e5a1d950SNeel Natu vmx_clear_nmi_blocking(struct vmx *vmx, int vcpuid)
1210e5a1d950SNeel Natu {
1211e5a1d950SNeel Natu 	uint32_t gi;
1212e5a1d950SNeel Natu 
1213e5a1d950SNeel Natu 	VCPU_CTR0(vmx->vm, vcpuid, "Clear Virtual-NMI blocking");
1214e5a1d950SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1215e5a1d950SNeel Natu 	gi &= ~VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1216e5a1d950SNeel Natu 	vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1217e5a1d950SNeel Natu }
1218e5a1d950SNeel Natu 
1219366f6083SPeter Grehan static int
1220366f6083SPeter Grehan vmx_emulate_cr_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1221366f6083SPeter Grehan {
12223de83862SNeel Natu 	int cr, vmcs_guest_cr, vmcs_shadow_cr;
122380a902efSPeter Grehan 	uint64_t crval, regval, ones_mask, zeros_mask;
1224366f6083SPeter Grehan 	const struct vmxctx *vmxctx;
1225366f6083SPeter Grehan 
122639c21c2dSNeel Natu 	/* We only handle mov to %cr0 or %cr4 at this time */
122739c21c2dSNeel Natu 	if ((exitqual & 0xf0) != 0x00)
122839c21c2dSNeel Natu 		return (UNHANDLED);
122939c21c2dSNeel Natu 
123039c21c2dSNeel Natu 	cr = exitqual & 0xf;
123139c21c2dSNeel Natu 	if (cr != 0 && cr != 4)
1232366f6083SPeter Grehan 		return (UNHANDLED);
1233366f6083SPeter Grehan 
12346f0c167fSDimitry Andric 	regval = 0; /* silence gcc */
1235366f6083SPeter Grehan 	vmxctx = &vmx->ctx[vcpu];
1236366f6083SPeter Grehan 
1237366f6083SPeter Grehan 	/*
12383de83862SNeel Natu 	 * We must use vmcs_write() directly here because vmcs_setreg() will
1239366f6083SPeter Grehan 	 * call vmclear(vmcs) as a side-effect which we certainly don't want.
1240366f6083SPeter Grehan 	 */
1241366f6083SPeter Grehan 	switch ((exitqual >> 8) & 0xf) {
1242366f6083SPeter Grehan 	case 0:
1243366f6083SPeter Grehan 		regval = vmxctx->guest_rax;
1244366f6083SPeter Grehan 		break;
1245366f6083SPeter Grehan 	case 1:
1246366f6083SPeter Grehan 		regval = vmxctx->guest_rcx;
1247366f6083SPeter Grehan 		break;
1248366f6083SPeter Grehan 	case 2:
1249366f6083SPeter Grehan 		regval = vmxctx->guest_rdx;
1250366f6083SPeter Grehan 		break;
1251366f6083SPeter Grehan 	case 3:
1252366f6083SPeter Grehan 		regval = vmxctx->guest_rbx;
1253366f6083SPeter Grehan 		break;
1254366f6083SPeter Grehan 	case 4:
12553de83862SNeel Natu 		regval = vmcs_read(VMCS_GUEST_RSP);
1256366f6083SPeter Grehan 		break;
1257366f6083SPeter Grehan 	case 5:
1258366f6083SPeter Grehan 		regval = vmxctx->guest_rbp;
1259366f6083SPeter Grehan 		break;
1260366f6083SPeter Grehan 	case 6:
1261366f6083SPeter Grehan 		regval = vmxctx->guest_rsi;
1262366f6083SPeter Grehan 		break;
1263366f6083SPeter Grehan 	case 7:
1264366f6083SPeter Grehan 		regval = vmxctx->guest_rdi;
1265366f6083SPeter Grehan 		break;
1266366f6083SPeter Grehan 	case 8:
1267366f6083SPeter Grehan 		regval = vmxctx->guest_r8;
1268366f6083SPeter Grehan 		break;
1269366f6083SPeter Grehan 	case 9:
1270366f6083SPeter Grehan 		regval = vmxctx->guest_r9;
1271366f6083SPeter Grehan 		break;
1272366f6083SPeter Grehan 	case 10:
1273366f6083SPeter Grehan 		regval = vmxctx->guest_r10;
1274366f6083SPeter Grehan 		break;
1275366f6083SPeter Grehan 	case 11:
1276366f6083SPeter Grehan 		regval = vmxctx->guest_r11;
1277366f6083SPeter Grehan 		break;
1278366f6083SPeter Grehan 	case 12:
1279366f6083SPeter Grehan 		regval = vmxctx->guest_r12;
1280366f6083SPeter Grehan 		break;
1281366f6083SPeter Grehan 	case 13:
1282366f6083SPeter Grehan 		regval = vmxctx->guest_r13;
1283366f6083SPeter Grehan 		break;
1284366f6083SPeter Grehan 	case 14:
1285366f6083SPeter Grehan 		regval = vmxctx->guest_r14;
1286366f6083SPeter Grehan 		break;
1287366f6083SPeter Grehan 	case 15:
1288366f6083SPeter Grehan 		regval = vmxctx->guest_r15;
1289366f6083SPeter Grehan 		break;
1290366f6083SPeter Grehan 	}
1291366f6083SPeter Grehan 
129239c21c2dSNeel Natu 	if (cr == 0) {
129339c21c2dSNeel Natu 		ones_mask = cr0_ones_mask;
129439c21c2dSNeel Natu 		zeros_mask = cr0_zeros_mask;
129539c21c2dSNeel Natu 		vmcs_guest_cr = VMCS_GUEST_CR0;
1296aaaa0656SPeter Grehan 		vmcs_shadow_cr = VMCS_CR0_SHADOW;
129739c21c2dSNeel Natu 	} else {
129839c21c2dSNeel Natu 		ones_mask = cr4_ones_mask;
129939c21c2dSNeel Natu 		zeros_mask = cr4_zeros_mask;
130039c21c2dSNeel Natu 		vmcs_guest_cr = VMCS_GUEST_CR4;
1301aaaa0656SPeter Grehan 		vmcs_shadow_cr = VMCS_CR4_SHADOW;
130239c21c2dSNeel Natu 	}
13033de83862SNeel Natu 	vmcs_write(vmcs_shadow_cr, regval);
1304aaaa0656SPeter Grehan 
130580a902efSPeter Grehan 	crval = regval | ones_mask;
130680a902efSPeter Grehan 	crval &= ~zeros_mask;
13073de83862SNeel Natu 	vmcs_write(vmcs_guest_cr, crval);
1308366f6083SPeter Grehan 
130980a902efSPeter Grehan 	if (cr == 0 && regval & CR0_PG) {
131080a902efSPeter Grehan 		uint64_t efer, entry_ctls;
131180a902efSPeter Grehan 
131280a902efSPeter Grehan 		/*
131380a902efSPeter Grehan 		 * If CR0.PG is 1 and EFER.LME is 1 then EFER.LMA and
131480a902efSPeter Grehan 		 * the "IA-32e mode guest" bit in VM-entry control must be
131580a902efSPeter Grehan 		 * equal.
131680a902efSPeter Grehan 		 */
13173de83862SNeel Natu 		efer = vmcs_read(VMCS_GUEST_IA32_EFER);
131880a902efSPeter Grehan 		if (efer & EFER_LME) {
131980a902efSPeter Grehan 			efer |= EFER_LMA;
13203de83862SNeel Natu 			vmcs_write(VMCS_GUEST_IA32_EFER, efer);
13213de83862SNeel Natu 			entry_ctls = vmcs_read(VMCS_ENTRY_CTLS);
132280a902efSPeter Grehan 			entry_ctls |= VM_ENTRY_GUEST_LMA;
13233de83862SNeel Natu 			vmcs_write(VMCS_ENTRY_CTLS, entry_ctls);
132480a902efSPeter Grehan 		}
132580a902efSPeter Grehan 	}
132680a902efSPeter Grehan 
1327366f6083SPeter Grehan 	return (HANDLED);
1328366f6083SPeter Grehan }
1329366f6083SPeter Grehan 
1330366f6083SPeter Grehan static int
1331318224bbSNeel Natu ept_fault_type(uint64_t ept_qual)
1332a2da7af6SNeel Natu {
1333318224bbSNeel Natu 	int fault_type;
1334a2da7af6SNeel Natu 
1335318224bbSNeel Natu 	if (ept_qual & EPT_VIOLATION_DATA_WRITE)
1336318224bbSNeel Natu 		fault_type = VM_PROT_WRITE;
1337318224bbSNeel Natu 	else if (ept_qual & EPT_VIOLATION_INST_FETCH)
1338318224bbSNeel Natu 		fault_type = VM_PROT_EXECUTE;
1339318224bbSNeel Natu 	else
1340318224bbSNeel Natu 		fault_type= VM_PROT_READ;
1341318224bbSNeel Natu 
1342318224bbSNeel Natu 	return (fault_type);
1343318224bbSNeel Natu }
1344318224bbSNeel Natu 
1345318224bbSNeel Natu static boolean_t
1346318224bbSNeel Natu ept_emulation_fault(uint64_t ept_qual)
1347318224bbSNeel Natu {
1348318224bbSNeel Natu 	int read, write;
1349318224bbSNeel Natu 
1350318224bbSNeel Natu 	/* EPT fault on an instruction fetch doesn't make sense here */
1351a2da7af6SNeel Natu 	if (ept_qual & EPT_VIOLATION_INST_FETCH)
1352318224bbSNeel Natu 		return (FALSE);
1353a2da7af6SNeel Natu 
1354318224bbSNeel Natu 	/* EPT fault must be a read fault or a write fault */
1355a2da7af6SNeel Natu 	read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0;
1356a2da7af6SNeel Natu 	write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0;
13573b2b0011SPeter Grehan 	if ((read | write) == 0)
1358318224bbSNeel Natu 		return (FALSE);
1359a2da7af6SNeel Natu 
1360a2da7af6SNeel Natu 	/*
13613b2b0011SPeter Grehan 	 * The EPT violation must have been caused by accessing a
13623b2b0011SPeter Grehan 	 * guest-physical address that is a translation of a guest-linear
13633b2b0011SPeter Grehan 	 * address.
1364a2da7af6SNeel Natu 	 */
1365a2da7af6SNeel Natu 	if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 ||
1366a2da7af6SNeel Natu 	    (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) {
1367318224bbSNeel Natu 		return (FALSE);
1368a2da7af6SNeel Natu 	}
1369a2da7af6SNeel Natu 
1370318224bbSNeel Natu 	return (TRUE);
1371a2da7af6SNeel Natu }
1372a2da7af6SNeel Natu 
1373a2da7af6SNeel Natu static int
137488c4b8d1SNeel Natu vmx_handle_apic_write(struct vlapic *vlapic, uint64_t qual)
137588c4b8d1SNeel Natu {
137688c4b8d1SNeel Natu 	int error, handled, offset;
137788c4b8d1SNeel Natu 	bool retu;
137888c4b8d1SNeel Natu 
137988c4b8d1SNeel Natu 	if (!virtual_interrupt_delivery)
138088c4b8d1SNeel Natu 		return (UNHANDLED);
138188c4b8d1SNeel Natu 
138288c4b8d1SNeel Natu 	handled = 1;
138388c4b8d1SNeel Natu 	offset = APIC_WRITE_OFFSET(qual);
138488c4b8d1SNeel Natu 	switch (offset) {
138588c4b8d1SNeel Natu 	case APIC_OFFSET_ID:
138688c4b8d1SNeel Natu 		vlapic_id_write_handler(vlapic);
138788c4b8d1SNeel Natu 		break;
138888c4b8d1SNeel Natu 	case APIC_OFFSET_LDR:
138988c4b8d1SNeel Natu 		vlapic_ldr_write_handler(vlapic);
139088c4b8d1SNeel Natu 		break;
139188c4b8d1SNeel Natu 	case APIC_OFFSET_DFR:
139288c4b8d1SNeel Natu 		vlapic_dfr_write_handler(vlapic);
139388c4b8d1SNeel Natu 		break;
139488c4b8d1SNeel Natu 	case APIC_OFFSET_SVR:
139588c4b8d1SNeel Natu 		vlapic_svr_write_handler(vlapic);
139688c4b8d1SNeel Natu 		break;
139788c4b8d1SNeel Natu 	case APIC_OFFSET_ESR:
139888c4b8d1SNeel Natu 		vlapic_esr_write_handler(vlapic);
139988c4b8d1SNeel Natu 		break;
140088c4b8d1SNeel Natu 	case APIC_OFFSET_ICR_LOW:
140188c4b8d1SNeel Natu 		retu = false;
140288c4b8d1SNeel Natu 		error = vlapic_icrlo_write_handler(vlapic, &retu);
140388c4b8d1SNeel Natu 		if (error != 0 || retu)
140488c4b8d1SNeel Natu 			handled = 0;
140588c4b8d1SNeel Natu 		break;
140688c4b8d1SNeel Natu 	case APIC_OFFSET_CMCI_LVT:
140788c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
140888c4b8d1SNeel Natu 		vlapic_lvt_write_handler(vlapic, offset);
140988c4b8d1SNeel Natu 		break;
141088c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_ICR:
141188c4b8d1SNeel Natu 		vlapic_icrtmr_write_handler(vlapic);
141288c4b8d1SNeel Natu 		break;
141388c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_DCR:
141488c4b8d1SNeel Natu 		vlapic_dcr_write_handler(vlapic);
141588c4b8d1SNeel Natu 		break;
141688c4b8d1SNeel Natu 	default:
141788c4b8d1SNeel Natu 		handled = 0;
141888c4b8d1SNeel Natu 		break;
141988c4b8d1SNeel Natu 	}
142088c4b8d1SNeel Natu 	return (handled);
142188c4b8d1SNeel Natu }
142288c4b8d1SNeel Natu 
142388c4b8d1SNeel Natu static bool
142488c4b8d1SNeel Natu apic_access_fault(uint64_t gpa)
142588c4b8d1SNeel Natu {
142688c4b8d1SNeel Natu 
142788c4b8d1SNeel Natu 	if (virtual_interrupt_delivery &&
142888c4b8d1SNeel Natu 	    (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE))
142988c4b8d1SNeel Natu 		return (true);
143088c4b8d1SNeel Natu 	else
143188c4b8d1SNeel Natu 		return (false);
143288c4b8d1SNeel Natu }
143388c4b8d1SNeel Natu 
143488c4b8d1SNeel Natu static int
143588c4b8d1SNeel Natu vmx_handle_apic_access(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
143688c4b8d1SNeel Natu {
143788c4b8d1SNeel Natu 	uint64_t qual;
143888c4b8d1SNeel Natu 	int access_type, offset, allowed;
143988c4b8d1SNeel Natu 
144088c4b8d1SNeel Natu 	if (!virtual_interrupt_delivery)
144188c4b8d1SNeel Natu 		return (UNHANDLED);
144288c4b8d1SNeel Natu 
144388c4b8d1SNeel Natu 	qual = vmexit->u.vmx.exit_qualification;
144488c4b8d1SNeel Natu 	access_type = APIC_ACCESS_TYPE(qual);
144588c4b8d1SNeel Natu 	offset = APIC_ACCESS_OFFSET(qual);
144688c4b8d1SNeel Natu 
144788c4b8d1SNeel Natu 	allowed = 0;
144888c4b8d1SNeel Natu 	if (access_type == 0) {
144988c4b8d1SNeel Natu 		/*
145088c4b8d1SNeel Natu 		 * Read data access to the following registers is expected.
145188c4b8d1SNeel Natu 		 */
145288c4b8d1SNeel Natu 		switch (offset) {
145388c4b8d1SNeel Natu 		case APIC_OFFSET_APR:
145488c4b8d1SNeel Natu 		case APIC_OFFSET_PPR:
145588c4b8d1SNeel Natu 		case APIC_OFFSET_RRR:
145688c4b8d1SNeel Natu 		case APIC_OFFSET_CMCI_LVT:
145788c4b8d1SNeel Natu 		case APIC_OFFSET_TIMER_CCR:
145888c4b8d1SNeel Natu 			allowed = 1;
145988c4b8d1SNeel Natu 			break;
146088c4b8d1SNeel Natu 		default:
146188c4b8d1SNeel Natu 			break;
146288c4b8d1SNeel Natu 		}
146388c4b8d1SNeel Natu 	} else if (access_type == 1) {
146488c4b8d1SNeel Natu 		/*
146588c4b8d1SNeel Natu 		 * Write data access to the following registers is expected.
146688c4b8d1SNeel Natu 		 */
146788c4b8d1SNeel Natu 		switch (offset) {
146888c4b8d1SNeel Natu 		case APIC_OFFSET_VER:
146988c4b8d1SNeel Natu 		case APIC_OFFSET_APR:
147088c4b8d1SNeel Natu 		case APIC_OFFSET_PPR:
147188c4b8d1SNeel Natu 		case APIC_OFFSET_RRR:
147288c4b8d1SNeel Natu 		case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
147388c4b8d1SNeel Natu 		case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
147488c4b8d1SNeel Natu 		case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
147588c4b8d1SNeel Natu 		case APIC_OFFSET_CMCI_LVT:
147688c4b8d1SNeel Natu 		case APIC_OFFSET_TIMER_CCR:
147788c4b8d1SNeel Natu 			allowed = 1;
147888c4b8d1SNeel Natu 			break;
147988c4b8d1SNeel Natu 		default:
148088c4b8d1SNeel Natu 			break;
148188c4b8d1SNeel Natu 		}
148288c4b8d1SNeel Natu 	}
148388c4b8d1SNeel Natu 
148488c4b8d1SNeel Natu 	if (allowed) {
148588c4b8d1SNeel Natu 		vmexit->exitcode = VM_EXITCODE_INST_EMUL;
148688c4b8d1SNeel Natu 		vmexit->u.inst_emul.gpa = DEFAULT_APIC_BASE + offset;
148788c4b8d1SNeel Natu 		vmexit->u.inst_emul.gla = VIE_INVALID_GLA;
148888c4b8d1SNeel Natu 		vmexit->u.inst_emul.cr3 = vmcs_guest_cr3();
148988c4b8d1SNeel Natu 	}
149088c4b8d1SNeel Natu 
149188c4b8d1SNeel Natu 	/*
149288c4b8d1SNeel Natu 	 * Regardless of whether the APIC-access is allowed this handler
149388c4b8d1SNeel Natu 	 * always returns UNHANDLED:
149488c4b8d1SNeel Natu 	 * - if the access is allowed then it is handled by emulating the
149588c4b8d1SNeel Natu 	 *   instruction that caused the VM-exit (outside the critical section)
149688c4b8d1SNeel Natu 	 * - if the access is not allowed then it will be converted to an
149788c4b8d1SNeel Natu 	 *   exitcode of VM_EXITCODE_VMX and will be dealt with in userland.
149888c4b8d1SNeel Natu 	 */
149988c4b8d1SNeel Natu 	return (UNHANDLED);
150088c4b8d1SNeel Natu }
150188c4b8d1SNeel Natu 
150288c4b8d1SNeel Natu static int
1503366f6083SPeter Grehan vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1504366f6083SPeter Grehan {
1505f76fc5d4SNeel Natu 	int error, handled;
1506366f6083SPeter Grehan 	struct vmxctx *vmxctx;
150788c4b8d1SNeel Natu 	struct vlapic *vlapic;
1508e5a1d950SNeel Natu 	uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, reason;
15093de83862SNeel Natu 	uint64_t qual, gpa;
1510becd9849SNeel Natu 	bool retu;
1511366f6083SPeter Grehan 
1512160471d2SNeel Natu 	CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0);
1513c308b23bSNeel Natu 	CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0);
1514160471d2SNeel Natu 
1515366f6083SPeter Grehan 	handled = 0;
1516366f6083SPeter Grehan 	vmxctx = &vmx->ctx[vcpu];
15170492757cSNeel Natu 
1518366f6083SPeter Grehan 	qual = vmexit->u.vmx.exit_qualification;
1519318224bbSNeel Natu 	reason = vmexit->u.vmx.exit_reason;
1520366f6083SPeter Grehan 	vmexit->exitcode = VM_EXITCODE_BOGUS;
1521366f6083SPeter Grehan 
152261592433SNeel Natu 	vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1);
152361592433SNeel Natu 
1524318224bbSNeel Natu 	/*
1525318224bbSNeel Natu 	 * VM exits that could be triggered during event injection on the
1526318224bbSNeel Natu 	 * previous VM entry need to be handled specially by re-injecting
1527318224bbSNeel Natu 	 * the event.
1528318224bbSNeel Natu 	 *
1529318224bbSNeel Natu 	 * See "Information for VM Exits During Event Delivery" in Intel SDM
1530318224bbSNeel Natu 	 * for details.
1531318224bbSNeel Natu 	 */
1532318224bbSNeel Natu 	switch (reason) {
1533318224bbSNeel Natu 	case EXIT_REASON_EPT_FAULT:
1534318224bbSNeel Natu 	case EXIT_REASON_EPT_MISCONFIG:
153588c4b8d1SNeel Natu 	case EXIT_REASON_APIC_ACCESS:
1536318224bbSNeel Natu 	case EXIT_REASON_TASK_SWITCH:
1537318224bbSNeel Natu 	case EXIT_REASON_EXCEPTION:
1538318224bbSNeel Natu 		idtvec_info = vmcs_idt_vectoring_info();
1539318224bbSNeel Natu 		if (idtvec_info & VMCS_IDT_VEC_VALID) {
1540318224bbSNeel Natu 			idtvec_info &= ~(1 << 12); /* clear undefined bit */
15413de83862SNeel Natu 			vmcs_write(VMCS_ENTRY_INTR_INFO, idtvec_info);
1542318224bbSNeel Natu 			if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
1543318224bbSNeel Natu 				idtvec_err = vmcs_idt_vectoring_err();
15443de83862SNeel Natu 				vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR,
15453de83862SNeel Natu 				    idtvec_err);
1546318224bbSNeel Natu 			}
1547160471d2SNeel Natu 			/*
1548160471d2SNeel Natu 			 * If 'virtual NMIs' are being used and the VM-exit
1549160471d2SNeel Natu 			 * happened while injecting an NMI during the previous
1550160471d2SNeel Natu 			 * VM-entry, then clear "blocking by NMI" in the Guest
1551160471d2SNeel Natu 			 * Interruptibility-state.
1552160471d2SNeel Natu 			 */
1553160471d2SNeel Natu 			if ((idtvec_info & VMCS_INTR_T_MASK) ==
1554160471d2SNeel Natu 			    VMCS_INTR_T_NMI) {
1555e5a1d950SNeel Natu 				 vmx_clear_nmi_blocking(vmx, vcpu);
1556160471d2SNeel Natu 			}
15573de83862SNeel Natu 			vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
1558318224bbSNeel Natu 		}
1559318224bbSNeel Natu 	default:
1560e5a1d950SNeel Natu 		idtvec_info = 0;
1561318224bbSNeel Natu 		break;
1562318224bbSNeel Natu 	}
1563318224bbSNeel Natu 
1564318224bbSNeel Natu 	switch (reason) {
1565366f6083SPeter Grehan 	case EXIT_REASON_CR_ACCESS:
1566b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CR_ACCESS, 1);
1567366f6083SPeter Grehan 		handled = vmx_emulate_cr_access(vmx, vcpu, qual);
1568366f6083SPeter Grehan 		break;
1569366f6083SPeter Grehan 	case EXIT_REASON_RDMSR:
1570b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RDMSR, 1);
1571becd9849SNeel Natu 		retu = false;
1572366f6083SPeter Grehan 		ecx = vmxctx->guest_rcx;
1573becd9849SNeel Natu 		error = emulate_rdmsr(vmx->vm, vcpu, ecx, &retu);
1574b42206f3SNeel Natu 		if (error) {
1575366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_RDMSR;
1576366f6083SPeter Grehan 			vmexit->u.msr.code = ecx;
1577becd9849SNeel Natu 		} else if (!retu) {
1578b42206f3SNeel Natu 			handled = 1;
1579becd9849SNeel Natu 		} else {
1580becd9849SNeel Natu 			/* Return to userspace with a valid exitcode */
1581becd9849SNeel Natu 			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
1582becd9849SNeel Natu 			    ("emulate_wrmsr retu with bogus exitcode"));
1583becd9849SNeel Natu 		}
1584366f6083SPeter Grehan 		break;
1585366f6083SPeter Grehan 	case EXIT_REASON_WRMSR:
1586b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_WRMSR, 1);
1587becd9849SNeel Natu 		retu = false;
1588366f6083SPeter Grehan 		eax = vmxctx->guest_rax;
1589366f6083SPeter Grehan 		ecx = vmxctx->guest_rcx;
1590366f6083SPeter Grehan 		edx = vmxctx->guest_rdx;
1591b42206f3SNeel Natu 		error = emulate_wrmsr(vmx->vm, vcpu, ecx,
1592becd9849SNeel Natu 		    (uint64_t)edx << 32 | eax, &retu);
1593b42206f3SNeel Natu 		if (error) {
1594366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_WRMSR;
1595366f6083SPeter Grehan 			vmexit->u.msr.code = ecx;
1596366f6083SPeter Grehan 			vmexit->u.msr.wval = (uint64_t)edx << 32 | eax;
1597becd9849SNeel Natu 		} else if (!retu) {
1598b42206f3SNeel Natu 			handled = 1;
1599becd9849SNeel Natu 		} else {
1600becd9849SNeel Natu 			/* Return to userspace with a valid exitcode */
1601becd9849SNeel Natu 			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
1602becd9849SNeel Natu 			    ("emulate_wrmsr retu with bogus exitcode"));
1603becd9849SNeel Natu 		}
1604366f6083SPeter Grehan 		break;
1605366f6083SPeter Grehan 	case EXIT_REASON_HLT:
1606f76fc5d4SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_HLT, 1);
1607366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_HLT;
16083de83862SNeel Natu 		vmexit->u.hlt.rflags = vmcs_read(VMCS_GUEST_RFLAGS);
1609366f6083SPeter Grehan 		break;
1610366f6083SPeter Grehan 	case EXIT_REASON_MTF:
1611b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_MTRAP, 1);
1612366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_MTRAP;
1613366f6083SPeter Grehan 		break;
1614366f6083SPeter Grehan 	case EXIT_REASON_PAUSE:
1615b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_PAUSE, 1);
1616366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_PAUSE;
1617366f6083SPeter Grehan 		break;
1618366f6083SPeter Grehan 	case EXIT_REASON_INTR_WINDOW:
1619b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INTR_WINDOW, 1);
1620366f6083SPeter Grehan 		vmx_clear_int_window_exiting(vmx, vcpu);
1621b5aaf7b2SNeel Natu 		return (1);
1622366f6083SPeter Grehan 	case EXIT_REASON_EXT_INTR:
1623366f6083SPeter Grehan 		/*
1624366f6083SPeter Grehan 		 * External interrupts serve only to cause VM exits and allow
1625366f6083SPeter Grehan 		 * the host interrupt handler to run.
1626366f6083SPeter Grehan 		 *
1627366f6083SPeter Grehan 		 * If this external interrupt triggers a virtual interrupt
1628366f6083SPeter Grehan 		 * to a VM, then that state will be recorded by the
1629366f6083SPeter Grehan 		 * host interrupt handler in the VM's softc. We will inject
1630366f6083SPeter Grehan 		 * this virtual interrupt during the subsequent VM enter.
1631366f6083SPeter Grehan 		 */
1632f7d47425SNeel Natu 		intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
1633160471d2SNeel Natu 		KASSERT((intr_info & VMCS_INTR_VALID) != 0 &&
1634160471d2SNeel Natu 		    (intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_HWINTR,
1635f7d47425SNeel Natu 		    ("VM exit interruption info invalid: %#x", intr_info));
1636f7d47425SNeel Natu 		vmx_trigger_hostintr(intr_info & 0xff);
1637366f6083SPeter Grehan 
1638366f6083SPeter Grehan 		/*
1639366f6083SPeter Grehan 		 * This is special. We want to treat this as an 'handled'
1640366f6083SPeter Grehan 		 * VM-exit but not increment the instruction pointer.
1641366f6083SPeter Grehan 		 */
1642366f6083SPeter Grehan 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1);
1643366f6083SPeter Grehan 		return (1);
1644366f6083SPeter Grehan 	case EXIT_REASON_NMI_WINDOW:
1645366f6083SPeter Grehan 		/* Exit to allow the pending virtual NMI to be injected */
164648b2d828SNeel Natu 		if (vm_nmi_pending(vmx->vm, vcpu))
164748b2d828SNeel Natu 			vmx_inject_nmi(vmx, vcpu);
1648366f6083SPeter Grehan 		vmx_clear_nmi_window_exiting(vmx, vcpu);
164948b2d828SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NMI_WINDOW, 1);
1650366f6083SPeter Grehan 		return (1);
1651366f6083SPeter Grehan 	case EXIT_REASON_INOUT:
1652b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INOUT, 1);
1653366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_INOUT;
1654366f6083SPeter Grehan 		vmexit->u.inout.bytes = (qual & 0x7) + 1;
1655366f6083SPeter Grehan 		vmexit->u.inout.in = (qual & 0x8) ? 1 : 0;
1656366f6083SPeter Grehan 		vmexit->u.inout.string = (qual & 0x10) ? 1 : 0;
1657366f6083SPeter Grehan 		vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0;
1658366f6083SPeter Grehan 		vmexit->u.inout.port = (uint16_t)(qual >> 16);
1659366f6083SPeter Grehan 		vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax);
1660366f6083SPeter Grehan 		break;
1661366f6083SPeter Grehan 	case EXIT_REASON_CPUID:
1662b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CPUID, 1);
1663a2da7af6SNeel Natu 		handled = vmx_handle_cpuid(vmx->vm, vcpu, vmxctx);
1664366f6083SPeter Grehan 		break;
1665e5a1d950SNeel Natu 	case EXIT_REASON_EXCEPTION:
1666c308b23bSNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXCEPTION, 1);
1667e5a1d950SNeel Natu 		intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
1668e5a1d950SNeel Natu 		KASSERT((intr_info & VMCS_INTR_VALID) != 0,
1669e5a1d950SNeel Natu 		    ("VM exit interruption info invalid: %#x", intr_info));
1670c308b23bSNeel Natu 
1671e5a1d950SNeel Natu 		/*
1672e5a1d950SNeel Natu 		 * If Virtual NMIs control is 1 and the VM-exit is due to a
1673e5a1d950SNeel Natu 		 * fault encountered during the execution of IRET then we must
1674e5a1d950SNeel Natu 		 * restore the state of "virtual-NMI blocking" before resuming
1675e5a1d950SNeel Natu 		 * the guest.
1676e5a1d950SNeel Natu 		 *
1677e5a1d950SNeel Natu 		 * See "Resuming Guest Software after Handling an Exception".
1678e5a1d950SNeel Natu 		 */
1679e5a1d950SNeel Natu 		if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
1680e5a1d950SNeel Natu 		    (intr_info & 0xff) != IDT_DF &&
1681e5a1d950SNeel Natu 		    (intr_info & EXIT_QUAL_NMIUDTI) != 0)
1682e5a1d950SNeel Natu 			vmx_restore_nmi_blocking(vmx, vcpu);
1683c308b23bSNeel Natu 
1684c308b23bSNeel Natu 		/*
1685c308b23bSNeel Natu 		 * If the NMI-exiting VM execution control is set to '1'
1686c308b23bSNeel Natu 		 * then an NMI in non-root operation causes a VM-exit.
1687c308b23bSNeel Natu 		 * NMI blocking is in effect for this logical processor so
1688c308b23bSNeel Natu 		 * it is sufficient to simply vector to the NMI handler via
1689c308b23bSNeel Natu 		 * a software interrupt.
1690c308b23bSNeel Natu 		 */
1691c308b23bSNeel Natu 		if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI) {
1692c308b23bSNeel Natu 			KASSERT((intr_info & 0xff) == IDT_NMI, ("VM exit due "
1693c308b23bSNeel Natu 			    "to NMI has invalid vector: %#x", intr_info));
1694c308b23bSNeel Natu 			VCPU_CTR0(vmx->vm, vcpu, "Vectoring to NMI handler");
1695c308b23bSNeel Natu 			__asm __volatile("int $2");
1696c308b23bSNeel Natu 			return (1);
1697c308b23bSNeel Natu 		}
1698e5a1d950SNeel Natu 		break;
1699cd942e0fSPeter Grehan 	case EXIT_REASON_EPT_FAULT:
1700b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EPT_FAULT, 1);
1701318224bbSNeel Natu 		/*
1702318224bbSNeel Natu 		 * If 'gpa' lies within the address space allocated to
1703318224bbSNeel Natu 		 * memory then this must be a nested page fault otherwise
1704318224bbSNeel Natu 		 * this must be an instruction that accesses MMIO space.
1705318224bbSNeel Natu 		 */
1706a2da7af6SNeel Natu 		gpa = vmcs_gpa();
170788c4b8d1SNeel Natu 		if (vm_mem_allocated(vmx->vm, gpa) || apic_access_fault(gpa)) {
1708cd942e0fSPeter Grehan 			vmexit->exitcode = VM_EXITCODE_PAGING;
170913ec9371SPeter Grehan 			vmexit->u.paging.gpa = gpa;
1710318224bbSNeel Natu 			vmexit->u.paging.fault_type = ept_fault_type(qual);
1711318224bbSNeel Natu 		} else if (ept_emulation_fault(qual)) {
1712318224bbSNeel Natu 			vmexit->exitcode = VM_EXITCODE_INST_EMUL;
1713318224bbSNeel Natu 			vmexit->u.inst_emul.gpa = gpa;
1714318224bbSNeel Natu 			vmexit->u.inst_emul.gla = vmcs_gla();
1715318224bbSNeel Natu 			vmexit->u.inst_emul.cr3 = vmcs_guest_cr3();
1716a2da7af6SNeel Natu 		}
1717e5a1d950SNeel Natu 		/*
1718e5a1d950SNeel Natu 		 * If Virtual NMIs control is 1 and the VM-exit is due to an
1719e5a1d950SNeel Natu 		 * EPT fault during the execution of IRET then we must restore
1720e5a1d950SNeel Natu 		 * the state of "virtual-NMI blocking" before resuming.
1721e5a1d950SNeel Natu 		 *
1722e5a1d950SNeel Natu 		 * See description of "NMI unblocking due to IRET" in
1723e5a1d950SNeel Natu 		 * "Exit Qualification for EPT Violations".
1724e5a1d950SNeel Natu 		 */
1725e5a1d950SNeel Natu 		if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
1726e5a1d950SNeel Natu 		    (qual & EXIT_QUAL_NMIUDTI) != 0)
1727e5a1d950SNeel Natu 			vmx_restore_nmi_blocking(vmx, vcpu);
1728cd942e0fSPeter Grehan 		break;
1729*30b94db8SNeel Natu 	case EXIT_REASON_VIRTUALIZED_EOI:
1730*30b94db8SNeel Natu 		vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI;
1731*30b94db8SNeel Natu 		vmexit->u.ioapic_eoi.vector = qual & 0xFF;
1732*30b94db8SNeel Natu 		vmexit->inst_length = 0;	/* trap-like */
1733*30b94db8SNeel Natu 		break;
173488c4b8d1SNeel Natu 	case EXIT_REASON_APIC_ACCESS:
173588c4b8d1SNeel Natu 		handled = vmx_handle_apic_access(vmx, vcpu, vmexit);
173688c4b8d1SNeel Natu 		break;
173788c4b8d1SNeel Natu 	case EXIT_REASON_APIC_WRITE:
173888c4b8d1SNeel Natu 		/*
173988c4b8d1SNeel Natu 		 * APIC-write VM exit is trap-like so the %rip is already
174088c4b8d1SNeel Natu 		 * pointing to the next instruction.
174188c4b8d1SNeel Natu 		 */
174288c4b8d1SNeel Natu 		vmexit->inst_length = 0;
174388c4b8d1SNeel Natu 		vlapic = vm_lapic(vmx->vm, vcpu);
174488c4b8d1SNeel Natu 		handled = vmx_handle_apic_write(vlapic, qual);
174588c4b8d1SNeel Natu 		break;
1746366f6083SPeter Grehan 	default:
1747b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_UNKNOWN, 1);
1748366f6083SPeter Grehan 		break;
1749366f6083SPeter Grehan 	}
1750366f6083SPeter Grehan 
1751366f6083SPeter Grehan 	if (handled) {
1752366f6083SPeter Grehan 		/*
1753366f6083SPeter Grehan 		 * It is possible that control is returned to userland
1754366f6083SPeter Grehan 		 * even though we were able to handle the VM exit in the
1755eeefa4e4SNeel Natu 		 * kernel.
1756366f6083SPeter Grehan 		 *
1757366f6083SPeter Grehan 		 * In such a case we want to make sure that the userland
1758366f6083SPeter Grehan 		 * restarts guest execution at the instruction *after*
1759366f6083SPeter Grehan 		 * the one we just processed. Therefore we update the
1760366f6083SPeter Grehan 		 * guest rip in the VMCS and in 'vmexit'.
1761366f6083SPeter Grehan 		 */
1762366f6083SPeter Grehan 		vmexit->rip += vmexit->inst_length;
1763366f6083SPeter Grehan 		vmexit->inst_length = 0;
17643de83862SNeel Natu 		vmcs_write(VMCS_GUEST_RIP, vmexit->rip);
1765366f6083SPeter Grehan 	} else {
1766366f6083SPeter Grehan 		if (vmexit->exitcode == VM_EXITCODE_BOGUS) {
1767366f6083SPeter Grehan 			/*
1768366f6083SPeter Grehan 			 * If this VM exit was not claimed by anybody then
1769366f6083SPeter Grehan 			 * treat it as a generic VMX exit.
1770366f6083SPeter Grehan 			 */
1771366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_VMX;
17720492757cSNeel Natu 			vmexit->u.vmx.status = VM_SUCCESS;
1773c308b23bSNeel Natu 			vmexit->u.vmx.inst_type = 0;
1774c308b23bSNeel Natu 			vmexit->u.vmx.inst_error = 0;
1775366f6083SPeter Grehan 		} else {
1776366f6083SPeter Grehan 			/*
1777366f6083SPeter Grehan 			 * The exitcode and collateral have been populated.
1778366f6083SPeter Grehan 			 * The VM exit will be processed further in userland.
1779366f6083SPeter Grehan 			 */
1780366f6083SPeter Grehan 		}
1781366f6083SPeter Grehan 	}
1782366f6083SPeter Grehan 	return (handled);
1783366f6083SPeter Grehan }
1784366f6083SPeter Grehan 
17850492757cSNeel Natu static __inline int
17860492757cSNeel Natu vmx_exit_astpending(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1787366f6083SPeter Grehan {
17880492757cSNeel Natu 
17890492757cSNeel Natu 	vmexit->rip = vmcs_guest_rip();
17900492757cSNeel Natu 	vmexit->inst_length = 0;
17910492757cSNeel Natu 	vmexit->exitcode = VM_EXITCODE_BOGUS;
17920492757cSNeel Natu 	vmx_astpending_trace(vmx, vcpu, vmexit->rip);
17930492757cSNeel Natu 	vmm_stat_incr(vmx->vm, vcpu, VMEXIT_ASTPENDING, 1);
17940492757cSNeel Natu 
17950492757cSNeel Natu 	return (HANDLED);
17960492757cSNeel Natu }
17970492757cSNeel Natu 
17980492757cSNeel Natu static __inline int
17995b8a8cd1SNeel Natu vmx_exit_rendezvous(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
18005b8a8cd1SNeel Natu {
18015b8a8cd1SNeel Natu 
18025b8a8cd1SNeel Natu 	vmexit->rip = vmcs_guest_rip();
18035b8a8cd1SNeel Natu 	vmexit->inst_length = 0;
18045b8a8cd1SNeel Natu 	vmexit->exitcode = VM_EXITCODE_RENDEZVOUS;
18055b8a8cd1SNeel Natu 	vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RENDEZVOUS, 1);
18065b8a8cd1SNeel Natu 
18075b8a8cd1SNeel Natu 	return (UNHANDLED);
18085b8a8cd1SNeel Natu }
18095b8a8cd1SNeel Natu 
18105b8a8cd1SNeel Natu static __inline int
18110492757cSNeel Natu vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit)
18120492757cSNeel Natu {
18130492757cSNeel Natu 
18140492757cSNeel Natu 	KASSERT(vmxctx->inst_fail_status != VM_SUCCESS,
18150492757cSNeel Natu 	    ("vmx_exit_inst_error: invalid inst_fail_status %d",
18160492757cSNeel Natu 	    vmxctx->inst_fail_status));
18170492757cSNeel Natu 
18180492757cSNeel Natu 	vmexit->inst_length = 0;
18190492757cSNeel Natu 	vmexit->exitcode = VM_EXITCODE_VMX;
18200492757cSNeel Natu 	vmexit->u.vmx.status = vmxctx->inst_fail_status;
18210492757cSNeel Natu 	vmexit->u.vmx.inst_error = vmcs_instruction_error();
18220492757cSNeel Natu 	vmexit->u.vmx.exit_reason = ~0;
18230492757cSNeel Natu 	vmexit->u.vmx.exit_qualification = ~0;
18240492757cSNeel Natu 
18250492757cSNeel Natu 	switch (rc) {
18260492757cSNeel Natu 	case VMX_VMRESUME_ERROR:
18270492757cSNeel Natu 	case VMX_VMLAUNCH_ERROR:
18280492757cSNeel Natu 	case VMX_INVEPT_ERROR:
18290492757cSNeel Natu 		vmexit->u.vmx.inst_type = rc;
18300492757cSNeel Natu 		break;
18310492757cSNeel Natu 	default:
18320492757cSNeel Natu 		panic("vm_exit_inst_error: vmx_enter_guest returned %d", rc);
18330492757cSNeel Natu 	}
18340492757cSNeel Natu 
18350492757cSNeel Natu 	return (UNHANDLED);
18360492757cSNeel Natu }
18370492757cSNeel Natu 
18380492757cSNeel Natu static int
18395b8a8cd1SNeel Natu vmx_run(void *arg, int vcpu, register_t startrip, pmap_t pmap,
18405b8a8cd1SNeel Natu     void *rendezvous_cookie)
18410492757cSNeel Natu {
18420492757cSNeel Natu 	int rc, handled, launched;
1843366f6083SPeter Grehan 	struct vmx *vmx;
18445b8a8cd1SNeel Natu 	struct vm *vm;
1845366f6083SPeter Grehan 	struct vmxctx *vmxctx;
1846366f6083SPeter Grehan 	struct vmcs *vmcs;
184798ed632cSNeel Natu 	struct vm_exit *vmexit;
1848de5ea6b6SNeel Natu 	struct vlapic *vlapic;
184979c59630SNeel Natu 	uint64_t rip;
185079c59630SNeel Natu 	uint32_t exit_reason;
1851366f6083SPeter Grehan 
1852366f6083SPeter Grehan 	vmx = arg;
18535b8a8cd1SNeel Natu 	vm = vmx->vm;
1854366f6083SPeter Grehan 	vmcs = &vmx->vmcs[vcpu];
1855366f6083SPeter Grehan 	vmxctx = &vmx->ctx[vcpu];
18565b8a8cd1SNeel Natu 	vlapic = vm_lapic(vm, vcpu);
18575b8a8cd1SNeel Natu 	vmexit = vm_exitinfo(vm, vcpu);
18580492757cSNeel Natu 	launched = 0;
185998ed632cSNeel Natu 
1860318224bbSNeel Natu 	KASSERT(vmxctx->pmap == pmap,
1861318224bbSNeel Natu 	    ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap));
1862318224bbSNeel Natu 	KASSERT(vmxctx->eptp == vmx->eptp,
1863318224bbSNeel Natu 	    ("eptp %p different than ctx eptp %#lx", eptp, vmxctx->eptp));
1864318224bbSNeel Natu 
1865366f6083SPeter Grehan 	VMPTRLD(vmcs);
1866366f6083SPeter Grehan 
1867366f6083SPeter Grehan 	/*
1868366f6083SPeter Grehan 	 * XXX
1869366f6083SPeter Grehan 	 * We do this every time because we may setup the virtual machine
1870366f6083SPeter Grehan 	 * from a different process than the one that actually runs it.
1871366f6083SPeter Grehan 	 *
1872366f6083SPeter Grehan 	 * If the life of a virtual machine was spent entirely in the context
1873c847a506SNeel Natu 	 * of a single process we could do this once in vmx_vminit().
1874366f6083SPeter Grehan 	 */
18753de83862SNeel Natu 	vmcs_write(VMCS_HOST_CR3, rcr3());
1876366f6083SPeter Grehan 
18770492757cSNeel Natu 	vmcs_write(VMCS_GUEST_RIP, startrip);
18780492757cSNeel Natu 	vmx_set_pcpu_defaults(vmx, vcpu);
1879366f6083SPeter Grehan 	do {
18800492757cSNeel Natu 		/*
18810492757cSNeel Natu 		 * Interrupts are disabled from this point on until the
18820492757cSNeel Natu 		 * guest starts executing. This is done for the following
18830492757cSNeel Natu 		 * reasons:
18840492757cSNeel Natu 		 *
18850492757cSNeel Natu 		 * If an AST is asserted on this thread after the check below,
18860492757cSNeel Natu 		 * then the IPI_AST notification will not be lost, because it
18870492757cSNeel Natu 		 * will cause a VM exit due to external interrupt as soon as
18880492757cSNeel Natu 		 * the guest state is loaded.
18890492757cSNeel Natu 		 *
18900492757cSNeel Natu 		 * A posted interrupt after 'vmx_inject_interrupts()' will
18910492757cSNeel Natu 		 * not be "lost" because it will be held pending in the host
18920492757cSNeel Natu 		 * APIC because interrupts are disabled. The pending interrupt
18930492757cSNeel Natu 		 * will be recognized as soon as the guest state is loaded.
18940492757cSNeel Natu 		 *
18950492757cSNeel Natu 		 * The same reasoning applies to the IPI generated by
18960492757cSNeel Natu 		 * pmap_invalidate_ept().
18970492757cSNeel Natu 		 */
18980492757cSNeel Natu 		disable_intr();
18990492757cSNeel Natu 		if (curthread->td_flags & (TDF_ASTPENDING | TDF_NEEDRESCHED)) {
19000492757cSNeel Natu 			enable_intr();
19010492757cSNeel Natu 			handled = vmx_exit_astpending(vmx, vcpu, vmexit);
19020492757cSNeel Natu 			break;
19030492757cSNeel Natu 		}
19040492757cSNeel Natu 
19055b8a8cd1SNeel Natu 		if (vcpu_rendezvous_pending(rendezvous_cookie)) {
19065b8a8cd1SNeel Natu 			enable_intr();
19075b8a8cd1SNeel Natu 			handled = vmx_exit_rendezvous(vmx, vcpu, vmexit);
19085b8a8cd1SNeel Natu 			break;
19095b8a8cd1SNeel Natu 		}
19105b8a8cd1SNeel Natu 
1911de5ea6b6SNeel Natu 		vmx_inject_interrupts(vmx, vcpu, vlapic);
1912366f6083SPeter Grehan 		vmx_run_trace(vmx, vcpu);
19130492757cSNeel Natu 		rc = vmx_enter_guest(vmxctx, launched);
191479c59630SNeel Natu 
1915366f6083SPeter Grehan 		enable_intr();
191679c59630SNeel Natu 
191779c59630SNeel Natu 		/* Collect some information for VM exit processing */
191879c59630SNeel Natu 		vmexit->rip = rip = vmcs_guest_rip();
191979c59630SNeel Natu 		vmexit->inst_length = vmexit_instruction_length();
192079c59630SNeel Natu 		vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason();
192179c59630SNeel Natu 		vmexit->u.vmx.exit_qualification = vmcs_exit_qualification();
192279c59630SNeel Natu 
19230492757cSNeel Natu 		if (rc == VMX_GUEST_VMEXIT) {
19240492757cSNeel Natu 			launched = 1;
19250492757cSNeel Natu 			handled = vmx_exit_process(vmx, vcpu, vmexit);
19260492757cSNeel Natu 		} else {
19270492757cSNeel Natu 			handled = vmx_exit_inst_error(vmxctx, rc, vmexit);
1928eeefa4e4SNeel Natu 		}
1929366f6083SPeter Grehan 
193079c59630SNeel Natu 		vmx_exit_trace(vmx, vcpu, rip, exit_reason, handled);
1931eeefa4e4SNeel Natu 	} while (handled);
1932366f6083SPeter Grehan 
1933366f6083SPeter Grehan 	/*
1934366f6083SPeter Grehan 	 * If a VM exit has been handled then the exitcode must be BOGUS
1935366f6083SPeter Grehan 	 * If a VM exit is not handled then the exitcode must not be BOGUS
1936366f6083SPeter Grehan 	 */
1937366f6083SPeter Grehan 	if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) ||
1938366f6083SPeter Grehan 	    (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) {
1939366f6083SPeter Grehan 		panic("Mismatch between handled (%d) and exitcode (%d)",
1940366f6083SPeter Grehan 		      handled, vmexit->exitcode);
1941366f6083SPeter Grehan 	}
1942366f6083SPeter Grehan 
1943b5aaf7b2SNeel Natu 	if (!handled)
19445b8a8cd1SNeel Natu 		vmm_stat_incr(vm, vcpu, VMEXIT_USERSPACE, 1);
1945b5aaf7b2SNeel Natu 
19465b8a8cd1SNeel Natu 	VCPU_CTR1(vm, vcpu, "returning from vmx_run: exitcode %d",
19470492757cSNeel Natu 	    vmexit->exitcode);
1948366f6083SPeter Grehan 
1949366f6083SPeter Grehan 	VMCLEAR(vmcs);
1950366f6083SPeter Grehan 	return (0);
1951366f6083SPeter Grehan }
1952366f6083SPeter Grehan 
1953366f6083SPeter Grehan static void
1954366f6083SPeter Grehan vmx_vmcleanup(void *arg)
1955366f6083SPeter Grehan {
195645e51299SNeel Natu 	int i, error;
1957366f6083SPeter Grehan 	struct vmx *vmx = arg;
1958366f6083SPeter Grehan 
195988c4b8d1SNeel Natu 	if (virtual_interrupt_delivery)
196088c4b8d1SNeel Natu 		vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
196188c4b8d1SNeel Natu 
196245e51299SNeel Natu 	for (i = 0; i < VM_MAXCPU; i++)
196345e51299SNeel Natu 		vpid_free(vmx->state[i].vpid);
196445e51299SNeel Natu 
1965366f6083SPeter Grehan 	/*
1966366f6083SPeter Grehan 	 * XXXSMP we also need to clear the VMCS active on the other vcpus.
1967366f6083SPeter Grehan 	 */
1968366f6083SPeter Grehan 	error = vmclear(&vmx->vmcs[0]);
1969366f6083SPeter Grehan 	if (error != 0)
1970366f6083SPeter Grehan 		panic("vmx_vmcleanup: vmclear error %d on vcpu 0", error);
1971366f6083SPeter Grehan 
1972366f6083SPeter Grehan 	free(vmx, M_VMX);
1973366f6083SPeter Grehan 
1974366f6083SPeter Grehan 	return;
1975366f6083SPeter Grehan }
1976366f6083SPeter Grehan 
1977366f6083SPeter Grehan static register_t *
1978366f6083SPeter Grehan vmxctx_regptr(struct vmxctx *vmxctx, int reg)
1979366f6083SPeter Grehan {
1980366f6083SPeter Grehan 
1981366f6083SPeter Grehan 	switch (reg) {
1982366f6083SPeter Grehan 	case VM_REG_GUEST_RAX:
1983366f6083SPeter Grehan 		return (&vmxctx->guest_rax);
1984366f6083SPeter Grehan 	case VM_REG_GUEST_RBX:
1985366f6083SPeter Grehan 		return (&vmxctx->guest_rbx);
1986366f6083SPeter Grehan 	case VM_REG_GUEST_RCX:
1987366f6083SPeter Grehan 		return (&vmxctx->guest_rcx);
1988366f6083SPeter Grehan 	case VM_REG_GUEST_RDX:
1989366f6083SPeter Grehan 		return (&vmxctx->guest_rdx);
1990366f6083SPeter Grehan 	case VM_REG_GUEST_RSI:
1991366f6083SPeter Grehan 		return (&vmxctx->guest_rsi);
1992366f6083SPeter Grehan 	case VM_REG_GUEST_RDI:
1993366f6083SPeter Grehan 		return (&vmxctx->guest_rdi);
1994366f6083SPeter Grehan 	case VM_REG_GUEST_RBP:
1995366f6083SPeter Grehan 		return (&vmxctx->guest_rbp);
1996366f6083SPeter Grehan 	case VM_REG_GUEST_R8:
1997366f6083SPeter Grehan 		return (&vmxctx->guest_r8);
1998366f6083SPeter Grehan 	case VM_REG_GUEST_R9:
1999366f6083SPeter Grehan 		return (&vmxctx->guest_r9);
2000366f6083SPeter Grehan 	case VM_REG_GUEST_R10:
2001366f6083SPeter Grehan 		return (&vmxctx->guest_r10);
2002366f6083SPeter Grehan 	case VM_REG_GUEST_R11:
2003366f6083SPeter Grehan 		return (&vmxctx->guest_r11);
2004366f6083SPeter Grehan 	case VM_REG_GUEST_R12:
2005366f6083SPeter Grehan 		return (&vmxctx->guest_r12);
2006366f6083SPeter Grehan 	case VM_REG_GUEST_R13:
2007366f6083SPeter Grehan 		return (&vmxctx->guest_r13);
2008366f6083SPeter Grehan 	case VM_REG_GUEST_R14:
2009366f6083SPeter Grehan 		return (&vmxctx->guest_r14);
2010366f6083SPeter Grehan 	case VM_REG_GUEST_R15:
2011366f6083SPeter Grehan 		return (&vmxctx->guest_r15);
2012366f6083SPeter Grehan 	default:
2013366f6083SPeter Grehan 		break;
2014366f6083SPeter Grehan 	}
2015366f6083SPeter Grehan 	return (NULL);
2016366f6083SPeter Grehan }
2017366f6083SPeter Grehan 
2018366f6083SPeter Grehan static int
2019366f6083SPeter Grehan vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval)
2020366f6083SPeter Grehan {
2021366f6083SPeter Grehan 	register_t *regp;
2022366f6083SPeter Grehan 
2023366f6083SPeter Grehan 	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
2024366f6083SPeter Grehan 		*retval = *regp;
2025366f6083SPeter Grehan 		return (0);
2026366f6083SPeter Grehan 	} else
2027366f6083SPeter Grehan 		return (EINVAL);
2028366f6083SPeter Grehan }
2029366f6083SPeter Grehan 
2030366f6083SPeter Grehan static int
2031366f6083SPeter Grehan vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val)
2032366f6083SPeter Grehan {
2033366f6083SPeter Grehan 	register_t *regp;
2034366f6083SPeter Grehan 
2035366f6083SPeter Grehan 	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
2036366f6083SPeter Grehan 		*regp = val;
2037366f6083SPeter Grehan 		return (0);
2038366f6083SPeter Grehan 	} else
2039366f6083SPeter Grehan 		return (EINVAL);
2040366f6083SPeter Grehan }
2041366f6083SPeter Grehan 
2042366f6083SPeter Grehan static int
2043aaaa0656SPeter Grehan vmx_shadow_reg(int reg)
2044aaaa0656SPeter Grehan {
2045aaaa0656SPeter Grehan 	int shreg;
2046aaaa0656SPeter Grehan 
2047aaaa0656SPeter Grehan 	shreg = -1;
2048aaaa0656SPeter Grehan 
2049aaaa0656SPeter Grehan 	switch (reg) {
2050aaaa0656SPeter Grehan 	case VM_REG_GUEST_CR0:
2051aaaa0656SPeter Grehan 		shreg = VMCS_CR0_SHADOW;
2052aaaa0656SPeter Grehan                 break;
2053aaaa0656SPeter Grehan         case VM_REG_GUEST_CR4:
2054aaaa0656SPeter Grehan 		shreg = VMCS_CR4_SHADOW;
2055aaaa0656SPeter Grehan 		break;
2056aaaa0656SPeter Grehan 	default:
2057aaaa0656SPeter Grehan 		break;
2058aaaa0656SPeter Grehan 	}
2059aaaa0656SPeter Grehan 
2060aaaa0656SPeter Grehan 	return (shreg);
2061aaaa0656SPeter Grehan }
2062aaaa0656SPeter Grehan 
2063aaaa0656SPeter Grehan static int
2064366f6083SPeter Grehan vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval)
2065366f6083SPeter Grehan {
2066d3c11f40SPeter Grehan 	int running, hostcpu;
2067366f6083SPeter Grehan 	struct vmx *vmx = arg;
2068366f6083SPeter Grehan 
2069d3c11f40SPeter Grehan 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
2070d3c11f40SPeter Grehan 	if (running && hostcpu != curcpu)
2071d3c11f40SPeter Grehan 		panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu);
2072d3c11f40SPeter Grehan 
2073366f6083SPeter Grehan 	if (vmxctx_getreg(&vmx->ctx[vcpu], reg, retval) == 0)
2074366f6083SPeter Grehan 		return (0);
2075366f6083SPeter Grehan 
2076d3c11f40SPeter Grehan 	return (vmcs_getreg(&vmx->vmcs[vcpu], running, reg, retval));
2077366f6083SPeter Grehan }
2078366f6083SPeter Grehan 
2079366f6083SPeter Grehan static int
2080366f6083SPeter Grehan vmx_setreg(void *arg, int vcpu, int reg, uint64_t val)
2081366f6083SPeter Grehan {
2082aaaa0656SPeter Grehan 	int error, hostcpu, running, shadow;
2083366f6083SPeter Grehan 	uint64_t ctls;
2084366f6083SPeter Grehan 	struct vmx *vmx = arg;
2085366f6083SPeter Grehan 
2086d3c11f40SPeter Grehan 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
2087d3c11f40SPeter Grehan 	if (running && hostcpu != curcpu)
2088d3c11f40SPeter Grehan 		panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu);
2089d3c11f40SPeter Grehan 
2090366f6083SPeter Grehan 	if (vmxctx_setreg(&vmx->ctx[vcpu], reg, val) == 0)
2091366f6083SPeter Grehan 		return (0);
2092366f6083SPeter Grehan 
2093d3c11f40SPeter Grehan 	error = vmcs_setreg(&vmx->vmcs[vcpu], running, reg, val);
2094366f6083SPeter Grehan 
2095366f6083SPeter Grehan 	if (error == 0) {
2096366f6083SPeter Grehan 		/*
2097366f6083SPeter Grehan 		 * If the "load EFER" VM-entry control is 1 then the
2098366f6083SPeter Grehan 		 * value of EFER.LMA must be identical to "IA-32e mode guest"
2099366f6083SPeter Grehan 		 * bit in the VM-entry control.
2100366f6083SPeter Grehan 		 */
2101366f6083SPeter Grehan 		if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 &&
2102366f6083SPeter Grehan 		    (reg == VM_REG_GUEST_EFER)) {
2103d3c11f40SPeter Grehan 			vmcs_getreg(&vmx->vmcs[vcpu], running,
2104366f6083SPeter Grehan 				    VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls);
2105366f6083SPeter Grehan 			if (val & EFER_LMA)
2106366f6083SPeter Grehan 				ctls |= VM_ENTRY_GUEST_LMA;
2107366f6083SPeter Grehan 			else
2108366f6083SPeter Grehan 				ctls &= ~VM_ENTRY_GUEST_LMA;
2109d3c11f40SPeter Grehan 			vmcs_setreg(&vmx->vmcs[vcpu], running,
2110366f6083SPeter Grehan 				    VMCS_IDENT(VMCS_ENTRY_CTLS), ctls);
2111366f6083SPeter Grehan 		}
2112aaaa0656SPeter Grehan 
2113aaaa0656SPeter Grehan 		shadow = vmx_shadow_reg(reg);
2114aaaa0656SPeter Grehan 		if (shadow > 0) {
2115aaaa0656SPeter Grehan 			/*
2116aaaa0656SPeter Grehan 			 * Store the unmodified value in the shadow
2117aaaa0656SPeter Grehan 			 */
2118aaaa0656SPeter Grehan 			error = vmcs_setreg(&vmx->vmcs[vcpu], running,
2119aaaa0656SPeter Grehan 				    VMCS_IDENT(shadow), val);
2120aaaa0656SPeter Grehan 		}
2121366f6083SPeter Grehan 	}
2122366f6083SPeter Grehan 
2123366f6083SPeter Grehan 	return (error);
2124366f6083SPeter Grehan }
2125366f6083SPeter Grehan 
2126366f6083SPeter Grehan static int
2127366f6083SPeter Grehan vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
2128366f6083SPeter Grehan {
2129366f6083SPeter Grehan 	struct vmx *vmx = arg;
2130366f6083SPeter Grehan 
2131366f6083SPeter Grehan 	return (vmcs_getdesc(&vmx->vmcs[vcpu], reg, desc));
2132366f6083SPeter Grehan }
2133366f6083SPeter Grehan 
2134366f6083SPeter Grehan static int
2135366f6083SPeter Grehan vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
2136366f6083SPeter Grehan {
2137366f6083SPeter Grehan 	struct vmx *vmx = arg;
2138366f6083SPeter Grehan 
2139366f6083SPeter Grehan 	return (vmcs_setdesc(&vmx->vmcs[vcpu], reg, desc));
2140366f6083SPeter Grehan }
2141366f6083SPeter Grehan 
2142366f6083SPeter Grehan static int
2143366f6083SPeter Grehan vmx_inject(void *arg, int vcpu, int type, int vector, uint32_t code,
2144366f6083SPeter Grehan 	   int code_valid)
2145366f6083SPeter Grehan {
2146366f6083SPeter Grehan 	int error;
2147eeefa4e4SNeel Natu 	uint64_t info;
2148366f6083SPeter Grehan 	struct vmx *vmx = arg;
2149366f6083SPeter Grehan 	struct vmcs *vmcs = &vmx->vmcs[vcpu];
2150366f6083SPeter Grehan 
2151366f6083SPeter Grehan 	static uint32_t type_map[VM_EVENT_MAX] = {
2152366f6083SPeter Grehan 		0x1,		/* VM_EVENT_NONE */
2153366f6083SPeter Grehan 		0x0,		/* VM_HW_INTR */
2154366f6083SPeter Grehan 		0x2,		/* VM_NMI */
2155366f6083SPeter Grehan 		0x3,		/* VM_HW_EXCEPTION */
2156366f6083SPeter Grehan 		0x4,		/* VM_SW_INTR */
2157366f6083SPeter Grehan 		0x5,		/* VM_PRIV_SW_EXCEPTION */
2158366f6083SPeter Grehan 		0x6,		/* VM_SW_EXCEPTION */
2159366f6083SPeter Grehan 	};
2160366f6083SPeter Grehan 
2161eeefa4e4SNeel Natu 	/*
2162eeefa4e4SNeel Natu 	 * If there is already an exception pending to be delivered to the
2163eeefa4e4SNeel Natu 	 * vcpu then just return.
2164eeefa4e4SNeel Natu 	 */
2165d3c11f40SPeter Grehan 	error = vmcs_getreg(vmcs, 0, VMCS_IDENT(VMCS_ENTRY_INTR_INFO), &info);
2166eeefa4e4SNeel Natu 	if (error)
2167eeefa4e4SNeel Natu 		return (error);
2168eeefa4e4SNeel Natu 
2169160471d2SNeel Natu 	if (info & VMCS_INTR_VALID)
2170eeefa4e4SNeel Natu 		return (EAGAIN);
2171eeefa4e4SNeel Natu 
2172366f6083SPeter Grehan 	info = vector | (type_map[type] << 8) | (code_valid ? 1 << 11 : 0);
2173160471d2SNeel Natu 	info |= VMCS_INTR_VALID;
2174d3c11f40SPeter Grehan 	error = vmcs_setreg(vmcs, 0, VMCS_IDENT(VMCS_ENTRY_INTR_INFO), info);
2175366f6083SPeter Grehan 	if (error != 0)
2176366f6083SPeter Grehan 		return (error);
2177366f6083SPeter Grehan 
2178366f6083SPeter Grehan 	if (code_valid) {
2179d3c11f40SPeter Grehan 		error = vmcs_setreg(vmcs, 0,
2180366f6083SPeter Grehan 				    VMCS_IDENT(VMCS_ENTRY_EXCEPTION_ERROR),
2181366f6083SPeter Grehan 				    code);
2182366f6083SPeter Grehan 	}
2183366f6083SPeter Grehan 	return (error);
2184366f6083SPeter Grehan }
2185366f6083SPeter Grehan 
2186366f6083SPeter Grehan static int
2187366f6083SPeter Grehan vmx_getcap(void *arg, int vcpu, int type, int *retval)
2188366f6083SPeter Grehan {
2189366f6083SPeter Grehan 	struct vmx *vmx = arg;
2190366f6083SPeter Grehan 	int vcap;
2191366f6083SPeter Grehan 	int ret;
2192366f6083SPeter Grehan 
2193366f6083SPeter Grehan 	ret = ENOENT;
2194366f6083SPeter Grehan 
2195366f6083SPeter Grehan 	vcap = vmx->cap[vcpu].set;
2196366f6083SPeter Grehan 
2197366f6083SPeter Grehan 	switch (type) {
2198366f6083SPeter Grehan 	case VM_CAP_HALT_EXIT:
2199366f6083SPeter Grehan 		if (cap_halt_exit)
2200366f6083SPeter Grehan 			ret = 0;
2201366f6083SPeter Grehan 		break;
2202366f6083SPeter Grehan 	case VM_CAP_PAUSE_EXIT:
2203366f6083SPeter Grehan 		if (cap_pause_exit)
2204366f6083SPeter Grehan 			ret = 0;
2205366f6083SPeter Grehan 		break;
2206366f6083SPeter Grehan 	case VM_CAP_MTRAP_EXIT:
2207366f6083SPeter Grehan 		if (cap_monitor_trap)
2208366f6083SPeter Grehan 			ret = 0;
2209366f6083SPeter Grehan 		break;
2210366f6083SPeter Grehan 	case VM_CAP_UNRESTRICTED_GUEST:
2211366f6083SPeter Grehan 		if (cap_unrestricted_guest)
2212366f6083SPeter Grehan 			ret = 0;
2213366f6083SPeter Grehan 		break;
221449cc03daSNeel Natu 	case VM_CAP_ENABLE_INVPCID:
221549cc03daSNeel Natu 		if (cap_invpcid)
221649cc03daSNeel Natu 			ret = 0;
221749cc03daSNeel Natu 		break;
2218366f6083SPeter Grehan 	default:
2219366f6083SPeter Grehan 		break;
2220366f6083SPeter Grehan 	}
2221366f6083SPeter Grehan 
2222366f6083SPeter Grehan 	if (ret == 0)
2223366f6083SPeter Grehan 		*retval = (vcap & (1 << type)) ? 1 : 0;
2224366f6083SPeter Grehan 
2225366f6083SPeter Grehan 	return (ret);
2226366f6083SPeter Grehan }
2227366f6083SPeter Grehan 
2228366f6083SPeter Grehan static int
2229366f6083SPeter Grehan vmx_setcap(void *arg, int vcpu, int type, int val)
2230366f6083SPeter Grehan {
2231366f6083SPeter Grehan 	struct vmx *vmx = arg;
2232366f6083SPeter Grehan 	struct vmcs *vmcs = &vmx->vmcs[vcpu];
2233366f6083SPeter Grehan 	uint32_t baseval;
2234366f6083SPeter Grehan 	uint32_t *pptr;
2235366f6083SPeter Grehan 	int error;
2236366f6083SPeter Grehan 	int flag;
2237366f6083SPeter Grehan 	int reg;
2238366f6083SPeter Grehan 	int retval;
2239366f6083SPeter Grehan 
2240366f6083SPeter Grehan 	retval = ENOENT;
2241366f6083SPeter Grehan 	pptr = NULL;
2242366f6083SPeter Grehan 
2243366f6083SPeter Grehan 	switch (type) {
2244366f6083SPeter Grehan 	case VM_CAP_HALT_EXIT:
2245366f6083SPeter Grehan 		if (cap_halt_exit) {
2246366f6083SPeter Grehan 			retval = 0;
2247366f6083SPeter Grehan 			pptr = &vmx->cap[vcpu].proc_ctls;
2248366f6083SPeter Grehan 			baseval = *pptr;
2249366f6083SPeter Grehan 			flag = PROCBASED_HLT_EXITING;
2250366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
2251366f6083SPeter Grehan 		}
2252366f6083SPeter Grehan 		break;
2253366f6083SPeter Grehan 	case VM_CAP_MTRAP_EXIT:
2254366f6083SPeter Grehan 		if (cap_monitor_trap) {
2255366f6083SPeter Grehan 			retval = 0;
2256366f6083SPeter Grehan 			pptr = &vmx->cap[vcpu].proc_ctls;
2257366f6083SPeter Grehan 			baseval = *pptr;
2258366f6083SPeter Grehan 			flag = PROCBASED_MTF;
2259366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
2260366f6083SPeter Grehan 		}
2261366f6083SPeter Grehan 		break;
2262366f6083SPeter Grehan 	case VM_CAP_PAUSE_EXIT:
2263366f6083SPeter Grehan 		if (cap_pause_exit) {
2264366f6083SPeter Grehan 			retval = 0;
2265366f6083SPeter Grehan 			pptr = &vmx->cap[vcpu].proc_ctls;
2266366f6083SPeter Grehan 			baseval = *pptr;
2267366f6083SPeter Grehan 			flag = PROCBASED_PAUSE_EXITING;
2268366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
2269366f6083SPeter Grehan 		}
2270366f6083SPeter Grehan 		break;
2271366f6083SPeter Grehan 	case VM_CAP_UNRESTRICTED_GUEST:
2272366f6083SPeter Grehan 		if (cap_unrestricted_guest) {
2273366f6083SPeter Grehan 			retval = 0;
227449cc03daSNeel Natu 			pptr = &vmx->cap[vcpu].proc_ctls2;
227549cc03daSNeel Natu 			baseval = *pptr;
2276366f6083SPeter Grehan 			flag = PROCBASED2_UNRESTRICTED_GUEST;
2277366f6083SPeter Grehan 			reg = VMCS_SEC_PROC_BASED_CTLS;
2278366f6083SPeter Grehan 		}
2279366f6083SPeter Grehan 		break;
228049cc03daSNeel Natu 	case VM_CAP_ENABLE_INVPCID:
228149cc03daSNeel Natu 		if (cap_invpcid) {
228249cc03daSNeel Natu 			retval = 0;
228349cc03daSNeel Natu 			pptr = &vmx->cap[vcpu].proc_ctls2;
228449cc03daSNeel Natu 			baseval = *pptr;
228549cc03daSNeel Natu 			flag = PROCBASED2_ENABLE_INVPCID;
228649cc03daSNeel Natu 			reg = VMCS_SEC_PROC_BASED_CTLS;
228749cc03daSNeel Natu 		}
228849cc03daSNeel Natu 		break;
2289366f6083SPeter Grehan 	default:
2290366f6083SPeter Grehan 		break;
2291366f6083SPeter Grehan 	}
2292366f6083SPeter Grehan 
2293366f6083SPeter Grehan 	if (retval == 0) {
2294366f6083SPeter Grehan 		if (val) {
2295366f6083SPeter Grehan 			baseval |= flag;
2296366f6083SPeter Grehan 		} else {
2297366f6083SPeter Grehan 			baseval &= ~flag;
2298366f6083SPeter Grehan 		}
2299366f6083SPeter Grehan 		VMPTRLD(vmcs);
2300366f6083SPeter Grehan 		error = vmwrite(reg, baseval);
2301366f6083SPeter Grehan 		VMCLEAR(vmcs);
2302366f6083SPeter Grehan 
2303366f6083SPeter Grehan 		if (error) {
2304366f6083SPeter Grehan 			retval = error;
2305366f6083SPeter Grehan 		} else {
2306366f6083SPeter Grehan 			/*
2307366f6083SPeter Grehan 			 * Update optional stored flags, and record
2308366f6083SPeter Grehan 			 * setting
2309366f6083SPeter Grehan 			 */
2310366f6083SPeter Grehan 			if (pptr != NULL) {
2311366f6083SPeter Grehan 				*pptr = baseval;
2312366f6083SPeter Grehan 			}
2313366f6083SPeter Grehan 
2314366f6083SPeter Grehan 			if (val) {
2315366f6083SPeter Grehan 				vmx->cap[vcpu].set |= (1 << type);
2316366f6083SPeter Grehan 			} else {
2317366f6083SPeter Grehan 				vmx->cap[vcpu].set &= ~(1 << type);
2318366f6083SPeter Grehan 			}
2319366f6083SPeter Grehan 		}
2320366f6083SPeter Grehan 	}
2321366f6083SPeter Grehan 
2322366f6083SPeter Grehan         return (retval);
2323366f6083SPeter Grehan }
2324366f6083SPeter Grehan 
232588c4b8d1SNeel Natu struct vlapic_vtx {
232688c4b8d1SNeel Natu 	struct vlapic	vlapic;
2327176666c2SNeel Natu 	struct pir_desc	*pir_desc;
2328*30b94db8SNeel Natu 	struct vmx	*vmx;
232988c4b8d1SNeel Natu };
233088c4b8d1SNeel Natu 
233188c4b8d1SNeel Natu #define	VMX_CTR_PIR(vm, vcpuid, pir_desc, notify, vector, level, msg)	\
233288c4b8d1SNeel Natu do {									\
233388c4b8d1SNeel Natu 	VCPU_CTR2(vm, vcpuid, msg " assert %s-triggered vector %d",	\
233488c4b8d1SNeel Natu 	    level ? "level" : "edge", vector);				\
233588c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir0 0x%016lx", pir_desc->pir[0]);	\
233688c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir1 0x%016lx", pir_desc->pir[1]);	\
233788c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir2 0x%016lx", pir_desc->pir[2]);	\
233888c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir3 0x%016lx", pir_desc->pir[3]);	\
233988c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " notify: %s", notify ? "yes" : "no");\
234088c4b8d1SNeel Natu } while (0)
234188c4b8d1SNeel Natu 
234288c4b8d1SNeel Natu /*
234388c4b8d1SNeel Natu  * vlapic->ops handlers that utilize the APICv hardware assist described in
234488c4b8d1SNeel Natu  * Chapter 29 of the Intel SDM.
234588c4b8d1SNeel Natu  */
234688c4b8d1SNeel Natu static int
234788c4b8d1SNeel Natu vmx_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
234888c4b8d1SNeel Natu {
234988c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
235088c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
235188c4b8d1SNeel Natu 	uint64_t mask;
235288c4b8d1SNeel Natu 	int idx, notify;
235388c4b8d1SNeel Natu 
235488c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
2355176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
235688c4b8d1SNeel Natu 
235788c4b8d1SNeel Natu 	/*
235888c4b8d1SNeel Natu 	 * Keep track of interrupt requests in the PIR descriptor. This is
235988c4b8d1SNeel Natu 	 * because the virtual APIC page pointed to by the VMCS cannot be
236088c4b8d1SNeel Natu 	 * modified if the vcpu is running.
236188c4b8d1SNeel Natu 	 */
236288c4b8d1SNeel Natu 	idx = vector / 64;
236388c4b8d1SNeel Natu 	mask = 1UL << (vector % 64);
236488c4b8d1SNeel Natu 	atomic_set_long(&pir_desc->pir[idx], mask);
236588c4b8d1SNeel Natu 	notify = atomic_cmpset_long(&pir_desc->pending, 0, 1);
236688c4b8d1SNeel Natu 
236788c4b8d1SNeel Natu 	VMX_CTR_PIR(vlapic->vm, vlapic->vcpuid, pir_desc, notify, vector,
236888c4b8d1SNeel Natu 	    level, "vmx_set_intr_ready");
236988c4b8d1SNeel Natu 	return (notify);
237088c4b8d1SNeel Natu }
237188c4b8d1SNeel Natu 
237288c4b8d1SNeel Natu static int
237388c4b8d1SNeel Natu vmx_pending_intr(struct vlapic *vlapic, int *vecptr)
237488c4b8d1SNeel Natu {
237588c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
237688c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
237788c4b8d1SNeel Natu 	struct LAPIC *lapic;
237888c4b8d1SNeel Natu 	uint64_t pending, pirval;
237988c4b8d1SNeel Natu 	uint32_t ppr, vpr;
238088c4b8d1SNeel Natu 	int i;
238188c4b8d1SNeel Natu 
238288c4b8d1SNeel Natu 	/*
238388c4b8d1SNeel Natu 	 * This function is only expected to be called from the 'HLT' exit
238488c4b8d1SNeel Natu 	 * handler which does not care about the vector that is pending.
238588c4b8d1SNeel Natu 	 */
238688c4b8d1SNeel Natu 	KASSERT(vecptr == NULL, ("vmx_pending_intr: vecptr must be NULL"));
238788c4b8d1SNeel Natu 
238888c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
2389176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
239088c4b8d1SNeel Natu 
239188c4b8d1SNeel Natu 	pending = atomic_load_acq_long(&pir_desc->pending);
239288c4b8d1SNeel Natu 	if (!pending)
239388c4b8d1SNeel Natu 		return (0);	/* common case */
239488c4b8d1SNeel Natu 
239588c4b8d1SNeel Natu 	/*
239688c4b8d1SNeel Natu 	 * If there is an interrupt pending then it will be recognized only
239788c4b8d1SNeel Natu 	 * if its priority is greater than the processor priority.
239888c4b8d1SNeel Natu 	 *
239988c4b8d1SNeel Natu 	 * Special case: if the processor priority is zero then any pending
240088c4b8d1SNeel Natu 	 * interrupt will be recognized.
240188c4b8d1SNeel Natu 	 */
240288c4b8d1SNeel Natu 	lapic = vlapic->apic_page;
240388c4b8d1SNeel Natu 	ppr = lapic->ppr & 0xf0;
240488c4b8d1SNeel Natu 	if (ppr == 0)
240588c4b8d1SNeel Natu 		return (1);
240688c4b8d1SNeel Natu 
240788c4b8d1SNeel Natu 	VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "HLT with non-zero PPR %d",
240888c4b8d1SNeel Natu 	    lapic->ppr);
240988c4b8d1SNeel Natu 
241088c4b8d1SNeel Natu 	for (i = 3; i >= 0; i--) {
241188c4b8d1SNeel Natu 		pirval = pir_desc->pir[i];
241288c4b8d1SNeel Natu 		if (pirval != 0) {
241388c4b8d1SNeel Natu 			vpr = (i * 64 + flsl(pirval) - 1) & 0xf0;
241488c4b8d1SNeel Natu 			return (vpr > ppr);
241588c4b8d1SNeel Natu 		}
241688c4b8d1SNeel Natu 	}
241788c4b8d1SNeel Natu 	return (0);
241888c4b8d1SNeel Natu }
241988c4b8d1SNeel Natu 
242088c4b8d1SNeel Natu static void
242188c4b8d1SNeel Natu vmx_intr_accepted(struct vlapic *vlapic, int vector)
242288c4b8d1SNeel Natu {
242388c4b8d1SNeel Natu 
242488c4b8d1SNeel Natu 	panic("vmx_intr_accepted: not expected to be called");
242588c4b8d1SNeel Natu }
242688c4b8d1SNeel Natu 
2427176666c2SNeel Natu static void
2428*30b94db8SNeel Natu vmx_set_tmr(struct vlapic *vlapic, int vector, bool level)
2429*30b94db8SNeel Natu {
2430*30b94db8SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
2431*30b94db8SNeel Natu 	struct vmx *vmx;
2432*30b94db8SNeel Natu 	struct vmcs *vmcs;
2433*30b94db8SNeel Natu 	uint64_t mask, val;
2434*30b94db8SNeel Natu 
2435*30b94db8SNeel Natu 	KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
2436*30b94db8SNeel Natu 	KASSERT(!vcpu_is_running(vlapic->vm, vlapic->vcpuid, NULL),
2437*30b94db8SNeel Natu 	    ("vmx_set_tmr: vcpu cannot be running"));
2438*30b94db8SNeel Natu 
2439*30b94db8SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
2440*30b94db8SNeel Natu 	vmx = vlapic_vtx->vmx;
2441*30b94db8SNeel Natu 	vmcs = &vmx->vmcs[vlapic->vcpuid];
2442*30b94db8SNeel Natu 	mask = 1UL << (vector % 64);
2443*30b94db8SNeel Natu 
2444*30b94db8SNeel Natu 	VMPTRLD(vmcs);
2445*30b94db8SNeel Natu 	val = vmcs_read(VMCS_EOI_EXIT(vector));
2446*30b94db8SNeel Natu 	if (level)
2447*30b94db8SNeel Natu 		val |= mask;
2448*30b94db8SNeel Natu 	else
2449*30b94db8SNeel Natu 		val &= ~mask;
2450*30b94db8SNeel Natu 	vmcs_write(VMCS_EOI_EXIT(vector), val);
2451*30b94db8SNeel Natu 	VMCLEAR(vmcs);
2452*30b94db8SNeel Natu }
2453*30b94db8SNeel Natu 
2454*30b94db8SNeel Natu static void
2455176666c2SNeel Natu vmx_post_intr(struct vlapic *vlapic, int hostcpu)
2456176666c2SNeel Natu {
2457176666c2SNeel Natu 
2458176666c2SNeel Natu 	ipi_cpu(hostcpu, pirvec);
2459176666c2SNeel Natu }
2460176666c2SNeel Natu 
246188c4b8d1SNeel Natu /*
246288c4b8d1SNeel Natu  * Transfer the pending interrupts in the PIR descriptor to the IRR
246388c4b8d1SNeel Natu  * in the virtual APIC page.
246488c4b8d1SNeel Natu  */
246588c4b8d1SNeel Natu static void
246688c4b8d1SNeel Natu vmx_inject_pir(struct vlapic *vlapic)
246788c4b8d1SNeel Natu {
246888c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
246988c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
247088c4b8d1SNeel Natu 	struct LAPIC *lapic;
247188c4b8d1SNeel Natu 	uint64_t val, pirval;
247288c4b8d1SNeel Natu 	int rvi, pirbase;
247388c4b8d1SNeel Natu 	uint16_t intr_status_old, intr_status_new;
247488c4b8d1SNeel Natu 
247588c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
2476176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
247788c4b8d1SNeel Natu 	if (atomic_cmpset_long(&pir_desc->pending, 1, 0) == 0) {
247888c4b8d1SNeel Natu 		VCPU_CTR0(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
247988c4b8d1SNeel Natu 		    "no posted interrupt pending");
248088c4b8d1SNeel Natu 		return;
248188c4b8d1SNeel Natu 	}
248288c4b8d1SNeel Natu 
248388c4b8d1SNeel Natu 	pirval = 0;
248488c4b8d1SNeel Natu 	lapic = vlapic->apic_page;
248588c4b8d1SNeel Natu 
248688c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[0]);
248788c4b8d1SNeel Natu 	if (val != 0) {
248888c4b8d1SNeel Natu 		lapic->irr0 |= val;
248988c4b8d1SNeel Natu 		lapic->irr1 |= val >> 32;
249088c4b8d1SNeel Natu 		pirbase = 0;
249188c4b8d1SNeel Natu 		pirval = val;
249288c4b8d1SNeel Natu 	}
249388c4b8d1SNeel Natu 
249488c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[1]);
249588c4b8d1SNeel Natu 	if (val != 0) {
249688c4b8d1SNeel Natu 		lapic->irr2 |= val;
249788c4b8d1SNeel Natu 		lapic->irr3 |= val >> 32;
249888c4b8d1SNeel Natu 		pirbase = 64;
249988c4b8d1SNeel Natu 		pirval = val;
250088c4b8d1SNeel Natu 	}
250188c4b8d1SNeel Natu 
250288c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[2]);
250388c4b8d1SNeel Natu 	if (val != 0) {
250488c4b8d1SNeel Natu 		lapic->irr4 |= val;
250588c4b8d1SNeel Natu 		lapic->irr5 |= val >> 32;
250688c4b8d1SNeel Natu 		pirbase = 128;
250788c4b8d1SNeel Natu 		pirval = val;
250888c4b8d1SNeel Natu 	}
250988c4b8d1SNeel Natu 
251088c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[3]);
251188c4b8d1SNeel Natu 	if (val != 0) {
251288c4b8d1SNeel Natu 		lapic->irr6 |= val;
251388c4b8d1SNeel Natu 		lapic->irr7 |= val >> 32;
251488c4b8d1SNeel Natu 		pirbase = 192;
251588c4b8d1SNeel Natu 		pirval = val;
251688c4b8d1SNeel Natu 	}
251788c4b8d1SNeel Natu 	VLAPIC_CTR_IRR(vlapic, "vmx_inject_pir");
251888c4b8d1SNeel Natu 
251988c4b8d1SNeel Natu 	/*
252088c4b8d1SNeel Natu 	 * Update RVI so the processor can evaluate pending virtual
252188c4b8d1SNeel Natu 	 * interrupts on VM-entry.
252288c4b8d1SNeel Natu 	 */
252388c4b8d1SNeel Natu 	if (pirval != 0) {
252488c4b8d1SNeel Natu 		rvi = pirbase + flsl(pirval) - 1;
252588c4b8d1SNeel Natu 		intr_status_old = vmcs_read(VMCS_GUEST_INTR_STATUS);
252688c4b8d1SNeel Natu 		intr_status_new = (intr_status_old & 0xFF00) | rvi;
252788c4b8d1SNeel Natu 		if (intr_status_new > intr_status_old) {
252888c4b8d1SNeel Natu 			vmcs_write(VMCS_GUEST_INTR_STATUS, intr_status_new);
252988c4b8d1SNeel Natu 			VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
253088c4b8d1SNeel Natu 			    "guest_intr_status changed from 0x%04x to 0x%04x",
253188c4b8d1SNeel Natu 			    intr_status_old, intr_status_new);
253288c4b8d1SNeel Natu 		}
253388c4b8d1SNeel Natu 	}
253488c4b8d1SNeel Natu }
253588c4b8d1SNeel Natu 
2536de5ea6b6SNeel Natu static struct vlapic *
2537de5ea6b6SNeel Natu vmx_vlapic_init(void *arg, int vcpuid)
2538de5ea6b6SNeel Natu {
2539de5ea6b6SNeel Natu 	struct vmx *vmx;
2540de5ea6b6SNeel Natu 	struct vlapic *vlapic;
2541176666c2SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
2542de5ea6b6SNeel Natu 
2543de5ea6b6SNeel Natu 	vmx = arg;
2544de5ea6b6SNeel Natu 
254588c4b8d1SNeel Natu 	vlapic = malloc(sizeof(struct vlapic_vtx), M_VLAPIC, M_WAITOK | M_ZERO);
2546de5ea6b6SNeel Natu 	vlapic->vm = vmx->vm;
2547de5ea6b6SNeel Natu 	vlapic->vcpuid = vcpuid;
2548de5ea6b6SNeel Natu 	vlapic->apic_page = (struct LAPIC *)&vmx->apic_page[vcpuid];
2549de5ea6b6SNeel Natu 
2550176666c2SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
2551176666c2SNeel Natu 	vlapic_vtx->pir_desc = &vmx->pir_desc[vcpuid];
2552*30b94db8SNeel Natu 	vlapic_vtx->vmx = vmx;
2553176666c2SNeel Natu 
255488c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
255588c4b8d1SNeel Natu 		vlapic->ops.set_intr_ready = vmx_set_intr_ready;
255688c4b8d1SNeel Natu 		vlapic->ops.pending_intr = vmx_pending_intr;
255788c4b8d1SNeel Natu 		vlapic->ops.intr_accepted = vmx_intr_accepted;
2558*30b94db8SNeel Natu 		vlapic->ops.set_tmr = vmx_set_tmr;
255988c4b8d1SNeel Natu 	}
256088c4b8d1SNeel Natu 
2561176666c2SNeel Natu 	if (posted_interrupts)
2562176666c2SNeel Natu 		vlapic->ops.post_intr = vmx_post_intr;
2563176666c2SNeel Natu 
2564de5ea6b6SNeel Natu 	vlapic_init(vlapic);
2565de5ea6b6SNeel Natu 
2566de5ea6b6SNeel Natu 	return (vlapic);
2567de5ea6b6SNeel Natu }
2568de5ea6b6SNeel Natu 
2569de5ea6b6SNeel Natu static void
2570de5ea6b6SNeel Natu vmx_vlapic_cleanup(void *arg, struct vlapic *vlapic)
2571de5ea6b6SNeel Natu {
2572de5ea6b6SNeel Natu 
2573de5ea6b6SNeel Natu 	vlapic_cleanup(vlapic);
2574de5ea6b6SNeel Natu 	free(vlapic, M_VLAPIC);
2575de5ea6b6SNeel Natu }
2576de5ea6b6SNeel Natu 
2577366f6083SPeter Grehan struct vmm_ops vmm_ops_intel = {
2578366f6083SPeter Grehan 	vmx_init,
2579366f6083SPeter Grehan 	vmx_cleanup,
258063e62d39SJohn Baldwin 	vmx_restore,
2581366f6083SPeter Grehan 	vmx_vminit,
2582366f6083SPeter Grehan 	vmx_run,
2583366f6083SPeter Grehan 	vmx_vmcleanup,
2584366f6083SPeter Grehan 	vmx_getreg,
2585366f6083SPeter Grehan 	vmx_setreg,
2586366f6083SPeter Grehan 	vmx_getdesc,
2587366f6083SPeter Grehan 	vmx_setdesc,
2588366f6083SPeter Grehan 	vmx_inject,
2589366f6083SPeter Grehan 	vmx_getcap,
2590318224bbSNeel Natu 	vmx_setcap,
2591318224bbSNeel Natu 	ept_vmspace_alloc,
2592318224bbSNeel Natu 	ept_vmspace_free,
2593de5ea6b6SNeel Natu 	vmx_vlapic_init,
2594de5ea6b6SNeel Natu 	vmx_vlapic_cleanup,
2595366f6083SPeter Grehan };
2596