xref: /freebsd/sys/amd64/vmm/intel/vmx.c (revision 201b1ccc22c1a188b58b9ba7c4b7b3157705d46d)
1366f6083SPeter Grehan /*-
2366f6083SPeter Grehan  * Copyright (c) 2011 NetApp, Inc.
3366f6083SPeter Grehan  * All rights reserved.
4366f6083SPeter Grehan  *
5366f6083SPeter Grehan  * Redistribution and use in source and binary forms, with or without
6366f6083SPeter Grehan  * modification, are permitted provided that the following conditions
7366f6083SPeter Grehan  * are met:
8366f6083SPeter Grehan  * 1. Redistributions of source code must retain the above copyright
9366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer.
10366f6083SPeter Grehan  * 2. Redistributions in binary form must reproduce the above copyright
11366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer in the
12366f6083SPeter Grehan  *    documentation and/or other materials provided with the distribution.
13366f6083SPeter Grehan  *
14366f6083SPeter Grehan  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
15366f6083SPeter Grehan  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16366f6083SPeter Grehan  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17366f6083SPeter Grehan  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
18366f6083SPeter Grehan  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19366f6083SPeter Grehan  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20366f6083SPeter Grehan  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21366f6083SPeter Grehan  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22366f6083SPeter Grehan  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23366f6083SPeter Grehan  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24366f6083SPeter Grehan  * SUCH DAMAGE.
25366f6083SPeter Grehan  *
26366f6083SPeter Grehan  * $FreeBSD$
27366f6083SPeter Grehan  */
28366f6083SPeter Grehan 
29366f6083SPeter Grehan #include <sys/cdefs.h>
30366f6083SPeter Grehan __FBSDID("$FreeBSD$");
31366f6083SPeter Grehan 
32366f6083SPeter Grehan #include <sys/param.h>
33366f6083SPeter Grehan #include <sys/systm.h>
34366f6083SPeter Grehan #include <sys/smp.h>
35366f6083SPeter Grehan #include <sys/kernel.h>
36366f6083SPeter Grehan #include <sys/malloc.h>
37366f6083SPeter Grehan #include <sys/pcpu.h>
38366f6083SPeter Grehan #include <sys/proc.h>
393565b59eSNeel Natu #include <sys/sysctl.h>
40366f6083SPeter Grehan 
41366f6083SPeter Grehan #include <vm/vm.h>
42366f6083SPeter Grehan #include <vm/pmap.h>
43366f6083SPeter Grehan 
44366f6083SPeter Grehan #include <machine/psl.h>
45366f6083SPeter Grehan #include <machine/cpufunc.h>
468b287612SJohn Baldwin #include <machine/md_var.h>
47366f6083SPeter Grehan #include <machine/segments.h>
48176666c2SNeel Natu #include <machine/smp.h>
49608f97c3SPeter Grehan #include <machine/specialreg.h>
50366f6083SPeter Grehan #include <machine/vmparam.h>
51366f6083SPeter Grehan 
52366f6083SPeter Grehan #include <machine/vmm.h>
53dc506506SNeel Natu #include <machine/vmm_dev.h>
54b01c2033SNeel Natu #include "vmm_host.h"
55762fd208STycho Nightingale #include "vmm_ioport.h"
56176666c2SNeel Natu #include "vmm_ipi.h"
57366f6083SPeter Grehan #include "vmm_msr.h"
58366f6083SPeter Grehan #include "vmm_ktr.h"
59366f6083SPeter Grehan #include "vmm_stat.h"
600775fbb4STycho Nightingale #include "vatpic.h"
61de5ea6b6SNeel Natu #include "vlapic.h"
62de5ea6b6SNeel Natu #include "vlapic_priv.h"
63366f6083SPeter Grehan 
64366f6083SPeter Grehan #include "vmx_msr.h"
65366f6083SPeter Grehan #include "ept.h"
66366f6083SPeter Grehan #include "vmx_cpufunc.h"
67366f6083SPeter Grehan #include "vmx.h"
68366f6083SPeter Grehan #include "x86.h"
69366f6083SPeter Grehan #include "vmx_controls.h"
70366f6083SPeter Grehan 
71366f6083SPeter Grehan #define	PINBASED_CTLS_ONE_SETTING					\
72366f6083SPeter Grehan 	(PINBASED_EXTINT_EXITING	|				\
73366f6083SPeter Grehan 	 PINBASED_NMI_EXITING		|				\
74366f6083SPeter Grehan 	 PINBASED_VIRTUAL_NMI)
75366f6083SPeter Grehan #define	PINBASED_CTLS_ZERO_SETTING	0
76366f6083SPeter Grehan 
77366f6083SPeter Grehan #define PROCBASED_CTLS_WINDOW_SETTING					\
78366f6083SPeter Grehan 	(PROCBASED_INT_WINDOW_EXITING	|				\
79366f6083SPeter Grehan 	 PROCBASED_NMI_WINDOW_EXITING)
80366f6083SPeter Grehan 
81366f6083SPeter Grehan #define	PROCBASED_CTLS_ONE_SETTING 					\
82366f6083SPeter Grehan 	(PROCBASED_SECONDARY_CONTROLS	|				\
83366f6083SPeter Grehan 	 PROCBASED_IO_EXITING		|				\
84366f6083SPeter Grehan 	 PROCBASED_MSR_BITMAPS		|				\
85366f6083SPeter Grehan 	 PROCBASED_CTLS_WINDOW_SETTING)
86366f6083SPeter Grehan #define	PROCBASED_CTLS_ZERO_SETTING	\
87366f6083SPeter Grehan 	(PROCBASED_CR3_LOAD_EXITING |	\
88366f6083SPeter Grehan 	PROCBASED_CR3_STORE_EXITING |	\
89366f6083SPeter Grehan 	PROCBASED_IO_BITMAPS)
90366f6083SPeter Grehan 
91366f6083SPeter Grehan #define	PROCBASED_CTLS2_ONE_SETTING	PROCBASED2_ENABLE_EPT
92366f6083SPeter Grehan #define	PROCBASED_CTLS2_ZERO_SETTING	0
93366f6083SPeter Grehan 
94608f97c3SPeter Grehan #define VM_EXIT_CTLS_ONE_SETTING_NO_PAT					\
95366f6083SPeter Grehan 	(VM_EXIT_HOST_LMA			|			\
96366f6083SPeter Grehan 	VM_EXIT_SAVE_EFER			|			\
97366f6083SPeter Grehan 	VM_EXIT_LOAD_EFER)
98608f97c3SPeter Grehan 
99608f97c3SPeter Grehan #define	VM_EXIT_CTLS_ONE_SETTING					\
100608f97c3SPeter Grehan 	(VM_EXIT_CTLS_ONE_SETTING_NO_PAT       	|			\
101f7d47425SNeel Natu 	VM_EXIT_ACKNOWLEDGE_INTERRUPT		|			\
102608f97c3SPeter Grehan 	VM_EXIT_SAVE_PAT			|			\
103608f97c3SPeter Grehan 	VM_EXIT_LOAD_PAT)
104366f6083SPeter Grehan #define	VM_EXIT_CTLS_ZERO_SETTING	VM_EXIT_SAVE_DEBUG_CONTROLS
105366f6083SPeter Grehan 
106608f97c3SPeter Grehan #define	VM_ENTRY_CTLS_ONE_SETTING_NO_PAT	VM_ENTRY_LOAD_EFER
107608f97c3SPeter Grehan 
108366f6083SPeter Grehan #define	VM_ENTRY_CTLS_ONE_SETTING					\
109608f97c3SPeter Grehan 	(VM_ENTRY_CTLS_ONE_SETTING_NO_PAT     	|			\
110608f97c3SPeter Grehan 	VM_ENTRY_LOAD_PAT)
111366f6083SPeter Grehan #define	VM_ENTRY_CTLS_ZERO_SETTING					\
112366f6083SPeter Grehan 	(VM_ENTRY_LOAD_DEBUG_CONTROLS		|			\
113366f6083SPeter Grehan 	VM_ENTRY_INTO_SMM			|			\
114366f6083SPeter Grehan 	VM_ENTRY_DEACTIVATE_DUAL_MONITOR)
115366f6083SPeter Grehan 
116366f6083SPeter Grehan #define	guest_msr_rw(vmx, msr) \
117366f6083SPeter Grehan 	msr_bitmap_change_access((vmx)->msr_bitmap, (msr), MSR_BITMAP_ACCESS_RW)
118366f6083SPeter Grehan 
119159dd56fSNeel Natu #define	guest_msr_ro(vmx, msr) \
120159dd56fSNeel Natu     msr_bitmap_change_access((vmx)->msr_bitmap, (msr), MSR_BITMAP_ACCESS_READ)
121159dd56fSNeel Natu 
122366f6083SPeter Grehan #define	HANDLED		1
123366f6083SPeter Grehan #define	UNHANDLED	0
124366f6083SPeter Grehan 
125de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VMX, "vmx", "vmx");
126de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
127366f6083SPeter Grehan 
1283565b59eSNeel Natu SYSCTL_DECL(_hw_vmm);
1293565b59eSNeel Natu SYSCTL_NODE(_hw_vmm, OID_AUTO, vmx, CTLFLAG_RW, NULL, NULL);
1303565b59eSNeel Natu 
131b3996dd4SJohn Baldwin int vmxon_enabled[MAXCPU];
132366f6083SPeter Grehan static char vmxon_region[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE);
133366f6083SPeter Grehan 
134366f6083SPeter Grehan static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2;
135366f6083SPeter Grehan static uint32_t exit_ctls, entry_ctls;
136366f6083SPeter Grehan 
137366f6083SPeter Grehan static uint64_t cr0_ones_mask, cr0_zeros_mask;
1383565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_ones_mask, CTLFLAG_RD,
1393565b59eSNeel Natu 	     &cr0_ones_mask, 0, NULL);
1403565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_zeros_mask, CTLFLAG_RD,
1413565b59eSNeel Natu 	     &cr0_zeros_mask, 0, NULL);
1423565b59eSNeel Natu 
143366f6083SPeter Grehan static uint64_t cr4_ones_mask, cr4_zeros_mask;
1443565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_ones_mask, CTLFLAG_RD,
1453565b59eSNeel Natu 	     &cr4_ones_mask, 0, NULL);
1463565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_zeros_mask, CTLFLAG_RD,
1473565b59eSNeel Natu 	     &cr4_zeros_mask, 0, NULL);
148366f6083SPeter Grehan 
149608f97c3SPeter Grehan static int vmx_no_patmsr;
150608f97c3SPeter Grehan 
1513565b59eSNeel Natu static int vmx_initialized;
1523565b59eSNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, initialized, CTLFLAG_RD,
1533565b59eSNeel Natu 	   &vmx_initialized, 0, "Intel VMX initialized");
1543565b59eSNeel Natu 
155366f6083SPeter Grehan /*
156366f6083SPeter Grehan  * Optional capabilities
157366f6083SPeter Grehan  */
158366f6083SPeter Grehan static int cap_halt_exit;
159366f6083SPeter Grehan static int cap_pause_exit;
160366f6083SPeter Grehan static int cap_unrestricted_guest;
161366f6083SPeter Grehan static int cap_monitor_trap;
16249cc03daSNeel Natu static int cap_invpcid;
163366f6083SPeter Grehan 
16488c4b8d1SNeel Natu static int virtual_interrupt_delivery;
16588c4b8d1SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, virtual_interrupt_delivery, CTLFLAG_RD,
16688c4b8d1SNeel Natu     &virtual_interrupt_delivery, 0, "APICv virtual interrupt delivery support");
16788c4b8d1SNeel Natu 
168176666c2SNeel Natu static int posted_interrupts;
169176666c2SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupts, CTLFLAG_RD,
170176666c2SNeel Natu     &posted_interrupts, 0, "APICv posted interrupt support");
171176666c2SNeel Natu 
172176666c2SNeel Natu static int pirvec;
173176666c2SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupt_vector, CTLFLAG_RD,
174176666c2SNeel Natu     &pirvec, 0, "APICv posted interrupt vector");
175176666c2SNeel Natu 
17645e51299SNeel Natu static struct unrhdr *vpid_unr;
17745e51299SNeel Natu static u_int vpid_alloc_failed;
17845e51299SNeel Natu SYSCTL_UINT(_hw_vmm_vmx, OID_AUTO, vpid_alloc_failed, CTLFLAG_RD,
17945e51299SNeel Natu 	    &vpid_alloc_failed, 0, NULL);
18045e51299SNeel Natu 
18188c4b8d1SNeel Natu /*
18288c4b8d1SNeel Natu  * Use the last page below 4GB as the APIC access address. This address is
18388c4b8d1SNeel Natu  * occupied by the boot firmware so it is guaranteed that it will not conflict
18488c4b8d1SNeel Natu  * with a page in system memory.
18588c4b8d1SNeel Natu  */
18688c4b8d1SNeel Natu #define	APIC_ACCESS_ADDRESS	0xFFFFF000
18788c4b8d1SNeel Natu 
18888c4b8d1SNeel Natu static void vmx_inject_pir(struct vlapic *vlapic);
18988c4b8d1SNeel Natu 
190366f6083SPeter Grehan #ifdef KTR
191366f6083SPeter Grehan static const char *
192366f6083SPeter Grehan exit_reason_to_str(int reason)
193366f6083SPeter Grehan {
194366f6083SPeter Grehan 	static char reasonbuf[32];
195366f6083SPeter Grehan 
196366f6083SPeter Grehan 	switch (reason) {
197366f6083SPeter Grehan 	case EXIT_REASON_EXCEPTION:
198366f6083SPeter Grehan 		return "exception";
199366f6083SPeter Grehan 	case EXIT_REASON_EXT_INTR:
200366f6083SPeter Grehan 		return "extint";
201366f6083SPeter Grehan 	case EXIT_REASON_TRIPLE_FAULT:
202366f6083SPeter Grehan 		return "triplefault";
203366f6083SPeter Grehan 	case EXIT_REASON_INIT:
204366f6083SPeter Grehan 		return "init";
205366f6083SPeter Grehan 	case EXIT_REASON_SIPI:
206366f6083SPeter Grehan 		return "sipi";
207366f6083SPeter Grehan 	case EXIT_REASON_IO_SMI:
208366f6083SPeter Grehan 		return "iosmi";
209366f6083SPeter Grehan 	case EXIT_REASON_SMI:
210366f6083SPeter Grehan 		return "smi";
211366f6083SPeter Grehan 	case EXIT_REASON_INTR_WINDOW:
212366f6083SPeter Grehan 		return "intrwindow";
213366f6083SPeter Grehan 	case EXIT_REASON_NMI_WINDOW:
214366f6083SPeter Grehan 		return "nmiwindow";
215366f6083SPeter Grehan 	case EXIT_REASON_TASK_SWITCH:
216366f6083SPeter Grehan 		return "taskswitch";
217366f6083SPeter Grehan 	case EXIT_REASON_CPUID:
218366f6083SPeter Grehan 		return "cpuid";
219366f6083SPeter Grehan 	case EXIT_REASON_GETSEC:
220366f6083SPeter Grehan 		return "getsec";
221366f6083SPeter Grehan 	case EXIT_REASON_HLT:
222366f6083SPeter Grehan 		return "hlt";
223366f6083SPeter Grehan 	case EXIT_REASON_INVD:
224366f6083SPeter Grehan 		return "invd";
225366f6083SPeter Grehan 	case EXIT_REASON_INVLPG:
226366f6083SPeter Grehan 		return "invlpg";
227366f6083SPeter Grehan 	case EXIT_REASON_RDPMC:
228366f6083SPeter Grehan 		return "rdpmc";
229366f6083SPeter Grehan 	case EXIT_REASON_RDTSC:
230366f6083SPeter Grehan 		return "rdtsc";
231366f6083SPeter Grehan 	case EXIT_REASON_RSM:
232366f6083SPeter Grehan 		return "rsm";
233366f6083SPeter Grehan 	case EXIT_REASON_VMCALL:
234366f6083SPeter Grehan 		return "vmcall";
235366f6083SPeter Grehan 	case EXIT_REASON_VMCLEAR:
236366f6083SPeter Grehan 		return "vmclear";
237366f6083SPeter Grehan 	case EXIT_REASON_VMLAUNCH:
238366f6083SPeter Grehan 		return "vmlaunch";
239366f6083SPeter Grehan 	case EXIT_REASON_VMPTRLD:
240366f6083SPeter Grehan 		return "vmptrld";
241366f6083SPeter Grehan 	case EXIT_REASON_VMPTRST:
242366f6083SPeter Grehan 		return "vmptrst";
243366f6083SPeter Grehan 	case EXIT_REASON_VMREAD:
244366f6083SPeter Grehan 		return "vmread";
245366f6083SPeter Grehan 	case EXIT_REASON_VMRESUME:
246366f6083SPeter Grehan 		return "vmresume";
247366f6083SPeter Grehan 	case EXIT_REASON_VMWRITE:
248366f6083SPeter Grehan 		return "vmwrite";
249366f6083SPeter Grehan 	case EXIT_REASON_VMXOFF:
250366f6083SPeter Grehan 		return "vmxoff";
251366f6083SPeter Grehan 	case EXIT_REASON_VMXON:
252366f6083SPeter Grehan 		return "vmxon";
253366f6083SPeter Grehan 	case EXIT_REASON_CR_ACCESS:
254366f6083SPeter Grehan 		return "craccess";
255366f6083SPeter Grehan 	case EXIT_REASON_DR_ACCESS:
256366f6083SPeter Grehan 		return "draccess";
257366f6083SPeter Grehan 	case EXIT_REASON_INOUT:
258366f6083SPeter Grehan 		return "inout";
259366f6083SPeter Grehan 	case EXIT_REASON_RDMSR:
260366f6083SPeter Grehan 		return "rdmsr";
261366f6083SPeter Grehan 	case EXIT_REASON_WRMSR:
262366f6083SPeter Grehan 		return "wrmsr";
263366f6083SPeter Grehan 	case EXIT_REASON_INVAL_VMCS:
264366f6083SPeter Grehan 		return "invalvmcs";
265366f6083SPeter Grehan 	case EXIT_REASON_INVAL_MSR:
266366f6083SPeter Grehan 		return "invalmsr";
267366f6083SPeter Grehan 	case EXIT_REASON_MWAIT:
268366f6083SPeter Grehan 		return "mwait";
269366f6083SPeter Grehan 	case EXIT_REASON_MTF:
270366f6083SPeter Grehan 		return "mtf";
271366f6083SPeter Grehan 	case EXIT_REASON_MONITOR:
272366f6083SPeter Grehan 		return "monitor";
273366f6083SPeter Grehan 	case EXIT_REASON_PAUSE:
274366f6083SPeter Grehan 		return "pause";
275366f6083SPeter Grehan 	case EXIT_REASON_MCE:
276366f6083SPeter Grehan 		return "mce";
277366f6083SPeter Grehan 	case EXIT_REASON_TPR:
278366f6083SPeter Grehan 		return "tpr";
27988c4b8d1SNeel Natu 	case EXIT_REASON_APIC_ACCESS:
28088c4b8d1SNeel Natu 		return "apic-access";
281366f6083SPeter Grehan 	case EXIT_REASON_GDTR_IDTR:
282366f6083SPeter Grehan 		return "gdtridtr";
283366f6083SPeter Grehan 	case EXIT_REASON_LDTR_TR:
284366f6083SPeter Grehan 		return "ldtrtr";
285366f6083SPeter Grehan 	case EXIT_REASON_EPT_FAULT:
286366f6083SPeter Grehan 		return "eptfault";
287366f6083SPeter Grehan 	case EXIT_REASON_EPT_MISCONFIG:
288366f6083SPeter Grehan 		return "eptmisconfig";
289366f6083SPeter Grehan 	case EXIT_REASON_INVEPT:
290366f6083SPeter Grehan 		return "invept";
291366f6083SPeter Grehan 	case EXIT_REASON_RDTSCP:
292366f6083SPeter Grehan 		return "rdtscp";
293366f6083SPeter Grehan 	case EXIT_REASON_VMX_PREEMPT:
294366f6083SPeter Grehan 		return "vmxpreempt";
295366f6083SPeter Grehan 	case EXIT_REASON_INVVPID:
296366f6083SPeter Grehan 		return "invvpid";
297366f6083SPeter Grehan 	case EXIT_REASON_WBINVD:
298366f6083SPeter Grehan 		return "wbinvd";
299366f6083SPeter Grehan 	case EXIT_REASON_XSETBV:
300366f6083SPeter Grehan 		return "xsetbv";
30188c4b8d1SNeel Natu 	case EXIT_REASON_APIC_WRITE:
30288c4b8d1SNeel Natu 		return "apic-write";
303366f6083SPeter Grehan 	default:
304366f6083SPeter Grehan 		snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason);
305366f6083SPeter Grehan 		return (reasonbuf);
306366f6083SPeter Grehan 	}
307366f6083SPeter Grehan }
308366f6083SPeter Grehan #endif	/* KTR */
309366f6083SPeter Grehan 
310159dd56fSNeel Natu static int
311159dd56fSNeel Natu vmx_allow_x2apic_msrs(struct vmx *vmx)
312159dd56fSNeel Natu {
313159dd56fSNeel Natu 	int i, error;
314159dd56fSNeel Natu 
315159dd56fSNeel Natu 	error = 0;
316159dd56fSNeel Natu 
317159dd56fSNeel Natu 	/*
318159dd56fSNeel Natu 	 * Allow readonly access to the following x2APIC MSRs from the guest.
319159dd56fSNeel Natu 	 */
320159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ID);
321159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_VERSION);
322159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LDR);
323159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_SVR);
324159dd56fSNeel Natu 
325159dd56fSNeel Natu 	for (i = 0; i < 8; i++)
326159dd56fSNeel Natu 		error += guest_msr_ro(vmx, MSR_APIC_ISR0 + i);
327159dd56fSNeel Natu 
328159dd56fSNeel Natu 	for (i = 0; i < 8; i++)
329159dd56fSNeel Natu 		error += guest_msr_ro(vmx, MSR_APIC_TMR0 + i);
330159dd56fSNeel Natu 
331159dd56fSNeel Natu 	for (i = 0; i < 8; i++)
332159dd56fSNeel Natu 		error += guest_msr_ro(vmx, MSR_APIC_IRR0 + i);
333159dd56fSNeel Natu 
334159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ESR);
335159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_TIMER);
336159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_THERMAL);
337159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_PCINT);
338159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT0);
339159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT1);
340159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_ERROR);
341159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ICR_TIMER);
342159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_DCR_TIMER);
343159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ICR);
344159dd56fSNeel Natu 
345159dd56fSNeel Natu 	/*
346159dd56fSNeel Natu 	 * Allow TPR, EOI and SELF_IPI MSRs to be read and written by the guest.
347159dd56fSNeel Natu 	 *
348159dd56fSNeel Natu 	 * These registers get special treatment described in the section
349159dd56fSNeel Natu 	 * "Virtualizing MSR-Based APIC Accesses".
350159dd56fSNeel Natu 	 */
351159dd56fSNeel Natu 	error += guest_msr_rw(vmx, MSR_APIC_TPR);
352159dd56fSNeel Natu 	error += guest_msr_rw(vmx, MSR_APIC_EOI);
353159dd56fSNeel Natu 	error += guest_msr_rw(vmx, MSR_APIC_SELF_IPI);
354159dd56fSNeel Natu 
355159dd56fSNeel Natu 	return (error);
356159dd56fSNeel Natu }
357159dd56fSNeel Natu 
358366f6083SPeter Grehan u_long
359366f6083SPeter Grehan vmx_fix_cr0(u_long cr0)
360366f6083SPeter Grehan {
361366f6083SPeter Grehan 
362366f6083SPeter Grehan 	return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask);
363366f6083SPeter Grehan }
364366f6083SPeter Grehan 
365366f6083SPeter Grehan u_long
366366f6083SPeter Grehan vmx_fix_cr4(u_long cr4)
367366f6083SPeter Grehan {
368366f6083SPeter Grehan 
369366f6083SPeter Grehan 	return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask);
370366f6083SPeter Grehan }
371366f6083SPeter Grehan 
372366f6083SPeter Grehan static void
37345e51299SNeel Natu vpid_free(int vpid)
37445e51299SNeel Natu {
37545e51299SNeel Natu 	if (vpid < 0 || vpid > 0xffff)
37645e51299SNeel Natu 		panic("vpid_free: invalid vpid %d", vpid);
37745e51299SNeel Natu 
37845e51299SNeel Natu 	/*
37945e51299SNeel Natu 	 * VPIDs [0,VM_MAXCPU] are special and are not allocated from
38045e51299SNeel Natu 	 * the unit number allocator.
38145e51299SNeel Natu 	 */
38245e51299SNeel Natu 
38345e51299SNeel Natu 	if (vpid > VM_MAXCPU)
38445e51299SNeel Natu 		free_unr(vpid_unr, vpid);
38545e51299SNeel Natu }
38645e51299SNeel Natu 
38745e51299SNeel Natu static void
38845e51299SNeel Natu vpid_alloc(uint16_t *vpid, int num)
38945e51299SNeel Natu {
39045e51299SNeel Natu 	int i, x;
39145e51299SNeel Natu 
39245e51299SNeel Natu 	if (num <= 0 || num > VM_MAXCPU)
39345e51299SNeel Natu 		panic("invalid number of vpids requested: %d", num);
39445e51299SNeel Natu 
39545e51299SNeel Natu 	/*
39645e51299SNeel Natu 	 * If the "enable vpid" execution control is not enabled then the
39745e51299SNeel Natu 	 * VPID is required to be 0 for all vcpus.
39845e51299SNeel Natu 	 */
39945e51299SNeel Natu 	if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) == 0) {
40045e51299SNeel Natu 		for (i = 0; i < num; i++)
40145e51299SNeel Natu 			vpid[i] = 0;
40245e51299SNeel Natu 		return;
40345e51299SNeel Natu 	}
40445e51299SNeel Natu 
40545e51299SNeel Natu 	/*
40645e51299SNeel Natu 	 * Allocate a unique VPID for each vcpu from the unit number allocator.
40745e51299SNeel Natu 	 */
40845e51299SNeel Natu 	for (i = 0; i < num; i++) {
40945e51299SNeel Natu 		x = alloc_unr(vpid_unr);
41045e51299SNeel Natu 		if (x == -1)
41145e51299SNeel Natu 			break;
41245e51299SNeel Natu 		else
41345e51299SNeel Natu 			vpid[i] = x;
41445e51299SNeel Natu 	}
41545e51299SNeel Natu 
41645e51299SNeel Natu 	if (i < num) {
41745e51299SNeel Natu 		atomic_add_int(&vpid_alloc_failed, 1);
41845e51299SNeel Natu 
41945e51299SNeel Natu 		/*
42045e51299SNeel Natu 		 * If the unit number allocator does not have enough unique
42145e51299SNeel Natu 		 * VPIDs then we need to allocate from the [1,VM_MAXCPU] range.
42245e51299SNeel Natu 		 *
42345e51299SNeel Natu 		 * These VPIDs are not be unique across VMs but this does not
42445e51299SNeel Natu 		 * affect correctness because the combined mappings are also
42545e51299SNeel Natu 		 * tagged with the EP4TA which is unique for each VM.
42645e51299SNeel Natu 		 *
42745e51299SNeel Natu 		 * It is still sub-optimal because the invvpid will invalidate
42845e51299SNeel Natu 		 * combined mappings for a particular VPID across all EP4TAs.
42945e51299SNeel Natu 		 */
43045e51299SNeel Natu 		while (i-- > 0)
43145e51299SNeel Natu 			vpid_free(vpid[i]);
43245e51299SNeel Natu 
43345e51299SNeel Natu 		for (i = 0; i < num; i++)
43445e51299SNeel Natu 			vpid[i] = i + 1;
43545e51299SNeel Natu 	}
43645e51299SNeel Natu }
43745e51299SNeel Natu 
43845e51299SNeel Natu static void
43945e51299SNeel Natu vpid_init(void)
44045e51299SNeel Natu {
44145e51299SNeel Natu 	/*
44245e51299SNeel Natu 	 * VPID 0 is required when the "enable VPID" execution control is
44345e51299SNeel Natu 	 * disabled.
44445e51299SNeel Natu 	 *
44545e51299SNeel Natu 	 * VPIDs [1,VM_MAXCPU] are used as the "overflow namespace" when the
44645e51299SNeel Natu 	 * unit number allocator does not have sufficient unique VPIDs to
44745e51299SNeel Natu 	 * satisfy the allocation.
44845e51299SNeel Natu 	 *
44945e51299SNeel Natu 	 * The remaining VPIDs are managed by the unit number allocator.
45045e51299SNeel Natu 	 */
45145e51299SNeel Natu 	vpid_unr = new_unrhdr(VM_MAXCPU + 1, 0xffff, NULL);
45245e51299SNeel Natu }
45345e51299SNeel Natu 
45445e51299SNeel Natu static void
455366f6083SPeter Grehan msr_save_area_init(struct msr_entry *g_area, int *g_count)
456366f6083SPeter Grehan {
457366f6083SPeter Grehan 	int cnt;
458366f6083SPeter Grehan 
459366f6083SPeter Grehan 	static struct msr_entry guest_msrs[] = {
460366f6083SPeter Grehan 		{ MSR_KGSBASE, 0, 0 },
461366f6083SPeter Grehan 	};
462366f6083SPeter Grehan 
463366f6083SPeter Grehan 	cnt = sizeof(guest_msrs) / sizeof(guest_msrs[0]);
464366f6083SPeter Grehan 	if (cnt > GUEST_MSR_MAX_ENTRIES)
465366f6083SPeter Grehan 		panic("guest msr save area overrun");
466366f6083SPeter Grehan 	bcopy(guest_msrs, g_area, sizeof(guest_msrs));
467366f6083SPeter Grehan 	*g_count = cnt;
468366f6083SPeter Grehan }
469366f6083SPeter Grehan 
470366f6083SPeter Grehan static void
471366f6083SPeter Grehan vmx_disable(void *arg __unused)
472366f6083SPeter Grehan {
473366f6083SPeter Grehan 	struct invvpid_desc invvpid_desc = { 0 };
474366f6083SPeter Grehan 	struct invept_desc invept_desc = { 0 };
475366f6083SPeter Grehan 
476366f6083SPeter Grehan 	if (vmxon_enabled[curcpu]) {
477366f6083SPeter Grehan 		/*
478366f6083SPeter Grehan 		 * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b.
479366f6083SPeter Grehan 		 *
480366f6083SPeter Grehan 		 * VMXON or VMXOFF are not required to invalidate any TLB
481366f6083SPeter Grehan 		 * caching structures. This prevents potential retention of
482366f6083SPeter Grehan 		 * cached information in the TLB between distinct VMX episodes.
483366f6083SPeter Grehan 		 */
484366f6083SPeter Grehan 		invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc);
485366f6083SPeter Grehan 		invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc);
486366f6083SPeter Grehan 		vmxoff();
487366f6083SPeter Grehan 	}
488366f6083SPeter Grehan 	load_cr4(rcr4() & ~CR4_VMXE);
489366f6083SPeter Grehan }
490366f6083SPeter Grehan 
491366f6083SPeter Grehan static int
492366f6083SPeter Grehan vmx_cleanup(void)
493366f6083SPeter Grehan {
494366f6083SPeter Grehan 
495176666c2SNeel Natu 	if (pirvec != 0)
496176666c2SNeel Natu 		vmm_ipi_free(pirvec);
497176666c2SNeel Natu 
49845e51299SNeel Natu 	if (vpid_unr != NULL) {
49945e51299SNeel Natu 		delete_unrhdr(vpid_unr);
50045e51299SNeel Natu 		vpid_unr = NULL;
50145e51299SNeel Natu 	}
50245e51299SNeel Natu 
503366f6083SPeter Grehan 	smp_rendezvous(NULL, vmx_disable, NULL, NULL);
504366f6083SPeter Grehan 
505366f6083SPeter Grehan 	return (0);
506366f6083SPeter Grehan }
507366f6083SPeter Grehan 
508366f6083SPeter Grehan static void
509366f6083SPeter Grehan vmx_enable(void *arg __unused)
510366f6083SPeter Grehan {
511366f6083SPeter Grehan 	int error;
512366f6083SPeter Grehan 
513366f6083SPeter Grehan 	load_cr4(rcr4() | CR4_VMXE);
514366f6083SPeter Grehan 
515366f6083SPeter Grehan 	*(uint32_t *)vmxon_region[curcpu] = vmx_revision();
516366f6083SPeter Grehan 	error = vmxon(vmxon_region[curcpu]);
517366f6083SPeter Grehan 	if (error == 0)
518366f6083SPeter Grehan 		vmxon_enabled[curcpu] = 1;
519366f6083SPeter Grehan }
520366f6083SPeter Grehan 
52163e62d39SJohn Baldwin static void
52263e62d39SJohn Baldwin vmx_restore(void)
52363e62d39SJohn Baldwin {
52463e62d39SJohn Baldwin 
52563e62d39SJohn Baldwin 	if (vmxon_enabled[curcpu])
52663e62d39SJohn Baldwin 		vmxon(vmxon_region[curcpu]);
52763e62d39SJohn Baldwin }
52863e62d39SJohn Baldwin 
529366f6083SPeter Grehan static int
530add611fdSNeel Natu vmx_init(int ipinum)
531366f6083SPeter Grehan {
53288c4b8d1SNeel Natu 	int error, use_tpr_shadow;
5334bff7fadSNeel Natu 	uint64_t fixed0, fixed1, feature_control;
53488c4b8d1SNeel Natu 	uint32_t tmp, procbased2_vid_bits;
535366f6083SPeter Grehan 
536366f6083SPeter Grehan 	/* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */
5378b287612SJohn Baldwin 	if (!(cpu_feature2 & CPUID2_VMX)) {
538366f6083SPeter Grehan 		printf("vmx_init: processor does not support VMX operation\n");
539366f6083SPeter Grehan 		return (ENXIO);
540366f6083SPeter Grehan 	}
541366f6083SPeter Grehan 
5424bff7fadSNeel Natu 	/*
5434bff7fadSNeel Natu 	 * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits
5444bff7fadSNeel Natu 	 * are set (bits 0 and 2 respectively).
5454bff7fadSNeel Natu 	 */
5464bff7fadSNeel Natu 	feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
547150369abSNeel Natu 	if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 0 ||
548150369abSNeel Natu 	    (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) {
5494bff7fadSNeel Natu 		printf("vmx_init: VMX operation disabled by BIOS\n");
5504bff7fadSNeel Natu 		return (ENXIO);
5514bff7fadSNeel Natu 	}
5524bff7fadSNeel Natu 
553366f6083SPeter Grehan 	/* Check support for primary processor-based VM-execution controls */
554366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
555366f6083SPeter Grehan 			       MSR_VMX_TRUE_PROCBASED_CTLS,
556366f6083SPeter Grehan 			       PROCBASED_CTLS_ONE_SETTING,
557366f6083SPeter Grehan 			       PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls);
558366f6083SPeter Grehan 	if (error) {
559366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired primary "
560366f6083SPeter Grehan 		       "processor-based controls\n");
561366f6083SPeter Grehan 		return (error);
562366f6083SPeter Grehan 	}
563366f6083SPeter Grehan 
564366f6083SPeter Grehan 	/* Clear the processor-based ctl bits that are set on demand */
565366f6083SPeter Grehan 	procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING;
566366f6083SPeter Grehan 
567366f6083SPeter Grehan 	/* Check support for secondary processor-based VM-execution controls */
568366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
569366f6083SPeter Grehan 			       MSR_VMX_PROCBASED_CTLS2,
570366f6083SPeter Grehan 			       PROCBASED_CTLS2_ONE_SETTING,
571366f6083SPeter Grehan 			       PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2);
572366f6083SPeter Grehan 	if (error) {
573366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired secondary "
574366f6083SPeter Grehan 		       "processor-based controls\n");
575366f6083SPeter Grehan 		return (error);
576366f6083SPeter Grehan 	}
577366f6083SPeter Grehan 
578366f6083SPeter Grehan 	/* Check support for VPID */
579366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
580366f6083SPeter Grehan 			       PROCBASED2_ENABLE_VPID, 0, &tmp);
581366f6083SPeter Grehan 	if (error == 0)
582366f6083SPeter Grehan 		procbased_ctls2 |= PROCBASED2_ENABLE_VPID;
583366f6083SPeter Grehan 
584366f6083SPeter Grehan 	/* Check support for pin-based VM-execution controls */
585366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
586366f6083SPeter Grehan 			       MSR_VMX_TRUE_PINBASED_CTLS,
587366f6083SPeter Grehan 			       PINBASED_CTLS_ONE_SETTING,
588366f6083SPeter Grehan 			       PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls);
589366f6083SPeter Grehan 	if (error) {
590366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired "
591366f6083SPeter Grehan 		       "pin-based controls\n");
592366f6083SPeter Grehan 		return (error);
593366f6083SPeter Grehan 	}
594366f6083SPeter Grehan 
595366f6083SPeter Grehan 	/* Check support for VM-exit controls */
596366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS,
597366f6083SPeter Grehan 			       VM_EXIT_CTLS_ONE_SETTING,
598366f6083SPeter Grehan 			       VM_EXIT_CTLS_ZERO_SETTING,
599366f6083SPeter Grehan 			       &exit_ctls);
600366f6083SPeter Grehan 	if (error) {
601608f97c3SPeter Grehan 		/* Try again without the PAT MSR bits */
602608f97c3SPeter Grehan 		error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS,
603608f97c3SPeter Grehan 				       MSR_VMX_TRUE_EXIT_CTLS,
604608f97c3SPeter Grehan 				       VM_EXIT_CTLS_ONE_SETTING_NO_PAT,
605608f97c3SPeter Grehan 				       VM_EXIT_CTLS_ZERO_SETTING,
606608f97c3SPeter Grehan 				       &exit_ctls);
607608f97c3SPeter Grehan 		if (error) {
608366f6083SPeter Grehan 			printf("vmx_init: processor does not support desired "
609366f6083SPeter Grehan 			       "exit controls\n");
610366f6083SPeter Grehan 			return (error);
611608f97c3SPeter Grehan 		} else {
612608f97c3SPeter Grehan 			if (bootverbose)
613608f97c3SPeter Grehan 				printf("vmm: PAT MSR access not supported\n");
614608f97c3SPeter Grehan 			guest_msr_valid(MSR_PAT);
615608f97c3SPeter Grehan 			vmx_no_patmsr = 1;
616608f97c3SPeter Grehan 		}
617366f6083SPeter Grehan 	}
618366f6083SPeter Grehan 
619366f6083SPeter Grehan 	/* Check support for VM-entry controls */
620608f97c3SPeter Grehan 	if (!vmx_no_patmsr) {
621608f97c3SPeter Grehan 		error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS,
622608f97c3SPeter Grehan 				       MSR_VMX_TRUE_ENTRY_CTLS,
623366f6083SPeter Grehan 				       VM_ENTRY_CTLS_ONE_SETTING,
624366f6083SPeter Grehan 				       VM_ENTRY_CTLS_ZERO_SETTING,
625366f6083SPeter Grehan 				       &entry_ctls);
626608f97c3SPeter Grehan 	} else {
627608f97c3SPeter Grehan 		error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS,
628608f97c3SPeter Grehan 				       MSR_VMX_TRUE_ENTRY_CTLS,
629608f97c3SPeter Grehan 				       VM_ENTRY_CTLS_ONE_SETTING_NO_PAT,
630608f97c3SPeter Grehan 				       VM_ENTRY_CTLS_ZERO_SETTING,
631608f97c3SPeter Grehan 				       &entry_ctls);
632608f97c3SPeter Grehan 	}
633608f97c3SPeter Grehan 
634366f6083SPeter Grehan 	if (error) {
635366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired "
636366f6083SPeter Grehan 		       "entry controls\n");
637366f6083SPeter Grehan 		       return (error);
638366f6083SPeter Grehan 	}
639366f6083SPeter Grehan 
640366f6083SPeter Grehan 	/*
641366f6083SPeter Grehan 	 * Check support for optional features by testing them
642366f6083SPeter Grehan 	 * as individual bits
643366f6083SPeter Grehan 	 */
644366f6083SPeter Grehan 	cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
645366f6083SPeter Grehan 					MSR_VMX_TRUE_PROCBASED_CTLS,
646366f6083SPeter Grehan 					PROCBASED_HLT_EXITING, 0,
647366f6083SPeter Grehan 					&tmp) == 0);
648366f6083SPeter Grehan 
649366f6083SPeter Grehan 	cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
650366f6083SPeter Grehan 					MSR_VMX_PROCBASED_CTLS,
651366f6083SPeter Grehan 					PROCBASED_MTF, 0,
652366f6083SPeter Grehan 					&tmp) == 0);
653366f6083SPeter Grehan 
654366f6083SPeter Grehan 	cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
655366f6083SPeter Grehan 					 MSR_VMX_TRUE_PROCBASED_CTLS,
656366f6083SPeter Grehan 					 PROCBASED_PAUSE_EXITING, 0,
657366f6083SPeter Grehan 					 &tmp) == 0);
658366f6083SPeter Grehan 
659366f6083SPeter Grehan 	cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
660366f6083SPeter Grehan 					MSR_VMX_PROCBASED_CTLS2,
661366f6083SPeter Grehan 					PROCBASED2_UNRESTRICTED_GUEST, 0,
662366f6083SPeter Grehan 				        &tmp) == 0);
663366f6083SPeter Grehan 
66449cc03daSNeel Natu 	cap_invpcid = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
66549cc03daSNeel Natu 	    MSR_VMX_PROCBASED_CTLS2, PROCBASED2_ENABLE_INVPCID, 0,
66649cc03daSNeel Natu 	    &tmp) == 0);
66749cc03daSNeel Natu 
66888c4b8d1SNeel Natu 	/*
66988c4b8d1SNeel Natu 	 * Check support for virtual interrupt delivery.
67088c4b8d1SNeel Natu 	 */
67188c4b8d1SNeel Natu 	procbased2_vid_bits = (PROCBASED2_VIRTUALIZE_APIC_ACCESSES |
67288c4b8d1SNeel Natu 	    PROCBASED2_VIRTUALIZE_X2APIC_MODE |
67388c4b8d1SNeel Natu 	    PROCBASED2_APIC_REGISTER_VIRTUALIZATION |
67488c4b8d1SNeel Natu 	    PROCBASED2_VIRTUAL_INTERRUPT_DELIVERY);
67588c4b8d1SNeel Natu 
67688c4b8d1SNeel Natu 	use_tpr_shadow = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
67788c4b8d1SNeel Natu 	    MSR_VMX_TRUE_PROCBASED_CTLS, PROCBASED_USE_TPR_SHADOW, 0,
67888c4b8d1SNeel Natu 	    &tmp) == 0);
67988c4b8d1SNeel Natu 
68088c4b8d1SNeel Natu 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
68188c4b8d1SNeel Natu 	    procbased2_vid_bits, 0, &tmp);
68288c4b8d1SNeel Natu 	if (error == 0 && use_tpr_shadow) {
68388c4b8d1SNeel Natu 		virtual_interrupt_delivery = 1;
68488c4b8d1SNeel Natu 		TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_vid",
68588c4b8d1SNeel Natu 		    &virtual_interrupt_delivery);
68688c4b8d1SNeel Natu 	}
68788c4b8d1SNeel Natu 
68888c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
68988c4b8d1SNeel Natu 		procbased_ctls |= PROCBASED_USE_TPR_SHADOW;
69088c4b8d1SNeel Natu 		procbased_ctls2 |= procbased2_vid_bits;
69188c4b8d1SNeel Natu 		procbased_ctls2 &= ~PROCBASED2_VIRTUALIZE_X2APIC_MODE;
692176666c2SNeel Natu 
693176666c2SNeel Natu 		/*
694176666c2SNeel Natu 		 * Check for Posted Interrupts only if Virtual Interrupt
695176666c2SNeel Natu 		 * Delivery is enabled.
696176666c2SNeel Natu 		 */
697176666c2SNeel Natu 		error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
698176666c2SNeel Natu 		    MSR_VMX_TRUE_PINBASED_CTLS, PINBASED_POSTED_INTERRUPT, 0,
699176666c2SNeel Natu 		    &tmp);
700176666c2SNeel Natu 		if (error == 0) {
701176666c2SNeel Natu 			pirvec = vmm_ipi_alloc();
702176666c2SNeel Natu 			if (pirvec == 0) {
703176666c2SNeel Natu 				if (bootverbose) {
704176666c2SNeel Natu 					printf("vmx_init: unable to allocate "
705176666c2SNeel Natu 					    "posted interrupt vector\n");
70688c4b8d1SNeel Natu 				}
707176666c2SNeel Natu 			} else {
708176666c2SNeel Natu 				posted_interrupts = 1;
709176666c2SNeel Natu 				TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_pir",
710176666c2SNeel Natu 				    &posted_interrupts);
711176666c2SNeel Natu 			}
712176666c2SNeel Natu 		}
713176666c2SNeel Natu 	}
714176666c2SNeel Natu 
715176666c2SNeel Natu 	if (posted_interrupts)
716176666c2SNeel Natu 		    pinbased_ctls |= PINBASED_POSTED_INTERRUPT;
71749cc03daSNeel Natu 
718366f6083SPeter Grehan 	/* Initialize EPT */
719add611fdSNeel Natu 	error = ept_init(ipinum);
720366f6083SPeter Grehan 	if (error) {
721366f6083SPeter Grehan 		printf("vmx_init: ept initialization failed (%d)\n", error);
722366f6083SPeter Grehan 		return (error);
723366f6083SPeter Grehan 	}
724366f6083SPeter Grehan 
725366f6083SPeter Grehan 	/*
726366f6083SPeter Grehan 	 * Stash the cr0 and cr4 bits that must be fixed to 0 or 1
727366f6083SPeter Grehan 	 */
728366f6083SPeter Grehan 	fixed0 = rdmsr(MSR_VMX_CR0_FIXED0);
729366f6083SPeter Grehan 	fixed1 = rdmsr(MSR_VMX_CR0_FIXED1);
730366f6083SPeter Grehan 	cr0_ones_mask = fixed0 & fixed1;
731366f6083SPeter Grehan 	cr0_zeros_mask = ~fixed0 & ~fixed1;
732366f6083SPeter Grehan 
733366f6083SPeter Grehan 	/*
734366f6083SPeter Grehan 	 * CR0_PE and CR0_PG can be set to zero in VMX non-root operation
735366f6083SPeter Grehan 	 * if unrestricted guest execution is allowed.
736366f6083SPeter Grehan 	 */
737366f6083SPeter Grehan 	if (cap_unrestricted_guest)
738366f6083SPeter Grehan 		cr0_ones_mask &= ~(CR0_PG | CR0_PE);
739366f6083SPeter Grehan 
740366f6083SPeter Grehan 	/*
741366f6083SPeter Grehan 	 * Do not allow the guest to set CR0_NW or CR0_CD.
742366f6083SPeter Grehan 	 */
743366f6083SPeter Grehan 	cr0_zeros_mask |= (CR0_NW | CR0_CD);
744366f6083SPeter Grehan 
745366f6083SPeter Grehan 	fixed0 = rdmsr(MSR_VMX_CR4_FIXED0);
746366f6083SPeter Grehan 	fixed1 = rdmsr(MSR_VMX_CR4_FIXED1);
747366f6083SPeter Grehan 	cr4_ones_mask = fixed0 & fixed1;
748366f6083SPeter Grehan 	cr4_zeros_mask = ~fixed0 & ~fixed1;
749366f6083SPeter Grehan 
75045e51299SNeel Natu 	vpid_init();
75145e51299SNeel Natu 
752366f6083SPeter Grehan 	/* enable VMX operation */
753366f6083SPeter Grehan 	smp_rendezvous(NULL, vmx_enable, NULL, NULL);
754366f6083SPeter Grehan 
7553565b59eSNeel Natu 	vmx_initialized = 1;
7563565b59eSNeel Natu 
757366f6083SPeter Grehan 	return (0);
758366f6083SPeter Grehan }
759366f6083SPeter Grehan 
760f7d47425SNeel Natu static void
761f7d47425SNeel Natu vmx_trigger_hostintr(int vector)
762f7d47425SNeel Natu {
763f7d47425SNeel Natu 	uintptr_t func;
764f7d47425SNeel Natu 	struct gate_descriptor *gd;
765f7d47425SNeel Natu 
766f7d47425SNeel Natu 	gd = &idt[vector];
767f7d47425SNeel Natu 
768f7d47425SNeel Natu 	KASSERT(vector >= 32 && vector <= 255, ("vmx_trigger_hostintr: "
769f7d47425SNeel Natu 	    "invalid vector %d", vector));
770f7d47425SNeel Natu 	KASSERT(gd->gd_p == 1, ("gate descriptor for vector %d not present",
771f7d47425SNeel Natu 	    vector));
772f7d47425SNeel Natu 	KASSERT(gd->gd_type == SDT_SYSIGT, ("gate descriptor for vector %d "
773f7d47425SNeel Natu 	    "has invalid type %d", vector, gd->gd_type));
774f7d47425SNeel Natu 	KASSERT(gd->gd_dpl == SEL_KPL, ("gate descriptor for vector %d "
775f7d47425SNeel Natu 	    "has invalid dpl %d", vector, gd->gd_dpl));
776f7d47425SNeel Natu 	KASSERT(gd->gd_selector == GSEL(GCODE_SEL, SEL_KPL), ("gate descriptor "
777f7d47425SNeel Natu 	    "for vector %d has invalid selector %d", vector, gd->gd_selector));
778f7d47425SNeel Natu 	KASSERT(gd->gd_ist == 0, ("gate descriptor for vector %d has invalid "
779f7d47425SNeel Natu 	    "IST %d", vector, gd->gd_ist));
780f7d47425SNeel Natu 
781f7d47425SNeel Natu 	func = ((long)gd->gd_hioffset << 16 | gd->gd_looffset);
782f7d47425SNeel Natu 	vmx_call_isr(func);
783f7d47425SNeel Natu }
784f7d47425SNeel Natu 
785366f6083SPeter Grehan static int
786aaaa0656SPeter Grehan vmx_setup_cr_shadow(int which, struct vmcs *vmcs, uint32_t initial)
787366f6083SPeter Grehan {
78839c21c2dSNeel Natu 	int error, mask_ident, shadow_ident;
789aaaa0656SPeter Grehan 	uint64_t mask_value;
790366f6083SPeter Grehan 
79139c21c2dSNeel Natu 	if (which != 0 && which != 4)
79239c21c2dSNeel Natu 		panic("vmx_setup_cr_shadow: unknown cr%d", which);
79339c21c2dSNeel Natu 
79439c21c2dSNeel Natu 	if (which == 0) {
79539c21c2dSNeel Natu 		mask_ident = VMCS_CR0_MASK;
79639c21c2dSNeel Natu 		mask_value = cr0_ones_mask | cr0_zeros_mask;
79739c21c2dSNeel Natu 		shadow_ident = VMCS_CR0_SHADOW;
79839c21c2dSNeel Natu 	} else {
79939c21c2dSNeel Natu 		mask_ident = VMCS_CR4_MASK;
80039c21c2dSNeel Natu 		mask_value = cr4_ones_mask | cr4_zeros_mask;
80139c21c2dSNeel Natu 		shadow_ident = VMCS_CR4_SHADOW;
80239c21c2dSNeel Natu 	}
80339c21c2dSNeel Natu 
804d3c11f40SPeter Grehan 	error = vmcs_setreg(vmcs, 0, VMCS_IDENT(mask_ident), mask_value);
805366f6083SPeter Grehan 	if (error)
806366f6083SPeter Grehan 		return (error);
807366f6083SPeter Grehan 
808aaaa0656SPeter Grehan 	error = vmcs_setreg(vmcs, 0, VMCS_IDENT(shadow_ident), initial);
809366f6083SPeter Grehan 	if (error)
810366f6083SPeter Grehan 		return (error);
811366f6083SPeter Grehan 
812366f6083SPeter Grehan 	return (0);
813366f6083SPeter Grehan }
814aaaa0656SPeter Grehan #define	vmx_setup_cr0_shadow(vmcs,init)	vmx_setup_cr_shadow(0, (vmcs), (init))
815aaaa0656SPeter Grehan #define	vmx_setup_cr4_shadow(vmcs,init)	vmx_setup_cr_shadow(4, (vmcs), (init))
816366f6083SPeter Grehan 
817366f6083SPeter Grehan static void *
818318224bbSNeel Natu vmx_vminit(struct vm *vm, pmap_t pmap)
819366f6083SPeter Grehan {
82045e51299SNeel Natu 	uint16_t vpid[VM_MAXCPU];
821366f6083SPeter Grehan 	int i, error, guest_msr_count;
822366f6083SPeter Grehan 	struct vmx *vmx;
823c847a506SNeel Natu 	struct vmcs *vmcs;
824366f6083SPeter Grehan 
825366f6083SPeter Grehan 	vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO);
826366f6083SPeter Grehan 	if ((uintptr_t)vmx & PAGE_MASK) {
827366f6083SPeter Grehan 		panic("malloc of struct vmx not aligned on %d byte boundary",
828366f6083SPeter Grehan 		      PAGE_SIZE);
829366f6083SPeter Grehan 	}
830366f6083SPeter Grehan 	vmx->vm = vm;
831366f6083SPeter Grehan 
832318224bbSNeel Natu 	vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pml4));
833318224bbSNeel Natu 
834366f6083SPeter Grehan 	/*
835366f6083SPeter Grehan 	 * Clean up EPTP-tagged guest physical and combined mappings
836366f6083SPeter Grehan 	 *
837366f6083SPeter Grehan 	 * VMX transitions are not required to invalidate any guest physical
838366f6083SPeter Grehan 	 * mappings. So, it may be possible for stale guest physical mappings
839366f6083SPeter Grehan 	 * to be present in the processor TLBs.
840366f6083SPeter Grehan 	 *
841366f6083SPeter Grehan 	 * Combined mappings for this EP4TA are also invalidated for all VPIDs.
842366f6083SPeter Grehan 	 */
843318224bbSNeel Natu 	ept_invalidate_mappings(vmx->eptp);
844366f6083SPeter Grehan 
845366f6083SPeter Grehan 	msr_bitmap_initialize(vmx->msr_bitmap);
846366f6083SPeter Grehan 
847366f6083SPeter Grehan 	/*
848366f6083SPeter Grehan 	 * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE.
849366f6083SPeter Grehan 	 * The guest FSBASE and GSBASE are saved and restored during
850366f6083SPeter Grehan 	 * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are
851366f6083SPeter Grehan 	 * always restored from the vmcs host state area on vm-exit.
852366f6083SPeter Grehan 	 *
8531fb0ea3fSPeter Grehan 	 * The SYSENTER_CS/ESP/EIP MSRs are identical to FS/GSBASE in
8541fb0ea3fSPeter Grehan 	 * how they are saved/restored so can be directly accessed by the
8551fb0ea3fSPeter Grehan 	 * guest.
8561fb0ea3fSPeter Grehan 	 *
857366f6083SPeter Grehan 	 * Guest KGSBASE is saved and restored in the guest MSR save area.
858366f6083SPeter Grehan 	 * Host KGSBASE is restored before returning to userland from the pcb.
859366f6083SPeter Grehan 	 * There will be a window of time when we are executing in the host
860366f6083SPeter Grehan 	 * kernel context with a value of KGSBASE from the guest. This is ok
861366f6083SPeter Grehan 	 * because the value of KGSBASE is inconsequential in kernel context.
862366f6083SPeter Grehan 	 *
863366f6083SPeter Grehan 	 * MSR_EFER is saved and restored in the guest VMCS area on a
864366f6083SPeter Grehan 	 * VM exit and entry respectively. It is also restored from the
865366f6083SPeter Grehan 	 * host VMCS area on a VM exit.
866366f6083SPeter Grehan 	 */
867366f6083SPeter Grehan 	if (guest_msr_rw(vmx, MSR_GSBASE) ||
868366f6083SPeter Grehan 	    guest_msr_rw(vmx, MSR_FSBASE) ||
8691fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_CS_MSR) ||
8701fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_ESP_MSR) ||
8711fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_EIP_MSR) ||
872366f6083SPeter Grehan 	    guest_msr_rw(vmx, MSR_KGSBASE) ||
873608f97c3SPeter Grehan 	    guest_msr_rw(vmx, MSR_EFER))
874366f6083SPeter Grehan 		panic("vmx_vminit: error setting guest msr access");
875366f6083SPeter Grehan 
876608f97c3SPeter Grehan 	/*
877608f97c3SPeter Grehan 	 * MSR_PAT is saved and restored in the guest VMCS are on a VM exit
878608f97c3SPeter Grehan 	 * and entry respectively. It is also restored from the host VMCS
879608f97c3SPeter Grehan 	 * area on a VM exit. However, if running on a system with no
880608f97c3SPeter Grehan 	 * MSR_PAT save/restore support, leave access disabled so accesses
881608f97c3SPeter Grehan 	 * will be trapped.
882608f97c3SPeter Grehan 	 */
883608f97c3SPeter Grehan 	if (!vmx_no_patmsr && guest_msr_rw(vmx, MSR_PAT))
884608f97c3SPeter Grehan 		panic("vmx_vminit: error setting guest pat msr access");
885608f97c3SPeter Grehan 
88645e51299SNeel Natu 	vpid_alloc(vpid, VM_MAXCPU);
88745e51299SNeel Natu 
88888c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
88988c4b8d1SNeel Natu 		error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE,
89088c4b8d1SNeel Natu 		    APIC_ACCESS_ADDRESS);
89188c4b8d1SNeel Natu 		/* XXX this should really return an error to the caller */
89288c4b8d1SNeel Natu 		KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error));
89388c4b8d1SNeel Natu 	}
89488c4b8d1SNeel Natu 
895366f6083SPeter Grehan 	for (i = 0; i < VM_MAXCPU; i++) {
896c847a506SNeel Natu 		vmcs = &vmx->vmcs[i];
897c847a506SNeel Natu 		vmcs->identifier = vmx_revision();
898c847a506SNeel Natu 		error = vmclear(vmcs);
899366f6083SPeter Grehan 		if (error != 0) {
900366f6083SPeter Grehan 			panic("vmx_vminit: vmclear error %d on vcpu %d\n",
901366f6083SPeter Grehan 			      error, i);
902366f6083SPeter Grehan 		}
903366f6083SPeter Grehan 
904c847a506SNeel Natu 		error = vmcs_init(vmcs);
905c847a506SNeel Natu 		KASSERT(error == 0, ("vmcs_init error %d", error));
906366f6083SPeter Grehan 
907c847a506SNeel Natu 		VMPTRLD(vmcs);
908c847a506SNeel Natu 		error = 0;
909c847a506SNeel Natu 		error += vmwrite(VMCS_HOST_RSP, (u_long)&vmx->ctx[i]);
910c847a506SNeel Natu 		error += vmwrite(VMCS_EPTP, vmx->eptp);
911c847a506SNeel Natu 		error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls);
912c847a506SNeel Natu 		error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls);
913c847a506SNeel Natu 		error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2);
914c847a506SNeel Natu 		error += vmwrite(VMCS_EXIT_CTLS, exit_ctls);
915c847a506SNeel Natu 		error += vmwrite(VMCS_ENTRY_CTLS, entry_ctls);
916c847a506SNeel Natu 		error += vmwrite(VMCS_MSR_BITMAP, vtophys(vmx->msr_bitmap));
917c847a506SNeel Natu 		error += vmwrite(VMCS_VPID, vpid[i]);
91888c4b8d1SNeel Natu 		if (virtual_interrupt_delivery) {
91988c4b8d1SNeel Natu 			error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS);
92088c4b8d1SNeel Natu 			error += vmwrite(VMCS_VIRTUAL_APIC,
92188c4b8d1SNeel Natu 			    vtophys(&vmx->apic_page[i]));
92288c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT0, 0);
92388c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT1, 0);
92488c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT2, 0);
92588c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT3, 0);
92688c4b8d1SNeel Natu 		}
927176666c2SNeel Natu 		if (posted_interrupts) {
928176666c2SNeel Natu 			error += vmwrite(VMCS_PIR_VECTOR, pirvec);
929176666c2SNeel Natu 			error += vmwrite(VMCS_PIR_DESC,
930176666c2SNeel Natu 			    vtophys(&vmx->pir_desc[i]));
931176666c2SNeel Natu 		}
932c847a506SNeel Natu 		VMCLEAR(vmcs);
933c847a506SNeel Natu 		KASSERT(error == 0, ("vmx_vminit: error customizing the vmcs"));
934366f6083SPeter Grehan 
935366f6083SPeter Grehan 		vmx->cap[i].set = 0;
936366f6083SPeter Grehan 		vmx->cap[i].proc_ctls = procbased_ctls;
93749cc03daSNeel Natu 		vmx->cap[i].proc_ctls2 = procbased_ctls2;
938366f6083SPeter Grehan 
939366f6083SPeter Grehan 		vmx->state[i].lastcpu = -1;
94045e51299SNeel Natu 		vmx->state[i].vpid = vpid[i];
941366f6083SPeter Grehan 
942366f6083SPeter Grehan 		msr_save_area_init(vmx->guest_msrs[i], &guest_msr_count);
943366f6083SPeter Grehan 
944c847a506SNeel Natu 		error = vmcs_set_msr_save(vmcs, vtophys(vmx->guest_msrs[i]),
945366f6083SPeter Grehan 		    guest_msr_count);
946366f6083SPeter Grehan 		if (error != 0)
947366f6083SPeter Grehan 			panic("vmcs_set_msr_save error %d", error);
948366f6083SPeter Grehan 
949aaaa0656SPeter Grehan 		/*
950aaaa0656SPeter Grehan 		 * Set up the CR0/4 shadows, and init the read shadow
951aaaa0656SPeter Grehan 		 * to the power-on register value from the Intel Sys Arch.
952aaaa0656SPeter Grehan 		 *  CR0 - 0x60000010
953aaaa0656SPeter Grehan 		 *  CR4 - 0
954aaaa0656SPeter Grehan 		 */
955c847a506SNeel Natu 		error = vmx_setup_cr0_shadow(vmcs, 0x60000010);
95639c21c2dSNeel Natu 		if (error != 0)
95739c21c2dSNeel Natu 			panic("vmx_setup_cr0_shadow %d", error);
95839c21c2dSNeel Natu 
959c847a506SNeel Natu 		error = vmx_setup_cr4_shadow(vmcs, 0);
96039c21c2dSNeel Natu 		if (error != 0)
96139c21c2dSNeel Natu 			panic("vmx_setup_cr4_shadow %d", error);
962318224bbSNeel Natu 
963318224bbSNeel Natu 		vmx->ctx[i].pmap = pmap;
964366f6083SPeter Grehan 	}
965366f6083SPeter Grehan 
966366f6083SPeter Grehan 	return (vmx);
967366f6083SPeter Grehan }
968366f6083SPeter Grehan 
969366f6083SPeter Grehan static int
970a2da7af6SNeel Natu vmx_handle_cpuid(struct vm *vm, int vcpu, struct vmxctx *vmxctx)
971366f6083SPeter Grehan {
972366f6083SPeter Grehan 	int handled, func;
973366f6083SPeter Grehan 
974366f6083SPeter Grehan 	func = vmxctx->guest_rax;
975366f6083SPeter Grehan 
976a2da7af6SNeel Natu 	handled = x86_emulate_cpuid(vm, vcpu,
977a2da7af6SNeel Natu 				    (uint32_t*)(&vmxctx->guest_rax),
978a2da7af6SNeel Natu 				    (uint32_t*)(&vmxctx->guest_rbx),
979a2da7af6SNeel Natu 				    (uint32_t*)(&vmxctx->guest_rcx),
980a2da7af6SNeel Natu 				    (uint32_t*)(&vmxctx->guest_rdx));
981366f6083SPeter Grehan 	return (handled);
982366f6083SPeter Grehan }
983366f6083SPeter Grehan 
984366f6083SPeter Grehan static __inline void
985366f6083SPeter Grehan vmx_run_trace(struct vmx *vmx, int vcpu)
986366f6083SPeter Grehan {
987366f6083SPeter Grehan #ifdef KTR
988513c8d33SNeel Natu 	VCPU_CTR1(vmx->vm, vcpu, "Resume execution at %#lx", vmcs_guest_rip());
989366f6083SPeter Grehan #endif
990366f6083SPeter Grehan }
991366f6083SPeter Grehan 
992366f6083SPeter Grehan static __inline void
993366f6083SPeter Grehan vmx_exit_trace(struct vmx *vmx, int vcpu, uint64_t rip, uint32_t exit_reason,
994eeefa4e4SNeel Natu 	       int handled)
995366f6083SPeter Grehan {
996366f6083SPeter Grehan #ifdef KTR
997513c8d33SNeel Natu 	VCPU_CTR3(vmx->vm, vcpu, "%s %s vmexit at 0x%0lx",
998366f6083SPeter Grehan 		 handled ? "handled" : "unhandled",
999366f6083SPeter Grehan 		 exit_reason_to_str(exit_reason), rip);
1000eeefa4e4SNeel Natu #endif
1001eeefa4e4SNeel Natu }
1002366f6083SPeter Grehan 
1003eeefa4e4SNeel Natu static __inline void
1004eeefa4e4SNeel Natu vmx_astpending_trace(struct vmx *vmx, int vcpu, uint64_t rip)
1005eeefa4e4SNeel Natu {
1006eeefa4e4SNeel Natu #ifdef KTR
1007513c8d33SNeel Natu 	VCPU_CTR1(vmx->vm, vcpu, "astpending vmexit at 0x%0lx", rip);
1008366f6083SPeter Grehan #endif
1009366f6083SPeter Grehan }
1010366f6083SPeter Grehan 
1011953c2c47SNeel Natu static VMM_STAT_INTEL(VCPU_INVVPID_SAVED, "Number of vpid invalidations saved");
1012953c2c47SNeel Natu 
10133de83862SNeel Natu static void
1014953c2c47SNeel Natu vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu, pmap_t pmap)
1015366f6083SPeter Grehan {
1016366f6083SPeter Grehan 	struct vmxstate *vmxstate;
1017953c2c47SNeel Natu 	struct invvpid_desc invvpid_desc;
1018366f6083SPeter Grehan 
1019366f6083SPeter Grehan 	vmxstate = &vmx->state[vcpu];
1020953c2c47SNeel Natu 	if (vmxstate->lastcpu == curcpu)
10213de83862SNeel Natu 		return;
1022366f6083SPeter Grehan 
1023953c2c47SNeel Natu 	vmxstate->lastcpu = curcpu;
1024953c2c47SNeel Natu 
1025366f6083SPeter Grehan 	vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1);
1026366f6083SPeter Grehan 
10273de83862SNeel Natu 	vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase());
10283de83862SNeel Natu 	vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase());
10293de83862SNeel Natu 	vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase());
1030366f6083SPeter Grehan 
1031366f6083SPeter Grehan 	/*
1032366f6083SPeter Grehan 	 * If we are using VPIDs then invalidate all mappings tagged with 'vpid'
1033366f6083SPeter Grehan 	 *
1034366f6083SPeter Grehan 	 * We do this because this vcpu was executing on a different host
1035366f6083SPeter Grehan 	 * cpu when it last ran. We do not track whether it invalidated
1036366f6083SPeter Grehan 	 * mappings associated with its 'vpid' during that run. So we must
1037366f6083SPeter Grehan 	 * assume that the mappings associated with 'vpid' on 'curcpu' are
1038366f6083SPeter Grehan 	 * stale and invalidate them.
1039366f6083SPeter Grehan 	 *
1040366f6083SPeter Grehan 	 * Note that we incur this penalty only when the scheduler chooses to
1041366f6083SPeter Grehan 	 * move the thread associated with this vcpu between host cpus.
1042366f6083SPeter Grehan 	 *
1043366f6083SPeter Grehan 	 * Note also that this will invalidate mappings tagged with 'vpid'
1044366f6083SPeter Grehan 	 * for "all" EP4TAs.
1045366f6083SPeter Grehan 	 */
1046366f6083SPeter Grehan 	if (vmxstate->vpid != 0) {
1047953c2c47SNeel Natu 		if (pmap->pm_eptgen == vmx->eptgen[curcpu]) {
1048953c2c47SNeel Natu 			invvpid_desc._res1 = 0;
1049953c2c47SNeel Natu 			invvpid_desc._res2 = 0;
1050366f6083SPeter Grehan 			invvpid_desc.vpid = vmxstate->vpid;
10510e30c5c0SWarner Losh 			invvpid_desc.linear_addr = 0;
1052366f6083SPeter Grehan 			invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc);
1053953c2c47SNeel Natu 		} else {
1054953c2c47SNeel Natu 			/*
1055953c2c47SNeel Natu 			 * The invvpid can be skipped if an invept is going to
1056953c2c47SNeel Natu 			 * be performed before entering the guest. The invept
1057953c2c47SNeel Natu 			 * will invalidate combined mappings tagged with
1058953c2c47SNeel Natu 			 * 'vmx->eptp' for all vpids.
1059953c2c47SNeel Natu 			 */
1060953c2c47SNeel Natu 			vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_SAVED, 1);
1061953c2c47SNeel Natu 		}
1062366f6083SPeter Grehan 	}
1063366f6083SPeter Grehan }
1064366f6083SPeter Grehan 
1065366f6083SPeter Grehan /*
1066366f6083SPeter Grehan  * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set.
1067366f6083SPeter Grehan  */
1068366f6083SPeter Grehan CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0);
1069366f6083SPeter Grehan 
1070366f6083SPeter Grehan static void __inline
1071366f6083SPeter Grehan vmx_set_int_window_exiting(struct vmx *vmx, int vcpu)
1072366f6083SPeter Grehan {
1073366f6083SPeter Grehan 
107448b2d828SNeel Natu 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) {
1075366f6083SPeter Grehan 		vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING;
10763de83862SNeel Natu 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
107748b2d828SNeel Natu 		VCPU_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting");
107848b2d828SNeel Natu 	}
1079366f6083SPeter Grehan }
1080366f6083SPeter Grehan 
1081366f6083SPeter Grehan static void __inline
1082366f6083SPeter Grehan vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu)
1083366f6083SPeter Grehan {
1084366f6083SPeter Grehan 
108548b2d828SNeel Natu 	KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0,
108648b2d828SNeel Natu 	    ("intr_window_exiting not set: %#x", vmx->cap[vcpu].proc_ctls));
1087366f6083SPeter Grehan 	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING;
10883de83862SNeel Natu 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
108948b2d828SNeel Natu 	VCPU_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting");
1090366f6083SPeter Grehan }
1091366f6083SPeter Grehan 
1092366f6083SPeter Grehan static void __inline
1093366f6083SPeter Grehan vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu)
1094366f6083SPeter Grehan {
1095366f6083SPeter Grehan 
109648b2d828SNeel Natu 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) {
1097366f6083SPeter Grehan 		vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING;
10983de83862SNeel Natu 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
109948b2d828SNeel Natu 		VCPU_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting");
110048b2d828SNeel Natu 	}
1101366f6083SPeter Grehan }
1102366f6083SPeter Grehan 
1103366f6083SPeter Grehan static void __inline
1104366f6083SPeter Grehan vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu)
1105366f6083SPeter Grehan {
1106366f6083SPeter Grehan 
110748b2d828SNeel Natu 	KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0,
110848b2d828SNeel Natu 	    ("nmi_window_exiting not set %#x", vmx->cap[vcpu].proc_ctls));
1109366f6083SPeter Grehan 	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING;
11103de83862SNeel Natu 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
111148b2d828SNeel Natu 	VCPU_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting");
1112366f6083SPeter Grehan }
1113366f6083SPeter Grehan 
111448b2d828SNeel Natu #define	NMI_BLOCKING	(VMCS_INTERRUPTIBILITY_NMI_BLOCKING |		\
111548b2d828SNeel Natu 			 VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
111648b2d828SNeel Natu #define	HWINTR_BLOCKING	(VMCS_INTERRUPTIBILITY_STI_BLOCKING |		\
111748b2d828SNeel Natu 			 VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
111848b2d828SNeel Natu 
111948b2d828SNeel Natu static void
1120366f6083SPeter Grehan vmx_inject_nmi(struct vmx *vmx, int vcpu)
1121366f6083SPeter Grehan {
112248b2d828SNeel Natu 	uint32_t gi, info;
1123366f6083SPeter Grehan 
112448b2d828SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
112548b2d828SNeel Natu 	KASSERT((gi & NMI_BLOCKING) == 0, ("vmx_inject_nmi: invalid guest "
112648b2d828SNeel Natu 	    "interruptibility-state %#x", gi));
1127366f6083SPeter Grehan 
112848b2d828SNeel Natu 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
112948b2d828SNeel Natu 	KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_nmi: invalid "
113048b2d828SNeel Natu 	    "VM-entry interruption information %#x", info));
1131366f6083SPeter Grehan 
1132366f6083SPeter Grehan 	/*
1133366f6083SPeter Grehan 	 * Inject the virtual NMI. The vector must be the NMI IDT entry
1134366f6083SPeter Grehan 	 * or the VMCS entry check will fail.
1135366f6083SPeter Grehan 	 */
113648b2d828SNeel Natu 	info = IDT_NMI | VMCS_INTR_T_NMI | VMCS_INTR_VALID;
11373de83862SNeel Natu 	vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1138366f6083SPeter Grehan 
1139513c8d33SNeel Natu 	VCPU_CTR0(vmx->vm, vcpu, "Injecting vNMI");
1140366f6083SPeter Grehan 
1141366f6083SPeter Grehan 	/* Clear the request */
1142f352ff0cSNeel Natu 	vm_nmi_clear(vmx->vm, vcpu);
1143366f6083SPeter Grehan }
1144366f6083SPeter Grehan 
1145366f6083SPeter Grehan static void
1146de5ea6b6SNeel Natu vmx_inject_interrupts(struct vmx *vmx, int vcpu, struct vlapic *vlapic)
1147366f6083SPeter Grehan {
1148dc506506SNeel Natu 	struct vm_exception exc;
11490775fbb4STycho Nightingale 	int vector, need_nmi_exiting, extint_pending;
115048b2d828SNeel Natu 	uint64_t rflags;
115148b2d828SNeel Natu 	uint32_t gi, info;
1152366f6083SPeter Grehan 
1153dc506506SNeel Natu 	if (vm_exception_pending(vmx->vm, vcpu, &exc)) {
1154dc506506SNeel Natu 		KASSERT(exc.vector >= 0 && exc.vector < 32,
1155dc506506SNeel Natu 		    ("%s: invalid exception vector %d", __func__, exc.vector));
1156dc506506SNeel Natu 
1157dc506506SNeel Natu 		info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1158dc506506SNeel Natu 		KASSERT((info & VMCS_INTR_VALID) == 0, ("%s: cannot inject "
1159dc506506SNeel Natu 		     "pending exception %d: %#x", __func__, exc.vector, info));
1160dc506506SNeel Natu 
1161dc506506SNeel Natu 		info = exc.vector | VMCS_INTR_T_HWEXCEPTION | VMCS_INTR_VALID;
1162dc506506SNeel Natu 		if (exc.error_code_valid) {
1163dc506506SNeel Natu 			info |= VMCS_INTR_DEL_ERRCODE;
1164dc506506SNeel Natu 			vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR, exc.error_code);
1165dc506506SNeel Natu 		}
1166dc506506SNeel Natu 		vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1167dc506506SNeel Natu 	}
1168dc506506SNeel Natu 
116948b2d828SNeel Natu 	if (vm_nmi_pending(vmx->vm, vcpu)) {
1170366f6083SPeter Grehan 		/*
117148b2d828SNeel Natu 		 * If there are no conditions blocking NMI injection then
117248b2d828SNeel Natu 		 * inject it directly here otherwise enable "NMI window
117348b2d828SNeel Natu 		 * exiting" to inject it as soon as we can.
1174eeefa4e4SNeel Natu 		 *
117548b2d828SNeel Natu 		 * We also check for STI_BLOCKING because some implementations
117648b2d828SNeel Natu 		 * don't allow NMI injection in this case. If we are running
117748b2d828SNeel Natu 		 * on a processor that doesn't have this restriction it will
117848b2d828SNeel Natu 		 * immediately exit and the NMI will be injected in the
117948b2d828SNeel Natu 		 * "NMI window exiting" handler.
1180366f6083SPeter Grehan 		 */
118148b2d828SNeel Natu 		need_nmi_exiting = 1;
118248b2d828SNeel Natu 		gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
118348b2d828SNeel Natu 		if ((gi & (HWINTR_BLOCKING | NMI_BLOCKING)) == 0) {
11843de83862SNeel Natu 			info = vmcs_read(VMCS_ENTRY_INTR_INFO);
118548b2d828SNeel Natu 			if ((info & VMCS_INTR_VALID) == 0) {
118648b2d828SNeel Natu 				vmx_inject_nmi(vmx, vcpu);
118748b2d828SNeel Natu 				need_nmi_exiting = 0;
118848b2d828SNeel Natu 			} else {
118948b2d828SNeel Natu 				VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI "
119048b2d828SNeel Natu 				    "due to VM-entry intr info %#x", info);
119148b2d828SNeel Natu 			}
119248b2d828SNeel Natu 		} else {
119348b2d828SNeel Natu 			VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI due to "
119448b2d828SNeel Natu 			    "Guest Interruptibility-state %#x", gi);
119548b2d828SNeel Natu 		}
1196eeefa4e4SNeel Natu 
119748b2d828SNeel Natu 		if (need_nmi_exiting)
119848b2d828SNeel Natu 			vmx_set_nmi_window_exiting(vmx, vcpu);
119948b2d828SNeel Natu 	}
1200366f6083SPeter Grehan 
12010775fbb4STycho Nightingale 	extint_pending = vm_extint_pending(vmx->vm, vcpu);
12020775fbb4STycho Nightingale 
12030775fbb4STycho Nightingale 	if (!extint_pending && virtual_interrupt_delivery) {
120488c4b8d1SNeel Natu 		vmx_inject_pir(vlapic);
120588c4b8d1SNeel Natu 		return;
120688c4b8d1SNeel Natu 	}
120788c4b8d1SNeel Natu 
120848b2d828SNeel Natu 	/*
120936736912SNeel Natu 	 * If interrupt-window exiting is already in effect then don't bother
121036736912SNeel Natu 	 * checking for pending interrupts. This is just an optimization and
121136736912SNeel Natu 	 * not needed for correctness.
121248b2d828SNeel Natu 	 */
121336736912SNeel Natu 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) {
121436736912SNeel Natu 		VCPU_CTR0(vmx->vm, vcpu, "Skip interrupt injection due to "
121536736912SNeel Natu 		    "pending int_window_exiting");
121648b2d828SNeel Natu 		return;
121736736912SNeel Natu 	}
121848b2d828SNeel Natu 
12190775fbb4STycho Nightingale 	if (!extint_pending) {
1220366f6083SPeter Grehan 		/* Ask the local apic for a vector to inject */
12214d1e82a8SNeel Natu 		if (!vlapic_pending_intr(vlapic, &vector))
1222366f6083SPeter Grehan 			return;
12230775fbb4STycho Nightingale 	} else {
12240775fbb4STycho Nightingale 		/* Ask the legacy pic for a vector to inject */
12250775fbb4STycho Nightingale 		vatpic_pending_intr(vmx->vm, &vector);
12260775fbb4STycho Nightingale 	}
1227366f6083SPeter Grehan 
122848b2d828SNeel Natu 	KASSERT(vector >= 32 && vector <= 255, ("invalid vector %d", vector));
1229366f6083SPeter Grehan 
1230366f6083SPeter Grehan 	/* Check RFLAGS.IF and the interruptibility state of the guest */
12313de83862SNeel Natu 	rflags = vmcs_read(VMCS_GUEST_RFLAGS);
123236736912SNeel Natu 	if ((rflags & PSL_I) == 0) {
123336736912SNeel Natu 		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
123436736912SNeel Natu 		    "rflags %#lx", vector, rflags);
1235366f6083SPeter Grehan 		goto cantinject;
123636736912SNeel Natu 	}
1237366f6083SPeter Grehan 
123848b2d828SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
123936736912SNeel Natu 	if (gi & HWINTR_BLOCKING) {
124036736912SNeel Natu 		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
124136736912SNeel Natu 		    "Guest Interruptibility-state %#x", vector, gi);
1242366f6083SPeter Grehan 		goto cantinject;
124336736912SNeel Natu 	}
124436736912SNeel Natu 
124536736912SNeel Natu 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
124636736912SNeel Natu 	if (info & VMCS_INTR_VALID) {
124736736912SNeel Natu 		/*
124836736912SNeel Natu 		 * This is expected and could happen for multiple reasons:
124936736912SNeel Natu 		 * - A vectoring VM-entry was aborted due to astpending
125036736912SNeel Natu 		 * - A VM-exit happened during event injection.
1251dc506506SNeel Natu 		 * - An exception was injected above.
125236736912SNeel Natu 		 * - An NMI was injected above or after "NMI window exiting"
125336736912SNeel Natu 		 */
125436736912SNeel Natu 		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
125536736912SNeel Natu 		    "VM-entry intr info %#x", vector, info);
125636736912SNeel Natu 		goto cantinject;
125736736912SNeel Natu 	}
1258366f6083SPeter Grehan 
1259366f6083SPeter Grehan 	/* Inject the interrupt */
1260160471d2SNeel Natu 	info = VMCS_INTR_T_HWINTR | VMCS_INTR_VALID;
1261366f6083SPeter Grehan 	info |= vector;
12623de83862SNeel Natu 	vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1263366f6083SPeter Grehan 
12640775fbb4STycho Nightingale 	if (!extint_pending) {
1265366f6083SPeter Grehan 		/* Update the Local APIC ISR */
1266de5ea6b6SNeel Natu 		vlapic_intr_accepted(vlapic, vector);
12670775fbb4STycho Nightingale 	} else {
12680775fbb4STycho Nightingale 		vm_extint_clear(vmx->vm, vcpu);
12690775fbb4STycho Nightingale 		vatpic_intr_accepted(vmx->vm, vector);
12700775fbb4STycho Nightingale 
12710775fbb4STycho Nightingale 		/*
12720775fbb4STycho Nightingale 		 * After we accepted the current ExtINT the PIC may
12730775fbb4STycho Nightingale 		 * have posted another one.  If that is the case, set
12740775fbb4STycho Nightingale 		 * the Interrupt Window Exiting execution control so
12750775fbb4STycho Nightingale 		 * we can inject that one too.
12760775fbb4STycho Nightingale 		 */
12770775fbb4STycho Nightingale 		if (vm_extint_pending(vmx->vm, vcpu))
12780775fbb4STycho Nightingale 			vmx_set_int_window_exiting(vmx, vcpu);
12790775fbb4STycho Nightingale 	}
1280366f6083SPeter Grehan 
1281513c8d33SNeel Natu 	VCPU_CTR1(vmx->vm, vcpu, "Injecting hwintr at vector %d", vector);
1282366f6083SPeter Grehan 
1283366f6083SPeter Grehan 	return;
1284366f6083SPeter Grehan 
1285366f6083SPeter Grehan cantinject:
1286366f6083SPeter Grehan 	/*
1287366f6083SPeter Grehan 	 * Set the Interrupt Window Exiting execution control so we can inject
1288366f6083SPeter Grehan 	 * the interrupt as soon as blocking condition goes away.
1289366f6083SPeter Grehan 	 */
1290366f6083SPeter Grehan 	vmx_set_int_window_exiting(vmx, vcpu);
1291366f6083SPeter Grehan }
1292366f6083SPeter Grehan 
1293e5a1d950SNeel Natu /*
1294e5a1d950SNeel Natu  * If the Virtual NMIs execution control is '1' then the logical processor
1295e5a1d950SNeel Natu  * tracks virtual-NMI blocking in the Guest Interruptibility-state field of
1296e5a1d950SNeel Natu  * the VMCS. An IRET instruction in VMX non-root operation will remove any
1297e5a1d950SNeel Natu  * virtual-NMI blocking.
1298e5a1d950SNeel Natu  *
1299e5a1d950SNeel Natu  * This unblocking occurs even if the IRET causes a fault. In this case the
1300e5a1d950SNeel Natu  * hypervisor needs to restore virtual-NMI blocking before resuming the guest.
1301e5a1d950SNeel Natu  */
1302e5a1d950SNeel Natu static void
1303e5a1d950SNeel Natu vmx_restore_nmi_blocking(struct vmx *vmx, int vcpuid)
1304e5a1d950SNeel Natu {
1305e5a1d950SNeel Natu 	uint32_t gi;
1306e5a1d950SNeel Natu 
1307e5a1d950SNeel Natu 	VCPU_CTR0(vmx->vm, vcpuid, "Restore Virtual-NMI blocking");
1308e5a1d950SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1309e5a1d950SNeel Natu 	gi |= VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1310e5a1d950SNeel Natu 	vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1311e5a1d950SNeel Natu }
1312e5a1d950SNeel Natu 
1313e5a1d950SNeel Natu static void
1314e5a1d950SNeel Natu vmx_clear_nmi_blocking(struct vmx *vmx, int vcpuid)
1315e5a1d950SNeel Natu {
1316e5a1d950SNeel Natu 	uint32_t gi;
1317e5a1d950SNeel Natu 
1318e5a1d950SNeel Natu 	VCPU_CTR0(vmx->vm, vcpuid, "Clear Virtual-NMI blocking");
1319e5a1d950SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1320e5a1d950SNeel Natu 	gi &= ~VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1321e5a1d950SNeel Natu 	vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1322e5a1d950SNeel Natu }
1323e5a1d950SNeel Natu 
1324366f6083SPeter Grehan static int
1325a0efd3fbSJohn Baldwin vmx_emulate_xsetbv(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1326abb023fbSJohn Baldwin {
1327abb023fbSJohn Baldwin 	struct vmxctx *vmxctx;
1328abb023fbSJohn Baldwin 	uint64_t xcrval;
1329abb023fbSJohn Baldwin 	const struct xsave_limits *limits;
1330abb023fbSJohn Baldwin 
1331abb023fbSJohn Baldwin 	vmxctx = &vmx->ctx[vcpu];
1332abb023fbSJohn Baldwin 	limits = vmm_get_xsave_limits();
1333abb023fbSJohn Baldwin 
1334a0efd3fbSJohn Baldwin 	/*
1335a0efd3fbSJohn Baldwin 	 * Note that the processor raises a GP# fault on its own if
1336a0efd3fbSJohn Baldwin 	 * xsetbv is executed for CPL != 0, so we do not have to
1337a0efd3fbSJohn Baldwin 	 * emulate that fault here.
1338a0efd3fbSJohn Baldwin 	 */
1339a0efd3fbSJohn Baldwin 
1340a0efd3fbSJohn Baldwin 	/* Only xcr0 is supported. */
1341a0efd3fbSJohn Baldwin 	if (vmxctx->guest_rcx != 0) {
1342dc506506SNeel Natu 		vm_inject_gp(vmx->vm, vcpu);
1343a0efd3fbSJohn Baldwin 		return (HANDLED);
1344a0efd3fbSJohn Baldwin 	}
1345a0efd3fbSJohn Baldwin 
1346a0efd3fbSJohn Baldwin 	/* We only handle xcr0 if both the host and guest have XSAVE enabled. */
1347a0efd3fbSJohn Baldwin 	if (!limits->xsave_enabled || !(vmcs_read(VMCS_GUEST_CR4) & CR4_XSAVE)) {
1348dc506506SNeel Natu 		vm_inject_ud(vmx->vm, vcpu);
1349a0efd3fbSJohn Baldwin 		return (HANDLED);
1350a0efd3fbSJohn Baldwin 	}
1351abb023fbSJohn Baldwin 
1352abb023fbSJohn Baldwin 	xcrval = vmxctx->guest_rdx << 32 | (vmxctx->guest_rax & 0xffffffff);
1353a0efd3fbSJohn Baldwin 	if ((xcrval & ~limits->xcr0_allowed) != 0) {
1354dc506506SNeel Natu 		vm_inject_gp(vmx->vm, vcpu);
1355a0efd3fbSJohn Baldwin 		return (HANDLED);
1356a0efd3fbSJohn Baldwin 	}
1357abb023fbSJohn Baldwin 
1358a0efd3fbSJohn Baldwin 	if (!(xcrval & XFEATURE_ENABLED_X87)) {
1359dc506506SNeel Natu 		vm_inject_gp(vmx->vm, vcpu);
1360a0efd3fbSJohn Baldwin 		return (HANDLED);
1361a0efd3fbSJohn Baldwin 	}
1362abb023fbSJohn Baldwin 
1363abb023fbSJohn Baldwin 	if ((xcrval & (XFEATURE_ENABLED_AVX | XFEATURE_ENABLED_SSE)) ==
1364a0efd3fbSJohn Baldwin 	    XFEATURE_ENABLED_AVX) {
1365dc506506SNeel Natu 		vm_inject_gp(vmx->vm, vcpu);
1366a0efd3fbSJohn Baldwin 		return (HANDLED);
1367a0efd3fbSJohn Baldwin 	}
1368abb023fbSJohn Baldwin 
1369abb023fbSJohn Baldwin 	/*
1370abb023fbSJohn Baldwin 	 * This runs "inside" vmrun() with the guest's FPU state, so
1371abb023fbSJohn Baldwin 	 * modifying xcr0 directly modifies the guest's xcr0, not the
1372abb023fbSJohn Baldwin 	 * host's.
1373abb023fbSJohn Baldwin 	 */
1374abb023fbSJohn Baldwin 	load_xcr(0, xcrval);
1375abb023fbSJohn Baldwin 	return (HANDLED);
1376abb023fbSJohn Baldwin }
1377abb023fbSJohn Baldwin 
1378abb023fbSJohn Baldwin static int
1379366f6083SPeter Grehan vmx_emulate_cr_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1380366f6083SPeter Grehan {
13813de83862SNeel Natu 	int cr, vmcs_guest_cr, vmcs_shadow_cr;
138280a902efSPeter Grehan 	uint64_t crval, regval, ones_mask, zeros_mask;
1383366f6083SPeter Grehan 	const struct vmxctx *vmxctx;
1384366f6083SPeter Grehan 
138539c21c2dSNeel Natu 	/* We only handle mov to %cr0 or %cr4 at this time */
138639c21c2dSNeel Natu 	if ((exitqual & 0xf0) != 0x00)
138739c21c2dSNeel Natu 		return (UNHANDLED);
138839c21c2dSNeel Natu 
138939c21c2dSNeel Natu 	cr = exitqual & 0xf;
139039c21c2dSNeel Natu 	if (cr != 0 && cr != 4)
1391366f6083SPeter Grehan 		return (UNHANDLED);
1392366f6083SPeter Grehan 
13936f0c167fSDimitry Andric 	regval = 0; /* silence gcc */
1394366f6083SPeter Grehan 	vmxctx = &vmx->ctx[vcpu];
1395366f6083SPeter Grehan 
1396366f6083SPeter Grehan 	/*
13973de83862SNeel Natu 	 * We must use vmcs_write() directly here because vmcs_setreg() will
1398366f6083SPeter Grehan 	 * call vmclear(vmcs) as a side-effect which we certainly don't want.
1399366f6083SPeter Grehan 	 */
1400366f6083SPeter Grehan 	switch ((exitqual >> 8) & 0xf) {
1401366f6083SPeter Grehan 	case 0:
1402366f6083SPeter Grehan 		regval = vmxctx->guest_rax;
1403366f6083SPeter Grehan 		break;
1404366f6083SPeter Grehan 	case 1:
1405366f6083SPeter Grehan 		regval = vmxctx->guest_rcx;
1406366f6083SPeter Grehan 		break;
1407366f6083SPeter Grehan 	case 2:
1408366f6083SPeter Grehan 		regval = vmxctx->guest_rdx;
1409366f6083SPeter Grehan 		break;
1410366f6083SPeter Grehan 	case 3:
1411366f6083SPeter Grehan 		regval = vmxctx->guest_rbx;
1412366f6083SPeter Grehan 		break;
1413366f6083SPeter Grehan 	case 4:
14143de83862SNeel Natu 		regval = vmcs_read(VMCS_GUEST_RSP);
1415366f6083SPeter Grehan 		break;
1416366f6083SPeter Grehan 	case 5:
1417366f6083SPeter Grehan 		regval = vmxctx->guest_rbp;
1418366f6083SPeter Grehan 		break;
1419366f6083SPeter Grehan 	case 6:
1420366f6083SPeter Grehan 		regval = vmxctx->guest_rsi;
1421366f6083SPeter Grehan 		break;
1422366f6083SPeter Grehan 	case 7:
1423366f6083SPeter Grehan 		regval = vmxctx->guest_rdi;
1424366f6083SPeter Grehan 		break;
1425366f6083SPeter Grehan 	case 8:
1426366f6083SPeter Grehan 		regval = vmxctx->guest_r8;
1427366f6083SPeter Grehan 		break;
1428366f6083SPeter Grehan 	case 9:
1429366f6083SPeter Grehan 		regval = vmxctx->guest_r9;
1430366f6083SPeter Grehan 		break;
1431366f6083SPeter Grehan 	case 10:
1432366f6083SPeter Grehan 		regval = vmxctx->guest_r10;
1433366f6083SPeter Grehan 		break;
1434366f6083SPeter Grehan 	case 11:
1435366f6083SPeter Grehan 		regval = vmxctx->guest_r11;
1436366f6083SPeter Grehan 		break;
1437366f6083SPeter Grehan 	case 12:
1438366f6083SPeter Grehan 		regval = vmxctx->guest_r12;
1439366f6083SPeter Grehan 		break;
1440366f6083SPeter Grehan 	case 13:
1441366f6083SPeter Grehan 		regval = vmxctx->guest_r13;
1442366f6083SPeter Grehan 		break;
1443366f6083SPeter Grehan 	case 14:
1444366f6083SPeter Grehan 		regval = vmxctx->guest_r14;
1445366f6083SPeter Grehan 		break;
1446366f6083SPeter Grehan 	case 15:
1447366f6083SPeter Grehan 		regval = vmxctx->guest_r15;
1448366f6083SPeter Grehan 		break;
1449366f6083SPeter Grehan 	}
1450366f6083SPeter Grehan 
145139c21c2dSNeel Natu 	if (cr == 0) {
145239c21c2dSNeel Natu 		ones_mask = cr0_ones_mask;
145339c21c2dSNeel Natu 		zeros_mask = cr0_zeros_mask;
145439c21c2dSNeel Natu 		vmcs_guest_cr = VMCS_GUEST_CR0;
1455aaaa0656SPeter Grehan 		vmcs_shadow_cr = VMCS_CR0_SHADOW;
145639c21c2dSNeel Natu 	} else {
145739c21c2dSNeel Natu 		ones_mask = cr4_ones_mask;
145839c21c2dSNeel Natu 		zeros_mask = cr4_zeros_mask;
145939c21c2dSNeel Natu 		vmcs_guest_cr = VMCS_GUEST_CR4;
1460aaaa0656SPeter Grehan 		vmcs_shadow_cr = VMCS_CR4_SHADOW;
146139c21c2dSNeel Natu 	}
14623de83862SNeel Natu 	vmcs_write(vmcs_shadow_cr, regval);
1463aaaa0656SPeter Grehan 
146480a902efSPeter Grehan 	crval = regval | ones_mask;
146580a902efSPeter Grehan 	crval &= ~zeros_mask;
14663de83862SNeel Natu 	vmcs_write(vmcs_guest_cr, crval);
1467366f6083SPeter Grehan 
146880a902efSPeter Grehan 	if (cr == 0 && regval & CR0_PG) {
146980a902efSPeter Grehan 		uint64_t efer, entry_ctls;
147080a902efSPeter Grehan 
147180a902efSPeter Grehan 		/*
147280a902efSPeter Grehan 		 * If CR0.PG is 1 and EFER.LME is 1 then EFER.LMA and
147380a902efSPeter Grehan 		 * the "IA-32e mode guest" bit in VM-entry control must be
147480a902efSPeter Grehan 		 * equal.
147580a902efSPeter Grehan 		 */
14763de83862SNeel Natu 		efer = vmcs_read(VMCS_GUEST_IA32_EFER);
147780a902efSPeter Grehan 		if (efer & EFER_LME) {
147880a902efSPeter Grehan 			efer |= EFER_LMA;
14793de83862SNeel Natu 			vmcs_write(VMCS_GUEST_IA32_EFER, efer);
14803de83862SNeel Natu 			entry_ctls = vmcs_read(VMCS_ENTRY_CTLS);
148180a902efSPeter Grehan 			entry_ctls |= VM_ENTRY_GUEST_LMA;
14823de83862SNeel Natu 			vmcs_write(VMCS_ENTRY_CTLS, entry_ctls);
148380a902efSPeter Grehan 		}
148480a902efSPeter Grehan 	}
148580a902efSPeter Grehan 
1486366f6083SPeter Grehan 	return (HANDLED);
1487366f6083SPeter Grehan }
1488366f6083SPeter Grehan 
148900f3efe1SJohn Baldwin static enum vie_cpu_mode
149000f3efe1SJohn Baldwin vmx_cpu_mode(void)
149100f3efe1SJohn Baldwin {
149200f3efe1SJohn Baldwin 
149300f3efe1SJohn Baldwin 	if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LMA)
149400f3efe1SJohn Baldwin 		return (CPU_MODE_64BIT);
149500f3efe1SJohn Baldwin 	else
149600f3efe1SJohn Baldwin 		return (CPU_MODE_COMPATIBILITY);
149700f3efe1SJohn Baldwin }
149800f3efe1SJohn Baldwin 
149900f3efe1SJohn Baldwin static enum vie_paging_mode
150000f3efe1SJohn Baldwin vmx_paging_mode(void)
150100f3efe1SJohn Baldwin {
150200f3efe1SJohn Baldwin 
150300f3efe1SJohn Baldwin 	if (!(vmcs_read(VMCS_GUEST_CR0) & CR0_PG))
150400f3efe1SJohn Baldwin 		return (PAGING_MODE_FLAT);
150500f3efe1SJohn Baldwin 	if (!(vmcs_read(VMCS_GUEST_CR4) & CR4_PAE))
150600f3efe1SJohn Baldwin 		return (PAGING_MODE_32);
150700f3efe1SJohn Baldwin 	if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LME)
150800f3efe1SJohn Baldwin 		return (PAGING_MODE_64);
150900f3efe1SJohn Baldwin 	else
151000f3efe1SJohn Baldwin 		return (PAGING_MODE_PAE);
151100f3efe1SJohn Baldwin }
151200f3efe1SJohn Baldwin 
1513366f6083SPeter Grehan static int
1514318224bbSNeel Natu ept_fault_type(uint64_t ept_qual)
1515a2da7af6SNeel Natu {
1516318224bbSNeel Natu 	int fault_type;
1517a2da7af6SNeel Natu 
1518318224bbSNeel Natu 	if (ept_qual & EPT_VIOLATION_DATA_WRITE)
1519318224bbSNeel Natu 		fault_type = VM_PROT_WRITE;
1520318224bbSNeel Natu 	else if (ept_qual & EPT_VIOLATION_INST_FETCH)
1521318224bbSNeel Natu 		fault_type = VM_PROT_EXECUTE;
1522318224bbSNeel Natu 	else
1523318224bbSNeel Natu 		fault_type= VM_PROT_READ;
1524318224bbSNeel Natu 
1525318224bbSNeel Natu 	return (fault_type);
1526318224bbSNeel Natu }
1527318224bbSNeel Natu 
1528318224bbSNeel Natu static boolean_t
1529318224bbSNeel Natu ept_emulation_fault(uint64_t ept_qual)
1530318224bbSNeel Natu {
1531318224bbSNeel Natu 	int read, write;
1532318224bbSNeel Natu 
1533318224bbSNeel Natu 	/* EPT fault on an instruction fetch doesn't make sense here */
1534a2da7af6SNeel Natu 	if (ept_qual & EPT_VIOLATION_INST_FETCH)
1535318224bbSNeel Natu 		return (FALSE);
1536a2da7af6SNeel Natu 
1537318224bbSNeel Natu 	/* EPT fault must be a read fault or a write fault */
1538a2da7af6SNeel Natu 	read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0;
1539a2da7af6SNeel Natu 	write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0;
15403b2b0011SPeter Grehan 	if ((read | write) == 0)
1541318224bbSNeel Natu 		return (FALSE);
1542a2da7af6SNeel Natu 
1543a2da7af6SNeel Natu 	/*
15443b2b0011SPeter Grehan 	 * The EPT violation must have been caused by accessing a
15453b2b0011SPeter Grehan 	 * guest-physical address that is a translation of a guest-linear
15463b2b0011SPeter Grehan 	 * address.
1547a2da7af6SNeel Natu 	 */
1548a2da7af6SNeel Natu 	if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 ||
1549a2da7af6SNeel Natu 	    (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) {
1550318224bbSNeel Natu 		return (FALSE);
1551a2da7af6SNeel Natu 	}
1552a2da7af6SNeel Natu 
1553318224bbSNeel Natu 	return (TRUE);
1554a2da7af6SNeel Natu }
1555a2da7af6SNeel Natu 
1556159dd56fSNeel Natu static __inline int
1557159dd56fSNeel Natu apic_access_virtualization(struct vmx *vmx, int vcpuid)
1558159dd56fSNeel Natu {
1559159dd56fSNeel Natu 	uint32_t proc_ctls2;
1560159dd56fSNeel Natu 
1561159dd56fSNeel Natu 	proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
1562159dd56fSNeel Natu 	return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) ? 1 : 0);
1563159dd56fSNeel Natu }
1564159dd56fSNeel Natu 
1565159dd56fSNeel Natu static __inline int
1566159dd56fSNeel Natu x2apic_virtualization(struct vmx *vmx, int vcpuid)
1567159dd56fSNeel Natu {
1568159dd56fSNeel Natu 	uint32_t proc_ctls2;
1569159dd56fSNeel Natu 
1570159dd56fSNeel Natu 	proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
1571159dd56fSNeel Natu 	return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_X2APIC_MODE) ? 1 : 0);
1572159dd56fSNeel Natu }
1573159dd56fSNeel Natu 
1574a2da7af6SNeel Natu static int
1575159dd56fSNeel Natu vmx_handle_apic_write(struct vmx *vmx, int vcpuid, struct vlapic *vlapic,
1576159dd56fSNeel Natu     uint64_t qual)
157788c4b8d1SNeel Natu {
157888c4b8d1SNeel Natu 	int error, handled, offset;
1579159dd56fSNeel Natu 	uint32_t *apic_regs, vector;
158088c4b8d1SNeel Natu 	bool retu;
158188c4b8d1SNeel Natu 
1582a0efd3fbSJohn Baldwin 	handled = HANDLED;
158388c4b8d1SNeel Natu 	offset = APIC_WRITE_OFFSET(qual);
1584159dd56fSNeel Natu 
1585159dd56fSNeel Natu 	if (!apic_access_virtualization(vmx, vcpuid)) {
1586159dd56fSNeel Natu 		/*
1587159dd56fSNeel Natu 		 * In general there should not be any APIC write VM-exits
1588159dd56fSNeel Natu 		 * unless APIC-access virtualization is enabled.
1589159dd56fSNeel Natu 		 *
1590159dd56fSNeel Natu 		 * However self-IPI virtualization can legitimately trigger
1591159dd56fSNeel Natu 		 * an APIC-write VM-exit so treat it specially.
1592159dd56fSNeel Natu 		 */
1593159dd56fSNeel Natu 		if (x2apic_virtualization(vmx, vcpuid) &&
1594159dd56fSNeel Natu 		    offset == APIC_OFFSET_SELF_IPI) {
1595159dd56fSNeel Natu 			apic_regs = (uint32_t *)(vlapic->apic_page);
1596159dd56fSNeel Natu 			vector = apic_regs[APIC_OFFSET_SELF_IPI / 4];
1597159dd56fSNeel Natu 			vlapic_self_ipi_handler(vlapic, vector);
1598159dd56fSNeel Natu 			return (HANDLED);
1599159dd56fSNeel Natu 		} else
1600159dd56fSNeel Natu 			return (UNHANDLED);
1601159dd56fSNeel Natu 	}
1602159dd56fSNeel Natu 
160388c4b8d1SNeel Natu 	switch (offset) {
160488c4b8d1SNeel Natu 	case APIC_OFFSET_ID:
160588c4b8d1SNeel Natu 		vlapic_id_write_handler(vlapic);
160688c4b8d1SNeel Natu 		break;
160788c4b8d1SNeel Natu 	case APIC_OFFSET_LDR:
160888c4b8d1SNeel Natu 		vlapic_ldr_write_handler(vlapic);
160988c4b8d1SNeel Natu 		break;
161088c4b8d1SNeel Natu 	case APIC_OFFSET_DFR:
161188c4b8d1SNeel Natu 		vlapic_dfr_write_handler(vlapic);
161288c4b8d1SNeel Natu 		break;
161388c4b8d1SNeel Natu 	case APIC_OFFSET_SVR:
161488c4b8d1SNeel Natu 		vlapic_svr_write_handler(vlapic);
161588c4b8d1SNeel Natu 		break;
161688c4b8d1SNeel Natu 	case APIC_OFFSET_ESR:
161788c4b8d1SNeel Natu 		vlapic_esr_write_handler(vlapic);
161888c4b8d1SNeel Natu 		break;
161988c4b8d1SNeel Natu 	case APIC_OFFSET_ICR_LOW:
162088c4b8d1SNeel Natu 		retu = false;
162188c4b8d1SNeel Natu 		error = vlapic_icrlo_write_handler(vlapic, &retu);
162288c4b8d1SNeel Natu 		if (error != 0 || retu)
1623a0efd3fbSJohn Baldwin 			handled = UNHANDLED;
162488c4b8d1SNeel Natu 		break;
162588c4b8d1SNeel Natu 	case APIC_OFFSET_CMCI_LVT:
162688c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
162788c4b8d1SNeel Natu 		vlapic_lvt_write_handler(vlapic, offset);
162888c4b8d1SNeel Natu 		break;
162988c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_ICR:
163088c4b8d1SNeel Natu 		vlapic_icrtmr_write_handler(vlapic);
163188c4b8d1SNeel Natu 		break;
163288c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_DCR:
163388c4b8d1SNeel Natu 		vlapic_dcr_write_handler(vlapic);
163488c4b8d1SNeel Natu 		break;
163588c4b8d1SNeel Natu 	default:
1636a0efd3fbSJohn Baldwin 		handled = UNHANDLED;
163788c4b8d1SNeel Natu 		break;
163888c4b8d1SNeel Natu 	}
163988c4b8d1SNeel Natu 	return (handled);
164088c4b8d1SNeel Natu }
164188c4b8d1SNeel Natu 
164288c4b8d1SNeel Natu static bool
1643159dd56fSNeel Natu apic_access_fault(struct vmx *vmx, int vcpuid, uint64_t gpa)
164488c4b8d1SNeel Natu {
164588c4b8d1SNeel Natu 
1646159dd56fSNeel Natu 	if (apic_access_virtualization(vmx, vcpuid) &&
164788c4b8d1SNeel Natu 	    (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE))
164888c4b8d1SNeel Natu 		return (true);
164988c4b8d1SNeel Natu 	else
165088c4b8d1SNeel Natu 		return (false);
165188c4b8d1SNeel Natu }
165288c4b8d1SNeel Natu 
165388c4b8d1SNeel Natu static int
165488c4b8d1SNeel Natu vmx_handle_apic_access(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
165588c4b8d1SNeel Natu {
165688c4b8d1SNeel Natu 	uint64_t qual;
165788c4b8d1SNeel Natu 	int access_type, offset, allowed;
165888c4b8d1SNeel Natu 
1659159dd56fSNeel Natu 	if (!apic_access_virtualization(vmx, vcpuid))
166088c4b8d1SNeel Natu 		return (UNHANDLED);
166188c4b8d1SNeel Natu 
166288c4b8d1SNeel Natu 	qual = vmexit->u.vmx.exit_qualification;
166388c4b8d1SNeel Natu 	access_type = APIC_ACCESS_TYPE(qual);
166488c4b8d1SNeel Natu 	offset = APIC_ACCESS_OFFSET(qual);
166588c4b8d1SNeel Natu 
166688c4b8d1SNeel Natu 	allowed = 0;
166788c4b8d1SNeel Natu 	if (access_type == 0) {
166888c4b8d1SNeel Natu 		/*
166988c4b8d1SNeel Natu 		 * Read data access to the following registers is expected.
167088c4b8d1SNeel Natu 		 */
167188c4b8d1SNeel Natu 		switch (offset) {
167288c4b8d1SNeel Natu 		case APIC_OFFSET_APR:
167388c4b8d1SNeel Natu 		case APIC_OFFSET_PPR:
167488c4b8d1SNeel Natu 		case APIC_OFFSET_RRR:
167588c4b8d1SNeel Natu 		case APIC_OFFSET_CMCI_LVT:
167688c4b8d1SNeel Natu 		case APIC_OFFSET_TIMER_CCR:
167788c4b8d1SNeel Natu 			allowed = 1;
167888c4b8d1SNeel Natu 			break;
167988c4b8d1SNeel Natu 		default:
168088c4b8d1SNeel Natu 			break;
168188c4b8d1SNeel Natu 		}
168288c4b8d1SNeel Natu 	} else if (access_type == 1) {
168388c4b8d1SNeel Natu 		/*
168488c4b8d1SNeel Natu 		 * Write data access to the following registers is expected.
168588c4b8d1SNeel Natu 		 */
168688c4b8d1SNeel Natu 		switch (offset) {
168788c4b8d1SNeel Natu 		case APIC_OFFSET_VER:
168888c4b8d1SNeel Natu 		case APIC_OFFSET_APR:
168988c4b8d1SNeel Natu 		case APIC_OFFSET_PPR:
169088c4b8d1SNeel Natu 		case APIC_OFFSET_RRR:
169188c4b8d1SNeel Natu 		case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
169288c4b8d1SNeel Natu 		case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
169388c4b8d1SNeel Natu 		case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
169488c4b8d1SNeel Natu 		case APIC_OFFSET_CMCI_LVT:
169588c4b8d1SNeel Natu 		case APIC_OFFSET_TIMER_CCR:
169688c4b8d1SNeel Natu 			allowed = 1;
169788c4b8d1SNeel Natu 			break;
169888c4b8d1SNeel Natu 		default:
169988c4b8d1SNeel Natu 			break;
170088c4b8d1SNeel Natu 		}
170188c4b8d1SNeel Natu 	}
170288c4b8d1SNeel Natu 
170388c4b8d1SNeel Natu 	if (allowed) {
170488c4b8d1SNeel Natu 		vmexit->exitcode = VM_EXITCODE_INST_EMUL;
170588c4b8d1SNeel Natu 		vmexit->u.inst_emul.gpa = DEFAULT_APIC_BASE + offset;
170688c4b8d1SNeel Natu 		vmexit->u.inst_emul.gla = VIE_INVALID_GLA;
170788c4b8d1SNeel Natu 		vmexit->u.inst_emul.cr3 = vmcs_guest_cr3();
170800f3efe1SJohn Baldwin 		vmexit->u.inst_emul.cpu_mode = vmx_cpu_mode();
170900f3efe1SJohn Baldwin 		vmexit->u.inst_emul.paging_mode = vmx_paging_mode();
171088c4b8d1SNeel Natu 	}
171188c4b8d1SNeel Natu 
171288c4b8d1SNeel Natu 	/*
171388c4b8d1SNeel Natu 	 * Regardless of whether the APIC-access is allowed this handler
171488c4b8d1SNeel Natu 	 * always returns UNHANDLED:
171588c4b8d1SNeel Natu 	 * - if the access is allowed then it is handled by emulating the
171688c4b8d1SNeel Natu 	 *   instruction that caused the VM-exit (outside the critical section)
171788c4b8d1SNeel Natu 	 * - if the access is not allowed then it will be converted to an
171888c4b8d1SNeel Natu 	 *   exitcode of VM_EXITCODE_VMX and will be dealt with in userland.
171988c4b8d1SNeel Natu 	 */
172088c4b8d1SNeel Natu 	return (UNHANDLED);
172188c4b8d1SNeel Natu }
172288c4b8d1SNeel Natu 
172388c4b8d1SNeel Natu static int
1724366f6083SPeter Grehan vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1725366f6083SPeter Grehan {
1726f76fc5d4SNeel Natu 	int error, handled;
1727366f6083SPeter Grehan 	struct vmxctx *vmxctx;
172888c4b8d1SNeel Natu 	struct vlapic *vlapic;
1729e5a1d950SNeel Natu 	uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, reason;
17303de83862SNeel Natu 	uint64_t qual, gpa;
1731becd9849SNeel Natu 	bool retu;
1732366f6083SPeter Grehan 
1733160471d2SNeel Natu 	CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0);
1734c308b23bSNeel Natu 	CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0);
1735160471d2SNeel Natu 
1736a0efd3fbSJohn Baldwin 	handled = UNHANDLED;
1737366f6083SPeter Grehan 	vmxctx = &vmx->ctx[vcpu];
17380492757cSNeel Natu 
1739366f6083SPeter Grehan 	qual = vmexit->u.vmx.exit_qualification;
1740318224bbSNeel Natu 	reason = vmexit->u.vmx.exit_reason;
1741366f6083SPeter Grehan 	vmexit->exitcode = VM_EXITCODE_BOGUS;
1742366f6083SPeter Grehan 
174361592433SNeel Natu 	vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1);
174461592433SNeel Natu 
1745318224bbSNeel Natu 	/*
1746318224bbSNeel Natu 	 * VM exits that could be triggered during event injection on the
1747318224bbSNeel Natu 	 * previous VM entry need to be handled specially by re-injecting
1748318224bbSNeel Natu 	 * the event.
1749318224bbSNeel Natu 	 *
1750318224bbSNeel Natu 	 * See "Information for VM Exits During Event Delivery" in Intel SDM
1751318224bbSNeel Natu 	 * for details.
1752318224bbSNeel Natu 	 */
1753318224bbSNeel Natu 	switch (reason) {
1754318224bbSNeel Natu 	case EXIT_REASON_EPT_FAULT:
1755318224bbSNeel Natu 	case EXIT_REASON_EPT_MISCONFIG:
175688c4b8d1SNeel Natu 	case EXIT_REASON_APIC_ACCESS:
1757318224bbSNeel Natu 	case EXIT_REASON_TASK_SWITCH:
1758318224bbSNeel Natu 	case EXIT_REASON_EXCEPTION:
1759318224bbSNeel Natu 		idtvec_info = vmcs_idt_vectoring_info();
1760318224bbSNeel Natu 		if (idtvec_info & VMCS_IDT_VEC_VALID) {
1761318224bbSNeel Natu 			idtvec_info &= ~(1 << 12); /* clear undefined bit */
17623de83862SNeel Natu 			vmcs_write(VMCS_ENTRY_INTR_INFO, idtvec_info);
1763318224bbSNeel Natu 			if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
1764318224bbSNeel Natu 				idtvec_err = vmcs_idt_vectoring_err();
17653de83862SNeel Natu 				vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR,
17663de83862SNeel Natu 				    idtvec_err);
1767318224bbSNeel Natu 			}
1768160471d2SNeel Natu 			/*
1769160471d2SNeel Natu 			 * If 'virtual NMIs' are being used and the VM-exit
1770160471d2SNeel Natu 			 * happened while injecting an NMI during the previous
1771160471d2SNeel Natu 			 * VM-entry, then clear "blocking by NMI" in the Guest
1772160471d2SNeel Natu 			 * Interruptibility-state.
1773160471d2SNeel Natu 			 */
1774160471d2SNeel Natu 			if ((idtvec_info & VMCS_INTR_T_MASK) ==
1775160471d2SNeel Natu 			    VMCS_INTR_T_NMI) {
1776e5a1d950SNeel Natu 				 vmx_clear_nmi_blocking(vmx, vcpu);
1777160471d2SNeel Natu 			}
17783de83862SNeel Natu 			vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
1779318224bbSNeel Natu 		}
1780318224bbSNeel Natu 	default:
1781e5a1d950SNeel Natu 		idtvec_info = 0;
1782318224bbSNeel Natu 		break;
1783318224bbSNeel Natu 	}
1784318224bbSNeel Natu 
1785318224bbSNeel Natu 	switch (reason) {
1786366f6083SPeter Grehan 	case EXIT_REASON_CR_ACCESS:
1787b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CR_ACCESS, 1);
1788366f6083SPeter Grehan 		handled = vmx_emulate_cr_access(vmx, vcpu, qual);
1789366f6083SPeter Grehan 		break;
1790366f6083SPeter Grehan 	case EXIT_REASON_RDMSR:
1791b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RDMSR, 1);
1792becd9849SNeel Natu 		retu = false;
1793366f6083SPeter Grehan 		ecx = vmxctx->guest_rcx;
1794becd9849SNeel Natu 		error = emulate_rdmsr(vmx->vm, vcpu, ecx, &retu);
1795b42206f3SNeel Natu 		if (error) {
1796366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_RDMSR;
1797366f6083SPeter Grehan 			vmexit->u.msr.code = ecx;
1798becd9849SNeel Natu 		} else if (!retu) {
1799a0efd3fbSJohn Baldwin 			handled = HANDLED;
1800becd9849SNeel Natu 		} else {
1801becd9849SNeel Natu 			/* Return to userspace with a valid exitcode */
1802becd9849SNeel Natu 			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
1803becd9849SNeel Natu 			    ("emulate_wrmsr retu with bogus exitcode"));
1804becd9849SNeel Natu 		}
1805366f6083SPeter Grehan 		break;
1806366f6083SPeter Grehan 	case EXIT_REASON_WRMSR:
1807b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_WRMSR, 1);
1808becd9849SNeel Natu 		retu = false;
1809366f6083SPeter Grehan 		eax = vmxctx->guest_rax;
1810366f6083SPeter Grehan 		ecx = vmxctx->guest_rcx;
1811366f6083SPeter Grehan 		edx = vmxctx->guest_rdx;
1812b42206f3SNeel Natu 		error = emulate_wrmsr(vmx->vm, vcpu, ecx,
1813becd9849SNeel Natu 		    (uint64_t)edx << 32 | eax, &retu);
1814b42206f3SNeel Natu 		if (error) {
1815366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_WRMSR;
1816366f6083SPeter Grehan 			vmexit->u.msr.code = ecx;
1817366f6083SPeter Grehan 			vmexit->u.msr.wval = (uint64_t)edx << 32 | eax;
1818becd9849SNeel Natu 		} else if (!retu) {
1819a0efd3fbSJohn Baldwin 			handled = HANDLED;
1820becd9849SNeel Natu 		} else {
1821becd9849SNeel Natu 			/* Return to userspace with a valid exitcode */
1822becd9849SNeel Natu 			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
1823becd9849SNeel Natu 			    ("emulate_wrmsr retu with bogus exitcode"));
1824becd9849SNeel Natu 		}
1825366f6083SPeter Grehan 		break;
1826366f6083SPeter Grehan 	case EXIT_REASON_HLT:
1827f76fc5d4SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_HLT, 1);
1828366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_HLT;
18293de83862SNeel Natu 		vmexit->u.hlt.rflags = vmcs_read(VMCS_GUEST_RFLAGS);
1830366f6083SPeter Grehan 		break;
1831366f6083SPeter Grehan 	case EXIT_REASON_MTF:
1832b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_MTRAP, 1);
1833366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_MTRAP;
1834366f6083SPeter Grehan 		break;
1835366f6083SPeter Grehan 	case EXIT_REASON_PAUSE:
1836b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_PAUSE, 1);
1837366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_PAUSE;
1838366f6083SPeter Grehan 		break;
1839366f6083SPeter Grehan 	case EXIT_REASON_INTR_WINDOW:
1840b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INTR_WINDOW, 1);
1841366f6083SPeter Grehan 		vmx_clear_int_window_exiting(vmx, vcpu);
1842b5aaf7b2SNeel Natu 		return (1);
1843366f6083SPeter Grehan 	case EXIT_REASON_EXT_INTR:
1844366f6083SPeter Grehan 		/*
1845366f6083SPeter Grehan 		 * External interrupts serve only to cause VM exits and allow
1846366f6083SPeter Grehan 		 * the host interrupt handler to run.
1847366f6083SPeter Grehan 		 *
1848366f6083SPeter Grehan 		 * If this external interrupt triggers a virtual interrupt
1849366f6083SPeter Grehan 		 * to a VM, then that state will be recorded by the
1850366f6083SPeter Grehan 		 * host interrupt handler in the VM's softc. We will inject
1851366f6083SPeter Grehan 		 * this virtual interrupt during the subsequent VM enter.
1852366f6083SPeter Grehan 		 */
1853f7d47425SNeel Natu 		intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
1854722b6744SJohn Baldwin 
1855722b6744SJohn Baldwin 		/*
1856722b6744SJohn Baldwin 		 * XXX: Ignore this exit if VMCS_INTR_VALID is not set.
1857ad3e3687SJohn Baldwin 		 * This appears to be a bug in VMware Fusion?
1858722b6744SJohn Baldwin 		 */
1859722b6744SJohn Baldwin 		if (!(intr_info & VMCS_INTR_VALID))
1860722b6744SJohn Baldwin 			return (1);
1861160471d2SNeel Natu 		KASSERT((intr_info & VMCS_INTR_VALID) != 0 &&
1862160471d2SNeel Natu 		    (intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_HWINTR,
1863f7d47425SNeel Natu 		    ("VM exit interruption info invalid: %#x", intr_info));
1864f7d47425SNeel Natu 		vmx_trigger_hostintr(intr_info & 0xff);
1865366f6083SPeter Grehan 
1866366f6083SPeter Grehan 		/*
1867366f6083SPeter Grehan 		 * This is special. We want to treat this as an 'handled'
1868366f6083SPeter Grehan 		 * VM-exit but not increment the instruction pointer.
1869366f6083SPeter Grehan 		 */
1870366f6083SPeter Grehan 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1);
1871366f6083SPeter Grehan 		return (1);
1872366f6083SPeter Grehan 	case EXIT_REASON_NMI_WINDOW:
1873366f6083SPeter Grehan 		/* Exit to allow the pending virtual NMI to be injected */
187448b2d828SNeel Natu 		if (vm_nmi_pending(vmx->vm, vcpu))
187548b2d828SNeel Natu 			vmx_inject_nmi(vmx, vcpu);
1876366f6083SPeter Grehan 		vmx_clear_nmi_window_exiting(vmx, vcpu);
187748b2d828SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NMI_WINDOW, 1);
1878366f6083SPeter Grehan 		return (1);
1879366f6083SPeter Grehan 	case EXIT_REASON_INOUT:
1880b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INOUT, 1);
1881366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_INOUT;
1882366f6083SPeter Grehan 		vmexit->u.inout.bytes = (qual & 0x7) + 1;
1883366f6083SPeter Grehan 		vmexit->u.inout.in = (qual & 0x8) ? 1 : 0;
1884366f6083SPeter Grehan 		vmexit->u.inout.string = (qual & 0x10) ? 1 : 0;
1885366f6083SPeter Grehan 		vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0;
1886366f6083SPeter Grehan 		vmexit->u.inout.port = (uint16_t)(qual >> 16);
1887366f6083SPeter Grehan 		vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax);
1888762fd208STycho Nightingale 		error = emulate_ioport(vmx->vm, vcpu, vmexit);
1889762fd208STycho Nightingale 		if (error == 0)  {
1890762fd208STycho Nightingale 			handled = 1;
1891762fd208STycho Nightingale 			vmxctx->guest_rax = vmexit->u.inout.eax;
1892762fd208STycho Nightingale 		}
1893366f6083SPeter Grehan 		break;
1894366f6083SPeter Grehan 	case EXIT_REASON_CPUID:
1895b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CPUID, 1);
1896a2da7af6SNeel Natu 		handled = vmx_handle_cpuid(vmx->vm, vcpu, vmxctx);
1897366f6083SPeter Grehan 		break;
1898e5a1d950SNeel Natu 	case EXIT_REASON_EXCEPTION:
1899c308b23bSNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXCEPTION, 1);
1900e5a1d950SNeel Natu 		intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
1901e5a1d950SNeel Natu 		KASSERT((intr_info & VMCS_INTR_VALID) != 0,
1902e5a1d950SNeel Natu 		    ("VM exit interruption info invalid: %#x", intr_info));
1903c308b23bSNeel Natu 
1904e5a1d950SNeel Natu 		/*
1905e5a1d950SNeel Natu 		 * If Virtual NMIs control is 1 and the VM-exit is due to a
1906e5a1d950SNeel Natu 		 * fault encountered during the execution of IRET then we must
1907e5a1d950SNeel Natu 		 * restore the state of "virtual-NMI blocking" before resuming
1908e5a1d950SNeel Natu 		 * the guest.
1909e5a1d950SNeel Natu 		 *
1910e5a1d950SNeel Natu 		 * See "Resuming Guest Software after Handling an Exception".
1911e5a1d950SNeel Natu 		 */
1912e5a1d950SNeel Natu 		if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
1913e5a1d950SNeel Natu 		    (intr_info & 0xff) != IDT_DF &&
1914e5a1d950SNeel Natu 		    (intr_info & EXIT_QUAL_NMIUDTI) != 0)
1915e5a1d950SNeel Natu 			vmx_restore_nmi_blocking(vmx, vcpu);
1916c308b23bSNeel Natu 
1917c308b23bSNeel Natu 		/*
191862fbd7c2SNeel Natu 		 * The NMI has already been handled in vmx_exit_handle_nmi().
1919c308b23bSNeel Natu 		 */
192062fbd7c2SNeel Natu 		if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI)
1921c308b23bSNeel Natu 			return (1);
1922e5a1d950SNeel Natu 		break;
1923cd942e0fSPeter Grehan 	case EXIT_REASON_EPT_FAULT:
1924318224bbSNeel Natu 		/*
1925318224bbSNeel Natu 		 * If 'gpa' lies within the address space allocated to
1926318224bbSNeel Natu 		 * memory then this must be a nested page fault otherwise
1927318224bbSNeel Natu 		 * this must be an instruction that accesses MMIO space.
1928318224bbSNeel Natu 		 */
1929a2da7af6SNeel Natu 		gpa = vmcs_gpa();
1930159dd56fSNeel Natu 		if (vm_mem_allocated(vmx->vm, gpa) ||
1931159dd56fSNeel Natu 		    apic_access_fault(vmx, vcpu, gpa)) {
1932cd942e0fSPeter Grehan 			vmexit->exitcode = VM_EXITCODE_PAGING;
193313ec9371SPeter Grehan 			vmexit->u.paging.gpa = gpa;
1934318224bbSNeel Natu 			vmexit->u.paging.fault_type = ept_fault_type(qual);
1935bf73979dSNeel Natu 			vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NESTED_FAULT, 1);
1936318224bbSNeel Natu 		} else if (ept_emulation_fault(qual)) {
1937318224bbSNeel Natu 			vmexit->exitcode = VM_EXITCODE_INST_EMUL;
1938318224bbSNeel Natu 			vmexit->u.inst_emul.gpa = gpa;
1939318224bbSNeel Natu 			vmexit->u.inst_emul.gla = vmcs_gla();
1940318224bbSNeel Natu 			vmexit->u.inst_emul.cr3 = vmcs_guest_cr3();
194100f3efe1SJohn Baldwin 			vmexit->u.inst_emul.cpu_mode = vmx_cpu_mode();
194200f3efe1SJohn Baldwin 			vmexit->u.inst_emul.paging_mode = vmx_paging_mode();
1943bf73979dSNeel Natu 			vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INST_EMUL, 1);
1944a2da7af6SNeel Natu 		}
1945e5a1d950SNeel Natu 		/*
1946e5a1d950SNeel Natu 		 * If Virtual NMIs control is 1 and the VM-exit is due to an
1947e5a1d950SNeel Natu 		 * EPT fault during the execution of IRET then we must restore
1948e5a1d950SNeel Natu 		 * the state of "virtual-NMI blocking" before resuming.
1949e5a1d950SNeel Natu 		 *
1950e5a1d950SNeel Natu 		 * See description of "NMI unblocking due to IRET" in
1951e5a1d950SNeel Natu 		 * "Exit Qualification for EPT Violations".
1952e5a1d950SNeel Natu 		 */
1953e5a1d950SNeel Natu 		if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
1954e5a1d950SNeel Natu 		    (qual & EXIT_QUAL_NMIUDTI) != 0)
1955e5a1d950SNeel Natu 			vmx_restore_nmi_blocking(vmx, vcpu);
1956cd942e0fSPeter Grehan 		break;
195730b94db8SNeel Natu 	case EXIT_REASON_VIRTUALIZED_EOI:
195830b94db8SNeel Natu 		vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI;
195930b94db8SNeel Natu 		vmexit->u.ioapic_eoi.vector = qual & 0xFF;
196030b94db8SNeel Natu 		vmexit->inst_length = 0;	/* trap-like */
196130b94db8SNeel Natu 		break;
196288c4b8d1SNeel Natu 	case EXIT_REASON_APIC_ACCESS:
196388c4b8d1SNeel Natu 		handled = vmx_handle_apic_access(vmx, vcpu, vmexit);
196488c4b8d1SNeel Natu 		break;
196588c4b8d1SNeel Natu 	case EXIT_REASON_APIC_WRITE:
196688c4b8d1SNeel Natu 		/*
196788c4b8d1SNeel Natu 		 * APIC-write VM exit is trap-like so the %rip is already
196888c4b8d1SNeel Natu 		 * pointing to the next instruction.
196988c4b8d1SNeel Natu 		 */
197088c4b8d1SNeel Natu 		vmexit->inst_length = 0;
197188c4b8d1SNeel Natu 		vlapic = vm_lapic(vmx->vm, vcpu);
1972159dd56fSNeel Natu 		handled = vmx_handle_apic_write(vmx, vcpu, vlapic, qual);
197388c4b8d1SNeel Natu 		break;
1974abb023fbSJohn Baldwin 	case EXIT_REASON_XSETBV:
1975a0efd3fbSJohn Baldwin 		handled = vmx_emulate_xsetbv(vmx, vcpu, vmexit);
1976abb023fbSJohn Baldwin 		break;
1977366f6083SPeter Grehan 	default:
1978b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_UNKNOWN, 1);
1979366f6083SPeter Grehan 		break;
1980366f6083SPeter Grehan 	}
1981366f6083SPeter Grehan 
1982366f6083SPeter Grehan 	if (handled) {
1983366f6083SPeter Grehan 		/*
1984366f6083SPeter Grehan 		 * It is possible that control is returned to userland
1985366f6083SPeter Grehan 		 * even though we were able to handle the VM exit in the
1986eeefa4e4SNeel Natu 		 * kernel.
1987366f6083SPeter Grehan 		 *
1988366f6083SPeter Grehan 		 * In such a case we want to make sure that the userland
1989366f6083SPeter Grehan 		 * restarts guest execution at the instruction *after*
1990366f6083SPeter Grehan 		 * the one we just processed. Therefore we update the
1991366f6083SPeter Grehan 		 * guest rip in the VMCS and in 'vmexit'.
1992366f6083SPeter Grehan 		 */
1993366f6083SPeter Grehan 		vmexit->rip += vmexit->inst_length;
1994366f6083SPeter Grehan 		vmexit->inst_length = 0;
19953de83862SNeel Natu 		vmcs_write(VMCS_GUEST_RIP, vmexit->rip);
1996366f6083SPeter Grehan 	} else {
1997366f6083SPeter Grehan 		if (vmexit->exitcode == VM_EXITCODE_BOGUS) {
1998366f6083SPeter Grehan 			/*
1999366f6083SPeter Grehan 			 * If this VM exit was not claimed by anybody then
2000366f6083SPeter Grehan 			 * treat it as a generic VMX exit.
2001366f6083SPeter Grehan 			 */
2002366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_VMX;
20030492757cSNeel Natu 			vmexit->u.vmx.status = VM_SUCCESS;
2004c308b23bSNeel Natu 			vmexit->u.vmx.inst_type = 0;
2005c308b23bSNeel Natu 			vmexit->u.vmx.inst_error = 0;
2006366f6083SPeter Grehan 		} else {
2007366f6083SPeter Grehan 			/*
2008366f6083SPeter Grehan 			 * The exitcode and collateral have been populated.
2009366f6083SPeter Grehan 			 * The VM exit will be processed further in userland.
2010366f6083SPeter Grehan 			 */
2011366f6083SPeter Grehan 		}
2012366f6083SPeter Grehan 	}
2013366f6083SPeter Grehan 	return (handled);
2014366f6083SPeter Grehan }
2015366f6083SPeter Grehan 
20160492757cSNeel Natu static __inline int
20170492757cSNeel Natu vmx_exit_astpending(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
2018366f6083SPeter Grehan {
20190492757cSNeel Natu 
20200492757cSNeel Natu 	vmexit->rip = vmcs_guest_rip();
20210492757cSNeel Natu 	vmexit->inst_length = 0;
20220492757cSNeel Natu 	vmexit->exitcode = VM_EXITCODE_BOGUS;
20230492757cSNeel Natu 	vmx_astpending_trace(vmx, vcpu, vmexit->rip);
20240492757cSNeel Natu 	vmm_stat_incr(vmx->vm, vcpu, VMEXIT_ASTPENDING, 1);
20250492757cSNeel Natu 
20260492757cSNeel Natu 	return (HANDLED);
20270492757cSNeel Natu }
20280492757cSNeel Natu 
20290492757cSNeel Natu static __inline int
20305b8a8cd1SNeel Natu vmx_exit_rendezvous(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
20315b8a8cd1SNeel Natu {
20325b8a8cd1SNeel Natu 
20335b8a8cd1SNeel Natu 	vmexit->rip = vmcs_guest_rip();
20345b8a8cd1SNeel Natu 	vmexit->inst_length = 0;
20355b8a8cd1SNeel Natu 	vmexit->exitcode = VM_EXITCODE_RENDEZVOUS;
20365b8a8cd1SNeel Natu 	vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RENDEZVOUS, 1);
20375b8a8cd1SNeel Natu 
20385b8a8cd1SNeel Natu 	return (UNHANDLED);
20395b8a8cd1SNeel Natu }
20405b8a8cd1SNeel Natu 
20415b8a8cd1SNeel Natu static __inline int
2042b15a09c0SNeel Natu vmx_exit_suspended(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
2043b15a09c0SNeel Natu {
2044b15a09c0SNeel Natu 
2045b15a09c0SNeel Natu 	vmexit->rip = vmcs_guest_rip();
2046b15a09c0SNeel Natu 	vmexit->inst_length = 0;
2047b15a09c0SNeel Natu 	vmexit->exitcode = VM_EXITCODE_SUSPENDED;
2048b15a09c0SNeel Natu 	return (UNHANDLED);
2049b15a09c0SNeel Natu }
2050b15a09c0SNeel Natu 
2051b15a09c0SNeel Natu static __inline int
20520492757cSNeel Natu vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit)
20530492757cSNeel Natu {
20540492757cSNeel Natu 
20550492757cSNeel Natu 	KASSERT(vmxctx->inst_fail_status != VM_SUCCESS,
20560492757cSNeel Natu 	    ("vmx_exit_inst_error: invalid inst_fail_status %d",
20570492757cSNeel Natu 	    vmxctx->inst_fail_status));
20580492757cSNeel Natu 
20590492757cSNeel Natu 	vmexit->inst_length = 0;
20600492757cSNeel Natu 	vmexit->exitcode = VM_EXITCODE_VMX;
20610492757cSNeel Natu 	vmexit->u.vmx.status = vmxctx->inst_fail_status;
20620492757cSNeel Natu 	vmexit->u.vmx.inst_error = vmcs_instruction_error();
20630492757cSNeel Natu 	vmexit->u.vmx.exit_reason = ~0;
20640492757cSNeel Natu 	vmexit->u.vmx.exit_qualification = ~0;
20650492757cSNeel Natu 
20660492757cSNeel Natu 	switch (rc) {
20670492757cSNeel Natu 	case VMX_VMRESUME_ERROR:
20680492757cSNeel Natu 	case VMX_VMLAUNCH_ERROR:
20690492757cSNeel Natu 	case VMX_INVEPT_ERROR:
20700492757cSNeel Natu 		vmexit->u.vmx.inst_type = rc;
20710492757cSNeel Natu 		break;
20720492757cSNeel Natu 	default:
20730492757cSNeel Natu 		panic("vm_exit_inst_error: vmx_enter_guest returned %d", rc);
20740492757cSNeel Natu 	}
20750492757cSNeel Natu 
20760492757cSNeel Natu 	return (UNHANDLED);
20770492757cSNeel Natu }
20780492757cSNeel Natu 
207962fbd7c2SNeel Natu /*
208062fbd7c2SNeel Natu  * If the NMI-exiting VM execution control is set to '1' then an NMI in
208162fbd7c2SNeel Natu  * non-root operation causes a VM-exit. NMI blocking is in effect so it is
208262fbd7c2SNeel Natu  * sufficient to simply vector to the NMI handler via a software interrupt.
208362fbd7c2SNeel Natu  * However, this must be done before maskable interrupts are enabled
208462fbd7c2SNeel Natu  * otherwise the "iret" issued by an interrupt handler will incorrectly
208562fbd7c2SNeel Natu  * clear NMI blocking.
208662fbd7c2SNeel Natu  */
208762fbd7c2SNeel Natu static __inline void
208862fbd7c2SNeel Natu vmx_exit_handle_nmi(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
208962fbd7c2SNeel Natu {
209062fbd7c2SNeel Natu 	uint32_t intr_info;
209162fbd7c2SNeel Natu 
209262fbd7c2SNeel Natu 	KASSERT((read_rflags() & PSL_I) == 0, ("interrupts enabled"));
209362fbd7c2SNeel Natu 
209462fbd7c2SNeel Natu 	if (vmexit->u.vmx.exit_reason != EXIT_REASON_EXCEPTION)
209562fbd7c2SNeel Natu 		return;
209662fbd7c2SNeel Natu 
209762fbd7c2SNeel Natu 	intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
209862fbd7c2SNeel Natu 	KASSERT((intr_info & VMCS_INTR_VALID) != 0,
209962fbd7c2SNeel Natu 	    ("VM exit interruption info invalid: %#x", intr_info));
210062fbd7c2SNeel Natu 
210162fbd7c2SNeel Natu 	if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI) {
210262fbd7c2SNeel Natu 		KASSERT((intr_info & 0xff) == IDT_NMI, ("VM exit due "
210362fbd7c2SNeel Natu 		    "to NMI has invalid vector: %#x", intr_info));
210462fbd7c2SNeel Natu 		VCPU_CTR0(vmx->vm, vcpuid, "Vectoring to NMI handler");
210562fbd7c2SNeel Natu 		__asm __volatile("int $2");
210662fbd7c2SNeel Natu 	}
210762fbd7c2SNeel Natu }
210862fbd7c2SNeel Natu 
21090492757cSNeel Natu static int
21105b8a8cd1SNeel Natu vmx_run(void *arg, int vcpu, register_t startrip, pmap_t pmap,
2111b15a09c0SNeel Natu     void *rendezvous_cookie, void *suspend_cookie)
21120492757cSNeel Natu {
21130492757cSNeel Natu 	int rc, handled, launched;
2114366f6083SPeter Grehan 	struct vmx *vmx;
21155b8a8cd1SNeel Natu 	struct vm *vm;
2116366f6083SPeter Grehan 	struct vmxctx *vmxctx;
2117366f6083SPeter Grehan 	struct vmcs *vmcs;
211898ed632cSNeel Natu 	struct vm_exit *vmexit;
2119de5ea6b6SNeel Natu 	struct vlapic *vlapic;
212079c59630SNeel Natu 	uint64_t rip;
212179c59630SNeel Natu 	uint32_t exit_reason;
2122366f6083SPeter Grehan 
2123366f6083SPeter Grehan 	vmx = arg;
21245b8a8cd1SNeel Natu 	vm = vmx->vm;
2125366f6083SPeter Grehan 	vmcs = &vmx->vmcs[vcpu];
2126366f6083SPeter Grehan 	vmxctx = &vmx->ctx[vcpu];
21275b8a8cd1SNeel Natu 	vlapic = vm_lapic(vm, vcpu);
21285b8a8cd1SNeel Natu 	vmexit = vm_exitinfo(vm, vcpu);
21290492757cSNeel Natu 	launched = 0;
213098ed632cSNeel Natu 
2131318224bbSNeel Natu 	KASSERT(vmxctx->pmap == pmap,
2132318224bbSNeel Natu 	    ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap));
2133318224bbSNeel Natu 
2134366f6083SPeter Grehan 	VMPTRLD(vmcs);
2135366f6083SPeter Grehan 
2136366f6083SPeter Grehan 	/*
2137366f6083SPeter Grehan 	 * XXX
2138366f6083SPeter Grehan 	 * We do this every time because we may setup the virtual machine
2139366f6083SPeter Grehan 	 * from a different process than the one that actually runs it.
2140366f6083SPeter Grehan 	 *
2141366f6083SPeter Grehan 	 * If the life of a virtual machine was spent entirely in the context
2142c847a506SNeel Natu 	 * of a single process we could do this once in vmx_vminit().
2143366f6083SPeter Grehan 	 */
21443de83862SNeel Natu 	vmcs_write(VMCS_HOST_CR3, rcr3());
2145366f6083SPeter Grehan 
21460492757cSNeel Natu 	vmcs_write(VMCS_GUEST_RIP, startrip);
2147953c2c47SNeel Natu 	vmx_set_pcpu_defaults(vmx, vcpu, pmap);
2148366f6083SPeter Grehan 	do {
21490492757cSNeel Natu 		/*
21500492757cSNeel Natu 		 * Interrupts are disabled from this point on until the
21510492757cSNeel Natu 		 * guest starts executing. This is done for the following
21520492757cSNeel Natu 		 * reasons:
21530492757cSNeel Natu 		 *
21540492757cSNeel Natu 		 * If an AST is asserted on this thread after the check below,
21550492757cSNeel Natu 		 * then the IPI_AST notification will not be lost, because it
21560492757cSNeel Natu 		 * will cause a VM exit due to external interrupt as soon as
21570492757cSNeel Natu 		 * the guest state is loaded.
21580492757cSNeel Natu 		 *
21590492757cSNeel Natu 		 * A posted interrupt after 'vmx_inject_interrupts()' will
21600492757cSNeel Natu 		 * not be "lost" because it will be held pending in the host
21610492757cSNeel Natu 		 * APIC because interrupts are disabled. The pending interrupt
21620492757cSNeel Natu 		 * will be recognized as soon as the guest state is loaded.
21630492757cSNeel Natu 		 *
21640492757cSNeel Natu 		 * The same reasoning applies to the IPI generated by
21650492757cSNeel Natu 		 * pmap_invalidate_ept().
21660492757cSNeel Natu 		 */
21670492757cSNeel Natu 		disable_intr();
2168b15a09c0SNeel Natu 		if (vcpu_suspended(suspend_cookie)) {
21690492757cSNeel Natu 			enable_intr();
2170b15a09c0SNeel Natu 			handled = vmx_exit_suspended(vmx, vcpu, vmexit);
21710492757cSNeel Natu 			break;
21720492757cSNeel Natu 		}
21730492757cSNeel Natu 
21745b8a8cd1SNeel Natu 		if (vcpu_rendezvous_pending(rendezvous_cookie)) {
21755b8a8cd1SNeel Natu 			enable_intr();
21765b8a8cd1SNeel Natu 			handled = vmx_exit_rendezvous(vmx, vcpu, vmexit);
21775b8a8cd1SNeel Natu 			break;
21785b8a8cd1SNeel Natu 		}
21795b8a8cd1SNeel Natu 
2180b15a09c0SNeel Natu 		if (curthread->td_flags & (TDF_ASTPENDING | TDF_NEEDRESCHED)) {
2181b15a09c0SNeel Natu 			enable_intr();
2182b15a09c0SNeel Natu 			handled = vmx_exit_astpending(vmx, vcpu, vmexit);
2183b15a09c0SNeel Natu 			break;
2184b15a09c0SNeel Natu 		}
2185b15a09c0SNeel Natu 
2186de5ea6b6SNeel Natu 		vmx_inject_interrupts(vmx, vcpu, vlapic);
2187366f6083SPeter Grehan 		vmx_run_trace(vmx, vcpu);
2188953c2c47SNeel Natu 		rc = vmx_enter_guest(vmxctx, vmx, launched);
218979c59630SNeel Natu 
219079c59630SNeel Natu 		/* Collect some information for VM exit processing */
219179c59630SNeel Natu 		vmexit->rip = rip = vmcs_guest_rip();
219279c59630SNeel Natu 		vmexit->inst_length = vmexit_instruction_length();
219379c59630SNeel Natu 		vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason();
219479c59630SNeel Natu 		vmexit->u.vmx.exit_qualification = vmcs_exit_qualification();
219579c59630SNeel Natu 
21960492757cSNeel Natu 		if (rc == VMX_GUEST_VMEXIT) {
219762fbd7c2SNeel Natu 			vmx_exit_handle_nmi(vmx, vcpu, vmexit);
219862fbd7c2SNeel Natu 			enable_intr();
21990492757cSNeel Natu 			handled = vmx_exit_process(vmx, vcpu, vmexit);
22000492757cSNeel Natu 		} else {
220162fbd7c2SNeel Natu 			enable_intr();
22020492757cSNeel Natu 			handled = vmx_exit_inst_error(vmxctx, rc, vmexit);
2203eeefa4e4SNeel Natu 		}
220462fbd7c2SNeel Natu 		launched = 1;
220579c59630SNeel Natu 		vmx_exit_trace(vmx, vcpu, rip, exit_reason, handled);
2206eeefa4e4SNeel Natu 	} while (handled);
2207366f6083SPeter Grehan 
2208366f6083SPeter Grehan 	/*
2209366f6083SPeter Grehan 	 * If a VM exit has been handled then the exitcode must be BOGUS
2210366f6083SPeter Grehan 	 * If a VM exit is not handled then the exitcode must not be BOGUS
2211366f6083SPeter Grehan 	 */
2212366f6083SPeter Grehan 	if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) ||
2213366f6083SPeter Grehan 	    (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) {
2214366f6083SPeter Grehan 		panic("Mismatch between handled (%d) and exitcode (%d)",
2215366f6083SPeter Grehan 		      handled, vmexit->exitcode);
2216366f6083SPeter Grehan 	}
2217366f6083SPeter Grehan 
2218b5aaf7b2SNeel Natu 	if (!handled)
22195b8a8cd1SNeel Natu 		vmm_stat_incr(vm, vcpu, VMEXIT_USERSPACE, 1);
2220b5aaf7b2SNeel Natu 
22215b8a8cd1SNeel Natu 	VCPU_CTR1(vm, vcpu, "returning from vmx_run: exitcode %d",
22220492757cSNeel Natu 	    vmexit->exitcode);
2223366f6083SPeter Grehan 
2224366f6083SPeter Grehan 	VMCLEAR(vmcs);
2225366f6083SPeter Grehan 	return (0);
2226366f6083SPeter Grehan }
2227366f6083SPeter Grehan 
2228366f6083SPeter Grehan static void
2229366f6083SPeter Grehan vmx_vmcleanup(void *arg)
2230366f6083SPeter Grehan {
223145e51299SNeel Natu 	int i, error;
2232366f6083SPeter Grehan 	struct vmx *vmx = arg;
2233366f6083SPeter Grehan 
2234159dd56fSNeel Natu 	if (apic_access_virtualization(vmx, 0))
223588c4b8d1SNeel Natu 		vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
223688c4b8d1SNeel Natu 
223745e51299SNeel Natu 	for (i = 0; i < VM_MAXCPU; i++)
223845e51299SNeel Natu 		vpid_free(vmx->state[i].vpid);
223945e51299SNeel Natu 
2240366f6083SPeter Grehan 	/*
2241366f6083SPeter Grehan 	 * XXXSMP we also need to clear the VMCS active on the other vcpus.
2242366f6083SPeter Grehan 	 */
2243366f6083SPeter Grehan 	error = vmclear(&vmx->vmcs[0]);
2244366f6083SPeter Grehan 	if (error != 0)
2245366f6083SPeter Grehan 		panic("vmx_vmcleanup: vmclear error %d on vcpu 0", error);
2246366f6083SPeter Grehan 
2247366f6083SPeter Grehan 	free(vmx, M_VMX);
2248366f6083SPeter Grehan 
2249366f6083SPeter Grehan 	return;
2250366f6083SPeter Grehan }
2251366f6083SPeter Grehan 
2252366f6083SPeter Grehan static register_t *
2253366f6083SPeter Grehan vmxctx_regptr(struct vmxctx *vmxctx, int reg)
2254366f6083SPeter Grehan {
2255366f6083SPeter Grehan 
2256366f6083SPeter Grehan 	switch (reg) {
2257366f6083SPeter Grehan 	case VM_REG_GUEST_RAX:
2258366f6083SPeter Grehan 		return (&vmxctx->guest_rax);
2259366f6083SPeter Grehan 	case VM_REG_GUEST_RBX:
2260366f6083SPeter Grehan 		return (&vmxctx->guest_rbx);
2261366f6083SPeter Grehan 	case VM_REG_GUEST_RCX:
2262366f6083SPeter Grehan 		return (&vmxctx->guest_rcx);
2263366f6083SPeter Grehan 	case VM_REG_GUEST_RDX:
2264366f6083SPeter Grehan 		return (&vmxctx->guest_rdx);
2265366f6083SPeter Grehan 	case VM_REG_GUEST_RSI:
2266366f6083SPeter Grehan 		return (&vmxctx->guest_rsi);
2267366f6083SPeter Grehan 	case VM_REG_GUEST_RDI:
2268366f6083SPeter Grehan 		return (&vmxctx->guest_rdi);
2269366f6083SPeter Grehan 	case VM_REG_GUEST_RBP:
2270366f6083SPeter Grehan 		return (&vmxctx->guest_rbp);
2271366f6083SPeter Grehan 	case VM_REG_GUEST_R8:
2272366f6083SPeter Grehan 		return (&vmxctx->guest_r8);
2273366f6083SPeter Grehan 	case VM_REG_GUEST_R9:
2274366f6083SPeter Grehan 		return (&vmxctx->guest_r9);
2275366f6083SPeter Grehan 	case VM_REG_GUEST_R10:
2276366f6083SPeter Grehan 		return (&vmxctx->guest_r10);
2277366f6083SPeter Grehan 	case VM_REG_GUEST_R11:
2278366f6083SPeter Grehan 		return (&vmxctx->guest_r11);
2279366f6083SPeter Grehan 	case VM_REG_GUEST_R12:
2280366f6083SPeter Grehan 		return (&vmxctx->guest_r12);
2281366f6083SPeter Grehan 	case VM_REG_GUEST_R13:
2282366f6083SPeter Grehan 		return (&vmxctx->guest_r13);
2283366f6083SPeter Grehan 	case VM_REG_GUEST_R14:
2284366f6083SPeter Grehan 		return (&vmxctx->guest_r14);
2285366f6083SPeter Grehan 	case VM_REG_GUEST_R15:
2286366f6083SPeter Grehan 		return (&vmxctx->guest_r15);
2287366f6083SPeter Grehan 	default:
2288366f6083SPeter Grehan 		break;
2289366f6083SPeter Grehan 	}
2290366f6083SPeter Grehan 	return (NULL);
2291366f6083SPeter Grehan }
2292366f6083SPeter Grehan 
2293366f6083SPeter Grehan static int
2294366f6083SPeter Grehan vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval)
2295366f6083SPeter Grehan {
2296366f6083SPeter Grehan 	register_t *regp;
2297366f6083SPeter Grehan 
2298366f6083SPeter Grehan 	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
2299366f6083SPeter Grehan 		*retval = *regp;
2300366f6083SPeter Grehan 		return (0);
2301366f6083SPeter Grehan 	} else
2302366f6083SPeter Grehan 		return (EINVAL);
2303366f6083SPeter Grehan }
2304366f6083SPeter Grehan 
2305366f6083SPeter Grehan static int
2306366f6083SPeter Grehan vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val)
2307366f6083SPeter Grehan {
2308366f6083SPeter Grehan 	register_t *regp;
2309366f6083SPeter Grehan 
2310366f6083SPeter Grehan 	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
2311366f6083SPeter Grehan 		*regp = val;
2312366f6083SPeter Grehan 		return (0);
2313366f6083SPeter Grehan 	} else
2314366f6083SPeter Grehan 		return (EINVAL);
2315366f6083SPeter Grehan }
2316366f6083SPeter Grehan 
2317366f6083SPeter Grehan static int
2318aaaa0656SPeter Grehan vmx_shadow_reg(int reg)
2319aaaa0656SPeter Grehan {
2320aaaa0656SPeter Grehan 	int shreg;
2321aaaa0656SPeter Grehan 
2322aaaa0656SPeter Grehan 	shreg = -1;
2323aaaa0656SPeter Grehan 
2324aaaa0656SPeter Grehan 	switch (reg) {
2325aaaa0656SPeter Grehan 	case VM_REG_GUEST_CR0:
2326aaaa0656SPeter Grehan 		shreg = VMCS_CR0_SHADOW;
2327aaaa0656SPeter Grehan                 break;
2328aaaa0656SPeter Grehan         case VM_REG_GUEST_CR4:
2329aaaa0656SPeter Grehan 		shreg = VMCS_CR4_SHADOW;
2330aaaa0656SPeter Grehan 		break;
2331aaaa0656SPeter Grehan 	default:
2332aaaa0656SPeter Grehan 		break;
2333aaaa0656SPeter Grehan 	}
2334aaaa0656SPeter Grehan 
2335aaaa0656SPeter Grehan 	return (shreg);
2336aaaa0656SPeter Grehan }
2337aaaa0656SPeter Grehan 
2338aaaa0656SPeter Grehan static int
2339366f6083SPeter Grehan vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval)
2340366f6083SPeter Grehan {
2341d3c11f40SPeter Grehan 	int running, hostcpu;
2342366f6083SPeter Grehan 	struct vmx *vmx = arg;
2343366f6083SPeter Grehan 
2344d3c11f40SPeter Grehan 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
2345d3c11f40SPeter Grehan 	if (running && hostcpu != curcpu)
2346d3c11f40SPeter Grehan 		panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu);
2347d3c11f40SPeter Grehan 
2348366f6083SPeter Grehan 	if (vmxctx_getreg(&vmx->ctx[vcpu], reg, retval) == 0)
2349366f6083SPeter Grehan 		return (0);
2350366f6083SPeter Grehan 
2351d3c11f40SPeter Grehan 	return (vmcs_getreg(&vmx->vmcs[vcpu], running, reg, retval));
2352366f6083SPeter Grehan }
2353366f6083SPeter Grehan 
2354366f6083SPeter Grehan static int
2355366f6083SPeter Grehan vmx_setreg(void *arg, int vcpu, int reg, uint64_t val)
2356366f6083SPeter Grehan {
2357aaaa0656SPeter Grehan 	int error, hostcpu, running, shadow;
2358366f6083SPeter Grehan 	uint64_t ctls;
2359366f6083SPeter Grehan 	struct vmx *vmx = arg;
2360366f6083SPeter Grehan 
2361d3c11f40SPeter Grehan 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
2362d3c11f40SPeter Grehan 	if (running && hostcpu != curcpu)
2363d3c11f40SPeter Grehan 		panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu);
2364d3c11f40SPeter Grehan 
2365366f6083SPeter Grehan 	if (vmxctx_setreg(&vmx->ctx[vcpu], reg, val) == 0)
2366366f6083SPeter Grehan 		return (0);
2367366f6083SPeter Grehan 
2368d3c11f40SPeter Grehan 	error = vmcs_setreg(&vmx->vmcs[vcpu], running, reg, val);
2369366f6083SPeter Grehan 
2370366f6083SPeter Grehan 	if (error == 0) {
2371366f6083SPeter Grehan 		/*
2372366f6083SPeter Grehan 		 * If the "load EFER" VM-entry control is 1 then the
2373366f6083SPeter Grehan 		 * value of EFER.LMA must be identical to "IA-32e mode guest"
2374366f6083SPeter Grehan 		 * bit in the VM-entry control.
2375366f6083SPeter Grehan 		 */
2376366f6083SPeter Grehan 		if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 &&
2377366f6083SPeter Grehan 		    (reg == VM_REG_GUEST_EFER)) {
2378d3c11f40SPeter Grehan 			vmcs_getreg(&vmx->vmcs[vcpu], running,
2379366f6083SPeter Grehan 				    VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls);
2380366f6083SPeter Grehan 			if (val & EFER_LMA)
2381366f6083SPeter Grehan 				ctls |= VM_ENTRY_GUEST_LMA;
2382366f6083SPeter Grehan 			else
2383366f6083SPeter Grehan 				ctls &= ~VM_ENTRY_GUEST_LMA;
2384d3c11f40SPeter Grehan 			vmcs_setreg(&vmx->vmcs[vcpu], running,
2385366f6083SPeter Grehan 				    VMCS_IDENT(VMCS_ENTRY_CTLS), ctls);
2386366f6083SPeter Grehan 		}
2387aaaa0656SPeter Grehan 
2388aaaa0656SPeter Grehan 		shadow = vmx_shadow_reg(reg);
2389aaaa0656SPeter Grehan 		if (shadow > 0) {
2390aaaa0656SPeter Grehan 			/*
2391aaaa0656SPeter Grehan 			 * Store the unmodified value in the shadow
2392aaaa0656SPeter Grehan 			 */
2393aaaa0656SPeter Grehan 			error = vmcs_setreg(&vmx->vmcs[vcpu], running,
2394aaaa0656SPeter Grehan 				    VMCS_IDENT(shadow), val);
2395aaaa0656SPeter Grehan 		}
2396366f6083SPeter Grehan 	}
2397366f6083SPeter Grehan 
2398366f6083SPeter Grehan 	return (error);
2399366f6083SPeter Grehan }
2400366f6083SPeter Grehan 
2401366f6083SPeter Grehan static int
2402366f6083SPeter Grehan vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
2403366f6083SPeter Grehan {
2404366f6083SPeter Grehan 	struct vmx *vmx = arg;
2405366f6083SPeter Grehan 
2406366f6083SPeter Grehan 	return (vmcs_getdesc(&vmx->vmcs[vcpu], reg, desc));
2407366f6083SPeter Grehan }
2408366f6083SPeter Grehan 
2409366f6083SPeter Grehan static int
2410366f6083SPeter Grehan vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
2411366f6083SPeter Grehan {
2412366f6083SPeter Grehan 	struct vmx *vmx = arg;
2413366f6083SPeter Grehan 
2414366f6083SPeter Grehan 	return (vmcs_setdesc(&vmx->vmcs[vcpu], reg, desc));
2415366f6083SPeter Grehan }
2416366f6083SPeter Grehan 
2417366f6083SPeter Grehan static int
2418366f6083SPeter Grehan vmx_getcap(void *arg, int vcpu, int type, int *retval)
2419366f6083SPeter Grehan {
2420366f6083SPeter Grehan 	struct vmx *vmx = arg;
2421366f6083SPeter Grehan 	int vcap;
2422366f6083SPeter Grehan 	int ret;
2423366f6083SPeter Grehan 
2424366f6083SPeter Grehan 	ret = ENOENT;
2425366f6083SPeter Grehan 
2426366f6083SPeter Grehan 	vcap = vmx->cap[vcpu].set;
2427366f6083SPeter Grehan 
2428366f6083SPeter Grehan 	switch (type) {
2429366f6083SPeter Grehan 	case VM_CAP_HALT_EXIT:
2430366f6083SPeter Grehan 		if (cap_halt_exit)
2431366f6083SPeter Grehan 			ret = 0;
2432366f6083SPeter Grehan 		break;
2433366f6083SPeter Grehan 	case VM_CAP_PAUSE_EXIT:
2434366f6083SPeter Grehan 		if (cap_pause_exit)
2435366f6083SPeter Grehan 			ret = 0;
2436366f6083SPeter Grehan 		break;
2437366f6083SPeter Grehan 	case VM_CAP_MTRAP_EXIT:
2438366f6083SPeter Grehan 		if (cap_monitor_trap)
2439366f6083SPeter Grehan 			ret = 0;
2440366f6083SPeter Grehan 		break;
2441366f6083SPeter Grehan 	case VM_CAP_UNRESTRICTED_GUEST:
2442366f6083SPeter Grehan 		if (cap_unrestricted_guest)
2443366f6083SPeter Grehan 			ret = 0;
2444366f6083SPeter Grehan 		break;
244549cc03daSNeel Natu 	case VM_CAP_ENABLE_INVPCID:
244649cc03daSNeel Natu 		if (cap_invpcid)
244749cc03daSNeel Natu 			ret = 0;
244849cc03daSNeel Natu 		break;
2449366f6083SPeter Grehan 	default:
2450366f6083SPeter Grehan 		break;
2451366f6083SPeter Grehan 	}
2452366f6083SPeter Grehan 
2453366f6083SPeter Grehan 	if (ret == 0)
2454366f6083SPeter Grehan 		*retval = (vcap & (1 << type)) ? 1 : 0;
2455366f6083SPeter Grehan 
2456366f6083SPeter Grehan 	return (ret);
2457366f6083SPeter Grehan }
2458366f6083SPeter Grehan 
2459366f6083SPeter Grehan static int
2460366f6083SPeter Grehan vmx_setcap(void *arg, int vcpu, int type, int val)
2461366f6083SPeter Grehan {
2462366f6083SPeter Grehan 	struct vmx *vmx = arg;
2463366f6083SPeter Grehan 	struct vmcs *vmcs = &vmx->vmcs[vcpu];
2464366f6083SPeter Grehan 	uint32_t baseval;
2465366f6083SPeter Grehan 	uint32_t *pptr;
2466366f6083SPeter Grehan 	int error;
2467366f6083SPeter Grehan 	int flag;
2468366f6083SPeter Grehan 	int reg;
2469366f6083SPeter Grehan 	int retval;
2470366f6083SPeter Grehan 
2471366f6083SPeter Grehan 	retval = ENOENT;
2472366f6083SPeter Grehan 	pptr = NULL;
2473366f6083SPeter Grehan 
2474366f6083SPeter Grehan 	switch (type) {
2475366f6083SPeter Grehan 	case VM_CAP_HALT_EXIT:
2476366f6083SPeter Grehan 		if (cap_halt_exit) {
2477366f6083SPeter Grehan 			retval = 0;
2478366f6083SPeter Grehan 			pptr = &vmx->cap[vcpu].proc_ctls;
2479366f6083SPeter Grehan 			baseval = *pptr;
2480366f6083SPeter Grehan 			flag = PROCBASED_HLT_EXITING;
2481366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
2482366f6083SPeter Grehan 		}
2483366f6083SPeter Grehan 		break;
2484366f6083SPeter Grehan 	case VM_CAP_MTRAP_EXIT:
2485366f6083SPeter Grehan 		if (cap_monitor_trap) {
2486366f6083SPeter Grehan 			retval = 0;
2487366f6083SPeter Grehan 			pptr = &vmx->cap[vcpu].proc_ctls;
2488366f6083SPeter Grehan 			baseval = *pptr;
2489366f6083SPeter Grehan 			flag = PROCBASED_MTF;
2490366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
2491366f6083SPeter Grehan 		}
2492366f6083SPeter Grehan 		break;
2493366f6083SPeter Grehan 	case VM_CAP_PAUSE_EXIT:
2494366f6083SPeter Grehan 		if (cap_pause_exit) {
2495366f6083SPeter Grehan 			retval = 0;
2496366f6083SPeter Grehan 			pptr = &vmx->cap[vcpu].proc_ctls;
2497366f6083SPeter Grehan 			baseval = *pptr;
2498366f6083SPeter Grehan 			flag = PROCBASED_PAUSE_EXITING;
2499366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
2500366f6083SPeter Grehan 		}
2501366f6083SPeter Grehan 		break;
2502366f6083SPeter Grehan 	case VM_CAP_UNRESTRICTED_GUEST:
2503366f6083SPeter Grehan 		if (cap_unrestricted_guest) {
2504366f6083SPeter Grehan 			retval = 0;
250549cc03daSNeel Natu 			pptr = &vmx->cap[vcpu].proc_ctls2;
250649cc03daSNeel Natu 			baseval = *pptr;
2507366f6083SPeter Grehan 			flag = PROCBASED2_UNRESTRICTED_GUEST;
2508366f6083SPeter Grehan 			reg = VMCS_SEC_PROC_BASED_CTLS;
2509366f6083SPeter Grehan 		}
2510366f6083SPeter Grehan 		break;
251149cc03daSNeel Natu 	case VM_CAP_ENABLE_INVPCID:
251249cc03daSNeel Natu 		if (cap_invpcid) {
251349cc03daSNeel Natu 			retval = 0;
251449cc03daSNeel Natu 			pptr = &vmx->cap[vcpu].proc_ctls2;
251549cc03daSNeel Natu 			baseval = *pptr;
251649cc03daSNeel Natu 			flag = PROCBASED2_ENABLE_INVPCID;
251749cc03daSNeel Natu 			reg = VMCS_SEC_PROC_BASED_CTLS;
251849cc03daSNeel Natu 		}
251949cc03daSNeel Natu 		break;
2520366f6083SPeter Grehan 	default:
2521366f6083SPeter Grehan 		break;
2522366f6083SPeter Grehan 	}
2523366f6083SPeter Grehan 
2524366f6083SPeter Grehan 	if (retval == 0) {
2525366f6083SPeter Grehan 		if (val) {
2526366f6083SPeter Grehan 			baseval |= flag;
2527366f6083SPeter Grehan 		} else {
2528366f6083SPeter Grehan 			baseval &= ~flag;
2529366f6083SPeter Grehan 		}
2530366f6083SPeter Grehan 		VMPTRLD(vmcs);
2531366f6083SPeter Grehan 		error = vmwrite(reg, baseval);
2532366f6083SPeter Grehan 		VMCLEAR(vmcs);
2533366f6083SPeter Grehan 
2534366f6083SPeter Grehan 		if (error) {
2535366f6083SPeter Grehan 			retval = error;
2536366f6083SPeter Grehan 		} else {
2537366f6083SPeter Grehan 			/*
2538366f6083SPeter Grehan 			 * Update optional stored flags, and record
2539366f6083SPeter Grehan 			 * setting
2540366f6083SPeter Grehan 			 */
2541366f6083SPeter Grehan 			if (pptr != NULL) {
2542366f6083SPeter Grehan 				*pptr = baseval;
2543366f6083SPeter Grehan 			}
2544366f6083SPeter Grehan 
2545366f6083SPeter Grehan 			if (val) {
2546366f6083SPeter Grehan 				vmx->cap[vcpu].set |= (1 << type);
2547366f6083SPeter Grehan 			} else {
2548366f6083SPeter Grehan 				vmx->cap[vcpu].set &= ~(1 << type);
2549366f6083SPeter Grehan 			}
2550366f6083SPeter Grehan 		}
2551366f6083SPeter Grehan 	}
2552366f6083SPeter Grehan 
2553366f6083SPeter Grehan         return (retval);
2554366f6083SPeter Grehan }
2555366f6083SPeter Grehan 
255688c4b8d1SNeel Natu struct vlapic_vtx {
255788c4b8d1SNeel Natu 	struct vlapic	vlapic;
2558176666c2SNeel Natu 	struct pir_desc	*pir_desc;
255930b94db8SNeel Natu 	struct vmx	*vmx;
256088c4b8d1SNeel Natu };
256188c4b8d1SNeel Natu 
256288c4b8d1SNeel Natu #define	VMX_CTR_PIR(vm, vcpuid, pir_desc, notify, vector, level, msg)	\
256388c4b8d1SNeel Natu do {									\
256488c4b8d1SNeel Natu 	VCPU_CTR2(vm, vcpuid, msg " assert %s-triggered vector %d",	\
256588c4b8d1SNeel Natu 	    level ? "level" : "edge", vector);				\
256688c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir0 0x%016lx", pir_desc->pir[0]);	\
256788c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir1 0x%016lx", pir_desc->pir[1]);	\
256888c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir2 0x%016lx", pir_desc->pir[2]);	\
256988c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir3 0x%016lx", pir_desc->pir[3]);	\
257088c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " notify: %s", notify ? "yes" : "no");\
257188c4b8d1SNeel Natu } while (0)
257288c4b8d1SNeel Natu 
257388c4b8d1SNeel Natu /*
257488c4b8d1SNeel Natu  * vlapic->ops handlers that utilize the APICv hardware assist described in
257588c4b8d1SNeel Natu  * Chapter 29 of the Intel SDM.
257688c4b8d1SNeel Natu  */
257788c4b8d1SNeel Natu static int
257888c4b8d1SNeel Natu vmx_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
257988c4b8d1SNeel Natu {
258088c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
258188c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
258288c4b8d1SNeel Natu 	uint64_t mask;
258388c4b8d1SNeel Natu 	int idx, notify;
258488c4b8d1SNeel Natu 
258588c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
2586176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
258788c4b8d1SNeel Natu 
258888c4b8d1SNeel Natu 	/*
258988c4b8d1SNeel Natu 	 * Keep track of interrupt requests in the PIR descriptor. This is
259088c4b8d1SNeel Natu 	 * because the virtual APIC page pointed to by the VMCS cannot be
259188c4b8d1SNeel Natu 	 * modified if the vcpu is running.
259288c4b8d1SNeel Natu 	 */
259388c4b8d1SNeel Natu 	idx = vector / 64;
259488c4b8d1SNeel Natu 	mask = 1UL << (vector % 64);
259588c4b8d1SNeel Natu 	atomic_set_long(&pir_desc->pir[idx], mask);
259688c4b8d1SNeel Natu 	notify = atomic_cmpset_long(&pir_desc->pending, 0, 1);
259788c4b8d1SNeel Natu 
259888c4b8d1SNeel Natu 	VMX_CTR_PIR(vlapic->vm, vlapic->vcpuid, pir_desc, notify, vector,
259988c4b8d1SNeel Natu 	    level, "vmx_set_intr_ready");
260088c4b8d1SNeel Natu 	return (notify);
260188c4b8d1SNeel Natu }
260288c4b8d1SNeel Natu 
260388c4b8d1SNeel Natu static int
260488c4b8d1SNeel Natu vmx_pending_intr(struct vlapic *vlapic, int *vecptr)
260588c4b8d1SNeel Natu {
260688c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
260788c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
260888c4b8d1SNeel Natu 	struct LAPIC *lapic;
260988c4b8d1SNeel Natu 	uint64_t pending, pirval;
261088c4b8d1SNeel Natu 	uint32_t ppr, vpr;
261188c4b8d1SNeel Natu 	int i;
261288c4b8d1SNeel Natu 
261388c4b8d1SNeel Natu 	/*
261488c4b8d1SNeel Natu 	 * This function is only expected to be called from the 'HLT' exit
261588c4b8d1SNeel Natu 	 * handler which does not care about the vector that is pending.
261688c4b8d1SNeel Natu 	 */
261788c4b8d1SNeel Natu 	KASSERT(vecptr == NULL, ("vmx_pending_intr: vecptr must be NULL"));
261888c4b8d1SNeel Natu 
261988c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
2620176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
262188c4b8d1SNeel Natu 
262288c4b8d1SNeel Natu 	pending = atomic_load_acq_long(&pir_desc->pending);
262388c4b8d1SNeel Natu 	if (!pending)
262488c4b8d1SNeel Natu 		return (0);	/* common case */
262588c4b8d1SNeel Natu 
262688c4b8d1SNeel Natu 	/*
262788c4b8d1SNeel Natu 	 * If there is an interrupt pending then it will be recognized only
262888c4b8d1SNeel Natu 	 * if its priority is greater than the processor priority.
262988c4b8d1SNeel Natu 	 *
263088c4b8d1SNeel Natu 	 * Special case: if the processor priority is zero then any pending
263188c4b8d1SNeel Natu 	 * interrupt will be recognized.
263288c4b8d1SNeel Natu 	 */
263388c4b8d1SNeel Natu 	lapic = vlapic->apic_page;
263488c4b8d1SNeel Natu 	ppr = lapic->ppr & 0xf0;
263588c4b8d1SNeel Natu 	if (ppr == 0)
263688c4b8d1SNeel Natu 		return (1);
263788c4b8d1SNeel Natu 
263888c4b8d1SNeel Natu 	VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "HLT with non-zero PPR %d",
263988c4b8d1SNeel Natu 	    lapic->ppr);
264088c4b8d1SNeel Natu 
264188c4b8d1SNeel Natu 	for (i = 3; i >= 0; i--) {
264288c4b8d1SNeel Natu 		pirval = pir_desc->pir[i];
264388c4b8d1SNeel Natu 		if (pirval != 0) {
264488c4b8d1SNeel Natu 			vpr = (i * 64 + flsl(pirval) - 1) & 0xf0;
264588c4b8d1SNeel Natu 			return (vpr > ppr);
264688c4b8d1SNeel Natu 		}
264788c4b8d1SNeel Natu 	}
264888c4b8d1SNeel Natu 	return (0);
264988c4b8d1SNeel Natu }
265088c4b8d1SNeel Natu 
265188c4b8d1SNeel Natu static void
265288c4b8d1SNeel Natu vmx_intr_accepted(struct vlapic *vlapic, int vector)
265388c4b8d1SNeel Natu {
265488c4b8d1SNeel Natu 
265588c4b8d1SNeel Natu 	panic("vmx_intr_accepted: not expected to be called");
265688c4b8d1SNeel Natu }
265788c4b8d1SNeel Natu 
2658176666c2SNeel Natu static void
265930b94db8SNeel Natu vmx_set_tmr(struct vlapic *vlapic, int vector, bool level)
266030b94db8SNeel Natu {
266130b94db8SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
266230b94db8SNeel Natu 	struct vmx *vmx;
266330b94db8SNeel Natu 	struct vmcs *vmcs;
266430b94db8SNeel Natu 	uint64_t mask, val;
266530b94db8SNeel Natu 
266630b94db8SNeel Natu 	KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
266730b94db8SNeel Natu 	KASSERT(!vcpu_is_running(vlapic->vm, vlapic->vcpuid, NULL),
266830b94db8SNeel Natu 	    ("vmx_set_tmr: vcpu cannot be running"));
266930b94db8SNeel Natu 
267030b94db8SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
267130b94db8SNeel Natu 	vmx = vlapic_vtx->vmx;
267230b94db8SNeel Natu 	vmcs = &vmx->vmcs[vlapic->vcpuid];
267330b94db8SNeel Natu 	mask = 1UL << (vector % 64);
267430b94db8SNeel Natu 
267530b94db8SNeel Natu 	VMPTRLD(vmcs);
267630b94db8SNeel Natu 	val = vmcs_read(VMCS_EOI_EXIT(vector));
267730b94db8SNeel Natu 	if (level)
267830b94db8SNeel Natu 		val |= mask;
267930b94db8SNeel Natu 	else
268030b94db8SNeel Natu 		val &= ~mask;
268130b94db8SNeel Natu 	vmcs_write(VMCS_EOI_EXIT(vector), val);
268230b94db8SNeel Natu 	VMCLEAR(vmcs);
268330b94db8SNeel Natu }
268430b94db8SNeel Natu 
268530b94db8SNeel Natu static void
2686159dd56fSNeel Natu vmx_enable_x2apic_mode(struct vlapic *vlapic)
2687159dd56fSNeel Natu {
2688159dd56fSNeel Natu 	struct vmx *vmx;
2689159dd56fSNeel Natu 	struct vmcs *vmcs;
2690159dd56fSNeel Natu 	uint32_t proc_ctls2;
2691159dd56fSNeel Natu 	int vcpuid, error;
2692159dd56fSNeel Natu 
2693159dd56fSNeel Natu 	vcpuid = vlapic->vcpuid;
2694159dd56fSNeel Natu 	vmx = ((struct vlapic_vtx *)vlapic)->vmx;
2695159dd56fSNeel Natu 	vmcs = &vmx->vmcs[vcpuid];
2696159dd56fSNeel Natu 
2697159dd56fSNeel Natu 	proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
2698159dd56fSNeel Natu 	KASSERT((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) != 0,
2699159dd56fSNeel Natu 	    ("%s: invalid proc_ctls2 %#x", __func__, proc_ctls2));
2700159dd56fSNeel Natu 
2701159dd56fSNeel Natu 	proc_ctls2 &= ~PROCBASED2_VIRTUALIZE_APIC_ACCESSES;
2702159dd56fSNeel Natu 	proc_ctls2 |= PROCBASED2_VIRTUALIZE_X2APIC_MODE;
2703159dd56fSNeel Natu 	vmx->cap[vcpuid].proc_ctls2 = proc_ctls2;
2704159dd56fSNeel Natu 
2705159dd56fSNeel Natu 	VMPTRLD(vmcs);
2706159dd56fSNeel Natu 	vmcs_write(VMCS_SEC_PROC_BASED_CTLS, proc_ctls2);
2707159dd56fSNeel Natu 	VMCLEAR(vmcs);
2708159dd56fSNeel Natu 
2709159dd56fSNeel Natu 	if (vlapic->vcpuid == 0) {
2710159dd56fSNeel Natu 		/*
2711159dd56fSNeel Natu 		 * The nested page table mappings are shared by all vcpus
2712159dd56fSNeel Natu 		 * so unmap the APIC access page just once.
2713159dd56fSNeel Natu 		 */
2714159dd56fSNeel Natu 		error = vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
2715159dd56fSNeel Natu 		KASSERT(error == 0, ("%s: vm_unmap_mmio error %d",
2716159dd56fSNeel Natu 		    __func__, error));
2717159dd56fSNeel Natu 
2718159dd56fSNeel Natu 		/*
2719159dd56fSNeel Natu 		 * The MSR bitmap is shared by all vcpus so modify it only
2720159dd56fSNeel Natu 		 * once in the context of vcpu 0.
2721159dd56fSNeel Natu 		 */
2722159dd56fSNeel Natu 		error = vmx_allow_x2apic_msrs(vmx);
2723159dd56fSNeel Natu 		KASSERT(error == 0, ("%s: vmx_allow_x2apic_msrs error %d",
2724159dd56fSNeel Natu 		    __func__, error));
2725159dd56fSNeel Natu 	}
2726159dd56fSNeel Natu }
2727159dd56fSNeel Natu 
2728159dd56fSNeel Natu static void
2729176666c2SNeel Natu vmx_post_intr(struct vlapic *vlapic, int hostcpu)
2730176666c2SNeel Natu {
2731176666c2SNeel Natu 
2732176666c2SNeel Natu 	ipi_cpu(hostcpu, pirvec);
2733176666c2SNeel Natu }
2734176666c2SNeel Natu 
273588c4b8d1SNeel Natu /*
273688c4b8d1SNeel Natu  * Transfer the pending interrupts in the PIR descriptor to the IRR
273788c4b8d1SNeel Natu  * in the virtual APIC page.
273888c4b8d1SNeel Natu  */
273988c4b8d1SNeel Natu static void
274088c4b8d1SNeel Natu vmx_inject_pir(struct vlapic *vlapic)
274188c4b8d1SNeel Natu {
274288c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
274388c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
274488c4b8d1SNeel Natu 	struct LAPIC *lapic;
274588c4b8d1SNeel Natu 	uint64_t val, pirval;
27460e30c5c0SWarner Losh 	int rvi, pirbase = -1;
274788c4b8d1SNeel Natu 	uint16_t intr_status_old, intr_status_new;
274888c4b8d1SNeel Natu 
274988c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
2750176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
275188c4b8d1SNeel Natu 	if (atomic_cmpset_long(&pir_desc->pending, 1, 0) == 0) {
275288c4b8d1SNeel Natu 		VCPU_CTR0(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
275388c4b8d1SNeel Natu 		    "no posted interrupt pending");
275488c4b8d1SNeel Natu 		return;
275588c4b8d1SNeel Natu 	}
275688c4b8d1SNeel Natu 
275788c4b8d1SNeel Natu 	pirval = 0;
2758*201b1cccSPeter Grehan 	pirbase = -1;
275988c4b8d1SNeel Natu 	lapic = vlapic->apic_page;
276088c4b8d1SNeel Natu 
276188c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[0]);
276288c4b8d1SNeel Natu 	if (val != 0) {
276388c4b8d1SNeel Natu 		lapic->irr0 |= val;
276488c4b8d1SNeel Natu 		lapic->irr1 |= val >> 32;
276588c4b8d1SNeel Natu 		pirbase = 0;
276688c4b8d1SNeel Natu 		pirval = val;
276788c4b8d1SNeel Natu 	}
276888c4b8d1SNeel Natu 
276988c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[1]);
277088c4b8d1SNeel Natu 	if (val != 0) {
277188c4b8d1SNeel Natu 		lapic->irr2 |= val;
277288c4b8d1SNeel Natu 		lapic->irr3 |= val >> 32;
277388c4b8d1SNeel Natu 		pirbase = 64;
277488c4b8d1SNeel Natu 		pirval = val;
277588c4b8d1SNeel Natu 	}
277688c4b8d1SNeel Natu 
277788c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[2]);
277888c4b8d1SNeel Natu 	if (val != 0) {
277988c4b8d1SNeel Natu 		lapic->irr4 |= val;
278088c4b8d1SNeel Natu 		lapic->irr5 |= val >> 32;
278188c4b8d1SNeel Natu 		pirbase = 128;
278288c4b8d1SNeel Natu 		pirval = val;
278388c4b8d1SNeel Natu 	}
278488c4b8d1SNeel Natu 
278588c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[3]);
278688c4b8d1SNeel Natu 	if (val != 0) {
278788c4b8d1SNeel Natu 		lapic->irr6 |= val;
278888c4b8d1SNeel Natu 		lapic->irr7 |= val >> 32;
278988c4b8d1SNeel Natu 		pirbase = 192;
279088c4b8d1SNeel Natu 		pirval = val;
279188c4b8d1SNeel Natu 	}
2792*201b1cccSPeter Grehan 
279388c4b8d1SNeel Natu 	VLAPIC_CTR_IRR(vlapic, "vmx_inject_pir");
279488c4b8d1SNeel Natu 
279588c4b8d1SNeel Natu 	/*
279688c4b8d1SNeel Natu 	 * Update RVI so the processor can evaluate pending virtual
279788c4b8d1SNeel Natu 	 * interrupts on VM-entry.
2798*201b1cccSPeter Grehan 	 *
2799*201b1cccSPeter Grehan 	 * It is possible for pirval to be 0 here, even though the
2800*201b1cccSPeter Grehan 	 * pending bit has been set. The scenario is:
2801*201b1cccSPeter Grehan 	 * CPU-Y is sending a posted interrupt to CPU-X, which
2802*201b1cccSPeter Grehan 	 * is running a guest and processing posted interrupts in h/w.
2803*201b1cccSPeter Grehan 	 * CPU-X will eventually exit and the state seen in s/w is
2804*201b1cccSPeter Grehan 	 * the pending bit set, but no PIR bits set.
2805*201b1cccSPeter Grehan 	 *
2806*201b1cccSPeter Grehan 	 *      CPU-X                      CPU-Y
2807*201b1cccSPeter Grehan 	 *   (vm running)                (host running)
2808*201b1cccSPeter Grehan 	 *   rx posted interrupt
2809*201b1cccSPeter Grehan 	 *   CLEAR pending bit
2810*201b1cccSPeter Grehan 	 *				 SET PIR bit
2811*201b1cccSPeter Grehan 	 *   READ/CLEAR PIR bits
2812*201b1cccSPeter Grehan 	 *				 SET pending bit
2813*201b1cccSPeter Grehan 	 *   (vm exit)
2814*201b1cccSPeter Grehan 	 *   pending bit set, PIR 0
281588c4b8d1SNeel Natu 	 */
281688c4b8d1SNeel Natu 	if (pirval != 0) {
281788c4b8d1SNeel Natu 		rvi = pirbase + flsl(pirval) - 1;
281888c4b8d1SNeel Natu 		intr_status_old = vmcs_read(VMCS_GUEST_INTR_STATUS);
281988c4b8d1SNeel Natu 		intr_status_new = (intr_status_old & 0xFF00) | rvi;
282088c4b8d1SNeel Natu 		if (intr_status_new > intr_status_old) {
282188c4b8d1SNeel Natu 			vmcs_write(VMCS_GUEST_INTR_STATUS, intr_status_new);
282288c4b8d1SNeel Natu 			VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
282388c4b8d1SNeel Natu 			    "guest_intr_status changed from 0x%04x to 0x%04x",
282488c4b8d1SNeel Natu 			    intr_status_old, intr_status_new);
282588c4b8d1SNeel Natu 		}
282688c4b8d1SNeel Natu 	}
282788c4b8d1SNeel Natu }
282888c4b8d1SNeel Natu 
2829de5ea6b6SNeel Natu static struct vlapic *
2830de5ea6b6SNeel Natu vmx_vlapic_init(void *arg, int vcpuid)
2831de5ea6b6SNeel Natu {
2832de5ea6b6SNeel Natu 	struct vmx *vmx;
2833de5ea6b6SNeel Natu 	struct vlapic *vlapic;
2834176666c2SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
2835de5ea6b6SNeel Natu 
2836de5ea6b6SNeel Natu 	vmx = arg;
2837de5ea6b6SNeel Natu 
283888c4b8d1SNeel Natu 	vlapic = malloc(sizeof(struct vlapic_vtx), M_VLAPIC, M_WAITOK | M_ZERO);
2839de5ea6b6SNeel Natu 	vlapic->vm = vmx->vm;
2840de5ea6b6SNeel Natu 	vlapic->vcpuid = vcpuid;
2841de5ea6b6SNeel Natu 	vlapic->apic_page = (struct LAPIC *)&vmx->apic_page[vcpuid];
2842de5ea6b6SNeel Natu 
2843176666c2SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
2844176666c2SNeel Natu 	vlapic_vtx->pir_desc = &vmx->pir_desc[vcpuid];
284530b94db8SNeel Natu 	vlapic_vtx->vmx = vmx;
2846176666c2SNeel Natu 
284788c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
284888c4b8d1SNeel Natu 		vlapic->ops.set_intr_ready = vmx_set_intr_ready;
284988c4b8d1SNeel Natu 		vlapic->ops.pending_intr = vmx_pending_intr;
285088c4b8d1SNeel Natu 		vlapic->ops.intr_accepted = vmx_intr_accepted;
285130b94db8SNeel Natu 		vlapic->ops.set_tmr = vmx_set_tmr;
2852159dd56fSNeel Natu 		vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode;
285388c4b8d1SNeel Natu 	}
285488c4b8d1SNeel Natu 
2855176666c2SNeel Natu 	if (posted_interrupts)
2856176666c2SNeel Natu 		vlapic->ops.post_intr = vmx_post_intr;
2857176666c2SNeel Natu 
2858de5ea6b6SNeel Natu 	vlapic_init(vlapic);
2859de5ea6b6SNeel Natu 
2860de5ea6b6SNeel Natu 	return (vlapic);
2861de5ea6b6SNeel Natu }
2862de5ea6b6SNeel Natu 
2863de5ea6b6SNeel Natu static void
2864de5ea6b6SNeel Natu vmx_vlapic_cleanup(void *arg, struct vlapic *vlapic)
2865de5ea6b6SNeel Natu {
2866de5ea6b6SNeel Natu 
2867de5ea6b6SNeel Natu 	vlapic_cleanup(vlapic);
2868de5ea6b6SNeel Natu 	free(vlapic, M_VLAPIC);
2869de5ea6b6SNeel Natu }
2870de5ea6b6SNeel Natu 
2871366f6083SPeter Grehan struct vmm_ops vmm_ops_intel = {
2872366f6083SPeter Grehan 	vmx_init,
2873366f6083SPeter Grehan 	vmx_cleanup,
287463e62d39SJohn Baldwin 	vmx_restore,
2875366f6083SPeter Grehan 	vmx_vminit,
2876366f6083SPeter Grehan 	vmx_run,
2877366f6083SPeter Grehan 	vmx_vmcleanup,
2878366f6083SPeter Grehan 	vmx_getreg,
2879366f6083SPeter Grehan 	vmx_setreg,
2880366f6083SPeter Grehan 	vmx_getdesc,
2881366f6083SPeter Grehan 	vmx_setdesc,
2882366f6083SPeter Grehan 	vmx_getcap,
2883318224bbSNeel Natu 	vmx_setcap,
2884318224bbSNeel Natu 	ept_vmspace_alloc,
2885318224bbSNeel Natu 	ept_vmspace_free,
2886de5ea6b6SNeel Natu 	vmx_vlapic_init,
2887de5ea6b6SNeel Natu 	vmx_vlapic_cleanup,
2888366f6083SPeter Grehan };
2889