xref: /freebsd/sys/amd64/vmm/intel/vmx.c (revision bd50262f705c4fed70ea94d16a0f19b5f5497cf2)
1366f6083SPeter Grehan /*-
2c49761ddSPedro F. Giffuni  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3c49761ddSPedro F. Giffuni  *
4366f6083SPeter Grehan  * Copyright (c) 2011 NetApp, Inc.
5366f6083SPeter Grehan  * All rights reserved.
6366f6083SPeter Grehan  *
7366f6083SPeter Grehan  * Redistribution and use in source and binary forms, with or without
8366f6083SPeter Grehan  * modification, are permitted provided that the following conditions
9366f6083SPeter Grehan  * are met:
10366f6083SPeter Grehan  * 1. Redistributions of source code must retain the above copyright
11366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer.
12366f6083SPeter Grehan  * 2. Redistributions in binary form must reproduce the above copyright
13366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer in the
14366f6083SPeter Grehan  *    documentation and/or other materials provided with the distribution.
15366f6083SPeter Grehan  *
16366f6083SPeter Grehan  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17366f6083SPeter Grehan  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18366f6083SPeter Grehan  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19366f6083SPeter Grehan  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20366f6083SPeter Grehan  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21366f6083SPeter Grehan  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22366f6083SPeter Grehan  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23366f6083SPeter Grehan  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24366f6083SPeter Grehan  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25366f6083SPeter Grehan  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26366f6083SPeter Grehan  * SUCH DAMAGE.
27366f6083SPeter Grehan  *
28366f6083SPeter Grehan  * $FreeBSD$
29366f6083SPeter Grehan  */
30366f6083SPeter Grehan 
31366f6083SPeter Grehan #include <sys/cdefs.h>
32366f6083SPeter Grehan __FBSDID("$FreeBSD$");
33366f6083SPeter Grehan 
34366f6083SPeter Grehan #include <sys/param.h>
35366f6083SPeter Grehan #include <sys/systm.h>
36366f6083SPeter Grehan #include <sys/smp.h>
37366f6083SPeter Grehan #include <sys/kernel.h>
38366f6083SPeter Grehan #include <sys/malloc.h>
39366f6083SPeter Grehan #include <sys/pcpu.h>
40366f6083SPeter Grehan #include <sys/proc.h>
413565b59eSNeel Natu #include <sys/sysctl.h>
42366f6083SPeter Grehan 
43366f6083SPeter Grehan #include <vm/vm.h>
44366f6083SPeter Grehan #include <vm/pmap.h>
45366f6083SPeter Grehan 
46366f6083SPeter Grehan #include <machine/psl.h>
47366f6083SPeter Grehan #include <machine/cpufunc.h>
488b287612SJohn Baldwin #include <machine/md_var.h>
49366f6083SPeter Grehan #include <machine/segments.h>
50176666c2SNeel Natu #include <machine/smp.h>
51608f97c3SPeter Grehan #include <machine/specialreg.h>
52366f6083SPeter Grehan #include <machine/vmparam.h>
53366f6083SPeter Grehan 
54366f6083SPeter Grehan #include <machine/vmm.h>
55dc506506SNeel Natu #include <machine/vmm_dev.h>
56e813a873SNeel Natu #include <machine/vmm_instruction_emul.h>
57c3498942SNeel Natu #include "vmm_lapic.h"
58b01c2033SNeel Natu #include "vmm_host.h"
59762fd208STycho Nightingale #include "vmm_ioport.h"
60366f6083SPeter Grehan #include "vmm_ktr.h"
61366f6083SPeter Grehan #include "vmm_stat.h"
620775fbb4STycho Nightingale #include "vatpic.h"
63de5ea6b6SNeel Natu #include "vlapic.h"
64de5ea6b6SNeel Natu #include "vlapic_priv.h"
65366f6083SPeter Grehan 
66366f6083SPeter Grehan #include "ept.h"
67366f6083SPeter Grehan #include "vmx_cpufunc.h"
68366f6083SPeter Grehan #include "vmx.h"
69c3498942SNeel Natu #include "vmx_msr.h"
70366f6083SPeter Grehan #include "x86.h"
71366f6083SPeter Grehan #include "vmx_controls.h"
72366f6083SPeter Grehan 
73366f6083SPeter Grehan #define	PINBASED_CTLS_ONE_SETTING					\
74366f6083SPeter Grehan 	(PINBASED_EXTINT_EXITING	|				\
75366f6083SPeter Grehan 	 PINBASED_NMI_EXITING		|				\
76366f6083SPeter Grehan 	 PINBASED_VIRTUAL_NMI)
77366f6083SPeter Grehan #define	PINBASED_CTLS_ZERO_SETTING	0
78366f6083SPeter Grehan 
79366f6083SPeter Grehan #define PROCBASED_CTLS_WINDOW_SETTING					\
80366f6083SPeter Grehan 	(PROCBASED_INT_WINDOW_EXITING	|				\
81366f6083SPeter Grehan 	 PROCBASED_NMI_WINDOW_EXITING)
82366f6083SPeter Grehan 
83366f6083SPeter Grehan #define	PROCBASED_CTLS_ONE_SETTING 					\
84366f6083SPeter Grehan 	(PROCBASED_SECONDARY_CONTROLS	|				\
8565145c7fSNeel Natu 	 PROCBASED_MWAIT_EXITING	|				\
8665145c7fSNeel Natu 	 PROCBASED_MONITOR_EXITING	|				\
87366f6083SPeter Grehan 	 PROCBASED_IO_EXITING		|				\
88366f6083SPeter Grehan 	 PROCBASED_MSR_BITMAPS		|				\
89594db002STycho Nightingale 	 PROCBASED_CTLS_WINDOW_SETTING	|				\
90594db002STycho Nightingale 	 PROCBASED_CR8_LOAD_EXITING	|				\
91594db002STycho Nightingale 	 PROCBASED_CR8_STORE_EXITING)
92366f6083SPeter Grehan #define	PROCBASED_CTLS_ZERO_SETTING	\
93366f6083SPeter Grehan 	(PROCBASED_CR3_LOAD_EXITING |	\
94366f6083SPeter Grehan 	PROCBASED_CR3_STORE_EXITING |	\
95366f6083SPeter Grehan 	PROCBASED_IO_BITMAPS)
96366f6083SPeter Grehan 
97366f6083SPeter Grehan #define	PROCBASED_CTLS2_ONE_SETTING	PROCBASED2_ENABLE_EPT
98366f6083SPeter Grehan #define	PROCBASED_CTLS2_ZERO_SETTING	0
99366f6083SPeter Grehan 
100d72978ecSNeel Natu #define	VM_EXIT_CTLS_ONE_SETTING					\
101366f6083SPeter Grehan 	(VM_EXIT_HOST_LMA			|			\
102366f6083SPeter Grehan 	VM_EXIT_SAVE_EFER			|			\
103d72978ecSNeel Natu 	VM_EXIT_LOAD_EFER			|			\
104a318f7ddSNeel Natu 	VM_EXIT_ACKNOWLEDGE_INTERRUPT)
105d72978ecSNeel Natu 
106366f6083SPeter Grehan #define	VM_EXIT_CTLS_ZERO_SETTING	VM_EXIT_SAVE_DEBUG_CONTROLS
107366f6083SPeter Grehan 
108a318f7ddSNeel Natu #define	VM_ENTRY_CTLS_ONE_SETTING	(VM_ENTRY_LOAD_EFER)
109608f97c3SPeter Grehan 
110366f6083SPeter Grehan #define	VM_ENTRY_CTLS_ZERO_SETTING					\
111366f6083SPeter Grehan 	(VM_ENTRY_LOAD_DEBUG_CONTROLS		|			\
112366f6083SPeter Grehan 	VM_ENTRY_INTO_SMM			|			\
113366f6083SPeter Grehan 	VM_ENTRY_DEACTIVATE_DUAL_MONITOR)
114366f6083SPeter Grehan 
115366f6083SPeter Grehan #define	HANDLED		1
116366f6083SPeter Grehan #define	UNHANDLED	0
117366f6083SPeter Grehan 
118de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VMX, "vmx", "vmx");
119de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
120366f6083SPeter Grehan 
1213565b59eSNeel Natu SYSCTL_DECL(_hw_vmm);
1223565b59eSNeel Natu SYSCTL_NODE(_hw_vmm, OID_AUTO, vmx, CTLFLAG_RW, NULL, NULL);
1233565b59eSNeel Natu 
124b3996dd4SJohn Baldwin int vmxon_enabled[MAXCPU];
125366f6083SPeter Grehan static char vmxon_region[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE);
126366f6083SPeter Grehan 
127366f6083SPeter Grehan static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2;
128366f6083SPeter Grehan static uint32_t exit_ctls, entry_ctls;
129366f6083SPeter Grehan 
130366f6083SPeter Grehan static uint64_t cr0_ones_mask, cr0_zeros_mask;
1313565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_ones_mask, CTLFLAG_RD,
1323565b59eSNeel Natu 	     &cr0_ones_mask, 0, NULL);
1333565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_zeros_mask, CTLFLAG_RD,
1343565b59eSNeel Natu 	     &cr0_zeros_mask, 0, NULL);
1353565b59eSNeel Natu 
136366f6083SPeter Grehan static uint64_t cr4_ones_mask, cr4_zeros_mask;
1373565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_ones_mask, CTLFLAG_RD,
1383565b59eSNeel Natu 	     &cr4_ones_mask, 0, NULL);
1393565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_zeros_mask, CTLFLAG_RD,
1403565b59eSNeel Natu 	     &cr4_zeros_mask, 0, NULL);
141366f6083SPeter Grehan 
1423565b59eSNeel Natu static int vmx_initialized;
1433565b59eSNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, initialized, CTLFLAG_RD,
1443565b59eSNeel Natu 	   &vmx_initialized, 0, "Intel VMX initialized");
1453565b59eSNeel Natu 
146366f6083SPeter Grehan /*
147366f6083SPeter Grehan  * Optional capabilities
148366f6083SPeter Grehan  */
14906fc6db9SJohn Baldwin static SYSCTL_NODE(_hw_vmm_vmx, OID_AUTO, cap, CTLFLAG_RW, NULL, NULL);
15006fc6db9SJohn Baldwin 
151366f6083SPeter Grehan static int cap_halt_exit;
15206fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, halt_exit, CTLFLAG_RD, &cap_halt_exit, 0,
15306fc6db9SJohn Baldwin     "HLT triggers a VM-exit");
15406fc6db9SJohn Baldwin 
155366f6083SPeter Grehan static int cap_pause_exit;
15606fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, pause_exit, CTLFLAG_RD, &cap_pause_exit,
15706fc6db9SJohn Baldwin     0, "PAUSE triggers a VM-exit");
15806fc6db9SJohn Baldwin 
159366f6083SPeter Grehan static int cap_unrestricted_guest;
16006fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, unrestricted_guest, CTLFLAG_RD,
16106fc6db9SJohn Baldwin     &cap_unrestricted_guest, 0, "Unrestricted guests");
16206fc6db9SJohn Baldwin 
163366f6083SPeter Grehan static int cap_monitor_trap;
16406fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, monitor_trap, CTLFLAG_RD,
16506fc6db9SJohn Baldwin     &cap_monitor_trap, 0, "Monitor trap flag");
16606fc6db9SJohn Baldwin 
16749cc03daSNeel Natu static int cap_invpcid;
16806fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, invpcid, CTLFLAG_RD, &cap_invpcid,
16906fc6db9SJohn Baldwin     0, "Guests are allowed to use INVPCID");
170366f6083SPeter Grehan 
17188c4b8d1SNeel Natu static int virtual_interrupt_delivery;
17206fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, virtual_interrupt_delivery, CTLFLAG_RD,
17388c4b8d1SNeel Natu     &virtual_interrupt_delivery, 0, "APICv virtual interrupt delivery support");
17488c4b8d1SNeel Natu 
175176666c2SNeel Natu static int posted_interrupts;
17606fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, posted_interrupts, CTLFLAG_RD,
177176666c2SNeel Natu     &posted_interrupts, 0, "APICv posted interrupt support");
178176666c2SNeel Natu 
17918a2b08eSNeel Natu static int pirvec = -1;
180176666c2SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupt_vector, CTLFLAG_RD,
181176666c2SNeel Natu     &pirvec, 0, "APICv posted interrupt vector");
182176666c2SNeel Natu 
18345e51299SNeel Natu static struct unrhdr *vpid_unr;
18445e51299SNeel Natu static u_int vpid_alloc_failed;
18545e51299SNeel Natu SYSCTL_UINT(_hw_vmm_vmx, OID_AUTO, vpid_alloc_failed, CTLFLAG_RD,
18645e51299SNeel Natu 	    &vpid_alloc_failed, 0, NULL);
18745e51299SNeel Natu 
18888c4b8d1SNeel Natu /*
18988c4b8d1SNeel Natu  * Use the last page below 4GB as the APIC access address. This address is
19088c4b8d1SNeel Natu  * occupied by the boot firmware so it is guaranteed that it will not conflict
19188c4b8d1SNeel Natu  * with a page in system memory.
19288c4b8d1SNeel Natu  */
19388c4b8d1SNeel Natu #define	APIC_ACCESS_ADDRESS	0xFFFFF000
19488c4b8d1SNeel Natu 
195d17b5104SNeel Natu static int vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc);
196d17b5104SNeel Natu static int vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval);
197c3498942SNeel Natu static int vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val);
19888c4b8d1SNeel Natu static void vmx_inject_pir(struct vlapic *vlapic);
19988c4b8d1SNeel Natu 
200366f6083SPeter Grehan #ifdef KTR
201366f6083SPeter Grehan static const char *
202366f6083SPeter Grehan exit_reason_to_str(int reason)
203366f6083SPeter Grehan {
204366f6083SPeter Grehan 	static char reasonbuf[32];
205366f6083SPeter Grehan 
206366f6083SPeter Grehan 	switch (reason) {
207366f6083SPeter Grehan 	case EXIT_REASON_EXCEPTION:
208366f6083SPeter Grehan 		return "exception";
209366f6083SPeter Grehan 	case EXIT_REASON_EXT_INTR:
210366f6083SPeter Grehan 		return "extint";
211366f6083SPeter Grehan 	case EXIT_REASON_TRIPLE_FAULT:
212366f6083SPeter Grehan 		return "triplefault";
213366f6083SPeter Grehan 	case EXIT_REASON_INIT:
214366f6083SPeter Grehan 		return "init";
215366f6083SPeter Grehan 	case EXIT_REASON_SIPI:
216366f6083SPeter Grehan 		return "sipi";
217366f6083SPeter Grehan 	case EXIT_REASON_IO_SMI:
218366f6083SPeter Grehan 		return "iosmi";
219366f6083SPeter Grehan 	case EXIT_REASON_SMI:
220366f6083SPeter Grehan 		return "smi";
221366f6083SPeter Grehan 	case EXIT_REASON_INTR_WINDOW:
222366f6083SPeter Grehan 		return "intrwindow";
223366f6083SPeter Grehan 	case EXIT_REASON_NMI_WINDOW:
224366f6083SPeter Grehan 		return "nmiwindow";
225366f6083SPeter Grehan 	case EXIT_REASON_TASK_SWITCH:
226366f6083SPeter Grehan 		return "taskswitch";
227366f6083SPeter Grehan 	case EXIT_REASON_CPUID:
228366f6083SPeter Grehan 		return "cpuid";
229366f6083SPeter Grehan 	case EXIT_REASON_GETSEC:
230366f6083SPeter Grehan 		return "getsec";
231366f6083SPeter Grehan 	case EXIT_REASON_HLT:
232366f6083SPeter Grehan 		return "hlt";
233366f6083SPeter Grehan 	case EXIT_REASON_INVD:
234366f6083SPeter Grehan 		return "invd";
235366f6083SPeter Grehan 	case EXIT_REASON_INVLPG:
236366f6083SPeter Grehan 		return "invlpg";
237366f6083SPeter Grehan 	case EXIT_REASON_RDPMC:
238366f6083SPeter Grehan 		return "rdpmc";
239366f6083SPeter Grehan 	case EXIT_REASON_RDTSC:
240366f6083SPeter Grehan 		return "rdtsc";
241366f6083SPeter Grehan 	case EXIT_REASON_RSM:
242366f6083SPeter Grehan 		return "rsm";
243366f6083SPeter Grehan 	case EXIT_REASON_VMCALL:
244366f6083SPeter Grehan 		return "vmcall";
245366f6083SPeter Grehan 	case EXIT_REASON_VMCLEAR:
246366f6083SPeter Grehan 		return "vmclear";
247366f6083SPeter Grehan 	case EXIT_REASON_VMLAUNCH:
248366f6083SPeter Grehan 		return "vmlaunch";
249366f6083SPeter Grehan 	case EXIT_REASON_VMPTRLD:
250366f6083SPeter Grehan 		return "vmptrld";
251366f6083SPeter Grehan 	case EXIT_REASON_VMPTRST:
252366f6083SPeter Grehan 		return "vmptrst";
253366f6083SPeter Grehan 	case EXIT_REASON_VMREAD:
254366f6083SPeter Grehan 		return "vmread";
255366f6083SPeter Grehan 	case EXIT_REASON_VMRESUME:
256366f6083SPeter Grehan 		return "vmresume";
257366f6083SPeter Grehan 	case EXIT_REASON_VMWRITE:
258366f6083SPeter Grehan 		return "vmwrite";
259366f6083SPeter Grehan 	case EXIT_REASON_VMXOFF:
260366f6083SPeter Grehan 		return "vmxoff";
261366f6083SPeter Grehan 	case EXIT_REASON_VMXON:
262366f6083SPeter Grehan 		return "vmxon";
263366f6083SPeter Grehan 	case EXIT_REASON_CR_ACCESS:
264366f6083SPeter Grehan 		return "craccess";
265366f6083SPeter Grehan 	case EXIT_REASON_DR_ACCESS:
266366f6083SPeter Grehan 		return "draccess";
267366f6083SPeter Grehan 	case EXIT_REASON_INOUT:
268366f6083SPeter Grehan 		return "inout";
269366f6083SPeter Grehan 	case EXIT_REASON_RDMSR:
270366f6083SPeter Grehan 		return "rdmsr";
271366f6083SPeter Grehan 	case EXIT_REASON_WRMSR:
272366f6083SPeter Grehan 		return "wrmsr";
273366f6083SPeter Grehan 	case EXIT_REASON_INVAL_VMCS:
274366f6083SPeter Grehan 		return "invalvmcs";
275366f6083SPeter Grehan 	case EXIT_REASON_INVAL_MSR:
276366f6083SPeter Grehan 		return "invalmsr";
277366f6083SPeter Grehan 	case EXIT_REASON_MWAIT:
278366f6083SPeter Grehan 		return "mwait";
279366f6083SPeter Grehan 	case EXIT_REASON_MTF:
280366f6083SPeter Grehan 		return "mtf";
281366f6083SPeter Grehan 	case EXIT_REASON_MONITOR:
282366f6083SPeter Grehan 		return "monitor";
283366f6083SPeter Grehan 	case EXIT_REASON_PAUSE:
284366f6083SPeter Grehan 		return "pause";
285b0538143SNeel Natu 	case EXIT_REASON_MCE_DURING_ENTRY:
286b0538143SNeel Natu 		return "mce-during-entry";
287366f6083SPeter Grehan 	case EXIT_REASON_TPR:
288366f6083SPeter Grehan 		return "tpr";
28988c4b8d1SNeel Natu 	case EXIT_REASON_APIC_ACCESS:
29088c4b8d1SNeel Natu 		return "apic-access";
291366f6083SPeter Grehan 	case EXIT_REASON_GDTR_IDTR:
292366f6083SPeter Grehan 		return "gdtridtr";
293366f6083SPeter Grehan 	case EXIT_REASON_LDTR_TR:
294366f6083SPeter Grehan 		return "ldtrtr";
295366f6083SPeter Grehan 	case EXIT_REASON_EPT_FAULT:
296366f6083SPeter Grehan 		return "eptfault";
297366f6083SPeter Grehan 	case EXIT_REASON_EPT_MISCONFIG:
298366f6083SPeter Grehan 		return "eptmisconfig";
299366f6083SPeter Grehan 	case EXIT_REASON_INVEPT:
300366f6083SPeter Grehan 		return "invept";
301366f6083SPeter Grehan 	case EXIT_REASON_RDTSCP:
302366f6083SPeter Grehan 		return "rdtscp";
303366f6083SPeter Grehan 	case EXIT_REASON_VMX_PREEMPT:
304366f6083SPeter Grehan 		return "vmxpreempt";
305366f6083SPeter Grehan 	case EXIT_REASON_INVVPID:
306366f6083SPeter Grehan 		return "invvpid";
307366f6083SPeter Grehan 	case EXIT_REASON_WBINVD:
308366f6083SPeter Grehan 		return "wbinvd";
309366f6083SPeter Grehan 	case EXIT_REASON_XSETBV:
310366f6083SPeter Grehan 		return "xsetbv";
31188c4b8d1SNeel Natu 	case EXIT_REASON_APIC_WRITE:
31288c4b8d1SNeel Natu 		return "apic-write";
313366f6083SPeter Grehan 	default:
314366f6083SPeter Grehan 		snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason);
315366f6083SPeter Grehan 		return (reasonbuf);
316366f6083SPeter Grehan 	}
317366f6083SPeter Grehan }
318366f6083SPeter Grehan #endif	/* KTR */
319366f6083SPeter Grehan 
320159dd56fSNeel Natu static int
321159dd56fSNeel Natu vmx_allow_x2apic_msrs(struct vmx *vmx)
322159dd56fSNeel Natu {
323159dd56fSNeel Natu 	int i, error;
324159dd56fSNeel Natu 
325159dd56fSNeel Natu 	error = 0;
326159dd56fSNeel Natu 
327159dd56fSNeel Natu 	/*
328159dd56fSNeel Natu 	 * Allow readonly access to the following x2APIC MSRs from the guest.
329159dd56fSNeel Natu 	 */
330159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ID);
331159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_VERSION);
332159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LDR);
333159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_SVR);
334159dd56fSNeel Natu 
335159dd56fSNeel Natu 	for (i = 0; i < 8; i++)
336159dd56fSNeel Natu 		error += guest_msr_ro(vmx, MSR_APIC_ISR0 + i);
337159dd56fSNeel Natu 
338159dd56fSNeel Natu 	for (i = 0; i < 8; i++)
339159dd56fSNeel Natu 		error += guest_msr_ro(vmx, MSR_APIC_TMR0 + i);
340159dd56fSNeel Natu 
341159dd56fSNeel Natu 	for (i = 0; i < 8; i++)
342159dd56fSNeel Natu 		error += guest_msr_ro(vmx, MSR_APIC_IRR0 + i);
343159dd56fSNeel Natu 
344159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ESR);
345159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_TIMER);
346159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_THERMAL);
347159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_PCINT);
348159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT0);
349159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT1);
350159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_ERROR);
351159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ICR_TIMER);
352159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_DCR_TIMER);
353159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ICR);
354159dd56fSNeel Natu 
355159dd56fSNeel Natu 	/*
356159dd56fSNeel Natu 	 * Allow TPR, EOI and SELF_IPI MSRs to be read and written by the guest.
357159dd56fSNeel Natu 	 *
358159dd56fSNeel Natu 	 * These registers get special treatment described in the section
359159dd56fSNeel Natu 	 * "Virtualizing MSR-Based APIC Accesses".
360159dd56fSNeel Natu 	 */
361159dd56fSNeel Natu 	error += guest_msr_rw(vmx, MSR_APIC_TPR);
362159dd56fSNeel Natu 	error += guest_msr_rw(vmx, MSR_APIC_EOI);
363159dd56fSNeel Natu 	error += guest_msr_rw(vmx, MSR_APIC_SELF_IPI);
364159dd56fSNeel Natu 
365159dd56fSNeel Natu 	return (error);
366159dd56fSNeel Natu }
367159dd56fSNeel Natu 
368366f6083SPeter Grehan u_long
369366f6083SPeter Grehan vmx_fix_cr0(u_long cr0)
370366f6083SPeter Grehan {
371366f6083SPeter Grehan 
372366f6083SPeter Grehan 	return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask);
373366f6083SPeter Grehan }
374366f6083SPeter Grehan 
375366f6083SPeter Grehan u_long
376366f6083SPeter Grehan vmx_fix_cr4(u_long cr4)
377366f6083SPeter Grehan {
378366f6083SPeter Grehan 
379366f6083SPeter Grehan 	return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask);
380366f6083SPeter Grehan }
381366f6083SPeter Grehan 
382366f6083SPeter Grehan static void
38345e51299SNeel Natu vpid_free(int vpid)
38445e51299SNeel Natu {
38545e51299SNeel Natu 	if (vpid < 0 || vpid > 0xffff)
38645e51299SNeel Natu 		panic("vpid_free: invalid vpid %d", vpid);
38745e51299SNeel Natu 
38845e51299SNeel Natu 	/*
38945e51299SNeel Natu 	 * VPIDs [0,VM_MAXCPU] are special and are not allocated from
39045e51299SNeel Natu 	 * the unit number allocator.
39145e51299SNeel Natu 	 */
39245e51299SNeel Natu 
39345e51299SNeel Natu 	if (vpid > VM_MAXCPU)
39445e51299SNeel Natu 		free_unr(vpid_unr, vpid);
39545e51299SNeel Natu }
39645e51299SNeel Natu 
39745e51299SNeel Natu static void
39845e51299SNeel Natu vpid_alloc(uint16_t *vpid, int num)
39945e51299SNeel Natu {
40045e51299SNeel Natu 	int i, x;
40145e51299SNeel Natu 
40245e51299SNeel Natu 	if (num <= 0 || num > VM_MAXCPU)
40345e51299SNeel Natu 		panic("invalid number of vpids requested: %d", num);
40445e51299SNeel Natu 
40545e51299SNeel Natu 	/*
40645e51299SNeel Natu 	 * If the "enable vpid" execution control is not enabled then the
40745e51299SNeel Natu 	 * VPID is required to be 0 for all vcpus.
40845e51299SNeel Natu 	 */
40945e51299SNeel Natu 	if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) == 0) {
41045e51299SNeel Natu 		for (i = 0; i < num; i++)
41145e51299SNeel Natu 			vpid[i] = 0;
41245e51299SNeel Natu 		return;
41345e51299SNeel Natu 	}
41445e51299SNeel Natu 
41545e51299SNeel Natu 	/*
41645e51299SNeel Natu 	 * Allocate a unique VPID for each vcpu from the unit number allocator.
41745e51299SNeel Natu 	 */
41845e51299SNeel Natu 	for (i = 0; i < num; i++) {
41945e51299SNeel Natu 		x = alloc_unr(vpid_unr);
42045e51299SNeel Natu 		if (x == -1)
42145e51299SNeel Natu 			break;
42245e51299SNeel Natu 		else
42345e51299SNeel Natu 			vpid[i] = x;
42445e51299SNeel Natu 	}
42545e51299SNeel Natu 
42645e51299SNeel Natu 	if (i < num) {
42745e51299SNeel Natu 		atomic_add_int(&vpid_alloc_failed, 1);
42845e51299SNeel Natu 
42945e51299SNeel Natu 		/*
43045e51299SNeel Natu 		 * If the unit number allocator does not have enough unique
43145e51299SNeel Natu 		 * VPIDs then we need to allocate from the [1,VM_MAXCPU] range.
43245e51299SNeel Natu 		 *
43345e51299SNeel Natu 		 * These VPIDs are not be unique across VMs but this does not
43445e51299SNeel Natu 		 * affect correctness because the combined mappings are also
43545e51299SNeel Natu 		 * tagged with the EP4TA which is unique for each VM.
43645e51299SNeel Natu 		 *
43745e51299SNeel Natu 		 * It is still sub-optimal because the invvpid will invalidate
43845e51299SNeel Natu 		 * combined mappings for a particular VPID across all EP4TAs.
43945e51299SNeel Natu 		 */
44045e51299SNeel Natu 		while (i-- > 0)
44145e51299SNeel Natu 			vpid_free(vpid[i]);
44245e51299SNeel Natu 
44345e51299SNeel Natu 		for (i = 0; i < num; i++)
44445e51299SNeel Natu 			vpid[i] = i + 1;
44545e51299SNeel Natu 	}
44645e51299SNeel Natu }
44745e51299SNeel Natu 
44845e51299SNeel Natu static void
44945e51299SNeel Natu vpid_init(void)
45045e51299SNeel Natu {
45145e51299SNeel Natu 	/*
45245e51299SNeel Natu 	 * VPID 0 is required when the "enable VPID" execution control is
45345e51299SNeel Natu 	 * disabled.
45445e51299SNeel Natu 	 *
45545e51299SNeel Natu 	 * VPIDs [1,VM_MAXCPU] are used as the "overflow namespace" when the
45645e51299SNeel Natu 	 * unit number allocator does not have sufficient unique VPIDs to
45745e51299SNeel Natu 	 * satisfy the allocation.
45845e51299SNeel Natu 	 *
45945e51299SNeel Natu 	 * The remaining VPIDs are managed by the unit number allocator.
46045e51299SNeel Natu 	 */
46145e51299SNeel Natu 	vpid_unr = new_unrhdr(VM_MAXCPU + 1, 0xffff, NULL);
46245e51299SNeel Natu }
46345e51299SNeel Natu 
46445e51299SNeel Natu static void
465366f6083SPeter Grehan vmx_disable(void *arg __unused)
466366f6083SPeter Grehan {
467366f6083SPeter Grehan 	struct invvpid_desc invvpid_desc = { 0 };
468366f6083SPeter Grehan 	struct invept_desc invept_desc = { 0 };
469366f6083SPeter Grehan 
470366f6083SPeter Grehan 	if (vmxon_enabled[curcpu]) {
471366f6083SPeter Grehan 		/*
472366f6083SPeter Grehan 		 * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b.
473366f6083SPeter Grehan 		 *
474366f6083SPeter Grehan 		 * VMXON or VMXOFF are not required to invalidate any TLB
475366f6083SPeter Grehan 		 * caching structures. This prevents potential retention of
476366f6083SPeter Grehan 		 * cached information in the TLB between distinct VMX episodes.
477366f6083SPeter Grehan 		 */
478366f6083SPeter Grehan 		invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc);
479366f6083SPeter Grehan 		invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc);
480366f6083SPeter Grehan 		vmxoff();
481366f6083SPeter Grehan 	}
482366f6083SPeter Grehan 	load_cr4(rcr4() & ~CR4_VMXE);
483366f6083SPeter Grehan }
484366f6083SPeter Grehan 
485366f6083SPeter Grehan static int
486366f6083SPeter Grehan vmx_cleanup(void)
487366f6083SPeter Grehan {
488366f6083SPeter Grehan 
48918a2b08eSNeel Natu 	if (pirvec >= 0)
49018a2b08eSNeel Natu 		lapic_ipi_free(pirvec);
491176666c2SNeel Natu 
49245e51299SNeel Natu 	if (vpid_unr != NULL) {
49345e51299SNeel Natu 		delete_unrhdr(vpid_unr);
49445e51299SNeel Natu 		vpid_unr = NULL;
49545e51299SNeel Natu 	}
49645e51299SNeel Natu 
497366f6083SPeter Grehan 	smp_rendezvous(NULL, vmx_disable, NULL, NULL);
498366f6083SPeter Grehan 
499366f6083SPeter Grehan 	return (0);
500366f6083SPeter Grehan }
501366f6083SPeter Grehan 
502366f6083SPeter Grehan static void
503366f6083SPeter Grehan vmx_enable(void *arg __unused)
504366f6083SPeter Grehan {
505366f6083SPeter Grehan 	int error;
50611669a68STycho Nightingale 	uint64_t feature_control;
50711669a68STycho Nightingale 
50811669a68STycho Nightingale 	feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
50911669a68STycho Nightingale 	if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 0 ||
51011669a68STycho Nightingale 	    (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) {
51111669a68STycho Nightingale 		wrmsr(MSR_IA32_FEATURE_CONTROL,
51211669a68STycho Nightingale 		    feature_control | IA32_FEATURE_CONTROL_VMX_EN |
51311669a68STycho Nightingale 		    IA32_FEATURE_CONTROL_LOCK);
51411669a68STycho Nightingale 	}
515366f6083SPeter Grehan 
516366f6083SPeter Grehan 	load_cr4(rcr4() | CR4_VMXE);
517366f6083SPeter Grehan 
518366f6083SPeter Grehan 	*(uint32_t *)vmxon_region[curcpu] = vmx_revision();
519366f6083SPeter Grehan 	error = vmxon(vmxon_region[curcpu]);
520366f6083SPeter Grehan 	if (error == 0)
521366f6083SPeter Grehan 		vmxon_enabled[curcpu] = 1;
522366f6083SPeter Grehan }
523366f6083SPeter Grehan 
52463e62d39SJohn Baldwin static void
52563e62d39SJohn Baldwin vmx_restore(void)
52663e62d39SJohn Baldwin {
52763e62d39SJohn Baldwin 
52863e62d39SJohn Baldwin 	if (vmxon_enabled[curcpu])
52963e62d39SJohn Baldwin 		vmxon(vmxon_region[curcpu]);
53063e62d39SJohn Baldwin }
53163e62d39SJohn Baldwin 
532366f6083SPeter Grehan static int
533add611fdSNeel Natu vmx_init(int ipinum)
534366f6083SPeter Grehan {
53588c4b8d1SNeel Natu 	int error, use_tpr_shadow;
536d17b5104SNeel Natu 	uint64_t basic, fixed0, fixed1, feature_control;
53788c4b8d1SNeel Natu 	uint32_t tmp, procbased2_vid_bits;
538366f6083SPeter Grehan 
539366f6083SPeter Grehan 	/* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */
5408b287612SJohn Baldwin 	if (!(cpu_feature2 & CPUID2_VMX)) {
541366f6083SPeter Grehan 		printf("vmx_init: processor does not support VMX operation\n");
542366f6083SPeter Grehan 		return (ENXIO);
543366f6083SPeter Grehan 	}
544366f6083SPeter Grehan 
5454bff7fadSNeel Natu 	/*
5464bff7fadSNeel Natu 	 * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits
5474bff7fadSNeel Natu 	 * are set (bits 0 and 2 respectively).
5484bff7fadSNeel Natu 	 */
5494bff7fadSNeel Natu 	feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
55011669a68STycho Nightingale 	if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 1 &&
551150369abSNeel Natu 	    (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) {
5524bff7fadSNeel Natu 		printf("vmx_init: VMX operation disabled by BIOS\n");
5534bff7fadSNeel Natu 		return (ENXIO);
5544bff7fadSNeel Natu 	}
5554bff7fadSNeel Natu 
556d17b5104SNeel Natu 	/*
557d17b5104SNeel Natu 	 * Verify capabilities MSR_VMX_BASIC:
558d17b5104SNeel Natu 	 * - bit 54 indicates support for INS/OUTS decoding
559d17b5104SNeel Natu 	 */
560d17b5104SNeel Natu 	basic = rdmsr(MSR_VMX_BASIC);
561d17b5104SNeel Natu 	if ((basic & (1UL << 54)) == 0) {
562d17b5104SNeel Natu 		printf("vmx_init: processor does not support desired basic "
563d17b5104SNeel Natu 		    "capabilities\n");
564d17b5104SNeel Natu 		return (EINVAL);
565d17b5104SNeel Natu 	}
566d17b5104SNeel Natu 
567366f6083SPeter Grehan 	/* Check support for primary processor-based VM-execution controls */
568366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
569366f6083SPeter Grehan 			       MSR_VMX_TRUE_PROCBASED_CTLS,
570366f6083SPeter Grehan 			       PROCBASED_CTLS_ONE_SETTING,
571366f6083SPeter Grehan 			       PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls);
572366f6083SPeter Grehan 	if (error) {
573366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired primary "
574366f6083SPeter Grehan 		       "processor-based controls\n");
575366f6083SPeter Grehan 		return (error);
576366f6083SPeter Grehan 	}
577366f6083SPeter Grehan 
578366f6083SPeter Grehan 	/* Clear the processor-based ctl bits that are set on demand */
579366f6083SPeter Grehan 	procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING;
580366f6083SPeter Grehan 
581366f6083SPeter Grehan 	/* Check support for secondary processor-based VM-execution controls */
582366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
583366f6083SPeter Grehan 			       MSR_VMX_PROCBASED_CTLS2,
584366f6083SPeter Grehan 			       PROCBASED_CTLS2_ONE_SETTING,
585366f6083SPeter Grehan 			       PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2);
586366f6083SPeter Grehan 	if (error) {
587366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired secondary "
588366f6083SPeter Grehan 		       "processor-based controls\n");
589366f6083SPeter Grehan 		return (error);
590366f6083SPeter Grehan 	}
591366f6083SPeter Grehan 
592366f6083SPeter Grehan 	/* Check support for VPID */
593366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
594366f6083SPeter Grehan 			       PROCBASED2_ENABLE_VPID, 0, &tmp);
595366f6083SPeter Grehan 	if (error == 0)
596366f6083SPeter Grehan 		procbased_ctls2 |= PROCBASED2_ENABLE_VPID;
597366f6083SPeter Grehan 
598366f6083SPeter Grehan 	/* Check support for pin-based VM-execution controls */
599366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
600366f6083SPeter Grehan 			       MSR_VMX_TRUE_PINBASED_CTLS,
601366f6083SPeter Grehan 			       PINBASED_CTLS_ONE_SETTING,
602366f6083SPeter Grehan 			       PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls);
603366f6083SPeter Grehan 	if (error) {
604366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired "
605366f6083SPeter Grehan 		       "pin-based controls\n");
606366f6083SPeter Grehan 		return (error);
607366f6083SPeter Grehan 	}
608366f6083SPeter Grehan 
609366f6083SPeter Grehan 	/* Check support for VM-exit controls */
610366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS,
611366f6083SPeter Grehan 			       VM_EXIT_CTLS_ONE_SETTING,
612366f6083SPeter Grehan 			       VM_EXIT_CTLS_ZERO_SETTING,
613366f6083SPeter Grehan 			       &exit_ctls);
614366f6083SPeter Grehan 	if (error) {
615366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired "
616366f6083SPeter Grehan 		    "exit controls\n");
617366f6083SPeter Grehan 		return (error);
618366f6083SPeter Grehan 	}
619366f6083SPeter Grehan 
620366f6083SPeter Grehan 	/* Check support for VM-entry controls */
621d72978ecSNeel Natu 	error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS, MSR_VMX_TRUE_ENTRY_CTLS,
622d72978ecSNeel Natu 	    VM_ENTRY_CTLS_ONE_SETTING, VM_ENTRY_CTLS_ZERO_SETTING,
623366f6083SPeter Grehan 	    &entry_ctls);
624366f6083SPeter Grehan 	if (error) {
625366f6083SPeter Grehan 		printf("vmx_init: processor does not support desired "
626366f6083SPeter Grehan 		    "entry controls\n");
627366f6083SPeter Grehan 		return (error);
628366f6083SPeter Grehan 	}
629366f6083SPeter Grehan 
630366f6083SPeter Grehan 	/*
631366f6083SPeter Grehan 	 * Check support for optional features by testing them
632366f6083SPeter Grehan 	 * as individual bits
633366f6083SPeter Grehan 	 */
634366f6083SPeter Grehan 	cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
635366f6083SPeter Grehan 					MSR_VMX_TRUE_PROCBASED_CTLS,
636366f6083SPeter Grehan 					PROCBASED_HLT_EXITING, 0,
637366f6083SPeter Grehan 					&tmp) == 0);
638366f6083SPeter Grehan 
639366f6083SPeter Grehan 	cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
640366f6083SPeter Grehan 					MSR_VMX_PROCBASED_CTLS,
641366f6083SPeter Grehan 					PROCBASED_MTF, 0,
642366f6083SPeter Grehan 					&tmp) == 0);
643366f6083SPeter Grehan 
644366f6083SPeter Grehan 	cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
645366f6083SPeter Grehan 					 MSR_VMX_TRUE_PROCBASED_CTLS,
646366f6083SPeter Grehan 					 PROCBASED_PAUSE_EXITING, 0,
647366f6083SPeter Grehan 					 &tmp) == 0);
648366f6083SPeter Grehan 
649366f6083SPeter Grehan 	cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
650366f6083SPeter Grehan 					MSR_VMX_PROCBASED_CTLS2,
651366f6083SPeter Grehan 					PROCBASED2_UNRESTRICTED_GUEST, 0,
652366f6083SPeter Grehan 				        &tmp) == 0);
653366f6083SPeter Grehan 
65449cc03daSNeel Natu 	cap_invpcid = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
65549cc03daSNeel Natu 	    MSR_VMX_PROCBASED_CTLS2, PROCBASED2_ENABLE_INVPCID, 0,
65649cc03daSNeel Natu 	    &tmp) == 0);
65749cc03daSNeel Natu 
65888c4b8d1SNeel Natu 	/*
65988c4b8d1SNeel Natu 	 * Check support for virtual interrupt delivery.
66088c4b8d1SNeel Natu 	 */
66188c4b8d1SNeel Natu 	procbased2_vid_bits = (PROCBASED2_VIRTUALIZE_APIC_ACCESSES |
66288c4b8d1SNeel Natu 	    PROCBASED2_VIRTUALIZE_X2APIC_MODE |
66388c4b8d1SNeel Natu 	    PROCBASED2_APIC_REGISTER_VIRTUALIZATION |
66488c4b8d1SNeel Natu 	    PROCBASED2_VIRTUAL_INTERRUPT_DELIVERY);
66588c4b8d1SNeel Natu 
66688c4b8d1SNeel Natu 	use_tpr_shadow = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
66788c4b8d1SNeel Natu 	    MSR_VMX_TRUE_PROCBASED_CTLS, PROCBASED_USE_TPR_SHADOW, 0,
66888c4b8d1SNeel Natu 	    &tmp) == 0);
66988c4b8d1SNeel Natu 
67088c4b8d1SNeel Natu 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
67188c4b8d1SNeel Natu 	    procbased2_vid_bits, 0, &tmp);
67288c4b8d1SNeel Natu 	if (error == 0 && use_tpr_shadow) {
67388c4b8d1SNeel Natu 		virtual_interrupt_delivery = 1;
67488c4b8d1SNeel Natu 		TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_vid",
67588c4b8d1SNeel Natu 		    &virtual_interrupt_delivery);
67688c4b8d1SNeel Natu 	}
67788c4b8d1SNeel Natu 
67888c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
67988c4b8d1SNeel Natu 		procbased_ctls |= PROCBASED_USE_TPR_SHADOW;
68088c4b8d1SNeel Natu 		procbased_ctls2 |= procbased2_vid_bits;
68188c4b8d1SNeel Natu 		procbased_ctls2 &= ~PROCBASED2_VIRTUALIZE_X2APIC_MODE;
682176666c2SNeel Natu 
683176666c2SNeel Natu 		/*
684594db002STycho Nightingale 		 * No need to emulate accesses to %CR8 if virtual
685594db002STycho Nightingale 		 * interrupt delivery is enabled.
686594db002STycho Nightingale 		 */
687594db002STycho Nightingale 		procbased_ctls &= ~PROCBASED_CR8_LOAD_EXITING;
688594db002STycho Nightingale 		procbased_ctls &= ~PROCBASED_CR8_STORE_EXITING;
689594db002STycho Nightingale 
690594db002STycho Nightingale 		/*
691176666c2SNeel Natu 		 * Check for Posted Interrupts only if Virtual Interrupt
692176666c2SNeel Natu 		 * Delivery is enabled.
693176666c2SNeel Natu 		 */
694176666c2SNeel Natu 		error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
695176666c2SNeel Natu 		    MSR_VMX_TRUE_PINBASED_CTLS, PINBASED_POSTED_INTERRUPT, 0,
696176666c2SNeel Natu 		    &tmp);
697176666c2SNeel Natu 		if (error == 0) {
698*bd50262fSKonstantin Belousov 			pirvec = lapic_ipi_alloc(pti ? &IDTVEC(justreturn1_pti) :
699*bd50262fSKonstantin Belousov 			    &IDTVEC(justreturn));
70018a2b08eSNeel Natu 			if (pirvec < 0) {
701176666c2SNeel Natu 				if (bootverbose) {
702176666c2SNeel Natu 					printf("vmx_init: unable to allocate "
703176666c2SNeel Natu 					    "posted interrupt vector\n");
70488c4b8d1SNeel Natu 				}
705176666c2SNeel Natu 			} else {
706176666c2SNeel Natu 				posted_interrupts = 1;
707176666c2SNeel Natu 				TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_pir",
708176666c2SNeel Natu 				    &posted_interrupts);
709176666c2SNeel Natu 			}
710176666c2SNeel Natu 		}
711176666c2SNeel Natu 	}
712176666c2SNeel Natu 
713176666c2SNeel Natu 	if (posted_interrupts)
714176666c2SNeel Natu 		    pinbased_ctls |= PINBASED_POSTED_INTERRUPT;
71549cc03daSNeel Natu 
716366f6083SPeter Grehan 	/* Initialize EPT */
717add611fdSNeel Natu 	error = ept_init(ipinum);
718366f6083SPeter Grehan 	if (error) {
719366f6083SPeter Grehan 		printf("vmx_init: ept initialization failed (%d)\n", error);
720366f6083SPeter Grehan 		return (error);
721366f6083SPeter Grehan 	}
722366f6083SPeter Grehan 
723366f6083SPeter Grehan 	/*
724366f6083SPeter Grehan 	 * Stash the cr0 and cr4 bits that must be fixed to 0 or 1
725366f6083SPeter Grehan 	 */
726366f6083SPeter Grehan 	fixed0 = rdmsr(MSR_VMX_CR0_FIXED0);
727366f6083SPeter Grehan 	fixed1 = rdmsr(MSR_VMX_CR0_FIXED1);
728366f6083SPeter Grehan 	cr0_ones_mask = fixed0 & fixed1;
729366f6083SPeter Grehan 	cr0_zeros_mask = ~fixed0 & ~fixed1;
730366f6083SPeter Grehan 
731366f6083SPeter Grehan 	/*
732366f6083SPeter Grehan 	 * CR0_PE and CR0_PG can be set to zero in VMX non-root operation
733366f6083SPeter Grehan 	 * if unrestricted guest execution is allowed.
734366f6083SPeter Grehan 	 */
735366f6083SPeter Grehan 	if (cap_unrestricted_guest)
736366f6083SPeter Grehan 		cr0_ones_mask &= ~(CR0_PG | CR0_PE);
737366f6083SPeter Grehan 
738366f6083SPeter Grehan 	/*
739366f6083SPeter Grehan 	 * Do not allow the guest to set CR0_NW or CR0_CD.
740366f6083SPeter Grehan 	 */
741366f6083SPeter Grehan 	cr0_zeros_mask |= (CR0_NW | CR0_CD);
742366f6083SPeter Grehan 
743366f6083SPeter Grehan 	fixed0 = rdmsr(MSR_VMX_CR4_FIXED0);
744366f6083SPeter Grehan 	fixed1 = rdmsr(MSR_VMX_CR4_FIXED1);
745366f6083SPeter Grehan 	cr4_ones_mask = fixed0 & fixed1;
746366f6083SPeter Grehan 	cr4_zeros_mask = ~fixed0 & ~fixed1;
747366f6083SPeter Grehan 
74845e51299SNeel Natu 	vpid_init();
74945e51299SNeel Natu 
750c3498942SNeel Natu 	vmx_msr_init();
751c3498942SNeel Natu 
752366f6083SPeter Grehan 	/* enable VMX operation */
753366f6083SPeter Grehan 	smp_rendezvous(NULL, vmx_enable, NULL, NULL);
754366f6083SPeter Grehan 
7553565b59eSNeel Natu 	vmx_initialized = 1;
7563565b59eSNeel Natu 
757366f6083SPeter Grehan 	return (0);
758366f6083SPeter Grehan }
759366f6083SPeter Grehan 
760f7d47425SNeel Natu static void
761f7d47425SNeel Natu vmx_trigger_hostintr(int vector)
762f7d47425SNeel Natu {
763f7d47425SNeel Natu 	uintptr_t func;
764f7d47425SNeel Natu 	struct gate_descriptor *gd;
765f7d47425SNeel Natu 
766f7d47425SNeel Natu 	gd = &idt[vector];
767f7d47425SNeel Natu 
768f7d47425SNeel Natu 	KASSERT(vector >= 32 && vector <= 255, ("vmx_trigger_hostintr: "
769f7d47425SNeel Natu 	    "invalid vector %d", vector));
770f7d47425SNeel Natu 	KASSERT(gd->gd_p == 1, ("gate descriptor for vector %d not present",
771f7d47425SNeel Natu 	    vector));
772f7d47425SNeel Natu 	KASSERT(gd->gd_type == SDT_SYSIGT, ("gate descriptor for vector %d "
773f7d47425SNeel Natu 	    "has invalid type %d", vector, gd->gd_type));
774f7d47425SNeel Natu 	KASSERT(gd->gd_dpl == SEL_KPL, ("gate descriptor for vector %d "
775f7d47425SNeel Natu 	    "has invalid dpl %d", vector, gd->gd_dpl));
776f7d47425SNeel Natu 	KASSERT(gd->gd_selector == GSEL(GCODE_SEL, SEL_KPL), ("gate descriptor "
777f7d47425SNeel Natu 	    "for vector %d has invalid selector %d", vector, gd->gd_selector));
778f7d47425SNeel Natu 	KASSERT(gd->gd_ist == 0, ("gate descriptor for vector %d has invalid "
779f7d47425SNeel Natu 	    "IST %d", vector, gd->gd_ist));
780f7d47425SNeel Natu 
781f7d47425SNeel Natu 	func = ((long)gd->gd_hioffset << 16 | gd->gd_looffset);
782f7d47425SNeel Natu 	vmx_call_isr(func);
783f7d47425SNeel Natu }
784f7d47425SNeel Natu 
785366f6083SPeter Grehan static int
786aaaa0656SPeter Grehan vmx_setup_cr_shadow(int which, struct vmcs *vmcs, uint32_t initial)
787366f6083SPeter Grehan {
78839c21c2dSNeel Natu 	int error, mask_ident, shadow_ident;
789aaaa0656SPeter Grehan 	uint64_t mask_value;
790366f6083SPeter Grehan 
79139c21c2dSNeel Natu 	if (which != 0 && which != 4)
79239c21c2dSNeel Natu 		panic("vmx_setup_cr_shadow: unknown cr%d", which);
79339c21c2dSNeel Natu 
79439c21c2dSNeel Natu 	if (which == 0) {
79539c21c2dSNeel Natu 		mask_ident = VMCS_CR0_MASK;
79639c21c2dSNeel Natu 		mask_value = cr0_ones_mask | cr0_zeros_mask;
79739c21c2dSNeel Natu 		shadow_ident = VMCS_CR0_SHADOW;
79839c21c2dSNeel Natu 	} else {
79939c21c2dSNeel Natu 		mask_ident = VMCS_CR4_MASK;
80039c21c2dSNeel Natu 		mask_value = cr4_ones_mask | cr4_zeros_mask;
80139c21c2dSNeel Natu 		shadow_ident = VMCS_CR4_SHADOW;
80239c21c2dSNeel Natu 	}
80339c21c2dSNeel Natu 
804d3c11f40SPeter Grehan 	error = vmcs_setreg(vmcs, 0, VMCS_IDENT(mask_ident), mask_value);
805366f6083SPeter Grehan 	if (error)
806366f6083SPeter Grehan 		return (error);
807366f6083SPeter Grehan 
808aaaa0656SPeter Grehan 	error = vmcs_setreg(vmcs, 0, VMCS_IDENT(shadow_ident), initial);
809366f6083SPeter Grehan 	if (error)
810366f6083SPeter Grehan 		return (error);
811366f6083SPeter Grehan 
812366f6083SPeter Grehan 	return (0);
813366f6083SPeter Grehan }
814aaaa0656SPeter Grehan #define	vmx_setup_cr0_shadow(vmcs,init)	vmx_setup_cr_shadow(0, (vmcs), (init))
815aaaa0656SPeter Grehan #define	vmx_setup_cr4_shadow(vmcs,init)	vmx_setup_cr_shadow(4, (vmcs), (init))
816366f6083SPeter Grehan 
817366f6083SPeter Grehan static void *
818318224bbSNeel Natu vmx_vminit(struct vm *vm, pmap_t pmap)
819366f6083SPeter Grehan {
82045e51299SNeel Natu 	uint16_t vpid[VM_MAXCPU];
821c3498942SNeel Natu 	int i, error;
822366f6083SPeter Grehan 	struct vmx *vmx;
823c847a506SNeel Natu 	struct vmcs *vmcs;
824b0538143SNeel Natu 	uint32_t exc_bitmap;
825366f6083SPeter Grehan 
826366f6083SPeter Grehan 	vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO);
827366f6083SPeter Grehan 	if ((uintptr_t)vmx & PAGE_MASK) {
828366f6083SPeter Grehan 		panic("malloc of struct vmx not aligned on %d byte boundary",
829366f6083SPeter Grehan 		      PAGE_SIZE);
830366f6083SPeter Grehan 	}
831366f6083SPeter Grehan 	vmx->vm = vm;
832366f6083SPeter Grehan 
833318224bbSNeel Natu 	vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pml4));
834318224bbSNeel Natu 
835366f6083SPeter Grehan 	/*
836366f6083SPeter Grehan 	 * Clean up EPTP-tagged guest physical and combined mappings
837366f6083SPeter Grehan 	 *
838366f6083SPeter Grehan 	 * VMX transitions are not required to invalidate any guest physical
839366f6083SPeter Grehan 	 * mappings. So, it may be possible for stale guest physical mappings
840366f6083SPeter Grehan 	 * to be present in the processor TLBs.
841366f6083SPeter Grehan 	 *
842366f6083SPeter Grehan 	 * Combined mappings for this EP4TA are also invalidated for all VPIDs.
843366f6083SPeter Grehan 	 */
844318224bbSNeel Natu 	ept_invalidate_mappings(vmx->eptp);
845366f6083SPeter Grehan 
846366f6083SPeter Grehan 	msr_bitmap_initialize(vmx->msr_bitmap);
847366f6083SPeter Grehan 
848366f6083SPeter Grehan 	/*
849366f6083SPeter Grehan 	 * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE.
850366f6083SPeter Grehan 	 * The guest FSBASE and GSBASE are saved and restored during
851366f6083SPeter Grehan 	 * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are
852366f6083SPeter Grehan 	 * always restored from the vmcs host state area on vm-exit.
853366f6083SPeter Grehan 	 *
8541fb0ea3fSPeter Grehan 	 * The SYSENTER_CS/ESP/EIP MSRs are identical to FS/GSBASE in
8551fb0ea3fSPeter Grehan 	 * how they are saved/restored so can be directly accessed by the
8561fb0ea3fSPeter Grehan 	 * guest.
8571fb0ea3fSPeter Grehan 	 *
858366f6083SPeter Grehan 	 * MSR_EFER is saved and restored in the guest VMCS area on a
859366f6083SPeter Grehan 	 * VM exit and entry respectively. It is also restored from the
860366f6083SPeter Grehan 	 * host VMCS area on a VM exit.
8618d1d7a9eSPeter Grehan 	 *
862277bdd99STycho Nightingale 	 * The TSC MSR is exposed read-only. Writes are disallowed as
863277bdd99STycho Nightingale 	 * that will impact the host TSC.  If the guest does a write
864277bdd99STycho Nightingale 	 * the "use TSC offsetting" execution control is enabled and the
865277bdd99STycho Nightingale 	 * difference between the host TSC and the guest TSC is written
866277bdd99STycho Nightingale 	 * into the TSC offset in the VMCS.
867366f6083SPeter Grehan 	 */
868366f6083SPeter Grehan 	if (guest_msr_rw(vmx, MSR_GSBASE) ||
869366f6083SPeter Grehan 	    guest_msr_rw(vmx, MSR_FSBASE) ||
8701fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_CS_MSR) ||
8711fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_ESP_MSR) ||
8721fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_EIP_MSR) ||
8738d1d7a9eSPeter Grehan 	    guest_msr_rw(vmx, MSR_EFER) ||
8748d1d7a9eSPeter Grehan 	    guest_msr_ro(vmx, MSR_TSC))
875366f6083SPeter Grehan 		panic("vmx_vminit: error setting guest msr access");
876366f6083SPeter Grehan 
87745e51299SNeel Natu 	vpid_alloc(vpid, VM_MAXCPU);
87845e51299SNeel Natu 
87988c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
88088c4b8d1SNeel Natu 		error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE,
88188c4b8d1SNeel Natu 		    APIC_ACCESS_ADDRESS);
88288c4b8d1SNeel Natu 		/* XXX this should really return an error to the caller */
88388c4b8d1SNeel Natu 		KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error));
88488c4b8d1SNeel Natu 	}
88588c4b8d1SNeel Natu 
886366f6083SPeter Grehan 	for (i = 0; i < VM_MAXCPU; i++) {
887c847a506SNeel Natu 		vmcs = &vmx->vmcs[i];
888c847a506SNeel Natu 		vmcs->identifier = vmx_revision();
889c847a506SNeel Natu 		error = vmclear(vmcs);
890366f6083SPeter Grehan 		if (error != 0) {
891366f6083SPeter Grehan 			panic("vmx_vminit: vmclear error %d on vcpu %d\n",
892366f6083SPeter Grehan 			      error, i);
893366f6083SPeter Grehan 		}
894366f6083SPeter Grehan 
895c3498942SNeel Natu 		vmx_msr_guest_init(vmx, i);
896c3498942SNeel Natu 
897c847a506SNeel Natu 		error = vmcs_init(vmcs);
898c847a506SNeel Natu 		KASSERT(error == 0, ("vmcs_init error %d", error));
899366f6083SPeter Grehan 
900c847a506SNeel Natu 		VMPTRLD(vmcs);
901c847a506SNeel Natu 		error = 0;
902c847a506SNeel Natu 		error += vmwrite(VMCS_HOST_RSP, (u_long)&vmx->ctx[i]);
903c847a506SNeel Natu 		error += vmwrite(VMCS_EPTP, vmx->eptp);
904c847a506SNeel Natu 		error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls);
905c847a506SNeel Natu 		error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls);
906c847a506SNeel Natu 		error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2);
907c847a506SNeel Natu 		error += vmwrite(VMCS_EXIT_CTLS, exit_ctls);
908c847a506SNeel Natu 		error += vmwrite(VMCS_ENTRY_CTLS, entry_ctls);
909c847a506SNeel Natu 		error += vmwrite(VMCS_MSR_BITMAP, vtophys(vmx->msr_bitmap));
910c847a506SNeel Natu 		error += vmwrite(VMCS_VPID, vpid[i]);
911b0538143SNeel Natu 
912b0538143SNeel Natu 		/* exception bitmap */
913b0538143SNeel Natu 		if (vcpu_trace_exceptions(vm, i))
914b0538143SNeel Natu 			exc_bitmap = 0xffffffff;
915b0538143SNeel Natu 		else
916b0538143SNeel Natu 			exc_bitmap = 1 << IDT_MC;
917b0538143SNeel Natu 		error += vmwrite(VMCS_EXCEPTION_BITMAP, exc_bitmap);
918b0538143SNeel Natu 
91988c4b8d1SNeel Natu 		if (virtual_interrupt_delivery) {
92088c4b8d1SNeel Natu 			error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS);
92188c4b8d1SNeel Natu 			error += vmwrite(VMCS_VIRTUAL_APIC,
92288c4b8d1SNeel Natu 			    vtophys(&vmx->apic_page[i]));
92388c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT0, 0);
92488c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT1, 0);
92588c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT2, 0);
92688c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT3, 0);
92788c4b8d1SNeel Natu 		}
928176666c2SNeel Natu 		if (posted_interrupts) {
929176666c2SNeel Natu 			error += vmwrite(VMCS_PIR_VECTOR, pirvec);
930176666c2SNeel Natu 			error += vmwrite(VMCS_PIR_DESC,
931176666c2SNeel Natu 			    vtophys(&vmx->pir_desc[i]));
932176666c2SNeel Natu 		}
933c847a506SNeel Natu 		VMCLEAR(vmcs);
934c847a506SNeel Natu 		KASSERT(error == 0, ("vmx_vminit: error customizing the vmcs"));
935366f6083SPeter Grehan 
936366f6083SPeter Grehan 		vmx->cap[i].set = 0;
937366f6083SPeter Grehan 		vmx->cap[i].proc_ctls = procbased_ctls;
93849cc03daSNeel Natu 		vmx->cap[i].proc_ctls2 = procbased_ctls2;
939366f6083SPeter Grehan 
9402ce12423SNeel Natu 		vmx->state[i].nextrip = ~0;
9413527963bSNeel Natu 		vmx->state[i].lastcpu = NOCPU;
94245e51299SNeel Natu 		vmx->state[i].vpid = vpid[i];
943366f6083SPeter Grehan 
944aaaa0656SPeter Grehan 		/*
945aaaa0656SPeter Grehan 		 * Set up the CR0/4 shadows, and init the read shadow
946aaaa0656SPeter Grehan 		 * to the power-on register value from the Intel Sys Arch.
947aaaa0656SPeter Grehan 		 *  CR0 - 0x60000010
948aaaa0656SPeter Grehan 		 *  CR4 - 0
949aaaa0656SPeter Grehan 		 */
950c847a506SNeel Natu 		error = vmx_setup_cr0_shadow(vmcs, 0x60000010);
95139c21c2dSNeel Natu 		if (error != 0)
95239c21c2dSNeel Natu 			panic("vmx_setup_cr0_shadow %d", error);
95339c21c2dSNeel Natu 
954c847a506SNeel Natu 		error = vmx_setup_cr4_shadow(vmcs, 0);
95539c21c2dSNeel Natu 		if (error != 0)
95639c21c2dSNeel Natu 			panic("vmx_setup_cr4_shadow %d", error);
957318224bbSNeel Natu 
958318224bbSNeel Natu 		vmx->ctx[i].pmap = pmap;
959366f6083SPeter Grehan 	}
960366f6083SPeter Grehan 
961366f6083SPeter Grehan 	return (vmx);
962366f6083SPeter Grehan }
963366f6083SPeter Grehan 
964366f6083SPeter Grehan static int
965a2da7af6SNeel Natu vmx_handle_cpuid(struct vm *vm, int vcpu, struct vmxctx *vmxctx)
966366f6083SPeter Grehan {
967366f6083SPeter Grehan 	int handled, func;
968366f6083SPeter Grehan 
969366f6083SPeter Grehan 	func = vmxctx->guest_rax;
970366f6083SPeter Grehan 
971a2da7af6SNeel Natu 	handled = x86_emulate_cpuid(vm, vcpu,
972a2da7af6SNeel Natu 				    (uint32_t*)(&vmxctx->guest_rax),
973a2da7af6SNeel Natu 				    (uint32_t*)(&vmxctx->guest_rbx),
974a2da7af6SNeel Natu 				    (uint32_t*)(&vmxctx->guest_rcx),
975a2da7af6SNeel Natu 				    (uint32_t*)(&vmxctx->guest_rdx));
976366f6083SPeter Grehan 	return (handled);
977366f6083SPeter Grehan }
978366f6083SPeter Grehan 
979366f6083SPeter Grehan static __inline void
980366f6083SPeter Grehan vmx_run_trace(struct vmx *vmx, int vcpu)
981366f6083SPeter Grehan {
982366f6083SPeter Grehan #ifdef KTR
983513c8d33SNeel Natu 	VCPU_CTR1(vmx->vm, vcpu, "Resume execution at %#lx", vmcs_guest_rip());
984366f6083SPeter Grehan #endif
985366f6083SPeter Grehan }
986366f6083SPeter Grehan 
987366f6083SPeter Grehan static __inline void
988366f6083SPeter Grehan vmx_exit_trace(struct vmx *vmx, int vcpu, uint64_t rip, uint32_t exit_reason,
989eeefa4e4SNeel Natu 	       int handled)
990366f6083SPeter Grehan {
991366f6083SPeter Grehan #ifdef KTR
992513c8d33SNeel Natu 	VCPU_CTR3(vmx->vm, vcpu, "%s %s vmexit at 0x%0lx",
993366f6083SPeter Grehan 		 handled ? "handled" : "unhandled",
994366f6083SPeter Grehan 		 exit_reason_to_str(exit_reason), rip);
995eeefa4e4SNeel Natu #endif
996eeefa4e4SNeel Natu }
997366f6083SPeter Grehan 
998eeefa4e4SNeel Natu static __inline void
999eeefa4e4SNeel Natu vmx_astpending_trace(struct vmx *vmx, int vcpu, uint64_t rip)
1000eeefa4e4SNeel Natu {
1001eeefa4e4SNeel Natu #ifdef KTR
1002513c8d33SNeel Natu 	VCPU_CTR1(vmx->vm, vcpu, "astpending vmexit at 0x%0lx", rip);
1003366f6083SPeter Grehan #endif
1004366f6083SPeter Grehan }
1005366f6083SPeter Grehan 
1006953c2c47SNeel Natu static VMM_STAT_INTEL(VCPU_INVVPID_SAVED, "Number of vpid invalidations saved");
10073527963bSNeel Natu static VMM_STAT_INTEL(VCPU_INVVPID_DONE, "Number of vpid invalidations done");
1008953c2c47SNeel Natu 
10093527963bSNeel Natu /*
10103527963bSNeel Natu  * Invalidate guest mappings identified by its vpid from the TLB.
10113527963bSNeel Natu  */
10123527963bSNeel Natu static __inline void
10133527963bSNeel Natu vmx_invvpid(struct vmx *vmx, int vcpu, pmap_t pmap, int running)
1014366f6083SPeter Grehan {
1015366f6083SPeter Grehan 	struct vmxstate *vmxstate;
1016953c2c47SNeel Natu 	struct invvpid_desc invvpid_desc;
1017366f6083SPeter Grehan 
1018366f6083SPeter Grehan 	vmxstate = &vmx->state[vcpu];
10193527963bSNeel Natu 	if (vmxstate->vpid == 0)
10203de83862SNeel Natu 		return;
1021366f6083SPeter Grehan 
10223527963bSNeel Natu 	if (!running) {
10233527963bSNeel Natu 		/*
10243527963bSNeel Natu 		 * Set the 'lastcpu' to an invalid host cpu.
10253527963bSNeel Natu 		 *
10263527963bSNeel Natu 		 * This will invalidate TLB entries tagged with the vcpu's
10273527963bSNeel Natu 		 * vpid the next time it runs via vmx_set_pcpu_defaults().
10283527963bSNeel Natu 		 */
10293527963bSNeel Natu 		vmxstate->lastcpu = NOCPU;
10303527963bSNeel Natu 		return;
10313527963bSNeel Natu 	}
1032953c2c47SNeel Natu 
10333527963bSNeel Natu 	KASSERT(curthread->td_critnest > 0, ("%s: vcpu %d running outside "
10343527963bSNeel Natu 	    "critical section", __func__, vcpu));
1035366f6083SPeter Grehan 
1036366f6083SPeter Grehan 	/*
10373527963bSNeel Natu 	 * Invalidate all mappings tagged with 'vpid'
1038366f6083SPeter Grehan 	 *
1039366f6083SPeter Grehan 	 * We do this because this vcpu was executing on a different host
1040366f6083SPeter Grehan 	 * cpu when it last ran. We do not track whether it invalidated
1041366f6083SPeter Grehan 	 * mappings associated with its 'vpid' during that run. So we must
1042366f6083SPeter Grehan 	 * assume that the mappings associated with 'vpid' on 'curcpu' are
1043366f6083SPeter Grehan 	 * stale and invalidate them.
1044366f6083SPeter Grehan 	 *
1045366f6083SPeter Grehan 	 * Note that we incur this penalty only when the scheduler chooses to
1046366f6083SPeter Grehan 	 * move the thread associated with this vcpu between host cpus.
1047366f6083SPeter Grehan 	 *
1048366f6083SPeter Grehan 	 * Note also that this will invalidate mappings tagged with 'vpid'
1049366f6083SPeter Grehan 	 * for "all" EP4TAs.
1050366f6083SPeter Grehan 	 */
1051953c2c47SNeel Natu 	if (pmap->pm_eptgen == vmx->eptgen[curcpu]) {
1052953c2c47SNeel Natu 		invvpid_desc._res1 = 0;
1053953c2c47SNeel Natu 		invvpid_desc._res2 = 0;
1054366f6083SPeter Grehan 		invvpid_desc.vpid = vmxstate->vpid;
10550e30c5c0SWarner Losh 		invvpid_desc.linear_addr = 0;
1056366f6083SPeter Grehan 		invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc);
10573527963bSNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_DONE, 1);
1058953c2c47SNeel Natu 	} else {
1059953c2c47SNeel Natu 		/*
1060953c2c47SNeel Natu 		 * The invvpid can be skipped if an invept is going to
1061953c2c47SNeel Natu 		 * be performed before entering the guest. The invept
1062953c2c47SNeel Natu 		 * will invalidate combined mappings tagged with
1063953c2c47SNeel Natu 		 * 'vmx->eptp' for all vpids.
1064953c2c47SNeel Natu 		 */
1065953c2c47SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_SAVED, 1);
1066953c2c47SNeel Natu 	}
1067366f6083SPeter Grehan }
10683527963bSNeel Natu 
10693527963bSNeel Natu static void
10703527963bSNeel Natu vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu, pmap_t pmap)
10713527963bSNeel Natu {
10723527963bSNeel Natu 	struct vmxstate *vmxstate;
10733527963bSNeel Natu 
10743527963bSNeel Natu 	vmxstate = &vmx->state[vcpu];
10753527963bSNeel Natu 	if (vmxstate->lastcpu == curcpu)
10763527963bSNeel Natu 		return;
10773527963bSNeel Natu 
10783527963bSNeel Natu 	vmxstate->lastcpu = curcpu;
10793527963bSNeel Natu 
10803527963bSNeel Natu 	vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1);
10813527963bSNeel Natu 
10823527963bSNeel Natu 	vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase());
10833527963bSNeel Natu 	vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase());
10843527963bSNeel Natu 	vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase());
10853527963bSNeel Natu 	vmx_invvpid(vmx, vcpu, pmap, 1);
1086366f6083SPeter Grehan }
1087366f6083SPeter Grehan 
1088366f6083SPeter Grehan /*
1089366f6083SPeter Grehan  * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set.
1090366f6083SPeter Grehan  */
1091366f6083SPeter Grehan CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0);
1092366f6083SPeter Grehan 
1093366f6083SPeter Grehan static void __inline
1094366f6083SPeter Grehan vmx_set_int_window_exiting(struct vmx *vmx, int vcpu)
1095366f6083SPeter Grehan {
1096366f6083SPeter Grehan 
109748b2d828SNeel Natu 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) {
1098366f6083SPeter Grehan 		vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING;
10993de83862SNeel Natu 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
110048b2d828SNeel Natu 		VCPU_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting");
110148b2d828SNeel Natu 	}
1102366f6083SPeter Grehan }
1103366f6083SPeter Grehan 
1104366f6083SPeter Grehan static void __inline
1105366f6083SPeter Grehan vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu)
1106366f6083SPeter Grehan {
1107366f6083SPeter Grehan 
110848b2d828SNeel Natu 	KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0,
110948b2d828SNeel Natu 	    ("intr_window_exiting not set: %#x", vmx->cap[vcpu].proc_ctls));
1110366f6083SPeter Grehan 	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING;
11113de83862SNeel Natu 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
111248b2d828SNeel Natu 	VCPU_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting");
1113366f6083SPeter Grehan }
1114366f6083SPeter Grehan 
1115366f6083SPeter Grehan static void __inline
1116366f6083SPeter Grehan vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu)
1117366f6083SPeter Grehan {
1118366f6083SPeter Grehan 
111948b2d828SNeel Natu 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) {
1120366f6083SPeter Grehan 		vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING;
11213de83862SNeel Natu 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
112248b2d828SNeel Natu 		VCPU_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting");
112348b2d828SNeel Natu 	}
1124366f6083SPeter Grehan }
1125366f6083SPeter Grehan 
1126366f6083SPeter Grehan static void __inline
1127366f6083SPeter Grehan vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu)
1128366f6083SPeter Grehan {
1129366f6083SPeter Grehan 
113048b2d828SNeel Natu 	KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0,
113148b2d828SNeel Natu 	    ("nmi_window_exiting not set %#x", vmx->cap[vcpu].proc_ctls));
1132366f6083SPeter Grehan 	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING;
11333de83862SNeel Natu 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
113448b2d828SNeel Natu 	VCPU_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting");
1135366f6083SPeter Grehan }
1136366f6083SPeter Grehan 
1137277bdd99STycho Nightingale int
1138277bdd99STycho Nightingale vmx_set_tsc_offset(struct vmx *vmx, int vcpu, uint64_t offset)
1139277bdd99STycho Nightingale {
1140277bdd99STycho Nightingale 	int error;
1141277bdd99STycho Nightingale 
1142277bdd99STycho Nightingale 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_TSC_OFFSET) == 0) {
1143277bdd99STycho Nightingale 		vmx->cap[vcpu].proc_ctls |= PROCBASED_TSC_OFFSET;
1144277bdd99STycho Nightingale 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1145277bdd99STycho Nightingale 		VCPU_CTR0(vmx->vm, vcpu, "Enabling TSC offsetting");
1146277bdd99STycho Nightingale 	}
1147277bdd99STycho Nightingale 
1148277bdd99STycho Nightingale 	error = vmwrite(VMCS_TSC_OFFSET, offset);
1149277bdd99STycho Nightingale 
1150277bdd99STycho Nightingale 	return (error);
1151277bdd99STycho Nightingale }
1152277bdd99STycho Nightingale 
115348b2d828SNeel Natu #define	NMI_BLOCKING	(VMCS_INTERRUPTIBILITY_NMI_BLOCKING |		\
115448b2d828SNeel Natu 			 VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
115548b2d828SNeel Natu #define	HWINTR_BLOCKING	(VMCS_INTERRUPTIBILITY_STI_BLOCKING |		\
115648b2d828SNeel Natu 			 VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
115748b2d828SNeel Natu 
115848b2d828SNeel Natu static void
1159366f6083SPeter Grehan vmx_inject_nmi(struct vmx *vmx, int vcpu)
1160366f6083SPeter Grehan {
116148b2d828SNeel Natu 	uint32_t gi, info;
1162366f6083SPeter Grehan 
116348b2d828SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
116448b2d828SNeel Natu 	KASSERT((gi & NMI_BLOCKING) == 0, ("vmx_inject_nmi: invalid guest "
116548b2d828SNeel Natu 	    "interruptibility-state %#x", gi));
1166366f6083SPeter Grehan 
116748b2d828SNeel Natu 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
116848b2d828SNeel Natu 	KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_nmi: invalid "
116948b2d828SNeel Natu 	    "VM-entry interruption information %#x", info));
1170366f6083SPeter Grehan 
1171366f6083SPeter Grehan 	/*
1172366f6083SPeter Grehan 	 * Inject the virtual NMI. The vector must be the NMI IDT entry
1173366f6083SPeter Grehan 	 * or the VMCS entry check will fail.
1174366f6083SPeter Grehan 	 */
117548b2d828SNeel Natu 	info = IDT_NMI | VMCS_INTR_T_NMI | VMCS_INTR_VALID;
11763de83862SNeel Natu 	vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1177366f6083SPeter Grehan 
1178513c8d33SNeel Natu 	VCPU_CTR0(vmx->vm, vcpu, "Injecting vNMI");
1179366f6083SPeter Grehan 
1180366f6083SPeter Grehan 	/* Clear the request */
1181f352ff0cSNeel Natu 	vm_nmi_clear(vmx->vm, vcpu);
1182366f6083SPeter Grehan }
1183366f6083SPeter Grehan 
1184366f6083SPeter Grehan static void
11852ce12423SNeel Natu vmx_inject_interrupts(struct vmx *vmx, int vcpu, struct vlapic *vlapic,
11862ce12423SNeel Natu     uint64_t guestrip)
1187366f6083SPeter Grehan {
11880775fbb4STycho Nightingale 	int vector, need_nmi_exiting, extint_pending;
1189091d4532SNeel Natu 	uint64_t rflags, entryinfo;
119048b2d828SNeel Natu 	uint32_t gi, info;
1191366f6083SPeter Grehan 
11922ce12423SNeel Natu 	if (vmx->state[vcpu].nextrip != guestrip) {
11932ce12423SNeel Natu 		gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
11942ce12423SNeel Natu 		if (gi & HWINTR_BLOCKING) {
11952ce12423SNeel Natu 			VCPU_CTR2(vmx->vm, vcpu, "Guest interrupt blocking "
11962ce12423SNeel Natu 			    "cleared due to rip change: %#lx/%#lx",
11972ce12423SNeel Natu 			    vmx->state[vcpu].nextrip, guestrip);
11982ce12423SNeel Natu 			gi &= ~HWINTR_BLOCKING;
11992ce12423SNeel Natu 			vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
12002ce12423SNeel Natu 		}
12012ce12423SNeel Natu 	}
12022ce12423SNeel Natu 
1203091d4532SNeel Natu 	if (vm_entry_intinfo(vmx->vm, vcpu, &entryinfo)) {
1204091d4532SNeel Natu 		KASSERT((entryinfo & VMCS_INTR_VALID) != 0, ("%s: entry "
1205091d4532SNeel Natu 		    "intinfo is not valid: %#lx", __func__, entryinfo));
1206dc506506SNeel Natu 
1207dc506506SNeel Natu 		info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1208dc506506SNeel Natu 		KASSERT((info & VMCS_INTR_VALID) == 0, ("%s: cannot inject "
1209019008ebSNeel Natu 		     "pending exception: %#lx/%#x", __func__, entryinfo, info));
1210dc506506SNeel Natu 
1211091d4532SNeel Natu 		info = entryinfo;
1212091d4532SNeel Natu 		vector = info & 0xff;
1213091d4532SNeel Natu 		if (vector == IDT_BP || vector == IDT_OF) {
1214091d4532SNeel Natu 			/*
1215091d4532SNeel Natu 			 * VT-x requires #BP and #OF to be injected as software
1216091d4532SNeel Natu 			 * exceptions.
1217091d4532SNeel Natu 			 */
1218091d4532SNeel Natu 			info &= ~VMCS_INTR_T_MASK;
1219091d4532SNeel Natu 			info |= VMCS_INTR_T_SWEXCEPTION;
1220dc506506SNeel Natu 		}
1221091d4532SNeel Natu 
1222091d4532SNeel Natu 		if (info & VMCS_INTR_DEL_ERRCODE)
1223091d4532SNeel Natu 			vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR, entryinfo >> 32);
1224091d4532SNeel Natu 
1225dc506506SNeel Natu 		vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1226dc506506SNeel Natu 	}
1227dc506506SNeel Natu 
122848b2d828SNeel Natu 	if (vm_nmi_pending(vmx->vm, vcpu)) {
1229366f6083SPeter Grehan 		/*
123048b2d828SNeel Natu 		 * If there are no conditions blocking NMI injection then
123148b2d828SNeel Natu 		 * inject it directly here otherwise enable "NMI window
123248b2d828SNeel Natu 		 * exiting" to inject it as soon as we can.
1233eeefa4e4SNeel Natu 		 *
123448b2d828SNeel Natu 		 * We also check for STI_BLOCKING because some implementations
123548b2d828SNeel Natu 		 * don't allow NMI injection in this case. If we are running
123648b2d828SNeel Natu 		 * on a processor that doesn't have this restriction it will
123748b2d828SNeel Natu 		 * immediately exit and the NMI will be injected in the
123848b2d828SNeel Natu 		 * "NMI window exiting" handler.
1239366f6083SPeter Grehan 		 */
124048b2d828SNeel Natu 		need_nmi_exiting = 1;
124148b2d828SNeel Natu 		gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
124248b2d828SNeel Natu 		if ((gi & (HWINTR_BLOCKING | NMI_BLOCKING)) == 0) {
12433de83862SNeel Natu 			info = vmcs_read(VMCS_ENTRY_INTR_INFO);
124448b2d828SNeel Natu 			if ((info & VMCS_INTR_VALID) == 0) {
124548b2d828SNeel Natu 				vmx_inject_nmi(vmx, vcpu);
124648b2d828SNeel Natu 				need_nmi_exiting = 0;
124748b2d828SNeel Natu 			} else {
124848b2d828SNeel Natu 				VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI "
124948b2d828SNeel Natu 				    "due to VM-entry intr info %#x", info);
125048b2d828SNeel Natu 			}
125148b2d828SNeel Natu 		} else {
125248b2d828SNeel Natu 			VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI due to "
125348b2d828SNeel Natu 			    "Guest Interruptibility-state %#x", gi);
125448b2d828SNeel Natu 		}
1255eeefa4e4SNeel Natu 
125648b2d828SNeel Natu 		if (need_nmi_exiting)
125748b2d828SNeel Natu 			vmx_set_nmi_window_exiting(vmx, vcpu);
125848b2d828SNeel Natu 	}
1259366f6083SPeter Grehan 
12600775fbb4STycho Nightingale 	extint_pending = vm_extint_pending(vmx->vm, vcpu);
12610775fbb4STycho Nightingale 
12620775fbb4STycho Nightingale 	if (!extint_pending && virtual_interrupt_delivery) {
126388c4b8d1SNeel Natu 		vmx_inject_pir(vlapic);
126488c4b8d1SNeel Natu 		return;
126588c4b8d1SNeel Natu 	}
126688c4b8d1SNeel Natu 
126748b2d828SNeel Natu 	/*
126836736912SNeel Natu 	 * If interrupt-window exiting is already in effect then don't bother
126936736912SNeel Natu 	 * checking for pending interrupts. This is just an optimization and
127036736912SNeel Natu 	 * not needed for correctness.
127148b2d828SNeel Natu 	 */
127236736912SNeel Natu 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) {
127336736912SNeel Natu 		VCPU_CTR0(vmx->vm, vcpu, "Skip interrupt injection due to "
127436736912SNeel Natu 		    "pending int_window_exiting");
127548b2d828SNeel Natu 		return;
127636736912SNeel Natu 	}
127748b2d828SNeel Natu 
12780775fbb4STycho Nightingale 	if (!extint_pending) {
1279366f6083SPeter Grehan 		/* Ask the local apic for a vector to inject */
12804d1e82a8SNeel Natu 		if (!vlapic_pending_intr(vlapic, &vector))
1281366f6083SPeter Grehan 			return;
1282a026dc3fSTycho Nightingale 
1283a026dc3fSTycho Nightingale 		/*
1284a026dc3fSTycho Nightingale 		 * From the Intel SDM, Volume 3, Section "Maskable
1285a026dc3fSTycho Nightingale 		 * Hardware Interrupts":
1286a026dc3fSTycho Nightingale 		 * - maskable interrupt vectors [16,255] can be delivered
1287a026dc3fSTycho Nightingale 		 *   through the local APIC.
1288a026dc3fSTycho Nightingale 		*/
1289a026dc3fSTycho Nightingale 		KASSERT(vector >= 16 && vector <= 255,
1290a026dc3fSTycho Nightingale 		    ("invalid vector %d from local APIC", vector));
12910775fbb4STycho Nightingale 	} else {
12920775fbb4STycho Nightingale 		/* Ask the legacy pic for a vector to inject */
12930775fbb4STycho Nightingale 		vatpic_pending_intr(vmx->vm, &vector);
1294366f6083SPeter Grehan 
1295a026dc3fSTycho Nightingale 		/*
1296a026dc3fSTycho Nightingale 		 * From the Intel SDM, Volume 3, Section "Maskable
1297a026dc3fSTycho Nightingale 		 * Hardware Interrupts":
1298a026dc3fSTycho Nightingale 		 * - maskable interrupt vectors [0,255] can be delivered
1299a026dc3fSTycho Nightingale 		 *   through the INTR pin.
1300a026dc3fSTycho Nightingale 		 */
1301a026dc3fSTycho Nightingale 		KASSERT(vector >= 0 && vector <= 255,
1302a026dc3fSTycho Nightingale 		    ("invalid vector %d from INTR", vector));
1303a026dc3fSTycho Nightingale 	}
1304366f6083SPeter Grehan 
1305366f6083SPeter Grehan 	/* Check RFLAGS.IF and the interruptibility state of the guest */
13063de83862SNeel Natu 	rflags = vmcs_read(VMCS_GUEST_RFLAGS);
130736736912SNeel Natu 	if ((rflags & PSL_I) == 0) {
130836736912SNeel Natu 		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
130936736912SNeel Natu 		    "rflags %#lx", vector, rflags);
1310366f6083SPeter Grehan 		goto cantinject;
131136736912SNeel Natu 	}
1312366f6083SPeter Grehan 
131348b2d828SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
131436736912SNeel Natu 	if (gi & HWINTR_BLOCKING) {
131536736912SNeel Natu 		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
131636736912SNeel Natu 		    "Guest Interruptibility-state %#x", vector, gi);
1317366f6083SPeter Grehan 		goto cantinject;
131836736912SNeel Natu 	}
131936736912SNeel Natu 
132036736912SNeel Natu 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
132136736912SNeel Natu 	if (info & VMCS_INTR_VALID) {
132236736912SNeel Natu 		/*
132336736912SNeel Natu 		 * This is expected and could happen for multiple reasons:
132436736912SNeel Natu 		 * - A vectoring VM-entry was aborted due to astpending
132536736912SNeel Natu 		 * - A VM-exit happened during event injection.
1326dc506506SNeel Natu 		 * - An exception was injected above.
132736736912SNeel Natu 		 * - An NMI was injected above or after "NMI window exiting"
132836736912SNeel Natu 		 */
132936736912SNeel Natu 		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
133036736912SNeel Natu 		    "VM-entry intr info %#x", vector, info);
133136736912SNeel Natu 		goto cantinject;
133236736912SNeel Natu 	}
1333366f6083SPeter Grehan 
1334366f6083SPeter Grehan 	/* Inject the interrupt */
1335160471d2SNeel Natu 	info = VMCS_INTR_T_HWINTR | VMCS_INTR_VALID;
1336366f6083SPeter Grehan 	info |= vector;
13373de83862SNeel Natu 	vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1338366f6083SPeter Grehan 
13390775fbb4STycho Nightingale 	if (!extint_pending) {
1340366f6083SPeter Grehan 		/* Update the Local APIC ISR */
1341de5ea6b6SNeel Natu 		vlapic_intr_accepted(vlapic, vector);
13420775fbb4STycho Nightingale 	} else {
13430775fbb4STycho Nightingale 		vm_extint_clear(vmx->vm, vcpu);
13440775fbb4STycho Nightingale 		vatpic_intr_accepted(vmx->vm, vector);
13450775fbb4STycho Nightingale 
13460775fbb4STycho Nightingale 		/*
13470775fbb4STycho Nightingale 		 * After we accepted the current ExtINT the PIC may
13480775fbb4STycho Nightingale 		 * have posted another one.  If that is the case, set
13490775fbb4STycho Nightingale 		 * the Interrupt Window Exiting execution control so
13500775fbb4STycho Nightingale 		 * we can inject that one too.
13510494cb1bSNeel Natu 		 *
13520494cb1bSNeel Natu 		 * Also, interrupt window exiting allows us to inject any
13530494cb1bSNeel Natu 		 * pending APIC vector that was preempted by the ExtINT
13540494cb1bSNeel Natu 		 * as soon as possible. This applies both for the software
13550494cb1bSNeel Natu 		 * emulated vlapic and the hardware assisted virtual APIC.
13560775fbb4STycho Nightingale 		 */
13570775fbb4STycho Nightingale 		vmx_set_int_window_exiting(vmx, vcpu);
13580775fbb4STycho Nightingale 	}
1359366f6083SPeter Grehan 
1360513c8d33SNeel Natu 	VCPU_CTR1(vmx->vm, vcpu, "Injecting hwintr at vector %d", vector);
1361366f6083SPeter Grehan 
1362366f6083SPeter Grehan 	return;
1363366f6083SPeter Grehan 
1364366f6083SPeter Grehan cantinject:
1365366f6083SPeter Grehan 	/*
1366366f6083SPeter Grehan 	 * Set the Interrupt Window Exiting execution control so we can inject
1367366f6083SPeter Grehan 	 * the interrupt as soon as blocking condition goes away.
1368366f6083SPeter Grehan 	 */
1369366f6083SPeter Grehan 	vmx_set_int_window_exiting(vmx, vcpu);
1370366f6083SPeter Grehan }
1371366f6083SPeter Grehan 
1372e5a1d950SNeel Natu /*
1373e5a1d950SNeel Natu  * If the Virtual NMIs execution control is '1' then the logical processor
1374e5a1d950SNeel Natu  * tracks virtual-NMI blocking in the Guest Interruptibility-state field of
1375e5a1d950SNeel Natu  * the VMCS. An IRET instruction in VMX non-root operation will remove any
1376e5a1d950SNeel Natu  * virtual-NMI blocking.
1377e5a1d950SNeel Natu  *
1378e5a1d950SNeel Natu  * This unblocking occurs even if the IRET causes a fault. In this case the
1379e5a1d950SNeel Natu  * hypervisor needs to restore virtual-NMI blocking before resuming the guest.
1380e5a1d950SNeel Natu  */
1381e5a1d950SNeel Natu static void
1382e5a1d950SNeel Natu vmx_restore_nmi_blocking(struct vmx *vmx, int vcpuid)
1383e5a1d950SNeel Natu {
1384e5a1d950SNeel Natu 	uint32_t gi;
1385e5a1d950SNeel Natu 
1386e5a1d950SNeel Natu 	VCPU_CTR0(vmx->vm, vcpuid, "Restore Virtual-NMI blocking");
1387e5a1d950SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1388e5a1d950SNeel Natu 	gi |= VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1389e5a1d950SNeel Natu 	vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1390e5a1d950SNeel Natu }
1391e5a1d950SNeel Natu 
1392e5a1d950SNeel Natu static void
1393e5a1d950SNeel Natu vmx_clear_nmi_blocking(struct vmx *vmx, int vcpuid)
1394e5a1d950SNeel Natu {
1395e5a1d950SNeel Natu 	uint32_t gi;
1396e5a1d950SNeel Natu 
1397e5a1d950SNeel Natu 	VCPU_CTR0(vmx->vm, vcpuid, "Clear Virtual-NMI blocking");
1398e5a1d950SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1399e5a1d950SNeel Natu 	gi &= ~VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1400e5a1d950SNeel Natu 	vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1401e5a1d950SNeel Natu }
1402e5a1d950SNeel Natu 
1403091d4532SNeel Natu static void
1404091d4532SNeel Natu vmx_assert_nmi_blocking(struct vmx *vmx, int vcpuid)
1405091d4532SNeel Natu {
1406091d4532SNeel Natu 	uint32_t gi;
1407091d4532SNeel Natu 
1408091d4532SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1409091d4532SNeel Natu 	KASSERT(gi & VMCS_INTERRUPTIBILITY_NMI_BLOCKING,
1410091d4532SNeel Natu 	    ("NMI blocking is not in effect %#x", gi));
1411091d4532SNeel Natu }
1412091d4532SNeel Natu 
1413366f6083SPeter Grehan static int
1414a0efd3fbSJohn Baldwin vmx_emulate_xsetbv(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1415abb023fbSJohn Baldwin {
1416abb023fbSJohn Baldwin 	struct vmxctx *vmxctx;
1417abb023fbSJohn Baldwin 	uint64_t xcrval;
1418abb023fbSJohn Baldwin 	const struct xsave_limits *limits;
1419abb023fbSJohn Baldwin 
1420abb023fbSJohn Baldwin 	vmxctx = &vmx->ctx[vcpu];
1421abb023fbSJohn Baldwin 	limits = vmm_get_xsave_limits();
1422abb023fbSJohn Baldwin 
1423a0efd3fbSJohn Baldwin 	/*
1424a0efd3fbSJohn Baldwin 	 * Note that the processor raises a GP# fault on its own if
1425a0efd3fbSJohn Baldwin 	 * xsetbv is executed for CPL != 0, so we do not have to
1426a0efd3fbSJohn Baldwin 	 * emulate that fault here.
1427a0efd3fbSJohn Baldwin 	 */
1428a0efd3fbSJohn Baldwin 
1429a0efd3fbSJohn Baldwin 	/* Only xcr0 is supported. */
1430a0efd3fbSJohn Baldwin 	if (vmxctx->guest_rcx != 0) {
1431dc506506SNeel Natu 		vm_inject_gp(vmx->vm, vcpu);
1432a0efd3fbSJohn Baldwin 		return (HANDLED);
1433a0efd3fbSJohn Baldwin 	}
1434a0efd3fbSJohn Baldwin 
1435a0efd3fbSJohn Baldwin 	/* We only handle xcr0 if both the host and guest have XSAVE enabled. */
1436a0efd3fbSJohn Baldwin 	if (!limits->xsave_enabled || !(vmcs_read(VMCS_GUEST_CR4) & CR4_XSAVE)) {
1437dc506506SNeel Natu 		vm_inject_ud(vmx->vm, vcpu);
1438a0efd3fbSJohn Baldwin 		return (HANDLED);
1439a0efd3fbSJohn Baldwin 	}
1440abb023fbSJohn Baldwin 
1441abb023fbSJohn Baldwin 	xcrval = vmxctx->guest_rdx << 32 | (vmxctx->guest_rax & 0xffffffff);
1442a0efd3fbSJohn Baldwin 	if ((xcrval & ~limits->xcr0_allowed) != 0) {
1443dc506506SNeel Natu 		vm_inject_gp(vmx->vm, vcpu);
1444a0efd3fbSJohn Baldwin 		return (HANDLED);
1445a0efd3fbSJohn Baldwin 	}
1446abb023fbSJohn Baldwin 
1447a0efd3fbSJohn Baldwin 	if (!(xcrval & XFEATURE_ENABLED_X87)) {
1448dc506506SNeel Natu 		vm_inject_gp(vmx->vm, vcpu);
1449a0efd3fbSJohn Baldwin 		return (HANDLED);
1450a0efd3fbSJohn Baldwin 	}
1451abb023fbSJohn Baldwin 
145244a68c4eSJohn Baldwin 	/* AVX (YMM_Hi128) requires SSE. */
145344a68c4eSJohn Baldwin 	if (xcrval & XFEATURE_ENABLED_AVX &&
145444a68c4eSJohn Baldwin 	    (xcrval & XFEATURE_AVX) != XFEATURE_AVX) {
145544a68c4eSJohn Baldwin 		vm_inject_gp(vmx->vm, vcpu);
145644a68c4eSJohn Baldwin 		return (HANDLED);
145744a68c4eSJohn Baldwin 	}
145844a68c4eSJohn Baldwin 
145944a68c4eSJohn Baldwin 	/*
146044a68c4eSJohn Baldwin 	 * AVX512 requires base AVX (YMM_Hi128) as well as OpMask,
146144a68c4eSJohn Baldwin 	 * ZMM_Hi256, and Hi16_ZMM.
146244a68c4eSJohn Baldwin 	 */
146344a68c4eSJohn Baldwin 	if (xcrval & XFEATURE_AVX512 &&
146444a68c4eSJohn Baldwin 	    (xcrval & (XFEATURE_AVX512 | XFEATURE_AVX)) !=
146544a68c4eSJohn Baldwin 	    (XFEATURE_AVX512 | XFEATURE_AVX)) {
146644a68c4eSJohn Baldwin 		vm_inject_gp(vmx->vm, vcpu);
146744a68c4eSJohn Baldwin 		return (HANDLED);
146844a68c4eSJohn Baldwin 	}
146944a68c4eSJohn Baldwin 
147044a68c4eSJohn Baldwin 	/*
147144a68c4eSJohn Baldwin 	 * Intel MPX requires both bound register state flags to be
147244a68c4eSJohn Baldwin 	 * set.
147344a68c4eSJohn Baldwin 	 */
147444a68c4eSJohn Baldwin 	if (((xcrval & XFEATURE_ENABLED_BNDREGS) != 0) !=
147544a68c4eSJohn Baldwin 	    ((xcrval & XFEATURE_ENABLED_BNDCSR) != 0)) {
1476dc506506SNeel Natu 		vm_inject_gp(vmx->vm, vcpu);
1477a0efd3fbSJohn Baldwin 		return (HANDLED);
1478a0efd3fbSJohn Baldwin 	}
1479abb023fbSJohn Baldwin 
1480abb023fbSJohn Baldwin 	/*
1481abb023fbSJohn Baldwin 	 * This runs "inside" vmrun() with the guest's FPU state, so
1482abb023fbSJohn Baldwin 	 * modifying xcr0 directly modifies the guest's xcr0, not the
1483abb023fbSJohn Baldwin 	 * host's.
1484abb023fbSJohn Baldwin 	 */
1485abb023fbSJohn Baldwin 	load_xcr(0, xcrval);
1486abb023fbSJohn Baldwin 	return (HANDLED);
1487abb023fbSJohn Baldwin }
1488abb023fbSJohn Baldwin 
1489594db002STycho Nightingale static uint64_t
1490594db002STycho Nightingale vmx_get_guest_reg(struct vmx *vmx, int vcpu, int ident)
1491366f6083SPeter Grehan {
1492366f6083SPeter Grehan 	const struct vmxctx *vmxctx;
1493366f6083SPeter Grehan 
1494594db002STycho Nightingale 	vmxctx = &vmx->ctx[vcpu];
1495594db002STycho Nightingale 
1496594db002STycho Nightingale 	switch (ident) {
1497594db002STycho Nightingale 	case 0:
1498594db002STycho Nightingale 		return (vmxctx->guest_rax);
1499594db002STycho Nightingale 	case 1:
1500594db002STycho Nightingale 		return (vmxctx->guest_rcx);
1501594db002STycho Nightingale 	case 2:
1502594db002STycho Nightingale 		return (vmxctx->guest_rdx);
1503594db002STycho Nightingale 	case 3:
1504594db002STycho Nightingale 		return (vmxctx->guest_rbx);
1505594db002STycho Nightingale 	case 4:
1506594db002STycho Nightingale 		return (vmcs_read(VMCS_GUEST_RSP));
1507594db002STycho Nightingale 	case 5:
1508594db002STycho Nightingale 		return (vmxctx->guest_rbp);
1509594db002STycho Nightingale 	case 6:
1510594db002STycho Nightingale 		return (vmxctx->guest_rsi);
1511594db002STycho Nightingale 	case 7:
1512594db002STycho Nightingale 		return (vmxctx->guest_rdi);
1513594db002STycho Nightingale 	case 8:
1514594db002STycho Nightingale 		return (vmxctx->guest_r8);
1515594db002STycho Nightingale 	case 9:
1516594db002STycho Nightingale 		return (vmxctx->guest_r9);
1517594db002STycho Nightingale 	case 10:
1518594db002STycho Nightingale 		return (vmxctx->guest_r10);
1519594db002STycho Nightingale 	case 11:
1520594db002STycho Nightingale 		return (vmxctx->guest_r11);
1521594db002STycho Nightingale 	case 12:
1522594db002STycho Nightingale 		return (vmxctx->guest_r12);
1523594db002STycho Nightingale 	case 13:
1524594db002STycho Nightingale 		return (vmxctx->guest_r13);
1525594db002STycho Nightingale 	case 14:
1526594db002STycho Nightingale 		return (vmxctx->guest_r14);
1527594db002STycho Nightingale 	case 15:
1528594db002STycho Nightingale 		return (vmxctx->guest_r15);
1529594db002STycho Nightingale 	default:
1530594db002STycho Nightingale 		panic("invalid vmx register %d", ident);
1531594db002STycho Nightingale 	}
1532594db002STycho Nightingale }
1533594db002STycho Nightingale 
1534594db002STycho Nightingale static void
1535594db002STycho Nightingale vmx_set_guest_reg(struct vmx *vmx, int vcpu, int ident, uint64_t regval)
1536594db002STycho Nightingale {
1537594db002STycho Nightingale 	struct vmxctx *vmxctx;
1538594db002STycho Nightingale 
1539594db002STycho Nightingale 	vmxctx = &vmx->ctx[vcpu];
1540594db002STycho Nightingale 
1541594db002STycho Nightingale 	switch (ident) {
1542594db002STycho Nightingale 	case 0:
1543594db002STycho Nightingale 		vmxctx->guest_rax = regval;
1544594db002STycho Nightingale 		break;
1545594db002STycho Nightingale 	case 1:
1546594db002STycho Nightingale 		vmxctx->guest_rcx = regval;
1547594db002STycho Nightingale 		break;
1548594db002STycho Nightingale 	case 2:
1549594db002STycho Nightingale 		vmxctx->guest_rdx = regval;
1550594db002STycho Nightingale 		break;
1551594db002STycho Nightingale 	case 3:
1552594db002STycho Nightingale 		vmxctx->guest_rbx = regval;
1553594db002STycho Nightingale 		break;
1554594db002STycho Nightingale 	case 4:
1555594db002STycho Nightingale 		vmcs_write(VMCS_GUEST_RSP, regval);
1556594db002STycho Nightingale 		break;
1557594db002STycho Nightingale 	case 5:
1558594db002STycho Nightingale 		vmxctx->guest_rbp = regval;
1559594db002STycho Nightingale 		break;
1560594db002STycho Nightingale 	case 6:
1561594db002STycho Nightingale 		vmxctx->guest_rsi = regval;
1562594db002STycho Nightingale 		break;
1563594db002STycho Nightingale 	case 7:
1564594db002STycho Nightingale 		vmxctx->guest_rdi = regval;
1565594db002STycho Nightingale 		break;
1566594db002STycho Nightingale 	case 8:
1567594db002STycho Nightingale 		vmxctx->guest_r8 = regval;
1568594db002STycho Nightingale 		break;
1569594db002STycho Nightingale 	case 9:
1570594db002STycho Nightingale 		vmxctx->guest_r9 = regval;
1571594db002STycho Nightingale 		break;
1572594db002STycho Nightingale 	case 10:
1573594db002STycho Nightingale 		vmxctx->guest_r10 = regval;
1574594db002STycho Nightingale 		break;
1575594db002STycho Nightingale 	case 11:
1576594db002STycho Nightingale 		vmxctx->guest_r11 = regval;
1577594db002STycho Nightingale 		break;
1578594db002STycho Nightingale 	case 12:
1579594db002STycho Nightingale 		vmxctx->guest_r12 = regval;
1580594db002STycho Nightingale 		break;
1581594db002STycho Nightingale 	case 13:
1582594db002STycho Nightingale 		vmxctx->guest_r13 = regval;
1583594db002STycho Nightingale 		break;
1584594db002STycho Nightingale 	case 14:
1585594db002STycho Nightingale 		vmxctx->guest_r14 = regval;
1586594db002STycho Nightingale 		break;
1587594db002STycho Nightingale 	case 15:
1588594db002STycho Nightingale 		vmxctx->guest_r15 = regval;
1589594db002STycho Nightingale 		break;
1590594db002STycho Nightingale 	default:
1591594db002STycho Nightingale 		panic("invalid vmx register %d", ident);
1592594db002STycho Nightingale 	}
1593594db002STycho Nightingale }
1594594db002STycho Nightingale 
1595594db002STycho Nightingale static int
1596594db002STycho Nightingale vmx_emulate_cr0_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1597594db002STycho Nightingale {
1598594db002STycho Nightingale 	uint64_t crval, regval;
1599594db002STycho Nightingale 
1600594db002STycho Nightingale 	/* We only handle mov to %cr0 at this time */
160139c21c2dSNeel Natu 	if ((exitqual & 0xf0) != 0x00)
160239c21c2dSNeel Natu 		return (UNHANDLED);
160339c21c2dSNeel Natu 
1604594db002STycho Nightingale 	regval = vmx_get_guest_reg(vmx, vcpu, (exitqual >> 8) & 0xf);
1605366f6083SPeter Grehan 
1606594db002STycho Nightingale 	vmcs_write(VMCS_CR0_SHADOW, regval);
1607366f6083SPeter Grehan 
1608594db002STycho Nightingale 	crval = regval | cr0_ones_mask;
1609594db002STycho Nightingale 	crval &= ~cr0_zeros_mask;
1610594db002STycho Nightingale 	vmcs_write(VMCS_GUEST_CR0, crval);
1611366f6083SPeter Grehan 
1612594db002STycho Nightingale 	if (regval & CR0_PG) {
161380a902efSPeter Grehan 		uint64_t efer, entry_ctls;
161480a902efSPeter Grehan 
161580a902efSPeter Grehan 		/*
161680a902efSPeter Grehan 		 * If CR0.PG is 1 and EFER.LME is 1 then EFER.LMA and
161780a902efSPeter Grehan 		 * the "IA-32e mode guest" bit in VM-entry control must be
161880a902efSPeter Grehan 		 * equal.
161980a902efSPeter Grehan 		 */
16203de83862SNeel Natu 		efer = vmcs_read(VMCS_GUEST_IA32_EFER);
162180a902efSPeter Grehan 		if (efer & EFER_LME) {
162280a902efSPeter Grehan 			efer |= EFER_LMA;
16233de83862SNeel Natu 			vmcs_write(VMCS_GUEST_IA32_EFER, efer);
16243de83862SNeel Natu 			entry_ctls = vmcs_read(VMCS_ENTRY_CTLS);
162580a902efSPeter Grehan 			entry_ctls |= VM_ENTRY_GUEST_LMA;
16263de83862SNeel Natu 			vmcs_write(VMCS_ENTRY_CTLS, entry_ctls);
162780a902efSPeter Grehan 		}
162880a902efSPeter Grehan 	}
162980a902efSPeter Grehan 
1630366f6083SPeter Grehan 	return (HANDLED);
1631366f6083SPeter Grehan }
1632366f6083SPeter Grehan 
1633594db002STycho Nightingale static int
1634594db002STycho Nightingale vmx_emulate_cr4_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1635594db002STycho Nightingale {
1636594db002STycho Nightingale 	uint64_t crval, regval;
1637594db002STycho Nightingale 
1638594db002STycho Nightingale 	/* We only handle mov to %cr4 at this time */
1639594db002STycho Nightingale 	if ((exitqual & 0xf0) != 0x00)
1640594db002STycho Nightingale 		return (UNHANDLED);
1641594db002STycho Nightingale 
1642594db002STycho Nightingale 	regval = vmx_get_guest_reg(vmx, vcpu, (exitqual >> 8) & 0xf);
1643594db002STycho Nightingale 
1644594db002STycho Nightingale 	vmcs_write(VMCS_CR4_SHADOW, regval);
1645594db002STycho Nightingale 
1646594db002STycho Nightingale 	crval = regval | cr4_ones_mask;
1647594db002STycho Nightingale 	crval &= ~cr4_zeros_mask;
1648594db002STycho Nightingale 	vmcs_write(VMCS_GUEST_CR4, crval);
1649594db002STycho Nightingale 
1650594db002STycho Nightingale 	return (HANDLED);
1651594db002STycho Nightingale }
1652594db002STycho Nightingale 
1653594db002STycho Nightingale static int
1654594db002STycho Nightingale vmx_emulate_cr8_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1655594db002STycho Nightingale {
1656051f2bd1SNeel Natu 	struct vlapic *vlapic;
1657051f2bd1SNeel Natu 	uint64_t cr8;
1658051f2bd1SNeel Natu 	int regnum;
1659594db002STycho Nightingale 
1660594db002STycho Nightingale 	/* We only handle mov %cr8 to/from a register at this time. */
1661594db002STycho Nightingale 	if ((exitqual & 0xe0) != 0x00) {
1662594db002STycho Nightingale 		return (UNHANDLED);
1663594db002STycho Nightingale 	}
1664594db002STycho Nightingale 
1665051f2bd1SNeel Natu 	vlapic = vm_lapic(vmx->vm, vcpu);
1666051f2bd1SNeel Natu 	regnum = (exitqual >> 8) & 0xf;
1667594db002STycho Nightingale 	if (exitqual & 0x10) {
1668051f2bd1SNeel Natu 		cr8 = vlapic_get_cr8(vlapic);
1669051f2bd1SNeel Natu 		vmx_set_guest_reg(vmx, vcpu, regnum, cr8);
1670594db002STycho Nightingale 	} else {
1671051f2bd1SNeel Natu 		cr8 = vmx_get_guest_reg(vmx, vcpu, regnum);
1672051f2bd1SNeel Natu 		vlapic_set_cr8(vlapic, cr8);
1673594db002STycho Nightingale 	}
1674594db002STycho Nightingale 
1675594db002STycho Nightingale 	return (HANDLED);
1676594db002STycho Nightingale }
1677594db002STycho Nightingale 
1678e4c8a13dSNeel Natu /*
1679e4c8a13dSNeel Natu  * From section "Guest Register State" in the Intel SDM: CPL = SS.DPL
1680e4c8a13dSNeel Natu  */
1681e4c8a13dSNeel Natu static int
1682e4c8a13dSNeel Natu vmx_cpl(void)
1683e4c8a13dSNeel Natu {
1684e4c8a13dSNeel Natu 	uint32_t ssar;
1685e4c8a13dSNeel Natu 
1686e4c8a13dSNeel Natu 	ssar = vmcs_read(VMCS_GUEST_SS_ACCESS_RIGHTS);
1687e4c8a13dSNeel Natu 	return ((ssar >> 5) & 0x3);
1688e4c8a13dSNeel Natu }
1689e4c8a13dSNeel Natu 
1690e813a873SNeel Natu static enum vm_cpu_mode
169100f3efe1SJohn Baldwin vmx_cpu_mode(void)
169200f3efe1SJohn Baldwin {
1693b301b9e2SNeel Natu 	uint32_t csar;
169400f3efe1SJohn Baldwin 
1695b301b9e2SNeel Natu 	if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LMA) {
1696b301b9e2SNeel Natu 		csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS);
1697b301b9e2SNeel Natu 		if (csar & 0x2000)
1698b301b9e2SNeel Natu 			return (CPU_MODE_64BIT);	/* CS.L = 1 */
169900f3efe1SJohn Baldwin 		else
170000f3efe1SJohn Baldwin 			return (CPU_MODE_COMPATIBILITY);
1701b301b9e2SNeel Natu 	} else if (vmcs_read(VMCS_GUEST_CR0) & CR0_PE) {
1702b301b9e2SNeel Natu 		return (CPU_MODE_PROTECTED);
1703b301b9e2SNeel Natu 	} else {
1704b301b9e2SNeel Natu 		return (CPU_MODE_REAL);
1705b301b9e2SNeel Natu 	}
170600f3efe1SJohn Baldwin }
170700f3efe1SJohn Baldwin 
1708e813a873SNeel Natu static enum vm_paging_mode
170900f3efe1SJohn Baldwin vmx_paging_mode(void)
171000f3efe1SJohn Baldwin {
171100f3efe1SJohn Baldwin 
171200f3efe1SJohn Baldwin 	if (!(vmcs_read(VMCS_GUEST_CR0) & CR0_PG))
171300f3efe1SJohn Baldwin 		return (PAGING_MODE_FLAT);
171400f3efe1SJohn Baldwin 	if (!(vmcs_read(VMCS_GUEST_CR4) & CR4_PAE))
171500f3efe1SJohn Baldwin 		return (PAGING_MODE_32);
171600f3efe1SJohn Baldwin 	if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LME)
171700f3efe1SJohn Baldwin 		return (PAGING_MODE_64);
171800f3efe1SJohn Baldwin 	else
171900f3efe1SJohn Baldwin 		return (PAGING_MODE_PAE);
172000f3efe1SJohn Baldwin }
172100f3efe1SJohn Baldwin 
1722d17b5104SNeel Natu static uint64_t
1723d17b5104SNeel Natu inout_str_index(struct vmx *vmx, int vcpuid, int in)
1724d17b5104SNeel Natu {
1725d17b5104SNeel Natu 	uint64_t val;
1726d17b5104SNeel Natu 	int error;
1727d17b5104SNeel Natu 	enum vm_reg_name reg;
1728d17b5104SNeel Natu 
1729d17b5104SNeel Natu 	reg = in ? VM_REG_GUEST_RDI : VM_REG_GUEST_RSI;
1730d17b5104SNeel Natu 	error = vmx_getreg(vmx, vcpuid, reg, &val);
1731d17b5104SNeel Natu 	KASSERT(error == 0, ("%s: vmx_getreg error %d", __func__, error));
1732d17b5104SNeel Natu 	return (val);
1733d17b5104SNeel Natu }
1734d17b5104SNeel Natu 
1735d17b5104SNeel Natu static uint64_t
1736d17b5104SNeel Natu inout_str_count(struct vmx *vmx, int vcpuid, int rep)
1737d17b5104SNeel Natu {
1738d17b5104SNeel Natu 	uint64_t val;
1739d17b5104SNeel Natu 	int error;
1740d17b5104SNeel Natu 
1741d17b5104SNeel Natu 	if (rep) {
1742d17b5104SNeel Natu 		error = vmx_getreg(vmx, vcpuid, VM_REG_GUEST_RCX, &val);
1743d17b5104SNeel Natu 		KASSERT(!error, ("%s: vmx_getreg error %d", __func__, error));
1744d17b5104SNeel Natu 	} else {
1745d17b5104SNeel Natu 		val = 1;
1746d17b5104SNeel Natu 	}
1747d17b5104SNeel Natu 	return (val);
1748d17b5104SNeel Natu }
1749d17b5104SNeel Natu 
1750d17b5104SNeel Natu static int
1751d17b5104SNeel Natu inout_str_addrsize(uint32_t inst_info)
1752d17b5104SNeel Natu {
1753d17b5104SNeel Natu 	uint32_t size;
1754d17b5104SNeel Natu 
1755d17b5104SNeel Natu 	size = (inst_info >> 7) & 0x7;
1756d17b5104SNeel Natu 	switch (size) {
1757d17b5104SNeel Natu 	case 0:
1758d17b5104SNeel Natu 		return (2);	/* 16 bit */
1759d17b5104SNeel Natu 	case 1:
1760d17b5104SNeel Natu 		return (4);	/* 32 bit */
1761d17b5104SNeel Natu 	case 2:
1762d17b5104SNeel Natu 		return (8);	/* 64 bit */
1763d17b5104SNeel Natu 	default:
1764d17b5104SNeel Natu 		panic("%s: invalid size encoding %d", __func__, size);
1765d17b5104SNeel Natu 	}
1766d17b5104SNeel Natu }
1767d17b5104SNeel Natu 
1768d17b5104SNeel Natu static void
1769d17b5104SNeel Natu inout_str_seginfo(struct vmx *vmx, int vcpuid, uint32_t inst_info, int in,
1770d17b5104SNeel Natu     struct vm_inout_str *vis)
1771d17b5104SNeel Natu {
1772d17b5104SNeel Natu 	int error, s;
1773d17b5104SNeel Natu 
1774d17b5104SNeel Natu 	if (in) {
1775d17b5104SNeel Natu 		vis->seg_name = VM_REG_GUEST_ES;
1776d17b5104SNeel Natu 	} else {
1777d17b5104SNeel Natu 		s = (inst_info >> 15) & 0x7;
1778d17b5104SNeel Natu 		vis->seg_name = vm_segment_name(s);
1779d17b5104SNeel Natu 	}
1780d17b5104SNeel Natu 
1781d17b5104SNeel Natu 	error = vmx_getdesc(vmx, vcpuid, vis->seg_name, &vis->seg_desc);
1782d17b5104SNeel Natu 	KASSERT(error == 0, ("%s: vmx_getdesc error %d", __func__, error));
1783d17b5104SNeel Natu }
1784d17b5104SNeel Natu 
1785e4c8a13dSNeel Natu static void
1786e813a873SNeel Natu vmx_paging_info(struct vm_guest_paging *paging)
1787e813a873SNeel Natu {
1788e813a873SNeel Natu 	paging->cr3 = vmcs_guest_cr3();
1789e813a873SNeel Natu 	paging->cpl = vmx_cpl();
1790e813a873SNeel Natu 	paging->cpu_mode = vmx_cpu_mode();
1791e813a873SNeel Natu 	paging->paging_mode = vmx_paging_mode();
1792e813a873SNeel Natu }
1793e813a873SNeel Natu 
1794e813a873SNeel Natu static void
1795e4c8a13dSNeel Natu vmexit_inst_emul(struct vm_exit *vmexit, uint64_t gpa, uint64_t gla)
1796e4c8a13dSNeel Natu {
1797f7a9f178SNeel Natu 	struct vm_guest_paging *paging;
1798f7a9f178SNeel Natu 	uint32_t csar;
1799f7a9f178SNeel Natu 
1800f7a9f178SNeel Natu 	paging = &vmexit->u.inst_emul.paging;
1801f7a9f178SNeel Natu 
1802e4c8a13dSNeel Natu 	vmexit->exitcode = VM_EXITCODE_INST_EMUL;
18031c73ea3eSNeel Natu 	vmexit->inst_length = 0;
1804e4c8a13dSNeel Natu 	vmexit->u.inst_emul.gpa = gpa;
1805e4c8a13dSNeel Natu 	vmexit->u.inst_emul.gla = gla;
1806f7a9f178SNeel Natu 	vmx_paging_info(paging);
1807f7a9f178SNeel Natu 	switch (paging->cpu_mode) {
1808e4f605eeSTycho Nightingale 	case CPU_MODE_REAL:
1809e4f605eeSTycho Nightingale 		vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE);
1810e4f605eeSTycho Nightingale 		vmexit->u.inst_emul.cs_d = 0;
1811e4f605eeSTycho Nightingale 		break;
1812f7a9f178SNeel Natu 	case CPU_MODE_PROTECTED:
1813f7a9f178SNeel Natu 	case CPU_MODE_COMPATIBILITY:
1814e4f605eeSTycho Nightingale 		vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE);
1815f7a9f178SNeel Natu 		csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS);
1816f7a9f178SNeel Natu 		vmexit->u.inst_emul.cs_d = SEG_DESC_DEF32(csar);
1817f7a9f178SNeel Natu 		break;
1818f7a9f178SNeel Natu 	default:
1819e4f605eeSTycho Nightingale 		vmexit->u.inst_emul.cs_base = 0;
1820f7a9f178SNeel Natu 		vmexit->u.inst_emul.cs_d = 0;
1821f7a9f178SNeel Natu 		break;
1822f7a9f178SNeel Natu 	}
1823c2a875f9SNeel Natu 	vie_init(&vmexit->u.inst_emul.vie, NULL, 0);
1824e4c8a13dSNeel Natu }
1825e4c8a13dSNeel Natu 
1826366f6083SPeter Grehan static int
1827318224bbSNeel Natu ept_fault_type(uint64_t ept_qual)
1828a2da7af6SNeel Natu {
1829318224bbSNeel Natu 	int fault_type;
1830a2da7af6SNeel Natu 
1831318224bbSNeel Natu 	if (ept_qual & EPT_VIOLATION_DATA_WRITE)
1832318224bbSNeel Natu 		fault_type = VM_PROT_WRITE;
1833318224bbSNeel Natu 	else if (ept_qual & EPT_VIOLATION_INST_FETCH)
1834318224bbSNeel Natu 		fault_type = VM_PROT_EXECUTE;
1835318224bbSNeel Natu 	else
1836318224bbSNeel Natu 		fault_type= VM_PROT_READ;
1837318224bbSNeel Natu 
1838318224bbSNeel Natu 	return (fault_type);
1839318224bbSNeel Natu }
1840318224bbSNeel Natu 
1841318224bbSNeel Natu static boolean_t
1842318224bbSNeel Natu ept_emulation_fault(uint64_t ept_qual)
1843318224bbSNeel Natu {
1844318224bbSNeel Natu 	int read, write;
1845318224bbSNeel Natu 
1846318224bbSNeel Natu 	/* EPT fault on an instruction fetch doesn't make sense here */
1847a2da7af6SNeel Natu 	if (ept_qual & EPT_VIOLATION_INST_FETCH)
1848318224bbSNeel Natu 		return (FALSE);
1849a2da7af6SNeel Natu 
1850318224bbSNeel Natu 	/* EPT fault must be a read fault or a write fault */
1851a2da7af6SNeel Natu 	read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0;
1852a2da7af6SNeel Natu 	write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0;
18533b2b0011SPeter Grehan 	if ((read | write) == 0)
1854318224bbSNeel Natu 		return (FALSE);
1855a2da7af6SNeel Natu 
1856a2da7af6SNeel Natu 	/*
18573b2b0011SPeter Grehan 	 * The EPT violation must have been caused by accessing a
18583b2b0011SPeter Grehan 	 * guest-physical address that is a translation of a guest-linear
18593b2b0011SPeter Grehan 	 * address.
1860a2da7af6SNeel Natu 	 */
1861a2da7af6SNeel Natu 	if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 ||
1862a2da7af6SNeel Natu 	    (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) {
1863318224bbSNeel Natu 		return (FALSE);
1864a2da7af6SNeel Natu 	}
1865a2da7af6SNeel Natu 
1866318224bbSNeel Natu 	return (TRUE);
1867a2da7af6SNeel Natu }
1868a2da7af6SNeel Natu 
1869159dd56fSNeel Natu static __inline int
1870159dd56fSNeel Natu apic_access_virtualization(struct vmx *vmx, int vcpuid)
1871159dd56fSNeel Natu {
1872159dd56fSNeel Natu 	uint32_t proc_ctls2;
1873159dd56fSNeel Natu 
1874159dd56fSNeel Natu 	proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
1875159dd56fSNeel Natu 	return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) ? 1 : 0);
1876159dd56fSNeel Natu }
1877159dd56fSNeel Natu 
1878159dd56fSNeel Natu static __inline int
1879159dd56fSNeel Natu x2apic_virtualization(struct vmx *vmx, int vcpuid)
1880159dd56fSNeel Natu {
1881159dd56fSNeel Natu 	uint32_t proc_ctls2;
1882159dd56fSNeel Natu 
1883159dd56fSNeel Natu 	proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
1884159dd56fSNeel Natu 	return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_X2APIC_MODE) ? 1 : 0);
1885159dd56fSNeel Natu }
1886159dd56fSNeel Natu 
1887a2da7af6SNeel Natu static int
1888159dd56fSNeel Natu vmx_handle_apic_write(struct vmx *vmx, int vcpuid, struct vlapic *vlapic,
1889159dd56fSNeel Natu     uint64_t qual)
189088c4b8d1SNeel Natu {
189188c4b8d1SNeel Natu 	int error, handled, offset;
1892159dd56fSNeel Natu 	uint32_t *apic_regs, vector;
189388c4b8d1SNeel Natu 	bool retu;
189488c4b8d1SNeel Natu 
1895a0efd3fbSJohn Baldwin 	handled = HANDLED;
189688c4b8d1SNeel Natu 	offset = APIC_WRITE_OFFSET(qual);
1897159dd56fSNeel Natu 
1898159dd56fSNeel Natu 	if (!apic_access_virtualization(vmx, vcpuid)) {
1899159dd56fSNeel Natu 		/*
1900159dd56fSNeel Natu 		 * In general there should not be any APIC write VM-exits
1901159dd56fSNeel Natu 		 * unless APIC-access virtualization is enabled.
1902159dd56fSNeel Natu 		 *
1903159dd56fSNeel Natu 		 * However self-IPI virtualization can legitimately trigger
1904159dd56fSNeel Natu 		 * an APIC-write VM-exit so treat it specially.
1905159dd56fSNeel Natu 		 */
1906159dd56fSNeel Natu 		if (x2apic_virtualization(vmx, vcpuid) &&
1907159dd56fSNeel Natu 		    offset == APIC_OFFSET_SELF_IPI) {
1908159dd56fSNeel Natu 			apic_regs = (uint32_t *)(vlapic->apic_page);
1909159dd56fSNeel Natu 			vector = apic_regs[APIC_OFFSET_SELF_IPI / 4];
1910159dd56fSNeel Natu 			vlapic_self_ipi_handler(vlapic, vector);
1911159dd56fSNeel Natu 			return (HANDLED);
1912159dd56fSNeel Natu 		} else
1913159dd56fSNeel Natu 			return (UNHANDLED);
1914159dd56fSNeel Natu 	}
1915159dd56fSNeel Natu 
191688c4b8d1SNeel Natu 	switch (offset) {
191788c4b8d1SNeel Natu 	case APIC_OFFSET_ID:
191888c4b8d1SNeel Natu 		vlapic_id_write_handler(vlapic);
191988c4b8d1SNeel Natu 		break;
192088c4b8d1SNeel Natu 	case APIC_OFFSET_LDR:
192188c4b8d1SNeel Natu 		vlapic_ldr_write_handler(vlapic);
192288c4b8d1SNeel Natu 		break;
192388c4b8d1SNeel Natu 	case APIC_OFFSET_DFR:
192488c4b8d1SNeel Natu 		vlapic_dfr_write_handler(vlapic);
192588c4b8d1SNeel Natu 		break;
192688c4b8d1SNeel Natu 	case APIC_OFFSET_SVR:
192788c4b8d1SNeel Natu 		vlapic_svr_write_handler(vlapic);
192888c4b8d1SNeel Natu 		break;
192988c4b8d1SNeel Natu 	case APIC_OFFSET_ESR:
193088c4b8d1SNeel Natu 		vlapic_esr_write_handler(vlapic);
193188c4b8d1SNeel Natu 		break;
193288c4b8d1SNeel Natu 	case APIC_OFFSET_ICR_LOW:
193388c4b8d1SNeel Natu 		retu = false;
193488c4b8d1SNeel Natu 		error = vlapic_icrlo_write_handler(vlapic, &retu);
193588c4b8d1SNeel Natu 		if (error != 0 || retu)
1936a0efd3fbSJohn Baldwin 			handled = UNHANDLED;
193788c4b8d1SNeel Natu 		break;
193888c4b8d1SNeel Natu 	case APIC_OFFSET_CMCI_LVT:
193988c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
194088c4b8d1SNeel Natu 		vlapic_lvt_write_handler(vlapic, offset);
194188c4b8d1SNeel Natu 		break;
194288c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_ICR:
194388c4b8d1SNeel Natu 		vlapic_icrtmr_write_handler(vlapic);
194488c4b8d1SNeel Natu 		break;
194588c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_DCR:
194688c4b8d1SNeel Natu 		vlapic_dcr_write_handler(vlapic);
194788c4b8d1SNeel Natu 		break;
194888c4b8d1SNeel Natu 	default:
1949a0efd3fbSJohn Baldwin 		handled = UNHANDLED;
195088c4b8d1SNeel Natu 		break;
195188c4b8d1SNeel Natu 	}
195288c4b8d1SNeel Natu 	return (handled);
195388c4b8d1SNeel Natu }
195488c4b8d1SNeel Natu 
195588c4b8d1SNeel Natu static bool
1956159dd56fSNeel Natu apic_access_fault(struct vmx *vmx, int vcpuid, uint64_t gpa)
195788c4b8d1SNeel Natu {
195888c4b8d1SNeel Natu 
1959159dd56fSNeel Natu 	if (apic_access_virtualization(vmx, vcpuid) &&
196088c4b8d1SNeel Natu 	    (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE))
196188c4b8d1SNeel Natu 		return (true);
196288c4b8d1SNeel Natu 	else
196388c4b8d1SNeel Natu 		return (false);
196488c4b8d1SNeel Natu }
196588c4b8d1SNeel Natu 
196688c4b8d1SNeel Natu static int
196788c4b8d1SNeel Natu vmx_handle_apic_access(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
196888c4b8d1SNeel Natu {
196988c4b8d1SNeel Natu 	uint64_t qual;
197088c4b8d1SNeel Natu 	int access_type, offset, allowed;
197188c4b8d1SNeel Natu 
1972159dd56fSNeel Natu 	if (!apic_access_virtualization(vmx, vcpuid))
197388c4b8d1SNeel Natu 		return (UNHANDLED);
197488c4b8d1SNeel Natu 
197588c4b8d1SNeel Natu 	qual = vmexit->u.vmx.exit_qualification;
197688c4b8d1SNeel Natu 	access_type = APIC_ACCESS_TYPE(qual);
197788c4b8d1SNeel Natu 	offset = APIC_ACCESS_OFFSET(qual);
197888c4b8d1SNeel Natu 
197988c4b8d1SNeel Natu 	allowed = 0;
198088c4b8d1SNeel Natu 	if (access_type == 0) {
198188c4b8d1SNeel Natu 		/*
198288c4b8d1SNeel Natu 		 * Read data access to the following registers is expected.
198388c4b8d1SNeel Natu 		 */
198488c4b8d1SNeel Natu 		switch (offset) {
198588c4b8d1SNeel Natu 		case APIC_OFFSET_APR:
198688c4b8d1SNeel Natu 		case APIC_OFFSET_PPR:
198788c4b8d1SNeel Natu 		case APIC_OFFSET_RRR:
198888c4b8d1SNeel Natu 		case APIC_OFFSET_CMCI_LVT:
198988c4b8d1SNeel Natu 		case APIC_OFFSET_TIMER_CCR:
199088c4b8d1SNeel Natu 			allowed = 1;
199188c4b8d1SNeel Natu 			break;
199288c4b8d1SNeel Natu 		default:
199388c4b8d1SNeel Natu 			break;
199488c4b8d1SNeel Natu 		}
199588c4b8d1SNeel Natu 	} else if (access_type == 1) {
199688c4b8d1SNeel Natu 		/*
199788c4b8d1SNeel Natu 		 * Write data access to the following registers is expected.
199888c4b8d1SNeel Natu 		 */
199988c4b8d1SNeel Natu 		switch (offset) {
200088c4b8d1SNeel Natu 		case APIC_OFFSET_VER:
200188c4b8d1SNeel Natu 		case APIC_OFFSET_APR:
200288c4b8d1SNeel Natu 		case APIC_OFFSET_PPR:
200388c4b8d1SNeel Natu 		case APIC_OFFSET_RRR:
200488c4b8d1SNeel Natu 		case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
200588c4b8d1SNeel Natu 		case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
200688c4b8d1SNeel Natu 		case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
200788c4b8d1SNeel Natu 		case APIC_OFFSET_CMCI_LVT:
200888c4b8d1SNeel Natu 		case APIC_OFFSET_TIMER_CCR:
200988c4b8d1SNeel Natu 			allowed = 1;
201088c4b8d1SNeel Natu 			break;
201188c4b8d1SNeel Natu 		default:
201288c4b8d1SNeel Natu 			break;
201388c4b8d1SNeel Natu 		}
201488c4b8d1SNeel Natu 	}
201588c4b8d1SNeel Natu 
201688c4b8d1SNeel Natu 	if (allowed) {
2017e4c8a13dSNeel Natu 		vmexit_inst_emul(vmexit, DEFAULT_APIC_BASE + offset,
2018e4c8a13dSNeel Natu 		    VIE_INVALID_GLA);
201988c4b8d1SNeel Natu 	}
202088c4b8d1SNeel Natu 
202188c4b8d1SNeel Natu 	/*
202288c4b8d1SNeel Natu 	 * Regardless of whether the APIC-access is allowed this handler
202388c4b8d1SNeel Natu 	 * always returns UNHANDLED:
202488c4b8d1SNeel Natu 	 * - if the access is allowed then it is handled by emulating the
202588c4b8d1SNeel Natu 	 *   instruction that caused the VM-exit (outside the critical section)
202688c4b8d1SNeel Natu 	 * - if the access is not allowed then it will be converted to an
202788c4b8d1SNeel Natu 	 *   exitcode of VM_EXITCODE_VMX and will be dealt with in userland.
202888c4b8d1SNeel Natu 	 */
202988c4b8d1SNeel Natu 	return (UNHANDLED);
203088c4b8d1SNeel Natu }
203188c4b8d1SNeel Natu 
20323d5444c8SNeel Natu static enum task_switch_reason
20333d5444c8SNeel Natu vmx_task_switch_reason(uint64_t qual)
20343d5444c8SNeel Natu {
20353d5444c8SNeel Natu 	int reason;
20363d5444c8SNeel Natu 
20373d5444c8SNeel Natu 	reason = (qual >> 30) & 0x3;
20383d5444c8SNeel Natu 	switch (reason) {
20393d5444c8SNeel Natu 	case 0:
20403d5444c8SNeel Natu 		return (TSR_CALL);
20413d5444c8SNeel Natu 	case 1:
20423d5444c8SNeel Natu 		return (TSR_IRET);
20433d5444c8SNeel Natu 	case 2:
20443d5444c8SNeel Natu 		return (TSR_JMP);
20453d5444c8SNeel Natu 	case 3:
20463d5444c8SNeel Natu 		return (TSR_IDT_GATE);
20473d5444c8SNeel Natu 	default:
20483d5444c8SNeel Natu 		panic("%s: invalid reason %d", __func__, reason);
20493d5444c8SNeel Natu 	}
20503d5444c8SNeel Natu }
20513d5444c8SNeel Natu 
205288c4b8d1SNeel Natu static int
2053c3498942SNeel Natu emulate_wrmsr(struct vmx *vmx, int vcpuid, u_int num, uint64_t val, bool *retu)
2054c3498942SNeel Natu {
2055c3498942SNeel Natu 	int error;
2056c3498942SNeel Natu 
2057c3498942SNeel Natu 	if (lapic_msr(num))
2058c3498942SNeel Natu 		error = lapic_wrmsr(vmx->vm, vcpuid, num, val, retu);
2059c3498942SNeel Natu 	else
2060c3498942SNeel Natu 		error = vmx_wrmsr(vmx, vcpuid, num, val, retu);
2061c3498942SNeel Natu 
2062c3498942SNeel Natu 	return (error);
2063c3498942SNeel Natu }
2064c3498942SNeel Natu 
2065c3498942SNeel Natu static int
2066c3498942SNeel Natu emulate_rdmsr(struct vmx *vmx, int vcpuid, u_int num, bool *retu)
2067c3498942SNeel Natu {
2068c3498942SNeel Natu 	struct vmxctx *vmxctx;
2069c3498942SNeel Natu 	uint64_t result;
2070c3498942SNeel Natu 	uint32_t eax, edx;
2071c3498942SNeel Natu 	int error;
2072c3498942SNeel Natu 
2073c3498942SNeel Natu 	if (lapic_msr(num))
2074c3498942SNeel Natu 		error = lapic_rdmsr(vmx->vm, vcpuid, num, &result, retu);
2075c3498942SNeel Natu 	else
2076c3498942SNeel Natu 		error = vmx_rdmsr(vmx, vcpuid, num, &result, retu);
2077c3498942SNeel Natu 
2078c3498942SNeel Natu 	if (error == 0) {
2079c3498942SNeel Natu 		eax = result;
2080c3498942SNeel Natu 		vmxctx = &vmx->ctx[vcpuid];
2081c3498942SNeel Natu 		error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RAX, eax);
2082c3498942SNeel Natu 		KASSERT(error == 0, ("vmxctx_setreg(rax) error %d", error));
2083c3498942SNeel Natu 
2084c3498942SNeel Natu 		edx = result >> 32;
2085c3498942SNeel Natu 		error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RDX, edx);
2086c3498942SNeel Natu 		KASSERT(error == 0, ("vmxctx_setreg(rdx) error %d", error));
2087c3498942SNeel Natu 	}
2088c3498942SNeel Natu 
2089c3498942SNeel Natu 	return (error);
2090c3498942SNeel Natu }
2091c3498942SNeel Natu 
2092c3498942SNeel Natu static int
2093366f6083SPeter Grehan vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
2094366f6083SPeter Grehan {
2095c9c75df4SNeel Natu 	int error, errcode, errcode_valid, handled, in;
2096366f6083SPeter Grehan 	struct vmxctx *vmxctx;
209788c4b8d1SNeel Natu 	struct vlapic *vlapic;
2098d17b5104SNeel Natu 	struct vm_inout_str *vis;
20993d5444c8SNeel Natu 	struct vm_task_switch *ts;
2100d17b5104SNeel Natu 	uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, inst_info;
2101b0538143SNeel Natu 	uint32_t intr_type, intr_vec, reason;
2102091d4532SNeel Natu 	uint64_t exitintinfo, qual, gpa;
2103becd9849SNeel Natu 	bool retu;
2104366f6083SPeter Grehan 
2105160471d2SNeel Natu 	CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0);
2106c308b23bSNeel Natu 	CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0);
2107160471d2SNeel Natu 
2108a0efd3fbSJohn Baldwin 	handled = UNHANDLED;
2109366f6083SPeter Grehan 	vmxctx = &vmx->ctx[vcpu];
21100492757cSNeel Natu 
2111366f6083SPeter Grehan 	qual = vmexit->u.vmx.exit_qualification;
2112318224bbSNeel Natu 	reason = vmexit->u.vmx.exit_reason;
2113366f6083SPeter Grehan 	vmexit->exitcode = VM_EXITCODE_BOGUS;
2114366f6083SPeter Grehan 
211561592433SNeel Natu 	vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1);
211661592433SNeel Natu 
2117318224bbSNeel Natu 	/*
2118b0538143SNeel Natu 	 * VM-entry failures during or after loading guest state.
2119b0538143SNeel Natu 	 *
2120b0538143SNeel Natu 	 * These VM-exits are uncommon but must be handled specially
2121b0538143SNeel Natu 	 * as most VM-exit fields are not populated as usual.
2122b0538143SNeel Natu 	 */
2123b0538143SNeel Natu 	if (__predict_false(reason == EXIT_REASON_MCE_DURING_ENTRY)) {
2124b0538143SNeel Natu 		VCPU_CTR0(vmx->vm, vcpu, "Handling MCE during VM-entry");
2125b0538143SNeel Natu 		__asm __volatile("int $18");
2126b0538143SNeel Natu 		return (1);
2127b0538143SNeel Natu 	}
2128b0538143SNeel Natu 
2129b0538143SNeel Natu 	/*
21303d5444c8SNeel Natu 	 * VM exits that can be triggered during event delivery need to
21313d5444c8SNeel Natu 	 * be handled specially by re-injecting the event if the IDT
21323d5444c8SNeel Natu 	 * vectoring information field's valid bit is set.
2133318224bbSNeel Natu 	 *
2134318224bbSNeel Natu 	 * See "Information for VM Exits During Event Delivery" in Intel SDM
2135318224bbSNeel Natu 	 * for details.
2136318224bbSNeel Natu 	 */
2137318224bbSNeel Natu 	idtvec_info = vmcs_idt_vectoring_info();
2138318224bbSNeel Natu 	if (idtvec_info & VMCS_IDT_VEC_VALID) {
2139318224bbSNeel Natu 		idtvec_info &= ~(1 << 12); /* clear undefined bit */
2140091d4532SNeel Natu 		exitintinfo = idtvec_info;
2141318224bbSNeel Natu 		if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
2142318224bbSNeel Natu 			idtvec_err = vmcs_idt_vectoring_err();
2143091d4532SNeel Natu 			exitintinfo |= (uint64_t)idtvec_err << 32;
2144318224bbSNeel Natu 		}
2145091d4532SNeel Natu 		error = vm_exit_intinfo(vmx->vm, vcpu, exitintinfo);
2146091d4532SNeel Natu 		KASSERT(error == 0, ("%s: vm_set_intinfo error %d",
2147091d4532SNeel Natu 		    __func__, error));
2148091d4532SNeel Natu 
2149160471d2SNeel Natu 		/*
2150160471d2SNeel Natu 		 * If 'virtual NMIs' are being used and the VM-exit
2151160471d2SNeel Natu 		 * happened while injecting an NMI during the previous
2152091d4532SNeel Natu 		 * VM-entry, then clear "blocking by NMI" in the
2153091d4532SNeel Natu 		 * Guest Interruptibility-State so the NMI can be
2154091d4532SNeel Natu 		 * reinjected on the subsequent VM-entry.
2155091d4532SNeel Natu 		 *
2156091d4532SNeel Natu 		 * However, if the NMI was being delivered through a task
2157091d4532SNeel Natu 		 * gate, then the new task must start execution with NMIs
2158091d4532SNeel Natu 		 * blocked so don't clear NMI blocking in this case.
2159160471d2SNeel Natu 		 */
2160091d4532SNeel Natu 		intr_type = idtvec_info & VMCS_INTR_T_MASK;
2161091d4532SNeel Natu 		if (intr_type == VMCS_INTR_T_NMI) {
2162091d4532SNeel Natu 			if (reason != EXIT_REASON_TASK_SWITCH)
2163e5a1d950SNeel Natu 				vmx_clear_nmi_blocking(vmx, vcpu);
2164091d4532SNeel Natu 			else
2165091d4532SNeel Natu 				vmx_assert_nmi_blocking(vmx, vcpu);
2166160471d2SNeel Natu 		}
2167091d4532SNeel Natu 
2168091d4532SNeel Natu 		/*
2169091d4532SNeel Natu 		 * Update VM-entry instruction length if the event being
2170091d4532SNeel Natu 		 * delivered was a software interrupt or software exception.
2171091d4532SNeel Natu 		 */
2172091d4532SNeel Natu 		if (intr_type == VMCS_INTR_T_SWINTR ||
2173091d4532SNeel Natu 		    intr_type == VMCS_INTR_T_PRIV_SWEXCEPTION ||
2174091d4532SNeel Natu 		    intr_type == VMCS_INTR_T_SWEXCEPTION) {
21753de83862SNeel Natu 			vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
2176318224bbSNeel Natu 		}
2177318224bbSNeel Natu 	}
2178318224bbSNeel Natu 
2179318224bbSNeel Natu 	switch (reason) {
21803d5444c8SNeel Natu 	case EXIT_REASON_TASK_SWITCH:
21813d5444c8SNeel Natu 		ts = &vmexit->u.task_switch;
21823d5444c8SNeel Natu 		ts->tsssel = qual & 0xffff;
21833d5444c8SNeel Natu 		ts->reason = vmx_task_switch_reason(qual);
21843d5444c8SNeel Natu 		ts->ext = 0;
21853d5444c8SNeel Natu 		ts->errcode_valid = 0;
21863d5444c8SNeel Natu 		vmx_paging_info(&ts->paging);
21873d5444c8SNeel Natu 		/*
21883d5444c8SNeel Natu 		 * If the task switch was due to a CALL, JMP, IRET, software
21893d5444c8SNeel Natu 		 * interrupt (INT n) or software exception (INT3, INTO),
21903d5444c8SNeel Natu 		 * then the saved %rip references the instruction that caused
21913d5444c8SNeel Natu 		 * the task switch. The instruction length field in the VMCS
21923d5444c8SNeel Natu 		 * is valid in this case.
21933d5444c8SNeel Natu 		 *
21943d5444c8SNeel Natu 		 * In all other cases (e.g., NMI, hardware exception) the
21953d5444c8SNeel Natu 		 * saved %rip is one that would have been saved in the old TSS
21963d5444c8SNeel Natu 		 * had the task switch completed normally so the instruction
21973d5444c8SNeel Natu 		 * length field is not needed in this case and is explicitly
21983d5444c8SNeel Natu 		 * set to 0.
21993d5444c8SNeel Natu 		 */
22003d5444c8SNeel Natu 		if (ts->reason == TSR_IDT_GATE) {
22013d5444c8SNeel Natu 			KASSERT(idtvec_info & VMCS_IDT_VEC_VALID,
2202091d4532SNeel Natu 			    ("invalid idtvec_info %#x for IDT task switch",
22033d5444c8SNeel Natu 			    idtvec_info));
22043d5444c8SNeel Natu 			intr_type = idtvec_info & VMCS_INTR_T_MASK;
22053d5444c8SNeel Natu 			if (intr_type != VMCS_INTR_T_SWINTR &&
22063d5444c8SNeel Natu 			    intr_type != VMCS_INTR_T_SWEXCEPTION &&
22073d5444c8SNeel Natu 			    intr_type != VMCS_INTR_T_PRIV_SWEXCEPTION) {
22083d5444c8SNeel Natu 				/* Task switch triggered by external event */
22093d5444c8SNeel Natu 				ts->ext = 1;
22103d5444c8SNeel Natu 				vmexit->inst_length = 0;
22113d5444c8SNeel Natu 				if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
22123d5444c8SNeel Natu 					ts->errcode_valid = 1;
22133d5444c8SNeel Natu 					ts->errcode = vmcs_idt_vectoring_err();
22143d5444c8SNeel Natu 				}
22153d5444c8SNeel Natu 			}
22163d5444c8SNeel Natu 		}
22173d5444c8SNeel Natu 		vmexit->exitcode = VM_EXITCODE_TASK_SWITCH;
22183d5444c8SNeel Natu 		VCPU_CTR4(vmx->vm, vcpu, "task switch reason %d, tss 0x%04x, "
22193d5444c8SNeel Natu 		    "%s errcode 0x%016lx", ts->reason, ts->tsssel,
22203d5444c8SNeel Natu 		    ts->ext ? "external" : "internal",
22213d5444c8SNeel Natu 		    ((uint64_t)ts->errcode << 32) | ts->errcode_valid);
22223d5444c8SNeel Natu 		break;
2223366f6083SPeter Grehan 	case EXIT_REASON_CR_ACCESS:
2224b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CR_ACCESS, 1);
2225594db002STycho Nightingale 		switch (qual & 0xf) {
2226594db002STycho Nightingale 		case 0:
2227594db002STycho Nightingale 			handled = vmx_emulate_cr0_access(vmx, vcpu, qual);
2228594db002STycho Nightingale 			break;
2229594db002STycho Nightingale 		case 4:
2230594db002STycho Nightingale 			handled = vmx_emulate_cr4_access(vmx, vcpu, qual);
2231594db002STycho Nightingale 			break;
2232594db002STycho Nightingale 		case 8:
2233594db002STycho Nightingale 			handled = vmx_emulate_cr8_access(vmx, vcpu, qual);
2234594db002STycho Nightingale 			break;
2235594db002STycho Nightingale 		}
2236366f6083SPeter Grehan 		break;
2237366f6083SPeter Grehan 	case EXIT_REASON_RDMSR:
2238b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RDMSR, 1);
2239becd9849SNeel Natu 		retu = false;
2240366f6083SPeter Grehan 		ecx = vmxctx->guest_rcx;
22412cb97c9dSNeel Natu 		VCPU_CTR1(vmx->vm, vcpu, "rdmsr 0x%08x", ecx);
2242c3498942SNeel Natu 		error = emulate_rdmsr(vmx, vcpu, ecx, &retu);
2243b42206f3SNeel Natu 		if (error) {
2244366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_RDMSR;
2245366f6083SPeter Grehan 			vmexit->u.msr.code = ecx;
2246becd9849SNeel Natu 		} else if (!retu) {
2247a0efd3fbSJohn Baldwin 			handled = HANDLED;
2248becd9849SNeel Natu 		} else {
2249becd9849SNeel Natu 			/* Return to userspace with a valid exitcode */
2250becd9849SNeel Natu 			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
2251c3498942SNeel Natu 			    ("emulate_rdmsr retu with bogus exitcode"));
2252becd9849SNeel Natu 		}
2253366f6083SPeter Grehan 		break;
2254366f6083SPeter Grehan 	case EXIT_REASON_WRMSR:
2255b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_WRMSR, 1);
2256becd9849SNeel Natu 		retu = false;
2257366f6083SPeter Grehan 		eax = vmxctx->guest_rax;
2258366f6083SPeter Grehan 		ecx = vmxctx->guest_rcx;
2259366f6083SPeter Grehan 		edx = vmxctx->guest_rdx;
22602cb97c9dSNeel Natu 		VCPU_CTR2(vmx->vm, vcpu, "wrmsr 0x%08x value 0x%016lx",
22612cb97c9dSNeel Natu 		    ecx, (uint64_t)edx << 32 | eax);
2262c3498942SNeel Natu 		error = emulate_wrmsr(vmx, vcpu, ecx,
2263becd9849SNeel Natu 		    (uint64_t)edx << 32 | eax, &retu);
2264b42206f3SNeel Natu 		if (error) {
2265366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_WRMSR;
2266366f6083SPeter Grehan 			vmexit->u.msr.code = ecx;
2267366f6083SPeter Grehan 			vmexit->u.msr.wval = (uint64_t)edx << 32 | eax;
2268becd9849SNeel Natu 		} else if (!retu) {
2269a0efd3fbSJohn Baldwin 			handled = HANDLED;
2270becd9849SNeel Natu 		} else {
2271becd9849SNeel Natu 			/* Return to userspace with a valid exitcode */
2272becd9849SNeel Natu 			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
2273becd9849SNeel Natu 			    ("emulate_wrmsr retu with bogus exitcode"));
2274becd9849SNeel Natu 		}
2275366f6083SPeter Grehan 		break;
2276366f6083SPeter Grehan 	case EXIT_REASON_HLT:
2277f76fc5d4SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_HLT, 1);
2278366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_HLT;
22793de83862SNeel Natu 		vmexit->u.hlt.rflags = vmcs_read(VMCS_GUEST_RFLAGS);
2280366f6083SPeter Grehan 		break;
2281366f6083SPeter Grehan 	case EXIT_REASON_MTF:
2282b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_MTRAP, 1);
2283366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_MTRAP;
2284c9c75df4SNeel Natu 		vmexit->inst_length = 0;
2285366f6083SPeter Grehan 		break;
2286366f6083SPeter Grehan 	case EXIT_REASON_PAUSE:
2287b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_PAUSE, 1);
2288366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_PAUSE;
2289366f6083SPeter Grehan 		break;
2290366f6083SPeter Grehan 	case EXIT_REASON_INTR_WINDOW:
2291b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INTR_WINDOW, 1);
2292366f6083SPeter Grehan 		vmx_clear_int_window_exiting(vmx, vcpu);
2293b5aaf7b2SNeel Natu 		return (1);
2294366f6083SPeter Grehan 	case EXIT_REASON_EXT_INTR:
2295366f6083SPeter Grehan 		/*
2296366f6083SPeter Grehan 		 * External interrupts serve only to cause VM exits and allow
2297366f6083SPeter Grehan 		 * the host interrupt handler to run.
2298366f6083SPeter Grehan 		 *
2299366f6083SPeter Grehan 		 * If this external interrupt triggers a virtual interrupt
2300366f6083SPeter Grehan 		 * to a VM, then that state will be recorded by the
2301366f6083SPeter Grehan 		 * host interrupt handler in the VM's softc. We will inject
2302366f6083SPeter Grehan 		 * this virtual interrupt during the subsequent VM enter.
2303366f6083SPeter Grehan 		 */
2304f7d47425SNeel Natu 		intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
2305722b6744SJohn Baldwin 
2306722b6744SJohn Baldwin 		/*
2307722b6744SJohn Baldwin 		 * XXX: Ignore this exit if VMCS_INTR_VALID is not set.
2308ad3e3687SJohn Baldwin 		 * This appears to be a bug in VMware Fusion?
2309722b6744SJohn Baldwin 		 */
2310722b6744SJohn Baldwin 		if (!(intr_info & VMCS_INTR_VALID))
2311722b6744SJohn Baldwin 			return (1);
2312160471d2SNeel Natu 		KASSERT((intr_info & VMCS_INTR_VALID) != 0 &&
2313160471d2SNeel Natu 		    (intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_HWINTR,
2314f7d47425SNeel Natu 		    ("VM exit interruption info invalid: %#x", intr_info));
2315f7d47425SNeel Natu 		vmx_trigger_hostintr(intr_info & 0xff);
2316366f6083SPeter Grehan 
2317366f6083SPeter Grehan 		/*
2318366f6083SPeter Grehan 		 * This is special. We want to treat this as an 'handled'
2319366f6083SPeter Grehan 		 * VM-exit but not increment the instruction pointer.
2320366f6083SPeter Grehan 		 */
2321366f6083SPeter Grehan 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1);
2322366f6083SPeter Grehan 		return (1);
2323366f6083SPeter Grehan 	case EXIT_REASON_NMI_WINDOW:
2324366f6083SPeter Grehan 		/* Exit to allow the pending virtual NMI to be injected */
232548b2d828SNeel Natu 		if (vm_nmi_pending(vmx->vm, vcpu))
232648b2d828SNeel Natu 			vmx_inject_nmi(vmx, vcpu);
2327366f6083SPeter Grehan 		vmx_clear_nmi_window_exiting(vmx, vcpu);
232848b2d828SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NMI_WINDOW, 1);
2329366f6083SPeter Grehan 		return (1);
2330366f6083SPeter Grehan 	case EXIT_REASON_INOUT:
2331b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INOUT, 1);
2332366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_INOUT;
2333366f6083SPeter Grehan 		vmexit->u.inout.bytes = (qual & 0x7) + 1;
2334d17b5104SNeel Natu 		vmexit->u.inout.in = in = (qual & 0x8) ? 1 : 0;
2335366f6083SPeter Grehan 		vmexit->u.inout.string = (qual & 0x10) ? 1 : 0;
2336366f6083SPeter Grehan 		vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0;
2337366f6083SPeter Grehan 		vmexit->u.inout.port = (uint16_t)(qual >> 16);
2338366f6083SPeter Grehan 		vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax);
2339d17b5104SNeel Natu 		if (vmexit->u.inout.string) {
2340d17b5104SNeel Natu 			inst_info = vmcs_read(VMCS_EXIT_INSTRUCTION_INFO);
2341d17b5104SNeel Natu 			vmexit->exitcode = VM_EXITCODE_INOUT_STR;
2342d17b5104SNeel Natu 			vis = &vmexit->u.inout_str;
2343e813a873SNeel Natu 			vmx_paging_info(&vis->paging);
2344d17b5104SNeel Natu 			vis->rflags = vmcs_read(VMCS_GUEST_RFLAGS);
2345d17b5104SNeel Natu 			vis->cr0 = vmcs_read(VMCS_GUEST_CR0);
2346d17b5104SNeel Natu 			vis->index = inout_str_index(vmx, vcpu, in);
2347d17b5104SNeel Natu 			vis->count = inout_str_count(vmx, vcpu, vis->inout.rep);
2348d17b5104SNeel Natu 			vis->addrsize = inout_str_addrsize(inst_info);
2349d17b5104SNeel Natu 			inout_str_seginfo(vmx, vcpu, inst_info, in, vis);
2350762fd208STycho Nightingale 		}
2351366f6083SPeter Grehan 		break;
2352366f6083SPeter Grehan 	case EXIT_REASON_CPUID:
2353b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CPUID, 1);
2354a2da7af6SNeel Natu 		handled = vmx_handle_cpuid(vmx->vm, vcpu, vmxctx);
2355366f6083SPeter Grehan 		break;
2356e5a1d950SNeel Natu 	case EXIT_REASON_EXCEPTION:
2357c308b23bSNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXCEPTION, 1);
2358e5a1d950SNeel Natu 		intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
2359e5a1d950SNeel Natu 		KASSERT((intr_info & VMCS_INTR_VALID) != 0,
2360e5a1d950SNeel Natu 		    ("VM exit interruption info invalid: %#x", intr_info));
2361c308b23bSNeel Natu 
2362b0538143SNeel Natu 		intr_vec = intr_info & 0xff;
2363b0538143SNeel Natu 		intr_type = intr_info & VMCS_INTR_T_MASK;
2364b0538143SNeel Natu 
2365e5a1d950SNeel Natu 		/*
2366e5a1d950SNeel Natu 		 * If Virtual NMIs control is 1 and the VM-exit is due to a
2367e5a1d950SNeel Natu 		 * fault encountered during the execution of IRET then we must
2368e5a1d950SNeel Natu 		 * restore the state of "virtual-NMI blocking" before resuming
2369e5a1d950SNeel Natu 		 * the guest.
2370e5a1d950SNeel Natu 		 *
2371e5a1d950SNeel Natu 		 * See "Resuming Guest Software after Handling an Exception".
2372091d4532SNeel Natu 		 * See "Information for VM Exits Due to Vectored Events".
2373e5a1d950SNeel Natu 		 */
2374e5a1d950SNeel Natu 		if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
2375b0538143SNeel Natu 		    (intr_vec != IDT_DF) &&
2376e5a1d950SNeel Natu 		    (intr_info & EXIT_QUAL_NMIUDTI) != 0)
2377e5a1d950SNeel Natu 			vmx_restore_nmi_blocking(vmx, vcpu);
2378c308b23bSNeel Natu 
2379c308b23bSNeel Natu 		/*
238062fbd7c2SNeel Natu 		 * The NMI has already been handled in vmx_exit_handle_nmi().
2381c308b23bSNeel Natu 		 */
2382b0538143SNeel Natu 		if (intr_type == VMCS_INTR_T_NMI)
2383c308b23bSNeel Natu 			return (1);
2384b0538143SNeel Natu 
2385b0538143SNeel Natu 		/*
2386b0538143SNeel Natu 		 * Call the machine check handler by hand. Also don't reflect
2387b0538143SNeel Natu 		 * the machine check back into the guest.
2388b0538143SNeel Natu 		 */
2389b0538143SNeel Natu 		if (intr_vec == IDT_MC) {
2390b0538143SNeel Natu 			VCPU_CTR0(vmx->vm, vcpu, "Vectoring to MCE handler");
2391b0538143SNeel Natu 			__asm __volatile("int $18");
2392b0538143SNeel Natu 			return (1);
2393b0538143SNeel Natu 		}
2394b0538143SNeel Natu 
2395b0538143SNeel Natu 		if (intr_vec == IDT_PF) {
2396b0538143SNeel Natu 			error = vmxctx_setreg(vmxctx, VM_REG_GUEST_CR2, qual);
2397b0538143SNeel Natu 			KASSERT(error == 0, ("%s: vmxctx_setreg(cr2) error %d",
2398b0538143SNeel Natu 			    __func__, error));
2399b0538143SNeel Natu 		}
2400b0538143SNeel Natu 
2401b0538143SNeel Natu 		/*
2402b0538143SNeel Natu 		 * Software exceptions exhibit trap-like behavior. This in
2403b0538143SNeel Natu 		 * turn requires populating the VM-entry instruction length
2404b0538143SNeel Natu 		 * so that the %rip in the trap frame is past the INT3/INTO
2405b0538143SNeel Natu 		 * instruction.
2406b0538143SNeel Natu 		 */
2407b0538143SNeel Natu 		if (intr_type == VMCS_INTR_T_SWEXCEPTION)
2408b0538143SNeel Natu 			vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
2409b0538143SNeel Natu 
2410b0538143SNeel Natu 		/* Reflect all other exceptions back into the guest */
2411c9c75df4SNeel Natu 		errcode_valid = errcode = 0;
2412b0538143SNeel Natu 		if (intr_info & VMCS_INTR_DEL_ERRCODE) {
2413c9c75df4SNeel Natu 			errcode_valid = 1;
2414c9c75df4SNeel Natu 			errcode = vmcs_read(VMCS_EXIT_INTR_ERRCODE);
2415b0538143SNeel Natu 		}
2416b0538143SNeel Natu 		VCPU_CTR2(vmx->vm, vcpu, "Reflecting exception %d/%#x into "
2417c9c75df4SNeel Natu 		    "the guest", intr_vec, errcode);
2418c9c75df4SNeel Natu 		error = vm_inject_exception(vmx->vm, vcpu, intr_vec,
2419c9c75df4SNeel Natu 		    errcode_valid, errcode, 0);
2420b0538143SNeel Natu 		KASSERT(error == 0, ("%s: vm_inject_exception error %d",
2421b0538143SNeel Natu 		    __func__, error));
2422b0538143SNeel Natu 		return (1);
2423b0538143SNeel Natu 
2424cd942e0fSPeter Grehan 	case EXIT_REASON_EPT_FAULT:
2425318224bbSNeel Natu 		/*
2426318224bbSNeel Natu 		 * If 'gpa' lies within the address space allocated to
2427318224bbSNeel Natu 		 * memory then this must be a nested page fault otherwise
2428318224bbSNeel Natu 		 * this must be an instruction that accesses MMIO space.
2429318224bbSNeel Natu 		 */
2430a2da7af6SNeel Natu 		gpa = vmcs_gpa();
24319b1aa8d6SNeel Natu 		if (vm_mem_allocated(vmx->vm, vcpu, gpa) ||
2432159dd56fSNeel Natu 		    apic_access_fault(vmx, vcpu, gpa)) {
2433cd942e0fSPeter Grehan 			vmexit->exitcode = VM_EXITCODE_PAGING;
2434d087a399SNeel Natu 			vmexit->inst_length = 0;
243513ec9371SPeter Grehan 			vmexit->u.paging.gpa = gpa;
2436318224bbSNeel Natu 			vmexit->u.paging.fault_type = ept_fault_type(qual);
2437bf73979dSNeel Natu 			vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NESTED_FAULT, 1);
2438318224bbSNeel Natu 		} else if (ept_emulation_fault(qual)) {
2439e4c8a13dSNeel Natu 			vmexit_inst_emul(vmexit, gpa, vmcs_gla());
2440bf73979dSNeel Natu 			vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INST_EMUL, 1);
2441a2da7af6SNeel Natu 		}
2442e5a1d950SNeel Natu 		/*
2443e5a1d950SNeel Natu 		 * If Virtual NMIs control is 1 and the VM-exit is due to an
2444e5a1d950SNeel Natu 		 * EPT fault during the execution of IRET then we must restore
2445e5a1d950SNeel Natu 		 * the state of "virtual-NMI blocking" before resuming.
2446e5a1d950SNeel Natu 		 *
2447e5a1d950SNeel Natu 		 * See description of "NMI unblocking due to IRET" in
2448e5a1d950SNeel Natu 		 * "Exit Qualification for EPT Violations".
2449e5a1d950SNeel Natu 		 */
2450e5a1d950SNeel Natu 		if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
2451e5a1d950SNeel Natu 		    (qual & EXIT_QUAL_NMIUDTI) != 0)
2452e5a1d950SNeel Natu 			vmx_restore_nmi_blocking(vmx, vcpu);
2453cd942e0fSPeter Grehan 		break;
245430b94db8SNeel Natu 	case EXIT_REASON_VIRTUALIZED_EOI:
245530b94db8SNeel Natu 		vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI;
245630b94db8SNeel Natu 		vmexit->u.ioapic_eoi.vector = qual & 0xFF;
245730b94db8SNeel Natu 		vmexit->inst_length = 0;	/* trap-like */
245830b94db8SNeel Natu 		break;
245988c4b8d1SNeel Natu 	case EXIT_REASON_APIC_ACCESS:
246088c4b8d1SNeel Natu 		handled = vmx_handle_apic_access(vmx, vcpu, vmexit);
246188c4b8d1SNeel Natu 		break;
246288c4b8d1SNeel Natu 	case EXIT_REASON_APIC_WRITE:
246388c4b8d1SNeel Natu 		/*
246488c4b8d1SNeel Natu 		 * APIC-write VM exit is trap-like so the %rip is already
246588c4b8d1SNeel Natu 		 * pointing to the next instruction.
246688c4b8d1SNeel Natu 		 */
246788c4b8d1SNeel Natu 		vmexit->inst_length = 0;
246888c4b8d1SNeel Natu 		vlapic = vm_lapic(vmx->vm, vcpu);
2469159dd56fSNeel Natu 		handled = vmx_handle_apic_write(vmx, vcpu, vlapic, qual);
247088c4b8d1SNeel Natu 		break;
2471abb023fbSJohn Baldwin 	case EXIT_REASON_XSETBV:
2472a0efd3fbSJohn Baldwin 		handled = vmx_emulate_xsetbv(vmx, vcpu, vmexit);
2473abb023fbSJohn Baldwin 		break;
247465145c7fSNeel Natu 	case EXIT_REASON_MONITOR:
247565145c7fSNeel Natu 		vmexit->exitcode = VM_EXITCODE_MONITOR;
247665145c7fSNeel Natu 		break;
247765145c7fSNeel Natu 	case EXIT_REASON_MWAIT:
247865145c7fSNeel Natu 		vmexit->exitcode = VM_EXITCODE_MWAIT;
247965145c7fSNeel Natu 		break;
2480366f6083SPeter Grehan 	default:
2481b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_UNKNOWN, 1);
2482366f6083SPeter Grehan 		break;
2483366f6083SPeter Grehan 	}
2484366f6083SPeter Grehan 
2485366f6083SPeter Grehan 	if (handled) {
2486366f6083SPeter Grehan 		/*
2487366f6083SPeter Grehan 		 * It is possible that control is returned to userland
2488366f6083SPeter Grehan 		 * even though we were able to handle the VM exit in the
2489eeefa4e4SNeel Natu 		 * kernel.
2490366f6083SPeter Grehan 		 *
2491366f6083SPeter Grehan 		 * In such a case we want to make sure that the userland
2492366f6083SPeter Grehan 		 * restarts guest execution at the instruction *after*
2493366f6083SPeter Grehan 		 * the one we just processed. Therefore we update the
2494366f6083SPeter Grehan 		 * guest rip in the VMCS and in 'vmexit'.
2495366f6083SPeter Grehan 		 */
2496366f6083SPeter Grehan 		vmexit->rip += vmexit->inst_length;
2497366f6083SPeter Grehan 		vmexit->inst_length = 0;
24983de83862SNeel Natu 		vmcs_write(VMCS_GUEST_RIP, vmexit->rip);
2499366f6083SPeter Grehan 	} else {
2500366f6083SPeter Grehan 		if (vmexit->exitcode == VM_EXITCODE_BOGUS) {
2501366f6083SPeter Grehan 			/*
2502366f6083SPeter Grehan 			 * If this VM exit was not claimed by anybody then
2503366f6083SPeter Grehan 			 * treat it as a generic VMX exit.
2504366f6083SPeter Grehan 			 */
2505366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_VMX;
25060492757cSNeel Natu 			vmexit->u.vmx.status = VM_SUCCESS;
2507c308b23bSNeel Natu 			vmexit->u.vmx.inst_type = 0;
2508c308b23bSNeel Natu 			vmexit->u.vmx.inst_error = 0;
2509366f6083SPeter Grehan 		} else {
2510366f6083SPeter Grehan 			/*
2511366f6083SPeter Grehan 			 * The exitcode and collateral have been populated.
2512366f6083SPeter Grehan 			 * The VM exit will be processed further in userland.
2513366f6083SPeter Grehan 			 */
2514366f6083SPeter Grehan 		}
2515366f6083SPeter Grehan 	}
2516366f6083SPeter Grehan 	return (handled);
2517366f6083SPeter Grehan }
2518366f6083SPeter Grehan 
251940487465SNeel Natu static __inline void
25200492757cSNeel Natu vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit)
25210492757cSNeel Natu {
25220492757cSNeel Natu 
25230492757cSNeel Natu 	KASSERT(vmxctx->inst_fail_status != VM_SUCCESS,
25240492757cSNeel Natu 	    ("vmx_exit_inst_error: invalid inst_fail_status %d",
25250492757cSNeel Natu 	    vmxctx->inst_fail_status));
25260492757cSNeel Natu 
25270492757cSNeel Natu 	vmexit->inst_length = 0;
25280492757cSNeel Natu 	vmexit->exitcode = VM_EXITCODE_VMX;
25290492757cSNeel Natu 	vmexit->u.vmx.status = vmxctx->inst_fail_status;
25300492757cSNeel Natu 	vmexit->u.vmx.inst_error = vmcs_instruction_error();
25310492757cSNeel Natu 	vmexit->u.vmx.exit_reason = ~0;
25320492757cSNeel Natu 	vmexit->u.vmx.exit_qualification = ~0;
25330492757cSNeel Natu 
25340492757cSNeel Natu 	switch (rc) {
25350492757cSNeel Natu 	case VMX_VMRESUME_ERROR:
25360492757cSNeel Natu 	case VMX_VMLAUNCH_ERROR:
25370492757cSNeel Natu 	case VMX_INVEPT_ERROR:
25380492757cSNeel Natu 		vmexit->u.vmx.inst_type = rc;
25390492757cSNeel Natu 		break;
25400492757cSNeel Natu 	default:
25410492757cSNeel Natu 		panic("vm_exit_inst_error: vmx_enter_guest returned %d", rc);
25420492757cSNeel Natu 	}
25430492757cSNeel Natu }
25440492757cSNeel Natu 
254562fbd7c2SNeel Natu /*
254662fbd7c2SNeel Natu  * If the NMI-exiting VM execution control is set to '1' then an NMI in
254762fbd7c2SNeel Natu  * non-root operation causes a VM-exit. NMI blocking is in effect so it is
254862fbd7c2SNeel Natu  * sufficient to simply vector to the NMI handler via a software interrupt.
254962fbd7c2SNeel Natu  * However, this must be done before maskable interrupts are enabled
255062fbd7c2SNeel Natu  * otherwise the "iret" issued by an interrupt handler will incorrectly
255162fbd7c2SNeel Natu  * clear NMI blocking.
255262fbd7c2SNeel Natu  */
255362fbd7c2SNeel Natu static __inline void
255462fbd7c2SNeel Natu vmx_exit_handle_nmi(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
255562fbd7c2SNeel Natu {
255662fbd7c2SNeel Natu 	uint32_t intr_info;
255762fbd7c2SNeel Natu 
255862fbd7c2SNeel Natu 	KASSERT((read_rflags() & PSL_I) == 0, ("interrupts enabled"));
255962fbd7c2SNeel Natu 
256062fbd7c2SNeel Natu 	if (vmexit->u.vmx.exit_reason != EXIT_REASON_EXCEPTION)
256162fbd7c2SNeel Natu 		return;
256262fbd7c2SNeel Natu 
256362fbd7c2SNeel Natu 	intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
256462fbd7c2SNeel Natu 	KASSERT((intr_info & VMCS_INTR_VALID) != 0,
256562fbd7c2SNeel Natu 	    ("VM exit interruption info invalid: %#x", intr_info));
256662fbd7c2SNeel Natu 
256762fbd7c2SNeel Natu 	if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI) {
256862fbd7c2SNeel Natu 		KASSERT((intr_info & 0xff) == IDT_NMI, ("VM exit due "
256962fbd7c2SNeel Natu 		    "to NMI has invalid vector: %#x", intr_info));
257062fbd7c2SNeel Natu 		VCPU_CTR0(vmx->vm, vcpuid, "Vectoring to NMI handler");
257162fbd7c2SNeel Natu 		__asm __volatile("int $2");
257262fbd7c2SNeel Natu 	}
257362fbd7c2SNeel Natu }
257462fbd7c2SNeel Natu 
25750492757cSNeel Natu static int
25762ce12423SNeel Natu vmx_run(void *arg, int vcpu, register_t rip, pmap_t pmap,
2577248e6799SNeel Natu     struct vm_eventinfo *evinfo)
25780492757cSNeel Natu {
25790492757cSNeel Natu 	int rc, handled, launched;
2580366f6083SPeter Grehan 	struct vmx *vmx;
25815b8a8cd1SNeel Natu 	struct vm *vm;
2582366f6083SPeter Grehan 	struct vmxctx *vmxctx;
2583366f6083SPeter Grehan 	struct vmcs *vmcs;
258498ed632cSNeel Natu 	struct vm_exit *vmexit;
2585de5ea6b6SNeel Natu 	struct vlapic *vlapic;
258679c59630SNeel Natu 	uint32_t exit_reason;
2587366f6083SPeter Grehan 
2588366f6083SPeter Grehan 	vmx = arg;
25895b8a8cd1SNeel Natu 	vm = vmx->vm;
2590366f6083SPeter Grehan 	vmcs = &vmx->vmcs[vcpu];
2591366f6083SPeter Grehan 	vmxctx = &vmx->ctx[vcpu];
25925b8a8cd1SNeel Natu 	vlapic = vm_lapic(vm, vcpu);
25935b8a8cd1SNeel Natu 	vmexit = vm_exitinfo(vm, vcpu);
25940492757cSNeel Natu 	launched = 0;
259598ed632cSNeel Natu 
2596318224bbSNeel Natu 	KASSERT(vmxctx->pmap == pmap,
2597318224bbSNeel Natu 	    ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap));
2598318224bbSNeel Natu 
2599c3498942SNeel Natu 	vmx_msr_guest_enter(vmx, vcpu);
2600c3498942SNeel Natu 
2601366f6083SPeter Grehan 	VMPTRLD(vmcs);
2602366f6083SPeter Grehan 
2603366f6083SPeter Grehan 	/*
2604366f6083SPeter Grehan 	 * XXX
2605366f6083SPeter Grehan 	 * We do this every time because we may setup the virtual machine
2606366f6083SPeter Grehan 	 * from a different process than the one that actually runs it.
2607366f6083SPeter Grehan 	 *
2608366f6083SPeter Grehan 	 * If the life of a virtual machine was spent entirely in the context
2609c847a506SNeel Natu 	 * of a single process we could do this once in vmx_vminit().
2610366f6083SPeter Grehan 	 */
26113de83862SNeel Natu 	vmcs_write(VMCS_HOST_CR3, rcr3());
2612366f6083SPeter Grehan 
26132ce12423SNeel Natu 	vmcs_write(VMCS_GUEST_RIP, rip);
2614953c2c47SNeel Natu 	vmx_set_pcpu_defaults(vmx, vcpu, pmap);
2615366f6083SPeter Grehan 	do {
26162ce12423SNeel Natu 		KASSERT(vmcs_guest_rip() == rip, ("%s: vmcs guest rip mismatch "
26172ce12423SNeel Natu 		    "%#lx/%#lx", __func__, vmcs_guest_rip(), rip));
261840487465SNeel Natu 
26192ce12423SNeel Natu 		handled = UNHANDLED;
26200492757cSNeel Natu 		/*
26210492757cSNeel Natu 		 * Interrupts are disabled from this point on until the
26220492757cSNeel Natu 		 * guest starts executing. This is done for the following
26230492757cSNeel Natu 		 * reasons:
26240492757cSNeel Natu 		 *
26250492757cSNeel Natu 		 * If an AST is asserted on this thread after the check below,
26260492757cSNeel Natu 		 * then the IPI_AST notification will not be lost, because it
26270492757cSNeel Natu 		 * will cause a VM exit due to external interrupt as soon as
26280492757cSNeel Natu 		 * the guest state is loaded.
26290492757cSNeel Natu 		 *
26300492757cSNeel Natu 		 * A posted interrupt after 'vmx_inject_interrupts()' will
26310492757cSNeel Natu 		 * not be "lost" because it will be held pending in the host
26320492757cSNeel Natu 		 * APIC because interrupts are disabled. The pending interrupt
26330492757cSNeel Natu 		 * will be recognized as soon as the guest state is loaded.
26340492757cSNeel Natu 		 *
26350492757cSNeel Natu 		 * The same reasoning applies to the IPI generated by
26360492757cSNeel Natu 		 * pmap_invalidate_ept().
26370492757cSNeel Natu 		 */
26380492757cSNeel Natu 		disable_intr();
26392ce12423SNeel Natu 		vmx_inject_interrupts(vmx, vcpu, vlapic, rip);
2640091d4532SNeel Natu 
2641091d4532SNeel Natu 		/*
2642091d4532SNeel Natu 		 * Check for vcpu suspension after injecting events because
2643091d4532SNeel Natu 		 * vmx_inject_interrupts() can suspend the vcpu due to a
2644091d4532SNeel Natu 		 * triple fault.
2645091d4532SNeel Natu 		 */
2646248e6799SNeel Natu 		if (vcpu_suspended(evinfo)) {
26470492757cSNeel Natu 			enable_intr();
26482ce12423SNeel Natu 			vm_exit_suspended(vmx->vm, vcpu, rip);
26490492757cSNeel Natu 			break;
26500492757cSNeel Natu 		}
26510492757cSNeel Natu 
2652248e6799SNeel Natu 		if (vcpu_rendezvous_pending(evinfo)) {
26535b8a8cd1SNeel Natu 			enable_intr();
26542ce12423SNeel Natu 			vm_exit_rendezvous(vmx->vm, vcpu, rip);
26555b8a8cd1SNeel Natu 			break;
26565b8a8cd1SNeel Natu 		}
26575b8a8cd1SNeel Natu 
2658248e6799SNeel Natu 		if (vcpu_reqidle(evinfo)) {
2659248e6799SNeel Natu 			enable_intr();
2660248e6799SNeel Natu 			vm_exit_reqidle(vmx->vm, vcpu, rip);
2661248e6799SNeel Natu 			break;
2662248e6799SNeel Natu 		}
2663248e6799SNeel Natu 
2664f008d157SNeel Natu 		if (vcpu_should_yield(vm, vcpu)) {
2665b15a09c0SNeel Natu 			enable_intr();
26662ce12423SNeel Natu 			vm_exit_astpending(vmx->vm, vcpu, rip);
26672ce12423SNeel Natu 			vmx_astpending_trace(vmx, vcpu, rip);
266840487465SNeel Natu 			handled = HANDLED;
2669b15a09c0SNeel Natu 			break;
2670b15a09c0SNeel Natu 		}
2671b15a09c0SNeel Natu 
2672366f6083SPeter Grehan 		vmx_run_trace(vmx, vcpu);
2673953c2c47SNeel Natu 		rc = vmx_enter_guest(vmxctx, vmx, launched);
267479c59630SNeel Natu 
267579c59630SNeel Natu 		/* Collect some information for VM exit processing */
267679c59630SNeel Natu 		vmexit->rip = rip = vmcs_guest_rip();
267779c59630SNeel Natu 		vmexit->inst_length = vmexit_instruction_length();
267879c59630SNeel Natu 		vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason();
267979c59630SNeel Natu 		vmexit->u.vmx.exit_qualification = vmcs_exit_qualification();
268079c59630SNeel Natu 
26812ce12423SNeel Natu 		/* Update 'nextrip' */
26822ce12423SNeel Natu 		vmx->state[vcpu].nextrip = rip;
26832ce12423SNeel Natu 
26840492757cSNeel Natu 		if (rc == VMX_GUEST_VMEXIT) {
268562fbd7c2SNeel Natu 			vmx_exit_handle_nmi(vmx, vcpu, vmexit);
268662fbd7c2SNeel Natu 			enable_intr();
26870492757cSNeel Natu 			handled = vmx_exit_process(vmx, vcpu, vmexit);
26880492757cSNeel Natu 		} else {
268962fbd7c2SNeel Natu 			enable_intr();
269040487465SNeel Natu 			vmx_exit_inst_error(vmxctx, rc, vmexit);
2691eeefa4e4SNeel Natu 		}
269262fbd7c2SNeel Natu 		launched = 1;
269379c59630SNeel Natu 		vmx_exit_trace(vmx, vcpu, rip, exit_reason, handled);
26942ce12423SNeel Natu 		rip = vmexit->rip;
2695eeefa4e4SNeel Natu 	} while (handled);
2696366f6083SPeter Grehan 
2697366f6083SPeter Grehan 	/*
2698366f6083SPeter Grehan 	 * If a VM exit has been handled then the exitcode must be BOGUS
2699366f6083SPeter Grehan 	 * If a VM exit is not handled then the exitcode must not be BOGUS
2700366f6083SPeter Grehan 	 */
2701366f6083SPeter Grehan 	if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) ||
2702366f6083SPeter Grehan 	    (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) {
2703366f6083SPeter Grehan 		panic("Mismatch between handled (%d) and exitcode (%d)",
2704366f6083SPeter Grehan 		      handled, vmexit->exitcode);
2705366f6083SPeter Grehan 	}
2706366f6083SPeter Grehan 
2707b5aaf7b2SNeel Natu 	if (!handled)
27085b8a8cd1SNeel Natu 		vmm_stat_incr(vm, vcpu, VMEXIT_USERSPACE, 1);
2709b5aaf7b2SNeel Natu 
27105b8a8cd1SNeel Natu 	VCPU_CTR1(vm, vcpu, "returning from vmx_run: exitcode %d",
27110492757cSNeel Natu 	    vmexit->exitcode);
2712366f6083SPeter Grehan 
2713366f6083SPeter Grehan 	VMCLEAR(vmcs);
2714c3498942SNeel Natu 	vmx_msr_guest_exit(vmx, vcpu);
2715c3498942SNeel Natu 
2716366f6083SPeter Grehan 	return (0);
2717366f6083SPeter Grehan }
2718366f6083SPeter Grehan 
2719366f6083SPeter Grehan static void
2720366f6083SPeter Grehan vmx_vmcleanup(void *arg)
2721366f6083SPeter Grehan {
272263c9389aSNeel Natu 	int i;
2723366f6083SPeter Grehan 	struct vmx *vmx = arg;
2724366f6083SPeter Grehan 
2725159dd56fSNeel Natu 	if (apic_access_virtualization(vmx, 0))
272688c4b8d1SNeel Natu 		vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
272788c4b8d1SNeel Natu 
272845e51299SNeel Natu 	for (i = 0; i < VM_MAXCPU; i++)
272945e51299SNeel Natu 		vpid_free(vmx->state[i].vpid);
273045e51299SNeel Natu 
2731366f6083SPeter Grehan 	free(vmx, M_VMX);
2732366f6083SPeter Grehan 
2733366f6083SPeter Grehan 	return;
2734366f6083SPeter Grehan }
2735366f6083SPeter Grehan 
2736366f6083SPeter Grehan static register_t *
2737366f6083SPeter Grehan vmxctx_regptr(struct vmxctx *vmxctx, int reg)
2738366f6083SPeter Grehan {
2739366f6083SPeter Grehan 
2740366f6083SPeter Grehan 	switch (reg) {
2741366f6083SPeter Grehan 	case VM_REG_GUEST_RAX:
2742366f6083SPeter Grehan 		return (&vmxctx->guest_rax);
2743366f6083SPeter Grehan 	case VM_REG_GUEST_RBX:
2744366f6083SPeter Grehan 		return (&vmxctx->guest_rbx);
2745366f6083SPeter Grehan 	case VM_REG_GUEST_RCX:
2746366f6083SPeter Grehan 		return (&vmxctx->guest_rcx);
2747366f6083SPeter Grehan 	case VM_REG_GUEST_RDX:
2748366f6083SPeter Grehan 		return (&vmxctx->guest_rdx);
2749366f6083SPeter Grehan 	case VM_REG_GUEST_RSI:
2750366f6083SPeter Grehan 		return (&vmxctx->guest_rsi);
2751366f6083SPeter Grehan 	case VM_REG_GUEST_RDI:
2752366f6083SPeter Grehan 		return (&vmxctx->guest_rdi);
2753366f6083SPeter Grehan 	case VM_REG_GUEST_RBP:
2754366f6083SPeter Grehan 		return (&vmxctx->guest_rbp);
2755366f6083SPeter Grehan 	case VM_REG_GUEST_R8:
2756366f6083SPeter Grehan 		return (&vmxctx->guest_r8);
2757366f6083SPeter Grehan 	case VM_REG_GUEST_R9:
2758366f6083SPeter Grehan 		return (&vmxctx->guest_r9);
2759366f6083SPeter Grehan 	case VM_REG_GUEST_R10:
2760366f6083SPeter Grehan 		return (&vmxctx->guest_r10);
2761366f6083SPeter Grehan 	case VM_REG_GUEST_R11:
2762366f6083SPeter Grehan 		return (&vmxctx->guest_r11);
2763366f6083SPeter Grehan 	case VM_REG_GUEST_R12:
2764366f6083SPeter Grehan 		return (&vmxctx->guest_r12);
2765366f6083SPeter Grehan 	case VM_REG_GUEST_R13:
2766366f6083SPeter Grehan 		return (&vmxctx->guest_r13);
2767366f6083SPeter Grehan 	case VM_REG_GUEST_R14:
2768366f6083SPeter Grehan 		return (&vmxctx->guest_r14);
2769366f6083SPeter Grehan 	case VM_REG_GUEST_R15:
2770366f6083SPeter Grehan 		return (&vmxctx->guest_r15);
277137a723a5SNeel Natu 	case VM_REG_GUEST_CR2:
277237a723a5SNeel Natu 		return (&vmxctx->guest_cr2);
2773366f6083SPeter Grehan 	default:
2774366f6083SPeter Grehan 		break;
2775366f6083SPeter Grehan 	}
2776366f6083SPeter Grehan 	return (NULL);
2777366f6083SPeter Grehan }
2778366f6083SPeter Grehan 
2779366f6083SPeter Grehan static int
2780366f6083SPeter Grehan vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval)
2781366f6083SPeter Grehan {
2782366f6083SPeter Grehan 	register_t *regp;
2783366f6083SPeter Grehan 
2784366f6083SPeter Grehan 	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
2785366f6083SPeter Grehan 		*retval = *regp;
2786366f6083SPeter Grehan 		return (0);
2787366f6083SPeter Grehan 	} else
2788366f6083SPeter Grehan 		return (EINVAL);
2789366f6083SPeter Grehan }
2790366f6083SPeter Grehan 
2791366f6083SPeter Grehan static int
2792366f6083SPeter Grehan vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val)
2793366f6083SPeter Grehan {
2794366f6083SPeter Grehan 	register_t *regp;
2795366f6083SPeter Grehan 
2796366f6083SPeter Grehan 	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
2797366f6083SPeter Grehan 		*regp = val;
2798366f6083SPeter Grehan 		return (0);
2799366f6083SPeter Grehan 	} else
2800366f6083SPeter Grehan 		return (EINVAL);
2801366f6083SPeter Grehan }
2802366f6083SPeter Grehan 
2803366f6083SPeter Grehan static int
2804d1819632SNeel Natu vmx_get_intr_shadow(struct vmx *vmx, int vcpu, int running, uint64_t *retval)
2805d1819632SNeel Natu {
2806d1819632SNeel Natu 	uint64_t gi;
2807d1819632SNeel Natu 	int error;
2808d1819632SNeel Natu 
2809d1819632SNeel Natu 	error = vmcs_getreg(&vmx->vmcs[vcpu], running,
2810d1819632SNeel Natu 	    VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY), &gi);
2811d1819632SNeel Natu 	*retval = (gi & HWINTR_BLOCKING) ? 1 : 0;
2812d1819632SNeel Natu 	return (error);
2813d1819632SNeel Natu }
2814d1819632SNeel Natu 
2815d1819632SNeel Natu static int
2816d1819632SNeel Natu vmx_modify_intr_shadow(struct vmx *vmx, int vcpu, int running, uint64_t val)
2817d1819632SNeel Natu {
2818d1819632SNeel Natu 	struct vmcs *vmcs;
2819d1819632SNeel Natu 	uint64_t gi;
2820d1819632SNeel Natu 	int error, ident;
2821d1819632SNeel Natu 
2822d1819632SNeel Natu 	/*
2823d1819632SNeel Natu 	 * Forcing the vcpu into an interrupt shadow is not supported.
2824d1819632SNeel Natu 	 */
2825d1819632SNeel Natu 	if (val) {
2826d1819632SNeel Natu 		error = EINVAL;
2827d1819632SNeel Natu 		goto done;
2828d1819632SNeel Natu 	}
2829d1819632SNeel Natu 
2830d1819632SNeel Natu 	vmcs = &vmx->vmcs[vcpu];
2831d1819632SNeel Natu 	ident = VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY);
2832d1819632SNeel Natu 	error = vmcs_getreg(vmcs, running, ident, &gi);
2833d1819632SNeel Natu 	if (error == 0) {
2834d1819632SNeel Natu 		gi &= ~HWINTR_BLOCKING;
2835d1819632SNeel Natu 		error = vmcs_setreg(vmcs, running, ident, gi);
2836d1819632SNeel Natu 	}
2837d1819632SNeel Natu done:
2838d1819632SNeel Natu 	VCPU_CTR2(vmx->vm, vcpu, "Setting intr_shadow to %#lx %s", val,
2839d1819632SNeel Natu 	    error ? "failed" : "succeeded");
2840d1819632SNeel Natu 	return (error);
2841d1819632SNeel Natu }
2842d1819632SNeel Natu 
2843d1819632SNeel Natu static int
2844aaaa0656SPeter Grehan vmx_shadow_reg(int reg)
2845aaaa0656SPeter Grehan {
2846aaaa0656SPeter Grehan 	int shreg;
2847aaaa0656SPeter Grehan 
2848aaaa0656SPeter Grehan 	shreg = -1;
2849aaaa0656SPeter Grehan 
2850aaaa0656SPeter Grehan 	switch (reg) {
2851aaaa0656SPeter Grehan 	case VM_REG_GUEST_CR0:
2852aaaa0656SPeter Grehan 		shreg = VMCS_CR0_SHADOW;
2853aaaa0656SPeter Grehan                 break;
2854aaaa0656SPeter Grehan         case VM_REG_GUEST_CR4:
2855aaaa0656SPeter Grehan 		shreg = VMCS_CR4_SHADOW;
2856aaaa0656SPeter Grehan 		break;
2857aaaa0656SPeter Grehan 	default:
2858aaaa0656SPeter Grehan 		break;
2859aaaa0656SPeter Grehan 	}
2860aaaa0656SPeter Grehan 
2861aaaa0656SPeter Grehan 	return (shreg);
2862aaaa0656SPeter Grehan }
2863aaaa0656SPeter Grehan 
2864aaaa0656SPeter Grehan static int
2865366f6083SPeter Grehan vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval)
2866366f6083SPeter Grehan {
2867d3c11f40SPeter Grehan 	int running, hostcpu;
2868366f6083SPeter Grehan 	struct vmx *vmx = arg;
2869366f6083SPeter Grehan 
2870d3c11f40SPeter Grehan 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
2871d3c11f40SPeter Grehan 	if (running && hostcpu != curcpu)
2872d3c11f40SPeter Grehan 		panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu);
2873d3c11f40SPeter Grehan 
2874d1819632SNeel Natu 	if (reg == VM_REG_GUEST_INTR_SHADOW)
2875d1819632SNeel Natu 		return (vmx_get_intr_shadow(vmx, vcpu, running, retval));
2876d1819632SNeel Natu 
2877366f6083SPeter Grehan 	if (vmxctx_getreg(&vmx->ctx[vcpu], reg, retval) == 0)
2878366f6083SPeter Grehan 		return (0);
2879366f6083SPeter Grehan 
2880d3c11f40SPeter Grehan 	return (vmcs_getreg(&vmx->vmcs[vcpu], running, reg, retval));
2881366f6083SPeter Grehan }
2882366f6083SPeter Grehan 
2883366f6083SPeter Grehan static int
2884366f6083SPeter Grehan vmx_setreg(void *arg, int vcpu, int reg, uint64_t val)
2885366f6083SPeter Grehan {
2886aaaa0656SPeter Grehan 	int error, hostcpu, running, shadow;
2887366f6083SPeter Grehan 	uint64_t ctls;
28883527963bSNeel Natu 	pmap_t pmap;
2889366f6083SPeter Grehan 	struct vmx *vmx = arg;
2890366f6083SPeter Grehan 
2891d3c11f40SPeter Grehan 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
2892d3c11f40SPeter Grehan 	if (running && hostcpu != curcpu)
2893d3c11f40SPeter Grehan 		panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu);
2894d3c11f40SPeter Grehan 
2895d1819632SNeel Natu 	if (reg == VM_REG_GUEST_INTR_SHADOW)
2896d1819632SNeel Natu 		return (vmx_modify_intr_shadow(vmx, vcpu, running, val));
2897d1819632SNeel Natu 
2898366f6083SPeter Grehan 	if (vmxctx_setreg(&vmx->ctx[vcpu], reg, val) == 0)
2899366f6083SPeter Grehan 		return (0);
2900366f6083SPeter Grehan 
2901d3c11f40SPeter Grehan 	error = vmcs_setreg(&vmx->vmcs[vcpu], running, reg, val);
2902366f6083SPeter Grehan 
2903366f6083SPeter Grehan 	if (error == 0) {
2904366f6083SPeter Grehan 		/*
2905366f6083SPeter Grehan 		 * If the "load EFER" VM-entry control is 1 then the
2906366f6083SPeter Grehan 		 * value of EFER.LMA must be identical to "IA-32e mode guest"
2907366f6083SPeter Grehan 		 * bit in the VM-entry control.
2908366f6083SPeter Grehan 		 */
2909366f6083SPeter Grehan 		if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 &&
2910366f6083SPeter Grehan 		    (reg == VM_REG_GUEST_EFER)) {
2911d3c11f40SPeter Grehan 			vmcs_getreg(&vmx->vmcs[vcpu], running,
2912366f6083SPeter Grehan 				    VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls);
2913366f6083SPeter Grehan 			if (val & EFER_LMA)
2914366f6083SPeter Grehan 				ctls |= VM_ENTRY_GUEST_LMA;
2915366f6083SPeter Grehan 			else
2916366f6083SPeter Grehan 				ctls &= ~VM_ENTRY_GUEST_LMA;
2917d3c11f40SPeter Grehan 			vmcs_setreg(&vmx->vmcs[vcpu], running,
2918366f6083SPeter Grehan 				    VMCS_IDENT(VMCS_ENTRY_CTLS), ctls);
2919366f6083SPeter Grehan 		}
2920aaaa0656SPeter Grehan 
2921aaaa0656SPeter Grehan 		shadow = vmx_shadow_reg(reg);
2922aaaa0656SPeter Grehan 		if (shadow > 0) {
2923aaaa0656SPeter Grehan 			/*
2924aaaa0656SPeter Grehan 			 * Store the unmodified value in the shadow
2925aaaa0656SPeter Grehan 			 */
2926aaaa0656SPeter Grehan 			error = vmcs_setreg(&vmx->vmcs[vcpu], running,
2927aaaa0656SPeter Grehan 				    VMCS_IDENT(shadow), val);
2928aaaa0656SPeter Grehan 		}
29293527963bSNeel Natu 
29303527963bSNeel Natu 		if (reg == VM_REG_GUEST_CR3) {
29313527963bSNeel Natu 			/*
29323527963bSNeel Natu 			 * Invalidate the guest vcpu's TLB mappings to emulate
29333527963bSNeel Natu 			 * the behavior of updating %cr3.
29343527963bSNeel Natu 			 *
29353527963bSNeel Natu 			 * XXX the processor retains global mappings when %cr3
29363527963bSNeel Natu 			 * is updated but vmx_invvpid() does not.
29373527963bSNeel Natu 			 */
29383527963bSNeel Natu 			pmap = vmx->ctx[vcpu].pmap;
29393527963bSNeel Natu 			vmx_invvpid(vmx, vcpu, pmap, running);
29403527963bSNeel Natu 		}
2941366f6083SPeter Grehan 	}
2942366f6083SPeter Grehan 
2943366f6083SPeter Grehan 	return (error);
2944366f6083SPeter Grehan }
2945366f6083SPeter Grehan 
2946366f6083SPeter Grehan static int
2947366f6083SPeter Grehan vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
2948366f6083SPeter Grehan {
2949ba6f5e23SNeel Natu 	int hostcpu, running;
2950366f6083SPeter Grehan 	struct vmx *vmx = arg;
2951366f6083SPeter Grehan 
2952ba6f5e23SNeel Natu 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
2953ba6f5e23SNeel Natu 	if (running && hostcpu != curcpu)
2954ba6f5e23SNeel Natu 		panic("vmx_getdesc: %s%d is running", vm_name(vmx->vm), vcpu);
2955ba6f5e23SNeel Natu 
2956ba6f5e23SNeel Natu 	return (vmcs_getdesc(&vmx->vmcs[vcpu], running, reg, desc));
2957366f6083SPeter Grehan }
2958366f6083SPeter Grehan 
2959366f6083SPeter Grehan static int
2960366f6083SPeter Grehan vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
2961366f6083SPeter Grehan {
2962ba6f5e23SNeel Natu 	int hostcpu, running;
2963366f6083SPeter Grehan 	struct vmx *vmx = arg;
2964366f6083SPeter Grehan 
2965ba6f5e23SNeel Natu 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
2966ba6f5e23SNeel Natu 	if (running && hostcpu != curcpu)
2967ba6f5e23SNeel Natu 		panic("vmx_setdesc: %s%d is running", vm_name(vmx->vm), vcpu);
2968ba6f5e23SNeel Natu 
2969ba6f5e23SNeel Natu 	return (vmcs_setdesc(&vmx->vmcs[vcpu], running, reg, desc));
2970366f6083SPeter Grehan }
2971366f6083SPeter Grehan 
2972366f6083SPeter Grehan static int
2973366f6083SPeter Grehan vmx_getcap(void *arg, int vcpu, int type, int *retval)
2974366f6083SPeter Grehan {
2975366f6083SPeter Grehan 	struct vmx *vmx = arg;
2976366f6083SPeter Grehan 	int vcap;
2977366f6083SPeter Grehan 	int ret;
2978366f6083SPeter Grehan 
2979366f6083SPeter Grehan 	ret = ENOENT;
2980366f6083SPeter Grehan 
2981366f6083SPeter Grehan 	vcap = vmx->cap[vcpu].set;
2982366f6083SPeter Grehan 
2983366f6083SPeter Grehan 	switch (type) {
2984366f6083SPeter Grehan 	case VM_CAP_HALT_EXIT:
2985366f6083SPeter Grehan 		if (cap_halt_exit)
2986366f6083SPeter Grehan 			ret = 0;
2987366f6083SPeter Grehan 		break;
2988366f6083SPeter Grehan 	case VM_CAP_PAUSE_EXIT:
2989366f6083SPeter Grehan 		if (cap_pause_exit)
2990366f6083SPeter Grehan 			ret = 0;
2991366f6083SPeter Grehan 		break;
2992366f6083SPeter Grehan 	case VM_CAP_MTRAP_EXIT:
2993366f6083SPeter Grehan 		if (cap_monitor_trap)
2994366f6083SPeter Grehan 			ret = 0;
2995366f6083SPeter Grehan 		break;
2996366f6083SPeter Grehan 	case VM_CAP_UNRESTRICTED_GUEST:
2997366f6083SPeter Grehan 		if (cap_unrestricted_guest)
2998366f6083SPeter Grehan 			ret = 0;
2999366f6083SPeter Grehan 		break;
300049cc03daSNeel Natu 	case VM_CAP_ENABLE_INVPCID:
300149cc03daSNeel Natu 		if (cap_invpcid)
300249cc03daSNeel Natu 			ret = 0;
300349cc03daSNeel Natu 		break;
3004366f6083SPeter Grehan 	default:
3005366f6083SPeter Grehan 		break;
3006366f6083SPeter Grehan 	}
3007366f6083SPeter Grehan 
3008366f6083SPeter Grehan 	if (ret == 0)
3009366f6083SPeter Grehan 		*retval = (vcap & (1 << type)) ? 1 : 0;
3010366f6083SPeter Grehan 
3011366f6083SPeter Grehan 	return (ret);
3012366f6083SPeter Grehan }
3013366f6083SPeter Grehan 
3014366f6083SPeter Grehan static int
3015366f6083SPeter Grehan vmx_setcap(void *arg, int vcpu, int type, int val)
3016366f6083SPeter Grehan {
3017366f6083SPeter Grehan 	struct vmx *vmx = arg;
3018366f6083SPeter Grehan 	struct vmcs *vmcs = &vmx->vmcs[vcpu];
3019366f6083SPeter Grehan 	uint32_t baseval;
3020366f6083SPeter Grehan 	uint32_t *pptr;
3021366f6083SPeter Grehan 	int error;
3022366f6083SPeter Grehan 	int flag;
3023366f6083SPeter Grehan 	int reg;
3024366f6083SPeter Grehan 	int retval;
3025366f6083SPeter Grehan 
3026366f6083SPeter Grehan 	retval = ENOENT;
3027366f6083SPeter Grehan 	pptr = NULL;
3028366f6083SPeter Grehan 
3029366f6083SPeter Grehan 	switch (type) {
3030366f6083SPeter Grehan 	case VM_CAP_HALT_EXIT:
3031366f6083SPeter Grehan 		if (cap_halt_exit) {
3032366f6083SPeter Grehan 			retval = 0;
3033366f6083SPeter Grehan 			pptr = &vmx->cap[vcpu].proc_ctls;
3034366f6083SPeter Grehan 			baseval = *pptr;
3035366f6083SPeter Grehan 			flag = PROCBASED_HLT_EXITING;
3036366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
3037366f6083SPeter Grehan 		}
3038366f6083SPeter Grehan 		break;
3039366f6083SPeter Grehan 	case VM_CAP_MTRAP_EXIT:
3040366f6083SPeter Grehan 		if (cap_monitor_trap) {
3041366f6083SPeter Grehan 			retval = 0;
3042366f6083SPeter Grehan 			pptr = &vmx->cap[vcpu].proc_ctls;
3043366f6083SPeter Grehan 			baseval = *pptr;
3044366f6083SPeter Grehan 			flag = PROCBASED_MTF;
3045366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
3046366f6083SPeter Grehan 		}
3047366f6083SPeter Grehan 		break;
3048366f6083SPeter Grehan 	case VM_CAP_PAUSE_EXIT:
3049366f6083SPeter Grehan 		if (cap_pause_exit) {
3050366f6083SPeter Grehan 			retval = 0;
3051366f6083SPeter Grehan 			pptr = &vmx->cap[vcpu].proc_ctls;
3052366f6083SPeter Grehan 			baseval = *pptr;
3053366f6083SPeter Grehan 			flag = PROCBASED_PAUSE_EXITING;
3054366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
3055366f6083SPeter Grehan 		}
3056366f6083SPeter Grehan 		break;
3057366f6083SPeter Grehan 	case VM_CAP_UNRESTRICTED_GUEST:
3058366f6083SPeter Grehan 		if (cap_unrestricted_guest) {
3059366f6083SPeter Grehan 			retval = 0;
306049cc03daSNeel Natu 			pptr = &vmx->cap[vcpu].proc_ctls2;
306149cc03daSNeel Natu 			baseval = *pptr;
3062366f6083SPeter Grehan 			flag = PROCBASED2_UNRESTRICTED_GUEST;
3063366f6083SPeter Grehan 			reg = VMCS_SEC_PROC_BASED_CTLS;
3064366f6083SPeter Grehan 		}
3065366f6083SPeter Grehan 		break;
306649cc03daSNeel Natu 	case VM_CAP_ENABLE_INVPCID:
306749cc03daSNeel Natu 		if (cap_invpcid) {
306849cc03daSNeel Natu 			retval = 0;
306949cc03daSNeel Natu 			pptr = &vmx->cap[vcpu].proc_ctls2;
307049cc03daSNeel Natu 			baseval = *pptr;
307149cc03daSNeel Natu 			flag = PROCBASED2_ENABLE_INVPCID;
307249cc03daSNeel Natu 			reg = VMCS_SEC_PROC_BASED_CTLS;
307349cc03daSNeel Natu 		}
307449cc03daSNeel Natu 		break;
3075366f6083SPeter Grehan 	default:
3076366f6083SPeter Grehan 		break;
3077366f6083SPeter Grehan 	}
3078366f6083SPeter Grehan 
3079366f6083SPeter Grehan 	if (retval == 0) {
3080366f6083SPeter Grehan 		if (val) {
3081366f6083SPeter Grehan 			baseval |= flag;
3082366f6083SPeter Grehan 		} else {
3083366f6083SPeter Grehan 			baseval &= ~flag;
3084366f6083SPeter Grehan 		}
3085366f6083SPeter Grehan 		VMPTRLD(vmcs);
3086366f6083SPeter Grehan 		error = vmwrite(reg, baseval);
3087366f6083SPeter Grehan 		VMCLEAR(vmcs);
3088366f6083SPeter Grehan 
3089366f6083SPeter Grehan 		if (error) {
3090366f6083SPeter Grehan 			retval = error;
3091366f6083SPeter Grehan 		} else {
3092366f6083SPeter Grehan 			/*
3093366f6083SPeter Grehan 			 * Update optional stored flags, and record
3094366f6083SPeter Grehan 			 * setting
3095366f6083SPeter Grehan 			 */
3096366f6083SPeter Grehan 			if (pptr != NULL) {
3097366f6083SPeter Grehan 				*pptr = baseval;
3098366f6083SPeter Grehan 			}
3099366f6083SPeter Grehan 
3100366f6083SPeter Grehan 			if (val) {
3101366f6083SPeter Grehan 				vmx->cap[vcpu].set |= (1 << type);
3102366f6083SPeter Grehan 			} else {
3103366f6083SPeter Grehan 				vmx->cap[vcpu].set &= ~(1 << type);
3104366f6083SPeter Grehan 			}
3105366f6083SPeter Grehan 		}
3106366f6083SPeter Grehan 	}
3107366f6083SPeter Grehan 
3108366f6083SPeter Grehan         return (retval);
3109366f6083SPeter Grehan }
3110366f6083SPeter Grehan 
311188c4b8d1SNeel Natu struct vlapic_vtx {
311288c4b8d1SNeel Natu 	struct vlapic	vlapic;
3113176666c2SNeel Natu 	struct pir_desc	*pir_desc;
311430b94db8SNeel Natu 	struct vmx	*vmx;
311588c4b8d1SNeel Natu };
311688c4b8d1SNeel Natu 
311788c4b8d1SNeel Natu #define	VMX_CTR_PIR(vm, vcpuid, pir_desc, notify, vector, level, msg)	\
311888c4b8d1SNeel Natu do {									\
311988c4b8d1SNeel Natu 	VCPU_CTR2(vm, vcpuid, msg " assert %s-triggered vector %d",	\
312088c4b8d1SNeel Natu 	    level ? "level" : "edge", vector);				\
312188c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir0 0x%016lx", pir_desc->pir[0]);	\
312288c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir1 0x%016lx", pir_desc->pir[1]);	\
312388c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir2 0x%016lx", pir_desc->pir[2]);	\
312488c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir3 0x%016lx", pir_desc->pir[3]);	\
312588c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " notify: %s", notify ? "yes" : "no");\
312688c4b8d1SNeel Natu } while (0)
312788c4b8d1SNeel Natu 
312888c4b8d1SNeel Natu /*
312988c4b8d1SNeel Natu  * vlapic->ops handlers that utilize the APICv hardware assist described in
313088c4b8d1SNeel Natu  * Chapter 29 of the Intel SDM.
313188c4b8d1SNeel Natu  */
313288c4b8d1SNeel Natu static int
313388c4b8d1SNeel Natu vmx_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
313488c4b8d1SNeel Natu {
313588c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
313688c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
313788c4b8d1SNeel Natu 	uint64_t mask;
313888c4b8d1SNeel Natu 	int idx, notify;
313988c4b8d1SNeel Natu 
314088c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
3141176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
314288c4b8d1SNeel Natu 
314388c4b8d1SNeel Natu 	/*
314488c4b8d1SNeel Natu 	 * Keep track of interrupt requests in the PIR descriptor. This is
314588c4b8d1SNeel Natu 	 * because the virtual APIC page pointed to by the VMCS cannot be
314688c4b8d1SNeel Natu 	 * modified if the vcpu is running.
314788c4b8d1SNeel Natu 	 */
314888c4b8d1SNeel Natu 	idx = vector / 64;
314988c4b8d1SNeel Natu 	mask = 1UL << (vector % 64);
315088c4b8d1SNeel Natu 	atomic_set_long(&pir_desc->pir[idx], mask);
315188c4b8d1SNeel Natu 	notify = atomic_cmpset_long(&pir_desc->pending, 0, 1);
315288c4b8d1SNeel Natu 
315388c4b8d1SNeel Natu 	VMX_CTR_PIR(vlapic->vm, vlapic->vcpuid, pir_desc, notify, vector,
315488c4b8d1SNeel Natu 	    level, "vmx_set_intr_ready");
315588c4b8d1SNeel Natu 	return (notify);
315688c4b8d1SNeel Natu }
315788c4b8d1SNeel Natu 
315888c4b8d1SNeel Natu static int
315988c4b8d1SNeel Natu vmx_pending_intr(struct vlapic *vlapic, int *vecptr)
316088c4b8d1SNeel Natu {
316188c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
316288c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
316388c4b8d1SNeel Natu 	struct LAPIC *lapic;
316488c4b8d1SNeel Natu 	uint64_t pending, pirval;
316588c4b8d1SNeel Natu 	uint32_t ppr, vpr;
316688c4b8d1SNeel Natu 	int i;
316788c4b8d1SNeel Natu 
316888c4b8d1SNeel Natu 	/*
316988c4b8d1SNeel Natu 	 * This function is only expected to be called from the 'HLT' exit
317088c4b8d1SNeel Natu 	 * handler which does not care about the vector that is pending.
317188c4b8d1SNeel Natu 	 */
317288c4b8d1SNeel Natu 	KASSERT(vecptr == NULL, ("vmx_pending_intr: vecptr must be NULL"));
317388c4b8d1SNeel Natu 
317488c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
3175176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
317688c4b8d1SNeel Natu 
317788c4b8d1SNeel Natu 	pending = atomic_load_acq_long(&pir_desc->pending);
31789e33a616STycho Nightingale 	if (!pending) {
31799e33a616STycho Nightingale 		/*
31809e33a616STycho Nightingale 		 * While a virtual interrupt may have already been
31819e33a616STycho Nightingale 		 * processed the actual delivery maybe pending the
31829e33a616STycho Nightingale 		 * interruptibility of the guest.  Recognize a pending
31839e33a616STycho Nightingale 		 * interrupt by reevaluating virtual interrupts
31849e33a616STycho Nightingale 		 * following Section 29.2.1 in the Intel SDM Volume 3.
31859e33a616STycho Nightingale 		 */
31869e33a616STycho Nightingale 		uint64_t val;
31879e33a616STycho Nightingale 		uint8_t rvi, ppr;
31889e33a616STycho Nightingale 
31899e33a616STycho Nightingale 		vmx_getreg(vlapic_vtx->vmx, vlapic->vcpuid,
31909e33a616STycho Nightingale 		    VMCS_IDENT(VMCS_GUEST_INTR_STATUS), &val);
31919e33a616STycho Nightingale 		rvi = val & APIC_TPR_INT;
31929e33a616STycho Nightingale 		lapic = vlapic->apic_page;
31939e33a616STycho Nightingale 		ppr = lapic->ppr & APIC_TPR_INT;
31949e33a616STycho Nightingale 		if (rvi > ppr) {
31959e33a616STycho Nightingale 			return (1);
31969e33a616STycho Nightingale 		}
31979e33a616STycho Nightingale 
31989e33a616STycho Nightingale 		return (0);
31999e33a616STycho Nightingale 	}
320088c4b8d1SNeel Natu 
320188c4b8d1SNeel Natu 	/*
320288c4b8d1SNeel Natu 	 * If there is an interrupt pending then it will be recognized only
320388c4b8d1SNeel Natu 	 * if its priority is greater than the processor priority.
320488c4b8d1SNeel Natu 	 *
320588c4b8d1SNeel Natu 	 * Special case: if the processor priority is zero then any pending
320688c4b8d1SNeel Natu 	 * interrupt will be recognized.
320788c4b8d1SNeel Natu 	 */
320888c4b8d1SNeel Natu 	lapic = vlapic->apic_page;
32099e33a616STycho Nightingale 	ppr = lapic->ppr & APIC_TPR_INT;
321088c4b8d1SNeel Natu 	if (ppr == 0)
321188c4b8d1SNeel Natu 		return (1);
321288c4b8d1SNeel Natu 
321388c4b8d1SNeel Natu 	VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "HLT with non-zero PPR %d",
321488c4b8d1SNeel Natu 	    lapic->ppr);
321588c4b8d1SNeel Natu 
321688c4b8d1SNeel Natu 	for (i = 3; i >= 0; i--) {
321788c4b8d1SNeel Natu 		pirval = pir_desc->pir[i];
321888c4b8d1SNeel Natu 		if (pirval != 0) {
32199e33a616STycho Nightingale 			vpr = (i * 64 + flsl(pirval) - 1) & APIC_TPR_INT;
322088c4b8d1SNeel Natu 			return (vpr > ppr);
322188c4b8d1SNeel Natu 		}
322288c4b8d1SNeel Natu 	}
322388c4b8d1SNeel Natu 	return (0);
322488c4b8d1SNeel Natu }
322588c4b8d1SNeel Natu 
322688c4b8d1SNeel Natu static void
322788c4b8d1SNeel Natu vmx_intr_accepted(struct vlapic *vlapic, int vector)
322888c4b8d1SNeel Natu {
322988c4b8d1SNeel Natu 
323088c4b8d1SNeel Natu 	panic("vmx_intr_accepted: not expected to be called");
323188c4b8d1SNeel Natu }
323288c4b8d1SNeel Natu 
3233176666c2SNeel Natu static void
323430b94db8SNeel Natu vmx_set_tmr(struct vlapic *vlapic, int vector, bool level)
323530b94db8SNeel Natu {
323630b94db8SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
323730b94db8SNeel Natu 	struct vmx *vmx;
323830b94db8SNeel Natu 	struct vmcs *vmcs;
323930b94db8SNeel Natu 	uint64_t mask, val;
324030b94db8SNeel Natu 
324130b94db8SNeel Natu 	KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
324230b94db8SNeel Natu 	KASSERT(!vcpu_is_running(vlapic->vm, vlapic->vcpuid, NULL),
324330b94db8SNeel Natu 	    ("vmx_set_tmr: vcpu cannot be running"));
324430b94db8SNeel Natu 
324530b94db8SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
324630b94db8SNeel Natu 	vmx = vlapic_vtx->vmx;
324730b94db8SNeel Natu 	vmcs = &vmx->vmcs[vlapic->vcpuid];
324830b94db8SNeel Natu 	mask = 1UL << (vector % 64);
324930b94db8SNeel Natu 
325030b94db8SNeel Natu 	VMPTRLD(vmcs);
325130b94db8SNeel Natu 	val = vmcs_read(VMCS_EOI_EXIT(vector));
325230b94db8SNeel Natu 	if (level)
325330b94db8SNeel Natu 		val |= mask;
325430b94db8SNeel Natu 	else
325530b94db8SNeel Natu 		val &= ~mask;
325630b94db8SNeel Natu 	vmcs_write(VMCS_EOI_EXIT(vector), val);
325730b94db8SNeel Natu 	VMCLEAR(vmcs);
325830b94db8SNeel Natu }
325930b94db8SNeel Natu 
326030b94db8SNeel Natu static void
3261159dd56fSNeel Natu vmx_enable_x2apic_mode(struct vlapic *vlapic)
3262159dd56fSNeel Natu {
3263159dd56fSNeel Natu 	struct vmx *vmx;
3264159dd56fSNeel Natu 	struct vmcs *vmcs;
3265159dd56fSNeel Natu 	uint32_t proc_ctls2;
3266159dd56fSNeel Natu 	int vcpuid, error;
3267159dd56fSNeel Natu 
3268159dd56fSNeel Natu 	vcpuid = vlapic->vcpuid;
3269159dd56fSNeel Natu 	vmx = ((struct vlapic_vtx *)vlapic)->vmx;
3270159dd56fSNeel Natu 	vmcs = &vmx->vmcs[vcpuid];
3271159dd56fSNeel Natu 
3272159dd56fSNeel Natu 	proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
3273159dd56fSNeel Natu 	KASSERT((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) != 0,
3274159dd56fSNeel Natu 	    ("%s: invalid proc_ctls2 %#x", __func__, proc_ctls2));
3275159dd56fSNeel Natu 
3276159dd56fSNeel Natu 	proc_ctls2 &= ~PROCBASED2_VIRTUALIZE_APIC_ACCESSES;
3277159dd56fSNeel Natu 	proc_ctls2 |= PROCBASED2_VIRTUALIZE_X2APIC_MODE;
3278159dd56fSNeel Natu 	vmx->cap[vcpuid].proc_ctls2 = proc_ctls2;
3279159dd56fSNeel Natu 
3280159dd56fSNeel Natu 	VMPTRLD(vmcs);
3281159dd56fSNeel Natu 	vmcs_write(VMCS_SEC_PROC_BASED_CTLS, proc_ctls2);
3282159dd56fSNeel Natu 	VMCLEAR(vmcs);
3283159dd56fSNeel Natu 
3284159dd56fSNeel Natu 	if (vlapic->vcpuid == 0) {
3285159dd56fSNeel Natu 		/*
3286159dd56fSNeel Natu 		 * The nested page table mappings are shared by all vcpus
3287159dd56fSNeel Natu 		 * so unmap the APIC access page just once.
3288159dd56fSNeel Natu 		 */
3289159dd56fSNeel Natu 		error = vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
3290159dd56fSNeel Natu 		KASSERT(error == 0, ("%s: vm_unmap_mmio error %d",
3291159dd56fSNeel Natu 		    __func__, error));
3292159dd56fSNeel Natu 
3293159dd56fSNeel Natu 		/*
3294159dd56fSNeel Natu 		 * The MSR bitmap is shared by all vcpus so modify it only
3295159dd56fSNeel Natu 		 * once in the context of vcpu 0.
3296159dd56fSNeel Natu 		 */
3297159dd56fSNeel Natu 		error = vmx_allow_x2apic_msrs(vmx);
3298159dd56fSNeel Natu 		KASSERT(error == 0, ("%s: vmx_allow_x2apic_msrs error %d",
3299159dd56fSNeel Natu 		    __func__, error));
3300159dd56fSNeel Natu 	}
3301159dd56fSNeel Natu }
3302159dd56fSNeel Natu 
3303159dd56fSNeel Natu static void
3304176666c2SNeel Natu vmx_post_intr(struct vlapic *vlapic, int hostcpu)
3305176666c2SNeel Natu {
3306176666c2SNeel Natu 
3307176666c2SNeel Natu 	ipi_cpu(hostcpu, pirvec);
3308176666c2SNeel Natu }
3309176666c2SNeel Natu 
331088c4b8d1SNeel Natu /*
331188c4b8d1SNeel Natu  * Transfer the pending interrupts in the PIR descriptor to the IRR
331288c4b8d1SNeel Natu  * in the virtual APIC page.
331388c4b8d1SNeel Natu  */
331488c4b8d1SNeel Natu static void
331588c4b8d1SNeel Natu vmx_inject_pir(struct vlapic *vlapic)
331688c4b8d1SNeel Natu {
331788c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
331888c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
331988c4b8d1SNeel Natu 	struct LAPIC *lapic;
332088c4b8d1SNeel Natu 	uint64_t val, pirval;
33210e30c5c0SWarner Losh 	int rvi, pirbase = -1;
332288c4b8d1SNeel Natu 	uint16_t intr_status_old, intr_status_new;
332388c4b8d1SNeel Natu 
332488c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
3325176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
332688c4b8d1SNeel Natu 	if (atomic_cmpset_long(&pir_desc->pending, 1, 0) == 0) {
332788c4b8d1SNeel Natu 		VCPU_CTR0(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
332888c4b8d1SNeel Natu 		    "no posted interrupt pending");
332988c4b8d1SNeel Natu 		return;
333088c4b8d1SNeel Natu 	}
333188c4b8d1SNeel Natu 
333288c4b8d1SNeel Natu 	pirval = 0;
3333201b1cccSPeter Grehan 	pirbase = -1;
333488c4b8d1SNeel Natu 	lapic = vlapic->apic_page;
333588c4b8d1SNeel Natu 
333688c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[0]);
333788c4b8d1SNeel Natu 	if (val != 0) {
333888c4b8d1SNeel Natu 		lapic->irr0 |= val;
333988c4b8d1SNeel Natu 		lapic->irr1 |= val >> 32;
334088c4b8d1SNeel Natu 		pirbase = 0;
334188c4b8d1SNeel Natu 		pirval = val;
334288c4b8d1SNeel Natu 	}
334388c4b8d1SNeel Natu 
334488c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[1]);
334588c4b8d1SNeel Natu 	if (val != 0) {
334688c4b8d1SNeel Natu 		lapic->irr2 |= val;
334788c4b8d1SNeel Natu 		lapic->irr3 |= val >> 32;
334888c4b8d1SNeel Natu 		pirbase = 64;
334988c4b8d1SNeel Natu 		pirval = val;
335088c4b8d1SNeel Natu 	}
335188c4b8d1SNeel Natu 
335288c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[2]);
335388c4b8d1SNeel Natu 	if (val != 0) {
335488c4b8d1SNeel Natu 		lapic->irr4 |= val;
335588c4b8d1SNeel Natu 		lapic->irr5 |= val >> 32;
335688c4b8d1SNeel Natu 		pirbase = 128;
335788c4b8d1SNeel Natu 		pirval = val;
335888c4b8d1SNeel Natu 	}
335988c4b8d1SNeel Natu 
336088c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[3]);
336188c4b8d1SNeel Natu 	if (val != 0) {
336288c4b8d1SNeel Natu 		lapic->irr6 |= val;
336388c4b8d1SNeel Natu 		lapic->irr7 |= val >> 32;
336488c4b8d1SNeel Natu 		pirbase = 192;
336588c4b8d1SNeel Natu 		pirval = val;
336688c4b8d1SNeel Natu 	}
3367201b1cccSPeter Grehan 
336888c4b8d1SNeel Natu 	VLAPIC_CTR_IRR(vlapic, "vmx_inject_pir");
336988c4b8d1SNeel Natu 
337088c4b8d1SNeel Natu 	/*
337188c4b8d1SNeel Natu 	 * Update RVI so the processor can evaluate pending virtual
337288c4b8d1SNeel Natu 	 * interrupts on VM-entry.
3373201b1cccSPeter Grehan 	 *
3374201b1cccSPeter Grehan 	 * It is possible for pirval to be 0 here, even though the
3375201b1cccSPeter Grehan 	 * pending bit has been set. The scenario is:
3376201b1cccSPeter Grehan 	 * CPU-Y is sending a posted interrupt to CPU-X, which
3377201b1cccSPeter Grehan 	 * is running a guest and processing posted interrupts in h/w.
3378201b1cccSPeter Grehan 	 * CPU-X will eventually exit and the state seen in s/w is
3379201b1cccSPeter Grehan 	 * the pending bit set, but no PIR bits set.
3380201b1cccSPeter Grehan 	 *
3381201b1cccSPeter Grehan 	 *      CPU-X                      CPU-Y
3382201b1cccSPeter Grehan 	 *   (vm running)                (host running)
3383201b1cccSPeter Grehan 	 *   rx posted interrupt
3384201b1cccSPeter Grehan 	 *   CLEAR pending bit
3385201b1cccSPeter Grehan 	 *				 SET PIR bit
3386201b1cccSPeter Grehan 	 *   READ/CLEAR PIR bits
3387201b1cccSPeter Grehan 	 *				 SET pending bit
3388201b1cccSPeter Grehan 	 *   (vm exit)
3389201b1cccSPeter Grehan 	 *   pending bit set, PIR 0
339088c4b8d1SNeel Natu 	 */
339188c4b8d1SNeel Natu 	if (pirval != 0) {
339288c4b8d1SNeel Natu 		rvi = pirbase + flsl(pirval) - 1;
339388c4b8d1SNeel Natu 		intr_status_old = vmcs_read(VMCS_GUEST_INTR_STATUS);
339488c4b8d1SNeel Natu 		intr_status_new = (intr_status_old & 0xFF00) | rvi;
339588c4b8d1SNeel Natu 		if (intr_status_new > intr_status_old) {
339688c4b8d1SNeel Natu 			vmcs_write(VMCS_GUEST_INTR_STATUS, intr_status_new);
339788c4b8d1SNeel Natu 			VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
339888c4b8d1SNeel Natu 			    "guest_intr_status changed from 0x%04x to 0x%04x",
339988c4b8d1SNeel Natu 			    intr_status_old, intr_status_new);
340088c4b8d1SNeel Natu 		}
340188c4b8d1SNeel Natu 	}
340288c4b8d1SNeel Natu }
340388c4b8d1SNeel Natu 
3404de5ea6b6SNeel Natu static struct vlapic *
3405de5ea6b6SNeel Natu vmx_vlapic_init(void *arg, int vcpuid)
3406de5ea6b6SNeel Natu {
3407de5ea6b6SNeel Natu 	struct vmx *vmx;
3408de5ea6b6SNeel Natu 	struct vlapic *vlapic;
3409176666c2SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
3410de5ea6b6SNeel Natu 
3411de5ea6b6SNeel Natu 	vmx = arg;
3412de5ea6b6SNeel Natu 
341388c4b8d1SNeel Natu 	vlapic = malloc(sizeof(struct vlapic_vtx), M_VLAPIC, M_WAITOK | M_ZERO);
3414de5ea6b6SNeel Natu 	vlapic->vm = vmx->vm;
3415de5ea6b6SNeel Natu 	vlapic->vcpuid = vcpuid;
3416de5ea6b6SNeel Natu 	vlapic->apic_page = (struct LAPIC *)&vmx->apic_page[vcpuid];
3417de5ea6b6SNeel Natu 
3418176666c2SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
3419176666c2SNeel Natu 	vlapic_vtx->pir_desc = &vmx->pir_desc[vcpuid];
342030b94db8SNeel Natu 	vlapic_vtx->vmx = vmx;
3421176666c2SNeel Natu 
342288c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
342388c4b8d1SNeel Natu 		vlapic->ops.set_intr_ready = vmx_set_intr_ready;
342488c4b8d1SNeel Natu 		vlapic->ops.pending_intr = vmx_pending_intr;
342588c4b8d1SNeel Natu 		vlapic->ops.intr_accepted = vmx_intr_accepted;
342630b94db8SNeel Natu 		vlapic->ops.set_tmr = vmx_set_tmr;
3427159dd56fSNeel Natu 		vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode;
342888c4b8d1SNeel Natu 	}
342988c4b8d1SNeel Natu 
3430176666c2SNeel Natu 	if (posted_interrupts)
3431176666c2SNeel Natu 		vlapic->ops.post_intr = vmx_post_intr;
3432176666c2SNeel Natu 
3433de5ea6b6SNeel Natu 	vlapic_init(vlapic);
3434de5ea6b6SNeel Natu 
3435de5ea6b6SNeel Natu 	return (vlapic);
3436de5ea6b6SNeel Natu }
3437de5ea6b6SNeel Natu 
3438de5ea6b6SNeel Natu static void
3439de5ea6b6SNeel Natu vmx_vlapic_cleanup(void *arg, struct vlapic *vlapic)
3440de5ea6b6SNeel Natu {
3441de5ea6b6SNeel Natu 
3442de5ea6b6SNeel Natu 	vlapic_cleanup(vlapic);
3443de5ea6b6SNeel Natu 	free(vlapic, M_VLAPIC);
3444de5ea6b6SNeel Natu }
3445de5ea6b6SNeel Natu 
3446366f6083SPeter Grehan struct vmm_ops vmm_ops_intel = {
3447366f6083SPeter Grehan 	vmx_init,
3448366f6083SPeter Grehan 	vmx_cleanup,
344963e62d39SJohn Baldwin 	vmx_restore,
3450366f6083SPeter Grehan 	vmx_vminit,
3451366f6083SPeter Grehan 	vmx_run,
3452366f6083SPeter Grehan 	vmx_vmcleanup,
3453366f6083SPeter Grehan 	vmx_getreg,
3454366f6083SPeter Grehan 	vmx_setreg,
3455366f6083SPeter Grehan 	vmx_getdesc,
3456366f6083SPeter Grehan 	vmx_setdesc,
3457366f6083SPeter Grehan 	vmx_getcap,
3458318224bbSNeel Natu 	vmx_setcap,
3459318224bbSNeel Natu 	ept_vmspace_alloc,
3460318224bbSNeel Natu 	ept_vmspace_free,
3461de5ea6b6SNeel Natu 	vmx_vlapic_init,
3462de5ea6b6SNeel Natu 	vmx_vlapic_cleanup,
3463366f6083SPeter Grehan };
3464