xref: /freebsd/sys/amd64/vmm/intel/vmx.c (revision 3ccb02334bf5aee49a66dcff4b9229220bd0184b)
1366f6083SPeter Grehan /*-
24d846d26SWarner Losh  * SPDX-License-Identifier: BSD-2-Clause
3c49761ddSPedro F. Giffuni  *
4366f6083SPeter Grehan  * Copyright (c) 2011 NetApp, Inc.
5366f6083SPeter Grehan  * All rights reserved.
62c352febSJohn Baldwin  * Copyright (c) 2018 Joyent, Inc.
7366f6083SPeter Grehan  *
8366f6083SPeter Grehan  * Redistribution and use in source and binary forms, with or without
9366f6083SPeter Grehan  * modification, are permitted provided that the following conditions
10366f6083SPeter Grehan  * are met:
11366f6083SPeter Grehan  * 1. Redistributions of source code must retain the above copyright
12366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer.
13366f6083SPeter Grehan  * 2. Redistributions in binary form must reproduce the above copyright
14366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer in the
15366f6083SPeter Grehan  *    documentation and/or other materials provided with the distribution.
16366f6083SPeter Grehan  *
17366f6083SPeter Grehan  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
18366f6083SPeter Grehan  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19366f6083SPeter Grehan  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20366f6083SPeter Grehan  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
21366f6083SPeter Grehan  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22366f6083SPeter Grehan  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23366f6083SPeter Grehan  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24366f6083SPeter Grehan  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25366f6083SPeter Grehan  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26366f6083SPeter Grehan  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27366f6083SPeter Grehan  * SUCH DAMAGE.
28366f6083SPeter Grehan  */
29366f6083SPeter Grehan 
30366f6083SPeter Grehan #include <sys/cdefs.h>
31483d953aSJohn Baldwin #include "opt_bhyve_snapshot.h"
32483d953aSJohn Baldwin 
33366f6083SPeter Grehan #include <sys/param.h>
34366f6083SPeter Grehan #include <sys/systm.h>
35366f6083SPeter Grehan #include <sys/smp.h>
36366f6083SPeter Grehan #include <sys/kernel.h>
37366f6083SPeter Grehan #include <sys/malloc.h>
38366f6083SPeter Grehan #include <sys/pcpu.h>
39366f6083SPeter Grehan #include <sys/proc.h>
40b7924341SAndrew Turner #include <sys/reg.h>
416f5a9606SMark Johnston #include <sys/smr.h>
423565b59eSNeel Natu #include <sys/sysctl.h>
43366f6083SPeter Grehan 
44366f6083SPeter Grehan #include <vm/vm.h>
4574ac712fSMark Johnston #include <vm/vm_extern.h>
46366f6083SPeter Grehan #include <vm/pmap.h>
47366f6083SPeter Grehan 
48366f6083SPeter Grehan #include <machine/psl.h>
49366f6083SPeter Grehan #include <machine/cpufunc.h>
508b287612SJohn Baldwin #include <machine/md_var.h>
51366f6083SPeter Grehan #include <machine/segments.h>
52176666c2SNeel Natu #include <machine/smp.h>
53608f97c3SPeter Grehan #include <machine/specialreg.h>
54366f6083SPeter Grehan #include <machine/vmparam.h>
55366f6083SPeter Grehan 
56366f6083SPeter Grehan #include <machine/vmm.h>
57dc506506SNeel Natu #include <machine/vmm_dev.h>
58e813a873SNeel Natu #include <machine/vmm_instruction_emul.h>
59483d953aSJohn Baldwin #include <machine/vmm_snapshot.h>
60483d953aSJohn Baldwin 
61*3ccb0233SMark Johnston #include <dev/vmm/vmm_ktr.h>
62*3ccb0233SMark Johnston 
63c3498942SNeel Natu #include "vmm_lapic.h"
64b01c2033SNeel Natu #include "vmm_host.h"
65762fd208STycho Nightingale #include "vmm_ioport.h"
66366f6083SPeter Grehan #include "vmm_stat.h"
670775fbb4STycho Nightingale #include "vatpic.h"
68de5ea6b6SNeel Natu #include "vlapic.h"
69de5ea6b6SNeel Natu #include "vlapic_priv.h"
70366f6083SPeter Grehan 
71366f6083SPeter Grehan #include "ept.h"
72366f6083SPeter Grehan #include "vmx_cpufunc.h"
73366f6083SPeter Grehan #include "vmx.h"
74c3498942SNeel Natu #include "vmx_msr.h"
75366f6083SPeter Grehan #include "x86.h"
76366f6083SPeter Grehan #include "vmx_controls.h"
77366f6083SPeter Grehan 
78366f6083SPeter Grehan #define	PINBASED_CTLS_ONE_SETTING					\
79366f6083SPeter Grehan 	(PINBASED_EXTINT_EXITING	|				\
80366f6083SPeter Grehan 	 PINBASED_NMI_EXITING		|				\
81366f6083SPeter Grehan 	 PINBASED_VIRTUAL_NMI)
82366f6083SPeter Grehan #define	PINBASED_CTLS_ZERO_SETTING	0
83366f6083SPeter Grehan 
84366f6083SPeter Grehan #define PROCBASED_CTLS_WINDOW_SETTING					\
85366f6083SPeter Grehan 	(PROCBASED_INT_WINDOW_EXITING	|				\
86366f6083SPeter Grehan 	 PROCBASED_NMI_WINDOW_EXITING)
87366f6083SPeter Grehan 
88366f6083SPeter Grehan #define	PROCBASED_CTLS_ONE_SETTING					\
89366f6083SPeter Grehan 	(PROCBASED_SECONDARY_CONTROLS	|				\
9065145c7fSNeel Natu 	 PROCBASED_MWAIT_EXITING	|				\
9165145c7fSNeel Natu 	 PROCBASED_MONITOR_EXITING	|				\
92366f6083SPeter Grehan 	 PROCBASED_IO_EXITING		|				\
93366f6083SPeter Grehan 	 PROCBASED_MSR_BITMAPS		|				\
94594db002STycho Nightingale 	 PROCBASED_CTLS_WINDOW_SETTING	|				\
95594db002STycho Nightingale 	 PROCBASED_CR8_LOAD_EXITING	|				\
96594db002STycho Nightingale 	 PROCBASED_CR8_STORE_EXITING)
97366f6083SPeter Grehan #define	PROCBASED_CTLS_ZERO_SETTING	\
98366f6083SPeter Grehan 	(PROCBASED_CR3_LOAD_EXITING |	\
99366f6083SPeter Grehan 	PROCBASED_CR3_STORE_EXITING |	\
100366f6083SPeter Grehan 	PROCBASED_IO_BITMAPS)
101366f6083SPeter Grehan 
102366f6083SPeter Grehan #define	PROCBASED_CTLS2_ONE_SETTING	PROCBASED2_ENABLE_EPT
103366f6083SPeter Grehan #define	PROCBASED_CTLS2_ZERO_SETTING	0
104366f6083SPeter Grehan 
105d72978ecSNeel Natu #define	VM_EXIT_CTLS_ONE_SETTING					\
10665eefbe4SJohn Baldwin 	(VM_EXIT_SAVE_DEBUG_CONTROLS		|			\
10765eefbe4SJohn Baldwin 	VM_EXIT_HOST_LMA			|			\
108366f6083SPeter Grehan 	VM_EXIT_SAVE_EFER			|			\
109d72978ecSNeel Natu 	VM_EXIT_LOAD_EFER			|			\
110a318f7ddSNeel Natu 	VM_EXIT_ACKNOWLEDGE_INTERRUPT)
111d72978ecSNeel Natu 
11265eefbe4SJohn Baldwin #define	VM_EXIT_CTLS_ZERO_SETTING	0
113366f6083SPeter Grehan 
11465eefbe4SJohn Baldwin #define	VM_ENTRY_CTLS_ONE_SETTING					\
11565eefbe4SJohn Baldwin 	(VM_ENTRY_LOAD_DEBUG_CONTROLS		|			\
11665eefbe4SJohn Baldwin 	VM_ENTRY_LOAD_EFER)
117608f97c3SPeter Grehan 
118366f6083SPeter Grehan #define	VM_ENTRY_CTLS_ZERO_SETTING					\
11965eefbe4SJohn Baldwin 	(VM_ENTRY_INTO_SMM			|			\
120366f6083SPeter Grehan 	VM_ENTRY_DEACTIVATE_DUAL_MONITOR)
121366f6083SPeter Grehan 
122366f6083SPeter Grehan #define	HANDLED		1
123366f6083SPeter Grehan #define	UNHANDLED	0
124366f6083SPeter Grehan 
125de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VMX, "vmx", "vmx");
126de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
127366f6083SPeter Grehan 
12873abae44SJohn Baldwin bool vmx_have_msr_tsc_aux;
12973abae44SJohn Baldwin 
1303565b59eSNeel Natu SYSCTL_DECL(_hw_vmm);
131b40598c5SPawel Biernacki SYSCTL_NODE(_hw_vmm, OID_AUTO, vmx, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
132b40598c5SPawel Biernacki     NULL);
1333565b59eSNeel Natu 
134b3996dd4SJohn Baldwin int vmxon_enabled[MAXCPU];
13574ac712fSMark Johnston static uint8_t *vmxon_region;
136366f6083SPeter Grehan 
137366f6083SPeter Grehan static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2;
138366f6083SPeter Grehan static uint32_t exit_ctls, entry_ctls;
139366f6083SPeter Grehan 
140366f6083SPeter Grehan static uint64_t cr0_ones_mask, cr0_zeros_mask;
1413565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_ones_mask, CTLFLAG_RD,
1423565b59eSNeel Natu 	     &cr0_ones_mask, 0, NULL);
1433565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_zeros_mask, CTLFLAG_RD,
1443565b59eSNeel Natu 	     &cr0_zeros_mask, 0, NULL);
1453565b59eSNeel Natu 
146366f6083SPeter Grehan static uint64_t cr4_ones_mask, cr4_zeros_mask;
1473565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_ones_mask, CTLFLAG_RD,
1483565b59eSNeel Natu 	     &cr4_ones_mask, 0, NULL);
1493565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_zeros_mask, CTLFLAG_RD,
1503565b59eSNeel Natu 	     &cr4_zeros_mask, 0, NULL);
151366f6083SPeter Grehan 
1523565b59eSNeel Natu static int vmx_initialized;
1533565b59eSNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, initialized, CTLFLAG_RD,
1543565b59eSNeel Natu 	   &vmx_initialized, 0, "Intel VMX initialized");
1553565b59eSNeel Natu 
156366f6083SPeter Grehan /*
157366f6083SPeter Grehan  * Optional capabilities
158366f6083SPeter Grehan  */
159b40598c5SPawel Biernacki static SYSCTL_NODE(_hw_vmm_vmx, OID_AUTO, cap,
160b40598c5SPawel Biernacki     CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
161b40598c5SPawel Biernacki     NULL);
16206fc6db9SJohn Baldwin 
163366f6083SPeter Grehan static int cap_halt_exit;
16406fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, halt_exit, CTLFLAG_RD, &cap_halt_exit, 0,
16506fc6db9SJohn Baldwin     "HLT triggers a VM-exit");
16606fc6db9SJohn Baldwin 
167366f6083SPeter Grehan static int cap_pause_exit;
16806fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, pause_exit, CTLFLAG_RD, &cap_pause_exit,
16906fc6db9SJohn Baldwin     0, "PAUSE triggers a VM-exit");
17006fc6db9SJohn Baldwin 
1713ba952e1SCorvin Köhne static int cap_wbinvd_exit;
1723ba952e1SCorvin Köhne SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, wbinvd_exit, CTLFLAG_RD, &cap_wbinvd_exit,
1733ba952e1SCorvin Köhne     0, "WBINVD triggers a VM-exit");
1743ba952e1SCorvin Köhne 
175f5f5f1e7SPeter Grehan static int cap_rdpid;
176f5f5f1e7SPeter Grehan SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, rdpid, CTLFLAG_RD, &cap_rdpid, 0,
177f5f5f1e7SPeter Grehan     "Guests are allowed to use RDPID");
178f5f5f1e7SPeter Grehan 
179f5f5f1e7SPeter Grehan static int cap_rdtscp;
180f5f5f1e7SPeter Grehan SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, rdtscp, CTLFLAG_RD, &cap_rdtscp, 0,
181f5f5f1e7SPeter Grehan     "Guests are allowed to use RDTSCP");
182f5f5f1e7SPeter Grehan 
183366f6083SPeter Grehan static int cap_unrestricted_guest;
18406fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, unrestricted_guest, CTLFLAG_RD,
18506fc6db9SJohn Baldwin     &cap_unrestricted_guest, 0, "Unrestricted guests");
18606fc6db9SJohn Baldwin 
187366f6083SPeter Grehan static int cap_monitor_trap;
18806fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, monitor_trap, CTLFLAG_RD,
18906fc6db9SJohn Baldwin     &cap_monitor_trap, 0, "Monitor trap flag");
19006fc6db9SJohn Baldwin 
19149cc03daSNeel Natu static int cap_invpcid;
19206fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, invpcid, CTLFLAG_RD, &cap_invpcid,
19306fc6db9SJohn Baldwin     0, "Guests are allowed to use INVPCID");
194366f6083SPeter Grehan 
1951bc51badSMichael Reifenberger static int tpr_shadowing;
196f3ff0918SZhenlei Huang SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, tpr_shadowing,
197f3ff0918SZhenlei Huang     CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
1981bc51badSMichael Reifenberger     &tpr_shadowing, 0, "TPR shadowing support");
1991bc51badSMichael Reifenberger 
20088c4b8d1SNeel Natu static int virtual_interrupt_delivery;
201f3ff0918SZhenlei Huang SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, virtual_interrupt_delivery,
202f3ff0918SZhenlei Huang     CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
20388c4b8d1SNeel Natu     &virtual_interrupt_delivery, 0, "APICv virtual interrupt delivery support");
20488c4b8d1SNeel Natu 
205176666c2SNeel Natu static int posted_interrupts;
206f3ff0918SZhenlei Huang SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, posted_interrupts,
207f3ff0918SZhenlei Huang     CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
208176666c2SNeel Natu     &posted_interrupts, 0, "APICv posted interrupt support");
209176666c2SNeel Natu 
21018a2b08eSNeel Natu static int pirvec = -1;
211176666c2SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupt_vector, CTLFLAG_RD,
212176666c2SNeel Natu     &pirvec, 0, "APICv posted interrupt vector");
213176666c2SNeel Natu 
21445e51299SNeel Natu static struct unrhdr *vpid_unr;
21545e51299SNeel Natu static u_int vpid_alloc_failed;
21645e51299SNeel Natu SYSCTL_UINT(_hw_vmm_vmx, OID_AUTO, vpid_alloc_failed, CTLFLAG_RD,
21745e51299SNeel Natu 	    &vpid_alloc_failed, 0, NULL);
21845e51299SNeel Natu 
219d3588766SMark Johnston int guest_l1d_flush;
220f3ff0918SZhenlei Huang SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, l1d_flush, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
221c30578feSKonstantin Belousov     &guest_l1d_flush, 0, NULL);
222d3588766SMark Johnston int guest_l1d_flush_sw;
223f3ff0918SZhenlei Huang SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, l1d_flush_sw, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
224c1141fbaSKonstantin Belousov     &guest_l1d_flush_sw, 0, NULL);
225c30578feSKonstantin Belousov 
226c1141fbaSKonstantin Belousov static struct msr_entry msr_load_list[1] __aligned(16);
227c30578feSKonstantin Belousov 
22888c4b8d1SNeel Natu /*
2296ac73777STycho Nightingale  * The definitions of SDT probes for VMX.
2306ac73777STycho Nightingale  */
2316ac73777STycho Nightingale 
2326ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, entry,
2336ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2346ac73777STycho Nightingale 
2356ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, taskswitch,
2366ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "struct vm_task_switch *");
2376ac73777STycho Nightingale 
2386ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, craccess,
2396ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint64_t");
2406ac73777STycho Nightingale 
2416ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, rdmsr,
2426ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint32_t");
2436ac73777STycho Nightingale 
2446ac73777STycho Nightingale SDT_PROBE_DEFINE5(vmm, vmx, exit, wrmsr,
2456ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint32_t", "uint64_t");
2466ac73777STycho Nightingale 
2476ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, halt,
2486ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2496ac73777STycho Nightingale 
2506ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, mtrap,
2516ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2526ac73777STycho Nightingale 
2536ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, pause,
2546ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2556ac73777STycho Nightingale 
2566ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, intrwindow,
2576ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2586ac73777STycho Nightingale 
2596ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, interrupt,
2606ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint32_t");
2616ac73777STycho Nightingale 
2626ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, nmiwindow,
2636ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2646ac73777STycho Nightingale 
2656ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, inout,
2666ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2676ac73777STycho Nightingale 
2686ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, cpuid,
2696ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2706ac73777STycho Nightingale 
2716ac73777STycho Nightingale SDT_PROBE_DEFINE5(vmm, vmx, exit, exception,
2726ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint32_t", "int");
2736ac73777STycho Nightingale 
2746ac73777STycho Nightingale SDT_PROBE_DEFINE5(vmm, vmx, exit, nestedfault,
2756ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint64_t", "uint64_t");
2766ac73777STycho Nightingale 
2776ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, mmiofault,
2786ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint64_t");
2796ac73777STycho Nightingale 
2806ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, eoi,
2816ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2826ac73777STycho Nightingale 
2836ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, apicaccess,
2846ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2856ac73777STycho Nightingale 
2866ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, apicwrite,
2876ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "struct vlapic *");
2886ac73777STycho Nightingale 
2896ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, xsetbv,
2906ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2916ac73777STycho Nightingale 
2926ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, monitor,
2936ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2946ac73777STycho Nightingale 
2956ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, mwait,
2966ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2976ac73777STycho Nightingale 
29827d26457SAndrew Turner SDT_PROBE_DEFINE3(vmm, vmx, exit, vminsn,
29927d26457SAndrew Turner     "struct vmx *", "int", "struct vm_exit *");
30027d26457SAndrew Turner 
3016ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, unknown,
3026ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint32_t");
3036ac73777STycho Nightingale 
3046ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, return,
3056ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "int");
3066ac73777STycho Nightingale 
3076ac73777STycho Nightingale /*
30888c4b8d1SNeel Natu  * Use the last page below 4GB as the APIC access address. This address is
30988c4b8d1SNeel Natu  * occupied by the boot firmware so it is guaranteed that it will not conflict
31088c4b8d1SNeel Natu  * with a page in system memory.
31188c4b8d1SNeel Natu  */
31288c4b8d1SNeel Natu #define	APIC_ACCESS_ADDRESS	0xFFFFF000
31388c4b8d1SNeel Natu 
314869c8d19SJohn Baldwin static int vmx_getdesc(void *vcpui, int reg, struct seg_desc *desc);
315869c8d19SJohn Baldwin static int vmx_getreg(void *vcpui, int reg, uint64_t *retval);
316c3498942SNeel Natu static int vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val);
31788c4b8d1SNeel Natu static void vmx_inject_pir(struct vlapic *vlapic);
318483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT
319869c8d19SJohn Baldwin static int vmx_restore_tsc(void *vcpui, uint64_t now);
320483d953aSJohn Baldwin #endif
32188c4b8d1SNeel Natu 
322f5f5f1e7SPeter Grehan static inline bool
323f5f5f1e7SPeter Grehan host_has_rdpid(void)
324f5f5f1e7SPeter Grehan {
325f5f5f1e7SPeter Grehan 	return ((cpu_stdext_feature2 & CPUID_STDEXT2_RDPID) != 0);
326f5f5f1e7SPeter Grehan }
327f5f5f1e7SPeter Grehan 
328f5f5f1e7SPeter Grehan static inline bool
329f5f5f1e7SPeter Grehan host_has_rdtscp(void)
330f5f5f1e7SPeter Grehan {
331f5f5f1e7SPeter Grehan 	return ((amd_feature & AMDID_RDTSCP) != 0);
332f5f5f1e7SPeter Grehan }
333f5f5f1e7SPeter Grehan 
334366f6083SPeter Grehan #ifdef KTR
335366f6083SPeter Grehan static const char *
336366f6083SPeter Grehan exit_reason_to_str(int reason)
337366f6083SPeter Grehan {
338366f6083SPeter Grehan 	static char reasonbuf[32];
339366f6083SPeter Grehan 
340366f6083SPeter Grehan 	switch (reason) {
341366f6083SPeter Grehan 	case EXIT_REASON_EXCEPTION:
342366f6083SPeter Grehan 		return "exception";
343366f6083SPeter Grehan 	case EXIT_REASON_EXT_INTR:
344366f6083SPeter Grehan 		return "extint";
345366f6083SPeter Grehan 	case EXIT_REASON_TRIPLE_FAULT:
346366f6083SPeter Grehan 		return "triplefault";
347366f6083SPeter Grehan 	case EXIT_REASON_INIT:
348366f6083SPeter Grehan 		return "init";
349366f6083SPeter Grehan 	case EXIT_REASON_SIPI:
350366f6083SPeter Grehan 		return "sipi";
351366f6083SPeter Grehan 	case EXIT_REASON_IO_SMI:
352366f6083SPeter Grehan 		return "iosmi";
353366f6083SPeter Grehan 	case EXIT_REASON_SMI:
354366f6083SPeter Grehan 		return "smi";
355366f6083SPeter Grehan 	case EXIT_REASON_INTR_WINDOW:
356366f6083SPeter Grehan 		return "intrwindow";
357366f6083SPeter Grehan 	case EXIT_REASON_NMI_WINDOW:
358366f6083SPeter Grehan 		return "nmiwindow";
359366f6083SPeter Grehan 	case EXIT_REASON_TASK_SWITCH:
360366f6083SPeter Grehan 		return "taskswitch";
361366f6083SPeter Grehan 	case EXIT_REASON_CPUID:
362366f6083SPeter Grehan 		return "cpuid";
363366f6083SPeter Grehan 	case EXIT_REASON_GETSEC:
364366f6083SPeter Grehan 		return "getsec";
365366f6083SPeter Grehan 	case EXIT_REASON_HLT:
366366f6083SPeter Grehan 		return "hlt";
367366f6083SPeter Grehan 	case EXIT_REASON_INVD:
368366f6083SPeter Grehan 		return "invd";
369366f6083SPeter Grehan 	case EXIT_REASON_INVLPG:
370366f6083SPeter Grehan 		return "invlpg";
371366f6083SPeter Grehan 	case EXIT_REASON_RDPMC:
372366f6083SPeter Grehan 		return "rdpmc";
373366f6083SPeter Grehan 	case EXIT_REASON_RDTSC:
374366f6083SPeter Grehan 		return "rdtsc";
375366f6083SPeter Grehan 	case EXIT_REASON_RSM:
376366f6083SPeter Grehan 		return "rsm";
377366f6083SPeter Grehan 	case EXIT_REASON_VMCALL:
378366f6083SPeter Grehan 		return "vmcall";
379366f6083SPeter Grehan 	case EXIT_REASON_VMCLEAR:
380366f6083SPeter Grehan 		return "vmclear";
381366f6083SPeter Grehan 	case EXIT_REASON_VMLAUNCH:
382366f6083SPeter Grehan 		return "vmlaunch";
383366f6083SPeter Grehan 	case EXIT_REASON_VMPTRLD:
384366f6083SPeter Grehan 		return "vmptrld";
385366f6083SPeter Grehan 	case EXIT_REASON_VMPTRST:
386366f6083SPeter Grehan 		return "vmptrst";
387366f6083SPeter Grehan 	case EXIT_REASON_VMREAD:
388366f6083SPeter Grehan 		return "vmread";
389366f6083SPeter Grehan 	case EXIT_REASON_VMRESUME:
390366f6083SPeter Grehan 		return "vmresume";
391366f6083SPeter Grehan 	case EXIT_REASON_VMWRITE:
392366f6083SPeter Grehan 		return "vmwrite";
393366f6083SPeter Grehan 	case EXIT_REASON_VMXOFF:
394366f6083SPeter Grehan 		return "vmxoff";
395366f6083SPeter Grehan 	case EXIT_REASON_VMXON:
396366f6083SPeter Grehan 		return "vmxon";
397366f6083SPeter Grehan 	case EXIT_REASON_CR_ACCESS:
398366f6083SPeter Grehan 		return "craccess";
399366f6083SPeter Grehan 	case EXIT_REASON_DR_ACCESS:
400366f6083SPeter Grehan 		return "draccess";
401366f6083SPeter Grehan 	case EXIT_REASON_INOUT:
402366f6083SPeter Grehan 		return "inout";
403366f6083SPeter Grehan 	case EXIT_REASON_RDMSR:
404366f6083SPeter Grehan 		return "rdmsr";
405366f6083SPeter Grehan 	case EXIT_REASON_WRMSR:
406366f6083SPeter Grehan 		return "wrmsr";
407366f6083SPeter Grehan 	case EXIT_REASON_INVAL_VMCS:
408366f6083SPeter Grehan 		return "invalvmcs";
409366f6083SPeter Grehan 	case EXIT_REASON_INVAL_MSR:
410366f6083SPeter Grehan 		return "invalmsr";
411366f6083SPeter Grehan 	case EXIT_REASON_MWAIT:
412366f6083SPeter Grehan 		return "mwait";
413366f6083SPeter Grehan 	case EXIT_REASON_MTF:
414366f6083SPeter Grehan 		return "mtf";
415366f6083SPeter Grehan 	case EXIT_REASON_MONITOR:
416366f6083SPeter Grehan 		return "monitor";
417366f6083SPeter Grehan 	case EXIT_REASON_PAUSE:
418366f6083SPeter Grehan 		return "pause";
419b0538143SNeel Natu 	case EXIT_REASON_MCE_DURING_ENTRY:
420b0538143SNeel Natu 		return "mce-during-entry";
421366f6083SPeter Grehan 	case EXIT_REASON_TPR:
422366f6083SPeter Grehan 		return "tpr";
42388c4b8d1SNeel Natu 	case EXIT_REASON_APIC_ACCESS:
42488c4b8d1SNeel Natu 		return "apic-access";
425366f6083SPeter Grehan 	case EXIT_REASON_GDTR_IDTR:
426366f6083SPeter Grehan 		return "gdtridtr";
427366f6083SPeter Grehan 	case EXIT_REASON_LDTR_TR:
428366f6083SPeter Grehan 		return "ldtrtr";
429366f6083SPeter Grehan 	case EXIT_REASON_EPT_FAULT:
430366f6083SPeter Grehan 		return "eptfault";
431366f6083SPeter Grehan 	case EXIT_REASON_EPT_MISCONFIG:
432366f6083SPeter Grehan 		return "eptmisconfig";
433366f6083SPeter Grehan 	case EXIT_REASON_INVEPT:
434366f6083SPeter Grehan 		return "invept";
435366f6083SPeter Grehan 	case EXIT_REASON_RDTSCP:
436366f6083SPeter Grehan 		return "rdtscp";
437366f6083SPeter Grehan 	case EXIT_REASON_VMX_PREEMPT:
438366f6083SPeter Grehan 		return "vmxpreempt";
439366f6083SPeter Grehan 	case EXIT_REASON_INVVPID:
440366f6083SPeter Grehan 		return "invvpid";
441366f6083SPeter Grehan 	case EXIT_REASON_WBINVD:
442366f6083SPeter Grehan 		return "wbinvd";
443366f6083SPeter Grehan 	case EXIT_REASON_XSETBV:
444366f6083SPeter Grehan 		return "xsetbv";
44588c4b8d1SNeel Natu 	case EXIT_REASON_APIC_WRITE:
44688c4b8d1SNeel Natu 		return "apic-write";
447366f6083SPeter Grehan 	default:
448366f6083SPeter Grehan 		snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason);
449366f6083SPeter Grehan 		return (reasonbuf);
450366f6083SPeter Grehan 	}
451366f6083SPeter Grehan }
452366f6083SPeter Grehan #endif	/* KTR */
453366f6083SPeter Grehan 
454159dd56fSNeel Natu static int
455159dd56fSNeel Natu vmx_allow_x2apic_msrs(struct vmx *vmx)
456159dd56fSNeel Natu {
457159dd56fSNeel Natu 	int i, error;
458159dd56fSNeel Natu 
459159dd56fSNeel Natu 	error = 0;
460159dd56fSNeel Natu 
461159dd56fSNeel Natu 	/*
462159dd56fSNeel Natu 	 * Allow readonly access to the following x2APIC MSRs from the guest.
463159dd56fSNeel Natu 	 */
464159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ID);
465159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_VERSION);
466159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LDR);
467159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_SVR);
468159dd56fSNeel Natu 
469159dd56fSNeel Natu 	for (i = 0; i < 8; i++)
470159dd56fSNeel Natu 		error += guest_msr_ro(vmx, MSR_APIC_ISR0 + i);
471159dd56fSNeel Natu 
472159dd56fSNeel Natu 	for (i = 0; i < 8; i++)
473159dd56fSNeel Natu 		error += guest_msr_ro(vmx, MSR_APIC_TMR0 + i);
474159dd56fSNeel Natu 
475159dd56fSNeel Natu 	for (i = 0; i < 8; i++)
476159dd56fSNeel Natu 		error += guest_msr_ro(vmx, MSR_APIC_IRR0 + i);
477159dd56fSNeel Natu 
478159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ESR);
479159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_TIMER);
480159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_THERMAL);
481159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_PCINT);
482159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT0);
483159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT1);
484159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_ERROR);
485159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ICR_TIMER);
486159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_DCR_TIMER);
487159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ICR);
488159dd56fSNeel Natu 
489159dd56fSNeel Natu 	/*
490159dd56fSNeel Natu 	 * Allow TPR, EOI and SELF_IPI MSRs to be read and written by the guest.
491159dd56fSNeel Natu 	 *
492159dd56fSNeel Natu 	 * These registers get special treatment described in the section
493159dd56fSNeel Natu 	 * "Virtualizing MSR-Based APIC Accesses".
494159dd56fSNeel Natu 	 */
495159dd56fSNeel Natu 	error += guest_msr_rw(vmx, MSR_APIC_TPR);
496159dd56fSNeel Natu 	error += guest_msr_rw(vmx, MSR_APIC_EOI);
497159dd56fSNeel Natu 	error += guest_msr_rw(vmx, MSR_APIC_SELF_IPI);
498159dd56fSNeel Natu 
499159dd56fSNeel Natu 	return (error);
500159dd56fSNeel Natu }
501159dd56fSNeel Natu 
502366f6083SPeter Grehan u_long
503366f6083SPeter Grehan vmx_fix_cr0(u_long cr0)
504366f6083SPeter Grehan {
505366f6083SPeter Grehan 
506366f6083SPeter Grehan 	return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask);
507366f6083SPeter Grehan }
508366f6083SPeter Grehan 
509366f6083SPeter Grehan u_long
510366f6083SPeter Grehan vmx_fix_cr4(u_long cr4)
511366f6083SPeter Grehan {
512366f6083SPeter Grehan 
513366f6083SPeter Grehan 	return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask);
514366f6083SPeter Grehan }
515366f6083SPeter Grehan 
516366f6083SPeter Grehan static void
51745e51299SNeel Natu vpid_free(int vpid)
51845e51299SNeel Natu {
51945e51299SNeel Natu 	if (vpid < 0 || vpid > 0xffff)
52045e51299SNeel Natu 		panic("vpid_free: invalid vpid %d", vpid);
52145e51299SNeel Natu 
52245e51299SNeel Natu 	/*
523ee98f99dSJohn Baldwin 	 * VPIDs [0,vm_maxcpu] are special and are not allocated from
52445e51299SNeel Natu 	 * the unit number allocator.
52545e51299SNeel Natu 	 */
52645e51299SNeel Natu 
527ee98f99dSJohn Baldwin 	if (vpid > vm_maxcpu)
52845e51299SNeel Natu 		free_unr(vpid_unr, vpid);
52945e51299SNeel Natu }
53045e51299SNeel Natu 
53158eefc67SJohn Baldwin static uint16_t
53258eefc67SJohn Baldwin vpid_alloc(int vcpuid)
53345e51299SNeel Natu {
53458eefc67SJohn Baldwin 	int x;
53545e51299SNeel Natu 
53645e51299SNeel Natu 	/*
53745e51299SNeel Natu 	 * If the "enable vpid" execution control is not enabled then the
53845e51299SNeel Natu 	 * VPID is required to be 0 for all vcpus.
53945e51299SNeel Natu 	 */
54058eefc67SJohn Baldwin 	if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) == 0)
54158eefc67SJohn Baldwin 		return (0);
54245e51299SNeel Natu 
54345e51299SNeel Natu 	/*
54458eefc67SJohn Baldwin 	 * Try to allocate a unique VPID for each from the unit number
54558eefc67SJohn Baldwin 	 * allocator.
54645e51299SNeel Natu 	 */
54745e51299SNeel Natu 	x = alloc_unr(vpid_unr);
54845e51299SNeel Natu 
54958eefc67SJohn Baldwin 	if (x == -1) {
55045e51299SNeel Natu 		atomic_add_int(&vpid_alloc_failed, 1);
55145e51299SNeel Natu 
55245e51299SNeel Natu 		/*
55345e51299SNeel Natu 		 * If the unit number allocator does not have enough unique
554ee98f99dSJohn Baldwin 		 * VPIDs then we need to allocate from the [1,vm_maxcpu] range.
55545e51299SNeel Natu 		 *
55645e51299SNeel Natu 		 * These VPIDs are not be unique across VMs but this does not
55745e51299SNeel Natu 		 * affect correctness because the combined mappings are also
55845e51299SNeel Natu 		 * tagged with the EP4TA which is unique for each VM.
55945e51299SNeel Natu 		 *
56045e51299SNeel Natu 		 * It is still sub-optimal because the invvpid will invalidate
56145e51299SNeel Natu 		 * combined mappings for a particular VPID across all EP4TAs.
56245e51299SNeel Natu 		 */
56358eefc67SJohn Baldwin 		return (vcpuid + 1);
56445e51299SNeel Natu 	}
56558eefc67SJohn Baldwin 
56658eefc67SJohn Baldwin 	return (x);
56745e51299SNeel Natu }
56845e51299SNeel Natu 
56945e51299SNeel Natu static void
57045e51299SNeel Natu vpid_init(void)
57145e51299SNeel Natu {
57245e51299SNeel Natu 	/*
57345e51299SNeel Natu 	 * VPID 0 is required when the "enable VPID" execution control is
57445e51299SNeel Natu 	 * disabled.
57545e51299SNeel Natu 	 *
576ee98f99dSJohn Baldwin 	 * VPIDs [1,vm_maxcpu] are used as the "overflow namespace" when the
57745e51299SNeel Natu 	 * unit number allocator does not have sufficient unique VPIDs to
57845e51299SNeel Natu 	 * satisfy the allocation.
57945e51299SNeel Natu 	 *
58045e51299SNeel Natu 	 * The remaining VPIDs are managed by the unit number allocator.
58145e51299SNeel Natu 	 */
582ee98f99dSJohn Baldwin 	vpid_unr = new_unrhdr(vm_maxcpu + 1, 0xffff, NULL);
58345e51299SNeel Natu }
58445e51299SNeel Natu 
58545e51299SNeel Natu static void
586366f6083SPeter Grehan vmx_disable(void *arg __unused)
587366f6083SPeter Grehan {
588366f6083SPeter Grehan 	struct invvpid_desc invvpid_desc = { 0 };
589366f6083SPeter Grehan 	struct invept_desc invept_desc = { 0 };
590366f6083SPeter Grehan 
591366f6083SPeter Grehan 	if (vmxon_enabled[curcpu]) {
592366f6083SPeter Grehan 		/*
593366f6083SPeter Grehan 		 * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b.
594366f6083SPeter Grehan 		 *
595366f6083SPeter Grehan 		 * VMXON or VMXOFF are not required to invalidate any TLB
596366f6083SPeter Grehan 		 * caching structures. This prevents potential retention of
597366f6083SPeter Grehan 		 * cached information in the TLB between distinct VMX episodes.
598366f6083SPeter Grehan 		 */
599366f6083SPeter Grehan 		invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc);
600366f6083SPeter Grehan 		invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc);
601366f6083SPeter Grehan 		vmxoff();
602366f6083SPeter Grehan 	}
603366f6083SPeter Grehan 	load_cr4(rcr4() & ~CR4_VMXE);
604366f6083SPeter Grehan }
605366f6083SPeter Grehan 
606366f6083SPeter Grehan static int
60715add60dSPeter Grehan vmx_modcleanup(void)
608366f6083SPeter Grehan {
609366f6083SPeter Grehan 
61018a2b08eSNeel Natu 	if (pirvec >= 0)
61118a2b08eSNeel Natu 		lapic_ipi_free(pirvec);
612176666c2SNeel Natu 
61345e51299SNeel Natu 	if (vpid_unr != NULL) {
61445e51299SNeel Natu 		delete_unrhdr(vpid_unr);
61545e51299SNeel Natu 		vpid_unr = NULL;
61645e51299SNeel Natu 	}
61745e51299SNeel Natu 
618c1141fbaSKonstantin Belousov 	if (nmi_flush_l1d_sw == 1)
619c1141fbaSKonstantin Belousov 		nmi_flush_l1d_sw = 0;
620c1141fbaSKonstantin Belousov 
621366f6083SPeter Grehan 	smp_rendezvous(NULL, vmx_disable, NULL, NULL);
622b10e100dSCorvin Köhne 
623b10e100dSCorvin Köhne 	if (vmxon_region != NULL)
62474ac712fSMark Johnston 		kmem_free(vmxon_region, (mp_maxid + 1) * PAGE_SIZE);
625366f6083SPeter Grehan 
626366f6083SPeter Grehan 	return (0);
627366f6083SPeter Grehan }
628366f6083SPeter Grehan 
629366f6083SPeter Grehan static void
630366f6083SPeter Grehan vmx_enable(void *arg __unused)
631366f6083SPeter Grehan {
632366f6083SPeter Grehan 	int error;
63311669a68STycho Nightingale 	uint64_t feature_control;
63411669a68STycho Nightingale 
63511669a68STycho Nightingale 	feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
63611669a68STycho Nightingale 	if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 0 ||
63711669a68STycho Nightingale 	    (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) {
63811669a68STycho Nightingale 		wrmsr(MSR_IA32_FEATURE_CONTROL,
63911669a68STycho Nightingale 		    feature_control | IA32_FEATURE_CONTROL_VMX_EN |
64011669a68STycho Nightingale 		    IA32_FEATURE_CONTROL_LOCK);
64111669a68STycho Nightingale 	}
642366f6083SPeter Grehan 
643366f6083SPeter Grehan 	load_cr4(rcr4() | CR4_VMXE);
644366f6083SPeter Grehan 
64574ac712fSMark Johnston 	*(uint32_t *)&vmxon_region[curcpu * PAGE_SIZE] = vmx_revision();
64674ac712fSMark Johnston 	error = vmxon(&vmxon_region[curcpu * PAGE_SIZE]);
647366f6083SPeter Grehan 	if (error == 0)
648366f6083SPeter Grehan 		vmxon_enabled[curcpu] = 1;
649366f6083SPeter Grehan }
650366f6083SPeter Grehan 
65163e62d39SJohn Baldwin static void
65215add60dSPeter Grehan vmx_modresume(void)
65363e62d39SJohn Baldwin {
65463e62d39SJohn Baldwin 
65563e62d39SJohn Baldwin 	if (vmxon_enabled[curcpu])
65674ac712fSMark Johnston 		vmxon(&vmxon_region[curcpu * PAGE_SIZE]);
65763e62d39SJohn Baldwin }
65863e62d39SJohn Baldwin 
659366f6083SPeter Grehan static int
66015add60dSPeter Grehan vmx_modinit(int ipinum)
661366f6083SPeter Grehan {
6621bc51badSMichael Reifenberger 	int error;
663d17b5104SNeel Natu 	uint64_t basic, fixed0, fixed1, feature_control;
66488c4b8d1SNeel Natu 	uint32_t tmp, procbased2_vid_bits;
665366f6083SPeter Grehan 
666366f6083SPeter Grehan 	/* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */
6678b287612SJohn Baldwin 	if (!(cpu_feature2 & CPUID2_VMX)) {
66815add60dSPeter Grehan 		printf("vmx_modinit: processor does not support VMX "
66915add60dSPeter Grehan 		    "operation\n");
670366f6083SPeter Grehan 		return (ENXIO);
671366f6083SPeter Grehan 	}
672366f6083SPeter Grehan 
6734bff7fadSNeel Natu 	/*
6744bff7fadSNeel Natu 	 * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits
6754bff7fadSNeel Natu 	 * are set (bits 0 and 2 respectively).
6764bff7fadSNeel Natu 	 */
6774bff7fadSNeel Natu 	feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
67811669a68STycho Nightingale 	if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 1 &&
679150369abSNeel Natu 	    (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) {
68015add60dSPeter Grehan 		printf("vmx_modinit: VMX operation disabled by BIOS\n");
6814bff7fadSNeel Natu 		return (ENXIO);
6824bff7fadSNeel Natu 	}
6834bff7fadSNeel Natu 
684d17b5104SNeel Natu 	/*
685d17b5104SNeel Natu 	 * Verify capabilities MSR_VMX_BASIC:
686d17b5104SNeel Natu 	 * - bit 54 indicates support for INS/OUTS decoding
687d17b5104SNeel Natu 	 */
688d17b5104SNeel Natu 	basic = rdmsr(MSR_VMX_BASIC);
689d17b5104SNeel Natu 	if ((basic & (1UL << 54)) == 0) {
69015add60dSPeter Grehan 		printf("vmx_modinit: processor does not support desired basic "
691d17b5104SNeel Natu 		    "capabilities\n");
692d17b5104SNeel Natu 		return (EINVAL);
693d17b5104SNeel Natu 	}
694d17b5104SNeel Natu 
695366f6083SPeter Grehan 	/* Check support for primary processor-based VM-execution controls */
696366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
697366f6083SPeter Grehan 			       MSR_VMX_TRUE_PROCBASED_CTLS,
698366f6083SPeter Grehan 			       PROCBASED_CTLS_ONE_SETTING,
699366f6083SPeter Grehan 			       PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls);
700366f6083SPeter Grehan 	if (error) {
70115add60dSPeter Grehan 		printf("vmx_modinit: processor does not support desired "
70215add60dSPeter Grehan 		    "primary processor-based controls\n");
703366f6083SPeter Grehan 		return (error);
704366f6083SPeter Grehan 	}
705366f6083SPeter Grehan 
706366f6083SPeter Grehan 	/* Clear the processor-based ctl bits that are set on demand */
707366f6083SPeter Grehan 	procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING;
708366f6083SPeter Grehan 
709366f6083SPeter Grehan 	/* Check support for secondary processor-based VM-execution controls */
710366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
711366f6083SPeter Grehan 			       MSR_VMX_PROCBASED_CTLS2,
712366f6083SPeter Grehan 			       PROCBASED_CTLS2_ONE_SETTING,
713366f6083SPeter Grehan 			       PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2);
714366f6083SPeter Grehan 	if (error) {
71515add60dSPeter Grehan 		printf("vmx_modinit: processor does not support desired "
71615add60dSPeter Grehan 		    "secondary processor-based controls\n");
717366f6083SPeter Grehan 		return (error);
718366f6083SPeter Grehan 	}
719366f6083SPeter Grehan 
720366f6083SPeter Grehan 	/* Check support for VPID */
721366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
722366f6083SPeter Grehan 			       PROCBASED2_ENABLE_VPID, 0, &tmp);
723366f6083SPeter Grehan 	if (error == 0)
724366f6083SPeter Grehan 		procbased_ctls2 |= PROCBASED2_ENABLE_VPID;
725366f6083SPeter Grehan 
726366f6083SPeter Grehan 	/* Check support for pin-based VM-execution controls */
727366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
728366f6083SPeter Grehan 			       MSR_VMX_TRUE_PINBASED_CTLS,
729366f6083SPeter Grehan 			       PINBASED_CTLS_ONE_SETTING,
730366f6083SPeter Grehan 			       PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls);
731366f6083SPeter Grehan 	if (error) {
73215add60dSPeter Grehan 		printf("vmx_modinit: processor does not support desired "
733366f6083SPeter Grehan 		    "pin-based controls\n");
734366f6083SPeter Grehan 		return (error);
735366f6083SPeter Grehan 	}
736366f6083SPeter Grehan 
737366f6083SPeter Grehan 	/* Check support for VM-exit controls */
738366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS,
739366f6083SPeter Grehan 			       VM_EXIT_CTLS_ONE_SETTING,
740366f6083SPeter Grehan 			       VM_EXIT_CTLS_ZERO_SETTING,
741366f6083SPeter Grehan 			       &exit_ctls);
742366f6083SPeter Grehan 	if (error) {
74315add60dSPeter Grehan 		printf("vmx_modinit: processor does not support desired "
744366f6083SPeter Grehan 		    "exit controls\n");
745366f6083SPeter Grehan 		return (error);
746366f6083SPeter Grehan 	}
747366f6083SPeter Grehan 
748366f6083SPeter Grehan 	/* Check support for VM-entry controls */
749d72978ecSNeel Natu 	error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS, MSR_VMX_TRUE_ENTRY_CTLS,
750d72978ecSNeel Natu 	    VM_ENTRY_CTLS_ONE_SETTING, VM_ENTRY_CTLS_ZERO_SETTING,
751366f6083SPeter Grehan 	    &entry_ctls);
752366f6083SPeter Grehan 	if (error) {
75315add60dSPeter Grehan 		printf("vmx_modinit: processor does not support desired "
754366f6083SPeter Grehan 		    "entry controls\n");
755366f6083SPeter Grehan 		return (error);
756366f6083SPeter Grehan 	}
757366f6083SPeter Grehan 
758366f6083SPeter Grehan 	/*
759366f6083SPeter Grehan 	 * Check support for optional features by testing them
760366f6083SPeter Grehan 	 * as individual bits
761366f6083SPeter Grehan 	 */
762366f6083SPeter Grehan 	cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
763366f6083SPeter Grehan 					MSR_VMX_TRUE_PROCBASED_CTLS,
764366f6083SPeter Grehan 					PROCBASED_HLT_EXITING, 0,
765366f6083SPeter Grehan 					&tmp) == 0);
766366f6083SPeter Grehan 
767366f6083SPeter Grehan 	cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
768366f6083SPeter Grehan 					MSR_VMX_PROCBASED_CTLS,
769366f6083SPeter Grehan 					PROCBASED_MTF, 0,
770366f6083SPeter Grehan 					&tmp) == 0);
771366f6083SPeter Grehan 
772366f6083SPeter Grehan 	cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
773366f6083SPeter Grehan 					 MSR_VMX_TRUE_PROCBASED_CTLS,
774366f6083SPeter Grehan 					 PROCBASED_PAUSE_EXITING, 0,
775366f6083SPeter Grehan 					 &tmp) == 0);
776366f6083SPeter Grehan 
7773ba952e1SCorvin Köhne 	cap_wbinvd_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
7783ba952e1SCorvin Köhne 					MSR_VMX_PROCBASED_CTLS2,
7793ba952e1SCorvin Köhne 					PROCBASED2_WBINVD_EXITING,
7803ba952e1SCorvin Köhne 					0,
7813ba952e1SCorvin Köhne 					&tmp) == 0);
7823ba952e1SCorvin Köhne 
783f5f5f1e7SPeter Grehan 	/*
784f5f5f1e7SPeter Grehan 	 * Check support for RDPID and/or RDTSCP.
785f5f5f1e7SPeter Grehan 	 *
786f5f5f1e7SPeter Grehan 	 * Support a pass-through-based implementation of these via the
787f5f5f1e7SPeter Grehan 	 * "enable RDTSCP" VM-execution control and the "RDTSC exiting"
788f5f5f1e7SPeter Grehan 	 * VM-execution control.
789f5f5f1e7SPeter Grehan 	 *
790f5f5f1e7SPeter Grehan 	 * The "enable RDTSCP" VM-execution control applies to both RDPID
791f5f5f1e7SPeter Grehan 	 * and RDTSCP (see SDM volume 3, section 25.3, "Changes to
792f5f5f1e7SPeter Grehan 	 * Instruction Behavior in VMX Non-root operation"); this is why
793f5f5f1e7SPeter Grehan 	 * only this VM-execution control needs to be enabled in order to
794f5f5f1e7SPeter Grehan 	 * enable passing through whichever of RDPID and/or RDTSCP are
795f5f5f1e7SPeter Grehan 	 * supported by the host.
796f5f5f1e7SPeter Grehan 	 *
797f5f5f1e7SPeter Grehan 	 * The "RDTSC exiting" VM-execution control applies to both RDTSC
798f5f5f1e7SPeter Grehan 	 * and RDTSCP (again, per SDM volume 3, section 25.3), and is
799f5f5f1e7SPeter Grehan 	 * already set up for RDTSC and RDTSCP pass-through by the current
800f5f5f1e7SPeter Grehan 	 * implementation of RDTSC.
801f5f5f1e7SPeter Grehan 	 *
802f5f5f1e7SPeter Grehan 	 * Although RDPID and RDTSCP are optional capabilities, since there
803f5f5f1e7SPeter Grehan 	 * does not currently seem to be a use case for enabling/disabling
804f5f5f1e7SPeter Grehan 	 * these via libvmmapi, choose not to support this and, instead,
805f5f5f1e7SPeter Grehan 	 * just statically always enable or always disable this support
806f5f5f1e7SPeter Grehan 	 * across all vCPUs on all VMs. (Note that there may be some
807f5f5f1e7SPeter Grehan 	 * complications to providing this functionality, e.g., the MSR
808f5f5f1e7SPeter Grehan 	 * bitmap is currently per-VM rather than per-vCPU while the
809f5f5f1e7SPeter Grehan 	 * capability API wants to be able to control capabilities on a
810f5f5f1e7SPeter Grehan 	 * per-vCPU basis).
811f5f5f1e7SPeter Grehan 	 */
812f5f5f1e7SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
813f5f5f1e7SPeter Grehan 			       MSR_VMX_PROCBASED_CTLS2,
814f5f5f1e7SPeter Grehan 			       PROCBASED2_ENABLE_RDTSCP, 0, &tmp);
815f5f5f1e7SPeter Grehan 	cap_rdpid = error == 0 && host_has_rdpid();
816f5f5f1e7SPeter Grehan 	cap_rdtscp = error == 0 && host_has_rdtscp();
81773abae44SJohn Baldwin 	if (cap_rdpid || cap_rdtscp) {
818f5f5f1e7SPeter Grehan 		procbased_ctls2 |= PROCBASED2_ENABLE_RDTSCP;
81973abae44SJohn Baldwin 		vmx_have_msr_tsc_aux = true;
82073abae44SJohn Baldwin 	}
821f5f5f1e7SPeter Grehan 
822366f6083SPeter Grehan 	cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
823366f6083SPeter Grehan 					MSR_VMX_PROCBASED_CTLS2,
824366f6083SPeter Grehan 					PROCBASED2_UNRESTRICTED_GUEST, 0,
825366f6083SPeter Grehan 				        &tmp) == 0);
826366f6083SPeter Grehan 
82749cc03daSNeel Natu 	cap_invpcid = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
82849cc03daSNeel Natu 	    MSR_VMX_PROCBASED_CTLS2, PROCBASED2_ENABLE_INVPCID, 0,
82949cc03daSNeel Natu 	    &tmp) == 0);
83049cc03daSNeel Natu 
83188c4b8d1SNeel Natu 	/*
8321bc51badSMichael Reifenberger 	 * Check support for TPR shadow.
8331bc51badSMichael Reifenberger 	 */
8341bc51badSMichael Reifenberger 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
8351bc51badSMichael Reifenberger 	    MSR_VMX_TRUE_PROCBASED_CTLS, PROCBASED_USE_TPR_SHADOW, 0,
8361bc51badSMichael Reifenberger 	    &tmp);
8371bc51badSMichael Reifenberger 	if (error == 0) {
8381bc51badSMichael Reifenberger 		tpr_shadowing = 1;
839f3ff0918SZhenlei Huang #ifndef BURN_BRIDGES
8401bc51badSMichael Reifenberger 		TUNABLE_INT_FETCH("hw.vmm.vmx.use_tpr_shadowing",
8411bc51badSMichael Reifenberger 		    &tpr_shadowing);
842f3ff0918SZhenlei Huang #endif
843f3ff0918SZhenlei Huang 		TUNABLE_INT_FETCH("hw.vmm.vmx.cap.tpr_shadowing",
844f3ff0918SZhenlei Huang 		    &tpr_shadowing);
8451bc51badSMichael Reifenberger 	}
8461bc51badSMichael Reifenberger 
8471bc51badSMichael Reifenberger 	if (tpr_shadowing) {
8481bc51badSMichael Reifenberger 		procbased_ctls |= PROCBASED_USE_TPR_SHADOW;
8491bc51badSMichael Reifenberger 		procbased_ctls &= ~PROCBASED_CR8_LOAD_EXITING;
8501bc51badSMichael Reifenberger 		procbased_ctls &= ~PROCBASED_CR8_STORE_EXITING;
8511bc51badSMichael Reifenberger 	}
8521bc51badSMichael Reifenberger 
8531bc51badSMichael Reifenberger 	/*
85488c4b8d1SNeel Natu 	 * Check support for virtual interrupt delivery.
85588c4b8d1SNeel Natu 	 */
85688c4b8d1SNeel Natu 	procbased2_vid_bits = (PROCBASED2_VIRTUALIZE_APIC_ACCESSES |
85788c4b8d1SNeel Natu 	    PROCBASED2_VIRTUALIZE_X2APIC_MODE |
85888c4b8d1SNeel Natu 	    PROCBASED2_APIC_REGISTER_VIRTUALIZATION |
85988c4b8d1SNeel Natu 	    PROCBASED2_VIRTUAL_INTERRUPT_DELIVERY);
86088c4b8d1SNeel Natu 
86188c4b8d1SNeel Natu 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
86288c4b8d1SNeel Natu 	    procbased2_vid_bits, 0, &tmp);
8631bc51badSMichael Reifenberger 	if (error == 0 && tpr_shadowing) {
86488c4b8d1SNeel Natu 		virtual_interrupt_delivery = 1;
865f3ff0918SZhenlei Huang #ifndef BURN_BRIDGES
86688c4b8d1SNeel Natu 		TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_vid",
86788c4b8d1SNeel Natu 		    &virtual_interrupt_delivery);
868f3ff0918SZhenlei Huang #endif
869f3ff0918SZhenlei Huang 		TUNABLE_INT_FETCH("hw.vmm.vmx.cap.virtual_interrupt_delivery",
870f3ff0918SZhenlei Huang 		    &virtual_interrupt_delivery);
87188c4b8d1SNeel Natu 	}
87288c4b8d1SNeel Natu 
87388c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
87488c4b8d1SNeel Natu 		procbased_ctls |= PROCBASED_USE_TPR_SHADOW;
87588c4b8d1SNeel Natu 		procbased_ctls2 |= procbased2_vid_bits;
87688c4b8d1SNeel Natu 		procbased_ctls2 &= ~PROCBASED2_VIRTUALIZE_X2APIC_MODE;
877176666c2SNeel Natu 
878176666c2SNeel Natu 		/*
879176666c2SNeel Natu 		 * Check for Posted Interrupts only if Virtual Interrupt
880176666c2SNeel Natu 		 * Delivery is enabled.
881176666c2SNeel Natu 		 */
882176666c2SNeel Natu 		error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
883176666c2SNeel Natu 		    MSR_VMX_TRUE_PINBASED_CTLS, PINBASED_POSTED_INTERRUPT, 0,
884176666c2SNeel Natu 		    &tmp);
885176666c2SNeel Natu 		if (error == 0) {
886bd50262fSKonstantin Belousov 			pirvec = lapic_ipi_alloc(pti ? &IDTVEC(justreturn1_pti) :
887bd50262fSKonstantin Belousov 			    &IDTVEC(justreturn));
88818a2b08eSNeel Natu 			if (pirvec < 0) {
889176666c2SNeel Natu 				if (bootverbose) {
89015add60dSPeter Grehan 					printf("vmx_modinit: unable to "
89115add60dSPeter Grehan 					    "allocate posted interrupt "
89215add60dSPeter Grehan 					    "vector\n");
89388c4b8d1SNeel Natu 				}
894176666c2SNeel Natu 			} else {
895176666c2SNeel Natu 				posted_interrupts = 1;
896f3ff0918SZhenlei Huang #ifndef BURN_BRIDGES
897176666c2SNeel Natu 				TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_pir",
898176666c2SNeel Natu 				    &posted_interrupts);
899f3ff0918SZhenlei Huang #endif
900f3ff0918SZhenlei Huang 				TUNABLE_INT_FETCH("hw.vmm.vmx.cap.posted_interrupts",
901f3ff0918SZhenlei Huang 				    &posted_interrupts);
902176666c2SNeel Natu 			}
903176666c2SNeel Natu 		}
904176666c2SNeel Natu 	}
905176666c2SNeel Natu 
906176666c2SNeel Natu 	if (posted_interrupts)
907176666c2SNeel Natu 		    pinbased_ctls |= PINBASED_POSTED_INTERRUPT;
90849cc03daSNeel Natu 
909366f6083SPeter Grehan 	/* Initialize EPT */
910add611fdSNeel Natu 	error = ept_init(ipinum);
911366f6083SPeter Grehan 	if (error) {
91215add60dSPeter Grehan 		printf("vmx_modinit: ept initialization failed (%d)\n", error);
913366f6083SPeter Grehan 		return (error);
914366f6083SPeter Grehan 	}
915366f6083SPeter Grehan 
91623437573SKonstantin Belousov 	guest_l1d_flush = (cpu_ia32_arch_caps &
91723437573SKonstantin Belousov 	    IA32_ARCH_CAP_SKIP_L1DFL_VMENTRY) == 0;
918f3ff0918SZhenlei Huang #ifndef BURN_BRIDGES
919c30578feSKonstantin Belousov 	TUNABLE_INT_FETCH("hw.vmm.l1d_flush", &guest_l1d_flush);
920f3ff0918SZhenlei Huang #endif
921f3ff0918SZhenlei Huang 	TUNABLE_INT_FETCH("hw.vmm.vmx.l1d_flush", &guest_l1d_flush);
922c1141fbaSKonstantin Belousov 
923c1141fbaSKonstantin Belousov 	/*
924c1141fbaSKonstantin Belousov 	 * L1D cache flush is enabled.  Use IA32_FLUSH_CMD MSR when
925c1141fbaSKonstantin Belousov 	 * available.  Otherwise fall back to the software flush
926c1141fbaSKonstantin Belousov 	 * method which loads enough data from the kernel text to
927c1141fbaSKonstantin Belousov 	 * flush existing L1D content, both on VMX entry and on NMI
928c1141fbaSKonstantin Belousov 	 * return.
929c1141fbaSKonstantin Belousov 	 */
930c1141fbaSKonstantin Belousov 	if (guest_l1d_flush) {
931c1141fbaSKonstantin Belousov 		if ((cpu_stdext_feature3 & CPUID_STDEXT3_L1D_FLUSH) == 0) {
932c1141fbaSKonstantin Belousov 			guest_l1d_flush_sw = 1;
933f3ff0918SZhenlei Huang #ifndef BURN_BRIDGES
934c1141fbaSKonstantin Belousov 			TUNABLE_INT_FETCH("hw.vmm.l1d_flush_sw",
935c1141fbaSKonstantin Belousov 			    &guest_l1d_flush_sw);
936f3ff0918SZhenlei Huang #endif
937f3ff0918SZhenlei Huang 			TUNABLE_INT_FETCH("hw.vmm.vmx.l1d_flush_sw",
938f3ff0918SZhenlei Huang 			    &guest_l1d_flush_sw);
939c1141fbaSKonstantin Belousov 		}
940c1141fbaSKonstantin Belousov 		if (guest_l1d_flush_sw) {
941c1141fbaSKonstantin Belousov 			if (nmi_flush_l1d_sw <= 1)
942c1141fbaSKonstantin Belousov 				nmi_flush_l1d_sw = 1;
943c1141fbaSKonstantin Belousov 		} else {
944c1141fbaSKonstantin Belousov 			msr_load_list[0].index = MSR_IA32_FLUSH_CMD;
945c1141fbaSKonstantin Belousov 			msr_load_list[0].val = IA32_FLUSH_CMD_L1D;
946c1141fbaSKonstantin Belousov 		}
947c1141fbaSKonstantin Belousov 	}
948c30578feSKonstantin Belousov 
949366f6083SPeter Grehan 	/*
950366f6083SPeter Grehan 	 * Stash the cr0 and cr4 bits that must be fixed to 0 or 1
951366f6083SPeter Grehan 	 */
952366f6083SPeter Grehan 	fixed0 = rdmsr(MSR_VMX_CR0_FIXED0);
953366f6083SPeter Grehan 	fixed1 = rdmsr(MSR_VMX_CR0_FIXED1);
954366f6083SPeter Grehan 	cr0_ones_mask = fixed0 & fixed1;
955366f6083SPeter Grehan 	cr0_zeros_mask = ~fixed0 & ~fixed1;
956366f6083SPeter Grehan 
957366f6083SPeter Grehan 	/*
958366f6083SPeter Grehan 	 * CR0_PE and CR0_PG can be set to zero in VMX non-root operation
959366f6083SPeter Grehan 	 * if unrestricted guest execution is allowed.
960366f6083SPeter Grehan 	 */
961366f6083SPeter Grehan 	if (cap_unrestricted_guest)
962366f6083SPeter Grehan 		cr0_ones_mask &= ~(CR0_PG | CR0_PE);
963366f6083SPeter Grehan 
964366f6083SPeter Grehan 	/*
965366f6083SPeter Grehan 	 * Do not allow the guest to set CR0_NW or CR0_CD.
966366f6083SPeter Grehan 	 */
967366f6083SPeter Grehan 	cr0_zeros_mask |= (CR0_NW | CR0_CD);
968366f6083SPeter Grehan 
969366f6083SPeter Grehan 	fixed0 = rdmsr(MSR_VMX_CR4_FIXED0);
970366f6083SPeter Grehan 	fixed1 = rdmsr(MSR_VMX_CR4_FIXED1);
971366f6083SPeter Grehan 	cr4_ones_mask = fixed0 & fixed1;
972366f6083SPeter Grehan 	cr4_zeros_mask = ~fixed0 & ~fixed1;
973366f6083SPeter Grehan 
97445e51299SNeel Natu 	vpid_init();
97545e51299SNeel Natu 
976c3498942SNeel Natu 	vmx_msr_init();
977c3498942SNeel Natu 
978366f6083SPeter Grehan 	/* enable VMX operation */
97974ac712fSMark Johnston 	vmxon_region = kmem_malloc((mp_maxid + 1) * PAGE_SIZE,
98074ac712fSMark Johnston 	    M_WAITOK | M_ZERO);
981366f6083SPeter Grehan 	smp_rendezvous(NULL, vmx_enable, NULL, NULL);
982366f6083SPeter Grehan 
9833565b59eSNeel Natu 	vmx_initialized = 1;
9843565b59eSNeel Natu 
985366f6083SPeter Grehan 	return (0);
986366f6083SPeter Grehan }
987366f6083SPeter Grehan 
988f7d47425SNeel Natu static void
989f7d47425SNeel Natu vmx_trigger_hostintr(int vector)
990f7d47425SNeel Natu {
991f7d47425SNeel Natu 	uintptr_t func;
992f7d47425SNeel Natu 	struct gate_descriptor *gd;
993f7d47425SNeel Natu 
994f7d47425SNeel Natu 	gd = &idt[vector];
995f7d47425SNeel Natu 
996f7d47425SNeel Natu 	KASSERT(vector >= 32 && vector <= 255, ("vmx_trigger_hostintr: "
997f7d47425SNeel Natu 	    "invalid vector %d", vector));
998f7d47425SNeel Natu 	KASSERT(gd->gd_p == 1, ("gate descriptor for vector %d not present",
999f7d47425SNeel Natu 	    vector));
1000f7d47425SNeel Natu 	KASSERT(gd->gd_type == SDT_SYSIGT, ("gate descriptor for vector %d "
1001f7d47425SNeel Natu 	    "has invalid type %d", vector, gd->gd_type));
1002f7d47425SNeel Natu 	KASSERT(gd->gd_dpl == SEL_KPL, ("gate descriptor for vector %d "
1003f7d47425SNeel Natu 	    "has invalid dpl %d", vector, gd->gd_dpl));
1004f7d47425SNeel Natu 	KASSERT(gd->gd_selector == GSEL(GCODE_SEL, SEL_KPL), ("gate descriptor "
1005f7d47425SNeel Natu 	    "for vector %d has invalid selector %d", vector, gd->gd_selector));
1006f7d47425SNeel Natu 	KASSERT(gd->gd_ist == 0, ("gate descriptor for vector %d has invalid "
1007f7d47425SNeel Natu 	    "IST %d", vector, gd->gd_ist));
1008f7d47425SNeel Natu 
1009f7d47425SNeel Natu 	func = ((long)gd->gd_hioffset << 16 | gd->gd_looffset);
1010f7d47425SNeel Natu 	vmx_call_isr(func);
1011f7d47425SNeel Natu }
1012f7d47425SNeel Natu 
1013366f6083SPeter Grehan static int
1014aaaa0656SPeter Grehan vmx_setup_cr_shadow(int which, struct vmcs *vmcs, uint32_t initial)
1015366f6083SPeter Grehan {
101639c21c2dSNeel Natu 	int error, mask_ident, shadow_ident;
1017aaaa0656SPeter Grehan 	uint64_t mask_value;
1018366f6083SPeter Grehan 
101939c21c2dSNeel Natu 	if (which != 0 && which != 4)
102039c21c2dSNeel Natu 		panic("vmx_setup_cr_shadow: unknown cr%d", which);
102139c21c2dSNeel Natu 
102239c21c2dSNeel Natu 	if (which == 0) {
102339c21c2dSNeel Natu 		mask_ident = VMCS_CR0_MASK;
102439c21c2dSNeel Natu 		mask_value = cr0_ones_mask | cr0_zeros_mask;
102539c21c2dSNeel Natu 		shadow_ident = VMCS_CR0_SHADOW;
102639c21c2dSNeel Natu 	} else {
102739c21c2dSNeel Natu 		mask_ident = VMCS_CR4_MASK;
102839c21c2dSNeel Natu 		mask_value = cr4_ones_mask | cr4_zeros_mask;
102939c21c2dSNeel Natu 		shadow_ident = VMCS_CR4_SHADOW;
103039c21c2dSNeel Natu 	}
103139c21c2dSNeel Natu 
1032d3c11f40SPeter Grehan 	error = vmcs_setreg(vmcs, 0, VMCS_IDENT(mask_ident), mask_value);
1033366f6083SPeter Grehan 	if (error)
1034366f6083SPeter Grehan 		return (error);
1035366f6083SPeter Grehan 
1036aaaa0656SPeter Grehan 	error = vmcs_setreg(vmcs, 0, VMCS_IDENT(shadow_ident), initial);
1037366f6083SPeter Grehan 	if (error)
1038366f6083SPeter Grehan 		return (error);
1039366f6083SPeter Grehan 
1040366f6083SPeter Grehan 	return (0);
1041366f6083SPeter Grehan }
1042aaaa0656SPeter Grehan #define	vmx_setup_cr0_shadow(vmcs,init)	vmx_setup_cr_shadow(0, (vmcs), (init))
1043aaaa0656SPeter Grehan #define	vmx_setup_cr4_shadow(vmcs,init)	vmx_setup_cr_shadow(4, (vmcs), (init))
1044366f6083SPeter Grehan 
1045366f6083SPeter Grehan static void *
104615add60dSPeter Grehan vmx_init(struct vm *vm, pmap_t pmap)
1047366f6083SPeter Grehan {
1048d487cba3SCy Schubert 	int error __diagused;
1049366f6083SPeter Grehan 	struct vmx *vmx;
1050366f6083SPeter Grehan 
1051366f6083SPeter Grehan 	vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO);
1052366f6083SPeter Grehan 	vmx->vm = vm;
1053366f6083SPeter Grehan 
10549ce875d9SKonstantin Belousov 	vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pmltop));
1055318224bbSNeel Natu 
1056366f6083SPeter Grehan 	/*
1057366f6083SPeter Grehan 	 * Clean up EPTP-tagged guest physical and combined mappings
1058366f6083SPeter Grehan 	 *
1059366f6083SPeter Grehan 	 * VMX transitions are not required to invalidate any guest physical
1060366f6083SPeter Grehan 	 * mappings. So, it may be possible for stale guest physical mappings
1061366f6083SPeter Grehan 	 * to be present in the processor TLBs.
1062366f6083SPeter Grehan 	 *
1063366f6083SPeter Grehan 	 * Combined mappings for this EP4TA are also invalidated for all VPIDs.
1064366f6083SPeter Grehan 	 */
1065318224bbSNeel Natu 	ept_invalidate_mappings(vmx->eptp);
1066366f6083SPeter Grehan 
10670f00260cSJohn Baldwin 	vmx->msr_bitmap = malloc_aligned(PAGE_SIZE, PAGE_SIZE, M_VMX,
10680f00260cSJohn Baldwin 	    M_WAITOK | M_ZERO);
1069366f6083SPeter Grehan 	msr_bitmap_initialize(vmx->msr_bitmap);
1070366f6083SPeter Grehan 
1071366f6083SPeter Grehan 	/*
1072366f6083SPeter Grehan 	 * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE.
1073366f6083SPeter Grehan 	 * The guest FSBASE and GSBASE are saved and restored during
1074366f6083SPeter Grehan 	 * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are
1075366f6083SPeter Grehan 	 * always restored from the vmcs host state area on vm-exit.
1076366f6083SPeter Grehan 	 *
10771fb0ea3fSPeter Grehan 	 * The SYSENTER_CS/ESP/EIP MSRs are identical to FS/GSBASE in
10781fb0ea3fSPeter Grehan 	 * how they are saved/restored so can be directly accessed by the
10791fb0ea3fSPeter Grehan 	 * guest.
10801fb0ea3fSPeter Grehan 	 *
1081366f6083SPeter Grehan 	 * MSR_EFER is saved and restored in the guest VMCS area on a
1082366f6083SPeter Grehan 	 * VM exit and entry respectively. It is also restored from the
1083366f6083SPeter Grehan 	 * host VMCS area on a VM exit.
10848d1d7a9eSPeter Grehan 	 *
1085277bdd99STycho Nightingale 	 * The TSC MSR is exposed read-only. Writes are disallowed as
1086277bdd99STycho Nightingale 	 * that will impact the host TSC.  If the guest does a write
1087277bdd99STycho Nightingale 	 * the "use TSC offsetting" execution control is enabled and the
1088277bdd99STycho Nightingale 	 * difference between the host TSC and the guest TSC is written
1089277bdd99STycho Nightingale 	 * into the TSC offset in the VMCS.
1090f5f5f1e7SPeter Grehan 	 *
1091f5f5f1e7SPeter Grehan 	 * Guest TSC_AUX support is enabled if any of guest RDPID and/or
1092f5f5f1e7SPeter Grehan 	 * guest RDTSCP support are enabled (since, as per Table 2-2 in SDM
1093f5f5f1e7SPeter Grehan 	 * volume 4, TSC_AUX is supported if any of RDPID and/or RDTSCP are
1094f5f5f1e7SPeter Grehan 	 * supported). If guest TSC_AUX support is enabled, TSC_AUX is
1095f5f5f1e7SPeter Grehan 	 * exposed read-only so that the VMM can do one fewer MSR read per
1096f5f5f1e7SPeter Grehan 	 * exit than if this register were exposed read-write; the guest
1097f5f5f1e7SPeter Grehan 	 * restore value can be updated during guest writes (expected to be
1098f5f5f1e7SPeter Grehan 	 * rare) instead of during all exits (common).
1099366f6083SPeter Grehan 	 */
1100366f6083SPeter Grehan 	if (guest_msr_rw(vmx, MSR_GSBASE) ||
1101366f6083SPeter Grehan 	    guest_msr_rw(vmx, MSR_FSBASE) ||
11021fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_CS_MSR) ||
11031fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_ESP_MSR) ||
11041fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_EIP_MSR) ||
11058d1d7a9eSPeter Grehan 	    guest_msr_rw(vmx, MSR_EFER) ||
1106f5f5f1e7SPeter Grehan 	    guest_msr_ro(vmx, MSR_TSC) ||
1107f5f5f1e7SPeter Grehan 	    ((cap_rdpid || cap_rdtscp) && guest_msr_ro(vmx, MSR_TSC_AUX)))
110815add60dSPeter Grehan 		panic("vmx_init: error setting guest msr access");
1109366f6083SPeter Grehan 
111088c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
111188c4b8d1SNeel Natu 		error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE,
111288c4b8d1SNeel Natu 		    APIC_ACCESS_ADDRESS);
111388c4b8d1SNeel Natu 		/* XXX this should really return an error to the caller */
111488c4b8d1SNeel Natu 		KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error));
111588c4b8d1SNeel Natu 	}
111688c4b8d1SNeel Natu 
11171aa51504SJohn Baldwin 	vmx->pmap = pmap;
11181aa51504SJohn Baldwin 	return (vmx);
11191aa51504SJohn Baldwin }
11200f00260cSJohn Baldwin 
11211aa51504SJohn Baldwin static void *
1122950af9ffSJohn Baldwin vmx_vcpu_init(void *vmi, struct vcpu *vcpu1, int vcpuid)
11231aa51504SJohn Baldwin {
1124869c8d19SJohn Baldwin 	struct vmx *vmx = vmi;
11251aa51504SJohn Baldwin 	struct vmcs *vmcs;
11261aa51504SJohn Baldwin 	struct vmx_vcpu *vcpu;
11271aa51504SJohn Baldwin 	uint32_t exc_bitmap;
112858eefc67SJohn Baldwin 	uint16_t vpid;
11291aa51504SJohn Baldwin 	int error;
11301aa51504SJohn Baldwin 
113158eefc67SJohn Baldwin 	vpid = vpid_alloc(vcpuid);
113258eefc67SJohn Baldwin 
11331aa51504SJohn Baldwin 	vcpu = malloc(sizeof(*vcpu), M_VMX, M_WAITOK | M_ZERO);
1134869c8d19SJohn Baldwin 	vcpu->vmx = vmx;
1135950af9ffSJohn Baldwin 	vcpu->vcpu = vcpu1;
11361aa51504SJohn Baldwin 	vcpu->vcpuid = vcpuid;
11370f00260cSJohn Baldwin 	vcpu->vmcs = malloc_aligned(sizeof(*vmcs), PAGE_SIZE, M_VMX,
11380f00260cSJohn Baldwin 	    M_WAITOK | M_ZERO);
11390f00260cSJohn Baldwin 	vcpu->apic_page = malloc_aligned(PAGE_SIZE, PAGE_SIZE, M_VMX,
11400f00260cSJohn Baldwin 	    M_WAITOK | M_ZERO);
11411aa51504SJohn Baldwin 	vcpu->pir_desc = malloc_aligned(sizeof(*vcpu->pir_desc), 64, M_VMX,
11421aa51504SJohn Baldwin 	    M_WAITOK | M_ZERO);
11430f00260cSJohn Baldwin 
11440f00260cSJohn Baldwin 	vmcs = vcpu->vmcs;
1145c847a506SNeel Natu 	vmcs->identifier = vmx_revision();
1146c847a506SNeel Natu 	error = vmclear(vmcs);
1147366f6083SPeter Grehan 	if (error != 0) {
114815add60dSPeter Grehan 		panic("vmx_init: vmclear error %d on vcpu %d\n",
11491aa51504SJohn Baldwin 		    error, vcpuid);
1150366f6083SPeter Grehan 	}
1151366f6083SPeter Grehan 
11521aa51504SJohn Baldwin 	vmx_msr_guest_init(vmx, vcpu);
1153c3498942SNeel Natu 
1154c847a506SNeel Natu 	error = vmcs_init(vmcs);
1155c847a506SNeel Natu 	KASSERT(error == 0, ("vmcs_init error %d", error));
1156366f6083SPeter Grehan 
1157c847a506SNeel Natu 	VMPTRLD(vmcs);
1158c847a506SNeel Natu 	error = 0;
11590f00260cSJohn Baldwin 	error += vmwrite(VMCS_HOST_RSP, (u_long)&vcpu->ctx);
1160c847a506SNeel Natu 	error += vmwrite(VMCS_EPTP, vmx->eptp);
1161c847a506SNeel Natu 	error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls);
1162c847a506SNeel Natu 	error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls);
116380cb5d84SJohn Baldwin 	if (vcpu_trap_wbinvd(vcpu->vcpu)) {
11643ba952e1SCorvin Köhne 		KASSERT(cap_wbinvd_exit, ("WBINVD trap not available"));
11653ba952e1SCorvin Köhne 		procbased_ctls2 |= PROCBASED2_WBINVD_EXITING;
11663ba952e1SCorvin Köhne 	}
1167c847a506SNeel Natu 	error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2);
1168c847a506SNeel Natu 	error += vmwrite(VMCS_EXIT_CTLS, exit_ctls);
1169c847a506SNeel Natu 	error += vmwrite(VMCS_ENTRY_CTLS, entry_ctls);
1170c847a506SNeel Natu 	error += vmwrite(VMCS_MSR_BITMAP, vtophys(vmx->msr_bitmap));
117158eefc67SJohn Baldwin 	error += vmwrite(VMCS_VPID, vpid);
1172b0538143SNeel Natu 
1173c1141fbaSKonstantin Belousov 	if (guest_l1d_flush && !guest_l1d_flush_sw) {
1174c1141fbaSKonstantin Belousov 		vmcs_write(VMCS_ENTRY_MSR_LOAD, pmap_kextract(
1175c1141fbaSKonstantin Belousov 			(vm_offset_t)&msr_load_list[0]));
1176c1141fbaSKonstantin Belousov 		vmcs_write(VMCS_ENTRY_MSR_LOAD_COUNT,
1177c1141fbaSKonstantin Belousov 		    nitems(msr_load_list));
1178c1141fbaSKonstantin Belousov 		vmcs_write(VMCS_EXIT_MSR_STORE, 0);
1179c1141fbaSKonstantin Belousov 		vmcs_write(VMCS_EXIT_MSR_STORE_COUNT, 0);
1180c1141fbaSKonstantin Belousov 	}
1181c1141fbaSKonstantin Belousov 
1182b0538143SNeel Natu 	/* exception bitmap */
118380cb5d84SJohn Baldwin 	if (vcpu_trace_exceptions(vcpu->vcpu))
1184b0538143SNeel Natu 		exc_bitmap = 0xffffffff;
1185b0538143SNeel Natu 	else
1186b0538143SNeel Natu 		exc_bitmap = 1 << IDT_MC;
1187b0538143SNeel Natu 	error += vmwrite(VMCS_EXCEPTION_BITMAP, exc_bitmap);
1188b0538143SNeel Natu 
11890f00260cSJohn Baldwin 	vcpu->ctx.guest_dr6 = DBREG_DR6_RESERVED1;
11909e2154ffSJohn Baldwin 	error += vmwrite(VMCS_GUEST_DR7, DBREG_DR7_RESERVED1);
119165eefbe4SJohn Baldwin 
11921bc51badSMichael Reifenberger 	if (tpr_shadowing) {
11931aa51504SJohn Baldwin 		error += vmwrite(VMCS_VIRTUAL_APIC, vtophys(vcpu->apic_page));
11941bc51badSMichael Reifenberger 	}
11951bc51badSMichael Reifenberger 
11961bc51badSMichael Reifenberger 	if (virtual_interrupt_delivery) {
11971bc51badSMichael Reifenberger 		error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS);
119888c4b8d1SNeel Natu 		error += vmwrite(VMCS_EOI_EXIT0, 0);
119988c4b8d1SNeel Natu 		error += vmwrite(VMCS_EOI_EXIT1, 0);
120088c4b8d1SNeel Natu 		error += vmwrite(VMCS_EOI_EXIT2, 0);
120188c4b8d1SNeel Natu 		error += vmwrite(VMCS_EOI_EXIT3, 0);
120288c4b8d1SNeel Natu 	}
1203176666c2SNeel Natu 	if (posted_interrupts) {
1204176666c2SNeel Natu 		error += vmwrite(VMCS_PIR_VECTOR, pirvec);
12051aa51504SJohn Baldwin 		error += vmwrite(VMCS_PIR_DESC, vtophys(vcpu->pir_desc));
1206176666c2SNeel Natu 	}
1207c847a506SNeel Natu 	VMCLEAR(vmcs);
120815add60dSPeter Grehan 	KASSERT(error == 0, ("vmx_init: error customizing the vmcs"));
1209366f6083SPeter Grehan 
12100f00260cSJohn Baldwin 	vcpu->cap.set = 0;
12110f00260cSJohn Baldwin 	vcpu->cap.set |= cap_rdpid != 0 ? 1 << VM_CAP_RDPID : 0;
12120f00260cSJohn Baldwin 	vcpu->cap.set |= cap_rdtscp != 0 ? 1 << VM_CAP_RDTSCP : 0;
12130f00260cSJohn Baldwin 	vcpu->cap.proc_ctls = procbased_ctls;
12140f00260cSJohn Baldwin 	vcpu->cap.proc_ctls2 = procbased_ctls2;
12150f00260cSJohn Baldwin 	vcpu->cap.exc_bitmap = exc_bitmap;
1216366f6083SPeter Grehan 
12170f00260cSJohn Baldwin 	vcpu->state.nextrip = ~0;
12180f00260cSJohn Baldwin 	vcpu->state.lastcpu = NOCPU;
121958eefc67SJohn Baldwin 	vcpu->state.vpid = vpid;
1220366f6083SPeter Grehan 
1221aaaa0656SPeter Grehan 	/*
1222aaaa0656SPeter Grehan 	 * Set up the CR0/4 shadows, and init the read shadow
1223aaaa0656SPeter Grehan 	 * to the power-on register value from the Intel Sys Arch.
1224aaaa0656SPeter Grehan 	 *  CR0 - 0x60000010
1225aaaa0656SPeter Grehan 	 *  CR4 - 0
1226aaaa0656SPeter Grehan 	 */
1227c847a506SNeel Natu 	error = vmx_setup_cr0_shadow(vmcs, 0x60000010);
122839c21c2dSNeel Natu 	if (error != 0)
122939c21c2dSNeel Natu 		panic("vmx_setup_cr0_shadow %d", error);
123039c21c2dSNeel Natu 
1231c847a506SNeel Natu 	error = vmx_setup_cr4_shadow(vmcs, 0);
123239c21c2dSNeel Natu 	if (error != 0)
123339c21c2dSNeel Natu 		panic("vmx_setup_cr4_shadow %d", error);
1234318224bbSNeel Natu 
12351aa51504SJohn Baldwin 	vcpu->ctx.pmap = vmx->pmap;
1236366f6083SPeter Grehan 
12371aa51504SJohn Baldwin 	return (vcpu);
1238366f6083SPeter Grehan }
1239366f6083SPeter Grehan 
1240366f6083SPeter Grehan static int
124180cb5d84SJohn Baldwin vmx_handle_cpuid(struct vmx_vcpu *vcpu, struct vmxctx *vmxctx)
1242366f6083SPeter Grehan {
1243a3f2a9c5SJohn Baldwin 	int handled;
1244366f6083SPeter Grehan 
124580cb5d84SJohn Baldwin 	handled = x86_emulate_cpuid(vcpu->vcpu, (uint64_t *)&vmxctx->guest_rax,
1246a3f2a9c5SJohn Baldwin 	    (uint64_t *)&vmxctx->guest_rbx, (uint64_t *)&vmxctx->guest_rcx,
1247a3f2a9c5SJohn Baldwin 	    (uint64_t *)&vmxctx->guest_rdx);
1248366f6083SPeter Grehan 	return (handled);
1249366f6083SPeter Grehan }
1250366f6083SPeter Grehan 
1251366f6083SPeter Grehan static __inline void
1252869c8d19SJohn Baldwin vmx_run_trace(struct vmx_vcpu *vcpu)
1253366f6083SPeter Grehan {
125457e0119eSJohn Baldwin 	VMX_CTR1(vcpu, "Resume execution at %#lx", vmcs_guest_rip());
1255366f6083SPeter Grehan }
1256366f6083SPeter Grehan 
1257366f6083SPeter Grehan static __inline void
1258869c8d19SJohn Baldwin vmx_exit_trace(struct vmx_vcpu *vcpu, uint64_t rip, uint32_t exit_reason,
1259869c8d19SJohn Baldwin     int handled)
1260366f6083SPeter Grehan {
126157e0119eSJohn Baldwin 	VMX_CTR3(vcpu, "%s %s vmexit at 0x%0lx",
1262366f6083SPeter Grehan 		 handled ? "handled" : "unhandled",
1263366f6083SPeter Grehan 		 exit_reason_to_str(exit_reason), rip);
1264eeefa4e4SNeel Natu }
1265366f6083SPeter Grehan 
1266eeefa4e4SNeel Natu static __inline void
1267869c8d19SJohn Baldwin vmx_astpending_trace(struct vmx_vcpu *vcpu, uint64_t rip)
1268eeefa4e4SNeel Natu {
126957e0119eSJohn Baldwin 	VMX_CTR1(vcpu, "astpending vmexit at 0x%0lx", rip);
1270366f6083SPeter Grehan }
1271366f6083SPeter Grehan 
1272953c2c47SNeel Natu static VMM_STAT_INTEL(VCPU_INVVPID_SAVED, "Number of vpid invalidations saved");
12733527963bSNeel Natu static VMM_STAT_INTEL(VCPU_INVVPID_DONE, "Number of vpid invalidations done");
1274953c2c47SNeel Natu 
12753527963bSNeel Natu /*
12763527963bSNeel Natu  * Invalidate guest mappings identified by its vpid from the TLB.
12773527963bSNeel Natu  */
12783527963bSNeel Natu static __inline void
12791aa51504SJohn Baldwin vmx_invvpid(struct vmx *vmx, struct vmx_vcpu *vcpu, pmap_t pmap, int running)
1280366f6083SPeter Grehan {
1281366f6083SPeter Grehan 	struct vmxstate *vmxstate;
1282953c2c47SNeel Natu 	struct invvpid_desc invvpid_desc;
1283366f6083SPeter Grehan 
12841aa51504SJohn Baldwin 	vmxstate = &vcpu->state;
12853527963bSNeel Natu 	if (vmxstate->vpid == 0)
12863de83862SNeel Natu 		return;
1287366f6083SPeter Grehan 
12883527963bSNeel Natu 	if (!running) {
12893527963bSNeel Natu 		/*
12903527963bSNeel Natu 		 * Set the 'lastcpu' to an invalid host cpu.
12913527963bSNeel Natu 		 *
12923527963bSNeel Natu 		 * This will invalidate TLB entries tagged with the vcpu's
12933527963bSNeel Natu 		 * vpid the next time it runs via vmx_set_pcpu_defaults().
12943527963bSNeel Natu 		 */
12953527963bSNeel Natu 		vmxstate->lastcpu = NOCPU;
12963527963bSNeel Natu 		return;
12973527963bSNeel Natu 	}
1298953c2c47SNeel Natu 
12993527963bSNeel Natu 	KASSERT(curthread->td_critnest > 0, ("%s: vcpu %d running outside "
13001aa51504SJohn Baldwin 	    "critical section", __func__, vcpu->vcpuid));
1301366f6083SPeter Grehan 
1302366f6083SPeter Grehan 	/*
13033527963bSNeel Natu 	 * Invalidate all mappings tagged with 'vpid'
1304366f6083SPeter Grehan 	 *
1305366f6083SPeter Grehan 	 * We do this because this vcpu was executing on a different host
1306366f6083SPeter Grehan 	 * cpu when it last ran. We do not track whether it invalidated
1307366f6083SPeter Grehan 	 * mappings associated with its 'vpid' during that run. So we must
1308366f6083SPeter Grehan 	 * assume that the mappings associated with 'vpid' on 'curcpu' are
1309366f6083SPeter Grehan 	 * stale and invalidate them.
1310366f6083SPeter Grehan 	 *
1311366f6083SPeter Grehan 	 * Note that we incur this penalty only when the scheduler chooses to
1312366f6083SPeter Grehan 	 * move the thread associated with this vcpu between host cpus.
1313366f6083SPeter Grehan 	 *
1314366f6083SPeter Grehan 	 * Note also that this will invalidate mappings tagged with 'vpid'
1315366f6083SPeter Grehan 	 * for "all" EP4TAs.
1316366f6083SPeter Grehan 	 */
13176f5a9606SMark Johnston 	if (atomic_load_long(&pmap->pm_eptgen) == vmx->eptgen[curcpu]) {
1318953c2c47SNeel Natu 		invvpid_desc._res1 = 0;
1319953c2c47SNeel Natu 		invvpid_desc._res2 = 0;
1320366f6083SPeter Grehan 		invvpid_desc.vpid = vmxstate->vpid;
13210e30c5c0SWarner Losh 		invvpid_desc.linear_addr = 0;
1322366f6083SPeter Grehan 		invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc);
13233dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VCPU_INVVPID_DONE, 1);
1324953c2c47SNeel Natu 	} else {
1325953c2c47SNeel Natu 		/*
1326953c2c47SNeel Natu 		 * The invvpid can be skipped if an invept is going to
1327953c2c47SNeel Natu 		 * be performed before entering the guest. The invept
1328953c2c47SNeel Natu 		 * will invalidate combined mappings tagged with
1329953c2c47SNeel Natu 		 * 'vmx->eptp' for all vpids.
1330953c2c47SNeel Natu 		 */
13313dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VCPU_INVVPID_SAVED, 1);
1332953c2c47SNeel Natu 	}
1333366f6083SPeter Grehan }
13343527963bSNeel Natu 
13353527963bSNeel Natu static void
13361aa51504SJohn Baldwin vmx_set_pcpu_defaults(struct vmx *vmx, struct vmx_vcpu *vcpu, pmap_t pmap)
13373527963bSNeel Natu {
13383527963bSNeel Natu 	struct vmxstate *vmxstate;
13393527963bSNeel Natu 
13401aa51504SJohn Baldwin 	vmxstate = &vcpu->state;
13413527963bSNeel Natu 	if (vmxstate->lastcpu == curcpu)
13423527963bSNeel Natu 		return;
13433527963bSNeel Natu 
13443527963bSNeel Natu 	vmxstate->lastcpu = curcpu;
13453527963bSNeel Natu 
13463dc3d32aSJohn Baldwin 	vmm_stat_incr(vcpu->vcpu, VCPU_MIGRATIONS, 1);
13473527963bSNeel Natu 
13483527963bSNeel Natu 	vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase());
13493527963bSNeel Natu 	vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase());
13503527963bSNeel Natu 	vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase());
13513527963bSNeel Natu 	vmx_invvpid(vmx, vcpu, pmap, 1);
1352366f6083SPeter Grehan }
1353366f6083SPeter Grehan 
1354366f6083SPeter Grehan /*
1355366f6083SPeter Grehan  * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set.
1356366f6083SPeter Grehan  */
1357366f6083SPeter Grehan CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0);
1358366f6083SPeter Grehan 
1359366f6083SPeter Grehan static void __inline
1360869c8d19SJohn Baldwin vmx_set_int_window_exiting(struct vmx_vcpu *vcpu)
1361366f6083SPeter Grehan {
1362366f6083SPeter Grehan 
13631aa51504SJohn Baldwin 	if ((vcpu->cap.proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) {
13641aa51504SJohn Baldwin 		vcpu->cap.proc_ctls |= PROCBASED_INT_WINDOW_EXITING;
13651aa51504SJohn Baldwin 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls);
136657e0119eSJohn Baldwin 		VMX_CTR0(vcpu, "Enabling interrupt window exiting");
136748b2d828SNeel Natu 	}
1368366f6083SPeter Grehan }
1369366f6083SPeter Grehan 
1370366f6083SPeter Grehan static void __inline
1371869c8d19SJohn Baldwin vmx_clear_int_window_exiting(struct vmx_vcpu *vcpu)
1372366f6083SPeter Grehan {
1373366f6083SPeter Grehan 
13741aa51504SJohn Baldwin 	KASSERT((vcpu->cap.proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0,
13751aa51504SJohn Baldwin 	    ("intr_window_exiting not set: %#x", vcpu->cap.proc_ctls));
13761aa51504SJohn Baldwin 	vcpu->cap.proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING;
13771aa51504SJohn Baldwin 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls);
137857e0119eSJohn Baldwin 	VMX_CTR0(vcpu, "Disabling interrupt window exiting");
1379366f6083SPeter Grehan }
1380366f6083SPeter Grehan 
1381366f6083SPeter Grehan static void __inline
1382869c8d19SJohn Baldwin vmx_set_nmi_window_exiting(struct vmx_vcpu *vcpu)
1383366f6083SPeter Grehan {
1384366f6083SPeter Grehan 
13851aa51504SJohn Baldwin 	if ((vcpu->cap.proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) {
13861aa51504SJohn Baldwin 		vcpu->cap.proc_ctls |= PROCBASED_NMI_WINDOW_EXITING;
13871aa51504SJohn Baldwin 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls);
138857e0119eSJohn Baldwin 		VMX_CTR0(vcpu, "Enabling NMI window exiting");
138948b2d828SNeel Natu 	}
1390366f6083SPeter Grehan }
1391366f6083SPeter Grehan 
1392366f6083SPeter Grehan static void __inline
1393869c8d19SJohn Baldwin vmx_clear_nmi_window_exiting(struct vmx_vcpu *vcpu)
1394366f6083SPeter Grehan {
1395366f6083SPeter Grehan 
13961aa51504SJohn Baldwin 	KASSERT((vcpu->cap.proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0,
13971aa51504SJohn Baldwin 	    ("nmi_window_exiting not set %#x", vcpu->cap.proc_ctls));
13981aa51504SJohn Baldwin 	vcpu->cap.proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING;
13991aa51504SJohn Baldwin 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls);
140057e0119eSJohn Baldwin 	VMX_CTR0(vcpu, "Disabling NMI window exiting");
1401366f6083SPeter Grehan }
1402366f6083SPeter Grehan 
1403277bdd99STycho Nightingale int
140480cb5d84SJohn Baldwin vmx_set_tsc_offset(struct vmx_vcpu *vcpu, uint64_t offset)
1405277bdd99STycho Nightingale {
1406277bdd99STycho Nightingale 	int error;
1407277bdd99STycho Nightingale 
14081aa51504SJohn Baldwin 	if ((vcpu->cap.proc_ctls & PROCBASED_TSC_OFFSET) == 0) {
14091aa51504SJohn Baldwin 		vcpu->cap.proc_ctls |= PROCBASED_TSC_OFFSET;
14101aa51504SJohn Baldwin 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls);
141157e0119eSJohn Baldwin 		VMX_CTR0(vcpu, "Enabling TSC offsetting");
1412277bdd99STycho Nightingale 	}
1413277bdd99STycho Nightingale 
1414277bdd99STycho Nightingale 	error = vmwrite(VMCS_TSC_OFFSET, offset);
1415483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT
1416483d953aSJohn Baldwin 	if (error == 0)
141780cb5d84SJohn Baldwin 		vm_set_tsc_offset(vcpu->vcpu, offset);
1418483d953aSJohn Baldwin #endif
1419277bdd99STycho Nightingale 	return (error);
1420277bdd99STycho Nightingale }
1421277bdd99STycho Nightingale 
142248b2d828SNeel Natu #define	NMI_BLOCKING	(VMCS_INTERRUPTIBILITY_NMI_BLOCKING |		\
142348b2d828SNeel Natu 			 VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
142448b2d828SNeel Natu #define	HWINTR_BLOCKING	(VMCS_INTERRUPTIBILITY_STI_BLOCKING |		\
142548b2d828SNeel Natu 			 VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
142648b2d828SNeel Natu 
142748b2d828SNeel Natu static void
142880cb5d84SJohn Baldwin vmx_inject_nmi(struct vmx_vcpu *vcpu)
1429366f6083SPeter Grehan {
14305c272efaSRobert Wing 	uint32_t gi __diagused, info;
1431366f6083SPeter Grehan 
143248b2d828SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
143348b2d828SNeel Natu 	KASSERT((gi & NMI_BLOCKING) == 0, ("vmx_inject_nmi: invalid guest "
143448b2d828SNeel Natu 	    "interruptibility-state %#x", gi));
1435366f6083SPeter Grehan 
143648b2d828SNeel Natu 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
143748b2d828SNeel Natu 	KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_nmi: invalid "
143848b2d828SNeel Natu 	    "VM-entry interruption information %#x", info));
1439366f6083SPeter Grehan 
1440366f6083SPeter Grehan 	/*
1441366f6083SPeter Grehan 	 * Inject the virtual NMI. The vector must be the NMI IDT entry
1442366f6083SPeter Grehan 	 * or the VMCS entry check will fail.
1443366f6083SPeter Grehan 	 */
144448b2d828SNeel Natu 	info = IDT_NMI | VMCS_INTR_T_NMI | VMCS_INTR_VALID;
14453de83862SNeel Natu 	vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1446366f6083SPeter Grehan 
144757e0119eSJohn Baldwin 	VMX_CTR0(vcpu, "Injecting vNMI");
1448366f6083SPeter Grehan 
1449366f6083SPeter Grehan 	/* Clear the request */
145080cb5d84SJohn Baldwin 	vm_nmi_clear(vcpu->vcpu);
1451366f6083SPeter Grehan }
1452366f6083SPeter Grehan 
1453366f6083SPeter Grehan static void
145480cb5d84SJohn Baldwin vmx_inject_interrupts(struct vmx_vcpu *vcpu, struct vlapic *vlapic,
145580cb5d84SJohn Baldwin     uint64_t guestrip)
1456366f6083SPeter Grehan {
14570775fbb4STycho Nightingale 	int vector, need_nmi_exiting, extint_pending;
1458091d4532SNeel Natu 	uint64_t rflags, entryinfo;
145948b2d828SNeel Natu 	uint32_t gi, info;
1460366f6083SPeter Grehan 
1461fefac543SBojan Novković 	if (vcpu->cap.set & (1 << VM_CAP_MASK_HWINTR)) {
1462fefac543SBojan Novković 		return;
1463fefac543SBojan Novković 	}
1464fefac543SBojan Novković 
14651aa51504SJohn Baldwin 	if (vcpu->state.nextrip != guestrip) {
14662ce12423SNeel Natu 		gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
14672ce12423SNeel Natu 		if (gi & HWINTR_BLOCKING) {
146857e0119eSJohn Baldwin 			VMX_CTR2(vcpu, "Guest interrupt blocking "
14692ce12423SNeel Natu 			    "cleared due to rip change: %#lx/%#lx",
14701aa51504SJohn Baldwin 			    vcpu->state.nextrip, guestrip);
14712ce12423SNeel Natu 			gi &= ~HWINTR_BLOCKING;
14722ce12423SNeel Natu 			vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
14732ce12423SNeel Natu 		}
14742ce12423SNeel Natu 	}
14752ce12423SNeel Natu 
147680cb5d84SJohn Baldwin 	if (vm_entry_intinfo(vcpu->vcpu, &entryinfo)) {
1477091d4532SNeel Natu 		KASSERT((entryinfo & VMCS_INTR_VALID) != 0, ("%s: entry "
1478091d4532SNeel Natu 		    "intinfo is not valid: %#lx", __func__, entryinfo));
1479dc506506SNeel Natu 
1480dc506506SNeel Natu 		info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1481dc506506SNeel Natu 		KASSERT((info & VMCS_INTR_VALID) == 0, ("%s: cannot inject "
1482019008ebSNeel Natu 		     "pending exception: %#lx/%#x", __func__, entryinfo, info));
1483dc506506SNeel Natu 
1484091d4532SNeel Natu 		info = entryinfo;
1485091d4532SNeel Natu 		vector = info & 0xff;
1486091d4532SNeel Natu 		if (vector == IDT_BP || vector == IDT_OF) {
1487091d4532SNeel Natu 			/*
1488091d4532SNeel Natu 			 * VT-x requires #BP and #OF to be injected as software
1489091d4532SNeel Natu 			 * exceptions.
1490091d4532SNeel Natu 			 */
1491091d4532SNeel Natu 			info &= ~VMCS_INTR_T_MASK;
1492091d4532SNeel Natu 			info |= VMCS_INTR_T_SWEXCEPTION;
1493dc506506SNeel Natu 		}
1494091d4532SNeel Natu 
1495091d4532SNeel Natu 		if (info & VMCS_INTR_DEL_ERRCODE)
1496091d4532SNeel Natu 			vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR, entryinfo >> 32);
1497091d4532SNeel Natu 
1498dc506506SNeel Natu 		vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1499dc506506SNeel Natu 	}
1500dc506506SNeel Natu 
150180cb5d84SJohn Baldwin 	if (vm_nmi_pending(vcpu->vcpu)) {
1502366f6083SPeter Grehan 		/*
150348b2d828SNeel Natu 		 * If there are no conditions blocking NMI injection then
150448b2d828SNeel Natu 		 * inject it directly here otherwise enable "NMI window
150548b2d828SNeel Natu 		 * exiting" to inject it as soon as we can.
1506eeefa4e4SNeel Natu 		 *
150748b2d828SNeel Natu 		 * We also check for STI_BLOCKING because some implementations
150848b2d828SNeel Natu 		 * don't allow NMI injection in this case. If we are running
150948b2d828SNeel Natu 		 * on a processor that doesn't have this restriction it will
151048b2d828SNeel Natu 		 * immediately exit and the NMI will be injected in the
151148b2d828SNeel Natu 		 * "NMI window exiting" handler.
1512366f6083SPeter Grehan 		 */
151348b2d828SNeel Natu 		need_nmi_exiting = 1;
151448b2d828SNeel Natu 		gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
151548b2d828SNeel Natu 		if ((gi & (HWINTR_BLOCKING | NMI_BLOCKING)) == 0) {
15163de83862SNeel Natu 			info = vmcs_read(VMCS_ENTRY_INTR_INFO);
151748b2d828SNeel Natu 			if ((info & VMCS_INTR_VALID) == 0) {
151880cb5d84SJohn Baldwin 				vmx_inject_nmi(vcpu);
151948b2d828SNeel Natu 				need_nmi_exiting = 0;
152048b2d828SNeel Natu 			} else {
152157e0119eSJohn Baldwin 				VMX_CTR1(vcpu, "Cannot inject NMI "
152257e0119eSJohn Baldwin 				    "due to VM-entry intr info %#x", info);
152348b2d828SNeel Natu 			}
152448b2d828SNeel Natu 		} else {
152557e0119eSJohn Baldwin 			VMX_CTR1(vcpu, "Cannot inject NMI due to "
152657e0119eSJohn Baldwin 			    "Guest Interruptibility-state %#x", gi);
152748b2d828SNeel Natu 		}
1528eeefa4e4SNeel Natu 
152948b2d828SNeel Natu 		if (need_nmi_exiting)
1530869c8d19SJohn Baldwin 			vmx_set_nmi_window_exiting(vcpu);
153148b2d828SNeel Natu 	}
1532366f6083SPeter Grehan 
153380cb5d84SJohn Baldwin 	extint_pending = vm_extint_pending(vcpu->vcpu);
15340775fbb4STycho Nightingale 
15350775fbb4STycho Nightingale 	if (!extint_pending && virtual_interrupt_delivery) {
153688c4b8d1SNeel Natu 		vmx_inject_pir(vlapic);
153788c4b8d1SNeel Natu 		return;
153888c4b8d1SNeel Natu 	}
153988c4b8d1SNeel Natu 
154048b2d828SNeel Natu 	/*
154136736912SNeel Natu 	 * If interrupt-window exiting is already in effect then don't bother
154236736912SNeel Natu 	 * checking for pending interrupts. This is just an optimization and
154336736912SNeel Natu 	 * not needed for correctness.
154448b2d828SNeel Natu 	 */
15451aa51504SJohn Baldwin 	if ((vcpu->cap.proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) {
154657e0119eSJohn Baldwin 		VMX_CTR0(vcpu, "Skip interrupt injection due to "
154757e0119eSJohn Baldwin 		    "pending int_window_exiting");
154848b2d828SNeel Natu 		return;
154936736912SNeel Natu 	}
155048b2d828SNeel Natu 
15510775fbb4STycho Nightingale 	if (!extint_pending) {
1552366f6083SPeter Grehan 		/* Ask the local apic for a vector to inject */
15534d1e82a8SNeel Natu 		if (!vlapic_pending_intr(vlapic, &vector))
1554366f6083SPeter Grehan 			return;
1555a026dc3fSTycho Nightingale 
1556a026dc3fSTycho Nightingale 		/*
1557a026dc3fSTycho Nightingale 		 * From the Intel SDM, Volume 3, Section "Maskable
1558a026dc3fSTycho Nightingale 		 * Hardware Interrupts":
1559a026dc3fSTycho Nightingale 		 * - maskable interrupt vectors [16,255] can be delivered
1560a026dc3fSTycho Nightingale 		 *   through the local APIC.
1561a026dc3fSTycho Nightingale 		*/
1562a026dc3fSTycho Nightingale 		KASSERT(vector >= 16 && vector <= 255,
1563a026dc3fSTycho Nightingale 		    ("invalid vector %d from local APIC", vector));
15640775fbb4STycho Nightingale 	} else {
15650775fbb4STycho Nightingale 		/* Ask the legacy pic for a vector to inject */
156680cb5d84SJohn Baldwin 		vatpic_pending_intr(vcpu->vmx->vm, &vector);
1567366f6083SPeter Grehan 
1568a026dc3fSTycho Nightingale 		/*
1569a026dc3fSTycho Nightingale 		 * From the Intel SDM, Volume 3, Section "Maskable
1570a026dc3fSTycho Nightingale 		 * Hardware Interrupts":
1571a026dc3fSTycho Nightingale 		 * - maskable interrupt vectors [0,255] can be delivered
1572a026dc3fSTycho Nightingale 		 *   through the INTR pin.
1573a026dc3fSTycho Nightingale 		 */
1574a026dc3fSTycho Nightingale 		KASSERT(vector >= 0 && vector <= 255,
1575a026dc3fSTycho Nightingale 		    ("invalid vector %d from INTR", vector));
1576a026dc3fSTycho Nightingale 	}
1577366f6083SPeter Grehan 
1578366f6083SPeter Grehan 	/* Check RFLAGS.IF and the interruptibility state of the guest */
15793de83862SNeel Natu 	rflags = vmcs_read(VMCS_GUEST_RFLAGS);
158036736912SNeel Natu 	if ((rflags & PSL_I) == 0) {
158157e0119eSJohn Baldwin 		VMX_CTR2(vcpu, "Cannot inject vector %d due to "
158257e0119eSJohn Baldwin 		    "rflags %#lx", vector, rflags);
1583366f6083SPeter Grehan 		goto cantinject;
158436736912SNeel Natu 	}
1585366f6083SPeter Grehan 
158648b2d828SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
158736736912SNeel Natu 	if (gi & HWINTR_BLOCKING) {
158857e0119eSJohn Baldwin 		VMX_CTR2(vcpu, "Cannot inject vector %d due to "
158957e0119eSJohn Baldwin 		    "Guest Interruptibility-state %#x", vector, gi);
1590366f6083SPeter Grehan 		goto cantinject;
159136736912SNeel Natu 	}
159236736912SNeel Natu 
159336736912SNeel Natu 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
159436736912SNeel Natu 	if (info & VMCS_INTR_VALID) {
159536736912SNeel Natu 		/*
159636736912SNeel Natu 		 * This is expected and could happen for multiple reasons:
159736736912SNeel Natu 		 * - A vectoring VM-entry was aborted due to astpending
159836736912SNeel Natu 		 * - A VM-exit happened during event injection.
1599dc506506SNeel Natu 		 * - An exception was injected above.
160036736912SNeel Natu 		 * - An NMI was injected above or after "NMI window exiting"
160136736912SNeel Natu 		 */
160257e0119eSJohn Baldwin 		VMX_CTR2(vcpu, "Cannot inject vector %d due to "
160357e0119eSJohn Baldwin 		    "VM-entry intr info %#x", vector, info);
160436736912SNeel Natu 		goto cantinject;
160536736912SNeel Natu 	}
1606366f6083SPeter Grehan 
1607366f6083SPeter Grehan 	/* Inject the interrupt */
1608160471d2SNeel Natu 	info = VMCS_INTR_T_HWINTR | VMCS_INTR_VALID;
1609366f6083SPeter Grehan 	info |= vector;
16103de83862SNeel Natu 	vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1611366f6083SPeter Grehan 
16120775fbb4STycho Nightingale 	if (!extint_pending) {
1613366f6083SPeter Grehan 		/* Update the Local APIC ISR */
1614de5ea6b6SNeel Natu 		vlapic_intr_accepted(vlapic, vector);
16150775fbb4STycho Nightingale 	} else {
161680cb5d84SJohn Baldwin 		vm_extint_clear(vcpu->vcpu);
161780cb5d84SJohn Baldwin 		vatpic_intr_accepted(vcpu->vmx->vm, vector);
16180775fbb4STycho Nightingale 
16190775fbb4STycho Nightingale 		/*
16200775fbb4STycho Nightingale 		 * After we accepted the current ExtINT the PIC may
16210775fbb4STycho Nightingale 		 * have posted another one.  If that is the case, set
16220775fbb4STycho Nightingale 		 * the Interrupt Window Exiting execution control so
16230775fbb4STycho Nightingale 		 * we can inject that one too.
16240494cb1bSNeel Natu 		 *
16250494cb1bSNeel Natu 		 * Also, interrupt window exiting allows us to inject any
16260494cb1bSNeel Natu 		 * pending APIC vector that was preempted by the ExtINT
16270494cb1bSNeel Natu 		 * as soon as possible. This applies both for the software
16280494cb1bSNeel Natu 		 * emulated vlapic and the hardware assisted virtual APIC.
16290775fbb4STycho Nightingale 		 */
1630869c8d19SJohn Baldwin 		vmx_set_int_window_exiting(vcpu);
16310775fbb4STycho Nightingale 	}
1632366f6083SPeter Grehan 
163357e0119eSJohn Baldwin 	VMX_CTR1(vcpu, "Injecting hwintr at vector %d", vector);
1634366f6083SPeter Grehan 
1635366f6083SPeter Grehan 	return;
1636366f6083SPeter Grehan 
1637366f6083SPeter Grehan cantinject:
1638366f6083SPeter Grehan 	/*
1639366f6083SPeter Grehan 	 * Set the Interrupt Window Exiting execution control so we can inject
1640366f6083SPeter Grehan 	 * the interrupt as soon as blocking condition goes away.
1641366f6083SPeter Grehan 	 */
1642869c8d19SJohn Baldwin 	vmx_set_int_window_exiting(vcpu);
1643366f6083SPeter Grehan }
1644366f6083SPeter Grehan 
1645e5a1d950SNeel Natu /*
1646e5a1d950SNeel Natu  * If the Virtual NMIs execution control is '1' then the logical processor
1647e5a1d950SNeel Natu  * tracks virtual-NMI blocking in the Guest Interruptibility-state field of
1648e5a1d950SNeel Natu  * the VMCS. An IRET instruction in VMX non-root operation will remove any
1649e5a1d950SNeel Natu  * virtual-NMI blocking.
1650e5a1d950SNeel Natu  *
1651e5a1d950SNeel Natu  * This unblocking occurs even if the IRET causes a fault. In this case the
1652e5a1d950SNeel Natu  * hypervisor needs to restore virtual-NMI blocking before resuming the guest.
1653e5a1d950SNeel Natu  */
1654e5a1d950SNeel Natu static void
1655869c8d19SJohn Baldwin vmx_restore_nmi_blocking(struct vmx_vcpu *vcpu)
1656e5a1d950SNeel Natu {
1657e5a1d950SNeel Natu 	uint32_t gi;
1658e5a1d950SNeel Natu 
165957e0119eSJohn Baldwin 	VMX_CTR0(vcpu, "Restore Virtual-NMI blocking");
1660e5a1d950SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1661e5a1d950SNeel Natu 	gi |= VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1662e5a1d950SNeel Natu 	vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1663e5a1d950SNeel Natu }
1664e5a1d950SNeel Natu 
1665e5a1d950SNeel Natu static void
1666869c8d19SJohn Baldwin vmx_clear_nmi_blocking(struct vmx_vcpu *vcpu)
1667e5a1d950SNeel Natu {
1668e5a1d950SNeel Natu 	uint32_t gi;
1669e5a1d950SNeel Natu 
167057e0119eSJohn Baldwin 	VMX_CTR0(vcpu, "Clear Virtual-NMI blocking");
1671e5a1d950SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1672e5a1d950SNeel Natu 	gi &= ~VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1673e5a1d950SNeel Natu 	vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1674e5a1d950SNeel Natu }
1675e5a1d950SNeel Natu 
1676091d4532SNeel Natu static void
1677869c8d19SJohn Baldwin vmx_assert_nmi_blocking(struct vmx_vcpu *vcpu)
1678091d4532SNeel Natu {
16795c272efaSRobert Wing 	uint32_t gi __diagused;
1680091d4532SNeel Natu 
1681091d4532SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1682091d4532SNeel Natu 	KASSERT(gi & VMCS_INTERRUPTIBILITY_NMI_BLOCKING,
1683091d4532SNeel Natu 	    ("NMI blocking is not in effect %#x", gi));
1684091d4532SNeel Natu }
1685091d4532SNeel Natu 
1686366f6083SPeter Grehan static int
16871aa51504SJohn Baldwin vmx_emulate_xsetbv(struct vmx *vmx, struct vmx_vcpu *vcpu,
16881aa51504SJohn Baldwin     struct vm_exit *vmexit)
1689abb023fbSJohn Baldwin {
1690abb023fbSJohn Baldwin 	struct vmxctx *vmxctx;
1691abb023fbSJohn Baldwin 	uint64_t xcrval;
1692abb023fbSJohn Baldwin 	const struct xsave_limits *limits;
1693abb023fbSJohn Baldwin 
16941aa51504SJohn Baldwin 	vmxctx = &vcpu->ctx;
1695abb023fbSJohn Baldwin 	limits = vmm_get_xsave_limits();
1696abb023fbSJohn Baldwin 
1697a0efd3fbSJohn Baldwin 	/*
1698a0efd3fbSJohn Baldwin 	 * Note that the processor raises a GP# fault on its own if
1699a0efd3fbSJohn Baldwin 	 * xsetbv is executed for CPL != 0, so we do not have to
1700a0efd3fbSJohn Baldwin 	 * emulate that fault here.
1701a0efd3fbSJohn Baldwin 	 */
1702a0efd3fbSJohn Baldwin 
1703a0efd3fbSJohn Baldwin 	/* Only xcr0 is supported. */
1704a0efd3fbSJohn Baldwin 	if (vmxctx->guest_rcx != 0) {
1705d3956e46SJohn Baldwin 		vm_inject_gp(vcpu->vcpu);
1706a0efd3fbSJohn Baldwin 		return (HANDLED);
1707a0efd3fbSJohn Baldwin 	}
1708a0efd3fbSJohn Baldwin 
1709a0efd3fbSJohn Baldwin 	/* We only handle xcr0 if both the host and guest have XSAVE enabled. */
1710a0efd3fbSJohn Baldwin 	if (!limits->xsave_enabled || !(vmcs_read(VMCS_GUEST_CR4) & CR4_XSAVE)) {
1711d3956e46SJohn Baldwin 		vm_inject_ud(vcpu->vcpu);
1712a0efd3fbSJohn Baldwin 		return (HANDLED);
1713a0efd3fbSJohn Baldwin 	}
1714abb023fbSJohn Baldwin 
1715abb023fbSJohn Baldwin 	xcrval = vmxctx->guest_rdx << 32 | (vmxctx->guest_rax & 0xffffffff);
1716a0efd3fbSJohn Baldwin 	if ((xcrval & ~limits->xcr0_allowed) != 0) {
1717d3956e46SJohn Baldwin 		vm_inject_gp(vcpu->vcpu);
1718a0efd3fbSJohn Baldwin 		return (HANDLED);
1719a0efd3fbSJohn Baldwin 	}
1720abb023fbSJohn Baldwin 
1721a0efd3fbSJohn Baldwin 	if (!(xcrval & XFEATURE_ENABLED_X87)) {
1722d3956e46SJohn Baldwin 		vm_inject_gp(vcpu->vcpu);
1723a0efd3fbSJohn Baldwin 		return (HANDLED);
1724a0efd3fbSJohn Baldwin 	}
1725abb023fbSJohn Baldwin 
172644a68c4eSJohn Baldwin 	/* AVX (YMM_Hi128) requires SSE. */
172744a68c4eSJohn Baldwin 	if (xcrval & XFEATURE_ENABLED_AVX &&
172844a68c4eSJohn Baldwin 	    (xcrval & XFEATURE_AVX) != XFEATURE_AVX) {
1729d3956e46SJohn Baldwin 		vm_inject_gp(vcpu->vcpu);
173044a68c4eSJohn Baldwin 		return (HANDLED);
173144a68c4eSJohn Baldwin 	}
173244a68c4eSJohn Baldwin 
173344a68c4eSJohn Baldwin 	/*
173444a68c4eSJohn Baldwin 	 * AVX512 requires base AVX (YMM_Hi128) as well as OpMask,
173544a68c4eSJohn Baldwin 	 * ZMM_Hi256, and Hi16_ZMM.
173644a68c4eSJohn Baldwin 	 */
173744a68c4eSJohn Baldwin 	if (xcrval & XFEATURE_AVX512 &&
173844a68c4eSJohn Baldwin 	    (xcrval & (XFEATURE_AVX512 | XFEATURE_AVX)) !=
173944a68c4eSJohn Baldwin 	    (XFEATURE_AVX512 | XFEATURE_AVX)) {
1740d3956e46SJohn Baldwin 		vm_inject_gp(vcpu->vcpu);
174144a68c4eSJohn Baldwin 		return (HANDLED);
174244a68c4eSJohn Baldwin 	}
174344a68c4eSJohn Baldwin 
174444a68c4eSJohn Baldwin 	/*
174544a68c4eSJohn Baldwin 	 * Intel MPX requires both bound register state flags to be
174644a68c4eSJohn Baldwin 	 * set.
174744a68c4eSJohn Baldwin 	 */
174844a68c4eSJohn Baldwin 	if (((xcrval & XFEATURE_ENABLED_BNDREGS) != 0) !=
174944a68c4eSJohn Baldwin 	    ((xcrval & XFEATURE_ENABLED_BNDCSR) != 0)) {
1750d3956e46SJohn Baldwin 		vm_inject_gp(vcpu->vcpu);
1751a0efd3fbSJohn Baldwin 		return (HANDLED);
1752a0efd3fbSJohn Baldwin 	}
1753abb023fbSJohn Baldwin 
1754abb023fbSJohn Baldwin 	/*
1755abb023fbSJohn Baldwin 	 * This runs "inside" vmrun() with the guest's FPU state, so
1756abb023fbSJohn Baldwin 	 * modifying xcr0 directly modifies the guest's xcr0, not the
1757abb023fbSJohn Baldwin 	 * host's.
1758abb023fbSJohn Baldwin 	 */
1759abb023fbSJohn Baldwin 	load_xcr(0, xcrval);
1760abb023fbSJohn Baldwin 	return (HANDLED);
1761abb023fbSJohn Baldwin }
1762abb023fbSJohn Baldwin 
1763594db002STycho Nightingale static uint64_t
17641aa51504SJohn Baldwin vmx_get_guest_reg(struct vmx_vcpu *vcpu, int ident)
1765366f6083SPeter Grehan {
1766366f6083SPeter Grehan 	const struct vmxctx *vmxctx;
1767366f6083SPeter Grehan 
17681aa51504SJohn Baldwin 	vmxctx = &vcpu->ctx;
1769594db002STycho Nightingale 
1770594db002STycho Nightingale 	switch (ident) {
1771594db002STycho Nightingale 	case 0:
1772594db002STycho Nightingale 		return (vmxctx->guest_rax);
1773594db002STycho Nightingale 	case 1:
1774594db002STycho Nightingale 		return (vmxctx->guest_rcx);
1775594db002STycho Nightingale 	case 2:
1776594db002STycho Nightingale 		return (vmxctx->guest_rdx);
1777594db002STycho Nightingale 	case 3:
1778594db002STycho Nightingale 		return (vmxctx->guest_rbx);
1779594db002STycho Nightingale 	case 4:
1780594db002STycho Nightingale 		return (vmcs_read(VMCS_GUEST_RSP));
1781594db002STycho Nightingale 	case 5:
1782594db002STycho Nightingale 		return (vmxctx->guest_rbp);
1783594db002STycho Nightingale 	case 6:
1784594db002STycho Nightingale 		return (vmxctx->guest_rsi);
1785594db002STycho Nightingale 	case 7:
1786594db002STycho Nightingale 		return (vmxctx->guest_rdi);
1787594db002STycho Nightingale 	case 8:
1788594db002STycho Nightingale 		return (vmxctx->guest_r8);
1789594db002STycho Nightingale 	case 9:
1790594db002STycho Nightingale 		return (vmxctx->guest_r9);
1791594db002STycho Nightingale 	case 10:
1792594db002STycho Nightingale 		return (vmxctx->guest_r10);
1793594db002STycho Nightingale 	case 11:
1794594db002STycho Nightingale 		return (vmxctx->guest_r11);
1795594db002STycho Nightingale 	case 12:
1796594db002STycho Nightingale 		return (vmxctx->guest_r12);
1797594db002STycho Nightingale 	case 13:
1798594db002STycho Nightingale 		return (vmxctx->guest_r13);
1799594db002STycho Nightingale 	case 14:
1800594db002STycho Nightingale 		return (vmxctx->guest_r14);
1801594db002STycho Nightingale 	case 15:
1802594db002STycho Nightingale 		return (vmxctx->guest_r15);
1803594db002STycho Nightingale 	default:
1804594db002STycho Nightingale 		panic("invalid vmx register %d", ident);
1805594db002STycho Nightingale 	}
1806594db002STycho Nightingale }
1807594db002STycho Nightingale 
1808594db002STycho Nightingale static void
18091aa51504SJohn Baldwin vmx_set_guest_reg(struct vmx_vcpu *vcpu, int ident, uint64_t regval)
1810594db002STycho Nightingale {
1811594db002STycho Nightingale 	struct vmxctx *vmxctx;
1812594db002STycho Nightingale 
18131aa51504SJohn Baldwin 	vmxctx = &vcpu->ctx;
1814594db002STycho Nightingale 
1815594db002STycho Nightingale 	switch (ident) {
1816594db002STycho Nightingale 	case 0:
1817594db002STycho Nightingale 		vmxctx->guest_rax = regval;
1818594db002STycho Nightingale 		break;
1819594db002STycho Nightingale 	case 1:
1820594db002STycho Nightingale 		vmxctx->guest_rcx = regval;
1821594db002STycho Nightingale 		break;
1822594db002STycho Nightingale 	case 2:
1823594db002STycho Nightingale 		vmxctx->guest_rdx = regval;
1824594db002STycho Nightingale 		break;
1825594db002STycho Nightingale 	case 3:
1826594db002STycho Nightingale 		vmxctx->guest_rbx = regval;
1827594db002STycho Nightingale 		break;
1828594db002STycho Nightingale 	case 4:
1829594db002STycho Nightingale 		vmcs_write(VMCS_GUEST_RSP, regval);
1830594db002STycho Nightingale 		break;
1831594db002STycho Nightingale 	case 5:
1832594db002STycho Nightingale 		vmxctx->guest_rbp = regval;
1833594db002STycho Nightingale 		break;
1834594db002STycho Nightingale 	case 6:
1835594db002STycho Nightingale 		vmxctx->guest_rsi = regval;
1836594db002STycho Nightingale 		break;
1837594db002STycho Nightingale 	case 7:
1838594db002STycho Nightingale 		vmxctx->guest_rdi = regval;
1839594db002STycho Nightingale 		break;
1840594db002STycho Nightingale 	case 8:
1841594db002STycho Nightingale 		vmxctx->guest_r8 = regval;
1842594db002STycho Nightingale 		break;
1843594db002STycho Nightingale 	case 9:
1844594db002STycho Nightingale 		vmxctx->guest_r9 = regval;
1845594db002STycho Nightingale 		break;
1846594db002STycho Nightingale 	case 10:
1847594db002STycho Nightingale 		vmxctx->guest_r10 = regval;
1848594db002STycho Nightingale 		break;
1849594db002STycho Nightingale 	case 11:
1850594db002STycho Nightingale 		vmxctx->guest_r11 = regval;
1851594db002STycho Nightingale 		break;
1852594db002STycho Nightingale 	case 12:
1853594db002STycho Nightingale 		vmxctx->guest_r12 = regval;
1854594db002STycho Nightingale 		break;
1855594db002STycho Nightingale 	case 13:
1856594db002STycho Nightingale 		vmxctx->guest_r13 = regval;
1857594db002STycho Nightingale 		break;
1858594db002STycho Nightingale 	case 14:
1859594db002STycho Nightingale 		vmxctx->guest_r14 = regval;
1860594db002STycho Nightingale 		break;
1861594db002STycho Nightingale 	case 15:
1862594db002STycho Nightingale 		vmxctx->guest_r15 = regval;
1863594db002STycho Nightingale 		break;
1864594db002STycho Nightingale 	default:
1865594db002STycho Nightingale 		panic("invalid vmx register %d", ident);
1866594db002STycho Nightingale 	}
1867594db002STycho Nightingale }
1868594db002STycho Nightingale 
1869594db002STycho Nightingale static int
18701aa51504SJohn Baldwin vmx_emulate_cr0_access(struct vmx_vcpu *vcpu, uint64_t exitqual)
1871594db002STycho Nightingale {
1872594db002STycho Nightingale 	uint64_t crval, regval;
1873594db002STycho Nightingale 
1874594db002STycho Nightingale 	/* We only handle mov to %cr0 at this time */
187539c21c2dSNeel Natu 	if ((exitqual & 0xf0) != 0x00)
187639c21c2dSNeel Natu 		return (UNHANDLED);
187739c21c2dSNeel Natu 
18781aa51504SJohn Baldwin 	regval = vmx_get_guest_reg(vcpu, (exitqual >> 8) & 0xf);
1879366f6083SPeter Grehan 
1880594db002STycho Nightingale 	vmcs_write(VMCS_CR0_SHADOW, regval);
1881366f6083SPeter Grehan 
1882594db002STycho Nightingale 	crval = regval | cr0_ones_mask;
1883594db002STycho Nightingale 	crval &= ~cr0_zeros_mask;
1884594db002STycho Nightingale 	vmcs_write(VMCS_GUEST_CR0, crval);
1885366f6083SPeter Grehan 
1886594db002STycho Nightingale 	if (regval & CR0_PG) {
188780a902efSPeter Grehan 		uint64_t efer, entry_ctls;
188880a902efSPeter Grehan 
188980a902efSPeter Grehan 		/*
189080a902efSPeter Grehan 		 * If CR0.PG is 1 and EFER.LME is 1 then EFER.LMA and
189180a902efSPeter Grehan 		 * the "IA-32e mode guest" bit in VM-entry control must be
189280a902efSPeter Grehan 		 * equal.
189380a902efSPeter Grehan 		 */
18943de83862SNeel Natu 		efer = vmcs_read(VMCS_GUEST_IA32_EFER);
189580a902efSPeter Grehan 		if (efer & EFER_LME) {
189680a902efSPeter Grehan 			efer |= EFER_LMA;
18973de83862SNeel Natu 			vmcs_write(VMCS_GUEST_IA32_EFER, efer);
18983de83862SNeel Natu 			entry_ctls = vmcs_read(VMCS_ENTRY_CTLS);
189980a902efSPeter Grehan 			entry_ctls |= VM_ENTRY_GUEST_LMA;
19003de83862SNeel Natu 			vmcs_write(VMCS_ENTRY_CTLS, entry_ctls);
190180a902efSPeter Grehan 		}
190280a902efSPeter Grehan 	}
190380a902efSPeter Grehan 
1904366f6083SPeter Grehan 	return (HANDLED);
1905366f6083SPeter Grehan }
1906366f6083SPeter Grehan 
1907594db002STycho Nightingale static int
19081aa51504SJohn Baldwin vmx_emulate_cr4_access(struct vmx_vcpu *vcpu, uint64_t exitqual)
1909594db002STycho Nightingale {
1910594db002STycho Nightingale 	uint64_t crval, regval;
1911594db002STycho Nightingale 
1912594db002STycho Nightingale 	/* We only handle mov to %cr4 at this time */
1913594db002STycho Nightingale 	if ((exitqual & 0xf0) != 0x00)
1914594db002STycho Nightingale 		return (UNHANDLED);
1915594db002STycho Nightingale 
19161aa51504SJohn Baldwin 	regval = vmx_get_guest_reg(vcpu, (exitqual >> 8) & 0xf);
1917594db002STycho Nightingale 
1918594db002STycho Nightingale 	vmcs_write(VMCS_CR4_SHADOW, regval);
1919594db002STycho Nightingale 
1920594db002STycho Nightingale 	crval = regval | cr4_ones_mask;
1921594db002STycho Nightingale 	crval &= ~cr4_zeros_mask;
1922594db002STycho Nightingale 	vmcs_write(VMCS_GUEST_CR4, crval);
1923594db002STycho Nightingale 
1924594db002STycho Nightingale 	return (HANDLED);
1925594db002STycho Nightingale }
1926594db002STycho Nightingale 
1927594db002STycho Nightingale static int
19281aa51504SJohn Baldwin vmx_emulate_cr8_access(struct vmx *vmx, struct vmx_vcpu *vcpu,
19291aa51504SJohn Baldwin     uint64_t exitqual)
1930594db002STycho Nightingale {
1931051f2bd1SNeel Natu 	struct vlapic *vlapic;
1932051f2bd1SNeel Natu 	uint64_t cr8;
1933051f2bd1SNeel Natu 	int regnum;
1934594db002STycho Nightingale 
1935594db002STycho Nightingale 	/* We only handle mov %cr8 to/from a register at this time. */
1936594db002STycho Nightingale 	if ((exitqual & 0xe0) != 0x00) {
1937594db002STycho Nightingale 		return (UNHANDLED);
1938594db002STycho Nightingale 	}
1939594db002STycho Nightingale 
1940d3956e46SJohn Baldwin 	vlapic = vm_lapic(vcpu->vcpu);
1941051f2bd1SNeel Natu 	regnum = (exitqual >> 8) & 0xf;
1942594db002STycho Nightingale 	if (exitqual & 0x10) {
1943051f2bd1SNeel Natu 		cr8 = vlapic_get_cr8(vlapic);
19441aa51504SJohn Baldwin 		vmx_set_guest_reg(vcpu, regnum, cr8);
1945594db002STycho Nightingale 	} else {
19461aa51504SJohn Baldwin 		cr8 = vmx_get_guest_reg(vcpu, regnum);
1947051f2bd1SNeel Natu 		vlapic_set_cr8(vlapic, cr8);
1948594db002STycho Nightingale 	}
1949594db002STycho Nightingale 
1950594db002STycho Nightingale 	return (HANDLED);
1951594db002STycho Nightingale }
1952594db002STycho Nightingale 
1953e4c8a13dSNeel Natu /*
1954e4c8a13dSNeel Natu  * From section "Guest Register State" in the Intel SDM: CPL = SS.DPL
1955e4c8a13dSNeel Natu  */
1956e4c8a13dSNeel Natu static int
1957e4c8a13dSNeel Natu vmx_cpl(void)
1958e4c8a13dSNeel Natu {
1959e4c8a13dSNeel Natu 	uint32_t ssar;
1960e4c8a13dSNeel Natu 
1961e4c8a13dSNeel Natu 	ssar = vmcs_read(VMCS_GUEST_SS_ACCESS_RIGHTS);
1962e4c8a13dSNeel Natu 	return ((ssar >> 5) & 0x3);
1963e4c8a13dSNeel Natu }
1964e4c8a13dSNeel Natu 
1965e813a873SNeel Natu static enum vm_cpu_mode
196600f3efe1SJohn Baldwin vmx_cpu_mode(void)
196700f3efe1SJohn Baldwin {
1968b301b9e2SNeel Natu 	uint32_t csar;
196900f3efe1SJohn Baldwin 
1970b301b9e2SNeel Natu 	if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LMA) {
1971b301b9e2SNeel Natu 		csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS);
1972b301b9e2SNeel Natu 		if (csar & 0x2000)
1973b301b9e2SNeel Natu 			return (CPU_MODE_64BIT);	/* CS.L = 1 */
197400f3efe1SJohn Baldwin 		else
197500f3efe1SJohn Baldwin 			return (CPU_MODE_COMPATIBILITY);
1976b301b9e2SNeel Natu 	} else if (vmcs_read(VMCS_GUEST_CR0) & CR0_PE) {
1977b301b9e2SNeel Natu 		return (CPU_MODE_PROTECTED);
1978b301b9e2SNeel Natu 	} else {
1979b301b9e2SNeel Natu 		return (CPU_MODE_REAL);
1980b301b9e2SNeel Natu 	}
198100f3efe1SJohn Baldwin }
198200f3efe1SJohn Baldwin 
1983e813a873SNeel Natu static enum vm_paging_mode
198400f3efe1SJohn Baldwin vmx_paging_mode(void)
198500f3efe1SJohn Baldwin {
1986f3eb12e4SKonstantin Belousov 	uint64_t cr4;
198700f3efe1SJohn Baldwin 
198800f3efe1SJohn Baldwin 	if (!(vmcs_read(VMCS_GUEST_CR0) & CR0_PG))
198900f3efe1SJohn Baldwin 		return (PAGING_MODE_FLAT);
1990f3eb12e4SKonstantin Belousov 	cr4 = vmcs_read(VMCS_GUEST_CR4);
1991f3eb12e4SKonstantin Belousov 	if (!(cr4 & CR4_PAE))
199200f3efe1SJohn Baldwin 		return (PAGING_MODE_32);
1993f3eb12e4SKonstantin Belousov 	if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LME) {
1994f3eb12e4SKonstantin Belousov 		if (!(cr4 & CR4_LA57))
199500f3efe1SJohn Baldwin 			return (PAGING_MODE_64);
1996f3eb12e4SKonstantin Belousov 		return (PAGING_MODE_64_LA57);
1997f3eb12e4SKonstantin Belousov 	} else
199800f3efe1SJohn Baldwin 		return (PAGING_MODE_PAE);
199900f3efe1SJohn Baldwin }
200000f3efe1SJohn Baldwin 
2001d17b5104SNeel Natu static uint64_t
2002869c8d19SJohn Baldwin inout_str_index(struct vmx_vcpu *vcpu, int in)
2003d17b5104SNeel Natu {
2004d17b5104SNeel Natu 	uint64_t val;
20055c272efaSRobert Wing 	int error __diagused;
2006d17b5104SNeel Natu 	enum vm_reg_name reg;
2007d17b5104SNeel Natu 
2008d17b5104SNeel Natu 	reg = in ? VM_REG_GUEST_RDI : VM_REG_GUEST_RSI;
2009869c8d19SJohn Baldwin 	error = vmx_getreg(vcpu, reg, &val);
2010d17b5104SNeel Natu 	KASSERT(error == 0, ("%s: vmx_getreg error %d", __func__, error));
2011d17b5104SNeel Natu 	return (val);
2012d17b5104SNeel Natu }
2013d17b5104SNeel Natu 
2014d17b5104SNeel Natu static uint64_t
2015869c8d19SJohn Baldwin inout_str_count(struct vmx_vcpu *vcpu, int rep)
2016d17b5104SNeel Natu {
2017d17b5104SNeel Natu 	uint64_t val;
20185c272efaSRobert Wing 	int error __diagused;
2019d17b5104SNeel Natu 
2020d17b5104SNeel Natu 	if (rep) {
2021869c8d19SJohn Baldwin 		error = vmx_getreg(vcpu, VM_REG_GUEST_RCX, &val);
2022d17b5104SNeel Natu 		KASSERT(!error, ("%s: vmx_getreg error %d", __func__, error));
2023d17b5104SNeel Natu 	} else {
2024d17b5104SNeel Natu 		val = 1;
2025d17b5104SNeel Natu 	}
2026d17b5104SNeel Natu 	return (val);
2027d17b5104SNeel Natu }
2028d17b5104SNeel Natu 
2029d17b5104SNeel Natu static int
2030d17b5104SNeel Natu inout_str_addrsize(uint32_t inst_info)
2031d17b5104SNeel Natu {
2032d17b5104SNeel Natu 	uint32_t size;
2033d17b5104SNeel Natu 
2034d17b5104SNeel Natu 	size = (inst_info >> 7) & 0x7;
2035d17b5104SNeel Natu 	switch (size) {
2036d17b5104SNeel Natu 	case 0:
2037d17b5104SNeel Natu 		return (2);	/* 16 bit */
2038d17b5104SNeel Natu 	case 1:
2039d17b5104SNeel Natu 		return (4);	/* 32 bit */
2040d17b5104SNeel Natu 	case 2:
2041d17b5104SNeel Natu 		return (8);	/* 64 bit */
2042d17b5104SNeel Natu 	default:
2043d17b5104SNeel Natu 		panic("%s: invalid size encoding %d", __func__, size);
2044d17b5104SNeel Natu 	}
2045d17b5104SNeel Natu }
2046d17b5104SNeel Natu 
2047d17b5104SNeel Natu static void
2048869c8d19SJohn Baldwin inout_str_seginfo(struct vmx_vcpu *vcpu, uint32_t inst_info, int in,
2049869c8d19SJohn Baldwin     struct vm_inout_str *vis)
2050d17b5104SNeel Natu {
20515c272efaSRobert Wing 	int error __diagused, s;
2052d17b5104SNeel Natu 
2053d17b5104SNeel Natu 	if (in) {
2054d17b5104SNeel Natu 		vis->seg_name = VM_REG_GUEST_ES;
2055d17b5104SNeel Natu 	} else {
2056d17b5104SNeel Natu 		s = (inst_info >> 15) & 0x7;
2057d17b5104SNeel Natu 		vis->seg_name = vm_segment_name(s);
2058d17b5104SNeel Natu 	}
2059d17b5104SNeel Natu 
2060869c8d19SJohn Baldwin 	error = vmx_getdesc(vcpu, vis->seg_name, &vis->seg_desc);
2061d17b5104SNeel Natu 	KASSERT(error == 0, ("%s: vmx_getdesc error %d", __func__, error));
2062d17b5104SNeel Natu }
2063d17b5104SNeel Natu 
2064e4c8a13dSNeel Natu static void
2065e813a873SNeel Natu vmx_paging_info(struct vm_guest_paging *paging)
2066e813a873SNeel Natu {
2067e813a873SNeel Natu 	paging->cr3 = vmcs_guest_cr3();
2068e813a873SNeel Natu 	paging->cpl = vmx_cpl();
2069e813a873SNeel Natu 	paging->cpu_mode = vmx_cpu_mode();
2070e813a873SNeel Natu 	paging->paging_mode = vmx_paging_mode();
2071e813a873SNeel Natu }
2072e813a873SNeel Natu 
2073e813a873SNeel Natu static void
2074e4c8a13dSNeel Natu vmexit_inst_emul(struct vm_exit *vmexit, uint64_t gpa, uint64_t gla)
2075e4c8a13dSNeel Natu {
2076f7a9f178SNeel Natu 	struct vm_guest_paging *paging;
2077f7a9f178SNeel Natu 	uint32_t csar;
2078f7a9f178SNeel Natu 
2079f7a9f178SNeel Natu 	paging = &vmexit->u.inst_emul.paging;
2080f7a9f178SNeel Natu 
2081e4c8a13dSNeel Natu 	vmexit->exitcode = VM_EXITCODE_INST_EMUL;
20821c73ea3eSNeel Natu 	vmexit->inst_length = 0;
2083e4c8a13dSNeel Natu 	vmexit->u.inst_emul.gpa = gpa;
2084e4c8a13dSNeel Natu 	vmexit->u.inst_emul.gla = gla;
2085f7a9f178SNeel Natu 	vmx_paging_info(paging);
2086f7a9f178SNeel Natu 	switch (paging->cpu_mode) {
2087e4f605eeSTycho Nightingale 	case CPU_MODE_REAL:
2088e4f605eeSTycho Nightingale 		vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE);
2089e4f605eeSTycho Nightingale 		vmexit->u.inst_emul.cs_d = 0;
2090e4f605eeSTycho Nightingale 		break;
2091f7a9f178SNeel Natu 	case CPU_MODE_PROTECTED:
2092f7a9f178SNeel Natu 	case CPU_MODE_COMPATIBILITY:
2093e4f605eeSTycho Nightingale 		vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE);
2094f7a9f178SNeel Natu 		csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS);
2095f7a9f178SNeel Natu 		vmexit->u.inst_emul.cs_d = SEG_DESC_DEF32(csar);
2096f7a9f178SNeel Natu 		break;
2097f7a9f178SNeel Natu 	default:
2098e4f605eeSTycho Nightingale 		vmexit->u.inst_emul.cs_base = 0;
2099f7a9f178SNeel Natu 		vmexit->u.inst_emul.cs_d = 0;
2100f7a9f178SNeel Natu 		break;
2101f7a9f178SNeel Natu 	}
2102c2a875f9SNeel Natu 	vie_init(&vmexit->u.inst_emul.vie, NULL, 0);
2103e4c8a13dSNeel Natu }
2104e4c8a13dSNeel Natu 
2105366f6083SPeter Grehan static int
2106318224bbSNeel Natu ept_fault_type(uint64_t ept_qual)
2107a2da7af6SNeel Natu {
2108318224bbSNeel Natu 	int fault_type;
2109a2da7af6SNeel Natu 
2110318224bbSNeel Natu 	if (ept_qual & EPT_VIOLATION_DATA_WRITE)
2111318224bbSNeel Natu 		fault_type = VM_PROT_WRITE;
2112318224bbSNeel Natu 	else if (ept_qual & EPT_VIOLATION_INST_FETCH)
2113318224bbSNeel Natu 		fault_type = VM_PROT_EXECUTE;
2114318224bbSNeel Natu 	else
2115318224bbSNeel Natu 		fault_type= VM_PROT_READ;
2116318224bbSNeel Natu 
2117318224bbSNeel Natu 	return (fault_type);
2118318224bbSNeel Natu }
2119318224bbSNeel Natu 
2120490d56c5SEd Maste static bool
2121318224bbSNeel Natu ept_emulation_fault(uint64_t ept_qual)
2122318224bbSNeel Natu {
2123318224bbSNeel Natu 	int read, write;
2124318224bbSNeel Natu 
2125318224bbSNeel Natu 	/* EPT fault on an instruction fetch doesn't make sense here */
2126a2da7af6SNeel Natu 	if (ept_qual & EPT_VIOLATION_INST_FETCH)
2127490d56c5SEd Maste 		return (false);
2128a2da7af6SNeel Natu 
2129318224bbSNeel Natu 	/* EPT fault must be a read fault or a write fault */
2130a2da7af6SNeel Natu 	read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0;
2131a2da7af6SNeel Natu 	write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0;
21323b2b0011SPeter Grehan 	if ((read | write) == 0)
2133490d56c5SEd Maste 		return (false);
2134a2da7af6SNeel Natu 
2135a2da7af6SNeel Natu 	/*
21363b2b0011SPeter Grehan 	 * The EPT violation must have been caused by accessing a
21373b2b0011SPeter Grehan 	 * guest-physical address that is a translation of a guest-linear
21383b2b0011SPeter Grehan 	 * address.
2139a2da7af6SNeel Natu 	 */
2140a2da7af6SNeel Natu 	if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 ||
2141a2da7af6SNeel Natu 	    (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) {
2142490d56c5SEd Maste 		return (false);
2143a2da7af6SNeel Natu 	}
2144a2da7af6SNeel Natu 
2145490d56c5SEd Maste 	return (true);
2146a2da7af6SNeel Natu }
2147a2da7af6SNeel Natu 
2148159dd56fSNeel Natu static __inline int
21491aa51504SJohn Baldwin apic_access_virtualization(struct vmx_vcpu *vcpu)
2150159dd56fSNeel Natu {
2151159dd56fSNeel Natu 	uint32_t proc_ctls2;
2152159dd56fSNeel Natu 
21531aa51504SJohn Baldwin 	proc_ctls2 = vcpu->cap.proc_ctls2;
2154159dd56fSNeel Natu 	return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) ? 1 : 0);
2155159dd56fSNeel Natu }
2156159dd56fSNeel Natu 
2157159dd56fSNeel Natu static __inline int
21581aa51504SJohn Baldwin x2apic_virtualization(struct vmx_vcpu *vcpu)
2159159dd56fSNeel Natu {
2160159dd56fSNeel Natu 	uint32_t proc_ctls2;
2161159dd56fSNeel Natu 
21621aa51504SJohn Baldwin 	proc_ctls2 = vcpu->cap.proc_ctls2;
2163159dd56fSNeel Natu 	return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_X2APIC_MODE) ? 1 : 0);
2164159dd56fSNeel Natu }
2165159dd56fSNeel Natu 
2166a2da7af6SNeel Natu static int
21671aa51504SJohn Baldwin vmx_handle_apic_write(struct vmx_vcpu *vcpu, struct vlapic *vlapic,
2168159dd56fSNeel Natu     uint64_t qual)
216988c4b8d1SNeel Natu {
217088c4b8d1SNeel Natu 	int error, handled, offset;
2171159dd56fSNeel Natu 	uint32_t *apic_regs, vector;
217288c4b8d1SNeel Natu 	bool retu;
217388c4b8d1SNeel Natu 
2174a0efd3fbSJohn Baldwin 	handled = HANDLED;
217588c4b8d1SNeel Natu 	offset = APIC_WRITE_OFFSET(qual);
2176159dd56fSNeel Natu 
21771aa51504SJohn Baldwin 	if (!apic_access_virtualization(vcpu)) {
2178159dd56fSNeel Natu 		/*
2179159dd56fSNeel Natu 		 * In general there should not be any APIC write VM-exits
2180159dd56fSNeel Natu 		 * unless APIC-access virtualization is enabled.
2181159dd56fSNeel Natu 		 *
2182159dd56fSNeel Natu 		 * However self-IPI virtualization can legitimately trigger
2183159dd56fSNeel Natu 		 * an APIC-write VM-exit so treat it specially.
2184159dd56fSNeel Natu 		 */
21851aa51504SJohn Baldwin 		if (x2apic_virtualization(vcpu) &&
2186159dd56fSNeel Natu 		    offset == APIC_OFFSET_SELF_IPI) {
2187159dd56fSNeel Natu 			apic_regs = (uint32_t *)(vlapic->apic_page);
2188159dd56fSNeel Natu 			vector = apic_regs[APIC_OFFSET_SELF_IPI / 4];
2189159dd56fSNeel Natu 			vlapic_self_ipi_handler(vlapic, vector);
2190159dd56fSNeel Natu 			return (HANDLED);
2191159dd56fSNeel Natu 		} else
2192159dd56fSNeel Natu 			return (UNHANDLED);
2193159dd56fSNeel Natu 	}
2194159dd56fSNeel Natu 
219588c4b8d1SNeel Natu 	switch (offset) {
219688c4b8d1SNeel Natu 	case APIC_OFFSET_ID:
219788c4b8d1SNeel Natu 		vlapic_id_write_handler(vlapic);
219888c4b8d1SNeel Natu 		break;
219988c4b8d1SNeel Natu 	case APIC_OFFSET_LDR:
220088c4b8d1SNeel Natu 		vlapic_ldr_write_handler(vlapic);
220188c4b8d1SNeel Natu 		break;
220288c4b8d1SNeel Natu 	case APIC_OFFSET_DFR:
220388c4b8d1SNeel Natu 		vlapic_dfr_write_handler(vlapic);
220488c4b8d1SNeel Natu 		break;
220588c4b8d1SNeel Natu 	case APIC_OFFSET_SVR:
220688c4b8d1SNeel Natu 		vlapic_svr_write_handler(vlapic);
220788c4b8d1SNeel Natu 		break;
220888c4b8d1SNeel Natu 	case APIC_OFFSET_ESR:
220988c4b8d1SNeel Natu 		vlapic_esr_write_handler(vlapic);
221088c4b8d1SNeel Natu 		break;
221188c4b8d1SNeel Natu 	case APIC_OFFSET_ICR_LOW:
221288c4b8d1SNeel Natu 		retu = false;
221388c4b8d1SNeel Natu 		error = vlapic_icrlo_write_handler(vlapic, &retu);
221488c4b8d1SNeel Natu 		if (error != 0 || retu)
2215a0efd3fbSJohn Baldwin 			handled = UNHANDLED;
221688c4b8d1SNeel Natu 		break;
221788c4b8d1SNeel Natu 	case APIC_OFFSET_CMCI_LVT:
221888c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
221988c4b8d1SNeel Natu 		vlapic_lvt_write_handler(vlapic, offset);
222088c4b8d1SNeel Natu 		break;
222188c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_ICR:
222288c4b8d1SNeel Natu 		vlapic_icrtmr_write_handler(vlapic);
222388c4b8d1SNeel Natu 		break;
222488c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_DCR:
222588c4b8d1SNeel Natu 		vlapic_dcr_write_handler(vlapic);
222688c4b8d1SNeel Natu 		break;
222788c4b8d1SNeel Natu 	default:
2228a0efd3fbSJohn Baldwin 		handled = UNHANDLED;
222988c4b8d1SNeel Natu 		break;
223088c4b8d1SNeel Natu 	}
223188c4b8d1SNeel Natu 	return (handled);
223288c4b8d1SNeel Natu }
223388c4b8d1SNeel Natu 
223488c4b8d1SNeel Natu static bool
22351aa51504SJohn Baldwin apic_access_fault(struct vmx_vcpu *vcpu, uint64_t gpa)
223688c4b8d1SNeel Natu {
223788c4b8d1SNeel Natu 
22381aa51504SJohn Baldwin 	if (apic_access_virtualization(vcpu) &&
223988c4b8d1SNeel Natu 	    (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE))
224088c4b8d1SNeel Natu 		return (true);
224188c4b8d1SNeel Natu 	else
224288c4b8d1SNeel Natu 		return (false);
224388c4b8d1SNeel Natu }
224488c4b8d1SNeel Natu 
224588c4b8d1SNeel Natu static int
22461aa51504SJohn Baldwin vmx_handle_apic_access(struct vmx_vcpu *vcpu, struct vm_exit *vmexit)
224788c4b8d1SNeel Natu {
224888c4b8d1SNeel Natu 	uint64_t qual;
224988c4b8d1SNeel Natu 	int access_type, offset, allowed;
225088c4b8d1SNeel Natu 
22511aa51504SJohn Baldwin 	if (!apic_access_virtualization(vcpu))
225288c4b8d1SNeel Natu 		return (UNHANDLED);
225388c4b8d1SNeel Natu 
225488c4b8d1SNeel Natu 	qual = vmexit->u.vmx.exit_qualification;
225588c4b8d1SNeel Natu 	access_type = APIC_ACCESS_TYPE(qual);
225688c4b8d1SNeel Natu 	offset = APIC_ACCESS_OFFSET(qual);
225788c4b8d1SNeel Natu 
225888c4b8d1SNeel Natu 	allowed = 0;
225988c4b8d1SNeel Natu 	if (access_type == 0) {
226088c4b8d1SNeel Natu 		/*
226188c4b8d1SNeel Natu 		 * Read data access to the following registers is expected.
226288c4b8d1SNeel Natu 		 */
226388c4b8d1SNeel Natu 		switch (offset) {
226488c4b8d1SNeel Natu 		case APIC_OFFSET_APR:
226588c4b8d1SNeel Natu 		case APIC_OFFSET_PPR:
226688c4b8d1SNeel Natu 		case APIC_OFFSET_RRR:
226788c4b8d1SNeel Natu 		case APIC_OFFSET_CMCI_LVT:
226888c4b8d1SNeel Natu 		case APIC_OFFSET_TIMER_CCR:
226988c4b8d1SNeel Natu 			allowed = 1;
227088c4b8d1SNeel Natu 			break;
227188c4b8d1SNeel Natu 		default:
227288c4b8d1SNeel Natu 			break;
227388c4b8d1SNeel Natu 		}
227488c4b8d1SNeel Natu 	} else if (access_type == 1) {
227588c4b8d1SNeel Natu 		/*
227688c4b8d1SNeel Natu 		 * Write data access to the following registers is expected.
227788c4b8d1SNeel Natu 		 */
227888c4b8d1SNeel Natu 		switch (offset) {
227988c4b8d1SNeel Natu 		case APIC_OFFSET_VER:
228088c4b8d1SNeel Natu 		case APIC_OFFSET_APR:
228188c4b8d1SNeel Natu 		case APIC_OFFSET_PPR:
228288c4b8d1SNeel Natu 		case APIC_OFFSET_RRR:
228388c4b8d1SNeel Natu 		case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
228488c4b8d1SNeel Natu 		case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
228588c4b8d1SNeel Natu 		case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
228688c4b8d1SNeel Natu 		case APIC_OFFSET_CMCI_LVT:
228788c4b8d1SNeel Natu 		case APIC_OFFSET_TIMER_CCR:
228888c4b8d1SNeel Natu 			allowed = 1;
228988c4b8d1SNeel Natu 			break;
229088c4b8d1SNeel Natu 		default:
229188c4b8d1SNeel Natu 			break;
229288c4b8d1SNeel Natu 		}
229388c4b8d1SNeel Natu 	}
229488c4b8d1SNeel Natu 
229588c4b8d1SNeel Natu 	if (allowed) {
2296e4c8a13dSNeel Natu 		vmexit_inst_emul(vmexit, DEFAULT_APIC_BASE + offset,
2297e4c8a13dSNeel Natu 		    VIE_INVALID_GLA);
229888c4b8d1SNeel Natu 	}
229988c4b8d1SNeel Natu 
230088c4b8d1SNeel Natu 	/*
230188c4b8d1SNeel Natu 	 * Regardless of whether the APIC-access is allowed this handler
230288c4b8d1SNeel Natu 	 * always returns UNHANDLED:
230388c4b8d1SNeel Natu 	 * - if the access is allowed then it is handled by emulating the
230488c4b8d1SNeel Natu 	 *   instruction that caused the VM-exit (outside the critical section)
230588c4b8d1SNeel Natu 	 * - if the access is not allowed then it will be converted to an
230688c4b8d1SNeel Natu 	 *   exitcode of VM_EXITCODE_VMX and will be dealt with in userland.
230788c4b8d1SNeel Natu 	 */
230888c4b8d1SNeel Natu 	return (UNHANDLED);
230988c4b8d1SNeel Natu }
231088c4b8d1SNeel Natu 
23113d5444c8SNeel Natu static enum task_switch_reason
23123d5444c8SNeel Natu vmx_task_switch_reason(uint64_t qual)
23133d5444c8SNeel Natu {
23143d5444c8SNeel Natu 	int reason;
23153d5444c8SNeel Natu 
23163d5444c8SNeel Natu 	reason = (qual >> 30) & 0x3;
23173d5444c8SNeel Natu 	switch (reason) {
23183d5444c8SNeel Natu 	case 0:
23193d5444c8SNeel Natu 		return (TSR_CALL);
23203d5444c8SNeel Natu 	case 1:
23213d5444c8SNeel Natu 		return (TSR_IRET);
23223d5444c8SNeel Natu 	case 2:
23233d5444c8SNeel Natu 		return (TSR_JMP);
23243d5444c8SNeel Natu 	case 3:
23253d5444c8SNeel Natu 		return (TSR_IDT_GATE);
23263d5444c8SNeel Natu 	default:
23273d5444c8SNeel Natu 		panic("%s: invalid reason %d", __func__, reason);
23283d5444c8SNeel Natu 	}
23293d5444c8SNeel Natu }
23303d5444c8SNeel Natu 
233188c4b8d1SNeel Natu static int
233280cb5d84SJohn Baldwin emulate_wrmsr(struct vmx_vcpu *vcpu, u_int num, uint64_t val, bool *retu)
2333c3498942SNeel Natu {
2334c3498942SNeel Natu 	int error;
2335c3498942SNeel Natu 
2336c3498942SNeel Natu 	if (lapic_msr(num))
233780cb5d84SJohn Baldwin 		error = lapic_wrmsr(vcpu->vcpu, num, val, retu);
2338c3498942SNeel Natu 	else
233980cb5d84SJohn Baldwin 		error = vmx_wrmsr(vcpu, num, val, retu);
2340c3498942SNeel Natu 
2341c3498942SNeel Natu 	return (error);
2342c3498942SNeel Natu }
2343c3498942SNeel Natu 
2344c3498942SNeel Natu static int
234580cb5d84SJohn Baldwin emulate_rdmsr(struct vmx_vcpu *vcpu, u_int num, bool *retu)
2346c3498942SNeel Natu {
2347c3498942SNeel Natu 	struct vmxctx *vmxctx;
2348c3498942SNeel Natu 	uint64_t result;
2349c3498942SNeel Natu 	uint32_t eax, edx;
2350c3498942SNeel Natu 	int error;
2351c3498942SNeel Natu 
2352c3498942SNeel Natu 	if (lapic_msr(num))
235380cb5d84SJohn Baldwin 		error = lapic_rdmsr(vcpu->vcpu, num, &result, retu);
2354c3498942SNeel Natu 	else
235580cb5d84SJohn Baldwin 		error = vmx_rdmsr(vcpu, num, &result, retu);
2356c3498942SNeel Natu 
2357c3498942SNeel Natu 	if (error == 0) {
2358c3498942SNeel Natu 		eax = result;
23591aa51504SJohn Baldwin 		vmxctx = &vcpu->ctx;
2360c3498942SNeel Natu 		error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RAX, eax);
2361c3498942SNeel Natu 		KASSERT(error == 0, ("vmxctx_setreg(rax) error %d", error));
2362c3498942SNeel Natu 
2363c3498942SNeel Natu 		edx = result >> 32;
2364c3498942SNeel Natu 		error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RDX, edx);
2365c3498942SNeel Natu 		KASSERT(error == 0, ("vmxctx_setreg(rdx) error %d", error));
2366c3498942SNeel Natu 	}
2367c3498942SNeel Natu 
2368c3498942SNeel Natu 	return (error);
2369c3498942SNeel Natu }
2370c3498942SNeel Natu 
2371c3498942SNeel Natu static int
23721aa51504SJohn Baldwin vmx_exit_process(struct vmx *vmx, struct vmx_vcpu *vcpu, struct vm_exit *vmexit)
2373366f6083SPeter Grehan {
2374c9c75df4SNeel Natu 	int error, errcode, errcode_valid, handled, in;
2375366f6083SPeter Grehan 	struct vmxctx *vmxctx;
237688c4b8d1SNeel Natu 	struct vlapic *vlapic;
2377d17b5104SNeel Natu 	struct vm_inout_str *vis;
23783d5444c8SNeel Natu 	struct vm_task_switch *ts;
2379d17b5104SNeel Natu 	uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, inst_info;
2380b0538143SNeel Natu 	uint32_t intr_type, intr_vec, reason;
2381091d4532SNeel Natu 	uint64_t exitintinfo, qual, gpa;
23822ee1a18dSDmitry Chagin #ifdef KDTRACE_HOOKS
23831aa51504SJohn Baldwin 	int vcpuid;
23842ee1a18dSDmitry Chagin #endif
2385becd9849SNeel Natu 	bool retu;
2386366f6083SPeter Grehan 
2387160471d2SNeel Natu 	CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0);
2388c308b23bSNeel Natu 	CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0);
2389160471d2SNeel Natu 
2390a0efd3fbSJohn Baldwin 	handled = UNHANDLED;
23911aa51504SJohn Baldwin 	vmxctx = &vcpu->ctx;
23922ee1a18dSDmitry Chagin #ifdef KDTRACE_HOOKS
23931aa51504SJohn Baldwin 	vcpuid = vcpu->vcpuid;
23942ee1a18dSDmitry Chagin #endif
23950492757cSNeel Natu 
2396366f6083SPeter Grehan 	qual = vmexit->u.vmx.exit_qualification;
2397318224bbSNeel Natu 	reason = vmexit->u.vmx.exit_reason;
2398366f6083SPeter Grehan 	vmexit->exitcode = VM_EXITCODE_BOGUS;
2399366f6083SPeter Grehan 
24003dc3d32aSJohn Baldwin 	vmm_stat_incr(vcpu->vcpu, VMEXIT_COUNT, 1);
24011aa51504SJohn Baldwin 	SDT_PROBE3(vmm, vmx, exit, entry, vmx, vcpuid, vmexit);
240261592433SNeel Natu 
2403318224bbSNeel Natu 	/*
2404b0538143SNeel Natu 	 * VM-entry failures during or after loading guest state.
2405b0538143SNeel Natu 	 *
2406b0538143SNeel Natu 	 * These VM-exits are uncommon but must be handled specially
2407b0538143SNeel Natu 	 * as most VM-exit fields are not populated as usual.
2408b0538143SNeel Natu 	 */
2409b0538143SNeel Natu 	if (__predict_false(reason == EXIT_REASON_MCE_DURING_ENTRY)) {
241057e0119eSJohn Baldwin 		VMX_CTR0(vcpu, "Handling MCE during VM-entry");
2411b0538143SNeel Natu 		__asm __volatile("int $18");
2412b0538143SNeel Natu 		return (1);
2413b0538143SNeel Natu 	}
2414b0538143SNeel Natu 
2415b0538143SNeel Natu 	/*
24163d5444c8SNeel Natu 	 * VM exits that can be triggered during event delivery need to
24173d5444c8SNeel Natu 	 * be handled specially by re-injecting the event if the IDT
24183d5444c8SNeel Natu 	 * vectoring information field's valid bit is set.
2419318224bbSNeel Natu 	 *
2420318224bbSNeel Natu 	 * See "Information for VM Exits During Event Delivery" in Intel SDM
2421318224bbSNeel Natu 	 * for details.
2422318224bbSNeel Natu 	 */
2423318224bbSNeel Natu 	idtvec_info = vmcs_idt_vectoring_info();
2424318224bbSNeel Natu 	if (idtvec_info & VMCS_IDT_VEC_VALID) {
2425318224bbSNeel Natu 		idtvec_info &= ~(1 << 12); /* clear undefined bit */
2426091d4532SNeel Natu 		exitintinfo = idtvec_info;
2427318224bbSNeel Natu 		if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
2428318224bbSNeel Natu 			idtvec_err = vmcs_idt_vectoring_err();
2429091d4532SNeel Natu 			exitintinfo |= (uint64_t)idtvec_err << 32;
2430318224bbSNeel Natu 		}
243180cb5d84SJohn Baldwin 		error = vm_exit_intinfo(vcpu->vcpu, exitintinfo);
2432091d4532SNeel Natu 		KASSERT(error == 0, ("%s: vm_set_intinfo error %d",
2433091d4532SNeel Natu 		    __func__, error));
2434091d4532SNeel Natu 
2435160471d2SNeel Natu 		/*
2436160471d2SNeel Natu 		 * If 'virtual NMIs' are being used and the VM-exit
2437160471d2SNeel Natu 		 * happened while injecting an NMI during the previous
2438091d4532SNeel Natu 		 * VM-entry, then clear "blocking by NMI" in the
2439091d4532SNeel Natu 		 * Guest Interruptibility-State so the NMI can be
2440091d4532SNeel Natu 		 * reinjected on the subsequent VM-entry.
2441091d4532SNeel Natu 		 *
2442091d4532SNeel Natu 		 * However, if the NMI was being delivered through a task
2443091d4532SNeel Natu 		 * gate, then the new task must start execution with NMIs
2444091d4532SNeel Natu 		 * blocked so don't clear NMI blocking in this case.
2445160471d2SNeel Natu 		 */
2446091d4532SNeel Natu 		intr_type = idtvec_info & VMCS_INTR_T_MASK;
2447091d4532SNeel Natu 		if (intr_type == VMCS_INTR_T_NMI) {
2448091d4532SNeel Natu 			if (reason != EXIT_REASON_TASK_SWITCH)
2449869c8d19SJohn Baldwin 				vmx_clear_nmi_blocking(vcpu);
2450091d4532SNeel Natu 			else
2451869c8d19SJohn Baldwin 				vmx_assert_nmi_blocking(vcpu);
2452160471d2SNeel Natu 		}
2453091d4532SNeel Natu 
2454091d4532SNeel Natu 		/*
2455091d4532SNeel Natu 		 * Update VM-entry instruction length if the event being
2456091d4532SNeel Natu 		 * delivered was a software interrupt or software exception.
2457091d4532SNeel Natu 		 */
2458091d4532SNeel Natu 		if (intr_type == VMCS_INTR_T_SWINTR ||
2459091d4532SNeel Natu 		    intr_type == VMCS_INTR_T_PRIV_SWEXCEPTION ||
2460091d4532SNeel Natu 		    intr_type == VMCS_INTR_T_SWEXCEPTION) {
24613de83862SNeel Natu 			vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
2462318224bbSNeel Natu 		}
2463318224bbSNeel Natu 	}
2464318224bbSNeel Natu 
2465318224bbSNeel Natu 	switch (reason) {
24663d5444c8SNeel Natu 	case EXIT_REASON_TASK_SWITCH:
24673d5444c8SNeel Natu 		ts = &vmexit->u.task_switch;
24683d5444c8SNeel Natu 		ts->tsssel = qual & 0xffff;
24693d5444c8SNeel Natu 		ts->reason = vmx_task_switch_reason(qual);
24703d5444c8SNeel Natu 		ts->ext = 0;
24713d5444c8SNeel Natu 		ts->errcode_valid = 0;
24723d5444c8SNeel Natu 		vmx_paging_info(&ts->paging);
24733d5444c8SNeel Natu 		/*
24743d5444c8SNeel Natu 		 * If the task switch was due to a CALL, JMP, IRET, software
24753d5444c8SNeel Natu 		 * interrupt (INT n) or software exception (INT3, INTO),
24763d5444c8SNeel Natu 		 * then the saved %rip references the instruction that caused
24773d5444c8SNeel Natu 		 * the task switch. The instruction length field in the VMCS
24783d5444c8SNeel Natu 		 * is valid in this case.
24793d5444c8SNeel Natu 		 *
24803d5444c8SNeel Natu 		 * In all other cases (e.g., NMI, hardware exception) the
24813d5444c8SNeel Natu 		 * saved %rip is one that would have been saved in the old TSS
24823d5444c8SNeel Natu 		 * had the task switch completed normally so the instruction
24833d5444c8SNeel Natu 		 * length field is not needed in this case and is explicitly
24843d5444c8SNeel Natu 		 * set to 0.
24853d5444c8SNeel Natu 		 */
24863d5444c8SNeel Natu 		if (ts->reason == TSR_IDT_GATE) {
24873d5444c8SNeel Natu 			KASSERT(idtvec_info & VMCS_IDT_VEC_VALID,
2488091d4532SNeel Natu 			    ("invalid idtvec_info %#x for IDT task switch",
24893d5444c8SNeel Natu 			    idtvec_info));
24903d5444c8SNeel Natu 			intr_type = idtvec_info & VMCS_INTR_T_MASK;
24913d5444c8SNeel Natu 			if (intr_type != VMCS_INTR_T_SWINTR &&
24923d5444c8SNeel Natu 			    intr_type != VMCS_INTR_T_SWEXCEPTION &&
24933d5444c8SNeel Natu 			    intr_type != VMCS_INTR_T_PRIV_SWEXCEPTION) {
24943d5444c8SNeel Natu 				/* Task switch triggered by external event */
24953d5444c8SNeel Natu 				ts->ext = 1;
24963d5444c8SNeel Natu 				vmexit->inst_length = 0;
24973d5444c8SNeel Natu 				if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
24983d5444c8SNeel Natu 					ts->errcode_valid = 1;
24993d5444c8SNeel Natu 					ts->errcode = vmcs_idt_vectoring_err();
25003d5444c8SNeel Natu 				}
25013d5444c8SNeel Natu 			}
25023d5444c8SNeel Natu 		}
25033d5444c8SNeel Natu 		vmexit->exitcode = VM_EXITCODE_TASK_SWITCH;
25041aa51504SJohn Baldwin 		SDT_PROBE4(vmm, vmx, exit, taskswitch, vmx, vcpuid, vmexit, ts);
250557e0119eSJohn Baldwin 		VMX_CTR4(vcpu, "task switch reason %d, tss 0x%04x, "
25063d5444c8SNeel Natu 		    "%s errcode 0x%016lx", ts->reason, ts->tsssel,
25073d5444c8SNeel Natu 		    ts->ext ? "external" : "internal",
25083d5444c8SNeel Natu 		    ((uint64_t)ts->errcode << 32) | ts->errcode_valid);
25093d5444c8SNeel Natu 		break;
2510366f6083SPeter Grehan 	case EXIT_REASON_CR_ACCESS:
25113dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_CR_ACCESS, 1);
25121aa51504SJohn Baldwin 		SDT_PROBE4(vmm, vmx, exit, craccess, vmx, vcpuid, vmexit, qual);
2513594db002STycho Nightingale 		switch (qual & 0xf) {
2514594db002STycho Nightingale 		case 0:
25151aa51504SJohn Baldwin 			handled = vmx_emulate_cr0_access(vcpu, qual);
2516594db002STycho Nightingale 			break;
2517594db002STycho Nightingale 		case 4:
25181aa51504SJohn Baldwin 			handled = vmx_emulate_cr4_access(vcpu, qual);
2519594db002STycho Nightingale 			break;
2520594db002STycho Nightingale 		case 8:
2521594db002STycho Nightingale 			handled = vmx_emulate_cr8_access(vmx, vcpu, qual);
2522594db002STycho Nightingale 			break;
2523594db002STycho Nightingale 		}
2524366f6083SPeter Grehan 		break;
2525366f6083SPeter Grehan 	case EXIT_REASON_RDMSR:
25263dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_RDMSR, 1);
2527becd9849SNeel Natu 		retu = false;
2528366f6083SPeter Grehan 		ecx = vmxctx->guest_rcx;
252957e0119eSJohn Baldwin 		VMX_CTR1(vcpu, "rdmsr 0x%08x", ecx);
25301aa51504SJohn Baldwin 		SDT_PROBE4(vmm, vmx, exit, rdmsr, vmx, vcpuid, vmexit, ecx);
253180cb5d84SJohn Baldwin 		error = emulate_rdmsr(vcpu, ecx, &retu);
2532b42206f3SNeel Natu 		if (error) {
2533366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_RDMSR;
2534366f6083SPeter Grehan 			vmexit->u.msr.code = ecx;
2535becd9849SNeel Natu 		} else if (!retu) {
2536a0efd3fbSJohn Baldwin 			handled = HANDLED;
2537becd9849SNeel Natu 		} else {
2538becd9849SNeel Natu 			/* Return to userspace with a valid exitcode */
2539becd9849SNeel Natu 			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
2540c3498942SNeel Natu 			    ("emulate_rdmsr retu with bogus exitcode"));
2541becd9849SNeel Natu 		}
2542366f6083SPeter Grehan 		break;
2543366f6083SPeter Grehan 	case EXIT_REASON_WRMSR:
25443dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_WRMSR, 1);
2545becd9849SNeel Natu 		retu = false;
2546366f6083SPeter Grehan 		eax = vmxctx->guest_rax;
2547366f6083SPeter Grehan 		ecx = vmxctx->guest_rcx;
2548366f6083SPeter Grehan 		edx = vmxctx->guest_rdx;
254957e0119eSJohn Baldwin 		VMX_CTR2(vcpu, "wrmsr 0x%08x value 0x%016lx",
25502cb97c9dSNeel Natu 		    ecx, (uint64_t)edx << 32 | eax);
25511aa51504SJohn Baldwin 		SDT_PROBE5(vmm, vmx, exit, wrmsr, vmx, vmexit, vcpuid, ecx,
25526ac73777STycho Nightingale 		    (uint64_t)edx << 32 | eax);
255380cb5d84SJohn Baldwin 		error = emulate_wrmsr(vcpu, ecx, (uint64_t)edx << 32 | eax,
255480cb5d84SJohn Baldwin 		    &retu);
2555b42206f3SNeel Natu 		if (error) {
2556366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_WRMSR;
2557366f6083SPeter Grehan 			vmexit->u.msr.code = ecx;
2558366f6083SPeter Grehan 			vmexit->u.msr.wval = (uint64_t)edx << 32 | eax;
2559becd9849SNeel Natu 		} else if (!retu) {
2560a0efd3fbSJohn Baldwin 			handled = HANDLED;
2561becd9849SNeel Natu 		} else {
2562becd9849SNeel Natu 			/* Return to userspace with a valid exitcode */
2563becd9849SNeel Natu 			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
2564becd9849SNeel Natu 			    ("emulate_wrmsr retu with bogus exitcode"));
2565becd9849SNeel Natu 		}
2566366f6083SPeter Grehan 		break;
2567366f6083SPeter Grehan 	case EXIT_REASON_HLT:
25683dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_HLT, 1);
25691aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, halt, vmx, vcpuid, vmexit);
2570366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_HLT;
25713de83862SNeel Natu 		vmexit->u.hlt.rflags = vmcs_read(VMCS_GUEST_RFLAGS);
2572490768e2STycho Nightingale 		if (virtual_interrupt_delivery)
2573490768e2STycho Nightingale 			vmexit->u.hlt.intr_status =
2574490768e2STycho Nightingale 			    vmcs_read(VMCS_GUEST_INTR_STATUS);
2575490768e2STycho Nightingale 		else
2576490768e2STycho Nightingale 			vmexit->u.hlt.intr_status = 0;
2577366f6083SPeter Grehan 		break;
2578366f6083SPeter Grehan 	case EXIT_REASON_MTF:
25793dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_MTRAP, 1);
25801aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, mtrap, vmx, vcpuid, vmexit);
2581366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_MTRAP;
2582c9c75df4SNeel Natu 		vmexit->inst_length = 0;
2583366f6083SPeter Grehan 		break;
2584366f6083SPeter Grehan 	case EXIT_REASON_PAUSE:
25853dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_PAUSE, 1);
25861aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, pause, vmx, vcpuid, vmexit);
2587366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_PAUSE;
2588366f6083SPeter Grehan 		break;
2589366f6083SPeter Grehan 	case EXIT_REASON_INTR_WINDOW:
25903dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_INTR_WINDOW, 1);
25911aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, intrwindow, vmx, vcpuid, vmexit);
2592869c8d19SJohn Baldwin 		vmx_clear_int_window_exiting(vcpu);
2593b5aaf7b2SNeel Natu 		return (1);
2594366f6083SPeter Grehan 	case EXIT_REASON_EXT_INTR:
2595366f6083SPeter Grehan 		/*
2596366f6083SPeter Grehan 		 * External interrupts serve only to cause VM exits and allow
2597366f6083SPeter Grehan 		 * the host interrupt handler to run.
2598366f6083SPeter Grehan 		 *
2599366f6083SPeter Grehan 		 * If this external interrupt triggers a virtual interrupt
2600366f6083SPeter Grehan 		 * to a VM, then that state will be recorded by the
2601366f6083SPeter Grehan 		 * host interrupt handler in the VM's softc. We will inject
2602366f6083SPeter Grehan 		 * this virtual interrupt during the subsequent VM enter.
2603366f6083SPeter Grehan 		 */
2604f7d47425SNeel Natu 		intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
26056ac73777STycho Nightingale 		SDT_PROBE4(vmm, vmx, exit, interrupt,
26061aa51504SJohn Baldwin 		    vmx, vcpuid, vmexit, intr_info);
2607722b6744SJohn Baldwin 
2608722b6744SJohn Baldwin 		/*
2609722b6744SJohn Baldwin 		 * XXX: Ignore this exit if VMCS_INTR_VALID is not set.
2610ad3e3687SJohn Baldwin 		 * This appears to be a bug in VMware Fusion?
2611722b6744SJohn Baldwin 		 */
2612722b6744SJohn Baldwin 		if (!(intr_info & VMCS_INTR_VALID))
2613722b6744SJohn Baldwin 			return (1);
2614160471d2SNeel Natu 		KASSERT((intr_info & VMCS_INTR_VALID) != 0 &&
2615160471d2SNeel Natu 		    (intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_HWINTR,
2616f7d47425SNeel Natu 		    ("VM exit interruption info invalid: %#x", intr_info));
2617f7d47425SNeel Natu 		vmx_trigger_hostintr(intr_info & 0xff);
2618366f6083SPeter Grehan 
2619366f6083SPeter Grehan 		/*
2620366f6083SPeter Grehan 		 * This is special. We want to treat this as an 'handled'
2621366f6083SPeter Grehan 		 * VM-exit but not increment the instruction pointer.
2622366f6083SPeter Grehan 		 */
26233dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_EXTINT, 1);
2624366f6083SPeter Grehan 		return (1);
2625366f6083SPeter Grehan 	case EXIT_REASON_NMI_WINDOW:
26261aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, nmiwindow, vmx, vcpuid, vmexit);
2627366f6083SPeter Grehan 		/* Exit to allow the pending virtual NMI to be injected */
262880cb5d84SJohn Baldwin 		if (vm_nmi_pending(vcpu->vcpu))
262980cb5d84SJohn Baldwin 			vmx_inject_nmi(vcpu);
2630869c8d19SJohn Baldwin 		vmx_clear_nmi_window_exiting(vcpu);
26313dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_NMI_WINDOW, 1);
2632366f6083SPeter Grehan 		return (1);
2633366f6083SPeter Grehan 	case EXIT_REASON_INOUT:
26343dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_INOUT, 1);
2635366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_INOUT;
2636366f6083SPeter Grehan 		vmexit->u.inout.bytes = (qual & 0x7) + 1;
2637d17b5104SNeel Natu 		vmexit->u.inout.in = in = (qual & 0x8) ? 1 : 0;
2638366f6083SPeter Grehan 		vmexit->u.inout.string = (qual & 0x10) ? 1 : 0;
2639366f6083SPeter Grehan 		vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0;
2640366f6083SPeter Grehan 		vmexit->u.inout.port = (uint16_t)(qual >> 16);
2641366f6083SPeter Grehan 		vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax);
2642d17b5104SNeel Natu 		if (vmexit->u.inout.string) {
2643d17b5104SNeel Natu 			inst_info = vmcs_read(VMCS_EXIT_INSTRUCTION_INFO);
2644d17b5104SNeel Natu 			vmexit->exitcode = VM_EXITCODE_INOUT_STR;
2645d17b5104SNeel Natu 			vis = &vmexit->u.inout_str;
2646e813a873SNeel Natu 			vmx_paging_info(&vis->paging);
2647d17b5104SNeel Natu 			vis->rflags = vmcs_read(VMCS_GUEST_RFLAGS);
2648d17b5104SNeel Natu 			vis->cr0 = vmcs_read(VMCS_GUEST_CR0);
2649869c8d19SJohn Baldwin 			vis->index = inout_str_index(vcpu, in);
2650869c8d19SJohn Baldwin 			vis->count = inout_str_count(vcpu, vis->inout.rep);
2651d17b5104SNeel Natu 			vis->addrsize = inout_str_addrsize(inst_info);
2652869c8d19SJohn Baldwin 			inout_str_seginfo(vcpu, inst_info, in, vis);
2653762fd208STycho Nightingale 		}
26541aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, inout, vmx, vcpuid, vmexit);
2655366f6083SPeter Grehan 		break;
2656366f6083SPeter Grehan 	case EXIT_REASON_CPUID:
26573dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_CPUID, 1);
26581aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, cpuid, vmx, vcpuid, vmexit);
265980cb5d84SJohn Baldwin 		handled = vmx_handle_cpuid(vcpu, vmxctx);
2660366f6083SPeter Grehan 		break;
2661e5a1d950SNeel Natu 	case EXIT_REASON_EXCEPTION:
26623dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_EXCEPTION, 1);
2663e5a1d950SNeel Natu 		intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
2664e5a1d950SNeel Natu 		KASSERT((intr_info & VMCS_INTR_VALID) != 0,
2665e5a1d950SNeel Natu 		    ("VM exit interruption info invalid: %#x", intr_info));
2666c308b23bSNeel Natu 
2667b0538143SNeel Natu 		intr_vec = intr_info & 0xff;
2668b0538143SNeel Natu 		intr_type = intr_info & VMCS_INTR_T_MASK;
2669b0538143SNeel Natu 
2670e5a1d950SNeel Natu 		/*
2671e5a1d950SNeel Natu 		 * If Virtual NMIs control is 1 and the VM-exit is due to a
2672e5a1d950SNeel Natu 		 * fault encountered during the execution of IRET then we must
2673e5a1d950SNeel Natu 		 * restore the state of "virtual-NMI blocking" before resuming
2674e5a1d950SNeel Natu 		 * the guest.
2675e5a1d950SNeel Natu 		 *
2676e5a1d950SNeel Natu 		 * See "Resuming Guest Software after Handling an Exception".
2677091d4532SNeel Natu 		 * See "Information for VM Exits Due to Vectored Events".
2678e5a1d950SNeel Natu 		 */
2679e5a1d950SNeel Natu 		if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
2680b0538143SNeel Natu 		    (intr_vec != IDT_DF) &&
2681e5a1d950SNeel Natu 		    (intr_info & EXIT_QUAL_NMIUDTI) != 0)
2682869c8d19SJohn Baldwin 			vmx_restore_nmi_blocking(vcpu);
2683c308b23bSNeel Natu 
2684c308b23bSNeel Natu 		/*
268562fbd7c2SNeel Natu 		 * The NMI has already been handled in vmx_exit_handle_nmi().
2686c308b23bSNeel Natu 		 */
2687b0538143SNeel Natu 		if (intr_type == VMCS_INTR_T_NMI)
2688c308b23bSNeel Natu 			return (1);
2689b0538143SNeel Natu 
2690b0538143SNeel Natu 		/*
2691b0538143SNeel Natu 		 * Call the machine check handler by hand. Also don't reflect
2692b0538143SNeel Natu 		 * the machine check back into the guest.
2693b0538143SNeel Natu 		 */
2694b0538143SNeel Natu 		if (intr_vec == IDT_MC) {
269557e0119eSJohn Baldwin 			VMX_CTR0(vcpu, "Vectoring to MCE handler");
2696b0538143SNeel Natu 			__asm __volatile("int $18");
2697b0538143SNeel Natu 			return (1);
2698b0538143SNeel Natu 		}
2699b0538143SNeel Natu 
2700cbd03a9dSJohn Baldwin 		/*
2701cbd03a9dSJohn Baldwin 		 * If the hypervisor has requested user exits for
2702cbd03a9dSJohn Baldwin 		 * debug exceptions, bounce them out to userland.
2703cbd03a9dSJohn Baldwin 		 */
2704cbd03a9dSJohn Baldwin 		if (intr_type == VMCS_INTR_T_SWEXCEPTION && intr_vec == IDT_BP &&
27051aa51504SJohn Baldwin 		    (vcpu->cap.set & (1 << VM_CAP_BPT_EXIT))) {
2706cbd03a9dSJohn Baldwin 			vmexit->exitcode = VM_EXITCODE_BPT;
2707cbd03a9dSJohn Baldwin 			vmexit->u.bpt.inst_length = vmexit->inst_length;
2708cbd03a9dSJohn Baldwin 			vmexit->inst_length = 0;
2709cbd03a9dSJohn Baldwin 			break;
2710cbd03a9dSJohn Baldwin 		}
2711cbd03a9dSJohn Baldwin 
2712b0538143SNeel Natu 		if (intr_vec == IDT_PF) {
2713b0538143SNeel Natu 			error = vmxctx_setreg(vmxctx, VM_REG_GUEST_CR2, qual);
2714b0538143SNeel Natu 			KASSERT(error == 0, ("%s: vmxctx_setreg(cr2) error %d",
2715b0538143SNeel Natu 			    __func__, error));
2716b0538143SNeel Natu 		}
2717b0538143SNeel Natu 
2718b0538143SNeel Natu 		/*
2719b0538143SNeel Natu 		 * Software exceptions exhibit trap-like behavior. This in
2720b0538143SNeel Natu 		 * turn requires populating the VM-entry instruction length
2721b0538143SNeel Natu 		 * so that the %rip in the trap frame is past the INT3/INTO
2722b0538143SNeel Natu 		 * instruction.
2723b0538143SNeel Natu 		 */
2724b0538143SNeel Natu 		if (intr_type == VMCS_INTR_T_SWEXCEPTION)
2725b0538143SNeel Natu 			vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
2726b0538143SNeel Natu 
2727b0538143SNeel Natu 		/* Reflect all other exceptions back into the guest */
2728c9c75df4SNeel Natu 		errcode_valid = errcode = 0;
2729b0538143SNeel Natu 		if (intr_info & VMCS_INTR_DEL_ERRCODE) {
2730c9c75df4SNeel Natu 			errcode_valid = 1;
2731c9c75df4SNeel Natu 			errcode = vmcs_read(VMCS_EXIT_INTR_ERRCODE);
2732b0538143SNeel Natu 		}
273357e0119eSJohn Baldwin 		VMX_CTR2(vcpu, "Reflecting exception %d/%#x into "
2734c9c75df4SNeel Natu 		    "the guest", intr_vec, errcode);
27356ac73777STycho Nightingale 		SDT_PROBE5(vmm, vmx, exit, exception,
27361aa51504SJohn Baldwin 		    vmx, vcpuid, vmexit, intr_vec, errcode);
2737d3956e46SJohn Baldwin 		error = vm_inject_exception(vcpu->vcpu, intr_vec,
2738c9c75df4SNeel Natu 		    errcode_valid, errcode, 0);
2739b0538143SNeel Natu 		KASSERT(error == 0, ("%s: vm_inject_exception error %d",
2740b0538143SNeel Natu 		    __func__, error));
2741b0538143SNeel Natu 		return (1);
2742b0538143SNeel Natu 
2743cd942e0fSPeter Grehan 	case EXIT_REASON_EPT_FAULT:
2744318224bbSNeel Natu 		/*
2745318224bbSNeel Natu 		 * If 'gpa' lies within the address space allocated to
2746318224bbSNeel Natu 		 * memory then this must be a nested page fault otherwise
2747318224bbSNeel Natu 		 * this must be an instruction that accesses MMIO space.
2748318224bbSNeel Natu 		 */
2749a2da7af6SNeel Natu 		gpa = vmcs_gpa();
275080cb5d84SJohn Baldwin 		if (vm_mem_allocated(vcpu->vcpu, gpa) ||
27511aa51504SJohn Baldwin 		    apic_access_fault(vcpu, gpa)) {
2752cd942e0fSPeter Grehan 			vmexit->exitcode = VM_EXITCODE_PAGING;
2753d087a399SNeel Natu 			vmexit->inst_length = 0;
275413ec9371SPeter Grehan 			vmexit->u.paging.gpa = gpa;
2755318224bbSNeel Natu 			vmexit->u.paging.fault_type = ept_fault_type(qual);
27563dc3d32aSJohn Baldwin 			vmm_stat_incr(vcpu->vcpu, VMEXIT_NESTED_FAULT, 1);
27576ac73777STycho Nightingale 			SDT_PROBE5(vmm, vmx, exit, nestedfault,
27581aa51504SJohn Baldwin 			    vmx, vcpuid, vmexit, gpa, qual);
2759318224bbSNeel Natu 		} else if (ept_emulation_fault(qual)) {
2760e4c8a13dSNeel Natu 			vmexit_inst_emul(vmexit, gpa, vmcs_gla());
27613dc3d32aSJohn Baldwin 			vmm_stat_incr(vcpu->vcpu, VMEXIT_INST_EMUL, 1);
27626ac73777STycho Nightingale 			SDT_PROBE4(vmm, vmx, exit, mmiofault,
27631aa51504SJohn Baldwin 			    vmx, vcpuid, vmexit, gpa);
2764a2da7af6SNeel Natu 		}
2765e5a1d950SNeel Natu 		/*
2766e5a1d950SNeel Natu 		 * If Virtual NMIs control is 1 and the VM-exit is due to an
2767e5a1d950SNeel Natu 		 * EPT fault during the execution of IRET then we must restore
2768e5a1d950SNeel Natu 		 * the state of "virtual-NMI blocking" before resuming.
2769e5a1d950SNeel Natu 		 *
2770e5a1d950SNeel Natu 		 * See description of "NMI unblocking due to IRET" in
2771e5a1d950SNeel Natu 		 * "Exit Qualification for EPT Violations".
2772e5a1d950SNeel Natu 		 */
2773e5a1d950SNeel Natu 		if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
2774e5a1d950SNeel Natu 		    (qual & EXIT_QUAL_NMIUDTI) != 0)
2775869c8d19SJohn Baldwin 			vmx_restore_nmi_blocking(vcpu);
2776cd942e0fSPeter Grehan 		break;
277730b94db8SNeel Natu 	case EXIT_REASON_VIRTUALIZED_EOI:
277830b94db8SNeel Natu 		vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI;
277930b94db8SNeel Natu 		vmexit->u.ioapic_eoi.vector = qual & 0xFF;
27801aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, eoi, vmx, vcpuid, vmexit);
278130b94db8SNeel Natu 		vmexit->inst_length = 0;	/* trap-like */
278230b94db8SNeel Natu 		break;
278388c4b8d1SNeel Natu 	case EXIT_REASON_APIC_ACCESS:
27841aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, apicaccess, vmx, vcpuid, vmexit);
27851aa51504SJohn Baldwin 		handled = vmx_handle_apic_access(vcpu, vmexit);
278688c4b8d1SNeel Natu 		break;
278788c4b8d1SNeel Natu 	case EXIT_REASON_APIC_WRITE:
278888c4b8d1SNeel Natu 		/*
278988c4b8d1SNeel Natu 		 * APIC-write VM exit is trap-like so the %rip is already
279088c4b8d1SNeel Natu 		 * pointing to the next instruction.
279188c4b8d1SNeel Natu 		 */
279288c4b8d1SNeel Natu 		vmexit->inst_length = 0;
2793d3956e46SJohn Baldwin 		vlapic = vm_lapic(vcpu->vcpu);
27946ac73777STycho Nightingale 		SDT_PROBE4(vmm, vmx, exit, apicwrite,
27951aa51504SJohn Baldwin 		    vmx, vcpuid, vmexit, vlapic);
27961aa51504SJohn Baldwin 		handled = vmx_handle_apic_write(vcpu, vlapic, qual);
279788c4b8d1SNeel Natu 		break;
2798abb023fbSJohn Baldwin 	case EXIT_REASON_XSETBV:
27991aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, xsetbv, vmx, vcpuid, vmexit);
2800a0efd3fbSJohn Baldwin 		handled = vmx_emulate_xsetbv(vmx, vcpu, vmexit);
2801abb023fbSJohn Baldwin 		break;
280265145c7fSNeel Natu 	case EXIT_REASON_MONITOR:
28031aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, monitor, vmx, vcpuid, vmexit);
280465145c7fSNeel Natu 		vmexit->exitcode = VM_EXITCODE_MONITOR;
280565145c7fSNeel Natu 		break;
280665145c7fSNeel Natu 	case EXIT_REASON_MWAIT:
28071aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, mwait, vmx, vcpuid, vmexit);
280865145c7fSNeel Natu 		vmexit->exitcode = VM_EXITCODE_MWAIT;
280965145c7fSNeel Natu 		break;
28101bc51badSMichael Reifenberger 	case EXIT_REASON_TPR:
2811d3956e46SJohn Baldwin 		vlapic = vm_lapic(vcpu->vcpu);
28121bc51badSMichael Reifenberger 		vlapic_sync_tpr(vlapic);
28131bc51badSMichael Reifenberger 		vmexit->inst_length = 0;
28141bc51badSMichael Reifenberger 		handled = HANDLED;
28151bc51badSMichael Reifenberger 		break;
281627d26457SAndrew Turner 	case EXIT_REASON_VMCALL:
281727d26457SAndrew Turner 	case EXIT_REASON_VMCLEAR:
281827d26457SAndrew Turner 	case EXIT_REASON_VMLAUNCH:
281927d26457SAndrew Turner 	case EXIT_REASON_VMPTRLD:
282027d26457SAndrew Turner 	case EXIT_REASON_VMPTRST:
282127d26457SAndrew Turner 	case EXIT_REASON_VMREAD:
282227d26457SAndrew Turner 	case EXIT_REASON_VMRESUME:
282327d26457SAndrew Turner 	case EXIT_REASON_VMWRITE:
282427d26457SAndrew Turner 	case EXIT_REASON_VMXOFF:
282527d26457SAndrew Turner 	case EXIT_REASON_VMXON:
28261aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, vminsn, vmx, vcpuid, vmexit);
282727d26457SAndrew Turner 		vmexit->exitcode = VM_EXITCODE_VMINSN;
282827d26457SAndrew Turner 		break;
28294eadbef9SCorvin Köhne 	case EXIT_REASON_INVD:
28303ba952e1SCorvin Köhne 	case EXIT_REASON_WBINVD:
28314eadbef9SCorvin Köhne 		/* ignore exit */
28323ba952e1SCorvin Köhne 		handled = HANDLED;
28333ba952e1SCorvin Köhne 		break;
2834366f6083SPeter Grehan 	default:
28356ac73777STycho Nightingale 		SDT_PROBE4(vmm, vmx, exit, unknown,
28361aa51504SJohn Baldwin 		    vmx, vcpuid, vmexit, reason);
28373dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_UNKNOWN, 1);
2838366f6083SPeter Grehan 		break;
2839366f6083SPeter Grehan 	}
2840366f6083SPeter Grehan 
2841366f6083SPeter Grehan 	if (handled) {
2842366f6083SPeter Grehan 		/*
2843366f6083SPeter Grehan 		 * It is possible that control is returned to userland
2844366f6083SPeter Grehan 		 * even though we were able to handle the VM exit in the
2845eeefa4e4SNeel Natu 		 * kernel.
2846366f6083SPeter Grehan 		 *
2847366f6083SPeter Grehan 		 * In such a case we want to make sure that the userland
2848366f6083SPeter Grehan 		 * restarts guest execution at the instruction *after*
2849366f6083SPeter Grehan 		 * the one we just processed. Therefore we update the
2850366f6083SPeter Grehan 		 * guest rip in the VMCS and in 'vmexit'.
2851366f6083SPeter Grehan 		 */
2852366f6083SPeter Grehan 		vmexit->rip += vmexit->inst_length;
2853366f6083SPeter Grehan 		vmexit->inst_length = 0;
28543de83862SNeel Natu 		vmcs_write(VMCS_GUEST_RIP, vmexit->rip);
2855366f6083SPeter Grehan 	} else {
2856366f6083SPeter Grehan 		if (vmexit->exitcode == VM_EXITCODE_BOGUS) {
2857366f6083SPeter Grehan 			/*
2858366f6083SPeter Grehan 			 * If this VM exit was not claimed by anybody then
2859366f6083SPeter Grehan 			 * treat it as a generic VMX exit.
2860366f6083SPeter Grehan 			 */
2861366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_VMX;
28620492757cSNeel Natu 			vmexit->u.vmx.status = VM_SUCCESS;
2863c308b23bSNeel Natu 			vmexit->u.vmx.inst_type = 0;
2864c308b23bSNeel Natu 			vmexit->u.vmx.inst_error = 0;
2865366f6083SPeter Grehan 		} else {
2866366f6083SPeter Grehan 			/*
2867366f6083SPeter Grehan 			 * The exitcode and collateral have been populated.
2868366f6083SPeter Grehan 			 * The VM exit will be processed further in userland.
2869366f6083SPeter Grehan 			 */
2870366f6083SPeter Grehan 		}
2871366f6083SPeter Grehan 	}
28726ac73777STycho Nightingale 
28736ac73777STycho Nightingale 	SDT_PROBE4(vmm, vmx, exit, return,
28741aa51504SJohn Baldwin 	    vmx, vcpuid, vmexit, handled);
2875366f6083SPeter Grehan 	return (handled);
2876366f6083SPeter Grehan }
2877366f6083SPeter Grehan 
287840487465SNeel Natu static __inline void
28790492757cSNeel Natu vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit)
28800492757cSNeel Natu {
28810492757cSNeel Natu 
28820492757cSNeel Natu 	KASSERT(vmxctx->inst_fail_status != VM_SUCCESS,
28830492757cSNeel Natu 	    ("vmx_exit_inst_error: invalid inst_fail_status %d",
28840492757cSNeel Natu 	    vmxctx->inst_fail_status));
28850492757cSNeel Natu 
28860492757cSNeel Natu 	vmexit->inst_length = 0;
28870492757cSNeel Natu 	vmexit->exitcode = VM_EXITCODE_VMX;
28880492757cSNeel Natu 	vmexit->u.vmx.status = vmxctx->inst_fail_status;
28890492757cSNeel Natu 	vmexit->u.vmx.inst_error = vmcs_instruction_error();
28900492757cSNeel Natu 	vmexit->u.vmx.exit_reason = ~0;
28910492757cSNeel Natu 	vmexit->u.vmx.exit_qualification = ~0;
28920492757cSNeel Natu 
28930492757cSNeel Natu 	switch (rc) {
28940492757cSNeel Natu 	case VMX_VMRESUME_ERROR:
28950492757cSNeel Natu 	case VMX_VMLAUNCH_ERROR:
28960492757cSNeel Natu 		vmexit->u.vmx.inst_type = rc;
28970492757cSNeel Natu 		break;
28980492757cSNeel Natu 	default:
28990492757cSNeel Natu 		panic("vm_exit_inst_error: vmx_enter_guest returned %d", rc);
29000492757cSNeel Natu 	}
29010492757cSNeel Natu }
29020492757cSNeel Natu 
290362fbd7c2SNeel Natu /*
290462fbd7c2SNeel Natu  * If the NMI-exiting VM execution control is set to '1' then an NMI in
290562fbd7c2SNeel Natu  * non-root operation causes a VM-exit. NMI blocking is in effect so it is
290662fbd7c2SNeel Natu  * sufficient to simply vector to the NMI handler via a software interrupt.
290762fbd7c2SNeel Natu  * However, this must be done before maskable interrupts are enabled
290862fbd7c2SNeel Natu  * otherwise the "iret" issued by an interrupt handler will incorrectly
290962fbd7c2SNeel Natu  * clear NMI blocking.
291062fbd7c2SNeel Natu  */
291162fbd7c2SNeel Natu static __inline void
2912869c8d19SJohn Baldwin vmx_exit_handle_nmi(struct vmx_vcpu *vcpu, struct vm_exit *vmexit)
291362fbd7c2SNeel Natu {
291462fbd7c2SNeel Natu 	uint32_t intr_info;
291562fbd7c2SNeel Natu 
291662fbd7c2SNeel Natu 	KASSERT((read_rflags() & PSL_I) == 0, ("interrupts enabled"));
291762fbd7c2SNeel Natu 
291862fbd7c2SNeel Natu 	if (vmexit->u.vmx.exit_reason != EXIT_REASON_EXCEPTION)
291962fbd7c2SNeel Natu 		return;
292062fbd7c2SNeel Natu 
292162fbd7c2SNeel Natu 	intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
292262fbd7c2SNeel Natu 	KASSERT((intr_info & VMCS_INTR_VALID) != 0,
292362fbd7c2SNeel Natu 	    ("VM exit interruption info invalid: %#x", intr_info));
292462fbd7c2SNeel Natu 
292562fbd7c2SNeel Natu 	if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI) {
292662fbd7c2SNeel Natu 		KASSERT((intr_info & 0xff) == IDT_NMI, ("VM exit due "
292762fbd7c2SNeel Natu 		    "to NMI has invalid vector: %#x", intr_info));
292857e0119eSJohn Baldwin 		VMX_CTR0(vcpu, "Vectoring to NMI handler");
292962fbd7c2SNeel Natu 		__asm __volatile("int $2");
293062fbd7c2SNeel Natu 	}
293162fbd7c2SNeel Natu }
293262fbd7c2SNeel Natu 
293365eefbe4SJohn Baldwin static __inline void
293465eefbe4SJohn Baldwin vmx_dr_enter_guest(struct vmxctx *vmxctx)
293565eefbe4SJohn Baldwin {
293665eefbe4SJohn Baldwin 	register_t rflags;
293765eefbe4SJohn Baldwin 
293865eefbe4SJohn Baldwin 	/* Save host control debug registers. */
293965eefbe4SJohn Baldwin 	vmxctx->host_dr7 = rdr7();
294065eefbe4SJohn Baldwin 	vmxctx->host_debugctl = rdmsr(MSR_DEBUGCTLMSR);
294165eefbe4SJohn Baldwin 
294265eefbe4SJohn Baldwin 	/*
294365eefbe4SJohn Baldwin 	 * Disable debugging in DR7 and DEBUGCTL to avoid triggering
294465eefbe4SJohn Baldwin 	 * exceptions in the host based on the guest DRx values.  The
294565eefbe4SJohn Baldwin 	 * guest DR7 and DEBUGCTL are saved/restored in the VMCS.
294665eefbe4SJohn Baldwin 	 */
294765eefbe4SJohn Baldwin 	load_dr7(0);
294865eefbe4SJohn Baldwin 	wrmsr(MSR_DEBUGCTLMSR, 0);
294965eefbe4SJohn Baldwin 
295065eefbe4SJohn Baldwin 	/*
295165eefbe4SJohn Baldwin 	 * Disable single stepping the kernel to avoid corrupting the
295265eefbe4SJohn Baldwin 	 * guest DR6.  A debugger might still be able to corrupt the
295365eefbe4SJohn Baldwin 	 * guest DR6 by setting a breakpoint after this point and then
295465eefbe4SJohn Baldwin 	 * single stepping.
295565eefbe4SJohn Baldwin 	 */
295665eefbe4SJohn Baldwin 	rflags = read_rflags();
295765eefbe4SJohn Baldwin 	vmxctx->host_tf = rflags & PSL_T;
295865eefbe4SJohn Baldwin 	write_rflags(rflags & ~PSL_T);
295965eefbe4SJohn Baldwin 
296065eefbe4SJohn Baldwin 	/* Save host debug registers. */
296165eefbe4SJohn Baldwin 	vmxctx->host_dr0 = rdr0();
296265eefbe4SJohn Baldwin 	vmxctx->host_dr1 = rdr1();
296365eefbe4SJohn Baldwin 	vmxctx->host_dr2 = rdr2();
296465eefbe4SJohn Baldwin 	vmxctx->host_dr3 = rdr3();
296565eefbe4SJohn Baldwin 	vmxctx->host_dr6 = rdr6();
296665eefbe4SJohn Baldwin 
296765eefbe4SJohn Baldwin 	/* Restore guest debug registers. */
296865eefbe4SJohn Baldwin 	load_dr0(vmxctx->guest_dr0);
296965eefbe4SJohn Baldwin 	load_dr1(vmxctx->guest_dr1);
297065eefbe4SJohn Baldwin 	load_dr2(vmxctx->guest_dr2);
297165eefbe4SJohn Baldwin 	load_dr3(vmxctx->guest_dr3);
297265eefbe4SJohn Baldwin 	load_dr6(vmxctx->guest_dr6);
297365eefbe4SJohn Baldwin }
297465eefbe4SJohn Baldwin 
297565eefbe4SJohn Baldwin static __inline void
297665eefbe4SJohn Baldwin vmx_dr_leave_guest(struct vmxctx *vmxctx)
297765eefbe4SJohn Baldwin {
297865eefbe4SJohn Baldwin 
297965eefbe4SJohn Baldwin 	/* Save guest debug registers. */
298065eefbe4SJohn Baldwin 	vmxctx->guest_dr0 = rdr0();
298165eefbe4SJohn Baldwin 	vmxctx->guest_dr1 = rdr1();
298265eefbe4SJohn Baldwin 	vmxctx->guest_dr2 = rdr2();
298365eefbe4SJohn Baldwin 	vmxctx->guest_dr3 = rdr3();
298465eefbe4SJohn Baldwin 	vmxctx->guest_dr6 = rdr6();
298565eefbe4SJohn Baldwin 
298665eefbe4SJohn Baldwin 	/*
298765eefbe4SJohn Baldwin 	 * Restore host debug registers.  Restore DR7, DEBUGCTL, and
298865eefbe4SJohn Baldwin 	 * PSL_T last.
298965eefbe4SJohn Baldwin 	 */
299065eefbe4SJohn Baldwin 	load_dr0(vmxctx->host_dr0);
299165eefbe4SJohn Baldwin 	load_dr1(vmxctx->host_dr1);
299265eefbe4SJohn Baldwin 	load_dr2(vmxctx->host_dr2);
299365eefbe4SJohn Baldwin 	load_dr3(vmxctx->host_dr3);
299465eefbe4SJohn Baldwin 	load_dr6(vmxctx->host_dr6);
299565eefbe4SJohn Baldwin 	wrmsr(MSR_DEBUGCTLMSR, vmxctx->host_debugctl);
299665eefbe4SJohn Baldwin 	load_dr7(vmxctx->host_dr7);
299765eefbe4SJohn Baldwin 	write_rflags(read_rflags() | vmxctx->host_tf);
299865eefbe4SJohn Baldwin }
299965eefbe4SJohn Baldwin 
30008e2cbc56SMark Johnston static __inline void
30018e2cbc56SMark Johnston vmx_pmap_activate(struct vmx *vmx, pmap_t pmap)
30028e2cbc56SMark Johnston {
30038e2cbc56SMark Johnston 	long eptgen;
30048e2cbc56SMark Johnston 	int cpu;
30058e2cbc56SMark Johnston 
30068e2cbc56SMark Johnston 	cpu = curcpu;
30078e2cbc56SMark Johnston 
30088e2cbc56SMark Johnston 	CPU_SET_ATOMIC(cpu, &pmap->pm_active);
30096f5a9606SMark Johnston 	smr_enter(pmap->pm_eptsmr);
30108e2cbc56SMark Johnston 	eptgen = atomic_load_long(&pmap->pm_eptgen);
30118e2cbc56SMark Johnston 	if (eptgen != vmx->eptgen[cpu]) {
30128e2cbc56SMark Johnston 		vmx->eptgen[cpu] = eptgen;
30138e2cbc56SMark Johnston 		invept(INVEPT_TYPE_SINGLE_CONTEXT,
30148e2cbc56SMark Johnston 		    (struct invept_desc){ .eptp = vmx->eptp, ._res = 0 });
30158e2cbc56SMark Johnston 	}
30168e2cbc56SMark Johnston }
30178e2cbc56SMark Johnston 
30188e2cbc56SMark Johnston static __inline void
30198e2cbc56SMark Johnston vmx_pmap_deactivate(struct vmx *vmx, pmap_t pmap)
30208e2cbc56SMark Johnston {
30216f5a9606SMark Johnston 	smr_exit(pmap->pm_eptsmr);
30228e2cbc56SMark Johnston 	CPU_CLR_ATOMIC(curcpu, &pmap->pm_active);
30238e2cbc56SMark Johnston }
30248e2cbc56SMark Johnston 
30250492757cSNeel Natu static int
3026869c8d19SJohn Baldwin vmx_run(void *vcpui, register_t rip, pmap_t pmap, struct vm_eventinfo *evinfo)
30270492757cSNeel Natu {
302880cb5d84SJohn Baldwin 	int rc, handled, launched;
3029366f6083SPeter Grehan 	struct vmx *vmx;
30301aa51504SJohn Baldwin 	struct vmx_vcpu *vcpu;
3031366f6083SPeter Grehan 	struct vmxctx *vmxctx;
3032366f6083SPeter Grehan 	struct vmcs *vmcs;
303398ed632cSNeel Natu 	struct vm_exit *vmexit;
3034de5ea6b6SNeel Natu 	struct vlapic *vlapic;
303579c59630SNeel Natu 	uint32_t exit_reason;
3036b843f9beSJohn Baldwin 	struct region_descriptor gdtr, idtr;
3037b843f9beSJohn Baldwin 	uint16_t ldt_sel;
3038366f6083SPeter Grehan 
30391aa51504SJohn Baldwin 	vcpu = vcpui;
3040869c8d19SJohn Baldwin 	vmx = vcpu->vmx;
30411aa51504SJohn Baldwin 	vmcs = vcpu->vmcs;
30421aa51504SJohn Baldwin 	vmxctx = &vcpu->ctx;
3043d3956e46SJohn Baldwin 	vlapic = vm_lapic(vcpu->vcpu);
304480cb5d84SJohn Baldwin 	vmexit = vm_exitinfo(vcpu->vcpu);
30450492757cSNeel Natu 	launched = 0;
304698ed632cSNeel Natu 
3047318224bbSNeel Natu 	KASSERT(vmxctx->pmap == pmap,
3048318224bbSNeel Natu 	    ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap));
3049318224bbSNeel Natu 
305080cb5d84SJohn Baldwin 	vmx_msr_guest_enter(vcpu);
3051c3498942SNeel Natu 
3052366f6083SPeter Grehan 	VMPTRLD(vmcs);
3053366f6083SPeter Grehan 
3054366f6083SPeter Grehan 	/*
3055366f6083SPeter Grehan 	 * XXX
3056366f6083SPeter Grehan 	 * We do this every time because we may setup the virtual machine
3057366f6083SPeter Grehan 	 * from a different process than the one that actually runs it.
3058366f6083SPeter Grehan 	 *
3059366f6083SPeter Grehan 	 * If the life of a virtual machine was spent entirely in the context
306015add60dSPeter Grehan 	 * of a single process we could do this once in vmx_init().
3061366f6083SPeter Grehan 	 */
30623de83862SNeel Natu 	vmcs_write(VMCS_HOST_CR3, rcr3());
3063366f6083SPeter Grehan 
30642ce12423SNeel Natu 	vmcs_write(VMCS_GUEST_RIP, rip);
3065953c2c47SNeel Natu 	vmx_set_pcpu_defaults(vmx, vcpu, pmap);
3066366f6083SPeter Grehan 	do {
30672ce12423SNeel Natu 		KASSERT(vmcs_guest_rip() == rip, ("%s: vmcs guest rip mismatch "
30682ce12423SNeel Natu 		    "%#lx/%#lx", __func__, vmcs_guest_rip(), rip));
306940487465SNeel Natu 
30702ce12423SNeel Natu 		handled = UNHANDLED;
30710492757cSNeel Natu 		/*
30720492757cSNeel Natu 		 * Interrupts are disabled from this point on until the
30730492757cSNeel Natu 		 * guest starts executing. This is done for the following
30740492757cSNeel Natu 		 * reasons:
30750492757cSNeel Natu 		 *
30760492757cSNeel Natu 		 * If an AST is asserted on this thread after the check below,
30770492757cSNeel Natu 		 * then the IPI_AST notification will not be lost, because it
30780492757cSNeel Natu 		 * will cause a VM exit due to external interrupt as soon as
30790492757cSNeel Natu 		 * the guest state is loaded.
30800492757cSNeel Natu 		 *
30810492757cSNeel Natu 		 * A posted interrupt after 'vmx_inject_interrupts()' will
30820492757cSNeel Natu 		 * not be "lost" because it will be held pending in the host
30830492757cSNeel Natu 		 * APIC because interrupts are disabled. The pending interrupt
30840492757cSNeel Natu 		 * will be recognized as soon as the guest state is loaded.
30850492757cSNeel Natu 		 *
30860492757cSNeel Natu 		 * The same reasoning applies to the IPI generated by
30870492757cSNeel Natu 		 * pmap_invalidate_ept().
30880492757cSNeel Natu 		 */
30890492757cSNeel Natu 		disable_intr();
309080cb5d84SJohn Baldwin 		vmx_inject_interrupts(vcpu, vlapic, rip);
3091091d4532SNeel Natu 
3092091d4532SNeel Natu 		/*
3093091d4532SNeel Natu 		 * Check for vcpu suspension after injecting events because
3094091d4532SNeel Natu 		 * vmx_inject_interrupts() can suspend the vcpu due to a
3095091d4532SNeel Natu 		 * triple fault.
3096091d4532SNeel Natu 		 */
3097248e6799SNeel Natu 		if (vcpu_suspended(evinfo)) {
30980492757cSNeel Natu 			enable_intr();
309980cb5d84SJohn Baldwin 			vm_exit_suspended(vcpu->vcpu, rip);
31000492757cSNeel Natu 			break;
31010492757cSNeel Natu 		}
31020492757cSNeel Natu 
3103892feec2SCorvin Köhne 		if (vcpu_rendezvous_pending(vcpu->vcpu, evinfo)) {
31045b8a8cd1SNeel Natu 			enable_intr();
310580cb5d84SJohn Baldwin 			vm_exit_rendezvous(vcpu->vcpu, rip);
31065b8a8cd1SNeel Natu 			break;
31075b8a8cd1SNeel Natu 		}
31085b8a8cd1SNeel Natu 
3109248e6799SNeel Natu 		if (vcpu_reqidle(evinfo)) {
3110248e6799SNeel Natu 			enable_intr();
311180cb5d84SJohn Baldwin 			vm_exit_reqidle(vcpu->vcpu, rip);
3112248e6799SNeel Natu 			break;
3113248e6799SNeel Natu 		}
3114248e6799SNeel Natu 
311580cb5d84SJohn Baldwin 		if (vcpu_should_yield(vcpu->vcpu)) {
3116b15a09c0SNeel Natu 			enable_intr();
311780cb5d84SJohn Baldwin 			vm_exit_astpending(vcpu->vcpu, rip);
3118869c8d19SJohn Baldwin 			vmx_astpending_trace(vcpu, rip);
311940487465SNeel Natu 			handled = HANDLED;
3120b15a09c0SNeel Natu 			break;
3121b15a09c0SNeel Natu 		}
3122b15a09c0SNeel Natu 
312380cb5d84SJohn Baldwin 		if (vcpu_debugged(vcpu->vcpu)) {
3124fc276d92SJohn Baldwin 			enable_intr();
312580cb5d84SJohn Baldwin 			vm_exit_debug(vcpu->vcpu, rip);
3126fc276d92SJohn Baldwin 			break;
3127fc276d92SJohn Baldwin 		}
3128fc276d92SJohn Baldwin 
3129b843f9beSJohn Baldwin 		/*
31301bc51badSMichael Reifenberger 		 * If TPR Shadowing is enabled, the TPR Threshold
31311bc51badSMichael Reifenberger 		 * must be updated right before entering the guest.
31321bc51badSMichael Reifenberger 		 */
31331bc51badSMichael Reifenberger 		if (tpr_shadowing && !virtual_interrupt_delivery) {
31341aa51504SJohn Baldwin 			if ((vcpu->cap.proc_ctls & PROCBASED_USE_TPR_SHADOW) != 0) {
31351bc51badSMichael Reifenberger 				vmcs_write(VMCS_TPR_THRESHOLD, vlapic_get_cr8(vlapic));
31361bc51badSMichael Reifenberger 			}
31371bc51badSMichael Reifenberger 		}
31381bc51badSMichael Reifenberger 
31391bc51badSMichael Reifenberger 		/*
3140b843f9beSJohn Baldwin 		 * VM exits restore the base address but not the
3141b843f9beSJohn Baldwin 		 * limits of GDTR and IDTR.  The VMCS only stores the
3142b843f9beSJohn Baldwin 		 * base address, so VM exits set the limits to 0xffff.
3143b843f9beSJohn Baldwin 		 * Save and restore the full GDTR and IDTR to restore
3144b843f9beSJohn Baldwin 		 * the limits.
3145b843f9beSJohn Baldwin 		 *
3146b843f9beSJohn Baldwin 		 * The VMCS does not save the LDTR at all, and VM
3147b843f9beSJohn Baldwin 		 * exits clear LDTR as if a NULL selector were loaded.
3148b843f9beSJohn Baldwin 		 * The userspace hypervisor probably doesn't use a
3149b843f9beSJohn Baldwin 		 * LDT, but save and restore it to be safe.
3150b843f9beSJohn Baldwin 		 */
3151b843f9beSJohn Baldwin 		sgdt(&gdtr);
3152b843f9beSJohn Baldwin 		sidt(&idtr);
3153b843f9beSJohn Baldwin 		ldt_sel = sldt();
3154b843f9beSJohn Baldwin 
3155f5f5f1e7SPeter Grehan 		/*
3156f5f5f1e7SPeter Grehan 		 * The TSC_AUX MSR must be saved/restored while interrupts
3157f5f5f1e7SPeter Grehan 		 * are disabled so that it is not possible for the guest
3158f5f5f1e7SPeter Grehan 		 * TSC_AUX MSR value to be overwritten by the resume
3159f5f5f1e7SPeter Grehan 		 * portion of the IPI_SUSPEND codepath. This is why the
3160f5f5f1e7SPeter Grehan 		 * transition of this MSR is handled separately from those
3161f5f5f1e7SPeter Grehan 		 * handled by vmx_msr_guest_{enter,exit}(), which are ok to
3162f5f5f1e7SPeter Grehan 		 * be transitioned with preemption disabled but interrupts
3163f5f5f1e7SPeter Grehan 		 * enabled.
3164f5f5f1e7SPeter Grehan 		 *
3165f5f5f1e7SPeter Grehan 		 * These vmx_msr_guest_{enter,exit}_tsc_aux() calls can be
3166f5f5f1e7SPeter Grehan 		 * anywhere in this loop so long as they happen with
3167f5f5f1e7SPeter Grehan 		 * interrupts disabled. This location is chosen for
3168f5f5f1e7SPeter Grehan 		 * simplicity.
3169f5f5f1e7SPeter Grehan 		 */
3170f5f5f1e7SPeter Grehan 		vmx_msr_guest_enter_tsc_aux(vmx, vcpu);
3171f5f5f1e7SPeter Grehan 
317265eefbe4SJohn Baldwin 		vmx_dr_enter_guest(vmxctx);
317379c59630SNeel Natu 
31748e2cbc56SMark Johnston 		/*
31758e2cbc56SMark Johnston 		 * Mark the EPT as active on this host CPU and invalidate
31768e2cbc56SMark Johnston 		 * EPTP-tagged TLB entries if required.
31778e2cbc56SMark Johnston 		 */
31788e2cbc56SMark Johnston 		vmx_pmap_activate(vmx, pmap);
31798e2cbc56SMark Johnston 
3180869c8d19SJohn Baldwin 		vmx_run_trace(vcpu);
31818e2cbc56SMark Johnston 		rc = vmx_enter_guest(vmxctx, vmx, launched);
31828e2cbc56SMark Johnston 
31838e2cbc56SMark Johnston 		vmx_pmap_deactivate(vmx, pmap);
31848e2cbc56SMark Johnston 		vmx_dr_leave_guest(vmxctx);
3185f5f5f1e7SPeter Grehan 		vmx_msr_guest_exit_tsc_aux(vmx, vcpu);
3186f5f5f1e7SPeter Grehan 
3187b843f9beSJohn Baldwin 		bare_lgdt(&gdtr);
3188b843f9beSJohn Baldwin 		lidt(&idtr);
3189b843f9beSJohn Baldwin 		lldt(ldt_sel);
3190b843f9beSJohn Baldwin 
319179c59630SNeel Natu 		/* Collect some information for VM exit processing */
319279c59630SNeel Natu 		vmexit->rip = rip = vmcs_guest_rip();
319379c59630SNeel Natu 		vmexit->inst_length = vmexit_instruction_length();
319479c59630SNeel Natu 		vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason();
319579c59630SNeel Natu 		vmexit->u.vmx.exit_qualification = vmcs_exit_qualification();
319679c59630SNeel Natu 
31972ce12423SNeel Natu 		/* Update 'nextrip' */
31981aa51504SJohn Baldwin 		vcpu->state.nextrip = rip;
31992ce12423SNeel Natu 
32000492757cSNeel Natu 		if (rc == VMX_GUEST_VMEXIT) {
3201869c8d19SJohn Baldwin 			vmx_exit_handle_nmi(vcpu, vmexit);
320262fbd7c2SNeel Natu 			enable_intr();
32030492757cSNeel Natu 			handled = vmx_exit_process(vmx, vcpu, vmexit);
32040492757cSNeel Natu 		} else {
320562fbd7c2SNeel Natu 			enable_intr();
320640487465SNeel Natu 			vmx_exit_inst_error(vmxctx, rc, vmexit);
3207eeefa4e4SNeel Natu 		}
320862fbd7c2SNeel Natu 		launched = 1;
3209869c8d19SJohn Baldwin 		vmx_exit_trace(vcpu, rip, exit_reason, handled);
32102ce12423SNeel Natu 		rip = vmexit->rip;
3211eeefa4e4SNeel Natu 	} while (handled);
3212366f6083SPeter Grehan 
3213366f6083SPeter Grehan 	/*
3214366f6083SPeter Grehan 	 * If a VM exit has been handled then the exitcode must be BOGUS
3215366f6083SPeter Grehan 	 * If a VM exit is not handled then the exitcode must not be BOGUS
3216366f6083SPeter Grehan 	 */
3217366f6083SPeter Grehan 	if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) ||
3218366f6083SPeter Grehan 	    (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) {
3219366f6083SPeter Grehan 		panic("Mismatch between handled (%d) and exitcode (%d)",
3220366f6083SPeter Grehan 		      handled, vmexit->exitcode);
3221366f6083SPeter Grehan 	}
3222366f6083SPeter Grehan 
322357e0119eSJohn Baldwin 	VMX_CTR1(vcpu, "returning from vmx_run: exitcode %d",
32240492757cSNeel Natu 	    vmexit->exitcode);
3225366f6083SPeter Grehan 
3226366f6083SPeter Grehan 	VMCLEAR(vmcs);
322780cb5d84SJohn Baldwin 	vmx_msr_guest_exit(vcpu);
3228c3498942SNeel Natu 
3229366f6083SPeter Grehan 	return (0);
3230366f6083SPeter Grehan }
3231366f6083SPeter Grehan 
3232366f6083SPeter Grehan static void
3233869c8d19SJohn Baldwin vmx_vcpu_cleanup(void *vcpui)
3234366f6083SPeter Grehan {
32351aa51504SJohn Baldwin 	struct vmx_vcpu *vcpu = vcpui;
3236366f6083SPeter Grehan 
32370f00260cSJohn Baldwin 	vpid_free(vcpu->state.vpid);
32380f00260cSJohn Baldwin 	free(vcpu->pir_desc, M_VMX);
32390f00260cSJohn Baldwin 	free(vcpu->apic_page, M_VMX);
32400f00260cSJohn Baldwin 	free(vcpu->vmcs, M_VMX);
32411aa51504SJohn Baldwin 	free(vcpu, M_VMX);
32420f00260cSJohn Baldwin }
324345e51299SNeel Natu 
32441aa51504SJohn Baldwin static void
3245869c8d19SJohn Baldwin vmx_cleanup(void *vmi)
32461aa51504SJohn Baldwin {
3247869c8d19SJohn Baldwin 	struct vmx *vmx = vmi;
32481aa51504SJohn Baldwin 
32491aa51504SJohn Baldwin 	if (virtual_interrupt_delivery)
32501aa51504SJohn Baldwin 		vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
32511aa51504SJohn Baldwin 
32520f00260cSJohn Baldwin 	free(vmx->msr_bitmap, M_VMX);
3253366f6083SPeter Grehan 	free(vmx, M_VMX);
3254366f6083SPeter Grehan 
3255366f6083SPeter Grehan 	return;
3256366f6083SPeter Grehan }
3257366f6083SPeter Grehan 
3258366f6083SPeter Grehan static register_t *
3259366f6083SPeter Grehan vmxctx_regptr(struct vmxctx *vmxctx, int reg)
3260366f6083SPeter Grehan {
3261366f6083SPeter Grehan 
3262366f6083SPeter Grehan 	switch (reg) {
3263366f6083SPeter Grehan 	case VM_REG_GUEST_RAX:
3264366f6083SPeter Grehan 		return (&vmxctx->guest_rax);
3265366f6083SPeter Grehan 	case VM_REG_GUEST_RBX:
3266366f6083SPeter Grehan 		return (&vmxctx->guest_rbx);
3267366f6083SPeter Grehan 	case VM_REG_GUEST_RCX:
3268366f6083SPeter Grehan 		return (&vmxctx->guest_rcx);
3269366f6083SPeter Grehan 	case VM_REG_GUEST_RDX:
3270366f6083SPeter Grehan 		return (&vmxctx->guest_rdx);
3271366f6083SPeter Grehan 	case VM_REG_GUEST_RSI:
3272366f6083SPeter Grehan 		return (&vmxctx->guest_rsi);
3273366f6083SPeter Grehan 	case VM_REG_GUEST_RDI:
3274366f6083SPeter Grehan 		return (&vmxctx->guest_rdi);
3275366f6083SPeter Grehan 	case VM_REG_GUEST_RBP:
3276366f6083SPeter Grehan 		return (&vmxctx->guest_rbp);
3277366f6083SPeter Grehan 	case VM_REG_GUEST_R8:
3278366f6083SPeter Grehan 		return (&vmxctx->guest_r8);
3279366f6083SPeter Grehan 	case VM_REG_GUEST_R9:
3280366f6083SPeter Grehan 		return (&vmxctx->guest_r9);
3281366f6083SPeter Grehan 	case VM_REG_GUEST_R10:
3282366f6083SPeter Grehan 		return (&vmxctx->guest_r10);
3283366f6083SPeter Grehan 	case VM_REG_GUEST_R11:
3284366f6083SPeter Grehan 		return (&vmxctx->guest_r11);
3285366f6083SPeter Grehan 	case VM_REG_GUEST_R12:
3286366f6083SPeter Grehan 		return (&vmxctx->guest_r12);
3287366f6083SPeter Grehan 	case VM_REG_GUEST_R13:
3288366f6083SPeter Grehan 		return (&vmxctx->guest_r13);
3289366f6083SPeter Grehan 	case VM_REG_GUEST_R14:
3290366f6083SPeter Grehan 		return (&vmxctx->guest_r14);
3291366f6083SPeter Grehan 	case VM_REG_GUEST_R15:
3292366f6083SPeter Grehan 		return (&vmxctx->guest_r15);
329337a723a5SNeel Natu 	case VM_REG_GUEST_CR2:
329437a723a5SNeel Natu 		return (&vmxctx->guest_cr2);
329565eefbe4SJohn Baldwin 	case VM_REG_GUEST_DR0:
329665eefbe4SJohn Baldwin 		return (&vmxctx->guest_dr0);
329765eefbe4SJohn Baldwin 	case VM_REG_GUEST_DR1:
329865eefbe4SJohn Baldwin 		return (&vmxctx->guest_dr1);
329965eefbe4SJohn Baldwin 	case VM_REG_GUEST_DR2:
330065eefbe4SJohn Baldwin 		return (&vmxctx->guest_dr2);
330165eefbe4SJohn Baldwin 	case VM_REG_GUEST_DR3:
330265eefbe4SJohn Baldwin 		return (&vmxctx->guest_dr3);
330365eefbe4SJohn Baldwin 	case VM_REG_GUEST_DR6:
330465eefbe4SJohn Baldwin 		return (&vmxctx->guest_dr6);
3305366f6083SPeter Grehan 	default:
3306366f6083SPeter Grehan 		break;
3307366f6083SPeter Grehan 	}
3308366f6083SPeter Grehan 	return (NULL);
3309366f6083SPeter Grehan }
3310366f6083SPeter Grehan 
3311366f6083SPeter Grehan static int
3312366f6083SPeter Grehan vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval)
3313366f6083SPeter Grehan {
3314366f6083SPeter Grehan 	register_t *regp;
3315366f6083SPeter Grehan 
3316366f6083SPeter Grehan 	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
3317366f6083SPeter Grehan 		*retval = *regp;
3318366f6083SPeter Grehan 		return (0);
3319366f6083SPeter Grehan 	} else
3320366f6083SPeter Grehan 		return (EINVAL);
3321366f6083SPeter Grehan }
3322366f6083SPeter Grehan 
3323366f6083SPeter Grehan static int
3324366f6083SPeter Grehan vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val)
3325366f6083SPeter Grehan {
3326366f6083SPeter Grehan 	register_t *regp;
3327366f6083SPeter Grehan 
3328366f6083SPeter Grehan 	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
3329366f6083SPeter Grehan 		*regp = val;
3330366f6083SPeter Grehan 		return (0);
3331366f6083SPeter Grehan 	} else
3332366f6083SPeter Grehan 		return (EINVAL);
3333366f6083SPeter Grehan }
3334366f6083SPeter Grehan 
3335366f6083SPeter Grehan static int
33361aa51504SJohn Baldwin vmx_get_intr_shadow(struct vmx_vcpu *vcpu, int running, uint64_t *retval)
3337d1819632SNeel Natu {
3338d1819632SNeel Natu 	uint64_t gi;
3339d1819632SNeel Natu 	int error;
3340d1819632SNeel Natu 
33411aa51504SJohn Baldwin 	error = vmcs_getreg(vcpu->vmcs, running,
3342d1819632SNeel Natu 	    VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY), &gi);
3343d1819632SNeel Natu 	*retval = (gi & HWINTR_BLOCKING) ? 1 : 0;
3344d1819632SNeel Natu 	return (error);
3345d1819632SNeel Natu }
3346d1819632SNeel Natu 
3347d1819632SNeel Natu static int
3348869c8d19SJohn Baldwin vmx_modify_intr_shadow(struct vmx_vcpu *vcpu, int running, uint64_t val)
3349d1819632SNeel Natu {
3350d1819632SNeel Natu 	struct vmcs *vmcs;
3351d1819632SNeel Natu 	uint64_t gi;
3352d1819632SNeel Natu 	int error, ident;
3353d1819632SNeel Natu 
3354d1819632SNeel Natu 	/*
3355d1819632SNeel Natu 	 * Forcing the vcpu into an interrupt shadow is not supported.
3356d1819632SNeel Natu 	 */
3357d1819632SNeel Natu 	if (val) {
3358d1819632SNeel Natu 		error = EINVAL;
3359d1819632SNeel Natu 		goto done;
3360d1819632SNeel Natu 	}
3361d1819632SNeel Natu 
33621aa51504SJohn Baldwin 	vmcs = vcpu->vmcs;
3363d1819632SNeel Natu 	ident = VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY);
3364d1819632SNeel Natu 	error = vmcs_getreg(vmcs, running, ident, &gi);
3365d1819632SNeel Natu 	if (error == 0) {
3366d1819632SNeel Natu 		gi &= ~HWINTR_BLOCKING;
3367d1819632SNeel Natu 		error = vmcs_setreg(vmcs, running, ident, gi);
3368d1819632SNeel Natu 	}
3369d1819632SNeel Natu done:
337057e0119eSJohn Baldwin 	VMX_CTR2(vcpu, "Setting intr_shadow to %#lx %s", val,
337157e0119eSJohn Baldwin 	    error ? "failed" : "succeeded");
3372d1819632SNeel Natu 	return (error);
3373d1819632SNeel Natu }
3374d1819632SNeel Natu 
3375d1819632SNeel Natu static int
3376aaaa0656SPeter Grehan vmx_shadow_reg(int reg)
3377aaaa0656SPeter Grehan {
3378aaaa0656SPeter Grehan 	int shreg;
3379aaaa0656SPeter Grehan 
3380aaaa0656SPeter Grehan 	shreg = -1;
3381aaaa0656SPeter Grehan 
3382aaaa0656SPeter Grehan 	switch (reg) {
3383aaaa0656SPeter Grehan 	case VM_REG_GUEST_CR0:
3384aaaa0656SPeter Grehan 		shreg = VMCS_CR0_SHADOW;
3385aaaa0656SPeter Grehan 		break;
3386aaaa0656SPeter Grehan 	case VM_REG_GUEST_CR4:
3387aaaa0656SPeter Grehan 		shreg = VMCS_CR4_SHADOW;
3388aaaa0656SPeter Grehan 		break;
3389aaaa0656SPeter Grehan 	default:
3390aaaa0656SPeter Grehan 		break;
3391aaaa0656SPeter Grehan 	}
3392aaaa0656SPeter Grehan 
3393aaaa0656SPeter Grehan 	return (shreg);
3394aaaa0656SPeter Grehan }
3395aaaa0656SPeter Grehan 
3396aaaa0656SPeter Grehan static int
3397869c8d19SJohn Baldwin vmx_getreg(void *vcpui, int reg, uint64_t *retval)
3398366f6083SPeter Grehan {
3399d3c11f40SPeter Grehan 	int running, hostcpu;
34001aa51504SJohn Baldwin 	struct vmx_vcpu *vcpu = vcpui;
3401869c8d19SJohn Baldwin 	struct vmx *vmx = vcpu->vmx;
3402366f6083SPeter Grehan 
340380cb5d84SJohn Baldwin 	running = vcpu_is_running(vcpu->vcpu, &hostcpu);
3404d3c11f40SPeter Grehan 	if (running && hostcpu != curcpu)
34051aa51504SJohn Baldwin 		panic("vmx_getreg: %s%d is running", vm_name(vmx->vm),
34061aa51504SJohn Baldwin 		    vcpu->vcpuid);
3407d3c11f40SPeter Grehan 
3408f493ea65SMark Johnston 	switch (reg) {
3409f493ea65SMark Johnston 	case VM_REG_GUEST_INTR_SHADOW:
34101aa51504SJohn Baldwin 		return (vmx_get_intr_shadow(vcpu, running, retval));
3411f493ea65SMark Johnston 	case VM_REG_GUEST_KGS_BASE:
3412f493ea65SMark Johnston 		*retval = vcpu->guest_msrs[IDX_MSR_KGSBASE];
3413f493ea65SMark Johnston 		return (0);
3414f493ea65SMark Johnston 	case VM_REG_GUEST_TPR:
3415f493ea65SMark Johnston 		*retval = vlapic_get_cr8(vm_lapic(vcpu->vcpu));
3416f493ea65SMark Johnston 		return (0);
3417f493ea65SMark Johnston 	}
3418d1819632SNeel Natu 
34191aa51504SJohn Baldwin 	if (vmxctx_getreg(&vcpu->ctx, reg, retval) == 0)
3420366f6083SPeter Grehan 		return (0);
3421366f6083SPeter Grehan 
34221aa51504SJohn Baldwin 	return (vmcs_getreg(vcpu->vmcs, running, reg, retval));
3423366f6083SPeter Grehan }
3424366f6083SPeter Grehan 
3425366f6083SPeter Grehan static int
3426869c8d19SJohn Baldwin vmx_setreg(void *vcpui, int reg, uint64_t val)
3427366f6083SPeter Grehan {
3428aaaa0656SPeter Grehan 	int error, hostcpu, running, shadow;
3429366f6083SPeter Grehan 	uint64_t ctls;
34303527963bSNeel Natu 	pmap_t pmap;
34311aa51504SJohn Baldwin 	struct vmx_vcpu *vcpu = vcpui;
3432869c8d19SJohn Baldwin 	struct vmx *vmx = vcpu->vmx;
3433366f6083SPeter Grehan 
343480cb5d84SJohn Baldwin 	running = vcpu_is_running(vcpu->vcpu, &hostcpu);
3435d3c11f40SPeter Grehan 	if (running && hostcpu != curcpu)
34361aa51504SJohn Baldwin 		panic("vmx_setreg: %s%d is running", vm_name(vmx->vm),
34371aa51504SJohn Baldwin 		    vcpu->vcpuid);
3438d3c11f40SPeter Grehan 
3439d1819632SNeel Natu 	if (reg == VM_REG_GUEST_INTR_SHADOW)
3440869c8d19SJohn Baldwin 		return (vmx_modify_intr_shadow(vcpu, running, val));
3441d1819632SNeel Natu 
34421aa51504SJohn Baldwin 	if (vmxctx_setreg(&vcpu->ctx, reg, val) == 0)
3443366f6083SPeter Grehan 		return (0);
3444366f6083SPeter Grehan 
344509860d44SEd Maste 	/* Do not permit user write access to VMCS fields by offset. */
344609860d44SEd Maste 	if (reg < 0)
344709860d44SEd Maste 		return (EINVAL);
344809860d44SEd Maste 
34491aa51504SJohn Baldwin 	error = vmcs_setreg(vcpu->vmcs, running, reg, val);
3450366f6083SPeter Grehan 
3451366f6083SPeter Grehan 	if (error == 0) {
3452366f6083SPeter Grehan 		/*
3453366f6083SPeter Grehan 		 * If the "load EFER" VM-entry control is 1 then the
3454366f6083SPeter Grehan 		 * value of EFER.LMA must be identical to "IA-32e mode guest"
3455366f6083SPeter Grehan 		 * bit in the VM-entry control.
3456366f6083SPeter Grehan 		 */
3457366f6083SPeter Grehan 		if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 &&
3458366f6083SPeter Grehan 		    (reg == VM_REG_GUEST_EFER)) {
34591aa51504SJohn Baldwin 			vmcs_getreg(vcpu->vmcs, running,
3460366f6083SPeter Grehan 				    VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls);
3461366f6083SPeter Grehan 			if (val & EFER_LMA)
3462366f6083SPeter Grehan 				ctls |= VM_ENTRY_GUEST_LMA;
3463366f6083SPeter Grehan 			else
3464366f6083SPeter Grehan 				ctls &= ~VM_ENTRY_GUEST_LMA;
34651aa51504SJohn Baldwin 			vmcs_setreg(vcpu->vmcs, running,
3466366f6083SPeter Grehan 				    VMCS_IDENT(VMCS_ENTRY_CTLS), ctls);
3467366f6083SPeter Grehan 		}
3468aaaa0656SPeter Grehan 
3469aaaa0656SPeter Grehan 		shadow = vmx_shadow_reg(reg);
3470aaaa0656SPeter Grehan 		if (shadow > 0) {
3471aaaa0656SPeter Grehan 			/*
3472aaaa0656SPeter Grehan 			 * Store the unmodified value in the shadow
3473aaaa0656SPeter Grehan 			 */
34741aa51504SJohn Baldwin 			error = vmcs_setreg(vcpu->vmcs, running,
3475aaaa0656SPeter Grehan 				    VMCS_IDENT(shadow), val);
3476aaaa0656SPeter Grehan 		}
34773527963bSNeel Natu 
34783527963bSNeel Natu 		if (reg == VM_REG_GUEST_CR3) {
34793527963bSNeel Natu 			/*
34803527963bSNeel Natu 			 * Invalidate the guest vcpu's TLB mappings to emulate
34813527963bSNeel Natu 			 * the behavior of updating %cr3.
34823527963bSNeel Natu 			 *
34833527963bSNeel Natu 			 * XXX the processor retains global mappings when %cr3
34843527963bSNeel Natu 			 * is updated but vmx_invvpid() does not.
34853527963bSNeel Natu 			 */
34861aa51504SJohn Baldwin 			pmap = vcpu->ctx.pmap;
34873527963bSNeel Natu 			vmx_invvpid(vmx, vcpu, pmap, running);
34883527963bSNeel Natu 		}
3489366f6083SPeter Grehan 	}
3490366f6083SPeter Grehan 
3491366f6083SPeter Grehan 	return (error);
3492366f6083SPeter Grehan }
3493366f6083SPeter Grehan 
3494366f6083SPeter Grehan static int
3495869c8d19SJohn Baldwin vmx_getdesc(void *vcpui, int reg, struct seg_desc *desc)
3496366f6083SPeter Grehan {
3497ba6f5e23SNeel Natu 	int hostcpu, running;
34981aa51504SJohn Baldwin 	struct vmx_vcpu *vcpu = vcpui;
3499869c8d19SJohn Baldwin 	struct vmx *vmx = vcpu->vmx;
3500366f6083SPeter Grehan 
350180cb5d84SJohn Baldwin 	running = vcpu_is_running(vcpu->vcpu, &hostcpu);
3502ba6f5e23SNeel Natu 	if (running && hostcpu != curcpu)
35031aa51504SJohn Baldwin 		panic("vmx_getdesc: %s%d is running", vm_name(vmx->vm),
35041aa51504SJohn Baldwin 		    vcpu->vcpuid);
3505ba6f5e23SNeel Natu 
35061aa51504SJohn Baldwin 	return (vmcs_getdesc(vcpu->vmcs, running, reg, desc));
3507366f6083SPeter Grehan }
3508366f6083SPeter Grehan 
3509366f6083SPeter Grehan static int
3510869c8d19SJohn Baldwin vmx_setdesc(void *vcpui, int reg, struct seg_desc *desc)
3511366f6083SPeter Grehan {
3512ba6f5e23SNeel Natu 	int hostcpu, running;
35131aa51504SJohn Baldwin 	struct vmx_vcpu *vcpu = vcpui;
3514869c8d19SJohn Baldwin 	struct vmx *vmx = vcpu->vmx;
3515366f6083SPeter Grehan 
351680cb5d84SJohn Baldwin 	running = vcpu_is_running(vcpu->vcpu, &hostcpu);
3517ba6f5e23SNeel Natu 	if (running && hostcpu != curcpu)
35181aa51504SJohn Baldwin 		panic("vmx_setdesc: %s%d is running", vm_name(vmx->vm),
35191aa51504SJohn Baldwin 		    vcpu->vcpuid);
3520ba6f5e23SNeel Natu 
35211aa51504SJohn Baldwin 	return (vmcs_setdesc(vcpu->vmcs, running, reg, desc));
3522366f6083SPeter Grehan }
3523366f6083SPeter Grehan 
3524366f6083SPeter Grehan static int
3525869c8d19SJohn Baldwin vmx_getcap(void *vcpui, int type, int *retval)
3526366f6083SPeter Grehan {
35271aa51504SJohn Baldwin 	struct vmx_vcpu *vcpu = vcpui;
3528366f6083SPeter Grehan 	int vcap;
3529366f6083SPeter Grehan 	int ret;
3530366f6083SPeter Grehan 
3531366f6083SPeter Grehan 	ret = ENOENT;
3532366f6083SPeter Grehan 
35331aa51504SJohn Baldwin 	vcap = vcpu->cap.set;
3534366f6083SPeter Grehan 
3535366f6083SPeter Grehan 	switch (type) {
3536366f6083SPeter Grehan 	case VM_CAP_HALT_EXIT:
3537366f6083SPeter Grehan 		if (cap_halt_exit)
3538366f6083SPeter Grehan 			ret = 0;
3539366f6083SPeter Grehan 		break;
3540366f6083SPeter Grehan 	case VM_CAP_PAUSE_EXIT:
3541366f6083SPeter Grehan 		if (cap_pause_exit)
3542366f6083SPeter Grehan 			ret = 0;
3543366f6083SPeter Grehan 		break;
3544366f6083SPeter Grehan 	case VM_CAP_MTRAP_EXIT:
3545366f6083SPeter Grehan 		if (cap_monitor_trap)
3546366f6083SPeter Grehan 			ret = 0;
3547366f6083SPeter Grehan 		break;
3548f5f5f1e7SPeter Grehan 	case VM_CAP_RDPID:
3549f5f5f1e7SPeter Grehan 		if (cap_rdpid)
3550f5f5f1e7SPeter Grehan 			ret = 0;
3551f5f5f1e7SPeter Grehan 		break;
3552f5f5f1e7SPeter Grehan 	case VM_CAP_RDTSCP:
3553f5f5f1e7SPeter Grehan 		if (cap_rdtscp)
3554f5f5f1e7SPeter Grehan 			ret = 0;
3555f5f5f1e7SPeter Grehan 		break;
3556366f6083SPeter Grehan 	case VM_CAP_UNRESTRICTED_GUEST:
3557366f6083SPeter Grehan 		if (cap_unrestricted_guest)
3558366f6083SPeter Grehan 			ret = 0;
3559366f6083SPeter Grehan 		break;
356049cc03daSNeel Natu 	case VM_CAP_ENABLE_INVPCID:
356149cc03daSNeel Natu 		if (cap_invpcid)
356249cc03daSNeel Natu 			ret = 0;
356349cc03daSNeel Natu 		break;
3564cbd03a9dSJohn Baldwin 	case VM_CAP_BPT_EXIT:
35650bda8d3eSCorvin Köhne 	case VM_CAP_IPI_EXIT:
3566cbd03a9dSJohn Baldwin 		ret = 0;
3567cbd03a9dSJohn Baldwin 		break;
3568366f6083SPeter Grehan 	default:
3569366f6083SPeter Grehan 		break;
3570366f6083SPeter Grehan 	}
3571366f6083SPeter Grehan 
3572366f6083SPeter Grehan 	if (ret == 0)
3573366f6083SPeter Grehan 		*retval = (vcap & (1 << type)) ? 1 : 0;
3574366f6083SPeter Grehan 
3575366f6083SPeter Grehan 	return (ret);
3576366f6083SPeter Grehan }
3577366f6083SPeter Grehan 
3578366f6083SPeter Grehan static int
3579869c8d19SJohn Baldwin vmx_setcap(void *vcpui, int type, int val)
3580366f6083SPeter Grehan {
35811aa51504SJohn Baldwin 	struct vmx_vcpu *vcpu = vcpui;
35821aa51504SJohn Baldwin 	struct vmcs *vmcs = vcpu->vmcs;
35830bda8d3eSCorvin Köhne 	struct vlapic *vlapic;
3584366f6083SPeter Grehan 	uint32_t baseval;
3585366f6083SPeter Grehan 	uint32_t *pptr;
3586366f6083SPeter Grehan 	int error;
3587366f6083SPeter Grehan 	int flag;
3588366f6083SPeter Grehan 	int reg;
3589366f6083SPeter Grehan 	int retval;
3590366f6083SPeter Grehan 
3591366f6083SPeter Grehan 	retval = ENOENT;
3592366f6083SPeter Grehan 	pptr = NULL;
3593366f6083SPeter Grehan 
3594366f6083SPeter Grehan 	switch (type) {
3595366f6083SPeter Grehan 	case VM_CAP_HALT_EXIT:
3596366f6083SPeter Grehan 		if (cap_halt_exit) {
3597366f6083SPeter Grehan 			retval = 0;
35981aa51504SJohn Baldwin 			pptr = &vcpu->cap.proc_ctls;
3599366f6083SPeter Grehan 			baseval = *pptr;
3600366f6083SPeter Grehan 			flag = PROCBASED_HLT_EXITING;
3601366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
3602366f6083SPeter Grehan 		}
3603366f6083SPeter Grehan 		break;
3604366f6083SPeter Grehan 	case VM_CAP_MTRAP_EXIT:
3605366f6083SPeter Grehan 		if (cap_monitor_trap) {
3606366f6083SPeter Grehan 			retval = 0;
36071aa51504SJohn Baldwin 			pptr = &vcpu->cap.proc_ctls;
3608366f6083SPeter Grehan 			baseval = *pptr;
3609366f6083SPeter Grehan 			flag = PROCBASED_MTF;
3610366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
3611366f6083SPeter Grehan 		}
3612366f6083SPeter Grehan 		break;
3613366f6083SPeter Grehan 	case VM_CAP_PAUSE_EXIT:
3614366f6083SPeter Grehan 		if (cap_pause_exit) {
3615366f6083SPeter Grehan 			retval = 0;
36161aa51504SJohn Baldwin 			pptr = &vcpu->cap.proc_ctls;
3617366f6083SPeter Grehan 			baseval = *pptr;
3618366f6083SPeter Grehan 			flag = PROCBASED_PAUSE_EXITING;
3619366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
3620366f6083SPeter Grehan 		}
3621366f6083SPeter Grehan 		break;
3622f5f5f1e7SPeter Grehan 	case VM_CAP_RDPID:
3623f5f5f1e7SPeter Grehan 	case VM_CAP_RDTSCP:
3624f5f5f1e7SPeter Grehan 		if (cap_rdpid || cap_rdtscp)
3625f5f5f1e7SPeter Grehan 			/*
3626f5f5f1e7SPeter Grehan 			 * Choose not to support enabling/disabling
3627f5f5f1e7SPeter Grehan 			 * RDPID/RDTSCP via libvmmapi since, as per the
362815add60dSPeter Grehan 			 * discussion in vmx_modinit(), RDPID/RDTSCP are
3629f5f5f1e7SPeter Grehan 			 * either always enabled or always disabled.
3630f5f5f1e7SPeter Grehan 			 */
3631f5f5f1e7SPeter Grehan 			error = EOPNOTSUPP;
3632f5f5f1e7SPeter Grehan 		break;
3633366f6083SPeter Grehan 	case VM_CAP_UNRESTRICTED_GUEST:
3634366f6083SPeter Grehan 		if (cap_unrestricted_guest) {
3635366f6083SPeter Grehan 			retval = 0;
36361aa51504SJohn Baldwin 			pptr = &vcpu->cap.proc_ctls2;
363749cc03daSNeel Natu 			baseval = *pptr;
3638366f6083SPeter Grehan 			flag = PROCBASED2_UNRESTRICTED_GUEST;
3639366f6083SPeter Grehan 			reg = VMCS_SEC_PROC_BASED_CTLS;
3640366f6083SPeter Grehan 		}
3641366f6083SPeter Grehan 		break;
364249cc03daSNeel Natu 	case VM_CAP_ENABLE_INVPCID:
364349cc03daSNeel Natu 		if (cap_invpcid) {
364449cc03daSNeel Natu 			retval = 0;
36451aa51504SJohn Baldwin 			pptr = &vcpu->cap.proc_ctls2;
364649cc03daSNeel Natu 			baseval = *pptr;
364749cc03daSNeel Natu 			flag = PROCBASED2_ENABLE_INVPCID;
364849cc03daSNeel Natu 			reg = VMCS_SEC_PROC_BASED_CTLS;
364949cc03daSNeel Natu 		}
365049cc03daSNeel Natu 		break;
3651cbd03a9dSJohn Baldwin 	case VM_CAP_BPT_EXIT:
3652cbd03a9dSJohn Baldwin 		retval = 0;
3653cbd03a9dSJohn Baldwin 
3654cbd03a9dSJohn Baldwin 		/* Don't change the bitmap if we are tracing all exceptions. */
36551aa51504SJohn Baldwin 		if (vcpu->cap.exc_bitmap != 0xffffffff) {
36561aa51504SJohn Baldwin 			pptr = &vcpu->cap.exc_bitmap;
3657cbd03a9dSJohn Baldwin 			baseval = *pptr;
3658cbd03a9dSJohn Baldwin 			flag = (1 << IDT_BP);
3659cbd03a9dSJohn Baldwin 			reg = VMCS_EXCEPTION_BITMAP;
3660cbd03a9dSJohn Baldwin 		}
3661cbd03a9dSJohn Baldwin 		break;
36620bda8d3eSCorvin Köhne 	case VM_CAP_IPI_EXIT:
36630bda8d3eSCorvin Köhne 		retval = 0;
36640bda8d3eSCorvin Köhne 
3665d3956e46SJohn Baldwin 		vlapic = vm_lapic(vcpu->vcpu);
36660bda8d3eSCorvin Köhne 		vlapic->ipi_exit = val;
36670bda8d3eSCorvin Köhne 		break;
3668fefac543SBojan Novković 	case VM_CAP_MASK_HWINTR:
3669fefac543SBojan Novković 		retval = 0;
3670fefac543SBojan Novković 		break;
3671366f6083SPeter Grehan 	default:
3672366f6083SPeter Grehan 		break;
3673366f6083SPeter Grehan 	}
3674366f6083SPeter Grehan 
3675cbd03a9dSJohn Baldwin 	if (retval)
3676cbd03a9dSJohn Baldwin 		return (retval);
3677cbd03a9dSJohn Baldwin 
3678cbd03a9dSJohn Baldwin 	if (pptr != NULL) {
3679366f6083SPeter Grehan 		if (val) {
3680366f6083SPeter Grehan 			baseval |= flag;
3681366f6083SPeter Grehan 		} else {
3682366f6083SPeter Grehan 			baseval &= ~flag;
3683366f6083SPeter Grehan 		}
3684366f6083SPeter Grehan 		VMPTRLD(vmcs);
3685366f6083SPeter Grehan 		error = vmwrite(reg, baseval);
3686366f6083SPeter Grehan 		VMCLEAR(vmcs);
3687366f6083SPeter Grehan 
3688cbd03a9dSJohn Baldwin 		if (error)
3689cbd03a9dSJohn Baldwin 			return (error);
3690cbd03a9dSJohn Baldwin 
3691366f6083SPeter Grehan 		/*
3692366f6083SPeter Grehan 		 * Update optional stored flags, and record
3693366f6083SPeter Grehan 		 * setting
3694366f6083SPeter Grehan 		 */
3695366f6083SPeter Grehan 		*pptr = baseval;
3696366f6083SPeter Grehan 	}
3697366f6083SPeter Grehan 
3698366f6083SPeter Grehan 	if (val) {
36991aa51504SJohn Baldwin 		vcpu->cap.set |= (1 << type);
3700366f6083SPeter Grehan 	} else {
37011aa51504SJohn Baldwin 		vcpu->cap.set &= ~(1 << type);
3702366f6083SPeter Grehan 	}
3703366f6083SPeter Grehan 
3704cbd03a9dSJohn Baldwin 	return (0);
3705366f6083SPeter Grehan }
3706366f6083SPeter Grehan 
370715add60dSPeter Grehan static struct vmspace *
370815add60dSPeter Grehan vmx_vmspace_alloc(vm_offset_t min, vm_offset_t max)
370915add60dSPeter Grehan {
371015add60dSPeter Grehan 	return (ept_vmspace_alloc(min, max));
371115add60dSPeter Grehan }
371215add60dSPeter Grehan 
371315add60dSPeter Grehan static void
371415add60dSPeter Grehan vmx_vmspace_free(struct vmspace *vmspace)
371515add60dSPeter Grehan {
371615add60dSPeter Grehan 	ept_vmspace_free(vmspace);
371715add60dSPeter Grehan }
371815add60dSPeter Grehan 
371988c4b8d1SNeel Natu struct vlapic_vtx {
372088c4b8d1SNeel Natu 	struct vlapic	vlapic;
3721176666c2SNeel Natu 	struct pir_desc	*pir_desc;
37221aa51504SJohn Baldwin 	struct vmx_vcpu	*vcpu;
37232c352febSJohn Baldwin 	u_int	pending_prio;
372488c4b8d1SNeel Natu };
372588c4b8d1SNeel Natu 
37262c352febSJohn Baldwin #define VPR_PRIO_BIT(vpr)	(1 << ((vpr) >> 4))
37272c352febSJohn Baldwin 
3728d030f941SJohn Baldwin #define	VMX_CTR_PIR(vlapic, pir_desc, notify, vector, level, msg)	\
372988c4b8d1SNeel Natu do {									\
3730d030f941SJohn Baldwin 	VLAPIC_CTR2(vlapic, msg " assert %s-triggered vector %d",	\
373188c4b8d1SNeel Natu 	    level ? "level" : "edge", vector);				\
3732d030f941SJohn Baldwin 	VLAPIC_CTR1(vlapic, msg " pir0 0x%016lx", pir_desc->pir[0]);	\
3733d030f941SJohn Baldwin 	VLAPIC_CTR1(vlapic, msg " pir1 0x%016lx", pir_desc->pir[1]);	\
3734d030f941SJohn Baldwin 	VLAPIC_CTR1(vlapic, msg " pir2 0x%016lx", pir_desc->pir[2]);	\
3735d030f941SJohn Baldwin 	VLAPIC_CTR1(vlapic, msg " pir3 0x%016lx", pir_desc->pir[3]);	\
3736d030f941SJohn Baldwin 	VLAPIC_CTR1(vlapic, msg " notify: %s", notify ? "yes" : "no");	\
373788c4b8d1SNeel Natu } while (0)
373888c4b8d1SNeel Natu 
373988c4b8d1SNeel Natu /*
374088c4b8d1SNeel Natu  * vlapic->ops handlers that utilize the APICv hardware assist described in
374188c4b8d1SNeel Natu  * Chapter 29 of the Intel SDM.
374288c4b8d1SNeel Natu  */
374388c4b8d1SNeel Natu static int
374488c4b8d1SNeel Natu vmx_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
374588c4b8d1SNeel Natu {
374688c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
374788c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
374888c4b8d1SNeel Natu 	uint64_t mask;
37492c352febSJohn Baldwin 	int idx, notify = 0;
375088c4b8d1SNeel Natu 
375188c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
3752176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
375388c4b8d1SNeel Natu 
375488c4b8d1SNeel Natu 	/*
375588c4b8d1SNeel Natu 	 * Keep track of interrupt requests in the PIR descriptor. This is
375688c4b8d1SNeel Natu 	 * because the virtual APIC page pointed to by the VMCS cannot be
375788c4b8d1SNeel Natu 	 * modified if the vcpu is running.
375888c4b8d1SNeel Natu 	 */
375988c4b8d1SNeel Natu 	idx = vector / 64;
376088c4b8d1SNeel Natu 	mask = 1UL << (vector % 64);
376188c4b8d1SNeel Natu 	atomic_set_long(&pir_desc->pir[idx], mask);
37622c352febSJohn Baldwin 
37632c352febSJohn Baldwin 	/*
37642c352febSJohn Baldwin 	 * A notification is required whenever the 'pending' bit makes a
37652c352febSJohn Baldwin 	 * transition from 0->1.
37662c352febSJohn Baldwin 	 *
37672c352febSJohn Baldwin 	 * Even if the 'pending' bit is already asserted, notification about
37682c352febSJohn Baldwin 	 * the incoming interrupt may still be necessary.  For example, if a
37692c352febSJohn Baldwin 	 * vCPU is HLTed with a high PPR, a low priority interrupt would cause
37702c352febSJohn Baldwin 	 * the 0->1 'pending' transition with a notification, but the vCPU
37712c352febSJohn Baldwin 	 * would ignore the interrupt for the time being.  The same vCPU would
37722c352febSJohn Baldwin 	 * need to then be notified if a high-priority interrupt arrived which
37732c352febSJohn Baldwin 	 * satisfied the PPR.
37742c352febSJohn Baldwin 	 *
37752c352febSJohn Baldwin 	 * The priorities of interrupts injected while 'pending' is asserted
37762c352febSJohn Baldwin 	 * are tracked in a custom bitfield 'pending_prio'.  Should the
37772c352febSJohn Baldwin 	 * to-be-injected interrupt exceed the priorities already present, the
37782c352febSJohn Baldwin 	 * notification is sent.  The priorities recorded in 'pending_prio' are
37792c352febSJohn Baldwin 	 * cleared whenever the 'pending' bit makes another 0->1 transition.
37802c352febSJohn Baldwin 	 */
37812c352febSJohn Baldwin 	if (atomic_cmpset_long(&pir_desc->pending, 0, 1) != 0) {
37822c352febSJohn Baldwin 		notify = 1;
37832c352febSJohn Baldwin 		vlapic_vtx->pending_prio = 0;
37842c352febSJohn Baldwin 	} else {
37852c352febSJohn Baldwin 		const u_int old_prio = vlapic_vtx->pending_prio;
37862c352febSJohn Baldwin 		const u_int prio_bit = VPR_PRIO_BIT(vector & APIC_TPR_INT);
37872c352febSJohn Baldwin 
37882c352febSJohn Baldwin 		if ((old_prio & prio_bit) == 0 && prio_bit > old_prio) {
37892c352febSJohn Baldwin 			atomic_set_int(&vlapic_vtx->pending_prio, prio_bit);
37902c352febSJohn Baldwin 			notify = 1;
37912c352febSJohn Baldwin 		}
37922c352febSJohn Baldwin 	}
379388c4b8d1SNeel Natu 
3794d030f941SJohn Baldwin 	VMX_CTR_PIR(vlapic, pir_desc, notify, vector, level,
3795d030f941SJohn Baldwin 	    "vmx_set_intr_ready");
379688c4b8d1SNeel Natu 	return (notify);
379788c4b8d1SNeel Natu }
379888c4b8d1SNeel Natu 
379988c4b8d1SNeel Natu static int
380088c4b8d1SNeel Natu vmx_pending_intr(struct vlapic *vlapic, int *vecptr)
380188c4b8d1SNeel Natu {
380288c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
380388c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
380488c4b8d1SNeel Natu 	struct LAPIC *lapic;
380588c4b8d1SNeel Natu 	uint64_t pending, pirval;
38060912408aSVitaliy Gusev 	uint8_t ppr, vpr, rvi;
38070912408aSVitaliy Gusev 	struct vm_exit *vmexit;
380888c4b8d1SNeel Natu 	int i;
380988c4b8d1SNeel Natu 
381088c4b8d1SNeel Natu 	/*
381188c4b8d1SNeel Natu 	 * This function is only expected to be called from the 'HLT' exit
381288c4b8d1SNeel Natu 	 * handler which does not care about the vector that is pending.
381388c4b8d1SNeel Natu 	 */
381488c4b8d1SNeel Natu 	KASSERT(vecptr == NULL, ("vmx_pending_intr: vecptr must be NULL"));
381588c4b8d1SNeel Natu 
381688c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
3817176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
38180912408aSVitaliy Gusev 	lapic = vlapic->apic_page;
381988c4b8d1SNeel Natu 
38209e33a616STycho Nightingale 	/*
38219e33a616STycho Nightingale 	 * While a virtual interrupt may have already been
38229e33a616STycho Nightingale 	 * processed the actual delivery maybe pending the
38239e33a616STycho Nightingale 	 * interruptibility of the guest.  Recognize a pending
38249e33a616STycho Nightingale 	 * interrupt by reevaluating virtual interrupts
38250912408aSVitaliy Gusev 	 * following Section 30.2.1 in the Intel SDM Volume 3.
38269e33a616STycho Nightingale 	 */
382780cb5d84SJohn Baldwin 	vmexit = vm_exitinfo(vlapic->vcpu);
3828490768e2STycho Nightingale 	KASSERT(vmexit->exitcode == VM_EXITCODE_HLT,
3829490768e2STycho Nightingale 	    ("vmx_pending_intr: exitcode not 'HLT'"));
3830490768e2STycho Nightingale 	rvi = vmexit->u.hlt.intr_status & APIC_TPR_INT;
38319e33a616STycho Nightingale 	ppr = lapic->ppr & APIC_TPR_INT;
38320912408aSVitaliy Gusev 	if (rvi > ppr)
38339e33a616STycho Nightingale 		return (1);
38349e33a616STycho Nightingale 
38350912408aSVitaliy Gusev 	pending = atomic_load_acq_long(&pir_desc->pending);
38360912408aSVitaliy Gusev 	if (!pending)
38379e33a616STycho Nightingale 		return (0);
383888c4b8d1SNeel Natu 
383988c4b8d1SNeel Natu 	/*
384088c4b8d1SNeel Natu 	 * If there is an interrupt pending then it will be recognized only
384188c4b8d1SNeel Natu 	 * if its priority is greater than the processor priority.
384288c4b8d1SNeel Natu 	 *
384388c4b8d1SNeel Natu 	 * Special case: if the processor priority is zero then any pending
384488c4b8d1SNeel Natu 	 * interrupt will be recognized.
384588c4b8d1SNeel Natu 	 */
384688c4b8d1SNeel Natu 	if (ppr == 0)
384788c4b8d1SNeel Natu 		return (1);
384888c4b8d1SNeel Natu 
3849d030f941SJohn Baldwin 	VLAPIC_CTR1(vlapic, "HLT with non-zero PPR %d", lapic->ppr);
385088c4b8d1SNeel Natu 
38512c352febSJohn Baldwin 	vpr = 0;
385288c4b8d1SNeel Natu 	for (i = 3; i >= 0; i--) {
385388c4b8d1SNeel Natu 		pirval = pir_desc->pir[i];
385488c4b8d1SNeel Natu 		if (pirval != 0) {
38559e33a616STycho Nightingale 			vpr = (i * 64 + flsl(pirval) - 1) & APIC_TPR_INT;
38562c352febSJohn Baldwin 			break;
385788c4b8d1SNeel Natu 		}
385888c4b8d1SNeel Natu 	}
38592c352febSJohn Baldwin 
38602c352febSJohn Baldwin 	/*
38612c352febSJohn Baldwin 	 * If the highest-priority pending interrupt falls short of the
38622c352febSJohn Baldwin 	 * processor priority of this vCPU, ensure that 'pending_prio' does not
38632c352febSJohn Baldwin 	 * have any stale bits which would preclude a higher-priority interrupt
38642c352febSJohn Baldwin 	 * from incurring a notification later.
38652c352febSJohn Baldwin 	 */
38662c352febSJohn Baldwin 	if (vpr <= ppr) {
38672c352febSJohn Baldwin 		const u_int prio_bit = VPR_PRIO_BIT(vpr);
38682c352febSJohn Baldwin 		const u_int old = vlapic_vtx->pending_prio;
38692c352febSJohn Baldwin 
38702c352febSJohn Baldwin 		if (old > prio_bit && (old & prio_bit) == 0) {
38712c352febSJohn Baldwin 			vlapic_vtx->pending_prio = prio_bit;
38722c352febSJohn Baldwin 		}
387388c4b8d1SNeel Natu 		return (0);
387488c4b8d1SNeel Natu 	}
38752c352febSJohn Baldwin 	return (1);
38762c352febSJohn Baldwin }
387788c4b8d1SNeel Natu 
387888c4b8d1SNeel Natu static void
387988c4b8d1SNeel Natu vmx_intr_accepted(struct vlapic *vlapic, int vector)
388088c4b8d1SNeel Natu {
388188c4b8d1SNeel Natu 
388288c4b8d1SNeel Natu 	panic("vmx_intr_accepted: not expected to be called");
388388c4b8d1SNeel Natu }
388488c4b8d1SNeel Natu 
3885176666c2SNeel Natu static void
388630b94db8SNeel Natu vmx_set_tmr(struct vlapic *vlapic, int vector, bool level)
388730b94db8SNeel Natu {
388830b94db8SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
388930b94db8SNeel Natu 	struct vmcs *vmcs;
389030b94db8SNeel Natu 	uint64_t mask, val;
389130b94db8SNeel Natu 
389230b94db8SNeel Natu 	KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
389380cb5d84SJohn Baldwin 	KASSERT(!vcpu_is_running(vlapic->vcpu, NULL),
389430b94db8SNeel Natu 	    ("vmx_set_tmr: vcpu cannot be running"));
389530b94db8SNeel Natu 
389630b94db8SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
38971aa51504SJohn Baldwin 	vmcs = vlapic_vtx->vcpu->vmcs;
389830b94db8SNeel Natu 	mask = 1UL << (vector % 64);
389930b94db8SNeel Natu 
390030b94db8SNeel Natu 	VMPTRLD(vmcs);
390130b94db8SNeel Natu 	val = vmcs_read(VMCS_EOI_EXIT(vector));
390230b94db8SNeel Natu 	if (level)
390330b94db8SNeel Natu 		val |= mask;
390430b94db8SNeel Natu 	else
390530b94db8SNeel Natu 		val &= ~mask;
390630b94db8SNeel Natu 	vmcs_write(VMCS_EOI_EXIT(vector), val);
390730b94db8SNeel Natu 	VMCLEAR(vmcs);
390830b94db8SNeel Natu }
390930b94db8SNeel Natu 
391030b94db8SNeel Natu static void
39111bc51badSMichael Reifenberger vmx_enable_x2apic_mode_ts(struct vlapic *vlapic)
39121bc51badSMichael Reifenberger {
39131aa51504SJohn Baldwin 	struct vlapic_vtx *vlapic_vtx;
39140f00260cSJohn Baldwin 	struct vmx_vcpu *vcpu;
39151bc51badSMichael Reifenberger 	struct vmcs *vmcs;
39161bc51badSMichael Reifenberger 	uint32_t proc_ctls;
39171bc51badSMichael Reifenberger 
39181aa51504SJohn Baldwin 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
39191aa51504SJohn Baldwin 	vcpu = vlapic_vtx->vcpu;
39200f00260cSJohn Baldwin 	vmcs = vcpu->vmcs;
39211bc51badSMichael Reifenberger 
39220f00260cSJohn Baldwin 	proc_ctls = vcpu->cap.proc_ctls;
39231bc51badSMichael Reifenberger 	proc_ctls &= ~PROCBASED_USE_TPR_SHADOW;
39241bc51badSMichael Reifenberger 	proc_ctls |= PROCBASED_CR8_LOAD_EXITING;
39251bc51badSMichael Reifenberger 	proc_ctls |= PROCBASED_CR8_STORE_EXITING;
39260f00260cSJohn Baldwin 	vcpu->cap.proc_ctls = proc_ctls;
39271bc51badSMichael Reifenberger 
39281bc51badSMichael Reifenberger 	VMPTRLD(vmcs);
39291bc51badSMichael Reifenberger 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, proc_ctls);
39301bc51badSMichael Reifenberger 	VMCLEAR(vmcs);
39311bc51badSMichael Reifenberger }
39321bc51badSMichael Reifenberger 
39331bc51badSMichael Reifenberger static void
39341bc51badSMichael Reifenberger vmx_enable_x2apic_mode_vid(struct vlapic *vlapic)
3935159dd56fSNeel Natu {
39361aa51504SJohn Baldwin 	struct vlapic_vtx *vlapic_vtx;
3937159dd56fSNeel Natu 	struct vmx *vmx;
39380f00260cSJohn Baldwin 	struct vmx_vcpu *vcpu;
3939159dd56fSNeel Natu 	struct vmcs *vmcs;
3940159dd56fSNeel Natu 	uint32_t proc_ctls2;
39411aa51504SJohn Baldwin 	int error __diagused;
3942159dd56fSNeel Natu 
39431aa51504SJohn Baldwin 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
39441aa51504SJohn Baldwin 	vcpu = vlapic_vtx->vcpu;
3945869c8d19SJohn Baldwin 	vmx = vcpu->vmx;
39460f00260cSJohn Baldwin 	vmcs = vcpu->vmcs;
3947159dd56fSNeel Natu 
39480f00260cSJohn Baldwin 	proc_ctls2 = vcpu->cap.proc_ctls2;
3949159dd56fSNeel Natu 	KASSERT((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) != 0,
3950159dd56fSNeel Natu 	    ("%s: invalid proc_ctls2 %#x", __func__, proc_ctls2));
3951159dd56fSNeel Natu 
3952159dd56fSNeel Natu 	proc_ctls2 &= ~PROCBASED2_VIRTUALIZE_APIC_ACCESSES;
3953159dd56fSNeel Natu 	proc_ctls2 |= PROCBASED2_VIRTUALIZE_X2APIC_MODE;
39540f00260cSJohn Baldwin 	vcpu->cap.proc_ctls2 = proc_ctls2;
3955159dd56fSNeel Natu 
3956159dd56fSNeel Natu 	VMPTRLD(vmcs);
3957159dd56fSNeel Natu 	vmcs_write(VMCS_SEC_PROC_BASED_CTLS, proc_ctls2);
3958159dd56fSNeel Natu 	VMCLEAR(vmcs);
3959159dd56fSNeel Natu 
3960159dd56fSNeel Natu 	if (vlapic->vcpuid == 0) {
3961159dd56fSNeel Natu 		/*
3962159dd56fSNeel Natu 		 * The nested page table mappings are shared by all vcpus
3963159dd56fSNeel Natu 		 * so unmap the APIC access page just once.
3964159dd56fSNeel Natu 		 */
3965159dd56fSNeel Natu 		error = vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
3966159dd56fSNeel Natu 		KASSERT(error == 0, ("%s: vm_unmap_mmio error %d",
3967159dd56fSNeel Natu 		    __func__, error));
3968159dd56fSNeel Natu 
3969159dd56fSNeel Natu 		/*
3970159dd56fSNeel Natu 		 * The MSR bitmap is shared by all vcpus so modify it only
3971159dd56fSNeel Natu 		 * once in the context of vcpu 0.
3972159dd56fSNeel Natu 		 */
3973159dd56fSNeel Natu 		error = vmx_allow_x2apic_msrs(vmx);
3974159dd56fSNeel Natu 		KASSERT(error == 0, ("%s: vmx_allow_x2apic_msrs error %d",
3975159dd56fSNeel Natu 		    __func__, error));
3976159dd56fSNeel Natu 	}
3977159dd56fSNeel Natu }
3978159dd56fSNeel Natu 
3979159dd56fSNeel Natu static void
3980176666c2SNeel Natu vmx_post_intr(struct vlapic *vlapic, int hostcpu)
3981176666c2SNeel Natu {
3982176666c2SNeel Natu 
3983176666c2SNeel Natu 	ipi_cpu(hostcpu, pirvec);
3984176666c2SNeel Natu }
3985176666c2SNeel Natu 
398688c4b8d1SNeel Natu /*
398788c4b8d1SNeel Natu  * Transfer the pending interrupts in the PIR descriptor to the IRR
398888c4b8d1SNeel Natu  * in the virtual APIC page.
398988c4b8d1SNeel Natu  */
399088c4b8d1SNeel Natu static void
399188c4b8d1SNeel Natu vmx_inject_pir(struct vlapic *vlapic)
399288c4b8d1SNeel Natu {
399388c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
399488c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
399588c4b8d1SNeel Natu 	struct LAPIC *lapic;
399688c4b8d1SNeel Natu 	uint64_t val, pirval;
39970e30c5c0SWarner Losh 	int rvi, pirbase = -1;
399888c4b8d1SNeel Natu 	uint16_t intr_status_old, intr_status_new;
399988c4b8d1SNeel Natu 
400088c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
4001176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
400288c4b8d1SNeel Natu 	if (atomic_cmpset_long(&pir_desc->pending, 1, 0) == 0) {
4003d030f941SJohn Baldwin 		VLAPIC_CTR0(vlapic, "vmx_inject_pir: "
400488c4b8d1SNeel Natu 		    "no posted interrupt pending");
400588c4b8d1SNeel Natu 		return;
400688c4b8d1SNeel Natu 	}
400788c4b8d1SNeel Natu 
400888c4b8d1SNeel Natu 	pirval = 0;
4009201b1cccSPeter Grehan 	pirbase = -1;
401088c4b8d1SNeel Natu 	lapic = vlapic->apic_page;
401188c4b8d1SNeel Natu 
401288c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[0]);
401388c4b8d1SNeel Natu 	if (val != 0) {
401488c4b8d1SNeel Natu 		lapic->irr0 |= val;
401588c4b8d1SNeel Natu 		lapic->irr1 |= val >> 32;
401688c4b8d1SNeel Natu 		pirbase = 0;
401788c4b8d1SNeel Natu 		pirval = val;
401888c4b8d1SNeel Natu 	}
401988c4b8d1SNeel Natu 
402088c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[1]);
402188c4b8d1SNeel Natu 	if (val != 0) {
402288c4b8d1SNeel Natu 		lapic->irr2 |= val;
402388c4b8d1SNeel Natu 		lapic->irr3 |= val >> 32;
402488c4b8d1SNeel Natu 		pirbase = 64;
402588c4b8d1SNeel Natu 		pirval = val;
402688c4b8d1SNeel Natu 	}
402788c4b8d1SNeel Natu 
402888c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[2]);
402988c4b8d1SNeel Natu 	if (val != 0) {
403088c4b8d1SNeel Natu 		lapic->irr4 |= val;
403188c4b8d1SNeel Natu 		lapic->irr5 |= val >> 32;
403288c4b8d1SNeel Natu 		pirbase = 128;
403388c4b8d1SNeel Natu 		pirval = val;
403488c4b8d1SNeel Natu 	}
403588c4b8d1SNeel Natu 
403688c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[3]);
403788c4b8d1SNeel Natu 	if (val != 0) {
403888c4b8d1SNeel Natu 		lapic->irr6 |= val;
403988c4b8d1SNeel Natu 		lapic->irr7 |= val >> 32;
404088c4b8d1SNeel Natu 		pirbase = 192;
404188c4b8d1SNeel Natu 		pirval = val;
404288c4b8d1SNeel Natu 	}
4043201b1cccSPeter Grehan 
404488c4b8d1SNeel Natu 	VLAPIC_CTR_IRR(vlapic, "vmx_inject_pir");
404588c4b8d1SNeel Natu 
404688c4b8d1SNeel Natu 	/*
404788c4b8d1SNeel Natu 	 * Update RVI so the processor can evaluate pending virtual
404888c4b8d1SNeel Natu 	 * interrupts on VM-entry.
4049201b1cccSPeter Grehan 	 *
4050201b1cccSPeter Grehan 	 * It is possible for pirval to be 0 here, even though the
4051201b1cccSPeter Grehan 	 * pending bit has been set. The scenario is:
4052201b1cccSPeter Grehan 	 * CPU-Y is sending a posted interrupt to CPU-X, which
4053201b1cccSPeter Grehan 	 * is running a guest and processing posted interrupts in h/w.
4054201b1cccSPeter Grehan 	 * CPU-X will eventually exit and the state seen in s/w is
4055201b1cccSPeter Grehan 	 * the pending bit set, but no PIR bits set.
4056201b1cccSPeter Grehan 	 *
4057201b1cccSPeter Grehan 	 *      CPU-X                      CPU-Y
4058201b1cccSPeter Grehan 	 *   (vm running)                (host running)
4059201b1cccSPeter Grehan 	 *   rx posted interrupt
4060201b1cccSPeter Grehan 	 *   CLEAR pending bit
4061201b1cccSPeter Grehan 	 *				 SET PIR bit
4062201b1cccSPeter Grehan 	 *   READ/CLEAR PIR bits
4063201b1cccSPeter Grehan 	 *				 SET pending bit
4064201b1cccSPeter Grehan 	 *   (vm exit)
4065201b1cccSPeter Grehan 	 *   pending bit set, PIR 0
406688c4b8d1SNeel Natu 	 */
406788c4b8d1SNeel Natu 	if (pirval != 0) {
406888c4b8d1SNeel Natu 		rvi = pirbase + flsl(pirval) - 1;
406988c4b8d1SNeel Natu 		intr_status_old = vmcs_read(VMCS_GUEST_INTR_STATUS);
407088c4b8d1SNeel Natu 		intr_status_new = (intr_status_old & 0xFF00) | rvi;
407188c4b8d1SNeel Natu 		if (intr_status_new > intr_status_old) {
407288c4b8d1SNeel Natu 			vmcs_write(VMCS_GUEST_INTR_STATUS, intr_status_new);
4073d030f941SJohn Baldwin 			VLAPIC_CTR2(vlapic, "vmx_inject_pir: "
407488c4b8d1SNeel Natu 			    "guest_intr_status changed from 0x%04x to 0x%04x",
407588c4b8d1SNeel Natu 			    intr_status_old, intr_status_new);
407688c4b8d1SNeel Natu 		}
407788c4b8d1SNeel Natu 	}
407888c4b8d1SNeel Natu }
407988c4b8d1SNeel Natu 
4080de5ea6b6SNeel Natu static struct vlapic *
4081869c8d19SJohn Baldwin vmx_vlapic_init(void *vcpui)
4082de5ea6b6SNeel Natu {
4083de5ea6b6SNeel Natu 	struct vmx *vmx;
40841aa51504SJohn Baldwin 	struct vmx_vcpu *vcpu;
4085de5ea6b6SNeel Natu 	struct vlapic *vlapic;
4086176666c2SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
4087de5ea6b6SNeel Natu 
40881aa51504SJohn Baldwin 	vcpu = vcpui;
4089869c8d19SJohn Baldwin 	vmx = vcpu->vmx;
4090de5ea6b6SNeel Natu 
409188c4b8d1SNeel Natu 	vlapic = malloc(sizeof(struct vlapic_vtx), M_VLAPIC, M_WAITOK | M_ZERO);
4092de5ea6b6SNeel Natu 	vlapic->vm = vmx->vm;
4093950af9ffSJohn Baldwin 	vlapic->vcpu = vcpu->vcpu;
40941aa51504SJohn Baldwin 	vlapic->vcpuid = vcpu->vcpuid;
40951aa51504SJohn Baldwin 	vlapic->apic_page = (struct LAPIC *)vcpu->apic_page;
4096de5ea6b6SNeel Natu 
4097176666c2SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
40981aa51504SJohn Baldwin 	vlapic_vtx->pir_desc = vcpu->pir_desc;
40991aa51504SJohn Baldwin 	vlapic_vtx->vcpu = vcpu;
4100176666c2SNeel Natu 
41011bc51badSMichael Reifenberger 	if (tpr_shadowing) {
41021bc51badSMichael Reifenberger 		vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode_ts;
41031bc51badSMichael Reifenberger 	}
41041bc51badSMichael Reifenberger 
410588c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
410688c4b8d1SNeel Natu 		vlapic->ops.set_intr_ready = vmx_set_intr_ready;
410788c4b8d1SNeel Natu 		vlapic->ops.pending_intr = vmx_pending_intr;
410888c4b8d1SNeel Natu 		vlapic->ops.intr_accepted = vmx_intr_accepted;
410930b94db8SNeel Natu 		vlapic->ops.set_tmr = vmx_set_tmr;
41101bc51badSMichael Reifenberger 		vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode_vid;
411188c4b8d1SNeel Natu 	}
411288c4b8d1SNeel Natu 
4113176666c2SNeel Natu 	if (posted_interrupts)
4114176666c2SNeel Natu 		vlapic->ops.post_intr = vmx_post_intr;
4115176666c2SNeel Natu 
4116de5ea6b6SNeel Natu 	vlapic_init(vlapic);
4117de5ea6b6SNeel Natu 
4118de5ea6b6SNeel Natu 	return (vlapic);
4119de5ea6b6SNeel Natu }
4120de5ea6b6SNeel Natu 
4121de5ea6b6SNeel Natu static void
4122869c8d19SJohn Baldwin vmx_vlapic_cleanup(struct vlapic *vlapic)
4123de5ea6b6SNeel Natu {
4124de5ea6b6SNeel Natu 
4125de5ea6b6SNeel Natu 	vlapic_cleanup(vlapic);
4126de5ea6b6SNeel Natu 	free(vlapic, M_VLAPIC);
4127de5ea6b6SNeel Natu }
4128de5ea6b6SNeel Natu 
4129483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT
4130483d953aSJohn Baldwin static int
4131869c8d19SJohn Baldwin vmx_vcpu_snapshot(void *vcpui, struct vm_snapshot_meta *meta)
4132483d953aSJohn Baldwin {
4133483d953aSJohn Baldwin 	struct vmcs *vmcs;
4134483d953aSJohn Baldwin 	struct vmx *vmx;
413539ec056eSJohn Baldwin 	struct vmx_vcpu *vcpu;
413639ec056eSJohn Baldwin 	struct vmxctx *vmxctx;
4137483d953aSJohn Baldwin 	int err, run, hostcpu;
4138483d953aSJohn Baldwin 
4139483d953aSJohn Baldwin 	err = 0;
4140869c8d19SJohn Baldwin 	vcpu = vcpui;
4141869c8d19SJohn Baldwin 	vmx = vcpu->vmx;
414239ec056eSJohn Baldwin 	vmcs = vcpu->vmcs;
4143483d953aSJohn Baldwin 
414480cb5d84SJohn Baldwin 	run = vcpu_is_running(vcpu->vcpu, &hostcpu);
4145483d953aSJohn Baldwin 	if (run && hostcpu != curcpu) {
414639ec056eSJohn Baldwin 		printf("%s: %s%d is running", __func__, vm_name(vmx->vm),
41471aa51504SJohn Baldwin 		    vcpu->vcpuid);
4148483d953aSJohn Baldwin 		return (EINVAL);
4149483d953aSJohn Baldwin 	}
4150483d953aSJohn Baldwin 
4151483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CR0, meta);
4152483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CR3, meta);
4153483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CR4, meta);
4154483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_DR7, meta);
4155483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_RSP, meta);
4156483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_RIP, meta);
4157483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_RFLAGS, meta);
4158483d953aSJohn Baldwin 
4159483d953aSJohn Baldwin 	/* Guest segments */
4160483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_ES, meta);
4161483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_ES, meta);
4162483d953aSJohn Baldwin 
4163483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CS, meta);
4164483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_CS, meta);
4165483d953aSJohn Baldwin 
4166483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_SS, meta);
4167483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_SS, meta);
4168483d953aSJohn Baldwin 
4169483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_DS, meta);
4170483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_DS, meta);
4171483d953aSJohn Baldwin 
4172483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_FS, meta);
4173483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_FS, meta);
4174483d953aSJohn Baldwin 
4175483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_GS, meta);
4176483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_GS, meta);
4177483d953aSJohn Baldwin 
4178483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_TR, meta);
4179483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_TR, meta);
4180483d953aSJohn Baldwin 
4181483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_LDTR, meta);
4182483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_LDTR, meta);
4183483d953aSJohn Baldwin 
4184483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_EFER, meta);
4185483d953aSJohn Baldwin 
4186483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_IDTR, meta);
4187483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_GDTR, meta);
4188483d953aSJohn Baldwin 
4189483d953aSJohn Baldwin 	/* Guest page tables */
4190483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE0, meta);
4191483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE1, meta);
4192483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE2, meta);
4193483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE3, meta);
4194483d953aSJohn Baldwin 
4195483d953aSJohn Baldwin 	/* Other guest state */
4196483d953aSJohn Baldwin 	err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_IA32_SYSENTER_CS, meta);
4197483d953aSJohn Baldwin 	err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_IA32_SYSENTER_ESP, meta);
4198483d953aSJohn Baldwin 	err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_IA32_SYSENTER_EIP, meta);
4199483d953aSJohn Baldwin 	err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_INTERRUPTIBILITY, meta);
4200483d953aSJohn Baldwin 	err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_ACTIVITY, meta);
4201483d953aSJohn Baldwin 	err += vmcs_snapshot_any(vmcs, run, VMCS_ENTRY_CTLS, meta);
4202483d953aSJohn Baldwin 	err += vmcs_snapshot_any(vmcs, run, VMCS_EXIT_CTLS, meta);
420339ec056eSJohn Baldwin 	if (err != 0)
420439ec056eSJohn Baldwin 		goto done;
4205483d953aSJohn Baldwin 
420639ec056eSJohn Baldwin 	SNAPSHOT_BUF_OR_LEAVE(vcpu->guest_msrs,
420739ec056eSJohn Baldwin 	    sizeof(vcpu->guest_msrs), meta, err, done);
420839ec056eSJohn Baldwin 
4209c543e09fSVitaliy Gusev 	SNAPSHOT_BUF_OR_LEAVE(vcpu->pir_desc,
4210c543e09fSVitaliy Gusev 	    sizeof(*vcpu->pir_desc), meta, err, done);
4211c543e09fSVitaliy Gusev 
4212683ea4d2SVitaliy Gusev 	SNAPSHOT_BUF_OR_LEAVE(&vcpu->mtrr,
4213683ea4d2SVitaliy Gusev 	    sizeof(vcpu->mtrr), meta, err, done);
4214683ea4d2SVitaliy Gusev 
421539ec056eSJohn Baldwin 	vmxctx = &vcpu->ctx;
421639ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rdi, meta, err, done);
421739ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rsi, meta, err, done);
421839ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rdx, meta, err, done);
421939ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rcx, meta, err, done);
422039ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r8, meta, err, done);
422139ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r9, meta, err, done);
422239ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rax, meta, err, done);
422339ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rbx, meta, err, done);
422439ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rbp, meta, err, done);
422539ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r10, meta, err, done);
422639ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r11, meta, err, done);
422739ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r12, meta, err, done);
422839ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r13, meta, err, done);
422939ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r14, meta, err, done);
423039ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r15, meta, err, done);
423139ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_cr2, meta, err, done);
423239ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr0, meta, err, done);
423339ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr1, meta, err, done);
423439ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr2, meta, err, done);
423539ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr3, meta, err, done);
423639ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr6, meta, err, done);
423739ec056eSJohn Baldwin 
423839ec056eSJohn Baldwin done:
4239483d953aSJohn Baldwin 	return (err);
4240483d953aSJohn Baldwin }
4241483d953aSJohn Baldwin 
4242483d953aSJohn Baldwin static int
4243869c8d19SJohn Baldwin vmx_restore_tsc(void *vcpui, uint64_t offset)
4244483d953aSJohn Baldwin {
42451aa51504SJohn Baldwin 	struct vmx_vcpu *vcpu = vcpui;
4246869c8d19SJohn Baldwin 	struct vmcs *vmcs;
4247869c8d19SJohn Baldwin 	struct vmx *vmx;
4248483d953aSJohn Baldwin 	int error, running, hostcpu;
4249483d953aSJohn Baldwin 
4250869c8d19SJohn Baldwin 	vmx = vcpu->vmx;
42511aa51504SJohn Baldwin 	vmcs = vcpu->vmcs;
4252483d953aSJohn Baldwin 
425380cb5d84SJohn Baldwin 	running = vcpu_is_running(vcpu->vcpu, &hostcpu);
4254483d953aSJohn Baldwin 	if (running && hostcpu != curcpu) {
42551aa51504SJohn Baldwin 		printf("%s: %s%d is running", __func__, vm_name(vmx->vm),
42561aa51504SJohn Baldwin 		    vcpu->vcpuid);
4257483d953aSJohn Baldwin 		return (EINVAL);
4258483d953aSJohn Baldwin 	}
4259483d953aSJohn Baldwin 
4260483d953aSJohn Baldwin 	if (!running)
4261483d953aSJohn Baldwin 		VMPTRLD(vmcs);
4262483d953aSJohn Baldwin 
426380cb5d84SJohn Baldwin 	error = vmx_set_tsc_offset(vcpu, offset);
4264483d953aSJohn Baldwin 
4265483d953aSJohn Baldwin 	if (!running)
4266483d953aSJohn Baldwin 		VMCLEAR(vmcs);
4267483d953aSJohn Baldwin 	return (error);
4268483d953aSJohn Baldwin }
4269483d953aSJohn Baldwin #endif
4270483d953aSJohn Baldwin 
427115add60dSPeter Grehan const struct vmm_ops vmm_ops_intel = {
427215add60dSPeter Grehan 	.modinit	= vmx_modinit,
427315add60dSPeter Grehan 	.modcleanup	= vmx_modcleanup,
427415add60dSPeter Grehan 	.modresume	= vmx_modresume,
427513a7c4d4SMark Johnston 	.init		= vmx_init,
427615add60dSPeter Grehan 	.run		= vmx_run,
427713a7c4d4SMark Johnston 	.cleanup	= vmx_cleanup,
42781aa51504SJohn Baldwin 	.vcpu_init	= vmx_vcpu_init,
42791aa51504SJohn Baldwin 	.vcpu_cleanup	= vmx_vcpu_cleanup,
428015add60dSPeter Grehan 	.getreg		= vmx_getreg,
428115add60dSPeter Grehan 	.setreg		= vmx_setreg,
428215add60dSPeter Grehan 	.getdesc	= vmx_getdesc,
428315add60dSPeter Grehan 	.setdesc	= vmx_setdesc,
428415add60dSPeter Grehan 	.getcap		= vmx_getcap,
428515add60dSPeter Grehan 	.setcap		= vmx_setcap,
428615add60dSPeter Grehan 	.vmspace_alloc	= vmx_vmspace_alloc,
428715add60dSPeter Grehan 	.vmspace_free	= vmx_vmspace_free,
428813a7c4d4SMark Johnston 	.vlapic_init	= vmx_vlapic_init,
428913a7c4d4SMark Johnston 	.vlapic_cleanup	= vmx_vlapic_cleanup,
4290483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT
429139ec056eSJohn Baldwin 	.vcpu_snapshot	= vmx_vcpu_snapshot,
429215add60dSPeter Grehan 	.restore_tsc	= vmx_restore_tsc,
4293483d953aSJohn Baldwin #endif
4294366f6083SPeter Grehan };
4295