xref: /freebsd/sys/amd64/vmm/intel/vmx.c (revision 2ee1a18d51ed68ee34df7dcfd05f6cfc16110202)
1366f6083SPeter Grehan /*-
2c49761ddSPedro F. Giffuni  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3c49761ddSPedro F. Giffuni  *
4366f6083SPeter Grehan  * Copyright (c) 2011 NetApp, Inc.
5366f6083SPeter Grehan  * All rights reserved.
62c352febSJohn Baldwin  * Copyright (c) 2018 Joyent, Inc.
7366f6083SPeter Grehan  *
8366f6083SPeter Grehan  * Redistribution and use in source and binary forms, with or without
9366f6083SPeter Grehan  * modification, are permitted provided that the following conditions
10366f6083SPeter Grehan  * are met:
11366f6083SPeter Grehan  * 1. Redistributions of source code must retain the above copyright
12366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer.
13366f6083SPeter Grehan  * 2. Redistributions in binary form must reproduce the above copyright
14366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer in the
15366f6083SPeter Grehan  *    documentation and/or other materials provided with the distribution.
16366f6083SPeter Grehan  *
17366f6083SPeter Grehan  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
18366f6083SPeter Grehan  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19366f6083SPeter Grehan  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20366f6083SPeter Grehan  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
21366f6083SPeter Grehan  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22366f6083SPeter Grehan  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23366f6083SPeter Grehan  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24366f6083SPeter Grehan  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25366f6083SPeter Grehan  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26366f6083SPeter Grehan  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27366f6083SPeter Grehan  * SUCH DAMAGE.
28366f6083SPeter Grehan  *
29366f6083SPeter Grehan  * $FreeBSD$
30366f6083SPeter Grehan  */
31366f6083SPeter Grehan 
32366f6083SPeter Grehan #include <sys/cdefs.h>
33366f6083SPeter Grehan __FBSDID("$FreeBSD$");
34366f6083SPeter Grehan 
35483d953aSJohn Baldwin #include "opt_bhyve_snapshot.h"
36483d953aSJohn Baldwin 
37366f6083SPeter Grehan #include <sys/param.h>
38366f6083SPeter Grehan #include <sys/systm.h>
39366f6083SPeter Grehan #include <sys/smp.h>
40366f6083SPeter Grehan #include <sys/kernel.h>
41366f6083SPeter Grehan #include <sys/malloc.h>
42366f6083SPeter Grehan #include <sys/pcpu.h>
43366f6083SPeter Grehan #include <sys/proc.h>
44b7924341SAndrew Turner #include <sys/reg.h>
456f5a9606SMark Johnston #include <sys/smr.h>
463565b59eSNeel Natu #include <sys/sysctl.h>
47366f6083SPeter Grehan 
48366f6083SPeter Grehan #include <vm/vm.h>
49366f6083SPeter Grehan #include <vm/pmap.h>
50366f6083SPeter Grehan 
51366f6083SPeter Grehan #include <machine/psl.h>
52366f6083SPeter Grehan #include <machine/cpufunc.h>
538b287612SJohn Baldwin #include <machine/md_var.h>
54366f6083SPeter Grehan #include <machine/segments.h>
55176666c2SNeel Natu #include <machine/smp.h>
56608f97c3SPeter Grehan #include <machine/specialreg.h>
57366f6083SPeter Grehan #include <machine/vmparam.h>
58366f6083SPeter Grehan 
59366f6083SPeter Grehan #include <machine/vmm.h>
60dc506506SNeel Natu #include <machine/vmm_dev.h>
61e813a873SNeel Natu #include <machine/vmm_instruction_emul.h>
62483d953aSJohn Baldwin #include <machine/vmm_snapshot.h>
63483d953aSJohn Baldwin 
64c3498942SNeel Natu #include "vmm_lapic.h"
65b01c2033SNeel Natu #include "vmm_host.h"
66762fd208STycho Nightingale #include "vmm_ioport.h"
67366f6083SPeter Grehan #include "vmm_ktr.h"
68366f6083SPeter Grehan #include "vmm_stat.h"
690775fbb4STycho Nightingale #include "vatpic.h"
70de5ea6b6SNeel Natu #include "vlapic.h"
71de5ea6b6SNeel Natu #include "vlapic_priv.h"
72366f6083SPeter Grehan 
73366f6083SPeter Grehan #include "ept.h"
74366f6083SPeter Grehan #include "vmx_cpufunc.h"
75366f6083SPeter Grehan #include "vmx.h"
76c3498942SNeel Natu #include "vmx_msr.h"
77366f6083SPeter Grehan #include "x86.h"
78366f6083SPeter Grehan #include "vmx_controls.h"
79366f6083SPeter Grehan 
80366f6083SPeter Grehan #define	PINBASED_CTLS_ONE_SETTING					\
81366f6083SPeter Grehan 	(PINBASED_EXTINT_EXITING	|				\
82366f6083SPeter Grehan 	 PINBASED_NMI_EXITING		|				\
83366f6083SPeter Grehan 	 PINBASED_VIRTUAL_NMI)
84366f6083SPeter Grehan #define	PINBASED_CTLS_ZERO_SETTING	0
85366f6083SPeter Grehan 
86366f6083SPeter Grehan #define PROCBASED_CTLS_WINDOW_SETTING					\
87366f6083SPeter Grehan 	(PROCBASED_INT_WINDOW_EXITING	|				\
88366f6083SPeter Grehan 	 PROCBASED_NMI_WINDOW_EXITING)
89366f6083SPeter Grehan 
90366f6083SPeter Grehan #define	PROCBASED_CTLS_ONE_SETTING					\
91366f6083SPeter Grehan 	(PROCBASED_SECONDARY_CONTROLS	|				\
9265145c7fSNeel Natu 	 PROCBASED_MWAIT_EXITING	|				\
9365145c7fSNeel Natu 	 PROCBASED_MONITOR_EXITING	|				\
94366f6083SPeter Grehan 	 PROCBASED_IO_EXITING		|				\
95366f6083SPeter Grehan 	 PROCBASED_MSR_BITMAPS		|				\
96594db002STycho Nightingale 	 PROCBASED_CTLS_WINDOW_SETTING	|				\
97594db002STycho Nightingale 	 PROCBASED_CR8_LOAD_EXITING	|				\
98594db002STycho Nightingale 	 PROCBASED_CR8_STORE_EXITING)
99366f6083SPeter Grehan #define	PROCBASED_CTLS_ZERO_SETTING	\
100366f6083SPeter Grehan 	(PROCBASED_CR3_LOAD_EXITING |	\
101366f6083SPeter Grehan 	PROCBASED_CR3_STORE_EXITING |	\
102366f6083SPeter Grehan 	PROCBASED_IO_BITMAPS)
103366f6083SPeter Grehan 
104366f6083SPeter Grehan #define	PROCBASED_CTLS2_ONE_SETTING	PROCBASED2_ENABLE_EPT
105366f6083SPeter Grehan #define	PROCBASED_CTLS2_ZERO_SETTING	0
106366f6083SPeter Grehan 
107d72978ecSNeel Natu #define	VM_EXIT_CTLS_ONE_SETTING					\
10865eefbe4SJohn Baldwin 	(VM_EXIT_SAVE_DEBUG_CONTROLS		|			\
10965eefbe4SJohn Baldwin 	VM_EXIT_HOST_LMA			|			\
110366f6083SPeter Grehan 	VM_EXIT_SAVE_EFER			|			\
111d72978ecSNeel Natu 	VM_EXIT_LOAD_EFER			|			\
112a318f7ddSNeel Natu 	VM_EXIT_ACKNOWLEDGE_INTERRUPT)
113d72978ecSNeel Natu 
11465eefbe4SJohn Baldwin #define	VM_EXIT_CTLS_ZERO_SETTING	0
115366f6083SPeter Grehan 
11665eefbe4SJohn Baldwin #define	VM_ENTRY_CTLS_ONE_SETTING					\
11765eefbe4SJohn Baldwin 	(VM_ENTRY_LOAD_DEBUG_CONTROLS		|			\
11865eefbe4SJohn Baldwin 	VM_ENTRY_LOAD_EFER)
119608f97c3SPeter Grehan 
120366f6083SPeter Grehan #define	VM_ENTRY_CTLS_ZERO_SETTING					\
12165eefbe4SJohn Baldwin 	(VM_ENTRY_INTO_SMM			|			\
122366f6083SPeter Grehan 	VM_ENTRY_DEACTIVATE_DUAL_MONITOR)
123366f6083SPeter Grehan 
124366f6083SPeter Grehan #define	HANDLED		1
125366f6083SPeter Grehan #define	UNHANDLED	0
126366f6083SPeter Grehan 
127de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VMX, "vmx", "vmx");
128de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
129366f6083SPeter Grehan 
13073abae44SJohn Baldwin bool vmx_have_msr_tsc_aux;
13173abae44SJohn Baldwin 
1323565b59eSNeel Natu SYSCTL_DECL(_hw_vmm);
133b40598c5SPawel Biernacki SYSCTL_NODE(_hw_vmm, OID_AUTO, vmx, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
134b40598c5SPawel Biernacki     NULL);
1353565b59eSNeel Natu 
136b3996dd4SJohn Baldwin int vmxon_enabled[MAXCPU];
137366f6083SPeter Grehan static char vmxon_region[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE);
138366f6083SPeter Grehan 
139366f6083SPeter Grehan static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2;
140366f6083SPeter Grehan static uint32_t exit_ctls, entry_ctls;
141366f6083SPeter Grehan 
142366f6083SPeter Grehan static uint64_t cr0_ones_mask, cr0_zeros_mask;
1433565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_ones_mask, CTLFLAG_RD,
1443565b59eSNeel Natu 	     &cr0_ones_mask, 0, NULL);
1453565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_zeros_mask, CTLFLAG_RD,
1463565b59eSNeel Natu 	     &cr0_zeros_mask, 0, NULL);
1473565b59eSNeel Natu 
148366f6083SPeter Grehan static uint64_t cr4_ones_mask, cr4_zeros_mask;
1493565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_ones_mask, CTLFLAG_RD,
1503565b59eSNeel Natu 	     &cr4_ones_mask, 0, NULL);
1513565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_zeros_mask, CTLFLAG_RD,
1523565b59eSNeel Natu 	     &cr4_zeros_mask, 0, NULL);
153366f6083SPeter Grehan 
1543565b59eSNeel Natu static int vmx_initialized;
1553565b59eSNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, initialized, CTLFLAG_RD,
1563565b59eSNeel Natu 	   &vmx_initialized, 0, "Intel VMX initialized");
1573565b59eSNeel Natu 
158366f6083SPeter Grehan /*
159366f6083SPeter Grehan  * Optional capabilities
160366f6083SPeter Grehan  */
161b40598c5SPawel Biernacki static SYSCTL_NODE(_hw_vmm_vmx, OID_AUTO, cap,
162b40598c5SPawel Biernacki     CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
163b40598c5SPawel Biernacki     NULL);
16406fc6db9SJohn Baldwin 
165366f6083SPeter Grehan static int cap_halt_exit;
16606fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, halt_exit, CTLFLAG_RD, &cap_halt_exit, 0,
16706fc6db9SJohn Baldwin     "HLT triggers a VM-exit");
16806fc6db9SJohn Baldwin 
169366f6083SPeter Grehan static int cap_pause_exit;
17006fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, pause_exit, CTLFLAG_RD, &cap_pause_exit,
17106fc6db9SJohn Baldwin     0, "PAUSE triggers a VM-exit");
17206fc6db9SJohn Baldwin 
1733ba952e1SCorvin Köhne static int cap_wbinvd_exit;
1743ba952e1SCorvin Köhne SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, wbinvd_exit, CTLFLAG_RD, &cap_wbinvd_exit,
1753ba952e1SCorvin Köhne     0, "WBINVD triggers a VM-exit");
1763ba952e1SCorvin Köhne 
177f5f5f1e7SPeter Grehan static int cap_rdpid;
178f5f5f1e7SPeter Grehan SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, rdpid, CTLFLAG_RD, &cap_rdpid, 0,
179f5f5f1e7SPeter Grehan     "Guests are allowed to use RDPID");
180f5f5f1e7SPeter Grehan 
181f5f5f1e7SPeter Grehan static int cap_rdtscp;
182f5f5f1e7SPeter Grehan SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, rdtscp, CTLFLAG_RD, &cap_rdtscp, 0,
183f5f5f1e7SPeter Grehan     "Guests are allowed to use RDTSCP");
184f5f5f1e7SPeter Grehan 
185366f6083SPeter Grehan static int cap_unrestricted_guest;
18606fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, unrestricted_guest, CTLFLAG_RD,
18706fc6db9SJohn Baldwin     &cap_unrestricted_guest, 0, "Unrestricted guests");
18806fc6db9SJohn Baldwin 
189366f6083SPeter Grehan static int cap_monitor_trap;
19006fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, monitor_trap, CTLFLAG_RD,
19106fc6db9SJohn Baldwin     &cap_monitor_trap, 0, "Monitor trap flag");
19206fc6db9SJohn Baldwin 
19349cc03daSNeel Natu static int cap_invpcid;
19406fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, invpcid, CTLFLAG_RD, &cap_invpcid,
19506fc6db9SJohn Baldwin     0, "Guests are allowed to use INVPCID");
196366f6083SPeter Grehan 
1971bc51badSMichael Reifenberger static int tpr_shadowing;
1981bc51badSMichael Reifenberger SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, tpr_shadowing, CTLFLAG_RD,
1991bc51badSMichael Reifenberger     &tpr_shadowing, 0, "TPR shadowing support");
2001bc51badSMichael Reifenberger 
20188c4b8d1SNeel Natu static int virtual_interrupt_delivery;
20206fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, virtual_interrupt_delivery, CTLFLAG_RD,
20388c4b8d1SNeel Natu     &virtual_interrupt_delivery, 0, "APICv virtual interrupt delivery support");
20488c4b8d1SNeel Natu 
205176666c2SNeel Natu static int posted_interrupts;
20606fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, posted_interrupts, CTLFLAG_RD,
207176666c2SNeel Natu     &posted_interrupts, 0, "APICv posted interrupt support");
208176666c2SNeel Natu 
20918a2b08eSNeel Natu static int pirvec = -1;
210176666c2SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupt_vector, CTLFLAG_RD,
211176666c2SNeel Natu     &pirvec, 0, "APICv posted interrupt vector");
212176666c2SNeel Natu 
21345e51299SNeel Natu static struct unrhdr *vpid_unr;
21445e51299SNeel Natu static u_int vpid_alloc_failed;
21545e51299SNeel Natu SYSCTL_UINT(_hw_vmm_vmx, OID_AUTO, vpid_alloc_failed, CTLFLAG_RD,
21645e51299SNeel Natu 	    &vpid_alloc_failed, 0, NULL);
21745e51299SNeel Natu 
218d3588766SMark Johnston int guest_l1d_flush;
219c30578feSKonstantin Belousov SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, l1d_flush, CTLFLAG_RD,
220c30578feSKonstantin Belousov     &guest_l1d_flush, 0, NULL);
221d3588766SMark Johnston int guest_l1d_flush_sw;
222c1141fbaSKonstantin Belousov SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, l1d_flush_sw, CTLFLAG_RD,
223c1141fbaSKonstantin Belousov     &guest_l1d_flush_sw, 0, NULL);
224c30578feSKonstantin Belousov 
225c1141fbaSKonstantin Belousov static struct msr_entry msr_load_list[1] __aligned(16);
226c30578feSKonstantin Belousov 
22788c4b8d1SNeel Natu /*
2286ac73777STycho Nightingale  * The definitions of SDT probes for VMX.
2296ac73777STycho Nightingale  */
2306ac73777STycho Nightingale 
2316ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, entry,
2326ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2336ac73777STycho Nightingale 
2346ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, taskswitch,
2356ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "struct vm_task_switch *");
2366ac73777STycho Nightingale 
2376ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, craccess,
2386ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint64_t");
2396ac73777STycho Nightingale 
2406ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, rdmsr,
2416ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint32_t");
2426ac73777STycho Nightingale 
2436ac73777STycho Nightingale SDT_PROBE_DEFINE5(vmm, vmx, exit, wrmsr,
2446ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint32_t", "uint64_t");
2456ac73777STycho Nightingale 
2466ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, halt,
2476ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2486ac73777STycho Nightingale 
2496ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, mtrap,
2506ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2516ac73777STycho Nightingale 
2526ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, pause,
2536ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2546ac73777STycho Nightingale 
2556ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, intrwindow,
2566ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2576ac73777STycho Nightingale 
2586ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, interrupt,
2596ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint32_t");
2606ac73777STycho Nightingale 
2616ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, nmiwindow,
2626ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2636ac73777STycho Nightingale 
2646ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, inout,
2656ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2666ac73777STycho Nightingale 
2676ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, cpuid,
2686ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2696ac73777STycho Nightingale 
2706ac73777STycho Nightingale SDT_PROBE_DEFINE5(vmm, vmx, exit, exception,
2716ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint32_t", "int");
2726ac73777STycho Nightingale 
2736ac73777STycho Nightingale SDT_PROBE_DEFINE5(vmm, vmx, exit, nestedfault,
2746ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint64_t", "uint64_t");
2756ac73777STycho Nightingale 
2766ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, mmiofault,
2776ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint64_t");
2786ac73777STycho Nightingale 
2796ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, eoi,
2806ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2816ac73777STycho Nightingale 
2826ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, apicaccess,
2836ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2846ac73777STycho Nightingale 
2856ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, apicwrite,
2866ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "struct vlapic *");
2876ac73777STycho Nightingale 
2886ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, xsetbv,
2896ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2906ac73777STycho Nightingale 
2916ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, monitor,
2926ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2936ac73777STycho Nightingale 
2946ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, mwait,
2956ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2966ac73777STycho Nightingale 
29727d26457SAndrew Turner SDT_PROBE_DEFINE3(vmm, vmx, exit, vminsn,
29827d26457SAndrew Turner     "struct vmx *", "int", "struct vm_exit *");
29927d26457SAndrew Turner 
3006ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, unknown,
3016ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint32_t");
3026ac73777STycho Nightingale 
3036ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, return,
3046ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "int");
3056ac73777STycho Nightingale 
3066ac73777STycho Nightingale /*
30788c4b8d1SNeel Natu  * Use the last page below 4GB as the APIC access address. This address is
30888c4b8d1SNeel Natu  * occupied by the boot firmware so it is guaranteed that it will not conflict
30988c4b8d1SNeel Natu  * with a page in system memory.
31088c4b8d1SNeel Natu  */
31188c4b8d1SNeel Natu #define	APIC_ACCESS_ADDRESS	0xFFFFF000
31288c4b8d1SNeel Natu 
313869c8d19SJohn Baldwin static int vmx_getdesc(void *vcpui, int reg, struct seg_desc *desc);
314869c8d19SJohn Baldwin static int vmx_getreg(void *vcpui, int reg, uint64_t *retval);
315c3498942SNeel Natu static int vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val);
31688c4b8d1SNeel Natu static void vmx_inject_pir(struct vlapic *vlapic);
317483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT
318869c8d19SJohn Baldwin static int vmx_restore_tsc(void *vcpui, uint64_t now);
319483d953aSJohn Baldwin #endif
32088c4b8d1SNeel Natu 
321f5f5f1e7SPeter Grehan static inline bool
322f5f5f1e7SPeter Grehan host_has_rdpid(void)
323f5f5f1e7SPeter Grehan {
324f5f5f1e7SPeter Grehan 	return ((cpu_stdext_feature2 & CPUID_STDEXT2_RDPID) != 0);
325f5f5f1e7SPeter Grehan }
326f5f5f1e7SPeter Grehan 
327f5f5f1e7SPeter Grehan static inline bool
328f5f5f1e7SPeter Grehan host_has_rdtscp(void)
329f5f5f1e7SPeter Grehan {
330f5f5f1e7SPeter Grehan 	return ((amd_feature & AMDID_RDTSCP) != 0);
331f5f5f1e7SPeter Grehan }
332f5f5f1e7SPeter Grehan 
333366f6083SPeter Grehan #ifdef KTR
334366f6083SPeter Grehan static const char *
335366f6083SPeter Grehan exit_reason_to_str(int reason)
336366f6083SPeter Grehan {
337366f6083SPeter Grehan 	static char reasonbuf[32];
338366f6083SPeter Grehan 
339366f6083SPeter Grehan 	switch (reason) {
340366f6083SPeter Grehan 	case EXIT_REASON_EXCEPTION:
341366f6083SPeter Grehan 		return "exception";
342366f6083SPeter Grehan 	case EXIT_REASON_EXT_INTR:
343366f6083SPeter Grehan 		return "extint";
344366f6083SPeter Grehan 	case EXIT_REASON_TRIPLE_FAULT:
345366f6083SPeter Grehan 		return "triplefault";
346366f6083SPeter Grehan 	case EXIT_REASON_INIT:
347366f6083SPeter Grehan 		return "init";
348366f6083SPeter Grehan 	case EXIT_REASON_SIPI:
349366f6083SPeter Grehan 		return "sipi";
350366f6083SPeter Grehan 	case EXIT_REASON_IO_SMI:
351366f6083SPeter Grehan 		return "iosmi";
352366f6083SPeter Grehan 	case EXIT_REASON_SMI:
353366f6083SPeter Grehan 		return "smi";
354366f6083SPeter Grehan 	case EXIT_REASON_INTR_WINDOW:
355366f6083SPeter Grehan 		return "intrwindow";
356366f6083SPeter Grehan 	case EXIT_REASON_NMI_WINDOW:
357366f6083SPeter Grehan 		return "nmiwindow";
358366f6083SPeter Grehan 	case EXIT_REASON_TASK_SWITCH:
359366f6083SPeter Grehan 		return "taskswitch";
360366f6083SPeter Grehan 	case EXIT_REASON_CPUID:
361366f6083SPeter Grehan 		return "cpuid";
362366f6083SPeter Grehan 	case EXIT_REASON_GETSEC:
363366f6083SPeter Grehan 		return "getsec";
364366f6083SPeter Grehan 	case EXIT_REASON_HLT:
365366f6083SPeter Grehan 		return "hlt";
366366f6083SPeter Grehan 	case EXIT_REASON_INVD:
367366f6083SPeter Grehan 		return "invd";
368366f6083SPeter Grehan 	case EXIT_REASON_INVLPG:
369366f6083SPeter Grehan 		return "invlpg";
370366f6083SPeter Grehan 	case EXIT_REASON_RDPMC:
371366f6083SPeter Grehan 		return "rdpmc";
372366f6083SPeter Grehan 	case EXIT_REASON_RDTSC:
373366f6083SPeter Grehan 		return "rdtsc";
374366f6083SPeter Grehan 	case EXIT_REASON_RSM:
375366f6083SPeter Grehan 		return "rsm";
376366f6083SPeter Grehan 	case EXIT_REASON_VMCALL:
377366f6083SPeter Grehan 		return "vmcall";
378366f6083SPeter Grehan 	case EXIT_REASON_VMCLEAR:
379366f6083SPeter Grehan 		return "vmclear";
380366f6083SPeter Grehan 	case EXIT_REASON_VMLAUNCH:
381366f6083SPeter Grehan 		return "vmlaunch";
382366f6083SPeter Grehan 	case EXIT_REASON_VMPTRLD:
383366f6083SPeter Grehan 		return "vmptrld";
384366f6083SPeter Grehan 	case EXIT_REASON_VMPTRST:
385366f6083SPeter Grehan 		return "vmptrst";
386366f6083SPeter Grehan 	case EXIT_REASON_VMREAD:
387366f6083SPeter Grehan 		return "vmread";
388366f6083SPeter Grehan 	case EXIT_REASON_VMRESUME:
389366f6083SPeter Grehan 		return "vmresume";
390366f6083SPeter Grehan 	case EXIT_REASON_VMWRITE:
391366f6083SPeter Grehan 		return "vmwrite";
392366f6083SPeter Grehan 	case EXIT_REASON_VMXOFF:
393366f6083SPeter Grehan 		return "vmxoff";
394366f6083SPeter Grehan 	case EXIT_REASON_VMXON:
395366f6083SPeter Grehan 		return "vmxon";
396366f6083SPeter Grehan 	case EXIT_REASON_CR_ACCESS:
397366f6083SPeter Grehan 		return "craccess";
398366f6083SPeter Grehan 	case EXIT_REASON_DR_ACCESS:
399366f6083SPeter Grehan 		return "draccess";
400366f6083SPeter Grehan 	case EXIT_REASON_INOUT:
401366f6083SPeter Grehan 		return "inout";
402366f6083SPeter Grehan 	case EXIT_REASON_RDMSR:
403366f6083SPeter Grehan 		return "rdmsr";
404366f6083SPeter Grehan 	case EXIT_REASON_WRMSR:
405366f6083SPeter Grehan 		return "wrmsr";
406366f6083SPeter Grehan 	case EXIT_REASON_INVAL_VMCS:
407366f6083SPeter Grehan 		return "invalvmcs";
408366f6083SPeter Grehan 	case EXIT_REASON_INVAL_MSR:
409366f6083SPeter Grehan 		return "invalmsr";
410366f6083SPeter Grehan 	case EXIT_REASON_MWAIT:
411366f6083SPeter Grehan 		return "mwait";
412366f6083SPeter Grehan 	case EXIT_REASON_MTF:
413366f6083SPeter Grehan 		return "mtf";
414366f6083SPeter Grehan 	case EXIT_REASON_MONITOR:
415366f6083SPeter Grehan 		return "monitor";
416366f6083SPeter Grehan 	case EXIT_REASON_PAUSE:
417366f6083SPeter Grehan 		return "pause";
418b0538143SNeel Natu 	case EXIT_REASON_MCE_DURING_ENTRY:
419b0538143SNeel Natu 		return "mce-during-entry";
420366f6083SPeter Grehan 	case EXIT_REASON_TPR:
421366f6083SPeter Grehan 		return "tpr";
42288c4b8d1SNeel Natu 	case EXIT_REASON_APIC_ACCESS:
42388c4b8d1SNeel Natu 		return "apic-access";
424366f6083SPeter Grehan 	case EXIT_REASON_GDTR_IDTR:
425366f6083SPeter Grehan 		return "gdtridtr";
426366f6083SPeter Grehan 	case EXIT_REASON_LDTR_TR:
427366f6083SPeter Grehan 		return "ldtrtr";
428366f6083SPeter Grehan 	case EXIT_REASON_EPT_FAULT:
429366f6083SPeter Grehan 		return "eptfault";
430366f6083SPeter Grehan 	case EXIT_REASON_EPT_MISCONFIG:
431366f6083SPeter Grehan 		return "eptmisconfig";
432366f6083SPeter Grehan 	case EXIT_REASON_INVEPT:
433366f6083SPeter Grehan 		return "invept";
434366f6083SPeter Grehan 	case EXIT_REASON_RDTSCP:
435366f6083SPeter Grehan 		return "rdtscp";
436366f6083SPeter Grehan 	case EXIT_REASON_VMX_PREEMPT:
437366f6083SPeter Grehan 		return "vmxpreempt";
438366f6083SPeter Grehan 	case EXIT_REASON_INVVPID:
439366f6083SPeter Grehan 		return "invvpid";
440366f6083SPeter Grehan 	case EXIT_REASON_WBINVD:
441366f6083SPeter Grehan 		return "wbinvd";
442366f6083SPeter Grehan 	case EXIT_REASON_XSETBV:
443366f6083SPeter Grehan 		return "xsetbv";
44488c4b8d1SNeel Natu 	case EXIT_REASON_APIC_WRITE:
44588c4b8d1SNeel Natu 		return "apic-write";
446366f6083SPeter Grehan 	default:
447366f6083SPeter Grehan 		snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason);
448366f6083SPeter Grehan 		return (reasonbuf);
449366f6083SPeter Grehan 	}
450366f6083SPeter Grehan }
451366f6083SPeter Grehan #endif	/* KTR */
452366f6083SPeter Grehan 
453159dd56fSNeel Natu static int
454159dd56fSNeel Natu vmx_allow_x2apic_msrs(struct vmx *vmx)
455159dd56fSNeel Natu {
456159dd56fSNeel Natu 	int i, error;
457159dd56fSNeel Natu 
458159dd56fSNeel Natu 	error = 0;
459159dd56fSNeel Natu 
460159dd56fSNeel Natu 	/*
461159dd56fSNeel Natu 	 * Allow readonly access to the following x2APIC MSRs from the guest.
462159dd56fSNeel Natu 	 */
463159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ID);
464159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_VERSION);
465159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LDR);
466159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_SVR);
467159dd56fSNeel Natu 
468159dd56fSNeel Natu 	for (i = 0; i < 8; i++)
469159dd56fSNeel Natu 		error += guest_msr_ro(vmx, MSR_APIC_ISR0 + i);
470159dd56fSNeel Natu 
471159dd56fSNeel Natu 	for (i = 0; i < 8; i++)
472159dd56fSNeel Natu 		error += guest_msr_ro(vmx, MSR_APIC_TMR0 + i);
473159dd56fSNeel Natu 
474159dd56fSNeel Natu 	for (i = 0; i < 8; i++)
475159dd56fSNeel Natu 		error += guest_msr_ro(vmx, MSR_APIC_IRR0 + i);
476159dd56fSNeel Natu 
477159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ESR);
478159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_TIMER);
479159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_THERMAL);
480159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_PCINT);
481159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT0);
482159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT1);
483159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_ERROR);
484159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ICR_TIMER);
485159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_DCR_TIMER);
486159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ICR);
487159dd56fSNeel Natu 
488159dd56fSNeel Natu 	/*
489159dd56fSNeel Natu 	 * Allow TPR, EOI and SELF_IPI MSRs to be read and written by the guest.
490159dd56fSNeel Natu 	 *
491159dd56fSNeel Natu 	 * These registers get special treatment described in the section
492159dd56fSNeel Natu 	 * "Virtualizing MSR-Based APIC Accesses".
493159dd56fSNeel Natu 	 */
494159dd56fSNeel Natu 	error += guest_msr_rw(vmx, MSR_APIC_TPR);
495159dd56fSNeel Natu 	error += guest_msr_rw(vmx, MSR_APIC_EOI);
496159dd56fSNeel Natu 	error += guest_msr_rw(vmx, MSR_APIC_SELF_IPI);
497159dd56fSNeel Natu 
498159dd56fSNeel Natu 	return (error);
499159dd56fSNeel Natu }
500159dd56fSNeel Natu 
501366f6083SPeter Grehan u_long
502366f6083SPeter Grehan vmx_fix_cr0(u_long cr0)
503366f6083SPeter Grehan {
504366f6083SPeter Grehan 
505366f6083SPeter Grehan 	return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask);
506366f6083SPeter Grehan }
507366f6083SPeter Grehan 
508366f6083SPeter Grehan u_long
509366f6083SPeter Grehan vmx_fix_cr4(u_long cr4)
510366f6083SPeter Grehan {
511366f6083SPeter Grehan 
512366f6083SPeter Grehan 	return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask);
513366f6083SPeter Grehan }
514366f6083SPeter Grehan 
515366f6083SPeter Grehan static void
51645e51299SNeel Natu vpid_free(int vpid)
51745e51299SNeel Natu {
51845e51299SNeel Natu 	if (vpid < 0 || vpid > 0xffff)
51945e51299SNeel Natu 		panic("vpid_free: invalid vpid %d", vpid);
52045e51299SNeel Natu 
52145e51299SNeel Natu 	/*
522ee98f99dSJohn Baldwin 	 * VPIDs [0,vm_maxcpu] are special and are not allocated from
52345e51299SNeel Natu 	 * the unit number allocator.
52445e51299SNeel Natu 	 */
52545e51299SNeel Natu 
526ee98f99dSJohn Baldwin 	if (vpid > vm_maxcpu)
52745e51299SNeel Natu 		free_unr(vpid_unr, vpid);
52845e51299SNeel Natu }
52945e51299SNeel Natu 
53058eefc67SJohn Baldwin static uint16_t
53158eefc67SJohn Baldwin vpid_alloc(int vcpuid)
53245e51299SNeel Natu {
53358eefc67SJohn Baldwin 	int x;
53445e51299SNeel Natu 
53545e51299SNeel Natu 	/*
53645e51299SNeel Natu 	 * If the "enable vpid" execution control is not enabled then the
53745e51299SNeel Natu 	 * VPID is required to be 0 for all vcpus.
53845e51299SNeel Natu 	 */
53958eefc67SJohn Baldwin 	if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) == 0)
54058eefc67SJohn Baldwin 		return (0);
54145e51299SNeel Natu 
54245e51299SNeel Natu 	/*
54358eefc67SJohn Baldwin 	 * Try to allocate a unique VPID for each from the unit number
54458eefc67SJohn Baldwin 	 * allocator.
54545e51299SNeel Natu 	 */
54645e51299SNeel Natu 	x = alloc_unr(vpid_unr);
54745e51299SNeel Natu 
54858eefc67SJohn Baldwin 	if (x == -1) {
54945e51299SNeel Natu 		atomic_add_int(&vpid_alloc_failed, 1);
55045e51299SNeel Natu 
55145e51299SNeel Natu 		/*
55245e51299SNeel Natu 		 * If the unit number allocator does not have enough unique
553ee98f99dSJohn Baldwin 		 * VPIDs then we need to allocate from the [1,vm_maxcpu] range.
55445e51299SNeel Natu 		 *
55545e51299SNeel Natu 		 * These VPIDs are not be unique across VMs but this does not
55645e51299SNeel Natu 		 * affect correctness because the combined mappings are also
55745e51299SNeel Natu 		 * tagged with the EP4TA which is unique for each VM.
55845e51299SNeel Natu 		 *
55945e51299SNeel Natu 		 * It is still sub-optimal because the invvpid will invalidate
56045e51299SNeel Natu 		 * combined mappings for a particular VPID across all EP4TAs.
56145e51299SNeel Natu 		 */
56258eefc67SJohn Baldwin 		return (vcpuid + 1);
56345e51299SNeel Natu 	}
56458eefc67SJohn Baldwin 
56558eefc67SJohn Baldwin 	return (x);
56645e51299SNeel Natu }
56745e51299SNeel Natu 
56845e51299SNeel Natu static void
56945e51299SNeel Natu vpid_init(void)
57045e51299SNeel Natu {
57145e51299SNeel Natu 	/*
57245e51299SNeel Natu 	 * VPID 0 is required when the "enable VPID" execution control is
57345e51299SNeel Natu 	 * disabled.
57445e51299SNeel Natu 	 *
575ee98f99dSJohn Baldwin 	 * VPIDs [1,vm_maxcpu] are used as the "overflow namespace" when the
57645e51299SNeel Natu 	 * unit number allocator does not have sufficient unique VPIDs to
57745e51299SNeel Natu 	 * satisfy the allocation.
57845e51299SNeel Natu 	 *
57945e51299SNeel Natu 	 * The remaining VPIDs are managed by the unit number allocator.
58045e51299SNeel Natu 	 */
581ee98f99dSJohn Baldwin 	vpid_unr = new_unrhdr(vm_maxcpu + 1, 0xffff, NULL);
58245e51299SNeel Natu }
58345e51299SNeel Natu 
58445e51299SNeel Natu static void
585366f6083SPeter Grehan vmx_disable(void *arg __unused)
586366f6083SPeter Grehan {
587366f6083SPeter Grehan 	struct invvpid_desc invvpid_desc = { 0 };
588366f6083SPeter Grehan 	struct invept_desc invept_desc = { 0 };
589366f6083SPeter Grehan 
590366f6083SPeter Grehan 	if (vmxon_enabled[curcpu]) {
591366f6083SPeter Grehan 		/*
592366f6083SPeter Grehan 		 * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b.
593366f6083SPeter Grehan 		 *
594366f6083SPeter Grehan 		 * VMXON or VMXOFF are not required to invalidate any TLB
595366f6083SPeter Grehan 		 * caching structures. This prevents potential retention of
596366f6083SPeter Grehan 		 * cached information in the TLB between distinct VMX episodes.
597366f6083SPeter Grehan 		 */
598366f6083SPeter Grehan 		invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc);
599366f6083SPeter Grehan 		invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc);
600366f6083SPeter Grehan 		vmxoff();
601366f6083SPeter Grehan 	}
602366f6083SPeter Grehan 	load_cr4(rcr4() & ~CR4_VMXE);
603366f6083SPeter Grehan }
604366f6083SPeter Grehan 
605366f6083SPeter Grehan static int
60615add60dSPeter Grehan vmx_modcleanup(void)
607366f6083SPeter Grehan {
608366f6083SPeter Grehan 
60918a2b08eSNeel Natu 	if (pirvec >= 0)
61018a2b08eSNeel Natu 		lapic_ipi_free(pirvec);
611176666c2SNeel Natu 
61245e51299SNeel Natu 	if (vpid_unr != NULL) {
61345e51299SNeel Natu 		delete_unrhdr(vpid_unr);
61445e51299SNeel Natu 		vpid_unr = NULL;
61545e51299SNeel Natu 	}
61645e51299SNeel Natu 
617c1141fbaSKonstantin Belousov 	if (nmi_flush_l1d_sw == 1)
618c1141fbaSKonstantin Belousov 		nmi_flush_l1d_sw = 0;
619c1141fbaSKonstantin Belousov 
620366f6083SPeter Grehan 	smp_rendezvous(NULL, vmx_disable, NULL, NULL);
621366f6083SPeter Grehan 
622366f6083SPeter Grehan 	return (0);
623366f6083SPeter Grehan }
624366f6083SPeter Grehan 
625366f6083SPeter Grehan static void
626366f6083SPeter Grehan vmx_enable(void *arg __unused)
627366f6083SPeter Grehan {
628366f6083SPeter Grehan 	int error;
62911669a68STycho Nightingale 	uint64_t feature_control;
63011669a68STycho Nightingale 
63111669a68STycho Nightingale 	feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
63211669a68STycho Nightingale 	if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 0 ||
63311669a68STycho Nightingale 	    (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) {
63411669a68STycho Nightingale 		wrmsr(MSR_IA32_FEATURE_CONTROL,
63511669a68STycho Nightingale 		    feature_control | IA32_FEATURE_CONTROL_VMX_EN |
63611669a68STycho Nightingale 		    IA32_FEATURE_CONTROL_LOCK);
63711669a68STycho Nightingale 	}
638366f6083SPeter Grehan 
639366f6083SPeter Grehan 	load_cr4(rcr4() | CR4_VMXE);
640366f6083SPeter Grehan 
641366f6083SPeter Grehan 	*(uint32_t *)vmxon_region[curcpu] = vmx_revision();
642366f6083SPeter Grehan 	error = vmxon(vmxon_region[curcpu]);
643366f6083SPeter Grehan 	if (error == 0)
644366f6083SPeter Grehan 		vmxon_enabled[curcpu] = 1;
645366f6083SPeter Grehan }
646366f6083SPeter Grehan 
64763e62d39SJohn Baldwin static void
64815add60dSPeter Grehan vmx_modresume(void)
64963e62d39SJohn Baldwin {
65063e62d39SJohn Baldwin 
65163e62d39SJohn Baldwin 	if (vmxon_enabled[curcpu])
65263e62d39SJohn Baldwin 		vmxon(vmxon_region[curcpu]);
65363e62d39SJohn Baldwin }
65463e62d39SJohn Baldwin 
655366f6083SPeter Grehan static int
65615add60dSPeter Grehan vmx_modinit(int ipinum)
657366f6083SPeter Grehan {
6581bc51badSMichael Reifenberger 	int error;
659d17b5104SNeel Natu 	uint64_t basic, fixed0, fixed1, feature_control;
66088c4b8d1SNeel Natu 	uint32_t tmp, procbased2_vid_bits;
661366f6083SPeter Grehan 
662366f6083SPeter Grehan 	/* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */
6638b287612SJohn Baldwin 	if (!(cpu_feature2 & CPUID2_VMX)) {
66415add60dSPeter Grehan 		printf("vmx_modinit: processor does not support VMX "
66515add60dSPeter Grehan 		    "operation\n");
666366f6083SPeter Grehan 		return (ENXIO);
667366f6083SPeter Grehan 	}
668366f6083SPeter Grehan 
6694bff7fadSNeel Natu 	/*
6704bff7fadSNeel Natu 	 * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits
6714bff7fadSNeel Natu 	 * are set (bits 0 and 2 respectively).
6724bff7fadSNeel Natu 	 */
6734bff7fadSNeel Natu 	feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
67411669a68STycho Nightingale 	if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 1 &&
675150369abSNeel Natu 	    (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) {
67615add60dSPeter Grehan 		printf("vmx_modinit: VMX operation disabled by BIOS\n");
6774bff7fadSNeel Natu 		return (ENXIO);
6784bff7fadSNeel Natu 	}
6794bff7fadSNeel Natu 
680d17b5104SNeel Natu 	/*
681d17b5104SNeel Natu 	 * Verify capabilities MSR_VMX_BASIC:
682d17b5104SNeel Natu 	 * - bit 54 indicates support for INS/OUTS decoding
683d17b5104SNeel Natu 	 */
684d17b5104SNeel Natu 	basic = rdmsr(MSR_VMX_BASIC);
685d17b5104SNeel Natu 	if ((basic & (1UL << 54)) == 0) {
68615add60dSPeter Grehan 		printf("vmx_modinit: processor does not support desired basic "
687d17b5104SNeel Natu 		    "capabilities\n");
688d17b5104SNeel Natu 		return (EINVAL);
689d17b5104SNeel Natu 	}
690d17b5104SNeel Natu 
691366f6083SPeter Grehan 	/* Check support for primary processor-based VM-execution controls */
692366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
693366f6083SPeter Grehan 			       MSR_VMX_TRUE_PROCBASED_CTLS,
694366f6083SPeter Grehan 			       PROCBASED_CTLS_ONE_SETTING,
695366f6083SPeter Grehan 			       PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls);
696366f6083SPeter Grehan 	if (error) {
69715add60dSPeter Grehan 		printf("vmx_modinit: processor does not support desired "
69815add60dSPeter Grehan 		    "primary processor-based controls\n");
699366f6083SPeter Grehan 		return (error);
700366f6083SPeter Grehan 	}
701366f6083SPeter Grehan 
702366f6083SPeter Grehan 	/* Clear the processor-based ctl bits that are set on demand */
703366f6083SPeter Grehan 	procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING;
704366f6083SPeter Grehan 
705366f6083SPeter Grehan 	/* Check support for secondary processor-based VM-execution controls */
706366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
707366f6083SPeter Grehan 			       MSR_VMX_PROCBASED_CTLS2,
708366f6083SPeter Grehan 			       PROCBASED_CTLS2_ONE_SETTING,
709366f6083SPeter Grehan 			       PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2);
710366f6083SPeter Grehan 	if (error) {
71115add60dSPeter Grehan 		printf("vmx_modinit: processor does not support desired "
71215add60dSPeter Grehan 		    "secondary processor-based controls\n");
713366f6083SPeter Grehan 		return (error);
714366f6083SPeter Grehan 	}
715366f6083SPeter Grehan 
716366f6083SPeter Grehan 	/* Check support for VPID */
717366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
718366f6083SPeter Grehan 			       PROCBASED2_ENABLE_VPID, 0, &tmp);
719366f6083SPeter Grehan 	if (error == 0)
720366f6083SPeter Grehan 		procbased_ctls2 |= PROCBASED2_ENABLE_VPID;
721366f6083SPeter Grehan 
722366f6083SPeter Grehan 	/* Check support for pin-based VM-execution controls */
723366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
724366f6083SPeter Grehan 			       MSR_VMX_TRUE_PINBASED_CTLS,
725366f6083SPeter Grehan 			       PINBASED_CTLS_ONE_SETTING,
726366f6083SPeter Grehan 			       PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls);
727366f6083SPeter Grehan 	if (error) {
72815add60dSPeter Grehan 		printf("vmx_modinit: processor does not support desired "
729366f6083SPeter Grehan 		    "pin-based controls\n");
730366f6083SPeter Grehan 		return (error);
731366f6083SPeter Grehan 	}
732366f6083SPeter Grehan 
733366f6083SPeter Grehan 	/* Check support for VM-exit controls */
734366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS,
735366f6083SPeter Grehan 			       VM_EXIT_CTLS_ONE_SETTING,
736366f6083SPeter Grehan 			       VM_EXIT_CTLS_ZERO_SETTING,
737366f6083SPeter Grehan 			       &exit_ctls);
738366f6083SPeter Grehan 	if (error) {
73915add60dSPeter Grehan 		printf("vmx_modinit: processor does not support desired "
740366f6083SPeter Grehan 		    "exit controls\n");
741366f6083SPeter Grehan 		return (error);
742366f6083SPeter Grehan 	}
743366f6083SPeter Grehan 
744366f6083SPeter Grehan 	/* Check support for VM-entry controls */
745d72978ecSNeel Natu 	error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS, MSR_VMX_TRUE_ENTRY_CTLS,
746d72978ecSNeel Natu 	    VM_ENTRY_CTLS_ONE_SETTING, VM_ENTRY_CTLS_ZERO_SETTING,
747366f6083SPeter Grehan 	    &entry_ctls);
748366f6083SPeter Grehan 	if (error) {
74915add60dSPeter Grehan 		printf("vmx_modinit: processor does not support desired "
750366f6083SPeter Grehan 		    "entry controls\n");
751366f6083SPeter Grehan 		return (error);
752366f6083SPeter Grehan 	}
753366f6083SPeter Grehan 
754366f6083SPeter Grehan 	/*
755366f6083SPeter Grehan 	 * Check support for optional features by testing them
756366f6083SPeter Grehan 	 * as individual bits
757366f6083SPeter Grehan 	 */
758366f6083SPeter Grehan 	cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
759366f6083SPeter Grehan 					MSR_VMX_TRUE_PROCBASED_CTLS,
760366f6083SPeter Grehan 					PROCBASED_HLT_EXITING, 0,
761366f6083SPeter Grehan 					&tmp) == 0);
762366f6083SPeter Grehan 
763366f6083SPeter Grehan 	cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
764366f6083SPeter Grehan 					MSR_VMX_PROCBASED_CTLS,
765366f6083SPeter Grehan 					PROCBASED_MTF, 0,
766366f6083SPeter Grehan 					&tmp) == 0);
767366f6083SPeter Grehan 
768366f6083SPeter Grehan 	cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
769366f6083SPeter Grehan 					 MSR_VMX_TRUE_PROCBASED_CTLS,
770366f6083SPeter Grehan 					 PROCBASED_PAUSE_EXITING, 0,
771366f6083SPeter Grehan 					 &tmp) == 0);
772366f6083SPeter Grehan 
7733ba952e1SCorvin Köhne 	cap_wbinvd_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
7743ba952e1SCorvin Köhne 					MSR_VMX_PROCBASED_CTLS2,
7753ba952e1SCorvin Köhne 					PROCBASED2_WBINVD_EXITING,
7763ba952e1SCorvin Köhne 					0,
7773ba952e1SCorvin Köhne 					&tmp) == 0);
7783ba952e1SCorvin Köhne 
779f5f5f1e7SPeter Grehan 	/*
780f5f5f1e7SPeter Grehan 	 * Check support for RDPID and/or RDTSCP.
781f5f5f1e7SPeter Grehan 	 *
782f5f5f1e7SPeter Grehan 	 * Support a pass-through-based implementation of these via the
783f5f5f1e7SPeter Grehan 	 * "enable RDTSCP" VM-execution control and the "RDTSC exiting"
784f5f5f1e7SPeter Grehan 	 * VM-execution control.
785f5f5f1e7SPeter Grehan 	 *
786f5f5f1e7SPeter Grehan 	 * The "enable RDTSCP" VM-execution control applies to both RDPID
787f5f5f1e7SPeter Grehan 	 * and RDTSCP (see SDM volume 3, section 25.3, "Changes to
788f5f5f1e7SPeter Grehan 	 * Instruction Behavior in VMX Non-root operation"); this is why
789f5f5f1e7SPeter Grehan 	 * only this VM-execution control needs to be enabled in order to
790f5f5f1e7SPeter Grehan 	 * enable passing through whichever of RDPID and/or RDTSCP are
791f5f5f1e7SPeter Grehan 	 * supported by the host.
792f5f5f1e7SPeter Grehan 	 *
793f5f5f1e7SPeter Grehan 	 * The "RDTSC exiting" VM-execution control applies to both RDTSC
794f5f5f1e7SPeter Grehan 	 * and RDTSCP (again, per SDM volume 3, section 25.3), and is
795f5f5f1e7SPeter Grehan 	 * already set up for RDTSC and RDTSCP pass-through by the current
796f5f5f1e7SPeter Grehan 	 * implementation of RDTSC.
797f5f5f1e7SPeter Grehan 	 *
798f5f5f1e7SPeter Grehan 	 * Although RDPID and RDTSCP are optional capabilities, since there
799f5f5f1e7SPeter Grehan 	 * does not currently seem to be a use case for enabling/disabling
800f5f5f1e7SPeter Grehan 	 * these via libvmmapi, choose not to support this and, instead,
801f5f5f1e7SPeter Grehan 	 * just statically always enable or always disable this support
802f5f5f1e7SPeter Grehan 	 * across all vCPUs on all VMs. (Note that there may be some
803f5f5f1e7SPeter Grehan 	 * complications to providing this functionality, e.g., the MSR
804f5f5f1e7SPeter Grehan 	 * bitmap is currently per-VM rather than per-vCPU while the
805f5f5f1e7SPeter Grehan 	 * capability API wants to be able to control capabilities on a
806f5f5f1e7SPeter Grehan 	 * per-vCPU basis).
807f5f5f1e7SPeter Grehan 	 */
808f5f5f1e7SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
809f5f5f1e7SPeter Grehan 			       MSR_VMX_PROCBASED_CTLS2,
810f5f5f1e7SPeter Grehan 			       PROCBASED2_ENABLE_RDTSCP, 0, &tmp);
811f5f5f1e7SPeter Grehan 	cap_rdpid = error == 0 && host_has_rdpid();
812f5f5f1e7SPeter Grehan 	cap_rdtscp = error == 0 && host_has_rdtscp();
81373abae44SJohn Baldwin 	if (cap_rdpid || cap_rdtscp) {
814f5f5f1e7SPeter Grehan 		procbased_ctls2 |= PROCBASED2_ENABLE_RDTSCP;
81573abae44SJohn Baldwin 		vmx_have_msr_tsc_aux = true;
81673abae44SJohn Baldwin 	}
817f5f5f1e7SPeter Grehan 
818366f6083SPeter Grehan 	cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
819366f6083SPeter Grehan 					MSR_VMX_PROCBASED_CTLS2,
820366f6083SPeter Grehan 					PROCBASED2_UNRESTRICTED_GUEST, 0,
821366f6083SPeter Grehan 				        &tmp) == 0);
822366f6083SPeter Grehan 
82349cc03daSNeel Natu 	cap_invpcid = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
82449cc03daSNeel Natu 	    MSR_VMX_PROCBASED_CTLS2, PROCBASED2_ENABLE_INVPCID, 0,
82549cc03daSNeel Natu 	    &tmp) == 0);
82649cc03daSNeel Natu 
82788c4b8d1SNeel Natu 	/*
8281bc51badSMichael Reifenberger 	 * Check support for TPR shadow.
8291bc51badSMichael Reifenberger 	 */
8301bc51badSMichael Reifenberger 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
8311bc51badSMichael Reifenberger 	    MSR_VMX_TRUE_PROCBASED_CTLS, PROCBASED_USE_TPR_SHADOW, 0,
8321bc51badSMichael Reifenberger 	    &tmp);
8331bc51badSMichael Reifenberger 	if (error == 0) {
8341bc51badSMichael Reifenberger 		tpr_shadowing = 1;
8351bc51badSMichael Reifenberger 		TUNABLE_INT_FETCH("hw.vmm.vmx.use_tpr_shadowing",
8361bc51badSMichael Reifenberger 		    &tpr_shadowing);
8371bc51badSMichael Reifenberger 	}
8381bc51badSMichael Reifenberger 
8391bc51badSMichael Reifenberger 	if (tpr_shadowing) {
8401bc51badSMichael Reifenberger 		procbased_ctls |= PROCBASED_USE_TPR_SHADOW;
8411bc51badSMichael Reifenberger 		procbased_ctls &= ~PROCBASED_CR8_LOAD_EXITING;
8421bc51badSMichael Reifenberger 		procbased_ctls &= ~PROCBASED_CR8_STORE_EXITING;
8431bc51badSMichael Reifenberger 	}
8441bc51badSMichael Reifenberger 
8451bc51badSMichael Reifenberger 	/*
84688c4b8d1SNeel Natu 	 * Check support for virtual interrupt delivery.
84788c4b8d1SNeel Natu 	 */
84888c4b8d1SNeel Natu 	procbased2_vid_bits = (PROCBASED2_VIRTUALIZE_APIC_ACCESSES |
84988c4b8d1SNeel Natu 	    PROCBASED2_VIRTUALIZE_X2APIC_MODE |
85088c4b8d1SNeel Natu 	    PROCBASED2_APIC_REGISTER_VIRTUALIZATION |
85188c4b8d1SNeel Natu 	    PROCBASED2_VIRTUAL_INTERRUPT_DELIVERY);
85288c4b8d1SNeel Natu 
85388c4b8d1SNeel Natu 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
85488c4b8d1SNeel Natu 	    procbased2_vid_bits, 0, &tmp);
8551bc51badSMichael Reifenberger 	if (error == 0 && tpr_shadowing) {
85688c4b8d1SNeel Natu 		virtual_interrupt_delivery = 1;
85788c4b8d1SNeel Natu 		TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_vid",
85888c4b8d1SNeel Natu 		    &virtual_interrupt_delivery);
85988c4b8d1SNeel Natu 	}
86088c4b8d1SNeel Natu 
86188c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
86288c4b8d1SNeel Natu 		procbased_ctls |= PROCBASED_USE_TPR_SHADOW;
86388c4b8d1SNeel Natu 		procbased_ctls2 |= procbased2_vid_bits;
86488c4b8d1SNeel Natu 		procbased_ctls2 &= ~PROCBASED2_VIRTUALIZE_X2APIC_MODE;
865176666c2SNeel Natu 
866176666c2SNeel Natu 		/*
867176666c2SNeel Natu 		 * Check for Posted Interrupts only if Virtual Interrupt
868176666c2SNeel Natu 		 * Delivery is enabled.
869176666c2SNeel Natu 		 */
870176666c2SNeel Natu 		error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
871176666c2SNeel Natu 		    MSR_VMX_TRUE_PINBASED_CTLS, PINBASED_POSTED_INTERRUPT, 0,
872176666c2SNeel Natu 		    &tmp);
873176666c2SNeel Natu 		if (error == 0) {
874bd50262fSKonstantin Belousov 			pirvec = lapic_ipi_alloc(pti ? &IDTVEC(justreturn1_pti) :
875bd50262fSKonstantin Belousov 			    &IDTVEC(justreturn));
87618a2b08eSNeel Natu 			if (pirvec < 0) {
877176666c2SNeel Natu 				if (bootverbose) {
87815add60dSPeter Grehan 					printf("vmx_modinit: unable to "
87915add60dSPeter Grehan 					    "allocate posted interrupt "
88015add60dSPeter Grehan 					    "vector\n");
88188c4b8d1SNeel Natu 				}
882176666c2SNeel Natu 			} else {
883176666c2SNeel Natu 				posted_interrupts = 1;
884176666c2SNeel Natu 				TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_pir",
885176666c2SNeel Natu 				    &posted_interrupts);
886176666c2SNeel Natu 			}
887176666c2SNeel Natu 		}
888176666c2SNeel Natu 	}
889176666c2SNeel Natu 
890176666c2SNeel Natu 	if (posted_interrupts)
891176666c2SNeel Natu 		    pinbased_ctls |= PINBASED_POSTED_INTERRUPT;
89249cc03daSNeel Natu 
893366f6083SPeter Grehan 	/* Initialize EPT */
894add611fdSNeel Natu 	error = ept_init(ipinum);
895366f6083SPeter Grehan 	if (error) {
89615add60dSPeter Grehan 		printf("vmx_modinit: ept initialization failed (%d)\n", error);
897366f6083SPeter Grehan 		return (error);
898366f6083SPeter Grehan 	}
899366f6083SPeter Grehan 
90023437573SKonstantin Belousov 	guest_l1d_flush = (cpu_ia32_arch_caps &
90123437573SKonstantin Belousov 	    IA32_ARCH_CAP_SKIP_L1DFL_VMENTRY) == 0;
902c30578feSKonstantin Belousov 	TUNABLE_INT_FETCH("hw.vmm.l1d_flush", &guest_l1d_flush);
903c1141fbaSKonstantin Belousov 
904c1141fbaSKonstantin Belousov 	/*
905c1141fbaSKonstantin Belousov 	 * L1D cache flush is enabled.  Use IA32_FLUSH_CMD MSR when
906c1141fbaSKonstantin Belousov 	 * available.  Otherwise fall back to the software flush
907c1141fbaSKonstantin Belousov 	 * method which loads enough data from the kernel text to
908c1141fbaSKonstantin Belousov 	 * flush existing L1D content, both on VMX entry and on NMI
909c1141fbaSKonstantin Belousov 	 * return.
910c1141fbaSKonstantin Belousov 	 */
911c1141fbaSKonstantin Belousov 	if (guest_l1d_flush) {
912c1141fbaSKonstantin Belousov 		if ((cpu_stdext_feature3 & CPUID_STDEXT3_L1D_FLUSH) == 0) {
913c1141fbaSKonstantin Belousov 			guest_l1d_flush_sw = 1;
914c1141fbaSKonstantin Belousov 			TUNABLE_INT_FETCH("hw.vmm.l1d_flush_sw",
915c1141fbaSKonstantin Belousov 			    &guest_l1d_flush_sw);
916c1141fbaSKonstantin Belousov 		}
917c1141fbaSKonstantin Belousov 		if (guest_l1d_flush_sw) {
918c1141fbaSKonstantin Belousov 			if (nmi_flush_l1d_sw <= 1)
919c1141fbaSKonstantin Belousov 				nmi_flush_l1d_sw = 1;
920c1141fbaSKonstantin Belousov 		} else {
921c1141fbaSKonstantin Belousov 			msr_load_list[0].index = MSR_IA32_FLUSH_CMD;
922c1141fbaSKonstantin Belousov 			msr_load_list[0].val = IA32_FLUSH_CMD_L1D;
923c1141fbaSKonstantin Belousov 		}
924c1141fbaSKonstantin Belousov 	}
925c30578feSKonstantin Belousov 
926366f6083SPeter Grehan 	/*
927366f6083SPeter Grehan 	 * Stash the cr0 and cr4 bits that must be fixed to 0 or 1
928366f6083SPeter Grehan 	 */
929366f6083SPeter Grehan 	fixed0 = rdmsr(MSR_VMX_CR0_FIXED0);
930366f6083SPeter Grehan 	fixed1 = rdmsr(MSR_VMX_CR0_FIXED1);
931366f6083SPeter Grehan 	cr0_ones_mask = fixed0 & fixed1;
932366f6083SPeter Grehan 	cr0_zeros_mask = ~fixed0 & ~fixed1;
933366f6083SPeter Grehan 
934366f6083SPeter Grehan 	/*
935366f6083SPeter Grehan 	 * CR0_PE and CR0_PG can be set to zero in VMX non-root operation
936366f6083SPeter Grehan 	 * if unrestricted guest execution is allowed.
937366f6083SPeter Grehan 	 */
938366f6083SPeter Grehan 	if (cap_unrestricted_guest)
939366f6083SPeter Grehan 		cr0_ones_mask &= ~(CR0_PG | CR0_PE);
940366f6083SPeter Grehan 
941366f6083SPeter Grehan 	/*
942366f6083SPeter Grehan 	 * Do not allow the guest to set CR0_NW or CR0_CD.
943366f6083SPeter Grehan 	 */
944366f6083SPeter Grehan 	cr0_zeros_mask |= (CR0_NW | CR0_CD);
945366f6083SPeter Grehan 
946366f6083SPeter Grehan 	fixed0 = rdmsr(MSR_VMX_CR4_FIXED0);
947366f6083SPeter Grehan 	fixed1 = rdmsr(MSR_VMX_CR4_FIXED1);
948366f6083SPeter Grehan 	cr4_ones_mask = fixed0 & fixed1;
949366f6083SPeter Grehan 	cr4_zeros_mask = ~fixed0 & ~fixed1;
950366f6083SPeter Grehan 
95145e51299SNeel Natu 	vpid_init();
95245e51299SNeel Natu 
953c3498942SNeel Natu 	vmx_msr_init();
954c3498942SNeel Natu 
955366f6083SPeter Grehan 	/* enable VMX operation */
956366f6083SPeter Grehan 	smp_rendezvous(NULL, vmx_enable, NULL, NULL);
957366f6083SPeter Grehan 
9583565b59eSNeel Natu 	vmx_initialized = 1;
9593565b59eSNeel Natu 
960366f6083SPeter Grehan 	return (0);
961366f6083SPeter Grehan }
962366f6083SPeter Grehan 
963f7d47425SNeel Natu static void
964f7d47425SNeel Natu vmx_trigger_hostintr(int vector)
965f7d47425SNeel Natu {
966f7d47425SNeel Natu 	uintptr_t func;
967f7d47425SNeel Natu 	struct gate_descriptor *gd;
968f7d47425SNeel Natu 
969f7d47425SNeel Natu 	gd = &idt[vector];
970f7d47425SNeel Natu 
971f7d47425SNeel Natu 	KASSERT(vector >= 32 && vector <= 255, ("vmx_trigger_hostintr: "
972f7d47425SNeel Natu 	    "invalid vector %d", vector));
973f7d47425SNeel Natu 	KASSERT(gd->gd_p == 1, ("gate descriptor for vector %d not present",
974f7d47425SNeel Natu 	    vector));
975f7d47425SNeel Natu 	KASSERT(gd->gd_type == SDT_SYSIGT, ("gate descriptor for vector %d "
976f7d47425SNeel Natu 	    "has invalid type %d", vector, gd->gd_type));
977f7d47425SNeel Natu 	KASSERT(gd->gd_dpl == SEL_KPL, ("gate descriptor for vector %d "
978f7d47425SNeel Natu 	    "has invalid dpl %d", vector, gd->gd_dpl));
979f7d47425SNeel Natu 	KASSERT(gd->gd_selector == GSEL(GCODE_SEL, SEL_KPL), ("gate descriptor "
980f7d47425SNeel Natu 	    "for vector %d has invalid selector %d", vector, gd->gd_selector));
981f7d47425SNeel Natu 	KASSERT(gd->gd_ist == 0, ("gate descriptor for vector %d has invalid "
982f7d47425SNeel Natu 	    "IST %d", vector, gd->gd_ist));
983f7d47425SNeel Natu 
984f7d47425SNeel Natu 	func = ((long)gd->gd_hioffset << 16 | gd->gd_looffset);
985f7d47425SNeel Natu 	vmx_call_isr(func);
986f7d47425SNeel Natu }
987f7d47425SNeel Natu 
988366f6083SPeter Grehan static int
989aaaa0656SPeter Grehan vmx_setup_cr_shadow(int which, struct vmcs *vmcs, uint32_t initial)
990366f6083SPeter Grehan {
99139c21c2dSNeel Natu 	int error, mask_ident, shadow_ident;
992aaaa0656SPeter Grehan 	uint64_t mask_value;
993366f6083SPeter Grehan 
99439c21c2dSNeel Natu 	if (which != 0 && which != 4)
99539c21c2dSNeel Natu 		panic("vmx_setup_cr_shadow: unknown cr%d", which);
99639c21c2dSNeel Natu 
99739c21c2dSNeel Natu 	if (which == 0) {
99839c21c2dSNeel Natu 		mask_ident = VMCS_CR0_MASK;
99939c21c2dSNeel Natu 		mask_value = cr0_ones_mask | cr0_zeros_mask;
100039c21c2dSNeel Natu 		shadow_ident = VMCS_CR0_SHADOW;
100139c21c2dSNeel Natu 	} else {
100239c21c2dSNeel Natu 		mask_ident = VMCS_CR4_MASK;
100339c21c2dSNeel Natu 		mask_value = cr4_ones_mask | cr4_zeros_mask;
100439c21c2dSNeel Natu 		shadow_ident = VMCS_CR4_SHADOW;
100539c21c2dSNeel Natu 	}
100639c21c2dSNeel Natu 
1007d3c11f40SPeter Grehan 	error = vmcs_setreg(vmcs, 0, VMCS_IDENT(mask_ident), mask_value);
1008366f6083SPeter Grehan 	if (error)
1009366f6083SPeter Grehan 		return (error);
1010366f6083SPeter Grehan 
1011aaaa0656SPeter Grehan 	error = vmcs_setreg(vmcs, 0, VMCS_IDENT(shadow_ident), initial);
1012366f6083SPeter Grehan 	if (error)
1013366f6083SPeter Grehan 		return (error);
1014366f6083SPeter Grehan 
1015366f6083SPeter Grehan 	return (0);
1016366f6083SPeter Grehan }
1017aaaa0656SPeter Grehan #define	vmx_setup_cr0_shadow(vmcs,init)	vmx_setup_cr_shadow(0, (vmcs), (init))
1018aaaa0656SPeter Grehan #define	vmx_setup_cr4_shadow(vmcs,init)	vmx_setup_cr_shadow(4, (vmcs), (init))
1019366f6083SPeter Grehan 
1020366f6083SPeter Grehan static void *
102115add60dSPeter Grehan vmx_init(struct vm *vm, pmap_t pmap)
1022366f6083SPeter Grehan {
1023d487cba3SCy Schubert 	int error __diagused;
1024366f6083SPeter Grehan 	struct vmx *vmx;
1025366f6083SPeter Grehan 
1026366f6083SPeter Grehan 	vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO);
1027366f6083SPeter Grehan 	vmx->vm = vm;
1028366f6083SPeter Grehan 
10299ce875d9SKonstantin Belousov 	vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pmltop));
1030318224bbSNeel Natu 
1031366f6083SPeter Grehan 	/*
1032366f6083SPeter Grehan 	 * Clean up EPTP-tagged guest physical and combined mappings
1033366f6083SPeter Grehan 	 *
1034366f6083SPeter Grehan 	 * VMX transitions are not required to invalidate any guest physical
1035366f6083SPeter Grehan 	 * mappings. So, it may be possible for stale guest physical mappings
1036366f6083SPeter Grehan 	 * to be present in the processor TLBs.
1037366f6083SPeter Grehan 	 *
1038366f6083SPeter Grehan 	 * Combined mappings for this EP4TA are also invalidated for all VPIDs.
1039366f6083SPeter Grehan 	 */
1040318224bbSNeel Natu 	ept_invalidate_mappings(vmx->eptp);
1041366f6083SPeter Grehan 
10420f00260cSJohn Baldwin 	vmx->msr_bitmap = malloc_aligned(PAGE_SIZE, PAGE_SIZE, M_VMX,
10430f00260cSJohn Baldwin 	    M_WAITOK | M_ZERO);
1044366f6083SPeter Grehan 	msr_bitmap_initialize(vmx->msr_bitmap);
1045366f6083SPeter Grehan 
1046366f6083SPeter Grehan 	/*
1047366f6083SPeter Grehan 	 * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE.
1048366f6083SPeter Grehan 	 * The guest FSBASE and GSBASE are saved and restored during
1049366f6083SPeter Grehan 	 * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are
1050366f6083SPeter Grehan 	 * always restored from the vmcs host state area on vm-exit.
1051366f6083SPeter Grehan 	 *
10521fb0ea3fSPeter Grehan 	 * The SYSENTER_CS/ESP/EIP MSRs are identical to FS/GSBASE in
10531fb0ea3fSPeter Grehan 	 * how they are saved/restored so can be directly accessed by the
10541fb0ea3fSPeter Grehan 	 * guest.
10551fb0ea3fSPeter Grehan 	 *
1056366f6083SPeter Grehan 	 * MSR_EFER is saved and restored in the guest VMCS area on a
1057366f6083SPeter Grehan 	 * VM exit and entry respectively. It is also restored from the
1058366f6083SPeter Grehan 	 * host VMCS area on a VM exit.
10598d1d7a9eSPeter Grehan 	 *
1060277bdd99STycho Nightingale 	 * The TSC MSR is exposed read-only. Writes are disallowed as
1061277bdd99STycho Nightingale 	 * that will impact the host TSC.  If the guest does a write
1062277bdd99STycho Nightingale 	 * the "use TSC offsetting" execution control is enabled and the
1063277bdd99STycho Nightingale 	 * difference between the host TSC and the guest TSC is written
1064277bdd99STycho Nightingale 	 * into the TSC offset in the VMCS.
1065f5f5f1e7SPeter Grehan 	 *
1066f5f5f1e7SPeter Grehan 	 * Guest TSC_AUX support is enabled if any of guest RDPID and/or
1067f5f5f1e7SPeter Grehan 	 * guest RDTSCP support are enabled (since, as per Table 2-2 in SDM
1068f5f5f1e7SPeter Grehan 	 * volume 4, TSC_AUX is supported if any of RDPID and/or RDTSCP are
1069f5f5f1e7SPeter Grehan 	 * supported). If guest TSC_AUX support is enabled, TSC_AUX is
1070f5f5f1e7SPeter Grehan 	 * exposed read-only so that the VMM can do one fewer MSR read per
1071f5f5f1e7SPeter Grehan 	 * exit than if this register were exposed read-write; the guest
1072f5f5f1e7SPeter Grehan 	 * restore value can be updated during guest writes (expected to be
1073f5f5f1e7SPeter Grehan 	 * rare) instead of during all exits (common).
1074366f6083SPeter Grehan 	 */
1075366f6083SPeter Grehan 	if (guest_msr_rw(vmx, MSR_GSBASE) ||
1076366f6083SPeter Grehan 	    guest_msr_rw(vmx, MSR_FSBASE) ||
10771fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_CS_MSR) ||
10781fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_ESP_MSR) ||
10791fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_EIP_MSR) ||
10808d1d7a9eSPeter Grehan 	    guest_msr_rw(vmx, MSR_EFER) ||
1081f5f5f1e7SPeter Grehan 	    guest_msr_ro(vmx, MSR_TSC) ||
1082f5f5f1e7SPeter Grehan 	    ((cap_rdpid || cap_rdtscp) && guest_msr_ro(vmx, MSR_TSC_AUX)))
108315add60dSPeter Grehan 		panic("vmx_init: error setting guest msr access");
1084366f6083SPeter Grehan 
108588c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
108688c4b8d1SNeel Natu 		error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE,
108788c4b8d1SNeel Natu 		    APIC_ACCESS_ADDRESS);
108888c4b8d1SNeel Natu 		/* XXX this should really return an error to the caller */
108988c4b8d1SNeel Natu 		KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error));
109088c4b8d1SNeel Natu 	}
109188c4b8d1SNeel Natu 
10921aa51504SJohn Baldwin 	vmx->pmap = pmap;
10931aa51504SJohn Baldwin 	return (vmx);
10941aa51504SJohn Baldwin }
10950f00260cSJohn Baldwin 
10961aa51504SJohn Baldwin static void *
1097950af9ffSJohn Baldwin vmx_vcpu_init(void *vmi, struct vcpu *vcpu1, int vcpuid)
10981aa51504SJohn Baldwin {
1099869c8d19SJohn Baldwin 	struct vmx *vmx = vmi;
11001aa51504SJohn Baldwin 	struct vmcs *vmcs;
11011aa51504SJohn Baldwin 	struct vmx_vcpu *vcpu;
11021aa51504SJohn Baldwin 	uint32_t exc_bitmap;
110358eefc67SJohn Baldwin 	uint16_t vpid;
11041aa51504SJohn Baldwin 	int error;
11051aa51504SJohn Baldwin 
110658eefc67SJohn Baldwin 	vpid = vpid_alloc(vcpuid);
110758eefc67SJohn Baldwin 
11081aa51504SJohn Baldwin 	vcpu = malloc(sizeof(*vcpu), M_VMX, M_WAITOK | M_ZERO);
1109869c8d19SJohn Baldwin 	vcpu->vmx = vmx;
1110950af9ffSJohn Baldwin 	vcpu->vcpu = vcpu1;
11111aa51504SJohn Baldwin 	vcpu->vcpuid = vcpuid;
11120f00260cSJohn Baldwin 	vcpu->vmcs = malloc_aligned(sizeof(*vmcs), PAGE_SIZE, M_VMX,
11130f00260cSJohn Baldwin 	    M_WAITOK | M_ZERO);
11140f00260cSJohn Baldwin 	vcpu->apic_page = malloc_aligned(PAGE_SIZE, PAGE_SIZE, M_VMX,
11150f00260cSJohn Baldwin 	    M_WAITOK | M_ZERO);
11161aa51504SJohn Baldwin 	vcpu->pir_desc = malloc_aligned(sizeof(*vcpu->pir_desc), 64, M_VMX,
11171aa51504SJohn Baldwin 	    M_WAITOK | M_ZERO);
11180f00260cSJohn Baldwin 
11190f00260cSJohn Baldwin 	vmcs = vcpu->vmcs;
1120c847a506SNeel Natu 	vmcs->identifier = vmx_revision();
1121c847a506SNeel Natu 	error = vmclear(vmcs);
1122366f6083SPeter Grehan 	if (error != 0) {
112315add60dSPeter Grehan 		panic("vmx_init: vmclear error %d on vcpu %d\n",
11241aa51504SJohn Baldwin 		    error, vcpuid);
1125366f6083SPeter Grehan 	}
1126366f6083SPeter Grehan 
11271aa51504SJohn Baldwin 	vmx_msr_guest_init(vmx, vcpu);
1128c3498942SNeel Natu 
1129c847a506SNeel Natu 	error = vmcs_init(vmcs);
1130c847a506SNeel Natu 	KASSERT(error == 0, ("vmcs_init error %d", error));
1131366f6083SPeter Grehan 
1132c847a506SNeel Natu 	VMPTRLD(vmcs);
1133c847a506SNeel Natu 	error = 0;
11340f00260cSJohn Baldwin 	error += vmwrite(VMCS_HOST_RSP, (u_long)&vcpu->ctx);
1135c847a506SNeel Natu 	error += vmwrite(VMCS_EPTP, vmx->eptp);
1136c847a506SNeel Natu 	error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls);
1137c847a506SNeel Natu 	error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls);
113880cb5d84SJohn Baldwin 	if (vcpu_trap_wbinvd(vcpu->vcpu)) {
11393ba952e1SCorvin Köhne 		KASSERT(cap_wbinvd_exit, ("WBINVD trap not available"));
11403ba952e1SCorvin Köhne 		procbased_ctls2 |= PROCBASED2_WBINVD_EXITING;
11413ba952e1SCorvin Köhne 	}
1142c847a506SNeel Natu 	error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2);
1143c847a506SNeel Natu 	error += vmwrite(VMCS_EXIT_CTLS, exit_ctls);
1144c847a506SNeel Natu 	error += vmwrite(VMCS_ENTRY_CTLS, entry_ctls);
1145c847a506SNeel Natu 	error += vmwrite(VMCS_MSR_BITMAP, vtophys(vmx->msr_bitmap));
114658eefc67SJohn Baldwin 	error += vmwrite(VMCS_VPID, vpid);
1147b0538143SNeel Natu 
1148c1141fbaSKonstantin Belousov 	if (guest_l1d_flush && !guest_l1d_flush_sw) {
1149c1141fbaSKonstantin Belousov 		vmcs_write(VMCS_ENTRY_MSR_LOAD, pmap_kextract(
1150c1141fbaSKonstantin Belousov 			(vm_offset_t)&msr_load_list[0]));
1151c1141fbaSKonstantin Belousov 		vmcs_write(VMCS_ENTRY_MSR_LOAD_COUNT,
1152c1141fbaSKonstantin Belousov 		    nitems(msr_load_list));
1153c1141fbaSKonstantin Belousov 		vmcs_write(VMCS_EXIT_MSR_STORE, 0);
1154c1141fbaSKonstantin Belousov 		vmcs_write(VMCS_EXIT_MSR_STORE_COUNT, 0);
1155c1141fbaSKonstantin Belousov 	}
1156c1141fbaSKonstantin Belousov 
1157b0538143SNeel Natu 	/* exception bitmap */
115880cb5d84SJohn Baldwin 	if (vcpu_trace_exceptions(vcpu->vcpu))
1159b0538143SNeel Natu 		exc_bitmap = 0xffffffff;
1160b0538143SNeel Natu 	else
1161b0538143SNeel Natu 		exc_bitmap = 1 << IDT_MC;
1162b0538143SNeel Natu 	error += vmwrite(VMCS_EXCEPTION_BITMAP, exc_bitmap);
1163b0538143SNeel Natu 
11640f00260cSJohn Baldwin 	vcpu->ctx.guest_dr6 = DBREG_DR6_RESERVED1;
11659e2154ffSJohn Baldwin 	error += vmwrite(VMCS_GUEST_DR7, DBREG_DR7_RESERVED1);
116665eefbe4SJohn Baldwin 
11671bc51badSMichael Reifenberger 	if (tpr_shadowing) {
11681aa51504SJohn Baldwin 		error += vmwrite(VMCS_VIRTUAL_APIC, vtophys(vcpu->apic_page));
11691bc51badSMichael Reifenberger 	}
11701bc51badSMichael Reifenberger 
11711bc51badSMichael Reifenberger 	if (virtual_interrupt_delivery) {
11721bc51badSMichael Reifenberger 		error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS);
117388c4b8d1SNeel Natu 		error += vmwrite(VMCS_EOI_EXIT0, 0);
117488c4b8d1SNeel Natu 		error += vmwrite(VMCS_EOI_EXIT1, 0);
117588c4b8d1SNeel Natu 		error += vmwrite(VMCS_EOI_EXIT2, 0);
117688c4b8d1SNeel Natu 		error += vmwrite(VMCS_EOI_EXIT3, 0);
117788c4b8d1SNeel Natu 	}
1178176666c2SNeel Natu 	if (posted_interrupts) {
1179176666c2SNeel Natu 		error += vmwrite(VMCS_PIR_VECTOR, pirvec);
11801aa51504SJohn Baldwin 		error += vmwrite(VMCS_PIR_DESC, vtophys(vcpu->pir_desc));
1181176666c2SNeel Natu 	}
1182c847a506SNeel Natu 	VMCLEAR(vmcs);
118315add60dSPeter Grehan 	KASSERT(error == 0, ("vmx_init: error customizing the vmcs"));
1184366f6083SPeter Grehan 
11850f00260cSJohn Baldwin 	vcpu->cap.set = 0;
11860f00260cSJohn Baldwin 	vcpu->cap.set |= cap_rdpid != 0 ? 1 << VM_CAP_RDPID : 0;
11870f00260cSJohn Baldwin 	vcpu->cap.set |= cap_rdtscp != 0 ? 1 << VM_CAP_RDTSCP : 0;
11880f00260cSJohn Baldwin 	vcpu->cap.proc_ctls = procbased_ctls;
11890f00260cSJohn Baldwin 	vcpu->cap.proc_ctls2 = procbased_ctls2;
11900f00260cSJohn Baldwin 	vcpu->cap.exc_bitmap = exc_bitmap;
1191366f6083SPeter Grehan 
11920f00260cSJohn Baldwin 	vcpu->state.nextrip = ~0;
11930f00260cSJohn Baldwin 	vcpu->state.lastcpu = NOCPU;
119458eefc67SJohn Baldwin 	vcpu->state.vpid = vpid;
1195366f6083SPeter Grehan 
1196aaaa0656SPeter Grehan 	/*
1197aaaa0656SPeter Grehan 	 * Set up the CR0/4 shadows, and init the read shadow
1198aaaa0656SPeter Grehan 	 * to the power-on register value from the Intel Sys Arch.
1199aaaa0656SPeter Grehan 	 *  CR0 - 0x60000010
1200aaaa0656SPeter Grehan 	 *  CR4 - 0
1201aaaa0656SPeter Grehan 	 */
1202c847a506SNeel Natu 	error = vmx_setup_cr0_shadow(vmcs, 0x60000010);
120339c21c2dSNeel Natu 	if (error != 0)
120439c21c2dSNeel Natu 		panic("vmx_setup_cr0_shadow %d", error);
120539c21c2dSNeel Natu 
1206c847a506SNeel Natu 	error = vmx_setup_cr4_shadow(vmcs, 0);
120739c21c2dSNeel Natu 	if (error != 0)
120839c21c2dSNeel Natu 		panic("vmx_setup_cr4_shadow %d", error);
1209318224bbSNeel Natu 
12101aa51504SJohn Baldwin 	vcpu->ctx.pmap = vmx->pmap;
1211366f6083SPeter Grehan 
12121aa51504SJohn Baldwin 	return (vcpu);
1213366f6083SPeter Grehan }
1214366f6083SPeter Grehan 
1215366f6083SPeter Grehan static int
121680cb5d84SJohn Baldwin vmx_handle_cpuid(struct vmx_vcpu *vcpu, struct vmxctx *vmxctx)
1217366f6083SPeter Grehan {
1218a3f2a9c5SJohn Baldwin 	int handled;
1219366f6083SPeter Grehan 
122080cb5d84SJohn Baldwin 	handled = x86_emulate_cpuid(vcpu->vcpu, (uint64_t *)&vmxctx->guest_rax,
1221a3f2a9c5SJohn Baldwin 	    (uint64_t *)&vmxctx->guest_rbx, (uint64_t *)&vmxctx->guest_rcx,
1222a3f2a9c5SJohn Baldwin 	    (uint64_t *)&vmxctx->guest_rdx);
1223366f6083SPeter Grehan 	return (handled);
1224366f6083SPeter Grehan }
1225366f6083SPeter Grehan 
1226366f6083SPeter Grehan static __inline void
1227869c8d19SJohn Baldwin vmx_run_trace(struct vmx_vcpu *vcpu)
1228366f6083SPeter Grehan {
122957e0119eSJohn Baldwin 	VMX_CTR1(vcpu, "Resume execution at %#lx", vmcs_guest_rip());
1230366f6083SPeter Grehan }
1231366f6083SPeter Grehan 
1232366f6083SPeter Grehan static __inline void
1233869c8d19SJohn Baldwin vmx_exit_trace(struct vmx_vcpu *vcpu, uint64_t rip, uint32_t exit_reason,
1234869c8d19SJohn Baldwin     int handled)
1235366f6083SPeter Grehan {
123657e0119eSJohn Baldwin 	VMX_CTR3(vcpu, "%s %s vmexit at 0x%0lx",
1237366f6083SPeter Grehan 		 handled ? "handled" : "unhandled",
1238366f6083SPeter Grehan 		 exit_reason_to_str(exit_reason), rip);
1239eeefa4e4SNeel Natu }
1240366f6083SPeter Grehan 
1241eeefa4e4SNeel Natu static __inline void
1242869c8d19SJohn Baldwin vmx_astpending_trace(struct vmx_vcpu *vcpu, uint64_t rip)
1243eeefa4e4SNeel Natu {
124457e0119eSJohn Baldwin 	VMX_CTR1(vcpu, "astpending vmexit at 0x%0lx", rip);
1245366f6083SPeter Grehan }
1246366f6083SPeter Grehan 
1247953c2c47SNeel Natu static VMM_STAT_INTEL(VCPU_INVVPID_SAVED, "Number of vpid invalidations saved");
12483527963bSNeel Natu static VMM_STAT_INTEL(VCPU_INVVPID_DONE, "Number of vpid invalidations done");
1249953c2c47SNeel Natu 
12503527963bSNeel Natu /*
12513527963bSNeel Natu  * Invalidate guest mappings identified by its vpid from the TLB.
12523527963bSNeel Natu  */
12533527963bSNeel Natu static __inline void
12541aa51504SJohn Baldwin vmx_invvpid(struct vmx *vmx, struct vmx_vcpu *vcpu, pmap_t pmap, int running)
1255366f6083SPeter Grehan {
1256366f6083SPeter Grehan 	struct vmxstate *vmxstate;
1257953c2c47SNeel Natu 	struct invvpid_desc invvpid_desc;
1258366f6083SPeter Grehan 
12591aa51504SJohn Baldwin 	vmxstate = &vcpu->state;
12603527963bSNeel Natu 	if (vmxstate->vpid == 0)
12613de83862SNeel Natu 		return;
1262366f6083SPeter Grehan 
12633527963bSNeel Natu 	if (!running) {
12643527963bSNeel Natu 		/*
12653527963bSNeel Natu 		 * Set the 'lastcpu' to an invalid host cpu.
12663527963bSNeel Natu 		 *
12673527963bSNeel Natu 		 * This will invalidate TLB entries tagged with the vcpu's
12683527963bSNeel Natu 		 * vpid the next time it runs via vmx_set_pcpu_defaults().
12693527963bSNeel Natu 		 */
12703527963bSNeel Natu 		vmxstate->lastcpu = NOCPU;
12713527963bSNeel Natu 		return;
12723527963bSNeel Natu 	}
1273953c2c47SNeel Natu 
12743527963bSNeel Natu 	KASSERT(curthread->td_critnest > 0, ("%s: vcpu %d running outside "
12751aa51504SJohn Baldwin 	    "critical section", __func__, vcpu->vcpuid));
1276366f6083SPeter Grehan 
1277366f6083SPeter Grehan 	/*
12783527963bSNeel Natu 	 * Invalidate all mappings tagged with 'vpid'
1279366f6083SPeter Grehan 	 *
1280366f6083SPeter Grehan 	 * We do this because this vcpu was executing on a different host
1281366f6083SPeter Grehan 	 * cpu when it last ran. We do not track whether it invalidated
1282366f6083SPeter Grehan 	 * mappings associated with its 'vpid' during that run. So we must
1283366f6083SPeter Grehan 	 * assume that the mappings associated with 'vpid' on 'curcpu' are
1284366f6083SPeter Grehan 	 * stale and invalidate them.
1285366f6083SPeter Grehan 	 *
1286366f6083SPeter Grehan 	 * Note that we incur this penalty only when the scheduler chooses to
1287366f6083SPeter Grehan 	 * move the thread associated with this vcpu between host cpus.
1288366f6083SPeter Grehan 	 *
1289366f6083SPeter Grehan 	 * Note also that this will invalidate mappings tagged with 'vpid'
1290366f6083SPeter Grehan 	 * for "all" EP4TAs.
1291366f6083SPeter Grehan 	 */
12926f5a9606SMark Johnston 	if (atomic_load_long(&pmap->pm_eptgen) == vmx->eptgen[curcpu]) {
1293953c2c47SNeel Natu 		invvpid_desc._res1 = 0;
1294953c2c47SNeel Natu 		invvpid_desc._res2 = 0;
1295366f6083SPeter Grehan 		invvpid_desc.vpid = vmxstate->vpid;
12960e30c5c0SWarner Losh 		invvpid_desc.linear_addr = 0;
1297366f6083SPeter Grehan 		invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc);
12983dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VCPU_INVVPID_DONE, 1);
1299953c2c47SNeel Natu 	} else {
1300953c2c47SNeel Natu 		/*
1301953c2c47SNeel Natu 		 * The invvpid can be skipped if an invept is going to
1302953c2c47SNeel Natu 		 * be performed before entering the guest. The invept
1303953c2c47SNeel Natu 		 * will invalidate combined mappings tagged with
1304953c2c47SNeel Natu 		 * 'vmx->eptp' for all vpids.
1305953c2c47SNeel Natu 		 */
13063dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VCPU_INVVPID_SAVED, 1);
1307953c2c47SNeel Natu 	}
1308366f6083SPeter Grehan }
13093527963bSNeel Natu 
13103527963bSNeel Natu static void
13111aa51504SJohn Baldwin vmx_set_pcpu_defaults(struct vmx *vmx, struct vmx_vcpu *vcpu, pmap_t pmap)
13123527963bSNeel Natu {
13133527963bSNeel Natu 	struct vmxstate *vmxstate;
13143527963bSNeel Natu 
13151aa51504SJohn Baldwin 	vmxstate = &vcpu->state;
13163527963bSNeel Natu 	if (vmxstate->lastcpu == curcpu)
13173527963bSNeel Natu 		return;
13183527963bSNeel Natu 
13193527963bSNeel Natu 	vmxstate->lastcpu = curcpu;
13203527963bSNeel Natu 
13213dc3d32aSJohn Baldwin 	vmm_stat_incr(vcpu->vcpu, VCPU_MIGRATIONS, 1);
13223527963bSNeel Natu 
13233527963bSNeel Natu 	vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase());
13243527963bSNeel Natu 	vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase());
13253527963bSNeel Natu 	vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase());
13263527963bSNeel Natu 	vmx_invvpid(vmx, vcpu, pmap, 1);
1327366f6083SPeter Grehan }
1328366f6083SPeter Grehan 
1329366f6083SPeter Grehan /*
1330366f6083SPeter Grehan  * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set.
1331366f6083SPeter Grehan  */
1332366f6083SPeter Grehan CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0);
1333366f6083SPeter Grehan 
1334366f6083SPeter Grehan static void __inline
1335869c8d19SJohn Baldwin vmx_set_int_window_exiting(struct vmx_vcpu *vcpu)
1336366f6083SPeter Grehan {
1337366f6083SPeter Grehan 
13381aa51504SJohn Baldwin 	if ((vcpu->cap.proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) {
13391aa51504SJohn Baldwin 		vcpu->cap.proc_ctls |= PROCBASED_INT_WINDOW_EXITING;
13401aa51504SJohn Baldwin 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls);
134157e0119eSJohn Baldwin 		VMX_CTR0(vcpu, "Enabling interrupt window exiting");
134248b2d828SNeel Natu 	}
1343366f6083SPeter Grehan }
1344366f6083SPeter Grehan 
1345366f6083SPeter Grehan static void __inline
1346869c8d19SJohn Baldwin vmx_clear_int_window_exiting(struct vmx_vcpu *vcpu)
1347366f6083SPeter Grehan {
1348366f6083SPeter Grehan 
13491aa51504SJohn Baldwin 	KASSERT((vcpu->cap.proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0,
13501aa51504SJohn Baldwin 	    ("intr_window_exiting not set: %#x", vcpu->cap.proc_ctls));
13511aa51504SJohn Baldwin 	vcpu->cap.proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING;
13521aa51504SJohn Baldwin 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls);
135357e0119eSJohn Baldwin 	VMX_CTR0(vcpu, "Disabling interrupt window exiting");
1354366f6083SPeter Grehan }
1355366f6083SPeter Grehan 
1356366f6083SPeter Grehan static void __inline
1357869c8d19SJohn Baldwin vmx_set_nmi_window_exiting(struct vmx_vcpu *vcpu)
1358366f6083SPeter Grehan {
1359366f6083SPeter Grehan 
13601aa51504SJohn Baldwin 	if ((vcpu->cap.proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) {
13611aa51504SJohn Baldwin 		vcpu->cap.proc_ctls |= PROCBASED_NMI_WINDOW_EXITING;
13621aa51504SJohn Baldwin 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls);
136357e0119eSJohn Baldwin 		VMX_CTR0(vcpu, "Enabling NMI window exiting");
136448b2d828SNeel Natu 	}
1365366f6083SPeter Grehan }
1366366f6083SPeter Grehan 
1367366f6083SPeter Grehan static void __inline
1368869c8d19SJohn Baldwin vmx_clear_nmi_window_exiting(struct vmx_vcpu *vcpu)
1369366f6083SPeter Grehan {
1370366f6083SPeter Grehan 
13711aa51504SJohn Baldwin 	KASSERT((vcpu->cap.proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0,
13721aa51504SJohn Baldwin 	    ("nmi_window_exiting not set %#x", vcpu->cap.proc_ctls));
13731aa51504SJohn Baldwin 	vcpu->cap.proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING;
13741aa51504SJohn Baldwin 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls);
137557e0119eSJohn Baldwin 	VMX_CTR0(vcpu, "Disabling NMI window exiting");
1376366f6083SPeter Grehan }
1377366f6083SPeter Grehan 
1378277bdd99STycho Nightingale int
137980cb5d84SJohn Baldwin vmx_set_tsc_offset(struct vmx_vcpu *vcpu, uint64_t offset)
1380277bdd99STycho Nightingale {
1381277bdd99STycho Nightingale 	int error;
1382277bdd99STycho Nightingale 
13831aa51504SJohn Baldwin 	if ((vcpu->cap.proc_ctls & PROCBASED_TSC_OFFSET) == 0) {
13841aa51504SJohn Baldwin 		vcpu->cap.proc_ctls |= PROCBASED_TSC_OFFSET;
13851aa51504SJohn Baldwin 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls);
138657e0119eSJohn Baldwin 		VMX_CTR0(vcpu, "Enabling TSC offsetting");
1387277bdd99STycho Nightingale 	}
1388277bdd99STycho Nightingale 
1389277bdd99STycho Nightingale 	error = vmwrite(VMCS_TSC_OFFSET, offset);
1390483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT
1391483d953aSJohn Baldwin 	if (error == 0)
139280cb5d84SJohn Baldwin 		vm_set_tsc_offset(vcpu->vcpu, offset);
1393483d953aSJohn Baldwin #endif
1394277bdd99STycho Nightingale 	return (error);
1395277bdd99STycho Nightingale }
1396277bdd99STycho Nightingale 
139748b2d828SNeel Natu #define	NMI_BLOCKING	(VMCS_INTERRUPTIBILITY_NMI_BLOCKING |		\
139848b2d828SNeel Natu 			 VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
139948b2d828SNeel Natu #define	HWINTR_BLOCKING	(VMCS_INTERRUPTIBILITY_STI_BLOCKING |		\
140048b2d828SNeel Natu 			 VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
140148b2d828SNeel Natu 
140248b2d828SNeel Natu static void
140380cb5d84SJohn Baldwin vmx_inject_nmi(struct vmx_vcpu *vcpu)
1404366f6083SPeter Grehan {
14055c272efaSRobert Wing 	uint32_t gi __diagused, info;
1406366f6083SPeter Grehan 
140748b2d828SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
140848b2d828SNeel Natu 	KASSERT((gi & NMI_BLOCKING) == 0, ("vmx_inject_nmi: invalid guest "
140948b2d828SNeel Natu 	    "interruptibility-state %#x", gi));
1410366f6083SPeter Grehan 
141148b2d828SNeel Natu 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
141248b2d828SNeel Natu 	KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_nmi: invalid "
141348b2d828SNeel Natu 	    "VM-entry interruption information %#x", info));
1414366f6083SPeter Grehan 
1415366f6083SPeter Grehan 	/*
1416366f6083SPeter Grehan 	 * Inject the virtual NMI. The vector must be the NMI IDT entry
1417366f6083SPeter Grehan 	 * or the VMCS entry check will fail.
1418366f6083SPeter Grehan 	 */
141948b2d828SNeel Natu 	info = IDT_NMI | VMCS_INTR_T_NMI | VMCS_INTR_VALID;
14203de83862SNeel Natu 	vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1421366f6083SPeter Grehan 
142257e0119eSJohn Baldwin 	VMX_CTR0(vcpu, "Injecting vNMI");
1423366f6083SPeter Grehan 
1424366f6083SPeter Grehan 	/* Clear the request */
142580cb5d84SJohn Baldwin 	vm_nmi_clear(vcpu->vcpu);
1426366f6083SPeter Grehan }
1427366f6083SPeter Grehan 
1428366f6083SPeter Grehan static void
142980cb5d84SJohn Baldwin vmx_inject_interrupts(struct vmx_vcpu *vcpu, struct vlapic *vlapic,
143080cb5d84SJohn Baldwin     uint64_t guestrip)
1431366f6083SPeter Grehan {
14320775fbb4STycho Nightingale 	int vector, need_nmi_exiting, extint_pending;
1433091d4532SNeel Natu 	uint64_t rflags, entryinfo;
143448b2d828SNeel Natu 	uint32_t gi, info;
1435366f6083SPeter Grehan 
14361aa51504SJohn Baldwin 	if (vcpu->state.nextrip != guestrip) {
14372ce12423SNeel Natu 		gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
14382ce12423SNeel Natu 		if (gi & HWINTR_BLOCKING) {
143957e0119eSJohn Baldwin 			VMX_CTR2(vcpu, "Guest interrupt blocking "
14402ce12423SNeel Natu 			    "cleared due to rip change: %#lx/%#lx",
14411aa51504SJohn Baldwin 			    vcpu->state.nextrip, guestrip);
14422ce12423SNeel Natu 			gi &= ~HWINTR_BLOCKING;
14432ce12423SNeel Natu 			vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
14442ce12423SNeel Natu 		}
14452ce12423SNeel Natu 	}
14462ce12423SNeel Natu 
144780cb5d84SJohn Baldwin 	if (vm_entry_intinfo(vcpu->vcpu, &entryinfo)) {
1448091d4532SNeel Natu 		KASSERT((entryinfo & VMCS_INTR_VALID) != 0, ("%s: entry "
1449091d4532SNeel Natu 		    "intinfo is not valid: %#lx", __func__, entryinfo));
1450dc506506SNeel Natu 
1451dc506506SNeel Natu 		info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1452dc506506SNeel Natu 		KASSERT((info & VMCS_INTR_VALID) == 0, ("%s: cannot inject "
1453019008ebSNeel Natu 		     "pending exception: %#lx/%#x", __func__, entryinfo, info));
1454dc506506SNeel Natu 
1455091d4532SNeel Natu 		info = entryinfo;
1456091d4532SNeel Natu 		vector = info & 0xff;
1457091d4532SNeel Natu 		if (vector == IDT_BP || vector == IDT_OF) {
1458091d4532SNeel Natu 			/*
1459091d4532SNeel Natu 			 * VT-x requires #BP and #OF to be injected as software
1460091d4532SNeel Natu 			 * exceptions.
1461091d4532SNeel Natu 			 */
1462091d4532SNeel Natu 			info &= ~VMCS_INTR_T_MASK;
1463091d4532SNeel Natu 			info |= VMCS_INTR_T_SWEXCEPTION;
1464dc506506SNeel Natu 		}
1465091d4532SNeel Natu 
1466091d4532SNeel Natu 		if (info & VMCS_INTR_DEL_ERRCODE)
1467091d4532SNeel Natu 			vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR, entryinfo >> 32);
1468091d4532SNeel Natu 
1469dc506506SNeel Natu 		vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1470dc506506SNeel Natu 	}
1471dc506506SNeel Natu 
147280cb5d84SJohn Baldwin 	if (vm_nmi_pending(vcpu->vcpu)) {
1473366f6083SPeter Grehan 		/*
147448b2d828SNeel Natu 		 * If there are no conditions blocking NMI injection then
147548b2d828SNeel Natu 		 * inject it directly here otherwise enable "NMI window
147648b2d828SNeel Natu 		 * exiting" to inject it as soon as we can.
1477eeefa4e4SNeel Natu 		 *
147848b2d828SNeel Natu 		 * We also check for STI_BLOCKING because some implementations
147948b2d828SNeel Natu 		 * don't allow NMI injection in this case. If we are running
148048b2d828SNeel Natu 		 * on a processor that doesn't have this restriction it will
148148b2d828SNeel Natu 		 * immediately exit and the NMI will be injected in the
148248b2d828SNeel Natu 		 * "NMI window exiting" handler.
1483366f6083SPeter Grehan 		 */
148448b2d828SNeel Natu 		need_nmi_exiting = 1;
148548b2d828SNeel Natu 		gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
148648b2d828SNeel Natu 		if ((gi & (HWINTR_BLOCKING | NMI_BLOCKING)) == 0) {
14873de83862SNeel Natu 			info = vmcs_read(VMCS_ENTRY_INTR_INFO);
148848b2d828SNeel Natu 			if ((info & VMCS_INTR_VALID) == 0) {
148980cb5d84SJohn Baldwin 				vmx_inject_nmi(vcpu);
149048b2d828SNeel Natu 				need_nmi_exiting = 0;
149148b2d828SNeel Natu 			} else {
149257e0119eSJohn Baldwin 				VMX_CTR1(vcpu, "Cannot inject NMI "
149357e0119eSJohn Baldwin 				    "due to VM-entry intr info %#x", info);
149448b2d828SNeel Natu 			}
149548b2d828SNeel Natu 		} else {
149657e0119eSJohn Baldwin 			VMX_CTR1(vcpu, "Cannot inject NMI due to "
149757e0119eSJohn Baldwin 			    "Guest Interruptibility-state %#x", gi);
149848b2d828SNeel Natu 		}
1499eeefa4e4SNeel Natu 
150048b2d828SNeel Natu 		if (need_nmi_exiting)
1501869c8d19SJohn Baldwin 			vmx_set_nmi_window_exiting(vcpu);
150248b2d828SNeel Natu 	}
1503366f6083SPeter Grehan 
150480cb5d84SJohn Baldwin 	extint_pending = vm_extint_pending(vcpu->vcpu);
15050775fbb4STycho Nightingale 
15060775fbb4STycho Nightingale 	if (!extint_pending && virtual_interrupt_delivery) {
150788c4b8d1SNeel Natu 		vmx_inject_pir(vlapic);
150888c4b8d1SNeel Natu 		return;
150988c4b8d1SNeel Natu 	}
151088c4b8d1SNeel Natu 
151148b2d828SNeel Natu 	/*
151236736912SNeel Natu 	 * If interrupt-window exiting is already in effect then don't bother
151336736912SNeel Natu 	 * checking for pending interrupts. This is just an optimization and
151436736912SNeel Natu 	 * not needed for correctness.
151548b2d828SNeel Natu 	 */
15161aa51504SJohn Baldwin 	if ((vcpu->cap.proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) {
151757e0119eSJohn Baldwin 		VMX_CTR0(vcpu, "Skip interrupt injection due to "
151857e0119eSJohn Baldwin 		    "pending int_window_exiting");
151948b2d828SNeel Natu 		return;
152036736912SNeel Natu 	}
152148b2d828SNeel Natu 
15220775fbb4STycho Nightingale 	if (!extint_pending) {
1523366f6083SPeter Grehan 		/* Ask the local apic for a vector to inject */
15244d1e82a8SNeel Natu 		if (!vlapic_pending_intr(vlapic, &vector))
1525366f6083SPeter Grehan 			return;
1526a026dc3fSTycho Nightingale 
1527a026dc3fSTycho Nightingale 		/*
1528a026dc3fSTycho Nightingale 		 * From the Intel SDM, Volume 3, Section "Maskable
1529a026dc3fSTycho Nightingale 		 * Hardware Interrupts":
1530a026dc3fSTycho Nightingale 		 * - maskable interrupt vectors [16,255] can be delivered
1531a026dc3fSTycho Nightingale 		 *   through the local APIC.
1532a026dc3fSTycho Nightingale 		*/
1533a026dc3fSTycho Nightingale 		KASSERT(vector >= 16 && vector <= 255,
1534a026dc3fSTycho Nightingale 		    ("invalid vector %d from local APIC", vector));
15350775fbb4STycho Nightingale 	} else {
15360775fbb4STycho Nightingale 		/* Ask the legacy pic for a vector to inject */
153780cb5d84SJohn Baldwin 		vatpic_pending_intr(vcpu->vmx->vm, &vector);
1538366f6083SPeter Grehan 
1539a026dc3fSTycho Nightingale 		/*
1540a026dc3fSTycho Nightingale 		 * From the Intel SDM, Volume 3, Section "Maskable
1541a026dc3fSTycho Nightingale 		 * Hardware Interrupts":
1542a026dc3fSTycho Nightingale 		 * - maskable interrupt vectors [0,255] can be delivered
1543a026dc3fSTycho Nightingale 		 *   through the INTR pin.
1544a026dc3fSTycho Nightingale 		 */
1545a026dc3fSTycho Nightingale 		KASSERT(vector >= 0 && vector <= 255,
1546a026dc3fSTycho Nightingale 		    ("invalid vector %d from INTR", vector));
1547a026dc3fSTycho Nightingale 	}
1548366f6083SPeter Grehan 
1549366f6083SPeter Grehan 	/* Check RFLAGS.IF and the interruptibility state of the guest */
15503de83862SNeel Natu 	rflags = vmcs_read(VMCS_GUEST_RFLAGS);
155136736912SNeel Natu 	if ((rflags & PSL_I) == 0) {
155257e0119eSJohn Baldwin 		VMX_CTR2(vcpu, "Cannot inject vector %d due to "
155357e0119eSJohn Baldwin 		    "rflags %#lx", vector, rflags);
1554366f6083SPeter Grehan 		goto cantinject;
155536736912SNeel Natu 	}
1556366f6083SPeter Grehan 
155748b2d828SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
155836736912SNeel Natu 	if (gi & HWINTR_BLOCKING) {
155957e0119eSJohn Baldwin 		VMX_CTR2(vcpu, "Cannot inject vector %d due to "
156057e0119eSJohn Baldwin 		    "Guest Interruptibility-state %#x", vector, gi);
1561366f6083SPeter Grehan 		goto cantinject;
156236736912SNeel Natu 	}
156336736912SNeel Natu 
156436736912SNeel Natu 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
156536736912SNeel Natu 	if (info & VMCS_INTR_VALID) {
156636736912SNeel Natu 		/*
156736736912SNeel Natu 		 * This is expected and could happen for multiple reasons:
156836736912SNeel Natu 		 * - A vectoring VM-entry was aborted due to astpending
156936736912SNeel Natu 		 * - A VM-exit happened during event injection.
1570dc506506SNeel Natu 		 * - An exception was injected above.
157136736912SNeel Natu 		 * - An NMI was injected above or after "NMI window exiting"
157236736912SNeel Natu 		 */
157357e0119eSJohn Baldwin 		VMX_CTR2(vcpu, "Cannot inject vector %d due to "
157457e0119eSJohn Baldwin 		    "VM-entry intr info %#x", vector, info);
157536736912SNeel Natu 		goto cantinject;
157636736912SNeel Natu 	}
1577366f6083SPeter Grehan 
1578366f6083SPeter Grehan 	/* Inject the interrupt */
1579160471d2SNeel Natu 	info = VMCS_INTR_T_HWINTR | VMCS_INTR_VALID;
1580366f6083SPeter Grehan 	info |= vector;
15813de83862SNeel Natu 	vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1582366f6083SPeter Grehan 
15830775fbb4STycho Nightingale 	if (!extint_pending) {
1584366f6083SPeter Grehan 		/* Update the Local APIC ISR */
1585de5ea6b6SNeel Natu 		vlapic_intr_accepted(vlapic, vector);
15860775fbb4STycho Nightingale 	} else {
158780cb5d84SJohn Baldwin 		vm_extint_clear(vcpu->vcpu);
158880cb5d84SJohn Baldwin 		vatpic_intr_accepted(vcpu->vmx->vm, vector);
15890775fbb4STycho Nightingale 
15900775fbb4STycho Nightingale 		/*
15910775fbb4STycho Nightingale 		 * After we accepted the current ExtINT the PIC may
15920775fbb4STycho Nightingale 		 * have posted another one.  If that is the case, set
15930775fbb4STycho Nightingale 		 * the Interrupt Window Exiting execution control so
15940775fbb4STycho Nightingale 		 * we can inject that one too.
15950494cb1bSNeel Natu 		 *
15960494cb1bSNeel Natu 		 * Also, interrupt window exiting allows us to inject any
15970494cb1bSNeel Natu 		 * pending APIC vector that was preempted by the ExtINT
15980494cb1bSNeel Natu 		 * as soon as possible. This applies both for the software
15990494cb1bSNeel Natu 		 * emulated vlapic and the hardware assisted virtual APIC.
16000775fbb4STycho Nightingale 		 */
1601869c8d19SJohn Baldwin 		vmx_set_int_window_exiting(vcpu);
16020775fbb4STycho Nightingale 	}
1603366f6083SPeter Grehan 
160457e0119eSJohn Baldwin 	VMX_CTR1(vcpu, "Injecting hwintr at vector %d", vector);
1605366f6083SPeter Grehan 
1606366f6083SPeter Grehan 	return;
1607366f6083SPeter Grehan 
1608366f6083SPeter Grehan cantinject:
1609366f6083SPeter Grehan 	/*
1610366f6083SPeter Grehan 	 * Set the Interrupt Window Exiting execution control so we can inject
1611366f6083SPeter Grehan 	 * the interrupt as soon as blocking condition goes away.
1612366f6083SPeter Grehan 	 */
1613869c8d19SJohn Baldwin 	vmx_set_int_window_exiting(vcpu);
1614366f6083SPeter Grehan }
1615366f6083SPeter Grehan 
1616e5a1d950SNeel Natu /*
1617e5a1d950SNeel Natu  * If the Virtual NMIs execution control is '1' then the logical processor
1618e5a1d950SNeel Natu  * tracks virtual-NMI blocking in the Guest Interruptibility-state field of
1619e5a1d950SNeel Natu  * the VMCS. An IRET instruction in VMX non-root operation will remove any
1620e5a1d950SNeel Natu  * virtual-NMI blocking.
1621e5a1d950SNeel Natu  *
1622e5a1d950SNeel Natu  * This unblocking occurs even if the IRET causes a fault. In this case the
1623e5a1d950SNeel Natu  * hypervisor needs to restore virtual-NMI blocking before resuming the guest.
1624e5a1d950SNeel Natu  */
1625e5a1d950SNeel Natu static void
1626869c8d19SJohn Baldwin vmx_restore_nmi_blocking(struct vmx_vcpu *vcpu)
1627e5a1d950SNeel Natu {
1628e5a1d950SNeel Natu 	uint32_t gi;
1629e5a1d950SNeel Natu 
163057e0119eSJohn Baldwin 	VMX_CTR0(vcpu, "Restore Virtual-NMI blocking");
1631e5a1d950SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1632e5a1d950SNeel Natu 	gi |= VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1633e5a1d950SNeel Natu 	vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1634e5a1d950SNeel Natu }
1635e5a1d950SNeel Natu 
1636e5a1d950SNeel Natu static void
1637869c8d19SJohn Baldwin vmx_clear_nmi_blocking(struct vmx_vcpu *vcpu)
1638e5a1d950SNeel Natu {
1639e5a1d950SNeel Natu 	uint32_t gi;
1640e5a1d950SNeel Natu 
164157e0119eSJohn Baldwin 	VMX_CTR0(vcpu, "Clear Virtual-NMI blocking");
1642e5a1d950SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1643e5a1d950SNeel Natu 	gi &= ~VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1644e5a1d950SNeel Natu 	vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1645e5a1d950SNeel Natu }
1646e5a1d950SNeel Natu 
1647091d4532SNeel Natu static void
1648869c8d19SJohn Baldwin vmx_assert_nmi_blocking(struct vmx_vcpu *vcpu)
1649091d4532SNeel Natu {
16505c272efaSRobert Wing 	uint32_t gi __diagused;
1651091d4532SNeel Natu 
1652091d4532SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1653091d4532SNeel Natu 	KASSERT(gi & VMCS_INTERRUPTIBILITY_NMI_BLOCKING,
1654091d4532SNeel Natu 	    ("NMI blocking is not in effect %#x", gi));
1655091d4532SNeel Natu }
1656091d4532SNeel Natu 
1657366f6083SPeter Grehan static int
16581aa51504SJohn Baldwin vmx_emulate_xsetbv(struct vmx *vmx, struct vmx_vcpu *vcpu,
16591aa51504SJohn Baldwin     struct vm_exit *vmexit)
1660abb023fbSJohn Baldwin {
1661abb023fbSJohn Baldwin 	struct vmxctx *vmxctx;
1662abb023fbSJohn Baldwin 	uint64_t xcrval;
1663abb023fbSJohn Baldwin 	const struct xsave_limits *limits;
1664abb023fbSJohn Baldwin 
16651aa51504SJohn Baldwin 	vmxctx = &vcpu->ctx;
1666abb023fbSJohn Baldwin 	limits = vmm_get_xsave_limits();
1667abb023fbSJohn Baldwin 
1668a0efd3fbSJohn Baldwin 	/*
1669a0efd3fbSJohn Baldwin 	 * Note that the processor raises a GP# fault on its own if
1670a0efd3fbSJohn Baldwin 	 * xsetbv is executed for CPL != 0, so we do not have to
1671a0efd3fbSJohn Baldwin 	 * emulate that fault here.
1672a0efd3fbSJohn Baldwin 	 */
1673a0efd3fbSJohn Baldwin 
1674a0efd3fbSJohn Baldwin 	/* Only xcr0 is supported. */
1675a0efd3fbSJohn Baldwin 	if (vmxctx->guest_rcx != 0) {
1676d3956e46SJohn Baldwin 		vm_inject_gp(vcpu->vcpu);
1677a0efd3fbSJohn Baldwin 		return (HANDLED);
1678a0efd3fbSJohn Baldwin 	}
1679a0efd3fbSJohn Baldwin 
1680a0efd3fbSJohn Baldwin 	/* We only handle xcr0 if both the host and guest have XSAVE enabled. */
1681a0efd3fbSJohn Baldwin 	if (!limits->xsave_enabled || !(vmcs_read(VMCS_GUEST_CR4) & CR4_XSAVE)) {
1682d3956e46SJohn Baldwin 		vm_inject_ud(vcpu->vcpu);
1683a0efd3fbSJohn Baldwin 		return (HANDLED);
1684a0efd3fbSJohn Baldwin 	}
1685abb023fbSJohn Baldwin 
1686abb023fbSJohn Baldwin 	xcrval = vmxctx->guest_rdx << 32 | (vmxctx->guest_rax & 0xffffffff);
1687a0efd3fbSJohn Baldwin 	if ((xcrval & ~limits->xcr0_allowed) != 0) {
1688d3956e46SJohn Baldwin 		vm_inject_gp(vcpu->vcpu);
1689a0efd3fbSJohn Baldwin 		return (HANDLED);
1690a0efd3fbSJohn Baldwin 	}
1691abb023fbSJohn Baldwin 
1692a0efd3fbSJohn Baldwin 	if (!(xcrval & XFEATURE_ENABLED_X87)) {
1693d3956e46SJohn Baldwin 		vm_inject_gp(vcpu->vcpu);
1694a0efd3fbSJohn Baldwin 		return (HANDLED);
1695a0efd3fbSJohn Baldwin 	}
1696abb023fbSJohn Baldwin 
169744a68c4eSJohn Baldwin 	/* AVX (YMM_Hi128) requires SSE. */
169844a68c4eSJohn Baldwin 	if (xcrval & XFEATURE_ENABLED_AVX &&
169944a68c4eSJohn Baldwin 	    (xcrval & XFEATURE_AVX) != XFEATURE_AVX) {
1700d3956e46SJohn Baldwin 		vm_inject_gp(vcpu->vcpu);
170144a68c4eSJohn Baldwin 		return (HANDLED);
170244a68c4eSJohn Baldwin 	}
170344a68c4eSJohn Baldwin 
170444a68c4eSJohn Baldwin 	/*
170544a68c4eSJohn Baldwin 	 * AVX512 requires base AVX (YMM_Hi128) as well as OpMask,
170644a68c4eSJohn Baldwin 	 * ZMM_Hi256, and Hi16_ZMM.
170744a68c4eSJohn Baldwin 	 */
170844a68c4eSJohn Baldwin 	if (xcrval & XFEATURE_AVX512 &&
170944a68c4eSJohn Baldwin 	    (xcrval & (XFEATURE_AVX512 | XFEATURE_AVX)) !=
171044a68c4eSJohn Baldwin 	    (XFEATURE_AVX512 | XFEATURE_AVX)) {
1711d3956e46SJohn Baldwin 		vm_inject_gp(vcpu->vcpu);
171244a68c4eSJohn Baldwin 		return (HANDLED);
171344a68c4eSJohn Baldwin 	}
171444a68c4eSJohn Baldwin 
171544a68c4eSJohn Baldwin 	/*
171644a68c4eSJohn Baldwin 	 * Intel MPX requires both bound register state flags to be
171744a68c4eSJohn Baldwin 	 * set.
171844a68c4eSJohn Baldwin 	 */
171944a68c4eSJohn Baldwin 	if (((xcrval & XFEATURE_ENABLED_BNDREGS) != 0) !=
172044a68c4eSJohn Baldwin 	    ((xcrval & XFEATURE_ENABLED_BNDCSR) != 0)) {
1721d3956e46SJohn Baldwin 		vm_inject_gp(vcpu->vcpu);
1722a0efd3fbSJohn Baldwin 		return (HANDLED);
1723a0efd3fbSJohn Baldwin 	}
1724abb023fbSJohn Baldwin 
1725abb023fbSJohn Baldwin 	/*
1726abb023fbSJohn Baldwin 	 * This runs "inside" vmrun() with the guest's FPU state, so
1727abb023fbSJohn Baldwin 	 * modifying xcr0 directly modifies the guest's xcr0, not the
1728abb023fbSJohn Baldwin 	 * host's.
1729abb023fbSJohn Baldwin 	 */
1730abb023fbSJohn Baldwin 	load_xcr(0, xcrval);
1731abb023fbSJohn Baldwin 	return (HANDLED);
1732abb023fbSJohn Baldwin }
1733abb023fbSJohn Baldwin 
1734594db002STycho Nightingale static uint64_t
17351aa51504SJohn Baldwin vmx_get_guest_reg(struct vmx_vcpu *vcpu, int ident)
1736366f6083SPeter Grehan {
1737366f6083SPeter Grehan 	const struct vmxctx *vmxctx;
1738366f6083SPeter Grehan 
17391aa51504SJohn Baldwin 	vmxctx = &vcpu->ctx;
1740594db002STycho Nightingale 
1741594db002STycho Nightingale 	switch (ident) {
1742594db002STycho Nightingale 	case 0:
1743594db002STycho Nightingale 		return (vmxctx->guest_rax);
1744594db002STycho Nightingale 	case 1:
1745594db002STycho Nightingale 		return (vmxctx->guest_rcx);
1746594db002STycho Nightingale 	case 2:
1747594db002STycho Nightingale 		return (vmxctx->guest_rdx);
1748594db002STycho Nightingale 	case 3:
1749594db002STycho Nightingale 		return (vmxctx->guest_rbx);
1750594db002STycho Nightingale 	case 4:
1751594db002STycho Nightingale 		return (vmcs_read(VMCS_GUEST_RSP));
1752594db002STycho Nightingale 	case 5:
1753594db002STycho Nightingale 		return (vmxctx->guest_rbp);
1754594db002STycho Nightingale 	case 6:
1755594db002STycho Nightingale 		return (vmxctx->guest_rsi);
1756594db002STycho Nightingale 	case 7:
1757594db002STycho Nightingale 		return (vmxctx->guest_rdi);
1758594db002STycho Nightingale 	case 8:
1759594db002STycho Nightingale 		return (vmxctx->guest_r8);
1760594db002STycho Nightingale 	case 9:
1761594db002STycho Nightingale 		return (vmxctx->guest_r9);
1762594db002STycho Nightingale 	case 10:
1763594db002STycho Nightingale 		return (vmxctx->guest_r10);
1764594db002STycho Nightingale 	case 11:
1765594db002STycho Nightingale 		return (vmxctx->guest_r11);
1766594db002STycho Nightingale 	case 12:
1767594db002STycho Nightingale 		return (vmxctx->guest_r12);
1768594db002STycho Nightingale 	case 13:
1769594db002STycho Nightingale 		return (vmxctx->guest_r13);
1770594db002STycho Nightingale 	case 14:
1771594db002STycho Nightingale 		return (vmxctx->guest_r14);
1772594db002STycho Nightingale 	case 15:
1773594db002STycho Nightingale 		return (vmxctx->guest_r15);
1774594db002STycho Nightingale 	default:
1775594db002STycho Nightingale 		panic("invalid vmx register %d", ident);
1776594db002STycho Nightingale 	}
1777594db002STycho Nightingale }
1778594db002STycho Nightingale 
1779594db002STycho Nightingale static void
17801aa51504SJohn Baldwin vmx_set_guest_reg(struct vmx_vcpu *vcpu, int ident, uint64_t regval)
1781594db002STycho Nightingale {
1782594db002STycho Nightingale 	struct vmxctx *vmxctx;
1783594db002STycho Nightingale 
17841aa51504SJohn Baldwin 	vmxctx = &vcpu->ctx;
1785594db002STycho Nightingale 
1786594db002STycho Nightingale 	switch (ident) {
1787594db002STycho Nightingale 	case 0:
1788594db002STycho Nightingale 		vmxctx->guest_rax = regval;
1789594db002STycho Nightingale 		break;
1790594db002STycho Nightingale 	case 1:
1791594db002STycho Nightingale 		vmxctx->guest_rcx = regval;
1792594db002STycho Nightingale 		break;
1793594db002STycho Nightingale 	case 2:
1794594db002STycho Nightingale 		vmxctx->guest_rdx = regval;
1795594db002STycho Nightingale 		break;
1796594db002STycho Nightingale 	case 3:
1797594db002STycho Nightingale 		vmxctx->guest_rbx = regval;
1798594db002STycho Nightingale 		break;
1799594db002STycho Nightingale 	case 4:
1800594db002STycho Nightingale 		vmcs_write(VMCS_GUEST_RSP, regval);
1801594db002STycho Nightingale 		break;
1802594db002STycho Nightingale 	case 5:
1803594db002STycho Nightingale 		vmxctx->guest_rbp = regval;
1804594db002STycho Nightingale 		break;
1805594db002STycho Nightingale 	case 6:
1806594db002STycho Nightingale 		vmxctx->guest_rsi = regval;
1807594db002STycho Nightingale 		break;
1808594db002STycho Nightingale 	case 7:
1809594db002STycho Nightingale 		vmxctx->guest_rdi = regval;
1810594db002STycho Nightingale 		break;
1811594db002STycho Nightingale 	case 8:
1812594db002STycho Nightingale 		vmxctx->guest_r8 = regval;
1813594db002STycho Nightingale 		break;
1814594db002STycho Nightingale 	case 9:
1815594db002STycho Nightingale 		vmxctx->guest_r9 = regval;
1816594db002STycho Nightingale 		break;
1817594db002STycho Nightingale 	case 10:
1818594db002STycho Nightingale 		vmxctx->guest_r10 = regval;
1819594db002STycho Nightingale 		break;
1820594db002STycho Nightingale 	case 11:
1821594db002STycho Nightingale 		vmxctx->guest_r11 = regval;
1822594db002STycho Nightingale 		break;
1823594db002STycho Nightingale 	case 12:
1824594db002STycho Nightingale 		vmxctx->guest_r12 = regval;
1825594db002STycho Nightingale 		break;
1826594db002STycho Nightingale 	case 13:
1827594db002STycho Nightingale 		vmxctx->guest_r13 = regval;
1828594db002STycho Nightingale 		break;
1829594db002STycho Nightingale 	case 14:
1830594db002STycho Nightingale 		vmxctx->guest_r14 = regval;
1831594db002STycho Nightingale 		break;
1832594db002STycho Nightingale 	case 15:
1833594db002STycho Nightingale 		vmxctx->guest_r15 = regval;
1834594db002STycho Nightingale 		break;
1835594db002STycho Nightingale 	default:
1836594db002STycho Nightingale 		panic("invalid vmx register %d", ident);
1837594db002STycho Nightingale 	}
1838594db002STycho Nightingale }
1839594db002STycho Nightingale 
1840594db002STycho Nightingale static int
18411aa51504SJohn Baldwin vmx_emulate_cr0_access(struct vmx_vcpu *vcpu, uint64_t exitqual)
1842594db002STycho Nightingale {
1843594db002STycho Nightingale 	uint64_t crval, regval;
1844594db002STycho Nightingale 
1845594db002STycho Nightingale 	/* We only handle mov to %cr0 at this time */
184639c21c2dSNeel Natu 	if ((exitqual & 0xf0) != 0x00)
184739c21c2dSNeel Natu 		return (UNHANDLED);
184839c21c2dSNeel Natu 
18491aa51504SJohn Baldwin 	regval = vmx_get_guest_reg(vcpu, (exitqual >> 8) & 0xf);
1850366f6083SPeter Grehan 
1851594db002STycho Nightingale 	vmcs_write(VMCS_CR0_SHADOW, regval);
1852366f6083SPeter Grehan 
1853594db002STycho Nightingale 	crval = regval | cr0_ones_mask;
1854594db002STycho Nightingale 	crval &= ~cr0_zeros_mask;
1855594db002STycho Nightingale 	vmcs_write(VMCS_GUEST_CR0, crval);
1856366f6083SPeter Grehan 
1857594db002STycho Nightingale 	if (regval & CR0_PG) {
185880a902efSPeter Grehan 		uint64_t efer, entry_ctls;
185980a902efSPeter Grehan 
186080a902efSPeter Grehan 		/*
186180a902efSPeter Grehan 		 * If CR0.PG is 1 and EFER.LME is 1 then EFER.LMA and
186280a902efSPeter Grehan 		 * the "IA-32e mode guest" bit in VM-entry control must be
186380a902efSPeter Grehan 		 * equal.
186480a902efSPeter Grehan 		 */
18653de83862SNeel Natu 		efer = vmcs_read(VMCS_GUEST_IA32_EFER);
186680a902efSPeter Grehan 		if (efer & EFER_LME) {
186780a902efSPeter Grehan 			efer |= EFER_LMA;
18683de83862SNeel Natu 			vmcs_write(VMCS_GUEST_IA32_EFER, efer);
18693de83862SNeel Natu 			entry_ctls = vmcs_read(VMCS_ENTRY_CTLS);
187080a902efSPeter Grehan 			entry_ctls |= VM_ENTRY_GUEST_LMA;
18713de83862SNeel Natu 			vmcs_write(VMCS_ENTRY_CTLS, entry_ctls);
187280a902efSPeter Grehan 		}
187380a902efSPeter Grehan 	}
187480a902efSPeter Grehan 
1875366f6083SPeter Grehan 	return (HANDLED);
1876366f6083SPeter Grehan }
1877366f6083SPeter Grehan 
1878594db002STycho Nightingale static int
18791aa51504SJohn Baldwin vmx_emulate_cr4_access(struct vmx_vcpu *vcpu, uint64_t exitqual)
1880594db002STycho Nightingale {
1881594db002STycho Nightingale 	uint64_t crval, regval;
1882594db002STycho Nightingale 
1883594db002STycho Nightingale 	/* We only handle mov to %cr4 at this time */
1884594db002STycho Nightingale 	if ((exitqual & 0xf0) != 0x00)
1885594db002STycho Nightingale 		return (UNHANDLED);
1886594db002STycho Nightingale 
18871aa51504SJohn Baldwin 	regval = vmx_get_guest_reg(vcpu, (exitqual >> 8) & 0xf);
1888594db002STycho Nightingale 
1889594db002STycho Nightingale 	vmcs_write(VMCS_CR4_SHADOW, regval);
1890594db002STycho Nightingale 
1891594db002STycho Nightingale 	crval = regval | cr4_ones_mask;
1892594db002STycho Nightingale 	crval &= ~cr4_zeros_mask;
1893594db002STycho Nightingale 	vmcs_write(VMCS_GUEST_CR4, crval);
1894594db002STycho Nightingale 
1895594db002STycho Nightingale 	return (HANDLED);
1896594db002STycho Nightingale }
1897594db002STycho Nightingale 
1898594db002STycho Nightingale static int
18991aa51504SJohn Baldwin vmx_emulate_cr8_access(struct vmx *vmx, struct vmx_vcpu *vcpu,
19001aa51504SJohn Baldwin     uint64_t exitqual)
1901594db002STycho Nightingale {
1902051f2bd1SNeel Natu 	struct vlapic *vlapic;
1903051f2bd1SNeel Natu 	uint64_t cr8;
1904051f2bd1SNeel Natu 	int regnum;
1905594db002STycho Nightingale 
1906594db002STycho Nightingale 	/* We only handle mov %cr8 to/from a register at this time. */
1907594db002STycho Nightingale 	if ((exitqual & 0xe0) != 0x00) {
1908594db002STycho Nightingale 		return (UNHANDLED);
1909594db002STycho Nightingale 	}
1910594db002STycho Nightingale 
1911d3956e46SJohn Baldwin 	vlapic = vm_lapic(vcpu->vcpu);
1912051f2bd1SNeel Natu 	regnum = (exitqual >> 8) & 0xf;
1913594db002STycho Nightingale 	if (exitqual & 0x10) {
1914051f2bd1SNeel Natu 		cr8 = vlapic_get_cr8(vlapic);
19151aa51504SJohn Baldwin 		vmx_set_guest_reg(vcpu, regnum, cr8);
1916594db002STycho Nightingale 	} else {
19171aa51504SJohn Baldwin 		cr8 = vmx_get_guest_reg(vcpu, regnum);
1918051f2bd1SNeel Natu 		vlapic_set_cr8(vlapic, cr8);
1919594db002STycho Nightingale 	}
1920594db002STycho Nightingale 
1921594db002STycho Nightingale 	return (HANDLED);
1922594db002STycho Nightingale }
1923594db002STycho Nightingale 
1924e4c8a13dSNeel Natu /*
1925e4c8a13dSNeel Natu  * From section "Guest Register State" in the Intel SDM: CPL = SS.DPL
1926e4c8a13dSNeel Natu  */
1927e4c8a13dSNeel Natu static int
1928e4c8a13dSNeel Natu vmx_cpl(void)
1929e4c8a13dSNeel Natu {
1930e4c8a13dSNeel Natu 	uint32_t ssar;
1931e4c8a13dSNeel Natu 
1932e4c8a13dSNeel Natu 	ssar = vmcs_read(VMCS_GUEST_SS_ACCESS_RIGHTS);
1933e4c8a13dSNeel Natu 	return ((ssar >> 5) & 0x3);
1934e4c8a13dSNeel Natu }
1935e4c8a13dSNeel Natu 
1936e813a873SNeel Natu static enum vm_cpu_mode
193700f3efe1SJohn Baldwin vmx_cpu_mode(void)
193800f3efe1SJohn Baldwin {
1939b301b9e2SNeel Natu 	uint32_t csar;
194000f3efe1SJohn Baldwin 
1941b301b9e2SNeel Natu 	if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LMA) {
1942b301b9e2SNeel Natu 		csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS);
1943b301b9e2SNeel Natu 		if (csar & 0x2000)
1944b301b9e2SNeel Natu 			return (CPU_MODE_64BIT);	/* CS.L = 1 */
194500f3efe1SJohn Baldwin 		else
194600f3efe1SJohn Baldwin 			return (CPU_MODE_COMPATIBILITY);
1947b301b9e2SNeel Natu 	} else if (vmcs_read(VMCS_GUEST_CR0) & CR0_PE) {
1948b301b9e2SNeel Natu 		return (CPU_MODE_PROTECTED);
1949b301b9e2SNeel Natu 	} else {
1950b301b9e2SNeel Natu 		return (CPU_MODE_REAL);
1951b301b9e2SNeel Natu 	}
195200f3efe1SJohn Baldwin }
195300f3efe1SJohn Baldwin 
1954e813a873SNeel Natu static enum vm_paging_mode
195500f3efe1SJohn Baldwin vmx_paging_mode(void)
195600f3efe1SJohn Baldwin {
1957f3eb12e4SKonstantin Belousov 	uint64_t cr4;
195800f3efe1SJohn Baldwin 
195900f3efe1SJohn Baldwin 	if (!(vmcs_read(VMCS_GUEST_CR0) & CR0_PG))
196000f3efe1SJohn Baldwin 		return (PAGING_MODE_FLAT);
1961f3eb12e4SKonstantin Belousov 	cr4 = vmcs_read(VMCS_GUEST_CR4);
1962f3eb12e4SKonstantin Belousov 	if (!(cr4 & CR4_PAE))
196300f3efe1SJohn Baldwin 		return (PAGING_MODE_32);
1964f3eb12e4SKonstantin Belousov 	if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LME) {
1965f3eb12e4SKonstantin Belousov 		if (!(cr4 & CR4_LA57))
196600f3efe1SJohn Baldwin 			return (PAGING_MODE_64);
1967f3eb12e4SKonstantin Belousov 		return (PAGING_MODE_64_LA57);
1968f3eb12e4SKonstantin Belousov 	} else
196900f3efe1SJohn Baldwin 		return (PAGING_MODE_PAE);
197000f3efe1SJohn Baldwin }
197100f3efe1SJohn Baldwin 
1972d17b5104SNeel Natu static uint64_t
1973869c8d19SJohn Baldwin inout_str_index(struct vmx_vcpu *vcpu, int in)
1974d17b5104SNeel Natu {
1975d17b5104SNeel Natu 	uint64_t val;
19765c272efaSRobert Wing 	int error __diagused;
1977d17b5104SNeel Natu 	enum vm_reg_name reg;
1978d17b5104SNeel Natu 
1979d17b5104SNeel Natu 	reg = in ? VM_REG_GUEST_RDI : VM_REG_GUEST_RSI;
1980869c8d19SJohn Baldwin 	error = vmx_getreg(vcpu, reg, &val);
1981d17b5104SNeel Natu 	KASSERT(error == 0, ("%s: vmx_getreg error %d", __func__, error));
1982d17b5104SNeel Natu 	return (val);
1983d17b5104SNeel Natu }
1984d17b5104SNeel Natu 
1985d17b5104SNeel Natu static uint64_t
1986869c8d19SJohn Baldwin inout_str_count(struct vmx_vcpu *vcpu, int rep)
1987d17b5104SNeel Natu {
1988d17b5104SNeel Natu 	uint64_t val;
19895c272efaSRobert Wing 	int error __diagused;
1990d17b5104SNeel Natu 
1991d17b5104SNeel Natu 	if (rep) {
1992869c8d19SJohn Baldwin 		error = vmx_getreg(vcpu, VM_REG_GUEST_RCX, &val);
1993d17b5104SNeel Natu 		KASSERT(!error, ("%s: vmx_getreg error %d", __func__, error));
1994d17b5104SNeel Natu 	} else {
1995d17b5104SNeel Natu 		val = 1;
1996d17b5104SNeel Natu 	}
1997d17b5104SNeel Natu 	return (val);
1998d17b5104SNeel Natu }
1999d17b5104SNeel Natu 
2000d17b5104SNeel Natu static int
2001d17b5104SNeel Natu inout_str_addrsize(uint32_t inst_info)
2002d17b5104SNeel Natu {
2003d17b5104SNeel Natu 	uint32_t size;
2004d17b5104SNeel Natu 
2005d17b5104SNeel Natu 	size = (inst_info >> 7) & 0x7;
2006d17b5104SNeel Natu 	switch (size) {
2007d17b5104SNeel Natu 	case 0:
2008d17b5104SNeel Natu 		return (2);	/* 16 bit */
2009d17b5104SNeel Natu 	case 1:
2010d17b5104SNeel Natu 		return (4);	/* 32 bit */
2011d17b5104SNeel Natu 	case 2:
2012d17b5104SNeel Natu 		return (8);	/* 64 bit */
2013d17b5104SNeel Natu 	default:
2014d17b5104SNeel Natu 		panic("%s: invalid size encoding %d", __func__, size);
2015d17b5104SNeel Natu 	}
2016d17b5104SNeel Natu }
2017d17b5104SNeel Natu 
2018d17b5104SNeel Natu static void
2019869c8d19SJohn Baldwin inout_str_seginfo(struct vmx_vcpu *vcpu, uint32_t inst_info, int in,
2020869c8d19SJohn Baldwin     struct vm_inout_str *vis)
2021d17b5104SNeel Natu {
20225c272efaSRobert Wing 	int error __diagused, s;
2023d17b5104SNeel Natu 
2024d17b5104SNeel Natu 	if (in) {
2025d17b5104SNeel Natu 		vis->seg_name = VM_REG_GUEST_ES;
2026d17b5104SNeel Natu 	} else {
2027d17b5104SNeel Natu 		s = (inst_info >> 15) & 0x7;
2028d17b5104SNeel Natu 		vis->seg_name = vm_segment_name(s);
2029d17b5104SNeel Natu 	}
2030d17b5104SNeel Natu 
2031869c8d19SJohn Baldwin 	error = vmx_getdesc(vcpu, vis->seg_name, &vis->seg_desc);
2032d17b5104SNeel Natu 	KASSERT(error == 0, ("%s: vmx_getdesc error %d", __func__, error));
2033d17b5104SNeel Natu }
2034d17b5104SNeel Natu 
2035e4c8a13dSNeel Natu static void
2036e813a873SNeel Natu vmx_paging_info(struct vm_guest_paging *paging)
2037e813a873SNeel Natu {
2038e813a873SNeel Natu 	paging->cr3 = vmcs_guest_cr3();
2039e813a873SNeel Natu 	paging->cpl = vmx_cpl();
2040e813a873SNeel Natu 	paging->cpu_mode = vmx_cpu_mode();
2041e813a873SNeel Natu 	paging->paging_mode = vmx_paging_mode();
2042e813a873SNeel Natu }
2043e813a873SNeel Natu 
2044e813a873SNeel Natu static void
2045e4c8a13dSNeel Natu vmexit_inst_emul(struct vm_exit *vmexit, uint64_t gpa, uint64_t gla)
2046e4c8a13dSNeel Natu {
2047f7a9f178SNeel Natu 	struct vm_guest_paging *paging;
2048f7a9f178SNeel Natu 	uint32_t csar;
2049f7a9f178SNeel Natu 
2050f7a9f178SNeel Natu 	paging = &vmexit->u.inst_emul.paging;
2051f7a9f178SNeel Natu 
2052e4c8a13dSNeel Natu 	vmexit->exitcode = VM_EXITCODE_INST_EMUL;
20531c73ea3eSNeel Natu 	vmexit->inst_length = 0;
2054e4c8a13dSNeel Natu 	vmexit->u.inst_emul.gpa = gpa;
2055e4c8a13dSNeel Natu 	vmexit->u.inst_emul.gla = gla;
2056f7a9f178SNeel Natu 	vmx_paging_info(paging);
2057f7a9f178SNeel Natu 	switch (paging->cpu_mode) {
2058e4f605eeSTycho Nightingale 	case CPU_MODE_REAL:
2059e4f605eeSTycho Nightingale 		vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE);
2060e4f605eeSTycho Nightingale 		vmexit->u.inst_emul.cs_d = 0;
2061e4f605eeSTycho Nightingale 		break;
2062f7a9f178SNeel Natu 	case CPU_MODE_PROTECTED:
2063f7a9f178SNeel Natu 	case CPU_MODE_COMPATIBILITY:
2064e4f605eeSTycho Nightingale 		vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE);
2065f7a9f178SNeel Natu 		csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS);
2066f7a9f178SNeel Natu 		vmexit->u.inst_emul.cs_d = SEG_DESC_DEF32(csar);
2067f7a9f178SNeel Natu 		break;
2068f7a9f178SNeel Natu 	default:
2069e4f605eeSTycho Nightingale 		vmexit->u.inst_emul.cs_base = 0;
2070f7a9f178SNeel Natu 		vmexit->u.inst_emul.cs_d = 0;
2071f7a9f178SNeel Natu 		break;
2072f7a9f178SNeel Natu 	}
2073c2a875f9SNeel Natu 	vie_init(&vmexit->u.inst_emul.vie, NULL, 0);
2074e4c8a13dSNeel Natu }
2075e4c8a13dSNeel Natu 
2076366f6083SPeter Grehan static int
2077318224bbSNeel Natu ept_fault_type(uint64_t ept_qual)
2078a2da7af6SNeel Natu {
2079318224bbSNeel Natu 	int fault_type;
2080a2da7af6SNeel Natu 
2081318224bbSNeel Natu 	if (ept_qual & EPT_VIOLATION_DATA_WRITE)
2082318224bbSNeel Natu 		fault_type = VM_PROT_WRITE;
2083318224bbSNeel Natu 	else if (ept_qual & EPT_VIOLATION_INST_FETCH)
2084318224bbSNeel Natu 		fault_type = VM_PROT_EXECUTE;
2085318224bbSNeel Natu 	else
2086318224bbSNeel Natu 		fault_type= VM_PROT_READ;
2087318224bbSNeel Natu 
2088318224bbSNeel Natu 	return (fault_type);
2089318224bbSNeel Natu }
2090318224bbSNeel Natu 
2091490d56c5SEd Maste static bool
2092318224bbSNeel Natu ept_emulation_fault(uint64_t ept_qual)
2093318224bbSNeel Natu {
2094318224bbSNeel Natu 	int read, write;
2095318224bbSNeel Natu 
2096318224bbSNeel Natu 	/* EPT fault on an instruction fetch doesn't make sense here */
2097a2da7af6SNeel Natu 	if (ept_qual & EPT_VIOLATION_INST_FETCH)
2098490d56c5SEd Maste 		return (false);
2099a2da7af6SNeel Natu 
2100318224bbSNeel Natu 	/* EPT fault must be a read fault or a write fault */
2101a2da7af6SNeel Natu 	read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0;
2102a2da7af6SNeel Natu 	write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0;
21033b2b0011SPeter Grehan 	if ((read | write) == 0)
2104490d56c5SEd Maste 		return (false);
2105a2da7af6SNeel Natu 
2106a2da7af6SNeel Natu 	/*
21073b2b0011SPeter Grehan 	 * The EPT violation must have been caused by accessing a
21083b2b0011SPeter Grehan 	 * guest-physical address that is a translation of a guest-linear
21093b2b0011SPeter Grehan 	 * address.
2110a2da7af6SNeel Natu 	 */
2111a2da7af6SNeel Natu 	if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 ||
2112a2da7af6SNeel Natu 	    (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) {
2113490d56c5SEd Maste 		return (false);
2114a2da7af6SNeel Natu 	}
2115a2da7af6SNeel Natu 
2116490d56c5SEd Maste 	return (true);
2117a2da7af6SNeel Natu }
2118a2da7af6SNeel Natu 
2119159dd56fSNeel Natu static __inline int
21201aa51504SJohn Baldwin apic_access_virtualization(struct vmx_vcpu *vcpu)
2121159dd56fSNeel Natu {
2122159dd56fSNeel Natu 	uint32_t proc_ctls2;
2123159dd56fSNeel Natu 
21241aa51504SJohn Baldwin 	proc_ctls2 = vcpu->cap.proc_ctls2;
2125159dd56fSNeel Natu 	return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) ? 1 : 0);
2126159dd56fSNeel Natu }
2127159dd56fSNeel Natu 
2128159dd56fSNeel Natu static __inline int
21291aa51504SJohn Baldwin x2apic_virtualization(struct vmx_vcpu *vcpu)
2130159dd56fSNeel Natu {
2131159dd56fSNeel Natu 	uint32_t proc_ctls2;
2132159dd56fSNeel Natu 
21331aa51504SJohn Baldwin 	proc_ctls2 = vcpu->cap.proc_ctls2;
2134159dd56fSNeel Natu 	return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_X2APIC_MODE) ? 1 : 0);
2135159dd56fSNeel Natu }
2136159dd56fSNeel Natu 
2137a2da7af6SNeel Natu static int
21381aa51504SJohn Baldwin vmx_handle_apic_write(struct vmx_vcpu *vcpu, struct vlapic *vlapic,
2139159dd56fSNeel Natu     uint64_t qual)
214088c4b8d1SNeel Natu {
214188c4b8d1SNeel Natu 	int error, handled, offset;
2142159dd56fSNeel Natu 	uint32_t *apic_regs, vector;
214388c4b8d1SNeel Natu 	bool retu;
214488c4b8d1SNeel Natu 
2145a0efd3fbSJohn Baldwin 	handled = HANDLED;
214688c4b8d1SNeel Natu 	offset = APIC_WRITE_OFFSET(qual);
2147159dd56fSNeel Natu 
21481aa51504SJohn Baldwin 	if (!apic_access_virtualization(vcpu)) {
2149159dd56fSNeel Natu 		/*
2150159dd56fSNeel Natu 		 * In general there should not be any APIC write VM-exits
2151159dd56fSNeel Natu 		 * unless APIC-access virtualization is enabled.
2152159dd56fSNeel Natu 		 *
2153159dd56fSNeel Natu 		 * However self-IPI virtualization can legitimately trigger
2154159dd56fSNeel Natu 		 * an APIC-write VM-exit so treat it specially.
2155159dd56fSNeel Natu 		 */
21561aa51504SJohn Baldwin 		if (x2apic_virtualization(vcpu) &&
2157159dd56fSNeel Natu 		    offset == APIC_OFFSET_SELF_IPI) {
2158159dd56fSNeel Natu 			apic_regs = (uint32_t *)(vlapic->apic_page);
2159159dd56fSNeel Natu 			vector = apic_regs[APIC_OFFSET_SELF_IPI / 4];
2160159dd56fSNeel Natu 			vlapic_self_ipi_handler(vlapic, vector);
2161159dd56fSNeel Natu 			return (HANDLED);
2162159dd56fSNeel Natu 		} else
2163159dd56fSNeel Natu 			return (UNHANDLED);
2164159dd56fSNeel Natu 	}
2165159dd56fSNeel Natu 
216688c4b8d1SNeel Natu 	switch (offset) {
216788c4b8d1SNeel Natu 	case APIC_OFFSET_ID:
216888c4b8d1SNeel Natu 		vlapic_id_write_handler(vlapic);
216988c4b8d1SNeel Natu 		break;
217088c4b8d1SNeel Natu 	case APIC_OFFSET_LDR:
217188c4b8d1SNeel Natu 		vlapic_ldr_write_handler(vlapic);
217288c4b8d1SNeel Natu 		break;
217388c4b8d1SNeel Natu 	case APIC_OFFSET_DFR:
217488c4b8d1SNeel Natu 		vlapic_dfr_write_handler(vlapic);
217588c4b8d1SNeel Natu 		break;
217688c4b8d1SNeel Natu 	case APIC_OFFSET_SVR:
217788c4b8d1SNeel Natu 		vlapic_svr_write_handler(vlapic);
217888c4b8d1SNeel Natu 		break;
217988c4b8d1SNeel Natu 	case APIC_OFFSET_ESR:
218088c4b8d1SNeel Natu 		vlapic_esr_write_handler(vlapic);
218188c4b8d1SNeel Natu 		break;
218288c4b8d1SNeel Natu 	case APIC_OFFSET_ICR_LOW:
218388c4b8d1SNeel Natu 		retu = false;
218488c4b8d1SNeel Natu 		error = vlapic_icrlo_write_handler(vlapic, &retu);
218588c4b8d1SNeel Natu 		if (error != 0 || retu)
2186a0efd3fbSJohn Baldwin 			handled = UNHANDLED;
218788c4b8d1SNeel Natu 		break;
218888c4b8d1SNeel Natu 	case APIC_OFFSET_CMCI_LVT:
218988c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
219088c4b8d1SNeel Natu 		vlapic_lvt_write_handler(vlapic, offset);
219188c4b8d1SNeel Natu 		break;
219288c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_ICR:
219388c4b8d1SNeel Natu 		vlapic_icrtmr_write_handler(vlapic);
219488c4b8d1SNeel Natu 		break;
219588c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_DCR:
219688c4b8d1SNeel Natu 		vlapic_dcr_write_handler(vlapic);
219788c4b8d1SNeel Natu 		break;
219888c4b8d1SNeel Natu 	default:
2199a0efd3fbSJohn Baldwin 		handled = UNHANDLED;
220088c4b8d1SNeel Natu 		break;
220188c4b8d1SNeel Natu 	}
220288c4b8d1SNeel Natu 	return (handled);
220388c4b8d1SNeel Natu }
220488c4b8d1SNeel Natu 
220588c4b8d1SNeel Natu static bool
22061aa51504SJohn Baldwin apic_access_fault(struct vmx_vcpu *vcpu, uint64_t gpa)
220788c4b8d1SNeel Natu {
220888c4b8d1SNeel Natu 
22091aa51504SJohn Baldwin 	if (apic_access_virtualization(vcpu) &&
221088c4b8d1SNeel Natu 	    (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE))
221188c4b8d1SNeel Natu 		return (true);
221288c4b8d1SNeel Natu 	else
221388c4b8d1SNeel Natu 		return (false);
221488c4b8d1SNeel Natu }
221588c4b8d1SNeel Natu 
221688c4b8d1SNeel Natu static int
22171aa51504SJohn Baldwin vmx_handle_apic_access(struct vmx_vcpu *vcpu, struct vm_exit *vmexit)
221888c4b8d1SNeel Natu {
221988c4b8d1SNeel Natu 	uint64_t qual;
222088c4b8d1SNeel Natu 	int access_type, offset, allowed;
222188c4b8d1SNeel Natu 
22221aa51504SJohn Baldwin 	if (!apic_access_virtualization(vcpu))
222388c4b8d1SNeel Natu 		return (UNHANDLED);
222488c4b8d1SNeel Natu 
222588c4b8d1SNeel Natu 	qual = vmexit->u.vmx.exit_qualification;
222688c4b8d1SNeel Natu 	access_type = APIC_ACCESS_TYPE(qual);
222788c4b8d1SNeel Natu 	offset = APIC_ACCESS_OFFSET(qual);
222888c4b8d1SNeel Natu 
222988c4b8d1SNeel Natu 	allowed = 0;
223088c4b8d1SNeel Natu 	if (access_type == 0) {
223188c4b8d1SNeel Natu 		/*
223288c4b8d1SNeel Natu 		 * Read data access to the following registers is expected.
223388c4b8d1SNeel Natu 		 */
223488c4b8d1SNeel Natu 		switch (offset) {
223588c4b8d1SNeel Natu 		case APIC_OFFSET_APR:
223688c4b8d1SNeel Natu 		case APIC_OFFSET_PPR:
223788c4b8d1SNeel Natu 		case APIC_OFFSET_RRR:
223888c4b8d1SNeel Natu 		case APIC_OFFSET_CMCI_LVT:
223988c4b8d1SNeel Natu 		case APIC_OFFSET_TIMER_CCR:
224088c4b8d1SNeel Natu 			allowed = 1;
224188c4b8d1SNeel Natu 			break;
224288c4b8d1SNeel Natu 		default:
224388c4b8d1SNeel Natu 			break;
224488c4b8d1SNeel Natu 		}
224588c4b8d1SNeel Natu 	} else if (access_type == 1) {
224688c4b8d1SNeel Natu 		/*
224788c4b8d1SNeel Natu 		 * Write data access to the following registers is expected.
224888c4b8d1SNeel Natu 		 */
224988c4b8d1SNeel Natu 		switch (offset) {
225088c4b8d1SNeel Natu 		case APIC_OFFSET_VER:
225188c4b8d1SNeel Natu 		case APIC_OFFSET_APR:
225288c4b8d1SNeel Natu 		case APIC_OFFSET_PPR:
225388c4b8d1SNeel Natu 		case APIC_OFFSET_RRR:
225488c4b8d1SNeel Natu 		case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
225588c4b8d1SNeel Natu 		case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
225688c4b8d1SNeel Natu 		case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
225788c4b8d1SNeel Natu 		case APIC_OFFSET_CMCI_LVT:
225888c4b8d1SNeel Natu 		case APIC_OFFSET_TIMER_CCR:
225988c4b8d1SNeel Natu 			allowed = 1;
226088c4b8d1SNeel Natu 			break;
226188c4b8d1SNeel Natu 		default:
226288c4b8d1SNeel Natu 			break;
226388c4b8d1SNeel Natu 		}
226488c4b8d1SNeel Natu 	}
226588c4b8d1SNeel Natu 
226688c4b8d1SNeel Natu 	if (allowed) {
2267e4c8a13dSNeel Natu 		vmexit_inst_emul(vmexit, DEFAULT_APIC_BASE + offset,
2268e4c8a13dSNeel Natu 		    VIE_INVALID_GLA);
226988c4b8d1SNeel Natu 	}
227088c4b8d1SNeel Natu 
227188c4b8d1SNeel Natu 	/*
227288c4b8d1SNeel Natu 	 * Regardless of whether the APIC-access is allowed this handler
227388c4b8d1SNeel Natu 	 * always returns UNHANDLED:
227488c4b8d1SNeel Natu 	 * - if the access is allowed then it is handled by emulating the
227588c4b8d1SNeel Natu 	 *   instruction that caused the VM-exit (outside the critical section)
227688c4b8d1SNeel Natu 	 * - if the access is not allowed then it will be converted to an
227788c4b8d1SNeel Natu 	 *   exitcode of VM_EXITCODE_VMX and will be dealt with in userland.
227888c4b8d1SNeel Natu 	 */
227988c4b8d1SNeel Natu 	return (UNHANDLED);
228088c4b8d1SNeel Natu }
228188c4b8d1SNeel Natu 
22823d5444c8SNeel Natu static enum task_switch_reason
22833d5444c8SNeel Natu vmx_task_switch_reason(uint64_t qual)
22843d5444c8SNeel Natu {
22853d5444c8SNeel Natu 	int reason;
22863d5444c8SNeel Natu 
22873d5444c8SNeel Natu 	reason = (qual >> 30) & 0x3;
22883d5444c8SNeel Natu 	switch (reason) {
22893d5444c8SNeel Natu 	case 0:
22903d5444c8SNeel Natu 		return (TSR_CALL);
22913d5444c8SNeel Natu 	case 1:
22923d5444c8SNeel Natu 		return (TSR_IRET);
22933d5444c8SNeel Natu 	case 2:
22943d5444c8SNeel Natu 		return (TSR_JMP);
22953d5444c8SNeel Natu 	case 3:
22963d5444c8SNeel Natu 		return (TSR_IDT_GATE);
22973d5444c8SNeel Natu 	default:
22983d5444c8SNeel Natu 		panic("%s: invalid reason %d", __func__, reason);
22993d5444c8SNeel Natu 	}
23003d5444c8SNeel Natu }
23013d5444c8SNeel Natu 
230288c4b8d1SNeel Natu static int
230380cb5d84SJohn Baldwin emulate_wrmsr(struct vmx_vcpu *vcpu, u_int num, uint64_t val, bool *retu)
2304c3498942SNeel Natu {
2305c3498942SNeel Natu 	int error;
2306c3498942SNeel Natu 
2307c3498942SNeel Natu 	if (lapic_msr(num))
230880cb5d84SJohn Baldwin 		error = lapic_wrmsr(vcpu->vcpu, num, val, retu);
2309c3498942SNeel Natu 	else
231080cb5d84SJohn Baldwin 		error = vmx_wrmsr(vcpu, num, val, retu);
2311c3498942SNeel Natu 
2312c3498942SNeel Natu 	return (error);
2313c3498942SNeel Natu }
2314c3498942SNeel Natu 
2315c3498942SNeel Natu static int
231680cb5d84SJohn Baldwin emulate_rdmsr(struct vmx_vcpu *vcpu, u_int num, bool *retu)
2317c3498942SNeel Natu {
2318c3498942SNeel Natu 	struct vmxctx *vmxctx;
2319c3498942SNeel Natu 	uint64_t result;
2320c3498942SNeel Natu 	uint32_t eax, edx;
2321c3498942SNeel Natu 	int error;
2322c3498942SNeel Natu 
2323c3498942SNeel Natu 	if (lapic_msr(num))
232480cb5d84SJohn Baldwin 		error = lapic_rdmsr(vcpu->vcpu, num, &result, retu);
2325c3498942SNeel Natu 	else
232680cb5d84SJohn Baldwin 		error = vmx_rdmsr(vcpu, num, &result, retu);
2327c3498942SNeel Natu 
2328c3498942SNeel Natu 	if (error == 0) {
2329c3498942SNeel Natu 		eax = result;
23301aa51504SJohn Baldwin 		vmxctx = &vcpu->ctx;
2331c3498942SNeel Natu 		error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RAX, eax);
2332c3498942SNeel Natu 		KASSERT(error == 0, ("vmxctx_setreg(rax) error %d", error));
2333c3498942SNeel Natu 
2334c3498942SNeel Natu 		edx = result >> 32;
2335c3498942SNeel Natu 		error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RDX, edx);
2336c3498942SNeel Natu 		KASSERT(error == 0, ("vmxctx_setreg(rdx) error %d", error));
2337c3498942SNeel Natu 	}
2338c3498942SNeel Natu 
2339c3498942SNeel Natu 	return (error);
2340c3498942SNeel Natu }
2341c3498942SNeel Natu 
2342c3498942SNeel Natu static int
23431aa51504SJohn Baldwin vmx_exit_process(struct vmx *vmx, struct vmx_vcpu *vcpu, struct vm_exit *vmexit)
2344366f6083SPeter Grehan {
2345c9c75df4SNeel Natu 	int error, errcode, errcode_valid, handled, in;
2346366f6083SPeter Grehan 	struct vmxctx *vmxctx;
234788c4b8d1SNeel Natu 	struct vlapic *vlapic;
2348d17b5104SNeel Natu 	struct vm_inout_str *vis;
23493d5444c8SNeel Natu 	struct vm_task_switch *ts;
2350d17b5104SNeel Natu 	uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, inst_info;
2351b0538143SNeel Natu 	uint32_t intr_type, intr_vec, reason;
2352091d4532SNeel Natu 	uint64_t exitintinfo, qual, gpa;
2353*2ee1a18dSDmitry Chagin #ifdef KDTRACE_HOOKS
23541aa51504SJohn Baldwin 	int vcpuid;
2355*2ee1a18dSDmitry Chagin #endif
2356becd9849SNeel Natu 	bool retu;
2357366f6083SPeter Grehan 
2358160471d2SNeel Natu 	CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0);
2359c308b23bSNeel Natu 	CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0);
2360160471d2SNeel Natu 
2361a0efd3fbSJohn Baldwin 	handled = UNHANDLED;
23621aa51504SJohn Baldwin 	vmxctx = &vcpu->ctx;
2363*2ee1a18dSDmitry Chagin #ifdef KDTRACE_HOOKS
23641aa51504SJohn Baldwin 	vcpuid = vcpu->vcpuid;
2365*2ee1a18dSDmitry Chagin #endif
23660492757cSNeel Natu 
2367366f6083SPeter Grehan 	qual = vmexit->u.vmx.exit_qualification;
2368318224bbSNeel Natu 	reason = vmexit->u.vmx.exit_reason;
2369366f6083SPeter Grehan 	vmexit->exitcode = VM_EXITCODE_BOGUS;
2370366f6083SPeter Grehan 
23713dc3d32aSJohn Baldwin 	vmm_stat_incr(vcpu->vcpu, VMEXIT_COUNT, 1);
23721aa51504SJohn Baldwin 	SDT_PROBE3(vmm, vmx, exit, entry, vmx, vcpuid, vmexit);
237361592433SNeel Natu 
2374318224bbSNeel Natu 	/*
2375b0538143SNeel Natu 	 * VM-entry failures during or after loading guest state.
2376b0538143SNeel Natu 	 *
2377b0538143SNeel Natu 	 * These VM-exits are uncommon but must be handled specially
2378b0538143SNeel Natu 	 * as most VM-exit fields are not populated as usual.
2379b0538143SNeel Natu 	 */
2380b0538143SNeel Natu 	if (__predict_false(reason == EXIT_REASON_MCE_DURING_ENTRY)) {
238157e0119eSJohn Baldwin 		VMX_CTR0(vcpu, "Handling MCE during VM-entry");
2382b0538143SNeel Natu 		__asm __volatile("int $18");
2383b0538143SNeel Natu 		return (1);
2384b0538143SNeel Natu 	}
2385b0538143SNeel Natu 
2386b0538143SNeel Natu 	/*
23873d5444c8SNeel Natu 	 * VM exits that can be triggered during event delivery need to
23883d5444c8SNeel Natu 	 * be handled specially by re-injecting the event if the IDT
23893d5444c8SNeel Natu 	 * vectoring information field's valid bit is set.
2390318224bbSNeel Natu 	 *
2391318224bbSNeel Natu 	 * See "Information for VM Exits During Event Delivery" in Intel SDM
2392318224bbSNeel Natu 	 * for details.
2393318224bbSNeel Natu 	 */
2394318224bbSNeel Natu 	idtvec_info = vmcs_idt_vectoring_info();
2395318224bbSNeel Natu 	if (idtvec_info & VMCS_IDT_VEC_VALID) {
2396318224bbSNeel Natu 		idtvec_info &= ~(1 << 12); /* clear undefined bit */
2397091d4532SNeel Natu 		exitintinfo = idtvec_info;
2398318224bbSNeel Natu 		if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
2399318224bbSNeel Natu 			idtvec_err = vmcs_idt_vectoring_err();
2400091d4532SNeel Natu 			exitintinfo |= (uint64_t)idtvec_err << 32;
2401318224bbSNeel Natu 		}
240280cb5d84SJohn Baldwin 		error = vm_exit_intinfo(vcpu->vcpu, exitintinfo);
2403091d4532SNeel Natu 		KASSERT(error == 0, ("%s: vm_set_intinfo error %d",
2404091d4532SNeel Natu 		    __func__, error));
2405091d4532SNeel Natu 
2406160471d2SNeel Natu 		/*
2407160471d2SNeel Natu 		 * If 'virtual NMIs' are being used and the VM-exit
2408160471d2SNeel Natu 		 * happened while injecting an NMI during the previous
2409091d4532SNeel Natu 		 * VM-entry, then clear "blocking by NMI" in the
2410091d4532SNeel Natu 		 * Guest Interruptibility-State so the NMI can be
2411091d4532SNeel Natu 		 * reinjected on the subsequent VM-entry.
2412091d4532SNeel Natu 		 *
2413091d4532SNeel Natu 		 * However, if the NMI was being delivered through a task
2414091d4532SNeel Natu 		 * gate, then the new task must start execution with NMIs
2415091d4532SNeel Natu 		 * blocked so don't clear NMI blocking in this case.
2416160471d2SNeel Natu 		 */
2417091d4532SNeel Natu 		intr_type = idtvec_info & VMCS_INTR_T_MASK;
2418091d4532SNeel Natu 		if (intr_type == VMCS_INTR_T_NMI) {
2419091d4532SNeel Natu 			if (reason != EXIT_REASON_TASK_SWITCH)
2420869c8d19SJohn Baldwin 				vmx_clear_nmi_blocking(vcpu);
2421091d4532SNeel Natu 			else
2422869c8d19SJohn Baldwin 				vmx_assert_nmi_blocking(vcpu);
2423160471d2SNeel Natu 		}
2424091d4532SNeel Natu 
2425091d4532SNeel Natu 		/*
2426091d4532SNeel Natu 		 * Update VM-entry instruction length if the event being
2427091d4532SNeel Natu 		 * delivered was a software interrupt or software exception.
2428091d4532SNeel Natu 		 */
2429091d4532SNeel Natu 		if (intr_type == VMCS_INTR_T_SWINTR ||
2430091d4532SNeel Natu 		    intr_type == VMCS_INTR_T_PRIV_SWEXCEPTION ||
2431091d4532SNeel Natu 		    intr_type == VMCS_INTR_T_SWEXCEPTION) {
24323de83862SNeel Natu 			vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
2433318224bbSNeel Natu 		}
2434318224bbSNeel Natu 	}
2435318224bbSNeel Natu 
2436318224bbSNeel Natu 	switch (reason) {
24373d5444c8SNeel Natu 	case EXIT_REASON_TASK_SWITCH:
24383d5444c8SNeel Natu 		ts = &vmexit->u.task_switch;
24393d5444c8SNeel Natu 		ts->tsssel = qual & 0xffff;
24403d5444c8SNeel Natu 		ts->reason = vmx_task_switch_reason(qual);
24413d5444c8SNeel Natu 		ts->ext = 0;
24423d5444c8SNeel Natu 		ts->errcode_valid = 0;
24433d5444c8SNeel Natu 		vmx_paging_info(&ts->paging);
24443d5444c8SNeel Natu 		/*
24453d5444c8SNeel Natu 		 * If the task switch was due to a CALL, JMP, IRET, software
24463d5444c8SNeel Natu 		 * interrupt (INT n) or software exception (INT3, INTO),
24473d5444c8SNeel Natu 		 * then the saved %rip references the instruction that caused
24483d5444c8SNeel Natu 		 * the task switch. The instruction length field in the VMCS
24493d5444c8SNeel Natu 		 * is valid in this case.
24503d5444c8SNeel Natu 		 *
24513d5444c8SNeel Natu 		 * In all other cases (e.g., NMI, hardware exception) the
24523d5444c8SNeel Natu 		 * saved %rip is one that would have been saved in the old TSS
24533d5444c8SNeel Natu 		 * had the task switch completed normally so the instruction
24543d5444c8SNeel Natu 		 * length field is not needed in this case and is explicitly
24553d5444c8SNeel Natu 		 * set to 0.
24563d5444c8SNeel Natu 		 */
24573d5444c8SNeel Natu 		if (ts->reason == TSR_IDT_GATE) {
24583d5444c8SNeel Natu 			KASSERT(idtvec_info & VMCS_IDT_VEC_VALID,
2459091d4532SNeel Natu 			    ("invalid idtvec_info %#x for IDT task switch",
24603d5444c8SNeel Natu 			    idtvec_info));
24613d5444c8SNeel Natu 			intr_type = idtvec_info & VMCS_INTR_T_MASK;
24623d5444c8SNeel Natu 			if (intr_type != VMCS_INTR_T_SWINTR &&
24633d5444c8SNeel Natu 			    intr_type != VMCS_INTR_T_SWEXCEPTION &&
24643d5444c8SNeel Natu 			    intr_type != VMCS_INTR_T_PRIV_SWEXCEPTION) {
24653d5444c8SNeel Natu 				/* Task switch triggered by external event */
24663d5444c8SNeel Natu 				ts->ext = 1;
24673d5444c8SNeel Natu 				vmexit->inst_length = 0;
24683d5444c8SNeel Natu 				if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
24693d5444c8SNeel Natu 					ts->errcode_valid = 1;
24703d5444c8SNeel Natu 					ts->errcode = vmcs_idt_vectoring_err();
24713d5444c8SNeel Natu 				}
24723d5444c8SNeel Natu 			}
24733d5444c8SNeel Natu 		}
24743d5444c8SNeel Natu 		vmexit->exitcode = VM_EXITCODE_TASK_SWITCH;
24751aa51504SJohn Baldwin 		SDT_PROBE4(vmm, vmx, exit, taskswitch, vmx, vcpuid, vmexit, ts);
247657e0119eSJohn Baldwin 		VMX_CTR4(vcpu, "task switch reason %d, tss 0x%04x, "
24773d5444c8SNeel Natu 		    "%s errcode 0x%016lx", ts->reason, ts->tsssel,
24783d5444c8SNeel Natu 		    ts->ext ? "external" : "internal",
24793d5444c8SNeel Natu 		    ((uint64_t)ts->errcode << 32) | ts->errcode_valid);
24803d5444c8SNeel Natu 		break;
2481366f6083SPeter Grehan 	case EXIT_REASON_CR_ACCESS:
24823dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_CR_ACCESS, 1);
24831aa51504SJohn Baldwin 		SDT_PROBE4(vmm, vmx, exit, craccess, vmx, vcpuid, vmexit, qual);
2484594db002STycho Nightingale 		switch (qual & 0xf) {
2485594db002STycho Nightingale 		case 0:
24861aa51504SJohn Baldwin 			handled = vmx_emulate_cr0_access(vcpu, qual);
2487594db002STycho Nightingale 			break;
2488594db002STycho Nightingale 		case 4:
24891aa51504SJohn Baldwin 			handled = vmx_emulate_cr4_access(vcpu, qual);
2490594db002STycho Nightingale 			break;
2491594db002STycho Nightingale 		case 8:
2492594db002STycho Nightingale 			handled = vmx_emulate_cr8_access(vmx, vcpu, qual);
2493594db002STycho Nightingale 			break;
2494594db002STycho Nightingale 		}
2495366f6083SPeter Grehan 		break;
2496366f6083SPeter Grehan 	case EXIT_REASON_RDMSR:
24973dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_RDMSR, 1);
2498becd9849SNeel Natu 		retu = false;
2499366f6083SPeter Grehan 		ecx = vmxctx->guest_rcx;
250057e0119eSJohn Baldwin 		VMX_CTR1(vcpu, "rdmsr 0x%08x", ecx);
25011aa51504SJohn Baldwin 		SDT_PROBE4(vmm, vmx, exit, rdmsr, vmx, vcpuid, vmexit, ecx);
250280cb5d84SJohn Baldwin 		error = emulate_rdmsr(vcpu, ecx, &retu);
2503b42206f3SNeel Natu 		if (error) {
2504366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_RDMSR;
2505366f6083SPeter Grehan 			vmexit->u.msr.code = ecx;
2506becd9849SNeel Natu 		} else if (!retu) {
2507a0efd3fbSJohn Baldwin 			handled = HANDLED;
2508becd9849SNeel Natu 		} else {
2509becd9849SNeel Natu 			/* Return to userspace with a valid exitcode */
2510becd9849SNeel Natu 			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
2511c3498942SNeel Natu 			    ("emulate_rdmsr retu with bogus exitcode"));
2512becd9849SNeel Natu 		}
2513366f6083SPeter Grehan 		break;
2514366f6083SPeter Grehan 	case EXIT_REASON_WRMSR:
25153dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_WRMSR, 1);
2516becd9849SNeel Natu 		retu = false;
2517366f6083SPeter Grehan 		eax = vmxctx->guest_rax;
2518366f6083SPeter Grehan 		ecx = vmxctx->guest_rcx;
2519366f6083SPeter Grehan 		edx = vmxctx->guest_rdx;
252057e0119eSJohn Baldwin 		VMX_CTR2(vcpu, "wrmsr 0x%08x value 0x%016lx",
25212cb97c9dSNeel Natu 		    ecx, (uint64_t)edx << 32 | eax);
25221aa51504SJohn Baldwin 		SDT_PROBE5(vmm, vmx, exit, wrmsr, vmx, vmexit, vcpuid, ecx,
25236ac73777STycho Nightingale 		    (uint64_t)edx << 32 | eax);
252480cb5d84SJohn Baldwin 		error = emulate_wrmsr(vcpu, ecx, (uint64_t)edx << 32 | eax,
252580cb5d84SJohn Baldwin 		    &retu);
2526b42206f3SNeel Natu 		if (error) {
2527366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_WRMSR;
2528366f6083SPeter Grehan 			vmexit->u.msr.code = ecx;
2529366f6083SPeter Grehan 			vmexit->u.msr.wval = (uint64_t)edx << 32 | eax;
2530becd9849SNeel Natu 		} else if (!retu) {
2531a0efd3fbSJohn Baldwin 			handled = HANDLED;
2532becd9849SNeel Natu 		} else {
2533becd9849SNeel Natu 			/* Return to userspace with a valid exitcode */
2534becd9849SNeel Natu 			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
2535becd9849SNeel Natu 			    ("emulate_wrmsr retu with bogus exitcode"));
2536becd9849SNeel Natu 		}
2537366f6083SPeter Grehan 		break;
2538366f6083SPeter Grehan 	case EXIT_REASON_HLT:
25393dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_HLT, 1);
25401aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, halt, vmx, vcpuid, vmexit);
2541366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_HLT;
25423de83862SNeel Natu 		vmexit->u.hlt.rflags = vmcs_read(VMCS_GUEST_RFLAGS);
2543490768e2STycho Nightingale 		if (virtual_interrupt_delivery)
2544490768e2STycho Nightingale 			vmexit->u.hlt.intr_status =
2545490768e2STycho Nightingale 			    vmcs_read(VMCS_GUEST_INTR_STATUS);
2546490768e2STycho Nightingale 		else
2547490768e2STycho Nightingale 			vmexit->u.hlt.intr_status = 0;
2548366f6083SPeter Grehan 		break;
2549366f6083SPeter Grehan 	case EXIT_REASON_MTF:
25503dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_MTRAP, 1);
25511aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, mtrap, vmx, vcpuid, vmexit);
2552366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_MTRAP;
2553c9c75df4SNeel Natu 		vmexit->inst_length = 0;
2554366f6083SPeter Grehan 		break;
2555366f6083SPeter Grehan 	case EXIT_REASON_PAUSE:
25563dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_PAUSE, 1);
25571aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, pause, vmx, vcpuid, vmexit);
2558366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_PAUSE;
2559366f6083SPeter Grehan 		break;
2560366f6083SPeter Grehan 	case EXIT_REASON_INTR_WINDOW:
25613dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_INTR_WINDOW, 1);
25621aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, intrwindow, vmx, vcpuid, vmexit);
2563869c8d19SJohn Baldwin 		vmx_clear_int_window_exiting(vcpu);
2564b5aaf7b2SNeel Natu 		return (1);
2565366f6083SPeter Grehan 	case EXIT_REASON_EXT_INTR:
2566366f6083SPeter Grehan 		/*
2567366f6083SPeter Grehan 		 * External interrupts serve only to cause VM exits and allow
2568366f6083SPeter Grehan 		 * the host interrupt handler to run.
2569366f6083SPeter Grehan 		 *
2570366f6083SPeter Grehan 		 * If this external interrupt triggers a virtual interrupt
2571366f6083SPeter Grehan 		 * to a VM, then that state will be recorded by the
2572366f6083SPeter Grehan 		 * host interrupt handler in the VM's softc. We will inject
2573366f6083SPeter Grehan 		 * this virtual interrupt during the subsequent VM enter.
2574366f6083SPeter Grehan 		 */
2575f7d47425SNeel Natu 		intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
25766ac73777STycho Nightingale 		SDT_PROBE4(vmm, vmx, exit, interrupt,
25771aa51504SJohn Baldwin 		    vmx, vcpuid, vmexit, intr_info);
2578722b6744SJohn Baldwin 
2579722b6744SJohn Baldwin 		/*
2580722b6744SJohn Baldwin 		 * XXX: Ignore this exit if VMCS_INTR_VALID is not set.
2581ad3e3687SJohn Baldwin 		 * This appears to be a bug in VMware Fusion?
2582722b6744SJohn Baldwin 		 */
2583722b6744SJohn Baldwin 		if (!(intr_info & VMCS_INTR_VALID))
2584722b6744SJohn Baldwin 			return (1);
2585160471d2SNeel Natu 		KASSERT((intr_info & VMCS_INTR_VALID) != 0 &&
2586160471d2SNeel Natu 		    (intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_HWINTR,
2587f7d47425SNeel Natu 		    ("VM exit interruption info invalid: %#x", intr_info));
2588f7d47425SNeel Natu 		vmx_trigger_hostintr(intr_info & 0xff);
2589366f6083SPeter Grehan 
2590366f6083SPeter Grehan 		/*
2591366f6083SPeter Grehan 		 * This is special. We want to treat this as an 'handled'
2592366f6083SPeter Grehan 		 * VM-exit but not increment the instruction pointer.
2593366f6083SPeter Grehan 		 */
25943dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_EXTINT, 1);
2595366f6083SPeter Grehan 		return (1);
2596366f6083SPeter Grehan 	case EXIT_REASON_NMI_WINDOW:
25971aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, nmiwindow, vmx, vcpuid, vmexit);
2598366f6083SPeter Grehan 		/* Exit to allow the pending virtual NMI to be injected */
259980cb5d84SJohn Baldwin 		if (vm_nmi_pending(vcpu->vcpu))
260080cb5d84SJohn Baldwin 			vmx_inject_nmi(vcpu);
2601869c8d19SJohn Baldwin 		vmx_clear_nmi_window_exiting(vcpu);
26023dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_NMI_WINDOW, 1);
2603366f6083SPeter Grehan 		return (1);
2604366f6083SPeter Grehan 	case EXIT_REASON_INOUT:
26053dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_INOUT, 1);
2606366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_INOUT;
2607366f6083SPeter Grehan 		vmexit->u.inout.bytes = (qual & 0x7) + 1;
2608d17b5104SNeel Natu 		vmexit->u.inout.in = in = (qual & 0x8) ? 1 : 0;
2609366f6083SPeter Grehan 		vmexit->u.inout.string = (qual & 0x10) ? 1 : 0;
2610366f6083SPeter Grehan 		vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0;
2611366f6083SPeter Grehan 		vmexit->u.inout.port = (uint16_t)(qual >> 16);
2612366f6083SPeter Grehan 		vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax);
2613d17b5104SNeel Natu 		if (vmexit->u.inout.string) {
2614d17b5104SNeel Natu 			inst_info = vmcs_read(VMCS_EXIT_INSTRUCTION_INFO);
2615d17b5104SNeel Natu 			vmexit->exitcode = VM_EXITCODE_INOUT_STR;
2616d17b5104SNeel Natu 			vis = &vmexit->u.inout_str;
2617e813a873SNeel Natu 			vmx_paging_info(&vis->paging);
2618d17b5104SNeel Natu 			vis->rflags = vmcs_read(VMCS_GUEST_RFLAGS);
2619d17b5104SNeel Natu 			vis->cr0 = vmcs_read(VMCS_GUEST_CR0);
2620869c8d19SJohn Baldwin 			vis->index = inout_str_index(vcpu, in);
2621869c8d19SJohn Baldwin 			vis->count = inout_str_count(vcpu, vis->inout.rep);
2622d17b5104SNeel Natu 			vis->addrsize = inout_str_addrsize(inst_info);
2623869c8d19SJohn Baldwin 			inout_str_seginfo(vcpu, inst_info, in, vis);
2624762fd208STycho Nightingale 		}
26251aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, inout, vmx, vcpuid, vmexit);
2626366f6083SPeter Grehan 		break;
2627366f6083SPeter Grehan 	case EXIT_REASON_CPUID:
26283dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_CPUID, 1);
26291aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, cpuid, vmx, vcpuid, vmexit);
263080cb5d84SJohn Baldwin 		handled = vmx_handle_cpuid(vcpu, vmxctx);
2631366f6083SPeter Grehan 		break;
2632e5a1d950SNeel Natu 	case EXIT_REASON_EXCEPTION:
26333dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_EXCEPTION, 1);
2634e5a1d950SNeel Natu 		intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
2635e5a1d950SNeel Natu 		KASSERT((intr_info & VMCS_INTR_VALID) != 0,
2636e5a1d950SNeel Natu 		    ("VM exit interruption info invalid: %#x", intr_info));
2637c308b23bSNeel Natu 
2638b0538143SNeel Natu 		intr_vec = intr_info & 0xff;
2639b0538143SNeel Natu 		intr_type = intr_info & VMCS_INTR_T_MASK;
2640b0538143SNeel Natu 
2641e5a1d950SNeel Natu 		/*
2642e5a1d950SNeel Natu 		 * If Virtual NMIs control is 1 and the VM-exit is due to a
2643e5a1d950SNeel Natu 		 * fault encountered during the execution of IRET then we must
2644e5a1d950SNeel Natu 		 * restore the state of "virtual-NMI blocking" before resuming
2645e5a1d950SNeel Natu 		 * the guest.
2646e5a1d950SNeel Natu 		 *
2647e5a1d950SNeel Natu 		 * See "Resuming Guest Software after Handling an Exception".
2648091d4532SNeel Natu 		 * See "Information for VM Exits Due to Vectored Events".
2649e5a1d950SNeel Natu 		 */
2650e5a1d950SNeel Natu 		if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
2651b0538143SNeel Natu 		    (intr_vec != IDT_DF) &&
2652e5a1d950SNeel Natu 		    (intr_info & EXIT_QUAL_NMIUDTI) != 0)
2653869c8d19SJohn Baldwin 			vmx_restore_nmi_blocking(vcpu);
2654c308b23bSNeel Natu 
2655c308b23bSNeel Natu 		/*
265662fbd7c2SNeel Natu 		 * The NMI has already been handled in vmx_exit_handle_nmi().
2657c308b23bSNeel Natu 		 */
2658b0538143SNeel Natu 		if (intr_type == VMCS_INTR_T_NMI)
2659c308b23bSNeel Natu 			return (1);
2660b0538143SNeel Natu 
2661b0538143SNeel Natu 		/*
2662b0538143SNeel Natu 		 * Call the machine check handler by hand. Also don't reflect
2663b0538143SNeel Natu 		 * the machine check back into the guest.
2664b0538143SNeel Natu 		 */
2665b0538143SNeel Natu 		if (intr_vec == IDT_MC) {
266657e0119eSJohn Baldwin 			VMX_CTR0(vcpu, "Vectoring to MCE handler");
2667b0538143SNeel Natu 			__asm __volatile("int $18");
2668b0538143SNeel Natu 			return (1);
2669b0538143SNeel Natu 		}
2670b0538143SNeel Natu 
2671cbd03a9dSJohn Baldwin 		/*
2672cbd03a9dSJohn Baldwin 		 * If the hypervisor has requested user exits for
2673cbd03a9dSJohn Baldwin 		 * debug exceptions, bounce them out to userland.
2674cbd03a9dSJohn Baldwin 		 */
2675cbd03a9dSJohn Baldwin 		if (intr_type == VMCS_INTR_T_SWEXCEPTION && intr_vec == IDT_BP &&
26761aa51504SJohn Baldwin 		    (vcpu->cap.set & (1 << VM_CAP_BPT_EXIT))) {
2677cbd03a9dSJohn Baldwin 			vmexit->exitcode = VM_EXITCODE_BPT;
2678cbd03a9dSJohn Baldwin 			vmexit->u.bpt.inst_length = vmexit->inst_length;
2679cbd03a9dSJohn Baldwin 			vmexit->inst_length = 0;
2680cbd03a9dSJohn Baldwin 			break;
2681cbd03a9dSJohn Baldwin 		}
2682cbd03a9dSJohn Baldwin 
2683b0538143SNeel Natu 		if (intr_vec == IDT_PF) {
2684b0538143SNeel Natu 			error = vmxctx_setreg(vmxctx, VM_REG_GUEST_CR2, qual);
2685b0538143SNeel Natu 			KASSERT(error == 0, ("%s: vmxctx_setreg(cr2) error %d",
2686b0538143SNeel Natu 			    __func__, error));
2687b0538143SNeel Natu 		}
2688b0538143SNeel Natu 
2689b0538143SNeel Natu 		/*
2690b0538143SNeel Natu 		 * Software exceptions exhibit trap-like behavior. This in
2691b0538143SNeel Natu 		 * turn requires populating the VM-entry instruction length
2692b0538143SNeel Natu 		 * so that the %rip in the trap frame is past the INT3/INTO
2693b0538143SNeel Natu 		 * instruction.
2694b0538143SNeel Natu 		 */
2695b0538143SNeel Natu 		if (intr_type == VMCS_INTR_T_SWEXCEPTION)
2696b0538143SNeel Natu 			vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
2697b0538143SNeel Natu 
2698b0538143SNeel Natu 		/* Reflect all other exceptions back into the guest */
2699c9c75df4SNeel Natu 		errcode_valid = errcode = 0;
2700b0538143SNeel Natu 		if (intr_info & VMCS_INTR_DEL_ERRCODE) {
2701c9c75df4SNeel Natu 			errcode_valid = 1;
2702c9c75df4SNeel Natu 			errcode = vmcs_read(VMCS_EXIT_INTR_ERRCODE);
2703b0538143SNeel Natu 		}
270457e0119eSJohn Baldwin 		VMX_CTR2(vcpu, "Reflecting exception %d/%#x into "
2705c9c75df4SNeel Natu 		    "the guest", intr_vec, errcode);
27066ac73777STycho Nightingale 		SDT_PROBE5(vmm, vmx, exit, exception,
27071aa51504SJohn Baldwin 		    vmx, vcpuid, vmexit, intr_vec, errcode);
2708d3956e46SJohn Baldwin 		error = vm_inject_exception(vcpu->vcpu, intr_vec,
2709c9c75df4SNeel Natu 		    errcode_valid, errcode, 0);
2710b0538143SNeel Natu 		KASSERT(error == 0, ("%s: vm_inject_exception error %d",
2711b0538143SNeel Natu 		    __func__, error));
2712b0538143SNeel Natu 		return (1);
2713b0538143SNeel Natu 
2714cd942e0fSPeter Grehan 	case EXIT_REASON_EPT_FAULT:
2715318224bbSNeel Natu 		/*
2716318224bbSNeel Natu 		 * If 'gpa' lies within the address space allocated to
2717318224bbSNeel Natu 		 * memory then this must be a nested page fault otherwise
2718318224bbSNeel Natu 		 * this must be an instruction that accesses MMIO space.
2719318224bbSNeel Natu 		 */
2720a2da7af6SNeel Natu 		gpa = vmcs_gpa();
272180cb5d84SJohn Baldwin 		if (vm_mem_allocated(vcpu->vcpu, gpa) ||
27221aa51504SJohn Baldwin 		    apic_access_fault(vcpu, gpa)) {
2723cd942e0fSPeter Grehan 			vmexit->exitcode = VM_EXITCODE_PAGING;
2724d087a399SNeel Natu 			vmexit->inst_length = 0;
272513ec9371SPeter Grehan 			vmexit->u.paging.gpa = gpa;
2726318224bbSNeel Natu 			vmexit->u.paging.fault_type = ept_fault_type(qual);
27273dc3d32aSJohn Baldwin 			vmm_stat_incr(vcpu->vcpu, VMEXIT_NESTED_FAULT, 1);
27286ac73777STycho Nightingale 			SDT_PROBE5(vmm, vmx, exit, nestedfault,
27291aa51504SJohn Baldwin 			    vmx, vcpuid, vmexit, gpa, qual);
2730318224bbSNeel Natu 		} else if (ept_emulation_fault(qual)) {
2731e4c8a13dSNeel Natu 			vmexit_inst_emul(vmexit, gpa, vmcs_gla());
27323dc3d32aSJohn Baldwin 			vmm_stat_incr(vcpu->vcpu, VMEXIT_INST_EMUL, 1);
27336ac73777STycho Nightingale 			SDT_PROBE4(vmm, vmx, exit, mmiofault,
27341aa51504SJohn Baldwin 			    vmx, vcpuid, vmexit, gpa);
2735a2da7af6SNeel Natu 		}
2736e5a1d950SNeel Natu 		/*
2737e5a1d950SNeel Natu 		 * If Virtual NMIs control is 1 and the VM-exit is due to an
2738e5a1d950SNeel Natu 		 * EPT fault during the execution of IRET then we must restore
2739e5a1d950SNeel Natu 		 * the state of "virtual-NMI blocking" before resuming.
2740e5a1d950SNeel Natu 		 *
2741e5a1d950SNeel Natu 		 * See description of "NMI unblocking due to IRET" in
2742e5a1d950SNeel Natu 		 * "Exit Qualification for EPT Violations".
2743e5a1d950SNeel Natu 		 */
2744e5a1d950SNeel Natu 		if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
2745e5a1d950SNeel Natu 		    (qual & EXIT_QUAL_NMIUDTI) != 0)
2746869c8d19SJohn Baldwin 			vmx_restore_nmi_blocking(vcpu);
2747cd942e0fSPeter Grehan 		break;
274830b94db8SNeel Natu 	case EXIT_REASON_VIRTUALIZED_EOI:
274930b94db8SNeel Natu 		vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI;
275030b94db8SNeel Natu 		vmexit->u.ioapic_eoi.vector = qual & 0xFF;
27511aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, eoi, vmx, vcpuid, vmexit);
275230b94db8SNeel Natu 		vmexit->inst_length = 0;	/* trap-like */
275330b94db8SNeel Natu 		break;
275488c4b8d1SNeel Natu 	case EXIT_REASON_APIC_ACCESS:
27551aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, apicaccess, vmx, vcpuid, vmexit);
27561aa51504SJohn Baldwin 		handled = vmx_handle_apic_access(vcpu, vmexit);
275788c4b8d1SNeel Natu 		break;
275888c4b8d1SNeel Natu 	case EXIT_REASON_APIC_WRITE:
275988c4b8d1SNeel Natu 		/*
276088c4b8d1SNeel Natu 		 * APIC-write VM exit is trap-like so the %rip is already
276188c4b8d1SNeel Natu 		 * pointing to the next instruction.
276288c4b8d1SNeel Natu 		 */
276388c4b8d1SNeel Natu 		vmexit->inst_length = 0;
2764d3956e46SJohn Baldwin 		vlapic = vm_lapic(vcpu->vcpu);
27656ac73777STycho Nightingale 		SDT_PROBE4(vmm, vmx, exit, apicwrite,
27661aa51504SJohn Baldwin 		    vmx, vcpuid, vmexit, vlapic);
27671aa51504SJohn Baldwin 		handled = vmx_handle_apic_write(vcpu, vlapic, qual);
276888c4b8d1SNeel Natu 		break;
2769abb023fbSJohn Baldwin 	case EXIT_REASON_XSETBV:
27701aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, xsetbv, vmx, vcpuid, vmexit);
2771a0efd3fbSJohn Baldwin 		handled = vmx_emulate_xsetbv(vmx, vcpu, vmexit);
2772abb023fbSJohn Baldwin 		break;
277365145c7fSNeel Natu 	case EXIT_REASON_MONITOR:
27741aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, monitor, vmx, vcpuid, vmexit);
277565145c7fSNeel Natu 		vmexit->exitcode = VM_EXITCODE_MONITOR;
277665145c7fSNeel Natu 		break;
277765145c7fSNeel Natu 	case EXIT_REASON_MWAIT:
27781aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, mwait, vmx, vcpuid, vmexit);
277965145c7fSNeel Natu 		vmexit->exitcode = VM_EXITCODE_MWAIT;
278065145c7fSNeel Natu 		break;
27811bc51badSMichael Reifenberger 	case EXIT_REASON_TPR:
2782d3956e46SJohn Baldwin 		vlapic = vm_lapic(vcpu->vcpu);
27831bc51badSMichael Reifenberger 		vlapic_sync_tpr(vlapic);
27841bc51badSMichael Reifenberger 		vmexit->inst_length = 0;
27851bc51badSMichael Reifenberger 		handled = HANDLED;
27861bc51badSMichael Reifenberger 		break;
278727d26457SAndrew Turner 	case EXIT_REASON_VMCALL:
278827d26457SAndrew Turner 	case EXIT_REASON_VMCLEAR:
278927d26457SAndrew Turner 	case EXIT_REASON_VMLAUNCH:
279027d26457SAndrew Turner 	case EXIT_REASON_VMPTRLD:
279127d26457SAndrew Turner 	case EXIT_REASON_VMPTRST:
279227d26457SAndrew Turner 	case EXIT_REASON_VMREAD:
279327d26457SAndrew Turner 	case EXIT_REASON_VMRESUME:
279427d26457SAndrew Turner 	case EXIT_REASON_VMWRITE:
279527d26457SAndrew Turner 	case EXIT_REASON_VMXOFF:
279627d26457SAndrew Turner 	case EXIT_REASON_VMXON:
27971aa51504SJohn Baldwin 		SDT_PROBE3(vmm, vmx, exit, vminsn, vmx, vcpuid, vmexit);
279827d26457SAndrew Turner 		vmexit->exitcode = VM_EXITCODE_VMINSN;
279927d26457SAndrew Turner 		break;
28004eadbef9SCorvin Köhne 	case EXIT_REASON_INVD:
28013ba952e1SCorvin Köhne 	case EXIT_REASON_WBINVD:
28024eadbef9SCorvin Köhne 		/* ignore exit */
28033ba952e1SCorvin Köhne 		handled = HANDLED;
28043ba952e1SCorvin Köhne 		break;
2805366f6083SPeter Grehan 	default:
28066ac73777STycho Nightingale 		SDT_PROBE4(vmm, vmx, exit, unknown,
28071aa51504SJohn Baldwin 		    vmx, vcpuid, vmexit, reason);
28083dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu->vcpu, VMEXIT_UNKNOWN, 1);
2809366f6083SPeter Grehan 		break;
2810366f6083SPeter Grehan 	}
2811366f6083SPeter Grehan 
2812366f6083SPeter Grehan 	if (handled) {
2813366f6083SPeter Grehan 		/*
2814366f6083SPeter Grehan 		 * It is possible that control is returned to userland
2815366f6083SPeter Grehan 		 * even though we were able to handle the VM exit in the
2816eeefa4e4SNeel Natu 		 * kernel.
2817366f6083SPeter Grehan 		 *
2818366f6083SPeter Grehan 		 * In such a case we want to make sure that the userland
2819366f6083SPeter Grehan 		 * restarts guest execution at the instruction *after*
2820366f6083SPeter Grehan 		 * the one we just processed. Therefore we update the
2821366f6083SPeter Grehan 		 * guest rip in the VMCS and in 'vmexit'.
2822366f6083SPeter Grehan 		 */
2823366f6083SPeter Grehan 		vmexit->rip += vmexit->inst_length;
2824366f6083SPeter Grehan 		vmexit->inst_length = 0;
28253de83862SNeel Natu 		vmcs_write(VMCS_GUEST_RIP, vmexit->rip);
2826366f6083SPeter Grehan 	} else {
2827366f6083SPeter Grehan 		if (vmexit->exitcode == VM_EXITCODE_BOGUS) {
2828366f6083SPeter Grehan 			/*
2829366f6083SPeter Grehan 			 * If this VM exit was not claimed by anybody then
2830366f6083SPeter Grehan 			 * treat it as a generic VMX exit.
2831366f6083SPeter Grehan 			 */
2832366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_VMX;
28330492757cSNeel Natu 			vmexit->u.vmx.status = VM_SUCCESS;
2834c308b23bSNeel Natu 			vmexit->u.vmx.inst_type = 0;
2835c308b23bSNeel Natu 			vmexit->u.vmx.inst_error = 0;
2836366f6083SPeter Grehan 		} else {
2837366f6083SPeter Grehan 			/*
2838366f6083SPeter Grehan 			 * The exitcode and collateral have been populated.
2839366f6083SPeter Grehan 			 * The VM exit will be processed further in userland.
2840366f6083SPeter Grehan 			 */
2841366f6083SPeter Grehan 		}
2842366f6083SPeter Grehan 	}
28436ac73777STycho Nightingale 
28446ac73777STycho Nightingale 	SDT_PROBE4(vmm, vmx, exit, return,
28451aa51504SJohn Baldwin 	    vmx, vcpuid, vmexit, handled);
2846366f6083SPeter Grehan 	return (handled);
2847366f6083SPeter Grehan }
2848366f6083SPeter Grehan 
284940487465SNeel Natu static __inline void
28500492757cSNeel Natu vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit)
28510492757cSNeel Natu {
28520492757cSNeel Natu 
28530492757cSNeel Natu 	KASSERT(vmxctx->inst_fail_status != VM_SUCCESS,
28540492757cSNeel Natu 	    ("vmx_exit_inst_error: invalid inst_fail_status %d",
28550492757cSNeel Natu 	    vmxctx->inst_fail_status));
28560492757cSNeel Natu 
28570492757cSNeel Natu 	vmexit->inst_length = 0;
28580492757cSNeel Natu 	vmexit->exitcode = VM_EXITCODE_VMX;
28590492757cSNeel Natu 	vmexit->u.vmx.status = vmxctx->inst_fail_status;
28600492757cSNeel Natu 	vmexit->u.vmx.inst_error = vmcs_instruction_error();
28610492757cSNeel Natu 	vmexit->u.vmx.exit_reason = ~0;
28620492757cSNeel Natu 	vmexit->u.vmx.exit_qualification = ~0;
28630492757cSNeel Natu 
28640492757cSNeel Natu 	switch (rc) {
28650492757cSNeel Natu 	case VMX_VMRESUME_ERROR:
28660492757cSNeel Natu 	case VMX_VMLAUNCH_ERROR:
28670492757cSNeel Natu 		vmexit->u.vmx.inst_type = rc;
28680492757cSNeel Natu 		break;
28690492757cSNeel Natu 	default:
28700492757cSNeel Natu 		panic("vm_exit_inst_error: vmx_enter_guest returned %d", rc);
28710492757cSNeel Natu 	}
28720492757cSNeel Natu }
28730492757cSNeel Natu 
287462fbd7c2SNeel Natu /*
287562fbd7c2SNeel Natu  * If the NMI-exiting VM execution control is set to '1' then an NMI in
287662fbd7c2SNeel Natu  * non-root operation causes a VM-exit. NMI blocking is in effect so it is
287762fbd7c2SNeel Natu  * sufficient to simply vector to the NMI handler via a software interrupt.
287862fbd7c2SNeel Natu  * However, this must be done before maskable interrupts are enabled
287962fbd7c2SNeel Natu  * otherwise the "iret" issued by an interrupt handler will incorrectly
288062fbd7c2SNeel Natu  * clear NMI blocking.
288162fbd7c2SNeel Natu  */
288262fbd7c2SNeel Natu static __inline void
2883869c8d19SJohn Baldwin vmx_exit_handle_nmi(struct vmx_vcpu *vcpu, struct vm_exit *vmexit)
288462fbd7c2SNeel Natu {
288562fbd7c2SNeel Natu 	uint32_t intr_info;
288662fbd7c2SNeel Natu 
288762fbd7c2SNeel Natu 	KASSERT((read_rflags() & PSL_I) == 0, ("interrupts enabled"));
288862fbd7c2SNeel Natu 
288962fbd7c2SNeel Natu 	if (vmexit->u.vmx.exit_reason != EXIT_REASON_EXCEPTION)
289062fbd7c2SNeel Natu 		return;
289162fbd7c2SNeel Natu 
289262fbd7c2SNeel Natu 	intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
289362fbd7c2SNeel Natu 	KASSERT((intr_info & VMCS_INTR_VALID) != 0,
289462fbd7c2SNeel Natu 	    ("VM exit interruption info invalid: %#x", intr_info));
289562fbd7c2SNeel Natu 
289662fbd7c2SNeel Natu 	if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI) {
289762fbd7c2SNeel Natu 		KASSERT((intr_info & 0xff) == IDT_NMI, ("VM exit due "
289862fbd7c2SNeel Natu 		    "to NMI has invalid vector: %#x", intr_info));
289957e0119eSJohn Baldwin 		VMX_CTR0(vcpu, "Vectoring to NMI handler");
290062fbd7c2SNeel Natu 		__asm __volatile("int $2");
290162fbd7c2SNeel Natu 	}
290262fbd7c2SNeel Natu }
290362fbd7c2SNeel Natu 
290465eefbe4SJohn Baldwin static __inline void
290565eefbe4SJohn Baldwin vmx_dr_enter_guest(struct vmxctx *vmxctx)
290665eefbe4SJohn Baldwin {
290765eefbe4SJohn Baldwin 	register_t rflags;
290865eefbe4SJohn Baldwin 
290965eefbe4SJohn Baldwin 	/* Save host control debug registers. */
291065eefbe4SJohn Baldwin 	vmxctx->host_dr7 = rdr7();
291165eefbe4SJohn Baldwin 	vmxctx->host_debugctl = rdmsr(MSR_DEBUGCTLMSR);
291265eefbe4SJohn Baldwin 
291365eefbe4SJohn Baldwin 	/*
291465eefbe4SJohn Baldwin 	 * Disable debugging in DR7 and DEBUGCTL to avoid triggering
291565eefbe4SJohn Baldwin 	 * exceptions in the host based on the guest DRx values.  The
291665eefbe4SJohn Baldwin 	 * guest DR7 and DEBUGCTL are saved/restored in the VMCS.
291765eefbe4SJohn Baldwin 	 */
291865eefbe4SJohn Baldwin 	load_dr7(0);
291965eefbe4SJohn Baldwin 	wrmsr(MSR_DEBUGCTLMSR, 0);
292065eefbe4SJohn Baldwin 
292165eefbe4SJohn Baldwin 	/*
292265eefbe4SJohn Baldwin 	 * Disable single stepping the kernel to avoid corrupting the
292365eefbe4SJohn Baldwin 	 * guest DR6.  A debugger might still be able to corrupt the
292465eefbe4SJohn Baldwin 	 * guest DR6 by setting a breakpoint after this point and then
292565eefbe4SJohn Baldwin 	 * single stepping.
292665eefbe4SJohn Baldwin 	 */
292765eefbe4SJohn Baldwin 	rflags = read_rflags();
292865eefbe4SJohn Baldwin 	vmxctx->host_tf = rflags & PSL_T;
292965eefbe4SJohn Baldwin 	write_rflags(rflags & ~PSL_T);
293065eefbe4SJohn Baldwin 
293165eefbe4SJohn Baldwin 	/* Save host debug registers. */
293265eefbe4SJohn Baldwin 	vmxctx->host_dr0 = rdr0();
293365eefbe4SJohn Baldwin 	vmxctx->host_dr1 = rdr1();
293465eefbe4SJohn Baldwin 	vmxctx->host_dr2 = rdr2();
293565eefbe4SJohn Baldwin 	vmxctx->host_dr3 = rdr3();
293665eefbe4SJohn Baldwin 	vmxctx->host_dr6 = rdr6();
293765eefbe4SJohn Baldwin 
293865eefbe4SJohn Baldwin 	/* Restore guest debug registers. */
293965eefbe4SJohn Baldwin 	load_dr0(vmxctx->guest_dr0);
294065eefbe4SJohn Baldwin 	load_dr1(vmxctx->guest_dr1);
294165eefbe4SJohn Baldwin 	load_dr2(vmxctx->guest_dr2);
294265eefbe4SJohn Baldwin 	load_dr3(vmxctx->guest_dr3);
294365eefbe4SJohn Baldwin 	load_dr6(vmxctx->guest_dr6);
294465eefbe4SJohn Baldwin }
294565eefbe4SJohn Baldwin 
294665eefbe4SJohn Baldwin static __inline void
294765eefbe4SJohn Baldwin vmx_dr_leave_guest(struct vmxctx *vmxctx)
294865eefbe4SJohn Baldwin {
294965eefbe4SJohn Baldwin 
295065eefbe4SJohn Baldwin 	/* Save guest debug registers. */
295165eefbe4SJohn Baldwin 	vmxctx->guest_dr0 = rdr0();
295265eefbe4SJohn Baldwin 	vmxctx->guest_dr1 = rdr1();
295365eefbe4SJohn Baldwin 	vmxctx->guest_dr2 = rdr2();
295465eefbe4SJohn Baldwin 	vmxctx->guest_dr3 = rdr3();
295565eefbe4SJohn Baldwin 	vmxctx->guest_dr6 = rdr6();
295665eefbe4SJohn Baldwin 
295765eefbe4SJohn Baldwin 	/*
295865eefbe4SJohn Baldwin 	 * Restore host debug registers.  Restore DR7, DEBUGCTL, and
295965eefbe4SJohn Baldwin 	 * PSL_T last.
296065eefbe4SJohn Baldwin 	 */
296165eefbe4SJohn Baldwin 	load_dr0(vmxctx->host_dr0);
296265eefbe4SJohn Baldwin 	load_dr1(vmxctx->host_dr1);
296365eefbe4SJohn Baldwin 	load_dr2(vmxctx->host_dr2);
296465eefbe4SJohn Baldwin 	load_dr3(vmxctx->host_dr3);
296565eefbe4SJohn Baldwin 	load_dr6(vmxctx->host_dr6);
296665eefbe4SJohn Baldwin 	wrmsr(MSR_DEBUGCTLMSR, vmxctx->host_debugctl);
296765eefbe4SJohn Baldwin 	load_dr7(vmxctx->host_dr7);
296865eefbe4SJohn Baldwin 	write_rflags(read_rflags() | vmxctx->host_tf);
296965eefbe4SJohn Baldwin }
297065eefbe4SJohn Baldwin 
29718e2cbc56SMark Johnston static __inline void
29728e2cbc56SMark Johnston vmx_pmap_activate(struct vmx *vmx, pmap_t pmap)
29738e2cbc56SMark Johnston {
29748e2cbc56SMark Johnston 	long eptgen;
29758e2cbc56SMark Johnston 	int cpu;
29768e2cbc56SMark Johnston 
29778e2cbc56SMark Johnston 	cpu = curcpu;
29788e2cbc56SMark Johnston 
29798e2cbc56SMark Johnston 	CPU_SET_ATOMIC(cpu, &pmap->pm_active);
29806f5a9606SMark Johnston 	smr_enter(pmap->pm_eptsmr);
29818e2cbc56SMark Johnston 	eptgen = atomic_load_long(&pmap->pm_eptgen);
29828e2cbc56SMark Johnston 	if (eptgen != vmx->eptgen[cpu]) {
29838e2cbc56SMark Johnston 		vmx->eptgen[cpu] = eptgen;
29848e2cbc56SMark Johnston 		invept(INVEPT_TYPE_SINGLE_CONTEXT,
29858e2cbc56SMark Johnston 		    (struct invept_desc){ .eptp = vmx->eptp, ._res = 0 });
29868e2cbc56SMark Johnston 	}
29878e2cbc56SMark Johnston }
29888e2cbc56SMark Johnston 
29898e2cbc56SMark Johnston static __inline void
29908e2cbc56SMark Johnston vmx_pmap_deactivate(struct vmx *vmx, pmap_t pmap)
29918e2cbc56SMark Johnston {
29926f5a9606SMark Johnston 	smr_exit(pmap->pm_eptsmr);
29938e2cbc56SMark Johnston 	CPU_CLR_ATOMIC(curcpu, &pmap->pm_active);
29948e2cbc56SMark Johnston }
29958e2cbc56SMark Johnston 
29960492757cSNeel Natu static int
2997869c8d19SJohn Baldwin vmx_run(void *vcpui, register_t rip, pmap_t pmap, struct vm_eventinfo *evinfo)
29980492757cSNeel Natu {
299980cb5d84SJohn Baldwin 	int rc, handled, launched;
3000366f6083SPeter Grehan 	struct vmx *vmx;
30011aa51504SJohn Baldwin 	struct vmx_vcpu *vcpu;
3002366f6083SPeter Grehan 	struct vmxctx *vmxctx;
3003366f6083SPeter Grehan 	struct vmcs *vmcs;
300498ed632cSNeel Natu 	struct vm_exit *vmexit;
3005de5ea6b6SNeel Natu 	struct vlapic *vlapic;
300679c59630SNeel Natu 	uint32_t exit_reason;
3007b843f9beSJohn Baldwin 	struct region_descriptor gdtr, idtr;
3008b843f9beSJohn Baldwin 	uint16_t ldt_sel;
3009366f6083SPeter Grehan 
30101aa51504SJohn Baldwin 	vcpu = vcpui;
3011869c8d19SJohn Baldwin 	vmx = vcpu->vmx;
30121aa51504SJohn Baldwin 	vmcs = vcpu->vmcs;
30131aa51504SJohn Baldwin 	vmxctx = &vcpu->ctx;
3014d3956e46SJohn Baldwin 	vlapic = vm_lapic(vcpu->vcpu);
301580cb5d84SJohn Baldwin 	vmexit = vm_exitinfo(vcpu->vcpu);
30160492757cSNeel Natu 	launched = 0;
301798ed632cSNeel Natu 
3018318224bbSNeel Natu 	KASSERT(vmxctx->pmap == pmap,
3019318224bbSNeel Natu 	    ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap));
3020318224bbSNeel Natu 
302180cb5d84SJohn Baldwin 	vmx_msr_guest_enter(vcpu);
3022c3498942SNeel Natu 
3023366f6083SPeter Grehan 	VMPTRLD(vmcs);
3024366f6083SPeter Grehan 
3025366f6083SPeter Grehan 	/*
3026366f6083SPeter Grehan 	 * XXX
3027366f6083SPeter Grehan 	 * We do this every time because we may setup the virtual machine
3028366f6083SPeter Grehan 	 * from a different process than the one that actually runs it.
3029366f6083SPeter Grehan 	 *
3030366f6083SPeter Grehan 	 * If the life of a virtual machine was spent entirely in the context
303115add60dSPeter Grehan 	 * of a single process we could do this once in vmx_init().
3032366f6083SPeter Grehan 	 */
30333de83862SNeel Natu 	vmcs_write(VMCS_HOST_CR3, rcr3());
3034366f6083SPeter Grehan 
30352ce12423SNeel Natu 	vmcs_write(VMCS_GUEST_RIP, rip);
3036953c2c47SNeel Natu 	vmx_set_pcpu_defaults(vmx, vcpu, pmap);
3037366f6083SPeter Grehan 	do {
30382ce12423SNeel Natu 		KASSERT(vmcs_guest_rip() == rip, ("%s: vmcs guest rip mismatch "
30392ce12423SNeel Natu 		    "%#lx/%#lx", __func__, vmcs_guest_rip(), rip));
304040487465SNeel Natu 
30412ce12423SNeel Natu 		handled = UNHANDLED;
30420492757cSNeel Natu 		/*
30430492757cSNeel Natu 		 * Interrupts are disabled from this point on until the
30440492757cSNeel Natu 		 * guest starts executing. This is done for the following
30450492757cSNeel Natu 		 * reasons:
30460492757cSNeel Natu 		 *
30470492757cSNeel Natu 		 * If an AST is asserted on this thread after the check below,
30480492757cSNeel Natu 		 * then the IPI_AST notification will not be lost, because it
30490492757cSNeel Natu 		 * will cause a VM exit due to external interrupt as soon as
30500492757cSNeel Natu 		 * the guest state is loaded.
30510492757cSNeel Natu 		 *
30520492757cSNeel Natu 		 * A posted interrupt after 'vmx_inject_interrupts()' will
30530492757cSNeel Natu 		 * not be "lost" because it will be held pending in the host
30540492757cSNeel Natu 		 * APIC because interrupts are disabled. The pending interrupt
30550492757cSNeel Natu 		 * will be recognized as soon as the guest state is loaded.
30560492757cSNeel Natu 		 *
30570492757cSNeel Natu 		 * The same reasoning applies to the IPI generated by
30580492757cSNeel Natu 		 * pmap_invalidate_ept().
30590492757cSNeel Natu 		 */
30600492757cSNeel Natu 		disable_intr();
306180cb5d84SJohn Baldwin 		vmx_inject_interrupts(vcpu, vlapic, rip);
3062091d4532SNeel Natu 
3063091d4532SNeel Natu 		/*
3064091d4532SNeel Natu 		 * Check for vcpu suspension after injecting events because
3065091d4532SNeel Natu 		 * vmx_inject_interrupts() can suspend the vcpu due to a
3066091d4532SNeel Natu 		 * triple fault.
3067091d4532SNeel Natu 		 */
3068248e6799SNeel Natu 		if (vcpu_suspended(evinfo)) {
30690492757cSNeel Natu 			enable_intr();
307080cb5d84SJohn Baldwin 			vm_exit_suspended(vcpu->vcpu, rip);
30710492757cSNeel Natu 			break;
30720492757cSNeel Natu 		}
30730492757cSNeel Natu 
3074248e6799SNeel Natu 		if (vcpu_rendezvous_pending(evinfo)) {
30755b8a8cd1SNeel Natu 			enable_intr();
307680cb5d84SJohn Baldwin 			vm_exit_rendezvous(vcpu->vcpu, rip);
30775b8a8cd1SNeel Natu 			break;
30785b8a8cd1SNeel Natu 		}
30795b8a8cd1SNeel Natu 
3080248e6799SNeel Natu 		if (vcpu_reqidle(evinfo)) {
3081248e6799SNeel Natu 			enable_intr();
308280cb5d84SJohn Baldwin 			vm_exit_reqidle(vcpu->vcpu, rip);
3083248e6799SNeel Natu 			break;
3084248e6799SNeel Natu 		}
3085248e6799SNeel Natu 
308680cb5d84SJohn Baldwin 		if (vcpu_should_yield(vcpu->vcpu)) {
3087b15a09c0SNeel Natu 			enable_intr();
308880cb5d84SJohn Baldwin 			vm_exit_astpending(vcpu->vcpu, rip);
3089869c8d19SJohn Baldwin 			vmx_astpending_trace(vcpu, rip);
309040487465SNeel Natu 			handled = HANDLED;
3091b15a09c0SNeel Natu 			break;
3092b15a09c0SNeel Natu 		}
3093b15a09c0SNeel Natu 
309480cb5d84SJohn Baldwin 		if (vcpu_debugged(vcpu->vcpu)) {
3095fc276d92SJohn Baldwin 			enable_intr();
309680cb5d84SJohn Baldwin 			vm_exit_debug(vcpu->vcpu, rip);
3097fc276d92SJohn Baldwin 			break;
3098fc276d92SJohn Baldwin 		}
3099fc276d92SJohn Baldwin 
3100b843f9beSJohn Baldwin 		/*
31011bc51badSMichael Reifenberger 		 * If TPR Shadowing is enabled, the TPR Threshold
31021bc51badSMichael Reifenberger 		 * must be updated right before entering the guest.
31031bc51badSMichael Reifenberger 		 */
31041bc51badSMichael Reifenberger 		if (tpr_shadowing && !virtual_interrupt_delivery) {
31051aa51504SJohn Baldwin 			if ((vcpu->cap.proc_ctls & PROCBASED_USE_TPR_SHADOW) != 0) {
31061bc51badSMichael Reifenberger 				vmcs_write(VMCS_TPR_THRESHOLD, vlapic_get_cr8(vlapic));
31071bc51badSMichael Reifenberger 			}
31081bc51badSMichael Reifenberger 		}
31091bc51badSMichael Reifenberger 
31101bc51badSMichael Reifenberger 		/*
3111b843f9beSJohn Baldwin 		 * VM exits restore the base address but not the
3112b843f9beSJohn Baldwin 		 * limits of GDTR and IDTR.  The VMCS only stores the
3113b843f9beSJohn Baldwin 		 * base address, so VM exits set the limits to 0xffff.
3114b843f9beSJohn Baldwin 		 * Save and restore the full GDTR and IDTR to restore
3115b843f9beSJohn Baldwin 		 * the limits.
3116b843f9beSJohn Baldwin 		 *
3117b843f9beSJohn Baldwin 		 * The VMCS does not save the LDTR at all, and VM
3118b843f9beSJohn Baldwin 		 * exits clear LDTR as if a NULL selector were loaded.
3119b843f9beSJohn Baldwin 		 * The userspace hypervisor probably doesn't use a
3120b843f9beSJohn Baldwin 		 * LDT, but save and restore it to be safe.
3121b843f9beSJohn Baldwin 		 */
3122b843f9beSJohn Baldwin 		sgdt(&gdtr);
3123b843f9beSJohn Baldwin 		sidt(&idtr);
3124b843f9beSJohn Baldwin 		ldt_sel = sldt();
3125b843f9beSJohn Baldwin 
3126f5f5f1e7SPeter Grehan 		/*
3127f5f5f1e7SPeter Grehan 		 * The TSC_AUX MSR must be saved/restored while interrupts
3128f5f5f1e7SPeter Grehan 		 * are disabled so that it is not possible for the guest
3129f5f5f1e7SPeter Grehan 		 * TSC_AUX MSR value to be overwritten by the resume
3130f5f5f1e7SPeter Grehan 		 * portion of the IPI_SUSPEND codepath. This is why the
3131f5f5f1e7SPeter Grehan 		 * transition of this MSR is handled separately from those
3132f5f5f1e7SPeter Grehan 		 * handled by vmx_msr_guest_{enter,exit}(), which are ok to
3133f5f5f1e7SPeter Grehan 		 * be transitioned with preemption disabled but interrupts
3134f5f5f1e7SPeter Grehan 		 * enabled.
3135f5f5f1e7SPeter Grehan 		 *
3136f5f5f1e7SPeter Grehan 		 * These vmx_msr_guest_{enter,exit}_tsc_aux() calls can be
3137f5f5f1e7SPeter Grehan 		 * anywhere in this loop so long as they happen with
3138f5f5f1e7SPeter Grehan 		 * interrupts disabled. This location is chosen for
3139f5f5f1e7SPeter Grehan 		 * simplicity.
3140f5f5f1e7SPeter Grehan 		 */
3141f5f5f1e7SPeter Grehan 		vmx_msr_guest_enter_tsc_aux(vmx, vcpu);
3142f5f5f1e7SPeter Grehan 
314365eefbe4SJohn Baldwin 		vmx_dr_enter_guest(vmxctx);
314479c59630SNeel Natu 
31458e2cbc56SMark Johnston 		/*
31468e2cbc56SMark Johnston 		 * Mark the EPT as active on this host CPU and invalidate
31478e2cbc56SMark Johnston 		 * EPTP-tagged TLB entries if required.
31488e2cbc56SMark Johnston 		 */
31498e2cbc56SMark Johnston 		vmx_pmap_activate(vmx, pmap);
31508e2cbc56SMark Johnston 
3151869c8d19SJohn Baldwin 		vmx_run_trace(vcpu);
31528e2cbc56SMark Johnston 		rc = vmx_enter_guest(vmxctx, vmx, launched);
31538e2cbc56SMark Johnston 
31548e2cbc56SMark Johnston 		vmx_pmap_deactivate(vmx, pmap);
31558e2cbc56SMark Johnston 		vmx_dr_leave_guest(vmxctx);
3156f5f5f1e7SPeter Grehan 		vmx_msr_guest_exit_tsc_aux(vmx, vcpu);
3157f5f5f1e7SPeter Grehan 
3158b843f9beSJohn Baldwin 		bare_lgdt(&gdtr);
3159b843f9beSJohn Baldwin 		lidt(&idtr);
3160b843f9beSJohn Baldwin 		lldt(ldt_sel);
3161b843f9beSJohn Baldwin 
316279c59630SNeel Natu 		/* Collect some information for VM exit processing */
316379c59630SNeel Natu 		vmexit->rip = rip = vmcs_guest_rip();
316479c59630SNeel Natu 		vmexit->inst_length = vmexit_instruction_length();
316579c59630SNeel Natu 		vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason();
316679c59630SNeel Natu 		vmexit->u.vmx.exit_qualification = vmcs_exit_qualification();
316779c59630SNeel Natu 
31682ce12423SNeel Natu 		/* Update 'nextrip' */
31691aa51504SJohn Baldwin 		vcpu->state.nextrip = rip;
31702ce12423SNeel Natu 
31710492757cSNeel Natu 		if (rc == VMX_GUEST_VMEXIT) {
3172869c8d19SJohn Baldwin 			vmx_exit_handle_nmi(vcpu, vmexit);
317362fbd7c2SNeel Natu 			enable_intr();
31740492757cSNeel Natu 			handled = vmx_exit_process(vmx, vcpu, vmexit);
31750492757cSNeel Natu 		} else {
317662fbd7c2SNeel Natu 			enable_intr();
317740487465SNeel Natu 			vmx_exit_inst_error(vmxctx, rc, vmexit);
3178eeefa4e4SNeel Natu 		}
317962fbd7c2SNeel Natu 		launched = 1;
3180869c8d19SJohn Baldwin 		vmx_exit_trace(vcpu, rip, exit_reason, handled);
31812ce12423SNeel Natu 		rip = vmexit->rip;
3182eeefa4e4SNeel Natu 	} while (handled);
3183366f6083SPeter Grehan 
3184366f6083SPeter Grehan 	/*
3185366f6083SPeter Grehan 	 * If a VM exit has been handled then the exitcode must be BOGUS
3186366f6083SPeter Grehan 	 * If a VM exit is not handled then the exitcode must not be BOGUS
3187366f6083SPeter Grehan 	 */
3188366f6083SPeter Grehan 	if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) ||
3189366f6083SPeter Grehan 	    (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) {
3190366f6083SPeter Grehan 		panic("Mismatch between handled (%d) and exitcode (%d)",
3191366f6083SPeter Grehan 		      handled, vmexit->exitcode);
3192366f6083SPeter Grehan 	}
3193366f6083SPeter Grehan 
319457e0119eSJohn Baldwin 	VMX_CTR1(vcpu, "returning from vmx_run: exitcode %d",
31950492757cSNeel Natu 	    vmexit->exitcode);
3196366f6083SPeter Grehan 
3197366f6083SPeter Grehan 	VMCLEAR(vmcs);
319880cb5d84SJohn Baldwin 	vmx_msr_guest_exit(vcpu);
3199c3498942SNeel Natu 
3200366f6083SPeter Grehan 	return (0);
3201366f6083SPeter Grehan }
3202366f6083SPeter Grehan 
3203366f6083SPeter Grehan static void
3204869c8d19SJohn Baldwin vmx_vcpu_cleanup(void *vcpui)
3205366f6083SPeter Grehan {
32061aa51504SJohn Baldwin 	struct vmx_vcpu *vcpu = vcpui;
3207366f6083SPeter Grehan 
32080f00260cSJohn Baldwin 	vpid_free(vcpu->state.vpid);
32090f00260cSJohn Baldwin 	free(vcpu->pir_desc, M_VMX);
32100f00260cSJohn Baldwin 	free(vcpu->apic_page, M_VMX);
32110f00260cSJohn Baldwin 	free(vcpu->vmcs, M_VMX);
32121aa51504SJohn Baldwin 	free(vcpu, M_VMX);
32130f00260cSJohn Baldwin }
321445e51299SNeel Natu 
32151aa51504SJohn Baldwin static void
3216869c8d19SJohn Baldwin vmx_cleanup(void *vmi)
32171aa51504SJohn Baldwin {
3218869c8d19SJohn Baldwin 	struct vmx *vmx = vmi;
32191aa51504SJohn Baldwin 
32201aa51504SJohn Baldwin 	if (virtual_interrupt_delivery)
32211aa51504SJohn Baldwin 		vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
32221aa51504SJohn Baldwin 
32230f00260cSJohn Baldwin 	free(vmx->msr_bitmap, M_VMX);
3224366f6083SPeter Grehan 	free(vmx, M_VMX);
3225366f6083SPeter Grehan 
3226366f6083SPeter Grehan 	return;
3227366f6083SPeter Grehan }
3228366f6083SPeter Grehan 
3229366f6083SPeter Grehan static register_t *
3230366f6083SPeter Grehan vmxctx_regptr(struct vmxctx *vmxctx, int reg)
3231366f6083SPeter Grehan {
3232366f6083SPeter Grehan 
3233366f6083SPeter Grehan 	switch (reg) {
3234366f6083SPeter Grehan 	case VM_REG_GUEST_RAX:
3235366f6083SPeter Grehan 		return (&vmxctx->guest_rax);
3236366f6083SPeter Grehan 	case VM_REG_GUEST_RBX:
3237366f6083SPeter Grehan 		return (&vmxctx->guest_rbx);
3238366f6083SPeter Grehan 	case VM_REG_GUEST_RCX:
3239366f6083SPeter Grehan 		return (&vmxctx->guest_rcx);
3240366f6083SPeter Grehan 	case VM_REG_GUEST_RDX:
3241366f6083SPeter Grehan 		return (&vmxctx->guest_rdx);
3242366f6083SPeter Grehan 	case VM_REG_GUEST_RSI:
3243366f6083SPeter Grehan 		return (&vmxctx->guest_rsi);
3244366f6083SPeter Grehan 	case VM_REG_GUEST_RDI:
3245366f6083SPeter Grehan 		return (&vmxctx->guest_rdi);
3246366f6083SPeter Grehan 	case VM_REG_GUEST_RBP:
3247366f6083SPeter Grehan 		return (&vmxctx->guest_rbp);
3248366f6083SPeter Grehan 	case VM_REG_GUEST_R8:
3249366f6083SPeter Grehan 		return (&vmxctx->guest_r8);
3250366f6083SPeter Grehan 	case VM_REG_GUEST_R9:
3251366f6083SPeter Grehan 		return (&vmxctx->guest_r9);
3252366f6083SPeter Grehan 	case VM_REG_GUEST_R10:
3253366f6083SPeter Grehan 		return (&vmxctx->guest_r10);
3254366f6083SPeter Grehan 	case VM_REG_GUEST_R11:
3255366f6083SPeter Grehan 		return (&vmxctx->guest_r11);
3256366f6083SPeter Grehan 	case VM_REG_GUEST_R12:
3257366f6083SPeter Grehan 		return (&vmxctx->guest_r12);
3258366f6083SPeter Grehan 	case VM_REG_GUEST_R13:
3259366f6083SPeter Grehan 		return (&vmxctx->guest_r13);
3260366f6083SPeter Grehan 	case VM_REG_GUEST_R14:
3261366f6083SPeter Grehan 		return (&vmxctx->guest_r14);
3262366f6083SPeter Grehan 	case VM_REG_GUEST_R15:
3263366f6083SPeter Grehan 		return (&vmxctx->guest_r15);
326437a723a5SNeel Natu 	case VM_REG_GUEST_CR2:
326537a723a5SNeel Natu 		return (&vmxctx->guest_cr2);
326665eefbe4SJohn Baldwin 	case VM_REG_GUEST_DR0:
326765eefbe4SJohn Baldwin 		return (&vmxctx->guest_dr0);
326865eefbe4SJohn Baldwin 	case VM_REG_GUEST_DR1:
326965eefbe4SJohn Baldwin 		return (&vmxctx->guest_dr1);
327065eefbe4SJohn Baldwin 	case VM_REG_GUEST_DR2:
327165eefbe4SJohn Baldwin 		return (&vmxctx->guest_dr2);
327265eefbe4SJohn Baldwin 	case VM_REG_GUEST_DR3:
327365eefbe4SJohn Baldwin 		return (&vmxctx->guest_dr3);
327465eefbe4SJohn Baldwin 	case VM_REG_GUEST_DR6:
327565eefbe4SJohn Baldwin 		return (&vmxctx->guest_dr6);
3276366f6083SPeter Grehan 	default:
3277366f6083SPeter Grehan 		break;
3278366f6083SPeter Grehan 	}
3279366f6083SPeter Grehan 	return (NULL);
3280366f6083SPeter Grehan }
3281366f6083SPeter Grehan 
3282366f6083SPeter Grehan static int
3283366f6083SPeter Grehan vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval)
3284366f6083SPeter Grehan {
3285366f6083SPeter Grehan 	register_t *regp;
3286366f6083SPeter Grehan 
3287366f6083SPeter Grehan 	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
3288366f6083SPeter Grehan 		*retval = *regp;
3289366f6083SPeter Grehan 		return (0);
3290366f6083SPeter Grehan 	} else
3291366f6083SPeter Grehan 		return (EINVAL);
3292366f6083SPeter Grehan }
3293366f6083SPeter Grehan 
3294366f6083SPeter Grehan static int
3295366f6083SPeter Grehan vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val)
3296366f6083SPeter Grehan {
3297366f6083SPeter Grehan 	register_t *regp;
3298366f6083SPeter Grehan 
3299366f6083SPeter Grehan 	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
3300366f6083SPeter Grehan 		*regp = val;
3301366f6083SPeter Grehan 		return (0);
3302366f6083SPeter Grehan 	} else
3303366f6083SPeter Grehan 		return (EINVAL);
3304366f6083SPeter Grehan }
3305366f6083SPeter Grehan 
3306366f6083SPeter Grehan static int
33071aa51504SJohn Baldwin vmx_get_intr_shadow(struct vmx_vcpu *vcpu, int running, uint64_t *retval)
3308d1819632SNeel Natu {
3309d1819632SNeel Natu 	uint64_t gi;
3310d1819632SNeel Natu 	int error;
3311d1819632SNeel Natu 
33121aa51504SJohn Baldwin 	error = vmcs_getreg(vcpu->vmcs, running,
3313d1819632SNeel Natu 	    VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY), &gi);
3314d1819632SNeel Natu 	*retval = (gi & HWINTR_BLOCKING) ? 1 : 0;
3315d1819632SNeel Natu 	return (error);
3316d1819632SNeel Natu }
3317d1819632SNeel Natu 
3318d1819632SNeel Natu static int
3319869c8d19SJohn Baldwin vmx_modify_intr_shadow(struct vmx_vcpu *vcpu, int running, uint64_t val)
3320d1819632SNeel Natu {
3321d1819632SNeel Natu 	struct vmcs *vmcs;
3322d1819632SNeel Natu 	uint64_t gi;
3323d1819632SNeel Natu 	int error, ident;
3324d1819632SNeel Natu 
3325d1819632SNeel Natu 	/*
3326d1819632SNeel Natu 	 * Forcing the vcpu into an interrupt shadow is not supported.
3327d1819632SNeel Natu 	 */
3328d1819632SNeel Natu 	if (val) {
3329d1819632SNeel Natu 		error = EINVAL;
3330d1819632SNeel Natu 		goto done;
3331d1819632SNeel Natu 	}
3332d1819632SNeel Natu 
33331aa51504SJohn Baldwin 	vmcs = vcpu->vmcs;
3334d1819632SNeel Natu 	ident = VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY);
3335d1819632SNeel Natu 	error = vmcs_getreg(vmcs, running, ident, &gi);
3336d1819632SNeel Natu 	if (error == 0) {
3337d1819632SNeel Natu 		gi &= ~HWINTR_BLOCKING;
3338d1819632SNeel Natu 		error = vmcs_setreg(vmcs, running, ident, gi);
3339d1819632SNeel Natu 	}
3340d1819632SNeel Natu done:
334157e0119eSJohn Baldwin 	VMX_CTR2(vcpu, "Setting intr_shadow to %#lx %s", val,
334257e0119eSJohn Baldwin 	    error ? "failed" : "succeeded");
3343d1819632SNeel Natu 	return (error);
3344d1819632SNeel Natu }
3345d1819632SNeel Natu 
3346d1819632SNeel Natu static int
3347aaaa0656SPeter Grehan vmx_shadow_reg(int reg)
3348aaaa0656SPeter Grehan {
3349aaaa0656SPeter Grehan 	int shreg;
3350aaaa0656SPeter Grehan 
3351aaaa0656SPeter Grehan 	shreg = -1;
3352aaaa0656SPeter Grehan 
3353aaaa0656SPeter Grehan 	switch (reg) {
3354aaaa0656SPeter Grehan 	case VM_REG_GUEST_CR0:
3355aaaa0656SPeter Grehan 		shreg = VMCS_CR0_SHADOW;
3356aaaa0656SPeter Grehan 		break;
3357aaaa0656SPeter Grehan 	case VM_REG_GUEST_CR4:
3358aaaa0656SPeter Grehan 		shreg = VMCS_CR4_SHADOW;
3359aaaa0656SPeter Grehan 		break;
3360aaaa0656SPeter Grehan 	default:
3361aaaa0656SPeter Grehan 		break;
3362aaaa0656SPeter Grehan 	}
3363aaaa0656SPeter Grehan 
3364aaaa0656SPeter Grehan 	return (shreg);
3365aaaa0656SPeter Grehan }
3366aaaa0656SPeter Grehan 
3367aaaa0656SPeter Grehan static int
3368869c8d19SJohn Baldwin vmx_getreg(void *vcpui, int reg, uint64_t *retval)
3369366f6083SPeter Grehan {
3370d3c11f40SPeter Grehan 	int running, hostcpu;
33711aa51504SJohn Baldwin 	struct vmx_vcpu *vcpu = vcpui;
3372869c8d19SJohn Baldwin 	struct vmx *vmx = vcpu->vmx;
3373366f6083SPeter Grehan 
337480cb5d84SJohn Baldwin 	running = vcpu_is_running(vcpu->vcpu, &hostcpu);
3375d3c11f40SPeter Grehan 	if (running && hostcpu != curcpu)
33761aa51504SJohn Baldwin 		panic("vmx_getreg: %s%d is running", vm_name(vmx->vm),
33771aa51504SJohn Baldwin 		    vcpu->vcpuid);
3378d3c11f40SPeter Grehan 
3379d1819632SNeel Natu 	if (reg == VM_REG_GUEST_INTR_SHADOW)
33801aa51504SJohn Baldwin 		return (vmx_get_intr_shadow(vcpu, running, retval));
3381d1819632SNeel Natu 
33821aa51504SJohn Baldwin 	if (vmxctx_getreg(&vcpu->ctx, reg, retval) == 0)
3383366f6083SPeter Grehan 		return (0);
3384366f6083SPeter Grehan 
33851aa51504SJohn Baldwin 	return (vmcs_getreg(vcpu->vmcs, running, reg, retval));
3386366f6083SPeter Grehan }
3387366f6083SPeter Grehan 
3388366f6083SPeter Grehan static int
3389869c8d19SJohn Baldwin vmx_setreg(void *vcpui, int reg, uint64_t val)
3390366f6083SPeter Grehan {
3391aaaa0656SPeter Grehan 	int error, hostcpu, running, shadow;
3392366f6083SPeter Grehan 	uint64_t ctls;
33933527963bSNeel Natu 	pmap_t pmap;
33941aa51504SJohn Baldwin 	struct vmx_vcpu *vcpu = vcpui;
3395869c8d19SJohn Baldwin 	struct vmx *vmx = vcpu->vmx;
3396366f6083SPeter Grehan 
339780cb5d84SJohn Baldwin 	running = vcpu_is_running(vcpu->vcpu, &hostcpu);
3398d3c11f40SPeter Grehan 	if (running && hostcpu != curcpu)
33991aa51504SJohn Baldwin 		panic("vmx_setreg: %s%d is running", vm_name(vmx->vm),
34001aa51504SJohn Baldwin 		    vcpu->vcpuid);
3401d3c11f40SPeter Grehan 
3402d1819632SNeel Natu 	if (reg == VM_REG_GUEST_INTR_SHADOW)
3403869c8d19SJohn Baldwin 		return (vmx_modify_intr_shadow(vcpu, running, val));
3404d1819632SNeel Natu 
34051aa51504SJohn Baldwin 	if (vmxctx_setreg(&vcpu->ctx, reg, val) == 0)
3406366f6083SPeter Grehan 		return (0);
3407366f6083SPeter Grehan 
340809860d44SEd Maste 	/* Do not permit user write access to VMCS fields by offset. */
340909860d44SEd Maste 	if (reg < 0)
341009860d44SEd Maste 		return (EINVAL);
341109860d44SEd Maste 
34121aa51504SJohn Baldwin 	error = vmcs_setreg(vcpu->vmcs, running, reg, val);
3413366f6083SPeter Grehan 
3414366f6083SPeter Grehan 	if (error == 0) {
3415366f6083SPeter Grehan 		/*
3416366f6083SPeter Grehan 		 * If the "load EFER" VM-entry control is 1 then the
3417366f6083SPeter Grehan 		 * value of EFER.LMA must be identical to "IA-32e mode guest"
3418366f6083SPeter Grehan 		 * bit in the VM-entry control.
3419366f6083SPeter Grehan 		 */
3420366f6083SPeter Grehan 		if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 &&
3421366f6083SPeter Grehan 		    (reg == VM_REG_GUEST_EFER)) {
34221aa51504SJohn Baldwin 			vmcs_getreg(vcpu->vmcs, running,
3423366f6083SPeter Grehan 				    VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls);
3424366f6083SPeter Grehan 			if (val & EFER_LMA)
3425366f6083SPeter Grehan 				ctls |= VM_ENTRY_GUEST_LMA;
3426366f6083SPeter Grehan 			else
3427366f6083SPeter Grehan 				ctls &= ~VM_ENTRY_GUEST_LMA;
34281aa51504SJohn Baldwin 			vmcs_setreg(vcpu->vmcs, running,
3429366f6083SPeter Grehan 				    VMCS_IDENT(VMCS_ENTRY_CTLS), ctls);
3430366f6083SPeter Grehan 		}
3431aaaa0656SPeter Grehan 
3432aaaa0656SPeter Grehan 		shadow = vmx_shadow_reg(reg);
3433aaaa0656SPeter Grehan 		if (shadow > 0) {
3434aaaa0656SPeter Grehan 			/*
3435aaaa0656SPeter Grehan 			 * Store the unmodified value in the shadow
3436aaaa0656SPeter Grehan 			 */
34371aa51504SJohn Baldwin 			error = vmcs_setreg(vcpu->vmcs, running,
3438aaaa0656SPeter Grehan 				    VMCS_IDENT(shadow), val);
3439aaaa0656SPeter Grehan 		}
34403527963bSNeel Natu 
34413527963bSNeel Natu 		if (reg == VM_REG_GUEST_CR3) {
34423527963bSNeel Natu 			/*
34433527963bSNeel Natu 			 * Invalidate the guest vcpu's TLB mappings to emulate
34443527963bSNeel Natu 			 * the behavior of updating %cr3.
34453527963bSNeel Natu 			 *
34463527963bSNeel Natu 			 * XXX the processor retains global mappings when %cr3
34473527963bSNeel Natu 			 * is updated but vmx_invvpid() does not.
34483527963bSNeel Natu 			 */
34491aa51504SJohn Baldwin 			pmap = vcpu->ctx.pmap;
34503527963bSNeel Natu 			vmx_invvpid(vmx, vcpu, pmap, running);
34513527963bSNeel Natu 		}
3452366f6083SPeter Grehan 	}
3453366f6083SPeter Grehan 
3454366f6083SPeter Grehan 	return (error);
3455366f6083SPeter Grehan }
3456366f6083SPeter Grehan 
3457366f6083SPeter Grehan static int
3458869c8d19SJohn Baldwin vmx_getdesc(void *vcpui, int reg, struct seg_desc *desc)
3459366f6083SPeter Grehan {
3460ba6f5e23SNeel Natu 	int hostcpu, running;
34611aa51504SJohn Baldwin 	struct vmx_vcpu *vcpu = vcpui;
3462869c8d19SJohn Baldwin 	struct vmx *vmx = vcpu->vmx;
3463366f6083SPeter Grehan 
346480cb5d84SJohn Baldwin 	running = vcpu_is_running(vcpu->vcpu, &hostcpu);
3465ba6f5e23SNeel Natu 	if (running && hostcpu != curcpu)
34661aa51504SJohn Baldwin 		panic("vmx_getdesc: %s%d is running", vm_name(vmx->vm),
34671aa51504SJohn Baldwin 		    vcpu->vcpuid);
3468ba6f5e23SNeel Natu 
34691aa51504SJohn Baldwin 	return (vmcs_getdesc(vcpu->vmcs, running, reg, desc));
3470366f6083SPeter Grehan }
3471366f6083SPeter Grehan 
3472366f6083SPeter Grehan static int
3473869c8d19SJohn Baldwin vmx_setdesc(void *vcpui, int reg, struct seg_desc *desc)
3474366f6083SPeter Grehan {
3475ba6f5e23SNeel Natu 	int hostcpu, running;
34761aa51504SJohn Baldwin 	struct vmx_vcpu *vcpu = vcpui;
3477869c8d19SJohn Baldwin 	struct vmx *vmx = vcpu->vmx;
3478366f6083SPeter Grehan 
347980cb5d84SJohn Baldwin 	running = vcpu_is_running(vcpu->vcpu, &hostcpu);
3480ba6f5e23SNeel Natu 	if (running && hostcpu != curcpu)
34811aa51504SJohn Baldwin 		panic("vmx_setdesc: %s%d is running", vm_name(vmx->vm),
34821aa51504SJohn Baldwin 		    vcpu->vcpuid);
3483ba6f5e23SNeel Natu 
34841aa51504SJohn Baldwin 	return (vmcs_setdesc(vcpu->vmcs, running, reg, desc));
3485366f6083SPeter Grehan }
3486366f6083SPeter Grehan 
3487366f6083SPeter Grehan static int
3488869c8d19SJohn Baldwin vmx_getcap(void *vcpui, int type, int *retval)
3489366f6083SPeter Grehan {
34901aa51504SJohn Baldwin 	struct vmx_vcpu *vcpu = vcpui;
3491366f6083SPeter Grehan 	int vcap;
3492366f6083SPeter Grehan 	int ret;
3493366f6083SPeter Grehan 
3494366f6083SPeter Grehan 	ret = ENOENT;
3495366f6083SPeter Grehan 
34961aa51504SJohn Baldwin 	vcap = vcpu->cap.set;
3497366f6083SPeter Grehan 
3498366f6083SPeter Grehan 	switch (type) {
3499366f6083SPeter Grehan 	case VM_CAP_HALT_EXIT:
3500366f6083SPeter Grehan 		if (cap_halt_exit)
3501366f6083SPeter Grehan 			ret = 0;
3502366f6083SPeter Grehan 		break;
3503366f6083SPeter Grehan 	case VM_CAP_PAUSE_EXIT:
3504366f6083SPeter Grehan 		if (cap_pause_exit)
3505366f6083SPeter Grehan 			ret = 0;
3506366f6083SPeter Grehan 		break;
3507366f6083SPeter Grehan 	case VM_CAP_MTRAP_EXIT:
3508366f6083SPeter Grehan 		if (cap_monitor_trap)
3509366f6083SPeter Grehan 			ret = 0;
3510366f6083SPeter Grehan 		break;
3511f5f5f1e7SPeter Grehan 	case VM_CAP_RDPID:
3512f5f5f1e7SPeter Grehan 		if (cap_rdpid)
3513f5f5f1e7SPeter Grehan 			ret = 0;
3514f5f5f1e7SPeter Grehan 		break;
3515f5f5f1e7SPeter Grehan 	case VM_CAP_RDTSCP:
3516f5f5f1e7SPeter Grehan 		if (cap_rdtscp)
3517f5f5f1e7SPeter Grehan 			ret = 0;
3518f5f5f1e7SPeter Grehan 		break;
3519366f6083SPeter Grehan 	case VM_CAP_UNRESTRICTED_GUEST:
3520366f6083SPeter Grehan 		if (cap_unrestricted_guest)
3521366f6083SPeter Grehan 			ret = 0;
3522366f6083SPeter Grehan 		break;
352349cc03daSNeel Natu 	case VM_CAP_ENABLE_INVPCID:
352449cc03daSNeel Natu 		if (cap_invpcid)
352549cc03daSNeel Natu 			ret = 0;
352649cc03daSNeel Natu 		break;
3527cbd03a9dSJohn Baldwin 	case VM_CAP_BPT_EXIT:
35280bda8d3eSCorvin Köhne 	case VM_CAP_IPI_EXIT:
3529cbd03a9dSJohn Baldwin 		ret = 0;
3530cbd03a9dSJohn Baldwin 		break;
3531366f6083SPeter Grehan 	default:
3532366f6083SPeter Grehan 		break;
3533366f6083SPeter Grehan 	}
3534366f6083SPeter Grehan 
3535366f6083SPeter Grehan 	if (ret == 0)
3536366f6083SPeter Grehan 		*retval = (vcap & (1 << type)) ? 1 : 0;
3537366f6083SPeter Grehan 
3538366f6083SPeter Grehan 	return (ret);
3539366f6083SPeter Grehan }
3540366f6083SPeter Grehan 
3541366f6083SPeter Grehan static int
3542869c8d19SJohn Baldwin vmx_setcap(void *vcpui, int type, int val)
3543366f6083SPeter Grehan {
35441aa51504SJohn Baldwin 	struct vmx_vcpu *vcpu = vcpui;
35451aa51504SJohn Baldwin 	struct vmcs *vmcs = vcpu->vmcs;
35460bda8d3eSCorvin Köhne 	struct vlapic *vlapic;
3547366f6083SPeter Grehan 	uint32_t baseval;
3548366f6083SPeter Grehan 	uint32_t *pptr;
3549366f6083SPeter Grehan 	int error;
3550366f6083SPeter Grehan 	int flag;
3551366f6083SPeter Grehan 	int reg;
3552366f6083SPeter Grehan 	int retval;
3553366f6083SPeter Grehan 
3554366f6083SPeter Grehan 	retval = ENOENT;
3555366f6083SPeter Grehan 	pptr = NULL;
3556366f6083SPeter Grehan 
3557366f6083SPeter Grehan 	switch (type) {
3558366f6083SPeter Grehan 	case VM_CAP_HALT_EXIT:
3559366f6083SPeter Grehan 		if (cap_halt_exit) {
3560366f6083SPeter Grehan 			retval = 0;
35611aa51504SJohn Baldwin 			pptr = &vcpu->cap.proc_ctls;
3562366f6083SPeter Grehan 			baseval = *pptr;
3563366f6083SPeter Grehan 			flag = PROCBASED_HLT_EXITING;
3564366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
3565366f6083SPeter Grehan 		}
3566366f6083SPeter Grehan 		break;
3567366f6083SPeter Grehan 	case VM_CAP_MTRAP_EXIT:
3568366f6083SPeter Grehan 		if (cap_monitor_trap) {
3569366f6083SPeter Grehan 			retval = 0;
35701aa51504SJohn Baldwin 			pptr = &vcpu->cap.proc_ctls;
3571366f6083SPeter Grehan 			baseval = *pptr;
3572366f6083SPeter Grehan 			flag = PROCBASED_MTF;
3573366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
3574366f6083SPeter Grehan 		}
3575366f6083SPeter Grehan 		break;
3576366f6083SPeter Grehan 	case VM_CAP_PAUSE_EXIT:
3577366f6083SPeter Grehan 		if (cap_pause_exit) {
3578366f6083SPeter Grehan 			retval = 0;
35791aa51504SJohn Baldwin 			pptr = &vcpu->cap.proc_ctls;
3580366f6083SPeter Grehan 			baseval = *pptr;
3581366f6083SPeter Grehan 			flag = PROCBASED_PAUSE_EXITING;
3582366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
3583366f6083SPeter Grehan 		}
3584366f6083SPeter Grehan 		break;
3585f5f5f1e7SPeter Grehan 	case VM_CAP_RDPID:
3586f5f5f1e7SPeter Grehan 	case VM_CAP_RDTSCP:
3587f5f5f1e7SPeter Grehan 		if (cap_rdpid || cap_rdtscp)
3588f5f5f1e7SPeter Grehan 			/*
3589f5f5f1e7SPeter Grehan 			 * Choose not to support enabling/disabling
3590f5f5f1e7SPeter Grehan 			 * RDPID/RDTSCP via libvmmapi since, as per the
359115add60dSPeter Grehan 			 * discussion in vmx_modinit(), RDPID/RDTSCP are
3592f5f5f1e7SPeter Grehan 			 * either always enabled or always disabled.
3593f5f5f1e7SPeter Grehan 			 */
3594f5f5f1e7SPeter Grehan 			error = EOPNOTSUPP;
3595f5f5f1e7SPeter Grehan 		break;
3596366f6083SPeter Grehan 	case VM_CAP_UNRESTRICTED_GUEST:
3597366f6083SPeter Grehan 		if (cap_unrestricted_guest) {
3598366f6083SPeter Grehan 			retval = 0;
35991aa51504SJohn Baldwin 			pptr = &vcpu->cap.proc_ctls2;
360049cc03daSNeel Natu 			baseval = *pptr;
3601366f6083SPeter Grehan 			flag = PROCBASED2_UNRESTRICTED_GUEST;
3602366f6083SPeter Grehan 			reg = VMCS_SEC_PROC_BASED_CTLS;
3603366f6083SPeter Grehan 		}
3604366f6083SPeter Grehan 		break;
360549cc03daSNeel Natu 	case VM_CAP_ENABLE_INVPCID:
360649cc03daSNeel Natu 		if (cap_invpcid) {
360749cc03daSNeel Natu 			retval = 0;
36081aa51504SJohn Baldwin 			pptr = &vcpu->cap.proc_ctls2;
360949cc03daSNeel Natu 			baseval = *pptr;
361049cc03daSNeel Natu 			flag = PROCBASED2_ENABLE_INVPCID;
361149cc03daSNeel Natu 			reg = VMCS_SEC_PROC_BASED_CTLS;
361249cc03daSNeel Natu 		}
361349cc03daSNeel Natu 		break;
3614cbd03a9dSJohn Baldwin 	case VM_CAP_BPT_EXIT:
3615cbd03a9dSJohn Baldwin 		retval = 0;
3616cbd03a9dSJohn Baldwin 
3617cbd03a9dSJohn Baldwin 		/* Don't change the bitmap if we are tracing all exceptions. */
36181aa51504SJohn Baldwin 		if (vcpu->cap.exc_bitmap != 0xffffffff) {
36191aa51504SJohn Baldwin 			pptr = &vcpu->cap.exc_bitmap;
3620cbd03a9dSJohn Baldwin 			baseval = *pptr;
3621cbd03a9dSJohn Baldwin 			flag = (1 << IDT_BP);
3622cbd03a9dSJohn Baldwin 			reg = VMCS_EXCEPTION_BITMAP;
3623cbd03a9dSJohn Baldwin 		}
3624cbd03a9dSJohn Baldwin 		break;
36250bda8d3eSCorvin Köhne 	case VM_CAP_IPI_EXIT:
36260bda8d3eSCorvin Köhne 		retval = 0;
36270bda8d3eSCorvin Köhne 
3628d3956e46SJohn Baldwin 		vlapic = vm_lapic(vcpu->vcpu);
36290bda8d3eSCorvin Köhne 		vlapic->ipi_exit = val;
36300bda8d3eSCorvin Köhne 		break;
3631366f6083SPeter Grehan 	default:
3632366f6083SPeter Grehan 		break;
3633366f6083SPeter Grehan 	}
3634366f6083SPeter Grehan 
3635cbd03a9dSJohn Baldwin 	if (retval)
3636cbd03a9dSJohn Baldwin 		return (retval);
3637cbd03a9dSJohn Baldwin 
3638cbd03a9dSJohn Baldwin 	if (pptr != NULL) {
3639366f6083SPeter Grehan 		if (val) {
3640366f6083SPeter Grehan 			baseval |= flag;
3641366f6083SPeter Grehan 		} else {
3642366f6083SPeter Grehan 			baseval &= ~flag;
3643366f6083SPeter Grehan 		}
3644366f6083SPeter Grehan 		VMPTRLD(vmcs);
3645366f6083SPeter Grehan 		error = vmwrite(reg, baseval);
3646366f6083SPeter Grehan 		VMCLEAR(vmcs);
3647366f6083SPeter Grehan 
3648cbd03a9dSJohn Baldwin 		if (error)
3649cbd03a9dSJohn Baldwin 			return (error);
3650cbd03a9dSJohn Baldwin 
3651366f6083SPeter Grehan 		/*
3652366f6083SPeter Grehan 		 * Update optional stored flags, and record
3653366f6083SPeter Grehan 		 * setting
3654366f6083SPeter Grehan 		 */
3655366f6083SPeter Grehan 		*pptr = baseval;
3656366f6083SPeter Grehan 	}
3657366f6083SPeter Grehan 
3658366f6083SPeter Grehan 	if (val) {
36591aa51504SJohn Baldwin 		vcpu->cap.set |= (1 << type);
3660366f6083SPeter Grehan 	} else {
36611aa51504SJohn Baldwin 		vcpu->cap.set &= ~(1 << type);
3662366f6083SPeter Grehan 	}
3663366f6083SPeter Grehan 
3664cbd03a9dSJohn Baldwin 	return (0);
3665366f6083SPeter Grehan }
3666366f6083SPeter Grehan 
366715add60dSPeter Grehan static struct vmspace *
366815add60dSPeter Grehan vmx_vmspace_alloc(vm_offset_t min, vm_offset_t max)
366915add60dSPeter Grehan {
367015add60dSPeter Grehan 	return (ept_vmspace_alloc(min, max));
367115add60dSPeter Grehan }
367215add60dSPeter Grehan 
367315add60dSPeter Grehan static void
367415add60dSPeter Grehan vmx_vmspace_free(struct vmspace *vmspace)
367515add60dSPeter Grehan {
367615add60dSPeter Grehan 	ept_vmspace_free(vmspace);
367715add60dSPeter Grehan }
367815add60dSPeter Grehan 
367988c4b8d1SNeel Natu struct vlapic_vtx {
368088c4b8d1SNeel Natu 	struct vlapic	vlapic;
3681176666c2SNeel Natu 	struct pir_desc	*pir_desc;
36821aa51504SJohn Baldwin 	struct vmx_vcpu	*vcpu;
36832c352febSJohn Baldwin 	u_int	pending_prio;
368488c4b8d1SNeel Natu };
368588c4b8d1SNeel Natu 
36862c352febSJohn Baldwin #define VPR_PRIO_BIT(vpr)	(1 << ((vpr) >> 4))
36872c352febSJohn Baldwin 
3688d030f941SJohn Baldwin #define	VMX_CTR_PIR(vlapic, pir_desc, notify, vector, level, msg)	\
368988c4b8d1SNeel Natu do {									\
3690d030f941SJohn Baldwin 	VLAPIC_CTR2(vlapic, msg " assert %s-triggered vector %d",	\
369188c4b8d1SNeel Natu 	    level ? "level" : "edge", vector);				\
3692d030f941SJohn Baldwin 	VLAPIC_CTR1(vlapic, msg " pir0 0x%016lx", pir_desc->pir[0]);	\
3693d030f941SJohn Baldwin 	VLAPIC_CTR1(vlapic, msg " pir1 0x%016lx", pir_desc->pir[1]);	\
3694d030f941SJohn Baldwin 	VLAPIC_CTR1(vlapic, msg " pir2 0x%016lx", pir_desc->pir[2]);	\
3695d030f941SJohn Baldwin 	VLAPIC_CTR1(vlapic, msg " pir3 0x%016lx", pir_desc->pir[3]);	\
3696d030f941SJohn Baldwin 	VLAPIC_CTR1(vlapic, msg " notify: %s", notify ? "yes" : "no");	\
369788c4b8d1SNeel Natu } while (0)
369888c4b8d1SNeel Natu 
369988c4b8d1SNeel Natu /*
370088c4b8d1SNeel Natu  * vlapic->ops handlers that utilize the APICv hardware assist described in
370188c4b8d1SNeel Natu  * Chapter 29 of the Intel SDM.
370288c4b8d1SNeel Natu  */
370388c4b8d1SNeel Natu static int
370488c4b8d1SNeel Natu vmx_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
370588c4b8d1SNeel Natu {
370688c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
370788c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
370888c4b8d1SNeel Natu 	uint64_t mask;
37092c352febSJohn Baldwin 	int idx, notify = 0;
371088c4b8d1SNeel Natu 
371188c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
3712176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
371388c4b8d1SNeel Natu 
371488c4b8d1SNeel Natu 	/*
371588c4b8d1SNeel Natu 	 * Keep track of interrupt requests in the PIR descriptor. This is
371688c4b8d1SNeel Natu 	 * because the virtual APIC page pointed to by the VMCS cannot be
371788c4b8d1SNeel Natu 	 * modified if the vcpu is running.
371888c4b8d1SNeel Natu 	 */
371988c4b8d1SNeel Natu 	idx = vector / 64;
372088c4b8d1SNeel Natu 	mask = 1UL << (vector % 64);
372188c4b8d1SNeel Natu 	atomic_set_long(&pir_desc->pir[idx], mask);
37222c352febSJohn Baldwin 
37232c352febSJohn Baldwin 	/*
37242c352febSJohn Baldwin 	 * A notification is required whenever the 'pending' bit makes a
37252c352febSJohn Baldwin 	 * transition from 0->1.
37262c352febSJohn Baldwin 	 *
37272c352febSJohn Baldwin 	 * Even if the 'pending' bit is already asserted, notification about
37282c352febSJohn Baldwin 	 * the incoming interrupt may still be necessary.  For example, if a
37292c352febSJohn Baldwin 	 * vCPU is HLTed with a high PPR, a low priority interrupt would cause
37302c352febSJohn Baldwin 	 * the 0->1 'pending' transition with a notification, but the vCPU
37312c352febSJohn Baldwin 	 * would ignore the interrupt for the time being.  The same vCPU would
37322c352febSJohn Baldwin 	 * need to then be notified if a high-priority interrupt arrived which
37332c352febSJohn Baldwin 	 * satisfied the PPR.
37342c352febSJohn Baldwin 	 *
37352c352febSJohn Baldwin 	 * The priorities of interrupts injected while 'pending' is asserted
37362c352febSJohn Baldwin 	 * are tracked in a custom bitfield 'pending_prio'.  Should the
37372c352febSJohn Baldwin 	 * to-be-injected interrupt exceed the priorities already present, the
37382c352febSJohn Baldwin 	 * notification is sent.  The priorities recorded in 'pending_prio' are
37392c352febSJohn Baldwin 	 * cleared whenever the 'pending' bit makes another 0->1 transition.
37402c352febSJohn Baldwin 	 */
37412c352febSJohn Baldwin 	if (atomic_cmpset_long(&pir_desc->pending, 0, 1) != 0) {
37422c352febSJohn Baldwin 		notify = 1;
37432c352febSJohn Baldwin 		vlapic_vtx->pending_prio = 0;
37442c352febSJohn Baldwin 	} else {
37452c352febSJohn Baldwin 		const u_int old_prio = vlapic_vtx->pending_prio;
37462c352febSJohn Baldwin 		const u_int prio_bit = VPR_PRIO_BIT(vector & APIC_TPR_INT);
37472c352febSJohn Baldwin 
37482c352febSJohn Baldwin 		if ((old_prio & prio_bit) == 0 && prio_bit > old_prio) {
37492c352febSJohn Baldwin 			atomic_set_int(&vlapic_vtx->pending_prio, prio_bit);
37502c352febSJohn Baldwin 			notify = 1;
37512c352febSJohn Baldwin 		}
37522c352febSJohn Baldwin 	}
375388c4b8d1SNeel Natu 
3754d030f941SJohn Baldwin 	VMX_CTR_PIR(vlapic, pir_desc, notify, vector, level,
3755d030f941SJohn Baldwin 	    "vmx_set_intr_ready");
375688c4b8d1SNeel Natu 	return (notify);
375788c4b8d1SNeel Natu }
375888c4b8d1SNeel Natu 
375988c4b8d1SNeel Natu static int
376088c4b8d1SNeel Natu vmx_pending_intr(struct vlapic *vlapic, int *vecptr)
376188c4b8d1SNeel Natu {
376288c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
376388c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
376488c4b8d1SNeel Natu 	struct LAPIC *lapic;
376588c4b8d1SNeel Natu 	uint64_t pending, pirval;
376688c4b8d1SNeel Natu 	uint32_t ppr, vpr;
376788c4b8d1SNeel Natu 	int i;
376888c4b8d1SNeel Natu 
376988c4b8d1SNeel Natu 	/*
377088c4b8d1SNeel Natu 	 * This function is only expected to be called from the 'HLT' exit
377188c4b8d1SNeel Natu 	 * handler which does not care about the vector that is pending.
377288c4b8d1SNeel Natu 	 */
377388c4b8d1SNeel Natu 	KASSERT(vecptr == NULL, ("vmx_pending_intr: vecptr must be NULL"));
377488c4b8d1SNeel Natu 
377588c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
3776176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
377788c4b8d1SNeel Natu 
377888c4b8d1SNeel Natu 	pending = atomic_load_acq_long(&pir_desc->pending);
37799e33a616STycho Nightingale 	if (!pending) {
37809e33a616STycho Nightingale 		/*
37819e33a616STycho Nightingale 		 * While a virtual interrupt may have already been
37829e33a616STycho Nightingale 		 * processed the actual delivery maybe pending the
37839e33a616STycho Nightingale 		 * interruptibility of the guest.  Recognize a pending
37849e33a616STycho Nightingale 		 * interrupt by reevaluating virtual interrupts
37859e33a616STycho Nightingale 		 * following Section 29.2.1 in the Intel SDM Volume 3.
37869e33a616STycho Nightingale 		 */
3787490768e2STycho Nightingale 		struct vm_exit *vmexit;
37889e33a616STycho Nightingale 		uint8_t rvi, ppr;
37899e33a616STycho Nightingale 
379080cb5d84SJohn Baldwin 		vmexit = vm_exitinfo(vlapic->vcpu);
3791490768e2STycho Nightingale 		KASSERT(vmexit->exitcode == VM_EXITCODE_HLT,
3792490768e2STycho Nightingale 		    ("vmx_pending_intr: exitcode not 'HLT'"));
3793490768e2STycho Nightingale 		rvi = vmexit->u.hlt.intr_status & APIC_TPR_INT;
37949e33a616STycho Nightingale 		lapic = vlapic->apic_page;
37959e33a616STycho Nightingale 		ppr = lapic->ppr & APIC_TPR_INT;
37969e33a616STycho Nightingale 		if (rvi > ppr) {
37979e33a616STycho Nightingale 			return (1);
37989e33a616STycho Nightingale 		}
37999e33a616STycho Nightingale 
38009e33a616STycho Nightingale 		return (0);
38019e33a616STycho Nightingale 	}
380288c4b8d1SNeel Natu 
380388c4b8d1SNeel Natu 	/*
380488c4b8d1SNeel Natu 	 * If there is an interrupt pending then it will be recognized only
380588c4b8d1SNeel Natu 	 * if its priority is greater than the processor priority.
380688c4b8d1SNeel Natu 	 *
380788c4b8d1SNeel Natu 	 * Special case: if the processor priority is zero then any pending
380888c4b8d1SNeel Natu 	 * interrupt will be recognized.
380988c4b8d1SNeel Natu 	 */
381088c4b8d1SNeel Natu 	lapic = vlapic->apic_page;
38119e33a616STycho Nightingale 	ppr = lapic->ppr & APIC_TPR_INT;
381288c4b8d1SNeel Natu 	if (ppr == 0)
381388c4b8d1SNeel Natu 		return (1);
381488c4b8d1SNeel Natu 
3815d030f941SJohn Baldwin 	VLAPIC_CTR1(vlapic, "HLT with non-zero PPR %d", lapic->ppr);
381688c4b8d1SNeel Natu 
38172c352febSJohn Baldwin 	vpr = 0;
381888c4b8d1SNeel Natu 	for (i = 3; i >= 0; i--) {
381988c4b8d1SNeel Natu 		pirval = pir_desc->pir[i];
382088c4b8d1SNeel Natu 		if (pirval != 0) {
38219e33a616STycho Nightingale 			vpr = (i * 64 + flsl(pirval) - 1) & APIC_TPR_INT;
38222c352febSJohn Baldwin 			break;
382388c4b8d1SNeel Natu 		}
382488c4b8d1SNeel Natu 	}
38252c352febSJohn Baldwin 
38262c352febSJohn Baldwin 	/*
38272c352febSJohn Baldwin 	 * If the highest-priority pending interrupt falls short of the
38282c352febSJohn Baldwin 	 * processor priority of this vCPU, ensure that 'pending_prio' does not
38292c352febSJohn Baldwin 	 * have any stale bits which would preclude a higher-priority interrupt
38302c352febSJohn Baldwin 	 * from incurring a notification later.
38312c352febSJohn Baldwin 	 */
38322c352febSJohn Baldwin 	if (vpr <= ppr) {
38332c352febSJohn Baldwin 		const u_int prio_bit = VPR_PRIO_BIT(vpr);
38342c352febSJohn Baldwin 		const u_int old = vlapic_vtx->pending_prio;
38352c352febSJohn Baldwin 
38362c352febSJohn Baldwin 		if (old > prio_bit && (old & prio_bit) == 0) {
38372c352febSJohn Baldwin 			vlapic_vtx->pending_prio = prio_bit;
38382c352febSJohn Baldwin 		}
383988c4b8d1SNeel Natu 		return (0);
384088c4b8d1SNeel Natu 	}
38412c352febSJohn Baldwin 	return (1);
38422c352febSJohn Baldwin }
384388c4b8d1SNeel Natu 
384488c4b8d1SNeel Natu static void
384588c4b8d1SNeel Natu vmx_intr_accepted(struct vlapic *vlapic, int vector)
384688c4b8d1SNeel Natu {
384788c4b8d1SNeel Natu 
384888c4b8d1SNeel Natu 	panic("vmx_intr_accepted: not expected to be called");
384988c4b8d1SNeel Natu }
385088c4b8d1SNeel Natu 
3851176666c2SNeel Natu static void
385230b94db8SNeel Natu vmx_set_tmr(struct vlapic *vlapic, int vector, bool level)
385330b94db8SNeel Natu {
385430b94db8SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
385530b94db8SNeel Natu 	struct vmcs *vmcs;
385630b94db8SNeel Natu 	uint64_t mask, val;
385730b94db8SNeel Natu 
385830b94db8SNeel Natu 	KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
385980cb5d84SJohn Baldwin 	KASSERT(!vcpu_is_running(vlapic->vcpu, NULL),
386030b94db8SNeel Natu 	    ("vmx_set_tmr: vcpu cannot be running"));
386130b94db8SNeel Natu 
386230b94db8SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
38631aa51504SJohn Baldwin 	vmcs = vlapic_vtx->vcpu->vmcs;
386430b94db8SNeel Natu 	mask = 1UL << (vector % 64);
386530b94db8SNeel Natu 
386630b94db8SNeel Natu 	VMPTRLD(vmcs);
386730b94db8SNeel Natu 	val = vmcs_read(VMCS_EOI_EXIT(vector));
386830b94db8SNeel Natu 	if (level)
386930b94db8SNeel Natu 		val |= mask;
387030b94db8SNeel Natu 	else
387130b94db8SNeel Natu 		val &= ~mask;
387230b94db8SNeel Natu 	vmcs_write(VMCS_EOI_EXIT(vector), val);
387330b94db8SNeel Natu 	VMCLEAR(vmcs);
387430b94db8SNeel Natu }
387530b94db8SNeel Natu 
387630b94db8SNeel Natu static void
38771bc51badSMichael Reifenberger vmx_enable_x2apic_mode_ts(struct vlapic *vlapic)
38781bc51badSMichael Reifenberger {
38791aa51504SJohn Baldwin 	struct vlapic_vtx *vlapic_vtx;
38800f00260cSJohn Baldwin 	struct vmx_vcpu *vcpu;
38811bc51badSMichael Reifenberger 	struct vmcs *vmcs;
38821bc51badSMichael Reifenberger 	uint32_t proc_ctls;
38831bc51badSMichael Reifenberger 
38841aa51504SJohn Baldwin 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
38851aa51504SJohn Baldwin 	vcpu = vlapic_vtx->vcpu;
38860f00260cSJohn Baldwin 	vmcs = vcpu->vmcs;
38871bc51badSMichael Reifenberger 
38880f00260cSJohn Baldwin 	proc_ctls = vcpu->cap.proc_ctls;
38891bc51badSMichael Reifenberger 	proc_ctls &= ~PROCBASED_USE_TPR_SHADOW;
38901bc51badSMichael Reifenberger 	proc_ctls |= PROCBASED_CR8_LOAD_EXITING;
38911bc51badSMichael Reifenberger 	proc_ctls |= PROCBASED_CR8_STORE_EXITING;
38920f00260cSJohn Baldwin 	vcpu->cap.proc_ctls = proc_ctls;
38931bc51badSMichael Reifenberger 
38941bc51badSMichael Reifenberger 	VMPTRLD(vmcs);
38951bc51badSMichael Reifenberger 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, proc_ctls);
38961bc51badSMichael Reifenberger 	VMCLEAR(vmcs);
38971bc51badSMichael Reifenberger }
38981bc51badSMichael Reifenberger 
38991bc51badSMichael Reifenberger static void
39001bc51badSMichael Reifenberger vmx_enable_x2apic_mode_vid(struct vlapic *vlapic)
3901159dd56fSNeel Natu {
39021aa51504SJohn Baldwin 	struct vlapic_vtx *vlapic_vtx;
3903159dd56fSNeel Natu 	struct vmx *vmx;
39040f00260cSJohn Baldwin 	struct vmx_vcpu *vcpu;
3905159dd56fSNeel Natu 	struct vmcs *vmcs;
3906159dd56fSNeel Natu 	uint32_t proc_ctls2;
39071aa51504SJohn Baldwin 	int error __diagused;
3908159dd56fSNeel Natu 
39091aa51504SJohn Baldwin 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
39101aa51504SJohn Baldwin 	vcpu = vlapic_vtx->vcpu;
3911869c8d19SJohn Baldwin 	vmx = vcpu->vmx;
39120f00260cSJohn Baldwin 	vmcs = vcpu->vmcs;
3913159dd56fSNeel Natu 
39140f00260cSJohn Baldwin 	proc_ctls2 = vcpu->cap.proc_ctls2;
3915159dd56fSNeel Natu 	KASSERT((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) != 0,
3916159dd56fSNeel Natu 	    ("%s: invalid proc_ctls2 %#x", __func__, proc_ctls2));
3917159dd56fSNeel Natu 
3918159dd56fSNeel Natu 	proc_ctls2 &= ~PROCBASED2_VIRTUALIZE_APIC_ACCESSES;
3919159dd56fSNeel Natu 	proc_ctls2 |= PROCBASED2_VIRTUALIZE_X2APIC_MODE;
39200f00260cSJohn Baldwin 	vcpu->cap.proc_ctls2 = proc_ctls2;
3921159dd56fSNeel Natu 
3922159dd56fSNeel Natu 	VMPTRLD(vmcs);
3923159dd56fSNeel Natu 	vmcs_write(VMCS_SEC_PROC_BASED_CTLS, proc_ctls2);
3924159dd56fSNeel Natu 	VMCLEAR(vmcs);
3925159dd56fSNeel Natu 
3926159dd56fSNeel Natu 	if (vlapic->vcpuid == 0) {
3927159dd56fSNeel Natu 		/*
3928159dd56fSNeel Natu 		 * The nested page table mappings are shared by all vcpus
3929159dd56fSNeel Natu 		 * so unmap the APIC access page just once.
3930159dd56fSNeel Natu 		 */
3931159dd56fSNeel Natu 		error = vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
3932159dd56fSNeel Natu 		KASSERT(error == 0, ("%s: vm_unmap_mmio error %d",
3933159dd56fSNeel Natu 		    __func__, error));
3934159dd56fSNeel Natu 
3935159dd56fSNeel Natu 		/*
3936159dd56fSNeel Natu 		 * The MSR bitmap is shared by all vcpus so modify it only
3937159dd56fSNeel Natu 		 * once in the context of vcpu 0.
3938159dd56fSNeel Natu 		 */
3939159dd56fSNeel Natu 		error = vmx_allow_x2apic_msrs(vmx);
3940159dd56fSNeel Natu 		KASSERT(error == 0, ("%s: vmx_allow_x2apic_msrs error %d",
3941159dd56fSNeel Natu 		    __func__, error));
3942159dd56fSNeel Natu 	}
3943159dd56fSNeel Natu }
3944159dd56fSNeel Natu 
3945159dd56fSNeel Natu static void
3946176666c2SNeel Natu vmx_post_intr(struct vlapic *vlapic, int hostcpu)
3947176666c2SNeel Natu {
3948176666c2SNeel Natu 
3949176666c2SNeel Natu 	ipi_cpu(hostcpu, pirvec);
3950176666c2SNeel Natu }
3951176666c2SNeel Natu 
395288c4b8d1SNeel Natu /*
395388c4b8d1SNeel Natu  * Transfer the pending interrupts in the PIR descriptor to the IRR
395488c4b8d1SNeel Natu  * in the virtual APIC page.
395588c4b8d1SNeel Natu  */
395688c4b8d1SNeel Natu static void
395788c4b8d1SNeel Natu vmx_inject_pir(struct vlapic *vlapic)
395888c4b8d1SNeel Natu {
395988c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
396088c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
396188c4b8d1SNeel Natu 	struct LAPIC *lapic;
396288c4b8d1SNeel Natu 	uint64_t val, pirval;
39630e30c5c0SWarner Losh 	int rvi, pirbase = -1;
396488c4b8d1SNeel Natu 	uint16_t intr_status_old, intr_status_new;
396588c4b8d1SNeel Natu 
396688c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
3967176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
396888c4b8d1SNeel Natu 	if (atomic_cmpset_long(&pir_desc->pending, 1, 0) == 0) {
3969d030f941SJohn Baldwin 		VLAPIC_CTR0(vlapic, "vmx_inject_pir: "
397088c4b8d1SNeel Natu 		    "no posted interrupt pending");
397188c4b8d1SNeel Natu 		return;
397288c4b8d1SNeel Natu 	}
397388c4b8d1SNeel Natu 
397488c4b8d1SNeel Natu 	pirval = 0;
3975201b1cccSPeter Grehan 	pirbase = -1;
397688c4b8d1SNeel Natu 	lapic = vlapic->apic_page;
397788c4b8d1SNeel Natu 
397888c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[0]);
397988c4b8d1SNeel Natu 	if (val != 0) {
398088c4b8d1SNeel Natu 		lapic->irr0 |= val;
398188c4b8d1SNeel Natu 		lapic->irr1 |= val >> 32;
398288c4b8d1SNeel Natu 		pirbase = 0;
398388c4b8d1SNeel Natu 		pirval = val;
398488c4b8d1SNeel Natu 	}
398588c4b8d1SNeel Natu 
398688c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[1]);
398788c4b8d1SNeel Natu 	if (val != 0) {
398888c4b8d1SNeel Natu 		lapic->irr2 |= val;
398988c4b8d1SNeel Natu 		lapic->irr3 |= val >> 32;
399088c4b8d1SNeel Natu 		pirbase = 64;
399188c4b8d1SNeel Natu 		pirval = val;
399288c4b8d1SNeel Natu 	}
399388c4b8d1SNeel Natu 
399488c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[2]);
399588c4b8d1SNeel Natu 	if (val != 0) {
399688c4b8d1SNeel Natu 		lapic->irr4 |= val;
399788c4b8d1SNeel Natu 		lapic->irr5 |= val >> 32;
399888c4b8d1SNeel Natu 		pirbase = 128;
399988c4b8d1SNeel Natu 		pirval = val;
400088c4b8d1SNeel Natu 	}
400188c4b8d1SNeel Natu 
400288c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[3]);
400388c4b8d1SNeel Natu 	if (val != 0) {
400488c4b8d1SNeel Natu 		lapic->irr6 |= val;
400588c4b8d1SNeel Natu 		lapic->irr7 |= val >> 32;
400688c4b8d1SNeel Natu 		pirbase = 192;
400788c4b8d1SNeel Natu 		pirval = val;
400888c4b8d1SNeel Natu 	}
4009201b1cccSPeter Grehan 
401088c4b8d1SNeel Natu 	VLAPIC_CTR_IRR(vlapic, "vmx_inject_pir");
401188c4b8d1SNeel Natu 
401288c4b8d1SNeel Natu 	/*
401388c4b8d1SNeel Natu 	 * Update RVI so the processor can evaluate pending virtual
401488c4b8d1SNeel Natu 	 * interrupts on VM-entry.
4015201b1cccSPeter Grehan 	 *
4016201b1cccSPeter Grehan 	 * It is possible for pirval to be 0 here, even though the
4017201b1cccSPeter Grehan 	 * pending bit has been set. The scenario is:
4018201b1cccSPeter Grehan 	 * CPU-Y is sending a posted interrupt to CPU-X, which
4019201b1cccSPeter Grehan 	 * is running a guest and processing posted interrupts in h/w.
4020201b1cccSPeter Grehan 	 * CPU-X will eventually exit and the state seen in s/w is
4021201b1cccSPeter Grehan 	 * the pending bit set, but no PIR bits set.
4022201b1cccSPeter Grehan 	 *
4023201b1cccSPeter Grehan 	 *      CPU-X                      CPU-Y
4024201b1cccSPeter Grehan 	 *   (vm running)                (host running)
4025201b1cccSPeter Grehan 	 *   rx posted interrupt
4026201b1cccSPeter Grehan 	 *   CLEAR pending bit
4027201b1cccSPeter Grehan 	 *				 SET PIR bit
4028201b1cccSPeter Grehan 	 *   READ/CLEAR PIR bits
4029201b1cccSPeter Grehan 	 *				 SET pending bit
4030201b1cccSPeter Grehan 	 *   (vm exit)
4031201b1cccSPeter Grehan 	 *   pending bit set, PIR 0
403288c4b8d1SNeel Natu 	 */
403388c4b8d1SNeel Natu 	if (pirval != 0) {
403488c4b8d1SNeel Natu 		rvi = pirbase + flsl(pirval) - 1;
403588c4b8d1SNeel Natu 		intr_status_old = vmcs_read(VMCS_GUEST_INTR_STATUS);
403688c4b8d1SNeel Natu 		intr_status_new = (intr_status_old & 0xFF00) | rvi;
403788c4b8d1SNeel Natu 		if (intr_status_new > intr_status_old) {
403888c4b8d1SNeel Natu 			vmcs_write(VMCS_GUEST_INTR_STATUS, intr_status_new);
4039d030f941SJohn Baldwin 			VLAPIC_CTR2(vlapic, "vmx_inject_pir: "
404088c4b8d1SNeel Natu 			    "guest_intr_status changed from 0x%04x to 0x%04x",
404188c4b8d1SNeel Natu 			    intr_status_old, intr_status_new);
404288c4b8d1SNeel Natu 		}
404388c4b8d1SNeel Natu 	}
404488c4b8d1SNeel Natu }
404588c4b8d1SNeel Natu 
4046de5ea6b6SNeel Natu static struct vlapic *
4047869c8d19SJohn Baldwin vmx_vlapic_init(void *vcpui)
4048de5ea6b6SNeel Natu {
4049de5ea6b6SNeel Natu 	struct vmx *vmx;
40501aa51504SJohn Baldwin 	struct vmx_vcpu *vcpu;
4051de5ea6b6SNeel Natu 	struct vlapic *vlapic;
4052176666c2SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
4053de5ea6b6SNeel Natu 
40541aa51504SJohn Baldwin 	vcpu = vcpui;
4055869c8d19SJohn Baldwin 	vmx = vcpu->vmx;
4056de5ea6b6SNeel Natu 
405788c4b8d1SNeel Natu 	vlapic = malloc(sizeof(struct vlapic_vtx), M_VLAPIC, M_WAITOK | M_ZERO);
4058de5ea6b6SNeel Natu 	vlapic->vm = vmx->vm;
4059950af9ffSJohn Baldwin 	vlapic->vcpu = vcpu->vcpu;
40601aa51504SJohn Baldwin 	vlapic->vcpuid = vcpu->vcpuid;
40611aa51504SJohn Baldwin 	vlapic->apic_page = (struct LAPIC *)vcpu->apic_page;
4062de5ea6b6SNeel Natu 
4063176666c2SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
40641aa51504SJohn Baldwin 	vlapic_vtx->pir_desc = vcpu->pir_desc;
40651aa51504SJohn Baldwin 	vlapic_vtx->vcpu = vcpu;
4066176666c2SNeel Natu 
40671bc51badSMichael Reifenberger 	if (tpr_shadowing) {
40681bc51badSMichael Reifenberger 		vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode_ts;
40691bc51badSMichael Reifenberger 	}
40701bc51badSMichael Reifenberger 
407188c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
407288c4b8d1SNeel Natu 		vlapic->ops.set_intr_ready = vmx_set_intr_ready;
407388c4b8d1SNeel Natu 		vlapic->ops.pending_intr = vmx_pending_intr;
407488c4b8d1SNeel Natu 		vlapic->ops.intr_accepted = vmx_intr_accepted;
407530b94db8SNeel Natu 		vlapic->ops.set_tmr = vmx_set_tmr;
40761bc51badSMichael Reifenberger 		vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode_vid;
407788c4b8d1SNeel Natu 	}
407888c4b8d1SNeel Natu 
4079176666c2SNeel Natu 	if (posted_interrupts)
4080176666c2SNeel Natu 		vlapic->ops.post_intr = vmx_post_intr;
4081176666c2SNeel Natu 
4082de5ea6b6SNeel Natu 	vlapic_init(vlapic);
4083de5ea6b6SNeel Natu 
4084de5ea6b6SNeel Natu 	return (vlapic);
4085de5ea6b6SNeel Natu }
4086de5ea6b6SNeel Natu 
4087de5ea6b6SNeel Natu static void
4088869c8d19SJohn Baldwin vmx_vlapic_cleanup(struct vlapic *vlapic)
4089de5ea6b6SNeel Natu {
4090de5ea6b6SNeel Natu 
4091de5ea6b6SNeel Natu 	vlapic_cleanup(vlapic);
4092de5ea6b6SNeel Natu 	free(vlapic, M_VLAPIC);
4093de5ea6b6SNeel Natu }
4094de5ea6b6SNeel Natu 
4095483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT
4096483d953aSJohn Baldwin static int
4097869c8d19SJohn Baldwin vmx_snapshot(void *vmi, struct vm_snapshot_meta *meta)
4098483d953aSJohn Baldwin {
409939ec056eSJohn Baldwin 	return (0);
4100483d953aSJohn Baldwin }
4101483d953aSJohn Baldwin 
4102483d953aSJohn Baldwin static int
4103869c8d19SJohn Baldwin vmx_vcpu_snapshot(void *vcpui, struct vm_snapshot_meta *meta)
4104483d953aSJohn Baldwin {
4105483d953aSJohn Baldwin 	struct vmcs *vmcs;
4106483d953aSJohn Baldwin 	struct vmx *vmx;
410739ec056eSJohn Baldwin 	struct vmx_vcpu *vcpu;
410839ec056eSJohn Baldwin 	struct vmxctx *vmxctx;
4109483d953aSJohn Baldwin 	int err, run, hostcpu;
4110483d953aSJohn Baldwin 
4111483d953aSJohn Baldwin 	err = 0;
4112869c8d19SJohn Baldwin 	vcpu = vcpui;
4113869c8d19SJohn Baldwin 	vmx = vcpu->vmx;
411439ec056eSJohn Baldwin 	vmcs = vcpu->vmcs;
4115483d953aSJohn Baldwin 
411680cb5d84SJohn Baldwin 	run = vcpu_is_running(vcpu->vcpu, &hostcpu);
4117483d953aSJohn Baldwin 	if (run && hostcpu != curcpu) {
411839ec056eSJohn Baldwin 		printf("%s: %s%d is running", __func__, vm_name(vmx->vm),
41191aa51504SJohn Baldwin 		    vcpu->vcpuid);
4120483d953aSJohn Baldwin 		return (EINVAL);
4121483d953aSJohn Baldwin 	}
4122483d953aSJohn Baldwin 
4123483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CR0, meta);
4124483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CR3, meta);
4125483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CR4, meta);
4126483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_DR7, meta);
4127483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_RSP, meta);
4128483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_RIP, meta);
4129483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_RFLAGS, meta);
4130483d953aSJohn Baldwin 
4131483d953aSJohn Baldwin 	/* Guest segments */
4132483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_ES, meta);
4133483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_ES, meta);
4134483d953aSJohn Baldwin 
4135483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CS, meta);
4136483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_CS, meta);
4137483d953aSJohn Baldwin 
4138483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_SS, meta);
4139483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_SS, meta);
4140483d953aSJohn Baldwin 
4141483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_DS, meta);
4142483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_DS, meta);
4143483d953aSJohn Baldwin 
4144483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_FS, meta);
4145483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_FS, meta);
4146483d953aSJohn Baldwin 
4147483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_GS, meta);
4148483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_GS, meta);
4149483d953aSJohn Baldwin 
4150483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_TR, meta);
4151483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_TR, meta);
4152483d953aSJohn Baldwin 
4153483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_LDTR, meta);
4154483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_LDTR, meta);
4155483d953aSJohn Baldwin 
4156483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_EFER, meta);
4157483d953aSJohn Baldwin 
4158483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_IDTR, meta);
4159483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_GDTR, meta);
4160483d953aSJohn Baldwin 
4161483d953aSJohn Baldwin 	/* Guest page tables */
4162483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE0, meta);
4163483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE1, meta);
4164483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE2, meta);
4165483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE3, meta);
4166483d953aSJohn Baldwin 
4167483d953aSJohn Baldwin 	/* Other guest state */
4168483d953aSJohn Baldwin 	err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_IA32_SYSENTER_CS, meta);
4169483d953aSJohn Baldwin 	err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_IA32_SYSENTER_ESP, meta);
4170483d953aSJohn Baldwin 	err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_IA32_SYSENTER_EIP, meta);
4171483d953aSJohn Baldwin 	err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_INTERRUPTIBILITY, meta);
4172483d953aSJohn Baldwin 	err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_ACTIVITY, meta);
4173483d953aSJohn Baldwin 	err += vmcs_snapshot_any(vmcs, run, VMCS_ENTRY_CTLS, meta);
4174483d953aSJohn Baldwin 	err += vmcs_snapshot_any(vmcs, run, VMCS_EXIT_CTLS, meta);
417539ec056eSJohn Baldwin 	if (err != 0)
417639ec056eSJohn Baldwin 		goto done;
4177483d953aSJohn Baldwin 
417839ec056eSJohn Baldwin 	SNAPSHOT_BUF_OR_LEAVE(vcpu->guest_msrs,
417939ec056eSJohn Baldwin 	    sizeof(vcpu->guest_msrs), meta, err, done);
418039ec056eSJohn Baldwin 
418139ec056eSJohn Baldwin 	vmxctx = &vcpu->ctx;
418239ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rdi, meta, err, done);
418339ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rsi, meta, err, done);
418439ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rdx, meta, err, done);
418539ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rcx, meta, err, done);
418639ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r8, meta, err, done);
418739ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r9, meta, err, done);
418839ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rax, meta, err, done);
418939ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rbx, meta, err, done);
419039ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rbp, meta, err, done);
419139ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r10, meta, err, done);
419239ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r11, meta, err, done);
419339ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r12, meta, err, done);
419439ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r13, meta, err, done);
419539ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r14, meta, err, done);
419639ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r15, meta, err, done);
419739ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_cr2, meta, err, done);
419839ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr0, meta, err, done);
419939ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr1, meta, err, done);
420039ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr2, meta, err, done);
420139ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr3, meta, err, done);
420239ec056eSJohn Baldwin 	SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr6, meta, err, done);
420339ec056eSJohn Baldwin 
420439ec056eSJohn Baldwin done:
4205483d953aSJohn Baldwin 	return (err);
4206483d953aSJohn Baldwin }
4207483d953aSJohn Baldwin 
4208483d953aSJohn Baldwin static int
4209869c8d19SJohn Baldwin vmx_restore_tsc(void *vcpui, uint64_t offset)
4210483d953aSJohn Baldwin {
42111aa51504SJohn Baldwin 	struct vmx_vcpu *vcpu = vcpui;
4212869c8d19SJohn Baldwin 	struct vmcs *vmcs;
4213869c8d19SJohn Baldwin 	struct vmx *vmx;
4214483d953aSJohn Baldwin 	int error, running, hostcpu;
4215483d953aSJohn Baldwin 
4216869c8d19SJohn Baldwin 	vmx = vcpu->vmx;
42171aa51504SJohn Baldwin 	vmcs = vcpu->vmcs;
4218483d953aSJohn Baldwin 
421980cb5d84SJohn Baldwin 	running = vcpu_is_running(vcpu->vcpu, &hostcpu);
4220483d953aSJohn Baldwin 	if (running && hostcpu != curcpu) {
42211aa51504SJohn Baldwin 		printf("%s: %s%d is running", __func__, vm_name(vmx->vm),
42221aa51504SJohn Baldwin 		    vcpu->vcpuid);
4223483d953aSJohn Baldwin 		return (EINVAL);
4224483d953aSJohn Baldwin 	}
4225483d953aSJohn Baldwin 
4226483d953aSJohn Baldwin 	if (!running)
4227483d953aSJohn Baldwin 		VMPTRLD(vmcs);
4228483d953aSJohn Baldwin 
422980cb5d84SJohn Baldwin 	error = vmx_set_tsc_offset(vcpu, offset);
4230483d953aSJohn Baldwin 
4231483d953aSJohn Baldwin 	if (!running)
4232483d953aSJohn Baldwin 		VMCLEAR(vmcs);
4233483d953aSJohn Baldwin 	return (error);
4234483d953aSJohn Baldwin }
4235483d953aSJohn Baldwin #endif
4236483d953aSJohn Baldwin 
423715add60dSPeter Grehan const struct vmm_ops vmm_ops_intel = {
423815add60dSPeter Grehan 	.modinit	= vmx_modinit,
423915add60dSPeter Grehan 	.modcleanup	= vmx_modcleanup,
424015add60dSPeter Grehan 	.modresume	= vmx_modresume,
424113a7c4d4SMark Johnston 	.init		= vmx_init,
424215add60dSPeter Grehan 	.run		= vmx_run,
424313a7c4d4SMark Johnston 	.cleanup	= vmx_cleanup,
42441aa51504SJohn Baldwin 	.vcpu_init	= vmx_vcpu_init,
42451aa51504SJohn Baldwin 	.vcpu_cleanup	= vmx_vcpu_cleanup,
424615add60dSPeter Grehan 	.getreg		= vmx_getreg,
424715add60dSPeter Grehan 	.setreg		= vmx_setreg,
424815add60dSPeter Grehan 	.getdesc	= vmx_getdesc,
424915add60dSPeter Grehan 	.setdesc	= vmx_setdesc,
425015add60dSPeter Grehan 	.getcap		= vmx_getcap,
425115add60dSPeter Grehan 	.setcap		= vmx_setcap,
425215add60dSPeter Grehan 	.vmspace_alloc	= vmx_vmspace_alloc,
425315add60dSPeter Grehan 	.vmspace_free	= vmx_vmspace_free,
425413a7c4d4SMark Johnston 	.vlapic_init	= vmx_vlapic_init,
425513a7c4d4SMark Johnston 	.vlapic_cleanup	= vmx_vlapic_cleanup,
4256483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT
425715add60dSPeter Grehan 	.snapshot	= vmx_snapshot,
425839ec056eSJohn Baldwin 	.vcpu_snapshot	= vmx_vcpu_snapshot,
425915add60dSPeter Grehan 	.restore_tsc	= vmx_restore_tsc,
4260483d953aSJohn Baldwin #endif
4261366f6083SPeter Grehan };
4262