xref: /freebsd/sys/amd64/vmm/intel/vmx.c (revision 3ba952e1a2179c232402c82d5c7587159b15a8dd)
1366f6083SPeter Grehan /*-
2c49761ddSPedro F. Giffuni  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3c49761ddSPedro F. Giffuni  *
4366f6083SPeter Grehan  * Copyright (c) 2011 NetApp, Inc.
5366f6083SPeter Grehan  * All rights reserved.
62c352febSJohn Baldwin  * Copyright (c) 2018 Joyent, Inc.
7366f6083SPeter Grehan  *
8366f6083SPeter Grehan  * Redistribution and use in source and binary forms, with or without
9366f6083SPeter Grehan  * modification, are permitted provided that the following conditions
10366f6083SPeter Grehan  * are met:
11366f6083SPeter Grehan  * 1. Redistributions of source code must retain the above copyright
12366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer.
13366f6083SPeter Grehan  * 2. Redistributions in binary form must reproduce the above copyright
14366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer in the
15366f6083SPeter Grehan  *    documentation and/or other materials provided with the distribution.
16366f6083SPeter Grehan  *
17366f6083SPeter Grehan  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
18366f6083SPeter Grehan  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19366f6083SPeter Grehan  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20366f6083SPeter Grehan  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
21366f6083SPeter Grehan  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22366f6083SPeter Grehan  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23366f6083SPeter Grehan  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24366f6083SPeter Grehan  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25366f6083SPeter Grehan  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26366f6083SPeter Grehan  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27366f6083SPeter Grehan  * SUCH DAMAGE.
28366f6083SPeter Grehan  *
29366f6083SPeter Grehan  * $FreeBSD$
30366f6083SPeter Grehan  */
31366f6083SPeter Grehan 
32366f6083SPeter Grehan #include <sys/cdefs.h>
33366f6083SPeter Grehan __FBSDID("$FreeBSD$");
34366f6083SPeter Grehan 
35483d953aSJohn Baldwin #include "opt_bhyve_snapshot.h"
36483d953aSJohn Baldwin 
37366f6083SPeter Grehan #include <sys/param.h>
38366f6083SPeter Grehan #include <sys/systm.h>
39366f6083SPeter Grehan #include <sys/smp.h>
40366f6083SPeter Grehan #include <sys/kernel.h>
41366f6083SPeter Grehan #include <sys/malloc.h>
42366f6083SPeter Grehan #include <sys/pcpu.h>
43366f6083SPeter Grehan #include <sys/proc.h>
44b7924341SAndrew Turner #include <sys/reg.h>
456f5a9606SMark Johnston #include <sys/smr.h>
463565b59eSNeel Natu #include <sys/sysctl.h>
47366f6083SPeter Grehan 
48366f6083SPeter Grehan #include <vm/vm.h>
49366f6083SPeter Grehan #include <vm/pmap.h>
50366f6083SPeter Grehan 
51366f6083SPeter Grehan #include <machine/psl.h>
52366f6083SPeter Grehan #include <machine/cpufunc.h>
538b287612SJohn Baldwin #include <machine/md_var.h>
54366f6083SPeter Grehan #include <machine/segments.h>
55176666c2SNeel Natu #include <machine/smp.h>
56608f97c3SPeter Grehan #include <machine/specialreg.h>
57366f6083SPeter Grehan #include <machine/vmparam.h>
58366f6083SPeter Grehan 
59366f6083SPeter Grehan #include <machine/vmm.h>
60dc506506SNeel Natu #include <machine/vmm_dev.h>
61e813a873SNeel Natu #include <machine/vmm_instruction_emul.h>
62483d953aSJohn Baldwin #include <machine/vmm_snapshot.h>
63483d953aSJohn Baldwin 
64c3498942SNeel Natu #include "vmm_lapic.h"
65b01c2033SNeel Natu #include "vmm_host.h"
66762fd208STycho Nightingale #include "vmm_ioport.h"
67366f6083SPeter Grehan #include "vmm_ktr.h"
68366f6083SPeter Grehan #include "vmm_stat.h"
690775fbb4STycho Nightingale #include "vatpic.h"
70de5ea6b6SNeel Natu #include "vlapic.h"
71de5ea6b6SNeel Natu #include "vlapic_priv.h"
72366f6083SPeter Grehan 
73366f6083SPeter Grehan #include "ept.h"
74366f6083SPeter Grehan #include "vmx_cpufunc.h"
75366f6083SPeter Grehan #include "vmx.h"
76c3498942SNeel Natu #include "vmx_msr.h"
77366f6083SPeter Grehan #include "x86.h"
78366f6083SPeter Grehan #include "vmx_controls.h"
79366f6083SPeter Grehan 
80366f6083SPeter Grehan #define	PINBASED_CTLS_ONE_SETTING					\
81366f6083SPeter Grehan 	(PINBASED_EXTINT_EXITING	|				\
82366f6083SPeter Grehan 	 PINBASED_NMI_EXITING		|				\
83366f6083SPeter Grehan 	 PINBASED_VIRTUAL_NMI)
84366f6083SPeter Grehan #define	PINBASED_CTLS_ZERO_SETTING	0
85366f6083SPeter Grehan 
86366f6083SPeter Grehan #define PROCBASED_CTLS_WINDOW_SETTING					\
87366f6083SPeter Grehan 	(PROCBASED_INT_WINDOW_EXITING	|				\
88366f6083SPeter Grehan 	 PROCBASED_NMI_WINDOW_EXITING)
89366f6083SPeter Grehan 
90366f6083SPeter Grehan #define	PROCBASED_CTLS_ONE_SETTING					\
91366f6083SPeter Grehan 	(PROCBASED_SECONDARY_CONTROLS	|				\
9265145c7fSNeel Natu 	 PROCBASED_MWAIT_EXITING	|				\
9365145c7fSNeel Natu 	 PROCBASED_MONITOR_EXITING	|				\
94366f6083SPeter Grehan 	 PROCBASED_IO_EXITING		|				\
95366f6083SPeter Grehan 	 PROCBASED_MSR_BITMAPS		|				\
96594db002STycho Nightingale 	 PROCBASED_CTLS_WINDOW_SETTING	|				\
97594db002STycho Nightingale 	 PROCBASED_CR8_LOAD_EXITING	|				\
98594db002STycho Nightingale 	 PROCBASED_CR8_STORE_EXITING)
99366f6083SPeter Grehan #define	PROCBASED_CTLS_ZERO_SETTING	\
100366f6083SPeter Grehan 	(PROCBASED_CR3_LOAD_EXITING |	\
101366f6083SPeter Grehan 	PROCBASED_CR3_STORE_EXITING |	\
102366f6083SPeter Grehan 	PROCBASED_IO_BITMAPS)
103366f6083SPeter Grehan 
104366f6083SPeter Grehan #define	PROCBASED_CTLS2_ONE_SETTING	PROCBASED2_ENABLE_EPT
105366f6083SPeter Grehan #define	PROCBASED_CTLS2_ZERO_SETTING	0
106366f6083SPeter Grehan 
107d72978ecSNeel Natu #define	VM_EXIT_CTLS_ONE_SETTING					\
10865eefbe4SJohn Baldwin 	(VM_EXIT_SAVE_DEBUG_CONTROLS		|			\
10965eefbe4SJohn Baldwin 	VM_EXIT_HOST_LMA			|			\
110366f6083SPeter Grehan 	VM_EXIT_SAVE_EFER			|			\
111d72978ecSNeel Natu 	VM_EXIT_LOAD_EFER			|			\
112a318f7ddSNeel Natu 	VM_EXIT_ACKNOWLEDGE_INTERRUPT)
113d72978ecSNeel Natu 
11465eefbe4SJohn Baldwin #define	VM_EXIT_CTLS_ZERO_SETTING	0
115366f6083SPeter Grehan 
11665eefbe4SJohn Baldwin #define	VM_ENTRY_CTLS_ONE_SETTING					\
11765eefbe4SJohn Baldwin 	(VM_ENTRY_LOAD_DEBUG_CONTROLS		|			\
11865eefbe4SJohn Baldwin 	VM_ENTRY_LOAD_EFER)
119608f97c3SPeter Grehan 
120366f6083SPeter Grehan #define	VM_ENTRY_CTLS_ZERO_SETTING					\
12165eefbe4SJohn Baldwin 	(VM_ENTRY_INTO_SMM			|			\
122366f6083SPeter Grehan 	VM_ENTRY_DEACTIVATE_DUAL_MONITOR)
123366f6083SPeter Grehan 
124366f6083SPeter Grehan #define	HANDLED		1
125366f6083SPeter Grehan #define	UNHANDLED	0
126366f6083SPeter Grehan 
127de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VMX, "vmx", "vmx");
128de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
129366f6083SPeter Grehan 
1303565b59eSNeel Natu SYSCTL_DECL(_hw_vmm);
131b40598c5SPawel Biernacki SYSCTL_NODE(_hw_vmm, OID_AUTO, vmx, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
132b40598c5SPawel Biernacki     NULL);
1333565b59eSNeel Natu 
134b3996dd4SJohn Baldwin int vmxon_enabled[MAXCPU];
135366f6083SPeter Grehan static char vmxon_region[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE);
136366f6083SPeter Grehan 
137366f6083SPeter Grehan static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2;
138366f6083SPeter Grehan static uint32_t exit_ctls, entry_ctls;
139366f6083SPeter Grehan 
140366f6083SPeter Grehan static uint64_t cr0_ones_mask, cr0_zeros_mask;
1413565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_ones_mask, CTLFLAG_RD,
1423565b59eSNeel Natu 	     &cr0_ones_mask, 0, NULL);
1433565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_zeros_mask, CTLFLAG_RD,
1443565b59eSNeel Natu 	     &cr0_zeros_mask, 0, NULL);
1453565b59eSNeel Natu 
146366f6083SPeter Grehan static uint64_t cr4_ones_mask, cr4_zeros_mask;
1473565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_ones_mask, CTLFLAG_RD,
1483565b59eSNeel Natu 	     &cr4_ones_mask, 0, NULL);
1493565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_zeros_mask, CTLFLAG_RD,
1503565b59eSNeel Natu 	     &cr4_zeros_mask, 0, NULL);
151366f6083SPeter Grehan 
1523565b59eSNeel Natu static int vmx_initialized;
1533565b59eSNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, initialized, CTLFLAG_RD,
1543565b59eSNeel Natu 	   &vmx_initialized, 0, "Intel VMX initialized");
1553565b59eSNeel Natu 
156366f6083SPeter Grehan /*
157366f6083SPeter Grehan  * Optional capabilities
158366f6083SPeter Grehan  */
159b40598c5SPawel Biernacki static SYSCTL_NODE(_hw_vmm_vmx, OID_AUTO, cap,
160b40598c5SPawel Biernacki     CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
161b40598c5SPawel Biernacki     NULL);
16206fc6db9SJohn Baldwin 
163366f6083SPeter Grehan static int cap_halt_exit;
16406fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, halt_exit, CTLFLAG_RD, &cap_halt_exit, 0,
16506fc6db9SJohn Baldwin     "HLT triggers a VM-exit");
16606fc6db9SJohn Baldwin 
167366f6083SPeter Grehan static int cap_pause_exit;
16806fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, pause_exit, CTLFLAG_RD, &cap_pause_exit,
16906fc6db9SJohn Baldwin     0, "PAUSE triggers a VM-exit");
17006fc6db9SJohn Baldwin 
171*3ba952e1SCorvin Köhne static int cap_wbinvd_exit;
172*3ba952e1SCorvin Köhne SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, wbinvd_exit, CTLFLAG_RD, &cap_wbinvd_exit,
173*3ba952e1SCorvin Köhne     0, "WBINVD triggers a VM-exit");
174*3ba952e1SCorvin Köhne 
175f5f5f1e7SPeter Grehan static int cap_rdpid;
176f5f5f1e7SPeter Grehan SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, rdpid, CTLFLAG_RD, &cap_rdpid, 0,
177f5f5f1e7SPeter Grehan     "Guests are allowed to use RDPID");
178f5f5f1e7SPeter Grehan 
179f5f5f1e7SPeter Grehan static int cap_rdtscp;
180f5f5f1e7SPeter Grehan SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, rdtscp, CTLFLAG_RD, &cap_rdtscp, 0,
181f5f5f1e7SPeter Grehan     "Guests are allowed to use RDTSCP");
182f5f5f1e7SPeter Grehan 
183366f6083SPeter Grehan static int cap_unrestricted_guest;
18406fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, unrestricted_guest, CTLFLAG_RD,
18506fc6db9SJohn Baldwin     &cap_unrestricted_guest, 0, "Unrestricted guests");
18606fc6db9SJohn Baldwin 
187366f6083SPeter Grehan static int cap_monitor_trap;
18806fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, monitor_trap, CTLFLAG_RD,
18906fc6db9SJohn Baldwin     &cap_monitor_trap, 0, "Monitor trap flag");
19006fc6db9SJohn Baldwin 
19149cc03daSNeel Natu static int cap_invpcid;
19206fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, invpcid, CTLFLAG_RD, &cap_invpcid,
19306fc6db9SJohn Baldwin     0, "Guests are allowed to use INVPCID");
194366f6083SPeter Grehan 
1951bc51badSMichael Reifenberger static int tpr_shadowing;
1961bc51badSMichael Reifenberger SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, tpr_shadowing, CTLFLAG_RD,
1971bc51badSMichael Reifenberger     &tpr_shadowing, 0, "TPR shadowing support");
1981bc51badSMichael Reifenberger 
19988c4b8d1SNeel Natu static int virtual_interrupt_delivery;
20006fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, virtual_interrupt_delivery, CTLFLAG_RD,
20188c4b8d1SNeel Natu     &virtual_interrupt_delivery, 0, "APICv virtual interrupt delivery support");
20288c4b8d1SNeel Natu 
203176666c2SNeel Natu static int posted_interrupts;
20406fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, posted_interrupts, CTLFLAG_RD,
205176666c2SNeel Natu     &posted_interrupts, 0, "APICv posted interrupt support");
206176666c2SNeel Natu 
20718a2b08eSNeel Natu static int pirvec = -1;
208176666c2SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupt_vector, CTLFLAG_RD,
209176666c2SNeel Natu     &pirvec, 0, "APICv posted interrupt vector");
210176666c2SNeel Natu 
21145e51299SNeel Natu static struct unrhdr *vpid_unr;
21245e51299SNeel Natu static u_int vpid_alloc_failed;
21345e51299SNeel Natu SYSCTL_UINT(_hw_vmm_vmx, OID_AUTO, vpid_alloc_failed, CTLFLAG_RD,
21445e51299SNeel Natu 	    &vpid_alloc_failed, 0, NULL);
21545e51299SNeel Natu 
216d3588766SMark Johnston int guest_l1d_flush;
217c30578feSKonstantin Belousov SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, l1d_flush, CTLFLAG_RD,
218c30578feSKonstantin Belousov     &guest_l1d_flush, 0, NULL);
219d3588766SMark Johnston int guest_l1d_flush_sw;
220c1141fbaSKonstantin Belousov SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, l1d_flush_sw, CTLFLAG_RD,
221c1141fbaSKonstantin Belousov     &guest_l1d_flush_sw, 0, NULL);
222c30578feSKonstantin Belousov 
223c1141fbaSKonstantin Belousov static struct msr_entry msr_load_list[1] __aligned(16);
224c30578feSKonstantin Belousov 
22588c4b8d1SNeel Natu /*
2266ac73777STycho Nightingale  * The definitions of SDT probes for VMX.
2276ac73777STycho Nightingale  */
2286ac73777STycho Nightingale 
2296ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, entry,
2306ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2316ac73777STycho Nightingale 
2326ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, taskswitch,
2336ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "struct vm_task_switch *");
2346ac73777STycho Nightingale 
2356ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, craccess,
2366ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint64_t");
2376ac73777STycho Nightingale 
2386ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, rdmsr,
2396ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint32_t");
2406ac73777STycho Nightingale 
2416ac73777STycho Nightingale SDT_PROBE_DEFINE5(vmm, vmx, exit, wrmsr,
2426ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint32_t", "uint64_t");
2436ac73777STycho Nightingale 
2446ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, halt,
2456ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2466ac73777STycho Nightingale 
2476ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, mtrap,
2486ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2496ac73777STycho Nightingale 
2506ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, pause,
2516ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2526ac73777STycho Nightingale 
2536ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, intrwindow,
2546ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2556ac73777STycho Nightingale 
2566ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, interrupt,
2576ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint32_t");
2586ac73777STycho Nightingale 
2596ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, nmiwindow,
2606ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2616ac73777STycho Nightingale 
2626ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, inout,
2636ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2646ac73777STycho Nightingale 
2656ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, cpuid,
2666ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2676ac73777STycho Nightingale 
2686ac73777STycho Nightingale SDT_PROBE_DEFINE5(vmm, vmx, exit, exception,
2696ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint32_t", "int");
2706ac73777STycho Nightingale 
2716ac73777STycho Nightingale SDT_PROBE_DEFINE5(vmm, vmx, exit, nestedfault,
2726ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint64_t", "uint64_t");
2736ac73777STycho Nightingale 
2746ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, mmiofault,
2756ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint64_t");
2766ac73777STycho Nightingale 
2776ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, eoi,
2786ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2796ac73777STycho Nightingale 
2806ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, apicaccess,
2816ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2826ac73777STycho Nightingale 
2836ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, apicwrite,
2846ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "struct vlapic *");
2856ac73777STycho Nightingale 
2866ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, xsetbv,
2876ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2886ac73777STycho Nightingale 
2896ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, monitor,
2906ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2916ac73777STycho Nightingale 
2926ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, mwait,
2936ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *");
2946ac73777STycho Nightingale 
29527d26457SAndrew Turner SDT_PROBE_DEFINE3(vmm, vmx, exit, vminsn,
29627d26457SAndrew Turner     "struct vmx *", "int", "struct vm_exit *");
29727d26457SAndrew Turner 
2986ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, unknown,
2996ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "uint32_t");
3006ac73777STycho Nightingale 
3016ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, return,
3026ac73777STycho Nightingale     "struct vmx *", "int", "struct vm_exit *", "int");
3036ac73777STycho Nightingale 
3046ac73777STycho Nightingale /*
30588c4b8d1SNeel Natu  * Use the last page below 4GB as the APIC access address. This address is
30688c4b8d1SNeel Natu  * occupied by the boot firmware so it is guaranteed that it will not conflict
30788c4b8d1SNeel Natu  * with a page in system memory.
30888c4b8d1SNeel Natu  */
30988c4b8d1SNeel Natu #define	APIC_ACCESS_ADDRESS	0xFFFFF000
31088c4b8d1SNeel Natu 
311d17b5104SNeel Natu static int vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc);
312d17b5104SNeel Natu static int vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval);
313c3498942SNeel Natu static int vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val);
31488c4b8d1SNeel Natu static void vmx_inject_pir(struct vlapic *vlapic);
315483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT
316483d953aSJohn Baldwin static int vmx_restore_tsc(void *arg, int vcpu, uint64_t now);
317483d953aSJohn Baldwin #endif
31888c4b8d1SNeel Natu 
319f5f5f1e7SPeter Grehan static inline bool
320f5f5f1e7SPeter Grehan host_has_rdpid(void)
321f5f5f1e7SPeter Grehan {
322f5f5f1e7SPeter Grehan 	return ((cpu_stdext_feature2 & CPUID_STDEXT2_RDPID) != 0);
323f5f5f1e7SPeter Grehan }
324f5f5f1e7SPeter Grehan 
325f5f5f1e7SPeter Grehan static inline bool
326f5f5f1e7SPeter Grehan host_has_rdtscp(void)
327f5f5f1e7SPeter Grehan {
328f5f5f1e7SPeter Grehan 	return ((amd_feature & AMDID_RDTSCP) != 0);
329f5f5f1e7SPeter Grehan }
330f5f5f1e7SPeter Grehan 
331366f6083SPeter Grehan #ifdef KTR
332366f6083SPeter Grehan static const char *
333366f6083SPeter Grehan exit_reason_to_str(int reason)
334366f6083SPeter Grehan {
335366f6083SPeter Grehan 	static char reasonbuf[32];
336366f6083SPeter Grehan 
337366f6083SPeter Grehan 	switch (reason) {
338366f6083SPeter Grehan 	case EXIT_REASON_EXCEPTION:
339366f6083SPeter Grehan 		return "exception";
340366f6083SPeter Grehan 	case EXIT_REASON_EXT_INTR:
341366f6083SPeter Grehan 		return "extint";
342366f6083SPeter Grehan 	case EXIT_REASON_TRIPLE_FAULT:
343366f6083SPeter Grehan 		return "triplefault";
344366f6083SPeter Grehan 	case EXIT_REASON_INIT:
345366f6083SPeter Grehan 		return "init";
346366f6083SPeter Grehan 	case EXIT_REASON_SIPI:
347366f6083SPeter Grehan 		return "sipi";
348366f6083SPeter Grehan 	case EXIT_REASON_IO_SMI:
349366f6083SPeter Grehan 		return "iosmi";
350366f6083SPeter Grehan 	case EXIT_REASON_SMI:
351366f6083SPeter Grehan 		return "smi";
352366f6083SPeter Grehan 	case EXIT_REASON_INTR_WINDOW:
353366f6083SPeter Grehan 		return "intrwindow";
354366f6083SPeter Grehan 	case EXIT_REASON_NMI_WINDOW:
355366f6083SPeter Grehan 		return "nmiwindow";
356366f6083SPeter Grehan 	case EXIT_REASON_TASK_SWITCH:
357366f6083SPeter Grehan 		return "taskswitch";
358366f6083SPeter Grehan 	case EXIT_REASON_CPUID:
359366f6083SPeter Grehan 		return "cpuid";
360366f6083SPeter Grehan 	case EXIT_REASON_GETSEC:
361366f6083SPeter Grehan 		return "getsec";
362366f6083SPeter Grehan 	case EXIT_REASON_HLT:
363366f6083SPeter Grehan 		return "hlt";
364366f6083SPeter Grehan 	case EXIT_REASON_INVD:
365366f6083SPeter Grehan 		return "invd";
366366f6083SPeter Grehan 	case EXIT_REASON_INVLPG:
367366f6083SPeter Grehan 		return "invlpg";
368366f6083SPeter Grehan 	case EXIT_REASON_RDPMC:
369366f6083SPeter Grehan 		return "rdpmc";
370366f6083SPeter Grehan 	case EXIT_REASON_RDTSC:
371366f6083SPeter Grehan 		return "rdtsc";
372366f6083SPeter Grehan 	case EXIT_REASON_RSM:
373366f6083SPeter Grehan 		return "rsm";
374366f6083SPeter Grehan 	case EXIT_REASON_VMCALL:
375366f6083SPeter Grehan 		return "vmcall";
376366f6083SPeter Grehan 	case EXIT_REASON_VMCLEAR:
377366f6083SPeter Grehan 		return "vmclear";
378366f6083SPeter Grehan 	case EXIT_REASON_VMLAUNCH:
379366f6083SPeter Grehan 		return "vmlaunch";
380366f6083SPeter Grehan 	case EXIT_REASON_VMPTRLD:
381366f6083SPeter Grehan 		return "vmptrld";
382366f6083SPeter Grehan 	case EXIT_REASON_VMPTRST:
383366f6083SPeter Grehan 		return "vmptrst";
384366f6083SPeter Grehan 	case EXIT_REASON_VMREAD:
385366f6083SPeter Grehan 		return "vmread";
386366f6083SPeter Grehan 	case EXIT_REASON_VMRESUME:
387366f6083SPeter Grehan 		return "vmresume";
388366f6083SPeter Grehan 	case EXIT_REASON_VMWRITE:
389366f6083SPeter Grehan 		return "vmwrite";
390366f6083SPeter Grehan 	case EXIT_REASON_VMXOFF:
391366f6083SPeter Grehan 		return "vmxoff";
392366f6083SPeter Grehan 	case EXIT_REASON_VMXON:
393366f6083SPeter Grehan 		return "vmxon";
394366f6083SPeter Grehan 	case EXIT_REASON_CR_ACCESS:
395366f6083SPeter Grehan 		return "craccess";
396366f6083SPeter Grehan 	case EXIT_REASON_DR_ACCESS:
397366f6083SPeter Grehan 		return "draccess";
398366f6083SPeter Grehan 	case EXIT_REASON_INOUT:
399366f6083SPeter Grehan 		return "inout";
400366f6083SPeter Grehan 	case EXIT_REASON_RDMSR:
401366f6083SPeter Grehan 		return "rdmsr";
402366f6083SPeter Grehan 	case EXIT_REASON_WRMSR:
403366f6083SPeter Grehan 		return "wrmsr";
404366f6083SPeter Grehan 	case EXIT_REASON_INVAL_VMCS:
405366f6083SPeter Grehan 		return "invalvmcs";
406366f6083SPeter Grehan 	case EXIT_REASON_INVAL_MSR:
407366f6083SPeter Grehan 		return "invalmsr";
408366f6083SPeter Grehan 	case EXIT_REASON_MWAIT:
409366f6083SPeter Grehan 		return "mwait";
410366f6083SPeter Grehan 	case EXIT_REASON_MTF:
411366f6083SPeter Grehan 		return "mtf";
412366f6083SPeter Grehan 	case EXIT_REASON_MONITOR:
413366f6083SPeter Grehan 		return "monitor";
414366f6083SPeter Grehan 	case EXIT_REASON_PAUSE:
415366f6083SPeter Grehan 		return "pause";
416b0538143SNeel Natu 	case EXIT_REASON_MCE_DURING_ENTRY:
417b0538143SNeel Natu 		return "mce-during-entry";
418366f6083SPeter Grehan 	case EXIT_REASON_TPR:
419366f6083SPeter Grehan 		return "tpr";
42088c4b8d1SNeel Natu 	case EXIT_REASON_APIC_ACCESS:
42188c4b8d1SNeel Natu 		return "apic-access";
422366f6083SPeter Grehan 	case EXIT_REASON_GDTR_IDTR:
423366f6083SPeter Grehan 		return "gdtridtr";
424366f6083SPeter Grehan 	case EXIT_REASON_LDTR_TR:
425366f6083SPeter Grehan 		return "ldtrtr";
426366f6083SPeter Grehan 	case EXIT_REASON_EPT_FAULT:
427366f6083SPeter Grehan 		return "eptfault";
428366f6083SPeter Grehan 	case EXIT_REASON_EPT_MISCONFIG:
429366f6083SPeter Grehan 		return "eptmisconfig";
430366f6083SPeter Grehan 	case EXIT_REASON_INVEPT:
431366f6083SPeter Grehan 		return "invept";
432366f6083SPeter Grehan 	case EXIT_REASON_RDTSCP:
433366f6083SPeter Grehan 		return "rdtscp";
434366f6083SPeter Grehan 	case EXIT_REASON_VMX_PREEMPT:
435366f6083SPeter Grehan 		return "vmxpreempt";
436366f6083SPeter Grehan 	case EXIT_REASON_INVVPID:
437366f6083SPeter Grehan 		return "invvpid";
438366f6083SPeter Grehan 	case EXIT_REASON_WBINVD:
439366f6083SPeter Grehan 		return "wbinvd";
440366f6083SPeter Grehan 	case EXIT_REASON_XSETBV:
441366f6083SPeter Grehan 		return "xsetbv";
44288c4b8d1SNeel Natu 	case EXIT_REASON_APIC_WRITE:
44388c4b8d1SNeel Natu 		return "apic-write";
444366f6083SPeter Grehan 	default:
445366f6083SPeter Grehan 		snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason);
446366f6083SPeter Grehan 		return (reasonbuf);
447366f6083SPeter Grehan 	}
448366f6083SPeter Grehan }
449366f6083SPeter Grehan #endif	/* KTR */
450366f6083SPeter Grehan 
451159dd56fSNeel Natu static int
452159dd56fSNeel Natu vmx_allow_x2apic_msrs(struct vmx *vmx)
453159dd56fSNeel Natu {
454159dd56fSNeel Natu 	int i, error;
455159dd56fSNeel Natu 
456159dd56fSNeel Natu 	error = 0;
457159dd56fSNeel Natu 
458159dd56fSNeel Natu 	/*
459159dd56fSNeel Natu 	 * Allow readonly access to the following x2APIC MSRs from the guest.
460159dd56fSNeel Natu 	 */
461159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ID);
462159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_VERSION);
463159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LDR);
464159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_SVR);
465159dd56fSNeel Natu 
466159dd56fSNeel Natu 	for (i = 0; i < 8; i++)
467159dd56fSNeel Natu 		error += guest_msr_ro(vmx, MSR_APIC_ISR0 + i);
468159dd56fSNeel Natu 
469159dd56fSNeel Natu 	for (i = 0; i < 8; i++)
470159dd56fSNeel Natu 		error += guest_msr_ro(vmx, MSR_APIC_TMR0 + i);
471159dd56fSNeel Natu 
472159dd56fSNeel Natu 	for (i = 0; i < 8; i++)
473159dd56fSNeel Natu 		error += guest_msr_ro(vmx, MSR_APIC_IRR0 + i);
474159dd56fSNeel Natu 
475159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ESR);
476159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_TIMER);
477159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_THERMAL);
478159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_PCINT);
479159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT0);
480159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT1);
481159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_LVT_ERROR);
482159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ICR_TIMER);
483159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_DCR_TIMER);
484159dd56fSNeel Natu 	error += guest_msr_ro(vmx, MSR_APIC_ICR);
485159dd56fSNeel Natu 
486159dd56fSNeel Natu 	/*
487159dd56fSNeel Natu 	 * Allow TPR, EOI and SELF_IPI MSRs to be read and written by the guest.
488159dd56fSNeel Natu 	 *
489159dd56fSNeel Natu 	 * These registers get special treatment described in the section
490159dd56fSNeel Natu 	 * "Virtualizing MSR-Based APIC Accesses".
491159dd56fSNeel Natu 	 */
492159dd56fSNeel Natu 	error += guest_msr_rw(vmx, MSR_APIC_TPR);
493159dd56fSNeel Natu 	error += guest_msr_rw(vmx, MSR_APIC_EOI);
494159dd56fSNeel Natu 	error += guest_msr_rw(vmx, MSR_APIC_SELF_IPI);
495159dd56fSNeel Natu 
496159dd56fSNeel Natu 	return (error);
497159dd56fSNeel Natu }
498159dd56fSNeel Natu 
499366f6083SPeter Grehan u_long
500366f6083SPeter Grehan vmx_fix_cr0(u_long cr0)
501366f6083SPeter Grehan {
502366f6083SPeter Grehan 
503366f6083SPeter Grehan 	return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask);
504366f6083SPeter Grehan }
505366f6083SPeter Grehan 
506366f6083SPeter Grehan u_long
507366f6083SPeter Grehan vmx_fix_cr4(u_long cr4)
508366f6083SPeter Grehan {
509366f6083SPeter Grehan 
510366f6083SPeter Grehan 	return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask);
511366f6083SPeter Grehan }
512366f6083SPeter Grehan 
513366f6083SPeter Grehan static void
51445e51299SNeel Natu vpid_free(int vpid)
51545e51299SNeel Natu {
51645e51299SNeel Natu 	if (vpid < 0 || vpid > 0xffff)
51745e51299SNeel Natu 		panic("vpid_free: invalid vpid %d", vpid);
51845e51299SNeel Natu 
51945e51299SNeel Natu 	/*
52045e51299SNeel Natu 	 * VPIDs [0,VM_MAXCPU] are special and are not allocated from
52145e51299SNeel Natu 	 * the unit number allocator.
52245e51299SNeel Natu 	 */
52345e51299SNeel Natu 
52445e51299SNeel Natu 	if (vpid > VM_MAXCPU)
52545e51299SNeel Natu 		free_unr(vpid_unr, vpid);
52645e51299SNeel Natu }
52745e51299SNeel Natu 
52845e51299SNeel Natu static void
52945e51299SNeel Natu vpid_alloc(uint16_t *vpid, int num)
53045e51299SNeel Natu {
53145e51299SNeel Natu 	int i, x;
53245e51299SNeel Natu 
53345e51299SNeel Natu 	if (num <= 0 || num > VM_MAXCPU)
53445e51299SNeel Natu 		panic("invalid number of vpids requested: %d", num);
53545e51299SNeel Natu 
53645e51299SNeel Natu 	/*
53745e51299SNeel Natu 	 * If the "enable vpid" execution control is not enabled then the
53845e51299SNeel Natu 	 * VPID is required to be 0 for all vcpus.
53945e51299SNeel Natu 	 */
54045e51299SNeel Natu 	if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) == 0) {
54145e51299SNeel Natu 		for (i = 0; i < num; i++)
54245e51299SNeel Natu 			vpid[i] = 0;
54345e51299SNeel Natu 		return;
54445e51299SNeel Natu 	}
54545e51299SNeel Natu 
54645e51299SNeel Natu 	/*
54745e51299SNeel Natu 	 * Allocate a unique VPID for each vcpu from the unit number allocator.
54845e51299SNeel Natu 	 */
54945e51299SNeel Natu 	for (i = 0; i < num; i++) {
55045e51299SNeel Natu 		x = alloc_unr(vpid_unr);
55145e51299SNeel Natu 		if (x == -1)
55245e51299SNeel Natu 			break;
55345e51299SNeel Natu 		else
55445e51299SNeel Natu 			vpid[i] = x;
55545e51299SNeel Natu 	}
55645e51299SNeel Natu 
55745e51299SNeel Natu 	if (i < num) {
55845e51299SNeel Natu 		atomic_add_int(&vpid_alloc_failed, 1);
55945e51299SNeel Natu 
56045e51299SNeel Natu 		/*
56145e51299SNeel Natu 		 * If the unit number allocator does not have enough unique
56245e51299SNeel Natu 		 * VPIDs then we need to allocate from the [1,VM_MAXCPU] range.
56345e51299SNeel Natu 		 *
56445e51299SNeel Natu 		 * These VPIDs are not be unique across VMs but this does not
56545e51299SNeel Natu 		 * affect correctness because the combined mappings are also
56645e51299SNeel Natu 		 * tagged with the EP4TA which is unique for each VM.
56745e51299SNeel Natu 		 *
56845e51299SNeel Natu 		 * It is still sub-optimal because the invvpid will invalidate
56945e51299SNeel Natu 		 * combined mappings for a particular VPID across all EP4TAs.
57045e51299SNeel Natu 		 */
57145e51299SNeel Natu 		while (i-- > 0)
57245e51299SNeel Natu 			vpid_free(vpid[i]);
57345e51299SNeel Natu 
57445e51299SNeel Natu 		for (i = 0; i < num; i++)
57545e51299SNeel Natu 			vpid[i] = i + 1;
57645e51299SNeel Natu 	}
57745e51299SNeel Natu }
57845e51299SNeel Natu 
57945e51299SNeel Natu static void
58045e51299SNeel Natu vpid_init(void)
58145e51299SNeel Natu {
58245e51299SNeel Natu 	/*
58345e51299SNeel Natu 	 * VPID 0 is required when the "enable VPID" execution control is
58445e51299SNeel Natu 	 * disabled.
58545e51299SNeel Natu 	 *
58645e51299SNeel Natu 	 * VPIDs [1,VM_MAXCPU] are used as the "overflow namespace" when the
58745e51299SNeel Natu 	 * unit number allocator does not have sufficient unique VPIDs to
58845e51299SNeel Natu 	 * satisfy the allocation.
58945e51299SNeel Natu 	 *
59045e51299SNeel Natu 	 * The remaining VPIDs are managed by the unit number allocator.
59145e51299SNeel Natu 	 */
59245e51299SNeel Natu 	vpid_unr = new_unrhdr(VM_MAXCPU + 1, 0xffff, NULL);
59345e51299SNeel Natu }
59445e51299SNeel Natu 
59545e51299SNeel Natu static void
596366f6083SPeter Grehan vmx_disable(void *arg __unused)
597366f6083SPeter Grehan {
598366f6083SPeter Grehan 	struct invvpid_desc invvpid_desc = { 0 };
599366f6083SPeter Grehan 	struct invept_desc invept_desc = { 0 };
600366f6083SPeter Grehan 
601366f6083SPeter Grehan 	if (vmxon_enabled[curcpu]) {
602366f6083SPeter Grehan 		/*
603366f6083SPeter Grehan 		 * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b.
604366f6083SPeter Grehan 		 *
605366f6083SPeter Grehan 		 * VMXON or VMXOFF are not required to invalidate any TLB
606366f6083SPeter Grehan 		 * caching structures. This prevents potential retention of
607366f6083SPeter Grehan 		 * cached information in the TLB between distinct VMX episodes.
608366f6083SPeter Grehan 		 */
609366f6083SPeter Grehan 		invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc);
610366f6083SPeter Grehan 		invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc);
611366f6083SPeter Grehan 		vmxoff();
612366f6083SPeter Grehan 	}
613366f6083SPeter Grehan 	load_cr4(rcr4() & ~CR4_VMXE);
614366f6083SPeter Grehan }
615366f6083SPeter Grehan 
616366f6083SPeter Grehan static int
61715add60dSPeter Grehan vmx_modcleanup(void)
618366f6083SPeter Grehan {
619366f6083SPeter Grehan 
62018a2b08eSNeel Natu 	if (pirvec >= 0)
62118a2b08eSNeel Natu 		lapic_ipi_free(pirvec);
622176666c2SNeel Natu 
62345e51299SNeel Natu 	if (vpid_unr != NULL) {
62445e51299SNeel Natu 		delete_unrhdr(vpid_unr);
62545e51299SNeel Natu 		vpid_unr = NULL;
62645e51299SNeel Natu 	}
62745e51299SNeel Natu 
628c1141fbaSKonstantin Belousov 	if (nmi_flush_l1d_sw == 1)
629c1141fbaSKonstantin Belousov 		nmi_flush_l1d_sw = 0;
630c1141fbaSKonstantin Belousov 
631366f6083SPeter Grehan 	smp_rendezvous(NULL, vmx_disable, NULL, NULL);
632366f6083SPeter Grehan 
633366f6083SPeter Grehan 	return (0);
634366f6083SPeter Grehan }
635366f6083SPeter Grehan 
636366f6083SPeter Grehan static void
637366f6083SPeter Grehan vmx_enable(void *arg __unused)
638366f6083SPeter Grehan {
639366f6083SPeter Grehan 	int error;
64011669a68STycho Nightingale 	uint64_t feature_control;
64111669a68STycho Nightingale 
64211669a68STycho Nightingale 	feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
64311669a68STycho Nightingale 	if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 0 ||
64411669a68STycho Nightingale 	    (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) {
64511669a68STycho Nightingale 		wrmsr(MSR_IA32_FEATURE_CONTROL,
64611669a68STycho Nightingale 		    feature_control | IA32_FEATURE_CONTROL_VMX_EN |
64711669a68STycho Nightingale 		    IA32_FEATURE_CONTROL_LOCK);
64811669a68STycho Nightingale 	}
649366f6083SPeter Grehan 
650366f6083SPeter Grehan 	load_cr4(rcr4() | CR4_VMXE);
651366f6083SPeter Grehan 
652366f6083SPeter Grehan 	*(uint32_t *)vmxon_region[curcpu] = vmx_revision();
653366f6083SPeter Grehan 	error = vmxon(vmxon_region[curcpu]);
654366f6083SPeter Grehan 	if (error == 0)
655366f6083SPeter Grehan 		vmxon_enabled[curcpu] = 1;
656366f6083SPeter Grehan }
657366f6083SPeter Grehan 
65863e62d39SJohn Baldwin static void
65915add60dSPeter Grehan vmx_modresume(void)
66063e62d39SJohn Baldwin {
66163e62d39SJohn Baldwin 
66263e62d39SJohn Baldwin 	if (vmxon_enabled[curcpu])
66363e62d39SJohn Baldwin 		vmxon(vmxon_region[curcpu]);
66463e62d39SJohn Baldwin }
66563e62d39SJohn Baldwin 
666366f6083SPeter Grehan static int
66715add60dSPeter Grehan vmx_modinit(int ipinum)
668366f6083SPeter Grehan {
6691bc51badSMichael Reifenberger 	int error;
670d17b5104SNeel Natu 	uint64_t basic, fixed0, fixed1, feature_control;
67188c4b8d1SNeel Natu 	uint32_t tmp, procbased2_vid_bits;
672366f6083SPeter Grehan 
673366f6083SPeter Grehan 	/* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */
6748b287612SJohn Baldwin 	if (!(cpu_feature2 & CPUID2_VMX)) {
67515add60dSPeter Grehan 		printf("vmx_modinit: processor does not support VMX "
67615add60dSPeter Grehan 		    "operation\n");
677366f6083SPeter Grehan 		return (ENXIO);
678366f6083SPeter Grehan 	}
679366f6083SPeter Grehan 
6804bff7fadSNeel Natu 	/*
6814bff7fadSNeel Natu 	 * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits
6824bff7fadSNeel Natu 	 * are set (bits 0 and 2 respectively).
6834bff7fadSNeel Natu 	 */
6844bff7fadSNeel Natu 	feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
68511669a68STycho Nightingale 	if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 1 &&
686150369abSNeel Natu 	    (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) {
68715add60dSPeter Grehan 		printf("vmx_modinit: VMX operation disabled by BIOS\n");
6884bff7fadSNeel Natu 		return (ENXIO);
6894bff7fadSNeel Natu 	}
6904bff7fadSNeel Natu 
691d17b5104SNeel Natu 	/*
692d17b5104SNeel Natu 	 * Verify capabilities MSR_VMX_BASIC:
693d17b5104SNeel Natu 	 * - bit 54 indicates support for INS/OUTS decoding
694d17b5104SNeel Natu 	 */
695d17b5104SNeel Natu 	basic = rdmsr(MSR_VMX_BASIC);
696d17b5104SNeel Natu 	if ((basic & (1UL << 54)) == 0) {
69715add60dSPeter Grehan 		printf("vmx_modinit: processor does not support desired basic "
698d17b5104SNeel Natu 		    "capabilities\n");
699d17b5104SNeel Natu 		return (EINVAL);
700d17b5104SNeel Natu 	}
701d17b5104SNeel Natu 
702366f6083SPeter Grehan 	/* Check support for primary processor-based VM-execution controls */
703366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
704366f6083SPeter Grehan 			       MSR_VMX_TRUE_PROCBASED_CTLS,
705366f6083SPeter Grehan 			       PROCBASED_CTLS_ONE_SETTING,
706366f6083SPeter Grehan 			       PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls);
707366f6083SPeter Grehan 	if (error) {
70815add60dSPeter Grehan 		printf("vmx_modinit: processor does not support desired "
70915add60dSPeter Grehan 		    "primary processor-based controls\n");
710366f6083SPeter Grehan 		return (error);
711366f6083SPeter Grehan 	}
712366f6083SPeter Grehan 
713366f6083SPeter Grehan 	/* Clear the processor-based ctl bits that are set on demand */
714366f6083SPeter Grehan 	procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING;
715366f6083SPeter Grehan 
716366f6083SPeter Grehan 	/* Check support for secondary processor-based VM-execution controls */
717366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
718366f6083SPeter Grehan 			       MSR_VMX_PROCBASED_CTLS2,
719366f6083SPeter Grehan 			       PROCBASED_CTLS2_ONE_SETTING,
720366f6083SPeter Grehan 			       PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2);
721366f6083SPeter Grehan 	if (error) {
72215add60dSPeter Grehan 		printf("vmx_modinit: processor does not support desired "
72315add60dSPeter Grehan 		    "secondary processor-based controls\n");
724366f6083SPeter Grehan 		return (error);
725366f6083SPeter Grehan 	}
726366f6083SPeter Grehan 
727366f6083SPeter Grehan 	/* Check support for VPID */
728366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
729366f6083SPeter Grehan 			       PROCBASED2_ENABLE_VPID, 0, &tmp);
730366f6083SPeter Grehan 	if (error == 0)
731366f6083SPeter Grehan 		procbased_ctls2 |= PROCBASED2_ENABLE_VPID;
732366f6083SPeter Grehan 
733366f6083SPeter Grehan 	/* Check support for pin-based VM-execution controls */
734366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
735366f6083SPeter Grehan 			       MSR_VMX_TRUE_PINBASED_CTLS,
736366f6083SPeter Grehan 			       PINBASED_CTLS_ONE_SETTING,
737366f6083SPeter Grehan 			       PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls);
738366f6083SPeter Grehan 	if (error) {
73915add60dSPeter Grehan 		printf("vmx_modinit: processor does not support desired "
740366f6083SPeter Grehan 		    "pin-based controls\n");
741366f6083SPeter Grehan 		return (error);
742366f6083SPeter Grehan 	}
743366f6083SPeter Grehan 
744366f6083SPeter Grehan 	/* Check support for VM-exit controls */
745366f6083SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS,
746366f6083SPeter Grehan 			       VM_EXIT_CTLS_ONE_SETTING,
747366f6083SPeter Grehan 			       VM_EXIT_CTLS_ZERO_SETTING,
748366f6083SPeter Grehan 			       &exit_ctls);
749366f6083SPeter Grehan 	if (error) {
75015add60dSPeter Grehan 		printf("vmx_modinit: processor does not support desired "
751366f6083SPeter Grehan 		    "exit controls\n");
752366f6083SPeter Grehan 		return (error);
753366f6083SPeter Grehan 	}
754366f6083SPeter Grehan 
755366f6083SPeter Grehan 	/* Check support for VM-entry controls */
756d72978ecSNeel Natu 	error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS, MSR_VMX_TRUE_ENTRY_CTLS,
757d72978ecSNeel Natu 	    VM_ENTRY_CTLS_ONE_SETTING, VM_ENTRY_CTLS_ZERO_SETTING,
758366f6083SPeter Grehan 	    &entry_ctls);
759366f6083SPeter Grehan 	if (error) {
76015add60dSPeter Grehan 		printf("vmx_modinit: processor does not support desired "
761366f6083SPeter Grehan 		    "entry controls\n");
762366f6083SPeter Grehan 		return (error);
763366f6083SPeter Grehan 	}
764366f6083SPeter Grehan 
765366f6083SPeter Grehan 	/*
766366f6083SPeter Grehan 	 * Check support for optional features by testing them
767366f6083SPeter Grehan 	 * as individual bits
768366f6083SPeter Grehan 	 */
769366f6083SPeter Grehan 	cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
770366f6083SPeter Grehan 					MSR_VMX_TRUE_PROCBASED_CTLS,
771366f6083SPeter Grehan 					PROCBASED_HLT_EXITING, 0,
772366f6083SPeter Grehan 					&tmp) == 0);
773366f6083SPeter Grehan 
774366f6083SPeter Grehan 	cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
775366f6083SPeter Grehan 					MSR_VMX_PROCBASED_CTLS,
776366f6083SPeter Grehan 					PROCBASED_MTF, 0,
777366f6083SPeter Grehan 					&tmp) == 0);
778366f6083SPeter Grehan 
779366f6083SPeter Grehan 	cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
780366f6083SPeter Grehan 					 MSR_VMX_TRUE_PROCBASED_CTLS,
781366f6083SPeter Grehan 					 PROCBASED_PAUSE_EXITING, 0,
782366f6083SPeter Grehan 					 &tmp) == 0);
783366f6083SPeter Grehan 
784*3ba952e1SCorvin Köhne 	cap_wbinvd_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
785*3ba952e1SCorvin Köhne 					MSR_VMX_PROCBASED_CTLS2,
786*3ba952e1SCorvin Köhne 					PROCBASED2_WBINVD_EXITING,
787*3ba952e1SCorvin Köhne 					0,
788*3ba952e1SCorvin Köhne 					&tmp) == 0);
789*3ba952e1SCorvin Köhne 
790f5f5f1e7SPeter Grehan 	/*
791f5f5f1e7SPeter Grehan 	 * Check support for RDPID and/or RDTSCP.
792f5f5f1e7SPeter Grehan 	 *
793f5f5f1e7SPeter Grehan 	 * Support a pass-through-based implementation of these via the
794f5f5f1e7SPeter Grehan 	 * "enable RDTSCP" VM-execution control and the "RDTSC exiting"
795f5f5f1e7SPeter Grehan 	 * VM-execution control.
796f5f5f1e7SPeter Grehan 	 *
797f5f5f1e7SPeter Grehan 	 * The "enable RDTSCP" VM-execution control applies to both RDPID
798f5f5f1e7SPeter Grehan 	 * and RDTSCP (see SDM volume 3, section 25.3, "Changes to
799f5f5f1e7SPeter Grehan 	 * Instruction Behavior in VMX Non-root operation"); this is why
800f5f5f1e7SPeter Grehan 	 * only this VM-execution control needs to be enabled in order to
801f5f5f1e7SPeter Grehan 	 * enable passing through whichever of RDPID and/or RDTSCP are
802f5f5f1e7SPeter Grehan 	 * supported by the host.
803f5f5f1e7SPeter Grehan 	 *
804f5f5f1e7SPeter Grehan 	 * The "RDTSC exiting" VM-execution control applies to both RDTSC
805f5f5f1e7SPeter Grehan 	 * and RDTSCP (again, per SDM volume 3, section 25.3), and is
806f5f5f1e7SPeter Grehan 	 * already set up for RDTSC and RDTSCP pass-through by the current
807f5f5f1e7SPeter Grehan 	 * implementation of RDTSC.
808f5f5f1e7SPeter Grehan 	 *
809f5f5f1e7SPeter Grehan 	 * Although RDPID and RDTSCP are optional capabilities, since there
810f5f5f1e7SPeter Grehan 	 * does not currently seem to be a use case for enabling/disabling
811f5f5f1e7SPeter Grehan 	 * these via libvmmapi, choose not to support this and, instead,
812f5f5f1e7SPeter Grehan 	 * just statically always enable or always disable this support
813f5f5f1e7SPeter Grehan 	 * across all vCPUs on all VMs. (Note that there may be some
814f5f5f1e7SPeter Grehan 	 * complications to providing this functionality, e.g., the MSR
815f5f5f1e7SPeter Grehan 	 * bitmap is currently per-VM rather than per-vCPU while the
816f5f5f1e7SPeter Grehan 	 * capability API wants to be able to control capabilities on a
817f5f5f1e7SPeter Grehan 	 * per-vCPU basis).
818f5f5f1e7SPeter Grehan 	 */
819f5f5f1e7SPeter Grehan 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
820f5f5f1e7SPeter Grehan 			       MSR_VMX_PROCBASED_CTLS2,
821f5f5f1e7SPeter Grehan 			       PROCBASED2_ENABLE_RDTSCP, 0, &tmp);
822f5f5f1e7SPeter Grehan 	cap_rdpid = error == 0 && host_has_rdpid();
823f5f5f1e7SPeter Grehan 	cap_rdtscp = error == 0 && host_has_rdtscp();
824f5f5f1e7SPeter Grehan 	if (cap_rdpid || cap_rdtscp)
825f5f5f1e7SPeter Grehan 		procbased_ctls2 |= PROCBASED2_ENABLE_RDTSCP;
826f5f5f1e7SPeter Grehan 
827366f6083SPeter Grehan 	cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
828366f6083SPeter Grehan 					MSR_VMX_PROCBASED_CTLS2,
829366f6083SPeter Grehan 					PROCBASED2_UNRESTRICTED_GUEST, 0,
830366f6083SPeter Grehan 				        &tmp) == 0);
831366f6083SPeter Grehan 
83249cc03daSNeel Natu 	cap_invpcid = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
83349cc03daSNeel Natu 	    MSR_VMX_PROCBASED_CTLS2, PROCBASED2_ENABLE_INVPCID, 0,
83449cc03daSNeel Natu 	    &tmp) == 0);
83549cc03daSNeel Natu 
83688c4b8d1SNeel Natu 	/*
8371bc51badSMichael Reifenberger 	 * Check support for TPR shadow.
8381bc51badSMichael Reifenberger 	 */
8391bc51badSMichael Reifenberger 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
8401bc51badSMichael Reifenberger 	    MSR_VMX_TRUE_PROCBASED_CTLS, PROCBASED_USE_TPR_SHADOW, 0,
8411bc51badSMichael Reifenberger 	    &tmp);
8421bc51badSMichael Reifenberger 	if (error == 0) {
8431bc51badSMichael Reifenberger 		tpr_shadowing = 1;
8441bc51badSMichael Reifenberger 		TUNABLE_INT_FETCH("hw.vmm.vmx.use_tpr_shadowing",
8451bc51badSMichael Reifenberger 		    &tpr_shadowing);
8461bc51badSMichael Reifenberger 	}
8471bc51badSMichael Reifenberger 
8481bc51badSMichael Reifenberger 	if (tpr_shadowing) {
8491bc51badSMichael Reifenberger 		procbased_ctls |= PROCBASED_USE_TPR_SHADOW;
8501bc51badSMichael Reifenberger 		procbased_ctls &= ~PROCBASED_CR8_LOAD_EXITING;
8511bc51badSMichael Reifenberger 		procbased_ctls &= ~PROCBASED_CR8_STORE_EXITING;
8521bc51badSMichael Reifenberger 	}
8531bc51badSMichael Reifenberger 
8541bc51badSMichael Reifenberger 	/*
85588c4b8d1SNeel Natu 	 * Check support for virtual interrupt delivery.
85688c4b8d1SNeel Natu 	 */
85788c4b8d1SNeel Natu 	procbased2_vid_bits = (PROCBASED2_VIRTUALIZE_APIC_ACCESSES |
85888c4b8d1SNeel Natu 	    PROCBASED2_VIRTUALIZE_X2APIC_MODE |
85988c4b8d1SNeel Natu 	    PROCBASED2_APIC_REGISTER_VIRTUALIZATION |
86088c4b8d1SNeel Natu 	    PROCBASED2_VIRTUAL_INTERRUPT_DELIVERY);
86188c4b8d1SNeel Natu 
86288c4b8d1SNeel Natu 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
86388c4b8d1SNeel Natu 	    procbased2_vid_bits, 0, &tmp);
8641bc51badSMichael Reifenberger 	if (error == 0 && tpr_shadowing) {
86588c4b8d1SNeel Natu 		virtual_interrupt_delivery = 1;
86688c4b8d1SNeel Natu 		TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_vid",
86788c4b8d1SNeel Natu 		    &virtual_interrupt_delivery);
86888c4b8d1SNeel Natu 	}
86988c4b8d1SNeel Natu 
87088c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
87188c4b8d1SNeel Natu 		procbased_ctls |= PROCBASED_USE_TPR_SHADOW;
87288c4b8d1SNeel Natu 		procbased_ctls2 |= procbased2_vid_bits;
87388c4b8d1SNeel Natu 		procbased_ctls2 &= ~PROCBASED2_VIRTUALIZE_X2APIC_MODE;
874176666c2SNeel Natu 
875176666c2SNeel Natu 		/*
876176666c2SNeel Natu 		 * Check for Posted Interrupts only if Virtual Interrupt
877176666c2SNeel Natu 		 * Delivery is enabled.
878176666c2SNeel Natu 		 */
879176666c2SNeel Natu 		error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
880176666c2SNeel Natu 		    MSR_VMX_TRUE_PINBASED_CTLS, PINBASED_POSTED_INTERRUPT, 0,
881176666c2SNeel Natu 		    &tmp);
882176666c2SNeel Natu 		if (error == 0) {
883bd50262fSKonstantin Belousov 			pirvec = lapic_ipi_alloc(pti ? &IDTVEC(justreturn1_pti) :
884bd50262fSKonstantin Belousov 			    &IDTVEC(justreturn));
88518a2b08eSNeel Natu 			if (pirvec < 0) {
886176666c2SNeel Natu 				if (bootverbose) {
88715add60dSPeter Grehan 					printf("vmx_modinit: unable to "
88815add60dSPeter Grehan 					    "allocate posted interrupt "
88915add60dSPeter Grehan 					    "vector\n");
89088c4b8d1SNeel Natu 				}
891176666c2SNeel Natu 			} else {
892176666c2SNeel Natu 				posted_interrupts = 1;
893176666c2SNeel Natu 				TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_pir",
894176666c2SNeel Natu 				    &posted_interrupts);
895176666c2SNeel Natu 			}
896176666c2SNeel Natu 		}
897176666c2SNeel Natu 	}
898176666c2SNeel Natu 
899176666c2SNeel Natu 	if (posted_interrupts)
900176666c2SNeel Natu 		    pinbased_ctls |= PINBASED_POSTED_INTERRUPT;
90149cc03daSNeel Natu 
902366f6083SPeter Grehan 	/* Initialize EPT */
903add611fdSNeel Natu 	error = ept_init(ipinum);
904366f6083SPeter Grehan 	if (error) {
90515add60dSPeter Grehan 		printf("vmx_modinit: ept initialization failed (%d)\n", error);
906366f6083SPeter Grehan 		return (error);
907366f6083SPeter Grehan 	}
908366f6083SPeter Grehan 
90923437573SKonstantin Belousov 	guest_l1d_flush = (cpu_ia32_arch_caps &
91023437573SKonstantin Belousov 	    IA32_ARCH_CAP_SKIP_L1DFL_VMENTRY) == 0;
911c30578feSKonstantin Belousov 	TUNABLE_INT_FETCH("hw.vmm.l1d_flush", &guest_l1d_flush);
912c1141fbaSKonstantin Belousov 
913c1141fbaSKonstantin Belousov 	/*
914c1141fbaSKonstantin Belousov 	 * L1D cache flush is enabled.  Use IA32_FLUSH_CMD MSR when
915c1141fbaSKonstantin Belousov 	 * available.  Otherwise fall back to the software flush
916c1141fbaSKonstantin Belousov 	 * method which loads enough data from the kernel text to
917c1141fbaSKonstantin Belousov 	 * flush existing L1D content, both on VMX entry and on NMI
918c1141fbaSKonstantin Belousov 	 * return.
919c1141fbaSKonstantin Belousov 	 */
920c1141fbaSKonstantin Belousov 	if (guest_l1d_flush) {
921c1141fbaSKonstantin Belousov 		if ((cpu_stdext_feature3 & CPUID_STDEXT3_L1D_FLUSH) == 0) {
922c1141fbaSKonstantin Belousov 			guest_l1d_flush_sw = 1;
923c1141fbaSKonstantin Belousov 			TUNABLE_INT_FETCH("hw.vmm.l1d_flush_sw",
924c1141fbaSKonstantin Belousov 			    &guest_l1d_flush_sw);
925c1141fbaSKonstantin Belousov 		}
926c1141fbaSKonstantin Belousov 		if (guest_l1d_flush_sw) {
927c1141fbaSKonstantin Belousov 			if (nmi_flush_l1d_sw <= 1)
928c1141fbaSKonstantin Belousov 				nmi_flush_l1d_sw = 1;
929c1141fbaSKonstantin Belousov 		} else {
930c1141fbaSKonstantin Belousov 			msr_load_list[0].index = MSR_IA32_FLUSH_CMD;
931c1141fbaSKonstantin Belousov 			msr_load_list[0].val = IA32_FLUSH_CMD_L1D;
932c1141fbaSKonstantin Belousov 		}
933c1141fbaSKonstantin Belousov 	}
934c30578feSKonstantin Belousov 
935366f6083SPeter Grehan 	/*
936366f6083SPeter Grehan 	 * Stash the cr0 and cr4 bits that must be fixed to 0 or 1
937366f6083SPeter Grehan 	 */
938366f6083SPeter Grehan 	fixed0 = rdmsr(MSR_VMX_CR0_FIXED0);
939366f6083SPeter Grehan 	fixed1 = rdmsr(MSR_VMX_CR0_FIXED1);
940366f6083SPeter Grehan 	cr0_ones_mask = fixed0 & fixed1;
941366f6083SPeter Grehan 	cr0_zeros_mask = ~fixed0 & ~fixed1;
942366f6083SPeter Grehan 
943366f6083SPeter Grehan 	/*
944366f6083SPeter Grehan 	 * CR0_PE and CR0_PG can be set to zero in VMX non-root operation
945366f6083SPeter Grehan 	 * if unrestricted guest execution is allowed.
946366f6083SPeter Grehan 	 */
947366f6083SPeter Grehan 	if (cap_unrestricted_guest)
948366f6083SPeter Grehan 		cr0_ones_mask &= ~(CR0_PG | CR0_PE);
949366f6083SPeter Grehan 
950366f6083SPeter Grehan 	/*
951366f6083SPeter Grehan 	 * Do not allow the guest to set CR0_NW or CR0_CD.
952366f6083SPeter Grehan 	 */
953366f6083SPeter Grehan 	cr0_zeros_mask |= (CR0_NW | CR0_CD);
954366f6083SPeter Grehan 
955366f6083SPeter Grehan 	fixed0 = rdmsr(MSR_VMX_CR4_FIXED0);
956366f6083SPeter Grehan 	fixed1 = rdmsr(MSR_VMX_CR4_FIXED1);
957366f6083SPeter Grehan 	cr4_ones_mask = fixed0 & fixed1;
958366f6083SPeter Grehan 	cr4_zeros_mask = ~fixed0 & ~fixed1;
959366f6083SPeter Grehan 
96045e51299SNeel Natu 	vpid_init();
96145e51299SNeel Natu 
962c3498942SNeel Natu 	vmx_msr_init();
963c3498942SNeel Natu 
964366f6083SPeter Grehan 	/* enable VMX operation */
965366f6083SPeter Grehan 	smp_rendezvous(NULL, vmx_enable, NULL, NULL);
966366f6083SPeter Grehan 
9673565b59eSNeel Natu 	vmx_initialized = 1;
9683565b59eSNeel Natu 
969366f6083SPeter Grehan 	return (0);
970366f6083SPeter Grehan }
971366f6083SPeter Grehan 
972f7d47425SNeel Natu static void
973f7d47425SNeel Natu vmx_trigger_hostintr(int vector)
974f7d47425SNeel Natu {
975f7d47425SNeel Natu 	uintptr_t func;
976f7d47425SNeel Natu 	struct gate_descriptor *gd;
977f7d47425SNeel Natu 
978f7d47425SNeel Natu 	gd = &idt[vector];
979f7d47425SNeel Natu 
980f7d47425SNeel Natu 	KASSERT(vector >= 32 && vector <= 255, ("vmx_trigger_hostintr: "
981f7d47425SNeel Natu 	    "invalid vector %d", vector));
982f7d47425SNeel Natu 	KASSERT(gd->gd_p == 1, ("gate descriptor for vector %d not present",
983f7d47425SNeel Natu 	    vector));
984f7d47425SNeel Natu 	KASSERT(gd->gd_type == SDT_SYSIGT, ("gate descriptor for vector %d "
985f7d47425SNeel Natu 	    "has invalid type %d", vector, gd->gd_type));
986f7d47425SNeel Natu 	KASSERT(gd->gd_dpl == SEL_KPL, ("gate descriptor for vector %d "
987f7d47425SNeel Natu 	    "has invalid dpl %d", vector, gd->gd_dpl));
988f7d47425SNeel Natu 	KASSERT(gd->gd_selector == GSEL(GCODE_SEL, SEL_KPL), ("gate descriptor "
989f7d47425SNeel Natu 	    "for vector %d has invalid selector %d", vector, gd->gd_selector));
990f7d47425SNeel Natu 	KASSERT(gd->gd_ist == 0, ("gate descriptor for vector %d has invalid "
991f7d47425SNeel Natu 	    "IST %d", vector, gd->gd_ist));
992f7d47425SNeel Natu 
993f7d47425SNeel Natu 	func = ((long)gd->gd_hioffset << 16 | gd->gd_looffset);
994f7d47425SNeel Natu 	vmx_call_isr(func);
995f7d47425SNeel Natu }
996f7d47425SNeel Natu 
997366f6083SPeter Grehan static int
998aaaa0656SPeter Grehan vmx_setup_cr_shadow(int which, struct vmcs *vmcs, uint32_t initial)
999366f6083SPeter Grehan {
100039c21c2dSNeel Natu 	int error, mask_ident, shadow_ident;
1001aaaa0656SPeter Grehan 	uint64_t mask_value;
1002366f6083SPeter Grehan 
100339c21c2dSNeel Natu 	if (which != 0 && which != 4)
100439c21c2dSNeel Natu 		panic("vmx_setup_cr_shadow: unknown cr%d", which);
100539c21c2dSNeel Natu 
100639c21c2dSNeel Natu 	if (which == 0) {
100739c21c2dSNeel Natu 		mask_ident = VMCS_CR0_MASK;
100839c21c2dSNeel Natu 		mask_value = cr0_ones_mask | cr0_zeros_mask;
100939c21c2dSNeel Natu 		shadow_ident = VMCS_CR0_SHADOW;
101039c21c2dSNeel Natu 	} else {
101139c21c2dSNeel Natu 		mask_ident = VMCS_CR4_MASK;
101239c21c2dSNeel Natu 		mask_value = cr4_ones_mask | cr4_zeros_mask;
101339c21c2dSNeel Natu 		shadow_ident = VMCS_CR4_SHADOW;
101439c21c2dSNeel Natu 	}
101539c21c2dSNeel Natu 
1016d3c11f40SPeter Grehan 	error = vmcs_setreg(vmcs, 0, VMCS_IDENT(mask_ident), mask_value);
1017366f6083SPeter Grehan 	if (error)
1018366f6083SPeter Grehan 		return (error);
1019366f6083SPeter Grehan 
1020aaaa0656SPeter Grehan 	error = vmcs_setreg(vmcs, 0, VMCS_IDENT(shadow_ident), initial);
1021366f6083SPeter Grehan 	if (error)
1022366f6083SPeter Grehan 		return (error);
1023366f6083SPeter Grehan 
1024366f6083SPeter Grehan 	return (0);
1025366f6083SPeter Grehan }
1026aaaa0656SPeter Grehan #define	vmx_setup_cr0_shadow(vmcs,init)	vmx_setup_cr_shadow(0, (vmcs), (init))
1027aaaa0656SPeter Grehan #define	vmx_setup_cr4_shadow(vmcs,init)	vmx_setup_cr_shadow(4, (vmcs), (init))
1028366f6083SPeter Grehan 
1029366f6083SPeter Grehan static void *
103015add60dSPeter Grehan vmx_init(struct vm *vm, pmap_t pmap)
1031366f6083SPeter Grehan {
103245e51299SNeel Natu 	uint16_t vpid[VM_MAXCPU];
1033c3498942SNeel Natu 	int i, error;
1034366f6083SPeter Grehan 	struct vmx *vmx;
1035c847a506SNeel Natu 	struct vmcs *vmcs;
1036b0538143SNeel Natu 	uint32_t exc_bitmap;
1037a488c9c9SRodney W. Grimes 	uint16_t maxcpus;
1038366f6083SPeter Grehan 
1039366f6083SPeter Grehan 	vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO);
1040366f6083SPeter Grehan 	if ((uintptr_t)vmx & PAGE_MASK) {
1041366f6083SPeter Grehan 		panic("malloc of struct vmx not aligned on %d byte boundary",
1042366f6083SPeter Grehan 		      PAGE_SIZE);
1043366f6083SPeter Grehan 	}
1044366f6083SPeter Grehan 	vmx->vm = vm;
1045366f6083SPeter Grehan 
10469ce875d9SKonstantin Belousov 	vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pmltop));
1047318224bbSNeel Natu 
1048366f6083SPeter Grehan 	/*
1049366f6083SPeter Grehan 	 * Clean up EPTP-tagged guest physical and combined mappings
1050366f6083SPeter Grehan 	 *
1051366f6083SPeter Grehan 	 * VMX transitions are not required to invalidate any guest physical
1052366f6083SPeter Grehan 	 * mappings. So, it may be possible for stale guest physical mappings
1053366f6083SPeter Grehan 	 * to be present in the processor TLBs.
1054366f6083SPeter Grehan 	 *
1055366f6083SPeter Grehan 	 * Combined mappings for this EP4TA are also invalidated for all VPIDs.
1056366f6083SPeter Grehan 	 */
1057318224bbSNeel Natu 	ept_invalidate_mappings(vmx->eptp);
1058366f6083SPeter Grehan 
1059366f6083SPeter Grehan 	msr_bitmap_initialize(vmx->msr_bitmap);
1060366f6083SPeter Grehan 
1061366f6083SPeter Grehan 	/*
1062366f6083SPeter Grehan 	 * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE.
1063366f6083SPeter Grehan 	 * The guest FSBASE and GSBASE are saved and restored during
1064366f6083SPeter Grehan 	 * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are
1065366f6083SPeter Grehan 	 * always restored from the vmcs host state area on vm-exit.
1066366f6083SPeter Grehan 	 *
10671fb0ea3fSPeter Grehan 	 * The SYSENTER_CS/ESP/EIP MSRs are identical to FS/GSBASE in
10681fb0ea3fSPeter Grehan 	 * how they are saved/restored so can be directly accessed by the
10691fb0ea3fSPeter Grehan 	 * guest.
10701fb0ea3fSPeter Grehan 	 *
1071366f6083SPeter Grehan 	 * MSR_EFER is saved and restored in the guest VMCS area on a
1072366f6083SPeter Grehan 	 * VM exit and entry respectively. It is also restored from the
1073366f6083SPeter Grehan 	 * host VMCS area on a VM exit.
10748d1d7a9eSPeter Grehan 	 *
1075277bdd99STycho Nightingale 	 * The TSC MSR is exposed read-only. Writes are disallowed as
1076277bdd99STycho Nightingale 	 * that will impact the host TSC.  If the guest does a write
1077277bdd99STycho Nightingale 	 * the "use TSC offsetting" execution control is enabled and the
1078277bdd99STycho Nightingale 	 * difference between the host TSC and the guest TSC is written
1079277bdd99STycho Nightingale 	 * into the TSC offset in the VMCS.
1080f5f5f1e7SPeter Grehan 	 *
1081f5f5f1e7SPeter Grehan 	 * Guest TSC_AUX support is enabled if any of guest RDPID and/or
1082f5f5f1e7SPeter Grehan 	 * guest RDTSCP support are enabled (since, as per Table 2-2 in SDM
1083f5f5f1e7SPeter Grehan 	 * volume 4, TSC_AUX is supported if any of RDPID and/or RDTSCP are
1084f5f5f1e7SPeter Grehan 	 * supported). If guest TSC_AUX support is enabled, TSC_AUX is
1085f5f5f1e7SPeter Grehan 	 * exposed read-only so that the VMM can do one fewer MSR read per
1086f5f5f1e7SPeter Grehan 	 * exit than if this register were exposed read-write; the guest
1087f5f5f1e7SPeter Grehan 	 * restore value can be updated during guest writes (expected to be
1088f5f5f1e7SPeter Grehan 	 * rare) instead of during all exits (common).
1089366f6083SPeter Grehan 	 */
1090366f6083SPeter Grehan 	if (guest_msr_rw(vmx, MSR_GSBASE) ||
1091366f6083SPeter Grehan 	    guest_msr_rw(vmx, MSR_FSBASE) ||
10921fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_CS_MSR) ||
10931fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_ESP_MSR) ||
10941fb0ea3fSPeter Grehan 	    guest_msr_rw(vmx, MSR_SYSENTER_EIP_MSR) ||
10958d1d7a9eSPeter Grehan 	    guest_msr_rw(vmx, MSR_EFER) ||
1096f5f5f1e7SPeter Grehan 	    guest_msr_ro(vmx, MSR_TSC) ||
1097f5f5f1e7SPeter Grehan 	    ((cap_rdpid || cap_rdtscp) && guest_msr_ro(vmx, MSR_TSC_AUX)))
109815add60dSPeter Grehan 		panic("vmx_init: error setting guest msr access");
1099366f6083SPeter Grehan 
110045e51299SNeel Natu 	vpid_alloc(vpid, VM_MAXCPU);
110145e51299SNeel Natu 
110288c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
110388c4b8d1SNeel Natu 		error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE,
110488c4b8d1SNeel Natu 		    APIC_ACCESS_ADDRESS);
110588c4b8d1SNeel Natu 		/* XXX this should really return an error to the caller */
110688c4b8d1SNeel Natu 		KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error));
110788c4b8d1SNeel Natu 	}
110888c4b8d1SNeel Natu 
1109a488c9c9SRodney W. Grimes 	maxcpus = vm_get_maxcpus(vm);
1110a488c9c9SRodney W. Grimes 	for (i = 0; i < maxcpus; i++) {
1111c847a506SNeel Natu 		vmcs = &vmx->vmcs[i];
1112c847a506SNeel Natu 		vmcs->identifier = vmx_revision();
1113c847a506SNeel Natu 		error = vmclear(vmcs);
1114366f6083SPeter Grehan 		if (error != 0) {
111515add60dSPeter Grehan 			panic("vmx_init: vmclear error %d on vcpu %d\n",
1116366f6083SPeter Grehan 			      error, i);
1117366f6083SPeter Grehan 		}
1118366f6083SPeter Grehan 
1119c3498942SNeel Natu 		vmx_msr_guest_init(vmx, i);
1120c3498942SNeel Natu 
1121c847a506SNeel Natu 		error = vmcs_init(vmcs);
1122c847a506SNeel Natu 		KASSERT(error == 0, ("vmcs_init error %d", error));
1123366f6083SPeter Grehan 
1124c847a506SNeel Natu 		VMPTRLD(vmcs);
1125c847a506SNeel Natu 		error = 0;
1126c847a506SNeel Natu 		error += vmwrite(VMCS_HOST_RSP, (u_long)&vmx->ctx[i]);
1127c847a506SNeel Natu 		error += vmwrite(VMCS_EPTP, vmx->eptp);
1128c847a506SNeel Natu 		error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls);
1129c847a506SNeel Natu 		error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls);
1130*3ba952e1SCorvin Köhne 		if (vcpu_trap_wbinvd(vm, i)) {
1131*3ba952e1SCorvin Köhne 			KASSERT(cap_wbinvd_exit, ("WBINVD trap not available"));
1132*3ba952e1SCorvin Köhne 			procbased_ctls2 |= PROCBASED2_WBINVD_EXITING;
1133*3ba952e1SCorvin Köhne 		}
1134c847a506SNeel Natu 		error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2);
1135c847a506SNeel Natu 		error += vmwrite(VMCS_EXIT_CTLS, exit_ctls);
1136c847a506SNeel Natu 		error += vmwrite(VMCS_ENTRY_CTLS, entry_ctls);
1137c847a506SNeel Natu 		error += vmwrite(VMCS_MSR_BITMAP, vtophys(vmx->msr_bitmap));
1138c847a506SNeel Natu 		error += vmwrite(VMCS_VPID, vpid[i]);
1139b0538143SNeel Natu 
1140c1141fbaSKonstantin Belousov 		if (guest_l1d_flush && !guest_l1d_flush_sw) {
1141c1141fbaSKonstantin Belousov 			vmcs_write(VMCS_ENTRY_MSR_LOAD, pmap_kextract(
1142c1141fbaSKonstantin Belousov 			    (vm_offset_t)&msr_load_list[0]));
1143c1141fbaSKonstantin Belousov 			vmcs_write(VMCS_ENTRY_MSR_LOAD_COUNT,
1144c1141fbaSKonstantin Belousov 			    nitems(msr_load_list));
1145c1141fbaSKonstantin Belousov 			vmcs_write(VMCS_EXIT_MSR_STORE, 0);
1146c1141fbaSKonstantin Belousov 			vmcs_write(VMCS_EXIT_MSR_STORE_COUNT, 0);
1147c1141fbaSKonstantin Belousov 		}
1148c1141fbaSKonstantin Belousov 
1149b0538143SNeel Natu 		/* exception bitmap */
1150b0538143SNeel Natu 		if (vcpu_trace_exceptions(vm, i))
1151b0538143SNeel Natu 			exc_bitmap = 0xffffffff;
1152b0538143SNeel Natu 		else
1153b0538143SNeel Natu 			exc_bitmap = 1 << IDT_MC;
1154b0538143SNeel Natu 		error += vmwrite(VMCS_EXCEPTION_BITMAP, exc_bitmap);
1155b0538143SNeel Natu 
11569e2154ffSJohn Baldwin 		vmx->ctx[i].guest_dr6 = DBREG_DR6_RESERVED1;
11579e2154ffSJohn Baldwin 		error += vmwrite(VMCS_GUEST_DR7, DBREG_DR7_RESERVED1);
115865eefbe4SJohn Baldwin 
11591bc51badSMichael Reifenberger 		if (tpr_shadowing) {
116088c4b8d1SNeel Natu 			error += vmwrite(VMCS_VIRTUAL_APIC,
116188c4b8d1SNeel Natu 			    vtophys(&vmx->apic_page[i]));
11621bc51badSMichael Reifenberger 		}
11631bc51badSMichael Reifenberger 
11641bc51badSMichael Reifenberger 		if (virtual_interrupt_delivery) {
11651bc51badSMichael Reifenberger 			error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS);
116688c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT0, 0);
116788c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT1, 0);
116888c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT2, 0);
116988c4b8d1SNeel Natu 			error += vmwrite(VMCS_EOI_EXIT3, 0);
117088c4b8d1SNeel Natu 		}
1171176666c2SNeel Natu 		if (posted_interrupts) {
1172176666c2SNeel Natu 			error += vmwrite(VMCS_PIR_VECTOR, pirvec);
1173176666c2SNeel Natu 			error += vmwrite(VMCS_PIR_DESC,
1174176666c2SNeel Natu 			    vtophys(&vmx->pir_desc[i]));
1175176666c2SNeel Natu 		}
1176c847a506SNeel Natu 		VMCLEAR(vmcs);
117715add60dSPeter Grehan 		KASSERT(error == 0, ("vmx_init: error customizing the vmcs"));
1178366f6083SPeter Grehan 
1179366f6083SPeter Grehan 		vmx->cap[i].set = 0;
1180f5f5f1e7SPeter Grehan 		vmx->cap[i].set |= cap_rdpid != 0 ? 1 << VM_CAP_RDPID : 0;
1181f5f5f1e7SPeter Grehan 		vmx->cap[i].set |= cap_rdtscp != 0 ? 1 << VM_CAP_RDTSCP : 0;
1182366f6083SPeter Grehan 		vmx->cap[i].proc_ctls = procbased_ctls;
118349cc03daSNeel Natu 		vmx->cap[i].proc_ctls2 = procbased_ctls2;
1184cbd03a9dSJohn Baldwin 		vmx->cap[i].exc_bitmap = exc_bitmap;
1185366f6083SPeter Grehan 
11862ce12423SNeel Natu 		vmx->state[i].nextrip = ~0;
11873527963bSNeel Natu 		vmx->state[i].lastcpu = NOCPU;
118845e51299SNeel Natu 		vmx->state[i].vpid = vpid[i];
1189366f6083SPeter Grehan 
1190aaaa0656SPeter Grehan 		/*
1191aaaa0656SPeter Grehan 		 * Set up the CR0/4 shadows, and init the read shadow
1192aaaa0656SPeter Grehan 		 * to the power-on register value from the Intel Sys Arch.
1193aaaa0656SPeter Grehan 		 *  CR0 - 0x60000010
1194aaaa0656SPeter Grehan 		 *  CR4 - 0
1195aaaa0656SPeter Grehan 		 */
1196c847a506SNeel Natu 		error = vmx_setup_cr0_shadow(vmcs, 0x60000010);
119739c21c2dSNeel Natu 		if (error != 0)
119839c21c2dSNeel Natu 			panic("vmx_setup_cr0_shadow %d", error);
119939c21c2dSNeel Natu 
1200c847a506SNeel Natu 		error = vmx_setup_cr4_shadow(vmcs, 0);
120139c21c2dSNeel Natu 		if (error != 0)
120239c21c2dSNeel Natu 			panic("vmx_setup_cr4_shadow %d", error);
1203318224bbSNeel Natu 
1204318224bbSNeel Natu 		vmx->ctx[i].pmap = pmap;
1205366f6083SPeter Grehan 	}
1206366f6083SPeter Grehan 
1207366f6083SPeter Grehan 	return (vmx);
1208366f6083SPeter Grehan }
1209366f6083SPeter Grehan 
1210366f6083SPeter Grehan static int
1211a2da7af6SNeel Natu vmx_handle_cpuid(struct vm *vm, int vcpu, struct vmxctx *vmxctx)
1212366f6083SPeter Grehan {
1213a3f2a9c5SJohn Baldwin 	int handled;
1214366f6083SPeter Grehan 
1215a3f2a9c5SJohn Baldwin 	handled = x86_emulate_cpuid(vm, vcpu, (uint64_t *)&vmxctx->guest_rax,
1216a3f2a9c5SJohn Baldwin 	    (uint64_t *)&vmxctx->guest_rbx, (uint64_t *)&vmxctx->guest_rcx,
1217a3f2a9c5SJohn Baldwin 	    (uint64_t *)&vmxctx->guest_rdx);
1218366f6083SPeter Grehan 	return (handled);
1219366f6083SPeter Grehan }
1220366f6083SPeter Grehan 
1221366f6083SPeter Grehan static __inline void
1222366f6083SPeter Grehan vmx_run_trace(struct vmx *vmx, int vcpu)
1223366f6083SPeter Grehan {
1224366f6083SPeter Grehan #ifdef KTR
1225513c8d33SNeel Natu 	VCPU_CTR1(vmx->vm, vcpu, "Resume execution at %#lx", vmcs_guest_rip());
1226366f6083SPeter Grehan #endif
1227366f6083SPeter Grehan }
1228366f6083SPeter Grehan 
1229366f6083SPeter Grehan static __inline void
1230366f6083SPeter Grehan vmx_exit_trace(struct vmx *vmx, int vcpu, uint64_t rip, uint32_t exit_reason,
1231eeefa4e4SNeel Natu 	       int handled)
1232366f6083SPeter Grehan {
1233366f6083SPeter Grehan #ifdef KTR
1234513c8d33SNeel Natu 	VCPU_CTR3(vmx->vm, vcpu, "%s %s vmexit at 0x%0lx",
1235366f6083SPeter Grehan 		 handled ? "handled" : "unhandled",
1236366f6083SPeter Grehan 		 exit_reason_to_str(exit_reason), rip);
1237eeefa4e4SNeel Natu #endif
1238eeefa4e4SNeel Natu }
1239366f6083SPeter Grehan 
1240eeefa4e4SNeel Natu static __inline void
1241eeefa4e4SNeel Natu vmx_astpending_trace(struct vmx *vmx, int vcpu, uint64_t rip)
1242eeefa4e4SNeel Natu {
1243eeefa4e4SNeel Natu #ifdef KTR
1244513c8d33SNeel Natu 	VCPU_CTR1(vmx->vm, vcpu, "astpending vmexit at 0x%0lx", rip);
1245366f6083SPeter Grehan #endif
1246366f6083SPeter Grehan }
1247366f6083SPeter Grehan 
1248953c2c47SNeel Natu static VMM_STAT_INTEL(VCPU_INVVPID_SAVED, "Number of vpid invalidations saved");
12493527963bSNeel Natu static VMM_STAT_INTEL(VCPU_INVVPID_DONE, "Number of vpid invalidations done");
1250953c2c47SNeel Natu 
12513527963bSNeel Natu /*
12523527963bSNeel Natu  * Invalidate guest mappings identified by its vpid from the TLB.
12533527963bSNeel Natu  */
12543527963bSNeel Natu static __inline void
12553527963bSNeel Natu vmx_invvpid(struct vmx *vmx, int vcpu, pmap_t pmap, int running)
1256366f6083SPeter Grehan {
1257366f6083SPeter Grehan 	struct vmxstate *vmxstate;
1258953c2c47SNeel Natu 	struct invvpid_desc invvpid_desc;
1259366f6083SPeter Grehan 
1260366f6083SPeter Grehan 	vmxstate = &vmx->state[vcpu];
12613527963bSNeel Natu 	if (vmxstate->vpid == 0)
12623de83862SNeel Natu 		return;
1263366f6083SPeter Grehan 
12643527963bSNeel Natu 	if (!running) {
12653527963bSNeel Natu 		/*
12663527963bSNeel Natu 		 * Set the 'lastcpu' to an invalid host cpu.
12673527963bSNeel Natu 		 *
12683527963bSNeel Natu 		 * This will invalidate TLB entries tagged with the vcpu's
12693527963bSNeel Natu 		 * vpid the next time it runs via vmx_set_pcpu_defaults().
12703527963bSNeel Natu 		 */
12713527963bSNeel Natu 		vmxstate->lastcpu = NOCPU;
12723527963bSNeel Natu 		return;
12733527963bSNeel Natu 	}
1274953c2c47SNeel Natu 
12753527963bSNeel Natu 	KASSERT(curthread->td_critnest > 0, ("%s: vcpu %d running outside "
12763527963bSNeel Natu 	    "critical section", __func__, vcpu));
1277366f6083SPeter Grehan 
1278366f6083SPeter Grehan 	/*
12793527963bSNeel Natu 	 * Invalidate all mappings tagged with 'vpid'
1280366f6083SPeter Grehan 	 *
1281366f6083SPeter Grehan 	 * We do this because this vcpu was executing on a different host
1282366f6083SPeter Grehan 	 * cpu when it last ran. We do not track whether it invalidated
1283366f6083SPeter Grehan 	 * mappings associated with its 'vpid' during that run. So we must
1284366f6083SPeter Grehan 	 * assume that the mappings associated with 'vpid' on 'curcpu' are
1285366f6083SPeter Grehan 	 * stale and invalidate them.
1286366f6083SPeter Grehan 	 *
1287366f6083SPeter Grehan 	 * Note that we incur this penalty only when the scheduler chooses to
1288366f6083SPeter Grehan 	 * move the thread associated with this vcpu between host cpus.
1289366f6083SPeter Grehan 	 *
1290366f6083SPeter Grehan 	 * Note also that this will invalidate mappings tagged with 'vpid'
1291366f6083SPeter Grehan 	 * for "all" EP4TAs.
1292366f6083SPeter Grehan 	 */
12936f5a9606SMark Johnston 	if (atomic_load_long(&pmap->pm_eptgen) == vmx->eptgen[curcpu]) {
1294953c2c47SNeel Natu 		invvpid_desc._res1 = 0;
1295953c2c47SNeel Natu 		invvpid_desc._res2 = 0;
1296366f6083SPeter Grehan 		invvpid_desc.vpid = vmxstate->vpid;
12970e30c5c0SWarner Losh 		invvpid_desc.linear_addr = 0;
1298366f6083SPeter Grehan 		invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc);
12993527963bSNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_DONE, 1);
1300953c2c47SNeel Natu 	} else {
1301953c2c47SNeel Natu 		/*
1302953c2c47SNeel Natu 		 * The invvpid can be skipped if an invept is going to
1303953c2c47SNeel Natu 		 * be performed before entering the guest. The invept
1304953c2c47SNeel Natu 		 * will invalidate combined mappings tagged with
1305953c2c47SNeel Natu 		 * 'vmx->eptp' for all vpids.
1306953c2c47SNeel Natu 		 */
1307953c2c47SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_SAVED, 1);
1308953c2c47SNeel Natu 	}
1309366f6083SPeter Grehan }
13103527963bSNeel Natu 
13113527963bSNeel Natu static void
13123527963bSNeel Natu vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu, pmap_t pmap)
13133527963bSNeel Natu {
13143527963bSNeel Natu 	struct vmxstate *vmxstate;
13153527963bSNeel Natu 
13163527963bSNeel Natu 	vmxstate = &vmx->state[vcpu];
13173527963bSNeel Natu 	if (vmxstate->lastcpu == curcpu)
13183527963bSNeel Natu 		return;
13193527963bSNeel Natu 
13203527963bSNeel Natu 	vmxstate->lastcpu = curcpu;
13213527963bSNeel Natu 
13223527963bSNeel Natu 	vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1);
13233527963bSNeel Natu 
13243527963bSNeel Natu 	vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase());
13253527963bSNeel Natu 	vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase());
13263527963bSNeel Natu 	vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase());
13273527963bSNeel Natu 	vmx_invvpid(vmx, vcpu, pmap, 1);
1328366f6083SPeter Grehan }
1329366f6083SPeter Grehan 
1330366f6083SPeter Grehan /*
1331366f6083SPeter Grehan  * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set.
1332366f6083SPeter Grehan  */
1333366f6083SPeter Grehan CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0);
1334366f6083SPeter Grehan 
1335366f6083SPeter Grehan static void __inline
1336366f6083SPeter Grehan vmx_set_int_window_exiting(struct vmx *vmx, int vcpu)
1337366f6083SPeter Grehan {
1338366f6083SPeter Grehan 
133948b2d828SNeel Natu 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) {
1340366f6083SPeter Grehan 		vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING;
13413de83862SNeel Natu 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
134248b2d828SNeel Natu 		VCPU_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting");
134348b2d828SNeel Natu 	}
1344366f6083SPeter Grehan }
1345366f6083SPeter Grehan 
1346366f6083SPeter Grehan static void __inline
1347366f6083SPeter Grehan vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu)
1348366f6083SPeter Grehan {
1349366f6083SPeter Grehan 
135048b2d828SNeel Natu 	KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0,
135148b2d828SNeel Natu 	    ("intr_window_exiting not set: %#x", vmx->cap[vcpu].proc_ctls));
1352366f6083SPeter Grehan 	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING;
13533de83862SNeel Natu 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
135448b2d828SNeel Natu 	VCPU_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting");
1355366f6083SPeter Grehan }
1356366f6083SPeter Grehan 
1357366f6083SPeter Grehan static void __inline
1358366f6083SPeter Grehan vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu)
1359366f6083SPeter Grehan {
1360366f6083SPeter Grehan 
136148b2d828SNeel Natu 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) {
1362366f6083SPeter Grehan 		vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING;
13633de83862SNeel Natu 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
136448b2d828SNeel Natu 		VCPU_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting");
136548b2d828SNeel Natu 	}
1366366f6083SPeter Grehan }
1367366f6083SPeter Grehan 
1368366f6083SPeter Grehan static void __inline
1369366f6083SPeter Grehan vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu)
1370366f6083SPeter Grehan {
1371366f6083SPeter Grehan 
137248b2d828SNeel Natu 	KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0,
137348b2d828SNeel Natu 	    ("nmi_window_exiting not set %#x", vmx->cap[vcpu].proc_ctls));
1374366f6083SPeter Grehan 	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING;
13753de83862SNeel Natu 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
137648b2d828SNeel Natu 	VCPU_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting");
1377366f6083SPeter Grehan }
1378366f6083SPeter Grehan 
1379277bdd99STycho Nightingale int
1380277bdd99STycho Nightingale vmx_set_tsc_offset(struct vmx *vmx, int vcpu, uint64_t offset)
1381277bdd99STycho Nightingale {
1382277bdd99STycho Nightingale 	int error;
1383277bdd99STycho Nightingale 
1384277bdd99STycho Nightingale 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_TSC_OFFSET) == 0) {
1385277bdd99STycho Nightingale 		vmx->cap[vcpu].proc_ctls |= PROCBASED_TSC_OFFSET;
1386277bdd99STycho Nightingale 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1387277bdd99STycho Nightingale 		VCPU_CTR0(vmx->vm, vcpu, "Enabling TSC offsetting");
1388277bdd99STycho Nightingale 	}
1389277bdd99STycho Nightingale 
1390277bdd99STycho Nightingale 	error = vmwrite(VMCS_TSC_OFFSET, offset);
1391483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT
1392483d953aSJohn Baldwin 	if (error == 0)
1393483d953aSJohn Baldwin 		error = vm_set_tsc_offset(vmx->vm, vcpu, offset);
1394483d953aSJohn Baldwin #endif
1395277bdd99STycho Nightingale 	return (error);
1396277bdd99STycho Nightingale }
1397277bdd99STycho Nightingale 
139848b2d828SNeel Natu #define	NMI_BLOCKING	(VMCS_INTERRUPTIBILITY_NMI_BLOCKING |		\
139948b2d828SNeel Natu 			 VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
140048b2d828SNeel Natu #define	HWINTR_BLOCKING	(VMCS_INTERRUPTIBILITY_STI_BLOCKING |		\
140148b2d828SNeel Natu 			 VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
140248b2d828SNeel Natu 
140348b2d828SNeel Natu static void
1404366f6083SPeter Grehan vmx_inject_nmi(struct vmx *vmx, int vcpu)
1405366f6083SPeter Grehan {
14065c272efaSRobert Wing 	uint32_t gi __diagused, info;
1407366f6083SPeter Grehan 
140848b2d828SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
140948b2d828SNeel Natu 	KASSERT((gi & NMI_BLOCKING) == 0, ("vmx_inject_nmi: invalid guest "
141048b2d828SNeel Natu 	    "interruptibility-state %#x", gi));
1411366f6083SPeter Grehan 
141248b2d828SNeel Natu 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
141348b2d828SNeel Natu 	KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_nmi: invalid "
141448b2d828SNeel Natu 	    "VM-entry interruption information %#x", info));
1415366f6083SPeter Grehan 
1416366f6083SPeter Grehan 	/*
1417366f6083SPeter Grehan 	 * Inject the virtual NMI. The vector must be the NMI IDT entry
1418366f6083SPeter Grehan 	 * or the VMCS entry check will fail.
1419366f6083SPeter Grehan 	 */
142048b2d828SNeel Natu 	info = IDT_NMI | VMCS_INTR_T_NMI | VMCS_INTR_VALID;
14213de83862SNeel Natu 	vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1422366f6083SPeter Grehan 
1423513c8d33SNeel Natu 	VCPU_CTR0(vmx->vm, vcpu, "Injecting vNMI");
1424366f6083SPeter Grehan 
1425366f6083SPeter Grehan 	/* Clear the request */
1426f352ff0cSNeel Natu 	vm_nmi_clear(vmx->vm, vcpu);
1427366f6083SPeter Grehan }
1428366f6083SPeter Grehan 
1429366f6083SPeter Grehan static void
14302ce12423SNeel Natu vmx_inject_interrupts(struct vmx *vmx, int vcpu, struct vlapic *vlapic,
14312ce12423SNeel Natu     uint64_t guestrip)
1432366f6083SPeter Grehan {
14330775fbb4STycho Nightingale 	int vector, need_nmi_exiting, extint_pending;
1434091d4532SNeel Natu 	uint64_t rflags, entryinfo;
143548b2d828SNeel Natu 	uint32_t gi, info;
1436366f6083SPeter Grehan 
14372ce12423SNeel Natu 	if (vmx->state[vcpu].nextrip != guestrip) {
14382ce12423SNeel Natu 		gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
14392ce12423SNeel Natu 		if (gi & HWINTR_BLOCKING) {
14402ce12423SNeel Natu 			VCPU_CTR2(vmx->vm, vcpu, "Guest interrupt blocking "
14412ce12423SNeel Natu 			    "cleared due to rip change: %#lx/%#lx",
14422ce12423SNeel Natu 			    vmx->state[vcpu].nextrip, guestrip);
14432ce12423SNeel Natu 			gi &= ~HWINTR_BLOCKING;
14442ce12423SNeel Natu 			vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
14452ce12423SNeel Natu 		}
14462ce12423SNeel Natu 	}
14472ce12423SNeel Natu 
1448091d4532SNeel Natu 	if (vm_entry_intinfo(vmx->vm, vcpu, &entryinfo)) {
1449091d4532SNeel Natu 		KASSERT((entryinfo & VMCS_INTR_VALID) != 0, ("%s: entry "
1450091d4532SNeel Natu 		    "intinfo is not valid: %#lx", __func__, entryinfo));
1451dc506506SNeel Natu 
1452dc506506SNeel Natu 		info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1453dc506506SNeel Natu 		KASSERT((info & VMCS_INTR_VALID) == 0, ("%s: cannot inject "
1454019008ebSNeel Natu 		     "pending exception: %#lx/%#x", __func__, entryinfo, info));
1455dc506506SNeel Natu 
1456091d4532SNeel Natu 		info = entryinfo;
1457091d4532SNeel Natu 		vector = info & 0xff;
1458091d4532SNeel Natu 		if (vector == IDT_BP || vector == IDT_OF) {
1459091d4532SNeel Natu 			/*
1460091d4532SNeel Natu 			 * VT-x requires #BP and #OF to be injected as software
1461091d4532SNeel Natu 			 * exceptions.
1462091d4532SNeel Natu 			 */
1463091d4532SNeel Natu 			info &= ~VMCS_INTR_T_MASK;
1464091d4532SNeel Natu 			info |= VMCS_INTR_T_SWEXCEPTION;
1465dc506506SNeel Natu 		}
1466091d4532SNeel Natu 
1467091d4532SNeel Natu 		if (info & VMCS_INTR_DEL_ERRCODE)
1468091d4532SNeel Natu 			vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR, entryinfo >> 32);
1469091d4532SNeel Natu 
1470dc506506SNeel Natu 		vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1471dc506506SNeel Natu 	}
1472dc506506SNeel Natu 
147348b2d828SNeel Natu 	if (vm_nmi_pending(vmx->vm, vcpu)) {
1474366f6083SPeter Grehan 		/*
147548b2d828SNeel Natu 		 * If there are no conditions blocking NMI injection then
147648b2d828SNeel Natu 		 * inject it directly here otherwise enable "NMI window
147748b2d828SNeel Natu 		 * exiting" to inject it as soon as we can.
1478eeefa4e4SNeel Natu 		 *
147948b2d828SNeel Natu 		 * We also check for STI_BLOCKING because some implementations
148048b2d828SNeel Natu 		 * don't allow NMI injection in this case. If we are running
148148b2d828SNeel Natu 		 * on a processor that doesn't have this restriction it will
148248b2d828SNeel Natu 		 * immediately exit and the NMI will be injected in the
148348b2d828SNeel Natu 		 * "NMI window exiting" handler.
1484366f6083SPeter Grehan 		 */
148548b2d828SNeel Natu 		need_nmi_exiting = 1;
148648b2d828SNeel Natu 		gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
148748b2d828SNeel Natu 		if ((gi & (HWINTR_BLOCKING | NMI_BLOCKING)) == 0) {
14883de83862SNeel Natu 			info = vmcs_read(VMCS_ENTRY_INTR_INFO);
148948b2d828SNeel Natu 			if ((info & VMCS_INTR_VALID) == 0) {
149048b2d828SNeel Natu 				vmx_inject_nmi(vmx, vcpu);
149148b2d828SNeel Natu 				need_nmi_exiting = 0;
149248b2d828SNeel Natu 			} else {
149348b2d828SNeel Natu 				VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI "
149448b2d828SNeel Natu 				    "due to VM-entry intr info %#x", info);
149548b2d828SNeel Natu 			}
149648b2d828SNeel Natu 		} else {
149748b2d828SNeel Natu 			VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI due to "
149848b2d828SNeel Natu 			    "Guest Interruptibility-state %#x", gi);
149948b2d828SNeel Natu 		}
1500eeefa4e4SNeel Natu 
150148b2d828SNeel Natu 		if (need_nmi_exiting)
150248b2d828SNeel Natu 			vmx_set_nmi_window_exiting(vmx, vcpu);
150348b2d828SNeel Natu 	}
1504366f6083SPeter Grehan 
15050775fbb4STycho Nightingale 	extint_pending = vm_extint_pending(vmx->vm, vcpu);
15060775fbb4STycho Nightingale 
15070775fbb4STycho Nightingale 	if (!extint_pending && virtual_interrupt_delivery) {
150888c4b8d1SNeel Natu 		vmx_inject_pir(vlapic);
150988c4b8d1SNeel Natu 		return;
151088c4b8d1SNeel Natu 	}
151188c4b8d1SNeel Natu 
151248b2d828SNeel Natu 	/*
151336736912SNeel Natu 	 * If interrupt-window exiting is already in effect then don't bother
151436736912SNeel Natu 	 * checking for pending interrupts. This is just an optimization and
151536736912SNeel Natu 	 * not needed for correctness.
151648b2d828SNeel Natu 	 */
151736736912SNeel Natu 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) {
151836736912SNeel Natu 		VCPU_CTR0(vmx->vm, vcpu, "Skip interrupt injection due to "
151936736912SNeel Natu 		    "pending int_window_exiting");
152048b2d828SNeel Natu 		return;
152136736912SNeel Natu 	}
152248b2d828SNeel Natu 
15230775fbb4STycho Nightingale 	if (!extint_pending) {
1524366f6083SPeter Grehan 		/* Ask the local apic for a vector to inject */
15254d1e82a8SNeel Natu 		if (!vlapic_pending_intr(vlapic, &vector))
1526366f6083SPeter Grehan 			return;
1527a026dc3fSTycho Nightingale 
1528a026dc3fSTycho Nightingale 		/*
1529a026dc3fSTycho Nightingale 		 * From the Intel SDM, Volume 3, Section "Maskable
1530a026dc3fSTycho Nightingale 		 * Hardware Interrupts":
1531a026dc3fSTycho Nightingale 		 * - maskable interrupt vectors [16,255] can be delivered
1532a026dc3fSTycho Nightingale 		 *   through the local APIC.
1533a026dc3fSTycho Nightingale 		*/
1534a026dc3fSTycho Nightingale 		KASSERT(vector >= 16 && vector <= 255,
1535a026dc3fSTycho Nightingale 		    ("invalid vector %d from local APIC", vector));
15360775fbb4STycho Nightingale 	} else {
15370775fbb4STycho Nightingale 		/* Ask the legacy pic for a vector to inject */
15380775fbb4STycho Nightingale 		vatpic_pending_intr(vmx->vm, &vector);
1539366f6083SPeter Grehan 
1540a026dc3fSTycho Nightingale 		/*
1541a026dc3fSTycho Nightingale 		 * From the Intel SDM, Volume 3, Section "Maskable
1542a026dc3fSTycho Nightingale 		 * Hardware Interrupts":
1543a026dc3fSTycho Nightingale 		 * - maskable interrupt vectors [0,255] can be delivered
1544a026dc3fSTycho Nightingale 		 *   through the INTR pin.
1545a026dc3fSTycho Nightingale 		 */
1546a026dc3fSTycho Nightingale 		KASSERT(vector >= 0 && vector <= 255,
1547a026dc3fSTycho Nightingale 		    ("invalid vector %d from INTR", vector));
1548a026dc3fSTycho Nightingale 	}
1549366f6083SPeter Grehan 
1550366f6083SPeter Grehan 	/* Check RFLAGS.IF and the interruptibility state of the guest */
15513de83862SNeel Natu 	rflags = vmcs_read(VMCS_GUEST_RFLAGS);
155236736912SNeel Natu 	if ((rflags & PSL_I) == 0) {
155336736912SNeel Natu 		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
155436736912SNeel Natu 		    "rflags %#lx", vector, rflags);
1555366f6083SPeter Grehan 		goto cantinject;
155636736912SNeel Natu 	}
1557366f6083SPeter Grehan 
155848b2d828SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
155936736912SNeel Natu 	if (gi & HWINTR_BLOCKING) {
156036736912SNeel Natu 		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
156136736912SNeel Natu 		    "Guest Interruptibility-state %#x", vector, gi);
1562366f6083SPeter Grehan 		goto cantinject;
156336736912SNeel Natu 	}
156436736912SNeel Natu 
156536736912SNeel Natu 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
156636736912SNeel Natu 	if (info & VMCS_INTR_VALID) {
156736736912SNeel Natu 		/*
156836736912SNeel Natu 		 * This is expected and could happen for multiple reasons:
156936736912SNeel Natu 		 * - A vectoring VM-entry was aborted due to astpending
157036736912SNeel Natu 		 * - A VM-exit happened during event injection.
1571dc506506SNeel Natu 		 * - An exception was injected above.
157236736912SNeel Natu 		 * - An NMI was injected above or after "NMI window exiting"
157336736912SNeel Natu 		 */
157436736912SNeel Natu 		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
157536736912SNeel Natu 		    "VM-entry intr info %#x", vector, info);
157636736912SNeel Natu 		goto cantinject;
157736736912SNeel Natu 	}
1578366f6083SPeter Grehan 
1579366f6083SPeter Grehan 	/* Inject the interrupt */
1580160471d2SNeel Natu 	info = VMCS_INTR_T_HWINTR | VMCS_INTR_VALID;
1581366f6083SPeter Grehan 	info |= vector;
15823de83862SNeel Natu 	vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1583366f6083SPeter Grehan 
15840775fbb4STycho Nightingale 	if (!extint_pending) {
1585366f6083SPeter Grehan 		/* Update the Local APIC ISR */
1586de5ea6b6SNeel Natu 		vlapic_intr_accepted(vlapic, vector);
15870775fbb4STycho Nightingale 	} else {
15880775fbb4STycho Nightingale 		vm_extint_clear(vmx->vm, vcpu);
15890775fbb4STycho Nightingale 		vatpic_intr_accepted(vmx->vm, vector);
15900775fbb4STycho Nightingale 
15910775fbb4STycho Nightingale 		/*
15920775fbb4STycho Nightingale 		 * After we accepted the current ExtINT the PIC may
15930775fbb4STycho Nightingale 		 * have posted another one.  If that is the case, set
15940775fbb4STycho Nightingale 		 * the Interrupt Window Exiting execution control so
15950775fbb4STycho Nightingale 		 * we can inject that one too.
15960494cb1bSNeel Natu 		 *
15970494cb1bSNeel Natu 		 * Also, interrupt window exiting allows us to inject any
15980494cb1bSNeel Natu 		 * pending APIC vector that was preempted by the ExtINT
15990494cb1bSNeel Natu 		 * as soon as possible. This applies both for the software
16000494cb1bSNeel Natu 		 * emulated vlapic and the hardware assisted virtual APIC.
16010775fbb4STycho Nightingale 		 */
16020775fbb4STycho Nightingale 		vmx_set_int_window_exiting(vmx, vcpu);
16030775fbb4STycho Nightingale 	}
1604366f6083SPeter Grehan 
1605513c8d33SNeel Natu 	VCPU_CTR1(vmx->vm, vcpu, "Injecting hwintr at vector %d", vector);
1606366f6083SPeter Grehan 
1607366f6083SPeter Grehan 	return;
1608366f6083SPeter Grehan 
1609366f6083SPeter Grehan cantinject:
1610366f6083SPeter Grehan 	/*
1611366f6083SPeter Grehan 	 * Set the Interrupt Window Exiting execution control so we can inject
1612366f6083SPeter Grehan 	 * the interrupt as soon as blocking condition goes away.
1613366f6083SPeter Grehan 	 */
1614366f6083SPeter Grehan 	vmx_set_int_window_exiting(vmx, vcpu);
1615366f6083SPeter Grehan }
1616366f6083SPeter Grehan 
1617e5a1d950SNeel Natu /*
1618e5a1d950SNeel Natu  * If the Virtual NMIs execution control is '1' then the logical processor
1619e5a1d950SNeel Natu  * tracks virtual-NMI blocking in the Guest Interruptibility-state field of
1620e5a1d950SNeel Natu  * the VMCS. An IRET instruction in VMX non-root operation will remove any
1621e5a1d950SNeel Natu  * virtual-NMI blocking.
1622e5a1d950SNeel Natu  *
1623e5a1d950SNeel Natu  * This unblocking occurs even if the IRET causes a fault. In this case the
1624e5a1d950SNeel Natu  * hypervisor needs to restore virtual-NMI blocking before resuming the guest.
1625e5a1d950SNeel Natu  */
1626e5a1d950SNeel Natu static void
1627e5a1d950SNeel Natu vmx_restore_nmi_blocking(struct vmx *vmx, int vcpuid)
1628e5a1d950SNeel Natu {
1629e5a1d950SNeel Natu 	uint32_t gi;
1630e5a1d950SNeel Natu 
1631e5a1d950SNeel Natu 	VCPU_CTR0(vmx->vm, vcpuid, "Restore Virtual-NMI blocking");
1632e5a1d950SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1633e5a1d950SNeel Natu 	gi |= VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1634e5a1d950SNeel Natu 	vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1635e5a1d950SNeel Natu }
1636e5a1d950SNeel Natu 
1637e5a1d950SNeel Natu static void
1638e5a1d950SNeel Natu vmx_clear_nmi_blocking(struct vmx *vmx, int vcpuid)
1639e5a1d950SNeel Natu {
1640e5a1d950SNeel Natu 	uint32_t gi;
1641e5a1d950SNeel Natu 
1642e5a1d950SNeel Natu 	VCPU_CTR0(vmx->vm, vcpuid, "Clear Virtual-NMI blocking");
1643e5a1d950SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1644e5a1d950SNeel Natu 	gi &= ~VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1645e5a1d950SNeel Natu 	vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1646e5a1d950SNeel Natu }
1647e5a1d950SNeel Natu 
1648091d4532SNeel Natu static void
1649091d4532SNeel Natu vmx_assert_nmi_blocking(struct vmx *vmx, int vcpuid)
1650091d4532SNeel Natu {
16515c272efaSRobert Wing 	uint32_t gi __diagused;
1652091d4532SNeel Natu 
1653091d4532SNeel Natu 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1654091d4532SNeel Natu 	KASSERT(gi & VMCS_INTERRUPTIBILITY_NMI_BLOCKING,
1655091d4532SNeel Natu 	    ("NMI blocking is not in effect %#x", gi));
1656091d4532SNeel Natu }
1657091d4532SNeel Natu 
1658366f6083SPeter Grehan static int
1659a0efd3fbSJohn Baldwin vmx_emulate_xsetbv(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1660abb023fbSJohn Baldwin {
1661abb023fbSJohn Baldwin 	struct vmxctx *vmxctx;
1662abb023fbSJohn Baldwin 	uint64_t xcrval;
1663abb023fbSJohn Baldwin 	const struct xsave_limits *limits;
1664abb023fbSJohn Baldwin 
1665abb023fbSJohn Baldwin 	vmxctx = &vmx->ctx[vcpu];
1666abb023fbSJohn Baldwin 	limits = vmm_get_xsave_limits();
1667abb023fbSJohn Baldwin 
1668a0efd3fbSJohn Baldwin 	/*
1669a0efd3fbSJohn Baldwin 	 * Note that the processor raises a GP# fault on its own if
1670a0efd3fbSJohn Baldwin 	 * xsetbv is executed for CPL != 0, so we do not have to
1671a0efd3fbSJohn Baldwin 	 * emulate that fault here.
1672a0efd3fbSJohn Baldwin 	 */
1673a0efd3fbSJohn Baldwin 
1674a0efd3fbSJohn Baldwin 	/* Only xcr0 is supported. */
1675a0efd3fbSJohn Baldwin 	if (vmxctx->guest_rcx != 0) {
1676dc506506SNeel Natu 		vm_inject_gp(vmx->vm, vcpu);
1677a0efd3fbSJohn Baldwin 		return (HANDLED);
1678a0efd3fbSJohn Baldwin 	}
1679a0efd3fbSJohn Baldwin 
1680a0efd3fbSJohn Baldwin 	/* We only handle xcr0 if both the host and guest have XSAVE enabled. */
1681a0efd3fbSJohn Baldwin 	if (!limits->xsave_enabled || !(vmcs_read(VMCS_GUEST_CR4) & CR4_XSAVE)) {
1682dc506506SNeel Natu 		vm_inject_ud(vmx->vm, vcpu);
1683a0efd3fbSJohn Baldwin 		return (HANDLED);
1684a0efd3fbSJohn Baldwin 	}
1685abb023fbSJohn Baldwin 
1686abb023fbSJohn Baldwin 	xcrval = vmxctx->guest_rdx << 32 | (vmxctx->guest_rax & 0xffffffff);
1687a0efd3fbSJohn Baldwin 	if ((xcrval & ~limits->xcr0_allowed) != 0) {
1688dc506506SNeel Natu 		vm_inject_gp(vmx->vm, vcpu);
1689a0efd3fbSJohn Baldwin 		return (HANDLED);
1690a0efd3fbSJohn Baldwin 	}
1691abb023fbSJohn Baldwin 
1692a0efd3fbSJohn Baldwin 	if (!(xcrval & XFEATURE_ENABLED_X87)) {
1693dc506506SNeel Natu 		vm_inject_gp(vmx->vm, vcpu);
1694a0efd3fbSJohn Baldwin 		return (HANDLED);
1695a0efd3fbSJohn Baldwin 	}
1696abb023fbSJohn Baldwin 
169744a68c4eSJohn Baldwin 	/* AVX (YMM_Hi128) requires SSE. */
169844a68c4eSJohn Baldwin 	if (xcrval & XFEATURE_ENABLED_AVX &&
169944a68c4eSJohn Baldwin 	    (xcrval & XFEATURE_AVX) != XFEATURE_AVX) {
170044a68c4eSJohn Baldwin 		vm_inject_gp(vmx->vm, vcpu);
170144a68c4eSJohn Baldwin 		return (HANDLED);
170244a68c4eSJohn Baldwin 	}
170344a68c4eSJohn Baldwin 
170444a68c4eSJohn Baldwin 	/*
170544a68c4eSJohn Baldwin 	 * AVX512 requires base AVX (YMM_Hi128) as well as OpMask,
170644a68c4eSJohn Baldwin 	 * ZMM_Hi256, and Hi16_ZMM.
170744a68c4eSJohn Baldwin 	 */
170844a68c4eSJohn Baldwin 	if (xcrval & XFEATURE_AVX512 &&
170944a68c4eSJohn Baldwin 	    (xcrval & (XFEATURE_AVX512 | XFEATURE_AVX)) !=
171044a68c4eSJohn Baldwin 	    (XFEATURE_AVX512 | XFEATURE_AVX)) {
171144a68c4eSJohn Baldwin 		vm_inject_gp(vmx->vm, vcpu);
171244a68c4eSJohn Baldwin 		return (HANDLED);
171344a68c4eSJohn Baldwin 	}
171444a68c4eSJohn Baldwin 
171544a68c4eSJohn Baldwin 	/*
171644a68c4eSJohn Baldwin 	 * Intel MPX requires both bound register state flags to be
171744a68c4eSJohn Baldwin 	 * set.
171844a68c4eSJohn Baldwin 	 */
171944a68c4eSJohn Baldwin 	if (((xcrval & XFEATURE_ENABLED_BNDREGS) != 0) !=
172044a68c4eSJohn Baldwin 	    ((xcrval & XFEATURE_ENABLED_BNDCSR) != 0)) {
1721dc506506SNeel Natu 		vm_inject_gp(vmx->vm, vcpu);
1722a0efd3fbSJohn Baldwin 		return (HANDLED);
1723a0efd3fbSJohn Baldwin 	}
1724abb023fbSJohn Baldwin 
1725abb023fbSJohn Baldwin 	/*
1726abb023fbSJohn Baldwin 	 * This runs "inside" vmrun() with the guest's FPU state, so
1727abb023fbSJohn Baldwin 	 * modifying xcr0 directly modifies the guest's xcr0, not the
1728abb023fbSJohn Baldwin 	 * host's.
1729abb023fbSJohn Baldwin 	 */
1730abb023fbSJohn Baldwin 	load_xcr(0, xcrval);
1731abb023fbSJohn Baldwin 	return (HANDLED);
1732abb023fbSJohn Baldwin }
1733abb023fbSJohn Baldwin 
1734594db002STycho Nightingale static uint64_t
1735594db002STycho Nightingale vmx_get_guest_reg(struct vmx *vmx, int vcpu, int ident)
1736366f6083SPeter Grehan {
1737366f6083SPeter Grehan 	const struct vmxctx *vmxctx;
1738366f6083SPeter Grehan 
1739594db002STycho Nightingale 	vmxctx = &vmx->ctx[vcpu];
1740594db002STycho Nightingale 
1741594db002STycho Nightingale 	switch (ident) {
1742594db002STycho Nightingale 	case 0:
1743594db002STycho Nightingale 		return (vmxctx->guest_rax);
1744594db002STycho Nightingale 	case 1:
1745594db002STycho Nightingale 		return (vmxctx->guest_rcx);
1746594db002STycho Nightingale 	case 2:
1747594db002STycho Nightingale 		return (vmxctx->guest_rdx);
1748594db002STycho Nightingale 	case 3:
1749594db002STycho Nightingale 		return (vmxctx->guest_rbx);
1750594db002STycho Nightingale 	case 4:
1751594db002STycho Nightingale 		return (vmcs_read(VMCS_GUEST_RSP));
1752594db002STycho Nightingale 	case 5:
1753594db002STycho Nightingale 		return (vmxctx->guest_rbp);
1754594db002STycho Nightingale 	case 6:
1755594db002STycho Nightingale 		return (vmxctx->guest_rsi);
1756594db002STycho Nightingale 	case 7:
1757594db002STycho Nightingale 		return (vmxctx->guest_rdi);
1758594db002STycho Nightingale 	case 8:
1759594db002STycho Nightingale 		return (vmxctx->guest_r8);
1760594db002STycho Nightingale 	case 9:
1761594db002STycho Nightingale 		return (vmxctx->guest_r9);
1762594db002STycho Nightingale 	case 10:
1763594db002STycho Nightingale 		return (vmxctx->guest_r10);
1764594db002STycho Nightingale 	case 11:
1765594db002STycho Nightingale 		return (vmxctx->guest_r11);
1766594db002STycho Nightingale 	case 12:
1767594db002STycho Nightingale 		return (vmxctx->guest_r12);
1768594db002STycho Nightingale 	case 13:
1769594db002STycho Nightingale 		return (vmxctx->guest_r13);
1770594db002STycho Nightingale 	case 14:
1771594db002STycho Nightingale 		return (vmxctx->guest_r14);
1772594db002STycho Nightingale 	case 15:
1773594db002STycho Nightingale 		return (vmxctx->guest_r15);
1774594db002STycho Nightingale 	default:
1775594db002STycho Nightingale 		panic("invalid vmx register %d", ident);
1776594db002STycho Nightingale 	}
1777594db002STycho Nightingale }
1778594db002STycho Nightingale 
1779594db002STycho Nightingale static void
1780594db002STycho Nightingale vmx_set_guest_reg(struct vmx *vmx, int vcpu, int ident, uint64_t regval)
1781594db002STycho Nightingale {
1782594db002STycho Nightingale 	struct vmxctx *vmxctx;
1783594db002STycho Nightingale 
1784594db002STycho Nightingale 	vmxctx = &vmx->ctx[vcpu];
1785594db002STycho Nightingale 
1786594db002STycho Nightingale 	switch (ident) {
1787594db002STycho Nightingale 	case 0:
1788594db002STycho Nightingale 		vmxctx->guest_rax = regval;
1789594db002STycho Nightingale 		break;
1790594db002STycho Nightingale 	case 1:
1791594db002STycho Nightingale 		vmxctx->guest_rcx = regval;
1792594db002STycho Nightingale 		break;
1793594db002STycho Nightingale 	case 2:
1794594db002STycho Nightingale 		vmxctx->guest_rdx = regval;
1795594db002STycho Nightingale 		break;
1796594db002STycho Nightingale 	case 3:
1797594db002STycho Nightingale 		vmxctx->guest_rbx = regval;
1798594db002STycho Nightingale 		break;
1799594db002STycho Nightingale 	case 4:
1800594db002STycho Nightingale 		vmcs_write(VMCS_GUEST_RSP, regval);
1801594db002STycho Nightingale 		break;
1802594db002STycho Nightingale 	case 5:
1803594db002STycho Nightingale 		vmxctx->guest_rbp = regval;
1804594db002STycho Nightingale 		break;
1805594db002STycho Nightingale 	case 6:
1806594db002STycho Nightingale 		vmxctx->guest_rsi = regval;
1807594db002STycho Nightingale 		break;
1808594db002STycho Nightingale 	case 7:
1809594db002STycho Nightingale 		vmxctx->guest_rdi = regval;
1810594db002STycho Nightingale 		break;
1811594db002STycho Nightingale 	case 8:
1812594db002STycho Nightingale 		vmxctx->guest_r8 = regval;
1813594db002STycho Nightingale 		break;
1814594db002STycho Nightingale 	case 9:
1815594db002STycho Nightingale 		vmxctx->guest_r9 = regval;
1816594db002STycho Nightingale 		break;
1817594db002STycho Nightingale 	case 10:
1818594db002STycho Nightingale 		vmxctx->guest_r10 = regval;
1819594db002STycho Nightingale 		break;
1820594db002STycho Nightingale 	case 11:
1821594db002STycho Nightingale 		vmxctx->guest_r11 = regval;
1822594db002STycho Nightingale 		break;
1823594db002STycho Nightingale 	case 12:
1824594db002STycho Nightingale 		vmxctx->guest_r12 = regval;
1825594db002STycho Nightingale 		break;
1826594db002STycho Nightingale 	case 13:
1827594db002STycho Nightingale 		vmxctx->guest_r13 = regval;
1828594db002STycho Nightingale 		break;
1829594db002STycho Nightingale 	case 14:
1830594db002STycho Nightingale 		vmxctx->guest_r14 = regval;
1831594db002STycho Nightingale 		break;
1832594db002STycho Nightingale 	case 15:
1833594db002STycho Nightingale 		vmxctx->guest_r15 = regval;
1834594db002STycho Nightingale 		break;
1835594db002STycho Nightingale 	default:
1836594db002STycho Nightingale 		panic("invalid vmx register %d", ident);
1837594db002STycho Nightingale 	}
1838594db002STycho Nightingale }
1839594db002STycho Nightingale 
1840594db002STycho Nightingale static int
1841594db002STycho Nightingale vmx_emulate_cr0_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1842594db002STycho Nightingale {
1843594db002STycho Nightingale 	uint64_t crval, regval;
1844594db002STycho Nightingale 
1845594db002STycho Nightingale 	/* We only handle mov to %cr0 at this time */
184639c21c2dSNeel Natu 	if ((exitqual & 0xf0) != 0x00)
184739c21c2dSNeel Natu 		return (UNHANDLED);
184839c21c2dSNeel Natu 
1849594db002STycho Nightingale 	regval = vmx_get_guest_reg(vmx, vcpu, (exitqual >> 8) & 0xf);
1850366f6083SPeter Grehan 
1851594db002STycho Nightingale 	vmcs_write(VMCS_CR0_SHADOW, regval);
1852366f6083SPeter Grehan 
1853594db002STycho Nightingale 	crval = regval | cr0_ones_mask;
1854594db002STycho Nightingale 	crval &= ~cr0_zeros_mask;
1855594db002STycho Nightingale 	vmcs_write(VMCS_GUEST_CR0, crval);
1856366f6083SPeter Grehan 
1857594db002STycho Nightingale 	if (regval & CR0_PG) {
185880a902efSPeter Grehan 		uint64_t efer, entry_ctls;
185980a902efSPeter Grehan 
186080a902efSPeter Grehan 		/*
186180a902efSPeter Grehan 		 * If CR0.PG is 1 and EFER.LME is 1 then EFER.LMA and
186280a902efSPeter Grehan 		 * the "IA-32e mode guest" bit in VM-entry control must be
186380a902efSPeter Grehan 		 * equal.
186480a902efSPeter Grehan 		 */
18653de83862SNeel Natu 		efer = vmcs_read(VMCS_GUEST_IA32_EFER);
186680a902efSPeter Grehan 		if (efer & EFER_LME) {
186780a902efSPeter Grehan 			efer |= EFER_LMA;
18683de83862SNeel Natu 			vmcs_write(VMCS_GUEST_IA32_EFER, efer);
18693de83862SNeel Natu 			entry_ctls = vmcs_read(VMCS_ENTRY_CTLS);
187080a902efSPeter Grehan 			entry_ctls |= VM_ENTRY_GUEST_LMA;
18713de83862SNeel Natu 			vmcs_write(VMCS_ENTRY_CTLS, entry_ctls);
187280a902efSPeter Grehan 		}
187380a902efSPeter Grehan 	}
187480a902efSPeter Grehan 
1875366f6083SPeter Grehan 	return (HANDLED);
1876366f6083SPeter Grehan }
1877366f6083SPeter Grehan 
1878594db002STycho Nightingale static int
1879594db002STycho Nightingale vmx_emulate_cr4_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1880594db002STycho Nightingale {
1881594db002STycho Nightingale 	uint64_t crval, regval;
1882594db002STycho Nightingale 
1883594db002STycho Nightingale 	/* We only handle mov to %cr4 at this time */
1884594db002STycho Nightingale 	if ((exitqual & 0xf0) != 0x00)
1885594db002STycho Nightingale 		return (UNHANDLED);
1886594db002STycho Nightingale 
1887594db002STycho Nightingale 	regval = vmx_get_guest_reg(vmx, vcpu, (exitqual >> 8) & 0xf);
1888594db002STycho Nightingale 
1889594db002STycho Nightingale 	vmcs_write(VMCS_CR4_SHADOW, regval);
1890594db002STycho Nightingale 
1891594db002STycho Nightingale 	crval = regval | cr4_ones_mask;
1892594db002STycho Nightingale 	crval &= ~cr4_zeros_mask;
1893594db002STycho Nightingale 	vmcs_write(VMCS_GUEST_CR4, crval);
1894594db002STycho Nightingale 
1895594db002STycho Nightingale 	return (HANDLED);
1896594db002STycho Nightingale }
1897594db002STycho Nightingale 
1898594db002STycho Nightingale static int
1899594db002STycho Nightingale vmx_emulate_cr8_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1900594db002STycho Nightingale {
1901051f2bd1SNeel Natu 	struct vlapic *vlapic;
1902051f2bd1SNeel Natu 	uint64_t cr8;
1903051f2bd1SNeel Natu 	int regnum;
1904594db002STycho Nightingale 
1905594db002STycho Nightingale 	/* We only handle mov %cr8 to/from a register at this time. */
1906594db002STycho Nightingale 	if ((exitqual & 0xe0) != 0x00) {
1907594db002STycho Nightingale 		return (UNHANDLED);
1908594db002STycho Nightingale 	}
1909594db002STycho Nightingale 
1910051f2bd1SNeel Natu 	vlapic = vm_lapic(vmx->vm, vcpu);
1911051f2bd1SNeel Natu 	regnum = (exitqual >> 8) & 0xf;
1912594db002STycho Nightingale 	if (exitqual & 0x10) {
1913051f2bd1SNeel Natu 		cr8 = vlapic_get_cr8(vlapic);
1914051f2bd1SNeel Natu 		vmx_set_guest_reg(vmx, vcpu, regnum, cr8);
1915594db002STycho Nightingale 	} else {
1916051f2bd1SNeel Natu 		cr8 = vmx_get_guest_reg(vmx, vcpu, regnum);
1917051f2bd1SNeel Natu 		vlapic_set_cr8(vlapic, cr8);
1918594db002STycho Nightingale 	}
1919594db002STycho Nightingale 
1920594db002STycho Nightingale 	return (HANDLED);
1921594db002STycho Nightingale }
1922594db002STycho Nightingale 
1923e4c8a13dSNeel Natu /*
1924e4c8a13dSNeel Natu  * From section "Guest Register State" in the Intel SDM: CPL = SS.DPL
1925e4c8a13dSNeel Natu  */
1926e4c8a13dSNeel Natu static int
1927e4c8a13dSNeel Natu vmx_cpl(void)
1928e4c8a13dSNeel Natu {
1929e4c8a13dSNeel Natu 	uint32_t ssar;
1930e4c8a13dSNeel Natu 
1931e4c8a13dSNeel Natu 	ssar = vmcs_read(VMCS_GUEST_SS_ACCESS_RIGHTS);
1932e4c8a13dSNeel Natu 	return ((ssar >> 5) & 0x3);
1933e4c8a13dSNeel Natu }
1934e4c8a13dSNeel Natu 
1935e813a873SNeel Natu static enum vm_cpu_mode
193600f3efe1SJohn Baldwin vmx_cpu_mode(void)
193700f3efe1SJohn Baldwin {
1938b301b9e2SNeel Natu 	uint32_t csar;
193900f3efe1SJohn Baldwin 
1940b301b9e2SNeel Natu 	if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LMA) {
1941b301b9e2SNeel Natu 		csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS);
1942b301b9e2SNeel Natu 		if (csar & 0x2000)
1943b301b9e2SNeel Natu 			return (CPU_MODE_64BIT);	/* CS.L = 1 */
194400f3efe1SJohn Baldwin 		else
194500f3efe1SJohn Baldwin 			return (CPU_MODE_COMPATIBILITY);
1946b301b9e2SNeel Natu 	} else if (vmcs_read(VMCS_GUEST_CR0) & CR0_PE) {
1947b301b9e2SNeel Natu 		return (CPU_MODE_PROTECTED);
1948b301b9e2SNeel Natu 	} else {
1949b301b9e2SNeel Natu 		return (CPU_MODE_REAL);
1950b301b9e2SNeel Natu 	}
195100f3efe1SJohn Baldwin }
195200f3efe1SJohn Baldwin 
1953e813a873SNeel Natu static enum vm_paging_mode
195400f3efe1SJohn Baldwin vmx_paging_mode(void)
195500f3efe1SJohn Baldwin {
1956f3eb12e4SKonstantin Belousov 	uint64_t cr4;
195700f3efe1SJohn Baldwin 
195800f3efe1SJohn Baldwin 	if (!(vmcs_read(VMCS_GUEST_CR0) & CR0_PG))
195900f3efe1SJohn Baldwin 		return (PAGING_MODE_FLAT);
1960f3eb12e4SKonstantin Belousov 	cr4 = vmcs_read(VMCS_GUEST_CR4);
1961f3eb12e4SKonstantin Belousov 	if (!(cr4 & CR4_PAE))
196200f3efe1SJohn Baldwin 		return (PAGING_MODE_32);
1963f3eb12e4SKonstantin Belousov 	if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LME) {
1964f3eb12e4SKonstantin Belousov 		if (!(cr4 & CR4_LA57))
196500f3efe1SJohn Baldwin 			return (PAGING_MODE_64);
1966f3eb12e4SKonstantin Belousov 		return (PAGING_MODE_64_LA57);
1967f3eb12e4SKonstantin Belousov 	} else
196800f3efe1SJohn Baldwin 		return (PAGING_MODE_PAE);
196900f3efe1SJohn Baldwin }
197000f3efe1SJohn Baldwin 
1971d17b5104SNeel Natu static uint64_t
1972d17b5104SNeel Natu inout_str_index(struct vmx *vmx, int vcpuid, int in)
1973d17b5104SNeel Natu {
1974d17b5104SNeel Natu 	uint64_t val;
19755c272efaSRobert Wing 	int error __diagused;
1976d17b5104SNeel Natu 	enum vm_reg_name reg;
1977d17b5104SNeel Natu 
1978d17b5104SNeel Natu 	reg = in ? VM_REG_GUEST_RDI : VM_REG_GUEST_RSI;
1979d17b5104SNeel Natu 	error = vmx_getreg(vmx, vcpuid, reg, &val);
1980d17b5104SNeel Natu 	KASSERT(error == 0, ("%s: vmx_getreg error %d", __func__, error));
1981d17b5104SNeel Natu 	return (val);
1982d17b5104SNeel Natu }
1983d17b5104SNeel Natu 
1984d17b5104SNeel Natu static uint64_t
1985d17b5104SNeel Natu inout_str_count(struct vmx *vmx, int vcpuid, int rep)
1986d17b5104SNeel Natu {
1987d17b5104SNeel Natu 	uint64_t val;
19885c272efaSRobert Wing 	int error __diagused;
1989d17b5104SNeel Natu 
1990d17b5104SNeel Natu 	if (rep) {
1991d17b5104SNeel Natu 		error = vmx_getreg(vmx, vcpuid, VM_REG_GUEST_RCX, &val);
1992d17b5104SNeel Natu 		KASSERT(!error, ("%s: vmx_getreg error %d", __func__, error));
1993d17b5104SNeel Natu 	} else {
1994d17b5104SNeel Natu 		val = 1;
1995d17b5104SNeel Natu 	}
1996d17b5104SNeel Natu 	return (val);
1997d17b5104SNeel Natu }
1998d17b5104SNeel Natu 
1999d17b5104SNeel Natu static int
2000d17b5104SNeel Natu inout_str_addrsize(uint32_t inst_info)
2001d17b5104SNeel Natu {
2002d17b5104SNeel Natu 	uint32_t size;
2003d17b5104SNeel Natu 
2004d17b5104SNeel Natu 	size = (inst_info >> 7) & 0x7;
2005d17b5104SNeel Natu 	switch (size) {
2006d17b5104SNeel Natu 	case 0:
2007d17b5104SNeel Natu 		return (2);	/* 16 bit */
2008d17b5104SNeel Natu 	case 1:
2009d17b5104SNeel Natu 		return (4);	/* 32 bit */
2010d17b5104SNeel Natu 	case 2:
2011d17b5104SNeel Natu 		return (8);	/* 64 bit */
2012d17b5104SNeel Natu 	default:
2013d17b5104SNeel Natu 		panic("%s: invalid size encoding %d", __func__, size);
2014d17b5104SNeel Natu 	}
2015d17b5104SNeel Natu }
2016d17b5104SNeel Natu 
2017d17b5104SNeel Natu static void
2018d17b5104SNeel Natu inout_str_seginfo(struct vmx *vmx, int vcpuid, uint32_t inst_info, int in,
2019d17b5104SNeel Natu     struct vm_inout_str *vis)
2020d17b5104SNeel Natu {
20215c272efaSRobert Wing 	int error __diagused, s;
2022d17b5104SNeel Natu 
2023d17b5104SNeel Natu 	if (in) {
2024d17b5104SNeel Natu 		vis->seg_name = VM_REG_GUEST_ES;
2025d17b5104SNeel Natu 	} else {
2026d17b5104SNeel Natu 		s = (inst_info >> 15) & 0x7;
2027d17b5104SNeel Natu 		vis->seg_name = vm_segment_name(s);
2028d17b5104SNeel Natu 	}
2029d17b5104SNeel Natu 
2030d17b5104SNeel Natu 	error = vmx_getdesc(vmx, vcpuid, vis->seg_name, &vis->seg_desc);
2031d17b5104SNeel Natu 	KASSERT(error == 0, ("%s: vmx_getdesc error %d", __func__, error));
2032d17b5104SNeel Natu }
2033d17b5104SNeel Natu 
2034e4c8a13dSNeel Natu static void
2035e813a873SNeel Natu vmx_paging_info(struct vm_guest_paging *paging)
2036e813a873SNeel Natu {
2037e813a873SNeel Natu 	paging->cr3 = vmcs_guest_cr3();
2038e813a873SNeel Natu 	paging->cpl = vmx_cpl();
2039e813a873SNeel Natu 	paging->cpu_mode = vmx_cpu_mode();
2040e813a873SNeel Natu 	paging->paging_mode = vmx_paging_mode();
2041e813a873SNeel Natu }
2042e813a873SNeel Natu 
2043e813a873SNeel Natu static void
2044e4c8a13dSNeel Natu vmexit_inst_emul(struct vm_exit *vmexit, uint64_t gpa, uint64_t gla)
2045e4c8a13dSNeel Natu {
2046f7a9f178SNeel Natu 	struct vm_guest_paging *paging;
2047f7a9f178SNeel Natu 	uint32_t csar;
2048f7a9f178SNeel Natu 
2049f7a9f178SNeel Natu 	paging = &vmexit->u.inst_emul.paging;
2050f7a9f178SNeel Natu 
2051e4c8a13dSNeel Natu 	vmexit->exitcode = VM_EXITCODE_INST_EMUL;
20521c73ea3eSNeel Natu 	vmexit->inst_length = 0;
2053e4c8a13dSNeel Natu 	vmexit->u.inst_emul.gpa = gpa;
2054e4c8a13dSNeel Natu 	vmexit->u.inst_emul.gla = gla;
2055f7a9f178SNeel Natu 	vmx_paging_info(paging);
2056f7a9f178SNeel Natu 	switch (paging->cpu_mode) {
2057e4f605eeSTycho Nightingale 	case CPU_MODE_REAL:
2058e4f605eeSTycho Nightingale 		vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE);
2059e4f605eeSTycho Nightingale 		vmexit->u.inst_emul.cs_d = 0;
2060e4f605eeSTycho Nightingale 		break;
2061f7a9f178SNeel Natu 	case CPU_MODE_PROTECTED:
2062f7a9f178SNeel Natu 	case CPU_MODE_COMPATIBILITY:
2063e4f605eeSTycho Nightingale 		vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE);
2064f7a9f178SNeel Natu 		csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS);
2065f7a9f178SNeel Natu 		vmexit->u.inst_emul.cs_d = SEG_DESC_DEF32(csar);
2066f7a9f178SNeel Natu 		break;
2067f7a9f178SNeel Natu 	default:
2068e4f605eeSTycho Nightingale 		vmexit->u.inst_emul.cs_base = 0;
2069f7a9f178SNeel Natu 		vmexit->u.inst_emul.cs_d = 0;
2070f7a9f178SNeel Natu 		break;
2071f7a9f178SNeel Natu 	}
2072c2a875f9SNeel Natu 	vie_init(&vmexit->u.inst_emul.vie, NULL, 0);
2073e4c8a13dSNeel Natu }
2074e4c8a13dSNeel Natu 
2075366f6083SPeter Grehan static int
2076318224bbSNeel Natu ept_fault_type(uint64_t ept_qual)
2077a2da7af6SNeel Natu {
2078318224bbSNeel Natu 	int fault_type;
2079a2da7af6SNeel Natu 
2080318224bbSNeel Natu 	if (ept_qual & EPT_VIOLATION_DATA_WRITE)
2081318224bbSNeel Natu 		fault_type = VM_PROT_WRITE;
2082318224bbSNeel Natu 	else if (ept_qual & EPT_VIOLATION_INST_FETCH)
2083318224bbSNeel Natu 		fault_type = VM_PROT_EXECUTE;
2084318224bbSNeel Natu 	else
2085318224bbSNeel Natu 		fault_type= VM_PROT_READ;
2086318224bbSNeel Natu 
2087318224bbSNeel Natu 	return (fault_type);
2088318224bbSNeel Natu }
2089318224bbSNeel Natu 
2090490d56c5SEd Maste static bool
2091318224bbSNeel Natu ept_emulation_fault(uint64_t ept_qual)
2092318224bbSNeel Natu {
2093318224bbSNeel Natu 	int read, write;
2094318224bbSNeel Natu 
2095318224bbSNeel Natu 	/* EPT fault on an instruction fetch doesn't make sense here */
2096a2da7af6SNeel Natu 	if (ept_qual & EPT_VIOLATION_INST_FETCH)
2097490d56c5SEd Maste 		return (false);
2098a2da7af6SNeel Natu 
2099318224bbSNeel Natu 	/* EPT fault must be a read fault or a write fault */
2100a2da7af6SNeel Natu 	read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0;
2101a2da7af6SNeel Natu 	write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0;
21023b2b0011SPeter Grehan 	if ((read | write) == 0)
2103490d56c5SEd Maste 		return (false);
2104a2da7af6SNeel Natu 
2105a2da7af6SNeel Natu 	/*
21063b2b0011SPeter Grehan 	 * The EPT violation must have been caused by accessing a
21073b2b0011SPeter Grehan 	 * guest-physical address that is a translation of a guest-linear
21083b2b0011SPeter Grehan 	 * address.
2109a2da7af6SNeel Natu 	 */
2110a2da7af6SNeel Natu 	if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 ||
2111a2da7af6SNeel Natu 	    (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) {
2112490d56c5SEd Maste 		return (false);
2113a2da7af6SNeel Natu 	}
2114a2da7af6SNeel Natu 
2115490d56c5SEd Maste 	return (true);
2116a2da7af6SNeel Natu }
2117a2da7af6SNeel Natu 
2118159dd56fSNeel Natu static __inline int
2119159dd56fSNeel Natu apic_access_virtualization(struct vmx *vmx, int vcpuid)
2120159dd56fSNeel Natu {
2121159dd56fSNeel Natu 	uint32_t proc_ctls2;
2122159dd56fSNeel Natu 
2123159dd56fSNeel Natu 	proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
2124159dd56fSNeel Natu 	return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) ? 1 : 0);
2125159dd56fSNeel Natu }
2126159dd56fSNeel Natu 
2127159dd56fSNeel Natu static __inline int
2128159dd56fSNeel Natu x2apic_virtualization(struct vmx *vmx, int vcpuid)
2129159dd56fSNeel Natu {
2130159dd56fSNeel Natu 	uint32_t proc_ctls2;
2131159dd56fSNeel Natu 
2132159dd56fSNeel Natu 	proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
2133159dd56fSNeel Natu 	return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_X2APIC_MODE) ? 1 : 0);
2134159dd56fSNeel Natu }
2135159dd56fSNeel Natu 
2136a2da7af6SNeel Natu static int
2137159dd56fSNeel Natu vmx_handle_apic_write(struct vmx *vmx, int vcpuid, struct vlapic *vlapic,
2138159dd56fSNeel Natu     uint64_t qual)
213988c4b8d1SNeel Natu {
214088c4b8d1SNeel Natu 	int error, handled, offset;
2141159dd56fSNeel Natu 	uint32_t *apic_regs, vector;
214288c4b8d1SNeel Natu 	bool retu;
214388c4b8d1SNeel Natu 
2144a0efd3fbSJohn Baldwin 	handled = HANDLED;
214588c4b8d1SNeel Natu 	offset = APIC_WRITE_OFFSET(qual);
2146159dd56fSNeel Natu 
2147159dd56fSNeel Natu 	if (!apic_access_virtualization(vmx, vcpuid)) {
2148159dd56fSNeel Natu 		/*
2149159dd56fSNeel Natu 		 * In general there should not be any APIC write VM-exits
2150159dd56fSNeel Natu 		 * unless APIC-access virtualization is enabled.
2151159dd56fSNeel Natu 		 *
2152159dd56fSNeel Natu 		 * However self-IPI virtualization can legitimately trigger
2153159dd56fSNeel Natu 		 * an APIC-write VM-exit so treat it specially.
2154159dd56fSNeel Natu 		 */
2155159dd56fSNeel Natu 		if (x2apic_virtualization(vmx, vcpuid) &&
2156159dd56fSNeel Natu 		    offset == APIC_OFFSET_SELF_IPI) {
2157159dd56fSNeel Natu 			apic_regs = (uint32_t *)(vlapic->apic_page);
2158159dd56fSNeel Natu 			vector = apic_regs[APIC_OFFSET_SELF_IPI / 4];
2159159dd56fSNeel Natu 			vlapic_self_ipi_handler(vlapic, vector);
2160159dd56fSNeel Natu 			return (HANDLED);
2161159dd56fSNeel Natu 		} else
2162159dd56fSNeel Natu 			return (UNHANDLED);
2163159dd56fSNeel Natu 	}
2164159dd56fSNeel Natu 
216588c4b8d1SNeel Natu 	switch (offset) {
216688c4b8d1SNeel Natu 	case APIC_OFFSET_ID:
216788c4b8d1SNeel Natu 		vlapic_id_write_handler(vlapic);
216888c4b8d1SNeel Natu 		break;
216988c4b8d1SNeel Natu 	case APIC_OFFSET_LDR:
217088c4b8d1SNeel Natu 		vlapic_ldr_write_handler(vlapic);
217188c4b8d1SNeel Natu 		break;
217288c4b8d1SNeel Natu 	case APIC_OFFSET_DFR:
217388c4b8d1SNeel Natu 		vlapic_dfr_write_handler(vlapic);
217488c4b8d1SNeel Natu 		break;
217588c4b8d1SNeel Natu 	case APIC_OFFSET_SVR:
217688c4b8d1SNeel Natu 		vlapic_svr_write_handler(vlapic);
217788c4b8d1SNeel Natu 		break;
217888c4b8d1SNeel Natu 	case APIC_OFFSET_ESR:
217988c4b8d1SNeel Natu 		vlapic_esr_write_handler(vlapic);
218088c4b8d1SNeel Natu 		break;
218188c4b8d1SNeel Natu 	case APIC_OFFSET_ICR_LOW:
218288c4b8d1SNeel Natu 		retu = false;
218388c4b8d1SNeel Natu 		error = vlapic_icrlo_write_handler(vlapic, &retu);
218488c4b8d1SNeel Natu 		if (error != 0 || retu)
2185a0efd3fbSJohn Baldwin 			handled = UNHANDLED;
218688c4b8d1SNeel Natu 		break;
218788c4b8d1SNeel Natu 	case APIC_OFFSET_CMCI_LVT:
218888c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
218988c4b8d1SNeel Natu 		vlapic_lvt_write_handler(vlapic, offset);
219088c4b8d1SNeel Natu 		break;
219188c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_ICR:
219288c4b8d1SNeel Natu 		vlapic_icrtmr_write_handler(vlapic);
219388c4b8d1SNeel Natu 		break;
219488c4b8d1SNeel Natu 	case APIC_OFFSET_TIMER_DCR:
219588c4b8d1SNeel Natu 		vlapic_dcr_write_handler(vlapic);
219688c4b8d1SNeel Natu 		break;
219788c4b8d1SNeel Natu 	default:
2198a0efd3fbSJohn Baldwin 		handled = UNHANDLED;
219988c4b8d1SNeel Natu 		break;
220088c4b8d1SNeel Natu 	}
220188c4b8d1SNeel Natu 	return (handled);
220288c4b8d1SNeel Natu }
220388c4b8d1SNeel Natu 
220488c4b8d1SNeel Natu static bool
2205159dd56fSNeel Natu apic_access_fault(struct vmx *vmx, int vcpuid, uint64_t gpa)
220688c4b8d1SNeel Natu {
220788c4b8d1SNeel Natu 
2208159dd56fSNeel Natu 	if (apic_access_virtualization(vmx, vcpuid) &&
220988c4b8d1SNeel Natu 	    (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE))
221088c4b8d1SNeel Natu 		return (true);
221188c4b8d1SNeel Natu 	else
221288c4b8d1SNeel Natu 		return (false);
221388c4b8d1SNeel Natu }
221488c4b8d1SNeel Natu 
221588c4b8d1SNeel Natu static int
221688c4b8d1SNeel Natu vmx_handle_apic_access(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
221788c4b8d1SNeel Natu {
221888c4b8d1SNeel Natu 	uint64_t qual;
221988c4b8d1SNeel Natu 	int access_type, offset, allowed;
222088c4b8d1SNeel Natu 
2221159dd56fSNeel Natu 	if (!apic_access_virtualization(vmx, vcpuid))
222288c4b8d1SNeel Natu 		return (UNHANDLED);
222388c4b8d1SNeel Natu 
222488c4b8d1SNeel Natu 	qual = vmexit->u.vmx.exit_qualification;
222588c4b8d1SNeel Natu 	access_type = APIC_ACCESS_TYPE(qual);
222688c4b8d1SNeel Natu 	offset = APIC_ACCESS_OFFSET(qual);
222788c4b8d1SNeel Natu 
222888c4b8d1SNeel Natu 	allowed = 0;
222988c4b8d1SNeel Natu 	if (access_type == 0) {
223088c4b8d1SNeel Natu 		/*
223188c4b8d1SNeel Natu 		 * Read data access to the following registers is expected.
223288c4b8d1SNeel Natu 		 */
223388c4b8d1SNeel Natu 		switch (offset) {
223488c4b8d1SNeel Natu 		case APIC_OFFSET_APR:
223588c4b8d1SNeel Natu 		case APIC_OFFSET_PPR:
223688c4b8d1SNeel Natu 		case APIC_OFFSET_RRR:
223788c4b8d1SNeel Natu 		case APIC_OFFSET_CMCI_LVT:
223888c4b8d1SNeel Natu 		case APIC_OFFSET_TIMER_CCR:
223988c4b8d1SNeel Natu 			allowed = 1;
224088c4b8d1SNeel Natu 			break;
224188c4b8d1SNeel Natu 		default:
224288c4b8d1SNeel Natu 			break;
224388c4b8d1SNeel Natu 		}
224488c4b8d1SNeel Natu 	} else if (access_type == 1) {
224588c4b8d1SNeel Natu 		/*
224688c4b8d1SNeel Natu 		 * Write data access to the following registers is expected.
224788c4b8d1SNeel Natu 		 */
224888c4b8d1SNeel Natu 		switch (offset) {
224988c4b8d1SNeel Natu 		case APIC_OFFSET_VER:
225088c4b8d1SNeel Natu 		case APIC_OFFSET_APR:
225188c4b8d1SNeel Natu 		case APIC_OFFSET_PPR:
225288c4b8d1SNeel Natu 		case APIC_OFFSET_RRR:
225388c4b8d1SNeel Natu 		case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
225488c4b8d1SNeel Natu 		case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
225588c4b8d1SNeel Natu 		case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
225688c4b8d1SNeel Natu 		case APIC_OFFSET_CMCI_LVT:
225788c4b8d1SNeel Natu 		case APIC_OFFSET_TIMER_CCR:
225888c4b8d1SNeel Natu 			allowed = 1;
225988c4b8d1SNeel Natu 			break;
226088c4b8d1SNeel Natu 		default:
226188c4b8d1SNeel Natu 			break;
226288c4b8d1SNeel Natu 		}
226388c4b8d1SNeel Natu 	}
226488c4b8d1SNeel Natu 
226588c4b8d1SNeel Natu 	if (allowed) {
2266e4c8a13dSNeel Natu 		vmexit_inst_emul(vmexit, DEFAULT_APIC_BASE + offset,
2267e4c8a13dSNeel Natu 		    VIE_INVALID_GLA);
226888c4b8d1SNeel Natu 	}
226988c4b8d1SNeel Natu 
227088c4b8d1SNeel Natu 	/*
227188c4b8d1SNeel Natu 	 * Regardless of whether the APIC-access is allowed this handler
227288c4b8d1SNeel Natu 	 * always returns UNHANDLED:
227388c4b8d1SNeel Natu 	 * - if the access is allowed then it is handled by emulating the
227488c4b8d1SNeel Natu 	 *   instruction that caused the VM-exit (outside the critical section)
227588c4b8d1SNeel Natu 	 * - if the access is not allowed then it will be converted to an
227688c4b8d1SNeel Natu 	 *   exitcode of VM_EXITCODE_VMX and will be dealt with in userland.
227788c4b8d1SNeel Natu 	 */
227888c4b8d1SNeel Natu 	return (UNHANDLED);
227988c4b8d1SNeel Natu }
228088c4b8d1SNeel Natu 
22813d5444c8SNeel Natu static enum task_switch_reason
22823d5444c8SNeel Natu vmx_task_switch_reason(uint64_t qual)
22833d5444c8SNeel Natu {
22843d5444c8SNeel Natu 	int reason;
22853d5444c8SNeel Natu 
22863d5444c8SNeel Natu 	reason = (qual >> 30) & 0x3;
22873d5444c8SNeel Natu 	switch (reason) {
22883d5444c8SNeel Natu 	case 0:
22893d5444c8SNeel Natu 		return (TSR_CALL);
22903d5444c8SNeel Natu 	case 1:
22913d5444c8SNeel Natu 		return (TSR_IRET);
22923d5444c8SNeel Natu 	case 2:
22933d5444c8SNeel Natu 		return (TSR_JMP);
22943d5444c8SNeel Natu 	case 3:
22953d5444c8SNeel Natu 		return (TSR_IDT_GATE);
22963d5444c8SNeel Natu 	default:
22973d5444c8SNeel Natu 		panic("%s: invalid reason %d", __func__, reason);
22983d5444c8SNeel Natu 	}
22993d5444c8SNeel Natu }
23003d5444c8SNeel Natu 
230188c4b8d1SNeel Natu static int
2302c3498942SNeel Natu emulate_wrmsr(struct vmx *vmx, int vcpuid, u_int num, uint64_t val, bool *retu)
2303c3498942SNeel Natu {
2304c3498942SNeel Natu 	int error;
2305c3498942SNeel Natu 
2306c3498942SNeel Natu 	if (lapic_msr(num))
2307c3498942SNeel Natu 		error = lapic_wrmsr(vmx->vm, vcpuid, num, val, retu);
2308c3498942SNeel Natu 	else
2309c3498942SNeel Natu 		error = vmx_wrmsr(vmx, vcpuid, num, val, retu);
2310c3498942SNeel Natu 
2311c3498942SNeel Natu 	return (error);
2312c3498942SNeel Natu }
2313c3498942SNeel Natu 
2314c3498942SNeel Natu static int
2315c3498942SNeel Natu emulate_rdmsr(struct vmx *vmx, int vcpuid, u_int num, bool *retu)
2316c3498942SNeel Natu {
2317c3498942SNeel Natu 	struct vmxctx *vmxctx;
2318c3498942SNeel Natu 	uint64_t result;
2319c3498942SNeel Natu 	uint32_t eax, edx;
2320c3498942SNeel Natu 	int error;
2321c3498942SNeel Natu 
2322c3498942SNeel Natu 	if (lapic_msr(num))
2323c3498942SNeel Natu 		error = lapic_rdmsr(vmx->vm, vcpuid, num, &result, retu);
2324c3498942SNeel Natu 	else
2325c3498942SNeel Natu 		error = vmx_rdmsr(vmx, vcpuid, num, &result, retu);
2326c3498942SNeel Natu 
2327c3498942SNeel Natu 	if (error == 0) {
2328c3498942SNeel Natu 		eax = result;
2329c3498942SNeel Natu 		vmxctx = &vmx->ctx[vcpuid];
2330c3498942SNeel Natu 		error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RAX, eax);
2331c3498942SNeel Natu 		KASSERT(error == 0, ("vmxctx_setreg(rax) error %d", error));
2332c3498942SNeel Natu 
2333c3498942SNeel Natu 		edx = result >> 32;
2334c3498942SNeel Natu 		error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RDX, edx);
2335c3498942SNeel Natu 		KASSERT(error == 0, ("vmxctx_setreg(rdx) error %d", error));
2336c3498942SNeel Natu 	}
2337c3498942SNeel Natu 
2338c3498942SNeel Natu 	return (error);
2339c3498942SNeel Natu }
2340c3498942SNeel Natu 
2341c3498942SNeel Natu static int
2342366f6083SPeter Grehan vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
2343366f6083SPeter Grehan {
2344c9c75df4SNeel Natu 	int error, errcode, errcode_valid, handled, in;
2345366f6083SPeter Grehan 	struct vmxctx *vmxctx;
234688c4b8d1SNeel Natu 	struct vlapic *vlapic;
2347d17b5104SNeel Natu 	struct vm_inout_str *vis;
23483d5444c8SNeel Natu 	struct vm_task_switch *ts;
2349d17b5104SNeel Natu 	uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, inst_info;
2350b0538143SNeel Natu 	uint32_t intr_type, intr_vec, reason;
2351091d4532SNeel Natu 	uint64_t exitintinfo, qual, gpa;
2352becd9849SNeel Natu 	bool retu;
2353366f6083SPeter Grehan 
2354160471d2SNeel Natu 	CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0);
2355c308b23bSNeel Natu 	CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0);
2356160471d2SNeel Natu 
2357a0efd3fbSJohn Baldwin 	handled = UNHANDLED;
2358366f6083SPeter Grehan 	vmxctx = &vmx->ctx[vcpu];
23590492757cSNeel Natu 
2360366f6083SPeter Grehan 	qual = vmexit->u.vmx.exit_qualification;
2361318224bbSNeel Natu 	reason = vmexit->u.vmx.exit_reason;
2362366f6083SPeter Grehan 	vmexit->exitcode = VM_EXITCODE_BOGUS;
2363366f6083SPeter Grehan 
236461592433SNeel Natu 	vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1);
23656ac73777STycho Nightingale 	SDT_PROBE3(vmm, vmx, exit, entry, vmx, vcpu, vmexit);
236661592433SNeel Natu 
2367318224bbSNeel Natu 	/*
2368b0538143SNeel Natu 	 * VM-entry failures during or after loading guest state.
2369b0538143SNeel Natu 	 *
2370b0538143SNeel Natu 	 * These VM-exits are uncommon but must be handled specially
2371b0538143SNeel Natu 	 * as most VM-exit fields are not populated as usual.
2372b0538143SNeel Natu 	 */
2373b0538143SNeel Natu 	if (__predict_false(reason == EXIT_REASON_MCE_DURING_ENTRY)) {
2374b0538143SNeel Natu 		VCPU_CTR0(vmx->vm, vcpu, "Handling MCE during VM-entry");
2375b0538143SNeel Natu 		__asm __volatile("int $18");
2376b0538143SNeel Natu 		return (1);
2377b0538143SNeel Natu 	}
2378b0538143SNeel Natu 
2379b0538143SNeel Natu 	/*
23803d5444c8SNeel Natu 	 * VM exits that can be triggered during event delivery need to
23813d5444c8SNeel Natu 	 * be handled specially by re-injecting the event if the IDT
23823d5444c8SNeel Natu 	 * vectoring information field's valid bit is set.
2383318224bbSNeel Natu 	 *
2384318224bbSNeel Natu 	 * See "Information for VM Exits During Event Delivery" in Intel SDM
2385318224bbSNeel Natu 	 * for details.
2386318224bbSNeel Natu 	 */
2387318224bbSNeel Natu 	idtvec_info = vmcs_idt_vectoring_info();
2388318224bbSNeel Natu 	if (idtvec_info & VMCS_IDT_VEC_VALID) {
2389318224bbSNeel Natu 		idtvec_info &= ~(1 << 12); /* clear undefined bit */
2390091d4532SNeel Natu 		exitintinfo = idtvec_info;
2391318224bbSNeel Natu 		if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
2392318224bbSNeel Natu 			idtvec_err = vmcs_idt_vectoring_err();
2393091d4532SNeel Natu 			exitintinfo |= (uint64_t)idtvec_err << 32;
2394318224bbSNeel Natu 		}
2395091d4532SNeel Natu 		error = vm_exit_intinfo(vmx->vm, vcpu, exitintinfo);
2396091d4532SNeel Natu 		KASSERT(error == 0, ("%s: vm_set_intinfo error %d",
2397091d4532SNeel Natu 		    __func__, error));
2398091d4532SNeel Natu 
2399160471d2SNeel Natu 		/*
2400160471d2SNeel Natu 		 * If 'virtual NMIs' are being used and the VM-exit
2401160471d2SNeel Natu 		 * happened while injecting an NMI during the previous
2402091d4532SNeel Natu 		 * VM-entry, then clear "blocking by NMI" in the
2403091d4532SNeel Natu 		 * Guest Interruptibility-State so the NMI can be
2404091d4532SNeel Natu 		 * reinjected on the subsequent VM-entry.
2405091d4532SNeel Natu 		 *
2406091d4532SNeel Natu 		 * However, if the NMI was being delivered through a task
2407091d4532SNeel Natu 		 * gate, then the new task must start execution with NMIs
2408091d4532SNeel Natu 		 * blocked so don't clear NMI blocking in this case.
2409160471d2SNeel Natu 		 */
2410091d4532SNeel Natu 		intr_type = idtvec_info & VMCS_INTR_T_MASK;
2411091d4532SNeel Natu 		if (intr_type == VMCS_INTR_T_NMI) {
2412091d4532SNeel Natu 			if (reason != EXIT_REASON_TASK_SWITCH)
2413e5a1d950SNeel Natu 				vmx_clear_nmi_blocking(vmx, vcpu);
2414091d4532SNeel Natu 			else
2415091d4532SNeel Natu 				vmx_assert_nmi_blocking(vmx, vcpu);
2416160471d2SNeel Natu 		}
2417091d4532SNeel Natu 
2418091d4532SNeel Natu 		/*
2419091d4532SNeel Natu 		 * Update VM-entry instruction length if the event being
2420091d4532SNeel Natu 		 * delivered was a software interrupt or software exception.
2421091d4532SNeel Natu 		 */
2422091d4532SNeel Natu 		if (intr_type == VMCS_INTR_T_SWINTR ||
2423091d4532SNeel Natu 		    intr_type == VMCS_INTR_T_PRIV_SWEXCEPTION ||
2424091d4532SNeel Natu 		    intr_type == VMCS_INTR_T_SWEXCEPTION) {
24253de83862SNeel Natu 			vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
2426318224bbSNeel Natu 		}
2427318224bbSNeel Natu 	}
2428318224bbSNeel Natu 
2429318224bbSNeel Natu 	switch (reason) {
24303d5444c8SNeel Natu 	case EXIT_REASON_TASK_SWITCH:
24313d5444c8SNeel Natu 		ts = &vmexit->u.task_switch;
24323d5444c8SNeel Natu 		ts->tsssel = qual & 0xffff;
24333d5444c8SNeel Natu 		ts->reason = vmx_task_switch_reason(qual);
24343d5444c8SNeel Natu 		ts->ext = 0;
24353d5444c8SNeel Natu 		ts->errcode_valid = 0;
24363d5444c8SNeel Natu 		vmx_paging_info(&ts->paging);
24373d5444c8SNeel Natu 		/*
24383d5444c8SNeel Natu 		 * If the task switch was due to a CALL, JMP, IRET, software
24393d5444c8SNeel Natu 		 * interrupt (INT n) or software exception (INT3, INTO),
24403d5444c8SNeel Natu 		 * then the saved %rip references the instruction that caused
24413d5444c8SNeel Natu 		 * the task switch. The instruction length field in the VMCS
24423d5444c8SNeel Natu 		 * is valid in this case.
24433d5444c8SNeel Natu 		 *
24443d5444c8SNeel Natu 		 * In all other cases (e.g., NMI, hardware exception) the
24453d5444c8SNeel Natu 		 * saved %rip is one that would have been saved in the old TSS
24463d5444c8SNeel Natu 		 * had the task switch completed normally so the instruction
24473d5444c8SNeel Natu 		 * length field is not needed in this case and is explicitly
24483d5444c8SNeel Natu 		 * set to 0.
24493d5444c8SNeel Natu 		 */
24503d5444c8SNeel Natu 		if (ts->reason == TSR_IDT_GATE) {
24513d5444c8SNeel Natu 			KASSERT(idtvec_info & VMCS_IDT_VEC_VALID,
2452091d4532SNeel Natu 			    ("invalid idtvec_info %#x for IDT task switch",
24533d5444c8SNeel Natu 			    idtvec_info));
24543d5444c8SNeel Natu 			intr_type = idtvec_info & VMCS_INTR_T_MASK;
24553d5444c8SNeel Natu 			if (intr_type != VMCS_INTR_T_SWINTR &&
24563d5444c8SNeel Natu 			    intr_type != VMCS_INTR_T_SWEXCEPTION &&
24573d5444c8SNeel Natu 			    intr_type != VMCS_INTR_T_PRIV_SWEXCEPTION) {
24583d5444c8SNeel Natu 				/* Task switch triggered by external event */
24593d5444c8SNeel Natu 				ts->ext = 1;
24603d5444c8SNeel Natu 				vmexit->inst_length = 0;
24613d5444c8SNeel Natu 				if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
24623d5444c8SNeel Natu 					ts->errcode_valid = 1;
24633d5444c8SNeel Natu 					ts->errcode = vmcs_idt_vectoring_err();
24643d5444c8SNeel Natu 				}
24653d5444c8SNeel Natu 			}
24663d5444c8SNeel Natu 		}
24673d5444c8SNeel Natu 		vmexit->exitcode = VM_EXITCODE_TASK_SWITCH;
24686ac73777STycho Nightingale 		SDT_PROBE4(vmm, vmx, exit, taskswitch, vmx, vcpu, vmexit, ts);
24693d5444c8SNeel Natu 		VCPU_CTR4(vmx->vm, vcpu, "task switch reason %d, tss 0x%04x, "
24703d5444c8SNeel Natu 		    "%s errcode 0x%016lx", ts->reason, ts->tsssel,
24713d5444c8SNeel Natu 		    ts->ext ? "external" : "internal",
24723d5444c8SNeel Natu 		    ((uint64_t)ts->errcode << 32) | ts->errcode_valid);
24733d5444c8SNeel Natu 		break;
2474366f6083SPeter Grehan 	case EXIT_REASON_CR_ACCESS:
2475b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CR_ACCESS, 1);
24766ac73777STycho Nightingale 		SDT_PROBE4(vmm, vmx, exit, craccess, vmx, vcpu, vmexit, qual);
2477594db002STycho Nightingale 		switch (qual & 0xf) {
2478594db002STycho Nightingale 		case 0:
2479594db002STycho Nightingale 			handled = vmx_emulate_cr0_access(vmx, vcpu, qual);
2480594db002STycho Nightingale 			break;
2481594db002STycho Nightingale 		case 4:
2482594db002STycho Nightingale 			handled = vmx_emulate_cr4_access(vmx, vcpu, qual);
2483594db002STycho Nightingale 			break;
2484594db002STycho Nightingale 		case 8:
2485594db002STycho Nightingale 			handled = vmx_emulate_cr8_access(vmx, vcpu, qual);
2486594db002STycho Nightingale 			break;
2487594db002STycho Nightingale 		}
2488366f6083SPeter Grehan 		break;
2489366f6083SPeter Grehan 	case EXIT_REASON_RDMSR:
2490b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RDMSR, 1);
2491becd9849SNeel Natu 		retu = false;
2492366f6083SPeter Grehan 		ecx = vmxctx->guest_rcx;
24932cb97c9dSNeel Natu 		VCPU_CTR1(vmx->vm, vcpu, "rdmsr 0x%08x", ecx);
24946ac73777STycho Nightingale 		SDT_PROBE4(vmm, vmx, exit, rdmsr, vmx, vcpu, vmexit, ecx);
2495c3498942SNeel Natu 		error = emulate_rdmsr(vmx, vcpu, ecx, &retu);
2496b42206f3SNeel Natu 		if (error) {
2497366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_RDMSR;
2498366f6083SPeter Grehan 			vmexit->u.msr.code = ecx;
2499becd9849SNeel Natu 		} else if (!retu) {
2500a0efd3fbSJohn Baldwin 			handled = HANDLED;
2501becd9849SNeel Natu 		} else {
2502becd9849SNeel Natu 			/* Return to userspace with a valid exitcode */
2503becd9849SNeel Natu 			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
2504c3498942SNeel Natu 			    ("emulate_rdmsr retu with bogus exitcode"));
2505becd9849SNeel Natu 		}
2506366f6083SPeter Grehan 		break;
2507366f6083SPeter Grehan 	case EXIT_REASON_WRMSR:
2508b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_WRMSR, 1);
2509becd9849SNeel Natu 		retu = false;
2510366f6083SPeter Grehan 		eax = vmxctx->guest_rax;
2511366f6083SPeter Grehan 		ecx = vmxctx->guest_rcx;
2512366f6083SPeter Grehan 		edx = vmxctx->guest_rdx;
25132cb97c9dSNeel Natu 		VCPU_CTR2(vmx->vm, vcpu, "wrmsr 0x%08x value 0x%016lx",
25142cb97c9dSNeel Natu 		    ecx, (uint64_t)edx << 32 | eax);
25156ac73777STycho Nightingale 		SDT_PROBE5(vmm, vmx, exit, wrmsr, vmx, vmexit, vcpu, ecx,
25166ac73777STycho Nightingale 		    (uint64_t)edx << 32 | eax);
2517c3498942SNeel Natu 		error = emulate_wrmsr(vmx, vcpu, ecx,
2518becd9849SNeel Natu 		    (uint64_t)edx << 32 | eax, &retu);
2519b42206f3SNeel Natu 		if (error) {
2520366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_WRMSR;
2521366f6083SPeter Grehan 			vmexit->u.msr.code = ecx;
2522366f6083SPeter Grehan 			vmexit->u.msr.wval = (uint64_t)edx << 32 | eax;
2523becd9849SNeel Natu 		} else if (!retu) {
2524a0efd3fbSJohn Baldwin 			handled = HANDLED;
2525becd9849SNeel Natu 		} else {
2526becd9849SNeel Natu 			/* Return to userspace with a valid exitcode */
2527becd9849SNeel Natu 			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
2528becd9849SNeel Natu 			    ("emulate_wrmsr retu with bogus exitcode"));
2529becd9849SNeel Natu 		}
2530366f6083SPeter Grehan 		break;
2531366f6083SPeter Grehan 	case EXIT_REASON_HLT:
2532f76fc5d4SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_HLT, 1);
25336ac73777STycho Nightingale 		SDT_PROBE3(vmm, vmx, exit, halt, vmx, vcpu, vmexit);
2534366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_HLT;
25353de83862SNeel Natu 		vmexit->u.hlt.rflags = vmcs_read(VMCS_GUEST_RFLAGS);
2536490768e2STycho Nightingale 		if (virtual_interrupt_delivery)
2537490768e2STycho Nightingale 			vmexit->u.hlt.intr_status =
2538490768e2STycho Nightingale 			    vmcs_read(VMCS_GUEST_INTR_STATUS);
2539490768e2STycho Nightingale 		else
2540490768e2STycho Nightingale 			vmexit->u.hlt.intr_status = 0;
2541366f6083SPeter Grehan 		break;
2542366f6083SPeter Grehan 	case EXIT_REASON_MTF:
2543b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_MTRAP, 1);
25446ac73777STycho Nightingale 		SDT_PROBE3(vmm, vmx, exit, mtrap, vmx, vcpu, vmexit);
2545366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_MTRAP;
2546c9c75df4SNeel Natu 		vmexit->inst_length = 0;
2547366f6083SPeter Grehan 		break;
2548366f6083SPeter Grehan 	case EXIT_REASON_PAUSE:
2549b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_PAUSE, 1);
25506ac73777STycho Nightingale 		SDT_PROBE3(vmm, vmx, exit, pause, vmx, vcpu, vmexit);
2551366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_PAUSE;
2552366f6083SPeter Grehan 		break;
2553366f6083SPeter Grehan 	case EXIT_REASON_INTR_WINDOW:
2554b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INTR_WINDOW, 1);
25556ac73777STycho Nightingale 		SDT_PROBE3(vmm, vmx, exit, intrwindow, vmx, vcpu, vmexit);
2556366f6083SPeter Grehan 		vmx_clear_int_window_exiting(vmx, vcpu);
2557b5aaf7b2SNeel Natu 		return (1);
2558366f6083SPeter Grehan 	case EXIT_REASON_EXT_INTR:
2559366f6083SPeter Grehan 		/*
2560366f6083SPeter Grehan 		 * External interrupts serve only to cause VM exits and allow
2561366f6083SPeter Grehan 		 * the host interrupt handler to run.
2562366f6083SPeter Grehan 		 *
2563366f6083SPeter Grehan 		 * If this external interrupt triggers a virtual interrupt
2564366f6083SPeter Grehan 		 * to a VM, then that state will be recorded by the
2565366f6083SPeter Grehan 		 * host interrupt handler in the VM's softc. We will inject
2566366f6083SPeter Grehan 		 * this virtual interrupt during the subsequent VM enter.
2567366f6083SPeter Grehan 		 */
2568f7d47425SNeel Natu 		intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
25696ac73777STycho Nightingale 		SDT_PROBE4(vmm, vmx, exit, interrupt,
25706ac73777STycho Nightingale 		    vmx, vcpu, vmexit, intr_info);
2571722b6744SJohn Baldwin 
2572722b6744SJohn Baldwin 		/*
2573722b6744SJohn Baldwin 		 * XXX: Ignore this exit if VMCS_INTR_VALID is not set.
2574ad3e3687SJohn Baldwin 		 * This appears to be a bug in VMware Fusion?
2575722b6744SJohn Baldwin 		 */
2576722b6744SJohn Baldwin 		if (!(intr_info & VMCS_INTR_VALID))
2577722b6744SJohn Baldwin 			return (1);
2578160471d2SNeel Natu 		KASSERT((intr_info & VMCS_INTR_VALID) != 0 &&
2579160471d2SNeel Natu 		    (intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_HWINTR,
2580f7d47425SNeel Natu 		    ("VM exit interruption info invalid: %#x", intr_info));
2581f7d47425SNeel Natu 		vmx_trigger_hostintr(intr_info & 0xff);
2582366f6083SPeter Grehan 
2583366f6083SPeter Grehan 		/*
2584366f6083SPeter Grehan 		 * This is special. We want to treat this as an 'handled'
2585366f6083SPeter Grehan 		 * VM-exit but not increment the instruction pointer.
2586366f6083SPeter Grehan 		 */
2587366f6083SPeter Grehan 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1);
2588366f6083SPeter Grehan 		return (1);
2589366f6083SPeter Grehan 	case EXIT_REASON_NMI_WINDOW:
25906ac73777STycho Nightingale 		SDT_PROBE3(vmm, vmx, exit, nmiwindow, vmx, vcpu, vmexit);
2591366f6083SPeter Grehan 		/* Exit to allow the pending virtual NMI to be injected */
259248b2d828SNeel Natu 		if (vm_nmi_pending(vmx->vm, vcpu))
259348b2d828SNeel Natu 			vmx_inject_nmi(vmx, vcpu);
2594366f6083SPeter Grehan 		vmx_clear_nmi_window_exiting(vmx, vcpu);
259548b2d828SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NMI_WINDOW, 1);
2596366f6083SPeter Grehan 		return (1);
2597366f6083SPeter Grehan 	case EXIT_REASON_INOUT:
2598b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INOUT, 1);
2599366f6083SPeter Grehan 		vmexit->exitcode = VM_EXITCODE_INOUT;
2600366f6083SPeter Grehan 		vmexit->u.inout.bytes = (qual & 0x7) + 1;
2601d17b5104SNeel Natu 		vmexit->u.inout.in = in = (qual & 0x8) ? 1 : 0;
2602366f6083SPeter Grehan 		vmexit->u.inout.string = (qual & 0x10) ? 1 : 0;
2603366f6083SPeter Grehan 		vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0;
2604366f6083SPeter Grehan 		vmexit->u.inout.port = (uint16_t)(qual >> 16);
2605366f6083SPeter Grehan 		vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax);
2606d17b5104SNeel Natu 		if (vmexit->u.inout.string) {
2607d17b5104SNeel Natu 			inst_info = vmcs_read(VMCS_EXIT_INSTRUCTION_INFO);
2608d17b5104SNeel Natu 			vmexit->exitcode = VM_EXITCODE_INOUT_STR;
2609d17b5104SNeel Natu 			vis = &vmexit->u.inout_str;
2610e813a873SNeel Natu 			vmx_paging_info(&vis->paging);
2611d17b5104SNeel Natu 			vis->rflags = vmcs_read(VMCS_GUEST_RFLAGS);
2612d17b5104SNeel Natu 			vis->cr0 = vmcs_read(VMCS_GUEST_CR0);
2613d17b5104SNeel Natu 			vis->index = inout_str_index(vmx, vcpu, in);
2614d17b5104SNeel Natu 			vis->count = inout_str_count(vmx, vcpu, vis->inout.rep);
2615d17b5104SNeel Natu 			vis->addrsize = inout_str_addrsize(inst_info);
2616d17b5104SNeel Natu 			inout_str_seginfo(vmx, vcpu, inst_info, in, vis);
2617762fd208STycho Nightingale 		}
26186ac73777STycho Nightingale 		SDT_PROBE3(vmm, vmx, exit, inout, vmx, vcpu, vmexit);
2619366f6083SPeter Grehan 		break;
2620366f6083SPeter Grehan 	case EXIT_REASON_CPUID:
2621b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CPUID, 1);
26226ac73777STycho Nightingale 		SDT_PROBE3(vmm, vmx, exit, cpuid, vmx, vcpu, vmexit);
2623a2da7af6SNeel Natu 		handled = vmx_handle_cpuid(vmx->vm, vcpu, vmxctx);
2624366f6083SPeter Grehan 		break;
2625e5a1d950SNeel Natu 	case EXIT_REASON_EXCEPTION:
2626c308b23bSNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXCEPTION, 1);
2627e5a1d950SNeel Natu 		intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
2628e5a1d950SNeel Natu 		KASSERT((intr_info & VMCS_INTR_VALID) != 0,
2629e5a1d950SNeel Natu 		    ("VM exit interruption info invalid: %#x", intr_info));
2630c308b23bSNeel Natu 
2631b0538143SNeel Natu 		intr_vec = intr_info & 0xff;
2632b0538143SNeel Natu 		intr_type = intr_info & VMCS_INTR_T_MASK;
2633b0538143SNeel Natu 
2634e5a1d950SNeel Natu 		/*
2635e5a1d950SNeel Natu 		 * If Virtual NMIs control is 1 and the VM-exit is due to a
2636e5a1d950SNeel Natu 		 * fault encountered during the execution of IRET then we must
2637e5a1d950SNeel Natu 		 * restore the state of "virtual-NMI blocking" before resuming
2638e5a1d950SNeel Natu 		 * the guest.
2639e5a1d950SNeel Natu 		 *
2640e5a1d950SNeel Natu 		 * See "Resuming Guest Software after Handling an Exception".
2641091d4532SNeel Natu 		 * See "Information for VM Exits Due to Vectored Events".
2642e5a1d950SNeel Natu 		 */
2643e5a1d950SNeel Natu 		if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
2644b0538143SNeel Natu 		    (intr_vec != IDT_DF) &&
2645e5a1d950SNeel Natu 		    (intr_info & EXIT_QUAL_NMIUDTI) != 0)
2646e5a1d950SNeel Natu 			vmx_restore_nmi_blocking(vmx, vcpu);
2647c308b23bSNeel Natu 
2648c308b23bSNeel Natu 		/*
264962fbd7c2SNeel Natu 		 * The NMI has already been handled in vmx_exit_handle_nmi().
2650c308b23bSNeel Natu 		 */
2651b0538143SNeel Natu 		if (intr_type == VMCS_INTR_T_NMI)
2652c308b23bSNeel Natu 			return (1);
2653b0538143SNeel Natu 
2654b0538143SNeel Natu 		/*
2655b0538143SNeel Natu 		 * Call the machine check handler by hand. Also don't reflect
2656b0538143SNeel Natu 		 * the machine check back into the guest.
2657b0538143SNeel Natu 		 */
2658b0538143SNeel Natu 		if (intr_vec == IDT_MC) {
2659b0538143SNeel Natu 			VCPU_CTR0(vmx->vm, vcpu, "Vectoring to MCE handler");
2660b0538143SNeel Natu 			__asm __volatile("int $18");
2661b0538143SNeel Natu 			return (1);
2662b0538143SNeel Natu 		}
2663b0538143SNeel Natu 
2664cbd03a9dSJohn Baldwin 		/*
2665cbd03a9dSJohn Baldwin 		 * If the hypervisor has requested user exits for
2666cbd03a9dSJohn Baldwin 		 * debug exceptions, bounce them out to userland.
2667cbd03a9dSJohn Baldwin 		 */
2668cbd03a9dSJohn Baldwin 		if (intr_type == VMCS_INTR_T_SWEXCEPTION && intr_vec == IDT_BP &&
2669cbd03a9dSJohn Baldwin 		    (vmx->cap[vcpu].set & (1 << VM_CAP_BPT_EXIT))) {
2670cbd03a9dSJohn Baldwin 			vmexit->exitcode = VM_EXITCODE_BPT;
2671cbd03a9dSJohn Baldwin 			vmexit->u.bpt.inst_length = vmexit->inst_length;
2672cbd03a9dSJohn Baldwin 			vmexit->inst_length = 0;
2673cbd03a9dSJohn Baldwin 			break;
2674cbd03a9dSJohn Baldwin 		}
2675cbd03a9dSJohn Baldwin 
2676b0538143SNeel Natu 		if (intr_vec == IDT_PF) {
2677b0538143SNeel Natu 			error = vmxctx_setreg(vmxctx, VM_REG_GUEST_CR2, qual);
2678b0538143SNeel Natu 			KASSERT(error == 0, ("%s: vmxctx_setreg(cr2) error %d",
2679b0538143SNeel Natu 			    __func__, error));
2680b0538143SNeel Natu 		}
2681b0538143SNeel Natu 
2682b0538143SNeel Natu 		/*
2683b0538143SNeel Natu 		 * Software exceptions exhibit trap-like behavior. This in
2684b0538143SNeel Natu 		 * turn requires populating the VM-entry instruction length
2685b0538143SNeel Natu 		 * so that the %rip in the trap frame is past the INT3/INTO
2686b0538143SNeel Natu 		 * instruction.
2687b0538143SNeel Natu 		 */
2688b0538143SNeel Natu 		if (intr_type == VMCS_INTR_T_SWEXCEPTION)
2689b0538143SNeel Natu 			vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
2690b0538143SNeel Natu 
2691b0538143SNeel Natu 		/* Reflect all other exceptions back into the guest */
2692c9c75df4SNeel Natu 		errcode_valid = errcode = 0;
2693b0538143SNeel Natu 		if (intr_info & VMCS_INTR_DEL_ERRCODE) {
2694c9c75df4SNeel Natu 			errcode_valid = 1;
2695c9c75df4SNeel Natu 			errcode = vmcs_read(VMCS_EXIT_INTR_ERRCODE);
2696b0538143SNeel Natu 		}
2697b0538143SNeel Natu 		VCPU_CTR2(vmx->vm, vcpu, "Reflecting exception %d/%#x into "
2698c9c75df4SNeel Natu 		    "the guest", intr_vec, errcode);
26996ac73777STycho Nightingale 		SDT_PROBE5(vmm, vmx, exit, exception,
27006ac73777STycho Nightingale 		    vmx, vcpu, vmexit, intr_vec, errcode);
2701c9c75df4SNeel Natu 		error = vm_inject_exception(vmx->vm, vcpu, intr_vec,
2702c9c75df4SNeel Natu 		    errcode_valid, errcode, 0);
2703b0538143SNeel Natu 		KASSERT(error == 0, ("%s: vm_inject_exception error %d",
2704b0538143SNeel Natu 		    __func__, error));
2705b0538143SNeel Natu 		return (1);
2706b0538143SNeel Natu 
2707cd942e0fSPeter Grehan 	case EXIT_REASON_EPT_FAULT:
2708318224bbSNeel Natu 		/*
2709318224bbSNeel Natu 		 * If 'gpa' lies within the address space allocated to
2710318224bbSNeel Natu 		 * memory then this must be a nested page fault otherwise
2711318224bbSNeel Natu 		 * this must be an instruction that accesses MMIO space.
2712318224bbSNeel Natu 		 */
2713a2da7af6SNeel Natu 		gpa = vmcs_gpa();
27149b1aa8d6SNeel Natu 		if (vm_mem_allocated(vmx->vm, vcpu, gpa) ||
2715159dd56fSNeel Natu 		    apic_access_fault(vmx, vcpu, gpa)) {
2716cd942e0fSPeter Grehan 			vmexit->exitcode = VM_EXITCODE_PAGING;
2717d087a399SNeel Natu 			vmexit->inst_length = 0;
271813ec9371SPeter Grehan 			vmexit->u.paging.gpa = gpa;
2719318224bbSNeel Natu 			vmexit->u.paging.fault_type = ept_fault_type(qual);
2720bf73979dSNeel Natu 			vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NESTED_FAULT, 1);
27216ac73777STycho Nightingale 			SDT_PROBE5(vmm, vmx, exit, nestedfault,
27226ac73777STycho Nightingale 			    vmx, vcpu, vmexit, gpa, qual);
2723318224bbSNeel Natu 		} else if (ept_emulation_fault(qual)) {
2724e4c8a13dSNeel Natu 			vmexit_inst_emul(vmexit, gpa, vmcs_gla());
2725bf73979dSNeel Natu 			vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INST_EMUL, 1);
27266ac73777STycho Nightingale 			SDT_PROBE4(vmm, vmx, exit, mmiofault,
27276ac73777STycho Nightingale 			    vmx, vcpu, vmexit, gpa);
2728a2da7af6SNeel Natu 		}
2729e5a1d950SNeel Natu 		/*
2730e5a1d950SNeel Natu 		 * If Virtual NMIs control is 1 and the VM-exit is due to an
2731e5a1d950SNeel Natu 		 * EPT fault during the execution of IRET then we must restore
2732e5a1d950SNeel Natu 		 * the state of "virtual-NMI blocking" before resuming.
2733e5a1d950SNeel Natu 		 *
2734e5a1d950SNeel Natu 		 * See description of "NMI unblocking due to IRET" in
2735e5a1d950SNeel Natu 		 * "Exit Qualification for EPT Violations".
2736e5a1d950SNeel Natu 		 */
2737e5a1d950SNeel Natu 		if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
2738e5a1d950SNeel Natu 		    (qual & EXIT_QUAL_NMIUDTI) != 0)
2739e5a1d950SNeel Natu 			vmx_restore_nmi_blocking(vmx, vcpu);
2740cd942e0fSPeter Grehan 		break;
274130b94db8SNeel Natu 	case EXIT_REASON_VIRTUALIZED_EOI:
274230b94db8SNeel Natu 		vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI;
274330b94db8SNeel Natu 		vmexit->u.ioapic_eoi.vector = qual & 0xFF;
27446ac73777STycho Nightingale 		SDT_PROBE3(vmm, vmx, exit, eoi, vmx, vcpu, vmexit);
274530b94db8SNeel Natu 		vmexit->inst_length = 0;	/* trap-like */
274630b94db8SNeel Natu 		break;
274788c4b8d1SNeel Natu 	case EXIT_REASON_APIC_ACCESS:
27486ac73777STycho Nightingale 		SDT_PROBE3(vmm, vmx, exit, apicaccess, vmx, vcpu, vmexit);
274988c4b8d1SNeel Natu 		handled = vmx_handle_apic_access(vmx, vcpu, vmexit);
275088c4b8d1SNeel Natu 		break;
275188c4b8d1SNeel Natu 	case EXIT_REASON_APIC_WRITE:
275288c4b8d1SNeel Natu 		/*
275388c4b8d1SNeel Natu 		 * APIC-write VM exit is trap-like so the %rip is already
275488c4b8d1SNeel Natu 		 * pointing to the next instruction.
275588c4b8d1SNeel Natu 		 */
275688c4b8d1SNeel Natu 		vmexit->inst_length = 0;
275788c4b8d1SNeel Natu 		vlapic = vm_lapic(vmx->vm, vcpu);
27586ac73777STycho Nightingale 		SDT_PROBE4(vmm, vmx, exit, apicwrite,
27596ac73777STycho Nightingale 		    vmx, vcpu, vmexit, vlapic);
2760159dd56fSNeel Natu 		handled = vmx_handle_apic_write(vmx, vcpu, vlapic, qual);
276188c4b8d1SNeel Natu 		break;
2762abb023fbSJohn Baldwin 	case EXIT_REASON_XSETBV:
27636ac73777STycho Nightingale 		SDT_PROBE3(vmm, vmx, exit, xsetbv, vmx, vcpu, vmexit);
2764a0efd3fbSJohn Baldwin 		handled = vmx_emulate_xsetbv(vmx, vcpu, vmexit);
2765abb023fbSJohn Baldwin 		break;
276665145c7fSNeel Natu 	case EXIT_REASON_MONITOR:
27676ac73777STycho Nightingale 		SDT_PROBE3(vmm, vmx, exit, monitor, vmx, vcpu, vmexit);
276865145c7fSNeel Natu 		vmexit->exitcode = VM_EXITCODE_MONITOR;
276965145c7fSNeel Natu 		break;
277065145c7fSNeel Natu 	case EXIT_REASON_MWAIT:
27716ac73777STycho Nightingale 		SDT_PROBE3(vmm, vmx, exit, mwait, vmx, vcpu, vmexit);
277265145c7fSNeel Natu 		vmexit->exitcode = VM_EXITCODE_MWAIT;
277365145c7fSNeel Natu 		break;
27741bc51badSMichael Reifenberger 	case EXIT_REASON_TPR:
27751bc51badSMichael Reifenberger 		vlapic = vm_lapic(vmx->vm, vcpu);
27761bc51badSMichael Reifenberger 		vlapic_sync_tpr(vlapic);
27771bc51badSMichael Reifenberger 		vmexit->inst_length = 0;
27781bc51badSMichael Reifenberger 		handled = HANDLED;
27791bc51badSMichael Reifenberger 		break;
278027d26457SAndrew Turner 	case EXIT_REASON_VMCALL:
278127d26457SAndrew Turner 	case EXIT_REASON_VMCLEAR:
278227d26457SAndrew Turner 	case EXIT_REASON_VMLAUNCH:
278327d26457SAndrew Turner 	case EXIT_REASON_VMPTRLD:
278427d26457SAndrew Turner 	case EXIT_REASON_VMPTRST:
278527d26457SAndrew Turner 	case EXIT_REASON_VMREAD:
278627d26457SAndrew Turner 	case EXIT_REASON_VMRESUME:
278727d26457SAndrew Turner 	case EXIT_REASON_VMWRITE:
278827d26457SAndrew Turner 	case EXIT_REASON_VMXOFF:
278927d26457SAndrew Turner 	case EXIT_REASON_VMXON:
279027d26457SAndrew Turner 		SDT_PROBE3(vmm, vmx, exit, vminsn, vmx, vcpu, vmexit);
279127d26457SAndrew Turner 		vmexit->exitcode = VM_EXITCODE_VMINSN;
279227d26457SAndrew Turner 		break;
2793*3ba952e1SCorvin Köhne 	case EXIT_REASON_WBINVD:
2794*3ba952e1SCorvin Köhne 		/* ignore WBINVD */
2795*3ba952e1SCorvin Köhne 		handled = HANDLED;
2796*3ba952e1SCorvin Köhne 		break;
2797366f6083SPeter Grehan 	default:
27986ac73777STycho Nightingale 		SDT_PROBE4(vmm, vmx, exit, unknown,
27996ac73777STycho Nightingale 		    vmx, vcpu, vmexit, reason);
2800b5aaf7b2SNeel Natu 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_UNKNOWN, 1);
2801366f6083SPeter Grehan 		break;
2802366f6083SPeter Grehan 	}
2803366f6083SPeter Grehan 
2804366f6083SPeter Grehan 	if (handled) {
2805366f6083SPeter Grehan 		/*
2806366f6083SPeter Grehan 		 * It is possible that control is returned to userland
2807366f6083SPeter Grehan 		 * even though we were able to handle the VM exit in the
2808eeefa4e4SNeel Natu 		 * kernel.
2809366f6083SPeter Grehan 		 *
2810366f6083SPeter Grehan 		 * In such a case we want to make sure that the userland
2811366f6083SPeter Grehan 		 * restarts guest execution at the instruction *after*
2812366f6083SPeter Grehan 		 * the one we just processed. Therefore we update the
2813366f6083SPeter Grehan 		 * guest rip in the VMCS and in 'vmexit'.
2814366f6083SPeter Grehan 		 */
2815366f6083SPeter Grehan 		vmexit->rip += vmexit->inst_length;
2816366f6083SPeter Grehan 		vmexit->inst_length = 0;
28173de83862SNeel Natu 		vmcs_write(VMCS_GUEST_RIP, vmexit->rip);
2818366f6083SPeter Grehan 	} else {
2819366f6083SPeter Grehan 		if (vmexit->exitcode == VM_EXITCODE_BOGUS) {
2820366f6083SPeter Grehan 			/*
2821366f6083SPeter Grehan 			 * If this VM exit was not claimed by anybody then
2822366f6083SPeter Grehan 			 * treat it as a generic VMX exit.
2823366f6083SPeter Grehan 			 */
2824366f6083SPeter Grehan 			vmexit->exitcode = VM_EXITCODE_VMX;
28250492757cSNeel Natu 			vmexit->u.vmx.status = VM_SUCCESS;
2826c308b23bSNeel Natu 			vmexit->u.vmx.inst_type = 0;
2827c308b23bSNeel Natu 			vmexit->u.vmx.inst_error = 0;
2828366f6083SPeter Grehan 		} else {
2829366f6083SPeter Grehan 			/*
2830366f6083SPeter Grehan 			 * The exitcode and collateral have been populated.
2831366f6083SPeter Grehan 			 * The VM exit will be processed further in userland.
2832366f6083SPeter Grehan 			 */
2833366f6083SPeter Grehan 		}
2834366f6083SPeter Grehan 	}
28356ac73777STycho Nightingale 
28366ac73777STycho Nightingale 	SDT_PROBE4(vmm, vmx, exit, return,
28376ac73777STycho Nightingale 	    vmx, vcpu, vmexit, handled);
2838366f6083SPeter Grehan 	return (handled);
2839366f6083SPeter Grehan }
2840366f6083SPeter Grehan 
284140487465SNeel Natu static __inline void
28420492757cSNeel Natu vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit)
28430492757cSNeel Natu {
28440492757cSNeel Natu 
28450492757cSNeel Natu 	KASSERT(vmxctx->inst_fail_status != VM_SUCCESS,
28460492757cSNeel Natu 	    ("vmx_exit_inst_error: invalid inst_fail_status %d",
28470492757cSNeel Natu 	    vmxctx->inst_fail_status));
28480492757cSNeel Natu 
28490492757cSNeel Natu 	vmexit->inst_length = 0;
28500492757cSNeel Natu 	vmexit->exitcode = VM_EXITCODE_VMX;
28510492757cSNeel Natu 	vmexit->u.vmx.status = vmxctx->inst_fail_status;
28520492757cSNeel Natu 	vmexit->u.vmx.inst_error = vmcs_instruction_error();
28530492757cSNeel Natu 	vmexit->u.vmx.exit_reason = ~0;
28540492757cSNeel Natu 	vmexit->u.vmx.exit_qualification = ~0;
28550492757cSNeel Natu 
28560492757cSNeel Natu 	switch (rc) {
28570492757cSNeel Natu 	case VMX_VMRESUME_ERROR:
28580492757cSNeel Natu 	case VMX_VMLAUNCH_ERROR:
28590492757cSNeel Natu 		vmexit->u.vmx.inst_type = rc;
28600492757cSNeel Natu 		break;
28610492757cSNeel Natu 	default:
28620492757cSNeel Natu 		panic("vm_exit_inst_error: vmx_enter_guest returned %d", rc);
28630492757cSNeel Natu 	}
28640492757cSNeel Natu }
28650492757cSNeel Natu 
286662fbd7c2SNeel Natu /*
286762fbd7c2SNeel Natu  * If the NMI-exiting VM execution control is set to '1' then an NMI in
286862fbd7c2SNeel Natu  * non-root operation causes a VM-exit. NMI blocking is in effect so it is
286962fbd7c2SNeel Natu  * sufficient to simply vector to the NMI handler via a software interrupt.
287062fbd7c2SNeel Natu  * However, this must be done before maskable interrupts are enabled
287162fbd7c2SNeel Natu  * otherwise the "iret" issued by an interrupt handler will incorrectly
287262fbd7c2SNeel Natu  * clear NMI blocking.
287362fbd7c2SNeel Natu  */
287462fbd7c2SNeel Natu static __inline void
287562fbd7c2SNeel Natu vmx_exit_handle_nmi(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
287662fbd7c2SNeel Natu {
287762fbd7c2SNeel Natu 	uint32_t intr_info;
287862fbd7c2SNeel Natu 
287962fbd7c2SNeel Natu 	KASSERT((read_rflags() & PSL_I) == 0, ("interrupts enabled"));
288062fbd7c2SNeel Natu 
288162fbd7c2SNeel Natu 	if (vmexit->u.vmx.exit_reason != EXIT_REASON_EXCEPTION)
288262fbd7c2SNeel Natu 		return;
288362fbd7c2SNeel Natu 
288462fbd7c2SNeel Natu 	intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
288562fbd7c2SNeel Natu 	KASSERT((intr_info & VMCS_INTR_VALID) != 0,
288662fbd7c2SNeel Natu 	    ("VM exit interruption info invalid: %#x", intr_info));
288762fbd7c2SNeel Natu 
288862fbd7c2SNeel Natu 	if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI) {
288962fbd7c2SNeel Natu 		KASSERT((intr_info & 0xff) == IDT_NMI, ("VM exit due "
289062fbd7c2SNeel Natu 		    "to NMI has invalid vector: %#x", intr_info));
289162fbd7c2SNeel Natu 		VCPU_CTR0(vmx->vm, vcpuid, "Vectoring to NMI handler");
289262fbd7c2SNeel Natu 		__asm __volatile("int $2");
289362fbd7c2SNeel Natu 	}
289462fbd7c2SNeel Natu }
289562fbd7c2SNeel Natu 
289665eefbe4SJohn Baldwin static __inline void
289765eefbe4SJohn Baldwin vmx_dr_enter_guest(struct vmxctx *vmxctx)
289865eefbe4SJohn Baldwin {
289965eefbe4SJohn Baldwin 	register_t rflags;
290065eefbe4SJohn Baldwin 
290165eefbe4SJohn Baldwin 	/* Save host control debug registers. */
290265eefbe4SJohn Baldwin 	vmxctx->host_dr7 = rdr7();
290365eefbe4SJohn Baldwin 	vmxctx->host_debugctl = rdmsr(MSR_DEBUGCTLMSR);
290465eefbe4SJohn Baldwin 
290565eefbe4SJohn Baldwin 	/*
290665eefbe4SJohn Baldwin 	 * Disable debugging in DR7 and DEBUGCTL to avoid triggering
290765eefbe4SJohn Baldwin 	 * exceptions in the host based on the guest DRx values.  The
290865eefbe4SJohn Baldwin 	 * guest DR7 and DEBUGCTL are saved/restored in the VMCS.
290965eefbe4SJohn Baldwin 	 */
291065eefbe4SJohn Baldwin 	load_dr7(0);
291165eefbe4SJohn Baldwin 	wrmsr(MSR_DEBUGCTLMSR, 0);
291265eefbe4SJohn Baldwin 
291365eefbe4SJohn Baldwin 	/*
291465eefbe4SJohn Baldwin 	 * Disable single stepping the kernel to avoid corrupting the
291565eefbe4SJohn Baldwin 	 * guest DR6.  A debugger might still be able to corrupt the
291665eefbe4SJohn Baldwin 	 * guest DR6 by setting a breakpoint after this point and then
291765eefbe4SJohn Baldwin 	 * single stepping.
291865eefbe4SJohn Baldwin 	 */
291965eefbe4SJohn Baldwin 	rflags = read_rflags();
292065eefbe4SJohn Baldwin 	vmxctx->host_tf = rflags & PSL_T;
292165eefbe4SJohn Baldwin 	write_rflags(rflags & ~PSL_T);
292265eefbe4SJohn Baldwin 
292365eefbe4SJohn Baldwin 	/* Save host debug registers. */
292465eefbe4SJohn Baldwin 	vmxctx->host_dr0 = rdr0();
292565eefbe4SJohn Baldwin 	vmxctx->host_dr1 = rdr1();
292665eefbe4SJohn Baldwin 	vmxctx->host_dr2 = rdr2();
292765eefbe4SJohn Baldwin 	vmxctx->host_dr3 = rdr3();
292865eefbe4SJohn Baldwin 	vmxctx->host_dr6 = rdr6();
292965eefbe4SJohn Baldwin 
293065eefbe4SJohn Baldwin 	/* Restore guest debug registers. */
293165eefbe4SJohn Baldwin 	load_dr0(vmxctx->guest_dr0);
293265eefbe4SJohn Baldwin 	load_dr1(vmxctx->guest_dr1);
293365eefbe4SJohn Baldwin 	load_dr2(vmxctx->guest_dr2);
293465eefbe4SJohn Baldwin 	load_dr3(vmxctx->guest_dr3);
293565eefbe4SJohn Baldwin 	load_dr6(vmxctx->guest_dr6);
293665eefbe4SJohn Baldwin }
293765eefbe4SJohn Baldwin 
293865eefbe4SJohn Baldwin static __inline void
293965eefbe4SJohn Baldwin vmx_dr_leave_guest(struct vmxctx *vmxctx)
294065eefbe4SJohn Baldwin {
294165eefbe4SJohn Baldwin 
294265eefbe4SJohn Baldwin 	/* Save guest debug registers. */
294365eefbe4SJohn Baldwin 	vmxctx->guest_dr0 = rdr0();
294465eefbe4SJohn Baldwin 	vmxctx->guest_dr1 = rdr1();
294565eefbe4SJohn Baldwin 	vmxctx->guest_dr2 = rdr2();
294665eefbe4SJohn Baldwin 	vmxctx->guest_dr3 = rdr3();
294765eefbe4SJohn Baldwin 	vmxctx->guest_dr6 = rdr6();
294865eefbe4SJohn Baldwin 
294965eefbe4SJohn Baldwin 	/*
295065eefbe4SJohn Baldwin 	 * Restore host debug registers.  Restore DR7, DEBUGCTL, and
295165eefbe4SJohn Baldwin 	 * PSL_T last.
295265eefbe4SJohn Baldwin 	 */
295365eefbe4SJohn Baldwin 	load_dr0(vmxctx->host_dr0);
295465eefbe4SJohn Baldwin 	load_dr1(vmxctx->host_dr1);
295565eefbe4SJohn Baldwin 	load_dr2(vmxctx->host_dr2);
295665eefbe4SJohn Baldwin 	load_dr3(vmxctx->host_dr3);
295765eefbe4SJohn Baldwin 	load_dr6(vmxctx->host_dr6);
295865eefbe4SJohn Baldwin 	wrmsr(MSR_DEBUGCTLMSR, vmxctx->host_debugctl);
295965eefbe4SJohn Baldwin 	load_dr7(vmxctx->host_dr7);
296065eefbe4SJohn Baldwin 	write_rflags(read_rflags() | vmxctx->host_tf);
296165eefbe4SJohn Baldwin }
296265eefbe4SJohn Baldwin 
29638e2cbc56SMark Johnston static __inline void
29648e2cbc56SMark Johnston vmx_pmap_activate(struct vmx *vmx, pmap_t pmap)
29658e2cbc56SMark Johnston {
29668e2cbc56SMark Johnston 	long eptgen;
29678e2cbc56SMark Johnston 	int cpu;
29688e2cbc56SMark Johnston 
29698e2cbc56SMark Johnston 	cpu = curcpu;
29708e2cbc56SMark Johnston 
29718e2cbc56SMark Johnston 	CPU_SET_ATOMIC(cpu, &pmap->pm_active);
29726f5a9606SMark Johnston 	smr_enter(pmap->pm_eptsmr);
29738e2cbc56SMark Johnston 	eptgen = atomic_load_long(&pmap->pm_eptgen);
29748e2cbc56SMark Johnston 	if (eptgen != vmx->eptgen[cpu]) {
29758e2cbc56SMark Johnston 		vmx->eptgen[cpu] = eptgen;
29768e2cbc56SMark Johnston 		invept(INVEPT_TYPE_SINGLE_CONTEXT,
29778e2cbc56SMark Johnston 		    (struct invept_desc){ .eptp = vmx->eptp, ._res = 0 });
29788e2cbc56SMark Johnston 	}
29798e2cbc56SMark Johnston }
29808e2cbc56SMark Johnston 
29818e2cbc56SMark Johnston static __inline void
29828e2cbc56SMark Johnston vmx_pmap_deactivate(struct vmx *vmx, pmap_t pmap)
29838e2cbc56SMark Johnston {
29846f5a9606SMark Johnston 	smr_exit(pmap->pm_eptsmr);
29858e2cbc56SMark Johnston 	CPU_CLR_ATOMIC(curcpu, &pmap->pm_active);
29868e2cbc56SMark Johnston }
29878e2cbc56SMark Johnston 
29880492757cSNeel Natu static int
29892ce12423SNeel Natu vmx_run(void *arg, int vcpu, register_t rip, pmap_t pmap,
2990248e6799SNeel Natu     struct vm_eventinfo *evinfo)
29910492757cSNeel Natu {
29920492757cSNeel Natu 	int rc, handled, launched;
2993366f6083SPeter Grehan 	struct vmx *vmx;
29945b8a8cd1SNeel Natu 	struct vm *vm;
2995366f6083SPeter Grehan 	struct vmxctx *vmxctx;
2996366f6083SPeter Grehan 	struct vmcs *vmcs;
299798ed632cSNeel Natu 	struct vm_exit *vmexit;
2998de5ea6b6SNeel Natu 	struct vlapic *vlapic;
299979c59630SNeel Natu 	uint32_t exit_reason;
3000b843f9beSJohn Baldwin 	struct region_descriptor gdtr, idtr;
3001b843f9beSJohn Baldwin 	uint16_t ldt_sel;
3002366f6083SPeter Grehan 
3003366f6083SPeter Grehan 	vmx = arg;
30045b8a8cd1SNeel Natu 	vm = vmx->vm;
3005366f6083SPeter Grehan 	vmcs = &vmx->vmcs[vcpu];
3006366f6083SPeter Grehan 	vmxctx = &vmx->ctx[vcpu];
30075b8a8cd1SNeel Natu 	vlapic = vm_lapic(vm, vcpu);
30085b8a8cd1SNeel Natu 	vmexit = vm_exitinfo(vm, vcpu);
30090492757cSNeel Natu 	launched = 0;
301098ed632cSNeel Natu 
3011318224bbSNeel Natu 	KASSERT(vmxctx->pmap == pmap,
3012318224bbSNeel Natu 	    ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap));
3013318224bbSNeel Natu 
3014c3498942SNeel Natu 	vmx_msr_guest_enter(vmx, vcpu);
3015c3498942SNeel Natu 
3016366f6083SPeter Grehan 	VMPTRLD(vmcs);
3017366f6083SPeter Grehan 
3018366f6083SPeter Grehan 	/*
3019366f6083SPeter Grehan 	 * XXX
3020366f6083SPeter Grehan 	 * We do this every time because we may setup the virtual machine
3021366f6083SPeter Grehan 	 * from a different process than the one that actually runs it.
3022366f6083SPeter Grehan 	 *
3023366f6083SPeter Grehan 	 * If the life of a virtual machine was spent entirely in the context
302415add60dSPeter Grehan 	 * of a single process we could do this once in vmx_init().
3025366f6083SPeter Grehan 	 */
30263de83862SNeel Natu 	vmcs_write(VMCS_HOST_CR3, rcr3());
3027366f6083SPeter Grehan 
30282ce12423SNeel Natu 	vmcs_write(VMCS_GUEST_RIP, rip);
3029953c2c47SNeel Natu 	vmx_set_pcpu_defaults(vmx, vcpu, pmap);
3030366f6083SPeter Grehan 	do {
30312ce12423SNeel Natu 		KASSERT(vmcs_guest_rip() == rip, ("%s: vmcs guest rip mismatch "
30322ce12423SNeel Natu 		    "%#lx/%#lx", __func__, vmcs_guest_rip(), rip));
303340487465SNeel Natu 
30342ce12423SNeel Natu 		handled = UNHANDLED;
30350492757cSNeel Natu 		/*
30360492757cSNeel Natu 		 * Interrupts are disabled from this point on until the
30370492757cSNeel Natu 		 * guest starts executing. This is done for the following
30380492757cSNeel Natu 		 * reasons:
30390492757cSNeel Natu 		 *
30400492757cSNeel Natu 		 * If an AST is asserted on this thread after the check below,
30410492757cSNeel Natu 		 * then the IPI_AST notification will not be lost, because it
30420492757cSNeel Natu 		 * will cause a VM exit due to external interrupt as soon as
30430492757cSNeel Natu 		 * the guest state is loaded.
30440492757cSNeel Natu 		 *
30450492757cSNeel Natu 		 * A posted interrupt after 'vmx_inject_interrupts()' will
30460492757cSNeel Natu 		 * not be "lost" because it will be held pending in the host
30470492757cSNeel Natu 		 * APIC because interrupts are disabled. The pending interrupt
30480492757cSNeel Natu 		 * will be recognized as soon as the guest state is loaded.
30490492757cSNeel Natu 		 *
30500492757cSNeel Natu 		 * The same reasoning applies to the IPI generated by
30510492757cSNeel Natu 		 * pmap_invalidate_ept().
30520492757cSNeel Natu 		 */
30530492757cSNeel Natu 		disable_intr();
30542ce12423SNeel Natu 		vmx_inject_interrupts(vmx, vcpu, vlapic, rip);
3055091d4532SNeel Natu 
3056091d4532SNeel Natu 		/*
3057091d4532SNeel Natu 		 * Check for vcpu suspension after injecting events because
3058091d4532SNeel Natu 		 * vmx_inject_interrupts() can suspend the vcpu due to a
3059091d4532SNeel Natu 		 * triple fault.
3060091d4532SNeel Natu 		 */
3061248e6799SNeel Natu 		if (vcpu_suspended(evinfo)) {
30620492757cSNeel Natu 			enable_intr();
30632ce12423SNeel Natu 			vm_exit_suspended(vmx->vm, vcpu, rip);
30640492757cSNeel Natu 			break;
30650492757cSNeel Natu 		}
30660492757cSNeel Natu 
3067248e6799SNeel Natu 		if (vcpu_rendezvous_pending(evinfo)) {
30685b8a8cd1SNeel Natu 			enable_intr();
30692ce12423SNeel Natu 			vm_exit_rendezvous(vmx->vm, vcpu, rip);
30705b8a8cd1SNeel Natu 			break;
30715b8a8cd1SNeel Natu 		}
30725b8a8cd1SNeel Natu 
3073248e6799SNeel Natu 		if (vcpu_reqidle(evinfo)) {
3074248e6799SNeel Natu 			enable_intr();
3075248e6799SNeel Natu 			vm_exit_reqidle(vmx->vm, vcpu, rip);
3076248e6799SNeel Natu 			break;
3077248e6799SNeel Natu 		}
3078248e6799SNeel Natu 
3079f008d157SNeel Natu 		if (vcpu_should_yield(vm, vcpu)) {
3080b15a09c0SNeel Natu 			enable_intr();
30812ce12423SNeel Natu 			vm_exit_astpending(vmx->vm, vcpu, rip);
30822ce12423SNeel Natu 			vmx_astpending_trace(vmx, vcpu, rip);
308340487465SNeel Natu 			handled = HANDLED;
3084b15a09c0SNeel Natu 			break;
3085b15a09c0SNeel Natu 		}
3086b15a09c0SNeel Natu 
3087fc276d92SJohn Baldwin 		if (vcpu_debugged(vm, vcpu)) {
3088fc276d92SJohn Baldwin 			enable_intr();
3089fc276d92SJohn Baldwin 			vm_exit_debug(vmx->vm, vcpu, rip);
3090fc276d92SJohn Baldwin 			break;
3091fc276d92SJohn Baldwin 		}
3092fc276d92SJohn Baldwin 
3093b843f9beSJohn Baldwin 		/*
30941bc51badSMichael Reifenberger 		 * If TPR Shadowing is enabled, the TPR Threshold
30951bc51badSMichael Reifenberger 		 * must be updated right before entering the guest.
30961bc51badSMichael Reifenberger 		 */
30971bc51badSMichael Reifenberger 		if (tpr_shadowing && !virtual_interrupt_delivery) {
30981bc51badSMichael Reifenberger 			if ((vmx->cap[vcpu].proc_ctls & PROCBASED_USE_TPR_SHADOW) != 0) {
30991bc51badSMichael Reifenberger 				vmcs_write(VMCS_TPR_THRESHOLD, vlapic_get_cr8(vlapic));
31001bc51badSMichael Reifenberger 			}
31011bc51badSMichael Reifenberger 		}
31021bc51badSMichael Reifenberger 
31031bc51badSMichael Reifenberger 		/*
3104b843f9beSJohn Baldwin 		 * VM exits restore the base address but not the
3105b843f9beSJohn Baldwin 		 * limits of GDTR and IDTR.  The VMCS only stores the
3106b843f9beSJohn Baldwin 		 * base address, so VM exits set the limits to 0xffff.
3107b843f9beSJohn Baldwin 		 * Save and restore the full GDTR and IDTR to restore
3108b843f9beSJohn Baldwin 		 * the limits.
3109b843f9beSJohn Baldwin 		 *
3110b843f9beSJohn Baldwin 		 * The VMCS does not save the LDTR at all, and VM
3111b843f9beSJohn Baldwin 		 * exits clear LDTR as if a NULL selector were loaded.
3112b843f9beSJohn Baldwin 		 * The userspace hypervisor probably doesn't use a
3113b843f9beSJohn Baldwin 		 * LDT, but save and restore it to be safe.
3114b843f9beSJohn Baldwin 		 */
3115b843f9beSJohn Baldwin 		sgdt(&gdtr);
3116b843f9beSJohn Baldwin 		sidt(&idtr);
3117b843f9beSJohn Baldwin 		ldt_sel = sldt();
3118b843f9beSJohn Baldwin 
3119f5f5f1e7SPeter Grehan 		/*
3120f5f5f1e7SPeter Grehan 		 * The TSC_AUX MSR must be saved/restored while interrupts
3121f5f5f1e7SPeter Grehan 		 * are disabled so that it is not possible for the guest
3122f5f5f1e7SPeter Grehan 		 * TSC_AUX MSR value to be overwritten by the resume
3123f5f5f1e7SPeter Grehan 		 * portion of the IPI_SUSPEND codepath. This is why the
3124f5f5f1e7SPeter Grehan 		 * transition of this MSR is handled separately from those
3125f5f5f1e7SPeter Grehan 		 * handled by vmx_msr_guest_{enter,exit}(), which are ok to
3126f5f5f1e7SPeter Grehan 		 * be transitioned with preemption disabled but interrupts
3127f5f5f1e7SPeter Grehan 		 * enabled.
3128f5f5f1e7SPeter Grehan 		 *
3129f5f5f1e7SPeter Grehan 		 * These vmx_msr_guest_{enter,exit}_tsc_aux() calls can be
3130f5f5f1e7SPeter Grehan 		 * anywhere in this loop so long as they happen with
3131f5f5f1e7SPeter Grehan 		 * interrupts disabled. This location is chosen for
3132f5f5f1e7SPeter Grehan 		 * simplicity.
3133f5f5f1e7SPeter Grehan 		 */
3134f5f5f1e7SPeter Grehan 		vmx_msr_guest_enter_tsc_aux(vmx, vcpu);
3135f5f5f1e7SPeter Grehan 
313665eefbe4SJohn Baldwin 		vmx_dr_enter_guest(vmxctx);
313779c59630SNeel Natu 
31388e2cbc56SMark Johnston 		/*
31398e2cbc56SMark Johnston 		 * Mark the EPT as active on this host CPU and invalidate
31408e2cbc56SMark Johnston 		 * EPTP-tagged TLB entries if required.
31418e2cbc56SMark Johnston 		 */
31428e2cbc56SMark Johnston 		vmx_pmap_activate(vmx, pmap);
31438e2cbc56SMark Johnston 
31448e2cbc56SMark Johnston 		vmx_run_trace(vmx, vcpu);
31458e2cbc56SMark Johnston 		rc = vmx_enter_guest(vmxctx, vmx, launched);
31468e2cbc56SMark Johnston 
31478e2cbc56SMark Johnston 		vmx_pmap_deactivate(vmx, pmap);
31488e2cbc56SMark Johnston 		vmx_dr_leave_guest(vmxctx);
3149f5f5f1e7SPeter Grehan 		vmx_msr_guest_exit_tsc_aux(vmx, vcpu);
3150f5f5f1e7SPeter Grehan 
3151b843f9beSJohn Baldwin 		bare_lgdt(&gdtr);
3152b843f9beSJohn Baldwin 		lidt(&idtr);
3153b843f9beSJohn Baldwin 		lldt(ldt_sel);
3154b843f9beSJohn Baldwin 
315579c59630SNeel Natu 		/* Collect some information for VM exit processing */
315679c59630SNeel Natu 		vmexit->rip = rip = vmcs_guest_rip();
315779c59630SNeel Natu 		vmexit->inst_length = vmexit_instruction_length();
315879c59630SNeel Natu 		vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason();
315979c59630SNeel Natu 		vmexit->u.vmx.exit_qualification = vmcs_exit_qualification();
316079c59630SNeel Natu 
31612ce12423SNeel Natu 		/* Update 'nextrip' */
31622ce12423SNeel Natu 		vmx->state[vcpu].nextrip = rip;
31632ce12423SNeel Natu 
31640492757cSNeel Natu 		if (rc == VMX_GUEST_VMEXIT) {
316562fbd7c2SNeel Natu 			vmx_exit_handle_nmi(vmx, vcpu, vmexit);
316662fbd7c2SNeel Natu 			enable_intr();
31670492757cSNeel Natu 			handled = vmx_exit_process(vmx, vcpu, vmexit);
31680492757cSNeel Natu 		} else {
316962fbd7c2SNeel Natu 			enable_intr();
317040487465SNeel Natu 			vmx_exit_inst_error(vmxctx, rc, vmexit);
3171eeefa4e4SNeel Natu 		}
317262fbd7c2SNeel Natu 		launched = 1;
317379c59630SNeel Natu 		vmx_exit_trace(vmx, vcpu, rip, exit_reason, handled);
31742ce12423SNeel Natu 		rip = vmexit->rip;
3175eeefa4e4SNeel Natu 	} while (handled);
3176366f6083SPeter Grehan 
3177366f6083SPeter Grehan 	/*
3178366f6083SPeter Grehan 	 * If a VM exit has been handled then the exitcode must be BOGUS
3179366f6083SPeter Grehan 	 * If a VM exit is not handled then the exitcode must not be BOGUS
3180366f6083SPeter Grehan 	 */
3181366f6083SPeter Grehan 	if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) ||
3182366f6083SPeter Grehan 	    (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) {
3183366f6083SPeter Grehan 		panic("Mismatch between handled (%d) and exitcode (%d)",
3184366f6083SPeter Grehan 		      handled, vmexit->exitcode);
3185366f6083SPeter Grehan 	}
3186366f6083SPeter Grehan 
3187b5aaf7b2SNeel Natu 	if (!handled)
31885b8a8cd1SNeel Natu 		vmm_stat_incr(vm, vcpu, VMEXIT_USERSPACE, 1);
3189b5aaf7b2SNeel Natu 
31905b8a8cd1SNeel Natu 	VCPU_CTR1(vm, vcpu, "returning from vmx_run: exitcode %d",
31910492757cSNeel Natu 	    vmexit->exitcode);
3192366f6083SPeter Grehan 
3193366f6083SPeter Grehan 	VMCLEAR(vmcs);
3194c3498942SNeel Natu 	vmx_msr_guest_exit(vmx, vcpu);
3195c3498942SNeel Natu 
3196366f6083SPeter Grehan 	return (0);
3197366f6083SPeter Grehan }
3198366f6083SPeter Grehan 
3199366f6083SPeter Grehan static void
320015add60dSPeter Grehan vmx_cleanup(void *arg)
3201366f6083SPeter Grehan {
320263c9389aSNeel Natu 	int i;
3203366f6083SPeter Grehan 	struct vmx *vmx = arg;
3204a488c9c9SRodney W. Grimes 	uint16_t maxcpus;
3205366f6083SPeter Grehan 
3206159dd56fSNeel Natu 	if (apic_access_virtualization(vmx, 0))
320788c4b8d1SNeel Natu 		vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
320888c4b8d1SNeel Natu 
3209a488c9c9SRodney W. Grimes 	maxcpus = vm_get_maxcpus(vmx->vm);
3210a488c9c9SRodney W. Grimes 	for (i = 0; i < maxcpus; i++)
321145e51299SNeel Natu 		vpid_free(vmx->state[i].vpid);
321245e51299SNeel Natu 
3213366f6083SPeter Grehan 	free(vmx, M_VMX);
3214366f6083SPeter Grehan 
3215366f6083SPeter Grehan 	return;
3216366f6083SPeter Grehan }
3217366f6083SPeter Grehan 
3218366f6083SPeter Grehan static register_t *
3219366f6083SPeter Grehan vmxctx_regptr(struct vmxctx *vmxctx, int reg)
3220366f6083SPeter Grehan {
3221366f6083SPeter Grehan 
3222366f6083SPeter Grehan 	switch (reg) {
3223366f6083SPeter Grehan 	case VM_REG_GUEST_RAX:
3224366f6083SPeter Grehan 		return (&vmxctx->guest_rax);
3225366f6083SPeter Grehan 	case VM_REG_GUEST_RBX:
3226366f6083SPeter Grehan 		return (&vmxctx->guest_rbx);
3227366f6083SPeter Grehan 	case VM_REG_GUEST_RCX:
3228366f6083SPeter Grehan 		return (&vmxctx->guest_rcx);
3229366f6083SPeter Grehan 	case VM_REG_GUEST_RDX:
3230366f6083SPeter Grehan 		return (&vmxctx->guest_rdx);
3231366f6083SPeter Grehan 	case VM_REG_GUEST_RSI:
3232366f6083SPeter Grehan 		return (&vmxctx->guest_rsi);
3233366f6083SPeter Grehan 	case VM_REG_GUEST_RDI:
3234366f6083SPeter Grehan 		return (&vmxctx->guest_rdi);
3235366f6083SPeter Grehan 	case VM_REG_GUEST_RBP:
3236366f6083SPeter Grehan 		return (&vmxctx->guest_rbp);
3237366f6083SPeter Grehan 	case VM_REG_GUEST_R8:
3238366f6083SPeter Grehan 		return (&vmxctx->guest_r8);
3239366f6083SPeter Grehan 	case VM_REG_GUEST_R9:
3240366f6083SPeter Grehan 		return (&vmxctx->guest_r9);
3241366f6083SPeter Grehan 	case VM_REG_GUEST_R10:
3242366f6083SPeter Grehan 		return (&vmxctx->guest_r10);
3243366f6083SPeter Grehan 	case VM_REG_GUEST_R11:
3244366f6083SPeter Grehan 		return (&vmxctx->guest_r11);
3245366f6083SPeter Grehan 	case VM_REG_GUEST_R12:
3246366f6083SPeter Grehan 		return (&vmxctx->guest_r12);
3247366f6083SPeter Grehan 	case VM_REG_GUEST_R13:
3248366f6083SPeter Grehan 		return (&vmxctx->guest_r13);
3249366f6083SPeter Grehan 	case VM_REG_GUEST_R14:
3250366f6083SPeter Grehan 		return (&vmxctx->guest_r14);
3251366f6083SPeter Grehan 	case VM_REG_GUEST_R15:
3252366f6083SPeter Grehan 		return (&vmxctx->guest_r15);
325337a723a5SNeel Natu 	case VM_REG_GUEST_CR2:
325437a723a5SNeel Natu 		return (&vmxctx->guest_cr2);
325565eefbe4SJohn Baldwin 	case VM_REG_GUEST_DR0:
325665eefbe4SJohn Baldwin 		return (&vmxctx->guest_dr0);
325765eefbe4SJohn Baldwin 	case VM_REG_GUEST_DR1:
325865eefbe4SJohn Baldwin 		return (&vmxctx->guest_dr1);
325965eefbe4SJohn Baldwin 	case VM_REG_GUEST_DR2:
326065eefbe4SJohn Baldwin 		return (&vmxctx->guest_dr2);
326165eefbe4SJohn Baldwin 	case VM_REG_GUEST_DR3:
326265eefbe4SJohn Baldwin 		return (&vmxctx->guest_dr3);
326365eefbe4SJohn Baldwin 	case VM_REG_GUEST_DR6:
326465eefbe4SJohn Baldwin 		return (&vmxctx->guest_dr6);
3265366f6083SPeter Grehan 	default:
3266366f6083SPeter Grehan 		break;
3267366f6083SPeter Grehan 	}
3268366f6083SPeter Grehan 	return (NULL);
3269366f6083SPeter Grehan }
3270366f6083SPeter Grehan 
3271366f6083SPeter Grehan static int
3272366f6083SPeter Grehan vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval)
3273366f6083SPeter Grehan {
3274366f6083SPeter Grehan 	register_t *regp;
3275366f6083SPeter Grehan 
3276366f6083SPeter Grehan 	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
3277366f6083SPeter Grehan 		*retval = *regp;
3278366f6083SPeter Grehan 		return (0);
3279366f6083SPeter Grehan 	} else
3280366f6083SPeter Grehan 		return (EINVAL);
3281366f6083SPeter Grehan }
3282366f6083SPeter Grehan 
3283366f6083SPeter Grehan static int
3284366f6083SPeter Grehan vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val)
3285366f6083SPeter Grehan {
3286366f6083SPeter Grehan 	register_t *regp;
3287366f6083SPeter Grehan 
3288366f6083SPeter Grehan 	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
3289366f6083SPeter Grehan 		*regp = val;
3290366f6083SPeter Grehan 		return (0);
3291366f6083SPeter Grehan 	} else
3292366f6083SPeter Grehan 		return (EINVAL);
3293366f6083SPeter Grehan }
3294366f6083SPeter Grehan 
3295366f6083SPeter Grehan static int
3296d1819632SNeel Natu vmx_get_intr_shadow(struct vmx *vmx, int vcpu, int running, uint64_t *retval)
3297d1819632SNeel Natu {
3298d1819632SNeel Natu 	uint64_t gi;
3299d1819632SNeel Natu 	int error;
3300d1819632SNeel Natu 
3301d1819632SNeel Natu 	error = vmcs_getreg(&vmx->vmcs[vcpu], running,
3302d1819632SNeel Natu 	    VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY), &gi);
3303d1819632SNeel Natu 	*retval = (gi & HWINTR_BLOCKING) ? 1 : 0;
3304d1819632SNeel Natu 	return (error);
3305d1819632SNeel Natu }
3306d1819632SNeel Natu 
3307d1819632SNeel Natu static int
3308d1819632SNeel Natu vmx_modify_intr_shadow(struct vmx *vmx, int vcpu, int running, uint64_t val)
3309d1819632SNeel Natu {
3310d1819632SNeel Natu 	struct vmcs *vmcs;
3311d1819632SNeel Natu 	uint64_t gi;
3312d1819632SNeel Natu 	int error, ident;
3313d1819632SNeel Natu 
3314d1819632SNeel Natu 	/*
3315d1819632SNeel Natu 	 * Forcing the vcpu into an interrupt shadow is not supported.
3316d1819632SNeel Natu 	 */
3317d1819632SNeel Natu 	if (val) {
3318d1819632SNeel Natu 		error = EINVAL;
3319d1819632SNeel Natu 		goto done;
3320d1819632SNeel Natu 	}
3321d1819632SNeel Natu 
3322d1819632SNeel Natu 	vmcs = &vmx->vmcs[vcpu];
3323d1819632SNeel Natu 	ident = VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY);
3324d1819632SNeel Natu 	error = vmcs_getreg(vmcs, running, ident, &gi);
3325d1819632SNeel Natu 	if (error == 0) {
3326d1819632SNeel Natu 		gi &= ~HWINTR_BLOCKING;
3327d1819632SNeel Natu 		error = vmcs_setreg(vmcs, running, ident, gi);
3328d1819632SNeel Natu 	}
3329d1819632SNeel Natu done:
3330d1819632SNeel Natu 	VCPU_CTR2(vmx->vm, vcpu, "Setting intr_shadow to %#lx %s", val,
3331d1819632SNeel Natu 	    error ? "failed" : "succeeded");
3332d1819632SNeel Natu 	return (error);
3333d1819632SNeel Natu }
3334d1819632SNeel Natu 
3335d1819632SNeel Natu static int
3336aaaa0656SPeter Grehan vmx_shadow_reg(int reg)
3337aaaa0656SPeter Grehan {
3338aaaa0656SPeter Grehan 	int shreg;
3339aaaa0656SPeter Grehan 
3340aaaa0656SPeter Grehan 	shreg = -1;
3341aaaa0656SPeter Grehan 
3342aaaa0656SPeter Grehan 	switch (reg) {
3343aaaa0656SPeter Grehan 	case VM_REG_GUEST_CR0:
3344aaaa0656SPeter Grehan 		shreg = VMCS_CR0_SHADOW;
3345aaaa0656SPeter Grehan 		break;
3346aaaa0656SPeter Grehan 	case VM_REG_GUEST_CR4:
3347aaaa0656SPeter Grehan 		shreg = VMCS_CR4_SHADOW;
3348aaaa0656SPeter Grehan 		break;
3349aaaa0656SPeter Grehan 	default:
3350aaaa0656SPeter Grehan 		break;
3351aaaa0656SPeter Grehan 	}
3352aaaa0656SPeter Grehan 
3353aaaa0656SPeter Grehan 	return (shreg);
3354aaaa0656SPeter Grehan }
3355aaaa0656SPeter Grehan 
3356aaaa0656SPeter Grehan static int
3357366f6083SPeter Grehan vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval)
3358366f6083SPeter Grehan {
3359d3c11f40SPeter Grehan 	int running, hostcpu;
3360366f6083SPeter Grehan 	struct vmx *vmx = arg;
3361366f6083SPeter Grehan 
3362d3c11f40SPeter Grehan 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
3363d3c11f40SPeter Grehan 	if (running && hostcpu != curcpu)
3364d3c11f40SPeter Grehan 		panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu);
3365d3c11f40SPeter Grehan 
3366d1819632SNeel Natu 	if (reg == VM_REG_GUEST_INTR_SHADOW)
3367d1819632SNeel Natu 		return (vmx_get_intr_shadow(vmx, vcpu, running, retval));
3368d1819632SNeel Natu 
3369366f6083SPeter Grehan 	if (vmxctx_getreg(&vmx->ctx[vcpu], reg, retval) == 0)
3370366f6083SPeter Grehan 		return (0);
3371366f6083SPeter Grehan 
3372d3c11f40SPeter Grehan 	return (vmcs_getreg(&vmx->vmcs[vcpu], running, reg, retval));
3373366f6083SPeter Grehan }
3374366f6083SPeter Grehan 
3375366f6083SPeter Grehan static int
3376366f6083SPeter Grehan vmx_setreg(void *arg, int vcpu, int reg, uint64_t val)
3377366f6083SPeter Grehan {
3378aaaa0656SPeter Grehan 	int error, hostcpu, running, shadow;
3379366f6083SPeter Grehan 	uint64_t ctls;
33803527963bSNeel Natu 	pmap_t pmap;
3381366f6083SPeter Grehan 	struct vmx *vmx = arg;
3382366f6083SPeter Grehan 
3383d3c11f40SPeter Grehan 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
3384d3c11f40SPeter Grehan 	if (running && hostcpu != curcpu)
3385d3c11f40SPeter Grehan 		panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu);
3386d3c11f40SPeter Grehan 
3387d1819632SNeel Natu 	if (reg == VM_REG_GUEST_INTR_SHADOW)
3388d1819632SNeel Natu 		return (vmx_modify_intr_shadow(vmx, vcpu, running, val));
3389d1819632SNeel Natu 
3390366f6083SPeter Grehan 	if (vmxctx_setreg(&vmx->ctx[vcpu], reg, val) == 0)
3391366f6083SPeter Grehan 		return (0);
3392366f6083SPeter Grehan 
339309860d44SEd Maste 	/* Do not permit user write access to VMCS fields by offset. */
339409860d44SEd Maste 	if (reg < 0)
339509860d44SEd Maste 		return (EINVAL);
339609860d44SEd Maste 
3397d3c11f40SPeter Grehan 	error = vmcs_setreg(&vmx->vmcs[vcpu], running, reg, val);
3398366f6083SPeter Grehan 
3399366f6083SPeter Grehan 	if (error == 0) {
3400366f6083SPeter Grehan 		/*
3401366f6083SPeter Grehan 		 * If the "load EFER" VM-entry control is 1 then the
3402366f6083SPeter Grehan 		 * value of EFER.LMA must be identical to "IA-32e mode guest"
3403366f6083SPeter Grehan 		 * bit in the VM-entry control.
3404366f6083SPeter Grehan 		 */
3405366f6083SPeter Grehan 		if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 &&
3406366f6083SPeter Grehan 		    (reg == VM_REG_GUEST_EFER)) {
3407d3c11f40SPeter Grehan 			vmcs_getreg(&vmx->vmcs[vcpu], running,
3408366f6083SPeter Grehan 				    VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls);
3409366f6083SPeter Grehan 			if (val & EFER_LMA)
3410366f6083SPeter Grehan 				ctls |= VM_ENTRY_GUEST_LMA;
3411366f6083SPeter Grehan 			else
3412366f6083SPeter Grehan 				ctls &= ~VM_ENTRY_GUEST_LMA;
3413d3c11f40SPeter Grehan 			vmcs_setreg(&vmx->vmcs[vcpu], running,
3414366f6083SPeter Grehan 				    VMCS_IDENT(VMCS_ENTRY_CTLS), ctls);
3415366f6083SPeter Grehan 		}
3416aaaa0656SPeter Grehan 
3417aaaa0656SPeter Grehan 		shadow = vmx_shadow_reg(reg);
3418aaaa0656SPeter Grehan 		if (shadow > 0) {
3419aaaa0656SPeter Grehan 			/*
3420aaaa0656SPeter Grehan 			 * Store the unmodified value in the shadow
3421aaaa0656SPeter Grehan 			 */
3422aaaa0656SPeter Grehan 			error = vmcs_setreg(&vmx->vmcs[vcpu], running,
3423aaaa0656SPeter Grehan 				    VMCS_IDENT(shadow), val);
3424aaaa0656SPeter Grehan 		}
34253527963bSNeel Natu 
34263527963bSNeel Natu 		if (reg == VM_REG_GUEST_CR3) {
34273527963bSNeel Natu 			/*
34283527963bSNeel Natu 			 * Invalidate the guest vcpu's TLB mappings to emulate
34293527963bSNeel Natu 			 * the behavior of updating %cr3.
34303527963bSNeel Natu 			 *
34313527963bSNeel Natu 			 * XXX the processor retains global mappings when %cr3
34323527963bSNeel Natu 			 * is updated but vmx_invvpid() does not.
34333527963bSNeel Natu 			 */
34343527963bSNeel Natu 			pmap = vmx->ctx[vcpu].pmap;
34353527963bSNeel Natu 			vmx_invvpid(vmx, vcpu, pmap, running);
34363527963bSNeel Natu 		}
3437366f6083SPeter Grehan 	}
3438366f6083SPeter Grehan 
3439366f6083SPeter Grehan 	return (error);
3440366f6083SPeter Grehan }
3441366f6083SPeter Grehan 
3442366f6083SPeter Grehan static int
3443366f6083SPeter Grehan vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
3444366f6083SPeter Grehan {
3445ba6f5e23SNeel Natu 	int hostcpu, running;
3446366f6083SPeter Grehan 	struct vmx *vmx = arg;
3447366f6083SPeter Grehan 
3448ba6f5e23SNeel Natu 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
3449ba6f5e23SNeel Natu 	if (running && hostcpu != curcpu)
3450ba6f5e23SNeel Natu 		panic("vmx_getdesc: %s%d is running", vm_name(vmx->vm), vcpu);
3451ba6f5e23SNeel Natu 
3452ba6f5e23SNeel Natu 	return (vmcs_getdesc(&vmx->vmcs[vcpu], running, reg, desc));
3453366f6083SPeter Grehan }
3454366f6083SPeter Grehan 
3455366f6083SPeter Grehan static int
3456366f6083SPeter Grehan vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
3457366f6083SPeter Grehan {
3458ba6f5e23SNeel Natu 	int hostcpu, running;
3459366f6083SPeter Grehan 	struct vmx *vmx = arg;
3460366f6083SPeter Grehan 
3461ba6f5e23SNeel Natu 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
3462ba6f5e23SNeel Natu 	if (running && hostcpu != curcpu)
3463ba6f5e23SNeel Natu 		panic("vmx_setdesc: %s%d is running", vm_name(vmx->vm), vcpu);
3464ba6f5e23SNeel Natu 
3465ba6f5e23SNeel Natu 	return (vmcs_setdesc(&vmx->vmcs[vcpu], running, reg, desc));
3466366f6083SPeter Grehan }
3467366f6083SPeter Grehan 
3468366f6083SPeter Grehan static int
3469366f6083SPeter Grehan vmx_getcap(void *arg, int vcpu, int type, int *retval)
3470366f6083SPeter Grehan {
3471366f6083SPeter Grehan 	struct vmx *vmx = arg;
3472366f6083SPeter Grehan 	int vcap;
3473366f6083SPeter Grehan 	int ret;
3474366f6083SPeter Grehan 
3475366f6083SPeter Grehan 	ret = ENOENT;
3476366f6083SPeter Grehan 
3477366f6083SPeter Grehan 	vcap = vmx->cap[vcpu].set;
3478366f6083SPeter Grehan 
3479366f6083SPeter Grehan 	switch (type) {
3480366f6083SPeter Grehan 	case VM_CAP_HALT_EXIT:
3481366f6083SPeter Grehan 		if (cap_halt_exit)
3482366f6083SPeter Grehan 			ret = 0;
3483366f6083SPeter Grehan 		break;
3484366f6083SPeter Grehan 	case VM_CAP_PAUSE_EXIT:
3485366f6083SPeter Grehan 		if (cap_pause_exit)
3486366f6083SPeter Grehan 			ret = 0;
3487366f6083SPeter Grehan 		break;
3488366f6083SPeter Grehan 	case VM_CAP_MTRAP_EXIT:
3489366f6083SPeter Grehan 		if (cap_monitor_trap)
3490366f6083SPeter Grehan 			ret = 0;
3491366f6083SPeter Grehan 		break;
3492f5f5f1e7SPeter Grehan 	case VM_CAP_RDPID:
3493f5f5f1e7SPeter Grehan 		if (cap_rdpid)
3494f5f5f1e7SPeter Grehan 			ret = 0;
3495f5f5f1e7SPeter Grehan 		break;
3496f5f5f1e7SPeter Grehan 	case VM_CAP_RDTSCP:
3497f5f5f1e7SPeter Grehan 		if (cap_rdtscp)
3498f5f5f1e7SPeter Grehan 			ret = 0;
3499f5f5f1e7SPeter Grehan 		break;
3500366f6083SPeter Grehan 	case VM_CAP_UNRESTRICTED_GUEST:
3501366f6083SPeter Grehan 		if (cap_unrestricted_guest)
3502366f6083SPeter Grehan 			ret = 0;
3503366f6083SPeter Grehan 		break;
350449cc03daSNeel Natu 	case VM_CAP_ENABLE_INVPCID:
350549cc03daSNeel Natu 		if (cap_invpcid)
350649cc03daSNeel Natu 			ret = 0;
350749cc03daSNeel Natu 		break;
3508cbd03a9dSJohn Baldwin 	case VM_CAP_BPT_EXIT:
3509cbd03a9dSJohn Baldwin 		ret = 0;
3510cbd03a9dSJohn Baldwin 		break;
3511366f6083SPeter Grehan 	default:
3512366f6083SPeter Grehan 		break;
3513366f6083SPeter Grehan 	}
3514366f6083SPeter Grehan 
3515366f6083SPeter Grehan 	if (ret == 0)
3516366f6083SPeter Grehan 		*retval = (vcap & (1 << type)) ? 1 : 0;
3517366f6083SPeter Grehan 
3518366f6083SPeter Grehan 	return (ret);
3519366f6083SPeter Grehan }
3520366f6083SPeter Grehan 
3521366f6083SPeter Grehan static int
3522366f6083SPeter Grehan vmx_setcap(void *arg, int vcpu, int type, int val)
3523366f6083SPeter Grehan {
3524366f6083SPeter Grehan 	struct vmx *vmx = arg;
3525366f6083SPeter Grehan 	struct vmcs *vmcs = &vmx->vmcs[vcpu];
3526366f6083SPeter Grehan 	uint32_t baseval;
3527366f6083SPeter Grehan 	uint32_t *pptr;
3528366f6083SPeter Grehan 	int error;
3529366f6083SPeter Grehan 	int flag;
3530366f6083SPeter Grehan 	int reg;
3531366f6083SPeter Grehan 	int retval;
3532366f6083SPeter Grehan 
3533366f6083SPeter Grehan 	retval = ENOENT;
3534366f6083SPeter Grehan 	pptr = NULL;
3535366f6083SPeter Grehan 
3536366f6083SPeter Grehan 	switch (type) {
3537366f6083SPeter Grehan 	case VM_CAP_HALT_EXIT:
3538366f6083SPeter Grehan 		if (cap_halt_exit) {
3539366f6083SPeter Grehan 			retval = 0;
3540366f6083SPeter Grehan 			pptr = &vmx->cap[vcpu].proc_ctls;
3541366f6083SPeter Grehan 			baseval = *pptr;
3542366f6083SPeter Grehan 			flag = PROCBASED_HLT_EXITING;
3543366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
3544366f6083SPeter Grehan 		}
3545366f6083SPeter Grehan 		break;
3546366f6083SPeter Grehan 	case VM_CAP_MTRAP_EXIT:
3547366f6083SPeter Grehan 		if (cap_monitor_trap) {
3548366f6083SPeter Grehan 			retval = 0;
3549366f6083SPeter Grehan 			pptr = &vmx->cap[vcpu].proc_ctls;
3550366f6083SPeter Grehan 			baseval = *pptr;
3551366f6083SPeter Grehan 			flag = PROCBASED_MTF;
3552366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
3553366f6083SPeter Grehan 		}
3554366f6083SPeter Grehan 		break;
3555366f6083SPeter Grehan 	case VM_CAP_PAUSE_EXIT:
3556366f6083SPeter Grehan 		if (cap_pause_exit) {
3557366f6083SPeter Grehan 			retval = 0;
3558366f6083SPeter Grehan 			pptr = &vmx->cap[vcpu].proc_ctls;
3559366f6083SPeter Grehan 			baseval = *pptr;
3560366f6083SPeter Grehan 			flag = PROCBASED_PAUSE_EXITING;
3561366f6083SPeter Grehan 			reg = VMCS_PRI_PROC_BASED_CTLS;
3562366f6083SPeter Grehan 		}
3563366f6083SPeter Grehan 		break;
3564f5f5f1e7SPeter Grehan 	case VM_CAP_RDPID:
3565f5f5f1e7SPeter Grehan 	case VM_CAP_RDTSCP:
3566f5f5f1e7SPeter Grehan 		if (cap_rdpid || cap_rdtscp)
3567f5f5f1e7SPeter Grehan 			/*
3568f5f5f1e7SPeter Grehan 			 * Choose not to support enabling/disabling
3569f5f5f1e7SPeter Grehan 			 * RDPID/RDTSCP via libvmmapi since, as per the
357015add60dSPeter Grehan 			 * discussion in vmx_modinit(), RDPID/RDTSCP are
3571f5f5f1e7SPeter Grehan 			 * either always enabled or always disabled.
3572f5f5f1e7SPeter Grehan 			 */
3573f5f5f1e7SPeter Grehan 			error = EOPNOTSUPP;
3574f5f5f1e7SPeter Grehan 		break;
3575366f6083SPeter Grehan 	case VM_CAP_UNRESTRICTED_GUEST:
3576366f6083SPeter Grehan 		if (cap_unrestricted_guest) {
3577366f6083SPeter Grehan 			retval = 0;
357849cc03daSNeel Natu 			pptr = &vmx->cap[vcpu].proc_ctls2;
357949cc03daSNeel Natu 			baseval = *pptr;
3580366f6083SPeter Grehan 			flag = PROCBASED2_UNRESTRICTED_GUEST;
3581366f6083SPeter Grehan 			reg = VMCS_SEC_PROC_BASED_CTLS;
3582366f6083SPeter Grehan 		}
3583366f6083SPeter Grehan 		break;
358449cc03daSNeel Natu 	case VM_CAP_ENABLE_INVPCID:
358549cc03daSNeel Natu 		if (cap_invpcid) {
358649cc03daSNeel Natu 			retval = 0;
358749cc03daSNeel Natu 			pptr = &vmx->cap[vcpu].proc_ctls2;
358849cc03daSNeel Natu 			baseval = *pptr;
358949cc03daSNeel Natu 			flag = PROCBASED2_ENABLE_INVPCID;
359049cc03daSNeel Natu 			reg = VMCS_SEC_PROC_BASED_CTLS;
359149cc03daSNeel Natu 		}
359249cc03daSNeel Natu 		break;
3593cbd03a9dSJohn Baldwin 	case VM_CAP_BPT_EXIT:
3594cbd03a9dSJohn Baldwin 		retval = 0;
3595cbd03a9dSJohn Baldwin 
3596cbd03a9dSJohn Baldwin 		/* Don't change the bitmap if we are tracing all exceptions. */
3597cbd03a9dSJohn Baldwin 		if (vmx->cap[vcpu].exc_bitmap != 0xffffffff) {
3598cbd03a9dSJohn Baldwin 			pptr = &vmx->cap[vcpu].exc_bitmap;
3599cbd03a9dSJohn Baldwin 			baseval = *pptr;
3600cbd03a9dSJohn Baldwin 			flag = (1 << IDT_BP);
3601cbd03a9dSJohn Baldwin 			reg = VMCS_EXCEPTION_BITMAP;
3602cbd03a9dSJohn Baldwin 		}
3603cbd03a9dSJohn Baldwin 		break;
3604366f6083SPeter Grehan 	default:
3605366f6083SPeter Grehan 		break;
3606366f6083SPeter Grehan 	}
3607366f6083SPeter Grehan 
3608cbd03a9dSJohn Baldwin 	if (retval)
3609cbd03a9dSJohn Baldwin 		return (retval);
3610cbd03a9dSJohn Baldwin 
3611cbd03a9dSJohn Baldwin 	if (pptr != NULL) {
3612366f6083SPeter Grehan 		if (val) {
3613366f6083SPeter Grehan 			baseval |= flag;
3614366f6083SPeter Grehan 		} else {
3615366f6083SPeter Grehan 			baseval &= ~flag;
3616366f6083SPeter Grehan 		}
3617366f6083SPeter Grehan 		VMPTRLD(vmcs);
3618366f6083SPeter Grehan 		error = vmwrite(reg, baseval);
3619366f6083SPeter Grehan 		VMCLEAR(vmcs);
3620366f6083SPeter Grehan 
3621cbd03a9dSJohn Baldwin 		if (error)
3622cbd03a9dSJohn Baldwin 			return (error);
3623cbd03a9dSJohn Baldwin 
3624366f6083SPeter Grehan 		/*
3625366f6083SPeter Grehan 		 * Update optional stored flags, and record
3626366f6083SPeter Grehan 		 * setting
3627366f6083SPeter Grehan 		 */
3628366f6083SPeter Grehan 		*pptr = baseval;
3629366f6083SPeter Grehan 	}
3630366f6083SPeter Grehan 
3631366f6083SPeter Grehan 	if (val) {
3632366f6083SPeter Grehan 		vmx->cap[vcpu].set |= (1 << type);
3633366f6083SPeter Grehan 	} else {
3634366f6083SPeter Grehan 		vmx->cap[vcpu].set &= ~(1 << type);
3635366f6083SPeter Grehan 	}
3636366f6083SPeter Grehan 
3637cbd03a9dSJohn Baldwin 	return (0);
3638366f6083SPeter Grehan }
3639366f6083SPeter Grehan 
364015add60dSPeter Grehan static struct vmspace *
364115add60dSPeter Grehan vmx_vmspace_alloc(vm_offset_t min, vm_offset_t max)
364215add60dSPeter Grehan {
364315add60dSPeter Grehan 	return (ept_vmspace_alloc(min, max));
364415add60dSPeter Grehan }
364515add60dSPeter Grehan 
364615add60dSPeter Grehan static void
364715add60dSPeter Grehan vmx_vmspace_free(struct vmspace *vmspace)
364815add60dSPeter Grehan {
364915add60dSPeter Grehan 	ept_vmspace_free(vmspace);
365015add60dSPeter Grehan }
365115add60dSPeter Grehan 
365288c4b8d1SNeel Natu struct vlapic_vtx {
365388c4b8d1SNeel Natu 	struct vlapic	vlapic;
3654176666c2SNeel Natu 	struct pir_desc	*pir_desc;
365530b94db8SNeel Natu 	struct vmx	*vmx;
36562c352febSJohn Baldwin 	u_int	pending_prio;
365788c4b8d1SNeel Natu };
365888c4b8d1SNeel Natu 
36592c352febSJohn Baldwin #define VPR_PRIO_BIT(vpr)	(1 << ((vpr) >> 4))
36602c352febSJohn Baldwin 
366188c4b8d1SNeel Natu #define	VMX_CTR_PIR(vm, vcpuid, pir_desc, notify, vector, level, msg)	\
366288c4b8d1SNeel Natu do {									\
366388c4b8d1SNeel Natu 	VCPU_CTR2(vm, vcpuid, msg " assert %s-triggered vector %d",	\
366488c4b8d1SNeel Natu 	    level ? "level" : "edge", vector);				\
366588c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir0 0x%016lx", pir_desc->pir[0]);	\
366688c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir1 0x%016lx", pir_desc->pir[1]);	\
366788c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir2 0x%016lx", pir_desc->pir[2]);	\
366888c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " pir3 0x%016lx", pir_desc->pir[3]);	\
366988c4b8d1SNeel Natu 	VCPU_CTR1(vm, vcpuid, msg " notify: %s", notify ? "yes" : "no");\
367088c4b8d1SNeel Natu } while (0)
367188c4b8d1SNeel Natu 
367288c4b8d1SNeel Natu /*
367388c4b8d1SNeel Natu  * vlapic->ops handlers that utilize the APICv hardware assist described in
367488c4b8d1SNeel Natu  * Chapter 29 of the Intel SDM.
367588c4b8d1SNeel Natu  */
367688c4b8d1SNeel Natu static int
367788c4b8d1SNeel Natu vmx_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
367888c4b8d1SNeel Natu {
367988c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
368088c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
368188c4b8d1SNeel Natu 	uint64_t mask;
36822c352febSJohn Baldwin 	int idx, notify = 0;
368388c4b8d1SNeel Natu 
368488c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
3685176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
368688c4b8d1SNeel Natu 
368788c4b8d1SNeel Natu 	/*
368888c4b8d1SNeel Natu 	 * Keep track of interrupt requests in the PIR descriptor. This is
368988c4b8d1SNeel Natu 	 * because the virtual APIC page pointed to by the VMCS cannot be
369088c4b8d1SNeel Natu 	 * modified if the vcpu is running.
369188c4b8d1SNeel Natu 	 */
369288c4b8d1SNeel Natu 	idx = vector / 64;
369388c4b8d1SNeel Natu 	mask = 1UL << (vector % 64);
369488c4b8d1SNeel Natu 	atomic_set_long(&pir_desc->pir[idx], mask);
36952c352febSJohn Baldwin 
36962c352febSJohn Baldwin 	/*
36972c352febSJohn Baldwin 	 * A notification is required whenever the 'pending' bit makes a
36982c352febSJohn Baldwin 	 * transition from 0->1.
36992c352febSJohn Baldwin 	 *
37002c352febSJohn Baldwin 	 * Even if the 'pending' bit is already asserted, notification about
37012c352febSJohn Baldwin 	 * the incoming interrupt may still be necessary.  For example, if a
37022c352febSJohn Baldwin 	 * vCPU is HLTed with a high PPR, a low priority interrupt would cause
37032c352febSJohn Baldwin 	 * the 0->1 'pending' transition with a notification, but the vCPU
37042c352febSJohn Baldwin 	 * would ignore the interrupt for the time being.  The same vCPU would
37052c352febSJohn Baldwin 	 * need to then be notified if a high-priority interrupt arrived which
37062c352febSJohn Baldwin 	 * satisfied the PPR.
37072c352febSJohn Baldwin 	 *
37082c352febSJohn Baldwin 	 * The priorities of interrupts injected while 'pending' is asserted
37092c352febSJohn Baldwin 	 * are tracked in a custom bitfield 'pending_prio'.  Should the
37102c352febSJohn Baldwin 	 * to-be-injected interrupt exceed the priorities already present, the
37112c352febSJohn Baldwin 	 * notification is sent.  The priorities recorded in 'pending_prio' are
37122c352febSJohn Baldwin 	 * cleared whenever the 'pending' bit makes another 0->1 transition.
37132c352febSJohn Baldwin 	 */
37142c352febSJohn Baldwin 	if (atomic_cmpset_long(&pir_desc->pending, 0, 1) != 0) {
37152c352febSJohn Baldwin 		notify = 1;
37162c352febSJohn Baldwin 		vlapic_vtx->pending_prio = 0;
37172c352febSJohn Baldwin 	} else {
37182c352febSJohn Baldwin 		const u_int old_prio = vlapic_vtx->pending_prio;
37192c352febSJohn Baldwin 		const u_int prio_bit = VPR_PRIO_BIT(vector & APIC_TPR_INT);
37202c352febSJohn Baldwin 
37212c352febSJohn Baldwin 		if ((old_prio & prio_bit) == 0 && prio_bit > old_prio) {
37222c352febSJohn Baldwin 			atomic_set_int(&vlapic_vtx->pending_prio, prio_bit);
37232c352febSJohn Baldwin 			notify = 1;
37242c352febSJohn Baldwin 		}
37252c352febSJohn Baldwin 	}
372688c4b8d1SNeel Natu 
372788c4b8d1SNeel Natu 	VMX_CTR_PIR(vlapic->vm, vlapic->vcpuid, pir_desc, notify, vector,
372888c4b8d1SNeel Natu 	    level, "vmx_set_intr_ready");
372988c4b8d1SNeel Natu 	return (notify);
373088c4b8d1SNeel Natu }
373188c4b8d1SNeel Natu 
373288c4b8d1SNeel Natu static int
373388c4b8d1SNeel Natu vmx_pending_intr(struct vlapic *vlapic, int *vecptr)
373488c4b8d1SNeel Natu {
373588c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
373688c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
373788c4b8d1SNeel Natu 	struct LAPIC *lapic;
373888c4b8d1SNeel Natu 	uint64_t pending, pirval;
373988c4b8d1SNeel Natu 	uint32_t ppr, vpr;
374088c4b8d1SNeel Natu 	int i;
374188c4b8d1SNeel Natu 
374288c4b8d1SNeel Natu 	/*
374388c4b8d1SNeel Natu 	 * This function is only expected to be called from the 'HLT' exit
374488c4b8d1SNeel Natu 	 * handler which does not care about the vector that is pending.
374588c4b8d1SNeel Natu 	 */
374688c4b8d1SNeel Natu 	KASSERT(vecptr == NULL, ("vmx_pending_intr: vecptr must be NULL"));
374788c4b8d1SNeel Natu 
374888c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
3749176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
375088c4b8d1SNeel Natu 
375188c4b8d1SNeel Natu 	pending = atomic_load_acq_long(&pir_desc->pending);
37529e33a616STycho Nightingale 	if (!pending) {
37539e33a616STycho Nightingale 		/*
37549e33a616STycho Nightingale 		 * While a virtual interrupt may have already been
37559e33a616STycho Nightingale 		 * processed the actual delivery maybe pending the
37569e33a616STycho Nightingale 		 * interruptibility of the guest.  Recognize a pending
37579e33a616STycho Nightingale 		 * interrupt by reevaluating virtual interrupts
37589e33a616STycho Nightingale 		 * following Section 29.2.1 in the Intel SDM Volume 3.
37599e33a616STycho Nightingale 		 */
3760490768e2STycho Nightingale 		struct vm_exit *vmexit;
37619e33a616STycho Nightingale 		uint8_t rvi, ppr;
37629e33a616STycho Nightingale 
3763490768e2STycho Nightingale 		vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid);
3764490768e2STycho Nightingale 		KASSERT(vmexit->exitcode == VM_EXITCODE_HLT,
3765490768e2STycho Nightingale 		    ("vmx_pending_intr: exitcode not 'HLT'"));
3766490768e2STycho Nightingale 		rvi = vmexit->u.hlt.intr_status & APIC_TPR_INT;
37679e33a616STycho Nightingale 		lapic = vlapic->apic_page;
37689e33a616STycho Nightingale 		ppr = lapic->ppr & APIC_TPR_INT;
37699e33a616STycho Nightingale 		if (rvi > ppr) {
37709e33a616STycho Nightingale 			return (1);
37719e33a616STycho Nightingale 		}
37729e33a616STycho Nightingale 
37739e33a616STycho Nightingale 		return (0);
37749e33a616STycho Nightingale 	}
377588c4b8d1SNeel Natu 
377688c4b8d1SNeel Natu 	/*
377788c4b8d1SNeel Natu 	 * If there is an interrupt pending then it will be recognized only
377888c4b8d1SNeel Natu 	 * if its priority is greater than the processor priority.
377988c4b8d1SNeel Natu 	 *
378088c4b8d1SNeel Natu 	 * Special case: if the processor priority is zero then any pending
378188c4b8d1SNeel Natu 	 * interrupt will be recognized.
378288c4b8d1SNeel Natu 	 */
378388c4b8d1SNeel Natu 	lapic = vlapic->apic_page;
37849e33a616STycho Nightingale 	ppr = lapic->ppr & APIC_TPR_INT;
378588c4b8d1SNeel Natu 	if (ppr == 0)
378688c4b8d1SNeel Natu 		return (1);
378788c4b8d1SNeel Natu 
378888c4b8d1SNeel Natu 	VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "HLT with non-zero PPR %d",
378988c4b8d1SNeel Natu 	    lapic->ppr);
379088c4b8d1SNeel Natu 
37912c352febSJohn Baldwin 	vpr = 0;
379288c4b8d1SNeel Natu 	for (i = 3; i >= 0; i--) {
379388c4b8d1SNeel Natu 		pirval = pir_desc->pir[i];
379488c4b8d1SNeel Natu 		if (pirval != 0) {
37959e33a616STycho Nightingale 			vpr = (i * 64 + flsl(pirval) - 1) & APIC_TPR_INT;
37962c352febSJohn Baldwin 			break;
379788c4b8d1SNeel Natu 		}
379888c4b8d1SNeel Natu 	}
37992c352febSJohn Baldwin 
38002c352febSJohn Baldwin 	/*
38012c352febSJohn Baldwin 	 * If the highest-priority pending interrupt falls short of the
38022c352febSJohn Baldwin 	 * processor priority of this vCPU, ensure that 'pending_prio' does not
38032c352febSJohn Baldwin 	 * have any stale bits which would preclude a higher-priority interrupt
38042c352febSJohn Baldwin 	 * from incurring a notification later.
38052c352febSJohn Baldwin 	 */
38062c352febSJohn Baldwin 	if (vpr <= ppr) {
38072c352febSJohn Baldwin 		const u_int prio_bit = VPR_PRIO_BIT(vpr);
38082c352febSJohn Baldwin 		const u_int old = vlapic_vtx->pending_prio;
38092c352febSJohn Baldwin 
38102c352febSJohn Baldwin 		if (old > prio_bit && (old & prio_bit) == 0) {
38112c352febSJohn Baldwin 			vlapic_vtx->pending_prio = prio_bit;
38122c352febSJohn Baldwin 		}
381388c4b8d1SNeel Natu 		return (0);
381488c4b8d1SNeel Natu 	}
38152c352febSJohn Baldwin 	return (1);
38162c352febSJohn Baldwin }
381788c4b8d1SNeel Natu 
381888c4b8d1SNeel Natu static void
381988c4b8d1SNeel Natu vmx_intr_accepted(struct vlapic *vlapic, int vector)
382088c4b8d1SNeel Natu {
382188c4b8d1SNeel Natu 
382288c4b8d1SNeel Natu 	panic("vmx_intr_accepted: not expected to be called");
382388c4b8d1SNeel Natu }
382488c4b8d1SNeel Natu 
3825176666c2SNeel Natu static void
382630b94db8SNeel Natu vmx_set_tmr(struct vlapic *vlapic, int vector, bool level)
382730b94db8SNeel Natu {
382830b94db8SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
382930b94db8SNeel Natu 	struct vmx *vmx;
383030b94db8SNeel Natu 	struct vmcs *vmcs;
383130b94db8SNeel Natu 	uint64_t mask, val;
383230b94db8SNeel Natu 
383330b94db8SNeel Natu 	KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
383430b94db8SNeel Natu 	KASSERT(!vcpu_is_running(vlapic->vm, vlapic->vcpuid, NULL),
383530b94db8SNeel Natu 	    ("vmx_set_tmr: vcpu cannot be running"));
383630b94db8SNeel Natu 
383730b94db8SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
383830b94db8SNeel Natu 	vmx = vlapic_vtx->vmx;
383930b94db8SNeel Natu 	vmcs = &vmx->vmcs[vlapic->vcpuid];
384030b94db8SNeel Natu 	mask = 1UL << (vector % 64);
384130b94db8SNeel Natu 
384230b94db8SNeel Natu 	VMPTRLD(vmcs);
384330b94db8SNeel Natu 	val = vmcs_read(VMCS_EOI_EXIT(vector));
384430b94db8SNeel Natu 	if (level)
384530b94db8SNeel Natu 		val |= mask;
384630b94db8SNeel Natu 	else
384730b94db8SNeel Natu 		val &= ~mask;
384830b94db8SNeel Natu 	vmcs_write(VMCS_EOI_EXIT(vector), val);
384930b94db8SNeel Natu 	VMCLEAR(vmcs);
385030b94db8SNeel Natu }
385130b94db8SNeel Natu 
385230b94db8SNeel Natu static void
38531bc51badSMichael Reifenberger vmx_enable_x2apic_mode_ts(struct vlapic *vlapic)
38541bc51badSMichael Reifenberger {
38551bc51badSMichael Reifenberger 	struct vmx *vmx;
38561bc51badSMichael Reifenberger 	struct vmcs *vmcs;
38571bc51badSMichael Reifenberger 	uint32_t proc_ctls;
38581bc51badSMichael Reifenberger 	int vcpuid;
38591bc51badSMichael Reifenberger 
38601bc51badSMichael Reifenberger 	vcpuid = vlapic->vcpuid;
38611bc51badSMichael Reifenberger 	vmx = ((struct vlapic_vtx *)vlapic)->vmx;
38621bc51badSMichael Reifenberger 	vmcs = &vmx->vmcs[vcpuid];
38631bc51badSMichael Reifenberger 
38641bc51badSMichael Reifenberger 	proc_ctls = vmx->cap[vcpuid].proc_ctls;
38651bc51badSMichael Reifenberger 	proc_ctls &= ~PROCBASED_USE_TPR_SHADOW;
38661bc51badSMichael Reifenberger 	proc_ctls |= PROCBASED_CR8_LOAD_EXITING;
38671bc51badSMichael Reifenberger 	proc_ctls |= PROCBASED_CR8_STORE_EXITING;
38681bc51badSMichael Reifenberger 	vmx->cap[vcpuid].proc_ctls = proc_ctls;
38691bc51badSMichael Reifenberger 
38701bc51badSMichael Reifenberger 	VMPTRLD(vmcs);
38711bc51badSMichael Reifenberger 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, proc_ctls);
38721bc51badSMichael Reifenberger 	VMCLEAR(vmcs);
38731bc51badSMichael Reifenberger }
38741bc51badSMichael Reifenberger 
38751bc51badSMichael Reifenberger static void
38761bc51badSMichael Reifenberger vmx_enable_x2apic_mode_vid(struct vlapic *vlapic)
3877159dd56fSNeel Natu {
3878159dd56fSNeel Natu 	struct vmx *vmx;
3879159dd56fSNeel Natu 	struct vmcs *vmcs;
3880159dd56fSNeel Natu 	uint32_t proc_ctls2;
38815c272efaSRobert Wing 	int vcpuid, error __diagused;
3882159dd56fSNeel Natu 
3883159dd56fSNeel Natu 	vcpuid = vlapic->vcpuid;
3884159dd56fSNeel Natu 	vmx = ((struct vlapic_vtx *)vlapic)->vmx;
3885159dd56fSNeel Natu 	vmcs = &vmx->vmcs[vcpuid];
3886159dd56fSNeel Natu 
3887159dd56fSNeel Natu 	proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
3888159dd56fSNeel Natu 	KASSERT((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) != 0,
3889159dd56fSNeel Natu 	    ("%s: invalid proc_ctls2 %#x", __func__, proc_ctls2));
3890159dd56fSNeel Natu 
3891159dd56fSNeel Natu 	proc_ctls2 &= ~PROCBASED2_VIRTUALIZE_APIC_ACCESSES;
3892159dd56fSNeel Natu 	proc_ctls2 |= PROCBASED2_VIRTUALIZE_X2APIC_MODE;
3893159dd56fSNeel Natu 	vmx->cap[vcpuid].proc_ctls2 = proc_ctls2;
3894159dd56fSNeel Natu 
3895159dd56fSNeel Natu 	VMPTRLD(vmcs);
3896159dd56fSNeel Natu 	vmcs_write(VMCS_SEC_PROC_BASED_CTLS, proc_ctls2);
3897159dd56fSNeel Natu 	VMCLEAR(vmcs);
3898159dd56fSNeel Natu 
3899159dd56fSNeel Natu 	if (vlapic->vcpuid == 0) {
3900159dd56fSNeel Natu 		/*
3901159dd56fSNeel Natu 		 * The nested page table mappings are shared by all vcpus
3902159dd56fSNeel Natu 		 * so unmap the APIC access page just once.
3903159dd56fSNeel Natu 		 */
3904159dd56fSNeel Natu 		error = vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
3905159dd56fSNeel Natu 		KASSERT(error == 0, ("%s: vm_unmap_mmio error %d",
3906159dd56fSNeel Natu 		    __func__, error));
3907159dd56fSNeel Natu 
3908159dd56fSNeel Natu 		/*
3909159dd56fSNeel Natu 		 * The MSR bitmap is shared by all vcpus so modify it only
3910159dd56fSNeel Natu 		 * once in the context of vcpu 0.
3911159dd56fSNeel Natu 		 */
3912159dd56fSNeel Natu 		error = vmx_allow_x2apic_msrs(vmx);
3913159dd56fSNeel Natu 		KASSERT(error == 0, ("%s: vmx_allow_x2apic_msrs error %d",
3914159dd56fSNeel Natu 		    __func__, error));
3915159dd56fSNeel Natu 	}
3916159dd56fSNeel Natu }
3917159dd56fSNeel Natu 
3918159dd56fSNeel Natu static void
3919176666c2SNeel Natu vmx_post_intr(struct vlapic *vlapic, int hostcpu)
3920176666c2SNeel Natu {
3921176666c2SNeel Natu 
3922176666c2SNeel Natu 	ipi_cpu(hostcpu, pirvec);
3923176666c2SNeel Natu }
3924176666c2SNeel Natu 
392588c4b8d1SNeel Natu /*
392688c4b8d1SNeel Natu  * Transfer the pending interrupts in the PIR descriptor to the IRR
392788c4b8d1SNeel Natu  * in the virtual APIC page.
392888c4b8d1SNeel Natu  */
392988c4b8d1SNeel Natu static void
393088c4b8d1SNeel Natu vmx_inject_pir(struct vlapic *vlapic)
393188c4b8d1SNeel Natu {
393288c4b8d1SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
393388c4b8d1SNeel Natu 	struct pir_desc *pir_desc;
393488c4b8d1SNeel Natu 	struct LAPIC *lapic;
393588c4b8d1SNeel Natu 	uint64_t val, pirval;
39360e30c5c0SWarner Losh 	int rvi, pirbase = -1;
393788c4b8d1SNeel Natu 	uint16_t intr_status_old, intr_status_new;
393888c4b8d1SNeel Natu 
393988c4b8d1SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
3940176666c2SNeel Natu 	pir_desc = vlapic_vtx->pir_desc;
394188c4b8d1SNeel Natu 	if (atomic_cmpset_long(&pir_desc->pending, 1, 0) == 0) {
394288c4b8d1SNeel Natu 		VCPU_CTR0(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
394388c4b8d1SNeel Natu 		    "no posted interrupt pending");
394488c4b8d1SNeel Natu 		return;
394588c4b8d1SNeel Natu 	}
394688c4b8d1SNeel Natu 
394788c4b8d1SNeel Natu 	pirval = 0;
3948201b1cccSPeter Grehan 	pirbase = -1;
394988c4b8d1SNeel Natu 	lapic = vlapic->apic_page;
395088c4b8d1SNeel Natu 
395188c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[0]);
395288c4b8d1SNeel Natu 	if (val != 0) {
395388c4b8d1SNeel Natu 		lapic->irr0 |= val;
395488c4b8d1SNeel Natu 		lapic->irr1 |= val >> 32;
395588c4b8d1SNeel Natu 		pirbase = 0;
395688c4b8d1SNeel Natu 		pirval = val;
395788c4b8d1SNeel Natu 	}
395888c4b8d1SNeel Natu 
395988c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[1]);
396088c4b8d1SNeel Natu 	if (val != 0) {
396188c4b8d1SNeel Natu 		lapic->irr2 |= val;
396288c4b8d1SNeel Natu 		lapic->irr3 |= val >> 32;
396388c4b8d1SNeel Natu 		pirbase = 64;
396488c4b8d1SNeel Natu 		pirval = val;
396588c4b8d1SNeel Natu 	}
396688c4b8d1SNeel Natu 
396788c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[2]);
396888c4b8d1SNeel Natu 	if (val != 0) {
396988c4b8d1SNeel Natu 		lapic->irr4 |= val;
397088c4b8d1SNeel Natu 		lapic->irr5 |= val >> 32;
397188c4b8d1SNeel Natu 		pirbase = 128;
397288c4b8d1SNeel Natu 		pirval = val;
397388c4b8d1SNeel Natu 	}
397488c4b8d1SNeel Natu 
397588c4b8d1SNeel Natu 	val = atomic_readandclear_long(&pir_desc->pir[3]);
397688c4b8d1SNeel Natu 	if (val != 0) {
397788c4b8d1SNeel Natu 		lapic->irr6 |= val;
397888c4b8d1SNeel Natu 		lapic->irr7 |= val >> 32;
397988c4b8d1SNeel Natu 		pirbase = 192;
398088c4b8d1SNeel Natu 		pirval = val;
398188c4b8d1SNeel Natu 	}
3982201b1cccSPeter Grehan 
398388c4b8d1SNeel Natu 	VLAPIC_CTR_IRR(vlapic, "vmx_inject_pir");
398488c4b8d1SNeel Natu 
398588c4b8d1SNeel Natu 	/*
398688c4b8d1SNeel Natu 	 * Update RVI so the processor can evaluate pending virtual
398788c4b8d1SNeel Natu 	 * interrupts on VM-entry.
3988201b1cccSPeter Grehan 	 *
3989201b1cccSPeter Grehan 	 * It is possible for pirval to be 0 here, even though the
3990201b1cccSPeter Grehan 	 * pending bit has been set. The scenario is:
3991201b1cccSPeter Grehan 	 * CPU-Y is sending a posted interrupt to CPU-X, which
3992201b1cccSPeter Grehan 	 * is running a guest and processing posted interrupts in h/w.
3993201b1cccSPeter Grehan 	 * CPU-X will eventually exit and the state seen in s/w is
3994201b1cccSPeter Grehan 	 * the pending bit set, but no PIR bits set.
3995201b1cccSPeter Grehan 	 *
3996201b1cccSPeter Grehan 	 *      CPU-X                      CPU-Y
3997201b1cccSPeter Grehan 	 *   (vm running)                (host running)
3998201b1cccSPeter Grehan 	 *   rx posted interrupt
3999201b1cccSPeter Grehan 	 *   CLEAR pending bit
4000201b1cccSPeter Grehan 	 *				 SET PIR bit
4001201b1cccSPeter Grehan 	 *   READ/CLEAR PIR bits
4002201b1cccSPeter Grehan 	 *				 SET pending bit
4003201b1cccSPeter Grehan 	 *   (vm exit)
4004201b1cccSPeter Grehan 	 *   pending bit set, PIR 0
400588c4b8d1SNeel Natu 	 */
400688c4b8d1SNeel Natu 	if (pirval != 0) {
400788c4b8d1SNeel Natu 		rvi = pirbase + flsl(pirval) - 1;
400888c4b8d1SNeel Natu 		intr_status_old = vmcs_read(VMCS_GUEST_INTR_STATUS);
400988c4b8d1SNeel Natu 		intr_status_new = (intr_status_old & 0xFF00) | rvi;
401088c4b8d1SNeel Natu 		if (intr_status_new > intr_status_old) {
401188c4b8d1SNeel Natu 			vmcs_write(VMCS_GUEST_INTR_STATUS, intr_status_new);
401288c4b8d1SNeel Natu 			VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
401388c4b8d1SNeel Natu 			    "guest_intr_status changed from 0x%04x to 0x%04x",
401488c4b8d1SNeel Natu 			    intr_status_old, intr_status_new);
401588c4b8d1SNeel Natu 		}
401688c4b8d1SNeel Natu 	}
401788c4b8d1SNeel Natu }
401888c4b8d1SNeel Natu 
4019de5ea6b6SNeel Natu static struct vlapic *
4020de5ea6b6SNeel Natu vmx_vlapic_init(void *arg, int vcpuid)
4021de5ea6b6SNeel Natu {
4022de5ea6b6SNeel Natu 	struct vmx *vmx;
4023de5ea6b6SNeel Natu 	struct vlapic *vlapic;
4024176666c2SNeel Natu 	struct vlapic_vtx *vlapic_vtx;
4025de5ea6b6SNeel Natu 
4026de5ea6b6SNeel Natu 	vmx = arg;
4027de5ea6b6SNeel Natu 
402888c4b8d1SNeel Natu 	vlapic = malloc(sizeof(struct vlapic_vtx), M_VLAPIC, M_WAITOK | M_ZERO);
4029de5ea6b6SNeel Natu 	vlapic->vm = vmx->vm;
4030de5ea6b6SNeel Natu 	vlapic->vcpuid = vcpuid;
4031de5ea6b6SNeel Natu 	vlapic->apic_page = (struct LAPIC *)&vmx->apic_page[vcpuid];
4032de5ea6b6SNeel Natu 
4033176666c2SNeel Natu 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
4034176666c2SNeel Natu 	vlapic_vtx->pir_desc = &vmx->pir_desc[vcpuid];
403530b94db8SNeel Natu 	vlapic_vtx->vmx = vmx;
4036176666c2SNeel Natu 
40371bc51badSMichael Reifenberger 	if (tpr_shadowing) {
40381bc51badSMichael Reifenberger 		vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode_ts;
40391bc51badSMichael Reifenberger 	}
40401bc51badSMichael Reifenberger 
404188c4b8d1SNeel Natu 	if (virtual_interrupt_delivery) {
404288c4b8d1SNeel Natu 		vlapic->ops.set_intr_ready = vmx_set_intr_ready;
404388c4b8d1SNeel Natu 		vlapic->ops.pending_intr = vmx_pending_intr;
404488c4b8d1SNeel Natu 		vlapic->ops.intr_accepted = vmx_intr_accepted;
404530b94db8SNeel Natu 		vlapic->ops.set_tmr = vmx_set_tmr;
40461bc51badSMichael Reifenberger 		vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode_vid;
404788c4b8d1SNeel Natu 	}
404888c4b8d1SNeel Natu 
4049176666c2SNeel Natu 	if (posted_interrupts)
4050176666c2SNeel Natu 		vlapic->ops.post_intr = vmx_post_intr;
4051176666c2SNeel Natu 
4052de5ea6b6SNeel Natu 	vlapic_init(vlapic);
4053de5ea6b6SNeel Natu 
4054de5ea6b6SNeel Natu 	return (vlapic);
4055de5ea6b6SNeel Natu }
4056de5ea6b6SNeel Natu 
4057de5ea6b6SNeel Natu static void
4058de5ea6b6SNeel Natu vmx_vlapic_cleanup(void *arg, struct vlapic *vlapic)
4059de5ea6b6SNeel Natu {
4060de5ea6b6SNeel Natu 
4061de5ea6b6SNeel Natu 	vlapic_cleanup(vlapic);
4062de5ea6b6SNeel Natu 	free(vlapic, M_VLAPIC);
4063de5ea6b6SNeel Natu }
4064de5ea6b6SNeel Natu 
4065483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT
4066483d953aSJohn Baldwin static int
406715add60dSPeter Grehan vmx_snapshot(void *arg, struct vm_snapshot_meta *meta)
4068483d953aSJohn Baldwin {
4069483d953aSJohn Baldwin 	struct vmx *vmx;
4070483d953aSJohn Baldwin 	struct vmxctx *vmxctx;
4071483d953aSJohn Baldwin 	int i;
4072483d953aSJohn Baldwin 	int ret;
4073483d953aSJohn Baldwin 
4074483d953aSJohn Baldwin 	vmx = arg;
4075483d953aSJohn Baldwin 
4076483d953aSJohn Baldwin 	KASSERT(vmx != NULL, ("%s: arg was NULL", __func__));
4077483d953aSJohn Baldwin 
4078483d953aSJohn Baldwin 	for (i = 0; i < VM_MAXCPU; i++) {
4079483d953aSJohn Baldwin 		SNAPSHOT_BUF_OR_LEAVE(vmx->guest_msrs[i],
4080483d953aSJohn Baldwin 		      sizeof(vmx->guest_msrs[i]), meta, ret, done);
4081483d953aSJohn Baldwin 
4082483d953aSJohn Baldwin 		vmxctx = &vmx->ctx[i];
4083483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rdi, meta, ret, done);
4084483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rsi, meta, ret, done);
4085483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rdx, meta, ret, done);
4086483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rcx, meta, ret, done);
4087483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r8, meta, ret, done);
4088483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r9, meta, ret, done);
4089483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rax, meta, ret, done);
4090483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rbx, meta, ret, done);
4091483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rbp, meta, ret, done);
4092483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r10, meta, ret, done);
4093483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r11, meta, ret, done);
4094483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r12, meta, ret, done);
4095483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r13, meta, ret, done);
4096483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r14, meta, ret, done);
4097483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r15, meta, ret, done);
4098483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_cr2, meta, ret, done);
4099483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr0, meta, ret, done);
4100483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr1, meta, ret, done);
4101483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr2, meta, ret, done);
4102483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr3, meta, ret, done);
4103483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr6, meta, ret, done);
4104483d953aSJohn Baldwin 	}
4105483d953aSJohn Baldwin 
4106483d953aSJohn Baldwin done:
4107483d953aSJohn Baldwin 	return (ret);
4108483d953aSJohn Baldwin }
4109483d953aSJohn Baldwin 
4110483d953aSJohn Baldwin static int
411115add60dSPeter Grehan vmx_vmcx_snapshot(void *arg, struct vm_snapshot_meta *meta, int vcpu)
4112483d953aSJohn Baldwin {
4113483d953aSJohn Baldwin 	struct vmcs *vmcs;
4114483d953aSJohn Baldwin 	struct vmx *vmx;
4115483d953aSJohn Baldwin 	int err, run, hostcpu;
4116483d953aSJohn Baldwin 
4117483d953aSJohn Baldwin 	vmx = (struct vmx *)arg;
4118483d953aSJohn Baldwin 	err = 0;
4119483d953aSJohn Baldwin 
4120483d953aSJohn Baldwin 	KASSERT(arg != NULL, ("%s: arg was NULL", __func__));
4121483d953aSJohn Baldwin 	vmcs = &vmx->vmcs[vcpu];
4122483d953aSJohn Baldwin 
4123483d953aSJohn Baldwin 	run = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
4124483d953aSJohn Baldwin 	if (run && hostcpu != curcpu) {
4125483d953aSJohn Baldwin 		printf("%s: %s%d is running", __func__, vm_name(vmx->vm), vcpu);
4126483d953aSJohn Baldwin 		return (EINVAL);
4127483d953aSJohn Baldwin 	}
4128483d953aSJohn Baldwin 
4129483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CR0, meta);
4130483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CR3, meta);
4131483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CR4, meta);
4132483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_DR7, meta);
4133483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_RSP, meta);
4134483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_RIP, meta);
4135483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_RFLAGS, meta);
4136483d953aSJohn Baldwin 
4137483d953aSJohn Baldwin 	/* Guest segments */
4138483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_ES, meta);
4139483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_ES, meta);
4140483d953aSJohn Baldwin 
4141483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CS, meta);
4142483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_CS, meta);
4143483d953aSJohn Baldwin 
4144483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_SS, meta);
4145483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_SS, meta);
4146483d953aSJohn Baldwin 
4147483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_DS, meta);
4148483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_DS, meta);
4149483d953aSJohn Baldwin 
4150483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_FS, meta);
4151483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_FS, meta);
4152483d953aSJohn Baldwin 
4153483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_GS, meta);
4154483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_GS, meta);
4155483d953aSJohn Baldwin 
4156483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_TR, meta);
4157483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_TR, meta);
4158483d953aSJohn Baldwin 
4159483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_LDTR, meta);
4160483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_LDTR, meta);
4161483d953aSJohn Baldwin 
4162483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_EFER, meta);
4163483d953aSJohn Baldwin 
4164483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_IDTR, meta);
4165483d953aSJohn Baldwin 	err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_GDTR, meta);
4166483d953aSJohn Baldwin 
4167483d953aSJohn Baldwin 	/* Guest page tables */
4168483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE0, meta);
4169483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE1, meta);
4170483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE2, meta);
4171483d953aSJohn Baldwin 	err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE3, meta);
4172483d953aSJohn Baldwin 
4173483d953aSJohn Baldwin 	/* Other guest state */
4174483d953aSJohn Baldwin 	err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_IA32_SYSENTER_CS, meta);
4175483d953aSJohn Baldwin 	err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_IA32_SYSENTER_ESP, meta);
4176483d953aSJohn Baldwin 	err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_IA32_SYSENTER_EIP, meta);
4177483d953aSJohn Baldwin 	err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_INTERRUPTIBILITY, meta);
4178483d953aSJohn Baldwin 	err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_ACTIVITY, meta);
4179483d953aSJohn Baldwin 	err += vmcs_snapshot_any(vmcs, run, VMCS_ENTRY_CTLS, meta);
4180483d953aSJohn Baldwin 	err += vmcs_snapshot_any(vmcs, run, VMCS_EXIT_CTLS, meta);
4181483d953aSJohn Baldwin 
4182483d953aSJohn Baldwin 	return (err);
4183483d953aSJohn Baldwin }
4184483d953aSJohn Baldwin 
4185483d953aSJohn Baldwin static int
4186483d953aSJohn Baldwin vmx_restore_tsc(void *arg, int vcpu, uint64_t offset)
4187483d953aSJohn Baldwin {
4188483d953aSJohn Baldwin 	struct vmcs *vmcs;
4189483d953aSJohn Baldwin 	struct vmx *vmx = (struct vmx *)arg;
4190483d953aSJohn Baldwin 	int error, running, hostcpu;
4191483d953aSJohn Baldwin 
4192483d953aSJohn Baldwin 	KASSERT(arg != NULL, ("%s: arg was NULL", __func__));
4193483d953aSJohn Baldwin 	vmcs = &vmx->vmcs[vcpu];
4194483d953aSJohn Baldwin 
4195483d953aSJohn Baldwin 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
4196483d953aSJohn Baldwin 	if (running && hostcpu != curcpu) {
4197483d953aSJohn Baldwin 		printf("%s: %s%d is running", __func__, vm_name(vmx->vm), vcpu);
4198483d953aSJohn Baldwin 		return (EINVAL);
4199483d953aSJohn Baldwin 	}
4200483d953aSJohn Baldwin 
4201483d953aSJohn Baldwin 	if (!running)
4202483d953aSJohn Baldwin 		VMPTRLD(vmcs);
4203483d953aSJohn Baldwin 
4204483d953aSJohn Baldwin 	error = vmx_set_tsc_offset(vmx, vcpu, offset);
4205483d953aSJohn Baldwin 
4206483d953aSJohn Baldwin 	if (!running)
4207483d953aSJohn Baldwin 		VMCLEAR(vmcs);
4208483d953aSJohn Baldwin 	return (error);
4209483d953aSJohn Baldwin }
4210483d953aSJohn Baldwin #endif
4211483d953aSJohn Baldwin 
421215add60dSPeter Grehan const struct vmm_ops vmm_ops_intel = {
421315add60dSPeter Grehan 	.modinit	= vmx_modinit,
421415add60dSPeter Grehan 	.modcleanup	= vmx_modcleanup,
421515add60dSPeter Grehan 	.modresume	= vmx_modresume,
421613a7c4d4SMark Johnston 	.init		= vmx_init,
421715add60dSPeter Grehan 	.run		= vmx_run,
421813a7c4d4SMark Johnston 	.cleanup	= vmx_cleanup,
421915add60dSPeter Grehan 	.getreg		= vmx_getreg,
422015add60dSPeter Grehan 	.setreg		= vmx_setreg,
422115add60dSPeter Grehan 	.getdesc	= vmx_getdesc,
422215add60dSPeter Grehan 	.setdesc	= vmx_setdesc,
422315add60dSPeter Grehan 	.getcap		= vmx_getcap,
422415add60dSPeter Grehan 	.setcap		= vmx_setcap,
422515add60dSPeter Grehan 	.vmspace_alloc	= vmx_vmspace_alloc,
422615add60dSPeter Grehan 	.vmspace_free	= vmx_vmspace_free,
422713a7c4d4SMark Johnston 	.vlapic_init	= vmx_vlapic_init,
422813a7c4d4SMark Johnston 	.vlapic_cleanup	= vmx_vlapic_cleanup,
4229483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT
423015add60dSPeter Grehan 	.snapshot	= vmx_snapshot,
423115add60dSPeter Grehan 	.vmcx_snapshot	= vmx_vmcx_snapshot,
423215add60dSPeter Grehan 	.restore_tsc	= vmx_restore_tsc,
4233483d953aSJohn Baldwin #endif
4234366f6083SPeter Grehan };
4235