1366f6083SPeter Grehan /*- 2366f6083SPeter Grehan * Copyright (c) 2011 NetApp, Inc. 3366f6083SPeter Grehan * All rights reserved. 4366f6083SPeter Grehan * 5366f6083SPeter Grehan * Redistribution and use in source and binary forms, with or without 6366f6083SPeter Grehan * modification, are permitted provided that the following conditions 7366f6083SPeter Grehan * are met: 8366f6083SPeter Grehan * 1. Redistributions of source code must retain the above copyright 9366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer. 10366f6083SPeter Grehan * 2. Redistributions in binary form must reproduce the above copyright 11366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer in the 12366f6083SPeter Grehan * documentation and/or other materials provided with the distribution. 13366f6083SPeter Grehan * 14366f6083SPeter Grehan * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 15366f6083SPeter Grehan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16366f6083SPeter Grehan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17366f6083SPeter Grehan * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 18366f6083SPeter Grehan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19366f6083SPeter Grehan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20366f6083SPeter Grehan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21366f6083SPeter Grehan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22366f6083SPeter Grehan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23366f6083SPeter Grehan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24366f6083SPeter Grehan * SUCH DAMAGE. 25366f6083SPeter Grehan * 26366f6083SPeter Grehan * $FreeBSD$ 27366f6083SPeter Grehan */ 28366f6083SPeter Grehan 29366f6083SPeter Grehan #include <sys/cdefs.h> 30366f6083SPeter Grehan __FBSDID("$FreeBSD$"); 31366f6083SPeter Grehan 32366f6083SPeter Grehan #include <sys/param.h> 33366f6083SPeter Grehan #include <sys/systm.h> 34366f6083SPeter Grehan #include <sys/smp.h> 35366f6083SPeter Grehan #include <sys/kernel.h> 36366f6083SPeter Grehan #include <sys/malloc.h> 37366f6083SPeter Grehan #include <sys/pcpu.h> 38366f6083SPeter Grehan #include <sys/proc.h> 393565b59eSNeel Natu #include <sys/sysctl.h> 40366f6083SPeter Grehan 41366f6083SPeter Grehan #include <vm/vm.h> 42366f6083SPeter Grehan #include <vm/pmap.h> 43366f6083SPeter Grehan 44366f6083SPeter Grehan #include <machine/psl.h> 45366f6083SPeter Grehan #include <machine/cpufunc.h> 468b287612SJohn Baldwin #include <machine/md_var.h> 47366f6083SPeter Grehan #include <machine/segments.h> 48176666c2SNeel Natu #include <machine/smp.h> 49608f97c3SPeter Grehan #include <machine/specialreg.h> 50366f6083SPeter Grehan #include <machine/vmparam.h> 51366f6083SPeter Grehan 52366f6083SPeter Grehan #include <machine/vmm.h> 53b01c2033SNeel Natu #include "vmm_host.h" 54176666c2SNeel Natu #include "vmm_ipi.h" 55366f6083SPeter Grehan #include "vmm_msr.h" 56366f6083SPeter Grehan #include "vmm_ktr.h" 57366f6083SPeter Grehan #include "vmm_stat.h" 58de5ea6b6SNeel Natu #include "vlapic.h" 59de5ea6b6SNeel Natu #include "vlapic_priv.h" 60366f6083SPeter Grehan 61366f6083SPeter Grehan #include "vmx_msr.h" 62366f6083SPeter Grehan #include "ept.h" 63366f6083SPeter Grehan #include "vmx_cpufunc.h" 64366f6083SPeter Grehan #include "vmx.h" 65366f6083SPeter Grehan #include "x86.h" 66366f6083SPeter Grehan #include "vmx_controls.h" 67366f6083SPeter Grehan 68366f6083SPeter Grehan #define PINBASED_CTLS_ONE_SETTING \ 69366f6083SPeter Grehan (PINBASED_EXTINT_EXITING | \ 70366f6083SPeter Grehan PINBASED_NMI_EXITING | \ 71366f6083SPeter Grehan PINBASED_VIRTUAL_NMI) 72366f6083SPeter Grehan #define PINBASED_CTLS_ZERO_SETTING 0 73366f6083SPeter Grehan 74366f6083SPeter Grehan #define PROCBASED_CTLS_WINDOW_SETTING \ 75366f6083SPeter Grehan (PROCBASED_INT_WINDOW_EXITING | \ 76366f6083SPeter Grehan PROCBASED_NMI_WINDOW_EXITING) 77366f6083SPeter Grehan 78366f6083SPeter Grehan #define PROCBASED_CTLS_ONE_SETTING \ 79366f6083SPeter Grehan (PROCBASED_SECONDARY_CONTROLS | \ 80366f6083SPeter Grehan PROCBASED_IO_EXITING | \ 81366f6083SPeter Grehan PROCBASED_MSR_BITMAPS | \ 82366f6083SPeter Grehan PROCBASED_CTLS_WINDOW_SETTING) 83366f6083SPeter Grehan #define PROCBASED_CTLS_ZERO_SETTING \ 84366f6083SPeter Grehan (PROCBASED_CR3_LOAD_EXITING | \ 85366f6083SPeter Grehan PROCBASED_CR3_STORE_EXITING | \ 86366f6083SPeter Grehan PROCBASED_IO_BITMAPS) 87366f6083SPeter Grehan 88366f6083SPeter Grehan #define PROCBASED_CTLS2_ONE_SETTING PROCBASED2_ENABLE_EPT 89366f6083SPeter Grehan #define PROCBASED_CTLS2_ZERO_SETTING 0 90366f6083SPeter Grehan 91608f97c3SPeter Grehan #define VM_EXIT_CTLS_ONE_SETTING_NO_PAT \ 92366f6083SPeter Grehan (VM_EXIT_HOST_LMA | \ 93366f6083SPeter Grehan VM_EXIT_SAVE_EFER | \ 94366f6083SPeter Grehan VM_EXIT_LOAD_EFER) 95608f97c3SPeter Grehan 96608f97c3SPeter Grehan #define VM_EXIT_CTLS_ONE_SETTING \ 97608f97c3SPeter Grehan (VM_EXIT_CTLS_ONE_SETTING_NO_PAT | \ 98f7d47425SNeel Natu VM_EXIT_ACKNOWLEDGE_INTERRUPT | \ 99608f97c3SPeter Grehan VM_EXIT_SAVE_PAT | \ 100608f97c3SPeter Grehan VM_EXIT_LOAD_PAT) 101366f6083SPeter Grehan #define VM_EXIT_CTLS_ZERO_SETTING VM_EXIT_SAVE_DEBUG_CONTROLS 102366f6083SPeter Grehan 103608f97c3SPeter Grehan #define VM_ENTRY_CTLS_ONE_SETTING_NO_PAT VM_ENTRY_LOAD_EFER 104608f97c3SPeter Grehan 105366f6083SPeter Grehan #define VM_ENTRY_CTLS_ONE_SETTING \ 106608f97c3SPeter Grehan (VM_ENTRY_CTLS_ONE_SETTING_NO_PAT | \ 107608f97c3SPeter Grehan VM_ENTRY_LOAD_PAT) 108366f6083SPeter Grehan #define VM_ENTRY_CTLS_ZERO_SETTING \ 109366f6083SPeter Grehan (VM_ENTRY_LOAD_DEBUG_CONTROLS | \ 110366f6083SPeter Grehan VM_ENTRY_INTO_SMM | \ 111366f6083SPeter Grehan VM_ENTRY_DEACTIVATE_DUAL_MONITOR) 112366f6083SPeter Grehan 113366f6083SPeter Grehan #define guest_msr_rw(vmx, msr) \ 114366f6083SPeter Grehan msr_bitmap_change_access((vmx)->msr_bitmap, (msr), MSR_BITMAP_ACCESS_RW) 115366f6083SPeter Grehan 116366f6083SPeter Grehan #define HANDLED 1 117366f6083SPeter Grehan #define UNHANDLED 0 118366f6083SPeter Grehan 119de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VMX, "vmx", "vmx"); 120de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic"); 121366f6083SPeter Grehan 1223565b59eSNeel Natu SYSCTL_DECL(_hw_vmm); 1233565b59eSNeel Natu SYSCTL_NODE(_hw_vmm, OID_AUTO, vmx, CTLFLAG_RW, NULL, NULL); 1243565b59eSNeel Natu 125b3996dd4SJohn Baldwin int vmxon_enabled[MAXCPU]; 126366f6083SPeter Grehan static char vmxon_region[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE); 127366f6083SPeter Grehan 128366f6083SPeter Grehan static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2; 129366f6083SPeter Grehan static uint32_t exit_ctls, entry_ctls; 130366f6083SPeter Grehan 131366f6083SPeter Grehan static uint64_t cr0_ones_mask, cr0_zeros_mask; 1323565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_ones_mask, CTLFLAG_RD, 1333565b59eSNeel Natu &cr0_ones_mask, 0, NULL); 1343565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_zeros_mask, CTLFLAG_RD, 1353565b59eSNeel Natu &cr0_zeros_mask, 0, NULL); 1363565b59eSNeel Natu 137366f6083SPeter Grehan static uint64_t cr4_ones_mask, cr4_zeros_mask; 1383565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_ones_mask, CTLFLAG_RD, 1393565b59eSNeel Natu &cr4_ones_mask, 0, NULL); 1403565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_zeros_mask, CTLFLAG_RD, 1413565b59eSNeel Natu &cr4_zeros_mask, 0, NULL); 142366f6083SPeter Grehan 143608f97c3SPeter Grehan static int vmx_no_patmsr; 144608f97c3SPeter Grehan 1453565b59eSNeel Natu static int vmx_initialized; 1463565b59eSNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, initialized, CTLFLAG_RD, 1473565b59eSNeel Natu &vmx_initialized, 0, "Intel VMX initialized"); 1483565b59eSNeel Natu 149366f6083SPeter Grehan /* 150366f6083SPeter Grehan * Optional capabilities 151366f6083SPeter Grehan */ 152366f6083SPeter Grehan static int cap_halt_exit; 153366f6083SPeter Grehan static int cap_pause_exit; 154366f6083SPeter Grehan static int cap_unrestricted_guest; 155366f6083SPeter Grehan static int cap_monitor_trap; 15649cc03daSNeel Natu static int cap_invpcid; 157366f6083SPeter Grehan 15888c4b8d1SNeel Natu static int virtual_interrupt_delivery; 15988c4b8d1SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, virtual_interrupt_delivery, CTLFLAG_RD, 16088c4b8d1SNeel Natu &virtual_interrupt_delivery, 0, "APICv virtual interrupt delivery support"); 16188c4b8d1SNeel Natu 162176666c2SNeel Natu static int posted_interrupts; 163176666c2SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupts, CTLFLAG_RD, 164176666c2SNeel Natu &posted_interrupts, 0, "APICv posted interrupt support"); 165176666c2SNeel Natu 166176666c2SNeel Natu static int pirvec; 167176666c2SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupt_vector, CTLFLAG_RD, 168176666c2SNeel Natu &pirvec, 0, "APICv posted interrupt vector"); 169176666c2SNeel Natu 17045e51299SNeel Natu static struct unrhdr *vpid_unr; 17145e51299SNeel Natu static u_int vpid_alloc_failed; 17245e51299SNeel Natu SYSCTL_UINT(_hw_vmm_vmx, OID_AUTO, vpid_alloc_failed, CTLFLAG_RD, 17345e51299SNeel Natu &vpid_alloc_failed, 0, NULL); 17445e51299SNeel Natu 17588c4b8d1SNeel Natu /* 17688c4b8d1SNeel Natu * Use the last page below 4GB as the APIC access address. This address is 17788c4b8d1SNeel Natu * occupied by the boot firmware so it is guaranteed that it will not conflict 17888c4b8d1SNeel Natu * with a page in system memory. 17988c4b8d1SNeel Natu */ 18088c4b8d1SNeel Natu #define APIC_ACCESS_ADDRESS 0xFFFFF000 18188c4b8d1SNeel Natu 18288c4b8d1SNeel Natu static void vmx_inject_pir(struct vlapic *vlapic); 18388c4b8d1SNeel Natu 184366f6083SPeter Grehan #ifdef KTR 185366f6083SPeter Grehan static const char * 186366f6083SPeter Grehan exit_reason_to_str(int reason) 187366f6083SPeter Grehan { 188366f6083SPeter Grehan static char reasonbuf[32]; 189366f6083SPeter Grehan 190366f6083SPeter Grehan switch (reason) { 191366f6083SPeter Grehan case EXIT_REASON_EXCEPTION: 192366f6083SPeter Grehan return "exception"; 193366f6083SPeter Grehan case EXIT_REASON_EXT_INTR: 194366f6083SPeter Grehan return "extint"; 195366f6083SPeter Grehan case EXIT_REASON_TRIPLE_FAULT: 196366f6083SPeter Grehan return "triplefault"; 197366f6083SPeter Grehan case EXIT_REASON_INIT: 198366f6083SPeter Grehan return "init"; 199366f6083SPeter Grehan case EXIT_REASON_SIPI: 200366f6083SPeter Grehan return "sipi"; 201366f6083SPeter Grehan case EXIT_REASON_IO_SMI: 202366f6083SPeter Grehan return "iosmi"; 203366f6083SPeter Grehan case EXIT_REASON_SMI: 204366f6083SPeter Grehan return "smi"; 205366f6083SPeter Grehan case EXIT_REASON_INTR_WINDOW: 206366f6083SPeter Grehan return "intrwindow"; 207366f6083SPeter Grehan case EXIT_REASON_NMI_WINDOW: 208366f6083SPeter Grehan return "nmiwindow"; 209366f6083SPeter Grehan case EXIT_REASON_TASK_SWITCH: 210366f6083SPeter Grehan return "taskswitch"; 211366f6083SPeter Grehan case EXIT_REASON_CPUID: 212366f6083SPeter Grehan return "cpuid"; 213366f6083SPeter Grehan case EXIT_REASON_GETSEC: 214366f6083SPeter Grehan return "getsec"; 215366f6083SPeter Grehan case EXIT_REASON_HLT: 216366f6083SPeter Grehan return "hlt"; 217366f6083SPeter Grehan case EXIT_REASON_INVD: 218366f6083SPeter Grehan return "invd"; 219366f6083SPeter Grehan case EXIT_REASON_INVLPG: 220366f6083SPeter Grehan return "invlpg"; 221366f6083SPeter Grehan case EXIT_REASON_RDPMC: 222366f6083SPeter Grehan return "rdpmc"; 223366f6083SPeter Grehan case EXIT_REASON_RDTSC: 224366f6083SPeter Grehan return "rdtsc"; 225366f6083SPeter Grehan case EXIT_REASON_RSM: 226366f6083SPeter Grehan return "rsm"; 227366f6083SPeter Grehan case EXIT_REASON_VMCALL: 228366f6083SPeter Grehan return "vmcall"; 229366f6083SPeter Grehan case EXIT_REASON_VMCLEAR: 230366f6083SPeter Grehan return "vmclear"; 231366f6083SPeter Grehan case EXIT_REASON_VMLAUNCH: 232366f6083SPeter Grehan return "vmlaunch"; 233366f6083SPeter Grehan case EXIT_REASON_VMPTRLD: 234366f6083SPeter Grehan return "vmptrld"; 235366f6083SPeter Grehan case EXIT_REASON_VMPTRST: 236366f6083SPeter Grehan return "vmptrst"; 237366f6083SPeter Grehan case EXIT_REASON_VMREAD: 238366f6083SPeter Grehan return "vmread"; 239366f6083SPeter Grehan case EXIT_REASON_VMRESUME: 240366f6083SPeter Grehan return "vmresume"; 241366f6083SPeter Grehan case EXIT_REASON_VMWRITE: 242366f6083SPeter Grehan return "vmwrite"; 243366f6083SPeter Grehan case EXIT_REASON_VMXOFF: 244366f6083SPeter Grehan return "vmxoff"; 245366f6083SPeter Grehan case EXIT_REASON_VMXON: 246366f6083SPeter Grehan return "vmxon"; 247366f6083SPeter Grehan case EXIT_REASON_CR_ACCESS: 248366f6083SPeter Grehan return "craccess"; 249366f6083SPeter Grehan case EXIT_REASON_DR_ACCESS: 250366f6083SPeter Grehan return "draccess"; 251366f6083SPeter Grehan case EXIT_REASON_INOUT: 252366f6083SPeter Grehan return "inout"; 253366f6083SPeter Grehan case EXIT_REASON_RDMSR: 254366f6083SPeter Grehan return "rdmsr"; 255366f6083SPeter Grehan case EXIT_REASON_WRMSR: 256366f6083SPeter Grehan return "wrmsr"; 257366f6083SPeter Grehan case EXIT_REASON_INVAL_VMCS: 258366f6083SPeter Grehan return "invalvmcs"; 259366f6083SPeter Grehan case EXIT_REASON_INVAL_MSR: 260366f6083SPeter Grehan return "invalmsr"; 261366f6083SPeter Grehan case EXIT_REASON_MWAIT: 262366f6083SPeter Grehan return "mwait"; 263366f6083SPeter Grehan case EXIT_REASON_MTF: 264366f6083SPeter Grehan return "mtf"; 265366f6083SPeter Grehan case EXIT_REASON_MONITOR: 266366f6083SPeter Grehan return "monitor"; 267366f6083SPeter Grehan case EXIT_REASON_PAUSE: 268366f6083SPeter Grehan return "pause"; 269366f6083SPeter Grehan case EXIT_REASON_MCE: 270366f6083SPeter Grehan return "mce"; 271366f6083SPeter Grehan case EXIT_REASON_TPR: 272366f6083SPeter Grehan return "tpr"; 27388c4b8d1SNeel Natu case EXIT_REASON_APIC_ACCESS: 27488c4b8d1SNeel Natu return "apic-access"; 275366f6083SPeter Grehan case EXIT_REASON_GDTR_IDTR: 276366f6083SPeter Grehan return "gdtridtr"; 277366f6083SPeter Grehan case EXIT_REASON_LDTR_TR: 278366f6083SPeter Grehan return "ldtrtr"; 279366f6083SPeter Grehan case EXIT_REASON_EPT_FAULT: 280366f6083SPeter Grehan return "eptfault"; 281366f6083SPeter Grehan case EXIT_REASON_EPT_MISCONFIG: 282366f6083SPeter Grehan return "eptmisconfig"; 283366f6083SPeter Grehan case EXIT_REASON_INVEPT: 284366f6083SPeter Grehan return "invept"; 285366f6083SPeter Grehan case EXIT_REASON_RDTSCP: 286366f6083SPeter Grehan return "rdtscp"; 287366f6083SPeter Grehan case EXIT_REASON_VMX_PREEMPT: 288366f6083SPeter Grehan return "vmxpreempt"; 289366f6083SPeter Grehan case EXIT_REASON_INVVPID: 290366f6083SPeter Grehan return "invvpid"; 291366f6083SPeter Grehan case EXIT_REASON_WBINVD: 292366f6083SPeter Grehan return "wbinvd"; 293366f6083SPeter Grehan case EXIT_REASON_XSETBV: 294366f6083SPeter Grehan return "xsetbv"; 29588c4b8d1SNeel Natu case EXIT_REASON_APIC_WRITE: 29688c4b8d1SNeel Natu return "apic-write"; 297366f6083SPeter Grehan default: 298366f6083SPeter Grehan snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason); 299366f6083SPeter Grehan return (reasonbuf); 300366f6083SPeter Grehan } 301366f6083SPeter Grehan } 302366f6083SPeter Grehan #endif /* KTR */ 303366f6083SPeter Grehan 304366f6083SPeter Grehan u_long 305366f6083SPeter Grehan vmx_fix_cr0(u_long cr0) 306366f6083SPeter Grehan { 307366f6083SPeter Grehan 308366f6083SPeter Grehan return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask); 309366f6083SPeter Grehan } 310366f6083SPeter Grehan 311366f6083SPeter Grehan u_long 312366f6083SPeter Grehan vmx_fix_cr4(u_long cr4) 313366f6083SPeter Grehan { 314366f6083SPeter Grehan 315366f6083SPeter Grehan return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask); 316366f6083SPeter Grehan } 317366f6083SPeter Grehan 318366f6083SPeter Grehan static void 31945e51299SNeel Natu vpid_free(int vpid) 32045e51299SNeel Natu { 32145e51299SNeel Natu if (vpid < 0 || vpid > 0xffff) 32245e51299SNeel Natu panic("vpid_free: invalid vpid %d", vpid); 32345e51299SNeel Natu 32445e51299SNeel Natu /* 32545e51299SNeel Natu * VPIDs [0,VM_MAXCPU] are special and are not allocated from 32645e51299SNeel Natu * the unit number allocator. 32745e51299SNeel Natu */ 32845e51299SNeel Natu 32945e51299SNeel Natu if (vpid > VM_MAXCPU) 33045e51299SNeel Natu free_unr(vpid_unr, vpid); 33145e51299SNeel Natu } 33245e51299SNeel Natu 33345e51299SNeel Natu static void 33445e51299SNeel Natu vpid_alloc(uint16_t *vpid, int num) 33545e51299SNeel Natu { 33645e51299SNeel Natu int i, x; 33745e51299SNeel Natu 33845e51299SNeel Natu if (num <= 0 || num > VM_MAXCPU) 33945e51299SNeel Natu panic("invalid number of vpids requested: %d", num); 34045e51299SNeel Natu 34145e51299SNeel Natu /* 34245e51299SNeel Natu * If the "enable vpid" execution control is not enabled then the 34345e51299SNeel Natu * VPID is required to be 0 for all vcpus. 34445e51299SNeel Natu */ 34545e51299SNeel Natu if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) == 0) { 34645e51299SNeel Natu for (i = 0; i < num; i++) 34745e51299SNeel Natu vpid[i] = 0; 34845e51299SNeel Natu return; 34945e51299SNeel Natu } 35045e51299SNeel Natu 35145e51299SNeel Natu /* 35245e51299SNeel Natu * Allocate a unique VPID for each vcpu from the unit number allocator. 35345e51299SNeel Natu */ 35445e51299SNeel Natu for (i = 0; i < num; i++) { 35545e51299SNeel Natu x = alloc_unr(vpid_unr); 35645e51299SNeel Natu if (x == -1) 35745e51299SNeel Natu break; 35845e51299SNeel Natu else 35945e51299SNeel Natu vpid[i] = x; 36045e51299SNeel Natu } 36145e51299SNeel Natu 36245e51299SNeel Natu if (i < num) { 36345e51299SNeel Natu atomic_add_int(&vpid_alloc_failed, 1); 36445e51299SNeel Natu 36545e51299SNeel Natu /* 36645e51299SNeel Natu * If the unit number allocator does not have enough unique 36745e51299SNeel Natu * VPIDs then we need to allocate from the [1,VM_MAXCPU] range. 36845e51299SNeel Natu * 36945e51299SNeel Natu * These VPIDs are not be unique across VMs but this does not 37045e51299SNeel Natu * affect correctness because the combined mappings are also 37145e51299SNeel Natu * tagged with the EP4TA which is unique for each VM. 37245e51299SNeel Natu * 37345e51299SNeel Natu * It is still sub-optimal because the invvpid will invalidate 37445e51299SNeel Natu * combined mappings for a particular VPID across all EP4TAs. 37545e51299SNeel Natu */ 37645e51299SNeel Natu while (i-- > 0) 37745e51299SNeel Natu vpid_free(vpid[i]); 37845e51299SNeel Natu 37945e51299SNeel Natu for (i = 0; i < num; i++) 38045e51299SNeel Natu vpid[i] = i + 1; 38145e51299SNeel Natu } 38245e51299SNeel Natu } 38345e51299SNeel Natu 38445e51299SNeel Natu static void 38545e51299SNeel Natu vpid_init(void) 38645e51299SNeel Natu { 38745e51299SNeel Natu /* 38845e51299SNeel Natu * VPID 0 is required when the "enable VPID" execution control is 38945e51299SNeel Natu * disabled. 39045e51299SNeel Natu * 39145e51299SNeel Natu * VPIDs [1,VM_MAXCPU] are used as the "overflow namespace" when the 39245e51299SNeel Natu * unit number allocator does not have sufficient unique VPIDs to 39345e51299SNeel Natu * satisfy the allocation. 39445e51299SNeel Natu * 39545e51299SNeel Natu * The remaining VPIDs are managed by the unit number allocator. 39645e51299SNeel Natu */ 39745e51299SNeel Natu vpid_unr = new_unrhdr(VM_MAXCPU + 1, 0xffff, NULL); 39845e51299SNeel Natu } 39945e51299SNeel Natu 40045e51299SNeel Natu static void 401366f6083SPeter Grehan msr_save_area_init(struct msr_entry *g_area, int *g_count) 402366f6083SPeter Grehan { 403366f6083SPeter Grehan int cnt; 404366f6083SPeter Grehan 405366f6083SPeter Grehan static struct msr_entry guest_msrs[] = { 406366f6083SPeter Grehan { MSR_KGSBASE, 0, 0 }, 407366f6083SPeter Grehan }; 408366f6083SPeter Grehan 409366f6083SPeter Grehan cnt = sizeof(guest_msrs) / sizeof(guest_msrs[0]); 410366f6083SPeter Grehan if (cnt > GUEST_MSR_MAX_ENTRIES) 411366f6083SPeter Grehan panic("guest msr save area overrun"); 412366f6083SPeter Grehan bcopy(guest_msrs, g_area, sizeof(guest_msrs)); 413366f6083SPeter Grehan *g_count = cnt; 414366f6083SPeter Grehan } 415366f6083SPeter Grehan 416366f6083SPeter Grehan static void 417366f6083SPeter Grehan vmx_disable(void *arg __unused) 418366f6083SPeter Grehan { 419366f6083SPeter Grehan struct invvpid_desc invvpid_desc = { 0 }; 420366f6083SPeter Grehan struct invept_desc invept_desc = { 0 }; 421366f6083SPeter Grehan 422366f6083SPeter Grehan if (vmxon_enabled[curcpu]) { 423366f6083SPeter Grehan /* 424366f6083SPeter Grehan * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b. 425366f6083SPeter Grehan * 426366f6083SPeter Grehan * VMXON or VMXOFF are not required to invalidate any TLB 427366f6083SPeter Grehan * caching structures. This prevents potential retention of 428366f6083SPeter Grehan * cached information in the TLB between distinct VMX episodes. 429366f6083SPeter Grehan */ 430366f6083SPeter Grehan invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc); 431366f6083SPeter Grehan invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc); 432366f6083SPeter Grehan vmxoff(); 433366f6083SPeter Grehan } 434366f6083SPeter Grehan load_cr4(rcr4() & ~CR4_VMXE); 435366f6083SPeter Grehan } 436366f6083SPeter Grehan 437366f6083SPeter Grehan static int 438366f6083SPeter Grehan vmx_cleanup(void) 439366f6083SPeter Grehan { 440366f6083SPeter Grehan 441176666c2SNeel Natu if (pirvec != 0) 442176666c2SNeel Natu vmm_ipi_free(pirvec); 443176666c2SNeel Natu 44445e51299SNeel Natu if (vpid_unr != NULL) { 44545e51299SNeel Natu delete_unrhdr(vpid_unr); 44645e51299SNeel Natu vpid_unr = NULL; 44745e51299SNeel Natu } 44845e51299SNeel Natu 449366f6083SPeter Grehan smp_rendezvous(NULL, vmx_disable, NULL, NULL); 450366f6083SPeter Grehan 451366f6083SPeter Grehan return (0); 452366f6083SPeter Grehan } 453366f6083SPeter Grehan 454366f6083SPeter Grehan static void 455366f6083SPeter Grehan vmx_enable(void *arg __unused) 456366f6083SPeter Grehan { 457366f6083SPeter Grehan int error; 458366f6083SPeter Grehan 459366f6083SPeter Grehan load_cr4(rcr4() | CR4_VMXE); 460366f6083SPeter Grehan 461366f6083SPeter Grehan *(uint32_t *)vmxon_region[curcpu] = vmx_revision(); 462366f6083SPeter Grehan error = vmxon(vmxon_region[curcpu]); 463366f6083SPeter Grehan if (error == 0) 464366f6083SPeter Grehan vmxon_enabled[curcpu] = 1; 465366f6083SPeter Grehan } 466366f6083SPeter Grehan 46763e62d39SJohn Baldwin static void 46863e62d39SJohn Baldwin vmx_restore(void) 46963e62d39SJohn Baldwin { 47063e62d39SJohn Baldwin 47163e62d39SJohn Baldwin if (vmxon_enabled[curcpu]) 47263e62d39SJohn Baldwin vmxon(vmxon_region[curcpu]); 47363e62d39SJohn Baldwin } 47463e62d39SJohn Baldwin 475366f6083SPeter Grehan static int 476add611fdSNeel Natu vmx_init(int ipinum) 477366f6083SPeter Grehan { 47888c4b8d1SNeel Natu int error, use_tpr_shadow; 4794bff7fadSNeel Natu uint64_t fixed0, fixed1, feature_control; 48088c4b8d1SNeel Natu uint32_t tmp, procbased2_vid_bits; 481366f6083SPeter Grehan 482366f6083SPeter Grehan /* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */ 4838b287612SJohn Baldwin if (!(cpu_feature2 & CPUID2_VMX)) { 484366f6083SPeter Grehan printf("vmx_init: processor does not support VMX operation\n"); 485366f6083SPeter Grehan return (ENXIO); 486366f6083SPeter Grehan } 487366f6083SPeter Grehan 4884bff7fadSNeel Natu /* 4894bff7fadSNeel Natu * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits 4904bff7fadSNeel Natu * are set (bits 0 and 2 respectively). 4914bff7fadSNeel Natu */ 4924bff7fadSNeel Natu feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL); 493150369abSNeel Natu if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 0 || 494150369abSNeel Natu (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) { 4954bff7fadSNeel Natu printf("vmx_init: VMX operation disabled by BIOS\n"); 4964bff7fadSNeel Natu return (ENXIO); 4974bff7fadSNeel Natu } 4984bff7fadSNeel Natu 499366f6083SPeter Grehan /* Check support for primary processor-based VM-execution controls */ 500366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 501366f6083SPeter Grehan MSR_VMX_TRUE_PROCBASED_CTLS, 502366f6083SPeter Grehan PROCBASED_CTLS_ONE_SETTING, 503366f6083SPeter Grehan PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls); 504366f6083SPeter Grehan if (error) { 505366f6083SPeter Grehan printf("vmx_init: processor does not support desired primary " 506366f6083SPeter Grehan "processor-based controls\n"); 507366f6083SPeter Grehan return (error); 508366f6083SPeter Grehan } 509366f6083SPeter Grehan 510366f6083SPeter Grehan /* Clear the processor-based ctl bits that are set on demand */ 511366f6083SPeter Grehan procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING; 512366f6083SPeter Grehan 513366f6083SPeter Grehan /* Check support for secondary processor-based VM-execution controls */ 514366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 515366f6083SPeter Grehan MSR_VMX_PROCBASED_CTLS2, 516366f6083SPeter Grehan PROCBASED_CTLS2_ONE_SETTING, 517366f6083SPeter Grehan PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2); 518366f6083SPeter Grehan if (error) { 519366f6083SPeter Grehan printf("vmx_init: processor does not support desired secondary " 520366f6083SPeter Grehan "processor-based controls\n"); 521366f6083SPeter Grehan return (error); 522366f6083SPeter Grehan } 523366f6083SPeter Grehan 524366f6083SPeter Grehan /* Check support for VPID */ 525366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2, 526366f6083SPeter Grehan PROCBASED2_ENABLE_VPID, 0, &tmp); 527366f6083SPeter Grehan if (error == 0) 528366f6083SPeter Grehan procbased_ctls2 |= PROCBASED2_ENABLE_VPID; 529366f6083SPeter Grehan 530366f6083SPeter Grehan /* Check support for pin-based VM-execution controls */ 531366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS, 532366f6083SPeter Grehan MSR_VMX_TRUE_PINBASED_CTLS, 533366f6083SPeter Grehan PINBASED_CTLS_ONE_SETTING, 534366f6083SPeter Grehan PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls); 535366f6083SPeter Grehan if (error) { 536366f6083SPeter Grehan printf("vmx_init: processor does not support desired " 537366f6083SPeter Grehan "pin-based controls\n"); 538366f6083SPeter Grehan return (error); 539366f6083SPeter Grehan } 540366f6083SPeter Grehan 541366f6083SPeter Grehan /* Check support for VM-exit controls */ 542366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS, 543366f6083SPeter Grehan VM_EXIT_CTLS_ONE_SETTING, 544366f6083SPeter Grehan VM_EXIT_CTLS_ZERO_SETTING, 545366f6083SPeter Grehan &exit_ctls); 546366f6083SPeter Grehan if (error) { 547608f97c3SPeter Grehan /* Try again without the PAT MSR bits */ 548608f97c3SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, 549608f97c3SPeter Grehan MSR_VMX_TRUE_EXIT_CTLS, 550608f97c3SPeter Grehan VM_EXIT_CTLS_ONE_SETTING_NO_PAT, 551608f97c3SPeter Grehan VM_EXIT_CTLS_ZERO_SETTING, 552608f97c3SPeter Grehan &exit_ctls); 553608f97c3SPeter Grehan if (error) { 554366f6083SPeter Grehan printf("vmx_init: processor does not support desired " 555366f6083SPeter Grehan "exit controls\n"); 556366f6083SPeter Grehan return (error); 557608f97c3SPeter Grehan } else { 558608f97c3SPeter Grehan if (bootverbose) 559608f97c3SPeter Grehan printf("vmm: PAT MSR access not supported\n"); 560608f97c3SPeter Grehan guest_msr_valid(MSR_PAT); 561608f97c3SPeter Grehan vmx_no_patmsr = 1; 562608f97c3SPeter Grehan } 563366f6083SPeter Grehan } 564366f6083SPeter Grehan 565366f6083SPeter Grehan /* Check support for VM-entry controls */ 566608f97c3SPeter Grehan if (!vmx_no_patmsr) { 567608f97c3SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS, 568608f97c3SPeter Grehan MSR_VMX_TRUE_ENTRY_CTLS, 569366f6083SPeter Grehan VM_ENTRY_CTLS_ONE_SETTING, 570366f6083SPeter Grehan VM_ENTRY_CTLS_ZERO_SETTING, 571366f6083SPeter Grehan &entry_ctls); 572608f97c3SPeter Grehan } else { 573608f97c3SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS, 574608f97c3SPeter Grehan MSR_VMX_TRUE_ENTRY_CTLS, 575608f97c3SPeter Grehan VM_ENTRY_CTLS_ONE_SETTING_NO_PAT, 576608f97c3SPeter Grehan VM_ENTRY_CTLS_ZERO_SETTING, 577608f97c3SPeter Grehan &entry_ctls); 578608f97c3SPeter Grehan } 579608f97c3SPeter Grehan 580366f6083SPeter Grehan if (error) { 581366f6083SPeter Grehan printf("vmx_init: processor does not support desired " 582366f6083SPeter Grehan "entry controls\n"); 583366f6083SPeter Grehan return (error); 584366f6083SPeter Grehan } 585366f6083SPeter Grehan 586366f6083SPeter Grehan /* 587366f6083SPeter Grehan * Check support for optional features by testing them 588366f6083SPeter Grehan * as individual bits 589366f6083SPeter Grehan */ 590366f6083SPeter Grehan cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 591366f6083SPeter Grehan MSR_VMX_TRUE_PROCBASED_CTLS, 592366f6083SPeter Grehan PROCBASED_HLT_EXITING, 0, 593366f6083SPeter Grehan &tmp) == 0); 594366f6083SPeter Grehan 595366f6083SPeter Grehan cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 596366f6083SPeter Grehan MSR_VMX_PROCBASED_CTLS, 597366f6083SPeter Grehan PROCBASED_MTF, 0, 598366f6083SPeter Grehan &tmp) == 0); 599366f6083SPeter Grehan 600366f6083SPeter Grehan cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 601366f6083SPeter Grehan MSR_VMX_TRUE_PROCBASED_CTLS, 602366f6083SPeter Grehan PROCBASED_PAUSE_EXITING, 0, 603366f6083SPeter Grehan &tmp) == 0); 604366f6083SPeter Grehan 605366f6083SPeter Grehan cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 606366f6083SPeter Grehan MSR_VMX_PROCBASED_CTLS2, 607366f6083SPeter Grehan PROCBASED2_UNRESTRICTED_GUEST, 0, 608366f6083SPeter Grehan &tmp) == 0); 609366f6083SPeter Grehan 61049cc03daSNeel Natu cap_invpcid = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 61149cc03daSNeel Natu MSR_VMX_PROCBASED_CTLS2, PROCBASED2_ENABLE_INVPCID, 0, 61249cc03daSNeel Natu &tmp) == 0); 61349cc03daSNeel Natu 61488c4b8d1SNeel Natu /* 61588c4b8d1SNeel Natu * Check support for virtual interrupt delivery. 61688c4b8d1SNeel Natu */ 61788c4b8d1SNeel Natu procbased2_vid_bits = (PROCBASED2_VIRTUALIZE_APIC_ACCESSES | 61888c4b8d1SNeel Natu PROCBASED2_VIRTUALIZE_X2APIC_MODE | 61988c4b8d1SNeel Natu PROCBASED2_APIC_REGISTER_VIRTUALIZATION | 62088c4b8d1SNeel Natu PROCBASED2_VIRTUAL_INTERRUPT_DELIVERY); 62188c4b8d1SNeel Natu 62288c4b8d1SNeel Natu use_tpr_shadow = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 62388c4b8d1SNeel Natu MSR_VMX_TRUE_PROCBASED_CTLS, PROCBASED_USE_TPR_SHADOW, 0, 62488c4b8d1SNeel Natu &tmp) == 0); 62588c4b8d1SNeel Natu 62688c4b8d1SNeel Natu error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2, 62788c4b8d1SNeel Natu procbased2_vid_bits, 0, &tmp); 62888c4b8d1SNeel Natu if (error == 0 && use_tpr_shadow) { 62988c4b8d1SNeel Natu virtual_interrupt_delivery = 1; 63088c4b8d1SNeel Natu TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_vid", 63188c4b8d1SNeel Natu &virtual_interrupt_delivery); 63288c4b8d1SNeel Natu } 63388c4b8d1SNeel Natu 63488c4b8d1SNeel Natu if (virtual_interrupt_delivery) { 63588c4b8d1SNeel Natu procbased_ctls |= PROCBASED_USE_TPR_SHADOW; 63688c4b8d1SNeel Natu procbased_ctls2 |= procbased2_vid_bits; 63788c4b8d1SNeel Natu procbased_ctls2 &= ~PROCBASED2_VIRTUALIZE_X2APIC_MODE; 638176666c2SNeel Natu 639176666c2SNeel Natu /* 640176666c2SNeel Natu * Check for Posted Interrupts only if Virtual Interrupt 641176666c2SNeel Natu * Delivery is enabled. 642176666c2SNeel Natu */ 643176666c2SNeel Natu error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS, 644176666c2SNeel Natu MSR_VMX_TRUE_PINBASED_CTLS, PINBASED_POSTED_INTERRUPT, 0, 645176666c2SNeel Natu &tmp); 646176666c2SNeel Natu if (error == 0) { 647176666c2SNeel Natu pirvec = vmm_ipi_alloc(); 648176666c2SNeel Natu if (pirvec == 0) { 649176666c2SNeel Natu if (bootverbose) { 650176666c2SNeel Natu printf("vmx_init: unable to allocate " 651176666c2SNeel Natu "posted interrupt vector\n"); 65288c4b8d1SNeel Natu } 653176666c2SNeel Natu } else { 654176666c2SNeel Natu posted_interrupts = 1; 655176666c2SNeel Natu TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_pir", 656176666c2SNeel Natu &posted_interrupts); 657176666c2SNeel Natu } 658176666c2SNeel Natu } 659176666c2SNeel Natu } 660176666c2SNeel Natu 661176666c2SNeel Natu if (posted_interrupts) 662176666c2SNeel Natu pinbased_ctls |= PINBASED_POSTED_INTERRUPT; 66349cc03daSNeel Natu 664366f6083SPeter Grehan /* Initialize EPT */ 665add611fdSNeel Natu error = ept_init(ipinum); 666366f6083SPeter Grehan if (error) { 667366f6083SPeter Grehan printf("vmx_init: ept initialization failed (%d)\n", error); 668366f6083SPeter Grehan return (error); 669366f6083SPeter Grehan } 670366f6083SPeter Grehan 671366f6083SPeter Grehan /* 672366f6083SPeter Grehan * Stash the cr0 and cr4 bits that must be fixed to 0 or 1 673366f6083SPeter Grehan */ 674366f6083SPeter Grehan fixed0 = rdmsr(MSR_VMX_CR0_FIXED0); 675366f6083SPeter Grehan fixed1 = rdmsr(MSR_VMX_CR0_FIXED1); 676366f6083SPeter Grehan cr0_ones_mask = fixed0 & fixed1; 677366f6083SPeter Grehan cr0_zeros_mask = ~fixed0 & ~fixed1; 678366f6083SPeter Grehan 679366f6083SPeter Grehan /* 680366f6083SPeter Grehan * CR0_PE and CR0_PG can be set to zero in VMX non-root operation 681366f6083SPeter Grehan * if unrestricted guest execution is allowed. 682366f6083SPeter Grehan */ 683366f6083SPeter Grehan if (cap_unrestricted_guest) 684366f6083SPeter Grehan cr0_ones_mask &= ~(CR0_PG | CR0_PE); 685366f6083SPeter Grehan 686366f6083SPeter Grehan /* 687366f6083SPeter Grehan * Do not allow the guest to set CR0_NW or CR0_CD. 688366f6083SPeter Grehan */ 689366f6083SPeter Grehan cr0_zeros_mask |= (CR0_NW | CR0_CD); 690366f6083SPeter Grehan 691366f6083SPeter Grehan fixed0 = rdmsr(MSR_VMX_CR4_FIXED0); 692366f6083SPeter Grehan fixed1 = rdmsr(MSR_VMX_CR4_FIXED1); 693366f6083SPeter Grehan cr4_ones_mask = fixed0 & fixed1; 694366f6083SPeter Grehan cr4_zeros_mask = ~fixed0 & ~fixed1; 695366f6083SPeter Grehan 69645e51299SNeel Natu vpid_init(); 69745e51299SNeel Natu 698366f6083SPeter Grehan /* enable VMX operation */ 699366f6083SPeter Grehan smp_rendezvous(NULL, vmx_enable, NULL, NULL); 700366f6083SPeter Grehan 7013565b59eSNeel Natu vmx_initialized = 1; 7023565b59eSNeel Natu 703366f6083SPeter Grehan return (0); 704366f6083SPeter Grehan } 705366f6083SPeter Grehan 706f7d47425SNeel Natu static void 707f7d47425SNeel Natu vmx_trigger_hostintr(int vector) 708f7d47425SNeel Natu { 709f7d47425SNeel Natu uintptr_t func; 710f7d47425SNeel Natu struct gate_descriptor *gd; 711f7d47425SNeel Natu 712f7d47425SNeel Natu gd = &idt[vector]; 713f7d47425SNeel Natu 714f7d47425SNeel Natu KASSERT(vector >= 32 && vector <= 255, ("vmx_trigger_hostintr: " 715f7d47425SNeel Natu "invalid vector %d", vector)); 716f7d47425SNeel Natu KASSERT(gd->gd_p == 1, ("gate descriptor for vector %d not present", 717f7d47425SNeel Natu vector)); 718f7d47425SNeel Natu KASSERT(gd->gd_type == SDT_SYSIGT, ("gate descriptor for vector %d " 719f7d47425SNeel Natu "has invalid type %d", vector, gd->gd_type)); 720f7d47425SNeel Natu KASSERT(gd->gd_dpl == SEL_KPL, ("gate descriptor for vector %d " 721f7d47425SNeel Natu "has invalid dpl %d", vector, gd->gd_dpl)); 722f7d47425SNeel Natu KASSERT(gd->gd_selector == GSEL(GCODE_SEL, SEL_KPL), ("gate descriptor " 723f7d47425SNeel Natu "for vector %d has invalid selector %d", vector, gd->gd_selector)); 724f7d47425SNeel Natu KASSERT(gd->gd_ist == 0, ("gate descriptor for vector %d has invalid " 725f7d47425SNeel Natu "IST %d", vector, gd->gd_ist)); 726f7d47425SNeel Natu 727f7d47425SNeel Natu func = ((long)gd->gd_hioffset << 16 | gd->gd_looffset); 728f7d47425SNeel Natu vmx_call_isr(func); 729f7d47425SNeel Natu } 730f7d47425SNeel Natu 731366f6083SPeter Grehan static int 732aaaa0656SPeter Grehan vmx_setup_cr_shadow(int which, struct vmcs *vmcs, uint32_t initial) 733366f6083SPeter Grehan { 73439c21c2dSNeel Natu int error, mask_ident, shadow_ident; 735aaaa0656SPeter Grehan uint64_t mask_value; 736366f6083SPeter Grehan 73739c21c2dSNeel Natu if (which != 0 && which != 4) 73839c21c2dSNeel Natu panic("vmx_setup_cr_shadow: unknown cr%d", which); 73939c21c2dSNeel Natu 74039c21c2dSNeel Natu if (which == 0) { 74139c21c2dSNeel Natu mask_ident = VMCS_CR0_MASK; 74239c21c2dSNeel Natu mask_value = cr0_ones_mask | cr0_zeros_mask; 74339c21c2dSNeel Natu shadow_ident = VMCS_CR0_SHADOW; 74439c21c2dSNeel Natu } else { 74539c21c2dSNeel Natu mask_ident = VMCS_CR4_MASK; 74639c21c2dSNeel Natu mask_value = cr4_ones_mask | cr4_zeros_mask; 74739c21c2dSNeel Natu shadow_ident = VMCS_CR4_SHADOW; 74839c21c2dSNeel Natu } 74939c21c2dSNeel Natu 750d3c11f40SPeter Grehan error = vmcs_setreg(vmcs, 0, VMCS_IDENT(mask_ident), mask_value); 751366f6083SPeter Grehan if (error) 752366f6083SPeter Grehan return (error); 753366f6083SPeter Grehan 754aaaa0656SPeter Grehan error = vmcs_setreg(vmcs, 0, VMCS_IDENT(shadow_ident), initial); 755366f6083SPeter Grehan if (error) 756366f6083SPeter Grehan return (error); 757366f6083SPeter Grehan 758366f6083SPeter Grehan return (0); 759366f6083SPeter Grehan } 760aaaa0656SPeter Grehan #define vmx_setup_cr0_shadow(vmcs,init) vmx_setup_cr_shadow(0, (vmcs), (init)) 761aaaa0656SPeter Grehan #define vmx_setup_cr4_shadow(vmcs,init) vmx_setup_cr_shadow(4, (vmcs), (init)) 762366f6083SPeter Grehan 763366f6083SPeter Grehan static void * 764318224bbSNeel Natu vmx_vminit(struct vm *vm, pmap_t pmap) 765366f6083SPeter Grehan { 76645e51299SNeel Natu uint16_t vpid[VM_MAXCPU]; 767366f6083SPeter Grehan int i, error, guest_msr_count; 768366f6083SPeter Grehan struct vmx *vmx; 769c847a506SNeel Natu struct vmcs *vmcs; 770366f6083SPeter Grehan 771366f6083SPeter Grehan vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO); 772366f6083SPeter Grehan if ((uintptr_t)vmx & PAGE_MASK) { 773366f6083SPeter Grehan panic("malloc of struct vmx not aligned on %d byte boundary", 774366f6083SPeter Grehan PAGE_SIZE); 775366f6083SPeter Grehan } 776366f6083SPeter Grehan vmx->vm = vm; 777366f6083SPeter Grehan 778318224bbSNeel Natu vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pml4)); 779318224bbSNeel Natu 780366f6083SPeter Grehan /* 781366f6083SPeter Grehan * Clean up EPTP-tagged guest physical and combined mappings 782366f6083SPeter Grehan * 783366f6083SPeter Grehan * VMX transitions are not required to invalidate any guest physical 784366f6083SPeter Grehan * mappings. So, it may be possible for stale guest physical mappings 785366f6083SPeter Grehan * to be present in the processor TLBs. 786366f6083SPeter Grehan * 787366f6083SPeter Grehan * Combined mappings for this EP4TA are also invalidated for all VPIDs. 788366f6083SPeter Grehan */ 789318224bbSNeel Natu ept_invalidate_mappings(vmx->eptp); 790366f6083SPeter Grehan 791366f6083SPeter Grehan msr_bitmap_initialize(vmx->msr_bitmap); 792366f6083SPeter Grehan 793366f6083SPeter Grehan /* 794366f6083SPeter Grehan * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE. 795366f6083SPeter Grehan * The guest FSBASE and GSBASE are saved and restored during 796366f6083SPeter Grehan * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are 797366f6083SPeter Grehan * always restored from the vmcs host state area on vm-exit. 798366f6083SPeter Grehan * 7991fb0ea3fSPeter Grehan * The SYSENTER_CS/ESP/EIP MSRs are identical to FS/GSBASE in 8001fb0ea3fSPeter Grehan * how they are saved/restored so can be directly accessed by the 8011fb0ea3fSPeter Grehan * guest. 8021fb0ea3fSPeter Grehan * 803366f6083SPeter Grehan * Guest KGSBASE is saved and restored in the guest MSR save area. 804366f6083SPeter Grehan * Host KGSBASE is restored before returning to userland from the pcb. 805366f6083SPeter Grehan * There will be a window of time when we are executing in the host 806366f6083SPeter Grehan * kernel context with a value of KGSBASE from the guest. This is ok 807366f6083SPeter Grehan * because the value of KGSBASE is inconsequential in kernel context. 808366f6083SPeter Grehan * 809366f6083SPeter Grehan * MSR_EFER is saved and restored in the guest VMCS area on a 810366f6083SPeter Grehan * VM exit and entry respectively. It is also restored from the 811366f6083SPeter Grehan * host VMCS area on a VM exit. 812366f6083SPeter Grehan */ 813366f6083SPeter Grehan if (guest_msr_rw(vmx, MSR_GSBASE) || 814366f6083SPeter Grehan guest_msr_rw(vmx, MSR_FSBASE) || 8151fb0ea3fSPeter Grehan guest_msr_rw(vmx, MSR_SYSENTER_CS_MSR) || 8161fb0ea3fSPeter Grehan guest_msr_rw(vmx, MSR_SYSENTER_ESP_MSR) || 8171fb0ea3fSPeter Grehan guest_msr_rw(vmx, MSR_SYSENTER_EIP_MSR) || 818366f6083SPeter Grehan guest_msr_rw(vmx, MSR_KGSBASE) || 819608f97c3SPeter Grehan guest_msr_rw(vmx, MSR_EFER)) 820366f6083SPeter Grehan panic("vmx_vminit: error setting guest msr access"); 821366f6083SPeter Grehan 822608f97c3SPeter Grehan /* 823608f97c3SPeter Grehan * MSR_PAT is saved and restored in the guest VMCS are on a VM exit 824608f97c3SPeter Grehan * and entry respectively. It is also restored from the host VMCS 825608f97c3SPeter Grehan * area on a VM exit. However, if running on a system with no 826608f97c3SPeter Grehan * MSR_PAT save/restore support, leave access disabled so accesses 827608f97c3SPeter Grehan * will be trapped. 828608f97c3SPeter Grehan */ 829608f97c3SPeter Grehan if (!vmx_no_patmsr && guest_msr_rw(vmx, MSR_PAT)) 830608f97c3SPeter Grehan panic("vmx_vminit: error setting guest pat msr access"); 831608f97c3SPeter Grehan 83245e51299SNeel Natu vpid_alloc(vpid, VM_MAXCPU); 83345e51299SNeel Natu 83488c4b8d1SNeel Natu if (virtual_interrupt_delivery) { 83588c4b8d1SNeel Natu error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE, 83688c4b8d1SNeel Natu APIC_ACCESS_ADDRESS); 83788c4b8d1SNeel Natu /* XXX this should really return an error to the caller */ 83888c4b8d1SNeel Natu KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error)); 83988c4b8d1SNeel Natu } 84088c4b8d1SNeel Natu 841366f6083SPeter Grehan for (i = 0; i < VM_MAXCPU; i++) { 842c847a506SNeel Natu vmcs = &vmx->vmcs[i]; 843c847a506SNeel Natu vmcs->identifier = vmx_revision(); 844c847a506SNeel Natu error = vmclear(vmcs); 845366f6083SPeter Grehan if (error != 0) { 846366f6083SPeter Grehan panic("vmx_vminit: vmclear error %d on vcpu %d\n", 847366f6083SPeter Grehan error, i); 848366f6083SPeter Grehan } 849366f6083SPeter Grehan 850c847a506SNeel Natu error = vmcs_init(vmcs); 851c847a506SNeel Natu KASSERT(error == 0, ("vmcs_init error %d", error)); 852366f6083SPeter Grehan 853c847a506SNeel Natu VMPTRLD(vmcs); 854c847a506SNeel Natu error = 0; 855c847a506SNeel Natu error += vmwrite(VMCS_HOST_RSP, (u_long)&vmx->ctx[i]); 856c847a506SNeel Natu error += vmwrite(VMCS_EPTP, vmx->eptp); 857c847a506SNeel Natu error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls); 858c847a506SNeel Natu error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls); 859c847a506SNeel Natu error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2); 860c847a506SNeel Natu error += vmwrite(VMCS_EXIT_CTLS, exit_ctls); 861c847a506SNeel Natu error += vmwrite(VMCS_ENTRY_CTLS, entry_ctls); 862c847a506SNeel Natu error += vmwrite(VMCS_MSR_BITMAP, vtophys(vmx->msr_bitmap)); 863c847a506SNeel Natu error += vmwrite(VMCS_VPID, vpid[i]); 86488c4b8d1SNeel Natu if (virtual_interrupt_delivery) { 86588c4b8d1SNeel Natu error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS); 86688c4b8d1SNeel Natu error += vmwrite(VMCS_VIRTUAL_APIC, 86788c4b8d1SNeel Natu vtophys(&vmx->apic_page[i])); 86888c4b8d1SNeel Natu error += vmwrite(VMCS_EOI_EXIT0, 0); 86988c4b8d1SNeel Natu error += vmwrite(VMCS_EOI_EXIT1, 0); 87088c4b8d1SNeel Natu error += vmwrite(VMCS_EOI_EXIT2, 0); 87188c4b8d1SNeel Natu error += vmwrite(VMCS_EOI_EXIT3, 0); 87288c4b8d1SNeel Natu } 873176666c2SNeel Natu if (posted_interrupts) { 874176666c2SNeel Natu error += vmwrite(VMCS_PIR_VECTOR, pirvec); 875176666c2SNeel Natu error += vmwrite(VMCS_PIR_DESC, 876176666c2SNeel Natu vtophys(&vmx->pir_desc[i])); 877176666c2SNeel Natu } 878c847a506SNeel Natu VMCLEAR(vmcs); 879c847a506SNeel Natu KASSERT(error == 0, ("vmx_vminit: error customizing the vmcs")); 880366f6083SPeter Grehan 881366f6083SPeter Grehan vmx->cap[i].set = 0; 882366f6083SPeter Grehan vmx->cap[i].proc_ctls = procbased_ctls; 88349cc03daSNeel Natu vmx->cap[i].proc_ctls2 = procbased_ctls2; 884366f6083SPeter Grehan 885366f6083SPeter Grehan vmx->state[i].lastcpu = -1; 88645e51299SNeel Natu vmx->state[i].vpid = vpid[i]; 887366f6083SPeter Grehan 888366f6083SPeter Grehan msr_save_area_init(vmx->guest_msrs[i], &guest_msr_count); 889366f6083SPeter Grehan 890c847a506SNeel Natu error = vmcs_set_msr_save(vmcs, vtophys(vmx->guest_msrs[i]), 891366f6083SPeter Grehan guest_msr_count); 892366f6083SPeter Grehan if (error != 0) 893366f6083SPeter Grehan panic("vmcs_set_msr_save error %d", error); 894366f6083SPeter Grehan 895aaaa0656SPeter Grehan /* 896aaaa0656SPeter Grehan * Set up the CR0/4 shadows, and init the read shadow 897aaaa0656SPeter Grehan * to the power-on register value from the Intel Sys Arch. 898aaaa0656SPeter Grehan * CR0 - 0x60000010 899aaaa0656SPeter Grehan * CR4 - 0 900aaaa0656SPeter Grehan */ 901c847a506SNeel Natu error = vmx_setup_cr0_shadow(vmcs, 0x60000010); 90239c21c2dSNeel Natu if (error != 0) 90339c21c2dSNeel Natu panic("vmx_setup_cr0_shadow %d", error); 90439c21c2dSNeel Natu 905c847a506SNeel Natu error = vmx_setup_cr4_shadow(vmcs, 0); 90639c21c2dSNeel Natu if (error != 0) 90739c21c2dSNeel Natu panic("vmx_setup_cr4_shadow %d", error); 908318224bbSNeel Natu 909318224bbSNeel Natu vmx->ctx[i].pmap = pmap; 910318224bbSNeel Natu vmx->ctx[i].eptp = vmx->eptp; 911366f6083SPeter Grehan } 912366f6083SPeter Grehan 913366f6083SPeter Grehan return (vmx); 914366f6083SPeter Grehan } 915366f6083SPeter Grehan 916366f6083SPeter Grehan static int 917a2da7af6SNeel Natu vmx_handle_cpuid(struct vm *vm, int vcpu, struct vmxctx *vmxctx) 918366f6083SPeter Grehan { 919366f6083SPeter Grehan int handled, func; 920366f6083SPeter Grehan 921366f6083SPeter Grehan func = vmxctx->guest_rax; 922366f6083SPeter Grehan 923a2da7af6SNeel Natu handled = x86_emulate_cpuid(vm, vcpu, 924a2da7af6SNeel Natu (uint32_t*)(&vmxctx->guest_rax), 925a2da7af6SNeel Natu (uint32_t*)(&vmxctx->guest_rbx), 926a2da7af6SNeel Natu (uint32_t*)(&vmxctx->guest_rcx), 927a2da7af6SNeel Natu (uint32_t*)(&vmxctx->guest_rdx)); 928366f6083SPeter Grehan return (handled); 929366f6083SPeter Grehan } 930366f6083SPeter Grehan 931366f6083SPeter Grehan static __inline void 932366f6083SPeter Grehan vmx_run_trace(struct vmx *vmx, int vcpu) 933366f6083SPeter Grehan { 934366f6083SPeter Grehan #ifdef KTR 935513c8d33SNeel Natu VCPU_CTR1(vmx->vm, vcpu, "Resume execution at %#lx", vmcs_guest_rip()); 936366f6083SPeter Grehan #endif 937366f6083SPeter Grehan } 938366f6083SPeter Grehan 939366f6083SPeter Grehan static __inline void 940366f6083SPeter Grehan vmx_exit_trace(struct vmx *vmx, int vcpu, uint64_t rip, uint32_t exit_reason, 941eeefa4e4SNeel Natu int handled) 942366f6083SPeter Grehan { 943366f6083SPeter Grehan #ifdef KTR 944513c8d33SNeel Natu VCPU_CTR3(vmx->vm, vcpu, "%s %s vmexit at 0x%0lx", 945366f6083SPeter Grehan handled ? "handled" : "unhandled", 946366f6083SPeter Grehan exit_reason_to_str(exit_reason), rip); 947eeefa4e4SNeel Natu #endif 948eeefa4e4SNeel Natu } 949366f6083SPeter Grehan 950eeefa4e4SNeel Natu static __inline void 951eeefa4e4SNeel Natu vmx_astpending_trace(struct vmx *vmx, int vcpu, uint64_t rip) 952eeefa4e4SNeel Natu { 953eeefa4e4SNeel Natu #ifdef KTR 954513c8d33SNeel Natu VCPU_CTR1(vmx->vm, vcpu, "astpending vmexit at 0x%0lx", rip); 955366f6083SPeter Grehan #endif 956366f6083SPeter Grehan } 957366f6083SPeter Grehan 9583de83862SNeel Natu static void 959366f6083SPeter Grehan vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu) 960366f6083SPeter Grehan { 9613de83862SNeel Natu int lastcpu; 962366f6083SPeter Grehan struct vmxstate *vmxstate; 963366f6083SPeter Grehan struct invvpid_desc invvpid_desc = { 0 }; 964366f6083SPeter Grehan 965366f6083SPeter Grehan vmxstate = &vmx->state[vcpu]; 966366f6083SPeter Grehan lastcpu = vmxstate->lastcpu; 967366f6083SPeter Grehan vmxstate->lastcpu = curcpu; 968366f6083SPeter Grehan 9693de83862SNeel Natu if (lastcpu == curcpu) 9703de83862SNeel Natu return; 971366f6083SPeter Grehan 972366f6083SPeter Grehan vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1); 973366f6083SPeter Grehan 9743de83862SNeel Natu vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase()); 9753de83862SNeel Natu vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase()); 9763de83862SNeel Natu vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase()); 977366f6083SPeter Grehan 978366f6083SPeter Grehan /* 979366f6083SPeter Grehan * If we are using VPIDs then invalidate all mappings tagged with 'vpid' 980366f6083SPeter Grehan * 981366f6083SPeter Grehan * We do this because this vcpu was executing on a different host 982366f6083SPeter Grehan * cpu when it last ran. We do not track whether it invalidated 983366f6083SPeter Grehan * mappings associated with its 'vpid' during that run. So we must 984366f6083SPeter Grehan * assume that the mappings associated with 'vpid' on 'curcpu' are 985366f6083SPeter Grehan * stale and invalidate them. 986366f6083SPeter Grehan * 987366f6083SPeter Grehan * Note that we incur this penalty only when the scheduler chooses to 988366f6083SPeter Grehan * move the thread associated with this vcpu between host cpus. 989366f6083SPeter Grehan * 990366f6083SPeter Grehan * Note also that this will invalidate mappings tagged with 'vpid' 991366f6083SPeter Grehan * for "all" EP4TAs. 992366f6083SPeter Grehan */ 993366f6083SPeter Grehan if (vmxstate->vpid != 0) { 994366f6083SPeter Grehan invvpid_desc.vpid = vmxstate->vpid; 995366f6083SPeter Grehan invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc); 996366f6083SPeter Grehan } 997366f6083SPeter Grehan } 998366f6083SPeter Grehan 999366f6083SPeter Grehan /* 1000366f6083SPeter Grehan * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set. 1001366f6083SPeter Grehan */ 1002366f6083SPeter Grehan CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0); 1003366f6083SPeter Grehan 1004366f6083SPeter Grehan static void __inline 1005366f6083SPeter Grehan vmx_set_int_window_exiting(struct vmx *vmx, int vcpu) 1006366f6083SPeter Grehan { 1007366f6083SPeter Grehan 100848b2d828SNeel Natu if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) { 1009366f6083SPeter Grehan vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING; 10103de83862SNeel Natu vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 101148b2d828SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting"); 101248b2d828SNeel Natu } 1013366f6083SPeter Grehan } 1014366f6083SPeter Grehan 1015366f6083SPeter Grehan static void __inline 1016366f6083SPeter Grehan vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu) 1017366f6083SPeter Grehan { 1018366f6083SPeter Grehan 101948b2d828SNeel Natu KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0, 102048b2d828SNeel Natu ("intr_window_exiting not set: %#x", vmx->cap[vcpu].proc_ctls)); 1021366f6083SPeter Grehan vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING; 10223de83862SNeel Natu vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 102348b2d828SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting"); 1024366f6083SPeter Grehan } 1025366f6083SPeter Grehan 1026366f6083SPeter Grehan static void __inline 1027366f6083SPeter Grehan vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu) 1028366f6083SPeter Grehan { 1029366f6083SPeter Grehan 103048b2d828SNeel Natu if ((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) { 1031366f6083SPeter Grehan vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING; 10323de83862SNeel Natu vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 103348b2d828SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting"); 103448b2d828SNeel Natu } 1035366f6083SPeter Grehan } 1036366f6083SPeter Grehan 1037366f6083SPeter Grehan static void __inline 1038366f6083SPeter Grehan vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu) 1039366f6083SPeter Grehan { 1040366f6083SPeter Grehan 104148b2d828SNeel Natu KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0, 104248b2d828SNeel Natu ("nmi_window_exiting not set %#x", vmx->cap[vcpu].proc_ctls)); 1043366f6083SPeter Grehan vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING; 10443de83862SNeel Natu vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 104548b2d828SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting"); 1046366f6083SPeter Grehan } 1047366f6083SPeter Grehan 104848b2d828SNeel Natu #define NMI_BLOCKING (VMCS_INTERRUPTIBILITY_NMI_BLOCKING | \ 104948b2d828SNeel Natu VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING) 105048b2d828SNeel Natu #define HWINTR_BLOCKING (VMCS_INTERRUPTIBILITY_STI_BLOCKING | \ 105148b2d828SNeel Natu VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING) 105248b2d828SNeel Natu 105348b2d828SNeel Natu static void 1054366f6083SPeter Grehan vmx_inject_nmi(struct vmx *vmx, int vcpu) 1055366f6083SPeter Grehan { 105648b2d828SNeel Natu uint32_t gi, info; 1057366f6083SPeter Grehan 105848b2d828SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 105948b2d828SNeel Natu KASSERT((gi & NMI_BLOCKING) == 0, ("vmx_inject_nmi: invalid guest " 106048b2d828SNeel Natu "interruptibility-state %#x", gi)); 1061366f6083SPeter Grehan 106248b2d828SNeel Natu info = vmcs_read(VMCS_ENTRY_INTR_INFO); 106348b2d828SNeel Natu KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_nmi: invalid " 106448b2d828SNeel Natu "VM-entry interruption information %#x", info)); 1065366f6083SPeter Grehan 1066366f6083SPeter Grehan /* 1067366f6083SPeter Grehan * Inject the virtual NMI. The vector must be the NMI IDT entry 1068366f6083SPeter Grehan * or the VMCS entry check will fail. 1069366f6083SPeter Grehan */ 107048b2d828SNeel Natu info = IDT_NMI | VMCS_INTR_T_NMI | VMCS_INTR_VALID; 10713de83862SNeel Natu vmcs_write(VMCS_ENTRY_INTR_INFO, info); 1072366f6083SPeter Grehan 1073513c8d33SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Injecting vNMI"); 1074366f6083SPeter Grehan 1075366f6083SPeter Grehan /* Clear the request */ 1076f352ff0cSNeel Natu vm_nmi_clear(vmx->vm, vcpu); 1077366f6083SPeter Grehan } 1078366f6083SPeter Grehan 1079366f6083SPeter Grehan static void 1080de5ea6b6SNeel Natu vmx_inject_interrupts(struct vmx *vmx, int vcpu, struct vlapic *vlapic) 1081366f6083SPeter Grehan { 108248b2d828SNeel Natu int vector, need_nmi_exiting; 108348b2d828SNeel Natu uint64_t rflags; 108448b2d828SNeel Natu uint32_t gi, info; 1085366f6083SPeter Grehan 108648b2d828SNeel Natu if (vm_nmi_pending(vmx->vm, vcpu)) { 1087366f6083SPeter Grehan /* 108848b2d828SNeel Natu * If there are no conditions blocking NMI injection then 108948b2d828SNeel Natu * inject it directly here otherwise enable "NMI window 109048b2d828SNeel Natu * exiting" to inject it as soon as we can. 1091eeefa4e4SNeel Natu * 109248b2d828SNeel Natu * We also check for STI_BLOCKING because some implementations 109348b2d828SNeel Natu * don't allow NMI injection in this case. If we are running 109448b2d828SNeel Natu * on a processor that doesn't have this restriction it will 109548b2d828SNeel Natu * immediately exit and the NMI will be injected in the 109648b2d828SNeel Natu * "NMI window exiting" handler. 1097366f6083SPeter Grehan */ 109848b2d828SNeel Natu need_nmi_exiting = 1; 109948b2d828SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 110048b2d828SNeel Natu if ((gi & (HWINTR_BLOCKING | NMI_BLOCKING)) == 0) { 11013de83862SNeel Natu info = vmcs_read(VMCS_ENTRY_INTR_INFO); 110248b2d828SNeel Natu if ((info & VMCS_INTR_VALID) == 0) { 110348b2d828SNeel Natu vmx_inject_nmi(vmx, vcpu); 110448b2d828SNeel Natu need_nmi_exiting = 0; 110548b2d828SNeel Natu } else { 110648b2d828SNeel Natu VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI " 110748b2d828SNeel Natu "due to VM-entry intr info %#x", info); 110848b2d828SNeel Natu } 110948b2d828SNeel Natu } else { 111048b2d828SNeel Natu VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI due to " 111148b2d828SNeel Natu "Guest Interruptibility-state %#x", gi); 111248b2d828SNeel Natu } 1113eeefa4e4SNeel Natu 111448b2d828SNeel Natu if (need_nmi_exiting) 111548b2d828SNeel Natu vmx_set_nmi_window_exiting(vmx, vcpu); 111648b2d828SNeel Natu } 1117366f6083SPeter Grehan 111888c4b8d1SNeel Natu if (virtual_interrupt_delivery) { 111988c4b8d1SNeel Natu vmx_inject_pir(vlapic); 112088c4b8d1SNeel Natu return; 112188c4b8d1SNeel Natu } 112288c4b8d1SNeel Natu 112348b2d828SNeel Natu /* 112436736912SNeel Natu * If interrupt-window exiting is already in effect then don't bother 112536736912SNeel Natu * checking for pending interrupts. This is just an optimization and 112636736912SNeel Natu * not needed for correctness. 112748b2d828SNeel Natu */ 112836736912SNeel Natu if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) { 112936736912SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Skip interrupt injection due to " 113036736912SNeel Natu "pending int_window_exiting"); 113148b2d828SNeel Natu return; 113236736912SNeel Natu } 113348b2d828SNeel Natu 1134366f6083SPeter Grehan /* Ask the local apic for a vector to inject */ 11354d1e82a8SNeel Natu if (!vlapic_pending_intr(vlapic, &vector)) 1136366f6083SPeter Grehan return; 1137366f6083SPeter Grehan 113848b2d828SNeel Natu KASSERT(vector >= 32 && vector <= 255, ("invalid vector %d", vector)); 1139366f6083SPeter Grehan 1140366f6083SPeter Grehan /* Check RFLAGS.IF and the interruptibility state of the guest */ 11413de83862SNeel Natu rflags = vmcs_read(VMCS_GUEST_RFLAGS); 114236736912SNeel Natu if ((rflags & PSL_I) == 0) { 114336736912SNeel Natu VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to " 114436736912SNeel Natu "rflags %#lx", vector, rflags); 1145366f6083SPeter Grehan goto cantinject; 114636736912SNeel Natu } 1147366f6083SPeter Grehan 114848b2d828SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 114936736912SNeel Natu if (gi & HWINTR_BLOCKING) { 115036736912SNeel Natu VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to " 115136736912SNeel Natu "Guest Interruptibility-state %#x", vector, gi); 1152366f6083SPeter Grehan goto cantinject; 115336736912SNeel Natu } 115436736912SNeel Natu 115536736912SNeel Natu info = vmcs_read(VMCS_ENTRY_INTR_INFO); 115636736912SNeel Natu if (info & VMCS_INTR_VALID) { 115736736912SNeel Natu /* 115836736912SNeel Natu * This is expected and could happen for multiple reasons: 115936736912SNeel Natu * - A vectoring VM-entry was aborted due to astpending 116036736912SNeel Natu * - A VM-exit happened during event injection. 116136736912SNeel Natu * - An NMI was injected above or after "NMI window exiting" 116236736912SNeel Natu */ 116336736912SNeel Natu VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to " 116436736912SNeel Natu "VM-entry intr info %#x", vector, info); 116536736912SNeel Natu goto cantinject; 116636736912SNeel Natu } 1167366f6083SPeter Grehan 1168366f6083SPeter Grehan /* Inject the interrupt */ 1169160471d2SNeel Natu info = VMCS_INTR_T_HWINTR | VMCS_INTR_VALID; 1170366f6083SPeter Grehan info |= vector; 11713de83862SNeel Natu vmcs_write(VMCS_ENTRY_INTR_INFO, info); 1172366f6083SPeter Grehan 1173366f6083SPeter Grehan /* Update the Local APIC ISR */ 1174de5ea6b6SNeel Natu vlapic_intr_accepted(vlapic, vector); 1175366f6083SPeter Grehan 1176513c8d33SNeel Natu VCPU_CTR1(vmx->vm, vcpu, "Injecting hwintr at vector %d", vector); 1177366f6083SPeter Grehan 1178366f6083SPeter Grehan return; 1179366f6083SPeter Grehan 1180366f6083SPeter Grehan cantinject: 1181366f6083SPeter Grehan /* 1182366f6083SPeter Grehan * Set the Interrupt Window Exiting execution control so we can inject 1183366f6083SPeter Grehan * the interrupt as soon as blocking condition goes away. 1184366f6083SPeter Grehan */ 1185366f6083SPeter Grehan vmx_set_int_window_exiting(vmx, vcpu); 1186366f6083SPeter Grehan } 1187366f6083SPeter Grehan 1188e5a1d950SNeel Natu /* 1189e5a1d950SNeel Natu * If the Virtual NMIs execution control is '1' then the logical processor 1190e5a1d950SNeel Natu * tracks virtual-NMI blocking in the Guest Interruptibility-state field of 1191e5a1d950SNeel Natu * the VMCS. An IRET instruction in VMX non-root operation will remove any 1192e5a1d950SNeel Natu * virtual-NMI blocking. 1193e5a1d950SNeel Natu * 1194e5a1d950SNeel Natu * This unblocking occurs even if the IRET causes a fault. In this case the 1195e5a1d950SNeel Natu * hypervisor needs to restore virtual-NMI blocking before resuming the guest. 1196e5a1d950SNeel Natu */ 1197e5a1d950SNeel Natu static void 1198e5a1d950SNeel Natu vmx_restore_nmi_blocking(struct vmx *vmx, int vcpuid) 1199e5a1d950SNeel Natu { 1200e5a1d950SNeel Natu uint32_t gi; 1201e5a1d950SNeel Natu 1202e5a1d950SNeel Natu VCPU_CTR0(vmx->vm, vcpuid, "Restore Virtual-NMI blocking"); 1203e5a1d950SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 1204e5a1d950SNeel Natu gi |= VMCS_INTERRUPTIBILITY_NMI_BLOCKING; 1205e5a1d950SNeel Natu vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi); 1206e5a1d950SNeel Natu } 1207e5a1d950SNeel Natu 1208e5a1d950SNeel Natu static void 1209e5a1d950SNeel Natu vmx_clear_nmi_blocking(struct vmx *vmx, int vcpuid) 1210e5a1d950SNeel Natu { 1211e5a1d950SNeel Natu uint32_t gi; 1212e5a1d950SNeel Natu 1213e5a1d950SNeel Natu VCPU_CTR0(vmx->vm, vcpuid, "Clear Virtual-NMI blocking"); 1214e5a1d950SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 1215e5a1d950SNeel Natu gi &= ~VMCS_INTERRUPTIBILITY_NMI_BLOCKING; 1216e5a1d950SNeel Natu vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi); 1217e5a1d950SNeel Natu } 1218e5a1d950SNeel Natu 1219366f6083SPeter Grehan static int 1220366f6083SPeter Grehan vmx_emulate_cr_access(struct vmx *vmx, int vcpu, uint64_t exitqual) 1221366f6083SPeter Grehan { 12223de83862SNeel Natu int cr, vmcs_guest_cr, vmcs_shadow_cr; 122380a902efSPeter Grehan uint64_t crval, regval, ones_mask, zeros_mask; 1224366f6083SPeter Grehan const struct vmxctx *vmxctx; 1225366f6083SPeter Grehan 122639c21c2dSNeel Natu /* We only handle mov to %cr0 or %cr4 at this time */ 122739c21c2dSNeel Natu if ((exitqual & 0xf0) != 0x00) 122839c21c2dSNeel Natu return (UNHANDLED); 122939c21c2dSNeel Natu 123039c21c2dSNeel Natu cr = exitqual & 0xf; 123139c21c2dSNeel Natu if (cr != 0 && cr != 4) 1232366f6083SPeter Grehan return (UNHANDLED); 1233366f6083SPeter Grehan 12346f0c167fSDimitry Andric regval = 0; /* silence gcc */ 1235366f6083SPeter Grehan vmxctx = &vmx->ctx[vcpu]; 1236366f6083SPeter Grehan 1237366f6083SPeter Grehan /* 12383de83862SNeel Natu * We must use vmcs_write() directly here because vmcs_setreg() will 1239366f6083SPeter Grehan * call vmclear(vmcs) as a side-effect which we certainly don't want. 1240366f6083SPeter Grehan */ 1241366f6083SPeter Grehan switch ((exitqual >> 8) & 0xf) { 1242366f6083SPeter Grehan case 0: 1243366f6083SPeter Grehan regval = vmxctx->guest_rax; 1244366f6083SPeter Grehan break; 1245366f6083SPeter Grehan case 1: 1246366f6083SPeter Grehan regval = vmxctx->guest_rcx; 1247366f6083SPeter Grehan break; 1248366f6083SPeter Grehan case 2: 1249366f6083SPeter Grehan regval = vmxctx->guest_rdx; 1250366f6083SPeter Grehan break; 1251366f6083SPeter Grehan case 3: 1252366f6083SPeter Grehan regval = vmxctx->guest_rbx; 1253366f6083SPeter Grehan break; 1254366f6083SPeter Grehan case 4: 12553de83862SNeel Natu regval = vmcs_read(VMCS_GUEST_RSP); 1256366f6083SPeter Grehan break; 1257366f6083SPeter Grehan case 5: 1258366f6083SPeter Grehan regval = vmxctx->guest_rbp; 1259366f6083SPeter Grehan break; 1260366f6083SPeter Grehan case 6: 1261366f6083SPeter Grehan regval = vmxctx->guest_rsi; 1262366f6083SPeter Grehan break; 1263366f6083SPeter Grehan case 7: 1264366f6083SPeter Grehan regval = vmxctx->guest_rdi; 1265366f6083SPeter Grehan break; 1266366f6083SPeter Grehan case 8: 1267366f6083SPeter Grehan regval = vmxctx->guest_r8; 1268366f6083SPeter Grehan break; 1269366f6083SPeter Grehan case 9: 1270366f6083SPeter Grehan regval = vmxctx->guest_r9; 1271366f6083SPeter Grehan break; 1272366f6083SPeter Grehan case 10: 1273366f6083SPeter Grehan regval = vmxctx->guest_r10; 1274366f6083SPeter Grehan break; 1275366f6083SPeter Grehan case 11: 1276366f6083SPeter Grehan regval = vmxctx->guest_r11; 1277366f6083SPeter Grehan break; 1278366f6083SPeter Grehan case 12: 1279366f6083SPeter Grehan regval = vmxctx->guest_r12; 1280366f6083SPeter Grehan break; 1281366f6083SPeter Grehan case 13: 1282366f6083SPeter Grehan regval = vmxctx->guest_r13; 1283366f6083SPeter Grehan break; 1284366f6083SPeter Grehan case 14: 1285366f6083SPeter Grehan regval = vmxctx->guest_r14; 1286366f6083SPeter Grehan break; 1287366f6083SPeter Grehan case 15: 1288366f6083SPeter Grehan regval = vmxctx->guest_r15; 1289366f6083SPeter Grehan break; 1290366f6083SPeter Grehan } 1291366f6083SPeter Grehan 129239c21c2dSNeel Natu if (cr == 0) { 129339c21c2dSNeel Natu ones_mask = cr0_ones_mask; 129439c21c2dSNeel Natu zeros_mask = cr0_zeros_mask; 129539c21c2dSNeel Natu vmcs_guest_cr = VMCS_GUEST_CR0; 1296aaaa0656SPeter Grehan vmcs_shadow_cr = VMCS_CR0_SHADOW; 129739c21c2dSNeel Natu } else { 129839c21c2dSNeel Natu ones_mask = cr4_ones_mask; 129939c21c2dSNeel Natu zeros_mask = cr4_zeros_mask; 130039c21c2dSNeel Natu vmcs_guest_cr = VMCS_GUEST_CR4; 1301aaaa0656SPeter Grehan vmcs_shadow_cr = VMCS_CR4_SHADOW; 130239c21c2dSNeel Natu } 13033de83862SNeel Natu vmcs_write(vmcs_shadow_cr, regval); 1304aaaa0656SPeter Grehan 130580a902efSPeter Grehan crval = regval | ones_mask; 130680a902efSPeter Grehan crval &= ~zeros_mask; 13073de83862SNeel Natu vmcs_write(vmcs_guest_cr, crval); 1308366f6083SPeter Grehan 130980a902efSPeter Grehan if (cr == 0 && regval & CR0_PG) { 131080a902efSPeter Grehan uint64_t efer, entry_ctls; 131180a902efSPeter Grehan 131280a902efSPeter Grehan /* 131380a902efSPeter Grehan * If CR0.PG is 1 and EFER.LME is 1 then EFER.LMA and 131480a902efSPeter Grehan * the "IA-32e mode guest" bit in VM-entry control must be 131580a902efSPeter Grehan * equal. 131680a902efSPeter Grehan */ 13173de83862SNeel Natu efer = vmcs_read(VMCS_GUEST_IA32_EFER); 131880a902efSPeter Grehan if (efer & EFER_LME) { 131980a902efSPeter Grehan efer |= EFER_LMA; 13203de83862SNeel Natu vmcs_write(VMCS_GUEST_IA32_EFER, efer); 13213de83862SNeel Natu entry_ctls = vmcs_read(VMCS_ENTRY_CTLS); 132280a902efSPeter Grehan entry_ctls |= VM_ENTRY_GUEST_LMA; 13233de83862SNeel Natu vmcs_write(VMCS_ENTRY_CTLS, entry_ctls); 132480a902efSPeter Grehan } 132580a902efSPeter Grehan } 132680a902efSPeter Grehan 1327366f6083SPeter Grehan return (HANDLED); 1328366f6083SPeter Grehan } 1329366f6083SPeter Grehan 1330366f6083SPeter Grehan static int 1331318224bbSNeel Natu ept_fault_type(uint64_t ept_qual) 1332a2da7af6SNeel Natu { 1333318224bbSNeel Natu int fault_type; 1334a2da7af6SNeel Natu 1335318224bbSNeel Natu if (ept_qual & EPT_VIOLATION_DATA_WRITE) 1336318224bbSNeel Natu fault_type = VM_PROT_WRITE; 1337318224bbSNeel Natu else if (ept_qual & EPT_VIOLATION_INST_FETCH) 1338318224bbSNeel Natu fault_type = VM_PROT_EXECUTE; 1339318224bbSNeel Natu else 1340318224bbSNeel Natu fault_type= VM_PROT_READ; 1341318224bbSNeel Natu 1342318224bbSNeel Natu return (fault_type); 1343318224bbSNeel Natu } 1344318224bbSNeel Natu 1345318224bbSNeel Natu static boolean_t 1346318224bbSNeel Natu ept_emulation_fault(uint64_t ept_qual) 1347318224bbSNeel Natu { 1348318224bbSNeel Natu int read, write; 1349318224bbSNeel Natu 1350318224bbSNeel Natu /* EPT fault on an instruction fetch doesn't make sense here */ 1351a2da7af6SNeel Natu if (ept_qual & EPT_VIOLATION_INST_FETCH) 1352318224bbSNeel Natu return (FALSE); 1353a2da7af6SNeel Natu 1354318224bbSNeel Natu /* EPT fault must be a read fault or a write fault */ 1355a2da7af6SNeel Natu read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0; 1356a2da7af6SNeel Natu write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0; 13573b2b0011SPeter Grehan if ((read | write) == 0) 1358318224bbSNeel Natu return (FALSE); 1359a2da7af6SNeel Natu 1360a2da7af6SNeel Natu /* 13613b2b0011SPeter Grehan * The EPT violation must have been caused by accessing a 13623b2b0011SPeter Grehan * guest-physical address that is a translation of a guest-linear 13633b2b0011SPeter Grehan * address. 1364a2da7af6SNeel Natu */ 1365a2da7af6SNeel Natu if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 || 1366a2da7af6SNeel Natu (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) { 1367318224bbSNeel Natu return (FALSE); 1368a2da7af6SNeel Natu } 1369a2da7af6SNeel Natu 1370318224bbSNeel Natu return (TRUE); 1371a2da7af6SNeel Natu } 1372a2da7af6SNeel Natu 1373a2da7af6SNeel Natu static int 137488c4b8d1SNeel Natu vmx_handle_apic_write(struct vlapic *vlapic, uint64_t qual) 137588c4b8d1SNeel Natu { 137688c4b8d1SNeel Natu int error, handled, offset; 137788c4b8d1SNeel Natu bool retu; 137888c4b8d1SNeel Natu 137988c4b8d1SNeel Natu if (!virtual_interrupt_delivery) 138088c4b8d1SNeel Natu return (UNHANDLED); 138188c4b8d1SNeel Natu 138288c4b8d1SNeel Natu handled = 1; 138388c4b8d1SNeel Natu offset = APIC_WRITE_OFFSET(qual); 138488c4b8d1SNeel Natu switch (offset) { 138588c4b8d1SNeel Natu case APIC_OFFSET_ID: 138688c4b8d1SNeel Natu vlapic_id_write_handler(vlapic); 138788c4b8d1SNeel Natu break; 138888c4b8d1SNeel Natu case APIC_OFFSET_LDR: 138988c4b8d1SNeel Natu vlapic_ldr_write_handler(vlapic); 139088c4b8d1SNeel Natu break; 139188c4b8d1SNeel Natu case APIC_OFFSET_DFR: 139288c4b8d1SNeel Natu vlapic_dfr_write_handler(vlapic); 139388c4b8d1SNeel Natu break; 139488c4b8d1SNeel Natu case APIC_OFFSET_SVR: 139588c4b8d1SNeel Natu vlapic_svr_write_handler(vlapic); 139688c4b8d1SNeel Natu break; 139788c4b8d1SNeel Natu case APIC_OFFSET_ESR: 139888c4b8d1SNeel Natu vlapic_esr_write_handler(vlapic); 139988c4b8d1SNeel Natu break; 140088c4b8d1SNeel Natu case APIC_OFFSET_ICR_LOW: 140188c4b8d1SNeel Natu retu = false; 140288c4b8d1SNeel Natu error = vlapic_icrlo_write_handler(vlapic, &retu); 140388c4b8d1SNeel Natu if (error != 0 || retu) 140488c4b8d1SNeel Natu handled = 0; 140588c4b8d1SNeel Natu break; 140688c4b8d1SNeel Natu case APIC_OFFSET_CMCI_LVT: 140788c4b8d1SNeel Natu case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT: 140888c4b8d1SNeel Natu vlapic_lvt_write_handler(vlapic, offset); 140988c4b8d1SNeel Natu break; 141088c4b8d1SNeel Natu case APIC_OFFSET_TIMER_ICR: 141188c4b8d1SNeel Natu vlapic_icrtmr_write_handler(vlapic); 141288c4b8d1SNeel Natu break; 141388c4b8d1SNeel Natu case APIC_OFFSET_TIMER_DCR: 141488c4b8d1SNeel Natu vlapic_dcr_write_handler(vlapic); 141588c4b8d1SNeel Natu break; 141688c4b8d1SNeel Natu default: 141788c4b8d1SNeel Natu handled = 0; 141888c4b8d1SNeel Natu break; 141988c4b8d1SNeel Natu } 142088c4b8d1SNeel Natu return (handled); 142188c4b8d1SNeel Natu } 142288c4b8d1SNeel Natu 142388c4b8d1SNeel Natu static bool 142488c4b8d1SNeel Natu apic_access_fault(uint64_t gpa) 142588c4b8d1SNeel Natu { 142688c4b8d1SNeel Natu 142788c4b8d1SNeel Natu if (virtual_interrupt_delivery && 142888c4b8d1SNeel Natu (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE)) 142988c4b8d1SNeel Natu return (true); 143088c4b8d1SNeel Natu else 143188c4b8d1SNeel Natu return (false); 143288c4b8d1SNeel Natu } 143388c4b8d1SNeel Natu 143488c4b8d1SNeel Natu static int 143588c4b8d1SNeel Natu vmx_handle_apic_access(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit) 143688c4b8d1SNeel Natu { 143788c4b8d1SNeel Natu uint64_t qual; 143888c4b8d1SNeel Natu int access_type, offset, allowed; 143988c4b8d1SNeel Natu 144088c4b8d1SNeel Natu if (!virtual_interrupt_delivery) 144188c4b8d1SNeel Natu return (UNHANDLED); 144288c4b8d1SNeel Natu 144388c4b8d1SNeel Natu qual = vmexit->u.vmx.exit_qualification; 144488c4b8d1SNeel Natu access_type = APIC_ACCESS_TYPE(qual); 144588c4b8d1SNeel Natu offset = APIC_ACCESS_OFFSET(qual); 144688c4b8d1SNeel Natu 144788c4b8d1SNeel Natu allowed = 0; 144888c4b8d1SNeel Natu if (access_type == 0) { 144988c4b8d1SNeel Natu /* 145088c4b8d1SNeel Natu * Read data access to the following registers is expected. 145188c4b8d1SNeel Natu */ 145288c4b8d1SNeel Natu switch (offset) { 145388c4b8d1SNeel Natu case APIC_OFFSET_APR: 145488c4b8d1SNeel Natu case APIC_OFFSET_PPR: 145588c4b8d1SNeel Natu case APIC_OFFSET_RRR: 145688c4b8d1SNeel Natu case APIC_OFFSET_CMCI_LVT: 145788c4b8d1SNeel Natu case APIC_OFFSET_TIMER_CCR: 145888c4b8d1SNeel Natu allowed = 1; 145988c4b8d1SNeel Natu break; 146088c4b8d1SNeel Natu default: 146188c4b8d1SNeel Natu break; 146288c4b8d1SNeel Natu } 146388c4b8d1SNeel Natu } else if (access_type == 1) { 146488c4b8d1SNeel Natu /* 146588c4b8d1SNeel Natu * Write data access to the following registers is expected. 146688c4b8d1SNeel Natu */ 146788c4b8d1SNeel Natu switch (offset) { 146888c4b8d1SNeel Natu case APIC_OFFSET_VER: 146988c4b8d1SNeel Natu case APIC_OFFSET_APR: 147088c4b8d1SNeel Natu case APIC_OFFSET_PPR: 147188c4b8d1SNeel Natu case APIC_OFFSET_RRR: 147288c4b8d1SNeel Natu case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7: 147388c4b8d1SNeel Natu case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7: 147488c4b8d1SNeel Natu case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7: 147588c4b8d1SNeel Natu case APIC_OFFSET_CMCI_LVT: 147688c4b8d1SNeel Natu case APIC_OFFSET_TIMER_CCR: 147788c4b8d1SNeel Natu allowed = 1; 147888c4b8d1SNeel Natu break; 147988c4b8d1SNeel Natu default: 148088c4b8d1SNeel Natu break; 148188c4b8d1SNeel Natu } 148288c4b8d1SNeel Natu } 148388c4b8d1SNeel Natu 148488c4b8d1SNeel Natu if (allowed) { 148588c4b8d1SNeel Natu vmexit->exitcode = VM_EXITCODE_INST_EMUL; 148688c4b8d1SNeel Natu vmexit->u.inst_emul.gpa = DEFAULT_APIC_BASE + offset; 148788c4b8d1SNeel Natu vmexit->u.inst_emul.gla = VIE_INVALID_GLA; 148888c4b8d1SNeel Natu vmexit->u.inst_emul.cr3 = vmcs_guest_cr3(); 148988c4b8d1SNeel Natu } 149088c4b8d1SNeel Natu 149188c4b8d1SNeel Natu /* 149288c4b8d1SNeel Natu * Regardless of whether the APIC-access is allowed this handler 149388c4b8d1SNeel Natu * always returns UNHANDLED: 149488c4b8d1SNeel Natu * - if the access is allowed then it is handled by emulating the 149588c4b8d1SNeel Natu * instruction that caused the VM-exit (outside the critical section) 149688c4b8d1SNeel Natu * - if the access is not allowed then it will be converted to an 149788c4b8d1SNeel Natu * exitcode of VM_EXITCODE_VMX and will be dealt with in userland. 149888c4b8d1SNeel Natu */ 149988c4b8d1SNeel Natu return (UNHANDLED); 150088c4b8d1SNeel Natu } 150188c4b8d1SNeel Natu 150288c4b8d1SNeel Natu static int 1503366f6083SPeter Grehan vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) 1504366f6083SPeter Grehan { 1505f76fc5d4SNeel Natu int error, handled; 1506366f6083SPeter Grehan struct vmxctx *vmxctx; 150788c4b8d1SNeel Natu struct vlapic *vlapic; 1508e5a1d950SNeel Natu uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, reason; 15093de83862SNeel Natu uint64_t qual, gpa; 1510becd9849SNeel Natu bool retu; 1511366f6083SPeter Grehan 1512160471d2SNeel Natu CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0); 1513c308b23bSNeel Natu CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0); 1514160471d2SNeel Natu 1515366f6083SPeter Grehan handled = 0; 1516366f6083SPeter Grehan vmxctx = &vmx->ctx[vcpu]; 15170492757cSNeel Natu 1518366f6083SPeter Grehan qual = vmexit->u.vmx.exit_qualification; 1519318224bbSNeel Natu reason = vmexit->u.vmx.exit_reason; 1520366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_BOGUS; 1521366f6083SPeter Grehan 152261592433SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1); 152361592433SNeel Natu 1524318224bbSNeel Natu /* 1525318224bbSNeel Natu * VM exits that could be triggered during event injection on the 1526318224bbSNeel Natu * previous VM entry need to be handled specially by re-injecting 1527318224bbSNeel Natu * the event. 1528318224bbSNeel Natu * 1529318224bbSNeel Natu * See "Information for VM Exits During Event Delivery" in Intel SDM 1530318224bbSNeel Natu * for details. 1531318224bbSNeel Natu */ 1532318224bbSNeel Natu switch (reason) { 1533318224bbSNeel Natu case EXIT_REASON_EPT_FAULT: 1534318224bbSNeel Natu case EXIT_REASON_EPT_MISCONFIG: 153588c4b8d1SNeel Natu case EXIT_REASON_APIC_ACCESS: 1536318224bbSNeel Natu case EXIT_REASON_TASK_SWITCH: 1537318224bbSNeel Natu case EXIT_REASON_EXCEPTION: 1538318224bbSNeel Natu idtvec_info = vmcs_idt_vectoring_info(); 1539318224bbSNeel Natu if (idtvec_info & VMCS_IDT_VEC_VALID) { 1540318224bbSNeel Natu idtvec_info &= ~(1 << 12); /* clear undefined bit */ 15413de83862SNeel Natu vmcs_write(VMCS_ENTRY_INTR_INFO, idtvec_info); 1542318224bbSNeel Natu if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) { 1543318224bbSNeel Natu idtvec_err = vmcs_idt_vectoring_err(); 15443de83862SNeel Natu vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR, 15453de83862SNeel Natu idtvec_err); 1546318224bbSNeel Natu } 1547160471d2SNeel Natu /* 1548160471d2SNeel Natu * If 'virtual NMIs' are being used and the VM-exit 1549160471d2SNeel Natu * happened while injecting an NMI during the previous 1550160471d2SNeel Natu * VM-entry, then clear "blocking by NMI" in the Guest 1551160471d2SNeel Natu * Interruptibility-state. 1552160471d2SNeel Natu */ 1553160471d2SNeel Natu if ((idtvec_info & VMCS_INTR_T_MASK) == 1554160471d2SNeel Natu VMCS_INTR_T_NMI) { 1555e5a1d950SNeel Natu vmx_clear_nmi_blocking(vmx, vcpu); 1556160471d2SNeel Natu } 15573de83862SNeel Natu vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length); 1558318224bbSNeel Natu } 1559318224bbSNeel Natu default: 1560e5a1d950SNeel Natu idtvec_info = 0; 1561318224bbSNeel Natu break; 1562318224bbSNeel Natu } 1563318224bbSNeel Natu 1564318224bbSNeel Natu switch (reason) { 1565366f6083SPeter Grehan case EXIT_REASON_CR_ACCESS: 1566b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CR_ACCESS, 1); 1567366f6083SPeter Grehan handled = vmx_emulate_cr_access(vmx, vcpu, qual); 1568366f6083SPeter Grehan break; 1569366f6083SPeter Grehan case EXIT_REASON_RDMSR: 1570b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RDMSR, 1); 1571becd9849SNeel Natu retu = false; 1572366f6083SPeter Grehan ecx = vmxctx->guest_rcx; 1573becd9849SNeel Natu error = emulate_rdmsr(vmx->vm, vcpu, ecx, &retu); 1574b42206f3SNeel Natu if (error) { 1575366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_RDMSR; 1576366f6083SPeter Grehan vmexit->u.msr.code = ecx; 1577becd9849SNeel Natu } else if (!retu) { 1578b42206f3SNeel Natu handled = 1; 1579becd9849SNeel Natu } else { 1580becd9849SNeel Natu /* Return to userspace with a valid exitcode */ 1581becd9849SNeel Natu KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS, 1582becd9849SNeel Natu ("emulate_wrmsr retu with bogus exitcode")); 1583becd9849SNeel Natu } 1584366f6083SPeter Grehan break; 1585366f6083SPeter Grehan case EXIT_REASON_WRMSR: 1586b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_WRMSR, 1); 1587becd9849SNeel Natu retu = false; 1588366f6083SPeter Grehan eax = vmxctx->guest_rax; 1589366f6083SPeter Grehan ecx = vmxctx->guest_rcx; 1590366f6083SPeter Grehan edx = vmxctx->guest_rdx; 1591b42206f3SNeel Natu error = emulate_wrmsr(vmx->vm, vcpu, ecx, 1592becd9849SNeel Natu (uint64_t)edx << 32 | eax, &retu); 1593b42206f3SNeel Natu if (error) { 1594366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_WRMSR; 1595366f6083SPeter Grehan vmexit->u.msr.code = ecx; 1596366f6083SPeter Grehan vmexit->u.msr.wval = (uint64_t)edx << 32 | eax; 1597becd9849SNeel Natu } else if (!retu) { 1598b42206f3SNeel Natu handled = 1; 1599becd9849SNeel Natu } else { 1600becd9849SNeel Natu /* Return to userspace with a valid exitcode */ 1601becd9849SNeel Natu KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS, 1602becd9849SNeel Natu ("emulate_wrmsr retu with bogus exitcode")); 1603becd9849SNeel Natu } 1604366f6083SPeter Grehan break; 1605366f6083SPeter Grehan case EXIT_REASON_HLT: 1606f76fc5d4SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_HLT, 1); 1607366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_HLT; 16083de83862SNeel Natu vmexit->u.hlt.rflags = vmcs_read(VMCS_GUEST_RFLAGS); 1609366f6083SPeter Grehan break; 1610366f6083SPeter Grehan case EXIT_REASON_MTF: 1611b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_MTRAP, 1); 1612366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_MTRAP; 1613366f6083SPeter Grehan break; 1614366f6083SPeter Grehan case EXIT_REASON_PAUSE: 1615b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_PAUSE, 1); 1616366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_PAUSE; 1617366f6083SPeter Grehan break; 1618366f6083SPeter Grehan case EXIT_REASON_INTR_WINDOW: 1619b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INTR_WINDOW, 1); 1620366f6083SPeter Grehan vmx_clear_int_window_exiting(vmx, vcpu); 1621b5aaf7b2SNeel Natu return (1); 1622366f6083SPeter Grehan case EXIT_REASON_EXT_INTR: 1623366f6083SPeter Grehan /* 1624366f6083SPeter Grehan * External interrupts serve only to cause VM exits and allow 1625366f6083SPeter Grehan * the host interrupt handler to run. 1626366f6083SPeter Grehan * 1627366f6083SPeter Grehan * If this external interrupt triggers a virtual interrupt 1628366f6083SPeter Grehan * to a VM, then that state will be recorded by the 1629366f6083SPeter Grehan * host interrupt handler in the VM's softc. We will inject 1630366f6083SPeter Grehan * this virtual interrupt during the subsequent VM enter. 1631366f6083SPeter Grehan */ 1632f7d47425SNeel Natu intr_info = vmcs_read(VMCS_EXIT_INTR_INFO); 1633160471d2SNeel Natu KASSERT((intr_info & VMCS_INTR_VALID) != 0 && 1634160471d2SNeel Natu (intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_HWINTR, 1635f7d47425SNeel Natu ("VM exit interruption info invalid: %#x", intr_info)); 1636f7d47425SNeel Natu vmx_trigger_hostintr(intr_info & 0xff); 1637366f6083SPeter Grehan 1638366f6083SPeter Grehan /* 1639366f6083SPeter Grehan * This is special. We want to treat this as an 'handled' 1640366f6083SPeter Grehan * VM-exit but not increment the instruction pointer. 1641366f6083SPeter Grehan */ 1642366f6083SPeter Grehan vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1); 1643366f6083SPeter Grehan return (1); 1644366f6083SPeter Grehan case EXIT_REASON_NMI_WINDOW: 1645366f6083SPeter Grehan /* Exit to allow the pending virtual NMI to be injected */ 164648b2d828SNeel Natu if (vm_nmi_pending(vmx->vm, vcpu)) 164748b2d828SNeel Natu vmx_inject_nmi(vmx, vcpu); 1648366f6083SPeter Grehan vmx_clear_nmi_window_exiting(vmx, vcpu); 164948b2d828SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NMI_WINDOW, 1); 1650366f6083SPeter Grehan return (1); 1651366f6083SPeter Grehan case EXIT_REASON_INOUT: 1652b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INOUT, 1); 1653366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_INOUT; 1654366f6083SPeter Grehan vmexit->u.inout.bytes = (qual & 0x7) + 1; 1655366f6083SPeter Grehan vmexit->u.inout.in = (qual & 0x8) ? 1 : 0; 1656366f6083SPeter Grehan vmexit->u.inout.string = (qual & 0x10) ? 1 : 0; 1657366f6083SPeter Grehan vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0; 1658366f6083SPeter Grehan vmexit->u.inout.port = (uint16_t)(qual >> 16); 1659366f6083SPeter Grehan vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax); 1660366f6083SPeter Grehan break; 1661366f6083SPeter Grehan case EXIT_REASON_CPUID: 1662b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CPUID, 1); 1663a2da7af6SNeel Natu handled = vmx_handle_cpuid(vmx->vm, vcpu, vmxctx); 1664366f6083SPeter Grehan break; 1665e5a1d950SNeel Natu case EXIT_REASON_EXCEPTION: 1666c308b23bSNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXCEPTION, 1); 1667e5a1d950SNeel Natu intr_info = vmcs_read(VMCS_EXIT_INTR_INFO); 1668e5a1d950SNeel Natu KASSERT((intr_info & VMCS_INTR_VALID) != 0, 1669e5a1d950SNeel Natu ("VM exit interruption info invalid: %#x", intr_info)); 1670c308b23bSNeel Natu 1671e5a1d950SNeel Natu /* 1672e5a1d950SNeel Natu * If Virtual NMIs control is 1 and the VM-exit is due to a 1673e5a1d950SNeel Natu * fault encountered during the execution of IRET then we must 1674e5a1d950SNeel Natu * restore the state of "virtual-NMI blocking" before resuming 1675e5a1d950SNeel Natu * the guest. 1676e5a1d950SNeel Natu * 1677e5a1d950SNeel Natu * See "Resuming Guest Software after Handling an Exception". 1678e5a1d950SNeel Natu */ 1679e5a1d950SNeel Natu if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 && 1680e5a1d950SNeel Natu (intr_info & 0xff) != IDT_DF && 1681e5a1d950SNeel Natu (intr_info & EXIT_QUAL_NMIUDTI) != 0) 1682e5a1d950SNeel Natu vmx_restore_nmi_blocking(vmx, vcpu); 1683c308b23bSNeel Natu 1684c308b23bSNeel Natu /* 1685c308b23bSNeel Natu * If the NMI-exiting VM execution control is set to '1' 1686c308b23bSNeel Natu * then an NMI in non-root operation causes a VM-exit. 1687c308b23bSNeel Natu * NMI blocking is in effect for this logical processor so 1688c308b23bSNeel Natu * it is sufficient to simply vector to the NMI handler via 1689c308b23bSNeel Natu * a software interrupt. 1690c308b23bSNeel Natu */ 1691c308b23bSNeel Natu if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI) { 1692c308b23bSNeel Natu KASSERT((intr_info & 0xff) == IDT_NMI, ("VM exit due " 1693c308b23bSNeel Natu "to NMI has invalid vector: %#x", intr_info)); 1694c308b23bSNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Vectoring to NMI handler"); 1695c308b23bSNeel Natu __asm __volatile("int $2"); 1696c308b23bSNeel Natu return (1); 1697c308b23bSNeel Natu } 1698e5a1d950SNeel Natu break; 1699cd942e0fSPeter Grehan case EXIT_REASON_EPT_FAULT: 1700b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EPT_FAULT, 1); 1701318224bbSNeel Natu /* 1702318224bbSNeel Natu * If 'gpa' lies within the address space allocated to 1703318224bbSNeel Natu * memory then this must be a nested page fault otherwise 1704318224bbSNeel Natu * this must be an instruction that accesses MMIO space. 1705318224bbSNeel Natu */ 1706a2da7af6SNeel Natu gpa = vmcs_gpa(); 170788c4b8d1SNeel Natu if (vm_mem_allocated(vmx->vm, gpa) || apic_access_fault(gpa)) { 1708cd942e0fSPeter Grehan vmexit->exitcode = VM_EXITCODE_PAGING; 170913ec9371SPeter Grehan vmexit->u.paging.gpa = gpa; 1710318224bbSNeel Natu vmexit->u.paging.fault_type = ept_fault_type(qual); 1711318224bbSNeel Natu } else if (ept_emulation_fault(qual)) { 1712318224bbSNeel Natu vmexit->exitcode = VM_EXITCODE_INST_EMUL; 1713318224bbSNeel Natu vmexit->u.inst_emul.gpa = gpa; 1714318224bbSNeel Natu vmexit->u.inst_emul.gla = vmcs_gla(); 1715318224bbSNeel Natu vmexit->u.inst_emul.cr3 = vmcs_guest_cr3(); 1716a2da7af6SNeel Natu } 1717e5a1d950SNeel Natu /* 1718e5a1d950SNeel Natu * If Virtual NMIs control is 1 and the VM-exit is due to an 1719e5a1d950SNeel Natu * EPT fault during the execution of IRET then we must restore 1720e5a1d950SNeel Natu * the state of "virtual-NMI blocking" before resuming. 1721e5a1d950SNeel Natu * 1722e5a1d950SNeel Natu * See description of "NMI unblocking due to IRET" in 1723e5a1d950SNeel Natu * "Exit Qualification for EPT Violations". 1724e5a1d950SNeel Natu */ 1725e5a1d950SNeel Natu if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 && 1726e5a1d950SNeel Natu (qual & EXIT_QUAL_NMIUDTI) != 0) 1727e5a1d950SNeel Natu vmx_restore_nmi_blocking(vmx, vcpu); 1728cd942e0fSPeter Grehan break; 1729*30b94db8SNeel Natu case EXIT_REASON_VIRTUALIZED_EOI: 1730*30b94db8SNeel Natu vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI; 1731*30b94db8SNeel Natu vmexit->u.ioapic_eoi.vector = qual & 0xFF; 1732*30b94db8SNeel Natu vmexit->inst_length = 0; /* trap-like */ 1733*30b94db8SNeel Natu break; 173488c4b8d1SNeel Natu case EXIT_REASON_APIC_ACCESS: 173588c4b8d1SNeel Natu handled = vmx_handle_apic_access(vmx, vcpu, vmexit); 173688c4b8d1SNeel Natu break; 173788c4b8d1SNeel Natu case EXIT_REASON_APIC_WRITE: 173888c4b8d1SNeel Natu /* 173988c4b8d1SNeel Natu * APIC-write VM exit is trap-like so the %rip is already 174088c4b8d1SNeel Natu * pointing to the next instruction. 174188c4b8d1SNeel Natu */ 174288c4b8d1SNeel Natu vmexit->inst_length = 0; 174388c4b8d1SNeel Natu vlapic = vm_lapic(vmx->vm, vcpu); 174488c4b8d1SNeel Natu handled = vmx_handle_apic_write(vlapic, qual); 174588c4b8d1SNeel Natu break; 1746366f6083SPeter Grehan default: 1747b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_UNKNOWN, 1); 1748366f6083SPeter Grehan break; 1749366f6083SPeter Grehan } 1750366f6083SPeter Grehan 1751366f6083SPeter Grehan if (handled) { 1752366f6083SPeter Grehan /* 1753366f6083SPeter Grehan * It is possible that control is returned to userland 1754366f6083SPeter Grehan * even though we were able to handle the VM exit in the 1755eeefa4e4SNeel Natu * kernel. 1756366f6083SPeter Grehan * 1757366f6083SPeter Grehan * In such a case we want to make sure that the userland 1758366f6083SPeter Grehan * restarts guest execution at the instruction *after* 1759366f6083SPeter Grehan * the one we just processed. Therefore we update the 1760366f6083SPeter Grehan * guest rip in the VMCS and in 'vmexit'. 1761366f6083SPeter Grehan */ 1762366f6083SPeter Grehan vmexit->rip += vmexit->inst_length; 1763366f6083SPeter Grehan vmexit->inst_length = 0; 17643de83862SNeel Natu vmcs_write(VMCS_GUEST_RIP, vmexit->rip); 1765366f6083SPeter Grehan } else { 1766366f6083SPeter Grehan if (vmexit->exitcode == VM_EXITCODE_BOGUS) { 1767366f6083SPeter Grehan /* 1768366f6083SPeter Grehan * If this VM exit was not claimed by anybody then 1769366f6083SPeter Grehan * treat it as a generic VMX exit. 1770366f6083SPeter Grehan */ 1771366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_VMX; 17720492757cSNeel Natu vmexit->u.vmx.status = VM_SUCCESS; 1773c308b23bSNeel Natu vmexit->u.vmx.inst_type = 0; 1774c308b23bSNeel Natu vmexit->u.vmx.inst_error = 0; 1775366f6083SPeter Grehan } else { 1776366f6083SPeter Grehan /* 1777366f6083SPeter Grehan * The exitcode and collateral have been populated. 1778366f6083SPeter Grehan * The VM exit will be processed further in userland. 1779366f6083SPeter Grehan */ 1780366f6083SPeter Grehan } 1781366f6083SPeter Grehan } 1782366f6083SPeter Grehan return (handled); 1783366f6083SPeter Grehan } 1784366f6083SPeter Grehan 17850492757cSNeel Natu static __inline int 17860492757cSNeel Natu vmx_exit_astpending(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) 1787366f6083SPeter Grehan { 17880492757cSNeel Natu 17890492757cSNeel Natu vmexit->rip = vmcs_guest_rip(); 17900492757cSNeel Natu vmexit->inst_length = 0; 17910492757cSNeel Natu vmexit->exitcode = VM_EXITCODE_BOGUS; 17920492757cSNeel Natu vmx_astpending_trace(vmx, vcpu, vmexit->rip); 17930492757cSNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_ASTPENDING, 1); 17940492757cSNeel Natu 17950492757cSNeel Natu return (HANDLED); 17960492757cSNeel Natu } 17970492757cSNeel Natu 17980492757cSNeel Natu static __inline int 17995b8a8cd1SNeel Natu vmx_exit_rendezvous(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) 18005b8a8cd1SNeel Natu { 18015b8a8cd1SNeel Natu 18025b8a8cd1SNeel Natu vmexit->rip = vmcs_guest_rip(); 18035b8a8cd1SNeel Natu vmexit->inst_length = 0; 18045b8a8cd1SNeel Natu vmexit->exitcode = VM_EXITCODE_RENDEZVOUS; 18055b8a8cd1SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RENDEZVOUS, 1); 18065b8a8cd1SNeel Natu 18075b8a8cd1SNeel Natu return (UNHANDLED); 18085b8a8cd1SNeel Natu } 18095b8a8cd1SNeel Natu 18105b8a8cd1SNeel Natu static __inline int 18110492757cSNeel Natu vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit) 18120492757cSNeel Natu { 18130492757cSNeel Natu 18140492757cSNeel Natu KASSERT(vmxctx->inst_fail_status != VM_SUCCESS, 18150492757cSNeel Natu ("vmx_exit_inst_error: invalid inst_fail_status %d", 18160492757cSNeel Natu vmxctx->inst_fail_status)); 18170492757cSNeel Natu 18180492757cSNeel Natu vmexit->inst_length = 0; 18190492757cSNeel Natu vmexit->exitcode = VM_EXITCODE_VMX; 18200492757cSNeel Natu vmexit->u.vmx.status = vmxctx->inst_fail_status; 18210492757cSNeel Natu vmexit->u.vmx.inst_error = vmcs_instruction_error(); 18220492757cSNeel Natu vmexit->u.vmx.exit_reason = ~0; 18230492757cSNeel Natu vmexit->u.vmx.exit_qualification = ~0; 18240492757cSNeel Natu 18250492757cSNeel Natu switch (rc) { 18260492757cSNeel Natu case VMX_VMRESUME_ERROR: 18270492757cSNeel Natu case VMX_VMLAUNCH_ERROR: 18280492757cSNeel Natu case VMX_INVEPT_ERROR: 18290492757cSNeel Natu vmexit->u.vmx.inst_type = rc; 18300492757cSNeel Natu break; 18310492757cSNeel Natu default: 18320492757cSNeel Natu panic("vm_exit_inst_error: vmx_enter_guest returned %d", rc); 18330492757cSNeel Natu } 18340492757cSNeel Natu 18350492757cSNeel Natu return (UNHANDLED); 18360492757cSNeel Natu } 18370492757cSNeel Natu 18380492757cSNeel Natu static int 18395b8a8cd1SNeel Natu vmx_run(void *arg, int vcpu, register_t startrip, pmap_t pmap, 18405b8a8cd1SNeel Natu void *rendezvous_cookie) 18410492757cSNeel Natu { 18420492757cSNeel Natu int rc, handled, launched; 1843366f6083SPeter Grehan struct vmx *vmx; 18445b8a8cd1SNeel Natu struct vm *vm; 1845366f6083SPeter Grehan struct vmxctx *vmxctx; 1846366f6083SPeter Grehan struct vmcs *vmcs; 184798ed632cSNeel Natu struct vm_exit *vmexit; 1848de5ea6b6SNeel Natu struct vlapic *vlapic; 184979c59630SNeel Natu uint64_t rip; 185079c59630SNeel Natu uint32_t exit_reason; 1851366f6083SPeter Grehan 1852366f6083SPeter Grehan vmx = arg; 18535b8a8cd1SNeel Natu vm = vmx->vm; 1854366f6083SPeter Grehan vmcs = &vmx->vmcs[vcpu]; 1855366f6083SPeter Grehan vmxctx = &vmx->ctx[vcpu]; 18565b8a8cd1SNeel Natu vlapic = vm_lapic(vm, vcpu); 18575b8a8cd1SNeel Natu vmexit = vm_exitinfo(vm, vcpu); 18580492757cSNeel Natu launched = 0; 185998ed632cSNeel Natu 1860318224bbSNeel Natu KASSERT(vmxctx->pmap == pmap, 1861318224bbSNeel Natu ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap)); 1862318224bbSNeel Natu KASSERT(vmxctx->eptp == vmx->eptp, 1863318224bbSNeel Natu ("eptp %p different than ctx eptp %#lx", eptp, vmxctx->eptp)); 1864318224bbSNeel Natu 1865366f6083SPeter Grehan VMPTRLD(vmcs); 1866366f6083SPeter Grehan 1867366f6083SPeter Grehan /* 1868366f6083SPeter Grehan * XXX 1869366f6083SPeter Grehan * We do this every time because we may setup the virtual machine 1870366f6083SPeter Grehan * from a different process than the one that actually runs it. 1871366f6083SPeter Grehan * 1872366f6083SPeter Grehan * If the life of a virtual machine was spent entirely in the context 1873c847a506SNeel Natu * of a single process we could do this once in vmx_vminit(). 1874366f6083SPeter Grehan */ 18753de83862SNeel Natu vmcs_write(VMCS_HOST_CR3, rcr3()); 1876366f6083SPeter Grehan 18770492757cSNeel Natu vmcs_write(VMCS_GUEST_RIP, startrip); 18780492757cSNeel Natu vmx_set_pcpu_defaults(vmx, vcpu); 1879366f6083SPeter Grehan do { 18800492757cSNeel Natu /* 18810492757cSNeel Natu * Interrupts are disabled from this point on until the 18820492757cSNeel Natu * guest starts executing. This is done for the following 18830492757cSNeel Natu * reasons: 18840492757cSNeel Natu * 18850492757cSNeel Natu * If an AST is asserted on this thread after the check below, 18860492757cSNeel Natu * then the IPI_AST notification will not be lost, because it 18870492757cSNeel Natu * will cause a VM exit due to external interrupt as soon as 18880492757cSNeel Natu * the guest state is loaded. 18890492757cSNeel Natu * 18900492757cSNeel Natu * A posted interrupt after 'vmx_inject_interrupts()' will 18910492757cSNeel Natu * not be "lost" because it will be held pending in the host 18920492757cSNeel Natu * APIC because interrupts are disabled. The pending interrupt 18930492757cSNeel Natu * will be recognized as soon as the guest state is loaded. 18940492757cSNeel Natu * 18950492757cSNeel Natu * The same reasoning applies to the IPI generated by 18960492757cSNeel Natu * pmap_invalidate_ept(). 18970492757cSNeel Natu */ 18980492757cSNeel Natu disable_intr(); 18990492757cSNeel Natu if (curthread->td_flags & (TDF_ASTPENDING | TDF_NEEDRESCHED)) { 19000492757cSNeel Natu enable_intr(); 19010492757cSNeel Natu handled = vmx_exit_astpending(vmx, vcpu, vmexit); 19020492757cSNeel Natu break; 19030492757cSNeel Natu } 19040492757cSNeel Natu 19055b8a8cd1SNeel Natu if (vcpu_rendezvous_pending(rendezvous_cookie)) { 19065b8a8cd1SNeel Natu enable_intr(); 19075b8a8cd1SNeel Natu handled = vmx_exit_rendezvous(vmx, vcpu, vmexit); 19085b8a8cd1SNeel Natu break; 19095b8a8cd1SNeel Natu } 19105b8a8cd1SNeel Natu 1911de5ea6b6SNeel Natu vmx_inject_interrupts(vmx, vcpu, vlapic); 1912366f6083SPeter Grehan vmx_run_trace(vmx, vcpu); 19130492757cSNeel Natu rc = vmx_enter_guest(vmxctx, launched); 191479c59630SNeel Natu 1915366f6083SPeter Grehan enable_intr(); 191679c59630SNeel Natu 191779c59630SNeel Natu /* Collect some information for VM exit processing */ 191879c59630SNeel Natu vmexit->rip = rip = vmcs_guest_rip(); 191979c59630SNeel Natu vmexit->inst_length = vmexit_instruction_length(); 192079c59630SNeel Natu vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason(); 192179c59630SNeel Natu vmexit->u.vmx.exit_qualification = vmcs_exit_qualification(); 192279c59630SNeel Natu 19230492757cSNeel Natu if (rc == VMX_GUEST_VMEXIT) { 19240492757cSNeel Natu launched = 1; 19250492757cSNeel Natu handled = vmx_exit_process(vmx, vcpu, vmexit); 19260492757cSNeel Natu } else { 19270492757cSNeel Natu handled = vmx_exit_inst_error(vmxctx, rc, vmexit); 1928eeefa4e4SNeel Natu } 1929366f6083SPeter Grehan 193079c59630SNeel Natu vmx_exit_trace(vmx, vcpu, rip, exit_reason, handled); 1931eeefa4e4SNeel Natu } while (handled); 1932366f6083SPeter Grehan 1933366f6083SPeter Grehan /* 1934366f6083SPeter Grehan * If a VM exit has been handled then the exitcode must be BOGUS 1935366f6083SPeter Grehan * If a VM exit is not handled then the exitcode must not be BOGUS 1936366f6083SPeter Grehan */ 1937366f6083SPeter Grehan if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) || 1938366f6083SPeter Grehan (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) { 1939366f6083SPeter Grehan panic("Mismatch between handled (%d) and exitcode (%d)", 1940366f6083SPeter Grehan handled, vmexit->exitcode); 1941366f6083SPeter Grehan } 1942366f6083SPeter Grehan 1943b5aaf7b2SNeel Natu if (!handled) 19445b8a8cd1SNeel Natu vmm_stat_incr(vm, vcpu, VMEXIT_USERSPACE, 1); 1945b5aaf7b2SNeel Natu 19465b8a8cd1SNeel Natu VCPU_CTR1(vm, vcpu, "returning from vmx_run: exitcode %d", 19470492757cSNeel Natu vmexit->exitcode); 1948366f6083SPeter Grehan 1949366f6083SPeter Grehan VMCLEAR(vmcs); 1950366f6083SPeter Grehan return (0); 1951366f6083SPeter Grehan } 1952366f6083SPeter Grehan 1953366f6083SPeter Grehan static void 1954366f6083SPeter Grehan vmx_vmcleanup(void *arg) 1955366f6083SPeter Grehan { 195645e51299SNeel Natu int i, error; 1957366f6083SPeter Grehan struct vmx *vmx = arg; 1958366f6083SPeter Grehan 195988c4b8d1SNeel Natu if (virtual_interrupt_delivery) 196088c4b8d1SNeel Natu vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE); 196188c4b8d1SNeel Natu 196245e51299SNeel Natu for (i = 0; i < VM_MAXCPU; i++) 196345e51299SNeel Natu vpid_free(vmx->state[i].vpid); 196445e51299SNeel Natu 1965366f6083SPeter Grehan /* 1966366f6083SPeter Grehan * XXXSMP we also need to clear the VMCS active on the other vcpus. 1967366f6083SPeter Grehan */ 1968366f6083SPeter Grehan error = vmclear(&vmx->vmcs[0]); 1969366f6083SPeter Grehan if (error != 0) 1970366f6083SPeter Grehan panic("vmx_vmcleanup: vmclear error %d on vcpu 0", error); 1971366f6083SPeter Grehan 1972366f6083SPeter Grehan free(vmx, M_VMX); 1973366f6083SPeter Grehan 1974366f6083SPeter Grehan return; 1975366f6083SPeter Grehan } 1976366f6083SPeter Grehan 1977366f6083SPeter Grehan static register_t * 1978366f6083SPeter Grehan vmxctx_regptr(struct vmxctx *vmxctx, int reg) 1979366f6083SPeter Grehan { 1980366f6083SPeter Grehan 1981366f6083SPeter Grehan switch (reg) { 1982366f6083SPeter Grehan case VM_REG_GUEST_RAX: 1983366f6083SPeter Grehan return (&vmxctx->guest_rax); 1984366f6083SPeter Grehan case VM_REG_GUEST_RBX: 1985366f6083SPeter Grehan return (&vmxctx->guest_rbx); 1986366f6083SPeter Grehan case VM_REG_GUEST_RCX: 1987366f6083SPeter Grehan return (&vmxctx->guest_rcx); 1988366f6083SPeter Grehan case VM_REG_GUEST_RDX: 1989366f6083SPeter Grehan return (&vmxctx->guest_rdx); 1990366f6083SPeter Grehan case VM_REG_GUEST_RSI: 1991366f6083SPeter Grehan return (&vmxctx->guest_rsi); 1992366f6083SPeter Grehan case VM_REG_GUEST_RDI: 1993366f6083SPeter Grehan return (&vmxctx->guest_rdi); 1994366f6083SPeter Grehan case VM_REG_GUEST_RBP: 1995366f6083SPeter Grehan return (&vmxctx->guest_rbp); 1996366f6083SPeter Grehan case VM_REG_GUEST_R8: 1997366f6083SPeter Grehan return (&vmxctx->guest_r8); 1998366f6083SPeter Grehan case VM_REG_GUEST_R9: 1999366f6083SPeter Grehan return (&vmxctx->guest_r9); 2000366f6083SPeter Grehan case VM_REG_GUEST_R10: 2001366f6083SPeter Grehan return (&vmxctx->guest_r10); 2002366f6083SPeter Grehan case VM_REG_GUEST_R11: 2003366f6083SPeter Grehan return (&vmxctx->guest_r11); 2004366f6083SPeter Grehan case VM_REG_GUEST_R12: 2005366f6083SPeter Grehan return (&vmxctx->guest_r12); 2006366f6083SPeter Grehan case VM_REG_GUEST_R13: 2007366f6083SPeter Grehan return (&vmxctx->guest_r13); 2008366f6083SPeter Grehan case VM_REG_GUEST_R14: 2009366f6083SPeter Grehan return (&vmxctx->guest_r14); 2010366f6083SPeter Grehan case VM_REG_GUEST_R15: 2011366f6083SPeter Grehan return (&vmxctx->guest_r15); 2012366f6083SPeter Grehan default: 2013366f6083SPeter Grehan break; 2014366f6083SPeter Grehan } 2015366f6083SPeter Grehan return (NULL); 2016366f6083SPeter Grehan } 2017366f6083SPeter Grehan 2018366f6083SPeter Grehan static int 2019366f6083SPeter Grehan vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval) 2020366f6083SPeter Grehan { 2021366f6083SPeter Grehan register_t *regp; 2022366f6083SPeter Grehan 2023366f6083SPeter Grehan if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) { 2024366f6083SPeter Grehan *retval = *regp; 2025366f6083SPeter Grehan return (0); 2026366f6083SPeter Grehan } else 2027366f6083SPeter Grehan return (EINVAL); 2028366f6083SPeter Grehan } 2029366f6083SPeter Grehan 2030366f6083SPeter Grehan static int 2031366f6083SPeter Grehan vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val) 2032366f6083SPeter Grehan { 2033366f6083SPeter Grehan register_t *regp; 2034366f6083SPeter Grehan 2035366f6083SPeter Grehan if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) { 2036366f6083SPeter Grehan *regp = val; 2037366f6083SPeter Grehan return (0); 2038366f6083SPeter Grehan } else 2039366f6083SPeter Grehan return (EINVAL); 2040366f6083SPeter Grehan } 2041366f6083SPeter Grehan 2042366f6083SPeter Grehan static int 2043aaaa0656SPeter Grehan vmx_shadow_reg(int reg) 2044aaaa0656SPeter Grehan { 2045aaaa0656SPeter Grehan int shreg; 2046aaaa0656SPeter Grehan 2047aaaa0656SPeter Grehan shreg = -1; 2048aaaa0656SPeter Grehan 2049aaaa0656SPeter Grehan switch (reg) { 2050aaaa0656SPeter Grehan case VM_REG_GUEST_CR0: 2051aaaa0656SPeter Grehan shreg = VMCS_CR0_SHADOW; 2052aaaa0656SPeter Grehan break; 2053aaaa0656SPeter Grehan case VM_REG_GUEST_CR4: 2054aaaa0656SPeter Grehan shreg = VMCS_CR4_SHADOW; 2055aaaa0656SPeter Grehan break; 2056aaaa0656SPeter Grehan default: 2057aaaa0656SPeter Grehan break; 2058aaaa0656SPeter Grehan } 2059aaaa0656SPeter Grehan 2060aaaa0656SPeter Grehan return (shreg); 2061aaaa0656SPeter Grehan } 2062aaaa0656SPeter Grehan 2063aaaa0656SPeter Grehan static int 2064366f6083SPeter Grehan vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval) 2065366f6083SPeter Grehan { 2066d3c11f40SPeter Grehan int running, hostcpu; 2067366f6083SPeter Grehan struct vmx *vmx = arg; 2068366f6083SPeter Grehan 2069d3c11f40SPeter Grehan running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); 2070d3c11f40SPeter Grehan if (running && hostcpu != curcpu) 2071d3c11f40SPeter Grehan panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu); 2072d3c11f40SPeter Grehan 2073366f6083SPeter Grehan if (vmxctx_getreg(&vmx->ctx[vcpu], reg, retval) == 0) 2074366f6083SPeter Grehan return (0); 2075366f6083SPeter Grehan 2076d3c11f40SPeter Grehan return (vmcs_getreg(&vmx->vmcs[vcpu], running, reg, retval)); 2077366f6083SPeter Grehan } 2078366f6083SPeter Grehan 2079366f6083SPeter Grehan static int 2080366f6083SPeter Grehan vmx_setreg(void *arg, int vcpu, int reg, uint64_t val) 2081366f6083SPeter Grehan { 2082aaaa0656SPeter Grehan int error, hostcpu, running, shadow; 2083366f6083SPeter Grehan uint64_t ctls; 2084366f6083SPeter Grehan struct vmx *vmx = arg; 2085366f6083SPeter Grehan 2086d3c11f40SPeter Grehan running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); 2087d3c11f40SPeter Grehan if (running && hostcpu != curcpu) 2088d3c11f40SPeter Grehan panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu); 2089d3c11f40SPeter Grehan 2090366f6083SPeter Grehan if (vmxctx_setreg(&vmx->ctx[vcpu], reg, val) == 0) 2091366f6083SPeter Grehan return (0); 2092366f6083SPeter Grehan 2093d3c11f40SPeter Grehan error = vmcs_setreg(&vmx->vmcs[vcpu], running, reg, val); 2094366f6083SPeter Grehan 2095366f6083SPeter Grehan if (error == 0) { 2096366f6083SPeter Grehan /* 2097366f6083SPeter Grehan * If the "load EFER" VM-entry control is 1 then the 2098366f6083SPeter Grehan * value of EFER.LMA must be identical to "IA-32e mode guest" 2099366f6083SPeter Grehan * bit in the VM-entry control. 2100366f6083SPeter Grehan */ 2101366f6083SPeter Grehan if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 && 2102366f6083SPeter Grehan (reg == VM_REG_GUEST_EFER)) { 2103d3c11f40SPeter Grehan vmcs_getreg(&vmx->vmcs[vcpu], running, 2104366f6083SPeter Grehan VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls); 2105366f6083SPeter Grehan if (val & EFER_LMA) 2106366f6083SPeter Grehan ctls |= VM_ENTRY_GUEST_LMA; 2107366f6083SPeter Grehan else 2108366f6083SPeter Grehan ctls &= ~VM_ENTRY_GUEST_LMA; 2109d3c11f40SPeter Grehan vmcs_setreg(&vmx->vmcs[vcpu], running, 2110366f6083SPeter Grehan VMCS_IDENT(VMCS_ENTRY_CTLS), ctls); 2111366f6083SPeter Grehan } 2112aaaa0656SPeter Grehan 2113aaaa0656SPeter Grehan shadow = vmx_shadow_reg(reg); 2114aaaa0656SPeter Grehan if (shadow > 0) { 2115aaaa0656SPeter Grehan /* 2116aaaa0656SPeter Grehan * Store the unmodified value in the shadow 2117aaaa0656SPeter Grehan */ 2118aaaa0656SPeter Grehan error = vmcs_setreg(&vmx->vmcs[vcpu], running, 2119aaaa0656SPeter Grehan VMCS_IDENT(shadow), val); 2120aaaa0656SPeter Grehan } 2121366f6083SPeter Grehan } 2122366f6083SPeter Grehan 2123366f6083SPeter Grehan return (error); 2124366f6083SPeter Grehan } 2125366f6083SPeter Grehan 2126366f6083SPeter Grehan static int 2127366f6083SPeter Grehan vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc) 2128366f6083SPeter Grehan { 2129366f6083SPeter Grehan struct vmx *vmx = arg; 2130366f6083SPeter Grehan 2131366f6083SPeter Grehan return (vmcs_getdesc(&vmx->vmcs[vcpu], reg, desc)); 2132366f6083SPeter Grehan } 2133366f6083SPeter Grehan 2134366f6083SPeter Grehan static int 2135366f6083SPeter Grehan vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc) 2136366f6083SPeter Grehan { 2137366f6083SPeter Grehan struct vmx *vmx = arg; 2138366f6083SPeter Grehan 2139366f6083SPeter Grehan return (vmcs_setdesc(&vmx->vmcs[vcpu], reg, desc)); 2140366f6083SPeter Grehan } 2141366f6083SPeter Grehan 2142366f6083SPeter Grehan static int 2143366f6083SPeter Grehan vmx_inject(void *arg, int vcpu, int type, int vector, uint32_t code, 2144366f6083SPeter Grehan int code_valid) 2145366f6083SPeter Grehan { 2146366f6083SPeter Grehan int error; 2147eeefa4e4SNeel Natu uint64_t info; 2148366f6083SPeter Grehan struct vmx *vmx = arg; 2149366f6083SPeter Grehan struct vmcs *vmcs = &vmx->vmcs[vcpu]; 2150366f6083SPeter Grehan 2151366f6083SPeter Grehan static uint32_t type_map[VM_EVENT_MAX] = { 2152366f6083SPeter Grehan 0x1, /* VM_EVENT_NONE */ 2153366f6083SPeter Grehan 0x0, /* VM_HW_INTR */ 2154366f6083SPeter Grehan 0x2, /* VM_NMI */ 2155366f6083SPeter Grehan 0x3, /* VM_HW_EXCEPTION */ 2156366f6083SPeter Grehan 0x4, /* VM_SW_INTR */ 2157366f6083SPeter Grehan 0x5, /* VM_PRIV_SW_EXCEPTION */ 2158366f6083SPeter Grehan 0x6, /* VM_SW_EXCEPTION */ 2159366f6083SPeter Grehan }; 2160366f6083SPeter Grehan 2161eeefa4e4SNeel Natu /* 2162eeefa4e4SNeel Natu * If there is already an exception pending to be delivered to the 2163eeefa4e4SNeel Natu * vcpu then just return. 2164eeefa4e4SNeel Natu */ 2165d3c11f40SPeter Grehan error = vmcs_getreg(vmcs, 0, VMCS_IDENT(VMCS_ENTRY_INTR_INFO), &info); 2166eeefa4e4SNeel Natu if (error) 2167eeefa4e4SNeel Natu return (error); 2168eeefa4e4SNeel Natu 2169160471d2SNeel Natu if (info & VMCS_INTR_VALID) 2170eeefa4e4SNeel Natu return (EAGAIN); 2171eeefa4e4SNeel Natu 2172366f6083SPeter Grehan info = vector | (type_map[type] << 8) | (code_valid ? 1 << 11 : 0); 2173160471d2SNeel Natu info |= VMCS_INTR_VALID; 2174d3c11f40SPeter Grehan error = vmcs_setreg(vmcs, 0, VMCS_IDENT(VMCS_ENTRY_INTR_INFO), info); 2175366f6083SPeter Grehan if (error != 0) 2176366f6083SPeter Grehan return (error); 2177366f6083SPeter Grehan 2178366f6083SPeter Grehan if (code_valid) { 2179d3c11f40SPeter Grehan error = vmcs_setreg(vmcs, 0, 2180366f6083SPeter Grehan VMCS_IDENT(VMCS_ENTRY_EXCEPTION_ERROR), 2181366f6083SPeter Grehan code); 2182366f6083SPeter Grehan } 2183366f6083SPeter Grehan return (error); 2184366f6083SPeter Grehan } 2185366f6083SPeter Grehan 2186366f6083SPeter Grehan static int 2187366f6083SPeter Grehan vmx_getcap(void *arg, int vcpu, int type, int *retval) 2188366f6083SPeter Grehan { 2189366f6083SPeter Grehan struct vmx *vmx = arg; 2190366f6083SPeter Grehan int vcap; 2191366f6083SPeter Grehan int ret; 2192366f6083SPeter Grehan 2193366f6083SPeter Grehan ret = ENOENT; 2194366f6083SPeter Grehan 2195366f6083SPeter Grehan vcap = vmx->cap[vcpu].set; 2196366f6083SPeter Grehan 2197366f6083SPeter Grehan switch (type) { 2198366f6083SPeter Grehan case VM_CAP_HALT_EXIT: 2199366f6083SPeter Grehan if (cap_halt_exit) 2200366f6083SPeter Grehan ret = 0; 2201366f6083SPeter Grehan break; 2202366f6083SPeter Grehan case VM_CAP_PAUSE_EXIT: 2203366f6083SPeter Grehan if (cap_pause_exit) 2204366f6083SPeter Grehan ret = 0; 2205366f6083SPeter Grehan break; 2206366f6083SPeter Grehan case VM_CAP_MTRAP_EXIT: 2207366f6083SPeter Grehan if (cap_monitor_trap) 2208366f6083SPeter Grehan ret = 0; 2209366f6083SPeter Grehan break; 2210366f6083SPeter Grehan case VM_CAP_UNRESTRICTED_GUEST: 2211366f6083SPeter Grehan if (cap_unrestricted_guest) 2212366f6083SPeter Grehan ret = 0; 2213366f6083SPeter Grehan break; 221449cc03daSNeel Natu case VM_CAP_ENABLE_INVPCID: 221549cc03daSNeel Natu if (cap_invpcid) 221649cc03daSNeel Natu ret = 0; 221749cc03daSNeel Natu break; 2218366f6083SPeter Grehan default: 2219366f6083SPeter Grehan break; 2220366f6083SPeter Grehan } 2221366f6083SPeter Grehan 2222366f6083SPeter Grehan if (ret == 0) 2223366f6083SPeter Grehan *retval = (vcap & (1 << type)) ? 1 : 0; 2224366f6083SPeter Grehan 2225366f6083SPeter Grehan return (ret); 2226366f6083SPeter Grehan } 2227366f6083SPeter Grehan 2228366f6083SPeter Grehan static int 2229366f6083SPeter Grehan vmx_setcap(void *arg, int vcpu, int type, int val) 2230366f6083SPeter Grehan { 2231366f6083SPeter Grehan struct vmx *vmx = arg; 2232366f6083SPeter Grehan struct vmcs *vmcs = &vmx->vmcs[vcpu]; 2233366f6083SPeter Grehan uint32_t baseval; 2234366f6083SPeter Grehan uint32_t *pptr; 2235366f6083SPeter Grehan int error; 2236366f6083SPeter Grehan int flag; 2237366f6083SPeter Grehan int reg; 2238366f6083SPeter Grehan int retval; 2239366f6083SPeter Grehan 2240366f6083SPeter Grehan retval = ENOENT; 2241366f6083SPeter Grehan pptr = NULL; 2242366f6083SPeter Grehan 2243366f6083SPeter Grehan switch (type) { 2244366f6083SPeter Grehan case VM_CAP_HALT_EXIT: 2245366f6083SPeter Grehan if (cap_halt_exit) { 2246366f6083SPeter Grehan retval = 0; 2247366f6083SPeter Grehan pptr = &vmx->cap[vcpu].proc_ctls; 2248366f6083SPeter Grehan baseval = *pptr; 2249366f6083SPeter Grehan flag = PROCBASED_HLT_EXITING; 2250366f6083SPeter Grehan reg = VMCS_PRI_PROC_BASED_CTLS; 2251366f6083SPeter Grehan } 2252366f6083SPeter Grehan break; 2253366f6083SPeter Grehan case VM_CAP_MTRAP_EXIT: 2254366f6083SPeter Grehan if (cap_monitor_trap) { 2255366f6083SPeter Grehan retval = 0; 2256366f6083SPeter Grehan pptr = &vmx->cap[vcpu].proc_ctls; 2257366f6083SPeter Grehan baseval = *pptr; 2258366f6083SPeter Grehan flag = PROCBASED_MTF; 2259366f6083SPeter Grehan reg = VMCS_PRI_PROC_BASED_CTLS; 2260366f6083SPeter Grehan } 2261366f6083SPeter Grehan break; 2262366f6083SPeter Grehan case VM_CAP_PAUSE_EXIT: 2263366f6083SPeter Grehan if (cap_pause_exit) { 2264366f6083SPeter Grehan retval = 0; 2265366f6083SPeter Grehan pptr = &vmx->cap[vcpu].proc_ctls; 2266366f6083SPeter Grehan baseval = *pptr; 2267366f6083SPeter Grehan flag = PROCBASED_PAUSE_EXITING; 2268366f6083SPeter Grehan reg = VMCS_PRI_PROC_BASED_CTLS; 2269366f6083SPeter Grehan } 2270366f6083SPeter Grehan break; 2271366f6083SPeter Grehan case VM_CAP_UNRESTRICTED_GUEST: 2272366f6083SPeter Grehan if (cap_unrestricted_guest) { 2273366f6083SPeter Grehan retval = 0; 227449cc03daSNeel Natu pptr = &vmx->cap[vcpu].proc_ctls2; 227549cc03daSNeel Natu baseval = *pptr; 2276366f6083SPeter Grehan flag = PROCBASED2_UNRESTRICTED_GUEST; 2277366f6083SPeter Grehan reg = VMCS_SEC_PROC_BASED_CTLS; 2278366f6083SPeter Grehan } 2279366f6083SPeter Grehan break; 228049cc03daSNeel Natu case VM_CAP_ENABLE_INVPCID: 228149cc03daSNeel Natu if (cap_invpcid) { 228249cc03daSNeel Natu retval = 0; 228349cc03daSNeel Natu pptr = &vmx->cap[vcpu].proc_ctls2; 228449cc03daSNeel Natu baseval = *pptr; 228549cc03daSNeel Natu flag = PROCBASED2_ENABLE_INVPCID; 228649cc03daSNeel Natu reg = VMCS_SEC_PROC_BASED_CTLS; 228749cc03daSNeel Natu } 228849cc03daSNeel Natu break; 2289366f6083SPeter Grehan default: 2290366f6083SPeter Grehan break; 2291366f6083SPeter Grehan } 2292366f6083SPeter Grehan 2293366f6083SPeter Grehan if (retval == 0) { 2294366f6083SPeter Grehan if (val) { 2295366f6083SPeter Grehan baseval |= flag; 2296366f6083SPeter Grehan } else { 2297366f6083SPeter Grehan baseval &= ~flag; 2298366f6083SPeter Grehan } 2299366f6083SPeter Grehan VMPTRLD(vmcs); 2300366f6083SPeter Grehan error = vmwrite(reg, baseval); 2301366f6083SPeter Grehan VMCLEAR(vmcs); 2302366f6083SPeter Grehan 2303366f6083SPeter Grehan if (error) { 2304366f6083SPeter Grehan retval = error; 2305366f6083SPeter Grehan } else { 2306366f6083SPeter Grehan /* 2307366f6083SPeter Grehan * Update optional stored flags, and record 2308366f6083SPeter Grehan * setting 2309366f6083SPeter Grehan */ 2310366f6083SPeter Grehan if (pptr != NULL) { 2311366f6083SPeter Grehan *pptr = baseval; 2312366f6083SPeter Grehan } 2313366f6083SPeter Grehan 2314366f6083SPeter Grehan if (val) { 2315366f6083SPeter Grehan vmx->cap[vcpu].set |= (1 << type); 2316366f6083SPeter Grehan } else { 2317366f6083SPeter Grehan vmx->cap[vcpu].set &= ~(1 << type); 2318366f6083SPeter Grehan } 2319366f6083SPeter Grehan } 2320366f6083SPeter Grehan } 2321366f6083SPeter Grehan 2322366f6083SPeter Grehan return (retval); 2323366f6083SPeter Grehan } 2324366f6083SPeter Grehan 232588c4b8d1SNeel Natu struct vlapic_vtx { 232688c4b8d1SNeel Natu struct vlapic vlapic; 2327176666c2SNeel Natu struct pir_desc *pir_desc; 2328*30b94db8SNeel Natu struct vmx *vmx; 232988c4b8d1SNeel Natu }; 233088c4b8d1SNeel Natu 233188c4b8d1SNeel Natu #define VMX_CTR_PIR(vm, vcpuid, pir_desc, notify, vector, level, msg) \ 233288c4b8d1SNeel Natu do { \ 233388c4b8d1SNeel Natu VCPU_CTR2(vm, vcpuid, msg " assert %s-triggered vector %d", \ 233488c4b8d1SNeel Natu level ? "level" : "edge", vector); \ 233588c4b8d1SNeel Natu VCPU_CTR1(vm, vcpuid, msg " pir0 0x%016lx", pir_desc->pir[0]); \ 233688c4b8d1SNeel Natu VCPU_CTR1(vm, vcpuid, msg " pir1 0x%016lx", pir_desc->pir[1]); \ 233788c4b8d1SNeel Natu VCPU_CTR1(vm, vcpuid, msg " pir2 0x%016lx", pir_desc->pir[2]); \ 233888c4b8d1SNeel Natu VCPU_CTR1(vm, vcpuid, msg " pir3 0x%016lx", pir_desc->pir[3]); \ 233988c4b8d1SNeel Natu VCPU_CTR1(vm, vcpuid, msg " notify: %s", notify ? "yes" : "no");\ 234088c4b8d1SNeel Natu } while (0) 234188c4b8d1SNeel Natu 234288c4b8d1SNeel Natu /* 234388c4b8d1SNeel Natu * vlapic->ops handlers that utilize the APICv hardware assist described in 234488c4b8d1SNeel Natu * Chapter 29 of the Intel SDM. 234588c4b8d1SNeel Natu */ 234688c4b8d1SNeel Natu static int 234788c4b8d1SNeel Natu vmx_set_intr_ready(struct vlapic *vlapic, int vector, bool level) 234888c4b8d1SNeel Natu { 234988c4b8d1SNeel Natu struct vlapic_vtx *vlapic_vtx; 235088c4b8d1SNeel Natu struct pir_desc *pir_desc; 235188c4b8d1SNeel Natu uint64_t mask; 235288c4b8d1SNeel Natu int idx, notify; 235388c4b8d1SNeel Natu 235488c4b8d1SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 2355176666c2SNeel Natu pir_desc = vlapic_vtx->pir_desc; 235688c4b8d1SNeel Natu 235788c4b8d1SNeel Natu /* 235888c4b8d1SNeel Natu * Keep track of interrupt requests in the PIR descriptor. This is 235988c4b8d1SNeel Natu * because the virtual APIC page pointed to by the VMCS cannot be 236088c4b8d1SNeel Natu * modified if the vcpu is running. 236188c4b8d1SNeel Natu */ 236288c4b8d1SNeel Natu idx = vector / 64; 236388c4b8d1SNeel Natu mask = 1UL << (vector % 64); 236488c4b8d1SNeel Natu atomic_set_long(&pir_desc->pir[idx], mask); 236588c4b8d1SNeel Natu notify = atomic_cmpset_long(&pir_desc->pending, 0, 1); 236688c4b8d1SNeel Natu 236788c4b8d1SNeel Natu VMX_CTR_PIR(vlapic->vm, vlapic->vcpuid, pir_desc, notify, vector, 236888c4b8d1SNeel Natu level, "vmx_set_intr_ready"); 236988c4b8d1SNeel Natu return (notify); 237088c4b8d1SNeel Natu } 237188c4b8d1SNeel Natu 237288c4b8d1SNeel Natu static int 237388c4b8d1SNeel Natu vmx_pending_intr(struct vlapic *vlapic, int *vecptr) 237488c4b8d1SNeel Natu { 237588c4b8d1SNeel Natu struct vlapic_vtx *vlapic_vtx; 237688c4b8d1SNeel Natu struct pir_desc *pir_desc; 237788c4b8d1SNeel Natu struct LAPIC *lapic; 237888c4b8d1SNeel Natu uint64_t pending, pirval; 237988c4b8d1SNeel Natu uint32_t ppr, vpr; 238088c4b8d1SNeel Natu int i; 238188c4b8d1SNeel Natu 238288c4b8d1SNeel Natu /* 238388c4b8d1SNeel Natu * This function is only expected to be called from the 'HLT' exit 238488c4b8d1SNeel Natu * handler which does not care about the vector that is pending. 238588c4b8d1SNeel Natu */ 238688c4b8d1SNeel Natu KASSERT(vecptr == NULL, ("vmx_pending_intr: vecptr must be NULL")); 238788c4b8d1SNeel Natu 238888c4b8d1SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 2389176666c2SNeel Natu pir_desc = vlapic_vtx->pir_desc; 239088c4b8d1SNeel Natu 239188c4b8d1SNeel Natu pending = atomic_load_acq_long(&pir_desc->pending); 239288c4b8d1SNeel Natu if (!pending) 239388c4b8d1SNeel Natu return (0); /* common case */ 239488c4b8d1SNeel Natu 239588c4b8d1SNeel Natu /* 239688c4b8d1SNeel Natu * If there is an interrupt pending then it will be recognized only 239788c4b8d1SNeel Natu * if its priority is greater than the processor priority. 239888c4b8d1SNeel Natu * 239988c4b8d1SNeel Natu * Special case: if the processor priority is zero then any pending 240088c4b8d1SNeel Natu * interrupt will be recognized. 240188c4b8d1SNeel Natu */ 240288c4b8d1SNeel Natu lapic = vlapic->apic_page; 240388c4b8d1SNeel Natu ppr = lapic->ppr & 0xf0; 240488c4b8d1SNeel Natu if (ppr == 0) 240588c4b8d1SNeel Natu return (1); 240688c4b8d1SNeel Natu 240788c4b8d1SNeel Natu VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "HLT with non-zero PPR %d", 240888c4b8d1SNeel Natu lapic->ppr); 240988c4b8d1SNeel Natu 241088c4b8d1SNeel Natu for (i = 3; i >= 0; i--) { 241188c4b8d1SNeel Natu pirval = pir_desc->pir[i]; 241288c4b8d1SNeel Natu if (pirval != 0) { 241388c4b8d1SNeel Natu vpr = (i * 64 + flsl(pirval) - 1) & 0xf0; 241488c4b8d1SNeel Natu return (vpr > ppr); 241588c4b8d1SNeel Natu } 241688c4b8d1SNeel Natu } 241788c4b8d1SNeel Natu return (0); 241888c4b8d1SNeel Natu } 241988c4b8d1SNeel Natu 242088c4b8d1SNeel Natu static void 242188c4b8d1SNeel Natu vmx_intr_accepted(struct vlapic *vlapic, int vector) 242288c4b8d1SNeel Natu { 242388c4b8d1SNeel Natu 242488c4b8d1SNeel Natu panic("vmx_intr_accepted: not expected to be called"); 242588c4b8d1SNeel Natu } 242688c4b8d1SNeel Natu 2427176666c2SNeel Natu static void 2428*30b94db8SNeel Natu vmx_set_tmr(struct vlapic *vlapic, int vector, bool level) 2429*30b94db8SNeel Natu { 2430*30b94db8SNeel Natu struct vlapic_vtx *vlapic_vtx; 2431*30b94db8SNeel Natu struct vmx *vmx; 2432*30b94db8SNeel Natu struct vmcs *vmcs; 2433*30b94db8SNeel Natu uint64_t mask, val; 2434*30b94db8SNeel Natu 2435*30b94db8SNeel Natu KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector)); 2436*30b94db8SNeel Natu KASSERT(!vcpu_is_running(vlapic->vm, vlapic->vcpuid, NULL), 2437*30b94db8SNeel Natu ("vmx_set_tmr: vcpu cannot be running")); 2438*30b94db8SNeel Natu 2439*30b94db8SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 2440*30b94db8SNeel Natu vmx = vlapic_vtx->vmx; 2441*30b94db8SNeel Natu vmcs = &vmx->vmcs[vlapic->vcpuid]; 2442*30b94db8SNeel Natu mask = 1UL << (vector % 64); 2443*30b94db8SNeel Natu 2444*30b94db8SNeel Natu VMPTRLD(vmcs); 2445*30b94db8SNeel Natu val = vmcs_read(VMCS_EOI_EXIT(vector)); 2446*30b94db8SNeel Natu if (level) 2447*30b94db8SNeel Natu val |= mask; 2448*30b94db8SNeel Natu else 2449*30b94db8SNeel Natu val &= ~mask; 2450*30b94db8SNeel Natu vmcs_write(VMCS_EOI_EXIT(vector), val); 2451*30b94db8SNeel Natu VMCLEAR(vmcs); 2452*30b94db8SNeel Natu } 2453*30b94db8SNeel Natu 2454*30b94db8SNeel Natu static void 2455176666c2SNeel Natu vmx_post_intr(struct vlapic *vlapic, int hostcpu) 2456176666c2SNeel Natu { 2457176666c2SNeel Natu 2458176666c2SNeel Natu ipi_cpu(hostcpu, pirvec); 2459176666c2SNeel Natu } 2460176666c2SNeel Natu 246188c4b8d1SNeel Natu /* 246288c4b8d1SNeel Natu * Transfer the pending interrupts in the PIR descriptor to the IRR 246388c4b8d1SNeel Natu * in the virtual APIC page. 246488c4b8d1SNeel Natu */ 246588c4b8d1SNeel Natu static void 246688c4b8d1SNeel Natu vmx_inject_pir(struct vlapic *vlapic) 246788c4b8d1SNeel Natu { 246888c4b8d1SNeel Natu struct vlapic_vtx *vlapic_vtx; 246988c4b8d1SNeel Natu struct pir_desc *pir_desc; 247088c4b8d1SNeel Natu struct LAPIC *lapic; 247188c4b8d1SNeel Natu uint64_t val, pirval; 247288c4b8d1SNeel Natu int rvi, pirbase; 247388c4b8d1SNeel Natu uint16_t intr_status_old, intr_status_new; 247488c4b8d1SNeel Natu 247588c4b8d1SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 2476176666c2SNeel Natu pir_desc = vlapic_vtx->pir_desc; 247788c4b8d1SNeel Natu if (atomic_cmpset_long(&pir_desc->pending, 1, 0) == 0) { 247888c4b8d1SNeel Natu VCPU_CTR0(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: " 247988c4b8d1SNeel Natu "no posted interrupt pending"); 248088c4b8d1SNeel Natu return; 248188c4b8d1SNeel Natu } 248288c4b8d1SNeel Natu 248388c4b8d1SNeel Natu pirval = 0; 248488c4b8d1SNeel Natu lapic = vlapic->apic_page; 248588c4b8d1SNeel Natu 248688c4b8d1SNeel Natu val = atomic_readandclear_long(&pir_desc->pir[0]); 248788c4b8d1SNeel Natu if (val != 0) { 248888c4b8d1SNeel Natu lapic->irr0 |= val; 248988c4b8d1SNeel Natu lapic->irr1 |= val >> 32; 249088c4b8d1SNeel Natu pirbase = 0; 249188c4b8d1SNeel Natu pirval = val; 249288c4b8d1SNeel Natu } 249388c4b8d1SNeel Natu 249488c4b8d1SNeel Natu val = atomic_readandclear_long(&pir_desc->pir[1]); 249588c4b8d1SNeel Natu if (val != 0) { 249688c4b8d1SNeel Natu lapic->irr2 |= val; 249788c4b8d1SNeel Natu lapic->irr3 |= val >> 32; 249888c4b8d1SNeel Natu pirbase = 64; 249988c4b8d1SNeel Natu pirval = val; 250088c4b8d1SNeel Natu } 250188c4b8d1SNeel Natu 250288c4b8d1SNeel Natu val = atomic_readandclear_long(&pir_desc->pir[2]); 250388c4b8d1SNeel Natu if (val != 0) { 250488c4b8d1SNeel Natu lapic->irr4 |= val; 250588c4b8d1SNeel Natu lapic->irr5 |= val >> 32; 250688c4b8d1SNeel Natu pirbase = 128; 250788c4b8d1SNeel Natu pirval = val; 250888c4b8d1SNeel Natu } 250988c4b8d1SNeel Natu 251088c4b8d1SNeel Natu val = atomic_readandclear_long(&pir_desc->pir[3]); 251188c4b8d1SNeel Natu if (val != 0) { 251288c4b8d1SNeel Natu lapic->irr6 |= val; 251388c4b8d1SNeel Natu lapic->irr7 |= val >> 32; 251488c4b8d1SNeel Natu pirbase = 192; 251588c4b8d1SNeel Natu pirval = val; 251688c4b8d1SNeel Natu } 251788c4b8d1SNeel Natu VLAPIC_CTR_IRR(vlapic, "vmx_inject_pir"); 251888c4b8d1SNeel Natu 251988c4b8d1SNeel Natu /* 252088c4b8d1SNeel Natu * Update RVI so the processor can evaluate pending virtual 252188c4b8d1SNeel Natu * interrupts on VM-entry. 252288c4b8d1SNeel Natu */ 252388c4b8d1SNeel Natu if (pirval != 0) { 252488c4b8d1SNeel Natu rvi = pirbase + flsl(pirval) - 1; 252588c4b8d1SNeel Natu intr_status_old = vmcs_read(VMCS_GUEST_INTR_STATUS); 252688c4b8d1SNeel Natu intr_status_new = (intr_status_old & 0xFF00) | rvi; 252788c4b8d1SNeel Natu if (intr_status_new > intr_status_old) { 252888c4b8d1SNeel Natu vmcs_write(VMCS_GUEST_INTR_STATUS, intr_status_new); 252988c4b8d1SNeel Natu VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: " 253088c4b8d1SNeel Natu "guest_intr_status changed from 0x%04x to 0x%04x", 253188c4b8d1SNeel Natu intr_status_old, intr_status_new); 253288c4b8d1SNeel Natu } 253388c4b8d1SNeel Natu } 253488c4b8d1SNeel Natu } 253588c4b8d1SNeel Natu 2536de5ea6b6SNeel Natu static struct vlapic * 2537de5ea6b6SNeel Natu vmx_vlapic_init(void *arg, int vcpuid) 2538de5ea6b6SNeel Natu { 2539de5ea6b6SNeel Natu struct vmx *vmx; 2540de5ea6b6SNeel Natu struct vlapic *vlapic; 2541176666c2SNeel Natu struct vlapic_vtx *vlapic_vtx; 2542de5ea6b6SNeel Natu 2543de5ea6b6SNeel Natu vmx = arg; 2544de5ea6b6SNeel Natu 254588c4b8d1SNeel Natu vlapic = malloc(sizeof(struct vlapic_vtx), M_VLAPIC, M_WAITOK | M_ZERO); 2546de5ea6b6SNeel Natu vlapic->vm = vmx->vm; 2547de5ea6b6SNeel Natu vlapic->vcpuid = vcpuid; 2548de5ea6b6SNeel Natu vlapic->apic_page = (struct LAPIC *)&vmx->apic_page[vcpuid]; 2549de5ea6b6SNeel Natu 2550176666c2SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 2551176666c2SNeel Natu vlapic_vtx->pir_desc = &vmx->pir_desc[vcpuid]; 2552*30b94db8SNeel Natu vlapic_vtx->vmx = vmx; 2553176666c2SNeel Natu 255488c4b8d1SNeel Natu if (virtual_interrupt_delivery) { 255588c4b8d1SNeel Natu vlapic->ops.set_intr_ready = vmx_set_intr_ready; 255688c4b8d1SNeel Natu vlapic->ops.pending_intr = vmx_pending_intr; 255788c4b8d1SNeel Natu vlapic->ops.intr_accepted = vmx_intr_accepted; 2558*30b94db8SNeel Natu vlapic->ops.set_tmr = vmx_set_tmr; 255988c4b8d1SNeel Natu } 256088c4b8d1SNeel Natu 2561176666c2SNeel Natu if (posted_interrupts) 2562176666c2SNeel Natu vlapic->ops.post_intr = vmx_post_intr; 2563176666c2SNeel Natu 2564de5ea6b6SNeel Natu vlapic_init(vlapic); 2565de5ea6b6SNeel Natu 2566de5ea6b6SNeel Natu return (vlapic); 2567de5ea6b6SNeel Natu } 2568de5ea6b6SNeel Natu 2569de5ea6b6SNeel Natu static void 2570de5ea6b6SNeel Natu vmx_vlapic_cleanup(void *arg, struct vlapic *vlapic) 2571de5ea6b6SNeel Natu { 2572de5ea6b6SNeel Natu 2573de5ea6b6SNeel Natu vlapic_cleanup(vlapic); 2574de5ea6b6SNeel Natu free(vlapic, M_VLAPIC); 2575de5ea6b6SNeel Natu } 2576de5ea6b6SNeel Natu 2577366f6083SPeter Grehan struct vmm_ops vmm_ops_intel = { 2578366f6083SPeter Grehan vmx_init, 2579366f6083SPeter Grehan vmx_cleanup, 258063e62d39SJohn Baldwin vmx_restore, 2581366f6083SPeter Grehan vmx_vminit, 2582366f6083SPeter Grehan vmx_run, 2583366f6083SPeter Grehan vmx_vmcleanup, 2584366f6083SPeter Grehan vmx_getreg, 2585366f6083SPeter Grehan vmx_setreg, 2586366f6083SPeter Grehan vmx_getdesc, 2587366f6083SPeter Grehan vmx_setdesc, 2588366f6083SPeter Grehan vmx_inject, 2589366f6083SPeter Grehan vmx_getcap, 2590318224bbSNeel Natu vmx_setcap, 2591318224bbSNeel Natu ept_vmspace_alloc, 2592318224bbSNeel Natu ept_vmspace_free, 2593de5ea6b6SNeel Natu vmx_vlapic_init, 2594de5ea6b6SNeel Natu vmx_vlapic_cleanup, 2595366f6083SPeter Grehan }; 2596