1366f6083SPeter Grehan /*- 2366f6083SPeter Grehan * Copyright (c) 2011 NetApp, Inc. 3366f6083SPeter Grehan * All rights reserved. 4366f6083SPeter Grehan * 5366f6083SPeter Grehan * Redistribution and use in source and binary forms, with or without 6366f6083SPeter Grehan * modification, are permitted provided that the following conditions 7366f6083SPeter Grehan * are met: 8366f6083SPeter Grehan * 1. Redistributions of source code must retain the above copyright 9366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer. 10366f6083SPeter Grehan * 2. Redistributions in binary form must reproduce the above copyright 11366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer in the 12366f6083SPeter Grehan * documentation and/or other materials provided with the distribution. 13366f6083SPeter Grehan * 14366f6083SPeter Grehan * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 15366f6083SPeter Grehan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16366f6083SPeter Grehan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17366f6083SPeter Grehan * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 18366f6083SPeter Grehan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19366f6083SPeter Grehan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20366f6083SPeter Grehan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21366f6083SPeter Grehan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22366f6083SPeter Grehan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23366f6083SPeter Grehan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24366f6083SPeter Grehan * SUCH DAMAGE. 25366f6083SPeter Grehan * 26366f6083SPeter Grehan * $FreeBSD$ 27366f6083SPeter Grehan */ 28366f6083SPeter Grehan 29366f6083SPeter Grehan #include <sys/cdefs.h> 30366f6083SPeter Grehan __FBSDID("$FreeBSD$"); 31366f6083SPeter Grehan 32366f6083SPeter Grehan #include <sys/param.h> 33366f6083SPeter Grehan #include <sys/systm.h> 34366f6083SPeter Grehan #include <sys/smp.h> 35366f6083SPeter Grehan #include <sys/kernel.h> 36366f6083SPeter Grehan #include <sys/malloc.h> 37366f6083SPeter Grehan #include <sys/pcpu.h> 38366f6083SPeter Grehan #include <sys/proc.h> 393565b59eSNeel Natu #include <sys/sysctl.h> 40366f6083SPeter Grehan 41366f6083SPeter Grehan #include <vm/vm.h> 42366f6083SPeter Grehan #include <vm/pmap.h> 43366f6083SPeter Grehan 44366f6083SPeter Grehan #include <machine/psl.h> 45366f6083SPeter Grehan #include <machine/cpufunc.h> 468b287612SJohn Baldwin #include <machine/md_var.h> 47366f6083SPeter Grehan #include <machine/segments.h> 48176666c2SNeel Natu #include <machine/smp.h> 49608f97c3SPeter Grehan #include <machine/specialreg.h> 50366f6083SPeter Grehan #include <machine/vmparam.h> 51366f6083SPeter Grehan 52366f6083SPeter Grehan #include <machine/vmm.h> 53dc506506SNeel Natu #include <machine/vmm_dev.h> 54b01c2033SNeel Natu #include "vmm_host.h" 55762fd208STycho Nightingale #include "vmm_ioport.h" 56176666c2SNeel Natu #include "vmm_ipi.h" 57366f6083SPeter Grehan #include "vmm_msr.h" 58366f6083SPeter Grehan #include "vmm_ktr.h" 59366f6083SPeter Grehan #include "vmm_stat.h" 600775fbb4STycho Nightingale #include "vatpic.h" 61de5ea6b6SNeel Natu #include "vlapic.h" 62de5ea6b6SNeel Natu #include "vlapic_priv.h" 63366f6083SPeter Grehan 64366f6083SPeter Grehan #include "vmx_msr.h" 65366f6083SPeter Grehan #include "ept.h" 66366f6083SPeter Grehan #include "vmx_cpufunc.h" 67366f6083SPeter Grehan #include "vmx.h" 68366f6083SPeter Grehan #include "x86.h" 69366f6083SPeter Grehan #include "vmx_controls.h" 70366f6083SPeter Grehan 71366f6083SPeter Grehan #define PINBASED_CTLS_ONE_SETTING \ 72366f6083SPeter Grehan (PINBASED_EXTINT_EXITING | \ 73366f6083SPeter Grehan PINBASED_NMI_EXITING | \ 74366f6083SPeter Grehan PINBASED_VIRTUAL_NMI) 75366f6083SPeter Grehan #define PINBASED_CTLS_ZERO_SETTING 0 76366f6083SPeter Grehan 77366f6083SPeter Grehan #define PROCBASED_CTLS_WINDOW_SETTING \ 78366f6083SPeter Grehan (PROCBASED_INT_WINDOW_EXITING | \ 79366f6083SPeter Grehan PROCBASED_NMI_WINDOW_EXITING) 80366f6083SPeter Grehan 81366f6083SPeter Grehan #define PROCBASED_CTLS_ONE_SETTING \ 82366f6083SPeter Grehan (PROCBASED_SECONDARY_CONTROLS | \ 83366f6083SPeter Grehan PROCBASED_IO_EXITING | \ 84366f6083SPeter Grehan PROCBASED_MSR_BITMAPS | \ 85366f6083SPeter Grehan PROCBASED_CTLS_WINDOW_SETTING) 86366f6083SPeter Grehan #define PROCBASED_CTLS_ZERO_SETTING \ 87366f6083SPeter Grehan (PROCBASED_CR3_LOAD_EXITING | \ 88366f6083SPeter Grehan PROCBASED_CR3_STORE_EXITING | \ 89366f6083SPeter Grehan PROCBASED_IO_BITMAPS) 90366f6083SPeter Grehan 91366f6083SPeter Grehan #define PROCBASED_CTLS2_ONE_SETTING PROCBASED2_ENABLE_EPT 92366f6083SPeter Grehan #define PROCBASED_CTLS2_ZERO_SETTING 0 93366f6083SPeter Grehan 94608f97c3SPeter Grehan #define VM_EXIT_CTLS_ONE_SETTING_NO_PAT \ 95366f6083SPeter Grehan (VM_EXIT_HOST_LMA | \ 96366f6083SPeter Grehan VM_EXIT_SAVE_EFER | \ 97366f6083SPeter Grehan VM_EXIT_LOAD_EFER) 98608f97c3SPeter Grehan 99608f97c3SPeter Grehan #define VM_EXIT_CTLS_ONE_SETTING \ 100608f97c3SPeter Grehan (VM_EXIT_CTLS_ONE_SETTING_NO_PAT | \ 101f7d47425SNeel Natu VM_EXIT_ACKNOWLEDGE_INTERRUPT | \ 102608f97c3SPeter Grehan VM_EXIT_SAVE_PAT | \ 103608f97c3SPeter Grehan VM_EXIT_LOAD_PAT) 104366f6083SPeter Grehan #define VM_EXIT_CTLS_ZERO_SETTING VM_EXIT_SAVE_DEBUG_CONTROLS 105366f6083SPeter Grehan 106608f97c3SPeter Grehan #define VM_ENTRY_CTLS_ONE_SETTING_NO_PAT VM_ENTRY_LOAD_EFER 107608f97c3SPeter Grehan 108366f6083SPeter Grehan #define VM_ENTRY_CTLS_ONE_SETTING \ 109608f97c3SPeter Grehan (VM_ENTRY_CTLS_ONE_SETTING_NO_PAT | \ 110608f97c3SPeter Grehan VM_ENTRY_LOAD_PAT) 111366f6083SPeter Grehan #define VM_ENTRY_CTLS_ZERO_SETTING \ 112366f6083SPeter Grehan (VM_ENTRY_LOAD_DEBUG_CONTROLS | \ 113366f6083SPeter Grehan VM_ENTRY_INTO_SMM | \ 114366f6083SPeter Grehan VM_ENTRY_DEACTIVATE_DUAL_MONITOR) 115366f6083SPeter Grehan 116366f6083SPeter Grehan #define guest_msr_rw(vmx, msr) \ 117366f6083SPeter Grehan msr_bitmap_change_access((vmx)->msr_bitmap, (msr), MSR_BITMAP_ACCESS_RW) 118366f6083SPeter Grehan 119159dd56fSNeel Natu #define guest_msr_ro(vmx, msr) \ 120159dd56fSNeel Natu msr_bitmap_change_access((vmx)->msr_bitmap, (msr), MSR_BITMAP_ACCESS_READ) 121159dd56fSNeel Natu 122366f6083SPeter Grehan #define HANDLED 1 123366f6083SPeter Grehan #define UNHANDLED 0 124366f6083SPeter Grehan 125de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VMX, "vmx", "vmx"); 126de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic"); 127366f6083SPeter Grehan 1283565b59eSNeel Natu SYSCTL_DECL(_hw_vmm); 1293565b59eSNeel Natu SYSCTL_NODE(_hw_vmm, OID_AUTO, vmx, CTLFLAG_RW, NULL, NULL); 1303565b59eSNeel Natu 131b3996dd4SJohn Baldwin int vmxon_enabled[MAXCPU]; 132366f6083SPeter Grehan static char vmxon_region[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE); 133366f6083SPeter Grehan 134366f6083SPeter Grehan static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2; 135366f6083SPeter Grehan static uint32_t exit_ctls, entry_ctls; 136366f6083SPeter Grehan 137366f6083SPeter Grehan static uint64_t cr0_ones_mask, cr0_zeros_mask; 1383565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_ones_mask, CTLFLAG_RD, 1393565b59eSNeel Natu &cr0_ones_mask, 0, NULL); 1403565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_zeros_mask, CTLFLAG_RD, 1413565b59eSNeel Natu &cr0_zeros_mask, 0, NULL); 1423565b59eSNeel Natu 143366f6083SPeter Grehan static uint64_t cr4_ones_mask, cr4_zeros_mask; 1443565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_ones_mask, CTLFLAG_RD, 1453565b59eSNeel Natu &cr4_ones_mask, 0, NULL); 1463565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_zeros_mask, CTLFLAG_RD, 1473565b59eSNeel Natu &cr4_zeros_mask, 0, NULL); 148366f6083SPeter Grehan 149608f97c3SPeter Grehan static int vmx_no_patmsr; 150608f97c3SPeter Grehan 1513565b59eSNeel Natu static int vmx_initialized; 1523565b59eSNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, initialized, CTLFLAG_RD, 1533565b59eSNeel Natu &vmx_initialized, 0, "Intel VMX initialized"); 1543565b59eSNeel Natu 155366f6083SPeter Grehan /* 156366f6083SPeter Grehan * Optional capabilities 157366f6083SPeter Grehan */ 158366f6083SPeter Grehan static int cap_halt_exit; 159366f6083SPeter Grehan static int cap_pause_exit; 160366f6083SPeter Grehan static int cap_unrestricted_guest; 161366f6083SPeter Grehan static int cap_monitor_trap; 16249cc03daSNeel Natu static int cap_invpcid; 163366f6083SPeter Grehan 16488c4b8d1SNeel Natu static int virtual_interrupt_delivery; 16588c4b8d1SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, virtual_interrupt_delivery, CTLFLAG_RD, 16688c4b8d1SNeel Natu &virtual_interrupt_delivery, 0, "APICv virtual interrupt delivery support"); 16788c4b8d1SNeel Natu 168176666c2SNeel Natu static int posted_interrupts; 169176666c2SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupts, CTLFLAG_RD, 170176666c2SNeel Natu &posted_interrupts, 0, "APICv posted interrupt support"); 171176666c2SNeel Natu 172176666c2SNeel Natu static int pirvec; 173176666c2SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupt_vector, CTLFLAG_RD, 174176666c2SNeel Natu &pirvec, 0, "APICv posted interrupt vector"); 175176666c2SNeel Natu 17645e51299SNeel Natu static struct unrhdr *vpid_unr; 17745e51299SNeel Natu static u_int vpid_alloc_failed; 17845e51299SNeel Natu SYSCTL_UINT(_hw_vmm_vmx, OID_AUTO, vpid_alloc_failed, CTLFLAG_RD, 17945e51299SNeel Natu &vpid_alloc_failed, 0, NULL); 18045e51299SNeel Natu 18188c4b8d1SNeel Natu /* 18288c4b8d1SNeel Natu * Use the last page below 4GB as the APIC access address. This address is 18388c4b8d1SNeel Natu * occupied by the boot firmware so it is guaranteed that it will not conflict 18488c4b8d1SNeel Natu * with a page in system memory. 18588c4b8d1SNeel Natu */ 18688c4b8d1SNeel Natu #define APIC_ACCESS_ADDRESS 0xFFFFF000 18788c4b8d1SNeel Natu 18888c4b8d1SNeel Natu static void vmx_inject_pir(struct vlapic *vlapic); 18988c4b8d1SNeel Natu 190366f6083SPeter Grehan #ifdef KTR 191366f6083SPeter Grehan static const char * 192366f6083SPeter Grehan exit_reason_to_str(int reason) 193366f6083SPeter Grehan { 194366f6083SPeter Grehan static char reasonbuf[32]; 195366f6083SPeter Grehan 196366f6083SPeter Grehan switch (reason) { 197366f6083SPeter Grehan case EXIT_REASON_EXCEPTION: 198366f6083SPeter Grehan return "exception"; 199366f6083SPeter Grehan case EXIT_REASON_EXT_INTR: 200366f6083SPeter Grehan return "extint"; 201366f6083SPeter Grehan case EXIT_REASON_TRIPLE_FAULT: 202366f6083SPeter Grehan return "triplefault"; 203366f6083SPeter Grehan case EXIT_REASON_INIT: 204366f6083SPeter Grehan return "init"; 205366f6083SPeter Grehan case EXIT_REASON_SIPI: 206366f6083SPeter Grehan return "sipi"; 207366f6083SPeter Grehan case EXIT_REASON_IO_SMI: 208366f6083SPeter Grehan return "iosmi"; 209366f6083SPeter Grehan case EXIT_REASON_SMI: 210366f6083SPeter Grehan return "smi"; 211366f6083SPeter Grehan case EXIT_REASON_INTR_WINDOW: 212366f6083SPeter Grehan return "intrwindow"; 213366f6083SPeter Grehan case EXIT_REASON_NMI_WINDOW: 214366f6083SPeter Grehan return "nmiwindow"; 215366f6083SPeter Grehan case EXIT_REASON_TASK_SWITCH: 216366f6083SPeter Grehan return "taskswitch"; 217366f6083SPeter Grehan case EXIT_REASON_CPUID: 218366f6083SPeter Grehan return "cpuid"; 219366f6083SPeter Grehan case EXIT_REASON_GETSEC: 220366f6083SPeter Grehan return "getsec"; 221366f6083SPeter Grehan case EXIT_REASON_HLT: 222366f6083SPeter Grehan return "hlt"; 223366f6083SPeter Grehan case EXIT_REASON_INVD: 224366f6083SPeter Grehan return "invd"; 225366f6083SPeter Grehan case EXIT_REASON_INVLPG: 226366f6083SPeter Grehan return "invlpg"; 227366f6083SPeter Grehan case EXIT_REASON_RDPMC: 228366f6083SPeter Grehan return "rdpmc"; 229366f6083SPeter Grehan case EXIT_REASON_RDTSC: 230366f6083SPeter Grehan return "rdtsc"; 231366f6083SPeter Grehan case EXIT_REASON_RSM: 232366f6083SPeter Grehan return "rsm"; 233366f6083SPeter Grehan case EXIT_REASON_VMCALL: 234366f6083SPeter Grehan return "vmcall"; 235366f6083SPeter Grehan case EXIT_REASON_VMCLEAR: 236366f6083SPeter Grehan return "vmclear"; 237366f6083SPeter Grehan case EXIT_REASON_VMLAUNCH: 238366f6083SPeter Grehan return "vmlaunch"; 239366f6083SPeter Grehan case EXIT_REASON_VMPTRLD: 240366f6083SPeter Grehan return "vmptrld"; 241366f6083SPeter Grehan case EXIT_REASON_VMPTRST: 242366f6083SPeter Grehan return "vmptrst"; 243366f6083SPeter Grehan case EXIT_REASON_VMREAD: 244366f6083SPeter Grehan return "vmread"; 245366f6083SPeter Grehan case EXIT_REASON_VMRESUME: 246366f6083SPeter Grehan return "vmresume"; 247366f6083SPeter Grehan case EXIT_REASON_VMWRITE: 248366f6083SPeter Grehan return "vmwrite"; 249366f6083SPeter Grehan case EXIT_REASON_VMXOFF: 250366f6083SPeter Grehan return "vmxoff"; 251366f6083SPeter Grehan case EXIT_REASON_VMXON: 252366f6083SPeter Grehan return "vmxon"; 253366f6083SPeter Grehan case EXIT_REASON_CR_ACCESS: 254366f6083SPeter Grehan return "craccess"; 255366f6083SPeter Grehan case EXIT_REASON_DR_ACCESS: 256366f6083SPeter Grehan return "draccess"; 257366f6083SPeter Grehan case EXIT_REASON_INOUT: 258366f6083SPeter Grehan return "inout"; 259366f6083SPeter Grehan case EXIT_REASON_RDMSR: 260366f6083SPeter Grehan return "rdmsr"; 261366f6083SPeter Grehan case EXIT_REASON_WRMSR: 262366f6083SPeter Grehan return "wrmsr"; 263366f6083SPeter Grehan case EXIT_REASON_INVAL_VMCS: 264366f6083SPeter Grehan return "invalvmcs"; 265366f6083SPeter Grehan case EXIT_REASON_INVAL_MSR: 266366f6083SPeter Grehan return "invalmsr"; 267366f6083SPeter Grehan case EXIT_REASON_MWAIT: 268366f6083SPeter Grehan return "mwait"; 269366f6083SPeter Grehan case EXIT_REASON_MTF: 270366f6083SPeter Grehan return "mtf"; 271366f6083SPeter Grehan case EXIT_REASON_MONITOR: 272366f6083SPeter Grehan return "monitor"; 273366f6083SPeter Grehan case EXIT_REASON_PAUSE: 274366f6083SPeter Grehan return "pause"; 275366f6083SPeter Grehan case EXIT_REASON_MCE: 276366f6083SPeter Grehan return "mce"; 277366f6083SPeter Grehan case EXIT_REASON_TPR: 278366f6083SPeter Grehan return "tpr"; 27988c4b8d1SNeel Natu case EXIT_REASON_APIC_ACCESS: 28088c4b8d1SNeel Natu return "apic-access"; 281366f6083SPeter Grehan case EXIT_REASON_GDTR_IDTR: 282366f6083SPeter Grehan return "gdtridtr"; 283366f6083SPeter Grehan case EXIT_REASON_LDTR_TR: 284366f6083SPeter Grehan return "ldtrtr"; 285366f6083SPeter Grehan case EXIT_REASON_EPT_FAULT: 286366f6083SPeter Grehan return "eptfault"; 287366f6083SPeter Grehan case EXIT_REASON_EPT_MISCONFIG: 288366f6083SPeter Grehan return "eptmisconfig"; 289366f6083SPeter Grehan case EXIT_REASON_INVEPT: 290366f6083SPeter Grehan return "invept"; 291366f6083SPeter Grehan case EXIT_REASON_RDTSCP: 292366f6083SPeter Grehan return "rdtscp"; 293366f6083SPeter Grehan case EXIT_REASON_VMX_PREEMPT: 294366f6083SPeter Grehan return "vmxpreempt"; 295366f6083SPeter Grehan case EXIT_REASON_INVVPID: 296366f6083SPeter Grehan return "invvpid"; 297366f6083SPeter Grehan case EXIT_REASON_WBINVD: 298366f6083SPeter Grehan return "wbinvd"; 299366f6083SPeter Grehan case EXIT_REASON_XSETBV: 300366f6083SPeter Grehan return "xsetbv"; 30188c4b8d1SNeel Natu case EXIT_REASON_APIC_WRITE: 30288c4b8d1SNeel Natu return "apic-write"; 303366f6083SPeter Grehan default: 304366f6083SPeter Grehan snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason); 305366f6083SPeter Grehan return (reasonbuf); 306366f6083SPeter Grehan } 307366f6083SPeter Grehan } 308366f6083SPeter Grehan #endif /* KTR */ 309366f6083SPeter Grehan 310159dd56fSNeel Natu static int 311159dd56fSNeel Natu vmx_allow_x2apic_msrs(struct vmx *vmx) 312159dd56fSNeel Natu { 313159dd56fSNeel Natu int i, error; 314159dd56fSNeel Natu 315159dd56fSNeel Natu error = 0; 316159dd56fSNeel Natu 317159dd56fSNeel Natu /* 318159dd56fSNeel Natu * Allow readonly access to the following x2APIC MSRs from the guest. 319159dd56fSNeel Natu */ 320159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_ID); 321159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_VERSION); 322159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LDR); 323159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_SVR); 324159dd56fSNeel Natu 325159dd56fSNeel Natu for (i = 0; i < 8; i++) 326159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_ISR0 + i); 327159dd56fSNeel Natu 328159dd56fSNeel Natu for (i = 0; i < 8; i++) 329159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_TMR0 + i); 330159dd56fSNeel Natu 331159dd56fSNeel Natu for (i = 0; i < 8; i++) 332159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_IRR0 + i); 333159dd56fSNeel Natu 334159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_ESR); 335159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_TIMER); 336159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_THERMAL); 337159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_PCINT); 338159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT0); 339159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT1); 340159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_ERROR); 341159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_ICR_TIMER); 342159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_DCR_TIMER); 343159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_ICR); 344159dd56fSNeel Natu 345159dd56fSNeel Natu /* 346159dd56fSNeel Natu * Allow TPR, EOI and SELF_IPI MSRs to be read and written by the guest. 347159dd56fSNeel Natu * 348159dd56fSNeel Natu * These registers get special treatment described in the section 349159dd56fSNeel Natu * "Virtualizing MSR-Based APIC Accesses". 350159dd56fSNeel Natu */ 351159dd56fSNeel Natu error += guest_msr_rw(vmx, MSR_APIC_TPR); 352159dd56fSNeel Natu error += guest_msr_rw(vmx, MSR_APIC_EOI); 353159dd56fSNeel Natu error += guest_msr_rw(vmx, MSR_APIC_SELF_IPI); 354159dd56fSNeel Natu 355159dd56fSNeel Natu return (error); 356159dd56fSNeel Natu } 357159dd56fSNeel Natu 358366f6083SPeter Grehan u_long 359366f6083SPeter Grehan vmx_fix_cr0(u_long cr0) 360366f6083SPeter Grehan { 361366f6083SPeter Grehan 362366f6083SPeter Grehan return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask); 363366f6083SPeter Grehan } 364366f6083SPeter Grehan 365366f6083SPeter Grehan u_long 366366f6083SPeter Grehan vmx_fix_cr4(u_long cr4) 367366f6083SPeter Grehan { 368366f6083SPeter Grehan 369366f6083SPeter Grehan return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask); 370366f6083SPeter Grehan } 371366f6083SPeter Grehan 372366f6083SPeter Grehan static void 37345e51299SNeel Natu vpid_free(int vpid) 37445e51299SNeel Natu { 37545e51299SNeel Natu if (vpid < 0 || vpid > 0xffff) 37645e51299SNeel Natu panic("vpid_free: invalid vpid %d", vpid); 37745e51299SNeel Natu 37845e51299SNeel Natu /* 37945e51299SNeel Natu * VPIDs [0,VM_MAXCPU] are special and are not allocated from 38045e51299SNeel Natu * the unit number allocator. 38145e51299SNeel Natu */ 38245e51299SNeel Natu 38345e51299SNeel Natu if (vpid > VM_MAXCPU) 38445e51299SNeel Natu free_unr(vpid_unr, vpid); 38545e51299SNeel Natu } 38645e51299SNeel Natu 38745e51299SNeel Natu static void 38845e51299SNeel Natu vpid_alloc(uint16_t *vpid, int num) 38945e51299SNeel Natu { 39045e51299SNeel Natu int i, x; 39145e51299SNeel Natu 39245e51299SNeel Natu if (num <= 0 || num > VM_MAXCPU) 39345e51299SNeel Natu panic("invalid number of vpids requested: %d", num); 39445e51299SNeel Natu 39545e51299SNeel Natu /* 39645e51299SNeel Natu * If the "enable vpid" execution control is not enabled then the 39745e51299SNeel Natu * VPID is required to be 0 for all vcpus. 39845e51299SNeel Natu */ 39945e51299SNeel Natu if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) == 0) { 40045e51299SNeel Natu for (i = 0; i < num; i++) 40145e51299SNeel Natu vpid[i] = 0; 40245e51299SNeel Natu return; 40345e51299SNeel Natu } 40445e51299SNeel Natu 40545e51299SNeel Natu /* 40645e51299SNeel Natu * Allocate a unique VPID for each vcpu from the unit number allocator. 40745e51299SNeel Natu */ 40845e51299SNeel Natu for (i = 0; i < num; i++) { 40945e51299SNeel Natu x = alloc_unr(vpid_unr); 41045e51299SNeel Natu if (x == -1) 41145e51299SNeel Natu break; 41245e51299SNeel Natu else 41345e51299SNeel Natu vpid[i] = x; 41445e51299SNeel Natu } 41545e51299SNeel Natu 41645e51299SNeel Natu if (i < num) { 41745e51299SNeel Natu atomic_add_int(&vpid_alloc_failed, 1); 41845e51299SNeel Natu 41945e51299SNeel Natu /* 42045e51299SNeel Natu * If the unit number allocator does not have enough unique 42145e51299SNeel Natu * VPIDs then we need to allocate from the [1,VM_MAXCPU] range. 42245e51299SNeel Natu * 42345e51299SNeel Natu * These VPIDs are not be unique across VMs but this does not 42445e51299SNeel Natu * affect correctness because the combined mappings are also 42545e51299SNeel Natu * tagged with the EP4TA which is unique for each VM. 42645e51299SNeel Natu * 42745e51299SNeel Natu * It is still sub-optimal because the invvpid will invalidate 42845e51299SNeel Natu * combined mappings for a particular VPID across all EP4TAs. 42945e51299SNeel Natu */ 43045e51299SNeel Natu while (i-- > 0) 43145e51299SNeel Natu vpid_free(vpid[i]); 43245e51299SNeel Natu 43345e51299SNeel Natu for (i = 0; i < num; i++) 43445e51299SNeel Natu vpid[i] = i + 1; 43545e51299SNeel Natu } 43645e51299SNeel Natu } 43745e51299SNeel Natu 43845e51299SNeel Natu static void 43945e51299SNeel Natu vpid_init(void) 44045e51299SNeel Natu { 44145e51299SNeel Natu /* 44245e51299SNeel Natu * VPID 0 is required when the "enable VPID" execution control is 44345e51299SNeel Natu * disabled. 44445e51299SNeel Natu * 44545e51299SNeel Natu * VPIDs [1,VM_MAXCPU] are used as the "overflow namespace" when the 44645e51299SNeel Natu * unit number allocator does not have sufficient unique VPIDs to 44745e51299SNeel Natu * satisfy the allocation. 44845e51299SNeel Natu * 44945e51299SNeel Natu * The remaining VPIDs are managed by the unit number allocator. 45045e51299SNeel Natu */ 45145e51299SNeel Natu vpid_unr = new_unrhdr(VM_MAXCPU + 1, 0xffff, NULL); 45245e51299SNeel Natu } 45345e51299SNeel Natu 45445e51299SNeel Natu static void 455366f6083SPeter Grehan msr_save_area_init(struct msr_entry *g_area, int *g_count) 456366f6083SPeter Grehan { 457366f6083SPeter Grehan int cnt; 458366f6083SPeter Grehan 459366f6083SPeter Grehan static struct msr_entry guest_msrs[] = { 460366f6083SPeter Grehan { MSR_KGSBASE, 0, 0 }, 461366f6083SPeter Grehan }; 462366f6083SPeter Grehan 463366f6083SPeter Grehan cnt = sizeof(guest_msrs) / sizeof(guest_msrs[0]); 464366f6083SPeter Grehan if (cnt > GUEST_MSR_MAX_ENTRIES) 465366f6083SPeter Grehan panic("guest msr save area overrun"); 466366f6083SPeter Grehan bcopy(guest_msrs, g_area, sizeof(guest_msrs)); 467366f6083SPeter Grehan *g_count = cnt; 468366f6083SPeter Grehan } 469366f6083SPeter Grehan 470366f6083SPeter Grehan static void 471366f6083SPeter Grehan vmx_disable(void *arg __unused) 472366f6083SPeter Grehan { 473366f6083SPeter Grehan struct invvpid_desc invvpid_desc = { 0 }; 474366f6083SPeter Grehan struct invept_desc invept_desc = { 0 }; 475366f6083SPeter Grehan 476366f6083SPeter Grehan if (vmxon_enabled[curcpu]) { 477366f6083SPeter Grehan /* 478366f6083SPeter Grehan * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b. 479366f6083SPeter Grehan * 480366f6083SPeter Grehan * VMXON or VMXOFF are not required to invalidate any TLB 481366f6083SPeter Grehan * caching structures. This prevents potential retention of 482366f6083SPeter Grehan * cached information in the TLB between distinct VMX episodes. 483366f6083SPeter Grehan */ 484366f6083SPeter Grehan invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc); 485366f6083SPeter Grehan invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc); 486366f6083SPeter Grehan vmxoff(); 487366f6083SPeter Grehan } 488366f6083SPeter Grehan load_cr4(rcr4() & ~CR4_VMXE); 489366f6083SPeter Grehan } 490366f6083SPeter Grehan 491366f6083SPeter Grehan static int 492366f6083SPeter Grehan vmx_cleanup(void) 493366f6083SPeter Grehan { 494366f6083SPeter Grehan 495176666c2SNeel Natu if (pirvec != 0) 496176666c2SNeel Natu vmm_ipi_free(pirvec); 497176666c2SNeel Natu 49845e51299SNeel Natu if (vpid_unr != NULL) { 49945e51299SNeel Natu delete_unrhdr(vpid_unr); 50045e51299SNeel Natu vpid_unr = NULL; 50145e51299SNeel Natu } 50245e51299SNeel Natu 503366f6083SPeter Grehan smp_rendezvous(NULL, vmx_disable, NULL, NULL); 504366f6083SPeter Grehan 505366f6083SPeter Grehan return (0); 506366f6083SPeter Grehan } 507366f6083SPeter Grehan 508366f6083SPeter Grehan static void 509366f6083SPeter Grehan vmx_enable(void *arg __unused) 510366f6083SPeter Grehan { 511366f6083SPeter Grehan int error; 512366f6083SPeter Grehan 513366f6083SPeter Grehan load_cr4(rcr4() | CR4_VMXE); 514366f6083SPeter Grehan 515366f6083SPeter Grehan *(uint32_t *)vmxon_region[curcpu] = vmx_revision(); 516366f6083SPeter Grehan error = vmxon(vmxon_region[curcpu]); 517366f6083SPeter Grehan if (error == 0) 518366f6083SPeter Grehan vmxon_enabled[curcpu] = 1; 519366f6083SPeter Grehan } 520366f6083SPeter Grehan 52163e62d39SJohn Baldwin static void 52263e62d39SJohn Baldwin vmx_restore(void) 52363e62d39SJohn Baldwin { 52463e62d39SJohn Baldwin 52563e62d39SJohn Baldwin if (vmxon_enabled[curcpu]) 52663e62d39SJohn Baldwin vmxon(vmxon_region[curcpu]); 52763e62d39SJohn Baldwin } 52863e62d39SJohn Baldwin 529366f6083SPeter Grehan static int 530add611fdSNeel Natu vmx_init(int ipinum) 531366f6083SPeter Grehan { 53288c4b8d1SNeel Natu int error, use_tpr_shadow; 5334bff7fadSNeel Natu uint64_t fixed0, fixed1, feature_control; 53488c4b8d1SNeel Natu uint32_t tmp, procbased2_vid_bits; 535366f6083SPeter Grehan 536366f6083SPeter Grehan /* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */ 5378b287612SJohn Baldwin if (!(cpu_feature2 & CPUID2_VMX)) { 538366f6083SPeter Grehan printf("vmx_init: processor does not support VMX operation\n"); 539366f6083SPeter Grehan return (ENXIO); 540366f6083SPeter Grehan } 541366f6083SPeter Grehan 5424bff7fadSNeel Natu /* 5434bff7fadSNeel Natu * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits 5444bff7fadSNeel Natu * are set (bits 0 and 2 respectively). 5454bff7fadSNeel Natu */ 5464bff7fadSNeel Natu feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL); 547150369abSNeel Natu if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 0 || 548150369abSNeel Natu (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) { 5494bff7fadSNeel Natu printf("vmx_init: VMX operation disabled by BIOS\n"); 5504bff7fadSNeel Natu return (ENXIO); 5514bff7fadSNeel Natu } 5524bff7fadSNeel Natu 553366f6083SPeter Grehan /* Check support for primary processor-based VM-execution controls */ 554366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 555366f6083SPeter Grehan MSR_VMX_TRUE_PROCBASED_CTLS, 556366f6083SPeter Grehan PROCBASED_CTLS_ONE_SETTING, 557366f6083SPeter Grehan PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls); 558366f6083SPeter Grehan if (error) { 559366f6083SPeter Grehan printf("vmx_init: processor does not support desired primary " 560366f6083SPeter Grehan "processor-based controls\n"); 561366f6083SPeter Grehan return (error); 562366f6083SPeter Grehan } 563366f6083SPeter Grehan 564366f6083SPeter Grehan /* Clear the processor-based ctl bits that are set on demand */ 565366f6083SPeter Grehan procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING; 566366f6083SPeter Grehan 567366f6083SPeter Grehan /* Check support for secondary processor-based VM-execution controls */ 568366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 569366f6083SPeter Grehan MSR_VMX_PROCBASED_CTLS2, 570366f6083SPeter Grehan PROCBASED_CTLS2_ONE_SETTING, 571366f6083SPeter Grehan PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2); 572366f6083SPeter Grehan if (error) { 573366f6083SPeter Grehan printf("vmx_init: processor does not support desired secondary " 574366f6083SPeter Grehan "processor-based controls\n"); 575366f6083SPeter Grehan return (error); 576366f6083SPeter Grehan } 577366f6083SPeter Grehan 578366f6083SPeter Grehan /* Check support for VPID */ 579366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2, 580366f6083SPeter Grehan PROCBASED2_ENABLE_VPID, 0, &tmp); 581366f6083SPeter Grehan if (error == 0) 582366f6083SPeter Grehan procbased_ctls2 |= PROCBASED2_ENABLE_VPID; 583366f6083SPeter Grehan 584366f6083SPeter Grehan /* Check support for pin-based VM-execution controls */ 585366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS, 586366f6083SPeter Grehan MSR_VMX_TRUE_PINBASED_CTLS, 587366f6083SPeter Grehan PINBASED_CTLS_ONE_SETTING, 588366f6083SPeter Grehan PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls); 589366f6083SPeter Grehan if (error) { 590366f6083SPeter Grehan printf("vmx_init: processor does not support desired " 591366f6083SPeter Grehan "pin-based controls\n"); 592366f6083SPeter Grehan return (error); 593366f6083SPeter Grehan } 594366f6083SPeter Grehan 595366f6083SPeter Grehan /* Check support for VM-exit controls */ 596366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS, 597366f6083SPeter Grehan VM_EXIT_CTLS_ONE_SETTING, 598366f6083SPeter Grehan VM_EXIT_CTLS_ZERO_SETTING, 599366f6083SPeter Grehan &exit_ctls); 600366f6083SPeter Grehan if (error) { 601608f97c3SPeter Grehan /* Try again without the PAT MSR bits */ 602608f97c3SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, 603608f97c3SPeter Grehan MSR_VMX_TRUE_EXIT_CTLS, 604608f97c3SPeter Grehan VM_EXIT_CTLS_ONE_SETTING_NO_PAT, 605608f97c3SPeter Grehan VM_EXIT_CTLS_ZERO_SETTING, 606608f97c3SPeter Grehan &exit_ctls); 607608f97c3SPeter Grehan if (error) { 608366f6083SPeter Grehan printf("vmx_init: processor does not support desired " 609366f6083SPeter Grehan "exit controls\n"); 610366f6083SPeter Grehan return (error); 611608f97c3SPeter Grehan } else { 612608f97c3SPeter Grehan if (bootverbose) 613608f97c3SPeter Grehan printf("vmm: PAT MSR access not supported\n"); 614608f97c3SPeter Grehan guest_msr_valid(MSR_PAT); 615608f97c3SPeter Grehan vmx_no_patmsr = 1; 616608f97c3SPeter Grehan } 617366f6083SPeter Grehan } 618366f6083SPeter Grehan 619366f6083SPeter Grehan /* Check support for VM-entry controls */ 620608f97c3SPeter Grehan if (!vmx_no_patmsr) { 621608f97c3SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS, 622608f97c3SPeter Grehan MSR_VMX_TRUE_ENTRY_CTLS, 623366f6083SPeter Grehan VM_ENTRY_CTLS_ONE_SETTING, 624366f6083SPeter Grehan VM_ENTRY_CTLS_ZERO_SETTING, 625366f6083SPeter Grehan &entry_ctls); 626608f97c3SPeter Grehan } else { 627608f97c3SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS, 628608f97c3SPeter Grehan MSR_VMX_TRUE_ENTRY_CTLS, 629608f97c3SPeter Grehan VM_ENTRY_CTLS_ONE_SETTING_NO_PAT, 630608f97c3SPeter Grehan VM_ENTRY_CTLS_ZERO_SETTING, 631608f97c3SPeter Grehan &entry_ctls); 632608f97c3SPeter Grehan } 633608f97c3SPeter Grehan 634366f6083SPeter Grehan if (error) { 635366f6083SPeter Grehan printf("vmx_init: processor does not support desired " 636366f6083SPeter Grehan "entry controls\n"); 637366f6083SPeter Grehan return (error); 638366f6083SPeter Grehan } 639366f6083SPeter Grehan 640366f6083SPeter Grehan /* 641366f6083SPeter Grehan * Check support for optional features by testing them 642366f6083SPeter Grehan * as individual bits 643366f6083SPeter Grehan */ 644366f6083SPeter Grehan cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 645366f6083SPeter Grehan MSR_VMX_TRUE_PROCBASED_CTLS, 646366f6083SPeter Grehan PROCBASED_HLT_EXITING, 0, 647366f6083SPeter Grehan &tmp) == 0); 648366f6083SPeter Grehan 649366f6083SPeter Grehan cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 650366f6083SPeter Grehan MSR_VMX_PROCBASED_CTLS, 651366f6083SPeter Grehan PROCBASED_MTF, 0, 652366f6083SPeter Grehan &tmp) == 0); 653366f6083SPeter Grehan 654366f6083SPeter Grehan cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 655366f6083SPeter Grehan MSR_VMX_TRUE_PROCBASED_CTLS, 656366f6083SPeter Grehan PROCBASED_PAUSE_EXITING, 0, 657366f6083SPeter Grehan &tmp) == 0); 658366f6083SPeter Grehan 659366f6083SPeter Grehan cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 660366f6083SPeter Grehan MSR_VMX_PROCBASED_CTLS2, 661366f6083SPeter Grehan PROCBASED2_UNRESTRICTED_GUEST, 0, 662366f6083SPeter Grehan &tmp) == 0); 663366f6083SPeter Grehan 66449cc03daSNeel Natu cap_invpcid = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 66549cc03daSNeel Natu MSR_VMX_PROCBASED_CTLS2, PROCBASED2_ENABLE_INVPCID, 0, 66649cc03daSNeel Natu &tmp) == 0); 66749cc03daSNeel Natu 66888c4b8d1SNeel Natu /* 66988c4b8d1SNeel Natu * Check support for virtual interrupt delivery. 67088c4b8d1SNeel Natu */ 67188c4b8d1SNeel Natu procbased2_vid_bits = (PROCBASED2_VIRTUALIZE_APIC_ACCESSES | 67288c4b8d1SNeel Natu PROCBASED2_VIRTUALIZE_X2APIC_MODE | 67388c4b8d1SNeel Natu PROCBASED2_APIC_REGISTER_VIRTUALIZATION | 67488c4b8d1SNeel Natu PROCBASED2_VIRTUAL_INTERRUPT_DELIVERY); 67588c4b8d1SNeel Natu 67688c4b8d1SNeel Natu use_tpr_shadow = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 67788c4b8d1SNeel Natu MSR_VMX_TRUE_PROCBASED_CTLS, PROCBASED_USE_TPR_SHADOW, 0, 67888c4b8d1SNeel Natu &tmp) == 0); 67988c4b8d1SNeel Natu 68088c4b8d1SNeel Natu error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2, 68188c4b8d1SNeel Natu procbased2_vid_bits, 0, &tmp); 68288c4b8d1SNeel Natu if (error == 0 && use_tpr_shadow) { 68388c4b8d1SNeel Natu virtual_interrupt_delivery = 1; 68488c4b8d1SNeel Natu TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_vid", 68588c4b8d1SNeel Natu &virtual_interrupt_delivery); 68688c4b8d1SNeel Natu } 68788c4b8d1SNeel Natu 68888c4b8d1SNeel Natu if (virtual_interrupt_delivery) { 68988c4b8d1SNeel Natu procbased_ctls |= PROCBASED_USE_TPR_SHADOW; 69088c4b8d1SNeel Natu procbased_ctls2 |= procbased2_vid_bits; 69188c4b8d1SNeel Natu procbased_ctls2 &= ~PROCBASED2_VIRTUALIZE_X2APIC_MODE; 692176666c2SNeel Natu 693176666c2SNeel Natu /* 694176666c2SNeel Natu * Check for Posted Interrupts only if Virtual Interrupt 695176666c2SNeel Natu * Delivery is enabled. 696176666c2SNeel Natu */ 697176666c2SNeel Natu error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS, 698176666c2SNeel Natu MSR_VMX_TRUE_PINBASED_CTLS, PINBASED_POSTED_INTERRUPT, 0, 699176666c2SNeel Natu &tmp); 700176666c2SNeel Natu if (error == 0) { 701176666c2SNeel Natu pirvec = vmm_ipi_alloc(); 702176666c2SNeel Natu if (pirvec == 0) { 703176666c2SNeel Natu if (bootverbose) { 704176666c2SNeel Natu printf("vmx_init: unable to allocate " 705176666c2SNeel Natu "posted interrupt vector\n"); 70688c4b8d1SNeel Natu } 707176666c2SNeel Natu } else { 708176666c2SNeel Natu posted_interrupts = 1; 709176666c2SNeel Natu TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_pir", 710176666c2SNeel Natu &posted_interrupts); 711176666c2SNeel Natu } 712176666c2SNeel Natu } 713176666c2SNeel Natu } 714176666c2SNeel Natu 715176666c2SNeel Natu if (posted_interrupts) 716176666c2SNeel Natu pinbased_ctls |= PINBASED_POSTED_INTERRUPT; 71749cc03daSNeel Natu 718366f6083SPeter Grehan /* Initialize EPT */ 719add611fdSNeel Natu error = ept_init(ipinum); 720366f6083SPeter Grehan if (error) { 721366f6083SPeter Grehan printf("vmx_init: ept initialization failed (%d)\n", error); 722366f6083SPeter Grehan return (error); 723366f6083SPeter Grehan } 724366f6083SPeter Grehan 725366f6083SPeter Grehan /* 726366f6083SPeter Grehan * Stash the cr0 and cr4 bits that must be fixed to 0 or 1 727366f6083SPeter Grehan */ 728366f6083SPeter Grehan fixed0 = rdmsr(MSR_VMX_CR0_FIXED0); 729366f6083SPeter Grehan fixed1 = rdmsr(MSR_VMX_CR0_FIXED1); 730366f6083SPeter Grehan cr0_ones_mask = fixed0 & fixed1; 731366f6083SPeter Grehan cr0_zeros_mask = ~fixed0 & ~fixed1; 732366f6083SPeter Grehan 733366f6083SPeter Grehan /* 734366f6083SPeter Grehan * CR0_PE and CR0_PG can be set to zero in VMX non-root operation 735366f6083SPeter Grehan * if unrestricted guest execution is allowed. 736366f6083SPeter Grehan */ 737366f6083SPeter Grehan if (cap_unrestricted_guest) 738366f6083SPeter Grehan cr0_ones_mask &= ~(CR0_PG | CR0_PE); 739366f6083SPeter Grehan 740366f6083SPeter Grehan /* 741366f6083SPeter Grehan * Do not allow the guest to set CR0_NW or CR0_CD. 742366f6083SPeter Grehan */ 743366f6083SPeter Grehan cr0_zeros_mask |= (CR0_NW | CR0_CD); 744366f6083SPeter Grehan 745366f6083SPeter Grehan fixed0 = rdmsr(MSR_VMX_CR4_FIXED0); 746366f6083SPeter Grehan fixed1 = rdmsr(MSR_VMX_CR4_FIXED1); 747366f6083SPeter Grehan cr4_ones_mask = fixed0 & fixed1; 748366f6083SPeter Grehan cr4_zeros_mask = ~fixed0 & ~fixed1; 749366f6083SPeter Grehan 75045e51299SNeel Natu vpid_init(); 75145e51299SNeel Natu 752366f6083SPeter Grehan /* enable VMX operation */ 753366f6083SPeter Grehan smp_rendezvous(NULL, vmx_enable, NULL, NULL); 754366f6083SPeter Grehan 7553565b59eSNeel Natu vmx_initialized = 1; 7563565b59eSNeel Natu 757366f6083SPeter Grehan return (0); 758366f6083SPeter Grehan } 759366f6083SPeter Grehan 760f7d47425SNeel Natu static void 761f7d47425SNeel Natu vmx_trigger_hostintr(int vector) 762f7d47425SNeel Natu { 763f7d47425SNeel Natu uintptr_t func; 764f7d47425SNeel Natu struct gate_descriptor *gd; 765f7d47425SNeel Natu 766f7d47425SNeel Natu gd = &idt[vector]; 767f7d47425SNeel Natu 768f7d47425SNeel Natu KASSERT(vector >= 32 && vector <= 255, ("vmx_trigger_hostintr: " 769f7d47425SNeel Natu "invalid vector %d", vector)); 770f7d47425SNeel Natu KASSERT(gd->gd_p == 1, ("gate descriptor for vector %d not present", 771f7d47425SNeel Natu vector)); 772f7d47425SNeel Natu KASSERT(gd->gd_type == SDT_SYSIGT, ("gate descriptor for vector %d " 773f7d47425SNeel Natu "has invalid type %d", vector, gd->gd_type)); 774f7d47425SNeel Natu KASSERT(gd->gd_dpl == SEL_KPL, ("gate descriptor for vector %d " 775f7d47425SNeel Natu "has invalid dpl %d", vector, gd->gd_dpl)); 776f7d47425SNeel Natu KASSERT(gd->gd_selector == GSEL(GCODE_SEL, SEL_KPL), ("gate descriptor " 777f7d47425SNeel Natu "for vector %d has invalid selector %d", vector, gd->gd_selector)); 778f7d47425SNeel Natu KASSERT(gd->gd_ist == 0, ("gate descriptor for vector %d has invalid " 779f7d47425SNeel Natu "IST %d", vector, gd->gd_ist)); 780f7d47425SNeel Natu 781f7d47425SNeel Natu func = ((long)gd->gd_hioffset << 16 | gd->gd_looffset); 782f7d47425SNeel Natu vmx_call_isr(func); 783f7d47425SNeel Natu } 784f7d47425SNeel Natu 785366f6083SPeter Grehan static int 786aaaa0656SPeter Grehan vmx_setup_cr_shadow(int which, struct vmcs *vmcs, uint32_t initial) 787366f6083SPeter Grehan { 78839c21c2dSNeel Natu int error, mask_ident, shadow_ident; 789aaaa0656SPeter Grehan uint64_t mask_value; 790366f6083SPeter Grehan 79139c21c2dSNeel Natu if (which != 0 && which != 4) 79239c21c2dSNeel Natu panic("vmx_setup_cr_shadow: unknown cr%d", which); 79339c21c2dSNeel Natu 79439c21c2dSNeel Natu if (which == 0) { 79539c21c2dSNeel Natu mask_ident = VMCS_CR0_MASK; 79639c21c2dSNeel Natu mask_value = cr0_ones_mask | cr0_zeros_mask; 79739c21c2dSNeel Natu shadow_ident = VMCS_CR0_SHADOW; 79839c21c2dSNeel Natu } else { 79939c21c2dSNeel Natu mask_ident = VMCS_CR4_MASK; 80039c21c2dSNeel Natu mask_value = cr4_ones_mask | cr4_zeros_mask; 80139c21c2dSNeel Natu shadow_ident = VMCS_CR4_SHADOW; 80239c21c2dSNeel Natu } 80339c21c2dSNeel Natu 804d3c11f40SPeter Grehan error = vmcs_setreg(vmcs, 0, VMCS_IDENT(mask_ident), mask_value); 805366f6083SPeter Grehan if (error) 806366f6083SPeter Grehan return (error); 807366f6083SPeter Grehan 808aaaa0656SPeter Grehan error = vmcs_setreg(vmcs, 0, VMCS_IDENT(shadow_ident), initial); 809366f6083SPeter Grehan if (error) 810366f6083SPeter Grehan return (error); 811366f6083SPeter Grehan 812366f6083SPeter Grehan return (0); 813366f6083SPeter Grehan } 814aaaa0656SPeter Grehan #define vmx_setup_cr0_shadow(vmcs,init) vmx_setup_cr_shadow(0, (vmcs), (init)) 815aaaa0656SPeter Grehan #define vmx_setup_cr4_shadow(vmcs,init) vmx_setup_cr_shadow(4, (vmcs), (init)) 816366f6083SPeter Grehan 817366f6083SPeter Grehan static void * 818318224bbSNeel Natu vmx_vminit(struct vm *vm, pmap_t pmap) 819366f6083SPeter Grehan { 82045e51299SNeel Natu uint16_t vpid[VM_MAXCPU]; 821366f6083SPeter Grehan int i, error, guest_msr_count; 822366f6083SPeter Grehan struct vmx *vmx; 823c847a506SNeel Natu struct vmcs *vmcs; 824366f6083SPeter Grehan 825366f6083SPeter Grehan vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO); 826366f6083SPeter Grehan if ((uintptr_t)vmx & PAGE_MASK) { 827366f6083SPeter Grehan panic("malloc of struct vmx not aligned on %d byte boundary", 828366f6083SPeter Grehan PAGE_SIZE); 829366f6083SPeter Grehan } 830366f6083SPeter Grehan vmx->vm = vm; 831366f6083SPeter Grehan 832318224bbSNeel Natu vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pml4)); 833318224bbSNeel Natu 834366f6083SPeter Grehan /* 835366f6083SPeter Grehan * Clean up EPTP-tagged guest physical and combined mappings 836366f6083SPeter Grehan * 837366f6083SPeter Grehan * VMX transitions are not required to invalidate any guest physical 838366f6083SPeter Grehan * mappings. So, it may be possible for stale guest physical mappings 839366f6083SPeter Grehan * to be present in the processor TLBs. 840366f6083SPeter Grehan * 841366f6083SPeter Grehan * Combined mappings for this EP4TA are also invalidated for all VPIDs. 842366f6083SPeter Grehan */ 843318224bbSNeel Natu ept_invalidate_mappings(vmx->eptp); 844366f6083SPeter Grehan 845366f6083SPeter Grehan msr_bitmap_initialize(vmx->msr_bitmap); 846366f6083SPeter Grehan 847366f6083SPeter Grehan /* 848366f6083SPeter Grehan * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE. 849366f6083SPeter Grehan * The guest FSBASE and GSBASE are saved and restored during 850366f6083SPeter Grehan * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are 851366f6083SPeter Grehan * always restored from the vmcs host state area on vm-exit. 852366f6083SPeter Grehan * 8531fb0ea3fSPeter Grehan * The SYSENTER_CS/ESP/EIP MSRs are identical to FS/GSBASE in 8541fb0ea3fSPeter Grehan * how they are saved/restored so can be directly accessed by the 8551fb0ea3fSPeter Grehan * guest. 8561fb0ea3fSPeter Grehan * 857366f6083SPeter Grehan * Guest KGSBASE is saved and restored in the guest MSR save area. 858366f6083SPeter Grehan * Host KGSBASE is restored before returning to userland from the pcb. 859366f6083SPeter Grehan * There will be a window of time when we are executing in the host 860366f6083SPeter Grehan * kernel context with a value of KGSBASE from the guest. This is ok 861366f6083SPeter Grehan * because the value of KGSBASE is inconsequential in kernel context. 862366f6083SPeter Grehan * 863366f6083SPeter Grehan * MSR_EFER is saved and restored in the guest VMCS area on a 864366f6083SPeter Grehan * VM exit and entry respectively. It is also restored from the 865366f6083SPeter Grehan * host VMCS area on a VM exit. 866366f6083SPeter Grehan */ 867366f6083SPeter Grehan if (guest_msr_rw(vmx, MSR_GSBASE) || 868366f6083SPeter Grehan guest_msr_rw(vmx, MSR_FSBASE) || 8691fb0ea3fSPeter Grehan guest_msr_rw(vmx, MSR_SYSENTER_CS_MSR) || 8701fb0ea3fSPeter Grehan guest_msr_rw(vmx, MSR_SYSENTER_ESP_MSR) || 8711fb0ea3fSPeter Grehan guest_msr_rw(vmx, MSR_SYSENTER_EIP_MSR) || 872366f6083SPeter Grehan guest_msr_rw(vmx, MSR_KGSBASE) || 873608f97c3SPeter Grehan guest_msr_rw(vmx, MSR_EFER)) 874366f6083SPeter Grehan panic("vmx_vminit: error setting guest msr access"); 875366f6083SPeter Grehan 876608f97c3SPeter Grehan /* 877608f97c3SPeter Grehan * MSR_PAT is saved and restored in the guest VMCS are on a VM exit 878608f97c3SPeter Grehan * and entry respectively. It is also restored from the host VMCS 879608f97c3SPeter Grehan * area on a VM exit. However, if running on a system with no 880608f97c3SPeter Grehan * MSR_PAT save/restore support, leave access disabled so accesses 881608f97c3SPeter Grehan * will be trapped. 882608f97c3SPeter Grehan */ 883608f97c3SPeter Grehan if (!vmx_no_patmsr && guest_msr_rw(vmx, MSR_PAT)) 884608f97c3SPeter Grehan panic("vmx_vminit: error setting guest pat msr access"); 885608f97c3SPeter Grehan 88645e51299SNeel Natu vpid_alloc(vpid, VM_MAXCPU); 88745e51299SNeel Natu 88888c4b8d1SNeel Natu if (virtual_interrupt_delivery) { 88988c4b8d1SNeel Natu error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE, 89088c4b8d1SNeel Natu APIC_ACCESS_ADDRESS); 89188c4b8d1SNeel Natu /* XXX this should really return an error to the caller */ 89288c4b8d1SNeel Natu KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error)); 89388c4b8d1SNeel Natu } 89488c4b8d1SNeel Natu 895366f6083SPeter Grehan for (i = 0; i < VM_MAXCPU; i++) { 896c847a506SNeel Natu vmcs = &vmx->vmcs[i]; 897c847a506SNeel Natu vmcs->identifier = vmx_revision(); 898c847a506SNeel Natu error = vmclear(vmcs); 899366f6083SPeter Grehan if (error != 0) { 900366f6083SPeter Grehan panic("vmx_vminit: vmclear error %d on vcpu %d\n", 901366f6083SPeter Grehan error, i); 902366f6083SPeter Grehan } 903366f6083SPeter Grehan 904c847a506SNeel Natu error = vmcs_init(vmcs); 905c847a506SNeel Natu KASSERT(error == 0, ("vmcs_init error %d", error)); 906366f6083SPeter Grehan 907c847a506SNeel Natu VMPTRLD(vmcs); 908c847a506SNeel Natu error = 0; 909c847a506SNeel Natu error += vmwrite(VMCS_HOST_RSP, (u_long)&vmx->ctx[i]); 910c847a506SNeel Natu error += vmwrite(VMCS_EPTP, vmx->eptp); 911c847a506SNeel Natu error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls); 912c847a506SNeel Natu error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls); 913c847a506SNeel Natu error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2); 914c847a506SNeel Natu error += vmwrite(VMCS_EXIT_CTLS, exit_ctls); 915c847a506SNeel Natu error += vmwrite(VMCS_ENTRY_CTLS, entry_ctls); 916c847a506SNeel Natu error += vmwrite(VMCS_MSR_BITMAP, vtophys(vmx->msr_bitmap)); 917c847a506SNeel Natu error += vmwrite(VMCS_VPID, vpid[i]); 91888c4b8d1SNeel Natu if (virtual_interrupt_delivery) { 91988c4b8d1SNeel Natu error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS); 92088c4b8d1SNeel Natu error += vmwrite(VMCS_VIRTUAL_APIC, 92188c4b8d1SNeel Natu vtophys(&vmx->apic_page[i])); 92288c4b8d1SNeel Natu error += vmwrite(VMCS_EOI_EXIT0, 0); 92388c4b8d1SNeel Natu error += vmwrite(VMCS_EOI_EXIT1, 0); 92488c4b8d1SNeel Natu error += vmwrite(VMCS_EOI_EXIT2, 0); 92588c4b8d1SNeel Natu error += vmwrite(VMCS_EOI_EXIT3, 0); 92688c4b8d1SNeel Natu } 927176666c2SNeel Natu if (posted_interrupts) { 928176666c2SNeel Natu error += vmwrite(VMCS_PIR_VECTOR, pirvec); 929176666c2SNeel Natu error += vmwrite(VMCS_PIR_DESC, 930176666c2SNeel Natu vtophys(&vmx->pir_desc[i])); 931176666c2SNeel Natu } 932c847a506SNeel Natu VMCLEAR(vmcs); 933c847a506SNeel Natu KASSERT(error == 0, ("vmx_vminit: error customizing the vmcs")); 934366f6083SPeter Grehan 935366f6083SPeter Grehan vmx->cap[i].set = 0; 936366f6083SPeter Grehan vmx->cap[i].proc_ctls = procbased_ctls; 93749cc03daSNeel Natu vmx->cap[i].proc_ctls2 = procbased_ctls2; 938366f6083SPeter Grehan 939366f6083SPeter Grehan vmx->state[i].lastcpu = -1; 94045e51299SNeel Natu vmx->state[i].vpid = vpid[i]; 941366f6083SPeter Grehan 942366f6083SPeter Grehan msr_save_area_init(vmx->guest_msrs[i], &guest_msr_count); 943366f6083SPeter Grehan 944c847a506SNeel Natu error = vmcs_set_msr_save(vmcs, vtophys(vmx->guest_msrs[i]), 945366f6083SPeter Grehan guest_msr_count); 946366f6083SPeter Grehan if (error != 0) 947366f6083SPeter Grehan panic("vmcs_set_msr_save error %d", error); 948366f6083SPeter Grehan 949aaaa0656SPeter Grehan /* 950aaaa0656SPeter Grehan * Set up the CR0/4 shadows, and init the read shadow 951aaaa0656SPeter Grehan * to the power-on register value from the Intel Sys Arch. 952aaaa0656SPeter Grehan * CR0 - 0x60000010 953aaaa0656SPeter Grehan * CR4 - 0 954aaaa0656SPeter Grehan */ 955c847a506SNeel Natu error = vmx_setup_cr0_shadow(vmcs, 0x60000010); 95639c21c2dSNeel Natu if (error != 0) 95739c21c2dSNeel Natu panic("vmx_setup_cr0_shadow %d", error); 95839c21c2dSNeel Natu 959c847a506SNeel Natu error = vmx_setup_cr4_shadow(vmcs, 0); 96039c21c2dSNeel Natu if (error != 0) 96139c21c2dSNeel Natu panic("vmx_setup_cr4_shadow %d", error); 962318224bbSNeel Natu 963318224bbSNeel Natu vmx->ctx[i].pmap = pmap; 964366f6083SPeter Grehan } 965366f6083SPeter Grehan 966366f6083SPeter Grehan return (vmx); 967366f6083SPeter Grehan } 968366f6083SPeter Grehan 969366f6083SPeter Grehan static int 970a2da7af6SNeel Natu vmx_handle_cpuid(struct vm *vm, int vcpu, struct vmxctx *vmxctx) 971366f6083SPeter Grehan { 972366f6083SPeter Grehan int handled, func; 973366f6083SPeter Grehan 974366f6083SPeter Grehan func = vmxctx->guest_rax; 975366f6083SPeter Grehan 976a2da7af6SNeel Natu handled = x86_emulate_cpuid(vm, vcpu, 977a2da7af6SNeel Natu (uint32_t*)(&vmxctx->guest_rax), 978a2da7af6SNeel Natu (uint32_t*)(&vmxctx->guest_rbx), 979a2da7af6SNeel Natu (uint32_t*)(&vmxctx->guest_rcx), 980a2da7af6SNeel Natu (uint32_t*)(&vmxctx->guest_rdx)); 981366f6083SPeter Grehan return (handled); 982366f6083SPeter Grehan } 983366f6083SPeter Grehan 984366f6083SPeter Grehan static __inline void 985366f6083SPeter Grehan vmx_run_trace(struct vmx *vmx, int vcpu) 986366f6083SPeter Grehan { 987366f6083SPeter Grehan #ifdef KTR 988513c8d33SNeel Natu VCPU_CTR1(vmx->vm, vcpu, "Resume execution at %#lx", vmcs_guest_rip()); 989366f6083SPeter Grehan #endif 990366f6083SPeter Grehan } 991366f6083SPeter Grehan 992366f6083SPeter Grehan static __inline void 993366f6083SPeter Grehan vmx_exit_trace(struct vmx *vmx, int vcpu, uint64_t rip, uint32_t exit_reason, 994eeefa4e4SNeel Natu int handled) 995366f6083SPeter Grehan { 996366f6083SPeter Grehan #ifdef KTR 997513c8d33SNeel Natu VCPU_CTR3(vmx->vm, vcpu, "%s %s vmexit at 0x%0lx", 998366f6083SPeter Grehan handled ? "handled" : "unhandled", 999366f6083SPeter Grehan exit_reason_to_str(exit_reason), rip); 1000eeefa4e4SNeel Natu #endif 1001eeefa4e4SNeel Natu } 1002366f6083SPeter Grehan 1003eeefa4e4SNeel Natu static __inline void 1004eeefa4e4SNeel Natu vmx_astpending_trace(struct vmx *vmx, int vcpu, uint64_t rip) 1005eeefa4e4SNeel Natu { 1006eeefa4e4SNeel Natu #ifdef KTR 1007513c8d33SNeel Natu VCPU_CTR1(vmx->vm, vcpu, "astpending vmexit at 0x%0lx", rip); 1008366f6083SPeter Grehan #endif 1009366f6083SPeter Grehan } 1010366f6083SPeter Grehan 1011953c2c47SNeel Natu static VMM_STAT_INTEL(VCPU_INVVPID_SAVED, "Number of vpid invalidations saved"); 1012953c2c47SNeel Natu 10133de83862SNeel Natu static void 1014953c2c47SNeel Natu vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu, pmap_t pmap) 1015366f6083SPeter Grehan { 1016366f6083SPeter Grehan struct vmxstate *vmxstate; 1017953c2c47SNeel Natu struct invvpid_desc invvpid_desc; 1018366f6083SPeter Grehan 1019366f6083SPeter Grehan vmxstate = &vmx->state[vcpu]; 1020953c2c47SNeel Natu if (vmxstate->lastcpu == curcpu) 10213de83862SNeel Natu return; 1022366f6083SPeter Grehan 1023953c2c47SNeel Natu vmxstate->lastcpu = curcpu; 1024953c2c47SNeel Natu 1025366f6083SPeter Grehan vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1); 1026366f6083SPeter Grehan 10273de83862SNeel Natu vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase()); 10283de83862SNeel Natu vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase()); 10293de83862SNeel Natu vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase()); 1030366f6083SPeter Grehan 1031366f6083SPeter Grehan /* 1032366f6083SPeter Grehan * If we are using VPIDs then invalidate all mappings tagged with 'vpid' 1033366f6083SPeter Grehan * 1034366f6083SPeter Grehan * We do this because this vcpu was executing on a different host 1035366f6083SPeter Grehan * cpu when it last ran. We do not track whether it invalidated 1036366f6083SPeter Grehan * mappings associated with its 'vpid' during that run. So we must 1037366f6083SPeter Grehan * assume that the mappings associated with 'vpid' on 'curcpu' are 1038366f6083SPeter Grehan * stale and invalidate them. 1039366f6083SPeter Grehan * 1040366f6083SPeter Grehan * Note that we incur this penalty only when the scheduler chooses to 1041366f6083SPeter Grehan * move the thread associated with this vcpu between host cpus. 1042366f6083SPeter Grehan * 1043366f6083SPeter Grehan * Note also that this will invalidate mappings tagged with 'vpid' 1044366f6083SPeter Grehan * for "all" EP4TAs. 1045366f6083SPeter Grehan */ 1046366f6083SPeter Grehan if (vmxstate->vpid != 0) { 1047953c2c47SNeel Natu if (pmap->pm_eptgen == vmx->eptgen[curcpu]) { 1048953c2c47SNeel Natu invvpid_desc._res1 = 0; 1049953c2c47SNeel Natu invvpid_desc._res2 = 0; 1050366f6083SPeter Grehan invvpid_desc.vpid = vmxstate->vpid; 10510e30c5c0SWarner Losh invvpid_desc.linear_addr = 0; 1052366f6083SPeter Grehan invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc); 1053953c2c47SNeel Natu } else { 1054953c2c47SNeel Natu /* 1055953c2c47SNeel Natu * The invvpid can be skipped if an invept is going to 1056953c2c47SNeel Natu * be performed before entering the guest. The invept 1057953c2c47SNeel Natu * will invalidate combined mappings tagged with 1058953c2c47SNeel Natu * 'vmx->eptp' for all vpids. 1059953c2c47SNeel Natu */ 1060953c2c47SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_SAVED, 1); 1061953c2c47SNeel Natu } 1062366f6083SPeter Grehan } 1063366f6083SPeter Grehan } 1064366f6083SPeter Grehan 1065366f6083SPeter Grehan /* 1066366f6083SPeter Grehan * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set. 1067366f6083SPeter Grehan */ 1068366f6083SPeter Grehan CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0); 1069366f6083SPeter Grehan 1070366f6083SPeter Grehan static void __inline 1071366f6083SPeter Grehan vmx_set_int_window_exiting(struct vmx *vmx, int vcpu) 1072366f6083SPeter Grehan { 1073366f6083SPeter Grehan 107448b2d828SNeel Natu if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) { 1075366f6083SPeter Grehan vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING; 10763de83862SNeel Natu vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 107748b2d828SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting"); 107848b2d828SNeel Natu } 1079366f6083SPeter Grehan } 1080366f6083SPeter Grehan 1081366f6083SPeter Grehan static void __inline 1082366f6083SPeter Grehan vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu) 1083366f6083SPeter Grehan { 1084366f6083SPeter Grehan 108548b2d828SNeel Natu KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0, 108648b2d828SNeel Natu ("intr_window_exiting not set: %#x", vmx->cap[vcpu].proc_ctls)); 1087366f6083SPeter Grehan vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING; 10883de83862SNeel Natu vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 108948b2d828SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting"); 1090366f6083SPeter Grehan } 1091366f6083SPeter Grehan 1092366f6083SPeter Grehan static void __inline 1093366f6083SPeter Grehan vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu) 1094366f6083SPeter Grehan { 1095366f6083SPeter Grehan 109648b2d828SNeel Natu if ((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) { 1097366f6083SPeter Grehan vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING; 10983de83862SNeel Natu vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 109948b2d828SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting"); 110048b2d828SNeel Natu } 1101366f6083SPeter Grehan } 1102366f6083SPeter Grehan 1103366f6083SPeter Grehan static void __inline 1104366f6083SPeter Grehan vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu) 1105366f6083SPeter Grehan { 1106366f6083SPeter Grehan 110748b2d828SNeel Natu KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0, 110848b2d828SNeel Natu ("nmi_window_exiting not set %#x", vmx->cap[vcpu].proc_ctls)); 1109366f6083SPeter Grehan vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING; 11103de83862SNeel Natu vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 111148b2d828SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting"); 1112366f6083SPeter Grehan } 1113366f6083SPeter Grehan 111448b2d828SNeel Natu #define NMI_BLOCKING (VMCS_INTERRUPTIBILITY_NMI_BLOCKING | \ 111548b2d828SNeel Natu VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING) 111648b2d828SNeel Natu #define HWINTR_BLOCKING (VMCS_INTERRUPTIBILITY_STI_BLOCKING | \ 111748b2d828SNeel Natu VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING) 111848b2d828SNeel Natu 111948b2d828SNeel Natu static void 1120366f6083SPeter Grehan vmx_inject_nmi(struct vmx *vmx, int vcpu) 1121366f6083SPeter Grehan { 112248b2d828SNeel Natu uint32_t gi, info; 1123366f6083SPeter Grehan 112448b2d828SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 112548b2d828SNeel Natu KASSERT((gi & NMI_BLOCKING) == 0, ("vmx_inject_nmi: invalid guest " 112648b2d828SNeel Natu "interruptibility-state %#x", gi)); 1127366f6083SPeter Grehan 112848b2d828SNeel Natu info = vmcs_read(VMCS_ENTRY_INTR_INFO); 112948b2d828SNeel Natu KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_nmi: invalid " 113048b2d828SNeel Natu "VM-entry interruption information %#x", info)); 1131366f6083SPeter Grehan 1132366f6083SPeter Grehan /* 1133366f6083SPeter Grehan * Inject the virtual NMI. The vector must be the NMI IDT entry 1134366f6083SPeter Grehan * or the VMCS entry check will fail. 1135366f6083SPeter Grehan */ 113648b2d828SNeel Natu info = IDT_NMI | VMCS_INTR_T_NMI | VMCS_INTR_VALID; 11373de83862SNeel Natu vmcs_write(VMCS_ENTRY_INTR_INFO, info); 1138366f6083SPeter Grehan 1139513c8d33SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Injecting vNMI"); 1140366f6083SPeter Grehan 1141366f6083SPeter Grehan /* Clear the request */ 1142f352ff0cSNeel Natu vm_nmi_clear(vmx->vm, vcpu); 1143366f6083SPeter Grehan } 1144366f6083SPeter Grehan 1145366f6083SPeter Grehan static void 1146de5ea6b6SNeel Natu vmx_inject_interrupts(struct vmx *vmx, int vcpu, struct vlapic *vlapic) 1147366f6083SPeter Grehan { 1148dc506506SNeel Natu struct vm_exception exc; 11490775fbb4STycho Nightingale int vector, need_nmi_exiting, extint_pending; 115048b2d828SNeel Natu uint64_t rflags; 115148b2d828SNeel Natu uint32_t gi, info; 1152366f6083SPeter Grehan 1153dc506506SNeel Natu if (vm_exception_pending(vmx->vm, vcpu, &exc)) { 1154dc506506SNeel Natu KASSERT(exc.vector >= 0 && exc.vector < 32, 1155dc506506SNeel Natu ("%s: invalid exception vector %d", __func__, exc.vector)); 1156dc506506SNeel Natu 1157dc506506SNeel Natu info = vmcs_read(VMCS_ENTRY_INTR_INFO); 1158dc506506SNeel Natu KASSERT((info & VMCS_INTR_VALID) == 0, ("%s: cannot inject " 1159dc506506SNeel Natu "pending exception %d: %#x", __func__, exc.vector, info)); 1160dc506506SNeel Natu 1161dc506506SNeel Natu info = exc.vector | VMCS_INTR_T_HWEXCEPTION | VMCS_INTR_VALID; 1162dc506506SNeel Natu if (exc.error_code_valid) { 1163dc506506SNeel Natu info |= VMCS_INTR_DEL_ERRCODE; 1164dc506506SNeel Natu vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR, exc.error_code); 1165dc506506SNeel Natu } 1166dc506506SNeel Natu vmcs_write(VMCS_ENTRY_INTR_INFO, info); 1167dc506506SNeel Natu } 1168dc506506SNeel Natu 116948b2d828SNeel Natu if (vm_nmi_pending(vmx->vm, vcpu)) { 1170366f6083SPeter Grehan /* 117148b2d828SNeel Natu * If there are no conditions blocking NMI injection then 117248b2d828SNeel Natu * inject it directly here otherwise enable "NMI window 117348b2d828SNeel Natu * exiting" to inject it as soon as we can. 1174eeefa4e4SNeel Natu * 117548b2d828SNeel Natu * We also check for STI_BLOCKING because some implementations 117648b2d828SNeel Natu * don't allow NMI injection in this case. If we are running 117748b2d828SNeel Natu * on a processor that doesn't have this restriction it will 117848b2d828SNeel Natu * immediately exit and the NMI will be injected in the 117948b2d828SNeel Natu * "NMI window exiting" handler. 1180366f6083SPeter Grehan */ 118148b2d828SNeel Natu need_nmi_exiting = 1; 118248b2d828SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 118348b2d828SNeel Natu if ((gi & (HWINTR_BLOCKING | NMI_BLOCKING)) == 0) { 11843de83862SNeel Natu info = vmcs_read(VMCS_ENTRY_INTR_INFO); 118548b2d828SNeel Natu if ((info & VMCS_INTR_VALID) == 0) { 118648b2d828SNeel Natu vmx_inject_nmi(vmx, vcpu); 118748b2d828SNeel Natu need_nmi_exiting = 0; 118848b2d828SNeel Natu } else { 118948b2d828SNeel Natu VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI " 119048b2d828SNeel Natu "due to VM-entry intr info %#x", info); 119148b2d828SNeel Natu } 119248b2d828SNeel Natu } else { 119348b2d828SNeel Natu VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI due to " 119448b2d828SNeel Natu "Guest Interruptibility-state %#x", gi); 119548b2d828SNeel Natu } 1196eeefa4e4SNeel Natu 119748b2d828SNeel Natu if (need_nmi_exiting) 119848b2d828SNeel Natu vmx_set_nmi_window_exiting(vmx, vcpu); 119948b2d828SNeel Natu } 1200366f6083SPeter Grehan 12010775fbb4STycho Nightingale extint_pending = vm_extint_pending(vmx->vm, vcpu); 12020775fbb4STycho Nightingale 12030775fbb4STycho Nightingale if (!extint_pending && virtual_interrupt_delivery) { 120488c4b8d1SNeel Natu vmx_inject_pir(vlapic); 120588c4b8d1SNeel Natu return; 120688c4b8d1SNeel Natu } 120788c4b8d1SNeel Natu 120848b2d828SNeel Natu /* 120936736912SNeel Natu * If interrupt-window exiting is already in effect then don't bother 121036736912SNeel Natu * checking for pending interrupts. This is just an optimization and 121136736912SNeel Natu * not needed for correctness. 121248b2d828SNeel Natu */ 121336736912SNeel Natu if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) { 121436736912SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Skip interrupt injection due to " 121536736912SNeel Natu "pending int_window_exiting"); 121648b2d828SNeel Natu return; 121736736912SNeel Natu } 121848b2d828SNeel Natu 12190775fbb4STycho Nightingale if (!extint_pending) { 1220366f6083SPeter Grehan /* Ask the local apic for a vector to inject */ 12214d1e82a8SNeel Natu if (!vlapic_pending_intr(vlapic, &vector)) 1222366f6083SPeter Grehan return; 12230775fbb4STycho Nightingale } else { 12240775fbb4STycho Nightingale /* Ask the legacy pic for a vector to inject */ 12250775fbb4STycho Nightingale vatpic_pending_intr(vmx->vm, &vector); 12260775fbb4STycho Nightingale } 1227366f6083SPeter Grehan 122848b2d828SNeel Natu KASSERT(vector >= 32 && vector <= 255, ("invalid vector %d", vector)); 1229366f6083SPeter Grehan 1230366f6083SPeter Grehan /* Check RFLAGS.IF and the interruptibility state of the guest */ 12313de83862SNeel Natu rflags = vmcs_read(VMCS_GUEST_RFLAGS); 123236736912SNeel Natu if ((rflags & PSL_I) == 0) { 123336736912SNeel Natu VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to " 123436736912SNeel Natu "rflags %#lx", vector, rflags); 1235366f6083SPeter Grehan goto cantinject; 123636736912SNeel Natu } 1237366f6083SPeter Grehan 123848b2d828SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 123936736912SNeel Natu if (gi & HWINTR_BLOCKING) { 124036736912SNeel Natu VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to " 124136736912SNeel Natu "Guest Interruptibility-state %#x", vector, gi); 1242366f6083SPeter Grehan goto cantinject; 124336736912SNeel Natu } 124436736912SNeel Natu 124536736912SNeel Natu info = vmcs_read(VMCS_ENTRY_INTR_INFO); 124636736912SNeel Natu if (info & VMCS_INTR_VALID) { 124736736912SNeel Natu /* 124836736912SNeel Natu * This is expected and could happen for multiple reasons: 124936736912SNeel Natu * - A vectoring VM-entry was aborted due to astpending 125036736912SNeel Natu * - A VM-exit happened during event injection. 1251dc506506SNeel Natu * - An exception was injected above. 125236736912SNeel Natu * - An NMI was injected above or after "NMI window exiting" 125336736912SNeel Natu */ 125436736912SNeel Natu VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to " 125536736912SNeel Natu "VM-entry intr info %#x", vector, info); 125636736912SNeel Natu goto cantinject; 125736736912SNeel Natu } 1258366f6083SPeter Grehan 1259366f6083SPeter Grehan /* Inject the interrupt */ 1260160471d2SNeel Natu info = VMCS_INTR_T_HWINTR | VMCS_INTR_VALID; 1261366f6083SPeter Grehan info |= vector; 12623de83862SNeel Natu vmcs_write(VMCS_ENTRY_INTR_INFO, info); 1263366f6083SPeter Grehan 12640775fbb4STycho Nightingale if (!extint_pending) { 1265366f6083SPeter Grehan /* Update the Local APIC ISR */ 1266de5ea6b6SNeel Natu vlapic_intr_accepted(vlapic, vector); 12670775fbb4STycho Nightingale } else { 12680775fbb4STycho Nightingale vm_extint_clear(vmx->vm, vcpu); 12690775fbb4STycho Nightingale vatpic_intr_accepted(vmx->vm, vector); 12700775fbb4STycho Nightingale 12710775fbb4STycho Nightingale /* 12720775fbb4STycho Nightingale * After we accepted the current ExtINT the PIC may 12730775fbb4STycho Nightingale * have posted another one. If that is the case, set 12740775fbb4STycho Nightingale * the Interrupt Window Exiting execution control so 12750775fbb4STycho Nightingale * we can inject that one too. 12760775fbb4STycho Nightingale */ 12770775fbb4STycho Nightingale if (vm_extint_pending(vmx->vm, vcpu)) 12780775fbb4STycho Nightingale vmx_set_int_window_exiting(vmx, vcpu); 12790775fbb4STycho Nightingale } 1280366f6083SPeter Grehan 1281513c8d33SNeel Natu VCPU_CTR1(vmx->vm, vcpu, "Injecting hwintr at vector %d", vector); 1282366f6083SPeter Grehan 1283366f6083SPeter Grehan return; 1284366f6083SPeter Grehan 1285366f6083SPeter Grehan cantinject: 1286366f6083SPeter Grehan /* 1287366f6083SPeter Grehan * Set the Interrupt Window Exiting execution control so we can inject 1288366f6083SPeter Grehan * the interrupt as soon as blocking condition goes away. 1289366f6083SPeter Grehan */ 1290366f6083SPeter Grehan vmx_set_int_window_exiting(vmx, vcpu); 1291366f6083SPeter Grehan } 1292366f6083SPeter Grehan 1293e5a1d950SNeel Natu /* 1294e5a1d950SNeel Natu * If the Virtual NMIs execution control is '1' then the logical processor 1295e5a1d950SNeel Natu * tracks virtual-NMI blocking in the Guest Interruptibility-state field of 1296e5a1d950SNeel Natu * the VMCS. An IRET instruction in VMX non-root operation will remove any 1297e5a1d950SNeel Natu * virtual-NMI blocking. 1298e5a1d950SNeel Natu * 1299e5a1d950SNeel Natu * This unblocking occurs even if the IRET causes a fault. In this case the 1300e5a1d950SNeel Natu * hypervisor needs to restore virtual-NMI blocking before resuming the guest. 1301e5a1d950SNeel Natu */ 1302e5a1d950SNeel Natu static void 1303e5a1d950SNeel Natu vmx_restore_nmi_blocking(struct vmx *vmx, int vcpuid) 1304e5a1d950SNeel Natu { 1305e5a1d950SNeel Natu uint32_t gi; 1306e5a1d950SNeel Natu 1307e5a1d950SNeel Natu VCPU_CTR0(vmx->vm, vcpuid, "Restore Virtual-NMI blocking"); 1308e5a1d950SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 1309e5a1d950SNeel Natu gi |= VMCS_INTERRUPTIBILITY_NMI_BLOCKING; 1310e5a1d950SNeel Natu vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi); 1311e5a1d950SNeel Natu } 1312e5a1d950SNeel Natu 1313e5a1d950SNeel Natu static void 1314e5a1d950SNeel Natu vmx_clear_nmi_blocking(struct vmx *vmx, int vcpuid) 1315e5a1d950SNeel Natu { 1316e5a1d950SNeel Natu uint32_t gi; 1317e5a1d950SNeel Natu 1318e5a1d950SNeel Natu VCPU_CTR0(vmx->vm, vcpuid, "Clear Virtual-NMI blocking"); 1319e5a1d950SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 1320e5a1d950SNeel Natu gi &= ~VMCS_INTERRUPTIBILITY_NMI_BLOCKING; 1321e5a1d950SNeel Natu vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi); 1322e5a1d950SNeel Natu } 1323e5a1d950SNeel Natu 1324366f6083SPeter Grehan static int 1325a0efd3fbSJohn Baldwin vmx_emulate_xsetbv(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) 1326abb023fbSJohn Baldwin { 1327abb023fbSJohn Baldwin struct vmxctx *vmxctx; 1328abb023fbSJohn Baldwin uint64_t xcrval; 1329abb023fbSJohn Baldwin const struct xsave_limits *limits; 1330abb023fbSJohn Baldwin 1331abb023fbSJohn Baldwin vmxctx = &vmx->ctx[vcpu]; 1332abb023fbSJohn Baldwin limits = vmm_get_xsave_limits(); 1333abb023fbSJohn Baldwin 1334a0efd3fbSJohn Baldwin /* 1335a0efd3fbSJohn Baldwin * Note that the processor raises a GP# fault on its own if 1336a0efd3fbSJohn Baldwin * xsetbv is executed for CPL != 0, so we do not have to 1337a0efd3fbSJohn Baldwin * emulate that fault here. 1338a0efd3fbSJohn Baldwin */ 1339a0efd3fbSJohn Baldwin 1340a0efd3fbSJohn Baldwin /* Only xcr0 is supported. */ 1341a0efd3fbSJohn Baldwin if (vmxctx->guest_rcx != 0) { 1342dc506506SNeel Natu vm_inject_gp(vmx->vm, vcpu); 1343a0efd3fbSJohn Baldwin return (HANDLED); 1344a0efd3fbSJohn Baldwin } 1345a0efd3fbSJohn Baldwin 1346a0efd3fbSJohn Baldwin /* We only handle xcr0 if both the host and guest have XSAVE enabled. */ 1347a0efd3fbSJohn Baldwin if (!limits->xsave_enabled || !(vmcs_read(VMCS_GUEST_CR4) & CR4_XSAVE)) { 1348dc506506SNeel Natu vm_inject_ud(vmx->vm, vcpu); 1349a0efd3fbSJohn Baldwin return (HANDLED); 1350a0efd3fbSJohn Baldwin } 1351abb023fbSJohn Baldwin 1352abb023fbSJohn Baldwin xcrval = vmxctx->guest_rdx << 32 | (vmxctx->guest_rax & 0xffffffff); 1353a0efd3fbSJohn Baldwin if ((xcrval & ~limits->xcr0_allowed) != 0) { 1354dc506506SNeel Natu vm_inject_gp(vmx->vm, vcpu); 1355a0efd3fbSJohn Baldwin return (HANDLED); 1356a0efd3fbSJohn Baldwin } 1357abb023fbSJohn Baldwin 1358a0efd3fbSJohn Baldwin if (!(xcrval & XFEATURE_ENABLED_X87)) { 1359dc506506SNeel Natu vm_inject_gp(vmx->vm, vcpu); 1360a0efd3fbSJohn Baldwin return (HANDLED); 1361a0efd3fbSJohn Baldwin } 1362abb023fbSJohn Baldwin 1363abb023fbSJohn Baldwin if ((xcrval & (XFEATURE_ENABLED_AVX | XFEATURE_ENABLED_SSE)) == 1364a0efd3fbSJohn Baldwin XFEATURE_ENABLED_AVX) { 1365dc506506SNeel Natu vm_inject_gp(vmx->vm, vcpu); 1366a0efd3fbSJohn Baldwin return (HANDLED); 1367a0efd3fbSJohn Baldwin } 1368abb023fbSJohn Baldwin 1369abb023fbSJohn Baldwin /* 1370abb023fbSJohn Baldwin * This runs "inside" vmrun() with the guest's FPU state, so 1371abb023fbSJohn Baldwin * modifying xcr0 directly modifies the guest's xcr0, not the 1372abb023fbSJohn Baldwin * host's. 1373abb023fbSJohn Baldwin */ 1374abb023fbSJohn Baldwin load_xcr(0, xcrval); 1375abb023fbSJohn Baldwin return (HANDLED); 1376abb023fbSJohn Baldwin } 1377abb023fbSJohn Baldwin 1378abb023fbSJohn Baldwin static int 1379366f6083SPeter Grehan vmx_emulate_cr_access(struct vmx *vmx, int vcpu, uint64_t exitqual) 1380366f6083SPeter Grehan { 13813de83862SNeel Natu int cr, vmcs_guest_cr, vmcs_shadow_cr; 138280a902efSPeter Grehan uint64_t crval, regval, ones_mask, zeros_mask; 1383366f6083SPeter Grehan const struct vmxctx *vmxctx; 1384366f6083SPeter Grehan 138539c21c2dSNeel Natu /* We only handle mov to %cr0 or %cr4 at this time */ 138639c21c2dSNeel Natu if ((exitqual & 0xf0) != 0x00) 138739c21c2dSNeel Natu return (UNHANDLED); 138839c21c2dSNeel Natu 138939c21c2dSNeel Natu cr = exitqual & 0xf; 139039c21c2dSNeel Natu if (cr != 0 && cr != 4) 1391366f6083SPeter Grehan return (UNHANDLED); 1392366f6083SPeter Grehan 13936f0c167fSDimitry Andric regval = 0; /* silence gcc */ 1394366f6083SPeter Grehan vmxctx = &vmx->ctx[vcpu]; 1395366f6083SPeter Grehan 1396366f6083SPeter Grehan /* 13973de83862SNeel Natu * We must use vmcs_write() directly here because vmcs_setreg() will 1398366f6083SPeter Grehan * call vmclear(vmcs) as a side-effect which we certainly don't want. 1399366f6083SPeter Grehan */ 1400366f6083SPeter Grehan switch ((exitqual >> 8) & 0xf) { 1401366f6083SPeter Grehan case 0: 1402366f6083SPeter Grehan regval = vmxctx->guest_rax; 1403366f6083SPeter Grehan break; 1404366f6083SPeter Grehan case 1: 1405366f6083SPeter Grehan regval = vmxctx->guest_rcx; 1406366f6083SPeter Grehan break; 1407366f6083SPeter Grehan case 2: 1408366f6083SPeter Grehan regval = vmxctx->guest_rdx; 1409366f6083SPeter Grehan break; 1410366f6083SPeter Grehan case 3: 1411366f6083SPeter Grehan regval = vmxctx->guest_rbx; 1412366f6083SPeter Grehan break; 1413366f6083SPeter Grehan case 4: 14143de83862SNeel Natu regval = vmcs_read(VMCS_GUEST_RSP); 1415366f6083SPeter Grehan break; 1416366f6083SPeter Grehan case 5: 1417366f6083SPeter Grehan regval = vmxctx->guest_rbp; 1418366f6083SPeter Grehan break; 1419366f6083SPeter Grehan case 6: 1420366f6083SPeter Grehan regval = vmxctx->guest_rsi; 1421366f6083SPeter Grehan break; 1422366f6083SPeter Grehan case 7: 1423366f6083SPeter Grehan regval = vmxctx->guest_rdi; 1424366f6083SPeter Grehan break; 1425366f6083SPeter Grehan case 8: 1426366f6083SPeter Grehan regval = vmxctx->guest_r8; 1427366f6083SPeter Grehan break; 1428366f6083SPeter Grehan case 9: 1429366f6083SPeter Grehan regval = vmxctx->guest_r9; 1430366f6083SPeter Grehan break; 1431366f6083SPeter Grehan case 10: 1432366f6083SPeter Grehan regval = vmxctx->guest_r10; 1433366f6083SPeter Grehan break; 1434366f6083SPeter Grehan case 11: 1435366f6083SPeter Grehan regval = vmxctx->guest_r11; 1436366f6083SPeter Grehan break; 1437366f6083SPeter Grehan case 12: 1438366f6083SPeter Grehan regval = vmxctx->guest_r12; 1439366f6083SPeter Grehan break; 1440366f6083SPeter Grehan case 13: 1441366f6083SPeter Grehan regval = vmxctx->guest_r13; 1442366f6083SPeter Grehan break; 1443366f6083SPeter Grehan case 14: 1444366f6083SPeter Grehan regval = vmxctx->guest_r14; 1445366f6083SPeter Grehan break; 1446366f6083SPeter Grehan case 15: 1447366f6083SPeter Grehan regval = vmxctx->guest_r15; 1448366f6083SPeter Grehan break; 1449366f6083SPeter Grehan } 1450366f6083SPeter Grehan 145139c21c2dSNeel Natu if (cr == 0) { 145239c21c2dSNeel Natu ones_mask = cr0_ones_mask; 145339c21c2dSNeel Natu zeros_mask = cr0_zeros_mask; 145439c21c2dSNeel Natu vmcs_guest_cr = VMCS_GUEST_CR0; 1455aaaa0656SPeter Grehan vmcs_shadow_cr = VMCS_CR0_SHADOW; 145639c21c2dSNeel Natu } else { 145739c21c2dSNeel Natu ones_mask = cr4_ones_mask; 145839c21c2dSNeel Natu zeros_mask = cr4_zeros_mask; 145939c21c2dSNeel Natu vmcs_guest_cr = VMCS_GUEST_CR4; 1460aaaa0656SPeter Grehan vmcs_shadow_cr = VMCS_CR4_SHADOW; 146139c21c2dSNeel Natu } 14623de83862SNeel Natu vmcs_write(vmcs_shadow_cr, regval); 1463aaaa0656SPeter Grehan 146480a902efSPeter Grehan crval = regval | ones_mask; 146580a902efSPeter Grehan crval &= ~zeros_mask; 14663de83862SNeel Natu vmcs_write(vmcs_guest_cr, crval); 1467366f6083SPeter Grehan 146880a902efSPeter Grehan if (cr == 0 && regval & CR0_PG) { 146980a902efSPeter Grehan uint64_t efer, entry_ctls; 147080a902efSPeter Grehan 147180a902efSPeter Grehan /* 147280a902efSPeter Grehan * If CR0.PG is 1 and EFER.LME is 1 then EFER.LMA and 147380a902efSPeter Grehan * the "IA-32e mode guest" bit in VM-entry control must be 147480a902efSPeter Grehan * equal. 147580a902efSPeter Grehan */ 14763de83862SNeel Natu efer = vmcs_read(VMCS_GUEST_IA32_EFER); 147780a902efSPeter Grehan if (efer & EFER_LME) { 147880a902efSPeter Grehan efer |= EFER_LMA; 14793de83862SNeel Natu vmcs_write(VMCS_GUEST_IA32_EFER, efer); 14803de83862SNeel Natu entry_ctls = vmcs_read(VMCS_ENTRY_CTLS); 148180a902efSPeter Grehan entry_ctls |= VM_ENTRY_GUEST_LMA; 14823de83862SNeel Natu vmcs_write(VMCS_ENTRY_CTLS, entry_ctls); 148380a902efSPeter Grehan } 148480a902efSPeter Grehan } 148580a902efSPeter Grehan 1486366f6083SPeter Grehan return (HANDLED); 1487366f6083SPeter Grehan } 1488366f6083SPeter Grehan 148900f3efe1SJohn Baldwin static enum vie_cpu_mode 149000f3efe1SJohn Baldwin vmx_cpu_mode(void) 149100f3efe1SJohn Baldwin { 149200f3efe1SJohn Baldwin 149300f3efe1SJohn Baldwin if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LMA) 149400f3efe1SJohn Baldwin return (CPU_MODE_64BIT); 149500f3efe1SJohn Baldwin else 149600f3efe1SJohn Baldwin return (CPU_MODE_COMPATIBILITY); 149700f3efe1SJohn Baldwin } 149800f3efe1SJohn Baldwin 149900f3efe1SJohn Baldwin static enum vie_paging_mode 150000f3efe1SJohn Baldwin vmx_paging_mode(void) 150100f3efe1SJohn Baldwin { 150200f3efe1SJohn Baldwin 150300f3efe1SJohn Baldwin if (!(vmcs_read(VMCS_GUEST_CR0) & CR0_PG)) 150400f3efe1SJohn Baldwin return (PAGING_MODE_FLAT); 150500f3efe1SJohn Baldwin if (!(vmcs_read(VMCS_GUEST_CR4) & CR4_PAE)) 150600f3efe1SJohn Baldwin return (PAGING_MODE_32); 150700f3efe1SJohn Baldwin if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LME) 150800f3efe1SJohn Baldwin return (PAGING_MODE_64); 150900f3efe1SJohn Baldwin else 151000f3efe1SJohn Baldwin return (PAGING_MODE_PAE); 151100f3efe1SJohn Baldwin } 151200f3efe1SJohn Baldwin 1513366f6083SPeter Grehan static int 1514318224bbSNeel Natu ept_fault_type(uint64_t ept_qual) 1515a2da7af6SNeel Natu { 1516318224bbSNeel Natu int fault_type; 1517a2da7af6SNeel Natu 1518318224bbSNeel Natu if (ept_qual & EPT_VIOLATION_DATA_WRITE) 1519318224bbSNeel Natu fault_type = VM_PROT_WRITE; 1520318224bbSNeel Natu else if (ept_qual & EPT_VIOLATION_INST_FETCH) 1521318224bbSNeel Natu fault_type = VM_PROT_EXECUTE; 1522318224bbSNeel Natu else 1523318224bbSNeel Natu fault_type= VM_PROT_READ; 1524318224bbSNeel Natu 1525318224bbSNeel Natu return (fault_type); 1526318224bbSNeel Natu } 1527318224bbSNeel Natu 1528318224bbSNeel Natu static boolean_t 1529318224bbSNeel Natu ept_emulation_fault(uint64_t ept_qual) 1530318224bbSNeel Natu { 1531318224bbSNeel Natu int read, write; 1532318224bbSNeel Natu 1533318224bbSNeel Natu /* EPT fault on an instruction fetch doesn't make sense here */ 1534a2da7af6SNeel Natu if (ept_qual & EPT_VIOLATION_INST_FETCH) 1535318224bbSNeel Natu return (FALSE); 1536a2da7af6SNeel Natu 1537318224bbSNeel Natu /* EPT fault must be a read fault or a write fault */ 1538a2da7af6SNeel Natu read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0; 1539a2da7af6SNeel Natu write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0; 15403b2b0011SPeter Grehan if ((read | write) == 0) 1541318224bbSNeel Natu return (FALSE); 1542a2da7af6SNeel Natu 1543a2da7af6SNeel Natu /* 15443b2b0011SPeter Grehan * The EPT violation must have been caused by accessing a 15453b2b0011SPeter Grehan * guest-physical address that is a translation of a guest-linear 15463b2b0011SPeter Grehan * address. 1547a2da7af6SNeel Natu */ 1548a2da7af6SNeel Natu if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 || 1549a2da7af6SNeel Natu (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) { 1550318224bbSNeel Natu return (FALSE); 1551a2da7af6SNeel Natu } 1552a2da7af6SNeel Natu 1553318224bbSNeel Natu return (TRUE); 1554a2da7af6SNeel Natu } 1555a2da7af6SNeel Natu 1556159dd56fSNeel Natu static __inline int 1557159dd56fSNeel Natu apic_access_virtualization(struct vmx *vmx, int vcpuid) 1558159dd56fSNeel Natu { 1559159dd56fSNeel Natu uint32_t proc_ctls2; 1560159dd56fSNeel Natu 1561159dd56fSNeel Natu proc_ctls2 = vmx->cap[vcpuid].proc_ctls2; 1562159dd56fSNeel Natu return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) ? 1 : 0); 1563159dd56fSNeel Natu } 1564159dd56fSNeel Natu 1565159dd56fSNeel Natu static __inline int 1566159dd56fSNeel Natu x2apic_virtualization(struct vmx *vmx, int vcpuid) 1567159dd56fSNeel Natu { 1568159dd56fSNeel Natu uint32_t proc_ctls2; 1569159dd56fSNeel Natu 1570159dd56fSNeel Natu proc_ctls2 = vmx->cap[vcpuid].proc_ctls2; 1571159dd56fSNeel Natu return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_X2APIC_MODE) ? 1 : 0); 1572159dd56fSNeel Natu } 1573159dd56fSNeel Natu 1574a2da7af6SNeel Natu static int 1575159dd56fSNeel Natu vmx_handle_apic_write(struct vmx *vmx, int vcpuid, struct vlapic *vlapic, 1576159dd56fSNeel Natu uint64_t qual) 157788c4b8d1SNeel Natu { 157888c4b8d1SNeel Natu int error, handled, offset; 1579159dd56fSNeel Natu uint32_t *apic_regs, vector; 158088c4b8d1SNeel Natu bool retu; 158188c4b8d1SNeel Natu 1582a0efd3fbSJohn Baldwin handled = HANDLED; 158388c4b8d1SNeel Natu offset = APIC_WRITE_OFFSET(qual); 1584159dd56fSNeel Natu 1585159dd56fSNeel Natu if (!apic_access_virtualization(vmx, vcpuid)) { 1586159dd56fSNeel Natu /* 1587159dd56fSNeel Natu * In general there should not be any APIC write VM-exits 1588159dd56fSNeel Natu * unless APIC-access virtualization is enabled. 1589159dd56fSNeel Natu * 1590159dd56fSNeel Natu * However self-IPI virtualization can legitimately trigger 1591159dd56fSNeel Natu * an APIC-write VM-exit so treat it specially. 1592159dd56fSNeel Natu */ 1593159dd56fSNeel Natu if (x2apic_virtualization(vmx, vcpuid) && 1594159dd56fSNeel Natu offset == APIC_OFFSET_SELF_IPI) { 1595159dd56fSNeel Natu apic_regs = (uint32_t *)(vlapic->apic_page); 1596159dd56fSNeel Natu vector = apic_regs[APIC_OFFSET_SELF_IPI / 4]; 1597159dd56fSNeel Natu vlapic_self_ipi_handler(vlapic, vector); 1598159dd56fSNeel Natu return (HANDLED); 1599159dd56fSNeel Natu } else 1600159dd56fSNeel Natu return (UNHANDLED); 1601159dd56fSNeel Natu } 1602159dd56fSNeel Natu 160388c4b8d1SNeel Natu switch (offset) { 160488c4b8d1SNeel Natu case APIC_OFFSET_ID: 160588c4b8d1SNeel Natu vlapic_id_write_handler(vlapic); 160688c4b8d1SNeel Natu break; 160788c4b8d1SNeel Natu case APIC_OFFSET_LDR: 160888c4b8d1SNeel Natu vlapic_ldr_write_handler(vlapic); 160988c4b8d1SNeel Natu break; 161088c4b8d1SNeel Natu case APIC_OFFSET_DFR: 161188c4b8d1SNeel Natu vlapic_dfr_write_handler(vlapic); 161288c4b8d1SNeel Natu break; 161388c4b8d1SNeel Natu case APIC_OFFSET_SVR: 161488c4b8d1SNeel Natu vlapic_svr_write_handler(vlapic); 161588c4b8d1SNeel Natu break; 161688c4b8d1SNeel Natu case APIC_OFFSET_ESR: 161788c4b8d1SNeel Natu vlapic_esr_write_handler(vlapic); 161888c4b8d1SNeel Natu break; 161988c4b8d1SNeel Natu case APIC_OFFSET_ICR_LOW: 162088c4b8d1SNeel Natu retu = false; 162188c4b8d1SNeel Natu error = vlapic_icrlo_write_handler(vlapic, &retu); 162288c4b8d1SNeel Natu if (error != 0 || retu) 1623a0efd3fbSJohn Baldwin handled = UNHANDLED; 162488c4b8d1SNeel Natu break; 162588c4b8d1SNeel Natu case APIC_OFFSET_CMCI_LVT: 162688c4b8d1SNeel Natu case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT: 162788c4b8d1SNeel Natu vlapic_lvt_write_handler(vlapic, offset); 162888c4b8d1SNeel Natu break; 162988c4b8d1SNeel Natu case APIC_OFFSET_TIMER_ICR: 163088c4b8d1SNeel Natu vlapic_icrtmr_write_handler(vlapic); 163188c4b8d1SNeel Natu break; 163288c4b8d1SNeel Natu case APIC_OFFSET_TIMER_DCR: 163388c4b8d1SNeel Natu vlapic_dcr_write_handler(vlapic); 163488c4b8d1SNeel Natu break; 163588c4b8d1SNeel Natu default: 1636a0efd3fbSJohn Baldwin handled = UNHANDLED; 163788c4b8d1SNeel Natu break; 163888c4b8d1SNeel Natu } 163988c4b8d1SNeel Natu return (handled); 164088c4b8d1SNeel Natu } 164188c4b8d1SNeel Natu 164288c4b8d1SNeel Natu static bool 1643159dd56fSNeel Natu apic_access_fault(struct vmx *vmx, int vcpuid, uint64_t gpa) 164488c4b8d1SNeel Natu { 164588c4b8d1SNeel Natu 1646159dd56fSNeel Natu if (apic_access_virtualization(vmx, vcpuid) && 164788c4b8d1SNeel Natu (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE)) 164888c4b8d1SNeel Natu return (true); 164988c4b8d1SNeel Natu else 165088c4b8d1SNeel Natu return (false); 165188c4b8d1SNeel Natu } 165288c4b8d1SNeel Natu 165388c4b8d1SNeel Natu static int 165488c4b8d1SNeel Natu vmx_handle_apic_access(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit) 165588c4b8d1SNeel Natu { 165688c4b8d1SNeel Natu uint64_t qual; 165788c4b8d1SNeel Natu int access_type, offset, allowed; 165888c4b8d1SNeel Natu 1659159dd56fSNeel Natu if (!apic_access_virtualization(vmx, vcpuid)) 166088c4b8d1SNeel Natu return (UNHANDLED); 166188c4b8d1SNeel Natu 166288c4b8d1SNeel Natu qual = vmexit->u.vmx.exit_qualification; 166388c4b8d1SNeel Natu access_type = APIC_ACCESS_TYPE(qual); 166488c4b8d1SNeel Natu offset = APIC_ACCESS_OFFSET(qual); 166588c4b8d1SNeel Natu 166688c4b8d1SNeel Natu allowed = 0; 166788c4b8d1SNeel Natu if (access_type == 0) { 166888c4b8d1SNeel Natu /* 166988c4b8d1SNeel Natu * Read data access to the following registers is expected. 167088c4b8d1SNeel Natu */ 167188c4b8d1SNeel Natu switch (offset) { 167288c4b8d1SNeel Natu case APIC_OFFSET_APR: 167388c4b8d1SNeel Natu case APIC_OFFSET_PPR: 167488c4b8d1SNeel Natu case APIC_OFFSET_RRR: 167588c4b8d1SNeel Natu case APIC_OFFSET_CMCI_LVT: 167688c4b8d1SNeel Natu case APIC_OFFSET_TIMER_CCR: 167788c4b8d1SNeel Natu allowed = 1; 167888c4b8d1SNeel Natu break; 167988c4b8d1SNeel Natu default: 168088c4b8d1SNeel Natu break; 168188c4b8d1SNeel Natu } 168288c4b8d1SNeel Natu } else if (access_type == 1) { 168388c4b8d1SNeel Natu /* 168488c4b8d1SNeel Natu * Write data access to the following registers is expected. 168588c4b8d1SNeel Natu */ 168688c4b8d1SNeel Natu switch (offset) { 168788c4b8d1SNeel Natu case APIC_OFFSET_VER: 168888c4b8d1SNeel Natu case APIC_OFFSET_APR: 168988c4b8d1SNeel Natu case APIC_OFFSET_PPR: 169088c4b8d1SNeel Natu case APIC_OFFSET_RRR: 169188c4b8d1SNeel Natu case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7: 169288c4b8d1SNeel Natu case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7: 169388c4b8d1SNeel Natu case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7: 169488c4b8d1SNeel Natu case APIC_OFFSET_CMCI_LVT: 169588c4b8d1SNeel Natu case APIC_OFFSET_TIMER_CCR: 169688c4b8d1SNeel Natu allowed = 1; 169788c4b8d1SNeel Natu break; 169888c4b8d1SNeel Natu default: 169988c4b8d1SNeel Natu break; 170088c4b8d1SNeel Natu } 170188c4b8d1SNeel Natu } 170288c4b8d1SNeel Natu 170388c4b8d1SNeel Natu if (allowed) { 170488c4b8d1SNeel Natu vmexit->exitcode = VM_EXITCODE_INST_EMUL; 170588c4b8d1SNeel Natu vmexit->u.inst_emul.gpa = DEFAULT_APIC_BASE + offset; 170688c4b8d1SNeel Natu vmexit->u.inst_emul.gla = VIE_INVALID_GLA; 170788c4b8d1SNeel Natu vmexit->u.inst_emul.cr3 = vmcs_guest_cr3(); 170800f3efe1SJohn Baldwin vmexit->u.inst_emul.cpu_mode = vmx_cpu_mode(); 170900f3efe1SJohn Baldwin vmexit->u.inst_emul.paging_mode = vmx_paging_mode(); 171088c4b8d1SNeel Natu } 171188c4b8d1SNeel Natu 171288c4b8d1SNeel Natu /* 171388c4b8d1SNeel Natu * Regardless of whether the APIC-access is allowed this handler 171488c4b8d1SNeel Natu * always returns UNHANDLED: 171588c4b8d1SNeel Natu * - if the access is allowed then it is handled by emulating the 171688c4b8d1SNeel Natu * instruction that caused the VM-exit (outside the critical section) 171788c4b8d1SNeel Natu * - if the access is not allowed then it will be converted to an 171888c4b8d1SNeel Natu * exitcode of VM_EXITCODE_VMX and will be dealt with in userland. 171988c4b8d1SNeel Natu */ 172088c4b8d1SNeel Natu return (UNHANDLED); 172188c4b8d1SNeel Natu } 172288c4b8d1SNeel Natu 172388c4b8d1SNeel Natu static int 1724366f6083SPeter Grehan vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) 1725366f6083SPeter Grehan { 1726f76fc5d4SNeel Natu int error, handled; 1727366f6083SPeter Grehan struct vmxctx *vmxctx; 172888c4b8d1SNeel Natu struct vlapic *vlapic; 1729e5a1d950SNeel Natu uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, reason; 17303de83862SNeel Natu uint64_t qual, gpa; 1731becd9849SNeel Natu bool retu; 1732366f6083SPeter Grehan 1733160471d2SNeel Natu CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0); 1734c308b23bSNeel Natu CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0); 1735160471d2SNeel Natu 1736a0efd3fbSJohn Baldwin handled = UNHANDLED; 1737366f6083SPeter Grehan vmxctx = &vmx->ctx[vcpu]; 17380492757cSNeel Natu 1739366f6083SPeter Grehan qual = vmexit->u.vmx.exit_qualification; 1740318224bbSNeel Natu reason = vmexit->u.vmx.exit_reason; 1741366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_BOGUS; 1742366f6083SPeter Grehan 174361592433SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1); 174461592433SNeel Natu 1745318224bbSNeel Natu /* 1746318224bbSNeel Natu * VM exits that could be triggered during event injection on the 1747318224bbSNeel Natu * previous VM entry need to be handled specially by re-injecting 1748318224bbSNeel Natu * the event. 1749318224bbSNeel Natu * 1750318224bbSNeel Natu * See "Information for VM Exits During Event Delivery" in Intel SDM 1751318224bbSNeel Natu * for details. 1752318224bbSNeel Natu */ 1753318224bbSNeel Natu switch (reason) { 1754318224bbSNeel Natu case EXIT_REASON_EPT_FAULT: 1755318224bbSNeel Natu case EXIT_REASON_EPT_MISCONFIG: 175688c4b8d1SNeel Natu case EXIT_REASON_APIC_ACCESS: 1757318224bbSNeel Natu case EXIT_REASON_TASK_SWITCH: 1758318224bbSNeel Natu case EXIT_REASON_EXCEPTION: 1759318224bbSNeel Natu idtvec_info = vmcs_idt_vectoring_info(); 1760318224bbSNeel Natu if (idtvec_info & VMCS_IDT_VEC_VALID) { 1761318224bbSNeel Natu idtvec_info &= ~(1 << 12); /* clear undefined bit */ 17623de83862SNeel Natu vmcs_write(VMCS_ENTRY_INTR_INFO, idtvec_info); 1763318224bbSNeel Natu if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) { 1764318224bbSNeel Natu idtvec_err = vmcs_idt_vectoring_err(); 17653de83862SNeel Natu vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR, 17663de83862SNeel Natu idtvec_err); 1767318224bbSNeel Natu } 1768160471d2SNeel Natu /* 1769160471d2SNeel Natu * If 'virtual NMIs' are being used and the VM-exit 1770160471d2SNeel Natu * happened while injecting an NMI during the previous 1771160471d2SNeel Natu * VM-entry, then clear "blocking by NMI" in the Guest 1772160471d2SNeel Natu * Interruptibility-state. 1773160471d2SNeel Natu */ 1774160471d2SNeel Natu if ((idtvec_info & VMCS_INTR_T_MASK) == 1775160471d2SNeel Natu VMCS_INTR_T_NMI) { 1776e5a1d950SNeel Natu vmx_clear_nmi_blocking(vmx, vcpu); 1777160471d2SNeel Natu } 17783de83862SNeel Natu vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length); 1779318224bbSNeel Natu } 1780318224bbSNeel Natu default: 1781e5a1d950SNeel Natu idtvec_info = 0; 1782318224bbSNeel Natu break; 1783318224bbSNeel Natu } 1784318224bbSNeel Natu 1785318224bbSNeel Natu switch (reason) { 1786366f6083SPeter Grehan case EXIT_REASON_CR_ACCESS: 1787b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CR_ACCESS, 1); 1788366f6083SPeter Grehan handled = vmx_emulate_cr_access(vmx, vcpu, qual); 1789366f6083SPeter Grehan break; 1790366f6083SPeter Grehan case EXIT_REASON_RDMSR: 1791b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RDMSR, 1); 1792becd9849SNeel Natu retu = false; 1793366f6083SPeter Grehan ecx = vmxctx->guest_rcx; 1794becd9849SNeel Natu error = emulate_rdmsr(vmx->vm, vcpu, ecx, &retu); 1795b42206f3SNeel Natu if (error) { 1796366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_RDMSR; 1797366f6083SPeter Grehan vmexit->u.msr.code = ecx; 1798becd9849SNeel Natu } else if (!retu) { 1799a0efd3fbSJohn Baldwin handled = HANDLED; 1800becd9849SNeel Natu } else { 1801becd9849SNeel Natu /* Return to userspace with a valid exitcode */ 1802becd9849SNeel Natu KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS, 1803becd9849SNeel Natu ("emulate_wrmsr retu with bogus exitcode")); 1804becd9849SNeel Natu } 1805366f6083SPeter Grehan break; 1806366f6083SPeter Grehan case EXIT_REASON_WRMSR: 1807b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_WRMSR, 1); 1808becd9849SNeel Natu retu = false; 1809366f6083SPeter Grehan eax = vmxctx->guest_rax; 1810366f6083SPeter Grehan ecx = vmxctx->guest_rcx; 1811366f6083SPeter Grehan edx = vmxctx->guest_rdx; 1812b42206f3SNeel Natu error = emulate_wrmsr(vmx->vm, vcpu, ecx, 1813becd9849SNeel Natu (uint64_t)edx << 32 | eax, &retu); 1814b42206f3SNeel Natu if (error) { 1815366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_WRMSR; 1816366f6083SPeter Grehan vmexit->u.msr.code = ecx; 1817366f6083SPeter Grehan vmexit->u.msr.wval = (uint64_t)edx << 32 | eax; 1818becd9849SNeel Natu } else if (!retu) { 1819a0efd3fbSJohn Baldwin handled = HANDLED; 1820becd9849SNeel Natu } else { 1821becd9849SNeel Natu /* Return to userspace with a valid exitcode */ 1822becd9849SNeel Natu KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS, 1823becd9849SNeel Natu ("emulate_wrmsr retu with bogus exitcode")); 1824becd9849SNeel Natu } 1825366f6083SPeter Grehan break; 1826366f6083SPeter Grehan case EXIT_REASON_HLT: 1827f76fc5d4SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_HLT, 1); 1828366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_HLT; 18293de83862SNeel Natu vmexit->u.hlt.rflags = vmcs_read(VMCS_GUEST_RFLAGS); 1830366f6083SPeter Grehan break; 1831366f6083SPeter Grehan case EXIT_REASON_MTF: 1832b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_MTRAP, 1); 1833366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_MTRAP; 1834366f6083SPeter Grehan break; 1835366f6083SPeter Grehan case EXIT_REASON_PAUSE: 1836b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_PAUSE, 1); 1837366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_PAUSE; 1838366f6083SPeter Grehan break; 1839366f6083SPeter Grehan case EXIT_REASON_INTR_WINDOW: 1840b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INTR_WINDOW, 1); 1841366f6083SPeter Grehan vmx_clear_int_window_exiting(vmx, vcpu); 1842b5aaf7b2SNeel Natu return (1); 1843366f6083SPeter Grehan case EXIT_REASON_EXT_INTR: 1844366f6083SPeter Grehan /* 1845366f6083SPeter Grehan * External interrupts serve only to cause VM exits and allow 1846366f6083SPeter Grehan * the host interrupt handler to run. 1847366f6083SPeter Grehan * 1848366f6083SPeter Grehan * If this external interrupt triggers a virtual interrupt 1849366f6083SPeter Grehan * to a VM, then that state will be recorded by the 1850366f6083SPeter Grehan * host interrupt handler in the VM's softc. We will inject 1851366f6083SPeter Grehan * this virtual interrupt during the subsequent VM enter. 1852366f6083SPeter Grehan */ 1853f7d47425SNeel Natu intr_info = vmcs_read(VMCS_EXIT_INTR_INFO); 1854722b6744SJohn Baldwin 1855722b6744SJohn Baldwin /* 1856722b6744SJohn Baldwin * XXX: Ignore this exit if VMCS_INTR_VALID is not set. 1857ad3e3687SJohn Baldwin * This appears to be a bug in VMware Fusion? 1858722b6744SJohn Baldwin */ 1859722b6744SJohn Baldwin if (!(intr_info & VMCS_INTR_VALID)) 1860722b6744SJohn Baldwin return (1); 1861160471d2SNeel Natu KASSERT((intr_info & VMCS_INTR_VALID) != 0 && 1862160471d2SNeel Natu (intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_HWINTR, 1863f7d47425SNeel Natu ("VM exit interruption info invalid: %#x", intr_info)); 1864f7d47425SNeel Natu vmx_trigger_hostintr(intr_info & 0xff); 1865366f6083SPeter Grehan 1866366f6083SPeter Grehan /* 1867366f6083SPeter Grehan * This is special. We want to treat this as an 'handled' 1868366f6083SPeter Grehan * VM-exit but not increment the instruction pointer. 1869366f6083SPeter Grehan */ 1870366f6083SPeter Grehan vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1); 1871366f6083SPeter Grehan return (1); 1872366f6083SPeter Grehan case EXIT_REASON_NMI_WINDOW: 1873366f6083SPeter Grehan /* Exit to allow the pending virtual NMI to be injected */ 187448b2d828SNeel Natu if (vm_nmi_pending(vmx->vm, vcpu)) 187548b2d828SNeel Natu vmx_inject_nmi(vmx, vcpu); 1876366f6083SPeter Grehan vmx_clear_nmi_window_exiting(vmx, vcpu); 187748b2d828SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NMI_WINDOW, 1); 1878366f6083SPeter Grehan return (1); 1879366f6083SPeter Grehan case EXIT_REASON_INOUT: 1880b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INOUT, 1); 1881366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_INOUT; 1882366f6083SPeter Grehan vmexit->u.inout.bytes = (qual & 0x7) + 1; 1883366f6083SPeter Grehan vmexit->u.inout.in = (qual & 0x8) ? 1 : 0; 1884366f6083SPeter Grehan vmexit->u.inout.string = (qual & 0x10) ? 1 : 0; 1885366f6083SPeter Grehan vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0; 1886366f6083SPeter Grehan vmexit->u.inout.port = (uint16_t)(qual >> 16); 1887366f6083SPeter Grehan vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax); 1888762fd208STycho Nightingale error = emulate_ioport(vmx->vm, vcpu, vmexit); 1889762fd208STycho Nightingale if (error == 0) { 1890762fd208STycho Nightingale handled = 1; 1891762fd208STycho Nightingale vmxctx->guest_rax = vmexit->u.inout.eax; 1892762fd208STycho Nightingale } 1893366f6083SPeter Grehan break; 1894366f6083SPeter Grehan case EXIT_REASON_CPUID: 1895b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CPUID, 1); 1896a2da7af6SNeel Natu handled = vmx_handle_cpuid(vmx->vm, vcpu, vmxctx); 1897366f6083SPeter Grehan break; 1898e5a1d950SNeel Natu case EXIT_REASON_EXCEPTION: 1899c308b23bSNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXCEPTION, 1); 1900e5a1d950SNeel Natu intr_info = vmcs_read(VMCS_EXIT_INTR_INFO); 1901e5a1d950SNeel Natu KASSERT((intr_info & VMCS_INTR_VALID) != 0, 1902e5a1d950SNeel Natu ("VM exit interruption info invalid: %#x", intr_info)); 1903c308b23bSNeel Natu 1904e5a1d950SNeel Natu /* 1905e5a1d950SNeel Natu * If Virtual NMIs control is 1 and the VM-exit is due to a 1906e5a1d950SNeel Natu * fault encountered during the execution of IRET then we must 1907e5a1d950SNeel Natu * restore the state of "virtual-NMI blocking" before resuming 1908e5a1d950SNeel Natu * the guest. 1909e5a1d950SNeel Natu * 1910e5a1d950SNeel Natu * See "Resuming Guest Software after Handling an Exception". 1911e5a1d950SNeel Natu */ 1912e5a1d950SNeel Natu if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 && 1913e5a1d950SNeel Natu (intr_info & 0xff) != IDT_DF && 1914e5a1d950SNeel Natu (intr_info & EXIT_QUAL_NMIUDTI) != 0) 1915e5a1d950SNeel Natu vmx_restore_nmi_blocking(vmx, vcpu); 1916c308b23bSNeel Natu 1917c308b23bSNeel Natu /* 191862fbd7c2SNeel Natu * The NMI has already been handled in vmx_exit_handle_nmi(). 1919c308b23bSNeel Natu */ 192062fbd7c2SNeel Natu if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI) 1921c308b23bSNeel Natu return (1); 1922e5a1d950SNeel Natu break; 1923cd942e0fSPeter Grehan case EXIT_REASON_EPT_FAULT: 1924318224bbSNeel Natu /* 1925318224bbSNeel Natu * If 'gpa' lies within the address space allocated to 1926318224bbSNeel Natu * memory then this must be a nested page fault otherwise 1927318224bbSNeel Natu * this must be an instruction that accesses MMIO space. 1928318224bbSNeel Natu */ 1929a2da7af6SNeel Natu gpa = vmcs_gpa(); 1930159dd56fSNeel Natu if (vm_mem_allocated(vmx->vm, gpa) || 1931159dd56fSNeel Natu apic_access_fault(vmx, vcpu, gpa)) { 1932cd942e0fSPeter Grehan vmexit->exitcode = VM_EXITCODE_PAGING; 193313ec9371SPeter Grehan vmexit->u.paging.gpa = gpa; 1934318224bbSNeel Natu vmexit->u.paging.fault_type = ept_fault_type(qual); 1935bf73979dSNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NESTED_FAULT, 1); 1936318224bbSNeel Natu } else if (ept_emulation_fault(qual)) { 1937318224bbSNeel Natu vmexit->exitcode = VM_EXITCODE_INST_EMUL; 1938318224bbSNeel Natu vmexit->u.inst_emul.gpa = gpa; 1939318224bbSNeel Natu vmexit->u.inst_emul.gla = vmcs_gla(); 1940318224bbSNeel Natu vmexit->u.inst_emul.cr3 = vmcs_guest_cr3(); 194100f3efe1SJohn Baldwin vmexit->u.inst_emul.cpu_mode = vmx_cpu_mode(); 194200f3efe1SJohn Baldwin vmexit->u.inst_emul.paging_mode = vmx_paging_mode(); 1943bf73979dSNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INST_EMUL, 1); 1944a2da7af6SNeel Natu } 1945e5a1d950SNeel Natu /* 1946e5a1d950SNeel Natu * If Virtual NMIs control is 1 and the VM-exit is due to an 1947e5a1d950SNeel Natu * EPT fault during the execution of IRET then we must restore 1948e5a1d950SNeel Natu * the state of "virtual-NMI blocking" before resuming. 1949e5a1d950SNeel Natu * 1950e5a1d950SNeel Natu * See description of "NMI unblocking due to IRET" in 1951e5a1d950SNeel Natu * "Exit Qualification for EPT Violations". 1952e5a1d950SNeel Natu */ 1953e5a1d950SNeel Natu if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 && 1954e5a1d950SNeel Natu (qual & EXIT_QUAL_NMIUDTI) != 0) 1955e5a1d950SNeel Natu vmx_restore_nmi_blocking(vmx, vcpu); 1956cd942e0fSPeter Grehan break; 195730b94db8SNeel Natu case EXIT_REASON_VIRTUALIZED_EOI: 195830b94db8SNeel Natu vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI; 195930b94db8SNeel Natu vmexit->u.ioapic_eoi.vector = qual & 0xFF; 196030b94db8SNeel Natu vmexit->inst_length = 0; /* trap-like */ 196130b94db8SNeel Natu break; 196288c4b8d1SNeel Natu case EXIT_REASON_APIC_ACCESS: 196388c4b8d1SNeel Natu handled = vmx_handle_apic_access(vmx, vcpu, vmexit); 196488c4b8d1SNeel Natu break; 196588c4b8d1SNeel Natu case EXIT_REASON_APIC_WRITE: 196688c4b8d1SNeel Natu /* 196788c4b8d1SNeel Natu * APIC-write VM exit is trap-like so the %rip is already 196888c4b8d1SNeel Natu * pointing to the next instruction. 196988c4b8d1SNeel Natu */ 197088c4b8d1SNeel Natu vmexit->inst_length = 0; 197188c4b8d1SNeel Natu vlapic = vm_lapic(vmx->vm, vcpu); 1972159dd56fSNeel Natu handled = vmx_handle_apic_write(vmx, vcpu, vlapic, qual); 197388c4b8d1SNeel Natu break; 1974abb023fbSJohn Baldwin case EXIT_REASON_XSETBV: 1975a0efd3fbSJohn Baldwin handled = vmx_emulate_xsetbv(vmx, vcpu, vmexit); 1976abb023fbSJohn Baldwin break; 1977366f6083SPeter Grehan default: 1978b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_UNKNOWN, 1); 1979366f6083SPeter Grehan break; 1980366f6083SPeter Grehan } 1981366f6083SPeter Grehan 1982366f6083SPeter Grehan if (handled) { 1983366f6083SPeter Grehan /* 1984366f6083SPeter Grehan * It is possible that control is returned to userland 1985366f6083SPeter Grehan * even though we were able to handle the VM exit in the 1986eeefa4e4SNeel Natu * kernel. 1987366f6083SPeter Grehan * 1988366f6083SPeter Grehan * In such a case we want to make sure that the userland 1989366f6083SPeter Grehan * restarts guest execution at the instruction *after* 1990366f6083SPeter Grehan * the one we just processed. Therefore we update the 1991366f6083SPeter Grehan * guest rip in the VMCS and in 'vmexit'. 1992366f6083SPeter Grehan */ 1993366f6083SPeter Grehan vmexit->rip += vmexit->inst_length; 1994366f6083SPeter Grehan vmexit->inst_length = 0; 19953de83862SNeel Natu vmcs_write(VMCS_GUEST_RIP, vmexit->rip); 1996366f6083SPeter Grehan } else { 1997366f6083SPeter Grehan if (vmexit->exitcode == VM_EXITCODE_BOGUS) { 1998366f6083SPeter Grehan /* 1999366f6083SPeter Grehan * If this VM exit was not claimed by anybody then 2000366f6083SPeter Grehan * treat it as a generic VMX exit. 2001366f6083SPeter Grehan */ 2002366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_VMX; 20030492757cSNeel Natu vmexit->u.vmx.status = VM_SUCCESS; 2004c308b23bSNeel Natu vmexit->u.vmx.inst_type = 0; 2005c308b23bSNeel Natu vmexit->u.vmx.inst_error = 0; 2006366f6083SPeter Grehan } else { 2007366f6083SPeter Grehan /* 2008366f6083SPeter Grehan * The exitcode and collateral have been populated. 2009366f6083SPeter Grehan * The VM exit will be processed further in userland. 2010366f6083SPeter Grehan */ 2011366f6083SPeter Grehan } 2012366f6083SPeter Grehan } 2013366f6083SPeter Grehan return (handled); 2014366f6083SPeter Grehan } 2015366f6083SPeter Grehan 20160492757cSNeel Natu static __inline int 20170492757cSNeel Natu vmx_exit_astpending(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) 2018366f6083SPeter Grehan { 20190492757cSNeel Natu 20200492757cSNeel Natu vmexit->rip = vmcs_guest_rip(); 20210492757cSNeel Natu vmexit->inst_length = 0; 20220492757cSNeel Natu vmexit->exitcode = VM_EXITCODE_BOGUS; 20230492757cSNeel Natu vmx_astpending_trace(vmx, vcpu, vmexit->rip); 20240492757cSNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_ASTPENDING, 1); 20250492757cSNeel Natu 20260492757cSNeel Natu return (HANDLED); 20270492757cSNeel Natu } 20280492757cSNeel Natu 20290492757cSNeel Natu static __inline int 20305b8a8cd1SNeel Natu vmx_exit_rendezvous(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) 20315b8a8cd1SNeel Natu { 20325b8a8cd1SNeel Natu 20335b8a8cd1SNeel Natu vmexit->rip = vmcs_guest_rip(); 20345b8a8cd1SNeel Natu vmexit->inst_length = 0; 20355b8a8cd1SNeel Natu vmexit->exitcode = VM_EXITCODE_RENDEZVOUS; 20365b8a8cd1SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RENDEZVOUS, 1); 20375b8a8cd1SNeel Natu 20385b8a8cd1SNeel Natu return (UNHANDLED); 20395b8a8cd1SNeel Natu } 20405b8a8cd1SNeel Natu 20415b8a8cd1SNeel Natu static __inline int 2042b15a09c0SNeel Natu vmx_exit_suspended(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) 2043b15a09c0SNeel Natu { 2044b15a09c0SNeel Natu 2045b15a09c0SNeel Natu vmexit->rip = vmcs_guest_rip(); 2046b15a09c0SNeel Natu vmexit->inst_length = 0; 2047b15a09c0SNeel Natu vmexit->exitcode = VM_EXITCODE_SUSPENDED; 2048b15a09c0SNeel Natu return (UNHANDLED); 2049b15a09c0SNeel Natu } 2050b15a09c0SNeel Natu 2051b15a09c0SNeel Natu static __inline int 20520492757cSNeel Natu vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit) 20530492757cSNeel Natu { 20540492757cSNeel Natu 20550492757cSNeel Natu KASSERT(vmxctx->inst_fail_status != VM_SUCCESS, 20560492757cSNeel Natu ("vmx_exit_inst_error: invalid inst_fail_status %d", 20570492757cSNeel Natu vmxctx->inst_fail_status)); 20580492757cSNeel Natu 20590492757cSNeel Natu vmexit->inst_length = 0; 20600492757cSNeel Natu vmexit->exitcode = VM_EXITCODE_VMX; 20610492757cSNeel Natu vmexit->u.vmx.status = vmxctx->inst_fail_status; 20620492757cSNeel Natu vmexit->u.vmx.inst_error = vmcs_instruction_error(); 20630492757cSNeel Natu vmexit->u.vmx.exit_reason = ~0; 20640492757cSNeel Natu vmexit->u.vmx.exit_qualification = ~0; 20650492757cSNeel Natu 20660492757cSNeel Natu switch (rc) { 20670492757cSNeel Natu case VMX_VMRESUME_ERROR: 20680492757cSNeel Natu case VMX_VMLAUNCH_ERROR: 20690492757cSNeel Natu case VMX_INVEPT_ERROR: 20700492757cSNeel Natu vmexit->u.vmx.inst_type = rc; 20710492757cSNeel Natu break; 20720492757cSNeel Natu default: 20730492757cSNeel Natu panic("vm_exit_inst_error: vmx_enter_guest returned %d", rc); 20740492757cSNeel Natu } 20750492757cSNeel Natu 20760492757cSNeel Natu return (UNHANDLED); 20770492757cSNeel Natu } 20780492757cSNeel Natu 207962fbd7c2SNeel Natu /* 208062fbd7c2SNeel Natu * If the NMI-exiting VM execution control is set to '1' then an NMI in 208162fbd7c2SNeel Natu * non-root operation causes a VM-exit. NMI blocking is in effect so it is 208262fbd7c2SNeel Natu * sufficient to simply vector to the NMI handler via a software interrupt. 208362fbd7c2SNeel Natu * However, this must be done before maskable interrupts are enabled 208462fbd7c2SNeel Natu * otherwise the "iret" issued by an interrupt handler will incorrectly 208562fbd7c2SNeel Natu * clear NMI blocking. 208662fbd7c2SNeel Natu */ 208762fbd7c2SNeel Natu static __inline void 208862fbd7c2SNeel Natu vmx_exit_handle_nmi(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit) 208962fbd7c2SNeel Natu { 209062fbd7c2SNeel Natu uint32_t intr_info; 209162fbd7c2SNeel Natu 209262fbd7c2SNeel Natu KASSERT((read_rflags() & PSL_I) == 0, ("interrupts enabled")); 209362fbd7c2SNeel Natu 209462fbd7c2SNeel Natu if (vmexit->u.vmx.exit_reason != EXIT_REASON_EXCEPTION) 209562fbd7c2SNeel Natu return; 209662fbd7c2SNeel Natu 209762fbd7c2SNeel Natu intr_info = vmcs_read(VMCS_EXIT_INTR_INFO); 209862fbd7c2SNeel Natu KASSERT((intr_info & VMCS_INTR_VALID) != 0, 209962fbd7c2SNeel Natu ("VM exit interruption info invalid: %#x", intr_info)); 210062fbd7c2SNeel Natu 210162fbd7c2SNeel Natu if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI) { 210262fbd7c2SNeel Natu KASSERT((intr_info & 0xff) == IDT_NMI, ("VM exit due " 210362fbd7c2SNeel Natu "to NMI has invalid vector: %#x", intr_info)); 210462fbd7c2SNeel Natu VCPU_CTR0(vmx->vm, vcpuid, "Vectoring to NMI handler"); 210562fbd7c2SNeel Natu __asm __volatile("int $2"); 210662fbd7c2SNeel Natu } 210762fbd7c2SNeel Natu } 210862fbd7c2SNeel Natu 21090492757cSNeel Natu static int 21105b8a8cd1SNeel Natu vmx_run(void *arg, int vcpu, register_t startrip, pmap_t pmap, 2111b15a09c0SNeel Natu void *rendezvous_cookie, void *suspend_cookie) 21120492757cSNeel Natu { 21130492757cSNeel Natu int rc, handled, launched; 2114366f6083SPeter Grehan struct vmx *vmx; 21155b8a8cd1SNeel Natu struct vm *vm; 2116366f6083SPeter Grehan struct vmxctx *vmxctx; 2117366f6083SPeter Grehan struct vmcs *vmcs; 211898ed632cSNeel Natu struct vm_exit *vmexit; 2119de5ea6b6SNeel Natu struct vlapic *vlapic; 212079c59630SNeel Natu uint64_t rip; 212179c59630SNeel Natu uint32_t exit_reason; 2122366f6083SPeter Grehan 2123366f6083SPeter Grehan vmx = arg; 21245b8a8cd1SNeel Natu vm = vmx->vm; 2125366f6083SPeter Grehan vmcs = &vmx->vmcs[vcpu]; 2126366f6083SPeter Grehan vmxctx = &vmx->ctx[vcpu]; 21275b8a8cd1SNeel Natu vlapic = vm_lapic(vm, vcpu); 21285b8a8cd1SNeel Natu vmexit = vm_exitinfo(vm, vcpu); 21290492757cSNeel Natu launched = 0; 213098ed632cSNeel Natu 2131318224bbSNeel Natu KASSERT(vmxctx->pmap == pmap, 2132318224bbSNeel Natu ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap)); 2133318224bbSNeel Natu 2134366f6083SPeter Grehan VMPTRLD(vmcs); 2135366f6083SPeter Grehan 2136366f6083SPeter Grehan /* 2137366f6083SPeter Grehan * XXX 2138366f6083SPeter Grehan * We do this every time because we may setup the virtual machine 2139366f6083SPeter Grehan * from a different process than the one that actually runs it. 2140366f6083SPeter Grehan * 2141366f6083SPeter Grehan * If the life of a virtual machine was spent entirely in the context 2142c847a506SNeel Natu * of a single process we could do this once in vmx_vminit(). 2143366f6083SPeter Grehan */ 21443de83862SNeel Natu vmcs_write(VMCS_HOST_CR3, rcr3()); 2145366f6083SPeter Grehan 21460492757cSNeel Natu vmcs_write(VMCS_GUEST_RIP, startrip); 2147953c2c47SNeel Natu vmx_set_pcpu_defaults(vmx, vcpu, pmap); 2148366f6083SPeter Grehan do { 21490492757cSNeel Natu /* 21500492757cSNeel Natu * Interrupts are disabled from this point on until the 21510492757cSNeel Natu * guest starts executing. This is done for the following 21520492757cSNeel Natu * reasons: 21530492757cSNeel Natu * 21540492757cSNeel Natu * If an AST is asserted on this thread after the check below, 21550492757cSNeel Natu * then the IPI_AST notification will not be lost, because it 21560492757cSNeel Natu * will cause a VM exit due to external interrupt as soon as 21570492757cSNeel Natu * the guest state is loaded. 21580492757cSNeel Natu * 21590492757cSNeel Natu * A posted interrupt after 'vmx_inject_interrupts()' will 21600492757cSNeel Natu * not be "lost" because it will be held pending in the host 21610492757cSNeel Natu * APIC because interrupts are disabled. The pending interrupt 21620492757cSNeel Natu * will be recognized as soon as the guest state is loaded. 21630492757cSNeel Natu * 21640492757cSNeel Natu * The same reasoning applies to the IPI generated by 21650492757cSNeel Natu * pmap_invalidate_ept(). 21660492757cSNeel Natu */ 21670492757cSNeel Natu disable_intr(); 2168b15a09c0SNeel Natu if (vcpu_suspended(suspend_cookie)) { 21690492757cSNeel Natu enable_intr(); 2170b15a09c0SNeel Natu handled = vmx_exit_suspended(vmx, vcpu, vmexit); 21710492757cSNeel Natu break; 21720492757cSNeel Natu } 21730492757cSNeel Natu 21745b8a8cd1SNeel Natu if (vcpu_rendezvous_pending(rendezvous_cookie)) { 21755b8a8cd1SNeel Natu enable_intr(); 21765b8a8cd1SNeel Natu handled = vmx_exit_rendezvous(vmx, vcpu, vmexit); 21775b8a8cd1SNeel Natu break; 21785b8a8cd1SNeel Natu } 21795b8a8cd1SNeel Natu 2180b15a09c0SNeel Natu if (curthread->td_flags & (TDF_ASTPENDING | TDF_NEEDRESCHED)) { 2181b15a09c0SNeel Natu enable_intr(); 2182b15a09c0SNeel Natu handled = vmx_exit_astpending(vmx, vcpu, vmexit); 2183b15a09c0SNeel Natu break; 2184b15a09c0SNeel Natu } 2185b15a09c0SNeel Natu 2186de5ea6b6SNeel Natu vmx_inject_interrupts(vmx, vcpu, vlapic); 2187366f6083SPeter Grehan vmx_run_trace(vmx, vcpu); 2188953c2c47SNeel Natu rc = vmx_enter_guest(vmxctx, vmx, launched); 218979c59630SNeel Natu 219079c59630SNeel Natu /* Collect some information for VM exit processing */ 219179c59630SNeel Natu vmexit->rip = rip = vmcs_guest_rip(); 219279c59630SNeel Natu vmexit->inst_length = vmexit_instruction_length(); 219379c59630SNeel Natu vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason(); 219479c59630SNeel Natu vmexit->u.vmx.exit_qualification = vmcs_exit_qualification(); 219579c59630SNeel Natu 21960492757cSNeel Natu if (rc == VMX_GUEST_VMEXIT) { 219762fbd7c2SNeel Natu vmx_exit_handle_nmi(vmx, vcpu, vmexit); 219862fbd7c2SNeel Natu enable_intr(); 21990492757cSNeel Natu handled = vmx_exit_process(vmx, vcpu, vmexit); 22000492757cSNeel Natu } else { 220162fbd7c2SNeel Natu enable_intr(); 22020492757cSNeel Natu handled = vmx_exit_inst_error(vmxctx, rc, vmexit); 2203eeefa4e4SNeel Natu } 220462fbd7c2SNeel Natu launched = 1; 220579c59630SNeel Natu vmx_exit_trace(vmx, vcpu, rip, exit_reason, handled); 2206eeefa4e4SNeel Natu } while (handled); 2207366f6083SPeter Grehan 2208366f6083SPeter Grehan /* 2209366f6083SPeter Grehan * If a VM exit has been handled then the exitcode must be BOGUS 2210366f6083SPeter Grehan * If a VM exit is not handled then the exitcode must not be BOGUS 2211366f6083SPeter Grehan */ 2212366f6083SPeter Grehan if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) || 2213366f6083SPeter Grehan (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) { 2214366f6083SPeter Grehan panic("Mismatch between handled (%d) and exitcode (%d)", 2215366f6083SPeter Grehan handled, vmexit->exitcode); 2216366f6083SPeter Grehan } 2217366f6083SPeter Grehan 2218b5aaf7b2SNeel Natu if (!handled) 22195b8a8cd1SNeel Natu vmm_stat_incr(vm, vcpu, VMEXIT_USERSPACE, 1); 2220b5aaf7b2SNeel Natu 22215b8a8cd1SNeel Natu VCPU_CTR1(vm, vcpu, "returning from vmx_run: exitcode %d", 22220492757cSNeel Natu vmexit->exitcode); 2223366f6083SPeter Grehan 2224366f6083SPeter Grehan VMCLEAR(vmcs); 2225366f6083SPeter Grehan return (0); 2226366f6083SPeter Grehan } 2227366f6083SPeter Grehan 2228366f6083SPeter Grehan static void 2229366f6083SPeter Grehan vmx_vmcleanup(void *arg) 2230366f6083SPeter Grehan { 223145e51299SNeel Natu int i, error; 2232366f6083SPeter Grehan struct vmx *vmx = arg; 2233366f6083SPeter Grehan 2234159dd56fSNeel Natu if (apic_access_virtualization(vmx, 0)) 223588c4b8d1SNeel Natu vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE); 223688c4b8d1SNeel Natu 223745e51299SNeel Natu for (i = 0; i < VM_MAXCPU; i++) 223845e51299SNeel Natu vpid_free(vmx->state[i].vpid); 223945e51299SNeel Natu 2240366f6083SPeter Grehan /* 2241366f6083SPeter Grehan * XXXSMP we also need to clear the VMCS active on the other vcpus. 2242366f6083SPeter Grehan */ 2243366f6083SPeter Grehan error = vmclear(&vmx->vmcs[0]); 2244366f6083SPeter Grehan if (error != 0) 2245366f6083SPeter Grehan panic("vmx_vmcleanup: vmclear error %d on vcpu 0", error); 2246366f6083SPeter Grehan 2247366f6083SPeter Grehan free(vmx, M_VMX); 2248366f6083SPeter Grehan 2249366f6083SPeter Grehan return; 2250366f6083SPeter Grehan } 2251366f6083SPeter Grehan 2252366f6083SPeter Grehan static register_t * 2253366f6083SPeter Grehan vmxctx_regptr(struct vmxctx *vmxctx, int reg) 2254366f6083SPeter Grehan { 2255366f6083SPeter Grehan 2256366f6083SPeter Grehan switch (reg) { 2257366f6083SPeter Grehan case VM_REG_GUEST_RAX: 2258366f6083SPeter Grehan return (&vmxctx->guest_rax); 2259366f6083SPeter Grehan case VM_REG_GUEST_RBX: 2260366f6083SPeter Grehan return (&vmxctx->guest_rbx); 2261366f6083SPeter Grehan case VM_REG_GUEST_RCX: 2262366f6083SPeter Grehan return (&vmxctx->guest_rcx); 2263366f6083SPeter Grehan case VM_REG_GUEST_RDX: 2264366f6083SPeter Grehan return (&vmxctx->guest_rdx); 2265366f6083SPeter Grehan case VM_REG_GUEST_RSI: 2266366f6083SPeter Grehan return (&vmxctx->guest_rsi); 2267366f6083SPeter Grehan case VM_REG_GUEST_RDI: 2268366f6083SPeter Grehan return (&vmxctx->guest_rdi); 2269366f6083SPeter Grehan case VM_REG_GUEST_RBP: 2270366f6083SPeter Grehan return (&vmxctx->guest_rbp); 2271366f6083SPeter Grehan case VM_REG_GUEST_R8: 2272366f6083SPeter Grehan return (&vmxctx->guest_r8); 2273366f6083SPeter Grehan case VM_REG_GUEST_R9: 2274366f6083SPeter Grehan return (&vmxctx->guest_r9); 2275366f6083SPeter Grehan case VM_REG_GUEST_R10: 2276366f6083SPeter Grehan return (&vmxctx->guest_r10); 2277366f6083SPeter Grehan case VM_REG_GUEST_R11: 2278366f6083SPeter Grehan return (&vmxctx->guest_r11); 2279366f6083SPeter Grehan case VM_REG_GUEST_R12: 2280366f6083SPeter Grehan return (&vmxctx->guest_r12); 2281366f6083SPeter Grehan case VM_REG_GUEST_R13: 2282366f6083SPeter Grehan return (&vmxctx->guest_r13); 2283366f6083SPeter Grehan case VM_REG_GUEST_R14: 2284366f6083SPeter Grehan return (&vmxctx->guest_r14); 2285366f6083SPeter Grehan case VM_REG_GUEST_R15: 2286366f6083SPeter Grehan return (&vmxctx->guest_r15); 2287366f6083SPeter Grehan default: 2288366f6083SPeter Grehan break; 2289366f6083SPeter Grehan } 2290366f6083SPeter Grehan return (NULL); 2291366f6083SPeter Grehan } 2292366f6083SPeter Grehan 2293366f6083SPeter Grehan static int 2294366f6083SPeter Grehan vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval) 2295366f6083SPeter Grehan { 2296366f6083SPeter Grehan register_t *regp; 2297366f6083SPeter Grehan 2298366f6083SPeter Grehan if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) { 2299366f6083SPeter Grehan *retval = *regp; 2300366f6083SPeter Grehan return (0); 2301366f6083SPeter Grehan } else 2302366f6083SPeter Grehan return (EINVAL); 2303366f6083SPeter Grehan } 2304366f6083SPeter Grehan 2305366f6083SPeter Grehan static int 2306366f6083SPeter Grehan vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val) 2307366f6083SPeter Grehan { 2308366f6083SPeter Grehan register_t *regp; 2309366f6083SPeter Grehan 2310366f6083SPeter Grehan if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) { 2311366f6083SPeter Grehan *regp = val; 2312366f6083SPeter Grehan return (0); 2313366f6083SPeter Grehan } else 2314366f6083SPeter Grehan return (EINVAL); 2315366f6083SPeter Grehan } 2316366f6083SPeter Grehan 2317366f6083SPeter Grehan static int 2318aaaa0656SPeter Grehan vmx_shadow_reg(int reg) 2319aaaa0656SPeter Grehan { 2320aaaa0656SPeter Grehan int shreg; 2321aaaa0656SPeter Grehan 2322aaaa0656SPeter Grehan shreg = -1; 2323aaaa0656SPeter Grehan 2324aaaa0656SPeter Grehan switch (reg) { 2325aaaa0656SPeter Grehan case VM_REG_GUEST_CR0: 2326aaaa0656SPeter Grehan shreg = VMCS_CR0_SHADOW; 2327aaaa0656SPeter Grehan break; 2328aaaa0656SPeter Grehan case VM_REG_GUEST_CR4: 2329aaaa0656SPeter Grehan shreg = VMCS_CR4_SHADOW; 2330aaaa0656SPeter Grehan break; 2331aaaa0656SPeter Grehan default: 2332aaaa0656SPeter Grehan break; 2333aaaa0656SPeter Grehan } 2334aaaa0656SPeter Grehan 2335aaaa0656SPeter Grehan return (shreg); 2336aaaa0656SPeter Grehan } 2337aaaa0656SPeter Grehan 2338aaaa0656SPeter Grehan static int 2339366f6083SPeter Grehan vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval) 2340366f6083SPeter Grehan { 2341d3c11f40SPeter Grehan int running, hostcpu; 2342366f6083SPeter Grehan struct vmx *vmx = arg; 2343366f6083SPeter Grehan 2344d3c11f40SPeter Grehan running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); 2345d3c11f40SPeter Grehan if (running && hostcpu != curcpu) 2346d3c11f40SPeter Grehan panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu); 2347d3c11f40SPeter Grehan 2348366f6083SPeter Grehan if (vmxctx_getreg(&vmx->ctx[vcpu], reg, retval) == 0) 2349366f6083SPeter Grehan return (0); 2350366f6083SPeter Grehan 2351d3c11f40SPeter Grehan return (vmcs_getreg(&vmx->vmcs[vcpu], running, reg, retval)); 2352366f6083SPeter Grehan } 2353366f6083SPeter Grehan 2354366f6083SPeter Grehan static int 2355366f6083SPeter Grehan vmx_setreg(void *arg, int vcpu, int reg, uint64_t val) 2356366f6083SPeter Grehan { 2357aaaa0656SPeter Grehan int error, hostcpu, running, shadow; 2358366f6083SPeter Grehan uint64_t ctls; 2359366f6083SPeter Grehan struct vmx *vmx = arg; 2360366f6083SPeter Grehan 2361d3c11f40SPeter Grehan running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); 2362d3c11f40SPeter Grehan if (running && hostcpu != curcpu) 2363d3c11f40SPeter Grehan panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu); 2364d3c11f40SPeter Grehan 2365366f6083SPeter Grehan if (vmxctx_setreg(&vmx->ctx[vcpu], reg, val) == 0) 2366366f6083SPeter Grehan return (0); 2367366f6083SPeter Grehan 2368d3c11f40SPeter Grehan error = vmcs_setreg(&vmx->vmcs[vcpu], running, reg, val); 2369366f6083SPeter Grehan 2370366f6083SPeter Grehan if (error == 0) { 2371366f6083SPeter Grehan /* 2372366f6083SPeter Grehan * If the "load EFER" VM-entry control is 1 then the 2373366f6083SPeter Grehan * value of EFER.LMA must be identical to "IA-32e mode guest" 2374366f6083SPeter Grehan * bit in the VM-entry control. 2375366f6083SPeter Grehan */ 2376366f6083SPeter Grehan if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 && 2377366f6083SPeter Grehan (reg == VM_REG_GUEST_EFER)) { 2378d3c11f40SPeter Grehan vmcs_getreg(&vmx->vmcs[vcpu], running, 2379366f6083SPeter Grehan VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls); 2380366f6083SPeter Grehan if (val & EFER_LMA) 2381366f6083SPeter Grehan ctls |= VM_ENTRY_GUEST_LMA; 2382366f6083SPeter Grehan else 2383366f6083SPeter Grehan ctls &= ~VM_ENTRY_GUEST_LMA; 2384d3c11f40SPeter Grehan vmcs_setreg(&vmx->vmcs[vcpu], running, 2385366f6083SPeter Grehan VMCS_IDENT(VMCS_ENTRY_CTLS), ctls); 2386366f6083SPeter Grehan } 2387aaaa0656SPeter Grehan 2388aaaa0656SPeter Grehan shadow = vmx_shadow_reg(reg); 2389aaaa0656SPeter Grehan if (shadow > 0) { 2390aaaa0656SPeter Grehan /* 2391aaaa0656SPeter Grehan * Store the unmodified value in the shadow 2392aaaa0656SPeter Grehan */ 2393aaaa0656SPeter Grehan error = vmcs_setreg(&vmx->vmcs[vcpu], running, 2394aaaa0656SPeter Grehan VMCS_IDENT(shadow), val); 2395aaaa0656SPeter Grehan } 2396366f6083SPeter Grehan } 2397366f6083SPeter Grehan 2398366f6083SPeter Grehan return (error); 2399366f6083SPeter Grehan } 2400366f6083SPeter Grehan 2401366f6083SPeter Grehan static int 2402366f6083SPeter Grehan vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc) 2403366f6083SPeter Grehan { 2404366f6083SPeter Grehan struct vmx *vmx = arg; 2405366f6083SPeter Grehan 2406366f6083SPeter Grehan return (vmcs_getdesc(&vmx->vmcs[vcpu], reg, desc)); 2407366f6083SPeter Grehan } 2408366f6083SPeter Grehan 2409366f6083SPeter Grehan static int 2410366f6083SPeter Grehan vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc) 2411366f6083SPeter Grehan { 2412366f6083SPeter Grehan struct vmx *vmx = arg; 2413366f6083SPeter Grehan 2414366f6083SPeter Grehan return (vmcs_setdesc(&vmx->vmcs[vcpu], reg, desc)); 2415366f6083SPeter Grehan } 2416366f6083SPeter Grehan 2417366f6083SPeter Grehan static int 2418366f6083SPeter Grehan vmx_getcap(void *arg, int vcpu, int type, int *retval) 2419366f6083SPeter Grehan { 2420366f6083SPeter Grehan struct vmx *vmx = arg; 2421366f6083SPeter Grehan int vcap; 2422366f6083SPeter Grehan int ret; 2423366f6083SPeter Grehan 2424366f6083SPeter Grehan ret = ENOENT; 2425366f6083SPeter Grehan 2426366f6083SPeter Grehan vcap = vmx->cap[vcpu].set; 2427366f6083SPeter Grehan 2428366f6083SPeter Grehan switch (type) { 2429366f6083SPeter Grehan case VM_CAP_HALT_EXIT: 2430366f6083SPeter Grehan if (cap_halt_exit) 2431366f6083SPeter Grehan ret = 0; 2432366f6083SPeter Grehan break; 2433366f6083SPeter Grehan case VM_CAP_PAUSE_EXIT: 2434366f6083SPeter Grehan if (cap_pause_exit) 2435366f6083SPeter Grehan ret = 0; 2436366f6083SPeter Grehan break; 2437366f6083SPeter Grehan case VM_CAP_MTRAP_EXIT: 2438366f6083SPeter Grehan if (cap_monitor_trap) 2439366f6083SPeter Grehan ret = 0; 2440366f6083SPeter Grehan break; 2441366f6083SPeter Grehan case VM_CAP_UNRESTRICTED_GUEST: 2442366f6083SPeter Grehan if (cap_unrestricted_guest) 2443366f6083SPeter Grehan ret = 0; 2444366f6083SPeter Grehan break; 244549cc03daSNeel Natu case VM_CAP_ENABLE_INVPCID: 244649cc03daSNeel Natu if (cap_invpcid) 244749cc03daSNeel Natu ret = 0; 244849cc03daSNeel Natu break; 2449366f6083SPeter Grehan default: 2450366f6083SPeter Grehan break; 2451366f6083SPeter Grehan } 2452366f6083SPeter Grehan 2453366f6083SPeter Grehan if (ret == 0) 2454366f6083SPeter Grehan *retval = (vcap & (1 << type)) ? 1 : 0; 2455366f6083SPeter Grehan 2456366f6083SPeter Grehan return (ret); 2457366f6083SPeter Grehan } 2458366f6083SPeter Grehan 2459366f6083SPeter Grehan static int 2460366f6083SPeter Grehan vmx_setcap(void *arg, int vcpu, int type, int val) 2461366f6083SPeter Grehan { 2462366f6083SPeter Grehan struct vmx *vmx = arg; 2463366f6083SPeter Grehan struct vmcs *vmcs = &vmx->vmcs[vcpu]; 2464366f6083SPeter Grehan uint32_t baseval; 2465366f6083SPeter Grehan uint32_t *pptr; 2466366f6083SPeter Grehan int error; 2467366f6083SPeter Grehan int flag; 2468366f6083SPeter Grehan int reg; 2469366f6083SPeter Grehan int retval; 2470366f6083SPeter Grehan 2471366f6083SPeter Grehan retval = ENOENT; 2472366f6083SPeter Grehan pptr = NULL; 2473366f6083SPeter Grehan 2474366f6083SPeter Grehan switch (type) { 2475366f6083SPeter Grehan case VM_CAP_HALT_EXIT: 2476366f6083SPeter Grehan if (cap_halt_exit) { 2477366f6083SPeter Grehan retval = 0; 2478366f6083SPeter Grehan pptr = &vmx->cap[vcpu].proc_ctls; 2479366f6083SPeter Grehan baseval = *pptr; 2480366f6083SPeter Grehan flag = PROCBASED_HLT_EXITING; 2481366f6083SPeter Grehan reg = VMCS_PRI_PROC_BASED_CTLS; 2482366f6083SPeter Grehan } 2483366f6083SPeter Grehan break; 2484366f6083SPeter Grehan case VM_CAP_MTRAP_EXIT: 2485366f6083SPeter Grehan if (cap_monitor_trap) { 2486366f6083SPeter Grehan retval = 0; 2487366f6083SPeter Grehan pptr = &vmx->cap[vcpu].proc_ctls; 2488366f6083SPeter Grehan baseval = *pptr; 2489366f6083SPeter Grehan flag = PROCBASED_MTF; 2490366f6083SPeter Grehan reg = VMCS_PRI_PROC_BASED_CTLS; 2491366f6083SPeter Grehan } 2492366f6083SPeter Grehan break; 2493366f6083SPeter Grehan case VM_CAP_PAUSE_EXIT: 2494366f6083SPeter Grehan if (cap_pause_exit) { 2495366f6083SPeter Grehan retval = 0; 2496366f6083SPeter Grehan pptr = &vmx->cap[vcpu].proc_ctls; 2497366f6083SPeter Grehan baseval = *pptr; 2498366f6083SPeter Grehan flag = PROCBASED_PAUSE_EXITING; 2499366f6083SPeter Grehan reg = VMCS_PRI_PROC_BASED_CTLS; 2500366f6083SPeter Grehan } 2501366f6083SPeter Grehan break; 2502366f6083SPeter Grehan case VM_CAP_UNRESTRICTED_GUEST: 2503366f6083SPeter Grehan if (cap_unrestricted_guest) { 2504366f6083SPeter Grehan retval = 0; 250549cc03daSNeel Natu pptr = &vmx->cap[vcpu].proc_ctls2; 250649cc03daSNeel Natu baseval = *pptr; 2507366f6083SPeter Grehan flag = PROCBASED2_UNRESTRICTED_GUEST; 2508366f6083SPeter Grehan reg = VMCS_SEC_PROC_BASED_CTLS; 2509366f6083SPeter Grehan } 2510366f6083SPeter Grehan break; 251149cc03daSNeel Natu case VM_CAP_ENABLE_INVPCID: 251249cc03daSNeel Natu if (cap_invpcid) { 251349cc03daSNeel Natu retval = 0; 251449cc03daSNeel Natu pptr = &vmx->cap[vcpu].proc_ctls2; 251549cc03daSNeel Natu baseval = *pptr; 251649cc03daSNeel Natu flag = PROCBASED2_ENABLE_INVPCID; 251749cc03daSNeel Natu reg = VMCS_SEC_PROC_BASED_CTLS; 251849cc03daSNeel Natu } 251949cc03daSNeel Natu break; 2520366f6083SPeter Grehan default: 2521366f6083SPeter Grehan break; 2522366f6083SPeter Grehan } 2523366f6083SPeter Grehan 2524366f6083SPeter Grehan if (retval == 0) { 2525366f6083SPeter Grehan if (val) { 2526366f6083SPeter Grehan baseval |= flag; 2527366f6083SPeter Grehan } else { 2528366f6083SPeter Grehan baseval &= ~flag; 2529366f6083SPeter Grehan } 2530366f6083SPeter Grehan VMPTRLD(vmcs); 2531366f6083SPeter Grehan error = vmwrite(reg, baseval); 2532366f6083SPeter Grehan VMCLEAR(vmcs); 2533366f6083SPeter Grehan 2534366f6083SPeter Grehan if (error) { 2535366f6083SPeter Grehan retval = error; 2536366f6083SPeter Grehan } else { 2537366f6083SPeter Grehan /* 2538366f6083SPeter Grehan * Update optional stored flags, and record 2539366f6083SPeter Grehan * setting 2540366f6083SPeter Grehan */ 2541366f6083SPeter Grehan if (pptr != NULL) { 2542366f6083SPeter Grehan *pptr = baseval; 2543366f6083SPeter Grehan } 2544366f6083SPeter Grehan 2545366f6083SPeter Grehan if (val) { 2546366f6083SPeter Grehan vmx->cap[vcpu].set |= (1 << type); 2547366f6083SPeter Grehan } else { 2548366f6083SPeter Grehan vmx->cap[vcpu].set &= ~(1 << type); 2549366f6083SPeter Grehan } 2550366f6083SPeter Grehan } 2551366f6083SPeter Grehan } 2552366f6083SPeter Grehan 2553366f6083SPeter Grehan return (retval); 2554366f6083SPeter Grehan } 2555366f6083SPeter Grehan 255688c4b8d1SNeel Natu struct vlapic_vtx { 255788c4b8d1SNeel Natu struct vlapic vlapic; 2558176666c2SNeel Natu struct pir_desc *pir_desc; 255930b94db8SNeel Natu struct vmx *vmx; 256088c4b8d1SNeel Natu }; 256188c4b8d1SNeel Natu 256288c4b8d1SNeel Natu #define VMX_CTR_PIR(vm, vcpuid, pir_desc, notify, vector, level, msg) \ 256388c4b8d1SNeel Natu do { \ 256488c4b8d1SNeel Natu VCPU_CTR2(vm, vcpuid, msg " assert %s-triggered vector %d", \ 256588c4b8d1SNeel Natu level ? "level" : "edge", vector); \ 256688c4b8d1SNeel Natu VCPU_CTR1(vm, vcpuid, msg " pir0 0x%016lx", pir_desc->pir[0]); \ 256788c4b8d1SNeel Natu VCPU_CTR1(vm, vcpuid, msg " pir1 0x%016lx", pir_desc->pir[1]); \ 256888c4b8d1SNeel Natu VCPU_CTR1(vm, vcpuid, msg " pir2 0x%016lx", pir_desc->pir[2]); \ 256988c4b8d1SNeel Natu VCPU_CTR1(vm, vcpuid, msg " pir3 0x%016lx", pir_desc->pir[3]); \ 257088c4b8d1SNeel Natu VCPU_CTR1(vm, vcpuid, msg " notify: %s", notify ? "yes" : "no");\ 257188c4b8d1SNeel Natu } while (0) 257288c4b8d1SNeel Natu 257388c4b8d1SNeel Natu /* 257488c4b8d1SNeel Natu * vlapic->ops handlers that utilize the APICv hardware assist described in 257588c4b8d1SNeel Natu * Chapter 29 of the Intel SDM. 257688c4b8d1SNeel Natu */ 257788c4b8d1SNeel Natu static int 257888c4b8d1SNeel Natu vmx_set_intr_ready(struct vlapic *vlapic, int vector, bool level) 257988c4b8d1SNeel Natu { 258088c4b8d1SNeel Natu struct vlapic_vtx *vlapic_vtx; 258188c4b8d1SNeel Natu struct pir_desc *pir_desc; 258288c4b8d1SNeel Natu uint64_t mask; 258388c4b8d1SNeel Natu int idx, notify; 258488c4b8d1SNeel Natu 258588c4b8d1SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 2586176666c2SNeel Natu pir_desc = vlapic_vtx->pir_desc; 258788c4b8d1SNeel Natu 258888c4b8d1SNeel Natu /* 258988c4b8d1SNeel Natu * Keep track of interrupt requests in the PIR descriptor. This is 259088c4b8d1SNeel Natu * because the virtual APIC page pointed to by the VMCS cannot be 259188c4b8d1SNeel Natu * modified if the vcpu is running. 259288c4b8d1SNeel Natu */ 259388c4b8d1SNeel Natu idx = vector / 64; 259488c4b8d1SNeel Natu mask = 1UL << (vector % 64); 259588c4b8d1SNeel Natu atomic_set_long(&pir_desc->pir[idx], mask); 259688c4b8d1SNeel Natu notify = atomic_cmpset_long(&pir_desc->pending, 0, 1); 259788c4b8d1SNeel Natu 259888c4b8d1SNeel Natu VMX_CTR_PIR(vlapic->vm, vlapic->vcpuid, pir_desc, notify, vector, 259988c4b8d1SNeel Natu level, "vmx_set_intr_ready"); 260088c4b8d1SNeel Natu return (notify); 260188c4b8d1SNeel Natu } 260288c4b8d1SNeel Natu 260388c4b8d1SNeel Natu static int 260488c4b8d1SNeel Natu vmx_pending_intr(struct vlapic *vlapic, int *vecptr) 260588c4b8d1SNeel Natu { 260688c4b8d1SNeel Natu struct vlapic_vtx *vlapic_vtx; 260788c4b8d1SNeel Natu struct pir_desc *pir_desc; 260888c4b8d1SNeel Natu struct LAPIC *lapic; 260988c4b8d1SNeel Natu uint64_t pending, pirval; 261088c4b8d1SNeel Natu uint32_t ppr, vpr; 261188c4b8d1SNeel Natu int i; 261288c4b8d1SNeel Natu 261388c4b8d1SNeel Natu /* 261488c4b8d1SNeel Natu * This function is only expected to be called from the 'HLT' exit 261588c4b8d1SNeel Natu * handler which does not care about the vector that is pending. 261688c4b8d1SNeel Natu */ 261788c4b8d1SNeel Natu KASSERT(vecptr == NULL, ("vmx_pending_intr: vecptr must be NULL")); 261888c4b8d1SNeel Natu 261988c4b8d1SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 2620176666c2SNeel Natu pir_desc = vlapic_vtx->pir_desc; 262188c4b8d1SNeel Natu 262288c4b8d1SNeel Natu pending = atomic_load_acq_long(&pir_desc->pending); 262388c4b8d1SNeel Natu if (!pending) 262488c4b8d1SNeel Natu return (0); /* common case */ 262588c4b8d1SNeel Natu 262688c4b8d1SNeel Natu /* 262788c4b8d1SNeel Natu * If there is an interrupt pending then it will be recognized only 262888c4b8d1SNeel Natu * if its priority is greater than the processor priority. 262988c4b8d1SNeel Natu * 263088c4b8d1SNeel Natu * Special case: if the processor priority is zero then any pending 263188c4b8d1SNeel Natu * interrupt will be recognized. 263288c4b8d1SNeel Natu */ 263388c4b8d1SNeel Natu lapic = vlapic->apic_page; 263488c4b8d1SNeel Natu ppr = lapic->ppr & 0xf0; 263588c4b8d1SNeel Natu if (ppr == 0) 263688c4b8d1SNeel Natu return (1); 263788c4b8d1SNeel Natu 263888c4b8d1SNeel Natu VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "HLT with non-zero PPR %d", 263988c4b8d1SNeel Natu lapic->ppr); 264088c4b8d1SNeel Natu 264188c4b8d1SNeel Natu for (i = 3; i >= 0; i--) { 264288c4b8d1SNeel Natu pirval = pir_desc->pir[i]; 264388c4b8d1SNeel Natu if (pirval != 0) { 264488c4b8d1SNeel Natu vpr = (i * 64 + flsl(pirval) - 1) & 0xf0; 264588c4b8d1SNeel Natu return (vpr > ppr); 264688c4b8d1SNeel Natu } 264788c4b8d1SNeel Natu } 264888c4b8d1SNeel Natu return (0); 264988c4b8d1SNeel Natu } 265088c4b8d1SNeel Natu 265188c4b8d1SNeel Natu static void 265288c4b8d1SNeel Natu vmx_intr_accepted(struct vlapic *vlapic, int vector) 265388c4b8d1SNeel Natu { 265488c4b8d1SNeel Natu 265588c4b8d1SNeel Natu panic("vmx_intr_accepted: not expected to be called"); 265688c4b8d1SNeel Natu } 265788c4b8d1SNeel Natu 2658176666c2SNeel Natu static void 265930b94db8SNeel Natu vmx_set_tmr(struct vlapic *vlapic, int vector, bool level) 266030b94db8SNeel Natu { 266130b94db8SNeel Natu struct vlapic_vtx *vlapic_vtx; 266230b94db8SNeel Natu struct vmx *vmx; 266330b94db8SNeel Natu struct vmcs *vmcs; 266430b94db8SNeel Natu uint64_t mask, val; 266530b94db8SNeel Natu 266630b94db8SNeel Natu KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector)); 266730b94db8SNeel Natu KASSERT(!vcpu_is_running(vlapic->vm, vlapic->vcpuid, NULL), 266830b94db8SNeel Natu ("vmx_set_tmr: vcpu cannot be running")); 266930b94db8SNeel Natu 267030b94db8SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 267130b94db8SNeel Natu vmx = vlapic_vtx->vmx; 267230b94db8SNeel Natu vmcs = &vmx->vmcs[vlapic->vcpuid]; 267330b94db8SNeel Natu mask = 1UL << (vector % 64); 267430b94db8SNeel Natu 267530b94db8SNeel Natu VMPTRLD(vmcs); 267630b94db8SNeel Natu val = vmcs_read(VMCS_EOI_EXIT(vector)); 267730b94db8SNeel Natu if (level) 267830b94db8SNeel Natu val |= mask; 267930b94db8SNeel Natu else 268030b94db8SNeel Natu val &= ~mask; 268130b94db8SNeel Natu vmcs_write(VMCS_EOI_EXIT(vector), val); 268230b94db8SNeel Natu VMCLEAR(vmcs); 268330b94db8SNeel Natu } 268430b94db8SNeel Natu 268530b94db8SNeel Natu static void 2686159dd56fSNeel Natu vmx_enable_x2apic_mode(struct vlapic *vlapic) 2687159dd56fSNeel Natu { 2688159dd56fSNeel Natu struct vmx *vmx; 2689159dd56fSNeel Natu struct vmcs *vmcs; 2690159dd56fSNeel Natu uint32_t proc_ctls2; 2691159dd56fSNeel Natu int vcpuid, error; 2692159dd56fSNeel Natu 2693159dd56fSNeel Natu vcpuid = vlapic->vcpuid; 2694159dd56fSNeel Natu vmx = ((struct vlapic_vtx *)vlapic)->vmx; 2695159dd56fSNeel Natu vmcs = &vmx->vmcs[vcpuid]; 2696159dd56fSNeel Natu 2697159dd56fSNeel Natu proc_ctls2 = vmx->cap[vcpuid].proc_ctls2; 2698159dd56fSNeel Natu KASSERT((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) != 0, 2699159dd56fSNeel Natu ("%s: invalid proc_ctls2 %#x", __func__, proc_ctls2)); 2700159dd56fSNeel Natu 2701159dd56fSNeel Natu proc_ctls2 &= ~PROCBASED2_VIRTUALIZE_APIC_ACCESSES; 2702159dd56fSNeel Natu proc_ctls2 |= PROCBASED2_VIRTUALIZE_X2APIC_MODE; 2703159dd56fSNeel Natu vmx->cap[vcpuid].proc_ctls2 = proc_ctls2; 2704159dd56fSNeel Natu 2705159dd56fSNeel Natu VMPTRLD(vmcs); 2706159dd56fSNeel Natu vmcs_write(VMCS_SEC_PROC_BASED_CTLS, proc_ctls2); 2707159dd56fSNeel Natu VMCLEAR(vmcs); 2708159dd56fSNeel Natu 2709159dd56fSNeel Natu if (vlapic->vcpuid == 0) { 2710159dd56fSNeel Natu /* 2711159dd56fSNeel Natu * The nested page table mappings are shared by all vcpus 2712159dd56fSNeel Natu * so unmap the APIC access page just once. 2713159dd56fSNeel Natu */ 2714159dd56fSNeel Natu error = vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE); 2715159dd56fSNeel Natu KASSERT(error == 0, ("%s: vm_unmap_mmio error %d", 2716159dd56fSNeel Natu __func__, error)); 2717159dd56fSNeel Natu 2718159dd56fSNeel Natu /* 2719159dd56fSNeel Natu * The MSR bitmap is shared by all vcpus so modify it only 2720159dd56fSNeel Natu * once in the context of vcpu 0. 2721159dd56fSNeel Natu */ 2722159dd56fSNeel Natu error = vmx_allow_x2apic_msrs(vmx); 2723159dd56fSNeel Natu KASSERT(error == 0, ("%s: vmx_allow_x2apic_msrs error %d", 2724159dd56fSNeel Natu __func__, error)); 2725159dd56fSNeel Natu } 2726159dd56fSNeel Natu } 2727159dd56fSNeel Natu 2728159dd56fSNeel Natu static void 2729176666c2SNeel Natu vmx_post_intr(struct vlapic *vlapic, int hostcpu) 2730176666c2SNeel Natu { 2731176666c2SNeel Natu 2732176666c2SNeel Natu ipi_cpu(hostcpu, pirvec); 2733176666c2SNeel Natu } 2734176666c2SNeel Natu 273588c4b8d1SNeel Natu /* 273688c4b8d1SNeel Natu * Transfer the pending interrupts in the PIR descriptor to the IRR 273788c4b8d1SNeel Natu * in the virtual APIC page. 273888c4b8d1SNeel Natu */ 273988c4b8d1SNeel Natu static void 274088c4b8d1SNeel Natu vmx_inject_pir(struct vlapic *vlapic) 274188c4b8d1SNeel Natu { 274288c4b8d1SNeel Natu struct vlapic_vtx *vlapic_vtx; 274388c4b8d1SNeel Natu struct pir_desc *pir_desc; 274488c4b8d1SNeel Natu struct LAPIC *lapic; 274588c4b8d1SNeel Natu uint64_t val, pirval; 27460e30c5c0SWarner Losh int rvi, pirbase = -1; 274788c4b8d1SNeel Natu uint16_t intr_status_old, intr_status_new; 274888c4b8d1SNeel Natu 274988c4b8d1SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 2750176666c2SNeel Natu pir_desc = vlapic_vtx->pir_desc; 275188c4b8d1SNeel Natu if (atomic_cmpset_long(&pir_desc->pending, 1, 0) == 0) { 275288c4b8d1SNeel Natu VCPU_CTR0(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: " 275388c4b8d1SNeel Natu "no posted interrupt pending"); 275488c4b8d1SNeel Natu return; 275588c4b8d1SNeel Natu } 275688c4b8d1SNeel Natu 275788c4b8d1SNeel Natu pirval = 0; 2758*201b1cccSPeter Grehan pirbase = -1; 275988c4b8d1SNeel Natu lapic = vlapic->apic_page; 276088c4b8d1SNeel Natu 276188c4b8d1SNeel Natu val = atomic_readandclear_long(&pir_desc->pir[0]); 276288c4b8d1SNeel Natu if (val != 0) { 276388c4b8d1SNeel Natu lapic->irr0 |= val; 276488c4b8d1SNeel Natu lapic->irr1 |= val >> 32; 276588c4b8d1SNeel Natu pirbase = 0; 276688c4b8d1SNeel Natu pirval = val; 276788c4b8d1SNeel Natu } 276888c4b8d1SNeel Natu 276988c4b8d1SNeel Natu val = atomic_readandclear_long(&pir_desc->pir[1]); 277088c4b8d1SNeel Natu if (val != 0) { 277188c4b8d1SNeel Natu lapic->irr2 |= val; 277288c4b8d1SNeel Natu lapic->irr3 |= val >> 32; 277388c4b8d1SNeel Natu pirbase = 64; 277488c4b8d1SNeel Natu pirval = val; 277588c4b8d1SNeel Natu } 277688c4b8d1SNeel Natu 277788c4b8d1SNeel Natu val = atomic_readandclear_long(&pir_desc->pir[2]); 277888c4b8d1SNeel Natu if (val != 0) { 277988c4b8d1SNeel Natu lapic->irr4 |= val; 278088c4b8d1SNeel Natu lapic->irr5 |= val >> 32; 278188c4b8d1SNeel Natu pirbase = 128; 278288c4b8d1SNeel Natu pirval = val; 278388c4b8d1SNeel Natu } 278488c4b8d1SNeel Natu 278588c4b8d1SNeel Natu val = atomic_readandclear_long(&pir_desc->pir[3]); 278688c4b8d1SNeel Natu if (val != 0) { 278788c4b8d1SNeel Natu lapic->irr6 |= val; 278888c4b8d1SNeel Natu lapic->irr7 |= val >> 32; 278988c4b8d1SNeel Natu pirbase = 192; 279088c4b8d1SNeel Natu pirval = val; 279188c4b8d1SNeel Natu } 2792*201b1cccSPeter Grehan 279388c4b8d1SNeel Natu VLAPIC_CTR_IRR(vlapic, "vmx_inject_pir"); 279488c4b8d1SNeel Natu 279588c4b8d1SNeel Natu /* 279688c4b8d1SNeel Natu * Update RVI so the processor can evaluate pending virtual 279788c4b8d1SNeel Natu * interrupts on VM-entry. 2798*201b1cccSPeter Grehan * 2799*201b1cccSPeter Grehan * It is possible for pirval to be 0 here, even though the 2800*201b1cccSPeter Grehan * pending bit has been set. The scenario is: 2801*201b1cccSPeter Grehan * CPU-Y is sending a posted interrupt to CPU-X, which 2802*201b1cccSPeter Grehan * is running a guest and processing posted interrupts in h/w. 2803*201b1cccSPeter Grehan * CPU-X will eventually exit and the state seen in s/w is 2804*201b1cccSPeter Grehan * the pending bit set, but no PIR bits set. 2805*201b1cccSPeter Grehan * 2806*201b1cccSPeter Grehan * CPU-X CPU-Y 2807*201b1cccSPeter Grehan * (vm running) (host running) 2808*201b1cccSPeter Grehan * rx posted interrupt 2809*201b1cccSPeter Grehan * CLEAR pending bit 2810*201b1cccSPeter Grehan * SET PIR bit 2811*201b1cccSPeter Grehan * READ/CLEAR PIR bits 2812*201b1cccSPeter Grehan * SET pending bit 2813*201b1cccSPeter Grehan * (vm exit) 2814*201b1cccSPeter Grehan * pending bit set, PIR 0 281588c4b8d1SNeel Natu */ 281688c4b8d1SNeel Natu if (pirval != 0) { 281788c4b8d1SNeel Natu rvi = pirbase + flsl(pirval) - 1; 281888c4b8d1SNeel Natu intr_status_old = vmcs_read(VMCS_GUEST_INTR_STATUS); 281988c4b8d1SNeel Natu intr_status_new = (intr_status_old & 0xFF00) | rvi; 282088c4b8d1SNeel Natu if (intr_status_new > intr_status_old) { 282188c4b8d1SNeel Natu vmcs_write(VMCS_GUEST_INTR_STATUS, intr_status_new); 282288c4b8d1SNeel Natu VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: " 282388c4b8d1SNeel Natu "guest_intr_status changed from 0x%04x to 0x%04x", 282488c4b8d1SNeel Natu intr_status_old, intr_status_new); 282588c4b8d1SNeel Natu } 282688c4b8d1SNeel Natu } 282788c4b8d1SNeel Natu } 282888c4b8d1SNeel Natu 2829de5ea6b6SNeel Natu static struct vlapic * 2830de5ea6b6SNeel Natu vmx_vlapic_init(void *arg, int vcpuid) 2831de5ea6b6SNeel Natu { 2832de5ea6b6SNeel Natu struct vmx *vmx; 2833de5ea6b6SNeel Natu struct vlapic *vlapic; 2834176666c2SNeel Natu struct vlapic_vtx *vlapic_vtx; 2835de5ea6b6SNeel Natu 2836de5ea6b6SNeel Natu vmx = arg; 2837de5ea6b6SNeel Natu 283888c4b8d1SNeel Natu vlapic = malloc(sizeof(struct vlapic_vtx), M_VLAPIC, M_WAITOK | M_ZERO); 2839de5ea6b6SNeel Natu vlapic->vm = vmx->vm; 2840de5ea6b6SNeel Natu vlapic->vcpuid = vcpuid; 2841de5ea6b6SNeel Natu vlapic->apic_page = (struct LAPIC *)&vmx->apic_page[vcpuid]; 2842de5ea6b6SNeel Natu 2843176666c2SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 2844176666c2SNeel Natu vlapic_vtx->pir_desc = &vmx->pir_desc[vcpuid]; 284530b94db8SNeel Natu vlapic_vtx->vmx = vmx; 2846176666c2SNeel Natu 284788c4b8d1SNeel Natu if (virtual_interrupt_delivery) { 284888c4b8d1SNeel Natu vlapic->ops.set_intr_ready = vmx_set_intr_ready; 284988c4b8d1SNeel Natu vlapic->ops.pending_intr = vmx_pending_intr; 285088c4b8d1SNeel Natu vlapic->ops.intr_accepted = vmx_intr_accepted; 285130b94db8SNeel Natu vlapic->ops.set_tmr = vmx_set_tmr; 2852159dd56fSNeel Natu vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode; 285388c4b8d1SNeel Natu } 285488c4b8d1SNeel Natu 2855176666c2SNeel Natu if (posted_interrupts) 2856176666c2SNeel Natu vlapic->ops.post_intr = vmx_post_intr; 2857176666c2SNeel Natu 2858de5ea6b6SNeel Natu vlapic_init(vlapic); 2859de5ea6b6SNeel Natu 2860de5ea6b6SNeel Natu return (vlapic); 2861de5ea6b6SNeel Natu } 2862de5ea6b6SNeel Natu 2863de5ea6b6SNeel Natu static void 2864de5ea6b6SNeel Natu vmx_vlapic_cleanup(void *arg, struct vlapic *vlapic) 2865de5ea6b6SNeel Natu { 2866de5ea6b6SNeel Natu 2867de5ea6b6SNeel Natu vlapic_cleanup(vlapic); 2868de5ea6b6SNeel Natu free(vlapic, M_VLAPIC); 2869de5ea6b6SNeel Natu } 2870de5ea6b6SNeel Natu 2871366f6083SPeter Grehan struct vmm_ops vmm_ops_intel = { 2872366f6083SPeter Grehan vmx_init, 2873366f6083SPeter Grehan vmx_cleanup, 287463e62d39SJohn Baldwin vmx_restore, 2875366f6083SPeter Grehan vmx_vminit, 2876366f6083SPeter Grehan vmx_run, 2877366f6083SPeter Grehan vmx_vmcleanup, 2878366f6083SPeter Grehan vmx_getreg, 2879366f6083SPeter Grehan vmx_setreg, 2880366f6083SPeter Grehan vmx_getdesc, 2881366f6083SPeter Grehan vmx_setdesc, 2882366f6083SPeter Grehan vmx_getcap, 2883318224bbSNeel Natu vmx_setcap, 2884318224bbSNeel Natu ept_vmspace_alloc, 2885318224bbSNeel Natu ept_vmspace_free, 2886de5ea6b6SNeel Natu vmx_vlapic_init, 2887de5ea6b6SNeel Natu vmx_vlapic_cleanup, 2888366f6083SPeter Grehan }; 2889