1366f6083SPeter Grehan /*- 2c49761ddSPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3c49761ddSPedro F. Giffuni * 4366f6083SPeter Grehan * Copyright (c) 2011 NetApp, Inc. 5366f6083SPeter Grehan * All rights reserved. 62c352febSJohn Baldwin * Copyright (c) 2018 Joyent, Inc. 7366f6083SPeter Grehan * 8366f6083SPeter Grehan * Redistribution and use in source and binary forms, with or without 9366f6083SPeter Grehan * modification, are permitted provided that the following conditions 10366f6083SPeter Grehan * are met: 11366f6083SPeter Grehan * 1. Redistributions of source code must retain the above copyright 12366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer. 13366f6083SPeter Grehan * 2. Redistributions in binary form must reproduce the above copyright 14366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer in the 15366f6083SPeter Grehan * documentation and/or other materials provided with the distribution. 16366f6083SPeter Grehan * 17366f6083SPeter Grehan * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 18366f6083SPeter Grehan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19366f6083SPeter Grehan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20366f6083SPeter Grehan * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 21366f6083SPeter Grehan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22366f6083SPeter Grehan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23366f6083SPeter Grehan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24366f6083SPeter Grehan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25366f6083SPeter Grehan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26366f6083SPeter Grehan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27366f6083SPeter Grehan * SUCH DAMAGE. 28366f6083SPeter Grehan * 29366f6083SPeter Grehan * $FreeBSD$ 30366f6083SPeter Grehan */ 31366f6083SPeter Grehan 32366f6083SPeter Grehan #include <sys/cdefs.h> 33366f6083SPeter Grehan __FBSDID("$FreeBSD$"); 34366f6083SPeter Grehan 35483d953aSJohn Baldwin #include "opt_bhyve_snapshot.h" 36483d953aSJohn Baldwin 37366f6083SPeter Grehan #include <sys/param.h> 38366f6083SPeter Grehan #include <sys/systm.h> 39366f6083SPeter Grehan #include <sys/smp.h> 40366f6083SPeter Grehan #include <sys/kernel.h> 41366f6083SPeter Grehan #include <sys/malloc.h> 42366f6083SPeter Grehan #include <sys/pcpu.h> 43366f6083SPeter Grehan #include <sys/proc.h> 44b7924341SAndrew Turner #include <sys/reg.h> 456f5a9606SMark Johnston #include <sys/smr.h> 463565b59eSNeel Natu #include <sys/sysctl.h> 47366f6083SPeter Grehan 48366f6083SPeter Grehan #include <vm/vm.h> 49366f6083SPeter Grehan #include <vm/pmap.h> 50366f6083SPeter Grehan 51366f6083SPeter Grehan #include <machine/psl.h> 52366f6083SPeter Grehan #include <machine/cpufunc.h> 538b287612SJohn Baldwin #include <machine/md_var.h> 54366f6083SPeter Grehan #include <machine/segments.h> 55176666c2SNeel Natu #include <machine/smp.h> 56608f97c3SPeter Grehan #include <machine/specialreg.h> 57366f6083SPeter Grehan #include <machine/vmparam.h> 58366f6083SPeter Grehan 59366f6083SPeter Grehan #include <machine/vmm.h> 60dc506506SNeel Natu #include <machine/vmm_dev.h> 61e813a873SNeel Natu #include <machine/vmm_instruction_emul.h> 62483d953aSJohn Baldwin #include <machine/vmm_snapshot.h> 63483d953aSJohn Baldwin 64c3498942SNeel Natu #include "vmm_lapic.h" 65b01c2033SNeel Natu #include "vmm_host.h" 66762fd208STycho Nightingale #include "vmm_ioport.h" 67366f6083SPeter Grehan #include "vmm_ktr.h" 68366f6083SPeter Grehan #include "vmm_stat.h" 690775fbb4STycho Nightingale #include "vatpic.h" 70de5ea6b6SNeel Natu #include "vlapic.h" 71de5ea6b6SNeel Natu #include "vlapic_priv.h" 72366f6083SPeter Grehan 73366f6083SPeter Grehan #include "ept.h" 74366f6083SPeter Grehan #include "vmx_cpufunc.h" 75366f6083SPeter Grehan #include "vmx.h" 76c3498942SNeel Natu #include "vmx_msr.h" 77366f6083SPeter Grehan #include "x86.h" 78366f6083SPeter Grehan #include "vmx_controls.h" 79366f6083SPeter Grehan 80366f6083SPeter Grehan #define PINBASED_CTLS_ONE_SETTING \ 81366f6083SPeter Grehan (PINBASED_EXTINT_EXITING | \ 82366f6083SPeter Grehan PINBASED_NMI_EXITING | \ 83366f6083SPeter Grehan PINBASED_VIRTUAL_NMI) 84366f6083SPeter Grehan #define PINBASED_CTLS_ZERO_SETTING 0 85366f6083SPeter Grehan 86366f6083SPeter Grehan #define PROCBASED_CTLS_WINDOW_SETTING \ 87366f6083SPeter Grehan (PROCBASED_INT_WINDOW_EXITING | \ 88366f6083SPeter Grehan PROCBASED_NMI_WINDOW_EXITING) 89366f6083SPeter Grehan 90366f6083SPeter Grehan #define PROCBASED_CTLS_ONE_SETTING \ 91366f6083SPeter Grehan (PROCBASED_SECONDARY_CONTROLS | \ 9265145c7fSNeel Natu PROCBASED_MWAIT_EXITING | \ 9365145c7fSNeel Natu PROCBASED_MONITOR_EXITING | \ 94366f6083SPeter Grehan PROCBASED_IO_EXITING | \ 95366f6083SPeter Grehan PROCBASED_MSR_BITMAPS | \ 96594db002STycho Nightingale PROCBASED_CTLS_WINDOW_SETTING | \ 97594db002STycho Nightingale PROCBASED_CR8_LOAD_EXITING | \ 98594db002STycho Nightingale PROCBASED_CR8_STORE_EXITING) 99366f6083SPeter Grehan #define PROCBASED_CTLS_ZERO_SETTING \ 100366f6083SPeter Grehan (PROCBASED_CR3_LOAD_EXITING | \ 101366f6083SPeter Grehan PROCBASED_CR3_STORE_EXITING | \ 102366f6083SPeter Grehan PROCBASED_IO_BITMAPS) 103366f6083SPeter Grehan 104366f6083SPeter Grehan #define PROCBASED_CTLS2_ONE_SETTING PROCBASED2_ENABLE_EPT 105366f6083SPeter Grehan #define PROCBASED_CTLS2_ZERO_SETTING 0 106366f6083SPeter Grehan 107d72978ecSNeel Natu #define VM_EXIT_CTLS_ONE_SETTING \ 10865eefbe4SJohn Baldwin (VM_EXIT_SAVE_DEBUG_CONTROLS | \ 10965eefbe4SJohn Baldwin VM_EXIT_HOST_LMA | \ 110366f6083SPeter Grehan VM_EXIT_SAVE_EFER | \ 111d72978ecSNeel Natu VM_EXIT_LOAD_EFER | \ 112a318f7ddSNeel Natu VM_EXIT_ACKNOWLEDGE_INTERRUPT) 113d72978ecSNeel Natu 11465eefbe4SJohn Baldwin #define VM_EXIT_CTLS_ZERO_SETTING 0 115366f6083SPeter Grehan 11665eefbe4SJohn Baldwin #define VM_ENTRY_CTLS_ONE_SETTING \ 11765eefbe4SJohn Baldwin (VM_ENTRY_LOAD_DEBUG_CONTROLS | \ 11865eefbe4SJohn Baldwin VM_ENTRY_LOAD_EFER) 119608f97c3SPeter Grehan 120366f6083SPeter Grehan #define VM_ENTRY_CTLS_ZERO_SETTING \ 12165eefbe4SJohn Baldwin (VM_ENTRY_INTO_SMM | \ 122366f6083SPeter Grehan VM_ENTRY_DEACTIVATE_DUAL_MONITOR) 123366f6083SPeter Grehan 124366f6083SPeter Grehan #define HANDLED 1 125366f6083SPeter Grehan #define UNHANDLED 0 126366f6083SPeter Grehan 127de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VMX, "vmx", "vmx"); 128de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic"); 129366f6083SPeter Grehan 1303565b59eSNeel Natu SYSCTL_DECL(_hw_vmm); 131b40598c5SPawel Biernacki SYSCTL_NODE(_hw_vmm, OID_AUTO, vmx, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 132b40598c5SPawel Biernacki NULL); 1333565b59eSNeel Natu 134b3996dd4SJohn Baldwin int vmxon_enabled[MAXCPU]; 135366f6083SPeter Grehan static char vmxon_region[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE); 136366f6083SPeter Grehan 137366f6083SPeter Grehan static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2; 138366f6083SPeter Grehan static uint32_t exit_ctls, entry_ctls; 139366f6083SPeter Grehan 140366f6083SPeter Grehan static uint64_t cr0_ones_mask, cr0_zeros_mask; 1413565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_ones_mask, CTLFLAG_RD, 1423565b59eSNeel Natu &cr0_ones_mask, 0, NULL); 1433565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_zeros_mask, CTLFLAG_RD, 1443565b59eSNeel Natu &cr0_zeros_mask, 0, NULL); 1453565b59eSNeel Natu 146366f6083SPeter Grehan static uint64_t cr4_ones_mask, cr4_zeros_mask; 1473565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_ones_mask, CTLFLAG_RD, 1483565b59eSNeel Natu &cr4_ones_mask, 0, NULL); 1493565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_zeros_mask, CTLFLAG_RD, 1503565b59eSNeel Natu &cr4_zeros_mask, 0, NULL); 151366f6083SPeter Grehan 1523565b59eSNeel Natu static int vmx_initialized; 1533565b59eSNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, initialized, CTLFLAG_RD, 1543565b59eSNeel Natu &vmx_initialized, 0, "Intel VMX initialized"); 1553565b59eSNeel Natu 156366f6083SPeter Grehan /* 157366f6083SPeter Grehan * Optional capabilities 158366f6083SPeter Grehan */ 159b40598c5SPawel Biernacki static SYSCTL_NODE(_hw_vmm_vmx, OID_AUTO, cap, 160b40598c5SPawel Biernacki CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 161b40598c5SPawel Biernacki NULL); 16206fc6db9SJohn Baldwin 163366f6083SPeter Grehan static int cap_halt_exit; 16406fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, halt_exit, CTLFLAG_RD, &cap_halt_exit, 0, 16506fc6db9SJohn Baldwin "HLT triggers a VM-exit"); 16606fc6db9SJohn Baldwin 167366f6083SPeter Grehan static int cap_pause_exit; 16806fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, pause_exit, CTLFLAG_RD, &cap_pause_exit, 16906fc6db9SJohn Baldwin 0, "PAUSE triggers a VM-exit"); 17006fc6db9SJohn Baldwin 171*3ba952e1SCorvin Köhne static int cap_wbinvd_exit; 172*3ba952e1SCorvin Köhne SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, wbinvd_exit, CTLFLAG_RD, &cap_wbinvd_exit, 173*3ba952e1SCorvin Köhne 0, "WBINVD triggers a VM-exit"); 174*3ba952e1SCorvin Köhne 175f5f5f1e7SPeter Grehan static int cap_rdpid; 176f5f5f1e7SPeter Grehan SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, rdpid, CTLFLAG_RD, &cap_rdpid, 0, 177f5f5f1e7SPeter Grehan "Guests are allowed to use RDPID"); 178f5f5f1e7SPeter Grehan 179f5f5f1e7SPeter Grehan static int cap_rdtscp; 180f5f5f1e7SPeter Grehan SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, rdtscp, CTLFLAG_RD, &cap_rdtscp, 0, 181f5f5f1e7SPeter Grehan "Guests are allowed to use RDTSCP"); 182f5f5f1e7SPeter Grehan 183366f6083SPeter Grehan static int cap_unrestricted_guest; 18406fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, unrestricted_guest, CTLFLAG_RD, 18506fc6db9SJohn Baldwin &cap_unrestricted_guest, 0, "Unrestricted guests"); 18606fc6db9SJohn Baldwin 187366f6083SPeter Grehan static int cap_monitor_trap; 18806fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, monitor_trap, CTLFLAG_RD, 18906fc6db9SJohn Baldwin &cap_monitor_trap, 0, "Monitor trap flag"); 19006fc6db9SJohn Baldwin 19149cc03daSNeel Natu static int cap_invpcid; 19206fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, invpcid, CTLFLAG_RD, &cap_invpcid, 19306fc6db9SJohn Baldwin 0, "Guests are allowed to use INVPCID"); 194366f6083SPeter Grehan 1951bc51badSMichael Reifenberger static int tpr_shadowing; 1961bc51badSMichael Reifenberger SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, tpr_shadowing, CTLFLAG_RD, 1971bc51badSMichael Reifenberger &tpr_shadowing, 0, "TPR shadowing support"); 1981bc51badSMichael Reifenberger 19988c4b8d1SNeel Natu static int virtual_interrupt_delivery; 20006fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, virtual_interrupt_delivery, CTLFLAG_RD, 20188c4b8d1SNeel Natu &virtual_interrupt_delivery, 0, "APICv virtual interrupt delivery support"); 20288c4b8d1SNeel Natu 203176666c2SNeel Natu static int posted_interrupts; 20406fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, posted_interrupts, CTLFLAG_RD, 205176666c2SNeel Natu &posted_interrupts, 0, "APICv posted interrupt support"); 206176666c2SNeel Natu 20718a2b08eSNeel Natu static int pirvec = -1; 208176666c2SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupt_vector, CTLFLAG_RD, 209176666c2SNeel Natu &pirvec, 0, "APICv posted interrupt vector"); 210176666c2SNeel Natu 21145e51299SNeel Natu static struct unrhdr *vpid_unr; 21245e51299SNeel Natu static u_int vpid_alloc_failed; 21345e51299SNeel Natu SYSCTL_UINT(_hw_vmm_vmx, OID_AUTO, vpid_alloc_failed, CTLFLAG_RD, 21445e51299SNeel Natu &vpid_alloc_failed, 0, NULL); 21545e51299SNeel Natu 216d3588766SMark Johnston int guest_l1d_flush; 217c30578feSKonstantin Belousov SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, l1d_flush, CTLFLAG_RD, 218c30578feSKonstantin Belousov &guest_l1d_flush, 0, NULL); 219d3588766SMark Johnston int guest_l1d_flush_sw; 220c1141fbaSKonstantin Belousov SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, l1d_flush_sw, CTLFLAG_RD, 221c1141fbaSKonstantin Belousov &guest_l1d_flush_sw, 0, NULL); 222c30578feSKonstantin Belousov 223c1141fbaSKonstantin Belousov static struct msr_entry msr_load_list[1] __aligned(16); 224c30578feSKonstantin Belousov 22588c4b8d1SNeel Natu /* 2266ac73777STycho Nightingale * The definitions of SDT probes for VMX. 2276ac73777STycho Nightingale */ 2286ac73777STycho Nightingale 2296ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, entry, 2306ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2316ac73777STycho Nightingale 2326ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, taskswitch, 2336ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *", "struct vm_task_switch *"); 2346ac73777STycho Nightingale 2356ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, craccess, 2366ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *", "uint64_t"); 2376ac73777STycho Nightingale 2386ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, rdmsr, 2396ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *", "uint32_t"); 2406ac73777STycho Nightingale 2416ac73777STycho Nightingale SDT_PROBE_DEFINE5(vmm, vmx, exit, wrmsr, 2426ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *", "uint32_t", "uint64_t"); 2436ac73777STycho Nightingale 2446ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, halt, 2456ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2466ac73777STycho Nightingale 2476ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, mtrap, 2486ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2496ac73777STycho Nightingale 2506ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, pause, 2516ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2526ac73777STycho Nightingale 2536ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, intrwindow, 2546ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2556ac73777STycho Nightingale 2566ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, interrupt, 2576ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *", "uint32_t"); 2586ac73777STycho Nightingale 2596ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, nmiwindow, 2606ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2616ac73777STycho Nightingale 2626ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, inout, 2636ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2646ac73777STycho Nightingale 2656ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, cpuid, 2666ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2676ac73777STycho Nightingale 2686ac73777STycho Nightingale SDT_PROBE_DEFINE5(vmm, vmx, exit, exception, 2696ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *", "uint32_t", "int"); 2706ac73777STycho Nightingale 2716ac73777STycho Nightingale SDT_PROBE_DEFINE5(vmm, vmx, exit, nestedfault, 2726ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *", "uint64_t", "uint64_t"); 2736ac73777STycho Nightingale 2746ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, mmiofault, 2756ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *", "uint64_t"); 2766ac73777STycho Nightingale 2776ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, eoi, 2786ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2796ac73777STycho Nightingale 2806ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, apicaccess, 2816ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2826ac73777STycho Nightingale 2836ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, apicwrite, 2846ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *", "struct vlapic *"); 2856ac73777STycho Nightingale 2866ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, xsetbv, 2876ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2886ac73777STycho Nightingale 2896ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, monitor, 2906ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2916ac73777STycho Nightingale 2926ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, mwait, 2936ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2946ac73777STycho Nightingale 29527d26457SAndrew Turner SDT_PROBE_DEFINE3(vmm, vmx, exit, vminsn, 29627d26457SAndrew Turner "struct vmx *", "int", "struct vm_exit *"); 29727d26457SAndrew Turner 2986ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, unknown, 2996ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *", "uint32_t"); 3006ac73777STycho Nightingale 3016ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, return, 3026ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *", "int"); 3036ac73777STycho Nightingale 3046ac73777STycho Nightingale /* 30588c4b8d1SNeel Natu * Use the last page below 4GB as the APIC access address. This address is 30688c4b8d1SNeel Natu * occupied by the boot firmware so it is guaranteed that it will not conflict 30788c4b8d1SNeel Natu * with a page in system memory. 30888c4b8d1SNeel Natu */ 30988c4b8d1SNeel Natu #define APIC_ACCESS_ADDRESS 0xFFFFF000 31088c4b8d1SNeel Natu 311d17b5104SNeel Natu static int vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc); 312d17b5104SNeel Natu static int vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval); 313c3498942SNeel Natu static int vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val); 31488c4b8d1SNeel Natu static void vmx_inject_pir(struct vlapic *vlapic); 315483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT 316483d953aSJohn Baldwin static int vmx_restore_tsc(void *arg, int vcpu, uint64_t now); 317483d953aSJohn Baldwin #endif 31888c4b8d1SNeel Natu 319f5f5f1e7SPeter Grehan static inline bool 320f5f5f1e7SPeter Grehan host_has_rdpid(void) 321f5f5f1e7SPeter Grehan { 322f5f5f1e7SPeter Grehan return ((cpu_stdext_feature2 & CPUID_STDEXT2_RDPID) != 0); 323f5f5f1e7SPeter Grehan } 324f5f5f1e7SPeter Grehan 325f5f5f1e7SPeter Grehan static inline bool 326f5f5f1e7SPeter Grehan host_has_rdtscp(void) 327f5f5f1e7SPeter Grehan { 328f5f5f1e7SPeter Grehan return ((amd_feature & AMDID_RDTSCP) != 0); 329f5f5f1e7SPeter Grehan } 330f5f5f1e7SPeter Grehan 331366f6083SPeter Grehan #ifdef KTR 332366f6083SPeter Grehan static const char * 333366f6083SPeter Grehan exit_reason_to_str(int reason) 334366f6083SPeter Grehan { 335366f6083SPeter Grehan static char reasonbuf[32]; 336366f6083SPeter Grehan 337366f6083SPeter Grehan switch (reason) { 338366f6083SPeter Grehan case EXIT_REASON_EXCEPTION: 339366f6083SPeter Grehan return "exception"; 340366f6083SPeter Grehan case EXIT_REASON_EXT_INTR: 341366f6083SPeter Grehan return "extint"; 342366f6083SPeter Grehan case EXIT_REASON_TRIPLE_FAULT: 343366f6083SPeter Grehan return "triplefault"; 344366f6083SPeter Grehan case EXIT_REASON_INIT: 345366f6083SPeter Grehan return "init"; 346366f6083SPeter Grehan case EXIT_REASON_SIPI: 347366f6083SPeter Grehan return "sipi"; 348366f6083SPeter Grehan case EXIT_REASON_IO_SMI: 349366f6083SPeter Grehan return "iosmi"; 350366f6083SPeter Grehan case EXIT_REASON_SMI: 351366f6083SPeter Grehan return "smi"; 352366f6083SPeter Grehan case EXIT_REASON_INTR_WINDOW: 353366f6083SPeter Grehan return "intrwindow"; 354366f6083SPeter Grehan case EXIT_REASON_NMI_WINDOW: 355366f6083SPeter Grehan return "nmiwindow"; 356366f6083SPeter Grehan case EXIT_REASON_TASK_SWITCH: 357366f6083SPeter Grehan return "taskswitch"; 358366f6083SPeter Grehan case EXIT_REASON_CPUID: 359366f6083SPeter Grehan return "cpuid"; 360366f6083SPeter Grehan case EXIT_REASON_GETSEC: 361366f6083SPeter Grehan return "getsec"; 362366f6083SPeter Grehan case EXIT_REASON_HLT: 363366f6083SPeter Grehan return "hlt"; 364366f6083SPeter Grehan case EXIT_REASON_INVD: 365366f6083SPeter Grehan return "invd"; 366366f6083SPeter Grehan case EXIT_REASON_INVLPG: 367366f6083SPeter Grehan return "invlpg"; 368366f6083SPeter Grehan case EXIT_REASON_RDPMC: 369366f6083SPeter Grehan return "rdpmc"; 370366f6083SPeter Grehan case EXIT_REASON_RDTSC: 371366f6083SPeter Grehan return "rdtsc"; 372366f6083SPeter Grehan case EXIT_REASON_RSM: 373366f6083SPeter Grehan return "rsm"; 374366f6083SPeter Grehan case EXIT_REASON_VMCALL: 375366f6083SPeter Grehan return "vmcall"; 376366f6083SPeter Grehan case EXIT_REASON_VMCLEAR: 377366f6083SPeter Grehan return "vmclear"; 378366f6083SPeter Grehan case EXIT_REASON_VMLAUNCH: 379366f6083SPeter Grehan return "vmlaunch"; 380366f6083SPeter Grehan case EXIT_REASON_VMPTRLD: 381366f6083SPeter Grehan return "vmptrld"; 382366f6083SPeter Grehan case EXIT_REASON_VMPTRST: 383366f6083SPeter Grehan return "vmptrst"; 384366f6083SPeter Grehan case EXIT_REASON_VMREAD: 385366f6083SPeter Grehan return "vmread"; 386366f6083SPeter Grehan case EXIT_REASON_VMRESUME: 387366f6083SPeter Grehan return "vmresume"; 388366f6083SPeter Grehan case EXIT_REASON_VMWRITE: 389366f6083SPeter Grehan return "vmwrite"; 390366f6083SPeter Grehan case EXIT_REASON_VMXOFF: 391366f6083SPeter Grehan return "vmxoff"; 392366f6083SPeter Grehan case EXIT_REASON_VMXON: 393366f6083SPeter Grehan return "vmxon"; 394366f6083SPeter Grehan case EXIT_REASON_CR_ACCESS: 395366f6083SPeter Grehan return "craccess"; 396366f6083SPeter Grehan case EXIT_REASON_DR_ACCESS: 397366f6083SPeter Grehan return "draccess"; 398366f6083SPeter Grehan case EXIT_REASON_INOUT: 399366f6083SPeter Grehan return "inout"; 400366f6083SPeter Grehan case EXIT_REASON_RDMSR: 401366f6083SPeter Grehan return "rdmsr"; 402366f6083SPeter Grehan case EXIT_REASON_WRMSR: 403366f6083SPeter Grehan return "wrmsr"; 404366f6083SPeter Grehan case EXIT_REASON_INVAL_VMCS: 405366f6083SPeter Grehan return "invalvmcs"; 406366f6083SPeter Grehan case EXIT_REASON_INVAL_MSR: 407366f6083SPeter Grehan return "invalmsr"; 408366f6083SPeter Grehan case EXIT_REASON_MWAIT: 409366f6083SPeter Grehan return "mwait"; 410366f6083SPeter Grehan case EXIT_REASON_MTF: 411366f6083SPeter Grehan return "mtf"; 412366f6083SPeter Grehan case EXIT_REASON_MONITOR: 413366f6083SPeter Grehan return "monitor"; 414366f6083SPeter Grehan case EXIT_REASON_PAUSE: 415366f6083SPeter Grehan return "pause"; 416b0538143SNeel Natu case EXIT_REASON_MCE_DURING_ENTRY: 417b0538143SNeel Natu return "mce-during-entry"; 418366f6083SPeter Grehan case EXIT_REASON_TPR: 419366f6083SPeter Grehan return "tpr"; 42088c4b8d1SNeel Natu case EXIT_REASON_APIC_ACCESS: 42188c4b8d1SNeel Natu return "apic-access"; 422366f6083SPeter Grehan case EXIT_REASON_GDTR_IDTR: 423366f6083SPeter Grehan return "gdtridtr"; 424366f6083SPeter Grehan case EXIT_REASON_LDTR_TR: 425366f6083SPeter Grehan return "ldtrtr"; 426366f6083SPeter Grehan case EXIT_REASON_EPT_FAULT: 427366f6083SPeter Grehan return "eptfault"; 428366f6083SPeter Grehan case EXIT_REASON_EPT_MISCONFIG: 429366f6083SPeter Grehan return "eptmisconfig"; 430366f6083SPeter Grehan case EXIT_REASON_INVEPT: 431366f6083SPeter Grehan return "invept"; 432366f6083SPeter Grehan case EXIT_REASON_RDTSCP: 433366f6083SPeter Grehan return "rdtscp"; 434366f6083SPeter Grehan case EXIT_REASON_VMX_PREEMPT: 435366f6083SPeter Grehan return "vmxpreempt"; 436366f6083SPeter Grehan case EXIT_REASON_INVVPID: 437366f6083SPeter Grehan return "invvpid"; 438366f6083SPeter Grehan case EXIT_REASON_WBINVD: 439366f6083SPeter Grehan return "wbinvd"; 440366f6083SPeter Grehan case EXIT_REASON_XSETBV: 441366f6083SPeter Grehan return "xsetbv"; 44288c4b8d1SNeel Natu case EXIT_REASON_APIC_WRITE: 44388c4b8d1SNeel Natu return "apic-write"; 444366f6083SPeter Grehan default: 445366f6083SPeter Grehan snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason); 446366f6083SPeter Grehan return (reasonbuf); 447366f6083SPeter Grehan } 448366f6083SPeter Grehan } 449366f6083SPeter Grehan #endif /* KTR */ 450366f6083SPeter Grehan 451159dd56fSNeel Natu static int 452159dd56fSNeel Natu vmx_allow_x2apic_msrs(struct vmx *vmx) 453159dd56fSNeel Natu { 454159dd56fSNeel Natu int i, error; 455159dd56fSNeel Natu 456159dd56fSNeel Natu error = 0; 457159dd56fSNeel Natu 458159dd56fSNeel Natu /* 459159dd56fSNeel Natu * Allow readonly access to the following x2APIC MSRs from the guest. 460159dd56fSNeel Natu */ 461159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_ID); 462159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_VERSION); 463159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LDR); 464159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_SVR); 465159dd56fSNeel Natu 466159dd56fSNeel Natu for (i = 0; i < 8; i++) 467159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_ISR0 + i); 468159dd56fSNeel Natu 469159dd56fSNeel Natu for (i = 0; i < 8; i++) 470159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_TMR0 + i); 471159dd56fSNeel Natu 472159dd56fSNeel Natu for (i = 0; i < 8; i++) 473159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_IRR0 + i); 474159dd56fSNeel Natu 475159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_ESR); 476159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_TIMER); 477159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_THERMAL); 478159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_PCINT); 479159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT0); 480159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT1); 481159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_ERROR); 482159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_ICR_TIMER); 483159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_DCR_TIMER); 484159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_ICR); 485159dd56fSNeel Natu 486159dd56fSNeel Natu /* 487159dd56fSNeel Natu * Allow TPR, EOI and SELF_IPI MSRs to be read and written by the guest. 488159dd56fSNeel Natu * 489159dd56fSNeel Natu * These registers get special treatment described in the section 490159dd56fSNeel Natu * "Virtualizing MSR-Based APIC Accesses". 491159dd56fSNeel Natu */ 492159dd56fSNeel Natu error += guest_msr_rw(vmx, MSR_APIC_TPR); 493159dd56fSNeel Natu error += guest_msr_rw(vmx, MSR_APIC_EOI); 494159dd56fSNeel Natu error += guest_msr_rw(vmx, MSR_APIC_SELF_IPI); 495159dd56fSNeel Natu 496159dd56fSNeel Natu return (error); 497159dd56fSNeel Natu } 498159dd56fSNeel Natu 499366f6083SPeter Grehan u_long 500366f6083SPeter Grehan vmx_fix_cr0(u_long cr0) 501366f6083SPeter Grehan { 502366f6083SPeter Grehan 503366f6083SPeter Grehan return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask); 504366f6083SPeter Grehan } 505366f6083SPeter Grehan 506366f6083SPeter Grehan u_long 507366f6083SPeter Grehan vmx_fix_cr4(u_long cr4) 508366f6083SPeter Grehan { 509366f6083SPeter Grehan 510366f6083SPeter Grehan return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask); 511366f6083SPeter Grehan } 512366f6083SPeter Grehan 513366f6083SPeter Grehan static void 51445e51299SNeel Natu vpid_free(int vpid) 51545e51299SNeel Natu { 51645e51299SNeel Natu if (vpid < 0 || vpid > 0xffff) 51745e51299SNeel Natu panic("vpid_free: invalid vpid %d", vpid); 51845e51299SNeel Natu 51945e51299SNeel Natu /* 52045e51299SNeel Natu * VPIDs [0,VM_MAXCPU] are special and are not allocated from 52145e51299SNeel Natu * the unit number allocator. 52245e51299SNeel Natu */ 52345e51299SNeel Natu 52445e51299SNeel Natu if (vpid > VM_MAXCPU) 52545e51299SNeel Natu free_unr(vpid_unr, vpid); 52645e51299SNeel Natu } 52745e51299SNeel Natu 52845e51299SNeel Natu static void 52945e51299SNeel Natu vpid_alloc(uint16_t *vpid, int num) 53045e51299SNeel Natu { 53145e51299SNeel Natu int i, x; 53245e51299SNeel Natu 53345e51299SNeel Natu if (num <= 0 || num > VM_MAXCPU) 53445e51299SNeel Natu panic("invalid number of vpids requested: %d", num); 53545e51299SNeel Natu 53645e51299SNeel Natu /* 53745e51299SNeel Natu * If the "enable vpid" execution control is not enabled then the 53845e51299SNeel Natu * VPID is required to be 0 for all vcpus. 53945e51299SNeel Natu */ 54045e51299SNeel Natu if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) == 0) { 54145e51299SNeel Natu for (i = 0; i < num; i++) 54245e51299SNeel Natu vpid[i] = 0; 54345e51299SNeel Natu return; 54445e51299SNeel Natu } 54545e51299SNeel Natu 54645e51299SNeel Natu /* 54745e51299SNeel Natu * Allocate a unique VPID for each vcpu from the unit number allocator. 54845e51299SNeel Natu */ 54945e51299SNeel Natu for (i = 0; i < num; i++) { 55045e51299SNeel Natu x = alloc_unr(vpid_unr); 55145e51299SNeel Natu if (x == -1) 55245e51299SNeel Natu break; 55345e51299SNeel Natu else 55445e51299SNeel Natu vpid[i] = x; 55545e51299SNeel Natu } 55645e51299SNeel Natu 55745e51299SNeel Natu if (i < num) { 55845e51299SNeel Natu atomic_add_int(&vpid_alloc_failed, 1); 55945e51299SNeel Natu 56045e51299SNeel Natu /* 56145e51299SNeel Natu * If the unit number allocator does not have enough unique 56245e51299SNeel Natu * VPIDs then we need to allocate from the [1,VM_MAXCPU] range. 56345e51299SNeel Natu * 56445e51299SNeel Natu * These VPIDs are not be unique across VMs but this does not 56545e51299SNeel Natu * affect correctness because the combined mappings are also 56645e51299SNeel Natu * tagged with the EP4TA which is unique for each VM. 56745e51299SNeel Natu * 56845e51299SNeel Natu * It is still sub-optimal because the invvpid will invalidate 56945e51299SNeel Natu * combined mappings for a particular VPID across all EP4TAs. 57045e51299SNeel Natu */ 57145e51299SNeel Natu while (i-- > 0) 57245e51299SNeel Natu vpid_free(vpid[i]); 57345e51299SNeel Natu 57445e51299SNeel Natu for (i = 0; i < num; i++) 57545e51299SNeel Natu vpid[i] = i + 1; 57645e51299SNeel Natu } 57745e51299SNeel Natu } 57845e51299SNeel Natu 57945e51299SNeel Natu static void 58045e51299SNeel Natu vpid_init(void) 58145e51299SNeel Natu { 58245e51299SNeel Natu /* 58345e51299SNeel Natu * VPID 0 is required when the "enable VPID" execution control is 58445e51299SNeel Natu * disabled. 58545e51299SNeel Natu * 58645e51299SNeel Natu * VPIDs [1,VM_MAXCPU] are used as the "overflow namespace" when the 58745e51299SNeel Natu * unit number allocator does not have sufficient unique VPIDs to 58845e51299SNeel Natu * satisfy the allocation. 58945e51299SNeel Natu * 59045e51299SNeel Natu * The remaining VPIDs are managed by the unit number allocator. 59145e51299SNeel Natu */ 59245e51299SNeel Natu vpid_unr = new_unrhdr(VM_MAXCPU + 1, 0xffff, NULL); 59345e51299SNeel Natu } 59445e51299SNeel Natu 59545e51299SNeel Natu static void 596366f6083SPeter Grehan vmx_disable(void *arg __unused) 597366f6083SPeter Grehan { 598366f6083SPeter Grehan struct invvpid_desc invvpid_desc = { 0 }; 599366f6083SPeter Grehan struct invept_desc invept_desc = { 0 }; 600366f6083SPeter Grehan 601366f6083SPeter Grehan if (vmxon_enabled[curcpu]) { 602366f6083SPeter Grehan /* 603366f6083SPeter Grehan * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b. 604366f6083SPeter Grehan * 605366f6083SPeter Grehan * VMXON or VMXOFF are not required to invalidate any TLB 606366f6083SPeter Grehan * caching structures. This prevents potential retention of 607366f6083SPeter Grehan * cached information in the TLB between distinct VMX episodes. 608366f6083SPeter Grehan */ 609366f6083SPeter Grehan invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc); 610366f6083SPeter Grehan invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc); 611366f6083SPeter Grehan vmxoff(); 612366f6083SPeter Grehan } 613366f6083SPeter Grehan load_cr4(rcr4() & ~CR4_VMXE); 614366f6083SPeter Grehan } 615366f6083SPeter Grehan 616366f6083SPeter Grehan static int 61715add60dSPeter Grehan vmx_modcleanup(void) 618366f6083SPeter Grehan { 619366f6083SPeter Grehan 62018a2b08eSNeel Natu if (pirvec >= 0) 62118a2b08eSNeel Natu lapic_ipi_free(pirvec); 622176666c2SNeel Natu 62345e51299SNeel Natu if (vpid_unr != NULL) { 62445e51299SNeel Natu delete_unrhdr(vpid_unr); 62545e51299SNeel Natu vpid_unr = NULL; 62645e51299SNeel Natu } 62745e51299SNeel Natu 628c1141fbaSKonstantin Belousov if (nmi_flush_l1d_sw == 1) 629c1141fbaSKonstantin Belousov nmi_flush_l1d_sw = 0; 630c1141fbaSKonstantin Belousov 631366f6083SPeter Grehan smp_rendezvous(NULL, vmx_disable, NULL, NULL); 632366f6083SPeter Grehan 633366f6083SPeter Grehan return (0); 634366f6083SPeter Grehan } 635366f6083SPeter Grehan 636366f6083SPeter Grehan static void 637366f6083SPeter Grehan vmx_enable(void *arg __unused) 638366f6083SPeter Grehan { 639366f6083SPeter Grehan int error; 64011669a68STycho Nightingale uint64_t feature_control; 64111669a68STycho Nightingale 64211669a68STycho Nightingale feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL); 64311669a68STycho Nightingale if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 0 || 64411669a68STycho Nightingale (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) { 64511669a68STycho Nightingale wrmsr(MSR_IA32_FEATURE_CONTROL, 64611669a68STycho Nightingale feature_control | IA32_FEATURE_CONTROL_VMX_EN | 64711669a68STycho Nightingale IA32_FEATURE_CONTROL_LOCK); 64811669a68STycho Nightingale } 649366f6083SPeter Grehan 650366f6083SPeter Grehan load_cr4(rcr4() | CR4_VMXE); 651366f6083SPeter Grehan 652366f6083SPeter Grehan *(uint32_t *)vmxon_region[curcpu] = vmx_revision(); 653366f6083SPeter Grehan error = vmxon(vmxon_region[curcpu]); 654366f6083SPeter Grehan if (error == 0) 655366f6083SPeter Grehan vmxon_enabled[curcpu] = 1; 656366f6083SPeter Grehan } 657366f6083SPeter Grehan 65863e62d39SJohn Baldwin static void 65915add60dSPeter Grehan vmx_modresume(void) 66063e62d39SJohn Baldwin { 66163e62d39SJohn Baldwin 66263e62d39SJohn Baldwin if (vmxon_enabled[curcpu]) 66363e62d39SJohn Baldwin vmxon(vmxon_region[curcpu]); 66463e62d39SJohn Baldwin } 66563e62d39SJohn Baldwin 666366f6083SPeter Grehan static int 66715add60dSPeter Grehan vmx_modinit(int ipinum) 668366f6083SPeter Grehan { 6691bc51badSMichael Reifenberger int error; 670d17b5104SNeel Natu uint64_t basic, fixed0, fixed1, feature_control; 67188c4b8d1SNeel Natu uint32_t tmp, procbased2_vid_bits; 672366f6083SPeter Grehan 673366f6083SPeter Grehan /* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */ 6748b287612SJohn Baldwin if (!(cpu_feature2 & CPUID2_VMX)) { 67515add60dSPeter Grehan printf("vmx_modinit: processor does not support VMX " 67615add60dSPeter Grehan "operation\n"); 677366f6083SPeter Grehan return (ENXIO); 678366f6083SPeter Grehan } 679366f6083SPeter Grehan 6804bff7fadSNeel Natu /* 6814bff7fadSNeel Natu * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits 6824bff7fadSNeel Natu * are set (bits 0 and 2 respectively). 6834bff7fadSNeel Natu */ 6844bff7fadSNeel Natu feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL); 68511669a68STycho Nightingale if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 1 && 686150369abSNeel Natu (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) { 68715add60dSPeter Grehan printf("vmx_modinit: VMX operation disabled by BIOS\n"); 6884bff7fadSNeel Natu return (ENXIO); 6894bff7fadSNeel Natu } 6904bff7fadSNeel Natu 691d17b5104SNeel Natu /* 692d17b5104SNeel Natu * Verify capabilities MSR_VMX_BASIC: 693d17b5104SNeel Natu * - bit 54 indicates support for INS/OUTS decoding 694d17b5104SNeel Natu */ 695d17b5104SNeel Natu basic = rdmsr(MSR_VMX_BASIC); 696d17b5104SNeel Natu if ((basic & (1UL << 54)) == 0) { 69715add60dSPeter Grehan printf("vmx_modinit: processor does not support desired basic " 698d17b5104SNeel Natu "capabilities\n"); 699d17b5104SNeel Natu return (EINVAL); 700d17b5104SNeel Natu } 701d17b5104SNeel Natu 702366f6083SPeter Grehan /* Check support for primary processor-based VM-execution controls */ 703366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 704366f6083SPeter Grehan MSR_VMX_TRUE_PROCBASED_CTLS, 705366f6083SPeter Grehan PROCBASED_CTLS_ONE_SETTING, 706366f6083SPeter Grehan PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls); 707366f6083SPeter Grehan if (error) { 70815add60dSPeter Grehan printf("vmx_modinit: processor does not support desired " 70915add60dSPeter Grehan "primary processor-based controls\n"); 710366f6083SPeter Grehan return (error); 711366f6083SPeter Grehan } 712366f6083SPeter Grehan 713366f6083SPeter Grehan /* Clear the processor-based ctl bits that are set on demand */ 714366f6083SPeter Grehan procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING; 715366f6083SPeter Grehan 716366f6083SPeter Grehan /* Check support for secondary processor-based VM-execution controls */ 717366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 718366f6083SPeter Grehan MSR_VMX_PROCBASED_CTLS2, 719366f6083SPeter Grehan PROCBASED_CTLS2_ONE_SETTING, 720366f6083SPeter Grehan PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2); 721366f6083SPeter Grehan if (error) { 72215add60dSPeter Grehan printf("vmx_modinit: processor does not support desired " 72315add60dSPeter Grehan "secondary processor-based controls\n"); 724366f6083SPeter Grehan return (error); 725366f6083SPeter Grehan } 726366f6083SPeter Grehan 727366f6083SPeter Grehan /* Check support for VPID */ 728366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2, 729366f6083SPeter Grehan PROCBASED2_ENABLE_VPID, 0, &tmp); 730366f6083SPeter Grehan if (error == 0) 731366f6083SPeter Grehan procbased_ctls2 |= PROCBASED2_ENABLE_VPID; 732366f6083SPeter Grehan 733366f6083SPeter Grehan /* Check support for pin-based VM-execution controls */ 734366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS, 735366f6083SPeter Grehan MSR_VMX_TRUE_PINBASED_CTLS, 736366f6083SPeter Grehan PINBASED_CTLS_ONE_SETTING, 737366f6083SPeter Grehan PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls); 738366f6083SPeter Grehan if (error) { 73915add60dSPeter Grehan printf("vmx_modinit: processor does not support desired " 740366f6083SPeter Grehan "pin-based controls\n"); 741366f6083SPeter Grehan return (error); 742366f6083SPeter Grehan } 743366f6083SPeter Grehan 744366f6083SPeter Grehan /* Check support for VM-exit controls */ 745366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS, 746366f6083SPeter Grehan VM_EXIT_CTLS_ONE_SETTING, 747366f6083SPeter Grehan VM_EXIT_CTLS_ZERO_SETTING, 748366f6083SPeter Grehan &exit_ctls); 749366f6083SPeter Grehan if (error) { 75015add60dSPeter Grehan printf("vmx_modinit: processor does not support desired " 751366f6083SPeter Grehan "exit controls\n"); 752366f6083SPeter Grehan return (error); 753366f6083SPeter Grehan } 754366f6083SPeter Grehan 755366f6083SPeter Grehan /* Check support for VM-entry controls */ 756d72978ecSNeel Natu error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS, MSR_VMX_TRUE_ENTRY_CTLS, 757d72978ecSNeel Natu VM_ENTRY_CTLS_ONE_SETTING, VM_ENTRY_CTLS_ZERO_SETTING, 758366f6083SPeter Grehan &entry_ctls); 759366f6083SPeter Grehan if (error) { 76015add60dSPeter Grehan printf("vmx_modinit: processor does not support desired " 761366f6083SPeter Grehan "entry controls\n"); 762366f6083SPeter Grehan return (error); 763366f6083SPeter Grehan } 764366f6083SPeter Grehan 765366f6083SPeter Grehan /* 766366f6083SPeter Grehan * Check support for optional features by testing them 767366f6083SPeter Grehan * as individual bits 768366f6083SPeter Grehan */ 769366f6083SPeter Grehan cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 770366f6083SPeter Grehan MSR_VMX_TRUE_PROCBASED_CTLS, 771366f6083SPeter Grehan PROCBASED_HLT_EXITING, 0, 772366f6083SPeter Grehan &tmp) == 0); 773366f6083SPeter Grehan 774366f6083SPeter Grehan cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 775366f6083SPeter Grehan MSR_VMX_PROCBASED_CTLS, 776366f6083SPeter Grehan PROCBASED_MTF, 0, 777366f6083SPeter Grehan &tmp) == 0); 778366f6083SPeter Grehan 779366f6083SPeter Grehan cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 780366f6083SPeter Grehan MSR_VMX_TRUE_PROCBASED_CTLS, 781366f6083SPeter Grehan PROCBASED_PAUSE_EXITING, 0, 782366f6083SPeter Grehan &tmp) == 0); 783366f6083SPeter Grehan 784*3ba952e1SCorvin Köhne cap_wbinvd_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 785*3ba952e1SCorvin Köhne MSR_VMX_PROCBASED_CTLS2, 786*3ba952e1SCorvin Köhne PROCBASED2_WBINVD_EXITING, 787*3ba952e1SCorvin Köhne 0, 788*3ba952e1SCorvin Köhne &tmp) == 0); 789*3ba952e1SCorvin Köhne 790f5f5f1e7SPeter Grehan /* 791f5f5f1e7SPeter Grehan * Check support for RDPID and/or RDTSCP. 792f5f5f1e7SPeter Grehan * 793f5f5f1e7SPeter Grehan * Support a pass-through-based implementation of these via the 794f5f5f1e7SPeter Grehan * "enable RDTSCP" VM-execution control and the "RDTSC exiting" 795f5f5f1e7SPeter Grehan * VM-execution control. 796f5f5f1e7SPeter Grehan * 797f5f5f1e7SPeter Grehan * The "enable RDTSCP" VM-execution control applies to both RDPID 798f5f5f1e7SPeter Grehan * and RDTSCP (see SDM volume 3, section 25.3, "Changes to 799f5f5f1e7SPeter Grehan * Instruction Behavior in VMX Non-root operation"); this is why 800f5f5f1e7SPeter Grehan * only this VM-execution control needs to be enabled in order to 801f5f5f1e7SPeter Grehan * enable passing through whichever of RDPID and/or RDTSCP are 802f5f5f1e7SPeter Grehan * supported by the host. 803f5f5f1e7SPeter Grehan * 804f5f5f1e7SPeter Grehan * The "RDTSC exiting" VM-execution control applies to both RDTSC 805f5f5f1e7SPeter Grehan * and RDTSCP (again, per SDM volume 3, section 25.3), and is 806f5f5f1e7SPeter Grehan * already set up for RDTSC and RDTSCP pass-through by the current 807f5f5f1e7SPeter Grehan * implementation of RDTSC. 808f5f5f1e7SPeter Grehan * 809f5f5f1e7SPeter Grehan * Although RDPID and RDTSCP are optional capabilities, since there 810f5f5f1e7SPeter Grehan * does not currently seem to be a use case for enabling/disabling 811f5f5f1e7SPeter Grehan * these via libvmmapi, choose not to support this and, instead, 812f5f5f1e7SPeter Grehan * just statically always enable or always disable this support 813f5f5f1e7SPeter Grehan * across all vCPUs on all VMs. (Note that there may be some 814f5f5f1e7SPeter Grehan * complications to providing this functionality, e.g., the MSR 815f5f5f1e7SPeter Grehan * bitmap is currently per-VM rather than per-vCPU while the 816f5f5f1e7SPeter Grehan * capability API wants to be able to control capabilities on a 817f5f5f1e7SPeter Grehan * per-vCPU basis). 818f5f5f1e7SPeter Grehan */ 819f5f5f1e7SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 820f5f5f1e7SPeter Grehan MSR_VMX_PROCBASED_CTLS2, 821f5f5f1e7SPeter Grehan PROCBASED2_ENABLE_RDTSCP, 0, &tmp); 822f5f5f1e7SPeter Grehan cap_rdpid = error == 0 && host_has_rdpid(); 823f5f5f1e7SPeter Grehan cap_rdtscp = error == 0 && host_has_rdtscp(); 824f5f5f1e7SPeter Grehan if (cap_rdpid || cap_rdtscp) 825f5f5f1e7SPeter Grehan procbased_ctls2 |= PROCBASED2_ENABLE_RDTSCP; 826f5f5f1e7SPeter Grehan 827366f6083SPeter Grehan cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 828366f6083SPeter Grehan MSR_VMX_PROCBASED_CTLS2, 829366f6083SPeter Grehan PROCBASED2_UNRESTRICTED_GUEST, 0, 830366f6083SPeter Grehan &tmp) == 0); 831366f6083SPeter Grehan 83249cc03daSNeel Natu cap_invpcid = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 83349cc03daSNeel Natu MSR_VMX_PROCBASED_CTLS2, PROCBASED2_ENABLE_INVPCID, 0, 83449cc03daSNeel Natu &tmp) == 0); 83549cc03daSNeel Natu 83688c4b8d1SNeel Natu /* 8371bc51badSMichael Reifenberger * Check support for TPR shadow. 8381bc51badSMichael Reifenberger */ 8391bc51badSMichael Reifenberger error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 8401bc51badSMichael Reifenberger MSR_VMX_TRUE_PROCBASED_CTLS, PROCBASED_USE_TPR_SHADOW, 0, 8411bc51badSMichael Reifenberger &tmp); 8421bc51badSMichael Reifenberger if (error == 0) { 8431bc51badSMichael Reifenberger tpr_shadowing = 1; 8441bc51badSMichael Reifenberger TUNABLE_INT_FETCH("hw.vmm.vmx.use_tpr_shadowing", 8451bc51badSMichael Reifenberger &tpr_shadowing); 8461bc51badSMichael Reifenberger } 8471bc51badSMichael Reifenberger 8481bc51badSMichael Reifenberger if (tpr_shadowing) { 8491bc51badSMichael Reifenberger procbased_ctls |= PROCBASED_USE_TPR_SHADOW; 8501bc51badSMichael Reifenberger procbased_ctls &= ~PROCBASED_CR8_LOAD_EXITING; 8511bc51badSMichael Reifenberger procbased_ctls &= ~PROCBASED_CR8_STORE_EXITING; 8521bc51badSMichael Reifenberger } 8531bc51badSMichael Reifenberger 8541bc51badSMichael Reifenberger /* 85588c4b8d1SNeel Natu * Check support for virtual interrupt delivery. 85688c4b8d1SNeel Natu */ 85788c4b8d1SNeel Natu procbased2_vid_bits = (PROCBASED2_VIRTUALIZE_APIC_ACCESSES | 85888c4b8d1SNeel Natu PROCBASED2_VIRTUALIZE_X2APIC_MODE | 85988c4b8d1SNeel Natu PROCBASED2_APIC_REGISTER_VIRTUALIZATION | 86088c4b8d1SNeel Natu PROCBASED2_VIRTUAL_INTERRUPT_DELIVERY); 86188c4b8d1SNeel Natu 86288c4b8d1SNeel Natu error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2, 86388c4b8d1SNeel Natu procbased2_vid_bits, 0, &tmp); 8641bc51badSMichael Reifenberger if (error == 0 && tpr_shadowing) { 86588c4b8d1SNeel Natu virtual_interrupt_delivery = 1; 86688c4b8d1SNeel Natu TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_vid", 86788c4b8d1SNeel Natu &virtual_interrupt_delivery); 86888c4b8d1SNeel Natu } 86988c4b8d1SNeel Natu 87088c4b8d1SNeel Natu if (virtual_interrupt_delivery) { 87188c4b8d1SNeel Natu procbased_ctls |= PROCBASED_USE_TPR_SHADOW; 87288c4b8d1SNeel Natu procbased_ctls2 |= procbased2_vid_bits; 87388c4b8d1SNeel Natu procbased_ctls2 &= ~PROCBASED2_VIRTUALIZE_X2APIC_MODE; 874176666c2SNeel Natu 875176666c2SNeel Natu /* 876176666c2SNeel Natu * Check for Posted Interrupts only if Virtual Interrupt 877176666c2SNeel Natu * Delivery is enabled. 878176666c2SNeel Natu */ 879176666c2SNeel Natu error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS, 880176666c2SNeel Natu MSR_VMX_TRUE_PINBASED_CTLS, PINBASED_POSTED_INTERRUPT, 0, 881176666c2SNeel Natu &tmp); 882176666c2SNeel Natu if (error == 0) { 883bd50262fSKonstantin Belousov pirvec = lapic_ipi_alloc(pti ? &IDTVEC(justreturn1_pti) : 884bd50262fSKonstantin Belousov &IDTVEC(justreturn)); 88518a2b08eSNeel Natu if (pirvec < 0) { 886176666c2SNeel Natu if (bootverbose) { 88715add60dSPeter Grehan printf("vmx_modinit: unable to " 88815add60dSPeter Grehan "allocate posted interrupt " 88915add60dSPeter Grehan "vector\n"); 89088c4b8d1SNeel Natu } 891176666c2SNeel Natu } else { 892176666c2SNeel Natu posted_interrupts = 1; 893176666c2SNeel Natu TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_pir", 894176666c2SNeel Natu &posted_interrupts); 895176666c2SNeel Natu } 896176666c2SNeel Natu } 897176666c2SNeel Natu } 898176666c2SNeel Natu 899176666c2SNeel Natu if (posted_interrupts) 900176666c2SNeel Natu pinbased_ctls |= PINBASED_POSTED_INTERRUPT; 90149cc03daSNeel Natu 902366f6083SPeter Grehan /* Initialize EPT */ 903add611fdSNeel Natu error = ept_init(ipinum); 904366f6083SPeter Grehan if (error) { 90515add60dSPeter Grehan printf("vmx_modinit: ept initialization failed (%d)\n", error); 906366f6083SPeter Grehan return (error); 907366f6083SPeter Grehan } 908366f6083SPeter Grehan 90923437573SKonstantin Belousov guest_l1d_flush = (cpu_ia32_arch_caps & 91023437573SKonstantin Belousov IA32_ARCH_CAP_SKIP_L1DFL_VMENTRY) == 0; 911c30578feSKonstantin Belousov TUNABLE_INT_FETCH("hw.vmm.l1d_flush", &guest_l1d_flush); 912c1141fbaSKonstantin Belousov 913c1141fbaSKonstantin Belousov /* 914c1141fbaSKonstantin Belousov * L1D cache flush is enabled. Use IA32_FLUSH_CMD MSR when 915c1141fbaSKonstantin Belousov * available. Otherwise fall back to the software flush 916c1141fbaSKonstantin Belousov * method which loads enough data from the kernel text to 917c1141fbaSKonstantin Belousov * flush existing L1D content, both on VMX entry and on NMI 918c1141fbaSKonstantin Belousov * return. 919c1141fbaSKonstantin Belousov */ 920c1141fbaSKonstantin Belousov if (guest_l1d_flush) { 921c1141fbaSKonstantin Belousov if ((cpu_stdext_feature3 & CPUID_STDEXT3_L1D_FLUSH) == 0) { 922c1141fbaSKonstantin Belousov guest_l1d_flush_sw = 1; 923c1141fbaSKonstantin Belousov TUNABLE_INT_FETCH("hw.vmm.l1d_flush_sw", 924c1141fbaSKonstantin Belousov &guest_l1d_flush_sw); 925c1141fbaSKonstantin Belousov } 926c1141fbaSKonstantin Belousov if (guest_l1d_flush_sw) { 927c1141fbaSKonstantin Belousov if (nmi_flush_l1d_sw <= 1) 928c1141fbaSKonstantin Belousov nmi_flush_l1d_sw = 1; 929c1141fbaSKonstantin Belousov } else { 930c1141fbaSKonstantin Belousov msr_load_list[0].index = MSR_IA32_FLUSH_CMD; 931c1141fbaSKonstantin Belousov msr_load_list[0].val = IA32_FLUSH_CMD_L1D; 932c1141fbaSKonstantin Belousov } 933c1141fbaSKonstantin Belousov } 934c30578feSKonstantin Belousov 935366f6083SPeter Grehan /* 936366f6083SPeter Grehan * Stash the cr0 and cr4 bits that must be fixed to 0 or 1 937366f6083SPeter Grehan */ 938366f6083SPeter Grehan fixed0 = rdmsr(MSR_VMX_CR0_FIXED0); 939366f6083SPeter Grehan fixed1 = rdmsr(MSR_VMX_CR0_FIXED1); 940366f6083SPeter Grehan cr0_ones_mask = fixed0 & fixed1; 941366f6083SPeter Grehan cr0_zeros_mask = ~fixed0 & ~fixed1; 942366f6083SPeter Grehan 943366f6083SPeter Grehan /* 944366f6083SPeter Grehan * CR0_PE and CR0_PG can be set to zero in VMX non-root operation 945366f6083SPeter Grehan * if unrestricted guest execution is allowed. 946366f6083SPeter Grehan */ 947366f6083SPeter Grehan if (cap_unrestricted_guest) 948366f6083SPeter Grehan cr0_ones_mask &= ~(CR0_PG | CR0_PE); 949366f6083SPeter Grehan 950366f6083SPeter Grehan /* 951366f6083SPeter Grehan * Do not allow the guest to set CR0_NW or CR0_CD. 952366f6083SPeter Grehan */ 953366f6083SPeter Grehan cr0_zeros_mask |= (CR0_NW | CR0_CD); 954366f6083SPeter Grehan 955366f6083SPeter Grehan fixed0 = rdmsr(MSR_VMX_CR4_FIXED0); 956366f6083SPeter Grehan fixed1 = rdmsr(MSR_VMX_CR4_FIXED1); 957366f6083SPeter Grehan cr4_ones_mask = fixed0 & fixed1; 958366f6083SPeter Grehan cr4_zeros_mask = ~fixed0 & ~fixed1; 959366f6083SPeter Grehan 96045e51299SNeel Natu vpid_init(); 96145e51299SNeel Natu 962c3498942SNeel Natu vmx_msr_init(); 963c3498942SNeel Natu 964366f6083SPeter Grehan /* enable VMX operation */ 965366f6083SPeter Grehan smp_rendezvous(NULL, vmx_enable, NULL, NULL); 966366f6083SPeter Grehan 9673565b59eSNeel Natu vmx_initialized = 1; 9683565b59eSNeel Natu 969366f6083SPeter Grehan return (0); 970366f6083SPeter Grehan } 971366f6083SPeter Grehan 972f7d47425SNeel Natu static void 973f7d47425SNeel Natu vmx_trigger_hostintr(int vector) 974f7d47425SNeel Natu { 975f7d47425SNeel Natu uintptr_t func; 976f7d47425SNeel Natu struct gate_descriptor *gd; 977f7d47425SNeel Natu 978f7d47425SNeel Natu gd = &idt[vector]; 979f7d47425SNeel Natu 980f7d47425SNeel Natu KASSERT(vector >= 32 && vector <= 255, ("vmx_trigger_hostintr: " 981f7d47425SNeel Natu "invalid vector %d", vector)); 982f7d47425SNeel Natu KASSERT(gd->gd_p == 1, ("gate descriptor for vector %d not present", 983f7d47425SNeel Natu vector)); 984f7d47425SNeel Natu KASSERT(gd->gd_type == SDT_SYSIGT, ("gate descriptor for vector %d " 985f7d47425SNeel Natu "has invalid type %d", vector, gd->gd_type)); 986f7d47425SNeel Natu KASSERT(gd->gd_dpl == SEL_KPL, ("gate descriptor for vector %d " 987f7d47425SNeel Natu "has invalid dpl %d", vector, gd->gd_dpl)); 988f7d47425SNeel Natu KASSERT(gd->gd_selector == GSEL(GCODE_SEL, SEL_KPL), ("gate descriptor " 989f7d47425SNeel Natu "for vector %d has invalid selector %d", vector, gd->gd_selector)); 990f7d47425SNeel Natu KASSERT(gd->gd_ist == 0, ("gate descriptor for vector %d has invalid " 991f7d47425SNeel Natu "IST %d", vector, gd->gd_ist)); 992f7d47425SNeel Natu 993f7d47425SNeel Natu func = ((long)gd->gd_hioffset << 16 | gd->gd_looffset); 994f7d47425SNeel Natu vmx_call_isr(func); 995f7d47425SNeel Natu } 996f7d47425SNeel Natu 997366f6083SPeter Grehan static int 998aaaa0656SPeter Grehan vmx_setup_cr_shadow(int which, struct vmcs *vmcs, uint32_t initial) 999366f6083SPeter Grehan { 100039c21c2dSNeel Natu int error, mask_ident, shadow_ident; 1001aaaa0656SPeter Grehan uint64_t mask_value; 1002366f6083SPeter Grehan 100339c21c2dSNeel Natu if (which != 0 && which != 4) 100439c21c2dSNeel Natu panic("vmx_setup_cr_shadow: unknown cr%d", which); 100539c21c2dSNeel Natu 100639c21c2dSNeel Natu if (which == 0) { 100739c21c2dSNeel Natu mask_ident = VMCS_CR0_MASK; 100839c21c2dSNeel Natu mask_value = cr0_ones_mask | cr0_zeros_mask; 100939c21c2dSNeel Natu shadow_ident = VMCS_CR0_SHADOW; 101039c21c2dSNeel Natu } else { 101139c21c2dSNeel Natu mask_ident = VMCS_CR4_MASK; 101239c21c2dSNeel Natu mask_value = cr4_ones_mask | cr4_zeros_mask; 101339c21c2dSNeel Natu shadow_ident = VMCS_CR4_SHADOW; 101439c21c2dSNeel Natu } 101539c21c2dSNeel Natu 1016d3c11f40SPeter Grehan error = vmcs_setreg(vmcs, 0, VMCS_IDENT(mask_ident), mask_value); 1017366f6083SPeter Grehan if (error) 1018366f6083SPeter Grehan return (error); 1019366f6083SPeter Grehan 1020aaaa0656SPeter Grehan error = vmcs_setreg(vmcs, 0, VMCS_IDENT(shadow_ident), initial); 1021366f6083SPeter Grehan if (error) 1022366f6083SPeter Grehan return (error); 1023366f6083SPeter Grehan 1024366f6083SPeter Grehan return (0); 1025366f6083SPeter Grehan } 1026aaaa0656SPeter Grehan #define vmx_setup_cr0_shadow(vmcs,init) vmx_setup_cr_shadow(0, (vmcs), (init)) 1027aaaa0656SPeter Grehan #define vmx_setup_cr4_shadow(vmcs,init) vmx_setup_cr_shadow(4, (vmcs), (init)) 1028366f6083SPeter Grehan 1029366f6083SPeter Grehan static void * 103015add60dSPeter Grehan vmx_init(struct vm *vm, pmap_t pmap) 1031366f6083SPeter Grehan { 103245e51299SNeel Natu uint16_t vpid[VM_MAXCPU]; 1033c3498942SNeel Natu int i, error; 1034366f6083SPeter Grehan struct vmx *vmx; 1035c847a506SNeel Natu struct vmcs *vmcs; 1036b0538143SNeel Natu uint32_t exc_bitmap; 1037a488c9c9SRodney W. Grimes uint16_t maxcpus; 1038366f6083SPeter Grehan 1039366f6083SPeter Grehan vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO); 1040366f6083SPeter Grehan if ((uintptr_t)vmx & PAGE_MASK) { 1041366f6083SPeter Grehan panic("malloc of struct vmx not aligned on %d byte boundary", 1042366f6083SPeter Grehan PAGE_SIZE); 1043366f6083SPeter Grehan } 1044366f6083SPeter Grehan vmx->vm = vm; 1045366f6083SPeter Grehan 10469ce875d9SKonstantin Belousov vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pmltop)); 1047318224bbSNeel Natu 1048366f6083SPeter Grehan /* 1049366f6083SPeter Grehan * Clean up EPTP-tagged guest physical and combined mappings 1050366f6083SPeter Grehan * 1051366f6083SPeter Grehan * VMX transitions are not required to invalidate any guest physical 1052366f6083SPeter Grehan * mappings. So, it may be possible for stale guest physical mappings 1053366f6083SPeter Grehan * to be present in the processor TLBs. 1054366f6083SPeter Grehan * 1055366f6083SPeter Grehan * Combined mappings for this EP4TA are also invalidated for all VPIDs. 1056366f6083SPeter Grehan */ 1057318224bbSNeel Natu ept_invalidate_mappings(vmx->eptp); 1058366f6083SPeter Grehan 1059366f6083SPeter Grehan msr_bitmap_initialize(vmx->msr_bitmap); 1060366f6083SPeter Grehan 1061366f6083SPeter Grehan /* 1062366f6083SPeter Grehan * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE. 1063366f6083SPeter Grehan * The guest FSBASE and GSBASE are saved and restored during 1064366f6083SPeter Grehan * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are 1065366f6083SPeter Grehan * always restored from the vmcs host state area on vm-exit. 1066366f6083SPeter Grehan * 10671fb0ea3fSPeter Grehan * The SYSENTER_CS/ESP/EIP MSRs are identical to FS/GSBASE in 10681fb0ea3fSPeter Grehan * how they are saved/restored so can be directly accessed by the 10691fb0ea3fSPeter Grehan * guest. 10701fb0ea3fSPeter Grehan * 1071366f6083SPeter Grehan * MSR_EFER is saved and restored in the guest VMCS area on a 1072366f6083SPeter Grehan * VM exit and entry respectively. It is also restored from the 1073366f6083SPeter Grehan * host VMCS area on a VM exit. 10748d1d7a9eSPeter Grehan * 1075277bdd99STycho Nightingale * The TSC MSR is exposed read-only. Writes are disallowed as 1076277bdd99STycho Nightingale * that will impact the host TSC. If the guest does a write 1077277bdd99STycho Nightingale * the "use TSC offsetting" execution control is enabled and the 1078277bdd99STycho Nightingale * difference between the host TSC and the guest TSC is written 1079277bdd99STycho Nightingale * into the TSC offset in the VMCS. 1080f5f5f1e7SPeter Grehan * 1081f5f5f1e7SPeter Grehan * Guest TSC_AUX support is enabled if any of guest RDPID and/or 1082f5f5f1e7SPeter Grehan * guest RDTSCP support are enabled (since, as per Table 2-2 in SDM 1083f5f5f1e7SPeter Grehan * volume 4, TSC_AUX is supported if any of RDPID and/or RDTSCP are 1084f5f5f1e7SPeter Grehan * supported). If guest TSC_AUX support is enabled, TSC_AUX is 1085f5f5f1e7SPeter Grehan * exposed read-only so that the VMM can do one fewer MSR read per 1086f5f5f1e7SPeter Grehan * exit than if this register were exposed read-write; the guest 1087f5f5f1e7SPeter Grehan * restore value can be updated during guest writes (expected to be 1088f5f5f1e7SPeter Grehan * rare) instead of during all exits (common). 1089366f6083SPeter Grehan */ 1090366f6083SPeter Grehan if (guest_msr_rw(vmx, MSR_GSBASE) || 1091366f6083SPeter Grehan guest_msr_rw(vmx, MSR_FSBASE) || 10921fb0ea3fSPeter Grehan guest_msr_rw(vmx, MSR_SYSENTER_CS_MSR) || 10931fb0ea3fSPeter Grehan guest_msr_rw(vmx, MSR_SYSENTER_ESP_MSR) || 10941fb0ea3fSPeter Grehan guest_msr_rw(vmx, MSR_SYSENTER_EIP_MSR) || 10958d1d7a9eSPeter Grehan guest_msr_rw(vmx, MSR_EFER) || 1096f5f5f1e7SPeter Grehan guest_msr_ro(vmx, MSR_TSC) || 1097f5f5f1e7SPeter Grehan ((cap_rdpid || cap_rdtscp) && guest_msr_ro(vmx, MSR_TSC_AUX))) 109815add60dSPeter Grehan panic("vmx_init: error setting guest msr access"); 1099366f6083SPeter Grehan 110045e51299SNeel Natu vpid_alloc(vpid, VM_MAXCPU); 110145e51299SNeel Natu 110288c4b8d1SNeel Natu if (virtual_interrupt_delivery) { 110388c4b8d1SNeel Natu error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE, 110488c4b8d1SNeel Natu APIC_ACCESS_ADDRESS); 110588c4b8d1SNeel Natu /* XXX this should really return an error to the caller */ 110688c4b8d1SNeel Natu KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error)); 110788c4b8d1SNeel Natu } 110888c4b8d1SNeel Natu 1109a488c9c9SRodney W. Grimes maxcpus = vm_get_maxcpus(vm); 1110a488c9c9SRodney W. Grimes for (i = 0; i < maxcpus; i++) { 1111c847a506SNeel Natu vmcs = &vmx->vmcs[i]; 1112c847a506SNeel Natu vmcs->identifier = vmx_revision(); 1113c847a506SNeel Natu error = vmclear(vmcs); 1114366f6083SPeter Grehan if (error != 0) { 111515add60dSPeter Grehan panic("vmx_init: vmclear error %d on vcpu %d\n", 1116366f6083SPeter Grehan error, i); 1117366f6083SPeter Grehan } 1118366f6083SPeter Grehan 1119c3498942SNeel Natu vmx_msr_guest_init(vmx, i); 1120c3498942SNeel Natu 1121c847a506SNeel Natu error = vmcs_init(vmcs); 1122c847a506SNeel Natu KASSERT(error == 0, ("vmcs_init error %d", error)); 1123366f6083SPeter Grehan 1124c847a506SNeel Natu VMPTRLD(vmcs); 1125c847a506SNeel Natu error = 0; 1126c847a506SNeel Natu error += vmwrite(VMCS_HOST_RSP, (u_long)&vmx->ctx[i]); 1127c847a506SNeel Natu error += vmwrite(VMCS_EPTP, vmx->eptp); 1128c847a506SNeel Natu error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls); 1129c847a506SNeel Natu error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls); 1130*3ba952e1SCorvin Köhne if (vcpu_trap_wbinvd(vm, i)) { 1131*3ba952e1SCorvin Köhne KASSERT(cap_wbinvd_exit, ("WBINVD trap not available")); 1132*3ba952e1SCorvin Köhne procbased_ctls2 |= PROCBASED2_WBINVD_EXITING; 1133*3ba952e1SCorvin Köhne } 1134c847a506SNeel Natu error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2); 1135c847a506SNeel Natu error += vmwrite(VMCS_EXIT_CTLS, exit_ctls); 1136c847a506SNeel Natu error += vmwrite(VMCS_ENTRY_CTLS, entry_ctls); 1137c847a506SNeel Natu error += vmwrite(VMCS_MSR_BITMAP, vtophys(vmx->msr_bitmap)); 1138c847a506SNeel Natu error += vmwrite(VMCS_VPID, vpid[i]); 1139b0538143SNeel Natu 1140c1141fbaSKonstantin Belousov if (guest_l1d_flush && !guest_l1d_flush_sw) { 1141c1141fbaSKonstantin Belousov vmcs_write(VMCS_ENTRY_MSR_LOAD, pmap_kextract( 1142c1141fbaSKonstantin Belousov (vm_offset_t)&msr_load_list[0])); 1143c1141fbaSKonstantin Belousov vmcs_write(VMCS_ENTRY_MSR_LOAD_COUNT, 1144c1141fbaSKonstantin Belousov nitems(msr_load_list)); 1145c1141fbaSKonstantin Belousov vmcs_write(VMCS_EXIT_MSR_STORE, 0); 1146c1141fbaSKonstantin Belousov vmcs_write(VMCS_EXIT_MSR_STORE_COUNT, 0); 1147c1141fbaSKonstantin Belousov } 1148c1141fbaSKonstantin Belousov 1149b0538143SNeel Natu /* exception bitmap */ 1150b0538143SNeel Natu if (vcpu_trace_exceptions(vm, i)) 1151b0538143SNeel Natu exc_bitmap = 0xffffffff; 1152b0538143SNeel Natu else 1153b0538143SNeel Natu exc_bitmap = 1 << IDT_MC; 1154b0538143SNeel Natu error += vmwrite(VMCS_EXCEPTION_BITMAP, exc_bitmap); 1155b0538143SNeel Natu 11569e2154ffSJohn Baldwin vmx->ctx[i].guest_dr6 = DBREG_DR6_RESERVED1; 11579e2154ffSJohn Baldwin error += vmwrite(VMCS_GUEST_DR7, DBREG_DR7_RESERVED1); 115865eefbe4SJohn Baldwin 11591bc51badSMichael Reifenberger if (tpr_shadowing) { 116088c4b8d1SNeel Natu error += vmwrite(VMCS_VIRTUAL_APIC, 116188c4b8d1SNeel Natu vtophys(&vmx->apic_page[i])); 11621bc51badSMichael Reifenberger } 11631bc51badSMichael Reifenberger 11641bc51badSMichael Reifenberger if (virtual_interrupt_delivery) { 11651bc51badSMichael Reifenberger error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS); 116688c4b8d1SNeel Natu error += vmwrite(VMCS_EOI_EXIT0, 0); 116788c4b8d1SNeel Natu error += vmwrite(VMCS_EOI_EXIT1, 0); 116888c4b8d1SNeel Natu error += vmwrite(VMCS_EOI_EXIT2, 0); 116988c4b8d1SNeel Natu error += vmwrite(VMCS_EOI_EXIT3, 0); 117088c4b8d1SNeel Natu } 1171176666c2SNeel Natu if (posted_interrupts) { 1172176666c2SNeel Natu error += vmwrite(VMCS_PIR_VECTOR, pirvec); 1173176666c2SNeel Natu error += vmwrite(VMCS_PIR_DESC, 1174176666c2SNeel Natu vtophys(&vmx->pir_desc[i])); 1175176666c2SNeel Natu } 1176c847a506SNeel Natu VMCLEAR(vmcs); 117715add60dSPeter Grehan KASSERT(error == 0, ("vmx_init: error customizing the vmcs")); 1178366f6083SPeter Grehan 1179366f6083SPeter Grehan vmx->cap[i].set = 0; 1180f5f5f1e7SPeter Grehan vmx->cap[i].set |= cap_rdpid != 0 ? 1 << VM_CAP_RDPID : 0; 1181f5f5f1e7SPeter Grehan vmx->cap[i].set |= cap_rdtscp != 0 ? 1 << VM_CAP_RDTSCP : 0; 1182366f6083SPeter Grehan vmx->cap[i].proc_ctls = procbased_ctls; 118349cc03daSNeel Natu vmx->cap[i].proc_ctls2 = procbased_ctls2; 1184cbd03a9dSJohn Baldwin vmx->cap[i].exc_bitmap = exc_bitmap; 1185366f6083SPeter Grehan 11862ce12423SNeel Natu vmx->state[i].nextrip = ~0; 11873527963bSNeel Natu vmx->state[i].lastcpu = NOCPU; 118845e51299SNeel Natu vmx->state[i].vpid = vpid[i]; 1189366f6083SPeter Grehan 1190aaaa0656SPeter Grehan /* 1191aaaa0656SPeter Grehan * Set up the CR0/4 shadows, and init the read shadow 1192aaaa0656SPeter Grehan * to the power-on register value from the Intel Sys Arch. 1193aaaa0656SPeter Grehan * CR0 - 0x60000010 1194aaaa0656SPeter Grehan * CR4 - 0 1195aaaa0656SPeter Grehan */ 1196c847a506SNeel Natu error = vmx_setup_cr0_shadow(vmcs, 0x60000010); 119739c21c2dSNeel Natu if (error != 0) 119839c21c2dSNeel Natu panic("vmx_setup_cr0_shadow %d", error); 119939c21c2dSNeel Natu 1200c847a506SNeel Natu error = vmx_setup_cr4_shadow(vmcs, 0); 120139c21c2dSNeel Natu if (error != 0) 120239c21c2dSNeel Natu panic("vmx_setup_cr4_shadow %d", error); 1203318224bbSNeel Natu 1204318224bbSNeel Natu vmx->ctx[i].pmap = pmap; 1205366f6083SPeter Grehan } 1206366f6083SPeter Grehan 1207366f6083SPeter Grehan return (vmx); 1208366f6083SPeter Grehan } 1209366f6083SPeter Grehan 1210366f6083SPeter Grehan static int 1211a2da7af6SNeel Natu vmx_handle_cpuid(struct vm *vm, int vcpu, struct vmxctx *vmxctx) 1212366f6083SPeter Grehan { 1213a3f2a9c5SJohn Baldwin int handled; 1214366f6083SPeter Grehan 1215a3f2a9c5SJohn Baldwin handled = x86_emulate_cpuid(vm, vcpu, (uint64_t *)&vmxctx->guest_rax, 1216a3f2a9c5SJohn Baldwin (uint64_t *)&vmxctx->guest_rbx, (uint64_t *)&vmxctx->guest_rcx, 1217a3f2a9c5SJohn Baldwin (uint64_t *)&vmxctx->guest_rdx); 1218366f6083SPeter Grehan return (handled); 1219366f6083SPeter Grehan } 1220366f6083SPeter Grehan 1221366f6083SPeter Grehan static __inline void 1222366f6083SPeter Grehan vmx_run_trace(struct vmx *vmx, int vcpu) 1223366f6083SPeter Grehan { 1224366f6083SPeter Grehan #ifdef KTR 1225513c8d33SNeel Natu VCPU_CTR1(vmx->vm, vcpu, "Resume execution at %#lx", vmcs_guest_rip()); 1226366f6083SPeter Grehan #endif 1227366f6083SPeter Grehan } 1228366f6083SPeter Grehan 1229366f6083SPeter Grehan static __inline void 1230366f6083SPeter Grehan vmx_exit_trace(struct vmx *vmx, int vcpu, uint64_t rip, uint32_t exit_reason, 1231eeefa4e4SNeel Natu int handled) 1232366f6083SPeter Grehan { 1233366f6083SPeter Grehan #ifdef KTR 1234513c8d33SNeel Natu VCPU_CTR3(vmx->vm, vcpu, "%s %s vmexit at 0x%0lx", 1235366f6083SPeter Grehan handled ? "handled" : "unhandled", 1236366f6083SPeter Grehan exit_reason_to_str(exit_reason), rip); 1237eeefa4e4SNeel Natu #endif 1238eeefa4e4SNeel Natu } 1239366f6083SPeter Grehan 1240eeefa4e4SNeel Natu static __inline void 1241eeefa4e4SNeel Natu vmx_astpending_trace(struct vmx *vmx, int vcpu, uint64_t rip) 1242eeefa4e4SNeel Natu { 1243eeefa4e4SNeel Natu #ifdef KTR 1244513c8d33SNeel Natu VCPU_CTR1(vmx->vm, vcpu, "astpending vmexit at 0x%0lx", rip); 1245366f6083SPeter Grehan #endif 1246366f6083SPeter Grehan } 1247366f6083SPeter Grehan 1248953c2c47SNeel Natu static VMM_STAT_INTEL(VCPU_INVVPID_SAVED, "Number of vpid invalidations saved"); 12493527963bSNeel Natu static VMM_STAT_INTEL(VCPU_INVVPID_DONE, "Number of vpid invalidations done"); 1250953c2c47SNeel Natu 12513527963bSNeel Natu /* 12523527963bSNeel Natu * Invalidate guest mappings identified by its vpid from the TLB. 12533527963bSNeel Natu */ 12543527963bSNeel Natu static __inline void 12553527963bSNeel Natu vmx_invvpid(struct vmx *vmx, int vcpu, pmap_t pmap, int running) 1256366f6083SPeter Grehan { 1257366f6083SPeter Grehan struct vmxstate *vmxstate; 1258953c2c47SNeel Natu struct invvpid_desc invvpid_desc; 1259366f6083SPeter Grehan 1260366f6083SPeter Grehan vmxstate = &vmx->state[vcpu]; 12613527963bSNeel Natu if (vmxstate->vpid == 0) 12623de83862SNeel Natu return; 1263366f6083SPeter Grehan 12643527963bSNeel Natu if (!running) { 12653527963bSNeel Natu /* 12663527963bSNeel Natu * Set the 'lastcpu' to an invalid host cpu. 12673527963bSNeel Natu * 12683527963bSNeel Natu * This will invalidate TLB entries tagged with the vcpu's 12693527963bSNeel Natu * vpid the next time it runs via vmx_set_pcpu_defaults(). 12703527963bSNeel Natu */ 12713527963bSNeel Natu vmxstate->lastcpu = NOCPU; 12723527963bSNeel Natu return; 12733527963bSNeel Natu } 1274953c2c47SNeel Natu 12753527963bSNeel Natu KASSERT(curthread->td_critnest > 0, ("%s: vcpu %d running outside " 12763527963bSNeel Natu "critical section", __func__, vcpu)); 1277366f6083SPeter Grehan 1278366f6083SPeter Grehan /* 12793527963bSNeel Natu * Invalidate all mappings tagged with 'vpid' 1280366f6083SPeter Grehan * 1281366f6083SPeter Grehan * We do this because this vcpu was executing on a different host 1282366f6083SPeter Grehan * cpu when it last ran. We do not track whether it invalidated 1283366f6083SPeter Grehan * mappings associated with its 'vpid' during that run. So we must 1284366f6083SPeter Grehan * assume that the mappings associated with 'vpid' on 'curcpu' are 1285366f6083SPeter Grehan * stale and invalidate them. 1286366f6083SPeter Grehan * 1287366f6083SPeter Grehan * Note that we incur this penalty only when the scheduler chooses to 1288366f6083SPeter Grehan * move the thread associated with this vcpu between host cpus. 1289366f6083SPeter Grehan * 1290366f6083SPeter Grehan * Note also that this will invalidate mappings tagged with 'vpid' 1291366f6083SPeter Grehan * for "all" EP4TAs. 1292366f6083SPeter Grehan */ 12936f5a9606SMark Johnston if (atomic_load_long(&pmap->pm_eptgen) == vmx->eptgen[curcpu]) { 1294953c2c47SNeel Natu invvpid_desc._res1 = 0; 1295953c2c47SNeel Natu invvpid_desc._res2 = 0; 1296366f6083SPeter Grehan invvpid_desc.vpid = vmxstate->vpid; 12970e30c5c0SWarner Losh invvpid_desc.linear_addr = 0; 1298366f6083SPeter Grehan invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc); 12993527963bSNeel Natu vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_DONE, 1); 1300953c2c47SNeel Natu } else { 1301953c2c47SNeel Natu /* 1302953c2c47SNeel Natu * The invvpid can be skipped if an invept is going to 1303953c2c47SNeel Natu * be performed before entering the guest. The invept 1304953c2c47SNeel Natu * will invalidate combined mappings tagged with 1305953c2c47SNeel Natu * 'vmx->eptp' for all vpids. 1306953c2c47SNeel Natu */ 1307953c2c47SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_SAVED, 1); 1308953c2c47SNeel Natu } 1309366f6083SPeter Grehan } 13103527963bSNeel Natu 13113527963bSNeel Natu static void 13123527963bSNeel Natu vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu, pmap_t pmap) 13133527963bSNeel Natu { 13143527963bSNeel Natu struct vmxstate *vmxstate; 13153527963bSNeel Natu 13163527963bSNeel Natu vmxstate = &vmx->state[vcpu]; 13173527963bSNeel Natu if (vmxstate->lastcpu == curcpu) 13183527963bSNeel Natu return; 13193527963bSNeel Natu 13203527963bSNeel Natu vmxstate->lastcpu = curcpu; 13213527963bSNeel Natu 13223527963bSNeel Natu vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1); 13233527963bSNeel Natu 13243527963bSNeel Natu vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase()); 13253527963bSNeel Natu vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase()); 13263527963bSNeel Natu vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase()); 13273527963bSNeel Natu vmx_invvpid(vmx, vcpu, pmap, 1); 1328366f6083SPeter Grehan } 1329366f6083SPeter Grehan 1330366f6083SPeter Grehan /* 1331366f6083SPeter Grehan * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set. 1332366f6083SPeter Grehan */ 1333366f6083SPeter Grehan CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0); 1334366f6083SPeter Grehan 1335366f6083SPeter Grehan static void __inline 1336366f6083SPeter Grehan vmx_set_int_window_exiting(struct vmx *vmx, int vcpu) 1337366f6083SPeter Grehan { 1338366f6083SPeter Grehan 133948b2d828SNeel Natu if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) { 1340366f6083SPeter Grehan vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING; 13413de83862SNeel Natu vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 134248b2d828SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting"); 134348b2d828SNeel Natu } 1344366f6083SPeter Grehan } 1345366f6083SPeter Grehan 1346366f6083SPeter Grehan static void __inline 1347366f6083SPeter Grehan vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu) 1348366f6083SPeter Grehan { 1349366f6083SPeter Grehan 135048b2d828SNeel Natu KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0, 135148b2d828SNeel Natu ("intr_window_exiting not set: %#x", vmx->cap[vcpu].proc_ctls)); 1352366f6083SPeter Grehan vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING; 13533de83862SNeel Natu vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 135448b2d828SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting"); 1355366f6083SPeter Grehan } 1356366f6083SPeter Grehan 1357366f6083SPeter Grehan static void __inline 1358366f6083SPeter Grehan vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu) 1359366f6083SPeter Grehan { 1360366f6083SPeter Grehan 136148b2d828SNeel Natu if ((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) { 1362366f6083SPeter Grehan vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING; 13633de83862SNeel Natu vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 136448b2d828SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting"); 136548b2d828SNeel Natu } 1366366f6083SPeter Grehan } 1367366f6083SPeter Grehan 1368366f6083SPeter Grehan static void __inline 1369366f6083SPeter Grehan vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu) 1370366f6083SPeter Grehan { 1371366f6083SPeter Grehan 137248b2d828SNeel Natu KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0, 137348b2d828SNeel Natu ("nmi_window_exiting not set %#x", vmx->cap[vcpu].proc_ctls)); 1374366f6083SPeter Grehan vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING; 13753de83862SNeel Natu vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 137648b2d828SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting"); 1377366f6083SPeter Grehan } 1378366f6083SPeter Grehan 1379277bdd99STycho Nightingale int 1380277bdd99STycho Nightingale vmx_set_tsc_offset(struct vmx *vmx, int vcpu, uint64_t offset) 1381277bdd99STycho Nightingale { 1382277bdd99STycho Nightingale int error; 1383277bdd99STycho Nightingale 1384277bdd99STycho Nightingale if ((vmx->cap[vcpu].proc_ctls & PROCBASED_TSC_OFFSET) == 0) { 1385277bdd99STycho Nightingale vmx->cap[vcpu].proc_ctls |= PROCBASED_TSC_OFFSET; 1386277bdd99STycho Nightingale vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 1387277bdd99STycho Nightingale VCPU_CTR0(vmx->vm, vcpu, "Enabling TSC offsetting"); 1388277bdd99STycho Nightingale } 1389277bdd99STycho Nightingale 1390277bdd99STycho Nightingale error = vmwrite(VMCS_TSC_OFFSET, offset); 1391483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT 1392483d953aSJohn Baldwin if (error == 0) 1393483d953aSJohn Baldwin error = vm_set_tsc_offset(vmx->vm, vcpu, offset); 1394483d953aSJohn Baldwin #endif 1395277bdd99STycho Nightingale return (error); 1396277bdd99STycho Nightingale } 1397277bdd99STycho Nightingale 139848b2d828SNeel Natu #define NMI_BLOCKING (VMCS_INTERRUPTIBILITY_NMI_BLOCKING | \ 139948b2d828SNeel Natu VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING) 140048b2d828SNeel Natu #define HWINTR_BLOCKING (VMCS_INTERRUPTIBILITY_STI_BLOCKING | \ 140148b2d828SNeel Natu VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING) 140248b2d828SNeel Natu 140348b2d828SNeel Natu static void 1404366f6083SPeter Grehan vmx_inject_nmi(struct vmx *vmx, int vcpu) 1405366f6083SPeter Grehan { 14065c272efaSRobert Wing uint32_t gi __diagused, info; 1407366f6083SPeter Grehan 140848b2d828SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 140948b2d828SNeel Natu KASSERT((gi & NMI_BLOCKING) == 0, ("vmx_inject_nmi: invalid guest " 141048b2d828SNeel Natu "interruptibility-state %#x", gi)); 1411366f6083SPeter Grehan 141248b2d828SNeel Natu info = vmcs_read(VMCS_ENTRY_INTR_INFO); 141348b2d828SNeel Natu KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_nmi: invalid " 141448b2d828SNeel Natu "VM-entry interruption information %#x", info)); 1415366f6083SPeter Grehan 1416366f6083SPeter Grehan /* 1417366f6083SPeter Grehan * Inject the virtual NMI. The vector must be the NMI IDT entry 1418366f6083SPeter Grehan * or the VMCS entry check will fail. 1419366f6083SPeter Grehan */ 142048b2d828SNeel Natu info = IDT_NMI | VMCS_INTR_T_NMI | VMCS_INTR_VALID; 14213de83862SNeel Natu vmcs_write(VMCS_ENTRY_INTR_INFO, info); 1422366f6083SPeter Grehan 1423513c8d33SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Injecting vNMI"); 1424366f6083SPeter Grehan 1425366f6083SPeter Grehan /* Clear the request */ 1426f352ff0cSNeel Natu vm_nmi_clear(vmx->vm, vcpu); 1427366f6083SPeter Grehan } 1428366f6083SPeter Grehan 1429366f6083SPeter Grehan static void 14302ce12423SNeel Natu vmx_inject_interrupts(struct vmx *vmx, int vcpu, struct vlapic *vlapic, 14312ce12423SNeel Natu uint64_t guestrip) 1432366f6083SPeter Grehan { 14330775fbb4STycho Nightingale int vector, need_nmi_exiting, extint_pending; 1434091d4532SNeel Natu uint64_t rflags, entryinfo; 143548b2d828SNeel Natu uint32_t gi, info; 1436366f6083SPeter Grehan 14372ce12423SNeel Natu if (vmx->state[vcpu].nextrip != guestrip) { 14382ce12423SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 14392ce12423SNeel Natu if (gi & HWINTR_BLOCKING) { 14402ce12423SNeel Natu VCPU_CTR2(vmx->vm, vcpu, "Guest interrupt blocking " 14412ce12423SNeel Natu "cleared due to rip change: %#lx/%#lx", 14422ce12423SNeel Natu vmx->state[vcpu].nextrip, guestrip); 14432ce12423SNeel Natu gi &= ~HWINTR_BLOCKING; 14442ce12423SNeel Natu vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi); 14452ce12423SNeel Natu } 14462ce12423SNeel Natu } 14472ce12423SNeel Natu 1448091d4532SNeel Natu if (vm_entry_intinfo(vmx->vm, vcpu, &entryinfo)) { 1449091d4532SNeel Natu KASSERT((entryinfo & VMCS_INTR_VALID) != 0, ("%s: entry " 1450091d4532SNeel Natu "intinfo is not valid: %#lx", __func__, entryinfo)); 1451dc506506SNeel Natu 1452dc506506SNeel Natu info = vmcs_read(VMCS_ENTRY_INTR_INFO); 1453dc506506SNeel Natu KASSERT((info & VMCS_INTR_VALID) == 0, ("%s: cannot inject " 1454019008ebSNeel Natu "pending exception: %#lx/%#x", __func__, entryinfo, info)); 1455dc506506SNeel Natu 1456091d4532SNeel Natu info = entryinfo; 1457091d4532SNeel Natu vector = info & 0xff; 1458091d4532SNeel Natu if (vector == IDT_BP || vector == IDT_OF) { 1459091d4532SNeel Natu /* 1460091d4532SNeel Natu * VT-x requires #BP and #OF to be injected as software 1461091d4532SNeel Natu * exceptions. 1462091d4532SNeel Natu */ 1463091d4532SNeel Natu info &= ~VMCS_INTR_T_MASK; 1464091d4532SNeel Natu info |= VMCS_INTR_T_SWEXCEPTION; 1465dc506506SNeel Natu } 1466091d4532SNeel Natu 1467091d4532SNeel Natu if (info & VMCS_INTR_DEL_ERRCODE) 1468091d4532SNeel Natu vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR, entryinfo >> 32); 1469091d4532SNeel Natu 1470dc506506SNeel Natu vmcs_write(VMCS_ENTRY_INTR_INFO, info); 1471dc506506SNeel Natu } 1472dc506506SNeel Natu 147348b2d828SNeel Natu if (vm_nmi_pending(vmx->vm, vcpu)) { 1474366f6083SPeter Grehan /* 147548b2d828SNeel Natu * If there are no conditions blocking NMI injection then 147648b2d828SNeel Natu * inject it directly here otherwise enable "NMI window 147748b2d828SNeel Natu * exiting" to inject it as soon as we can. 1478eeefa4e4SNeel Natu * 147948b2d828SNeel Natu * We also check for STI_BLOCKING because some implementations 148048b2d828SNeel Natu * don't allow NMI injection in this case. If we are running 148148b2d828SNeel Natu * on a processor that doesn't have this restriction it will 148248b2d828SNeel Natu * immediately exit and the NMI will be injected in the 148348b2d828SNeel Natu * "NMI window exiting" handler. 1484366f6083SPeter Grehan */ 148548b2d828SNeel Natu need_nmi_exiting = 1; 148648b2d828SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 148748b2d828SNeel Natu if ((gi & (HWINTR_BLOCKING | NMI_BLOCKING)) == 0) { 14883de83862SNeel Natu info = vmcs_read(VMCS_ENTRY_INTR_INFO); 148948b2d828SNeel Natu if ((info & VMCS_INTR_VALID) == 0) { 149048b2d828SNeel Natu vmx_inject_nmi(vmx, vcpu); 149148b2d828SNeel Natu need_nmi_exiting = 0; 149248b2d828SNeel Natu } else { 149348b2d828SNeel Natu VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI " 149448b2d828SNeel Natu "due to VM-entry intr info %#x", info); 149548b2d828SNeel Natu } 149648b2d828SNeel Natu } else { 149748b2d828SNeel Natu VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI due to " 149848b2d828SNeel Natu "Guest Interruptibility-state %#x", gi); 149948b2d828SNeel Natu } 1500eeefa4e4SNeel Natu 150148b2d828SNeel Natu if (need_nmi_exiting) 150248b2d828SNeel Natu vmx_set_nmi_window_exiting(vmx, vcpu); 150348b2d828SNeel Natu } 1504366f6083SPeter Grehan 15050775fbb4STycho Nightingale extint_pending = vm_extint_pending(vmx->vm, vcpu); 15060775fbb4STycho Nightingale 15070775fbb4STycho Nightingale if (!extint_pending && virtual_interrupt_delivery) { 150888c4b8d1SNeel Natu vmx_inject_pir(vlapic); 150988c4b8d1SNeel Natu return; 151088c4b8d1SNeel Natu } 151188c4b8d1SNeel Natu 151248b2d828SNeel Natu /* 151336736912SNeel Natu * If interrupt-window exiting is already in effect then don't bother 151436736912SNeel Natu * checking for pending interrupts. This is just an optimization and 151536736912SNeel Natu * not needed for correctness. 151648b2d828SNeel Natu */ 151736736912SNeel Natu if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) { 151836736912SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Skip interrupt injection due to " 151936736912SNeel Natu "pending int_window_exiting"); 152048b2d828SNeel Natu return; 152136736912SNeel Natu } 152248b2d828SNeel Natu 15230775fbb4STycho Nightingale if (!extint_pending) { 1524366f6083SPeter Grehan /* Ask the local apic for a vector to inject */ 15254d1e82a8SNeel Natu if (!vlapic_pending_intr(vlapic, &vector)) 1526366f6083SPeter Grehan return; 1527a026dc3fSTycho Nightingale 1528a026dc3fSTycho Nightingale /* 1529a026dc3fSTycho Nightingale * From the Intel SDM, Volume 3, Section "Maskable 1530a026dc3fSTycho Nightingale * Hardware Interrupts": 1531a026dc3fSTycho Nightingale * - maskable interrupt vectors [16,255] can be delivered 1532a026dc3fSTycho Nightingale * through the local APIC. 1533a026dc3fSTycho Nightingale */ 1534a026dc3fSTycho Nightingale KASSERT(vector >= 16 && vector <= 255, 1535a026dc3fSTycho Nightingale ("invalid vector %d from local APIC", vector)); 15360775fbb4STycho Nightingale } else { 15370775fbb4STycho Nightingale /* Ask the legacy pic for a vector to inject */ 15380775fbb4STycho Nightingale vatpic_pending_intr(vmx->vm, &vector); 1539366f6083SPeter Grehan 1540a026dc3fSTycho Nightingale /* 1541a026dc3fSTycho Nightingale * From the Intel SDM, Volume 3, Section "Maskable 1542a026dc3fSTycho Nightingale * Hardware Interrupts": 1543a026dc3fSTycho Nightingale * - maskable interrupt vectors [0,255] can be delivered 1544a026dc3fSTycho Nightingale * through the INTR pin. 1545a026dc3fSTycho Nightingale */ 1546a026dc3fSTycho Nightingale KASSERT(vector >= 0 && vector <= 255, 1547a026dc3fSTycho Nightingale ("invalid vector %d from INTR", vector)); 1548a026dc3fSTycho Nightingale } 1549366f6083SPeter Grehan 1550366f6083SPeter Grehan /* Check RFLAGS.IF and the interruptibility state of the guest */ 15513de83862SNeel Natu rflags = vmcs_read(VMCS_GUEST_RFLAGS); 155236736912SNeel Natu if ((rflags & PSL_I) == 0) { 155336736912SNeel Natu VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to " 155436736912SNeel Natu "rflags %#lx", vector, rflags); 1555366f6083SPeter Grehan goto cantinject; 155636736912SNeel Natu } 1557366f6083SPeter Grehan 155848b2d828SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 155936736912SNeel Natu if (gi & HWINTR_BLOCKING) { 156036736912SNeel Natu VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to " 156136736912SNeel Natu "Guest Interruptibility-state %#x", vector, gi); 1562366f6083SPeter Grehan goto cantinject; 156336736912SNeel Natu } 156436736912SNeel Natu 156536736912SNeel Natu info = vmcs_read(VMCS_ENTRY_INTR_INFO); 156636736912SNeel Natu if (info & VMCS_INTR_VALID) { 156736736912SNeel Natu /* 156836736912SNeel Natu * This is expected and could happen for multiple reasons: 156936736912SNeel Natu * - A vectoring VM-entry was aborted due to astpending 157036736912SNeel Natu * - A VM-exit happened during event injection. 1571dc506506SNeel Natu * - An exception was injected above. 157236736912SNeel Natu * - An NMI was injected above or after "NMI window exiting" 157336736912SNeel Natu */ 157436736912SNeel Natu VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to " 157536736912SNeel Natu "VM-entry intr info %#x", vector, info); 157636736912SNeel Natu goto cantinject; 157736736912SNeel Natu } 1578366f6083SPeter Grehan 1579366f6083SPeter Grehan /* Inject the interrupt */ 1580160471d2SNeel Natu info = VMCS_INTR_T_HWINTR | VMCS_INTR_VALID; 1581366f6083SPeter Grehan info |= vector; 15823de83862SNeel Natu vmcs_write(VMCS_ENTRY_INTR_INFO, info); 1583366f6083SPeter Grehan 15840775fbb4STycho Nightingale if (!extint_pending) { 1585366f6083SPeter Grehan /* Update the Local APIC ISR */ 1586de5ea6b6SNeel Natu vlapic_intr_accepted(vlapic, vector); 15870775fbb4STycho Nightingale } else { 15880775fbb4STycho Nightingale vm_extint_clear(vmx->vm, vcpu); 15890775fbb4STycho Nightingale vatpic_intr_accepted(vmx->vm, vector); 15900775fbb4STycho Nightingale 15910775fbb4STycho Nightingale /* 15920775fbb4STycho Nightingale * After we accepted the current ExtINT the PIC may 15930775fbb4STycho Nightingale * have posted another one. If that is the case, set 15940775fbb4STycho Nightingale * the Interrupt Window Exiting execution control so 15950775fbb4STycho Nightingale * we can inject that one too. 15960494cb1bSNeel Natu * 15970494cb1bSNeel Natu * Also, interrupt window exiting allows us to inject any 15980494cb1bSNeel Natu * pending APIC vector that was preempted by the ExtINT 15990494cb1bSNeel Natu * as soon as possible. This applies both for the software 16000494cb1bSNeel Natu * emulated vlapic and the hardware assisted virtual APIC. 16010775fbb4STycho Nightingale */ 16020775fbb4STycho Nightingale vmx_set_int_window_exiting(vmx, vcpu); 16030775fbb4STycho Nightingale } 1604366f6083SPeter Grehan 1605513c8d33SNeel Natu VCPU_CTR1(vmx->vm, vcpu, "Injecting hwintr at vector %d", vector); 1606366f6083SPeter Grehan 1607366f6083SPeter Grehan return; 1608366f6083SPeter Grehan 1609366f6083SPeter Grehan cantinject: 1610366f6083SPeter Grehan /* 1611366f6083SPeter Grehan * Set the Interrupt Window Exiting execution control so we can inject 1612366f6083SPeter Grehan * the interrupt as soon as blocking condition goes away. 1613366f6083SPeter Grehan */ 1614366f6083SPeter Grehan vmx_set_int_window_exiting(vmx, vcpu); 1615366f6083SPeter Grehan } 1616366f6083SPeter Grehan 1617e5a1d950SNeel Natu /* 1618e5a1d950SNeel Natu * If the Virtual NMIs execution control is '1' then the logical processor 1619e5a1d950SNeel Natu * tracks virtual-NMI blocking in the Guest Interruptibility-state field of 1620e5a1d950SNeel Natu * the VMCS. An IRET instruction in VMX non-root operation will remove any 1621e5a1d950SNeel Natu * virtual-NMI blocking. 1622e5a1d950SNeel Natu * 1623e5a1d950SNeel Natu * This unblocking occurs even if the IRET causes a fault. In this case the 1624e5a1d950SNeel Natu * hypervisor needs to restore virtual-NMI blocking before resuming the guest. 1625e5a1d950SNeel Natu */ 1626e5a1d950SNeel Natu static void 1627e5a1d950SNeel Natu vmx_restore_nmi_blocking(struct vmx *vmx, int vcpuid) 1628e5a1d950SNeel Natu { 1629e5a1d950SNeel Natu uint32_t gi; 1630e5a1d950SNeel Natu 1631e5a1d950SNeel Natu VCPU_CTR0(vmx->vm, vcpuid, "Restore Virtual-NMI blocking"); 1632e5a1d950SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 1633e5a1d950SNeel Natu gi |= VMCS_INTERRUPTIBILITY_NMI_BLOCKING; 1634e5a1d950SNeel Natu vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi); 1635e5a1d950SNeel Natu } 1636e5a1d950SNeel Natu 1637e5a1d950SNeel Natu static void 1638e5a1d950SNeel Natu vmx_clear_nmi_blocking(struct vmx *vmx, int vcpuid) 1639e5a1d950SNeel Natu { 1640e5a1d950SNeel Natu uint32_t gi; 1641e5a1d950SNeel Natu 1642e5a1d950SNeel Natu VCPU_CTR0(vmx->vm, vcpuid, "Clear Virtual-NMI blocking"); 1643e5a1d950SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 1644e5a1d950SNeel Natu gi &= ~VMCS_INTERRUPTIBILITY_NMI_BLOCKING; 1645e5a1d950SNeel Natu vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi); 1646e5a1d950SNeel Natu } 1647e5a1d950SNeel Natu 1648091d4532SNeel Natu static void 1649091d4532SNeel Natu vmx_assert_nmi_blocking(struct vmx *vmx, int vcpuid) 1650091d4532SNeel Natu { 16515c272efaSRobert Wing uint32_t gi __diagused; 1652091d4532SNeel Natu 1653091d4532SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 1654091d4532SNeel Natu KASSERT(gi & VMCS_INTERRUPTIBILITY_NMI_BLOCKING, 1655091d4532SNeel Natu ("NMI blocking is not in effect %#x", gi)); 1656091d4532SNeel Natu } 1657091d4532SNeel Natu 1658366f6083SPeter Grehan static int 1659a0efd3fbSJohn Baldwin vmx_emulate_xsetbv(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) 1660abb023fbSJohn Baldwin { 1661abb023fbSJohn Baldwin struct vmxctx *vmxctx; 1662abb023fbSJohn Baldwin uint64_t xcrval; 1663abb023fbSJohn Baldwin const struct xsave_limits *limits; 1664abb023fbSJohn Baldwin 1665abb023fbSJohn Baldwin vmxctx = &vmx->ctx[vcpu]; 1666abb023fbSJohn Baldwin limits = vmm_get_xsave_limits(); 1667abb023fbSJohn Baldwin 1668a0efd3fbSJohn Baldwin /* 1669a0efd3fbSJohn Baldwin * Note that the processor raises a GP# fault on its own if 1670a0efd3fbSJohn Baldwin * xsetbv is executed for CPL != 0, so we do not have to 1671a0efd3fbSJohn Baldwin * emulate that fault here. 1672a0efd3fbSJohn Baldwin */ 1673a0efd3fbSJohn Baldwin 1674a0efd3fbSJohn Baldwin /* Only xcr0 is supported. */ 1675a0efd3fbSJohn Baldwin if (vmxctx->guest_rcx != 0) { 1676dc506506SNeel Natu vm_inject_gp(vmx->vm, vcpu); 1677a0efd3fbSJohn Baldwin return (HANDLED); 1678a0efd3fbSJohn Baldwin } 1679a0efd3fbSJohn Baldwin 1680a0efd3fbSJohn Baldwin /* We only handle xcr0 if both the host and guest have XSAVE enabled. */ 1681a0efd3fbSJohn Baldwin if (!limits->xsave_enabled || !(vmcs_read(VMCS_GUEST_CR4) & CR4_XSAVE)) { 1682dc506506SNeel Natu vm_inject_ud(vmx->vm, vcpu); 1683a0efd3fbSJohn Baldwin return (HANDLED); 1684a0efd3fbSJohn Baldwin } 1685abb023fbSJohn Baldwin 1686abb023fbSJohn Baldwin xcrval = vmxctx->guest_rdx << 32 | (vmxctx->guest_rax & 0xffffffff); 1687a0efd3fbSJohn Baldwin if ((xcrval & ~limits->xcr0_allowed) != 0) { 1688dc506506SNeel Natu vm_inject_gp(vmx->vm, vcpu); 1689a0efd3fbSJohn Baldwin return (HANDLED); 1690a0efd3fbSJohn Baldwin } 1691abb023fbSJohn Baldwin 1692a0efd3fbSJohn Baldwin if (!(xcrval & XFEATURE_ENABLED_X87)) { 1693dc506506SNeel Natu vm_inject_gp(vmx->vm, vcpu); 1694a0efd3fbSJohn Baldwin return (HANDLED); 1695a0efd3fbSJohn Baldwin } 1696abb023fbSJohn Baldwin 169744a68c4eSJohn Baldwin /* AVX (YMM_Hi128) requires SSE. */ 169844a68c4eSJohn Baldwin if (xcrval & XFEATURE_ENABLED_AVX && 169944a68c4eSJohn Baldwin (xcrval & XFEATURE_AVX) != XFEATURE_AVX) { 170044a68c4eSJohn Baldwin vm_inject_gp(vmx->vm, vcpu); 170144a68c4eSJohn Baldwin return (HANDLED); 170244a68c4eSJohn Baldwin } 170344a68c4eSJohn Baldwin 170444a68c4eSJohn Baldwin /* 170544a68c4eSJohn Baldwin * AVX512 requires base AVX (YMM_Hi128) as well as OpMask, 170644a68c4eSJohn Baldwin * ZMM_Hi256, and Hi16_ZMM. 170744a68c4eSJohn Baldwin */ 170844a68c4eSJohn Baldwin if (xcrval & XFEATURE_AVX512 && 170944a68c4eSJohn Baldwin (xcrval & (XFEATURE_AVX512 | XFEATURE_AVX)) != 171044a68c4eSJohn Baldwin (XFEATURE_AVX512 | XFEATURE_AVX)) { 171144a68c4eSJohn Baldwin vm_inject_gp(vmx->vm, vcpu); 171244a68c4eSJohn Baldwin return (HANDLED); 171344a68c4eSJohn Baldwin } 171444a68c4eSJohn Baldwin 171544a68c4eSJohn Baldwin /* 171644a68c4eSJohn Baldwin * Intel MPX requires both bound register state flags to be 171744a68c4eSJohn Baldwin * set. 171844a68c4eSJohn Baldwin */ 171944a68c4eSJohn Baldwin if (((xcrval & XFEATURE_ENABLED_BNDREGS) != 0) != 172044a68c4eSJohn Baldwin ((xcrval & XFEATURE_ENABLED_BNDCSR) != 0)) { 1721dc506506SNeel Natu vm_inject_gp(vmx->vm, vcpu); 1722a0efd3fbSJohn Baldwin return (HANDLED); 1723a0efd3fbSJohn Baldwin } 1724abb023fbSJohn Baldwin 1725abb023fbSJohn Baldwin /* 1726abb023fbSJohn Baldwin * This runs "inside" vmrun() with the guest's FPU state, so 1727abb023fbSJohn Baldwin * modifying xcr0 directly modifies the guest's xcr0, not the 1728abb023fbSJohn Baldwin * host's. 1729abb023fbSJohn Baldwin */ 1730abb023fbSJohn Baldwin load_xcr(0, xcrval); 1731abb023fbSJohn Baldwin return (HANDLED); 1732abb023fbSJohn Baldwin } 1733abb023fbSJohn Baldwin 1734594db002STycho Nightingale static uint64_t 1735594db002STycho Nightingale vmx_get_guest_reg(struct vmx *vmx, int vcpu, int ident) 1736366f6083SPeter Grehan { 1737366f6083SPeter Grehan const struct vmxctx *vmxctx; 1738366f6083SPeter Grehan 1739594db002STycho Nightingale vmxctx = &vmx->ctx[vcpu]; 1740594db002STycho Nightingale 1741594db002STycho Nightingale switch (ident) { 1742594db002STycho Nightingale case 0: 1743594db002STycho Nightingale return (vmxctx->guest_rax); 1744594db002STycho Nightingale case 1: 1745594db002STycho Nightingale return (vmxctx->guest_rcx); 1746594db002STycho Nightingale case 2: 1747594db002STycho Nightingale return (vmxctx->guest_rdx); 1748594db002STycho Nightingale case 3: 1749594db002STycho Nightingale return (vmxctx->guest_rbx); 1750594db002STycho Nightingale case 4: 1751594db002STycho Nightingale return (vmcs_read(VMCS_GUEST_RSP)); 1752594db002STycho Nightingale case 5: 1753594db002STycho Nightingale return (vmxctx->guest_rbp); 1754594db002STycho Nightingale case 6: 1755594db002STycho Nightingale return (vmxctx->guest_rsi); 1756594db002STycho Nightingale case 7: 1757594db002STycho Nightingale return (vmxctx->guest_rdi); 1758594db002STycho Nightingale case 8: 1759594db002STycho Nightingale return (vmxctx->guest_r8); 1760594db002STycho Nightingale case 9: 1761594db002STycho Nightingale return (vmxctx->guest_r9); 1762594db002STycho Nightingale case 10: 1763594db002STycho Nightingale return (vmxctx->guest_r10); 1764594db002STycho Nightingale case 11: 1765594db002STycho Nightingale return (vmxctx->guest_r11); 1766594db002STycho Nightingale case 12: 1767594db002STycho Nightingale return (vmxctx->guest_r12); 1768594db002STycho Nightingale case 13: 1769594db002STycho Nightingale return (vmxctx->guest_r13); 1770594db002STycho Nightingale case 14: 1771594db002STycho Nightingale return (vmxctx->guest_r14); 1772594db002STycho Nightingale case 15: 1773594db002STycho Nightingale return (vmxctx->guest_r15); 1774594db002STycho Nightingale default: 1775594db002STycho Nightingale panic("invalid vmx register %d", ident); 1776594db002STycho Nightingale } 1777594db002STycho Nightingale } 1778594db002STycho Nightingale 1779594db002STycho Nightingale static void 1780594db002STycho Nightingale vmx_set_guest_reg(struct vmx *vmx, int vcpu, int ident, uint64_t regval) 1781594db002STycho Nightingale { 1782594db002STycho Nightingale struct vmxctx *vmxctx; 1783594db002STycho Nightingale 1784594db002STycho Nightingale vmxctx = &vmx->ctx[vcpu]; 1785594db002STycho Nightingale 1786594db002STycho Nightingale switch (ident) { 1787594db002STycho Nightingale case 0: 1788594db002STycho Nightingale vmxctx->guest_rax = regval; 1789594db002STycho Nightingale break; 1790594db002STycho Nightingale case 1: 1791594db002STycho Nightingale vmxctx->guest_rcx = regval; 1792594db002STycho Nightingale break; 1793594db002STycho Nightingale case 2: 1794594db002STycho Nightingale vmxctx->guest_rdx = regval; 1795594db002STycho Nightingale break; 1796594db002STycho Nightingale case 3: 1797594db002STycho Nightingale vmxctx->guest_rbx = regval; 1798594db002STycho Nightingale break; 1799594db002STycho Nightingale case 4: 1800594db002STycho Nightingale vmcs_write(VMCS_GUEST_RSP, regval); 1801594db002STycho Nightingale break; 1802594db002STycho Nightingale case 5: 1803594db002STycho Nightingale vmxctx->guest_rbp = regval; 1804594db002STycho Nightingale break; 1805594db002STycho Nightingale case 6: 1806594db002STycho Nightingale vmxctx->guest_rsi = regval; 1807594db002STycho Nightingale break; 1808594db002STycho Nightingale case 7: 1809594db002STycho Nightingale vmxctx->guest_rdi = regval; 1810594db002STycho Nightingale break; 1811594db002STycho Nightingale case 8: 1812594db002STycho Nightingale vmxctx->guest_r8 = regval; 1813594db002STycho Nightingale break; 1814594db002STycho Nightingale case 9: 1815594db002STycho Nightingale vmxctx->guest_r9 = regval; 1816594db002STycho Nightingale break; 1817594db002STycho Nightingale case 10: 1818594db002STycho Nightingale vmxctx->guest_r10 = regval; 1819594db002STycho Nightingale break; 1820594db002STycho Nightingale case 11: 1821594db002STycho Nightingale vmxctx->guest_r11 = regval; 1822594db002STycho Nightingale break; 1823594db002STycho Nightingale case 12: 1824594db002STycho Nightingale vmxctx->guest_r12 = regval; 1825594db002STycho Nightingale break; 1826594db002STycho Nightingale case 13: 1827594db002STycho Nightingale vmxctx->guest_r13 = regval; 1828594db002STycho Nightingale break; 1829594db002STycho Nightingale case 14: 1830594db002STycho Nightingale vmxctx->guest_r14 = regval; 1831594db002STycho Nightingale break; 1832594db002STycho Nightingale case 15: 1833594db002STycho Nightingale vmxctx->guest_r15 = regval; 1834594db002STycho Nightingale break; 1835594db002STycho Nightingale default: 1836594db002STycho Nightingale panic("invalid vmx register %d", ident); 1837594db002STycho Nightingale } 1838594db002STycho Nightingale } 1839594db002STycho Nightingale 1840594db002STycho Nightingale static int 1841594db002STycho Nightingale vmx_emulate_cr0_access(struct vmx *vmx, int vcpu, uint64_t exitqual) 1842594db002STycho Nightingale { 1843594db002STycho Nightingale uint64_t crval, regval; 1844594db002STycho Nightingale 1845594db002STycho Nightingale /* We only handle mov to %cr0 at this time */ 184639c21c2dSNeel Natu if ((exitqual & 0xf0) != 0x00) 184739c21c2dSNeel Natu return (UNHANDLED); 184839c21c2dSNeel Natu 1849594db002STycho Nightingale regval = vmx_get_guest_reg(vmx, vcpu, (exitqual >> 8) & 0xf); 1850366f6083SPeter Grehan 1851594db002STycho Nightingale vmcs_write(VMCS_CR0_SHADOW, regval); 1852366f6083SPeter Grehan 1853594db002STycho Nightingale crval = regval | cr0_ones_mask; 1854594db002STycho Nightingale crval &= ~cr0_zeros_mask; 1855594db002STycho Nightingale vmcs_write(VMCS_GUEST_CR0, crval); 1856366f6083SPeter Grehan 1857594db002STycho Nightingale if (regval & CR0_PG) { 185880a902efSPeter Grehan uint64_t efer, entry_ctls; 185980a902efSPeter Grehan 186080a902efSPeter Grehan /* 186180a902efSPeter Grehan * If CR0.PG is 1 and EFER.LME is 1 then EFER.LMA and 186280a902efSPeter Grehan * the "IA-32e mode guest" bit in VM-entry control must be 186380a902efSPeter Grehan * equal. 186480a902efSPeter Grehan */ 18653de83862SNeel Natu efer = vmcs_read(VMCS_GUEST_IA32_EFER); 186680a902efSPeter Grehan if (efer & EFER_LME) { 186780a902efSPeter Grehan efer |= EFER_LMA; 18683de83862SNeel Natu vmcs_write(VMCS_GUEST_IA32_EFER, efer); 18693de83862SNeel Natu entry_ctls = vmcs_read(VMCS_ENTRY_CTLS); 187080a902efSPeter Grehan entry_ctls |= VM_ENTRY_GUEST_LMA; 18713de83862SNeel Natu vmcs_write(VMCS_ENTRY_CTLS, entry_ctls); 187280a902efSPeter Grehan } 187380a902efSPeter Grehan } 187480a902efSPeter Grehan 1875366f6083SPeter Grehan return (HANDLED); 1876366f6083SPeter Grehan } 1877366f6083SPeter Grehan 1878594db002STycho Nightingale static int 1879594db002STycho Nightingale vmx_emulate_cr4_access(struct vmx *vmx, int vcpu, uint64_t exitqual) 1880594db002STycho Nightingale { 1881594db002STycho Nightingale uint64_t crval, regval; 1882594db002STycho Nightingale 1883594db002STycho Nightingale /* We only handle mov to %cr4 at this time */ 1884594db002STycho Nightingale if ((exitqual & 0xf0) != 0x00) 1885594db002STycho Nightingale return (UNHANDLED); 1886594db002STycho Nightingale 1887594db002STycho Nightingale regval = vmx_get_guest_reg(vmx, vcpu, (exitqual >> 8) & 0xf); 1888594db002STycho Nightingale 1889594db002STycho Nightingale vmcs_write(VMCS_CR4_SHADOW, regval); 1890594db002STycho Nightingale 1891594db002STycho Nightingale crval = regval | cr4_ones_mask; 1892594db002STycho Nightingale crval &= ~cr4_zeros_mask; 1893594db002STycho Nightingale vmcs_write(VMCS_GUEST_CR4, crval); 1894594db002STycho Nightingale 1895594db002STycho Nightingale return (HANDLED); 1896594db002STycho Nightingale } 1897594db002STycho Nightingale 1898594db002STycho Nightingale static int 1899594db002STycho Nightingale vmx_emulate_cr8_access(struct vmx *vmx, int vcpu, uint64_t exitqual) 1900594db002STycho Nightingale { 1901051f2bd1SNeel Natu struct vlapic *vlapic; 1902051f2bd1SNeel Natu uint64_t cr8; 1903051f2bd1SNeel Natu int regnum; 1904594db002STycho Nightingale 1905594db002STycho Nightingale /* We only handle mov %cr8 to/from a register at this time. */ 1906594db002STycho Nightingale if ((exitqual & 0xe0) != 0x00) { 1907594db002STycho Nightingale return (UNHANDLED); 1908594db002STycho Nightingale } 1909594db002STycho Nightingale 1910051f2bd1SNeel Natu vlapic = vm_lapic(vmx->vm, vcpu); 1911051f2bd1SNeel Natu regnum = (exitqual >> 8) & 0xf; 1912594db002STycho Nightingale if (exitqual & 0x10) { 1913051f2bd1SNeel Natu cr8 = vlapic_get_cr8(vlapic); 1914051f2bd1SNeel Natu vmx_set_guest_reg(vmx, vcpu, regnum, cr8); 1915594db002STycho Nightingale } else { 1916051f2bd1SNeel Natu cr8 = vmx_get_guest_reg(vmx, vcpu, regnum); 1917051f2bd1SNeel Natu vlapic_set_cr8(vlapic, cr8); 1918594db002STycho Nightingale } 1919594db002STycho Nightingale 1920594db002STycho Nightingale return (HANDLED); 1921594db002STycho Nightingale } 1922594db002STycho Nightingale 1923e4c8a13dSNeel Natu /* 1924e4c8a13dSNeel Natu * From section "Guest Register State" in the Intel SDM: CPL = SS.DPL 1925e4c8a13dSNeel Natu */ 1926e4c8a13dSNeel Natu static int 1927e4c8a13dSNeel Natu vmx_cpl(void) 1928e4c8a13dSNeel Natu { 1929e4c8a13dSNeel Natu uint32_t ssar; 1930e4c8a13dSNeel Natu 1931e4c8a13dSNeel Natu ssar = vmcs_read(VMCS_GUEST_SS_ACCESS_RIGHTS); 1932e4c8a13dSNeel Natu return ((ssar >> 5) & 0x3); 1933e4c8a13dSNeel Natu } 1934e4c8a13dSNeel Natu 1935e813a873SNeel Natu static enum vm_cpu_mode 193600f3efe1SJohn Baldwin vmx_cpu_mode(void) 193700f3efe1SJohn Baldwin { 1938b301b9e2SNeel Natu uint32_t csar; 193900f3efe1SJohn Baldwin 1940b301b9e2SNeel Natu if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LMA) { 1941b301b9e2SNeel Natu csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS); 1942b301b9e2SNeel Natu if (csar & 0x2000) 1943b301b9e2SNeel Natu return (CPU_MODE_64BIT); /* CS.L = 1 */ 194400f3efe1SJohn Baldwin else 194500f3efe1SJohn Baldwin return (CPU_MODE_COMPATIBILITY); 1946b301b9e2SNeel Natu } else if (vmcs_read(VMCS_GUEST_CR0) & CR0_PE) { 1947b301b9e2SNeel Natu return (CPU_MODE_PROTECTED); 1948b301b9e2SNeel Natu } else { 1949b301b9e2SNeel Natu return (CPU_MODE_REAL); 1950b301b9e2SNeel Natu } 195100f3efe1SJohn Baldwin } 195200f3efe1SJohn Baldwin 1953e813a873SNeel Natu static enum vm_paging_mode 195400f3efe1SJohn Baldwin vmx_paging_mode(void) 195500f3efe1SJohn Baldwin { 1956f3eb12e4SKonstantin Belousov uint64_t cr4; 195700f3efe1SJohn Baldwin 195800f3efe1SJohn Baldwin if (!(vmcs_read(VMCS_GUEST_CR0) & CR0_PG)) 195900f3efe1SJohn Baldwin return (PAGING_MODE_FLAT); 1960f3eb12e4SKonstantin Belousov cr4 = vmcs_read(VMCS_GUEST_CR4); 1961f3eb12e4SKonstantin Belousov if (!(cr4 & CR4_PAE)) 196200f3efe1SJohn Baldwin return (PAGING_MODE_32); 1963f3eb12e4SKonstantin Belousov if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LME) { 1964f3eb12e4SKonstantin Belousov if (!(cr4 & CR4_LA57)) 196500f3efe1SJohn Baldwin return (PAGING_MODE_64); 1966f3eb12e4SKonstantin Belousov return (PAGING_MODE_64_LA57); 1967f3eb12e4SKonstantin Belousov } else 196800f3efe1SJohn Baldwin return (PAGING_MODE_PAE); 196900f3efe1SJohn Baldwin } 197000f3efe1SJohn Baldwin 1971d17b5104SNeel Natu static uint64_t 1972d17b5104SNeel Natu inout_str_index(struct vmx *vmx, int vcpuid, int in) 1973d17b5104SNeel Natu { 1974d17b5104SNeel Natu uint64_t val; 19755c272efaSRobert Wing int error __diagused; 1976d17b5104SNeel Natu enum vm_reg_name reg; 1977d17b5104SNeel Natu 1978d17b5104SNeel Natu reg = in ? VM_REG_GUEST_RDI : VM_REG_GUEST_RSI; 1979d17b5104SNeel Natu error = vmx_getreg(vmx, vcpuid, reg, &val); 1980d17b5104SNeel Natu KASSERT(error == 0, ("%s: vmx_getreg error %d", __func__, error)); 1981d17b5104SNeel Natu return (val); 1982d17b5104SNeel Natu } 1983d17b5104SNeel Natu 1984d17b5104SNeel Natu static uint64_t 1985d17b5104SNeel Natu inout_str_count(struct vmx *vmx, int vcpuid, int rep) 1986d17b5104SNeel Natu { 1987d17b5104SNeel Natu uint64_t val; 19885c272efaSRobert Wing int error __diagused; 1989d17b5104SNeel Natu 1990d17b5104SNeel Natu if (rep) { 1991d17b5104SNeel Natu error = vmx_getreg(vmx, vcpuid, VM_REG_GUEST_RCX, &val); 1992d17b5104SNeel Natu KASSERT(!error, ("%s: vmx_getreg error %d", __func__, error)); 1993d17b5104SNeel Natu } else { 1994d17b5104SNeel Natu val = 1; 1995d17b5104SNeel Natu } 1996d17b5104SNeel Natu return (val); 1997d17b5104SNeel Natu } 1998d17b5104SNeel Natu 1999d17b5104SNeel Natu static int 2000d17b5104SNeel Natu inout_str_addrsize(uint32_t inst_info) 2001d17b5104SNeel Natu { 2002d17b5104SNeel Natu uint32_t size; 2003d17b5104SNeel Natu 2004d17b5104SNeel Natu size = (inst_info >> 7) & 0x7; 2005d17b5104SNeel Natu switch (size) { 2006d17b5104SNeel Natu case 0: 2007d17b5104SNeel Natu return (2); /* 16 bit */ 2008d17b5104SNeel Natu case 1: 2009d17b5104SNeel Natu return (4); /* 32 bit */ 2010d17b5104SNeel Natu case 2: 2011d17b5104SNeel Natu return (8); /* 64 bit */ 2012d17b5104SNeel Natu default: 2013d17b5104SNeel Natu panic("%s: invalid size encoding %d", __func__, size); 2014d17b5104SNeel Natu } 2015d17b5104SNeel Natu } 2016d17b5104SNeel Natu 2017d17b5104SNeel Natu static void 2018d17b5104SNeel Natu inout_str_seginfo(struct vmx *vmx, int vcpuid, uint32_t inst_info, int in, 2019d17b5104SNeel Natu struct vm_inout_str *vis) 2020d17b5104SNeel Natu { 20215c272efaSRobert Wing int error __diagused, s; 2022d17b5104SNeel Natu 2023d17b5104SNeel Natu if (in) { 2024d17b5104SNeel Natu vis->seg_name = VM_REG_GUEST_ES; 2025d17b5104SNeel Natu } else { 2026d17b5104SNeel Natu s = (inst_info >> 15) & 0x7; 2027d17b5104SNeel Natu vis->seg_name = vm_segment_name(s); 2028d17b5104SNeel Natu } 2029d17b5104SNeel Natu 2030d17b5104SNeel Natu error = vmx_getdesc(vmx, vcpuid, vis->seg_name, &vis->seg_desc); 2031d17b5104SNeel Natu KASSERT(error == 0, ("%s: vmx_getdesc error %d", __func__, error)); 2032d17b5104SNeel Natu } 2033d17b5104SNeel Natu 2034e4c8a13dSNeel Natu static void 2035e813a873SNeel Natu vmx_paging_info(struct vm_guest_paging *paging) 2036e813a873SNeel Natu { 2037e813a873SNeel Natu paging->cr3 = vmcs_guest_cr3(); 2038e813a873SNeel Natu paging->cpl = vmx_cpl(); 2039e813a873SNeel Natu paging->cpu_mode = vmx_cpu_mode(); 2040e813a873SNeel Natu paging->paging_mode = vmx_paging_mode(); 2041e813a873SNeel Natu } 2042e813a873SNeel Natu 2043e813a873SNeel Natu static void 2044e4c8a13dSNeel Natu vmexit_inst_emul(struct vm_exit *vmexit, uint64_t gpa, uint64_t gla) 2045e4c8a13dSNeel Natu { 2046f7a9f178SNeel Natu struct vm_guest_paging *paging; 2047f7a9f178SNeel Natu uint32_t csar; 2048f7a9f178SNeel Natu 2049f7a9f178SNeel Natu paging = &vmexit->u.inst_emul.paging; 2050f7a9f178SNeel Natu 2051e4c8a13dSNeel Natu vmexit->exitcode = VM_EXITCODE_INST_EMUL; 20521c73ea3eSNeel Natu vmexit->inst_length = 0; 2053e4c8a13dSNeel Natu vmexit->u.inst_emul.gpa = gpa; 2054e4c8a13dSNeel Natu vmexit->u.inst_emul.gla = gla; 2055f7a9f178SNeel Natu vmx_paging_info(paging); 2056f7a9f178SNeel Natu switch (paging->cpu_mode) { 2057e4f605eeSTycho Nightingale case CPU_MODE_REAL: 2058e4f605eeSTycho Nightingale vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE); 2059e4f605eeSTycho Nightingale vmexit->u.inst_emul.cs_d = 0; 2060e4f605eeSTycho Nightingale break; 2061f7a9f178SNeel Natu case CPU_MODE_PROTECTED: 2062f7a9f178SNeel Natu case CPU_MODE_COMPATIBILITY: 2063e4f605eeSTycho Nightingale vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE); 2064f7a9f178SNeel Natu csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS); 2065f7a9f178SNeel Natu vmexit->u.inst_emul.cs_d = SEG_DESC_DEF32(csar); 2066f7a9f178SNeel Natu break; 2067f7a9f178SNeel Natu default: 2068e4f605eeSTycho Nightingale vmexit->u.inst_emul.cs_base = 0; 2069f7a9f178SNeel Natu vmexit->u.inst_emul.cs_d = 0; 2070f7a9f178SNeel Natu break; 2071f7a9f178SNeel Natu } 2072c2a875f9SNeel Natu vie_init(&vmexit->u.inst_emul.vie, NULL, 0); 2073e4c8a13dSNeel Natu } 2074e4c8a13dSNeel Natu 2075366f6083SPeter Grehan static int 2076318224bbSNeel Natu ept_fault_type(uint64_t ept_qual) 2077a2da7af6SNeel Natu { 2078318224bbSNeel Natu int fault_type; 2079a2da7af6SNeel Natu 2080318224bbSNeel Natu if (ept_qual & EPT_VIOLATION_DATA_WRITE) 2081318224bbSNeel Natu fault_type = VM_PROT_WRITE; 2082318224bbSNeel Natu else if (ept_qual & EPT_VIOLATION_INST_FETCH) 2083318224bbSNeel Natu fault_type = VM_PROT_EXECUTE; 2084318224bbSNeel Natu else 2085318224bbSNeel Natu fault_type= VM_PROT_READ; 2086318224bbSNeel Natu 2087318224bbSNeel Natu return (fault_type); 2088318224bbSNeel Natu } 2089318224bbSNeel Natu 2090490d56c5SEd Maste static bool 2091318224bbSNeel Natu ept_emulation_fault(uint64_t ept_qual) 2092318224bbSNeel Natu { 2093318224bbSNeel Natu int read, write; 2094318224bbSNeel Natu 2095318224bbSNeel Natu /* EPT fault on an instruction fetch doesn't make sense here */ 2096a2da7af6SNeel Natu if (ept_qual & EPT_VIOLATION_INST_FETCH) 2097490d56c5SEd Maste return (false); 2098a2da7af6SNeel Natu 2099318224bbSNeel Natu /* EPT fault must be a read fault or a write fault */ 2100a2da7af6SNeel Natu read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0; 2101a2da7af6SNeel Natu write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0; 21023b2b0011SPeter Grehan if ((read | write) == 0) 2103490d56c5SEd Maste return (false); 2104a2da7af6SNeel Natu 2105a2da7af6SNeel Natu /* 21063b2b0011SPeter Grehan * The EPT violation must have been caused by accessing a 21073b2b0011SPeter Grehan * guest-physical address that is a translation of a guest-linear 21083b2b0011SPeter Grehan * address. 2109a2da7af6SNeel Natu */ 2110a2da7af6SNeel Natu if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 || 2111a2da7af6SNeel Natu (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) { 2112490d56c5SEd Maste return (false); 2113a2da7af6SNeel Natu } 2114a2da7af6SNeel Natu 2115490d56c5SEd Maste return (true); 2116a2da7af6SNeel Natu } 2117a2da7af6SNeel Natu 2118159dd56fSNeel Natu static __inline int 2119159dd56fSNeel Natu apic_access_virtualization(struct vmx *vmx, int vcpuid) 2120159dd56fSNeel Natu { 2121159dd56fSNeel Natu uint32_t proc_ctls2; 2122159dd56fSNeel Natu 2123159dd56fSNeel Natu proc_ctls2 = vmx->cap[vcpuid].proc_ctls2; 2124159dd56fSNeel Natu return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) ? 1 : 0); 2125159dd56fSNeel Natu } 2126159dd56fSNeel Natu 2127159dd56fSNeel Natu static __inline int 2128159dd56fSNeel Natu x2apic_virtualization(struct vmx *vmx, int vcpuid) 2129159dd56fSNeel Natu { 2130159dd56fSNeel Natu uint32_t proc_ctls2; 2131159dd56fSNeel Natu 2132159dd56fSNeel Natu proc_ctls2 = vmx->cap[vcpuid].proc_ctls2; 2133159dd56fSNeel Natu return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_X2APIC_MODE) ? 1 : 0); 2134159dd56fSNeel Natu } 2135159dd56fSNeel Natu 2136a2da7af6SNeel Natu static int 2137159dd56fSNeel Natu vmx_handle_apic_write(struct vmx *vmx, int vcpuid, struct vlapic *vlapic, 2138159dd56fSNeel Natu uint64_t qual) 213988c4b8d1SNeel Natu { 214088c4b8d1SNeel Natu int error, handled, offset; 2141159dd56fSNeel Natu uint32_t *apic_regs, vector; 214288c4b8d1SNeel Natu bool retu; 214388c4b8d1SNeel Natu 2144a0efd3fbSJohn Baldwin handled = HANDLED; 214588c4b8d1SNeel Natu offset = APIC_WRITE_OFFSET(qual); 2146159dd56fSNeel Natu 2147159dd56fSNeel Natu if (!apic_access_virtualization(vmx, vcpuid)) { 2148159dd56fSNeel Natu /* 2149159dd56fSNeel Natu * In general there should not be any APIC write VM-exits 2150159dd56fSNeel Natu * unless APIC-access virtualization is enabled. 2151159dd56fSNeel Natu * 2152159dd56fSNeel Natu * However self-IPI virtualization can legitimately trigger 2153159dd56fSNeel Natu * an APIC-write VM-exit so treat it specially. 2154159dd56fSNeel Natu */ 2155159dd56fSNeel Natu if (x2apic_virtualization(vmx, vcpuid) && 2156159dd56fSNeel Natu offset == APIC_OFFSET_SELF_IPI) { 2157159dd56fSNeel Natu apic_regs = (uint32_t *)(vlapic->apic_page); 2158159dd56fSNeel Natu vector = apic_regs[APIC_OFFSET_SELF_IPI / 4]; 2159159dd56fSNeel Natu vlapic_self_ipi_handler(vlapic, vector); 2160159dd56fSNeel Natu return (HANDLED); 2161159dd56fSNeel Natu } else 2162159dd56fSNeel Natu return (UNHANDLED); 2163159dd56fSNeel Natu } 2164159dd56fSNeel Natu 216588c4b8d1SNeel Natu switch (offset) { 216688c4b8d1SNeel Natu case APIC_OFFSET_ID: 216788c4b8d1SNeel Natu vlapic_id_write_handler(vlapic); 216888c4b8d1SNeel Natu break; 216988c4b8d1SNeel Natu case APIC_OFFSET_LDR: 217088c4b8d1SNeel Natu vlapic_ldr_write_handler(vlapic); 217188c4b8d1SNeel Natu break; 217288c4b8d1SNeel Natu case APIC_OFFSET_DFR: 217388c4b8d1SNeel Natu vlapic_dfr_write_handler(vlapic); 217488c4b8d1SNeel Natu break; 217588c4b8d1SNeel Natu case APIC_OFFSET_SVR: 217688c4b8d1SNeel Natu vlapic_svr_write_handler(vlapic); 217788c4b8d1SNeel Natu break; 217888c4b8d1SNeel Natu case APIC_OFFSET_ESR: 217988c4b8d1SNeel Natu vlapic_esr_write_handler(vlapic); 218088c4b8d1SNeel Natu break; 218188c4b8d1SNeel Natu case APIC_OFFSET_ICR_LOW: 218288c4b8d1SNeel Natu retu = false; 218388c4b8d1SNeel Natu error = vlapic_icrlo_write_handler(vlapic, &retu); 218488c4b8d1SNeel Natu if (error != 0 || retu) 2185a0efd3fbSJohn Baldwin handled = UNHANDLED; 218688c4b8d1SNeel Natu break; 218788c4b8d1SNeel Natu case APIC_OFFSET_CMCI_LVT: 218888c4b8d1SNeel Natu case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT: 218988c4b8d1SNeel Natu vlapic_lvt_write_handler(vlapic, offset); 219088c4b8d1SNeel Natu break; 219188c4b8d1SNeel Natu case APIC_OFFSET_TIMER_ICR: 219288c4b8d1SNeel Natu vlapic_icrtmr_write_handler(vlapic); 219388c4b8d1SNeel Natu break; 219488c4b8d1SNeel Natu case APIC_OFFSET_TIMER_DCR: 219588c4b8d1SNeel Natu vlapic_dcr_write_handler(vlapic); 219688c4b8d1SNeel Natu break; 219788c4b8d1SNeel Natu default: 2198a0efd3fbSJohn Baldwin handled = UNHANDLED; 219988c4b8d1SNeel Natu break; 220088c4b8d1SNeel Natu } 220188c4b8d1SNeel Natu return (handled); 220288c4b8d1SNeel Natu } 220388c4b8d1SNeel Natu 220488c4b8d1SNeel Natu static bool 2205159dd56fSNeel Natu apic_access_fault(struct vmx *vmx, int vcpuid, uint64_t gpa) 220688c4b8d1SNeel Natu { 220788c4b8d1SNeel Natu 2208159dd56fSNeel Natu if (apic_access_virtualization(vmx, vcpuid) && 220988c4b8d1SNeel Natu (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE)) 221088c4b8d1SNeel Natu return (true); 221188c4b8d1SNeel Natu else 221288c4b8d1SNeel Natu return (false); 221388c4b8d1SNeel Natu } 221488c4b8d1SNeel Natu 221588c4b8d1SNeel Natu static int 221688c4b8d1SNeel Natu vmx_handle_apic_access(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit) 221788c4b8d1SNeel Natu { 221888c4b8d1SNeel Natu uint64_t qual; 221988c4b8d1SNeel Natu int access_type, offset, allowed; 222088c4b8d1SNeel Natu 2221159dd56fSNeel Natu if (!apic_access_virtualization(vmx, vcpuid)) 222288c4b8d1SNeel Natu return (UNHANDLED); 222388c4b8d1SNeel Natu 222488c4b8d1SNeel Natu qual = vmexit->u.vmx.exit_qualification; 222588c4b8d1SNeel Natu access_type = APIC_ACCESS_TYPE(qual); 222688c4b8d1SNeel Natu offset = APIC_ACCESS_OFFSET(qual); 222788c4b8d1SNeel Natu 222888c4b8d1SNeel Natu allowed = 0; 222988c4b8d1SNeel Natu if (access_type == 0) { 223088c4b8d1SNeel Natu /* 223188c4b8d1SNeel Natu * Read data access to the following registers is expected. 223288c4b8d1SNeel Natu */ 223388c4b8d1SNeel Natu switch (offset) { 223488c4b8d1SNeel Natu case APIC_OFFSET_APR: 223588c4b8d1SNeel Natu case APIC_OFFSET_PPR: 223688c4b8d1SNeel Natu case APIC_OFFSET_RRR: 223788c4b8d1SNeel Natu case APIC_OFFSET_CMCI_LVT: 223888c4b8d1SNeel Natu case APIC_OFFSET_TIMER_CCR: 223988c4b8d1SNeel Natu allowed = 1; 224088c4b8d1SNeel Natu break; 224188c4b8d1SNeel Natu default: 224288c4b8d1SNeel Natu break; 224388c4b8d1SNeel Natu } 224488c4b8d1SNeel Natu } else if (access_type == 1) { 224588c4b8d1SNeel Natu /* 224688c4b8d1SNeel Natu * Write data access to the following registers is expected. 224788c4b8d1SNeel Natu */ 224888c4b8d1SNeel Natu switch (offset) { 224988c4b8d1SNeel Natu case APIC_OFFSET_VER: 225088c4b8d1SNeel Natu case APIC_OFFSET_APR: 225188c4b8d1SNeel Natu case APIC_OFFSET_PPR: 225288c4b8d1SNeel Natu case APIC_OFFSET_RRR: 225388c4b8d1SNeel Natu case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7: 225488c4b8d1SNeel Natu case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7: 225588c4b8d1SNeel Natu case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7: 225688c4b8d1SNeel Natu case APIC_OFFSET_CMCI_LVT: 225788c4b8d1SNeel Natu case APIC_OFFSET_TIMER_CCR: 225888c4b8d1SNeel Natu allowed = 1; 225988c4b8d1SNeel Natu break; 226088c4b8d1SNeel Natu default: 226188c4b8d1SNeel Natu break; 226288c4b8d1SNeel Natu } 226388c4b8d1SNeel Natu } 226488c4b8d1SNeel Natu 226588c4b8d1SNeel Natu if (allowed) { 2266e4c8a13dSNeel Natu vmexit_inst_emul(vmexit, DEFAULT_APIC_BASE + offset, 2267e4c8a13dSNeel Natu VIE_INVALID_GLA); 226888c4b8d1SNeel Natu } 226988c4b8d1SNeel Natu 227088c4b8d1SNeel Natu /* 227188c4b8d1SNeel Natu * Regardless of whether the APIC-access is allowed this handler 227288c4b8d1SNeel Natu * always returns UNHANDLED: 227388c4b8d1SNeel Natu * - if the access is allowed then it is handled by emulating the 227488c4b8d1SNeel Natu * instruction that caused the VM-exit (outside the critical section) 227588c4b8d1SNeel Natu * - if the access is not allowed then it will be converted to an 227688c4b8d1SNeel Natu * exitcode of VM_EXITCODE_VMX and will be dealt with in userland. 227788c4b8d1SNeel Natu */ 227888c4b8d1SNeel Natu return (UNHANDLED); 227988c4b8d1SNeel Natu } 228088c4b8d1SNeel Natu 22813d5444c8SNeel Natu static enum task_switch_reason 22823d5444c8SNeel Natu vmx_task_switch_reason(uint64_t qual) 22833d5444c8SNeel Natu { 22843d5444c8SNeel Natu int reason; 22853d5444c8SNeel Natu 22863d5444c8SNeel Natu reason = (qual >> 30) & 0x3; 22873d5444c8SNeel Natu switch (reason) { 22883d5444c8SNeel Natu case 0: 22893d5444c8SNeel Natu return (TSR_CALL); 22903d5444c8SNeel Natu case 1: 22913d5444c8SNeel Natu return (TSR_IRET); 22923d5444c8SNeel Natu case 2: 22933d5444c8SNeel Natu return (TSR_JMP); 22943d5444c8SNeel Natu case 3: 22953d5444c8SNeel Natu return (TSR_IDT_GATE); 22963d5444c8SNeel Natu default: 22973d5444c8SNeel Natu panic("%s: invalid reason %d", __func__, reason); 22983d5444c8SNeel Natu } 22993d5444c8SNeel Natu } 23003d5444c8SNeel Natu 230188c4b8d1SNeel Natu static int 2302c3498942SNeel Natu emulate_wrmsr(struct vmx *vmx, int vcpuid, u_int num, uint64_t val, bool *retu) 2303c3498942SNeel Natu { 2304c3498942SNeel Natu int error; 2305c3498942SNeel Natu 2306c3498942SNeel Natu if (lapic_msr(num)) 2307c3498942SNeel Natu error = lapic_wrmsr(vmx->vm, vcpuid, num, val, retu); 2308c3498942SNeel Natu else 2309c3498942SNeel Natu error = vmx_wrmsr(vmx, vcpuid, num, val, retu); 2310c3498942SNeel Natu 2311c3498942SNeel Natu return (error); 2312c3498942SNeel Natu } 2313c3498942SNeel Natu 2314c3498942SNeel Natu static int 2315c3498942SNeel Natu emulate_rdmsr(struct vmx *vmx, int vcpuid, u_int num, bool *retu) 2316c3498942SNeel Natu { 2317c3498942SNeel Natu struct vmxctx *vmxctx; 2318c3498942SNeel Natu uint64_t result; 2319c3498942SNeel Natu uint32_t eax, edx; 2320c3498942SNeel Natu int error; 2321c3498942SNeel Natu 2322c3498942SNeel Natu if (lapic_msr(num)) 2323c3498942SNeel Natu error = lapic_rdmsr(vmx->vm, vcpuid, num, &result, retu); 2324c3498942SNeel Natu else 2325c3498942SNeel Natu error = vmx_rdmsr(vmx, vcpuid, num, &result, retu); 2326c3498942SNeel Natu 2327c3498942SNeel Natu if (error == 0) { 2328c3498942SNeel Natu eax = result; 2329c3498942SNeel Natu vmxctx = &vmx->ctx[vcpuid]; 2330c3498942SNeel Natu error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RAX, eax); 2331c3498942SNeel Natu KASSERT(error == 0, ("vmxctx_setreg(rax) error %d", error)); 2332c3498942SNeel Natu 2333c3498942SNeel Natu edx = result >> 32; 2334c3498942SNeel Natu error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RDX, edx); 2335c3498942SNeel Natu KASSERT(error == 0, ("vmxctx_setreg(rdx) error %d", error)); 2336c3498942SNeel Natu } 2337c3498942SNeel Natu 2338c3498942SNeel Natu return (error); 2339c3498942SNeel Natu } 2340c3498942SNeel Natu 2341c3498942SNeel Natu static int 2342366f6083SPeter Grehan vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) 2343366f6083SPeter Grehan { 2344c9c75df4SNeel Natu int error, errcode, errcode_valid, handled, in; 2345366f6083SPeter Grehan struct vmxctx *vmxctx; 234688c4b8d1SNeel Natu struct vlapic *vlapic; 2347d17b5104SNeel Natu struct vm_inout_str *vis; 23483d5444c8SNeel Natu struct vm_task_switch *ts; 2349d17b5104SNeel Natu uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, inst_info; 2350b0538143SNeel Natu uint32_t intr_type, intr_vec, reason; 2351091d4532SNeel Natu uint64_t exitintinfo, qual, gpa; 2352becd9849SNeel Natu bool retu; 2353366f6083SPeter Grehan 2354160471d2SNeel Natu CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0); 2355c308b23bSNeel Natu CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0); 2356160471d2SNeel Natu 2357a0efd3fbSJohn Baldwin handled = UNHANDLED; 2358366f6083SPeter Grehan vmxctx = &vmx->ctx[vcpu]; 23590492757cSNeel Natu 2360366f6083SPeter Grehan qual = vmexit->u.vmx.exit_qualification; 2361318224bbSNeel Natu reason = vmexit->u.vmx.exit_reason; 2362366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_BOGUS; 2363366f6083SPeter Grehan 236461592433SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1); 23656ac73777STycho Nightingale SDT_PROBE3(vmm, vmx, exit, entry, vmx, vcpu, vmexit); 236661592433SNeel Natu 2367318224bbSNeel Natu /* 2368b0538143SNeel Natu * VM-entry failures during or after loading guest state. 2369b0538143SNeel Natu * 2370b0538143SNeel Natu * These VM-exits are uncommon but must be handled specially 2371b0538143SNeel Natu * as most VM-exit fields are not populated as usual. 2372b0538143SNeel Natu */ 2373b0538143SNeel Natu if (__predict_false(reason == EXIT_REASON_MCE_DURING_ENTRY)) { 2374b0538143SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Handling MCE during VM-entry"); 2375b0538143SNeel Natu __asm __volatile("int $18"); 2376b0538143SNeel Natu return (1); 2377b0538143SNeel Natu } 2378b0538143SNeel Natu 2379b0538143SNeel Natu /* 23803d5444c8SNeel Natu * VM exits that can be triggered during event delivery need to 23813d5444c8SNeel Natu * be handled specially by re-injecting the event if the IDT 23823d5444c8SNeel Natu * vectoring information field's valid bit is set. 2383318224bbSNeel Natu * 2384318224bbSNeel Natu * See "Information for VM Exits During Event Delivery" in Intel SDM 2385318224bbSNeel Natu * for details. 2386318224bbSNeel Natu */ 2387318224bbSNeel Natu idtvec_info = vmcs_idt_vectoring_info(); 2388318224bbSNeel Natu if (idtvec_info & VMCS_IDT_VEC_VALID) { 2389318224bbSNeel Natu idtvec_info &= ~(1 << 12); /* clear undefined bit */ 2390091d4532SNeel Natu exitintinfo = idtvec_info; 2391318224bbSNeel Natu if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) { 2392318224bbSNeel Natu idtvec_err = vmcs_idt_vectoring_err(); 2393091d4532SNeel Natu exitintinfo |= (uint64_t)idtvec_err << 32; 2394318224bbSNeel Natu } 2395091d4532SNeel Natu error = vm_exit_intinfo(vmx->vm, vcpu, exitintinfo); 2396091d4532SNeel Natu KASSERT(error == 0, ("%s: vm_set_intinfo error %d", 2397091d4532SNeel Natu __func__, error)); 2398091d4532SNeel Natu 2399160471d2SNeel Natu /* 2400160471d2SNeel Natu * If 'virtual NMIs' are being used and the VM-exit 2401160471d2SNeel Natu * happened while injecting an NMI during the previous 2402091d4532SNeel Natu * VM-entry, then clear "blocking by NMI" in the 2403091d4532SNeel Natu * Guest Interruptibility-State so the NMI can be 2404091d4532SNeel Natu * reinjected on the subsequent VM-entry. 2405091d4532SNeel Natu * 2406091d4532SNeel Natu * However, if the NMI was being delivered through a task 2407091d4532SNeel Natu * gate, then the new task must start execution with NMIs 2408091d4532SNeel Natu * blocked so don't clear NMI blocking in this case. 2409160471d2SNeel Natu */ 2410091d4532SNeel Natu intr_type = idtvec_info & VMCS_INTR_T_MASK; 2411091d4532SNeel Natu if (intr_type == VMCS_INTR_T_NMI) { 2412091d4532SNeel Natu if (reason != EXIT_REASON_TASK_SWITCH) 2413e5a1d950SNeel Natu vmx_clear_nmi_blocking(vmx, vcpu); 2414091d4532SNeel Natu else 2415091d4532SNeel Natu vmx_assert_nmi_blocking(vmx, vcpu); 2416160471d2SNeel Natu } 2417091d4532SNeel Natu 2418091d4532SNeel Natu /* 2419091d4532SNeel Natu * Update VM-entry instruction length if the event being 2420091d4532SNeel Natu * delivered was a software interrupt or software exception. 2421091d4532SNeel Natu */ 2422091d4532SNeel Natu if (intr_type == VMCS_INTR_T_SWINTR || 2423091d4532SNeel Natu intr_type == VMCS_INTR_T_PRIV_SWEXCEPTION || 2424091d4532SNeel Natu intr_type == VMCS_INTR_T_SWEXCEPTION) { 24253de83862SNeel Natu vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length); 2426318224bbSNeel Natu } 2427318224bbSNeel Natu } 2428318224bbSNeel Natu 2429318224bbSNeel Natu switch (reason) { 24303d5444c8SNeel Natu case EXIT_REASON_TASK_SWITCH: 24313d5444c8SNeel Natu ts = &vmexit->u.task_switch; 24323d5444c8SNeel Natu ts->tsssel = qual & 0xffff; 24333d5444c8SNeel Natu ts->reason = vmx_task_switch_reason(qual); 24343d5444c8SNeel Natu ts->ext = 0; 24353d5444c8SNeel Natu ts->errcode_valid = 0; 24363d5444c8SNeel Natu vmx_paging_info(&ts->paging); 24373d5444c8SNeel Natu /* 24383d5444c8SNeel Natu * If the task switch was due to a CALL, JMP, IRET, software 24393d5444c8SNeel Natu * interrupt (INT n) or software exception (INT3, INTO), 24403d5444c8SNeel Natu * then the saved %rip references the instruction that caused 24413d5444c8SNeel Natu * the task switch. The instruction length field in the VMCS 24423d5444c8SNeel Natu * is valid in this case. 24433d5444c8SNeel Natu * 24443d5444c8SNeel Natu * In all other cases (e.g., NMI, hardware exception) the 24453d5444c8SNeel Natu * saved %rip is one that would have been saved in the old TSS 24463d5444c8SNeel Natu * had the task switch completed normally so the instruction 24473d5444c8SNeel Natu * length field is not needed in this case and is explicitly 24483d5444c8SNeel Natu * set to 0. 24493d5444c8SNeel Natu */ 24503d5444c8SNeel Natu if (ts->reason == TSR_IDT_GATE) { 24513d5444c8SNeel Natu KASSERT(idtvec_info & VMCS_IDT_VEC_VALID, 2452091d4532SNeel Natu ("invalid idtvec_info %#x for IDT task switch", 24533d5444c8SNeel Natu idtvec_info)); 24543d5444c8SNeel Natu intr_type = idtvec_info & VMCS_INTR_T_MASK; 24553d5444c8SNeel Natu if (intr_type != VMCS_INTR_T_SWINTR && 24563d5444c8SNeel Natu intr_type != VMCS_INTR_T_SWEXCEPTION && 24573d5444c8SNeel Natu intr_type != VMCS_INTR_T_PRIV_SWEXCEPTION) { 24583d5444c8SNeel Natu /* Task switch triggered by external event */ 24593d5444c8SNeel Natu ts->ext = 1; 24603d5444c8SNeel Natu vmexit->inst_length = 0; 24613d5444c8SNeel Natu if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) { 24623d5444c8SNeel Natu ts->errcode_valid = 1; 24633d5444c8SNeel Natu ts->errcode = vmcs_idt_vectoring_err(); 24643d5444c8SNeel Natu } 24653d5444c8SNeel Natu } 24663d5444c8SNeel Natu } 24673d5444c8SNeel Natu vmexit->exitcode = VM_EXITCODE_TASK_SWITCH; 24686ac73777STycho Nightingale SDT_PROBE4(vmm, vmx, exit, taskswitch, vmx, vcpu, vmexit, ts); 24693d5444c8SNeel Natu VCPU_CTR4(vmx->vm, vcpu, "task switch reason %d, tss 0x%04x, " 24703d5444c8SNeel Natu "%s errcode 0x%016lx", ts->reason, ts->tsssel, 24713d5444c8SNeel Natu ts->ext ? "external" : "internal", 24723d5444c8SNeel Natu ((uint64_t)ts->errcode << 32) | ts->errcode_valid); 24733d5444c8SNeel Natu break; 2474366f6083SPeter Grehan case EXIT_REASON_CR_ACCESS: 2475b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CR_ACCESS, 1); 24766ac73777STycho Nightingale SDT_PROBE4(vmm, vmx, exit, craccess, vmx, vcpu, vmexit, qual); 2477594db002STycho Nightingale switch (qual & 0xf) { 2478594db002STycho Nightingale case 0: 2479594db002STycho Nightingale handled = vmx_emulate_cr0_access(vmx, vcpu, qual); 2480594db002STycho Nightingale break; 2481594db002STycho Nightingale case 4: 2482594db002STycho Nightingale handled = vmx_emulate_cr4_access(vmx, vcpu, qual); 2483594db002STycho Nightingale break; 2484594db002STycho Nightingale case 8: 2485594db002STycho Nightingale handled = vmx_emulate_cr8_access(vmx, vcpu, qual); 2486594db002STycho Nightingale break; 2487594db002STycho Nightingale } 2488366f6083SPeter Grehan break; 2489366f6083SPeter Grehan case EXIT_REASON_RDMSR: 2490b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RDMSR, 1); 2491becd9849SNeel Natu retu = false; 2492366f6083SPeter Grehan ecx = vmxctx->guest_rcx; 24932cb97c9dSNeel Natu VCPU_CTR1(vmx->vm, vcpu, "rdmsr 0x%08x", ecx); 24946ac73777STycho Nightingale SDT_PROBE4(vmm, vmx, exit, rdmsr, vmx, vcpu, vmexit, ecx); 2495c3498942SNeel Natu error = emulate_rdmsr(vmx, vcpu, ecx, &retu); 2496b42206f3SNeel Natu if (error) { 2497366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_RDMSR; 2498366f6083SPeter Grehan vmexit->u.msr.code = ecx; 2499becd9849SNeel Natu } else if (!retu) { 2500a0efd3fbSJohn Baldwin handled = HANDLED; 2501becd9849SNeel Natu } else { 2502becd9849SNeel Natu /* Return to userspace with a valid exitcode */ 2503becd9849SNeel Natu KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS, 2504c3498942SNeel Natu ("emulate_rdmsr retu with bogus exitcode")); 2505becd9849SNeel Natu } 2506366f6083SPeter Grehan break; 2507366f6083SPeter Grehan case EXIT_REASON_WRMSR: 2508b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_WRMSR, 1); 2509becd9849SNeel Natu retu = false; 2510366f6083SPeter Grehan eax = vmxctx->guest_rax; 2511366f6083SPeter Grehan ecx = vmxctx->guest_rcx; 2512366f6083SPeter Grehan edx = vmxctx->guest_rdx; 25132cb97c9dSNeel Natu VCPU_CTR2(vmx->vm, vcpu, "wrmsr 0x%08x value 0x%016lx", 25142cb97c9dSNeel Natu ecx, (uint64_t)edx << 32 | eax); 25156ac73777STycho Nightingale SDT_PROBE5(vmm, vmx, exit, wrmsr, vmx, vmexit, vcpu, ecx, 25166ac73777STycho Nightingale (uint64_t)edx << 32 | eax); 2517c3498942SNeel Natu error = emulate_wrmsr(vmx, vcpu, ecx, 2518becd9849SNeel Natu (uint64_t)edx << 32 | eax, &retu); 2519b42206f3SNeel Natu if (error) { 2520366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_WRMSR; 2521366f6083SPeter Grehan vmexit->u.msr.code = ecx; 2522366f6083SPeter Grehan vmexit->u.msr.wval = (uint64_t)edx << 32 | eax; 2523becd9849SNeel Natu } else if (!retu) { 2524a0efd3fbSJohn Baldwin handled = HANDLED; 2525becd9849SNeel Natu } else { 2526becd9849SNeel Natu /* Return to userspace with a valid exitcode */ 2527becd9849SNeel Natu KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS, 2528becd9849SNeel Natu ("emulate_wrmsr retu with bogus exitcode")); 2529becd9849SNeel Natu } 2530366f6083SPeter Grehan break; 2531366f6083SPeter Grehan case EXIT_REASON_HLT: 2532f76fc5d4SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_HLT, 1); 25336ac73777STycho Nightingale SDT_PROBE3(vmm, vmx, exit, halt, vmx, vcpu, vmexit); 2534366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_HLT; 25353de83862SNeel Natu vmexit->u.hlt.rflags = vmcs_read(VMCS_GUEST_RFLAGS); 2536490768e2STycho Nightingale if (virtual_interrupt_delivery) 2537490768e2STycho Nightingale vmexit->u.hlt.intr_status = 2538490768e2STycho Nightingale vmcs_read(VMCS_GUEST_INTR_STATUS); 2539490768e2STycho Nightingale else 2540490768e2STycho Nightingale vmexit->u.hlt.intr_status = 0; 2541366f6083SPeter Grehan break; 2542366f6083SPeter Grehan case EXIT_REASON_MTF: 2543b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_MTRAP, 1); 25446ac73777STycho Nightingale SDT_PROBE3(vmm, vmx, exit, mtrap, vmx, vcpu, vmexit); 2545366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_MTRAP; 2546c9c75df4SNeel Natu vmexit->inst_length = 0; 2547366f6083SPeter Grehan break; 2548366f6083SPeter Grehan case EXIT_REASON_PAUSE: 2549b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_PAUSE, 1); 25506ac73777STycho Nightingale SDT_PROBE3(vmm, vmx, exit, pause, vmx, vcpu, vmexit); 2551366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_PAUSE; 2552366f6083SPeter Grehan break; 2553366f6083SPeter Grehan case EXIT_REASON_INTR_WINDOW: 2554b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INTR_WINDOW, 1); 25556ac73777STycho Nightingale SDT_PROBE3(vmm, vmx, exit, intrwindow, vmx, vcpu, vmexit); 2556366f6083SPeter Grehan vmx_clear_int_window_exiting(vmx, vcpu); 2557b5aaf7b2SNeel Natu return (1); 2558366f6083SPeter Grehan case EXIT_REASON_EXT_INTR: 2559366f6083SPeter Grehan /* 2560366f6083SPeter Grehan * External interrupts serve only to cause VM exits and allow 2561366f6083SPeter Grehan * the host interrupt handler to run. 2562366f6083SPeter Grehan * 2563366f6083SPeter Grehan * If this external interrupt triggers a virtual interrupt 2564366f6083SPeter Grehan * to a VM, then that state will be recorded by the 2565366f6083SPeter Grehan * host interrupt handler in the VM's softc. We will inject 2566366f6083SPeter Grehan * this virtual interrupt during the subsequent VM enter. 2567366f6083SPeter Grehan */ 2568f7d47425SNeel Natu intr_info = vmcs_read(VMCS_EXIT_INTR_INFO); 25696ac73777STycho Nightingale SDT_PROBE4(vmm, vmx, exit, interrupt, 25706ac73777STycho Nightingale vmx, vcpu, vmexit, intr_info); 2571722b6744SJohn Baldwin 2572722b6744SJohn Baldwin /* 2573722b6744SJohn Baldwin * XXX: Ignore this exit if VMCS_INTR_VALID is not set. 2574ad3e3687SJohn Baldwin * This appears to be a bug in VMware Fusion? 2575722b6744SJohn Baldwin */ 2576722b6744SJohn Baldwin if (!(intr_info & VMCS_INTR_VALID)) 2577722b6744SJohn Baldwin return (1); 2578160471d2SNeel Natu KASSERT((intr_info & VMCS_INTR_VALID) != 0 && 2579160471d2SNeel Natu (intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_HWINTR, 2580f7d47425SNeel Natu ("VM exit interruption info invalid: %#x", intr_info)); 2581f7d47425SNeel Natu vmx_trigger_hostintr(intr_info & 0xff); 2582366f6083SPeter Grehan 2583366f6083SPeter Grehan /* 2584366f6083SPeter Grehan * This is special. We want to treat this as an 'handled' 2585366f6083SPeter Grehan * VM-exit but not increment the instruction pointer. 2586366f6083SPeter Grehan */ 2587366f6083SPeter Grehan vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1); 2588366f6083SPeter Grehan return (1); 2589366f6083SPeter Grehan case EXIT_REASON_NMI_WINDOW: 25906ac73777STycho Nightingale SDT_PROBE3(vmm, vmx, exit, nmiwindow, vmx, vcpu, vmexit); 2591366f6083SPeter Grehan /* Exit to allow the pending virtual NMI to be injected */ 259248b2d828SNeel Natu if (vm_nmi_pending(vmx->vm, vcpu)) 259348b2d828SNeel Natu vmx_inject_nmi(vmx, vcpu); 2594366f6083SPeter Grehan vmx_clear_nmi_window_exiting(vmx, vcpu); 259548b2d828SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NMI_WINDOW, 1); 2596366f6083SPeter Grehan return (1); 2597366f6083SPeter Grehan case EXIT_REASON_INOUT: 2598b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INOUT, 1); 2599366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_INOUT; 2600366f6083SPeter Grehan vmexit->u.inout.bytes = (qual & 0x7) + 1; 2601d17b5104SNeel Natu vmexit->u.inout.in = in = (qual & 0x8) ? 1 : 0; 2602366f6083SPeter Grehan vmexit->u.inout.string = (qual & 0x10) ? 1 : 0; 2603366f6083SPeter Grehan vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0; 2604366f6083SPeter Grehan vmexit->u.inout.port = (uint16_t)(qual >> 16); 2605366f6083SPeter Grehan vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax); 2606d17b5104SNeel Natu if (vmexit->u.inout.string) { 2607d17b5104SNeel Natu inst_info = vmcs_read(VMCS_EXIT_INSTRUCTION_INFO); 2608d17b5104SNeel Natu vmexit->exitcode = VM_EXITCODE_INOUT_STR; 2609d17b5104SNeel Natu vis = &vmexit->u.inout_str; 2610e813a873SNeel Natu vmx_paging_info(&vis->paging); 2611d17b5104SNeel Natu vis->rflags = vmcs_read(VMCS_GUEST_RFLAGS); 2612d17b5104SNeel Natu vis->cr0 = vmcs_read(VMCS_GUEST_CR0); 2613d17b5104SNeel Natu vis->index = inout_str_index(vmx, vcpu, in); 2614d17b5104SNeel Natu vis->count = inout_str_count(vmx, vcpu, vis->inout.rep); 2615d17b5104SNeel Natu vis->addrsize = inout_str_addrsize(inst_info); 2616d17b5104SNeel Natu inout_str_seginfo(vmx, vcpu, inst_info, in, vis); 2617762fd208STycho Nightingale } 26186ac73777STycho Nightingale SDT_PROBE3(vmm, vmx, exit, inout, vmx, vcpu, vmexit); 2619366f6083SPeter Grehan break; 2620366f6083SPeter Grehan case EXIT_REASON_CPUID: 2621b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CPUID, 1); 26226ac73777STycho Nightingale SDT_PROBE3(vmm, vmx, exit, cpuid, vmx, vcpu, vmexit); 2623a2da7af6SNeel Natu handled = vmx_handle_cpuid(vmx->vm, vcpu, vmxctx); 2624366f6083SPeter Grehan break; 2625e5a1d950SNeel Natu case EXIT_REASON_EXCEPTION: 2626c308b23bSNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXCEPTION, 1); 2627e5a1d950SNeel Natu intr_info = vmcs_read(VMCS_EXIT_INTR_INFO); 2628e5a1d950SNeel Natu KASSERT((intr_info & VMCS_INTR_VALID) != 0, 2629e5a1d950SNeel Natu ("VM exit interruption info invalid: %#x", intr_info)); 2630c308b23bSNeel Natu 2631b0538143SNeel Natu intr_vec = intr_info & 0xff; 2632b0538143SNeel Natu intr_type = intr_info & VMCS_INTR_T_MASK; 2633b0538143SNeel Natu 2634e5a1d950SNeel Natu /* 2635e5a1d950SNeel Natu * If Virtual NMIs control is 1 and the VM-exit is due to a 2636e5a1d950SNeel Natu * fault encountered during the execution of IRET then we must 2637e5a1d950SNeel Natu * restore the state of "virtual-NMI blocking" before resuming 2638e5a1d950SNeel Natu * the guest. 2639e5a1d950SNeel Natu * 2640e5a1d950SNeel Natu * See "Resuming Guest Software after Handling an Exception". 2641091d4532SNeel Natu * See "Information for VM Exits Due to Vectored Events". 2642e5a1d950SNeel Natu */ 2643e5a1d950SNeel Natu if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 && 2644b0538143SNeel Natu (intr_vec != IDT_DF) && 2645e5a1d950SNeel Natu (intr_info & EXIT_QUAL_NMIUDTI) != 0) 2646e5a1d950SNeel Natu vmx_restore_nmi_blocking(vmx, vcpu); 2647c308b23bSNeel Natu 2648c308b23bSNeel Natu /* 264962fbd7c2SNeel Natu * The NMI has already been handled in vmx_exit_handle_nmi(). 2650c308b23bSNeel Natu */ 2651b0538143SNeel Natu if (intr_type == VMCS_INTR_T_NMI) 2652c308b23bSNeel Natu return (1); 2653b0538143SNeel Natu 2654b0538143SNeel Natu /* 2655b0538143SNeel Natu * Call the machine check handler by hand. Also don't reflect 2656b0538143SNeel Natu * the machine check back into the guest. 2657b0538143SNeel Natu */ 2658b0538143SNeel Natu if (intr_vec == IDT_MC) { 2659b0538143SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Vectoring to MCE handler"); 2660b0538143SNeel Natu __asm __volatile("int $18"); 2661b0538143SNeel Natu return (1); 2662b0538143SNeel Natu } 2663b0538143SNeel Natu 2664cbd03a9dSJohn Baldwin /* 2665cbd03a9dSJohn Baldwin * If the hypervisor has requested user exits for 2666cbd03a9dSJohn Baldwin * debug exceptions, bounce them out to userland. 2667cbd03a9dSJohn Baldwin */ 2668cbd03a9dSJohn Baldwin if (intr_type == VMCS_INTR_T_SWEXCEPTION && intr_vec == IDT_BP && 2669cbd03a9dSJohn Baldwin (vmx->cap[vcpu].set & (1 << VM_CAP_BPT_EXIT))) { 2670cbd03a9dSJohn Baldwin vmexit->exitcode = VM_EXITCODE_BPT; 2671cbd03a9dSJohn Baldwin vmexit->u.bpt.inst_length = vmexit->inst_length; 2672cbd03a9dSJohn Baldwin vmexit->inst_length = 0; 2673cbd03a9dSJohn Baldwin break; 2674cbd03a9dSJohn Baldwin } 2675cbd03a9dSJohn Baldwin 2676b0538143SNeel Natu if (intr_vec == IDT_PF) { 2677b0538143SNeel Natu error = vmxctx_setreg(vmxctx, VM_REG_GUEST_CR2, qual); 2678b0538143SNeel Natu KASSERT(error == 0, ("%s: vmxctx_setreg(cr2) error %d", 2679b0538143SNeel Natu __func__, error)); 2680b0538143SNeel Natu } 2681b0538143SNeel Natu 2682b0538143SNeel Natu /* 2683b0538143SNeel Natu * Software exceptions exhibit trap-like behavior. This in 2684b0538143SNeel Natu * turn requires populating the VM-entry instruction length 2685b0538143SNeel Natu * so that the %rip in the trap frame is past the INT3/INTO 2686b0538143SNeel Natu * instruction. 2687b0538143SNeel Natu */ 2688b0538143SNeel Natu if (intr_type == VMCS_INTR_T_SWEXCEPTION) 2689b0538143SNeel Natu vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length); 2690b0538143SNeel Natu 2691b0538143SNeel Natu /* Reflect all other exceptions back into the guest */ 2692c9c75df4SNeel Natu errcode_valid = errcode = 0; 2693b0538143SNeel Natu if (intr_info & VMCS_INTR_DEL_ERRCODE) { 2694c9c75df4SNeel Natu errcode_valid = 1; 2695c9c75df4SNeel Natu errcode = vmcs_read(VMCS_EXIT_INTR_ERRCODE); 2696b0538143SNeel Natu } 2697b0538143SNeel Natu VCPU_CTR2(vmx->vm, vcpu, "Reflecting exception %d/%#x into " 2698c9c75df4SNeel Natu "the guest", intr_vec, errcode); 26996ac73777STycho Nightingale SDT_PROBE5(vmm, vmx, exit, exception, 27006ac73777STycho Nightingale vmx, vcpu, vmexit, intr_vec, errcode); 2701c9c75df4SNeel Natu error = vm_inject_exception(vmx->vm, vcpu, intr_vec, 2702c9c75df4SNeel Natu errcode_valid, errcode, 0); 2703b0538143SNeel Natu KASSERT(error == 0, ("%s: vm_inject_exception error %d", 2704b0538143SNeel Natu __func__, error)); 2705b0538143SNeel Natu return (1); 2706b0538143SNeel Natu 2707cd942e0fSPeter Grehan case EXIT_REASON_EPT_FAULT: 2708318224bbSNeel Natu /* 2709318224bbSNeel Natu * If 'gpa' lies within the address space allocated to 2710318224bbSNeel Natu * memory then this must be a nested page fault otherwise 2711318224bbSNeel Natu * this must be an instruction that accesses MMIO space. 2712318224bbSNeel Natu */ 2713a2da7af6SNeel Natu gpa = vmcs_gpa(); 27149b1aa8d6SNeel Natu if (vm_mem_allocated(vmx->vm, vcpu, gpa) || 2715159dd56fSNeel Natu apic_access_fault(vmx, vcpu, gpa)) { 2716cd942e0fSPeter Grehan vmexit->exitcode = VM_EXITCODE_PAGING; 2717d087a399SNeel Natu vmexit->inst_length = 0; 271813ec9371SPeter Grehan vmexit->u.paging.gpa = gpa; 2719318224bbSNeel Natu vmexit->u.paging.fault_type = ept_fault_type(qual); 2720bf73979dSNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NESTED_FAULT, 1); 27216ac73777STycho Nightingale SDT_PROBE5(vmm, vmx, exit, nestedfault, 27226ac73777STycho Nightingale vmx, vcpu, vmexit, gpa, qual); 2723318224bbSNeel Natu } else if (ept_emulation_fault(qual)) { 2724e4c8a13dSNeel Natu vmexit_inst_emul(vmexit, gpa, vmcs_gla()); 2725bf73979dSNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INST_EMUL, 1); 27266ac73777STycho Nightingale SDT_PROBE4(vmm, vmx, exit, mmiofault, 27276ac73777STycho Nightingale vmx, vcpu, vmexit, gpa); 2728a2da7af6SNeel Natu } 2729e5a1d950SNeel Natu /* 2730e5a1d950SNeel Natu * If Virtual NMIs control is 1 and the VM-exit is due to an 2731e5a1d950SNeel Natu * EPT fault during the execution of IRET then we must restore 2732e5a1d950SNeel Natu * the state of "virtual-NMI blocking" before resuming. 2733e5a1d950SNeel Natu * 2734e5a1d950SNeel Natu * See description of "NMI unblocking due to IRET" in 2735e5a1d950SNeel Natu * "Exit Qualification for EPT Violations". 2736e5a1d950SNeel Natu */ 2737e5a1d950SNeel Natu if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 && 2738e5a1d950SNeel Natu (qual & EXIT_QUAL_NMIUDTI) != 0) 2739e5a1d950SNeel Natu vmx_restore_nmi_blocking(vmx, vcpu); 2740cd942e0fSPeter Grehan break; 274130b94db8SNeel Natu case EXIT_REASON_VIRTUALIZED_EOI: 274230b94db8SNeel Natu vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI; 274330b94db8SNeel Natu vmexit->u.ioapic_eoi.vector = qual & 0xFF; 27446ac73777STycho Nightingale SDT_PROBE3(vmm, vmx, exit, eoi, vmx, vcpu, vmexit); 274530b94db8SNeel Natu vmexit->inst_length = 0; /* trap-like */ 274630b94db8SNeel Natu break; 274788c4b8d1SNeel Natu case EXIT_REASON_APIC_ACCESS: 27486ac73777STycho Nightingale SDT_PROBE3(vmm, vmx, exit, apicaccess, vmx, vcpu, vmexit); 274988c4b8d1SNeel Natu handled = vmx_handle_apic_access(vmx, vcpu, vmexit); 275088c4b8d1SNeel Natu break; 275188c4b8d1SNeel Natu case EXIT_REASON_APIC_WRITE: 275288c4b8d1SNeel Natu /* 275388c4b8d1SNeel Natu * APIC-write VM exit is trap-like so the %rip is already 275488c4b8d1SNeel Natu * pointing to the next instruction. 275588c4b8d1SNeel Natu */ 275688c4b8d1SNeel Natu vmexit->inst_length = 0; 275788c4b8d1SNeel Natu vlapic = vm_lapic(vmx->vm, vcpu); 27586ac73777STycho Nightingale SDT_PROBE4(vmm, vmx, exit, apicwrite, 27596ac73777STycho Nightingale vmx, vcpu, vmexit, vlapic); 2760159dd56fSNeel Natu handled = vmx_handle_apic_write(vmx, vcpu, vlapic, qual); 276188c4b8d1SNeel Natu break; 2762abb023fbSJohn Baldwin case EXIT_REASON_XSETBV: 27636ac73777STycho Nightingale SDT_PROBE3(vmm, vmx, exit, xsetbv, vmx, vcpu, vmexit); 2764a0efd3fbSJohn Baldwin handled = vmx_emulate_xsetbv(vmx, vcpu, vmexit); 2765abb023fbSJohn Baldwin break; 276665145c7fSNeel Natu case EXIT_REASON_MONITOR: 27676ac73777STycho Nightingale SDT_PROBE3(vmm, vmx, exit, monitor, vmx, vcpu, vmexit); 276865145c7fSNeel Natu vmexit->exitcode = VM_EXITCODE_MONITOR; 276965145c7fSNeel Natu break; 277065145c7fSNeel Natu case EXIT_REASON_MWAIT: 27716ac73777STycho Nightingale SDT_PROBE3(vmm, vmx, exit, mwait, vmx, vcpu, vmexit); 277265145c7fSNeel Natu vmexit->exitcode = VM_EXITCODE_MWAIT; 277365145c7fSNeel Natu break; 27741bc51badSMichael Reifenberger case EXIT_REASON_TPR: 27751bc51badSMichael Reifenberger vlapic = vm_lapic(vmx->vm, vcpu); 27761bc51badSMichael Reifenberger vlapic_sync_tpr(vlapic); 27771bc51badSMichael Reifenberger vmexit->inst_length = 0; 27781bc51badSMichael Reifenberger handled = HANDLED; 27791bc51badSMichael Reifenberger break; 278027d26457SAndrew Turner case EXIT_REASON_VMCALL: 278127d26457SAndrew Turner case EXIT_REASON_VMCLEAR: 278227d26457SAndrew Turner case EXIT_REASON_VMLAUNCH: 278327d26457SAndrew Turner case EXIT_REASON_VMPTRLD: 278427d26457SAndrew Turner case EXIT_REASON_VMPTRST: 278527d26457SAndrew Turner case EXIT_REASON_VMREAD: 278627d26457SAndrew Turner case EXIT_REASON_VMRESUME: 278727d26457SAndrew Turner case EXIT_REASON_VMWRITE: 278827d26457SAndrew Turner case EXIT_REASON_VMXOFF: 278927d26457SAndrew Turner case EXIT_REASON_VMXON: 279027d26457SAndrew Turner SDT_PROBE3(vmm, vmx, exit, vminsn, vmx, vcpu, vmexit); 279127d26457SAndrew Turner vmexit->exitcode = VM_EXITCODE_VMINSN; 279227d26457SAndrew Turner break; 2793*3ba952e1SCorvin Köhne case EXIT_REASON_WBINVD: 2794*3ba952e1SCorvin Köhne /* ignore WBINVD */ 2795*3ba952e1SCorvin Köhne handled = HANDLED; 2796*3ba952e1SCorvin Köhne break; 2797366f6083SPeter Grehan default: 27986ac73777STycho Nightingale SDT_PROBE4(vmm, vmx, exit, unknown, 27996ac73777STycho Nightingale vmx, vcpu, vmexit, reason); 2800b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_UNKNOWN, 1); 2801366f6083SPeter Grehan break; 2802366f6083SPeter Grehan } 2803366f6083SPeter Grehan 2804366f6083SPeter Grehan if (handled) { 2805366f6083SPeter Grehan /* 2806366f6083SPeter Grehan * It is possible that control is returned to userland 2807366f6083SPeter Grehan * even though we were able to handle the VM exit in the 2808eeefa4e4SNeel Natu * kernel. 2809366f6083SPeter Grehan * 2810366f6083SPeter Grehan * In such a case we want to make sure that the userland 2811366f6083SPeter Grehan * restarts guest execution at the instruction *after* 2812366f6083SPeter Grehan * the one we just processed. Therefore we update the 2813366f6083SPeter Grehan * guest rip in the VMCS and in 'vmexit'. 2814366f6083SPeter Grehan */ 2815366f6083SPeter Grehan vmexit->rip += vmexit->inst_length; 2816366f6083SPeter Grehan vmexit->inst_length = 0; 28173de83862SNeel Natu vmcs_write(VMCS_GUEST_RIP, vmexit->rip); 2818366f6083SPeter Grehan } else { 2819366f6083SPeter Grehan if (vmexit->exitcode == VM_EXITCODE_BOGUS) { 2820366f6083SPeter Grehan /* 2821366f6083SPeter Grehan * If this VM exit was not claimed by anybody then 2822366f6083SPeter Grehan * treat it as a generic VMX exit. 2823366f6083SPeter Grehan */ 2824366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_VMX; 28250492757cSNeel Natu vmexit->u.vmx.status = VM_SUCCESS; 2826c308b23bSNeel Natu vmexit->u.vmx.inst_type = 0; 2827c308b23bSNeel Natu vmexit->u.vmx.inst_error = 0; 2828366f6083SPeter Grehan } else { 2829366f6083SPeter Grehan /* 2830366f6083SPeter Grehan * The exitcode and collateral have been populated. 2831366f6083SPeter Grehan * The VM exit will be processed further in userland. 2832366f6083SPeter Grehan */ 2833366f6083SPeter Grehan } 2834366f6083SPeter Grehan } 28356ac73777STycho Nightingale 28366ac73777STycho Nightingale SDT_PROBE4(vmm, vmx, exit, return, 28376ac73777STycho Nightingale vmx, vcpu, vmexit, handled); 2838366f6083SPeter Grehan return (handled); 2839366f6083SPeter Grehan } 2840366f6083SPeter Grehan 284140487465SNeel Natu static __inline void 28420492757cSNeel Natu vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit) 28430492757cSNeel Natu { 28440492757cSNeel Natu 28450492757cSNeel Natu KASSERT(vmxctx->inst_fail_status != VM_SUCCESS, 28460492757cSNeel Natu ("vmx_exit_inst_error: invalid inst_fail_status %d", 28470492757cSNeel Natu vmxctx->inst_fail_status)); 28480492757cSNeel Natu 28490492757cSNeel Natu vmexit->inst_length = 0; 28500492757cSNeel Natu vmexit->exitcode = VM_EXITCODE_VMX; 28510492757cSNeel Natu vmexit->u.vmx.status = vmxctx->inst_fail_status; 28520492757cSNeel Natu vmexit->u.vmx.inst_error = vmcs_instruction_error(); 28530492757cSNeel Natu vmexit->u.vmx.exit_reason = ~0; 28540492757cSNeel Natu vmexit->u.vmx.exit_qualification = ~0; 28550492757cSNeel Natu 28560492757cSNeel Natu switch (rc) { 28570492757cSNeel Natu case VMX_VMRESUME_ERROR: 28580492757cSNeel Natu case VMX_VMLAUNCH_ERROR: 28590492757cSNeel Natu vmexit->u.vmx.inst_type = rc; 28600492757cSNeel Natu break; 28610492757cSNeel Natu default: 28620492757cSNeel Natu panic("vm_exit_inst_error: vmx_enter_guest returned %d", rc); 28630492757cSNeel Natu } 28640492757cSNeel Natu } 28650492757cSNeel Natu 286662fbd7c2SNeel Natu /* 286762fbd7c2SNeel Natu * If the NMI-exiting VM execution control is set to '1' then an NMI in 286862fbd7c2SNeel Natu * non-root operation causes a VM-exit. NMI blocking is in effect so it is 286962fbd7c2SNeel Natu * sufficient to simply vector to the NMI handler via a software interrupt. 287062fbd7c2SNeel Natu * However, this must be done before maskable interrupts are enabled 287162fbd7c2SNeel Natu * otherwise the "iret" issued by an interrupt handler will incorrectly 287262fbd7c2SNeel Natu * clear NMI blocking. 287362fbd7c2SNeel Natu */ 287462fbd7c2SNeel Natu static __inline void 287562fbd7c2SNeel Natu vmx_exit_handle_nmi(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit) 287662fbd7c2SNeel Natu { 287762fbd7c2SNeel Natu uint32_t intr_info; 287862fbd7c2SNeel Natu 287962fbd7c2SNeel Natu KASSERT((read_rflags() & PSL_I) == 0, ("interrupts enabled")); 288062fbd7c2SNeel Natu 288162fbd7c2SNeel Natu if (vmexit->u.vmx.exit_reason != EXIT_REASON_EXCEPTION) 288262fbd7c2SNeel Natu return; 288362fbd7c2SNeel Natu 288462fbd7c2SNeel Natu intr_info = vmcs_read(VMCS_EXIT_INTR_INFO); 288562fbd7c2SNeel Natu KASSERT((intr_info & VMCS_INTR_VALID) != 0, 288662fbd7c2SNeel Natu ("VM exit interruption info invalid: %#x", intr_info)); 288762fbd7c2SNeel Natu 288862fbd7c2SNeel Natu if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI) { 288962fbd7c2SNeel Natu KASSERT((intr_info & 0xff) == IDT_NMI, ("VM exit due " 289062fbd7c2SNeel Natu "to NMI has invalid vector: %#x", intr_info)); 289162fbd7c2SNeel Natu VCPU_CTR0(vmx->vm, vcpuid, "Vectoring to NMI handler"); 289262fbd7c2SNeel Natu __asm __volatile("int $2"); 289362fbd7c2SNeel Natu } 289462fbd7c2SNeel Natu } 289562fbd7c2SNeel Natu 289665eefbe4SJohn Baldwin static __inline void 289765eefbe4SJohn Baldwin vmx_dr_enter_guest(struct vmxctx *vmxctx) 289865eefbe4SJohn Baldwin { 289965eefbe4SJohn Baldwin register_t rflags; 290065eefbe4SJohn Baldwin 290165eefbe4SJohn Baldwin /* Save host control debug registers. */ 290265eefbe4SJohn Baldwin vmxctx->host_dr7 = rdr7(); 290365eefbe4SJohn Baldwin vmxctx->host_debugctl = rdmsr(MSR_DEBUGCTLMSR); 290465eefbe4SJohn Baldwin 290565eefbe4SJohn Baldwin /* 290665eefbe4SJohn Baldwin * Disable debugging in DR7 and DEBUGCTL to avoid triggering 290765eefbe4SJohn Baldwin * exceptions in the host based on the guest DRx values. The 290865eefbe4SJohn Baldwin * guest DR7 and DEBUGCTL are saved/restored in the VMCS. 290965eefbe4SJohn Baldwin */ 291065eefbe4SJohn Baldwin load_dr7(0); 291165eefbe4SJohn Baldwin wrmsr(MSR_DEBUGCTLMSR, 0); 291265eefbe4SJohn Baldwin 291365eefbe4SJohn Baldwin /* 291465eefbe4SJohn Baldwin * Disable single stepping the kernel to avoid corrupting the 291565eefbe4SJohn Baldwin * guest DR6. A debugger might still be able to corrupt the 291665eefbe4SJohn Baldwin * guest DR6 by setting a breakpoint after this point and then 291765eefbe4SJohn Baldwin * single stepping. 291865eefbe4SJohn Baldwin */ 291965eefbe4SJohn Baldwin rflags = read_rflags(); 292065eefbe4SJohn Baldwin vmxctx->host_tf = rflags & PSL_T; 292165eefbe4SJohn Baldwin write_rflags(rflags & ~PSL_T); 292265eefbe4SJohn Baldwin 292365eefbe4SJohn Baldwin /* Save host debug registers. */ 292465eefbe4SJohn Baldwin vmxctx->host_dr0 = rdr0(); 292565eefbe4SJohn Baldwin vmxctx->host_dr1 = rdr1(); 292665eefbe4SJohn Baldwin vmxctx->host_dr2 = rdr2(); 292765eefbe4SJohn Baldwin vmxctx->host_dr3 = rdr3(); 292865eefbe4SJohn Baldwin vmxctx->host_dr6 = rdr6(); 292965eefbe4SJohn Baldwin 293065eefbe4SJohn Baldwin /* Restore guest debug registers. */ 293165eefbe4SJohn Baldwin load_dr0(vmxctx->guest_dr0); 293265eefbe4SJohn Baldwin load_dr1(vmxctx->guest_dr1); 293365eefbe4SJohn Baldwin load_dr2(vmxctx->guest_dr2); 293465eefbe4SJohn Baldwin load_dr3(vmxctx->guest_dr3); 293565eefbe4SJohn Baldwin load_dr6(vmxctx->guest_dr6); 293665eefbe4SJohn Baldwin } 293765eefbe4SJohn Baldwin 293865eefbe4SJohn Baldwin static __inline void 293965eefbe4SJohn Baldwin vmx_dr_leave_guest(struct vmxctx *vmxctx) 294065eefbe4SJohn Baldwin { 294165eefbe4SJohn Baldwin 294265eefbe4SJohn Baldwin /* Save guest debug registers. */ 294365eefbe4SJohn Baldwin vmxctx->guest_dr0 = rdr0(); 294465eefbe4SJohn Baldwin vmxctx->guest_dr1 = rdr1(); 294565eefbe4SJohn Baldwin vmxctx->guest_dr2 = rdr2(); 294665eefbe4SJohn Baldwin vmxctx->guest_dr3 = rdr3(); 294765eefbe4SJohn Baldwin vmxctx->guest_dr6 = rdr6(); 294865eefbe4SJohn Baldwin 294965eefbe4SJohn Baldwin /* 295065eefbe4SJohn Baldwin * Restore host debug registers. Restore DR7, DEBUGCTL, and 295165eefbe4SJohn Baldwin * PSL_T last. 295265eefbe4SJohn Baldwin */ 295365eefbe4SJohn Baldwin load_dr0(vmxctx->host_dr0); 295465eefbe4SJohn Baldwin load_dr1(vmxctx->host_dr1); 295565eefbe4SJohn Baldwin load_dr2(vmxctx->host_dr2); 295665eefbe4SJohn Baldwin load_dr3(vmxctx->host_dr3); 295765eefbe4SJohn Baldwin load_dr6(vmxctx->host_dr6); 295865eefbe4SJohn Baldwin wrmsr(MSR_DEBUGCTLMSR, vmxctx->host_debugctl); 295965eefbe4SJohn Baldwin load_dr7(vmxctx->host_dr7); 296065eefbe4SJohn Baldwin write_rflags(read_rflags() | vmxctx->host_tf); 296165eefbe4SJohn Baldwin } 296265eefbe4SJohn Baldwin 29638e2cbc56SMark Johnston static __inline void 29648e2cbc56SMark Johnston vmx_pmap_activate(struct vmx *vmx, pmap_t pmap) 29658e2cbc56SMark Johnston { 29668e2cbc56SMark Johnston long eptgen; 29678e2cbc56SMark Johnston int cpu; 29688e2cbc56SMark Johnston 29698e2cbc56SMark Johnston cpu = curcpu; 29708e2cbc56SMark Johnston 29718e2cbc56SMark Johnston CPU_SET_ATOMIC(cpu, &pmap->pm_active); 29726f5a9606SMark Johnston smr_enter(pmap->pm_eptsmr); 29738e2cbc56SMark Johnston eptgen = atomic_load_long(&pmap->pm_eptgen); 29748e2cbc56SMark Johnston if (eptgen != vmx->eptgen[cpu]) { 29758e2cbc56SMark Johnston vmx->eptgen[cpu] = eptgen; 29768e2cbc56SMark Johnston invept(INVEPT_TYPE_SINGLE_CONTEXT, 29778e2cbc56SMark Johnston (struct invept_desc){ .eptp = vmx->eptp, ._res = 0 }); 29788e2cbc56SMark Johnston } 29798e2cbc56SMark Johnston } 29808e2cbc56SMark Johnston 29818e2cbc56SMark Johnston static __inline void 29828e2cbc56SMark Johnston vmx_pmap_deactivate(struct vmx *vmx, pmap_t pmap) 29838e2cbc56SMark Johnston { 29846f5a9606SMark Johnston smr_exit(pmap->pm_eptsmr); 29858e2cbc56SMark Johnston CPU_CLR_ATOMIC(curcpu, &pmap->pm_active); 29868e2cbc56SMark Johnston } 29878e2cbc56SMark Johnston 29880492757cSNeel Natu static int 29892ce12423SNeel Natu vmx_run(void *arg, int vcpu, register_t rip, pmap_t pmap, 2990248e6799SNeel Natu struct vm_eventinfo *evinfo) 29910492757cSNeel Natu { 29920492757cSNeel Natu int rc, handled, launched; 2993366f6083SPeter Grehan struct vmx *vmx; 29945b8a8cd1SNeel Natu struct vm *vm; 2995366f6083SPeter Grehan struct vmxctx *vmxctx; 2996366f6083SPeter Grehan struct vmcs *vmcs; 299798ed632cSNeel Natu struct vm_exit *vmexit; 2998de5ea6b6SNeel Natu struct vlapic *vlapic; 299979c59630SNeel Natu uint32_t exit_reason; 3000b843f9beSJohn Baldwin struct region_descriptor gdtr, idtr; 3001b843f9beSJohn Baldwin uint16_t ldt_sel; 3002366f6083SPeter Grehan 3003366f6083SPeter Grehan vmx = arg; 30045b8a8cd1SNeel Natu vm = vmx->vm; 3005366f6083SPeter Grehan vmcs = &vmx->vmcs[vcpu]; 3006366f6083SPeter Grehan vmxctx = &vmx->ctx[vcpu]; 30075b8a8cd1SNeel Natu vlapic = vm_lapic(vm, vcpu); 30085b8a8cd1SNeel Natu vmexit = vm_exitinfo(vm, vcpu); 30090492757cSNeel Natu launched = 0; 301098ed632cSNeel Natu 3011318224bbSNeel Natu KASSERT(vmxctx->pmap == pmap, 3012318224bbSNeel Natu ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap)); 3013318224bbSNeel Natu 3014c3498942SNeel Natu vmx_msr_guest_enter(vmx, vcpu); 3015c3498942SNeel Natu 3016366f6083SPeter Grehan VMPTRLD(vmcs); 3017366f6083SPeter Grehan 3018366f6083SPeter Grehan /* 3019366f6083SPeter Grehan * XXX 3020366f6083SPeter Grehan * We do this every time because we may setup the virtual machine 3021366f6083SPeter Grehan * from a different process than the one that actually runs it. 3022366f6083SPeter Grehan * 3023366f6083SPeter Grehan * If the life of a virtual machine was spent entirely in the context 302415add60dSPeter Grehan * of a single process we could do this once in vmx_init(). 3025366f6083SPeter Grehan */ 30263de83862SNeel Natu vmcs_write(VMCS_HOST_CR3, rcr3()); 3027366f6083SPeter Grehan 30282ce12423SNeel Natu vmcs_write(VMCS_GUEST_RIP, rip); 3029953c2c47SNeel Natu vmx_set_pcpu_defaults(vmx, vcpu, pmap); 3030366f6083SPeter Grehan do { 30312ce12423SNeel Natu KASSERT(vmcs_guest_rip() == rip, ("%s: vmcs guest rip mismatch " 30322ce12423SNeel Natu "%#lx/%#lx", __func__, vmcs_guest_rip(), rip)); 303340487465SNeel Natu 30342ce12423SNeel Natu handled = UNHANDLED; 30350492757cSNeel Natu /* 30360492757cSNeel Natu * Interrupts are disabled from this point on until the 30370492757cSNeel Natu * guest starts executing. This is done for the following 30380492757cSNeel Natu * reasons: 30390492757cSNeel Natu * 30400492757cSNeel Natu * If an AST is asserted on this thread after the check below, 30410492757cSNeel Natu * then the IPI_AST notification will not be lost, because it 30420492757cSNeel Natu * will cause a VM exit due to external interrupt as soon as 30430492757cSNeel Natu * the guest state is loaded. 30440492757cSNeel Natu * 30450492757cSNeel Natu * A posted interrupt after 'vmx_inject_interrupts()' will 30460492757cSNeel Natu * not be "lost" because it will be held pending in the host 30470492757cSNeel Natu * APIC because interrupts are disabled. The pending interrupt 30480492757cSNeel Natu * will be recognized as soon as the guest state is loaded. 30490492757cSNeel Natu * 30500492757cSNeel Natu * The same reasoning applies to the IPI generated by 30510492757cSNeel Natu * pmap_invalidate_ept(). 30520492757cSNeel Natu */ 30530492757cSNeel Natu disable_intr(); 30542ce12423SNeel Natu vmx_inject_interrupts(vmx, vcpu, vlapic, rip); 3055091d4532SNeel Natu 3056091d4532SNeel Natu /* 3057091d4532SNeel Natu * Check for vcpu suspension after injecting events because 3058091d4532SNeel Natu * vmx_inject_interrupts() can suspend the vcpu due to a 3059091d4532SNeel Natu * triple fault. 3060091d4532SNeel Natu */ 3061248e6799SNeel Natu if (vcpu_suspended(evinfo)) { 30620492757cSNeel Natu enable_intr(); 30632ce12423SNeel Natu vm_exit_suspended(vmx->vm, vcpu, rip); 30640492757cSNeel Natu break; 30650492757cSNeel Natu } 30660492757cSNeel Natu 3067248e6799SNeel Natu if (vcpu_rendezvous_pending(evinfo)) { 30685b8a8cd1SNeel Natu enable_intr(); 30692ce12423SNeel Natu vm_exit_rendezvous(vmx->vm, vcpu, rip); 30705b8a8cd1SNeel Natu break; 30715b8a8cd1SNeel Natu } 30725b8a8cd1SNeel Natu 3073248e6799SNeel Natu if (vcpu_reqidle(evinfo)) { 3074248e6799SNeel Natu enable_intr(); 3075248e6799SNeel Natu vm_exit_reqidle(vmx->vm, vcpu, rip); 3076248e6799SNeel Natu break; 3077248e6799SNeel Natu } 3078248e6799SNeel Natu 3079f008d157SNeel Natu if (vcpu_should_yield(vm, vcpu)) { 3080b15a09c0SNeel Natu enable_intr(); 30812ce12423SNeel Natu vm_exit_astpending(vmx->vm, vcpu, rip); 30822ce12423SNeel Natu vmx_astpending_trace(vmx, vcpu, rip); 308340487465SNeel Natu handled = HANDLED; 3084b15a09c0SNeel Natu break; 3085b15a09c0SNeel Natu } 3086b15a09c0SNeel Natu 3087fc276d92SJohn Baldwin if (vcpu_debugged(vm, vcpu)) { 3088fc276d92SJohn Baldwin enable_intr(); 3089fc276d92SJohn Baldwin vm_exit_debug(vmx->vm, vcpu, rip); 3090fc276d92SJohn Baldwin break; 3091fc276d92SJohn Baldwin } 3092fc276d92SJohn Baldwin 3093b843f9beSJohn Baldwin /* 30941bc51badSMichael Reifenberger * If TPR Shadowing is enabled, the TPR Threshold 30951bc51badSMichael Reifenberger * must be updated right before entering the guest. 30961bc51badSMichael Reifenberger */ 30971bc51badSMichael Reifenberger if (tpr_shadowing && !virtual_interrupt_delivery) { 30981bc51badSMichael Reifenberger if ((vmx->cap[vcpu].proc_ctls & PROCBASED_USE_TPR_SHADOW) != 0) { 30991bc51badSMichael Reifenberger vmcs_write(VMCS_TPR_THRESHOLD, vlapic_get_cr8(vlapic)); 31001bc51badSMichael Reifenberger } 31011bc51badSMichael Reifenberger } 31021bc51badSMichael Reifenberger 31031bc51badSMichael Reifenberger /* 3104b843f9beSJohn Baldwin * VM exits restore the base address but not the 3105b843f9beSJohn Baldwin * limits of GDTR and IDTR. The VMCS only stores the 3106b843f9beSJohn Baldwin * base address, so VM exits set the limits to 0xffff. 3107b843f9beSJohn Baldwin * Save and restore the full GDTR and IDTR to restore 3108b843f9beSJohn Baldwin * the limits. 3109b843f9beSJohn Baldwin * 3110b843f9beSJohn Baldwin * The VMCS does not save the LDTR at all, and VM 3111b843f9beSJohn Baldwin * exits clear LDTR as if a NULL selector were loaded. 3112b843f9beSJohn Baldwin * The userspace hypervisor probably doesn't use a 3113b843f9beSJohn Baldwin * LDT, but save and restore it to be safe. 3114b843f9beSJohn Baldwin */ 3115b843f9beSJohn Baldwin sgdt(&gdtr); 3116b843f9beSJohn Baldwin sidt(&idtr); 3117b843f9beSJohn Baldwin ldt_sel = sldt(); 3118b843f9beSJohn Baldwin 3119f5f5f1e7SPeter Grehan /* 3120f5f5f1e7SPeter Grehan * The TSC_AUX MSR must be saved/restored while interrupts 3121f5f5f1e7SPeter Grehan * are disabled so that it is not possible for the guest 3122f5f5f1e7SPeter Grehan * TSC_AUX MSR value to be overwritten by the resume 3123f5f5f1e7SPeter Grehan * portion of the IPI_SUSPEND codepath. This is why the 3124f5f5f1e7SPeter Grehan * transition of this MSR is handled separately from those 3125f5f5f1e7SPeter Grehan * handled by vmx_msr_guest_{enter,exit}(), which are ok to 3126f5f5f1e7SPeter Grehan * be transitioned with preemption disabled but interrupts 3127f5f5f1e7SPeter Grehan * enabled. 3128f5f5f1e7SPeter Grehan * 3129f5f5f1e7SPeter Grehan * These vmx_msr_guest_{enter,exit}_tsc_aux() calls can be 3130f5f5f1e7SPeter Grehan * anywhere in this loop so long as they happen with 3131f5f5f1e7SPeter Grehan * interrupts disabled. This location is chosen for 3132f5f5f1e7SPeter Grehan * simplicity. 3133f5f5f1e7SPeter Grehan */ 3134f5f5f1e7SPeter Grehan vmx_msr_guest_enter_tsc_aux(vmx, vcpu); 3135f5f5f1e7SPeter Grehan 313665eefbe4SJohn Baldwin vmx_dr_enter_guest(vmxctx); 313779c59630SNeel Natu 31388e2cbc56SMark Johnston /* 31398e2cbc56SMark Johnston * Mark the EPT as active on this host CPU and invalidate 31408e2cbc56SMark Johnston * EPTP-tagged TLB entries if required. 31418e2cbc56SMark Johnston */ 31428e2cbc56SMark Johnston vmx_pmap_activate(vmx, pmap); 31438e2cbc56SMark Johnston 31448e2cbc56SMark Johnston vmx_run_trace(vmx, vcpu); 31458e2cbc56SMark Johnston rc = vmx_enter_guest(vmxctx, vmx, launched); 31468e2cbc56SMark Johnston 31478e2cbc56SMark Johnston vmx_pmap_deactivate(vmx, pmap); 31488e2cbc56SMark Johnston vmx_dr_leave_guest(vmxctx); 3149f5f5f1e7SPeter Grehan vmx_msr_guest_exit_tsc_aux(vmx, vcpu); 3150f5f5f1e7SPeter Grehan 3151b843f9beSJohn Baldwin bare_lgdt(&gdtr); 3152b843f9beSJohn Baldwin lidt(&idtr); 3153b843f9beSJohn Baldwin lldt(ldt_sel); 3154b843f9beSJohn Baldwin 315579c59630SNeel Natu /* Collect some information for VM exit processing */ 315679c59630SNeel Natu vmexit->rip = rip = vmcs_guest_rip(); 315779c59630SNeel Natu vmexit->inst_length = vmexit_instruction_length(); 315879c59630SNeel Natu vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason(); 315979c59630SNeel Natu vmexit->u.vmx.exit_qualification = vmcs_exit_qualification(); 316079c59630SNeel Natu 31612ce12423SNeel Natu /* Update 'nextrip' */ 31622ce12423SNeel Natu vmx->state[vcpu].nextrip = rip; 31632ce12423SNeel Natu 31640492757cSNeel Natu if (rc == VMX_GUEST_VMEXIT) { 316562fbd7c2SNeel Natu vmx_exit_handle_nmi(vmx, vcpu, vmexit); 316662fbd7c2SNeel Natu enable_intr(); 31670492757cSNeel Natu handled = vmx_exit_process(vmx, vcpu, vmexit); 31680492757cSNeel Natu } else { 316962fbd7c2SNeel Natu enable_intr(); 317040487465SNeel Natu vmx_exit_inst_error(vmxctx, rc, vmexit); 3171eeefa4e4SNeel Natu } 317262fbd7c2SNeel Natu launched = 1; 317379c59630SNeel Natu vmx_exit_trace(vmx, vcpu, rip, exit_reason, handled); 31742ce12423SNeel Natu rip = vmexit->rip; 3175eeefa4e4SNeel Natu } while (handled); 3176366f6083SPeter Grehan 3177366f6083SPeter Grehan /* 3178366f6083SPeter Grehan * If a VM exit has been handled then the exitcode must be BOGUS 3179366f6083SPeter Grehan * If a VM exit is not handled then the exitcode must not be BOGUS 3180366f6083SPeter Grehan */ 3181366f6083SPeter Grehan if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) || 3182366f6083SPeter Grehan (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) { 3183366f6083SPeter Grehan panic("Mismatch between handled (%d) and exitcode (%d)", 3184366f6083SPeter Grehan handled, vmexit->exitcode); 3185366f6083SPeter Grehan } 3186366f6083SPeter Grehan 3187b5aaf7b2SNeel Natu if (!handled) 31885b8a8cd1SNeel Natu vmm_stat_incr(vm, vcpu, VMEXIT_USERSPACE, 1); 3189b5aaf7b2SNeel Natu 31905b8a8cd1SNeel Natu VCPU_CTR1(vm, vcpu, "returning from vmx_run: exitcode %d", 31910492757cSNeel Natu vmexit->exitcode); 3192366f6083SPeter Grehan 3193366f6083SPeter Grehan VMCLEAR(vmcs); 3194c3498942SNeel Natu vmx_msr_guest_exit(vmx, vcpu); 3195c3498942SNeel Natu 3196366f6083SPeter Grehan return (0); 3197366f6083SPeter Grehan } 3198366f6083SPeter Grehan 3199366f6083SPeter Grehan static void 320015add60dSPeter Grehan vmx_cleanup(void *arg) 3201366f6083SPeter Grehan { 320263c9389aSNeel Natu int i; 3203366f6083SPeter Grehan struct vmx *vmx = arg; 3204a488c9c9SRodney W. Grimes uint16_t maxcpus; 3205366f6083SPeter Grehan 3206159dd56fSNeel Natu if (apic_access_virtualization(vmx, 0)) 320788c4b8d1SNeel Natu vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE); 320888c4b8d1SNeel Natu 3209a488c9c9SRodney W. Grimes maxcpus = vm_get_maxcpus(vmx->vm); 3210a488c9c9SRodney W. Grimes for (i = 0; i < maxcpus; i++) 321145e51299SNeel Natu vpid_free(vmx->state[i].vpid); 321245e51299SNeel Natu 3213366f6083SPeter Grehan free(vmx, M_VMX); 3214366f6083SPeter Grehan 3215366f6083SPeter Grehan return; 3216366f6083SPeter Grehan } 3217366f6083SPeter Grehan 3218366f6083SPeter Grehan static register_t * 3219366f6083SPeter Grehan vmxctx_regptr(struct vmxctx *vmxctx, int reg) 3220366f6083SPeter Grehan { 3221366f6083SPeter Grehan 3222366f6083SPeter Grehan switch (reg) { 3223366f6083SPeter Grehan case VM_REG_GUEST_RAX: 3224366f6083SPeter Grehan return (&vmxctx->guest_rax); 3225366f6083SPeter Grehan case VM_REG_GUEST_RBX: 3226366f6083SPeter Grehan return (&vmxctx->guest_rbx); 3227366f6083SPeter Grehan case VM_REG_GUEST_RCX: 3228366f6083SPeter Grehan return (&vmxctx->guest_rcx); 3229366f6083SPeter Grehan case VM_REG_GUEST_RDX: 3230366f6083SPeter Grehan return (&vmxctx->guest_rdx); 3231366f6083SPeter Grehan case VM_REG_GUEST_RSI: 3232366f6083SPeter Grehan return (&vmxctx->guest_rsi); 3233366f6083SPeter Grehan case VM_REG_GUEST_RDI: 3234366f6083SPeter Grehan return (&vmxctx->guest_rdi); 3235366f6083SPeter Grehan case VM_REG_GUEST_RBP: 3236366f6083SPeter Grehan return (&vmxctx->guest_rbp); 3237366f6083SPeter Grehan case VM_REG_GUEST_R8: 3238366f6083SPeter Grehan return (&vmxctx->guest_r8); 3239366f6083SPeter Grehan case VM_REG_GUEST_R9: 3240366f6083SPeter Grehan return (&vmxctx->guest_r9); 3241366f6083SPeter Grehan case VM_REG_GUEST_R10: 3242366f6083SPeter Grehan return (&vmxctx->guest_r10); 3243366f6083SPeter Grehan case VM_REG_GUEST_R11: 3244366f6083SPeter Grehan return (&vmxctx->guest_r11); 3245366f6083SPeter Grehan case VM_REG_GUEST_R12: 3246366f6083SPeter Grehan return (&vmxctx->guest_r12); 3247366f6083SPeter Grehan case VM_REG_GUEST_R13: 3248366f6083SPeter Grehan return (&vmxctx->guest_r13); 3249366f6083SPeter Grehan case VM_REG_GUEST_R14: 3250366f6083SPeter Grehan return (&vmxctx->guest_r14); 3251366f6083SPeter Grehan case VM_REG_GUEST_R15: 3252366f6083SPeter Grehan return (&vmxctx->guest_r15); 325337a723a5SNeel Natu case VM_REG_GUEST_CR2: 325437a723a5SNeel Natu return (&vmxctx->guest_cr2); 325565eefbe4SJohn Baldwin case VM_REG_GUEST_DR0: 325665eefbe4SJohn Baldwin return (&vmxctx->guest_dr0); 325765eefbe4SJohn Baldwin case VM_REG_GUEST_DR1: 325865eefbe4SJohn Baldwin return (&vmxctx->guest_dr1); 325965eefbe4SJohn Baldwin case VM_REG_GUEST_DR2: 326065eefbe4SJohn Baldwin return (&vmxctx->guest_dr2); 326165eefbe4SJohn Baldwin case VM_REG_GUEST_DR3: 326265eefbe4SJohn Baldwin return (&vmxctx->guest_dr3); 326365eefbe4SJohn Baldwin case VM_REG_GUEST_DR6: 326465eefbe4SJohn Baldwin return (&vmxctx->guest_dr6); 3265366f6083SPeter Grehan default: 3266366f6083SPeter Grehan break; 3267366f6083SPeter Grehan } 3268366f6083SPeter Grehan return (NULL); 3269366f6083SPeter Grehan } 3270366f6083SPeter Grehan 3271366f6083SPeter Grehan static int 3272366f6083SPeter Grehan vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval) 3273366f6083SPeter Grehan { 3274366f6083SPeter Grehan register_t *regp; 3275366f6083SPeter Grehan 3276366f6083SPeter Grehan if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) { 3277366f6083SPeter Grehan *retval = *regp; 3278366f6083SPeter Grehan return (0); 3279366f6083SPeter Grehan } else 3280366f6083SPeter Grehan return (EINVAL); 3281366f6083SPeter Grehan } 3282366f6083SPeter Grehan 3283366f6083SPeter Grehan static int 3284366f6083SPeter Grehan vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val) 3285366f6083SPeter Grehan { 3286366f6083SPeter Grehan register_t *regp; 3287366f6083SPeter Grehan 3288366f6083SPeter Grehan if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) { 3289366f6083SPeter Grehan *regp = val; 3290366f6083SPeter Grehan return (0); 3291366f6083SPeter Grehan } else 3292366f6083SPeter Grehan return (EINVAL); 3293366f6083SPeter Grehan } 3294366f6083SPeter Grehan 3295366f6083SPeter Grehan static int 3296d1819632SNeel Natu vmx_get_intr_shadow(struct vmx *vmx, int vcpu, int running, uint64_t *retval) 3297d1819632SNeel Natu { 3298d1819632SNeel Natu uint64_t gi; 3299d1819632SNeel Natu int error; 3300d1819632SNeel Natu 3301d1819632SNeel Natu error = vmcs_getreg(&vmx->vmcs[vcpu], running, 3302d1819632SNeel Natu VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY), &gi); 3303d1819632SNeel Natu *retval = (gi & HWINTR_BLOCKING) ? 1 : 0; 3304d1819632SNeel Natu return (error); 3305d1819632SNeel Natu } 3306d1819632SNeel Natu 3307d1819632SNeel Natu static int 3308d1819632SNeel Natu vmx_modify_intr_shadow(struct vmx *vmx, int vcpu, int running, uint64_t val) 3309d1819632SNeel Natu { 3310d1819632SNeel Natu struct vmcs *vmcs; 3311d1819632SNeel Natu uint64_t gi; 3312d1819632SNeel Natu int error, ident; 3313d1819632SNeel Natu 3314d1819632SNeel Natu /* 3315d1819632SNeel Natu * Forcing the vcpu into an interrupt shadow is not supported. 3316d1819632SNeel Natu */ 3317d1819632SNeel Natu if (val) { 3318d1819632SNeel Natu error = EINVAL; 3319d1819632SNeel Natu goto done; 3320d1819632SNeel Natu } 3321d1819632SNeel Natu 3322d1819632SNeel Natu vmcs = &vmx->vmcs[vcpu]; 3323d1819632SNeel Natu ident = VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY); 3324d1819632SNeel Natu error = vmcs_getreg(vmcs, running, ident, &gi); 3325d1819632SNeel Natu if (error == 0) { 3326d1819632SNeel Natu gi &= ~HWINTR_BLOCKING; 3327d1819632SNeel Natu error = vmcs_setreg(vmcs, running, ident, gi); 3328d1819632SNeel Natu } 3329d1819632SNeel Natu done: 3330d1819632SNeel Natu VCPU_CTR2(vmx->vm, vcpu, "Setting intr_shadow to %#lx %s", val, 3331d1819632SNeel Natu error ? "failed" : "succeeded"); 3332d1819632SNeel Natu return (error); 3333d1819632SNeel Natu } 3334d1819632SNeel Natu 3335d1819632SNeel Natu static int 3336aaaa0656SPeter Grehan vmx_shadow_reg(int reg) 3337aaaa0656SPeter Grehan { 3338aaaa0656SPeter Grehan int shreg; 3339aaaa0656SPeter Grehan 3340aaaa0656SPeter Grehan shreg = -1; 3341aaaa0656SPeter Grehan 3342aaaa0656SPeter Grehan switch (reg) { 3343aaaa0656SPeter Grehan case VM_REG_GUEST_CR0: 3344aaaa0656SPeter Grehan shreg = VMCS_CR0_SHADOW; 3345aaaa0656SPeter Grehan break; 3346aaaa0656SPeter Grehan case VM_REG_GUEST_CR4: 3347aaaa0656SPeter Grehan shreg = VMCS_CR4_SHADOW; 3348aaaa0656SPeter Grehan break; 3349aaaa0656SPeter Grehan default: 3350aaaa0656SPeter Grehan break; 3351aaaa0656SPeter Grehan } 3352aaaa0656SPeter Grehan 3353aaaa0656SPeter Grehan return (shreg); 3354aaaa0656SPeter Grehan } 3355aaaa0656SPeter Grehan 3356aaaa0656SPeter Grehan static int 3357366f6083SPeter Grehan vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval) 3358366f6083SPeter Grehan { 3359d3c11f40SPeter Grehan int running, hostcpu; 3360366f6083SPeter Grehan struct vmx *vmx = arg; 3361366f6083SPeter Grehan 3362d3c11f40SPeter Grehan running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); 3363d3c11f40SPeter Grehan if (running && hostcpu != curcpu) 3364d3c11f40SPeter Grehan panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu); 3365d3c11f40SPeter Grehan 3366d1819632SNeel Natu if (reg == VM_REG_GUEST_INTR_SHADOW) 3367d1819632SNeel Natu return (vmx_get_intr_shadow(vmx, vcpu, running, retval)); 3368d1819632SNeel Natu 3369366f6083SPeter Grehan if (vmxctx_getreg(&vmx->ctx[vcpu], reg, retval) == 0) 3370366f6083SPeter Grehan return (0); 3371366f6083SPeter Grehan 3372d3c11f40SPeter Grehan return (vmcs_getreg(&vmx->vmcs[vcpu], running, reg, retval)); 3373366f6083SPeter Grehan } 3374366f6083SPeter Grehan 3375366f6083SPeter Grehan static int 3376366f6083SPeter Grehan vmx_setreg(void *arg, int vcpu, int reg, uint64_t val) 3377366f6083SPeter Grehan { 3378aaaa0656SPeter Grehan int error, hostcpu, running, shadow; 3379366f6083SPeter Grehan uint64_t ctls; 33803527963bSNeel Natu pmap_t pmap; 3381366f6083SPeter Grehan struct vmx *vmx = arg; 3382366f6083SPeter Grehan 3383d3c11f40SPeter Grehan running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); 3384d3c11f40SPeter Grehan if (running && hostcpu != curcpu) 3385d3c11f40SPeter Grehan panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu); 3386d3c11f40SPeter Grehan 3387d1819632SNeel Natu if (reg == VM_REG_GUEST_INTR_SHADOW) 3388d1819632SNeel Natu return (vmx_modify_intr_shadow(vmx, vcpu, running, val)); 3389d1819632SNeel Natu 3390366f6083SPeter Grehan if (vmxctx_setreg(&vmx->ctx[vcpu], reg, val) == 0) 3391366f6083SPeter Grehan return (0); 3392366f6083SPeter Grehan 339309860d44SEd Maste /* Do not permit user write access to VMCS fields by offset. */ 339409860d44SEd Maste if (reg < 0) 339509860d44SEd Maste return (EINVAL); 339609860d44SEd Maste 3397d3c11f40SPeter Grehan error = vmcs_setreg(&vmx->vmcs[vcpu], running, reg, val); 3398366f6083SPeter Grehan 3399366f6083SPeter Grehan if (error == 0) { 3400366f6083SPeter Grehan /* 3401366f6083SPeter Grehan * If the "load EFER" VM-entry control is 1 then the 3402366f6083SPeter Grehan * value of EFER.LMA must be identical to "IA-32e mode guest" 3403366f6083SPeter Grehan * bit in the VM-entry control. 3404366f6083SPeter Grehan */ 3405366f6083SPeter Grehan if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 && 3406366f6083SPeter Grehan (reg == VM_REG_GUEST_EFER)) { 3407d3c11f40SPeter Grehan vmcs_getreg(&vmx->vmcs[vcpu], running, 3408366f6083SPeter Grehan VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls); 3409366f6083SPeter Grehan if (val & EFER_LMA) 3410366f6083SPeter Grehan ctls |= VM_ENTRY_GUEST_LMA; 3411366f6083SPeter Grehan else 3412366f6083SPeter Grehan ctls &= ~VM_ENTRY_GUEST_LMA; 3413d3c11f40SPeter Grehan vmcs_setreg(&vmx->vmcs[vcpu], running, 3414366f6083SPeter Grehan VMCS_IDENT(VMCS_ENTRY_CTLS), ctls); 3415366f6083SPeter Grehan } 3416aaaa0656SPeter Grehan 3417aaaa0656SPeter Grehan shadow = vmx_shadow_reg(reg); 3418aaaa0656SPeter Grehan if (shadow > 0) { 3419aaaa0656SPeter Grehan /* 3420aaaa0656SPeter Grehan * Store the unmodified value in the shadow 3421aaaa0656SPeter Grehan */ 3422aaaa0656SPeter Grehan error = vmcs_setreg(&vmx->vmcs[vcpu], running, 3423aaaa0656SPeter Grehan VMCS_IDENT(shadow), val); 3424aaaa0656SPeter Grehan } 34253527963bSNeel Natu 34263527963bSNeel Natu if (reg == VM_REG_GUEST_CR3) { 34273527963bSNeel Natu /* 34283527963bSNeel Natu * Invalidate the guest vcpu's TLB mappings to emulate 34293527963bSNeel Natu * the behavior of updating %cr3. 34303527963bSNeel Natu * 34313527963bSNeel Natu * XXX the processor retains global mappings when %cr3 34323527963bSNeel Natu * is updated but vmx_invvpid() does not. 34333527963bSNeel Natu */ 34343527963bSNeel Natu pmap = vmx->ctx[vcpu].pmap; 34353527963bSNeel Natu vmx_invvpid(vmx, vcpu, pmap, running); 34363527963bSNeel Natu } 3437366f6083SPeter Grehan } 3438366f6083SPeter Grehan 3439366f6083SPeter Grehan return (error); 3440366f6083SPeter Grehan } 3441366f6083SPeter Grehan 3442366f6083SPeter Grehan static int 3443366f6083SPeter Grehan vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc) 3444366f6083SPeter Grehan { 3445ba6f5e23SNeel Natu int hostcpu, running; 3446366f6083SPeter Grehan struct vmx *vmx = arg; 3447366f6083SPeter Grehan 3448ba6f5e23SNeel Natu running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); 3449ba6f5e23SNeel Natu if (running && hostcpu != curcpu) 3450ba6f5e23SNeel Natu panic("vmx_getdesc: %s%d is running", vm_name(vmx->vm), vcpu); 3451ba6f5e23SNeel Natu 3452ba6f5e23SNeel Natu return (vmcs_getdesc(&vmx->vmcs[vcpu], running, reg, desc)); 3453366f6083SPeter Grehan } 3454366f6083SPeter Grehan 3455366f6083SPeter Grehan static int 3456366f6083SPeter Grehan vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc) 3457366f6083SPeter Grehan { 3458ba6f5e23SNeel Natu int hostcpu, running; 3459366f6083SPeter Grehan struct vmx *vmx = arg; 3460366f6083SPeter Grehan 3461ba6f5e23SNeel Natu running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); 3462ba6f5e23SNeel Natu if (running && hostcpu != curcpu) 3463ba6f5e23SNeel Natu panic("vmx_setdesc: %s%d is running", vm_name(vmx->vm), vcpu); 3464ba6f5e23SNeel Natu 3465ba6f5e23SNeel Natu return (vmcs_setdesc(&vmx->vmcs[vcpu], running, reg, desc)); 3466366f6083SPeter Grehan } 3467366f6083SPeter Grehan 3468366f6083SPeter Grehan static int 3469366f6083SPeter Grehan vmx_getcap(void *arg, int vcpu, int type, int *retval) 3470366f6083SPeter Grehan { 3471366f6083SPeter Grehan struct vmx *vmx = arg; 3472366f6083SPeter Grehan int vcap; 3473366f6083SPeter Grehan int ret; 3474366f6083SPeter Grehan 3475366f6083SPeter Grehan ret = ENOENT; 3476366f6083SPeter Grehan 3477366f6083SPeter Grehan vcap = vmx->cap[vcpu].set; 3478366f6083SPeter Grehan 3479366f6083SPeter Grehan switch (type) { 3480366f6083SPeter Grehan case VM_CAP_HALT_EXIT: 3481366f6083SPeter Grehan if (cap_halt_exit) 3482366f6083SPeter Grehan ret = 0; 3483366f6083SPeter Grehan break; 3484366f6083SPeter Grehan case VM_CAP_PAUSE_EXIT: 3485366f6083SPeter Grehan if (cap_pause_exit) 3486366f6083SPeter Grehan ret = 0; 3487366f6083SPeter Grehan break; 3488366f6083SPeter Grehan case VM_CAP_MTRAP_EXIT: 3489366f6083SPeter Grehan if (cap_monitor_trap) 3490366f6083SPeter Grehan ret = 0; 3491366f6083SPeter Grehan break; 3492f5f5f1e7SPeter Grehan case VM_CAP_RDPID: 3493f5f5f1e7SPeter Grehan if (cap_rdpid) 3494f5f5f1e7SPeter Grehan ret = 0; 3495f5f5f1e7SPeter Grehan break; 3496f5f5f1e7SPeter Grehan case VM_CAP_RDTSCP: 3497f5f5f1e7SPeter Grehan if (cap_rdtscp) 3498f5f5f1e7SPeter Grehan ret = 0; 3499f5f5f1e7SPeter Grehan break; 3500366f6083SPeter Grehan case VM_CAP_UNRESTRICTED_GUEST: 3501366f6083SPeter Grehan if (cap_unrestricted_guest) 3502366f6083SPeter Grehan ret = 0; 3503366f6083SPeter Grehan break; 350449cc03daSNeel Natu case VM_CAP_ENABLE_INVPCID: 350549cc03daSNeel Natu if (cap_invpcid) 350649cc03daSNeel Natu ret = 0; 350749cc03daSNeel Natu break; 3508cbd03a9dSJohn Baldwin case VM_CAP_BPT_EXIT: 3509cbd03a9dSJohn Baldwin ret = 0; 3510cbd03a9dSJohn Baldwin break; 3511366f6083SPeter Grehan default: 3512366f6083SPeter Grehan break; 3513366f6083SPeter Grehan } 3514366f6083SPeter Grehan 3515366f6083SPeter Grehan if (ret == 0) 3516366f6083SPeter Grehan *retval = (vcap & (1 << type)) ? 1 : 0; 3517366f6083SPeter Grehan 3518366f6083SPeter Grehan return (ret); 3519366f6083SPeter Grehan } 3520366f6083SPeter Grehan 3521366f6083SPeter Grehan static int 3522366f6083SPeter Grehan vmx_setcap(void *arg, int vcpu, int type, int val) 3523366f6083SPeter Grehan { 3524366f6083SPeter Grehan struct vmx *vmx = arg; 3525366f6083SPeter Grehan struct vmcs *vmcs = &vmx->vmcs[vcpu]; 3526366f6083SPeter Grehan uint32_t baseval; 3527366f6083SPeter Grehan uint32_t *pptr; 3528366f6083SPeter Grehan int error; 3529366f6083SPeter Grehan int flag; 3530366f6083SPeter Grehan int reg; 3531366f6083SPeter Grehan int retval; 3532366f6083SPeter Grehan 3533366f6083SPeter Grehan retval = ENOENT; 3534366f6083SPeter Grehan pptr = NULL; 3535366f6083SPeter Grehan 3536366f6083SPeter Grehan switch (type) { 3537366f6083SPeter Grehan case VM_CAP_HALT_EXIT: 3538366f6083SPeter Grehan if (cap_halt_exit) { 3539366f6083SPeter Grehan retval = 0; 3540366f6083SPeter Grehan pptr = &vmx->cap[vcpu].proc_ctls; 3541366f6083SPeter Grehan baseval = *pptr; 3542366f6083SPeter Grehan flag = PROCBASED_HLT_EXITING; 3543366f6083SPeter Grehan reg = VMCS_PRI_PROC_BASED_CTLS; 3544366f6083SPeter Grehan } 3545366f6083SPeter Grehan break; 3546366f6083SPeter Grehan case VM_CAP_MTRAP_EXIT: 3547366f6083SPeter Grehan if (cap_monitor_trap) { 3548366f6083SPeter Grehan retval = 0; 3549366f6083SPeter Grehan pptr = &vmx->cap[vcpu].proc_ctls; 3550366f6083SPeter Grehan baseval = *pptr; 3551366f6083SPeter Grehan flag = PROCBASED_MTF; 3552366f6083SPeter Grehan reg = VMCS_PRI_PROC_BASED_CTLS; 3553366f6083SPeter Grehan } 3554366f6083SPeter Grehan break; 3555366f6083SPeter Grehan case VM_CAP_PAUSE_EXIT: 3556366f6083SPeter Grehan if (cap_pause_exit) { 3557366f6083SPeter Grehan retval = 0; 3558366f6083SPeter Grehan pptr = &vmx->cap[vcpu].proc_ctls; 3559366f6083SPeter Grehan baseval = *pptr; 3560366f6083SPeter Grehan flag = PROCBASED_PAUSE_EXITING; 3561366f6083SPeter Grehan reg = VMCS_PRI_PROC_BASED_CTLS; 3562366f6083SPeter Grehan } 3563366f6083SPeter Grehan break; 3564f5f5f1e7SPeter Grehan case VM_CAP_RDPID: 3565f5f5f1e7SPeter Grehan case VM_CAP_RDTSCP: 3566f5f5f1e7SPeter Grehan if (cap_rdpid || cap_rdtscp) 3567f5f5f1e7SPeter Grehan /* 3568f5f5f1e7SPeter Grehan * Choose not to support enabling/disabling 3569f5f5f1e7SPeter Grehan * RDPID/RDTSCP via libvmmapi since, as per the 357015add60dSPeter Grehan * discussion in vmx_modinit(), RDPID/RDTSCP are 3571f5f5f1e7SPeter Grehan * either always enabled or always disabled. 3572f5f5f1e7SPeter Grehan */ 3573f5f5f1e7SPeter Grehan error = EOPNOTSUPP; 3574f5f5f1e7SPeter Grehan break; 3575366f6083SPeter Grehan case VM_CAP_UNRESTRICTED_GUEST: 3576366f6083SPeter Grehan if (cap_unrestricted_guest) { 3577366f6083SPeter Grehan retval = 0; 357849cc03daSNeel Natu pptr = &vmx->cap[vcpu].proc_ctls2; 357949cc03daSNeel Natu baseval = *pptr; 3580366f6083SPeter Grehan flag = PROCBASED2_UNRESTRICTED_GUEST; 3581366f6083SPeter Grehan reg = VMCS_SEC_PROC_BASED_CTLS; 3582366f6083SPeter Grehan } 3583366f6083SPeter Grehan break; 358449cc03daSNeel Natu case VM_CAP_ENABLE_INVPCID: 358549cc03daSNeel Natu if (cap_invpcid) { 358649cc03daSNeel Natu retval = 0; 358749cc03daSNeel Natu pptr = &vmx->cap[vcpu].proc_ctls2; 358849cc03daSNeel Natu baseval = *pptr; 358949cc03daSNeel Natu flag = PROCBASED2_ENABLE_INVPCID; 359049cc03daSNeel Natu reg = VMCS_SEC_PROC_BASED_CTLS; 359149cc03daSNeel Natu } 359249cc03daSNeel Natu break; 3593cbd03a9dSJohn Baldwin case VM_CAP_BPT_EXIT: 3594cbd03a9dSJohn Baldwin retval = 0; 3595cbd03a9dSJohn Baldwin 3596cbd03a9dSJohn Baldwin /* Don't change the bitmap if we are tracing all exceptions. */ 3597cbd03a9dSJohn Baldwin if (vmx->cap[vcpu].exc_bitmap != 0xffffffff) { 3598cbd03a9dSJohn Baldwin pptr = &vmx->cap[vcpu].exc_bitmap; 3599cbd03a9dSJohn Baldwin baseval = *pptr; 3600cbd03a9dSJohn Baldwin flag = (1 << IDT_BP); 3601cbd03a9dSJohn Baldwin reg = VMCS_EXCEPTION_BITMAP; 3602cbd03a9dSJohn Baldwin } 3603cbd03a9dSJohn Baldwin break; 3604366f6083SPeter Grehan default: 3605366f6083SPeter Grehan break; 3606366f6083SPeter Grehan } 3607366f6083SPeter Grehan 3608cbd03a9dSJohn Baldwin if (retval) 3609cbd03a9dSJohn Baldwin return (retval); 3610cbd03a9dSJohn Baldwin 3611cbd03a9dSJohn Baldwin if (pptr != NULL) { 3612366f6083SPeter Grehan if (val) { 3613366f6083SPeter Grehan baseval |= flag; 3614366f6083SPeter Grehan } else { 3615366f6083SPeter Grehan baseval &= ~flag; 3616366f6083SPeter Grehan } 3617366f6083SPeter Grehan VMPTRLD(vmcs); 3618366f6083SPeter Grehan error = vmwrite(reg, baseval); 3619366f6083SPeter Grehan VMCLEAR(vmcs); 3620366f6083SPeter Grehan 3621cbd03a9dSJohn Baldwin if (error) 3622cbd03a9dSJohn Baldwin return (error); 3623cbd03a9dSJohn Baldwin 3624366f6083SPeter Grehan /* 3625366f6083SPeter Grehan * Update optional stored flags, and record 3626366f6083SPeter Grehan * setting 3627366f6083SPeter Grehan */ 3628366f6083SPeter Grehan *pptr = baseval; 3629366f6083SPeter Grehan } 3630366f6083SPeter Grehan 3631366f6083SPeter Grehan if (val) { 3632366f6083SPeter Grehan vmx->cap[vcpu].set |= (1 << type); 3633366f6083SPeter Grehan } else { 3634366f6083SPeter Grehan vmx->cap[vcpu].set &= ~(1 << type); 3635366f6083SPeter Grehan } 3636366f6083SPeter Grehan 3637cbd03a9dSJohn Baldwin return (0); 3638366f6083SPeter Grehan } 3639366f6083SPeter Grehan 364015add60dSPeter Grehan static struct vmspace * 364115add60dSPeter Grehan vmx_vmspace_alloc(vm_offset_t min, vm_offset_t max) 364215add60dSPeter Grehan { 364315add60dSPeter Grehan return (ept_vmspace_alloc(min, max)); 364415add60dSPeter Grehan } 364515add60dSPeter Grehan 364615add60dSPeter Grehan static void 364715add60dSPeter Grehan vmx_vmspace_free(struct vmspace *vmspace) 364815add60dSPeter Grehan { 364915add60dSPeter Grehan ept_vmspace_free(vmspace); 365015add60dSPeter Grehan } 365115add60dSPeter Grehan 365288c4b8d1SNeel Natu struct vlapic_vtx { 365388c4b8d1SNeel Natu struct vlapic vlapic; 3654176666c2SNeel Natu struct pir_desc *pir_desc; 365530b94db8SNeel Natu struct vmx *vmx; 36562c352febSJohn Baldwin u_int pending_prio; 365788c4b8d1SNeel Natu }; 365888c4b8d1SNeel Natu 36592c352febSJohn Baldwin #define VPR_PRIO_BIT(vpr) (1 << ((vpr) >> 4)) 36602c352febSJohn Baldwin 366188c4b8d1SNeel Natu #define VMX_CTR_PIR(vm, vcpuid, pir_desc, notify, vector, level, msg) \ 366288c4b8d1SNeel Natu do { \ 366388c4b8d1SNeel Natu VCPU_CTR2(vm, vcpuid, msg " assert %s-triggered vector %d", \ 366488c4b8d1SNeel Natu level ? "level" : "edge", vector); \ 366588c4b8d1SNeel Natu VCPU_CTR1(vm, vcpuid, msg " pir0 0x%016lx", pir_desc->pir[0]); \ 366688c4b8d1SNeel Natu VCPU_CTR1(vm, vcpuid, msg " pir1 0x%016lx", pir_desc->pir[1]); \ 366788c4b8d1SNeel Natu VCPU_CTR1(vm, vcpuid, msg " pir2 0x%016lx", pir_desc->pir[2]); \ 366888c4b8d1SNeel Natu VCPU_CTR1(vm, vcpuid, msg " pir3 0x%016lx", pir_desc->pir[3]); \ 366988c4b8d1SNeel Natu VCPU_CTR1(vm, vcpuid, msg " notify: %s", notify ? "yes" : "no");\ 367088c4b8d1SNeel Natu } while (0) 367188c4b8d1SNeel Natu 367288c4b8d1SNeel Natu /* 367388c4b8d1SNeel Natu * vlapic->ops handlers that utilize the APICv hardware assist described in 367488c4b8d1SNeel Natu * Chapter 29 of the Intel SDM. 367588c4b8d1SNeel Natu */ 367688c4b8d1SNeel Natu static int 367788c4b8d1SNeel Natu vmx_set_intr_ready(struct vlapic *vlapic, int vector, bool level) 367888c4b8d1SNeel Natu { 367988c4b8d1SNeel Natu struct vlapic_vtx *vlapic_vtx; 368088c4b8d1SNeel Natu struct pir_desc *pir_desc; 368188c4b8d1SNeel Natu uint64_t mask; 36822c352febSJohn Baldwin int idx, notify = 0; 368388c4b8d1SNeel Natu 368488c4b8d1SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 3685176666c2SNeel Natu pir_desc = vlapic_vtx->pir_desc; 368688c4b8d1SNeel Natu 368788c4b8d1SNeel Natu /* 368888c4b8d1SNeel Natu * Keep track of interrupt requests in the PIR descriptor. This is 368988c4b8d1SNeel Natu * because the virtual APIC page pointed to by the VMCS cannot be 369088c4b8d1SNeel Natu * modified if the vcpu is running. 369188c4b8d1SNeel Natu */ 369288c4b8d1SNeel Natu idx = vector / 64; 369388c4b8d1SNeel Natu mask = 1UL << (vector % 64); 369488c4b8d1SNeel Natu atomic_set_long(&pir_desc->pir[idx], mask); 36952c352febSJohn Baldwin 36962c352febSJohn Baldwin /* 36972c352febSJohn Baldwin * A notification is required whenever the 'pending' bit makes a 36982c352febSJohn Baldwin * transition from 0->1. 36992c352febSJohn Baldwin * 37002c352febSJohn Baldwin * Even if the 'pending' bit is already asserted, notification about 37012c352febSJohn Baldwin * the incoming interrupt may still be necessary. For example, if a 37022c352febSJohn Baldwin * vCPU is HLTed with a high PPR, a low priority interrupt would cause 37032c352febSJohn Baldwin * the 0->1 'pending' transition with a notification, but the vCPU 37042c352febSJohn Baldwin * would ignore the interrupt for the time being. The same vCPU would 37052c352febSJohn Baldwin * need to then be notified if a high-priority interrupt arrived which 37062c352febSJohn Baldwin * satisfied the PPR. 37072c352febSJohn Baldwin * 37082c352febSJohn Baldwin * The priorities of interrupts injected while 'pending' is asserted 37092c352febSJohn Baldwin * are tracked in a custom bitfield 'pending_prio'. Should the 37102c352febSJohn Baldwin * to-be-injected interrupt exceed the priorities already present, the 37112c352febSJohn Baldwin * notification is sent. The priorities recorded in 'pending_prio' are 37122c352febSJohn Baldwin * cleared whenever the 'pending' bit makes another 0->1 transition. 37132c352febSJohn Baldwin */ 37142c352febSJohn Baldwin if (atomic_cmpset_long(&pir_desc->pending, 0, 1) != 0) { 37152c352febSJohn Baldwin notify = 1; 37162c352febSJohn Baldwin vlapic_vtx->pending_prio = 0; 37172c352febSJohn Baldwin } else { 37182c352febSJohn Baldwin const u_int old_prio = vlapic_vtx->pending_prio; 37192c352febSJohn Baldwin const u_int prio_bit = VPR_PRIO_BIT(vector & APIC_TPR_INT); 37202c352febSJohn Baldwin 37212c352febSJohn Baldwin if ((old_prio & prio_bit) == 0 && prio_bit > old_prio) { 37222c352febSJohn Baldwin atomic_set_int(&vlapic_vtx->pending_prio, prio_bit); 37232c352febSJohn Baldwin notify = 1; 37242c352febSJohn Baldwin } 37252c352febSJohn Baldwin } 372688c4b8d1SNeel Natu 372788c4b8d1SNeel Natu VMX_CTR_PIR(vlapic->vm, vlapic->vcpuid, pir_desc, notify, vector, 372888c4b8d1SNeel Natu level, "vmx_set_intr_ready"); 372988c4b8d1SNeel Natu return (notify); 373088c4b8d1SNeel Natu } 373188c4b8d1SNeel Natu 373288c4b8d1SNeel Natu static int 373388c4b8d1SNeel Natu vmx_pending_intr(struct vlapic *vlapic, int *vecptr) 373488c4b8d1SNeel Natu { 373588c4b8d1SNeel Natu struct vlapic_vtx *vlapic_vtx; 373688c4b8d1SNeel Natu struct pir_desc *pir_desc; 373788c4b8d1SNeel Natu struct LAPIC *lapic; 373888c4b8d1SNeel Natu uint64_t pending, pirval; 373988c4b8d1SNeel Natu uint32_t ppr, vpr; 374088c4b8d1SNeel Natu int i; 374188c4b8d1SNeel Natu 374288c4b8d1SNeel Natu /* 374388c4b8d1SNeel Natu * This function is only expected to be called from the 'HLT' exit 374488c4b8d1SNeel Natu * handler which does not care about the vector that is pending. 374588c4b8d1SNeel Natu */ 374688c4b8d1SNeel Natu KASSERT(vecptr == NULL, ("vmx_pending_intr: vecptr must be NULL")); 374788c4b8d1SNeel Natu 374888c4b8d1SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 3749176666c2SNeel Natu pir_desc = vlapic_vtx->pir_desc; 375088c4b8d1SNeel Natu 375188c4b8d1SNeel Natu pending = atomic_load_acq_long(&pir_desc->pending); 37529e33a616STycho Nightingale if (!pending) { 37539e33a616STycho Nightingale /* 37549e33a616STycho Nightingale * While a virtual interrupt may have already been 37559e33a616STycho Nightingale * processed the actual delivery maybe pending the 37569e33a616STycho Nightingale * interruptibility of the guest. Recognize a pending 37579e33a616STycho Nightingale * interrupt by reevaluating virtual interrupts 37589e33a616STycho Nightingale * following Section 29.2.1 in the Intel SDM Volume 3. 37599e33a616STycho Nightingale */ 3760490768e2STycho Nightingale struct vm_exit *vmexit; 37619e33a616STycho Nightingale uint8_t rvi, ppr; 37629e33a616STycho Nightingale 3763490768e2STycho Nightingale vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid); 3764490768e2STycho Nightingale KASSERT(vmexit->exitcode == VM_EXITCODE_HLT, 3765490768e2STycho Nightingale ("vmx_pending_intr: exitcode not 'HLT'")); 3766490768e2STycho Nightingale rvi = vmexit->u.hlt.intr_status & APIC_TPR_INT; 37679e33a616STycho Nightingale lapic = vlapic->apic_page; 37689e33a616STycho Nightingale ppr = lapic->ppr & APIC_TPR_INT; 37699e33a616STycho Nightingale if (rvi > ppr) { 37709e33a616STycho Nightingale return (1); 37719e33a616STycho Nightingale } 37729e33a616STycho Nightingale 37739e33a616STycho Nightingale return (0); 37749e33a616STycho Nightingale } 377588c4b8d1SNeel Natu 377688c4b8d1SNeel Natu /* 377788c4b8d1SNeel Natu * If there is an interrupt pending then it will be recognized only 377888c4b8d1SNeel Natu * if its priority is greater than the processor priority. 377988c4b8d1SNeel Natu * 378088c4b8d1SNeel Natu * Special case: if the processor priority is zero then any pending 378188c4b8d1SNeel Natu * interrupt will be recognized. 378288c4b8d1SNeel Natu */ 378388c4b8d1SNeel Natu lapic = vlapic->apic_page; 37849e33a616STycho Nightingale ppr = lapic->ppr & APIC_TPR_INT; 378588c4b8d1SNeel Natu if (ppr == 0) 378688c4b8d1SNeel Natu return (1); 378788c4b8d1SNeel Natu 378888c4b8d1SNeel Natu VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "HLT with non-zero PPR %d", 378988c4b8d1SNeel Natu lapic->ppr); 379088c4b8d1SNeel Natu 37912c352febSJohn Baldwin vpr = 0; 379288c4b8d1SNeel Natu for (i = 3; i >= 0; i--) { 379388c4b8d1SNeel Natu pirval = pir_desc->pir[i]; 379488c4b8d1SNeel Natu if (pirval != 0) { 37959e33a616STycho Nightingale vpr = (i * 64 + flsl(pirval) - 1) & APIC_TPR_INT; 37962c352febSJohn Baldwin break; 379788c4b8d1SNeel Natu } 379888c4b8d1SNeel Natu } 37992c352febSJohn Baldwin 38002c352febSJohn Baldwin /* 38012c352febSJohn Baldwin * If the highest-priority pending interrupt falls short of the 38022c352febSJohn Baldwin * processor priority of this vCPU, ensure that 'pending_prio' does not 38032c352febSJohn Baldwin * have any stale bits which would preclude a higher-priority interrupt 38042c352febSJohn Baldwin * from incurring a notification later. 38052c352febSJohn Baldwin */ 38062c352febSJohn Baldwin if (vpr <= ppr) { 38072c352febSJohn Baldwin const u_int prio_bit = VPR_PRIO_BIT(vpr); 38082c352febSJohn Baldwin const u_int old = vlapic_vtx->pending_prio; 38092c352febSJohn Baldwin 38102c352febSJohn Baldwin if (old > prio_bit && (old & prio_bit) == 0) { 38112c352febSJohn Baldwin vlapic_vtx->pending_prio = prio_bit; 38122c352febSJohn Baldwin } 381388c4b8d1SNeel Natu return (0); 381488c4b8d1SNeel Natu } 38152c352febSJohn Baldwin return (1); 38162c352febSJohn Baldwin } 381788c4b8d1SNeel Natu 381888c4b8d1SNeel Natu static void 381988c4b8d1SNeel Natu vmx_intr_accepted(struct vlapic *vlapic, int vector) 382088c4b8d1SNeel Natu { 382188c4b8d1SNeel Natu 382288c4b8d1SNeel Natu panic("vmx_intr_accepted: not expected to be called"); 382388c4b8d1SNeel Natu } 382488c4b8d1SNeel Natu 3825176666c2SNeel Natu static void 382630b94db8SNeel Natu vmx_set_tmr(struct vlapic *vlapic, int vector, bool level) 382730b94db8SNeel Natu { 382830b94db8SNeel Natu struct vlapic_vtx *vlapic_vtx; 382930b94db8SNeel Natu struct vmx *vmx; 383030b94db8SNeel Natu struct vmcs *vmcs; 383130b94db8SNeel Natu uint64_t mask, val; 383230b94db8SNeel Natu 383330b94db8SNeel Natu KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector)); 383430b94db8SNeel Natu KASSERT(!vcpu_is_running(vlapic->vm, vlapic->vcpuid, NULL), 383530b94db8SNeel Natu ("vmx_set_tmr: vcpu cannot be running")); 383630b94db8SNeel Natu 383730b94db8SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 383830b94db8SNeel Natu vmx = vlapic_vtx->vmx; 383930b94db8SNeel Natu vmcs = &vmx->vmcs[vlapic->vcpuid]; 384030b94db8SNeel Natu mask = 1UL << (vector % 64); 384130b94db8SNeel Natu 384230b94db8SNeel Natu VMPTRLD(vmcs); 384330b94db8SNeel Natu val = vmcs_read(VMCS_EOI_EXIT(vector)); 384430b94db8SNeel Natu if (level) 384530b94db8SNeel Natu val |= mask; 384630b94db8SNeel Natu else 384730b94db8SNeel Natu val &= ~mask; 384830b94db8SNeel Natu vmcs_write(VMCS_EOI_EXIT(vector), val); 384930b94db8SNeel Natu VMCLEAR(vmcs); 385030b94db8SNeel Natu } 385130b94db8SNeel Natu 385230b94db8SNeel Natu static void 38531bc51badSMichael Reifenberger vmx_enable_x2apic_mode_ts(struct vlapic *vlapic) 38541bc51badSMichael Reifenberger { 38551bc51badSMichael Reifenberger struct vmx *vmx; 38561bc51badSMichael Reifenberger struct vmcs *vmcs; 38571bc51badSMichael Reifenberger uint32_t proc_ctls; 38581bc51badSMichael Reifenberger int vcpuid; 38591bc51badSMichael Reifenberger 38601bc51badSMichael Reifenberger vcpuid = vlapic->vcpuid; 38611bc51badSMichael Reifenberger vmx = ((struct vlapic_vtx *)vlapic)->vmx; 38621bc51badSMichael Reifenberger vmcs = &vmx->vmcs[vcpuid]; 38631bc51badSMichael Reifenberger 38641bc51badSMichael Reifenberger proc_ctls = vmx->cap[vcpuid].proc_ctls; 38651bc51badSMichael Reifenberger proc_ctls &= ~PROCBASED_USE_TPR_SHADOW; 38661bc51badSMichael Reifenberger proc_ctls |= PROCBASED_CR8_LOAD_EXITING; 38671bc51badSMichael Reifenberger proc_ctls |= PROCBASED_CR8_STORE_EXITING; 38681bc51badSMichael Reifenberger vmx->cap[vcpuid].proc_ctls = proc_ctls; 38691bc51badSMichael Reifenberger 38701bc51badSMichael Reifenberger VMPTRLD(vmcs); 38711bc51badSMichael Reifenberger vmcs_write(VMCS_PRI_PROC_BASED_CTLS, proc_ctls); 38721bc51badSMichael Reifenberger VMCLEAR(vmcs); 38731bc51badSMichael Reifenberger } 38741bc51badSMichael Reifenberger 38751bc51badSMichael Reifenberger static void 38761bc51badSMichael Reifenberger vmx_enable_x2apic_mode_vid(struct vlapic *vlapic) 3877159dd56fSNeel Natu { 3878159dd56fSNeel Natu struct vmx *vmx; 3879159dd56fSNeel Natu struct vmcs *vmcs; 3880159dd56fSNeel Natu uint32_t proc_ctls2; 38815c272efaSRobert Wing int vcpuid, error __diagused; 3882159dd56fSNeel Natu 3883159dd56fSNeel Natu vcpuid = vlapic->vcpuid; 3884159dd56fSNeel Natu vmx = ((struct vlapic_vtx *)vlapic)->vmx; 3885159dd56fSNeel Natu vmcs = &vmx->vmcs[vcpuid]; 3886159dd56fSNeel Natu 3887159dd56fSNeel Natu proc_ctls2 = vmx->cap[vcpuid].proc_ctls2; 3888159dd56fSNeel Natu KASSERT((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) != 0, 3889159dd56fSNeel Natu ("%s: invalid proc_ctls2 %#x", __func__, proc_ctls2)); 3890159dd56fSNeel Natu 3891159dd56fSNeel Natu proc_ctls2 &= ~PROCBASED2_VIRTUALIZE_APIC_ACCESSES; 3892159dd56fSNeel Natu proc_ctls2 |= PROCBASED2_VIRTUALIZE_X2APIC_MODE; 3893159dd56fSNeel Natu vmx->cap[vcpuid].proc_ctls2 = proc_ctls2; 3894159dd56fSNeel Natu 3895159dd56fSNeel Natu VMPTRLD(vmcs); 3896159dd56fSNeel Natu vmcs_write(VMCS_SEC_PROC_BASED_CTLS, proc_ctls2); 3897159dd56fSNeel Natu VMCLEAR(vmcs); 3898159dd56fSNeel Natu 3899159dd56fSNeel Natu if (vlapic->vcpuid == 0) { 3900159dd56fSNeel Natu /* 3901159dd56fSNeel Natu * The nested page table mappings are shared by all vcpus 3902159dd56fSNeel Natu * so unmap the APIC access page just once. 3903159dd56fSNeel Natu */ 3904159dd56fSNeel Natu error = vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE); 3905159dd56fSNeel Natu KASSERT(error == 0, ("%s: vm_unmap_mmio error %d", 3906159dd56fSNeel Natu __func__, error)); 3907159dd56fSNeel Natu 3908159dd56fSNeel Natu /* 3909159dd56fSNeel Natu * The MSR bitmap is shared by all vcpus so modify it only 3910159dd56fSNeel Natu * once in the context of vcpu 0. 3911159dd56fSNeel Natu */ 3912159dd56fSNeel Natu error = vmx_allow_x2apic_msrs(vmx); 3913159dd56fSNeel Natu KASSERT(error == 0, ("%s: vmx_allow_x2apic_msrs error %d", 3914159dd56fSNeel Natu __func__, error)); 3915159dd56fSNeel Natu } 3916159dd56fSNeel Natu } 3917159dd56fSNeel Natu 3918159dd56fSNeel Natu static void 3919176666c2SNeel Natu vmx_post_intr(struct vlapic *vlapic, int hostcpu) 3920176666c2SNeel Natu { 3921176666c2SNeel Natu 3922176666c2SNeel Natu ipi_cpu(hostcpu, pirvec); 3923176666c2SNeel Natu } 3924176666c2SNeel Natu 392588c4b8d1SNeel Natu /* 392688c4b8d1SNeel Natu * Transfer the pending interrupts in the PIR descriptor to the IRR 392788c4b8d1SNeel Natu * in the virtual APIC page. 392888c4b8d1SNeel Natu */ 392988c4b8d1SNeel Natu static void 393088c4b8d1SNeel Natu vmx_inject_pir(struct vlapic *vlapic) 393188c4b8d1SNeel Natu { 393288c4b8d1SNeel Natu struct vlapic_vtx *vlapic_vtx; 393388c4b8d1SNeel Natu struct pir_desc *pir_desc; 393488c4b8d1SNeel Natu struct LAPIC *lapic; 393588c4b8d1SNeel Natu uint64_t val, pirval; 39360e30c5c0SWarner Losh int rvi, pirbase = -1; 393788c4b8d1SNeel Natu uint16_t intr_status_old, intr_status_new; 393888c4b8d1SNeel Natu 393988c4b8d1SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 3940176666c2SNeel Natu pir_desc = vlapic_vtx->pir_desc; 394188c4b8d1SNeel Natu if (atomic_cmpset_long(&pir_desc->pending, 1, 0) == 0) { 394288c4b8d1SNeel Natu VCPU_CTR0(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: " 394388c4b8d1SNeel Natu "no posted interrupt pending"); 394488c4b8d1SNeel Natu return; 394588c4b8d1SNeel Natu } 394688c4b8d1SNeel Natu 394788c4b8d1SNeel Natu pirval = 0; 3948201b1cccSPeter Grehan pirbase = -1; 394988c4b8d1SNeel Natu lapic = vlapic->apic_page; 395088c4b8d1SNeel Natu 395188c4b8d1SNeel Natu val = atomic_readandclear_long(&pir_desc->pir[0]); 395288c4b8d1SNeel Natu if (val != 0) { 395388c4b8d1SNeel Natu lapic->irr0 |= val; 395488c4b8d1SNeel Natu lapic->irr1 |= val >> 32; 395588c4b8d1SNeel Natu pirbase = 0; 395688c4b8d1SNeel Natu pirval = val; 395788c4b8d1SNeel Natu } 395888c4b8d1SNeel Natu 395988c4b8d1SNeel Natu val = atomic_readandclear_long(&pir_desc->pir[1]); 396088c4b8d1SNeel Natu if (val != 0) { 396188c4b8d1SNeel Natu lapic->irr2 |= val; 396288c4b8d1SNeel Natu lapic->irr3 |= val >> 32; 396388c4b8d1SNeel Natu pirbase = 64; 396488c4b8d1SNeel Natu pirval = val; 396588c4b8d1SNeel Natu } 396688c4b8d1SNeel Natu 396788c4b8d1SNeel Natu val = atomic_readandclear_long(&pir_desc->pir[2]); 396888c4b8d1SNeel Natu if (val != 0) { 396988c4b8d1SNeel Natu lapic->irr4 |= val; 397088c4b8d1SNeel Natu lapic->irr5 |= val >> 32; 397188c4b8d1SNeel Natu pirbase = 128; 397288c4b8d1SNeel Natu pirval = val; 397388c4b8d1SNeel Natu } 397488c4b8d1SNeel Natu 397588c4b8d1SNeel Natu val = atomic_readandclear_long(&pir_desc->pir[3]); 397688c4b8d1SNeel Natu if (val != 0) { 397788c4b8d1SNeel Natu lapic->irr6 |= val; 397888c4b8d1SNeel Natu lapic->irr7 |= val >> 32; 397988c4b8d1SNeel Natu pirbase = 192; 398088c4b8d1SNeel Natu pirval = val; 398188c4b8d1SNeel Natu } 3982201b1cccSPeter Grehan 398388c4b8d1SNeel Natu VLAPIC_CTR_IRR(vlapic, "vmx_inject_pir"); 398488c4b8d1SNeel Natu 398588c4b8d1SNeel Natu /* 398688c4b8d1SNeel Natu * Update RVI so the processor can evaluate pending virtual 398788c4b8d1SNeel Natu * interrupts on VM-entry. 3988201b1cccSPeter Grehan * 3989201b1cccSPeter Grehan * It is possible for pirval to be 0 here, even though the 3990201b1cccSPeter Grehan * pending bit has been set. The scenario is: 3991201b1cccSPeter Grehan * CPU-Y is sending a posted interrupt to CPU-X, which 3992201b1cccSPeter Grehan * is running a guest and processing posted interrupts in h/w. 3993201b1cccSPeter Grehan * CPU-X will eventually exit and the state seen in s/w is 3994201b1cccSPeter Grehan * the pending bit set, but no PIR bits set. 3995201b1cccSPeter Grehan * 3996201b1cccSPeter Grehan * CPU-X CPU-Y 3997201b1cccSPeter Grehan * (vm running) (host running) 3998201b1cccSPeter Grehan * rx posted interrupt 3999201b1cccSPeter Grehan * CLEAR pending bit 4000201b1cccSPeter Grehan * SET PIR bit 4001201b1cccSPeter Grehan * READ/CLEAR PIR bits 4002201b1cccSPeter Grehan * SET pending bit 4003201b1cccSPeter Grehan * (vm exit) 4004201b1cccSPeter Grehan * pending bit set, PIR 0 400588c4b8d1SNeel Natu */ 400688c4b8d1SNeel Natu if (pirval != 0) { 400788c4b8d1SNeel Natu rvi = pirbase + flsl(pirval) - 1; 400888c4b8d1SNeel Natu intr_status_old = vmcs_read(VMCS_GUEST_INTR_STATUS); 400988c4b8d1SNeel Natu intr_status_new = (intr_status_old & 0xFF00) | rvi; 401088c4b8d1SNeel Natu if (intr_status_new > intr_status_old) { 401188c4b8d1SNeel Natu vmcs_write(VMCS_GUEST_INTR_STATUS, intr_status_new); 401288c4b8d1SNeel Natu VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: " 401388c4b8d1SNeel Natu "guest_intr_status changed from 0x%04x to 0x%04x", 401488c4b8d1SNeel Natu intr_status_old, intr_status_new); 401588c4b8d1SNeel Natu } 401688c4b8d1SNeel Natu } 401788c4b8d1SNeel Natu } 401888c4b8d1SNeel Natu 4019de5ea6b6SNeel Natu static struct vlapic * 4020de5ea6b6SNeel Natu vmx_vlapic_init(void *arg, int vcpuid) 4021de5ea6b6SNeel Natu { 4022de5ea6b6SNeel Natu struct vmx *vmx; 4023de5ea6b6SNeel Natu struct vlapic *vlapic; 4024176666c2SNeel Natu struct vlapic_vtx *vlapic_vtx; 4025de5ea6b6SNeel Natu 4026de5ea6b6SNeel Natu vmx = arg; 4027de5ea6b6SNeel Natu 402888c4b8d1SNeel Natu vlapic = malloc(sizeof(struct vlapic_vtx), M_VLAPIC, M_WAITOK | M_ZERO); 4029de5ea6b6SNeel Natu vlapic->vm = vmx->vm; 4030de5ea6b6SNeel Natu vlapic->vcpuid = vcpuid; 4031de5ea6b6SNeel Natu vlapic->apic_page = (struct LAPIC *)&vmx->apic_page[vcpuid]; 4032de5ea6b6SNeel Natu 4033176666c2SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 4034176666c2SNeel Natu vlapic_vtx->pir_desc = &vmx->pir_desc[vcpuid]; 403530b94db8SNeel Natu vlapic_vtx->vmx = vmx; 4036176666c2SNeel Natu 40371bc51badSMichael Reifenberger if (tpr_shadowing) { 40381bc51badSMichael Reifenberger vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode_ts; 40391bc51badSMichael Reifenberger } 40401bc51badSMichael Reifenberger 404188c4b8d1SNeel Natu if (virtual_interrupt_delivery) { 404288c4b8d1SNeel Natu vlapic->ops.set_intr_ready = vmx_set_intr_ready; 404388c4b8d1SNeel Natu vlapic->ops.pending_intr = vmx_pending_intr; 404488c4b8d1SNeel Natu vlapic->ops.intr_accepted = vmx_intr_accepted; 404530b94db8SNeel Natu vlapic->ops.set_tmr = vmx_set_tmr; 40461bc51badSMichael Reifenberger vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode_vid; 404788c4b8d1SNeel Natu } 404888c4b8d1SNeel Natu 4049176666c2SNeel Natu if (posted_interrupts) 4050176666c2SNeel Natu vlapic->ops.post_intr = vmx_post_intr; 4051176666c2SNeel Natu 4052de5ea6b6SNeel Natu vlapic_init(vlapic); 4053de5ea6b6SNeel Natu 4054de5ea6b6SNeel Natu return (vlapic); 4055de5ea6b6SNeel Natu } 4056de5ea6b6SNeel Natu 4057de5ea6b6SNeel Natu static void 4058de5ea6b6SNeel Natu vmx_vlapic_cleanup(void *arg, struct vlapic *vlapic) 4059de5ea6b6SNeel Natu { 4060de5ea6b6SNeel Natu 4061de5ea6b6SNeel Natu vlapic_cleanup(vlapic); 4062de5ea6b6SNeel Natu free(vlapic, M_VLAPIC); 4063de5ea6b6SNeel Natu } 4064de5ea6b6SNeel Natu 4065483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT 4066483d953aSJohn Baldwin static int 406715add60dSPeter Grehan vmx_snapshot(void *arg, struct vm_snapshot_meta *meta) 4068483d953aSJohn Baldwin { 4069483d953aSJohn Baldwin struct vmx *vmx; 4070483d953aSJohn Baldwin struct vmxctx *vmxctx; 4071483d953aSJohn Baldwin int i; 4072483d953aSJohn Baldwin int ret; 4073483d953aSJohn Baldwin 4074483d953aSJohn Baldwin vmx = arg; 4075483d953aSJohn Baldwin 4076483d953aSJohn Baldwin KASSERT(vmx != NULL, ("%s: arg was NULL", __func__)); 4077483d953aSJohn Baldwin 4078483d953aSJohn Baldwin for (i = 0; i < VM_MAXCPU; i++) { 4079483d953aSJohn Baldwin SNAPSHOT_BUF_OR_LEAVE(vmx->guest_msrs[i], 4080483d953aSJohn Baldwin sizeof(vmx->guest_msrs[i]), meta, ret, done); 4081483d953aSJohn Baldwin 4082483d953aSJohn Baldwin vmxctx = &vmx->ctx[i]; 4083483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rdi, meta, ret, done); 4084483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rsi, meta, ret, done); 4085483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rdx, meta, ret, done); 4086483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rcx, meta, ret, done); 4087483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r8, meta, ret, done); 4088483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r9, meta, ret, done); 4089483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rax, meta, ret, done); 4090483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rbx, meta, ret, done); 4091483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rbp, meta, ret, done); 4092483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r10, meta, ret, done); 4093483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r11, meta, ret, done); 4094483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r12, meta, ret, done); 4095483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r13, meta, ret, done); 4096483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r14, meta, ret, done); 4097483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r15, meta, ret, done); 4098483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_cr2, meta, ret, done); 4099483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr0, meta, ret, done); 4100483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr1, meta, ret, done); 4101483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr2, meta, ret, done); 4102483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr3, meta, ret, done); 4103483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr6, meta, ret, done); 4104483d953aSJohn Baldwin } 4105483d953aSJohn Baldwin 4106483d953aSJohn Baldwin done: 4107483d953aSJohn Baldwin return (ret); 4108483d953aSJohn Baldwin } 4109483d953aSJohn Baldwin 4110483d953aSJohn Baldwin static int 411115add60dSPeter Grehan vmx_vmcx_snapshot(void *arg, struct vm_snapshot_meta *meta, int vcpu) 4112483d953aSJohn Baldwin { 4113483d953aSJohn Baldwin struct vmcs *vmcs; 4114483d953aSJohn Baldwin struct vmx *vmx; 4115483d953aSJohn Baldwin int err, run, hostcpu; 4116483d953aSJohn Baldwin 4117483d953aSJohn Baldwin vmx = (struct vmx *)arg; 4118483d953aSJohn Baldwin err = 0; 4119483d953aSJohn Baldwin 4120483d953aSJohn Baldwin KASSERT(arg != NULL, ("%s: arg was NULL", __func__)); 4121483d953aSJohn Baldwin vmcs = &vmx->vmcs[vcpu]; 4122483d953aSJohn Baldwin 4123483d953aSJohn Baldwin run = vcpu_is_running(vmx->vm, vcpu, &hostcpu); 4124483d953aSJohn Baldwin if (run && hostcpu != curcpu) { 4125483d953aSJohn Baldwin printf("%s: %s%d is running", __func__, vm_name(vmx->vm), vcpu); 4126483d953aSJohn Baldwin return (EINVAL); 4127483d953aSJohn Baldwin } 4128483d953aSJohn Baldwin 4129483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CR0, meta); 4130483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CR3, meta); 4131483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CR4, meta); 4132483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_DR7, meta); 4133483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_RSP, meta); 4134483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_RIP, meta); 4135483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_RFLAGS, meta); 4136483d953aSJohn Baldwin 4137483d953aSJohn Baldwin /* Guest segments */ 4138483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_ES, meta); 4139483d953aSJohn Baldwin err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_ES, meta); 4140483d953aSJohn Baldwin 4141483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CS, meta); 4142483d953aSJohn Baldwin err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_CS, meta); 4143483d953aSJohn Baldwin 4144483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_SS, meta); 4145483d953aSJohn Baldwin err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_SS, meta); 4146483d953aSJohn Baldwin 4147483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_DS, meta); 4148483d953aSJohn Baldwin err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_DS, meta); 4149483d953aSJohn Baldwin 4150483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_FS, meta); 4151483d953aSJohn Baldwin err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_FS, meta); 4152483d953aSJohn Baldwin 4153483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_GS, meta); 4154483d953aSJohn Baldwin err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_GS, meta); 4155483d953aSJohn Baldwin 4156483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_TR, meta); 4157483d953aSJohn Baldwin err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_TR, meta); 4158483d953aSJohn Baldwin 4159483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_LDTR, meta); 4160483d953aSJohn Baldwin err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_LDTR, meta); 4161483d953aSJohn Baldwin 4162483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_EFER, meta); 4163483d953aSJohn Baldwin 4164483d953aSJohn Baldwin err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_IDTR, meta); 4165483d953aSJohn Baldwin err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_GDTR, meta); 4166483d953aSJohn Baldwin 4167483d953aSJohn Baldwin /* Guest page tables */ 4168483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE0, meta); 4169483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE1, meta); 4170483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE2, meta); 4171483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE3, meta); 4172483d953aSJohn Baldwin 4173483d953aSJohn Baldwin /* Other guest state */ 4174483d953aSJohn Baldwin err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_IA32_SYSENTER_CS, meta); 4175483d953aSJohn Baldwin err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_IA32_SYSENTER_ESP, meta); 4176483d953aSJohn Baldwin err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_IA32_SYSENTER_EIP, meta); 4177483d953aSJohn Baldwin err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_INTERRUPTIBILITY, meta); 4178483d953aSJohn Baldwin err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_ACTIVITY, meta); 4179483d953aSJohn Baldwin err += vmcs_snapshot_any(vmcs, run, VMCS_ENTRY_CTLS, meta); 4180483d953aSJohn Baldwin err += vmcs_snapshot_any(vmcs, run, VMCS_EXIT_CTLS, meta); 4181483d953aSJohn Baldwin 4182483d953aSJohn Baldwin return (err); 4183483d953aSJohn Baldwin } 4184483d953aSJohn Baldwin 4185483d953aSJohn Baldwin static int 4186483d953aSJohn Baldwin vmx_restore_tsc(void *arg, int vcpu, uint64_t offset) 4187483d953aSJohn Baldwin { 4188483d953aSJohn Baldwin struct vmcs *vmcs; 4189483d953aSJohn Baldwin struct vmx *vmx = (struct vmx *)arg; 4190483d953aSJohn Baldwin int error, running, hostcpu; 4191483d953aSJohn Baldwin 4192483d953aSJohn Baldwin KASSERT(arg != NULL, ("%s: arg was NULL", __func__)); 4193483d953aSJohn Baldwin vmcs = &vmx->vmcs[vcpu]; 4194483d953aSJohn Baldwin 4195483d953aSJohn Baldwin running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); 4196483d953aSJohn Baldwin if (running && hostcpu != curcpu) { 4197483d953aSJohn Baldwin printf("%s: %s%d is running", __func__, vm_name(vmx->vm), vcpu); 4198483d953aSJohn Baldwin return (EINVAL); 4199483d953aSJohn Baldwin } 4200483d953aSJohn Baldwin 4201483d953aSJohn Baldwin if (!running) 4202483d953aSJohn Baldwin VMPTRLD(vmcs); 4203483d953aSJohn Baldwin 4204483d953aSJohn Baldwin error = vmx_set_tsc_offset(vmx, vcpu, offset); 4205483d953aSJohn Baldwin 4206483d953aSJohn Baldwin if (!running) 4207483d953aSJohn Baldwin VMCLEAR(vmcs); 4208483d953aSJohn Baldwin return (error); 4209483d953aSJohn Baldwin } 4210483d953aSJohn Baldwin #endif 4211483d953aSJohn Baldwin 421215add60dSPeter Grehan const struct vmm_ops vmm_ops_intel = { 421315add60dSPeter Grehan .modinit = vmx_modinit, 421415add60dSPeter Grehan .modcleanup = vmx_modcleanup, 421515add60dSPeter Grehan .modresume = vmx_modresume, 421613a7c4d4SMark Johnston .init = vmx_init, 421715add60dSPeter Grehan .run = vmx_run, 421813a7c4d4SMark Johnston .cleanup = vmx_cleanup, 421915add60dSPeter Grehan .getreg = vmx_getreg, 422015add60dSPeter Grehan .setreg = vmx_setreg, 422115add60dSPeter Grehan .getdesc = vmx_getdesc, 422215add60dSPeter Grehan .setdesc = vmx_setdesc, 422315add60dSPeter Grehan .getcap = vmx_getcap, 422415add60dSPeter Grehan .setcap = vmx_setcap, 422515add60dSPeter Grehan .vmspace_alloc = vmx_vmspace_alloc, 422615add60dSPeter Grehan .vmspace_free = vmx_vmspace_free, 422713a7c4d4SMark Johnston .vlapic_init = vmx_vlapic_init, 422813a7c4d4SMark Johnston .vlapic_cleanup = vmx_vlapic_cleanup, 4229483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT 423015add60dSPeter Grehan .snapshot = vmx_snapshot, 423115add60dSPeter Grehan .vmcx_snapshot = vmx_vmcx_snapshot, 423215add60dSPeter Grehan .restore_tsc = vmx_restore_tsc, 4233483d953aSJohn Baldwin #endif 4234366f6083SPeter Grehan }; 4235