1366f6083SPeter Grehan /*- 24d846d26SWarner Losh * SPDX-License-Identifier: BSD-2-Clause 3c49761ddSPedro F. Giffuni * 4366f6083SPeter Grehan * Copyright (c) 2011 NetApp, Inc. 5366f6083SPeter Grehan * All rights reserved. 62c352febSJohn Baldwin * Copyright (c) 2018 Joyent, Inc. 7366f6083SPeter Grehan * 8366f6083SPeter Grehan * Redistribution and use in source and binary forms, with or without 9366f6083SPeter Grehan * modification, are permitted provided that the following conditions 10366f6083SPeter Grehan * are met: 11366f6083SPeter Grehan * 1. Redistributions of source code must retain the above copyright 12366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer. 13366f6083SPeter Grehan * 2. Redistributions in binary form must reproduce the above copyright 14366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer in the 15366f6083SPeter Grehan * documentation and/or other materials provided with the distribution. 16366f6083SPeter Grehan * 17366f6083SPeter Grehan * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 18366f6083SPeter Grehan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19366f6083SPeter Grehan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20366f6083SPeter Grehan * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 21366f6083SPeter Grehan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22366f6083SPeter Grehan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23366f6083SPeter Grehan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24366f6083SPeter Grehan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25366f6083SPeter Grehan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26366f6083SPeter Grehan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27366f6083SPeter Grehan * SUCH DAMAGE. 28366f6083SPeter Grehan */ 29366f6083SPeter Grehan 30366f6083SPeter Grehan #include <sys/cdefs.h> 31483d953aSJohn Baldwin #include "opt_bhyve_snapshot.h" 32483d953aSJohn Baldwin 33366f6083SPeter Grehan #include <sys/param.h> 34366f6083SPeter Grehan #include <sys/systm.h> 35366f6083SPeter Grehan #include <sys/smp.h> 36366f6083SPeter Grehan #include <sys/kernel.h> 37366f6083SPeter Grehan #include <sys/malloc.h> 38366f6083SPeter Grehan #include <sys/pcpu.h> 39366f6083SPeter Grehan #include <sys/proc.h> 40b7924341SAndrew Turner #include <sys/reg.h> 416f5a9606SMark Johnston #include <sys/smr.h> 423565b59eSNeel Natu #include <sys/sysctl.h> 43366f6083SPeter Grehan 44366f6083SPeter Grehan #include <vm/vm.h> 4574ac712fSMark Johnston #include <vm/vm_extern.h> 46366f6083SPeter Grehan #include <vm/pmap.h> 47366f6083SPeter Grehan 48366f6083SPeter Grehan #include <machine/psl.h> 49366f6083SPeter Grehan #include <machine/cpufunc.h> 508b287612SJohn Baldwin #include <machine/md_var.h> 51366f6083SPeter Grehan #include <machine/segments.h> 52176666c2SNeel Natu #include <machine/smp.h> 53608f97c3SPeter Grehan #include <machine/specialreg.h> 54366f6083SPeter Grehan #include <machine/vmparam.h> 55366f6083SPeter Grehan 56366f6083SPeter Grehan #include <machine/vmm.h> 57dc506506SNeel Natu #include <machine/vmm_dev.h> 58e813a873SNeel Natu #include <machine/vmm_instruction_emul.h> 59483d953aSJohn Baldwin #include <machine/vmm_snapshot.h> 60483d953aSJohn Baldwin 61*3ccb0233SMark Johnston #include <dev/vmm/vmm_ktr.h> 62*3ccb0233SMark Johnston 63c3498942SNeel Natu #include "vmm_lapic.h" 64b01c2033SNeel Natu #include "vmm_host.h" 65762fd208STycho Nightingale #include "vmm_ioport.h" 66366f6083SPeter Grehan #include "vmm_stat.h" 670775fbb4STycho Nightingale #include "vatpic.h" 68de5ea6b6SNeel Natu #include "vlapic.h" 69de5ea6b6SNeel Natu #include "vlapic_priv.h" 70366f6083SPeter Grehan 71366f6083SPeter Grehan #include "ept.h" 72366f6083SPeter Grehan #include "vmx_cpufunc.h" 73366f6083SPeter Grehan #include "vmx.h" 74c3498942SNeel Natu #include "vmx_msr.h" 75366f6083SPeter Grehan #include "x86.h" 76366f6083SPeter Grehan #include "vmx_controls.h" 77366f6083SPeter Grehan 78366f6083SPeter Grehan #define PINBASED_CTLS_ONE_SETTING \ 79366f6083SPeter Grehan (PINBASED_EXTINT_EXITING | \ 80366f6083SPeter Grehan PINBASED_NMI_EXITING | \ 81366f6083SPeter Grehan PINBASED_VIRTUAL_NMI) 82366f6083SPeter Grehan #define PINBASED_CTLS_ZERO_SETTING 0 83366f6083SPeter Grehan 84366f6083SPeter Grehan #define PROCBASED_CTLS_WINDOW_SETTING \ 85366f6083SPeter Grehan (PROCBASED_INT_WINDOW_EXITING | \ 86366f6083SPeter Grehan PROCBASED_NMI_WINDOW_EXITING) 87366f6083SPeter Grehan 88366f6083SPeter Grehan #define PROCBASED_CTLS_ONE_SETTING \ 89366f6083SPeter Grehan (PROCBASED_SECONDARY_CONTROLS | \ 9065145c7fSNeel Natu PROCBASED_MWAIT_EXITING | \ 9165145c7fSNeel Natu PROCBASED_MONITOR_EXITING | \ 92366f6083SPeter Grehan PROCBASED_IO_EXITING | \ 93366f6083SPeter Grehan PROCBASED_MSR_BITMAPS | \ 94594db002STycho Nightingale PROCBASED_CTLS_WINDOW_SETTING | \ 95594db002STycho Nightingale PROCBASED_CR8_LOAD_EXITING | \ 96594db002STycho Nightingale PROCBASED_CR8_STORE_EXITING) 97366f6083SPeter Grehan #define PROCBASED_CTLS_ZERO_SETTING \ 98366f6083SPeter Grehan (PROCBASED_CR3_LOAD_EXITING | \ 99366f6083SPeter Grehan PROCBASED_CR3_STORE_EXITING | \ 100366f6083SPeter Grehan PROCBASED_IO_BITMAPS) 101366f6083SPeter Grehan 102366f6083SPeter Grehan #define PROCBASED_CTLS2_ONE_SETTING PROCBASED2_ENABLE_EPT 103366f6083SPeter Grehan #define PROCBASED_CTLS2_ZERO_SETTING 0 104366f6083SPeter Grehan 105d72978ecSNeel Natu #define VM_EXIT_CTLS_ONE_SETTING \ 10665eefbe4SJohn Baldwin (VM_EXIT_SAVE_DEBUG_CONTROLS | \ 10765eefbe4SJohn Baldwin VM_EXIT_HOST_LMA | \ 108366f6083SPeter Grehan VM_EXIT_SAVE_EFER | \ 109d72978ecSNeel Natu VM_EXIT_LOAD_EFER | \ 110a318f7ddSNeel Natu VM_EXIT_ACKNOWLEDGE_INTERRUPT) 111d72978ecSNeel Natu 11265eefbe4SJohn Baldwin #define VM_EXIT_CTLS_ZERO_SETTING 0 113366f6083SPeter Grehan 11465eefbe4SJohn Baldwin #define VM_ENTRY_CTLS_ONE_SETTING \ 11565eefbe4SJohn Baldwin (VM_ENTRY_LOAD_DEBUG_CONTROLS | \ 11665eefbe4SJohn Baldwin VM_ENTRY_LOAD_EFER) 117608f97c3SPeter Grehan 118366f6083SPeter Grehan #define VM_ENTRY_CTLS_ZERO_SETTING \ 11965eefbe4SJohn Baldwin (VM_ENTRY_INTO_SMM | \ 120366f6083SPeter Grehan VM_ENTRY_DEACTIVATE_DUAL_MONITOR) 121366f6083SPeter Grehan 122366f6083SPeter Grehan #define HANDLED 1 123366f6083SPeter Grehan #define UNHANDLED 0 124366f6083SPeter Grehan 125de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VMX, "vmx", "vmx"); 126de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic"); 127366f6083SPeter Grehan 12873abae44SJohn Baldwin bool vmx_have_msr_tsc_aux; 12973abae44SJohn Baldwin 1303565b59eSNeel Natu SYSCTL_DECL(_hw_vmm); 131b40598c5SPawel Biernacki SYSCTL_NODE(_hw_vmm, OID_AUTO, vmx, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 132b40598c5SPawel Biernacki NULL); 1333565b59eSNeel Natu 134b3996dd4SJohn Baldwin int vmxon_enabled[MAXCPU]; 13574ac712fSMark Johnston static uint8_t *vmxon_region; 136366f6083SPeter Grehan 137366f6083SPeter Grehan static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2; 138366f6083SPeter Grehan static uint32_t exit_ctls, entry_ctls; 139366f6083SPeter Grehan 140366f6083SPeter Grehan static uint64_t cr0_ones_mask, cr0_zeros_mask; 1413565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_ones_mask, CTLFLAG_RD, 1423565b59eSNeel Natu &cr0_ones_mask, 0, NULL); 1433565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_zeros_mask, CTLFLAG_RD, 1443565b59eSNeel Natu &cr0_zeros_mask, 0, NULL); 1453565b59eSNeel Natu 146366f6083SPeter Grehan static uint64_t cr4_ones_mask, cr4_zeros_mask; 1473565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_ones_mask, CTLFLAG_RD, 1483565b59eSNeel Natu &cr4_ones_mask, 0, NULL); 1493565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_zeros_mask, CTLFLAG_RD, 1503565b59eSNeel Natu &cr4_zeros_mask, 0, NULL); 151366f6083SPeter Grehan 1523565b59eSNeel Natu static int vmx_initialized; 1533565b59eSNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, initialized, CTLFLAG_RD, 1543565b59eSNeel Natu &vmx_initialized, 0, "Intel VMX initialized"); 1553565b59eSNeel Natu 156366f6083SPeter Grehan /* 157366f6083SPeter Grehan * Optional capabilities 158366f6083SPeter Grehan */ 159b40598c5SPawel Biernacki static SYSCTL_NODE(_hw_vmm_vmx, OID_AUTO, cap, 160b40598c5SPawel Biernacki CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 161b40598c5SPawel Biernacki NULL); 16206fc6db9SJohn Baldwin 163366f6083SPeter Grehan static int cap_halt_exit; 16406fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, halt_exit, CTLFLAG_RD, &cap_halt_exit, 0, 16506fc6db9SJohn Baldwin "HLT triggers a VM-exit"); 16606fc6db9SJohn Baldwin 167366f6083SPeter Grehan static int cap_pause_exit; 16806fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, pause_exit, CTLFLAG_RD, &cap_pause_exit, 16906fc6db9SJohn Baldwin 0, "PAUSE triggers a VM-exit"); 17006fc6db9SJohn Baldwin 1713ba952e1SCorvin Köhne static int cap_wbinvd_exit; 1723ba952e1SCorvin Köhne SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, wbinvd_exit, CTLFLAG_RD, &cap_wbinvd_exit, 1733ba952e1SCorvin Köhne 0, "WBINVD triggers a VM-exit"); 1743ba952e1SCorvin Köhne 175f5f5f1e7SPeter Grehan static int cap_rdpid; 176f5f5f1e7SPeter Grehan SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, rdpid, CTLFLAG_RD, &cap_rdpid, 0, 177f5f5f1e7SPeter Grehan "Guests are allowed to use RDPID"); 178f5f5f1e7SPeter Grehan 179f5f5f1e7SPeter Grehan static int cap_rdtscp; 180f5f5f1e7SPeter Grehan SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, rdtscp, CTLFLAG_RD, &cap_rdtscp, 0, 181f5f5f1e7SPeter Grehan "Guests are allowed to use RDTSCP"); 182f5f5f1e7SPeter Grehan 183366f6083SPeter Grehan static int cap_unrestricted_guest; 18406fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, unrestricted_guest, CTLFLAG_RD, 18506fc6db9SJohn Baldwin &cap_unrestricted_guest, 0, "Unrestricted guests"); 18606fc6db9SJohn Baldwin 187366f6083SPeter Grehan static int cap_monitor_trap; 18806fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, monitor_trap, CTLFLAG_RD, 18906fc6db9SJohn Baldwin &cap_monitor_trap, 0, "Monitor trap flag"); 19006fc6db9SJohn Baldwin 19149cc03daSNeel Natu static int cap_invpcid; 19206fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, invpcid, CTLFLAG_RD, &cap_invpcid, 19306fc6db9SJohn Baldwin 0, "Guests are allowed to use INVPCID"); 194366f6083SPeter Grehan 1951bc51badSMichael Reifenberger static int tpr_shadowing; 196f3ff0918SZhenlei Huang SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, tpr_shadowing, 197f3ff0918SZhenlei Huang CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 1981bc51badSMichael Reifenberger &tpr_shadowing, 0, "TPR shadowing support"); 1991bc51badSMichael Reifenberger 20088c4b8d1SNeel Natu static int virtual_interrupt_delivery; 201f3ff0918SZhenlei Huang SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, virtual_interrupt_delivery, 202f3ff0918SZhenlei Huang CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 20388c4b8d1SNeel Natu &virtual_interrupt_delivery, 0, "APICv virtual interrupt delivery support"); 20488c4b8d1SNeel Natu 205176666c2SNeel Natu static int posted_interrupts; 206f3ff0918SZhenlei Huang SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, posted_interrupts, 207f3ff0918SZhenlei Huang CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 208176666c2SNeel Natu &posted_interrupts, 0, "APICv posted interrupt support"); 209176666c2SNeel Natu 21018a2b08eSNeel Natu static int pirvec = -1; 211176666c2SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupt_vector, CTLFLAG_RD, 212176666c2SNeel Natu &pirvec, 0, "APICv posted interrupt vector"); 213176666c2SNeel Natu 21445e51299SNeel Natu static struct unrhdr *vpid_unr; 21545e51299SNeel Natu static u_int vpid_alloc_failed; 21645e51299SNeel Natu SYSCTL_UINT(_hw_vmm_vmx, OID_AUTO, vpid_alloc_failed, CTLFLAG_RD, 21745e51299SNeel Natu &vpid_alloc_failed, 0, NULL); 21845e51299SNeel Natu 219d3588766SMark Johnston int guest_l1d_flush; 220f3ff0918SZhenlei Huang SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, l1d_flush, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 221c30578feSKonstantin Belousov &guest_l1d_flush, 0, NULL); 222d3588766SMark Johnston int guest_l1d_flush_sw; 223f3ff0918SZhenlei Huang SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, l1d_flush_sw, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 224c1141fbaSKonstantin Belousov &guest_l1d_flush_sw, 0, NULL); 225c30578feSKonstantin Belousov 226c1141fbaSKonstantin Belousov static struct msr_entry msr_load_list[1] __aligned(16); 227c30578feSKonstantin Belousov 22888c4b8d1SNeel Natu /* 2296ac73777STycho Nightingale * The definitions of SDT probes for VMX. 2306ac73777STycho Nightingale */ 2316ac73777STycho Nightingale 2326ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, entry, 2336ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2346ac73777STycho Nightingale 2356ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, taskswitch, 2366ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *", "struct vm_task_switch *"); 2376ac73777STycho Nightingale 2386ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, craccess, 2396ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *", "uint64_t"); 2406ac73777STycho Nightingale 2416ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, rdmsr, 2426ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *", "uint32_t"); 2436ac73777STycho Nightingale 2446ac73777STycho Nightingale SDT_PROBE_DEFINE5(vmm, vmx, exit, wrmsr, 2456ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *", "uint32_t", "uint64_t"); 2466ac73777STycho Nightingale 2476ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, halt, 2486ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2496ac73777STycho Nightingale 2506ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, mtrap, 2516ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2526ac73777STycho Nightingale 2536ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, pause, 2546ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2556ac73777STycho Nightingale 2566ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, intrwindow, 2576ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2586ac73777STycho Nightingale 2596ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, interrupt, 2606ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *", "uint32_t"); 2616ac73777STycho Nightingale 2626ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, nmiwindow, 2636ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2646ac73777STycho Nightingale 2656ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, inout, 2666ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2676ac73777STycho Nightingale 2686ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, cpuid, 2696ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2706ac73777STycho Nightingale 2716ac73777STycho Nightingale SDT_PROBE_DEFINE5(vmm, vmx, exit, exception, 2726ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *", "uint32_t", "int"); 2736ac73777STycho Nightingale 2746ac73777STycho Nightingale SDT_PROBE_DEFINE5(vmm, vmx, exit, nestedfault, 2756ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *", "uint64_t", "uint64_t"); 2766ac73777STycho Nightingale 2776ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, mmiofault, 2786ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *", "uint64_t"); 2796ac73777STycho Nightingale 2806ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, eoi, 2816ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2826ac73777STycho Nightingale 2836ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, apicaccess, 2846ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2856ac73777STycho Nightingale 2866ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, apicwrite, 2876ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *", "struct vlapic *"); 2886ac73777STycho Nightingale 2896ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, xsetbv, 2906ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2916ac73777STycho Nightingale 2926ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, monitor, 2936ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2946ac73777STycho Nightingale 2956ac73777STycho Nightingale SDT_PROBE_DEFINE3(vmm, vmx, exit, mwait, 2966ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *"); 2976ac73777STycho Nightingale 29827d26457SAndrew Turner SDT_PROBE_DEFINE3(vmm, vmx, exit, vminsn, 29927d26457SAndrew Turner "struct vmx *", "int", "struct vm_exit *"); 30027d26457SAndrew Turner 3016ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, unknown, 3026ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *", "uint32_t"); 3036ac73777STycho Nightingale 3046ac73777STycho Nightingale SDT_PROBE_DEFINE4(vmm, vmx, exit, return, 3056ac73777STycho Nightingale "struct vmx *", "int", "struct vm_exit *", "int"); 3066ac73777STycho Nightingale 3076ac73777STycho Nightingale /* 30888c4b8d1SNeel Natu * Use the last page below 4GB as the APIC access address. This address is 30988c4b8d1SNeel Natu * occupied by the boot firmware so it is guaranteed that it will not conflict 31088c4b8d1SNeel Natu * with a page in system memory. 31188c4b8d1SNeel Natu */ 31288c4b8d1SNeel Natu #define APIC_ACCESS_ADDRESS 0xFFFFF000 31388c4b8d1SNeel Natu 314869c8d19SJohn Baldwin static int vmx_getdesc(void *vcpui, int reg, struct seg_desc *desc); 315869c8d19SJohn Baldwin static int vmx_getreg(void *vcpui, int reg, uint64_t *retval); 316c3498942SNeel Natu static int vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val); 31788c4b8d1SNeel Natu static void vmx_inject_pir(struct vlapic *vlapic); 318483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT 319869c8d19SJohn Baldwin static int vmx_restore_tsc(void *vcpui, uint64_t now); 320483d953aSJohn Baldwin #endif 32188c4b8d1SNeel Natu 322f5f5f1e7SPeter Grehan static inline bool 323f5f5f1e7SPeter Grehan host_has_rdpid(void) 324f5f5f1e7SPeter Grehan { 325f5f5f1e7SPeter Grehan return ((cpu_stdext_feature2 & CPUID_STDEXT2_RDPID) != 0); 326f5f5f1e7SPeter Grehan } 327f5f5f1e7SPeter Grehan 328f5f5f1e7SPeter Grehan static inline bool 329f5f5f1e7SPeter Grehan host_has_rdtscp(void) 330f5f5f1e7SPeter Grehan { 331f5f5f1e7SPeter Grehan return ((amd_feature & AMDID_RDTSCP) != 0); 332f5f5f1e7SPeter Grehan } 333f5f5f1e7SPeter Grehan 334366f6083SPeter Grehan #ifdef KTR 335366f6083SPeter Grehan static const char * 336366f6083SPeter Grehan exit_reason_to_str(int reason) 337366f6083SPeter Grehan { 338366f6083SPeter Grehan static char reasonbuf[32]; 339366f6083SPeter Grehan 340366f6083SPeter Grehan switch (reason) { 341366f6083SPeter Grehan case EXIT_REASON_EXCEPTION: 342366f6083SPeter Grehan return "exception"; 343366f6083SPeter Grehan case EXIT_REASON_EXT_INTR: 344366f6083SPeter Grehan return "extint"; 345366f6083SPeter Grehan case EXIT_REASON_TRIPLE_FAULT: 346366f6083SPeter Grehan return "triplefault"; 347366f6083SPeter Grehan case EXIT_REASON_INIT: 348366f6083SPeter Grehan return "init"; 349366f6083SPeter Grehan case EXIT_REASON_SIPI: 350366f6083SPeter Grehan return "sipi"; 351366f6083SPeter Grehan case EXIT_REASON_IO_SMI: 352366f6083SPeter Grehan return "iosmi"; 353366f6083SPeter Grehan case EXIT_REASON_SMI: 354366f6083SPeter Grehan return "smi"; 355366f6083SPeter Grehan case EXIT_REASON_INTR_WINDOW: 356366f6083SPeter Grehan return "intrwindow"; 357366f6083SPeter Grehan case EXIT_REASON_NMI_WINDOW: 358366f6083SPeter Grehan return "nmiwindow"; 359366f6083SPeter Grehan case EXIT_REASON_TASK_SWITCH: 360366f6083SPeter Grehan return "taskswitch"; 361366f6083SPeter Grehan case EXIT_REASON_CPUID: 362366f6083SPeter Grehan return "cpuid"; 363366f6083SPeter Grehan case EXIT_REASON_GETSEC: 364366f6083SPeter Grehan return "getsec"; 365366f6083SPeter Grehan case EXIT_REASON_HLT: 366366f6083SPeter Grehan return "hlt"; 367366f6083SPeter Grehan case EXIT_REASON_INVD: 368366f6083SPeter Grehan return "invd"; 369366f6083SPeter Grehan case EXIT_REASON_INVLPG: 370366f6083SPeter Grehan return "invlpg"; 371366f6083SPeter Grehan case EXIT_REASON_RDPMC: 372366f6083SPeter Grehan return "rdpmc"; 373366f6083SPeter Grehan case EXIT_REASON_RDTSC: 374366f6083SPeter Grehan return "rdtsc"; 375366f6083SPeter Grehan case EXIT_REASON_RSM: 376366f6083SPeter Grehan return "rsm"; 377366f6083SPeter Grehan case EXIT_REASON_VMCALL: 378366f6083SPeter Grehan return "vmcall"; 379366f6083SPeter Grehan case EXIT_REASON_VMCLEAR: 380366f6083SPeter Grehan return "vmclear"; 381366f6083SPeter Grehan case EXIT_REASON_VMLAUNCH: 382366f6083SPeter Grehan return "vmlaunch"; 383366f6083SPeter Grehan case EXIT_REASON_VMPTRLD: 384366f6083SPeter Grehan return "vmptrld"; 385366f6083SPeter Grehan case EXIT_REASON_VMPTRST: 386366f6083SPeter Grehan return "vmptrst"; 387366f6083SPeter Grehan case EXIT_REASON_VMREAD: 388366f6083SPeter Grehan return "vmread"; 389366f6083SPeter Grehan case EXIT_REASON_VMRESUME: 390366f6083SPeter Grehan return "vmresume"; 391366f6083SPeter Grehan case EXIT_REASON_VMWRITE: 392366f6083SPeter Grehan return "vmwrite"; 393366f6083SPeter Grehan case EXIT_REASON_VMXOFF: 394366f6083SPeter Grehan return "vmxoff"; 395366f6083SPeter Grehan case EXIT_REASON_VMXON: 396366f6083SPeter Grehan return "vmxon"; 397366f6083SPeter Grehan case EXIT_REASON_CR_ACCESS: 398366f6083SPeter Grehan return "craccess"; 399366f6083SPeter Grehan case EXIT_REASON_DR_ACCESS: 400366f6083SPeter Grehan return "draccess"; 401366f6083SPeter Grehan case EXIT_REASON_INOUT: 402366f6083SPeter Grehan return "inout"; 403366f6083SPeter Grehan case EXIT_REASON_RDMSR: 404366f6083SPeter Grehan return "rdmsr"; 405366f6083SPeter Grehan case EXIT_REASON_WRMSR: 406366f6083SPeter Grehan return "wrmsr"; 407366f6083SPeter Grehan case EXIT_REASON_INVAL_VMCS: 408366f6083SPeter Grehan return "invalvmcs"; 409366f6083SPeter Grehan case EXIT_REASON_INVAL_MSR: 410366f6083SPeter Grehan return "invalmsr"; 411366f6083SPeter Grehan case EXIT_REASON_MWAIT: 412366f6083SPeter Grehan return "mwait"; 413366f6083SPeter Grehan case EXIT_REASON_MTF: 414366f6083SPeter Grehan return "mtf"; 415366f6083SPeter Grehan case EXIT_REASON_MONITOR: 416366f6083SPeter Grehan return "monitor"; 417366f6083SPeter Grehan case EXIT_REASON_PAUSE: 418366f6083SPeter Grehan return "pause"; 419b0538143SNeel Natu case EXIT_REASON_MCE_DURING_ENTRY: 420b0538143SNeel Natu return "mce-during-entry"; 421366f6083SPeter Grehan case EXIT_REASON_TPR: 422366f6083SPeter Grehan return "tpr"; 42388c4b8d1SNeel Natu case EXIT_REASON_APIC_ACCESS: 42488c4b8d1SNeel Natu return "apic-access"; 425366f6083SPeter Grehan case EXIT_REASON_GDTR_IDTR: 426366f6083SPeter Grehan return "gdtridtr"; 427366f6083SPeter Grehan case EXIT_REASON_LDTR_TR: 428366f6083SPeter Grehan return "ldtrtr"; 429366f6083SPeter Grehan case EXIT_REASON_EPT_FAULT: 430366f6083SPeter Grehan return "eptfault"; 431366f6083SPeter Grehan case EXIT_REASON_EPT_MISCONFIG: 432366f6083SPeter Grehan return "eptmisconfig"; 433366f6083SPeter Grehan case EXIT_REASON_INVEPT: 434366f6083SPeter Grehan return "invept"; 435366f6083SPeter Grehan case EXIT_REASON_RDTSCP: 436366f6083SPeter Grehan return "rdtscp"; 437366f6083SPeter Grehan case EXIT_REASON_VMX_PREEMPT: 438366f6083SPeter Grehan return "vmxpreempt"; 439366f6083SPeter Grehan case EXIT_REASON_INVVPID: 440366f6083SPeter Grehan return "invvpid"; 441366f6083SPeter Grehan case EXIT_REASON_WBINVD: 442366f6083SPeter Grehan return "wbinvd"; 443366f6083SPeter Grehan case EXIT_REASON_XSETBV: 444366f6083SPeter Grehan return "xsetbv"; 44588c4b8d1SNeel Natu case EXIT_REASON_APIC_WRITE: 44688c4b8d1SNeel Natu return "apic-write"; 447366f6083SPeter Grehan default: 448366f6083SPeter Grehan snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason); 449366f6083SPeter Grehan return (reasonbuf); 450366f6083SPeter Grehan } 451366f6083SPeter Grehan } 452366f6083SPeter Grehan #endif /* KTR */ 453366f6083SPeter Grehan 454159dd56fSNeel Natu static int 455159dd56fSNeel Natu vmx_allow_x2apic_msrs(struct vmx *vmx) 456159dd56fSNeel Natu { 457159dd56fSNeel Natu int i, error; 458159dd56fSNeel Natu 459159dd56fSNeel Natu error = 0; 460159dd56fSNeel Natu 461159dd56fSNeel Natu /* 462159dd56fSNeel Natu * Allow readonly access to the following x2APIC MSRs from the guest. 463159dd56fSNeel Natu */ 464159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_ID); 465159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_VERSION); 466159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LDR); 467159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_SVR); 468159dd56fSNeel Natu 469159dd56fSNeel Natu for (i = 0; i < 8; i++) 470159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_ISR0 + i); 471159dd56fSNeel Natu 472159dd56fSNeel Natu for (i = 0; i < 8; i++) 473159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_TMR0 + i); 474159dd56fSNeel Natu 475159dd56fSNeel Natu for (i = 0; i < 8; i++) 476159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_IRR0 + i); 477159dd56fSNeel Natu 478159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_ESR); 479159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_TIMER); 480159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_THERMAL); 481159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_PCINT); 482159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT0); 483159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT1); 484159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_ERROR); 485159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_ICR_TIMER); 486159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_DCR_TIMER); 487159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_ICR); 488159dd56fSNeel Natu 489159dd56fSNeel Natu /* 490159dd56fSNeel Natu * Allow TPR, EOI and SELF_IPI MSRs to be read and written by the guest. 491159dd56fSNeel Natu * 492159dd56fSNeel Natu * These registers get special treatment described in the section 493159dd56fSNeel Natu * "Virtualizing MSR-Based APIC Accesses". 494159dd56fSNeel Natu */ 495159dd56fSNeel Natu error += guest_msr_rw(vmx, MSR_APIC_TPR); 496159dd56fSNeel Natu error += guest_msr_rw(vmx, MSR_APIC_EOI); 497159dd56fSNeel Natu error += guest_msr_rw(vmx, MSR_APIC_SELF_IPI); 498159dd56fSNeel Natu 499159dd56fSNeel Natu return (error); 500159dd56fSNeel Natu } 501159dd56fSNeel Natu 502366f6083SPeter Grehan u_long 503366f6083SPeter Grehan vmx_fix_cr0(u_long cr0) 504366f6083SPeter Grehan { 505366f6083SPeter Grehan 506366f6083SPeter Grehan return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask); 507366f6083SPeter Grehan } 508366f6083SPeter Grehan 509366f6083SPeter Grehan u_long 510366f6083SPeter Grehan vmx_fix_cr4(u_long cr4) 511366f6083SPeter Grehan { 512366f6083SPeter Grehan 513366f6083SPeter Grehan return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask); 514366f6083SPeter Grehan } 515366f6083SPeter Grehan 516366f6083SPeter Grehan static void 51745e51299SNeel Natu vpid_free(int vpid) 51845e51299SNeel Natu { 51945e51299SNeel Natu if (vpid < 0 || vpid > 0xffff) 52045e51299SNeel Natu panic("vpid_free: invalid vpid %d", vpid); 52145e51299SNeel Natu 52245e51299SNeel Natu /* 523ee98f99dSJohn Baldwin * VPIDs [0,vm_maxcpu] are special and are not allocated from 52445e51299SNeel Natu * the unit number allocator. 52545e51299SNeel Natu */ 52645e51299SNeel Natu 527ee98f99dSJohn Baldwin if (vpid > vm_maxcpu) 52845e51299SNeel Natu free_unr(vpid_unr, vpid); 52945e51299SNeel Natu } 53045e51299SNeel Natu 53158eefc67SJohn Baldwin static uint16_t 53258eefc67SJohn Baldwin vpid_alloc(int vcpuid) 53345e51299SNeel Natu { 53458eefc67SJohn Baldwin int x; 53545e51299SNeel Natu 53645e51299SNeel Natu /* 53745e51299SNeel Natu * If the "enable vpid" execution control is not enabled then the 53845e51299SNeel Natu * VPID is required to be 0 for all vcpus. 53945e51299SNeel Natu */ 54058eefc67SJohn Baldwin if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) == 0) 54158eefc67SJohn Baldwin return (0); 54245e51299SNeel Natu 54345e51299SNeel Natu /* 54458eefc67SJohn Baldwin * Try to allocate a unique VPID for each from the unit number 54558eefc67SJohn Baldwin * allocator. 54645e51299SNeel Natu */ 54745e51299SNeel Natu x = alloc_unr(vpid_unr); 54845e51299SNeel Natu 54958eefc67SJohn Baldwin if (x == -1) { 55045e51299SNeel Natu atomic_add_int(&vpid_alloc_failed, 1); 55145e51299SNeel Natu 55245e51299SNeel Natu /* 55345e51299SNeel Natu * If the unit number allocator does not have enough unique 554ee98f99dSJohn Baldwin * VPIDs then we need to allocate from the [1,vm_maxcpu] range. 55545e51299SNeel Natu * 55645e51299SNeel Natu * These VPIDs are not be unique across VMs but this does not 55745e51299SNeel Natu * affect correctness because the combined mappings are also 55845e51299SNeel Natu * tagged with the EP4TA which is unique for each VM. 55945e51299SNeel Natu * 56045e51299SNeel Natu * It is still sub-optimal because the invvpid will invalidate 56145e51299SNeel Natu * combined mappings for a particular VPID across all EP4TAs. 56245e51299SNeel Natu */ 56358eefc67SJohn Baldwin return (vcpuid + 1); 56445e51299SNeel Natu } 56558eefc67SJohn Baldwin 56658eefc67SJohn Baldwin return (x); 56745e51299SNeel Natu } 56845e51299SNeel Natu 56945e51299SNeel Natu static void 57045e51299SNeel Natu vpid_init(void) 57145e51299SNeel Natu { 57245e51299SNeel Natu /* 57345e51299SNeel Natu * VPID 0 is required when the "enable VPID" execution control is 57445e51299SNeel Natu * disabled. 57545e51299SNeel Natu * 576ee98f99dSJohn Baldwin * VPIDs [1,vm_maxcpu] are used as the "overflow namespace" when the 57745e51299SNeel Natu * unit number allocator does not have sufficient unique VPIDs to 57845e51299SNeel Natu * satisfy the allocation. 57945e51299SNeel Natu * 58045e51299SNeel Natu * The remaining VPIDs are managed by the unit number allocator. 58145e51299SNeel Natu */ 582ee98f99dSJohn Baldwin vpid_unr = new_unrhdr(vm_maxcpu + 1, 0xffff, NULL); 58345e51299SNeel Natu } 58445e51299SNeel Natu 58545e51299SNeel Natu static void 586366f6083SPeter Grehan vmx_disable(void *arg __unused) 587366f6083SPeter Grehan { 588366f6083SPeter Grehan struct invvpid_desc invvpid_desc = { 0 }; 589366f6083SPeter Grehan struct invept_desc invept_desc = { 0 }; 590366f6083SPeter Grehan 591366f6083SPeter Grehan if (vmxon_enabled[curcpu]) { 592366f6083SPeter Grehan /* 593366f6083SPeter Grehan * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b. 594366f6083SPeter Grehan * 595366f6083SPeter Grehan * VMXON or VMXOFF are not required to invalidate any TLB 596366f6083SPeter Grehan * caching structures. This prevents potential retention of 597366f6083SPeter Grehan * cached information in the TLB between distinct VMX episodes. 598366f6083SPeter Grehan */ 599366f6083SPeter Grehan invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc); 600366f6083SPeter Grehan invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc); 601366f6083SPeter Grehan vmxoff(); 602366f6083SPeter Grehan } 603366f6083SPeter Grehan load_cr4(rcr4() & ~CR4_VMXE); 604366f6083SPeter Grehan } 605366f6083SPeter Grehan 606366f6083SPeter Grehan static int 60715add60dSPeter Grehan vmx_modcleanup(void) 608366f6083SPeter Grehan { 609366f6083SPeter Grehan 61018a2b08eSNeel Natu if (pirvec >= 0) 61118a2b08eSNeel Natu lapic_ipi_free(pirvec); 612176666c2SNeel Natu 61345e51299SNeel Natu if (vpid_unr != NULL) { 61445e51299SNeel Natu delete_unrhdr(vpid_unr); 61545e51299SNeel Natu vpid_unr = NULL; 61645e51299SNeel Natu } 61745e51299SNeel Natu 618c1141fbaSKonstantin Belousov if (nmi_flush_l1d_sw == 1) 619c1141fbaSKonstantin Belousov nmi_flush_l1d_sw = 0; 620c1141fbaSKonstantin Belousov 621366f6083SPeter Grehan smp_rendezvous(NULL, vmx_disable, NULL, NULL); 622b10e100dSCorvin Köhne 623b10e100dSCorvin Köhne if (vmxon_region != NULL) 62474ac712fSMark Johnston kmem_free(vmxon_region, (mp_maxid + 1) * PAGE_SIZE); 625366f6083SPeter Grehan 626366f6083SPeter Grehan return (0); 627366f6083SPeter Grehan } 628366f6083SPeter Grehan 629366f6083SPeter Grehan static void 630366f6083SPeter Grehan vmx_enable(void *arg __unused) 631366f6083SPeter Grehan { 632366f6083SPeter Grehan int error; 63311669a68STycho Nightingale uint64_t feature_control; 63411669a68STycho Nightingale 63511669a68STycho Nightingale feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL); 63611669a68STycho Nightingale if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 0 || 63711669a68STycho Nightingale (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) { 63811669a68STycho Nightingale wrmsr(MSR_IA32_FEATURE_CONTROL, 63911669a68STycho Nightingale feature_control | IA32_FEATURE_CONTROL_VMX_EN | 64011669a68STycho Nightingale IA32_FEATURE_CONTROL_LOCK); 64111669a68STycho Nightingale } 642366f6083SPeter Grehan 643366f6083SPeter Grehan load_cr4(rcr4() | CR4_VMXE); 644366f6083SPeter Grehan 64574ac712fSMark Johnston *(uint32_t *)&vmxon_region[curcpu * PAGE_SIZE] = vmx_revision(); 64674ac712fSMark Johnston error = vmxon(&vmxon_region[curcpu * PAGE_SIZE]); 647366f6083SPeter Grehan if (error == 0) 648366f6083SPeter Grehan vmxon_enabled[curcpu] = 1; 649366f6083SPeter Grehan } 650366f6083SPeter Grehan 65163e62d39SJohn Baldwin static void 65215add60dSPeter Grehan vmx_modresume(void) 65363e62d39SJohn Baldwin { 65463e62d39SJohn Baldwin 65563e62d39SJohn Baldwin if (vmxon_enabled[curcpu]) 65674ac712fSMark Johnston vmxon(&vmxon_region[curcpu * PAGE_SIZE]); 65763e62d39SJohn Baldwin } 65863e62d39SJohn Baldwin 659366f6083SPeter Grehan static int 66015add60dSPeter Grehan vmx_modinit(int ipinum) 661366f6083SPeter Grehan { 6621bc51badSMichael Reifenberger int error; 663d17b5104SNeel Natu uint64_t basic, fixed0, fixed1, feature_control; 66488c4b8d1SNeel Natu uint32_t tmp, procbased2_vid_bits; 665366f6083SPeter Grehan 666366f6083SPeter Grehan /* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */ 6678b287612SJohn Baldwin if (!(cpu_feature2 & CPUID2_VMX)) { 66815add60dSPeter Grehan printf("vmx_modinit: processor does not support VMX " 66915add60dSPeter Grehan "operation\n"); 670366f6083SPeter Grehan return (ENXIO); 671366f6083SPeter Grehan } 672366f6083SPeter Grehan 6734bff7fadSNeel Natu /* 6744bff7fadSNeel Natu * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits 6754bff7fadSNeel Natu * are set (bits 0 and 2 respectively). 6764bff7fadSNeel Natu */ 6774bff7fadSNeel Natu feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL); 67811669a68STycho Nightingale if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 1 && 679150369abSNeel Natu (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) { 68015add60dSPeter Grehan printf("vmx_modinit: VMX operation disabled by BIOS\n"); 6814bff7fadSNeel Natu return (ENXIO); 6824bff7fadSNeel Natu } 6834bff7fadSNeel Natu 684d17b5104SNeel Natu /* 685d17b5104SNeel Natu * Verify capabilities MSR_VMX_BASIC: 686d17b5104SNeel Natu * - bit 54 indicates support for INS/OUTS decoding 687d17b5104SNeel Natu */ 688d17b5104SNeel Natu basic = rdmsr(MSR_VMX_BASIC); 689d17b5104SNeel Natu if ((basic & (1UL << 54)) == 0) { 69015add60dSPeter Grehan printf("vmx_modinit: processor does not support desired basic " 691d17b5104SNeel Natu "capabilities\n"); 692d17b5104SNeel Natu return (EINVAL); 693d17b5104SNeel Natu } 694d17b5104SNeel Natu 695366f6083SPeter Grehan /* Check support for primary processor-based VM-execution controls */ 696366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 697366f6083SPeter Grehan MSR_VMX_TRUE_PROCBASED_CTLS, 698366f6083SPeter Grehan PROCBASED_CTLS_ONE_SETTING, 699366f6083SPeter Grehan PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls); 700366f6083SPeter Grehan if (error) { 70115add60dSPeter Grehan printf("vmx_modinit: processor does not support desired " 70215add60dSPeter Grehan "primary processor-based controls\n"); 703366f6083SPeter Grehan return (error); 704366f6083SPeter Grehan } 705366f6083SPeter Grehan 706366f6083SPeter Grehan /* Clear the processor-based ctl bits that are set on demand */ 707366f6083SPeter Grehan procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING; 708366f6083SPeter Grehan 709366f6083SPeter Grehan /* Check support for secondary processor-based VM-execution controls */ 710366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 711366f6083SPeter Grehan MSR_VMX_PROCBASED_CTLS2, 712366f6083SPeter Grehan PROCBASED_CTLS2_ONE_SETTING, 713366f6083SPeter Grehan PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2); 714366f6083SPeter Grehan if (error) { 71515add60dSPeter Grehan printf("vmx_modinit: processor does not support desired " 71615add60dSPeter Grehan "secondary processor-based controls\n"); 717366f6083SPeter Grehan return (error); 718366f6083SPeter Grehan } 719366f6083SPeter Grehan 720366f6083SPeter Grehan /* Check support for VPID */ 721366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2, 722366f6083SPeter Grehan PROCBASED2_ENABLE_VPID, 0, &tmp); 723366f6083SPeter Grehan if (error == 0) 724366f6083SPeter Grehan procbased_ctls2 |= PROCBASED2_ENABLE_VPID; 725366f6083SPeter Grehan 726366f6083SPeter Grehan /* Check support for pin-based VM-execution controls */ 727366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS, 728366f6083SPeter Grehan MSR_VMX_TRUE_PINBASED_CTLS, 729366f6083SPeter Grehan PINBASED_CTLS_ONE_SETTING, 730366f6083SPeter Grehan PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls); 731366f6083SPeter Grehan if (error) { 73215add60dSPeter Grehan printf("vmx_modinit: processor does not support desired " 733366f6083SPeter Grehan "pin-based controls\n"); 734366f6083SPeter Grehan return (error); 735366f6083SPeter Grehan } 736366f6083SPeter Grehan 737366f6083SPeter Grehan /* Check support for VM-exit controls */ 738366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS, 739366f6083SPeter Grehan VM_EXIT_CTLS_ONE_SETTING, 740366f6083SPeter Grehan VM_EXIT_CTLS_ZERO_SETTING, 741366f6083SPeter Grehan &exit_ctls); 742366f6083SPeter Grehan if (error) { 74315add60dSPeter Grehan printf("vmx_modinit: processor does not support desired " 744366f6083SPeter Grehan "exit controls\n"); 745366f6083SPeter Grehan return (error); 746366f6083SPeter Grehan } 747366f6083SPeter Grehan 748366f6083SPeter Grehan /* Check support for VM-entry controls */ 749d72978ecSNeel Natu error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS, MSR_VMX_TRUE_ENTRY_CTLS, 750d72978ecSNeel Natu VM_ENTRY_CTLS_ONE_SETTING, VM_ENTRY_CTLS_ZERO_SETTING, 751366f6083SPeter Grehan &entry_ctls); 752366f6083SPeter Grehan if (error) { 75315add60dSPeter Grehan printf("vmx_modinit: processor does not support desired " 754366f6083SPeter Grehan "entry controls\n"); 755366f6083SPeter Grehan return (error); 756366f6083SPeter Grehan } 757366f6083SPeter Grehan 758366f6083SPeter Grehan /* 759366f6083SPeter Grehan * Check support for optional features by testing them 760366f6083SPeter Grehan * as individual bits 761366f6083SPeter Grehan */ 762366f6083SPeter Grehan cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 763366f6083SPeter Grehan MSR_VMX_TRUE_PROCBASED_CTLS, 764366f6083SPeter Grehan PROCBASED_HLT_EXITING, 0, 765366f6083SPeter Grehan &tmp) == 0); 766366f6083SPeter Grehan 767366f6083SPeter Grehan cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 768366f6083SPeter Grehan MSR_VMX_PROCBASED_CTLS, 769366f6083SPeter Grehan PROCBASED_MTF, 0, 770366f6083SPeter Grehan &tmp) == 0); 771366f6083SPeter Grehan 772366f6083SPeter Grehan cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 773366f6083SPeter Grehan MSR_VMX_TRUE_PROCBASED_CTLS, 774366f6083SPeter Grehan PROCBASED_PAUSE_EXITING, 0, 775366f6083SPeter Grehan &tmp) == 0); 776366f6083SPeter Grehan 7773ba952e1SCorvin Köhne cap_wbinvd_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 7783ba952e1SCorvin Köhne MSR_VMX_PROCBASED_CTLS2, 7793ba952e1SCorvin Köhne PROCBASED2_WBINVD_EXITING, 7803ba952e1SCorvin Köhne 0, 7813ba952e1SCorvin Köhne &tmp) == 0); 7823ba952e1SCorvin Köhne 783f5f5f1e7SPeter Grehan /* 784f5f5f1e7SPeter Grehan * Check support for RDPID and/or RDTSCP. 785f5f5f1e7SPeter Grehan * 786f5f5f1e7SPeter Grehan * Support a pass-through-based implementation of these via the 787f5f5f1e7SPeter Grehan * "enable RDTSCP" VM-execution control and the "RDTSC exiting" 788f5f5f1e7SPeter Grehan * VM-execution control. 789f5f5f1e7SPeter Grehan * 790f5f5f1e7SPeter Grehan * The "enable RDTSCP" VM-execution control applies to both RDPID 791f5f5f1e7SPeter Grehan * and RDTSCP (see SDM volume 3, section 25.3, "Changes to 792f5f5f1e7SPeter Grehan * Instruction Behavior in VMX Non-root operation"); this is why 793f5f5f1e7SPeter Grehan * only this VM-execution control needs to be enabled in order to 794f5f5f1e7SPeter Grehan * enable passing through whichever of RDPID and/or RDTSCP are 795f5f5f1e7SPeter Grehan * supported by the host. 796f5f5f1e7SPeter Grehan * 797f5f5f1e7SPeter Grehan * The "RDTSC exiting" VM-execution control applies to both RDTSC 798f5f5f1e7SPeter Grehan * and RDTSCP (again, per SDM volume 3, section 25.3), and is 799f5f5f1e7SPeter Grehan * already set up for RDTSC and RDTSCP pass-through by the current 800f5f5f1e7SPeter Grehan * implementation of RDTSC. 801f5f5f1e7SPeter Grehan * 802f5f5f1e7SPeter Grehan * Although RDPID and RDTSCP are optional capabilities, since there 803f5f5f1e7SPeter Grehan * does not currently seem to be a use case for enabling/disabling 804f5f5f1e7SPeter Grehan * these via libvmmapi, choose not to support this and, instead, 805f5f5f1e7SPeter Grehan * just statically always enable or always disable this support 806f5f5f1e7SPeter Grehan * across all vCPUs on all VMs. (Note that there may be some 807f5f5f1e7SPeter Grehan * complications to providing this functionality, e.g., the MSR 808f5f5f1e7SPeter Grehan * bitmap is currently per-VM rather than per-vCPU while the 809f5f5f1e7SPeter Grehan * capability API wants to be able to control capabilities on a 810f5f5f1e7SPeter Grehan * per-vCPU basis). 811f5f5f1e7SPeter Grehan */ 812f5f5f1e7SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 813f5f5f1e7SPeter Grehan MSR_VMX_PROCBASED_CTLS2, 814f5f5f1e7SPeter Grehan PROCBASED2_ENABLE_RDTSCP, 0, &tmp); 815f5f5f1e7SPeter Grehan cap_rdpid = error == 0 && host_has_rdpid(); 816f5f5f1e7SPeter Grehan cap_rdtscp = error == 0 && host_has_rdtscp(); 81773abae44SJohn Baldwin if (cap_rdpid || cap_rdtscp) { 818f5f5f1e7SPeter Grehan procbased_ctls2 |= PROCBASED2_ENABLE_RDTSCP; 81973abae44SJohn Baldwin vmx_have_msr_tsc_aux = true; 82073abae44SJohn Baldwin } 821f5f5f1e7SPeter Grehan 822366f6083SPeter Grehan cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 823366f6083SPeter Grehan MSR_VMX_PROCBASED_CTLS2, 824366f6083SPeter Grehan PROCBASED2_UNRESTRICTED_GUEST, 0, 825366f6083SPeter Grehan &tmp) == 0); 826366f6083SPeter Grehan 82749cc03daSNeel Natu cap_invpcid = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 82849cc03daSNeel Natu MSR_VMX_PROCBASED_CTLS2, PROCBASED2_ENABLE_INVPCID, 0, 82949cc03daSNeel Natu &tmp) == 0); 83049cc03daSNeel Natu 83188c4b8d1SNeel Natu /* 8321bc51badSMichael Reifenberger * Check support for TPR shadow. 8331bc51badSMichael Reifenberger */ 8341bc51badSMichael Reifenberger error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 8351bc51badSMichael Reifenberger MSR_VMX_TRUE_PROCBASED_CTLS, PROCBASED_USE_TPR_SHADOW, 0, 8361bc51badSMichael Reifenberger &tmp); 8371bc51badSMichael Reifenberger if (error == 0) { 8381bc51badSMichael Reifenberger tpr_shadowing = 1; 839f3ff0918SZhenlei Huang #ifndef BURN_BRIDGES 8401bc51badSMichael Reifenberger TUNABLE_INT_FETCH("hw.vmm.vmx.use_tpr_shadowing", 8411bc51badSMichael Reifenberger &tpr_shadowing); 842f3ff0918SZhenlei Huang #endif 843f3ff0918SZhenlei Huang TUNABLE_INT_FETCH("hw.vmm.vmx.cap.tpr_shadowing", 844f3ff0918SZhenlei Huang &tpr_shadowing); 8451bc51badSMichael Reifenberger } 8461bc51badSMichael Reifenberger 8471bc51badSMichael Reifenberger if (tpr_shadowing) { 8481bc51badSMichael Reifenberger procbased_ctls |= PROCBASED_USE_TPR_SHADOW; 8491bc51badSMichael Reifenberger procbased_ctls &= ~PROCBASED_CR8_LOAD_EXITING; 8501bc51badSMichael Reifenberger procbased_ctls &= ~PROCBASED_CR8_STORE_EXITING; 8511bc51badSMichael Reifenberger } 8521bc51badSMichael Reifenberger 8531bc51badSMichael Reifenberger /* 85488c4b8d1SNeel Natu * Check support for virtual interrupt delivery. 85588c4b8d1SNeel Natu */ 85688c4b8d1SNeel Natu procbased2_vid_bits = (PROCBASED2_VIRTUALIZE_APIC_ACCESSES | 85788c4b8d1SNeel Natu PROCBASED2_VIRTUALIZE_X2APIC_MODE | 85888c4b8d1SNeel Natu PROCBASED2_APIC_REGISTER_VIRTUALIZATION | 85988c4b8d1SNeel Natu PROCBASED2_VIRTUAL_INTERRUPT_DELIVERY); 86088c4b8d1SNeel Natu 86188c4b8d1SNeel Natu error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2, 86288c4b8d1SNeel Natu procbased2_vid_bits, 0, &tmp); 8631bc51badSMichael Reifenberger if (error == 0 && tpr_shadowing) { 86488c4b8d1SNeel Natu virtual_interrupt_delivery = 1; 865f3ff0918SZhenlei Huang #ifndef BURN_BRIDGES 86688c4b8d1SNeel Natu TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_vid", 86788c4b8d1SNeel Natu &virtual_interrupt_delivery); 868f3ff0918SZhenlei Huang #endif 869f3ff0918SZhenlei Huang TUNABLE_INT_FETCH("hw.vmm.vmx.cap.virtual_interrupt_delivery", 870f3ff0918SZhenlei Huang &virtual_interrupt_delivery); 87188c4b8d1SNeel Natu } 87288c4b8d1SNeel Natu 87388c4b8d1SNeel Natu if (virtual_interrupt_delivery) { 87488c4b8d1SNeel Natu procbased_ctls |= PROCBASED_USE_TPR_SHADOW; 87588c4b8d1SNeel Natu procbased_ctls2 |= procbased2_vid_bits; 87688c4b8d1SNeel Natu procbased_ctls2 &= ~PROCBASED2_VIRTUALIZE_X2APIC_MODE; 877176666c2SNeel Natu 878176666c2SNeel Natu /* 879176666c2SNeel Natu * Check for Posted Interrupts only if Virtual Interrupt 880176666c2SNeel Natu * Delivery is enabled. 881176666c2SNeel Natu */ 882176666c2SNeel Natu error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS, 883176666c2SNeel Natu MSR_VMX_TRUE_PINBASED_CTLS, PINBASED_POSTED_INTERRUPT, 0, 884176666c2SNeel Natu &tmp); 885176666c2SNeel Natu if (error == 0) { 886bd50262fSKonstantin Belousov pirvec = lapic_ipi_alloc(pti ? &IDTVEC(justreturn1_pti) : 887bd50262fSKonstantin Belousov &IDTVEC(justreturn)); 88818a2b08eSNeel Natu if (pirvec < 0) { 889176666c2SNeel Natu if (bootverbose) { 89015add60dSPeter Grehan printf("vmx_modinit: unable to " 89115add60dSPeter Grehan "allocate posted interrupt " 89215add60dSPeter Grehan "vector\n"); 89388c4b8d1SNeel Natu } 894176666c2SNeel Natu } else { 895176666c2SNeel Natu posted_interrupts = 1; 896f3ff0918SZhenlei Huang #ifndef BURN_BRIDGES 897176666c2SNeel Natu TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_pir", 898176666c2SNeel Natu &posted_interrupts); 899f3ff0918SZhenlei Huang #endif 900f3ff0918SZhenlei Huang TUNABLE_INT_FETCH("hw.vmm.vmx.cap.posted_interrupts", 901f3ff0918SZhenlei Huang &posted_interrupts); 902176666c2SNeel Natu } 903176666c2SNeel Natu } 904176666c2SNeel Natu } 905176666c2SNeel Natu 906176666c2SNeel Natu if (posted_interrupts) 907176666c2SNeel Natu pinbased_ctls |= PINBASED_POSTED_INTERRUPT; 90849cc03daSNeel Natu 909366f6083SPeter Grehan /* Initialize EPT */ 910add611fdSNeel Natu error = ept_init(ipinum); 911366f6083SPeter Grehan if (error) { 91215add60dSPeter Grehan printf("vmx_modinit: ept initialization failed (%d)\n", error); 913366f6083SPeter Grehan return (error); 914366f6083SPeter Grehan } 915366f6083SPeter Grehan 91623437573SKonstantin Belousov guest_l1d_flush = (cpu_ia32_arch_caps & 91723437573SKonstantin Belousov IA32_ARCH_CAP_SKIP_L1DFL_VMENTRY) == 0; 918f3ff0918SZhenlei Huang #ifndef BURN_BRIDGES 919c30578feSKonstantin Belousov TUNABLE_INT_FETCH("hw.vmm.l1d_flush", &guest_l1d_flush); 920f3ff0918SZhenlei Huang #endif 921f3ff0918SZhenlei Huang TUNABLE_INT_FETCH("hw.vmm.vmx.l1d_flush", &guest_l1d_flush); 922c1141fbaSKonstantin Belousov 923c1141fbaSKonstantin Belousov /* 924c1141fbaSKonstantin Belousov * L1D cache flush is enabled. Use IA32_FLUSH_CMD MSR when 925c1141fbaSKonstantin Belousov * available. Otherwise fall back to the software flush 926c1141fbaSKonstantin Belousov * method which loads enough data from the kernel text to 927c1141fbaSKonstantin Belousov * flush existing L1D content, both on VMX entry and on NMI 928c1141fbaSKonstantin Belousov * return. 929c1141fbaSKonstantin Belousov */ 930c1141fbaSKonstantin Belousov if (guest_l1d_flush) { 931c1141fbaSKonstantin Belousov if ((cpu_stdext_feature3 & CPUID_STDEXT3_L1D_FLUSH) == 0) { 932c1141fbaSKonstantin Belousov guest_l1d_flush_sw = 1; 933f3ff0918SZhenlei Huang #ifndef BURN_BRIDGES 934c1141fbaSKonstantin Belousov TUNABLE_INT_FETCH("hw.vmm.l1d_flush_sw", 935c1141fbaSKonstantin Belousov &guest_l1d_flush_sw); 936f3ff0918SZhenlei Huang #endif 937f3ff0918SZhenlei Huang TUNABLE_INT_FETCH("hw.vmm.vmx.l1d_flush_sw", 938f3ff0918SZhenlei Huang &guest_l1d_flush_sw); 939c1141fbaSKonstantin Belousov } 940c1141fbaSKonstantin Belousov if (guest_l1d_flush_sw) { 941c1141fbaSKonstantin Belousov if (nmi_flush_l1d_sw <= 1) 942c1141fbaSKonstantin Belousov nmi_flush_l1d_sw = 1; 943c1141fbaSKonstantin Belousov } else { 944c1141fbaSKonstantin Belousov msr_load_list[0].index = MSR_IA32_FLUSH_CMD; 945c1141fbaSKonstantin Belousov msr_load_list[0].val = IA32_FLUSH_CMD_L1D; 946c1141fbaSKonstantin Belousov } 947c1141fbaSKonstantin Belousov } 948c30578feSKonstantin Belousov 949366f6083SPeter Grehan /* 950366f6083SPeter Grehan * Stash the cr0 and cr4 bits that must be fixed to 0 or 1 951366f6083SPeter Grehan */ 952366f6083SPeter Grehan fixed0 = rdmsr(MSR_VMX_CR0_FIXED0); 953366f6083SPeter Grehan fixed1 = rdmsr(MSR_VMX_CR0_FIXED1); 954366f6083SPeter Grehan cr0_ones_mask = fixed0 & fixed1; 955366f6083SPeter Grehan cr0_zeros_mask = ~fixed0 & ~fixed1; 956366f6083SPeter Grehan 957366f6083SPeter Grehan /* 958366f6083SPeter Grehan * CR0_PE and CR0_PG can be set to zero in VMX non-root operation 959366f6083SPeter Grehan * if unrestricted guest execution is allowed. 960366f6083SPeter Grehan */ 961366f6083SPeter Grehan if (cap_unrestricted_guest) 962366f6083SPeter Grehan cr0_ones_mask &= ~(CR0_PG | CR0_PE); 963366f6083SPeter Grehan 964366f6083SPeter Grehan /* 965366f6083SPeter Grehan * Do not allow the guest to set CR0_NW or CR0_CD. 966366f6083SPeter Grehan */ 967366f6083SPeter Grehan cr0_zeros_mask |= (CR0_NW | CR0_CD); 968366f6083SPeter Grehan 969366f6083SPeter Grehan fixed0 = rdmsr(MSR_VMX_CR4_FIXED0); 970366f6083SPeter Grehan fixed1 = rdmsr(MSR_VMX_CR4_FIXED1); 971366f6083SPeter Grehan cr4_ones_mask = fixed0 & fixed1; 972366f6083SPeter Grehan cr4_zeros_mask = ~fixed0 & ~fixed1; 973366f6083SPeter Grehan 97445e51299SNeel Natu vpid_init(); 97545e51299SNeel Natu 976c3498942SNeel Natu vmx_msr_init(); 977c3498942SNeel Natu 978366f6083SPeter Grehan /* enable VMX operation */ 97974ac712fSMark Johnston vmxon_region = kmem_malloc((mp_maxid + 1) * PAGE_SIZE, 98074ac712fSMark Johnston M_WAITOK | M_ZERO); 981366f6083SPeter Grehan smp_rendezvous(NULL, vmx_enable, NULL, NULL); 982366f6083SPeter Grehan 9833565b59eSNeel Natu vmx_initialized = 1; 9843565b59eSNeel Natu 985366f6083SPeter Grehan return (0); 986366f6083SPeter Grehan } 987366f6083SPeter Grehan 988f7d47425SNeel Natu static void 989f7d47425SNeel Natu vmx_trigger_hostintr(int vector) 990f7d47425SNeel Natu { 991f7d47425SNeel Natu uintptr_t func; 992f7d47425SNeel Natu struct gate_descriptor *gd; 993f7d47425SNeel Natu 994f7d47425SNeel Natu gd = &idt[vector]; 995f7d47425SNeel Natu 996f7d47425SNeel Natu KASSERT(vector >= 32 && vector <= 255, ("vmx_trigger_hostintr: " 997f7d47425SNeel Natu "invalid vector %d", vector)); 998f7d47425SNeel Natu KASSERT(gd->gd_p == 1, ("gate descriptor for vector %d not present", 999f7d47425SNeel Natu vector)); 1000f7d47425SNeel Natu KASSERT(gd->gd_type == SDT_SYSIGT, ("gate descriptor for vector %d " 1001f7d47425SNeel Natu "has invalid type %d", vector, gd->gd_type)); 1002f7d47425SNeel Natu KASSERT(gd->gd_dpl == SEL_KPL, ("gate descriptor for vector %d " 1003f7d47425SNeel Natu "has invalid dpl %d", vector, gd->gd_dpl)); 1004f7d47425SNeel Natu KASSERT(gd->gd_selector == GSEL(GCODE_SEL, SEL_KPL), ("gate descriptor " 1005f7d47425SNeel Natu "for vector %d has invalid selector %d", vector, gd->gd_selector)); 1006f7d47425SNeel Natu KASSERT(gd->gd_ist == 0, ("gate descriptor for vector %d has invalid " 1007f7d47425SNeel Natu "IST %d", vector, gd->gd_ist)); 1008f7d47425SNeel Natu 1009f7d47425SNeel Natu func = ((long)gd->gd_hioffset << 16 | gd->gd_looffset); 1010f7d47425SNeel Natu vmx_call_isr(func); 1011f7d47425SNeel Natu } 1012f7d47425SNeel Natu 1013366f6083SPeter Grehan static int 1014aaaa0656SPeter Grehan vmx_setup_cr_shadow(int which, struct vmcs *vmcs, uint32_t initial) 1015366f6083SPeter Grehan { 101639c21c2dSNeel Natu int error, mask_ident, shadow_ident; 1017aaaa0656SPeter Grehan uint64_t mask_value; 1018366f6083SPeter Grehan 101939c21c2dSNeel Natu if (which != 0 && which != 4) 102039c21c2dSNeel Natu panic("vmx_setup_cr_shadow: unknown cr%d", which); 102139c21c2dSNeel Natu 102239c21c2dSNeel Natu if (which == 0) { 102339c21c2dSNeel Natu mask_ident = VMCS_CR0_MASK; 102439c21c2dSNeel Natu mask_value = cr0_ones_mask | cr0_zeros_mask; 102539c21c2dSNeel Natu shadow_ident = VMCS_CR0_SHADOW; 102639c21c2dSNeel Natu } else { 102739c21c2dSNeel Natu mask_ident = VMCS_CR4_MASK; 102839c21c2dSNeel Natu mask_value = cr4_ones_mask | cr4_zeros_mask; 102939c21c2dSNeel Natu shadow_ident = VMCS_CR4_SHADOW; 103039c21c2dSNeel Natu } 103139c21c2dSNeel Natu 1032d3c11f40SPeter Grehan error = vmcs_setreg(vmcs, 0, VMCS_IDENT(mask_ident), mask_value); 1033366f6083SPeter Grehan if (error) 1034366f6083SPeter Grehan return (error); 1035366f6083SPeter Grehan 1036aaaa0656SPeter Grehan error = vmcs_setreg(vmcs, 0, VMCS_IDENT(shadow_ident), initial); 1037366f6083SPeter Grehan if (error) 1038366f6083SPeter Grehan return (error); 1039366f6083SPeter Grehan 1040366f6083SPeter Grehan return (0); 1041366f6083SPeter Grehan } 1042aaaa0656SPeter Grehan #define vmx_setup_cr0_shadow(vmcs,init) vmx_setup_cr_shadow(0, (vmcs), (init)) 1043aaaa0656SPeter Grehan #define vmx_setup_cr4_shadow(vmcs,init) vmx_setup_cr_shadow(4, (vmcs), (init)) 1044366f6083SPeter Grehan 1045366f6083SPeter Grehan static void * 104615add60dSPeter Grehan vmx_init(struct vm *vm, pmap_t pmap) 1047366f6083SPeter Grehan { 1048d487cba3SCy Schubert int error __diagused; 1049366f6083SPeter Grehan struct vmx *vmx; 1050366f6083SPeter Grehan 1051366f6083SPeter Grehan vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO); 1052366f6083SPeter Grehan vmx->vm = vm; 1053366f6083SPeter Grehan 10549ce875d9SKonstantin Belousov vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pmltop)); 1055318224bbSNeel Natu 1056366f6083SPeter Grehan /* 1057366f6083SPeter Grehan * Clean up EPTP-tagged guest physical and combined mappings 1058366f6083SPeter Grehan * 1059366f6083SPeter Grehan * VMX transitions are not required to invalidate any guest physical 1060366f6083SPeter Grehan * mappings. So, it may be possible for stale guest physical mappings 1061366f6083SPeter Grehan * to be present in the processor TLBs. 1062366f6083SPeter Grehan * 1063366f6083SPeter Grehan * Combined mappings for this EP4TA are also invalidated for all VPIDs. 1064366f6083SPeter Grehan */ 1065318224bbSNeel Natu ept_invalidate_mappings(vmx->eptp); 1066366f6083SPeter Grehan 10670f00260cSJohn Baldwin vmx->msr_bitmap = malloc_aligned(PAGE_SIZE, PAGE_SIZE, M_VMX, 10680f00260cSJohn Baldwin M_WAITOK | M_ZERO); 1069366f6083SPeter Grehan msr_bitmap_initialize(vmx->msr_bitmap); 1070366f6083SPeter Grehan 1071366f6083SPeter Grehan /* 1072366f6083SPeter Grehan * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE. 1073366f6083SPeter Grehan * The guest FSBASE and GSBASE are saved and restored during 1074366f6083SPeter Grehan * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are 1075366f6083SPeter Grehan * always restored from the vmcs host state area on vm-exit. 1076366f6083SPeter Grehan * 10771fb0ea3fSPeter Grehan * The SYSENTER_CS/ESP/EIP MSRs are identical to FS/GSBASE in 10781fb0ea3fSPeter Grehan * how they are saved/restored so can be directly accessed by the 10791fb0ea3fSPeter Grehan * guest. 10801fb0ea3fSPeter Grehan * 1081366f6083SPeter Grehan * MSR_EFER is saved and restored in the guest VMCS area on a 1082366f6083SPeter Grehan * VM exit and entry respectively. It is also restored from the 1083366f6083SPeter Grehan * host VMCS area on a VM exit. 10848d1d7a9eSPeter Grehan * 1085277bdd99STycho Nightingale * The TSC MSR is exposed read-only. Writes are disallowed as 1086277bdd99STycho Nightingale * that will impact the host TSC. If the guest does a write 1087277bdd99STycho Nightingale * the "use TSC offsetting" execution control is enabled and the 1088277bdd99STycho Nightingale * difference between the host TSC and the guest TSC is written 1089277bdd99STycho Nightingale * into the TSC offset in the VMCS. 1090f5f5f1e7SPeter Grehan * 1091f5f5f1e7SPeter Grehan * Guest TSC_AUX support is enabled if any of guest RDPID and/or 1092f5f5f1e7SPeter Grehan * guest RDTSCP support are enabled (since, as per Table 2-2 in SDM 1093f5f5f1e7SPeter Grehan * volume 4, TSC_AUX is supported if any of RDPID and/or RDTSCP are 1094f5f5f1e7SPeter Grehan * supported). If guest TSC_AUX support is enabled, TSC_AUX is 1095f5f5f1e7SPeter Grehan * exposed read-only so that the VMM can do one fewer MSR read per 1096f5f5f1e7SPeter Grehan * exit than if this register were exposed read-write; the guest 1097f5f5f1e7SPeter Grehan * restore value can be updated during guest writes (expected to be 1098f5f5f1e7SPeter Grehan * rare) instead of during all exits (common). 1099366f6083SPeter Grehan */ 1100366f6083SPeter Grehan if (guest_msr_rw(vmx, MSR_GSBASE) || 1101366f6083SPeter Grehan guest_msr_rw(vmx, MSR_FSBASE) || 11021fb0ea3fSPeter Grehan guest_msr_rw(vmx, MSR_SYSENTER_CS_MSR) || 11031fb0ea3fSPeter Grehan guest_msr_rw(vmx, MSR_SYSENTER_ESP_MSR) || 11041fb0ea3fSPeter Grehan guest_msr_rw(vmx, MSR_SYSENTER_EIP_MSR) || 11058d1d7a9eSPeter Grehan guest_msr_rw(vmx, MSR_EFER) || 1106f5f5f1e7SPeter Grehan guest_msr_ro(vmx, MSR_TSC) || 1107f5f5f1e7SPeter Grehan ((cap_rdpid || cap_rdtscp) && guest_msr_ro(vmx, MSR_TSC_AUX))) 110815add60dSPeter Grehan panic("vmx_init: error setting guest msr access"); 1109366f6083SPeter Grehan 111088c4b8d1SNeel Natu if (virtual_interrupt_delivery) { 111188c4b8d1SNeel Natu error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE, 111288c4b8d1SNeel Natu APIC_ACCESS_ADDRESS); 111388c4b8d1SNeel Natu /* XXX this should really return an error to the caller */ 111488c4b8d1SNeel Natu KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error)); 111588c4b8d1SNeel Natu } 111688c4b8d1SNeel Natu 11171aa51504SJohn Baldwin vmx->pmap = pmap; 11181aa51504SJohn Baldwin return (vmx); 11191aa51504SJohn Baldwin } 11200f00260cSJohn Baldwin 11211aa51504SJohn Baldwin static void * 1122950af9ffSJohn Baldwin vmx_vcpu_init(void *vmi, struct vcpu *vcpu1, int vcpuid) 11231aa51504SJohn Baldwin { 1124869c8d19SJohn Baldwin struct vmx *vmx = vmi; 11251aa51504SJohn Baldwin struct vmcs *vmcs; 11261aa51504SJohn Baldwin struct vmx_vcpu *vcpu; 11271aa51504SJohn Baldwin uint32_t exc_bitmap; 112858eefc67SJohn Baldwin uint16_t vpid; 11291aa51504SJohn Baldwin int error; 11301aa51504SJohn Baldwin 113158eefc67SJohn Baldwin vpid = vpid_alloc(vcpuid); 113258eefc67SJohn Baldwin 11331aa51504SJohn Baldwin vcpu = malloc(sizeof(*vcpu), M_VMX, M_WAITOK | M_ZERO); 1134869c8d19SJohn Baldwin vcpu->vmx = vmx; 1135950af9ffSJohn Baldwin vcpu->vcpu = vcpu1; 11361aa51504SJohn Baldwin vcpu->vcpuid = vcpuid; 11370f00260cSJohn Baldwin vcpu->vmcs = malloc_aligned(sizeof(*vmcs), PAGE_SIZE, M_VMX, 11380f00260cSJohn Baldwin M_WAITOK | M_ZERO); 11390f00260cSJohn Baldwin vcpu->apic_page = malloc_aligned(PAGE_SIZE, PAGE_SIZE, M_VMX, 11400f00260cSJohn Baldwin M_WAITOK | M_ZERO); 11411aa51504SJohn Baldwin vcpu->pir_desc = malloc_aligned(sizeof(*vcpu->pir_desc), 64, M_VMX, 11421aa51504SJohn Baldwin M_WAITOK | M_ZERO); 11430f00260cSJohn Baldwin 11440f00260cSJohn Baldwin vmcs = vcpu->vmcs; 1145c847a506SNeel Natu vmcs->identifier = vmx_revision(); 1146c847a506SNeel Natu error = vmclear(vmcs); 1147366f6083SPeter Grehan if (error != 0) { 114815add60dSPeter Grehan panic("vmx_init: vmclear error %d on vcpu %d\n", 11491aa51504SJohn Baldwin error, vcpuid); 1150366f6083SPeter Grehan } 1151366f6083SPeter Grehan 11521aa51504SJohn Baldwin vmx_msr_guest_init(vmx, vcpu); 1153c3498942SNeel Natu 1154c847a506SNeel Natu error = vmcs_init(vmcs); 1155c847a506SNeel Natu KASSERT(error == 0, ("vmcs_init error %d", error)); 1156366f6083SPeter Grehan 1157c847a506SNeel Natu VMPTRLD(vmcs); 1158c847a506SNeel Natu error = 0; 11590f00260cSJohn Baldwin error += vmwrite(VMCS_HOST_RSP, (u_long)&vcpu->ctx); 1160c847a506SNeel Natu error += vmwrite(VMCS_EPTP, vmx->eptp); 1161c847a506SNeel Natu error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls); 1162c847a506SNeel Natu error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls); 116380cb5d84SJohn Baldwin if (vcpu_trap_wbinvd(vcpu->vcpu)) { 11643ba952e1SCorvin Köhne KASSERT(cap_wbinvd_exit, ("WBINVD trap not available")); 11653ba952e1SCorvin Köhne procbased_ctls2 |= PROCBASED2_WBINVD_EXITING; 11663ba952e1SCorvin Köhne } 1167c847a506SNeel Natu error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2); 1168c847a506SNeel Natu error += vmwrite(VMCS_EXIT_CTLS, exit_ctls); 1169c847a506SNeel Natu error += vmwrite(VMCS_ENTRY_CTLS, entry_ctls); 1170c847a506SNeel Natu error += vmwrite(VMCS_MSR_BITMAP, vtophys(vmx->msr_bitmap)); 117158eefc67SJohn Baldwin error += vmwrite(VMCS_VPID, vpid); 1172b0538143SNeel Natu 1173c1141fbaSKonstantin Belousov if (guest_l1d_flush && !guest_l1d_flush_sw) { 1174c1141fbaSKonstantin Belousov vmcs_write(VMCS_ENTRY_MSR_LOAD, pmap_kextract( 1175c1141fbaSKonstantin Belousov (vm_offset_t)&msr_load_list[0])); 1176c1141fbaSKonstantin Belousov vmcs_write(VMCS_ENTRY_MSR_LOAD_COUNT, 1177c1141fbaSKonstantin Belousov nitems(msr_load_list)); 1178c1141fbaSKonstantin Belousov vmcs_write(VMCS_EXIT_MSR_STORE, 0); 1179c1141fbaSKonstantin Belousov vmcs_write(VMCS_EXIT_MSR_STORE_COUNT, 0); 1180c1141fbaSKonstantin Belousov } 1181c1141fbaSKonstantin Belousov 1182b0538143SNeel Natu /* exception bitmap */ 118380cb5d84SJohn Baldwin if (vcpu_trace_exceptions(vcpu->vcpu)) 1184b0538143SNeel Natu exc_bitmap = 0xffffffff; 1185b0538143SNeel Natu else 1186b0538143SNeel Natu exc_bitmap = 1 << IDT_MC; 1187b0538143SNeel Natu error += vmwrite(VMCS_EXCEPTION_BITMAP, exc_bitmap); 1188b0538143SNeel Natu 11890f00260cSJohn Baldwin vcpu->ctx.guest_dr6 = DBREG_DR6_RESERVED1; 11909e2154ffSJohn Baldwin error += vmwrite(VMCS_GUEST_DR7, DBREG_DR7_RESERVED1); 119165eefbe4SJohn Baldwin 11921bc51badSMichael Reifenberger if (tpr_shadowing) { 11931aa51504SJohn Baldwin error += vmwrite(VMCS_VIRTUAL_APIC, vtophys(vcpu->apic_page)); 11941bc51badSMichael Reifenberger } 11951bc51badSMichael Reifenberger 11961bc51badSMichael Reifenberger if (virtual_interrupt_delivery) { 11971bc51badSMichael Reifenberger error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS); 119888c4b8d1SNeel Natu error += vmwrite(VMCS_EOI_EXIT0, 0); 119988c4b8d1SNeel Natu error += vmwrite(VMCS_EOI_EXIT1, 0); 120088c4b8d1SNeel Natu error += vmwrite(VMCS_EOI_EXIT2, 0); 120188c4b8d1SNeel Natu error += vmwrite(VMCS_EOI_EXIT3, 0); 120288c4b8d1SNeel Natu } 1203176666c2SNeel Natu if (posted_interrupts) { 1204176666c2SNeel Natu error += vmwrite(VMCS_PIR_VECTOR, pirvec); 12051aa51504SJohn Baldwin error += vmwrite(VMCS_PIR_DESC, vtophys(vcpu->pir_desc)); 1206176666c2SNeel Natu } 1207c847a506SNeel Natu VMCLEAR(vmcs); 120815add60dSPeter Grehan KASSERT(error == 0, ("vmx_init: error customizing the vmcs")); 1209366f6083SPeter Grehan 12100f00260cSJohn Baldwin vcpu->cap.set = 0; 12110f00260cSJohn Baldwin vcpu->cap.set |= cap_rdpid != 0 ? 1 << VM_CAP_RDPID : 0; 12120f00260cSJohn Baldwin vcpu->cap.set |= cap_rdtscp != 0 ? 1 << VM_CAP_RDTSCP : 0; 12130f00260cSJohn Baldwin vcpu->cap.proc_ctls = procbased_ctls; 12140f00260cSJohn Baldwin vcpu->cap.proc_ctls2 = procbased_ctls2; 12150f00260cSJohn Baldwin vcpu->cap.exc_bitmap = exc_bitmap; 1216366f6083SPeter Grehan 12170f00260cSJohn Baldwin vcpu->state.nextrip = ~0; 12180f00260cSJohn Baldwin vcpu->state.lastcpu = NOCPU; 121958eefc67SJohn Baldwin vcpu->state.vpid = vpid; 1220366f6083SPeter Grehan 1221aaaa0656SPeter Grehan /* 1222aaaa0656SPeter Grehan * Set up the CR0/4 shadows, and init the read shadow 1223aaaa0656SPeter Grehan * to the power-on register value from the Intel Sys Arch. 1224aaaa0656SPeter Grehan * CR0 - 0x60000010 1225aaaa0656SPeter Grehan * CR4 - 0 1226aaaa0656SPeter Grehan */ 1227c847a506SNeel Natu error = vmx_setup_cr0_shadow(vmcs, 0x60000010); 122839c21c2dSNeel Natu if (error != 0) 122939c21c2dSNeel Natu panic("vmx_setup_cr0_shadow %d", error); 123039c21c2dSNeel Natu 1231c847a506SNeel Natu error = vmx_setup_cr4_shadow(vmcs, 0); 123239c21c2dSNeel Natu if (error != 0) 123339c21c2dSNeel Natu panic("vmx_setup_cr4_shadow %d", error); 1234318224bbSNeel Natu 12351aa51504SJohn Baldwin vcpu->ctx.pmap = vmx->pmap; 1236366f6083SPeter Grehan 12371aa51504SJohn Baldwin return (vcpu); 1238366f6083SPeter Grehan } 1239366f6083SPeter Grehan 1240366f6083SPeter Grehan static int 124180cb5d84SJohn Baldwin vmx_handle_cpuid(struct vmx_vcpu *vcpu, struct vmxctx *vmxctx) 1242366f6083SPeter Grehan { 1243a3f2a9c5SJohn Baldwin int handled; 1244366f6083SPeter Grehan 124580cb5d84SJohn Baldwin handled = x86_emulate_cpuid(vcpu->vcpu, (uint64_t *)&vmxctx->guest_rax, 1246a3f2a9c5SJohn Baldwin (uint64_t *)&vmxctx->guest_rbx, (uint64_t *)&vmxctx->guest_rcx, 1247a3f2a9c5SJohn Baldwin (uint64_t *)&vmxctx->guest_rdx); 1248366f6083SPeter Grehan return (handled); 1249366f6083SPeter Grehan } 1250366f6083SPeter Grehan 1251366f6083SPeter Grehan static __inline void 1252869c8d19SJohn Baldwin vmx_run_trace(struct vmx_vcpu *vcpu) 1253366f6083SPeter Grehan { 125457e0119eSJohn Baldwin VMX_CTR1(vcpu, "Resume execution at %#lx", vmcs_guest_rip()); 1255366f6083SPeter Grehan } 1256366f6083SPeter Grehan 1257366f6083SPeter Grehan static __inline void 1258869c8d19SJohn Baldwin vmx_exit_trace(struct vmx_vcpu *vcpu, uint64_t rip, uint32_t exit_reason, 1259869c8d19SJohn Baldwin int handled) 1260366f6083SPeter Grehan { 126157e0119eSJohn Baldwin VMX_CTR3(vcpu, "%s %s vmexit at 0x%0lx", 1262366f6083SPeter Grehan handled ? "handled" : "unhandled", 1263366f6083SPeter Grehan exit_reason_to_str(exit_reason), rip); 1264eeefa4e4SNeel Natu } 1265366f6083SPeter Grehan 1266eeefa4e4SNeel Natu static __inline void 1267869c8d19SJohn Baldwin vmx_astpending_trace(struct vmx_vcpu *vcpu, uint64_t rip) 1268eeefa4e4SNeel Natu { 126957e0119eSJohn Baldwin VMX_CTR1(vcpu, "astpending vmexit at 0x%0lx", rip); 1270366f6083SPeter Grehan } 1271366f6083SPeter Grehan 1272953c2c47SNeel Natu static VMM_STAT_INTEL(VCPU_INVVPID_SAVED, "Number of vpid invalidations saved"); 12733527963bSNeel Natu static VMM_STAT_INTEL(VCPU_INVVPID_DONE, "Number of vpid invalidations done"); 1274953c2c47SNeel Natu 12753527963bSNeel Natu /* 12763527963bSNeel Natu * Invalidate guest mappings identified by its vpid from the TLB. 12773527963bSNeel Natu */ 12783527963bSNeel Natu static __inline void 12791aa51504SJohn Baldwin vmx_invvpid(struct vmx *vmx, struct vmx_vcpu *vcpu, pmap_t pmap, int running) 1280366f6083SPeter Grehan { 1281366f6083SPeter Grehan struct vmxstate *vmxstate; 1282953c2c47SNeel Natu struct invvpid_desc invvpid_desc; 1283366f6083SPeter Grehan 12841aa51504SJohn Baldwin vmxstate = &vcpu->state; 12853527963bSNeel Natu if (vmxstate->vpid == 0) 12863de83862SNeel Natu return; 1287366f6083SPeter Grehan 12883527963bSNeel Natu if (!running) { 12893527963bSNeel Natu /* 12903527963bSNeel Natu * Set the 'lastcpu' to an invalid host cpu. 12913527963bSNeel Natu * 12923527963bSNeel Natu * This will invalidate TLB entries tagged with the vcpu's 12933527963bSNeel Natu * vpid the next time it runs via vmx_set_pcpu_defaults(). 12943527963bSNeel Natu */ 12953527963bSNeel Natu vmxstate->lastcpu = NOCPU; 12963527963bSNeel Natu return; 12973527963bSNeel Natu } 1298953c2c47SNeel Natu 12993527963bSNeel Natu KASSERT(curthread->td_critnest > 0, ("%s: vcpu %d running outside " 13001aa51504SJohn Baldwin "critical section", __func__, vcpu->vcpuid)); 1301366f6083SPeter Grehan 1302366f6083SPeter Grehan /* 13033527963bSNeel Natu * Invalidate all mappings tagged with 'vpid' 1304366f6083SPeter Grehan * 1305366f6083SPeter Grehan * We do this because this vcpu was executing on a different host 1306366f6083SPeter Grehan * cpu when it last ran. We do not track whether it invalidated 1307366f6083SPeter Grehan * mappings associated with its 'vpid' during that run. So we must 1308366f6083SPeter Grehan * assume that the mappings associated with 'vpid' on 'curcpu' are 1309366f6083SPeter Grehan * stale and invalidate them. 1310366f6083SPeter Grehan * 1311366f6083SPeter Grehan * Note that we incur this penalty only when the scheduler chooses to 1312366f6083SPeter Grehan * move the thread associated with this vcpu between host cpus. 1313366f6083SPeter Grehan * 1314366f6083SPeter Grehan * Note also that this will invalidate mappings tagged with 'vpid' 1315366f6083SPeter Grehan * for "all" EP4TAs. 1316366f6083SPeter Grehan */ 13176f5a9606SMark Johnston if (atomic_load_long(&pmap->pm_eptgen) == vmx->eptgen[curcpu]) { 1318953c2c47SNeel Natu invvpid_desc._res1 = 0; 1319953c2c47SNeel Natu invvpid_desc._res2 = 0; 1320366f6083SPeter Grehan invvpid_desc.vpid = vmxstate->vpid; 13210e30c5c0SWarner Losh invvpid_desc.linear_addr = 0; 1322366f6083SPeter Grehan invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc); 13233dc3d32aSJohn Baldwin vmm_stat_incr(vcpu->vcpu, VCPU_INVVPID_DONE, 1); 1324953c2c47SNeel Natu } else { 1325953c2c47SNeel Natu /* 1326953c2c47SNeel Natu * The invvpid can be skipped if an invept is going to 1327953c2c47SNeel Natu * be performed before entering the guest. The invept 1328953c2c47SNeel Natu * will invalidate combined mappings tagged with 1329953c2c47SNeel Natu * 'vmx->eptp' for all vpids. 1330953c2c47SNeel Natu */ 13313dc3d32aSJohn Baldwin vmm_stat_incr(vcpu->vcpu, VCPU_INVVPID_SAVED, 1); 1332953c2c47SNeel Natu } 1333366f6083SPeter Grehan } 13343527963bSNeel Natu 13353527963bSNeel Natu static void 13361aa51504SJohn Baldwin vmx_set_pcpu_defaults(struct vmx *vmx, struct vmx_vcpu *vcpu, pmap_t pmap) 13373527963bSNeel Natu { 13383527963bSNeel Natu struct vmxstate *vmxstate; 13393527963bSNeel Natu 13401aa51504SJohn Baldwin vmxstate = &vcpu->state; 13413527963bSNeel Natu if (vmxstate->lastcpu == curcpu) 13423527963bSNeel Natu return; 13433527963bSNeel Natu 13443527963bSNeel Natu vmxstate->lastcpu = curcpu; 13453527963bSNeel Natu 13463dc3d32aSJohn Baldwin vmm_stat_incr(vcpu->vcpu, VCPU_MIGRATIONS, 1); 13473527963bSNeel Natu 13483527963bSNeel Natu vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase()); 13493527963bSNeel Natu vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase()); 13503527963bSNeel Natu vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase()); 13513527963bSNeel Natu vmx_invvpid(vmx, vcpu, pmap, 1); 1352366f6083SPeter Grehan } 1353366f6083SPeter Grehan 1354366f6083SPeter Grehan /* 1355366f6083SPeter Grehan * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set. 1356366f6083SPeter Grehan */ 1357366f6083SPeter Grehan CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0); 1358366f6083SPeter Grehan 1359366f6083SPeter Grehan static void __inline 1360869c8d19SJohn Baldwin vmx_set_int_window_exiting(struct vmx_vcpu *vcpu) 1361366f6083SPeter Grehan { 1362366f6083SPeter Grehan 13631aa51504SJohn Baldwin if ((vcpu->cap.proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) { 13641aa51504SJohn Baldwin vcpu->cap.proc_ctls |= PROCBASED_INT_WINDOW_EXITING; 13651aa51504SJohn Baldwin vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls); 136657e0119eSJohn Baldwin VMX_CTR0(vcpu, "Enabling interrupt window exiting"); 136748b2d828SNeel Natu } 1368366f6083SPeter Grehan } 1369366f6083SPeter Grehan 1370366f6083SPeter Grehan static void __inline 1371869c8d19SJohn Baldwin vmx_clear_int_window_exiting(struct vmx_vcpu *vcpu) 1372366f6083SPeter Grehan { 1373366f6083SPeter Grehan 13741aa51504SJohn Baldwin KASSERT((vcpu->cap.proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0, 13751aa51504SJohn Baldwin ("intr_window_exiting not set: %#x", vcpu->cap.proc_ctls)); 13761aa51504SJohn Baldwin vcpu->cap.proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING; 13771aa51504SJohn Baldwin vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls); 137857e0119eSJohn Baldwin VMX_CTR0(vcpu, "Disabling interrupt window exiting"); 1379366f6083SPeter Grehan } 1380366f6083SPeter Grehan 1381366f6083SPeter Grehan static void __inline 1382869c8d19SJohn Baldwin vmx_set_nmi_window_exiting(struct vmx_vcpu *vcpu) 1383366f6083SPeter Grehan { 1384366f6083SPeter Grehan 13851aa51504SJohn Baldwin if ((vcpu->cap.proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) { 13861aa51504SJohn Baldwin vcpu->cap.proc_ctls |= PROCBASED_NMI_WINDOW_EXITING; 13871aa51504SJohn Baldwin vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls); 138857e0119eSJohn Baldwin VMX_CTR0(vcpu, "Enabling NMI window exiting"); 138948b2d828SNeel Natu } 1390366f6083SPeter Grehan } 1391366f6083SPeter Grehan 1392366f6083SPeter Grehan static void __inline 1393869c8d19SJohn Baldwin vmx_clear_nmi_window_exiting(struct vmx_vcpu *vcpu) 1394366f6083SPeter Grehan { 1395366f6083SPeter Grehan 13961aa51504SJohn Baldwin KASSERT((vcpu->cap.proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0, 13971aa51504SJohn Baldwin ("nmi_window_exiting not set %#x", vcpu->cap.proc_ctls)); 13981aa51504SJohn Baldwin vcpu->cap.proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING; 13991aa51504SJohn Baldwin vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls); 140057e0119eSJohn Baldwin VMX_CTR0(vcpu, "Disabling NMI window exiting"); 1401366f6083SPeter Grehan } 1402366f6083SPeter Grehan 1403277bdd99STycho Nightingale int 140480cb5d84SJohn Baldwin vmx_set_tsc_offset(struct vmx_vcpu *vcpu, uint64_t offset) 1405277bdd99STycho Nightingale { 1406277bdd99STycho Nightingale int error; 1407277bdd99STycho Nightingale 14081aa51504SJohn Baldwin if ((vcpu->cap.proc_ctls & PROCBASED_TSC_OFFSET) == 0) { 14091aa51504SJohn Baldwin vcpu->cap.proc_ctls |= PROCBASED_TSC_OFFSET; 14101aa51504SJohn Baldwin vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls); 141157e0119eSJohn Baldwin VMX_CTR0(vcpu, "Enabling TSC offsetting"); 1412277bdd99STycho Nightingale } 1413277bdd99STycho Nightingale 1414277bdd99STycho Nightingale error = vmwrite(VMCS_TSC_OFFSET, offset); 1415483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT 1416483d953aSJohn Baldwin if (error == 0) 141780cb5d84SJohn Baldwin vm_set_tsc_offset(vcpu->vcpu, offset); 1418483d953aSJohn Baldwin #endif 1419277bdd99STycho Nightingale return (error); 1420277bdd99STycho Nightingale } 1421277bdd99STycho Nightingale 142248b2d828SNeel Natu #define NMI_BLOCKING (VMCS_INTERRUPTIBILITY_NMI_BLOCKING | \ 142348b2d828SNeel Natu VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING) 142448b2d828SNeel Natu #define HWINTR_BLOCKING (VMCS_INTERRUPTIBILITY_STI_BLOCKING | \ 142548b2d828SNeel Natu VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING) 142648b2d828SNeel Natu 142748b2d828SNeel Natu static void 142880cb5d84SJohn Baldwin vmx_inject_nmi(struct vmx_vcpu *vcpu) 1429366f6083SPeter Grehan { 14305c272efaSRobert Wing uint32_t gi __diagused, info; 1431366f6083SPeter Grehan 143248b2d828SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 143348b2d828SNeel Natu KASSERT((gi & NMI_BLOCKING) == 0, ("vmx_inject_nmi: invalid guest " 143448b2d828SNeel Natu "interruptibility-state %#x", gi)); 1435366f6083SPeter Grehan 143648b2d828SNeel Natu info = vmcs_read(VMCS_ENTRY_INTR_INFO); 143748b2d828SNeel Natu KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_nmi: invalid " 143848b2d828SNeel Natu "VM-entry interruption information %#x", info)); 1439366f6083SPeter Grehan 1440366f6083SPeter Grehan /* 1441366f6083SPeter Grehan * Inject the virtual NMI. The vector must be the NMI IDT entry 1442366f6083SPeter Grehan * or the VMCS entry check will fail. 1443366f6083SPeter Grehan */ 144448b2d828SNeel Natu info = IDT_NMI | VMCS_INTR_T_NMI | VMCS_INTR_VALID; 14453de83862SNeel Natu vmcs_write(VMCS_ENTRY_INTR_INFO, info); 1446366f6083SPeter Grehan 144757e0119eSJohn Baldwin VMX_CTR0(vcpu, "Injecting vNMI"); 1448366f6083SPeter Grehan 1449366f6083SPeter Grehan /* Clear the request */ 145080cb5d84SJohn Baldwin vm_nmi_clear(vcpu->vcpu); 1451366f6083SPeter Grehan } 1452366f6083SPeter Grehan 1453366f6083SPeter Grehan static void 145480cb5d84SJohn Baldwin vmx_inject_interrupts(struct vmx_vcpu *vcpu, struct vlapic *vlapic, 145580cb5d84SJohn Baldwin uint64_t guestrip) 1456366f6083SPeter Grehan { 14570775fbb4STycho Nightingale int vector, need_nmi_exiting, extint_pending; 1458091d4532SNeel Natu uint64_t rflags, entryinfo; 145948b2d828SNeel Natu uint32_t gi, info; 1460366f6083SPeter Grehan 1461fefac543SBojan Novković if (vcpu->cap.set & (1 << VM_CAP_MASK_HWINTR)) { 1462fefac543SBojan Novković return; 1463fefac543SBojan Novković } 1464fefac543SBojan Novković 14651aa51504SJohn Baldwin if (vcpu->state.nextrip != guestrip) { 14662ce12423SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 14672ce12423SNeel Natu if (gi & HWINTR_BLOCKING) { 146857e0119eSJohn Baldwin VMX_CTR2(vcpu, "Guest interrupt blocking " 14692ce12423SNeel Natu "cleared due to rip change: %#lx/%#lx", 14701aa51504SJohn Baldwin vcpu->state.nextrip, guestrip); 14712ce12423SNeel Natu gi &= ~HWINTR_BLOCKING; 14722ce12423SNeel Natu vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi); 14732ce12423SNeel Natu } 14742ce12423SNeel Natu } 14752ce12423SNeel Natu 147680cb5d84SJohn Baldwin if (vm_entry_intinfo(vcpu->vcpu, &entryinfo)) { 1477091d4532SNeel Natu KASSERT((entryinfo & VMCS_INTR_VALID) != 0, ("%s: entry " 1478091d4532SNeel Natu "intinfo is not valid: %#lx", __func__, entryinfo)); 1479dc506506SNeel Natu 1480dc506506SNeel Natu info = vmcs_read(VMCS_ENTRY_INTR_INFO); 1481dc506506SNeel Natu KASSERT((info & VMCS_INTR_VALID) == 0, ("%s: cannot inject " 1482019008ebSNeel Natu "pending exception: %#lx/%#x", __func__, entryinfo, info)); 1483dc506506SNeel Natu 1484091d4532SNeel Natu info = entryinfo; 1485091d4532SNeel Natu vector = info & 0xff; 1486091d4532SNeel Natu if (vector == IDT_BP || vector == IDT_OF) { 1487091d4532SNeel Natu /* 1488091d4532SNeel Natu * VT-x requires #BP and #OF to be injected as software 1489091d4532SNeel Natu * exceptions. 1490091d4532SNeel Natu */ 1491091d4532SNeel Natu info &= ~VMCS_INTR_T_MASK; 1492091d4532SNeel Natu info |= VMCS_INTR_T_SWEXCEPTION; 1493dc506506SNeel Natu } 1494091d4532SNeel Natu 1495091d4532SNeel Natu if (info & VMCS_INTR_DEL_ERRCODE) 1496091d4532SNeel Natu vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR, entryinfo >> 32); 1497091d4532SNeel Natu 1498dc506506SNeel Natu vmcs_write(VMCS_ENTRY_INTR_INFO, info); 1499dc506506SNeel Natu } 1500dc506506SNeel Natu 150180cb5d84SJohn Baldwin if (vm_nmi_pending(vcpu->vcpu)) { 1502366f6083SPeter Grehan /* 150348b2d828SNeel Natu * If there are no conditions blocking NMI injection then 150448b2d828SNeel Natu * inject it directly here otherwise enable "NMI window 150548b2d828SNeel Natu * exiting" to inject it as soon as we can. 1506eeefa4e4SNeel Natu * 150748b2d828SNeel Natu * We also check for STI_BLOCKING because some implementations 150848b2d828SNeel Natu * don't allow NMI injection in this case. If we are running 150948b2d828SNeel Natu * on a processor that doesn't have this restriction it will 151048b2d828SNeel Natu * immediately exit and the NMI will be injected in the 151148b2d828SNeel Natu * "NMI window exiting" handler. 1512366f6083SPeter Grehan */ 151348b2d828SNeel Natu need_nmi_exiting = 1; 151448b2d828SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 151548b2d828SNeel Natu if ((gi & (HWINTR_BLOCKING | NMI_BLOCKING)) == 0) { 15163de83862SNeel Natu info = vmcs_read(VMCS_ENTRY_INTR_INFO); 151748b2d828SNeel Natu if ((info & VMCS_INTR_VALID) == 0) { 151880cb5d84SJohn Baldwin vmx_inject_nmi(vcpu); 151948b2d828SNeel Natu need_nmi_exiting = 0; 152048b2d828SNeel Natu } else { 152157e0119eSJohn Baldwin VMX_CTR1(vcpu, "Cannot inject NMI " 152257e0119eSJohn Baldwin "due to VM-entry intr info %#x", info); 152348b2d828SNeel Natu } 152448b2d828SNeel Natu } else { 152557e0119eSJohn Baldwin VMX_CTR1(vcpu, "Cannot inject NMI due to " 152657e0119eSJohn Baldwin "Guest Interruptibility-state %#x", gi); 152748b2d828SNeel Natu } 1528eeefa4e4SNeel Natu 152948b2d828SNeel Natu if (need_nmi_exiting) 1530869c8d19SJohn Baldwin vmx_set_nmi_window_exiting(vcpu); 153148b2d828SNeel Natu } 1532366f6083SPeter Grehan 153380cb5d84SJohn Baldwin extint_pending = vm_extint_pending(vcpu->vcpu); 15340775fbb4STycho Nightingale 15350775fbb4STycho Nightingale if (!extint_pending && virtual_interrupt_delivery) { 153688c4b8d1SNeel Natu vmx_inject_pir(vlapic); 153788c4b8d1SNeel Natu return; 153888c4b8d1SNeel Natu } 153988c4b8d1SNeel Natu 154048b2d828SNeel Natu /* 154136736912SNeel Natu * If interrupt-window exiting is already in effect then don't bother 154236736912SNeel Natu * checking for pending interrupts. This is just an optimization and 154336736912SNeel Natu * not needed for correctness. 154448b2d828SNeel Natu */ 15451aa51504SJohn Baldwin if ((vcpu->cap.proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) { 154657e0119eSJohn Baldwin VMX_CTR0(vcpu, "Skip interrupt injection due to " 154757e0119eSJohn Baldwin "pending int_window_exiting"); 154848b2d828SNeel Natu return; 154936736912SNeel Natu } 155048b2d828SNeel Natu 15510775fbb4STycho Nightingale if (!extint_pending) { 1552366f6083SPeter Grehan /* Ask the local apic for a vector to inject */ 15534d1e82a8SNeel Natu if (!vlapic_pending_intr(vlapic, &vector)) 1554366f6083SPeter Grehan return; 1555a026dc3fSTycho Nightingale 1556a026dc3fSTycho Nightingale /* 1557a026dc3fSTycho Nightingale * From the Intel SDM, Volume 3, Section "Maskable 1558a026dc3fSTycho Nightingale * Hardware Interrupts": 1559a026dc3fSTycho Nightingale * - maskable interrupt vectors [16,255] can be delivered 1560a026dc3fSTycho Nightingale * through the local APIC. 1561a026dc3fSTycho Nightingale */ 1562a026dc3fSTycho Nightingale KASSERT(vector >= 16 && vector <= 255, 1563a026dc3fSTycho Nightingale ("invalid vector %d from local APIC", vector)); 15640775fbb4STycho Nightingale } else { 15650775fbb4STycho Nightingale /* Ask the legacy pic for a vector to inject */ 156680cb5d84SJohn Baldwin vatpic_pending_intr(vcpu->vmx->vm, &vector); 1567366f6083SPeter Grehan 1568a026dc3fSTycho Nightingale /* 1569a026dc3fSTycho Nightingale * From the Intel SDM, Volume 3, Section "Maskable 1570a026dc3fSTycho Nightingale * Hardware Interrupts": 1571a026dc3fSTycho Nightingale * - maskable interrupt vectors [0,255] can be delivered 1572a026dc3fSTycho Nightingale * through the INTR pin. 1573a026dc3fSTycho Nightingale */ 1574a026dc3fSTycho Nightingale KASSERT(vector >= 0 && vector <= 255, 1575a026dc3fSTycho Nightingale ("invalid vector %d from INTR", vector)); 1576a026dc3fSTycho Nightingale } 1577366f6083SPeter Grehan 1578366f6083SPeter Grehan /* Check RFLAGS.IF and the interruptibility state of the guest */ 15793de83862SNeel Natu rflags = vmcs_read(VMCS_GUEST_RFLAGS); 158036736912SNeel Natu if ((rflags & PSL_I) == 0) { 158157e0119eSJohn Baldwin VMX_CTR2(vcpu, "Cannot inject vector %d due to " 158257e0119eSJohn Baldwin "rflags %#lx", vector, rflags); 1583366f6083SPeter Grehan goto cantinject; 158436736912SNeel Natu } 1585366f6083SPeter Grehan 158648b2d828SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 158736736912SNeel Natu if (gi & HWINTR_BLOCKING) { 158857e0119eSJohn Baldwin VMX_CTR2(vcpu, "Cannot inject vector %d due to " 158957e0119eSJohn Baldwin "Guest Interruptibility-state %#x", vector, gi); 1590366f6083SPeter Grehan goto cantinject; 159136736912SNeel Natu } 159236736912SNeel Natu 159336736912SNeel Natu info = vmcs_read(VMCS_ENTRY_INTR_INFO); 159436736912SNeel Natu if (info & VMCS_INTR_VALID) { 159536736912SNeel Natu /* 159636736912SNeel Natu * This is expected and could happen for multiple reasons: 159736736912SNeel Natu * - A vectoring VM-entry was aborted due to astpending 159836736912SNeel Natu * - A VM-exit happened during event injection. 1599dc506506SNeel Natu * - An exception was injected above. 160036736912SNeel Natu * - An NMI was injected above or after "NMI window exiting" 160136736912SNeel Natu */ 160257e0119eSJohn Baldwin VMX_CTR2(vcpu, "Cannot inject vector %d due to " 160357e0119eSJohn Baldwin "VM-entry intr info %#x", vector, info); 160436736912SNeel Natu goto cantinject; 160536736912SNeel Natu } 1606366f6083SPeter Grehan 1607366f6083SPeter Grehan /* Inject the interrupt */ 1608160471d2SNeel Natu info = VMCS_INTR_T_HWINTR | VMCS_INTR_VALID; 1609366f6083SPeter Grehan info |= vector; 16103de83862SNeel Natu vmcs_write(VMCS_ENTRY_INTR_INFO, info); 1611366f6083SPeter Grehan 16120775fbb4STycho Nightingale if (!extint_pending) { 1613366f6083SPeter Grehan /* Update the Local APIC ISR */ 1614de5ea6b6SNeel Natu vlapic_intr_accepted(vlapic, vector); 16150775fbb4STycho Nightingale } else { 161680cb5d84SJohn Baldwin vm_extint_clear(vcpu->vcpu); 161780cb5d84SJohn Baldwin vatpic_intr_accepted(vcpu->vmx->vm, vector); 16180775fbb4STycho Nightingale 16190775fbb4STycho Nightingale /* 16200775fbb4STycho Nightingale * After we accepted the current ExtINT the PIC may 16210775fbb4STycho Nightingale * have posted another one. If that is the case, set 16220775fbb4STycho Nightingale * the Interrupt Window Exiting execution control so 16230775fbb4STycho Nightingale * we can inject that one too. 16240494cb1bSNeel Natu * 16250494cb1bSNeel Natu * Also, interrupt window exiting allows us to inject any 16260494cb1bSNeel Natu * pending APIC vector that was preempted by the ExtINT 16270494cb1bSNeel Natu * as soon as possible. This applies both for the software 16280494cb1bSNeel Natu * emulated vlapic and the hardware assisted virtual APIC. 16290775fbb4STycho Nightingale */ 1630869c8d19SJohn Baldwin vmx_set_int_window_exiting(vcpu); 16310775fbb4STycho Nightingale } 1632366f6083SPeter Grehan 163357e0119eSJohn Baldwin VMX_CTR1(vcpu, "Injecting hwintr at vector %d", vector); 1634366f6083SPeter Grehan 1635366f6083SPeter Grehan return; 1636366f6083SPeter Grehan 1637366f6083SPeter Grehan cantinject: 1638366f6083SPeter Grehan /* 1639366f6083SPeter Grehan * Set the Interrupt Window Exiting execution control so we can inject 1640366f6083SPeter Grehan * the interrupt as soon as blocking condition goes away. 1641366f6083SPeter Grehan */ 1642869c8d19SJohn Baldwin vmx_set_int_window_exiting(vcpu); 1643366f6083SPeter Grehan } 1644366f6083SPeter Grehan 1645e5a1d950SNeel Natu /* 1646e5a1d950SNeel Natu * If the Virtual NMIs execution control is '1' then the logical processor 1647e5a1d950SNeel Natu * tracks virtual-NMI blocking in the Guest Interruptibility-state field of 1648e5a1d950SNeel Natu * the VMCS. An IRET instruction in VMX non-root operation will remove any 1649e5a1d950SNeel Natu * virtual-NMI blocking. 1650e5a1d950SNeel Natu * 1651e5a1d950SNeel Natu * This unblocking occurs even if the IRET causes a fault. In this case the 1652e5a1d950SNeel Natu * hypervisor needs to restore virtual-NMI blocking before resuming the guest. 1653e5a1d950SNeel Natu */ 1654e5a1d950SNeel Natu static void 1655869c8d19SJohn Baldwin vmx_restore_nmi_blocking(struct vmx_vcpu *vcpu) 1656e5a1d950SNeel Natu { 1657e5a1d950SNeel Natu uint32_t gi; 1658e5a1d950SNeel Natu 165957e0119eSJohn Baldwin VMX_CTR0(vcpu, "Restore Virtual-NMI blocking"); 1660e5a1d950SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 1661e5a1d950SNeel Natu gi |= VMCS_INTERRUPTIBILITY_NMI_BLOCKING; 1662e5a1d950SNeel Natu vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi); 1663e5a1d950SNeel Natu } 1664e5a1d950SNeel Natu 1665e5a1d950SNeel Natu static void 1666869c8d19SJohn Baldwin vmx_clear_nmi_blocking(struct vmx_vcpu *vcpu) 1667e5a1d950SNeel Natu { 1668e5a1d950SNeel Natu uint32_t gi; 1669e5a1d950SNeel Natu 167057e0119eSJohn Baldwin VMX_CTR0(vcpu, "Clear Virtual-NMI blocking"); 1671e5a1d950SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 1672e5a1d950SNeel Natu gi &= ~VMCS_INTERRUPTIBILITY_NMI_BLOCKING; 1673e5a1d950SNeel Natu vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi); 1674e5a1d950SNeel Natu } 1675e5a1d950SNeel Natu 1676091d4532SNeel Natu static void 1677869c8d19SJohn Baldwin vmx_assert_nmi_blocking(struct vmx_vcpu *vcpu) 1678091d4532SNeel Natu { 16795c272efaSRobert Wing uint32_t gi __diagused; 1680091d4532SNeel Natu 1681091d4532SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 1682091d4532SNeel Natu KASSERT(gi & VMCS_INTERRUPTIBILITY_NMI_BLOCKING, 1683091d4532SNeel Natu ("NMI blocking is not in effect %#x", gi)); 1684091d4532SNeel Natu } 1685091d4532SNeel Natu 1686366f6083SPeter Grehan static int 16871aa51504SJohn Baldwin vmx_emulate_xsetbv(struct vmx *vmx, struct vmx_vcpu *vcpu, 16881aa51504SJohn Baldwin struct vm_exit *vmexit) 1689abb023fbSJohn Baldwin { 1690abb023fbSJohn Baldwin struct vmxctx *vmxctx; 1691abb023fbSJohn Baldwin uint64_t xcrval; 1692abb023fbSJohn Baldwin const struct xsave_limits *limits; 1693abb023fbSJohn Baldwin 16941aa51504SJohn Baldwin vmxctx = &vcpu->ctx; 1695abb023fbSJohn Baldwin limits = vmm_get_xsave_limits(); 1696abb023fbSJohn Baldwin 1697a0efd3fbSJohn Baldwin /* 1698a0efd3fbSJohn Baldwin * Note that the processor raises a GP# fault on its own if 1699a0efd3fbSJohn Baldwin * xsetbv is executed for CPL != 0, so we do not have to 1700a0efd3fbSJohn Baldwin * emulate that fault here. 1701a0efd3fbSJohn Baldwin */ 1702a0efd3fbSJohn Baldwin 1703a0efd3fbSJohn Baldwin /* Only xcr0 is supported. */ 1704a0efd3fbSJohn Baldwin if (vmxctx->guest_rcx != 0) { 1705d3956e46SJohn Baldwin vm_inject_gp(vcpu->vcpu); 1706a0efd3fbSJohn Baldwin return (HANDLED); 1707a0efd3fbSJohn Baldwin } 1708a0efd3fbSJohn Baldwin 1709a0efd3fbSJohn Baldwin /* We only handle xcr0 if both the host and guest have XSAVE enabled. */ 1710a0efd3fbSJohn Baldwin if (!limits->xsave_enabled || !(vmcs_read(VMCS_GUEST_CR4) & CR4_XSAVE)) { 1711d3956e46SJohn Baldwin vm_inject_ud(vcpu->vcpu); 1712a0efd3fbSJohn Baldwin return (HANDLED); 1713a0efd3fbSJohn Baldwin } 1714abb023fbSJohn Baldwin 1715abb023fbSJohn Baldwin xcrval = vmxctx->guest_rdx << 32 | (vmxctx->guest_rax & 0xffffffff); 1716a0efd3fbSJohn Baldwin if ((xcrval & ~limits->xcr0_allowed) != 0) { 1717d3956e46SJohn Baldwin vm_inject_gp(vcpu->vcpu); 1718a0efd3fbSJohn Baldwin return (HANDLED); 1719a0efd3fbSJohn Baldwin } 1720abb023fbSJohn Baldwin 1721a0efd3fbSJohn Baldwin if (!(xcrval & XFEATURE_ENABLED_X87)) { 1722d3956e46SJohn Baldwin vm_inject_gp(vcpu->vcpu); 1723a0efd3fbSJohn Baldwin return (HANDLED); 1724a0efd3fbSJohn Baldwin } 1725abb023fbSJohn Baldwin 172644a68c4eSJohn Baldwin /* AVX (YMM_Hi128) requires SSE. */ 172744a68c4eSJohn Baldwin if (xcrval & XFEATURE_ENABLED_AVX && 172844a68c4eSJohn Baldwin (xcrval & XFEATURE_AVX) != XFEATURE_AVX) { 1729d3956e46SJohn Baldwin vm_inject_gp(vcpu->vcpu); 173044a68c4eSJohn Baldwin return (HANDLED); 173144a68c4eSJohn Baldwin } 173244a68c4eSJohn Baldwin 173344a68c4eSJohn Baldwin /* 173444a68c4eSJohn Baldwin * AVX512 requires base AVX (YMM_Hi128) as well as OpMask, 173544a68c4eSJohn Baldwin * ZMM_Hi256, and Hi16_ZMM. 173644a68c4eSJohn Baldwin */ 173744a68c4eSJohn Baldwin if (xcrval & XFEATURE_AVX512 && 173844a68c4eSJohn Baldwin (xcrval & (XFEATURE_AVX512 | XFEATURE_AVX)) != 173944a68c4eSJohn Baldwin (XFEATURE_AVX512 | XFEATURE_AVX)) { 1740d3956e46SJohn Baldwin vm_inject_gp(vcpu->vcpu); 174144a68c4eSJohn Baldwin return (HANDLED); 174244a68c4eSJohn Baldwin } 174344a68c4eSJohn Baldwin 174444a68c4eSJohn Baldwin /* 174544a68c4eSJohn Baldwin * Intel MPX requires both bound register state flags to be 174644a68c4eSJohn Baldwin * set. 174744a68c4eSJohn Baldwin */ 174844a68c4eSJohn Baldwin if (((xcrval & XFEATURE_ENABLED_BNDREGS) != 0) != 174944a68c4eSJohn Baldwin ((xcrval & XFEATURE_ENABLED_BNDCSR) != 0)) { 1750d3956e46SJohn Baldwin vm_inject_gp(vcpu->vcpu); 1751a0efd3fbSJohn Baldwin return (HANDLED); 1752a0efd3fbSJohn Baldwin } 1753abb023fbSJohn Baldwin 1754abb023fbSJohn Baldwin /* 1755abb023fbSJohn Baldwin * This runs "inside" vmrun() with the guest's FPU state, so 1756abb023fbSJohn Baldwin * modifying xcr0 directly modifies the guest's xcr0, not the 1757abb023fbSJohn Baldwin * host's. 1758abb023fbSJohn Baldwin */ 1759abb023fbSJohn Baldwin load_xcr(0, xcrval); 1760abb023fbSJohn Baldwin return (HANDLED); 1761abb023fbSJohn Baldwin } 1762abb023fbSJohn Baldwin 1763594db002STycho Nightingale static uint64_t 17641aa51504SJohn Baldwin vmx_get_guest_reg(struct vmx_vcpu *vcpu, int ident) 1765366f6083SPeter Grehan { 1766366f6083SPeter Grehan const struct vmxctx *vmxctx; 1767366f6083SPeter Grehan 17681aa51504SJohn Baldwin vmxctx = &vcpu->ctx; 1769594db002STycho Nightingale 1770594db002STycho Nightingale switch (ident) { 1771594db002STycho Nightingale case 0: 1772594db002STycho Nightingale return (vmxctx->guest_rax); 1773594db002STycho Nightingale case 1: 1774594db002STycho Nightingale return (vmxctx->guest_rcx); 1775594db002STycho Nightingale case 2: 1776594db002STycho Nightingale return (vmxctx->guest_rdx); 1777594db002STycho Nightingale case 3: 1778594db002STycho Nightingale return (vmxctx->guest_rbx); 1779594db002STycho Nightingale case 4: 1780594db002STycho Nightingale return (vmcs_read(VMCS_GUEST_RSP)); 1781594db002STycho Nightingale case 5: 1782594db002STycho Nightingale return (vmxctx->guest_rbp); 1783594db002STycho Nightingale case 6: 1784594db002STycho Nightingale return (vmxctx->guest_rsi); 1785594db002STycho Nightingale case 7: 1786594db002STycho Nightingale return (vmxctx->guest_rdi); 1787594db002STycho Nightingale case 8: 1788594db002STycho Nightingale return (vmxctx->guest_r8); 1789594db002STycho Nightingale case 9: 1790594db002STycho Nightingale return (vmxctx->guest_r9); 1791594db002STycho Nightingale case 10: 1792594db002STycho Nightingale return (vmxctx->guest_r10); 1793594db002STycho Nightingale case 11: 1794594db002STycho Nightingale return (vmxctx->guest_r11); 1795594db002STycho Nightingale case 12: 1796594db002STycho Nightingale return (vmxctx->guest_r12); 1797594db002STycho Nightingale case 13: 1798594db002STycho Nightingale return (vmxctx->guest_r13); 1799594db002STycho Nightingale case 14: 1800594db002STycho Nightingale return (vmxctx->guest_r14); 1801594db002STycho Nightingale case 15: 1802594db002STycho Nightingale return (vmxctx->guest_r15); 1803594db002STycho Nightingale default: 1804594db002STycho Nightingale panic("invalid vmx register %d", ident); 1805594db002STycho Nightingale } 1806594db002STycho Nightingale } 1807594db002STycho Nightingale 1808594db002STycho Nightingale static void 18091aa51504SJohn Baldwin vmx_set_guest_reg(struct vmx_vcpu *vcpu, int ident, uint64_t regval) 1810594db002STycho Nightingale { 1811594db002STycho Nightingale struct vmxctx *vmxctx; 1812594db002STycho Nightingale 18131aa51504SJohn Baldwin vmxctx = &vcpu->ctx; 1814594db002STycho Nightingale 1815594db002STycho Nightingale switch (ident) { 1816594db002STycho Nightingale case 0: 1817594db002STycho Nightingale vmxctx->guest_rax = regval; 1818594db002STycho Nightingale break; 1819594db002STycho Nightingale case 1: 1820594db002STycho Nightingale vmxctx->guest_rcx = regval; 1821594db002STycho Nightingale break; 1822594db002STycho Nightingale case 2: 1823594db002STycho Nightingale vmxctx->guest_rdx = regval; 1824594db002STycho Nightingale break; 1825594db002STycho Nightingale case 3: 1826594db002STycho Nightingale vmxctx->guest_rbx = regval; 1827594db002STycho Nightingale break; 1828594db002STycho Nightingale case 4: 1829594db002STycho Nightingale vmcs_write(VMCS_GUEST_RSP, regval); 1830594db002STycho Nightingale break; 1831594db002STycho Nightingale case 5: 1832594db002STycho Nightingale vmxctx->guest_rbp = regval; 1833594db002STycho Nightingale break; 1834594db002STycho Nightingale case 6: 1835594db002STycho Nightingale vmxctx->guest_rsi = regval; 1836594db002STycho Nightingale break; 1837594db002STycho Nightingale case 7: 1838594db002STycho Nightingale vmxctx->guest_rdi = regval; 1839594db002STycho Nightingale break; 1840594db002STycho Nightingale case 8: 1841594db002STycho Nightingale vmxctx->guest_r8 = regval; 1842594db002STycho Nightingale break; 1843594db002STycho Nightingale case 9: 1844594db002STycho Nightingale vmxctx->guest_r9 = regval; 1845594db002STycho Nightingale break; 1846594db002STycho Nightingale case 10: 1847594db002STycho Nightingale vmxctx->guest_r10 = regval; 1848594db002STycho Nightingale break; 1849594db002STycho Nightingale case 11: 1850594db002STycho Nightingale vmxctx->guest_r11 = regval; 1851594db002STycho Nightingale break; 1852594db002STycho Nightingale case 12: 1853594db002STycho Nightingale vmxctx->guest_r12 = regval; 1854594db002STycho Nightingale break; 1855594db002STycho Nightingale case 13: 1856594db002STycho Nightingale vmxctx->guest_r13 = regval; 1857594db002STycho Nightingale break; 1858594db002STycho Nightingale case 14: 1859594db002STycho Nightingale vmxctx->guest_r14 = regval; 1860594db002STycho Nightingale break; 1861594db002STycho Nightingale case 15: 1862594db002STycho Nightingale vmxctx->guest_r15 = regval; 1863594db002STycho Nightingale break; 1864594db002STycho Nightingale default: 1865594db002STycho Nightingale panic("invalid vmx register %d", ident); 1866594db002STycho Nightingale } 1867594db002STycho Nightingale } 1868594db002STycho Nightingale 1869594db002STycho Nightingale static int 18701aa51504SJohn Baldwin vmx_emulate_cr0_access(struct vmx_vcpu *vcpu, uint64_t exitqual) 1871594db002STycho Nightingale { 1872594db002STycho Nightingale uint64_t crval, regval; 1873594db002STycho Nightingale 1874594db002STycho Nightingale /* We only handle mov to %cr0 at this time */ 187539c21c2dSNeel Natu if ((exitqual & 0xf0) != 0x00) 187639c21c2dSNeel Natu return (UNHANDLED); 187739c21c2dSNeel Natu 18781aa51504SJohn Baldwin regval = vmx_get_guest_reg(vcpu, (exitqual >> 8) & 0xf); 1879366f6083SPeter Grehan 1880594db002STycho Nightingale vmcs_write(VMCS_CR0_SHADOW, regval); 1881366f6083SPeter Grehan 1882594db002STycho Nightingale crval = regval | cr0_ones_mask; 1883594db002STycho Nightingale crval &= ~cr0_zeros_mask; 1884594db002STycho Nightingale vmcs_write(VMCS_GUEST_CR0, crval); 1885366f6083SPeter Grehan 1886594db002STycho Nightingale if (regval & CR0_PG) { 188780a902efSPeter Grehan uint64_t efer, entry_ctls; 188880a902efSPeter Grehan 188980a902efSPeter Grehan /* 189080a902efSPeter Grehan * If CR0.PG is 1 and EFER.LME is 1 then EFER.LMA and 189180a902efSPeter Grehan * the "IA-32e mode guest" bit in VM-entry control must be 189280a902efSPeter Grehan * equal. 189380a902efSPeter Grehan */ 18943de83862SNeel Natu efer = vmcs_read(VMCS_GUEST_IA32_EFER); 189580a902efSPeter Grehan if (efer & EFER_LME) { 189680a902efSPeter Grehan efer |= EFER_LMA; 18973de83862SNeel Natu vmcs_write(VMCS_GUEST_IA32_EFER, efer); 18983de83862SNeel Natu entry_ctls = vmcs_read(VMCS_ENTRY_CTLS); 189980a902efSPeter Grehan entry_ctls |= VM_ENTRY_GUEST_LMA; 19003de83862SNeel Natu vmcs_write(VMCS_ENTRY_CTLS, entry_ctls); 190180a902efSPeter Grehan } 190280a902efSPeter Grehan } 190380a902efSPeter Grehan 1904366f6083SPeter Grehan return (HANDLED); 1905366f6083SPeter Grehan } 1906366f6083SPeter Grehan 1907594db002STycho Nightingale static int 19081aa51504SJohn Baldwin vmx_emulate_cr4_access(struct vmx_vcpu *vcpu, uint64_t exitqual) 1909594db002STycho Nightingale { 1910594db002STycho Nightingale uint64_t crval, regval; 1911594db002STycho Nightingale 1912594db002STycho Nightingale /* We only handle mov to %cr4 at this time */ 1913594db002STycho Nightingale if ((exitqual & 0xf0) != 0x00) 1914594db002STycho Nightingale return (UNHANDLED); 1915594db002STycho Nightingale 19161aa51504SJohn Baldwin regval = vmx_get_guest_reg(vcpu, (exitqual >> 8) & 0xf); 1917594db002STycho Nightingale 1918594db002STycho Nightingale vmcs_write(VMCS_CR4_SHADOW, regval); 1919594db002STycho Nightingale 1920594db002STycho Nightingale crval = regval | cr4_ones_mask; 1921594db002STycho Nightingale crval &= ~cr4_zeros_mask; 1922594db002STycho Nightingale vmcs_write(VMCS_GUEST_CR4, crval); 1923594db002STycho Nightingale 1924594db002STycho Nightingale return (HANDLED); 1925594db002STycho Nightingale } 1926594db002STycho Nightingale 1927594db002STycho Nightingale static int 19281aa51504SJohn Baldwin vmx_emulate_cr8_access(struct vmx *vmx, struct vmx_vcpu *vcpu, 19291aa51504SJohn Baldwin uint64_t exitqual) 1930594db002STycho Nightingale { 1931051f2bd1SNeel Natu struct vlapic *vlapic; 1932051f2bd1SNeel Natu uint64_t cr8; 1933051f2bd1SNeel Natu int regnum; 1934594db002STycho Nightingale 1935594db002STycho Nightingale /* We only handle mov %cr8 to/from a register at this time. */ 1936594db002STycho Nightingale if ((exitqual & 0xe0) != 0x00) { 1937594db002STycho Nightingale return (UNHANDLED); 1938594db002STycho Nightingale } 1939594db002STycho Nightingale 1940d3956e46SJohn Baldwin vlapic = vm_lapic(vcpu->vcpu); 1941051f2bd1SNeel Natu regnum = (exitqual >> 8) & 0xf; 1942594db002STycho Nightingale if (exitqual & 0x10) { 1943051f2bd1SNeel Natu cr8 = vlapic_get_cr8(vlapic); 19441aa51504SJohn Baldwin vmx_set_guest_reg(vcpu, regnum, cr8); 1945594db002STycho Nightingale } else { 19461aa51504SJohn Baldwin cr8 = vmx_get_guest_reg(vcpu, regnum); 1947051f2bd1SNeel Natu vlapic_set_cr8(vlapic, cr8); 1948594db002STycho Nightingale } 1949594db002STycho Nightingale 1950594db002STycho Nightingale return (HANDLED); 1951594db002STycho Nightingale } 1952594db002STycho Nightingale 1953e4c8a13dSNeel Natu /* 1954e4c8a13dSNeel Natu * From section "Guest Register State" in the Intel SDM: CPL = SS.DPL 1955e4c8a13dSNeel Natu */ 1956e4c8a13dSNeel Natu static int 1957e4c8a13dSNeel Natu vmx_cpl(void) 1958e4c8a13dSNeel Natu { 1959e4c8a13dSNeel Natu uint32_t ssar; 1960e4c8a13dSNeel Natu 1961e4c8a13dSNeel Natu ssar = vmcs_read(VMCS_GUEST_SS_ACCESS_RIGHTS); 1962e4c8a13dSNeel Natu return ((ssar >> 5) & 0x3); 1963e4c8a13dSNeel Natu } 1964e4c8a13dSNeel Natu 1965e813a873SNeel Natu static enum vm_cpu_mode 196600f3efe1SJohn Baldwin vmx_cpu_mode(void) 196700f3efe1SJohn Baldwin { 1968b301b9e2SNeel Natu uint32_t csar; 196900f3efe1SJohn Baldwin 1970b301b9e2SNeel Natu if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LMA) { 1971b301b9e2SNeel Natu csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS); 1972b301b9e2SNeel Natu if (csar & 0x2000) 1973b301b9e2SNeel Natu return (CPU_MODE_64BIT); /* CS.L = 1 */ 197400f3efe1SJohn Baldwin else 197500f3efe1SJohn Baldwin return (CPU_MODE_COMPATIBILITY); 1976b301b9e2SNeel Natu } else if (vmcs_read(VMCS_GUEST_CR0) & CR0_PE) { 1977b301b9e2SNeel Natu return (CPU_MODE_PROTECTED); 1978b301b9e2SNeel Natu } else { 1979b301b9e2SNeel Natu return (CPU_MODE_REAL); 1980b301b9e2SNeel Natu } 198100f3efe1SJohn Baldwin } 198200f3efe1SJohn Baldwin 1983e813a873SNeel Natu static enum vm_paging_mode 198400f3efe1SJohn Baldwin vmx_paging_mode(void) 198500f3efe1SJohn Baldwin { 1986f3eb12e4SKonstantin Belousov uint64_t cr4; 198700f3efe1SJohn Baldwin 198800f3efe1SJohn Baldwin if (!(vmcs_read(VMCS_GUEST_CR0) & CR0_PG)) 198900f3efe1SJohn Baldwin return (PAGING_MODE_FLAT); 1990f3eb12e4SKonstantin Belousov cr4 = vmcs_read(VMCS_GUEST_CR4); 1991f3eb12e4SKonstantin Belousov if (!(cr4 & CR4_PAE)) 199200f3efe1SJohn Baldwin return (PAGING_MODE_32); 1993f3eb12e4SKonstantin Belousov if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LME) { 1994f3eb12e4SKonstantin Belousov if (!(cr4 & CR4_LA57)) 199500f3efe1SJohn Baldwin return (PAGING_MODE_64); 1996f3eb12e4SKonstantin Belousov return (PAGING_MODE_64_LA57); 1997f3eb12e4SKonstantin Belousov } else 199800f3efe1SJohn Baldwin return (PAGING_MODE_PAE); 199900f3efe1SJohn Baldwin } 200000f3efe1SJohn Baldwin 2001d17b5104SNeel Natu static uint64_t 2002869c8d19SJohn Baldwin inout_str_index(struct vmx_vcpu *vcpu, int in) 2003d17b5104SNeel Natu { 2004d17b5104SNeel Natu uint64_t val; 20055c272efaSRobert Wing int error __diagused; 2006d17b5104SNeel Natu enum vm_reg_name reg; 2007d17b5104SNeel Natu 2008d17b5104SNeel Natu reg = in ? VM_REG_GUEST_RDI : VM_REG_GUEST_RSI; 2009869c8d19SJohn Baldwin error = vmx_getreg(vcpu, reg, &val); 2010d17b5104SNeel Natu KASSERT(error == 0, ("%s: vmx_getreg error %d", __func__, error)); 2011d17b5104SNeel Natu return (val); 2012d17b5104SNeel Natu } 2013d17b5104SNeel Natu 2014d17b5104SNeel Natu static uint64_t 2015869c8d19SJohn Baldwin inout_str_count(struct vmx_vcpu *vcpu, int rep) 2016d17b5104SNeel Natu { 2017d17b5104SNeel Natu uint64_t val; 20185c272efaSRobert Wing int error __diagused; 2019d17b5104SNeel Natu 2020d17b5104SNeel Natu if (rep) { 2021869c8d19SJohn Baldwin error = vmx_getreg(vcpu, VM_REG_GUEST_RCX, &val); 2022d17b5104SNeel Natu KASSERT(!error, ("%s: vmx_getreg error %d", __func__, error)); 2023d17b5104SNeel Natu } else { 2024d17b5104SNeel Natu val = 1; 2025d17b5104SNeel Natu } 2026d17b5104SNeel Natu return (val); 2027d17b5104SNeel Natu } 2028d17b5104SNeel Natu 2029d17b5104SNeel Natu static int 2030d17b5104SNeel Natu inout_str_addrsize(uint32_t inst_info) 2031d17b5104SNeel Natu { 2032d17b5104SNeel Natu uint32_t size; 2033d17b5104SNeel Natu 2034d17b5104SNeel Natu size = (inst_info >> 7) & 0x7; 2035d17b5104SNeel Natu switch (size) { 2036d17b5104SNeel Natu case 0: 2037d17b5104SNeel Natu return (2); /* 16 bit */ 2038d17b5104SNeel Natu case 1: 2039d17b5104SNeel Natu return (4); /* 32 bit */ 2040d17b5104SNeel Natu case 2: 2041d17b5104SNeel Natu return (8); /* 64 bit */ 2042d17b5104SNeel Natu default: 2043d17b5104SNeel Natu panic("%s: invalid size encoding %d", __func__, size); 2044d17b5104SNeel Natu } 2045d17b5104SNeel Natu } 2046d17b5104SNeel Natu 2047d17b5104SNeel Natu static void 2048869c8d19SJohn Baldwin inout_str_seginfo(struct vmx_vcpu *vcpu, uint32_t inst_info, int in, 2049869c8d19SJohn Baldwin struct vm_inout_str *vis) 2050d17b5104SNeel Natu { 20515c272efaSRobert Wing int error __diagused, s; 2052d17b5104SNeel Natu 2053d17b5104SNeel Natu if (in) { 2054d17b5104SNeel Natu vis->seg_name = VM_REG_GUEST_ES; 2055d17b5104SNeel Natu } else { 2056d17b5104SNeel Natu s = (inst_info >> 15) & 0x7; 2057d17b5104SNeel Natu vis->seg_name = vm_segment_name(s); 2058d17b5104SNeel Natu } 2059d17b5104SNeel Natu 2060869c8d19SJohn Baldwin error = vmx_getdesc(vcpu, vis->seg_name, &vis->seg_desc); 2061d17b5104SNeel Natu KASSERT(error == 0, ("%s: vmx_getdesc error %d", __func__, error)); 2062d17b5104SNeel Natu } 2063d17b5104SNeel Natu 2064e4c8a13dSNeel Natu static void 2065e813a873SNeel Natu vmx_paging_info(struct vm_guest_paging *paging) 2066e813a873SNeel Natu { 2067e813a873SNeel Natu paging->cr3 = vmcs_guest_cr3(); 2068e813a873SNeel Natu paging->cpl = vmx_cpl(); 2069e813a873SNeel Natu paging->cpu_mode = vmx_cpu_mode(); 2070e813a873SNeel Natu paging->paging_mode = vmx_paging_mode(); 2071e813a873SNeel Natu } 2072e813a873SNeel Natu 2073e813a873SNeel Natu static void 2074e4c8a13dSNeel Natu vmexit_inst_emul(struct vm_exit *vmexit, uint64_t gpa, uint64_t gla) 2075e4c8a13dSNeel Natu { 2076f7a9f178SNeel Natu struct vm_guest_paging *paging; 2077f7a9f178SNeel Natu uint32_t csar; 2078f7a9f178SNeel Natu 2079f7a9f178SNeel Natu paging = &vmexit->u.inst_emul.paging; 2080f7a9f178SNeel Natu 2081e4c8a13dSNeel Natu vmexit->exitcode = VM_EXITCODE_INST_EMUL; 20821c73ea3eSNeel Natu vmexit->inst_length = 0; 2083e4c8a13dSNeel Natu vmexit->u.inst_emul.gpa = gpa; 2084e4c8a13dSNeel Natu vmexit->u.inst_emul.gla = gla; 2085f7a9f178SNeel Natu vmx_paging_info(paging); 2086f7a9f178SNeel Natu switch (paging->cpu_mode) { 2087e4f605eeSTycho Nightingale case CPU_MODE_REAL: 2088e4f605eeSTycho Nightingale vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE); 2089e4f605eeSTycho Nightingale vmexit->u.inst_emul.cs_d = 0; 2090e4f605eeSTycho Nightingale break; 2091f7a9f178SNeel Natu case CPU_MODE_PROTECTED: 2092f7a9f178SNeel Natu case CPU_MODE_COMPATIBILITY: 2093e4f605eeSTycho Nightingale vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE); 2094f7a9f178SNeel Natu csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS); 2095f7a9f178SNeel Natu vmexit->u.inst_emul.cs_d = SEG_DESC_DEF32(csar); 2096f7a9f178SNeel Natu break; 2097f7a9f178SNeel Natu default: 2098e4f605eeSTycho Nightingale vmexit->u.inst_emul.cs_base = 0; 2099f7a9f178SNeel Natu vmexit->u.inst_emul.cs_d = 0; 2100f7a9f178SNeel Natu break; 2101f7a9f178SNeel Natu } 2102c2a875f9SNeel Natu vie_init(&vmexit->u.inst_emul.vie, NULL, 0); 2103e4c8a13dSNeel Natu } 2104e4c8a13dSNeel Natu 2105366f6083SPeter Grehan static int 2106318224bbSNeel Natu ept_fault_type(uint64_t ept_qual) 2107a2da7af6SNeel Natu { 2108318224bbSNeel Natu int fault_type; 2109a2da7af6SNeel Natu 2110318224bbSNeel Natu if (ept_qual & EPT_VIOLATION_DATA_WRITE) 2111318224bbSNeel Natu fault_type = VM_PROT_WRITE; 2112318224bbSNeel Natu else if (ept_qual & EPT_VIOLATION_INST_FETCH) 2113318224bbSNeel Natu fault_type = VM_PROT_EXECUTE; 2114318224bbSNeel Natu else 2115318224bbSNeel Natu fault_type= VM_PROT_READ; 2116318224bbSNeel Natu 2117318224bbSNeel Natu return (fault_type); 2118318224bbSNeel Natu } 2119318224bbSNeel Natu 2120490d56c5SEd Maste static bool 2121318224bbSNeel Natu ept_emulation_fault(uint64_t ept_qual) 2122318224bbSNeel Natu { 2123318224bbSNeel Natu int read, write; 2124318224bbSNeel Natu 2125318224bbSNeel Natu /* EPT fault on an instruction fetch doesn't make sense here */ 2126a2da7af6SNeel Natu if (ept_qual & EPT_VIOLATION_INST_FETCH) 2127490d56c5SEd Maste return (false); 2128a2da7af6SNeel Natu 2129318224bbSNeel Natu /* EPT fault must be a read fault or a write fault */ 2130a2da7af6SNeel Natu read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0; 2131a2da7af6SNeel Natu write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0; 21323b2b0011SPeter Grehan if ((read | write) == 0) 2133490d56c5SEd Maste return (false); 2134a2da7af6SNeel Natu 2135a2da7af6SNeel Natu /* 21363b2b0011SPeter Grehan * The EPT violation must have been caused by accessing a 21373b2b0011SPeter Grehan * guest-physical address that is a translation of a guest-linear 21383b2b0011SPeter Grehan * address. 2139a2da7af6SNeel Natu */ 2140a2da7af6SNeel Natu if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 || 2141a2da7af6SNeel Natu (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) { 2142490d56c5SEd Maste return (false); 2143a2da7af6SNeel Natu } 2144a2da7af6SNeel Natu 2145490d56c5SEd Maste return (true); 2146a2da7af6SNeel Natu } 2147a2da7af6SNeel Natu 2148159dd56fSNeel Natu static __inline int 21491aa51504SJohn Baldwin apic_access_virtualization(struct vmx_vcpu *vcpu) 2150159dd56fSNeel Natu { 2151159dd56fSNeel Natu uint32_t proc_ctls2; 2152159dd56fSNeel Natu 21531aa51504SJohn Baldwin proc_ctls2 = vcpu->cap.proc_ctls2; 2154159dd56fSNeel Natu return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) ? 1 : 0); 2155159dd56fSNeel Natu } 2156159dd56fSNeel Natu 2157159dd56fSNeel Natu static __inline int 21581aa51504SJohn Baldwin x2apic_virtualization(struct vmx_vcpu *vcpu) 2159159dd56fSNeel Natu { 2160159dd56fSNeel Natu uint32_t proc_ctls2; 2161159dd56fSNeel Natu 21621aa51504SJohn Baldwin proc_ctls2 = vcpu->cap.proc_ctls2; 2163159dd56fSNeel Natu return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_X2APIC_MODE) ? 1 : 0); 2164159dd56fSNeel Natu } 2165159dd56fSNeel Natu 2166a2da7af6SNeel Natu static int 21671aa51504SJohn Baldwin vmx_handle_apic_write(struct vmx_vcpu *vcpu, struct vlapic *vlapic, 2168159dd56fSNeel Natu uint64_t qual) 216988c4b8d1SNeel Natu { 217088c4b8d1SNeel Natu int error, handled, offset; 2171159dd56fSNeel Natu uint32_t *apic_regs, vector; 217288c4b8d1SNeel Natu bool retu; 217388c4b8d1SNeel Natu 2174a0efd3fbSJohn Baldwin handled = HANDLED; 217588c4b8d1SNeel Natu offset = APIC_WRITE_OFFSET(qual); 2176159dd56fSNeel Natu 21771aa51504SJohn Baldwin if (!apic_access_virtualization(vcpu)) { 2178159dd56fSNeel Natu /* 2179159dd56fSNeel Natu * In general there should not be any APIC write VM-exits 2180159dd56fSNeel Natu * unless APIC-access virtualization is enabled. 2181159dd56fSNeel Natu * 2182159dd56fSNeel Natu * However self-IPI virtualization can legitimately trigger 2183159dd56fSNeel Natu * an APIC-write VM-exit so treat it specially. 2184159dd56fSNeel Natu */ 21851aa51504SJohn Baldwin if (x2apic_virtualization(vcpu) && 2186159dd56fSNeel Natu offset == APIC_OFFSET_SELF_IPI) { 2187159dd56fSNeel Natu apic_regs = (uint32_t *)(vlapic->apic_page); 2188159dd56fSNeel Natu vector = apic_regs[APIC_OFFSET_SELF_IPI / 4]; 2189159dd56fSNeel Natu vlapic_self_ipi_handler(vlapic, vector); 2190159dd56fSNeel Natu return (HANDLED); 2191159dd56fSNeel Natu } else 2192159dd56fSNeel Natu return (UNHANDLED); 2193159dd56fSNeel Natu } 2194159dd56fSNeel Natu 219588c4b8d1SNeel Natu switch (offset) { 219688c4b8d1SNeel Natu case APIC_OFFSET_ID: 219788c4b8d1SNeel Natu vlapic_id_write_handler(vlapic); 219888c4b8d1SNeel Natu break; 219988c4b8d1SNeel Natu case APIC_OFFSET_LDR: 220088c4b8d1SNeel Natu vlapic_ldr_write_handler(vlapic); 220188c4b8d1SNeel Natu break; 220288c4b8d1SNeel Natu case APIC_OFFSET_DFR: 220388c4b8d1SNeel Natu vlapic_dfr_write_handler(vlapic); 220488c4b8d1SNeel Natu break; 220588c4b8d1SNeel Natu case APIC_OFFSET_SVR: 220688c4b8d1SNeel Natu vlapic_svr_write_handler(vlapic); 220788c4b8d1SNeel Natu break; 220888c4b8d1SNeel Natu case APIC_OFFSET_ESR: 220988c4b8d1SNeel Natu vlapic_esr_write_handler(vlapic); 221088c4b8d1SNeel Natu break; 221188c4b8d1SNeel Natu case APIC_OFFSET_ICR_LOW: 221288c4b8d1SNeel Natu retu = false; 221388c4b8d1SNeel Natu error = vlapic_icrlo_write_handler(vlapic, &retu); 221488c4b8d1SNeel Natu if (error != 0 || retu) 2215a0efd3fbSJohn Baldwin handled = UNHANDLED; 221688c4b8d1SNeel Natu break; 221788c4b8d1SNeel Natu case APIC_OFFSET_CMCI_LVT: 221888c4b8d1SNeel Natu case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT: 221988c4b8d1SNeel Natu vlapic_lvt_write_handler(vlapic, offset); 222088c4b8d1SNeel Natu break; 222188c4b8d1SNeel Natu case APIC_OFFSET_TIMER_ICR: 222288c4b8d1SNeel Natu vlapic_icrtmr_write_handler(vlapic); 222388c4b8d1SNeel Natu break; 222488c4b8d1SNeel Natu case APIC_OFFSET_TIMER_DCR: 222588c4b8d1SNeel Natu vlapic_dcr_write_handler(vlapic); 222688c4b8d1SNeel Natu break; 222788c4b8d1SNeel Natu default: 2228a0efd3fbSJohn Baldwin handled = UNHANDLED; 222988c4b8d1SNeel Natu break; 223088c4b8d1SNeel Natu } 223188c4b8d1SNeel Natu return (handled); 223288c4b8d1SNeel Natu } 223388c4b8d1SNeel Natu 223488c4b8d1SNeel Natu static bool 22351aa51504SJohn Baldwin apic_access_fault(struct vmx_vcpu *vcpu, uint64_t gpa) 223688c4b8d1SNeel Natu { 223788c4b8d1SNeel Natu 22381aa51504SJohn Baldwin if (apic_access_virtualization(vcpu) && 223988c4b8d1SNeel Natu (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE)) 224088c4b8d1SNeel Natu return (true); 224188c4b8d1SNeel Natu else 224288c4b8d1SNeel Natu return (false); 224388c4b8d1SNeel Natu } 224488c4b8d1SNeel Natu 224588c4b8d1SNeel Natu static int 22461aa51504SJohn Baldwin vmx_handle_apic_access(struct vmx_vcpu *vcpu, struct vm_exit *vmexit) 224788c4b8d1SNeel Natu { 224888c4b8d1SNeel Natu uint64_t qual; 224988c4b8d1SNeel Natu int access_type, offset, allowed; 225088c4b8d1SNeel Natu 22511aa51504SJohn Baldwin if (!apic_access_virtualization(vcpu)) 225288c4b8d1SNeel Natu return (UNHANDLED); 225388c4b8d1SNeel Natu 225488c4b8d1SNeel Natu qual = vmexit->u.vmx.exit_qualification; 225588c4b8d1SNeel Natu access_type = APIC_ACCESS_TYPE(qual); 225688c4b8d1SNeel Natu offset = APIC_ACCESS_OFFSET(qual); 225788c4b8d1SNeel Natu 225888c4b8d1SNeel Natu allowed = 0; 225988c4b8d1SNeel Natu if (access_type == 0) { 226088c4b8d1SNeel Natu /* 226188c4b8d1SNeel Natu * Read data access to the following registers is expected. 226288c4b8d1SNeel Natu */ 226388c4b8d1SNeel Natu switch (offset) { 226488c4b8d1SNeel Natu case APIC_OFFSET_APR: 226588c4b8d1SNeel Natu case APIC_OFFSET_PPR: 226688c4b8d1SNeel Natu case APIC_OFFSET_RRR: 226788c4b8d1SNeel Natu case APIC_OFFSET_CMCI_LVT: 226888c4b8d1SNeel Natu case APIC_OFFSET_TIMER_CCR: 226988c4b8d1SNeel Natu allowed = 1; 227088c4b8d1SNeel Natu break; 227188c4b8d1SNeel Natu default: 227288c4b8d1SNeel Natu break; 227388c4b8d1SNeel Natu } 227488c4b8d1SNeel Natu } else if (access_type == 1) { 227588c4b8d1SNeel Natu /* 227688c4b8d1SNeel Natu * Write data access to the following registers is expected. 227788c4b8d1SNeel Natu */ 227888c4b8d1SNeel Natu switch (offset) { 227988c4b8d1SNeel Natu case APIC_OFFSET_VER: 228088c4b8d1SNeel Natu case APIC_OFFSET_APR: 228188c4b8d1SNeel Natu case APIC_OFFSET_PPR: 228288c4b8d1SNeel Natu case APIC_OFFSET_RRR: 228388c4b8d1SNeel Natu case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7: 228488c4b8d1SNeel Natu case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7: 228588c4b8d1SNeel Natu case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7: 228688c4b8d1SNeel Natu case APIC_OFFSET_CMCI_LVT: 228788c4b8d1SNeel Natu case APIC_OFFSET_TIMER_CCR: 228888c4b8d1SNeel Natu allowed = 1; 228988c4b8d1SNeel Natu break; 229088c4b8d1SNeel Natu default: 229188c4b8d1SNeel Natu break; 229288c4b8d1SNeel Natu } 229388c4b8d1SNeel Natu } 229488c4b8d1SNeel Natu 229588c4b8d1SNeel Natu if (allowed) { 2296e4c8a13dSNeel Natu vmexit_inst_emul(vmexit, DEFAULT_APIC_BASE + offset, 2297e4c8a13dSNeel Natu VIE_INVALID_GLA); 229888c4b8d1SNeel Natu } 229988c4b8d1SNeel Natu 230088c4b8d1SNeel Natu /* 230188c4b8d1SNeel Natu * Regardless of whether the APIC-access is allowed this handler 230288c4b8d1SNeel Natu * always returns UNHANDLED: 230388c4b8d1SNeel Natu * - if the access is allowed then it is handled by emulating the 230488c4b8d1SNeel Natu * instruction that caused the VM-exit (outside the critical section) 230588c4b8d1SNeel Natu * - if the access is not allowed then it will be converted to an 230688c4b8d1SNeel Natu * exitcode of VM_EXITCODE_VMX and will be dealt with in userland. 230788c4b8d1SNeel Natu */ 230888c4b8d1SNeel Natu return (UNHANDLED); 230988c4b8d1SNeel Natu } 231088c4b8d1SNeel Natu 23113d5444c8SNeel Natu static enum task_switch_reason 23123d5444c8SNeel Natu vmx_task_switch_reason(uint64_t qual) 23133d5444c8SNeel Natu { 23143d5444c8SNeel Natu int reason; 23153d5444c8SNeel Natu 23163d5444c8SNeel Natu reason = (qual >> 30) & 0x3; 23173d5444c8SNeel Natu switch (reason) { 23183d5444c8SNeel Natu case 0: 23193d5444c8SNeel Natu return (TSR_CALL); 23203d5444c8SNeel Natu case 1: 23213d5444c8SNeel Natu return (TSR_IRET); 23223d5444c8SNeel Natu case 2: 23233d5444c8SNeel Natu return (TSR_JMP); 23243d5444c8SNeel Natu case 3: 23253d5444c8SNeel Natu return (TSR_IDT_GATE); 23263d5444c8SNeel Natu default: 23273d5444c8SNeel Natu panic("%s: invalid reason %d", __func__, reason); 23283d5444c8SNeel Natu } 23293d5444c8SNeel Natu } 23303d5444c8SNeel Natu 233188c4b8d1SNeel Natu static int 233280cb5d84SJohn Baldwin emulate_wrmsr(struct vmx_vcpu *vcpu, u_int num, uint64_t val, bool *retu) 2333c3498942SNeel Natu { 2334c3498942SNeel Natu int error; 2335c3498942SNeel Natu 2336c3498942SNeel Natu if (lapic_msr(num)) 233780cb5d84SJohn Baldwin error = lapic_wrmsr(vcpu->vcpu, num, val, retu); 2338c3498942SNeel Natu else 233980cb5d84SJohn Baldwin error = vmx_wrmsr(vcpu, num, val, retu); 2340c3498942SNeel Natu 2341c3498942SNeel Natu return (error); 2342c3498942SNeel Natu } 2343c3498942SNeel Natu 2344c3498942SNeel Natu static int 234580cb5d84SJohn Baldwin emulate_rdmsr(struct vmx_vcpu *vcpu, u_int num, bool *retu) 2346c3498942SNeel Natu { 2347c3498942SNeel Natu struct vmxctx *vmxctx; 2348c3498942SNeel Natu uint64_t result; 2349c3498942SNeel Natu uint32_t eax, edx; 2350c3498942SNeel Natu int error; 2351c3498942SNeel Natu 2352c3498942SNeel Natu if (lapic_msr(num)) 235380cb5d84SJohn Baldwin error = lapic_rdmsr(vcpu->vcpu, num, &result, retu); 2354c3498942SNeel Natu else 235580cb5d84SJohn Baldwin error = vmx_rdmsr(vcpu, num, &result, retu); 2356c3498942SNeel Natu 2357c3498942SNeel Natu if (error == 0) { 2358c3498942SNeel Natu eax = result; 23591aa51504SJohn Baldwin vmxctx = &vcpu->ctx; 2360c3498942SNeel Natu error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RAX, eax); 2361c3498942SNeel Natu KASSERT(error == 0, ("vmxctx_setreg(rax) error %d", error)); 2362c3498942SNeel Natu 2363c3498942SNeel Natu edx = result >> 32; 2364c3498942SNeel Natu error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RDX, edx); 2365c3498942SNeel Natu KASSERT(error == 0, ("vmxctx_setreg(rdx) error %d", error)); 2366c3498942SNeel Natu } 2367c3498942SNeel Natu 2368c3498942SNeel Natu return (error); 2369c3498942SNeel Natu } 2370c3498942SNeel Natu 2371c3498942SNeel Natu static int 23721aa51504SJohn Baldwin vmx_exit_process(struct vmx *vmx, struct vmx_vcpu *vcpu, struct vm_exit *vmexit) 2373366f6083SPeter Grehan { 2374c9c75df4SNeel Natu int error, errcode, errcode_valid, handled, in; 2375366f6083SPeter Grehan struct vmxctx *vmxctx; 237688c4b8d1SNeel Natu struct vlapic *vlapic; 2377d17b5104SNeel Natu struct vm_inout_str *vis; 23783d5444c8SNeel Natu struct vm_task_switch *ts; 2379d17b5104SNeel Natu uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, inst_info; 2380b0538143SNeel Natu uint32_t intr_type, intr_vec, reason; 2381091d4532SNeel Natu uint64_t exitintinfo, qual, gpa; 23822ee1a18dSDmitry Chagin #ifdef KDTRACE_HOOKS 23831aa51504SJohn Baldwin int vcpuid; 23842ee1a18dSDmitry Chagin #endif 2385becd9849SNeel Natu bool retu; 2386366f6083SPeter Grehan 2387160471d2SNeel Natu CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0); 2388c308b23bSNeel Natu CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0); 2389160471d2SNeel Natu 2390a0efd3fbSJohn Baldwin handled = UNHANDLED; 23911aa51504SJohn Baldwin vmxctx = &vcpu->ctx; 23922ee1a18dSDmitry Chagin #ifdef KDTRACE_HOOKS 23931aa51504SJohn Baldwin vcpuid = vcpu->vcpuid; 23942ee1a18dSDmitry Chagin #endif 23950492757cSNeel Natu 2396366f6083SPeter Grehan qual = vmexit->u.vmx.exit_qualification; 2397318224bbSNeel Natu reason = vmexit->u.vmx.exit_reason; 2398366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_BOGUS; 2399366f6083SPeter Grehan 24003dc3d32aSJohn Baldwin vmm_stat_incr(vcpu->vcpu, VMEXIT_COUNT, 1); 24011aa51504SJohn Baldwin SDT_PROBE3(vmm, vmx, exit, entry, vmx, vcpuid, vmexit); 240261592433SNeel Natu 2403318224bbSNeel Natu /* 2404b0538143SNeel Natu * VM-entry failures during or after loading guest state. 2405b0538143SNeel Natu * 2406b0538143SNeel Natu * These VM-exits are uncommon but must be handled specially 2407b0538143SNeel Natu * as most VM-exit fields are not populated as usual. 2408b0538143SNeel Natu */ 2409b0538143SNeel Natu if (__predict_false(reason == EXIT_REASON_MCE_DURING_ENTRY)) { 241057e0119eSJohn Baldwin VMX_CTR0(vcpu, "Handling MCE during VM-entry"); 2411b0538143SNeel Natu __asm __volatile("int $18"); 2412b0538143SNeel Natu return (1); 2413b0538143SNeel Natu } 2414b0538143SNeel Natu 2415b0538143SNeel Natu /* 24163d5444c8SNeel Natu * VM exits that can be triggered during event delivery need to 24173d5444c8SNeel Natu * be handled specially by re-injecting the event if the IDT 24183d5444c8SNeel Natu * vectoring information field's valid bit is set. 2419318224bbSNeel Natu * 2420318224bbSNeel Natu * See "Information for VM Exits During Event Delivery" in Intel SDM 2421318224bbSNeel Natu * for details. 2422318224bbSNeel Natu */ 2423318224bbSNeel Natu idtvec_info = vmcs_idt_vectoring_info(); 2424318224bbSNeel Natu if (idtvec_info & VMCS_IDT_VEC_VALID) { 2425318224bbSNeel Natu idtvec_info &= ~(1 << 12); /* clear undefined bit */ 2426091d4532SNeel Natu exitintinfo = idtvec_info; 2427318224bbSNeel Natu if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) { 2428318224bbSNeel Natu idtvec_err = vmcs_idt_vectoring_err(); 2429091d4532SNeel Natu exitintinfo |= (uint64_t)idtvec_err << 32; 2430318224bbSNeel Natu } 243180cb5d84SJohn Baldwin error = vm_exit_intinfo(vcpu->vcpu, exitintinfo); 2432091d4532SNeel Natu KASSERT(error == 0, ("%s: vm_set_intinfo error %d", 2433091d4532SNeel Natu __func__, error)); 2434091d4532SNeel Natu 2435160471d2SNeel Natu /* 2436160471d2SNeel Natu * If 'virtual NMIs' are being used and the VM-exit 2437160471d2SNeel Natu * happened while injecting an NMI during the previous 2438091d4532SNeel Natu * VM-entry, then clear "blocking by NMI" in the 2439091d4532SNeel Natu * Guest Interruptibility-State so the NMI can be 2440091d4532SNeel Natu * reinjected on the subsequent VM-entry. 2441091d4532SNeel Natu * 2442091d4532SNeel Natu * However, if the NMI was being delivered through a task 2443091d4532SNeel Natu * gate, then the new task must start execution with NMIs 2444091d4532SNeel Natu * blocked so don't clear NMI blocking in this case. 2445160471d2SNeel Natu */ 2446091d4532SNeel Natu intr_type = idtvec_info & VMCS_INTR_T_MASK; 2447091d4532SNeel Natu if (intr_type == VMCS_INTR_T_NMI) { 2448091d4532SNeel Natu if (reason != EXIT_REASON_TASK_SWITCH) 2449869c8d19SJohn Baldwin vmx_clear_nmi_blocking(vcpu); 2450091d4532SNeel Natu else 2451869c8d19SJohn Baldwin vmx_assert_nmi_blocking(vcpu); 2452160471d2SNeel Natu } 2453091d4532SNeel Natu 2454091d4532SNeel Natu /* 2455091d4532SNeel Natu * Update VM-entry instruction length if the event being 2456091d4532SNeel Natu * delivered was a software interrupt or software exception. 2457091d4532SNeel Natu */ 2458091d4532SNeel Natu if (intr_type == VMCS_INTR_T_SWINTR || 2459091d4532SNeel Natu intr_type == VMCS_INTR_T_PRIV_SWEXCEPTION || 2460091d4532SNeel Natu intr_type == VMCS_INTR_T_SWEXCEPTION) { 24613de83862SNeel Natu vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length); 2462318224bbSNeel Natu } 2463318224bbSNeel Natu } 2464318224bbSNeel Natu 2465318224bbSNeel Natu switch (reason) { 24663d5444c8SNeel Natu case EXIT_REASON_TASK_SWITCH: 24673d5444c8SNeel Natu ts = &vmexit->u.task_switch; 24683d5444c8SNeel Natu ts->tsssel = qual & 0xffff; 24693d5444c8SNeel Natu ts->reason = vmx_task_switch_reason(qual); 24703d5444c8SNeel Natu ts->ext = 0; 24713d5444c8SNeel Natu ts->errcode_valid = 0; 24723d5444c8SNeel Natu vmx_paging_info(&ts->paging); 24733d5444c8SNeel Natu /* 24743d5444c8SNeel Natu * If the task switch was due to a CALL, JMP, IRET, software 24753d5444c8SNeel Natu * interrupt (INT n) or software exception (INT3, INTO), 24763d5444c8SNeel Natu * then the saved %rip references the instruction that caused 24773d5444c8SNeel Natu * the task switch. The instruction length field in the VMCS 24783d5444c8SNeel Natu * is valid in this case. 24793d5444c8SNeel Natu * 24803d5444c8SNeel Natu * In all other cases (e.g., NMI, hardware exception) the 24813d5444c8SNeel Natu * saved %rip is one that would have been saved in the old TSS 24823d5444c8SNeel Natu * had the task switch completed normally so the instruction 24833d5444c8SNeel Natu * length field is not needed in this case and is explicitly 24843d5444c8SNeel Natu * set to 0. 24853d5444c8SNeel Natu */ 24863d5444c8SNeel Natu if (ts->reason == TSR_IDT_GATE) { 24873d5444c8SNeel Natu KASSERT(idtvec_info & VMCS_IDT_VEC_VALID, 2488091d4532SNeel Natu ("invalid idtvec_info %#x for IDT task switch", 24893d5444c8SNeel Natu idtvec_info)); 24903d5444c8SNeel Natu intr_type = idtvec_info & VMCS_INTR_T_MASK; 24913d5444c8SNeel Natu if (intr_type != VMCS_INTR_T_SWINTR && 24923d5444c8SNeel Natu intr_type != VMCS_INTR_T_SWEXCEPTION && 24933d5444c8SNeel Natu intr_type != VMCS_INTR_T_PRIV_SWEXCEPTION) { 24943d5444c8SNeel Natu /* Task switch triggered by external event */ 24953d5444c8SNeel Natu ts->ext = 1; 24963d5444c8SNeel Natu vmexit->inst_length = 0; 24973d5444c8SNeel Natu if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) { 24983d5444c8SNeel Natu ts->errcode_valid = 1; 24993d5444c8SNeel Natu ts->errcode = vmcs_idt_vectoring_err(); 25003d5444c8SNeel Natu } 25013d5444c8SNeel Natu } 25023d5444c8SNeel Natu } 25033d5444c8SNeel Natu vmexit->exitcode = VM_EXITCODE_TASK_SWITCH; 25041aa51504SJohn Baldwin SDT_PROBE4(vmm, vmx, exit, taskswitch, vmx, vcpuid, vmexit, ts); 250557e0119eSJohn Baldwin VMX_CTR4(vcpu, "task switch reason %d, tss 0x%04x, " 25063d5444c8SNeel Natu "%s errcode 0x%016lx", ts->reason, ts->tsssel, 25073d5444c8SNeel Natu ts->ext ? "external" : "internal", 25083d5444c8SNeel Natu ((uint64_t)ts->errcode << 32) | ts->errcode_valid); 25093d5444c8SNeel Natu break; 2510366f6083SPeter Grehan case EXIT_REASON_CR_ACCESS: 25113dc3d32aSJohn Baldwin vmm_stat_incr(vcpu->vcpu, VMEXIT_CR_ACCESS, 1); 25121aa51504SJohn Baldwin SDT_PROBE4(vmm, vmx, exit, craccess, vmx, vcpuid, vmexit, qual); 2513594db002STycho Nightingale switch (qual & 0xf) { 2514594db002STycho Nightingale case 0: 25151aa51504SJohn Baldwin handled = vmx_emulate_cr0_access(vcpu, qual); 2516594db002STycho Nightingale break; 2517594db002STycho Nightingale case 4: 25181aa51504SJohn Baldwin handled = vmx_emulate_cr4_access(vcpu, qual); 2519594db002STycho Nightingale break; 2520594db002STycho Nightingale case 8: 2521594db002STycho Nightingale handled = vmx_emulate_cr8_access(vmx, vcpu, qual); 2522594db002STycho Nightingale break; 2523594db002STycho Nightingale } 2524366f6083SPeter Grehan break; 2525366f6083SPeter Grehan case EXIT_REASON_RDMSR: 25263dc3d32aSJohn Baldwin vmm_stat_incr(vcpu->vcpu, VMEXIT_RDMSR, 1); 2527becd9849SNeel Natu retu = false; 2528366f6083SPeter Grehan ecx = vmxctx->guest_rcx; 252957e0119eSJohn Baldwin VMX_CTR1(vcpu, "rdmsr 0x%08x", ecx); 25301aa51504SJohn Baldwin SDT_PROBE4(vmm, vmx, exit, rdmsr, vmx, vcpuid, vmexit, ecx); 253180cb5d84SJohn Baldwin error = emulate_rdmsr(vcpu, ecx, &retu); 2532b42206f3SNeel Natu if (error) { 2533366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_RDMSR; 2534366f6083SPeter Grehan vmexit->u.msr.code = ecx; 2535becd9849SNeel Natu } else if (!retu) { 2536a0efd3fbSJohn Baldwin handled = HANDLED; 2537becd9849SNeel Natu } else { 2538becd9849SNeel Natu /* Return to userspace with a valid exitcode */ 2539becd9849SNeel Natu KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS, 2540c3498942SNeel Natu ("emulate_rdmsr retu with bogus exitcode")); 2541becd9849SNeel Natu } 2542366f6083SPeter Grehan break; 2543366f6083SPeter Grehan case EXIT_REASON_WRMSR: 25443dc3d32aSJohn Baldwin vmm_stat_incr(vcpu->vcpu, VMEXIT_WRMSR, 1); 2545becd9849SNeel Natu retu = false; 2546366f6083SPeter Grehan eax = vmxctx->guest_rax; 2547366f6083SPeter Grehan ecx = vmxctx->guest_rcx; 2548366f6083SPeter Grehan edx = vmxctx->guest_rdx; 254957e0119eSJohn Baldwin VMX_CTR2(vcpu, "wrmsr 0x%08x value 0x%016lx", 25502cb97c9dSNeel Natu ecx, (uint64_t)edx << 32 | eax); 25511aa51504SJohn Baldwin SDT_PROBE5(vmm, vmx, exit, wrmsr, vmx, vmexit, vcpuid, ecx, 25526ac73777STycho Nightingale (uint64_t)edx << 32 | eax); 255380cb5d84SJohn Baldwin error = emulate_wrmsr(vcpu, ecx, (uint64_t)edx << 32 | eax, 255480cb5d84SJohn Baldwin &retu); 2555b42206f3SNeel Natu if (error) { 2556366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_WRMSR; 2557366f6083SPeter Grehan vmexit->u.msr.code = ecx; 2558366f6083SPeter Grehan vmexit->u.msr.wval = (uint64_t)edx << 32 | eax; 2559becd9849SNeel Natu } else if (!retu) { 2560a0efd3fbSJohn Baldwin handled = HANDLED; 2561becd9849SNeel Natu } else { 2562becd9849SNeel Natu /* Return to userspace with a valid exitcode */ 2563becd9849SNeel Natu KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS, 2564becd9849SNeel Natu ("emulate_wrmsr retu with bogus exitcode")); 2565becd9849SNeel Natu } 2566366f6083SPeter Grehan break; 2567366f6083SPeter Grehan case EXIT_REASON_HLT: 25683dc3d32aSJohn Baldwin vmm_stat_incr(vcpu->vcpu, VMEXIT_HLT, 1); 25691aa51504SJohn Baldwin SDT_PROBE3(vmm, vmx, exit, halt, vmx, vcpuid, vmexit); 2570366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_HLT; 25713de83862SNeel Natu vmexit->u.hlt.rflags = vmcs_read(VMCS_GUEST_RFLAGS); 2572490768e2STycho Nightingale if (virtual_interrupt_delivery) 2573490768e2STycho Nightingale vmexit->u.hlt.intr_status = 2574490768e2STycho Nightingale vmcs_read(VMCS_GUEST_INTR_STATUS); 2575490768e2STycho Nightingale else 2576490768e2STycho Nightingale vmexit->u.hlt.intr_status = 0; 2577366f6083SPeter Grehan break; 2578366f6083SPeter Grehan case EXIT_REASON_MTF: 25793dc3d32aSJohn Baldwin vmm_stat_incr(vcpu->vcpu, VMEXIT_MTRAP, 1); 25801aa51504SJohn Baldwin SDT_PROBE3(vmm, vmx, exit, mtrap, vmx, vcpuid, vmexit); 2581366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_MTRAP; 2582c9c75df4SNeel Natu vmexit->inst_length = 0; 2583366f6083SPeter Grehan break; 2584366f6083SPeter Grehan case EXIT_REASON_PAUSE: 25853dc3d32aSJohn Baldwin vmm_stat_incr(vcpu->vcpu, VMEXIT_PAUSE, 1); 25861aa51504SJohn Baldwin SDT_PROBE3(vmm, vmx, exit, pause, vmx, vcpuid, vmexit); 2587366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_PAUSE; 2588366f6083SPeter Grehan break; 2589366f6083SPeter Grehan case EXIT_REASON_INTR_WINDOW: 25903dc3d32aSJohn Baldwin vmm_stat_incr(vcpu->vcpu, VMEXIT_INTR_WINDOW, 1); 25911aa51504SJohn Baldwin SDT_PROBE3(vmm, vmx, exit, intrwindow, vmx, vcpuid, vmexit); 2592869c8d19SJohn Baldwin vmx_clear_int_window_exiting(vcpu); 2593b5aaf7b2SNeel Natu return (1); 2594366f6083SPeter Grehan case EXIT_REASON_EXT_INTR: 2595366f6083SPeter Grehan /* 2596366f6083SPeter Grehan * External interrupts serve only to cause VM exits and allow 2597366f6083SPeter Grehan * the host interrupt handler to run. 2598366f6083SPeter Grehan * 2599366f6083SPeter Grehan * If this external interrupt triggers a virtual interrupt 2600366f6083SPeter Grehan * to a VM, then that state will be recorded by the 2601366f6083SPeter Grehan * host interrupt handler in the VM's softc. We will inject 2602366f6083SPeter Grehan * this virtual interrupt during the subsequent VM enter. 2603366f6083SPeter Grehan */ 2604f7d47425SNeel Natu intr_info = vmcs_read(VMCS_EXIT_INTR_INFO); 26056ac73777STycho Nightingale SDT_PROBE4(vmm, vmx, exit, interrupt, 26061aa51504SJohn Baldwin vmx, vcpuid, vmexit, intr_info); 2607722b6744SJohn Baldwin 2608722b6744SJohn Baldwin /* 2609722b6744SJohn Baldwin * XXX: Ignore this exit if VMCS_INTR_VALID is not set. 2610ad3e3687SJohn Baldwin * This appears to be a bug in VMware Fusion? 2611722b6744SJohn Baldwin */ 2612722b6744SJohn Baldwin if (!(intr_info & VMCS_INTR_VALID)) 2613722b6744SJohn Baldwin return (1); 2614160471d2SNeel Natu KASSERT((intr_info & VMCS_INTR_VALID) != 0 && 2615160471d2SNeel Natu (intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_HWINTR, 2616f7d47425SNeel Natu ("VM exit interruption info invalid: %#x", intr_info)); 2617f7d47425SNeel Natu vmx_trigger_hostintr(intr_info & 0xff); 2618366f6083SPeter Grehan 2619366f6083SPeter Grehan /* 2620366f6083SPeter Grehan * This is special. We want to treat this as an 'handled' 2621366f6083SPeter Grehan * VM-exit but not increment the instruction pointer. 2622366f6083SPeter Grehan */ 26233dc3d32aSJohn Baldwin vmm_stat_incr(vcpu->vcpu, VMEXIT_EXTINT, 1); 2624366f6083SPeter Grehan return (1); 2625366f6083SPeter Grehan case EXIT_REASON_NMI_WINDOW: 26261aa51504SJohn Baldwin SDT_PROBE3(vmm, vmx, exit, nmiwindow, vmx, vcpuid, vmexit); 2627366f6083SPeter Grehan /* Exit to allow the pending virtual NMI to be injected */ 262880cb5d84SJohn Baldwin if (vm_nmi_pending(vcpu->vcpu)) 262980cb5d84SJohn Baldwin vmx_inject_nmi(vcpu); 2630869c8d19SJohn Baldwin vmx_clear_nmi_window_exiting(vcpu); 26313dc3d32aSJohn Baldwin vmm_stat_incr(vcpu->vcpu, VMEXIT_NMI_WINDOW, 1); 2632366f6083SPeter Grehan return (1); 2633366f6083SPeter Grehan case EXIT_REASON_INOUT: 26343dc3d32aSJohn Baldwin vmm_stat_incr(vcpu->vcpu, VMEXIT_INOUT, 1); 2635366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_INOUT; 2636366f6083SPeter Grehan vmexit->u.inout.bytes = (qual & 0x7) + 1; 2637d17b5104SNeel Natu vmexit->u.inout.in = in = (qual & 0x8) ? 1 : 0; 2638366f6083SPeter Grehan vmexit->u.inout.string = (qual & 0x10) ? 1 : 0; 2639366f6083SPeter Grehan vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0; 2640366f6083SPeter Grehan vmexit->u.inout.port = (uint16_t)(qual >> 16); 2641366f6083SPeter Grehan vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax); 2642d17b5104SNeel Natu if (vmexit->u.inout.string) { 2643d17b5104SNeel Natu inst_info = vmcs_read(VMCS_EXIT_INSTRUCTION_INFO); 2644d17b5104SNeel Natu vmexit->exitcode = VM_EXITCODE_INOUT_STR; 2645d17b5104SNeel Natu vis = &vmexit->u.inout_str; 2646e813a873SNeel Natu vmx_paging_info(&vis->paging); 2647d17b5104SNeel Natu vis->rflags = vmcs_read(VMCS_GUEST_RFLAGS); 2648d17b5104SNeel Natu vis->cr0 = vmcs_read(VMCS_GUEST_CR0); 2649869c8d19SJohn Baldwin vis->index = inout_str_index(vcpu, in); 2650869c8d19SJohn Baldwin vis->count = inout_str_count(vcpu, vis->inout.rep); 2651d17b5104SNeel Natu vis->addrsize = inout_str_addrsize(inst_info); 2652869c8d19SJohn Baldwin inout_str_seginfo(vcpu, inst_info, in, vis); 2653762fd208STycho Nightingale } 26541aa51504SJohn Baldwin SDT_PROBE3(vmm, vmx, exit, inout, vmx, vcpuid, vmexit); 2655366f6083SPeter Grehan break; 2656366f6083SPeter Grehan case EXIT_REASON_CPUID: 26573dc3d32aSJohn Baldwin vmm_stat_incr(vcpu->vcpu, VMEXIT_CPUID, 1); 26581aa51504SJohn Baldwin SDT_PROBE3(vmm, vmx, exit, cpuid, vmx, vcpuid, vmexit); 265980cb5d84SJohn Baldwin handled = vmx_handle_cpuid(vcpu, vmxctx); 2660366f6083SPeter Grehan break; 2661e5a1d950SNeel Natu case EXIT_REASON_EXCEPTION: 26623dc3d32aSJohn Baldwin vmm_stat_incr(vcpu->vcpu, VMEXIT_EXCEPTION, 1); 2663e5a1d950SNeel Natu intr_info = vmcs_read(VMCS_EXIT_INTR_INFO); 2664e5a1d950SNeel Natu KASSERT((intr_info & VMCS_INTR_VALID) != 0, 2665e5a1d950SNeel Natu ("VM exit interruption info invalid: %#x", intr_info)); 2666c308b23bSNeel Natu 2667b0538143SNeel Natu intr_vec = intr_info & 0xff; 2668b0538143SNeel Natu intr_type = intr_info & VMCS_INTR_T_MASK; 2669b0538143SNeel Natu 2670e5a1d950SNeel Natu /* 2671e5a1d950SNeel Natu * If Virtual NMIs control is 1 and the VM-exit is due to a 2672e5a1d950SNeel Natu * fault encountered during the execution of IRET then we must 2673e5a1d950SNeel Natu * restore the state of "virtual-NMI blocking" before resuming 2674e5a1d950SNeel Natu * the guest. 2675e5a1d950SNeel Natu * 2676e5a1d950SNeel Natu * See "Resuming Guest Software after Handling an Exception". 2677091d4532SNeel Natu * See "Information for VM Exits Due to Vectored Events". 2678e5a1d950SNeel Natu */ 2679e5a1d950SNeel Natu if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 && 2680b0538143SNeel Natu (intr_vec != IDT_DF) && 2681e5a1d950SNeel Natu (intr_info & EXIT_QUAL_NMIUDTI) != 0) 2682869c8d19SJohn Baldwin vmx_restore_nmi_blocking(vcpu); 2683c308b23bSNeel Natu 2684c308b23bSNeel Natu /* 268562fbd7c2SNeel Natu * The NMI has already been handled in vmx_exit_handle_nmi(). 2686c308b23bSNeel Natu */ 2687b0538143SNeel Natu if (intr_type == VMCS_INTR_T_NMI) 2688c308b23bSNeel Natu return (1); 2689b0538143SNeel Natu 2690b0538143SNeel Natu /* 2691b0538143SNeel Natu * Call the machine check handler by hand. Also don't reflect 2692b0538143SNeel Natu * the machine check back into the guest. 2693b0538143SNeel Natu */ 2694b0538143SNeel Natu if (intr_vec == IDT_MC) { 269557e0119eSJohn Baldwin VMX_CTR0(vcpu, "Vectoring to MCE handler"); 2696b0538143SNeel Natu __asm __volatile("int $18"); 2697b0538143SNeel Natu return (1); 2698b0538143SNeel Natu } 2699b0538143SNeel Natu 2700cbd03a9dSJohn Baldwin /* 2701cbd03a9dSJohn Baldwin * If the hypervisor has requested user exits for 2702cbd03a9dSJohn Baldwin * debug exceptions, bounce them out to userland. 2703cbd03a9dSJohn Baldwin */ 2704cbd03a9dSJohn Baldwin if (intr_type == VMCS_INTR_T_SWEXCEPTION && intr_vec == IDT_BP && 27051aa51504SJohn Baldwin (vcpu->cap.set & (1 << VM_CAP_BPT_EXIT))) { 2706cbd03a9dSJohn Baldwin vmexit->exitcode = VM_EXITCODE_BPT; 2707cbd03a9dSJohn Baldwin vmexit->u.bpt.inst_length = vmexit->inst_length; 2708cbd03a9dSJohn Baldwin vmexit->inst_length = 0; 2709cbd03a9dSJohn Baldwin break; 2710cbd03a9dSJohn Baldwin } 2711cbd03a9dSJohn Baldwin 2712b0538143SNeel Natu if (intr_vec == IDT_PF) { 2713b0538143SNeel Natu error = vmxctx_setreg(vmxctx, VM_REG_GUEST_CR2, qual); 2714b0538143SNeel Natu KASSERT(error == 0, ("%s: vmxctx_setreg(cr2) error %d", 2715b0538143SNeel Natu __func__, error)); 2716b0538143SNeel Natu } 2717b0538143SNeel Natu 2718b0538143SNeel Natu /* 2719b0538143SNeel Natu * Software exceptions exhibit trap-like behavior. This in 2720b0538143SNeel Natu * turn requires populating the VM-entry instruction length 2721b0538143SNeel Natu * so that the %rip in the trap frame is past the INT3/INTO 2722b0538143SNeel Natu * instruction. 2723b0538143SNeel Natu */ 2724b0538143SNeel Natu if (intr_type == VMCS_INTR_T_SWEXCEPTION) 2725b0538143SNeel Natu vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length); 2726b0538143SNeel Natu 2727b0538143SNeel Natu /* Reflect all other exceptions back into the guest */ 2728c9c75df4SNeel Natu errcode_valid = errcode = 0; 2729b0538143SNeel Natu if (intr_info & VMCS_INTR_DEL_ERRCODE) { 2730c9c75df4SNeel Natu errcode_valid = 1; 2731c9c75df4SNeel Natu errcode = vmcs_read(VMCS_EXIT_INTR_ERRCODE); 2732b0538143SNeel Natu } 273357e0119eSJohn Baldwin VMX_CTR2(vcpu, "Reflecting exception %d/%#x into " 2734c9c75df4SNeel Natu "the guest", intr_vec, errcode); 27356ac73777STycho Nightingale SDT_PROBE5(vmm, vmx, exit, exception, 27361aa51504SJohn Baldwin vmx, vcpuid, vmexit, intr_vec, errcode); 2737d3956e46SJohn Baldwin error = vm_inject_exception(vcpu->vcpu, intr_vec, 2738c9c75df4SNeel Natu errcode_valid, errcode, 0); 2739b0538143SNeel Natu KASSERT(error == 0, ("%s: vm_inject_exception error %d", 2740b0538143SNeel Natu __func__, error)); 2741b0538143SNeel Natu return (1); 2742b0538143SNeel Natu 2743cd942e0fSPeter Grehan case EXIT_REASON_EPT_FAULT: 2744318224bbSNeel Natu /* 2745318224bbSNeel Natu * If 'gpa' lies within the address space allocated to 2746318224bbSNeel Natu * memory then this must be a nested page fault otherwise 2747318224bbSNeel Natu * this must be an instruction that accesses MMIO space. 2748318224bbSNeel Natu */ 2749a2da7af6SNeel Natu gpa = vmcs_gpa(); 275080cb5d84SJohn Baldwin if (vm_mem_allocated(vcpu->vcpu, gpa) || 27511aa51504SJohn Baldwin apic_access_fault(vcpu, gpa)) { 2752cd942e0fSPeter Grehan vmexit->exitcode = VM_EXITCODE_PAGING; 2753d087a399SNeel Natu vmexit->inst_length = 0; 275413ec9371SPeter Grehan vmexit->u.paging.gpa = gpa; 2755318224bbSNeel Natu vmexit->u.paging.fault_type = ept_fault_type(qual); 27563dc3d32aSJohn Baldwin vmm_stat_incr(vcpu->vcpu, VMEXIT_NESTED_FAULT, 1); 27576ac73777STycho Nightingale SDT_PROBE5(vmm, vmx, exit, nestedfault, 27581aa51504SJohn Baldwin vmx, vcpuid, vmexit, gpa, qual); 2759318224bbSNeel Natu } else if (ept_emulation_fault(qual)) { 2760e4c8a13dSNeel Natu vmexit_inst_emul(vmexit, gpa, vmcs_gla()); 27613dc3d32aSJohn Baldwin vmm_stat_incr(vcpu->vcpu, VMEXIT_INST_EMUL, 1); 27626ac73777STycho Nightingale SDT_PROBE4(vmm, vmx, exit, mmiofault, 27631aa51504SJohn Baldwin vmx, vcpuid, vmexit, gpa); 2764a2da7af6SNeel Natu } 2765e5a1d950SNeel Natu /* 2766e5a1d950SNeel Natu * If Virtual NMIs control is 1 and the VM-exit is due to an 2767e5a1d950SNeel Natu * EPT fault during the execution of IRET then we must restore 2768e5a1d950SNeel Natu * the state of "virtual-NMI blocking" before resuming. 2769e5a1d950SNeel Natu * 2770e5a1d950SNeel Natu * See description of "NMI unblocking due to IRET" in 2771e5a1d950SNeel Natu * "Exit Qualification for EPT Violations". 2772e5a1d950SNeel Natu */ 2773e5a1d950SNeel Natu if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 && 2774e5a1d950SNeel Natu (qual & EXIT_QUAL_NMIUDTI) != 0) 2775869c8d19SJohn Baldwin vmx_restore_nmi_blocking(vcpu); 2776cd942e0fSPeter Grehan break; 277730b94db8SNeel Natu case EXIT_REASON_VIRTUALIZED_EOI: 277830b94db8SNeel Natu vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI; 277930b94db8SNeel Natu vmexit->u.ioapic_eoi.vector = qual & 0xFF; 27801aa51504SJohn Baldwin SDT_PROBE3(vmm, vmx, exit, eoi, vmx, vcpuid, vmexit); 278130b94db8SNeel Natu vmexit->inst_length = 0; /* trap-like */ 278230b94db8SNeel Natu break; 278388c4b8d1SNeel Natu case EXIT_REASON_APIC_ACCESS: 27841aa51504SJohn Baldwin SDT_PROBE3(vmm, vmx, exit, apicaccess, vmx, vcpuid, vmexit); 27851aa51504SJohn Baldwin handled = vmx_handle_apic_access(vcpu, vmexit); 278688c4b8d1SNeel Natu break; 278788c4b8d1SNeel Natu case EXIT_REASON_APIC_WRITE: 278888c4b8d1SNeel Natu /* 278988c4b8d1SNeel Natu * APIC-write VM exit is trap-like so the %rip is already 279088c4b8d1SNeel Natu * pointing to the next instruction. 279188c4b8d1SNeel Natu */ 279288c4b8d1SNeel Natu vmexit->inst_length = 0; 2793d3956e46SJohn Baldwin vlapic = vm_lapic(vcpu->vcpu); 27946ac73777STycho Nightingale SDT_PROBE4(vmm, vmx, exit, apicwrite, 27951aa51504SJohn Baldwin vmx, vcpuid, vmexit, vlapic); 27961aa51504SJohn Baldwin handled = vmx_handle_apic_write(vcpu, vlapic, qual); 279788c4b8d1SNeel Natu break; 2798abb023fbSJohn Baldwin case EXIT_REASON_XSETBV: 27991aa51504SJohn Baldwin SDT_PROBE3(vmm, vmx, exit, xsetbv, vmx, vcpuid, vmexit); 2800a0efd3fbSJohn Baldwin handled = vmx_emulate_xsetbv(vmx, vcpu, vmexit); 2801abb023fbSJohn Baldwin break; 280265145c7fSNeel Natu case EXIT_REASON_MONITOR: 28031aa51504SJohn Baldwin SDT_PROBE3(vmm, vmx, exit, monitor, vmx, vcpuid, vmexit); 280465145c7fSNeel Natu vmexit->exitcode = VM_EXITCODE_MONITOR; 280565145c7fSNeel Natu break; 280665145c7fSNeel Natu case EXIT_REASON_MWAIT: 28071aa51504SJohn Baldwin SDT_PROBE3(vmm, vmx, exit, mwait, vmx, vcpuid, vmexit); 280865145c7fSNeel Natu vmexit->exitcode = VM_EXITCODE_MWAIT; 280965145c7fSNeel Natu break; 28101bc51badSMichael Reifenberger case EXIT_REASON_TPR: 2811d3956e46SJohn Baldwin vlapic = vm_lapic(vcpu->vcpu); 28121bc51badSMichael Reifenberger vlapic_sync_tpr(vlapic); 28131bc51badSMichael Reifenberger vmexit->inst_length = 0; 28141bc51badSMichael Reifenberger handled = HANDLED; 28151bc51badSMichael Reifenberger break; 281627d26457SAndrew Turner case EXIT_REASON_VMCALL: 281727d26457SAndrew Turner case EXIT_REASON_VMCLEAR: 281827d26457SAndrew Turner case EXIT_REASON_VMLAUNCH: 281927d26457SAndrew Turner case EXIT_REASON_VMPTRLD: 282027d26457SAndrew Turner case EXIT_REASON_VMPTRST: 282127d26457SAndrew Turner case EXIT_REASON_VMREAD: 282227d26457SAndrew Turner case EXIT_REASON_VMRESUME: 282327d26457SAndrew Turner case EXIT_REASON_VMWRITE: 282427d26457SAndrew Turner case EXIT_REASON_VMXOFF: 282527d26457SAndrew Turner case EXIT_REASON_VMXON: 28261aa51504SJohn Baldwin SDT_PROBE3(vmm, vmx, exit, vminsn, vmx, vcpuid, vmexit); 282727d26457SAndrew Turner vmexit->exitcode = VM_EXITCODE_VMINSN; 282827d26457SAndrew Turner break; 28294eadbef9SCorvin Köhne case EXIT_REASON_INVD: 28303ba952e1SCorvin Köhne case EXIT_REASON_WBINVD: 28314eadbef9SCorvin Köhne /* ignore exit */ 28323ba952e1SCorvin Köhne handled = HANDLED; 28333ba952e1SCorvin Köhne break; 2834366f6083SPeter Grehan default: 28356ac73777STycho Nightingale SDT_PROBE4(vmm, vmx, exit, unknown, 28361aa51504SJohn Baldwin vmx, vcpuid, vmexit, reason); 28373dc3d32aSJohn Baldwin vmm_stat_incr(vcpu->vcpu, VMEXIT_UNKNOWN, 1); 2838366f6083SPeter Grehan break; 2839366f6083SPeter Grehan } 2840366f6083SPeter Grehan 2841366f6083SPeter Grehan if (handled) { 2842366f6083SPeter Grehan /* 2843366f6083SPeter Grehan * It is possible that control is returned to userland 2844366f6083SPeter Grehan * even though we were able to handle the VM exit in the 2845eeefa4e4SNeel Natu * kernel. 2846366f6083SPeter Grehan * 2847366f6083SPeter Grehan * In such a case we want to make sure that the userland 2848366f6083SPeter Grehan * restarts guest execution at the instruction *after* 2849366f6083SPeter Grehan * the one we just processed. Therefore we update the 2850366f6083SPeter Grehan * guest rip in the VMCS and in 'vmexit'. 2851366f6083SPeter Grehan */ 2852366f6083SPeter Grehan vmexit->rip += vmexit->inst_length; 2853366f6083SPeter Grehan vmexit->inst_length = 0; 28543de83862SNeel Natu vmcs_write(VMCS_GUEST_RIP, vmexit->rip); 2855366f6083SPeter Grehan } else { 2856366f6083SPeter Grehan if (vmexit->exitcode == VM_EXITCODE_BOGUS) { 2857366f6083SPeter Grehan /* 2858366f6083SPeter Grehan * If this VM exit was not claimed by anybody then 2859366f6083SPeter Grehan * treat it as a generic VMX exit. 2860366f6083SPeter Grehan */ 2861366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_VMX; 28620492757cSNeel Natu vmexit->u.vmx.status = VM_SUCCESS; 2863c308b23bSNeel Natu vmexit->u.vmx.inst_type = 0; 2864c308b23bSNeel Natu vmexit->u.vmx.inst_error = 0; 2865366f6083SPeter Grehan } else { 2866366f6083SPeter Grehan /* 2867366f6083SPeter Grehan * The exitcode and collateral have been populated. 2868366f6083SPeter Grehan * The VM exit will be processed further in userland. 2869366f6083SPeter Grehan */ 2870366f6083SPeter Grehan } 2871366f6083SPeter Grehan } 28726ac73777STycho Nightingale 28736ac73777STycho Nightingale SDT_PROBE4(vmm, vmx, exit, return, 28741aa51504SJohn Baldwin vmx, vcpuid, vmexit, handled); 2875366f6083SPeter Grehan return (handled); 2876366f6083SPeter Grehan } 2877366f6083SPeter Grehan 287840487465SNeel Natu static __inline void 28790492757cSNeel Natu vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit) 28800492757cSNeel Natu { 28810492757cSNeel Natu 28820492757cSNeel Natu KASSERT(vmxctx->inst_fail_status != VM_SUCCESS, 28830492757cSNeel Natu ("vmx_exit_inst_error: invalid inst_fail_status %d", 28840492757cSNeel Natu vmxctx->inst_fail_status)); 28850492757cSNeel Natu 28860492757cSNeel Natu vmexit->inst_length = 0; 28870492757cSNeel Natu vmexit->exitcode = VM_EXITCODE_VMX; 28880492757cSNeel Natu vmexit->u.vmx.status = vmxctx->inst_fail_status; 28890492757cSNeel Natu vmexit->u.vmx.inst_error = vmcs_instruction_error(); 28900492757cSNeel Natu vmexit->u.vmx.exit_reason = ~0; 28910492757cSNeel Natu vmexit->u.vmx.exit_qualification = ~0; 28920492757cSNeel Natu 28930492757cSNeel Natu switch (rc) { 28940492757cSNeel Natu case VMX_VMRESUME_ERROR: 28950492757cSNeel Natu case VMX_VMLAUNCH_ERROR: 28960492757cSNeel Natu vmexit->u.vmx.inst_type = rc; 28970492757cSNeel Natu break; 28980492757cSNeel Natu default: 28990492757cSNeel Natu panic("vm_exit_inst_error: vmx_enter_guest returned %d", rc); 29000492757cSNeel Natu } 29010492757cSNeel Natu } 29020492757cSNeel Natu 290362fbd7c2SNeel Natu /* 290462fbd7c2SNeel Natu * If the NMI-exiting VM execution control is set to '1' then an NMI in 290562fbd7c2SNeel Natu * non-root operation causes a VM-exit. NMI blocking is in effect so it is 290662fbd7c2SNeel Natu * sufficient to simply vector to the NMI handler via a software interrupt. 290762fbd7c2SNeel Natu * However, this must be done before maskable interrupts are enabled 290862fbd7c2SNeel Natu * otherwise the "iret" issued by an interrupt handler will incorrectly 290962fbd7c2SNeel Natu * clear NMI blocking. 291062fbd7c2SNeel Natu */ 291162fbd7c2SNeel Natu static __inline void 2912869c8d19SJohn Baldwin vmx_exit_handle_nmi(struct vmx_vcpu *vcpu, struct vm_exit *vmexit) 291362fbd7c2SNeel Natu { 291462fbd7c2SNeel Natu uint32_t intr_info; 291562fbd7c2SNeel Natu 291662fbd7c2SNeel Natu KASSERT((read_rflags() & PSL_I) == 0, ("interrupts enabled")); 291762fbd7c2SNeel Natu 291862fbd7c2SNeel Natu if (vmexit->u.vmx.exit_reason != EXIT_REASON_EXCEPTION) 291962fbd7c2SNeel Natu return; 292062fbd7c2SNeel Natu 292162fbd7c2SNeel Natu intr_info = vmcs_read(VMCS_EXIT_INTR_INFO); 292262fbd7c2SNeel Natu KASSERT((intr_info & VMCS_INTR_VALID) != 0, 292362fbd7c2SNeel Natu ("VM exit interruption info invalid: %#x", intr_info)); 292462fbd7c2SNeel Natu 292562fbd7c2SNeel Natu if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI) { 292662fbd7c2SNeel Natu KASSERT((intr_info & 0xff) == IDT_NMI, ("VM exit due " 292762fbd7c2SNeel Natu "to NMI has invalid vector: %#x", intr_info)); 292857e0119eSJohn Baldwin VMX_CTR0(vcpu, "Vectoring to NMI handler"); 292962fbd7c2SNeel Natu __asm __volatile("int $2"); 293062fbd7c2SNeel Natu } 293162fbd7c2SNeel Natu } 293262fbd7c2SNeel Natu 293365eefbe4SJohn Baldwin static __inline void 293465eefbe4SJohn Baldwin vmx_dr_enter_guest(struct vmxctx *vmxctx) 293565eefbe4SJohn Baldwin { 293665eefbe4SJohn Baldwin register_t rflags; 293765eefbe4SJohn Baldwin 293865eefbe4SJohn Baldwin /* Save host control debug registers. */ 293965eefbe4SJohn Baldwin vmxctx->host_dr7 = rdr7(); 294065eefbe4SJohn Baldwin vmxctx->host_debugctl = rdmsr(MSR_DEBUGCTLMSR); 294165eefbe4SJohn Baldwin 294265eefbe4SJohn Baldwin /* 294365eefbe4SJohn Baldwin * Disable debugging in DR7 and DEBUGCTL to avoid triggering 294465eefbe4SJohn Baldwin * exceptions in the host based on the guest DRx values. The 294565eefbe4SJohn Baldwin * guest DR7 and DEBUGCTL are saved/restored in the VMCS. 294665eefbe4SJohn Baldwin */ 294765eefbe4SJohn Baldwin load_dr7(0); 294865eefbe4SJohn Baldwin wrmsr(MSR_DEBUGCTLMSR, 0); 294965eefbe4SJohn Baldwin 295065eefbe4SJohn Baldwin /* 295165eefbe4SJohn Baldwin * Disable single stepping the kernel to avoid corrupting the 295265eefbe4SJohn Baldwin * guest DR6. A debugger might still be able to corrupt the 295365eefbe4SJohn Baldwin * guest DR6 by setting a breakpoint after this point and then 295465eefbe4SJohn Baldwin * single stepping. 295565eefbe4SJohn Baldwin */ 295665eefbe4SJohn Baldwin rflags = read_rflags(); 295765eefbe4SJohn Baldwin vmxctx->host_tf = rflags & PSL_T; 295865eefbe4SJohn Baldwin write_rflags(rflags & ~PSL_T); 295965eefbe4SJohn Baldwin 296065eefbe4SJohn Baldwin /* Save host debug registers. */ 296165eefbe4SJohn Baldwin vmxctx->host_dr0 = rdr0(); 296265eefbe4SJohn Baldwin vmxctx->host_dr1 = rdr1(); 296365eefbe4SJohn Baldwin vmxctx->host_dr2 = rdr2(); 296465eefbe4SJohn Baldwin vmxctx->host_dr3 = rdr3(); 296565eefbe4SJohn Baldwin vmxctx->host_dr6 = rdr6(); 296665eefbe4SJohn Baldwin 296765eefbe4SJohn Baldwin /* Restore guest debug registers. */ 296865eefbe4SJohn Baldwin load_dr0(vmxctx->guest_dr0); 296965eefbe4SJohn Baldwin load_dr1(vmxctx->guest_dr1); 297065eefbe4SJohn Baldwin load_dr2(vmxctx->guest_dr2); 297165eefbe4SJohn Baldwin load_dr3(vmxctx->guest_dr3); 297265eefbe4SJohn Baldwin load_dr6(vmxctx->guest_dr6); 297365eefbe4SJohn Baldwin } 297465eefbe4SJohn Baldwin 297565eefbe4SJohn Baldwin static __inline void 297665eefbe4SJohn Baldwin vmx_dr_leave_guest(struct vmxctx *vmxctx) 297765eefbe4SJohn Baldwin { 297865eefbe4SJohn Baldwin 297965eefbe4SJohn Baldwin /* Save guest debug registers. */ 298065eefbe4SJohn Baldwin vmxctx->guest_dr0 = rdr0(); 298165eefbe4SJohn Baldwin vmxctx->guest_dr1 = rdr1(); 298265eefbe4SJohn Baldwin vmxctx->guest_dr2 = rdr2(); 298365eefbe4SJohn Baldwin vmxctx->guest_dr3 = rdr3(); 298465eefbe4SJohn Baldwin vmxctx->guest_dr6 = rdr6(); 298565eefbe4SJohn Baldwin 298665eefbe4SJohn Baldwin /* 298765eefbe4SJohn Baldwin * Restore host debug registers. Restore DR7, DEBUGCTL, and 298865eefbe4SJohn Baldwin * PSL_T last. 298965eefbe4SJohn Baldwin */ 299065eefbe4SJohn Baldwin load_dr0(vmxctx->host_dr0); 299165eefbe4SJohn Baldwin load_dr1(vmxctx->host_dr1); 299265eefbe4SJohn Baldwin load_dr2(vmxctx->host_dr2); 299365eefbe4SJohn Baldwin load_dr3(vmxctx->host_dr3); 299465eefbe4SJohn Baldwin load_dr6(vmxctx->host_dr6); 299565eefbe4SJohn Baldwin wrmsr(MSR_DEBUGCTLMSR, vmxctx->host_debugctl); 299665eefbe4SJohn Baldwin load_dr7(vmxctx->host_dr7); 299765eefbe4SJohn Baldwin write_rflags(read_rflags() | vmxctx->host_tf); 299865eefbe4SJohn Baldwin } 299965eefbe4SJohn Baldwin 30008e2cbc56SMark Johnston static __inline void 30018e2cbc56SMark Johnston vmx_pmap_activate(struct vmx *vmx, pmap_t pmap) 30028e2cbc56SMark Johnston { 30038e2cbc56SMark Johnston long eptgen; 30048e2cbc56SMark Johnston int cpu; 30058e2cbc56SMark Johnston 30068e2cbc56SMark Johnston cpu = curcpu; 30078e2cbc56SMark Johnston 30088e2cbc56SMark Johnston CPU_SET_ATOMIC(cpu, &pmap->pm_active); 30096f5a9606SMark Johnston smr_enter(pmap->pm_eptsmr); 30108e2cbc56SMark Johnston eptgen = atomic_load_long(&pmap->pm_eptgen); 30118e2cbc56SMark Johnston if (eptgen != vmx->eptgen[cpu]) { 30128e2cbc56SMark Johnston vmx->eptgen[cpu] = eptgen; 30138e2cbc56SMark Johnston invept(INVEPT_TYPE_SINGLE_CONTEXT, 30148e2cbc56SMark Johnston (struct invept_desc){ .eptp = vmx->eptp, ._res = 0 }); 30158e2cbc56SMark Johnston } 30168e2cbc56SMark Johnston } 30178e2cbc56SMark Johnston 30188e2cbc56SMark Johnston static __inline void 30198e2cbc56SMark Johnston vmx_pmap_deactivate(struct vmx *vmx, pmap_t pmap) 30208e2cbc56SMark Johnston { 30216f5a9606SMark Johnston smr_exit(pmap->pm_eptsmr); 30228e2cbc56SMark Johnston CPU_CLR_ATOMIC(curcpu, &pmap->pm_active); 30238e2cbc56SMark Johnston } 30248e2cbc56SMark Johnston 30250492757cSNeel Natu static int 3026869c8d19SJohn Baldwin vmx_run(void *vcpui, register_t rip, pmap_t pmap, struct vm_eventinfo *evinfo) 30270492757cSNeel Natu { 302880cb5d84SJohn Baldwin int rc, handled, launched; 3029366f6083SPeter Grehan struct vmx *vmx; 30301aa51504SJohn Baldwin struct vmx_vcpu *vcpu; 3031366f6083SPeter Grehan struct vmxctx *vmxctx; 3032366f6083SPeter Grehan struct vmcs *vmcs; 303398ed632cSNeel Natu struct vm_exit *vmexit; 3034de5ea6b6SNeel Natu struct vlapic *vlapic; 303579c59630SNeel Natu uint32_t exit_reason; 3036b843f9beSJohn Baldwin struct region_descriptor gdtr, idtr; 3037b843f9beSJohn Baldwin uint16_t ldt_sel; 3038366f6083SPeter Grehan 30391aa51504SJohn Baldwin vcpu = vcpui; 3040869c8d19SJohn Baldwin vmx = vcpu->vmx; 30411aa51504SJohn Baldwin vmcs = vcpu->vmcs; 30421aa51504SJohn Baldwin vmxctx = &vcpu->ctx; 3043d3956e46SJohn Baldwin vlapic = vm_lapic(vcpu->vcpu); 304480cb5d84SJohn Baldwin vmexit = vm_exitinfo(vcpu->vcpu); 30450492757cSNeel Natu launched = 0; 304698ed632cSNeel Natu 3047318224bbSNeel Natu KASSERT(vmxctx->pmap == pmap, 3048318224bbSNeel Natu ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap)); 3049318224bbSNeel Natu 305080cb5d84SJohn Baldwin vmx_msr_guest_enter(vcpu); 3051c3498942SNeel Natu 3052366f6083SPeter Grehan VMPTRLD(vmcs); 3053366f6083SPeter Grehan 3054366f6083SPeter Grehan /* 3055366f6083SPeter Grehan * XXX 3056366f6083SPeter Grehan * We do this every time because we may setup the virtual machine 3057366f6083SPeter Grehan * from a different process than the one that actually runs it. 3058366f6083SPeter Grehan * 3059366f6083SPeter Grehan * If the life of a virtual machine was spent entirely in the context 306015add60dSPeter Grehan * of a single process we could do this once in vmx_init(). 3061366f6083SPeter Grehan */ 30623de83862SNeel Natu vmcs_write(VMCS_HOST_CR3, rcr3()); 3063366f6083SPeter Grehan 30642ce12423SNeel Natu vmcs_write(VMCS_GUEST_RIP, rip); 3065953c2c47SNeel Natu vmx_set_pcpu_defaults(vmx, vcpu, pmap); 3066366f6083SPeter Grehan do { 30672ce12423SNeel Natu KASSERT(vmcs_guest_rip() == rip, ("%s: vmcs guest rip mismatch " 30682ce12423SNeel Natu "%#lx/%#lx", __func__, vmcs_guest_rip(), rip)); 306940487465SNeel Natu 30702ce12423SNeel Natu handled = UNHANDLED; 30710492757cSNeel Natu /* 30720492757cSNeel Natu * Interrupts are disabled from this point on until the 30730492757cSNeel Natu * guest starts executing. This is done for the following 30740492757cSNeel Natu * reasons: 30750492757cSNeel Natu * 30760492757cSNeel Natu * If an AST is asserted on this thread after the check below, 30770492757cSNeel Natu * then the IPI_AST notification will not be lost, because it 30780492757cSNeel Natu * will cause a VM exit due to external interrupt as soon as 30790492757cSNeel Natu * the guest state is loaded. 30800492757cSNeel Natu * 30810492757cSNeel Natu * A posted interrupt after 'vmx_inject_interrupts()' will 30820492757cSNeel Natu * not be "lost" because it will be held pending in the host 30830492757cSNeel Natu * APIC because interrupts are disabled. The pending interrupt 30840492757cSNeel Natu * will be recognized as soon as the guest state is loaded. 30850492757cSNeel Natu * 30860492757cSNeel Natu * The same reasoning applies to the IPI generated by 30870492757cSNeel Natu * pmap_invalidate_ept(). 30880492757cSNeel Natu */ 30890492757cSNeel Natu disable_intr(); 309080cb5d84SJohn Baldwin vmx_inject_interrupts(vcpu, vlapic, rip); 3091091d4532SNeel Natu 3092091d4532SNeel Natu /* 3093091d4532SNeel Natu * Check for vcpu suspension after injecting events because 3094091d4532SNeel Natu * vmx_inject_interrupts() can suspend the vcpu due to a 3095091d4532SNeel Natu * triple fault. 3096091d4532SNeel Natu */ 3097248e6799SNeel Natu if (vcpu_suspended(evinfo)) { 30980492757cSNeel Natu enable_intr(); 309980cb5d84SJohn Baldwin vm_exit_suspended(vcpu->vcpu, rip); 31000492757cSNeel Natu break; 31010492757cSNeel Natu } 31020492757cSNeel Natu 3103892feec2SCorvin Köhne if (vcpu_rendezvous_pending(vcpu->vcpu, evinfo)) { 31045b8a8cd1SNeel Natu enable_intr(); 310580cb5d84SJohn Baldwin vm_exit_rendezvous(vcpu->vcpu, rip); 31065b8a8cd1SNeel Natu break; 31075b8a8cd1SNeel Natu } 31085b8a8cd1SNeel Natu 3109248e6799SNeel Natu if (vcpu_reqidle(evinfo)) { 3110248e6799SNeel Natu enable_intr(); 311180cb5d84SJohn Baldwin vm_exit_reqidle(vcpu->vcpu, rip); 3112248e6799SNeel Natu break; 3113248e6799SNeel Natu } 3114248e6799SNeel Natu 311580cb5d84SJohn Baldwin if (vcpu_should_yield(vcpu->vcpu)) { 3116b15a09c0SNeel Natu enable_intr(); 311780cb5d84SJohn Baldwin vm_exit_astpending(vcpu->vcpu, rip); 3118869c8d19SJohn Baldwin vmx_astpending_trace(vcpu, rip); 311940487465SNeel Natu handled = HANDLED; 3120b15a09c0SNeel Natu break; 3121b15a09c0SNeel Natu } 3122b15a09c0SNeel Natu 312380cb5d84SJohn Baldwin if (vcpu_debugged(vcpu->vcpu)) { 3124fc276d92SJohn Baldwin enable_intr(); 312580cb5d84SJohn Baldwin vm_exit_debug(vcpu->vcpu, rip); 3126fc276d92SJohn Baldwin break; 3127fc276d92SJohn Baldwin } 3128fc276d92SJohn Baldwin 3129b843f9beSJohn Baldwin /* 31301bc51badSMichael Reifenberger * If TPR Shadowing is enabled, the TPR Threshold 31311bc51badSMichael Reifenberger * must be updated right before entering the guest. 31321bc51badSMichael Reifenberger */ 31331bc51badSMichael Reifenberger if (tpr_shadowing && !virtual_interrupt_delivery) { 31341aa51504SJohn Baldwin if ((vcpu->cap.proc_ctls & PROCBASED_USE_TPR_SHADOW) != 0) { 31351bc51badSMichael Reifenberger vmcs_write(VMCS_TPR_THRESHOLD, vlapic_get_cr8(vlapic)); 31361bc51badSMichael Reifenberger } 31371bc51badSMichael Reifenberger } 31381bc51badSMichael Reifenberger 31391bc51badSMichael Reifenberger /* 3140b843f9beSJohn Baldwin * VM exits restore the base address but not the 3141b843f9beSJohn Baldwin * limits of GDTR and IDTR. The VMCS only stores the 3142b843f9beSJohn Baldwin * base address, so VM exits set the limits to 0xffff. 3143b843f9beSJohn Baldwin * Save and restore the full GDTR and IDTR to restore 3144b843f9beSJohn Baldwin * the limits. 3145b843f9beSJohn Baldwin * 3146b843f9beSJohn Baldwin * The VMCS does not save the LDTR at all, and VM 3147b843f9beSJohn Baldwin * exits clear LDTR as if a NULL selector were loaded. 3148b843f9beSJohn Baldwin * The userspace hypervisor probably doesn't use a 3149b843f9beSJohn Baldwin * LDT, but save and restore it to be safe. 3150b843f9beSJohn Baldwin */ 3151b843f9beSJohn Baldwin sgdt(&gdtr); 3152b843f9beSJohn Baldwin sidt(&idtr); 3153b843f9beSJohn Baldwin ldt_sel = sldt(); 3154b843f9beSJohn Baldwin 3155f5f5f1e7SPeter Grehan /* 3156f5f5f1e7SPeter Grehan * The TSC_AUX MSR must be saved/restored while interrupts 3157f5f5f1e7SPeter Grehan * are disabled so that it is not possible for the guest 3158f5f5f1e7SPeter Grehan * TSC_AUX MSR value to be overwritten by the resume 3159f5f5f1e7SPeter Grehan * portion of the IPI_SUSPEND codepath. This is why the 3160f5f5f1e7SPeter Grehan * transition of this MSR is handled separately from those 3161f5f5f1e7SPeter Grehan * handled by vmx_msr_guest_{enter,exit}(), which are ok to 3162f5f5f1e7SPeter Grehan * be transitioned with preemption disabled but interrupts 3163f5f5f1e7SPeter Grehan * enabled. 3164f5f5f1e7SPeter Grehan * 3165f5f5f1e7SPeter Grehan * These vmx_msr_guest_{enter,exit}_tsc_aux() calls can be 3166f5f5f1e7SPeter Grehan * anywhere in this loop so long as they happen with 3167f5f5f1e7SPeter Grehan * interrupts disabled. This location is chosen for 3168f5f5f1e7SPeter Grehan * simplicity. 3169f5f5f1e7SPeter Grehan */ 3170f5f5f1e7SPeter Grehan vmx_msr_guest_enter_tsc_aux(vmx, vcpu); 3171f5f5f1e7SPeter Grehan 317265eefbe4SJohn Baldwin vmx_dr_enter_guest(vmxctx); 317379c59630SNeel Natu 31748e2cbc56SMark Johnston /* 31758e2cbc56SMark Johnston * Mark the EPT as active on this host CPU and invalidate 31768e2cbc56SMark Johnston * EPTP-tagged TLB entries if required. 31778e2cbc56SMark Johnston */ 31788e2cbc56SMark Johnston vmx_pmap_activate(vmx, pmap); 31798e2cbc56SMark Johnston 3180869c8d19SJohn Baldwin vmx_run_trace(vcpu); 31818e2cbc56SMark Johnston rc = vmx_enter_guest(vmxctx, vmx, launched); 31828e2cbc56SMark Johnston 31838e2cbc56SMark Johnston vmx_pmap_deactivate(vmx, pmap); 31848e2cbc56SMark Johnston vmx_dr_leave_guest(vmxctx); 3185f5f5f1e7SPeter Grehan vmx_msr_guest_exit_tsc_aux(vmx, vcpu); 3186f5f5f1e7SPeter Grehan 3187b843f9beSJohn Baldwin bare_lgdt(&gdtr); 3188b843f9beSJohn Baldwin lidt(&idtr); 3189b843f9beSJohn Baldwin lldt(ldt_sel); 3190b843f9beSJohn Baldwin 319179c59630SNeel Natu /* Collect some information for VM exit processing */ 319279c59630SNeel Natu vmexit->rip = rip = vmcs_guest_rip(); 319379c59630SNeel Natu vmexit->inst_length = vmexit_instruction_length(); 319479c59630SNeel Natu vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason(); 319579c59630SNeel Natu vmexit->u.vmx.exit_qualification = vmcs_exit_qualification(); 319679c59630SNeel Natu 31972ce12423SNeel Natu /* Update 'nextrip' */ 31981aa51504SJohn Baldwin vcpu->state.nextrip = rip; 31992ce12423SNeel Natu 32000492757cSNeel Natu if (rc == VMX_GUEST_VMEXIT) { 3201869c8d19SJohn Baldwin vmx_exit_handle_nmi(vcpu, vmexit); 320262fbd7c2SNeel Natu enable_intr(); 32030492757cSNeel Natu handled = vmx_exit_process(vmx, vcpu, vmexit); 32040492757cSNeel Natu } else { 320562fbd7c2SNeel Natu enable_intr(); 320640487465SNeel Natu vmx_exit_inst_error(vmxctx, rc, vmexit); 3207eeefa4e4SNeel Natu } 320862fbd7c2SNeel Natu launched = 1; 3209869c8d19SJohn Baldwin vmx_exit_trace(vcpu, rip, exit_reason, handled); 32102ce12423SNeel Natu rip = vmexit->rip; 3211eeefa4e4SNeel Natu } while (handled); 3212366f6083SPeter Grehan 3213366f6083SPeter Grehan /* 3214366f6083SPeter Grehan * If a VM exit has been handled then the exitcode must be BOGUS 3215366f6083SPeter Grehan * If a VM exit is not handled then the exitcode must not be BOGUS 3216366f6083SPeter Grehan */ 3217366f6083SPeter Grehan if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) || 3218366f6083SPeter Grehan (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) { 3219366f6083SPeter Grehan panic("Mismatch between handled (%d) and exitcode (%d)", 3220366f6083SPeter Grehan handled, vmexit->exitcode); 3221366f6083SPeter Grehan } 3222366f6083SPeter Grehan 322357e0119eSJohn Baldwin VMX_CTR1(vcpu, "returning from vmx_run: exitcode %d", 32240492757cSNeel Natu vmexit->exitcode); 3225366f6083SPeter Grehan 3226366f6083SPeter Grehan VMCLEAR(vmcs); 322780cb5d84SJohn Baldwin vmx_msr_guest_exit(vcpu); 3228c3498942SNeel Natu 3229366f6083SPeter Grehan return (0); 3230366f6083SPeter Grehan } 3231366f6083SPeter Grehan 3232366f6083SPeter Grehan static void 3233869c8d19SJohn Baldwin vmx_vcpu_cleanup(void *vcpui) 3234366f6083SPeter Grehan { 32351aa51504SJohn Baldwin struct vmx_vcpu *vcpu = vcpui; 3236366f6083SPeter Grehan 32370f00260cSJohn Baldwin vpid_free(vcpu->state.vpid); 32380f00260cSJohn Baldwin free(vcpu->pir_desc, M_VMX); 32390f00260cSJohn Baldwin free(vcpu->apic_page, M_VMX); 32400f00260cSJohn Baldwin free(vcpu->vmcs, M_VMX); 32411aa51504SJohn Baldwin free(vcpu, M_VMX); 32420f00260cSJohn Baldwin } 324345e51299SNeel Natu 32441aa51504SJohn Baldwin static void 3245869c8d19SJohn Baldwin vmx_cleanup(void *vmi) 32461aa51504SJohn Baldwin { 3247869c8d19SJohn Baldwin struct vmx *vmx = vmi; 32481aa51504SJohn Baldwin 32491aa51504SJohn Baldwin if (virtual_interrupt_delivery) 32501aa51504SJohn Baldwin vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE); 32511aa51504SJohn Baldwin 32520f00260cSJohn Baldwin free(vmx->msr_bitmap, M_VMX); 3253366f6083SPeter Grehan free(vmx, M_VMX); 3254366f6083SPeter Grehan 3255366f6083SPeter Grehan return; 3256366f6083SPeter Grehan } 3257366f6083SPeter Grehan 3258366f6083SPeter Grehan static register_t * 3259366f6083SPeter Grehan vmxctx_regptr(struct vmxctx *vmxctx, int reg) 3260366f6083SPeter Grehan { 3261366f6083SPeter Grehan 3262366f6083SPeter Grehan switch (reg) { 3263366f6083SPeter Grehan case VM_REG_GUEST_RAX: 3264366f6083SPeter Grehan return (&vmxctx->guest_rax); 3265366f6083SPeter Grehan case VM_REG_GUEST_RBX: 3266366f6083SPeter Grehan return (&vmxctx->guest_rbx); 3267366f6083SPeter Grehan case VM_REG_GUEST_RCX: 3268366f6083SPeter Grehan return (&vmxctx->guest_rcx); 3269366f6083SPeter Grehan case VM_REG_GUEST_RDX: 3270366f6083SPeter Grehan return (&vmxctx->guest_rdx); 3271366f6083SPeter Grehan case VM_REG_GUEST_RSI: 3272366f6083SPeter Grehan return (&vmxctx->guest_rsi); 3273366f6083SPeter Grehan case VM_REG_GUEST_RDI: 3274366f6083SPeter Grehan return (&vmxctx->guest_rdi); 3275366f6083SPeter Grehan case VM_REG_GUEST_RBP: 3276366f6083SPeter Grehan return (&vmxctx->guest_rbp); 3277366f6083SPeter Grehan case VM_REG_GUEST_R8: 3278366f6083SPeter Grehan return (&vmxctx->guest_r8); 3279366f6083SPeter Grehan case VM_REG_GUEST_R9: 3280366f6083SPeter Grehan return (&vmxctx->guest_r9); 3281366f6083SPeter Grehan case VM_REG_GUEST_R10: 3282366f6083SPeter Grehan return (&vmxctx->guest_r10); 3283366f6083SPeter Grehan case VM_REG_GUEST_R11: 3284366f6083SPeter Grehan return (&vmxctx->guest_r11); 3285366f6083SPeter Grehan case VM_REG_GUEST_R12: 3286366f6083SPeter Grehan return (&vmxctx->guest_r12); 3287366f6083SPeter Grehan case VM_REG_GUEST_R13: 3288366f6083SPeter Grehan return (&vmxctx->guest_r13); 3289366f6083SPeter Grehan case VM_REG_GUEST_R14: 3290366f6083SPeter Grehan return (&vmxctx->guest_r14); 3291366f6083SPeter Grehan case VM_REG_GUEST_R15: 3292366f6083SPeter Grehan return (&vmxctx->guest_r15); 329337a723a5SNeel Natu case VM_REG_GUEST_CR2: 329437a723a5SNeel Natu return (&vmxctx->guest_cr2); 329565eefbe4SJohn Baldwin case VM_REG_GUEST_DR0: 329665eefbe4SJohn Baldwin return (&vmxctx->guest_dr0); 329765eefbe4SJohn Baldwin case VM_REG_GUEST_DR1: 329865eefbe4SJohn Baldwin return (&vmxctx->guest_dr1); 329965eefbe4SJohn Baldwin case VM_REG_GUEST_DR2: 330065eefbe4SJohn Baldwin return (&vmxctx->guest_dr2); 330165eefbe4SJohn Baldwin case VM_REG_GUEST_DR3: 330265eefbe4SJohn Baldwin return (&vmxctx->guest_dr3); 330365eefbe4SJohn Baldwin case VM_REG_GUEST_DR6: 330465eefbe4SJohn Baldwin return (&vmxctx->guest_dr6); 3305366f6083SPeter Grehan default: 3306366f6083SPeter Grehan break; 3307366f6083SPeter Grehan } 3308366f6083SPeter Grehan return (NULL); 3309366f6083SPeter Grehan } 3310366f6083SPeter Grehan 3311366f6083SPeter Grehan static int 3312366f6083SPeter Grehan vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval) 3313366f6083SPeter Grehan { 3314366f6083SPeter Grehan register_t *regp; 3315366f6083SPeter Grehan 3316366f6083SPeter Grehan if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) { 3317366f6083SPeter Grehan *retval = *regp; 3318366f6083SPeter Grehan return (0); 3319366f6083SPeter Grehan } else 3320366f6083SPeter Grehan return (EINVAL); 3321366f6083SPeter Grehan } 3322366f6083SPeter Grehan 3323366f6083SPeter Grehan static int 3324366f6083SPeter Grehan vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val) 3325366f6083SPeter Grehan { 3326366f6083SPeter Grehan register_t *regp; 3327366f6083SPeter Grehan 3328366f6083SPeter Grehan if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) { 3329366f6083SPeter Grehan *regp = val; 3330366f6083SPeter Grehan return (0); 3331366f6083SPeter Grehan } else 3332366f6083SPeter Grehan return (EINVAL); 3333366f6083SPeter Grehan } 3334366f6083SPeter Grehan 3335366f6083SPeter Grehan static int 33361aa51504SJohn Baldwin vmx_get_intr_shadow(struct vmx_vcpu *vcpu, int running, uint64_t *retval) 3337d1819632SNeel Natu { 3338d1819632SNeel Natu uint64_t gi; 3339d1819632SNeel Natu int error; 3340d1819632SNeel Natu 33411aa51504SJohn Baldwin error = vmcs_getreg(vcpu->vmcs, running, 3342d1819632SNeel Natu VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY), &gi); 3343d1819632SNeel Natu *retval = (gi & HWINTR_BLOCKING) ? 1 : 0; 3344d1819632SNeel Natu return (error); 3345d1819632SNeel Natu } 3346d1819632SNeel Natu 3347d1819632SNeel Natu static int 3348869c8d19SJohn Baldwin vmx_modify_intr_shadow(struct vmx_vcpu *vcpu, int running, uint64_t val) 3349d1819632SNeel Natu { 3350d1819632SNeel Natu struct vmcs *vmcs; 3351d1819632SNeel Natu uint64_t gi; 3352d1819632SNeel Natu int error, ident; 3353d1819632SNeel Natu 3354d1819632SNeel Natu /* 3355d1819632SNeel Natu * Forcing the vcpu into an interrupt shadow is not supported. 3356d1819632SNeel Natu */ 3357d1819632SNeel Natu if (val) { 3358d1819632SNeel Natu error = EINVAL; 3359d1819632SNeel Natu goto done; 3360d1819632SNeel Natu } 3361d1819632SNeel Natu 33621aa51504SJohn Baldwin vmcs = vcpu->vmcs; 3363d1819632SNeel Natu ident = VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY); 3364d1819632SNeel Natu error = vmcs_getreg(vmcs, running, ident, &gi); 3365d1819632SNeel Natu if (error == 0) { 3366d1819632SNeel Natu gi &= ~HWINTR_BLOCKING; 3367d1819632SNeel Natu error = vmcs_setreg(vmcs, running, ident, gi); 3368d1819632SNeel Natu } 3369d1819632SNeel Natu done: 337057e0119eSJohn Baldwin VMX_CTR2(vcpu, "Setting intr_shadow to %#lx %s", val, 337157e0119eSJohn Baldwin error ? "failed" : "succeeded"); 3372d1819632SNeel Natu return (error); 3373d1819632SNeel Natu } 3374d1819632SNeel Natu 3375d1819632SNeel Natu static int 3376aaaa0656SPeter Grehan vmx_shadow_reg(int reg) 3377aaaa0656SPeter Grehan { 3378aaaa0656SPeter Grehan int shreg; 3379aaaa0656SPeter Grehan 3380aaaa0656SPeter Grehan shreg = -1; 3381aaaa0656SPeter Grehan 3382aaaa0656SPeter Grehan switch (reg) { 3383aaaa0656SPeter Grehan case VM_REG_GUEST_CR0: 3384aaaa0656SPeter Grehan shreg = VMCS_CR0_SHADOW; 3385aaaa0656SPeter Grehan break; 3386aaaa0656SPeter Grehan case VM_REG_GUEST_CR4: 3387aaaa0656SPeter Grehan shreg = VMCS_CR4_SHADOW; 3388aaaa0656SPeter Grehan break; 3389aaaa0656SPeter Grehan default: 3390aaaa0656SPeter Grehan break; 3391aaaa0656SPeter Grehan } 3392aaaa0656SPeter Grehan 3393aaaa0656SPeter Grehan return (shreg); 3394aaaa0656SPeter Grehan } 3395aaaa0656SPeter Grehan 3396aaaa0656SPeter Grehan static int 3397869c8d19SJohn Baldwin vmx_getreg(void *vcpui, int reg, uint64_t *retval) 3398366f6083SPeter Grehan { 3399d3c11f40SPeter Grehan int running, hostcpu; 34001aa51504SJohn Baldwin struct vmx_vcpu *vcpu = vcpui; 3401869c8d19SJohn Baldwin struct vmx *vmx = vcpu->vmx; 3402366f6083SPeter Grehan 340380cb5d84SJohn Baldwin running = vcpu_is_running(vcpu->vcpu, &hostcpu); 3404d3c11f40SPeter Grehan if (running && hostcpu != curcpu) 34051aa51504SJohn Baldwin panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), 34061aa51504SJohn Baldwin vcpu->vcpuid); 3407d3c11f40SPeter Grehan 3408f493ea65SMark Johnston switch (reg) { 3409f493ea65SMark Johnston case VM_REG_GUEST_INTR_SHADOW: 34101aa51504SJohn Baldwin return (vmx_get_intr_shadow(vcpu, running, retval)); 3411f493ea65SMark Johnston case VM_REG_GUEST_KGS_BASE: 3412f493ea65SMark Johnston *retval = vcpu->guest_msrs[IDX_MSR_KGSBASE]; 3413f493ea65SMark Johnston return (0); 3414f493ea65SMark Johnston case VM_REG_GUEST_TPR: 3415f493ea65SMark Johnston *retval = vlapic_get_cr8(vm_lapic(vcpu->vcpu)); 3416f493ea65SMark Johnston return (0); 3417f493ea65SMark Johnston } 3418d1819632SNeel Natu 34191aa51504SJohn Baldwin if (vmxctx_getreg(&vcpu->ctx, reg, retval) == 0) 3420366f6083SPeter Grehan return (0); 3421366f6083SPeter Grehan 34221aa51504SJohn Baldwin return (vmcs_getreg(vcpu->vmcs, running, reg, retval)); 3423366f6083SPeter Grehan } 3424366f6083SPeter Grehan 3425366f6083SPeter Grehan static int 3426869c8d19SJohn Baldwin vmx_setreg(void *vcpui, int reg, uint64_t val) 3427366f6083SPeter Grehan { 3428aaaa0656SPeter Grehan int error, hostcpu, running, shadow; 3429366f6083SPeter Grehan uint64_t ctls; 34303527963bSNeel Natu pmap_t pmap; 34311aa51504SJohn Baldwin struct vmx_vcpu *vcpu = vcpui; 3432869c8d19SJohn Baldwin struct vmx *vmx = vcpu->vmx; 3433366f6083SPeter Grehan 343480cb5d84SJohn Baldwin running = vcpu_is_running(vcpu->vcpu, &hostcpu); 3435d3c11f40SPeter Grehan if (running && hostcpu != curcpu) 34361aa51504SJohn Baldwin panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), 34371aa51504SJohn Baldwin vcpu->vcpuid); 3438d3c11f40SPeter Grehan 3439d1819632SNeel Natu if (reg == VM_REG_GUEST_INTR_SHADOW) 3440869c8d19SJohn Baldwin return (vmx_modify_intr_shadow(vcpu, running, val)); 3441d1819632SNeel Natu 34421aa51504SJohn Baldwin if (vmxctx_setreg(&vcpu->ctx, reg, val) == 0) 3443366f6083SPeter Grehan return (0); 3444366f6083SPeter Grehan 344509860d44SEd Maste /* Do not permit user write access to VMCS fields by offset. */ 344609860d44SEd Maste if (reg < 0) 344709860d44SEd Maste return (EINVAL); 344809860d44SEd Maste 34491aa51504SJohn Baldwin error = vmcs_setreg(vcpu->vmcs, running, reg, val); 3450366f6083SPeter Grehan 3451366f6083SPeter Grehan if (error == 0) { 3452366f6083SPeter Grehan /* 3453366f6083SPeter Grehan * If the "load EFER" VM-entry control is 1 then the 3454366f6083SPeter Grehan * value of EFER.LMA must be identical to "IA-32e mode guest" 3455366f6083SPeter Grehan * bit in the VM-entry control. 3456366f6083SPeter Grehan */ 3457366f6083SPeter Grehan if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 && 3458366f6083SPeter Grehan (reg == VM_REG_GUEST_EFER)) { 34591aa51504SJohn Baldwin vmcs_getreg(vcpu->vmcs, running, 3460366f6083SPeter Grehan VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls); 3461366f6083SPeter Grehan if (val & EFER_LMA) 3462366f6083SPeter Grehan ctls |= VM_ENTRY_GUEST_LMA; 3463366f6083SPeter Grehan else 3464366f6083SPeter Grehan ctls &= ~VM_ENTRY_GUEST_LMA; 34651aa51504SJohn Baldwin vmcs_setreg(vcpu->vmcs, running, 3466366f6083SPeter Grehan VMCS_IDENT(VMCS_ENTRY_CTLS), ctls); 3467366f6083SPeter Grehan } 3468aaaa0656SPeter Grehan 3469aaaa0656SPeter Grehan shadow = vmx_shadow_reg(reg); 3470aaaa0656SPeter Grehan if (shadow > 0) { 3471aaaa0656SPeter Grehan /* 3472aaaa0656SPeter Grehan * Store the unmodified value in the shadow 3473aaaa0656SPeter Grehan */ 34741aa51504SJohn Baldwin error = vmcs_setreg(vcpu->vmcs, running, 3475aaaa0656SPeter Grehan VMCS_IDENT(shadow), val); 3476aaaa0656SPeter Grehan } 34773527963bSNeel Natu 34783527963bSNeel Natu if (reg == VM_REG_GUEST_CR3) { 34793527963bSNeel Natu /* 34803527963bSNeel Natu * Invalidate the guest vcpu's TLB mappings to emulate 34813527963bSNeel Natu * the behavior of updating %cr3. 34823527963bSNeel Natu * 34833527963bSNeel Natu * XXX the processor retains global mappings when %cr3 34843527963bSNeel Natu * is updated but vmx_invvpid() does not. 34853527963bSNeel Natu */ 34861aa51504SJohn Baldwin pmap = vcpu->ctx.pmap; 34873527963bSNeel Natu vmx_invvpid(vmx, vcpu, pmap, running); 34883527963bSNeel Natu } 3489366f6083SPeter Grehan } 3490366f6083SPeter Grehan 3491366f6083SPeter Grehan return (error); 3492366f6083SPeter Grehan } 3493366f6083SPeter Grehan 3494366f6083SPeter Grehan static int 3495869c8d19SJohn Baldwin vmx_getdesc(void *vcpui, int reg, struct seg_desc *desc) 3496366f6083SPeter Grehan { 3497ba6f5e23SNeel Natu int hostcpu, running; 34981aa51504SJohn Baldwin struct vmx_vcpu *vcpu = vcpui; 3499869c8d19SJohn Baldwin struct vmx *vmx = vcpu->vmx; 3500366f6083SPeter Grehan 350180cb5d84SJohn Baldwin running = vcpu_is_running(vcpu->vcpu, &hostcpu); 3502ba6f5e23SNeel Natu if (running && hostcpu != curcpu) 35031aa51504SJohn Baldwin panic("vmx_getdesc: %s%d is running", vm_name(vmx->vm), 35041aa51504SJohn Baldwin vcpu->vcpuid); 3505ba6f5e23SNeel Natu 35061aa51504SJohn Baldwin return (vmcs_getdesc(vcpu->vmcs, running, reg, desc)); 3507366f6083SPeter Grehan } 3508366f6083SPeter Grehan 3509366f6083SPeter Grehan static int 3510869c8d19SJohn Baldwin vmx_setdesc(void *vcpui, int reg, struct seg_desc *desc) 3511366f6083SPeter Grehan { 3512ba6f5e23SNeel Natu int hostcpu, running; 35131aa51504SJohn Baldwin struct vmx_vcpu *vcpu = vcpui; 3514869c8d19SJohn Baldwin struct vmx *vmx = vcpu->vmx; 3515366f6083SPeter Grehan 351680cb5d84SJohn Baldwin running = vcpu_is_running(vcpu->vcpu, &hostcpu); 3517ba6f5e23SNeel Natu if (running && hostcpu != curcpu) 35181aa51504SJohn Baldwin panic("vmx_setdesc: %s%d is running", vm_name(vmx->vm), 35191aa51504SJohn Baldwin vcpu->vcpuid); 3520ba6f5e23SNeel Natu 35211aa51504SJohn Baldwin return (vmcs_setdesc(vcpu->vmcs, running, reg, desc)); 3522366f6083SPeter Grehan } 3523366f6083SPeter Grehan 3524366f6083SPeter Grehan static int 3525869c8d19SJohn Baldwin vmx_getcap(void *vcpui, int type, int *retval) 3526366f6083SPeter Grehan { 35271aa51504SJohn Baldwin struct vmx_vcpu *vcpu = vcpui; 3528366f6083SPeter Grehan int vcap; 3529366f6083SPeter Grehan int ret; 3530366f6083SPeter Grehan 3531366f6083SPeter Grehan ret = ENOENT; 3532366f6083SPeter Grehan 35331aa51504SJohn Baldwin vcap = vcpu->cap.set; 3534366f6083SPeter Grehan 3535366f6083SPeter Grehan switch (type) { 3536366f6083SPeter Grehan case VM_CAP_HALT_EXIT: 3537366f6083SPeter Grehan if (cap_halt_exit) 3538366f6083SPeter Grehan ret = 0; 3539366f6083SPeter Grehan break; 3540366f6083SPeter Grehan case VM_CAP_PAUSE_EXIT: 3541366f6083SPeter Grehan if (cap_pause_exit) 3542366f6083SPeter Grehan ret = 0; 3543366f6083SPeter Grehan break; 3544366f6083SPeter Grehan case VM_CAP_MTRAP_EXIT: 3545366f6083SPeter Grehan if (cap_monitor_trap) 3546366f6083SPeter Grehan ret = 0; 3547366f6083SPeter Grehan break; 3548f5f5f1e7SPeter Grehan case VM_CAP_RDPID: 3549f5f5f1e7SPeter Grehan if (cap_rdpid) 3550f5f5f1e7SPeter Grehan ret = 0; 3551f5f5f1e7SPeter Grehan break; 3552f5f5f1e7SPeter Grehan case VM_CAP_RDTSCP: 3553f5f5f1e7SPeter Grehan if (cap_rdtscp) 3554f5f5f1e7SPeter Grehan ret = 0; 3555f5f5f1e7SPeter Grehan break; 3556366f6083SPeter Grehan case VM_CAP_UNRESTRICTED_GUEST: 3557366f6083SPeter Grehan if (cap_unrestricted_guest) 3558366f6083SPeter Grehan ret = 0; 3559366f6083SPeter Grehan break; 356049cc03daSNeel Natu case VM_CAP_ENABLE_INVPCID: 356149cc03daSNeel Natu if (cap_invpcid) 356249cc03daSNeel Natu ret = 0; 356349cc03daSNeel Natu break; 3564cbd03a9dSJohn Baldwin case VM_CAP_BPT_EXIT: 35650bda8d3eSCorvin Köhne case VM_CAP_IPI_EXIT: 3566cbd03a9dSJohn Baldwin ret = 0; 3567cbd03a9dSJohn Baldwin break; 3568366f6083SPeter Grehan default: 3569366f6083SPeter Grehan break; 3570366f6083SPeter Grehan } 3571366f6083SPeter Grehan 3572366f6083SPeter Grehan if (ret == 0) 3573366f6083SPeter Grehan *retval = (vcap & (1 << type)) ? 1 : 0; 3574366f6083SPeter Grehan 3575366f6083SPeter Grehan return (ret); 3576366f6083SPeter Grehan } 3577366f6083SPeter Grehan 3578366f6083SPeter Grehan static int 3579869c8d19SJohn Baldwin vmx_setcap(void *vcpui, int type, int val) 3580366f6083SPeter Grehan { 35811aa51504SJohn Baldwin struct vmx_vcpu *vcpu = vcpui; 35821aa51504SJohn Baldwin struct vmcs *vmcs = vcpu->vmcs; 35830bda8d3eSCorvin Köhne struct vlapic *vlapic; 3584366f6083SPeter Grehan uint32_t baseval; 3585366f6083SPeter Grehan uint32_t *pptr; 3586366f6083SPeter Grehan int error; 3587366f6083SPeter Grehan int flag; 3588366f6083SPeter Grehan int reg; 3589366f6083SPeter Grehan int retval; 3590366f6083SPeter Grehan 3591366f6083SPeter Grehan retval = ENOENT; 3592366f6083SPeter Grehan pptr = NULL; 3593366f6083SPeter Grehan 3594366f6083SPeter Grehan switch (type) { 3595366f6083SPeter Grehan case VM_CAP_HALT_EXIT: 3596366f6083SPeter Grehan if (cap_halt_exit) { 3597366f6083SPeter Grehan retval = 0; 35981aa51504SJohn Baldwin pptr = &vcpu->cap.proc_ctls; 3599366f6083SPeter Grehan baseval = *pptr; 3600366f6083SPeter Grehan flag = PROCBASED_HLT_EXITING; 3601366f6083SPeter Grehan reg = VMCS_PRI_PROC_BASED_CTLS; 3602366f6083SPeter Grehan } 3603366f6083SPeter Grehan break; 3604366f6083SPeter Grehan case VM_CAP_MTRAP_EXIT: 3605366f6083SPeter Grehan if (cap_monitor_trap) { 3606366f6083SPeter Grehan retval = 0; 36071aa51504SJohn Baldwin pptr = &vcpu->cap.proc_ctls; 3608366f6083SPeter Grehan baseval = *pptr; 3609366f6083SPeter Grehan flag = PROCBASED_MTF; 3610366f6083SPeter Grehan reg = VMCS_PRI_PROC_BASED_CTLS; 3611366f6083SPeter Grehan } 3612366f6083SPeter Grehan break; 3613366f6083SPeter Grehan case VM_CAP_PAUSE_EXIT: 3614366f6083SPeter Grehan if (cap_pause_exit) { 3615366f6083SPeter Grehan retval = 0; 36161aa51504SJohn Baldwin pptr = &vcpu->cap.proc_ctls; 3617366f6083SPeter Grehan baseval = *pptr; 3618366f6083SPeter Grehan flag = PROCBASED_PAUSE_EXITING; 3619366f6083SPeter Grehan reg = VMCS_PRI_PROC_BASED_CTLS; 3620366f6083SPeter Grehan } 3621366f6083SPeter Grehan break; 3622f5f5f1e7SPeter Grehan case VM_CAP_RDPID: 3623f5f5f1e7SPeter Grehan case VM_CAP_RDTSCP: 3624f5f5f1e7SPeter Grehan if (cap_rdpid || cap_rdtscp) 3625f5f5f1e7SPeter Grehan /* 3626f5f5f1e7SPeter Grehan * Choose not to support enabling/disabling 3627f5f5f1e7SPeter Grehan * RDPID/RDTSCP via libvmmapi since, as per the 362815add60dSPeter Grehan * discussion in vmx_modinit(), RDPID/RDTSCP are 3629f5f5f1e7SPeter Grehan * either always enabled or always disabled. 3630f5f5f1e7SPeter Grehan */ 3631f5f5f1e7SPeter Grehan error = EOPNOTSUPP; 3632f5f5f1e7SPeter Grehan break; 3633366f6083SPeter Grehan case VM_CAP_UNRESTRICTED_GUEST: 3634366f6083SPeter Grehan if (cap_unrestricted_guest) { 3635366f6083SPeter Grehan retval = 0; 36361aa51504SJohn Baldwin pptr = &vcpu->cap.proc_ctls2; 363749cc03daSNeel Natu baseval = *pptr; 3638366f6083SPeter Grehan flag = PROCBASED2_UNRESTRICTED_GUEST; 3639366f6083SPeter Grehan reg = VMCS_SEC_PROC_BASED_CTLS; 3640366f6083SPeter Grehan } 3641366f6083SPeter Grehan break; 364249cc03daSNeel Natu case VM_CAP_ENABLE_INVPCID: 364349cc03daSNeel Natu if (cap_invpcid) { 364449cc03daSNeel Natu retval = 0; 36451aa51504SJohn Baldwin pptr = &vcpu->cap.proc_ctls2; 364649cc03daSNeel Natu baseval = *pptr; 364749cc03daSNeel Natu flag = PROCBASED2_ENABLE_INVPCID; 364849cc03daSNeel Natu reg = VMCS_SEC_PROC_BASED_CTLS; 364949cc03daSNeel Natu } 365049cc03daSNeel Natu break; 3651cbd03a9dSJohn Baldwin case VM_CAP_BPT_EXIT: 3652cbd03a9dSJohn Baldwin retval = 0; 3653cbd03a9dSJohn Baldwin 3654cbd03a9dSJohn Baldwin /* Don't change the bitmap if we are tracing all exceptions. */ 36551aa51504SJohn Baldwin if (vcpu->cap.exc_bitmap != 0xffffffff) { 36561aa51504SJohn Baldwin pptr = &vcpu->cap.exc_bitmap; 3657cbd03a9dSJohn Baldwin baseval = *pptr; 3658cbd03a9dSJohn Baldwin flag = (1 << IDT_BP); 3659cbd03a9dSJohn Baldwin reg = VMCS_EXCEPTION_BITMAP; 3660cbd03a9dSJohn Baldwin } 3661cbd03a9dSJohn Baldwin break; 36620bda8d3eSCorvin Köhne case VM_CAP_IPI_EXIT: 36630bda8d3eSCorvin Köhne retval = 0; 36640bda8d3eSCorvin Köhne 3665d3956e46SJohn Baldwin vlapic = vm_lapic(vcpu->vcpu); 36660bda8d3eSCorvin Köhne vlapic->ipi_exit = val; 36670bda8d3eSCorvin Köhne break; 3668fefac543SBojan Novković case VM_CAP_MASK_HWINTR: 3669fefac543SBojan Novković retval = 0; 3670fefac543SBojan Novković break; 3671366f6083SPeter Grehan default: 3672366f6083SPeter Grehan break; 3673366f6083SPeter Grehan } 3674366f6083SPeter Grehan 3675cbd03a9dSJohn Baldwin if (retval) 3676cbd03a9dSJohn Baldwin return (retval); 3677cbd03a9dSJohn Baldwin 3678cbd03a9dSJohn Baldwin if (pptr != NULL) { 3679366f6083SPeter Grehan if (val) { 3680366f6083SPeter Grehan baseval |= flag; 3681366f6083SPeter Grehan } else { 3682366f6083SPeter Grehan baseval &= ~flag; 3683366f6083SPeter Grehan } 3684366f6083SPeter Grehan VMPTRLD(vmcs); 3685366f6083SPeter Grehan error = vmwrite(reg, baseval); 3686366f6083SPeter Grehan VMCLEAR(vmcs); 3687366f6083SPeter Grehan 3688cbd03a9dSJohn Baldwin if (error) 3689cbd03a9dSJohn Baldwin return (error); 3690cbd03a9dSJohn Baldwin 3691366f6083SPeter Grehan /* 3692366f6083SPeter Grehan * Update optional stored flags, and record 3693366f6083SPeter Grehan * setting 3694366f6083SPeter Grehan */ 3695366f6083SPeter Grehan *pptr = baseval; 3696366f6083SPeter Grehan } 3697366f6083SPeter Grehan 3698366f6083SPeter Grehan if (val) { 36991aa51504SJohn Baldwin vcpu->cap.set |= (1 << type); 3700366f6083SPeter Grehan } else { 37011aa51504SJohn Baldwin vcpu->cap.set &= ~(1 << type); 3702366f6083SPeter Grehan } 3703366f6083SPeter Grehan 3704cbd03a9dSJohn Baldwin return (0); 3705366f6083SPeter Grehan } 3706366f6083SPeter Grehan 370715add60dSPeter Grehan static struct vmspace * 370815add60dSPeter Grehan vmx_vmspace_alloc(vm_offset_t min, vm_offset_t max) 370915add60dSPeter Grehan { 371015add60dSPeter Grehan return (ept_vmspace_alloc(min, max)); 371115add60dSPeter Grehan } 371215add60dSPeter Grehan 371315add60dSPeter Grehan static void 371415add60dSPeter Grehan vmx_vmspace_free(struct vmspace *vmspace) 371515add60dSPeter Grehan { 371615add60dSPeter Grehan ept_vmspace_free(vmspace); 371715add60dSPeter Grehan } 371815add60dSPeter Grehan 371988c4b8d1SNeel Natu struct vlapic_vtx { 372088c4b8d1SNeel Natu struct vlapic vlapic; 3721176666c2SNeel Natu struct pir_desc *pir_desc; 37221aa51504SJohn Baldwin struct vmx_vcpu *vcpu; 37232c352febSJohn Baldwin u_int pending_prio; 372488c4b8d1SNeel Natu }; 372588c4b8d1SNeel Natu 37262c352febSJohn Baldwin #define VPR_PRIO_BIT(vpr) (1 << ((vpr) >> 4)) 37272c352febSJohn Baldwin 3728d030f941SJohn Baldwin #define VMX_CTR_PIR(vlapic, pir_desc, notify, vector, level, msg) \ 372988c4b8d1SNeel Natu do { \ 3730d030f941SJohn Baldwin VLAPIC_CTR2(vlapic, msg " assert %s-triggered vector %d", \ 373188c4b8d1SNeel Natu level ? "level" : "edge", vector); \ 3732d030f941SJohn Baldwin VLAPIC_CTR1(vlapic, msg " pir0 0x%016lx", pir_desc->pir[0]); \ 3733d030f941SJohn Baldwin VLAPIC_CTR1(vlapic, msg " pir1 0x%016lx", pir_desc->pir[1]); \ 3734d030f941SJohn Baldwin VLAPIC_CTR1(vlapic, msg " pir2 0x%016lx", pir_desc->pir[2]); \ 3735d030f941SJohn Baldwin VLAPIC_CTR1(vlapic, msg " pir3 0x%016lx", pir_desc->pir[3]); \ 3736d030f941SJohn Baldwin VLAPIC_CTR1(vlapic, msg " notify: %s", notify ? "yes" : "no"); \ 373788c4b8d1SNeel Natu } while (0) 373888c4b8d1SNeel Natu 373988c4b8d1SNeel Natu /* 374088c4b8d1SNeel Natu * vlapic->ops handlers that utilize the APICv hardware assist described in 374188c4b8d1SNeel Natu * Chapter 29 of the Intel SDM. 374288c4b8d1SNeel Natu */ 374388c4b8d1SNeel Natu static int 374488c4b8d1SNeel Natu vmx_set_intr_ready(struct vlapic *vlapic, int vector, bool level) 374588c4b8d1SNeel Natu { 374688c4b8d1SNeel Natu struct vlapic_vtx *vlapic_vtx; 374788c4b8d1SNeel Natu struct pir_desc *pir_desc; 374888c4b8d1SNeel Natu uint64_t mask; 37492c352febSJohn Baldwin int idx, notify = 0; 375088c4b8d1SNeel Natu 375188c4b8d1SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 3752176666c2SNeel Natu pir_desc = vlapic_vtx->pir_desc; 375388c4b8d1SNeel Natu 375488c4b8d1SNeel Natu /* 375588c4b8d1SNeel Natu * Keep track of interrupt requests in the PIR descriptor. This is 375688c4b8d1SNeel Natu * because the virtual APIC page pointed to by the VMCS cannot be 375788c4b8d1SNeel Natu * modified if the vcpu is running. 375888c4b8d1SNeel Natu */ 375988c4b8d1SNeel Natu idx = vector / 64; 376088c4b8d1SNeel Natu mask = 1UL << (vector % 64); 376188c4b8d1SNeel Natu atomic_set_long(&pir_desc->pir[idx], mask); 37622c352febSJohn Baldwin 37632c352febSJohn Baldwin /* 37642c352febSJohn Baldwin * A notification is required whenever the 'pending' bit makes a 37652c352febSJohn Baldwin * transition from 0->1. 37662c352febSJohn Baldwin * 37672c352febSJohn Baldwin * Even if the 'pending' bit is already asserted, notification about 37682c352febSJohn Baldwin * the incoming interrupt may still be necessary. For example, if a 37692c352febSJohn Baldwin * vCPU is HLTed with a high PPR, a low priority interrupt would cause 37702c352febSJohn Baldwin * the 0->1 'pending' transition with a notification, but the vCPU 37712c352febSJohn Baldwin * would ignore the interrupt for the time being. The same vCPU would 37722c352febSJohn Baldwin * need to then be notified if a high-priority interrupt arrived which 37732c352febSJohn Baldwin * satisfied the PPR. 37742c352febSJohn Baldwin * 37752c352febSJohn Baldwin * The priorities of interrupts injected while 'pending' is asserted 37762c352febSJohn Baldwin * are tracked in a custom bitfield 'pending_prio'. Should the 37772c352febSJohn Baldwin * to-be-injected interrupt exceed the priorities already present, the 37782c352febSJohn Baldwin * notification is sent. The priorities recorded in 'pending_prio' are 37792c352febSJohn Baldwin * cleared whenever the 'pending' bit makes another 0->1 transition. 37802c352febSJohn Baldwin */ 37812c352febSJohn Baldwin if (atomic_cmpset_long(&pir_desc->pending, 0, 1) != 0) { 37822c352febSJohn Baldwin notify = 1; 37832c352febSJohn Baldwin vlapic_vtx->pending_prio = 0; 37842c352febSJohn Baldwin } else { 37852c352febSJohn Baldwin const u_int old_prio = vlapic_vtx->pending_prio; 37862c352febSJohn Baldwin const u_int prio_bit = VPR_PRIO_BIT(vector & APIC_TPR_INT); 37872c352febSJohn Baldwin 37882c352febSJohn Baldwin if ((old_prio & prio_bit) == 0 && prio_bit > old_prio) { 37892c352febSJohn Baldwin atomic_set_int(&vlapic_vtx->pending_prio, prio_bit); 37902c352febSJohn Baldwin notify = 1; 37912c352febSJohn Baldwin } 37922c352febSJohn Baldwin } 379388c4b8d1SNeel Natu 3794d030f941SJohn Baldwin VMX_CTR_PIR(vlapic, pir_desc, notify, vector, level, 3795d030f941SJohn Baldwin "vmx_set_intr_ready"); 379688c4b8d1SNeel Natu return (notify); 379788c4b8d1SNeel Natu } 379888c4b8d1SNeel Natu 379988c4b8d1SNeel Natu static int 380088c4b8d1SNeel Natu vmx_pending_intr(struct vlapic *vlapic, int *vecptr) 380188c4b8d1SNeel Natu { 380288c4b8d1SNeel Natu struct vlapic_vtx *vlapic_vtx; 380388c4b8d1SNeel Natu struct pir_desc *pir_desc; 380488c4b8d1SNeel Natu struct LAPIC *lapic; 380588c4b8d1SNeel Natu uint64_t pending, pirval; 38060912408aSVitaliy Gusev uint8_t ppr, vpr, rvi; 38070912408aSVitaliy Gusev struct vm_exit *vmexit; 380888c4b8d1SNeel Natu int i; 380988c4b8d1SNeel Natu 381088c4b8d1SNeel Natu /* 381188c4b8d1SNeel Natu * This function is only expected to be called from the 'HLT' exit 381288c4b8d1SNeel Natu * handler which does not care about the vector that is pending. 381388c4b8d1SNeel Natu */ 381488c4b8d1SNeel Natu KASSERT(vecptr == NULL, ("vmx_pending_intr: vecptr must be NULL")); 381588c4b8d1SNeel Natu 381688c4b8d1SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 3817176666c2SNeel Natu pir_desc = vlapic_vtx->pir_desc; 38180912408aSVitaliy Gusev lapic = vlapic->apic_page; 381988c4b8d1SNeel Natu 38209e33a616STycho Nightingale /* 38219e33a616STycho Nightingale * While a virtual interrupt may have already been 38229e33a616STycho Nightingale * processed the actual delivery maybe pending the 38239e33a616STycho Nightingale * interruptibility of the guest. Recognize a pending 38249e33a616STycho Nightingale * interrupt by reevaluating virtual interrupts 38250912408aSVitaliy Gusev * following Section 30.2.1 in the Intel SDM Volume 3. 38269e33a616STycho Nightingale */ 382780cb5d84SJohn Baldwin vmexit = vm_exitinfo(vlapic->vcpu); 3828490768e2STycho Nightingale KASSERT(vmexit->exitcode == VM_EXITCODE_HLT, 3829490768e2STycho Nightingale ("vmx_pending_intr: exitcode not 'HLT'")); 3830490768e2STycho Nightingale rvi = vmexit->u.hlt.intr_status & APIC_TPR_INT; 38319e33a616STycho Nightingale ppr = lapic->ppr & APIC_TPR_INT; 38320912408aSVitaliy Gusev if (rvi > ppr) 38339e33a616STycho Nightingale return (1); 38349e33a616STycho Nightingale 38350912408aSVitaliy Gusev pending = atomic_load_acq_long(&pir_desc->pending); 38360912408aSVitaliy Gusev if (!pending) 38379e33a616STycho Nightingale return (0); 383888c4b8d1SNeel Natu 383988c4b8d1SNeel Natu /* 384088c4b8d1SNeel Natu * If there is an interrupt pending then it will be recognized only 384188c4b8d1SNeel Natu * if its priority is greater than the processor priority. 384288c4b8d1SNeel Natu * 384388c4b8d1SNeel Natu * Special case: if the processor priority is zero then any pending 384488c4b8d1SNeel Natu * interrupt will be recognized. 384588c4b8d1SNeel Natu */ 384688c4b8d1SNeel Natu if (ppr == 0) 384788c4b8d1SNeel Natu return (1); 384888c4b8d1SNeel Natu 3849d030f941SJohn Baldwin VLAPIC_CTR1(vlapic, "HLT with non-zero PPR %d", lapic->ppr); 385088c4b8d1SNeel Natu 38512c352febSJohn Baldwin vpr = 0; 385288c4b8d1SNeel Natu for (i = 3; i >= 0; i--) { 385388c4b8d1SNeel Natu pirval = pir_desc->pir[i]; 385488c4b8d1SNeel Natu if (pirval != 0) { 38559e33a616STycho Nightingale vpr = (i * 64 + flsl(pirval) - 1) & APIC_TPR_INT; 38562c352febSJohn Baldwin break; 385788c4b8d1SNeel Natu } 385888c4b8d1SNeel Natu } 38592c352febSJohn Baldwin 38602c352febSJohn Baldwin /* 38612c352febSJohn Baldwin * If the highest-priority pending interrupt falls short of the 38622c352febSJohn Baldwin * processor priority of this vCPU, ensure that 'pending_prio' does not 38632c352febSJohn Baldwin * have any stale bits which would preclude a higher-priority interrupt 38642c352febSJohn Baldwin * from incurring a notification later. 38652c352febSJohn Baldwin */ 38662c352febSJohn Baldwin if (vpr <= ppr) { 38672c352febSJohn Baldwin const u_int prio_bit = VPR_PRIO_BIT(vpr); 38682c352febSJohn Baldwin const u_int old = vlapic_vtx->pending_prio; 38692c352febSJohn Baldwin 38702c352febSJohn Baldwin if (old > prio_bit && (old & prio_bit) == 0) { 38712c352febSJohn Baldwin vlapic_vtx->pending_prio = prio_bit; 38722c352febSJohn Baldwin } 387388c4b8d1SNeel Natu return (0); 387488c4b8d1SNeel Natu } 38752c352febSJohn Baldwin return (1); 38762c352febSJohn Baldwin } 387788c4b8d1SNeel Natu 387888c4b8d1SNeel Natu static void 387988c4b8d1SNeel Natu vmx_intr_accepted(struct vlapic *vlapic, int vector) 388088c4b8d1SNeel Natu { 388188c4b8d1SNeel Natu 388288c4b8d1SNeel Natu panic("vmx_intr_accepted: not expected to be called"); 388388c4b8d1SNeel Natu } 388488c4b8d1SNeel Natu 3885176666c2SNeel Natu static void 388630b94db8SNeel Natu vmx_set_tmr(struct vlapic *vlapic, int vector, bool level) 388730b94db8SNeel Natu { 388830b94db8SNeel Natu struct vlapic_vtx *vlapic_vtx; 388930b94db8SNeel Natu struct vmcs *vmcs; 389030b94db8SNeel Natu uint64_t mask, val; 389130b94db8SNeel Natu 389230b94db8SNeel Natu KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector)); 389380cb5d84SJohn Baldwin KASSERT(!vcpu_is_running(vlapic->vcpu, NULL), 389430b94db8SNeel Natu ("vmx_set_tmr: vcpu cannot be running")); 389530b94db8SNeel Natu 389630b94db8SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 38971aa51504SJohn Baldwin vmcs = vlapic_vtx->vcpu->vmcs; 389830b94db8SNeel Natu mask = 1UL << (vector % 64); 389930b94db8SNeel Natu 390030b94db8SNeel Natu VMPTRLD(vmcs); 390130b94db8SNeel Natu val = vmcs_read(VMCS_EOI_EXIT(vector)); 390230b94db8SNeel Natu if (level) 390330b94db8SNeel Natu val |= mask; 390430b94db8SNeel Natu else 390530b94db8SNeel Natu val &= ~mask; 390630b94db8SNeel Natu vmcs_write(VMCS_EOI_EXIT(vector), val); 390730b94db8SNeel Natu VMCLEAR(vmcs); 390830b94db8SNeel Natu } 390930b94db8SNeel Natu 391030b94db8SNeel Natu static void 39111bc51badSMichael Reifenberger vmx_enable_x2apic_mode_ts(struct vlapic *vlapic) 39121bc51badSMichael Reifenberger { 39131aa51504SJohn Baldwin struct vlapic_vtx *vlapic_vtx; 39140f00260cSJohn Baldwin struct vmx_vcpu *vcpu; 39151bc51badSMichael Reifenberger struct vmcs *vmcs; 39161bc51badSMichael Reifenberger uint32_t proc_ctls; 39171bc51badSMichael Reifenberger 39181aa51504SJohn Baldwin vlapic_vtx = (struct vlapic_vtx *)vlapic; 39191aa51504SJohn Baldwin vcpu = vlapic_vtx->vcpu; 39200f00260cSJohn Baldwin vmcs = vcpu->vmcs; 39211bc51badSMichael Reifenberger 39220f00260cSJohn Baldwin proc_ctls = vcpu->cap.proc_ctls; 39231bc51badSMichael Reifenberger proc_ctls &= ~PROCBASED_USE_TPR_SHADOW; 39241bc51badSMichael Reifenberger proc_ctls |= PROCBASED_CR8_LOAD_EXITING; 39251bc51badSMichael Reifenberger proc_ctls |= PROCBASED_CR8_STORE_EXITING; 39260f00260cSJohn Baldwin vcpu->cap.proc_ctls = proc_ctls; 39271bc51badSMichael Reifenberger 39281bc51badSMichael Reifenberger VMPTRLD(vmcs); 39291bc51badSMichael Reifenberger vmcs_write(VMCS_PRI_PROC_BASED_CTLS, proc_ctls); 39301bc51badSMichael Reifenberger VMCLEAR(vmcs); 39311bc51badSMichael Reifenberger } 39321bc51badSMichael Reifenberger 39331bc51badSMichael Reifenberger static void 39341bc51badSMichael Reifenberger vmx_enable_x2apic_mode_vid(struct vlapic *vlapic) 3935159dd56fSNeel Natu { 39361aa51504SJohn Baldwin struct vlapic_vtx *vlapic_vtx; 3937159dd56fSNeel Natu struct vmx *vmx; 39380f00260cSJohn Baldwin struct vmx_vcpu *vcpu; 3939159dd56fSNeel Natu struct vmcs *vmcs; 3940159dd56fSNeel Natu uint32_t proc_ctls2; 39411aa51504SJohn Baldwin int error __diagused; 3942159dd56fSNeel Natu 39431aa51504SJohn Baldwin vlapic_vtx = (struct vlapic_vtx *)vlapic; 39441aa51504SJohn Baldwin vcpu = vlapic_vtx->vcpu; 3945869c8d19SJohn Baldwin vmx = vcpu->vmx; 39460f00260cSJohn Baldwin vmcs = vcpu->vmcs; 3947159dd56fSNeel Natu 39480f00260cSJohn Baldwin proc_ctls2 = vcpu->cap.proc_ctls2; 3949159dd56fSNeel Natu KASSERT((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) != 0, 3950159dd56fSNeel Natu ("%s: invalid proc_ctls2 %#x", __func__, proc_ctls2)); 3951159dd56fSNeel Natu 3952159dd56fSNeel Natu proc_ctls2 &= ~PROCBASED2_VIRTUALIZE_APIC_ACCESSES; 3953159dd56fSNeel Natu proc_ctls2 |= PROCBASED2_VIRTUALIZE_X2APIC_MODE; 39540f00260cSJohn Baldwin vcpu->cap.proc_ctls2 = proc_ctls2; 3955159dd56fSNeel Natu 3956159dd56fSNeel Natu VMPTRLD(vmcs); 3957159dd56fSNeel Natu vmcs_write(VMCS_SEC_PROC_BASED_CTLS, proc_ctls2); 3958159dd56fSNeel Natu VMCLEAR(vmcs); 3959159dd56fSNeel Natu 3960159dd56fSNeel Natu if (vlapic->vcpuid == 0) { 3961159dd56fSNeel Natu /* 3962159dd56fSNeel Natu * The nested page table mappings are shared by all vcpus 3963159dd56fSNeel Natu * so unmap the APIC access page just once. 3964159dd56fSNeel Natu */ 3965159dd56fSNeel Natu error = vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE); 3966159dd56fSNeel Natu KASSERT(error == 0, ("%s: vm_unmap_mmio error %d", 3967159dd56fSNeel Natu __func__, error)); 3968159dd56fSNeel Natu 3969159dd56fSNeel Natu /* 3970159dd56fSNeel Natu * The MSR bitmap is shared by all vcpus so modify it only 3971159dd56fSNeel Natu * once in the context of vcpu 0. 3972159dd56fSNeel Natu */ 3973159dd56fSNeel Natu error = vmx_allow_x2apic_msrs(vmx); 3974159dd56fSNeel Natu KASSERT(error == 0, ("%s: vmx_allow_x2apic_msrs error %d", 3975159dd56fSNeel Natu __func__, error)); 3976159dd56fSNeel Natu } 3977159dd56fSNeel Natu } 3978159dd56fSNeel Natu 3979159dd56fSNeel Natu static void 3980176666c2SNeel Natu vmx_post_intr(struct vlapic *vlapic, int hostcpu) 3981176666c2SNeel Natu { 3982176666c2SNeel Natu 3983176666c2SNeel Natu ipi_cpu(hostcpu, pirvec); 3984176666c2SNeel Natu } 3985176666c2SNeel Natu 398688c4b8d1SNeel Natu /* 398788c4b8d1SNeel Natu * Transfer the pending interrupts in the PIR descriptor to the IRR 398888c4b8d1SNeel Natu * in the virtual APIC page. 398988c4b8d1SNeel Natu */ 399088c4b8d1SNeel Natu static void 399188c4b8d1SNeel Natu vmx_inject_pir(struct vlapic *vlapic) 399288c4b8d1SNeel Natu { 399388c4b8d1SNeel Natu struct vlapic_vtx *vlapic_vtx; 399488c4b8d1SNeel Natu struct pir_desc *pir_desc; 399588c4b8d1SNeel Natu struct LAPIC *lapic; 399688c4b8d1SNeel Natu uint64_t val, pirval; 39970e30c5c0SWarner Losh int rvi, pirbase = -1; 399888c4b8d1SNeel Natu uint16_t intr_status_old, intr_status_new; 399988c4b8d1SNeel Natu 400088c4b8d1SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 4001176666c2SNeel Natu pir_desc = vlapic_vtx->pir_desc; 400288c4b8d1SNeel Natu if (atomic_cmpset_long(&pir_desc->pending, 1, 0) == 0) { 4003d030f941SJohn Baldwin VLAPIC_CTR0(vlapic, "vmx_inject_pir: " 400488c4b8d1SNeel Natu "no posted interrupt pending"); 400588c4b8d1SNeel Natu return; 400688c4b8d1SNeel Natu } 400788c4b8d1SNeel Natu 400888c4b8d1SNeel Natu pirval = 0; 4009201b1cccSPeter Grehan pirbase = -1; 401088c4b8d1SNeel Natu lapic = vlapic->apic_page; 401188c4b8d1SNeel Natu 401288c4b8d1SNeel Natu val = atomic_readandclear_long(&pir_desc->pir[0]); 401388c4b8d1SNeel Natu if (val != 0) { 401488c4b8d1SNeel Natu lapic->irr0 |= val; 401588c4b8d1SNeel Natu lapic->irr1 |= val >> 32; 401688c4b8d1SNeel Natu pirbase = 0; 401788c4b8d1SNeel Natu pirval = val; 401888c4b8d1SNeel Natu } 401988c4b8d1SNeel Natu 402088c4b8d1SNeel Natu val = atomic_readandclear_long(&pir_desc->pir[1]); 402188c4b8d1SNeel Natu if (val != 0) { 402288c4b8d1SNeel Natu lapic->irr2 |= val; 402388c4b8d1SNeel Natu lapic->irr3 |= val >> 32; 402488c4b8d1SNeel Natu pirbase = 64; 402588c4b8d1SNeel Natu pirval = val; 402688c4b8d1SNeel Natu } 402788c4b8d1SNeel Natu 402888c4b8d1SNeel Natu val = atomic_readandclear_long(&pir_desc->pir[2]); 402988c4b8d1SNeel Natu if (val != 0) { 403088c4b8d1SNeel Natu lapic->irr4 |= val; 403188c4b8d1SNeel Natu lapic->irr5 |= val >> 32; 403288c4b8d1SNeel Natu pirbase = 128; 403388c4b8d1SNeel Natu pirval = val; 403488c4b8d1SNeel Natu } 403588c4b8d1SNeel Natu 403688c4b8d1SNeel Natu val = atomic_readandclear_long(&pir_desc->pir[3]); 403788c4b8d1SNeel Natu if (val != 0) { 403888c4b8d1SNeel Natu lapic->irr6 |= val; 403988c4b8d1SNeel Natu lapic->irr7 |= val >> 32; 404088c4b8d1SNeel Natu pirbase = 192; 404188c4b8d1SNeel Natu pirval = val; 404288c4b8d1SNeel Natu } 4043201b1cccSPeter Grehan 404488c4b8d1SNeel Natu VLAPIC_CTR_IRR(vlapic, "vmx_inject_pir"); 404588c4b8d1SNeel Natu 404688c4b8d1SNeel Natu /* 404788c4b8d1SNeel Natu * Update RVI so the processor can evaluate pending virtual 404888c4b8d1SNeel Natu * interrupts on VM-entry. 4049201b1cccSPeter Grehan * 4050201b1cccSPeter Grehan * It is possible for pirval to be 0 here, even though the 4051201b1cccSPeter Grehan * pending bit has been set. The scenario is: 4052201b1cccSPeter Grehan * CPU-Y is sending a posted interrupt to CPU-X, which 4053201b1cccSPeter Grehan * is running a guest and processing posted interrupts in h/w. 4054201b1cccSPeter Grehan * CPU-X will eventually exit and the state seen in s/w is 4055201b1cccSPeter Grehan * the pending bit set, but no PIR bits set. 4056201b1cccSPeter Grehan * 4057201b1cccSPeter Grehan * CPU-X CPU-Y 4058201b1cccSPeter Grehan * (vm running) (host running) 4059201b1cccSPeter Grehan * rx posted interrupt 4060201b1cccSPeter Grehan * CLEAR pending bit 4061201b1cccSPeter Grehan * SET PIR bit 4062201b1cccSPeter Grehan * READ/CLEAR PIR bits 4063201b1cccSPeter Grehan * SET pending bit 4064201b1cccSPeter Grehan * (vm exit) 4065201b1cccSPeter Grehan * pending bit set, PIR 0 406688c4b8d1SNeel Natu */ 406788c4b8d1SNeel Natu if (pirval != 0) { 406888c4b8d1SNeel Natu rvi = pirbase + flsl(pirval) - 1; 406988c4b8d1SNeel Natu intr_status_old = vmcs_read(VMCS_GUEST_INTR_STATUS); 407088c4b8d1SNeel Natu intr_status_new = (intr_status_old & 0xFF00) | rvi; 407188c4b8d1SNeel Natu if (intr_status_new > intr_status_old) { 407288c4b8d1SNeel Natu vmcs_write(VMCS_GUEST_INTR_STATUS, intr_status_new); 4073d030f941SJohn Baldwin VLAPIC_CTR2(vlapic, "vmx_inject_pir: " 407488c4b8d1SNeel Natu "guest_intr_status changed from 0x%04x to 0x%04x", 407588c4b8d1SNeel Natu intr_status_old, intr_status_new); 407688c4b8d1SNeel Natu } 407788c4b8d1SNeel Natu } 407888c4b8d1SNeel Natu } 407988c4b8d1SNeel Natu 4080de5ea6b6SNeel Natu static struct vlapic * 4081869c8d19SJohn Baldwin vmx_vlapic_init(void *vcpui) 4082de5ea6b6SNeel Natu { 4083de5ea6b6SNeel Natu struct vmx *vmx; 40841aa51504SJohn Baldwin struct vmx_vcpu *vcpu; 4085de5ea6b6SNeel Natu struct vlapic *vlapic; 4086176666c2SNeel Natu struct vlapic_vtx *vlapic_vtx; 4087de5ea6b6SNeel Natu 40881aa51504SJohn Baldwin vcpu = vcpui; 4089869c8d19SJohn Baldwin vmx = vcpu->vmx; 4090de5ea6b6SNeel Natu 409188c4b8d1SNeel Natu vlapic = malloc(sizeof(struct vlapic_vtx), M_VLAPIC, M_WAITOK | M_ZERO); 4092de5ea6b6SNeel Natu vlapic->vm = vmx->vm; 4093950af9ffSJohn Baldwin vlapic->vcpu = vcpu->vcpu; 40941aa51504SJohn Baldwin vlapic->vcpuid = vcpu->vcpuid; 40951aa51504SJohn Baldwin vlapic->apic_page = (struct LAPIC *)vcpu->apic_page; 4096de5ea6b6SNeel Natu 4097176666c2SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 40981aa51504SJohn Baldwin vlapic_vtx->pir_desc = vcpu->pir_desc; 40991aa51504SJohn Baldwin vlapic_vtx->vcpu = vcpu; 4100176666c2SNeel Natu 41011bc51badSMichael Reifenberger if (tpr_shadowing) { 41021bc51badSMichael Reifenberger vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode_ts; 41031bc51badSMichael Reifenberger } 41041bc51badSMichael Reifenberger 410588c4b8d1SNeel Natu if (virtual_interrupt_delivery) { 410688c4b8d1SNeel Natu vlapic->ops.set_intr_ready = vmx_set_intr_ready; 410788c4b8d1SNeel Natu vlapic->ops.pending_intr = vmx_pending_intr; 410888c4b8d1SNeel Natu vlapic->ops.intr_accepted = vmx_intr_accepted; 410930b94db8SNeel Natu vlapic->ops.set_tmr = vmx_set_tmr; 41101bc51badSMichael Reifenberger vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode_vid; 411188c4b8d1SNeel Natu } 411288c4b8d1SNeel Natu 4113176666c2SNeel Natu if (posted_interrupts) 4114176666c2SNeel Natu vlapic->ops.post_intr = vmx_post_intr; 4115176666c2SNeel Natu 4116de5ea6b6SNeel Natu vlapic_init(vlapic); 4117de5ea6b6SNeel Natu 4118de5ea6b6SNeel Natu return (vlapic); 4119de5ea6b6SNeel Natu } 4120de5ea6b6SNeel Natu 4121de5ea6b6SNeel Natu static void 4122869c8d19SJohn Baldwin vmx_vlapic_cleanup(struct vlapic *vlapic) 4123de5ea6b6SNeel Natu { 4124de5ea6b6SNeel Natu 4125de5ea6b6SNeel Natu vlapic_cleanup(vlapic); 4126de5ea6b6SNeel Natu free(vlapic, M_VLAPIC); 4127de5ea6b6SNeel Natu } 4128de5ea6b6SNeel Natu 4129483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT 4130483d953aSJohn Baldwin static int 4131869c8d19SJohn Baldwin vmx_vcpu_snapshot(void *vcpui, struct vm_snapshot_meta *meta) 4132483d953aSJohn Baldwin { 4133483d953aSJohn Baldwin struct vmcs *vmcs; 4134483d953aSJohn Baldwin struct vmx *vmx; 413539ec056eSJohn Baldwin struct vmx_vcpu *vcpu; 413639ec056eSJohn Baldwin struct vmxctx *vmxctx; 4137483d953aSJohn Baldwin int err, run, hostcpu; 4138483d953aSJohn Baldwin 4139483d953aSJohn Baldwin err = 0; 4140869c8d19SJohn Baldwin vcpu = vcpui; 4141869c8d19SJohn Baldwin vmx = vcpu->vmx; 414239ec056eSJohn Baldwin vmcs = vcpu->vmcs; 4143483d953aSJohn Baldwin 414480cb5d84SJohn Baldwin run = vcpu_is_running(vcpu->vcpu, &hostcpu); 4145483d953aSJohn Baldwin if (run && hostcpu != curcpu) { 414639ec056eSJohn Baldwin printf("%s: %s%d is running", __func__, vm_name(vmx->vm), 41471aa51504SJohn Baldwin vcpu->vcpuid); 4148483d953aSJohn Baldwin return (EINVAL); 4149483d953aSJohn Baldwin } 4150483d953aSJohn Baldwin 4151483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CR0, meta); 4152483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CR3, meta); 4153483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CR4, meta); 4154483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_DR7, meta); 4155483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_RSP, meta); 4156483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_RIP, meta); 4157483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_RFLAGS, meta); 4158483d953aSJohn Baldwin 4159483d953aSJohn Baldwin /* Guest segments */ 4160483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_ES, meta); 4161483d953aSJohn Baldwin err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_ES, meta); 4162483d953aSJohn Baldwin 4163483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CS, meta); 4164483d953aSJohn Baldwin err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_CS, meta); 4165483d953aSJohn Baldwin 4166483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_SS, meta); 4167483d953aSJohn Baldwin err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_SS, meta); 4168483d953aSJohn Baldwin 4169483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_DS, meta); 4170483d953aSJohn Baldwin err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_DS, meta); 4171483d953aSJohn Baldwin 4172483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_FS, meta); 4173483d953aSJohn Baldwin err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_FS, meta); 4174483d953aSJohn Baldwin 4175483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_GS, meta); 4176483d953aSJohn Baldwin err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_GS, meta); 4177483d953aSJohn Baldwin 4178483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_TR, meta); 4179483d953aSJohn Baldwin err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_TR, meta); 4180483d953aSJohn Baldwin 4181483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_LDTR, meta); 4182483d953aSJohn Baldwin err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_LDTR, meta); 4183483d953aSJohn Baldwin 4184483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_EFER, meta); 4185483d953aSJohn Baldwin 4186483d953aSJohn Baldwin err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_IDTR, meta); 4187483d953aSJohn Baldwin err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_GDTR, meta); 4188483d953aSJohn Baldwin 4189483d953aSJohn Baldwin /* Guest page tables */ 4190483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE0, meta); 4191483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE1, meta); 4192483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE2, meta); 4193483d953aSJohn Baldwin err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE3, meta); 4194483d953aSJohn Baldwin 4195483d953aSJohn Baldwin /* Other guest state */ 4196483d953aSJohn Baldwin err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_IA32_SYSENTER_CS, meta); 4197483d953aSJohn Baldwin err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_IA32_SYSENTER_ESP, meta); 4198483d953aSJohn Baldwin err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_IA32_SYSENTER_EIP, meta); 4199483d953aSJohn Baldwin err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_INTERRUPTIBILITY, meta); 4200483d953aSJohn Baldwin err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_ACTIVITY, meta); 4201483d953aSJohn Baldwin err += vmcs_snapshot_any(vmcs, run, VMCS_ENTRY_CTLS, meta); 4202483d953aSJohn Baldwin err += vmcs_snapshot_any(vmcs, run, VMCS_EXIT_CTLS, meta); 420339ec056eSJohn Baldwin if (err != 0) 420439ec056eSJohn Baldwin goto done; 4205483d953aSJohn Baldwin 420639ec056eSJohn Baldwin SNAPSHOT_BUF_OR_LEAVE(vcpu->guest_msrs, 420739ec056eSJohn Baldwin sizeof(vcpu->guest_msrs), meta, err, done); 420839ec056eSJohn Baldwin 4209c543e09fSVitaliy Gusev SNAPSHOT_BUF_OR_LEAVE(vcpu->pir_desc, 4210c543e09fSVitaliy Gusev sizeof(*vcpu->pir_desc), meta, err, done); 4211c543e09fSVitaliy Gusev 4212683ea4d2SVitaliy Gusev SNAPSHOT_BUF_OR_LEAVE(&vcpu->mtrr, 4213683ea4d2SVitaliy Gusev sizeof(vcpu->mtrr), meta, err, done); 4214683ea4d2SVitaliy Gusev 421539ec056eSJohn Baldwin vmxctx = &vcpu->ctx; 421639ec056eSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rdi, meta, err, done); 421739ec056eSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rsi, meta, err, done); 421839ec056eSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rdx, meta, err, done); 421939ec056eSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rcx, meta, err, done); 422039ec056eSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r8, meta, err, done); 422139ec056eSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r9, meta, err, done); 422239ec056eSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rax, meta, err, done); 422339ec056eSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rbx, meta, err, done); 422439ec056eSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rbp, meta, err, done); 422539ec056eSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r10, meta, err, done); 422639ec056eSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r11, meta, err, done); 422739ec056eSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r12, meta, err, done); 422839ec056eSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r13, meta, err, done); 422939ec056eSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r14, meta, err, done); 423039ec056eSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r15, meta, err, done); 423139ec056eSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_cr2, meta, err, done); 423239ec056eSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr0, meta, err, done); 423339ec056eSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr1, meta, err, done); 423439ec056eSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr2, meta, err, done); 423539ec056eSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr3, meta, err, done); 423639ec056eSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr6, meta, err, done); 423739ec056eSJohn Baldwin 423839ec056eSJohn Baldwin done: 4239483d953aSJohn Baldwin return (err); 4240483d953aSJohn Baldwin } 4241483d953aSJohn Baldwin 4242483d953aSJohn Baldwin static int 4243869c8d19SJohn Baldwin vmx_restore_tsc(void *vcpui, uint64_t offset) 4244483d953aSJohn Baldwin { 42451aa51504SJohn Baldwin struct vmx_vcpu *vcpu = vcpui; 4246869c8d19SJohn Baldwin struct vmcs *vmcs; 4247869c8d19SJohn Baldwin struct vmx *vmx; 4248483d953aSJohn Baldwin int error, running, hostcpu; 4249483d953aSJohn Baldwin 4250869c8d19SJohn Baldwin vmx = vcpu->vmx; 42511aa51504SJohn Baldwin vmcs = vcpu->vmcs; 4252483d953aSJohn Baldwin 425380cb5d84SJohn Baldwin running = vcpu_is_running(vcpu->vcpu, &hostcpu); 4254483d953aSJohn Baldwin if (running && hostcpu != curcpu) { 42551aa51504SJohn Baldwin printf("%s: %s%d is running", __func__, vm_name(vmx->vm), 42561aa51504SJohn Baldwin vcpu->vcpuid); 4257483d953aSJohn Baldwin return (EINVAL); 4258483d953aSJohn Baldwin } 4259483d953aSJohn Baldwin 4260483d953aSJohn Baldwin if (!running) 4261483d953aSJohn Baldwin VMPTRLD(vmcs); 4262483d953aSJohn Baldwin 426380cb5d84SJohn Baldwin error = vmx_set_tsc_offset(vcpu, offset); 4264483d953aSJohn Baldwin 4265483d953aSJohn Baldwin if (!running) 4266483d953aSJohn Baldwin VMCLEAR(vmcs); 4267483d953aSJohn Baldwin return (error); 4268483d953aSJohn Baldwin } 4269483d953aSJohn Baldwin #endif 4270483d953aSJohn Baldwin 427115add60dSPeter Grehan const struct vmm_ops vmm_ops_intel = { 427215add60dSPeter Grehan .modinit = vmx_modinit, 427315add60dSPeter Grehan .modcleanup = vmx_modcleanup, 427415add60dSPeter Grehan .modresume = vmx_modresume, 427513a7c4d4SMark Johnston .init = vmx_init, 427615add60dSPeter Grehan .run = vmx_run, 427713a7c4d4SMark Johnston .cleanup = vmx_cleanup, 42781aa51504SJohn Baldwin .vcpu_init = vmx_vcpu_init, 42791aa51504SJohn Baldwin .vcpu_cleanup = vmx_vcpu_cleanup, 428015add60dSPeter Grehan .getreg = vmx_getreg, 428115add60dSPeter Grehan .setreg = vmx_setreg, 428215add60dSPeter Grehan .getdesc = vmx_getdesc, 428315add60dSPeter Grehan .setdesc = vmx_setdesc, 428415add60dSPeter Grehan .getcap = vmx_getcap, 428515add60dSPeter Grehan .setcap = vmx_setcap, 428615add60dSPeter Grehan .vmspace_alloc = vmx_vmspace_alloc, 428715add60dSPeter Grehan .vmspace_free = vmx_vmspace_free, 428813a7c4d4SMark Johnston .vlapic_init = vmx_vlapic_init, 428913a7c4d4SMark Johnston .vlapic_cleanup = vmx_vlapic_cleanup, 4290483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT 429139ec056eSJohn Baldwin .vcpu_snapshot = vmx_vcpu_snapshot, 429215add60dSPeter Grehan .restore_tsc = vmx_restore_tsc, 4293483d953aSJohn Baldwin #endif 4294366f6083SPeter Grehan }; 4295