1366f6083SPeter Grehan /*- 2c49761ddSPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3c49761ddSPedro F. Giffuni * 4366f6083SPeter Grehan * Copyright (c) 2011 NetApp, Inc. 5366f6083SPeter Grehan * All rights reserved. 6366f6083SPeter Grehan * 7366f6083SPeter Grehan * Redistribution and use in source and binary forms, with or without 8366f6083SPeter Grehan * modification, are permitted provided that the following conditions 9366f6083SPeter Grehan * are met: 10366f6083SPeter Grehan * 1. Redistributions of source code must retain the above copyright 11366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer. 12366f6083SPeter Grehan * 2. Redistributions in binary form must reproduce the above copyright 13366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer in the 14366f6083SPeter Grehan * documentation and/or other materials provided with the distribution. 15366f6083SPeter Grehan * 16366f6083SPeter Grehan * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 17366f6083SPeter Grehan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18366f6083SPeter Grehan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19366f6083SPeter Grehan * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 20366f6083SPeter Grehan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21366f6083SPeter Grehan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22366f6083SPeter Grehan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23366f6083SPeter Grehan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24366f6083SPeter Grehan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25366f6083SPeter Grehan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26366f6083SPeter Grehan * SUCH DAMAGE. 27366f6083SPeter Grehan * 28366f6083SPeter Grehan * $FreeBSD$ 29366f6083SPeter Grehan */ 30366f6083SPeter Grehan 31366f6083SPeter Grehan #include <sys/cdefs.h> 32366f6083SPeter Grehan __FBSDID("$FreeBSD$"); 33366f6083SPeter Grehan 34366f6083SPeter Grehan #include <sys/param.h> 35366f6083SPeter Grehan #include <sys/systm.h> 36366f6083SPeter Grehan #include <sys/smp.h> 37366f6083SPeter Grehan #include <sys/kernel.h> 38366f6083SPeter Grehan #include <sys/malloc.h> 39366f6083SPeter Grehan #include <sys/pcpu.h> 40366f6083SPeter Grehan #include <sys/proc.h> 413565b59eSNeel Natu #include <sys/sysctl.h> 42366f6083SPeter Grehan 43366f6083SPeter Grehan #include <vm/vm.h> 44366f6083SPeter Grehan #include <vm/pmap.h> 45366f6083SPeter Grehan 46366f6083SPeter Grehan #include <machine/psl.h> 47366f6083SPeter Grehan #include <machine/cpufunc.h> 488b287612SJohn Baldwin #include <machine/md_var.h> 49366f6083SPeter Grehan #include <machine/segments.h> 50176666c2SNeel Natu #include <machine/smp.h> 51608f97c3SPeter Grehan #include <machine/specialreg.h> 52366f6083SPeter Grehan #include <machine/vmparam.h> 53366f6083SPeter Grehan 54366f6083SPeter Grehan #include <machine/vmm.h> 55dc506506SNeel Natu #include <machine/vmm_dev.h> 56e813a873SNeel Natu #include <machine/vmm_instruction_emul.h> 57c3498942SNeel Natu #include "vmm_lapic.h" 58b01c2033SNeel Natu #include "vmm_host.h" 59762fd208STycho Nightingale #include "vmm_ioport.h" 60366f6083SPeter Grehan #include "vmm_ktr.h" 61366f6083SPeter Grehan #include "vmm_stat.h" 620775fbb4STycho Nightingale #include "vatpic.h" 63de5ea6b6SNeel Natu #include "vlapic.h" 64de5ea6b6SNeel Natu #include "vlapic_priv.h" 65366f6083SPeter Grehan 66366f6083SPeter Grehan #include "ept.h" 67366f6083SPeter Grehan #include "vmx_cpufunc.h" 68366f6083SPeter Grehan #include "vmx.h" 69c3498942SNeel Natu #include "vmx_msr.h" 70366f6083SPeter Grehan #include "x86.h" 71366f6083SPeter Grehan #include "vmx_controls.h" 72366f6083SPeter Grehan 73366f6083SPeter Grehan #define PINBASED_CTLS_ONE_SETTING \ 74366f6083SPeter Grehan (PINBASED_EXTINT_EXITING | \ 75366f6083SPeter Grehan PINBASED_NMI_EXITING | \ 76366f6083SPeter Grehan PINBASED_VIRTUAL_NMI) 77366f6083SPeter Grehan #define PINBASED_CTLS_ZERO_SETTING 0 78366f6083SPeter Grehan 79366f6083SPeter Grehan #define PROCBASED_CTLS_WINDOW_SETTING \ 80366f6083SPeter Grehan (PROCBASED_INT_WINDOW_EXITING | \ 81366f6083SPeter Grehan PROCBASED_NMI_WINDOW_EXITING) 82366f6083SPeter Grehan 83366f6083SPeter Grehan #define PROCBASED_CTLS_ONE_SETTING \ 84366f6083SPeter Grehan (PROCBASED_SECONDARY_CONTROLS | \ 8565145c7fSNeel Natu PROCBASED_MWAIT_EXITING | \ 8665145c7fSNeel Natu PROCBASED_MONITOR_EXITING | \ 87366f6083SPeter Grehan PROCBASED_IO_EXITING | \ 88366f6083SPeter Grehan PROCBASED_MSR_BITMAPS | \ 89594db002STycho Nightingale PROCBASED_CTLS_WINDOW_SETTING | \ 90594db002STycho Nightingale PROCBASED_CR8_LOAD_EXITING | \ 91594db002STycho Nightingale PROCBASED_CR8_STORE_EXITING) 92366f6083SPeter Grehan #define PROCBASED_CTLS_ZERO_SETTING \ 93366f6083SPeter Grehan (PROCBASED_CR3_LOAD_EXITING | \ 94366f6083SPeter Grehan PROCBASED_CR3_STORE_EXITING | \ 95366f6083SPeter Grehan PROCBASED_IO_BITMAPS) 96366f6083SPeter Grehan 97366f6083SPeter Grehan #define PROCBASED_CTLS2_ONE_SETTING PROCBASED2_ENABLE_EPT 98366f6083SPeter Grehan #define PROCBASED_CTLS2_ZERO_SETTING 0 99366f6083SPeter Grehan 100d72978ecSNeel Natu #define VM_EXIT_CTLS_ONE_SETTING \ 101366f6083SPeter Grehan (VM_EXIT_HOST_LMA | \ 102366f6083SPeter Grehan VM_EXIT_SAVE_EFER | \ 103d72978ecSNeel Natu VM_EXIT_LOAD_EFER | \ 104a318f7ddSNeel Natu VM_EXIT_ACKNOWLEDGE_INTERRUPT) 105d72978ecSNeel Natu 106366f6083SPeter Grehan #define VM_EXIT_CTLS_ZERO_SETTING VM_EXIT_SAVE_DEBUG_CONTROLS 107366f6083SPeter Grehan 108a318f7ddSNeel Natu #define VM_ENTRY_CTLS_ONE_SETTING (VM_ENTRY_LOAD_EFER) 109608f97c3SPeter Grehan 110366f6083SPeter Grehan #define VM_ENTRY_CTLS_ZERO_SETTING \ 111366f6083SPeter Grehan (VM_ENTRY_LOAD_DEBUG_CONTROLS | \ 112366f6083SPeter Grehan VM_ENTRY_INTO_SMM | \ 113366f6083SPeter Grehan VM_ENTRY_DEACTIVATE_DUAL_MONITOR) 114366f6083SPeter Grehan 115366f6083SPeter Grehan #define HANDLED 1 116366f6083SPeter Grehan #define UNHANDLED 0 117366f6083SPeter Grehan 118de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VMX, "vmx", "vmx"); 119de5ea6b6SNeel Natu static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic"); 120366f6083SPeter Grehan 1213565b59eSNeel Natu SYSCTL_DECL(_hw_vmm); 1223565b59eSNeel Natu SYSCTL_NODE(_hw_vmm, OID_AUTO, vmx, CTLFLAG_RW, NULL, NULL); 1233565b59eSNeel Natu 124b3996dd4SJohn Baldwin int vmxon_enabled[MAXCPU]; 125366f6083SPeter Grehan static char vmxon_region[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE); 126366f6083SPeter Grehan 127366f6083SPeter Grehan static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2; 128366f6083SPeter Grehan static uint32_t exit_ctls, entry_ctls; 129366f6083SPeter Grehan 130366f6083SPeter Grehan static uint64_t cr0_ones_mask, cr0_zeros_mask; 1313565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_ones_mask, CTLFLAG_RD, 1323565b59eSNeel Natu &cr0_ones_mask, 0, NULL); 1333565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_zeros_mask, CTLFLAG_RD, 1343565b59eSNeel Natu &cr0_zeros_mask, 0, NULL); 1353565b59eSNeel Natu 136366f6083SPeter Grehan static uint64_t cr4_ones_mask, cr4_zeros_mask; 1373565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_ones_mask, CTLFLAG_RD, 1383565b59eSNeel Natu &cr4_ones_mask, 0, NULL); 1393565b59eSNeel Natu SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_zeros_mask, CTLFLAG_RD, 1403565b59eSNeel Natu &cr4_zeros_mask, 0, NULL); 141366f6083SPeter Grehan 1423565b59eSNeel Natu static int vmx_initialized; 1433565b59eSNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, initialized, CTLFLAG_RD, 1443565b59eSNeel Natu &vmx_initialized, 0, "Intel VMX initialized"); 1453565b59eSNeel Natu 146366f6083SPeter Grehan /* 147366f6083SPeter Grehan * Optional capabilities 148366f6083SPeter Grehan */ 14906fc6db9SJohn Baldwin static SYSCTL_NODE(_hw_vmm_vmx, OID_AUTO, cap, CTLFLAG_RW, NULL, NULL); 15006fc6db9SJohn Baldwin 151366f6083SPeter Grehan static int cap_halt_exit; 15206fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, halt_exit, CTLFLAG_RD, &cap_halt_exit, 0, 15306fc6db9SJohn Baldwin "HLT triggers a VM-exit"); 15406fc6db9SJohn Baldwin 155366f6083SPeter Grehan static int cap_pause_exit; 15606fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, pause_exit, CTLFLAG_RD, &cap_pause_exit, 15706fc6db9SJohn Baldwin 0, "PAUSE triggers a VM-exit"); 15806fc6db9SJohn Baldwin 159366f6083SPeter Grehan static int cap_unrestricted_guest; 16006fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, unrestricted_guest, CTLFLAG_RD, 16106fc6db9SJohn Baldwin &cap_unrestricted_guest, 0, "Unrestricted guests"); 16206fc6db9SJohn Baldwin 163366f6083SPeter Grehan static int cap_monitor_trap; 16406fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, monitor_trap, CTLFLAG_RD, 16506fc6db9SJohn Baldwin &cap_monitor_trap, 0, "Monitor trap flag"); 16606fc6db9SJohn Baldwin 16749cc03daSNeel Natu static int cap_invpcid; 16806fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, invpcid, CTLFLAG_RD, &cap_invpcid, 16906fc6db9SJohn Baldwin 0, "Guests are allowed to use INVPCID"); 170366f6083SPeter Grehan 17188c4b8d1SNeel Natu static int virtual_interrupt_delivery; 17206fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, virtual_interrupt_delivery, CTLFLAG_RD, 17388c4b8d1SNeel Natu &virtual_interrupt_delivery, 0, "APICv virtual interrupt delivery support"); 17488c4b8d1SNeel Natu 175176666c2SNeel Natu static int posted_interrupts; 17606fc6db9SJohn Baldwin SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, posted_interrupts, CTLFLAG_RD, 177176666c2SNeel Natu &posted_interrupts, 0, "APICv posted interrupt support"); 178176666c2SNeel Natu 17918a2b08eSNeel Natu static int pirvec = -1; 180176666c2SNeel Natu SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupt_vector, CTLFLAG_RD, 181176666c2SNeel Natu &pirvec, 0, "APICv posted interrupt vector"); 182176666c2SNeel Natu 18345e51299SNeel Natu static struct unrhdr *vpid_unr; 18445e51299SNeel Natu static u_int vpid_alloc_failed; 18545e51299SNeel Natu SYSCTL_UINT(_hw_vmm_vmx, OID_AUTO, vpid_alloc_failed, CTLFLAG_RD, 18645e51299SNeel Natu &vpid_alloc_failed, 0, NULL); 18745e51299SNeel Natu 18888c4b8d1SNeel Natu /* 18988c4b8d1SNeel Natu * Use the last page below 4GB as the APIC access address. This address is 19088c4b8d1SNeel Natu * occupied by the boot firmware so it is guaranteed that it will not conflict 19188c4b8d1SNeel Natu * with a page in system memory. 19288c4b8d1SNeel Natu */ 19388c4b8d1SNeel Natu #define APIC_ACCESS_ADDRESS 0xFFFFF000 19488c4b8d1SNeel Natu 195d17b5104SNeel Natu static int vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc); 196d17b5104SNeel Natu static int vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval); 197c3498942SNeel Natu static int vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val); 19888c4b8d1SNeel Natu static void vmx_inject_pir(struct vlapic *vlapic); 19988c4b8d1SNeel Natu 200366f6083SPeter Grehan #ifdef KTR 201366f6083SPeter Grehan static const char * 202366f6083SPeter Grehan exit_reason_to_str(int reason) 203366f6083SPeter Grehan { 204366f6083SPeter Grehan static char reasonbuf[32]; 205366f6083SPeter Grehan 206366f6083SPeter Grehan switch (reason) { 207366f6083SPeter Grehan case EXIT_REASON_EXCEPTION: 208366f6083SPeter Grehan return "exception"; 209366f6083SPeter Grehan case EXIT_REASON_EXT_INTR: 210366f6083SPeter Grehan return "extint"; 211366f6083SPeter Grehan case EXIT_REASON_TRIPLE_FAULT: 212366f6083SPeter Grehan return "triplefault"; 213366f6083SPeter Grehan case EXIT_REASON_INIT: 214366f6083SPeter Grehan return "init"; 215366f6083SPeter Grehan case EXIT_REASON_SIPI: 216366f6083SPeter Grehan return "sipi"; 217366f6083SPeter Grehan case EXIT_REASON_IO_SMI: 218366f6083SPeter Grehan return "iosmi"; 219366f6083SPeter Grehan case EXIT_REASON_SMI: 220366f6083SPeter Grehan return "smi"; 221366f6083SPeter Grehan case EXIT_REASON_INTR_WINDOW: 222366f6083SPeter Grehan return "intrwindow"; 223366f6083SPeter Grehan case EXIT_REASON_NMI_WINDOW: 224366f6083SPeter Grehan return "nmiwindow"; 225366f6083SPeter Grehan case EXIT_REASON_TASK_SWITCH: 226366f6083SPeter Grehan return "taskswitch"; 227366f6083SPeter Grehan case EXIT_REASON_CPUID: 228366f6083SPeter Grehan return "cpuid"; 229366f6083SPeter Grehan case EXIT_REASON_GETSEC: 230366f6083SPeter Grehan return "getsec"; 231366f6083SPeter Grehan case EXIT_REASON_HLT: 232366f6083SPeter Grehan return "hlt"; 233366f6083SPeter Grehan case EXIT_REASON_INVD: 234366f6083SPeter Grehan return "invd"; 235366f6083SPeter Grehan case EXIT_REASON_INVLPG: 236366f6083SPeter Grehan return "invlpg"; 237366f6083SPeter Grehan case EXIT_REASON_RDPMC: 238366f6083SPeter Grehan return "rdpmc"; 239366f6083SPeter Grehan case EXIT_REASON_RDTSC: 240366f6083SPeter Grehan return "rdtsc"; 241366f6083SPeter Grehan case EXIT_REASON_RSM: 242366f6083SPeter Grehan return "rsm"; 243366f6083SPeter Grehan case EXIT_REASON_VMCALL: 244366f6083SPeter Grehan return "vmcall"; 245366f6083SPeter Grehan case EXIT_REASON_VMCLEAR: 246366f6083SPeter Grehan return "vmclear"; 247366f6083SPeter Grehan case EXIT_REASON_VMLAUNCH: 248366f6083SPeter Grehan return "vmlaunch"; 249366f6083SPeter Grehan case EXIT_REASON_VMPTRLD: 250366f6083SPeter Grehan return "vmptrld"; 251366f6083SPeter Grehan case EXIT_REASON_VMPTRST: 252366f6083SPeter Grehan return "vmptrst"; 253366f6083SPeter Grehan case EXIT_REASON_VMREAD: 254366f6083SPeter Grehan return "vmread"; 255366f6083SPeter Grehan case EXIT_REASON_VMRESUME: 256366f6083SPeter Grehan return "vmresume"; 257366f6083SPeter Grehan case EXIT_REASON_VMWRITE: 258366f6083SPeter Grehan return "vmwrite"; 259366f6083SPeter Grehan case EXIT_REASON_VMXOFF: 260366f6083SPeter Grehan return "vmxoff"; 261366f6083SPeter Grehan case EXIT_REASON_VMXON: 262366f6083SPeter Grehan return "vmxon"; 263366f6083SPeter Grehan case EXIT_REASON_CR_ACCESS: 264366f6083SPeter Grehan return "craccess"; 265366f6083SPeter Grehan case EXIT_REASON_DR_ACCESS: 266366f6083SPeter Grehan return "draccess"; 267366f6083SPeter Grehan case EXIT_REASON_INOUT: 268366f6083SPeter Grehan return "inout"; 269366f6083SPeter Grehan case EXIT_REASON_RDMSR: 270366f6083SPeter Grehan return "rdmsr"; 271366f6083SPeter Grehan case EXIT_REASON_WRMSR: 272366f6083SPeter Grehan return "wrmsr"; 273366f6083SPeter Grehan case EXIT_REASON_INVAL_VMCS: 274366f6083SPeter Grehan return "invalvmcs"; 275366f6083SPeter Grehan case EXIT_REASON_INVAL_MSR: 276366f6083SPeter Grehan return "invalmsr"; 277366f6083SPeter Grehan case EXIT_REASON_MWAIT: 278366f6083SPeter Grehan return "mwait"; 279366f6083SPeter Grehan case EXIT_REASON_MTF: 280366f6083SPeter Grehan return "mtf"; 281366f6083SPeter Grehan case EXIT_REASON_MONITOR: 282366f6083SPeter Grehan return "monitor"; 283366f6083SPeter Grehan case EXIT_REASON_PAUSE: 284366f6083SPeter Grehan return "pause"; 285b0538143SNeel Natu case EXIT_REASON_MCE_DURING_ENTRY: 286b0538143SNeel Natu return "mce-during-entry"; 287366f6083SPeter Grehan case EXIT_REASON_TPR: 288366f6083SPeter Grehan return "tpr"; 28988c4b8d1SNeel Natu case EXIT_REASON_APIC_ACCESS: 29088c4b8d1SNeel Natu return "apic-access"; 291366f6083SPeter Grehan case EXIT_REASON_GDTR_IDTR: 292366f6083SPeter Grehan return "gdtridtr"; 293366f6083SPeter Grehan case EXIT_REASON_LDTR_TR: 294366f6083SPeter Grehan return "ldtrtr"; 295366f6083SPeter Grehan case EXIT_REASON_EPT_FAULT: 296366f6083SPeter Grehan return "eptfault"; 297366f6083SPeter Grehan case EXIT_REASON_EPT_MISCONFIG: 298366f6083SPeter Grehan return "eptmisconfig"; 299366f6083SPeter Grehan case EXIT_REASON_INVEPT: 300366f6083SPeter Grehan return "invept"; 301366f6083SPeter Grehan case EXIT_REASON_RDTSCP: 302366f6083SPeter Grehan return "rdtscp"; 303366f6083SPeter Grehan case EXIT_REASON_VMX_PREEMPT: 304366f6083SPeter Grehan return "vmxpreempt"; 305366f6083SPeter Grehan case EXIT_REASON_INVVPID: 306366f6083SPeter Grehan return "invvpid"; 307366f6083SPeter Grehan case EXIT_REASON_WBINVD: 308366f6083SPeter Grehan return "wbinvd"; 309366f6083SPeter Grehan case EXIT_REASON_XSETBV: 310366f6083SPeter Grehan return "xsetbv"; 31188c4b8d1SNeel Natu case EXIT_REASON_APIC_WRITE: 31288c4b8d1SNeel Natu return "apic-write"; 313366f6083SPeter Grehan default: 314366f6083SPeter Grehan snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason); 315366f6083SPeter Grehan return (reasonbuf); 316366f6083SPeter Grehan } 317366f6083SPeter Grehan } 318366f6083SPeter Grehan #endif /* KTR */ 319366f6083SPeter Grehan 320159dd56fSNeel Natu static int 321159dd56fSNeel Natu vmx_allow_x2apic_msrs(struct vmx *vmx) 322159dd56fSNeel Natu { 323159dd56fSNeel Natu int i, error; 324159dd56fSNeel Natu 325159dd56fSNeel Natu error = 0; 326159dd56fSNeel Natu 327159dd56fSNeel Natu /* 328159dd56fSNeel Natu * Allow readonly access to the following x2APIC MSRs from the guest. 329159dd56fSNeel Natu */ 330159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_ID); 331159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_VERSION); 332159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LDR); 333159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_SVR); 334159dd56fSNeel Natu 335159dd56fSNeel Natu for (i = 0; i < 8; i++) 336159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_ISR0 + i); 337159dd56fSNeel Natu 338159dd56fSNeel Natu for (i = 0; i < 8; i++) 339159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_TMR0 + i); 340159dd56fSNeel Natu 341159dd56fSNeel Natu for (i = 0; i < 8; i++) 342159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_IRR0 + i); 343159dd56fSNeel Natu 344159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_ESR); 345159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_TIMER); 346159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_THERMAL); 347159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_PCINT); 348159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT0); 349159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT1); 350159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_LVT_ERROR); 351159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_ICR_TIMER); 352159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_DCR_TIMER); 353159dd56fSNeel Natu error += guest_msr_ro(vmx, MSR_APIC_ICR); 354159dd56fSNeel Natu 355159dd56fSNeel Natu /* 356159dd56fSNeel Natu * Allow TPR, EOI and SELF_IPI MSRs to be read and written by the guest. 357159dd56fSNeel Natu * 358159dd56fSNeel Natu * These registers get special treatment described in the section 359159dd56fSNeel Natu * "Virtualizing MSR-Based APIC Accesses". 360159dd56fSNeel Natu */ 361159dd56fSNeel Natu error += guest_msr_rw(vmx, MSR_APIC_TPR); 362159dd56fSNeel Natu error += guest_msr_rw(vmx, MSR_APIC_EOI); 363159dd56fSNeel Natu error += guest_msr_rw(vmx, MSR_APIC_SELF_IPI); 364159dd56fSNeel Natu 365159dd56fSNeel Natu return (error); 366159dd56fSNeel Natu } 367159dd56fSNeel Natu 368366f6083SPeter Grehan u_long 369366f6083SPeter Grehan vmx_fix_cr0(u_long cr0) 370366f6083SPeter Grehan { 371366f6083SPeter Grehan 372366f6083SPeter Grehan return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask); 373366f6083SPeter Grehan } 374366f6083SPeter Grehan 375366f6083SPeter Grehan u_long 376366f6083SPeter Grehan vmx_fix_cr4(u_long cr4) 377366f6083SPeter Grehan { 378366f6083SPeter Grehan 379366f6083SPeter Grehan return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask); 380366f6083SPeter Grehan } 381366f6083SPeter Grehan 382366f6083SPeter Grehan static void 38345e51299SNeel Natu vpid_free(int vpid) 38445e51299SNeel Natu { 38545e51299SNeel Natu if (vpid < 0 || vpid > 0xffff) 38645e51299SNeel Natu panic("vpid_free: invalid vpid %d", vpid); 38745e51299SNeel Natu 38845e51299SNeel Natu /* 38945e51299SNeel Natu * VPIDs [0,VM_MAXCPU] are special and are not allocated from 39045e51299SNeel Natu * the unit number allocator. 39145e51299SNeel Natu */ 39245e51299SNeel Natu 39345e51299SNeel Natu if (vpid > VM_MAXCPU) 39445e51299SNeel Natu free_unr(vpid_unr, vpid); 39545e51299SNeel Natu } 39645e51299SNeel Natu 39745e51299SNeel Natu static void 39845e51299SNeel Natu vpid_alloc(uint16_t *vpid, int num) 39945e51299SNeel Natu { 40045e51299SNeel Natu int i, x; 40145e51299SNeel Natu 40245e51299SNeel Natu if (num <= 0 || num > VM_MAXCPU) 40345e51299SNeel Natu panic("invalid number of vpids requested: %d", num); 40445e51299SNeel Natu 40545e51299SNeel Natu /* 40645e51299SNeel Natu * If the "enable vpid" execution control is not enabled then the 40745e51299SNeel Natu * VPID is required to be 0 for all vcpus. 40845e51299SNeel Natu */ 40945e51299SNeel Natu if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) == 0) { 41045e51299SNeel Natu for (i = 0; i < num; i++) 41145e51299SNeel Natu vpid[i] = 0; 41245e51299SNeel Natu return; 41345e51299SNeel Natu } 41445e51299SNeel Natu 41545e51299SNeel Natu /* 41645e51299SNeel Natu * Allocate a unique VPID for each vcpu from the unit number allocator. 41745e51299SNeel Natu */ 41845e51299SNeel Natu for (i = 0; i < num; i++) { 41945e51299SNeel Natu x = alloc_unr(vpid_unr); 42045e51299SNeel Natu if (x == -1) 42145e51299SNeel Natu break; 42245e51299SNeel Natu else 42345e51299SNeel Natu vpid[i] = x; 42445e51299SNeel Natu } 42545e51299SNeel Natu 42645e51299SNeel Natu if (i < num) { 42745e51299SNeel Natu atomic_add_int(&vpid_alloc_failed, 1); 42845e51299SNeel Natu 42945e51299SNeel Natu /* 43045e51299SNeel Natu * If the unit number allocator does not have enough unique 43145e51299SNeel Natu * VPIDs then we need to allocate from the [1,VM_MAXCPU] range. 43245e51299SNeel Natu * 43345e51299SNeel Natu * These VPIDs are not be unique across VMs but this does not 43445e51299SNeel Natu * affect correctness because the combined mappings are also 43545e51299SNeel Natu * tagged with the EP4TA which is unique for each VM. 43645e51299SNeel Natu * 43745e51299SNeel Natu * It is still sub-optimal because the invvpid will invalidate 43845e51299SNeel Natu * combined mappings for a particular VPID across all EP4TAs. 43945e51299SNeel Natu */ 44045e51299SNeel Natu while (i-- > 0) 44145e51299SNeel Natu vpid_free(vpid[i]); 44245e51299SNeel Natu 44345e51299SNeel Natu for (i = 0; i < num; i++) 44445e51299SNeel Natu vpid[i] = i + 1; 44545e51299SNeel Natu } 44645e51299SNeel Natu } 44745e51299SNeel Natu 44845e51299SNeel Natu static void 44945e51299SNeel Natu vpid_init(void) 45045e51299SNeel Natu { 45145e51299SNeel Natu /* 45245e51299SNeel Natu * VPID 0 is required when the "enable VPID" execution control is 45345e51299SNeel Natu * disabled. 45445e51299SNeel Natu * 45545e51299SNeel Natu * VPIDs [1,VM_MAXCPU] are used as the "overflow namespace" when the 45645e51299SNeel Natu * unit number allocator does not have sufficient unique VPIDs to 45745e51299SNeel Natu * satisfy the allocation. 45845e51299SNeel Natu * 45945e51299SNeel Natu * The remaining VPIDs are managed by the unit number allocator. 46045e51299SNeel Natu */ 46145e51299SNeel Natu vpid_unr = new_unrhdr(VM_MAXCPU + 1, 0xffff, NULL); 46245e51299SNeel Natu } 46345e51299SNeel Natu 46445e51299SNeel Natu static void 465366f6083SPeter Grehan vmx_disable(void *arg __unused) 466366f6083SPeter Grehan { 467366f6083SPeter Grehan struct invvpid_desc invvpid_desc = { 0 }; 468366f6083SPeter Grehan struct invept_desc invept_desc = { 0 }; 469366f6083SPeter Grehan 470366f6083SPeter Grehan if (vmxon_enabled[curcpu]) { 471366f6083SPeter Grehan /* 472366f6083SPeter Grehan * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b. 473366f6083SPeter Grehan * 474366f6083SPeter Grehan * VMXON or VMXOFF are not required to invalidate any TLB 475366f6083SPeter Grehan * caching structures. This prevents potential retention of 476366f6083SPeter Grehan * cached information in the TLB between distinct VMX episodes. 477366f6083SPeter Grehan */ 478366f6083SPeter Grehan invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc); 479366f6083SPeter Grehan invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc); 480366f6083SPeter Grehan vmxoff(); 481366f6083SPeter Grehan } 482366f6083SPeter Grehan load_cr4(rcr4() & ~CR4_VMXE); 483366f6083SPeter Grehan } 484366f6083SPeter Grehan 485366f6083SPeter Grehan static int 486366f6083SPeter Grehan vmx_cleanup(void) 487366f6083SPeter Grehan { 488366f6083SPeter Grehan 48918a2b08eSNeel Natu if (pirvec >= 0) 49018a2b08eSNeel Natu lapic_ipi_free(pirvec); 491176666c2SNeel Natu 49245e51299SNeel Natu if (vpid_unr != NULL) { 49345e51299SNeel Natu delete_unrhdr(vpid_unr); 49445e51299SNeel Natu vpid_unr = NULL; 49545e51299SNeel Natu } 49645e51299SNeel Natu 497366f6083SPeter Grehan smp_rendezvous(NULL, vmx_disable, NULL, NULL); 498366f6083SPeter Grehan 499366f6083SPeter Grehan return (0); 500366f6083SPeter Grehan } 501366f6083SPeter Grehan 502366f6083SPeter Grehan static void 503366f6083SPeter Grehan vmx_enable(void *arg __unused) 504366f6083SPeter Grehan { 505366f6083SPeter Grehan int error; 50611669a68STycho Nightingale uint64_t feature_control; 50711669a68STycho Nightingale 50811669a68STycho Nightingale feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL); 50911669a68STycho Nightingale if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 0 || 51011669a68STycho Nightingale (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) { 51111669a68STycho Nightingale wrmsr(MSR_IA32_FEATURE_CONTROL, 51211669a68STycho Nightingale feature_control | IA32_FEATURE_CONTROL_VMX_EN | 51311669a68STycho Nightingale IA32_FEATURE_CONTROL_LOCK); 51411669a68STycho Nightingale } 515366f6083SPeter Grehan 516366f6083SPeter Grehan load_cr4(rcr4() | CR4_VMXE); 517366f6083SPeter Grehan 518366f6083SPeter Grehan *(uint32_t *)vmxon_region[curcpu] = vmx_revision(); 519366f6083SPeter Grehan error = vmxon(vmxon_region[curcpu]); 520366f6083SPeter Grehan if (error == 0) 521366f6083SPeter Grehan vmxon_enabled[curcpu] = 1; 522366f6083SPeter Grehan } 523366f6083SPeter Grehan 52463e62d39SJohn Baldwin static void 52563e62d39SJohn Baldwin vmx_restore(void) 52663e62d39SJohn Baldwin { 52763e62d39SJohn Baldwin 52863e62d39SJohn Baldwin if (vmxon_enabled[curcpu]) 52963e62d39SJohn Baldwin vmxon(vmxon_region[curcpu]); 53063e62d39SJohn Baldwin } 53163e62d39SJohn Baldwin 532366f6083SPeter Grehan static int 533add611fdSNeel Natu vmx_init(int ipinum) 534366f6083SPeter Grehan { 53588c4b8d1SNeel Natu int error, use_tpr_shadow; 536d17b5104SNeel Natu uint64_t basic, fixed0, fixed1, feature_control; 53788c4b8d1SNeel Natu uint32_t tmp, procbased2_vid_bits; 538366f6083SPeter Grehan 539366f6083SPeter Grehan /* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */ 5408b287612SJohn Baldwin if (!(cpu_feature2 & CPUID2_VMX)) { 541366f6083SPeter Grehan printf("vmx_init: processor does not support VMX operation\n"); 542366f6083SPeter Grehan return (ENXIO); 543366f6083SPeter Grehan } 544366f6083SPeter Grehan 5454bff7fadSNeel Natu /* 5464bff7fadSNeel Natu * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits 5474bff7fadSNeel Natu * are set (bits 0 and 2 respectively). 5484bff7fadSNeel Natu */ 5494bff7fadSNeel Natu feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL); 55011669a68STycho Nightingale if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 1 && 551150369abSNeel Natu (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) { 5524bff7fadSNeel Natu printf("vmx_init: VMX operation disabled by BIOS\n"); 5534bff7fadSNeel Natu return (ENXIO); 5544bff7fadSNeel Natu } 5554bff7fadSNeel Natu 556d17b5104SNeel Natu /* 557d17b5104SNeel Natu * Verify capabilities MSR_VMX_BASIC: 558d17b5104SNeel Natu * - bit 54 indicates support for INS/OUTS decoding 559d17b5104SNeel Natu */ 560d17b5104SNeel Natu basic = rdmsr(MSR_VMX_BASIC); 561d17b5104SNeel Natu if ((basic & (1UL << 54)) == 0) { 562d17b5104SNeel Natu printf("vmx_init: processor does not support desired basic " 563d17b5104SNeel Natu "capabilities\n"); 564d17b5104SNeel Natu return (EINVAL); 565d17b5104SNeel Natu } 566d17b5104SNeel Natu 567366f6083SPeter Grehan /* Check support for primary processor-based VM-execution controls */ 568366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 569366f6083SPeter Grehan MSR_VMX_TRUE_PROCBASED_CTLS, 570366f6083SPeter Grehan PROCBASED_CTLS_ONE_SETTING, 571366f6083SPeter Grehan PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls); 572366f6083SPeter Grehan if (error) { 573366f6083SPeter Grehan printf("vmx_init: processor does not support desired primary " 574366f6083SPeter Grehan "processor-based controls\n"); 575366f6083SPeter Grehan return (error); 576366f6083SPeter Grehan } 577366f6083SPeter Grehan 578366f6083SPeter Grehan /* Clear the processor-based ctl bits that are set on demand */ 579366f6083SPeter Grehan procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING; 580366f6083SPeter Grehan 581366f6083SPeter Grehan /* Check support for secondary processor-based VM-execution controls */ 582366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 583366f6083SPeter Grehan MSR_VMX_PROCBASED_CTLS2, 584366f6083SPeter Grehan PROCBASED_CTLS2_ONE_SETTING, 585366f6083SPeter Grehan PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2); 586366f6083SPeter Grehan if (error) { 587366f6083SPeter Grehan printf("vmx_init: processor does not support desired secondary " 588366f6083SPeter Grehan "processor-based controls\n"); 589366f6083SPeter Grehan return (error); 590366f6083SPeter Grehan } 591366f6083SPeter Grehan 592366f6083SPeter Grehan /* Check support for VPID */ 593366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2, 594366f6083SPeter Grehan PROCBASED2_ENABLE_VPID, 0, &tmp); 595366f6083SPeter Grehan if (error == 0) 596366f6083SPeter Grehan procbased_ctls2 |= PROCBASED2_ENABLE_VPID; 597366f6083SPeter Grehan 598366f6083SPeter Grehan /* Check support for pin-based VM-execution controls */ 599366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS, 600366f6083SPeter Grehan MSR_VMX_TRUE_PINBASED_CTLS, 601366f6083SPeter Grehan PINBASED_CTLS_ONE_SETTING, 602366f6083SPeter Grehan PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls); 603366f6083SPeter Grehan if (error) { 604366f6083SPeter Grehan printf("vmx_init: processor does not support desired " 605366f6083SPeter Grehan "pin-based controls\n"); 606366f6083SPeter Grehan return (error); 607366f6083SPeter Grehan } 608366f6083SPeter Grehan 609366f6083SPeter Grehan /* Check support for VM-exit controls */ 610366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS, 611366f6083SPeter Grehan VM_EXIT_CTLS_ONE_SETTING, 612366f6083SPeter Grehan VM_EXIT_CTLS_ZERO_SETTING, 613366f6083SPeter Grehan &exit_ctls); 614366f6083SPeter Grehan if (error) { 615366f6083SPeter Grehan printf("vmx_init: processor does not support desired " 616366f6083SPeter Grehan "exit controls\n"); 617366f6083SPeter Grehan return (error); 618366f6083SPeter Grehan } 619366f6083SPeter Grehan 620366f6083SPeter Grehan /* Check support for VM-entry controls */ 621d72978ecSNeel Natu error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS, MSR_VMX_TRUE_ENTRY_CTLS, 622d72978ecSNeel Natu VM_ENTRY_CTLS_ONE_SETTING, VM_ENTRY_CTLS_ZERO_SETTING, 623366f6083SPeter Grehan &entry_ctls); 624366f6083SPeter Grehan if (error) { 625366f6083SPeter Grehan printf("vmx_init: processor does not support desired " 626366f6083SPeter Grehan "entry controls\n"); 627366f6083SPeter Grehan return (error); 628366f6083SPeter Grehan } 629366f6083SPeter Grehan 630366f6083SPeter Grehan /* 631366f6083SPeter Grehan * Check support for optional features by testing them 632366f6083SPeter Grehan * as individual bits 633366f6083SPeter Grehan */ 634366f6083SPeter Grehan cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 635366f6083SPeter Grehan MSR_VMX_TRUE_PROCBASED_CTLS, 636366f6083SPeter Grehan PROCBASED_HLT_EXITING, 0, 637366f6083SPeter Grehan &tmp) == 0); 638366f6083SPeter Grehan 639366f6083SPeter Grehan cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 640366f6083SPeter Grehan MSR_VMX_PROCBASED_CTLS, 641366f6083SPeter Grehan PROCBASED_MTF, 0, 642366f6083SPeter Grehan &tmp) == 0); 643366f6083SPeter Grehan 644366f6083SPeter Grehan cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 645366f6083SPeter Grehan MSR_VMX_TRUE_PROCBASED_CTLS, 646366f6083SPeter Grehan PROCBASED_PAUSE_EXITING, 0, 647366f6083SPeter Grehan &tmp) == 0); 648366f6083SPeter Grehan 649366f6083SPeter Grehan cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 650366f6083SPeter Grehan MSR_VMX_PROCBASED_CTLS2, 651366f6083SPeter Grehan PROCBASED2_UNRESTRICTED_GUEST, 0, 652366f6083SPeter Grehan &tmp) == 0); 653366f6083SPeter Grehan 65449cc03daSNeel Natu cap_invpcid = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 65549cc03daSNeel Natu MSR_VMX_PROCBASED_CTLS2, PROCBASED2_ENABLE_INVPCID, 0, 65649cc03daSNeel Natu &tmp) == 0); 65749cc03daSNeel Natu 65888c4b8d1SNeel Natu /* 65988c4b8d1SNeel Natu * Check support for virtual interrupt delivery. 66088c4b8d1SNeel Natu */ 66188c4b8d1SNeel Natu procbased2_vid_bits = (PROCBASED2_VIRTUALIZE_APIC_ACCESSES | 66288c4b8d1SNeel Natu PROCBASED2_VIRTUALIZE_X2APIC_MODE | 66388c4b8d1SNeel Natu PROCBASED2_APIC_REGISTER_VIRTUALIZATION | 66488c4b8d1SNeel Natu PROCBASED2_VIRTUAL_INTERRUPT_DELIVERY); 66588c4b8d1SNeel Natu 66688c4b8d1SNeel Natu use_tpr_shadow = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 66788c4b8d1SNeel Natu MSR_VMX_TRUE_PROCBASED_CTLS, PROCBASED_USE_TPR_SHADOW, 0, 66888c4b8d1SNeel Natu &tmp) == 0); 66988c4b8d1SNeel Natu 67088c4b8d1SNeel Natu error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2, 67188c4b8d1SNeel Natu procbased2_vid_bits, 0, &tmp); 67288c4b8d1SNeel Natu if (error == 0 && use_tpr_shadow) { 67388c4b8d1SNeel Natu virtual_interrupt_delivery = 1; 67488c4b8d1SNeel Natu TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_vid", 67588c4b8d1SNeel Natu &virtual_interrupt_delivery); 67688c4b8d1SNeel Natu } 67788c4b8d1SNeel Natu 67888c4b8d1SNeel Natu if (virtual_interrupt_delivery) { 67988c4b8d1SNeel Natu procbased_ctls |= PROCBASED_USE_TPR_SHADOW; 68088c4b8d1SNeel Natu procbased_ctls2 |= procbased2_vid_bits; 68188c4b8d1SNeel Natu procbased_ctls2 &= ~PROCBASED2_VIRTUALIZE_X2APIC_MODE; 682176666c2SNeel Natu 683176666c2SNeel Natu /* 684594db002STycho Nightingale * No need to emulate accesses to %CR8 if virtual 685594db002STycho Nightingale * interrupt delivery is enabled. 686594db002STycho Nightingale */ 687594db002STycho Nightingale procbased_ctls &= ~PROCBASED_CR8_LOAD_EXITING; 688594db002STycho Nightingale procbased_ctls &= ~PROCBASED_CR8_STORE_EXITING; 689594db002STycho Nightingale 690594db002STycho Nightingale /* 691176666c2SNeel Natu * Check for Posted Interrupts only if Virtual Interrupt 692176666c2SNeel Natu * Delivery is enabled. 693176666c2SNeel Natu */ 694176666c2SNeel Natu error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS, 695176666c2SNeel Natu MSR_VMX_TRUE_PINBASED_CTLS, PINBASED_POSTED_INTERRUPT, 0, 696176666c2SNeel Natu &tmp); 697176666c2SNeel Natu if (error == 0) { 698*bd50262fSKonstantin Belousov pirvec = lapic_ipi_alloc(pti ? &IDTVEC(justreturn1_pti) : 699*bd50262fSKonstantin Belousov &IDTVEC(justreturn)); 70018a2b08eSNeel Natu if (pirvec < 0) { 701176666c2SNeel Natu if (bootverbose) { 702176666c2SNeel Natu printf("vmx_init: unable to allocate " 703176666c2SNeel Natu "posted interrupt vector\n"); 70488c4b8d1SNeel Natu } 705176666c2SNeel Natu } else { 706176666c2SNeel Natu posted_interrupts = 1; 707176666c2SNeel Natu TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_pir", 708176666c2SNeel Natu &posted_interrupts); 709176666c2SNeel Natu } 710176666c2SNeel Natu } 711176666c2SNeel Natu } 712176666c2SNeel Natu 713176666c2SNeel Natu if (posted_interrupts) 714176666c2SNeel Natu pinbased_ctls |= PINBASED_POSTED_INTERRUPT; 71549cc03daSNeel Natu 716366f6083SPeter Grehan /* Initialize EPT */ 717add611fdSNeel Natu error = ept_init(ipinum); 718366f6083SPeter Grehan if (error) { 719366f6083SPeter Grehan printf("vmx_init: ept initialization failed (%d)\n", error); 720366f6083SPeter Grehan return (error); 721366f6083SPeter Grehan } 722366f6083SPeter Grehan 723366f6083SPeter Grehan /* 724366f6083SPeter Grehan * Stash the cr0 and cr4 bits that must be fixed to 0 or 1 725366f6083SPeter Grehan */ 726366f6083SPeter Grehan fixed0 = rdmsr(MSR_VMX_CR0_FIXED0); 727366f6083SPeter Grehan fixed1 = rdmsr(MSR_VMX_CR0_FIXED1); 728366f6083SPeter Grehan cr0_ones_mask = fixed0 & fixed1; 729366f6083SPeter Grehan cr0_zeros_mask = ~fixed0 & ~fixed1; 730366f6083SPeter Grehan 731366f6083SPeter Grehan /* 732366f6083SPeter Grehan * CR0_PE and CR0_PG can be set to zero in VMX non-root operation 733366f6083SPeter Grehan * if unrestricted guest execution is allowed. 734366f6083SPeter Grehan */ 735366f6083SPeter Grehan if (cap_unrestricted_guest) 736366f6083SPeter Grehan cr0_ones_mask &= ~(CR0_PG | CR0_PE); 737366f6083SPeter Grehan 738366f6083SPeter Grehan /* 739366f6083SPeter Grehan * Do not allow the guest to set CR0_NW or CR0_CD. 740366f6083SPeter Grehan */ 741366f6083SPeter Grehan cr0_zeros_mask |= (CR0_NW | CR0_CD); 742366f6083SPeter Grehan 743366f6083SPeter Grehan fixed0 = rdmsr(MSR_VMX_CR4_FIXED0); 744366f6083SPeter Grehan fixed1 = rdmsr(MSR_VMX_CR4_FIXED1); 745366f6083SPeter Grehan cr4_ones_mask = fixed0 & fixed1; 746366f6083SPeter Grehan cr4_zeros_mask = ~fixed0 & ~fixed1; 747366f6083SPeter Grehan 74845e51299SNeel Natu vpid_init(); 74945e51299SNeel Natu 750c3498942SNeel Natu vmx_msr_init(); 751c3498942SNeel Natu 752366f6083SPeter Grehan /* enable VMX operation */ 753366f6083SPeter Grehan smp_rendezvous(NULL, vmx_enable, NULL, NULL); 754366f6083SPeter Grehan 7553565b59eSNeel Natu vmx_initialized = 1; 7563565b59eSNeel Natu 757366f6083SPeter Grehan return (0); 758366f6083SPeter Grehan } 759366f6083SPeter Grehan 760f7d47425SNeel Natu static void 761f7d47425SNeel Natu vmx_trigger_hostintr(int vector) 762f7d47425SNeel Natu { 763f7d47425SNeel Natu uintptr_t func; 764f7d47425SNeel Natu struct gate_descriptor *gd; 765f7d47425SNeel Natu 766f7d47425SNeel Natu gd = &idt[vector]; 767f7d47425SNeel Natu 768f7d47425SNeel Natu KASSERT(vector >= 32 && vector <= 255, ("vmx_trigger_hostintr: " 769f7d47425SNeel Natu "invalid vector %d", vector)); 770f7d47425SNeel Natu KASSERT(gd->gd_p == 1, ("gate descriptor for vector %d not present", 771f7d47425SNeel Natu vector)); 772f7d47425SNeel Natu KASSERT(gd->gd_type == SDT_SYSIGT, ("gate descriptor for vector %d " 773f7d47425SNeel Natu "has invalid type %d", vector, gd->gd_type)); 774f7d47425SNeel Natu KASSERT(gd->gd_dpl == SEL_KPL, ("gate descriptor for vector %d " 775f7d47425SNeel Natu "has invalid dpl %d", vector, gd->gd_dpl)); 776f7d47425SNeel Natu KASSERT(gd->gd_selector == GSEL(GCODE_SEL, SEL_KPL), ("gate descriptor " 777f7d47425SNeel Natu "for vector %d has invalid selector %d", vector, gd->gd_selector)); 778f7d47425SNeel Natu KASSERT(gd->gd_ist == 0, ("gate descriptor for vector %d has invalid " 779f7d47425SNeel Natu "IST %d", vector, gd->gd_ist)); 780f7d47425SNeel Natu 781f7d47425SNeel Natu func = ((long)gd->gd_hioffset << 16 | gd->gd_looffset); 782f7d47425SNeel Natu vmx_call_isr(func); 783f7d47425SNeel Natu } 784f7d47425SNeel Natu 785366f6083SPeter Grehan static int 786aaaa0656SPeter Grehan vmx_setup_cr_shadow(int which, struct vmcs *vmcs, uint32_t initial) 787366f6083SPeter Grehan { 78839c21c2dSNeel Natu int error, mask_ident, shadow_ident; 789aaaa0656SPeter Grehan uint64_t mask_value; 790366f6083SPeter Grehan 79139c21c2dSNeel Natu if (which != 0 && which != 4) 79239c21c2dSNeel Natu panic("vmx_setup_cr_shadow: unknown cr%d", which); 79339c21c2dSNeel Natu 79439c21c2dSNeel Natu if (which == 0) { 79539c21c2dSNeel Natu mask_ident = VMCS_CR0_MASK; 79639c21c2dSNeel Natu mask_value = cr0_ones_mask | cr0_zeros_mask; 79739c21c2dSNeel Natu shadow_ident = VMCS_CR0_SHADOW; 79839c21c2dSNeel Natu } else { 79939c21c2dSNeel Natu mask_ident = VMCS_CR4_MASK; 80039c21c2dSNeel Natu mask_value = cr4_ones_mask | cr4_zeros_mask; 80139c21c2dSNeel Natu shadow_ident = VMCS_CR4_SHADOW; 80239c21c2dSNeel Natu } 80339c21c2dSNeel Natu 804d3c11f40SPeter Grehan error = vmcs_setreg(vmcs, 0, VMCS_IDENT(mask_ident), mask_value); 805366f6083SPeter Grehan if (error) 806366f6083SPeter Grehan return (error); 807366f6083SPeter Grehan 808aaaa0656SPeter Grehan error = vmcs_setreg(vmcs, 0, VMCS_IDENT(shadow_ident), initial); 809366f6083SPeter Grehan if (error) 810366f6083SPeter Grehan return (error); 811366f6083SPeter Grehan 812366f6083SPeter Grehan return (0); 813366f6083SPeter Grehan } 814aaaa0656SPeter Grehan #define vmx_setup_cr0_shadow(vmcs,init) vmx_setup_cr_shadow(0, (vmcs), (init)) 815aaaa0656SPeter Grehan #define vmx_setup_cr4_shadow(vmcs,init) vmx_setup_cr_shadow(4, (vmcs), (init)) 816366f6083SPeter Grehan 817366f6083SPeter Grehan static void * 818318224bbSNeel Natu vmx_vminit(struct vm *vm, pmap_t pmap) 819366f6083SPeter Grehan { 82045e51299SNeel Natu uint16_t vpid[VM_MAXCPU]; 821c3498942SNeel Natu int i, error; 822366f6083SPeter Grehan struct vmx *vmx; 823c847a506SNeel Natu struct vmcs *vmcs; 824b0538143SNeel Natu uint32_t exc_bitmap; 825366f6083SPeter Grehan 826366f6083SPeter Grehan vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO); 827366f6083SPeter Grehan if ((uintptr_t)vmx & PAGE_MASK) { 828366f6083SPeter Grehan panic("malloc of struct vmx not aligned on %d byte boundary", 829366f6083SPeter Grehan PAGE_SIZE); 830366f6083SPeter Grehan } 831366f6083SPeter Grehan vmx->vm = vm; 832366f6083SPeter Grehan 833318224bbSNeel Natu vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pml4)); 834318224bbSNeel Natu 835366f6083SPeter Grehan /* 836366f6083SPeter Grehan * Clean up EPTP-tagged guest physical and combined mappings 837366f6083SPeter Grehan * 838366f6083SPeter Grehan * VMX transitions are not required to invalidate any guest physical 839366f6083SPeter Grehan * mappings. So, it may be possible for stale guest physical mappings 840366f6083SPeter Grehan * to be present in the processor TLBs. 841366f6083SPeter Grehan * 842366f6083SPeter Grehan * Combined mappings for this EP4TA are also invalidated for all VPIDs. 843366f6083SPeter Grehan */ 844318224bbSNeel Natu ept_invalidate_mappings(vmx->eptp); 845366f6083SPeter Grehan 846366f6083SPeter Grehan msr_bitmap_initialize(vmx->msr_bitmap); 847366f6083SPeter Grehan 848366f6083SPeter Grehan /* 849366f6083SPeter Grehan * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE. 850366f6083SPeter Grehan * The guest FSBASE and GSBASE are saved and restored during 851366f6083SPeter Grehan * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are 852366f6083SPeter Grehan * always restored from the vmcs host state area on vm-exit. 853366f6083SPeter Grehan * 8541fb0ea3fSPeter Grehan * The SYSENTER_CS/ESP/EIP MSRs are identical to FS/GSBASE in 8551fb0ea3fSPeter Grehan * how they are saved/restored so can be directly accessed by the 8561fb0ea3fSPeter Grehan * guest. 8571fb0ea3fSPeter Grehan * 858366f6083SPeter Grehan * MSR_EFER is saved and restored in the guest VMCS area on a 859366f6083SPeter Grehan * VM exit and entry respectively. It is also restored from the 860366f6083SPeter Grehan * host VMCS area on a VM exit. 8618d1d7a9eSPeter Grehan * 862277bdd99STycho Nightingale * The TSC MSR is exposed read-only. Writes are disallowed as 863277bdd99STycho Nightingale * that will impact the host TSC. If the guest does a write 864277bdd99STycho Nightingale * the "use TSC offsetting" execution control is enabled and the 865277bdd99STycho Nightingale * difference between the host TSC and the guest TSC is written 866277bdd99STycho Nightingale * into the TSC offset in the VMCS. 867366f6083SPeter Grehan */ 868366f6083SPeter Grehan if (guest_msr_rw(vmx, MSR_GSBASE) || 869366f6083SPeter Grehan guest_msr_rw(vmx, MSR_FSBASE) || 8701fb0ea3fSPeter Grehan guest_msr_rw(vmx, MSR_SYSENTER_CS_MSR) || 8711fb0ea3fSPeter Grehan guest_msr_rw(vmx, MSR_SYSENTER_ESP_MSR) || 8721fb0ea3fSPeter Grehan guest_msr_rw(vmx, MSR_SYSENTER_EIP_MSR) || 8738d1d7a9eSPeter Grehan guest_msr_rw(vmx, MSR_EFER) || 8748d1d7a9eSPeter Grehan guest_msr_ro(vmx, MSR_TSC)) 875366f6083SPeter Grehan panic("vmx_vminit: error setting guest msr access"); 876366f6083SPeter Grehan 87745e51299SNeel Natu vpid_alloc(vpid, VM_MAXCPU); 87845e51299SNeel Natu 87988c4b8d1SNeel Natu if (virtual_interrupt_delivery) { 88088c4b8d1SNeel Natu error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE, 88188c4b8d1SNeel Natu APIC_ACCESS_ADDRESS); 88288c4b8d1SNeel Natu /* XXX this should really return an error to the caller */ 88388c4b8d1SNeel Natu KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error)); 88488c4b8d1SNeel Natu } 88588c4b8d1SNeel Natu 886366f6083SPeter Grehan for (i = 0; i < VM_MAXCPU; i++) { 887c847a506SNeel Natu vmcs = &vmx->vmcs[i]; 888c847a506SNeel Natu vmcs->identifier = vmx_revision(); 889c847a506SNeel Natu error = vmclear(vmcs); 890366f6083SPeter Grehan if (error != 0) { 891366f6083SPeter Grehan panic("vmx_vminit: vmclear error %d on vcpu %d\n", 892366f6083SPeter Grehan error, i); 893366f6083SPeter Grehan } 894366f6083SPeter Grehan 895c3498942SNeel Natu vmx_msr_guest_init(vmx, i); 896c3498942SNeel Natu 897c847a506SNeel Natu error = vmcs_init(vmcs); 898c847a506SNeel Natu KASSERT(error == 0, ("vmcs_init error %d", error)); 899366f6083SPeter Grehan 900c847a506SNeel Natu VMPTRLD(vmcs); 901c847a506SNeel Natu error = 0; 902c847a506SNeel Natu error += vmwrite(VMCS_HOST_RSP, (u_long)&vmx->ctx[i]); 903c847a506SNeel Natu error += vmwrite(VMCS_EPTP, vmx->eptp); 904c847a506SNeel Natu error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls); 905c847a506SNeel Natu error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls); 906c847a506SNeel Natu error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2); 907c847a506SNeel Natu error += vmwrite(VMCS_EXIT_CTLS, exit_ctls); 908c847a506SNeel Natu error += vmwrite(VMCS_ENTRY_CTLS, entry_ctls); 909c847a506SNeel Natu error += vmwrite(VMCS_MSR_BITMAP, vtophys(vmx->msr_bitmap)); 910c847a506SNeel Natu error += vmwrite(VMCS_VPID, vpid[i]); 911b0538143SNeel Natu 912b0538143SNeel Natu /* exception bitmap */ 913b0538143SNeel Natu if (vcpu_trace_exceptions(vm, i)) 914b0538143SNeel Natu exc_bitmap = 0xffffffff; 915b0538143SNeel Natu else 916b0538143SNeel Natu exc_bitmap = 1 << IDT_MC; 917b0538143SNeel Natu error += vmwrite(VMCS_EXCEPTION_BITMAP, exc_bitmap); 918b0538143SNeel Natu 91988c4b8d1SNeel Natu if (virtual_interrupt_delivery) { 92088c4b8d1SNeel Natu error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS); 92188c4b8d1SNeel Natu error += vmwrite(VMCS_VIRTUAL_APIC, 92288c4b8d1SNeel Natu vtophys(&vmx->apic_page[i])); 92388c4b8d1SNeel Natu error += vmwrite(VMCS_EOI_EXIT0, 0); 92488c4b8d1SNeel Natu error += vmwrite(VMCS_EOI_EXIT1, 0); 92588c4b8d1SNeel Natu error += vmwrite(VMCS_EOI_EXIT2, 0); 92688c4b8d1SNeel Natu error += vmwrite(VMCS_EOI_EXIT3, 0); 92788c4b8d1SNeel Natu } 928176666c2SNeel Natu if (posted_interrupts) { 929176666c2SNeel Natu error += vmwrite(VMCS_PIR_VECTOR, pirvec); 930176666c2SNeel Natu error += vmwrite(VMCS_PIR_DESC, 931176666c2SNeel Natu vtophys(&vmx->pir_desc[i])); 932176666c2SNeel Natu } 933c847a506SNeel Natu VMCLEAR(vmcs); 934c847a506SNeel Natu KASSERT(error == 0, ("vmx_vminit: error customizing the vmcs")); 935366f6083SPeter Grehan 936366f6083SPeter Grehan vmx->cap[i].set = 0; 937366f6083SPeter Grehan vmx->cap[i].proc_ctls = procbased_ctls; 93849cc03daSNeel Natu vmx->cap[i].proc_ctls2 = procbased_ctls2; 939366f6083SPeter Grehan 9402ce12423SNeel Natu vmx->state[i].nextrip = ~0; 9413527963bSNeel Natu vmx->state[i].lastcpu = NOCPU; 94245e51299SNeel Natu vmx->state[i].vpid = vpid[i]; 943366f6083SPeter Grehan 944aaaa0656SPeter Grehan /* 945aaaa0656SPeter Grehan * Set up the CR0/4 shadows, and init the read shadow 946aaaa0656SPeter Grehan * to the power-on register value from the Intel Sys Arch. 947aaaa0656SPeter Grehan * CR0 - 0x60000010 948aaaa0656SPeter Grehan * CR4 - 0 949aaaa0656SPeter Grehan */ 950c847a506SNeel Natu error = vmx_setup_cr0_shadow(vmcs, 0x60000010); 95139c21c2dSNeel Natu if (error != 0) 95239c21c2dSNeel Natu panic("vmx_setup_cr0_shadow %d", error); 95339c21c2dSNeel Natu 954c847a506SNeel Natu error = vmx_setup_cr4_shadow(vmcs, 0); 95539c21c2dSNeel Natu if (error != 0) 95639c21c2dSNeel Natu panic("vmx_setup_cr4_shadow %d", error); 957318224bbSNeel Natu 958318224bbSNeel Natu vmx->ctx[i].pmap = pmap; 959366f6083SPeter Grehan } 960366f6083SPeter Grehan 961366f6083SPeter Grehan return (vmx); 962366f6083SPeter Grehan } 963366f6083SPeter Grehan 964366f6083SPeter Grehan static int 965a2da7af6SNeel Natu vmx_handle_cpuid(struct vm *vm, int vcpu, struct vmxctx *vmxctx) 966366f6083SPeter Grehan { 967366f6083SPeter Grehan int handled, func; 968366f6083SPeter Grehan 969366f6083SPeter Grehan func = vmxctx->guest_rax; 970366f6083SPeter Grehan 971a2da7af6SNeel Natu handled = x86_emulate_cpuid(vm, vcpu, 972a2da7af6SNeel Natu (uint32_t*)(&vmxctx->guest_rax), 973a2da7af6SNeel Natu (uint32_t*)(&vmxctx->guest_rbx), 974a2da7af6SNeel Natu (uint32_t*)(&vmxctx->guest_rcx), 975a2da7af6SNeel Natu (uint32_t*)(&vmxctx->guest_rdx)); 976366f6083SPeter Grehan return (handled); 977366f6083SPeter Grehan } 978366f6083SPeter Grehan 979366f6083SPeter Grehan static __inline void 980366f6083SPeter Grehan vmx_run_trace(struct vmx *vmx, int vcpu) 981366f6083SPeter Grehan { 982366f6083SPeter Grehan #ifdef KTR 983513c8d33SNeel Natu VCPU_CTR1(vmx->vm, vcpu, "Resume execution at %#lx", vmcs_guest_rip()); 984366f6083SPeter Grehan #endif 985366f6083SPeter Grehan } 986366f6083SPeter Grehan 987366f6083SPeter Grehan static __inline void 988366f6083SPeter Grehan vmx_exit_trace(struct vmx *vmx, int vcpu, uint64_t rip, uint32_t exit_reason, 989eeefa4e4SNeel Natu int handled) 990366f6083SPeter Grehan { 991366f6083SPeter Grehan #ifdef KTR 992513c8d33SNeel Natu VCPU_CTR3(vmx->vm, vcpu, "%s %s vmexit at 0x%0lx", 993366f6083SPeter Grehan handled ? "handled" : "unhandled", 994366f6083SPeter Grehan exit_reason_to_str(exit_reason), rip); 995eeefa4e4SNeel Natu #endif 996eeefa4e4SNeel Natu } 997366f6083SPeter Grehan 998eeefa4e4SNeel Natu static __inline void 999eeefa4e4SNeel Natu vmx_astpending_trace(struct vmx *vmx, int vcpu, uint64_t rip) 1000eeefa4e4SNeel Natu { 1001eeefa4e4SNeel Natu #ifdef KTR 1002513c8d33SNeel Natu VCPU_CTR1(vmx->vm, vcpu, "astpending vmexit at 0x%0lx", rip); 1003366f6083SPeter Grehan #endif 1004366f6083SPeter Grehan } 1005366f6083SPeter Grehan 1006953c2c47SNeel Natu static VMM_STAT_INTEL(VCPU_INVVPID_SAVED, "Number of vpid invalidations saved"); 10073527963bSNeel Natu static VMM_STAT_INTEL(VCPU_INVVPID_DONE, "Number of vpid invalidations done"); 1008953c2c47SNeel Natu 10093527963bSNeel Natu /* 10103527963bSNeel Natu * Invalidate guest mappings identified by its vpid from the TLB. 10113527963bSNeel Natu */ 10123527963bSNeel Natu static __inline void 10133527963bSNeel Natu vmx_invvpid(struct vmx *vmx, int vcpu, pmap_t pmap, int running) 1014366f6083SPeter Grehan { 1015366f6083SPeter Grehan struct vmxstate *vmxstate; 1016953c2c47SNeel Natu struct invvpid_desc invvpid_desc; 1017366f6083SPeter Grehan 1018366f6083SPeter Grehan vmxstate = &vmx->state[vcpu]; 10193527963bSNeel Natu if (vmxstate->vpid == 0) 10203de83862SNeel Natu return; 1021366f6083SPeter Grehan 10223527963bSNeel Natu if (!running) { 10233527963bSNeel Natu /* 10243527963bSNeel Natu * Set the 'lastcpu' to an invalid host cpu. 10253527963bSNeel Natu * 10263527963bSNeel Natu * This will invalidate TLB entries tagged with the vcpu's 10273527963bSNeel Natu * vpid the next time it runs via vmx_set_pcpu_defaults(). 10283527963bSNeel Natu */ 10293527963bSNeel Natu vmxstate->lastcpu = NOCPU; 10303527963bSNeel Natu return; 10313527963bSNeel Natu } 1032953c2c47SNeel Natu 10333527963bSNeel Natu KASSERT(curthread->td_critnest > 0, ("%s: vcpu %d running outside " 10343527963bSNeel Natu "critical section", __func__, vcpu)); 1035366f6083SPeter Grehan 1036366f6083SPeter Grehan /* 10373527963bSNeel Natu * Invalidate all mappings tagged with 'vpid' 1038366f6083SPeter Grehan * 1039366f6083SPeter Grehan * We do this because this vcpu was executing on a different host 1040366f6083SPeter Grehan * cpu when it last ran. We do not track whether it invalidated 1041366f6083SPeter Grehan * mappings associated with its 'vpid' during that run. So we must 1042366f6083SPeter Grehan * assume that the mappings associated with 'vpid' on 'curcpu' are 1043366f6083SPeter Grehan * stale and invalidate them. 1044366f6083SPeter Grehan * 1045366f6083SPeter Grehan * Note that we incur this penalty only when the scheduler chooses to 1046366f6083SPeter Grehan * move the thread associated with this vcpu between host cpus. 1047366f6083SPeter Grehan * 1048366f6083SPeter Grehan * Note also that this will invalidate mappings tagged with 'vpid' 1049366f6083SPeter Grehan * for "all" EP4TAs. 1050366f6083SPeter Grehan */ 1051953c2c47SNeel Natu if (pmap->pm_eptgen == vmx->eptgen[curcpu]) { 1052953c2c47SNeel Natu invvpid_desc._res1 = 0; 1053953c2c47SNeel Natu invvpid_desc._res2 = 0; 1054366f6083SPeter Grehan invvpid_desc.vpid = vmxstate->vpid; 10550e30c5c0SWarner Losh invvpid_desc.linear_addr = 0; 1056366f6083SPeter Grehan invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc); 10573527963bSNeel Natu vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_DONE, 1); 1058953c2c47SNeel Natu } else { 1059953c2c47SNeel Natu /* 1060953c2c47SNeel Natu * The invvpid can be skipped if an invept is going to 1061953c2c47SNeel Natu * be performed before entering the guest. The invept 1062953c2c47SNeel Natu * will invalidate combined mappings tagged with 1063953c2c47SNeel Natu * 'vmx->eptp' for all vpids. 1064953c2c47SNeel Natu */ 1065953c2c47SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_SAVED, 1); 1066953c2c47SNeel Natu } 1067366f6083SPeter Grehan } 10683527963bSNeel Natu 10693527963bSNeel Natu static void 10703527963bSNeel Natu vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu, pmap_t pmap) 10713527963bSNeel Natu { 10723527963bSNeel Natu struct vmxstate *vmxstate; 10733527963bSNeel Natu 10743527963bSNeel Natu vmxstate = &vmx->state[vcpu]; 10753527963bSNeel Natu if (vmxstate->lastcpu == curcpu) 10763527963bSNeel Natu return; 10773527963bSNeel Natu 10783527963bSNeel Natu vmxstate->lastcpu = curcpu; 10793527963bSNeel Natu 10803527963bSNeel Natu vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1); 10813527963bSNeel Natu 10823527963bSNeel Natu vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase()); 10833527963bSNeel Natu vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase()); 10843527963bSNeel Natu vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase()); 10853527963bSNeel Natu vmx_invvpid(vmx, vcpu, pmap, 1); 1086366f6083SPeter Grehan } 1087366f6083SPeter Grehan 1088366f6083SPeter Grehan /* 1089366f6083SPeter Grehan * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set. 1090366f6083SPeter Grehan */ 1091366f6083SPeter Grehan CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0); 1092366f6083SPeter Grehan 1093366f6083SPeter Grehan static void __inline 1094366f6083SPeter Grehan vmx_set_int_window_exiting(struct vmx *vmx, int vcpu) 1095366f6083SPeter Grehan { 1096366f6083SPeter Grehan 109748b2d828SNeel Natu if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) { 1098366f6083SPeter Grehan vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING; 10993de83862SNeel Natu vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 110048b2d828SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting"); 110148b2d828SNeel Natu } 1102366f6083SPeter Grehan } 1103366f6083SPeter Grehan 1104366f6083SPeter Grehan static void __inline 1105366f6083SPeter Grehan vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu) 1106366f6083SPeter Grehan { 1107366f6083SPeter Grehan 110848b2d828SNeel Natu KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0, 110948b2d828SNeel Natu ("intr_window_exiting not set: %#x", vmx->cap[vcpu].proc_ctls)); 1110366f6083SPeter Grehan vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING; 11113de83862SNeel Natu vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 111248b2d828SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting"); 1113366f6083SPeter Grehan } 1114366f6083SPeter Grehan 1115366f6083SPeter Grehan static void __inline 1116366f6083SPeter Grehan vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu) 1117366f6083SPeter Grehan { 1118366f6083SPeter Grehan 111948b2d828SNeel Natu if ((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) { 1120366f6083SPeter Grehan vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING; 11213de83862SNeel Natu vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 112248b2d828SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting"); 112348b2d828SNeel Natu } 1124366f6083SPeter Grehan } 1125366f6083SPeter Grehan 1126366f6083SPeter Grehan static void __inline 1127366f6083SPeter Grehan vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu) 1128366f6083SPeter Grehan { 1129366f6083SPeter Grehan 113048b2d828SNeel Natu KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0, 113148b2d828SNeel Natu ("nmi_window_exiting not set %#x", vmx->cap[vcpu].proc_ctls)); 1132366f6083SPeter Grehan vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING; 11333de83862SNeel Natu vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 113448b2d828SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting"); 1135366f6083SPeter Grehan } 1136366f6083SPeter Grehan 1137277bdd99STycho Nightingale int 1138277bdd99STycho Nightingale vmx_set_tsc_offset(struct vmx *vmx, int vcpu, uint64_t offset) 1139277bdd99STycho Nightingale { 1140277bdd99STycho Nightingale int error; 1141277bdd99STycho Nightingale 1142277bdd99STycho Nightingale if ((vmx->cap[vcpu].proc_ctls & PROCBASED_TSC_OFFSET) == 0) { 1143277bdd99STycho Nightingale vmx->cap[vcpu].proc_ctls |= PROCBASED_TSC_OFFSET; 1144277bdd99STycho Nightingale vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 1145277bdd99STycho Nightingale VCPU_CTR0(vmx->vm, vcpu, "Enabling TSC offsetting"); 1146277bdd99STycho Nightingale } 1147277bdd99STycho Nightingale 1148277bdd99STycho Nightingale error = vmwrite(VMCS_TSC_OFFSET, offset); 1149277bdd99STycho Nightingale 1150277bdd99STycho Nightingale return (error); 1151277bdd99STycho Nightingale } 1152277bdd99STycho Nightingale 115348b2d828SNeel Natu #define NMI_BLOCKING (VMCS_INTERRUPTIBILITY_NMI_BLOCKING | \ 115448b2d828SNeel Natu VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING) 115548b2d828SNeel Natu #define HWINTR_BLOCKING (VMCS_INTERRUPTIBILITY_STI_BLOCKING | \ 115648b2d828SNeel Natu VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING) 115748b2d828SNeel Natu 115848b2d828SNeel Natu static void 1159366f6083SPeter Grehan vmx_inject_nmi(struct vmx *vmx, int vcpu) 1160366f6083SPeter Grehan { 116148b2d828SNeel Natu uint32_t gi, info; 1162366f6083SPeter Grehan 116348b2d828SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 116448b2d828SNeel Natu KASSERT((gi & NMI_BLOCKING) == 0, ("vmx_inject_nmi: invalid guest " 116548b2d828SNeel Natu "interruptibility-state %#x", gi)); 1166366f6083SPeter Grehan 116748b2d828SNeel Natu info = vmcs_read(VMCS_ENTRY_INTR_INFO); 116848b2d828SNeel Natu KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_nmi: invalid " 116948b2d828SNeel Natu "VM-entry interruption information %#x", info)); 1170366f6083SPeter Grehan 1171366f6083SPeter Grehan /* 1172366f6083SPeter Grehan * Inject the virtual NMI. The vector must be the NMI IDT entry 1173366f6083SPeter Grehan * or the VMCS entry check will fail. 1174366f6083SPeter Grehan */ 117548b2d828SNeel Natu info = IDT_NMI | VMCS_INTR_T_NMI | VMCS_INTR_VALID; 11763de83862SNeel Natu vmcs_write(VMCS_ENTRY_INTR_INFO, info); 1177366f6083SPeter Grehan 1178513c8d33SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Injecting vNMI"); 1179366f6083SPeter Grehan 1180366f6083SPeter Grehan /* Clear the request */ 1181f352ff0cSNeel Natu vm_nmi_clear(vmx->vm, vcpu); 1182366f6083SPeter Grehan } 1183366f6083SPeter Grehan 1184366f6083SPeter Grehan static void 11852ce12423SNeel Natu vmx_inject_interrupts(struct vmx *vmx, int vcpu, struct vlapic *vlapic, 11862ce12423SNeel Natu uint64_t guestrip) 1187366f6083SPeter Grehan { 11880775fbb4STycho Nightingale int vector, need_nmi_exiting, extint_pending; 1189091d4532SNeel Natu uint64_t rflags, entryinfo; 119048b2d828SNeel Natu uint32_t gi, info; 1191366f6083SPeter Grehan 11922ce12423SNeel Natu if (vmx->state[vcpu].nextrip != guestrip) { 11932ce12423SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 11942ce12423SNeel Natu if (gi & HWINTR_BLOCKING) { 11952ce12423SNeel Natu VCPU_CTR2(vmx->vm, vcpu, "Guest interrupt blocking " 11962ce12423SNeel Natu "cleared due to rip change: %#lx/%#lx", 11972ce12423SNeel Natu vmx->state[vcpu].nextrip, guestrip); 11982ce12423SNeel Natu gi &= ~HWINTR_BLOCKING; 11992ce12423SNeel Natu vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi); 12002ce12423SNeel Natu } 12012ce12423SNeel Natu } 12022ce12423SNeel Natu 1203091d4532SNeel Natu if (vm_entry_intinfo(vmx->vm, vcpu, &entryinfo)) { 1204091d4532SNeel Natu KASSERT((entryinfo & VMCS_INTR_VALID) != 0, ("%s: entry " 1205091d4532SNeel Natu "intinfo is not valid: %#lx", __func__, entryinfo)); 1206dc506506SNeel Natu 1207dc506506SNeel Natu info = vmcs_read(VMCS_ENTRY_INTR_INFO); 1208dc506506SNeel Natu KASSERT((info & VMCS_INTR_VALID) == 0, ("%s: cannot inject " 1209019008ebSNeel Natu "pending exception: %#lx/%#x", __func__, entryinfo, info)); 1210dc506506SNeel Natu 1211091d4532SNeel Natu info = entryinfo; 1212091d4532SNeel Natu vector = info & 0xff; 1213091d4532SNeel Natu if (vector == IDT_BP || vector == IDT_OF) { 1214091d4532SNeel Natu /* 1215091d4532SNeel Natu * VT-x requires #BP and #OF to be injected as software 1216091d4532SNeel Natu * exceptions. 1217091d4532SNeel Natu */ 1218091d4532SNeel Natu info &= ~VMCS_INTR_T_MASK; 1219091d4532SNeel Natu info |= VMCS_INTR_T_SWEXCEPTION; 1220dc506506SNeel Natu } 1221091d4532SNeel Natu 1222091d4532SNeel Natu if (info & VMCS_INTR_DEL_ERRCODE) 1223091d4532SNeel Natu vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR, entryinfo >> 32); 1224091d4532SNeel Natu 1225dc506506SNeel Natu vmcs_write(VMCS_ENTRY_INTR_INFO, info); 1226dc506506SNeel Natu } 1227dc506506SNeel Natu 122848b2d828SNeel Natu if (vm_nmi_pending(vmx->vm, vcpu)) { 1229366f6083SPeter Grehan /* 123048b2d828SNeel Natu * If there are no conditions blocking NMI injection then 123148b2d828SNeel Natu * inject it directly here otherwise enable "NMI window 123248b2d828SNeel Natu * exiting" to inject it as soon as we can. 1233eeefa4e4SNeel Natu * 123448b2d828SNeel Natu * We also check for STI_BLOCKING because some implementations 123548b2d828SNeel Natu * don't allow NMI injection in this case. If we are running 123648b2d828SNeel Natu * on a processor that doesn't have this restriction it will 123748b2d828SNeel Natu * immediately exit and the NMI will be injected in the 123848b2d828SNeel Natu * "NMI window exiting" handler. 1239366f6083SPeter Grehan */ 124048b2d828SNeel Natu need_nmi_exiting = 1; 124148b2d828SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 124248b2d828SNeel Natu if ((gi & (HWINTR_BLOCKING | NMI_BLOCKING)) == 0) { 12433de83862SNeel Natu info = vmcs_read(VMCS_ENTRY_INTR_INFO); 124448b2d828SNeel Natu if ((info & VMCS_INTR_VALID) == 0) { 124548b2d828SNeel Natu vmx_inject_nmi(vmx, vcpu); 124648b2d828SNeel Natu need_nmi_exiting = 0; 124748b2d828SNeel Natu } else { 124848b2d828SNeel Natu VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI " 124948b2d828SNeel Natu "due to VM-entry intr info %#x", info); 125048b2d828SNeel Natu } 125148b2d828SNeel Natu } else { 125248b2d828SNeel Natu VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI due to " 125348b2d828SNeel Natu "Guest Interruptibility-state %#x", gi); 125448b2d828SNeel Natu } 1255eeefa4e4SNeel Natu 125648b2d828SNeel Natu if (need_nmi_exiting) 125748b2d828SNeel Natu vmx_set_nmi_window_exiting(vmx, vcpu); 125848b2d828SNeel Natu } 1259366f6083SPeter Grehan 12600775fbb4STycho Nightingale extint_pending = vm_extint_pending(vmx->vm, vcpu); 12610775fbb4STycho Nightingale 12620775fbb4STycho Nightingale if (!extint_pending && virtual_interrupt_delivery) { 126388c4b8d1SNeel Natu vmx_inject_pir(vlapic); 126488c4b8d1SNeel Natu return; 126588c4b8d1SNeel Natu } 126688c4b8d1SNeel Natu 126748b2d828SNeel Natu /* 126836736912SNeel Natu * If interrupt-window exiting is already in effect then don't bother 126936736912SNeel Natu * checking for pending interrupts. This is just an optimization and 127036736912SNeel Natu * not needed for correctness. 127148b2d828SNeel Natu */ 127236736912SNeel Natu if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) { 127336736912SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Skip interrupt injection due to " 127436736912SNeel Natu "pending int_window_exiting"); 127548b2d828SNeel Natu return; 127636736912SNeel Natu } 127748b2d828SNeel Natu 12780775fbb4STycho Nightingale if (!extint_pending) { 1279366f6083SPeter Grehan /* Ask the local apic for a vector to inject */ 12804d1e82a8SNeel Natu if (!vlapic_pending_intr(vlapic, &vector)) 1281366f6083SPeter Grehan return; 1282a026dc3fSTycho Nightingale 1283a026dc3fSTycho Nightingale /* 1284a026dc3fSTycho Nightingale * From the Intel SDM, Volume 3, Section "Maskable 1285a026dc3fSTycho Nightingale * Hardware Interrupts": 1286a026dc3fSTycho Nightingale * - maskable interrupt vectors [16,255] can be delivered 1287a026dc3fSTycho Nightingale * through the local APIC. 1288a026dc3fSTycho Nightingale */ 1289a026dc3fSTycho Nightingale KASSERT(vector >= 16 && vector <= 255, 1290a026dc3fSTycho Nightingale ("invalid vector %d from local APIC", vector)); 12910775fbb4STycho Nightingale } else { 12920775fbb4STycho Nightingale /* Ask the legacy pic for a vector to inject */ 12930775fbb4STycho Nightingale vatpic_pending_intr(vmx->vm, &vector); 1294366f6083SPeter Grehan 1295a026dc3fSTycho Nightingale /* 1296a026dc3fSTycho Nightingale * From the Intel SDM, Volume 3, Section "Maskable 1297a026dc3fSTycho Nightingale * Hardware Interrupts": 1298a026dc3fSTycho Nightingale * - maskable interrupt vectors [0,255] can be delivered 1299a026dc3fSTycho Nightingale * through the INTR pin. 1300a026dc3fSTycho Nightingale */ 1301a026dc3fSTycho Nightingale KASSERT(vector >= 0 && vector <= 255, 1302a026dc3fSTycho Nightingale ("invalid vector %d from INTR", vector)); 1303a026dc3fSTycho Nightingale } 1304366f6083SPeter Grehan 1305366f6083SPeter Grehan /* Check RFLAGS.IF and the interruptibility state of the guest */ 13063de83862SNeel Natu rflags = vmcs_read(VMCS_GUEST_RFLAGS); 130736736912SNeel Natu if ((rflags & PSL_I) == 0) { 130836736912SNeel Natu VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to " 130936736912SNeel Natu "rflags %#lx", vector, rflags); 1310366f6083SPeter Grehan goto cantinject; 131136736912SNeel Natu } 1312366f6083SPeter Grehan 131348b2d828SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 131436736912SNeel Natu if (gi & HWINTR_BLOCKING) { 131536736912SNeel Natu VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to " 131636736912SNeel Natu "Guest Interruptibility-state %#x", vector, gi); 1317366f6083SPeter Grehan goto cantinject; 131836736912SNeel Natu } 131936736912SNeel Natu 132036736912SNeel Natu info = vmcs_read(VMCS_ENTRY_INTR_INFO); 132136736912SNeel Natu if (info & VMCS_INTR_VALID) { 132236736912SNeel Natu /* 132336736912SNeel Natu * This is expected and could happen for multiple reasons: 132436736912SNeel Natu * - A vectoring VM-entry was aborted due to astpending 132536736912SNeel Natu * - A VM-exit happened during event injection. 1326dc506506SNeel Natu * - An exception was injected above. 132736736912SNeel Natu * - An NMI was injected above or after "NMI window exiting" 132836736912SNeel Natu */ 132936736912SNeel Natu VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to " 133036736912SNeel Natu "VM-entry intr info %#x", vector, info); 133136736912SNeel Natu goto cantinject; 133236736912SNeel Natu } 1333366f6083SPeter Grehan 1334366f6083SPeter Grehan /* Inject the interrupt */ 1335160471d2SNeel Natu info = VMCS_INTR_T_HWINTR | VMCS_INTR_VALID; 1336366f6083SPeter Grehan info |= vector; 13373de83862SNeel Natu vmcs_write(VMCS_ENTRY_INTR_INFO, info); 1338366f6083SPeter Grehan 13390775fbb4STycho Nightingale if (!extint_pending) { 1340366f6083SPeter Grehan /* Update the Local APIC ISR */ 1341de5ea6b6SNeel Natu vlapic_intr_accepted(vlapic, vector); 13420775fbb4STycho Nightingale } else { 13430775fbb4STycho Nightingale vm_extint_clear(vmx->vm, vcpu); 13440775fbb4STycho Nightingale vatpic_intr_accepted(vmx->vm, vector); 13450775fbb4STycho Nightingale 13460775fbb4STycho Nightingale /* 13470775fbb4STycho Nightingale * After we accepted the current ExtINT the PIC may 13480775fbb4STycho Nightingale * have posted another one. If that is the case, set 13490775fbb4STycho Nightingale * the Interrupt Window Exiting execution control so 13500775fbb4STycho Nightingale * we can inject that one too. 13510494cb1bSNeel Natu * 13520494cb1bSNeel Natu * Also, interrupt window exiting allows us to inject any 13530494cb1bSNeel Natu * pending APIC vector that was preempted by the ExtINT 13540494cb1bSNeel Natu * as soon as possible. This applies both for the software 13550494cb1bSNeel Natu * emulated vlapic and the hardware assisted virtual APIC. 13560775fbb4STycho Nightingale */ 13570775fbb4STycho Nightingale vmx_set_int_window_exiting(vmx, vcpu); 13580775fbb4STycho Nightingale } 1359366f6083SPeter Grehan 1360513c8d33SNeel Natu VCPU_CTR1(vmx->vm, vcpu, "Injecting hwintr at vector %d", vector); 1361366f6083SPeter Grehan 1362366f6083SPeter Grehan return; 1363366f6083SPeter Grehan 1364366f6083SPeter Grehan cantinject: 1365366f6083SPeter Grehan /* 1366366f6083SPeter Grehan * Set the Interrupt Window Exiting execution control so we can inject 1367366f6083SPeter Grehan * the interrupt as soon as blocking condition goes away. 1368366f6083SPeter Grehan */ 1369366f6083SPeter Grehan vmx_set_int_window_exiting(vmx, vcpu); 1370366f6083SPeter Grehan } 1371366f6083SPeter Grehan 1372e5a1d950SNeel Natu /* 1373e5a1d950SNeel Natu * If the Virtual NMIs execution control is '1' then the logical processor 1374e5a1d950SNeel Natu * tracks virtual-NMI blocking in the Guest Interruptibility-state field of 1375e5a1d950SNeel Natu * the VMCS. An IRET instruction in VMX non-root operation will remove any 1376e5a1d950SNeel Natu * virtual-NMI blocking. 1377e5a1d950SNeel Natu * 1378e5a1d950SNeel Natu * This unblocking occurs even if the IRET causes a fault. In this case the 1379e5a1d950SNeel Natu * hypervisor needs to restore virtual-NMI blocking before resuming the guest. 1380e5a1d950SNeel Natu */ 1381e5a1d950SNeel Natu static void 1382e5a1d950SNeel Natu vmx_restore_nmi_blocking(struct vmx *vmx, int vcpuid) 1383e5a1d950SNeel Natu { 1384e5a1d950SNeel Natu uint32_t gi; 1385e5a1d950SNeel Natu 1386e5a1d950SNeel Natu VCPU_CTR0(vmx->vm, vcpuid, "Restore Virtual-NMI blocking"); 1387e5a1d950SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 1388e5a1d950SNeel Natu gi |= VMCS_INTERRUPTIBILITY_NMI_BLOCKING; 1389e5a1d950SNeel Natu vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi); 1390e5a1d950SNeel Natu } 1391e5a1d950SNeel Natu 1392e5a1d950SNeel Natu static void 1393e5a1d950SNeel Natu vmx_clear_nmi_blocking(struct vmx *vmx, int vcpuid) 1394e5a1d950SNeel Natu { 1395e5a1d950SNeel Natu uint32_t gi; 1396e5a1d950SNeel Natu 1397e5a1d950SNeel Natu VCPU_CTR0(vmx->vm, vcpuid, "Clear Virtual-NMI blocking"); 1398e5a1d950SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 1399e5a1d950SNeel Natu gi &= ~VMCS_INTERRUPTIBILITY_NMI_BLOCKING; 1400e5a1d950SNeel Natu vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi); 1401e5a1d950SNeel Natu } 1402e5a1d950SNeel Natu 1403091d4532SNeel Natu static void 1404091d4532SNeel Natu vmx_assert_nmi_blocking(struct vmx *vmx, int vcpuid) 1405091d4532SNeel Natu { 1406091d4532SNeel Natu uint32_t gi; 1407091d4532SNeel Natu 1408091d4532SNeel Natu gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 1409091d4532SNeel Natu KASSERT(gi & VMCS_INTERRUPTIBILITY_NMI_BLOCKING, 1410091d4532SNeel Natu ("NMI blocking is not in effect %#x", gi)); 1411091d4532SNeel Natu } 1412091d4532SNeel Natu 1413366f6083SPeter Grehan static int 1414a0efd3fbSJohn Baldwin vmx_emulate_xsetbv(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) 1415abb023fbSJohn Baldwin { 1416abb023fbSJohn Baldwin struct vmxctx *vmxctx; 1417abb023fbSJohn Baldwin uint64_t xcrval; 1418abb023fbSJohn Baldwin const struct xsave_limits *limits; 1419abb023fbSJohn Baldwin 1420abb023fbSJohn Baldwin vmxctx = &vmx->ctx[vcpu]; 1421abb023fbSJohn Baldwin limits = vmm_get_xsave_limits(); 1422abb023fbSJohn Baldwin 1423a0efd3fbSJohn Baldwin /* 1424a0efd3fbSJohn Baldwin * Note that the processor raises a GP# fault on its own if 1425a0efd3fbSJohn Baldwin * xsetbv is executed for CPL != 0, so we do not have to 1426a0efd3fbSJohn Baldwin * emulate that fault here. 1427a0efd3fbSJohn Baldwin */ 1428a0efd3fbSJohn Baldwin 1429a0efd3fbSJohn Baldwin /* Only xcr0 is supported. */ 1430a0efd3fbSJohn Baldwin if (vmxctx->guest_rcx != 0) { 1431dc506506SNeel Natu vm_inject_gp(vmx->vm, vcpu); 1432a0efd3fbSJohn Baldwin return (HANDLED); 1433a0efd3fbSJohn Baldwin } 1434a0efd3fbSJohn Baldwin 1435a0efd3fbSJohn Baldwin /* We only handle xcr0 if both the host and guest have XSAVE enabled. */ 1436a0efd3fbSJohn Baldwin if (!limits->xsave_enabled || !(vmcs_read(VMCS_GUEST_CR4) & CR4_XSAVE)) { 1437dc506506SNeel Natu vm_inject_ud(vmx->vm, vcpu); 1438a0efd3fbSJohn Baldwin return (HANDLED); 1439a0efd3fbSJohn Baldwin } 1440abb023fbSJohn Baldwin 1441abb023fbSJohn Baldwin xcrval = vmxctx->guest_rdx << 32 | (vmxctx->guest_rax & 0xffffffff); 1442a0efd3fbSJohn Baldwin if ((xcrval & ~limits->xcr0_allowed) != 0) { 1443dc506506SNeel Natu vm_inject_gp(vmx->vm, vcpu); 1444a0efd3fbSJohn Baldwin return (HANDLED); 1445a0efd3fbSJohn Baldwin } 1446abb023fbSJohn Baldwin 1447a0efd3fbSJohn Baldwin if (!(xcrval & XFEATURE_ENABLED_X87)) { 1448dc506506SNeel Natu vm_inject_gp(vmx->vm, vcpu); 1449a0efd3fbSJohn Baldwin return (HANDLED); 1450a0efd3fbSJohn Baldwin } 1451abb023fbSJohn Baldwin 145244a68c4eSJohn Baldwin /* AVX (YMM_Hi128) requires SSE. */ 145344a68c4eSJohn Baldwin if (xcrval & XFEATURE_ENABLED_AVX && 145444a68c4eSJohn Baldwin (xcrval & XFEATURE_AVX) != XFEATURE_AVX) { 145544a68c4eSJohn Baldwin vm_inject_gp(vmx->vm, vcpu); 145644a68c4eSJohn Baldwin return (HANDLED); 145744a68c4eSJohn Baldwin } 145844a68c4eSJohn Baldwin 145944a68c4eSJohn Baldwin /* 146044a68c4eSJohn Baldwin * AVX512 requires base AVX (YMM_Hi128) as well as OpMask, 146144a68c4eSJohn Baldwin * ZMM_Hi256, and Hi16_ZMM. 146244a68c4eSJohn Baldwin */ 146344a68c4eSJohn Baldwin if (xcrval & XFEATURE_AVX512 && 146444a68c4eSJohn Baldwin (xcrval & (XFEATURE_AVX512 | XFEATURE_AVX)) != 146544a68c4eSJohn Baldwin (XFEATURE_AVX512 | XFEATURE_AVX)) { 146644a68c4eSJohn Baldwin vm_inject_gp(vmx->vm, vcpu); 146744a68c4eSJohn Baldwin return (HANDLED); 146844a68c4eSJohn Baldwin } 146944a68c4eSJohn Baldwin 147044a68c4eSJohn Baldwin /* 147144a68c4eSJohn Baldwin * Intel MPX requires both bound register state flags to be 147244a68c4eSJohn Baldwin * set. 147344a68c4eSJohn Baldwin */ 147444a68c4eSJohn Baldwin if (((xcrval & XFEATURE_ENABLED_BNDREGS) != 0) != 147544a68c4eSJohn Baldwin ((xcrval & XFEATURE_ENABLED_BNDCSR) != 0)) { 1476dc506506SNeel Natu vm_inject_gp(vmx->vm, vcpu); 1477a0efd3fbSJohn Baldwin return (HANDLED); 1478a0efd3fbSJohn Baldwin } 1479abb023fbSJohn Baldwin 1480abb023fbSJohn Baldwin /* 1481abb023fbSJohn Baldwin * This runs "inside" vmrun() with the guest's FPU state, so 1482abb023fbSJohn Baldwin * modifying xcr0 directly modifies the guest's xcr0, not the 1483abb023fbSJohn Baldwin * host's. 1484abb023fbSJohn Baldwin */ 1485abb023fbSJohn Baldwin load_xcr(0, xcrval); 1486abb023fbSJohn Baldwin return (HANDLED); 1487abb023fbSJohn Baldwin } 1488abb023fbSJohn Baldwin 1489594db002STycho Nightingale static uint64_t 1490594db002STycho Nightingale vmx_get_guest_reg(struct vmx *vmx, int vcpu, int ident) 1491366f6083SPeter Grehan { 1492366f6083SPeter Grehan const struct vmxctx *vmxctx; 1493366f6083SPeter Grehan 1494594db002STycho Nightingale vmxctx = &vmx->ctx[vcpu]; 1495594db002STycho Nightingale 1496594db002STycho Nightingale switch (ident) { 1497594db002STycho Nightingale case 0: 1498594db002STycho Nightingale return (vmxctx->guest_rax); 1499594db002STycho Nightingale case 1: 1500594db002STycho Nightingale return (vmxctx->guest_rcx); 1501594db002STycho Nightingale case 2: 1502594db002STycho Nightingale return (vmxctx->guest_rdx); 1503594db002STycho Nightingale case 3: 1504594db002STycho Nightingale return (vmxctx->guest_rbx); 1505594db002STycho Nightingale case 4: 1506594db002STycho Nightingale return (vmcs_read(VMCS_GUEST_RSP)); 1507594db002STycho Nightingale case 5: 1508594db002STycho Nightingale return (vmxctx->guest_rbp); 1509594db002STycho Nightingale case 6: 1510594db002STycho Nightingale return (vmxctx->guest_rsi); 1511594db002STycho Nightingale case 7: 1512594db002STycho Nightingale return (vmxctx->guest_rdi); 1513594db002STycho Nightingale case 8: 1514594db002STycho Nightingale return (vmxctx->guest_r8); 1515594db002STycho Nightingale case 9: 1516594db002STycho Nightingale return (vmxctx->guest_r9); 1517594db002STycho Nightingale case 10: 1518594db002STycho Nightingale return (vmxctx->guest_r10); 1519594db002STycho Nightingale case 11: 1520594db002STycho Nightingale return (vmxctx->guest_r11); 1521594db002STycho Nightingale case 12: 1522594db002STycho Nightingale return (vmxctx->guest_r12); 1523594db002STycho Nightingale case 13: 1524594db002STycho Nightingale return (vmxctx->guest_r13); 1525594db002STycho Nightingale case 14: 1526594db002STycho Nightingale return (vmxctx->guest_r14); 1527594db002STycho Nightingale case 15: 1528594db002STycho Nightingale return (vmxctx->guest_r15); 1529594db002STycho Nightingale default: 1530594db002STycho Nightingale panic("invalid vmx register %d", ident); 1531594db002STycho Nightingale } 1532594db002STycho Nightingale } 1533594db002STycho Nightingale 1534594db002STycho Nightingale static void 1535594db002STycho Nightingale vmx_set_guest_reg(struct vmx *vmx, int vcpu, int ident, uint64_t regval) 1536594db002STycho Nightingale { 1537594db002STycho Nightingale struct vmxctx *vmxctx; 1538594db002STycho Nightingale 1539594db002STycho Nightingale vmxctx = &vmx->ctx[vcpu]; 1540594db002STycho Nightingale 1541594db002STycho Nightingale switch (ident) { 1542594db002STycho Nightingale case 0: 1543594db002STycho Nightingale vmxctx->guest_rax = regval; 1544594db002STycho Nightingale break; 1545594db002STycho Nightingale case 1: 1546594db002STycho Nightingale vmxctx->guest_rcx = regval; 1547594db002STycho Nightingale break; 1548594db002STycho Nightingale case 2: 1549594db002STycho Nightingale vmxctx->guest_rdx = regval; 1550594db002STycho Nightingale break; 1551594db002STycho Nightingale case 3: 1552594db002STycho Nightingale vmxctx->guest_rbx = regval; 1553594db002STycho Nightingale break; 1554594db002STycho Nightingale case 4: 1555594db002STycho Nightingale vmcs_write(VMCS_GUEST_RSP, regval); 1556594db002STycho Nightingale break; 1557594db002STycho Nightingale case 5: 1558594db002STycho Nightingale vmxctx->guest_rbp = regval; 1559594db002STycho Nightingale break; 1560594db002STycho Nightingale case 6: 1561594db002STycho Nightingale vmxctx->guest_rsi = regval; 1562594db002STycho Nightingale break; 1563594db002STycho Nightingale case 7: 1564594db002STycho Nightingale vmxctx->guest_rdi = regval; 1565594db002STycho Nightingale break; 1566594db002STycho Nightingale case 8: 1567594db002STycho Nightingale vmxctx->guest_r8 = regval; 1568594db002STycho Nightingale break; 1569594db002STycho Nightingale case 9: 1570594db002STycho Nightingale vmxctx->guest_r9 = regval; 1571594db002STycho Nightingale break; 1572594db002STycho Nightingale case 10: 1573594db002STycho Nightingale vmxctx->guest_r10 = regval; 1574594db002STycho Nightingale break; 1575594db002STycho Nightingale case 11: 1576594db002STycho Nightingale vmxctx->guest_r11 = regval; 1577594db002STycho Nightingale break; 1578594db002STycho Nightingale case 12: 1579594db002STycho Nightingale vmxctx->guest_r12 = regval; 1580594db002STycho Nightingale break; 1581594db002STycho Nightingale case 13: 1582594db002STycho Nightingale vmxctx->guest_r13 = regval; 1583594db002STycho Nightingale break; 1584594db002STycho Nightingale case 14: 1585594db002STycho Nightingale vmxctx->guest_r14 = regval; 1586594db002STycho Nightingale break; 1587594db002STycho Nightingale case 15: 1588594db002STycho Nightingale vmxctx->guest_r15 = regval; 1589594db002STycho Nightingale break; 1590594db002STycho Nightingale default: 1591594db002STycho Nightingale panic("invalid vmx register %d", ident); 1592594db002STycho Nightingale } 1593594db002STycho Nightingale } 1594594db002STycho Nightingale 1595594db002STycho Nightingale static int 1596594db002STycho Nightingale vmx_emulate_cr0_access(struct vmx *vmx, int vcpu, uint64_t exitqual) 1597594db002STycho Nightingale { 1598594db002STycho Nightingale uint64_t crval, regval; 1599594db002STycho Nightingale 1600594db002STycho Nightingale /* We only handle mov to %cr0 at this time */ 160139c21c2dSNeel Natu if ((exitqual & 0xf0) != 0x00) 160239c21c2dSNeel Natu return (UNHANDLED); 160339c21c2dSNeel Natu 1604594db002STycho Nightingale regval = vmx_get_guest_reg(vmx, vcpu, (exitqual >> 8) & 0xf); 1605366f6083SPeter Grehan 1606594db002STycho Nightingale vmcs_write(VMCS_CR0_SHADOW, regval); 1607366f6083SPeter Grehan 1608594db002STycho Nightingale crval = regval | cr0_ones_mask; 1609594db002STycho Nightingale crval &= ~cr0_zeros_mask; 1610594db002STycho Nightingale vmcs_write(VMCS_GUEST_CR0, crval); 1611366f6083SPeter Grehan 1612594db002STycho Nightingale if (regval & CR0_PG) { 161380a902efSPeter Grehan uint64_t efer, entry_ctls; 161480a902efSPeter Grehan 161580a902efSPeter Grehan /* 161680a902efSPeter Grehan * If CR0.PG is 1 and EFER.LME is 1 then EFER.LMA and 161780a902efSPeter Grehan * the "IA-32e mode guest" bit in VM-entry control must be 161880a902efSPeter Grehan * equal. 161980a902efSPeter Grehan */ 16203de83862SNeel Natu efer = vmcs_read(VMCS_GUEST_IA32_EFER); 162180a902efSPeter Grehan if (efer & EFER_LME) { 162280a902efSPeter Grehan efer |= EFER_LMA; 16233de83862SNeel Natu vmcs_write(VMCS_GUEST_IA32_EFER, efer); 16243de83862SNeel Natu entry_ctls = vmcs_read(VMCS_ENTRY_CTLS); 162580a902efSPeter Grehan entry_ctls |= VM_ENTRY_GUEST_LMA; 16263de83862SNeel Natu vmcs_write(VMCS_ENTRY_CTLS, entry_ctls); 162780a902efSPeter Grehan } 162880a902efSPeter Grehan } 162980a902efSPeter Grehan 1630366f6083SPeter Grehan return (HANDLED); 1631366f6083SPeter Grehan } 1632366f6083SPeter Grehan 1633594db002STycho Nightingale static int 1634594db002STycho Nightingale vmx_emulate_cr4_access(struct vmx *vmx, int vcpu, uint64_t exitqual) 1635594db002STycho Nightingale { 1636594db002STycho Nightingale uint64_t crval, regval; 1637594db002STycho Nightingale 1638594db002STycho Nightingale /* We only handle mov to %cr4 at this time */ 1639594db002STycho Nightingale if ((exitqual & 0xf0) != 0x00) 1640594db002STycho Nightingale return (UNHANDLED); 1641594db002STycho Nightingale 1642594db002STycho Nightingale regval = vmx_get_guest_reg(vmx, vcpu, (exitqual >> 8) & 0xf); 1643594db002STycho Nightingale 1644594db002STycho Nightingale vmcs_write(VMCS_CR4_SHADOW, regval); 1645594db002STycho Nightingale 1646594db002STycho Nightingale crval = regval | cr4_ones_mask; 1647594db002STycho Nightingale crval &= ~cr4_zeros_mask; 1648594db002STycho Nightingale vmcs_write(VMCS_GUEST_CR4, crval); 1649594db002STycho Nightingale 1650594db002STycho Nightingale return (HANDLED); 1651594db002STycho Nightingale } 1652594db002STycho Nightingale 1653594db002STycho Nightingale static int 1654594db002STycho Nightingale vmx_emulate_cr8_access(struct vmx *vmx, int vcpu, uint64_t exitqual) 1655594db002STycho Nightingale { 1656051f2bd1SNeel Natu struct vlapic *vlapic; 1657051f2bd1SNeel Natu uint64_t cr8; 1658051f2bd1SNeel Natu int regnum; 1659594db002STycho Nightingale 1660594db002STycho Nightingale /* We only handle mov %cr8 to/from a register at this time. */ 1661594db002STycho Nightingale if ((exitqual & 0xe0) != 0x00) { 1662594db002STycho Nightingale return (UNHANDLED); 1663594db002STycho Nightingale } 1664594db002STycho Nightingale 1665051f2bd1SNeel Natu vlapic = vm_lapic(vmx->vm, vcpu); 1666051f2bd1SNeel Natu regnum = (exitqual >> 8) & 0xf; 1667594db002STycho Nightingale if (exitqual & 0x10) { 1668051f2bd1SNeel Natu cr8 = vlapic_get_cr8(vlapic); 1669051f2bd1SNeel Natu vmx_set_guest_reg(vmx, vcpu, regnum, cr8); 1670594db002STycho Nightingale } else { 1671051f2bd1SNeel Natu cr8 = vmx_get_guest_reg(vmx, vcpu, regnum); 1672051f2bd1SNeel Natu vlapic_set_cr8(vlapic, cr8); 1673594db002STycho Nightingale } 1674594db002STycho Nightingale 1675594db002STycho Nightingale return (HANDLED); 1676594db002STycho Nightingale } 1677594db002STycho Nightingale 1678e4c8a13dSNeel Natu /* 1679e4c8a13dSNeel Natu * From section "Guest Register State" in the Intel SDM: CPL = SS.DPL 1680e4c8a13dSNeel Natu */ 1681e4c8a13dSNeel Natu static int 1682e4c8a13dSNeel Natu vmx_cpl(void) 1683e4c8a13dSNeel Natu { 1684e4c8a13dSNeel Natu uint32_t ssar; 1685e4c8a13dSNeel Natu 1686e4c8a13dSNeel Natu ssar = vmcs_read(VMCS_GUEST_SS_ACCESS_RIGHTS); 1687e4c8a13dSNeel Natu return ((ssar >> 5) & 0x3); 1688e4c8a13dSNeel Natu } 1689e4c8a13dSNeel Natu 1690e813a873SNeel Natu static enum vm_cpu_mode 169100f3efe1SJohn Baldwin vmx_cpu_mode(void) 169200f3efe1SJohn Baldwin { 1693b301b9e2SNeel Natu uint32_t csar; 169400f3efe1SJohn Baldwin 1695b301b9e2SNeel Natu if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LMA) { 1696b301b9e2SNeel Natu csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS); 1697b301b9e2SNeel Natu if (csar & 0x2000) 1698b301b9e2SNeel Natu return (CPU_MODE_64BIT); /* CS.L = 1 */ 169900f3efe1SJohn Baldwin else 170000f3efe1SJohn Baldwin return (CPU_MODE_COMPATIBILITY); 1701b301b9e2SNeel Natu } else if (vmcs_read(VMCS_GUEST_CR0) & CR0_PE) { 1702b301b9e2SNeel Natu return (CPU_MODE_PROTECTED); 1703b301b9e2SNeel Natu } else { 1704b301b9e2SNeel Natu return (CPU_MODE_REAL); 1705b301b9e2SNeel Natu } 170600f3efe1SJohn Baldwin } 170700f3efe1SJohn Baldwin 1708e813a873SNeel Natu static enum vm_paging_mode 170900f3efe1SJohn Baldwin vmx_paging_mode(void) 171000f3efe1SJohn Baldwin { 171100f3efe1SJohn Baldwin 171200f3efe1SJohn Baldwin if (!(vmcs_read(VMCS_GUEST_CR0) & CR0_PG)) 171300f3efe1SJohn Baldwin return (PAGING_MODE_FLAT); 171400f3efe1SJohn Baldwin if (!(vmcs_read(VMCS_GUEST_CR4) & CR4_PAE)) 171500f3efe1SJohn Baldwin return (PAGING_MODE_32); 171600f3efe1SJohn Baldwin if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LME) 171700f3efe1SJohn Baldwin return (PAGING_MODE_64); 171800f3efe1SJohn Baldwin else 171900f3efe1SJohn Baldwin return (PAGING_MODE_PAE); 172000f3efe1SJohn Baldwin } 172100f3efe1SJohn Baldwin 1722d17b5104SNeel Natu static uint64_t 1723d17b5104SNeel Natu inout_str_index(struct vmx *vmx, int vcpuid, int in) 1724d17b5104SNeel Natu { 1725d17b5104SNeel Natu uint64_t val; 1726d17b5104SNeel Natu int error; 1727d17b5104SNeel Natu enum vm_reg_name reg; 1728d17b5104SNeel Natu 1729d17b5104SNeel Natu reg = in ? VM_REG_GUEST_RDI : VM_REG_GUEST_RSI; 1730d17b5104SNeel Natu error = vmx_getreg(vmx, vcpuid, reg, &val); 1731d17b5104SNeel Natu KASSERT(error == 0, ("%s: vmx_getreg error %d", __func__, error)); 1732d17b5104SNeel Natu return (val); 1733d17b5104SNeel Natu } 1734d17b5104SNeel Natu 1735d17b5104SNeel Natu static uint64_t 1736d17b5104SNeel Natu inout_str_count(struct vmx *vmx, int vcpuid, int rep) 1737d17b5104SNeel Natu { 1738d17b5104SNeel Natu uint64_t val; 1739d17b5104SNeel Natu int error; 1740d17b5104SNeel Natu 1741d17b5104SNeel Natu if (rep) { 1742d17b5104SNeel Natu error = vmx_getreg(vmx, vcpuid, VM_REG_GUEST_RCX, &val); 1743d17b5104SNeel Natu KASSERT(!error, ("%s: vmx_getreg error %d", __func__, error)); 1744d17b5104SNeel Natu } else { 1745d17b5104SNeel Natu val = 1; 1746d17b5104SNeel Natu } 1747d17b5104SNeel Natu return (val); 1748d17b5104SNeel Natu } 1749d17b5104SNeel Natu 1750d17b5104SNeel Natu static int 1751d17b5104SNeel Natu inout_str_addrsize(uint32_t inst_info) 1752d17b5104SNeel Natu { 1753d17b5104SNeel Natu uint32_t size; 1754d17b5104SNeel Natu 1755d17b5104SNeel Natu size = (inst_info >> 7) & 0x7; 1756d17b5104SNeel Natu switch (size) { 1757d17b5104SNeel Natu case 0: 1758d17b5104SNeel Natu return (2); /* 16 bit */ 1759d17b5104SNeel Natu case 1: 1760d17b5104SNeel Natu return (4); /* 32 bit */ 1761d17b5104SNeel Natu case 2: 1762d17b5104SNeel Natu return (8); /* 64 bit */ 1763d17b5104SNeel Natu default: 1764d17b5104SNeel Natu panic("%s: invalid size encoding %d", __func__, size); 1765d17b5104SNeel Natu } 1766d17b5104SNeel Natu } 1767d17b5104SNeel Natu 1768d17b5104SNeel Natu static void 1769d17b5104SNeel Natu inout_str_seginfo(struct vmx *vmx, int vcpuid, uint32_t inst_info, int in, 1770d17b5104SNeel Natu struct vm_inout_str *vis) 1771d17b5104SNeel Natu { 1772d17b5104SNeel Natu int error, s; 1773d17b5104SNeel Natu 1774d17b5104SNeel Natu if (in) { 1775d17b5104SNeel Natu vis->seg_name = VM_REG_GUEST_ES; 1776d17b5104SNeel Natu } else { 1777d17b5104SNeel Natu s = (inst_info >> 15) & 0x7; 1778d17b5104SNeel Natu vis->seg_name = vm_segment_name(s); 1779d17b5104SNeel Natu } 1780d17b5104SNeel Natu 1781d17b5104SNeel Natu error = vmx_getdesc(vmx, vcpuid, vis->seg_name, &vis->seg_desc); 1782d17b5104SNeel Natu KASSERT(error == 0, ("%s: vmx_getdesc error %d", __func__, error)); 1783d17b5104SNeel Natu } 1784d17b5104SNeel Natu 1785e4c8a13dSNeel Natu static void 1786e813a873SNeel Natu vmx_paging_info(struct vm_guest_paging *paging) 1787e813a873SNeel Natu { 1788e813a873SNeel Natu paging->cr3 = vmcs_guest_cr3(); 1789e813a873SNeel Natu paging->cpl = vmx_cpl(); 1790e813a873SNeel Natu paging->cpu_mode = vmx_cpu_mode(); 1791e813a873SNeel Natu paging->paging_mode = vmx_paging_mode(); 1792e813a873SNeel Natu } 1793e813a873SNeel Natu 1794e813a873SNeel Natu static void 1795e4c8a13dSNeel Natu vmexit_inst_emul(struct vm_exit *vmexit, uint64_t gpa, uint64_t gla) 1796e4c8a13dSNeel Natu { 1797f7a9f178SNeel Natu struct vm_guest_paging *paging; 1798f7a9f178SNeel Natu uint32_t csar; 1799f7a9f178SNeel Natu 1800f7a9f178SNeel Natu paging = &vmexit->u.inst_emul.paging; 1801f7a9f178SNeel Natu 1802e4c8a13dSNeel Natu vmexit->exitcode = VM_EXITCODE_INST_EMUL; 18031c73ea3eSNeel Natu vmexit->inst_length = 0; 1804e4c8a13dSNeel Natu vmexit->u.inst_emul.gpa = gpa; 1805e4c8a13dSNeel Natu vmexit->u.inst_emul.gla = gla; 1806f7a9f178SNeel Natu vmx_paging_info(paging); 1807f7a9f178SNeel Natu switch (paging->cpu_mode) { 1808e4f605eeSTycho Nightingale case CPU_MODE_REAL: 1809e4f605eeSTycho Nightingale vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE); 1810e4f605eeSTycho Nightingale vmexit->u.inst_emul.cs_d = 0; 1811e4f605eeSTycho Nightingale break; 1812f7a9f178SNeel Natu case CPU_MODE_PROTECTED: 1813f7a9f178SNeel Natu case CPU_MODE_COMPATIBILITY: 1814e4f605eeSTycho Nightingale vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE); 1815f7a9f178SNeel Natu csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS); 1816f7a9f178SNeel Natu vmexit->u.inst_emul.cs_d = SEG_DESC_DEF32(csar); 1817f7a9f178SNeel Natu break; 1818f7a9f178SNeel Natu default: 1819e4f605eeSTycho Nightingale vmexit->u.inst_emul.cs_base = 0; 1820f7a9f178SNeel Natu vmexit->u.inst_emul.cs_d = 0; 1821f7a9f178SNeel Natu break; 1822f7a9f178SNeel Natu } 1823c2a875f9SNeel Natu vie_init(&vmexit->u.inst_emul.vie, NULL, 0); 1824e4c8a13dSNeel Natu } 1825e4c8a13dSNeel Natu 1826366f6083SPeter Grehan static int 1827318224bbSNeel Natu ept_fault_type(uint64_t ept_qual) 1828a2da7af6SNeel Natu { 1829318224bbSNeel Natu int fault_type; 1830a2da7af6SNeel Natu 1831318224bbSNeel Natu if (ept_qual & EPT_VIOLATION_DATA_WRITE) 1832318224bbSNeel Natu fault_type = VM_PROT_WRITE; 1833318224bbSNeel Natu else if (ept_qual & EPT_VIOLATION_INST_FETCH) 1834318224bbSNeel Natu fault_type = VM_PROT_EXECUTE; 1835318224bbSNeel Natu else 1836318224bbSNeel Natu fault_type= VM_PROT_READ; 1837318224bbSNeel Natu 1838318224bbSNeel Natu return (fault_type); 1839318224bbSNeel Natu } 1840318224bbSNeel Natu 1841318224bbSNeel Natu static boolean_t 1842318224bbSNeel Natu ept_emulation_fault(uint64_t ept_qual) 1843318224bbSNeel Natu { 1844318224bbSNeel Natu int read, write; 1845318224bbSNeel Natu 1846318224bbSNeel Natu /* EPT fault on an instruction fetch doesn't make sense here */ 1847a2da7af6SNeel Natu if (ept_qual & EPT_VIOLATION_INST_FETCH) 1848318224bbSNeel Natu return (FALSE); 1849a2da7af6SNeel Natu 1850318224bbSNeel Natu /* EPT fault must be a read fault or a write fault */ 1851a2da7af6SNeel Natu read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0; 1852a2da7af6SNeel Natu write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0; 18533b2b0011SPeter Grehan if ((read | write) == 0) 1854318224bbSNeel Natu return (FALSE); 1855a2da7af6SNeel Natu 1856a2da7af6SNeel Natu /* 18573b2b0011SPeter Grehan * The EPT violation must have been caused by accessing a 18583b2b0011SPeter Grehan * guest-physical address that is a translation of a guest-linear 18593b2b0011SPeter Grehan * address. 1860a2da7af6SNeel Natu */ 1861a2da7af6SNeel Natu if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 || 1862a2da7af6SNeel Natu (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) { 1863318224bbSNeel Natu return (FALSE); 1864a2da7af6SNeel Natu } 1865a2da7af6SNeel Natu 1866318224bbSNeel Natu return (TRUE); 1867a2da7af6SNeel Natu } 1868a2da7af6SNeel Natu 1869159dd56fSNeel Natu static __inline int 1870159dd56fSNeel Natu apic_access_virtualization(struct vmx *vmx, int vcpuid) 1871159dd56fSNeel Natu { 1872159dd56fSNeel Natu uint32_t proc_ctls2; 1873159dd56fSNeel Natu 1874159dd56fSNeel Natu proc_ctls2 = vmx->cap[vcpuid].proc_ctls2; 1875159dd56fSNeel Natu return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) ? 1 : 0); 1876159dd56fSNeel Natu } 1877159dd56fSNeel Natu 1878159dd56fSNeel Natu static __inline int 1879159dd56fSNeel Natu x2apic_virtualization(struct vmx *vmx, int vcpuid) 1880159dd56fSNeel Natu { 1881159dd56fSNeel Natu uint32_t proc_ctls2; 1882159dd56fSNeel Natu 1883159dd56fSNeel Natu proc_ctls2 = vmx->cap[vcpuid].proc_ctls2; 1884159dd56fSNeel Natu return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_X2APIC_MODE) ? 1 : 0); 1885159dd56fSNeel Natu } 1886159dd56fSNeel Natu 1887a2da7af6SNeel Natu static int 1888159dd56fSNeel Natu vmx_handle_apic_write(struct vmx *vmx, int vcpuid, struct vlapic *vlapic, 1889159dd56fSNeel Natu uint64_t qual) 189088c4b8d1SNeel Natu { 189188c4b8d1SNeel Natu int error, handled, offset; 1892159dd56fSNeel Natu uint32_t *apic_regs, vector; 189388c4b8d1SNeel Natu bool retu; 189488c4b8d1SNeel Natu 1895a0efd3fbSJohn Baldwin handled = HANDLED; 189688c4b8d1SNeel Natu offset = APIC_WRITE_OFFSET(qual); 1897159dd56fSNeel Natu 1898159dd56fSNeel Natu if (!apic_access_virtualization(vmx, vcpuid)) { 1899159dd56fSNeel Natu /* 1900159dd56fSNeel Natu * In general there should not be any APIC write VM-exits 1901159dd56fSNeel Natu * unless APIC-access virtualization is enabled. 1902159dd56fSNeel Natu * 1903159dd56fSNeel Natu * However self-IPI virtualization can legitimately trigger 1904159dd56fSNeel Natu * an APIC-write VM-exit so treat it specially. 1905159dd56fSNeel Natu */ 1906159dd56fSNeel Natu if (x2apic_virtualization(vmx, vcpuid) && 1907159dd56fSNeel Natu offset == APIC_OFFSET_SELF_IPI) { 1908159dd56fSNeel Natu apic_regs = (uint32_t *)(vlapic->apic_page); 1909159dd56fSNeel Natu vector = apic_regs[APIC_OFFSET_SELF_IPI / 4]; 1910159dd56fSNeel Natu vlapic_self_ipi_handler(vlapic, vector); 1911159dd56fSNeel Natu return (HANDLED); 1912159dd56fSNeel Natu } else 1913159dd56fSNeel Natu return (UNHANDLED); 1914159dd56fSNeel Natu } 1915159dd56fSNeel Natu 191688c4b8d1SNeel Natu switch (offset) { 191788c4b8d1SNeel Natu case APIC_OFFSET_ID: 191888c4b8d1SNeel Natu vlapic_id_write_handler(vlapic); 191988c4b8d1SNeel Natu break; 192088c4b8d1SNeel Natu case APIC_OFFSET_LDR: 192188c4b8d1SNeel Natu vlapic_ldr_write_handler(vlapic); 192288c4b8d1SNeel Natu break; 192388c4b8d1SNeel Natu case APIC_OFFSET_DFR: 192488c4b8d1SNeel Natu vlapic_dfr_write_handler(vlapic); 192588c4b8d1SNeel Natu break; 192688c4b8d1SNeel Natu case APIC_OFFSET_SVR: 192788c4b8d1SNeel Natu vlapic_svr_write_handler(vlapic); 192888c4b8d1SNeel Natu break; 192988c4b8d1SNeel Natu case APIC_OFFSET_ESR: 193088c4b8d1SNeel Natu vlapic_esr_write_handler(vlapic); 193188c4b8d1SNeel Natu break; 193288c4b8d1SNeel Natu case APIC_OFFSET_ICR_LOW: 193388c4b8d1SNeel Natu retu = false; 193488c4b8d1SNeel Natu error = vlapic_icrlo_write_handler(vlapic, &retu); 193588c4b8d1SNeel Natu if (error != 0 || retu) 1936a0efd3fbSJohn Baldwin handled = UNHANDLED; 193788c4b8d1SNeel Natu break; 193888c4b8d1SNeel Natu case APIC_OFFSET_CMCI_LVT: 193988c4b8d1SNeel Natu case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT: 194088c4b8d1SNeel Natu vlapic_lvt_write_handler(vlapic, offset); 194188c4b8d1SNeel Natu break; 194288c4b8d1SNeel Natu case APIC_OFFSET_TIMER_ICR: 194388c4b8d1SNeel Natu vlapic_icrtmr_write_handler(vlapic); 194488c4b8d1SNeel Natu break; 194588c4b8d1SNeel Natu case APIC_OFFSET_TIMER_DCR: 194688c4b8d1SNeel Natu vlapic_dcr_write_handler(vlapic); 194788c4b8d1SNeel Natu break; 194888c4b8d1SNeel Natu default: 1949a0efd3fbSJohn Baldwin handled = UNHANDLED; 195088c4b8d1SNeel Natu break; 195188c4b8d1SNeel Natu } 195288c4b8d1SNeel Natu return (handled); 195388c4b8d1SNeel Natu } 195488c4b8d1SNeel Natu 195588c4b8d1SNeel Natu static bool 1956159dd56fSNeel Natu apic_access_fault(struct vmx *vmx, int vcpuid, uint64_t gpa) 195788c4b8d1SNeel Natu { 195888c4b8d1SNeel Natu 1959159dd56fSNeel Natu if (apic_access_virtualization(vmx, vcpuid) && 196088c4b8d1SNeel Natu (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE)) 196188c4b8d1SNeel Natu return (true); 196288c4b8d1SNeel Natu else 196388c4b8d1SNeel Natu return (false); 196488c4b8d1SNeel Natu } 196588c4b8d1SNeel Natu 196688c4b8d1SNeel Natu static int 196788c4b8d1SNeel Natu vmx_handle_apic_access(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit) 196888c4b8d1SNeel Natu { 196988c4b8d1SNeel Natu uint64_t qual; 197088c4b8d1SNeel Natu int access_type, offset, allowed; 197188c4b8d1SNeel Natu 1972159dd56fSNeel Natu if (!apic_access_virtualization(vmx, vcpuid)) 197388c4b8d1SNeel Natu return (UNHANDLED); 197488c4b8d1SNeel Natu 197588c4b8d1SNeel Natu qual = vmexit->u.vmx.exit_qualification; 197688c4b8d1SNeel Natu access_type = APIC_ACCESS_TYPE(qual); 197788c4b8d1SNeel Natu offset = APIC_ACCESS_OFFSET(qual); 197888c4b8d1SNeel Natu 197988c4b8d1SNeel Natu allowed = 0; 198088c4b8d1SNeel Natu if (access_type == 0) { 198188c4b8d1SNeel Natu /* 198288c4b8d1SNeel Natu * Read data access to the following registers is expected. 198388c4b8d1SNeel Natu */ 198488c4b8d1SNeel Natu switch (offset) { 198588c4b8d1SNeel Natu case APIC_OFFSET_APR: 198688c4b8d1SNeel Natu case APIC_OFFSET_PPR: 198788c4b8d1SNeel Natu case APIC_OFFSET_RRR: 198888c4b8d1SNeel Natu case APIC_OFFSET_CMCI_LVT: 198988c4b8d1SNeel Natu case APIC_OFFSET_TIMER_CCR: 199088c4b8d1SNeel Natu allowed = 1; 199188c4b8d1SNeel Natu break; 199288c4b8d1SNeel Natu default: 199388c4b8d1SNeel Natu break; 199488c4b8d1SNeel Natu } 199588c4b8d1SNeel Natu } else if (access_type == 1) { 199688c4b8d1SNeel Natu /* 199788c4b8d1SNeel Natu * Write data access to the following registers is expected. 199888c4b8d1SNeel Natu */ 199988c4b8d1SNeel Natu switch (offset) { 200088c4b8d1SNeel Natu case APIC_OFFSET_VER: 200188c4b8d1SNeel Natu case APIC_OFFSET_APR: 200288c4b8d1SNeel Natu case APIC_OFFSET_PPR: 200388c4b8d1SNeel Natu case APIC_OFFSET_RRR: 200488c4b8d1SNeel Natu case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7: 200588c4b8d1SNeel Natu case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7: 200688c4b8d1SNeel Natu case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7: 200788c4b8d1SNeel Natu case APIC_OFFSET_CMCI_LVT: 200888c4b8d1SNeel Natu case APIC_OFFSET_TIMER_CCR: 200988c4b8d1SNeel Natu allowed = 1; 201088c4b8d1SNeel Natu break; 201188c4b8d1SNeel Natu default: 201288c4b8d1SNeel Natu break; 201388c4b8d1SNeel Natu } 201488c4b8d1SNeel Natu } 201588c4b8d1SNeel Natu 201688c4b8d1SNeel Natu if (allowed) { 2017e4c8a13dSNeel Natu vmexit_inst_emul(vmexit, DEFAULT_APIC_BASE + offset, 2018e4c8a13dSNeel Natu VIE_INVALID_GLA); 201988c4b8d1SNeel Natu } 202088c4b8d1SNeel Natu 202188c4b8d1SNeel Natu /* 202288c4b8d1SNeel Natu * Regardless of whether the APIC-access is allowed this handler 202388c4b8d1SNeel Natu * always returns UNHANDLED: 202488c4b8d1SNeel Natu * - if the access is allowed then it is handled by emulating the 202588c4b8d1SNeel Natu * instruction that caused the VM-exit (outside the critical section) 202688c4b8d1SNeel Natu * - if the access is not allowed then it will be converted to an 202788c4b8d1SNeel Natu * exitcode of VM_EXITCODE_VMX and will be dealt with in userland. 202888c4b8d1SNeel Natu */ 202988c4b8d1SNeel Natu return (UNHANDLED); 203088c4b8d1SNeel Natu } 203188c4b8d1SNeel Natu 20323d5444c8SNeel Natu static enum task_switch_reason 20333d5444c8SNeel Natu vmx_task_switch_reason(uint64_t qual) 20343d5444c8SNeel Natu { 20353d5444c8SNeel Natu int reason; 20363d5444c8SNeel Natu 20373d5444c8SNeel Natu reason = (qual >> 30) & 0x3; 20383d5444c8SNeel Natu switch (reason) { 20393d5444c8SNeel Natu case 0: 20403d5444c8SNeel Natu return (TSR_CALL); 20413d5444c8SNeel Natu case 1: 20423d5444c8SNeel Natu return (TSR_IRET); 20433d5444c8SNeel Natu case 2: 20443d5444c8SNeel Natu return (TSR_JMP); 20453d5444c8SNeel Natu case 3: 20463d5444c8SNeel Natu return (TSR_IDT_GATE); 20473d5444c8SNeel Natu default: 20483d5444c8SNeel Natu panic("%s: invalid reason %d", __func__, reason); 20493d5444c8SNeel Natu } 20503d5444c8SNeel Natu } 20513d5444c8SNeel Natu 205288c4b8d1SNeel Natu static int 2053c3498942SNeel Natu emulate_wrmsr(struct vmx *vmx, int vcpuid, u_int num, uint64_t val, bool *retu) 2054c3498942SNeel Natu { 2055c3498942SNeel Natu int error; 2056c3498942SNeel Natu 2057c3498942SNeel Natu if (lapic_msr(num)) 2058c3498942SNeel Natu error = lapic_wrmsr(vmx->vm, vcpuid, num, val, retu); 2059c3498942SNeel Natu else 2060c3498942SNeel Natu error = vmx_wrmsr(vmx, vcpuid, num, val, retu); 2061c3498942SNeel Natu 2062c3498942SNeel Natu return (error); 2063c3498942SNeel Natu } 2064c3498942SNeel Natu 2065c3498942SNeel Natu static int 2066c3498942SNeel Natu emulate_rdmsr(struct vmx *vmx, int vcpuid, u_int num, bool *retu) 2067c3498942SNeel Natu { 2068c3498942SNeel Natu struct vmxctx *vmxctx; 2069c3498942SNeel Natu uint64_t result; 2070c3498942SNeel Natu uint32_t eax, edx; 2071c3498942SNeel Natu int error; 2072c3498942SNeel Natu 2073c3498942SNeel Natu if (lapic_msr(num)) 2074c3498942SNeel Natu error = lapic_rdmsr(vmx->vm, vcpuid, num, &result, retu); 2075c3498942SNeel Natu else 2076c3498942SNeel Natu error = vmx_rdmsr(vmx, vcpuid, num, &result, retu); 2077c3498942SNeel Natu 2078c3498942SNeel Natu if (error == 0) { 2079c3498942SNeel Natu eax = result; 2080c3498942SNeel Natu vmxctx = &vmx->ctx[vcpuid]; 2081c3498942SNeel Natu error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RAX, eax); 2082c3498942SNeel Natu KASSERT(error == 0, ("vmxctx_setreg(rax) error %d", error)); 2083c3498942SNeel Natu 2084c3498942SNeel Natu edx = result >> 32; 2085c3498942SNeel Natu error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RDX, edx); 2086c3498942SNeel Natu KASSERT(error == 0, ("vmxctx_setreg(rdx) error %d", error)); 2087c3498942SNeel Natu } 2088c3498942SNeel Natu 2089c3498942SNeel Natu return (error); 2090c3498942SNeel Natu } 2091c3498942SNeel Natu 2092c3498942SNeel Natu static int 2093366f6083SPeter Grehan vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) 2094366f6083SPeter Grehan { 2095c9c75df4SNeel Natu int error, errcode, errcode_valid, handled, in; 2096366f6083SPeter Grehan struct vmxctx *vmxctx; 209788c4b8d1SNeel Natu struct vlapic *vlapic; 2098d17b5104SNeel Natu struct vm_inout_str *vis; 20993d5444c8SNeel Natu struct vm_task_switch *ts; 2100d17b5104SNeel Natu uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, inst_info; 2101b0538143SNeel Natu uint32_t intr_type, intr_vec, reason; 2102091d4532SNeel Natu uint64_t exitintinfo, qual, gpa; 2103becd9849SNeel Natu bool retu; 2104366f6083SPeter Grehan 2105160471d2SNeel Natu CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0); 2106c308b23bSNeel Natu CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0); 2107160471d2SNeel Natu 2108a0efd3fbSJohn Baldwin handled = UNHANDLED; 2109366f6083SPeter Grehan vmxctx = &vmx->ctx[vcpu]; 21100492757cSNeel Natu 2111366f6083SPeter Grehan qual = vmexit->u.vmx.exit_qualification; 2112318224bbSNeel Natu reason = vmexit->u.vmx.exit_reason; 2113366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_BOGUS; 2114366f6083SPeter Grehan 211561592433SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1); 211661592433SNeel Natu 2117318224bbSNeel Natu /* 2118b0538143SNeel Natu * VM-entry failures during or after loading guest state. 2119b0538143SNeel Natu * 2120b0538143SNeel Natu * These VM-exits are uncommon but must be handled specially 2121b0538143SNeel Natu * as most VM-exit fields are not populated as usual. 2122b0538143SNeel Natu */ 2123b0538143SNeel Natu if (__predict_false(reason == EXIT_REASON_MCE_DURING_ENTRY)) { 2124b0538143SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Handling MCE during VM-entry"); 2125b0538143SNeel Natu __asm __volatile("int $18"); 2126b0538143SNeel Natu return (1); 2127b0538143SNeel Natu } 2128b0538143SNeel Natu 2129b0538143SNeel Natu /* 21303d5444c8SNeel Natu * VM exits that can be triggered during event delivery need to 21313d5444c8SNeel Natu * be handled specially by re-injecting the event if the IDT 21323d5444c8SNeel Natu * vectoring information field's valid bit is set. 2133318224bbSNeel Natu * 2134318224bbSNeel Natu * See "Information for VM Exits During Event Delivery" in Intel SDM 2135318224bbSNeel Natu * for details. 2136318224bbSNeel Natu */ 2137318224bbSNeel Natu idtvec_info = vmcs_idt_vectoring_info(); 2138318224bbSNeel Natu if (idtvec_info & VMCS_IDT_VEC_VALID) { 2139318224bbSNeel Natu idtvec_info &= ~(1 << 12); /* clear undefined bit */ 2140091d4532SNeel Natu exitintinfo = idtvec_info; 2141318224bbSNeel Natu if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) { 2142318224bbSNeel Natu idtvec_err = vmcs_idt_vectoring_err(); 2143091d4532SNeel Natu exitintinfo |= (uint64_t)idtvec_err << 32; 2144318224bbSNeel Natu } 2145091d4532SNeel Natu error = vm_exit_intinfo(vmx->vm, vcpu, exitintinfo); 2146091d4532SNeel Natu KASSERT(error == 0, ("%s: vm_set_intinfo error %d", 2147091d4532SNeel Natu __func__, error)); 2148091d4532SNeel Natu 2149160471d2SNeel Natu /* 2150160471d2SNeel Natu * If 'virtual NMIs' are being used and the VM-exit 2151160471d2SNeel Natu * happened while injecting an NMI during the previous 2152091d4532SNeel Natu * VM-entry, then clear "blocking by NMI" in the 2153091d4532SNeel Natu * Guest Interruptibility-State so the NMI can be 2154091d4532SNeel Natu * reinjected on the subsequent VM-entry. 2155091d4532SNeel Natu * 2156091d4532SNeel Natu * However, if the NMI was being delivered through a task 2157091d4532SNeel Natu * gate, then the new task must start execution with NMIs 2158091d4532SNeel Natu * blocked so don't clear NMI blocking in this case. 2159160471d2SNeel Natu */ 2160091d4532SNeel Natu intr_type = idtvec_info & VMCS_INTR_T_MASK; 2161091d4532SNeel Natu if (intr_type == VMCS_INTR_T_NMI) { 2162091d4532SNeel Natu if (reason != EXIT_REASON_TASK_SWITCH) 2163e5a1d950SNeel Natu vmx_clear_nmi_blocking(vmx, vcpu); 2164091d4532SNeel Natu else 2165091d4532SNeel Natu vmx_assert_nmi_blocking(vmx, vcpu); 2166160471d2SNeel Natu } 2167091d4532SNeel Natu 2168091d4532SNeel Natu /* 2169091d4532SNeel Natu * Update VM-entry instruction length if the event being 2170091d4532SNeel Natu * delivered was a software interrupt or software exception. 2171091d4532SNeel Natu */ 2172091d4532SNeel Natu if (intr_type == VMCS_INTR_T_SWINTR || 2173091d4532SNeel Natu intr_type == VMCS_INTR_T_PRIV_SWEXCEPTION || 2174091d4532SNeel Natu intr_type == VMCS_INTR_T_SWEXCEPTION) { 21753de83862SNeel Natu vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length); 2176318224bbSNeel Natu } 2177318224bbSNeel Natu } 2178318224bbSNeel Natu 2179318224bbSNeel Natu switch (reason) { 21803d5444c8SNeel Natu case EXIT_REASON_TASK_SWITCH: 21813d5444c8SNeel Natu ts = &vmexit->u.task_switch; 21823d5444c8SNeel Natu ts->tsssel = qual & 0xffff; 21833d5444c8SNeel Natu ts->reason = vmx_task_switch_reason(qual); 21843d5444c8SNeel Natu ts->ext = 0; 21853d5444c8SNeel Natu ts->errcode_valid = 0; 21863d5444c8SNeel Natu vmx_paging_info(&ts->paging); 21873d5444c8SNeel Natu /* 21883d5444c8SNeel Natu * If the task switch was due to a CALL, JMP, IRET, software 21893d5444c8SNeel Natu * interrupt (INT n) or software exception (INT3, INTO), 21903d5444c8SNeel Natu * then the saved %rip references the instruction that caused 21913d5444c8SNeel Natu * the task switch. The instruction length field in the VMCS 21923d5444c8SNeel Natu * is valid in this case. 21933d5444c8SNeel Natu * 21943d5444c8SNeel Natu * In all other cases (e.g., NMI, hardware exception) the 21953d5444c8SNeel Natu * saved %rip is one that would have been saved in the old TSS 21963d5444c8SNeel Natu * had the task switch completed normally so the instruction 21973d5444c8SNeel Natu * length field is not needed in this case and is explicitly 21983d5444c8SNeel Natu * set to 0. 21993d5444c8SNeel Natu */ 22003d5444c8SNeel Natu if (ts->reason == TSR_IDT_GATE) { 22013d5444c8SNeel Natu KASSERT(idtvec_info & VMCS_IDT_VEC_VALID, 2202091d4532SNeel Natu ("invalid idtvec_info %#x for IDT task switch", 22033d5444c8SNeel Natu idtvec_info)); 22043d5444c8SNeel Natu intr_type = idtvec_info & VMCS_INTR_T_MASK; 22053d5444c8SNeel Natu if (intr_type != VMCS_INTR_T_SWINTR && 22063d5444c8SNeel Natu intr_type != VMCS_INTR_T_SWEXCEPTION && 22073d5444c8SNeel Natu intr_type != VMCS_INTR_T_PRIV_SWEXCEPTION) { 22083d5444c8SNeel Natu /* Task switch triggered by external event */ 22093d5444c8SNeel Natu ts->ext = 1; 22103d5444c8SNeel Natu vmexit->inst_length = 0; 22113d5444c8SNeel Natu if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) { 22123d5444c8SNeel Natu ts->errcode_valid = 1; 22133d5444c8SNeel Natu ts->errcode = vmcs_idt_vectoring_err(); 22143d5444c8SNeel Natu } 22153d5444c8SNeel Natu } 22163d5444c8SNeel Natu } 22173d5444c8SNeel Natu vmexit->exitcode = VM_EXITCODE_TASK_SWITCH; 22183d5444c8SNeel Natu VCPU_CTR4(vmx->vm, vcpu, "task switch reason %d, tss 0x%04x, " 22193d5444c8SNeel Natu "%s errcode 0x%016lx", ts->reason, ts->tsssel, 22203d5444c8SNeel Natu ts->ext ? "external" : "internal", 22213d5444c8SNeel Natu ((uint64_t)ts->errcode << 32) | ts->errcode_valid); 22223d5444c8SNeel Natu break; 2223366f6083SPeter Grehan case EXIT_REASON_CR_ACCESS: 2224b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CR_ACCESS, 1); 2225594db002STycho Nightingale switch (qual & 0xf) { 2226594db002STycho Nightingale case 0: 2227594db002STycho Nightingale handled = vmx_emulate_cr0_access(vmx, vcpu, qual); 2228594db002STycho Nightingale break; 2229594db002STycho Nightingale case 4: 2230594db002STycho Nightingale handled = vmx_emulate_cr4_access(vmx, vcpu, qual); 2231594db002STycho Nightingale break; 2232594db002STycho Nightingale case 8: 2233594db002STycho Nightingale handled = vmx_emulate_cr8_access(vmx, vcpu, qual); 2234594db002STycho Nightingale break; 2235594db002STycho Nightingale } 2236366f6083SPeter Grehan break; 2237366f6083SPeter Grehan case EXIT_REASON_RDMSR: 2238b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RDMSR, 1); 2239becd9849SNeel Natu retu = false; 2240366f6083SPeter Grehan ecx = vmxctx->guest_rcx; 22412cb97c9dSNeel Natu VCPU_CTR1(vmx->vm, vcpu, "rdmsr 0x%08x", ecx); 2242c3498942SNeel Natu error = emulate_rdmsr(vmx, vcpu, ecx, &retu); 2243b42206f3SNeel Natu if (error) { 2244366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_RDMSR; 2245366f6083SPeter Grehan vmexit->u.msr.code = ecx; 2246becd9849SNeel Natu } else if (!retu) { 2247a0efd3fbSJohn Baldwin handled = HANDLED; 2248becd9849SNeel Natu } else { 2249becd9849SNeel Natu /* Return to userspace with a valid exitcode */ 2250becd9849SNeel Natu KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS, 2251c3498942SNeel Natu ("emulate_rdmsr retu with bogus exitcode")); 2252becd9849SNeel Natu } 2253366f6083SPeter Grehan break; 2254366f6083SPeter Grehan case EXIT_REASON_WRMSR: 2255b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_WRMSR, 1); 2256becd9849SNeel Natu retu = false; 2257366f6083SPeter Grehan eax = vmxctx->guest_rax; 2258366f6083SPeter Grehan ecx = vmxctx->guest_rcx; 2259366f6083SPeter Grehan edx = vmxctx->guest_rdx; 22602cb97c9dSNeel Natu VCPU_CTR2(vmx->vm, vcpu, "wrmsr 0x%08x value 0x%016lx", 22612cb97c9dSNeel Natu ecx, (uint64_t)edx << 32 | eax); 2262c3498942SNeel Natu error = emulate_wrmsr(vmx, vcpu, ecx, 2263becd9849SNeel Natu (uint64_t)edx << 32 | eax, &retu); 2264b42206f3SNeel Natu if (error) { 2265366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_WRMSR; 2266366f6083SPeter Grehan vmexit->u.msr.code = ecx; 2267366f6083SPeter Grehan vmexit->u.msr.wval = (uint64_t)edx << 32 | eax; 2268becd9849SNeel Natu } else if (!retu) { 2269a0efd3fbSJohn Baldwin handled = HANDLED; 2270becd9849SNeel Natu } else { 2271becd9849SNeel Natu /* Return to userspace with a valid exitcode */ 2272becd9849SNeel Natu KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS, 2273becd9849SNeel Natu ("emulate_wrmsr retu with bogus exitcode")); 2274becd9849SNeel Natu } 2275366f6083SPeter Grehan break; 2276366f6083SPeter Grehan case EXIT_REASON_HLT: 2277f76fc5d4SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_HLT, 1); 2278366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_HLT; 22793de83862SNeel Natu vmexit->u.hlt.rflags = vmcs_read(VMCS_GUEST_RFLAGS); 2280366f6083SPeter Grehan break; 2281366f6083SPeter Grehan case EXIT_REASON_MTF: 2282b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_MTRAP, 1); 2283366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_MTRAP; 2284c9c75df4SNeel Natu vmexit->inst_length = 0; 2285366f6083SPeter Grehan break; 2286366f6083SPeter Grehan case EXIT_REASON_PAUSE: 2287b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_PAUSE, 1); 2288366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_PAUSE; 2289366f6083SPeter Grehan break; 2290366f6083SPeter Grehan case EXIT_REASON_INTR_WINDOW: 2291b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INTR_WINDOW, 1); 2292366f6083SPeter Grehan vmx_clear_int_window_exiting(vmx, vcpu); 2293b5aaf7b2SNeel Natu return (1); 2294366f6083SPeter Grehan case EXIT_REASON_EXT_INTR: 2295366f6083SPeter Grehan /* 2296366f6083SPeter Grehan * External interrupts serve only to cause VM exits and allow 2297366f6083SPeter Grehan * the host interrupt handler to run. 2298366f6083SPeter Grehan * 2299366f6083SPeter Grehan * If this external interrupt triggers a virtual interrupt 2300366f6083SPeter Grehan * to a VM, then that state will be recorded by the 2301366f6083SPeter Grehan * host interrupt handler in the VM's softc. We will inject 2302366f6083SPeter Grehan * this virtual interrupt during the subsequent VM enter. 2303366f6083SPeter Grehan */ 2304f7d47425SNeel Natu intr_info = vmcs_read(VMCS_EXIT_INTR_INFO); 2305722b6744SJohn Baldwin 2306722b6744SJohn Baldwin /* 2307722b6744SJohn Baldwin * XXX: Ignore this exit if VMCS_INTR_VALID is not set. 2308ad3e3687SJohn Baldwin * This appears to be a bug in VMware Fusion? 2309722b6744SJohn Baldwin */ 2310722b6744SJohn Baldwin if (!(intr_info & VMCS_INTR_VALID)) 2311722b6744SJohn Baldwin return (1); 2312160471d2SNeel Natu KASSERT((intr_info & VMCS_INTR_VALID) != 0 && 2313160471d2SNeel Natu (intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_HWINTR, 2314f7d47425SNeel Natu ("VM exit interruption info invalid: %#x", intr_info)); 2315f7d47425SNeel Natu vmx_trigger_hostintr(intr_info & 0xff); 2316366f6083SPeter Grehan 2317366f6083SPeter Grehan /* 2318366f6083SPeter Grehan * This is special. We want to treat this as an 'handled' 2319366f6083SPeter Grehan * VM-exit but not increment the instruction pointer. 2320366f6083SPeter Grehan */ 2321366f6083SPeter Grehan vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1); 2322366f6083SPeter Grehan return (1); 2323366f6083SPeter Grehan case EXIT_REASON_NMI_WINDOW: 2324366f6083SPeter Grehan /* Exit to allow the pending virtual NMI to be injected */ 232548b2d828SNeel Natu if (vm_nmi_pending(vmx->vm, vcpu)) 232648b2d828SNeel Natu vmx_inject_nmi(vmx, vcpu); 2327366f6083SPeter Grehan vmx_clear_nmi_window_exiting(vmx, vcpu); 232848b2d828SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NMI_WINDOW, 1); 2329366f6083SPeter Grehan return (1); 2330366f6083SPeter Grehan case EXIT_REASON_INOUT: 2331b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INOUT, 1); 2332366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_INOUT; 2333366f6083SPeter Grehan vmexit->u.inout.bytes = (qual & 0x7) + 1; 2334d17b5104SNeel Natu vmexit->u.inout.in = in = (qual & 0x8) ? 1 : 0; 2335366f6083SPeter Grehan vmexit->u.inout.string = (qual & 0x10) ? 1 : 0; 2336366f6083SPeter Grehan vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0; 2337366f6083SPeter Grehan vmexit->u.inout.port = (uint16_t)(qual >> 16); 2338366f6083SPeter Grehan vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax); 2339d17b5104SNeel Natu if (vmexit->u.inout.string) { 2340d17b5104SNeel Natu inst_info = vmcs_read(VMCS_EXIT_INSTRUCTION_INFO); 2341d17b5104SNeel Natu vmexit->exitcode = VM_EXITCODE_INOUT_STR; 2342d17b5104SNeel Natu vis = &vmexit->u.inout_str; 2343e813a873SNeel Natu vmx_paging_info(&vis->paging); 2344d17b5104SNeel Natu vis->rflags = vmcs_read(VMCS_GUEST_RFLAGS); 2345d17b5104SNeel Natu vis->cr0 = vmcs_read(VMCS_GUEST_CR0); 2346d17b5104SNeel Natu vis->index = inout_str_index(vmx, vcpu, in); 2347d17b5104SNeel Natu vis->count = inout_str_count(vmx, vcpu, vis->inout.rep); 2348d17b5104SNeel Natu vis->addrsize = inout_str_addrsize(inst_info); 2349d17b5104SNeel Natu inout_str_seginfo(vmx, vcpu, inst_info, in, vis); 2350762fd208STycho Nightingale } 2351366f6083SPeter Grehan break; 2352366f6083SPeter Grehan case EXIT_REASON_CPUID: 2353b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CPUID, 1); 2354a2da7af6SNeel Natu handled = vmx_handle_cpuid(vmx->vm, vcpu, vmxctx); 2355366f6083SPeter Grehan break; 2356e5a1d950SNeel Natu case EXIT_REASON_EXCEPTION: 2357c308b23bSNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXCEPTION, 1); 2358e5a1d950SNeel Natu intr_info = vmcs_read(VMCS_EXIT_INTR_INFO); 2359e5a1d950SNeel Natu KASSERT((intr_info & VMCS_INTR_VALID) != 0, 2360e5a1d950SNeel Natu ("VM exit interruption info invalid: %#x", intr_info)); 2361c308b23bSNeel Natu 2362b0538143SNeel Natu intr_vec = intr_info & 0xff; 2363b0538143SNeel Natu intr_type = intr_info & VMCS_INTR_T_MASK; 2364b0538143SNeel Natu 2365e5a1d950SNeel Natu /* 2366e5a1d950SNeel Natu * If Virtual NMIs control is 1 and the VM-exit is due to a 2367e5a1d950SNeel Natu * fault encountered during the execution of IRET then we must 2368e5a1d950SNeel Natu * restore the state of "virtual-NMI blocking" before resuming 2369e5a1d950SNeel Natu * the guest. 2370e5a1d950SNeel Natu * 2371e5a1d950SNeel Natu * See "Resuming Guest Software after Handling an Exception". 2372091d4532SNeel Natu * See "Information for VM Exits Due to Vectored Events". 2373e5a1d950SNeel Natu */ 2374e5a1d950SNeel Natu if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 && 2375b0538143SNeel Natu (intr_vec != IDT_DF) && 2376e5a1d950SNeel Natu (intr_info & EXIT_QUAL_NMIUDTI) != 0) 2377e5a1d950SNeel Natu vmx_restore_nmi_blocking(vmx, vcpu); 2378c308b23bSNeel Natu 2379c308b23bSNeel Natu /* 238062fbd7c2SNeel Natu * The NMI has already been handled in vmx_exit_handle_nmi(). 2381c308b23bSNeel Natu */ 2382b0538143SNeel Natu if (intr_type == VMCS_INTR_T_NMI) 2383c308b23bSNeel Natu return (1); 2384b0538143SNeel Natu 2385b0538143SNeel Natu /* 2386b0538143SNeel Natu * Call the machine check handler by hand. Also don't reflect 2387b0538143SNeel Natu * the machine check back into the guest. 2388b0538143SNeel Natu */ 2389b0538143SNeel Natu if (intr_vec == IDT_MC) { 2390b0538143SNeel Natu VCPU_CTR0(vmx->vm, vcpu, "Vectoring to MCE handler"); 2391b0538143SNeel Natu __asm __volatile("int $18"); 2392b0538143SNeel Natu return (1); 2393b0538143SNeel Natu } 2394b0538143SNeel Natu 2395b0538143SNeel Natu if (intr_vec == IDT_PF) { 2396b0538143SNeel Natu error = vmxctx_setreg(vmxctx, VM_REG_GUEST_CR2, qual); 2397b0538143SNeel Natu KASSERT(error == 0, ("%s: vmxctx_setreg(cr2) error %d", 2398b0538143SNeel Natu __func__, error)); 2399b0538143SNeel Natu } 2400b0538143SNeel Natu 2401b0538143SNeel Natu /* 2402b0538143SNeel Natu * Software exceptions exhibit trap-like behavior. This in 2403b0538143SNeel Natu * turn requires populating the VM-entry instruction length 2404b0538143SNeel Natu * so that the %rip in the trap frame is past the INT3/INTO 2405b0538143SNeel Natu * instruction. 2406b0538143SNeel Natu */ 2407b0538143SNeel Natu if (intr_type == VMCS_INTR_T_SWEXCEPTION) 2408b0538143SNeel Natu vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length); 2409b0538143SNeel Natu 2410b0538143SNeel Natu /* Reflect all other exceptions back into the guest */ 2411c9c75df4SNeel Natu errcode_valid = errcode = 0; 2412b0538143SNeel Natu if (intr_info & VMCS_INTR_DEL_ERRCODE) { 2413c9c75df4SNeel Natu errcode_valid = 1; 2414c9c75df4SNeel Natu errcode = vmcs_read(VMCS_EXIT_INTR_ERRCODE); 2415b0538143SNeel Natu } 2416b0538143SNeel Natu VCPU_CTR2(vmx->vm, vcpu, "Reflecting exception %d/%#x into " 2417c9c75df4SNeel Natu "the guest", intr_vec, errcode); 2418c9c75df4SNeel Natu error = vm_inject_exception(vmx->vm, vcpu, intr_vec, 2419c9c75df4SNeel Natu errcode_valid, errcode, 0); 2420b0538143SNeel Natu KASSERT(error == 0, ("%s: vm_inject_exception error %d", 2421b0538143SNeel Natu __func__, error)); 2422b0538143SNeel Natu return (1); 2423b0538143SNeel Natu 2424cd942e0fSPeter Grehan case EXIT_REASON_EPT_FAULT: 2425318224bbSNeel Natu /* 2426318224bbSNeel Natu * If 'gpa' lies within the address space allocated to 2427318224bbSNeel Natu * memory then this must be a nested page fault otherwise 2428318224bbSNeel Natu * this must be an instruction that accesses MMIO space. 2429318224bbSNeel Natu */ 2430a2da7af6SNeel Natu gpa = vmcs_gpa(); 24319b1aa8d6SNeel Natu if (vm_mem_allocated(vmx->vm, vcpu, gpa) || 2432159dd56fSNeel Natu apic_access_fault(vmx, vcpu, gpa)) { 2433cd942e0fSPeter Grehan vmexit->exitcode = VM_EXITCODE_PAGING; 2434d087a399SNeel Natu vmexit->inst_length = 0; 243513ec9371SPeter Grehan vmexit->u.paging.gpa = gpa; 2436318224bbSNeel Natu vmexit->u.paging.fault_type = ept_fault_type(qual); 2437bf73979dSNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NESTED_FAULT, 1); 2438318224bbSNeel Natu } else if (ept_emulation_fault(qual)) { 2439e4c8a13dSNeel Natu vmexit_inst_emul(vmexit, gpa, vmcs_gla()); 2440bf73979dSNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INST_EMUL, 1); 2441a2da7af6SNeel Natu } 2442e5a1d950SNeel Natu /* 2443e5a1d950SNeel Natu * If Virtual NMIs control is 1 and the VM-exit is due to an 2444e5a1d950SNeel Natu * EPT fault during the execution of IRET then we must restore 2445e5a1d950SNeel Natu * the state of "virtual-NMI blocking" before resuming. 2446e5a1d950SNeel Natu * 2447e5a1d950SNeel Natu * See description of "NMI unblocking due to IRET" in 2448e5a1d950SNeel Natu * "Exit Qualification for EPT Violations". 2449e5a1d950SNeel Natu */ 2450e5a1d950SNeel Natu if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 && 2451e5a1d950SNeel Natu (qual & EXIT_QUAL_NMIUDTI) != 0) 2452e5a1d950SNeel Natu vmx_restore_nmi_blocking(vmx, vcpu); 2453cd942e0fSPeter Grehan break; 245430b94db8SNeel Natu case EXIT_REASON_VIRTUALIZED_EOI: 245530b94db8SNeel Natu vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI; 245630b94db8SNeel Natu vmexit->u.ioapic_eoi.vector = qual & 0xFF; 245730b94db8SNeel Natu vmexit->inst_length = 0; /* trap-like */ 245830b94db8SNeel Natu break; 245988c4b8d1SNeel Natu case EXIT_REASON_APIC_ACCESS: 246088c4b8d1SNeel Natu handled = vmx_handle_apic_access(vmx, vcpu, vmexit); 246188c4b8d1SNeel Natu break; 246288c4b8d1SNeel Natu case EXIT_REASON_APIC_WRITE: 246388c4b8d1SNeel Natu /* 246488c4b8d1SNeel Natu * APIC-write VM exit is trap-like so the %rip is already 246588c4b8d1SNeel Natu * pointing to the next instruction. 246688c4b8d1SNeel Natu */ 246788c4b8d1SNeel Natu vmexit->inst_length = 0; 246888c4b8d1SNeel Natu vlapic = vm_lapic(vmx->vm, vcpu); 2469159dd56fSNeel Natu handled = vmx_handle_apic_write(vmx, vcpu, vlapic, qual); 247088c4b8d1SNeel Natu break; 2471abb023fbSJohn Baldwin case EXIT_REASON_XSETBV: 2472a0efd3fbSJohn Baldwin handled = vmx_emulate_xsetbv(vmx, vcpu, vmexit); 2473abb023fbSJohn Baldwin break; 247465145c7fSNeel Natu case EXIT_REASON_MONITOR: 247565145c7fSNeel Natu vmexit->exitcode = VM_EXITCODE_MONITOR; 247665145c7fSNeel Natu break; 247765145c7fSNeel Natu case EXIT_REASON_MWAIT: 247865145c7fSNeel Natu vmexit->exitcode = VM_EXITCODE_MWAIT; 247965145c7fSNeel Natu break; 2480366f6083SPeter Grehan default: 2481b5aaf7b2SNeel Natu vmm_stat_incr(vmx->vm, vcpu, VMEXIT_UNKNOWN, 1); 2482366f6083SPeter Grehan break; 2483366f6083SPeter Grehan } 2484366f6083SPeter Grehan 2485366f6083SPeter Grehan if (handled) { 2486366f6083SPeter Grehan /* 2487366f6083SPeter Grehan * It is possible that control is returned to userland 2488366f6083SPeter Grehan * even though we were able to handle the VM exit in the 2489eeefa4e4SNeel Natu * kernel. 2490366f6083SPeter Grehan * 2491366f6083SPeter Grehan * In such a case we want to make sure that the userland 2492366f6083SPeter Grehan * restarts guest execution at the instruction *after* 2493366f6083SPeter Grehan * the one we just processed. Therefore we update the 2494366f6083SPeter Grehan * guest rip in the VMCS and in 'vmexit'. 2495366f6083SPeter Grehan */ 2496366f6083SPeter Grehan vmexit->rip += vmexit->inst_length; 2497366f6083SPeter Grehan vmexit->inst_length = 0; 24983de83862SNeel Natu vmcs_write(VMCS_GUEST_RIP, vmexit->rip); 2499366f6083SPeter Grehan } else { 2500366f6083SPeter Grehan if (vmexit->exitcode == VM_EXITCODE_BOGUS) { 2501366f6083SPeter Grehan /* 2502366f6083SPeter Grehan * If this VM exit was not claimed by anybody then 2503366f6083SPeter Grehan * treat it as a generic VMX exit. 2504366f6083SPeter Grehan */ 2505366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_VMX; 25060492757cSNeel Natu vmexit->u.vmx.status = VM_SUCCESS; 2507c308b23bSNeel Natu vmexit->u.vmx.inst_type = 0; 2508c308b23bSNeel Natu vmexit->u.vmx.inst_error = 0; 2509366f6083SPeter Grehan } else { 2510366f6083SPeter Grehan /* 2511366f6083SPeter Grehan * The exitcode and collateral have been populated. 2512366f6083SPeter Grehan * The VM exit will be processed further in userland. 2513366f6083SPeter Grehan */ 2514366f6083SPeter Grehan } 2515366f6083SPeter Grehan } 2516366f6083SPeter Grehan return (handled); 2517366f6083SPeter Grehan } 2518366f6083SPeter Grehan 251940487465SNeel Natu static __inline void 25200492757cSNeel Natu vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit) 25210492757cSNeel Natu { 25220492757cSNeel Natu 25230492757cSNeel Natu KASSERT(vmxctx->inst_fail_status != VM_SUCCESS, 25240492757cSNeel Natu ("vmx_exit_inst_error: invalid inst_fail_status %d", 25250492757cSNeel Natu vmxctx->inst_fail_status)); 25260492757cSNeel Natu 25270492757cSNeel Natu vmexit->inst_length = 0; 25280492757cSNeel Natu vmexit->exitcode = VM_EXITCODE_VMX; 25290492757cSNeel Natu vmexit->u.vmx.status = vmxctx->inst_fail_status; 25300492757cSNeel Natu vmexit->u.vmx.inst_error = vmcs_instruction_error(); 25310492757cSNeel Natu vmexit->u.vmx.exit_reason = ~0; 25320492757cSNeel Natu vmexit->u.vmx.exit_qualification = ~0; 25330492757cSNeel Natu 25340492757cSNeel Natu switch (rc) { 25350492757cSNeel Natu case VMX_VMRESUME_ERROR: 25360492757cSNeel Natu case VMX_VMLAUNCH_ERROR: 25370492757cSNeel Natu case VMX_INVEPT_ERROR: 25380492757cSNeel Natu vmexit->u.vmx.inst_type = rc; 25390492757cSNeel Natu break; 25400492757cSNeel Natu default: 25410492757cSNeel Natu panic("vm_exit_inst_error: vmx_enter_guest returned %d", rc); 25420492757cSNeel Natu } 25430492757cSNeel Natu } 25440492757cSNeel Natu 254562fbd7c2SNeel Natu /* 254662fbd7c2SNeel Natu * If the NMI-exiting VM execution control is set to '1' then an NMI in 254762fbd7c2SNeel Natu * non-root operation causes a VM-exit. NMI blocking is in effect so it is 254862fbd7c2SNeel Natu * sufficient to simply vector to the NMI handler via a software interrupt. 254962fbd7c2SNeel Natu * However, this must be done before maskable interrupts are enabled 255062fbd7c2SNeel Natu * otherwise the "iret" issued by an interrupt handler will incorrectly 255162fbd7c2SNeel Natu * clear NMI blocking. 255262fbd7c2SNeel Natu */ 255362fbd7c2SNeel Natu static __inline void 255462fbd7c2SNeel Natu vmx_exit_handle_nmi(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit) 255562fbd7c2SNeel Natu { 255662fbd7c2SNeel Natu uint32_t intr_info; 255762fbd7c2SNeel Natu 255862fbd7c2SNeel Natu KASSERT((read_rflags() & PSL_I) == 0, ("interrupts enabled")); 255962fbd7c2SNeel Natu 256062fbd7c2SNeel Natu if (vmexit->u.vmx.exit_reason != EXIT_REASON_EXCEPTION) 256162fbd7c2SNeel Natu return; 256262fbd7c2SNeel Natu 256362fbd7c2SNeel Natu intr_info = vmcs_read(VMCS_EXIT_INTR_INFO); 256462fbd7c2SNeel Natu KASSERT((intr_info & VMCS_INTR_VALID) != 0, 256562fbd7c2SNeel Natu ("VM exit interruption info invalid: %#x", intr_info)); 256662fbd7c2SNeel Natu 256762fbd7c2SNeel Natu if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI) { 256862fbd7c2SNeel Natu KASSERT((intr_info & 0xff) == IDT_NMI, ("VM exit due " 256962fbd7c2SNeel Natu "to NMI has invalid vector: %#x", intr_info)); 257062fbd7c2SNeel Natu VCPU_CTR0(vmx->vm, vcpuid, "Vectoring to NMI handler"); 257162fbd7c2SNeel Natu __asm __volatile("int $2"); 257262fbd7c2SNeel Natu } 257362fbd7c2SNeel Natu } 257462fbd7c2SNeel Natu 25750492757cSNeel Natu static int 25762ce12423SNeel Natu vmx_run(void *arg, int vcpu, register_t rip, pmap_t pmap, 2577248e6799SNeel Natu struct vm_eventinfo *evinfo) 25780492757cSNeel Natu { 25790492757cSNeel Natu int rc, handled, launched; 2580366f6083SPeter Grehan struct vmx *vmx; 25815b8a8cd1SNeel Natu struct vm *vm; 2582366f6083SPeter Grehan struct vmxctx *vmxctx; 2583366f6083SPeter Grehan struct vmcs *vmcs; 258498ed632cSNeel Natu struct vm_exit *vmexit; 2585de5ea6b6SNeel Natu struct vlapic *vlapic; 258679c59630SNeel Natu uint32_t exit_reason; 2587366f6083SPeter Grehan 2588366f6083SPeter Grehan vmx = arg; 25895b8a8cd1SNeel Natu vm = vmx->vm; 2590366f6083SPeter Grehan vmcs = &vmx->vmcs[vcpu]; 2591366f6083SPeter Grehan vmxctx = &vmx->ctx[vcpu]; 25925b8a8cd1SNeel Natu vlapic = vm_lapic(vm, vcpu); 25935b8a8cd1SNeel Natu vmexit = vm_exitinfo(vm, vcpu); 25940492757cSNeel Natu launched = 0; 259598ed632cSNeel Natu 2596318224bbSNeel Natu KASSERT(vmxctx->pmap == pmap, 2597318224bbSNeel Natu ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap)); 2598318224bbSNeel Natu 2599c3498942SNeel Natu vmx_msr_guest_enter(vmx, vcpu); 2600c3498942SNeel Natu 2601366f6083SPeter Grehan VMPTRLD(vmcs); 2602366f6083SPeter Grehan 2603366f6083SPeter Grehan /* 2604366f6083SPeter Grehan * XXX 2605366f6083SPeter Grehan * We do this every time because we may setup the virtual machine 2606366f6083SPeter Grehan * from a different process than the one that actually runs it. 2607366f6083SPeter Grehan * 2608366f6083SPeter Grehan * If the life of a virtual machine was spent entirely in the context 2609c847a506SNeel Natu * of a single process we could do this once in vmx_vminit(). 2610366f6083SPeter Grehan */ 26113de83862SNeel Natu vmcs_write(VMCS_HOST_CR3, rcr3()); 2612366f6083SPeter Grehan 26132ce12423SNeel Natu vmcs_write(VMCS_GUEST_RIP, rip); 2614953c2c47SNeel Natu vmx_set_pcpu_defaults(vmx, vcpu, pmap); 2615366f6083SPeter Grehan do { 26162ce12423SNeel Natu KASSERT(vmcs_guest_rip() == rip, ("%s: vmcs guest rip mismatch " 26172ce12423SNeel Natu "%#lx/%#lx", __func__, vmcs_guest_rip(), rip)); 261840487465SNeel Natu 26192ce12423SNeel Natu handled = UNHANDLED; 26200492757cSNeel Natu /* 26210492757cSNeel Natu * Interrupts are disabled from this point on until the 26220492757cSNeel Natu * guest starts executing. This is done for the following 26230492757cSNeel Natu * reasons: 26240492757cSNeel Natu * 26250492757cSNeel Natu * If an AST is asserted on this thread after the check below, 26260492757cSNeel Natu * then the IPI_AST notification will not be lost, because it 26270492757cSNeel Natu * will cause a VM exit due to external interrupt as soon as 26280492757cSNeel Natu * the guest state is loaded. 26290492757cSNeel Natu * 26300492757cSNeel Natu * A posted interrupt after 'vmx_inject_interrupts()' will 26310492757cSNeel Natu * not be "lost" because it will be held pending in the host 26320492757cSNeel Natu * APIC because interrupts are disabled. The pending interrupt 26330492757cSNeel Natu * will be recognized as soon as the guest state is loaded. 26340492757cSNeel Natu * 26350492757cSNeel Natu * The same reasoning applies to the IPI generated by 26360492757cSNeel Natu * pmap_invalidate_ept(). 26370492757cSNeel Natu */ 26380492757cSNeel Natu disable_intr(); 26392ce12423SNeel Natu vmx_inject_interrupts(vmx, vcpu, vlapic, rip); 2640091d4532SNeel Natu 2641091d4532SNeel Natu /* 2642091d4532SNeel Natu * Check for vcpu suspension after injecting events because 2643091d4532SNeel Natu * vmx_inject_interrupts() can suspend the vcpu due to a 2644091d4532SNeel Natu * triple fault. 2645091d4532SNeel Natu */ 2646248e6799SNeel Natu if (vcpu_suspended(evinfo)) { 26470492757cSNeel Natu enable_intr(); 26482ce12423SNeel Natu vm_exit_suspended(vmx->vm, vcpu, rip); 26490492757cSNeel Natu break; 26500492757cSNeel Natu } 26510492757cSNeel Natu 2652248e6799SNeel Natu if (vcpu_rendezvous_pending(evinfo)) { 26535b8a8cd1SNeel Natu enable_intr(); 26542ce12423SNeel Natu vm_exit_rendezvous(vmx->vm, vcpu, rip); 26555b8a8cd1SNeel Natu break; 26565b8a8cd1SNeel Natu } 26575b8a8cd1SNeel Natu 2658248e6799SNeel Natu if (vcpu_reqidle(evinfo)) { 2659248e6799SNeel Natu enable_intr(); 2660248e6799SNeel Natu vm_exit_reqidle(vmx->vm, vcpu, rip); 2661248e6799SNeel Natu break; 2662248e6799SNeel Natu } 2663248e6799SNeel Natu 2664f008d157SNeel Natu if (vcpu_should_yield(vm, vcpu)) { 2665b15a09c0SNeel Natu enable_intr(); 26662ce12423SNeel Natu vm_exit_astpending(vmx->vm, vcpu, rip); 26672ce12423SNeel Natu vmx_astpending_trace(vmx, vcpu, rip); 266840487465SNeel Natu handled = HANDLED; 2669b15a09c0SNeel Natu break; 2670b15a09c0SNeel Natu } 2671b15a09c0SNeel Natu 2672366f6083SPeter Grehan vmx_run_trace(vmx, vcpu); 2673953c2c47SNeel Natu rc = vmx_enter_guest(vmxctx, vmx, launched); 267479c59630SNeel Natu 267579c59630SNeel Natu /* Collect some information for VM exit processing */ 267679c59630SNeel Natu vmexit->rip = rip = vmcs_guest_rip(); 267779c59630SNeel Natu vmexit->inst_length = vmexit_instruction_length(); 267879c59630SNeel Natu vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason(); 267979c59630SNeel Natu vmexit->u.vmx.exit_qualification = vmcs_exit_qualification(); 268079c59630SNeel Natu 26812ce12423SNeel Natu /* Update 'nextrip' */ 26822ce12423SNeel Natu vmx->state[vcpu].nextrip = rip; 26832ce12423SNeel Natu 26840492757cSNeel Natu if (rc == VMX_GUEST_VMEXIT) { 268562fbd7c2SNeel Natu vmx_exit_handle_nmi(vmx, vcpu, vmexit); 268662fbd7c2SNeel Natu enable_intr(); 26870492757cSNeel Natu handled = vmx_exit_process(vmx, vcpu, vmexit); 26880492757cSNeel Natu } else { 268962fbd7c2SNeel Natu enable_intr(); 269040487465SNeel Natu vmx_exit_inst_error(vmxctx, rc, vmexit); 2691eeefa4e4SNeel Natu } 269262fbd7c2SNeel Natu launched = 1; 269379c59630SNeel Natu vmx_exit_trace(vmx, vcpu, rip, exit_reason, handled); 26942ce12423SNeel Natu rip = vmexit->rip; 2695eeefa4e4SNeel Natu } while (handled); 2696366f6083SPeter Grehan 2697366f6083SPeter Grehan /* 2698366f6083SPeter Grehan * If a VM exit has been handled then the exitcode must be BOGUS 2699366f6083SPeter Grehan * If a VM exit is not handled then the exitcode must not be BOGUS 2700366f6083SPeter Grehan */ 2701366f6083SPeter Grehan if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) || 2702366f6083SPeter Grehan (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) { 2703366f6083SPeter Grehan panic("Mismatch between handled (%d) and exitcode (%d)", 2704366f6083SPeter Grehan handled, vmexit->exitcode); 2705366f6083SPeter Grehan } 2706366f6083SPeter Grehan 2707b5aaf7b2SNeel Natu if (!handled) 27085b8a8cd1SNeel Natu vmm_stat_incr(vm, vcpu, VMEXIT_USERSPACE, 1); 2709b5aaf7b2SNeel Natu 27105b8a8cd1SNeel Natu VCPU_CTR1(vm, vcpu, "returning from vmx_run: exitcode %d", 27110492757cSNeel Natu vmexit->exitcode); 2712366f6083SPeter Grehan 2713366f6083SPeter Grehan VMCLEAR(vmcs); 2714c3498942SNeel Natu vmx_msr_guest_exit(vmx, vcpu); 2715c3498942SNeel Natu 2716366f6083SPeter Grehan return (0); 2717366f6083SPeter Grehan } 2718366f6083SPeter Grehan 2719366f6083SPeter Grehan static void 2720366f6083SPeter Grehan vmx_vmcleanup(void *arg) 2721366f6083SPeter Grehan { 272263c9389aSNeel Natu int i; 2723366f6083SPeter Grehan struct vmx *vmx = arg; 2724366f6083SPeter Grehan 2725159dd56fSNeel Natu if (apic_access_virtualization(vmx, 0)) 272688c4b8d1SNeel Natu vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE); 272788c4b8d1SNeel Natu 272845e51299SNeel Natu for (i = 0; i < VM_MAXCPU; i++) 272945e51299SNeel Natu vpid_free(vmx->state[i].vpid); 273045e51299SNeel Natu 2731366f6083SPeter Grehan free(vmx, M_VMX); 2732366f6083SPeter Grehan 2733366f6083SPeter Grehan return; 2734366f6083SPeter Grehan } 2735366f6083SPeter Grehan 2736366f6083SPeter Grehan static register_t * 2737366f6083SPeter Grehan vmxctx_regptr(struct vmxctx *vmxctx, int reg) 2738366f6083SPeter Grehan { 2739366f6083SPeter Grehan 2740366f6083SPeter Grehan switch (reg) { 2741366f6083SPeter Grehan case VM_REG_GUEST_RAX: 2742366f6083SPeter Grehan return (&vmxctx->guest_rax); 2743366f6083SPeter Grehan case VM_REG_GUEST_RBX: 2744366f6083SPeter Grehan return (&vmxctx->guest_rbx); 2745366f6083SPeter Grehan case VM_REG_GUEST_RCX: 2746366f6083SPeter Grehan return (&vmxctx->guest_rcx); 2747366f6083SPeter Grehan case VM_REG_GUEST_RDX: 2748366f6083SPeter Grehan return (&vmxctx->guest_rdx); 2749366f6083SPeter Grehan case VM_REG_GUEST_RSI: 2750366f6083SPeter Grehan return (&vmxctx->guest_rsi); 2751366f6083SPeter Grehan case VM_REG_GUEST_RDI: 2752366f6083SPeter Grehan return (&vmxctx->guest_rdi); 2753366f6083SPeter Grehan case VM_REG_GUEST_RBP: 2754366f6083SPeter Grehan return (&vmxctx->guest_rbp); 2755366f6083SPeter Grehan case VM_REG_GUEST_R8: 2756366f6083SPeter Grehan return (&vmxctx->guest_r8); 2757366f6083SPeter Grehan case VM_REG_GUEST_R9: 2758366f6083SPeter Grehan return (&vmxctx->guest_r9); 2759366f6083SPeter Grehan case VM_REG_GUEST_R10: 2760366f6083SPeter Grehan return (&vmxctx->guest_r10); 2761366f6083SPeter Grehan case VM_REG_GUEST_R11: 2762366f6083SPeter Grehan return (&vmxctx->guest_r11); 2763366f6083SPeter Grehan case VM_REG_GUEST_R12: 2764366f6083SPeter Grehan return (&vmxctx->guest_r12); 2765366f6083SPeter Grehan case VM_REG_GUEST_R13: 2766366f6083SPeter Grehan return (&vmxctx->guest_r13); 2767366f6083SPeter Grehan case VM_REG_GUEST_R14: 2768366f6083SPeter Grehan return (&vmxctx->guest_r14); 2769366f6083SPeter Grehan case VM_REG_GUEST_R15: 2770366f6083SPeter Grehan return (&vmxctx->guest_r15); 277137a723a5SNeel Natu case VM_REG_GUEST_CR2: 277237a723a5SNeel Natu return (&vmxctx->guest_cr2); 2773366f6083SPeter Grehan default: 2774366f6083SPeter Grehan break; 2775366f6083SPeter Grehan } 2776366f6083SPeter Grehan return (NULL); 2777366f6083SPeter Grehan } 2778366f6083SPeter Grehan 2779366f6083SPeter Grehan static int 2780366f6083SPeter Grehan vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval) 2781366f6083SPeter Grehan { 2782366f6083SPeter Grehan register_t *regp; 2783366f6083SPeter Grehan 2784366f6083SPeter Grehan if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) { 2785366f6083SPeter Grehan *retval = *regp; 2786366f6083SPeter Grehan return (0); 2787366f6083SPeter Grehan } else 2788366f6083SPeter Grehan return (EINVAL); 2789366f6083SPeter Grehan } 2790366f6083SPeter Grehan 2791366f6083SPeter Grehan static int 2792366f6083SPeter Grehan vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val) 2793366f6083SPeter Grehan { 2794366f6083SPeter Grehan register_t *regp; 2795366f6083SPeter Grehan 2796366f6083SPeter Grehan if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) { 2797366f6083SPeter Grehan *regp = val; 2798366f6083SPeter Grehan return (0); 2799366f6083SPeter Grehan } else 2800366f6083SPeter Grehan return (EINVAL); 2801366f6083SPeter Grehan } 2802366f6083SPeter Grehan 2803366f6083SPeter Grehan static int 2804d1819632SNeel Natu vmx_get_intr_shadow(struct vmx *vmx, int vcpu, int running, uint64_t *retval) 2805d1819632SNeel Natu { 2806d1819632SNeel Natu uint64_t gi; 2807d1819632SNeel Natu int error; 2808d1819632SNeel Natu 2809d1819632SNeel Natu error = vmcs_getreg(&vmx->vmcs[vcpu], running, 2810d1819632SNeel Natu VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY), &gi); 2811d1819632SNeel Natu *retval = (gi & HWINTR_BLOCKING) ? 1 : 0; 2812d1819632SNeel Natu return (error); 2813d1819632SNeel Natu } 2814d1819632SNeel Natu 2815d1819632SNeel Natu static int 2816d1819632SNeel Natu vmx_modify_intr_shadow(struct vmx *vmx, int vcpu, int running, uint64_t val) 2817d1819632SNeel Natu { 2818d1819632SNeel Natu struct vmcs *vmcs; 2819d1819632SNeel Natu uint64_t gi; 2820d1819632SNeel Natu int error, ident; 2821d1819632SNeel Natu 2822d1819632SNeel Natu /* 2823d1819632SNeel Natu * Forcing the vcpu into an interrupt shadow is not supported. 2824d1819632SNeel Natu */ 2825d1819632SNeel Natu if (val) { 2826d1819632SNeel Natu error = EINVAL; 2827d1819632SNeel Natu goto done; 2828d1819632SNeel Natu } 2829d1819632SNeel Natu 2830d1819632SNeel Natu vmcs = &vmx->vmcs[vcpu]; 2831d1819632SNeel Natu ident = VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY); 2832d1819632SNeel Natu error = vmcs_getreg(vmcs, running, ident, &gi); 2833d1819632SNeel Natu if (error == 0) { 2834d1819632SNeel Natu gi &= ~HWINTR_BLOCKING; 2835d1819632SNeel Natu error = vmcs_setreg(vmcs, running, ident, gi); 2836d1819632SNeel Natu } 2837d1819632SNeel Natu done: 2838d1819632SNeel Natu VCPU_CTR2(vmx->vm, vcpu, "Setting intr_shadow to %#lx %s", val, 2839d1819632SNeel Natu error ? "failed" : "succeeded"); 2840d1819632SNeel Natu return (error); 2841d1819632SNeel Natu } 2842d1819632SNeel Natu 2843d1819632SNeel Natu static int 2844aaaa0656SPeter Grehan vmx_shadow_reg(int reg) 2845aaaa0656SPeter Grehan { 2846aaaa0656SPeter Grehan int shreg; 2847aaaa0656SPeter Grehan 2848aaaa0656SPeter Grehan shreg = -1; 2849aaaa0656SPeter Grehan 2850aaaa0656SPeter Grehan switch (reg) { 2851aaaa0656SPeter Grehan case VM_REG_GUEST_CR0: 2852aaaa0656SPeter Grehan shreg = VMCS_CR0_SHADOW; 2853aaaa0656SPeter Grehan break; 2854aaaa0656SPeter Grehan case VM_REG_GUEST_CR4: 2855aaaa0656SPeter Grehan shreg = VMCS_CR4_SHADOW; 2856aaaa0656SPeter Grehan break; 2857aaaa0656SPeter Grehan default: 2858aaaa0656SPeter Grehan break; 2859aaaa0656SPeter Grehan } 2860aaaa0656SPeter Grehan 2861aaaa0656SPeter Grehan return (shreg); 2862aaaa0656SPeter Grehan } 2863aaaa0656SPeter Grehan 2864aaaa0656SPeter Grehan static int 2865366f6083SPeter Grehan vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval) 2866366f6083SPeter Grehan { 2867d3c11f40SPeter Grehan int running, hostcpu; 2868366f6083SPeter Grehan struct vmx *vmx = arg; 2869366f6083SPeter Grehan 2870d3c11f40SPeter Grehan running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); 2871d3c11f40SPeter Grehan if (running && hostcpu != curcpu) 2872d3c11f40SPeter Grehan panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu); 2873d3c11f40SPeter Grehan 2874d1819632SNeel Natu if (reg == VM_REG_GUEST_INTR_SHADOW) 2875d1819632SNeel Natu return (vmx_get_intr_shadow(vmx, vcpu, running, retval)); 2876d1819632SNeel Natu 2877366f6083SPeter Grehan if (vmxctx_getreg(&vmx->ctx[vcpu], reg, retval) == 0) 2878366f6083SPeter Grehan return (0); 2879366f6083SPeter Grehan 2880d3c11f40SPeter Grehan return (vmcs_getreg(&vmx->vmcs[vcpu], running, reg, retval)); 2881366f6083SPeter Grehan } 2882366f6083SPeter Grehan 2883366f6083SPeter Grehan static int 2884366f6083SPeter Grehan vmx_setreg(void *arg, int vcpu, int reg, uint64_t val) 2885366f6083SPeter Grehan { 2886aaaa0656SPeter Grehan int error, hostcpu, running, shadow; 2887366f6083SPeter Grehan uint64_t ctls; 28883527963bSNeel Natu pmap_t pmap; 2889366f6083SPeter Grehan struct vmx *vmx = arg; 2890366f6083SPeter Grehan 2891d3c11f40SPeter Grehan running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); 2892d3c11f40SPeter Grehan if (running && hostcpu != curcpu) 2893d3c11f40SPeter Grehan panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu); 2894d3c11f40SPeter Grehan 2895d1819632SNeel Natu if (reg == VM_REG_GUEST_INTR_SHADOW) 2896d1819632SNeel Natu return (vmx_modify_intr_shadow(vmx, vcpu, running, val)); 2897d1819632SNeel Natu 2898366f6083SPeter Grehan if (vmxctx_setreg(&vmx->ctx[vcpu], reg, val) == 0) 2899366f6083SPeter Grehan return (0); 2900366f6083SPeter Grehan 2901d3c11f40SPeter Grehan error = vmcs_setreg(&vmx->vmcs[vcpu], running, reg, val); 2902366f6083SPeter Grehan 2903366f6083SPeter Grehan if (error == 0) { 2904366f6083SPeter Grehan /* 2905366f6083SPeter Grehan * If the "load EFER" VM-entry control is 1 then the 2906366f6083SPeter Grehan * value of EFER.LMA must be identical to "IA-32e mode guest" 2907366f6083SPeter Grehan * bit in the VM-entry control. 2908366f6083SPeter Grehan */ 2909366f6083SPeter Grehan if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 && 2910366f6083SPeter Grehan (reg == VM_REG_GUEST_EFER)) { 2911d3c11f40SPeter Grehan vmcs_getreg(&vmx->vmcs[vcpu], running, 2912366f6083SPeter Grehan VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls); 2913366f6083SPeter Grehan if (val & EFER_LMA) 2914366f6083SPeter Grehan ctls |= VM_ENTRY_GUEST_LMA; 2915366f6083SPeter Grehan else 2916366f6083SPeter Grehan ctls &= ~VM_ENTRY_GUEST_LMA; 2917d3c11f40SPeter Grehan vmcs_setreg(&vmx->vmcs[vcpu], running, 2918366f6083SPeter Grehan VMCS_IDENT(VMCS_ENTRY_CTLS), ctls); 2919366f6083SPeter Grehan } 2920aaaa0656SPeter Grehan 2921aaaa0656SPeter Grehan shadow = vmx_shadow_reg(reg); 2922aaaa0656SPeter Grehan if (shadow > 0) { 2923aaaa0656SPeter Grehan /* 2924aaaa0656SPeter Grehan * Store the unmodified value in the shadow 2925aaaa0656SPeter Grehan */ 2926aaaa0656SPeter Grehan error = vmcs_setreg(&vmx->vmcs[vcpu], running, 2927aaaa0656SPeter Grehan VMCS_IDENT(shadow), val); 2928aaaa0656SPeter Grehan } 29293527963bSNeel Natu 29303527963bSNeel Natu if (reg == VM_REG_GUEST_CR3) { 29313527963bSNeel Natu /* 29323527963bSNeel Natu * Invalidate the guest vcpu's TLB mappings to emulate 29333527963bSNeel Natu * the behavior of updating %cr3. 29343527963bSNeel Natu * 29353527963bSNeel Natu * XXX the processor retains global mappings when %cr3 29363527963bSNeel Natu * is updated but vmx_invvpid() does not. 29373527963bSNeel Natu */ 29383527963bSNeel Natu pmap = vmx->ctx[vcpu].pmap; 29393527963bSNeel Natu vmx_invvpid(vmx, vcpu, pmap, running); 29403527963bSNeel Natu } 2941366f6083SPeter Grehan } 2942366f6083SPeter Grehan 2943366f6083SPeter Grehan return (error); 2944366f6083SPeter Grehan } 2945366f6083SPeter Grehan 2946366f6083SPeter Grehan static int 2947366f6083SPeter Grehan vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc) 2948366f6083SPeter Grehan { 2949ba6f5e23SNeel Natu int hostcpu, running; 2950366f6083SPeter Grehan struct vmx *vmx = arg; 2951366f6083SPeter Grehan 2952ba6f5e23SNeel Natu running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); 2953ba6f5e23SNeel Natu if (running && hostcpu != curcpu) 2954ba6f5e23SNeel Natu panic("vmx_getdesc: %s%d is running", vm_name(vmx->vm), vcpu); 2955ba6f5e23SNeel Natu 2956ba6f5e23SNeel Natu return (vmcs_getdesc(&vmx->vmcs[vcpu], running, reg, desc)); 2957366f6083SPeter Grehan } 2958366f6083SPeter Grehan 2959366f6083SPeter Grehan static int 2960366f6083SPeter Grehan vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc) 2961366f6083SPeter Grehan { 2962ba6f5e23SNeel Natu int hostcpu, running; 2963366f6083SPeter Grehan struct vmx *vmx = arg; 2964366f6083SPeter Grehan 2965ba6f5e23SNeel Natu running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); 2966ba6f5e23SNeel Natu if (running && hostcpu != curcpu) 2967ba6f5e23SNeel Natu panic("vmx_setdesc: %s%d is running", vm_name(vmx->vm), vcpu); 2968ba6f5e23SNeel Natu 2969ba6f5e23SNeel Natu return (vmcs_setdesc(&vmx->vmcs[vcpu], running, reg, desc)); 2970366f6083SPeter Grehan } 2971366f6083SPeter Grehan 2972366f6083SPeter Grehan static int 2973366f6083SPeter Grehan vmx_getcap(void *arg, int vcpu, int type, int *retval) 2974366f6083SPeter Grehan { 2975366f6083SPeter Grehan struct vmx *vmx = arg; 2976366f6083SPeter Grehan int vcap; 2977366f6083SPeter Grehan int ret; 2978366f6083SPeter Grehan 2979366f6083SPeter Grehan ret = ENOENT; 2980366f6083SPeter Grehan 2981366f6083SPeter Grehan vcap = vmx->cap[vcpu].set; 2982366f6083SPeter Grehan 2983366f6083SPeter Grehan switch (type) { 2984366f6083SPeter Grehan case VM_CAP_HALT_EXIT: 2985366f6083SPeter Grehan if (cap_halt_exit) 2986366f6083SPeter Grehan ret = 0; 2987366f6083SPeter Grehan break; 2988366f6083SPeter Grehan case VM_CAP_PAUSE_EXIT: 2989366f6083SPeter Grehan if (cap_pause_exit) 2990366f6083SPeter Grehan ret = 0; 2991366f6083SPeter Grehan break; 2992366f6083SPeter Grehan case VM_CAP_MTRAP_EXIT: 2993366f6083SPeter Grehan if (cap_monitor_trap) 2994366f6083SPeter Grehan ret = 0; 2995366f6083SPeter Grehan break; 2996366f6083SPeter Grehan case VM_CAP_UNRESTRICTED_GUEST: 2997366f6083SPeter Grehan if (cap_unrestricted_guest) 2998366f6083SPeter Grehan ret = 0; 2999366f6083SPeter Grehan break; 300049cc03daSNeel Natu case VM_CAP_ENABLE_INVPCID: 300149cc03daSNeel Natu if (cap_invpcid) 300249cc03daSNeel Natu ret = 0; 300349cc03daSNeel Natu break; 3004366f6083SPeter Grehan default: 3005366f6083SPeter Grehan break; 3006366f6083SPeter Grehan } 3007366f6083SPeter Grehan 3008366f6083SPeter Grehan if (ret == 0) 3009366f6083SPeter Grehan *retval = (vcap & (1 << type)) ? 1 : 0; 3010366f6083SPeter Grehan 3011366f6083SPeter Grehan return (ret); 3012366f6083SPeter Grehan } 3013366f6083SPeter Grehan 3014366f6083SPeter Grehan static int 3015366f6083SPeter Grehan vmx_setcap(void *arg, int vcpu, int type, int val) 3016366f6083SPeter Grehan { 3017366f6083SPeter Grehan struct vmx *vmx = arg; 3018366f6083SPeter Grehan struct vmcs *vmcs = &vmx->vmcs[vcpu]; 3019366f6083SPeter Grehan uint32_t baseval; 3020366f6083SPeter Grehan uint32_t *pptr; 3021366f6083SPeter Grehan int error; 3022366f6083SPeter Grehan int flag; 3023366f6083SPeter Grehan int reg; 3024366f6083SPeter Grehan int retval; 3025366f6083SPeter Grehan 3026366f6083SPeter Grehan retval = ENOENT; 3027366f6083SPeter Grehan pptr = NULL; 3028366f6083SPeter Grehan 3029366f6083SPeter Grehan switch (type) { 3030366f6083SPeter Grehan case VM_CAP_HALT_EXIT: 3031366f6083SPeter Grehan if (cap_halt_exit) { 3032366f6083SPeter Grehan retval = 0; 3033366f6083SPeter Grehan pptr = &vmx->cap[vcpu].proc_ctls; 3034366f6083SPeter Grehan baseval = *pptr; 3035366f6083SPeter Grehan flag = PROCBASED_HLT_EXITING; 3036366f6083SPeter Grehan reg = VMCS_PRI_PROC_BASED_CTLS; 3037366f6083SPeter Grehan } 3038366f6083SPeter Grehan break; 3039366f6083SPeter Grehan case VM_CAP_MTRAP_EXIT: 3040366f6083SPeter Grehan if (cap_monitor_trap) { 3041366f6083SPeter Grehan retval = 0; 3042366f6083SPeter Grehan pptr = &vmx->cap[vcpu].proc_ctls; 3043366f6083SPeter Grehan baseval = *pptr; 3044366f6083SPeter Grehan flag = PROCBASED_MTF; 3045366f6083SPeter Grehan reg = VMCS_PRI_PROC_BASED_CTLS; 3046366f6083SPeter Grehan } 3047366f6083SPeter Grehan break; 3048366f6083SPeter Grehan case VM_CAP_PAUSE_EXIT: 3049366f6083SPeter Grehan if (cap_pause_exit) { 3050366f6083SPeter Grehan retval = 0; 3051366f6083SPeter Grehan pptr = &vmx->cap[vcpu].proc_ctls; 3052366f6083SPeter Grehan baseval = *pptr; 3053366f6083SPeter Grehan flag = PROCBASED_PAUSE_EXITING; 3054366f6083SPeter Grehan reg = VMCS_PRI_PROC_BASED_CTLS; 3055366f6083SPeter Grehan } 3056366f6083SPeter Grehan break; 3057366f6083SPeter Grehan case VM_CAP_UNRESTRICTED_GUEST: 3058366f6083SPeter Grehan if (cap_unrestricted_guest) { 3059366f6083SPeter Grehan retval = 0; 306049cc03daSNeel Natu pptr = &vmx->cap[vcpu].proc_ctls2; 306149cc03daSNeel Natu baseval = *pptr; 3062366f6083SPeter Grehan flag = PROCBASED2_UNRESTRICTED_GUEST; 3063366f6083SPeter Grehan reg = VMCS_SEC_PROC_BASED_CTLS; 3064366f6083SPeter Grehan } 3065366f6083SPeter Grehan break; 306649cc03daSNeel Natu case VM_CAP_ENABLE_INVPCID: 306749cc03daSNeel Natu if (cap_invpcid) { 306849cc03daSNeel Natu retval = 0; 306949cc03daSNeel Natu pptr = &vmx->cap[vcpu].proc_ctls2; 307049cc03daSNeel Natu baseval = *pptr; 307149cc03daSNeel Natu flag = PROCBASED2_ENABLE_INVPCID; 307249cc03daSNeel Natu reg = VMCS_SEC_PROC_BASED_CTLS; 307349cc03daSNeel Natu } 307449cc03daSNeel Natu break; 3075366f6083SPeter Grehan default: 3076366f6083SPeter Grehan break; 3077366f6083SPeter Grehan } 3078366f6083SPeter Grehan 3079366f6083SPeter Grehan if (retval == 0) { 3080366f6083SPeter Grehan if (val) { 3081366f6083SPeter Grehan baseval |= flag; 3082366f6083SPeter Grehan } else { 3083366f6083SPeter Grehan baseval &= ~flag; 3084366f6083SPeter Grehan } 3085366f6083SPeter Grehan VMPTRLD(vmcs); 3086366f6083SPeter Grehan error = vmwrite(reg, baseval); 3087366f6083SPeter Grehan VMCLEAR(vmcs); 3088366f6083SPeter Grehan 3089366f6083SPeter Grehan if (error) { 3090366f6083SPeter Grehan retval = error; 3091366f6083SPeter Grehan } else { 3092366f6083SPeter Grehan /* 3093366f6083SPeter Grehan * Update optional stored flags, and record 3094366f6083SPeter Grehan * setting 3095366f6083SPeter Grehan */ 3096366f6083SPeter Grehan if (pptr != NULL) { 3097366f6083SPeter Grehan *pptr = baseval; 3098366f6083SPeter Grehan } 3099366f6083SPeter Grehan 3100366f6083SPeter Grehan if (val) { 3101366f6083SPeter Grehan vmx->cap[vcpu].set |= (1 << type); 3102366f6083SPeter Grehan } else { 3103366f6083SPeter Grehan vmx->cap[vcpu].set &= ~(1 << type); 3104366f6083SPeter Grehan } 3105366f6083SPeter Grehan } 3106366f6083SPeter Grehan } 3107366f6083SPeter Grehan 3108366f6083SPeter Grehan return (retval); 3109366f6083SPeter Grehan } 3110366f6083SPeter Grehan 311188c4b8d1SNeel Natu struct vlapic_vtx { 311288c4b8d1SNeel Natu struct vlapic vlapic; 3113176666c2SNeel Natu struct pir_desc *pir_desc; 311430b94db8SNeel Natu struct vmx *vmx; 311588c4b8d1SNeel Natu }; 311688c4b8d1SNeel Natu 311788c4b8d1SNeel Natu #define VMX_CTR_PIR(vm, vcpuid, pir_desc, notify, vector, level, msg) \ 311888c4b8d1SNeel Natu do { \ 311988c4b8d1SNeel Natu VCPU_CTR2(vm, vcpuid, msg " assert %s-triggered vector %d", \ 312088c4b8d1SNeel Natu level ? "level" : "edge", vector); \ 312188c4b8d1SNeel Natu VCPU_CTR1(vm, vcpuid, msg " pir0 0x%016lx", pir_desc->pir[0]); \ 312288c4b8d1SNeel Natu VCPU_CTR1(vm, vcpuid, msg " pir1 0x%016lx", pir_desc->pir[1]); \ 312388c4b8d1SNeel Natu VCPU_CTR1(vm, vcpuid, msg " pir2 0x%016lx", pir_desc->pir[2]); \ 312488c4b8d1SNeel Natu VCPU_CTR1(vm, vcpuid, msg " pir3 0x%016lx", pir_desc->pir[3]); \ 312588c4b8d1SNeel Natu VCPU_CTR1(vm, vcpuid, msg " notify: %s", notify ? "yes" : "no");\ 312688c4b8d1SNeel Natu } while (0) 312788c4b8d1SNeel Natu 312888c4b8d1SNeel Natu /* 312988c4b8d1SNeel Natu * vlapic->ops handlers that utilize the APICv hardware assist described in 313088c4b8d1SNeel Natu * Chapter 29 of the Intel SDM. 313188c4b8d1SNeel Natu */ 313288c4b8d1SNeel Natu static int 313388c4b8d1SNeel Natu vmx_set_intr_ready(struct vlapic *vlapic, int vector, bool level) 313488c4b8d1SNeel Natu { 313588c4b8d1SNeel Natu struct vlapic_vtx *vlapic_vtx; 313688c4b8d1SNeel Natu struct pir_desc *pir_desc; 313788c4b8d1SNeel Natu uint64_t mask; 313888c4b8d1SNeel Natu int idx, notify; 313988c4b8d1SNeel Natu 314088c4b8d1SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 3141176666c2SNeel Natu pir_desc = vlapic_vtx->pir_desc; 314288c4b8d1SNeel Natu 314388c4b8d1SNeel Natu /* 314488c4b8d1SNeel Natu * Keep track of interrupt requests in the PIR descriptor. This is 314588c4b8d1SNeel Natu * because the virtual APIC page pointed to by the VMCS cannot be 314688c4b8d1SNeel Natu * modified if the vcpu is running. 314788c4b8d1SNeel Natu */ 314888c4b8d1SNeel Natu idx = vector / 64; 314988c4b8d1SNeel Natu mask = 1UL << (vector % 64); 315088c4b8d1SNeel Natu atomic_set_long(&pir_desc->pir[idx], mask); 315188c4b8d1SNeel Natu notify = atomic_cmpset_long(&pir_desc->pending, 0, 1); 315288c4b8d1SNeel Natu 315388c4b8d1SNeel Natu VMX_CTR_PIR(vlapic->vm, vlapic->vcpuid, pir_desc, notify, vector, 315488c4b8d1SNeel Natu level, "vmx_set_intr_ready"); 315588c4b8d1SNeel Natu return (notify); 315688c4b8d1SNeel Natu } 315788c4b8d1SNeel Natu 315888c4b8d1SNeel Natu static int 315988c4b8d1SNeel Natu vmx_pending_intr(struct vlapic *vlapic, int *vecptr) 316088c4b8d1SNeel Natu { 316188c4b8d1SNeel Natu struct vlapic_vtx *vlapic_vtx; 316288c4b8d1SNeel Natu struct pir_desc *pir_desc; 316388c4b8d1SNeel Natu struct LAPIC *lapic; 316488c4b8d1SNeel Natu uint64_t pending, pirval; 316588c4b8d1SNeel Natu uint32_t ppr, vpr; 316688c4b8d1SNeel Natu int i; 316788c4b8d1SNeel Natu 316888c4b8d1SNeel Natu /* 316988c4b8d1SNeel Natu * This function is only expected to be called from the 'HLT' exit 317088c4b8d1SNeel Natu * handler which does not care about the vector that is pending. 317188c4b8d1SNeel Natu */ 317288c4b8d1SNeel Natu KASSERT(vecptr == NULL, ("vmx_pending_intr: vecptr must be NULL")); 317388c4b8d1SNeel Natu 317488c4b8d1SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 3175176666c2SNeel Natu pir_desc = vlapic_vtx->pir_desc; 317688c4b8d1SNeel Natu 317788c4b8d1SNeel Natu pending = atomic_load_acq_long(&pir_desc->pending); 31789e33a616STycho Nightingale if (!pending) { 31799e33a616STycho Nightingale /* 31809e33a616STycho Nightingale * While a virtual interrupt may have already been 31819e33a616STycho Nightingale * processed the actual delivery maybe pending the 31829e33a616STycho Nightingale * interruptibility of the guest. Recognize a pending 31839e33a616STycho Nightingale * interrupt by reevaluating virtual interrupts 31849e33a616STycho Nightingale * following Section 29.2.1 in the Intel SDM Volume 3. 31859e33a616STycho Nightingale */ 31869e33a616STycho Nightingale uint64_t val; 31879e33a616STycho Nightingale uint8_t rvi, ppr; 31889e33a616STycho Nightingale 31899e33a616STycho Nightingale vmx_getreg(vlapic_vtx->vmx, vlapic->vcpuid, 31909e33a616STycho Nightingale VMCS_IDENT(VMCS_GUEST_INTR_STATUS), &val); 31919e33a616STycho Nightingale rvi = val & APIC_TPR_INT; 31929e33a616STycho Nightingale lapic = vlapic->apic_page; 31939e33a616STycho Nightingale ppr = lapic->ppr & APIC_TPR_INT; 31949e33a616STycho Nightingale if (rvi > ppr) { 31959e33a616STycho Nightingale return (1); 31969e33a616STycho Nightingale } 31979e33a616STycho Nightingale 31989e33a616STycho Nightingale return (0); 31999e33a616STycho Nightingale } 320088c4b8d1SNeel Natu 320188c4b8d1SNeel Natu /* 320288c4b8d1SNeel Natu * If there is an interrupt pending then it will be recognized only 320388c4b8d1SNeel Natu * if its priority is greater than the processor priority. 320488c4b8d1SNeel Natu * 320588c4b8d1SNeel Natu * Special case: if the processor priority is zero then any pending 320688c4b8d1SNeel Natu * interrupt will be recognized. 320788c4b8d1SNeel Natu */ 320888c4b8d1SNeel Natu lapic = vlapic->apic_page; 32099e33a616STycho Nightingale ppr = lapic->ppr & APIC_TPR_INT; 321088c4b8d1SNeel Natu if (ppr == 0) 321188c4b8d1SNeel Natu return (1); 321288c4b8d1SNeel Natu 321388c4b8d1SNeel Natu VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "HLT with non-zero PPR %d", 321488c4b8d1SNeel Natu lapic->ppr); 321588c4b8d1SNeel Natu 321688c4b8d1SNeel Natu for (i = 3; i >= 0; i--) { 321788c4b8d1SNeel Natu pirval = pir_desc->pir[i]; 321888c4b8d1SNeel Natu if (pirval != 0) { 32199e33a616STycho Nightingale vpr = (i * 64 + flsl(pirval) - 1) & APIC_TPR_INT; 322088c4b8d1SNeel Natu return (vpr > ppr); 322188c4b8d1SNeel Natu } 322288c4b8d1SNeel Natu } 322388c4b8d1SNeel Natu return (0); 322488c4b8d1SNeel Natu } 322588c4b8d1SNeel Natu 322688c4b8d1SNeel Natu static void 322788c4b8d1SNeel Natu vmx_intr_accepted(struct vlapic *vlapic, int vector) 322888c4b8d1SNeel Natu { 322988c4b8d1SNeel Natu 323088c4b8d1SNeel Natu panic("vmx_intr_accepted: not expected to be called"); 323188c4b8d1SNeel Natu } 323288c4b8d1SNeel Natu 3233176666c2SNeel Natu static void 323430b94db8SNeel Natu vmx_set_tmr(struct vlapic *vlapic, int vector, bool level) 323530b94db8SNeel Natu { 323630b94db8SNeel Natu struct vlapic_vtx *vlapic_vtx; 323730b94db8SNeel Natu struct vmx *vmx; 323830b94db8SNeel Natu struct vmcs *vmcs; 323930b94db8SNeel Natu uint64_t mask, val; 324030b94db8SNeel Natu 324130b94db8SNeel Natu KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector)); 324230b94db8SNeel Natu KASSERT(!vcpu_is_running(vlapic->vm, vlapic->vcpuid, NULL), 324330b94db8SNeel Natu ("vmx_set_tmr: vcpu cannot be running")); 324430b94db8SNeel Natu 324530b94db8SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 324630b94db8SNeel Natu vmx = vlapic_vtx->vmx; 324730b94db8SNeel Natu vmcs = &vmx->vmcs[vlapic->vcpuid]; 324830b94db8SNeel Natu mask = 1UL << (vector % 64); 324930b94db8SNeel Natu 325030b94db8SNeel Natu VMPTRLD(vmcs); 325130b94db8SNeel Natu val = vmcs_read(VMCS_EOI_EXIT(vector)); 325230b94db8SNeel Natu if (level) 325330b94db8SNeel Natu val |= mask; 325430b94db8SNeel Natu else 325530b94db8SNeel Natu val &= ~mask; 325630b94db8SNeel Natu vmcs_write(VMCS_EOI_EXIT(vector), val); 325730b94db8SNeel Natu VMCLEAR(vmcs); 325830b94db8SNeel Natu } 325930b94db8SNeel Natu 326030b94db8SNeel Natu static void 3261159dd56fSNeel Natu vmx_enable_x2apic_mode(struct vlapic *vlapic) 3262159dd56fSNeel Natu { 3263159dd56fSNeel Natu struct vmx *vmx; 3264159dd56fSNeel Natu struct vmcs *vmcs; 3265159dd56fSNeel Natu uint32_t proc_ctls2; 3266159dd56fSNeel Natu int vcpuid, error; 3267159dd56fSNeel Natu 3268159dd56fSNeel Natu vcpuid = vlapic->vcpuid; 3269159dd56fSNeel Natu vmx = ((struct vlapic_vtx *)vlapic)->vmx; 3270159dd56fSNeel Natu vmcs = &vmx->vmcs[vcpuid]; 3271159dd56fSNeel Natu 3272159dd56fSNeel Natu proc_ctls2 = vmx->cap[vcpuid].proc_ctls2; 3273159dd56fSNeel Natu KASSERT((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) != 0, 3274159dd56fSNeel Natu ("%s: invalid proc_ctls2 %#x", __func__, proc_ctls2)); 3275159dd56fSNeel Natu 3276159dd56fSNeel Natu proc_ctls2 &= ~PROCBASED2_VIRTUALIZE_APIC_ACCESSES; 3277159dd56fSNeel Natu proc_ctls2 |= PROCBASED2_VIRTUALIZE_X2APIC_MODE; 3278159dd56fSNeel Natu vmx->cap[vcpuid].proc_ctls2 = proc_ctls2; 3279159dd56fSNeel Natu 3280159dd56fSNeel Natu VMPTRLD(vmcs); 3281159dd56fSNeel Natu vmcs_write(VMCS_SEC_PROC_BASED_CTLS, proc_ctls2); 3282159dd56fSNeel Natu VMCLEAR(vmcs); 3283159dd56fSNeel Natu 3284159dd56fSNeel Natu if (vlapic->vcpuid == 0) { 3285159dd56fSNeel Natu /* 3286159dd56fSNeel Natu * The nested page table mappings are shared by all vcpus 3287159dd56fSNeel Natu * so unmap the APIC access page just once. 3288159dd56fSNeel Natu */ 3289159dd56fSNeel Natu error = vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE); 3290159dd56fSNeel Natu KASSERT(error == 0, ("%s: vm_unmap_mmio error %d", 3291159dd56fSNeel Natu __func__, error)); 3292159dd56fSNeel Natu 3293159dd56fSNeel Natu /* 3294159dd56fSNeel Natu * The MSR bitmap is shared by all vcpus so modify it only 3295159dd56fSNeel Natu * once in the context of vcpu 0. 3296159dd56fSNeel Natu */ 3297159dd56fSNeel Natu error = vmx_allow_x2apic_msrs(vmx); 3298159dd56fSNeel Natu KASSERT(error == 0, ("%s: vmx_allow_x2apic_msrs error %d", 3299159dd56fSNeel Natu __func__, error)); 3300159dd56fSNeel Natu } 3301159dd56fSNeel Natu } 3302159dd56fSNeel Natu 3303159dd56fSNeel Natu static void 3304176666c2SNeel Natu vmx_post_intr(struct vlapic *vlapic, int hostcpu) 3305176666c2SNeel Natu { 3306176666c2SNeel Natu 3307176666c2SNeel Natu ipi_cpu(hostcpu, pirvec); 3308176666c2SNeel Natu } 3309176666c2SNeel Natu 331088c4b8d1SNeel Natu /* 331188c4b8d1SNeel Natu * Transfer the pending interrupts in the PIR descriptor to the IRR 331288c4b8d1SNeel Natu * in the virtual APIC page. 331388c4b8d1SNeel Natu */ 331488c4b8d1SNeel Natu static void 331588c4b8d1SNeel Natu vmx_inject_pir(struct vlapic *vlapic) 331688c4b8d1SNeel Natu { 331788c4b8d1SNeel Natu struct vlapic_vtx *vlapic_vtx; 331888c4b8d1SNeel Natu struct pir_desc *pir_desc; 331988c4b8d1SNeel Natu struct LAPIC *lapic; 332088c4b8d1SNeel Natu uint64_t val, pirval; 33210e30c5c0SWarner Losh int rvi, pirbase = -1; 332288c4b8d1SNeel Natu uint16_t intr_status_old, intr_status_new; 332388c4b8d1SNeel Natu 332488c4b8d1SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 3325176666c2SNeel Natu pir_desc = vlapic_vtx->pir_desc; 332688c4b8d1SNeel Natu if (atomic_cmpset_long(&pir_desc->pending, 1, 0) == 0) { 332788c4b8d1SNeel Natu VCPU_CTR0(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: " 332888c4b8d1SNeel Natu "no posted interrupt pending"); 332988c4b8d1SNeel Natu return; 333088c4b8d1SNeel Natu } 333188c4b8d1SNeel Natu 333288c4b8d1SNeel Natu pirval = 0; 3333201b1cccSPeter Grehan pirbase = -1; 333488c4b8d1SNeel Natu lapic = vlapic->apic_page; 333588c4b8d1SNeel Natu 333688c4b8d1SNeel Natu val = atomic_readandclear_long(&pir_desc->pir[0]); 333788c4b8d1SNeel Natu if (val != 0) { 333888c4b8d1SNeel Natu lapic->irr0 |= val; 333988c4b8d1SNeel Natu lapic->irr1 |= val >> 32; 334088c4b8d1SNeel Natu pirbase = 0; 334188c4b8d1SNeel Natu pirval = val; 334288c4b8d1SNeel Natu } 334388c4b8d1SNeel Natu 334488c4b8d1SNeel Natu val = atomic_readandclear_long(&pir_desc->pir[1]); 334588c4b8d1SNeel Natu if (val != 0) { 334688c4b8d1SNeel Natu lapic->irr2 |= val; 334788c4b8d1SNeel Natu lapic->irr3 |= val >> 32; 334888c4b8d1SNeel Natu pirbase = 64; 334988c4b8d1SNeel Natu pirval = val; 335088c4b8d1SNeel Natu } 335188c4b8d1SNeel Natu 335288c4b8d1SNeel Natu val = atomic_readandclear_long(&pir_desc->pir[2]); 335388c4b8d1SNeel Natu if (val != 0) { 335488c4b8d1SNeel Natu lapic->irr4 |= val; 335588c4b8d1SNeel Natu lapic->irr5 |= val >> 32; 335688c4b8d1SNeel Natu pirbase = 128; 335788c4b8d1SNeel Natu pirval = val; 335888c4b8d1SNeel Natu } 335988c4b8d1SNeel Natu 336088c4b8d1SNeel Natu val = atomic_readandclear_long(&pir_desc->pir[3]); 336188c4b8d1SNeel Natu if (val != 0) { 336288c4b8d1SNeel Natu lapic->irr6 |= val; 336388c4b8d1SNeel Natu lapic->irr7 |= val >> 32; 336488c4b8d1SNeel Natu pirbase = 192; 336588c4b8d1SNeel Natu pirval = val; 336688c4b8d1SNeel Natu } 3367201b1cccSPeter Grehan 336888c4b8d1SNeel Natu VLAPIC_CTR_IRR(vlapic, "vmx_inject_pir"); 336988c4b8d1SNeel Natu 337088c4b8d1SNeel Natu /* 337188c4b8d1SNeel Natu * Update RVI so the processor can evaluate pending virtual 337288c4b8d1SNeel Natu * interrupts on VM-entry. 3373201b1cccSPeter Grehan * 3374201b1cccSPeter Grehan * It is possible for pirval to be 0 here, even though the 3375201b1cccSPeter Grehan * pending bit has been set. The scenario is: 3376201b1cccSPeter Grehan * CPU-Y is sending a posted interrupt to CPU-X, which 3377201b1cccSPeter Grehan * is running a guest and processing posted interrupts in h/w. 3378201b1cccSPeter Grehan * CPU-X will eventually exit and the state seen in s/w is 3379201b1cccSPeter Grehan * the pending bit set, but no PIR bits set. 3380201b1cccSPeter Grehan * 3381201b1cccSPeter Grehan * CPU-X CPU-Y 3382201b1cccSPeter Grehan * (vm running) (host running) 3383201b1cccSPeter Grehan * rx posted interrupt 3384201b1cccSPeter Grehan * CLEAR pending bit 3385201b1cccSPeter Grehan * SET PIR bit 3386201b1cccSPeter Grehan * READ/CLEAR PIR bits 3387201b1cccSPeter Grehan * SET pending bit 3388201b1cccSPeter Grehan * (vm exit) 3389201b1cccSPeter Grehan * pending bit set, PIR 0 339088c4b8d1SNeel Natu */ 339188c4b8d1SNeel Natu if (pirval != 0) { 339288c4b8d1SNeel Natu rvi = pirbase + flsl(pirval) - 1; 339388c4b8d1SNeel Natu intr_status_old = vmcs_read(VMCS_GUEST_INTR_STATUS); 339488c4b8d1SNeel Natu intr_status_new = (intr_status_old & 0xFF00) | rvi; 339588c4b8d1SNeel Natu if (intr_status_new > intr_status_old) { 339688c4b8d1SNeel Natu vmcs_write(VMCS_GUEST_INTR_STATUS, intr_status_new); 339788c4b8d1SNeel Natu VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: " 339888c4b8d1SNeel Natu "guest_intr_status changed from 0x%04x to 0x%04x", 339988c4b8d1SNeel Natu intr_status_old, intr_status_new); 340088c4b8d1SNeel Natu } 340188c4b8d1SNeel Natu } 340288c4b8d1SNeel Natu } 340388c4b8d1SNeel Natu 3404de5ea6b6SNeel Natu static struct vlapic * 3405de5ea6b6SNeel Natu vmx_vlapic_init(void *arg, int vcpuid) 3406de5ea6b6SNeel Natu { 3407de5ea6b6SNeel Natu struct vmx *vmx; 3408de5ea6b6SNeel Natu struct vlapic *vlapic; 3409176666c2SNeel Natu struct vlapic_vtx *vlapic_vtx; 3410de5ea6b6SNeel Natu 3411de5ea6b6SNeel Natu vmx = arg; 3412de5ea6b6SNeel Natu 341388c4b8d1SNeel Natu vlapic = malloc(sizeof(struct vlapic_vtx), M_VLAPIC, M_WAITOK | M_ZERO); 3414de5ea6b6SNeel Natu vlapic->vm = vmx->vm; 3415de5ea6b6SNeel Natu vlapic->vcpuid = vcpuid; 3416de5ea6b6SNeel Natu vlapic->apic_page = (struct LAPIC *)&vmx->apic_page[vcpuid]; 3417de5ea6b6SNeel Natu 3418176666c2SNeel Natu vlapic_vtx = (struct vlapic_vtx *)vlapic; 3419176666c2SNeel Natu vlapic_vtx->pir_desc = &vmx->pir_desc[vcpuid]; 342030b94db8SNeel Natu vlapic_vtx->vmx = vmx; 3421176666c2SNeel Natu 342288c4b8d1SNeel Natu if (virtual_interrupt_delivery) { 342388c4b8d1SNeel Natu vlapic->ops.set_intr_ready = vmx_set_intr_ready; 342488c4b8d1SNeel Natu vlapic->ops.pending_intr = vmx_pending_intr; 342588c4b8d1SNeel Natu vlapic->ops.intr_accepted = vmx_intr_accepted; 342630b94db8SNeel Natu vlapic->ops.set_tmr = vmx_set_tmr; 3427159dd56fSNeel Natu vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode; 342888c4b8d1SNeel Natu } 342988c4b8d1SNeel Natu 3430176666c2SNeel Natu if (posted_interrupts) 3431176666c2SNeel Natu vlapic->ops.post_intr = vmx_post_intr; 3432176666c2SNeel Natu 3433de5ea6b6SNeel Natu vlapic_init(vlapic); 3434de5ea6b6SNeel Natu 3435de5ea6b6SNeel Natu return (vlapic); 3436de5ea6b6SNeel Natu } 3437de5ea6b6SNeel Natu 3438de5ea6b6SNeel Natu static void 3439de5ea6b6SNeel Natu vmx_vlapic_cleanup(void *arg, struct vlapic *vlapic) 3440de5ea6b6SNeel Natu { 3441de5ea6b6SNeel Natu 3442de5ea6b6SNeel Natu vlapic_cleanup(vlapic); 3443de5ea6b6SNeel Natu free(vlapic, M_VLAPIC); 3444de5ea6b6SNeel Natu } 3445de5ea6b6SNeel Natu 3446366f6083SPeter Grehan struct vmm_ops vmm_ops_intel = { 3447366f6083SPeter Grehan vmx_init, 3448366f6083SPeter Grehan vmx_cleanup, 344963e62d39SJohn Baldwin vmx_restore, 3450366f6083SPeter Grehan vmx_vminit, 3451366f6083SPeter Grehan vmx_run, 3452366f6083SPeter Grehan vmx_vmcleanup, 3453366f6083SPeter Grehan vmx_getreg, 3454366f6083SPeter Grehan vmx_setreg, 3455366f6083SPeter Grehan vmx_getdesc, 3456366f6083SPeter Grehan vmx_setdesc, 3457366f6083SPeter Grehan vmx_getcap, 3458318224bbSNeel Natu vmx_setcap, 3459318224bbSNeel Natu ept_vmspace_alloc, 3460318224bbSNeel Natu ept_vmspace_free, 3461de5ea6b6SNeel Natu vmx_vlapic_init, 3462de5ea6b6SNeel Natu vmx_vlapic_cleanup, 3463366f6083SPeter Grehan }; 3464