1366f6083SPeter Grehan /*- 2366f6083SPeter Grehan * Copyright (c) 2011 NetApp, Inc. 3366f6083SPeter Grehan * All rights reserved. 4366f6083SPeter Grehan * 5366f6083SPeter Grehan * Redistribution and use in source and binary forms, with or without 6366f6083SPeter Grehan * modification, are permitted provided that the following conditions 7366f6083SPeter Grehan * are met: 8366f6083SPeter Grehan * 1. Redistributions of source code must retain the above copyright 9366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer. 10366f6083SPeter Grehan * 2. Redistributions in binary form must reproduce the above copyright 11366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer in the 12366f6083SPeter Grehan * documentation and/or other materials provided with the distribution. 13366f6083SPeter Grehan * 14366f6083SPeter Grehan * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 15366f6083SPeter Grehan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16366f6083SPeter Grehan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17366f6083SPeter Grehan * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 18366f6083SPeter Grehan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19366f6083SPeter Grehan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20366f6083SPeter Grehan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21366f6083SPeter Grehan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22366f6083SPeter Grehan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23366f6083SPeter Grehan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24366f6083SPeter Grehan * SUCH DAMAGE. 25366f6083SPeter Grehan * 26366f6083SPeter Grehan * $FreeBSD$ 27366f6083SPeter Grehan */ 28366f6083SPeter Grehan 29366f6083SPeter Grehan #include <sys/cdefs.h> 30366f6083SPeter Grehan __FBSDID("$FreeBSD$"); 31366f6083SPeter Grehan 32366f6083SPeter Grehan #include <sys/param.h> 33366f6083SPeter Grehan #include <sys/systm.h> 34366f6083SPeter Grehan #include <sys/smp.h> 35366f6083SPeter Grehan #include <sys/kernel.h> 36366f6083SPeter Grehan #include <sys/malloc.h> 37366f6083SPeter Grehan #include <sys/pcpu.h> 38366f6083SPeter Grehan #include <sys/proc.h> 39366f6083SPeter Grehan 40366f6083SPeter Grehan #include <vm/vm.h> 41366f6083SPeter Grehan #include <vm/pmap.h> 42366f6083SPeter Grehan 43366f6083SPeter Grehan #include <machine/psl.h> 44366f6083SPeter Grehan #include <machine/cpufunc.h> 458b287612SJohn Baldwin #include <machine/md_var.h> 46366f6083SPeter Grehan #include <machine/pmap.h> 47366f6083SPeter Grehan #include <machine/segments.h> 48608f97c3SPeter Grehan #include <machine/specialreg.h> 49366f6083SPeter Grehan #include <machine/vmparam.h> 50366f6083SPeter Grehan 51366f6083SPeter Grehan #include <machine/vmm.h> 52366f6083SPeter Grehan #include "vmm_lapic.h" 53366f6083SPeter Grehan #include "vmm_msr.h" 54366f6083SPeter Grehan #include "vmm_ktr.h" 55366f6083SPeter Grehan #include "vmm_stat.h" 56366f6083SPeter Grehan 57366f6083SPeter Grehan #include "vmx_msr.h" 58366f6083SPeter Grehan #include "ept.h" 59366f6083SPeter Grehan #include "vmx_cpufunc.h" 60366f6083SPeter Grehan #include "vmx.h" 61366f6083SPeter Grehan #include "x86.h" 62366f6083SPeter Grehan #include "vmx_controls.h" 63366f6083SPeter Grehan 64366f6083SPeter Grehan #define CR4_VMXE (1UL << 13) 65366f6083SPeter Grehan 66366f6083SPeter Grehan #define PINBASED_CTLS_ONE_SETTING \ 67366f6083SPeter Grehan (PINBASED_EXTINT_EXITING | \ 68366f6083SPeter Grehan PINBASED_NMI_EXITING | \ 69366f6083SPeter Grehan PINBASED_VIRTUAL_NMI) 70366f6083SPeter Grehan #define PINBASED_CTLS_ZERO_SETTING 0 71366f6083SPeter Grehan 72366f6083SPeter Grehan #define PROCBASED_CTLS_WINDOW_SETTING \ 73366f6083SPeter Grehan (PROCBASED_INT_WINDOW_EXITING | \ 74366f6083SPeter Grehan PROCBASED_NMI_WINDOW_EXITING) 75366f6083SPeter Grehan 76366f6083SPeter Grehan #define PROCBASED_CTLS_ONE_SETTING \ 77366f6083SPeter Grehan (PROCBASED_SECONDARY_CONTROLS | \ 78366f6083SPeter Grehan PROCBASED_IO_EXITING | \ 79366f6083SPeter Grehan PROCBASED_MSR_BITMAPS | \ 80366f6083SPeter Grehan PROCBASED_CTLS_WINDOW_SETTING) 81366f6083SPeter Grehan #define PROCBASED_CTLS_ZERO_SETTING \ 82366f6083SPeter Grehan (PROCBASED_CR3_LOAD_EXITING | \ 83366f6083SPeter Grehan PROCBASED_CR3_STORE_EXITING | \ 84366f6083SPeter Grehan PROCBASED_IO_BITMAPS) 85366f6083SPeter Grehan 86366f6083SPeter Grehan #define PROCBASED_CTLS2_ONE_SETTING PROCBASED2_ENABLE_EPT 87366f6083SPeter Grehan #define PROCBASED_CTLS2_ZERO_SETTING 0 88366f6083SPeter Grehan 89608f97c3SPeter Grehan #define VM_EXIT_CTLS_ONE_SETTING_NO_PAT \ 90366f6083SPeter Grehan (VM_EXIT_HOST_LMA | \ 91366f6083SPeter Grehan VM_EXIT_SAVE_EFER | \ 92366f6083SPeter Grehan VM_EXIT_LOAD_EFER) 93608f97c3SPeter Grehan 94608f97c3SPeter Grehan #define VM_EXIT_CTLS_ONE_SETTING \ 95608f97c3SPeter Grehan (VM_EXIT_CTLS_ONE_SETTING_NO_PAT | \ 96608f97c3SPeter Grehan VM_EXIT_SAVE_PAT | \ 97608f97c3SPeter Grehan VM_EXIT_LOAD_PAT) 98366f6083SPeter Grehan #define VM_EXIT_CTLS_ZERO_SETTING VM_EXIT_SAVE_DEBUG_CONTROLS 99366f6083SPeter Grehan 100608f97c3SPeter Grehan #define VM_ENTRY_CTLS_ONE_SETTING_NO_PAT VM_ENTRY_LOAD_EFER 101608f97c3SPeter Grehan 102366f6083SPeter Grehan #define VM_ENTRY_CTLS_ONE_SETTING \ 103608f97c3SPeter Grehan (VM_ENTRY_CTLS_ONE_SETTING_NO_PAT | \ 104608f97c3SPeter Grehan VM_ENTRY_LOAD_PAT) 105366f6083SPeter Grehan #define VM_ENTRY_CTLS_ZERO_SETTING \ 106366f6083SPeter Grehan (VM_ENTRY_LOAD_DEBUG_CONTROLS | \ 107366f6083SPeter Grehan VM_ENTRY_INTO_SMM | \ 108366f6083SPeter Grehan VM_ENTRY_DEACTIVATE_DUAL_MONITOR) 109366f6083SPeter Grehan 110366f6083SPeter Grehan #define guest_msr_rw(vmx, msr) \ 111366f6083SPeter Grehan msr_bitmap_change_access((vmx)->msr_bitmap, (msr), MSR_BITMAP_ACCESS_RW) 112366f6083SPeter Grehan 113366f6083SPeter Grehan #define HANDLED 1 114366f6083SPeter Grehan #define UNHANDLED 0 115366f6083SPeter Grehan 116366f6083SPeter Grehan MALLOC_DEFINE(M_VMX, "vmx", "vmx"); 117366f6083SPeter Grehan 118366f6083SPeter Grehan extern struct pcpu __pcpu[]; 119366f6083SPeter Grehan 120b3996dd4SJohn Baldwin int vmxon_enabled[MAXCPU]; 121366f6083SPeter Grehan static char vmxon_region[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE); 122366f6083SPeter Grehan 123366f6083SPeter Grehan static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2; 124366f6083SPeter Grehan static uint32_t exit_ctls, entry_ctls; 125366f6083SPeter Grehan 126366f6083SPeter Grehan static uint64_t cr0_ones_mask, cr0_zeros_mask; 127366f6083SPeter Grehan static uint64_t cr4_ones_mask, cr4_zeros_mask; 128366f6083SPeter Grehan 129366f6083SPeter Grehan static volatile u_int nextvpid; 130366f6083SPeter Grehan 131608f97c3SPeter Grehan static int vmx_no_patmsr; 132608f97c3SPeter Grehan 133366f6083SPeter Grehan /* 134366f6083SPeter Grehan * Virtual NMI blocking conditions. 135366f6083SPeter Grehan * 136366f6083SPeter Grehan * Some processor implementations also require NMI to be blocked if 137366f6083SPeter Grehan * the STI_BLOCKING bit is set. It is possible to detect this at runtime 138366f6083SPeter Grehan * based on the (exit_reason,exit_qual) tuple being set to 139366f6083SPeter Grehan * (EXIT_REASON_INVAL_VMCS, EXIT_QUAL_NMI_WHILE_STI_BLOCKING). 140366f6083SPeter Grehan * 141366f6083SPeter Grehan * We take the easy way out and also include STI_BLOCKING as one of the 142366f6083SPeter Grehan * gating items for vNMI injection. 143366f6083SPeter Grehan */ 144366f6083SPeter Grehan static uint64_t nmi_blocking_bits = VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING | 145366f6083SPeter Grehan VMCS_INTERRUPTIBILITY_NMI_BLOCKING | 146366f6083SPeter Grehan VMCS_INTERRUPTIBILITY_STI_BLOCKING; 147366f6083SPeter Grehan 148366f6083SPeter Grehan /* 149366f6083SPeter Grehan * Optional capabilities 150366f6083SPeter Grehan */ 151366f6083SPeter Grehan static int cap_halt_exit; 152366f6083SPeter Grehan static int cap_pause_exit; 153366f6083SPeter Grehan static int cap_unrestricted_guest; 154366f6083SPeter Grehan static int cap_monitor_trap; 155366f6083SPeter Grehan 156366f6083SPeter Grehan /* statistics */ 157366f6083SPeter Grehan static VMM_STAT_DEFINE(VCPU_MIGRATIONS, "vcpu migration across host cpus"); 158366f6083SPeter Grehan static VMM_STAT_DEFINE(VMEXIT_EXTINT, "vm exits due to external interrupt"); 159366f6083SPeter Grehan 160366f6083SPeter Grehan #ifdef KTR 161366f6083SPeter Grehan static const char * 162366f6083SPeter Grehan exit_reason_to_str(int reason) 163366f6083SPeter Grehan { 164366f6083SPeter Grehan static char reasonbuf[32]; 165366f6083SPeter Grehan 166366f6083SPeter Grehan switch (reason) { 167366f6083SPeter Grehan case EXIT_REASON_EXCEPTION: 168366f6083SPeter Grehan return "exception"; 169366f6083SPeter Grehan case EXIT_REASON_EXT_INTR: 170366f6083SPeter Grehan return "extint"; 171366f6083SPeter Grehan case EXIT_REASON_TRIPLE_FAULT: 172366f6083SPeter Grehan return "triplefault"; 173366f6083SPeter Grehan case EXIT_REASON_INIT: 174366f6083SPeter Grehan return "init"; 175366f6083SPeter Grehan case EXIT_REASON_SIPI: 176366f6083SPeter Grehan return "sipi"; 177366f6083SPeter Grehan case EXIT_REASON_IO_SMI: 178366f6083SPeter Grehan return "iosmi"; 179366f6083SPeter Grehan case EXIT_REASON_SMI: 180366f6083SPeter Grehan return "smi"; 181366f6083SPeter Grehan case EXIT_REASON_INTR_WINDOW: 182366f6083SPeter Grehan return "intrwindow"; 183366f6083SPeter Grehan case EXIT_REASON_NMI_WINDOW: 184366f6083SPeter Grehan return "nmiwindow"; 185366f6083SPeter Grehan case EXIT_REASON_TASK_SWITCH: 186366f6083SPeter Grehan return "taskswitch"; 187366f6083SPeter Grehan case EXIT_REASON_CPUID: 188366f6083SPeter Grehan return "cpuid"; 189366f6083SPeter Grehan case EXIT_REASON_GETSEC: 190366f6083SPeter Grehan return "getsec"; 191366f6083SPeter Grehan case EXIT_REASON_HLT: 192366f6083SPeter Grehan return "hlt"; 193366f6083SPeter Grehan case EXIT_REASON_INVD: 194366f6083SPeter Grehan return "invd"; 195366f6083SPeter Grehan case EXIT_REASON_INVLPG: 196366f6083SPeter Grehan return "invlpg"; 197366f6083SPeter Grehan case EXIT_REASON_RDPMC: 198366f6083SPeter Grehan return "rdpmc"; 199366f6083SPeter Grehan case EXIT_REASON_RDTSC: 200366f6083SPeter Grehan return "rdtsc"; 201366f6083SPeter Grehan case EXIT_REASON_RSM: 202366f6083SPeter Grehan return "rsm"; 203366f6083SPeter Grehan case EXIT_REASON_VMCALL: 204366f6083SPeter Grehan return "vmcall"; 205366f6083SPeter Grehan case EXIT_REASON_VMCLEAR: 206366f6083SPeter Grehan return "vmclear"; 207366f6083SPeter Grehan case EXIT_REASON_VMLAUNCH: 208366f6083SPeter Grehan return "vmlaunch"; 209366f6083SPeter Grehan case EXIT_REASON_VMPTRLD: 210366f6083SPeter Grehan return "vmptrld"; 211366f6083SPeter Grehan case EXIT_REASON_VMPTRST: 212366f6083SPeter Grehan return "vmptrst"; 213366f6083SPeter Grehan case EXIT_REASON_VMREAD: 214366f6083SPeter Grehan return "vmread"; 215366f6083SPeter Grehan case EXIT_REASON_VMRESUME: 216366f6083SPeter Grehan return "vmresume"; 217366f6083SPeter Grehan case EXIT_REASON_VMWRITE: 218366f6083SPeter Grehan return "vmwrite"; 219366f6083SPeter Grehan case EXIT_REASON_VMXOFF: 220366f6083SPeter Grehan return "vmxoff"; 221366f6083SPeter Grehan case EXIT_REASON_VMXON: 222366f6083SPeter Grehan return "vmxon"; 223366f6083SPeter Grehan case EXIT_REASON_CR_ACCESS: 224366f6083SPeter Grehan return "craccess"; 225366f6083SPeter Grehan case EXIT_REASON_DR_ACCESS: 226366f6083SPeter Grehan return "draccess"; 227366f6083SPeter Grehan case EXIT_REASON_INOUT: 228366f6083SPeter Grehan return "inout"; 229366f6083SPeter Grehan case EXIT_REASON_RDMSR: 230366f6083SPeter Grehan return "rdmsr"; 231366f6083SPeter Grehan case EXIT_REASON_WRMSR: 232366f6083SPeter Grehan return "wrmsr"; 233366f6083SPeter Grehan case EXIT_REASON_INVAL_VMCS: 234366f6083SPeter Grehan return "invalvmcs"; 235366f6083SPeter Grehan case EXIT_REASON_INVAL_MSR: 236366f6083SPeter Grehan return "invalmsr"; 237366f6083SPeter Grehan case EXIT_REASON_MWAIT: 238366f6083SPeter Grehan return "mwait"; 239366f6083SPeter Grehan case EXIT_REASON_MTF: 240366f6083SPeter Grehan return "mtf"; 241366f6083SPeter Grehan case EXIT_REASON_MONITOR: 242366f6083SPeter Grehan return "monitor"; 243366f6083SPeter Grehan case EXIT_REASON_PAUSE: 244366f6083SPeter Grehan return "pause"; 245366f6083SPeter Grehan case EXIT_REASON_MCE: 246366f6083SPeter Grehan return "mce"; 247366f6083SPeter Grehan case EXIT_REASON_TPR: 248366f6083SPeter Grehan return "tpr"; 249366f6083SPeter Grehan case EXIT_REASON_APIC: 250366f6083SPeter Grehan return "apic"; 251366f6083SPeter Grehan case EXIT_REASON_GDTR_IDTR: 252366f6083SPeter Grehan return "gdtridtr"; 253366f6083SPeter Grehan case EXIT_REASON_LDTR_TR: 254366f6083SPeter Grehan return "ldtrtr"; 255366f6083SPeter Grehan case EXIT_REASON_EPT_FAULT: 256366f6083SPeter Grehan return "eptfault"; 257366f6083SPeter Grehan case EXIT_REASON_EPT_MISCONFIG: 258366f6083SPeter Grehan return "eptmisconfig"; 259366f6083SPeter Grehan case EXIT_REASON_INVEPT: 260366f6083SPeter Grehan return "invept"; 261366f6083SPeter Grehan case EXIT_REASON_RDTSCP: 262366f6083SPeter Grehan return "rdtscp"; 263366f6083SPeter Grehan case EXIT_REASON_VMX_PREEMPT: 264366f6083SPeter Grehan return "vmxpreempt"; 265366f6083SPeter Grehan case EXIT_REASON_INVVPID: 266366f6083SPeter Grehan return "invvpid"; 267366f6083SPeter Grehan case EXIT_REASON_WBINVD: 268366f6083SPeter Grehan return "wbinvd"; 269366f6083SPeter Grehan case EXIT_REASON_XSETBV: 270366f6083SPeter Grehan return "xsetbv"; 271366f6083SPeter Grehan default: 272366f6083SPeter Grehan snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason); 273366f6083SPeter Grehan return (reasonbuf); 274366f6083SPeter Grehan } 275366f6083SPeter Grehan } 276366f6083SPeter Grehan 277366f6083SPeter Grehan #ifdef SETJMP_TRACE 278366f6083SPeter Grehan static const char * 279366f6083SPeter Grehan vmx_setjmp_rc2str(int rc) 280366f6083SPeter Grehan { 281366f6083SPeter Grehan switch (rc) { 282366f6083SPeter Grehan case VMX_RETURN_DIRECT: 283366f6083SPeter Grehan return "direct"; 284366f6083SPeter Grehan case VMX_RETURN_LONGJMP: 285366f6083SPeter Grehan return "longjmp"; 286366f6083SPeter Grehan case VMX_RETURN_VMRESUME: 287366f6083SPeter Grehan return "vmresume"; 288366f6083SPeter Grehan case VMX_RETURN_VMLAUNCH: 289366f6083SPeter Grehan return "vmlaunch"; 290366f6083SPeter Grehan default: 291366f6083SPeter Grehan return "unknown"; 292366f6083SPeter Grehan } 293366f6083SPeter Grehan } 294366f6083SPeter Grehan 295366f6083SPeter Grehan #define SETJMP_TRACE(vmx, vcpu, vmxctx, regname) \ 296366f6083SPeter Grehan VMM_CTR1((vmx)->vm, (vcpu), "setjmp trace " #regname " 0x%016lx", \ 297366f6083SPeter Grehan (vmxctx)->regname) 298366f6083SPeter Grehan 299366f6083SPeter Grehan static void 300366f6083SPeter Grehan vmx_setjmp_trace(struct vmx *vmx, int vcpu, struct vmxctx *vmxctx, int rc) 301366f6083SPeter Grehan { 302366f6083SPeter Grehan uint64_t host_rip, host_rsp; 303366f6083SPeter Grehan 304366f6083SPeter Grehan if (vmxctx != &vmx->ctx[vcpu]) 305366f6083SPeter Grehan panic("vmx_setjmp_trace: invalid vmxctx %p; should be %p", 306366f6083SPeter Grehan vmxctx, &vmx->ctx[vcpu]); 307366f6083SPeter Grehan 308366f6083SPeter Grehan VMM_CTR1((vmx)->vm, (vcpu), "vmxctx = %p", vmxctx); 309366f6083SPeter Grehan VMM_CTR2((vmx)->vm, (vcpu), "setjmp return code %s(%d)", 310366f6083SPeter Grehan vmx_setjmp_rc2str(rc), rc); 311366f6083SPeter Grehan 312366f6083SPeter Grehan host_rsp = host_rip = ~0; 313366f6083SPeter Grehan vmread(VMCS_HOST_RIP, &host_rip); 314366f6083SPeter Grehan vmread(VMCS_HOST_RSP, &host_rsp); 315366f6083SPeter Grehan VMM_CTR2((vmx)->vm, (vcpu), "vmcs host_rip 0x%016lx, host_rsp 0x%016lx", 316366f6083SPeter Grehan host_rip, host_rsp); 317366f6083SPeter Grehan 318366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, host_r15); 319366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, host_r14); 320366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, host_r13); 321366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, host_r12); 322366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, host_rbp); 323366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, host_rsp); 324366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, host_rbx); 325366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, host_rip); 326366f6083SPeter Grehan 327366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rdi); 328366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rsi); 329366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rdx); 330366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rcx); 331366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r8); 332366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r9); 333366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rax); 334366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rbx); 335366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rbp); 336366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r10); 337366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r11); 338366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r12); 339366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r13); 340366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r14); 341366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r15); 342366f6083SPeter Grehan SETJMP_TRACE(vmx, vcpu, vmxctx, guest_cr2); 343366f6083SPeter Grehan } 344366f6083SPeter Grehan #endif 345366f6083SPeter Grehan #else 346366f6083SPeter Grehan static void __inline 347366f6083SPeter Grehan vmx_setjmp_trace(struct vmx *vmx, int vcpu, struct vmxctx *vmxctx, int rc) 348366f6083SPeter Grehan { 349366f6083SPeter Grehan return; 350366f6083SPeter Grehan } 351366f6083SPeter Grehan #endif /* KTR */ 352366f6083SPeter Grehan 353366f6083SPeter Grehan u_long 354366f6083SPeter Grehan vmx_fix_cr0(u_long cr0) 355366f6083SPeter Grehan { 356366f6083SPeter Grehan 357366f6083SPeter Grehan return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask); 358366f6083SPeter Grehan } 359366f6083SPeter Grehan 360366f6083SPeter Grehan u_long 361366f6083SPeter Grehan vmx_fix_cr4(u_long cr4) 362366f6083SPeter Grehan { 363366f6083SPeter Grehan 364366f6083SPeter Grehan return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask); 365366f6083SPeter Grehan } 366366f6083SPeter Grehan 367366f6083SPeter Grehan static void 368366f6083SPeter Grehan msr_save_area_init(struct msr_entry *g_area, int *g_count) 369366f6083SPeter Grehan { 370366f6083SPeter Grehan int cnt; 371366f6083SPeter Grehan 372366f6083SPeter Grehan static struct msr_entry guest_msrs[] = { 373366f6083SPeter Grehan { MSR_KGSBASE, 0, 0 }, 374366f6083SPeter Grehan }; 375366f6083SPeter Grehan 376366f6083SPeter Grehan cnt = sizeof(guest_msrs) / sizeof(guest_msrs[0]); 377366f6083SPeter Grehan if (cnt > GUEST_MSR_MAX_ENTRIES) 378366f6083SPeter Grehan panic("guest msr save area overrun"); 379366f6083SPeter Grehan bcopy(guest_msrs, g_area, sizeof(guest_msrs)); 380366f6083SPeter Grehan *g_count = cnt; 381366f6083SPeter Grehan } 382366f6083SPeter Grehan 383366f6083SPeter Grehan static void 384366f6083SPeter Grehan vmx_disable(void *arg __unused) 385366f6083SPeter Grehan { 386366f6083SPeter Grehan struct invvpid_desc invvpid_desc = { 0 }; 387366f6083SPeter Grehan struct invept_desc invept_desc = { 0 }; 388366f6083SPeter Grehan 389366f6083SPeter Grehan if (vmxon_enabled[curcpu]) { 390366f6083SPeter Grehan /* 391366f6083SPeter Grehan * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b. 392366f6083SPeter Grehan * 393366f6083SPeter Grehan * VMXON or VMXOFF are not required to invalidate any TLB 394366f6083SPeter Grehan * caching structures. This prevents potential retention of 395366f6083SPeter Grehan * cached information in the TLB between distinct VMX episodes. 396366f6083SPeter Grehan */ 397366f6083SPeter Grehan invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc); 398366f6083SPeter Grehan invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc); 399366f6083SPeter Grehan vmxoff(); 400366f6083SPeter Grehan } 401366f6083SPeter Grehan load_cr4(rcr4() & ~CR4_VMXE); 402366f6083SPeter Grehan } 403366f6083SPeter Grehan 404366f6083SPeter Grehan static int 405366f6083SPeter Grehan vmx_cleanup(void) 406366f6083SPeter Grehan { 407366f6083SPeter Grehan 408366f6083SPeter Grehan smp_rendezvous(NULL, vmx_disable, NULL, NULL); 409366f6083SPeter Grehan 410366f6083SPeter Grehan return (0); 411366f6083SPeter Grehan } 412366f6083SPeter Grehan 413366f6083SPeter Grehan static void 414366f6083SPeter Grehan vmx_enable(void *arg __unused) 415366f6083SPeter Grehan { 416366f6083SPeter Grehan int error; 417366f6083SPeter Grehan 418366f6083SPeter Grehan load_cr4(rcr4() | CR4_VMXE); 419366f6083SPeter Grehan 420366f6083SPeter Grehan *(uint32_t *)vmxon_region[curcpu] = vmx_revision(); 421366f6083SPeter Grehan error = vmxon(vmxon_region[curcpu]); 422366f6083SPeter Grehan if (error == 0) 423366f6083SPeter Grehan vmxon_enabled[curcpu] = 1; 424366f6083SPeter Grehan } 425366f6083SPeter Grehan 426366f6083SPeter Grehan static int 427366f6083SPeter Grehan vmx_init(void) 428366f6083SPeter Grehan { 429366f6083SPeter Grehan int error; 4304bff7fadSNeel Natu uint64_t fixed0, fixed1, feature_control; 431366f6083SPeter Grehan uint32_t tmp; 432366f6083SPeter Grehan 433366f6083SPeter Grehan /* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */ 4348b287612SJohn Baldwin if (!(cpu_feature2 & CPUID2_VMX)) { 435366f6083SPeter Grehan printf("vmx_init: processor does not support VMX operation\n"); 436366f6083SPeter Grehan return (ENXIO); 437366f6083SPeter Grehan } 438366f6083SPeter Grehan 4394bff7fadSNeel Natu /* 4404bff7fadSNeel Natu * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits 4414bff7fadSNeel Natu * are set (bits 0 and 2 respectively). 4424bff7fadSNeel Natu */ 4434bff7fadSNeel Natu feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL); 4444bff7fadSNeel Natu if ((feature_control & 0x5) != 0x5) { 4454bff7fadSNeel Natu printf("vmx_init: VMX operation disabled by BIOS\n"); 4464bff7fadSNeel Natu return (ENXIO); 4474bff7fadSNeel Natu } 4484bff7fadSNeel Natu 449366f6083SPeter Grehan /* Check support for primary processor-based VM-execution controls */ 450366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 451366f6083SPeter Grehan MSR_VMX_TRUE_PROCBASED_CTLS, 452366f6083SPeter Grehan PROCBASED_CTLS_ONE_SETTING, 453366f6083SPeter Grehan PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls); 454366f6083SPeter Grehan if (error) { 455366f6083SPeter Grehan printf("vmx_init: processor does not support desired primary " 456366f6083SPeter Grehan "processor-based controls\n"); 457366f6083SPeter Grehan return (error); 458366f6083SPeter Grehan } 459366f6083SPeter Grehan 460366f6083SPeter Grehan /* Clear the processor-based ctl bits that are set on demand */ 461366f6083SPeter Grehan procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING; 462366f6083SPeter Grehan 463366f6083SPeter Grehan /* Check support for secondary processor-based VM-execution controls */ 464366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 465366f6083SPeter Grehan MSR_VMX_PROCBASED_CTLS2, 466366f6083SPeter Grehan PROCBASED_CTLS2_ONE_SETTING, 467366f6083SPeter Grehan PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2); 468366f6083SPeter Grehan if (error) { 469366f6083SPeter Grehan printf("vmx_init: processor does not support desired secondary " 470366f6083SPeter Grehan "processor-based controls\n"); 471366f6083SPeter Grehan return (error); 472366f6083SPeter Grehan } 473366f6083SPeter Grehan 474366f6083SPeter Grehan /* Check support for VPID */ 475366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2, 476366f6083SPeter Grehan PROCBASED2_ENABLE_VPID, 0, &tmp); 477366f6083SPeter Grehan if (error == 0) 478366f6083SPeter Grehan procbased_ctls2 |= PROCBASED2_ENABLE_VPID; 479366f6083SPeter Grehan 480366f6083SPeter Grehan /* Check support for pin-based VM-execution controls */ 481366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS, 482366f6083SPeter Grehan MSR_VMX_TRUE_PINBASED_CTLS, 483366f6083SPeter Grehan PINBASED_CTLS_ONE_SETTING, 484366f6083SPeter Grehan PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls); 485366f6083SPeter Grehan if (error) { 486366f6083SPeter Grehan printf("vmx_init: processor does not support desired " 487366f6083SPeter Grehan "pin-based controls\n"); 488366f6083SPeter Grehan return (error); 489366f6083SPeter Grehan } 490366f6083SPeter Grehan 491366f6083SPeter Grehan /* Check support for VM-exit controls */ 492366f6083SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS, 493366f6083SPeter Grehan VM_EXIT_CTLS_ONE_SETTING, 494366f6083SPeter Grehan VM_EXIT_CTLS_ZERO_SETTING, 495366f6083SPeter Grehan &exit_ctls); 496366f6083SPeter Grehan if (error) { 497608f97c3SPeter Grehan /* Try again without the PAT MSR bits */ 498608f97c3SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, 499608f97c3SPeter Grehan MSR_VMX_TRUE_EXIT_CTLS, 500608f97c3SPeter Grehan VM_EXIT_CTLS_ONE_SETTING_NO_PAT, 501608f97c3SPeter Grehan VM_EXIT_CTLS_ZERO_SETTING, 502608f97c3SPeter Grehan &exit_ctls); 503608f97c3SPeter Grehan if (error) { 504366f6083SPeter Grehan printf("vmx_init: processor does not support desired " 505366f6083SPeter Grehan "exit controls\n"); 506366f6083SPeter Grehan return (error); 507608f97c3SPeter Grehan } else { 508608f97c3SPeter Grehan if (bootverbose) 509608f97c3SPeter Grehan printf("vmm: PAT MSR access not supported\n"); 510608f97c3SPeter Grehan guest_msr_valid(MSR_PAT); 511608f97c3SPeter Grehan vmx_no_patmsr = 1; 512608f97c3SPeter Grehan } 513366f6083SPeter Grehan } 514366f6083SPeter Grehan 515366f6083SPeter Grehan /* Check support for VM-entry controls */ 516608f97c3SPeter Grehan if (!vmx_no_patmsr) { 517608f97c3SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS, 518608f97c3SPeter Grehan MSR_VMX_TRUE_ENTRY_CTLS, 519366f6083SPeter Grehan VM_ENTRY_CTLS_ONE_SETTING, 520366f6083SPeter Grehan VM_ENTRY_CTLS_ZERO_SETTING, 521366f6083SPeter Grehan &entry_ctls); 522608f97c3SPeter Grehan } else { 523608f97c3SPeter Grehan error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS, 524608f97c3SPeter Grehan MSR_VMX_TRUE_ENTRY_CTLS, 525608f97c3SPeter Grehan VM_ENTRY_CTLS_ONE_SETTING_NO_PAT, 526608f97c3SPeter Grehan VM_ENTRY_CTLS_ZERO_SETTING, 527608f97c3SPeter Grehan &entry_ctls); 528608f97c3SPeter Grehan } 529608f97c3SPeter Grehan 530366f6083SPeter Grehan if (error) { 531366f6083SPeter Grehan printf("vmx_init: processor does not support desired " 532366f6083SPeter Grehan "entry controls\n"); 533366f6083SPeter Grehan return (error); 534366f6083SPeter Grehan } 535366f6083SPeter Grehan 536366f6083SPeter Grehan /* 537366f6083SPeter Grehan * Check support for optional features by testing them 538366f6083SPeter Grehan * as individual bits 539366f6083SPeter Grehan */ 540366f6083SPeter Grehan cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 541366f6083SPeter Grehan MSR_VMX_TRUE_PROCBASED_CTLS, 542366f6083SPeter Grehan PROCBASED_HLT_EXITING, 0, 543366f6083SPeter Grehan &tmp) == 0); 544366f6083SPeter Grehan 545366f6083SPeter Grehan cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 546366f6083SPeter Grehan MSR_VMX_PROCBASED_CTLS, 547366f6083SPeter Grehan PROCBASED_MTF, 0, 548366f6083SPeter Grehan &tmp) == 0); 549366f6083SPeter Grehan 550366f6083SPeter Grehan cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 551366f6083SPeter Grehan MSR_VMX_TRUE_PROCBASED_CTLS, 552366f6083SPeter Grehan PROCBASED_PAUSE_EXITING, 0, 553366f6083SPeter Grehan &tmp) == 0); 554366f6083SPeter Grehan 555366f6083SPeter Grehan cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 556366f6083SPeter Grehan MSR_VMX_PROCBASED_CTLS2, 557366f6083SPeter Grehan PROCBASED2_UNRESTRICTED_GUEST, 0, 558366f6083SPeter Grehan &tmp) == 0); 559366f6083SPeter Grehan 560366f6083SPeter Grehan /* Initialize EPT */ 561366f6083SPeter Grehan error = ept_init(); 562366f6083SPeter Grehan if (error) { 563366f6083SPeter Grehan printf("vmx_init: ept initialization failed (%d)\n", error); 564366f6083SPeter Grehan return (error); 565366f6083SPeter Grehan } 566366f6083SPeter Grehan 567366f6083SPeter Grehan /* 568366f6083SPeter Grehan * Stash the cr0 and cr4 bits that must be fixed to 0 or 1 569366f6083SPeter Grehan */ 570366f6083SPeter Grehan fixed0 = rdmsr(MSR_VMX_CR0_FIXED0); 571366f6083SPeter Grehan fixed1 = rdmsr(MSR_VMX_CR0_FIXED1); 572366f6083SPeter Grehan cr0_ones_mask = fixed0 & fixed1; 573366f6083SPeter Grehan cr0_zeros_mask = ~fixed0 & ~fixed1; 574366f6083SPeter Grehan 575366f6083SPeter Grehan /* 576366f6083SPeter Grehan * CR0_PE and CR0_PG can be set to zero in VMX non-root operation 577366f6083SPeter Grehan * if unrestricted guest execution is allowed. 578366f6083SPeter Grehan */ 579366f6083SPeter Grehan if (cap_unrestricted_guest) 580366f6083SPeter Grehan cr0_ones_mask &= ~(CR0_PG | CR0_PE); 581366f6083SPeter Grehan 582366f6083SPeter Grehan /* 583366f6083SPeter Grehan * Do not allow the guest to set CR0_NW or CR0_CD. 584366f6083SPeter Grehan */ 585366f6083SPeter Grehan cr0_zeros_mask |= (CR0_NW | CR0_CD); 586366f6083SPeter Grehan 587366f6083SPeter Grehan fixed0 = rdmsr(MSR_VMX_CR4_FIXED0); 588366f6083SPeter Grehan fixed1 = rdmsr(MSR_VMX_CR4_FIXED1); 589366f6083SPeter Grehan cr4_ones_mask = fixed0 & fixed1; 590366f6083SPeter Grehan cr4_zeros_mask = ~fixed0 & ~fixed1; 591366f6083SPeter Grehan 592366f6083SPeter Grehan /* enable VMX operation */ 593366f6083SPeter Grehan smp_rendezvous(NULL, vmx_enable, NULL, NULL); 594366f6083SPeter Grehan 595366f6083SPeter Grehan return (0); 596366f6083SPeter Grehan } 597366f6083SPeter Grehan 598366f6083SPeter Grehan /* 599366f6083SPeter Grehan * If this processor does not support VPIDs then simply return 0. 600366f6083SPeter Grehan * 601366f6083SPeter Grehan * Otherwise generate the next value of VPID to use. Any value is alright 602366f6083SPeter Grehan * as long as it is non-zero. 603366f6083SPeter Grehan * 604366f6083SPeter Grehan * We always execute in VMX non-root context with EPT enabled. Thus all 605366f6083SPeter Grehan * combined mappings are tagged with the (EP4TA, VPID, PCID) tuple. This 606366f6083SPeter Grehan * in turn means that multiple VMs can share the same VPID as long as 607366f6083SPeter Grehan * they have distinct EPT page tables. 608366f6083SPeter Grehan * 609366f6083SPeter Grehan * XXX 610366f6083SPeter Grehan * We should optimize this so that it returns VPIDs that are not in 611366f6083SPeter Grehan * use. Then we will not unnecessarily invalidate mappings in 612366f6083SPeter Grehan * vmx_set_pcpu_defaults() just because two or more vcpus happen to 613366f6083SPeter Grehan * use the same 'vpid'. 614366f6083SPeter Grehan */ 615366f6083SPeter Grehan static uint16_t 616366f6083SPeter Grehan vmx_vpid(void) 617366f6083SPeter Grehan { 618366f6083SPeter Grehan uint16_t vpid = 0; 619366f6083SPeter Grehan 620366f6083SPeter Grehan if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) != 0) { 621366f6083SPeter Grehan do { 622366f6083SPeter Grehan vpid = atomic_fetchadd_int(&nextvpid, 1); 623366f6083SPeter Grehan } while (vpid == 0); 624366f6083SPeter Grehan } 625366f6083SPeter Grehan 626366f6083SPeter Grehan return (vpid); 627366f6083SPeter Grehan } 628366f6083SPeter Grehan 629366f6083SPeter Grehan static int 630*39c21c2dSNeel Natu vmx_setup_cr_shadow(int which, struct vmcs *vmcs) 631366f6083SPeter Grehan { 632*39c21c2dSNeel Natu int error, mask_ident, shadow_ident; 633*39c21c2dSNeel Natu uint64_t mask_value, shadow_value; 634366f6083SPeter Grehan 635*39c21c2dSNeel Natu if (which != 0 && which != 4) 636*39c21c2dSNeel Natu panic("vmx_setup_cr_shadow: unknown cr%d", which); 637*39c21c2dSNeel Natu 638*39c21c2dSNeel Natu if (which == 0) { 639*39c21c2dSNeel Natu mask_ident = VMCS_CR0_MASK; 640*39c21c2dSNeel Natu mask_value = cr0_ones_mask | cr0_zeros_mask; 641*39c21c2dSNeel Natu shadow_ident = VMCS_CR0_SHADOW; 642*39c21c2dSNeel Natu shadow_value = cr0_ones_mask; 643*39c21c2dSNeel Natu } else { 644*39c21c2dSNeel Natu mask_ident = VMCS_CR4_MASK; 645*39c21c2dSNeel Natu mask_value = cr4_ones_mask | cr4_zeros_mask; 646*39c21c2dSNeel Natu shadow_ident = VMCS_CR4_SHADOW; 647*39c21c2dSNeel Natu shadow_value = cr4_ones_mask; 648*39c21c2dSNeel Natu } 649*39c21c2dSNeel Natu 650*39c21c2dSNeel Natu error = vmcs_setreg(vmcs, VMCS_IDENT(mask_ident), mask_value); 651366f6083SPeter Grehan if (error) 652366f6083SPeter Grehan return (error); 653366f6083SPeter Grehan 654*39c21c2dSNeel Natu error = vmcs_setreg(vmcs, VMCS_IDENT(shadow_ident), shadow_value); 655366f6083SPeter Grehan if (error) 656366f6083SPeter Grehan return (error); 657366f6083SPeter Grehan 658366f6083SPeter Grehan return (0); 659366f6083SPeter Grehan } 660*39c21c2dSNeel Natu #define vmx_setup_cr0_shadow(vmcs) vmx_setup_cr_shadow(0, (vmcs)) 661*39c21c2dSNeel Natu #define vmx_setup_cr4_shadow(vmcs) vmx_setup_cr_shadow(4, (vmcs)) 662366f6083SPeter Grehan 663366f6083SPeter Grehan static void * 664366f6083SPeter Grehan vmx_vminit(struct vm *vm) 665366f6083SPeter Grehan { 666366f6083SPeter Grehan uint16_t vpid; 667366f6083SPeter Grehan int i, error, guest_msr_count; 668366f6083SPeter Grehan struct vmx *vmx; 669366f6083SPeter Grehan 670366f6083SPeter Grehan vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO); 671366f6083SPeter Grehan if ((uintptr_t)vmx & PAGE_MASK) { 672366f6083SPeter Grehan panic("malloc of struct vmx not aligned on %d byte boundary", 673366f6083SPeter Grehan PAGE_SIZE); 674366f6083SPeter Grehan } 675366f6083SPeter Grehan vmx->vm = vm; 676366f6083SPeter Grehan 677366f6083SPeter Grehan /* 678366f6083SPeter Grehan * Clean up EPTP-tagged guest physical and combined mappings 679366f6083SPeter Grehan * 680366f6083SPeter Grehan * VMX transitions are not required to invalidate any guest physical 681366f6083SPeter Grehan * mappings. So, it may be possible for stale guest physical mappings 682366f6083SPeter Grehan * to be present in the processor TLBs. 683366f6083SPeter Grehan * 684366f6083SPeter Grehan * Combined mappings for this EP4TA are also invalidated for all VPIDs. 685366f6083SPeter Grehan */ 686366f6083SPeter Grehan ept_invalidate_mappings(vtophys(vmx->pml4ept)); 687366f6083SPeter Grehan 688366f6083SPeter Grehan msr_bitmap_initialize(vmx->msr_bitmap); 689366f6083SPeter Grehan 690366f6083SPeter Grehan /* 691366f6083SPeter Grehan * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE. 692366f6083SPeter Grehan * The guest FSBASE and GSBASE are saved and restored during 693366f6083SPeter Grehan * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are 694366f6083SPeter Grehan * always restored from the vmcs host state area on vm-exit. 695366f6083SPeter Grehan * 696366f6083SPeter Grehan * Guest KGSBASE is saved and restored in the guest MSR save area. 697366f6083SPeter Grehan * Host KGSBASE is restored before returning to userland from the pcb. 698366f6083SPeter Grehan * There will be a window of time when we are executing in the host 699366f6083SPeter Grehan * kernel context with a value of KGSBASE from the guest. This is ok 700366f6083SPeter Grehan * because the value of KGSBASE is inconsequential in kernel context. 701366f6083SPeter Grehan * 702366f6083SPeter Grehan * MSR_EFER is saved and restored in the guest VMCS area on a 703366f6083SPeter Grehan * VM exit and entry respectively. It is also restored from the 704366f6083SPeter Grehan * host VMCS area on a VM exit. 705366f6083SPeter Grehan */ 706366f6083SPeter Grehan if (guest_msr_rw(vmx, MSR_GSBASE) || 707366f6083SPeter Grehan guest_msr_rw(vmx, MSR_FSBASE) || 708366f6083SPeter Grehan guest_msr_rw(vmx, MSR_KGSBASE) || 709608f97c3SPeter Grehan guest_msr_rw(vmx, MSR_EFER)) 710366f6083SPeter Grehan panic("vmx_vminit: error setting guest msr access"); 711366f6083SPeter Grehan 712608f97c3SPeter Grehan /* 713608f97c3SPeter Grehan * MSR_PAT is saved and restored in the guest VMCS are on a VM exit 714608f97c3SPeter Grehan * and entry respectively. It is also restored from the host VMCS 715608f97c3SPeter Grehan * area on a VM exit. However, if running on a system with no 716608f97c3SPeter Grehan * MSR_PAT save/restore support, leave access disabled so accesses 717608f97c3SPeter Grehan * will be trapped. 718608f97c3SPeter Grehan */ 719608f97c3SPeter Grehan if (!vmx_no_patmsr && guest_msr_rw(vmx, MSR_PAT)) 720608f97c3SPeter Grehan panic("vmx_vminit: error setting guest pat msr access"); 721608f97c3SPeter Grehan 722366f6083SPeter Grehan for (i = 0; i < VM_MAXCPU; i++) { 723366f6083SPeter Grehan vmx->vmcs[i].identifier = vmx_revision(); 724366f6083SPeter Grehan error = vmclear(&vmx->vmcs[i]); 725366f6083SPeter Grehan if (error != 0) { 726366f6083SPeter Grehan panic("vmx_vminit: vmclear error %d on vcpu %d\n", 727366f6083SPeter Grehan error, i); 728366f6083SPeter Grehan } 729366f6083SPeter Grehan 730366f6083SPeter Grehan vpid = vmx_vpid(); 731366f6083SPeter Grehan 732366f6083SPeter Grehan error = vmcs_set_defaults(&vmx->vmcs[i], 733366f6083SPeter Grehan (u_long)vmx_longjmp, 734366f6083SPeter Grehan (u_long)&vmx->ctx[i], 735366f6083SPeter Grehan vtophys(vmx->pml4ept), 736366f6083SPeter Grehan pinbased_ctls, 737366f6083SPeter Grehan procbased_ctls, 738366f6083SPeter Grehan procbased_ctls2, 739366f6083SPeter Grehan exit_ctls, entry_ctls, 740366f6083SPeter Grehan vtophys(vmx->msr_bitmap), 741366f6083SPeter Grehan vpid); 742366f6083SPeter Grehan 743366f6083SPeter Grehan if (error != 0) 744366f6083SPeter Grehan panic("vmx_vminit: vmcs_set_defaults error %d", error); 745366f6083SPeter Grehan 746366f6083SPeter Grehan vmx->cap[i].set = 0; 747366f6083SPeter Grehan vmx->cap[i].proc_ctls = procbased_ctls; 748366f6083SPeter Grehan 749366f6083SPeter Grehan vmx->state[i].request_nmi = 0; 750366f6083SPeter Grehan vmx->state[i].lastcpu = -1; 751366f6083SPeter Grehan vmx->state[i].vpid = vpid; 752366f6083SPeter Grehan 753366f6083SPeter Grehan msr_save_area_init(vmx->guest_msrs[i], &guest_msr_count); 754366f6083SPeter Grehan 755366f6083SPeter Grehan error = vmcs_set_msr_save(&vmx->vmcs[i], 756366f6083SPeter Grehan vtophys(vmx->guest_msrs[i]), 757366f6083SPeter Grehan guest_msr_count); 758366f6083SPeter Grehan if (error != 0) 759366f6083SPeter Grehan panic("vmcs_set_msr_save error %d", error); 760366f6083SPeter Grehan 761366f6083SPeter Grehan error = vmx_setup_cr0_shadow(&vmx->vmcs[i]); 762*39c21c2dSNeel Natu if (error != 0) 763*39c21c2dSNeel Natu panic("vmx_setup_cr0_shadow %d", error); 764*39c21c2dSNeel Natu 765*39c21c2dSNeel Natu error = vmx_setup_cr4_shadow(&vmx->vmcs[i]); 766*39c21c2dSNeel Natu if (error != 0) 767*39c21c2dSNeel Natu panic("vmx_setup_cr4_shadow %d", error); 768366f6083SPeter Grehan } 769366f6083SPeter Grehan 770366f6083SPeter Grehan return (vmx); 771366f6083SPeter Grehan } 772366f6083SPeter Grehan 773366f6083SPeter Grehan static int 7748b287612SJohn Baldwin vmx_handle_cpuid(int vcpu, struct vmxctx *vmxctx) 775366f6083SPeter Grehan { 776366f6083SPeter Grehan int handled, func; 777366f6083SPeter Grehan 778366f6083SPeter Grehan func = vmxctx->guest_rax; 779366f6083SPeter Grehan 780366f6083SPeter Grehan handled = x86_emulate_cpuid((uint32_t*)(&vmxctx->guest_rax), 781366f6083SPeter Grehan (uint32_t*)(&vmxctx->guest_rbx), (uint32_t*)(&vmxctx->guest_rcx), 7828b287612SJohn Baldwin (uint32_t*)(&vmxctx->guest_rdx), vcpu); 783366f6083SPeter Grehan #if 0 784366f6083SPeter Grehan printf("%s: func %x rax %lx rbx %lx rcx %lx rdx %lx handled %d\n", 785366f6083SPeter Grehan __func__, func, vmxctx->guest_rax, vmxctx->guest_rbx, 786366f6083SPeter Grehan vmxctx->guest_rcx, vmxctx->guest_rdx, handled); 787366f6083SPeter Grehan #endif 788366f6083SPeter Grehan 789366f6083SPeter Grehan return (handled); 790366f6083SPeter Grehan } 791366f6083SPeter Grehan 792366f6083SPeter Grehan static __inline void 793366f6083SPeter Grehan vmx_run_trace(struct vmx *vmx, int vcpu) 794366f6083SPeter Grehan { 795366f6083SPeter Grehan #ifdef KTR 796366f6083SPeter Grehan VMM_CTR1(vmx->vm, vcpu, "Resume execution at 0x%0lx", vmcs_guest_rip()); 797366f6083SPeter Grehan #endif 798366f6083SPeter Grehan } 799366f6083SPeter Grehan 800366f6083SPeter Grehan static __inline void 801366f6083SPeter Grehan vmx_exit_trace(struct vmx *vmx, int vcpu, uint64_t rip, uint32_t exit_reason, 802366f6083SPeter Grehan int handled, int astpending) 803366f6083SPeter Grehan { 804366f6083SPeter Grehan #ifdef KTR 805366f6083SPeter Grehan VMM_CTR3(vmx->vm, vcpu, "%s %s vmexit at 0x%0lx", 806366f6083SPeter Grehan handled ? "handled" : "unhandled", 807366f6083SPeter Grehan exit_reason_to_str(exit_reason), rip); 808366f6083SPeter Grehan 809366f6083SPeter Grehan if (astpending) 810366f6083SPeter Grehan VMM_CTR0(vmx->vm, vcpu, "astpending"); 811366f6083SPeter Grehan #endif 812366f6083SPeter Grehan } 813366f6083SPeter Grehan 814366f6083SPeter Grehan static int 815366f6083SPeter Grehan vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu) 816366f6083SPeter Grehan { 817366f6083SPeter Grehan int error, lastcpu; 818366f6083SPeter Grehan struct vmxstate *vmxstate; 819366f6083SPeter Grehan struct invvpid_desc invvpid_desc = { 0 }; 820366f6083SPeter Grehan 821366f6083SPeter Grehan vmxstate = &vmx->state[vcpu]; 822366f6083SPeter Grehan lastcpu = vmxstate->lastcpu; 823366f6083SPeter Grehan vmxstate->lastcpu = curcpu; 824366f6083SPeter Grehan 825366f6083SPeter Grehan if (lastcpu == curcpu) { 826366f6083SPeter Grehan error = 0; 827366f6083SPeter Grehan goto done; 828366f6083SPeter Grehan } 829366f6083SPeter Grehan 830366f6083SPeter Grehan vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1); 831366f6083SPeter Grehan 832366f6083SPeter Grehan error = vmwrite(VMCS_HOST_TR_BASE, (u_long)PCPU_GET(tssp)); 833366f6083SPeter Grehan if (error != 0) 834366f6083SPeter Grehan goto done; 835366f6083SPeter Grehan 836366f6083SPeter Grehan error = vmwrite(VMCS_HOST_GDTR_BASE, (u_long)&gdt[NGDT * curcpu]); 837366f6083SPeter Grehan if (error != 0) 838366f6083SPeter Grehan goto done; 839366f6083SPeter Grehan 840366f6083SPeter Grehan error = vmwrite(VMCS_HOST_GS_BASE, (u_long)&__pcpu[curcpu]); 841366f6083SPeter Grehan if (error != 0) 842366f6083SPeter Grehan goto done; 843366f6083SPeter Grehan 844366f6083SPeter Grehan /* 845366f6083SPeter Grehan * If we are using VPIDs then invalidate all mappings tagged with 'vpid' 846366f6083SPeter Grehan * 847366f6083SPeter Grehan * We do this because this vcpu was executing on a different host 848366f6083SPeter Grehan * cpu when it last ran. We do not track whether it invalidated 849366f6083SPeter Grehan * mappings associated with its 'vpid' during that run. So we must 850366f6083SPeter Grehan * assume that the mappings associated with 'vpid' on 'curcpu' are 851366f6083SPeter Grehan * stale and invalidate them. 852366f6083SPeter Grehan * 853366f6083SPeter Grehan * Note that we incur this penalty only when the scheduler chooses to 854366f6083SPeter Grehan * move the thread associated with this vcpu between host cpus. 855366f6083SPeter Grehan * 856366f6083SPeter Grehan * Note also that this will invalidate mappings tagged with 'vpid' 857366f6083SPeter Grehan * for "all" EP4TAs. 858366f6083SPeter Grehan */ 859366f6083SPeter Grehan if (vmxstate->vpid != 0) { 860366f6083SPeter Grehan invvpid_desc.vpid = vmxstate->vpid; 861366f6083SPeter Grehan invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc); 862366f6083SPeter Grehan } 863366f6083SPeter Grehan done: 864366f6083SPeter Grehan return (error); 865366f6083SPeter Grehan } 866366f6083SPeter Grehan 867366f6083SPeter Grehan static void 868366f6083SPeter Grehan vm_exit_update_rip(struct vm_exit *vmexit) 869366f6083SPeter Grehan { 870366f6083SPeter Grehan int error; 871366f6083SPeter Grehan 872366f6083SPeter Grehan error = vmwrite(VMCS_GUEST_RIP, vmexit->rip + vmexit->inst_length); 873366f6083SPeter Grehan if (error) 874366f6083SPeter Grehan panic("vmx_run: error %d writing to VMCS_GUEST_RIP", error); 875366f6083SPeter Grehan } 876366f6083SPeter Grehan 877366f6083SPeter Grehan /* 878366f6083SPeter Grehan * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set. 879366f6083SPeter Grehan */ 880366f6083SPeter Grehan CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0); 881366f6083SPeter Grehan 882366f6083SPeter Grehan static void __inline 883366f6083SPeter Grehan vmx_set_int_window_exiting(struct vmx *vmx, int vcpu) 884366f6083SPeter Grehan { 885366f6083SPeter Grehan int error; 886366f6083SPeter Grehan 887366f6083SPeter Grehan vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING; 888366f6083SPeter Grehan 889366f6083SPeter Grehan error = vmwrite(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 890366f6083SPeter Grehan if (error) 891366f6083SPeter Grehan panic("vmx_set_int_window_exiting: vmwrite error %d", error); 892366f6083SPeter Grehan } 893366f6083SPeter Grehan 894366f6083SPeter Grehan static void __inline 895366f6083SPeter Grehan vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu) 896366f6083SPeter Grehan { 897366f6083SPeter Grehan int error; 898366f6083SPeter Grehan 899366f6083SPeter Grehan vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING; 900366f6083SPeter Grehan 901366f6083SPeter Grehan error = vmwrite(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 902366f6083SPeter Grehan if (error) 903366f6083SPeter Grehan panic("vmx_clear_int_window_exiting: vmwrite error %d", error); 904366f6083SPeter Grehan } 905366f6083SPeter Grehan 906366f6083SPeter Grehan static void __inline 907366f6083SPeter Grehan vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu) 908366f6083SPeter Grehan { 909366f6083SPeter Grehan int error; 910366f6083SPeter Grehan 911366f6083SPeter Grehan vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING; 912366f6083SPeter Grehan 913366f6083SPeter Grehan error = vmwrite(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 914366f6083SPeter Grehan if (error) 915366f6083SPeter Grehan panic("vmx_set_nmi_window_exiting: vmwrite error %d", error); 916366f6083SPeter Grehan } 917366f6083SPeter Grehan 918366f6083SPeter Grehan static void __inline 919366f6083SPeter Grehan vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu) 920366f6083SPeter Grehan { 921366f6083SPeter Grehan int error; 922366f6083SPeter Grehan 923366f6083SPeter Grehan vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING; 924366f6083SPeter Grehan 925366f6083SPeter Grehan error = vmwrite(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 926366f6083SPeter Grehan if (error) 927366f6083SPeter Grehan panic("vmx_clear_nmi_window_exiting: vmwrite error %d", error); 928366f6083SPeter Grehan } 929366f6083SPeter Grehan 930366f6083SPeter Grehan static int 931366f6083SPeter Grehan vmx_inject_nmi(struct vmx *vmx, int vcpu) 932366f6083SPeter Grehan { 933366f6083SPeter Grehan int error; 934366f6083SPeter Grehan uint64_t info, interruptibility; 935366f6083SPeter Grehan 936366f6083SPeter Grehan /* Bail out if no NMI requested */ 937366f6083SPeter Grehan if (vmx->state[vcpu].request_nmi == 0) 938366f6083SPeter Grehan return (0); 939366f6083SPeter Grehan 940366f6083SPeter Grehan error = vmread(VMCS_GUEST_INTERRUPTIBILITY, &interruptibility); 941366f6083SPeter Grehan if (error) { 942366f6083SPeter Grehan panic("vmx_inject_nmi: vmread(interruptibility) %d", 943366f6083SPeter Grehan error); 944366f6083SPeter Grehan } 945366f6083SPeter Grehan if (interruptibility & nmi_blocking_bits) 946366f6083SPeter Grehan goto nmiblocked; 947366f6083SPeter Grehan 948366f6083SPeter Grehan /* 949366f6083SPeter Grehan * Inject the virtual NMI. The vector must be the NMI IDT entry 950366f6083SPeter Grehan * or the VMCS entry check will fail. 951366f6083SPeter Grehan */ 952366f6083SPeter Grehan info = VMCS_INTERRUPTION_INFO_NMI | VMCS_INTERRUPTION_INFO_VALID; 953366f6083SPeter Grehan info |= IDT_NMI; 954366f6083SPeter Grehan 955366f6083SPeter Grehan error = vmwrite(VMCS_ENTRY_INTR_INFO, info); 956366f6083SPeter Grehan if (error) 957366f6083SPeter Grehan panic("vmx_inject_nmi: vmwrite(intrinfo) %d", error); 958366f6083SPeter Grehan 959366f6083SPeter Grehan VMM_CTR0(vmx->vm, vcpu, "Injecting vNMI"); 960366f6083SPeter Grehan 961366f6083SPeter Grehan /* Clear the request */ 962366f6083SPeter Grehan vmx->state[vcpu].request_nmi = 0; 963366f6083SPeter Grehan return (1); 964366f6083SPeter Grehan 965366f6083SPeter Grehan nmiblocked: 966366f6083SPeter Grehan /* 967366f6083SPeter Grehan * Set the NMI Window Exiting execution control so we can inject 968366f6083SPeter Grehan * the virtual NMI as soon as blocking condition goes away. 969366f6083SPeter Grehan */ 970366f6083SPeter Grehan vmx_set_nmi_window_exiting(vmx, vcpu); 971366f6083SPeter Grehan 972366f6083SPeter Grehan VMM_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting"); 973366f6083SPeter Grehan return (1); 974366f6083SPeter Grehan } 975366f6083SPeter Grehan 976366f6083SPeter Grehan static void 977366f6083SPeter Grehan vmx_inject_interrupts(struct vmx *vmx, int vcpu) 978366f6083SPeter Grehan { 979366f6083SPeter Grehan int error, vector; 980366f6083SPeter Grehan uint64_t info, rflags, interruptibility; 981366f6083SPeter Grehan 982366f6083SPeter Grehan const int HWINTR_BLOCKED = VMCS_INTERRUPTIBILITY_STI_BLOCKING | 983366f6083SPeter Grehan VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING; 984366f6083SPeter Grehan 985366f6083SPeter Grehan #if 1 986366f6083SPeter Grehan /* 987366f6083SPeter Grehan * XXX 988366f6083SPeter Grehan * If an event is being injected from userland then just return. 989366f6083SPeter Grehan * For e.g. we may inject a breakpoint exception to cause the 990366f6083SPeter Grehan * guest to enter the debugger so we can inspect its state. 991366f6083SPeter Grehan */ 992366f6083SPeter Grehan error = vmread(VMCS_ENTRY_INTR_INFO, &info); 993366f6083SPeter Grehan if (error) 994366f6083SPeter Grehan panic("vmx_inject_interrupts: vmread(intrinfo) %d", error); 995366f6083SPeter Grehan if (info & VMCS_INTERRUPTION_INFO_VALID) 996366f6083SPeter Grehan return; 997366f6083SPeter Grehan #endif 998366f6083SPeter Grehan /* 999366f6083SPeter Grehan * NMI injection has priority so deal with those first 1000366f6083SPeter Grehan */ 1001366f6083SPeter Grehan if (vmx_inject_nmi(vmx, vcpu)) 1002366f6083SPeter Grehan return; 1003366f6083SPeter Grehan 1004366f6083SPeter Grehan /* Ask the local apic for a vector to inject */ 1005366f6083SPeter Grehan vector = lapic_pending_intr(vmx->vm, vcpu); 1006366f6083SPeter Grehan if (vector < 0) 1007366f6083SPeter Grehan return; 1008366f6083SPeter Grehan 1009366f6083SPeter Grehan if (vector < 32 || vector > 255) 1010366f6083SPeter Grehan panic("vmx_inject_interrupts: invalid vector %d\n", vector); 1011366f6083SPeter Grehan 1012366f6083SPeter Grehan /* Check RFLAGS.IF and the interruptibility state of the guest */ 1013366f6083SPeter Grehan error = vmread(VMCS_GUEST_RFLAGS, &rflags); 1014366f6083SPeter Grehan if (error) 1015366f6083SPeter Grehan panic("vmx_inject_interrupts: vmread(rflags) %d", error); 1016366f6083SPeter Grehan 1017366f6083SPeter Grehan if ((rflags & PSL_I) == 0) 1018366f6083SPeter Grehan goto cantinject; 1019366f6083SPeter Grehan 1020366f6083SPeter Grehan error = vmread(VMCS_GUEST_INTERRUPTIBILITY, &interruptibility); 1021366f6083SPeter Grehan if (error) { 1022366f6083SPeter Grehan panic("vmx_inject_interrupts: vmread(interruptibility) %d", 1023366f6083SPeter Grehan error); 1024366f6083SPeter Grehan } 1025366f6083SPeter Grehan if (interruptibility & HWINTR_BLOCKED) 1026366f6083SPeter Grehan goto cantinject; 1027366f6083SPeter Grehan 1028366f6083SPeter Grehan /* Inject the interrupt */ 1029366f6083SPeter Grehan info = VMCS_INTERRUPTION_INFO_HW_INTR | VMCS_INTERRUPTION_INFO_VALID; 1030366f6083SPeter Grehan info |= vector; 1031366f6083SPeter Grehan error = vmwrite(VMCS_ENTRY_INTR_INFO, info); 1032366f6083SPeter Grehan if (error) 1033366f6083SPeter Grehan panic("vmx_inject_interrupts: vmwrite(intrinfo) %d", error); 1034366f6083SPeter Grehan 1035366f6083SPeter Grehan /* Update the Local APIC ISR */ 1036366f6083SPeter Grehan lapic_intr_accepted(vmx->vm, vcpu, vector); 1037366f6083SPeter Grehan 1038366f6083SPeter Grehan VMM_CTR1(vmx->vm, vcpu, "Injecting hwintr at vector %d", vector); 1039366f6083SPeter Grehan 1040366f6083SPeter Grehan return; 1041366f6083SPeter Grehan 1042366f6083SPeter Grehan cantinject: 1043366f6083SPeter Grehan /* 1044366f6083SPeter Grehan * Set the Interrupt Window Exiting execution control so we can inject 1045366f6083SPeter Grehan * the interrupt as soon as blocking condition goes away. 1046366f6083SPeter Grehan */ 1047366f6083SPeter Grehan vmx_set_int_window_exiting(vmx, vcpu); 1048366f6083SPeter Grehan 1049366f6083SPeter Grehan VMM_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting"); 1050366f6083SPeter Grehan } 1051366f6083SPeter Grehan 1052366f6083SPeter Grehan static int 1053366f6083SPeter Grehan vmx_emulate_cr_access(struct vmx *vmx, int vcpu, uint64_t exitqual) 1054366f6083SPeter Grehan { 1055*39c21c2dSNeel Natu int error, cr, vmcs_guest_cr; 1056*39c21c2dSNeel Natu uint64_t regval, ones_mask, zeros_mask; 1057366f6083SPeter Grehan const struct vmxctx *vmxctx; 1058366f6083SPeter Grehan 1059*39c21c2dSNeel Natu /* We only handle mov to %cr0 or %cr4 at this time */ 1060*39c21c2dSNeel Natu if ((exitqual & 0xf0) != 0x00) 1061*39c21c2dSNeel Natu return (UNHANDLED); 1062*39c21c2dSNeel Natu 1063*39c21c2dSNeel Natu cr = exitqual & 0xf; 1064*39c21c2dSNeel Natu if (cr != 0 && cr != 4) 1065366f6083SPeter Grehan return (UNHANDLED); 1066366f6083SPeter Grehan 1067366f6083SPeter Grehan vmxctx = &vmx->ctx[vcpu]; 1068366f6083SPeter Grehan 1069366f6083SPeter Grehan /* 1070366f6083SPeter Grehan * We must use vmwrite() directly here because vmcs_setreg() will 1071366f6083SPeter Grehan * call vmclear(vmcs) as a side-effect which we certainly don't want. 1072366f6083SPeter Grehan */ 1073366f6083SPeter Grehan switch ((exitqual >> 8) & 0xf) { 1074366f6083SPeter Grehan case 0: 1075366f6083SPeter Grehan regval = vmxctx->guest_rax; 1076366f6083SPeter Grehan break; 1077366f6083SPeter Grehan case 1: 1078366f6083SPeter Grehan regval = vmxctx->guest_rcx; 1079366f6083SPeter Grehan break; 1080366f6083SPeter Grehan case 2: 1081366f6083SPeter Grehan regval = vmxctx->guest_rdx; 1082366f6083SPeter Grehan break; 1083366f6083SPeter Grehan case 3: 1084366f6083SPeter Grehan regval = vmxctx->guest_rbx; 1085366f6083SPeter Grehan break; 1086366f6083SPeter Grehan case 4: 1087366f6083SPeter Grehan error = vmread(VMCS_GUEST_RSP, ®val); 1088366f6083SPeter Grehan if (error) { 1089366f6083SPeter Grehan panic("vmx_emulate_cr_access: " 1090366f6083SPeter Grehan "error %d reading guest rsp", error); 1091366f6083SPeter Grehan } 1092366f6083SPeter Grehan break; 1093366f6083SPeter Grehan case 5: 1094366f6083SPeter Grehan regval = vmxctx->guest_rbp; 1095366f6083SPeter Grehan break; 1096366f6083SPeter Grehan case 6: 1097366f6083SPeter Grehan regval = vmxctx->guest_rsi; 1098366f6083SPeter Grehan break; 1099366f6083SPeter Grehan case 7: 1100366f6083SPeter Grehan regval = vmxctx->guest_rdi; 1101366f6083SPeter Grehan break; 1102366f6083SPeter Grehan case 8: 1103366f6083SPeter Grehan regval = vmxctx->guest_r8; 1104366f6083SPeter Grehan break; 1105366f6083SPeter Grehan case 9: 1106366f6083SPeter Grehan regval = vmxctx->guest_r9; 1107366f6083SPeter Grehan break; 1108366f6083SPeter Grehan case 10: 1109366f6083SPeter Grehan regval = vmxctx->guest_r10; 1110366f6083SPeter Grehan break; 1111366f6083SPeter Grehan case 11: 1112366f6083SPeter Grehan regval = vmxctx->guest_r11; 1113366f6083SPeter Grehan break; 1114366f6083SPeter Grehan case 12: 1115366f6083SPeter Grehan regval = vmxctx->guest_r12; 1116366f6083SPeter Grehan break; 1117366f6083SPeter Grehan case 13: 1118366f6083SPeter Grehan regval = vmxctx->guest_r13; 1119366f6083SPeter Grehan break; 1120366f6083SPeter Grehan case 14: 1121366f6083SPeter Grehan regval = vmxctx->guest_r14; 1122366f6083SPeter Grehan break; 1123366f6083SPeter Grehan case 15: 1124366f6083SPeter Grehan regval = vmxctx->guest_r15; 1125366f6083SPeter Grehan break; 1126366f6083SPeter Grehan } 1127366f6083SPeter Grehan 1128*39c21c2dSNeel Natu if (cr == 0) { 1129*39c21c2dSNeel Natu ones_mask = cr0_ones_mask; 1130*39c21c2dSNeel Natu zeros_mask = cr0_zeros_mask; 1131*39c21c2dSNeel Natu vmcs_guest_cr = VMCS_GUEST_CR0; 1132*39c21c2dSNeel Natu } else { 1133*39c21c2dSNeel Natu ones_mask = cr4_ones_mask; 1134*39c21c2dSNeel Natu zeros_mask = cr4_zeros_mask; 1135*39c21c2dSNeel Natu vmcs_guest_cr = VMCS_GUEST_CR4; 1136*39c21c2dSNeel Natu } 1137*39c21c2dSNeel Natu regval |= ones_mask; 1138*39c21c2dSNeel Natu regval &= ~zeros_mask; 1139*39c21c2dSNeel Natu error = vmwrite(vmcs_guest_cr, regval); 1140*39c21c2dSNeel Natu if (error) { 1141*39c21c2dSNeel Natu panic("vmx_emulate_cr_access: error %d writing cr%d", 1142*39c21c2dSNeel Natu error, cr); 1143*39c21c2dSNeel Natu } 1144366f6083SPeter Grehan 1145366f6083SPeter Grehan return (HANDLED); 1146366f6083SPeter Grehan } 1147366f6083SPeter Grehan 1148366f6083SPeter Grehan static int 1149366f6083SPeter Grehan vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) 1150366f6083SPeter Grehan { 1151366f6083SPeter Grehan int handled; 1152366f6083SPeter Grehan struct vmcs *vmcs; 1153366f6083SPeter Grehan struct vmxctx *vmxctx; 1154366f6083SPeter Grehan uint32_t eax, ecx, edx; 1155366f6083SPeter Grehan uint64_t qual; 1156366f6083SPeter Grehan 1157366f6083SPeter Grehan handled = 0; 1158366f6083SPeter Grehan vmcs = &vmx->vmcs[vcpu]; 1159366f6083SPeter Grehan vmxctx = &vmx->ctx[vcpu]; 1160366f6083SPeter Grehan qual = vmexit->u.vmx.exit_qualification; 1161366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_BOGUS; 1162366f6083SPeter Grehan 1163366f6083SPeter Grehan switch (vmexit->u.vmx.exit_reason) { 1164366f6083SPeter Grehan case EXIT_REASON_CR_ACCESS: 1165366f6083SPeter Grehan handled = vmx_emulate_cr_access(vmx, vcpu, qual); 1166366f6083SPeter Grehan break; 1167366f6083SPeter Grehan case EXIT_REASON_RDMSR: 1168366f6083SPeter Grehan ecx = vmxctx->guest_rcx; 1169366f6083SPeter Grehan handled = emulate_rdmsr(vmx->vm, vcpu, ecx); 1170366f6083SPeter Grehan if (!handled) { 1171366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_RDMSR; 1172366f6083SPeter Grehan vmexit->u.msr.code = ecx; 1173366f6083SPeter Grehan } 1174366f6083SPeter Grehan break; 1175366f6083SPeter Grehan case EXIT_REASON_WRMSR: 1176366f6083SPeter Grehan eax = vmxctx->guest_rax; 1177366f6083SPeter Grehan ecx = vmxctx->guest_rcx; 1178366f6083SPeter Grehan edx = vmxctx->guest_rdx; 1179366f6083SPeter Grehan handled = emulate_wrmsr(vmx->vm, vcpu, ecx, 1180366f6083SPeter Grehan (uint64_t)edx << 32 | eax); 1181366f6083SPeter Grehan if (!handled) { 1182366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_WRMSR; 1183366f6083SPeter Grehan vmexit->u.msr.code = ecx; 1184366f6083SPeter Grehan vmexit->u.msr.wval = (uint64_t)edx << 32 | eax; 1185366f6083SPeter Grehan } 1186366f6083SPeter Grehan break; 1187366f6083SPeter Grehan case EXIT_REASON_HLT: 1188366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_HLT; 1189366f6083SPeter Grehan break; 1190366f6083SPeter Grehan case EXIT_REASON_MTF: 1191366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_MTRAP; 1192366f6083SPeter Grehan break; 1193366f6083SPeter Grehan case EXIT_REASON_PAUSE: 1194366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_PAUSE; 1195366f6083SPeter Grehan break; 1196366f6083SPeter Grehan case EXIT_REASON_INTR_WINDOW: 1197366f6083SPeter Grehan vmx_clear_int_window_exiting(vmx, vcpu); 1198366f6083SPeter Grehan VMM_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting"); 1199366f6083SPeter Grehan /* FALLTHRU */ 1200366f6083SPeter Grehan case EXIT_REASON_EXT_INTR: 1201366f6083SPeter Grehan /* 1202366f6083SPeter Grehan * External interrupts serve only to cause VM exits and allow 1203366f6083SPeter Grehan * the host interrupt handler to run. 1204366f6083SPeter Grehan * 1205366f6083SPeter Grehan * If this external interrupt triggers a virtual interrupt 1206366f6083SPeter Grehan * to a VM, then that state will be recorded by the 1207366f6083SPeter Grehan * host interrupt handler in the VM's softc. We will inject 1208366f6083SPeter Grehan * this virtual interrupt during the subsequent VM enter. 1209366f6083SPeter Grehan */ 1210366f6083SPeter Grehan 1211366f6083SPeter Grehan /* 1212366f6083SPeter Grehan * This is special. We want to treat this as an 'handled' 1213366f6083SPeter Grehan * VM-exit but not increment the instruction pointer. 1214366f6083SPeter Grehan */ 1215366f6083SPeter Grehan vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1); 1216366f6083SPeter Grehan return (1); 1217366f6083SPeter Grehan case EXIT_REASON_NMI_WINDOW: 1218366f6083SPeter Grehan /* Exit to allow the pending virtual NMI to be injected */ 1219366f6083SPeter Grehan vmx_clear_nmi_window_exiting(vmx, vcpu); 1220366f6083SPeter Grehan VMM_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting"); 1221366f6083SPeter Grehan return (1); 1222366f6083SPeter Grehan case EXIT_REASON_INOUT: 1223366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_INOUT; 1224366f6083SPeter Grehan vmexit->u.inout.bytes = (qual & 0x7) + 1; 1225366f6083SPeter Grehan vmexit->u.inout.in = (qual & 0x8) ? 1 : 0; 1226366f6083SPeter Grehan vmexit->u.inout.string = (qual & 0x10) ? 1 : 0; 1227366f6083SPeter Grehan vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0; 1228366f6083SPeter Grehan vmexit->u.inout.port = (uint16_t)(qual >> 16); 1229366f6083SPeter Grehan vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax); 1230366f6083SPeter Grehan break; 1231366f6083SPeter Grehan case EXIT_REASON_CPUID: 12328b287612SJohn Baldwin handled = vmx_handle_cpuid(vcpu, vmxctx); 1233366f6083SPeter Grehan break; 1234cd942e0fSPeter Grehan case EXIT_REASON_EPT_FAULT: 1235cd942e0fSPeter Grehan vmexit->exitcode = VM_EXITCODE_PAGING; 1236cd942e0fSPeter Grehan vmexit->u.paging.cr3 = vmcs_guest_cr3(); 1237cd942e0fSPeter Grehan break; 1238366f6083SPeter Grehan default: 1239366f6083SPeter Grehan break; 1240366f6083SPeter Grehan } 1241366f6083SPeter Grehan 1242366f6083SPeter Grehan if (handled) { 1243366f6083SPeter Grehan /* 1244366f6083SPeter Grehan * It is possible that control is returned to userland 1245366f6083SPeter Grehan * even though we were able to handle the VM exit in the 1246366f6083SPeter Grehan * kernel (for e.g. 'astpending' is set in the run loop). 1247366f6083SPeter Grehan * 1248366f6083SPeter Grehan * In such a case we want to make sure that the userland 1249366f6083SPeter Grehan * restarts guest execution at the instruction *after* 1250366f6083SPeter Grehan * the one we just processed. Therefore we update the 1251366f6083SPeter Grehan * guest rip in the VMCS and in 'vmexit'. 1252366f6083SPeter Grehan */ 1253366f6083SPeter Grehan vm_exit_update_rip(vmexit); 1254366f6083SPeter Grehan vmexit->rip += vmexit->inst_length; 1255366f6083SPeter Grehan vmexit->inst_length = 0; 1256366f6083SPeter Grehan } else { 1257366f6083SPeter Grehan if (vmexit->exitcode == VM_EXITCODE_BOGUS) { 1258366f6083SPeter Grehan /* 1259366f6083SPeter Grehan * If this VM exit was not claimed by anybody then 1260366f6083SPeter Grehan * treat it as a generic VMX exit. 1261366f6083SPeter Grehan */ 1262366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_VMX; 1263366f6083SPeter Grehan vmexit->u.vmx.error = 0; 1264366f6083SPeter Grehan } else { 1265366f6083SPeter Grehan /* 1266366f6083SPeter Grehan * The exitcode and collateral have been populated. 1267366f6083SPeter Grehan * The VM exit will be processed further in userland. 1268366f6083SPeter Grehan */ 1269366f6083SPeter Grehan } 1270366f6083SPeter Grehan } 1271366f6083SPeter Grehan return (handled); 1272366f6083SPeter Grehan } 1273366f6083SPeter Grehan 1274366f6083SPeter Grehan static int 1275366f6083SPeter Grehan vmx_run(void *arg, int vcpu, register_t rip, struct vm_exit *vmexit) 1276366f6083SPeter Grehan { 1277ad54f374SNeel Natu int error, vie, rc, handled, astpending; 1278366f6083SPeter Grehan uint32_t exit_reason; 1279366f6083SPeter Grehan struct vmx *vmx; 1280366f6083SPeter Grehan struct vmxctx *vmxctx; 1281366f6083SPeter Grehan struct vmcs *vmcs; 1282366f6083SPeter Grehan 1283366f6083SPeter Grehan vmx = arg; 1284366f6083SPeter Grehan vmcs = &vmx->vmcs[vcpu]; 1285366f6083SPeter Grehan vmxctx = &vmx->ctx[vcpu]; 1286ad54f374SNeel Natu vmxctx->launched = 0; 1287366f6083SPeter Grehan 1288366f6083SPeter Grehan /* 1289366f6083SPeter Grehan * XXX Can we avoid doing this every time we do a vm run? 1290366f6083SPeter Grehan */ 1291366f6083SPeter Grehan VMPTRLD(vmcs); 1292366f6083SPeter Grehan 1293366f6083SPeter Grehan /* 1294366f6083SPeter Grehan * XXX 1295366f6083SPeter Grehan * We do this every time because we may setup the virtual machine 1296366f6083SPeter Grehan * from a different process than the one that actually runs it. 1297366f6083SPeter Grehan * 1298366f6083SPeter Grehan * If the life of a virtual machine was spent entirely in the context 1299366f6083SPeter Grehan * of a single process we could do this once in vmcs_set_defaults(). 1300366f6083SPeter Grehan */ 1301366f6083SPeter Grehan if ((error = vmwrite(VMCS_HOST_CR3, rcr3())) != 0) 1302366f6083SPeter Grehan panic("vmx_run: error %d writing to VMCS_HOST_CR3", error); 1303366f6083SPeter Grehan 1304366f6083SPeter Grehan if ((error = vmwrite(VMCS_GUEST_RIP, rip)) != 0) 1305366f6083SPeter Grehan panic("vmx_run: error %d writing to VMCS_GUEST_RIP", error); 1306366f6083SPeter Grehan 1307366f6083SPeter Grehan if ((error = vmx_set_pcpu_defaults(vmx, vcpu)) != 0) 1308366f6083SPeter Grehan panic("vmx_run: error %d setting up pcpu defaults", error); 1309366f6083SPeter Grehan 1310366f6083SPeter Grehan do { 1311366f6083SPeter Grehan lapic_timer_tick(vmx->vm, vcpu); 1312366f6083SPeter Grehan vmx_inject_interrupts(vmx, vcpu); 1313366f6083SPeter Grehan vmx_run_trace(vmx, vcpu); 1314366f6083SPeter Grehan rc = vmx_setjmp(vmxctx); 1315366f6083SPeter Grehan #ifdef SETJMP_TRACE 1316366f6083SPeter Grehan vmx_setjmp_trace(vmx, vcpu, vmxctx, rc); 1317366f6083SPeter Grehan #endif 1318366f6083SPeter Grehan switch (rc) { 1319366f6083SPeter Grehan case VMX_RETURN_DIRECT: 1320ad54f374SNeel Natu if (vmxctx->launched == 0) { 1321ad54f374SNeel Natu vmxctx->launched = 1; 1322366f6083SPeter Grehan vmx_launch(vmxctx); 1323366f6083SPeter Grehan } else 1324366f6083SPeter Grehan vmx_resume(vmxctx); 1325366f6083SPeter Grehan panic("vmx_launch/resume should not return"); 1326366f6083SPeter Grehan break; 1327366f6083SPeter Grehan case VMX_RETURN_LONGJMP: 1328366f6083SPeter Grehan break; /* vm exit */ 1329366f6083SPeter Grehan case VMX_RETURN_VMRESUME: 1330366f6083SPeter Grehan vie = vmcs_instruction_error(); 1331366f6083SPeter Grehan if (vmxctx->launch_error == VM_FAIL_INVALID || 1332366f6083SPeter Grehan vie != VMRESUME_WITH_NON_LAUNCHED_VMCS) { 1333366f6083SPeter Grehan printf("vmresume error %d vmcs inst error %d\n", 1334366f6083SPeter Grehan vmxctx->launch_error, vie); 1335366f6083SPeter Grehan goto err_exit; 1336366f6083SPeter Grehan } 1337366f6083SPeter Grehan vmx_launch(vmxctx); /* try to launch the guest */ 1338366f6083SPeter Grehan panic("vmx_launch should not return"); 1339366f6083SPeter Grehan break; 1340366f6083SPeter Grehan case VMX_RETURN_VMLAUNCH: 1341366f6083SPeter Grehan vie = vmcs_instruction_error(); 1342366f6083SPeter Grehan #if 1 1343366f6083SPeter Grehan printf("vmlaunch error %d vmcs inst error %d\n", 1344366f6083SPeter Grehan vmxctx->launch_error, vie); 1345366f6083SPeter Grehan #endif 1346366f6083SPeter Grehan goto err_exit; 1347366f6083SPeter Grehan default: 1348366f6083SPeter Grehan panic("vmx_setjmp returned %d", rc); 1349366f6083SPeter Grehan } 1350366f6083SPeter Grehan 1351366f6083SPeter Grehan /* 1352366f6083SPeter Grehan * XXX locking? 1353366f6083SPeter Grehan * See comments in exception.S about checking for ASTs 1354366f6083SPeter Grehan * atomically while interrupts are disabled. But it is 1355366f6083SPeter Grehan * not clear that they apply in our case. 1356366f6083SPeter Grehan */ 1357366f6083SPeter Grehan astpending = curthread->td_flags & TDF_ASTPENDING; 1358366f6083SPeter Grehan 1359366f6083SPeter Grehan /* enable interrupts */ 1360366f6083SPeter Grehan enable_intr(); 1361366f6083SPeter Grehan 1362366f6083SPeter Grehan /* collect some basic information for VM exit processing */ 1363366f6083SPeter Grehan vmexit->rip = rip = vmcs_guest_rip(); 1364366f6083SPeter Grehan vmexit->inst_length = vmexit_instruction_length(); 1365366f6083SPeter Grehan vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason(); 1366366f6083SPeter Grehan vmexit->u.vmx.exit_qualification = vmcs_exit_qualification(); 1367366f6083SPeter Grehan 1368366f6083SPeter Grehan handled = vmx_exit_process(vmx, vcpu, vmexit); 1369366f6083SPeter Grehan 1370366f6083SPeter Grehan vmx_exit_trace(vmx, vcpu, rip, exit_reason, handled, 1371366f6083SPeter Grehan astpending); 1372366f6083SPeter Grehan } while (handled && !astpending); 1373366f6083SPeter Grehan 1374366f6083SPeter Grehan /* 1375366f6083SPeter Grehan * If a VM exit has been handled then the exitcode must be BOGUS 1376366f6083SPeter Grehan * If a VM exit is not handled then the exitcode must not be BOGUS 1377366f6083SPeter Grehan */ 1378366f6083SPeter Grehan if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) || 1379366f6083SPeter Grehan (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) { 1380366f6083SPeter Grehan panic("Mismatch between handled (%d) and exitcode (%d)", 1381366f6083SPeter Grehan handled, vmexit->exitcode); 1382366f6083SPeter Grehan } 1383366f6083SPeter Grehan 1384366f6083SPeter Grehan VMM_CTR1(vmx->vm, vcpu, "goto userland: exitcode %d",vmexit->exitcode); 1385366f6083SPeter Grehan 1386366f6083SPeter Grehan /* 1387366f6083SPeter Grehan * XXX 1388366f6083SPeter Grehan * We need to do this to ensure that any VMCS state cached by the 1389366f6083SPeter Grehan * processor is flushed to memory. We need to do this in case the 1390366f6083SPeter Grehan * VM moves to a different cpu the next time it runs. 1391366f6083SPeter Grehan * 1392366f6083SPeter Grehan * Can we avoid doing this? 1393366f6083SPeter Grehan */ 1394366f6083SPeter Grehan VMCLEAR(vmcs); 1395366f6083SPeter Grehan return (0); 1396366f6083SPeter Grehan 1397366f6083SPeter Grehan err_exit: 1398366f6083SPeter Grehan vmexit->exitcode = VM_EXITCODE_VMX; 1399366f6083SPeter Grehan vmexit->u.vmx.exit_reason = (uint32_t)-1; 1400366f6083SPeter Grehan vmexit->u.vmx.exit_qualification = (uint32_t)-1; 1401366f6083SPeter Grehan vmexit->u.vmx.error = vie; 1402366f6083SPeter Grehan VMCLEAR(vmcs); 1403366f6083SPeter Grehan return (ENOEXEC); 1404366f6083SPeter Grehan } 1405366f6083SPeter Grehan 1406366f6083SPeter Grehan static void 1407366f6083SPeter Grehan vmx_vmcleanup(void *arg) 1408366f6083SPeter Grehan { 1409366f6083SPeter Grehan int error; 1410366f6083SPeter Grehan struct vmx *vmx = arg; 1411366f6083SPeter Grehan 1412366f6083SPeter Grehan /* 1413366f6083SPeter Grehan * XXXSMP we also need to clear the VMCS active on the other vcpus. 1414366f6083SPeter Grehan */ 1415366f6083SPeter Grehan error = vmclear(&vmx->vmcs[0]); 1416366f6083SPeter Grehan if (error != 0) 1417366f6083SPeter Grehan panic("vmx_vmcleanup: vmclear error %d on vcpu 0", error); 1418366f6083SPeter Grehan 1419366f6083SPeter Grehan ept_vmcleanup(vmx); 1420366f6083SPeter Grehan free(vmx, M_VMX); 1421366f6083SPeter Grehan 1422366f6083SPeter Grehan return; 1423366f6083SPeter Grehan } 1424366f6083SPeter Grehan 1425366f6083SPeter Grehan static register_t * 1426366f6083SPeter Grehan vmxctx_regptr(struct vmxctx *vmxctx, int reg) 1427366f6083SPeter Grehan { 1428366f6083SPeter Grehan 1429366f6083SPeter Grehan switch (reg) { 1430366f6083SPeter Grehan case VM_REG_GUEST_RAX: 1431366f6083SPeter Grehan return (&vmxctx->guest_rax); 1432366f6083SPeter Grehan case VM_REG_GUEST_RBX: 1433366f6083SPeter Grehan return (&vmxctx->guest_rbx); 1434366f6083SPeter Grehan case VM_REG_GUEST_RCX: 1435366f6083SPeter Grehan return (&vmxctx->guest_rcx); 1436366f6083SPeter Grehan case VM_REG_GUEST_RDX: 1437366f6083SPeter Grehan return (&vmxctx->guest_rdx); 1438366f6083SPeter Grehan case VM_REG_GUEST_RSI: 1439366f6083SPeter Grehan return (&vmxctx->guest_rsi); 1440366f6083SPeter Grehan case VM_REG_GUEST_RDI: 1441366f6083SPeter Grehan return (&vmxctx->guest_rdi); 1442366f6083SPeter Grehan case VM_REG_GUEST_RBP: 1443366f6083SPeter Grehan return (&vmxctx->guest_rbp); 1444366f6083SPeter Grehan case VM_REG_GUEST_R8: 1445366f6083SPeter Grehan return (&vmxctx->guest_r8); 1446366f6083SPeter Grehan case VM_REG_GUEST_R9: 1447366f6083SPeter Grehan return (&vmxctx->guest_r9); 1448366f6083SPeter Grehan case VM_REG_GUEST_R10: 1449366f6083SPeter Grehan return (&vmxctx->guest_r10); 1450366f6083SPeter Grehan case VM_REG_GUEST_R11: 1451366f6083SPeter Grehan return (&vmxctx->guest_r11); 1452366f6083SPeter Grehan case VM_REG_GUEST_R12: 1453366f6083SPeter Grehan return (&vmxctx->guest_r12); 1454366f6083SPeter Grehan case VM_REG_GUEST_R13: 1455366f6083SPeter Grehan return (&vmxctx->guest_r13); 1456366f6083SPeter Grehan case VM_REG_GUEST_R14: 1457366f6083SPeter Grehan return (&vmxctx->guest_r14); 1458366f6083SPeter Grehan case VM_REG_GUEST_R15: 1459366f6083SPeter Grehan return (&vmxctx->guest_r15); 1460366f6083SPeter Grehan default: 1461366f6083SPeter Grehan break; 1462366f6083SPeter Grehan } 1463366f6083SPeter Grehan return (NULL); 1464366f6083SPeter Grehan } 1465366f6083SPeter Grehan 1466366f6083SPeter Grehan static int 1467366f6083SPeter Grehan vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval) 1468366f6083SPeter Grehan { 1469366f6083SPeter Grehan register_t *regp; 1470366f6083SPeter Grehan 1471366f6083SPeter Grehan if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) { 1472366f6083SPeter Grehan *retval = *regp; 1473366f6083SPeter Grehan return (0); 1474366f6083SPeter Grehan } else 1475366f6083SPeter Grehan return (EINVAL); 1476366f6083SPeter Grehan } 1477366f6083SPeter Grehan 1478366f6083SPeter Grehan static int 1479366f6083SPeter Grehan vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val) 1480366f6083SPeter Grehan { 1481366f6083SPeter Grehan register_t *regp; 1482366f6083SPeter Grehan 1483366f6083SPeter Grehan if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) { 1484366f6083SPeter Grehan *regp = val; 1485366f6083SPeter Grehan return (0); 1486366f6083SPeter Grehan } else 1487366f6083SPeter Grehan return (EINVAL); 1488366f6083SPeter Grehan } 1489366f6083SPeter Grehan 1490366f6083SPeter Grehan static int 1491366f6083SPeter Grehan vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval) 1492366f6083SPeter Grehan { 1493366f6083SPeter Grehan struct vmx *vmx = arg; 1494366f6083SPeter Grehan 1495366f6083SPeter Grehan if (vmxctx_getreg(&vmx->ctx[vcpu], reg, retval) == 0) 1496366f6083SPeter Grehan return (0); 1497366f6083SPeter Grehan 1498366f6083SPeter Grehan /* 1499366f6083SPeter Grehan * If the vcpu is running then don't mess with the VMCS. 1500366f6083SPeter Grehan * 1501366f6083SPeter Grehan * vmcs_getreg will VMCLEAR the vmcs when it is done which will cause 1502366f6083SPeter Grehan * the subsequent vmlaunch/vmresume to fail. 1503366f6083SPeter Grehan */ 1504366f6083SPeter Grehan if (vcpu_is_running(vmx->vm, vcpu, NULL)) 1505366f6083SPeter Grehan panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu); 1506366f6083SPeter Grehan 1507366f6083SPeter Grehan return (vmcs_getreg(&vmx->vmcs[vcpu], reg, retval)); 1508366f6083SPeter Grehan } 1509366f6083SPeter Grehan 1510366f6083SPeter Grehan static int 1511366f6083SPeter Grehan vmx_setreg(void *arg, int vcpu, int reg, uint64_t val) 1512366f6083SPeter Grehan { 1513366f6083SPeter Grehan int error; 1514366f6083SPeter Grehan uint64_t ctls; 1515366f6083SPeter Grehan struct vmx *vmx = arg; 1516366f6083SPeter Grehan 1517366f6083SPeter Grehan /* 1518366f6083SPeter Grehan * XXX Allow caller to set contents of the guest registers saved in 1519366f6083SPeter Grehan * the 'vmxctx' even though the vcpu might be running. We need this 1520366f6083SPeter Grehan * specifically to support the rdmsr emulation that will set the 1521366f6083SPeter Grehan * %eax and %edx registers during vm exit processing. 1522366f6083SPeter Grehan */ 1523366f6083SPeter Grehan if (vmxctx_setreg(&vmx->ctx[vcpu], reg, val) == 0) 1524366f6083SPeter Grehan return (0); 1525366f6083SPeter Grehan 1526366f6083SPeter Grehan /* 1527366f6083SPeter Grehan * If the vcpu is running then don't mess with the VMCS. 1528366f6083SPeter Grehan * 1529366f6083SPeter Grehan * vmcs_setreg will VMCLEAR the vmcs when it is done which will cause 1530366f6083SPeter Grehan * the subsequent vmlaunch/vmresume to fail. 1531366f6083SPeter Grehan */ 1532366f6083SPeter Grehan if (vcpu_is_running(vmx->vm, vcpu, NULL)) 1533366f6083SPeter Grehan panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu); 1534366f6083SPeter Grehan 1535366f6083SPeter Grehan error = vmcs_setreg(&vmx->vmcs[vcpu], reg, val); 1536366f6083SPeter Grehan 1537366f6083SPeter Grehan if (error == 0) { 1538366f6083SPeter Grehan /* 1539366f6083SPeter Grehan * If the "load EFER" VM-entry control is 1 then the 1540366f6083SPeter Grehan * value of EFER.LMA must be identical to "IA-32e mode guest" 1541366f6083SPeter Grehan * bit in the VM-entry control. 1542366f6083SPeter Grehan */ 1543366f6083SPeter Grehan if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 && 1544366f6083SPeter Grehan (reg == VM_REG_GUEST_EFER)) { 1545366f6083SPeter Grehan vmcs_getreg(&vmx->vmcs[vcpu], 1546366f6083SPeter Grehan VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls); 1547366f6083SPeter Grehan if (val & EFER_LMA) 1548366f6083SPeter Grehan ctls |= VM_ENTRY_GUEST_LMA; 1549366f6083SPeter Grehan else 1550366f6083SPeter Grehan ctls &= ~VM_ENTRY_GUEST_LMA; 1551366f6083SPeter Grehan vmcs_setreg(&vmx->vmcs[vcpu], 1552366f6083SPeter Grehan VMCS_IDENT(VMCS_ENTRY_CTLS), ctls); 1553366f6083SPeter Grehan } 1554366f6083SPeter Grehan } 1555366f6083SPeter Grehan 1556366f6083SPeter Grehan return (error); 1557366f6083SPeter Grehan } 1558366f6083SPeter Grehan 1559366f6083SPeter Grehan static int 1560366f6083SPeter Grehan vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc) 1561366f6083SPeter Grehan { 1562366f6083SPeter Grehan struct vmx *vmx = arg; 1563366f6083SPeter Grehan 1564366f6083SPeter Grehan return (vmcs_getdesc(&vmx->vmcs[vcpu], reg, desc)); 1565366f6083SPeter Grehan } 1566366f6083SPeter Grehan 1567366f6083SPeter Grehan static int 1568366f6083SPeter Grehan vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc) 1569366f6083SPeter Grehan { 1570366f6083SPeter Grehan struct vmx *vmx = arg; 1571366f6083SPeter Grehan 1572366f6083SPeter Grehan return (vmcs_setdesc(&vmx->vmcs[vcpu], reg, desc)); 1573366f6083SPeter Grehan } 1574366f6083SPeter Grehan 1575366f6083SPeter Grehan static int 1576366f6083SPeter Grehan vmx_inject(void *arg, int vcpu, int type, int vector, uint32_t code, 1577366f6083SPeter Grehan int code_valid) 1578366f6083SPeter Grehan { 1579366f6083SPeter Grehan int error; 1580366f6083SPeter Grehan uint32_t info; 1581366f6083SPeter Grehan struct vmx *vmx = arg; 1582366f6083SPeter Grehan struct vmcs *vmcs = &vmx->vmcs[vcpu]; 1583366f6083SPeter Grehan 1584366f6083SPeter Grehan static uint32_t type_map[VM_EVENT_MAX] = { 1585366f6083SPeter Grehan 0x1, /* VM_EVENT_NONE */ 1586366f6083SPeter Grehan 0x0, /* VM_HW_INTR */ 1587366f6083SPeter Grehan 0x2, /* VM_NMI */ 1588366f6083SPeter Grehan 0x3, /* VM_HW_EXCEPTION */ 1589366f6083SPeter Grehan 0x4, /* VM_SW_INTR */ 1590366f6083SPeter Grehan 0x5, /* VM_PRIV_SW_EXCEPTION */ 1591366f6083SPeter Grehan 0x6, /* VM_SW_EXCEPTION */ 1592366f6083SPeter Grehan }; 1593366f6083SPeter Grehan 1594366f6083SPeter Grehan info = vector | (type_map[type] << 8) | (code_valid ? 1 << 11 : 0); 1595366f6083SPeter Grehan info |= VMCS_INTERRUPTION_INFO_VALID; 1596366f6083SPeter Grehan error = vmcs_setreg(vmcs, VMCS_IDENT(VMCS_ENTRY_INTR_INFO), info); 1597366f6083SPeter Grehan if (error != 0) 1598366f6083SPeter Grehan return (error); 1599366f6083SPeter Grehan 1600366f6083SPeter Grehan if (code_valid) { 1601366f6083SPeter Grehan error = vmcs_setreg(vmcs, 1602366f6083SPeter Grehan VMCS_IDENT(VMCS_ENTRY_EXCEPTION_ERROR), 1603366f6083SPeter Grehan code); 1604366f6083SPeter Grehan } 1605366f6083SPeter Grehan return (error); 1606366f6083SPeter Grehan } 1607366f6083SPeter Grehan 1608366f6083SPeter Grehan static int 1609366f6083SPeter Grehan vmx_nmi(void *arg, int vcpu) 1610366f6083SPeter Grehan { 1611366f6083SPeter Grehan struct vmx *vmx = arg; 1612366f6083SPeter Grehan 1613366f6083SPeter Grehan atomic_set_int(&vmx->state[vcpu].request_nmi, 1); 1614366f6083SPeter Grehan 1615366f6083SPeter Grehan return (0); 1616366f6083SPeter Grehan } 1617366f6083SPeter Grehan 1618366f6083SPeter Grehan static int 1619366f6083SPeter Grehan vmx_getcap(void *arg, int vcpu, int type, int *retval) 1620366f6083SPeter Grehan { 1621366f6083SPeter Grehan struct vmx *vmx = arg; 1622366f6083SPeter Grehan int vcap; 1623366f6083SPeter Grehan int ret; 1624366f6083SPeter Grehan 1625366f6083SPeter Grehan ret = ENOENT; 1626366f6083SPeter Grehan 1627366f6083SPeter Grehan vcap = vmx->cap[vcpu].set; 1628366f6083SPeter Grehan 1629366f6083SPeter Grehan switch (type) { 1630366f6083SPeter Grehan case VM_CAP_HALT_EXIT: 1631366f6083SPeter Grehan if (cap_halt_exit) 1632366f6083SPeter Grehan ret = 0; 1633366f6083SPeter Grehan break; 1634366f6083SPeter Grehan case VM_CAP_PAUSE_EXIT: 1635366f6083SPeter Grehan if (cap_pause_exit) 1636366f6083SPeter Grehan ret = 0; 1637366f6083SPeter Grehan break; 1638366f6083SPeter Grehan case VM_CAP_MTRAP_EXIT: 1639366f6083SPeter Grehan if (cap_monitor_trap) 1640366f6083SPeter Grehan ret = 0; 1641366f6083SPeter Grehan break; 1642366f6083SPeter Grehan case VM_CAP_UNRESTRICTED_GUEST: 1643366f6083SPeter Grehan if (cap_unrestricted_guest) 1644366f6083SPeter Grehan ret = 0; 1645366f6083SPeter Grehan break; 1646366f6083SPeter Grehan default: 1647366f6083SPeter Grehan break; 1648366f6083SPeter Grehan } 1649366f6083SPeter Grehan 1650366f6083SPeter Grehan if (ret == 0) 1651366f6083SPeter Grehan *retval = (vcap & (1 << type)) ? 1 : 0; 1652366f6083SPeter Grehan 1653366f6083SPeter Grehan return (ret); 1654366f6083SPeter Grehan } 1655366f6083SPeter Grehan 1656366f6083SPeter Grehan static int 1657366f6083SPeter Grehan vmx_setcap(void *arg, int vcpu, int type, int val) 1658366f6083SPeter Grehan { 1659366f6083SPeter Grehan struct vmx *vmx = arg; 1660366f6083SPeter Grehan struct vmcs *vmcs = &vmx->vmcs[vcpu]; 1661366f6083SPeter Grehan uint32_t baseval; 1662366f6083SPeter Grehan uint32_t *pptr; 1663366f6083SPeter Grehan int error; 1664366f6083SPeter Grehan int flag; 1665366f6083SPeter Grehan int reg; 1666366f6083SPeter Grehan int retval; 1667366f6083SPeter Grehan 1668366f6083SPeter Grehan retval = ENOENT; 1669366f6083SPeter Grehan pptr = NULL; 1670366f6083SPeter Grehan 1671366f6083SPeter Grehan switch (type) { 1672366f6083SPeter Grehan case VM_CAP_HALT_EXIT: 1673366f6083SPeter Grehan if (cap_halt_exit) { 1674366f6083SPeter Grehan retval = 0; 1675366f6083SPeter Grehan pptr = &vmx->cap[vcpu].proc_ctls; 1676366f6083SPeter Grehan baseval = *pptr; 1677366f6083SPeter Grehan flag = PROCBASED_HLT_EXITING; 1678366f6083SPeter Grehan reg = VMCS_PRI_PROC_BASED_CTLS; 1679366f6083SPeter Grehan } 1680366f6083SPeter Grehan break; 1681366f6083SPeter Grehan case VM_CAP_MTRAP_EXIT: 1682366f6083SPeter Grehan if (cap_monitor_trap) { 1683366f6083SPeter Grehan retval = 0; 1684366f6083SPeter Grehan pptr = &vmx->cap[vcpu].proc_ctls; 1685366f6083SPeter Grehan baseval = *pptr; 1686366f6083SPeter Grehan flag = PROCBASED_MTF; 1687366f6083SPeter Grehan reg = VMCS_PRI_PROC_BASED_CTLS; 1688366f6083SPeter Grehan } 1689366f6083SPeter Grehan break; 1690366f6083SPeter Grehan case VM_CAP_PAUSE_EXIT: 1691366f6083SPeter Grehan if (cap_pause_exit) { 1692366f6083SPeter Grehan retval = 0; 1693366f6083SPeter Grehan pptr = &vmx->cap[vcpu].proc_ctls; 1694366f6083SPeter Grehan baseval = *pptr; 1695366f6083SPeter Grehan flag = PROCBASED_PAUSE_EXITING; 1696366f6083SPeter Grehan reg = VMCS_PRI_PROC_BASED_CTLS; 1697366f6083SPeter Grehan } 1698366f6083SPeter Grehan break; 1699366f6083SPeter Grehan case VM_CAP_UNRESTRICTED_GUEST: 1700366f6083SPeter Grehan if (cap_unrestricted_guest) { 1701366f6083SPeter Grehan retval = 0; 1702366f6083SPeter Grehan baseval = procbased_ctls2; 1703366f6083SPeter Grehan flag = PROCBASED2_UNRESTRICTED_GUEST; 1704366f6083SPeter Grehan reg = VMCS_SEC_PROC_BASED_CTLS; 1705366f6083SPeter Grehan } 1706366f6083SPeter Grehan break; 1707366f6083SPeter Grehan default: 1708366f6083SPeter Grehan break; 1709366f6083SPeter Grehan } 1710366f6083SPeter Grehan 1711366f6083SPeter Grehan if (retval == 0) { 1712366f6083SPeter Grehan if (val) { 1713366f6083SPeter Grehan baseval |= flag; 1714366f6083SPeter Grehan } else { 1715366f6083SPeter Grehan baseval &= ~flag; 1716366f6083SPeter Grehan } 1717366f6083SPeter Grehan VMPTRLD(vmcs); 1718366f6083SPeter Grehan error = vmwrite(reg, baseval); 1719366f6083SPeter Grehan VMCLEAR(vmcs); 1720366f6083SPeter Grehan 1721366f6083SPeter Grehan if (error) { 1722366f6083SPeter Grehan retval = error; 1723366f6083SPeter Grehan } else { 1724366f6083SPeter Grehan /* 1725366f6083SPeter Grehan * Update optional stored flags, and record 1726366f6083SPeter Grehan * setting 1727366f6083SPeter Grehan */ 1728366f6083SPeter Grehan if (pptr != NULL) { 1729366f6083SPeter Grehan *pptr = baseval; 1730366f6083SPeter Grehan } 1731366f6083SPeter Grehan 1732366f6083SPeter Grehan if (val) { 1733366f6083SPeter Grehan vmx->cap[vcpu].set |= (1 << type); 1734366f6083SPeter Grehan } else { 1735366f6083SPeter Grehan vmx->cap[vcpu].set &= ~(1 << type); 1736366f6083SPeter Grehan } 1737366f6083SPeter Grehan } 1738366f6083SPeter Grehan } 1739366f6083SPeter Grehan 1740366f6083SPeter Grehan return (retval); 1741366f6083SPeter Grehan } 1742366f6083SPeter Grehan 1743366f6083SPeter Grehan struct vmm_ops vmm_ops_intel = { 1744366f6083SPeter Grehan vmx_init, 1745366f6083SPeter Grehan vmx_cleanup, 1746366f6083SPeter Grehan vmx_vminit, 1747366f6083SPeter Grehan vmx_run, 1748366f6083SPeter Grehan vmx_vmcleanup, 1749366f6083SPeter Grehan ept_vmmmap, 1750366f6083SPeter Grehan vmx_getreg, 1751366f6083SPeter Grehan vmx_setreg, 1752366f6083SPeter Grehan vmx_getdesc, 1753366f6083SPeter Grehan vmx_setdesc, 1754366f6083SPeter Grehan vmx_inject, 1755366f6083SPeter Grehan vmx_nmi, 1756366f6083SPeter Grehan vmx_getcap, 1757366f6083SPeter Grehan vmx_setcap 1758366f6083SPeter Grehan }; 1759