1366f6083SPeter Grehan /*- 2c49761ddSPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3c49761ddSPedro F. Giffuni * 4366f6083SPeter Grehan * Copyright (c) 2011 NetApp, Inc. 5366f6083SPeter Grehan * All rights reserved. 6366f6083SPeter Grehan * 7366f6083SPeter Grehan * Redistribution and use in source and binary forms, with or without 8366f6083SPeter Grehan * modification, are permitted provided that the following conditions 9366f6083SPeter Grehan * are met: 10366f6083SPeter Grehan * 1. Redistributions of source code must retain the above copyright 11366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer. 12366f6083SPeter Grehan * 2. Redistributions in binary form must reproduce the above copyright 13366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer in the 14366f6083SPeter Grehan * documentation and/or other materials provided with the distribution. 15366f6083SPeter Grehan * 16366f6083SPeter Grehan * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 17366f6083SPeter Grehan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18366f6083SPeter Grehan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19366f6083SPeter Grehan * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 20366f6083SPeter Grehan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21366f6083SPeter Grehan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22366f6083SPeter Grehan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23366f6083SPeter Grehan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24366f6083SPeter Grehan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25366f6083SPeter Grehan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26366f6083SPeter Grehan * SUCH DAMAGE. 27366f6083SPeter Grehan * 28366f6083SPeter Grehan * $FreeBSD$ 29366f6083SPeter Grehan */ 30366f6083SPeter Grehan 31366f6083SPeter Grehan #include <sys/cdefs.h> 32366f6083SPeter Grehan __FBSDID("$FreeBSD$"); 33366f6083SPeter Grehan 34366f6083SPeter Grehan #include <sys/param.h> 3538f1b189SPeter Grehan #include <sys/systm.h> 36366f6083SPeter Grehan #include <sys/kernel.h> 37366f6083SPeter Grehan #include <sys/module.h> 38366f6083SPeter Grehan #include <sys/sysctl.h> 39366f6083SPeter Grehan #include <sys/malloc.h> 40366f6083SPeter Grehan #include <sys/pcpu.h> 41366f6083SPeter Grehan #include <sys/lock.h> 42366f6083SPeter Grehan #include <sys/mutex.h> 43366f6083SPeter Grehan #include <sys/proc.h> 44318224bbSNeel Natu #include <sys/rwlock.h> 45366f6083SPeter Grehan #include <sys/sched.h> 46366f6083SPeter Grehan #include <sys/smp.h> 47366f6083SPeter Grehan #include <sys/systm.h> 48366f6083SPeter Grehan 49366f6083SPeter Grehan #include <vm/vm.h> 50318224bbSNeel Natu #include <vm/vm_object.h> 51318224bbSNeel Natu #include <vm/vm_page.h> 52318224bbSNeel Natu #include <vm/pmap.h> 53318224bbSNeel Natu #include <vm/vm_map.h> 54318224bbSNeel Natu #include <vm/vm_extern.h> 55318224bbSNeel Natu #include <vm/vm_param.h> 56366f6083SPeter Grehan 5763e62d39SJohn Baldwin #include <machine/cpu.h> 58366f6083SPeter Grehan #include <machine/pcb.h> 5975dd3366SNeel Natu #include <machine/smp.h> 60bd50262fSKonstantin Belousov #include <machine/md_var.h> 611c052192SNeel Natu #include <x86/psl.h> 6234a6b2d6SJohn Baldwin #include <x86/apicreg.h> 63366f6083SPeter Grehan 64366f6083SPeter Grehan #include <machine/vmm.h> 65565bbb86SNeel Natu #include <machine/vmm_dev.h> 66e813a873SNeel Natu #include <machine/vmm_instruction_emul.h> 67565bbb86SNeel Natu 68d17b5104SNeel Natu #include "vmm_ioport.h" 69318224bbSNeel Natu #include "vmm_ktr.h" 70b01c2033SNeel Natu #include "vmm_host.h" 71366f6083SPeter Grehan #include "vmm_mem.h" 72366f6083SPeter Grehan #include "vmm_util.h" 73762fd208STycho Nightingale #include "vatpic.h" 74e883c9bbSTycho Nightingale #include "vatpit.h" 7508e3ff32SNeel Natu #include "vhpet.h" 76565bbb86SNeel Natu #include "vioapic.h" 77366f6083SPeter Grehan #include "vlapic.h" 78160ef77aSNeel Natu #include "vpmtmr.h" 790dafa5cdSNeel Natu #include "vrtc.h" 80366f6083SPeter Grehan #include "vmm_stat.h" 81f76fc5d4SNeel Natu #include "vmm_lapic.h" 82366f6083SPeter Grehan 83366f6083SPeter Grehan #include "io/ppt.h" 84366f6083SPeter Grehan #include "io/iommu.h" 85366f6083SPeter Grehan 86366f6083SPeter Grehan struct vlapic; 87366f6083SPeter Grehan 885fcf252fSNeel Natu /* 895fcf252fSNeel Natu * Initialization: 905fcf252fSNeel Natu * (a) allocated when vcpu is created 915fcf252fSNeel Natu * (i) initialized when vcpu is created and when it is reinitialized 925fcf252fSNeel Natu * (o) initialized the first time the vcpu is created 935fcf252fSNeel Natu * (x) initialized before use 945fcf252fSNeel Natu */ 95366f6083SPeter Grehan struct vcpu { 965fcf252fSNeel Natu struct mtx mtx; /* (o) protects 'state' and 'hostcpu' */ 975fcf252fSNeel Natu enum vcpu_state state; /* (o) vcpu state */ 985fcf252fSNeel Natu int hostcpu; /* (o) vcpu's host cpu */ 99248e6799SNeel Natu int reqidle; /* (i) request vcpu to idle */ 1005fcf252fSNeel Natu struct vlapic *vlapic; /* (i) APIC device model */ 1015fcf252fSNeel Natu enum x2apic_state x2apic_state; /* (i) APIC mode */ 102091d4532SNeel Natu uint64_t exitintinfo; /* (i) events pending at VM exit */ 1035fcf252fSNeel Natu int nmi_pending; /* (i) NMI pending */ 1045fcf252fSNeel Natu int extint_pending; /* (i) INTR pending */ 1055fcf252fSNeel Natu int exception_pending; /* (i) exception pending */ 106c9c75df4SNeel Natu int exc_vector; /* (x) exception collateral */ 107c9c75df4SNeel Natu int exc_errcode_valid; 108c9c75df4SNeel Natu uint32_t exc_errcode; 1095fcf252fSNeel Natu struct savefpu *guestfpu; /* (a,i) guest fpu state */ 1105fcf252fSNeel Natu uint64_t guest_xcr0; /* (i) guest %xcr0 register */ 1115fcf252fSNeel Natu void *stats; /* (a,i) statistics */ 1125fcf252fSNeel Natu struct vm_exit exitinfo; /* (x) exit reason and collateral */ 113d087a399SNeel Natu uint64_t nextrip; /* (x) next instruction to execute */ 114366f6083SPeter Grehan }; 115366f6083SPeter Grehan 1165fcf252fSNeel Natu #define vcpu_lock_initialized(v) mtx_initialized(&((v)->mtx)) 117f76fc5d4SNeel Natu #define vcpu_lock_init(v) mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN) 118f76fc5d4SNeel Natu #define vcpu_lock(v) mtx_lock_spin(&((v)->mtx)) 119f76fc5d4SNeel Natu #define vcpu_unlock(v) mtx_unlock_spin(&((v)->mtx)) 120318224bbSNeel Natu #define vcpu_assert_locked(v) mtx_assert(&((v)->mtx), MA_OWNED) 12175dd3366SNeel Natu 122318224bbSNeel Natu struct mem_seg { 1239b1aa8d6SNeel Natu size_t len; 1249b1aa8d6SNeel Natu bool sysmem; 1259b1aa8d6SNeel Natu struct vm_object *object; 1269b1aa8d6SNeel Natu }; 1276bcf245eSMarcel Moolenaar #define VM_MAX_MEMSEGS 3 1289b1aa8d6SNeel Natu 1299b1aa8d6SNeel Natu struct mem_map { 130318224bbSNeel Natu vm_paddr_t gpa; 131318224bbSNeel Natu size_t len; 1329b1aa8d6SNeel Natu vm_ooffset_t segoff; 1339b1aa8d6SNeel Natu int segid; 1349b1aa8d6SNeel Natu int prot; 1359b1aa8d6SNeel Natu int flags; 136318224bbSNeel Natu }; 1379b1aa8d6SNeel Natu #define VM_MAX_MEMMAPS 4 138366f6083SPeter Grehan 139366f6083SPeter Grehan /* 1405fcf252fSNeel Natu * Initialization: 1415fcf252fSNeel Natu * (o) initialized the first time the VM is created 1425fcf252fSNeel Natu * (i) initialized when VM is created and when it is reinitialized 1435fcf252fSNeel Natu * (x) initialized before use 144366f6083SPeter Grehan */ 1455fcf252fSNeel Natu struct vm { 1465fcf252fSNeel Natu void *cookie; /* (i) cpu-specific data */ 1475fcf252fSNeel Natu void *iommu; /* (x) iommu-specific data */ 1485fcf252fSNeel Natu struct vhpet *vhpet; /* (i) virtual HPET */ 1495fcf252fSNeel Natu struct vioapic *vioapic; /* (i) virtual ioapic */ 1505fcf252fSNeel Natu struct vatpic *vatpic; /* (i) virtual atpic */ 1515fcf252fSNeel Natu struct vatpit *vatpit; /* (i) virtual atpit */ 152160ef77aSNeel Natu struct vpmtmr *vpmtmr; /* (i) virtual ACPI PM timer */ 1530dafa5cdSNeel Natu struct vrtc *vrtc; /* (o) virtual RTC */ 1545fcf252fSNeel Natu volatile cpuset_t active_cpus; /* (i) active vcpus */ 155fc276d92SJohn Baldwin volatile cpuset_t debug_cpus; /* (i) vcpus stopped for debug */ 1565fcf252fSNeel Natu int suspend; /* (i) stop VM execution */ 1575fcf252fSNeel Natu volatile cpuset_t suspended_cpus; /* (i) suspended vcpus */ 1585fcf252fSNeel Natu volatile cpuset_t halted_cpus; /* (x) cpus in a hard halt */ 1595fcf252fSNeel Natu cpuset_t rendezvous_req_cpus; /* (x) rendezvous requested */ 1605fcf252fSNeel Natu cpuset_t rendezvous_done_cpus; /* (x) rendezvous finished */ 1615fcf252fSNeel Natu void *rendezvous_arg; /* (x) rendezvous func/arg */ 1625b8a8cd1SNeel Natu vm_rendezvous_func_t rendezvous_func; 1635fcf252fSNeel Natu struct mtx rendezvous_mtx; /* (o) rendezvous lock */ 1649b1aa8d6SNeel Natu struct mem_map mem_maps[VM_MAX_MEMMAPS]; /* (i) guest address space */ 1659b1aa8d6SNeel Natu struct mem_seg mem_segs[VM_MAX_MEMSEGS]; /* (o) guest memory regions */ 1665fcf252fSNeel Natu struct vmspace *vmspace; /* (o) guest's address space */ 1675fcf252fSNeel Natu char name[VM_MAX_NAMELEN]; /* (o) virtual machine name */ 1685fcf252fSNeel Natu struct vcpu vcpu[VM_MAXCPU]; /* (i) guest vcpus */ 16901d822d3SRodney W. Grimes /* The following describe the vm cpu topology */ 17001d822d3SRodney W. Grimes uint16_t sockets; /* (o) num of sockets */ 17101d822d3SRodney W. Grimes uint16_t cores; /* (o) num of cores/socket */ 17201d822d3SRodney W. Grimes uint16_t threads; /* (o) num of threads/core */ 17301d822d3SRodney W. Grimes uint16_t maxcpus; /* (o) max pluggable cpus */ 174366f6083SPeter Grehan }; 175366f6083SPeter Grehan 176d5408b1dSNeel Natu static int vmm_initialized; 177d5408b1dSNeel Natu 178366f6083SPeter Grehan static struct vmm_ops *ops; 179add611fdSNeel Natu #define VMM_INIT(num) (ops != NULL ? (*ops->init)(num) : 0) 180366f6083SPeter Grehan #define VMM_CLEANUP() (ops != NULL ? (*ops->cleanup)() : 0) 18163e62d39SJohn Baldwin #define VMM_RESUME() (ops != NULL ? (*ops->resume)() : 0) 182366f6083SPeter Grehan 183318224bbSNeel Natu #define VMINIT(vm, pmap) (ops != NULL ? (*ops->vminit)(vm, pmap): NULL) 184248e6799SNeel Natu #define VMRUN(vmi, vcpu, rip, pmap, evinfo) \ 185248e6799SNeel Natu (ops != NULL ? (*ops->vmrun)(vmi, vcpu, rip, pmap, evinfo) : ENXIO) 186366f6083SPeter Grehan #define VMCLEANUP(vmi) (ops != NULL ? (*ops->vmcleanup)(vmi) : NULL) 187318224bbSNeel Natu #define VMSPACE_ALLOC(min, max) \ 188318224bbSNeel Natu (ops != NULL ? (*ops->vmspace_alloc)(min, max) : NULL) 189318224bbSNeel Natu #define VMSPACE_FREE(vmspace) \ 190318224bbSNeel Natu (ops != NULL ? (*ops->vmspace_free)(vmspace) : ENXIO) 191366f6083SPeter Grehan #define VMGETREG(vmi, vcpu, num, retval) \ 192366f6083SPeter Grehan (ops != NULL ? (*ops->vmgetreg)(vmi, vcpu, num, retval) : ENXIO) 193366f6083SPeter Grehan #define VMSETREG(vmi, vcpu, num, val) \ 194366f6083SPeter Grehan (ops != NULL ? (*ops->vmsetreg)(vmi, vcpu, num, val) : ENXIO) 195366f6083SPeter Grehan #define VMGETDESC(vmi, vcpu, num, desc) \ 196366f6083SPeter Grehan (ops != NULL ? (*ops->vmgetdesc)(vmi, vcpu, num, desc) : ENXIO) 197366f6083SPeter Grehan #define VMSETDESC(vmi, vcpu, num, desc) \ 198366f6083SPeter Grehan (ops != NULL ? (*ops->vmsetdesc)(vmi, vcpu, num, desc) : ENXIO) 199366f6083SPeter Grehan #define VMGETCAP(vmi, vcpu, num, retval) \ 200366f6083SPeter Grehan (ops != NULL ? (*ops->vmgetcap)(vmi, vcpu, num, retval) : ENXIO) 201366f6083SPeter Grehan #define VMSETCAP(vmi, vcpu, num, val) \ 202366f6083SPeter Grehan (ops != NULL ? (*ops->vmsetcap)(vmi, vcpu, num, val) : ENXIO) 203de5ea6b6SNeel Natu #define VLAPIC_INIT(vmi, vcpu) \ 204de5ea6b6SNeel Natu (ops != NULL ? (*ops->vlapic_init)(vmi, vcpu) : NULL) 205de5ea6b6SNeel Natu #define VLAPIC_CLEANUP(vmi, vlapic) \ 206de5ea6b6SNeel Natu (ops != NULL ? (*ops->vlapic_cleanup)(vmi, vlapic) : NULL) 207366f6083SPeter Grehan 208014a52f3SNeel Natu #define fpu_start_emulating() load_cr0(rcr0() | CR0_TS) 209014a52f3SNeel Natu #define fpu_stop_emulating() clts() 210366f6083SPeter Grehan 2116ac73777STycho Nightingale SDT_PROVIDER_DEFINE(vmm); 2126ac73777STycho Nightingale 213366f6083SPeter Grehan static MALLOC_DEFINE(M_VM, "vm", "vm"); 214366f6083SPeter Grehan 215366f6083SPeter Grehan /* statistics */ 21661592433SNeel Natu static VMM_STAT(VCPU_TOTAL_RUNTIME, "vcpu total runtime"); 217366f6083SPeter Grehan 218add611fdSNeel Natu SYSCTL_NODE(_hw, OID_AUTO, vmm, CTLFLAG_RW, NULL, NULL); 219add611fdSNeel Natu 220055fc2cbSNeel Natu /* 221055fc2cbSNeel Natu * Halt the guest if all vcpus are executing a HLT instruction with 222055fc2cbSNeel Natu * interrupts disabled. 223055fc2cbSNeel Natu */ 224055fc2cbSNeel Natu static int halt_detection_enabled = 1; 225055fc2cbSNeel Natu SYSCTL_INT(_hw_vmm, OID_AUTO, halt_detection, CTLFLAG_RDTUN, 226055fc2cbSNeel Natu &halt_detection_enabled, 0, 227055fc2cbSNeel Natu "Halt VM if all vcpus execute HLT with interrupts disabled"); 228055fc2cbSNeel Natu 229978f3da1SAndriy Gapon static int vmm_ipinum; 230add611fdSNeel Natu SYSCTL_INT(_hw_vmm, OID_AUTO, ipinum, CTLFLAG_RD, &vmm_ipinum, 0, 231add611fdSNeel Natu "IPI vector used for vcpu notifications"); 232add611fdSNeel Natu 233b0538143SNeel Natu static int trace_guest_exceptions; 234b0538143SNeel Natu SYSCTL_INT(_hw_vmm, OID_AUTO, trace_guest_exceptions, CTLFLAG_RDTUN, 235b0538143SNeel Natu &trace_guest_exceptions, 0, 236b0538143SNeel Natu "Trap into hypervisor on all guest exceptions and reflect them back"); 237b0538143SNeel Natu 2389b1aa8d6SNeel Natu static void vm_free_memmap(struct vm *vm, int ident); 2399b1aa8d6SNeel Natu static bool sysmem_mapping(struct vm *vm, struct mem_map *mm); 240248e6799SNeel Natu static void vcpu_notify_event_locked(struct vcpu *vcpu, bool lapic_intr); 241248e6799SNeel Natu 242248e6799SNeel Natu #ifdef KTR 243248e6799SNeel Natu static const char * 244248e6799SNeel Natu vcpu_state2str(enum vcpu_state state) 245248e6799SNeel Natu { 246248e6799SNeel Natu 247248e6799SNeel Natu switch (state) { 248248e6799SNeel Natu case VCPU_IDLE: 249248e6799SNeel Natu return ("idle"); 250248e6799SNeel Natu case VCPU_FROZEN: 251248e6799SNeel Natu return ("frozen"); 252248e6799SNeel Natu case VCPU_RUNNING: 253248e6799SNeel Natu return ("running"); 254248e6799SNeel Natu case VCPU_SLEEPING: 255248e6799SNeel Natu return ("sleeping"); 256248e6799SNeel Natu default: 257248e6799SNeel Natu return ("unknown"); 258248e6799SNeel Natu } 259248e6799SNeel Natu } 260248e6799SNeel Natu #endif 261248e6799SNeel Natu 262366f6083SPeter Grehan static void 2635fcf252fSNeel Natu vcpu_cleanup(struct vm *vm, int i, bool destroy) 264366f6083SPeter Grehan { 265de5ea6b6SNeel Natu struct vcpu *vcpu = &vm->vcpu[i]; 266de5ea6b6SNeel Natu 267de5ea6b6SNeel Natu VLAPIC_CLEANUP(vm->cookie, vcpu->vlapic); 2685fcf252fSNeel Natu if (destroy) { 269366f6083SPeter Grehan vmm_stat_free(vcpu->stats); 27038f1b189SPeter Grehan fpu_save_area_free(vcpu->guestfpu); 271366f6083SPeter Grehan } 2725fcf252fSNeel Natu } 273366f6083SPeter Grehan 274366f6083SPeter Grehan static void 2755fcf252fSNeel Natu vcpu_init(struct vm *vm, int vcpu_id, bool create) 276366f6083SPeter Grehan { 277366f6083SPeter Grehan struct vcpu *vcpu; 278366f6083SPeter Grehan 2795fcf252fSNeel Natu KASSERT(vcpu_id >= 0 && vcpu_id < VM_MAXCPU, 2805fcf252fSNeel Natu ("vcpu_init: invalid vcpu %d", vcpu_id)); 2815fcf252fSNeel Natu 282366f6083SPeter Grehan vcpu = &vm->vcpu[vcpu_id]; 283366f6083SPeter Grehan 2845fcf252fSNeel Natu if (create) { 2855fcf252fSNeel Natu KASSERT(!vcpu_lock_initialized(vcpu), ("vcpu %d already " 2865fcf252fSNeel Natu "initialized", vcpu_id)); 28775dd3366SNeel Natu vcpu_lock_init(vcpu); 2885fcf252fSNeel Natu vcpu->state = VCPU_IDLE; 28975dd3366SNeel Natu vcpu->hostcpu = NOCPU; 2905fcf252fSNeel Natu vcpu->guestfpu = fpu_save_area_alloc(); 2915fcf252fSNeel Natu vcpu->stats = vmm_stat_alloc(); 2925fcf252fSNeel Natu } 2935fcf252fSNeel Natu 294de5ea6b6SNeel Natu vcpu->vlapic = VLAPIC_INIT(vm->cookie, vcpu_id); 29552e5c8a2SNeel Natu vm_set_x2apic_state(vm, vcpu_id, X2APIC_DISABLED); 296248e6799SNeel Natu vcpu->reqidle = 0; 297091d4532SNeel Natu vcpu->exitintinfo = 0; 2985fcf252fSNeel Natu vcpu->nmi_pending = 0; 2995fcf252fSNeel Natu vcpu->extint_pending = 0; 3005fcf252fSNeel Natu vcpu->exception_pending = 0; 301abb023fbSJohn Baldwin vcpu->guest_xcr0 = XFEATURE_ENABLED_X87; 30238f1b189SPeter Grehan fpu_save_area_reset(vcpu->guestfpu); 3035fcf252fSNeel Natu vmm_stat_init(vcpu->stats); 304366f6083SPeter Grehan } 305366f6083SPeter Grehan 306b0538143SNeel Natu int 307b0538143SNeel Natu vcpu_trace_exceptions(struct vm *vm, int vcpuid) 308b0538143SNeel Natu { 309b0538143SNeel Natu 310b0538143SNeel Natu return (trace_guest_exceptions); 311b0538143SNeel Natu } 312b0538143SNeel Natu 31398ed632cSNeel Natu struct vm_exit * 31498ed632cSNeel Natu vm_exitinfo(struct vm *vm, int cpuid) 31598ed632cSNeel Natu { 31698ed632cSNeel Natu struct vcpu *vcpu; 31798ed632cSNeel Natu 31898ed632cSNeel Natu if (cpuid < 0 || cpuid >= VM_MAXCPU) 31998ed632cSNeel Natu panic("vm_exitinfo: invalid cpuid %d", cpuid); 32098ed632cSNeel Natu 32198ed632cSNeel Natu vcpu = &vm->vcpu[cpuid]; 32298ed632cSNeel Natu 32398ed632cSNeel Natu return (&vcpu->exitinfo); 32498ed632cSNeel Natu } 32598ed632cSNeel Natu 32663e62d39SJohn Baldwin static void 32763e62d39SJohn Baldwin vmm_resume(void) 32863e62d39SJohn Baldwin { 32963e62d39SJohn Baldwin VMM_RESUME(); 33063e62d39SJohn Baldwin } 33163e62d39SJohn Baldwin 332366f6083SPeter Grehan static int 333366f6083SPeter Grehan vmm_init(void) 334366f6083SPeter Grehan { 335366f6083SPeter Grehan int error; 336366f6083SPeter Grehan 337b01c2033SNeel Natu vmm_host_state_init(); 338add611fdSNeel Natu 339bd50262fSKonstantin Belousov vmm_ipinum = lapic_ipi_alloc(pti ? &IDTVEC(justreturn1_pti) : 340bd50262fSKonstantin Belousov &IDTVEC(justreturn)); 34118a2b08eSNeel Natu if (vmm_ipinum < 0) 342add611fdSNeel Natu vmm_ipinum = IPI_AST; 343366f6083SPeter Grehan 344366f6083SPeter Grehan error = vmm_mem_init(); 345366f6083SPeter Grehan if (error) 346366f6083SPeter Grehan return (error); 347366f6083SPeter Grehan 348366f6083SPeter Grehan if (vmm_is_intel()) 349366f6083SPeter Grehan ops = &vmm_ops_intel; 350366f6083SPeter Grehan else if (vmm_is_amd()) 351366f6083SPeter Grehan ops = &vmm_ops_amd; 352366f6083SPeter Grehan else 353366f6083SPeter Grehan return (ENXIO); 354366f6083SPeter Grehan 35563e62d39SJohn Baldwin vmm_resume_p = vmm_resume; 356366f6083SPeter Grehan 357add611fdSNeel Natu return (VMM_INIT(vmm_ipinum)); 358366f6083SPeter Grehan } 359366f6083SPeter Grehan 360366f6083SPeter Grehan static int 361366f6083SPeter Grehan vmm_handler(module_t mod, int what, void *arg) 362366f6083SPeter Grehan { 363366f6083SPeter Grehan int error; 364366f6083SPeter Grehan 365366f6083SPeter Grehan switch (what) { 366366f6083SPeter Grehan case MOD_LOAD: 367366f6083SPeter Grehan vmmdev_init(); 368366f6083SPeter Grehan error = vmm_init(); 369d5408b1dSNeel Natu if (error == 0) 370d5408b1dSNeel Natu vmm_initialized = 1; 371366f6083SPeter Grehan break; 372366f6083SPeter Grehan case MOD_UNLOAD: 373cdc5b9e7SNeel Natu error = vmmdev_cleanup(); 374cdc5b9e7SNeel Natu if (error == 0) { 37563e62d39SJohn Baldwin vmm_resume_p = NULL; 376366f6083SPeter Grehan iommu_cleanup(); 377add611fdSNeel Natu if (vmm_ipinum != IPI_AST) 37818a2b08eSNeel Natu lapic_ipi_free(vmm_ipinum); 379366f6083SPeter Grehan error = VMM_CLEANUP(); 38081ef6611SPeter Grehan /* 38181ef6611SPeter Grehan * Something bad happened - prevent new 38281ef6611SPeter Grehan * VMs from being created 38381ef6611SPeter Grehan */ 38481ef6611SPeter Grehan if (error) 385d5408b1dSNeel Natu vmm_initialized = 0; 38681ef6611SPeter Grehan } 387366f6083SPeter Grehan break; 388366f6083SPeter Grehan default: 389366f6083SPeter Grehan error = 0; 390366f6083SPeter Grehan break; 391366f6083SPeter Grehan } 392366f6083SPeter Grehan return (error); 393366f6083SPeter Grehan } 394366f6083SPeter Grehan 395366f6083SPeter Grehan static moduledata_t vmm_kmod = { 396366f6083SPeter Grehan "vmm", 397366f6083SPeter Grehan vmm_handler, 398366f6083SPeter Grehan NULL 399366f6083SPeter Grehan }; 400366f6083SPeter Grehan 401366f6083SPeter Grehan /* 402e3f0800bSNeel Natu * vmm initialization has the following dependencies: 403e3f0800bSNeel Natu * 404e3f0800bSNeel Natu * - VT-x initialization requires smp_rendezvous() and therefore must happen 405e3f0800bSNeel Natu * after SMP is fully functional (after SI_SUB_SMP). 406366f6083SPeter Grehan */ 407e3f0800bSNeel Natu DECLARE_MODULE(vmm, vmm_kmod, SI_SUB_SMP + 1, SI_ORDER_ANY); 408366f6083SPeter Grehan MODULE_VERSION(vmm, 1); 409366f6083SPeter Grehan 4105fcf252fSNeel Natu static void 4115fcf252fSNeel Natu vm_init(struct vm *vm, bool create) 4125fcf252fSNeel Natu { 4135fcf252fSNeel Natu int i; 4145fcf252fSNeel Natu 4155fcf252fSNeel Natu vm->cookie = VMINIT(vm, vmspace_pmap(vm->vmspace)); 4165fcf252fSNeel Natu vm->iommu = NULL; 4175fcf252fSNeel Natu vm->vioapic = vioapic_init(vm); 4185fcf252fSNeel Natu vm->vhpet = vhpet_init(vm); 4195fcf252fSNeel Natu vm->vatpic = vatpic_init(vm); 4205fcf252fSNeel Natu vm->vatpit = vatpit_init(vm); 421160ef77aSNeel Natu vm->vpmtmr = vpmtmr_init(vm); 4220dafa5cdSNeel Natu if (create) 4230dafa5cdSNeel Natu vm->vrtc = vrtc_init(vm); 4245fcf252fSNeel Natu 4255fcf252fSNeel Natu CPU_ZERO(&vm->active_cpus); 426fc276d92SJohn Baldwin CPU_ZERO(&vm->debug_cpus); 4275fcf252fSNeel Natu 4285fcf252fSNeel Natu vm->suspend = 0; 4295fcf252fSNeel Natu CPU_ZERO(&vm->suspended_cpus); 4305fcf252fSNeel Natu 4315fcf252fSNeel Natu for (i = 0; i < VM_MAXCPU; i++) 4325fcf252fSNeel Natu vcpu_init(vm, i, create); 4335fcf252fSNeel Natu } 4345fcf252fSNeel Natu 43501d822d3SRodney W. Grimes /* 43601d822d3SRodney W. Grimes * The default CPU topology is a single thread per package. 43701d822d3SRodney W. Grimes */ 43801d822d3SRodney W. Grimes u_int cores_per_package = 1; 43901d822d3SRodney W. Grimes u_int threads_per_core = 1; 44001d822d3SRodney W. Grimes 441d5408b1dSNeel Natu int 442d5408b1dSNeel Natu vm_create(const char *name, struct vm **retvm) 443366f6083SPeter Grehan { 444366f6083SPeter Grehan struct vm *vm; 445318224bbSNeel Natu struct vmspace *vmspace; 446366f6083SPeter Grehan 447d5408b1dSNeel Natu /* 448d5408b1dSNeel Natu * If vmm.ko could not be successfully initialized then don't attempt 449d5408b1dSNeel Natu * to create the virtual machine. 450d5408b1dSNeel Natu */ 451d5408b1dSNeel Natu if (!vmm_initialized) 452d5408b1dSNeel Natu return (ENXIO); 453d5408b1dSNeel Natu 454366f6083SPeter Grehan if (name == NULL || strlen(name) >= VM_MAX_NAMELEN) 455d5408b1dSNeel Natu return (EINVAL); 456366f6083SPeter Grehan 457526c8885SPeter Grehan vmspace = VMSPACE_ALLOC(0, VM_MAXUSER_ADDRESS); 458318224bbSNeel Natu if (vmspace == NULL) 459318224bbSNeel Natu return (ENOMEM); 460318224bbSNeel Natu 461366f6083SPeter Grehan vm = malloc(sizeof(struct vm), M_VM, M_WAITOK | M_ZERO); 462366f6083SPeter Grehan strcpy(vm->name, name); 46388c4b8d1SNeel Natu vm->vmspace = vmspace; 4645b8a8cd1SNeel Natu mtx_init(&vm->rendezvous_mtx, "vm rendezvous lock", 0, MTX_DEF); 465366f6083SPeter Grehan 46601d822d3SRodney W. Grimes vm->sockets = 1; 46701d822d3SRodney W. Grimes vm->cores = cores_per_package; /* XXX backwards compatibility */ 46801d822d3SRodney W. Grimes vm->threads = threads_per_core; /* XXX backwards compatibility */ 46901d822d3SRodney W. Grimes vm->maxcpus = 0; /* XXX not implemented */ 47001d822d3SRodney W. Grimes 4715fcf252fSNeel Natu vm_init(vm, true); 472366f6083SPeter Grehan 473d5408b1dSNeel Natu *retvm = vm; 474d5408b1dSNeel Natu return (0); 475366f6083SPeter Grehan } 476366f6083SPeter Grehan 47701d822d3SRodney W. Grimes void 47801d822d3SRodney W. Grimes vm_get_topology(struct vm *vm, uint16_t *sockets, uint16_t *cores, 47901d822d3SRodney W. Grimes uint16_t *threads, uint16_t *maxcpus) 48001d822d3SRodney W. Grimes { 48101d822d3SRodney W. Grimes *sockets = vm->sockets; 48201d822d3SRodney W. Grimes *cores = vm->cores; 48301d822d3SRodney W. Grimes *threads = vm->threads; 48401d822d3SRodney W. Grimes *maxcpus = vm->maxcpus; 48501d822d3SRodney W. Grimes } 48601d822d3SRodney W. Grimes 48701d822d3SRodney W. Grimes int 48801d822d3SRodney W. Grimes vm_set_topology(struct vm *vm, uint16_t sockets, uint16_t cores, 48901d822d3SRodney W. Grimes uint16_t threads, uint16_t maxcpus) 49001d822d3SRodney W. Grimes { 49101d822d3SRodney W. Grimes if (maxcpus != 0) 49201d822d3SRodney W. Grimes return (EINVAL); /* XXX remove when supported */ 49301d822d3SRodney W. Grimes if ((sockets * cores * threads) > VM_MAXCPU) 49401d822d3SRodney W. Grimes return (EINVAL); 49501d822d3SRodney W. Grimes /* XXX need to check sockets * cores * threads == vCPU, how? */ 49601d822d3SRodney W. Grimes vm->sockets = sockets; 49701d822d3SRodney W. Grimes vm->cores = cores; 49801d822d3SRodney W. Grimes vm->threads = threads; 49901d822d3SRodney W. Grimes vm->maxcpus = maxcpus; 50001d822d3SRodney W. Grimes return(0); 50101d822d3SRodney W. Grimes } 50201d822d3SRodney W. Grimes 503f7d51510SNeel Natu static void 5045fcf252fSNeel Natu vm_cleanup(struct vm *vm, bool destroy) 505366f6083SPeter Grehan { 5069b1aa8d6SNeel Natu struct mem_map *mm; 507366f6083SPeter Grehan int i; 508366f6083SPeter Grehan 509366f6083SPeter Grehan ppt_unassign_all(vm); 510366f6083SPeter Grehan 511318224bbSNeel Natu if (vm->iommu != NULL) 512318224bbSNeel Natu iommu_destroy_domain(vm->iommu); 513318224bbSNeel Natu 5140dafa5cdSNeel Natu if (destroy) 5150dafa5cdSNeel Natu vrtc_cleanup(vm->vrtc); 5160dafa5cdSNeel Natu else 5170dafa5cdSNeel Natu vrtc_reset(vm->vrtc); 518160ef77aSNeel Natu vpmtmr_cleanup(vm->vpmtmr); 519e883c9bbSTycho Nightingale vatpit_cleanup(vm->vatpit); 52008e3ff32SNeel Natu vhpet_cleanup(vm->vhpet); 521762fd208STycho Nightingale vatpic_cleanup(vm->vatpic); 52208e3ff32SNeel Natu vioapic_cleanup(vm->vioapic); 52308e3ff32SNeel Natu 5245fcf252fSNeel Natu for (i = 0; i < VM_MAXCPU; i++) 5255fcf252fSNeel Natu vcpu_cleanup(vm, i, destroy); 5265fcf252fSNeel Natu 5275fcf252fSNeel Natu VMCLEANUP(vm->cookie); 5285fcf252fSNeel Natu 5299b1aa8d6SNeel Natu /* 5309b1aa8d6SNeel Natu * System memory is removed from the guest address space only when 5319b1aa8d6SNeel Natu * the VM is destroyed. This is because the mapping remains the same 5329b1aa8d6SNeel Natu * across VM reset. 5339b1aa8d6SNeel Natu * 5349b1aa8d6SNeel Natu * Device memory can be relocated by the guest (e.g. using PCI BARs) 5359b1aa8d6SNeel Natu * so those mappings are removed on a VM reset. 5369b1aa8d6SNeel Natu */ 5379b1aa8d6SNeel Natu for (i = 0; i < VM_MAX_MEMMAPS; i++) { 5389b1aa8d6SNeel Natu mm = &vm->mem_maps[i]; 5399b1aa8d6SNeel Natu if (destroy || !sysmem_mapping(vm, mm)) 5409b1aa8d6SNeel Natu vm_free_memmap(vm, i); 5419b1aa8d6SNeel Natu } 542f7d51510SNeel Natu 5439b1aa8d6SNeel Natu if (destroy) { 5449b1aa8d6SNeel Natu for (i = 0; i < VM_MAX_MEMSEGS; i++) 5459b1aa8d6SNeel Natu vm_free_memseg(vm, i); 546366f6083SPeter Grehan 547318224bbSNeel Natu VMSPACE_FREE(vm->vmspace); 5485fcf252fSNeel Natu vm->vmspace = NULL; 5495fcf252fSNeel Natu } 5505fcf252fSNeel Natu } 551366f6083SPeter Grehan 5525fcf252fSNeel Natu void 5535fcf252fSNeel Natu vm_destroy(struct vm *vm) 5545fcf252fSNeel Natu { 5555fcf252fSNeel Natu vm_cleanup(vm, true); 556366f6083SPeter Grehan free(vm, M_VM); 557366f6083SPeter Grehan } 558366f6083SPeter Grehan 5595fcf252fSNeel Natu int 5605fcf252fSNeel Natu vm_reinit(struct vm *vm) 5615fcf252fSNeel Natu { 5625fcf252fSNeel Natu int error; 5635fcf252fSNeel Natu 5645fcf252fSNeel Natu /* 5655fcf252fSNeel Natu * A virtual machine can be reset only if all vcpus are suspended. 5665fcf252fSNeel Natu */ 5675fcf252fSNeel Natu if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) { 5685fcf252fSNeel Natu vm_cleanup(vm, false); 5695fcf252fSNeel Natu vm_init(vm, false); 5705fcf252fSNeel Natu error = 0; 5715fcf252fSNeel Natu } else { 5725fcf252fSNeel Natu error = EBUSY; 5735fcf252fSNeel Natu } 5745fcf252fSNeel Natu 5755fcf252fSNeel Natu return (error); 5765fcf252fSNeel Natu } 5775fcf252fSNeel Natu 578366f6083SPeter Grehan const char * 579366f6083SPeter Grehan vm_name(struct vm *vm) 580366f6083SPeter Grehan { 581366f6083SPeter Grehan return (vm->name); 582366f6083SPeter Grehan } 583366f6083SPeter Grehan 584366f6083SPeter Grehan int 585366f6083SPeter Grehan vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa) 586366f6083SPeter Grehan { 587318224bbSNeel Natu vm_object_t obj; 588366f6083SPeter Grehan 589318224bbSNeel Natu if ((obj = vmm_mmio_alloc(vm->vmspace, gpa, len, hpa)) == NULL) 590318224bbSNeel Natu return (ENOMEM); 591318224bbSNeel Natu else 592318224bbSNeel Natu return (0); 593366f6083SPeter Grehan } 594366f6083SPeter Grehan 595366f6083SPeter Grehan int 596366f6083SPeter Grehan vm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len) 597366f6083SPeter Grehan { 598366f6083SPeter Grehan 599318224bbSNeel Natu vmm_mmio_free(vm->vmspace, gpa, len); 600318224bbSNeel Natu return (0); 601366f6083SPeter Grehan } 602366f6083SPeter Grehan 6039b1aa8d6SNeel Natu /* 6049b1aa8d6SNeel Natu * Return 'true' if 'gpa' is allocated in the guest address space. 6059b1aa8d6SNeel Natu * 6069b1aa8d6SNeel Natu * This function is called in the context of a running vcpu which acts as 6079b1aa8d6SNeel Natu * an implicit lock on 'vm->mem_maps[]'. 6089b1aa8d6SNeel Natu */ 6099b1aa8d6SNeel Natu bool 6109b1aa8d6SNeel Natu vm_mem_allocated(struct vm *vm, int vcpuid, vm_paddr_t gpa) 611366f6083SPeter Grehan { 6129b1aa8d6SNeel Natu struct mem_map *mm; 613341f19c9SNeel Natu int i; 614341f19c9SNeel Natu 6159b1aa8d6SNeel Natu #ifdef INVARIANTS 6169b1aa8d6SNeel Natu int hostcpu, state; 6179b1aa8d6SNeel Natu state = vcpu_get_state(vm, vcpuid, &hostcpu); 6189b1aa8d6SNeel Natu KASSERT(state == VCPU_RUNNING && hostcpu == curcpu, 6199b1aa8d6SNeel Natu ("%s: invalid vcpu state %d/%d", __func__, state, hostcpu)); 6209b1aa8d6SNeel Natu #endif 6219b1aa8d6SNeel Natu 6229b1aa8d6SNeel Natu for (i = 0; i < VM_MAX_MEMMAPS; i++) { 6239b1aa8d6SNeel Natu mm = &vm->mem_maps[i]; 6249b1aa8d6SNeel Natu if (mm->len != 0 && gpa >= mm->gpa && gpa < mm->gpa + mm->len) 6259b1aa8d6SNeel Natu return (true); /* 'gpa' is sysmem or devmem */ 626341f19c9SNeel Natu } 627341f19c9SNeel Natu 628318224bbSNeel Natu if (ppt_is_mmio(vm, gpa)) 6299b1aa8d6SNeel Natu return (true); /* 'gpa' is pci passthru mmio */ 630318224bbSNeel Natu 6319b1aa8d6SNeel Natu return (false); 632341f19c9SNeel Natu } 633341f19c9SNeel Natu 634341f19c9SNeel Natu int 6359b1aa8d6SNeel Natu vm_alloc_memseg(struct vm *vm, int ident, size_t len, bool sysmem) 636341f19c9SNeel Natu { 637318224bbSNeel Natu struct mem_seg *seg; 6389b1aa8d6SNeel Natu vm_object_t obj; 639366f6083SPeter Grehan 6409b1aa8d6SNeel Natu if (ident < 0 || ident >= VM_MAX_MEMSEGS) 641341f19c9SNeel Natu return (EINVAL); 642341f19c9SNeel Natu 6439b1aa8d6SNeel Natu if (len == 0 || (len & PAGE_MASK)) 6449b1aa8d6SNeel Natu return (EINVAL); 645341f19c9SNeel Natu 6469b1aa8d6SNeel Natu seg = &vm->mem_segs[ident]; 6479b1aa8d6SNeel Natu if (seg->object != NULL) { 6489b1aa8d6SNeel Natu if (seg->len == len && seg->sysmem == sysmem) 6499b1aa8d6SNeel Natu return (EEXIST); 6509b1aa8d6SNeel Natu else 6519b1aa8d6SNeel Natu return (EINVAL); 652341f19c9SNeel Natu } 653341f19c9SNeel Natu 6549b1aa8d6SNeel Natu obj = vm_object_allocate(OBJT_DEFAULT, len >> PAGE_SHIFT); 6559b1aa8d6SNeel Natu if (obj == NULL) 656318224bbSNeel Natu return (ENOMEM); 657318224bbSNeel Natu 658318224bbSNeel Natu seg->len = len; 6599b1aa8d6SNeel Natu seg->object = obj; 6609b1aa8d6SNeel Natu seg->sysmem = sysmem; 661366f6083SPeter Grehan return (0); 662366f6083SPeter Grehan } 663366f6083SPeter Grehan 6649b1aa8d6SNeel Natu int 6659b1aa8d6SNeel Natu vm_get_memseg(struct vm *vm, int ident, size_t *len, bool *sysmem, 6669b1aa8d6SNeel Natu vm_object_t *objptr) 667477867a0SNeel Natu { 6689b1aa8d6SNeel Natu struct mem_seg *seg; 669477867a0SNeel Natu 6709b1aa8d6SNeel Natu if (ident < 0 || ident >= VM_MAX_MEMSEGS) 6719b1aa8d6SNeel Natu return (EINVAL); 6729b1aa8d6SNeel Natu 6739b1aa8d6SNeel Natu seg = &vm->mem_segs[ident]; 6749b1aa8d6SNeel Natu if (len) 6759b1aa8d6SNeel Natu *len = seg->len; 6769b1aa8d6SNeel Natu if (sysmem) 6779b1aa8d6SNeel Natu *sysmem = seg->sysmem; 6789b1aa8d6SNeel Natu if (objptr) 6799b1aa8d6SNeel Natu *objptr = seg->object; 6809b1aa8d6SNeel Natu return (0); 681477867a0SNeel Natu } 6829b1aa8d6SNeel Natu 6839b1aa8d6SNeel Natu void 6849b1aa8d6SNeel Natu vm_free_memseg(struct vm *vm, int ident) 6859b1aa8d6SNeel Natu { 6869b1aa8d6SNeel Natu struct mem_seg *seg; 6879b1aa8d6SNeel Natu 6889b1aa8d6SNeel Natu KASSERT(ident >= 0 && ident < VM_MAX_MEMSEGS, 6899b1aa8d6SNeel Natu ("%s: invalid memseg ident %d", __func__, ident)); 6909b1aa8d6SNeel Natu 6919b1aa8d6SNeel Natu seg = &vm->mem_segs[ident]; 6929b1aa8d6SNeel Natu if (seg->object != NULL) { 6939b1aa8d6SNeel Natu vm_object_deallocate(seg->object); 6949b1aa8d6SNeel Natu bzero(seg, sizeof(struct mem_seg)); 6959b1aa8d6SNeel Natu } 6969b1aa8d6SNeel Natu } 6979b1aa8d6SNeel Natu 6989b1aa8d6SNeel Natu int 6999b1aa8d6SNeel Natu vm_mmap_memseg(struct vm *vm, vm_paddr_t gpa, int segid, vm_ooffset_t first, 7009b1aa8d6SNeel Natu size_t len, int prot, int flags) 7019b1aa8d6SNeel Natu { 7029b1aa8d6SNeel Natu struct mem_seg *seg; 7039b1aa8d6SNeel Natu struct mem_map *m, *map; 7049b1aa8d6SNeel Natu vm_ooffset_t last; 7059b1aa8d6SNeel Natu int i, error; 7069b1aa8d6SNeel Natu 7079b1aa8d6SNeel Natu if (prot == 0 || (prot & ~(VM_PROT_ALL)) != 0) 7089b1aa8d6SNeel Natu return (EINVAL); 7099b1aa8d6SNeel Natu 7109b1aa8d6SNeel Natu if (flags & ~VM_MEMMAP_F_WIRED) 7119b1aa8d6SNeel Natu return (EINVAL); 7129b1aa8d6SNeel Natu 7139b1aa8d6SNeel Natu if (segid < 0 || segid >= VM_MAX_MEMSEGS) 7149b1aa8d6SNeel Natu return (EINVAL); 7159b1aa8d6SNeel Natu 7169b1aa8d6SNeel Natu seg = &vm->mem_segs[segid]; 7179b1aa8d6SNeel Natu if (seg->object == NULL) 7189b1aa8d6SNeel Natu return (EINVAL); 7199b1aa8d6SNeel Natu 7209b1aa8d6SNeel Natu last = first + len; 7219b1aa8d6SNeel Natu if (first < 0 || first >= last || last > seg->len) 7229b1aa8d6SNeel Natu return (EINVAL); 7239b1aa8d6SNeel Natu 7249b1aa8d6SNeel Natu if ((gpa | first | last) & PAGE_MASK) 7259b1aa8d6SNeel Natu return (EINVAL); 7269b1aa8d6SNeel Natu 7279b1aa8d6SNeel Natu map = NULL; 7289b1aa8d6SNeel Natu for (i = 0; i < VM_MAX_MEMMAPS; i++) { 7299b1aa8d6SNeel Natu m = &vm->mem_maps[i]; 7309b1aa8d6SNeel Natu if (m->len == 0) { 7319b1aa8d6SNeel Natu map = m; 7329b1aa8d6SNeel Natu break; 7339b1aa8d6SNeel Natu } 7349b1aa8d6SNeel Natu } 7359b1aa8d6SNeel Natu 7369b1aa8d6SNeel Natu if (map == NULL) 7379b1aa8d6SNeel Natu return (ENOSPC); 7389b1aa8d6SNeel Natu 7399b1aa8d6SNeel Natu error = vm_map_find(&vm->vmspace->vm_map, seg->object, first, &gpa, 7409b1aa8d6SNeel Natu len, 0, VMFS_NO_SPACE, prot, prot, 0); 7419b1aa8d6SNeel Natu if (error != KERN_SUCCESS) 7429b1aa8d6SNeel Natu return (EFAULT); 7439b1aa8d6SNeel Natu 7449b1aa8d6SNeel Natu vm_object_reference(seg->object); 7459b1aa8d6SNeel Natu 7469b1aa8d6SNeel Natu if (flags & VM_MEMMAP_F_WIRED) { 7479b1aa8d6SNeel Natu error = vm_map_wire(&vm->vmspace->vm_map, gpa, gpa + len, 7489b1aa8d6SNeel Natu VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 7499b1aa8d6SNeel Natu if (error != KERN_SUCCESS) { 7509b1aa8d6SNeel Natu vm_map_remove(&vm->vmspace->vm_map, gpa, gpa + len); 7519b1aa8d6SNeel Natu return (EFAULT); 7529b1aa8d6SNeel Natu } 7539b1aa8d6SNeel Natu } 7549b1aa8d6SNeel Natu 7559b1aa8d6SNeel Natu map->gpa = gpa; 7569b1aa8d6SNeel Natu map->len = len; 7579b1aa8d6SNeel Natu map->segoff = first; 7589b1aa8d6SNeel Natu map->segid = segid; 7599b1aa8d6SNeel Natu map->prot = prot; 7609b1aa8d6SNeel Natu map->flags = flags; 7619b1aa8d6SNeel Natu return (0); 7629b1aa8d6SNeel Natu } 7639b1aa8d6SNeel Natu 7649b1aa8d6SNeel Natu int 7659b1aa8d6SNeel Natu vm_mmap_getnext(struct vm *vm, vm_paddr_t *gpa, int *segid, 7669b1aa8d6SNeel Natu vm_ooffset_t *segoff, size_t *len, int *prot, int *flags) 7679b1aa8d6SNeel Natu { 7689b1aa8d6SNeel Natu struct mem_map *mm, *mmnext; 7699b1aa8d6SNeel Natu int i; 7709b1aa8d6SNeel Natu 7719b1aa8d6SNeel Natu mmnext = NULL; 7729b1aa8d6SNeel Natu for (i = 0; i < VM_MAX_MEMMAPS; i++) { 7739b1aa8d6SNeel Natu mm = &vm->mem_maps[i]; 7749b1aa8d6SNeel Natu if (mm->len == 0 || mm->gpa < *gpa) 7759b1aa8d6SNeel Natu continue; 7769b1aa8d6SNeel Natu if (mmnext == NULL || mm->gpa < mmnext->gpa) 7779b1aa8d6SNeel Natu mmnext = mm; 7789b1aa8d6SNeel Natu } 7799b1aa8d6SNeel Natu 7809b1aa8d6SNeel Natu if (mmnext != NULL) { 7819b1aa8d6SNeel Natu *gpa = mmnext->gpa; 7829b1aa8d6SNeel Natu if (segid) 7839b1aa8d6SNeel Natu *segid = mmnext->segid; 7849b1aa8d6SNeel Natu if (segoff) 7859b1aa8d6SNeel Natu *segoff = mmnext->segoff; 7869b1aa8d6SNeel Natu if (len) 7879b1aa8d6SNeel Natu *len = mmnext->len; 7889b1aa8d6SNeel Natu if (prot) 7899b1aa8d6SNeel Natu *prot = mmnext->prot; 7909b1aa8d6SNeel Natu if (flags) 7919b1aa8d6SNeel Natu *flags = mmnext->flags; 7929b1aa8d6SNeel Natu return (0); 7939b1aa8d6SNeel Natu } else { 7949b1aa8d6SNeel Natu return (ENOENT); 7959b1aa8d6SNeel Natu } 796477867a0SNeel Natu } 797477867a0SNeel Natu 798318224bbSNeel Natu static void 7999b1aa8d6SNeel Natu vm_free_memmap(struct vm *vm, int ident) 800366f6083SPeter Grehan { 8019b1aa8d6SNeel Natu struct mem_map *mm; 8029b1aa8d6SNeel Natu int error; 8034db4fb2cSNeel Natu 8049b1aa8d6SNeel Natu mm = &vm->mem_maps[ident]; 8059b1aa8d6SNeel Natu if (mm->len) { 8069b1aa8d6SNeel Natu error = vm_map_remove(&vm->vmspace->vm_map, mm->gpa, 8079b1aa8d6SNeel Natu mm->gpa + mm->len); 8089b1aa8d6SNeel Natu KASSERT(error == KERN_SUCCESS, ("%s: vm_map_remove error %d", 8099b1aa8d6SNeel Natu __func__, error)); 8109b1aa8d6SNeel Natu bzero(mm, sizeof(struct mem_map)); 811318224bbSNeel Natu } 812318224bbSNeel Natu } 813318224bbSNeel Natu 8149b1aa8d6SNeel Natu static __inline bool 8159b1aa8d6SNeel Natu sysmem_mapping(struct vm *vm, struct mem_map *mm) 816318224bbSNeel Natu { 817318224bbSNeel Natu 8189b1aa8d6SNeel Natu if (mm->len != 0 && vm->mem_segs[mm->segid].sysmem) 8199b1aa8d6SNeel Natu return (true); 8209b1aa8d6SNeel Natu else 8219b1aa8d6SNeel Natu return (false); 822318224bbSNeel Natu } 823318224bbSNeel Natu 824*147d12a7SAntoine Brodin vm_paddr_t 825*147d12a7SAntoine Brodin vmm_sysmem_maxaddr(struct vm *vm) 8269b1aa8d6SNeel Natu { 8279b1aa8d6SNeel Natu struct mem_map *mm; 8289b1aa8d6SNeel Natu vm_paddr_t maxaddr; 8299b1aa8d6SNeel Natu int i; 830318224bbSNeel Natu 8319b1aa8d6SNeel Natu maxaddr = 0; 8329b1aa8d6SNeel Natu for (i = 0; i < VM_MAX_MEMMAPS; i++) { 8339b1aa8d6SNeel Natu mm = &vm->mem_maps[i]; 8349b1aa8d6SNeel Natu if (sysmem_mapping(vm, mm)) { 8359b1aa8d6SNeel Natu if (maxaddr < mm->gpa + mm->len) 8369b1aa8d6SNeel Natu maxaddr = mm->gpa + mm->len; 8379b1aa8d6SNeel Natu } 8389b1aa8d6SNeel Natu } 8399b1aa8d6SNeel Natu return (maxaddr); 840318224bbSNeel Natu } 841318224bbSNeel Natu 842318224bbSNeel Natu static void 843318224bbSNeel Natu vm_iommu_modify(struct vm *vm, boolean_t map) 844318224bbSNeel Natu { 845318224bbSNeel Natu int i, sz; 846318224bbSNeel Natu vm_paddr_t gpa, hpa; 8479b1aa8d6SNeel Natu struct mem_map *mm; 848318224bbSNeel Natu void *vp, *cookie, *host_domain; 849318224bbSNeel Natu 850318224bbSNeel Natu sz = PAGE_SIZE; 851318224bbSNeel Natu host_domain = iommu_host_domain(); 852318224bbSNeel Natu 8539b1aa8d6SNeel Natu for (i = 0; i < VM_MAX_MEMMAPS; i++) { 8549b1aa8d6SNeel Natu mm = &vm->mem_maps[i]; 8559b1aa8d6SNeel Natu if (!sysmem_mapping(vm, mm)) 8569b1aa8d6SNeel Natu continue; 857318224bbSNeel Natu 8589b1aa8d6SNeel Natu if (map) { 8599b1aa8d6SNeel Natu KASSERT((mm->flags & VM_MEMMAP_F_IOMMU) == 0, 8609b1aa8d6SNeel Natu ("iommu map found invalid memmap %#lx/%#lx/%#x", 8619b1aa8d6SNeel Natu mm->gpa, mm->len, mm->flags)); 8629b1aa8d6SNeel Natu if ((mm->flags & VM_MEMMAP_F_WIRED) == 0) 8639b1aa8d6SNeel Natu continue; 8649b1aa8d6SNeel Natu mm->flags |= VM_MEMMAP_F_IOMMU; 8659b1aa8d6SNeel Natu } else { 8669b1aa8d6SNeel Natu if ((mm->flags & VM_MEMMAP_F_IOMMU) == 0) 8679b1aa8d6SNeel Natu continue; 8689b1aa8d6SNeel Natu mm->flags &= ~VM_MEMMAP_F_IOMMU; 8699b1aa8d6SNeel Natu KASSERT((mm->flags & VM_MEMMAP_F_WIRED) != 0, 8709b1aa8d6SNeel Natu ("iommu unmap found invalid memmap %#lx/%#lx/%#x", 8719b1aa8d6SNeel Natu mm->gpa, mm->len, mm->flags)); 8729b1aa8d6SNeel Natu } 8739b1aa8d6SNeel Natu 8749b1aa8d6SNeel Natu gpa = mm->gpa; 8759b1aa8d6SNeel Natu while (gpa < mm->gpa + mm->len) { 8769b1aa8d6SNeel Natu vp = vm_gpa_hold(vm, -1, gpa, PAGE_SIZE, VM_PROT_WRITE, 877318224bbSNeel Natu &cookie); 878318224bbSNeel Natu KASSERT(vp != NULL, ("vm(%s) could not map gpa %#lx", 879318224bbSNeel Natu vm_name(vm), gpa)); 880318224bbSNeel Natu 881318224bbSNeel Natu vm_gpa_release(cookie); 882318224bbSNeel Natu 883318224bbSNeel Natu hpa = DMAP_TO_PHYS((uintptr_t)vp); 884318224bbSNeel Natu if (map) { 885318224bbSNeel Natu iommu_create_mapping(vm->iommu, gpa, hpa, sz); 886318224bbSNeel Natu iommu_remove_mapping(host_domain, hpa, sz); 887318224bbSNeel Natu } else { 888318224bbSNeel Natu iommu_remove_mapping(vm->iommu, gpa, sz); 889318224bbSNeel Natu iommu_create_mapping(host_domain, hpa, hpa, sz); 890318224bbSNeel Natu } 891318224bbSNeel Natu 892318224bbSNeel Natu gpa += PAGE_SIZE; 893318224bbSNeel Natu } 894318224bbSNeel Natu } 895318224bbSNeel Natu 896318224bbSNeel Natu /* 897318224bbSNeel Natu * Invalidate the cached translations associated with the domain 898318224bbSNeel Natu * from which pages were removed. 899318224bbSNeel Natu */ 900318224bbSNeel Natu if (map) 901318224bbSNeel Natu iommu_invalidate_tlb(host_domain); 902318224bbSNeel Natu else 903318224bbSNeel Natu iommu_invalidate_tlb(vm->iommu); 904318224bbSNeel Natu } 905318224bbSNeel Natu 906318224bbSNeel Natu #define vm_iommu_unmap(vm) vm_iommu_modify((vm), FALSE) 907318224bbSNeel Natu #define vm_iommu_map(vm) vm_iommu_modify((vm), TRUE) 908318224bbSNeel Natu 909318224bbSNeel Natu int 910318224bbSNeel Natu vm_unassign_pptdev(struct vm *vm, int bus, int slot, int func) 911318224bbSNeel Natu { 912318224bbSNeel Natu int error; 913318224bbSNeel Natu 914318224bbSNeel Natu error = ppt_unassign_device(vm, bus, slot, func); 915318224bbSNeel Natu if (error) 916318224bbSNeel Natu return (error); 917318224bbSNeel Natu 9189b1aa8d6SNeel Natu if (ppt_assigned_devices(vm) == 0) 919318224bbSNeel Natu vm_iommu_unmap(vm); 9209b1aa8d6SNeel Natu 921318224bbSNeel Natu return (0); 922318224bbSNeel Natu } 923318224bbSNeel Natu 924318224bbSNeel Natu int 925318224bbSNeel Natu vm_assign_pptdev(struct vm *vm, int bus, int slot, int func) 926318224bbSNeel Natu { 927318224bbSNeel Natu int error; 928318224bbSNeel Natu vm_paddr_t maxaddr; 929318224bbSNeel Natu 9309b1aa8d6SNeel Natu /* Set up the IOMMU to do the 'gpa' to 'hpa' translation */ 93151f45d01SNeel Natu if (ppt_assigned_devices(vm) == 0) { 932318224bbSNeel Natu KASSERT(vm->iommu == NULL, 933318224bbSNeel Natu ("vm_assign_pptdev: iommu must be NULL")); 934*147d12a7SAntoine Brodin maxaddr = vmm_sysmem_maxaddr(vm); 935318224bbSNeel Natu vm->iommu = iommu_create_domain(maxaddr); 936ffe1b10dSJohn Baldwin if (vm->iommu == NULL) 937ffe1b10dSJohn Baldwin return (ENXIO); 938318224bbSNeel Natu vm_iommu_map(vm); 939318224bbSNeel Natu } 940318224bbSNeel Natu 941318224bbSNeel Natu error = ppt_assign_device(vm, bus, slot, func); 942318224bbSNeel Natu return (error); 943318224bbSNeel Natu } 944318224bbSNeel Natu 945318224bbSNeel Natu void * 9469b1aa8d6SNeel Natu vm_gpa_hold(struct vm *vm, int vcpuid, vm_paddr_t gpa, size_t len, int reqprot, 947318224bbSNeel Natu void **cookie) 948318224bbSNeel Natu { 9499b1aa8d6SNeel Natu int i, count, pageoff; 9509b1aa8d6SNeel Natu struct mem_map *mm; 951318224bbSNeel Natu vm_page_t m; 9529b1aa8d6SNeel Natu #ifdef INVARIANTS 9539b1aa8d6SNeel Natu /* 9549b1aa8d6SNeel Natu * All vcpus are frozen by ioctls that modify the memory map 9559b1aa8d6SNeel Natu * (e.g. VM_MMAP_MEMSEG). Therefore 'vm->memmap[]' stability is 9569b1aa8d6SNeel Natu * guaranteed if at least one vcpu is in the VCPU_FROZEN state. 9579b1aa8d6SNeel Natu */ 9589b1aa8d6SNeel Natu int state; 9592de70600SJohn Baldwin KASSERT(vcpuid >= -1 && vcpuid < VM_MAXCPU, ("%s: invalid vcpuid %d", 9609b1aa8d6SNeel Natu __func__, vcpuid)); 9619b1aa8d6SNeel Natu for (i = 0; i < VM_MAXCPU; i++) { 9629b1aa8d6SNeel Natu if (vcpuid != -1 && vcpuid != i) 9639b1aa8d6SNeel Natu continue; 9649b1aa8d6SNeel Natu state = vcpu_get_state(vm, i, NULL); 9659b1aa8d6SNeel Natu KASSERT(state == VCPU_FROZEN, ("%s: invalid vcpu state %d", 9669b1aa8d6SNeel Natu __func__, state)); 9679b1aa8d6SNeel Natu } 9689b1aa8d6SNeel Natu #endif 969318224bbSNeel Natu pageoff = gpa & PAGE_MASK; 970318224bbSNeel Natu if (len > PAGE_SIZE - pageoff) 971318224bbSNeel Natu panic("vm_gpa_hold: invalid gpa/len: 0x%016lx/%lu", gpa, len); 972318224bbSNeel Natu 9739b1aa8d6SNeel Natu count = 0; 9749b1aa8d6SNeel Natu for (i = 0; i < VM_MAX_MEMMAPS; i++) { 9759b1aa8d6SNeel Natu mm = &vm->mem_maps[i]; 9769b1aa8d6SNeel Natu if (sysmem_mapping(vm, mm) && gpa >= mm->gpa && 9779b1aa8d6SNeel Natu gpa < mm->gpa + mm->len) { 978318224bbSNeel Natu count = vm_fault_quick_hold_pages(&vm->vmspace->vm_map, 979318224bbSNeel Natu trunc_page(gpa), PAGE_SIZE, reqprot, &m, 1); 9809b1aa8d6SNeel Natu break; 9819b1aa8d6SNeel Natu } 9829b1aa8d6SNeel Natu } 983318224bbSNeel Natu 984318224bbSNeel Natu if (count == 1) { 985318224bbSNeel Natu *cookie = m; 986318224bbSNeel Natu return ((void *)(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)) + pageoff)); 987318224bbSNeel Natu } else { 988318224bbSNeel Natu *cookie = NULL; 989318224bbSNeel Natu return (NULL); 990318224bbSNeel Natu } 991318224bbSNeel Natu } 992318224bbSNeel Natu 993318224bbSNeel Natu void 994318224bbSNeel Natu vm_gpa_release(void *cookie) 995318224bbSNeel Natu { 996318224bbSNeel Natu vm_page_t m = cookie; 997318224bbSNeel Natu 998318224bbSNeel Natu vm_page_lock(m); 999318224bbSNeel Natu vm_page_unhold(m); 1000318224bbSNeel Natu vm_page_unlock(m); 1001366f6083SPeter Grehan } 1002366f6083SPeter Grehan 1003366f6083SPeter Grehan int 1004366f6083SPeter Grehan vm_get_register(struct vm *vm, int vcpu, int reg, uint64_t *retval) 1005366f6083SPeter Grehan { 1006366f6083SPeter Grehan 1007366f6083SPeter Grehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 1008366f6083SPeter Grehan return (EINVAL); 1009366f6083SPeter Grehan 1010366f6083SPeter Grehan if (reg >= VM_REG_LAST) 1011366f6083SPeter Grehan return (EINVAL); 1012366f6083SPeter Grehan 1013366f6083SPeter Grehan return (VMGETREG(vm->cookie, vcpu, reg, retval)); 1014366f6083SPeter Grehan } 1015366f6083SPeter Grehan 1016366f6083SPeter Grehan int 1017d087a399SNeel Natu vm_set_register(struct vm *vm, int vcpuid, int reg, uint64_t val) 1018366f6083SPeter Grehan { 1019d087a399SNeel Natu struct vcpu *vcpu; 1020d087a399SNeel Natu int error; 1021366f6083SPeter Grehan 1022d087a399SNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1023366f6083SPeter Grehan return (EINVAL); 1024366f6083SPeter Grehan 1025366f6083SPeter Grehan if (reg >= VM_REG_LAST) 1026366f6083SPeter Grehan return (EINVAL); 1027366f6083SPeter Grehan 1028d087a399SNeel Natu error = VMSETREG(vm->cookie, vcpuid, reg, val); 1029d087a399SNeel Natu if (error || reg != VM_REG_GUEST_RIP) 1030d087a399SNeel Natu return (error); 1031d087a399SNeel Natu 1032d087a399SNeel Natu /* Set 'nextrip' to match the value of %rip */ 1033d087a399SNeel Natu VCPU_CTR1(vm, vcpuid, "Setting nextrip to %#lx", val); 1034d087a399SNeel Natu vcpu = &vm->vcpu[vcpuid]; 1035d087a399SNeel Natu vcpu->nextrip = val; 1036d087a399SNeel Natu return (0); 1037366f6083SPeter Grehan } 1038366f6083SPeter Grehan 1039366f6083SPeter Grehan static boolean_t 1040366f6083SPeter Grehan is_descriptor_table(int reg) 1041366f6083SPeter Grehan { 1042366f6083SPeter Grehan 1043366f6083SPeter Grehan switch (reg) { 1044366f6083SPeter Grehan case VM_REG_GUEST_IDTR: 1045366f6083SPeter Grehan case VM_REG_GUEST_GDTR: 1046366f6083SPeter Grehan return (TRUE); 1047366f6083SPeter Grehan default: 1048366f6083SPeter Grehan return (FALSE); 1049366f6083SPeter Grehan } 1050366f6083SPeter Grehan } 1051366f6083SPeter Grehan 1052366f6083SPeter Grehan static boolean_t 1053366f6083SPeter Grehan is_segment_register(int reg) 1054366f6083SPeter Grehan { 1055366f6083SPeter Grehan 1056366f6083SPeter Grehan switch (reg) { 1057366f6083SPeter Grehan case VM_REG_GUEST_ES: 1058366f6083SPeter Grehan case VM_REG_GUEST_CS: 1059366f6083SPeter Grehan case VM_REG_GUEST_SS: 1060366f6083SPeter Grehan case VM_REG_GUEST_DS: 1061366f6083SPeter Grehan case VM_REG_GUEST_FS: 1062366f6083SPeter Grehan case VM_REG_GUEST_GS: 1063366f6083SPeter Grehan case VM_REG_GUEST_TR: 1064366f6083SPeter Grehan case VM_REG_GUEST_LDTR: 1065366f6083SPeter Grehan return (TRUE); 1066366f6083SPeter Grehan default: 1067366f6083SPeter Grehan return (FALSE); 1068366f6083SPeter Grehan } 1069366f6083SPeter Grehan } 1070366f6083SPeter Grehan 1071366f6083SPeter Grehan int 1072366f6083SPeter Grehan vm_get_seg_desc(struct vm *vm, int vcpu, int reg, 1073366f6083SPeter Grehan struct seg_desc *desc) 1074366f6083SPeter Grehan { 1075366f6083SPeter Grehan 1076366f6083SPeter Grehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 1077366f6083SPeter Grehan return (EINVAL); 1078366f6083SPeter Grehan 1079366f6083SPeter Grehan if (!is_segment_register(reg) && !is_descriptor_table(reg)) 1080366f6083SPeter Grehan return (EINVAL); 1081366f6083SPeter Grehan 1082366f6083SPeter Grehan return (VMGETDESC(vm->cookie, vcpu, reg, desc)); 1083366f6083SPeter Grehan } 1084366f6083SPeter Grehan 1085366f6083SPeter Grehan int 1086366f6083SPeter Grehan vm_set_seg_desc(struct vm *vm, int vcpu, int reg, 1087366f6083SPeter Grehan struct seg_desc *desc) 1088366f6083SPeter Grehan { 1089366f6083SPeter Grehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 1090366f6083SPeter Grehan return (EINVAL); 1091366f6083SPeter Grehan 1092366f6083SPeter Grehan if (!is_segment_register(reg) && !is_descriptor_table(reg)) 1093366f6083SPeter Grehan return (EINVAL); 1094366f6083SPeter Grehan 1095366f6083SPeter Grehan return (VMSETDESC(vm->cookie, vcpu, reg, desc)); 1096366f6083SPeter Grehan } 1097366f6083SPeter Grehan 1098366f6083SPeter Grehan static void 1099366f6083SPeter Grehan restore_guest_fpustate(struct vcpu *vcpu) 1100366f6083SPeter Grehan { 1101366f6083SPeter Grehan 110238f1b189SPeter Grehan /* flush host state to the pcb */ 110338f1b189SPeter Grehan fpuexit(curthread); 1104bd8572e0SNeel Natu 1105bd8572e0SNeel Natu /* restore guest FPU state */ 1106366f6083SPeter Grehan fpu_stop_emulating(); 110738f1b189SPeter Grehan fpurestore(vcpu->guestfpu); 1108bd8572e0SNeel Natu 1109abb023fbSJohn Baldwin /* restore guest XCR0 if XSAVE is enabled in the host */ 1110abb023fbSJohn Baldwin if (rcr4() & CR4_XSAVE) 1111abb023fbSJohn Baldwin load_xcr(0, vcpu->guest_xcr0); 1112abb023fbSJohn Baldwin 1113bd8572e0SNeel Natu /* 1114bd8572e0SNeel Natu * The FPU is now "dirty" with the guest's state so turn on emulation 1115bd8572e0SNeel Natu * to trap any access to the FPU by the host. 1116bd8572e0SNeel Natu */ 1117bd8572e0SNeel Natu fpu_start_emulating(); 1118366f6083SPeter Grehan } 1119366f6083SPeter Grehan 1120366f6083SPeter Grehan static void 1121366f6083SPeter Grehan save_guest_fpustate(struct vcpu *vcpu) 1122366f6083SPeter Grehan { 1123366f6083SPeter Grehan 1124bd8572e0SNeel Natu if ((rcr0() & CR0_TS) == 0) 1125bd8572e0SNeel Natu panic("fpu emulation not enabled in host!"); 1126bd8572e0SNeel Natu 1127abb023fbSJohn Baldwin /* save guest XCR0 and restore host XCR0 */ 1128abb023fbSJohn Baldwin if (rcr4() & CR4_XSAVE) { 1129abb023fbSJohn Baldwin vcpu->guest_xcr0 = rxcr(0); 1130abb023fbSJohn Baldwin load_xcr(0, vmm_get_host_xcr0()); 1131abb023fbSJohn Baldwin } 1132abb023fbSJohn Baldwin 1133bd8572e0SNeel Natu /* save guest FPU state */ 1134bd8572e0SNeel Natu fpu_stop_emulating(); 113538f1b189SPeter Grehan fpusave(vcpu->guestfpu); 1136366f6083SPeter Grehan fpu_start_emulating(); 1137366f6083SPeter Grehan } 1138366f6083SPeter Grehan 113961592433SNeel Natu static VMM_STAT(VCPU_IDLE_TICKS, "number of ticks vcpu was idle"); 1140f76fc5d4SNeel Natu 1141318224bbSNeel Natu static int 1142248e6799SNeel Natu vcpu_set_state_locked(struct vm *vm, int vcpuid, enum vcpu_state newstate, 1143f80330a8SNeel Natu bool from_idle) 1144366f6083SPeter Grehan { 1145248e6799SNeel Natu struct vcpu *vcpu; 1146318224bbSNeel Natu int error; 1147366f6083SPeter Grehan 1148248e6799SNeel Natu vcpu = &vm->vcpu[vcpuid]; 1149318224bbSNeel Natu vcpu_assert_locked(vcpu); 1150366f6083SPeter Grehan 1151f76fc5d4SNeel Natu /* 1152f80330a8SNeel Natu * State transitions from the vmmdev_ioctl() must always begin from 1153f80330a8SNeel Natu * the VCPU_IDLE state. This guarantees that there is only a single 1154f80330a8SNeel Natu * ioctl() operating on a vcpu at any point. 1155f80330a8SNeel Natu */ 1156f80330a8SNeel Natu if (from_idle) { 1157248e6799SNeel Natu while (vcpu->state != VCPU_IDLE) { 1158248e6799SNeel Natu vcpu->reqidle = 1; 1159248e6799SNeel Natu vcpu_notify_event_locked(vcpu, false); 1160248e6799SNeel Natu VCPU_CTR1(vm, vcpuid, "vcpu state change from %s to " 1161248e6799SNeel Natu "idle requested", vcpu_state2str(vcpu->state)); 1162f80330a8SNeel Natu msleep_spin(&vcpu->state, &vcpu->mtx, "vmstat", hz); 1163248e6799SNeel Natu } 1164f80330a8SNeel Natu } else { 1165f80330a8SNeel Natu KASSERT(vcpu->state != VCPU_IDLE, ("invalid transition from " 1166f80330a8SNeel Natu "vcpu idle state")); 1167f80330a8SNeel Natu } 1168f80330a8SNeel Natu 1169ef39d7e9SNeel Natu if (vcpu->state == VCPU_RUNNING) { 1170ef39d7e9SNeel Natu KASSERT(vcpu->hostcpu == curcpu, ("curcpu %d and hostcpu %d " 1171ef39d7e9SNeel Natu "mismatch for running vcpu", curcpu, vcpu->hostcpu)); 1172ef39d7e9SNeel Natu } else { 1173ef39d7e9SNeel Natu KASSERT(vcpu->hostcpu == NOCPU, ("Invalid hostcpu %d for a " 1174ef39d7e9SNeel Natu "vcpu that is not running", vcpu->hostcpu)); 1175ef39d7e9SNeel Natu } 1176ef39d7e9SNeel Natu 1177f80330a8SNeel Natu /* 1178318224bbSNeel Natu * The following state transitions are allowed: 1179318224bbSNeel Natu * IDLE -> FROZEN -> IDLE 1180318224bbSNeel Natu * FROZEN -> RUNNING -> FROZEN 1181318224bbSNeel Natu * FROZEN -> SLEEPING -> FROZEN 1182f76fc5d4SNeel Natu */ 1183318224bbSNeel Natu switch (vcpu->state) { 1184318224bbSNeel Natu case VCPU_IDLE: 1185318224bbSNeel Natu case VCPU_RUNNING: 1186318224bbSNeel Natu case VCPU_SLEEPING: 1187318224bbSNeel Natu error = (newstate != VCPU_FROZEN); 1188318224bbSNeel Natu break; 1189318224bbSNeel Natu case VCPU_FROZEN: 1190318224bbSNeel Natu error = (newstate == VCPU_FROZEN); 1191318224bbSNeel Natu break; 1192318224bbSNeel Natu default: 1193318224bbSNeel Natu error = 1; 1194318224bbSNeel Natu break; 1195318224bbSNeel Natu } 1196318224bbSNeel Natu 1197f80330a8SNeel Natu if (error) 1198f80330a8SNeel Natu return (EBUSY); 1199318224bbSNeel Natu 1200248e6799SNeel Natu VCPU_CTR2(vm, vcpuid, "vcpu state changed from %s to %s", 1201248e6799SNeel Natu vcpu_state2str(vcpu->state), vcpu_state2str(newstate)); 1202248e6799SNeel Natu 1203f80330a8SNeel Natu vcpu->state = newstate; 1204ef39d7e9SNeel Natu if (newstate == VCPU_RUNNING) 1205ef39d7e9SNeel Natu vcpu->hostcpu = curcpu; 1206ef39d7e9SNeel Natu else 1207ef39d7e9SNeel Natu vcpu->hostcpu = NOCPU; 1208ef39d7e9SNeel Natu 1209f80330a8SNeel Natu if (newstate == VCPU_IDLE) 1210f80330a8SNeel Natu wakeup(&vcpu->state); 1211f80330a8SNeel Natu 1212f80330a8SNeel Natu return (0); 1213318224bbSNeel Natu } 1214318224bbSNeel Natu 1215318224bbSNeel Natu static void 1216318224bbSNeel Natu vcpu_require_state(struct vm *vm, int vcpuid, enum vcpu_state newstate) 1217318224bbSNeel Natu { 1218318224bbSNeel Natu int error; 1219318224bbSNeel Natu 1220f80330a8SNeel Natu if ((error = vcpu_set_state(vm, vcpuid, newstate, false)) != 0) 1221318224bbSNeel Natu panic("Error %d setting state to %d\n", error, newstate); 1222318224bbSNeel Natu } 1223318224bbSNeel Natu 1224318224bbSNeel Natu static void 1225248e6799SNeel Natu vcpu_require_state_locked(struct vm *vm, int vcpuid, enum vcpu_state newstate) 1226318224bbSNeel Natu { 1227318224bbSNeel Natu int error; 1228318224bbSNeel Natu 1229248e6799SNeel Natu if ((error = vcpu_set_state_locked(vm, vcpuid, newstate, false)) != 0) 1230318224bbSNeel Natu panic("Error %d setting state to %d", error, newstate); 1231318224bbSNeel Natu } 1232318224bbSNeel Natu 12335b8a8cd1SNeel Natu static void 12345b8a8cd1SNeel Natu vm_set_rendezvous_func(struct vm *vm, vm_rendezvous_func_t func) 12355b8a8cd1SNeel Natu { 12365b8a8cd1SNeel Natu 12375b8a8cd1SNeel Natu KASSERT(mtx_owned(&vm->rendezvous_mtx), ("rendezvous_mtx not locked")); 12385b8a8cd1SNeel Natu 12395b8a8cd1SNeel Natu /* 12405b8a8cd1SNeel Natu * Update 'rendezvous_func' and execute a write memory barrier to 12415b8a8cd1SNeel Natu * ensure that it is visible across all host cpus. This is not needed 12425b8a8cd1SNeel Natu * for correctness but it does ensure that all the vcpus will notice 12435b8a8cd1SNeel Natu * that the rendezvous is requested immediately. 12445b8a8cd1SNeel Natu */ 12455b8a8cd1SNeel Natu vm->rendezvous_func = func; 12465b8a8cd1SNeel Natu wmb(); 12475b8a8cd1SNeel Natu } 12485b8a8cd1SNeel Natu 12495b8a8cd1SNeel Natu #define RENDEZVOUS_CTR0(vm, vcpuid, fmt) \ 12505b8a8cd1SNeel Natu do { \ 12515b8a8cd1SNeel Natu if (vcpuid >= 0) \ 12525b8a8cd1SNeel Natu VCPU_CTR0(vm, vcpuid, fmt); \ 12535b8a8cd1SNeel Natu else \ 12545b8a8cd1SNeel Natu VM_CTR0(vm, fmt); \ 12555b8a8cd1SNeel Natu } while (0) 12565b8a8cd1SNeel Natu 12575b8a8cd1SNeel Natu static void 12585b8a8cd1SNeel Natu vm_handle_rendezvous(struct vm *vm, int vcpuid) 12595b8a8cd1SNeel Natu { 12605b8a8cd1SNeel Natu 12615b8a8cd1SNeel Natu KASSERT(vcpuid == -1 || (vcpuid >= 0 && vcpuid < VM_MAXCPU), 12625b8a8cd1SNeel Natu ("vm_handle_rendezvous: invalid vcpuid %d", vcpuid)); 12635b8a8cd1SNeel Natu 12645b8a8cd1SNeel Natu mtx_lock(&vm->rendezvous_mtx); 12655b8a8cd1SNeel Natu while (vm->rendezvous_func != NULL) { 126622d822c6SNeel Natu /* 'rendezvous_req_cpus' must be a subset of 'active_cpus' */ 126722d822c6SNeel Natu CPU_AND(&vm->rendezvous_req_cpus, &vm->active_cpus); 126822d822c6SNeel Natu 12695b8a8cd1SNeel Natu if (vcpuid != -1 && 127022d822c6SNeel Natu CPU_ISSET(vcpuid, &vm->rendezvous_req_cpus) && 127122d822c6SNeel Natu !CPU_ISSET(vcpuid, &vm->rendezvous_done_cpus)) { 12725b8a8cd1SNeel Natu VCPU_CTR0(vm, vcpuid, "Calling rendezvous func"); 12735b8a8cd1SNeel Natu (*vm->rendezvous_func)(vm, vcpuid, vm->rendezvous_arg); 12745b8a8cd1SNeel Natu CPU_SET(vcpuid, &vm->rendezvous_done_cpus); 12755b8a8cd1SNeel Natu } 12765b8a8cd1SNeel Natu if (CPU_CMP(&vm->rendezvous_req_cpus, 12775b8a8cd1SNeel Natu &vm->rendezvous_done_cpus) == 0) { 12785b8a8cd1SNeel Natu VCPU_CTR0(vm, vcpuid, "Rendezvous completed"); 12795b8a8cd1SNeel Natu vm_set_rendezvous_func(vm, NULL); 12805b8a8cd1SNeel Natu wakeup(&vm->rendezvous_func); 12815b8a8cd1SNeel Natu break; 12825b8a8cd1SNeel Natu } 12835b8a8cd1SNeel Natu RENDEZVOUS_CTR0(vm, vcpuid, "Wait for rendezvous completion"); 12845b8a8cd1SNeel Natu mtx_sleep(&vm->rendezvous_func, &vm->rendezvous_mtx, 0, 12855b8a8cd1SNeel Natu "vmrndv", 0); 12865b8a8cd1SNeel Natu } 12875b8a8cd1SNeel Natu mtx_unlock(&vm->rendezvous_mtx); 12885b8a8cd1SNeel Natu } 12895b8a8cd1SNeel Natu 1290318224bbSNeel Natu /* 1291318224bbSNeel Natu * Emulate a guest 'hlt' by sleeping until the vcpu is ready to run. 1292318224bbSNeel Natu */ 1293318224bbSNeel Natu static int 1294becd9849SNeel Natu vm_handle_hlt(struct vm *vm, int vcpuid, bool intr_disabled, bool *retu) 1295318224bbSNeel Natu { 1296318224bbSNeel Natu struct vcpu *vcpu; 1297c6a0cc2eSNeel Natu const char *wmesg; 12982ce12423SNeel Natu int t, vcpu_halted, vm_halted; 1299e50ce2aaSNeel Natu 1300e50ce2aaSNeel Natu KASSERT(!CPU_ISSET(vcpuid, &vm->halted_cpus), ("vcpu already halted")); 1301318224bbSNeel Natu 1302318224bbSNeel Natu vcpu = &vm->vcpu[vcpuid]; 1303e50ce2aaSNeel Natu vcpu_halted = 0; 1304e50ce2aaSNeel Natu vm_halted = 0; 1305318224bbSNeel Natu 1306f76fc5d4SNeel Natu vcpu_lock(vcpu); 1307c6a0cc2eSNeel Natu while (1) { 1308f76fc5d4SNeel Natu /* 1309f76fc5d4SNeel Natu * Do a final check for pending NMI or interrupts before 1310c6a0cc2eSNeel Natu * really putting this thread to sleep. Also check for 1311c6a0cc2eSNeel Natu * software events that would cause this vcpu to wakeup. 1312f76fc5d4SNeel Natu * 1313c6a0cc2eSNeel Natu * These interrupts/events could have happened after the 1314c6a0cc2eSNeel Natu * vcpu returned from VMRUN() and before it acquired the 1315c6a0cc2eSNeel Natu * vcpu lock above. 1316f76fc5d4SNeel Natu */ 1317248e6799SNeel Natu if (vm->rendezvous_func != NULL || vm->suspend || vcpu->reqidle) 1318c6a0cc2eSNeel Natu break; 1319c6a0cc2eSNeel Natu if (vm_nmi_pending(vm, vcpuid)) 1320c6a0cc2eSNeel Natu break; 1321c6a0cc2eSNeel Natu if (!intr_disabled) { 1322c6a0cc2eSNeel Natu if (vm_extint_pending(vm, vcpuid) || 1323c6a0cc2eSNeel Natu vlapic_pending_intr(vcpu->vlapic, NULL)) { 1324c6a0cc2eSNeel Natu break; 1325c6a0cc2eSNeel Natu } 1326c6a0cc2eSNeel Natu } 1327c6a0cc2eSNeel Natu 1328f008d157SNeel Natu /* Don't go to sleep if the vcpu thread needs to yield */ 1329f008d157SNeel Natu if (vcpu_should_yield(vm, vcpuid)) 1330f008d157SNeel Natu break; 1331f008d157SNeel Natu 1332fc276d92SJohn Baldwin if (vcpu_debugged(vm, vcpuid)) 1333fc276d92SJohn Baldwin break; 1334fc276d92SJohn Baldwin 1335e50ce2aaSNeel Natu /* 1336e50ce2aaSNeel Natu * Some Linux guests implement "halt" by having all vcpus 1337e50ce2aaSNeel Natu * execute HLT with interrupts disabled. 'halted_cpus' keeps 1338e50ce2aaSNeel Natu * track of the vcpus that have entered this state. When all 1339e50ce2aaSNeel Natu * vcpus enter the halted state the virtual machine is halted. 1340e50ce2aaSNeel Natu */ 1341e50ce2aaSNeel Natu if (intr_disabled) { 1342c6a0cc2eSNeel Natu wmesg = "vmhalt"; 1343e50ce2aaSNeel Natu VCPU_CTR0(vm, vcpuid, "Halted"); 1344055fc2cbSNeel Natu if (!vcpu_halted && halt_detection_enabled) { 1345e50ce2aaSNeel Natu vcpu_halted = 1; 1346e50ce2aaSNeel Natu CPU_SET_ATOMIC(vcpuid, &vm->halted_cpus); 1347e50ce2aaSNeel Natu } 1348e50ce2aaSNeel Natu if (CPU_CMP(&vm->halted_cpus, &vm->active_cpus) == 0) { 1349e50ce2aaSNeel Natu vm_halted = 1; 1350e50ce2aaSNeel Natu break; 1351e50ce2aaSNeel Natu } 1352e50ce2aaSNeel Natu } else { 1353e50ce2aaSNeel Natu wmesg = "vmidle"; 1354e50ce2aaSNeel Natu } 1355c6a0cc2eSNeel Natu 1356f76fc5d4SNeel Natu t = ticks; 1357248e6799SNeel Natu vcpu_require_state_locked(vm, vcpuid, VCPU_SLEEPING); 1358f008d157SNeel Natu /* 1359f008d157SNeel Natu * XXX msleep_spin() cannot be interrupted by signals so 1360f008d157SNeel Natu * wake up periodically to check pending signals. 1361f008d157SNeel Natu */ 1362f008d157SNeel Natu msleep_spin(vcpu, &vcpu->mtx, wmesg, hz); 1363248e6799SNeel Natu vcpu_require_state_locked(vm, vcpuid, VCPU_FROZEN); 1364f76fc5d4SNeel Natu vmm_stat_incr(vm, vcpuid, VCPU_IDLE_TICKS, ticks - t); 1365f76fc5d4SNeel Natu } 1366e50ce2aaSNeel Natu 1367e50ce2aaSNeel Natu if (vcpu_halted) 1368e50ce2aaSNeel Natu CPU_CLR_ATOMIC(vcpuid, &vm->halted_cpus); 1369e50ce2aaSNeel Natu 1370f76fc5d4SNeel Natu vcpu_unlock(vcpu); 1371f76fc5d4SNeel Natu 1372e50ce2aaSNeel Natu if (vm_halted) 1373e50ce2aaSNeel Natu vm_suspend(vm, VM_SUSPEND_HALT); 1374e50ce2aaSNeel Natu 1375318224bbSNeel Natu return (0); 1376318224bbSNeel Natu } 1377318224bbSNeel Natu 1378318224bbSNeel Natu static int 1379becd9849SNeel Natu vm_handle_paging(struct vm *vm, int vcpuid, bool *retu) 1380318224bbSNeel Natu { 1381318224bbSNeel Natu int rv, ftype; 1382318224bbSNeel Natu struct vm_map *map; 1383318224bbSNeel Natu struct vcpu *vcpu; 1384318224bbSNeel Natu struct vm_exit *vme; 1385318224bbSNeel Natu 1386318224bbSNeel Natu vcpu = &vm->vcpu[vcpuid]; 1387318224bbSNeel Natu vme = &vcpu->exitinfo; 1388318224bbSNeel Natu 1389d087a399SNeel Natu KASSERT(vme->inst_length == 0, ("%s: invalid inst_length %d", 1390d087a399SNeel Natu __func__, vme->inst_length)); 1391d087a399SNeel Natu 1392318224bbSNeel Natu ftype = vme->u.paging.fault_type; 1393318224bbSNeel Natu KASSERT(ftype == VM_PROT_READ || 1394318224bbSNeel Natu ftype == VM_PROT_WRITE || ftype == VM_PROT_EXECUTE, 1395318224bbSNeel Natu ("vm_handle_paging: invalid fault_type %d", ftype)); 1396318224bbSNeel Natu 1397318224bbSNeel Natu if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) { 1398318224bbSNeel Natu rv = pmap_emulate_accessed_dirty(vmspace_pmap(vm->vmspace), 1399318224bbSNeel Natu vme->u.paging.gpa, ftype); 14009d8d8e3eSNeel Natu if (rv == 0) { 14019d8d8e3eSNeel Natu VCPU_CTR2(vm, vcpuid, "%s bit emulation for gpa %#lx", 14029d8d8e3eSNeel Natu ftype == VM_PROT_READ ? "accessed" : "dirty", 14039d8d8e3eSNeel Natu vme->u.paging.gpa); 1404318224bbSNeel Natu goto done; 1405318224bbSNeel Natu } 14069d8d8e3eSNeel Natu } 1407318224bbSNeel Natu 1408318224bbSNeel Natu map = &vm->vmspace->vm_map; 1409318224bbSNeel Natu rv = vm_fault(map, vme->u.paging.gpa, ftype, VM_FAULT_NORMAL); 1410318224bbSNeel Natu 1411513c8d33SNeel Natu VCPU_CTR3(vm, vcpuid, "vm_handle_paging rv = %d, gpa = %#lx, " 1412513c8d33SNeel Natu "ftype = %d", rv, vme->u.paging.gpa, ftype); 1413318224bbSNeel Natu 1414318224bbSNeel Natu if (rv != KERN_SUCCESS) 1415318224bbSNeel Natu return (EFAULT); 1416318224bbSNeel Natu done: 1417318224bbSNeel Natu return (0); 1418318224bbSNeel Natu } 1419318224bbSNeel Natu 1420318224bbSNeel Natu static int 1421becd9849SNeel Natu vm_handle_inst_emul(struct vm *vm, int vcpuid, bool *retu) 1422318224bbSNeel Natu { 1423318224bbSNeel Natu struct vie *vie; 1424318224bbSNeel Natu struct vcpu *vcpu; 1425318224bbSNeel Natu struct vm_exit *vme; 1426e4f605eeSTycho Nightingale uint64_t gla, gpa, cs_base; 1427e813a873SNeel Natu struct vm_guest_paging *paging; 1428565bbb86SNeel Natu mem_region_read_t mread; 1429565bbb86SNeel Natu mem_region_write_t mwrite; 1430f7a9f178SNeel Natu enum vm_cpu_mode cpu_mode; 14311c73ea3eSNeel Natu int cs_d, error, fault; 1432318224bbSNeel Natu 1433318224bbSNeel Natu vcpu = &vm->vcpu[vcpuid]; 1434318224bbSNeel Natu vme = &vcpu->exitinfo; 1435318224bbSNeel Natu 14361c73ea3eSNeel Natu KASSERT(vme->inst_length == 0, ("%s: invalid inst_length %d", 14371c73ea3eSNeel Natu __func__, vme->inst_length)); 14381c73ea3eSNeel Natu 1439318224bbSNeel Natu gla = vme->u.inst_emul.gla; 1440318224bbSNeel Natu gpa = vme->u.inst_emul.gpa; 1441e4f605eeSTycho Nightingale cs_base = vme->u.inst_emul.cs_base; 1442f7a9f178SNeel Natu cs_d = vme->u.inst_emul.cs_d; 1443318224bbSNeel Natu vie = &vme->u.inst_emul.vie; 1444e813a873SNeel Natu paging = &vme->u.inst_emul.paging; 1445f7a9f178SNeel Natu cpu_mode = paging->cpu_mode; 1446318224bbSNeel Natu 14479d8d8e3eSNeel Natu VCPU_CTR1(vm, vcpuid, "inst_emul fault accessing gpa %#lx", gpa); 14489d8d8e3eSNeel Natu 1449318224bbSNeel Natu /* Fetch, decode and emulate the faulting instruction */ 1450c2a875f9SNeel Natu if (vie->num_valid == 0) { 1451e4f605eeSTycho Nightingale error = vmm_fetch_instruction(vm, vcpuid, paging, vme->rip + 14521c73ea3eSNeel Natu cs_base, VIE_INST_SIZE, vie, &fault); 1453c2a875f9SNeel Natu } else { 1454c2a875f9SNeel Natu /* 1455c2a875f9SNeel Natu * The instruction bytes have already been copied into 'vie' 1456c2a875f9SNeel Natu */ 14579c4d5478SNeel Natu error = fault = 0; 1458c2a875f9SNeel Natu } 14599c4d5478SNeel Natu if (error || fault) 14609c4d5478SNeel Natu return (error); 1461318224bbSNeel Natu 1462c07a0648SNeel Natu if (vmm_decode_instruction(vm, vcpuid, gla, cpu_mode, cs_d, vie) != 0) { 1463c07a0648SNeel Natu VCPU_CTR1(vm, vcpuid, "Error decoding instruction at %#lx", 1464c07a0648SNeel Natu vme->rip + cs_base); 1465c07a0648SNeel Natu *retu = true; /* dump instruction bytes in userspace */ 1466c07a0648SNeel Natu return (0); 1467c07a0648SNeel Natu } 1468318224bbSNeel Natu 1469a0b78f09SPeter Grehan /* 14701c73ea3eSNeel Natu * Update 'nextrip' based on the length of the emulated instruction. 1471a0b78f09SPeter Grehan */ 1472a0b78f09SPeter Grehan vme->inst_length = vie->num_processed; 1473d087a399SNeel Natu vcpu->nextrip += vie->num_processed; 14741c73ea3eSNeel Natu VCPU_CTR1(vm, vcpuid, "nextrip updated to %#lx after instruction " 14751c73ea3eSNeel Natu "decoding", vcpu->nextrip); 1476a0b78f09SPeter Grehan 147708e3ff32SNeel Natu /* return to userland unless this is an in-kernel emulated device */ 1478565bbb86SNeel Natu if (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE) { 1479565bbb86SNeel Natu mread = lapic_mmio_read; 1480565bbb86SNeel Natu mwrite = lapic_mmio_write; 1481565bbb86SNeel Natu } else if (gpa >= VIOAPIC_BASE && gpa < VIOAPIC_BASE + VIOAPIC_SIZE) { 1482565bbb86SNeel Natu mread = vioapic_mmio_read; 1483565bbb86SNeel Natu mwrite = vioapic_mmio_write; 148408e3ff32SNeel Natu } else if (gpa >= VHPET_BASE && gpa < VHPET_BASE + VHPET_SIZE) { 148508e3ff32SNeel Natu mread = vhpet_mmio_read; 148608e3ff32SNeel Natu mwrite = vhpet_mmio_write; 1487565bbb86SNeel Natu } else { 1488becd9849SNeel Natu *retu = true; 1489318224bbSNeel Natu return (0); 1490318224bbSNeel Natu } 1491318224bbSNeel Natu 1492d665d229SNeel Natu error = vmm_emulate_instruction(vm, vcpuid, gpa, vie, paging, 1493d665d229SNeel Natu mread, mwrite, retu); 1494318224bbSNeel Natu 1495318224bbSNeel Natu return (error); 1496318224bbSNeel Natu } 1497318224bbSNeel Natu 1498b15a09c0SNeel Natu static int 1499b15a09c0SNeel Natu vm_handle_suspend(struct vm *vm, int vcpuid, bool *retu) 1500b15a09c0SNeel Natu { 1501b15a09c0SNeel Natu int i, done; 1502b15a09c0SNeel Natu struct vcpu *vcpu; 1503b15a09c0SNeel Natu 1504b15a09c0SNeel Natu done = 0; 1505b15a09c0SNeel Natu vcpu = &vm->vcpu[vcpuid]; 1506b15a09c0SNeel Natu 1507b15a09c0SNeel Natu CPU_SET_ATOMIC(vcpuid, &vm->suspended_cpus); 1508b15a09c0SNeel Natu 1509b15a09c0SNeel Natu /* 1510b15a09c0SNeel Natu * Wait until all 'active_cpus' have suspended themselves. 1511b15a09c0SNeel Natu * 1512b15a09c0SNeel Natu * Since a VM may be suspended at any time including when one or 1513b15a09c0SNeel Natu * more vcpus are doing a rendezvous we need to call the rendezvous 1514b15a09c0SNeel Natu * handler while we are waiting to prevent a deadlock. 1515b15a09c0SNeel Natu */ 1516b15a09c0SNeel Natu vcpu_lock(vcpu); 1517b15a09c0SNeel Natu while (1) { 1518b15a09c0SNeel Natu if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) { 1519b15a09c0SNeel Natu VCPU_CTR0(vm, vcpuid, "All vcpus suspended"); 1520b15a09c0SNeel Natu break; 1521b15a09c0SNeel Natu } 1522b15a09c0SNeel Natu 1523b15a09c0SNeel Natu if (vm->rendezvous_func == NULL) { 1524b15a09c0SNeel Natu VCPU_CTR0(vm, vcpuid, "Sleeping during suspend"); 1525248e6799SNeel Natu vcpu_require_state_locked(vm, vcpuid, VCPU_SLEEPING); 1526b15a09c0SNeel Natu msleep_spin(vcpu, &vcpu->mtx, "vmsusp", hz); 1527248e6799SNeel Natu vcpu_require_state_locked(vm, vcpuid, VCPU_FROZEN); 1528b15a09c0SNeel Natu } else { 1529b15a09c0SNeel Natu VCPU_CTR0(vm, vcpuid, "Rendezvous during suspend"); 1530b15a09c0SNeel Natu vcpu_unlock(vcpu); 1531b15a09c0SNeel Natu vm_handle_rendezvous(vm, vcpuid); 1532b15a09c0SNeel Natu vcpu_lock(vcpu); 1533b15a09c0SNeel Natu } 1534b15a09c0SNeel Natu } 1535b15a09c0SNeel Natu vcpu_unlock(vcpu); 1536b15a09c0SNeel Natu 1537b15a09c0SNeel Natu /* 1538b15a09c0SNeel Natu * Wakeup the other sleeping vcpus and return to userspace. 1539b15a09c0SNeel Natu */ 1540b15a09c0SNeel Natu for (i = 0; i < VM_MAXCPU; i++) { 1541b15a09c0SNeel Natu if (CPU_ISSET(i, &vm->suspended_cpus)) { 1542b15a09c0SNeel Natu vcpu_notify_event(vm, i, false); 1543b15a09c0SNeel Natu } 1544b15a09c0SNeel Natu } 1545b15a09c0SNeel Natu 1546b15a09c0SNeel Natu *retu = true; 1547b15a09c0SNeel Natu return (0); 1548b15a09c0SNeel Natu } 1549b15a09c0SNeel Natu 1550248e6799SNeel Natu static int 1551248e6799SNeel Natu vm_handle_reqidle(struct vm *vm, int vcpuid, bool *retu) 1552248e6799SNeel Natu { 1553248e6799SNeel Natu struct vcpu *vcpu = &vm->vcpu[vcpuid]; 1554248e6799SNeel Natu 1555248e6799SNeel Natu vcpu_lock(vcpu); 1556248e6799SNeel Natu KASSERT(vcpu->reqidle, ("invalid vcpu reqidle %d", vcpu->reqidle)); 1557248e6799SNeel Natu vcpu->reqidle = 0; 1558248e6799SNeel Natu vcpu_unlock(vcpu); 1559248e6799SNeel Natu *retu = true; 1560248e6799SNeel Natu return (0); 1561248e6799SNeel Natu } 1562248e6799SNeel Natu 1563b15a09c0SNeel Natu int 1564f0fdcfe2SNeel Natu vm_suspend(struct vm *vm, enum vm_suspend_how how) 1565b15a09c0SNeel Natu { 1566f0fdcfe2SNeel Natu int i; 1567b15a09c0SNeel Natu 1568f0fdcfe2SNeel Natu if (how <= VM_SUSPEND_NONE || how >= VM_SUSPEND_LAST) 1569f0fdcfe2SNeel Natu return (EINVAL); 1570f0fdcfe2SNeel Natu 1571f0fdcfe2SNeel Natu if (atomic_cmpset_int(&vm->suspend, 0, how) == 0) { 1572f0fdcfe2SNeel Natu VM_CTR2(vm, "virtual machine already suspended %d/%d", 1573f0fdcfe2SNeel Natu vm->suspend, how); 1574b15a09c0SNeel Natu return (EALREADY); 1575b15a09c0SNeel Natu } 1576f0fdcfe2SNeel Natu 1577f0fdcfe2SNeel Natu VM_CTR1(vm, "virtual machine successfully suspended %d", how); 1578f0fdcfe2SNeel Natu 1579f0fdcfe2SNeel Natu /* 1580f0fdcfe2SNeel Natu * Notify all active vcpus that they are now suspended. 1581f0fdcfe2SNeel Natu */ 1582f0fdcfe2SNeel Natu for (i = 0; i < VM_MAXCPU; i++) { 1583f0fdcfe2SNeel Natu if (CPU_ISSET(i, &vm->active_cpus)) 1584f0fdcfe2SNeel Natu vcpu_notify_event(vm, i, false); 1585f0fdcfe2SNeel Natu } 1586f0fdcfe2SNeel Natu 1587f0fdcfe2SNeel Natu return (0); 1588f0fdcfe2SNeel Natu } 1589f0fdcfe2SNeel Natu 1590f0fdcfe2SNeel Natu void 1591f0fdcfe2SNeel Natu vm_exit_suspended(struct vm *vm, int vcpuid, uint64_t rip) 1592f0fdcfe2SNeel Natu { 1593f0fdcfe2SNeel Natu struct vm_exit *vmexit; 1594f0fdcfe2SNeel Natu 1595f0fdcfe2SNeel Natu KASSERT(vm->suspend > VM_SUSPEND_NONE && vm->suspend < VM_SUSPEND_LAST, 1596f0fdcfe2SNeel Natu ("vm_exit_suspended: invalid suspend type %d", vm->suspend)); 1597f0fdcfe2SNeel Natu 1598f0fdcfe2SNeel Natu vmexit = vm_exitinfo(vm, vcpuid); 1599f0fdcfe2SNeel Natu vmexit->rip = rip; 1600f0fdcfe2SNeel Natu vmexit->inst_length = 0; 1601f0fdcfe2SNeel Natu vmexit->exitcode = VM_EXITCODE_SUSPENDED; 1602f0fdcfe2SNeel Natu vmexit->u.suspended.how = vm->suspend; 1603b15a09c0SNeel Natu } 1604b15a09c0SNeel Natu 160540487465SNeel Natu void 1606fc276d92SJohn Baldwin vm_exit_debug(struct vm *vm, int vcpuid, uint64_t rip) 1607fc276d92SJohn Baldwin { 1608fc276d92SJohn Baldwin struct vm_exit *vmexit; 1609fc276d92SJohn Baldwin 1610fc276d92SJohn Baldwin vmexit = vm_exitinfo(vm, vcpuid); 1611fc276d92SJohn Baldwin vmexit->rip = rip; 1612fc276d92SJohn Baldwin vmexit->inst_length = 0; 1613fc276d92SJohn Baldwin vmexit->exitcode = VM_EXITCODE_DEBUG; 1614fc276d92SJohn Baldwin } 1615fc276d92SJohn Baldwin 1616fc276d92SJohn Baldwin void 161740487465SNeel Natu vm_exit_rendezvous(struct vm *vm, int vcpuid, uint64_t rip) 161840487465SNeel Natu { 161940487465SNeel Natu struct vm_exit *vmexit; 162040487465SNeel Natu 162140487465SNeel Natu KASSERT(vm->rendezvous_func != NULL, ("rendezvous not in progress")); 162240487465SNeel Natu 162340487465SNeel Natu vmexit = vm_exitinfo(vm, vcpuid); 162440487465SNeel Natu vmexit->rip = rip; 162540487465SNeel Natu vmexit->inst_length = 0; 162640487465SNeel Natu vmexit->exitcode = VM_EXITCODE_RENDEZVOUS; 162740487465SNeel Natu vmm_stat_incr(vm, vcpuid, VMEXIT_RENDEZVOUS, 1); 162840487465SNeel Natu } 162940487465SNeel Natu 163040487465SNeel Natu void 1631248e6799SNeel Natu vm_exit_reqidle(struct vm *vm, int vcpuid, uint64_t rip) 1632248e6799SNeel Natu { 1633248e6799SNeel Natu struct vm_exit *vmexit; 1634248e6799SNeel Natu 1635248e6799SNeel Natu vmexit = vm_exitinfo(vm, vcpuid); 1636248e6799SNeel Natu vmexit->rip = rip; 1637248e6799SNeel Natu vmexit->inst_length = 0; 1638248e6799SNeel Natu vmexit->exitcode = VM_EXITCODE_REQIDLE; 1639248e6799SNeel Natu vmm_stat_incr(vm, vcpuid, VMEXIT_REQIDLE, 1); 1640248e6799SNeel Natu } 1641248e6799SNeel Natu 1642248e6799SNeel Natu void 164340487465SNeel Natu vm_exit_astpending(struct vm *vm, int vcpuid, uint64_t rip) 164440487465SNeel Natu { 164540487465SNeel Natu struct vm_exit *vmexit; 164640487465SNeel Natu 164740487465SNeel Natu vmexit = vm_exitinfo(vm, vcpuid); 164840487465SNeel Natu vmexit->rip = rip; 164940487465SNeel Natu vmexit->inst_length = 0; 165040487465SNeel Natu vmexit->exitcode = VM_EXITCODE_BOGUS; 165140487465SNeel Natu vmm_stat_incr(vm, vcpuid, VMEXIT_ASTPENDING, 1); 165240487465SNeel Natu } 165340487465SNeel Natu 1654318224bbSNeel Natu int 1655318224bbSNeel Natu vm_run(struct vm *vm, struct vm_run *vmrun) 1656318224bbSNeel Natu { 1657248e6799SNeel Natu struct vm_eventinfo evinfo; 1658318224bbSNeel Natu int error, vcpuid; 1659318224bbSNeel Natu struct vcpu *vcpu; 1660318224bbSNeel Natu struct pcb *pcb; 1661d087a399SNeel Natu uint64_t tscval; 1662318224bbSNeel Natu struct vm_exit *vme; 1663becd9849SNeel Natu bool retu, intr_disabled; 1664318224bbSNeel Natu pmap_t pmap; 1665318224bbSNeel Natu 1666318224bbSNeel Natu vcpuid = vmrun->cpuid; 1667318224bbSNeel Natu 1668318224bbSNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1669318224bbSNeel Natu return (EINVAL); 1670318224bbSNeel Natu 167195ebc360SNeel Natu if (!CPU_ISSET(vcpuid, &vm->active_cpus)) 167295ebc360SNeel Natu return (EINVAL); 167395ebc360SNeel Natu 167495ebc360SNeel Natu if (CPU_ISSET(vcpuid, &vm->suspended_cpus)) 167595ebc360SNeel Natu return (EINVAL); 167695ebc360SNeel Natu 1677318224bbSNeel Natu pmap = vmspace_pmap(vm->vmspace); 1678318224bbSNeel Natu vcpu = &vm->vcpu[vcpuid]; 1679318224bbSNeel Natu vme = &vcpu->exitinfo; 1680248e6799SNeel Natu evinfo.rptr = &vm->rendezvous_func; 1681248e6799SNeel Natu evinfo.sptr = &vm->suspend; 1682248e6799SNeel Natu evinfo.iptr = &vcpu->reqidle; 1683318224bbSNeel Natu restart: 1684318224bbSNeel Natu critical_enter(); 1685318224bbSNeel Natu 1686318224bbSNeel Natu KASSERT(!CPU_ISSET(curcpu, &pmap->pm_active), 1687318224bbSNeel Natu ("vm_run: absurd pm_active")); 1688318224bbSNeel Natu 1689318224bbSNeel Natu tscval = rdtsc(); 1690318224bbSNeel Natu 1691318224bbSNeel Natu pcb = PCPU_GET(curpcb); 1692318224bbSNeel Natu set_pcb_flags(pcb, PCB_FULL_IRET); 1693318224bbSNeel Natu 1694318224bbSNeel Natu restore_guest_fpustate(vcpu); 1695318224bbSNeel Natu 1696318224bbSNeel Natu vcpu_require_state(vm, vcpuid, VCPU_RUNNING); 1697248e6799SNeel Natu error = VMRUN(vm->cookie, vcpuid, vcpu->nextrip, pmap, &evinfo); 1698318224bbSNeel Natu vcpu_require_state(vm, vcpuid, VCPU_FROZEN); 1699318224bbSNeel Natu 1700318224bbSNeel Natu save_guest_fpustate(vcpu); 1701318224bbSNeel Natu 1702318224bbSNeel Natu vmm_stat_incr(vm, vcpuid, VCPU_TOTAL_RUNTIME, rdtsc() - tscval); 1703318224bbSNeel Natu 1704318224bbSNeel Natu critical_exit(); 1705318224bbSNeel Natu 1706318224bbSNeel Natu if (error == 0) { 1707becd9849SNeel Natu retu = false; 1708d087a399SNeel Natu vcpu->nextrip = vme->rip + vme->inst_length; 1709318224bbSNeel Natu switch (vme->exitcode) { 1710248e6799SNeel Natu case VM_EXITCODE_REQIDLE: 1711248e6799SNeel Natu error = vm_handle_reqidle(vm, vcpuid, &retu); 1712248e6799SNeel Natu break; 1713b15a09c0SNeel Natu case VM_EXITCODE_SUSPENDED: 1714b15a09c0SNeel Natu error = vm_handle_suspend(vm, vcpuid, &retu); 1715b15a09c0SNeel Natu break; 171630b94db8SNeel Natu case VM_EXITCODE_IOAPIC_EOI: 171730b94db8SNeel Natu vioapic_process_eoi(vm, vcpuid, 171830b94db8SNeel Natu vme->u.ioapic_eoi.vector); 171930b94db8SNeel Natu break; 17205b8a8cd1SNeel Natu case VM_EXITCODE_RENDEZVOUS: 17215b8a8cd1SNeel Natu vm_handle_rendezvous(vm, vcpuid); 17225b8a8cd1SNeel Natu error = 0; 17235b8a8cd1SNeel Natu break; 1724318224bbSNeel Natu case VM_EXITCODE_HLT: 1725becd9849SNeel Natu intr_disabled = ((vme->u.hlt.rflags & PSL_I) == 0); 17261c052192SNeel Natu error = vm_handle_hlt(vm, vcpuid, intr_disabled, &retu); 1727318224bbSNeel Natu break; 1728318224bbSNeel Natu case VM_EXITCODE_PAGING: 1729318224bbSNeel Natu error = vm_handle_paging(vm, vcpuid, &retu); 1730318224bbSNeel Natu break; 1731318224bbSNeel Natu case VM_EXITCODE_INST_EMUL: 1732318224bbSNeel Natu error = vm_handle_inst_emul(vm, vcpuid, &retu); 1733318224bbSNeel Natu break; 1734d17b5104SNeel Natu case VM_EXITCODE_INOUT: 1735d17b5104SNeel Natu case VM_EXITCODE_INOUT_STR: 1736d17b5104SNeel Natu error = vm_handle_inout(vm, vcpuid, vme, &retu); 1737d17b5104SNeel Natu break; 173865145c7fSNeel Natu case VM_EXITCODE_MONITOR: 173965145c7fSNeel Natu case VM_EXITCODE_MWAIT: 174065145c7fSNeel Natu vm_inject_ud(vm, vcpuid); 174165145c7fSNeel Natu break; 1742318224bbSNeel Natu default: 1743becd9849SNeel Natu retu = true; /* handled in userland */ 1744318224bbSNeel Natu break; 1745318224bbSNeel Natu } 1746318224bbSNeel Natu } 1747318224bbSNeel Natu 1748d087a399SNeel Natu if (error == 0 && retu == false) 1749f76fc5d4SNeel Natu goto restart; 1750f76fc5d4SNeel Natu 1751248e6799SNeel Natu VCPU_CTR2(vm, vcpuid, "retu %d/%d", error, vme->exitcode); 1752248e6799SNeel Natu 1753318224bbSNeel Natu /* copy the exit information */ 1754318224bbSNeel Natu bcopy(vme, &vmrun->vm_exit, sizeof(struct vm_exit)); 1755366f6083SPeter Grehan return (error); 1756366f6083SPeter Grehan } 1757366f6083SPeter Grehan 1758366f6083SPeter Grehan int 1759c9c75df4SNeel Natu vm_restart_instruction(void *arg, int vcpuid) 1760c9c75df4SNeel Natu { 1761d087a399SNeel Natu struct vm *vm; 1762c9c75df4SNeel Natu struct vcpu *vcpu; 1763d087a399SNeel Natu enum vcpu_state state; 1764d087a399SNeel Natu uint64_t rip; 1765d087a399SNeel Natu int error; 1766c9c75df4SNeel Natu 1767d087a399SNeel Natu vm = arg; 1768c9c75df4SNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1769c9c75df4SNeel Natu return (EINVAL); 1770c9c75df4SNeel Natu 1771c9c75df4SNeel Natu vcpu = &vm->vcpu[vcpuid]; 1772d087a399SNeel Natu state = vcpu_get_state(vm, vcpuid, NULL); 1773d087a399SNeel Natu if (state == VCPU_RUNNING) { 1774d087a399SNeel Natu /* 1775d087a399SNeel Natu * When a vcpu is "running" the next instruction is determined 1776d087a399SNeel Natu * by adding 'rip' and 'inst_length' in the vcpu's 'exitinfo'. 1777d087a399SNeel Natu * Thus setting 'inst_length' to zero will cause the current 1778d087a399SNeel Natu * instruction to be restarted. 1779d087a399SNeel Natu */ 1780c9c75df4SNeel Natu vcpu->exitinfo.inst_length = 0; 1781d087a399SNeel Natu VCPU_CTR1(vm, vcpuid, "restarting instruction at %#lx by " 1782d087a399SNeel Natu "setting inst_length to zero", vcpu->exitinfo.rip); 1783d087a399SNeel Natu } else if (state == VCPU_FROZEN) { 1784d087a399SNeel Natu /* 1785d087a399SNeel Natu * When a vcpu is "frozen" it is outside the critical section 1786d087a399SNeel Natu * around VMRUN() and 'nextrip' points to the next instruction. 1787d087a399SNeel Natu * Thus instruction restart is achieved by setting 'nextrip' 1788d087a399SNeel Natu * to the vcpu's %rip. 1789d087a399SNeel Natu */ 1790d087a399SNeel Natu error = vm_get_register(vm, vcpuid, VM_REG_GUEST_RIP, &rip); 1791d087a399SNeel Natu KASSERT(!error, ("%s: error %d getting rip", __func__, error)); 1792d087a399SNeel Natu VCPU_CTR2(vm, vcpuid, "restarting instruction by updating " 1793d087a399SNeel Natu "nextrip from %#lx to %#lx", vcpu->nextrip, rip); 1794d087a399SNeel Natu vcpu->nextrip = rip; 1795d087a399SNeel Natu } else { 1796d087a399SNeel Natu panic("%s: invalid state %d", __func__, state); 1797d087a399SNeel Natu } 1798c9c75df4SNeel Natu return (0); 1799c9c75df4SNeel Natu } 1800c9c75df4SNeel Natu 1801c9c75df4SNeel Natu int 1802091d4532SNeel Natu vm_exit_intinfo(struct vm *vm, int vcpuid, uint64_t info) 1803091d4532SNeel Natu { 1804091d4532SNeel Natu struct vcpu *vcpu; 1805091d4532SNeel Natu int type, vector; 1806091d4532SNeel Natu 1807091d4532SNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1808091d4532SNeel Natu return (EINVAL); 1809091d4532SNeel Natu 1810091d4532SNeel Natu vcpu = &vm->vcpu[vcpuid]; 1811091d4532SNeel Natu 1812091d4532SNeel Natu if (info & VM_INTINFO_VALID) { 1813091d4532SNeel Natu type = info & VM_INTINFO_TYPE; 1814091d4532SNeel Natu vector = info & 0xff; 1815091d4532SNeel Natu if (type == VM_INTINFO_NMI && vector != IDT_NMI) 1816091d4532SNeel Natu return (EINVAL); 1817091d4532SNeel Natu if (type == VM_INTINFO_HWEXCEPTION && vector >= 32) 1818091d4532SNeel Natu return (EINVAL); 1819091d4532SNeel Natu if (info & VM_INTINFO_RSVD) 1820091d4532SNeel Natu return (EINVAL); 1821091d4532SNeel Natu } else { 1822091d4532SNeel Natu info = 0; 1823091d4532SNeel Natu } 1824091d4532SNeel Natu VCPU_CTR2(vm, vcpuid, "%s: info1(%#lx)", __func__, info); 1825091d4532SNeel Natu vcpu->exitintinfo = info; 1826091d4532SNeel Natu return (0); 1827091d4532SNeel Natu } 1828091d4532SNeel Natu 1829091d4532SNeel Natu enum exc_class { 1830091d4532SNeel Natu EXC_BENIGN, 1831091d4532SNeel Natu EXC_CONTRIBUTORY, 1832091d4532SNeel Natu EXC_PAGEFAULT 1833091d4532SNeel Natu }; 1834091d4532SNeel Natu 1835091d4532SNeel Natu #define IDT_VE 20 /* Virtualization Exception (Intel specific) */ 1836091d4532SNeel Natu 1837091d4532SNeel Natu static enum exc_class 1838091d4532SNeel Natu exception_class(uint64_t info) 1839091d4532SNeel Natu { 1840091d4532SNeel Natu int type, vector; 1841091d4532SNeel Natu 1842091d4532SNeel Natu KASSERT(info & VM_INTINFO_VALID, ("intinfo must be valid: %#lx", info)); 1843091d4532SNeel Natu type = info & VM_INTINFO_TYPE; 1844091d4532SNeel Natu vector = info & 0xff; 1845091d4532SNeel Natu 1846091d4532SNeel Natu /* Table 6-4, "Interrupt and Exception Classes", Intel SDM, Vol 3 */ 1847091d4532SNeel Natu switch (type) { 1848091d4532SNeel Natu case VM_INTINFO_HWINTR: 1849091d4532SNeel Natu case VM_INTINFO_SWINTR: 1850091d4532SNeel Natu case VM_INTINFO_NMI: 1851091d4532SNeel Natu return (EXC_BENIGN); 1852091d4532SNeel Natu default: 1853091d4532SNeel Natu /* 1854091d4532SNeel Natu * Hardware exception. 1855091d4532SNeel Natu * 1856091d4532SNeel Natu * SVM and VT-x use identical type values to represent NMI, 1857091d4532SNeel Natu * hardware interrupt and software interrupt. 1858091d4532SNeel Natu * 1859091d4532SNeel Natu * SVM uses type '3' for all exceptions. VT-x uses type '3' 1860091d4532SNeel Natu * for exceptions except #BP and #OF. #BP and #OF use a type 1861091d4532SNeel Natu * value of '5' or '6'. Therefore we don't check for explicit 1862091d4532SNeel Natu * values of 'type' to classify 'intinfo' into a hardware 1863091d4532SNeel Natu * exception. 1864091d4532SNeel Natu */ 1865091d4532SNeel Natu break; 1866091d4532SNeel Natu } 1867091d4532SNeel Natu 1868091d4532SNeel Natu switch (vector) { 1869091d4532SNeel Natu case IDT_PF: 1870091d4532SNeel Natu case IDT_VE: 1871091d4532SNeel Natu return (EXC_PAGEFAULT); 1872091d4532SNeel Natu case IDT_DE: 1873091d4532SNeel Natu case IDT_TS: 1874091d4532SNeel Natu case IDT_NP: 1875091d4532SNeel Natu case IDT_SS: 1876091d4532SNeel Natu case IDT_GP: 1877091d4532SNeel Natu return (EXC_CONTRIBUTORY); 1878091d4532SNeel Natu default: 1879091d4532SNeel Natu return (EXC_BENIGN); 1880091d4532SNeel Natu } 1881091d4532SNeel Natu } 1882091d4532SNeel Natu 1883091d4532SNeel Natu static int 1884091d4532SNeel Natu nested_fault(struct vm *vm, int vcpuid, uint64_t info1, uint64_t info2, 1885091d4532SNeel Natu uint64_t *retinfo) 1886091d4532SNeel Natu { 1887091d4532SNeel Natu enum exc_class exc1, exc2; 1888091d4532SNeel Natu int type1, vector1; 1889091d4532SNeel Natu 1890091d4532SNeel Natu KASSERT(info1 & VM_INTINFO_VALID, ("info1 %#lx is not valid", info1)); 1891091d4532SNeel Natu KASSERT(info2 & VM_INTINFO_VALID, ("info2 %#lx is not valid", info2)); 1892091d4532SNeel Natu 1893091d4532SNeel Natu /* 1894091d4532SNeel Natu * If an exception occurs while attempting to call the double-fault 1895091d4532SNeel Natu * handler the processor enters shutdown mode (aka triple fault). 1896091d4532SNeel Natu */ 1897091d4532SNeel Natu type1 = info1 & VM_INTINFO_TYPE; 1898091d4532SNeel Natu vector1 = info1 & 0xff; 1899091d4532SNeel Natu if (type1 == VM_INTINFO_HWEXCEPTION && vector1 == IDT_DF) { 1900091d4532SNeel Natu VCPU_CTR2(vm, vcpuid, "triple fault: info1(%#lx), info2(%#lx)", 1901091d4532SNeel Natu info1, info2); 1902091d4532SNeel Natu vm_suspend(vm, VM_SUSPEND_TRIPLEFAULT); 1903091d4532SNeel Natu *retinfo = 0; 1904091d4532SNeel Natu return (0); 1905091d4532SNeel Natu } 1906091d4532SNeel Natu 1907091d4532SNeel Natu /* 1908091d4532SNeel Natu * Table 6-5 "Conditions for Generating a Double Fault", Intel SDM, Vol3 1909091d4532SNeel Natu */ 1910091d4532SNeel Natu exc1 = exception_class(info1); 1911091d4532SNeel Natu exc2 = exception_class(info2); 1912091d4532SNeel Natu if ((exc1 == EXC_CONTRIBUTORY && exc2 == EXC_CONTRIBUTORY) || 1913091d4532SNeel Natu (exc1 == EXC_PAGEFAULT && exc2 != EXC_BENIGN)) { 1914091d4532SNeel Natu /* Convert nested fault into a double fault. */ 1915091d4532SNeel Natu *retinfo = IDT_DF; 1916091d4532SNeel Natu *retinfo |= VM_INTINFO_VALID | VM_INTINFO_HWEXCEPTION; 1917091d4532SNeel Natu *retinfo |= VM_INTINFO_DEL_ERRCODE; 1918091d4532SNeel Natu } else { 1919091d4532SNeel Natu /* Handle exceptions serially */ 1920091d4532SNeel Natu *retinfo = info2; 1921091d4532SNeel Natu } 1922091d4532SNeel Natu return (1); 1923091d4532SNeel Natu } 1924091d4532SNeel Natu 1925091d4532SNeel Natu static uint64_t 1926091d4532SNeel Natu vcpu_exception_intinfo(struct vcpu *vcpu) 1927091d4532SNeel Natu { 1928091d4532SNeel Natu uint64_t info = 0; 1929091d4532SNeel Natu 1930091d4532SNeel Natu if (vcpu->exception_pending) { 1931c9c75df4SNeel Natu info = vcpu->exc_vector & 0xff; 1932091d4532SNeel Natu info |= VM_INTINFO_VALID | VM_INTINFO_HWEXCEPTION; 1933c9c75df4SNeel Natu if (vcpu->exc_errcode_valid) { 1934091d4532SNeel Natu info |= VM_INTINFO_DEL_ERRCODE; 1935c9c75df4SNeel Natu info |= (uint64_t)vcpu->exc_errcode << 32; 1936091d4532SNeel Natu } 1937091d4532SNeel Natu } 1938091d4532SNeel Natu return (info); 1939091d4532SNeel Natu } 1940091d4532SNeel Natu 1941091d4532SNeel Natu int 1942091d4532SNeel Natu vm_entry_intinfo(struct vm *vm, int vcpuid, uint64_t *retinfo) 1943091d4532SNeel Natu { 1944091d4532SNeel Natu struct vcpu *vcpu; 1945091d4532SNeel Natu uint64_t info1, info2; 1946091d4532SNeel Natu int valid; 1947091d4532SNeel Natu 1948091d4532SNeel Natu KASSERT(vcpuid >= 0 && vcpuid < VM_MAXCPU, ("invalid vcpu %d", vcpuid)); 1949091d4532SNeel Natu 1950091d4532SNeel Natu vcpu = &vm->vcpu[vcpuid]; 1951091d4532SNeel Natu 1952091d4532SNeel Natu info1 = vcpu->exitintinfo; 1953091d4532SNeel Natu vcpu->exitintinfo = 0; 1954091d4532SNeel Natu 1955091d4532SNeel Natu info2 = 0; 1956091d4532SNeel Natu if (vcpu->exception_pending) { 1957091d4532SNeel Natu info2 = vcpu_exception_intinfo(vcpu); 1958091d4532SNeel Natu vcpu->exception_pending = 0; 1959091d4532SNeel Natu VCPU_CTR2(vm, vcpuid, "Exception %d delivered: %#lx", 1960c9c75df4SNeel Natu vcpu->exc_vector, info2); 1961091d4532SNeel Natu } 1962091d4532SNeel Natu 1963091d4532SNeel Natu if ((info1 & VM_INTINFO_VALID) && (info2 & VM_INTINFO_VALID)) { 1964091d4532SNeel Natu valid = nested_fault(vm, vcpuid, info1, info2, retinfo); 1965091d4532SNeel Natu } else if (info1 & VM_INTINFO_VALID) { 1966091d4532SNeel Natu *retinfo = info1; 1967091d4532SNeel Natu valid = 1; 1968091d4532SNeel Natu } else if (info2 & VM_INTINFO_VALID) { 1969091d4532SNeel Natu *retinfo = info2; 1970091d4532SNeel Natu valid = 1; 1971091d4532SNeel Natu } else { 1972091d4532SNeel Natu valid = 0; 1973091d4532SNeel Natu } 1974091d4532SNeel Natu 1975091d4532SNeel Natu if (valid) { 1976091d4532SNeel Natu VCPU_CTR4(vm, vcpuid, "%s: info1(%#lx), info2(%#lx), " 1977091d4532SNeel Natu "retinfo(%#lx)", __func__, info1, info2, *retinfo); 1978091d4532SNeel Natu } 1979091d4532SNeel Natu 1980091d4532SNeel Natu return (valid); 1981091d4532SNeel Natu } 1982091d4532SNeel Natu 1983091d4532SNeel Natu int 1984091d4532SNeel Natu vm_get_intinfo(struct vm *vm, int vcpuid, uint64_t *info1, uint64_t *info2) 1985091d4532SNeel Natu { 1986091d4532SNeel Natu struct vcpu *vcpu; 1987091d4532SNeel Natu 1988091d4532SNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1989091d4532SNeel Natu return (EINVAL); 1990091d4532SNeel Natu 1991091d4532SNeel Natu vcpu = &vm->vcpu[vcpuid]; 1992091d4532SNeel Natu *info1 = vcpu->exitintinfo; 1993091d4532SNeel Natu *info2 = vcpu_exception_intinfo(vcpu); 1994091d4532SNeel Natu return (0); 1995091d4532SNeel Natu } 1996091d4532SNeel Natu 1997091d4532SNeel Natu int 1998c9c75df4SNeel Natu vm_inject_exception(struct vm *vm, int vcpuid, int vector, int errcode_valid, 1999c9c75df4SNeel Natu uint32_t errcode, int restart_instruction) 2000366f6083SPeter Grehan { 2001dc506506SNeel Natu struct vcpu *vcpu; 200247b9935dSNeel Natu uint64_t regval; 20032ce12423SNeel Natu int error; 2004dc506506SNeel Natu 2005366f6083SPeter Grehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 2006366f6083SPeter Grehan return (EINVAL); 2007366f6083SPeter Grehan 2008c9c75df4SNeel Natu if (vector < 0 || vector >= 32) 2009366f6083SPeter Grehan return (EINVAL); 2010366f6083SPeter Grehan 2011091d4532SNeel Natu /* 2012091d4532SNeel Natu * A double fault exception should never be injected directly into 2013091d4532SNeel Natu * the guest. It is a derived exception that results from specific 2014091d4532SNeel Natu * combinations of nested faults. 2015091d4532SNeel Natu */ 2016c9c75df4SNeel Natu if (vector == IDT_DF) 2017091d4532SNeel Natu return (EINVAL); 2018091d4532SNeel Natu 2019dc506506SNeel Natu vcpu = &vm->vcpu[vcpuid]; 2020366f6083SPeter Grehan 2021dc506506SNeel Natu if (vcpu->exception_pending) { 2022dc506506SNeel Natu VCPU_CTR2(vm, vcpuid, "Unable to inject exception %d due to " 2023c9c75df4SNeel Natu "pending exception %d", vector, vcpu->exc_vector); 2024dc506506SNeel Natu return (EBUSY); 2025dc506506SNeel Natu } 2026dc506506SNeel Natu 202747b9935dSNeel Natu if (errcode_valid) { 202847b9935dSNeel Natu /* 202947b9935dSNeel Natu * Exceptions don't deliver an error code in real mode. 203047b9935dSNeel Natu */ 203147b9935dSNeel Natu error = vm_get_register(vm, vcpuid, VM_REG_GUEST_CR0, ®val); 203247b9935dSNeel Natu KASSERT(!error, ("%s: error %d getting CR0", __func__, error)); 203347b9935dSNeel Natu if (!(regval & CR0_PE)) 203447b9935dSNeel Natu errcode_valid = 0; 203547b9935dSNeel Natu } 203647b9935dSNeel Natu 20372ce12423SNeel Natu /* 20382ce12423SNeel Natu * From section 26.6.1 "Interruptibility State" in Intel SDM: 20392ce12423SNeel Natu * 20402ce12423SNeel Natu * Event blocking by "STI" or "MOV SS" is cleared after guest executes 20412ce12423SNeel Natu * one instruction or incurs an exception. 20422ce12423SNeel Natu */ 20432ce12423SNeel Natu error = vm_set_register(vm, vcpuid, VM_REG_GUEST_INTR_SHADOW, 0); 20442ce12423SNeel Natu KASSERT(error == 0, ("%s: error %d clearing interrupt shadow", 20452ce12423SNeel Natu __func__, error)); 20462ce12423SNeel Natu 2047c9c75df4SNeel Natu if (restart_instruction) 2048c9c75df4SNeel Natu vm_restart_instruction(vm, vcpuid); 2049c9c75df4SNeel Natu 2050dc506506SNeel Natu vcpu->exception_pending = 1; 2051c9c75df4SNeel Natu vcpu->exc_vector = vector; 2052c9c75df4SNeel Natu vcpu->exc_errcode = errcode; 2053c9c75df4SNeel Natu vcpu->exc_errcode_valid = errcode_valid; 2054c9c75df4SNeel Natu VCPU_CTR1(vm, vcpuid, "Exception %d pending", vector); 2055dc506506SNeel Natu return (0); 2056dc506506SNeel Natu } 2057dc506506SNeel Natu 2058d37f2adbSNeel Natu void 2059d37f2adbSNeel Natu vm_inject_fault(void *vmarg, int vcpuid, int vector, int errcode_valid, 2060d37f2adbSNeel Natu int errcode) 2061dc506506SNeel Natu { 2062d37f2adbSNeel Natu struct vm *vm; 2063c9c75df4SNeel Natu int error, restart_instruction; 2064dc506506SNeel Natu 2065d37f2adbSNeel Natu vm = vmarg; 2066c9c75df4SNeel Natu restart_instruction = 1; 2067d37f2adbSNeel Natu 2068c9c75df4SNeel Natu error = vm_inject_exception(vm, vcpuid, vector, errcode_valid, 2069c9c75df4SNeel Natu errcode, restart_instruction); 2070dc506506SNeel Natu KASSERT(error == 0, ("vm_inject_exception error %d", error)); 2071dc506506SNeel Natu } 2072dc506506SNeel Natu 2073dc506506SNeel Natu void 2074d37f2adbSNeel Natu vm_inject_pf(void *vmarg, int vcpuid, int error_code, uint64_t cr2) 2075fd949af6SNeel Natu { 2076d37f2adbSNeel Natu struct vm *vm; 207737a723a5SNeel Natu int error; 207837a723a5SNeel Natu 2079d37f2adbSNeel Natu vm = vmarg; 208037a723a5SNeel Natu VCPU_CTR2(vm, vcpuid, "Injecting page fault: error_code %#x, cr2 %#lx", 208137a723a5SNeel Natu error_code, cr2); 208237a723a5SNeel Natu 208337a723a5SNeel Natu error = vm_set_register(vm, vcpuid, VM_REG_GUEST_CR2, cr2); 208437a723a5SNeel Natu KASSERT(error == 0, ("vm_set_register(cr2) error %d", error)); 2085fd949af6SNeel Natu 2086d37f2adbSNeel Natu vm_inject_fault(vm, vcpuid, IDT_PF, 1, error_code); 2087366f6083SPeter Grehan } 2088366f6083SPeter Grehan 208961592433SNeel Natu static VMM_STAT(VCPU_NMI_COUNT, "number of NMIs delivered to vcpu"); 2090366f6083SPeter Grehan 2091f352ff0cSNeel Natu int 2092f352ff0cSNeel Natu vm_inject_nmi(struct vm *vm, int vcpuid) 2093f352ff0cSNeel Natu { 2094f352ff0cSNeel Natu struct vcpu *vcpu; 2095f352ff0cSNeel Natu 2096f352ff0cSNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 2097366f6083SPeter Grehan return (EINVAL); 2098366f6083SPeter Grehan 2099f352ff0cSNeel Natu vcpu = &vm->vcpu[vcpuid]; 2100f352ff0cSNeel Natu 2101f352ff0cSNeel Natu vcpu->nmi_pending = 1; 2102de5ea6b6SNeel Natu vcpu_notify_event(vm, vcpuid, false); 2103f352ff0cSNeel Natu return (0); 2104f352ff0cSNeel Natu } 2105f352ff0cSNeel Natu 2106f352ff0cSNeel Natu int 2107f352ff0cSNeel Natu vm_nmi_pending(struct vm *vm, int vcpuid) 2108f352ff0cSNeel Natu { 2109f352ff0cSNeel Natu struct vcpu *vcpu; 2110f352ff0cSNeel Natu 2111f352ff0cSNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 2112f352ff0cSNeel Natu panic("vm_nmi_pending: invalid vcpuid %d", vcpuid); 2113f352ff0cSNeel Natu 2114f352ff0cSNeel Natu vcpu = &vm->vcpu[vcpuid]; 2115f352ff0cSNeel Natu 2116f352ff0cSNeel Natu return (vcpu->nmi_pending); 2117f352ff0cSNeel Natu } 2118f352ff0cSNeel Natu 2119f352ff0cSNeel Natu void 2120f352ff0cSNeel Natu vm_nmi_clear(struct vm *vm, int vcpuid) 2121f352ff0cSNeel Natu { 2122f352ff0cSNeel Natu struct vcpu *vcpu; 2123f352ff0cSNeel Natu 2124f352ff0cSNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 2125f352ff0cSNeel Natu panic("vm_nmi_pending: invalid vcpuid %d", vcpuid); 2126f352ff0cSNeel Natu 2127f352ff0cSNeel Natu vcpu = &vm->vcpu[vcpuid]; 2128f352ff0cSNeel Natu 2129f352ff0cSNeel Natu if (vcpu->nmi_pending == 0) 2130f352ff0cSNeel Natu panic("vm_nmi_clear: inconsistent nmi_pending state"); 2131f352ff0cSNeel Natu 2132f352ff0cSNeel Natu vcpu->nmi_pending = 0; 2133f352ff0cSNeel Natu vmm_stat_incr(vm, vcpuid, VCPU_NMI_COUNT, 1); 2134366f6083SPeter Grehan } 2135366f6083SPeter Grehan 21360775fbb4STycho Nightingale static VMM_STAT(VCPU_EXTINT_COUNT, "number of ExtINTs delivered to vcpu"); 21370775fbb4STycho Nightingale 21380775fbb4STycho Nightingale int 21390775fbb4STycho Nightingale vm_inject_extint(struct vm *vm, int vcpuid) 21400775fbb4STycho Nightingale { 21410775fbb4STycho Nightingale struct vcpu *vcpu; 21420775fbb4STycho Nightingale 21430775fbb4STycho Nightingale if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 21440775fbb4STycho Nightingale return (EINVAL); 21450775fbb4STycho Nightingale 21460775fbb4STycho Nightingale vcpu = &vm->vcpu[vcpuid]; 21470775fbb4STycho Nightingale 21480775fbb4STycho Nightingale vcpu->extint_pending = 1; 21490775fbb4STycho Nightingale vcpu_notify_event(vm, vcpuid, false); 21500775fbb4STycho Nightingale return (0); 21510775fbb4STycho Nightingale } 21520775fbb4STycho Nightingale 21530775fbb4STycho Nightingale int 21540775fbb4STycho Nightingale vm_extint_pending(struct vm *vm, int vcpuid) 21550775fbb4STycho Nightingale { 21560775fbb4STycho Nightingale struct vcpu *vcpu; 21570775fbb4STycho Nightingale 21580775fbb4STycho Nightingale if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 21590775fbb4STycho Nightingale panic("vm_extint_pending: invalid vcpuid %d", vcpuid); 21600775fbb4STycho Nightingale 21610775fbb4STycho Nightingale vcpu = &vm->vcpu[vcpuid]; 21620775fbb4STycho Nightingale 21630775fbb4STycho Nightingale return (vcpu->extint_pending); 21640775fbb4STycho Nightingale } 21650775fbb4STycho Nightingale 21660775fbb4STycho Nightingale void 21670775fbb4STycho Nightingale vm_extint_clear(struct vm *vm, int vcpuid) 21680775fbb4STycho Nightingale { 21690775fbb4STycho Nightingale struct vcpu *vcpu; 21700775fbb4STycho Nightingale 21710775fbb4STycho Nightingale if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 21720775fbb4STycho Nightingale panic("vm_extint_pending: invalid vcpuid %d", vcpuid); 21730775fbb4STycho Nightingale 21740775fbb4STycho Nightingale vcpu = &vm->vcpu[vcpuid]; 21750775fbb4STycho Nightingale 21760775fbb4STycho Nightingale if (vcpu->extint_pending == 0) 21770775fbb4STycho Nightingale panic("vm_extint_clear: inconsistent extint_pending state"); 21780775fbb4STycho Nightingale 21790775fbb4STycho Nightingale vcpu->extint_pending = 0; 21800775fbb4STycho Nightingale vmm_stat_incr(vm, vcpuid, VCPU_EXTINT_COUNT, 1); 21810775fbb4STycho Nightingale } 21820775fbb4STycho Nightingale 2183366f6083SPeter Grehan int 2184366f6083SPeter Grehan vm_get_capability(struct vm *vm, int vcpu, int type, int *retval) 2185366f6083SPeter Grehan { 2186366f6083SPeter Grehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 2187366f6083SPeter Grehan return (EINVAL); 2188366f6083SPeter Grehan 2189366f6083SPeter Grehan if (type < 0 || type >= VM_CAP_MAX) 2190366f6083SPeter Grehan return (EINVAL); 2191366f6083SPeter Grehan 2192366f6083SPeter Grehan return (VMGETCAP(vm->cookie, vcpu, type, retval)); 2193366f6083SPeter Grehan } 2194366f6083SPeter Grehan 2195366f6083SPeter Grehan int 2196366f6083SPeter Grehan vm_set_capability(struct vm *vm, int vcpu, int type, int val) 2197366f6083SPeter Grehan { 2198366f6083SPeter Grehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 2199366f6083SPeter Grehan return (EINVAL); 2200366f6083SPeter Grehan 2201366f6083SPeter Grehan if (type < 0 || type >= VM_CAP_MAX) 2202366f6083SPeter Grehan return (EINVAL); 2203366f6083SPeter Grehan 2204366f6083SPeter Grehan return (VMSETCAP(vm->cookie, vcpu, type, val)); 2205366f6083SPeter Grehan } 2206366f6083SPeter Grehan 2207366f6083SPeter Grehan struct vlapic * 2208366f6083SPeter Grehan vm_lapic(struct vm *vm, int cpu) 2209366f6083SPeter Grehan { 2210366f6083SPeter Grehan return (vm->vcpu[cpu].vlapic); 2211366f6083SPeter Grehan } 2212366f6083SPeter Grehan 2213565bbb86SNeel Natu struct vioapic * 2214565bbb86SNeel Natu vm_ioapic(struct vm *vm) 2215565bbb86SNeel Natu { 2216565bbb86SNeel Natu 2217565bbb86SNeel Natu return (vm->vioapic); 2218565bbb86SNeel Natu } 2219565bbb86SNeel Natu 222008e3ff32SNeel Natu struct vhpet * 222108e3ff32SNeel Natu vm_hpet(struct vm *vm) 222208e3ff32SNeel Natu { 222308e3ff32SNeel Natu 222408e3ff32SNeel Natu return (vm->vhpet); 222508e3ff32SNeel Natu } 222608e3ff32SNeel Natu 2227366f6083SPeter Grehan boolean_t 2228366f6083SPeter Grehan vmm_is_pptdev(int bus, int slot, int func) 2229366f6083SPeter Grehan { 223007044a96SNeel Natu int found, i, n; 223107044a96SNeel Natu int b, s, f; 2232366f6083SPeter Grehan char *val, *cp, *cp2; 2233366f6083SPeter Grehan 2234366f6083SPeter Grehan /* 223507044a96SNeel Natu * XXX 223607044a96SNeel Natu * The length of an environment variable is limited to 128 bytes which 223707044a96SNeel Natu * puts an upper limit on the number of passthru devices that may be 223807044a96SNeel Natu * specified using a single environment variable. 223907044a96SNeel Natu * 224007044a96SNeel Natu * Work around this by scanning multiple environment variable 224107044a96SNeel Natu * names instead of a single one - yuck! 2242366f6083SPeter Grehan */ 224307044a96SNeel Natu const char *names[] = { "pptdevs", "pptdevs2", "pptdevs3", NULL }; 224407044a96SNeel Natu 224507044a96SNeel Natu /* set pptdevs="1/2/3 4/5/6 7/8/9 10/11/12" */ 2246366f6083SPeter Grehan found = 0; 224707044a96SNeel Natu for (i = 0; names[i] != NULL && !found; i++) { 22482be111bfSDavide Italiano cp = val = kern_getenv(names[i]); 2249366f6083SPeter Grehan while (cp != NULL && *cp != '\0') { 2250366f6083SPeter Grehan if ((cp2 = strchr(cp, ' ')) != NULL) 2251366f6083SPeter Grehan *cp2 = '\0'; 2252366f6083SPeter Grehan 2253366f6083SPeter Grehan n = sscanf(cp, "%d/%d/%d", &b, &s, &f); 2254366f6083SPeter Grehan if (n == 3 && bus == b && slot == s && func == f) { 2255366f6083SPeter Grehan found = 1; 2256366f6083SPeter Grehan break; 2257366f6083SPeter Grehan } 2258366f6083SPeter Grehan 2259366f6083SPeter Grehan if (cp2 != NULL) 2260366f6083SPeter Grehan *cp2++ = ' '; 2261366f6083SPeter Grehan 2262366f6083SPeter Grehan cp = cp2; 2263366f6083SPeter Grehan } 2264366f6083SPeter Grehan freeenv(val); 226507044a96SNeel Natu } 2266366f6083SPeter Grehan return (found); 2267366f6083SPeter Grehan } 2268366f6083SPeter Grehan 2269366f6083SPeter Grehan void * 2270366f6083SPeter Grehan vm_iommu_domain(struct vm *vm) 2271366f6083SPeter Grehan { 2272366f6083SPeter Grehan 2273366f6083SPeter Grehan return (vm->iommu); 2274366f6083SPeter Grehan } 2275366f6083SPeter Grehan 227675dd3366SNeel Natu int 2277f80330a8SNeel Natu vcpu_set_state(struct vm *vm, int vcpuid, enum vcpu_state newstate, 2278f80330a8SNeel Natu bool from_idle) 2279366f6083SPeter Grehan { 228075dd3366SNeel Natu int error; 2281366f6083SPeter Grehan struct vcpu *vcpu; 2282366f6083SPeter Grehan 2283366f6083SPeter Grehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 2284366f6083SPeter Grehan panic("vm_set_run_state: invalid vcpuid %d", vcpuid); 2285366f6083SPeter Grehan 2286366f6083SPeter Grehan vcpu = &vm->vcpu[vcpuid]; 2287366f6083SPeter Grehan 228875dd3366SNeel Natu vcpu_lock(vcpu); 2289248e6799SNeel Natu error = vcpu_set_state_locked(vm, vcpuid, newstate, from_idle); 229075dd3366SNeel Natu vcpu_unlock(vcpu); 229175dd3366SNeel Natu 229275dd3366SNeel Natu return (error); 229375dd3366SNeel Natu } 229475dd3366SNeel Natu 229575dd3366SNeel Natu enum vcpu_state 2296d3c11f40SPeter Grehan vcpu_get_state(struct vm *vm, int vcpuid, int *hostcpu) 2297366f6083SPeter Grehan { 2298366f6083SPeter Grehan struct vcpu *vcpu; 229975dd3366SNeel Natu enum vcpu_state state; 2300366f6083SPeter Grehan 2301366f6083SPeter Grehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 2302366f6083SPeter Grehan panic("vm_get_run_state: invalid vcpuid %d", vcpuid); 2303366f6083SPeter Grehan 2304366f6083SPeter Grehan vcpu = &vm->vcpu[vcpuid]; 2305366f6083SPeter Grehan 230675dd3366SNeel Natu vcpu_lock(vcpu); 230775dd3366SNeel Natu state = vcpu->state; 2308d3c11f40SPeter Grehan if (hostcpu != NULL) 2309d3c11f40SPeter Grehan *hostcpu = vcpu->hostcpu; 231075dd3366SNeel Natu vcpu_unlock(vcpu); 2311366f6083SPeter Grehan 231275dd3366SNeel Natu return (state); 2313366f6083SPeter Grehan } 2314366f6083SPeter Grehan 231595ebc360SNeel Natu int 2316366f6083SPeter Grehan vm_activate_cpu(struct vm *vm, int vcpuid) 2317366f6083SPeter Grehan { 2318366f6083SPeter Grehan 231995ebc360SNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 232095ebc360SNeel Natu return (EINVAL); 232195ebc360SNeel Natu 232295ebc360SNeel Natu if (CPU_ISSET(vcpuid, &vm->active_cpus)) 232395ebc360SNeel Natu return (EBUSY); 232422d822c6SNeel Natu 232522d822c6SNeel Natu VCPU_CTR0(vm, vcpuid, "activated"); 232622d822c6SNeel Natu CPU_SET_ATOMIC(vcpuid, &vm->active_cpus); 232795ebc360SNeel Natu return (0); 2328366f6083SPeter Grehan } 2329366f6083SPeter Grehan 2330fc276d92SJohn Baldwin int 2331fc276d92SJohn Baldwin vm_suspend_cpu(struct vm *vm, int vcpuid) 2332fc276d92SJohn Baldwin { 2333fc276d92SJohn Baldwin int i; 2334fc276d92SJohn Baldwin 2335fc276d92SJohn Baldwin if (vcpuid < -1 || vcpuid >= VM_MAXCPU) 2336fc276d92SJohn Baldwin return (EINVAL); 2337fc276d92SJohn Baldwin 2338fc276d92SJohn Baldwin if (vcpuid == -1) { 2339fc276d92SJohn Baldwin vm->debug_cpus = vm->active_cpus; 2340fc276d92SJohn Baldwin for (i = 0; i < VM_MAXCPU; i++) { 2341fc276d92SJohn Baldwin if (CPU_ISSET(i, &vm->active_cpus)) 2342fc276d92SJohn Baldwin vcpu_notify_event(vm, i, false); 2343fc276d92SJohn Baldwin } 2344fc276d92SJohn Baldwin } else { 2345fc276d92SJohn Baldwin if (!CPU_ISSET(vcpuid, &vm->active_cpus)) 2346fc276d92SJohn Baldwin return (EINVAL); 2347fc276d92SJohn Baldwin 2348fc276d92SJohn Baldwin CPU_SET_ATOMIC(vcpuid, &vm->debug_cpus); 2349fc276d92SJohn Baldwin vcpu_notify_event(vm, vcpuid, false); 2350fc276d92SJohn Baldwin } 2351fc276d92SJohn Baldwin return (0); 2352fc276d92SJohn Baldwin } 2353fc276d92SJohn Baldwin 2354fc276d92SJohn Baldwin int 2355fc276d92SJohn Baldwin vm_resume_cpu(struct vm *vm, int vcpuid) 2356fc276d92SJohn Baldwin { 2357fc276d92SJohn Baldwin 2358fc276d92SJohn Baldwin if (vcpuid < -1 || vcpuid >= VM_MAXCPU) 2359fc276d92SJohn Baldwin return (EINVAL); 2360fc276d92SJohn Baldwin 2361fc276d92SJohn Baldwin if (vcpuid == -1) { 2362fc276d92SJohn Baldwin CPU_ZERO(&vm->debug_cpus); 2363fc276d92SJohn Baldwin } else { 2364fc276d92SJohn Baldwin if (!CPU_ISSET(vcpuid, &vm->debug_cpus)) 2365fc276d92SJohn Baldwin return (EINVAL); 2366fc276d92SJohn Baldwin 2367fc276d92SJohn Baldwin CPU_CLR_ATOMIC(vcpuid, &vm->debug_cpus); 2368fc276d92SJohn Baldwin } 2369fc276d92SJohn Baldwin return (0); 2370fc276d92SJohn Baldwin } 2371fc276d92SJohn Baldwin 2372fc276d92SJohn Baldwin int 2373fc276d92SJohn Baldwin vcpu_debugged(struct vm *vm, int vcpuid) 2374fc276d92SJohn Baldwin { 2375fc276d92SJohn Baldwin 2376fc276d92SJohn Baldwin return (CPU_ISSET(vcpuid, &vm->debug_cpus)); 2377fc276d92SJohn Baldwin } 2378fc276d92SJohn Baldwin 2379a5615c90SPeter Grehan cpuset_t 2380366f6083SPeter Grehan vm_active_cpus(struct vm *vm) 2381366f6083SPeter Grehan { 2382366f6083SPeter Grehan 2383366f6083SPeter Grehan return (vm->active_cpus); 2384366f6083SPeter Grehan } 2385366f6083SPeter Grehan 238695ebc360SNeel Natu cpuset_t 2387fc276d92SJohn Baldwin vm_debug_cpus(struct vm *vm) 2388fc276d92SJohn Baldwin { 2389fc276d92SJohn Baldwin 2390fc276d92SJohn Baldwin return (vm->debug_cpus); 2391fc276d92SJohn Baldwin } 2392fc276d92SJohn Baldwin 2393fc276d92SJohn Baldwin cpuset_t 239495ebc360SNeel Natu vm_suspended_cpus(struct vm *vm) 239595ebc360SNeel Natu { 239695ebc360SNeel Natu 239795ebc360SNeel Natu return (vm->suspended_cpus); 239895ebc360SNeel Natu } 239995ebc360SNeel Natu 2400366f6083SPeter Grehan void * 2401366f6083SPeter Grehan vcpu_stats(struct vm *vm, int vcpuid) 2402366f6083SPeter Grehan { 2403366f6083SPeter Grehan 2404366f6083SPeter Grehan return (vm->vcpu[vcpuid].stats); 2405366f6083SPeter Grehan } 2406e9027382SNeel Natu 2407e9027382SNeel Natu int 2408e9027382SNeel Natu vm_get_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state *state) 2409e9027382SNeel Natu { 2410e9027382SNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 2411e9027382SNeel Natu return (EINVAL); 2412e9027382SNeel Natu 2413e9027382SNeel Natu *state = vm->vcpu[vcpuid].x2apic_state; 2414e9027382SNeel Natu 2415e9027382SNeel Natu return (0); 2416e9027382SNeel Natu } 2417e9027382SNeel Natu 2418e9027382SNeel Natu int 2419e9027382SNeel Natu vm_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state) 2420e9027382SNeel Natu { 2421e9027382SNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 2422e9027382SNeel Natu return (EINVAL); 2423e9027382SNeel Natu 24243f23d3caSNeel Natu if (state >= X2APIC_STATE_LAST) 2425e9027382SNeel Natu return (EINVAL); 2426e9027382SNeel Natu 2427e9027382SNeel Natu vm->vcpu[vcpuid].x2apic_state = state; 2428e9027382SNeel Natu 242973820fb0SNeel Natu vlapic_set_x2apic_state(vm, vcpuid, state); 243073820fb0SNeel Natu 2431e9027382SNeel Natu return (0); 2432e9027382SNeel Natu } 243375dd3366SNeel Natu 243422821874SNeel Natu /* 243522821874SNeel Natu * This function is called to ensure that a vcpu "sees" a pending event 243622821874SNeel Natu * as soon as possible: 243722821874SNeel Natu * - If the vcpu thread is sleeping then it is woken up. 243822821874SNeel Natu * - If the vcpu is running on a different host_cpu then an IPI will be directed 243922821874SNeel Natu * to the host_cpu to cause the vcpu to trap into the hypervisor. 244022821874SNeel Natu */ 2441248e6799SNeel Natu static void 2442248e6799SNeel Natu vcpu_notify_event_locked(struct vcpu *vcpu, bool lapic_intr) 244375dd3366SNeel Natu { 244475dd3366SNeel Natu int hostcpu; 244575dd3366SNeel Natu 244675dd3366SNeel Natu hostcpu = vcpu->hostcpu; 2447ef39d7e9SNeel Natu if (vcpu->state == VCPU_RUNNING) { 2448ef39d7e9SNeel Natu KASSERT(hostcpu != NOCPU, ("vcpu running on invalid hostcpu")); 2449de5ea6b6SNeel Natu if (hostcpu != curcpu) { 2450ef39d7e9SNeel Natu if (lapic_intr) { 2451add611fdSNeel Natu vlapic_post_intr(vcpu->vlapic, hostcpu, 2452add611fdSNeel Natu vmm_ipinum); 2453ef39d7e9SNeel Natu } else { 245475dd3366SNeel Natu ipi_cpu(hostcpu, vmm_ipinum); 245575dd3366SNeel Natu } 2456ef39d7e9SNeel Natu } else { 2457ef39d7e9SNeel Natu /* 2458ef39d7e9SNeel Natu * If the 'vcpu' is running on 'curcpu' then it must 2459ef39d7e9SNeel Natu * be sending a notification to itself (e.g. SELF_IPI). 2460ef39d7e9SNeel Natu * The pending event will be picked up when the vcpu 2461ef39d7e9SNeel Natu * transitions back to guest context. 2462ef39d7e9SNeel Natu */ 2463ef39d7e9SNeel Natu } 2464ef39d7e9SNeel Natu } else { 2465ef39d7e9SNeel Natu KASSERT(hostcpu == NOCPU, ("vcpu state %d not consistent " 2466ef39d7e9SNeel Natu "with hostcpu %d", vcpu->state, hostcpu)); 2467366f6083SPeter Grehan if (vcpu->state == VCPU_SLEEPING) 2468366f6083SPeter Grehan wakeup_one(vcpu); 2469366f6083SPeter Grehan } 2470248e6799SNeel Natu } 2471248e6799SNeel Natu 2472248e6799SNeel Natu void 2473248e6799SNeel Natu vcpu_notify_event(struct vm *vm, int vcpuid, bool lapic_intr) 2474248e6799SNeel Natu { 2475248e6799SNeel Natu struct vcpu *vcpu = &vm->vcpu[vcpuid]; 2476248e6799SNeel Natu 2477248e6799SNeel Natu vcpu_lock(vcpu); 2478248e6799SNeel Natu vcpu_notify_event_locked(vcpu, lapic_intr); 2479f76fc5d4SNeel Natu vcpu_unlock(vcpu); 2480f76fc5d4SNeel Natu } 2481318224bbSNeel Natu 2482318224bbSNeel Natu struct vmspace * 2483318224bbSNeel Natu vm_get_vmspace(struct vm *vm) 2484318224bbSNeel Natu { 2485318224bbSNeel Natu 2486318224bbSNeel Natu return (vm->vmspace); 2487318224bbSNeel Natu } 2488565bbb86SNeel Natu 2489565bbb86SNeel Natu int 2490565bbb86SNeel Natu vm_apicid2vcpuid(struct vm *vm, int apicid) 2491565bbb86SNeel Natu { 2492565bbb86SNeel Natu /* 2493565bbb86SNeel Natu * XXX apic id is assumed to be numerically identical to vcpu id 2494565bbb86SNeel Natu */ 2495565bbb86SNeel Natu return (apicid); 2496565bbb86SNeel Natu } 24975b8a8cd1SNeel Natu 24985b8a8cd1SNeel Natu void 24995b8a8cd1SNeel Natu vm_smp_rendezvous(struct vm *vm, int vcpuid, cpuset_t dest, 25005b8a8cd1SNeel Natu vm_rendezvous_func_t func, void *arg) 25015b8a8cd1SNeel Natu { 2502970955e4SNeel Natu int i; 2503970955e4SNeel Natu 25045b8a8cd1SNeel Natu /* 25055b8a8cd1SNeel Natu * Enforce that this function is called without any locks 25065b8a8cd1SNeel Natu */ 25075b8a8cd1SNeel Natu WITNESS_WARN(WARN_PANIC, NULL, "vm_smp_rendezvous"); 25085b8a8cd1SNeel Natu KASSERT(vcpuid == -1 || (vcpuid >= 0 && vcpuid < VM_MAXCPU), 25095b8a8cd1SNeel Natu ("vm_smp_rendezvous: invalid vcpuid %d", vcpuid)); 25105b8a8cd1SNeel Natu 25115b8a8cd1SNeel Natu restart: 25125b8a8cd1SNeel Natu mtx_lock(&vm->rendezvous_mtx); 25135b8a8cd1SNeel Natu if (vm->rendezvous_func != NULL) { 25145b8a8cd1SNeel Natu /* 25155b8a8cd1SNeel Natu * If a rendezvous is already in progress then we need to 25165b8a8cd1SNeel Natu * call the rendezvous handler in case this 'vcpuid' is one 25175b8a8cd1SNeel Natu * of the targets of the rendezvous. 25185b8a8cd1SNeel Natu */ 25195b8a8cd1SNeel Natu RENDEZVOUS_CTR0(vm, vcpuid, "Rendezvous already in progress"); 25205b8a8cd1SNeel Natu mtx_unlock(&vm->rendezvous_mtx); 25215b8a8cd1SNeel Natu vm_handle_rendezvous(vm, vcpuid); 25225b8a8cd1SNeel Natu goto restart; 25235b8a8cd1SNeel Natu } 25245b8a8cd1SNeel Natu KASSERT(vm->rendezvous_func == NULL, ("vm_smp_rendezvous: previous " 25255b8a8cd1SNeel Natu "rendezvous is still in progress")); 25265b8a8cd1SNeel Natu 25275b8a8cd1SNeel Natu RENDEZVOUS_CTR0(vm, vcpuid, "Initiating rendezvous"); 25285b8a8cd1SNeel Natu vm->rendezvous_req_cpus = dest; 25295b8a8cd1SNeel Natu CPU_ZERO(&vm->rendezvous_done_cpus); 25305b8a8cd1SNeel Natu vm->rendezvous_arg = arg; 25315b8a8cd1SNeel Natu vm_set_rendezvous_func(vm, func); 25325b8a8cd1SNeel Natu mtx_unlock(&vm->rendezvous_mtx); 25335b8a8cd1SNeel Natu 2534970955e4SNeel Natu /* 2535970955e4SNeel Natu * Wake up any sleeping vcpus and trigger a VM-exit in any running 2536970955e4SNeel Natu * vcpus so they handle the rendezvous as soon as possible. 2537970955e4SNeel Natu */ 2538970955e4SNeel Natu for (i = 0; i < VM_MAXCPU; i++) { 2539970955e4SNeel Natu if (CPU_ISSET(i, &dest)) 2540970955e4SNeel Natu vcpu_notify_event(vm, i, false); 2541970955e4SNeel Natu } 2542970955e4SNeel Natu 25435b8a8cd1SNeel Natu vm_handle_rendezvous(vm, vcpuid); 25445b8a8cd1SNeel Natu } 2545762fd208STycho Nightingale 2546762fd208STycho Nightingale struct vatpic * 2547762fd208STycho Nightingale vm_atpic(struct vm *vm) 2548762fd208STycho Nightingale { 2549762fd208STycho Nightingale return (vm->vatpic); 2550762fd208STycho Nightingale } 2551e883c9bbSTycho Nightingale 2552e883c9bbSTycho Nightingale struct vatpit * 2553e883c9bbSTycho Nightingale vm_atpit(struct vm *vm) 2554e883c9bbSTycho Nightingale { 2555e883c9bbSTycho Nightingale return (vm->vatpit); 2556e883c9bbSTycho Nightingale } 2557d17b5104SNeel Natu 2558160ef77aSNeel Natu struct vpmtmr * 2559160ef77aSNeel Natu vm_pmtmr(struct vm *vm) 2560160ef77aSNeel Natu { 2561160ef77aSNeel Natu 2562160ef77aSNeel Natu return (vm->vpmtmr); 2563160ef77aSNeel Natu } 2564160ef77aSNeel Natu 25650dafa5cdSNeel Natu struct vrtc * 25660dafa5cdSNeel Natu vm_rtc(struct vm *vm) 25670dafa5cdSNeel Natu { 25680dafa5cdSNeel Natu 25690dafa5cdSNeel Natu return (vm->vrtc); 25700dafa5cdSNeel Natu } 25710dafa5cdSNeel Natu 2572d17b5104SNeel Natu enum vm_reg_name 2573d17b5104SNeel Natu vm_segment_name(int seg) 2574d17b5104SNeel Natu { 2575d17b5104SNeel Natu static enum vm_reg_name seg_names[] = { 2576d17b5104SNeel Natu VM_REG_GUEST_ES, 2577d17b5104SNeel Natu VM_REG_GUEST_CS, 2578d17b5104SNeel Natu VM_REG_GUEST_SS, 2579d17b5104SNeel Natu VM_REG_GUEST_DS, 2580d17b5104SNeel Natu VM_REG_GUEST_FS, 2581d17b5104SNeel Natu VM_REG_GUEST_GS 2582d17b5104SNeel Natu }; 2583d17b5104SNeel Natu 2584d17b5104SNeel Natu KASSERT(seg >= 0 && seg < nitems(seg_names), 2585d17b5104SNeel Natu ("%s: invalid segment encoding %d", __func__, seg)); 2586d17b5104SNeel Natu return (seg_names[seg]); 2587d17b5104SNeel Natu } 2588cf1d80d8SPeter Grehan 2589d665d229SNeel Natu void 2590d665d229SNeel Natu vm_copy_teardown(struct vm *vm, int vcpuid, struct vm_copyinfo *copyinfo, 2591d665d229SNeel Natu int num_copyinfo) 2592d665d229SNeel Natu { 2593d665d229SNeel Natu int idx; 2594d665d229SNeel Natu 2595d665d229SNeel Natu for (idx = 0; idx < num_copyinfo; idx++) { 2596d665d229SNeel Natu if (copyinfo[idx].cookie != NULL) 2597d665d229SNeel Natu vm_gpa_release(copyinfo[idx].cookie); 2598d665d229SNeel Natu } 2599d665d229SNeel Natu bzero(copyinfo, num_copyinfo * sizeof(struct vm_copyinfo)); 2600d665d229SNeel Natu } 2601d665d229SNeel Natu 2602d665d229SNeel Natu int 2603d665d229SNeel Natu vm_copy_setup(struct vm *vm, int vcpuid, struct vm_guest_paging *paging, 2604d665d229SNeel Natu uint64_t gla, size_t len, int prot, struct vm_copyinfo *copyinfo, 26059c4d5478SNeel Natu int num_copyinfo, int *fault) 2606d665d229SNeel Natu { 2607d665d229SNeel Natu int error, idx, nused; 2608d665d229SNeel Natu size_t n, off, remaining; 2609d665d229SNeel Natu void *hva, *cookie; 2610d665d229SNeel Natu uint64_t gpa; 2611d665d229SNeel Natu 2612d665d229SNeel Natu bzero(copyinfo, sizeof(struct vm_copyinfo) * num_copyinfo); 2613d665d229SNeel Natu 2614d665d229SNeel Natu nused = 0; 2615d665d229SNeel Natu remaining = len; 2616d665d229SNeel Natu while (remaining > 0) { 2617d665d229SNeel Natu KASSERT(nused < num_copyinfo, ("insufficient vm_copyinfo")); 26189c4d5478SNeel Natu error = vm_gla2gpa(vm, vcpuid, paging, gla, prot, &gpa, fault); 26199c4d5478SNeel Natu if (error || *fault) 2620d665d229SNeel Natu return (error); 2621d665d229SNeel Natu off = gpa & PAGE_MASK; 2622d665d229SNeel Natu n = min(remaining, PAGE_SIZE - off); 2623d665d229SNeel Natu copyinfo[nused].gpa = gpa; 2624d665d229SNeel Natu copyinfo[nused].len = n; 2625d665d229SNeel Natu remaining -= n; 2626d665d229SNeel Natu gla += n; 2627d665d229SNeel Natu nused++; 2628d665d229SNeel Natu } 2629d665d229SNeel Natu 2630d665d229SNeel Natu for (idx = 0; idx < nused; idx++) { 26319b1aa8d6SNeel Natu hva = vm_gpa_hold(vm, vcpuid, copyinfo[idx].gpa, 26329b1aa8d6SNeel Natu copyinfo[idx].len, prot, &cookie); 2633d665d229SNeel Natu if (hva == NULL) 2634d665d229SNeel Natu break; 2635d665d229SNeel Natu copyinfo[idx].hva = hva; 2636d665d229SNeel Natu copyinfo[idx].cookie = cookie; 2637d665d229SNeel Natu } 2638d665d229SNeel Natu 2639d665d229SNeel Natu if (idx != nused) { 2640d665d229SNeel Natu vm_copy_teardown(vm, vcpuid, copyinfo, num_copyinfo); 26419c4d5478SNeel Natu return (EFAULT); 2642d665d229SNeel Natu } else { 26439c4d5478SNeel Natu *fault = 0; 2644d665d229SNeel Natu return (0); 2645d665d229SNeel Natu } 2646d665d229SNeel Natu } 2647d665d229SNeel Natu 2648d665d229SNeel Natu void 2649d665d229SNeel Natu vm_copyin(struct vm *vm, int vcpuid, struct vm_copyinfo *copyinfo, void *kaddr, 2650d665d229SNeel Natu size_t len) 2651d665d229SNeel Natu { 2652d665d229SNeel Natu char *dst; 2653d665d229SNeel Natu int idx; 2654d665d229SNeel Natu 2655d665d229SNeel Natu dst = kaddr; 2656d665d229SNeel Natu idx = 0; 2657d665d229SNeel Natu while (len > 0) { 2658d665d229SNeel Natu bcopy(copyinfo[idx].hva, dst, copyinfo[idx].len); 2659d665d229SNeel Natu len -= copyinfo[idx].len; 2660d665d229SNeel Natu dst += copyinfo[idx].len; 2661d665d229SNeel Natu idx++; 2662d665d229SNeel Natu } 2663d665d229SNeel Natu } 2664d665d229SNeel Natu 2665d665d229SNeel Natu void 2666d665d229SNeel Natu vm_copyout(struct vm *vm, int vcpuid, const void *kaddr, 2667d665d229SNeel Natu struct vm_copyinfo *copyinfo, size_t len) 2668d665d229SNeel Natu { 2669d665d229SNeel Natu const char *src; 2670d665d229SNeel Natu int idx; 2671d665d229SNeel Natu 2672d665d229SNeel Natu src = kaddr; 2673d665d229SNeel Natu idx = 0; 2674d665d229SNeel Natu while (len > 0) { 2675d665d229SNeel Natu bcopy(src, copyinfo[idx].hva, copyinfo[idx].len); 2676d665d229SNeel Natu len -= copyinfo[idx].len; 2677d665d229SNeel Natu src += copyinfo[idx].len; 2678d665d229SNeel Natu idx++; 2679d665d229SNeel Natu } 2680d665d229SNeel Natu } 2681cf1d80d8SPeter Grehan 2682cf1d80d8SPeter Grehan /* 2683cf1d80d8SPeter Grehan * Return the amount of in-use and wired memory for the VM. Since 2684cf1d80d8SPeter Grehan * these are global stats, only return the values with for vCPU 0 2685cf1d80d8SPeter Grehan */ 2686cf1d80d8SPeter Grehan VMM_STAT_DECLARE(VMM_MEM_RESIDENT); 2687cf1d80d8SPeter Grehan VMM_STAT_DECLARE(VMM_MEM_WIRED); 2688cf1d80d8SPeter Grehan 2689cf1d80d8SPeter Grehan static void 2690cf1d80d8SPeter Grehan vm_get_rescnt(struct vm *vm, int vcpu, struct vmm_stat_type *stat) 2691cf1d80d8SPeter Grehan { 2692cf1d80d8SPeter Grehan 2693cf1d80d8SPeter Grehan if (vcpu == 0) { 2694cf1d80d8SPeter Grehan vmm_stat_set(vm, vcpu, VMM_MEM_RESIDENT, 2695cf1d80d8SPeter Grehan PAGE_SIZE * vmspace_resident_count(vm->vmspace)); 2696cf1d80d8SPeter Grehan } 2697cf1d80d8SPeter Grehan } 2698cf1d80d8SPeter Grehan 2699cf1d80d8SPeter Grehan static void 2700cf1d80d8SPeter Grehan vm_get_wiredcnt(struct vm *vm, int vcpu, struct vmm_stat_type *stat) 2701cf1d80d8SPeter Grehan { 2702cf1d80d8SPeter Grehan 2703cf1d80d8SPeter Grehan if (vcpu == 0) { 2704cf1d80d8SPeter Grehan vmm_stat_set(vm, vcpu, VMM_MEM_WIRED, 2705cf1d80d8SPeter Grehan PAGE_SIZE * pmap_wired_count(vmspace_pmap(vm->vmspace))); 2706cf1d80d8SPeter Grehan } 2707cf1d80d8SPeter Grehan } 2708cf1d80d8SPeter Grehan 2709cf1d80d8SPeter Grehan VMM_STAT_FUNC(VMM_MEM_RESIDENT, "Resident memory", vm_get_rescnt); 2710cf1d80d8SPeter Grehan VMM_STAT_FUNC(VMM_MEM_WIRED, "Wired memory", vm_get_wiredcnt); 2711