1366f6083SPeter Grehan /*- 2c49761ddSPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3c49761ddSPedro F. Giffuni * 4366f6083SPeter Grehan * Copyright (c) 2011 NetApp, Inc. 5366f6083SPeter Grehan * All rights reserved. 6366f6083SPeter Grehan * 7366f6083SPeter Grehan * Redistribution and use in source and binary forms, with or without 8366f6083SPeter Grehan * modification, are permitted provided that the following conditions 9366f6083SPeter Grehan * are met: 10366f6083SPeter Grehan * 1. Redistributions of source code must retain the above copyright 11366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer. 12366f6083SPeter Grehan * 2. Redistributions in binary form must reproduce the above copyright 13366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer in the 14366f6083SPeter Grehan * documentation and/or other materials provided with the distribution. 15366f6083SPeter Grehan * 16366f6083SPeter Grehan * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 17366f6083SPeter Grehan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18366f6083SPeter Grehan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19366f6083SPeter Grehan * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 20366f6083SPeter Grehan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21366f6083SPeter Grehan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22366f6083SPeter Grehan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23366f6083SPeter Grehan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24366f6083SPeter Grehan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25366f6083SPeter Grehan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26366f6083SPeter Grehan * SUCH DAMAGE. 27366f6083SPeter Grehan * 28366f6083SPeter Grehan * $FreeBSD$ 29366f6083SPeter Grehan */ 30366f6083SPeter Grehan 31366f6083SPeter Grehan #include <sys/cdefs.h> 32366f6083SPeter Grehan __FBSDID("$FreeBSD$"); 33366f6083SPeter Grehan 34483d953aSJohn Baldwin #include "opt_bhyve_snapshot.h" 35483d953aSJohn Baldwin 36366f6083SPeter Grehan #include <sys/param.h> 3738f1b189SPeter Grehan #include <sys/systm.h> 38366f6083SPeter Grehan #include <sys/kernel.h> 39366f6083SPeter Grehan #include <sys/module.h> 40366f6083SPeter Grehan #include <sys/sysctl.h> 41366f6083SPeter Grehan #include <sys/malloc.h> 42366f6083SPeter Grehan #include <sys/pcpu.h> 43366f6083SPeter Grehan #include <sys/lock.h> 44366f6083SPeter Grehan #include <sys/mutex.h> 45366f6083SPeter Grehan #include <sys/proc.h> 46318224bbSNeel Natu #include <sys/rwlock.h> 47366f6083SPeter Grehan #include <sys/sched.h> 48366f6083SPeter Grehan #include <sys/smp.h> 49483d953aSJohn Baldwin #include <sys/vnode.h> 50366f6083SPeter Grehan 51366f6083SPeter Grehan #include <vm/vm.h> 523c48106aSKonstantin Belousov #include <vm/vm_param.h> 533c48106aSKonstantin Belousov #include <vm/vm_extern.h> 54318224bbSNeel Natu #include <vm/vm_object.h> 55318224bbSNeel Natu #include <vm/vm_page.h> 56318224bbSNeel Natu #include <vm/pmap.h> 57318224bbSNeel Natu #include <vm/vm_map.h> 58483d953aSJohn Baldwin #include <vm/vm_pager.h> 59483d953aSJohn Baldwin #include <vm/vm_kern.h> 60483d953aSJohn Baldwin #include <vm/vnode_pager.h> 61483d953aSJohn Baldwin #include <vm/swap_pager.h> 62483d953aSJohn Baldwin #include <vm/uma.h> 63366f6083SPeter Grehan 6463e62d39SJohn Baldwin #include <machine/cpu.h> 65366f6083SPeter Grehan #include <machine/pcb.h> 6675dd3366SNeel Natu #include <machine/smp.h> 67bd50262fSKonstantin Belousov #include <machine/md_var.h> 681c052192SNeel Natu #include <x86/psl.h> 6934a6b2d6SJohn Baldwin #include <x86/apicreg.h> 7015add60dSPeter Grehan #include <x86/ifunc.h> 71366f6083SPeter Grehan 72366f6083SPeter Grehan #include <machine/vmm.h> 73565bbb86SNeel Natu #include <machine/vmm_dev.h> 74e813a873SNeel Natu #include <machine/vmm_instruction_emul.h> 75483d953aSJohn Baldwin #include <machine/vmm_snapshot.h> 76565bbb86SNeel Natu 77d17b5104SNeel Natu #include "vmm_ioport.h" 78318224bbSNeel Natu #include "vmm_ktr.h" 79b01c2033SNeel Natu #include "vmm_host.h" 80366f6083SPeter Grehan #include "vmm_mem.h" 81366f6083SPeter Grehan #include "vmm_util.h" 82762fd208STycho Nightingale #include "vatpic.h" 83e883c9bbSTycho Nightingale #include "vatpit.h" 8408e3ff32SNeel Natu #include "vhpet.h" 85565bbb86SNeel Natu #include "vioapic.h" 86366f6083SPeter Grehan #include "vlapic.h" 87160ef77aSNeel Natu #include "vpmtmr.h" 880dafa5cdSNeel Natu #include "vrtc.h" 89366f6083SPeter Grehan #include "vmm_stat.h" 90f76fc5d4SNeel Natu #include "vmm_lapic.h" 91366f6083SPeter Grehan 92366f6083SPeter Grehan #include "io/ppt.h" 93366f6083SPeter Grehan #include "io/iommu.h" 94366f6083SPeter Grehan 95366f6083SPeter Grehan struct vlapic; 96366f6083SPeter Grehan 975fcf252fSNeel Natu /* 985fcf252fSNeel Natu * Initialization: 995fcf252fSNeel Natu * (a) allocated when vcpu is created 1005fcf252fSNeel Natu * (i) initialized when vcpu is created and when it is reinitialized 1015fcf252fSNeel Natu * (o) initialized the first time the vcpu is created 1025fcf252fSNeel Natu * (x) initialized before use 1035fcf252fSNeel Natu */ 104366f6083SPeter Grehan struct vcpu { 1055fcf252fSNeel Natu struct mtx mtx; /* (o) protects 'state' and 'hostcpu' */ 1065fcf252fSNeel Natu enum vcpu_state state; /* (o) vcpu state */ 107950af9ffSJohn Baldwin int vcpuid; /* (o) */ 1085fcf252fSNeel Natu int hostcpu; /* (o) vcpu's host cpu */ 109248e6799SNeel Natu int reqidle; /* (i) request vcpu to idle */ 110950af9ffSJohn Baldwin struct vm *vm; /* (o) */ 1111aa51504SJohn Baldwin void *cookie; /* (i) cpu-specific data */ 1125fcf252fSNeel Natu struct vlapic *vlapic; /* (i) APIC device model */ 1135fcf252fSNeel Natu enum x2apic_state x2apic_state; /* (i) APIC mode */ 114091d4532SNeel Natu uint64_t exitintinfo; /* (i) events pending at VM exit */ 1155fcf252fSNeel Natu int nmi_pending; /* (i) NMI pending */ 1165fcf252fSNeel Natu int extint_pending; /* (i) INTR pending */ 1175fcf252fSNeel Natu int exception_pending; /* (i) exception pending */ 118c9c75df4SNeel Natu int exc_vector; /* (x) exception collateral */ 119c9c75df4SNeel Natu int exc_errcode_valid; 120c9c75df4SNeel Natu uint32_t exc_errcode; 1215fcf252fSNeel Natu struct savefpu *guestfpu; /* (a,i) guest fpu state */ 1225fcf252fSNeel Natu uint64_t guest_xcr0; /* (i) guest %xcr0 register */ 1235fcf252fSNeel Natu void *stats; /* (a,i) statistics */ 1245fcf252fSNeel Natu struct vm_exit exitinfo; /* (x) exit reason and collateral */ 125d087a399SNeel Natu uint64_t nextrip; /* (x) next instruction to execute */ 126483d953aSJohn Baldwin uint64_t tsc_offset; /* (o) TSC offsetting */ 127366f6083SPeter Grehan }; 128366f6083SPeter Grehan 1295fcf252fSNeel Natu #define vcpu_lock_initialized(v) mtx_initialized(&((v)->mtx)) 130f76fc5d4SNeel Natu #define vcpu_lock_init(v) mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN) 131f76fc5d4SNeel Natu #define vcpu_lock(v) mtx_lock_spin(&((v)->mtx)) 132f76fc5d4SNeel Natu #define vcpu_unlock(v) mtx_unlock_spin(&((v)->mtx)) 133318224bbSNeel Natu #define vcpu_assert_locked(v) mtx_assert(&((v)->mtx), MA_OWNED) 13475dd3366SNeel Natu 135318224bbSNeel Natu struct mem_seg { 1369b1aa8d6SNeel Natu size_t len; 1379b1aa8d6SNeel Natu bool sysmem; 1389b1aa8d6SNeel Natu struct vm_object *object; 1399b1aa8d6SNeel Natu }; 140e47fe318SCorvin Köhne #define VM_MAX_MEMSEGS 4 1419b1aa8d6SNeel Natu 1429b1aa8d6SNeel Natu struct mem_map { 143318224bbSNeel Natu vm_paddr_t gpa; 144318224bbSNeel Natu size_t len; 1459b1aa8d6SNeel Natu vm_ooffset_t segoff; 1469b1aa8d6SNeel Natu int segid; 1479b1aa8d6SNeel Natu int prot; 1489b1aa8d6SNeel Natu int flags; 149318224bbSNeel Natu }; 15000d3723fSConrad Meyer #define VM_MAX_MEMMAPS 8 151366f6083SPeter Grehan 152366f6083SPeter Grehan /* 1535fcf252fSNeel Natu * Initialization: 1545fcf252fSNeel Natu * (o) initialized the first time the VM is created 1555fcf252fSNeel Natu * (i) initialized when VM is created and when it is reinitialized 1565fcf252fSNeel Natu * (x) initialized before use 157366f6083SPeter Grehan */ 1585fcf252fSNeel Natu struct vm { 1595fcf252fSNeel Natu void *cookie; /* (i) cpu-specific data */ 1605fcf252fSNeel Natu void *iommu; /* (x) iommu-specific data */ 1615fcf252fSNeel Natu struct vhpet *vhpet; /* (i) virtual HPET */ 1625fcf252fSNeel Natu struct vioapic *vioapic; /* (i) virtual ioapic */ 1635fcf252fSNeel Natu struct vatpic *vatpic; /* (i) virtual atpic */ 1645fcf252fSNeel Natu struct vatpit *vatpit; /* (i) virtual atpit */ 165160ef77aSNeel Natu struct vpmtmr *vpmtmr; /* (i) virtual ACPI PM timer */ 1660dafa5cdSNeel Natu struct vrtc *vrtc; /* (o) virtual RTC */ 1675fcf252fSNeel Natu volatile cpuset_t active_cpus; /* (i) active vcpus */ 168fc276d92SJohn Baldwin volatile cpuset_t debug_cpus; /* (i) vcpus stopped for debug */ 1695fcf252fSNeel Natu int suspend; /* (i) stop VM execution */ 1705fcf252fSNeel Natu volatile cpuset_t suspended_cpus; /* (i) suspended vcpus */ 1715fcf252fSNeel Natu volatile cpuset_t halted_cpus; /* (x) cpus in a hard halt */ 1725fcf252fSNeel Natu cpuset_t rendezvous_req_cpus; /* (x) rendezvous requested */ 1735fcf252fSNeel Natu cpuset_t rendezvous_done_cpus; /* (x) rendezvous finished */ 1745fcf252fSNeel Natu void *rendezvous_arg; /* (x) rendezvous func/arg */ 1755b8a8cd1SNeel Natu vm_rendezvous_func_t rendezvous_func; 1765fcf252fSNeel Natu struct mtx rendezvous_mtx; /* (o) rendezvous lock */ 1779b1aa8d6SNeel Natu struct mem_map mem_maps[VM_MAX_MEMMAPS]; /* (i) guest address space */ 1789b1aa8d6SNeel Natu struct mem_seg mem_segs[VM_MAX_MEMSEGS]; /* (o) guest memory regions */ 1795fcf252fSNeel Natu struct vmspace *vmspace; /* (o) guest's address space */ 180df95cc76SKa Ho Ng char name[VM_MAX_NAMELEN+1]; /* (o) virtual machine name */ 1815fcf252fSNeel Natu struct vcpu vcpu[VM_MAXCPU]; /* (i) guest vcpus */ 18201d822d3SRodney W. Grimes /* The following describe the vm cpu topology */ 18301d822d3SRodney W. Grimes uint16_t sockets; /* (o) num of sockets */ 18401d822d3SRodney W. Grimes uint16_t cores; /* (o) num of cores/socket */ 18501d822d3SRodney W. Grimes uint16_t threads; /* (o) num of threads/core */ 18601d822d3SRodney W. Grimes uint16_t maxcpus; /* (o) max pluggable cpus */ 187366f6083SPeter Grehan }; 188366f6083SPeter Grehan 189950af9ffSJohn Baldwin #define VMM_CTR0(vcpu, format) \ 190950af9ffSJohn Baldwin VCPU_CTR0((vcpu)->vm, (vcpu)->vcpuid, format) 191950af9ffSJohn Baldwin 192950af9ffSJohn Baldwin #define VMM_CTR1(vcpu, format, p1) \ 193950af9ffSJohn Baldwin VCPU_CTR1((vcpu)->vm, (vcpu)->vcpuid, format, p1) 194950af9ffSJohn Baldwin 195950af9ffSJohn Baldwin #define VMM_CTR2(vcpu, format, p1, p2) \ 196950af9ffSJohn Baldwin VCPU_CTR2((vcpu)->vm, (vcpu)->vcpuid, format, p1, p2) 197950af9ffSJohn Baldwin 198950af9ffSJohn Baldwin #define VMM_CTR3(vcpu, format, p1, p2, p3) \ 199950af9ffSJohn Baldwin VCPU_CTR3((vcpu)->vm, (vcpu)->vcpuid, format, p1, p2, p3) 200950af9ffSJohn Baldwin 201950af9ffSJohn Baldwin #define VMM_CTR4(vcpu, format, p1, p2, p3, p4) \ 202950af9ffSJohn Baldwin VCPU_CTR4((vcpu)->vm, (vcpu)->vcpuid, format, p1, p2, p3, p4) 203950af9ffSJohn Baldwin 204d5408b1dSNeel Natu static int vmm_initialized; 205d5408b1dSNeel Natu 20615add60dSPeter Grehan static void vmmops_panic(void); 207366f6083SPeter Grehan 20815add60dSPeter Grehan static void 20915add60dSPeter Grehan vmmops_panic(void) 21015add60dSPeter Grehan { 21115add60dSPeter Grehan panic("vmm_ops func called when !vmm_is_intel() && !vmm_is_svm()"); 21215add60dSPeter Grehan } 21315add60dSPeter Grehan 21415add60dSPeter Grehan #define DEFINE_VMMOPS_IFUNC(ret_type, opname, args) \ 21515add60dSPeter Grehan DEFINE_IFUNC(static, ret_type, vmmops_##opname, args) \ 21615add60dSPeter Grehan { \ 21715add60dSPeter Grehan if (vmm_is_intel()) \ 21815add60dSPeter Grehan return (vmm_ops_intel.opname); \ 21915add60dSPeter Grehan else if (vmm_is_svm()) \ 22015add60dSPeter Grehan return (vmm_ops_amd.opname); \ 22115add60dSPeter Grehan else \ 22215add60dSPeter Grehan return ((ret_type (*)args)vmmops_panic); \ 22315add60dSPeter Grehan } 22415add60dSPeter Grehan 22515add60dSPeter Grehan DEFINE_VMMOPS_IFUNC(int, modinit, (int ipinum)) 22615add60dSPeter Grehan DEFINE_VMMOPS_IFUNC(int, modcleanup, (void)) 22715add60dSPeter Grehan DEFINE_VMMOPS_IFUNC(void, modresume, (void)) 22815add60dSPeter Grehan DEFINE_VMMOPS_IFUNC(void *, init, (struct vm *vm, struct pmap *pmap)) 229869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(int, run, (void *vcpui, register_t rip, struct pmap *pmap, 230869c8d19SJohn Baldwin struct vm_eventinfo *info)) 23115add60dSPeter Grehan DEFINE_VMMOPS_IFUNC(void, cleanup, (void *vmi)) 232950af9ffSJohn Baldwin DEFINE_VMMOPS_IFUNC(void *, vcpu_init, (void *vmi, struct vcpu *vcpu, 233950af9ffSJohn Baldwin int vcpu_id)) 234869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(void, vcpu_cleanup, (void *vcpui)) 235869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(int, getreg, (void *vcpui, int num, uint64_t *retval)) 236869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(int, setreg, (void *vcpui, int num, uint64_t val)) 237869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(int, getdesc, (void *vcpui, int num, struct seg_desc *desc)) 238869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(int, setdesc, (void *vcpui, int num, struct seg_desc *desc)) 239869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(int, getcap, (void *vcpui, int num, int *retval)) 240869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(int, setcap, (void *vcpui, int num, int val)) 24115add60dSPeter Grehan DEFINE_VMMOPS_IFUNC(struct vmspace *, vmspace_alloc, (vm_offset_t min, 24215add60dSPeter Grehan vm_offset_t max)) 24315add60dSPeter Grehan DEFINE_VMMOPS_IFUNC(void, vmspace_free, (struct vmspace *vmspace)) 244869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(struct vlapic *, vlapic_init, (void *vcpui)) 245869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(void, vlapic_cleanup, (struct vlapic *vlapic)) 246483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT 247869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(int, snapshot, (void *vmi, struct vm_snapshot_meta *meta)) 248869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(int, vcpu_snapshot, (void *vcpui, 249869c8d19SJohn Baldwin struct vm_snapshot_meta *meta)) 250869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(int, restore_tsc, (void *vcpui, uint64_t now)) 251483d953aSJohn Baldwin #endif 252366f6083SPeter Grehan 253014a52f3SNeel Natu #define fpu_start_emulating() load_cr0(rcr0() | CR0_TS) 254014a52f3SNeel Natu #define fpu_stop_emulating() clts() 255366f6083SPeter Grehan 2566ac73777STycho Nightingale SDT_PROVIDER_DEFINE(vmm); 2576ac73777STycho Nightingale 258366f6083SPeter Grehan static MALLOC_DEFINE(M_VM, "vm", "vm"); 259366f6083SPeter Grehan 260366f6083SPeter Grehan /* statistics */ 26161592433SNeel Natu static VMM_STAT(VCPU_TOTAL_RUNTIME, "vcpu total runtime"); 262366f6083SPeter Grehan 263b40598c5SPawel Biernacki SYSCTL_NODE(_hw, OID_AUTO, vmm, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 264b40598c5SPawel Biernacki NULL); 265add611fdSNeel Natu 266055fc2cbSNeel Natu /* 267055fc2cbSNeel Natu * Halt the guest if all vcpus are executing a HLT instruction with 268055fc2cbSNeel Natu * interrupts disabled. 269055fc2cbSNeel Natu */ 270055fc2cbSNeel Natu static int halt_detection_enabled = 1; 271055fc2cbSNeel Natu SYSCTL_INT(_hw_vmm, OID_AUTO, halt_detection, CTLFLAG_RDTUN, 272055fc2cbSNeel Natu &halt_detection_enabled, 0, 273055fc2cbSNeel Natu "Halt VM if all vcpus execute HLT with interrupts disabled"); 274055fc2cbSNeel Natu 275978f3da1SAndriy Gapon static int vmm_ipinum; 276add611fdSNeel Natu SYSCTL_INT(_hw_vmm, OID_AUTO, ipinum, CTLFLAG_RD, &vmm_ipinum, 0, 277add611fdSNeel Natu "IPI vector used for vcpu notifications"); 278add611fdSNeel Natu 279b0538143SNeel Natu static int trace_guest_exceptions; 280b0538143SNeel Natu SYSCTL_INT(_hw_vmm, OID_AUTO, trace_guest_exceptions, CTLFLAG_RDTUN, 281b0538143SNeel Natu &trace_guest_exceptions, 0, 282b0538143SNeel Natu "Trap into hypervisor on all guest exceptions and reflect them back"); 283b0538143SNeel Natu 2843ba952e1SCorvin Köhne static int trap_wbinvd; 2853ba952e1SCorvin Köhne SYSCTL_INT(_hw_vmm, OID_AUTO, trap_wbinvd, CTLFLAG_RDTUN, &trap_wbinvd, 0, 2863ba952e1SCorvin Köhne "WBINVD triggers a VM-exit"); 2873ba952e1SCorvin Köhne 2889b1aa8d6SNeel Natu static void vm_free_memmap(struct vm *vm, int ident); 2899b1aa8d6SNeel Natu static bool sysmem_mapping(struct vm *vm, struct mem_map *mm); 290248e6799SNeel Natu static void vcpu_notify_event_locked(struct vcpu *vcpu, bool lapic_intr); 291248e6799SNeel Natu 292248e6799SNeel Natu #ifdef KTR 293248e6799SNeel Natu static const char * 294248e6799SNeel Natu vcpu_state2str(enum vcpu_state state) 295248e6799SNeel Natu { 296248e6799SNeel Natu 297248e6799SNeel Natu switch (state) { 298248e6799SNeel Natu case VCPU_IDLE: 299248e6799SNeel Natu return ("idle"); 300248e6799SNeel Natu case VCPU_FROZEN: 301248e6799SNeel Natu return ("frozen"); 302248e6799SNeel Natu case VCPU_RUNNING: 303248e6799SNeel Natu return ("running"); 304248e6799SNeel Natu case VCPU_SLEEPING: 305248e6799SNeel Natu return ("sleeping"); 306248e6799SNeel Natu default: 307248e6799SNeel Natu return ("unknown"); 308248e6799SNeel Natu } 309248e6799SNeel Natu } 310248e6799SNeel Natu #endif 311248e6799SNeel Natu 3121aa51504SJohn Baldwin static __inline void * 3131aa51504SJohn Baldwin vcpu_cookie(struct vm *vm, int i) 3141aa51504SJohn Baldwin { 3151aa51504SJohn Baldwin return (vm->vcpu[i].cookie); 3161aa51504SJohn Baldwin } 3171aa51504SJohn Baldwin 318366f6083SPeter Grehan static void 3195fcf252fSNeel Natu vcpu_cleanup(struct vm *vm, int i, bool destroy) 320366f6083SPeter Grehan { 321de5ea6b6SNeel Natu struct vcpu *vcpu = &vm->vcpu[i]; 322de5ea6b6SNeel Natu 323869c8d19SJohn Baldwin vmmops_vlapic_cleanup(vcpu->vlapic); 324869c8d19SJohn Baldwin vmmops_vcpu_cleanup(vcpu->cookie); 3251aa51504SJohn Baldwin vcpu->cookie = NULL; 3265fcf252fSNeel Natu if (destroy) { 327366f6083SPeter Grehan vmm_stat_free(vcpu->stats); 32838f1b189SPeter Grehan fpu_save_area_free(vcpu->guestfpu); 329366f6083SPeter Grehan } 3305fcf252fSNeel Natu } 331366f6083SPeter Grehan 332366f6083SPeter Grehan static void 3335fcf252fSNeel Natu vcpu_init(struct vm *vm, int vcpu_id, bool create) 334366f6083SPeter Grehan { 335366f6083SPeter Grehan struct vcpu *vcpu; 336366f6083SPeter Grehan 337a488c9c9SRodney W. Grimes KASSERT(vcpu_id >= 0 && vcpu_id < vm->maxcpus, 3385fcf252fSNeel Natu ("vcpu_init: invalid vcpu %d", vcpu_id)); 3395fcf252fSNeel Natu 340366f6083SPeter Grehan vcpu = &vm->vcpu[vcpu_id]; 341366f6083SPeter Grehan 3425fcf252fSNeel Natu if (create) { 3435fcf252fSNeel Natu KASSERT(!vcpu_lock_initialized(vcpu), ("vcpu %d already " 3445fcf252fSNeel Natu "initialized", vcpu_id)); 34575dd3366SNeel Natu vcpu_lock_init(vcpu); 3465fcf252fSNeel Natu vcpu->state = VCPU_IDLE; 34775dd3366SNeel Natu vcpu->hostcpu = NOCPU; 348950af9ffSJohn Baldwin vcpu->vcpuid = vcpu_id; 349950af9ffSJohn Baldwin vcpu->vm = vm; 3505fcf252fSNeel Natu vcpu->guestfpu = fpu_save_area_alloc(); 3515fcf252fSNeel Natu vcpu->stats = vmm_stat_alloc(); 352483d953aSJohn Baldwin vcpu->tsc_offset = 0; 3535fcf252fSNeel Natu } 3545fcf252fSNeel Natu 355950af9ffSJohn Baldwin vcpu->cookie = vmmops_vcpu_init(vm->cookie, vcpu, vcpu_id); 356869c8d19SJohn Baldwin vcpu->vlapic = vmmops_vlapic_init(vcpu->cookie); 35752e5c8a2SNeel Natu vm_set_x2apic_state(vm, vcpu_id, X2APIC_DISABLED); 358248e6799SNeel Natu vcpu->reqidle = 0; 359091d4532SNeel Natu vcpu->exitintinfo = 0; 3605fcf252fSNeel Natu vcpu->nmi_pending = 0; 3615fcf252fSNeel Natu vcpu->extint_pending = 0; 3625fcf252fSNeel Natu vcpu->exception_pending = 0; 363abb023fbSJohn Baldwin vcpu->guest_xcr0 = XFEATURE_ENABLED_X87; 36438f1b189SPeter Grehan fpu_save_area_reset(vcpu->guestfpu); 3655fcf252fSNeel Natu vmm_stat_init(vcpu->stats); 366366f6083SPeter Grehan } 367366f6083SPeter Grehan 368b0538143SNeel Natu int 369b0538143SNeel Natu vcpu_trace_exceptions(struct vm *vm, int vcpuid) 370b0538143SNeel Natu { 371b0538143SNeel Natu 372b0538143SNeel Natu return (trace_guest_exceptions); 373b0538143SNeel Natu } 374b0538143SNeel Natu 3753ba952e1SCorvin Köhne int 3763ba952e1SCorvin Köhne vcpu_trap_wbinvd(struct vm *vm, int vcpuid) 3773ba952e1SCorvin Köhne { 3783ba952e1SCorvin Köhne return (trap_wbinvd); 3793ba952e1SCorvin Köhne } 3803ba952e1SCorvin Köhne 38198ed632cSNeel Natu struct vm_exit * 38298ed632cSNeel Natu vm_exitinfo(struct vm *vm, int cpuid) 38398ed632cSNeel Natu { 38498ed632cSNeel Natu struct vcpu *vcpu; 38598ed632cSNeel Natu 386a488c9c9SRodney W. Grimes if (cpuid < 0 || cpuid >= vm->maxcpus) 38798ed632cSNeel Natu panic("vm_exitinfo: invalid cpuid %d", cpuid); 38898ed632cSNeel Natu 38998ed632cSNeel Natu vcpu = &vm->vcpu[cpuid]; 39098ed632cSNeel Natu 39198ed632cSNeel Natu return (&vcpu->exitinfo); 39298ed632cSNeel Natu } 39398ed632cSNeel Natu 394366f6083SPeter Grehan static int 395366f6083SPeter Grehan vmm_init(void) 396366f6083SPeter Grehan { 397366f6083SPeter Grehan int error; 398366f6083SPeter Grehan 39915add60dSPeter Grehan if (!vmm_is_hw_supported()) 40015add60dSPeter Grehan return (ENXIO); 40115add60dSPeter Grehan 402b01c2033SNeel Natu vmm_host_state_init(); 403add611fdSNeel Natu 404bd50262fSKonstantin Belousov vmm_ipinum = lapic_ipi_alloc(pti ? &IDTVEC(justreturn1_pti) : 405bd50262fSKonstantin Belousov &IDTVEC(justreturn)); 40618a2b08eSNeel Natu if (vmm_ipinum < 0) 407add611fdSNeel Natu vmm_ipinum = IPI_AST; 408366f6083SPeter Grehan 409366f6083SPeter Grehan error = vmm_mem_init(); 410366f6083SPeter Grehan if (error) 411366f6083SPeter Grehan return (error); 412366f6083SPeter Grehan 41315add60dSPeter Grehan vmm_resume_p = vmmops_modresume; 414366f6083SPeter Grehan 41515add60dSPeter Grehan return (vmmops_modinit(vmm_ipinum)); 416366f6083SPeter Grehan } 417366f6083SPeter Grehan 418366f6083SPeter Grehan static int 419366f6083SPeter Grehan vmm_handler(module_t mod, int what, void *arg) 420366f6083SPeter Grehan { 421366f6083SPeter Grehan int error; 422366f6083SPeter Grehan 423366f6083SPeter Grehan switch (what) { 424366f6083SPeter Grehan case MOD_LOAD: 42515add60dSPeter Grehan if (vmm_is_hw_supported()) { 426366f6083SPeter Grehan vmmdev_init(); 427366f6083SPeter Grehan error = vmm_init(); 428d5408b1dSNeel Natu if (error == 0) 429d5408b1dSNeel Natu vmm_initialized = 1; 43015add60dSPeter Grehan } else { 43115add60dSPeter Grehan error = ENXIO; 43215add60dSPeter Grehan } 433366f6083SPeter Grehan break; 434366f6083SPeter Grehan case MOD_UNLOAD: 43515add60dSPeter Grehan if (vmm_is_hw_supported()) { 436cdc5b9e7SNeel Natu error = vmmdev_cleanup(); 437cdc5b9e7SNeel Natu if (error == 0) { 43863e62d39SJohn Baldwin vmm_resume_p = NULL; 439366f6083SPeter Grehan iommu_cleanup(); 440add611fdSNeel Natu if (vmm_ipinum != IPI_AST) 44118a2b08eSNeel Natu lapic_ipi_free(vmm_ipinum); 44215add60dSPeter Grehan error = vmmops_modcleanup(); 44381ef6611SPeter Grehan /* 44481ef6611SPeter Grehan * Something bad happened - prevent new 44581ef6611SPeter Grehan * VMs from being created 44681ef6611SPeter Grehan */ 44781ef6611SPeter Grehan if (error) 448d5408b1dSNeel Natu vmm_initialized = 0; 44981ef6611SPeter Grehan } 45015add60dSPeter Grehan } else { 45115add60dSPeter Grehan error = 0; 45215add60dSPeter Grehan } 453366f6083SPeter Grehan break; 454366f6083SPeter Grehan default: 455366f6083SPeter Grehan error = 0; 456366f6083SPeter Grehan break; 457366f6083SPeter Grehan } 458366f6083SPeter Grehan return (error); 459366f6083SPeter Grehan } 460366f6083SPeter Grehan 461366f6083SPeter Grehan static moduledata_t vmm_kmod = { 462366f6083SPeter Grehan "vmm", 463366f6083SPeter Grehan vmm_handler, 464366f6083SPeter Grehan NULL 465366f6083SPeter Grehan }; 466366f6083SPeter Grehan 467366f6083SPeter Grehan /* 468e3f0800bSNeel Natu * vmm initialization has the following dependencies: 469e3f0800bSNeel Natu * 470e3f0800bSNeel Natu * - VT-x initialization requires smp_rendezvous() and therefore must happen 471e3f0800bSNeel Natu * after SMP is fully functional (after SI_SUB_SMP). 472366f6083SPeter Grehan */ 473e3f0800bSNeel Natu DECLARE_MODULE(vmm, vmm_kmod, SI_SUB_SMP + 1, SI_ORDER_ANY); 474366f6083SPeter Grehan MODULE_VERSION(vmm, 1); 475366f6083SPeter Grehan 4765fcf252fSNeel Natu static void 4775fcf252fSNeel Natu vm_init(struct vm *vm, bool create) 4785fcf252fSNeel Natu { 4795fcf252fSNeel Natu int i; 4805fcf252fSNeel Natu 48115add60dSPeter Grehan vm->cookie = vmmops_init(vm, vmspace_pmap(vm->vmspace)); 4825fcf252fSNeel Natu vm->iommu = NULL; 4835fcf252fSNeel Natu vm->vioapic = vioapic_init(vm); 4845fcf252fSNeel Natu vm->vhpet = vhpet_init(vm); 4855fcf252fSNeel Natu vm->vatpic = vatpic_init(vm); 4865fcf252fSNeel Natu vm->vatpit = vatpit_init(vm); 487160ef77aSNeel Natu vm->vpmtmr = vpmtmr_init(vm); 4880dafa5cdSNeel Natu if (create) 4890dafa5cdSNeel Natu vm->vrtc = vrtc_init(vm); 4905fcf252fSNeel Natu 4915fcf252fSNeel Natu CPU_ZERO(&vm->active_cpus); 492fc276d92SJohn Baldwin CPU_ZERO(&vm->debug_cpus); 4935fcf252fSNeel Natu 4945fcf252fSNeel Natu vm->suspend = 0; 4955fcf252fSNeel Natu CPU_ZERO(&vm->suspended_cpus); 4965fcf252fSNeel Natu 497a488c9c9SRodney W. Grimes for (i = 0; i < vm->maxcpus; i++) 4985fcf252fSNeel Natu vcpu_init(vm, i, create); 4995fcf252fSNeel Natu } 5005fcf252fSNeel Natu 50101d822d3SRodney W. Grimes /* 50201d822d3SRodney W. Grimes * The default CPU topology is a single thread per package. 50301d822d3SRodney W. Grimes */ 50401d822d3SRodney W. Grimes u_int cores_per_package = 1; 50501d822d3SRodney W. Grimes u_int threads_per_core = 1; 50601d822d3SRodney W. Grimes 507d5408b1dSNeel Natu int 508d5408b1dSNeel Natu vm_create(const char *name, struct vm **retvm) 509366f6083SPeter Grehan { 510366f6083SPeter Grehan struct vm *vm; 511318224bbSNeel Natu struct vmspace *vmspace; 512366f6083SPeter Grehan 513d5408b1dSNeel Natu /* 514d5408b1dSNeel Natu * If vmm.ko could not be successfully initialized then don't attempt 515d5408b1dSNeel Natu * to create the virtual machine. 516d5408b1dSNeel Natu */ 517d5408b1dSNeel Natu if (!vmm_initialized) 518d5408b1dSNeel Natu return (ENXIO); 519d5408b1dSNeel Natu 520df95cc76SKa Ho Ng if (name == NULL || strnlen(name, VM_MAX_NAMELEN + 1) == 521df95cc76SKa Ho Ng VM_MAX_NAMELEN + 1) 522d5408b1dSNeel Natu return (EINVAL); 523366f6083SPeter Grehan 5243c48106aSKonstantin Belousov vmspace = vmmops_vmspace_alloc(0, VM_MAXUSER_ADDRESS_LA48); 525318224bbSNeel Natu if (vmspace == NULL) 526318224bbSNeel Natu return (ENOMEM); 527318224bbSNeel Natu 528366f6083SPeter Grehan vm = malloc(sizeof(struct vm), M_VM, M_WAITOK | M_ZERO); 529366f6083SPeter Grehan strcpy(vm->name, name); 53088c4b8d1SNeel Natu vm->vmspace = vmspace; 5315b8a8cd1SNeel Natu mtx_init(&vm->rendezvous_mtx, "vm rendezvous lock", 0, MTX_DEF); 532366f6083SPeter Grehan 53301d822d3SRodney W. Grimes vm->sockets = 1; 53401d822d3SRodney W. Grimes vm->cores = cores_per_package; /* XXX backwards compatibility */ 53501d822d3SRodney W. Grimes vm->threads = threads_per_core; /* XXX backwards compatibility */ 536a488c9c9SRodney W. Grimes vm->maxcpus = VM_MAXCPU; /* XXX temp to keep code working */ 53701d822d3SRodney W. Grimes 5385fcf252fSNeel Natu vm_init(vm, true); 539366f6083SPeter Grehan 540d5408b1dSNeel Natu *retvm = vm; 541d5408b1dSNeel Natu return (0); 542366f6083SPeter Grehan } 543366f6083SPeter Grehan 54401d822d3SRodney W. Grimes void 54501d822d3SRodney W. Grimes vm_get_topology(struct vm *vm, uint16_t *sockets, uint16_t *cores, 54601d822d3SRodney W. Grimes uint16_t *threads, uint16_t *maxcpus) 54701d822d3SRodney W. Grimes { 54801d822d3SRodney W. Grimes *sockets = vm->sockets; 54901d822d3SRodney W. Grimes *cores = vm->cores; 55001d822d3SRodney W. Grimes *threads = vm->threads; 55101d822d3SRodney W. Grimes *maxcpus = vm->maxcpus; 55201d822d3SRodney W. Grimes } 55301d822d3SRodney W. Grimes 554a488c9c9SRodney W. Grimes uint16_t 555a488c9c9SRodney W. Grimes vm_get_maxcpus(struct vm *vm) 556a488c9c9SRodney W. Grimes { 557a488c9c9SRodney W. Grimes return (vm->maxcpus); 558a488c9c9SRodney W. Grimes } 559a488c9c9SRodney W. Grimes 56001d822d3SRodney W. Grimes int 56101d822d3SRodney W. Grimes vm_set_topology(struct vm *vm, uint16_t sockets, uint16_t cores, 56201d822d3SRodney W. Grimes uint16_t threads, uint16_t maxcpus) 56301d822d3SRodney W. Grimes { 56401d822d3SRodney W. Grimes if (maxcpus != 0) 56501d822d3SRodney W. Grimes return (EINVAL); /* XXX remove when supported */ 566a488c9c9SRodney W. Grimes if ((sockets * cores * threads) > vm->maxcpus) 56701d822d3SRodney W. Grimes return (EINVAL); 56801d822d3SRodney W. Grimes /* XXX need to check sockets * cores * threads == vCPU, how? */ 56901d822d3SRodney W. Grimes vm->sockets = sockets; 57001d822d3SRodney W. Grimes vm->cores = cores; 57101d822d3SRodney W. Grimes vm->threads = threads; 572a488c9c9SRodney W. Grimes vm->maxcpus = VM_MAXCPU; /* XXX temp to keep code working */ 57301d822d3SRodney W. Grimes return(0); 57401d822d3SRodney W. Grimes } 57501d822d3SRodney W. Grimes 576f7d51510SNeel Natu static void 5775fcf252fSNeel Natu vm_cleanup(struct vm *vm, bool destroy) 578366f6083SPeter Grehan { 5799b1aa8d6SNeel Natu struct mem_map *mm; 580366f6083SPeter Grehan int i; 581366f6083SPeter Grehan 582366f6083SPeter Grehan ppt_unassign_all(vm); 583366f6083SPeter Grehan 584318224bbSNeel Natu if (vm->iommu != NULL) 585318224bbSNeel Natu iommu_destroy_domain(vm->iommu); 586318224bbSNeel Natu 5870dafa5cdSNeel Natu if (destroy) 5880dafa5cdSNeel Natu vrtc_cleanup(vm->vrtc); 5890dafa5cdSNeel Natu else 5900dafa5cdSNeel Natu vrtc_reset(vm->vrtc); 591160ef77aSNeel Natu vpmtmr_cleanup(vm->vpmtmr); 592e883c9bbSTycho Nightingale vatpit_cleanup(vm->vatpit); 59308e3ff32SNeel Natu vhpet_cleanup(vm->vhpet); 594762fd208STycho Nightingale vatpic_cleanup(vm->vatpic); 59508e3ff32SNeel Natu vioapic_cleanup(vm->vioapic); 59608e3ff32SNeel Natu 597a488c9c9SRodney W. Grimes for (i = 0; i < vm->maxcpus; i++) 5985fcf252fSNeel Natu vcpu_cleanup(vm, i, destroy); 5995fcf252fSNeel Natu 60015add60dSPeter Grehan vmmops_cleanup(vm->cookie); 6015fcf252fSNeel Natu 6029b1aa8d6SNeel Natu /* 6039b1aa8d6SNeel Natu * System memory is removed from the guest address space only when 6049b1aa8d6SNeel Natu * the VM is destroyed. This is because the mapping remains the same 6059b1aa8d6SNeel Natu * across VM reset. 6069b1aa8d6SNeel Natu * 6079b1aa8d6SNeel Natu * Device memory can be relocated by the guest (e.g. using PCI BARs) 6089b1aa8d6SNeel Natu * so those mappings are removed on a VM reset. 6099b1aa8d6SNeel Natu */ 6109b1aa8d6SNeel Natu for (i = 0; i < VM_MAX_MEMMAPS; i++) { 6119b1aa8d6SNeel Natu mm = &vm->mem_maps[i]; 6129b1aa8d6SNeel Natu if (destroy || !sysmem_mapping(vm, mm)) 6139b1aa8d6SNeel Natu vm_free_memmap(vm, i); 6149b1aa8d6SNeel Natu } 615f7d51510SNeel Natu 6169b1aa8d6SNeel Natu if (destroy) { 6179b1aa8d6SNeel Natu for (i = 0; i < VM_MAX_MEMSEGS; i++) 6189b1aa8d6SNeel Natu vm_free_memseg(vm, i); 619366f6083SPeter Grehan 62015add60dSPeter Grehan vmmops_vmspace_free(vm->vmspace); 6215fcf252fSNeel Natu vm->vmspace = NULL; 6225fcf252fSNeel Natu } 6235fcf252fSNeel Natu } 624366f6083SPeter Grehan 6255fcf252fSNeel Natu void 6265fcf252fSNeel Natu vm_destroy(struct vm *vm) 6275fcf252fSNeel Natu { 6285fcf252fSNeel Natu vm_cleanup(vm, true); 629366f6083SPeter Grehan free(vm, M_VM); 630366f6083SPeter Grehan } 631366f6083SPeter Grehan 6325fcf252fSNeel Natu int 6335fcf252fSNeel Natu vm_reinit(struct vm *vm) 6345fcf252fSNeel Natu { 6355fcf252fSNeel Natu int error; 6365fcf252fSNeel Natu 6375fcf252fSNeel Natu /* 6385fcf252fSNeel Natu * A virtual machine can be reset only if all vcpus are suspended. 6395fcf252fSNeel Natu */ 6405fcf252fSNeel Natu if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) { 6415fcf252fSNeel Natu vm_cleanup(vm, false); 6425fcf252fSNeel Natu vm_init(vm, false); 6435fcf252fSNeel Natu error = 0; 6445fcf252fSNeel Natu } else { 6455fcf252fSNeel Natu error = EBUSY; 6465fcf252fSNeel Natu } 6475fcf252fSNeel Natu 6485fcf252fSNeel Natu return (error); 6495fcf252fSNeel Natu } 6505fcf252fSNeel Natu 651366f6083SPeter Grehan const char * 652366f6083SPeter Grehan vm_name(struct vm *vm) 653366f6083SPeter Grehan { 654366f6083SPeter Grehan return (vm->name); 655366f6083SPeter Grehan } 656366f6083SPeter Grehan 657366f6083SPeter Grehan int 658366f6083SPeter Grehan vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa) 659366f6083SPeter Grehan { 660318224bbSNeel Natu vm_object_t obj; 661366f6083SPeter Grehan 662318224bbSNeel Natu if ((obj = vmm_mmio_alloc(vm->vmspace, gpa, len, hpa)) == NULL) 663318224bbSNeel Natu return (ENOMEM); 664318224bbSNeel Natu else 665318224bbSNeel Natu return (0); 666366f6083SPeter Grehan } 667366f6083SPeter Grehan 668366f6083SPeter Grehan int 669366f6083SPeter Grehan vm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len) 670366f6083SPeter Grehan { 671366f6083SPeter Grehan 672318224bbSNeel Natu vmm_mmio_free(vm->vmspace, gpa, len); 673318224bbSNeel Natu return (0); 674366f6083SPeter Grehan } 675366f6083SPeter Grehan 6769b1aa8d6SNeel Natu /* 6779b1aa8d6SNeel Natu * Return 'true' if 'gpa' is allocated in the guest address space. 6789b1aa8d6SNeel Natu * 6799b1aa8d6SNeel Natu * This function is called in the context of a running vcpu which acts as 6809b1aa8d6SNeel Natu * an implicit lock on 'vm->mem_maps[]'. 6819b1aa8d6SNeel Natu */ 6829b1aa8d6SNeel Natu bool 6839b1aa8d6SNeel Natu vm_mem_allocated(struct vm *vm, int vcpuid, vm_paddr_t gpa) 684366f6083SPeter Grehan { 6859b1aa8d6SNeel Natu struct mem_map *mm; 686341f19c9SNeel Natu int i; 687341f19c9SNeel Natu 6889b1aa8d6SNeel Natu #ifdef INVARIANTS 6899b1aa8d6SNeel Natu int hostcpu, state; 6909b1aa8d6SNeel Natu state = vcpu_get_state(vm, vcpuid, &hostcpu); 6919b1aa8d6SNeel Natu KASSERT(state == VCPU_RUNNING && hostcpu == curcpu, 6929b1aa8d6SNeel Natu ("%s: invalid vcpu state %d/%d", __func__, state, hostcpu)); 6939b1aa8d6SNeel Natu #endif 6949b1aa8d6SNeel Natu 6959b1aa8d6SNeel Natu for (i = 0; i < VM_MAX_MEMMAPS; i++) { 6969b1aa8d6SNeel Natu mm = &vm->mem_maps[i]; 6979b1aa8d6SNeel Natu if (mm->len != 0 && gpa >= mm->gpa && gpa < mm->gpa + mm->len) 6989b1aa8d6SNeel Natu return (true); /* 'gpa' is sysmem or devmem */ 699341f19c9SNeel Natu } 700341f19c9SNeel Natu 701318224bbSNeel Natu if (ppt_is_mmio(vm, gpa)) 7029b1aa8d6SNeel Natu return (true); /* 'gpa' is pci passthru mmio */ 703318224bbSNeel Natu 7049b1aa8d6SNeel Natu return (false); 705341f19c9SNeel Natu } 706341f19c9SNeel Natu 707341f19c9SNeel Natu int 7089b1aa8d6SNeel Natu vm_alloc_memseg(struct vm *vm, int ident, size_t len, bool sysmem) 709341f19c9SNeel Natu { 710318224bbSNeel Natu struct mem_seg *seg; 7119b1aa8d6SNeel Natu vm_object_t obj; 712366f6083SPeter Grehan 7139b1aa8d6SNeel Natu if (ident < 0 || ident >= VM_MAX_MEMSEGS) 714341f19c9SNeel Natu return (EINVAL); 715341f19c9SNeel Natu 7169b1aa8d6SNeel Natu if (len == 0 || (len & PAGE_MASK)) 7179b1aa8d6SNeel Natu return (EINVAL); 718341f19c9SNeel Natu 7199b1aa8d6SNeel Natu seg = &vm->mem_segs[ident]; 7209b1aa8d6SNeel Natu if (seg->object != NULL) { 7219b1aa8d6SNeel Natu if (seg->len == len && seg->sysmem == sysmem) 7229b1aa8d6SNeel Natu return (EEXIST); 7239b1aa8d6SNeel Natu else 7249b1aa8d6SNeel Natu return (EINVAL); 725341f19c9SNeel Natu } 726341f19c9SNeel Natu 7276b389740SMark Johnston obj = vm_object_allocate(OBJT_SWAP, len >> PAGE_SHIFT); 7289b1aa8d6SNeel Natu if (obj == NULL) 729318224bbSNeel Natu return (ENOMEM); 730318224bbSNeel Natu 731318224bbSNeel Natu seg->len = len; 7329b1aa8d6SNeel Natu seg->object = obj; 7339b1aa8d6SNeel Natu seg->sysmem = sysmem; 734366f6083SPeter Grehan return (0); 735366f6083SPeter Grehan } 736366f6083SPeter Grehan 7379b1aa8d6SNeel Natu int 7389b1aa8d6SNeel Natu vm_get_memseg(struct vm *vm, int ident, size_t *len, bool *sysmem, 7399b1aa8d6SNeel Natu vm_object_t *objptr) 740477867a0SNeel Natu { 7419b1aa8d6SNeel Natu struct mem_seg *seg; 742477867a0SNeel Natu 7439b1aa8d6SNeel Natu if (ident < 0 || ident >= VM_MAX_MEMSEGS) 7449b1aa8d6SNeel Natu return (EINVAL); 7459b1aa8d6SNeel Natu 7469b1aa8d6SNeel Natu seg = &vm->mem_segs[ident]; 7479b1aa8d6SNeel Natu if (len) 7489b1aa8d6SNeel Natu *len = seg->len; 7499b1aa8d6SNeel Natu if (sysmem) 7509b1aa8d6SNeel Natu *sysmem = seg->sysmem; 7519b1aa8d6SNeel Natu if (objptr) 7529b1aa8d6SNeel Natu *objptr = seg->object; 7539b1aa8d6SNeel Natu return (0); 754477867a0SNeel Natu } 7559b1aa8d6SNeel Natu 7569b1aa8d6SNeel Natu void 7579b1aa8d6SNeel Natu vm_free_memseg(struct vm *vm, int ident) 7589b1aa8d6SNeel Natu { 7599b1aa8d6SNeel Natu struct mem_seg *seg; 7609b1aa8d6SNeel Natu 7619b1aa8d6SNeel Natu KASSERT(ident >= 0 && ident < VM_MAX_MEMSEGS, 7629b1aa8d6SNeel Natu ("%s: invalid memseg ident %d", __func__, ident)); 7639b1aa8d6SNeel Natu 7649b1aa8d6SNeel Natu seg = &vm->mem_segs[ident]; 7659b1aa8d6SNeel Natu if (seg->object != NULL) { 7669b1aa8d6SNeel Natu vm_object_deallocate(seg->object); 7679b1aa8d6SNeel Natu bzero(seg, sizeof(struct mem_seg)); 7689b1aa8d6SNeel Natu } 7699b1aa8d6SNeel Natu } 7709b1aa8d6SNeel Natu 7719b1aa8d6SNeel Natu int 7729b1aa8d6SNeel Natu vm_mmap_memseg(struct vm *vm, vm_paddr_t gpa, int segid, vm_ooffset_t first, 7739b1aa8d6SNeel Natu size_t len, int prot, int flags) 7749b1aa8d6SNeel Natu { 7759b1aa8d6SNeel Natu struct mem_seg *seg; 7769b1aa8d6SNeel Natu struct mem_map *m, *map; 7779b1aa8d6SNeel Natu vm_ooffset_t last; 7789b1aa8d6SNeel Natu int i, error; 7799b1aa8d6SNeel Natu 7809b1aa8d6SNeel Natu if (prot == 0 || (prot & ~(VM_PROT_ALL)) != 0) 7819b1aa8d6SNeel Natu return (EINVAL); 7829b1aa8d6SNeel Natu 7839b1aa8d6SNeel Natu if (flags & ~VM_MEMMAP_F_WIRED) 7849b1aa8d6SNeel Natu return (EINVAL); 7859b1aa8d6SNeel Natu 7869b1aa8d6SNeel Natu if (segid < 0 || segid >= VM_MAX_MEMSEGS) 7879b1aa8d6SNeel Natu return (EINVAL); 7889b1aa8d6SNeel Natu 7899b1aa8d6SNeel Natu seg = &vm->mem_segs[segid]; 7909b1aa8d6SNeel Natu if (seg->object == NULL) 7919b1aa8d6SNeel Natu return (EINVAL); 7929b1aa8d6SNeel Natu 7939b1aa8d6SNeel Natu last = first + len; 7949b1aa8d6SNeel Natu if (first < 0 || first >= last || last > seg->len) 7959b1aa8d6SNeel Natu return (EINVAL); 7969b1aa8d6SNeel Natu 7979b1aa8d6SNeel Natu if ((gpa | first | last) & PAGE_MASK) 7989b1aa8d6SNeel Natu return (EINVAL); 7999b1aa8d6SNeel Natu 8009b1aa8d6SNeel Natu map = NULL; 8019b1aa8d6SNeel Natu for (i = 0; i < VM_MAX_MEMMAPS; i++) { 8029b1aa8d6SNeel Natu m = &vm->mem_maps[i]; 8039b1aa8d6SNeel Natu if (m->len == 0) { 8049b1aa8d6SNeel Natu map = m; 8059b1aa8d6SNeel Natu break; 8069b1aa8d6SNeel Natu } 8079b1aa8d6SNeel Natu } 8089b1aa8d6SNeel Natu 8099b1aa8d6SNeel Natu if (map == NULL) 8109b1aa8d6SNeel Natu return (ENOSPC); 8119b1aa8d6SNeel Natu 8129b1aa8d6SNeel Natu error = vm_map_find(&vm->vmspace->vm_map, seg->object, first, &gpa, 8139b1aa8d6SNeel Natu len, 0, VMFS_NO_SPACE, prot, prot, 0); 8149b1aa8d6SNeel Natu if (error != KERN_SUCCESS) 8159b1aa8d6SNeel Natu return (EFAULT); 8169b1aa8d6SNeel Natu 8179b1aa8d6SNeel Natu vm_object_reference(seg->object); 8189b1aa8d6SNeel Natu 8199b1aa8d6SNeel Natu if (flags & VM_MEMMAP_F_WIRED) { 8209b1aa8d6SNeel Natu error = vm_map_wire(&vm->vmspace->vm_map, gpa, gpa + len, 8219b1aa8d6SNeel Natu VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 8229b1aa8d6SNeel Natu if (error != KERN_SUCCESS) { 8239b1aa8d6SNeel Natu vm_map_remove(&vm->vmspace->vm_map, gpa, gpa + len); 82454a3a114SMark Johnston return (error == KERN_RESOURCE_SHORTAGE ? ENOMEM : 82554a3a114SMark Johnston EFAULT); 8269b1aa8d6SNeel Natu } 8279b1aa8d6SNeel Natu } 8289b1aa8d6SNeel Natu 8299b1aa8d6SNeel Natu map->gpa = gpa; 8309b1aa8d6SNeel Natu map->len = len; 8319b1aa8d6SNeel Natu map->segoff = first; 8329b1aa8d6SNeel Natu map->segid = segid; 8339b1aa8d6SNeel Natu map->prot = prot; 8349b1aa8d6SNeel Natu map->flags = flags; 8359b1aa8d6SNeel Natu return (0); 8369b1aa8d6SNeel Natu } 8379b1aa8d6SNeel Natu 8389b1aa8d6SNeel Natu int 839f8a6ec2dSD Scott Phillips vm_munmap_memseg(struct vm *vm, vm_paddr_t gpa, size_t len) 840f8a6ec2dSD Scott Phillips { 841f8a6ec2dSD Scott Phillips struct mem_map *m; 842f8a6ec2dSD Scott Phillips int i; 843f8a6ec2dSD Scott Phillips 844f8a6ec2dSD Scott Phillips for (i = 0; i < VM_MAX_MEMMAPS; i++) { 845f8a6ec2dSD Scott Phillips m = &vm->mem_maps[i]; 846f8a6ec2dSD Scott Phillips if (m->gpa == gpa && m->len == len && 847f8a6ec2dSD Scott Phillips (m->flags & VM_MEMMAP_F_IOMMU) == 0) { 848f8a6ec2dSD Scott Phillips vm_free_memmap(vm, i); 849f8a6ec2dSD Scott Phillips return (0); 850f8a6ec2dSD Scott Phillips } 851f8a6ec2dSD Scott Phillips } 852f8a6ec2dSD Scott Phillips 853f8a6ec2dSD Scott Phillips return (EINVAL); 854f8a6ec2dSD Scott Phillips } 855f8a6ec2dSD Scott Phillips 856f8a6ec2dSD Scott Phillips int 8579b1aa8d6SNeel Natu vm_mmap_getnext(struct vm *vm, vm_paddr_t *gpa, int *segid, 8589b1aa8d6SNeel Natu vm_ooffset_t *segoff, size_t *len, int *prot, int *flags) 8599b1aa8d6SNeel Natu { 8609b1aa8d6SNeel Natu struct mem_map *mm, *mmnext; 8619b1aa8d6SNeel Natu int i; 8629b1aa8d6SNeel Natu 8639b1aa8d6SNeel Natu mmnext = NULL; 8649b1aa8d6SNeel Natu for (i = 0; i < VM_MAX_MEMMAPS; i++) { 8659b1aa8d6SNeel Natu mm = &vm->mem_maps[i]; 8669b1aa8d6SNeel Natu if (mm->len == 0 || mm->gpa < *gpa) 8679b1aa8d6SNeel Natu continue; 8689b1aa8d6SNeel Natu if (mmnext == NULL || mm->gpa < mmnext->gpa) 8699b1aa8d6SNeel Natu mmnext = mm; 8709b1aa8d6SNeel Natu } 8719b1aa8d6SNeel Natu 8729b1aa8d6SNeel Natu if (mmnext != NULL) { 8739b1aa8d6SNeel Natu *gpa = mmnext->gpa; 8749b1aa8d6SNeel Natu if (segid) 8759b1aa8d6SNeel Natu *segid = mmnext->segid; 8769b1aa8d6SNeel Natu if (segoff) 8779b1aa8d6SNeel Natu *segoff = mmnext->segoff; 8789b1aa8d6SNeel Natu if (len) 8799b1aa8d6SNeel Natu *len = mmnext->len; 8809b1aa8d6SNeel Natu if (prot) 8819b1aa8d6SNeel Natu *prot = mmnext->prot; 8829b1aa8d6SNeel Natu if (flags) 8839b1aa8d6SNeel Natu *flags = mmnext->flags; 8849b1aa8d6SNeel Natu return (0); 8859b1aa8d6SNeel Natu } else { 8869b1aa8d6SNeel Natu return (ENOENT); 8879b1aa8d6SNeel Natu } 888477867a0SNeel Natu } 889477867a0SNeel Natu 890318224bbSNeel Natu static void 8919b1aa8d6SNeel Natu vm_free_memmap(struct vm *vm, int ident) 892366f6083SPeter Grehan { 8939b1aa8d6SNeel Natu struct mem_map *mm; 89473505a10SRobert Wing int error __diagused; 8954db4fb2cSNeel Natu 8969b1aa8d6SNeel Natu mm = &vm->mem_maps[ident]; 8979b1aa8d6SNeel Natu if (mm->len) { 8989b1aa8d6SNeel Natu error = vm_map_remove(&vm->vmspace->vm_map, mm->gpa, 8999b1aa8d6SNeel Natu mm->gpa + mm->len); 9009b1aa8d6SNeel Natu KASSERT(error == KERN_SUCCESS, ("%s: vm_map_remove error %d", 9019b1aa8d6SNeel Natu __func__, error)); 9029b1aa8d6SNeel Natu bzero(mm, sizeof(struct mem_map)); 903318224bbSNeel Natu } 904318224bbSNeel Natu } 905318224bbSNeel Natu 9069b1aa8d6SNeel Natu static __inline bool 9079b1aa8d6SNeel Natu sysmem_mapping(struct vm *vm, struct mem_map *mm) 908318224bbSNeel Natu { 909318224bbSNeel Natu 9109b1aa8d6SNeel Natu if (mm->len != 0 && vm->mem_segs[mm->segid].sysmem) 9119b1aa8d6SNeel Natu return (true); 9129b1aa8d6SNeel Natu else 9139b1aa8d6SNeel Natu return (false); 914318224bbSNeel Natu } 915318224bbSNeel Natu 916147d12a7SAntoine Brodin vm_paddr_t 917147d12a7SAntoine Brodin vmm_sysmem_maxaddr(struct vm *vm) 9189b1aa8d6SNeel Natu { 9199b1aa8d6SNeel Natu struct mem_map *mm; 9209b1aa8d6SNeel Natu vm_paddr_t maxaddr; 9219b1aa8d6SNeel Natu int i; 922318224bbSNeel Natu 9239b1aa8d6SNeel Natu maxaddr = 0; 9249b1aa8d6SNeel Natu for (i = 0; i < VM_MAX_MEMMAPS; i++) { 9259b1aa8d6SNeel Natu mm = &vm->mem_maps[i]; 9269b1aa8d6SNeel Natu if (sysmem_mapping(vm, mm)) { 9279b1aa8d6SNeel Natu if (maxaddr < mm->gpa + mm->len) 9289b1aa8d6SNeel Natu maxaddr = mm->gpa + mm->len; 9299b1aa8d6SNeel Natu } 9309b1aa8d6SNeel Natu } 9319b1aa8d6SNeel Natu return (maxaddr); 932318224bbSNeel Natu } 933318224bbSNeel Natu 934318224bbSNeel Natu static void 935490d56c5SEd Maste vm_iommu_modify(struct vm *vm, bool map) 936318224bbSNeel Natu { 937318224bbSNeel Natu int i, sz; 938318224bbSNeel Natu vm_paddr_t gpa, hpa; 9399b1aa8d6SNeel Natu struct mem_map *mm; 940318224bbSNeel Natu void *vp, *cookie, *host_domain; 941318224bbSNeel Natu 942318224bbSNeel Natu sz = PAGE_SIZE; 943318224bbSNeel Natu host_domain = iommu_host_domain(); 944318224bbSNeel Natu 9459b1aa8d6SNeel Natu for (i = 0; i < VM_MAX_MEMMAPS; i++) { 9469b1aa8d6SNeel Natu mm = &vm->mem_maps[i]; 9479b1aa8d6SNeel Natu if (!sysmem_mapping(vm, mm)) 9489b1aa8d6SNeel Natu continue; 949318224bbSNeel Natu 9509b1aa8d6SNeel Natu if (map) { 9519b1aa8d6SNeel Natu KASSERT((mm->flags & VM_MEMMAP_F_IOMMU) == 0, 9529b1aa8d6SNeel Natu ("iommu map found invalid memmap %#lx/%#lx/%#x", 9539b1aa8d6SNeel Natu mm->gpa, mm->len, mm->flags)); 9549b1aa8d6SNeel Natu if ((mm->flags & VM_MEMMAP_F_WIRED) == 0) 9559b1aa8d6SNeel Natu continue; 9569b1aa8d6SNeel Natu mm->flags |= VM_MEMMAP_F_IOMMU; 9579b1aa8d6SNeel Natu } else { 9589b1aa8d6SNeel Natu if ((mm->flags & VM_MEMMAP_F_IOMMU) == 0) 9599b1aa8d6SNeel Natu continue; 9609b1aa8d6SNeel Natu mm->flags &= ~VM_MEMMAP_F_IOMMU; 9619b1aa8d6SNeel Natu KASSERT((mm->flags & VM_MEMMAP_F_WIRED) != 0, 9629b1aa8d6SNeel Natu ("iommu unmap found invalid memmap %#lx/%#lx/%#x", 9639b1aa8d6SNeel Natu mm->gpa, mm->len, mm->flags)); 9649b1aa8d6SNeel Natu } 9659b1aa8d6SNeel Natu 9669b1aa8d6SNeel Natu gpa = mm->gpa; 9679b1aa8d6SNeel Natu while (gpa < mm->gpa + mm->len) { 9689b1aa8d6SNeel Natu vp = vm_gpa_hold(vm, -1, gpa, PAGE_SIZE, VM_PROT_WRITE, 969318224bbSNeel Natu &cookie); 970318224bbSNeel Natu KASSERT(vp != NULL, ("vm(%s) could not map gpa %#lx", 971318224bbSNeel Natu vm_name(vm), gpa)); 972318224bbSNeel Natu 973318224bbSNeel Natu vm_gpa_release(cookie); 974318224bbSNeel Natu 975318224bbSNeel Natu hpa = DMAP_TO_PHYS((uintptr_t)vp); 976318224bbSNeel Natu if (map) { 977318224bbSNeel Natu iommu_create_mapping(vm->iommu, gpa, hpa, sz); 978318224bbSNeel Natu } else { 979318224bbSNeel Natu iommu_remove_mapping(vm->iommu, gpa, sz); 980318224bbSNeel Natu } 981318224bbSNeel Natu 982318224bbSNeel Natu gpa += PAGE_SIZE; 983318224bbSNeel Natu } 984318224bbSNeel Natu } 985318224bbSNeel Natu 986318224bbSNeel Natu /* 987318224bbSNeel Natu * Invalidate the cached translations associated with the domain 988318224bbSNeel Natu * from which pages were removed. 989318224bbSNeel Natu */ 990318224bbSNeel Natu if (map) 991318224bbSNeel Natu iommu_invalidate_tlb(host_domain); 992318224bbSNeel Natu else 993318224bbSNeel Natu iommu_invalidate_tlb(vm->iommu); 994318224bbSNeel Natu } 995318224bbSNeel Natu 996490d56c5SEd Maste #define vm_iommu_unmap(vm) vm_iommu_modify((vm), false) 997490d56c5SEd Maste #define vm_iommu_map(vm) vm_iommu_modify((vm), true) 998318224bbSNeel Natu 999318224bbSNeel Natu int 1000318224bbSNeel Natu vm_unassign_pptdev(struct vm *vm, int bus, int slot, int func) 1001318224bbSNeel Natu { 1002318224bbSNeel Natu int error; 1003318224bbSNeel Natu 1004318224bbSNeel Natu error = ppt_unassign_device(vm, bus, slot, func); 1005318224bbSNeel Natu if (error) 1006318224bbSNeel Natu return (error); 1007318224bbSNeel Natu 10089b1aa8d6SNeel Natu if (ppt_assigned_devices(vm) == 0) 1009318224bbSNeel Natu vm_iommu_unmap(vm); 10109b1aa8d6SNeel Natu 1011318224bbSNeel Natu return (0); 1012318224bbSNeel Natu } 1013318224bbSNeel Natu 1014318224bbSNeel Natu int 1015318224bbSNeel Natu vm_assign_pptdev(struct vm *vm, int bus, int slot, int func) 1016318224bbSNeel Natu { 1017318224bbSNeel Natu int error; 1018318224bbSNeel Natu vm_paddr_t maxaddr; 1019318224bbSNeel Natu 10209b1aa8d6SNeel Natu /* Set up the IOMMU to do the 'gpa' to 'hpa' translation */ 102151f45d01SNeel Natu if (ppt_assigned_devices(vm) == 0) { 1022318224bbSNeel Natu KASSERT(vm->iommu == NULL, 1023318224bbSNeel Natu ("vm_assign_pptdev: iommu must be NULL")); 1024147d12a7SAntoine Brodin maxaddr = vmm_sysmem_maxaddr(vm); 1025318224bbSNeel Natu vm->iommu = iommu_create_domain(maxaddr); 1026ffe1b10dSJohn Baldwin if (vm->iommu == NULL) 1027ffe1b10dSJohn Baldwin return (ENXIO); 1028318224bbSNeel Natu vm_iommu_map(vm); 1029318224bbSNeel Natu } 1030318224bbSNeel Natu 1031318224bbSNeel Natu error = ppt_assign_device(vm, bus, slot, func); 1032318224bbSNeel Natu return (error); 1033318224bbSNeel Natu } 1034318224bbSNeel Natu 1035318224bbSNeel Natu void * 10369b1aa8d6SNeel Natu vm_gpa_hold(struct vm *vm, int vcpuid, vm_paddr_t gpa, size_t len, int reqprot, 1037318224bbSNeel Natu void **cookie) 1038318224bbSNeel Natu { 10399b1aa8d6SNeel Natu int i, count, pageoff; 10409b1aa8d6SNeel Natu struct mem_map *mm; 1041318224bbSNeel Natu vm_page_t m; 10429b1aa8d6SNeel Natu #ifdef INVARIANTS 10439b1aa8d6SNeel Natu /* 10449b1aa8d6SNeel Natu * All vcpus are frozen by ioctls that modify the memory map 10459b1aa8d6SNeel Natu * (e.g. VM_MMAP_MEMSEG). Therefore 'vm->memmap[]' stability is 10469b1aa8d6SNeel Natu * guaranteed if at least one vcpu is in the VCPU_FROZEN state. 10479b1aa8d6SNeel Natu */ 10489b1aa8d6SNeel Natu int state; 1049a488c9c9SRodney W. Grimes KASSERT(vcpuid >= -1 && vcpuid < vm->maxcpus, ("%s: invalid vcpuid %d", 10509b1aa8d6SNeel Natu __func__, vcpuid)); 1051a488c9c9SRodney W. Grimes for (i = 0; i < vm->maxcpus; i++) { 10529b1aa8d6SNeel Natu if (vcpuid != -1 && vcpuid != i) 10539b1aa8d6SNeel Natu continue; 10549b1aa8d6SNeel Natu state = vcpu_get_state(vm, i, NULL); 10559b1aa8d6SNeel Natu KASSERT(state == VCPU_FROZEN, ("%s: invalid vcpu state %d", 10569b1aa8d6SNeel Natu __func__, state)); 10579b1aa8d6SNeel Natu } 10589b1aa8d6SNeel Natu #endif 1059318224bbSNeel Natu pageoff = gpa & PAGE_MASK; 1060318224bbSNeel Natu if (len > PAGE_SIZE - pageoff) 1061318224bbSNeel Natu panic("vm_gpa_hold: invalid gpa/len: 0x%016lx/%lu", gpa, len); 1062318224bbSNeel Natu 10639b1aa8d6SNeel Natu count = 0; 10649b1aa8d6SNeel Natu for (i = 0; i < VM_MAX_MEMMAPS; i++) { 10659b1aa8d6SNeel Natu mm = &vm->mem_maps[i]; 106646567b4fSPeter Grehan if (gpa >= mm->gpa && gpa < mm->gpa + mm->len) { 1067318224bbSNeel Natu count = vm_fault_quick_hold_pages(&vm->vmspace->vm_map, 1068318224bbSNeel Natu trunc_page(gpa), PAGE_SIZE, reqprot, &m, 1); 10699b1aa8d6SNeel Natu break; 10709b1aa8d6SNeel Natu } 10719b1aa8d6SNeel Natu } 1072318224bbSNeel Natu 1073318224bbSNeel Natu if (count == 1) { 1074318224bbSNeel Natu *cookie = m; 1075318224bbSNeel Natu return ((void *)(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)) + pageoff)); 1076318224bbSNeel Natu } else { 1077318224bbSNeel Natu *cookie = NULL; 1078318224bbSNeel Natu return (NULL); 1079318224bbSNeel Natu } 1080318224bbSNeel Natu } 1081318224bbSNeel Natu 1082318224bbSNeel Natu void 1083318224bbSNeel Natu vm_gpa_release(void *cookie) 1084318224bbSNeel Natu { 1085318224bbSNeel Natu vm_page_t m = cookie; 1086318224bbSNeel Natu 1087eeacb3b0SMark Johnston vm_page_unwire(m, PQ_ACTIVE); 1088366f6083SPeter Grehan } 1089366f6083SPeter Grehan 1090366f6083SPeter Grehan int 1091366f6083SPeter Grehan vm_get_register(struct vm *vm, int vcpu, int reg, uint64_t *retval) 1092366f6083SPeter Grehan { 1093366f6083SPeter Grehan 1094a488c9c9SRodney W. Grimes if (vcpu < 0 || vcpu >= vm->maxcpus) 1095366f6083SPeter Grehan return (EINVAL); 1096366f6083SPeter Grehan 1097366f6083SPeter Grehan if (reg >= VM_REG_LAST) 1098366f6083SPeter Grehan return (EINVAL); 1099366f6083SPeter Grehan 1100869c8d19SJohn Baldwin return (vmmops_getreg(vcpu_cookie(vm, vcpu), reg, retval)); 1101366f6083SPeter Grehan } 1102366f6083SPeter Grehan 1103366f6083SPeter Grehan int 1104d087a399SNeel Natu vm_set_register(struct vm *vm, int vcpuid, int reg, uint64_t val) 1105366f6083SPeter Grehan { 1106d087a399SNeel Natu struct vcpu *vcpu; 1107d087a399SNeel Natu int error; 1108366f6083SPeter Grehan 1109a488c9c9SRodney W. Grimes if (vcpuid < 0 || vcpuid >= vm->maxcpus) 1110366f6083SPeter Grehan return (EINVAL); 1111366f6083SPeter Grehan 1112366f6083SPeter Grehan if (reg >= VM_REG_LAST) 1113366f6083SPeter Grehan return (EINVAL); 1114366f6083SPeter Grehan 11151aa51504SJohn Baldwin vcpu = &vm->vcpu[vcpuid]; 1116869c8d19SJohn Baldwin error = vmmops_setreg(vcpu->cookie, reg, val); 1117d087a399SNeel Natu if (error || reg != VM_REG_GUEST_RIP) 1118d087a399SNeel Natu return (error); 1119d087a399SNeel Natu 1120d087a399SNeel Natu /* Set 'nextrip' to match the value of %rip */ 1121d087a399SNeel Natu VCPU_CTR1(vm, vcpuid, "Setting nextrip to %#lx", val); 1122d087a399SNeel Natu vcpu->nextrip = val; 1123d087a399SNeel Natu return (0); 1124366f6083SPeter Grehan } 1125366f6083SPeter Grehan 1126490d56c5SEd Maste static bool 1127366f6083SPeter Grehan is_descriptor_table(int reg) 1128366f6083SPeter Grehan { 1129366f6083SPeter Grehan 1130366f6083SPeter Grehan switch (reg) { 1131366f6083SPeter Grehan case VM_REG_GUEST_IDTR: 1132366f6083SPeter Grehan case VM_REG_GUEST_GDTR: 1133490d56c5SEd Maste return (true); 1134366f6083SPeter Grehan default: 1135490d56c5SEd Maste return (false); 1136366f6083SPeter Grehan } 1137366f6083SPeter Grehan } 1138366f6083SPeter Grehan 1139490d56c5SEd Maste static bool 1140366f6083SPeter Grehan is_segment_register(int reg) 1141366f6083SPeter Grehan { 1142366f6083SPeter Grehan 1143366f6083SPeter Grehan switch (reg) { 1144366f6083SPeter Grehan case VM_REG_GUEST_ES: 1145366f6083SPeter Grehan case VM_REG_GUEST_CS: 1146366f6083SPeter Grehan case VM_REG_GUEST_SS: 1147366f6083SPeter Grehan case VM_REG_GUEST_DS: 1148366f6083SPeter Grehan case VM_REG_GUEST_FS: 1149366f6083SPeter Grehan case VM_REG_GUEST_GS: 1150366f6083SPeter Grehan case VM_REG_GUEST_TR: 1151366f6083SPeter Grehan case VM_REG_GUEST_LDTR: 1152490d56c5SEd Maste return (true); 1153366f6083SPeter Grehan default: 1154490d56c5SEd Maste return (false); 1155366f6083SPeter Grehan } 1156366f6083SPeter Grehan } 1157366f6083SPeter Grehan 1158366f6083SPeter Grehan int 1159366f6083SPeter Grehan vm_get_seg_desc(struct vm *vm, int vcpu, int reg, 1160366f6083SPeter Grehan struct seg_desc *desc) 1161366f6083SPeter Grehan { 1162366f6083SPeter Grehan 1163a488c9c9SRodney W. Grimes if (vcpu < 0 || vcpu >= vm->maxcpus) 1164366f6083SPeter Grehan return (EINVAL); 1165366f6083SPeter Grehan 1166366f6083SPeter Grehan if (!is_segment_register(reg) && !is_descriptor_table(reg)) 1167366f6083SPeter Grehan return (EINVAL); 1168366f6083SPeter Grehan 1169869c8d19SJohn Baldwin return (vmmops_getdesc(vcpu_cookie(vm, vcpu), reg, desc)); 1170366f6083SPeter Grehan } 1171366f6083SPeter Grehan 1172366f6083SPeter Grehan int 1173366f6083SPeter Grehan vm_set_seg_desc(struct vm *vm, int vcpu, int reg, 1174366f6083SPeter Grehan struct seg_desc *desc) 1175366f6083SPeter Grehan { 1176a488c9c9SRodney W. Grimes if (vcpu < 0 || vcpu >= vm->maxcpus) 1177366f6083SPeter Grehan return (EINVAL); 1178366f6083SPeter Grehan 1179366f6083SPeter Grehan if (!is_segment_register(reg) && !is_descriptor_table(reg)) 1180366f6083SPeter Grehan return (EINVAL); 1181366f6083SPeter Grehan 1182869c8d19SJohn Baldwin return (vmmops_setdesc(vcpu_cookie(vm, vcpu), reg, desc)); 1183366f6083SPeter Grehan } 1184366f6083SPeter Grehan 1185366f6083SPeter Grehan static void 1186366f6083SPeter Grehan restore_guest_fpustate(struct vcpu *vcpu) 1187366f6083SPeter Grehan { 1188366f6083SPeter Grehan 118938f1b189SPeter Grehan /* flush host state to the pcb */ 119038f1b189SPeter Grehan fpuexit(curthread); 1191bd8572e0SNeel Natu 1192bd8572e0SNeel Natu /* restore guest FPU state */ 1193366f6083SPeter Grehan fpu_stop_emulating(); 119438f1b189SPeter Grehan fpurestore(vcpu->guestfpu); 1195bd8572e0SNeel Natu 1196abb023fbSJohn Baldwin /* restore guest XCR0 if XSAVE is enabled in the host */ 1197abb023fbSJohn Baldwin if (rcr4() & CR4_XSAVE) 1198abb023fbSJohn Baldwin load_xcr(0, vcpu->guest_xcr0); 1199abb023fbSJohn Baldwin 1200bd8572e0SNeel Natu /* 1201bd8572e0SNeel Natu * The FPU is now "dirty" with the guest's state so turn on emulation 1202bd8572e0SNeel Natu * to trap any access to the FPU by the host. 1203bd8572e0SNeel Natu */ 1204bd8572e0SNeel Natu fpu_start_emulating(); 1205366f6083SPeter Grehan } 1206366f6083SPeter Grehan 1207366f6083SPeter Grehan static void 1208366f6083SPeter Grehan save_guest_fpustate(struct vcpu *vcpu) 1209366f6083SPeter Grehan { 1210366f6083SPeter Grehan 1211bd8572e0SNeel Natu if ((rcr0() & CR0_TS) == 0) 1212bd8572e0SNeel Natu panic("fpu emulation not enabled in host!"); 1213bd8572e0SNeel Natu 1214abb023fbSJohn Baldwin /* save guest XCR0 and restore host XCR0 */ 1215abb023fbSJohn Baldwin if (rcr4() & CR4_XSAVE) { 1216abb023fbSJohn Baldwin vcpu->guest_xcr0 = rxcr(0); 1217abb023fbSJohn Baldwin load_xcr(0, vmm_get_host_xcr0()); 1218abb023fbSJohn Baldwin } 1219abb023fbSJohn Baldwin 1220bd8572e0SNeel Natu /* save guest FPU state */ 1221bd8572e0SNeel Natu fpu_stop_emulating(); 122238f1b189SPeter Grehan fpusave(vcpu->guestfpu); 1223366f6083SPeter Grehan fpu_start_emulating(); 1224366f6083SPeter Grehan } 1225366f6083SPeter Grehan 122661592433SNeel Natu static VMM_STAT(VCPU_IDLE_TICKS, "number of ticks vcpu was idle"); 1227f76fc5d4SNeel Natu 1228318224bbSNeel Natu static int 1229248e6799SNeel Natu vcpu_set_state_locked(struct vm *vm, int vcpuid, enum vcpu_state newstate, 1230f80330a8SNeel Natu bool from_idle) 1231366f6083SPeter Grehan { 1232248e6799SNeel Natu struct vcpu *vcpu; 1233318224bbSNeel Natu int error; 1234366f6083SPeter Grehan 1235248e6799SNeel Natu vcpu = &vm->vcpu[vcpuid]; 1236318224bbSNeel Natu vcpu_assert_locked(vcpu); 1237366f6083SPeter Grehan 1238f76fc5d4SNeel Natu /* 1239f80330a8SNeel Natu * State transitions from the vmmdev_ioctl() must always begin from 1240f80330a8SNeel Natu * the VCPU_IDLE state. This guarantees that there is only a single 1241f80330a8SNeel Natu * ioctl() operating on a vcpu at any point. 1242f80330a8SNeel Natu */ 1243f80330a8SNeel Natu if (from_idle) { 1244248e6799SNeel Natu while (vcpu->state != VCPU_IDLE) { 1245248e6799SNeel Natu vcpu->reqidle = 1; 1246248e6799SNeel Natu vcpu_notify_event_locked(vcpu, false); 1247248e6799SNeel Natu VCPU_CTR1(vm, vcpuid, "vcpu state change from %s to " 1248248e6799SNeel Natu "idle requested", vcpu_state2str(vcpu->state)); 1249f80330a8SNeel Natu msleep_spin(&vcpu->state, &vcpu->mtx, "vmstat", hz); 1250248e6799SNeel Natu } 1251f80330a8SNeel Natu } else { 1252f80330a8SNeel Natu KASSERT(vcpu->state != VCPU_IDLE, ("invalid transition from " 1253f80330a8SNeel Natu "vcpu idle state")); 1254f80330a8SNeel Natu } 1255f80330a8SNeel Natu 1256ef39d7e9SNeel Natu if (vcpu->state == VCPU_RUNNING) { 1257ef39d7e9SNeel Natu KASSERT(vcpu->hostcpu == curcpu, ("curcpu %d and hostcpu %d " 1258ef39d7e9SNeel Natu "mismatch for running vcpu", curcpu, vcpu->hostcpu)); 1259ef39d7e9SNeel Natu } else { 1260ef39d7e9SNeel Natu KASSERT(vcpu->hostcpu == NOCPU, ("Invalid hostcpu %d for a " 1261ef39d7e9SNeel Natu "vcpu that is not running", vcpu->hostcpu)); 1262ef39d7e9SNeel Natu } 1263ef39d7e9SNeel Natu 1264f80330a8SNeel Natu /* 1265318224bbSNeel Natu * The following state transitions are allowed: 1266318224bbSNeel Natu * IDLE -> FROZEN -> IDLE 1267318224bbSNeel Natu * FROZEN -> RUNNING -> FROZEN 1268318224bbSNeel Natu * FROZEN -> SLEEPING -> FROZEN 1269f76fc5d4SNeel Natu */ 1270318224bbSNeel Natu switch (vcpu->state) { 1271318224bbSNeel Natu case VCPU_IDLE: 1272318224bbSNeel Natu case VCPU_RUNNING: 1273318224bbSNeel Natu case VCPU_SLEEPING: 1274318224bbSNeel Natu error = (newstate != VCPU_FROZEN); 1275318224bbSNeel Natu break; 1276318224bbSNeel Natu case VCPU_FROZEN: 1277318224bbSNeel Natu error = (newstate == VCPU_FROZEN); 1278318224bbSNeel Natu break; 1279318224bbSNeel Natu default: 1280318224bbSNeel Natu error = 1; 1281318224bbSNeel Natu break; 1282318224bbSNeel Natu } 1283318224bbSNeel Natu 1284f80330a8SNeel Natu if (error) 1285f80330a8SNeel Natu return (EBUSY); 1286318224bbSNeel Natu 1287248e6799SNeel Natu VCPU_CTR2(vm, vcpuid, "vcpu state changed from %s to %s", 1288248e6799SNeel Natu vcpu_state2str(vcpu->state), vcpu_state2str(newstate)); 1289248e6799SNeel Natu 1290f80330a8SNeel Natu vcpu->state = newstate; 1291ef39d7e9SNeel Natu if (newstate == VCPU_RUNNING) 1292ef39d7e9SNeel Natu vcpu->hostcpu = curcpu; 1293ef39d7e9SNeel Natu else 1294ef39d7e9SNeel Natu vcpu->hostcpu = NOCPU; 1295ef39d7e9SNeel Natu 1296f80330a8SNeel Natu if (newstate == VCPU_IDLE) 1297f80330a8SNeel Natu wakeup(&vcpu->state); 1298f80330a8SNeel Natu 1299f80330a8SNeel Natu return (0); 1300318224bbSNeel Natu } 1301318224bbSNeel Natu 1302318224bbSNeel Natu static void 1303318224bbSNeel Natu vcpu_require_state(struct vm *vm, int vcpuid, enum vcpu_state newstate) 1304318224bbSNeel Natu { 1305318224bbSNeel Natu int error; 1306318224bbSNeel Natu 1307f80330a8SNeel Natu if ((error = vcpu_set_state(vm, vcpuid, newstate, false)) != 0) 1308318224bbSNeel Natu panic("Error %d setting state to %d\n", error, newstate); 1309318224bbSNeel Natu } 1310318224bbSNeel Natu 1311318224bbSNeel Natu static void 1312248e6799SNeel Natu vcpu_require_state_locked(struct vm *vm, int vcpuid, enum vcpu_state newstate) 1313318224bbSNeel Natu { 1314318224bbSNeel Natu int error; 1315318224bbSNeel Natu 1316248e6799SNeel Natu if ((error = vcpu_set_state_locked(vm, vcpuid, newstate, false)) != 0) 1317318224bbSNeel Natu panic("Error %d setting state to %d", error, newstate); 1318318224bbSNeel Natu } 1319318224bbSNeel Natu 13205b8a8cd1SNeel Natu #define RENDEZVOUS_CTR0(vm, vcpuid, fmt) \ 13215b8a8cd1SNeel Natu do { \ 13225b8a8cd1SNeel Natu if (vcpuid >= 0) \ 13235b8a8cd1SNeel Natu VCPU_CTR0(vm, vcpuid, fmt); \ 13245b8a8cd1SNeel Natu else \ 13255b8a8cd1SNeel Natu VM_CTR0(vm, fmt); \ 13265b8a8cd1SNeel Natu } while (0) 13275b8a8cd1SNeel Natu 1328b837daddSKonstantin Belousov static int 13295b8a8cd1SNeel Natu vm_handle_rendezvous(struct vm *vm, int vcpuid) 13305b8a8cd1SNeel Natu { 1331b837daddSKonstantin Belousov struct thread *td; 1332b837daddSKonstantin Belousov int error; 13335b8a8cd1SNeel Natu 1334a488c9c9SRodney W. Grimes KASSERT(vcpuid == -1 || (vcpuid >= 0 && vcpuid < vm->maxcpus), 13355b8a8cd1SNeel Natu ("vm_handle_rendezvous: invalid vcpuid %d", vcpuid)); 13365b8a8cd1SNeel Natu 1337b837daddSKonstantin Belousov error = 0; 1338b837daddSKonstantin Belousov td = curthread; 13395b8a8cd1SNeel Natu mtx_lock(&vm->rendezvous_mtx); 13405b8a8cd1SNeel Natu while (vm->rendezvous_func != NULL) { 134122d822c6SNeel Natu /* 'rendezvous_req_cpus' must be a subset of 'active_cpus' */ 1342e2650af1SStefan Eßer CPU_AND(&vm->rendezvous_req_cpus, &vm->rendezvous_req_cpus, &vm->active_cpus); 134322d822c6SNeel Natu 13445b8a8cd1SNeel Natu if (vcpuid != -1 && 134522d822c6SNeel Natu CPU_ISSET(vcpuid, &vm->rendezvous_req_cpus) && 134622d822c6SNeel Natu !CPU_ISSET(vcpuid, &vm->rendezvous_done_cpus)) { 13475b8a8cd1SNeel Natu VCPU_CTR0(vm, vcpuid, "Calling rendezvous func"); 13485b8a8cd1SNeel Natu (*vm->rendezvous_func)(vm, vcpuid, vm->rendezvous_arg); 13495b8a8cd1SNeel Natu CPU_SET(vcpuid, &vm->rendezvous_done_cpus); 13505b8a8cd1SNeel Natu } 13515b8a8cd1SNeel Natu if (CPU_CMP(&vm->rendezvous_req_cpus, 13525b8a8cd1SNeel Natu &vm->rendezvous_done_cpus) == 0) { 13535b8a8cd1SNeel Natu VCPU_CTR0(vm, vcpuid, "Rendezvous completed"); 1354869dbab7SAndriy Gapon vm->rendezvous_func = NULL; 13555b8a8cd1SNeel Natu wakeup(&vm->rendezvous_func); 13565b8a8cd1SNeel Natu break; 13575b8a8cd1SNeel Natu } 13585b8a8cd1SNeel Natu RENDEZVOUS_CTR0(vm, vcpuid, "Wait for rendezvous completion"); 13595b8a8cd1SNeel Natu mtx_sleep(&vm->rendezvous_func, &vm->rendezvous_mtx, 0, 1360b837daddSKonstantin Belousov "vmrndv", hz); 1361c6d31b83SKonstantin Belousov if (td_ast_pending(td, TDA_SUSPEND)) { 1362b837daddSKonstantin Belousov mtx_unlock(&vm->rendezvous_mtx); 1363b837daddSKonstantin Belousov error = thread_check_susp(td, true); 1364b837daddSKonstantin Belousov if (error != 0) 1365b837daddSKonstantin Belousov return (error); 1366b837daddSKonstantin Belousov mtx_lock(&vm->rendezvous_mtx); 1367b837daddSKonstantin Belousov } 13685b8a8cd1SNeel Natu } 13695b8a8cd1SNeel Natu mtx_unlock(&vm->rendezvous_mtx); 1370b837daddSKonstantin Belousov return (0); 13715b8a8cd1SNeel Natu } 13725b8a8cd1SNeel Natu 1373318224bbSNeel Natu /* 1374318224bbSNeel Natu * Emulate a guest 'hlt' by sleeping until the vcpu is ready to run. 1375318224bbSNeel Natu */ 1376318224bbSNeel Natu static int 1377becd9849SNeel Natu vm_handle_hlt(struct vm *vm, int vcpuid, bool intr_disabled, bool *retu) 1378318224bbSNeel Natu { 1379318224bbSNeel Natu struct vcpu *vcpu; 1380c6a0cc2eSNeel Natu const char *wmesg; 1381b837daddSKonstantin Belousov struct thread *td; 1382b837daddSKonstantin Belousov int error, t, vcpu_halted, vm_halted; 1383e50ce2aaSNeel Natu 1384e50ce2aaSNeel Natu KASSERT(!CPU_ISSET(vcpuid, &vm->halted_cpus), ("vcpu already halted")); 1385318224bbSNeel Natu 1386318224bbSNeel Natu vcpu = &vm->vcpu[vcpuid]; 1387e50ce2aaSNeel Natu vcpu_halted = 0; 1388e50ce2aaSNeel Natu vm_halted = 0; 1389b837daddSKonstantin Belousov error = 0; 1390b837daddSKonstantin Belousov td = curthread; 1391318224bbSNeel Natu 1392f76fc5d4SNeel Natu vcpu_lock(vcpu); 1393c6a0cc2eSNeel Natu while (1) { 1394f76fc5d4SNeel Natu /* 1395f76fc5d4SNeel Natu * Do a final check for pending NMI or interrupts before 1396c6a0cc2eSNeel Natu * really putting this thread to sleep. Also check for 1397c6a0cc2eSNeel Natu * software events that would cause this vcpu to wakeup. 1398f76fc5d4SNeel Natu * 1399c6a0cc2eSNeel Natu * These interrupts/events could have happened after the 140015add60dSPeter Grehan * vcpu returned from vmmops_run() and before it acquired the 1401c6a0cc2eSNeel Natu * vcpu lock above. 1402f76fc5d4SNeel Natu */ 1403248e6799SNeel Natu if (vm->rendezvous_func != NULL || vm->suspend || vcpu->reqidle) 1404c6a0cc2eSNeel Natu break; 1405c6a0cc2eSNeel Natu if (vm_nmi_pending(vm, vcpuid)) 1406c6a0cc2eSNeel Natu break; 1407c6a0cc2eSNeel Natu if (!intr_disabled) { 1408c6a0cc2eSNeel Natu if (vm_extint_pending(vm, vcpuid) || 1409c6a0cc2eSNeel Natu vlapic_pending_intr(vcpu->vlapic, NULL)) { 1410c6a0cc2eSNeel Natu break; 1411c6a0cc2eSNeel Natu } 1412c6a0cc2eSNeel Natu } 1413c6a0cc2eSNeel Natu 1414f008d157SNeel Natu /* Don't go to sleep if the vcpu thread needs to yield */ 1415f008d157SNeel Natu if (vcpu_should_yield(vm, vcpuid)) 1416f008d157SNeel Natu break; 1417f008d157SNeel Natu 1418fc276d92SJohn Baldwin if (vcpu_debugged(vm, vcpuid)) 1419fc276d92SJohn Baldwin break; 1420fc276d92SJohn Baldwin 1421e50ce2aaSNeel Natu /* 1422e50ce2aaSNeel Natu * Some Linux guests implement "halt" by having all vcpus 1423e50ce2aaSNeel Natu * execute HLT with interrupts disabled. 'halted_cpus' keeps 1424e50ce2aaSNeel Natu * track of the vcpus that have entered this state. When all 1425e50ce2aaSNeel Natu * vcpus enter the halted state the virtual machine is halted. 1426e50ce2aaSNeel Natu */ 1427e50ce2aaSNeel Natu if (intr_disabled) { 1428c6a0cc2eSNeel Natu wmesg = "vmhalt"; 1429e50ce2aaSNeel Natu VCPU_CTR0(vm, vcpuid, "Halted"); 1430055fc2cbSNeel Natu if (!vcpu_halted && halt_detection_enabled) { 1431e50ce2aaSNeel Natu vcpu_halted = 1; 1432e50ce2aaSNeel Natu CPU_SET_ATOMIC(vcpuid, &vm->halted_cpus); 1433e50ce2aaSNeel Natu } 1434e50ce2aaSNeel Natu if (CPU_CMP(&vm->halted_cpus, &vm->active_cpus) == 0) { 1435e50ce2aaSNeel Natu vm_halted = 1; 1436e50ce2aaSNeel Natu break; 1437e50ce2aaSNeel Natu } 1438e50ce2aaSNeel Natu } else { 1439e50ce2aaSNeel Natu wmesg = "vmidle"; 1440e50ce2aaSNeel Natu } 1441c6a0cc2eSNeel Natu 1442f76fc5d4SNeel Natu t = ticks; 1443248e6799SNeel Natu vcpu_require_state_locked(vm, vcpuid, VCPU_SLEEPING); 1444f008d157SNeel Natu /* 1445f008d157SNeel Natu * XXX msleep_spin() cannot be interrupted by signals so 1446f008d157SNeel Natu * wake up periodically to check pending signals. 1447f008d157SNeel Natu */ 1448f008d157SNeel Natu msleep_spin(vcpu, &vcpu->mtx, wmesg, hz); 1449248e6799SNeel Natu vcpu_require_state_locked(vm, vcpuid, VCPU_FROZEN); 14503dc3d32aSJohn Baldwin vmm_stat_incr(vcpu, VCPU_IDLE_TICKS, ticks - t); 1451c6d31b83SKonstantin Belousov if (td_ast_pending(td, TDA_SUSPEND)) { 1452b837daddSKonstantin Belousov vcpu_unlock(vcpu); 1453b837daddSKonstantin Belousov error = thread_check_susp(td, false); 14544d447b30SKonstantin Belousov if (error != 0) { 14554d447b30SKonstantin Belousov if (vcpu_halted) { 14564d447b30SKonstantin Belousov CPU_CLR_ATOMIC(vcpuid, 14574d447b30SKonstantin Belousov &vm->halted_cpus); 14584d447b30SKonstantin Belousov } 1459b837daddSKonstantin Belousov return (error); 14604d447b30SKonstantin Belousov } 1461b837daddSKonstantin Belousov vcpu_lock(vcpu); 1462b837daddSKonstantin Belousov } 1463f76fc5d4SNeel Natu } 1464e50ce2aaSNeel Natu 1465e50ce2aaSNeel Natu if (vcpu_halted) 1466e50ce2aaSNeel Natu CPU_CLR_ATOMIC(vcpuid, &vm->halted_cpus); 1467e50ce2aaSNeel Natu 1468f76fc5d4SNeel Natu vcpu_unlock(vcpu); 1469f76fc5d4SNeel Natu 1470e50ce2aaSNeel Natu if (vm_halted) 1471e50ce2aaSNeel Natu vm_suspend(vm, VM_SUSPEND_HALT); 1472e50ce2aaSNeel Natu 1473318224bbSNeel Natu return (0); 1474318224bbSNeel Natu } 1475318224bbSNeel Natu 1476318224bbSNeel Natu static int 1477becd9849SNeel Natu vm_handle_paging(struct vm *vm, int vcpuid, bool *retu) 1478318224bbSNeel Natu { 1479318224bbSNeel Natu int rv, ftype; 1480318224bbSNeel Natu struct vm_map *map; 1481318224bbSNeel Natu struct vcpu *vcpu; 1482318224bbSNeel Natu struct vm_exit *vme; 1483318224bbSNeel Natu 1484318224bbSNeel Natu vcpu = &vm->vcpu[vcpuid]; 1485318224bbSNeel Natu vme = &vcpu->exitinfo; 1486318224bbSNeel Natu 1487d087a399SNeel Natu KASSERT(vme->inst_length == 0, ("%s: invalid inst_length %d", 1488d087a399SNeel Natu __func__, vme->inst_length)); 1489d087a399SNeel Natu 1490318224bbSNeel Natu ftype = vme->u.paging.fault_type; 1491318224bbSNeel Natu KASSERT(ftype == VM_PROT_READ || 1492318224bbSNeel Natu ftype == VM_PROT_WRITE || ftype == VM_PROT_EXECUTE, 1493318224bbSNeel Natu ("vm_handle_paging: invalid fault_type %d", ftype)); 1494318224bbSNeel Natu 1495318224bbSNeel Natu if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) { 1496318224bbSNeel Natu rv = pmap_emulate_accessed_dirty(vmspace_pmap(vm->vmspace), 1497318224bbSNeel Natu vme->u.paging.gpa, ftype); 14989d8d8e3eSNeel Natu if (rv == 0) { 14999d8d8e3eSNeel Natu VCPU_CTR2(vm, vcpuid, "%s bit emulation for gpa %#lx", 15009d8d8e3eSNeel Natu ftype == VM_PROT_READ ? "accessed" : "dirty", 15019d8d8e3eSNeel Natu vme->u.paging.gpa); 1502318224bbSNeel Natu goto done; 1503318224bbSNeel Natu } 15049d8d8e3eSNeel Natu } 1505318224bbSNeel Natu 1506318224bbSNeel Natu map = &vm->vmspace->vm_map; 1507df08823dSKonstantin Belousov rv = vm_fault(map, vme->u.paging.gpa, ftype, VM_FAULT_NORMAL, NULL); 1508318224bbSNeel Natu 1509513c8d33SNeel Natu VCPU_CTR3(vm, vcpuid, "vm_handle_paging rv = %d, gpa = %#lx, " 1510513c8d33SNeel Natu "ftype = %d", rv, vme->u.paging.gpa, ftype); 1511318224bbSNeel Natu 1512318224bbSNeel Natu if (rv != KERN_SUCCESS) 1513318224bbSNeel Natu return (EFAULT); 1514318224bbSNeel Natu done: 1515318224bbSNeel Natu return (0); 1516318224bbSNeel Natu } 1517318224bbSNeel Natu 1518318224bbSNeel Natu static int 1519becd9849SNeel Natu vm_handle_inst_emul(struct vm *vm, int vcpuid, bool *retu) 1520318224bbSNeel Natu { 1521318224bbSNeel Natu struct vie *vie; 1522318224bbSNeel Natu struct vcpu *vcpu; 1523318224bbSNeel Natu struct vm_exit *vme; 1524e4f605eeSTycho Nightingale uint64_t gla, gpa, cs_base; 1525e813a873SNeel Natu struct vm_guest_paging *paging; 1526565bbb86SNeel Natu mem_region_read_t mread; 1527565bbb86SNeel Natu mem_region_write_t mwrite; 1528f7a9f178SNeel Natu enum vm_cpu_mode cpu_mode; 15291c73ea3eSNeel Natu int cs_d, error, fault; 1530318224bbSNeel Natu 1531318224bbSNeel Natu vcpu = &vm->vcpu[vcpuid]; 1532318224bbSNeel Natu vme = &vcpu->exitinfo; 1533318224bbSNeel Natu 15341c73ea3eSNeel Natu KASSERT(vme->inst_length == 0, ("%s: invalid inst_length %d", 15351c73ea3eSNeel Natu __func__, vme->inst_length)); 15361c73ea3eSNeel Natu 1537318224bbSNeel Natu gla = vme->u.inst_emul.gla; 1538318224bbSNeel Natu gpa = vme->u.inst_emul.gpa; 1539e4f605eeSTycho Nightingale cs_base = vme->u.inst_emul.cs_base; 1540f7a9f178SNeel Natu cs_d = vme->u.inst_emul.cs_d; 1541318224bbSNeel Natu vie = &vme->u.inst_emul.vie; 1542e813a873SNeel Natu paging = &vme->u.inst_emul.paging; 1543f7a9f178SNeel Natu cpu_mode = paging->cpu_mode; 1544318224bbSNeel Natu 15459d8d8e3eSNeel Natu VCPU_CTR1(vm, vcpuid, "inst_emul fault accessing gpa %#lx", gpa); 15469d8d8e3eSNeel Natu 1547318224bbSNeel Natu /* Fetch, decode and emulate the faulting instruction */ 1548c2a875f9SNeel Natu if (vie->num_valid == 0) { 1549e4f605eeSTycho Nightingale error = vmm_fetch_instruction(vm, vcpuid, paging, vme->rip + 15501c73ea3eSNeel Natu cs_base, VIE_INST_SIZE, vie, &fault); 1551c2a875f9SNeel Natu } else { 1552c2a875f9SNeel Natu /* 1553c2a875f9SNeel Natu * The instruction bytes have already been copied into 'vie' 1554c2a875f9SNeel Natu */ 15559c4d5478SNeel Natu error = fault = 0; 1556c2a875f9SNeel Natu } 15579c4d5478SNeel Natu if (error || fault) 15589c4d5478SNeel Natu return (error); 1559318224bbSNeel Natu 1560c07a0648SNeel Natu if (vmm_decode_instruction(vm, vcpuid, gla, cpu_mode, cs_d, vie) != 0) { 1561c07a0648SNeel Natu VCPU_CTR1(vm, vcpuid, "Error decoding instruction at %#lx", 1562c07a0648SNeel Natu vme->rip + cs_base); 1563c07a0648SNeel Natu *retu = true; /* dump instruction bytes in userspace */ 1564c07a0648SNeel Natu return (0); 1565c07a0648SNeel Natu } 1566318224bbSNeel Natu 1567a0b78f09SPeter Grehan /* 15681c73ea3eSNeel Natu * Update 'nextrip' based on the length of the emulated instruction. 1569a0b78f09SPeter Grehan */ 1570a0b78f09SPeter Grehan vme->inst_length = vie->num_processed; 1571d087a399SNeel Natu vcpu->nextrip += vie->num_processed; 15721c73ea3eSNeel Natu VCPU_CTR1(vm, vcpuid, "nextrip updated to %#lx after instruction " 15731c73ea3eSNeel Natu "decoding", vcpu->nextrip); 1574a0b78f09SPeter Grehan 157508e3ff32SNeel Natu /* return to userland unless this is an in-kernel emulated device */ 1576565bbb86SNeel Natu if (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE) { 1577565bbb86SNeel Natu mread = lapic_mmio_read; 1578565bbb86SNeel Natu mwrite = lapic_mmio_write; 1579565bbb86SNeel Natu } else if (gpa >= VIOAPIC_BASE && gpa < VIOAPIC_BASE + VIOAPIC_SIZE) { 1580565bbb86SNeel Natu mread = vioapic_mmio_read; 1581565bbb86SNeel Natu mwrite = vioapic_mmio_write; 158208e3ff32SNeel Natu } else if (gpa >= VHPET_BASE && gpa < VHPET_BASE + VHPET_SIZE) { 158308e3ff32SNeel Natu mread = vhpet_mmio_read; 158408e3ff32SNeel Natu mwrite = vhpet_mmio_write; 1585565bbb86SNeel Natu } else { 1586becd9849SNeel Natu *retu = true; 1587318224bbSNeel Natu return (0); 1588318224bbSNeel Natu } 1589318224bbSNeel Natu 1590d665d229SNeel Natu error = vmm_emulate_instruction(vm, vcpuid, gpa, vie, paging, 1591d665d229SNeel Natu mread, mwrite, retu); 1592318224bbSNeel Natu 1593318224bbSNeel Natu return (error); 1594318224bbSNeel Natu } 1595318224bbSNeel Natu 1596b15a09c0SNeel Natu static int 1597b15a09c0SNeel Natu vm_handle_suspend(struct vm *vm, int vcpuid, bool *retu) 1598b15a09c0SNeel Natu { 1599b837daddSKonstantin Belousov int error, i; 1600b15a09c0SNeel Natu struct vcpu *vcpu; 1601b837daddSKonstantin Belousov struct thread *td; 1602b15a09c0SNeel Natu 1603b837daddSKonstantin Belousov error = 0; 1604b15a09c0SNeel Natu vcpu = &vm->vcpu[vcpuid]; 1605b837daddSKonstantin Belousov td = curthread; 1606b15a09c0SNeel Natu 1607b15a09c0SNeel Natu CPU_SET_ATOMIC(vcpuid, &vm->suspended_cpus); 1608b15a09c0SNeel Natu 1609b15a09c0SNeel Natu /* 1610b15a09c0SNeel Natu * Wait until all 'active_cpus' have suspended themselves. 1611b15a09c0SNeel Natu * 1612b15a09c0SNeel Natu * Since a VM may be suspended at any time including when one or 1613b15a09c0SNeel Natu * more vcpus are doing a rendezvous we need to call the rendezvous 1614b15a09c0SNeel Natu * handler while we are waiting to prevent a deadlock. 1615b15a09c0SNeel Natu */ 1616b15a09c0SNeel Natu vcpu_lock(vcpu); 1617b837daddSKonstantin Belousov while (error == 0) { 1618b15a09c0SNeel Natu if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) { 1619b15a09c0SNeel Natu VCPU_CTR0(vm, vcpuid, "All vcpus suspended"); 1620b15a09c0SNeel Natu break; 1621b15a09c0SNeel Natu } 1622b15a09c0SNeel Natu 1623b15a09c0SNeel Natu if (vm->rendezvous_func == NULL) { 1624b15a09c0SNeel Natu VCPU_CTR0(vm, vcpuid, "Sleeping during suspend"); 1625248e6799SNeel Natu vcpu_require_state_locked(vm, vcpuid, VCPU_SLEEPING); 1626b15a09c0SNeel Natu msleep_spin(vcpu, &vcpu->mtx, "vmsusp", hz); 1627248e6799SNeel Natu vcpu_require_state_locked(vm, vcpuid, VCPU_FROZEN); 1628c6d31b83SKonstantin Belousov if (td_ast_pending(td, TDA_SUSPEND)) { 1629b837daddSKonstantin Belousov vcpu_unlock(vcpu); 1630b837daddSKonstantin Belousov error = thread_check_susp(td, false); 1631b837daddSKonstantin Belousov vcpu_lock(vcpu); 1632b837daddSKonstantin Belousov } 1633b15a09c0SNeel Natu } else { 1634b15a09c0SNeel Natu VCPU_CTR0(vm, vcpuid, "Rendezvous during suspend"); 1635b15a09c0SNeel Natu vcpu_unlock(vcpu); 1636b837daddSKonstantin Belousov error = vm_handle_rendezvous(vm, vcpuid); 1637b15a09c0SNeel Natu vcpu_lock(vcpu); 1638b15a09c0SNeel Natu } 1639b15a09c0SNeel Natu } 1640b15a09c0SNeel Natu vcpu_unlock(vcpu); 1641b15a09c0SNeel Natu 1642b15a09c0SNeel Natu /* 1643b15a09c0SNeel Natu * Wakeup the other sleeping vcpus and return to userspace. 1644b15a09c0SNeel Natu */ 1645a488c9c9SRodney W. Grimes for (i = 0; i < vm->maxcpus; i++) { 1646b15a09c0SNeel Natu if (CPU_ISSET(i, &vm->suspended_cpus)) { 1647b15a09c0SNeel Natu vcpu_notify_event(vm, i, false); 1648b15a09c0SNeel Natu } 1649b15a09c0SNeel Natu } 1650b15a09c0SNeel Natu 1651b15a09c0SNeel Natu *retu = true; 1652b837daddSKonstantin Belousov return (error); 1653b15a09c0SNeel Natu } 1654b15a09c0SNeel Natu 1655248e6799SNeel Natu static int 1656248e6799SNeel Natu vm_handle_reqidle(struct vm *vm, int vcpuid, bool *retu) 1657248e6799SNeel Natu { 1658248e6799SNeel Natu struct vcpu *vcpu = &vm->vcpu[vcpuid]; 1659248e6799SNeel Natu 1660248e6799SNeel Natu vcpu_lock(vcpu); 1661248e6799SNeel Natu KASSERT(vcpu->reqidle, ("invalid vcpu reqidle %d", vcpu->reqidle)); 1662248e6799SNeel Natu vcpu->reqidle = 0; 1663248e6799SNeel Natu vcpu_unlock(vcpu); 1664248e6799SNeel Natu *retu = true; 1665248e6799SNeel Natu return (0); 1666248e6799SNeel Natu } 1667248e6799SNeel Natu 1668b15a09c0SNeel Natu int 1669f0fdcfe2SNeel Natu vm_suspend(struct vm *vm, enum vm_suspend_how how) 1670b15a09c0SNeel Natu { 1671f0fdcfe2SNeel Natu int i; 1672b15a09c0SNeel Natu 1673f0fdcfe2SNeel Natu if (how <= VM_SUSPEND_NONE || how >= VM_SUSPEND_LAST) 1674f0fdcfe2SNeel Natu return (EINVAL); 1675f0fdcfe2SNeel Natu 1676f0fdcfe2SNeel Natu if (atomic_cmpset_int(&vm->suspend, 0, how) == 0) { 1677f0fdcfe2SNeel Natu VM_CTR2(vm, "virtual machine already suspended %d/%d", 1678f0fdcfe2SNeel Natu vm->suspend, how); 1679b15a09c0SNeel Natu return (EALREADY); 1680b15a09c0SNeel Natu } 1681f0fdcfe2SNeel Natu 1682f0fdcfe2SNeel Natu VM_CTR1(vm, "virtual machine successfully suspended %d", how); 1683f0fdcfe2SNeel Natu 1684f0fdcfe2SNeel Natu /* 1685f0fdcfe2SNeel Natu * Notify all active vcpus that they are now suspended. 1686f0fdcfe2SNeel Natu */ 1687a488c9c9SRodney W. Grimes for (i = 0; i < vm->maxcpus; i++) { 1688f0fdcfe2SNeel Natu if (CPU_ISSET(i, &vm->active_cpus)) 1689f0fdcfe2SNeel Natu vcpu_notify_event(vm, i, false); 1690f0fdcfe2SNeel Natu } 1691f0fdcfe2SNeel Natu 1692f0fdcfe2SNeel Natu return (0); 1693f0fdcfe2SNeel Natu } 1694f0fdcfe2SNeel Natu 1695f0fdcfe2SNeel Natu void 1696f0fdcfe2SNeel Natu vm_exit_suspended(struct vm *vm, int vcpuid, uint64_t rip) 1697f0fdcfe2SNeel Natu { 1698f0fdcfe2SNeel Natu struct vm_exit *vmexit; 1699f0fdcfe2SNeel Natu 1700f0fdcfe2SNeel Natu KASSERT(vm->suspend > VM_SUSPEND_NONE && vm->suspend < VM_SUSPEND_LAST, 1701f0fdcfe2SNeel Natu ("vm_exit_suspended: invalid suspend type %d", vm->suspend)); 1702f0fdcfe2SNeel Natu 1703f0fdcfe2SNeel Natu vmexit = vm_exitinfo(vm, vcpuid); 1704f0fdcfe2SNeel Natu vmexit->rip = rip; 1705f0fdcfe2SNeel Natu vmexit->inst_length = 0; 1706f0fdcfe2SNeel Natu vmexit->exitcode = VM_EXITCODE_SUSPENDED; 1707f0fdcfe2SNeel Natu vmexit->u.suspended.how = vm->suspend; 1708b15a09c0SNeel Natu } 1709b15a09c0SNeel Natu 171040487465SNeel Natu void 1711fc276d92SJohn Baldwin vm_exit_debug(struct vm *vm, int vcpuid, uint64_t rip) 1712fc276d92SJohn Baldwin { 1713fc276d92SJohn Baldwin struct vm_exit *vmexit; 1714fc276d92SJohn Baldwin 1715fc276d92SJohn Baldwin vmexit = vm_exitinfo(vm, vcpuid); 1716fc276d92SJohn Baldwin vmexit->rip = rip; 1717fc276d92SJohn Baldwin vmexit->inst_length = 0; 1718fc276d92SJohn Baldwin vmexit->exitcode = VM_EXITCODE_DEBUG; 1719fc276d92SJohn Baldwin } 1720fc276d92SJohn Baldwin 1721fc276d92SJohn Baldwin void 172240487465SNeel Natu vm_exit_rendezvous(struct vm *vm, int vcpuid, uint64_t rip) 172340487465SNeel Natu { 172440487465SNeel Natu struct vm_exit *vmexit; 172540487465SNeel Natu 172640487465SNeel Natu KASSERT(vm->rendezvous_func != NULL, ("rendezvous not in progress")); 172740487465SNeel Natu 172840487465SNeel Natu vmexit = vm_exitinfo(vm, vcpuid); 172940487465SNeel Natu vmexit->rip = rip; 173040487465SNeel Natu vmexit->inst_length = 0; 173140487465SNeel Natu vmexit->exitcode = VM_EXITCODE_RENDEZVOUS; 17323dc3d32aSJohn Baldwin vmm_stat_incr(vm_vcpu(vm, vcpuid), VMEXIT_RENDEZVOUS, 1); 173340487465SNeel Natu } 173440487465SNeel Natu 173540487465SNeel Natu void 1736248e6799SNeel Natu vm_exit_reqidle(struct vm *vm, int vcpuid, uint64_t rip) 1737248e6799SNeel Natu { 1738248e6799SNeel Natu struct vm_exit *vmexit; 1739248e6799SNeel Natu 1740248e6799SNeel Natu vmexit = vm_exitinfo(vm, vcpuid); 1741248e6799SNeel Natu vmexit->rip = rip; 1742248e6799SNeel Natu vmexit->inst_length = 0; 1743248e6799SNeel Natu vmexit->exitcode = VM_EXITCODE_REQIDLE; 17443dc3d32aSJohn Baldwin vmm_stat_incr(vm_vcpu(vm, vcpuid), VMEXIT_REQIDLE, 1); 1745248e6799SNeel Natu } 1746248e6799SNeel Natu 1747248e6799SNeel Natu void 174840487465SNeel Natu vm_exit_astpending(struct vm *vm, int vcpuid, uint64_t rip) 174940487465SNeel Natu { 175040487465SNeel Natu struct vm_exit *vmexit; 175140487465SNeel Natu 175240487465SNeel Natu vmexit = vm_exitinfo(vm, vcpuid); 175340487465SNeel Natu vmexit->rip = rip; 175440487465SNeel Natu vmexit->inst_length = 0; 175540487465SNeel Natu vmexit->exitcode = VM_EXITCODE_BOGUS; 17563dc3d32aSJohn Baldwin vmm_stat_incr(vm_vcpu(vm, vcpuid), VMEXIT_ASTPENDING, 1); 175740487465SNeel Natu } 175840487465SNeel Natu 1759318224bbSNeel Natu int 1760318224bbSNeel Natu vm_run(struct vm *vm, struct vm_run *vmrun) 1761318224bbSNeel Natu { 1762248e6799SNeel Natu struct vm_eventinfo evinfo; 1763318224bbSNeel Natu int error, vcpuid; 1764318224bbSNeel Natu struct vcpu *vcpu; 1765318224bbSNeel Natu struct pcb *pcb; 1766d087a399SNeel Natu uint64_t tscval; 1767318224bbSNeel Natu struct vm_exit *vme; 1768becd9849SNeel Natu bool retu, intr_disabled; 1769318224bbSNeel Natu pmap_t pmap; 1770318224bbSNeel Natu 1771318224bbSNeel Natu vcpuid = vmrun->cpuid; 1772318224bbSNeel Natu 1773a488c9c9SRodney W. Grimes if (vcpuid < 0 || vcpuid >= vm->maxcpus) 1774318224bbSNeel Natu return (EINVAL); 1775318224bbSNeel Natu 177695ebc360SNeel Natu if (!CPU_ISSET(vcpuid, &vm->active_cpus)) 177795ebc360SNeel Natu return (EINVAL); 177895ebc360SNeel Natu 177995ebc360SNeel Natu if (CPU_ISSET(vcpuid, &vm->suspended_cpus)) 178095ebc360SNeel Natu return (EINVAL); 178195ebc360SNeel Natu 1782318224bbSNeel Natu pmap = vmspace_pmap(vm->vmspace); 1783318224bbSNeel Natu vcpu = &vm->vcpu[vcpuid]; 1784318224bbSNeel Natu vme = &vcpu->exitinfo; 1785248e6799SNeel Natu evinfo.rptr = &vm->rendezvous_func; 1786248e6799SNeel Natu evinfo.sptr = &vm->suspend; 1787248e6799SNeel Natu evinfo.iptr = &vcpu->reqidle; 1788318224bbSNeel Natu restart: 1789318224bbSNeel Natu critical_enter(); 1790318224bbSNeel Natu 1791318224bbSNeel Natu KASSERT(!CPU_ISSET(curcpu, &pmap->pm_active), 1792318224bbSNeel Natu ("vm_run: absurd pm_active")); 1793318224bbSNeel Natu 1794318224bbSNeel Natu tscval = rdtsc(); 1795318224bbSNeel Natu 1796318224bbSNeel Natu pcb = PCPU_GET(curpcb); 1797318224bbSNeel Natu set_pcb_flags(pcb, PCB_FULL_IRET); 1798318224bbSNeel Natu 1799318224bbSNeel Natu restore_guest_fpustate(vcpu); 1800318224bbSNeel Natu 1801318224bbSNeel Natu vcpu_require_state(vm, vcpuid, VCPU_RUNNING); 1802869c8d19SJohn Baldwin error = vmmops_run(vcpu->cookie, vcpu->nextrip, pmap, &evinfo); 1803318224bbSNeel Natu vcpu_require_state(vm, vcpuid, VCPU_FROZEN); 1804318224bbSNeel Natu 1805318224bbSNeel Natu save_guest_fpustate(vcpu); 1806318224bbSNeel Natu 18073dc3d32aSJohn Baldwin vmm_stat_incr(vcpu, VCPU_TOTAL_RUNTIME, rdtsc() - tscval); 1808318224bbSNeel Natu 1809318224bbSNeel Natu critical_exit(); 1810318224bbSNeel Natu 1811318224bbSNeel Natu if (error == 0) { 1812becd9849SNeel Natu retu = false; 1813d087a399SNeel Natu vcpu->nextrip = vme->rip + vme->inst_length; 1814318224bbSNeel Natu switch (vme->exitcode) { 1815248e6799SNeel Natu case VM_EXITCODE_REQIDLE: 1816248e6799SNeel Natu error = vm_handle_reqidle(vm, vcpuid, &retu); 1817248e6799SNeel Natu break; 1818b15a09c0SNeel Natu case VM_EXITCODE_SUSPENDED: 1819b15a09c0SNeel Natu error = vm_handle_suspend(vm, vcpuid, &retu); 1820b15a09c0SNeel Natu break; 182130b94db8SNeel Natu case VM_EXITCODE_IOAPIC_EOI: 182230b94db8SNeel Natu vioapic_process_eoi(vm, vcpuid, 182330b94db8SNeel Natu vme->u.ioapic_eoi.vector); 182430b94db8SNeel Natu break; 18255b8a8cd1SNeel Natu case VM_EXITCODE_RENDEZVOUS: 1826b837daddSKonstantin Belousov error = vm_handle_rendezvous(vm, vcpuid); 18275b8a8cd1SNeel Natu break; 1828318224bbSNeel Natu case VM_EXITCODE_HLT: 1829becd9849SNeel Natu intr_disabled = ((vme->u.hlt.rflags & PSL_I) == 0); 18301c052192SNeel Natu error = vm_handle_hlt(vm, vcpuid, intr_disabled, &retu); 1831318224bbSNeel Natu break; 1832318224bbSNeel Natu case VM_EXITCODE_PAGING: 1833318224bbSNeel Natu error = vm_handle_paging(vm, vcpuid, &retu); 1834318224bbSNeel Natu break; 1835318224bbSNeel Natu case VM_EXITCODE_INST_EMUL: 1836318224bbSNeel Natu error = vm_handle_inst_emul(vm, vcpuid, &retu); 1837318224bbSNeel Natu break; 1838d17b5104SNeel Natu case VM_EXITCODE_INOUT: 1839d17b5104SNeel Natu case VM_EXITCODE_INOUT_STR: 1840d17b5104SNeel Natu error = vm_handle_inout(vm, vcpuid, vme, &retu); 1841d17b5104SNeel Natu break; 184265145c7fSNeel Natu case VM_EXITCODE_MONITOR: 184365145c7fSNeel Natu case VM_EXITCODE_MWAIT: 184427d26457SAndrew Turner case VM_EXITCODE_VMINSN: 184565145c7fSNeel Natu vm_inject_ud(vm, vcpuid); 184665145c7fSNeel Natu break; 1847318224bbSNeel Natu default: 1848becd9849SNeel Natu retu = true; /* handled in userland */ 1849318224bbSNeel Natu break; 1850318224bbSNeel Natu } 1851318224bbSNeel Natu } 1852318224bbSNeel Natu 18530bda8d3eSCorvin Köhne /* 18540bda8d3eSCorvin Köhne * VM_EXITCODE_INST_EMUL could access the apic which could transform the 18550bda8d3eSCorvin Köhne * exit code into VM_EXITCODE_IPI. 18560bda8d3eSCorvin Köhne */ 18570bda8d3eSCorvin Köhne if (error == 0 && vme->exitcode == VM_EXITCODE_IPI) { 18580bda8d3eSCorvin Köhne retu = false; 18590bda8d3eSCorvin Köhne error = vm_handle_ipi(vm, vcpuid, vme, &retu); 18600bda8d3eSCorvin Köhne } 18610bda8d3eSCorvin Köhne 1862d087a399SNeel Natu if (error == 0 && retu == false) 1863f76fc5d4SNeel Natu goto restart; 1864f76fc5d4SNeel Natu 18653dc3d32aSJohn Baldwin vmm_stat_incr(vcpu, VMEXIT_USERSPACE, 1); 1866248e6799SNeel Natu VCPU_CTR2(vm, vcpuid, "retu %d/%d", error, vme->exitcode); 1867248e6799SNeel Natu 1868318224bbSNeel Natu /* copy the exit information */ 1869318224bbSNeel Natu bcopy(vme, &vmrun->vm_exit, sizeof(struct vm_exit)); 1870366f6083SPeter Grehan return (error); 1871366f6083SPeter Grehan } 1872366f6083SPeter Grehan 1873366f6083SPeter Grehan int 1874c9c75df4SNeel Natu vm_restart_instruction(void *arg, int vcpuid) 1875c9c75df4SNeel Natu { 1876d087a399SNeel Natu struct vm *vm; 1877c9c75df4SNeel Natu struct vcpu *vcpu; 1878d087a399SNeel Natu enum vcpu_state state; 1879d087a399SNeel Natu uint64_t rip; 188073505a10SRobert Wing int error __diagused; 1881c9c75df4SNeel Natu 1882d087a399SNeel Natu vm = arg; 1883a488c9c9SRodney W. Grimes if (vcpuid < 0 || vcpuid >= vm->maxcpus) 1884c9c75df4SNeel Natu return (EINVAL); 1885c9c75df4SNeel Natu 1886c9c75df4SNeel Natu vcpu = &vm->vcpu[vcpuid]; 1887d087a399SNeel Natu state = vcpu_get_state(vm, vcpuid, NULL); 1888d087a399SNeel Natu if (state == VCPU_RUNNING) { 1889d087a399SNeel Natu /* 1890d087a399SNeel Natu * When a vcpu is "running" the next instruction is determined 1891d087a399SNeel Natu * by adding 'rip' and 'inst_length' in the vcpu's 'exitinfo'. 1892d087a399SNeel Natu * Thus setting 'inst_length' to zero will cause the current 1893d087a399SNeel Natu * instruction to be restarted. 1894d087a399SNeel Natu */ 1895c9c75df4SNeel Natu vcpu->exitinfo.inst_length = 0; 1896d087a399SNeel Natu VCPU_CTR1(vm, vcpuid, "restarting instruction at %#lx by " 1897d087a399SNeel Natu "setting inst_length to zero", vcpu->exitinfo.rip); 1898d087a399SNeel Natu } else if (state == VCPU_FROZEN) { 1899d087a399SNeel Natu /* 1900d087a399SNeel Natu * When a vcpu is "frozen" it is outside the critical section 190115add60dSPeter Grehan * around vmmops_run() and 'nextrip' points to the next 190215add60dSPeter Grehan * instruction. Thus instruction restart is achieved by setting 190315add60dSPeter Grehan * 'nextrip' to the vcpu's %rip. 1904d087a399SNeel Natu */ 1905d087a399SNeel Natu error = vm_get_register(vm, vcpuid, VM_REG_GUEST_RIP, &rip); 1906d087a399SNeel Natu KASSERT(!error, ("%s: error %d getting rip", __func__, error)); 1907d087a399SNeel Natu VCPU_CTR2(vm, vcpuid, "restarting instruction by updating " 1908d087a399SNeel Natu "nextrip from %#lx to %#lx", vcpu->nextrip, rip); 1909d087a399SNeel Natu vcpu->nextrip = rip; 1910d087a399SNeel Natu } else { 1911d087a399SNeel Natu panic("%s: invalid state %d", __func__, state); 1912d087a399SNeel Natu } 1913c9c75df4SNeel Natu return (0); 1914c9c75df4SNeel Natu } 1915c9c75df4SNeel Natu 1916c9c75df4SNeel Natu int 1917091d4532SNeel Natu vm_exit_intinfo(struct vm *vm, int vcpuid, uint64_t info) 1918091d4532SNeel Natu { 1919091d4532SNeel Natu struct vcpu *vcpu; 1920091d4532SNeel Natu int type, vector; 1921091d4532SNeel Natu 1922a488c9c9SRodney W. Grimes if (vcpuid < 0 || vcpuid >= vm->maxcpus) 1923091d4532SNeel Natu return (EINVAL); 1924091d4532SNeel Natu 1925091d4532SNeel Natu vcpu = &vm->vcpu[vcpuid]; 1926091d4532SNeel Natu 1927091d4532SNeel Natu if (info & VM_INTINFO_VALID) { 1928091d4532SNeel Natu type = info & VM_INTINFO_TYPE; 1929091d4532SNeel Natu vector = info & 0xff; 1930091d4532SNeel Natu if (type == VM_INTINFO_NMI && vector != IDT_NMI) 1931091d4532SNeel Natu return (EINVAL); 1932091d4532SNeel Natu if (type == VM_INTINFO_HWEXCEPTION && vector >= 32) 1933091d4532SNeel Natu return (EINVAL); 1934091d4532SNeel Natu if (info & VM_INTINFO_RSVD) 1935091d4532SNeel Natu return (EINVAL); 1936091d4532SNeel Natu } else { 1937091d4532SNeel Natu info = 0; 1938091d4532SNeel Natu } 1939091d4532SNeel Natu VCPU_CTR2(vm, vcpuid, "%s: info1(%#lx)", __func__, info); 1940091d4532SNeel Natu vcpu->exitintinfo = info; 1941091d4532SNeel Natu return (0); 1942091d4532SNeel Natu } 1943091d4532SNeel Natu 1944091d4532SNeel Natu enum exc_class { 1945091d4532SNeel Natu EXC_BENIGN, 1946091d4532SNeel Natu EXC_CONTRIBUTORY, 1947091d4532SNeel Natu EXC_PAGEFAULT 1948091d4532SNeel Natu }; 1949091d4532SNeel Natu 1950091d4532SNeel Natu #define IDT_VE 20 /* Virtualization Exception (Intel specific) */ 1951091d4532SNeel Natu 1952091d4532SNeel Natu static enum exc_class 1953091d4532SNeel Natu exception_class(uint64_t info) 1954091d4532SNeel Natu { 1955091d4532SNeel Natu int type, vector; 1956091d4532SNeel Natu 1957091d4532SNeel Natu KASSERT(info & VM_INTINFO_VALID, ("intinfo must be valid: %#lx", info)); 1958091d4532SNeel Natu type = info & VM_INTINFO_TYPE; 1959091d4532SNeel Natu vector = info & 0xff; 1960091d4532SNeel Natu 1961091d4532SNeel Natu /* Table 6-4, "Interrupt and Exception Classes", Intel SDM, Vol 3 */ 1962091d4532SNeel Natu switch (type) { 1963091d4532SNeel Natu case VM_INTINFO_HWINTR: 1964091d4532SNeel Natu case VM_INTINFO_SWINTR: 1965091d4532SNeel Natu case VM_INTINFO_NMI: 1966091d4532SNeel Natu return (EXC_BENIGN); 1967091d4532SNeel Natu default: 1968091d4532SNeel Natu /* 1969091d4532SNeel Natu * Hardware exception. 1970091d4532SNeel Natu * 1971091d4532SNeel Natu * SVM and VT-x use identical type values to represent NMI, 1972091d4532SNeel Natu * hardware interrupt and software interrupt. 1973091d4532SNeel Natu * 1974091d4532SNeel Natu * SVM uses type '3' for all exceptions. VT-x uses type '3' 1975091d4532SNeel Natu * for exceptions except #BP and #OF. #BP and #OF use a type 1976091d4532SNeel Natu * value of '5' or '6'. Therefore we don't check for explicit 1977091d4532SNeel Natu * values of 'type' to classify 'intinfo' into a hardware 1978091d4532SNeel Natu * exception. 1979091d4532SNeel Natu */ 1980091d4532SNeel Natu break; 1981091d4532SNeel Natu } 1982091d4532SNeel Natu 1983091d4532SNeel Natu switch (vector) { 1984091d4532SNeel Natu case IDT_PF: 1985091d4532SNeel Natu case IDT_VE: 1986091d4532SNeel Natu return (EXC_PAGEFAULT); 1987091d4532SNeel Natu case IDT_DE: 1988091d4532SNeel Natu case IDT_TS: 1989091d4532SNeel Natu case IDT_NP: 1990091d4532SNeel Natu case IDT_SS: 1991091d4532SNeel Natu case IDT_GP: 1992091d4532SNeel Natu return (EXC_CONTRIBUTORY); 1993091d4532SNeel Natu default: 1994091d4532SNeel Natu return (EXC_BENIGN); 1995091d4532SNeel Natu } 1996091d4532SNeel Natu } 1997091d4532SNeel Natu 1998091d4532SNeel Natu static int 1999091d4532SNeel Natu nested_fault(struct vm *vm, int vcpuid, uint64_t info1, uint64_t info2, 2000091d4532SNeel Natu uint64_t *retinfo) 2001091d4532SNeel Natu { 2002091d4532SNeel Natu enum exc_class exc1, exc2; 2003091d4532SNeel Natu int type1, vector1; 2004091d4532SNeel Natu 2005091d4532SNeel Natu KASSERT(info1 & VM_INTINFO_VALID, ("info1 %#lx is not valid", info1)); 2006091d4532SNeel Natu KASSERT(info2 & VM_INTINFO_VALID, ("info2 %#lx is not valid", info2)); 2007091d4532SNeel Natu 2008091d4532SNeel Natu /* 2009091d4532SNeel Natu * If an exception occurs while attempting to call the double-fault 2010091d4532SNeel Natu * handler the processor enters shutdown mode (aka triple fault). 2011091d4532SNeel Natu */ 2012091d4532SNeel Natu type1 = info1 & VM_INTINFO_TYPE; 2013091d4532SNeel Natu vector1 = info1 & 0xff; 2014091d4532SNeel Natu if (type1 == VM_INTINFO_HWEXCEPTION && vector1 == IDT_DF) { 2015091d4532SNeel Natu VCPU_CTR2(vm, vcpuid, "triple fault: info1(%#lx), info2(%#lx)", 2016091d4532SNeel Natu info1, info2); 2017091d4532SNeel Natu vm_suspend(vm, VM_SUSPEND_TRIPLEFAULT); 2018091d4532SNeel Natu *retinfo = 0; 2019091d4532SNeel Natu return (0); 2020091d4532SNeel Natu } 2021091d4532SNeel Natu 2022091d4532SNeel Natu /* 2023091d4532SNeel Natu * Table 6-5 "Conditions for Generating a Double Fault", Intel SDM, Vol3 2024091d4532SNeel Natu */ 2025091d4532SNeel Natu exc1 = exception_class(info1); 2026091d4532SNeel Natu exc2 = exception_class(info2); 2027091d4532SNeel Natu if ((exc1 == EXC_CONTRIBUTORY && exc2 == EXC_CONTRIBUTORY) || 2028091d4532SNeel Natu (exc1 == EXC_PAGEFAULT && exc2 != EXC_BENIGN)) { 2029091d4532SNeel Natu /* Convert nested fault into a double fault. */ 2030091d4532SNeel Natu *retinfo = IDT_DF; 2031091d4532SNeel Natu *retinfo |= VM_INTINFO_VALID | VM_INTINFO_HWEXCEPTION; 2032091d4532SNeel Natu *retinfo |= VM_INTINFO_DEL_ERRCODE; 2033091d4532SNeel Natu } else { 2034091d4532SNeel Natu /* Handle exceptions serially */ 2035091d4532SNeel Natu *retinfo = info2; 2036091d4532SNeel Natu } 2037091d4532SNeel Natu return (1); 2038091d4532SNeel Natu } 2039091d4532SNeel Natu 2040091d4532SNeel Natu static uint64_t 2041091d4532SNeel Natu vcpu_exception_intinfo(struct vcpu *vcpu) 2042091d4532SNeel Natu { 2043091d4532SNeel Natu uint64_t info = 0; 2044091d4532SNeel Natu 2045091d4532SNeel Natu if (vcpu->exception_pending) { 2046c9c75df4SNeel Natu info = vcpu->exc_vector & 0xff; 2047091d4532SNeel Natu info |= VM_INTINFO_VALID | VM_INTINFO_HWEXCEPTION; 2048c9c75df4SNeel Natu if (vcpu->exc_errcode_valid) { 2049091d4532SNeel Natu info |= VM_INTINFO_DEL_ERRCODE; 2050c9c75df4SNeel Natu info |= (uint64_t)vcpu->exc_errcode << 32; 2051091d4532SNeel Natu } 2052091d4532SNeel Natu } 2053091d4532SNeel Natu return (info); 2054091d4532SNeel Natu } 2055091d4532SNeel Natu 2056091d4532SNeel Natu int 2057091d4532SNeel Natu vm_entry_intinfo(struct vm *vm, int vcpuid, uint64_t *retinfo) 2058091d4532SNeel Natu { 2059091d4532SNeel Natu struct vcpu *vcpu; 2060091d4532SNeel Natu uint64_t info1, info2; 2061091d4532SNeel Natu int valid; 2062091d4532SNeel Natu 2063a488c9c9SRodney W. Grimes KASSERT(vcpuid >= 0 && 2064a488c9c9SRodney W. Grimes vcpuid < vm->maxcpus, ("invalid vcpu %d", vcpuid)); 2065091d4532SNeel Natu 2066091d4532SNeel Natu vcpu = &vm->vcpu[vcpuid]; 2067091d4532SNeel Natu 2068091d4532SNeel Natu info1 = vcpu->exitintinfo; 2069091d4532SNeel Natu vcpu->exitintinfo = 0; 2070091d4532SNeel Natu 2071091d4532SNeel Natu info2 = 0; 2072091d4532SNeel Natu if (vcpu->exception_pending) { 2073091d4532SNeel Natu info2 = vcpu_exception_intinfo(vcpu); 2074091d4532SNeel Natu vcpu->exception_pending = 0; 2075091d4532SNeel Natu VCPU_CTR2(vm, vcpuid, "Exception %d delivered: %#lx", 2076c9c75df4SNeel Natu vcpu->exc_vector, info2); 2077091d4532SNeel Natu } 2078091d4532SNeel Natu 2079091d4532SNeel Natu if ((info1 & VM_INTINFO_VALID) && (info2 & VM_INTINFO_VALID)) { 2080091d4532SNeel Natu valid = nested_fault(vm, vcpuid, info1, info2, retinfo); 2081091d4532SNeel Natu } else if (info1 & VM_INTINFO_VALID) { 2082091d4532SNeel Natu *retinfo = info1; 2083091d4532SNeel Natu valid = 1; 2084091d4532SNeel Natu } else if (info2 & VM_INTINFO_VALID) { 2085091d4532SNeel Natu *retinfo = info2; 2086091d4532SNeel Natu valid = 1; 2087091d4532SNeel Natu } else { 2088091d4532SNeel Natu valid = 0; 2089091d4532SNeel Natu } 2090091d4532SNeel Natu 2091091d4532SNeel Natu if (valid) { 2092091d4532SNeel Natu VCPU_CTR4(vm, vcpuid, "%s: info1(%#lx), info2(%#lx), " 2093091d4532SNeel Natu "retinfo(%#lx)", __func__, info1, info2, *retinfo); 2094091d4532SNeel Natu } 2095091d4532SNeel Natu 2096091d4532SNeel Natu return (valid); 2097091d4532SNeel Natu } 2098091d4532SNeel Natu 2099091d4532SNeel Natu int 2100091d4532SNeel Natu vm_get_intinfo(struct vm *vm, int vcpuid, uint64_t *info1, uint64_t *info2) 2101091d4532SNeel Natu { 2102091d4532SNeel Natu struct vcpu *vcpu; 2103091d4532SNeel Natu 2104a488c9c9SRodney W. Grimes if (vcpuid < 0 || vcpuid >= vm->maxcpus) 2105091d4532SNeel Natu return (EINVAL); 2106091d4532SNeel Natu 2107091d4532SNeel Natu vcpu = &vm->vcpu[vcpuid]; 2108091d4532SNeel Natu *info1 = vcpu->exitintinfo; 2109091d4532SNeel Natu *info2 = vcpu_exception_intinfo(vcpu); 2110091d4532SNeel Natu return (0); 2111091d4532SNeel Natu } 2112091d4532SNeel Natu 2113091d4532SNeel Natu int 2114c9c75df4SNeel Natu vm_inject_exception(struct vm *vm, int vcpuid, int vector, int errcode_valid, 2115c9c75df4SNeel Natu uint32_t errcode, int restart_instruction) 2116366f6083SPeter Grehan { 2117dc506506SNeel Natu struct vcpu *vcpu; 211847b9935dSNeel Natu uint64_t regval; 211973505a10SRobert Wing int error __diagused; 2120dc506506SNeel Natu 2121a488c9c9SRodney W. Grimes if (vcpuid < 0 || vcpuid >= vm->maxcpus) 2122366f6083SPeter Grehan return (EINVAL); 2123366f6083SPeter Grehan 2124c9c75df4SNeel Natu if (vector < 0 || vector >= 32) 2125366f6083SPeter Grehan return (EINVAL); 2126366f6083SPeter Grehan 2127091d4532SNeel Natu /* 2128091d4532SNeel Natu * A double fault exception should never be injected directly into 2129091d4532SNeel Natu * the guest. It is a derived exception that results from specific 2130091d4532SNeel Natu * combinations of nested faults. 2131091d4532SNeel Natu */ 2132c9c75df4SNeel Natu if (vector == IDT_DF) 2133091d4532SNeel Natu return (EINVAL); 2134091d4532SNeel Natu 2135dc506506SNeel Natu vcpu = &vm->vcpu[vcpuid]; 2136366f6083SPeter Grehan 2137dc506506SNeel Natu if (vcpu->exception_pending) { 2138dc506506SNeel Natu VCPU_CTR2(vm, vcpuid, "Unable to inject exception %d due to " 2139c9c75df4SNeel Natu "pending exception %d", vector, vcpu->exc_vector); 2140dc506506SNeel Natu return (EBUSY); 2141dc506506SNeel Natu } 2142dc506506SNeel Natu 214347b9935dSNeel Natu if (errcode_valid) { 214447b9935dSNeel Natu /* 214547b9935dSNeel Natu * Exceptions don't deliver an error code in real mode. 214647b9935dSNeel Natu */ 214747b9935dSNeel Natu error = vm_get_register(vm, vcpuid, VM_REG_GUEST_CR0, ®val); 214847b9935dSNeel Natu KASSERT(!error, ("%s: error %d getting CR0", __func__, error)); 214947b9935dSNeel Natu if (!(regval & CR0_PE)) 215047b9935dSNeel Natu errcode_valid = 0; 215147b9935dSNeel Natu } 215247b9935dSNeel Natu 21532ce12423SNeel Natu /* 21542ce12423SNeel Natu * From section 26.6.1 "Interruptibility State" in Intel SDM: 21552ce12423SNeel Natu * 21562ce12423SNeel Natu * Event blocking by "STI" or "MOV SS" is cleared after guest executes 21572ce12423SNeel Natu * one instruction or incurs an exception. 21582ce12423SNeel Natu */ 21592ce12423SNeel Natu error = vm_set_register(vm, vcpuid, VM_REG_GUEST_INTR_SHADOW, 0); 21602ce12423SNeel Natu KASSERT(error == 0, ("%s: error %d clearing interrupt shadow", 21612ce12423SNeel Natu __func__, error)); 21622ce12423SNeel Natu 2163c9c75df4SNeel Natu if (restart_instruction) 2164c9c75df4SNeel Natu vm_restart_instruction(vm, vcpuid); 2165c9c75df4SNeel Natu 2166dc506506SNeel Natu vcpu->exception_pending = 1; 2167c9c75df4SNeel Natu vcpu->exc_vector = vector; 2168c9c75df4SNeel Natu vcpu->exc_errcode = errcode; 2169c9c75df4SNeel Natu vcpu->exc_errcode_valid = errcode_valid; 2170c9c75df4SNeel Natu VCPU_CTR1(vm, vcpuid, "Exception %d pending", vector); 2171dc506506SNeel Natu return (0); 2172dc506506SNeel Natu } 2173dc506506SNeel Natu 2174d37f2adbSNeel Natu void 2175d37f2adbSNeel Natu vm_inject_fault(void *vmarg, int vcpuid, int vector, int errcode_valid, 2176d37f2adbSNeel Natu int errcode) 2177dc506506SNeel Natu { 2178d37f2adbSNeel Natu struct vm *vm; 217973505a10SRobert Wing int error __diagused, restart_instruction; 2180dc506506SNeel Natu 2181d37f2adbSNeel Natu vm = vmarg; 2182c9c75df4SNeel Natu restart_instruction = 1; 2183d37f2adbSNeel Natu 2184c9c75df4SNeel Natu error = vm_inject_exception(vm, vcpuid, vector, errcode_valid, 2185c9c75df4SNeel Natu errcode, restart_instruction); 2186dc506506SNeel Natu KASSERT(error == 0, ("vm_inject_exception error %d", error)); 2187dc506506SNeel Natu } 2188dc506506SNeel Natu 2189dc506506SNeel Natu void 2190d37f2adbSNeel Natu vm_inject_pf(void *vmarg, int vcpuid, int error_code, uint64_t cr2) 2191fd949af6SNeel Natu { 2192d37f2adbSNeel Natu struct vm *vm; 219373505a10SRobert Wing int error __diagused; 219437a723a5SNeel Natu 2195d37f2adbSNeel Natu vm = vmarg; 219637a723a5SNeel Natu VCPU_CTR2(vm, vcpuid, "Injecting page fault: error_code %#x, cr2 %#lx", 219737a723a5SNeel Natu error_code, cr2); 219837a723a5SNeel Natu 219937a723a5SNeel Natu error = vm_set_register(vm, vcpuid, VM_REG_GUEST_CR2, cr2); 220037a723a5SNeel Natu KASSERT(error == 0, ("vm_set_register(cr2) error %d", error)); 2201fd949af6SNeel Natu 2202d37f2adbSNeel Natu vm_inject_fault(vm, vcpuid, IDT_PF, 1, error_code); 2203366f6083SPeter Grehan } 2204366f6083SPeter Grehan 220561592433SNeel Natu static VMM_STAT(VCPU_NMI_COUNT, "number of NMIs delivered to vcpu"); 2206366f6083SPeter Grehan 2207f352ff0cSNeel Natu int 2208f352ff0cSNeel Natu vm_inject_nmi(struct vm *vm, int vcpuid) 2209f352ff0cSNeel Natu { 2210f352ff0cSNeel Natu struct vcpu *vcpu; 2211f352ff0cSNeel Natu 2212a488c9c9SRodney W. Grimes if (vcpuid < 0 || vcpuid >= vm->maxcpus) 2213366f6083SPeter Grehan return (EINVAL); 2214366f6083SPeter Grehan 2215f352ff0cSNeel Natu vcpu = &vm->vcpu[vcpuid]; 2216f352ff0cSNeel Natu 2217f352ff0cSNeel Natu vcpu->nmi_pending = 1; 2218de5ea6b6SNeel Natu vcpu_notify_event(vm, vcpuid, false); 2219f352ff0cSNeel Natu return (0); 2220f352ff0cSNeel Natu } 2221f352ff0cSNeel Natu 2222f352ff0cSNeel Natu int 2223f352ff0cSNeel Natu vm_nmi_pending(struct vm *vm, int vcpuid) 2224f352ff0cSNeel Natu { 2225f352ff0cSNeel Natu struct vcpu *vcpu; 2226f352ff0cSNeel Natu 2227a488c9c9SRodney W. Grimes if (vcpuid < 0 || vcpuid >= vm->maxcpus) 2228f352ff0cSNeel Natu panic("vm_nmi_pending: invalid vcpuid %d", vcpuid); 2229f352ff0cSNeel Natu 2230f352ff0cSNeel Natu vcpu = &vm->vcpu[vcpuid]; 2231f352ff0cSNeel Natu 2232f352ff0cSNeel Natu return (vcpu->nmi_pending); 2233f352ff0cSNeel Natu } 2234f352ff0cSNeel Natu 2235f352ff0cSNeel Natu void 2236f352ff0cSNeel Natu vm_nmi_clear(struct vm *vm, int vcpuid) 2237f352ff0cSNeel Natu { 2238f352ff0cSNeel Natu struct vcpu *vcpu; 2239f352ff0cSNeel Natu 2240a488c9c9SRodney W. Grimes if (vcpuid < 0 || vcpuid >= vm->maxcpus) 2241f352ff0cSNeel Natu panic("vm_nmi_pending: invalid vcpuid %d", vcpuid); 2242f352ff0cSNeel Natu 2243f352ff0cSNeel Natu vcpu = &vm->vcpu[vcpuid]; 2244f352ff0cSNeel Natu 2245f352ff0cSNeel Natu if (vcpu->nmi_pending == 0) 2246f352ff0cSNeel Natu panic("vm_nmi_clear: inconsistent nmi_pending state"); 2247f352ff0cSNeel Natu 2248f352ff0cSNeel Natu vcpu->nmi_pending = 0; 22493dc3d32aSJohn Baldwin vmm_stat_incr(vcpu, VCPU_NMI_COUNT, 1); 2250366f6083SPeter Grehan } 2251366f6083SPeter Grehan 22520775fbb4STycho Nightingale static VMM_STAT(VCPU_EXTINT_COUNT, "number of ExtINTs delivered to vcpu"); 22530775fbb4STycho Nightingale 22540775fbb4STycho Nightingale int 22550775fbb4STycho Nightingale vm_inject_extint(struct vm *vm, int vcpuid) 22560775fbb4STycho Nightingale { 22570775fbb4STycho Nightingale struct vcpu *vcpu; 22580775fbb4STycho Nightingale 2259a488c9c9SRodney W. Grimes if (vcpuid < 0 || vcpuid >= vm->maxcpus) 22600775fbb4STycho Nightingale return (EINVAL); 22610775fbb4STycho Nightingale 22620775fbb4STycho Nightingale vcpu = &vm->vcpu[vcpuid]; 22630775fbb4STycho Nightingale 22640775fbb4STycho Nightingale vcpu->extint_pending = 1; 22650775fbb4STycho Nightingale vcpu_notify_event(vm, vcpuid, false); 22660775fbb4STycho Nightingale return (0); 22670775fbb4STycho Nightingale } 22680775fbb4STycho Nightingale 22690775fbb4STycho Nightingale int 22700775fbb4STycho Nightingale vm_extint_pending(struct vm *vm, int vcpuid) 22710775fbb4STycho Nightingale { 22720775fbb4STycho Nightingale struct vcpu *vcpu; 22730775fbb4STycho Nightingale 2274a488c9c9SRodney W. Grimes if (vcpuid < 0 || vcpuid >= vm->maxcpus) 22750775fbb4STycho Nightingale panic("vm_extint_pending: invalid vcpuid %d", vcpuid); 22760775fbb4STycho Nightingale 22770775fbb4STycho Nightingale vcpu = &vm->vcpu[vcpuid]; 22780775fbb4STycho Nightingale 22790775fbb4STycho Nightingale return (vcpu->extint_pending); 22800775fbb4STycho Nightingale } 22810775fbb4STycho Nightingale 22820775fbb4STycho Nightingale void 22830775fbb4STycho Nightingale vm_extint_clear(struct vm *vm, int vcpuid) 22840775fbb4STycho Nightingale { 22850775fbb4STycho Nightingale struct vcpu *vcpu; 22860775fbb4STycho Nightingale 2287a488c9c9SRodney W. Grimes if (vcpuid < 0 || vcpuid >= vm->maxcpus) 22880775fbb4STycho Nightingale panic("vm_extint_pending: invalid vcpuid %d", vcpuid); 22890775fbb4STycho Nightingale 22900775fbb4STycho Nightingale vcpu = &vm->vcpu[vcpuid]; 22910775fbb4STycho Nightingale 22920775fbb4STycho Nightingale if (vcpu->extint_pending == 0) 22930775fbb4STycho Nightingale panic("vm_extint_clear: inconsistent extint_pending state"); 22940775fbb4STycho Nightingale 22950775fbb4STycho Nightingale vcpu->extint_pending = 0; 22963dc3d32aSJohn Baldwin vmm_stat_incr(vcpu, VCPU_EXTINT_COUNT, 1); 22970775fbb4STycho Nightingale } 22980775fbb4STycho Nightingale 2299366f6083SPeter Grehan int 2300366f6083SPeter Grehan vm_get_capability(struct vm *vm, int vcpu, int type, int *retval) 2301366f6083SPeter Grehan { 2302a488c9c9SRodney W. Grimes if (vcpu < 0 || vcpu >= vm->maxcpus) 2303366f6083SPeter Grehan return (EINVAL); 2304366f6083SPeter Grehan 2305366f6083SPeter Grehan if (type < 0 || type >= VM_CAP_MAX) 2306366f6083SPeter Grehan return (EINVAL); 2307366f6083SPeter Grehan 2308869c8d19SJohn Baldwin return (vmmops_getcap(vcpu_cookie(vm, vcpu), type, retval)); 2309366f6083SPeter Grehan } 2310366f6083SPeter Grehan 2311366f6083SPeter Grehan int 2312366f6083SPeter Grehan vm_set_capability(struct vm *vm, int vcpu, int type, int val) 2313366f6083SPeter Grehan { 2314a488c9c9SRodney W. Grimes if (vcpu < 0 || vcpu >= vm->maxcpus) 2315366f6083SPeter Grehan return (EINVAL); 2316366f6083SPeter Grehan 2317366f6083SPeter Grehan if (type < 0 || type >= VM_CAP_MAX) 2318366f6083SPeter Grehan return (EINVAL); 2319366f6083SPeter Grehan 2320869c8d19SJohn Baldwin return (vmmops_setcap(vcpu_cookie(vm, vcpu), type, val)); 2321366f6083SPeter Grehan } 2322366f6083SPeter Grehan 2323950af9ffSJohn Baldwin struct vm * 2324950af9ffSJohn Baldwin vcpu_vm(struct vcpu *vcpu) 2325950af9ffSJohn Baldwin { 2326950af9ffSJohn Baldwin return (vcpu->vm); 2327950af9ffSJohn Baldwin } 2328950af9ffSJohn Baldwin 2329950af9ffSJohn Baldwin int 2330950af9ffSJohn Baldwin vcpu_vcpuid(struct vcpu *vcpu) 2331950af9ffSJohn Baldwin { 2332950af9ffSJohn Baldwin return (vcpu->vcpuid); 2333950af9ffSJohn Baldwin } 2334950af9ffSJohn Baldwin 2335950af9ffSJohn Baldwin struct vcpu * 2336950af9ffSJohn Baldwin vm_vcpu(struct vm *vm, int vcpuid) 2337950af9ffSJohn Baldwin { 2338950af9ffSJohn Baldwin return (&vm->vcpu[vcpuid]); 2339950af9ffSJohn Baldwin } 2340950af9ffSJohn Baldwin 2341366f6083SPeter Grehan struct vlapic * 2342366f6083SPeter Grehan vm_lapic(struct vm *vm, int cpu) 2343366f6083SPeter Grehan { 2344366f6083SPeter Grehan return (vm->vcpu[cpu].vlapic); 2345366f6083SPeter Grehan } 2346366f6083SPeter Grehan 2347565bbb86SNeel Natu struct vioapic * 2348565bbb86SNeel Natu vm_ioapic(struct vm *vm) 2349565bbb86SNeel Natu { 2350565bbb86SNeel Natu 2351565bbb86SNeel Natu return (vm->vioapic); 2352565bbb86SNeel Natu } 2353565bbb86SNeel Natu 235408e3ff32SNeel Natu struct vhpet * 235508e3ff32SNeel Natu vm_hpet(struct vm *vm) 235608e3ff32SNeel Natu { 235708e3ff32SNeel Natu 235808e3ff32SNeel Natu return (vm->vhpet); 235908e3ff32SNeel Natu } 236008e3ff32SNeel Natu 2361490d56c5SEd Maste bool 2362366f6083SPeter Grehan vmm_is_pptdev(int bus, int slot, int func) 2363366f6083SPeter Grehan { 2364490d56c5SEd Maste int b, f, i, n, s; 2365366f6083SPeter Grehan char *val, *cp, *cp2; 2366490d56c5SEd Maste bool found; 2367366f6083SPeter Grehan 2368366f6083SPeter Grehan /* 236907044a96SNeel Natu * XXX 237007044a96SNeel Natu * The length of an environment variable is limited to 128 bytes which 237107044a96SNeel Natu * puts an upper limit on the number of passthru devices that may be 237207044a96SNeel Natu * specified using a single environment variable. 237307044a96SNeel Natu * 237407044a96SNeel Natu * Work around this by scanning multiple environment variable 237507044a96SNeel Natu * names instead of a single one - yuck! 2376366f6083SPeter Grehan */ 237707044a96SNeel Natu const char *names[] = { "pptdevs", "pptdevs2", "pptdevs3", NULL }; 237807044a96SNeel Natu 237907044a96SNeel Natu /* set pptdevs="1/2/3 4/5/6 7/8/9 10/11/12" */ 2380490d56c5SEd Maste found = false; 238107044a96SNeel Natu for (i = 0; names[i] != NULL && !found; i++) { 23822be111bfSDavide Italiano cp = val = kern_getenv(names[i]); 2383366f6083SPeter Grehan while (cp != NULL && *cp != '\0') { 2384366f6083SPeter Grehan if ((cp2 = strchr(cp, ' ')) != NULL) 2385366f6083SPeter Grehan *cp2 = '\0'; 2386366f6083SPeter Grehan 2387366f6083SPeter Grehan n = sscanf(cp, "%d/%d/%d", &b, &s, &f); 2388366f6083SPeter Grehan if (n == 3 && bus == b && slot == s && func == f) { 2389490d56c5SEd Maste found = true; 2390366f6083SPeter Grehan break; 2391366f6083SPeter Grehan } 2392366f6083SPeter Grehan 2393366f6083SPeter Grehan if (cp2 != NULL) 2394366f6083SPeter Grehan *cp2++ = ' '; 2395366f6083SPeter Grehan 2396366f6083SPeter Grehan cp = cp2; 2397366f6083SPeter Grehan } 2398366f6083SPeter Grehan freeenv(val); 239907044a96SNeel Natu } 2400366f6083SPeter Grehan return (found); 2401366f6083SPeter Grehan } 2402366f6083SPeter Grehan 2403366f6083SPeter Grehan void * 2404366f6083SPeter Grehan vm_iommu_domain(struct vm *vm) 2405366f6083SPeter Grehan { 2406366f6083SPeter Grehan 2407366f6083SPeter Grehan return (vm->iommu); 2408366f6083SPeter Grehan } 2409366f6083SPeter Grehan 241075dd3366SNeel Natu int 2411f80330a8SNeel Natu vcpu_set_state(struct vm *vm, int vcpuid, enum vcpu_state newstate, 2412f80330a8SNeel Natu bool from_idle) 2413366f6083SPeter Grehan { 241475dd3366SNeel Natu int error; 2415366f6083SPeter Grehan struct vcpu *vcpu; 2416366f6083SPeter Grehan 2417a488c9c9SRodney W. Grimes if (vcpuid < 0 || vcpuid >= vm->maxcpus) 2418366f6083SPeter Grehan panic("vm_set_run_state: invalid vcpuid %d", vcpuid); 2419366f6083SPeter Grehan 2420366f6083SPeter Grehan vcpu = &vm->vcpu[vcpuid]; 2421366f6083SPeter Grehan 242275dd3366SNeel Natu vcpu_lock(vcpu); 2423248e6799SNeel Natu error = vcpu_set_state_locked(vm, vcpuid, newstate, from_idle); 242475dd3366SNeel Natu vcpu_unlock(vcpu); 242575dd3366SNeel Natu 242675dd3366SNeel Natu return (error); 242775dd3366SNeel Natu } 242875dd3366SNeel Natu 242975dd3366SNeel Natu enum vcpu_state 2430d3c11f40SPeter Grehan vcpu_get_state(struct vm *vm, int vcpuid, int *hostcpu) 2431366f6083SPeter Grehan { 2432366f6083SPeter Grehan struct vcpu *vcpu; 243375dd3366SNeel Natu enum vcpu_state state; 2434366f6083SPeter Grehan 2435a488c9c9SRodney W. Grimes if (vcpuid < 0 || vcpuid >= vm->maxcpus) 2436366f6083SPeter Grehan panic("vm_get_run_state: invalid vcpuid %d", vcpuid); 2437366f6083SPeter Grehan 2438366f6083SPeter Grehan vcpu = &vm->vcpu[vcpuid]; 2439366f6083SPeter Grehan 244075dd3366SNeel Natu vcpu_lock(vcpu); 244175dd3366SNeel Natu state = vcpu->state; 2442d3c11f40SPeter Grehan if (hostcpu != NULL) 2443d3c11f40SPeter Grehan *hostcpu = vcpu->hostcpu; 244475dd3366SNeel Natu vcpu_unlock(vcpu); 2445366f6083SPeter Grehan 244675dd3366SNeel Natu return (state); 2447366f6083SPeter Grehan } 2448366f6083SPeter Grehan 244995ebc360SNeel Natu int 2450366f6083SPeter Grehan vm_activate_cpu(struct vm *vm, int vcpuid) 2451366f6083SPeter Grehan { 2452366f6083SPeter Grehan 2453a488c9c9SRodney W. Grimes if (vcpuid < 0 || vcpuid >= vm->maxcpus) 245495ebc360SNeel Natu return (EINVAL); 245595ebc360SNeel Natu 245695ebc360SNeel Natu if (CPU_ISSET(vcpuid, &vm->active_cpus)) 245795ebc360SNeel Natu return (EBUSY); 245822d822c6SNeel Natu 245922d822c6SNeel Natu VCPU_CTR0(vm, vcpuid, "activated"); 246022d822c6SNeel Natu CPU_SET_ATOMIC(vcpuid, &vm->active_cpus); 246195ebc360SNeel Natu return (0); 2462366f6083SPeter Grehan } 2463366f6083SPeter Grehan 2464fc276d92SJohn Baldwin int 2465fc276d92SJohn Baldwin vm_suspend_cpu(struct vm *vm, int vcpuid) 2466fc276d92SJohn Baldwin { 2467fc276d92SJohn Baldwin int i; 2468fc276d92SJohn Baldwin 2469a488c9c9SRodney W. Grimes if (vcpuid < -1 || vcpuid >= vm->maxcpus) 2470fc276d92SJohn Baldwin return (EINVAL); 2471fc276d92SJohn Baldwin 2472fc276d92SJohn Baldwin if (vcpuid == -1) { 2473fc276d92SJohn Baldwin vm->debug_cpus = vm->active_cpus; 2474a488c9c9SRodney W. Grimes for (i = 0; i < vm->maxcpus; i++) { 2475fc276d92SJohn Baldwin if (CPU_ISSET(i, &vm->active_cpus)) 2476fc276d92SJohn Baldwin vcpu_notify_event(vm, i, false); 2477fc276d92SJohn Baldwin } 2478fc276d92SJohn Baldwin } else { 2479fc276d92SJohn Baldwin if (!CPU_ISSET(vcpuid, &vm->active_cpus)) 2480fc276d92SJohn Baldwin return (EINVAL); 2481fc276d92SJohn Baldwin 2482fc276d92SJohn Baldwin CPU_SET_ATOMIC(vcpuid, &vm->debug_cpus); 2483fc276d92SJohn Baldwin vcpu_notify_event(vm, vcpuid, false); 2484fc276d92SJohn Baldwin } 2485fc276d92SJohn Baldwin return (0); 2486fc276d92SJohn Baldwin } 2487fc276d92SJohn Baldwin 2488fc276d92SJohn Baldwin int 2489fc276d92SJohn Baldwin vm_resume_cpu(struct vm *vm, int vcpuid) 2490fc276d92SJohn Baldwin { 2491fc276d92SJohn Baldwin 2492a488c9c9SRodney W. Grimes if (vcpuid < -1 || vcpuid >= vm->maxcpus) 2493fc276d92SJohn Baldwin return (EINVAL); 2494fc276d92SJohn Baldwin 2495fc276d92SJohn Baldwin if (vcpuid == -1) { 2496fc276d92SJohn Baldwin CPU_ZERO(&vm->debug_cpus); 2497fc276d92SJohn Baldwin } else { 2498fc276d92SJohn Baldwin if (!CPU_ISSET(vcpuid, &vm->debug_cpus)) 2499fc276d92SJohn Baldwin return (EINVAL); 2500fc276d92SJohn Baldwin 2501fc276d92SJohn Baldwin CPU_CLR_ATOMIC(vcpuid, &vm->debug_cpus); 2502fc276d92SJohn Baldwin } 2503fc276d92SJohn Baldwin return (0); 2504fc276d92SJohn Baldwin } 2505fc276d92SJohn Baldwin 2506fc276d92SJohn Baldwin int 2507fc276d92SJohn Baldwin vcpu_debugged(struct vm *vm, int vcpuid) 2508fc276d92SJohn Baldwin { 2509fc276d92SJohn Baldwin 2510fc276d92SJohn Baldwin return (CPU_ISSET(vcpuid, &vm->debug_cpus)); 2511fc276d92SJohn Baldwin } 2512fc276d92SJohn Baldwin 2513a5615c90SPeter Grehan cpuset_t 2514366f6083SPeter Grehan vm_active_cpus(struct vm *vm) 2515366f6083SPeter Grehan { 2516366f6083SPeter Grehan 2517366f6083SPeter Grehan return (vm->active_cpus); 2518366f6083SPeter Grehan } 2519366f6083SPeter Grehan 252095ebc360SNeel Natu cpuset_t 2521fc276d92SJohn Baldwin vm_debug_cpus(struct vm *vm) 2522fc276d92SJohn Baldwin { 2523fc276d92SJohn Baldwin 2524fc276d92SJohn Baldwin return (vm->debug_cpus); 2525fc276d92SJohn Baldwin } 2526fc276d92SJohn Baldwin 2527fc276d92SJohn Baldwin cpuset_t 252895ebc360SNeel Natu vm_suspended_cpus(struct vm *vm) 252995ebc360SNeel Natu { 253095ebc360SNeel Natu 253195ebc360SNeel Natu return (vm->suspended_cpus); 253295ebc360SNeel Natu } 253395ebc360SNeel Natu 2534366f6083SPeter Grehan void * 25353dc3d32aSJohn Baldwin vcpu_stats(struct vcpu *vcpu) 2536366f6083SPeter Grehan { 2537366f6083SPeter Grehan 25383dc3d32aSJohn Baldwin return (vcpu->stats); 2539366f6083SPeter Grehan } 2540e9027382SNeel Natu 2541e9027382SNeel Natu int 2542e9027382SNeel Natu vm_get_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state *state) 2543e9027382SNeel Natu { 2544a488c9c9SRodney W. Grimes if (vcpuid < 0 || vcpuid >= vm->maxcpus) 2545e9027382SNeel Natu return (EINVAL); 2546e9027382SNeel Natu 2547e9027382SNeel Natu *state = vm->vcpu[vcpuid].x2apic_state; 2548e9027382SNeel Natu 2549e9027382SNeel Natu return (0); 2550e9027382SNeel Natu } 2551e9027382SNeel Natu 2552e9027382SNeel Natu int 2553e9027382SNeel Natu vm_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state) 2554e9027382SNeel Natu { 2555a488c9c9SRodney W. Grimes if (vcpuid < 0 || vcpuid >= vm->maxcpus) 2556e9027382SNeel Natu return (EINVAL); 2557e9027382SNeel Natu 25583f23d3caSNeel Natu if (state >= X2APIC_STATE_LAST) 2559e9027382SNeel Natu return (EINVAL); 2560e9027382SNeel Natu 2561e9027382SNeel Natu vm->vcpu[vcpuid].x2apic_state = state; 2562e9027382SNeel Natu 256373820fb0SNeel Natu vlapic_set_x2apic_state(vm, vcpuid, state); 256473820fb0SNeel Natu 2565e9027382SNeel Natu return (0); 2566e9027382SNeel Natu } 256775dd3366SNeel Natu 256822821874SNeel Natu /* 256922821874SNeel Natu * This function is called to ensure that a vcpu "sees" a pending event 257022821874SNeel Natu * as soon as possible: 257122821874SNeel Natu * - If the vcpu thread is sleeping then it is woken up. 257222821874SNeel Natu * - If the vcpu is running on a different host_cpu then an IPI will be directed 257322821874SNeel Natu * to the host_cpu to cause the vcpu to trap into the hypervisor. 257422821874SNeel Natu */ 2575248e6799SNeel Natu static void 2576248e6799SNeel Natu vcpu_notify_event_locked(struct vcpu *vcpu, bool lapic_intr) 257775dd3366SNeel Natu { 257875dd3366SNeel Natu int hostcpu; 257975dd3366SNeel Natu 258075dd3366SNeel Natu hostcpu = vcpu->hostcpu; 2581ef39d7e9SNeel Natu if (vcpu->state == VCPU_RUNNING) { 2582ef39d7e9SNeel Natu KASSERT(hostcpu != NOCPU, ("vcpu running on invalid hostcpu")); 2583de5ea6b6SNeel Natu if (hostcpu != curcpu) { 2584ef39d7e9SNeel Natu if (lapic_intr) { 2585add611fdSNeel Natu vlapic_post_intr(vcpu->vlapic, hostcpu, 2586add611fdSNeel Natu vmm_ipinum); 2587ef39d7e9SNeel Natu } else { 258875dd3366SNeel Natu ipi_cpu(hostcpu, vmm_ipinum); 258975dd3366SNeel Natu } 2590ef39d7e9SNeel Natu } else { 2591ef39d7e9SNeel Natu /* 2592ef39d7e9SNeel Natu * If the 'vcpu' is running on 'curcpu' then it must 2593ef39d7e9SNeel Natu * be sending a notification to itself (e.g. SELF_IPI). 2594ef39d7e9SNeel Natu * The pending event will be picked up when the vcpu 2595ef39d7e9SNeel Natu * transitions back to guest context. 2596ef39d7e9SNeel Natu */ 2597ef39d7e9SNeel Natu } 2598ef39d7e9SNeel Natu } else { 2599ef39d7e9SNeel Natu KASSERT(hostcpu == NOCPU, ("vcpu state %d not consistent " 2600ef39d7e9SNeel Natu "with hostcpu %d", vcpu->state, hostcpu)); 2601366f6083SPeter Grehan if (vcpu->state == VCPU_SLEEPING) 2602366f6083SPeter Grehan wakeup_one(vcpu); 2603366f6083SPeter Grehan } 2604248e6799SNeel Natu } 2605248e6799SNeel Natu 2606248e6799SNeel Natu void 2607248e6799SNeel Natu vcpu_notify_event(struct vm *vm, int vcpuid, bool lapic_intr) 2608248e6799SNeel Natu { 2609248e6799SNeel Natu struct vcpu *vcpu = &vm->vcpu[vcpuid]; 2610248e6799SNeel Natu 2611248e6799SNeel Natu vcpu_lock(vcpu); 2612248e6799SNeel Natu vcpu_notify_event_locked(vcpu, lapic_intr); 2613f76fc5d4SNeel Natu vcpu_unlock(vcpu); 2614f76fc5d4SNeel Natu } 2615318224bbSNeel Natu 2616318224bbSNeel Natu struct vmspace * 2617318224bbSNeel Natu vm_get_vmspace(struct vm *vm) 2618318224bbSNeel Natu { 2619318224bbSNeel Natu 2620318224bbSNeel Natu return (vm->vmspace); 2621318224bbSNeel Natu } 2622565bbb86SNeel Natu 2623565bbb86SNeel Natu int 2624565bbb86SNeel Natu vm_apicid2vcpuid(struct vm *vm, int apicid) 2625565bbb86SNeel Natu { 2626565bbb86SNeel Natu /* 2627565bbb86SNeel Natu * XXX apic id is assumed to be numerically identical to vcpu id 2628565bbb86SNeel Natu */ 2629565bbb86SNeel Natu return (apicid); 2630565bbb86SNeel Natu } 26315b8a8cd1SNeel Natu 2632b837daddSKonstantin Belousov int 26335b8a8cd1SNeel Natu vm_smp_rendezvous(struct vm *vm, int vcpuid, cpuset_t dest, 26345b8a8cd1SNeel Natu vm_rendezvous_func_t func, void *arg) 26355b8a8cd1SNeel Natu { 2636b837daddSKonstantin Belousov int error, i; 2637970955e4SNeel Natu 26385b8a8cd1SNeel Natu /* 26395b8a8cd1SNeel Natu * Enforce that this function is called without any locks 26405b8a8cd1SNeel Natu */ 26415b8a8cd1SNeel Natu WITNESS_WARN(WARN_PANIC, NULL, "vm_smp_rendezvous"); 2642a488c9c9SRodney W. Grimes KASSERT(vcpuid == -1 || (vcpuid >= 0 && vcpuid < vm->maxcpus), 26435b8a8cd1SNeel Natu ("vm_smp_rendezvous: invalid vcpuid %d", vcpuid)); 26445b8a8cd1SNeel Natu 26455b8a8cd1SNeel Natu restart: 26465b8a8cd1SNeel Natu mtx_lock(&vm->rendezvous_mtx); 26475b8a8cd1SNeel Natu if (vm->rendezvous_func != NULL) { 26485b8a8cd1SNeel Natu /* 26495b8a8cd1SNeel Natu * If a rendezvous is already in progress then we need to 26505b8a8cd1SNeel Natu * call the rendezvous handler in case this 'vcpuid' is one 26515b8a8cd1SNeel Natu * of the targets of the rendezvous. 26525b8a8cd1SNeel Natu */ 26535b8a8cd1SNeel Natu RENDEZVOUS_CTR0(vm, vcpuid, "Rendezvous already in progress"); 26545b8a8cd1SNeel Natu mtx_unlock(&vm->rendezvous_mtx); 2655b837daddSKonstantin Belousov error = vm_handle_rendezvous(vm, vcpuid); 2656b837daddSKonstantin Belousov if (error != 0) 2657b837daddSKonstantin Belousov return (error); 26585b8a8cd1SNeel Natu goto restart; 26595b8a8cd1SNeel Natu } 26605b8a8cd1SNeel Natu KASSERT(vm->rendezvous_func == NULL, ("vm_smp_rendezvous: previous " 26615b8a8cd1SNeel Natu "rendezvous is still in progress")); 26625b8a8cd1SNeel Natu 26635b8a8cd1SNeel Natu RENDEZVOUS_CTR0(vm, vcpuid, "Initiating rendezvous"); 26645b8a8cd1SNeel Natu vm->rendezvous_req_cpus = dest; 26655b8a8cd1SNeel Natu CPU_ZERO(&vm->rendezvous_done_cpus); 26665b8a8cd1SNeel Natu vm->rendezvous_arg = arg; 2667869dbab7SAndriy Gapon vm->rendezvous_func = func; 26685b8a8cd1SNeel Natu mtx_unlock(&vm->rendezvous_mtx); 26695b8a8cd1SNeel Natu 2670970955e4SNeel Natu /* 2671970955e4SNeel Natu * Wake up any sleeping vcpus and trigger a VM-exit in any running 2672970955e4SNeel Natu * vcpus so they handle the rendezvous as soon as possible. 2673970955e4SNeel Natu */ 2674a488c9c9SRodney W. Grimes for (i = 0; i < vm->maxcpus; i++) { 2675970955e4SNeel Natu if (CPU_ISSET(i, &dest)) 2676970955e4SNeel Natu vcpu_notify_event(vm, i, false); 2677970955e4SNeel Natu } 2678970955e4SNeel Natu 2679b837daddSKonstantin Belousov return (vm_handle_rendezvous(vm, vcpuid)); 26805b8a8cd1SNeel Natu } 2681762fd208STycho Nightingale 2682762fd208STycho Nightingale struct vatpic * 2683762fd208STycho Nightingale vm_atpic(struct vm *vm) 2684762fd208STycho Nightingale { 2685762fd208STycho Nightingale return (vm->vatpic); 2686762fd208STycho Nightingale } 2687e883c9bbSTycho Nightingale 2688e883c9bbSTycho Nightingale struct vatpit * 2689e883c9bbSTycho Nightingale vm_atpit(struct vm *vm) 2690e883c9bbSTycho Nightingale { 2691e883c9bbSTycho Nightingale return (vm->vatpit); 2692e883c9bbSTycho Nightingale } 2693d17b5104SNeel Natu 2694160ef77aSNeel Natu struct vpmtmr * 2695160ef77aSNeel Natu vm_pmtmr(struct vm *vm) 2696160ef77aSNeel Natu { 2697160ef77aSNeel Natu 2698160ef77aSNeel Natu return (vm->vpmtmr); 2699160ef77aSNeel Natu } 2700160ef77aSNeel Natu 27010dafa5cdSNeel Natu struct vrtc * 27020dafa5cdSNeel Natu vm_rtc(struct vm *vm) 27030dafa5cdSNeel Natu { 27040dafa5cdSNeel Natu 27050dafa5cdSNeel Natu return (vm->vrtc); 27060dafa5cdSNeel Natu } 27070dafa5cdSNeel Natu 2708d17b5104SNeel Natu enum vm_reg_name 2709d17b5104SNeel Natu vm_segment_name(int seg) 2710d17b5104SNeel Natu { 2711d17b5104SNeel Natu static enum vm_reg_name seg_names[] = { 2712d17b5104SNeel Natu VM_REG_GUEST_ES, 2713d17b5104SNeel Natu VM_REG_GUEST_CS, 2714d17b5104SNeel Natu VM_REG_GUEST_SS, 2715d17b5104SNeel Natu VM_REG_GUEST_DS, 2716d17b5104SNeel Natu VM_REG_GUEST_FS, 2717d17b5104SNeel Natu VM_REG_GUEST_GS 2718d17b5104SNeel Natu }; 2719d17b5104SNeel Natu 2720d17b5104SNeel Natu KASSERT(seg >= 0 && seg < nitems(seg_names), 2721d17b5104SNeel Natu ("%s: invalid segment encoding %d", __func__, seg)); 2722d17b5104SNeel Natu return (seg_names[seg]); 2723d17b5104SNeel Natu } 2724cf1d80d8SPeter Grehan 2725d665d229SNeel Natu void 2726*2b4fe856SJohn Baldwin vm_copy_teardown(struct vm_copyinfo *copyinfo, int num_copyinfo) 2727d665d229SNeel Natu { 2728d665d229SNeel Natu int idx; 2729d665d229SNeel Natu 2730d665d229SNeel Natu for (idx = 0; idx < num_copyinfo; idx++) { 2731d665d229SNeel Natu if (copyinfo[idx].cookie != NULL) 2732d665d229SNeel Natu vm_gpa_release(copyinfo[idx].cookie); 2733d665d229SNeel Natu } 2734d665d229SNeel Natu bzero(copyinfo, num_copyinfo * sizeof(struct vm_copyinfo)); 2735d665d229SNeel Natu } 2736d665d229SNeel Natu 2737d665d229SNeel Natu int 2738d665d229SNeel Natu vm_copy_setup(struct vm *vm, int vcpuid, struct vm_guest_paging *paging, 2739d665d229SNeel Natu uint64_t gla, size_t len, int prot, struct vm_copyinfo *copyinfo, 27409c4d5478SNeel Natu int num_copyinfo, int *fault) 2741d665d229SNeel Natu { 2742d665d229SNeel Natu int error, idx, nused; 2743d665d229SNeel Natu size_t n, off, remaining; 2744d665d229SNeel Natu void *hva, *cookie; 2745d665d229SNeel Natu uint64_t gpa; 2746d665d229SNeel Natu 2747d665d229SNeel Natu bzero(copyinfo, sizeof(struct vm_copyinfo) * num_copyinfo); 2748d665d229SNeel Natu 2749d665d229SNeel Natu nused = 0; 2750d665d229SNeel Natu remaining = len; 2751d665d229SNeel Natu while (remaining > 0) { 2752d665d229SNeel Natu KASSERT(nused < num_copyinfo, ("insufficient vm_copyinfo")); 27539c4d5478SNeel Natu error = vm_gla2gpa(vm, vcpuid, paging, gla, prot, &gpa, fault); 27549c4d5478SNeel Natu if (error || *fault) 2755d665d229SNeel Natu return (error); 2756d665d229SNeel Natu off = gpa & PAGE_MASK; 2757d665d229SNeel Natu n = min(remaining, PAGE_SIZE - off); 2758d665d229SNeel Natu copyinfo[nused].gpa = gpa; 2759d665d229SNeel Natu copyinfo[nused].len = n; 2760d665d229SNeel Natu remaining -= n; 2761d665d229SNeel Natu gla += n; 2762d665d229SNeel Natu nused++; 2763d665d229SNeel Natu } 2764d665d229SNeel Natu 2765d665d229SNeel Natu for (idx = 0; idx < nused; idx++) { 27669b1aa8d6SNeel Natu hva = vm_gpa_hold(vm, vcpuid, copyinfo[idx].gpa, 27679b1aa8d6SNeel Natu copyinfo[idx].len, prot, &cookie); 2768d665d229SNeel Natu if (hva == NULL) 2769d665d229SNeel Natu break; 2770d665d229SNeel Natu copyinfo[idx].hva = hva; 2771d665d229SNeel Natu copyinfo[idx].cookie = cookie; 2772d665d229SNeel Natu } 2773d665d229SNeel Natu 2774d665d229SNeel Natu if (idx != nused) { 2775*2b4fe856SJohn Baldwin vm_copy_teardown(copyinfo, num_copyinfo); 27769c4d5478SNeel Natu return (EFAULT); 2777d665d229SNeel Natu } else { 27789c4d5478SNeel Natu *fault = 0; 2779d665d229SNeel Natu return (0); 2780d665d229SNeel Natu } 2781d665d229SNeel Natu } 2782d665d229SNeel Natu 2783d665d229SNeel Natu void 2784*2b4fe856SJohn Baldwin vm_copyin(struct vm_copyinfo *copyinfo, void *kaddr, size_t len) 2785d665d229SNeel Natu { 2786d665d229SNeel Natu char *dst; 2787d665d229SNeel Natu int idx; 2788d665d229SNeel Natu 2789d665d229SNeel Natu dst = kaddr; 2790d665d229SNeel Natu idx = 0; 2791d665d229SNeel Natu while (len > 0) { 2792d665d229SNeel Natu bcopy(copyinfo[idx].hva, dst, copyinfo[idx].len); 2793d665d229SNeel Natu len -= copyinfo[idx].len; 2794d665d229SNeel Natu dst += copyinfo[idx].len; 2795d665d229SNeel Natu idx++; 2796d665d229SNeel Natu } 2797d665d229SNeel Natu } 2798d665d229SNeel Natu 2799d665d229SNeel Natu void 2800*2b4fe856SJohn Baldwin vm_copyout(const void *kaddr, struct vm_copyinfo *copyinfo, size_t len) 2801d665d229SNeel Natu { 2802d665d229SNeel Natu const char *src; 2803d665d229SNeel Natu int idx; 2804d665d229SNeel Natu 2805d665d229SNeel Natu src = kaddr; 2806d665d229SNeel Natu idx = 0; 2807d665d229SNeel Natu while (len > 0) { 2808d665d229SNeel Natu bcopy(src, copyinfo[idx].hva, copyinfo[idx].len); 2809d665d229SNeel Natu len -= copyinfo[idx].len; 2810d665d229SNeel Natu src += copyinfo[idx].len; 2811d665d229SNeel Natu idx++; 2812d665d229SNeel Natu } 2813d665d229SNeel Natu } 2814cf1d80d8SPeter Grehan 2815cf1d80d8SPeter Grehan /* 2816cf1d80d8SPeter Grehan * Return the amount of in-use and wired memory for the VM. Since 2817cf1d80d8SPeter Grehan * these are global stats, only return the values with for vCPU 0 2818cf1d80d8SPeter Grehan */ 2819cf1d80d8SPeter Grehan VMM_STAT_DECLARE(VMM_MEM_RESIDENT); 2820cf1d80d8SPeter Grehan VMM_STAT_DECLARE(VMM_MEM_WIRED); 2821cf1d80d8SPeter Grehan 2822cf1d80d8SPeter Grehan static void 2823cf1d80d8SPeter Grehan vm_get_rescnt(struct vm *vm, int vcpu, struct vmm_stat_type *stat) 2824cf1d80d8SPeter Grehan { 2825cf1d80d8SPeter Grehan 2826cf1d80d8SPeter Grehan if (vcpu == 0) { 28273dc3d32aSJohn Baldwin vmm_stat_set(vm_vcpu(vm, vcpu), VMM_MEM_RESIDENT, 2828cf1d80d8SPeter Grehan PAGE_SIZE * vmspace_resident_count(vm->vmspace)); 2829cf1d80d8SPeter Grehan } 2830cf1d80d8SPeter Grehan } 2831cf1d80d8SPeter Grehan 2832cf1d80d8SPeter Grehan static void 2833cf1d80d8SPeter Grehan vm_get_wiredcnt(struct vm *vm, int vcpu, struct vmm_stat_type *stat) 2834cf1d80d8SPeter Grehan { 2835cf1d80d8SPeter Grehan 2836cf1d80d8SPeter Grehan if (vcpu == 0) { 28373dc3d32aSJohn Baldwin vmm_stat_set(vm_vcpu(vm, vcpu), VMM_MEM_WIRED, 2838cf1d80d8SPeter Grehan PAGE_SIZE * pmap_wired_count(vmspace_pmap(vm->vmspace))); 2839cf1d80d8SPeter Grehan } 2840cf1d80d8SPeter Grehan } 2841cf1d80d8SPeter Grehan 2842cf1d80d8SPeter Grehan VMM_STAT_FUNC(VMM_MEM_RESIDENT, "Resident memory", vm_get_rescnt); 2843cf1d80d8SPeter Grehan VMM_STAT_FUNC(VMM_MEM_WIRED, "Wired memory", vm_get_wiredcnt); 2844483d953aSJohn Baldwin 2845483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT 2846483d953aSJohn Baldwin static int 2847483d953aSJohn Baldwin vm_snapshot_vcpus(struct vm *vm, struct vm_snapshot_meta *meta) 2848483d953aSJohn Baldwin { 2849a7db532eSJohn Baldwin uint64_t tsc, now; 2850483d953aSJohn Baldwin int ret; 2851483d953aSJohn Baldwin struct vcpu *vcpu; 285235abc6c2SJohn Baldwin uint16_t i, maxcpus; 2853483d953aSJohn Baldwin 2854a7db532eSJohn Baldwin now = rdtsc(); 285535abc6c2SJohn Baldwin maxcpus = vm_get_maxcpus(vm); 285635abc6c2SJohn Baldwin for (i = 0; i < maxcpus; i++) { 2857483d953aSJohn Baldwin vcpu = &vm->vcpu[i]; 2858483d953aSJohn Baldwin 2859483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vcpu->x2apic_state, meta, ret, done); 2860483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vcpu->exitintinfo, meta, ret, done); 2861483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vcpu->exc_vector, meta, ret, done); 2862483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vcpu->exc_errcode_valid, meta, ret, done); 2863483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vcpu->exc_errcode, meta, ret, done); 2864483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vcpu->guest_xcr0, meta, ret, done); 2865483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vcpu->exitinfo, meta, ret, done); 2866483d953aSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(vcpu->nextrip, meta, ret, done); 2867a7db532eSJohn Baldwin 2868a7db532eSJohn Baldwin /* 2869a7db532eSJohn Baldwin * Save the absolute TSC value by adding now to tsc_offset. 2870483d953aSJohn Baldwin * 2871483d953aSJohn Baldwin * It will be turned turned back into an actual offset when the 2872483d953aSJohn Baldwin * TSC restore function is called 2873483d953aSJohn Baldwin */ 2874a7db532eSJohn Baldwin tsc = now + vcpu->tsc_offset; 2875a7db532eSJohn Baldwin SNAPSHOT_VAR_OR_LEAVE(tsc, meta, ret, done); 2876483d953aSJohn Baldwin } 2877483d953aSJohn Baldwin 2878483d953aSJohn Baldwin done: 2879483d953aSJohn Baldwin return (ret); 2880483d953aSJohn Baldwin } 2881483d953aSJohn Baldwin 2882483d953aSJohn Baldwin static int 2883483d953aSJohn Baldwin vm_snapshot_vm(struct vm *vm, struct vm_snapshot_meta *meta) 2884483d953aSJohn Baldwin { 2885483d953aSJohn Baldwin int ret; 2886483d953aSJohn Baldwin 2887483d953aSJohn Baldwin ret = vm_snapshot_vcpus(vm, meta); 2888a7db532eSJohn Baldwin if (ret != 0) 2889483d953aSJohn Baldwin goto done; 2890483d953aSJohn Baldwin 2891483d953aSJohn Baldwin done: 2892483d953aSJohn Baldwin return (ret); 2893483d953aSJohn Baldwin } 2894483d953aSJohn Baldwin 2895483d953aSJohn Baldwin static int 28961aa51504SJohn Baldwin vm_snapshot_vcpu(struct vm *vm, struct vm_snapshot_meta *meta) 2897483d953aSJohn Baldwin { 289835abc6c2SJohn Baldwin int error; 28991aa51504SJohn Baldwin struct vcpu *vcpu; 290035abc6c2SJohn Baldwin uint16_t i, maxcpus; 2901483d953aSJohn Baldwin 2902483d953aSJohn Baldwin error = 0; 2903483d953aSJohn Baldwin 290435abc6c2SJohn Baldwin maxcpus = vm_get_maxcpus(vm); 290535abc6c2SJohn Baldwin for (i = 0; i < maxcpus; i++) { 29061aa51504SJohn Baldwin vcpu = &vm->vcpu[i]; 29071aa51504SJohn Baldwin 2908869c8d19SJohn Baldwin error = vmmops_vcpu_snapshot(vcpu->cookie, meta); 2909483d953aSJohn Baldwin if (error != 0) { 2910483d953aSJohn Baldwin printf("%s: failed to snapshot vmcs/vmcb data for " 2911483d953aSJohn Baldwin "vCPU: %d; error: %d\n", __func__, i, error); 2912483d953aSJohn Baldwin goto done; 2913483d953aSJohn Baldwin } 2914483d953aSJohn Baldwin } 2915483d953aSJohn Baldwin 2916483d953aSJohn Baldwin done: 2917483d953aSJohn Baldwin return (error); 2918483d953aSJohn Baldwin } 2919483d953aSJohn Baldwin 2920483d953aSJohn Baldwin /* 2921483d953aSJohn Baldwin * Save kernel-side structures to user-space for snapshotting. 2922483d953aSJohn Baldwin */ 2923483d953aSJohn Baldwin int 2924483d953aSJohn Baldwin vm_snapshot_req(struct vm *vm, struct vm_snapshot_meta *meta) 2925483d953aSJohn Baldwin { 2926483d953aSJohn Baldwin int ret = 0; 2927483d953aSJohn Baldwin 2928483d953aSJohn Baldwin switch (meta->dev_req) { 2929483d953aSJohn Baldwin case STRUCT_VMX: 293015add60dSPeter Grehan ret = vmmops_snapshot(vm->cookie, meta); 2931483d953aSJohn Baldwin break; 2932483d953aSJohn Baldwin case STRUCT_VMCX: 29331aa51504SJohn Baldwin ret = vm_snapshot_vcpu(vm, meta); 2934483d953aSJohn Baldwin break; 2935483d953aSJohn Baldwin case STRUCT_VM: 2936483d953aSJohn Baldwin ret = vm_snapshot_vm(vm, meta); 2937483d953aSJohn Baldwin break; 2938483d953aSJohn Baldwin case STRUCT_VIOAPIC: 2939483d953aSJohn Baldwin ret = vioapic_snapshot(vm_ioapic(vm), meta); 2940483d953aSJohn Baldwin break; 2941483d953aSJohn Baldwin case STRUCT_VLAPIC: 2942483d953aSJohn Baldwin ret = vlapic_snapshot(vm, meta); 2943483d953aSJohn Baldwin break; 2944483d953aSJohn Baldwin case STRUCT_VHPET: 2945483d953aSJohn Baldwin ret = vhpet_snapshot(vm_hpet(vm), meta); 2946483d953aSJohn Baldwin break; 2947483d953aSJohn Baldwin case STRUCT_VATPIC: 2948483d953aSJohn Baldwin ret = vatpic_snapshot(vm_atpic(vm), meta); 2949483d953aSJohn Baldwin break; 2950483d953aSJohn Baldwin case STRUCT_VATPIT: 2951483d953aSJohn Baldwin ret = vatpit_snapshot(vm_atpit(vm), meta); 2952483d953aSJohn Baldwin break; 2953483d953aSJohn Baldwin case STRUCT_VPMTMR: 2954483d953aSJohn Baldwin ret = vpmtmr_snapshot(vm_pmtmr(vm), meta); 2955483d953aSJohn Baldwin break; 2956483d953aSJohn Baldwin case STRUCT_VRTC: 2957483d953aSJohn Baldwin ret = vrtc_snapshot(vm_rtc(vm), meta); 2958483d953aSJohn Baldwin break; 2959483d953aSJohn Baldwin default: 2960483d953aSJohn Baldwin printf("%s: failed to find the requested type %#x\n", 2961483d953aSJohn Baldwin __func__, meta->dev_req); 2962483d953aSJohn Baldwin ret = (EINVAL); 2963483d953aSJohn Baldwin } 2964483d953aSJohn Baldwin return (ret); 2965483d953aSJohn Baldwin } 2966483d953aSJohn Baldwin 2967483d953aSJohn Baldwin int 2968483d953aSJohn Baldwin vm_set_tsc_offset(struct vm *vm, int vcpuid, uint64_t offset) 2969483d953aSJohn Baldwin { 2970483d953aSJohn Baldwin struct vcpu *vcpu; 2971483d953aSJohn Baldwin 297235abc6c2SJohn Baldwin if (vcpuid < 0 || vcpuid >= vm_get_maxcpus(vm)) 2973483d953aSJohn Baldwin return (EINVAL); 2974483d953aSJohn Baldwin 2975483d953aSJohn Baldwin vcpu = &vm->vcpu[vcpuid]; 2976483d953aSJohn Baldwin vcpu->tsc_offset = offset; 2977483d953aSJohn Baldwin 2978483d953aSJohn Baldwin return (0); 2979483d953aSJohn Baldwin } 2980483d953aSJohn Baldwin 2981483d953aSJohn Baldwin int 2982483d953aSJohn Baldwin vm_restore_time(struct vm *vm) 2983483d953aSJohn Baldwin { 298435abc6c2SJohn Baldwin int error; 2985483d953aSJohn Baldwin uint64_t now; 2986483d953aSJohn Baldwin struct vcpu *vcpu; 298735abc6c2SJohn Baldwin uint16_t i, maxcpus; 2988483d953aSJohn Baldwin 2989483d953aSJohn Baldwin now = rdtsc(); 2990483d953aSJohn Baldwin 2991483d953aSJohn Baldwin error = vhpet_restore_time(vm_hpet(vm)); 2992483d953aSJohn Baldwin if (error) 2993483d953aSJohn Baldwin return (error); 2994483d953aSJohn Baldwin 299535abc6c2SJohn Baldwin maxcpus = vm_get_maxcpus(vm); 299635abc6c2SJohn Baldwin for (i = 0; i < maxcpus; i++) { 2997483d953aSJohn Baldwin vcpu = &vm->vcpu[i]; 2998483d953aSJohn Baldwin 2999869c8d19SJohn Baldwin error = vmmops_restore_tsc(vcpu->cookie, 30001aa51504SJohn Baldwin vcpu->tsc_offset - now); 3001483d953aSJohn Baldwin if (error) 3002483d953aSJohn Baldwin return (error); 3003483d953aSJohn Baldwin } 3004483d953aSJohn Baldwin 3005483d953aSJohn Baldwin return (0); 3006483d953aSJohn Baldwin } 3007483d953aSJohn Baldwin #endif 3008