xref: /freebsd/sys/amd64/vmm/vmm.c (revision 2b4fe856f44ded02f3450bac1782bb49b60b7dd5)
1366f6083SPeter Grehan /*-
2c49761ddSPedro F. Giffuni  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3c49761ddSPedro F. Giffuni  *
4366f6083SPeter Grehan  * Copyright (c) 2011 NetApp, Inc.
5366f6083SPeter Grehan  * All rights reserved.
6366f6083SPeter Grehan  *
7366f6083SPeter Grehan  * Redistribution and use in source and binary forms, with or without
8366f6083SPeter Grehan  * modification, are permitted provided that the following conditions
9366f6083SPeter Grehan  * are met:
10366f6083SPeter Grehan  * 1. Redistributions of source code must retain the above copyright
11366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer.
12366f6083SPeter Grehan  * 2. Redistributions in binary form must reproduce the above copyright
13366f6083SPeter Grehan  *    notice, this list of conditions and the following disclaimer in the
14366f6083SPeter Grehan  *    documentation and/or other materials provided with the distribution.
15366f6083SPeter Grehan  *
16366f6083SPeter Grehan  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17366f6083SPeter Grehan  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18366f6083SPeter Grehan  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19366f6083SPeter Grehan  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20366f6083SPeter Grehan  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21366f6083SPeter Grehan  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22366f6083SPeter Grehan  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23366f6083SPeter Grehan  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24366f6083SPeter Grehan  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25366f6083SPeter Grehan  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26366f6083SPeter Grehan  * SUCH DAMAGE.
27366f6083SPeter Grehan  *
28366f6083SPeter Grehan  * $FreeBSD$
29366f6083SPeter Grehan  */
30366f6083SPeter Grehan 
31366f6083SPeter Grehan #include <sys/cdefs.h>
32366f6083SPeter Grehan __FBSDID("$FreeBSD$");
33366f6083SPeter Grehan 
34483d953aSJohn Baldwin #include "opt_bhyve_snapshot.h"
35483d953aSJohn Baldwin 
36366f6083SPeter Grehan #include <sys/param.h>
3738f1b189SPeter Grehan #include <sys/systm.h>
38366f6083SPeter Grehan #include <sys/kernel.h>
39366f6083SPeter Grehan #include <sys/module.h>
40366f6083SPeter Grehan #include <sys/sysctl.h>
41366f6083SPeter Grehan #include <sys/malloc.h>
42366f6083SPeter Grehan #include <sys/pcpu.h>
43366f6083SPeter Grehan #include <sys/lock.h>
44366f6083SPeter Grehan #include <sys/mutex.h>
45366f6083SPeter Grehan #include <sys/proc.h>
46318224bbSNeel Natu #include <sys/rwlock.h>
47366f6083SPeter Grehan #include <sys/sched.h>
48366f6083SPeter Grehan #include <sys/smp.h>
49483d953aSJohn Baldwin #include <sys/vnode.h>
50366f6083SPeter Grehan 
51366f6083SPeter Grehan #include <vm/vm.h>
523c48106aSKonstantin Belousov #include <vm/vm_param.h>
533c48106aSKonstantin Belousov #include <vm/vm_extern.h>
54318224bbSNeel Natu #include <vm/vm_object.h>
55318224bbSNeel Natu #include <vm/vm_page.h>
56318224bbSNeel Natu #include <vm/pmap.h>
57318224bbSNeel Natu #include <vm/vm_map.h>
58483d953aSJohn Baldwin #include <vm/vm_pager.h>
59483d953aSJohn Baldwin #include <vm/vm_kern.h>
60483d953aSJohn Baldwin #include <vm/vnode_pager.h>
61483d953aSJohn Baldwin #include <vm/swap_pager.h>
62483d953aSJohn Baldwin #include <vm/uma.h>
63366f6083SPeter Grehan 
6463e62d39SJohn Baldwin #include <machine/cpu.h>
65366f6083SPeter Grehan #include <machine/pcb.h>
6675dd3366SNeel Natu #include <machine/smp.h>
67bd50262fSKonstantin Belousov #include <machine/md_var.h>
681c052192SNeel Natu #include <x86/psl.h>
6934a6b2d6SJohn Baldwin #include <x86/apicreg.h>
7015add60dSPeter Grehan #include <x86/ifunc.h>
71366f6083SPeter Grehan 
72366f6083SPeter Grehan #include <machine/vmm.h>
73565bbb86SNeel Natu #include <machine/vmm_dev.h>
74e813a873SNeel Natu #include <machine/vmm_instruction_emul.h>
75483d953aSJohn Baldwin #include <machine/vmm_snapshot.h>
76565bbb86SNeel Natu 
77d17b5104SNeel Natu #include "vmm_ioport.h"
78318224bbSNeel Natu #include "vmm_ktr.h"
79b01c2033SNeel Natu #include "vmm_host.h"
80366f6083SPeter Grehan #include "vmm_mem.h"
81366f6083SPeter Grehan #include "vmm_util.h"
82762fd208STycho Nightingale #include "vatpic.h"
83e883c9bbSTycho Nightingale #include "vatpit.h"
8408e3ff32SNeel Natu #include "vhpet.h"
85565bbb86SNeel Natu #include "vioapic.h"
86366f6083SPeter Grehan #include "vlapic.h"
87160ef77aSNeel Natu #include "vpmtmr.h"
880dafa5cdSNeel Natu #include "vrtc.h"
89366f6083SPeter Grehan #include "vmm_stat.h"
90f76fc5d4SNeel Natu #include "vmm_lapic.h"
91366f6083SPeter Grehan 
92366f6083SPeter Grehan #include "io/ppt.h"
93366f6083SPeter Grehan #include "io/iommu.h"
94366f6083SPeter Grehan 
95366f6083SPeter Grehan struct vlapic;
96366f6083SPeter Grehan 
975fcf252fSNeel Natu /*
985fcf252fSNeel Natu  * Initialization:
995fcf252fSNeel Natu  * (a) allocated when vcpu is created
1005fcf252fSNeel Natu  * (i) initialized when vcpu is created and when it is reinitialized
1015fcf252fSNeel Natu  * (o) initialized the first time the vcpu is created
1025fcf252fSNeel Natu  * (x) initialized before use
1035fcf252fSNeel Natu  */
104366f6083SPeter Grehan struct vcpu {
1055fcf252fSNeel Natu 	struct mtx 	mtx;		/* (o) protects 'state' and 'hostcpu' */
1065fcf252fSNeel Natu 	enum vcpu_state	state;		/* (o) vcpu state */
107950af9ffSJohn Baldwin 	int		vcpuid;		/* (o) */
1085fcf252fSNeel Natu 	int		hostcpu;	/* (o) vcpu's host cpu */
109248e6799SNeel Natu 	int		reqidle;	/* (i) request vcpu to idle */
110950af9ffSJohn Baldwin 	struct vm	*vm;		/* (o) */
1111aa51504SJohn Baldwin 	void		*cookie;	/* (i) cpu-specific data */
1125fcf252fSNeel Natu 	struct vlapic	*vlapic;	/* (i) APIC device model */
1135fcf252fSNeel Natu 	enum x2apic_state x2apic_state;	/* (i) APIC mode */
114091d4532SNeel Natu 	uint64_t	exitintinfo;	/* (i) events pending at VM exit */
1155fcf252fSNeel Natu 	int		nmi_pending;	/* (i) NMI pending */
1165fcf252fSNeel Natu 	int		extint_pending;	/* (i) INTR pending */
1175fcf252fSNeel Natu 	int	exception_pending;	/* (i) exception pending */
118c9c75df4SNeel Natu 	int	exc_vector;		/* (x) exception collateral */
119c9c75df4SNeel Natu 	int	exc_errcode_valid;
120c9c75df4SNeel Natu 	uint32_t exc_errcode;
1215fcf252fSNeel Natu 	struct savefpu	*guestfpu;	/* (a,i) guest fpu state */
1225fcf252fSNeel Natu 	uint64_t	guest_xcr0;	/* (i) guest %xcr0 register */
1235fcf252fSNeel Natu 	void		*stats;		/* (a,i) statistics */
1245fcf252fSNeel Natu 	struct vm_exit	exitinfo;	/* (x) exit reason and collateral */
125d087a399SNeel Natu 	uint64_t	nextrip;	/* (x) next instruction to execute */
126483d953aSJohn Baldwin 	uint64_t	tsc_offset;	/* (o) TSC offsetting */
127366f6083SPeter Grehan };
128366f6083SPeter Grehan 
1295fcf252fSNeel Natu #define	vcpu_lock_initialized(v) mtx_initialized(&((v)->mtx))
130f76fc5d4SNeel Natu #define	vcpu_lock_init(v)	mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN)
131f76fc5d4SNeel Natu #define	vcpu_lock(v)		mtx_lock_spin(&((v)->mtx))
132f76fc5d4SNeel Natu #define	vcpu_unlock(v)		mtx_unlock_spin(&((v)->mtx))
133318224bbSNeel Natu #define	vcpu_assert_locked(v)	mtx_assert(&((v)->mtx), MA_OWNED)
13475dd3366SNeel Natu 
135318224bbSNeel Natu struct mem_seg {
1369b1aa8d6SNeel Natu 	size_t	len;
1379b1aa8d6SNeel Natu 	bool	sysmem;
1389b1aa8d6SNeel Natu 	struct vm_object *object;
1399b1aa8d6SNeel Natu };
140e47fe318SCorvin Köhne #define	VM_MAX_MEMSEGS	4
1419b1aa8d6SNeel Natu 
1429b1aa8d6SNeel Natu struct mem_map {
143318224bbSNeel Natu 	vm_paddr_t	gpa;
144318224bbSNeel Natu 	size_t		len;
1459b1aa8d6SNeel Natu 	vm_ooffset_t	segoff;
1469b1aa8d6SNeel Natu 	int		segid;
1479b1aa8d6SNeel Natu 	int		prot;
1489b1aa8d6SNeel Natu 	int		flags;
149318224bbSNeel Natu };
15000d3723fSConrad Meyer #define	VM_MAX_MEMMAPS	8
151366f6083SPeter Grehan 
152366f6083SPeter Grehan /*
1535fcf252fSNeel Natu  * Initialization:
1545fcf252fSNeel Natu  * (o) initialized the first time the VM is created
1555fcf252fSNeel Natu  * (i) initialized when VM is created and when it is reinitialized
1565fcf252fSNeel Natu  * (x) initialized before use
157366f6083SPeter Grehan  */
1585fcf252fSNeel Natu struct vm {
1595fcf252fSNeel Natu 	void		*cookie;		/* (i) cpu-specific data */
1605fcf252fSNeel Natu 	void		*iommu;			/* (x) iommu-specific data */
1615fcf252fSNeel Natu 	struct vhpet	*vhpet;			/* (i) virtual HPET */
1625fcf252fSNeel Natu 	struct vioapic	*vioapic;		/* (i) virtual ioapic */
1635fcf252fSNeel Natu 	struct vatpic	*vatpic;		/* (i) virtual atpic */
1645fcf252fSNeel Natu 	struct vatpit	*vatpit;		/* (i) virtual atpit */
165160ef77aSNeel Natu 	struct vpmtmr	*vpmtmr;		/* (i) virtual ACPI PM timer */
1660dafa5cdSNeel Natu 	struct vrtc	*vrtc;			/* (o) virtual RTC */
1675fcf252fSNeel Natu 	volatile cpuset_t active_cpus;		/* (i) active vcpus */
168fc276d92SJohn Baldwin 	volatile cpuset_t debug_cpus;		/* (i) vcpus stopped for debug */
1695fcf252fSNeel Natu 	int		suspend;		/* (i) stop VM execution */
1705fcf252fSNeel Natu 	volatile cpuset_t suspended_cpus; 	/* (i) suspended vcpus */
1715fcf252fSNeel Natu 	volatile cpuset_t halted_cpus;		/* (x) cpus in a hard halt */
1725fcf252fSNeel Natu 	cpuset_t	rendezvous_req_cpus;	/* (x) rendezvous requested */
1735fcf252fSNeel Natu 	cpuset_t	rendezvous_done_cpus;	/* (x) rendezvous finished */
1745fcf252fSNeel Natu 	void		*rendezvous_arg;	/* (x) rendezvous func/arg */
1755b8a8cd1SNeel Natu 	vm_rendezvous_func_t rendezvous_func;
1765fcf252fSNeel Natu 	struct mtx	rendezvous_mtx;		/* (o) rendezvous lock */
1779b1aa8d6SNeel Natu 	struct mem_map	mem_maps[VM_MAX_MEMMAPS]; /* (i) guest address space */
1789b1aa8d6SNeel Natu 	struct mem_seg	mem_segs[VM_MAX_MEMSEGS]; /* (o) guest memory regions */
1795fcf252fSNeel Natu 	struct vmspace	*vmspace;		/* (o) guest's address space */
180df95cc76SKa Ho Ng 	char		name[VM_MAX_NAMELEN+1];	/* (o) virtual machine name */
1815fcf252fSNeel Natu 	struct vcpu	vcpu[VM_MAXCPU];	/* (i) guest vcpus */
18201d822d3SRodney W. Grimes 	/* The following describe the vm cpu topology */
18301d822d3SRodney W. Grimes 	uint16_t	sockets;		/* (o) num of sockets */
18401d822d3SRodney W. Grimes 	uint16_t	cores;			/* (o) num of cores/socket */
18501d822d3SRodney W. Grimes 	uint16_t	threads;		/* (o) num of threads/core */
18601d822d3SRodney W. Grimes 	uint16_t	maxcpus;		/* (o) max pluggable cpus */
187366f6083SPeter Grehan };
188366f6083SPeter Grehan 
189950af9ffSJohn Baldwin #define	VMM_CTR0(vcpu, format)						\
190950af9ffSJohn Baldwin 	VCPU_CTR0((vcpu)->vm, (vcpu)->vcpuid, format)
191950af9ffSJohn Baldwin 
192950af9ffSJohn Baldwin #define	VMM_CTR1(vcpu, format, p1)					\
193950af9ffSJohn Baldwin 	VCPU_CTR1((vcpu)->vm, (vcpu)->vcpuid, format, p1)
194950af9ffSJohn Baldwin 
195950af9ffSJohn Baldwin #define	VMM_CTR2(vcpu, format, p1, p2)					\
196950af9ffSJohn Baldwin 	VCPU_CTR2((vcpu)->vm, (vcpu)->vcpuid, format, p1, p2)
197950af9ffSJohn Baldwin 
198950af9ffSJohn Baldwin #define	VMM_CTR3(vcpu, format, p1, p2, p3)				\
199950af9ffSJohn Baldwin 	VCPU_CTR3((vcpu)->vm, (vcpu)->vcpuid, format, p1, p2, p3)
200950af9ffSJohn Baldwin 
201950af9ffSJohn Baldwin #define	VMM_CTR4(vcpu, format, p1, p2, p3, p4)				\
202950af9ffSJohn Baldwin 	VCPU_CTR4((vcpu)->vm, (vcpu)->vcpuid, format, p1, p2, p3, p4)
203950af9ffSJohn Baldwin 
204d5408b1dSNeel Natu static int vmm_initialized;
205d5408b1dSNeel Natu 
20615add60dSPeter Grehan static void	vmmops_panic(void);
207366f6083SPeter Grehan 
20815add60dSPeter Grehan static void
20915add60dSPeter Grehan vmmops_panic(void)
21015add60dSPeter Grehan {
21115add60dSPeter Grehan 	panic("vmm_ops func called when !vmm_is_intel() && !vmm_is_svm()");
21215add60dSPeter Grehan }
21315add60dSPeter Grehan 
21415add60dSPeter Grehan #define	DEFINE_VMMOPS_IFUNC(ret_type, opname, args)			\
21515add60dSPeter Grehan     DEFINE_IFUNC(static, ret_type, vmmops_##opname, args)		\
21615add60dSPeter Grehan     {									\
21715add60dSPeter Grehan     	if (vmm_is_intel())						\
21815add60dSPeter Grehan     		return (vmm_ops_intel.opname);				\
21915add60dSPeter Grehan     	else if (vmm_is_svm())						\
22015add60dSPeter Grehan     		return (vmm_ops_amd.opname);				\
22115add60dSPeter Grehan     	else								\
22215add60dSPeter Grehan     		return ((ret_type (*)args)vmmops_panic);		\
22315add60dSPeter Grehan     }
22415add60dSPeter Grehan 
22515add60dSPeter Grehan DEFINE_VMMOPS_IFUNC(int, modinit, (int ipinum))
22615add60dSPeter Grehan DEFINE_VMMOPS_IFUNC(int, modcleanup, (void))
22715add60dSPeter Grehan DEFINE_VMMOPS_IFUNC(void, modresume, (void))
22815add60dSPeter Grehan DEFINE_VMMOPS_IFUNC(void *, init, (struct vm *vm, struct pmap *pmap))
229869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(int, run, (void *vcpui, register_t rip, struct pmap *pmap,
230869c8d19SJohn Baldwin     struct vm_eventinfo *info))
23115add60dSPeter Grehan DEFINE_VMMOPS_IFUNC(void, cleanup, (void *vmi))
232950af9ffSJohn Baldwin DEFINE_VMMOPS_IFUNC(void *, vcpu_init, (void *vmi, struct vcpu *vcpu,
233950af9ffSJohn Baldwin     int vcpu_id))
234869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(void, vcpu_cleanup, (void *vcpui))
235869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(int, getreg, (void *vcpui, int num, uint64_t *retval))
236869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(int, setreg, (void *vcpui, int num, uint64_t val))
237869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(int, getdesc, (void *vcpui, int num, struct seg_desc *desc))
238869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(int, setdesc, (void *vcpui, int num, struct seg_desc *desc))
239869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(int, getcap, (void *vcpui, int num, int *retval))
240869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(int, setcap, (void *vcpui, int num, int val))
24115add60dSPeter Grehan DEFINE_VMMOPS_IFUNC(struct vmspace *, vmspace_alloc, (vm_offset_t min,
24215add60dSPeter Grehan     vm_offset_t max))
24315add60dSPeter Grehan DEFINE_VMMOPS_IFUNC(void, vmspace_free, (struct vmspace *vmspace))
244869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(struct vlapic *, vlapic_init, (void *vcpui))
245869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(void, vlapic_cleanup, (struct vlapic *vlapic))
246483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT
247869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(int, snapshot, (void *vmi, struct vm_snapshot_meta *meta))
248869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(int, vcpu_snapshot, (void *vcpui,
249869c8d19SJohn Baldwin     struct vm_snapshot_meta *meta))
250869c8d19SJohn Baldwin DEFINE_VMMOPS_IFUNC(int, restore_tsc, (void *vcpui, uint64_t now))
251483d953aSJohn Baldwin #endif
252366f6083SPeter Grehan 
253014a52f3SNeel Natu #define	fpu_start_emulating()	load_cr0(rcr0() | CR0_TS)
254014a52f3SNeel Natu #define	fpu_stop_emulating()	clts()
255366f6083SPeter Grehan 
2566ac73777STycho Nightingale SDT_PROVIDER_DEFINE(vmm);
2576ac73777STycho Nightingale 
258366f6083SPeter Grehan static MALLOC_DEFINE(M_VM, "vm", "vm");
259366f6083SPeter Grehan 
260366f6083SPeter Grehan /* statistics */
26161592433SNeel Natu static VMM_STAT(VCPU_TOTAL_RUNTIME, "vcpu total runtime");
262366f6083SPeter Grehan 
263b40598c5SPawel Biernacki SYSCTL_NODE(_hw, OID_AUTO, vmm, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
264b40598c5SPawel Biernacki     NULL);
265add611fdSNeel Natu 
266055fc2cbSNeel Natu /*
267055fc2cbSNeel Natu  * Halt the guest if all vcpus are executing a HLT instruction with
268055fc2cbSNeel Natu  * interrupts disabled.
269055fc2cbSNeel Natu  */
270055fc2cbSNeel Natu static int halt_detection_enabled = 1;
271055fc2cbSNeel Natu SYSCTL_INT(_hw_vmm, OID_AUTO, halt_detection, CTLFLAG_RDTUN,
272055fc2cbSNeel Natu     &halt_detection_enabled, 0,
273055fc2cbSNeel Natu     "Halt VM if all vcpus execute HLT with interrupts disabled");
274055fc2cbSNeel Natu 
275978f3da1SAndriy Gapon static int vmm_ipinum;
276add611fdSNeel Natu SYSCTL_INT(_hw_vmm, OID_AUTO, ipinum, CTLFLAG_RD, &vmm_ipinum, 0,
277add611fdSNeel Natu     "IPI vector used for vcpu notifications");
278add611fdSNeel Natu 
279b0538143SNeel Natu static int trace_guest_exceptions;
280b0538143SNeel Natu SYSCTL_INT(_hw_vmm, OID_AUTO, trace_guest_exceptions, CTLFLAG_RDTUN,
281b0538143SNeel Natu     &trace_guest_exceptions, 0,
282b0538143SNeel Natu     "Trap into hypervisor on all guest exceptions and reflect them back");
283b0538143SNeel Natu 
2843ba952e1SCorvin Köhne static int trap_wbinvd;
2853ba952e1SCorvin Köhne SYSCTL_INT(_hw_vmm, OID_AUTO, trap_wbinvd, CTLFLAG_RDTUN, &trap_wbinvd, 0,
2863ba952e1SCorvin Köhne     "WBINVD triggers a VM-exit");
2873ba952e1SCorvin Köhne 
2889b1aa8d6SNeel Natu static void vm_free_memmap(struct vm *vm, int ident);
2899b1aa8d6SNeel Natu static bool sysmem_mapping(struct vm *vm, struct mem_map *mm);
290248e6799SNeel Natu static void vcpu_notify_event_locked(struct vcpu *vcpu, bool lapic_intr);
291248e6799SNeel Natu 
292248e6799SNeel Natu #ifdef KTR
293248e6799SNeel Natu static const char *
294248e6799SNeel Natu vcpu_state2str(enum vcpu_state state)
295248e6799SNeel Natu {
296248e6799SNeel Natu 
297248e6799SNeel Natu 	switch (state) {
298248e6799SNeel Natu 	case VCPU_IDLE:
299248e6799SNeel Natu 		return ("idle");
300248e6799SNeel Natu 	case VCPU_FROZEN:
301248e6799SNeel Natu 		return ("frozen");
302248e6799SNeel Natu 	case VCPU_RUNNING:
303248e6799SNeel Natu 		return ("running");
304248e6799SNeel Natu 	case VCPU_SLEEPING:
305248e6799SNeel Natu 		return ("sleeping");
306248e6799SNeel Natu 	default:
307248e6799SNeel Natu 		return ("unknown");
308248e6799SNeel Natu 	}
309248e6799SNeel Natu }
310248e6799SNeel Natu #endif
311248e6799SNeel Natu 
3121aa51504SJohn Baldwin static __inline void *
3131aa51504SJohn Baldwin vcpu_cookie(struct vm *vm, int i)
3141aa51504SJohn Baldwin {
3151aa51504SJohn Baldwin 	return (vm->vcpu[i].cookie);
3161aa51504SJohn Baldwin }
3171aa51504SJohn Baldwin 
318366f6083SPeter Grehan static void
3195fcf252fSNeel Natu vcpu_cleanup(struct vm *vm, int i, bool destroy)
320366f6083SPeter Grehan {
321de5ea6b6SNeel Natu 	struct vcpu *vcpu = &vm->vcpu[i];
322de5ea6b6SNeel Natu 
323869c8d19SJohn Baldwin 	vmmops_vlapic_cleanup(vcpu->vlapic);
324869c8d19SJohn Baldwin 	vmmops_vcpu_cleanup(vcpu->cookie);
3251aa51504SJohn Baldwin 	vcpu->cookie = NULL;
3265fcf252fSNeel Natu 	if (destroy) {
327366f6083SPeter Grehan 		vmm_stat_free(vcpu->stats);
32838f1b189SPeter Grehan 		fpu_save_area_free(vcpu->guestfpu);
329366f6083SPeter Grehan 	}
3305fcf252fSNeel Natu }
331366f6083SPeter Grehan 
332366f6083SPeter Grehan static void
3335fcf252fSNeel Natu vcpu_init(struct vm *vm, int vcpu_id, bool create)
334366f6083SPeter Grehan {
335366f6083SPeter Grehan 	struct vcpu *vcpu;
336366f6083SPeter Grehan 
337a488c9c9SRodney W. Grimes 	KASSERT(vcpu_id >= 0 && vcpu_id < vm->maxcpus,
3385fcf252fSNeel Natu 	    ("vcpu_init: invalid vcpu %d", vcpu_id));
3395fcf252fSNeel Natu 
340366f6083SPeter Grehan 	vcpu = &vm->vcpu[vcpu_id];
341366f6083SPeter Grehan 
3425fcf252fSNeel Natu 	if (create) {
3435fcf252fSNeel Natu 		KASSERT(!vcpu_lock_initialized(vcpu), ("vcpu %d already "
3445fcf252fSNeel Natu 		    "initialized", vcpu_id));
34575dd3366SNeel Natu 		vcpu_lock_init(vcpu);
3465fcf252fSNeel Natu 		vcpu->state = VCPU_IDLE;
34775dd3366SNeel Natu 		vcpu->hostcpu = NOCPU;
348950af9ffSJohn Baldwin 		vcpu->vcpuid = vcpu_id;
349950af9ffSJohn Baldwin 		vcpu->vm = vm;
3505fcf252fSNeel Natu 		vcpu->guestfpu = fpu_save_area_alloc();
3515fcf252fSNeel Natu 		vcpu->stats = vmm_stat_alloc();
352483d953aSJohn Baldwin 		vcpu->tsc_offset = 0;
3535fcf252fSNeel Natu 	}
3545fcf252fSNeel Natu 
355950af9ffSJohn Baldwin 	vcpu->cookie = vmmops_vcpu_init(vm->cookie, vcpu, vcpu_id);
356869c8d19SJohn Baldwin 	vcpu->vlapic = vmmops_vlapic_init(vcpu->cookie);
35752e5c8a2SNeel Natu 	vm_set_x2apic_state(vm, vcpu_id, X2APIC_DISABLED);
358248e6799SNeel Natu 	vcpu->reqidle = 0;
359091d4532SNeel Natu 	vcpu->exitintinfo = 0;
3605fcf252fSNeel Natu 	vcpu->nmi_pending = 0;
3615fcf252fSNeel Natu 	vcpu->extint_pending = 0;
3625fcf252fSNeel Natu 	vcpu->exception_pending = 0;
363abb023fbSJohn Baldwin 	vcpu->guest_xcr0 = XFEATURE_ENABLED_X87;
36438f1b189SPeter Grehan 	fpu_save_area_reset(vcpu->guestfpu);
3655fcf252fSNeel Natu 	vmm_stat_init(vcpu->stats);
366366f6083SPeter Grehan }
367366f6083SPeter Grehan 
368b0538143SNeel Natu int
369b0538143SNeel Natu vcpu_trace_exceptions(struct vm *vm, int vcpuid)
370b0538143SNeel Natu {
371b0538143SNeel Natu 
372b0538143SNeel Natu 	return (trace_guest_exceptions);
373b0538143SNeel Natu }
374b0538143SNeel Natu 
3753ba952e1SCorvin Köhne int
3763ba952e1SCorvin Köhne vcpu_trap_wbinvd(struct vm *vm, int vcpuid)
3773ba952e1SCorvin Köhne {
3783ba952e1SCorvin Köhne 	return (trap_wbinvd);
3793ba952e1SCorvin Köhne }
3803ba952e1SCorvin Köhne 
38198ed632cSNeel Natu struct vm_exit *
38298ed632cSNeel Natu vm_exitinfo(struct vm *vm, int cpuid)
38398ed632cSNeel Natu {
38498ed632cSNeel Natu 	struct vcpu *vcpu;
38598ed632cSNeel Natu 
386a488c9c9SRodney W. Grimes 	if (cpuid < 0 || cpuid >= vm->maxcpus)
38798ed632cSNeel Natu 		panic("vm_exitinfo: invalid cpuid %d", cpuid);
38898ed632cSNeel Natu 
38998ed632cSNeel Natu 	vcpu = &vm->vcpu[cpuid];
39098ed632cSNeel Natu 
39198ed632cSNeel Natu 	return (&vcpu->exitinfo);
39298ed632cSNeel Natu }
39398ed632cSNeel Natu 
394366f6083SPeter Grehan static int
395366f6083SPeter Grehan vmm_init(void)
396366f6083SPeter Grehan {
397366f6083SPeter Grehan 	int error;
398366f6083SPeter Grehan 
39915add60dSPeter Grehan 	if (!vmm_is_hw_supported())
40015add60dSPeter Grehan 		return (ENXIO);
40115add60dSPeter Grehan 
402b01c2033SNeel Natu 	vmm_host_state_init();
403add611fdSNeel Natu 
404bd50262fSKonstantin Belousov 	vmm_ipinum = lapic_ipi_alloc(pti ? &IDTVEC(justreturn1_pti) :
405bd50262fSKonstantin Belousov 	    &IDTVEC(justreturn));
40618a2b08eSNeel Natu 	if (vmm_ipinum < 0)
407add611fdSNeel Natu 		vmm_ipinum = IPI_AST;
408366f6083SPeter Grehan 
409366f6083SPeter Grehan 	error = vmm_mem_init();
410366f6083SPeter Grehan 	if (error)
411366f6083SPeter Grehan 		return (error);
412366f6083SPeter Grehan 
41315add60dSPeter Grehan 	vmm_resume_p = vmmops_modresume;
414366f6083SPeter Grehan 
41515add60dSPeter Grehan 	return (vmmops_modinit(vmm_ipinum));
416366f6083SPeter Grehan }
417366f6083SPeter Grehan 
418366f6083SPeter Grehan static int
419366f6083SPeter Grehan vmm_handler(module_t mod, int what, void *arg)
420366f6083SPeter Grehan {
421366f6083SPeter Grehan 	int error;
422366f6083SPeter Grehan 
423366f6083SPeter Grehan 	switch (what) {
424366f6083SPeter Grehan 	case MOD_LOAD:
42515add60dSPeter Grehan 		if (vmm_is_hw_supported()) {
426366f6083SPeter Grehan 			vmmdev_init();
427366f6083SPeter Grehan 			error = vmm_init();
428d5408b1dSNeel Natu 			if (error == 0)
429d5408b1dSNeel Natu 				vmm_initialized = 1;
43015add60dSPeter Grehan 		} else {
43115add60dSPeter Grehan 			error = ENXIO;
43215add60dSPeter Grehan 		}
433366f6083SPeter Grehan 		break;
434366f6083SPeter Grehan 	case MOD_UNLOAD:
43515add60dSPeter Grehan 		if (vmm_is_hw_supported()) {
436cdc5b9e7SNeel Natu 			error = vmmdev_cleanup();
437cdc5b9e7SNeel Natu 			if (error == 0) {
43863e62d39SJohn Baldwin 				vmm_resume_p = NULL;
439366f6083SPeter Grehan 				iommu_cleanup();
440add611fdSNeel Natu 				if (vmm_ipinum != IPI_AST)
44118a2b08eSNeel Natu 					lapic_ipi_free(vmm_ipinum);
44215add60dSPeter Grehan 				error = vmmops_modcleanup();
44381ef6611SPeter Grehan 				/*
44481ef6611SPeter Grehan 				 * Something bad happened - prevent new
44581ef6611SPeter Grehan 				 * VMs from being created
44681ef6611SPeter Grehan 				 */
44781ef6611SPeter Grehan 				if (error)
448d5408b1dSNeel Natu 					vmm_initialized = 0;
44981ef6611SPeter Grehan 			}
45015add60dSPeter Grehan 		} else {
45115add60dSPeter Grehan 			error = 0;
45215add60dSPeter Grehan 		}
453366f6083SPeter Grehan 		break;
454366f6083SPeter Grehan 	default:
455366f6083SPeter Grehan 		error = 0;
456366f6083SPeter Grehan 		break;
457366f6083SPeter Grehan 	}
458366f6083SPeter Grehan 	return (error);
459366f6083SPeter Grehan }
460366f6083SPeter Grehan 
461366f6083SPeter Grehan static moduledata_t vmm_kmod = {
462366f6083SPeter Grehan 	"vmm",
463366f6083SPeter Grehan 	vmm_handler,
464366f6083SPeter Grehan 	NULL
465366f6083SPeter Grehan };
466366f6083SPeter Grehan 
467366f6083SPeter Grehan /*
468e3f0800bSNeel Natu  * vmm initialization has the following dependencies:
469e3f0800bSNeel Natu  *
470e3f0800bSNeel Natu  * - VT-x initialization requires smp_rendezvous() and therefore must happen
471e3f0800bSNeel Natu  *   after SMP is fully functional (after SI_SUB_SMP).
472366f6083SPeter Grehan  */
473e3f0800bSNeel Natu DECLARE_MODULE(vmm, vmm_kmod, SI_SUB_SMP + 1, SI_ORDER_ANY);
474366f6083SPeter Grehan MODULE_VERSION(vmm, 1);
475366f6083SPeter Grehan 
4765fcf252fSNeel Natu static void
4775fcf252fSNeel Natu vm_init(struct vm *vm, bool create)
4785fcf252fSNeel Natu {
4795fcf252fSNeel Natu 	int i;
4805fcf252fSNeel Natu 
48115add60dSPeter Grehan 	vm->cookie = vmmops_init(vm, vmspace_pmap(vm->vmspace));
4825fcf252fSNeel Natu 	vm->iommu = NULL;
4835fcf252fSNeel Natu 	vm->vioapic = vioapic_init(vm);
4845fcf252fSNeel Natu 	vm->vhpet = vhpet_init(vm);
4855fcf252fSNeel Natu 	vm->vatpic = vatpic_init(vm);
4865fcf252fSNeel Natu 	vm->vatpit = vatpit_init(vm);
487160ef77aSNeel Natu 	vm->vpmtmr = vpmtmr_init(vm);
4880dafa5cdSNeel Natu 	if (create)
4890dafa5cdSNeel Natu 		vm->vrtc = vrtc_init(vm);
4905fcf252fSNeel Natu 
4915fcf252fSNeel Natu 	CPU_ZERO(&vm->active_cpus);
492fc276d92SJohn Baldwin 	CPU_ZERO(&vm->debug_cpus);
4935fcf252fSNeel Natu 
4945fcf252fSNeel Natu 	vm->suspend = 0;
4955fcf252fSNeel Natu 	CPU_ZERO(&vm->suspended_cpus);
4965fcf252fSNeel Natu 
497a488c9c9SRodney W. Grimes 	for (i = 0; i < vm->maxcpus; i++)
4985fcf252fSNeel Natu 		vcpu_init(vm, i, create);
4995fcf252fSNeel Natu }
5005fcf252fSNeel Natu 
50101d822d3SRodney W. Grimes /*
50201d822d3SRodney W. Grimes  * The default CPU topology is a single thread per package.
50301d822d3SRodney W. Grimes  */
50401d822d3SRodney W. Grimes u_int cores_per_package = 1;
50501d822d3SRodney W. Grimes u_int threads_per_core = 1;
50601d822d3SRodney W. Grimes 
507d5408b1dSNeel Natu int
508d5408b1dSNeel Natu vm_create(const char *name, struct vm **retvm)
509366f6083SPeter Grehan {
510366f6083SPeter Grehan 	struct vm *vm;
511318224bbSNeel Natu 	struct vmspace *vmspace;
512366f6083SPeter Grehan 
513d5408b1dSNeel Natu 	/*
514d5408b1dSNeel Natu 	 * If vmm.ko could not be successfully initialized then don't attempt
515d5408b1dSNeel Natu 	 * to create the virtual machine.
516d5408b1dSNeel Natu 	 */
517d5408b1dSNeel Natu 	if (!vmm_initialized)
518d5408b1dSNeel Natu 		return (ENXIO);
519d5408b1dSNeel Natu 
520df95cc76SKa Ho Ng 	if (name == NULL || strnlen(name, VM_MAX_NAMELEN + 1) ==
521df95cc76SKa Ho Ng 	    VM_MAX_NAMELEN + 1)
522d5408b1dSNeel Natu 		return (EINVAL);
523366f6083SPeter Grehan 
5243c48106aSKonstantin Belousov 	vmspace = vmmops_vmspace_alloc(0, VM_MAXUSER_ADDRESS_LA48);
525318224bbSNeel Natu 	if (vmspace == NULL)
526318224bbSNeel Natu 		return (ENOMEM);
527318224bbSNeel Natu 
528366f6083SPeter Grehan 	vm = malloc(sizeof(struct vm), M_VM, M_WAITOK | M_ZERO);
529366f6083SPeter Grehan 	strcpy(vm->name, name);
53088c4b8d1SNeel Natu 	vm->vmspace = vmspace;
5315b8a8cd1SNeel Natu 	mtx_init(&vm->rendezvous_mtx, "vm rendezvous lock", 0, MTX_DEF);
532366f6083SPeter Grehan 
53301d822d3SRodney W. Grimes 	vm->sockets = 1;
53401d822d3SRodney W. Grimes 	vm->cores = cores_per_package;	/* XXX backwards compatibility */
53501d822d3SRodney W. Grimes 	vm->threads = threads_per_core;	/* XXX backwards compatibility */
536a488c9c9SRodney W. Grimes 	vm->maxcpus = VM_MAXCPU;	/* XXX temp to keep code working */
53701d822d3SRodney W. Grimes 
5385fcf252fSNeel Natu 	vm_init(vm, true);
539366f6083SPeter Grehan 
540d5408b1dSNeel Natu 	*retvm = vm;
541d5408b1dSNeel Natu 	return (0);
542366f6083SPeter Grehan }
543366f6083SPeter Grehan 
54401d822d3SRodney W. Grimes void
54501d822d3SRodney W. Grimes vm_get_topology(struct vm *vm, uint16_t *sockets, uint16_t *cores,
54601d822d3SRodney W. Grimes     uint16_t *threads, uint16_t *maxcpus)
54701d822d3SRodney W. Grimes {
54801d822d3SRodney W. Grimes 	*sockets = vm->sockets;
54901d822d3SRodney W. Grimes 	*cores = vm->cores;
55001d822d3SRodney W. Grimes 	*threads = vm->threads;
55101d822d3SRodney W. Grimes 	*maxcpus = vm->maxcpus;
55201d822d3SRodney W. Grimes }
55301d822d3SRodney W. Grimes 
554a488c9c9SRodney W. Grimes uint16_t
555a488c9c9SRodney W. Grimes vm_get_maxcpus(struct vm *vm)
556a488c9c9SRodney W. Grimes {
557a488c9c9SRodney W. Grimes 	return (vm->maxcpus);
558a488c9c9SRodney W. Grimes }
559a488c9c9SRodney W. Grimes 
56001d822d3SRodney W. Grimes int
56101d822d3SRodney W. Grimes vm_set_topology(struct vm *vm, uint16_t sockets, uint16_t cores,
56201d822d3SRodney W. Grimes     uint16_t threads, uint16_t maxcpus)
56301d822d3SRodney W. Grimes {
56401d822d3SRodney W. Grimes 	if (maxcpus != 0)
56501d822d3SRodney W. Grimes 		return (EINVAL);	/* XXX remove when supported */
566a488c9c9SRodney W. Grimes 	if ((sockets * cores * threads) > vm->maxcpus)
56701d822d3SRodney W. Grimes 		return (EINVAL);
56801d822d3SRodney W. Grimes 	/* XXX need to check sockets * cores * threads == vCPU, how? */
56901d822d3SRodney W. Grimes 	vm->sockets = sockets;
57001d822d3SRodney W. Grimes 	vm->cores = cores;
57101d822d3SRodney W. Grimes 	vm->threads = threads;
572a488c9c9SRodney W. Grimes 	vm->maxcpus = VM_MAXCPU;	/* XXX temp to keep code working */
57301d822d3SRodney W. Grimes 	return(0);
57401d822d3SRodney W. Grimes }
57501d822d3SRodney W. Grimes 
576f7d51510SNeel Natu static void
5775fcf252fSNeel Natu vm_cleanup(struct vm *vm, bool destroy)
578366f6083SPeter Grehan {
5799b1aa8d6SNeel Natu 	struct mem_map *mm;
580366f6083SPeter Grehan 	int i;
581366f6083SPeter Grehan 
582366f6083SPeter Grehan 	ppt_unassign_all(vm);
583366f6083SPeter Grehan 
584318224bbSNeel Natu 	if (vm->iommu != NULL)
585318224bbSNeel Natu 		iommu_destroy_domain(vm->iommu);
586318224bbSNeel Natu 
5870dafa5cdSNeel Natu 	if (destroy)
5880dafa5cdSNeel Natu 		vrtc_cleanup(vm->vrtc);
5890dafa5cdSNeel Natu 	else
5900dafa5cdSNeel Natu 		vrtc_reset(vm->vrtc);
591160ef77aSNeel Natu 	vpmtmr_cleanup(vm->vpmtmr);
592e883c9bbSTycho Nightingale 	vatpit_cleanup(vm->vatpit);
59308e3ff32SNeel Natu 	vhpet_cleanup(vm->vhpet);
594762fd208STycho Nightingale 	vatpic_cleanup(vm->vatpic);
59508e3ff32SNeel Natu 	vioapic_cleanup(vm->vioapic);
59608e3ff32SNeel Natu 
597a488c9c9SRodney W. Grimes 	for (i = 0; i < vm->maxcpus; i++)
5985fcf252fSNeel Natu 		vcpu_cleanup(vm, i, destroy);
5995fcf252fSNeel Natu 
60015add60dSPeter Grehan 	vmmops_cleanup(vm->cookie);
6015fcf252fSNeel Natu 
6029b1aa8d6SNeel Natu 	/*
6039b1aa8d6SNeel Natu 	 * System memory is removed from the guest address space only when
6049b1aa8d6SNeel Natu 	 * the VM is destroyed. This is because the mapping remains the same
6059b1aa8d6SNeel Natu 	 * across VM reset.
6069b1aa8d6SNeel Natu 	 *
6079b1aa8d6SNeel Natu 	 * Device memory can be relocated by the guest (e.g. using PCI BARs)
6089b1aa8d6SNeel Natu 	 * so those mappings are removed on a VM reset.
6099b1aa8d6SNeel Natu 	 */
6109b1aa8d6SNeel Natu 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
6119b1aa8d6SNeel Natu 		mm = &vm->mem_maps[i];
6129b1aa8d6SNeel Natu 		if (destroy || !sysmem_mapping(vm, mm))
6139b1aa8d6SNeel Natu 			vm_free_memmap(vm, i);
6149b1aa8d6SNeel Natu 	}
615f7d51510SNeel Natu 
6169b1aa8d6SNeel Natu 	if (destroy) {
6179b1aa8d6SNeel Natu 		for (i = 0; i < VM_MAX_MEMSEGS; i++)
6189b1aa8d6SNeel Natu 			vm_free_memseg(vm, i);
619366f6083SPeter Grehan 
62015add60dSPeter Grehan 		vmmops_vmspace_free(vm->vmspace);
6215fcf252fSNeel Natu 		vm->vmspace = NULL;
6225fcf252fSNeel Natu 	}
6235fcf252fSNeel Natu }
624366f6083SPeter Grehan 
6255fcf252fSNeel Natu void
6265fcf252fSNeel Natu vm_destroy(struct vm *vm)
6275fcf252fSNeel Natu {
6285fcf252fSNeel Natu 	vm_cleanup(vm, true);
629366f6083SPeter Grehan 	free(vm, M_VM);
630366f6083SPeter Grehan }
631366f6083SPeter Grehan 
6325fcf252fSNeel Natu int
6335fcf252fSNeel Natu vm_reinit(struct vm *vm)
6345fcf252fSNeel Natu {
6355fcf252fSNeel Natu 	int error;
6365fcf252fSNeel Natu 
6375fcf252fSNeel Natu 	/*
6385fcf252fSNeel Natu 	 * A virtual machine can be reset only if all vcpus are suspended.
6395fcf252fSNeel Natu 	 */
6405fcf252fSNeel Natu 	if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) {
6415fcf252fSNeel Natu 		vm_cleanup(vm, false);
6425fcf252fSNeel Natu 		vm_init(vm, false);
6435fcf252fSNeel Natu 		error = 0;
6445fcf252fSNeel Natu 	} else {
6455fcf252fSNeel Natu 		error = EBUSY;
6465fcf252fSNeel Natu 	}
6475fcf252fSNeel Natu 
6485fcf252fSNeel Natu 	return (error);
6495fcf252fSNeel Natu }
6505fcf252fSNeel Natu 
651366f6083SPeter Grehan const char *
652366f6083SPeter Grehan vm_name(struct vm *vm)
653366f6083SPeter Grehan {
654366f6083SPeter Grehan 	return (vm->name);
655366f6083SPeter Grehan }
656366f6083SPeter Grehan 
657366f6083SPeter Grehan int
658366f6083SPeter Grehan vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa)
659366f6083SPeter Grehan {
660318224bbSNeel Natu 	vm_object_t obj;
661366f6083SPeter Grehan 
662318224bbSNeel Natu 	if ((obj = vmm_mmio_alloc(vm->vmspace, gpa, len, hpa)) == NULL)
663318224bbSNeel Natu 		return (ENOMEM);
664318224bbSNeel Natu 	else
665318224bbSNeel Natu 		return (0);
666366f6083SPeter Grehan }
667366f6083SPeter Grehan 
668366f6083SPeter Grehan int
669366f6083SPeter Grehan vm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len)
670366f6083SPeter Grehan {
671366f6083SPeter Grehan 
672318224bbSNeel Natu 	vmm_mmio_free(vm->vmspace, gpa, len);
673318224bbSNeel Natu 	return (0);
674366f6083SPeter Grehan }
675366f6083SPeter Grehan 
6769b1aa8d6SNeel Natu /*
6779b1aa8d6SNeel Natu  * Return 'true' if 'gpa' is allocated in the guest address space.
6789b1aa8d6SNeel Natu  *
6799b1aa8d6SNeel Natu  * This function is called in the context of a running vcpu which acts as
6809b1aa8d6SNeel Natu  * an implicit lock on 'vm->mem_maps[]'.
6819b1aa8d6SNeel Natu  */
6829b1aa8d6SNeel Natu bool
6839b1aa8d6SNeel Natu vm_mem_allocated(struct vm *vm, int vcpuid, vm_paddr_t gpa)
684366f6083SPeter Grehan {
6859b1aa8d6SNeel Natu 	struct mem_map *mm;
686341f19c9SNeel Natu 	int i;
687341f19c9SNeel Natu 
6889b1aa8d6SNeel Natu #ifdef INVARIANTS
6899b1aa8d6SNeel Natu 	int hostcpu, state;
6909b1aa8d6SNeel Natu 	state = vcpu_get_state(vm, vcpuid, &hostcpu);
6919b1aa8d6SNeel Natu 	KASSERT(state == VCPU_RUNNING && hostcpu == curcpu,
6929b1aa8d6SNeel Natu 	    ("%s: invalid vcpu state %d/%d", __func__, state, hostcpu));
6939b1aa8d6SNeel Natu #endif
6949b1aa8d6SNeel Natu 
6959b1aa8d6SNeel Natu 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
6969b1aa8d6SNeel Natu 		mm = &vm->mem_maps[i];
6979b1aa8d6SNeel Natu 		if (mm->len != 0 && gpa >= mm->gpa && gpa < mm->gpa + mm->len)
6989b1aa8d6SNeel Natu 			return (true);		/* 'gpa' is sysmem or devmem */
699341f19c9SNeel Natu 	}
700341f19c9SNeel Natu 
701318224bbSNeel Natu 	if (ppt_is_mmio(vm, gpa))
7029b1aa8d6SNeel Natu 		return (true);			/* 'gpa' is pci passthru mmio */
703318224bbSNeel Natu 
7049b1aa8d6SNeel Natu 	return (false);
705341f19c9SNeel Natu }
706341f19c9SNeel Natu 
707341f19c9SNeel Natu int
7089b1aa8d6SNeel Natu vm_alloc_memseg(struct vm *vm, int ident, size_t len, bool sysmem)
709341f19c9SNeel Natu {
710318224bbSNeel Natu 	struct mem_seg *seg;
7119b1aa8d6SNeel Natu 	vm_object_t obj;
712366f6083SPeter Grehan 
7139b1aa8d6SNeel Natu 	if (ident < 0 || ident >= VM_MAX_MEMSEGS)
714341f19c9SNeel Natu 		return (EINVAL);
715341f19c9SNeel Natu 
7169b1aa8d6SNeel Natu 	if (len == 0 || (len & PAGE_MASK))
7179b1aa8d6SNeel Natu 		return (EINVAL);
718341f19c9SNeel Natu 
7199b1aa8d6SNeel Natu 	seg = &vm->mem_segs[ident];
7209b1aa8d6SNeel Natu 	if (seg->object != NULL) {
7219b1aa8d6SNeel Natu 		if (seg->len == len && seg->sysmem == sysmem)
7229b1aa8d6SNeel Natu 			return (EEXIST);
7239b1aa8d6SNeel Natu 		else
7249b1aa8d6SNeel Natu 			return (EINVAL);
725341f19c9SNeel Natu 	}
726341f19c9SNeel Natu 
7276b389740SMark Johnston 	obj = vm_object_allocate(OBJT_SWAP, len >> PAGE_SHIFT);
7289b1aa8d6SNeel Natu 	if (obj == NULL)
729318224bbSNeel Natu 		return (ENOMEM);
730318224bbSNeel Natu 
731318224bbSNeel Natu 	seg->len = len;
7329b1aa8d6SNeel Natu 	seg->object = obj;
7339b1aa8d6SNeel Natu 	seg->sysmem = sysmem;
734366f6083SPeter Grehan 	return (0);
735366f6083SPeter Grehan }
736366f6083SPeter Grehan 
7379b1aa8d6SNeel Natu int
7389b1aa8d6SNeel Natu vm_get_memseg(struct vm *vm, int ident, size_t *len, bool *sysmem,
7399b1aa8d6SNeel Natu     vm_object_t *objptr)
740477867a0SNeel Natu {
7419b1aa8d6SNeel Natu 	struct mem_seg *seg;
742477867a0SNeel Natu 
7439b1aa8d6SNeel Natu 	if (ident < 0 || ident >= VM_MAX_MEMSEGS)
7449b1aa8d6SNeel Natu 		return (EINVAL);
7459b1aa8d6SNeel Natu 
7469b1aa8d6SNeel Natu 	seg = &vm->mem_segs[ident];
7479b1aa8d6SNeel Natu 	if (len)
7489b1aa8d6SNeel Natu 		*len = seg->len;
7499b1aa8d6SNeel Natu 	if (sysmem)
7509b1aa8d6SNeel Natu 		*sysmem = seg->sysmem;
7519b1aa8d6SNeel Natu 	if (objptr)
7529b1aa8d6SNeel Natu 		*objptr = seg->object;
7539b1aa8d6SNeel Natu 	return (0);
754477867a0SNeel Natu }
7559b1aa8d6SNeel Natu 
7569b1aa8d6SNeel Natu void
7579b1aa8d6SNeel Natu vm_free_memseg(struct vm *vm, int ident)
7589b1aa8d6SNeel Natu {
7599b1aa8d6SNeel Natu 	struct mem_seg *seg;
7609b1aa8d6SNeel Natu 
7619b1aa8d6SNeel Natu 	KASSERT(ident >= 0 && ident < VM_MAX_MEMSEGS,
7629b1aa8d6SNeel Natu 	    ("%s: invalid memseg ident %d", __func__, ident));
7639b1aa8d6SNeel Natu 
7649b1aa8d6SNeel Natu 	seg = &vm->mem_segs[ident];
7659b1aa8d6SNeel Natu 	if (seg->object != NULL) {
7669b1aa8d6SNeel Natu 		vm_object_deallocate(seg->object);
7679b1aa8d6SNeel Natu 		bzero(seg, sizeof(struct mem_seg));
7689b1aa8d6SNeel Natu 	}
7699b1aa8d6SNeel Natu }
7709b1aa8d6SNeel Natu 
7719b1aa8d6SNeel Natu int
7729b1aa8d6SNeel Natu vm_mmap_memseg(struct vm *vm, vm_paddr_t gpa, int segid, vm_ooffset_t first,
7739b1aa8d6SNeel Natu     size_t len, int prot, int flags)
7749b1aa8d6SNeel Natu {
7759b1aa8d6SNeel Natu 	struct mem_seg *seg;
7769b1aa8d6SNeel Natu 	struct mem_map *m, *map;
7779b1aa8d6SNeel Natu 	vm_ooffset_t last;
7789b1aa8d6SNeel Natu 	int i, error;
7799b1aa8d6SNeel Natu 
7809b1aa8d6SNeel Natu 	if (prot == 0 || (prot & ~(VM_PROT_ALL)) != 0)
7819b1aa8d6SNeel Natu 		return (EINVAL);
7829b1aa8d6SNeel Natu 
7839b1aa8d6SNeel Natu 	if (flags & ~VM_MEMMAP_F_WIRED)
7849b1aa8d6SNeel Natu 		return (EINVAL);
7859b1aa8d6SNeel Natu 
7869b1aa8d6SNeel Natu 	if (segid < 0 || segid >= VM_MAX_MEMSEGS)
7879b1aa8d6SNeel Natu 		return (EINVAL);
7889b1aa8d6SNeel Natu 
7899b1aa8d6SNeel Natu 	seg = &vm->mem_segs[segid];
7909b1aa8d6SNeel Natu 	if (seg->object == NULL)
7919b1aa8d6SNeel Natu 		return (EINVAL);
7929b1aa8d6SNeel Natu 
7939b1aa8d6SNeel Natu 	last = first + len;
7949b1aa8d6SNeel Natu 	if (first < 0 || first >= last || last > seg->len)
7959b1aa8d6SNeel Natu 		return (EINVAL);
7969b1aa8d6SNeel Natu 
7979b1aa8d6SNeel Natu 	if ((gpa | first | last) & PAGE_MASK)
7989b1aa8d6SNeel Natu 		return (EINVAL);
7999b1aa8d6SNeel Natu 
8009b1aa8d6SNeel Natu 	map = NULL;
8019b1aa8d6SNeel Natu 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
8029b1aa8d6SNeel Natu 		m = &vm->mem_maps[i];
8039b1aa8d6SNeel Natu 		if (m->len == 0) {
8049b1aa8d6SNeel Natu 			map = m;
8059b1aa8d6SNeel Natu 			break;
8069b1aa8d6SNeel Natu 		}
8079b1aa8d6SNeel Natu 	}
8089b1aa8d6SNeel Natu 
8099b1aa8d6SNeel Natu 	if (map == NULL)
8109b1aa8d6SNeel Natu 		return (ENOSPC);
8119b1aa8d6SNeel Natu 
8129b1aa8d6SNeel Natu 	error = vm_map_find(&vm->vmspace->vm_map, seg->object, first, &gpa,
8139b1aa8d6SNeel Natu 	    len, 0, VMFS_NO_SPACE, prot, prot, 0);
8149b1aa8d6SNeel Natu 	if (error != KERN_SUCCESS)
8159b1aa8d6SNeel Natu 		return (EFAULT);
8169b1aa8d6SNeel Natu 
8179b1aa8d6SNeel Natu 	vm_object_reference(seg->object);
8189b1aa8d6SNeel Natu 
8199b1aa8d6SNeel Natu 	if (flags & VM_MEMMAP_F_WIRED) {
8209b1aa8d6SNeel Natu 		error = vm_map_wire(&vm->vmspace->vm_map, gpa, gpa + len,
8219b1aa8d6SNeel Natu 		    VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
8229b1aa8d6SNeel Natu 		if (error != KERN_SUCCESS) {
8239b1aa8d6SNeel Natu 			vm_map_remove(&vm->vmspace->vm_map, gpa, gpa + len);
82454a3a114SMark Johnston 			return (error == KERN_RESOURCE_SHORTAGE ? ENOMEM :
82554a3a114SMark Johnston 			    EFAULT);
8269b1aa8d6SNeel Natu 		}
8279b1aa8d6SNeel Natu 	}
8289b1aa8d6SNeel Natu 
8299b1aa8d6SNeel Natu 	map->gpa = gpa;
8309b1aa8d6SNeel Natu 	map->len = len;
8319b1aa8d6SNeel Natu 	map->segoff = first;
8329b1aa8d6SNeel Natu 	map->segid = segid;
8339b1aa8d6SNeel Natu 	map->prot = prot;
8349b1aa8d6SNeel Natu 	map->flags = flags;
8359b1aa8d6SNeel Natu 	return (0);
8369b1aa8d6SNeel Natu }
8379b1aa8d6SNeel Natu 
8389b1aa8d6SNeel Natu int
839f8a6ec2dSD Scott Phillips vm_munmap_memseg(struct vm *vm, vm_paddr_t gpa, size_t len)
840f8a6ec2dSD Scott Phillips {
841f8a6ec2dSD Scott Phillips 	struct mem_map *m;
842f8a6ec2dSD Scott Phillips 	int i;
843f8a6ec2dSD Scott Phillips 
844f8a6ec2dSD Scott Phillips 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
845f8a6ec2dSD Scott Phillips 		m = &vm->mem_maps[i];
846f8a6ec2dSD Scott Phillips 		if (m->gpa == gpa && m->len == len &&
847f8a6ec2dSD Scott Phillips 		    (m->flags & VM_MEMMAP_F_IOMMU) == 0) {
848f8a6ec2dSD Scott Phillips 			vm_free_memmap(vm, i);
849f8a6ec2dSD Scott Phillips 			return (0);
850f8a6ec2dSD Scott Phillips 		}
851f8a6ec2dSD Scott Phillips 	}
852f8a6ec2dSD Scott Phillips 
853f8a6ec2dSD Scott Phillips 	return (EINVAL);
854f8a6ec2dSD Scott Phillips }
855f8a6ec2dSD Scott Phillips 
856f8a6ec2dSD Scott Phillips int
8579b1aa8d6SNeel Natu vm_mmap_getnext(struct vm *vm, vm_paddr_t *gpa, int *segid,
8589b1aa8d6SNeel Natu     vm_ooffset_t *segoff, size_t *len, int *prot, int *flags)
8599b1aa8d6SNeel Natu {
8609b1aa8d6SNeel Natu 	struct mem_map *mm, *mmnext;
8619b1aa8d6SNeel Natu 	int i;
8629b1aa8d6SNeel Natu 
8639b1aa8d6SNeel Natu 	mmnext = NULL;
8649b1aa8d6SNeel Natu 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
8659b1aa8d6SNeel Natu 		mm = &vm->mem_maps[i];
8669b1aa8d6SNeel Natu 		if (mm->len == 0 || mm->gpa < *gpa)
8679b1aa8d6SNeel Natu 			continue;
8689b1aa8d6SNeel Natu 		if (mmnext == NULL || mm->gpa < mmnext->gpa)
8699b1aa8d6SNeel Natu 			mmnext = mm;
8709b1aa8d6SNeel Natu 	}
8719b1aa8d6SNeel Natu 
8729b1aa8d6SNeel Natu 	if (mmnext != NULL) {
8739b1aa8d6SNeel Natu 		*gpa = mmnext->gpa;
8749b1aa8d6SNeel Natu 		if (segid)
8759b1aa8d6SNeel Natu 			*segid = mmnext->segid;
8769b1aa8d6SNeel Natu 		if (segoff)
8779b1aa8d6SNeel Natu 			*segoff = mmnext->segoff;
8789b1aa8d6SNeel Natu 		if (len)
8799b1aa8d6SNeel Natu 			*len = mmnext->len;
8809b1aa8d6SNeel Natu 		if (prot)
8819b1aa8d6SNeel Natu 			*prot = mmnext->prot;
8829b1aa8d6SNeel Natu 		if (flags)
8839b1aa8d6SNeel Natu 			*flags = mmnext->flags;
8849b1aa8d6SNeel Natu 		return (0);
8859b1aa8d6SNeel Natu 	} else {
8869b1aa8d6SNeel Natu 		return (ENOENT);
8879b1aa8d6SNeel Natu 	}
888477867a0SNeel Natu }
889477867a0SNeel Natu 
890318224bbSNeel Natu static void
8919b1aa8d6SNeel Natu vm_free_memmap(struct vm *vm, int ident)
892366f6083SPeter Grehan {
8939b1aa8d6SNeel Natu 	struct mem_map *mm;
89473505a10SRobert Wing 	int error __diagused;
8954db4fb2cSNeel Natu 
8969b1aa8d6SNeel Natu 	mm = &vm->mem_maps[ident];
8979b1aa8d6SNeel Natu 	if (mm->len) {
8989b1aa8d6SNeel Natu 		error = vm_map_remove(&vm->vmspace->vm_map, mm->gpa,
8999b1aa8d6SNeel Natu 		    mm->gpa + mm->len);
9009b1aa8d6SNeel Natu 		KASSERT(error == KERN_SUCCESS, ("%s: vm_map_remove error %d",
9019b1aa8d6SNeel Natu 		    __func__, error));
9029b1aa8d6SNeel Natu 		bzero(mm, sizeof(struct mem_map));
903318224bbSNeel Natu 	}
904318224bbSNeel Natu }
905318224bbSNeel Natu 
9069b1aa8d6SNeel Natu static __inline bool
9079b1aa8d6SNeel Natu sysmem_mapping(struct vm *vm, struct mem_map *mm)
908318224bbSNeel Natu {
909318224bbSNeel Natu 
9109b1aa8d6SNeel Natu 	if (mm->len != 0 && vm->mem_segs[mm->segid].sysmem)
9119b1aa8d6SNeel Natu 		return (true);
9129b1aa8d6SNeel Natu 	else
9139b1aa8d6SNeel Natu 		return (false);
914318224bbSNeel Natu }
915318224bbSNeel Natu 
916147d12a7SAntoine Brodin vm_paddr_t
917147d12a7SAntoine Brodin vmm_sysmem_maxaddr(struct vm *vm)
9189b1aa8d6SNeel Natu {
9199b1aa8d6SNeel Natu 	struct mem_map *mm;
9209b1aa8d6SNeel Natu 	vm_paddr_t maxaddr;
9219b1aa8d6SNeel Natu 	int i;
922318224bbSNeel Natu 
9239b1aa8d6SNeel Natu 	maxaddr = 0;
9249b1aa8d6SNeel Natu 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
9259b1aa8d6SNeel Natu 		mm = &vm->mem_maps[i];
9269b1aa8d6SNeel Natu 		if (sysmem_mapping(vm, mm)) {
9279b1aa8d6SNeel Natu 			if (maxaddr < mm->gpa + mm->len)
9289b1aa8d6SNeel Natu 				maxaddr = mm->gpa + mm->len;
9299b1aa8d6SNeel Natu 		}
9309b1aa8d6SNeel Natu 	}
9319b1aa8d6SNeel Natu 	return (maxaddr);
932318224bbSNeel Natu }
933318224bbSNeel Natu 
934318224bbSNeel Natu static void
935490d56c5SEd Maste vm_iommu_modify(struct vm *vm, bool map)
936318224bbSNeel Natu {
937318224bbSNeel Natu 	int i, sz;
938318224bbSNeel Natu 	vm_paddr_t gpa, hpa;
9399b1aa8d6SNeel Natu 	struct mem_map *mm;
940318224bbSNeel Natu 	void *vp, *cookie, *host_domain;
941318224bbSNeel Natu 
942318224bbSNeel Natu 	sz = PAGE_SIZE;
943318224bbSNeel Natu 	host_domain = iommu_host_domain();
944318224bbSNeel Natu 
9459b1aa8d6SNeel Natu 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
9469b1aa8d6SNeel Natu 		mm = &vm->mem_maps[i];
9479b1aa8d6SNeel Natu 		if (!sysmem_mapping(vm, mm))
9489b1aa8d6SNeel Natu 			continue;
949318224bbSNeel Natu 
9509b1aa8d6SNeel Natu 		if (map) {
9519b1aa8d6SNeel Natu 			KASSERT((mm->flags & VM_MEMMAP_F_IOMMU) == 0,
9529b1aa8d6SNeel Natu 			    ("iommu map found invalid memmap %#lx/%#lx/%#x",
9539b1aa8d6SNeel Natu 			    mm->gpa, mm->len, mm->flags));
9549b1aa8d6SNeel Natu 			if ((mm->flags & VM_MEMMAP_F_WIRED) == 0)
9559b1aa8d6SNeel Natu 				continue;
9569b1aa8d6SNeel Natu 			mm->flags |= VM_MEMMAP_F_IOMMU;
9579b1aa8d6SNeel Natu 		} else {
9589b1aa8d6SNeel Natu 			if ((mm->flags & VM_MEMMAP_F_IOMMU) == 0)
9599b1aa8d6SNeel Natu 				continue;
9609b1aa8d6SNeel Natu 			mm->flags &= ~VM_MEMMAP_F_IOMMU;
9619b1aa8d6SNeel Natu 			KASSERT((mm->flags & VM_MEMMAP_F_WIRED) != 0,
9629b1aa8d6SNeel Natu 			    ("iommu unmap found invalid memmap %#lx/%#lx/%#x",
9639b1aa8d6SNeel Natu 			    mm->gpa, mm->len, mm->flags));
9649b1aa8d6SNeel Natu 		}
9659b1aa8d6SNeel Natu 
9669b1aa8d6SNeel Natu 		gpa = mm->gpa;
9679b1aa8d6SNeel Natu 		while (gpa < mm->gpa + mm->len) {
9689b1aa8d6SNeel Natu 			vp = vm_gpa_hold(vm, -1, gpa, PAGE_SIZE, VM_PROT_WRITE,
969318224bbSNeel Natu 					 &cookie);
970318224bbSNeel Natu 			KASSERT(vp != NULL, ("vm(%s) could not map gpa %#lx",
971318224bbSNeel Natu 			    vm_name(vm), gpa));
972318224bbSNeel Natu 
973318224bbSNeel Natu 			vm_gpa_release(cookie);
974318224bbSNeel Natu 
975318224bbSNeel Natu 			hpa = DMAP_TO_PHYS((uintptr_t)vp);
976318224bbSNeel Natu 			if (map) {
977318224bbSNeel Natu 				iommu_create_mapping(vm->iommu, gpa, hpa, sz);
978318224bbSNeel Natu 			} else {
979318224bbSNeel Natu 				iommu_remove_mapping(vm->iommu, gpa, sz);
980318224bbSNeel Natu 			}
981318224bbSNeel Natu 
982318224bbSNeel Natu 			gpa += PAGE_SIZE;
983318224bbSNeel Natu 		}
984318224bbSNeel Natu 	}
985318224bbSNeel Natu 
986318224bbSNeel Natu 	/*
987318224bbSNeel Natu 	 * Invalidate the cached translations associated with the domain
988318224bbSNeel Natu 	 * from which pages were removed.
989318224bbSNeel Natu 	 */
990318224bbSNeel Natu 	if (map)
991318224bbSNeel Natu 		iommu_invalidate_tlb(host_domain);
992318224bbSNeel Natu 	else
993318224bbSNeel Natu 		iommu_invalidate_tlb(vm->iommu);
994318224bbSNeel Natu }
995318224bbSNeel Natu 
996490d56c5SEd Maste #define	vm_iommu_unmap(vm)	vm_iommu_modify((vm), false)
997490d56c5SEd Maste #define	vm_iommu_map(vm)	vm_iommu_modify((vm), true)
998318224bbSNeel Natu 
999318224bbSNeel Natu int
1000318224bbSNeel Natu vm_unassign_pptdev(struct vm *vm, int bus, int slot, int func)
1001318224bbSNeel Natu {
1002318224bbSNeel Natu 	int error;
1003318224bbSNeel Natu 
1004318224bbSNeel Natu 	error = ppt_unassign_device(vm, bus, slot, func);
1005318224bbSNeel Natu 	if (error)
1006318224bbSNeel Natu 		return (error);
1007318224bbSNeel Natu 
10089b1aa8d6SNeel Natu 	if (ppt_assigned_devices(vm) == 0)
1009318224bbSNeel Natu 		vm_iommu_unmap(vm);
10109b1aa8d6SNeel Natu 
1011318224bbSNeel Natu 	return (0);
1012318224bbSNeel Natu }
1013318224bbSNeel Natu 
1014318224bbSNeel Natu int
1015318224bbSNeel Natu vm_assign_pptdev(struct vm *vm, int bus, int slot, int func)
1016318224bbSNeel Natu {
1017318224bbSNeel Natu 	int error;
1018318224bbSNeel Natu 	vm_paddr_t maxaddr;
1019318224bbSNeel Natu 
10209b1aa8d6SNeel Natu 	/* Set up the IOMMU to do the 'gpa' to 'hpa' translation */
102151f45d01SNeel Natu 	if (ppt_assigned_devices(vm) == 0) {
1022318224bbSNeel Natu 		KASSERT(vm->iommu == NULL,
1023318224bbSNeel Natu 		    ("vm_assign_pptdev: iommu must be NULL"));
1024147d12a7SAntoine Brodin 		maxaddr = vmm_sysmem_maxaddr(vm);
1025318224bbSNeel Natu 		vm->iommu = iommu_create_domain(maxaddr);
1026ffe1b10dSJohn Baldwin 		if (vm->iommu == NULL)
1027ffe1b10dSJohn Baldwin 			return (ENXIO);
1028318224bbSNeel Natu 		vm_iommu_map(vm);
1029318224bbSNeel Natu 	}
1030318224bbSNeel Natu 
1031318224bbSNeel Natu 	error = ppt_assign_device(vm, bus, slot, func);
1032318224bbSNeel Natu 	return (error);
1033318224bbSNeel Natu }
1034318224bbSNeel Natu 
1035318224bbSNeel Natu void *
10369b1aa8d6SNeel Natu vm_gpa_hold(struct vm *vm, int vcpuid, vm_paddr_t gpa, size_t len, int reqprot,
1037318224bbSNeel Natu 	    void **cookie)
1038318224bbSNeel Natu {
10399b1aa8d6SNeel Natu 	int i, count, pageoff;
10409b1aa8d6SNeel Natu 	struct mem_map *mm;
1041318224bbSNeel Natu 	vm_page_t m;
10429b1aa8d6SNeel Natu #ifdef INVARIANTS
10439b1aa8d6SNeel Natu 	/*
10449b1aa8d6SNeel Natu 	 * All vcpus are frozen by ioctls that modify the memory map
10459b1aa8d6SNeel Natu 	 * (e.g. VM_MMAP_MEMSEG). Therefore 'vm->memmap[]' stability is
10469b1aa8d6SNeel Natu 	 * guaranteed if at least one vcpu is in the VCPU_FROZEN state.
10479b1aa8d6SNeel Natu 	 */
10489b1aa8d6SNeel Natu 	int state;
1049a488c9c9SRodney W. Grimes 	KASSERT(vcpuid >= -1 && vcpuid < vm->maxcpus, ("%s: invalid vcpuid %d",
10509b1aa8d6SNeel Natu 	    __func__, vcpuid));
1051a488c9c9SRodney W. Grimes 	for (i = 0; i < vm->maxcpus; i++) {
10529b1aa8d6SNeel Natu 		if (vcpuid != -1 && vcpuid != i)
10539b1aa8d6SNeel Natu 			continue;
10549b1aa8d6SNeel Natu 		state = vcpu_get_state(vm, i, NULL);
10559b1aa8d6SNeel Natu 		KASSERT(state == VCPU_FROZEN, ("%s: invalid vcpu state %d",
10569b1aa8d6SNeel Natu 		    __func__, state));
10579b1aa8d6SNeel Natu 	}
10589b1aa8d6SNeel Natu #endif
1059318224bbSNeel Natu 	pageoff = gpa & PAGE_MASK;
1060318224bbSNeel Natu 	if (len > PAGE_SIZE - pageoff)
1061318224bbSNeel Natu 		panic("vm_gpa_hold: invalid gpa/len: 0x%016lx/%lu", gpa, len);
1062318224bbSNeel Natu 
10639b1aa8d6SNeel Natu 	count = 0;
10649b1aa8d6SNeel Natu 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
10659b1aa8d6SNeel Natu 		mm = &vm->mem_maps[i];
106646567b4fSPeter Grehan 		if (gpa >= mm->gpa && gpa < mm->gpa + mm->len) {
1067318224bbSNeel Natu 			count = vm_fault_quick_hold_pages(&vm->vmspace->vm_map,
1068318224bbSNeel Natu 			    trunc_page(gpa), PAGE_SIZE, reqprot, &m, 1);
10699b1aa8d6SNeel Natu 			break;
10709b1aa8d6SNeel Natu 		}
10719b1aa8d6SNeel Natu 	}
1072318224bbSNeel Natu 
1073318224bbSNeel Natu 	if (count == 1) {
1074318224bbSNeel Natu 		*cookie = m;
1075318224bbSNeel Natu 		return ((void *)(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)) + pageoff));
1076318224bbSNeel Natu 	} else {
1077318224bbSNeel Natu 		*cookie = NULL;
1078318224bbSNeel Natu 		return (NULL);
1079318224bbSNeel Natu 	}
1080318224bbSNeel Natu }
1081318224bbSNeel Natu 
1082318224bbSNeel Natu void
1083318224bbSNeel Natu vm_gpa_release(void *cookie)
1084318224bbSNeel Natu {
1085318224bbSNeel Natu 	vm_page_t m = cookie;
1086318224bbSNeel Natu 
1087eeacb3b0SMark Johnston 	vm_page_unwire(m, PQ_ACTIVE);
1088366f6083SPeter Grehan }
1089366f6083SPeter Grehan 
1090366f6083SPeter Grehan int
1091366f6083SPeter Grehan vm_get_register(struct vm *vm, int vcpu, int reg, uint64_t *retval)
1092366f6083SPeter Grehan {
1093366f6083SPeter Grehan 
1094a488c9c9SRodney W. Grimes 	if (vcpu < 0 || vcpu >= vm->maxcpus)
1095366f6083SPeter Grehan 		return (EINVAL);
1096366f6083SPeter Grehan 
1097366f6083SPeter Grehan 	if (reg >= VM_REG_LAST)
1098366f6083SPeter Grehan 		return (EINVAL);
1099366f6083SPeter Grehan 
1100869c8d19SJohn Baldwin 	return (vmmops_getreg(vcpu_cookie(vm, vcpu), reg, retval));
1101366f6083SPeter Grehan }
1102366f6083SPeter Grehan 
1103366f6083SPeter Grehan int
1104d087a399SNeel Natu vm_set_register(struct vm *vm, int vcpuid, int reg, uint64_t val)
1105366f6083SPeter Grehan {
1106d087a399SNeel Natu 	struct vcpu *vcpu;
1107d087a399SNeel Natu 	int error;
1108366f6083SPeter Grehan 
1109a488c9c9SRodney W. Grimes 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
1110366f6083SPeter Grehan 		return (EINVAL);
1111366f6083SPeter Grehan 
1112366f6083SPeter Grehan 	if (reg >= VM_REG_LAST)
1113366f6083SPeter Grehan 		return (EINVAL);
1114366f6083SPeter Grehan 
11151aa51504SJohn Baldwin 	vcpu = &vm->vcpu[vcpuid];
1116869c8d19SJohn Baldwin 	error = vmmops_setreg(vcpu->cookie, reg, val);
1117d087a399SNeel Natu 	if (error || reg != VM_REG_GUEST_RIP)
1118d087a399SNeel Natu 		return (error);
1119d087a399SNeel Natu 
1120d087a399SNeel Natu 	/* Set 'nextrip' to match the value of %rip */
1121d087a399SNeel Natu 	VCPU_CTR1(vm, vcpuid, "Setting nextrip to %#lx", val);
1122d087a399SNeel Natu 	vcpu->nextrip = val;
1123d087a399SNeel Natu 	return (0);
1124366f6083SPeter Grehan }
1125366f6083SPeter Grehan 
1126490d56c5SEd Maste static bool
1127366f6083SPeter Grehan is_descriptor_table(int reg)
1128366f6083SPeter Grehan {
1129366f6083SPeter Grehan 
1130366f6083SPeter Grehan 	switch (reg) {
1131366f6083SPeter Grehan 	case VM_REG_GUEST_IDTR:
1132366f6083SPeter Grehan 	case VM_REG_GUEST_GDTR:
1133490d56c5SEd Maste 		return (true);
1134366f6083SPeter Grehan 	default:
1135490d56c5SEd Maste 		return (false);
1136366f6083SPeter Grehan 	}
1137366f6083SPeter Grehan }
1138366f6083SPeter Grehan 
1139490d56c5SEd Maste static bool
1140366f6083SPeter Grehan is_segment_register(int reg)
1141366f6083SPeter Grehan {
1142366f6083SPeter Grehan 
1143366f6083SPeter Grehan 	switch (reg) {
1144366f6083SPeter Grehan 	case VM_REG_GUEST_ES:
1145366f6083SPeter Grehan 	case VM_REG_GUEST_CS:
1146366f6083SPeter Grehan 	case VM_REG_GUEST_SS:
1147366f6083SPeter Grehan 	case VM_REG_GUEST_DS:
1148366f6083SPeter Grehan 	case VM_REG_GUEST_FS:
1149366f6083SPeter Grehan 	case VM_REG_GUEST_GS:
1150366f6083SPeter Grehan 	case VM_REG_GUEST_TR:
1151366f6083SPeter Grehan 	case VM_REG_GUEST_LDTR:
1152490d56c5SEd Maste 		return (true);
1153366f6083SPeter Grehan 	default:
1154490d56c5SEd Maste 		return (false);
1155366f6083SPeter Grehan 	}
1156366f6083SPeter Grehan }
1157366f6083SPeter Grehan 
1158366f6083SPeter Grehan int
1159366f6083SPeter Grehan vm_get_seg_desc(struct vm *vm, int vcpu, int reg,
1160366f6083SPeter Grehan 		struct seg_desc *desc)
1161366f6083SPeter Grehan {
1162366f6083SPeter Grehan 
1163a488c9c9SRodney W. Grimes 	if (vcpu < 0 || vcpu >= vm->maxcpus)
1164366f6083SPeter Grehan 		return (EINVAL);
1165366f6083SPeter Grehan 
1166366f6083SPeter Grehan 	if (!is_segment_register(reg) && !is_descriptor_table(reg))
1167366f6083SPeter Grehan 		return (EINVAL);
1168366f6083SPeter Grehan 
1169869c8d19SJohn Baldwin 	return (vmmops_getdesc(vcpu_cookie(vm, vcpu), reg, desc));
1170366f6083SPeter Grehan }
1171366f6083SPeter Grehan 
1172366f6083SPeter Grehan int
1173366f6083SPeter Grehan vm_set_seg_desc(struct vm *vm, int vcpu, int reg,
1174366f6083SPeter Grehan 		struct seg_desc *desc)
1175366f6083SPeter Grehan {
1176a488c9c9SRodney W. Grimes 	if (vcpu < 0 || vcpu >= vm->maxcpus)
1177366f6083SPeter Grehan 		return (EINVAL);
1178366f6083SPeter Grehan 
1179366f6083SPeter Grehan 	if (!is_segment_register(reg) && !is_descriptor_table(reg))
1180366f6083SPeter Grehan 		return (EINVAL);
1181366f6083SPeter Grehan 
1182869c8d19SJohn Baldwin 	return (vmmops_setdesc(vcpu_cookie(vm, vcpu), reg, desc));
1183366f6083SPeter Grehan }
1184366f6083SPeter Grehan 
1185366f6083SPeter Grehan static void
1186366f6083SPeter Grehan restore_guest_fpustate(struct vcpu *vcpu)
1187366f6083SPeter Grehan {
1188366f6083SPeter Grehan 
118938f1b189SPeter Grehan 	/* flush host state to the pcb */
119038f1b189SPeter Grehan 	fpuexit(curthread);
1191bd8572e0SNeel Natu 
1192bd8572e0SNeel Natu 	/* restore guest FPU state */
1193366f6083SPeter Grehan 	fpu_stop_emulating();
119438f1b189SPeter Grehan 	fpurestore(vcpu->guestfpu);
1195bd8572e0SNeel Natu 
1196abb023fbSJohn Baldwin 	/* restore guest XCR0 if XSAVE is enabled in the host */
1197abb023fbSJohn Baldwin 	if (rcr4() & CR4_XSAVE)
1198abb023fbSJohn Baldwin 		load_xcr(0, vcpu->guest_xcr0);
1199abb023fbSJohn Baldwin 
1200bd8572e0SNeel Natu 	/*
1201bd8572e0SNeel Natu 	 * The FPU is now "dirty" with the guest's state so turn on emulation
1202bd8572e0SNeel Natu 	 * to trap any access to the FPU by the host.
1203bd8572e0SNeel Natu 	 */
1204bd8572e0SNeel Natu 	fpu_start_emulating();
1205366f6083SPeter Grehan }
1206366f6083SPeter Grehan 
1207366f6083SPeter Grehan static void
1208366f6083SPeter Grehan save_guest_fpustate(struct vcpu *vcpu)
1209366f6083SPeter Grehan {
1210366f6083SPeter Grehan 
1211bd8572e0SNeel Natu 	if ((rcr0() & CR0_TS) == 0)
1212bd8572e0SNeel Natu 		panic("fpu emulation not enabled in host!");
1213bd8572e0SNeel Natu 
1214abb023fbSJohn Baldwin 	/* save guest XCR0 and restore host XCR0 */
1215abb023fbSJohn Baldwin 	if (rcr4() & CR4_XSAVE) {
1216abb023fbSJohn Baldwin 		vcpu->guest_xcr0 = rxcr(0);
1217abb023fbSJohn Baldwin 		load_xcr(0, vmm_get_host_xcr0());
1218abb023fbSJohn Baldwin 	}
1219abb023fbSJohn Baldwin 
1220bd8572e0SNeel Natu 	/* save guest FPU state */
1221bd8572e0SNeel Natu 	fpu_stop_emulating();
122238f1b189SPeter Grehan 	fpusave(vcpu->guestfpu);
1223366f6083SPeter Grehan 	fpu_start_emulating();
1224366f6083SPeter Grehan }
1225366f6083SPeter Grehan 
122661592433SNeel Natu static VMM_STAT(VCPU_IDLE_TICKS, "number of ticks vcpu was idle");
1227f76fc5d4SNeel Natu 
1228318224bbSNeel Natu static int
1229248e6799SNeel Natu vcpu_set_state_locked(struct vm *vm, int vcpuid, enum vcpu_state newstate,
1230f80330a8SNeel Natu     bool from_idle)
1231366f6083SPeter Grehan {
1232248e6799SNeel Natu 	struct vcpu *vcpu;
1233318224bbSNeel Natu 	int error;
1234366f6083SPeter Grehan 
1235248e6799SNeel Natu 	vcpu = &vm->vcpu[vcpuid];
1236318224bbSNeel Natu 	vcpu_assert_locked(vcpu);
1237366f6083SPeter Grehan 
1238f76fc5d4SNeel Natu 	/*
1239f80330a8SNeel Natu 	 * State transitions from the vmmdev_ioctl() must always begin from
1240f80330a8SNeel Natu 	 * the VCPU_IDLE state. This guarantees that there is only a single
1241f80330a8SNeel Natu 	 * ioctl() operating on a vcpu at any point.
1242f80330a8SNeel Natu 	 */
1243f80330a8SNeel Natu 	if (from_idle) {
1244248e6799SNeel Natu 		while (vcpu->state != VCPU_IDLE) {
1245248e6799SNeel Natu 			vcpu->reqidle = 1;
1246248e6799SNeel Natu 			vcpu_notify_event_locked(vcpu, false);
1247248e6799SNeel Natu 			VCPU_CTR1(vm, vcpuid, "vcpu state change from %s to "
1248248e6799SNeel Natu 			    "idle requested", vcpu_state2str(vcpu->state));
1249f80330a8SNeel Natu 			msleep_spin(&vcpu->state, &vcpu->mtx, "vmstat", hz);
1250248e6799SNeel Natu 		}
1251f80330a8SNeel Natu 	} else {
1252f80330a8SNeel Natu 		KASSERT(vcpu->state != VCPU_IDLE, ("invalid transition from "
1253f80330a8SNeel Natu 		    "vcpu idle state"));
1254f80330a8SNeel Natu 	}
1255f80330a8SNeel Natu 
1256ef39d7e9SNeel Natu 	if (vcpu->state == VCPU_RUNNING) {
1257ef39d7e9SNeel Natu 		KASSERT(vcpu->hostcpu == curcpu, ("curcpu %d and hostcpu %d "
1258ef39d7e9SNeel Natu 		    "mismatch for running vcpu", curcpu, vcpu->hostcpu));
1259ef39d7e9SNeel Natu 	} else {
1260ef39d7e9SNeel Natu 		KASSERT(vcpu->hostcpu == NOCPU, ("Invalid hostcpu %d for a "
1261ef39d7e9SNeel Natu 		    "vcpu that is not running", vcpu->hostcpu));
1262ef39d7e9SNeel Natu 	}
1263ef39d7e9SNeel Natu 
1264f80330a8SNeel Natu 	/*
1265318224bbSNeel Natu 	 * The following state transitions are allowed:
1266318224bbSNeel Natu 	 * IDLE -> FROZEN -> IDLE
1267318224bbSNeel Natu 	 * FROZEN -> RUNNING -> FROZEN
1268318224bbSNeel Natu 	 * FROZEN -> SLEEPING -> FROZEN
1269f76fc5d4SNeel Natu 	 */
1270318224bbSNeel Natu 	switch (vcpu->state) {
1271318224bbSNeel Natu 	case VCPU_IDLE:
1272318224bbSNeel Natu 	case VCPU_RUNNING:
1273318224bbSNeel Natu 	case VCPU_SLEEPING:
1274318224bbSNeel Natu 		error = (newstate != VCPU_FROZEN);
1275318224bbSNeel Natu 		break;
1276318224bbSNeel Natu 	case VCPU_FROZEN:
1277318224bbSNeel Natu 		error = (newstate == VCPU_FROZEN);
1278318224bbSNeel Natu 		break;
1279318224bbSNeel Natu 	default:
1280318224bbSNeel Natu 		error = 1;
1281318224bbSNeel Natu 		break;
1282318224bbSNeel Natu 	}
1283318224bbSNeel Natu 
1284f80330a8SNeel Natu 	if (error)
1285f80330a8SNeel Natu 		return (EBUSY);
1286318224bbSNeel Natu 
1287248e6799SNeel Natu 	VCPU_CTR2(vm, vcpuid, "vcpu state changed from %s to %s",
1288248e6799SNeel Natu 	    vcpu_state2str(vcpu->state), vcpu_state2str(newstate));
1289248e6799SNeel Natu 
1290f80330a8SNeel Natu 	vcpu->state = newstate;
1291ef39d7e9SNeel Natu 	if (newstate == VCPU_RUNNING)
1292ef39d7e9SNeel Natu 		vcpu->hostcpu = curcpu;
1293ef39d7e9SNeel Natu 	else
1294ef39d7e9SNeel Natu 		vcpu->hostcpu = NOCPU;
1295ef39d7e9SNeel Natu 
1296f80330a8SNeel Natu 	if (newstate == VCPU_IDLE)
1297f80330a8SNeel Natu 		wakeup(&vcpu->state);
1298f80330a8SNeel Natu 
1299f80330a8SNeel Natu 	return (0);
1300318224bbSNeel Natu }
1301318224bbSNeel Natu 
1302318224bbSNeel Natu static void
1303318224bbSNeel Natu vcpu_require_state(struct vm *vm, int vcpuid, enum vcpu_state newstate)
1304318224bbSNeel Natu {
1305318224bbSNeel Natu 	int error;
1306318224bbSNeel Natu 
1307f80330a8SNeel Natu 	if ((error = vcpu_set_state(vm, vcpuid, newstate, false)) != 0)
1308318224bbSNeel Natu 		panic("Error %d setting state to %d\n", error, newstate);
1309318224bbSNeel Natu }
1310318224bbSNeel Natu 
1311318224bbSNeel Natu static void
1312248e6799SNeel Natu vcpu_require_state_locked(struct vm *vm, int vcpuid, enum vcpu_state newstate)
1313318224bbSNeel Natu {
1314318224bbSNeel Natu 	int error;
1315318224bbSNeel Natu 
1316248e6799SNeel Natu 	if ((error = vcpu_set_state_locked(vm, vcpuid, newstate, false)) != 0)
1317318224bbSNeel Natu 		panic("Error %d setting state to %d", error, newstate);
1318318224bbSNeel Natu }
1319318224bbSNeel Natu 
13205b8a8cd1SNeel Natu #define	RENDEZVOUS_CTR0(vm, vcpuid, fmt)				\
13215b8a8cd1SNeel Natu 	do {								\
13225b8a8cd1SNeel Natu 		if (vcpuid >= 0)					\
13235b8a8cd1SNeel Natu 			VCPU_CTR0(vm, vcpuid, fmt);			\
13245b8a8cd1SNeel Natu 		else							\
13255b8a8cd1SNeel Natu 			VM_CTR0(vm, fmt);				\
13265b8a8cd1SNeel Natu 	} while (0)
13275b8a8cd1SNeel Natu 
1328b837daddSKonstantin Belousov static int
13295b8a8cd1SNeel Natu vm_handle_rendezvous(struct vm *vm, int vcpuid)
13305b8a8cd1SNeel Natu {
1331b837daddSKonstantin Belousov 	struct thread *td;
1332b837daddSKonstantin Belousov 	int error;
13335b8a8cd1SNeel Natu 
1334a488c9c9SRodney W. Grimes 	KASSERT(vcpuid == -1 || (vcpuid >= 0 && vcpuid < vm->maxcpus),
13355b8a8cd1SNeel Natu 	    ("vm_handle_rendezvous: invalid vcpuid %d", vcpuid));
13365b8a8cd1SNeel Natu 
1337b837daddSKonstantin Belousov 	error = 0;
1338b837daddSKonstantin Belousov 	td = curthread;
13395b8a8cd1SNeel Natu 	mtx_lock(&vm->rendezvous_mtx);
13405b8a8cd1SNeel Natu 	while (vm->rendezvous_func != NULL) {
134122d822c6SNeel Natu 		/* 'rendezvous_req_cpus' must be a subset of 'active_cpus' */
1342e2650af1SStefan Eßer 		CPU_AND(&vm->rendezvous_req_cpus, &vm->rendezvous_req_cpus, &vm->active_cpus);
134322d822c6SNeel Natu 
13445b8a8cd1SNeel Natu 		if (vcpuid != -1 &&
134522d822c6SNeel Natu 		    CPU_ISSET(vcpuid, &vm->rendezvous_req_cpus) &&
134622d822c6SNeel Natu 		    !CPU_ISSET(vcpuid, &vm->rendezvous_done_cpus)) {
13475b8a8cd1SNeel Natu 			VCPU_CTR0(vm, vcpuid, "Calling rendezvous func");
13485b8a8cd1SNeel Natu 			(*vm->rendezvous_func)(vm, vcpuid, vm->rendezvous_arg);
13495b8a8cd1SNeel Natu 			CPU_SET(vcpuid, &vm->rendezvous_done_cpus);
13505b8a8cd1SNeel Natu 		}
13515b8a8cd1SNeel Natu 		if (CPU_CMP(&vm->rendezvous_req_cpus,
13525b8a8cd1SNeel Natu 		    &vm->rendezvous_done_cpus) == 0) {
13535b8a8cd1SNeel Natu 			VCPU_CTR0(vm, vcpuid, "Rendezvous completed");
1354869dbab7SAndriy Gapon 			vm->rendezvous_func = NULL;
13555b8a8cd1SNeel Natu 			wakeup(&vm->rendezvous_func);
13565b8a8cd1SNeel Natu 			break;
13575b8a8cd1SNeel Natu 		}
13585b8a8cd1SNeel Natu 		RENDEZVOUS_CTR0(vm, vcpuid, "Wait for rendezvous completion");
13595b8a8cd1SNeel Natu 		mtx_sleep(&vm->rendezvous_func, &vm->rendezvous_mtx, 0,
1360b837daddSKonstantin Belousov 		    "vmrndv", hz);
1361c6d31b83SKonstantin Belousov 		if (td_ast_pending(td, TDA_SUSPEND)) {
1362b837daddSKonstantin Belousov 			mtx_unlock(&vm->rendezvous_mtx);
1363b837daddSKonstantin Belousov 			error = thread_check_susp(td, true);
1364b837daddSKonstantin Belousov 			if (error != 0)
1365b837daddSKonstantin Belousov 				return (error);
1366b837daddSKonstantin Belousov 			mtx_lock(&vm->rendezvous_mtx);
1367b837daddSKonstantin Belousov 		}
13685b8a8cd1SNeel Natu 	}
13695b8a8cd1SNeel Natu 	mtx_unlock(&vm->rendezvous_mtx);
1370b837daddSKonstantin Belousov 	return (0);
13715b8a8cd1SNeel Natu }
13725b8a8cd1SNeel Natu 
1373318224bbSNeel Natu /*
1374318224bbSNeel Natu  * Emulate a guest 'hlt' by sleeping until the vcpu is ready to run.
1375318224bbSNeel Natu  */
1376318224bbSNeel Natu static int
1377becd9849SNeel Natu vm_handle_hlt(struct vm *vm, int vcpuid, bool intr_disabled, bool *retu)
1378318224bbSNeel Natu {
1379318224bbSNeel Natu 	struct vcpu *vcpu;
1380c6a0cc2eSNeel Natu 	const char *wmesg;
1381b837daddSKonstantin Belousov 	struct thread *td;
1382b837daddSKonstantin Belousov 	int error, t, vcpu_halted, vm_halted;
1383e50ce2aaSNeel Natu 
1384e50ce2aaSNeel Natu 	KASSERT(!CPU_ISSET(vcpuid, &vm->halted_cpus), ("vcpu already halted"));
1385318224bbSNeel Natu 
1386318224bbSNeel Natu 	vcpu = &vm->vcpu[vcpuid];
1387e50ce2aaSNeel Natu 	vcpu_halted = 0;
1388e50ce2aaSNeel Natu 	vm_halted = 0;
1389b837daddSKonstantin Belousov 	error = 0;
1390b837daddSKonstantin Belousov 	td = curthread;
1391318224bbSNeel Natu 
1392f76fc5d4SNeel Natu 	vcpu_lock(vcpu);
1393c6a0cc2eSNeel Natu 	while (1) {
1394f76fc5d4SNeel Natu 		/*
1395f76fc5d4SNeel Natu 		 * Do a final check for pending NMI or interrupts before
1396c6a0cc2eSNeel Natu 		 * really putting this thread to sleep. Also check for
1397c6a0cc2eSNeel Natu 		 * software events that would cause this vcpu to wakeup.
1398f76fc5d4SNeel Natu 		 *
1399c6a0cc2eSNeel Natu 		 * These interrupts/events could have happened after the
140015add60dSPeter Grehan 		 * vcpu returned from vmmops_run() and before it acquired the
1401c6a0cc2eSNeel Natu 		 * vcpu lock above.
1402f76fc5d4SNeel Natu 		 */
1403248e6799SNeel Natu 		if (vm->rendezvous_func != NULL || vm->suspend || vcpu->reqidle)
1404c6a0cc2eSNeel Natu 			break;
1405c6a0cc2eSNeel Natu 		if (vm_nmi_pending(vm, vcpuid))
1406c6a0cc2eSNeel Natu 			break;
1407c6a0cc2eSNeel Natu 		if (!intr_disabled) {
1408c6a0cc2eSNeel Natu 			if (vm_extint_pending(vm, vcpuid) ||
1409c6a0cc2eSNeel Natu 			    vlapic_pending_intr(vcpu->vlapic, NULL)) {
1410c6a0cc2eSNeel Natu 				break;
1411c6a0cc2eSNeel Natu 			}
1412c6a0cc2eSNeel Natu 		}
1413c6a0cc2eSNeel Natu 
1414f008d157SNeel Natu 		/* Don't go to sleep if the vcpu thread needs to yield */
1415f008d157SNeel Natu 		if (vcpu_should_yield(vm, vcpuid))
1416f008d157SNeel Natu 			break;
1417f008d157SNeel Natu 
1418fc276d92SJohn Baldwin 		if (vcpu_debugged(vm, vcpuid))
1419fc276d92SJohn Baldwin 			break;
1420fc276d92SJohn Baldwin 
1421e50ce2aaSNeel Natu 		/*
1422e50ce2aaSNeel Natu 		 * Some Linux guests implement "halt" by having all vcpus
1423e50ce2aaSNeel Natu 		 * execute HLT with interrupts disabled. 'halted_cpus' keeps
1424e50ce2aaSNeel Natu 		 * track of the vcpus that have entered this state. When all
1425e50ce2aaSNeel Natu 		 * vcpus enter the halted state the virtual machine is halted.
1426e50ce2aaSNeel Natu 		 */
1427e50ce2aaSNeel Natu 		if (intr_disabled) {
1428c6a0cc2eSNeel Natu 			wmesg = "vmhalt";
1429e50ce2aaSNeel Natu 			VCPU_CTR0(vm, vcpuid, "Halted");
1430055fc2cbSNeel Natu 			if (!vcpu_halted && halt_detection_enabled) {
1431e50ce2aaSNeel Natu 				vcpu_halted = 1;
1432e50ce2aaSNeel Natu 				CPU_SET_ATOMIC(vcpuid, &vm->halted_cpus);
1433e50ce2aaSNeel Natu 			}
1434e50ce2aaSNeel Natu 			if (CPU_CMP(&vm->halted_cpus, &vm->active_cpus) == 0) {
1435e50ce2aaSNeel Natu 				vm_halted = 1;
1436e50ce2aaSNeel Natu 				break;
1437e50ce2aaSNeel Natu 			}
1438e50ce2aaSNeel Natu 		} else {
1439e50ce2aaSNeel Natu 			wmesg = "vmidle";
1440e50ce2aaSNeel Natu 		}
1441c6a0cc2eSNeel Natu 
1442f76fc5d4SNeel Natu 		t = ticks;
1443248e6799SNeel Natu 		vcpu_require_state_locked(vm, vcpuid, VCPU_SLEEPING);
1444f008d157SNeel Natu 		/*
1445f008d157SNeel Natu 		 * XXX msleep_spin() cannot be interrupted by signals so
1446f008d157SNeel Natu 		 * wake up periodically to check pending signals.
1447f008d157SNeel Natu 		 */
1448f008d157SNeel Natu 		msleep_spin(vcpu, &vcpu->mtx, wmesg, hz);
1449248e6799SNeel Natu 		vcpu_require_state_locked(vm, vcpuid, VCPU_FROZEN);
14503dc3d32aSJohn Baldwin 		vmm_stat_incr(vcpu, VCPU_IDLE_TICKS, ticks - t);
1451c6d31b83SKonstantin Belousov 		if (td_ast_pending(td, TDA_SUSPEND)) {
1452b837daddSKonstantin Belousov 			vcpu_unlock(vcpu);
1453b837daddSKonstantin Belousov 			error = thread_check_susp(td, false);
14544d447b30SKonstantin Belousov 			if (error != 0) {
14554d447b30SKonstantin Belousov 				if (vcpu_halted) {
14564d447b30SKonstantin Belousov 					CPU_CLR_ATOMIC(vcpuid,
14574d447b30SKonstantin Belousov 					    &vm->halted_cpus);
14584d447b30SKonstantin Belousov 				}
1459b837daddSKonstantin Belousov 				return (error);
14604d447b30SKonstantin Belousov 			}
1461b837daddSKonstantin Belousov 			vcpu_lock(vcpu);
1462b837daddSKonstantin Belousov 		}
1463f76fc5d4SNeel Natu 	}
1464e50ce2aaSNeel Natu 
1465e50ce2aaSNeel Natu 	if (vcpu_halted)
1466e50ce2aaSNeel Natu 		CPU_CLR_ATOMIC(vcpuid, &vm->halted_cpus);
1467e50ce2aaSNeel Natu 
1468f76fc5d4SNeel Natu 	vcpu_unlock(vcpu);
1469f76fc5d4SNeel Natu 
1470e50ce2aaSNeel Natu 	if (vm_halted)
1471e50ce2aaSNeel Natu 		vm_suspend(vm, VM_SUSPEND_HALT);
1472e50ce2aaSNeel Natu 
1473318224bbSNeel Natu 	return (0);
1474318224bbSNeel Natu }
1475318224bbSNeel Natu 
1476318224bbSNeel Natu static int
1477becd9849SNeel Natu vm_handle_paging(struct vm *vm, int vcpuid, bool *retu)
1478318224bbSNeel Natu {
1479318224bbSNeel Natu 	int rv, ftype;
1480318224bbSNeel Natu 	struct vm_map *map;
1481318224bbSNeel Natu 	struct vcpu *vcpu;
1482318224bbSNeel Natu 	struct vm_exit *vme;
1483318224bbSNeel Natu 
1484318224bbSNeel Natu 	vcpu = &vm->vcpu[vcpuid];
1485318224bbSNeel Natu 	vme = &vcpu->exitinfo;
1486318224bbSNeel Natu 
1487d087a399SNeel Natu 	KASSERT(vme->inst_length == 0, ("%s: invalid inst_length %d",
1488d087a399SNeel Natu 	    __func__, vme->inst_length));
1489d087a399SNeel Natu 
1490318224bbSNeel Natu 	ftype = vme->u.paging.fault_type;
1491318224bbSNeel Natu 	KASSERT(ftype == VM_PROT_READ ||
1492318224bbSNeel Natu 	    ftype == VM_PROT_WRITE || ftype == VM_PROT_EXECUTE,
1493318224bbSNeel Natu 	    ("vm_handle_paging: invalid fault_type %d", ftype));
1494318224bbSNeel Natu 
1495318224bbSNeel Natu 	if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) {
1496318224bbSNeel Natu 		rv = pmap_emulate_accessed_dirty(vmspace_pmap(vm->vmspace),
1497318224bbSNeel Natu 		    vme->u.paging.gpa, ftype);
14989d8d8e3eSNeel Natu 		if (rv == 0) {
14999d8d8e3eSNeel Natu 			VCPU_CTR2(vm, vcpuid, "%s bit emulation for gpa %#lx",
15009d8d8e3eSNeel Natu 			    ftype == VM_PROT_READ ? "accessed" : "dirty",
15019d8d8e3eSNeel Natu 			    vme->u.paging.gpa);
1502318224bbSNeel Natu 			goto done;
1503318224bbSNeel Natu 		}
15049d8d8e3eSNeel Natu 	}
1505318224bbSNeel Natu 
1506318224bbSNeel Natu 	map = &vm->vmspace->vm_map;
1507df08823dSKonstantin Belousov 	rv = vm_fault(map, vme->u.paging.gpa, ftype, VM_FAULT_NORMAL, NULL);
1508318224bbSNeel Natu 
1509513c8d33SNeel Natu 	VCPU_CTR3(vm, vcpuid, "vm_handle_paging rv = %d, gpa = %#lx, "
1510513c8d33SNeel Natu 	    "ftype = %d", rv, vme->u.paging.gpa, ftype);
1511318224bbSNeel Natu 
1512318224bbSNeel Natu 	if (rv != KERN_SUCCESS)
1513318224bbSNeel Natu 		return (EFAULT);
1514318224bbSNeel Natu done:
1515318224bbSNeel Natu 	return (0);
1516318224bbSNeel Natu }
1517318224bbSNeel Natu 
1518318224bbSNeel Natu static int
1519becd9849SNeel Natu vm_handle_inst_emul(struct vm *vm, int vcpuid, bool *retu)
1520318224bbSNeel Natu {
1521318224bbSNeel Natu 	struct vie *vie;
1522318224bbSNeel Natu 	struct vcpu *vcpu;
1523318224bbSNeel Natu 	struct vm_exit *vme;
1524e4f605eeSTycho Nightingale 	uint64_t gla, gpa, cs_base;
1525e813a873SNeel Natu 	struct vm_guest_paging *paging;
1526565bbb86SNeel Natu 	mem_region_read_t mread;
1527565bbb86SNeel Natu 	mem_region_write_t mwrite;
1528f7a9f178SNeel Natu 	enum vm_cpu_mode cpu_mode;
15291c73ea3eSNeel Natu 	int cs_d, error, fault;
1530318224bbSNeel Natu 
1531318224bbSNeel Natu 	vcpu = &vm->vcpu[vcpuid];
1532318224bbSNeel Natu 	vme = &vcpu->exitinfo;
1533318224bbSNeel Natu 
15341c73ea3eSNeel Natu 	KASSERT(vme->inst_length == 0, ("%s: invalid inst_length %d",
15351c73ea3eSNeel Natu 	    __func__, vme->inst_length));
15361c73ea3eSNeel Natu 
1537318224bbSNeel Natu 	gla = vme->u.inst_emul.gla;
1538318224bbSNeel Natu 	gpa = vme->u.inst_emul.gpa;
1539e4f605eeSTycho Nightingale 	cs_base = vme->u.inst_emul.cs_base;
1540f7a9f178SNeel Natu 	cs_d = vme->u.inst_emul.cs_d;
1541318224bbSNeel Natu 	vie = &vme->u.inst_emul.vie;
1542e813a873SNeel Natu 	paging = &vme->u.inst_emul.paging;
1543f7a9f178SNeel Natu 	cpu_mode = paging->cpu_mode;
1544318224bbSNeel Natu 
15459d8d8e3eSNeel Natu 	VCPU_CTR1(vm, vcpuid, "inst_emul fault accessing gpa %#lx", gpa);
15469d8d8e3eSNeel Natu 
1547318224bbSNeel Natu 	/* Fetch, decode and emulate the faulting instruction */
1548c2a875f9SNeel Natu 	if (vie->num_valid == 0) {
1549e4f605eeSTycho Nightingale 		error = vmm_fetch_instruction(vm, vcpuid, paging, vme->rip +
15501c73ea3eSNeel Natu 		    cs_base, VIE_INST_SIZE, vie, &fault);
1551c2a875f9SNeel Natu 	} else {
1552c2a875f9SNeel Natu 		/*
1553c2a875f9SNeel Natu 		 * The instruction bytes have already been copied into 'vie'
1554c2a875f9SNeel Natu 		 */
15559c4d5478SNeel Natu 		error = fault = 0;
1556c2a875f9SNeel Natu 	}
15579c4d5478SNeel Natu 	if (error || fault)
15589c4d5478SNeel Natu 		return (error);
1559318224bbSNeel Natu 
1560c07a0648SNeel Natu 	if (vmm_decode_instruction(vm, vcpuid, gla, cpu_mode, cs_d, vie) != 0) {
1561c07a0648SNeel Natu 		VCPU_CTR1(vm, vcpuid, "Error decoding instruction at %#lx",
1562c07a0648SNeel Natu 		    vme->rip + cs_base);
1563c07a0648SNeel Natu 		*retu = true;	    /* dump instruction bytes in userspace */
1564c07a0648SNeel Natu 		return (0);
1565c07a0648SNeel Natu 	}
1566318224bbSNeel Natu 
1567a0b78f09SPeter Grehan 	/*
15681c73ea3eSNeel Natu 	 * Update 'nextrip' based on the length of the emulated instruction.
1569a0b78f09SPeter Grehan 	 */
1570a0b78f09SPeter Grehan 	vme->inst_length = vie->num_processed;
1571d087a399SNeel Natu 	vcpu->nextrip += vie->num_processed;
15721c73ea3eSNeel Natu 	VCPU_CTR1(vm, vcpuid, "nextrip updated to %#lx after instruction "
15731c73ea3eSNeel Natu 	    "decoding", vcpu->nextrip);
1574a0b78f09SPeter Grehan 
157508e3ff32SNeel Natu 	/* return to userland unless this is an in-kernel emulated device */
1576565bbb86SNeel Natu 	if (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE) {
1577565bbb86SNeel Natu 		mread = lapic_mmio_read;
1578565bbb86SNeel Natu 		mwrite = lapic_mmio_write;
1579565bbb86SNeel Natu 	} else if (gpa >= VIOAPIC_BASE && gpa < VIOAPIC_BASE + VIOAPIC_SIZE) {
1580565bbb86SNeel Natu 		mread = vioapic_mmio_read;
1581565bbb86SNeel Natu 		mwrite = vioapic_mmio_write;
158208e3ff32SNeel Natu 	} else if (gpa >= VHPET_BASE && gpa < VHPET_BASE + VHPET_SIZE) {
158308e3ff32SNeel Natu 		mread = vhpet_mmio_read;
158408e3ff32SNeel Natu 		mwrite = vhpet_mmio_write;
1585565bbb86SNeel Natu 	} else {
1586becd9849SNeel Natu 		*retu = true;
1587318224bbSNeel Natu 		return (0);
1588318224bbSNeel Natu 	}
1589318224bbSNeel Natu 
1590d665d229SNeel Natu 	error = vmm_emulate_instruction(vm, vcpuid, gpa, vie, paging,
1591d665d229SNeel Natu 	    mread, mwrite, retu);
1592318224bbSNeel Natu 
1593318224bbSNeel Natu 	return (error);
1594318224bbSNeel Natu }
1595318224bbSNeel Natu 
1596b15a09c0SNeel Natu static int
1597b15a09c0SNeel Natu vm_handle_suspend(struct vm *vm, int vcpuid, bool *retu)
1598b15a09c0SNeel Natu {
1599b837daddSKonstantin Belousov 	int error, i;
1600b15a09c0SNeel Natu 	struct vcpu *vcpu;
1601b837daddSKonstantin Belousov 	struct thread *td;
1602b15a09c0SNeel Natu 
1603b837daddSKonstantin Belousov 	error = 0;
1604b15a09c0SNeel Natu 	vcpu = &vm->vcpu[vcpuid];
1605b837daddSKonstantin Belousov 	td = curthread;
1606b15a09c0SNeel Natu 
1607b15a09c0SNeel Natu 	CPU_SET_ATOMIC(vcpuid, &vm->suspended_cpus);
1608b15a09c0SNeel Natu 
1609b15a09c0SNeel Natu 	/*
1610b15a09c0SNeel Natu 	 * Wait until all 'active_cpus' have suspended themselves.
1611b15a09c0SNeel Natu 	 *
1612b15a09c0SNeel Natu 	 * Since a VM may be suspended at any time including when one or
1613b15a09c0SNeel Natu 	 * more vcpus are doing a rendezvous we need to call the rendezvous
1614b15a09c0SNeel Natu 	 * handler while we are waiting to prevent a deadlock.
1615b15a09c0SNeel Natu 	 */
1616b15a09c0SNeel Natu 	vcpu_lock(vcpu);
1617b837daddSKonstantin Belousov 	while (error == 0) {
1618b15a09c0SNeel Natu 		if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) {
1619b15a09c0SNeel Natu 			VCPU_CTR0(vm, vcpuid, "All vcpus suspended");
1620b15a09c0SNeel Natu 			break;
1621b15a09c0SNeel Natu 		}
1622b15a09c0SNeel Natu 
1623b15a09c0SNeel Natu 		if (vm->rendezvous_func == NULL) {
1624b15a09c0SNeel Natu 			VCPU_CTR0(vm, vcpuid, "Sleeping during suspend");
1625248e6799SNeel Natu 			vcpu_require_state_locked(vm, vcpuid, VCPU_SLEEPING);
1626b15a09c0SNeel Natu 			msleep_spin(vcpu, &vcpu->mtx, "vmsusp", hz);
1627248e6799SNeel Natu 			vcpu_require_state_locked(vm, vcpuid, VCPU_FROZEN);
1628c6d31b83SKonstantin Belousov 			if (td_ast_pending(td, TDA_SUSPEND)) {
1629b837daddSKonstantin Belousov 				vcpu_unlock(vcpu);
1630b837daddSKonstantin Belousov 				error = thread_check_susp(td, false);
1631b837daddSKonstantin Belousov 				vcpu_lock(vcpu);
1632b837daddSKonstantin Belousov 			}
1633b15a09c0SNeel Natu 		} else {
1634b15a09c0SNeel Natu 			VCPU_CTR0(vm, vcpuid, "Rendezvous during suspend");
1635b15a09c0SNeel Natu 			vcpu_unlock(vcpu);
1636b837daddSKonstantin Belousov 			error = vm_handle_rendezvous(vm, vcpuid);
1637b15a09c0SNeel Natu 			vcpu_lock(vcpu);
1638b15a09c0SNeel Natu 		}
1639b15a09c0SNeel Natu 	}
1640b15a09c0SNeel Natu 	vcpu_unlock(vcpu);
1641b15a09c0SNeel Natu 
1642b15a09c0SNeel Natu 	/*
1643b15a09c0SNeel Natu 	 * Wakeup the other sleeping vcpus and return to userspace.
1644b15a09c0SNeel Natu 	 */
1645a488c9c9SRodney W. Grimes 	for (i = 0; i < vm->maxcpus; i++) {
1646b15a09c0SNeel Natu 		if (CPU_ISSET(i, &vm->suspended_cpus)) {
1647b15a09c0SNeel Natu 			vcpu_notify_event(vm, i, false);
1648b15a09c0SNeel Natu 		}
1649b15a09c0SNeel Natu 	}
1650b15a09c0SNeel Natu 
1651b15a09c0SNeel Natu 	*retu = true;
1652b837daddSKonstantin Belousov 	return (error);
1653b15a09c0SNeel Natu }
1654b15a09c0SNeel Natu 
1655248e6799SNeel Natu static int
1656248e6799SNeel Natu vm_handle_reqidle(struct vm *vm, int vcpuid, bool *retu)
1657248e6799SNeel Natu {
1658248e6799SNeel Natu 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
1659248e6799SNeel Natu 
1660248e6799SNeel Natu 	vcpu_lock(vcpu);
1661248e6799SNeel Natu 	KASSERT(vcpu->reqidle, ("invalid vcpu reqidle %d", vcpu->reqidle));
1662248e6799SNeel Natu 	vcpu->reqidle = 0;
1663248e6799SNeel Natu 	vcpu_unlock(vcpu);
1664248e6799SNeel Natu 	*retu = true;
1665248e6799SNeel Natu 	return (0);
1666248e6799SNeel Natu }
1667248e6799SNeel Natu 
1668b15a09c0SNeel Natu int
1669f0fdcfe2SNeel Natu vm_suspend(struct vm *vm, enum vm_suspend_how how)
1670b15a09c0SNeel Natu {
1671f0fdcfe2SNeel Natu 	int i;
1672b15a09c0SNeel Natu 
1673f0fdcfe2SNeel Natu 	if (how <= VM_SUSPEND_NONE || how >= VM_SUSPEND_LAST)
1674f0fdcfe2SNeel Natu 		return (EINVAL);
1675f0fdcfe2SNeel Natu 
1676f0fdcfe2SNeel Natu 	if (atomic_cmpset_int(&vm->suspend, 0, how) == 0) {
1677f0fdcfe2SNeel Natu 		VM_CTR2(vm, "virtual machine already suspended %d/%d",
1678f0fdcfe2SNeel Natu 		    vm->suspend, how);
1679b15a09c0SNeel Natu 		return (EALREADY);
1680b15a09c0SNeel Natu 	}
1681f0fdcfe2SNeel Natu 
1682f0fdcfe2SNeel Natu 	VM_CTR1(vm, "virtual machine successfully suspended %d", how);
1683f0fdcfe2SNeel Natu 
1684f0fdcfe2SNeel Natu 	/*
1685f0fdcfe2SNeel Natu 	 * Notify all active vcpus that they are now suspended.
1686f0fdcfe2SNeel Natu 	 */
1687a488c9c9SRodney W. Grimes 	for (i = 0; i < vm->maxcpus; i++) {
1688f0fdcfe2SNeel Natu 		if (CPU_ISSET(i, &vm->active_cpus))
1689f0fdcfe2SNeel Natu 			vcpu_notify_event(vm, i, false);
1690f0fdcfe2SNeel Natu 	}
1691f0fdcfe2SNeel Natu 
1692f0fdcfe2SNeel Natu 	return (0);
1693f0fdcfe2SNeel Natu }
1694f0fdcfe2SNeel Natu 
1695f0fdcfe2SNeel Natu void
1696f0fdcfe2SNeel Natu vm_exit_suspended(struct vm *vm, int vcpuid, uint64_t rip)
1697f0fdcfe2SNeel Natu {
1698f0fdcfe2SNeel Natu 	struct vm_exit *vmexit;
1699f0fdcfe2SNeel Natu 
1700f0fdcfe2SNeel Natu 	KASSERT(vm->suspend > VM_SUSPEND_NONE && vm->suspend < VM_SUSPEND_LAST,
1701f0fdcfe2SNeel Natu 	    ("vm_exit_suspended: invalid suspend type %d", vm->suspend));
1702f0fdcfe2SNeel Natu 
1703f0fdcfe2SNeel Natu 	vmexit = vm_exitinfo(vm, vcpuid);
1704f0fdcfe2SNeel Natu 	vmexit->rip = rip;
1705f0fdcfe2SNeel Natu 	vmexit->inst_length = 0;
1706f0fdcfe2SNeel Natu 	vmexit->exitcode = VM_EXITCODE_SUSPENDED;
1707f0fdcfe2SNeel Natu 	vmexit->u.suspended.how = vm->suspend;
1708b15a09c0SNeel Natu }
1709b15a09c0SNeel Natu 
171040487465SNeel Natu void
1711fc276d92SJohn Baldwin vm_exit_debug(struct vm *vm, int vcpuid, uint64_t rip)
1712fc276d92SJohn Baldwin {
1713fc276d92SJohn Baldwin 	struct vm_exit *vmexit;
1714fc276d92SJohn Baldwin 
1715fc276d92SJohn Baldwin 	vmexit = vm_exitinfo(vm, vcpuid);
1716fc276d92SJohn Baldwin 	vmexit->rip = rip;
1717fc276d92SJohn Baldwin 	vmexit->inst_length = 0;
1718fc276d92SJohn Baldwin 	vmexit->exitcode = VM_EXITCODE_DEBUG;
1719fc276d92SJohn Baldwin }
1720fc276d92SJohn Baldwin 
1721fc276d92SJohn Baldwin void
172240487465SNeel Natu vm_exit_rendezvous(struct vm *vm, int vcpuid, uint64_t rip)
172340487465SNeel Natu {
172440487465SNeel Natu 	struct vm_exit *vmexit;
172540487465SNeel Natu 
172640487465SNeel Natu 	KASSERT(vm->rendezvous_func != NULL, ("rendezvous not in progress"));
172740487465SNeel Natu 
172840487465SNeel Natu 	vmexit = vm_exitinfo(vm, vcpuid);
172940487465SNeel Natu 	vmexit->rip = rip;
173040487465SNeel Natu 	vmexit->inst_length = 0;
173140487465SNeel Natu 	vmexit->exitcode = VM_EXITCODE_RENDEZVOUS;
17323dc3d32aSJohn Baldwin 	vmm_stat_incr(vm_vcpu(vm, vcpuid), VMEXIT_RENDEZVOUS, 1);
173340487465SNeel Natu }
173440487465SNeel Natu 
173540487465SNeel Natu void
1736248e6799SNeel Natu vm_exit_reqidle(struct vm *vm, int vcpuid, uint64_t rip)
1737248e6799SNeel Natu {
1738248e6799SNeel Natu 	struct vm_exit *vmexit;
1739248e6799SNeel Natu 
1740248e6799SNeel Natu 	vmexit = vm_exitinfo(vm, vcpuid);
1741248e6799SNeel Natu 	vmexit->rip = rip;
1742248e6799SNeel Natu 	vmexit->inst_length = 0;
1743248e6799SNeel Natu 	vmexit->exitcode = VM_EXITCODE_REQIDLE;
17443dc3d32aSJohn Baldwin 	vmm_stat_incr(vm_vcpu(vm, vcpuid), VMEXIT_REQIDLE, 1);
1745248e6799SNeel Natu }
1746248e6799SNeel Natu 
1747248e6799SNeel Natu void
174840487465SNeel Natu vm_exit_astpending(struct vm *vm, int vcpuid, uint64_t rip)
174940487465SNeel Natu {
175040487465SNeel Natu 	struct vm_exit *vmexit;
175140487465SNeel Natu 
175240487465SNeel Natu 	vmexit = vm_exitinfo(vm, vcpuid);
175340487465SNeel Natu 	vmexit->rip = rip;
175440487465SNeel Natu 	vmexit->inst_length = 0;
175540487465SNeel Natu 	vmexit->exitcode = VM_EXITCODE_BOGUS;
17563dc3d32aSJohn Baldwin 	vmm_stat_incr(vm_vcpu(vm, vcpuid), VMEXIT_ASTPENDING, 1);
175740487465SNeel Natu }
175840487465SNeel Natu 
1759318224bbSNeel Natu int
1760318224bbSNeel Natu vm_run(struct vm *vm, struct vm_run *vmrun)
1761318224bbSNeel Natu {
1762248e6799SNeel Natu 	struct vm_eventinfo evinfo;
1763318224bbSNeel Natu 	int error, vcpuid;
1764318224bbSNeel Natu 	struct vcpu *vcpu;
1765318224bbSNeel Natu 	struct pcb *pcb;
1766d087a399SNeel Natu 	uint64_t tscval;
1767318224bbSNeel Natu 	struct vm_exit *vme;
1768becd9849SNeel Natu 	bool retu, intr_disabled;
1769318224bbSNeel Natu 	pmap_t pmap;
1770318224bbSNeel Natu 
1771318224bbSNeel Natu 	vcpuid = vmrun->cpuid;
1772318224bbSNeel Natu 
1773a488c9c9SRodney W. Grimes 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
1774318224bbSNeel Natu 		return (EINVAL);
1775318224bbSNeel Natu 
177695ebc360SNeel Natu 	if (!CPU_ISSET(vcpuid, &vm->active_cpus))
177795ebc360SNeel Natu 		return (EINVAL);
177895ebc360SNeel Natu 
177995ebc360SNeel Natu 	if (CPU_ISSET(vcpuid, &vm->suspended_cpus))
178095ebc360SNeel Natu 		return (EINVAL);
178195ebc360SNeel Natu 
1782318224bbSNeel Natu 	pmap = vmspace_pmap(vm->vmspace);
1783318224bbSNeel Natu 	vcpu = &vm->vcpu[vcpuid];
1784318224bbSNeel Natu 	vme = &vcpu->exitinfo;
1785248e6799SNeel Natu 	evinfo.rptr = &vm->rendezvous_func;
1786248e6799SNeel Natu 	evinfo.sptr = &vm->suspend;
1787248e6799SNeel Natu 	evinfo.iptr = &vcpu->reqidle;
1788318224bbSNeel Natu restart:
1789318224bbSNeel Natu 	critical_enter();
1790318224bbSNeel Natu 
1791318224bbSNeel Natu 	KASSERT(!CPU_ISSET(curcpu, &pmap->pm_active),
1792318224bbSNeel Natu 	    ("vm_run: absurd pm_active"));
1793318224bbSNeel Natu 
1794318224bbSNeel Natu 	tscval = rdtsc();
1795318224bbSNeel Natu 
1796318224bbSNeel Natu 	pcb = PCPU_GET(curpcb);
1797318224bbSNeel Natu 	set_pcb_flags(pcb, PCB_FULL_IRET);
1798318224bbSNeel Natu 
1799318224bbSNeel Natu 	restore_guest_fpustate(vcpu);
1800318224bbSNeel Natu 
1801318224bbSNeel Natu 	vcpu_require_state(vm, vcpuid, VCPU_RUNNING);
1802869c8d19SJohn Baldwin 	error = vmmops_run(vcpu->cookie, vcpu->nextrip, pmap, &evinfo);
1803318224bbSNeel Natu 	vcpu_require_state(vm, vcpuid, VCPU_FROZEN);
1804318224bbSNeel Natu 
1805318224bbSNeel Natu 	save_guest_fpustate(vcpu);
1806318224bbSNeel Natu 
18073dc3d32aSJohn Baldwin 	vmm_stat_incr(vcpu, VCPU_TOTAL_RUNTIME, rdtsc() - tscval);
1808318224bbSNeel Natu 
1809318224bbSNeel Natu 	critical_exit();
1810318224bbSNeel Natu 
1811318224bbSNeel Natu 	if (error == 0) {
1812becd9849SNeel Natu 		retu = false;
1813d087a399SNeel Natu 		vcpu->nextrip = vme->rip + vme->inst_length;
1814318224bbSNeel Natu 		switch (vme->exitcode) {
1815248e6799SNeel Natu 		case VM_EXITCODE_REQIDLE:
1816248e6799SNeel Natu 			error = vm_handle_reqidle(vm, vcpuid, &retu);
1817248e6799SNeel Natu 			break;
1818b15a09c0SNeel Natu 		case VM_EXITCODE_SUSPENDED:
1819b15a09c0SNeel Natu 			error = vm_handle_suspend(vm, vcpuid, &retu);
1820b15a09c0SNeel Natu 			break;
182130b94db8SNeel Natu 		case VM_EXITCODE_IOAPIC_EOI:
182230b94db8SNeel Natu 			vioapic_process_eoi(vm, vcpuid,
182330b94db8SNeel Natu 			    vme->u.ioapic_eoi.vector);
182430b94db8SNeel Natu 			break;
18255b8a8cd1SNeel Natu 		case VM_EXITCODE_RENDEZVOUS:
1826b837daddSKonstantin Belousov 			error = vm_handle_rendezvous(vm, vcpuid);
18275b8a8cd1SNeel Natu 			break;
1828318224bbSNeel Natu 		case VM_EXITCODE_HLT:
1829becd9849SNeel Natu 			intr_disabled = ((vme->u.hlt.rflags & PSL_I) == 0);
18301c052192SNeel Natu 			error = vm_handle_hlt(vm, vcpuid, intr_disabled, &retu);
1831318224bbSNeel Natu 			break;
1832318224bbSNeel Natu 		case VM_EXITCODE_PAGING:
1833318224bbSNeel Natu 			error = vm_handle_paging(vm, vcpuid, &retu);
1834318224bbSNeel Natu 			break;
1835318224bbSNeel Natu 		case VM_EXITCODE_INST_EMUL:
1836318224bbSNeel Natu 			error = vm_handle_inst_emul(vm, vcpuid, &retu);
1837318224bbSNeel Natu 			break;
1838d17b5104SNeel Natu 		case VM_EXITCODE_INOUT:
1839d17b5104SNeel Natu 		case VM_EXITCODE_INOUT_STR:
1840d17b5104SNeel Natu 			error = vm_handle_inout(vm, vcpuid, vme, &retu);
1841d17b5104SNeel Natu 			break;
184265145c7fSNeel Natu 		case VM_EXITCODE_MONITOR:
184365145c7fSNeel Natu 		case VM_EXITCODE_MWAIT:
184427d26457SAndrew Turner 		case VM_EXITCODE_VMINSN:
184565145c7fSNeel Natu 			vm_inject_ud(vm, vcpuid);
184665145c7fSNeel Natu 			break;
1847318224bbSNeel Natu 		default:
1848becd9849SNeel Natu 			retu = true;	/* handled in userland */
1849318224bbSNeel Natu 			break;
1850318224bbSNeel Natu 		}
1851318224bbSNeel Natu 	}
1852318224bbSNeel Natu 
18530bda8d3eSCorvin Köhne 	/*
18540bda8d3eSCorvin Köhne 	 * VM_EXITCODE_INST_EMUL could access the apic which could transform the
18550bda8d3eSCorvin Köhne 	 * exit code into VM_EXITCODE_IPI.
18560bda8d3eSCorvin Köhne 	 */
18570bda8d3eSCorvin Köhne 	if (error == 0 && vme->exitcode == VM_EXITCODE_IPI) {
18580bda8d3eSCorvin Köhne 		retu = false;
18590bda8d3eSCorvin Köhne 		error = vm_handle_ipi(vm, vcpuid, vme, &retu);
18600bda8d3eSCorvin Köhne 	}
18610bda8d3eSCorvin Köhne 
1862d087a399SNeel Natu 	if (error == 0 && retu == false)
1863f76fc5d4SNeel Natu 		goto restart;
1864f76fc5d4SNeel Natu 
18653dc3d32aSJohn Baldwin 	vmm_stat_incr(vcpu, VMEXIT_USERSPACE, 1);
1866248e6799SNeel Natu 	VCPU_CTR2(vm, vcpuid, "retu %d/%d", error, vme->exitcode);
1867248e6799SNeel Natu 
1868318224bbSNeel Natu 	/* copy the exit information */
1869318224bbSNeel Natu 	bcopy(vme, &vmrun->vm_exit, sizeof(struct vm_exit));
1870366f6083SPeter Grehan 	return (error);
1871366f6083SPeter Grehan }
1872366f6083SPeter Grehan 
1873366f6083SPeter Grehan int
1874c9c75df4SNeel Natu vm_restart_instruction(void *arg, int vcpuid)
1875c9c75df4SNeel Natu {
1876d087a399SNeel Natu 	struct vm *vm;
1877c9c75df4SNeel Natu 	struct vcpu *vcpu;
1878d087a399SNeel Natu 	enum vcpu_state state;
1879d087a399SNeel Natu 	uint64_t rip;
188073505a10SRobert Wing 	int error __diagused;
1881c9c75df4SNeel Natu 
1882d087a399SNeel Natu 	vm = arg;
1883a488c9c9SRodney W. Grimes 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
1884c9c75df4SNeel Natu 		return (EINVAL);
1885c9c75df4SNeel Natu 
1886c9c75df4SNeel Natu 	vcpu = &vm->vcpu[vcpuid];
1887d087a399SNeel Natu 	state = vcpu_get_state(vm, vcpuid, NULL);
1888d087a399SNeel Natu 	if (state == VCPU_RUNNING) {
1889d087a399SNeel Natu 		/*
1890d087a399SNeel Natu 		 * When a vcpu is "running" the next instruction is determined
1891d087a399SNeel Natu 		 * by adding 'rip' and 'inst_length' in the vcpu's 'exitinfo'.
1892d087a399SNeel Natu 		 * Thus setting 'inst_length' to zero will cause the current
1893d087a399SNeel Natu 		 * instruction to be restarted.
1894d087a399SNeel Natu 		 */
1895c9c75df4SNeel Natu 		vcpu->exitinfo.inst_length = 0;
1896d087a399SNeel Natu 		VCPU_CTR1(vm, vcpuid, "restarting instruction at %#lx by "
1897d087a399SNeel Natu 		    "setting inst_length to zero", vcpu->exitinfo.rip);
1898d087a399SNeel Natu 	} else if (state == VCPU_FROZEN) {
1899d087a399SNeel Natu 		/*
1900d087a399SNeel Natu 		 * When a vcpu is "frozen" it is outside the critical section
190115add60dSPeter Grehan 		 * around vmmops_run() and 'nextrip' points to the next
190215add60dSPeter Grehan 		 * instruction. Thus instruction restart is achieved by setting
190315add60dSPeter Grehan 		 * 'nextrip' to the vcpu's %rip.
1904d087a399SNeel Natu 		 */
1905d087a399SNeel Natu 		error = vm_get_register(vm, vcpuid, VM_REG_GUEST_RIP, &rip);
1906d087a399SNeel Natu 		KASSERT(!error, ("%s: error %d getting rip", __func__, error));
1907d087a399SNeel Natu 		VCPU_CTR2(vm, vcpuid, "restarting instruction by updating "
1908d087a399SNeel Natu 		    "nextrip from %#lx to %#lx", vcpu->nextrip, rip);
1909d087a399SNeel Natu 		vcpu->nextrip = rip;
1910d087a399SNeel Natu 	} else {
1911d087a399SNeel Natu 		panic("%s: invalid state %d", __func__, state);
1912d087a399SNeel Natu 	}
1913c9c75df4SNeel Natu 	return (0);
1914c9c75df4SNeel Natu }
1915c9c75df4SNeel Natu 
1916c9c75df4SNeel Natu int
1917091d4532SNeel Natu vm_exit_intinfo(struct vm *vm, int vcpuid, uint64_t info)
1918091d4532SNeel Natu {
1919091d4532SNeel Natu 	struct vcpu *vcpu;
1920091d4532SNeel Natu 	int type, vector;
1921091d4532SNeel Natu 
1922a488c9c9SRodney W. Grimes 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
1923091d4532SNeel Natu 		return (EINVAL);
1924091d4532SNeel Natu 
1925091d4532SNeel Natu 	vcpu = &vm->vcpu[vcpuid];
1926091d4532SNeel Natu 
1927091d4532SNeel Natu 	if (info & VM_INTINFO_VALID) {
1928091d4532SNeel Natu 		type = info & VM_INTINFO_TYPE;
1929091d4532SNeel Natu 		vector = info & 0xff;
1930091d4532SNeel Natu 		if (type == VM_INTINFO_NMI && vector != IDT_NMI)
1931091d4532SNeel Natu 			return (EINVAL);
1932091d4532SNeel Natu 		if (type == VM_INTINFO_HWEXCEPTION && vector >= 32)
1933091d4532SNeel Natu 			return (EINVAL);
1934091d4532SNeel Natu 		if (info & VM_INTINFO_RSVD)
1935091d4532SNeel Natu 			return (EINVAL);
1936091d4532SNeel Natu 	} else {
1937091d4532SNeel Natu 		info = 0;
1938091d4532SNeel Natu 	}
1939091d4532SNeel Natu 	VCPU_CTR2(vm, vcpuid, "%s: info1(%#lx)", __func__, info);
1940091d4532SNeel Natu 	vcpu->exitintinfo = info;
1941091d4532SNeel Natu 	return (0);
1942091d4532SNeel Natu }
1943091d4532SNeel Natu 
1944091d4532SNeel Natu enum exc_class {
1945091d4532SNeel Natu 	EXC_BENIGN,
1946091d4532SNeel Natu 	EXC_CONTRIBUTORY,
1947091d4532SNeel Natu 	EXC_PAGEFAULT
1948091d4532SNeel Natu };
1949091d4532SNeel Natu 
1950091d4532SNeel Natu #define	IDT_VE	20	/* Virtualization Exception (Intel specific) */
1951091d4532SNeel Natu 
1952091d4532SNeel Natu static enum exc_class
1953091d4532SNeel Natu exception_class(uint64_t info)
1954091d4532SNeel Natu {
1955091d4532SNeel Natu 	int type, vector;
1956091d4532SNeel Natu 
1957091d4532SNeel Natu 	KASSERT(info & VM_INTINFO_VALID, ("intinfo must be valid: %#lx", info));
1958091d4532SNeel Natu 	type = info & VM_INTINFO_TYPE;
1959091d4532SNeel Natu 	vector = info & 0xff;
1960091d4532SNeel Natu 
1961091d4532SNeel Natu 	/* Table 6-4, "Interrupt and Exception Classes", Intel SDM, Vol 3 */
1962091d4532SNeel Natu 	switch (type) {
1963091d4532SNeel Natu 	case VM_INTINFO_HWINTR:
1964091d4532SNeel Natu 	case VM_INTINFO_SWINTR:
1965091d4532SNeel Natu 	case VM_INTINFO_NMI:
1966091d4532SNeel Natu 		return (EXC_BENIGN);
1967091d4532SNeel Natu 	default:
1968091d4532SNeel Natu 		/*
1969091d4532SNeel Natu 		 * Hardware exception.
1970091d4532SNeel Natu 		 *
1971091d4532SNeel Natu 		 * SVM and VT-x use identical type values to represent NMI,
1972091d4532SNeel Natu 		 * hardware interrupt and software interrupt.
1973091d4532SNeel Natu 		 *
1974091d4532SNeel Natu 		 * SVM uses type '3' for all exceptions. VT-x uses type '3'
1975091d4532SNeel Natu 		 * for exceptions except #BP and #OF. #BP and #OF use a type
1976091d4532SNeel Natu 		 * value of '5' or '6'. Therefore we don't check for explicit
1977091d4532SNeel Natu 		 * values of 'type' to classify 'intinfo' into a hardware
1978091d4532SNeel Natu 		 * exception.
1979091d4532SNeel Natu 		 */
1980091d4532SNeel Natu 		break;
1981091d4532SNeel Natu 	}
1982091d4532SNeel Natu 
1983091d4532SNeel Natu 	switch (vector) {
1984091d4532SNeel Natu 	case IDT_PF:
1985091d4532SNeel Natu 	case IDT_VE:
1986091d4532SNeel Natu 		return (EXC_PAGEFAULT);
1987091d4532SNeel Natu 	case IDT_DE:
1988091d4532SNeel Natu 	case IDT_TS:
1989091d4532SNeel Natu 	case IDT_NP:
1990091d4532SNeel Natu 	case IDT_SS:
1991091d4532SNeel Natu 	case IDT_GP:
1992091d4532SNeel Natu 		return (EXC_CONTRIBUTORY);
1993091d4532SNeel Natu 	default:
1994091d4532SNeel Natu 		return (EXC_BENIGN);
1995091d4532SNeel Natu 	}
1996091d4532SNeel Natu }
1997091d4532SNeel Natu 
1998091d4532SNeel Natu static int
1999091d4532SNeel Natu nested_fault(struct vm *vm, int vcpuid, uint64_t info1, uint64_t info2,
2000091d4532SNeel Natu     uint64_t *retinfo)
2001091d4532SNeel Natu {
2002091d4532SNeel Natu 	enum exc_class exc1, exc2;
2003091d4532SNeel Natu 	int type1, vector1;
2004091d4532SNeel Natu 
2005091d4532SNeel Natu 	KASSERT(info1 & VM_INTINFO_VALID, ("info1 %#lx is not valid", info1));
2006091d4532SNeel Natu 	KASSERT(info2 & VM_INTINFO_VALID, ("info2 %#lx is not valid", info2));
2007091d4532SNeel Natu 
2008091d4532SNeel Natu 	/*
2009091d4532SNeel Natu 	 * If an exception occurs while attempting to call the double-fault
2010091d4532SNeel Natu 	 * handler the processor enters shutdown mode (aka triple fault).
2011091d4532SNeel Natu 	 */
2012091d4532SNeel Natu 	type1 = info1 & VM_INTINFO_TYPE;
2013091d4532SNeel Natu 	vector1 = info1 & 0xff;
2014091d4532SNeel Natu 	if (type1 == VM_INTINFO_HWEXCEPTION && vector1 == IDT_DF) {
2015091d4532SNeel Natu 		VCPU_CTR2(vm, vcpuid, "triple fault: info1(%#lx), info2(%#lx)",
2016091d4532SNeel Natu 		    info1, info2);
2017091d4532SNeel Natu 		vm_suspend(vm, VM_SUSPEND_TRIPLEFAULT);
2018091d4532SNeel Natu 		*retinfo = 0;
2019091d4532SNeel Natu 		return (0);
2020091d4532SNeel Natu 	}
2021091d4532SNeel Natu 
2022091d4532SNeel Natu 	/*
2023091d4532SNeel Natu 	 * Table 6-5 "Conditions for Generating a Double Fault", Intel SDM, Vol3
2024091d4532SNeel Natu 	 */
2025091d4532SNeel Natu 	exc1 = exception_class(info1);
2026091d4532SNeel Natu 	exc2 = exception_class(info2);
2027091d4532SNeel Natu 	if ((exc1 == EXC_CONTRIBUTORY && exc2 == EXC_CONTRIBUTORY) ||
2028091d4532SNeel Natu 	    (exc1 == EXC_PAGEFAULT && exc2 != EXC_BENIGN)) {
2029091d4532SNeel Natu 		/* Convert nested fault into a double fault. */
2030091d4532SNeel Natu 		*retinfo = IDT_DF;
2031091d4532SNeel Natu 		*retinfo |= VM_INTINFO_VALID | VM_INTINFO_HWEXCEPTION;
2032091d4532SNeel Natu 		*retinfo |= VM_INTINFO_DEL_ERRCODE;
2033091d4532SNeel Natu 	} else {
2034091d4532SNeel Natu 		/* Handle exceptions serially */
2035091d4532SNeel Natu 		*retinfo = info2;
2036091d4532SNeel Natu 	}
2037091d4532SNeel Natu 	return (1);
2038091d4532SNeel Natu }
2039091d4532SNeel Natu 
2040091d4532SNeel Natu static uint64_t
2041091d4532SNeel Natu vcpu_exception_intinfo(struct vcpu *vcpu)
2042091d4532SNeel Natu {
2043091d4532SNeel Natu 	uint64_t info = 0;
2044091d4532SNeel Natu 
2045091d4532SNeel Natu 	if (vcpu->exception_pending) {
2046c9c75df4SNeel Natu 		info = vcpu->exc_vector & 0xff;
2047091d4532SNeel Natu 		info |= VM_INTINFO_VALID | VM_INTINFO_HWEXCEPTION;
2048c9c75df4SNeel Natu 		if (vcpu->exc_errcode_valid) {
2049091d4532SNeel Natu 			info |= VM_INTINFO_DEL_ERRCODE;
2050c9c75df4SNeel Natu 			info |= (uint64_t)vcpu->exc_errcode << 32;
2051091d4532SNeel Natu 		}
2052091d4532SNeel Natu 	}
2053091d4532SNeel Natu 	return (info);
2054091d4532SNeel Natu }
2055091d4532SNeel Natu 
2056091d4532SNeel Natu int
2057091d4532SNeel Natu vm_entry_intinfo(struct vm *vm, int vcpuid, uint64_t *retinfo)
2058091d4532SNeel Natu {
2059091d4532SNeel Natu 	struct vcpu *vcpu;
2060091d4532SNeel Natu 	uint64_t info1, info2;
2061091d4532SNeel Natu 	int valid;
2062091d4532SNeel Natu 
2063a488c9c9SRodney W. Grimes 	KASSERT(vcpuid >= 0 &&
2064a488c9c9SRodney W. Grimes 	    vcpuid < vm->maxcpus, ("invalid vcpu %d", vcpuid));
2065091d4532SNeel Natu 
2066091d4532SNeel Natu 	vcpu = &vm->vcpu[vcpuid];
2067091d4532SNeel Natu 
2068091d4532SNeel Natu 	info1 = vcpu->exitintinfo;
2069091d4532SNeel Natu 	vcpu->exitintinfo = 0;
2070091d4532SNeel Natu 
2071091d4532SNeel Natu 	info2 = 0;
2072091d4532SNeel Natu 	if (vcpu->exception_pending) {
2073091d4532SNeel Natu 		info2 = vcpu_exception_intinfo(vcpu);
2074091d4532SNeel Natu 		vcpu->exception_pending = 0;
2075091d4532SNeel Natu 		VCPU_CTR2(vm, vcpuid, "Exception %d delivered: %#lx",
2076c9c75df4SNeel Natu 		    vcpu->exc_vector, info2);
2077091d4532SNeel Natu 	}
2078091d4532SNeel Natu 
2079091d4532SNeel Natu 	if ((info1 & VM_INTINFO_VALID) && (info2 & VM_INTINFO_VALID)) {
2080091d4532SNeel Natu 		valid = nested_fault(vm, vcpuid, info1, info2, retinfo);
2081091d4532SNeel Natu 	} else if (info1 & VM_INTINFO_VALID) {
2082091d4532SNeel Natu 		*retinfo = info1;
2083091d4532SNeel Natu 		valid = 1;
2084091d4532SNeel Natu 	} else if (info2 & VM_INTINFO_VALID) {
2085091d4532SNeel Natu 		*retinfo = info2;
2086091d4532SNeel Natu 		valid = 1;
2087091d4532SNeel Natu 	} else {
2088091d4532SNeel Natu 		valid = 0;
2089091d4532SNeel Natu 	}
2090091d4532SNeel Natu 
2091091d4532SNeel Natu 	if (valid) {
2092091d4532SNeel Natu 		VCPU_CTR4(vm, vcpuid, "%s: info1(%#lx), info2(%#lx), "
2093091d4532SNeel Natu 		    "retinfo(%#lx)", __func__, info1, info2, *retinfo);
2094091d4532SNeel Natu 	}
2095091d4532SNeel Natu 
2096091d4532SNeel Natu 	return (valid);
2097091d4532SNeel Natu }
2098091d4532SNeel Natu 
2099091d4532SNeel Natu int
2100091d4532SNeel Natu vm_get_intinfo(struct vm *vm, int vcpuid, uint64_t *info1, uint64_t *info2)
2101091d4532SNeel Natu {
2102091d4532SNeel Natu 	struct vcpu *vcpu;
2103091d4532SNeel Natu 
2104a488c9c9SRodney W. Grimes 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2105091d4532SNeel Natu 		return (EINVAL);
2106091d4532SNeel Natu 
2107091d4532SNeel Natu 	vcpu = &vm->vcpu[vcpuid];
2108091d4532SNeel Natu 	*info1 = vcpu->exitintinfo;
2109091d4532SNeel Natu 	*info2 = vcpu_exception_intinfo(vcpu);
2110091d4532SNeel Natu 	return (0);
2111091d4532SNeel Natu }
2112091d4532SNeel Natu 
2113091d4532SNeel Natu int
2114c9c75df4SNeel Natu vm_inject_exception(struct vm *vm, int vcpuid, int vector, int errcode_valid,
2115c9c75df4SNeel Natu     uint32_t errcode, int restart_instruction)
2116366f6083SPeter Grehan {
2117dc506506SNeel Natu 	struct vcpu *vcpu;
211847b9935dSNeel Natu 	uint64_t regval;
211973505a10SRobert Wing 	int error __diagused;
2120dc506506SNeel Natu 
2121a488c9c9SRodney W. Grimes 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2122366f6083SPeter Grehan 		return (EINVAL);
2123366f6083SPeter Grehan 
2124c9c75df4SNeel Natu 	if (vector < 0 || vector >= 32)
2125366f6083SPeter Grehan 		return (EINVAL);
2126366f6083SPeter Grehan 
2127091d4532SNeel Natu 	/*
2128091d4532SNeel Natu 	 * A double fault exception should never be injected directly into
2129091d4532SNeel Natu 	 * the guest. It is a derived exception that results from specific
2130091d4532SNeel Natu 	 * combinations of nested faults.
2131091d4532SNeel Natu 	 */
2132c9c75df4SNeel Natu 	if (vector == IDT_DF)
2133091d4532SNeel Natu 		return (EINVAL);
2134091d4532SNeel Natu 
2135dc506506SNeel Natu 	vcpu = &vm->vcpu[vcpuid];
2136366f6083SPeter Grehan 
2137dc506506SNeel Natu 	if (vcpu->exception_pending) {
2138dc506506SNeel Natu 		VCPU_CTR2(vm, vcpuid, "Unable to inject exception %d due to "
2139c9c75df4SNeel Natu 		    "pending exception %d", vector, vcpu->exc_vector);
2140dc506506SNeel Natu 		return (EBUSY);
2141dc506506SNeel Natu 	}
2142dc506506SNeel Natu 
214347b9935dSNeel Natu 	if (errcode_valid) {
214447b9935dSNeel Natu 		/*
214547b9935dSNeel Natu 		 * Exceptions don't deliver an error code in real mode.
214647b9935dSNeel Natu 		 */
214747b9935dSNeel Natu 		error = vm_get_register(vm, vcpuid, VM_REG_GUEST_CR0, &regval);
214847b9935dSNeel Natu 		KASSERT(!error, ("%s: error %d getting CR0", __func__, error));
214947b9935dSNeel Natu 		if (!(regval & CR0_PE))
215047b9935dSNeel Natu 			errcode_valid = 0;
215147b9935dSNeel Natu 	}
215247b9935dSNeel Natu 
21532ce12423SNeel Natu 	/*
21542ce12423SNeel Natu 	 * From section 26.6.1 "Interruptibility State" in Intel SDM:
21552ce12423SNeel Natu 	 *
21562ce12423SNeel Natu 	 * Event blocking by "STI" or "MOV SS" is cleared after guest executes
21572ce12423SNeel Natu 	 * one instruction or incurs an exception.
21582ce12423SNeel Natu 	 */
21592ce12423SNeel Natu 	error = vm_set_register(vm, vcpuid, VM_REG_GUEST_INTR_SHADOW, 0);
21602ce12423SNeel Natu 	KASSERT(error == 0, ("%s: error %d clearing interrupt shadow",
21612ce12423SNeel Natu 	    __func__, error));
21622ce12423SNeel Natu 
2163c9c75df4SNeel Natu 	if (restart_instruction)
2164c9c75df4SNeel Natu 		vm_restart_instruction(vm, vcpuid);
2165c9c75df4SNeel Natu 
2166dc506506SNeel Natu 	vcpu->exception_pending = 1;
2167c9c75df4SNeel Natu 	vcpu->exc_vector = vector;
2168c9c75df4SNeel Natu 	vcpu->exc_errcode = errcode;
2169c9c75df4SNeel Natu 	vcpu->exc_errcode_valid = errcode_valid;
2170c9c75df4SNeel Natu 	VCPU_CTR1(vm, vcpuid, "Exception %d pending", vector);
2171dc506506SNeel Natu 	return (0);
2172dc506506SNeel Natu }
2173dc506506SNeel Natu 
2174d37f2adbSNeel Natu void
2175d37f2adbSNeel Natu vm_inject_fault(void *vmarg, int vcpuid, int vector, int errcode_valid,
2176d37f2adbSNeel Natu     int errcode)
2177dc506506SNeel Natu {
2178d37f2adbSNeel Natu 	struct vm *vm;
217973505a10SRobert Wing 	int error __diagused, restart_instruction;
2180dc506506SNeel Natu 
2181d37f2adbSNeel Natu 	vm = vmarg;
2182c9c75df4SNeel Natu 	restart_instruction = 1;
2183d37f2adbSNeel Natu 
2184c9c75df4SNeel Natu 	error = vm_inject_exception(vm, vcpuid, vector, errcode_valid,
2185c9c75df4SNeel Natu 	    errcode, restart_instruction);
2186dc506506SNeel Natu 	KASSERT(error == 0, ("vm_inject_exception error %d", error));
2187dc506506SNeel Natu }
2188dc506506SNeel Natu 
2189dc506506SNeel Natu void
2190d37f2adbSNeel Natu vm_inject_pf(void *vmarg, int vcpuid, int error_code, uint64_t cr2)
2191fd949af6SNeel Natu {
2192d37f2adbSNeel Natu 	struct vm *vm;
219373505a10SRobert Wing 	int error __diagused;
219437a723a5SNeel Natu 
2195d37f2adbSNeel Natu 	vm = vmarg;
219637a723a5SNeel Natu 	VCPU_CTR2(vm, vcpuid, "Injecting page fault: error_code %#x, cr2 %#lx",
219737a723a5SNeel Natu 	    error_code, cr2);
219837a723a5SNeel Natu 
219937a723a5SNeel Natu 	error = vm_set_register(vm, vcpuid, VM_REG_GUEST_CR2, cr2);
220037a723a5SNeel Natu 	KASSERT(error == 0, ("vm_set_register(cr2) error %d", error));
2201fd949af6SNeel Natu 
2202d37f2adbSNeel Natu 	vm_inject_fault(vm, vcpuid, IDT_PF, 1, error_code);
2203366f6083SPeter Grehan }
2204366f6083SPeter Grehan 
220561592433SNeel Natu static VMM_STAT(VCPU_NMI_COUNT, "number of NMIs delivered to vcpu");
2206366f6083SPeter Grehan 
2207f352ff0cSNeel Natu int
2208f352ff0cSNeel Natu vm_inject_nmi(struct vm *vm, int vcpuid)
2209f352ff0cSNeel Natu {
2210f352ff0cSNeel Natu 	struct vcpu *vcpu;
2211f352ff0cSNeel Natu 
2212a488c9c9SRodney W. Grimes 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2213366f6083SPeter Grehan 		return (EINVAL);
2214366f6083SPeter Grehan 
2215f352ff0cSNeel Natu 	vcpu = &vm->vcpu[vcpuid];
2216f352ff0cSNeel Natu 
2217f352ff0cSNeel Natu 	vcpu->nmi_pending = 1;
2218de5ea6b6SNeel Natu 	vcpu_notify_event(vm, vcpuid, false);
2219f352ff0cSNeel Natu 	return (0);
2220f352ff0cSNeel Natu }
2221f352ff0cSNeel Natu 
2222f352ff0cSNeel Natu int
2223f352ff0cSNeel Natu vm_nmi_pending(struct vm *vm, int vcpuid)
2224f352ff0cSNeel Natu {
2225f352ff0cSNeel Natu 	struct vcpu *vcpu;
2226f352ff0cSNeel Natu 
2227a488c9c9SRodney W. Grimes 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2228f352ff0cSNeel Natu 		panic("vm_nmi_pending: invalid vcpuid %d", vcpuid);
2229f352ff0cSNeel Natu 
2230f352ff0cSNeel Natu 	vcpu = &vm->vcpu[vcpuid];
2231f352ff0cSNeel Natu 
2232f352ff0cSNeel Natu 	return (vcpu->nmi_pending);
2233f352ff0cSNeel Natu }
2234f352ff0cSNeel Natu 
2235f352ff0cSNeel Natu void
2236f352ff0cSNeel Natu vm_nmi_clear(struct vm *vm, int vcpuid)
2237f352ff0cSNeel Natu {
2238f352ff0cSNeel Natu 	struct vcpu *vcpu;
2239f352ff0cSNeel Natu 
2240a488c9c9SRodney W. Grimes 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2241f352ff0cSNeel Natu 		panic("vm_nmi_pending: invalid vcpuid %d", vcpuid);
2242f352ff0cSNeel Natu 
2243f352ff0cSNeel Natu 	vcpu = &vm->vcpu[vcpuid];
2244f352ff0cSNeel Natu 
2245f352ff0cSNeel Natu 	if (vcpu->nmi_pending == 0)
2246f352ff0cSNeel Natu 		panic("vm_nmi_clear: inconsistent nmi_pending state");
2247f352ff0cSNeel Natu 
2248f352ff0cSNeel Natu 	vcpu->nmi_pending = 0;
22493dc3d32aSJohn Baldwin 	vmm_stat_incr(vcpu, VCPU_NMI_COUNT, 1);
2250366f6083SPeter Grehan }
2251366f6083SPeter Grehan 
22520775fbb4STycho Nightingale static VMM_STAT(VCPU_EXTINT_COUNT, "number of ExtINTs delivered to vcpu");
22530775fbb4STycho Nightingale 
22540775fbb4STycho Nightingale int
22550775fbb4STycho Nightingale vm_inject_extint(struct vm *vm, int vcpuid)
22560775fbb4STycho Nightingale {
22570775fbb4STycho Nightingale 	struct vcpu *vcpu;
22580775fbb4STycho Nightingale 
2259a488c9c9SRodney W. Grimes 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
22600775fbb4STycho Nightingale 		return (EINVAL);
22610775fbb4STycho Nightingale 
22620775fbb4STycho Nightingale 	vcpu = &vm->vcpu[vcpuid];
22630775fbb4STycho Nightingale 
22640775fbb4STycho Nightingale 	vcpu->extint_pending = 1;
22650775fbb4STycho Nightingale 	vcpu_notify_event(vm, vcpuid, false);
22660775fbb4STycho Nightingale 	return (0);
22670775fbb4STycho Nightingale }
22680775fbb4STycho Nightingale 
22690775fbb4STycho Nightingale int
22700775fbb4STycho Nightingale vm_extint_pending(struct vm *vm, int vcpuid)
22710775fbb4STycho Nightingale {
22720775fbb4STycho Nightingale 	struct vcpu *vcpu;
22730775fbb4STycho Nightingale 
2274a488c9c9SRodney W. Grimes 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
22750775fbb4STycho Nightingale 		panic("vm_extint_pending: invalid vcpuid %d", vcpuid);
22760775fbb4STycho Nightingale 
22770775fbb4STycho Nightingale 	vcpu = &vm->vcpu[vcpuid];
22780775fbb4STycho Nightingale 
22790775fbb4STycho Nightingale 	return (vcpu->extint_pending);
22800775fbb4STycho Nightingale }
22810775fbb4STycho Nightingale 
22820775fbb4STycho Nightingale void
22830775fbb4STycho Nightingale vm_extint_clear(struct vm *vm, int vcpuid)
22840775fbb4STycho Nightingale {
22850775fbb4STycho Nightingale 	struct vcpu *vcpu;
22860775fbb4STycho Nightingale 
2287a488c9c9SRodney W. Grimes 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
22880775fbb4STycho Nightingale 		panic("vm_extint_pending: invalid vcpuid %d", vcpuid);
22890775fbb4STycho Nightingale 
22900775fbb4STycho Nightingale 	vcpu = &vm->vcpu[vcpuid];
22910775fbb4STycho Nightingale 
22920775fbb4STycho Nightingale 	if (vcpu->extint_pending == 0)
22930775fbb4STycho Nightingale 		panic("vm_extint_clear: inconsistent extint_pending state");
22940775fbb4STycho Nightingale 
22950775fbb4STycho Nightingale 	vcpu->extint_pending = 0;
22963dc3d32aSJohn Baldwin 	vmm_stat_incr(vcpu, VCPU_EXTINT_COUNT, 1);
22970775fbb4STycho Nightingale }
22980775fbb4STycho Nightingale 
2299366f6083SPeter Grehan int
2300366f6083SPeter Grehan vm_get_capability(struct vm *vm, int vcpu, int type, int *retval)
2301366f6083SPeter Grehan {
2302a488c9c9SRodney W. Grimes 	if (vcpu < 0 || vcpu >= vm->maxcpus)
2303366f6083SPeter Grehan 		return (EINVAL);
2304366f6083SPeter Grehan 
2305366f6083SPeter Grehan 	if (type < 0 || type >= VM_CAP_MAX)
2306366f6083SPeter Grehan 		return (EINVAL);
2307366f6083SPeter Grehan 
2308869c8d19SJohn Baldwin 	return (vmmops_getcap(vcpu_cookie(vm, vcpu), type, retval));
2309366f6083SPeter Grehan }
2310366f6083SPeter Grehan 
2311366f6083SPeter Grehan int
2312366f6083SPeter Grehan vm_set_capability(struct vm *vm, int vcpu, int type, int val)
2313366f6083SPeter Grehan {
2314a488c9c9SRodney W. Grimes 	if (vcpu < 0 || vcpu >= vm->maxcpus)
2315366f6083SPeter Grehan 		return (EINVAL);
2316366f6083SPeter Grehan 
2317366f6083SPeter Grehan 	if (type < 0 || type >= VM_CAP_MAX)
2318366f6083SPeter Grehan 		return (EINVAL);
2319366f6083SPeter Grehan 
2320869c8d19SJohn Baldwin 	return (vmmops_setcap(vcpu_cookie(vm, vcpu), type, val));
2321366f6083SPeter Grehan }
2322366f6083SPeter Grehan 
2323950af9ffSJohn Baldwin struct vm *
2324950af9ffSJohn Baldwin vcpu_vm(struct vcpu *vcpu)
2325950af9ffSJohn Baldwin {
2326950af9ffSJohn Baldwin 	return (vcpu->vm);
2327950af9ffSJohn Baldwin }
2328950af9ffSJohn Baldwin 
2329950af9ffSJohn Baldwin int
2330950af9ffSJohn Baldwin vcpu_vcpuid(struct vcpu *vcpu)
2331950af9ffSJohn Baldwin {
2332950af9ffSJohn Baldwin 	return (vcpu->vcpuid);
2333950af9ffSJohn Baldwin }
2334950af9ffSJohn Baldwin 
2335950af9ffSJohn Baldwin struct vcpu *
2336950af9ffSJohn Baldwin vm_vcpu(struct vm *vm, int vcpuid)
2337950af9ffSJohn Baldwin {
2338950af9ffSJohn Baldwin 	return (&vm->vcpu[vcpuid]);
2339950af9ffSJohn Baldwin }
2340950af9ffSJohn Baldwin 
2341366f6083SPeter Grehan struct vlapic *
2342366f6083SPeter Grehan vm_lapic(struct vm *vm, int cpu)
2343366f6083SPeter Grehan {
2344366f6083SPeter Grehan 	return (vm->vcpu[cpu].vlapic);
2345366f6083SPeter Grehan }
2346366f6083SPeter Grehan 
2347565bbb86SNeel Natu struct vioapic *
2348565bbb86SNeel Natu vm_ioapic(struct vm *vm)
2349565bbb86SNeel Natu {
2350565bbb86SNeel Natu 
2351565bbb86SNeel Natu 	return (vm->vioapic);
2352565bbb86SNeel Natu }
2353565bbb86SNeel Natu 
235408e3ff32SNeel Natu struct vhpet *
235508e3ff32SNeel Natu vm_hpet(struct vm *vm)
235608e3ff32SNeel Natu {
235708e3ff32SNeel Natu 
235808e3ff32SNeel Natu 	return (vm->vhpet);
235908e3ff32SNeel Natu }
236008e3ff32SNeel Natu 
2361490d56c5SEd Maste bool
2362366f6083SPeter Grehan vmm_is_pptdev(int bus, int slot, int func)
2363366f6083SPeter Grehan {
2364490d56c5SEd Maste 	int b, f, i, n, s;
2365366f6083SPeter Grehan 	char *val, *cp, *cp2;
2366490d56c5SEd Maste 	bool found;
2367366f6083SPeter Grehan 
2368366f6083SPeter Grehan 	/*
236907044a96SNeel Natu 	 * XXX
237007044a96SNeel Natu 	 * The length of an environment variable is limited to 128 bytes which
237107044a96SNeel Natu 	 * puts an upper limit on the number of passthru devices that may be
237207044a96SNeel Natu 	 * specified using a single environment variable.
237307044a96SNeel Natu 	 *
237407044a96SNeel Natu 	 * Work around this by scanning multiple environment variable
237507044a96SNeel Natu 	 * names instead of a single one - yuck!
2376366f6083SPeter Grehan 	 */
237707044a96SNeel Natu 	const char *names[] = { "pptdevs", "pptdevs2", "pptdevs3", NULL };
237807044a96SNeel Natu 
237907044a96SNeel Natu 	/* set pptdevs="1/2/3 4/5/6 7/8/9 10/11/12" */
2380490d56c5SEd Maste 	found = false;
238107044a96SNeel Natu 	for (i = 0; names[i] != NULL && !found; i++) {
23822be111bfSDavide Italiano 		cp = val = kern_getenv(names[i]);
2383366f6083SPeter Grehan 		while (cp != NULL && *cp != '\0') {
2384366f6083SPeter Grehan 			if ((cp2 = strchr(cp, ' ')) != NULL)
2385366f6083SPeter Grehan 				*cp2 = '\0';
2386366f6083SPeter Grehan 
2387366f6083SPeter Grehan 			n = sscanf(cp, "%d/%d/%d", &b, &s, &f);
2388366f6083SPeter Grehan 			if (n == 3 && bus == b && slot == s && func == f) {
2389490d56c5SEd Maste 				found = true;
2390366f6083SPeter Grehan 				break;
2391366f6083SPeter Grehan 			}
2392366f6083SPeter Grehan 
2393366f6083SPeter Grehan 			if (cp2 != NULL)
2394366f6083SPeter Grehan 				*cp2++ = ' ';
2395366f6083SPeter Grehan 
2396366f6083SPeter Grehan 			cp = cp2;
2397366f6083SPeter Grehan 		}
2398366f6083SPeter Grehan 		freeenv(val);
239907044a96SNeel Natu 	}
2400366f6083SPeter Grehan 	return (found);
2401366f6083SPeter Grehan }
2402366f6083SPeter Grehan 
2403366f6083SPeter Grehan void *
2404366f6083SPeter Grehan vm_iommu_domain(struct vm *vm)
2405366f6083SPeter Grehan {
2406366f6083SPeter Grehan 
2407366f6083SPeter Grehan 	return (vm->iommu);
2408366f6083SPeter Grehan }
2409366f6083SPeter Grehan 
241075dd3366SNeel Natu int
2411f80330a8SNeel Natu vcpu_set_state(struct vm *vm, int vcpuid, enum vcpu_state newstate,
2412f80330a8SNeel Natu     bool from_idle)
2413366f6083SPeter Grehan {
241475dd3366SNeel Natu 	int error;
2415366f6083SPeter Grehan 	struct vcpu *vcpu;
2416366f6083SPeter Grehan 
2417a488c9c9SRodney W. Grimes 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2418366f6083SPeter Grehan 		panic("vm_set_run_state: invalid vcpuid %d", vcpuid);
2419366f6083SPeter Grehan 
2420366f6083SPeter Grehan 	vcpu = &vm->vcpu[vcpuid];
2421366f6083SPeter Grehan 
242275dd3366SNeel Natu 	vcpu_lock(vcpu);
2423248e6799SNeel Natu 	error = vcpu_set_state_locked(vm, vcpuid, newstate, from_idle);
242475dd3366SNeel Natu 	vcpu_unlock(vcpu);
242575dd3366SNeel Natu 
242675dd3366SNeel Natu 	return (error);
242775dd3366SNeel Natu }
242875dd3366SNeel Natu 
242975dd3366SNeel Natu enum vcpu_state
2430d3c11f40SPeter Grehan vcpu_get_state(struct vm *vm, int vcpuid, int *hostcpu)
2431366f6083SPeter Grehan {
2432366f6083SPeter Grehan 	struct vcpu *vcpu;
243375dd3366SNeel Natu 	enum vcpu_state state;
2434366f6083SPeter Grehan 
2435a488c9c9SRodney W. Grimes 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2436366f6083SPeter Grehan 		panic("vm_get_run_state: invalid vcpuid %d", vcpuid);
2437366f6083SPeter Grehan 
2438366f6083SPeter Grehan 	vcpu = &vm->vcpu[vcpuid];
2439366f6083SPeter Grehan 
244075dd3366SNeel Natu 	vcpu_lock(vcpu);
244175dd3366SNeel Natu 	state = vcpu->state;
2442d3c11f40SPeter Grehan 	if (hostcpu != NULL)
2443d3c11f40SPeter Grehan 		*hostcpu = vcpu->hostcpu;
244475dd3366SNeel Natu 	vcpu_unlock(vcpu);
2445366f6083SPeter Grehan 
244675dd3366SNeel Natu 	return (state);
2447366f6083SPeter Grehan }
2448366f6083SPeter Grehan 
244995ebc360SNeel Natu int
2450366f6083SPeter Grehan vm_activate_cpu(struct vm *vm, int vcpuid)
2451366f6083SPeter Grehan {
2452366f6083SPeter Grehan 
2453a488c9c9SRodney W. Grimes 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
245495ebc360SNeel Natu 		return (EINVAL);
245595ebc360SNeel Natu 
245695ebc360SNeel Natu 	if (CPU_ISSET(vcpuid, &vm->active_cpus))
245795ebc360SNeel Natu 		return (EBUSY);
245822d822c6SNeel Natu 
245922d822c6SNeel Natu 	VCPU_CTR0(vm, vcpuid, "activated");
246022d822c6SNeel Natu 	CPU_SET_ATOMIC(vcpuid, &vm->active_cpus);
246195ebc360SNeel Natu 	return (0);
2462366f6083SPeter Grehan }
2463366f6083SPeter Grehan 
2464fc276d92SJohn Baldwin int
2465fc276d92SJohn Baldwin vm_suspend_cpu(struct vm *vm, int vcpuid)
2466fc276d92SJohn Baldwin {
2467fc276d92SJohn Baldwin 	int i;
2468fc276d92SJohn Baldwin 
2469a488c9c9SRodney W. Grimes 	if (vcpuid < -1 || vcpuid >= vm->maxcpus)
2470fc276d92SJohn Baldwin 		return (EINVAL);
2471fc276d92SJohn Baldwin 
2472fc276d92SJohn Baldwin 	if (vcpuid == -1) {
2473fc276d92SJohn Baldwin 		vm->debug_cpus = vm->active_cpus;
2474a488c9c9SRodney W. Grimes 		for (i = 0; i < vm->maxcpus; i++) {
2475fc276d92SJohn Baldwin 			if (CPU_ISSET(i, &vm->active_cpus))
2476fc276d92SJohn Baldwin 				vcpu_notify_event(vm, i, false);
2477fc276d92SJohn Baldwin 		}
2478fc276d92SJohn Baldwin 	} else {
2479fc276d92SJohn Baldwin 		if (!CPU_ISSET(vcpuid, &vm->active_cpus))
2480fc276d92SJohn Baldwin 			return (EINVAL);
2481fc276d92SJohn Baldwin 
2482fc276d92SJohn Baldwin 		CPU_SET_ATOMIC(vcpuid, &vm->debug_cpus);
2483fc276d92SJohn Baldwin 		vcpu_notify_event(vm, vcpuid, false);
2484fc276d92SJohn Baldwin 	}
2485fc276d92SJohn Baldwin 	return (0);
2486fc276d92SJohn Baldwin }
2487fc276d92SJohn Baldwin 
2488fc276d92SJohn Baldwin int
2489fc276d92SJohn Baldwin vm_resume_cpu(struct vm *vm, int vcpuid)
2490fc276d92SJohn Baldwin {
2491fc276d92SJohn Baldwin 
2492a488c9c9SRodney W. Grimes 	if (vcpuid < -1 || vcpuid >= vm->maxcpus)
2493fc276d92SJohn Baldwin 		return (EINVAL);
2494fc276d92SJohn Baldwin 
2495fc276d92SJohn Baldwin 	if (vcpuid == -1) {
2496fc276d92SJohn Baldwin 		CPU_ZERO(&vm->debug_cpus);
2497fc276d92SJohn Baldwin 	} else {
2498fc276d92SJohn Baldwin 		if (!CPU_ISSET(vcpuid, &vm->debug_cpus))
2499fc276d92SJohn Baldwin 			return (EINVAL);
2500fc276d92SJohn Baldwin 
2501fc276d92SJohn Baldwin 		CPU_CLR_ATOMIC(vcpuid, &vm->debug_cpus);
2502fc276d92SJohn Baldwin 	}
2503fc276d92SJohn Baldwin 	return (0);
2504fc276d92SJohn Baldwin }
2505fc276d92SJohn Baldwin 
2506fc276d92SJohn Baldwin int
2507fc276d92SJohn Baldwin vcpu_debugged(struct vm *vm, int vcpuid)
2508fc276d92SJohn Baldwin {
2509fc276d92SJohn Baldwin 
2510fc276d92SJohn Baldwin 	return (CPU_ISSET(vcpuid, &vm->debug_cpus));
2511fc276d92SJohn Baldwin }
2512fc276d92SJohn Baldwin 
2513a5615c90SPeter Grehan cpuset_t
2514366f6083SPeter Grehan vm_active_cpus(struct vm *vm)
2515366f6083SPeter Grehan {
2516366f6083SPeter Grehan 
2517366f6083SPeter Grehan 	return (vm->active_cpus);
2518366f6083SPeter Grehan }
2519366f6083SPeter Grehan 
252095ebc360SNeel Natu cpuset_t
2521fc276d92SJohn Baldwin vm_debug_cpus(struct vm *vm)
2522fc276d92SJohn Baldwin {
2523fc276d92SJohn Baldwin 
2524fc276d92SJohn Baldwin 	return (vm->debug_cpus);
2525fc276d92SJohn Baldwin }
2526fc276d92SJohn Baldwin 
2527fc276d92SJohn Baldwin cpuset_t
252895ebc360SNeel Natu vm_suspended_cpus(struct vm *vm)
252995ebc360SNeel Natu {
253095ebc360SNeel Natu 
253195ebc360SNeel Natu 	return (vm->suspended_cpus);
253295ebc360SNeel Natu }
253395ebc360SNeel Natu 
2534366f6083SPeter Grehan void *
25353dc3d32aSJohn Baldwin vcpu_stats(struct vcpu *vcpu)
2536366f6083SPeter Grehan {
2537366f6083SPeter Grehan 
25383dc3d32aSJohn Baldwin 	return (vcpu->stats);
2539366f6083SPeter Grehan }
2540e9027382SNeel Natu 
2541e9027382SNeel Natu int
2542e9027382SNeel Natu vm_get_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state *state)
2543e9027382SNeel Natu {
2544a488c9c9SRodney W. Grimes 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2545e9027382SNeel Natu 		return (EINVAL);
2546e9027382SNeel Natu 
2547e9027382SNeel Natu 	*state = vm->vcpu[vcpuid].x2apic_state;
2548e9027382SNeel Natu 
2549e9027382SNeel Natu 	return (0);
2550e9027382SNeel Natu }
2551e9027382SNeel Natu 
2552e9027382SNeel Natu int
2553e9027382SNeel Natu vm_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
2554e9027382SNeel Natu {
2555a488c9c9SRodney W. Grimes 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2556e9027382SNeel Natu 		return (EINVAL);
2557e9027382SNeel Natu 
25583f23d3caSNeel Natu 	if (state >= X2APIC_STATE_LAST)
2559e9027382SNeel Natu 		return (EINVAL);
2560e9027382SNeel Natu 
2561e9027382SNeel Natu 	vm->vcpu[vcpuid].x2apic_state = state;
2562e9027382SNeel Natu 
256373820fb0SNeel Natu 	vlapic_set_x2apic_state(vm, vcpuid, state);
256473820fb0SNeel Natu 
2565e9027382SNeel Natu 	return (0);
2566e9027382SNeel Natu }
256775dd3366SNeel Natu 
256822821874SNeel Natu /*
256922821874SNeel Natu  * This function is called to ensure that a vcpu "sees" a pending event
257022821874SNeel Natu  * as soon as possible:
257122821874SNeel Natu  * - If the vcpu thread is sleeping then it is woken up.
257222821874SNeel Natu  * - If the vcpu is running on a different host_cpu then an IPI will be directed
257322821874SNeel Natu  *   to the host_cpu to cause the vcpu to trap into the hypervisor.
257422821874SNeel Natu  */
2575248e6799SNeel Natu static void
2576248e6799SNeel Natu vcpu_notify_event_locked(struct vcpu *vcpu, bool lapic_intr)
257775dd3366SNeel Natu {
257875dd3366SNeel Natu 	int hostcpu;
257975dd3366SNeel Natu 
258075dd3366SNeel Natu 	hostcpu = vcpu->hostcpu;
2581ef39d7e9SNeel Natu 	if (vcpu->state == VCPU_RUNNING) {
2582ef39d7e9SNeel Natu 		KASSERT(hostcpu != NOCPU, ("vcpu running on invalid hostcpu"));
2583de5ea6b6SNeel Natu 		if (hostcpu != curcpu) {
2584ef39d7e9SNeel Natu 			if (lapic_intr) {
2585add611fdSNeel Natu 				vlapic_post_intr(vcpu->vlapic, hostcpu,
2586add611fdSNeel Natu 				    vmm_ipinum);
2587ef39d7e9SNeel Natu 			} else {
258875dd3366SNeel Natu 				ipi_cpu(hostcpu, vmm_ipinum);
258975dd3366SNeel Natu 			}
2590ef39d7e9SNeel Natu 		} else {
2591ef39d7e9SNeel Natu 			/*
2592ef39d7e9SNeel Natu 			 * If the 'vcpu' is running on 'curcpu' then it must
2593ef39d7e9SNeel Natu 			 * be sending a notification to itself (e.g. SELF_IPI).
2594ef39d7e9SNeel Natu 			 * The pending event will be picked up when the vcpu
2595ef39d7e9SNeel Natu 			 * transitions back to guest context.
2596ef39d7e9SNeel Natu 			 */
2597ef39d7e9SNeel Natu 		}
2598ef39d7e9SNeel Natu 	} else {
2599ef39d7e9SNeel Natu 		KASSERT(hostcpu == NOCPU, ("vcpu state %d not consistent "
2600ef39d7e9SNeel Natu 		    "with hostcpu %d", vcpu->state, hostcpu));
2601366f6083SPeter Grehan 		if (vcpu->state == VCPU_SLEEPING)
2602366f6083SPeter Grehan 			wakeup_one(vcpu);
2603366f6083SPeter Grehan 	}
2604248e6799SNeel Natu }
2605248e6799SNeel Natu 
2606248e6799SNeel Natu void
2607248e6799SNeel Natu vcpu_notify_event(struct vm *vm, int vcpuid, bool lapic_intr)
2608248e6799SNeel Natu {
2609248e6799SNeel Natu 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
2610248e6799SNeel Natu 
2611248e6799SNeel Natu 	vcpu_lock(vcpu);
2612248e6799SNeel Natu 	vcpu_notify_event_locked(vcpu, lapic_intr);
2613f76fc5d4SNeel Natu 	vcpu_unlock(vcpu);
2614f76fc5d4SNeel Natu }
2615318224bbSNeel Natu 
2616318224bbSNeel Natu struct vmspace *
2617318224bbSNeel Natu vm_get_vmspace(struct vm *vm)
2618318224bbSNeel Natu {
2619318224bbSNeel Natu 
2620318224bbSNeel Natu 	return (vm->vmspace);
2621318224bbSNeel Natu }
2622565bbb86SNeel Natu 
2623565bbb86SNeel Natu int
2624565bbb86SNeel Natu vm_apicid2vcpuid(struct vm *vm, int apicid)
2625565bbb86SNeel Natu {
2626565bbb86SNeel Natu 	/*
2627565bbb86SNeel Natu 	 * XXX apic id is assumed to be numerically identical to vcpu id
2628565bbb86SNeel Natu 	 */
2629565bbb86SNeel Natu 	return (apicid);
2630565bbb86SNeel Natu }
26315b8a8cd1SNeel Natu 
2632b837daddSKonstantin Belousov int
26335b8a8cd1SNeel Natu vm_smp_rendezvous(struct vm *vm, int vcpuid, cpuset_t dest,
26345b8a8cd1SNeel Natu     vm_rendezvous_func_t func, void *arg)
26355b8a8cd1SNeel Natu {
2636b837daddSKonstantin Belousov 	int error, i;
2637970955e4SNeel Natu 
26385b8a8cd1SNeel Natu 	/*
26395b8a8cd1SNeel Natu 	 * Enforce that this function is called without any locks
26405b8a8cd1SNeel Natu 	 */
26415b8a8cd1SNeel Natu 	WITNESS_WARN(WARN_PANIC, NULL, "vm_smp_rendezvous");
2642a488c9c9SRodney W. Grimes 	KASSERT(vcpuid == -1 || (vcpuid >= 0 && vcpuid < vm->maxcpus),
26435b8a8cd1SNeel Natu 	    ("vm_smp_rendezvous: invalid vcpuid %d", vcpuid));
26445b8a8cd1SNeel Natu 
26455b8a8cd1SNeel Natu restart:
26465b8a8cd1SNeel Natu 	mtx_lock(&vm->rendezvous_mtx);
26475b8a8cd1SNeel Natu 	if (vm->rendezvous_func != NULL) {
26485b8a8cd1SNeel Natu 		/*
26495b8a8cd1SNeel Natu 		 * If a rendezvous is already in progress then we need to
26505b8a8cd1SNeel Natu 		 * call the rendezvous handler in case this 'vcpuid' is one
26515b8a8cd1SNeel Natu 		 * of the targets of the rendezvous.
26525b8a8cd1SNeel Natu 		 */
26535b8a8cd1SNeel Natu 		RENDEZVOUS_CTR0(vm, vcpuid, "Rendezvous already in progress");
26545b8a8cd1SNeel Natu 		mtx_unlock(&vm->rendezvous_mtx);
2655b837daddSKonstantin Belousov 		error = vm_handle_rendezvous(vm, vcpuid);
2656b837daddSKonstantin Belousov 		if (error != 0)
2657b837daddSKonstantin Belousov 			return (error);
26585b8a8cd1SNeel Natu 		goto restart;
26595b8a8cd1SNeel Natu 	}
26605b8a8cd1SNeel Natu 	KASSERT(vm->rendezvous_func == NULL, ("vm_smp_rendezvous: previous "
26615b8a8cd1SNeel Natu 	    "rendezvous is still in progress"));
26625b8a8cd1SNeel Natu 
26635b8a8cd1SNeel Natu 	RENDEZVOUS_CTR0(vm, vcpuid, "Initiating rendezvous");
26645b8a8cd1SNeel Natu 	vm->rendezvous_req_cpus = dest;
26655b8a8cd1SNeel Natu 	CPU_ZERO(&vm->rendezvous_done_cpus);
26665b8a8cd1SNeel Natu 	vm->rendezvous_arg = arg;
2667869dbab7SAndriy Gapon 	vm->rendezvous_func = func;
26685b8a8cd1SNeel Natu 	mtx_unlock(&vm->rendezvous_mtx);
26695b8a8cd1SNeel Natu 
2670970955e4SNeel Natu 	/*
2671970955e4SNeel Natu 	 * Wake up any sleeping vcpus and trigger a VM-exit in any running
2672970955e4SNeel Natu 	 * vcpus so they handle the rendezvous as soon as possible.
2673970955e4SNeel Natu 	 */
2674a488c9c9SRodney W. Grimes 	for (i = 0; i < vm->maxcpus; i++) {
2675970955e4SNeel Natu 		if (CPU_ISSET(i, &dest))
2676970955e4SNeel Natu 			vcpu_notify_event(vm, i, false);
2677970955e4SNeel Natu 	}
2678970955e4SNeel Natu 
2679b837daddSKonstantin Belousov 	return (vm_handle_rendezvous(vm, vcpuid));
26805b8a8cd1SNeel Natu }
2681762fd208STycho Nightingale 
2682762fd208STycho Nightingale struct vatpic *
2683762fd208STycho Nightingale vm_atpic(struct vm *vm)
2684762fd208STycho Nightingale {
2685762fd208STycho Nightingale 	return (vm->vatpic);
2686762fd208STycho Nightingale }
2687e883c9bbSTycho Nightingale 
2688e883c9bbSTycho Nightingale struct vatpit *
2689e883c9bbSTycho Nightingale vm_atpit(struct vm *vm)
2690e883c9bbSTycho Nightingale {
2691e883c9bbSTycho Nightingale 	return (vm->vatpit);
2692e883c9bbSTycho Nightingale }
2693d17b5104SNeel Natu 
2694160ef77aSNeel Natu struct vpmtmr *
2695160ef77aSNeel Natu vm_pmtmr(struct vm *vm)
2696160ef77aSNeel Natu {
2697160ef77aSNeel Natu 
2698160ef77aSNeel Natu 	return (vm->vpmtmr);
2699160ef77aSNeel Natu }
2700160ef77aSNeel Natu 
27010dafa5cdSNeel Natu struct vrtc *
27020dafa5cdSNeel Natu vm_rtc(struct vm *vm)
27030dafa5cdSNeel Natu {
27040dafa5cdSNeel Natu 
27050dafa5cdSNeel Natu 	return (vm->vrtc);
27060dafa5cdSNeel Natu }
27070dafa5cdSNeel Natu 
2708d17b5104SNeel Natu enum vm_reg_name
2709d17b5104SNeel Natu vm_segment_name(int seg)
2710d17b5104SNeel Natu {
2711d17b5104SNeel Natu 	static enum vm_reg_name seg_names[] = {
2712d17b5104SNeel Natu 		VM_REG_GUEST_ES,
2713d17b5104SNeel Natu 		VM_REG_GUEST_CS,
2714d17b5104SNeel Natu 		VM_REG_GUEST_SS,
2715d17b5104SNeel Natu 		VM_REG_GUEST_DS,
2716d17b5104SNeel Natu 		VM_REG_GUEST_FS,
2717d17b5104SNeel Natu 		VM_REG_GUEST_GS
2718d17b5104SNeel Natu 	};
2719d17b5104SNeel Natu 
2720d17b5104SNeel Natu 	KASSERT(seg >= 0 && seg < nitems(seg_names),
2721d17b5104SNeel Natu 	    ("%s: invalid segment encoding %d", __func__, seg));
2722d17b5104SNeel Natu 	return (seg_names[seg]);
2723d17b5104SNeel Natu }
2724cf1d80d8SPeter Grehan 
2725d665d229SNeel Natu void
2726*2b4fe856SJohn Baldwin vm_copy_teardown(struct vm_copyinfo *copyinfo, int num_copyinfo)
2727d665d229SNeel Natu {
2728d665d229SNeel Natu 	int idx;
2729d665d229SNeel Natu 
2730d665d229SNeel Natu 	for (idx = 0; idx < num_copyinfo; idx++) {
2731d665d229SNeel Natu 		if (copyinfo[idx].cookie != NULL)
2732d665d229SNeel Natu 			vm_gpa_release(copyinfo[idx].cookie);
2733d665d229SNeel Natu 	}
2734d665d229SNeel Natu 	bzero(copyinfo, num_copyinfo * sizeof(struct vm_copyinfo));
2735d665d229SNeel Natu }
2736d665d229SNeel Natu 
2737d665d229SNeel Natu int
2738d665d229SNeel Natu vm_copy_setup(struct vm *vm, int vcpuid, struct vm_guest_paging *paging,
2739d665d229SNeel Natu     uint64_t gla, size_t len, int prot, struct vm_copyinfo *copyinfo,
27409c4d5478SNeel Natu     int num_copyinfo, int *fault)
2741d665d229SNeel Natu {
2742d665d229SNeel Natu 	int error, idx, nused;
2743d665d229SNeel Natu 	size_t n, off, remaining;
2744d665d229SNeel Natu 	void *hva, *cookie;
2745d665d229SNeel Natu 	uint64_t gpa;
2746d665d229SNeel Natu 
2747d665d229SNeel Natu 	bzero(copyinfo, sizeof(struct vm_copyinfo) * num_copyinfo);
2748d665d229SNeel Natu 
2749d665d229SNeel Natu 	nused = 0;
2750d665d229SNeel Natu 	remaining = len;
2751d665d229SNeel Natu 	while (remaining > 0) {
2752d665d229SNeel Natu 		KASSERT(nused < num_copyinfo, ("insufficient vm_copyinfo"));
27539c4d5478SNeel Natu 		error = vm_gla2gpa(vm, vcpuid, paging, gla, prot, &gpa, fault);
27549c4d5478SNeel Natu 		if (error || *fault)
2755d665d229SNeel Natu 			return (error);
2756d665d229SNeel Natu 		off = gpa & PAGE_MASK;
2757d665d229SNeel Natu 		n = min(remaining, PAGE_SIZE - off);
2758d665d229SNeel Natu 		copyinfo[nused].gpa = gpa;
2759d665d229SNeel Natu 		copyinfo[nused].len = n;
2760d665d229SNeel Natu 		remaining -= n;
2761d665d229SNeel Natu 		gla += n;
2762d665d229SNeel Natu 		nused++;
2763d665d229SNeel Natu 	}
2764d665d229SNeel Natu 
2765d665d229SNeel Natu 	for (idx = 0; idx < nused; idx++) {
27669b1aa8d6SNeel Natu 		hva = vm_gpa_hold(vm, vcpuid, copyinfo[idx].gpa,
27679b1aa8d6SNeel Natu 		    copyinfo[idx].len, prot, &cookie);
2768d665d229SNeel Natu 		if (hva == NULL)
2769d665d229SNeel Natu 			break;
2770d665d229SNeel Natu 		copyinfo[idx].hva = hva;
2771d665d229SNeel Natu 		copyinfo[idx].cookie = cookie;
2772d665d229SNeel Natu 	}
2773d665d229SNeel Natu 
2774d665d229SNeel Natu 	if (idx != nused) {
2775*2b4fe856SJohn Baldwin 		vm_copy_teardown(copyinfo, num_copyinfo);
27769c4d5478SNeel Natu 		return (EFAULT);
2777d665d229SNeel Natu 	} else {
27789c4d5478SNeel Natu 		*fault = 0;
2779d665d229SNeel Natu 		return (0);
2780d665d229SNeel Natu 	}
2781d665d229SNeel Natu }
2782d665d229SNeel Natu 
2783d665d229SNeel Natu void
2784*2b4fe856SJohn Baldwin vm_copyin(struct vm_copyinfo *copyinfo, void *kaddr, size_t len)
2785d665d229SNeel Natu {
2786d665d229SNeel Natu 	char *dst;
2787d665d229SNeel Natu 	int idx;
2788d665d229SNeel Natu 
2789d665d229SNeel Natu 	dst = kaddr;
2790d665d229SNeel Natu 	idx = 0;
2791d665d229SNeel Natu 	while (len > 0) {
2792d665d229SNeel Natu 		bcopy(copyinfo[idx].hva, dst, copyinfo[idx].len);
2793d665d229SNeel Natu 		len -= copyinfo[idx].len;
2794d665d229SNeel Natu 		dst += copyinfo[idx].len;
2795d665d229SNeel Natu 		idx++;
2796d665d229SNeel Natu 	}
2797d665d229SNeel Natu }
2798d665d229SNeel Natu 
2799d665d229SNeel Natu void
2800*2b4fe856SJohn Baldwin vm_copyout(const void *kaddr, struct vm_copyinfo *copyinfo, size_t len)
2801d665d229SNeel Natu {
2802d665d229SNeel Natu 	const char *src;
2803d665d229SNeel Natu 	int idx;
2804d665d229SNeel Natu 
2805d665d229SNeel Natu 	src = kaddr;
2806d665d229SNeel Natu 	idx = 0;
2807d665d229SNeel Natu 	while (len > 0) {
2808d665d229SNeel Natu 		bcopy(src, copyinfo[idx].hva, copyinfo[idx].len);
2809d665d229SNeel Natu 		len -= copyinfo[idx].len;
2810d665d229SNeel Natu 		src += copyinfo[idx].len;
2811d665d229SNeel Natu 		idx++;
2812d665d229SNeel Natu 	}
2813d665d229SNeel Natu }
2814cf1d80d8SPeter Grehan 
2815cf1d80d8SPeter Grehan /*
2816cf1d80d8SPeter Grehan  * Return the amount of in-use and wired memory for the VM. Since
2817cf1d80d8SPeter Grehan  * these are global stats, only return the values with for vCPU 0
2818cf1d80d8SPeter Grehan  */
2819cf1d80d8SPeter Grehan VMM_STAT_DECLARE(VMM_MEM_RESIDENT);
2820cf1d80d8SPeter Grehan VMM_STAT_DECLARE(VMM_MEM_WIRED);
2821cf1d80d8SPeter Grehan 
2822cf1d80d8SPeter Grehan static void
2823cf1d80d8SPeter Grehan vm_get_rescnt(struct vm *vm, int vcpu, struct vmm_stat_type *stat)
2824cf1d80d8SPeter Grehan {
2825cf1d80d8SPeter Grehan 
2826cf1d80d8SPeter Grehan 	if (vcpu == 0) {
28273dc3d32aSJohn Baldwin 		vmm_stat_set(vm_vcpu(vm, vcpu), VMM_MEM_RESIDENT,
2828cf1d80d8SPeter Grehan 	       	    PAGE_SIZE * vmspace_resident_count(vm->vmspace));
2829cf1d80d8SPeter Grehan 	}
2830cf1d80d8SPeter Grehan }
2831cf1d80d8SPeter Grehan 
2832cf1d80d8SPeter Grehan static void
2833cf1d80d8SPeter Grehan vm_get_wiredcnt(struct vm *vm, int vcpu, struct vmm_stat_type *stat)
2834cf1d80d8SPeter Grehan {
2835cf1d80d8SPeter Grehan 
2836cf1d80d8SPeter Grehan 	if (vcpu == 0) {
28373dc3d32aSJohn Baldwin 		vmm_stat_set(vm_vcpu(vm, vcpu), VMM_MEM_WIRED,
2838cf1d80d8SPeter Grehan 	      	    PAGE_SIZE * pmap_wired_count(vmspace_pmap(vm->vmspace)));
2839cf1d80d8SPeter Grehan 	}
2840cf1d80d8SPeter Grehan }
2841cf1d80d8SPeter Grehan 
2842cf1d80d8SPeter Grehan VMM_STAT_FUNC(VMM_MEM_RESIDENT, "Resident memory", vm_get_rescnt);
2843cf1d80d8SPeter Grehan VMM_STAT_FUNC(VMM_MEM_WIRED, "Wired memory", vm_get_wiredcnt);
2844483d953aSJohn Baldwin 
2845483d953aSJohn Baldwin #ifdef BHYVE_SNAPSHOT
2846483d953aSJohn Baldwin static int
2847483d953aSJohn Baldwin vm_snapshot_vcpus(struct vm *vm, struct vm_snapshot_meta *meta)
2848483d953aSJohn Baldwin {
2849a7db532eSJohn Baldwin 	uint64_t tsc, now;
2850483d953aSJohn Baldwin 	int ret;
2851483d953aSJohn Baldwin 	struct vcpu *vcpu;
285235abc6c2SJohn Baldwin 	uint16_t i, maxcpus;
2853483d953aSJohn Baldwin 
2854a7db532eSJohn Baldwin 	now = rdtsc();
285535abc6c2SJohn Baldwin 	maxcpus = vm_get_maxcpus(vm);
285635abc6c2SJohn Baldwin 	for (i = 0; i < maxcpus; i++) {
2857483d953aSJohn Baldwin 		vcpu = &vm->vcpu[i];
2858483d953aSJohn Baldwin 
2859483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vcpu->x2apic_state, meta, ret, done);
2860483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vcpu->exitintinfo, meta, ret, done);
2861483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vcpu->exc_vector, meta, ret, done);
2862483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vcpu->exc_errcode_valid, meta, ret, done);
2863483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vcpu->exc_errcode, meta, ret, done);
2864483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vcpu->guest_xcr0, meta, ret, done);
2865483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vcpu->exitinfo, meta, ret, done);
2866483d953aSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(vcpu->nextrip, meta, ret, done);
2867a7db532eSJohn Baldwin 
2868a7db532eSJohn Baldwin 		/*
2869a7db532eSJohn Baldwin 		 * Save the absolute TSC value by adding now to tsc_offset.
2870483d953aSJohn Baldwin 		 *
2871483d953aSJohn Baldwin 		 * It will be turned turned back into an actual offset when the
2872483d953aSJohn Baldwin 		 * TSC restore function is called
2873483d953aSJohn Baldwin 		 */
2874a7db532eSJohn Baldwin 		tsc = now + vcpu->tsc_offset;
2875a7db532eSJohn Baldwin 		SNAPSHOT_VAR_OR_LEAVE(tsc, meta, ret, done);
2876483d953aSJohn Baldwin 	}
2877483d953aSJohn Baldwin 
2878483d953aSJohn Baldwin done:
2879483d953aSJohn Baldwin 	return (ret);
2880483d953aSJohn Baldwin }
2881483d953aSJohn Baldwin 
2882483d953aSJohn Baldwin static int
2883483d953aSJohn Baldwin vm_snapshot_vm(struct vm *vm, struct vm_snapshot_meta *meta)
2884483d953aSJohn Baldwin {
2885483d953aSJohn Baldwin 	int ret;
2886483d953aSJohn Baldwin 
2887483d953aSJohn Baldwin 	ret = vm_snapshot_vcpus(vm, meta);
2888a7db532eSJohn Baldwin 	if (ret != 0)
2889483d953aSJohn Baldwin 		goto done;
2890483d953aSJohn Baldwin 
2891483d953aSJohn Baldwin done:
2892483d953aSJohn Baldwin 	return (ret);
2893483d953aSJohn Baldwin }
2894483d953aSJohn Baldwin 
2895483d953aSJohn Baldwin static int
28961aa51504SJohn Baldwin vm_snapshot_vcpu(struct vm *vm, struct vm_snapshot_meta *meta)
2897483d953aSJohn Baldwin {
289835abc6c2SJohn Baldwin 	int error;
28991aa51504SJohn Baldwin 	struct vcpu *vcpu;
290035abc6c2SJohn Baldwin 	uint16_t i, maxcpus;
2901483d953aSJohn Baldwin 
2902483d953aSJohn Baldwin 	error = 0;
2903483d953aSJohn Baldwin 
290435abc6c2SJohn Baldwin 	maxcpus = vm_get_maxcpus(vm);
290535abc6c2SJohn Baldwin 	for (i = 0; i < maxcpus; i++) {
29061aa51504SJohn Baldwin 		vcpu = &vm->vcpu[i];
29071aa51504SJohn Baldwin 
2908869c8d19SJohn Baldwin 		error = vmmops_vcpu_snapshot(vcpu->cookie, meta);
2909483d953aSJohn Baldwin 		if (error != 0) {
2910483d953aSJohn Baldwin 			printf("%s: failed to snapshot vmcs/vmcb data for "
2911483d953aSJohn Baldwin 			       "vCPU: %d; error: %d\n", __func__, i, error);
2912483d953aSJohn Baldwin 			goto done;
2913483d953aSJohn Baldwin 		}
2914483d953aSJohn Baldwin 	}
2915483d953aSJohn Baldwin 
2916483d953aSJohn Baldwin done:
2917483d953aSJohn Baldwin 	return (error);
2918483d953aSJohn Baldwin }
2919483d953aSJohn Baldwin 
2920483d953aSJohn Baldwin /*
2921483d953aSJohn Baldwin  * Save kernel-side structures to user-space for snapshotting.
2922483d953aSJohn Baldwin  */
2923483d953aSJohn Baldwin int
2924483d953aSJohn Baldwin vm_snapshot_req(struct vm *vm, struct vm_snapshot_meta *meta)
2925483d953aSJohn Baldwin {
2926483d953aSJohn Baldwin 	int ret = 0;
2927483d953aSJohn Baldwin 
2928483d953aSJohn Baldwin 	switch (meta->dev_req) {
2929483d953aSJohn Baldwin 	case STRUCT_VMX:
293015add60dSPeter Grehan 		ret = vmmops_snapshot(vm->cookie, meta);
2931483d953aSJohn Baldwin 		break;
2932483d953aSJohn Baldwin 	case STRUCT_VMCX:
29331aa51504SJohn Baldwin 		ret = vm_snapshot_vcpu(vm, meta);
2934483d953aSJohn Baldwin 		break;
2935483d953aSJohn Baldwin 	case STRUCT_VM:
2936483d953aSJohn Baldwin 		ret = vm_snapshot_vm(vm, meta);
2937483d953aSJohn Baldwin 		break;
2938483d953aSJohn Baldwin 	case STRUCT_VIOAPIC:
2939483d953aSJohn Baldwin 		ret = vioapic_snapshot(vm_ioapic(vm), meta);
2940483d953aSJohn Baldwin 		break;
2941483d953aSJohn Baldwin 	case STRUCT_VLAPIC:
2942483d953aSJohn Baldwin 		ret = vlapic_snapshot(vm, meta);
2943483d953aSJohn Baldwin 		break;
2944483d953aSJohn Baldwin 	case STRUCT_VHPET:
2945483d953aSJohn Baldwin 		ret = vhpet_snapshot(vm_hpet(vm), meta);
2946483d953aSJohn Baldwin 		break;
2947483d953aSJohn Baldwin 	case STRUCT_VATPIC:
2948483d953aSJohn Baldwin 		ret = vatpic_snapshot(vm_atpic(vm), meta);
2949483d953aSJohn Baldwin 		break;
2950483d953aSJohn Baldwin 	case STRUCT_VATPIT:
2951483d953aSJohn Baldwin 		ret = vatpit_snapshot(vm_atpit(vm), meta);
2952483d953aSJohn Baldwin 		break;
2953483d953aSJohn Baldwin 	case STRUCT_VPMTMR:
2954483d953aSJohn Baldwin 		ret = vpmtmr_snapshot(vm_pmtmr(vm), meta);
2955483d953aSJohn Baldwin 		break;
2956483d953aSJohn Baldwin 	case STRUCT_VRTC:
2957483d953aSJohn Baldwin 		ret = vrtc_snapshot(vm_rtc(vm), meta);
2958483d953aSJohn Baldwin 		break;
2959483d953aSJohn Baldwin 	default:
2960483d953aSJohn Baldwin 		printf("%s: failed to find the requested type %#x\n",
2961483d953aSJohn Baldwin 		       __func__, meta->dev_req);
2962483d953aSJohn Baldwin 		ret = (EINVAL);
2963483d953aSJohn Baldwin 	}
2964483d953aSJohn Baldwin 	return (ret);
2965483d953aSJohn Baldwin }
2966483d953aSJohn Baldwin 
2967483d953aSJohn Baldwin int
2968483d953aSJohn Baldwin vm_set_tsc_offset(struct vm *vm, int vcpuid, uint64_t offset)
2969483d953aSJohn Baldwin {
2970483d953aSJohn Baldwin 	struct vcpu *vcpu;
2971483d953aSJohn Baldwin 
297235abc6c2SJohn Baldwin 	if (vcpuid < 0 || vcpuid >= vm_get_maxcpus(vm))
2973483d953aSJohn Baldwin 		return (EINVAL);
2974483d953aSJohn Baldwin 
2975483d953aSJohn Baldwin 	vcpu = &vm->vcpu[vcpuid];
2976483d953aSJohn Baldwin 	vcpu->tsc_offset = offset;
2977483d953aSJohn Baldwin 
2978483d953aSJohn Baldwin 	return (0);
2979483d953aSJohn Baldwin }
2980483d953aSJohn Baldwin 
2981483d953aSJohn Baldwin int
2982483d953aSJohn Baldwin vm_restore_time(struct vm *vm)
2983483d953aSJohn Baldwin {
298435abc6c2SJohn Baldwin 	int error;
2985483d953aSJohn Baldwin 	uint64_t now;
2986483d953aSJohn Baldwin 	struct vcpu *vcpu;
298735abc6c2SJohn Baldwin 	uint16_t i, maxcpus;
2988483d953aSJohn Baldwin 
2989483d953aSJohn Baldwin 	now = rdtsc();
2990483d953aSJohn Baldwin 
2991483d953aSJohn Baldwin 	error = vhpet_restore_time(vm_hpet(vm));
2992483d953aSJohn Baldwin 	if (error)
2993483d953aSJohn Baldwin 		return (error);
2994483d953aSJohn Baldwin 
299535abc6c2SJohn Baldwin 	maxcpus = vm_get_maxcpus(vm);
299635abc6c2SJohn Baldwin 	for (i = 0; i < maxcpus; i++) {
2997483d953aSJohn Baldwin 		vcpu = &vm->vcpu[i];
2998483d953aSJohn Baldwin 
2999869c8d19SJohn Baldwin 		error = vmmops_restore_tsc(vcpu->cookie,
30001aa51504SJohn Baldwin 		    vcpu->tsc_offset - now);
3001483d953aSJohn Baldwin 		if (error)
3002483d953aSJohn Baldwin 			return (error);
3003483d953aSJohn Baldwin 	}
3004483d953aSJohn Baldwin 
3005483d953aSJohn Baldwin 	return (0);
3006483d953aSJohn Baldwin }
3007483d953aSJohn Baldwin #endif
3008