xref: /freebsd/sys/amd64/vmm/vmm.c (revision 924226fba12cc9a228c73b956e1b7fa24c60b055)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2011 NetApp, Inc.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  *
28  * $FreeBSD$
29  */
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include "opt_bhyve_snapshot.h"
35 
36 #include <sys/param.h>
37 #include <sys/systm.h>
38 #include <sys/kernel.h>
39 #include <sys/module.h>
40 #include <sys/sysctl.h>
41 #include <sys/malloc.h>
42 #include <sys/pcpu.h>
43 #include <sys/lock.h>
44 #include <sys/mutex.h>
45 #include <sys/proc.h>
46 #include <sys/rwlock.h>
47 #include <sys/sched.h>
48 #include <sys/smp.h>
49 #include <sys/vnode.h>
50 
51 #include <vm/vm.h>
52 #include <vm/vm_param.h>
53 #include <vm/vm_extern.h>
54 #include <vm/vm_object.h>
55 #include <vm/vm_page.h>
56 #include <vm/pmap.h>
57 #include <vm/vm_map.h>
58 #include <vm/vm_pager.h>
59 #include <vm/vm_kern.h>
60 #include <vm/vnode_pager.h>
61 #include <vm/swap_pager.h>
62 #include <vm/uma.h>
63 
64 #include <machine/cpu.h>
65 #include <machine/pcb.h>
66 #include <machine/smp.h>
67 #include <machine/md_var.h>
68 #include <x86/psl.h>
69 #include <x86/apicreg.h>
70 #include <x86/ifunc.h>
71 
72 #include <machine/vmm.h>
73 #include <machine/vmm_dev.h>
74 #include <machine/vmm_instruction_emul.h>
75 #include <machine/vmm_snapshot.h>
76 
77 #include "vmm_ioport.h"
78 #include "vmm_ktr.h"
79 #include "vmm_host.h"
80 #include "vmm_mem.h"
81 #include "vmm_util.h"
82 #include "vatpic.h"
83 #include "vatpit.h"
84 #include "vhpet.h"
85 #include "vioapic.h"
86 #include "vlapic.h"
87 #include "vpmtmr.h"
88 #include "vrtc.h"
89 #include "vmm_stat.h"
90 #include "vmm_lapic.h"
91 
92 #include "io/ppt.h"
93 #include "io/iommu.h"
94 
95 struct vlapic;
96 
97 /*
98  * Initialization:
99  * (a) allocated when vcpu is created
100  * (i) initialized when vcpu is created and when it is reinitialized
101  * (o) initialized the first time the vcpu is created
102  * (x) initialized before use
103  */
104 struct vcpu {
105 	struct mtx 	mtx;		/* (o) protects 'state' and 'hostcpu' */
106 	enum vcpu_state	state;		/* (o) vcpu state */
107 	int		hostcpu;	/* (o) vcpu's host cpu */
108 	int		reqidle;	/* (i) request vcpu to idle */
109 	struct vlapic	*vlapic;	/* (i) APIC device model */
110 	enum x2apic_state x2apic_state;	/* (i) APIC mode */
111 	uint64_t	exitintinfo;	/* (i) events pending at VM exit */
112 	int		nmi_pending;	/* (i) NMI pending */
113 	int		extint_pending;	/* (i) INTR pending */
114 	int	exception_pending;	/* (i) exception pending */
115 	int	exc_vector;		/* (x) exception collateral */
116 	int	exc_errcode_valid;
117 	uint32_t exc_errcode;
118 	struct savefpu	*guestfpu;	/* (a,i) guest fpu state */
119 	uint64_t	guest_xcr0;	/* (i) guest %xcr0 register */
120 	void		*stats;		/* (a,i) statistics */
121 	struct vm_exit	exitinfo;	/* (x) exit reason and collateral */
122 	uint64_t	nextrip;	/* (x) next instruction to execute */
123 	uint64_t	tsc_offset;	/* (o) TSC offsetting */
124 };
125 
126 #define	vcpu_lock_initialized(v) mtx_initialized(&((v)->mtx))
127 #define	vcpu_lock_init(v)	mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN)
128 #define	vcpu_lock(v)		mtx_lock_spin(&((v)->mtx))
129 #define	vcpu_unlock(v)		mtx_unlock_spin(&((v)->mtx))
130 #define	vcpu_assert_locked(v)	mtx_assert(&((v)->mtx), MA_OWNED)
131 
132 struct mem_seg {
133 	size_t	len;
134 	bool	sysmem;
135 	struct vm_object *object;
136 };
137 #define	VM_MAX_MEMSEGS	4
138 
139 struct mem_map {
140 	vm_paddr_t	gpa;
141 	size_t		len;
142 	vm_ooffset_t	segoff;
143 	int		segid;
144 	int		prot;
145 	int		flags;
146 };
147 #define	VM_MAX_MEMMAPS	8
148 
149 /*
150  * Initialization:
151  * (o) initialized the first time the VM is created
152  * (i) initialized when VM is created and when it is reinitialized
153  * (x) initialized before use
154  */
155 struct vm {
156 	void		*cookie;		/* (i) cpu-specific data */
157 	void		*iommu;			/* (x) iommu-specific data */
158 	struct vhpet	*vhpet;			/* (i) virtual HPET */
159 	struct vioapic	*vioapic;		/* (i) virtual ioapic */
160 	struct vatpic	*vatpic;		/* (i) virtual atpic */
161 	struct vatpit	*vatpit;		/* (i) virtual atpit */
162 	struct vpmtmr	*vpmtmr;		/* (i) virtual ACPI PM timer */
163 	struct vrtc	*vrtc;			/* (o) virtual RTC */
164 	volatile cpuset_t active_cpus;		/* (i) active vcpus */
165 	volatile cpuset_t debug_cpus;		/* (i) vcpus stopped for debug */
166 	int		suspend;		/* (i) stop VM execution */
167 	volatile cpuset_t suspended_cpus; 	/* (i) suspended vcpus */
168 	volatile cpuset_t halted_cpus;		/* (x) cpus in a hard halt */
169 	cpuset_t	rendezvous_req_cpus;	/* (x) rendezvous requested */
170 	cpuset_t	rendezvous_done_cpus;	/* (x) rendezvous finished */
171 	void		*rendezvous_arg;	/* (x) rendezvous func/arg */
172 	vm_rendezvous_func_t rendezvous_func;
173 	struct mtx	rendezvous_mtx;		/* (o) rendezvous lock */
174 	struct mem_map	mem_maps[VM_MAX_MEMMAPS]; /* (i) guest address space */
175 	struct mem_seg	mem_segs[VM_MAX_MEMSEGS]; /* (o) guest memory regions */
176 	struct vmspace	*vmspace;		/* (o) guest's address space */
177 	char		name[VM_MAX_NAMELEN+1];	/* (o) virtual machine name */
178 	struct vcpu	vcpu[VM_MAXCPU];	/* (i) guest vcpus */
179 	/* The following describe the vm cpu topology */
180 	uint16_t	sockets;		/* (o) num of sockets */
181 	uint16_t	cores;			/* (o) num of cores/socket */
182 	uint16_t	threads;		/* (o) num of threads/core */
183 	uint16_t	maxcpus;		/* (o) max pluggable cpus */
184 };
185 
186 static int vmm_initialized;
187 
188 static void	vmmops_panic(void);
189 
190 static void
191 vmmops_panic(void)
192 {
193 	panic("vmm_ops func called when !vmm_is_intel() && !vmm_is_svm()");
194 }
195 
196 #define	DEFINE_VMMOPS_IFUNC(ret_type, opname, args)			\
197     DEFINE_IFUNC(static, ret_type, vmmops_##opname, args)		\
198     {									\
199     	if (vmm_is_intel())						\
200     		return (vmm_ops_intel.opname);				\
201     	else if (vmm_is_svm())						\
202     		return (vmm_ops_amd.opname);				\
203     	else								\
204     		return ((ret_type (*)args)vmmops_panic);		\
205     }
206 
207 DEFINE_VMMOPS_IFUNC(int, modinit, (int ipinum))
208 DEFINE_VMMOPS_IFUNC(int, modcleanup, (void))
209 DEFINE_VMMOPS_IFUNC(void, modresume, (void))
210 DEFINE_VMMOPS_IFUNC(void *, init, (struct vm *vm, struct pmap *pmap))
211 DEFINE_VMMOPS_IFUNC(int, run, (void *vmi, int vcpu, register_t rip,
212     struct pmap *pmap, struct vm_eventinfo *info))
213 DEFINE_VMMOPS_IFUNC(void, cleanup, (void *vmi))
214 DEFINE_VMMOPS_IFUNC(int, getreg, (void *vmi, int vcpu, int num,
215     uint64_t *retval))
216 DEFINE_VMMOPS_IFUNC(int, setreg, (void *vmi, int vcpu, int num,
217     uint64_t val))
218 DEFINE_VMMOPS_IFUNC(int, getdesc, (void *vmi, int vcpu, int num,
219     struct seg_desc *desc))
220 DEFINE_VMMOPS_IFUNC(int, setdesc, (void *vmi, int vcpu, int num,
221     struct seg_desc *desc))
222 DEFINE_VMMOPS_IFUNC(int, getcap, (void *vmi, int vcpu, int num, int *retval))
223 DEFINE_VMMOPS_IFUNC(int, setcap, (void *vmi, int vcpu, int num, int val))
224 DEFINE_VMMOPS_IFUNC(struct vmspace *, vmspace_alloc, (vm_offset_t min,
225     vm_offset_t max))
226 DEFINE_VMMOPS_IFUNC(void, vmspace_free, (struct vmspace *vmspace))
227 DEFINE_VMMOPS_IFUNC(struct vlapic *, vlapic_init, (void *vmi, int vcpu))
228 DEFINE_VMMOPS_IFUNC(void, vlapic_cleanup, (void *vmi, struct vlapic *vlapic))
229 #ifdef BHYVE_SNAPSHOT
230 DEFINE_VMMOPS_IFUNC(int, snapshot, (void *vmi, struct vm_snapshot_meta
231     *meta))
232 DEFINE_VMMOPS_IFUNC(int, vmcx_snapshot, (void *vmi, struct vm_snapshot_meta
233     *meta, int vcpu))
234 DEFINE_VMMOPS_IFUNC(int, restore_tsc, (void *vmi, int vcpuid, uint64_t now))
235 #endif
236 
237 #define	fpu_start_emulating()	load_cr0(rcr0() | CR0_TS)
238 #define	fpu_stop_emulating()	clts()
239 
240 SDT_PROVIDER_DEFINE(vmm);
241 
242 static MALLOC_DEFINE(M_VM, "vm", "vm");
243 
244 /* statistics */
245 static VMM_STAT(VCPU_TOTAL_RUNTIME, "vcpu total runtime");
246 
247 SYSCTL_NODE(_hw, OID_AUTO, vmm, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
248     NULL);
249 
250 /*
251  * Halt the guest if all vcpus are executing a HLT instruction with
252  * interrupts disabled.
253  */
254 static int halt_detection_enabled = 1;
255 SYSCTL_INT(_hw_vmm, OID_AUTO, halt_detection, CTLFLAG_RDTUN,
256     &halt_detection_enabled, 0,
257     "Halt VM if all vcpus execute HLT with interrupts disabled");
258 
259 static int vmm_ipinum;
260 SYSCTL_INT(_hw_vmm, OID_AUTO, ipinum, CTLFLAG_RD, &vmm_ipinum, 0,
261     "IPI vector used for vcpu notifications");
262 
263 static int trace_guest_exceptions;
264 SYSCTL_INT(_hw_vmm, OID_AUTO, trace_guest_exceptions, CTLFLAG_RDTUN,
265     &trace_guest_exceptions, 0,
266     "Trap into hypervisor on all guest exceptions and reflect them back");
267 
268 static int trap_wbinvd;
269 SYSCTL_INT(_hw_vmm, OID_AUTO, trap_wbinvd, CTLFLAG_RDTUN, &trap_wbinvd, 0,
270     "WBINVD triggers a VM-exit");
271 
272 static void vm_free_memmap(struct vm *vm, int ident);
273 static bool sysmem_mapping(struct vm *vm, struct mem_map *mm);
274 static void vcpu_notify_event_locked(struct vcpu *vcpu, bool lapic_intr);
275 
276 #ifdef KTR
277 static const char *
278 vcpu_state2str(enum vcpu_state state)
279 {
280 
281 	switch (state) {
282 	case VCPU_IDLE:
283 		return ("idle");
284 	case VCPU_FROZEN:
285 		return ("frozen");
286 	case VCPU_RUNNING:
287 		return ("running");
288 	case VCPU_SLEEPING:
289 		return ("sleeping");
290 	default:
291 		return ("unknown");
292 	}
293 }
294 #endif
295 
296 static void
297 vcpu_cleanup(struct vm *vm, int i, bool destroy)
298 {
299 	struct vcpu *vcpu = &vm->vcpu[i];
300 
301 	vmmops_vlapic_cleanup(vm->cookie, vcpu->vlapic);
302 	if (destroy) {
303 		vmm_stat_free(vcpu->stats);
304 		fpu_save_area_free(vcpu->guestfpu);
305 	}
306 }
307 
308 static void
309 vcpu_init(struct vm *vm, int vcpu_id, bool create)
310 {
311 	struct vcpu *vcpu;
312 
313 	KASSERT(vcpu_id >= 0 && vcpu_id < vm->maxcpus,
314 	    ("vcpu_init: invalid vcpu %d", vcpu_id));
315 
316 	vcpu = &vm->vcpu[vcpu_id];
317 
318 	if (create) {
319 		KASSERT(!vcpu_lock_initialized(vcpu), ("vcpu %d already "
320 		    "initialized", vcpu_id));
321 		vcpu_lock_init(vcpu);
322 		vcpu->state = VCPU_IDLE;
323 		vcpu->hostcpu = NOCPU;
324 		vcpu->guestfpu = fpu_save_area_alloc();
325 		vcpu->stats = vmm_stat_alloc();
326 		vcpu->tsc_offset = 0;
327 	}
328 
329 	vcpu->vlapic = vmmops_vlapic_init(vm->cookie, vcpu_id);
330 	vm_set_x2apic_state(vm, vcpu_id, X2APIC_DISABLED);
331 	vcpu->reqidle = 0;
332 	vcpu->exitintinfo = 0;
333 	vcpu->nmi_pending = 0;
334 	vcpu->extint_pending = 0;
335 	vcpu->exception_pending = 0;
336 	vcpu->guest_xcr0 = XFEATURE_ENABLED_X87;
337 	fpu_save_area_reset(vcpu->guestfpu);
338 	vmm_stat_init(vcpu->stats);
339 }
340 
341 int
342 vcpu_trace_exceptions(struct vm *vm, int vcpuid)
343 {
344 
345 	return (trace_guest_exceptions);
346 }
347 
348 int
349 vcpu_trap_wbinvd(struct vm *vm, int vcpuid)
350 {
351 	return (trap_wbinvd);
352 }
353 
354 struct vm_exit *
355 vm_exitinfo(struct vm *vm, int cpuid)
356 {
357 	struct vcpu *vcpu;
358 
359 	if (cpuid < 0 || cpuid >= vm->maxcpus)
360 		panic("vm_exitinfo: invalid cpuid %d", cpuid);
361 
362 	vcpu = &vm->vcpu[cpuid];
363 
364 	return (&vcpu->exitinfo);
365 }
366 
367 static int
368 vmm_init(void)
369 {
370 	int error;
371 
372 	if (!vmm_is_hw_supported())
373 		return (ENXIO);
374 
375 	vmm_host_state_init();
376 
377 	vmm_ipinum = lapic_ipi_alloc(pti ? &IDTVEC(justreturn1_pti) :
378 	    &IDTVEC(justreturn));
379 	if (vmm_ipinum < 0)
380 		vmm_ipinum = IPI_AST;
381 
382 	error = vmm_mem_init();
383 	if (error)
384 		return (error);
385 
386 	vmm_resume_p = vmmops_modresume;
387 
388 	return (vmmops_modinit(vmm_ipinum));
389 }
390 
391 static int
392 vmm_handler(module_t mod, int what, void *arg)
393 {
394 	int error;
395 
396 	switch (what) {
397 	case MOD_LOAD:
398 		if (vmm_is_hw_supported()) {
399 			vmmdev_init();
400 			error = vmm_init();
401 			if (error == 0)
402 				vmm_initialized = 1;
403 		} else {
404 			error = ENXIO;
405 		}
406 		break;
407 	case MOD_UNLOAD:
408 		if (vmm_is_hw_supported()) {
409 			error = vmmdev_cleanup();
410 			if (error == 0) {
411 				vmm_resume_p = NULL;
412 				iommu_cleanup();
413 				if (vmm_ipinum != IPI_AST)
414 					lapic_ipi_free(vmm_ipinum);
415 				error = vmmops_modcleanup();
416 				/*
417 				 * Something bad happened - prevent new
418 				 * VMs from being created
419 				 */
420 				if (error)
421 					vmm_initialized = 0;
422 			}
423 		} else {
424 			error = 0;
425 		}
426 		break;
427 	default:
428 		error = 0;
429 		break;
430 	}
431 	return (error);
432 }
433 
434 static moduledata_t vmm_kmod = {
435 	"vmm",
436 	vmm_handler,
437 	NULL
438 };
439 
440 /*
441  * vmm initialization has the following dependencies:
442  *
443  * - VT-x initialization requires smp_rendezvous() and therefore must happen
444  *   after SMP is fully functional (after SI_SUB_SMP).
445  */
446 DECLARE_MODULE(vmm, vmm_kmod, SI_SUB_SMP + 1, SI_ORDER_ANY);
447 MODULE_VERSION(vmm, 1);
448 
449 static void
450 vm_init(struct vm *vm, bool create)
451 {
452 	int i;
453 
454 	vm->cookie = vmmops_init(vm, vmspace_pmap(vm->vmspace));
455 	vm->iommu = NULL;
456 	vm->vioapic = vioapic_init(vm);
457 	vm->vhpet = vhpet_init(vm);
458 	vm->vatpic = vatpic_init(vm);
459 	vm->vatpit = vatpit_init(vm);
460 	vm->vpmtmr = vpmtmr_init(vm);
461 	if (create)
462 		vm->vrtc = vrtc_init(vm);
463 
464 	CPU_ZERO(&vm->active_cpus);
465 	CPU_ZERO(&vm->debug_cpus);
466 
467 	vm->suspend = 0;
468 	CPU_ZERO(&vm->suspended_cpus);
469 
470 	for (i = 0; i < vm->maxcpus; i++)
471 		vcpu_init(vm, i, create);
472 }
473 
474 /*
475  * The default CPU topology is a single thread per package.
476  */
477 u_int cores_per_package = 1;
478 u_int threads_per_core = 1;
479 
480 int
481 vm_create(const char *name, struct vm **retvm)
482 {
483 	struct vm *vm;
484 	struct vmspace *vmspace;
485 
486 	/*
487 	 * If vmm.ko could not be successfully initialized then don't attempt
488 	 * to create the virtual machine.
489 	 */
490 	if (!vmm_initialized)
491 		return (ENXIO);
492 
493 	if (name == NULL || strnlen(name, VM_MAX_NAMELEN + 1) ==
494 	    VM_MAX_NAMELEN + 1)
495 		return (EINVAL);
496 
497 	vmspace = vmmops_vmspace_alloc(0, VM_MAXUSER_ADDRESS_LA48);
498 	if (vmspace == NULL)
499 		return (ENOMEM);
500 
501 	vm = malloc(sizeof(struct vm), M_VM, M_WAITOK | M_ZERO);
502 	strcpy(vm->name, name);
503 	vm->vmspace = vmspace;
504 	mtx_init(&vm->rendezvous_mtx, "vm rendezvous lock", 0, MTX_DEF);
505 
506 	vm->sockets = 1;
507 	vm->cores = cores_per_package;	/* XXX backwards compatibility */
508 	vm->threads = threads_per_core;	/* XXX backwards compatibility */
509 	vm->maxcpus = VM_MAXCPU;	/* XXX temp to keep code working */
510 
511 	vm_init(vm, true);
512 
513 	*retvm = vm;
514 	return (0);
515 }
516 
517 void
518 vm_get_topology(struct vm *vm, uint16_t *sockets, uint16_t *cores,
519     uint16_t *threads, uint16_t *maxcpus)
520 {
521 	*sockets = vm->sockets;
522 	*cores = vm->cores;
523 	*threads = vm->threads;
524 	*maxcpus = vm->maxcpus;
525 }
526 
527 uint16_t
528 vm_get_maxcpus(struct vm *vm)
529 {
530 	return (vm->maxcpus);
531 }
532 
533 int
534 vm_set_topology(struct vm *vm, uint16_t sockets, uint16_t cores,
535     uint16_t threads, uint16_t maxcpus)
536 {
537 	if (maxcpus != 0)
538 		return (EINVAL);	/* XXX remove when supported */
539 	if ((sockets * cores * threads) > vm->maxcpus)
540 		return (EINVAL);
541 	/* XXX need to check sockets * cores * threads == vCPU, how? */
542 	vm->sockets = sockets;
543 	vm->cores = cores;
544 	vm->threads = threads;
545 	vm->maxcpus = VM_MAXCPU;	/* XXX temp to keep code working */
546 	return(0);
547 }
548 
549 static void
550 vm_cleanup(struct vm *vm, bool destroy)
551 {
552 	struct mem_map *mm;
553 	int i;
554 
555 	ppt_unassign_all(vm);
556 
557 	if (vm->iommu != NULL)
558 		iommu_destroy_domain(vm->iommu);
559 
560 	if (destroy)
561 		vrtc_cleanup(vm->vrtc);
562 	else
563 		vrtc_reset(vm->vrtc);
564 	vpmtmr_cleanup(vm->vpmtmr);
565 	vatpit_cleanup(vm->vatpit);
566 	vhpet_cleanup(vm->vhpet);
567 	vatpic_cleanup(vm->vatpic);
568 	vioapic_cleanup(vm->vioapic);
569 
570 	for (i = 0; i < vm->maxcpus; i++)
571 		vcpu_cleanup(vm, i, destroy);
572 
573 	vmmops_cleanup(vm->cookie);
574 
575 	/*
576 	 * System memory is removed from the guest address space only when
577 	 * the VM is destroyed. This is because the mapping remains the same
578 	 * across VM reset.
579 	 *
580 	 * Device memory can be relocated by the guest (e.g. using PCI BARs)
581 	 * so those mappings are removed on a VM reset.
582 	 */
583 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
584 		mm = &vm->mem_maps[i];
585 		if (destroy || !sysmem_mapping(vm, mm))
586 			vm_free_memmap(vm, i);
587 	}
588 
589 	if (destroy) {
590 		for (i = 0; i < VM_MAX_MEMSEGS; i++)
591 			vm_free_memseg(vm, i);
592 
593 		vmmops_vmspace_free(vm->vmspace);
594 		vm->vmspace = NULL;
595 	}
596 }
597 
598 void
599 vm_destroy(struct vm *vm)
600 {
601 	vm_cleanup(vm, true);
602 	free(vm, M_VM);
603 }
604 
605 int
606 vm_reinit(struct vm *vm)
607 {
608 	int error;
609 
610 	/*
611 	 * A virtual machine can be reset only if all vcpus are suspended.
612 	 */
613 	if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) {
614 		vm_cleanup(vm, false);
615 		vm_init(vm, false);
616 		error = 0;
617 	} else {
618 		error = EBUSY;
619 	}
620 
621 	return (error);
622 }
623 
624 const char *
625 vm_name(struct vm *vm)
626 {
627 	return (vm->name);
628 }
629 
630 int
631 vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa)
632 {
633 	vm_object_t obj;
634 
635 	if ((obj = vmm_mmio_alloc(vm->vmspace, gpa, len, hpa)) == NULL)
636 		return (ENOMEM);
637 	else
638 		return (0);
639 }
640 
641 int
642 vm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len)
643 {
644 
645 	vmm_mmio_free(vm->vmspace, gpa, len);
646 	return (0);
647 }
648 
649 /*
650  * Return 'true' if 'gpa' is allocated in the guest address space.
651  *
652  * This function is called in the context of a running vcpu which acts as
653  * an implicit lock on 'vm->mem_maps[]'.
654  */
655 bool
656 vm_mem_allocated(struct vm *vm, int vcpuid, vm_paddr_t gpa)
657 {
658 	struct mem_map *mm;
659 	int i;
660 
661 #ifdef INVARIANTS
662 	int hostcpu, state;
663 	state = vcpu_get_state(vm, vcpuid, &hostcpu);
664 	KASSERT(state == VCPU_RUNNING && hostcpu == curcpu,
665 	    ("%s: invalid vcpu state %d/%d", __func__, state, hostcpu));
666 #endif
667 
668 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
669 		mm = &vm->mem_maps[i];
670 		if (mm->len != 0 && gpa >= mm->gpa && gpa < mm->gpa + mm->len)
671 			return (true);		/* 'gpa' is sysmem or devmem */
672 	}
673 
674 	if (ppt_is_mmio(vm, gpa))
675 		return (true);			/* 'gpa' is pci passthru mmio */
676 
677 	return (false);
678 }
679 
680 int
681 vm_alloc_memseg(struct vm *vm, int ident, size_t len, bool sysmem)
682 {
683 	struct mem_seg *seg;
684 	vm_object_t obj;
685 
686 	if (ident < 0 || ident >= VM_MAX_MEMSEGS)
687 		return (EINVAL);
688 
689 	if (len == 0 || (len & PAGE_MASK))
690 		return (EINVAL);
691 
692 	seg = &vm->mem_segs[ident];
693 	if (seg->object != NULL) {
694 		if (seg->len == len && seg->sysmem == sysmem)
695 			return (EEXIST);
696 		else
697 			return (EINVAL);
698 	}
699 
700 	obj = vm_object_allocate(OBJT_DEFAULT, len >> PAGE_SHIFT);
701 	if (obj == NULL)
702 		return (ENOMEM);
703 
704 	seg->len = len;
705 	seg->object = obj;
706 	seg->sysmem = sysmem;
707 	return (0);
708 }
709 
710 int
711 vm_get_memseg(struct vm *vm, int ident, size_t *len, bool *sysmem,
712     vm_object_t *objptr)
713 {
714 	struct mem_seg *seg;
715 
716 	if (ident < 0 || ident >= VM_MAX_MEMSEGS)
717 		return (EINVAL);
718 
719 	seg = &vm->mem_segs[ident];
720 	if (len)
721 		*len = seg->len;
722 	if (sysmem)
723 		*sysmem = seg->sysmem;
724 	if (objptr)
725 		*objptr = seg->object;
726 	return (0);
727 }
728 
729 void
730 vm_free_memseg(struct vm *vm, int ident)
731 {
732 	struct mem_seg *seg;
733 
734 	KASSERT(ident >= 0 && ident < VM_MAX_MEMSEGS,
735 	    ("%s: invalid memseg ident %d", __func__, ident));
736 
737 	seg = &vm->mem_segs[ident];
738 	if (seg->object != NULL) {
739 		vm_object_deallocate(seg->object);
740 		bzero(seg, sizeof(struct mem_seg));
741 	}
742 }
743 
744 int
745 vm_mmap_memseg(struct vm *vm, vm_paddr_t gpa, int segid, vm_ooffset_t first,
746     size_t len, int prot, int flags)
747 {
748 	struct mem_seg *seg;
749 	struct mem_map *m, *map;
750 	vm_ooffset_t last;
751 	int i, error;
752 
753 	if (prot == 0 || (prot & ~(VM_PROT_ALL)) != 0)
754 		return (EINVAL);
755 
756 	if (flags & ~VM_MEMMAP_F_WIRED)
757 		return (EINVAL);
758 
759 	if (segid < 0 || segid >= VM_MAX_MEMSEGS)
760 		return (EINVAL);
761 
762 	seg = &vm->mem_segs[segid];
763 	if (seg->object == NULL)
764 		return (EINVAL);
765 
766 	last = first + len;
767 	if (first < 0 || first >= last || last > seg->len)
768 		return (EINVAL);
769 
770 	if ((gpa | first | last) & PAGE_MASK)
771 		return (EINVAL);
772 
773 	map = NULL;
774 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
775 		m = &vm->mem_maps[i];
776 		if (m->len == 0) {
777 			map = m;
778 			break;
779 		}
780 	}
781 
782 	if (map == NULL)
783 		return (ENOSPC);
784 
785 	error = vm_map_find(&vm->vmspace->vm_map, seg->object, first, &gpa,
786 	    len, 0, VMFS_NO_SPACE, prot, prot, 0);
787 	if (error != KERN_SUCCESS)
788 		return (EFAULT);
789 
790 	vm_object_reference(seg->object);
791 
792 	if (flags & VM_MEMMAP_F_WIRED) {
793 		error = vm_map_wire(&vm->vmspace->vm_map, gpa, gpa + len,
794 		    VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
795 		if (error != KERN_SUCCESS) {
796 			vm_map_remove(&vm->vmspace->vm_map, gpa, gpa + len);
797 			return (error == KERN_RESOURCE_SHORTAGE ? ENOMEM :
798 			    EFAULT);
799 		}
800 	}
801 
802 	map->gpa = gpa;
803 	map->len = len;
804 	map->segoff = first;
805 	map->segid = segid;
806 	map->prot = prot;
807 	map->flags = flags;
808 	return (0);
809 }
810 
811 int
812 vm_munmap_memseg(struct vm *vm, vm_paddr_t gpa, size_t len)
813 {
814 	struct mem_map *m;
815 	int i;
816 
817 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
818 		m = &vm->mem_maps[i];
819 		if (m->gpa == gpa && m->len == len &&
820 		    (m->flags & VM_MEMMAP_F_IOMMU) == 0) {
821 			vm_free_memmap(vm, i);
822 			return (0);
823 		}
824 	}
825 
826 	return (EINVAL);
827 }
828 
829 int
830 vm_mmap_getnext(struct vm *vm, vm_paddr_t *gpa, int *segid,
831     vm_ooffset_t *segoff, size_t *len, int *prot, int *flags)
832 {
833 	struct mem_map *mm, *mmnext;
834 	int i;
835 
836 	mmnext = NULL;
837 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
838 		mm = &vm->mem_maps[i];
839 		if (mm->len == 0 || mm->gpa < *gpa)
840 			continue;
841 		if (mmnext == NULL || mm->gpa < mmnext->gpa)
842 			mmnext = mm;
843 	}
844 
845 	if (mmnext != NULL) {
846 		*gpa = mmnext->gpa;
847 		if (segid)
848 			*segid = mmnext->segid;
849 		if (segoff)
850 			*segoff = mmnext->segoff;
851 		if (len)
852 			*len = mmnext->len;
853 		if (prot)
854 			*prot = mmnext->prot;
855 		if (flags)
856 			*flags = mmnext->flags;
857 		return (0);
858 	} else {
859 		return (ENOENT);
860 	}
861 }
862 
863 static void
864 vm_free_memmap(struct vm *vm, int ident)
865 {
866 	struct mem_map *mm;
867 	int error __diagused;
868 
869 	mm = &vm->mem_maps[ident];
870 	if (mm->len) {
871 		error = vm_map_remove(&vm->vmspace->vm_map, mm->gpa,
872 		    mm->gpa + mm->len);
873 		KASSERT(error == KERN_SUCCESS, ("%s: vm_map_remove error %d",
874 		    __func__, error));
875 		bzero(mm, sizeof(struct mem_map));
876 	}
877 }
878 
879 static __inline bool
880 sysmem_mapping(struct vm *vm, struct mem_map *mm)
881 {
882 
883 	if (mm->len != 0 && vm->mem_segs[mm->segid].sysmem)
884 		return (true);
885 	else
886 		return (false);
887 }
888 
889 vm_paddr_t
890 vmm_sysmem_maxaddr(struct vm *vm)
891 {
892 	struct mem_map *mm;
893 	vm_paddr_t maxaddr;
894 	int i;
895 
896 	maxaddr = 0;
897 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
898 		mm = &vm->mem_maps[i];
899 		if (sysmem_mapping(vm, mm)) {
900 			if (maxaddr < mm->gpa + mm->len)
901 				maxaddr = mm->gpa + mm->len;
902 		}
903 	}
904 	return (maxaddr);
905 }
906 
907 static void
908 vm_iommu_modify(struct vm *vm, bool map)
909 {
910 	int i, sz;
911 	vm_paddr_t gpa, hpa;
912 	struct mem_map *mm;
913 	void *vp, *cookie, *host_domain;
914 
915 	sz = PAGE_SIZE;
916 	host_domain = iommu_host_domain();
917 
918 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
919 		mm = &vm->mem_maps[i];
920 		if (!sysmem_mapping(vm, mm))
921 			continue;
922 
923 		if (map) {
924 			KASSERT((mm->flags & VM_MEMMAP_F_IOMMU) == 0,
925 			    ("iommu map found invalid memmap %#lx/%#lx/%#x",
926 			    mm->gpa, mm->len, mm->flags));
927 			if ((mm->flags & VM_MEMMAP_F_WIRED) == 0)
928 				continue;
929 			mm->flags |= VM_MEMMAP_F_IOMMU;
930 		} else {
931 			if ((mm->flags & VM_MEMMAP_F_IOMMU) == 0)
932 				continue;
933 			mm->flags &= ~VM_MEMMAP_F_IOMMU;
934 			KASSERT((mm->flags & VM_MEMMAP_F_WIRED) != 0,
935 			    ("iommu unmap found invalid memmap %#lx/%#lx/%#x",
936 			    mm->gpa, mm->len, mm->flags));
937 		}
938 
939 		gpa = mm->gpa;
940 		while (gpa < mm->gpa + mm->len) {
941 			vp = vm_gpa_hold(vm, -1, gpa, PAGE_SIZE, VM_PROT_WRITE,
942 					 &cookie);
943 			KASSERT(vp != NULL, ("vm(%s) could not map gpa %#lx",
944 			    vm_name(vm), gpa));
945 
946 			vm_gpa_release(cookie);
947 
948 			hpa = DMAP_TO_PHYS((uintptr_t)vp);
949 			if (map) {
950 				iommu_create_mapping(vm->iommu, gpa, hpa, sz);
951 			} else {
952 				iommu_remove_mapping(vm->iommu, gpa, sz);
953 			}
954 
955 			gpa += PAGE_SIZE;
956 		}
957 	}
958 
959 	/*
960 	 * Invalidate the cached translations associated with the domain
961 	 * from which pages were removed.
962 	 */
963 	if (map)
964 		iommu_invalidate_tlb(host_domain);
965 	else
966 		iommu_invalidate_tlb(vm->iommu);
967 }
968 
969 #define	vm_iommu_unmap(vm)	vm_iommu_modify((vm), false)
970 #define	vm_iommu_map(vm)	vm_iommu_modify((vm), true)
971 
972 int
973 vm_unassign_pptdev(struct vm *vm, int bus, int slot, int func)
974 {
975 	int error;
976 
977 	error = ppt_unassign_device(vm, bus, slot, func);
978 	if (error)
979 		return (error);
980 
981 	if (ppt_assigned_devices(vm) == 0)
982 		vm_iommu_unmap(vm);
983 
984 	return (0);
985 }
986 
987 int
988 vm_assign_pptdev(struct vm *vm, int bus, int slot, int func)
989 {
990 	int error;
991 	vm_paddr_t maxaddr;
992 
993 	/* Set up the IOMMU to do the 'gpa' to 'hpa' translation */
994 	if (ppt_assigned_devices(vm) == 0) {
995 		KASSERT(vm->iommu == NULL,
996 		    ("vm_assign_pptdev: iommu must be NULL"));
997 		maxaddr = vmm_sysmem_maxaddr(vm);
998 		vm->iommu = iommu_create_domain(maxaddr);
999 		if (vm->iommu == NULL)
1000 			return (ENXIO);
1001 		vm_iommu_map(vm);
1002 	}
1003 
1004 	error = ppt_assign_device(vm, bus, slot, func);
1005 	return (error);
1006 }
1007 
1008 void *
1009 vm_gpa_hold(struct vm *vm, int vcpuid, vm_paddr_t gpa, size_t len, int reqprot,
1010 	    void **cookie)
1011 {
1012 	int i, count, pageoff;
1013 	struct mem_map *mm;
1014 	vm_page_t m;
1015 #ifdef INVARIANTS
1016 	/*
1017 	 * All vcpus are frozen by ioctls that modify the memory map
1018 	 * (e.g. VM_MMAP_MEMSEG). Therefore 'vm->memmap[]' stability is
1019 	 * guaranteed if at least one vcpu is in the VCPU_FROZEN state.
1020 	 */
1021 	int state;
1022 	KASSERT(vcpuid >= -1 && vcpuid < vm->maxcpus, ("%s: invalid vcpuid %d",
1023 	    __func__, vcpuid));
1024 	for (i = 0; i < vm->maxcpus; i++) {
1025 		if (vcpuid != -1 && vcpuid != i)
1026 			continue;
1027 		state = vcpu_get_state(vm, i, NULL);
1028 		KASSERT(state == VCPU_FROZEN, ("%s: invalid vcpu state %d",
1029 		    __func__, state));
1030 	}
1031 #endif
1032 	pageoff = gpa & PAGE_MASK;
1033 	if (len > PAGE_SIZE - pageoff)
1034 		panic("vm_gpa_hold: invalid gpa/len: 0x%016lx/%lu", gpa, len);
1035 
1036 	count = 0;
1037 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
1038 		mm = &vm->mem_maps[i];
1039 		if (gpa >= mm->gpa && gpa < mm->gpa + mm->len) {
1040 			count = vm_fault_quick_hold_pages(&vm->vmspace->vm_map,
1041 			    trunc_page(gpa), PAGE_SIZE, reqprot, &m, 1);
1042 			break;
1043 		}
1044 	}
1045 
1046 	if (count == 1) {
1047 		*cookie = m;
1048 		return ((void *)(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)) + pageoff));
1049 	} else {
1050 		*cookie = NULL;
1051 		return (NULL);
1052 	}
1053 }
1054 
1055 void
1056 vm_gpa_release(void *cookie)
1057 {
1058 	vm_page_t m = cookie;
1059 
1060 	vm_page_unwire(m, PQ_ACTIVE);
1061 }
1062 
1063 int
1064 vm_get_register(struct vm *vm, int vcpu, int reg, uint64_t *retval)
1065 {
1066 
1067 	if (vcpu < 0 || vcpu >= vm->maxcpus)
1068 		return (EINVAL);
1069 
1070 	if (reg >= VM_REG_LAST)
1071 		return (EINVAL);
1072 
1073 	return (vmmops_getreg(vm->cookie, vcpu, reg, retval));
1074 }
1075 
1076 int
1077 vm_set_register(struct vm *vm, int vcpuid, int reg, uint64_t val)
1078 {
1079 	struct vcpu *vcpu;
1080 	int error;
1081 
1082 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
1083 		return (EINVAL);
1084 
1085 	if (reg >= VM_REG_LAST)
1086 		return (EINVAL);
1087 
1088 	error = vmmops_setreg(vm->cookie, vcpuid, reg, val);
1089 	if (error || reg != VM_REG_GUEST_RIP)
1090 		return (error);
1091 
1092 	/* Set 'nextrip' to match the value of %rip */
1093 	VCPU_CTR1(vm, vcpuid, "Setting nextrip to %#lx", val);
1094 	vcpu = &vm->vcpu[vcpuid];
1095 	vcpu->nextrip = val;
1096 	return (0);
1097 }
1098 
1099 static bool
1100 is_descriptor_table(int reg)
1101 {
1102 
1103 	switch (reg) {
1104 	case VM_REG_GUEST_IDTR:
1105 	case VM_REG_GUEST_GDTR:
1106 		return (true);
1107 	default:
1108 		return (false);
1109 	}
1110 }
1111 
1112 static bool
1113 is_segment_register(int reg)
1114 {
1115 
1116 	switch (reg) {
1117 	case VM_REG_GUEST_ES:
1118 	case VM_REG_GUEST_CS:
1119 	case VM_REG_GUEST_SS:
1120 	case VM_REG_GUEST_DS:
1121 	case VM_REG_GUEST_FS:
1122 	case VM_REG_GUEST_GS:
1123 	case VM_REG_GUEST_TR:
1124 	case VM_REG_GUEST_LDTR:
1125 		return (true);
1126 	default:
1127 		return (false);
1128 	}
1129 }
1130 
1131 int
1132 vm_get_seg_desc(struct vm *vm, int vcpu, int reg,
1133 		struct seg_desc *desc)
1134 {
1135 
1136 	if (vcpu < 0 || vcpu >= vm->maxcpus)
1137 		return (EINVAL);
1138 
1139 	if (!is_segment_register(reg) && !is_descriptor_table(reg))
1140 		return (EINVAL);
1141 
1142 	return (vmmops_getdesc(vm->cookie, vcpu, reg, desc));
1143 }
1144 
1145 int
1146 vm_set_seg_desc(struct vm *vm, int vcpu, int reg,
1147 		struct seg_desc *desc)
1148 {
1149 	if (vcpu < 0 || vcpu >= vm->maxcpus)
1150 		return (EINVAL);
1151 
1152 	if (!is_segment_register(reg) && !is_descriptor_table(reg))
1153 		return (EINVAL);
1154 
1155 	return (vmmops_setdesc(vm->cookie, vcpu, reg, desc));
1156 }
1157 
1158 static void
1159 restore_guest_fpustate(struct vcpu *vcpu)
1160 {
1161 
1162 	/* flush host state to the pcb */
1163 	fpuexit(curthread);
1164 
1165 	/* restore guest FPU state */
1166 	fpu_stop_emulating();
1167 	fpurestore(vcpu->guestfpu);
1168 
1169 	/* restore guest XCR0 if XSAVE is enabled in the host */
1170 	if (rcr4() & CR4_XSAVE)
1171 		load_xcr(0, vcpu->guest_xcr0);
1172 
1173 	/*
1174 	 * The FPU is now "dirty" with the guest's state so turn on emulation
1175 	 * to trap any access to the FPU by the host.
1176 	 */
1177 	fpu_start_emulating();
1178 }
1179 
1180 static void
1181 save_guest_fpustate(struct vcpu *vcpu)
1182 {
1183 
1184 	if ((rcr0() & CR0_TS) == 0)
1185 		panic("fpu emulation not enabled in host!");
1186 
1187 	/* save guest XCR0 and restore host XCR0 */
1188 	if (rcr4() & CR4_XSAVE) {
1189 		vcpu->guest_xcr0 = rxcr(0);
1190 		load_xcr(0, vmm_get_host_xcr0());
1191 	}
1192 
1193 	/* save guest FPU state */
1194 	fpu_stop_emulating();
1195 	fpusave(vcpu->guestfpu);
1196 	fpu_start_emulating();
1197 }
1198 
1199 static VMM_STAT(VCPU_IDLE_TICKS, "number of ticks vcpu was idle");
1200 
1201 static int
1202 vcpu_set_state_locked(struct vm *vm, int vcpuid, enum vcpu_state newstate,
1203     bool from_idle)
1204 {
1205 	struct vcpu *vcpu;
1206 	int error;
1207 
1208 	vcpu = &vm->vcpu[vcpuid];
1209 	vcpu_assert_locked(vcpu);
1210 
1211 	/*
1212 	 * State transitions from the vmmdev_ioctl() must always begin from
1213 	 * the VCPU_IDLE state. This guarantees that there is only a single
1214 	 * ioctl() operating on a vcpu at any point.
1215 	 */
1216 	if (from_idle) {
1217 		while (vcpu->state != VCPU_IDLE) {
1218 			vcpu->reqidle = 1;
1219 			vcpu_notify_event_locked(vcpu, false);
1220 			VCPU_CTR1(vm, vcpuid, "vcpu state change from %s to "
1221 			    "idle requested", vcpu_state2str(vcpu->state));
1222 			msleep_spin(&vcpu->state, &vcpu->mtx, "vmstat", hz);
1223 		}
1224 	} else {
1225 		KASSERT(vcpu->state != VCPU_IDLE, ("invalid transition from "
1226 		    "vcpu idle state"));
1227 	}
1228 
1229 	if (vcpu->state == VCPU_RUNNING) {
1230 		KASSERT(vcpu->hostcpu == curcpu, ("curcpu %d and hostcpu %d "
1231 		    "mismatch for running vcpu", curcpu, vcpu->hostcpu));
1232 	} else {
1233 		KASSERT(vcpu->hostcpu == NOCPU, ("Invalid hostcpu %d for a "
1234 		    "vcpu that is not running", vcpu->hostcpu));
1235 	}
1236 
1237 	/*
1238 	 * The following state transitions are allowed:
1239 	 * IDLE -> FROZEN -> IDLE
1240 	 * FROZEN -> RUNNING -> FROZEN
1241 	 * FROZEN -> SLEEPING -> FROZEN
1242 	 */
1243 	switch (vcpu->state) {
1244 	case VCPU_IDLE:
1245 	case VCPU_RUNNING:
1246 	case VCPU_SLEEPING:
1247 		error = (newstate != VCPU_FROZEN);
1248 		break;
1249 	case VCPU_FROZEN:
1250 		error = (newstate == VCPU_FROZEN);
1251 		break;
1252 	default:
1253 		error = 1;
1254 		break;
1255 	}
1256 
1257 	if (error)
1258 		return (EBUSY);
1259 
1260 	VCPU_CTR2(vm, vcpuid, "vcpu state changed from %s to %s",
1261 	    vcpu_state2str(vcpu->state), vcpu_state2str(newstate));
1262 
1263 	vcpu->state = newstate;
1264 	if (newstate == VCPU_RUNNING)
1265 		vcpu->hostcpu = curcpu;
1266 	else
1267 		vcpu->hostcpu = NOCPU;
1268 
1269 	if (newstate == VCPU_IDLE)
1270 		wakeup(&vcpu->state);
1271 
1272 	return (0);
1273 }
1274 
1275 static void
1276 vcpu_require_state(struct vm *vm, int vcpuid, enum vcpu_state newstate)
1277 {
1278 	int error;
1279 
1280 	if ((error = vcpu_set_state(vm, vcpuid, newstate, false)) != 0)
1281 		panic("Error %d setting state to %d\n", error, newstate);
1282 }
1283 
1284 static void
1285 vcpu_require_state_locked(struct vm *vm, int vcpuid, enum vcpu_state newstate)
1286 {
1287 	int error;
1288 
1289 	if ((error = vcpu_set_state_locked(vm, vcpuid, newstate, false)) != 0)
1290 		panic("Error %d setting state to %d", error, newstate);
1291 }
1292 
1293 #define	RENDEZVOUS_CTR0(vm, vcpuid, fmt)				\
1294 	do {								\
1295 		if (vcpuid >= 0)					\
1296 			VCPU_CTR0(vm, vcpuid, fmt);			\
1297 		else							\
1298 			VM_CTR0(vm, fmt);				\
1299 	} while (0)
1300 
1301 static int
1302 vm_handle_rendezvous(struct vm *vm, int vcpuid)
1303 {
1304 	struct thread *td;
1305 	int error;
1306 
1307 	KASSERT(vcpuid == -1 || (vcpuid >= 0 && vcpuid < vm->maxcpus),
1308 	    ("vm_handle_rendezvous: invalid vcpuid %d", vcpuid));
1309 
1310 	error = 0;
1311 	td = curthread;
1312 	mtx_lock(&vm->rendezvous_mtx);
1313 	while (vm->rendezvous_func != NULL) {
1314 		/* 'rendezvous_req_cpus' must be a subset of 'active_cpus' */
1315 		CPU_AND(&vm->rendezvous_req_cpus, &vm->rendezvous_req_cpus, &vm->active_cpus);
1316 
1317 		if (vcpuid != -1 &&
1318 		    CPU_ISSET(vcpuid, &vm->rendezvous_req_cpus) &&
1319 		    !CPU_ISSET(vcpuid, &vm->rendezvous_done_cpus)) {
1320 			VCPU_CTR0(vm, vcpuid, "Calling rendezvous func");
1321 			(*vm->rendezvous_func)(vm, vcpuid, vm->rendezvous_arg);
1322 			CPU_SET(vcpuid, &vm->rendezvous_done_cpus);
1323 		}
1324 		if (CPU_CMP(&vm->rendezvous_req_cpus,
1325 		    &vm->rendezvous_done_cpus) == 0) {
1326 			VCPU_CTR0(vm, vcpuid, "Rendezvous completed");
1327 			vm->rendezvous_func = NULL;
1328 			wakeup(&vm->rendezvous_func);
1329 			break;
1330 		}
1331 		RENDEZVOUS_CTR0(vm, vcpuid, "Wait for rendezvous completion");
1332 		mtx_sleep(&vm->rendezvous_func, &vm->rendezvous_mtx, 0,
1333 		    "vmrndv", hz);
1334 		if ((td->td_flags & TDF_NEEDSUSPCHK) != 0) {
1335 			mtx_unlock(&vm->rendezvous_mtx);
1336 			error = thread_check_susp(td, true);
1337 			if (error != 0)
1338 				return (error);
1339 			mtx_lock(&vm->rendezvous_mtx);
1340 		}
1341 	}
1342 	mtx_unlock(&vm->rendezvous_mtx);
1343 	return (0);
1344 }
1345 
1346 /*
1347  * Emulate a guest 'hlt' by sleeping until the vcpu is ready to run.
1348  */
1349 static int
1350 vm_handle_hlt(struct vm *vm, int vcpuid, bool intr_disabled, bool *retu)
1351 {
1352 	struct vcpu *vcpu;
1353 	const char *wmesg;
1354 	struct thread *td;
1355 	int error, t, vcpu_halted, vm_halted;
1356 
1357 	KASSERT(!CPU_ISSET(vcpuid, &vm->halted_cpus), ("vcpu already halted"));
1358 
1359 	vcpu = &vm->vcpu[vcpuid];
1360 	vcpu_halted = 0;
1361 	vm_halted = 0;
1362 	error = 0;
1363 	td = curthread;
1364 
1365 	vcpu_lock(vcpu);
1366 	while (1) {
1367 		/*
1368 		 * Do a final check for pending NMI or interrupts before
1369 		 * really putting this thread to sleep. Also check for
1370 		 * software events that would cause this vcpu to wakeup.
1371 		 *
1372 		 * These interrupts/events could have happened after the
1373 		 * vcpu returned from vmmops_run() and before it acquired the
1374 		 * vcpu lock above.
1375 		 */
1376 		if (vm->rendezvous_func != NULL || vm->suspend || vcpu->reqidle)
1377 			break;
1378 		if (vm_nmi_pending(vm, vcpuid))
1379 			break;
1380 		if (!intr_disabled) {
1381 			if (vm_extint_pending(vm, vcpuid) ||
1382 			    vlapic_pending_intr(vcpu->vlapic, NULL)) {
1383 				break;
1384 			}
1385 		}
1386 
1387 		/* Don't go to sleep if the vcpu thread needs to yield */
1388 		if (vcpu_should_yield(vm, vcpuid))
1389 			break;
1390 
1391 		if (vcpu_debugged(vm, vcpuid))
1392 			break;
1393 
1394 		/*
1395 		 * Some Linux guests implement "halt" by having all vcpus
1396 		 * execute HLT with interrupts disabled. 'halted_cpus' keeps
1397 		 * track of the vcpus that have entered this state. When all
1398 		 * vcpus enter the halted state the virtual machine is halted.
1399 		 */
1400 		if (intr_disabled) {
1401 			wmesg = "vmhalt";
1402 			VCPU_CTR0(vm, vcpuid, "Halted");
1403 			if (!vcpu_halted && halt_detection_enabled) {
1404 				vcpu_halted = 1;
1405 				CPU_SET_ATOMIC(vcpuid, &vm->halted_cpus);
1406 			}
1407 			if (CPU_CMP(&vm->halted_cpus, &vm->active_cpus) == 0) {
1408 				vm_halted = 1;
1409 				break;
1410 			}
1411 		} else {
1412 			wmesg = "vmidle";
1413 		}
1414 
1415 		t = ticks;
1416 		vcpu_require_state_locked(vm, vcpuid, VCPU_SLEEPING);
1417 		/*
1418 		 * XXX msleep_spin() cannot be interrupted by signals so
1419 		 * wake up periodically to check pending signals.
1420 		 */
1421 		msleep_spin(vcpu, &vcpu->mtx, wmesg, hz);
1422 		vcpu_require_state_locked(vm, vcpuid, VCPU_FROZEN);
1423 		vmm_stat_incr(vm, vcpuid, VCPU_IDLE_TICKS, ticks - t);
1424 		if ((td->td_flags & TDF_NEEDSUSPCHK) != 0) {
1425 			vcpu_unlock(vcpu);
1426 			error = thread_check_susp(td, false);
1427 			if (error != 0)
1428 				return (error);
1429 			vcpu_lock(vcpu);
1430 		}
1431 	}
1432 
1433 	if (vcpu_halted)
1434 		CPU_CLR_ATOMIC(vcpuid, &vm->halted_cpus);
1435 
1436 	vcpu_unlock(vcpu);
1437 
1438 	if (vm_halted)
1439 		vm_suspend(vm, VM_SUSPEND_HALT);
1440 
1441 	return (0);
1442 }
1443 
1444 static int
1445 vm_handle_paging(struct vm *vm, int vcpuid, bool *retu)
1446 {
1447 	int rv, ftype;
1448 	struct vm_map *map;
1449 	struct vcpu *vcpu;
1450 	struct vm_exit *vme;
1451 
1452 	vcpu = &vm->vcpu[vcpuid];
1453 	vme = &vcpu->exitinfo;
1454 
1455 	KASSERT(vme->inst_length == 0, ("%s: invalid inst_length %d",
1456 	    __func__, vme->inst_length));
1457 
1458 	ftype = vme->u.paging.fault_type;
1459 	KASSERT(ftype == VM_PROT_READ ||
1460 	    ftype == VM_PROT_WRITE || ftype == VM_PROT_EXECUTE,
1461 	    ("vm_handle_paging: invalid fault_type %d", ftype));
1462 
1463 	if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) {
1464 		rv = pmap_emulate_accessed_dirty(vmspace_pmap(vm->vmspace),
1465 		    vme->u.paging.gpa, ftype);
1466 		if (rv == 0) {
1467 			VCPU_CTR2(vm, vcpuid, "%s bit emulation for gpa %#lx",
1468 			    ftype == VM_PROT_READ ? "accessed" : "dirty",
1469 			    vme->u.paging.gpa);
1470 			goto done;
1471 		}
1472 	}
1473 
1474 	map = &vm->vmspace->vm_map;
1475 	rv = vm_fault(map, vme->u.paging.gpa, ftype, VM_FAULT_NORMAL, NULL);
1476 
1477 	VCPU_CTR3(vm, vcpuid, "vm_handle_paging rv = %d, gpa = %#lx, "
1478 	    "ftype = %d", rv, vme->u.paging.gpa, ftype);
1479 
1480 	if (rv != KERN_SUCCESS)
1481 		return (EFAULT);
1482 done:
1483 	return (0);
1484 }
1485 
1486 static int
1487 vm_handle_inst_emul(struct vm *vm, int vcpuid, bool *retu)
1488 {
1489 	struct vie *vie;
1490 	struct vcpu *vcpu;
1491 	struct vm_exit *vme;
1492 	uint64_t gla, gpa, cs_base;
1493 	struct vm_guest_paging *paging;
1494 	mem_region_read_t mread;
1495 	mem_region_write_t mwrite;
1496 	enum vm_cpu_mode cpu_mode;
1497 	int cs_d, error, fault;
1498 
1499 	vcpu = &vm->vcpu[vcpuid];
1500 	vme = &vcpu->exitinfo;
1501 
1502 	KASSERT(vme->inst_length == 0, ("%s: invalid inst_length %d",
1503 	    __func__, vme->inst_length));
1504 
1505 	gla = vme->u.inst_emul.gla;
1506 	gpa = vme->u.inst_emul.gpa;
1507 	cs_base = vme->u.inst_emul.cs_base;
1508 	cs_d = vme->u.inst_emul.cs_d;
1509 	vie = &vme->u.inst_emul.vie;
1510 	paging = &vme->u.inst_emul.paging;
1511 	cpu_mode = paging->cpu_mode;
1512 
1513 	VCPU_CTR1(vm, vcpuid, "inst_emul fault accessing gpa %#lx", gpa);
1514 
1515 	/* Fetch, decode and emulate the faulting instruction */
1516 	if (vie->num_valid == 0) {
1517 		error = vmm_fetch_instruction(vm, vcpuid, paging, vme->rip +
1518 		    cs_base, VIE_INST_SIZE, vie, &fault);
1519 	} else {
1520 		/*
1521 		 * The instruction bytes have already been copied into 'vie'
1522 		 */
1523 		error = fault = 0;
1524 	}
1525 	if (error || fault)
1526 		return (error);
1527 
1528 	if (vmm_decode_instruction(vm, vcpuid, gla, cpu_mode, cs_d, vie) != 0) {
1529 		VCPU_CTR1(vm, vcpuid, "Error decoding instruction at %#lx",
1530 		    vme->rip + cs_base);
1531 		*retu = true;	    /* dump instruction bytes in userspace */
1532 		return (0);
1533 	}
1534 
1535 	/*
1536 	 * Update 'nextrip' based on the length of the emulated instruction.
1537 	 */
1538 	vme->inst_length = vie->num_processed;
1539 	vcpu->nextrip += vie->num_processed;
1540 	VCPU_CTR1(vm, vcpuid, "nextrip updated to %#lx after instruction "
1541 	    "decoding", vcpu->nextrip);
1542 
1543 	/* return to userland unless this is an in-kernel emulated device */
1544 	if (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE) {
1545 		mread = lapic_mmio_read;
1546 		mwrite = lapic_mmio_write;
1547 	} else if (gpa >= VIOAPIC_BASE && gpa < VIOAPIC_BASE + VIOAPIC_SIZE) {
1548 		mread = vioapic_mmio_read;
1549 		mwrite = vioapic_mmio_write;
1550 	} else if (gpa >= VHPET_BASE && gpa < VHPET_BASE + VHPET_SIZE) {
1551 		mread = vhpet_mmio_read;
1552 		mwrite = vhpet_mmio_write;
1553 	} else {
1554 		*retu = true;
1555 		return (0);
1556 	}
1557 
1558 	error = vmm_emulate_instruction(vm, vcpuid, gpa, vie, paging,
1559 	    mread, mwrite, retu);
1560 
1561 	return (error);
1562 }
1563 
1564 static int
1565 vm_handle_suspend(struct vm *vm, int vcpuid, bool *retu)
1566 {
1567 	int error, i;
1568 	struct vcpu *vcpu;
1569 	struct thread *td;
1570 
1571 	error = 0;
1572 	vcpu = &vm->vcpu[vcpuid];
1573 	td = curthread;
1574 
1575 	CPU_SET_ATOMIC(vcpuid, &vm->suspended_cpus);
1576 
1577 	/*
1578 	 * Wait until all 'active_cpus' have suspended themselves.
1579 	 *
1580 	 * Since a VM may be suspended at any time including when one or
1581 	 * more vcpus are doing a rendezvous we need to call the rendezvous
1582 	 * handler while we are waiting to prevent a deadlock.
1583 	 */
1584 	vcpu_lock(vcpu);
1585 	while (error == 0) {
1586 		if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) {
1587 			VCPU_CTR0(vm, vcpuid, "All vcpus suspended");
1588 			break;
1589 		}
1590 
1591 		if (vm->rendezvous_func == NULL) {
1592 			VCPU_CTR0(vm, vcpuid, "Sleeping during suspend");
1593 			vcpu_require_state_locked(vm, vcpuid, VCPU_SLEEPING);
1594 			msleep_spin(vcpu, &vcpu->mtx, "vmsusp", hz);
1595 			vcpu_require_state_locked(vm, vcpuid, VCPU_FROZEN);
1596 			if ((td->td_flags & TDF_NEEDSUSPCHK) != 0) {
1597 				vcpu_unlock(vcpu);
1598 				error = thread_check_susp(td, false);
1599 				vcpu_lock(vcpu);
1600 			}
1601 		} else {
1602 			VCPU_CTR0(vm, vcpuid, "Rendezvous during suspend");
1603 			vcpu_unlock(vcpu);
1604 			error = vm_handle_rendezvous(vm, vcpuid);
1605 			vcpu_lock(vcpu);
1606 		}
1607 	}
1608 	vcpu_unlock(vcpu);
1609 
1610 	/*
1611 	 * Wakeup the other sleeping vcpus and return to userspace.
1612 	 */
1613 	for (i = 0; i < vm->maxcpus; i++) {
1614 		if (CPU_ISSET(i, &vm->suspended_cpus)) {
1615 			vcpu_notify_event(vm, i, false);
1616 		}
1617 	}
1618 
1619 	*retu = true;
1620 	return (error);
1621 }
1622 
1623 static int
1624 vm_handle_reqidle(struct vm *vm, int vcpuid, bool *retu)
1625 {
1626 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
1627 
1628 	vcpu_lock(vcpu);
1629 	KASSERT(vcpu->reqidle, ("invalid vcpu reqidle %d", vcpu->reqidle));
1630 	vcpu->reqidle = 0;
1631 	vcpu_unlock(vcpu);
1632 	*retu = true;
1633 	return (0);
1634 }
1635 
1636 int
1637 vm_suspend(struct vm *vm, enum vm_suspend_how how)
1638 {
1639 	int i;
1640 
1641 	if (how <= VM_SUSPEND_NONE || how >= VM_SUSPEND_LAST)
1642 		return (EINVAL);
1643 
1644 	if (atomic_cmpset_int(&vm->suspend, 0, how) == 0) {
1645 		VM_CTR2(vm, "virtual machine already suspended %d/%d",
1646 		    vm->suspend, how);
1647 		return (EALREADY);
1648 	}
1649 
1650 	VM_CTR1(vm, "virtual machine successfully suspended %d", how);
1651 
1652 	/*
1653 	 * Notify all active vcpus that they are now suspended.
1654 	 */
1655 	for (i = 0; i < vm->maxcpus; i++) {
1656 		if (CPU_ISSET(i, &vm->active_cpus))
1657 			vcpu_notify_event(vm, i, false);
1658 	}
1659 
1660 	return (0);
1661 }
1662 
1663 void
1664 vm_exit_suspended(struct vm *vm, int vcpuid, uint64_t rip)
1665 {
1666 	struct vm_exit *vmexit;
1667 
1668 	KASSERT(vm->suspend > VM_SUSPEND_NONE && vm->suspend < VM_SUSPEND_LAST,
1669 	    ("vm_exit_suspended: invalid suspend type %d", vm->suspend));
1670 
1671 	vmexit = vm_exitinfo(vm, vcpuid);
1672 	vmexit->rip = rip;
1673 	vmexit->inst_length = 0;
1674 	vmexit->exitcode = VM_EXITCODE_SUSPENDED;
1675 	vmexit->u.suspended.how = vm->suspend;
1676 }
1677 
1678 void
1679 vm_exit_debug(struct vm *vm, int vcpuid, uint64_t rip)
1680 {
1681 	struct vm_exit *vmexit;
1682 
1683 	vmexit = vm_exitinfo(vm, vcpuid);
1684 	vmexit->rip = rip;
1685 	vmexit->inst_length = 0;
1686 	vmexit->exitcode = VM_EXITCODE_DEBUG;
1687 }
1688 
1689 void
1690 vm_exit_rendezvous(struct vm *vm, int vcpuid, uint64_t rip)
1691 {
1692 	struct vm_exit *vmexit;
1693 
1694 	KASSERT(vm->rendezvous_func != NULL, ("rendezvous not in progress"));
1695 
1696 	vmexit = vm_exitinfo(vm, vcpuid);
1697 	vmexit->rip = rip;
1698 	vmexit->inst_length = 0;
1699 	vmexit->exitcode = VM_EXITCODE_RENDEZVOUS;
1700 	vmm_stat_incr(vm, vcpuid, VMEXIT_RENDEZVOUS, 1);
1701 }
1702 
1703 void
1704 vm_exit_reqidle(struct vm *vm, int vcpuid, uint64_t rip)
1705 {
1706 	struct vm_exit *vmexit;
1707 
1708 	vmexit = vm_exitinfo(vm, vcpuid);
1709 	vmexit->rip = rip;
1710 	vmexit->inst_length = 0;
1711 	vmexit->exitcode = VM_EXITCODE_REQIDLE;
1712 	vmm_stat_incr(vm, vcpuid, VMEXIT_REQIDLE, 1);
1713 }
1714 
1715 void
1716 vm_exit_astpending(struct vm *vm, int vcpuid, uint64_t rip)
1717 {
1718 	struct vm_exit *vmexit;
1719 
1720 	vmexit = vm_exitinfo(vm, vcpuid);
1721 	vmexit->rip = rip;
1722 	vmexit->inst_length = 0;
1723 	vmexit->exitcode = VM_EXITCODE_BOGUS;
1724 	vmm_stat_incr(vm, vcpuid, VMEXIT_ASTPENDING, 1);
1725 }
1726 
1727 int
1728 vm_run(struct vm *vm, struct vm_run *vmrun)
1729 {
1730 	struct vm_eventinfo evinfo;
1731 	int error, vcpuid;
1732 	struct vcpu *vcpu;
1733 	struct pcb *pcb;
1734 	uint64_t tscval;
1735 	struct vm_exit *vme;
1736 	bool retu, intr_disabled;
1737 	pmap_t pmap;
1738 
1739 	vcpuid = vmrun->cpuid;
1740 
1741 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
1742 		return (EINVAL);
1743 
1744 	if (!CPU_ISSET(vcpuid, &vm->active_cpus))
1745 		return (EINVAL);
1746 
1747 	if (CPU_ISSET(vcpuid, &vm->suspended_cpus))
1748 		return (EINVAL);
1749 
1750 	pmap = vmspace_pmap(vm->vmspace);
1751 	vcpu = &vm->vcpu[vcpuid];
1752 	vme = &vcpu->exitinfo;
1753 	evinfo.rptr = &vm->rendezvous_func;
1754 	evinfo.sptr = &vm->suspend;
1755 	evinfo.iptr = &vcpu->reqidle;
1756 restart:
1757 	critical_enter();
1758 
1759 	KASSERT(!CPU_ISSET(curcpu, &pmap->pm_active),
1760 	    ("vm_run: absurd pm_active"));
1761 
1762 	tscval = rdtsc();
1763 
1764 	pcb = PCPU_GET(curpcb);
1765 	set_pcb_flags(pcb, PCB_FULL_IRET);
1766 
1767 	restore_guest_fpustate(vcpu);
1768 
1769 	vcpu_require_state(vm, vcpuid, VCPU_RUNNING);
1770 	error = vmmops_run(vm->cookie, vcpuid, vcpu->nextrip, pmap, &evinfo);
1771 	vcpu_require_state(vm, vcpuid, VCPU_FROZEN);
1772 
1773 	save_guest_fpustate(vcpu);
1774 
1775 	vmm_stat_incr(vm, vcpuid, VCPU_TOTAL_RUNTIME, rdtsc() - tscval);
1776 
1777 	critical_exit();
1778 
1779 	if (error == 0) {
1780 		retu = false;
1781 		vcpu->nextrip = vme->rip + vme->inst_length;
1782 		switch (vme->exitcode) {
1783 		case VM_EXITCODE_REQIDLE:
1784 			error = vm_handle_reqidle(vm, vcpuid, &retu);
1785 			break;
1786 		case VM_EXITCODE_SUSPENDED:
1787 			error = vm_handle_suspend(vm, vcpuid, &retu);
1788 			break;
1789 		case VM_EXITCODE_IOAPIC_EOI:
1790 			vioapic_process_eoi(vm, vcpuid,
1791 			    vme->u.ioapic_eoi.vector);
1792 			break;
1793 		case VM_EXITCODE_RENDEZVOUS:
1794 			error = vm_handle_rendezvous(vm, vcpuid);
1795 			break;
1796 		case VM_EXITCODE_HLT:
1797 			intr_disabled = ((vme->u.hlt.rflags & PSL_I) == 0);
1798 			error = vm_handle_hlt(vm, vcpuid, intr_disabled, &retu);
1799 			break;
1800 		case VM_EXITCODE_PAGING:
1801 			error = vm_handle_paging(vm, vcpuid, &retu);
1802 			break;
1803 		case VM_EXITCODE_INST_EMUL:
1804 			error = vm_handle_inst_emul(vm, vcpuid, &retu);
1805 			break;
1806 		case VM_EXITCODE_INOUT:
1807 		case VM_EXITCODE_INOUT_STR:
1808 			error = vm_handle_inout(vm, vcpuid, vme, &retu);
1809 			break;
1810 		case VM_EXITCODE_MONITOR:
1811 		case VM_EXITCODE_MWAIT:
1812 		case VM_EXITCODE_VMINSN:
1813 			vm_inject_ud(vm, vcpuid);
1814 			break;
1815 		default:
1816 			retu = true;	/* handled in userland */
1817 			break;
1818 		}
1819 	}
1820 
1821 	if (error == 0 && retu == false)
1822 		goto restart;
1823 
1824 	vmm_stat_incr(vm, vcpuid, VMEXIT_USERSPACE, 1);
1825 	VCPU_CTR2(vm, vcpuid, "retu %d/%d", error, vme->exitcode);
1826 
1827 	/* copy the exit information */
1828 	bcopy(vme, &vmrun->vm_exit, sizeof(struct vm_exit));
1829 	return (error);
1830 }
1831 
1832 int
1833 vm_restart_instruction(void *arg, int vcpuid)
1834 {
1835 	struct vm *vm;
1836 	struct vcpu *vcpu;
1837 	enum vcpu_state state;
1838 	uint64_t rip;
1839 	int error __diagused;
1840 
1841 	vm = arg;
1842 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
1843 		return (EINVAL);
1844 
1845 	vcpu = &vm->vcpu[vcpuid];
1846 	state = vcpu_get_state(vm, vcpuid, NULL);
1847 	if (state == VCPU_RUNNING) {
1848 		/*
1849 		 * When a vcpu is "running" the next instruction is determined
1850 		 * by adding 'rip' and 'inst_length' in the vcpu's 'exitinfo'.
1851 		 * Thus setting 'inst_length' to zero will cause the current
1852 		 * instruction to be restarted.
1853 		 */
1854 		vcpu->exitinfo.inst_length = 0;
1855 		VCPU_CTR1(vm, vcpuid, "restarting instruction at %#lx by "
1856 		    "setting inst_length to zero", vcpu->exitinfo.rip);
1857 	} else if (state == VCPU_FROZEN) {
1858 		/*
1859 		 * When a vcpu is "frozen" it is outside the critical section
1860 		 * around vmmops_run() and 'nextrip' points to the next
1861 		 * instruction. Thus instruction restart is achieved by setting
1862 		 * 'nextrip' to the vcpu's %rip.
1863 		 */
1864 		error = vm_get_register(vm, vcpuid, VM_REG_GUEST_RIP, &rip);
1865 		KASSERT(!error, ("%s: error %d getting rip", __func__, error));
1866 		VCPU_CTR2(vm, vcpuid, "restarting instruction by updating "
1867 		    "nextrip from %#lx to %#lx", vcpu->nextrip, rip);
1868 		vcpu->nextrip = rip;
1869 	} else {
1870 		panic("%s: invalid state %d", __func__, state);
1871 	}
1872 	return (0);
1873 }
1874 
1875 int
1876 vm_exit_intinfo(struct vm *vm, int vcpuid, uint64_t info)
1877 {
1878 	struct vcpu *vcpu;
1879 	int type, vector;
1880 
1881 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
1882 		return (EINVAL);
1883 
1884 	vcpu = &vm->vcpu[vcpuid];
1885 
1886 	if (info & VM_INTINFO_VALID) {
1887 		type = info & VM_INTINFO_TYPE;
1888 		vector = info & 0xff;
1889 		if (type == VM_INTINFO_NMI && vector != IDT_NMI)
1890 			return (EINVAL);
1891 		if (type == VM_INTINFO_HWEXCEPTION && vector >= 32)
1892 			return (EINVAL);
1893 		if (info & VM_INTINFO_RSVD)
1894 			return (EINVAL);
1895 	} else {
1896 		info = 0;
1897 	}
1898 	VCPU_CTR2(vm, vcpuid, "%s: info1(%#lx)", __func__, info);
1899 	vcpu->exitintinfo = info;
1900 	return (0);
1901 }
1902 
1903 enum exc_class {
1904 	EXC_BENIGN,
1905 	EXC_CONTRIBUTORY,
1906 	EXC_PAGEFAULT
1907 };
1908 
1909 #define	IDT_VE	20	/* Virtualization Exception (Intel specific) */
1910 
1911 static enum exc_class
1912 exception_class(uint64_t info)
1913 {
1914 	int type, vector;
1915 
1916 	KASSERT(info & VM_INTINFO_VALID, ("intinfo must be valid: %#lx", info));
1917 	type = info & VM_INTINFO_TYPE;
1918 	vector = info & 0xff;
1919 
1920 	/* Table 6-4, "Interrupt and Exception Classes", Intel SDM, Vol 3 */
1921 	switch (type) {
1922 	case VM_INTINFO_HWINTR:
1923 	case VM_INTINFO_SWINTR:
1924 	case VM_INTINFO_NMI:
1925 		return (EXC_BENIGN);
1926 	default:
1927 		/*
1928 		 * Hardware exception.
1929 		 *
1930 		 * SVM and VT-x use identical type values to represent NMI,
1931 		 * hardware interrupt and software interrupt.
1932 		 *
1933 		 * SVM uses type '3' for all exceptions. VT-x uses type '3'
1934 		 * for exceptions except #BP and #OF. #BP and #OF use a type
1935 		 * value of '5' or '6'. Therefore we don't check for explicit
1936 		 * values of 'type' to classify 'intinfo' into a hardware
1937 		 * exception.
1938 		 */
1939 		break;
1940 	}
1941 
1942 	switch (vector) {
1943 	case IDT_PF:
1944 	case IDT_VE:
1945 		return (EXC_PAGEFAULT);
1946 	case IDT_DE:
1947 	case IDT_TS:
1948 	case IDT_NP:
1949 	case IDT_SS:
1950 	case IDT_GP:
1951 		return (EXC_CONTRIBUTORY);
1952 	default:
1953 		return (EXC_BENIGN);
1954 	}
1955 }
1956 
1957 static int
1958 nested_fault(struct vm *vm, int vcpuid, uint64_t info1, uint64_t info2,
1959     uint64_t *retinfo)
1960 {
1961 	enum exc_class exc1, exc2;
1962 	int type1, vector1;
1963 
1964 	KASSERT(info1 & VM_INTINFO_VALID, ("info1 %#lx is not valid", info1));
1965 	KASSERT(info2 & VM_INTINFO_VALID, ("info2 %#lx is not valid", info2));
1966 
1967 	/*
1968 	 * If an exception occurs while attempting to call the double-fault
1969 	 * handler the processor enters shutdown mode (aka triple fault).
1970 	 */
1971 	type1 = info1 & VM_INTINFO_TYPE;
1972 	vector1 = info1 & 0xff;
1973 	if (type1 == VM_INTINFO_HWEXCEPTION && vector1 == IDT_DF) {
1974 		VCPU_CTR2(vm, vcpuid, "triple fault: info1(%#lx), info2(%#lx)",
1975 		    info1, info2);
1976 		vm_suspend(vm, VM_SUSPEND_TRIPLEFAULT);
1977 		*retinfo = 0;
1978 		return (0);
1979 	}
1980 
1981 	/*
1982 	 * Table 6-5 "Conditions for Generating a Double Fault", Intel SDM, Vol3
1983 	 */
1984 	exc1 = exception_class(info1);
1985 	exc2 = exception_class(info2);
1986 	if ((exc1 == EXC_CONTRIBUTORY && exc2 == EXC_CONTRIBUTORY) ||
1987 	    (exc1 == EXC_PAGEFAULT && exc2 != EXC_BENIGN)) {
1988 		/* Convert nested fault into a double fault. */
1989 		*retinfo = IDT_DF;
1990 		*retinfo |= VM_INTINFO_VALID | VM_INTINFO_HWEXCEPTION;
1991 		*retinfo |= VM_INTINFO_DEL_ERRCODE;
1992 	} else {
1993 		/* Handle exceptions serially */
1994 		*retinfo = info2;
1995 	}
1996 	return (1);
1997 }
1998 
1999 static uint64_t
2000 vcpu_exception_intinfo(struct vcpu *vcpu)
2001 {
2002 	uint64_t info = 0;
2003 
2004 	if (vcpu->exception_pending) {
2005 		info = vcpu->exc_vector & 0xff;
2006 		info |= VM_INTINFO_VALID | VM_INTINFO_HWEXCEPTION;
2007 		if (vcpu->exc_errcode_valid) {
2008 			info |= VM_INTINFO_DEL_ERRCODE;
2009 			info |= (uint64_t)vcpu->exc_errcode << 32;
2010 		}
2011 	}
2012 	return (info);
2013 }
2014 
2015 int
2016 vm_entry_intinfo(struct vm *vm, int vcpuid, uint64_t *retinfo)
2017 {
2018 	struct vcpu *vcpu;
2019 	uint64_t info1, info2;
2020 	int valid;
2021 
2022 	KASSERT(vcpuid >= 0 &&
2023 	    vcpuid < vm->maxcpus, ("invalid vcpu %d", vcpuid));
2024 
2025 	vcpu = &vm->vcpu[vcpuid];
2026 
2027 	info1 = vcpu->exitintinfo;
2028 	vcpu->exitintinfo = 0;
2029 
2030 	info2 = 0;
2031 	if (vcpu->exception_pending) {
2032 		info2 = vcpu_exception_intinfo(vcpu);
2033 		vcpu->exception_pending = 0;
2034 		VCPU_CTR2(vm, vcpuid, "Exception %d delivered: %#lx",
2035 		    vcpu->exc_vector, info2);
2036 	}
2037 
2038 	if ((info1 & VM_INTINFO_VALID) && (info2 & VM_INTINFO_VALID)) {
2039 		valid = nested_fault(vm, vcpuid, info1, info2, retinfo);
2040 	} else if (info1 & VM_INTINFO_VALID) {
2041 		*retinfo = info1;
2042 		valid = 1;
2043 	} else if (info2 & VM_INTINFO_VALID) {
2044 		*retinfo = info2;
2045 		valid = 1;
2046 	} else {
2047 		valid = 0;
2048 	}
2049 
2050 	if (valid) {
2051 		VCPU_CTR4(vm, vcpuid, "%s: info1(%#lx), info2(%#lx), "
2052 		    "retinfo(%#lx)", __func__, info1, info2, *retinfo);
2053 	}
2054 
2055 	return (valid);
2056 }
2057 
2058 int
2059 vm_get_intinfo(struct vm *vm, int vcpuid, uint64_t *info1, uint64_t *info2)
2060 {
2061 	struct vcpu *vcpu;
2062 
2063 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2064 		return (EINVAL);
2065 
2066 	vcpu = &vm->vcpu[vcpuid];
2067 	*info1 = vcpu->exitintinfo;
2068 	*info2 = vcpu_exception_intinfo(vcpu);
2069 	return (0);
2070 }
2071 
2072 int
2073 vm_inject_exception(struct vm *vm, int vcpuid, int vector, int errcode_valid,
2074     uint32_t errcode, int restart_instruction)
2075 {
2076 	struct vcpu *vcpu;
2077 	uint64_t regval;
2078 	int error __diagused;
2079 
2080 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2081 		return (EINVAL);
2082 
2083 	if (vector < 0 || vector >= 32)
2084 		return (EINVAL);
2085 
2086 	/*
2087 	 * A double fault exception should never be injected directly into
2088 	 * the guest. It is a derived exception that results from specific
2089 	 * combinations of nested faults.
2090 	 */
2091 	if (vector == IDT_DF)
2092 		return (EINVAL);
2093 
2094 	vcpu = &vm->vcpu[vcpuid];
2095 
2096 	if (vcpu->exception_pending) {
2097 		VCPU_CTR2(vm, vcpuid, "Unable to inject exception %d due to "
2098 		    "pending exception %d", vector, vcpu->exc_vector);
2099 		return (EBUSY);
2100 	}
2101 
2102 	if (errcode_valid) {
2103 		/*
2104 		 * Exceptions don't deliver an error code in real mode.
2105 		 */
2106 		error = vm_get_register(vm, vcpuid, VM_REG_GUEST_CR0, &regval);
2107 		KASSERT(!error, ("%s: error %d getting CR0", __func__, error));
2108 		if (!(regval & CR0_PE))
2109 			errcode_valid = 0;
2110 	}
2111 
2112 	/*
2113 	 * From section 26.6.1 "Interruptibility State" in Intel SDM:
2114 	 *
2115 	 * Event blocking by "STI" or "MOV SS" is cleared after guest executes
2116 	 * one instruction or incurs an exception.
2117 	 */
2118 	error = vm_set_register(vm, vcpuid, VM_REG_GUEST_INTR_SHADOW, 0);
2119 	KASSERT(error == 0, ("%s: error %d clearing interrupt shadow",
2120 	    __func__, error));
2121 
2122 	if (restart_instruction)
2123 		vm_restart_instruction(vm, vcpuid);
2124 
2125 	vcpu->exception_pending = 1;
2126 	vcpu->exc_vector = vector;
2127 	vcpu->exc_errcode = errcode;
2128 	vcpu->exc_errcode_valid = errcode_valid;
2129 	VCPU_CTR1(vm, vcpuid, "Exception %d pending", vector);
2130 	return (0);
2131 }
2132 
2133 void
2134 vm_inject_fault(void *vmarg, int vcpuid, int vector, int errcode_valid,
2135     int errcode)
2136 {
2137 	struct vm *vm;
2138 	int error __diagused, restart_instruction;
2139 
2140 	vm = vmarg;
2141 	restart_instruction = 1;
2142 
2143 	error = vm_inject_exception(vm, vcpuid, vector, errcode_valid,
2144 	    errcode, restart_instruction);
2145 	KASSERT(error == 0, ("vm_inject_exception error %d", error));
2146 }
2147 
2148 void
2149 vm_inject_pf(void *vmarg, int vcpuid, int error_code, uint64_t cr2)
2150 {
2151 	struct vm *vm;
2152 	int error __diagused;
2153 
2154 	vm = vmarg;
2155 	VCPU_CTR2(vm, vcpuid, "Injecting page fault: error_code %#x, cr2 %#lx",
2156 	    error_code, cr2);
2157 
2158 	error = vm_set_register(vm, vcpuid, VM_REG_GUEST_CR2, cr2);
2159 	KASSERT(error == 0, ("vm_set_register(cr2) error %d", error));
2160 
2161 	vm_inject_fault(vm, vcpuid, IDT_PF, 1, error_code);
2162 }
2163 
2164 static VMM_STAT(VCPU_NMI_COUNT, "number of NMIs delivered to vcpu");
2165 
2166 int
2167 vm_inject_nmi(struct vm *vm, int vcpuid)
2168 {
2169 	struct vcpu *vcpu;
2170 
2171 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2172 		return (EINVAL);
2173 
2174 	vcpu = &vm->vcpu[vcpuid];
2175 
2176 	vcpu->nmi_pending = 1;
2177 	vcpu_notify_event(vm, vcpuid, false);
2178 	return (0);
2179 }
2180 
2181 int
2182 vm_nmi_pending(struct vm *vm, int vcpuid)
2183 {
2184 	struct vcpu *vcpu;
2185 
2186 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2187 		panic("vm_nmi_pending: invalid vcpuid %d", vcpuid);
2188 
2189 	vcpu = &vm->vcpu[vcpuid];
2190 
2191 	return (vcpu->nmi_pending);
2192 }
2193 
2194 void
2195 vm_nmi_clear(struct vm *vm, int vcpuid)
2196 {
2197 	struct vcpu *vcpu;
2198 
2199 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2200 		panic("vm_nmi_pending: invalid vcpuid %d", vcpuid);
2201 
2202 	vcpu = &vm->vcpu[vcpuid];
2203 
2204 	if (vcpu->nmi_pending == 0)
2205 		panic("vm_nmi_clear: inconsistent nmi_pending state");
2206 
2207 	vcpu->nmi_pending = 0;
2208 	vmm_stat_incr(vm, vcpuid, VCPU_NMI_COUNT, 1);
2209 }
2210 
2211 static VMM_STAT(VCPU_EXTINT_COUNT, "number of ExtINTs delivered to vcpu");
2212 
2213 int
2214 vm_inject_extint(struct vm *vm, int vcpuid)
2215 {
2216 	struct vcpu *vcpu;
2217 
2218 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2219 		return (EINVAL);
2220 
2221 	vcpu = &vm->vcpu[vcpuid];
2222 
2223 	vcpu->extint_pending = 1;
2224 	vcpu_notify_event(vm, vcpuid, false);
2225 	return (0);
2226 }
2227 
2228 int
2229 vm_extint_pending(struct vm *vm, int vcpuid)
2230 {
2231 	struct vcpu *vcpu;
2232 
2233 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2234 		panic("vm_extint_pending: invalid vcpuid %d", vcpuid);
2235 
2236 	vcpu = &vm->vcpu[vcpuid];
2237 
2238 	return (vcpu->extint_pending);
2239 }
2240 
2241 void
2242 vm_extint_clear(struct vm *vm, int vcpuid)
2243 {
2244 	struct vcpu *vcpu;
2245 
2246 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2247 		panic("vm_extint_pending: invalid vcpuid %d", vcpuid);
2248 
2249 	vcpu = &vm->vcpu[vcpuid];
2250 
2251 	if (vcpu->extint_pending == 0)
2252 		panic("vm_extint_clear: inconsistent extint_pending state");
2253 
2254 	vcpu->extint_pending = 0;
2255 	vmm_stat_incr(vm, vcpuid, VCPU_EXTINT_COUNT, 1);
2256 }
2257 
2258 int
2259 vm_get_capability(struct vm *vm, int vcpu, int type, int *retval)
2260 {
2261 	if (vcpu < 0 || vcpu >= vm->maxcpus)
2262 		return (EINVAL);
2263 
2264 	if (type < 0 || type >= VM_CAP_MAX)
2265 		return (EINVAL);
2266 
2267 	return (vmmops_getcap(vm->cookie, vcpu, type, retval));
2268 }
2269 
2270 int
2271 vm_set_capability(struct vm *vm, int vcpu, int type, int val)
2272 {
2273 	if (vcpu < 0 || vcpu >= vm->maxcpus)
2274 		return (EINVAL);
2275 
2276 	if (type < 0 || type >= VM_CAP_MAX)
2277 		return (EINVAL);
2278 
2279 	return (vmmops_setcap(vm->cookie, vcpu, type, val));
2280 }
2281 
2282 struct vlapic *
2283 vm_lapic(struct vm *vm, int cpu)
2284 {
2285 	return (vm->vcpu[cpu].vlapic);
2286 }
2287 
2288 struct vioapic *
2289 vm_ioapic(struct vm *vm)
2290 {
2291 
2292 	return (vm->vioapic);
2293 }
2294 
2295 struct vhpet *
2296 vm_hpet(struct vm *vm)
2297 {
2298 
2299 	return (vm->vhpet);
2300 }
2301 
2302 bool
2303 vmm_is_pptdev(int bus, int slot, int func)
2304 {
2305 	int b, f, i, n, s;
2306 	char *val, *cp, *cp2;
2307 	bool found;
2308 
2309 	/*
2310 	 * XXX
2311 	 * The length of an environment variable is limited to 128 bytes which
2312 	 * puts an upper limit on the number of passthru devices that may be
2313 	 * specified using a single environment variable.
2314 	 *
2315 	 * Work around this by scanning multiple environment variable
2316 	 * names instead of a single one - yuck!
2317 	 */
2318 	const char *names[] = { "pptdevs", "pptdevs2", "pptdevs3", NULL };
2319 
2320 	/* set pptdevs="1/2/3 4/5/6 7/8/9 10/11/12" */
2321 	found = false;
2322 	for (i = 0; names[i] != NULL && !found; i++) {
2323 		cp = val = kern_getenv(names[i]);
2324 		while (cp != NULL && *cp != '\0') {
2325 			if ((cp2 = strchr(cp, ' ')) != NULL)
2326 				*cp2 = '\0';
2327 
2328 			n = sscanf(cp, "%d/%d/%d", &b, &s, &f);
2329 			if (n == 3 && bus == b && slot == s && func == f) {
2330 				found = true;
2331 				break;
2332 			}
2333 
2334 			if (cp2 != NULL)
2335 				*cp2++ = ' ';
2336 
2337 			cp = cp2;
2338 		}
2339 		freeenv(val);
2340 	}
2341 	return (found);
2342 }
2343 
2344 void *
2345 vm_iommu_domain(struct vm *vm)
2346 {
2347 
2348 	return (vm->iommu);
2349 }
2350 
2351 int
2352 vcpu_set_state(struct vm *vm, int vcpuid, enum vcpu_state newstate,
2353     bool from_idle)
2354 {
2355 	int error;
2356 	struct vcpu *vcpu;
2357 
2358 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2359 		panic("vm_set_run_state: invalid vcpuid %d", vcpuid);
2360 
2361 	vcpu = &vm->vcpu[vcpuid];
2362 
2363 	vcpu_lock(vcpu);
2364 	error = vcpu_set_state_locked(vm, vcpuid, newstate, from_idle);
2365 	vcpu_unlock(vcpu);
2366 
2367 	return (error);
2368 }
2369 
2370 enum vcpu_state
2371 vcpu_get_state(struct vm *vm, int vcpuid, int *hostcpu)
2372 {
2373 	struct vcpu *vcpu;
2374 	enum vcpu_state state;
2375 
2376 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2377 		panic("vm_get_run_state: invalid vcpuid %d", vcpuid);
2378 
2379 	vcpu = &vm->vcpu[vcpuid];
2380 
2381 	vcpu_lock(vcpu);
2382 	state = vcpu->state;
2383 	if (hostcpu != NULL)
2384 		*hostcpu = vcpu->hostcpu;
2385 	vcpu_unlock(vcpu);
2386 
2387 	return (state);
2388 }
2389 
2390 int
2391 vm_activate_cpu(struct vm *vm, int vcpuid)
2392 {
2393 
2394 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2395 		return (EINVAL);
2396 
2397 	if (CPU_ISSET(vcpuid, &vm->active_cpus))
2398 		return (EBUSY);
2399 
2400 	VCPU_CTR0(vm, vcpuid, "activated");
2401 	CPU_SET_ATOMIC(vcpuid, &vm->active_cpus);
2402 	return (0);
2403 }
2404 
2405 int
2406 vm_suspend_cpu(struct vm *vm, int vcpuid)
2407 {
2408 	int i;
2409 
2410 	if (vcpuid < -1 || vcpuid >= vm->maxcpus)
2411 		return (EINVAL);
2412 
2413 	if (vcpuid == -1) {
2414 		vm->debug_cpus = vm->active_cpus;
2415 		for (i = 0; i < vm->maxcpus; i++) {
2416 			if (CPU_ISSET(i, &vm->active_cpus))
2417 				vcpu_notify_event(vm, i, false);
2418 		}
2419 	} else {
2420 		if (!CPU_ISSET(vcpuid, &vm->active_cpus))
2421 			return (EINVAL);
2422 
2423 		CPU_SET_ATOMIC(vcpuid, &vm->debug_cpus);
2424 		vcpu_notify_event(vm, vcpuid, false);
2425 	}
2426 	return (0);
2427 }
2428 
2429 int
2430 vm_resume_cpu(struct vm *vm, int vcpuid)
2431 {
2432 
2433 	if (vcpuid < -1 || vcpuid >= vm->maxcpus)
2434 		return (EINVAL);
2435 
2436 	if (vcpuid == -1) {
2437 		CPU_ZERO(&vm->debug_cpus);
2438 	} else {
2439 		if (!CPU_ISSET(vcpuid, &vm->debug_cpus))
2440 			return (EINVAL);
2441 
2442 		CPU_CLR_ATOMIC(vcpuid, &vm->debug_cpus);
2443 	}
2444 	return (0);
2445 }
2446 
2447 int
2448 vcpu_debugged(struct vm *vm, int vcpuid)
2449 {
2450 
2451 	return (CPU_ISSET(vcpuid, &vm->debug_cpus));
2452 }
2453 
2454 cpuset_t
2455 vm_active_cpus(struct vm *vm)
2456 {
2457 
2458 	return (vm->active_cpus);
2459 }
2460 
2461 cpuset_t
2462 vm_debug_cpus(struct vm *vm)
2463 {
2464 
2465 	return (vm->debug_cpus);
2466 }
2467 
2468 cpuset_t
2469 vm_suspended_cpus(struct vm *vm)
2470 {
2471 
2472 	return (vm->suspended_cpus);
2473 }
2474 
2475 void *
2476 vcpu_stats(struct vm *vm, int vcpuid)
2477 {
2478 
2479 	return (vm->vcpu[vcpuid].stats);
2480 }
2481 
2482 int
2483 vm_get_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state *state)
2484 {
2485 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2486 		return (EINVAL);
2487 
2488 	*state = vm->vcpu[vcpuid].x2apic_state;
2489 
2490 	return (0);
2491 }
2492 
2493 int
2494 vm_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
2495 {
2496 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2497 		return (EINVAL);
2498 
2499 	if (state >= X2APIC_STATE_LAST)
2500 		return (EINVAL);
2501 
2502 	vm->vcpu[vcpuid].x2apic_state = state;
2503 
2504 	vlapic_set_x2apic_state(vm, vcpuid, state);
2505 
2506 	return (0);
2507 }
2508 
2509 /*
2510  * This function is called to ensure that a vcpu "sees" a pending event
2511  * as soon as possible:
2512  * - If the vcpu thread is sleeping then it is woken up.
2513  * - If the vcpu is running on a different host_cpu then an IPI will be directed
2514  *   to the host_cpu to cause the vcpu to trap into the hypervisor.
2515  */
2516 static void
2517 vcpu_notify_event_locked(struct vcpu *vcpu, bool lapic_intr)
2518 {
2519 	int hostcpu;
2520 
2521 	hostcpu = vcpu->hostcpu;
2522 	if (vcpu->state == VCPU_RUNNING) {
2523 		KASSERT(hostcpu != NOCPU, ("vcpu running on invalid hostcpu"));
2524 		if (hostcpu != curcpu) {
2525 			if (lapic_intr) {
2526 				vlapic_post_intr(vcpu->vlapic, hostcpu,
2527 				    vmm_ipinum);
2528 			} else {
2529 				ipi_cpu(hostcpu, vmm_ipinum);
2530 			}
2531 		} else {
2532 			/*
2533 			 * If the 'vcpu' is running on 'curcpu' then it must
2534 			 * be sending a notification to itself (e.g. SELF_IPI).
2535 			 * The pending event will be picked up when the vcpu
2536 			 * transitions back to guest context.
2537 			 */
2538 		}
2539 	} else {
2540 		KASSERT(hostcpu == NOCPU, ("vcpu state %d not consistent "
2541 		    "with hostcpu %d", vcpu->state, hostcpu));
2542 		if (vcpu->state == VCPU_SLEEPING)
2543 			wakeup_one(vcpu);
2544 	}
2545 }
2546 
2547 void
2548 vcpu_notify_event(struct vm *vm, int vcpuid, bool lapic_intr)
2549 {
2550 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
2551 
2552 	vcpu_lock(vcpu);
2553 	vcpu_notify_event_locked(vcpu, lapic_intr);
2554 	vcpu_unlock(vcpu);
2555 }
2556 
2557 struct vmspace *
2558 vm_get_vmspace(struct vm *vm)
2559 {
2560 
2561 	return (vm->vmspace);
2562 }
2563 
2564 int
2565 vm_apicid2vcpuid(struct vm *vm, int apicid)
2566 {
2567 	/*
2568 	 * XXX apic id is assumed to be numerically identical to vcpu id
2569 	 */
2570 	return (apicid);
2571 }
2572 
2573 int
2574 vm_smp_rendezvous(struct vm *vm, int vcpuid, cpuset_t dest,
2575     vm_rendezvous_func_t func, void *arg)
2576 {
2577 	int error, i;
2578 
2579 	/*
2580 	 * Enforce that this function is called without any locks
2581 	 */
2582 	WITNESS_WARN(WARN_PANIC, NULL, "vm_smp_rendezvous");
2583 	KASSERT(vcpuid == -1 || (vcpuid >= 0 && vcpuid < vm->maxcpus),
2584 	    ("vm_smp_rendezvous: invalid vcpuid %d", vcpuid));
2585 
2586 restart:
2587 	mtx_lock(&vm->rendezvous_mtx);
2588 	if (vm->rendezvous_func != NULL) {
2589 		/*
2590 		 * If a rendezvous is already in progress then we need to
2591 		 * call the rendezvous handler in case this 'vcpuid' is one
2592 		 * of the targets of the rendezvous.
2593 		 */
2594 		RENDEZVOUS_CTR0(vm, vcpuid, "Rendezvous already in progress");
2595 		mtx_unlock(&vm->rendezvous_mtx);
2596 		error = vm_handle_rendezvous(vm, vcpuid);
2597 		if (error != 0)
2598 			return (error);
2599 		goto restart;
2600 	}
2601 	KASSERT(vm->rendezvous_func == NULL, ("vm_smp_rendezvous: previous "
2602 	    "rendezvous is still in progress"));
2603 
2604 	RENDEZVOUS_CTR0(vm, vcpuid, "Initiating rendezvous");
2605 	vm->rendezvous_req_cpus = dest;
2606 	CPU_ZERO(&vm->rendezvous_done_cpus);
2607 	vm->rendezvous_arg = arg;
2608 	vm->rendezvous_func = func;
2609 	mtx_unlock(&vm->rendezvous_mtx);
2610 
2611 	/*
2612 	 * Wake up any sleeping vcpus and trigger a VM-exit in any running
2613 	 * vcpus so they handle the rendezvous as soon as possible.
2614 	 */
2615 	for (i = 0; i < vm->maxcpus; i++) {
2616 		if (CPU_ISSET(i, &dest))
2617 			vcpu_notify_event(vm, i, false);
2618 	}
2619 
2620 	return (vm_handle_rendezvous(vm, vcpuid));
2621 }
2622 
2623 struct vatpic *
2624 vm_atpic(struct vm *vm)
2625 {
2626 	return (vm->vatpic);
2627 }
2628 
2629 struct vatpit *
2630 vm_atpit(struct vm *vm)
2631 {
2632 	return (vm->vatpit);
2633 }
2634 
2635 struct vpmtmr *
2636 vm_pmtmr(struct vm *vm)
2637 {
2638 
2639 	return (vm->vpmtmr);
2640 }
2641 
2642 struct vrtc *
2643 vm_rtc(struct vm *vm)
2644 {
2645 
2646 	return (vm->vrtc);
2647 }
2648 
2649 enum vm_reg_name
2650 vm_segment_name(int seg)
2651 {
2652 	static enum vm_reg_name seg_names[] = {
2653 		VM_REG_GUEST_ES,
2654 		VM_REG_GUEST_CS,
2655 		VM_REG_GUEST_SS,
2656 		VM_REG_GUEST_DS,
2657 		VM_REG_GUEST_FS,
2658 		VM_REG_GUEST_GS
2659 	};
2660 
2661 	KASSERT(seg >= 0 && seg < nitems(seg_names),
2662 	    ("%s: invalid segment encoding %d", __func__, seg));
2663 	return (seg_names[seg]);
2664 }
2665 
2666 void
2667 vm_copy_teardown(struct vm *vm, int vcpuid, struct vm_copyinfo *copyinfo,
2668     int num_copyinfo)
2669 {
2670 	int idx;
2671 
2672 	for (idx = 0; idx < num_copyinfo; idx++) {
2673 		if (copyinfo[idx].cookie != NULL)
2674 			vm_gpa_release(copyinfo[idx].cookie);
2675 	}
2676 	bzero(copyinfo, num_copyinfo * sizeof(struct vm_copyinfo));
2677 }
2678 
2679 int
2680 vm_copy_setup(struct vm *vm, int vcpuid, struct vm_guest_paging *paging,
2681     uint64_t gla, size_t len, int prot, struct vm_copyinfo *copyinfo,
2682     int num_copyinfo, int *fault)
2683 {
2684 	int error, idx, nused;
2685 	size_t n, off, remaining;
2686 	void *hva, *cookie;
2687 	uint64_t gpa;
2688 
2689 	bzero(copyinfo, sizeof(struct vm_copyinfo) * num_copyinfo);
2690 
2691 	nused = 0;
2692 	remaining = len;
2693 	while (remaining > 0) {
2694 		KASSERT(nused < num_copyinfo, ("insufficient vm_copyinfo"));
2695 		error = vm_gla2gpa(vm, vcpuid, paging, gla, prot, &gpa, fault);
2696 		if (error || *fault)
2697 			return (error);
2698 		off = gpa & PAGE_MASK;
2699 		n = min(remaining, PAGE_SIZE - off);
2700 		copyinfo[nused].gpa = gpa;
2701 		copyinfo[nused].len = n;
2702 		remaining -= n;
2703 		gla += n;
2704 		nused++;
2705 	}
2706 
2707 	for (idx = 0; idx < nused; idx++) {
2708 		hva = vm_gpa_hold(vm, vcpuid, copyinfo[idx].gpa,
2709 		    copyinfo[idx].len, prot, &cookie);
2710 		if (hva == NULL)
2711 			break;
2712 		copyinfo[idx].hva = hva;
2713 		copyinfo[idx].cookie = cookie;
2714 	}
2715 
2716 	if (idx != nused) {
2717 		vm_copy_teardown(vm, vcpuid, copyinfo, num_copyinfo);
2718 		return (EFAULT);
2719 	} else {
2720 		*fault = 0;
2721 		return (0);
2722 	}
2723 }
2724 
2725 void
2726 vm_copyin(struct vm *vm, int vcpuid, struct vm_copyinfo *copyinfo, void *kaddr,
2727     size_t len)
2728 {
2729 	char *dst;
2730 	int idx;
2731 
2732 	dst = kaddr;
2733 	idx = 0;
2734 	while (len > 0) {
2735 		bcopy(copyinfo[idx].hva, dst, copyinfo[idx].len);
2736 		len -= copyinfo[idx].len;
2737 		dst += copyinfo[idx].len;
2738 		idx++;
2739 	}
2740 }
2741 
2742 void
2743 vm_copyout(struct vm *vm, int vcpuid, const void *kaddr,
2744     struct vm_copyinfo *copyinfo, size_t len)
2745 {
2746 	const char *src;
2747 	int idx;
2748 
2749 	src = kaddr;
2750 	idx = 0;
2751 	while (len > 0) {
2752 		bcopy(src, copyinfo[idx].hva, copyinfo[idx].len);
2753 		len -= copyinfo[idx].len;
2754 		src += copyinfo[idx].len;
2755 		idx++;
2756 	}
2757 }
2758 
2759 /*
2760  * Return the amount of in-use and wired memory for the VM. Since
2761  * these are global stats, only return the values with for vCPU 0
2762  */
2763 VMM_STAT_DECLARE(VMM_MEM_RESIDENT);
2764 VMM_STAT_DECLARE(VMM_MEM_WIRED);
2765 
2766 static void
2767 vm_get_rescnt(struct vm *vm, int vcpu, struct vmm_stat_type *stat)
2768 {
2769 
2770 	if (vcpu == 0) {
2771 		vmm_stat_set(vm, vcpu, VMM_MEM_RESIDENT,
2772 	       	    PAGE_SIZE * vmspace_resident_count(vm->vmspace));
2773 	}
2774 }
2775 
2776 static void
2777 vm_get_wiredcnt(struct vm *vm, int vcpu, struct vmm_stat_type *stat)
2778 {
2779 
2780 	if (vcpu == 0) {
2781 		vmm_stat_set(vm, vcpu, VMM_MEM_WIRED,
2782 	      	    PAGE_SIZE * pmap_wired_count(vmspace_pmap(vm->vmspace)));
2783 	}
2784 }
2785 
2786 VMM_STAT_FUNC(VMM_MEM_RESIDENT, "Resident memory", vm_get_rescnt);
2787 VMM_STAT_FUNC(VMM_MEM_WIRED, "Wired memory", vm_get_wiredcnt);
2788 
2789 #ifdef BHYVE_SNAPSHOT
2790 static int
2791 vm_snapshot_vcpus(struct vm *vm, struct vm_snapshot_meta *meta)
2792 {
2793 	int ret;
2794 	int i;
2795 	struct vcpu *vcpu;
2796 
2797 	for (i = 0; i < VM_MAXCPU; i++) {
2798 		vcpu = &vm->vcpu[i];
2799 
2800 		SNAPSHOT_VAR_OR_LEAVE(vcpu->x2apic_state, meta, ret, done);
2801 		SNAPSHOT_VAR_OR_LEAVE(vcpu->exitintinfo, meta, ret, done);
2802 		SNAPSHOT_VAR_OR_LEAVE(vcpu->exc_vector, meta, ret, done);
2803 		SNAPSHOT_VAR_OR_LEAVE(vcpu->exc_errcode_valid, meta, ret, done);
2804 		SNAPSHOT_VAR_OR_LEAVE(vcpu->exc_errcode, meta, ret, done);
2805 		SNAPSHOT_VAR_OR_LEAVE(vcpu->guest_xcr0, meta, ret, done);
2806 		SNAPSHOT_VAR_OR_LEAVE(vcpu->exitinfo, meta, ret, done);
2807 		SNAPSHOT_VAR_OR_LEAVE(vcpu->nextrip, meta, ret, done);
2808 		/* XXX we're cheating here, since the value of tsc_offset as
2809 		 * saved here is actually the value of the guest's TSC value.
2810 		 *
2811 		 * It will be turned turned back into an actual offset when the
2812 		 * TSC restore function is called
2813 		 */
2814 		SNAPSHOT_VAR_OR_LEAVE(vcpu->tsc_offset, meta, ret, done);
2815 	}
2816 
2817 done:
2818 	return (ret);
2819 }
2820 
2821 static int
2822 vm_snapshot_vm(struct vm *vm, struct vm_snapshot_meta *meta)
2823 {
2824 	int ret;
2825 	int i;
2826 	uint64_t now;
2827 
2828 	ret = 0;
2829 	now = rdtsc();
2830 
2831 	if (meta->op == VM_SNAPSHOT_SAVE) {
2832 		/* XXX make tsc_offset take the value TSC proper as seen by the
2833 		 * guest
2834 		 */
2835 		for (i = 0; i < VM_MAXCPU; i++)
2836 			vm->vcpu[i].tsc_offset += now;
2837 	}
2838 
2839 	ret = vm_snapshot_vcpus(vm, meta);
2840 	if (ret != 0) {
2841 		printf("%s: failed to copy vm data to user buffer", __func__);
2842 		goto done;
2843 	}
2844 
2845 	if (meta->op == VM_SNAPSHOT_SAVE) {
2846 		/* XXX turn tsc_offset back into an offset; actual value is only
2847 		 * required for restore; using it otherwise would be wrong
2848 		 */
2849 		for (i = 0; i < VM_MAXCPU; i++)
2850 			vm->vcpu[i].tsc_offset -= now;
2851 	}
2852 
2853 done:
2854 	return (ret);
2855 }
2856 
2857 static int
2858 vm_snapshot_vmcx(struct vm *vm, struct vm_snapshot_meta *meta)
2859 {
2860 	int i, error;
2861 
2862 	error = 0;
2863 
2864 	for (i = 0; i < VM_MAXCPU; i++) {
2865 		error = vmmops_vmcx_snapshot(vm->cookie, meta, i);
2866 		if (error != 0) {
2867 			printf("%s: failed to snapshot vmcs/vmcb data for "
2868 			       "vCPU: %d; error: %d\n", __func__, i, error);
2869 			goto done;
2870 		}
2871 	}
2872 
2873 done:
2874 	return (error);
2875 }
2876 
2877 /*
2878  * Save kernel-side structures to user-space for snapshotting.
2879  */
2880 int
2881 vm_snapshot_req(struct vm *vm, struct vm_snapshot_meta *meta)
2882 {
2883 	int ret = 0;
2884 
2885 	switch (meta->dev_req) {
2886 	case STRUCT_VMX:
2887 		ret = vmmops_snapshot(vm->cookie, meta);
2888 		break;
2889 	case STRUCT_VMCX:
2890 		ret = vm_snapshot_vmcx(vm, meta);
2891 		break;
2892 	case STRUCT_VM:
2893 		ret = vm_snapshot_vm(vm, meta);
2894 		break;
2895 	case STRUCT_VIOAPIC:
2896 		ret = vioapic_snapshot(vm_ioapic(vm), meta);
2897 		break;
2898 	case STRUCT_VLAPIC:
2899 		ret = vlapic_snapshot(vm, meta);
2900 		break;
2901 	case STRUCT_VHPET:
2902 		ret = vhpet_snapshot(vm_hpet(vm), meta);
2903 		break;
2904 	case STRUCT_VATPIC:
2905 		ret = vatpic_snapshot(vm_atpic(vm), meta);
2906 		break;
2907 	case STRUCT_VATPIT:
2908 		ret = vatpit_snapshot(vm_atpit(vm), meta);
2909 		break;
2910 	case STRUCT_VPMTMR:
2911 		ret = vpmtmr_snapshot(vm_pmtmr(vm), meta);
2912 		break;
2913 	case STRUCT_VRTC:
2914 		ret = vrtc_snapshot(vm_rtc(vm), meta);
2915 		break;
2916 	default:
2917 		printf("%s: failed to find the requested type %#x\n",
2918 		       __func__, meta->dev_req);
2919 		ret = (EINVAL);
2920 	}
2921 	return (ret);
2922 }
2923 
2924 int
2925 vm_set_tsc_offset(struct vm *vm, int vcpuid, uint64_t offset)
2926 {
2927 	struct vcpu *vcpu;
2928 
2929 	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
2930 		return (EINVAL);
2931 
2932 	vcpu = &vm->vcpu[vcpuid];
2933 	vcpu->tsc_offset = offset;
2934 
2935 	return (0);
2936 }
2937 
2938 int
2939 vm_restore_time(struct vm *vm)
2940 {
2941 	int error, i;
2942 	uint64_t now;
2943 	struct vcpu *vcpu;
2944 
2945 	now = rdtsc();
2946 
2947 	error = vhpet_restore_time(vm_hpet(vm));
2948 	if (error)
2949 		return (error);
2950 
2951 	for (i = 0; i < nitems(vm->vcpu); i++) {
2952 		vcpu = &vm->vcpu[i];
2953 
2954 		error = vmmops_restore_tsc(vm->cookie, i, vcpu->tsc_offset -
2955 		    now);
2956 		if (error)
2957 			return (error);
2958 	}
2959 
2960 	return (0);
2961 }
2962 #endif
2963